2026-06-24 10:58:41 +08:00

612 lines
20 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { NextResponse } from "next/server";
import { z } from "zod";
import { auth } from "@/app/(auth)/auth";
import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
import { createHuankeGemini } from "@/lib/ai/providers/huanke-gemini";
import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image";
import { createBflImage } from "@/lib/ai/providers/bfl-image";
import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
import { resolveModelRuntimeId } from "@/lib/ai/model-runtime";
import { createFalImage } from "@/lib/ai/providers/fal-image";
import { createSnapgenImage } from "@/lib/ai/providers/snapgen-image";
import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
import { createStepfunImage } from "@/lib/ai/providers/stepfun-image";
import { createMidjourney } from "@/lib/ai/providers/midjourney";
import { createRunway } from "@/lib/ai/providers/runway";
import { createKrea } from "@/lib/ai/providers/krea";
import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision";
import { isImageGenerationModelDescriptor } from "@/lib/ai/model-classification";
import { normalizeOptionalHexColor } from "@/lib/canvas/text-edit-colors";
import { createLogger } from "@/lib/observability/logger";
import { uploadFile } from "@/lib/storage/service";
export const runtime = "nodejs";
export const maxDuration = 1200;
const logger = createLogger("canvas-text-edit");
const TEXT_EDIT_SOURCE_ARCHIVE_TIMEOUT_MS = 20_000;
const TEXT_EDIT_MODEL_FALLBACK_LIMIT = 3;
const textItemSchema = z.object({
id: z.string().min(1),
originalText: z.string().min(1),
editedText: z.string().optional(),
textColor: z.string().optional(),
action: z.enum(["replace", "delete"]).optional(),
});
const requestSchema = z.object({
imageUrl: z.string().min(1, "图片地址不能为空"),
items: z.array(textItemSchema).min(1, "至少需要一条文字修改"),
});
const SUPPORTED_TEXT_EDIT_PROVIDER_TYPES = new Set([
"gemini",
"grsai-gpt-image",
"huanke-gemini",
"alibaba-image",
"bfl-image",
"ruxa-image",
"fal-image",
"snapgen-image",
"openai-image",
"stepfun-image",
"midjourney",
"runway",
"krea",
"volcano-vision",
]);
type TextEditModelCandidate = Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>[number];
function getTextEditModelCandidates(
models: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>
) {
return models
.filter(
(item) =>
isImageGenerationModelDescriptor({
providerType: item.provider.type,
modelId: item.model.modelId,
capabilities: item.model.capabilities,
supportsImageGen: item.model.supportsImageGen,
}) &&
item.model.isEnabled &&
item.provider.isEnabled &&
SUPPORTED_TEXT_EDIT_PROVIDER_TYPES.has(item.provider.type),
)
.sort((a, b) => {
const aScore =
(a.model.isDefaultTextEditModel ? 100 : 0) +
(a.model.isDefault ? 10 : 0) +
(a.model.modelId?.toLowerCase().includes("seedream") ? 5 : 0) +
(a.model.displayName?.includes("火山") ? 3 : 0);
const bScore =
(b.model.isDefaultTextEditModel ? 100 : 0) +
(b.model.isDefault ? 10 : 0) +
(b.model.modelId?.toLowerCase().includes("seedream") ? 5 : 0) +
(b.model.displayName?.includes("火山") ? 3 : 0);
return bScore - aScore;
});
}
function createTextEditProvider(
provider: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>[number]["provider"],
) {
const baseURL = provider.baseUrl;
const apiKey = provider.apiKey;
switch (provider.type) {
case "gemini":
case "grsai-gpt-image":
return createGeminiImage({ baseURL, apiKey });
case "huanke-gemini":
return createHuankeGemini({ baseURL, apiKey });
case "alibaba-image":
return createAlibabaImage({ baseURL, apiKey });
case "bfl-image":
return createBflImage({ baseURL, apiKey });
case "ruxa-image":
return createRuxaImage({ baseURL, apiKey });
case "fal-image":
return createFalImage({ baseURL, apiKey });
case "snapgen-image":
return createSnapgenImage({ baseURL, apiKey });
case "openai-image":
return createOpenAIImage({ baseURL, apiKey });
case "stepfun-image":
return createStepfunImage({ baseURL, apiKey });
case "midjourney":
return createMidjourney({ baseURL, apiKey });
case "runway":
return createRunway({ baseURL, apiKey });
case "krea":
return createKrea({ baseURL, apiKey });
case "volcano-vision":
return createVolcanoVision({ baseURL, apiKey });
default:
throw new Error(`当前文字编辑暂不支持服务商类型:${provider.type}`);
}
}
function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
return buffer.buffer.slice(
buffer.byteOffset,
buffer.byteOffset + buffer.byteLength
) as ArrayBuffer;
}
function inferImageExtension(contentType: string, imageUrl: string) {
const normalizedContentType = contentType.toLowerCase();
if (normalizedContentType.includes("image/jpeg")) return "jpg";
if (normalizedContentType.includes("image/png")) return "png";
if (normalizedContentType.includes("image/webp")) return "webp";
if (normalizedContentType.includes("image/gif")) return "gif";
const pathname = (() => {
try {
return new URL(imageUrl).pathname;
} catch {
return "";
}
})();
const match = pathname.match(/\.([a-z0-9]{2,5})$/i);
const ext = String(match?.[1] || "").toLowerCase();
return ["jpg", "jpeg", "png", "webp", "gif"].includes(ext) ? ext : "png";
}
async function archiveTextEditSourceImage(imageUrl: string) {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), TEXT_EDIT_SOURCE_ARCHIVE_TIMEOUT_MS);
try {
const response = await fetch(imageUrl, {
cache: "no-store",
signal: controller.signal,
});
if (!response.ok) {
throw new Error(`源图下载失败:${response.status} ${response.statusText}`);
}
const contentType = response.headers.get("content-type") || "";
if (contentType && !contentType.toLowerCase().startsWith("image/")) {
throw new Error(`源图类型异常:${contentType}`);
}
const buffer = Buffer.from(await response.arrayBuffer());
if (buffer.length === 0) {
throw new Error("源图为空文件");
}
const ext = inferImageExtension(contentType, imageUrl);
const uploaded = await uploadFile(
`canvas-text-edit-source-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`,
bufferToArrayBuffer(buffer)
);
return {
bytes: buffer.length,
url: uploaded.url,
};
} finally {
clearTimeout(timeoutId);
}
}
async function resolveTextEditSourceImageUrl(imageUrl: string) {
try {
const archived = await archiveTextEditSourceImage(imageUrl);
logger.info("source_image_archived", {
bytes: archived.bytes,
sourceHost: new URL(imageUrl).host,
targetHost: new URL(archived.url).host,
});
return archived.url;
} catch (error) {
logger.warn("source_image_archive_failed", {
error: error instanceof Error ? error.message : String(error),
});
return imageUrl;
}
}
function normalizeGeneratedImageUrl(value: unknown) {
const raw = String(value || "").trim();
if (!raw) return "";
return raw
.replace(/^["'([{<]+/, "")
.replace(/[)"'\]},;。]+$/, "")
.trim();
}
function isHttpImageUrl(value: unknown) {
const raw = normalizeGeneratedImageUrl(value);
if (!raw || !/^https?:\/\//i.test(raw)) {
return false;
}
try {
const parsed = new URL(raw);
return parsed.protocol === "http:" || parsed.protocol === "https:";
} catch {
return false;
}
}
function getComparableImageIdentity(value: unknown) {
const raw = normalizeGeneratedImageUrl(value);
if (!isHttpImageUrl(raw)) {
return "";
}
try {
const parsed = new URL(raw);
parsed.hash = "";
// Query params often include thumbnails/cache-busters. Same host+path still means
// the model returned the input asset instead of a newly generated image.
return `${parsed.protocol}//${parsed.host}${parsed.pathname}`.toLowerCase();
} catch {
return raw.split("#")[0]?.split("?")[0]?.trim().toLowerCase() || "";
}
}
function createExcludedImageIdentitySet(urls?: Iterable<string>) {
return new Set(
Array.from(urls || [])
.map(getComparableImageIdentity)
.filter(Boolean)
);
}
function isExcludedImageUrl(url: string, excludedIdentities: Set<string>) {
const identity = getComparableImageIdentity(url);
return Boolean(identity && excludedIdentities.has(identity));
}
function collectImageUrlsFromUnknown(value: unknown, output: Set<string>) {
if (value === null || value === undefined) return;
if (typeof value === "string") {
const raw = normalizeGeneratedImageUrl(value);
if (isHttpImageUrl(raw)) {
output.add(raw);
return;
}
const matches = raw.match(/https?:\/\/[^\s"'<>),\]}]+/gi) || [];
for (const match of matches) {
const normalized = normalizeGeneratedImageUrl(match);
if (isHttpImageUrl(normalized)) {
output.add(normalized);
}
}
return;
}
if (Array.isArray(value)) {
value.forEach((item) => collectImageUrlsFromUnknown(item, output));
return;
}
if (typeof value === "object") {
Object.entries(value as Record<string, unknown>).forEach(([key, entryValue]) => {
if (
/(url|uri|image|thumbnail|original|output|result|file)/i.test(key) ||
typeof entryValue === "object"
) {
collectImageUrlsFromUnknown(entryValue, output);
}
});
}
}
function pickGeneratedImageUrl(
urls: Iterable<string>,
excludedIdentities = new Set<string>()
) {
const candidates = Array.from(new Set(Array.from(urls).map(normalizeGeneratedImageUrl)))
.filter(isHttpImageUrl)
.filter((url) => !isExcludedImageUrl(url, excludedIdentities));
const imageLike = candidates.find((url) =>
/\.(avif|gif|jpe?g|png|webp)(?:[?#]|$)/i.test(url)
);
return imageLike || candidates[0] || "";
}
function parseImageFromGenerationText(
text: string,
excludedSourceUrls: string[] = []
): { originalUrl: string; thumbnailUrl: string } {
const markdown = String(text || "").trim();
const excludedIdentities = createExcludedImageIdentitySet(excludedSourceUrls);
const seenUrls = new Set<string>();
const strictMatch = markdown.match(/!\[thumb\|\|\|([^|]+)\|\|\|[^\]]*\]\(([^)]+)\)/);
if (strictMatch) {
const originalUrl = normalizeGeneratedImageUrl(strictMatch[1]);
const thumbnailUrl = normalizeGeneratedImageUrl(strictMatch[2]);
[originalUrl, thumbnailUrl].filter(isHttpImageUrl).forEach((url) => seenUrls.add(url));
if (isHttpImageUrl(originalUrl) && !isExcludedImageUrl(originalUrl, excludedIdentities)) {
return {
originalUrl,
thumbnailUrl:
isHttpImageUrl(thumbnailUrl) && !isExcludedImageUrl(thumbnailUrl, excludedIdentities)
? thumbnailUrl
: originalUrl,
};
}
}
const markdownUrls = new Set<string>();
const markdownImagePattern = /!\[[^\]]*\]\(([^)]+)\)/g;
let markdownMatch: RegExpExecArray | null;
while ((markdownMatch = markdownImagePattern.exec(markdown)) !== null) {
const url = normalizeGeneratedImageUrl(markdownMatch[1]);
if (isHttpImageUrl(url)) {
markdownUrls.add(url);
seenUrls.add(url);
}
}
const markdownUrl = pickGeneratedImageUrl(markdownUrls, excludedIdentities);
if (markdownUrl) {
return {
originalUrl: markdownUrl,
thumbnailUrl: markdownUrl,
};
}
const collectedUrls = new Set<string>();
try {
collectImageUrlsFromUnknown(JSON.parse(markdown), collectedUrls);
} catch {
collectImageUrlsFromUnknown(markdown, collectedUrls);
}
collectedUrls.forEach((url) => seenUrls.add(url));
const url = pickGeneratedImageUrl(collectedUrls, excludedIdentities);
if (url) {
return {
originalUrl: url,
thumbnailUrl: url,
};
}
if (Array.from(seenUrls).some((url) => isExcludedImageUrl(url, excludedIdentities))) {
throw new Error("模型返回的是输入原图,未完成文字替换");
}
throw new Error("未解析到生成图片");
}
function buildTextEditPrompt(
replacements: Array<{
originalText: string;
editedText: string;
textColor?: string;
}>,
deletions: Array<{ originalText: string }>,
) {
const sections: string[] = [
"请基于输入图片执行精确文字替换。",
"硬性要求:",
"1. 仅编辑下方列出的目标文字区域,不要修改其他主体、产品、背景、材质、配色、光影、图标、装饰元素与构图。",
"2. 必须保持原图排版、字重、笔画风格、字体形状、字号关系与文字所在位置尽可能一致,仅替换指定文字内容或指定文字颜色。",
"3. 输出图严格保持原图宽高比例,不裁切、不拉伸、不加边框。",
"4. 若目标文案更长,优先微调字距或字号,确保自然贴合原位置。",
"5. 如果指定了文字颜色,只允许修改该段文字本身的颜色,绝对禁止把产品、包装、背景、阴影、高光、边框或其他非文字区域一起染色。",
"6. 如未指定文字颜色,则尽量保持原图该文字本来的颜色与质感。",
"7. 删除文字时请仅移除对应文字并自然修复背景,不要改动周围产品外观。",
"8. 必须输出一张新生成的图片,不要直接返回输入原图或源图链接;如果无法完成替换,请明确失败,不要用原图冒充结果。",
];
if (replacements.length > 0) {
const replacementLines = replacements
.map((item, index) => {
if (item.textColor && item.originalText === item.editedText) {
return [
`${index + 1}. 原文: ${item.originalText}`,
" 文字内容保持不变",
` 仅将这段文字颜色改为: ${item.textColor.toUpperCase()}`,
" 注意: 只能改文字颜色,不要让产品、背景或其他元素变色。",
].join("\n");
}
if (item.textColor) {
return [
`${index + 1}. 原文: ${item.originalText}`,
` 替换为: ${item.editedText}`,
` 新文字颜色: ${item.textColor.toUpperCase()}`,
" 注意: 只对这段文字应用该颜色,其它区域颜色保持原样。",
].join("\n");
}
return [
`${index + 1}. 原文: ${item.originalText}`,
` 替换为: ${item.editedText}`,
" 文字颜色要求: 保持原图该文字原有颜色不变",
].join("\n");
})
.join("\n");
sections.push("替换清单:", replacementLines);
}
if (deletions.length > 0) {
const deletionLines = deletions
.map((item, index) => `${index + 1}. 删除文字: ${item.originalText}`)
.join("\n");
sections.push(
"删除清单:",
deletionLines,
"删除时请清除对应文字并自然修复背景,不保留残影或笔画痕迹,也不要影响产品主体颜色。",
);
}
sections.push("请返回一张最终完成的图片。");
return sections.join("\n");
}
async function generateTextEditImageWithFailover(params: {
candidates: TextEditModelCandidate[];
imageUrl: string;
sourceImageUrls: string[];
prompt: string;
}) {
let lastError: unknown = null;
const candidates = params.candidates.slice(0, TEXT_EDIT_MODEL_FALLBACK_LIMIT);
for (const selected of candidates) {
try {
logger.info("selected_model", {
providerType: selected.provider.type,
providerName: selected.provider.name,
modelId: selected.model.modelId,
displayName: selected.model.displayName,
isDefault: selected.model.isDefault,
isDefaultTextEditModel: selected.model.isDefaultTextEditModel,
});
const provider = createTextEditProvider(selected.provider);
const runtimeModelId = resolveModelRuntimeId(selected.model, {
providerType: selected.provider.type,
});
const result = await provider.chat(runtimeModelId).doGenerate({
messages: [
{
role: "user",
content: [
{ type: "image_url", image_url: { url: params.imageUrl } },
{ type: "text", text: params.prompt },
],
},
],
imageParams: {
imageCount: 1,
},
});
const resultText = String(result?.text || "").trim();
const finishReason = String(result?.finishReason || "").toLowerCase();
if (!resultText) {
throw new Error("模型未返回结果");
}
if (finishReason === "error" || resultText.startsWith("图片生成失败")) {
throw new Error(resultText);
}
const parsed = parseImageFromGenerationText(resultText, params.sourceImageUrls);
return {
...parsed,
model: selected.model.displayName,
};
} catch (error) {
lastError = error;
logger.warn("candidate_failed", {
providerType: selected.provider.type,
providerName: selected.provider.name,
modelId: selected.model.modelId,
displayName: selected.model.displayName,
error: error instanceof Error ? error.message : String(error),
});
}
}
throw lastError instanceof Error ? lastError : new Error("文字替换失败");
}
export async function POST(request: Request) {
try {
const session = await auth();
if (!session?.user?.id) {
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
}
const json = await request.json();
const parsed = requestSchema.safeParse(json);
if (!parsed.success) {
return NextResponse.json({ success: false, error: parsed.error.errors[0]?.message || "参数错误" }, { status: 400 });
}
const imageUrl = parsed.data.imageUrl.trim();
if (!/^https?:\/\//i.test(imageUrl)) {
return NextResponse.json({ success: false, error: "仅支持 URL 图片,不允许 base64 传输" }, { status: 400 });
}
const normalizedItems = parsed.data.items
.map((item) => ({
id: item.id,
originalText: item.originalText.trim(),
editedText: (item.editedText || "").trim(),
textColor: normalizeOptionalHexColor(item.textColor),
action: item.action,
}));
const replacements = normalizedItems
.filter(
(item) =>
item.originalText &&
item.action !== "delete" &&
item.editedText &&
(item.originalText !== item.editedText || Boolean(item.textColor)),
)
.map((item) => ({
id: item.id,
originalText: item.originalText,
editedText: item.editedText,
textColor: item.textColor,
}));
const deletions = normalizedItems
.filter(
(item) =>
item.originalText &&
(item.action === "delete" || item.editedText.length === 0),
)
.map((item) => ({
id: item.id,
originalText: item.originalText,
}));
if (replacements.length === 0 && deletions.length === 0) {
return NextResponse.json({ success: false, error: "未检测到有效的文字改动或删除" }, { status: 400 });
}
const models = await getEnabledAiModelsWithProvider();
const candidates = getTextEditModelCandidates(models);
if (candidates.length === 0) {
return NextResponse.json(
{
success: false,
error: "未配置可用的文字编辑生图模型,请在系统设置-AI 模型中启用一个支持图片生成的模型并设为默认文字编辑生图模型",
},
{ status: 500 },
);
}
const prompt = buildTextEditPrompt(replacements, deletions);
const generationSourceImageUrl = await resolveTextEditSourceImageUrl(imageUrl);
const { originalUrl, thumbnailUrl, model } = await generateTextEditImageWithFailover({
candidates,
imageUrl: generationSourceImageUrl,
sourceImageUrls: [imageUrl, generationSourceImageUrl],
prompt,
});
return NextResponse.json({
success: true,
data: {
originalUrl,
thumbnailUrl,
model,
},
});
} catch (error) {
logger.error("request_failed", error);
return NextResponse.json(
{
success: false,
error: error instanceof Error ? error.message : "文字替换失败",
},
{ status: 500 },
);
}
}