import { NextResponse } from "next/server"; import { z } from "zod"; import { auth } from "@/app/(auth)/auth"; import { getEnabledAiModelsWithProvider } from "@/lib/db/queries"; import { createGeminiImage } from "@/lib/ai/providers/gemini-image"; import { createHuankeGemini } from "@/lib/ai/providers/huanke-gemini"; import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image"; import { createBflImage } from "@/lib/ai/providers/bfl-image"; import { createRuxaImage } from "@/lib/ai/providers/ruxa-image"; import { resolveModelRuntimeId } from "@/lib/ai/model-runtime"; import { createFalImage } from "@/lib/ai/providers/fal-image"; import { createSnapgenImage } from "@/lib/ai/providers/snapgen-image"; import { createOpenAIImage } from "@/lib/ai/providers/openai-image"; import { createBananaImage } from "@/lib/ai/providers/bananaimage"; import { createVisionary } from "@/lib/ai/providers/visionary"; import { createAtlasCloud } from "@/lib/ai/providers/atlascloud"; import { createVip98k } from "@/lib/ai/providers/vip98k"; import { createStepfunImage } from "@/lib/ai/providers/stepfun-image"; import { createMidjourney } from "@/lib/ai/providers/midjourney"; import { createRunway } from "@/lib/ai/providers/runway"; import { createKrea } from "@/lib/ai/providers/krea"; import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision"; import { isImageGenerationModelDescriptor } from "@/lib/ai/model-classification"; import { normalizeOptionalHexColor } from "@/lib/canvas/text-edit-colors"; import { createLogger } from "@/lib/observability/logger"; import { uploadFile } from "@/lib/storage/service"; export const runtime = "nodejs"; export const maxDuration = 1200; const logger = createLogger("canvas-text-edit"); const TEXT_EDIT_SOURCE_ARCHIVE_TIMEOUT_MS = 120_000; const TEXT_EDIT_MODEL_FALLBACK_LIMIT = 3; const TEXT_EDIT_GENERATION_TIMEOUT_MS = 180_000; const textItemSchema = z.object({ id: z.string().min(1), originalText: z.string().min(1), editedText: z.string().optional(), textColor: z.string().optional(), action: z.enum(["replace", "delete"]).optional(), }); const requestSchema = z.object({ imageUrl: z.string().min(1, "图片地址不能为空"), items: z.array(textItemSchema).min(1, "至少需要一条文字修改"), }); const SUPPORTED_TEXT_EDIT_PROVIDER_TYPES = new Set([ "gemini", "grsai-gpt-image", "huanke-gemini", "alibaba-image", "bfl-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "vip98k", "stepfun-image", "midjourney", "runway", "krea", "volcano-vision", ]); type TextEditModelCandidate = Awaited>[number]; function isExplicitTextEditModel(candidate: TextEditModelCandidate) { const capabilities = candidate.model.capabilities as Record | null; const kind = String(capabilities?.kind || "").toLowerCase(); const modelId = String(candidate.model.modelId || "").toLowerCase(); return kind === "image-edit" || /(?:^|[/_-])edit(?:$|[/_-])/.test(modelId); } function getTextEditModelCandidates( models: Awaited> ) { return models .filter( (item) => isImageGenerationModelDescriptor({ providerType: item.provider.type, modelId: item.model.modelId, capabilities: item.model.capabilities, supportsImageGen: item.model.supportsImageGen, }) && item.model.isEnabled && item.provider.isEnabled && (!item.model.excludeFromFailover || item.model.isDefaultTextEditModel) && SUPPORTED_TEXT_EDIT_PROVIDER_TYPES.has(item.provider.type), ) .sort((a, b) => { const aScore = (a.model.isDefaultTextEditModel ? 100 : 0) + (isExplicitTextEditModel(a) ? 30 : 0) + (a.model.isDefault ? 10 : 0) + (a.model.modelId?.toLowerCase().includes("seedream") ? 5 : 0) + (a.model.displayName?.includes("火山") ? 3 : 0); const bScore = (b.model.isDefaultTextEditModel ? 100 : 0) + (isExplicitTextEditModel(b) ? 30 : 0) + (b.model.isDefault ? 10 : 0) + (b.model.modelId?.toLowerCase().includes("seedream") ? 5 : 0) + (b.model.displayName?.includes("火山") ? 3 : 0); return bScore - aScore; }); } function createTextEditProvider( provider: Awaited>[number]["provider"], ) { const baseURL = provider.baseUrl; const apiKey = provider.apiKey; switch (provider.type) { case "gemini": case "grsai-gpt-image": return createGeminiImage({ baseURL, apiKey }); case "huanke-gemini": return createHuankeGemini({ baseURL, apiKey }); case "alibaba-image": return createAlibabaImage({ baseURL, apiKey }); case "bfl-image": return createBflImage({ baseURL, apiKey }); case "ruxa-image": return createRuxaImage({ baseURL, apiKey }); case "fal-image": return createFalImage({ baseURL, apiKey }); case "snapgen-image": return createSnapgenImage({ baseURL, apiKey }); case "openai-image": return createOpenAIImage({ baseURL, apiKey }); case "bananaimage": return createBananaImage({ baseURL, apiKey }); case "visionary": return createVisionary({ baseURL, apiKey }); case "atlascloud": return createAtlasCloud({ baseURL, apiKey }); case "vip98k": return createVip98k({ baseURL, apiKey }); case "stepfun-image": return createStepfunImage({ baseURL, apiKey }); case "midjourney": return createMidjourney({ baseURL, apiKey }); case "runway": return createRunway({ baseURL, apiKey }); case "krea": return createKrea({ baseURL, apiKey }); case "volcano-vision": return createVolcanoVision({ baseURL, apiKey }); default: throw new Error(`当前文字编辑暂不支持服务商类型:${provider.type}`); } } function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer { return buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; } function inferImageExtension(contentType: string, imageUrl: string) { const normalizedContentType = contentType.toLowerCase(); if (normalizedContentType.includes("image/jpeg")) return "jpg"; if (normalizedContentType.includes("image/png")) return "png"; if (normalizedContentType.includes("image/webp")) return "webp"; if (normalizedContentType.includes("image/gif")) return "gif"; const pathname = (() => { try { return new URL(imageUrl).pathname; } catch { return ""; } })(); const match = pathname.match(/\.([a-z0-9]{2,5})$/i); const ext = String(match?.[1] || "").toLowerCase(); return ["jpg", "jpeg", "png", "webp", "gif"].includes(ext) ? ext : "png"; } async function archiveTextEditSourceImage(imageUrl: string) { const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), TEXT_EDIT_SOURCE_ARCHIVE_TIMEOUT_MS); try { const response = await fetch(imageUrl, { cache: "no-store", signal: controller.signal, }); if (!response.ok) { throw new Error(`源图下载失败:${response.status} ${response.statusText}`); } const contentType = response.headers.get("content-type") || ""; if (contentType && !contentType.toLowerCase().startsWith("image/")) { throw new Error(`源图类型异常:${contentType}`); } const buffer = Buffer.from(await response.arrayBuffer()); if (buffer.length === 0) { throw new Error("源图为空文件"); } const ext = inferImageExtension(contentType, imageUrl); const uploaded = await uploadFile( `canvas-text-edit-source-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`, bufferToArrayBuffer(buffer) ); return { bytes: buffer.length, url: uploaded.url, }; } finally { clearTimeout(timeoutId); } } async function resolveTextEditSourceImageUrl(imageUrl: string) { try { const archived = await archiveTextEditSourceImage(imageUrl); logger.info("source_image_archived", { bytes: archived.bytes, sourceHost: new URL(imageUrl).host, targetHost: new URL(archived.url).host, }); return archived.url; } catch (error) { logger.warn("source_image_archive_failed", { error: error instanceof Error ? error.message : String(error), sourceHost: (() => { try { return new URL(imageUrl).host; } catch { return "invalid-url"; } })(), }); return imageUrl; } } function normalizeGeneratedImageUrl(value: unknown) { const raw = String(value || "").trim(); if (!raw) return ""; return raw .replace(/^["'([{<]+/, "") .replace(/[)"'\]},;。]+$/, "") .trim(); } function isHttpImageUrl(value: unknown) { const raw = normalizeGeneratedImageUrl(value); if (!raw || !/^https?:\/\//i.test(raw)) { return false; } try { const parsed = new URL(raw); return parsed.protocol === "http:" || parsed.protocol === "https:"; } catch { return false; } } function getComparableImageIdentity(value: unknown) { const raw = normalizeGeneratedImageUrl(value); if (!isHttpImageUrl(raw)) { return ""; } try { const parsed = new URL(raw); parsed.hash = ""; // Query params often include thumbnails/cache-busters. Same host+path still means // the model returned the input asset instead of a newly generated image. return `${parsed.protocol}//${parsed.host}${parsed.pathname}`.toLowerCase(); } catch { return raw.split("#")[0]?.split("?")[0]?.trim().toLowerCase() || ""; } } function createExcludedImageIdentitySet(urls?: Iterable) { return new Set( Array.from(urls || []) .map(getComparableImageIdentity) .filter(Boolean) ); } function isExcludedImageUrl(url: string, excludedIdentities: Set) { const identity = getComparableImageIdentity(url); return Boolean(identity && excludedIdentities.has(identity)); } function collectImageUrlsFromUnknown(value: unknown, output: Set) { if (value === null || value === undefined) return; if (typeof value === "string") { const raw = normalizeGeneratedImageUrl(value); if (isHttpImageUrl(raw)) { output.add(raw); return; } const matches = raw.match(/https?:\/\/[^\s"'<>),\]}]+/gi) || []; for (const match of matches) { const normalized = normalizeGeneratedImageUrl(match); if (isHttpImageUrl(normalized)) { output.add(normalized); } } return; } if (Array.isArray(value)) { value.forEach((item) => collectImageUrlsFromUnknown(item, output)); return; } if (typeof value === "object") { Object.entries(value as Record).forEach(([key, entryValue]) => { if ( /(url|uri|image|thumbnail|original|output|result|file)/i.test(key) || typeof entryValue === "object" ) { collectImageUrlsFromUnknown(entryValue, output); } }); } } function pickGeneratedImageUrl( urls: Iterable, excludedIdentities = new Set() ) { const candidates = Array.from(new Set(Array.from(urls).map(normalizeGeneratedImageUrl))) .filter(isHttpImageUrl) .filter((url) => !isExcludedImageUrl(url, excludedIdentities)); const imageLike = candidates.find((url) => /\.(avif|gif|jpe?g|png|webp)(?:[?#]|$)/i.test(url) ); return imageLike || candidates[0] || ""; } function parseImageFromGenerationText( text: string, excludedSourceUrls: string[] = [] ): { originalUrl: string; thumbnailUrl: string } { const markdown = String(text || "").trim(); const excludedIdentities = createExcludedImageIdentitySet(excludedSourceUrls); const seenUrls = new Set(); const strictMatch = markdown.match(/!\[thumb\|\|\|([^|]+)\|\|\|[^\]]*\]\(([^)]+)\)/); if (strictMatch) { const originalUrl = normalizeGeneratedImageUrl(strictMatch[1]); const thumbnailUrl = normalizeGeneratedImageUrl(strictMatch[2]); [originalUrl, thumbnailUrl].filter(isHttpImageUrl).forEach((url) => seenUrls.add(url)); if (isHttpImageUrl(originalUrl) && !isExcludedImageUrl(originalUrl, excludedIdentities)) { return { originalUrl, thumbnailUrl: isHttpImageUrl(thumbnailUrl) && !isExcludedImageUrl(thumbnailUrl, excludedIdentities) ? thumbnailUrl : originalUrl, }; } } const markdownUrls = new Set(); const markdownImagePattern = /!\[[^\]]*\]\(([^)]+)\)/g; let markdownMatch: RegExpExecArray | null; while ((markdownMatch = markdownImagePattern.exec(markdown)) !== null) { const url = normalizeGeneratedImageUrl(markdownMatch[1]); if (isHttpImageUrl(url)) { markdownUrls.add(url); seenUrls.add(url); } } const markdownUrl = pickGeneratedImageUrl(markdownUrls, excludedIdentities); if (markdownUrl) { return { originalUrl: markdownUrl, thumbnailUrl: markdownUrl, }; } const collectedUrls = new Set(); try { collectImageUrlsFromUnknown(JSON.parse(markdown), collectedUrls); } catch { collectImageUrlsFromUnknown(markdown, collectedUrls); } collectedUrls.forEach((url) => seenUrls.add(url)); const url = pickGeneratedImageUrl(collectedUrls, excludedIdentities); if (url) { return { originalUrl: url, thumbnailUrl: url, }; } if (Array.from(seenUrls).some((url) => isExcludedImageUrl(url, excludedIdentities))) { throw new Error("模型返回的是输入原图,未完成文字替换"); } throw new Error("未解析到生成图片"); } function buildTextEditPrompt( replacements: Array<{ originalText: string; editedText: string; textColor?: string; }>, deletions: Array<{ originalText: string }>, ) { const sections: string[] = [ "请基于输入图片执行精确文字替换。", "硬性要求:", "1. 仅编辑下方列出的目标文字区域,不要修改其他主体、产品、背景、材质、配色、光影、图标、装饰元素与构图。", "2. 必须保持原图排版、字重、笔画风格、字体形状、字号关系与文字所在位置尽可能一致,仅替换指定文字内容或指定文字颜色。", "3. 输出图严格保持原图宽高比例,不裁切、不拉伸、不加边框。", "4. 若目标文案更长,优先微调字距或字号,确保自然贴合原位置。", "5. 如果指定了文字颜色,只允许修改该段文字本身的颜色,绝对禁止把产品、包装、背景、阴影、高光、边框或其他非文字区域一起染色。", "6. 如未指定文字颜色,则尽量保持原图该文字本来的颜色与质感。", "7. 删除文字时请仅移除对应文字并自然修复背景,不要改动周围产品外观。", "8. 必须输出一张新生成的图片,不要直接返回输入原图或源图链接;如果无法完成替换,请明确失败,不要用原图冒充结果。", ]; if (replacements.length > 0) { const replacementLines = replacements .map((item, index) => { if (item.textColor && item.originalText === item.editedText) { return [ `${index + 1}. 原文: ${item.originalText}`, " 文字内容保持不变", ` 仅将这段文字颜色改为: ${item.textColor.toUpperCase()}`, " 注意: 只能改文字颜色,不要让产品、背景或其他元素变色。", ].join("\n"); } if (item.textColor) { return [ `${index + 1}. 原文: ${item.originalText}`, ` 替换为: ${item.editedText}`, ` 新文字颜色: ${item.textColor.toUpperCase()}`, " 注意: 只对这段文字应用该颜色,其它区域颜色保持原样。", ].join("\n"); } return [ `${index + 1}. 原文: ${item.originalText}`, ` 替换为: ${item.editedText}`, " 文字颜色要求: 保持原图该文字原有颜色不变", ].join("\n"); }) .join("\n"); sections.push("替换清单:", replacementLines); } if (deletions.length > 0) { const deletionLines = deletions .map((item, index) => `${index + 1}. 删除文字: ${item.originalText}`) .join("\n"); sections.push( "删除清单:", deletionLines, "删除时请清除对应文字并自然修复背景,不保留残影或笔画痕迹,也不要影响产品主体颜色。", ); } sections.push("请返回一张最终完成的图片。"); return sections.join("\n"); } async function generateTextEditImageWithFailover(params: { candidates: TextEditModelCandidate[]; imageUrl: string; sourceImageUrls: string[]; prompt: string; }) { let lastError: unknown = null; const candidates = params.candidates.slice(0, TEXT_EDIT_MODEL_FALLBACK_LIMIT); for (const selected of candidates) { try { logger.info("selected_model", { providerType: selected.provider.type, providerName: selected.provider.name, modelId: selected.model.modelId, displayName: selected.model.displayName, isDefault: selected.model.isDefault, isDefaultTextEditModel: selected.model.isDefaultTextEditModel, }); const provider = createTextEditProvider(selected.provider); const runtimeModelId = resolveModelRuntimeId(selected.model, { providerType: selected.provider.type, }); const generationOptions: any = { messages: [ { role: "user", content: [ { type: "image_url", image_url: { url: params.imageUrl } }, { type: "text", text: params.prompt }, ], }, ], imageParams: { imageCount: 1, timeoutMs: TEXT_EDIT_GENERATION_TIMEOUT_MS, }, }; if (selected.provider.type === "atlascloud" || selected.provider.type === "vip98k") { generationOptions.imageGenConfig = selected.model.imageGenConfig || null; } const modelInstance = selected.provider.type === "vip98k" ? (provider as any).chat(runtimeModelId, { capabilities: selected.model.capabilities || null, imageGenConfig: selected.model.imageGenConfig || null, }) : provider.chat(runtimeModelId); const result = await modelInstance.doGenerate(generationOptions); const resultText = String(result?.text || "").trim(); const finishReason = String(result?.finishReason || "").toLowerCase(); if (!resultText) { throw new Error("模型未返回结果"); } if (finishReason === "error" || resultText.startsWith("图片生成失败")) { throw new Error(resultText); } const parsed = parseImageFromGenerationText(resultText, params.sourceImageUrls); return { ...parsed, model: selected.model.displayName, }; } catch (error) { lastError = error; logger.warn("candidate_failed", { providerType: selected.provider.type, providerName: selected.provider.name, modelId: selected.model.modelId, displayName: selected.model.displayName, error: error instanceof Error ? error.message : String(error), }); } } throw lastError instanceof Error ? lastError : new Error("文字替换失败"); } export async function POST(request: Request) { try { const session = await auth(); if (!session?.user?.id) { return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); } const json = await request.json(); const parsed = requestSchema.safeParse(json); if (!parsed.success) { return NextResponse.json({ success: false, error: parsed.error.errors[0]?.message || "参数错误" }, { status: 400 }); } const imageUrl = parsed.data.imageUrl.trim(); if (!/^https?:\/\//i.test(imageUrl)) { return NextResponse.json({ success: false, error: "仅支持 URL 图片,不允许 base64 传输" }, { status: 400 }); } const normalizedItems = parsed.data.items .map((item) => ({ id: item.id, originalText: item.originalText.trim(), editedText: (item.editedText || "").trim(), textColor: normalizeOptionalHexColor(item.textColor), action: item.action, })); const replacements = normalizedItems .filter( (item) => item.originalText && item.action !== "delete" && item.editedText && (item.originalText !== item.editedText || Boolean(item.textColor)), ) .map((item) => ({ id: item.id, originalText: item.originalText, editedText: item.editedText, textColor: item.textColor, })); const deletions = normalizedItems .filter( (item) => item.originalText && (item.action === "delete" || item.editedText.length === 0), ) .map((item) => ({ id: item.id, originalText: item.originalText, })); if (replacements.length === 0 && deletions.length === 0) { return NextResponse.json({ success: false, error: "未检测到有效的文字改动或删除" }, { status: 400 }); } const models = await getEnabledAiModelsWithProvider(); const candidates = getTextEditModelCandidates(models); if (candidates.length === 0) { return NextResponse.json( { success: false, error: "未配置可用的文字编辑生图模型,请在系统设置-AI 模型中启用一个支持图片生成的模型并设为默认文字编辑生图模型", }, { status: 500 }, ); } const prompt = buildTextEditPrompt(replacements, deletions); const generationSourceImageUrl = await resolveTextEditSourceImageUrl(imageUrl); const { originalUrl, thumbnailUrl, model } = await generateTextEditImageWithFailover({ candidates, imageUrl: generationSourceImageUrl, sourceImageUrls: [imageUrl, generationSourceImageUrl], prompt, }); return NextResponse.json({ success: true, data: { originalUrl, thumbnailUrl, model, }, }); } catch (error) { logger.error("request_failed", error); return NextResponse.json( { success: false, error: error instanceof Error ? error.message : "文字替换失败", }, { status: 500 }, ); } }