From f2d4f9e6982daf145ff22d4c038f741d2be95c0f Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 26 May 2026 22:34:12 +0800 Subject: [PATCH] refactor: modularize chat route helpers --- app/(chat)/api/chat/_lib/agent-output.ts | 106 + .../api/chat/_lib/agent-scene-workflow.ts | 540 +++ app/(chat)/api/chat/_lib/billing.ts | 107 + .../api/chat/_lib/detail-replacement.ts | 1150 ++++++ app/(chat)/api/chat/_lib/image-failover.ts | 259 ++ app/(chat)/api/chat/_lib/image-media.ts | 964 +++++ .../api/chat/_lib/message-preparation.ts | 169 + app/(chat)/api/chat/_lib/model-resolver.ts | 43 + app/(chat)/api/chat/_lib/video-models.ts | 275 ++ app/(chat)/api/chat/route.ts | 3664 +---------------- 10 files changed, 3694 insertions(+), 3583 deletions(-) create mode 100644 app/(chat)/api/chat/_lib/agent-output.ts create mode 100644 app/(chat)/api/chat/_lib/agent-scene-workflow.ts create mode 100644 app/(chat)/api/chat/_lib/billing.ts create mode 100644 app/(chat)/api/chat/_lib/detail-replacement.ts create mode 100644 app/(chat)/api/chat/_lib/image-failover.ts create mode 100644 app/(chat)/api/chat/_lib/image-media.ts create mode 100644 app/(chat)/api/chat/_lib/message-preparation.ts create mode 100644 app/(chat)/api/chat/_lib/model-resolver.ts create mode 100644 app/(chat)/api/chat/_lib/video-models.ts diff --git a/app/(chat)/api/chat/_lib/agent-output.ts b/app/(chat)/api/chat/_lib/agent-output.ts new file mode 100644 index 000000000..9b04ebc34 --- /dev/null +++ b/app/(chat)/api/chat/_lib/agent-output.ts @@ -0,0 +1,106 @@ +import type { PostRequestBody } from "../schema"; + +export function sanitizeAgentOutputSettingsSection(params: { + message?: string; + userImageParams?: PostRequestBody["imageParams"]; + hasUserCustomImageParams: boolean; +}): string | undefined { + const { message, userImageParams, hasUserCustomImageParams } = params; + if (!message?.trim()) return message; + + const outputSectionRegex = + /(?:###\s*输出设置|【\s*输出设置\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/; + if (!outputSectionRegex.test(message)) return message; + + if (!hasUserCustomImageParams) { + return message.replace( + outputSectionRegex, + "【输出设置】\n- 参数：按用户面板当前设置执行（未自动改写）" + ); + } + + const lines: string[] = []; + const aspectRatio = + typeof userImageParams?.aspectRatio === "string" ? userImageParams.aspectRatio.trim() : ""; + const imageSize = + typeof userImageParams?.imageSize === "string" ? userImageParams.imageSize.trim() : ""; + const imageCount = + typeof userImageParams?.imageCount === "number" ? userImageParams.imageCount : undefined; + + if (aspectRatio && aspectRatio !== "auto") { + lines.push(`- 宽高比：${aspectRatio}（来自用户面板设置）`); + } + if (imageSize && imageSize.toUpperCase() !== "2K") { + lines.push(`- 清晰度档位：${imageSize.toUpperCase()}（来自用户面板设置）`); + } + if (typeof imageCount === "number" && Number.isFinite(imageCount) && imageCount !== 1) { + lines.push(`- 出图数量：${Math.max(1, Math.round(imageCount))} 张（来自用户面板设置）`); + } + lines.push("- 其余参数：保持用户面板当前设置"); + + return message.replace(outputSectionRegex, `【输出设置】\n${lines.join("\n")}`); +} + +export function normalizeAgentReportHeadingLine(line: string): string { + const trimmed = line.trim(); + if (!trimmed) return line; + + const markdownHeading = trimmed.match(/^#{1,6}\s*(.+?)\s*$/); + const plainTitle = markdownHeading?.[1]?.trim() || trimmed; + const normalizedTitle = plainTitle + .replace(/^[^\u4e00-\u9fa5A-Za-z0-9]+/, "") + .trim(); + + const knownTitles = [ + "需求理解", + "图片分析", + "执行计划", + "优化后提示词", + "输出设置", + "脚本规划", + "分镜规划", + "执行说明", + "视频生成完成", + ]; + + if (knownTitles.some((title) => normalizedTitle === title)) { + return `【${normalizedTitle}】`; + } + + if (markdownHeading) { + return `【${normalizedTitle || plainTitle}】`; + } + return line; +} + +export function stripAgentFriendlyMediaEmbeds(message: string): string { + if (!message) return message; + + const withoutInlineMedia = message + .replace(/!\[[^\]]*]\((https?:\/\/[^\s)]+)\)/gi, "") + .replace(//gi, "") + .replace(/]*>/gi, ""); + + const cleanedLines = withoutInlineMedia + .split("\n") + .filter((line) => { + const trimmed = line.trim(); + if (!trimmed) return true; + if (/^https?:\/\/\S+$/i.test(trimmed)) return false; + if (/^\[[^\]]*]\(https?:\/\/[^\s)]+\)$/i.test(trimmed)) return false; + return true; + }) + .join("\n"); + + return cleanedLines.replace(/\n{3,}/g, "\n\n").trim(); +} + +export function normalizeAgentReportForDisplay(message?: string): string | undefined { + if (!message?.trim()) return message; + const stripped = stripAgentFriendlyMediaEmbeds(message); + if (!stripped) return undefined; + return stripped + .split("\n") + .map((line) => normalizeAgentReportHeadingLine(line)) + .join("\n"); +} diff --git a/app/(chat)/api/chat/_lib/agent-scene-workflow.ts b/app/(chat)/api/chat/_lib/agent-scene-workflow.ts new file mode 100644 index 000000000..df96bda85 --- /dev/null +++ b/app/(chat)/api/chat/_lib/agent-scene-workflow.ts @@ -0,0 +1,540 @@ +export type AgentPresetMessageImagePart = { + type: "file"; + url: string; + name: string; + mediaType: string; + assetId?: string; +}; + +export function extractCurrentMessageImageParts( + parts: Array>, +): AgentPresetMessageImagePart[] { + return parts + .filter( + (part): part is AgentPresetMessageImagePart => + part?.type === "file" && + typeof part?.url === "string" && + /^https?:\/\//i.test(part.url) && + typeof part?.mediaType === "string" && + part.mediaType.startsWith("image/") + ) + .map((part) => ({ + type: "file", + url: String(part.url).trim(), + name: String(part.name || "图片").trim() || "图片", + mediaType: String(part.mediaType || "image/jpeg").trim() || "image/jpeg", + assetId: + typeof part.assetId === "string" && part.assetId.trim() ? part.assetId.trim() : undefined, + })); +} + +export function getSceneUploadedImageBreakdown(params: { + uploadedImageParts: AgentPresetMessageImagePart[]; + fallbackPrimaryCount: number; + fallbackIncludeModelImage: boolean; +}) { + const normalizedNames = params.uploadedImageParts.map((part) => String(part.name || "").trim()); + const primaryCountFromNames = normalizedNames.filter((name) => name.startsWith("主SKU参考图")).length; + const secondaryCountFromNames = normalizedNames.filter((name) => name.startsWith("搭配SKU参考图")).length; + const hasModelFromNames = normalizedNames.some((name) => name.startsWith("品牌模特图")); + const hasNamedBreakdown = + primaryCountFromNames > 0 || secondaryCountFromNames > 0 || hasModelFromNames; + + if (hasNamedBreakdown) { + return { + primaryProductImageCount: primaryCountFromNames, + bundleProductImageCount: secondaryCountFromNames, + includeModelImage: hasModelFromNames, + }; + } + + const uploadedImageCount = params.uploadedImageParts.length; + const includeModelImage = + params.fallbackIncludeModelImage && uploadedImageCount > params.fallbackPrimaryCount; + const bundleProductImageCount = Math.max( + 0, + uploadedImageCount - params.fallbackPrimaryCount - (includeModelImage ? 1 : 0) + ); + const primaryProductImageCount = Math.max( + 0, + Math.min( + params.fallbackPrimaryCount, + uploadedImageCount - bundleProductImageCount - (includeModelImage ? 1 : 0) + ) + ); + + return { + primaryProductImageCount, + bundleProductImageCount, + includeModelImage, + }; +} + +export function resolveScenePromptOptimizationImageUrls(params: { + uploadedImageParts: AgentPresetMessageImagePart[]; + primaryProductImageCount: number; + bundleProductImageCount: number; + includeModelImage: boolean; +}) { + const namedPrimaryParts = params.uploadedImageParts.filter((part) => + String(part.name || "").trim().startsWith("主SKU参考图") + ); + const namedModelPart = params.uploadedImageParts.find((part) => + String(part.name || "").trim().startsWith("品牌模特图") + ); + + if (namedPrimaryParts.length > 0 || namedModelPart) { + return { + primaryImageUrl: namedPrimaryParts[0]?.url || "", + modelImageUrl: namedModelPart?.url || null, + }; + } + + const primaryImageUrl = + params.uploadedImageParts.slice(0, Math.max(0, params.primaryProductImageCount))[0]?.url || ""; + const modelIndex = params.primaryProductImageCount + params.bundleProductImageCount; + + return { + primaryImageUrl, + modelImageUrl: + params.includeModelImage && modelIndex < params.uploadedImageParts.length + ? params.uploadedImageParts[modelIndex]?.url || null + : null, + }; +} + +export function normalizeAgentSupplementPrompt(text: string, presetName?: string | null) { + const normalized = String(text || "").trim(); + if (!normalized) return ""; + const escapedPresetName = String(presetName || "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const genericPattern = escapedPresetName + ? new RegExp(`^请按「${escapedPresetName}」风格化配置顺序处理这\\d+张图片。?$`) + : /^请按「.+」风格化配置顺序处理这\d+张图片。?$/; + if (genericPattern.test(normalized)) { + return ""; + } + return normalized; +} + +export function normalizeAgentWorkflowPrompt(text: string) { + return String(text || "") + .split(/\n+/) + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter(Boolean) + .join("\n") + .trim(); +} + +export function parseAgentWorkflowPromptSections(text: string) { + const sectionMap = new Map(); + let activeSection = "__root__"; + const normalizedPrompt = normalizeAgentWorkflowPrompt(text); + if (!normalizedPrompt) { + return sectionMap; + } + + normalizedPrompt.split("\n").forEach((line) => { + const trimmedLine = line.trim(); + if (!trimmedLine) { + return; + } + const headerMatch = trimmedLine.match(/^【([^】]{1,30})】$/); + if (headerMatch) { + activeSection = headerMatch[1].trim(); + if (!sectionMap.has(activeSection)) { + sectionMap.set(activeSection, []); + } + return; + } + + const currentLines = sectionMap.get(activeSection) || []; + currentLines.push(trimmedLine); + sectionMap.set(activeSection, currentLines); + }); + + return sectionMap; +} + +export function buildAgentPromptSectionText( + title: string, + lines: string[], + options?: { bullet?: boolean } +) { + const normalizedLines = lines + .map((line) => String(line || "").trim()) + .filter(Boolean); + if (normalizedLines.length === 0) { + return ""; + } + + const sectionBody = options?.bullet + ? normalizedLines.map((line) => (line.startsWith("- ") ? line : `- ${line}`)).join("\n") + : normalizedLines.join("\n"); + + return `【${title}】\n${sectionBody}`; +} + +export function buildOptimizedAgentSceneWorkflowPrompt(params: { + workflowPrompt: string; + optimizationResult?: { + sceneGoal: string; + environmentSummary: string; + propSummary: string; + cameraLightingSummary: string; + characterSummary: string; + negativeConstraints: string[]; + conflicts: string[]; + } | null; +}) { + const normalizedWorkflowPrompt = normalizeAgentWorkflowPrompt(params.workflowPrompt); + if (!normalizedWorkflowPrompt) { + return ""; + } + + const optimizationResult = params.optimizationResult || null; + if (!optimizationResult) { + return normalizedWorkflowPrompt; + } + + const sectionMap = parseAgentWorkflowPromptSections(normalizedWorkflowPrompt); + const sceneGoalLines = [optimizationResult.sceneGoal, optimizationResult.environmentSummary].filter( + Boolean + ); + const propLines = [optimizationResult.propSummary].filter(Boolean); + const cameraLightingLines = [optimizationResult.cameraLightingSummary].filter(Boolean); + const characterLines = [optimizationResult.characterSummary].filter(Boolean); + const negativeLines = Array.from( + new Set([ + ...optimizationResult.negativeConstraints, + ...optimizationResult.conflicts.map((item) => `冲突校正：${item}`), + ]) + ); + + const optimizedSections = [ + buildAgentPromptSectionText("项目背景", sectionMap.get("项目背景") || sectionMap.get("__root__") || []), + buildAgentPromptSectionText("主SKU锁定", sectionMap.get("主SKU锁定") || []), + buildAgentPromptSectionText("搭配SKU锁定", sectionMap.get("搭配SKU锁定") || []), + buildAgentPromptSectionText("参考图", sectionMap.get("参考图") || []), + buildAgentPromptSectionText("场景创作总纲", sceneGoalLines), + buildAgentPromptSectionText("陈列与道具", propLines), + buildAgentPromptSectionText("机位与光线", cameraLightingLines), + buildAgentPromptSectionText("人物规则", characterLines), + buildAgentPromptSectionText("负面约束", negativeLines, { bullet: true }), + ].filter(Boolean); + + return optimizedSections.join("\n\n").trim(); +} + +const AGENT_PROMPT_STRUCTURAL_LABELS = new Set([ + "角色", + "技能", + "目标", + "约束条件", + "流程", + "输出", +]); + +const AGENT_PROMPT_LABEL_SEMANTIC_ALIASES = new Map([ + ["角色", "role"], + ["技能", "skills"], + ["目标", "goal"], + ["约束条件", "constraints"], + ["流程", "process"], + ["输出", "output"], + ["人群选择", "audience"], + ["人物", "audience"], + ["主题风格", "style"], + ["整体风格", "style"], + ["空间选择", "space"], + ["空间布局", "space"], + ["主空间", "space"], + ["子空间选择", "subspace"], + ["子空间", "subspace"], + ["硬装方案", "hardDecoration"], + ["硬装", "hardDecoration"], + ["软装方案", "softDecoration"], + ["软装", "softDecoration"], + ["光氛选择", "lighting"], + ["光源布置", "lighting"], + ["灯具", "lighting"], + ["机位选择", "camera"], + ["拍摄角度", "camera"], + ["配饰陈列", "accessories"], + ["放在产品内的小件配饰", "accessories"], + ["前景", "sceneLayout"], + ["中景", "sceneLayout"], + ["背景", "sceneLayout"], +]); + +const AGENT_SCENE_WORKFLOW_OWNED_SEMANTICS = new Set([ + "role", + "skills", + "goal", + "constraints", + "process", + "output", + "audience", + "style", + "space", + "subspace", + "hardDecoration", + "softDecoration", + "lighting", + "camera", + "accessories", + "sceneLayout", +]); + +export function normalizePromptLineForCompare(line: string) { + return String(line || "") + .replace(/\s+/g, " ") + .replace(/[：:]/g, ":") + .replace(/[。；;，,、!！?？]+$/g, "") + .trim() + .toLowerCase(); +} + +export function parsePromptLineLabel(line: string) { + const normalized = String(line || "").replace(/\s+/g, " ").trim(); + if (!normalized) return { label: "", value: "" }; + + const headerOnlyMatch = normalized.match(/^([^:：]{1,20})[:：]\s*$/); + if (headerOnlyMatch) { + return { + label: headerOnlyMatch[1].trim(), + value: "", + }; + } + + const colonMatch = normalized.match(/^([^:：]{1,20})[:：]\s*(.+)$/); + if (colonMatch) { + return { + label: colonMatch[1].trim(), + value: colonMatch[2].trim(), + }; + } + + const whitespaceMatch = normalized.match(/^([^\s]{1,20})\s+(.+)$/); + if (whitespaceMatch) { + return { + label: whitespaceMatch[1].trim(), + value: whitespaceMatch[2].trim(), + }; + } + + return { label: "", value: normalized }; +} + +export function normalizePromptLabel(label: string) { + return String(label || "") + .replace(/\s+/g, "") + .replace(/[：:]/g, "") + .trim(); +} + +export function getPromptSemanticLabel(label: string) { + const normalized = normalizePromptLabel(label); + return AGENT_PROMPT_LABEL_SEMANTIC_ALIASES.get(normalized) || normalized; +} + +export function collectPromptSemanticLabelsFromText(text: string) { + const semanticLabels = new Set(); + const normalizedText = String(text || "").replace(/\s+/g, " ").trim(); + if (!normalizedText) { + return semanticLabels; + } + + for (const [label, semantic] of AGENT_PROMPT_LABEL_SEMANTIC_ALIASES.entries()) { + if (!label) { + continue; + } + if (normalizedText.includes(`${label}：`) || normalizedText.includes(`${label}:`)) { + semanticLabels.add(semantic); + continue; + } + if (normalizedText.includes(`${label} `) || normalizedText.includes(label)) { + semanticLabels.add(semantic); + } + } + + return semanticLabels; +} + +export function shouldDropSceneSlotSemanticLine( + slotSemanticLabel: string, + workflowSemanticLabels: Set, +) { + if (!slotSemanticLabel || !AGENT_SCENE_WORKFLOW_OWNED_SEMANTICS.has(slotSemanticLabel)) { + return false; + } + + if (slotSemanticLabel === "sceneLayout") { + return [ + "style", + "space", + "subspace", + "hardDecoration", + "softDecoration", + "accessories", + "camera", + "lighting", + ].some((semantic) => workflowSemanticLabels.has(semantic)); + } + + return workflowSemanticLabels.has(slotSemanticLabel); +} + +export function dedupeAgentSlotPromptAgainstWorkflow(params: { + workflowPrompt?: string | null; + slotPrompt?: string | null; + modeType?: "scene_generation" | "detail_replacement" | "sku_replacement" | "generic"; +}) { + const dedupedSlotLines: string[] = []; + const seenExactLines = new Set(); + const workflowStructuralValues = new Map(); + const workflowSemanticLabels = new Set(); + + const rememberLine = (line: string) => { + const normalizedLine = normalizePromptLineForCompare(line); + if (!normalizedLine) { + return; + } + seenExactLines.add(normalizedLine); + }; + + const normalizedWorkflowLines = normalizeAgentWorkflowPrompt(params.workflowPrompt || "") + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + normalizedWorkflowLines.forEach((line) => { + const parsed = parsePromptLineLabel(line); + if (parsed.label) { + workflowSemanticLabels.add(getPromptSemanticLabel(parsed.label)); + } + collectPromptSemanticLabelsFromText(line).forEach((semantic) => { + workflowSemanticLabels.add(semantic); + }); + collectPromptSemanticLabelsFromText(parsed.value || "").forEach((semantic) => { + workflowSemanticLabels.add(semantic); + }); + if (parsed.label && AGENT_PROMPT_STRUCTURAL_LABELS.has(parsed.label)) { + workflowStructuralValues.set( + parsed.label, + normalizePromptLineForCompare(parsed.value || "") + ); + } + rememberLine(line); + }); + + const normalizedSlotLines = String(params.slotPrompt || "") + .split(/\n+/) + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter(Boolean); + let activeSlotSemanticLabel = ""; + normalizedSlotLines.forEach((line) => { + const normalizedLine = normalizePromptLineForCompare(line); + if (!normalizedLine || seenExactLines.has(normalizedLine)) { + return; + } + const parsed = parsePromptLineLabel(line); + const slotSemanticLabel = parsed.label ? getPromptSemanticLabel(parsed.label) : ""; + if (slotSemanticLabel) { + activeSlotSemanticLabel = slotSemanticLabel; + } + if ( + params.modeType === "scene_generation" && + shouldDropSceneSlotSemanticLine( + slotSemanticLabel || activeSlotSemanticLabel, + workflowSemanticLabels, + ) + ) { + return; + } + if (parsed.label && AGENT_PROMPT_STRUCTURAL_LABELS.has(parsed.label)) { + const normalizedValue = normalizePromptLineForCompare(parsed.value || ""); + const workflowValue = workflowStructuralValues.get(parsed.label); + if (workflowValue && workflowValue === normalizedValue) { + return; + } + } + seenExactLines.add(normalizedLine); + dedupedSlotLines.push(line); + }); + + return dedupedSlotLines.join("\n").trim(); +} + +export function buildAgentSlotPrompt(params: { + workflowPrompt?: string | null; + slotPrompt: string; + slotTitle?: string | null; + userSupplementPrompt?: string | null; + includeSlotTitle?: boolean; + modeType?: "scene_generation" | "detail_replacement" | "sku_replacement" | "generic"; +}) { + const lines: string[] = []; + const workflowPrompt = normalizeAgentWorkflowPrompt(params.workflowPrompt || ""); + if (workflowPrompt) { + lines.push(workflowPrompt); + } + const slotTitle = String(params.slotTitle || "").trim(); + if (params.includeSlotTitle !== false && slotTitle) { + lines.push(`【槽位】${slotTitle}`); + } + const dedupedSlotPrompt = dedupeAgentSlotPromptAgainstWorkflow({ + workflowPrompt, + slotPrompt: params.slotPrompt, + modeType: params.modeType || "generic", + }); + if (dedupedSlotPrompt) { + lines.push(dedupedSlotPrompt); + } + const supplement = String(params.userSupplementPrompt || "").trim(); + if (supplement) { + lines.push(`【补充要求】${supplement}`); + } + return lines.join("\n\n").trim(); +} + +export function buildAgentSceneWorkflowPrompt(params: { + workflowPrompt?: string | null; + slotPrompt?: string | null; + userSupplementPrompt?: string | null; + imageRolePrompt?: string | null; +}) { + const lines: string[] = []; + const workflowPrompt = normalizeAgentWorkflowPrompt(params.workflowPrompt || ""); + if (workflowPrompt) { + lines.push(workflowPrompt); + } + const slotPrompt = String(params.slotPrompt || "") + .split(/\n+/) + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter(Boolean) + .join("\n") + .trim(); + if (slotPrompt) { + lines.push(slotPrompt); + } + let imageRolePrompt = String(params.imageRolePrompt || "").trim(); + if (workflowPrompt.includes("品牌模特图：已选") || workflowPrompt.includes("固定模特图")) { + imageRolePrompt = imageRolePrompt + .replace( + /^如当前请求未附带固定模特图，仅在工作流明确要求人物时再安排辅助人物，且人物不能喧宾夺主。$/m, + "" + ) + .replace(/^当前没有固定模特图，不要凭空生成强人物主体。$/m, "") + .split(/\n+/) + .map((line) => line.trim()) + .filter(Boolean) + .join("\n"); + } + if (imageRolePrompt) { + lines.push(imageRolePrompt); + } + const supplement = String(params.userSupplementPrompt || "").trim(); + if (supplement) { + lines.push(`【补充要求】${supplement}`); + } + return lines.join("\n\n").trim(); +} diff --git a/app/(chat)/api/chat/_lib/billing.ts b/app/(chat)/api/chat/_lib/billing.ts new file mode 100644 index 000000000..a6c0ede9d --- /dev/null +++ b/app/(chat)/api/chat/_lib/billing.ts @@ -0,0 +1,107 @@ +import { getModelGenerationCreditCost } from "@/lib/ai/image-model-pricing"; +import { + consumeUserPoints, + getAiModelById, + getEnabledAiModelsWithProvider, + getPointsSystemConfig, + getUserById, +} from "@/lib/db/queries"; + +const MODEL_UUID_REGEX = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +export function getModelCreditCost( + modelConfig: any, + target: "image" | "video", + options?: { + duration?: number | null; + imageSize?: string | null; + resolution?: string | null; + }, +): number { + return getModelGenerationCreditCost(modelConfig, target, options); +} + +export async function resolveBillingModelByReference(modelRef: string): Promise { + if (!modelRef) return undefined; + + if (MODEL_UUID_REGEX.test(modelRef)) { + return await getAiModelById({ id: modelRef }).catch(() => undefined); + } + + const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []); + const matched = allModels.find( + ({ model }: any) => model.id === modelRef || model.modelId === modelRef, + ); + return matched?.model; +} + +export async function consumeGeneratedVideoPoints(params: { + userId: string; + chatId: string; + messageId: string; + modelRefs: Array; + duration?: number | null; + resolution?: string | null; +}) { + try { + const pointsConfig = await getPointsSystemConfig(); + if (!pointsConfig.enabled) return; + + const userData = await getUserById({ id: params.userId }).catch(() => undefined); + if (userData?.isHighVip) { + console.log(`[Points] Skip video points for VIP user=${params.userId}`); + return; + } + + let billingModel: any | undefined; + for (const ref of params.modelRefs) { + if (!ref) continue; + const resolved = await resolveBillingModelByReference(ref); + if (!resolved) continue; + billingModel = resolved; + if ( + getModelCreditCost(resolved, "video", { + duration: params.duration, + resolution: params.resolution, + }) > 0 + ) { + break; + } + } + + if (!billingModel) return; + + const unitCost = getModelCreditCost(billingModel, "video", { + duration: params.duration, + resolution: params.resolution, + }); + if (unitCost <= 0) return; + + const consumeResult = await consumeUserPoints({ + userId: params.userId, + amount: unitCost, + reason: "video_generation", + modelId: billingModel.id, + chatId: params.chatId, + messageId: params.messageId, + note: `模型(${billingModel.displayName || billingModel.modelId || billingModel.id}) 生视频 1 条${params.duration ? `（${params.duration} 秒）` : ""}${params.resolution ? `（${params.resolution}）` : ""}`, + }); + + if (!consumeResult.ok) { + console.warn( + `[Points] Failed to consume video points. user=${params.userId} model=${billingModel.id} cost=${unitCost}`, + ); + } else if (consumeResult.skipped) { + console.log( + `[Points] Skipped ${unitCost} video points for VIP user=${params.userId} model=${billingModel.id}`, + ); + } else { + console.log( + `[Points] Consumed ${unitCost} points for video. balance=${consumeResult.balance}`, + ); + } + } catch (error) { + console.error("[Points] Failed to consume points after video generation:", error); + } +} diff --git a/app/(chat)/api/chat/_lib/detail-replacement.ts b/app/(chat)/api/chat/_lib/detail-replacement.ts new file mode 100644 index 000000000..bd8b76ffa --- /dev/null +++ b/app/(chat)/api/chat/_lib/detail-replacement.ts @@ -0,0 +1,1150 @@ +import { generateText } from "ai"; +import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; +import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; +import { resolveDetailAgentLocalMoonshotModel } from "@/lib/ai/detail-agent-model-runtime"; +import { inlineVisionImages } from "@/lib/ai/vision-inline"; +import { getEnabledAiModelsWithProvider, getSiteSettings } from "@/lib/db/queries"; +import { uploadFile } from "@/lib/storage/service"; +import { bufferToArrayBuffer } from "./persistence"; +import { isVideoProviderType } from "./video-models"; +import type { AgentPresetMessageImagePart } from "./agent-scene-workflow"; +import type { PostRequestBody } from "../schema"; + +export async function createAgentImageThumbnailBuffer(buffer: Buffer): Promise { + const sharp = (await import("sharp")).default; + return sharp(buffer, { failOn: "none" }) + .rotate() + .resize(500, null, { + fit: "inside", + withoutEnlargement: true, + }) + .jpeg({ quality: 82 }) + .toBuffer(); +} + +const REPLACEMENT_OUTPUT_ASPECT_RATIOS = [ + "1:1", + "1:4", + "1:8", + "2:3", + "3:2", + "3:4", + "4:1", + "4:3", + "4:5", + "5:4", + "8:1", + "9:16", + "16:9", + "21:9", +] as const; + +export type ReplacementReferenceGeometry = { + width: number; + height: number; + aspectRatio: string; +}; + +export function parseReplacementAspectRatioValue(ratio: string) { + const [width, height] = ratio.split(":").map((item) => Number(item)); + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + return 1; + } + return width / height; +} + +export function inferClosestReplacementAspectRatio(width: number, height: number) { + const aspectValue = width / Math.max(1, height); + let closest = "1:1"; + let minDiff = Number.POSITIVE_INFINITY; + + for (const candidate of REPLACEMENT_OUTPUT_ASPECT_RATIOS) { + const diff = Math.abs(parseReplacementAspectRatioValue(candidate) - aspectValue); + if (diff < minDiff) { + minDiff = diff; + closest = candidate; + } + } + + return closest; +} + +export async function resolveReplacementReferenceGeometry( + referenceImageUrl: string, +): Promise { + try { + const sharp = (await import("sharp")).default; + const referenceBuffer = await fetchImageBufferFromUrl(referenceImageUrl); + const metadata = await sharp(referenceBuffer, { failOn: "none" }) + .rotate() + .metadata(); + const width = metadata.width; + const height = metadata.height; + + if (!width || !height) { + return null; + } + + return { + width, + height, + aspectRatio: inferClosestReplacementAspectRatio(width, height), + }; + } catch (error) { + console.warn("[Chat API] Failed to resolve replacement reference geometry:", { + referenceImageUrl, + error: error instanceof Error ? error.message : "unknown", + }); + return null; + } +} + +export async function normalizeReplacementResultImage(params: { + generatedImageUrl: string; + generatedThumbnailUrl?: string; + referenceImageUrl: string; + referenceGeometry?: ReplacementReferenceGeometry | null; + uploadPrefix: string; +}): Promise<{ + originalUrl: string; + thumbnailUrl: string; + imageSize: string; +}> { + const sharp = (await import("sharp")).default; + const generatedBuffer = await fetchImageBufferFromUrl(params.generatedImageUrl); + let targetWidth = params.referenceGeometry?.width; + let targetHeight = params.referenceGeometry?.height; + + if (!targetWidth || !targetHeight) { + const referenceBuffer = await fetchImageBufferFromUrl(params.referenceImageUrl); + const referenceMetadata = await sharp(referenceBuffer, { failOn: "none" }) + .rotate() + .metadata(); + targetWidth = referenceMetadata.width; + targetHeight = referenceMetadata.height; + } + + if (!targetWidth || !targetHeight) { + throw new Error("无法读取参考图尺寸"); + } + + const generatedMetadata = await sharp(generatedBuffer, { failOn: "none" }) + .rotate() + .metadata(); + if (generatedMetadata.width === targetWidth && generatedMetadata.height === targetHeight) { + return { + originalUrl: params.generatedImageUrl, + thumbnailUrl: params.generatedThumbnailUrl || params.generatedImageUrl, + imageSize: `${targetWidth}x${targetHeight}`, + }; + } + + const normalizedBuffer = await sharp(generatedBuffer, { failOn: "none" }) + .rotate() + .resize(targetWidth, targetHeight, { + // Keep the model output proportional when adapting it back to the reference page. + // `fill` stretches the picture and can squash products; `cover` behaves like a screenshot crop. + fit: "cover", + position: "centre", + withoutEnlargement: false, + }) + .png() + .toBuffer(); + const thumbnailBuffer = await createAgentImageThumbnailBuffer(normalizedBuffer); + + const uniqueSuffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + const originalFilename = `${params.uploadPrefix}-orig-${uniqueSuffix}.png`; + const thumbnailFilename = `${params.uploadPrefix}-thumb-${uniqueSuffix}.jpg`; + const [originalUpload, thumbnailUpload] = await Promise.all([ + uploadFile(originalFilename, bufferToArrayBuffer(normalizedBuffer)), + uploadFile(thumbnailFilename, bufferToArrayBuffer(thumbnailBuffer)), + ]); + + return { + originalUrl: originalUpload.url, + thumbnailUrl: thumbnailUpload.url, + imageSize: `${targetWidth}x${targetHeight}`, + }; +} + +export type AgentReplacementModePayload = NonNullable; +export type AgentModeSourceImagePayload = NonNullable; +export type AgentDetailReplacementStrategy = "scene_replace" | "product_only"; + +export type NormalizedAgentModeSourceImage = { + url: string; + label: string; + name: string; + assetId: string | null; +}; + +export type ReplacementReferenceDecision = { + shouldReplace: boolean; + containsFurnitureProduct: boolean; + reason: string; + detectionModelName?: string | null; + detectionSource: + | "local_moonshot" + | "configured_vision" + | "configured_qwen3_vl" + | "fallback_qwen3_vl" + | "safe_passthrough"; +}; + +export type ReplacementScenePlan = { + useModelImage: boolean; + modelImageReason: string; + sceneSummary: string; + cameraSummary: string; + placementSummary: string; + preserveSummary: string; + styleSummary: string; + avoidSummary: string; + generationPrompt: string; + selectedProductImageIndexes: number[]; + productSelectionReason: string; + analysisModelName?: string | null; + analysisSource: "local_moonshot" | "configured_vision" | "heuristic"; +}; + +export type ReplacementExecutionPlan = ReplacementReferenceDecision & { + scenePlan: ReplacementScenePlan; +}; + +export type ReplacementScenePlanningRuntime = + | { + kind: "local_moonshot"; + model: any; + modelName: string; + } + | { + kind: "configured_vision"; + model: any; + modelName: string; + } + | null; + +const QWEN_VL_MODEL_MATCHER = /(qwen|千问|通义).*(vl|vision)|qwen3-vl|qwen-vl/i; + +export function normalizeDetailReplacementStrategy( + value: unknown, +): AgentDetailReplacementStrategy { + return value === "product_only" ? "product_only" : "scene_replace"; +} + +export function truncateReplacementAnalysisText(value: unknown, maxLength = 240) { + return String(value || "").trim().replace(/\s+/g, " ").slice(0, maxLength); +} + +export function parseReplacementScenePlan( + text: string, + params: { + productImageCount: number; + hasModelImage: boolean; + }, +): ReplacementScenePlan | null { + const jsonText = extractJsonObjectText(text); + if (!jsonText) { + return null; + } + + try { + const parsed = JSON.parse(jsonText) as Record; + const useModelImageRaw = + normalizeBooleanLike(parsed.useModelImage) ?? + normalizeBooleanLike(parsed.needModelImage) ?? + normalizeBooleanLike(parsed.referenceNeedsModel) ?? + false; + + const generationPrompt = truncateReplacementAnalysisText( + parsed.generationPrompt || parsed.recommendedScenePrompt || parsed.scenePrompt, + 1200, + ); + + const scenePlan: ReplacementScenePlan = { + useModelImage: params.hasModelImage ? Boolean(useModelImageRaw) : false, + modelImageReason: truncateReplacementAnalysisText( + parsed.modelImageReason || parsed.modelReason, + 160, + ), + sceneSummary: truncateReplacementAnalysisText( + parsed.sceneSummary || parsed.sceneDescription, + 220, + ), + cameraSummary: truncateReplacementAnalysisText( + parsed.cameraSummary || parsed.cameraDescription, + 180, + ), + placementSummary: truncateReplacementAnalysisText( + parsed.placementSummary || parsed.productPlacement, + 180, + ), + preserveSummary: truncateReplacementAnalysisText( + parsed.preserveSummary || parsed.mustKeep || parsed.preserveInstruction, + 220, + ), + styleSummary: truncateReplacementAnalysisText( + parsed.styleSummary || parsed.styleInstruction || parsed.styleKeywords, + 220, + ), + avoidSummary: truncateReplacementAnalysisText( + parsed.avoidSummary || parsed.doNotInclude || parsed.avoidInstruction, + 220, + ), + generationPrompt, + selectedProductImageIndexes: Array.from( + { length: Math.min(Math.max(params.productImageCount, 1), 2) }, + (_, index) => index + 1, + ), + productSelectionReason: + params.productImageCount > 1 + ? "默认优先使用前两张产品图作为主视角与补充细节，其余角度不强制同时融入。" + : "默认使用当前唯一产品图作为主体参考。", + analysisModelName: null, + analysisSource: "heuristic", + }; + + if (!scenePlan.generationPrompt) { + scenePlan.generationPrompt = [ + scenePlan.sceneSummary, + scenePlan.cameraSummary, + scenePlan.placementSummary, + scenePlan.styleSummary, + ] + .filter(Boolean) + .join("；") + .slice(0, 1200); + } + + return scenePlan; + } catch (error) { + console.warn("[Chat API] Failed to parse replacement scene plan JSON:", error); + return null; + } +} + +export function buildHeuristicReplacementScenePlan(params: { + productImageCount: number; + hasModelImage: boolean; + shouldReplace: boolean; + reason: string; + replacementStrategy?: AgentDetailReplacementStrategy; +}): ReplacementScenePlan { + const isProductOnly = params.replacementStrategy === "product_only"; + return { + selectedProductImageIndexes: Array.from( + { length: Math.min(Math.max(params.productImageCount, 1), 2) }, + (_, index) => index + 1, + ), + productSelectionReason: + params.productImageCount > 1 + ? "默认优先使用前两张产品图作为主视角与补充细节，其余角度不强制同时融入。" + : "默认使用当前唯一产品图作为主体参考。", + useModelImage: false, + modelImageReason: params.hasModelImage + ? isProductOnly + ? "仅替换产品模式不使用模特图，默认锁定参考图原有人物与场景。" + : "前置分析未明确当前参考图存在人物展示需求，兜底阶段默认不使用模特图。" + : "当前未提供模特图。", + sceneSummary: params.shouldReplace + ? isProductOnly + ? "严格保留参考图的场景、文案和版式，只把产品展示区中的旧产品替换成输入产品。" + : "保留参考图版式骨架、构图和主体位置，但把旧场景视觉内容按工作流目标风格重建。" + : "当前参考图不满足替换条件。", + cameraSummary: "保持参考图原有机位、构图和景别，不做整体重画。", + placementSummary: isProductOnly + ? "产品必须落在参考图原有产品所在区域，保持原位置、透视和占比。" + : "产品应落在参考图当前主体所在区域，并围绕该区域同步重建新的场景内容。", + preserveSummary: isProductOnly + ? "除产品主体外，文字排版、标题区、分割线、留白、背景、灯光和装饰都必须保留。" + : "文字排版、标题区、分割线、留白和信息层级必须保留。", + styleSummary: isProductOnly + ? "延续参考图原有风格、背景和氛围，不引入新的场景改造要求。" + : "除版式骨架外，墙面、地面、背景家具、软装、灯光、摆件和空间氛围都要按工作流风格重建。", + avoidSummary: isProductOnly + ? "不要改动背景、文案、按钮、图标、排版和装饰，不要重建整个场景，不要把多个产品角度同时塞进一张图。" + : "不要只换产品不换场景，不要把参考图原有背景物件和空间氛围原样保留下来，不要把所有输入产品角度图同时强行出现在一张图中。", + generationPrompt: params.shouldReplace + ? isProductOnly + ? `仅替换参考图中的产品主体，严格保留其余背景、文案、排版与光影关系。${params.reason}` + : `以工作流目标风格重建当前这一屏的场景，只保留版式、机位、文字区与留白结构。${params.reason}` + : params.reason, + analysisModelName: null, + analysisSource: "heuristic", + }; +} + +export function normalizeAgentModeReferences( + payload?: PostRequestBody["agentModePayload"], +): AgentReplacementModePayload["references"] { + if (!payload?.references?.length) return []; + return payload.references + .map((item) => ({ + id: String(item.id || "").trim(), + label: String(item.label || "").trim() || "参考图", + imageUrl: String(item.imageUrl || "").trim(), + candidateId: String(item.candidateId || "").trim() || null, + sortOrder: Number.isFinite(Number(item.sortOrder)) + ? Math.max(0, Math.min(999, Math.round(Number(item.sortOrder)))) + : 0, + passthrough: item.passthrough === true, + assignedProductIndex: Number.isFinite(Number(item.assignedProductIndex)) + ? Math.max(0, Math.min(9, Math.round(Number(item.assignedProductIndex)))) + : null, + })) + .filter((item) => item.id && /^https?:\/\//i.test(item.imageUrl)) + .sort((left, right) => left.sortOrder - right.sortOrder); +} + +export function normalizeAgentModeSourceImage(value: unknown): NormalizedAgentModeSourceImage | null { + const record = + value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; + const url = String(record.url || "").trim(); + if (!/^https?:\/\//i.test(url)) { + return null; + } + + const label = String(record.label || record.name || "参考图").trim().slice(0, 255); + const name = String(record.name || record.label || label || "参考图").trim().slice(0, 255); + const assetId = String(record.assetId || "").trim() || null; + + return { + url, + label: label || name || "参考图", + name: name || label || "参考图", + assetId, + }; +} + +export function normalizeAgentModeSourceImages( + payload?: PostRequestBody["agentModePayload"], +): { + productImages: NormalizedAgentModeSourceImage[]; + modelImage: NormalizedAgentModeSourceImage | null; +} { + const sourceImages = payload?.sourceImages as AgentModeSourceImagePayload | undefined; + const productImages = Array.isArray(sourceImages?.productImages) + ? sourceImages.productImages + .map((item) => normalizeAgentModeSourceImage(item)) + .filter((item): item is NormalizedAgentModeSourceImage => Boolean(item)) + .slice(0, 16) + : []; + + return { + productImages, + modelImage: normalizeAgentModeSourceImage(sourceImages?.modelImage), + }; +} + +export function normalizeAgentModeDetailReplacementStrategy( + payload?: PostRequestBody["agentModePayload"], +): AgentDetailReplacementStrategy { + return normalizeDetailReplacementStrategy(payload?.replacementStrategy); +} + +export function resolveImagePartsByReferenceExclusion(params: { + uploadedImageParts: AgentPresetMessageImagePart[]; + referenceImageUrls: string[]; +}) { + const referenceUrlSet = new Set(params.referenceImageUrls.map((url) => String(url || "").trim())); + const modelPart = params.uploadedImageParts.find((part) => /模特|model/i.test(String(part.name || ""))); + const productParts = params.uploadedImageParts.filter((part) => { + const name = String(part.name || "").trim(); + if (referenceUrlSet.has(part.url)) return false; + if (/^详情参考图|^SKU参考图/i.test(name)) return false; + if (modelPart && part.url === modelPart.url) return false; + return true; + }); + + return { productParts, modelPart: modelPart || null }; +} + +export async function resolveDirectReplacementModelRef(params: { + selectedChatModel?: string | null; + finalModelId?: string | null; + models?: Awaited>; +}) { + const explicitModel = String(params.selectedChatModel || "").trim(); + if (explicitModel && explicitModel !== "auto") { + return explicitModel; + } + const routedModel = String(params.finalModelId || "").trim(); + if (routedModel && routedModel !== "auto") { + return routedModel; + } + + const models = params.models || (await getEnabledAiModelsWithProvider(false)); + const imageModels = models.filter( + ({ model, provider }) => + model.supportsImageGen && + !model.isSubModel && + !isVideoProviderType(provider?.type, model), + ); + const preferred = + imageModels.find(({ model }) => model.isDefault) || + imageModels.find(({ provider }) => String(provider?.type || "").includes("image")) || + imageModels[0]; + + return preferred?.model.id || explicitModel || routedModel || "auto"; +} + +export function normalizeBooleanLike(value: unknown): boolean | null { + if (typeof value === "boolean") { + return value; + } + + const normalized = String(value || "") + .trim() + .toLowerCase(); + if (!normalized) { + return null; + } + + if ( + normalized === "true" || + normalized === "yes" || + normalized === "y" || + normalized === "1" || + normalized === "是" + ) { + return true; + } + + if ( + normalized === "false" || + normalized === "no" || + normalized === "n" || + normalized === "0" || + normalized === "否" + ) { + return false; + } + + return null; +} + +export function extractJsonObjectText(text: string): string | null { + const normalized = String(text || "").trim(); + if (!normalized) { + return null; + } + + const fencedMatch = normalized.match(/```(?:json)?\s*([\s\S]*?)```/i); + if (fencedMatch?.[1]) { + const fencedContent = fencedMatch[1].trim(); + if (fencedContent.startsWith("{") && fencedContent.endsWith("}")) { + return fencedContent; + } + } + + if (normalized.startsWith("{") && normalized.endsWith("}")) { + return normalized; + } + + const firstBraceIndex = normalized.indexOf("{"); + const lastBraceIndex = normalized.lastIndexOf("}"); + if (firstBraceIndex >= 0 && lastBraceIndex > firstBraceIndex) { + return normalized.slice(firstBraceIndex, lastBraceIndex + 1); + } + + return null; +} + +export function parseReplacementReferenceDecision( + text: string, +): Omit | null { + const jsonText = extractJsonObjectText(text); + if (jsonText) { + try { + const parsed = JSON.parse(jsonText) as Record; + const shouldReplace = + normalizeBooleanLike(parsed.shouldReplace) ?? + normalizeBooleanLike(parsed.replace) ?? + normalizeBooleanLike(parsed.isReplaceable); + const containsFurnitureProduct = + normalizeBooleanLike(parsed.containsFurnitureProduct) ?? + normalizeBooleanLike(parsed.hasFurnitureProduct) ?? + shouldReplace; + const reason = String(parsed.reason || parsed.summary || parsed.note || "") + .trim() + .slice(0, 80); + + if (typeof shouldReplace === "boolean") { + return { + shouldReplace, + containsFurnitureProduct: Boolean(containsFurnitureProduct), + reason: + reason || + (shouldReplace + ? "检测到可替换的家具主体，继续执行替换" + : "未检测到可替换的家具主体，直接沿用参考图"), + }; + } + } catch (error) { + console.warn("[Chat API] Failed to parse replacement reference decision JSON:", error); + } + } + + const normalized = String(text || "") + .replace(/\s+/g, " ") + .trim(); + if (!normalized) { + return null; + } + + const shouldReplace = /shouldreplace["']?\s*[:=]\s*true|可替换|应该替换|存在可替换|检测到.*家具主体|yes/i.test( + normalized, + ); + const shouldKeep = /shouldreplace["']?\s*[:=]\s*false|不应替换|不要替换|沿用参考图|未检测到.*家具主体|无可替换|not replace|no/i.test( + normalized, + ); + + if (!shouldReplace && !shouldKeep) { + return null; + } + + return { + shouldReplace, + containsFurnitureProduct: shouldReplace, + reason: normalized.slice(0, 80), + }; +} + +export async function analyzeReplacementReferenceImage(params: { + imageUrl: string; + models?: Awaited>; + modeType?: "detail_replacement" | "sku_replacement"; + replacementStrategy?: AgentDetailReplacementStrategy; +}): Promise { + const isDetailReplacement = params.modeType === "detail_replacement"; + const isProductOnlyDetailReplacement = + isDetailReplacement && params.replacementStrategy === "product_only"; + const detectionPrompt = isProductOnlyDetailReplacement + ? [ + "你是电商详情图产品主体替换判定器。", + "请判断这张详情参考图里，是否存在清晰、主要、适合被新产品替换的商品主体、产品展示区或产品照片区域。", + "你的任务不是重建整张场景，也不是替换文字排版；只判断是否有明确的产品替换目标。", + "文字、标题、卖点标签、按钮、UI图标、分割线、品牌标识、留白和背景装饰不属于替换对象。", + "只有当图片几乎是纯文字页、参数表、品牌介绍页、没有明确产品展示区时，shouldReplace 才能为 false。", + '只输出 JSON，不要输出额外解释。格式：{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字"}', + ].join("\n") + : isDetailReplacement + ? [ + "你是电商详情参考图背景替换判定器。", + "请判断这张图片里，是否存在适合被重建/替换的背景图、产品场景图、商品主体图或大面积视觉图片区。", + "这不是只找家具主体：只要存在照片、插画、场景背景、产品展示图区、人物展示图区，都可以 shouldReplace=true。", + "必须保留的文字、标题、卖点标签、UI图标、按钮、分割线、品牌标识、留白和排版不属于替换对象。", + "只有当图片几乎是纯文字页、参数表、品牌介绍页、无明显背景图/场景图区/商品图片区时，shouldReplace 才能为 false。", + '只输出 JSON，不要输出额外解释。格式：{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字"}', + ].join("\n") + : [ + "你是家具商品参考图替换判定器。", + "请判断这张图片里，是否存在清晰、主要、适合被产品替换流程替换的家具商品主体。", + "家具主体包含：床、床垫、沙发、椅子、餐桌、茶几、柜子、电视柜、斗柜、床头柜、书桌等。", + "只有当家具商品是主要视觉主体、轮廓清晰、不是单纯环境陪衬时，shouldReplace 才能为 true。", + "以下情况必须判定 shouldReplace=false：纯文字页、参数说明页、品牌介绍页、材质/五金局部特写、只有模特没有家具、只有空间环境没有明确家具主体、家具很小只是背景、主体过多无法明确替换对象。", + '只输出 JSON，不要输出额外解释。格式：{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字"}', + ].join("\n"); + + const configuredModels = params.models || (await getEnabledAiModelsWithProvider()); + const configuredQwenVisionModel = configuredModels.find( + ({ model }) => + model.supportsVision && + !model.isSubModel && + QWEN_VL_MODEL_MATCHER.test(`${model.displayName} ${model.modelId}`), + ); + + if (configuredQwenVisionModel) { + try { + const languageModel = await getLanguageModelById(configuredQwenVisionModel.model.id); + if (languageModel) { + const { text } = await generateText({ + model: languageModel, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: detectionPrompt, + }, + { + type: "image", + image: params.imageUrl, + }, + ], + }, + ], + temperature: 0, + maxTokens: 160, + abortSignal: AbortSignal.timeout(45_000), + } as any); + + const parsed = parseReplacementReferenceDecision(text); + if (parsed) { + return { + ...parsed, + detectionModelName: + configuredQwenVisionModel.model.displayName || + configuredQwenVisionModel.model.modelId, + detectionSource: "configured_qwen3_vl", + }; + } + } + } catch (error) { + console.warn("[Chat API] Configured Qwen-VL reference analysis failed:", error); + } + } + + try { + const { callQwen3Vl } = await import("@/lib/ai/qwen3-vl"); + const fallbackResult = await callQwen3Vl({ + imageUrl: params.imageUrl, + prompt: detectionPrompt, + temperature: 0, + maxTokens: 160, + timeoutMs: 45_000, + retries: 1, + }); + const parsed = parseReplacementReferenceDecision(fallbackResult.text); + if (parsed) { + return { + ...parsed, + detectionModelName: fallbackResult.model, + detectionSource: "fallback_qwen3_vl", + }; + } + } catch (error) { + console.warn("[Chat API] Fallback Qwen3-VL reference analysis failed:", error); + } + + return { + shouldReplace: false, + containsFurnitureProduct: false, + reason: "视觉判断失败，为避免误替换，直接沿用参考图", + detectionModelName: null, + detectionSource: "safe_passthrough", + }; +} + +export async function resolveReplacementScenePlanningRuntime(params: { + models?: Awaited>; +}): Promise { + try { + const siteSettings = await getSiteSettings().catch(() => null); + const localDetailAgentResolved = await resolveDetailAgentLocalMoonshotModel({ + siteSettingModelId: siteSettings?.canvasDetailAgentModelId, + }); + if (localDetailAgentResolved) { + return { + kind: "local_moonshot", + model: localDetailAgentResolved.model, + modelName: + localDetailAgentResolved.modelRef.displayName || + localDetailAgentResolved.modelRef.modelId, + }; + } + } catch (error) { + console.warn("[Chat API] Failed to resolve local Moonshot scene planning model:", error); + } + + const configuredModels = params.models || (await getEnabledAiModelsWithProvider()); + const configuredVisionModel = + configuredModels.find( + ({ model }) => + model.supportsVision && + !model.isSubModel && + QWEN_VL_MODEL_MATCHER.test(`${model.displayName} ${model.modelId}`), + ) || + configuredModels.find(({ model }) => model.supportsVision && !model.isSubModel); + + if (!configuredVisionModel) { + return null; + } + + try { + const languageModel = await getLanguageModelById(configuredVisionModel.model.id); + if (!languageModel) { + return null; + } + return { + kind: "configured_vision", + model: languageModel, + modelName: + configuredVisionModel.model.displayName || + configuredVisionModel.model.modelId, + }; + } catch (error) { + console.warn("[Chat API] Failed to resolve configured replacement scene planning model:", { + modelId: configuredVisionModel.model.id, + error, + }); + return null; + } +} + +export async function analyzeReplacementExecutionPlan(params: { + modeType?: "detail_replacement" | "sku_replacement"; + replacementStrategy?: AgentDetailReplacementStrategy; + referenceImageUrl: string; + referenceLabel: string; + productImageUrls: string[]; + modelImageUrl?: string | null; + workflowPrompt?: string | null; + planningRuntime: ReplacementScenePlanningRuntime; + models?: Awaited>; +}): Promise { + const isDetailReplacement = params.modeType === "detail_replacement"; + const isProductOnlyDetailReplacement = + isDetailReplacement && params.replacementStrategy === "product_only"; + if (params.planningRuntime) { + try { + const replacementScenePlanningTimeoutMs = + params.planningRuntime.kind === "local_moonshot" ? 90_000 : 45_000; + const analysisImageUrls = [params.referenceImageUrl]; + const { inlineDataUrls } = await inlineVisionImages(analysisImageUrls, { + maxImages: 1, + maxSide: 900, + timeoutMs: 12000, + retries: 2, + loggerPrefix: "[Chat API] Replacement scene plan", + }).catch((error) => { + console.warn("[Chat API] Failed to inline replacement scene planning images:", error); + return { inlineDataUrls: [], skipped: [] }; + }); + const effectiveVisionImages = + inlineDataUrls.length > 0 ? inlineDataUrls : analysisImageUrls; + const workflowPromptExcerpt = isProductOnlyDetailReplacement + ? "" + : String(params.workflowPrompt || "") + .trim() + .replace(/\s+/g, " ") + .slice(0, 2200); + const analysisPrompt = [ + isProductOnlyDetailReplacement + ? "你是电商详情参考图中的“产品主体替换规划器”。" + : isDetailReplacement + ? "你是电商详情参考图背景替换流程中的“版式与背景区域规划器”。" + : "你是电商详情替换流程中的“场景理解规划器”。", + `当前要分析的参考图：${params.referenceLabel}`, + "你现在只分析这一张参考图本身，不分析产品图，不分析模特图。", + "产品图和模特图会在后续生图阶段单独提供给生图模型使用，你这里不要尝试理解它们的具体外观，也不要要求必须使用所有输入图。", + params.modelImageUrl && !isProductOnlyDetailReplacement + ? "你只需要根据当前参考图判断：这一屏是否存在真实的人物展示需求；如果没有人物需求，useModelImage 必须为 false。" + : "本轮没有模特图，useModelImage 必须为 false。", + params.modelImageUrl && !isProductOnlyDetailReplacement + ? "如果当前参考图存在人物展示需求，后续生图必须尽量复用提供模特图中的同一人物身份，不要明显换脸、换发型、换体型。" + : "", + "这张图片是唯一的详情参考图底板，它决定当前这一屏的版式、文字排版、UI图标、按钮、机位、构图、留白和信息层级。", + isProductOnlyDetailReplacement + ? "这张参考图要求最大程度原样保留。后续生图只允许替换其中的产品主体/产品展示区域，背景、文案、按钮、图标、分割线、品牌标识、留白、装饰和光影氛围都必须保留。" + : isDetailReplacement + ? "这张参考图不是要求原样保留的场景照片。后续生图只允许替换/重建背景图、产品场景图、商品主体图或人物展示图区域；文字、标题、卖点标签、UI图标、按钮、分割线、品牌标识、留白和排版必须保留。" + : "这张参考图不是要求原样保留的场景照片。原图里的沙发、床、柜体、墙面、地面、窗帘、灯具、摆件、桌面道具、背景家具和环境光氛围，都可以在后续生图阶段按前置工作流风格重建。", + "请完成以下任务：", + isProductOnlyDetailReplacement + ? "1. 判断 shouldReplace：当前参考图里是否存在清晰、主要、适合被新产品替换的商品主体、产品照片区域或产品展示区。纯文字/纯参数/纯品牌介绍且没有产品展示区时才为 false。" + : isDetailReplacement + ? "1. 判断 shouldReplace：当前参考图里是否存在适合被替换/重建的背景图、产品场景图、商品主体图、人物展示图或大面积视觉图片区。纯文字/纯参数/纯品牌介绍且没有视觉图片区时才为 false。" + : "1. 判断 shouldReplace：当前参考图里是否存在清晰、主要、适合被替换的家具商品主体。", + "2. 判断 useModelImage，并说明原因。", + "3. 提炼 sceneSummary、cameraSummary、placementSummary、preserveSummary、styleSummary、avoidSummary。", + isProductOnlyDetailReplacement + ? "4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，要求只替换产品主体，保留其余背景、文案、排版、装饰和光影完全不变。" + : isDetailReplacement + ? "4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，描述需要替换的背景/场景图片区、产品放置方式、模特使用规则，以及必须保留的文字/UI/图标/排版。" + : "4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，描述这一屏需要的场景、机位、主体位置、周边环境重建方向以及必须保留的内容。", + isProductOnlyDetailReplacement + ? "5. sceneSummary、styleSummary 和 generationPrompt 必须强调‘只替换产品，不动其他内容’，不要引导重建场景。" + : "5. sceneSummary、styleSummary 和 generationPrompt 必须优先描述“按前置工作流重建后的目标场景”，而不是只复述参考图当前已有场景。", + isProductOnlyDetailReplacement + ? "6. avoidSummary 里要明确避免改背景、改文字、改按钮、改图标、改版式、改留白、改装饰、改光影氛围。" + : "6. 如果前置工作流风格与当前参考图原场景冲突，必须保留版式骨架，但放弃原场景物件本身，以前置工作流指定的空间风格、软装、灯光和氛围为准。", + isProductOnlyDetailReplacement + ? "7. 不要根据用户提示词或工作流风格改造场景，只服务于产品主体替换。" + : "7. avoidSummary 里要明确避免“只替换产品、不重建场景”这类结果。", + workflowPromptExcerpt + ? `当前风格与场景限制（来自前置工作流与模式模板）:\n${workflowPromptExcerpt}` + : "", + "只返回严格 JSON，不要任何额外解释。", + `JSON 格式： +{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字","useModelImage":false,"modelImageReason":"不超过40字","sceneSummary":"不超过80字","cameraSummary":"不超过60字","placementSummary":"不超过60字","preserveSummary":"不超过80字","styleSummary":"不超过80字","avoidSummary":"不超过80字","generationPrompt":"不超过300字"}`, + ] + .filter(Boolean) + .join("\n"); + + const { text } = await generateText({ + model: params.planningRuntime.model, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: analysisPrompt, + }, + ...effectiveVisionImages.map((image) => ({ + type: "image" as const, + image, + })), + ], + }, + ], + system: + "你是高精度商品详情图场景规划器。你只分析当前参考图本身，负责理解这一屏的场景、机位、主体位置和人物需求，不要臆测或依赖其他输入图片。", + temperature: params.planningRuntime.kind === "local_moonshot" ? 1 : 0.1, + maxTokens: 1200, + abortSignal: AbortSignal.timeout(replacementScenePlanningTimeoutMs), + } as any); + + const parsedDecision = parseReplacementReferenceDecision(text); + const parsedScenePlan = parseReplacementScenePlan(text, { + productImageCount: params.productImageUrls.length, + hasModelImage: isProductOnlyDetailReplacement + ? false + : Boolean(params.modelImageUrl), + }); + + if (parsedDecision && parsedScenePlan) { + return { + ...parsedDecision, + detectionModelName: params.planningRuntime.modelName, + detectionSource: + params.planningRuntime.kind === "local_moonshot" + ? "local_moonshot" + : "configured_vision", + scenePlan: { + ...parsedScenePlan, + analysisModelName: params.planningRuntime.modelName, + analysisSource: + params.planningRuntime.kind === "local_moonshot" + ? "local_moonshot" + : "configured_vision", + }, + }; + } + } catch (error) { + console.warn("[Chat API] Replacement scene planning analysis failed:", { + referenceLabel: params.referenceLabel, + referenceImageUrl: params.referenceImageUrl, + error, + }); + } + } + + const fallbackDecision = await analyzeReplacementReferenceImage({ + imageUrl: params.referenceImageUrl, + models: params.models, + modeType: params.modeType, + replacementStrategy: params.replacementStrategy, + }); + return { + ...fallbackDecision, + scenePlan: buildHeuristicReplacementScenePlan({ + productImageCount: params.productImageUrls.length, + hasModelImage: isProductOnlyDetailReplacement + ? false + : Boolean(params.modelImageUrl), + shouldReplace: fallbackDecision.shouldReplace, + reason: fallbackDecision.reason, + replacementStrategy: params.replacementStrategy, + }), + }; +} + +export function buildAgentReplacementPrompt(params: { + basePrompt: string; + modeType: "detail_replacement" | "sku_replacement"; + replacementStrategy?: AgentDetailReplacementStrategy; + referenceLabel: string; + referenceIndex: number; + totalReferences: number; + hasModelImage: boolean; + productImageCount: number; + originalProductImageCount: number; + templateReferenceCount: number; + pairedProductIndex?: number; + scenePlan?: ReplacementScenePlan | null; + referenceGeometry?: ReplacementReferenceGeometry | null; +}) { + const imageOrderLines: string[] = []; + let nextImageIndex = 1; + + if (params.productImageCount > 0) { + const productRangeLabel = + params.productImageCount === 1 + ? `图片${nextImageIndex}` + : `图片${nextImageIndex}-${nextImageIndex + params.productImageCount - 1}`; + imageOrderLines.push( + `${productRangeLabel}：同一产品的白底图/细节图，必须综合理解所有产品图，不可只看第一张。` + ); + nextImageIndex += params.productImageCount; + } + + if (params.hasModelImage) { + imageOrderLines.push( + `图片${nextImageIndex}：模特图，仅在当前参考图本身包含人物展示时才允许参考。` + ); + nextImageIndex += 1; + } + + if (params.templateReferenceCount > 0) { + const templateRangeLabel = + params.templateReferenceCount === 1 + ? `图片${nextImageIndex}` + : `图片${nextImageIndex}-${nextImageIndex + params.templateReferenceCount - 1}`; + imageOrderLines.push( + `${templateRangeLabel}：模板补充参考图，只能补充风格/材质氛围，不能覆盖最终参考图版式。` + ); + nextImageIndex += params.templateReferenceCount; + } + + imageOrderLines.push( + `图片${nextImageIndex}：当前参考图底板（唯一版式基准图），必须保留它的文字排版、信息区、分割线、留白、机位和整体结构。` + ); + + const scenePlanLines = params.scenePlan + ? [ + "【本屏场景预分析】", + params.scenePlan.sceneSummary + ? `场景理解：${params.scenePlan.sceneSummary}` + : "", + params.scenePlan.cameraSummary + ? `机位构图：${params.scenePlan.cameraSummary}` + : "", + params.scenePlan.placementSummary + ? `主体位置：${params.scenePlan.placementSummary}` + : "", + params.scenePlan.preserveSummary + ? `必须保留：${params.scenePlan.preserveSummary}` + : "", + params.scenePlan.styleSummary + ? `风格约束：${params.scenePlan.styleSummary}` + : "", + params.scenePlan.avoidSummary + ? `禁止事项：${params.scenePlan.avoidSummary}` + : "", + params.originalProductImageCount > params.productImageCount + ? `原始共有 ${params.originalProductImageCount} 张产品角度图，本屏已筛选出最相关的 ${params.productImageCount} 张输入图；其余未入选产品图只是补充理解，不需要也不允许强行全部融入同一张结果图。` + : "", + params.scenePlan.productSelectionReason + ? `产品图筛选理由：${params.scenePlan.productSelectionReason}` + : "", + params.scenePlan.useModelImage + ? params.scenePlan.modelImageReason + ? `本屏允许使用模特图：${params.scenePlan.modelImageReason}` + : "本屏允许使用模特图，但仅在参考图当前这一屏确实存在人物展示需求时使用。" + : params.hasModelImage + ? params.scenePlan.modelImageReason + ? `本屏禁止使用模特图：${params.scenePlan.modelImageReason}` + : "本屏不需要模特图，不允许把模特图强行融入结果。" + : "", + params.scenePlan.generationPrompt + ? `本屏生成方向：${params.scenePlan.generationPrompt}` + : "", + ].filter(Boolean) + : []; + + const productDetailInstructionLines = + params.replacementStrategy === "product_only" + ? [ + "【执行模式】详情替换 / 仅替换产品", + `当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张详情参考图：${params.referenceLabel}`, + "【图片顺序规则】", + ...imageOrderLines, + ...scenePlanLines, + "你现在做的是‘仅替换产品’，不是重建场景，也不是重新设计整张详情图。", + "忽略用户补充提示词里的场景风格、氛围、空间改造要求；这一次只服务于产品主体替换。", + "最后一张参考图是唯一底板，除产品主体/产品展示区外，其余所有内容都必须保持不变。", + "必须严格保留：背景、人物、手部姿态、桌面摆件、灯光氛围、阴影关系、文字排版、标题、卖点标签、按钮、UI图标、分割线、品牌标识、留白、画布比例和信息层级。", + params.referenceGeometry + ? `当前参考图画布尺寸为 ${params.referenceGeometry.width}x${params.referenceGeometry.height}，输出必须保持这个页面画布比例。替换商品时只能等比缩放、透视匹配和自然融合，绝对禁止把商品、人物、文字、图标横向或纵向拉伸变形。` + : "输出必须保持当前参考图页面画布比例。替换商品时只能等比缩放、透视匹配和自然融合，绝对禁止把商品、人物、文字、图标横向或纵向拉伸变形。", + "镜头角度、透视关系、视平线、主体朝向和主体占比必须尽量跟随最后一张参考图，方便无缝替换当前产品区域。", + params.productImageCount > 1 + ? `前面的 ${params.productImageCount} 张商品输入图都属于同一产品的多角度/多细节白底图，请综合这些图片理解产品结构、材质、颜色、比例、边角、开孔、五金与细节，不可只参考第一张。` + : "前面的商品输入图就是当前要替换进去的产品白底图，请准确保持该产品的结构、材质、颜色、比例和细节。", + "你的任务只有一个：识别参考图中的旧产品主体或产品展示区，用输入产品替换它，并让新产品与原场景自然融合。", + "不要重建背景，不要重画空间，不要调整文案，不要修改按钮、图标、标签、装饰元素，也不要改变人物、道具和环境光氛围。", + "如果参考图中没有清晰可替换的产品主体或产品展示区，应直接沿用参考图，不要硬生成。", + "不要把多个产品角度同时塞进一张结果图，不要新增无关主体，不要删除任何关键版式或 UI 元素。", + ] + : [ + "【执行模式】详情替换", + `当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张详情参考图：${params.referenceLabel}`, + "【图片顺序规则】", + ...imageOrderLines, + ...scenePlanLines, + "你现在做的是“详情参考图背景/场景图替换”，不是重新设计整张详情图。", + "要把最后一张参考图理解为“页面版式底板 + 背景图/场景图区参考”，不是“旧场景照片必须原样保留”。", + "前面“工作流设定”里的场景组合、空间风格、硬装方案、软装方案、灯光氛围、人群设定，必须作为当前这一屏的强制场景改造要求生效。", + "场景风格必须优先执行前面用户提示词与工作流设定中给出的空间风格、材质、灯光和氛围描述，不要被参考图原有旧场景绑死。", + "执行时要基于这些场景组合要求，重建参考图中的背景图/产品场景图区域，使当前这一屏呈现出与第一步已选风格一致的新背景画面。", + "允许替换/重建的范围是参考图里的背景图、产品场景图、商品主体图和必要的人物展示图区；不允许改动版式层。", + "如果前面商品图、模特图与最后一张参考图底板存在冲突，始终以最后一张参考图底板的版式、机位、构图和信息层级为最高优先级。", + "最后一张参考图是唯一底板，必须完整保留这张图的文字排版、品牌字样、标题区、卖点区、UI图标、按钮、标签、装饰元素、分割线、留白、画布比例和信息层级。", + params.referenceGeometry + ? `当前参考图画布尺寸为 ${params.referenceGeometry.width}x${params.referenceGeometry.height}，输出必须保持这个页面画布比例。替换背景/商品时只能等比缩放后裁切或延展背景，绝对禁止把商品、人物、文字、图标横向或纵向拉伸变形。` + : "输出必须保持当前参考图页面画布比例。替换背景/商品时只能等比缩放后裁切或延展背景，绝对禁止把商品、人物、文字、图标横向或纵向拉伸变形。", + "镜头角度、透视关系、视平线、主体朝向和主体占比必须尽量跟随最后一张参考图，方便无缝替换当前版式，不要擅自改成完全不同的视角。", + "前面的商品图和模特图用于生成新的背景图/产品场景图内容，并按工作流设定替换参考图中的旧背景画面，但不能改变参考图的版式设计。", + "可被替换重建的内容包括但不限于：背景照片、产品展示照片、墙面、地面材质、沙发、床、背景柜、窗帘、灯具、地毯、挂画、摆件、桌面道具、空间色调、环境光氛围。", + params.productImageCount > 1 + ? `前面的 ${params.productImageCount} 张商品输入图都属于同一产品的多角度/多细节白底图，请综合这些图片理解产品结构、材质、颜色、比例、边角、开孔、五金与细节，不可只参考第一张。` + : "前面的商品输入图就是当前要替换进去的产品白底图，请准确保持该产品的结构、材质、颜色、比例和细节。", + "请先识别最后一张参考图中需要被替换的背景图/场景图区域，并在同一版式骨架下生成新的背景画面；不要只做局部抠图式换产品。", + "请严格保留最后一张参考图的版式结构、镜头关系、构图、文字排版、UI图标、标签、按钮、留白位置和信息层级；但旧背景图里的家具、背景物件、材质、灯光氛围、软装小物、空间质感需要按第一步工作流设定进行明显重建。", + "如果第一步工作流风格与参考图原场景冲突，必须保留版式骨架和UI层，但放弃原背景图物件本身，以第一步工作流风格为准。", + "前面的输入商品图用于提供商品主体、材质、颜色和细节特征；如有模特图，请只在参考图本身包含人物展示时再参考模特图。", + "如果提供了模特图且这一屏确实有人物展示需求，必须尽量复用同一人物身份与体态关系，不要明显换脸、换发型、换体型，服装风格也尽量保持一致。", + "绝对不要忽略最后一张参考图去生成一张无关的新场景图。", + "如果生成结果仍然保留了参考图原有的大部分背景图物件，只是把产品换掉，则视为执行失败；你必须让背景图/场景图发生清晰可见的变化。", + "绝对不要移动、删除、改写、重排参考图里的任何文字、UI图标、按钮、标签、装饰元素与排版元素。", + "输出结果必须保持最后一张参考图底板的原始纵横比与画布尺寸，不允许擅自改成长图、横图或裁掉任何版面区域。", + "如果参考图中没有可替换/重建的背景图、产品场景图或视觉图片区，应直接沿用参考图，不要硬生成。", + "不要改动参考图的整体布局，不要新增无关主体，不要删除关键版式或UI元素。", + ]; + + const skuInstructionLines = [ + "【执行模式】SKU替换", + `当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张 SKU 参考图：${params.referenceLabel}`, + "【图片顺序规则】", + ...imageOrderLines, + ...scenePlanLines, + params.pairedProductIndex !== undefined + ? `请优先使用第 ${params.pairedProductIndex + 1} 张输入商品图作为本张参考图的一对一替换主体。` + : "请使用输入商品图作为本张参考图的一对一替换主体。", + params.productImageCount > 1 + ? "其余商品输入图同样是这个产品的补充角度/细节参考，请一并综合理解产品信息，不可忽略。" + : "输入商品图就是当前产品主体，请保持产品信息准确一致。", + "如果前面商品图与最后一张参考图存在冲突，始终以最后一张参考图的机位、场景、光影和构图为准。", + "请严格保留最后一张参考图的场景、机位、透视、光影和构图关系，只替换商品主体本身。", + params.referenceGeometry + ? `当前参考图画布尺寸为 ${params.referenceGeometry.width}x${params.referenceGeometry.height}，输出必须保持这个画布比例。替换商品时只能等比缩放、透视匹配和自然融合，禁止把商品横向或纵向拉伸变形。` + : "输出必须保持当前参考图画布比例。替换商品时只能等比缩放、透视匹配和自然融合，禁止把商品横向或纵向拉伸变形。", + "输出结果必须保持最后一张参考图的原始纵横比与画布尺寸。", + "输出要自然融合，不要改变参考图原有风格和景别。", + ]; + + const instructionLines = + params.modeType === "detail_replacement" + ? productDetailInstructionLines + : skuInstructionLines; + + if (params.hasModelImage) { + instructionLines.push("如有模特图，请在需要人物展示时保持模特体态、穿搭和出镜关系一致。"); + } + + return [ + params.modeType === "detail_replacement" && + params.replacementStrategy === "product_only" + ? "" + : params.basePrompt, + instructionLines.join("\n"), + ] + .filter(Boolean) + .join("\n\n") + .trim(); +} diff --git a/app/(chat)/api/chat/_lib/image-failover.ts b/app/(chat)/api/chat/_lib/image-failover.ts new file mode 100644 index 000000000..4b2d54663 --- /dev/null +++ b/app/(chat)/api/chat/_lib/image-failover.ts @@ -0,0 +1,259 @@ +import { isFalGptImage2EditModelId } from "@/lib/ai/providers/fal-image"; +import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; +import type { ChatMessage } from "@/lib/types"; +import { convertMessagesWithImages } from "./image-media"; + +export const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([ + "alibaba-image", + "bfl-image", + "ruxa-image", + "fal-image", + "openai-image", + "stepfun-image", + "runway", + "krea", + "grsai-gpt-image", + "volcano-vision", + "gemini", + "huanke-gemini", +]); + +const IMAGE_FAILOVER_MODEL_COOLDOWN_MS = 30 * 60 * 1000; +const imageFailoverModelCooldown = new Map(); + +export function normalizeFailoverErrorMessage(error: unknown): string { + if (!error) return ""; + if (typeof error === "string") return error; + if (error instanceof Error) return error.message || String(error); + try { + return JSON.stringify(error); + } catch { + return String(error); + } +} + +export function shouldCooldownImageFallbackModel(errorMessage: string): boolean { + const msg = errorMessage.toLowerCase(); + return ( + msg.includes("model not found") || + msg.includes("model_not_found") || + msg.includes("no available channel for model") || + msg.includes("unknown model") || + msg.includes("invalid model") || + msg.includes("api key is missing") || + msg.includes("invalid api key") || + msg.includes("unauthorized") || + msg.includes("cannot convert argument to a bytestring") + ); +} + +export function isImageFallbackModelCoolingDown(modelId: string): boolean { + const snapshot = imageFailoverModelCooldown.get(modelId); + if (!snapshot) return false; + if (snapshot.until <= Date.now()) { + imageFailoverModelCooldown.delete(modelId); + return false; + } + return true; +} + +export function setImageFallbackModelCooldown(modelId: string, reason: string): void { + imageFailoverModelCooldown.set(modelId, { + until: Date.now() + IMAGE_FAILOVER_MODEL_COOLDOWN_MS, + reason: reason.slice(0, 500), + }); +} + +export function isImageFailoverExcludedModel(model: { excludeFromFailover?: boolean } | null | undefined) { + return Boolean(model?.excludeFromFailover); +} + +export type ImageFallbackCandidate = { + model: any; + provider: any; + configured: boolean; + sameProvider: boolean; +}; + +export function getImageFailoverConvertProvider(providerType?: string | null) { + switch (providerType) { + case "bfl-image": + case "ruxa-image": + case "fal-image": + case "openai-image": + case "stepfun-image": + case "midjourney": + case "runway": + case "krea": + case "gemini": + case "huanke-gemini": + return providerType; + case "grsai-gpt-image": + return "gemini"; + case "volcano-vision": + return "volcano"; + default: + return "alibaba-image"; + } +} + +export function buildImageFallbackCandidates( + allModels: Array<{ model: any; provider: any }>, + options: { + currentProviderId?: string; + currentModelId?: string; + currentModelKey?: string; + configuredFallbackIds?: string[]; + } +) { + const { currentProviderId, currentModelId, currentModelKey, configuredFallbackIds = [] } = options; + const configuredRefs = configuredFallbackIds + .map((item) => String(item || "").trim()) + .filter(Boolean); + const byModelRef = new Map(); + + const isEligible = (item: { model: any; provider: any }, sameProvider: boolean) => { + if ( + !item?.model?.supportsImageGen || + !item?.model?.isEnabled || + item?.model?.isSubModel || + isImageFailoverExcludedModel(item?.model) || + item?.model?.modelId === currentModelKey || + item?.model?.id === currentModelId + ) { + return false; + } + + const providerType = String(item?.provider?.type || ""); + if (!sameProvider && !IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(providerType)) { + return false; + } + if (isImageFallbackModelCoolingDown(item.model.id)) { + const cooldown = imageFailoverModelCooldown.get(item.model.id); + console.log( + `[Chat API] ⏭ Skip configured image fallback on cooldown: ${item.model.displayName} (${providerType}) - ${cooldown?.reason || "cooldown"}` + ); + return false; + } + return true; + }; + + allModels.forEach((item) => { + if (!item?.model) return; + byModelRef.set(String(item.model.id), item); + if (item.model.modelId) { + byModelRef.set(String(item.model.modelId), item); + } + }); + + const candidates: ImageFallbackCandidate[] = []; + const seenModelIds = new Set(); + const pushCandidate = (item: { model: any; provider: any } | undefined, configured: boolean) => { + if (!item?.model) return; + const sameProvider = Boolean(currentProviderId && item.provider?.id === currentProviderId); + if (!isEligible(item, sameProvider)) return; + const modelId = String(item.model.id || ""); + if (!modelId || seenModelIds.has(modelId)) return; + seenModelIds.add(modelId); + candidates.push({ + model: item.model, + provider: item.provider, + configured, + sameProvider, + }); + }; + + configuredRefs.forEach((modelRef) => { + pushCandidate(byModelRef.get(modelRef), true); + }); + + if (currentProviderId) { + allModels + .filter((item) => item.provider?.id === currentProviderId) + .forEach((item) => pushCandidate(item, false)); + } + + return candidates; +} + +export async function generateImageWithFailoverCandidate(params: { + candidate: ImageFallbackCandidate; + messages: ChatMessage[]; + imageParams: any; + fallbackImageParams: any; +}) { + const { candidate, messages, imageParams, fallbackImageParams } = params; + const providerType = String(candidate.provider?.type || ""); + const candidateRuntimeModelId = String(candidate.model?.modelId || candidate.model?.id || ""); + if ( + providerType === "fal-image" && + isFalGptImage2EditModelId(candidateRuntimeModelId) + ) { + throw new Error("FAL GPT Image 2 Edit 仅支持异步队列调用，已跳过同步故障迁移"); + } + const candidateLanguageModel = await getLanguageModelById(candidate.model.id); + if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") { + throw new Error(`备用模型不支持当前生图故障迁移调用：${candidate.model.displayName || candidate.model.id}`); + } + + const candidateMessages = await convertMessagesWithImages( + messages, + getImageFailoverConvertProvider(providerType) + ); + const candidateResult = await (candidateLanguageModel as any).doGenerate({ + messages: candidateMessages, + imageParams: imageParams || fallbackImageParams, + modelDisplayName: candidate.model.displayName, + overrideModelId: candidate.model.modelId, + }); + + if (candidateResult?.finishReason === "error") { + throw new Error(candidateResult.text || "图片生成失败"); + } + + return { + text: candidateResult.text || "图片生成失败", + usedModelId: String(candidate.model.modelId || candidate.model.id || ""), + displayName: String(candidate.model.displayName || candidate.model.modelId || candidate.model.id || ""), + }; +} + +export function getImageCrossProviderCandidates( + allModels: any[], + options: { + currentModelId: string; + currentProviderId?: string; + providerPriority: Record; + } +) { + return allModels + .filter((m) => { + if ( + !m?.model?.supportsImageGen || + !m?.model?.isEnabled || + m?.model?.isSubModel || + isImageFailoverExcludedModel(m?.model) + ) { + return false; + } + if (!IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(String(m?.provider?.type || ""))) { + return false; + } + if (m.model.id === options.currentModelId) return false; + if (options.currentProviderId && m.provider.id === options.currentProviderId) return false; + if (isImageFallbackModelCoolingDown(m.model.id)) { + const cooldown = imageFailoverModelCooldown.get(m.model.id); + console.log( + `[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "cooldown"}` + ); + return false; + } + return true; + }) + .sort((a, b) => { + const left = options.providerPriority[a.provider.type as string] ?? 99; + const right = options.providerPriority[b.provider.type as string] ?? 99; + if (left !== right) return left - right; + return (a.model.sortOrder ?? 0) - (b.model.sortOrder ?? 0); + }); +} diff --git a/app/(chat)/api/chat/_lib/image-media.ts b/app/(chat)/api/chat/_lib/image-media.ts new file mode 100644 index 000000000..4b42d5ade --- /dev/null +++ b/app/(chat)/api/chat/_lib/image-media.ts @@ -0,0 +1,964 @@ +import { convertToModelMessages } from "ai"; +import { inlineVisionImages } from "@/lib/ai/vision-inline"; +import { extractImageUrlsFromText } from "@/lib/image-url"; +import { parseImageAltMetadata } from "@/lib/ai/runtime/generated-image-persistence"; +import { uploadFile } from "@/lib/storage/service"; +import type { ChatMessage } from "@/lib/types"; +import { ANNOTATION_SYSTEM_PROMPT, hasAnnotatedImage } from "./message-preparation"; + +export function guessImageMediaType(url: string): string { + const cleanUrl = url.split("?")[0]?.toLowerCase() || ""; + if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "image/jpeg"; + if (cleanUrl.endsWith(".webp")) return "image/webp"; + if (cleanUrl.endsWith(".gif")) return "image/gif"; + if (cleanUrl.endsWith(".bmp")) return "image/bmp"; + if (cleanUrl.endsWith(".svg")) return "image/svg+xml"; + return "image/png"; +} + +export function injectHistoryImagesToLastUserMessage( + messages: ChatMessage[], + historyImages: string[] +): ChatMessage[] { + if (!historyImages || historyImages.length === 0) return messages; + + let lastUserIndex = -1; + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i]?.role === "user") { + lastUserIndex = i; + break; + } + } + + if (lastUserIndex < 0) return messages; + + const lastUserMessage = messages[lastUserIndex]; + const hasExistingImage = lastUserMessage.parts?.some( + (p: any) => p.type === "file" || p.type === "image" || p.type === "image_url" + ); + if (hasExistingImage) return messages; + + const imageParts = historyImages.map((url, index) => ({ + type: "file" as const, + url, + name: `history-image-${index + 1}`, + mediaType: guessImageMediaType(url), + mimeType: guessImageMediaType(url), + })); + + const updatedMessage: ChatMessage = { + ...lastUserMessage, + parts: [...lastUserMessage.parts, ...imageParts], + }; + + return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg)); +} + +export function isImageInputPart(part: any) { + return part?.type === "file" || part?.type === "image" || part?.type === "image_url"; +} + +export function normalizeResolvedTargetImageUrls(imageUrls?: string[] | null) { + return Array.from( + new Set( + (imageUrls || []) + .map((url) => String(url || "").trim()) + .filter((url) => /^https?:\/\//i.test(url)) + ) + ).slice(0, 8); +} + +export function summarizeTargetImageUrl(url?: string) { + const normalized = String(url || "").trim(); + if (!normalized) return undefined; + try { + const parsed = new URL(normalized); + return `${parsed.origin}${parsed.pathname}`.slice(0, 160); + } catch { + return normalized.slice(0, 160); + } +} + +export function applyResolvedTargetImagesToLastUserMessage( + messages: ChatMessage[], + targetImageUrls: string[] +): ChatMessage[] { + const normalizedTargetImageUrls = normalizeResolvedTargetImageUrls(targetImageUrls); + if (normalizedTargetImageUrls.length === 0) return messages; + + let lastUserIndex = -1; + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i]?.role === "user") { + lastUserIndex = i; + break; + } + } + + if (lastUserIndex < 0) return messages; + const lastUserMessage = messages[lastUserIndex]; + const nonImageParts = (lastUserMessage.parts || []).filter((part: any) => !isImageInputPart(part)); + const imageParts = normalizedTargetImageUrls.map((url, index) => ({ + type: "file" as const, + url, + name: `target-image-${index + 1}`, + mediaType: guessImageMediaType(url), + mimeType: guessImageMediaType(url), + })); + + const updatedMessage: ChatMessage = { + ...lastUserMessage, + parts: [...imageParts, ...nonImageParts], + }; + + return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg)); +} + +export function compactAutoDemand(text: string, maxLength = 56): string { + const normalized = text.replace(/\s+/g, " ").trim(); + if (!normalized) return "按你刚才的描述执行"; + if (normalized.length <= maxLength) return normalized; + return `${normalized.slice(0, maxLength)}...`; +} + +export function buildAutoModeFallbackMessage(params: { + text: string; + hasImage: boolean; + fromHistory: boolean; + imageSummary?: string; +}): string { + const demand = compactAutoDemand(params.text); + + if (params.hasImage) { + const sourceLine = params.fromHistory + ? "基于上一轮图片继续处理。" + : "基于当前上传图片处理。"; + + return [ + "### 需求理解", + `- ${demand}`, + "### 图片分析", + params.imageSummary + ? `- ${params.imageSummary.replace(/\s+/g, " ").trim()}` + : `- ${sourceLine}`, + "- 我会优先保持主体、构图和整体风格一致，只处理你指定的改动。", + "### 执行计划", + "- 先完成本轮核心改动，再检查边缘与光影一致性。", + "- 若结果偏差，我会按“保留A，修改B”的方式继续迭代。", + "### 输出设置", + "- 默认单图输出，优先稳定命中。", + ].join("\n"); + } + + return [ + "### 需求理解", + `- ${demand}`, + "### 执行计划", + "- 先锁定主体与场景，再细化风格、材质与光影。", + "### 输出设置", + "- 默认单图输出，优先稳定命中。", + ].join("\n"); +} + +export function sanitizePromptOnlyUserIntent(text: string): string { + return String(text || "") + .replace(/先?分析(?:一下)?(?:这张)?(?:图片|图像|这张图)?/g, "") + .replace(/帮我|给我|先给我|只给我|仅给我/g, "") + .replace(/优化(?:一下)?提示词|提示词优化|提示词|prompt/gi, "") + .replace(/不要生图|先别生图|不需要生图|不要生成|别生成|不直接生成/g, "") + .replace(/[，。；：,.!！?？]/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +export function buildPromptOnlyOptimizationSeed(params: { + text: string; + imageSummary?: string; +}): string { + const cleanedIntent = sanitizePromptOnlyUserIntent(params.text); + const summary = String(params.imageSummary || "").replace(/\s+/g, " ").trim(); + + if (cleanedIntent && summary) { + return `${cleanedIntent}。参考图关键特征：${summary}`; + } + if (summary) { + return `基于参考图生成一段可直接复现核心主体、场景、风格与光线特征的中文提示词。参考图关键特征：${summary}`; + } + if (cleanedIntent) { + return cleanedIntent; + } + return "基于当前参考图整理一段可直接使用的中文提示词。"; +} + +export function buildPromptOnlyReply(params: { + userText: string; + imageSummary?: string; + optimizedPrompt: string; +}): string { + const demand = sanitizePromptOnlyUserIntent(params.userText) || "你希望我先分析这张图并整理可直接使用的提示词"; + const summaryLines = String(params.imageSummary || "") + .split(/\r?\n/) + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter(Boolean); + const formattedSummary = + summaryLines.length > 0 + ? summaryLines.map((line) => `- ${line}`).join("\n") + : "- 我已基于当前图片梳理主体、场景、风格与光线特征。"; + + return [ + "【需求理解】", + `- ${demand}`, + "【图片分析】", + formattedSummary, + "【优化后提示词】", + `- ${params.optimizedPrompt.replace(/\s+/g, " ").trim()}`, + "【使用建议】", + "- 本轮仅输出优化后的提示词，不直接执行生图。", + ].join("\n"); +} + +export function hasStructuredAgentSections(message?: string): boolean { + if (!message) return false; + return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test( + message, + ); +} + +export function ensureStructuredAgentFriendlyMessage(params: { + message?: string; + text: string; + hasImage: boolean; + fromHistory: boolean; +}): string { + const existingMessage = String(params.message || "").trim(); + if (hasStructuredAgentSections(existingMessage)) { + return existingMessage; + } + + const fallback = buildAutoModeFallbackMessage({ + text: params.text, + hasImage: params.hasImage, + fromHistory: params.fromHistory, + }); + + if (!existingMessage) { + return fallback; + } + + const normalizedSupplement = existingMessage.replace(/\s+/g, " ").trim(); + if (!normalizedSupplement) { + return fallback; + } + + return fallback.replace( + "### 执行计划\n", + `### 执行计划\n- 补充说明：${normalizedSupplement}\n`, + ); +} + +export function extractMessageImageUrls(parts: any[]): string[] { + return (parts || []) + .map((part: any) => { + if (!part) return null; + if (part.type === "file") return part.url || part.data || null; + if (part.type === "image") return part.url || part.image || null; + if (part.type === "image_url") return part.image_url?.url || null; + return null; + }) + .filter((url: any): url is string => typeof url === "string" && /^https?:\/\//i.test(url)); +} + +export function extractMessageImageAssetIds(parts: any[]): string[] { + return Array.from( + new Set( + (parts || []) + .map((part: any) => { + if (!part || typeof part !== "object") return null; + return typeof part.assetId === "string" && part.assetId.trim() + ? part.assetId.trim() + : null; + }) + .filter((assetId: any): assetId is string => typeof assetId === "string" && assetId.length > 0), + ), + ); +} + +export function shouldEnrichImageAnalysisText(message?: string): boolean { + if (!message) return false; + if (!/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) return false; + return /(基于当前上传图片处理|基于你选中的原图处理|基于上一轮图片继续处理)/.test(message); +} + +export function injectImageAnalysisSummary(message: string, summary: string): string { + const normalizedSummaryLines = summary + .split(/\r?\n/) + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter(Boolean); + if (normalizedSummaryLines.length === 0) return message; + const lines = message.split("\n"); + const analysisHeaderIndex = lines.findIndex((line) => + /^(###\s*图片分析|【\s*图片分析\s*】)/.test(line.trim()) + ); + if (analysisHeaderIndex < 0) return message; + + for (let i = analysisHeaderIndex + 1; i < lines.length; i += 1) { + const current = lines[i]?.trim() || ""; + if (/^(###\s*|【\s*[^】]+\s*】)/.test(current)) break; + if (/^- /.test(current)) { + lines.splice(i, 1, ...normalizedSummaryLines.map((line) => `- ${line}`)); + return lines.join("\n"); + } + } + + const insertIndex = analysisHeaderIndex + 1; + lines.splice(insertIndex, 0, ...normalizedSummaryLines.map((line) => `- ${line}`)); + return lines.join("\n"); +} + +export function ensureImageAnalysisSummarySection(message: string, summary: string): string { + const normalizedSummaryLines = summary + .split(/\r?\n/) + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter(Boolean); + if (normalizedSummaryLines.length === 0) return message; + + if (/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) { + return injectImageAnalysisSummary(message, normalizedSummaryLines.join("\n")); + } + + const trimmed = message.trim(); + const summaryBlock = normalizedSummaryLines.map((line) => `- ${line}`).join("\n"); + if (!trimmed) return `【图片分析】\n${summaryBlock}`; + return `${trimmed}\n【图片分析】\n${summaryBlock}`; +} + +/** + * 自定义消息转换函数 + * 注意：gemini-image 和 huanke-gemini 都需要 URL 格式（不是 Base64） + * 因为这些 provider 内部会自行处理图片上传到 S3 + */ +export async function convertMessagesWithImages( + messages: ChatMessage[], + providerType: + | string + | { + providerType?: string; + providerBaseUrl?: string | null; + modelId?: string | null; + displayName?: string | null; + forceInlineImageUrls?: boolean; + loggerPrefix?: string; + }, +) { + type ImageConversionTarget = { + providerType?: string; + providerBaseUrl?: string | null; + modelId?: string | null; + displayName?: string | null; + forceInlineImageUrls?: boolean; + loggerPrefix?: string; + }; + + const normalizeTarget = ( + target: string | ImageConversionTarget, + ): Required> & + Omit => { + if (typeof target === "string") { + return { + providerType: target, + providerBaseUrl: null, + modelId: null, + displayName: null, + forceInlineImageUrls: false, + loggerPrefix: "[Chat API] Vision inline", + }; + } + + return { + providerType: String(target.providerType || ""), + providerBaseUrl: target.providerBaseUrl || null, + modelId: target.modelId || null, + displayName: target.displayName || null, + forceInlineImageUrls: target.forceInlineImageUrls === true, + loggerPrefix: target.loggerPrefix || "[Chat API] Vision inline", + }; + }; + + const extractImagePartUrl = (part: unknown): string | null => { + const partAny = part as any; + const candidateUrl = + partAny?.type === "file" + ? partAny.url || partAny.data + : partAny?.type === "image" + ? partAny.image || partAny.url + : partAny?.type === "image_url" + ? partAny.image_url?.url + : undefined; + + if (typeof candidateUrl !== "string") return null; + const normalized = candidateUrl.trim(); + if (!/^https?:\/\//i.test(normalized)) return null; + return normalized; + }; + + const replaceImagePartUrl = (part: unknown, nextUrl: string): unknown => { + const partAny = part as any; + if (partAny?.type === "file") { + return { + ...partAny, + url: nextUrl, + data: nextUrl, + mediaType: String(partAny.mediaType || "").startsWith("image/") + ? partAny.mediaType + : "image/jpeg", + mimeType: String(partAny.mimeType || "").startsWith("image/") + ? partAny.mimeType + : "image/jpeg", + }; + } + if (partAny?.type === "image") { + return { + ...partAny, + image: nextUrl, + url: nextUrl, + }; + } + if (partAny?.type === "image_url") { + return { + ...partAny, + image_url: { + ...(partAny.image_url || {}), + url: nextUrl, + }, + }; + } + return part; + }; + + const inlineImageUrlsInMessages = async ( + sourceMessages: ChatMessage[], + target: ReturnType, + ) => { + const uniqueUrls = Array.from( + new Set( + sourceMessages + .flatMap((message) => message.parts || []) + .map((part) => extractImagePartUrl(part)) + .filter((url): url is string => Boolean(url)), + ), + ); + + if (uniqueUrls.length === 0) { + return sourceMessages; + } + + const { inlineDataUrls } = await inlineVisionImages(uniqueUrls, { + maxImages: Math.min(8, uniqueUrls.length), + maxSide: 1200, + timeoutMs: 12000, + retries: 2, + loggerPrefix: target.loggerPrefix, + }).catch((error) => { + console.warn(`${target.loggerPrefix} failed to inline message images:`, error); + return { inlineDataUrls: [], skipped: [] }; + }); + + if (inlineDataUrls.length === 0) { + return sourceMessages; + } + + const inlineMap = new Map(); + for (let index = 0; index < inlineDataUrls.length; index += 1) { + const originalUrl = uniqueUrls[index]; + const inlineUrl = inlineDataUrls[index]; + if (originalUrl && inlineUrl) { + inlineMap.set(originalUrl, inlineUrl); + } + } + + if (inlineMap.size === 0) { + return sourceMessages; + } + + console.log(`${target.loggerPrefix} inlined chat message images:`, { + total: uniqueUrls.length, + inlined: inlineMap.size, + providerType: target.providerType, + modelId: target.modelId, + displayName: target.displayName, + }); + + return sourceMessages.map((message) => ({ + ...message, + parts: (message.parts || []).map((part) => { + const originalUrl = extractImagePartUrl(part); + if (!originalUrl) return part; + const inlineUrl = inlineMap.get(originalUrl); + return inlineUrl ? replaceImagePartUrl(part, inlineUrl) : part; + }) as typeof message.parts, + })) as ChatMessage[]; + }; + + const target = normalizeTarget(providerType); + let preparedMessages = messages; + + if (target.forceInlineImageUrls) { + preparedMessages = await inlineImageUrlsInMessages(preparedMessages, target); + } + + const normalizedProviderType = target.providerType; + + if ( + normalizedProviderType === "bfl-image" || + normalizedProviderType === "ruxa-image" || + normalizedProviderType === "fal-image" || + normalizedProviderType === "openai-image" || + normalizedProviderType === "stepfun-image" || + normalizedProviderType === "midjourney" || + normalizedProviderType === "runway" || + normalizedProviderType === "krea" + ) { + const convertedMessages = []; + + for (const message of preparedMessages) { + const content: any[] = []; + + for (const part of message.parts) { + const partAny = part as any; + + if (partAny.type === "text" && partAny.text) { + content.push({ + type: "text", + text: partAny.text, + }); + continue; + } + + const candidateUrl = + partAny.type === "file" + ? (partAny.url || partAny.data) + : partAny.type === "image" + ? (partAny.image || partAny.url) + : partAny.type === "image_url" + ? partAny.image_url?.url + : undefined; + + if (!candidateUrl || typeof candidateUrl !== "string") continue; + if (candidateUrl.startsWith("data:")) { + throw new Error("当前模型不支持 Base64 图片传输，请先上传图片后再编辑"); + } + if (!candidateUrl.startsWith("http")) continue; + + content.push({ + type: "image_url", + image_url: { + url: candidateUrl, + }, + }); + } + + convertedMessages.push({ + role: message.role, + content: content.length === 1 && content[0].type === "text" ? content[0].text : content, + }); + } + + return convertedMessages; + } + + // 所有 provider 都使用 URL 格式传输图片 + // gemini-image 和 huanke-gemini 内部会将 Base64 转换为 S3 URL + // 这里不再进行 Base64 转换，直接返回标准格式 + if (normalizedProviderType !== "gemini") { + return convertToModelMessages(preparedMessages); + } + + // 🆕 检测是否有标注图片 + const hasAnnotation = hasAnnotatedImage(messages); + if (hasAnnotation) { + console.log("[Message Convert] 🎨 检测到标注图片，将自动添加系统提示"); + } + + // 对于 Gemini，需要特殊处理图片 + const convertedMessages = []; + + for (const message of preparedMessages) { + const content: any[] = []; + + // 🆕 如果是用户消息且有标注图片，在文本前添加系统提示 + let hasAddedAnnotationPrompt = false; + + // 处理消息的各个部分 + for (const part of message.parts) { + if (part.type === "text" && part.text) { + let textContent = part.text; + + // 🆕 如果有标注图片且是用户消息，在用户文本前添加系统提示 + if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt) { + textContent = ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + textContent; + hasAddedAnnotationPrompt = true; + console.log("[Message Convert] 🎨 已注入标注提示词"); + } + + content.push({ + type: "text", + text: textContent, + }); + } else if (part.type === "file" && part.url) { + // 直接使用 URL 格式（不再转换为 Base64） + // gemini-image 和 huanke-gemini provider 内部会处理图片上传 + content.push({ + type: "image_url", + image_url: { + url: part.url, + }, + }); + console.log("[Message Convert] Image kept as URL format:", part.url.substring(0, 80) + "..."); + } + } + + // 🆕 如果有标注图片但消息中没有文本部分，添加一个纯提示文本 + if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt && content.length > 0) { + content.unshift({ + type: "text", + text: ANNOTATION_SYSTEM_PROMPT, + }); + console.log("[Message Convert] 🎨 已在消息开头注入标注提示词"); + } + + convertedMessages.push({ + role: message.role, + content: content.length === 1 && content[0].type === "text" + ? content[0].text + : content, + }); + } + + return convertedMessages; +} + +export function extractOriginalImageUrlsFromText(text: string): string[] { + return extractImageUrlsFromText(text); +} + +export function normalizeImageInputRef(rawValue: unknown, requestOrigin?: string): string | null { + if (typeof rawValue !== "string") return null; + const trimmed = rawValue.trim(); + if (!trimmed) return null; + + if (/^data:image\/[a-z0-9+.-]+;base64,/i.test(trimmed)) { + return trimmed.replace(/\s+/g, ""); + } + + if (/^https?:\/\//i.test(trimmed)) { + return trimmed; + } + + if (requestOrigin && trimmed.startsWith("/")) { + try { + return new URL(trimmed, requestOrigin).toString(); + } catch { + return null; + } + } + + return null; +} + +export function collectImageInputsFromMessageParts( + parts: Array> | undefined, + requestOrigin?: string, +): string[] { + if (!Array.isArray(parts) || parts.length === 0) return []; + + const result: string[] = []; + const pushInput = (rawValue: unknown) => { + const normalized = normalizeImageInputRef(rawValue, requestOrigin); + if (!normalized) return; + if (!result.includes(normalized)) { + result.push(normalized); + } + }; + + for (const part of parts) { + if (!part) continue; + + if (part.type === "file") { + pushInput(part.url); + pushInput(part.data); + continue; + } + + if (part.type === "image") { + pushInput(part.url); + pushInput(part.image); + pushInput(part.data); + continue; + } + + if (part.type === "image_url") { + pushInput(part.image_url?.url); + pushInput(part.url); + continue; + } + + if (part.type === "text" && typeof part.text === "string") { + const parsedUrls = extractOriginalImageUrlsFromText(part.text); + for (const parsedUrl of parsedUrls) { + pushInput(parsedUrl); + } + } + } + + return result; +} + +export function mimeTypeToExtension(mimeType: string) { + const normalized = mimeType.toLowerCase(); + if (normalized.includes("png")) return "png"; + if (normalized.includes("webp")) return "webp"; + if (normalized.includes("gif")) return "gif"; + if (normalized.includes("bmp")) return "bmp"; + return "jpg"; +} + +export function getExtensionFromUrl(url: string): string { + try { + const pathname = new URL(url).pathname; + const ext = pathname.split(".").pop()?.toLowerCase(); + if (!ext) return "jpg"; + if (["jpg", "jpeg", "png", "webp", "gif", "bmp"].includes(ext)) return ext; + } catch { + // ignore + } + return "jpg"; +} + +export async function uploadRemoteImageToStorage(imageUrl: string): Promise { + const response = await fetch(imageUrl, { + method: "GET", + signal: AbortSignal.timeout(30_000), + }); + + if (!response.ok) { + throw new Error(`Remote image fetch failed: ${response.status} ${response.statusText}`); + } + + const contentType = response.headers.get("content-type")?.toLowerCase() || ""; + if (contentType && !contentType.startsWith("image/")) { + throw new Error(`Remote url is not an image: ${contentType}`); + } + + const arrayBuffer = await response.arrayBuffer(); + if (!arrayBuffer || arrayBuffer.byteLength === 0) { + throw new Error("Remote image content is empty"); + } + + const ext = contentType ? mimeTypeToExtension(contentType) : getExtensionFromUrl(imageUrl); + const fileName = `remote-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`; + const uploaded = await uploadFile(fileName, arrayBuffer); + return uploaded.url; +} + +export async function isRemoteImageAccessible(imageUrl: string): Promise { + try { + const headResponse = await fetch(imageUrl, { + method: "HEAD", + signal: AbortSignal.timeout(10_000), + }); + + if (headResponse.ok) { + return true; + } + + if (headResponse.status !== 405 && headResponse.status !== 403) { + return false; + } + + const probeResponse = await fetch(imageUrl, { + method: "GET", + headers: { + Range: "bytes=0-0", + }, + signal: AbortSignal.timeout(10_000), + }); + return probeResponse.ok; + } catch { + return false; + } +} + +export async function ensureHostedImageUrl(imageUrl: string): Promise { + const normalizedUrl = imageUrl.trim(); + if (!normalizedUrl) { + throw new Error("Image URL is empty"); + } + + if (normalizedUrl.startsWith("data:image")) { + return uploadDataUrlImage(normalizedUrl); + } + + if (!/^https?:\/\//i.test(normalizedUrl)) { + throw new Error(`Unsupported image URL: ${normalizedUrl}`); + } + + if (/\/uploads\//i.test(normalizedUrl)) { + const accessible = await isRemoteImageAccessible(normalizedUrl); + if (accessible) { + return normalizedUrl; + } + } + + return uploadRemoteImageToStorage(normalizedUrl); +} + +export async function uploadDataUrlImage(dataUrl: string): Promise { + const match = dataUrl.match(/^data:([^;]+);base64,([\s\S]+)$/i); + if (!match) { + throw new Error("Invalid data URL image format"); + } + + const mimeType = match[1] || "image/jpeg"; + const base64Data = (match[2] || "").replace(/\s+/g, ""); + if (!base64Data) { + throw new Error("Empty base64 image content"); + } + + const buffer = Buffer.from(base64Data, "base64"); + const fileExt = mimeTypeToExtension(mimeType); + const fileName = `inline-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${fileExt}`; + const arrayBuffer = buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) as ArrayBuffer; + const uploaded = await uploadFile(fileName, arrayBuffer); + return uploaded.url; +} + +export async function normalizeAssistantImageMarkdown( + text: string, + imageSize = "2K" +): Promise { + if (!text) return text; + + const markdownDataUrlRegex = /!\[([^\]]*)\]\s*\(\s*(data:image\/[a-z0-9+.-]+;base64,[^)]+)\s*\)/gim; + const markdownMatches = [...text.matchAll(markdownDataUrlRegex)]; + let normalizedText = text; + const uploadedUrls: string[] = []; + + if (markdownMatches.length > 0) { + let rebuilt = ""; + let cursor = 0; + + for (const match of markdownMatches) { + const matchText = match[0]; + const dataUrl = match[2]; + const start = match.index ?? -1; + if (start < 0) continue; + + rebuilt += normalizedText.slice(cursor, start); + + try { + const uploadedUrl = await uploadDataUrlImage(dataUrl); + uploadedUrls.push(uploadedUrl); + rebuilt += `![thumb|||${uploadedUrl}|||${imageSize}](${uploadedUrl})`; + } catch (error) { + console.error("[Chat API] Failed to upload inline data URL image:", error); + rebuilt += matchText; + } + + cursor = start + matchText.length; + } + + rebuilt += normalizedText.slice(cursor); + normalizedText = rebuilt; + } + + if (normalizedText.includes("data:image")) { + const rawDataUrlRegex = /data:image\/[a-z0-9+.-]+;base64,[a-z0-9+/=\s]+/gim; + const rawMatches = normalizedText.match(rawDataUrlRegex) ?? []; + + for (const rawDataUrl of rawMatches) { + try { + const uploadedUrl = await uploadDataUrlImage(rawDataUrl); + uploadedUrls.push(uploadedUrl); + normalizedText = normalizedText.replace(rawDataUrl, uploadedUrl); + } catch (error) { + console.error("[Chat API] Failed to normalize raw data URL image:", error); + } + } + } + + const markdownHttpImageRegex = /!\[([^\]]*)\]\s*\(\s*(https?:\/\/[^)\s]+)\s*\)/gim; + const markdownHttpMatches = [...normalizedText.matchAll(markdownHttpImageRegex)]; + + if (markdownHttpMatches.length > 0) { + let rebuilt = ""; + let cursor = 0; + + for (const match of markdownHttpMatches) { + const matchText = match[0]; + const altText = match[1] || ""; + const thumbnailUrl = (match[2] || "").trim(); + const start = match.index ?? -1; + if (start < 0) continue; + + rebuilt += normalizedText.slice(cursor, start); + + try { + const { originalUrl, imageSize: parsedImageSize } = parseImageAltMetadata( + altText, + thumbnailUrl + ); + + let hostedThumbnailUrl = thumbnailUrl; + let hostedOriginalUrl = + originalUrl && originalUrl !== thumbnailUrl ? originalUrl : thumbnailUrl; + + try { + hostedThumbnailUrl = await ensureHostedImageUrl(thumbnailUrl); + } catch (error) { + console.error("[Chat API] Failed to normalize remote thumbnail URL:", error); + } + + if (originalUrl && originalUrl !== thumbnailUrl) { + try { + hostedOriginalUrl = await ensureHostedImageUrl(originalUrl); + } catch (error) { + console.error("[Chat API] Failed to normalize remote original URL:", error); + hostedOriginalUrl = hostedThumbnailUrl || originalUrl; + } + } else { + hostedOriginalUrl = hostedThumbnailUrl; + } + + const finalThumbnailUrl = hostedThumbnailUrl || thumbnailUrl; + const finalOriginalUrl = hostedOriginalUrl || finalThumbnailUrl; + + uploadedUrls.push(finalThumbnailUrl); + uploadedUrls.push(finalOriginalUrl); + rebuilt += `![thumb|||${finalOriginalUrl}|||${parsedImageSize || imageSize}](${finalThumbnailUrl})`; + } catch (error) { + console.error("[Chat API] Failed to parse remote image markdown metadata:", error); + rebuilt += matchText; + } + + cursor = start + matchText.length; + } + + rebuilt += normalizedText.slice(cursor); + normalizedText = rebuilt; + } + + const hasMarkdownImage = /!\[[^\]]*\]\((https?:\/\/[^)]+)\)/i.test(normalizedText); + if (!hasMarkdownImage && uploadedUrls.length > 0) { + const uniqueUrls = [...new Set(uploadedUrls)]; + console.log( + `[Chat API] Converted ${uniqueUrls.length} image link(s) to hosted link(s)` + ); + return uniqueUrls + .map((url) => `![thumb|||${url}|||${imageSize}](${url})`) + .join("\n"); + } + + return normalizedText; +} diff --git a/app/(chat)/api/chat/_lib/message-preparation.ts b/app/(chat)/api/chat/_lib/message-preparation.ts new file mode 100644 index 000000000..1fb03d059 --- /dev/null +++ b/app/(chat)/api/chat/_lib/message-preparation.ts @@ -0,0 +1,169 @@ +import { + analyzeOptimizedPromptMerge, + buildOptimizedPromptPreview, +} from "@/lib/ai/runtime/optimized-prompt"; +import type { ChatMessage } from "@/lib/types"; + +/** + * 将图片 URL 转换为 base64 格式 + * 用于将 S3 URL 的图片转换为 base64 发送给 Gemini API + */ +export async function convertImageUrlToBase64(url: string): Promise { + try { + console.log("[Image Convert] Fetching image from:", url); + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to fetch image: ${response.statusText}`); + } + + const arrayBuffer = await response.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + const base64 = buffer.toString("base64"); + + // 获取 MIME 类型 + const contentType = response.headers.get("content-type") || "image/jpeg"; + const dataUrl = `data:${contentType};base64,${base64}`; + + console.log("[Image Convert] Converted to base64, size:", base64.length); + return dataUrl; + } catch (error) { + console.error("[Image Convert] Failed:", error); + throw error; + } +} + +/** + * 检测消息中是否包含用户标注过的图片（文件名以 annotated- 开头） + */ +export function hasAnnotatedImage(messages: ChatMessage[]): boolean { + for (const message of messages) { + for (const part of message.parts) { + if (part.type === "file" && "name" in part && typeof part.name === "string") { + if (part.name.startsWith("annotated-")) { + return true; + } + } + } + } + return false; +} + +/** + * 标注图片的系统提示词（用户不可见，后台自动添加） + */ +export const ANNOTATION_SYSTEM_PROMPT = + "【重要】用户提供的图片上有标注线条（如箭头、圆圈、画笔痕迹或文字说明），这些标注是用户用来指示需要修改的区域或方向。请仔细理解标注的含义，按照标注指示进行编辑，但在生成的新图片中不要保留这些标注线条，输出干净的编辑结果。"; + +export function applyHiddenPromptToMessages( + messages: ChatMessage[], + hiddenPrompt?: string, +): ChatMessage[] { + if (!hiddenPrompt || !hiddenPrompt.trim()) return messages; + const cloned = messages.map((msg) => ({ + ...msg, + parts: msg.parts.map((part) => ({ ...part })), + })); + for (let i = cloned.length - 1; i >= 0; i--) { + const msg = cloned[i]; + if (msg.role !== "user") continue; + const textPart = msg.parts.find((part) => part.type === "text") as + | { type: "text"; text: string } + | undefined; + if (textPart) { + textPart.text = `${hiddenPrompt}\n\n${textPart.text}`.trim(); + } else { + msg.parts.unshift({ type: "text", text: hiddenPrompt }); + } + break; + } + return cloned; +} + +export function applyOptimizedPromptToLatestUserMessage( + messages: ChatMessage[], + optimizedPrompt?: string, + hiddenPromptPrefix = "", +): ChatMessage[] { + const prompt = optimizedPrompt?.trim(); + if (!prompt) return messages; + + const cloned = messages.map((msg) => ({ + ...msg, + parts: msg.parts.map((part) => ({ ...part })), + })); + + for (let i = cloned.length - 1; i >= 0; i -= 1) { + const msg = cloned[i]; + if (msg.role !== "user") continue; + + const textPart = msg.parts.find( + (part): part is { type: "text"; text: string } => part.type === "text", + ); + const originalText = textPart?.text?.trim() || ""; + const { normalizedOriginal, hasOriginalDemand, hasOriginalInOptimized } = + analyzeOptimizedPromptMerge(originalText, prompt, hiddenPromptPrefix); + const finalPrompt = + hasOriginalDemand && !hasOriginalInOptimized + ? [ + hiddenPromptPrefix.trim(), + `【核心目标（必须优先满足）】${normalizedOriginal}`, + `【执行优化细节】${prompt}`, + ] + .filter(Boolean) + .join("\n\n") + .trim() + : `${hiddenPromptPrefix}${prompt}`.trim(); + console.log("[Prompt Merge] Final image prompt prepared:", { + original: normalizedOriginal.substring(0, 120), + optimized: prompt.substring(0, 120), + merged: finalPrompt.substring(0, 180), + forcePreserveOriginal: hasOriginalDemand && !hasOriginalInOptimized, + }); + + if (textPart) { + textPart.text = finalPrompt; + } else { + msg.parts.unshift({ type: "text", text: finalPrompt }); + } + break; + } + + return cloned; +} + +export function injectOptimizedPromptSection(params: { + message?: string; + originalText: string; + optimizedPrompt?: string; + hiddenPromptPrefix?: string; +}): string | undefined { + const { message, originalText, optimizedPrompt, hiddenPromptPrefix } = params; + if (!message?.trim()) return message; + + const preview = buildOptimizedPromptPreview({ + originalText, + optimizedPrompt, + hiddenPromptPrefix, + }); + if (!preview) return message; + + const optimizedPromptSection = `【优化后提示词】\n${preview}`; + const optimizedSectionRegex = + /(?:###\s*优化后提示词|【\s*优化后提示词\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/; + if (optimizedSectionRegex.test(message)) { + return message.replace(optimizedSectionRegex, optimizedPromptSection); + } + + const executionSectionRegex = + /(?:###\s*执行计划|【\s*执行计划\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/; + if (executionSectionRegex.test(message)) { + return message.replace(executionSectionRegex, optimizedPromptSection); + } + + const outputSectionRegex = /(?:###\s*输出设置|【\s*输出设置\s*】)/; + if (outputSectionRegex.test(message)) { + return message.replace(outputSectionRegex, `${optimizedPromptSection}\n$&`); + } + + return `${message.trim()}\n\n${optimizedPromptSection}`; +} diff --git a/app/(chat)/api/chat/_lib/model-resolver.ts b/app/(chat)/api/chat/_lib/model-resolver.ts new file mode 100644 index 000000000..39160bac5 --- /dev/null +++ b/app/(chat)/api/chat/_lib/model-resolver.ts @@ -0,0 +1,43 @@ +export function resolveModelDisplayNameByModelId( + models: Array<{ model: any; provider?: any }> | undefined, + modelId: string | undefined, +): string { + if (!modelId || !models?.length) return ""; + const match = models.find( + (item) => item.model.modelId === modelId || item.model.id === modelId, + ); + return String(match?.model?.displayName || ""); +} + +export function resolveModelEntryByRuntimeModelId( + models: Array<{ model: any; provider?: any }> | undefined, + modelId: string | undefined, + fallbackRef?: string, +) { + const candidates = [modelId, fallbackRef] + .map((item) => String(item || "").trim()) + .filter(Boolean); + if (!candidates.length || !models?.length) return null; + + return ( + models.find((item) => + candidates.some( + (candidate) => item.model.modelId === candidate || item.model.id === candidate, + ), + ) || null + ); +} + +export function resolveModelRefByRuntimeModelId( + models: Array<{ model: any; provider?: any }> | undefined, + modelId: string | undefined, + fallbackRef?: string, +): string { + const normalizedModelId = String(modelId || "").trim(); + if (!normalizedModelId) return fallbackRef || ""; + + const match = models?.find( + (item) => item.model.modelId === normalizedModelId || item.model.id === normalizedModelId, + ); + return String(match?.model?.id || fallbackRef || normalizedModelId || ""); +} diff --git a/app/(chat)/api/chat/_lib/video-models.ts b/app/(chat)/api/chat/_lib/video-models.ts new file mode 100644 index 000000000..dce8055e3 --- /dev/null +++ b/app/(chat)/api/chat/_lib/video-models.ts @@ -0,0 +1,275 @@ +import { after } from "next/server"; +import { + getAiModelById, + getEnabledAiModelsWithProvider, + updateMessageById, +} from "@/lib/db/queries"; +import { + isImageGenerationModelDescriptor, + isVideoModelDescriptor, +} from "@/lib/ai/model-classification"; +import type { StoryboardModelCandidate } from "@/lib/video-agent/types"; + +export type VideoModeType = + | "text-to-video" + | "first-frame" + | "first-last-frame" + | "reference-images"; + +export function isVideoProviderType( + providerType?: string | null, + model?: { + modelId?: string | null; + capabilities?: unknown; + supportsImageGen?: boolean | null; + }, +): boolean { + return isVideoModelDescriptor({ + providerType, + modelId: model?.modelId, + capabilities: model?.capabilities, + supportsImageGen: model?.supportsImageGen, + }); +} + +export function isImageGenerationModelType( + providerType?: string | null, + model?: { + modelId?: string | null; + capabilities?: unknown; + supportsImageGen?: boolean | null; + }, +): boolean { + return isImageGenerationModelDescriptor({ + providerType, + modelId: model?.modelId, + capabilities: model?.capabilities, + supportsImageGen: model?.supportsImageGen, + }); +} + +export function inferSupportedVideoModesByModelId(modelId: string): VideoModeType[] { + const normalized = modelId.toLowerCase(); + + if ( + normalized === "gen4.5" || + normalized === "gen4_turbo" || + normalized === "gen4_aleph" || + normalized === "gen3a_turbo" || + normalized === "veo3" || + normalized === "veo3.1" || + normalized === "veo3.1_fast" + ) { + return ["text-to-video", "first-frame"]; + } + + if (normalized.includes("lite-t2v")) { + return ["text-to-video"]; + } + if (normalized.includes("lite-i2v")) { + return ["first-frame", "first-last-frame", "reference-images"]; + } + if (normalized.includes("pro-fast")) { + return ["text-to-video", "first-frame"]; + } + if (normalized.includes("-pro-") || normalized.endsWith("-pro")) { + return ["text-to-video", "first-frame", "first-last-frame"]; + } + + return ["text-to-video", "first-frame", "first-last-frame", "reference-images"]; +} + +export function getSupportedVideoModesForModel(model: any): VideoModeType[] { + const capabilityModes = (((model?.capabilities as any)?.supportedModes ?? []) as string[]).filter( + (mode): mode is VideoModeType => + mode === "text-to-video" || + mode === "first-frame" || + mode === "first-last-frame" || + mode === "reference-images", + ); + + if (capabilityModes.length > 0) { + return capabilityModes; + } + + const modelId = String(model?.modelId || ""); + return inferSupportedVideoModesByModelId(modelId); +} + +export function scoreVideoSubModelForMode(modelId: string, mode: VideoModeType): number { + const normalized = modelId.toLowerCase(); + const isProFast = normalized.includes("pro-fast"); + const isLiteT2V = normalized.includes("lite-t2v"); + const isLiteI2V = normalized.includes("lite-i2v"); + const isPro = (normalized.includes("-pro-") || normalized.endsWith("-pro")) && !isProFast; + const isV15Pro = normalized.includes("seedance-1-5-pro"); + + switch (mode) { + case "first-last-frame": + if (isV15Pro) return 150; + if (isPro) return 120; + if (isLiteI2V) return 90; + if (isProFast) return -1000; + if (isLiteT2V) return -1000; + return 10; + case "first-frame": + if (isV15Pro) return 140; + if (isProFast) return 130; + if (isPro) return 120; + if (isLiteI2V) return 100; + if (isLiteT2V) return 20; + return 10; + case "reference-images": + if (isV15Pro) return 150; + if (isLiteI2V) return 140; + if (isPro) return 110; + if (isProFast || isLiteT2V) return -1000; + return 10; + case "text-to-video": + default: + if (isV15Pro) return 140; + if (isLiteT2V) return 130; + if (isProFast) return 120; + if (isPro) return 100; + if (isLiteI2V) return 80; + return 10; + } +} + +export function selectBestVideoSubModelForMode( + subModels: Array<{ model: any }>, + mode: VideoModeType, +): { model: any } | null { + const compatible = subModels.filter(({ model }) => { + const supportedModes = getSupportedVideoModesForModel(model); + return supportedModes.includes(mode); + }); + + const candidates = compatible.length > 0 ? compatible : subModels; + if (candidates.length === 0) return null; + + return [...candidates].sort((a, b) => { + const scoreA = scoreVideoSubModelForMode(String(a.model?.modelId || ""), mode); + const scoreB = scoreVideoSubModelForMode(String(b.model?.modelId || ""), mode); + return scoreB - scoreA; + })[0]; +} + +export async function resolveStoryboardModelCandidates( + selectedModelRef: string, +): Promise { + const selectedModel = await getAiModelById({ id: selectedModelRef }).catch(() => undefined); + if (!selectedModel) return []; + + const configuredFallbackIds = [ + selectedModel.fallbackModelId1, + selectedModel.fallbackModelId2, + ].filter((item): item is string => Boolean(String(item || "").trim())); + + const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []); + const sameProvider = allModels + .filter( + ({ model, provider }: any) => + provider?.id === selectedModel.providerId && model?.id !== selectedModel.id && model?.isEnabled, + ) + .map(({ model }: any) => String(model?.id || "").trim()) + .filter(Boolean); + + const candidateIds = Array.from(new Set([selectedModel.id, ...configuredFallbackIds, ...sameProvider])); + const byModelRef = new Map(); + allModels + .map(({ model }: any) => model) + .filter(Boolean) + .forEach((model: any) => { + byModelRef.set(String(model.id), model); + if (model.modelId) { + byModelRef.set(String(model.modelId), model); + } + }); + + return candidateIds + .map((id) => byModelRef.get(id)) + .filter(Boolean) + .map((model: any) => ({ + id: String(model.id), + modelId: String(model.modelId || "").trim(), + displayName: String(model.displayName || model.modelId || "").trim(), + })) + .filter((item) => item.modelId); +} + +export function extractVideoUrlFromText(text: string): string | null { + const normalizedText = String(text || ""); + if (!normalizedText) return null; + + const htmlMatch = normalizedText.match(/]+src="([^"]+)"/i); + if (htmlMatch?.[1]) return htmlMatch[1]; + + const markdownMatch = normalizedText.match( + /\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i, + ); + if (markdownMatch?.[1]) return markdownMatch[1]; + + const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i); + return plainMatch?.[1] || null; +} + +export function isAlreadyArchivedVideoUrl(url: string): boolean { + return /\/uploads\//i.test(url); +} + +export function scheduleVideoArchive(params: { + chatId: string; + messageId: string; + text: string; + temporaryVideoUrl: string; + taskId?: string; + metadata?: Record | null; +}) { + const { chatId, messageId, temporaryVideoUrl, taskId, metadata } = params; + if (!temporaryVideoUrl || isAlreadyArchivedVideoUrl(temporaryVideoUrl)) { + return; + } + + const delayHoursRaw = Number(process.env.VIDEO_ARCHIVE_DELAY_HOURS || 48); + const delayHours = Number.isFinite(delayHoursRaw) + ? Math.max(1, Math.min(24 * 30, Math.round(delayHoursRaw))) + : 48; + const archiveAfterAtMs = Date.now() + delayHours * 60 * 60 * 1000; + + after(async () => { + try { + const nextMetadata = { + ...(metadata || {}), + temporaryVideoUrl, + storageStatus: "temporary", + archiveMode: "delayed_job", + archiveAfterAt: archiveAfterAtMs, + archiveDelayHours: delayHours, + archiveTaskId: taskId || messageId, + updatedAt: Date.now(), + }; + + await updateMessageById({ + messageId, + chatId, + videoStorageStatus: "temporary", + temporaryVideoUrl, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: nextMetadata as any, + }); + + console.log("[Chat API] 🕒 已登记延迟归档视频:", { + messageId, + archiveAfterAt: new Date(archiveAfterAtMs).toISOString(), + }); + } catch (error) { + console.error("[Chat API] ❌ 登记视频延迟归档失败（保留临时地址）:", { + messageId, + temporaryVideoUrl, + error, + }); + } + }); +} diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 29573cb86..4a2f88667 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -20,7 +20,6 @@ import { getUsage } from "tokenlens/helpers"; import { auth, type UserType } from "@/app/(auth)/auth"; import type { VisibilityType } from "@/components/visibility-selector"; import { entitlementsByUserType } from "@/lib/ai/entitlements"; -import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import type { ChatModel } from "@/lib/ai/models"; import { type RequestHints, systemPrompt } from "@/lib/ai/prompts"; import { myProvider } from "@/lib/ai/providers"; @@ -29,9 +28,7 @@ import { submitFalImageTask, } from "@/lib/ai/providers/fal-image"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; -import { resolveDetailAgentLocalMoonshotModel } from "@/lib/ai/detail-agent-model-runtime"; import { shouldForceInlineVisionInputs } from "@/lib/ai/model-compat"; -import { inlineVisionImages } from "@/lib/ai/vision-inline"; import { createDocument } from "@/lib/ai/tools/create-document"; import { getWeather } from "@/lib/ai/tools/get-weather"; import { requestSuggestions } from "@/lib/ai/tools/request-suggestions"; @@ -89,11 +86,6 @@ import { isGridConfirmationReply, type PendingGridConfirmation, } from "@/lib/ai/conversation-action-resolver"; -import { - isImageGenerationModelDescriptor, - isVideoModelDescriptor, -} from "@/lib/ai/model-classification"; -import { getModelGenerationCreditCost } from "@/lib/ai/image-model-pricing"; import { bootstrapConversationStateProtocol, composeConversationStateProtocol, @@ -127,7 +119,6 @@ import { getRequestErrorMessage, isRateLimitError, } from "@/lib/request-error"; -import { extractImageUrlsFromText } from "@/lib/image-url"; import { resolveClarificationBypass, resolveClarificationMessage, @@ -137,10 +128,7 @@ import { buildAgentToolVideoMarkdown, prependImageModelBadge, } from "@/lib/ai/runtime/agent-media-markdown"; -import { - parseImageAltMetadata, - saveGeneratedImages, -} from "@/lib/ai/runtime/generated-image-persistence"; +import { saveGeneratedImages } from "@/lib/ai/runtime/generated-image-persistence"; import { recordAiModelHealthFailure, recordAiModelHealthSuccess, @@ -187,10 +175,6 @@ import { safePersistDirectAgentVideoCompletedTurn, safePersistDirectAgentVideoFailedTurn, } from "@/lib/ai/runtime/direct-agent-working-state"; -import { - analyzeOptimizedPromptMerge, - buildOptimizedPromptPreview, -} from "@/lib/ai/runtime/optimized-prompt"; import { getDirectAgentToolErrorMessage, getDirectAgentToolLabel, @@ -216,7 +200,6 @@ import { } from "@/lib/ai/runtime/provider-health"; import { arbitrateAgentIntent } from "@/lib/ai/runtime/intent-arbiter"; import { buildAgentRoleIdentityInstruction } from "@/lib/ai/runtime/agent-role-profile"; -import { uploadFile } from "@/lib/storage/service"; import { getVideoStoryboardConfig } from "@/lib/video-agent/config"; import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline"; import { @@ -255,9 +238,76 @@ import { withOperationTimeout, } from "./_lib/errors"; import { - bufferToArrayBuffer, - normalizeAssistantPartsForPersistence, -} from "./_lib/persistence"; + applyResolvedTargetImagesToLastUserMessage, + buildAutoModeFallbackMessage, + buildPromptOnlyOptimizationSeed, + buildPromptOnlyReply, + collectImageInputsFromMessageParts, + convertMessagesWithImages, + ensureImageAnalysisSummarySection, + ensureStructuredAgentFriendlyMessage, + extractMessageImageAssetIds, + extractMessageImageUrls, + injectHistoryImagesToLastUserMessage, + normalizeAssistantImageMarkdown, + normalizeResolvedTargetImageUrls, + shouldEnrichImageAnalysisText, + summarizeTargetImageUrl, +} from "./_lib/image-media"; +import { + buildImageFallbackCandidates, + generateImageWithFailoverCandidate, + getImageCrossProviderCandidates, + IMAGE_FAILOVER_PROVIDER_ALLOWLIST, + normalizeFailoverErrorMessage, + setImageFallbackModelCooldown, + shouldCooldownImageFallbackModel, +} from "./_lib/image-failover"; +import { + buildAgentSceneWorkflowPrompt, + buildAgentSlotPrompt, + buildOptimizedAgentSceneWorkflowPrompt, + dedupeAgentSlotPromptAgainstWorkflow, + extractCurrentMessageImageParts, + getSceneUploadedImageBreakdown, + normalizeAgentSupplementPrompt, + resolveScenePromptOptimizationImageUrls, + type AgentPresetMessageImagePart, +} from "./_lib/agent-scene-workflow"; +import { + analyzeReplacementExecutionPlan, + buildAgentReplacementPrompt, + normalizeAgentModeDetailReplacementStrategy, + normalizeAgentModeReferences, + normalizeAgentModeSourceImages, + normalizeReplacementResultImage, + resolveDirectReplacementModelRef, + resolveImagePartsByReferenceExclusion, + resolveReplacementReferenceGeometry, + resolveReplacementScenePlanningRuntime, +} from "./_lib/detail-replacement"; +import { + normalizeAgentReportForDisplay, + sanitizeAgentOutputSettingsSection, +} from "./_lib/agent-output"; +import { + consumeGeneratedVideoPoints, + getModelCreditCost, +} from "./_lib/billing"; +import { + ANNOTATION_SYSTEM_PROMPT, + applyHiddenPromptToMessages, + applyOptimizedPromptToLatestUserMessage, + convertImageUrlToBase64, + hasAnnotatedImage, + injectOptimizedPromptSection, +} from "./_lib/message-preparation"; +import { + resolveModelDisplayNameByModelId, + resolveModelEntryByRuntimeModelId, + resolveModelRefByRuntimeModelId, +} from "./_lib/model-resolver"; +import { normalizeAssistantPartsForPersistence } from "./_lib/persistence"; import { buildAgentTargetLockPayload, createSseResponse, @@ -272,433 +322,22 @@ import { writeImageGenerationStatusChunk, writeProtocolDebugChunk, } from "./_lib/stream"; +import { + extractVideoUrlFromText, + getSupportedVideoModesForModel, + isImageGenerationModelType, + isVideoProviderType, + resolveStoryboardModelCandidates, + scheduleVideoArchive, + selectBestVideoSubModelForMode, + type VideoModeType, +} from "./_lib/video-models"; const AGENT_REPLACEMENT_PARALLEL_WORKER_COUNT = Math.max( 1, Math.min(8, Number(process.env.AGENT_REPLACEMENT_PARALLEL_WORKER_COUNT || 6) || 6), ); -/** - * 将图片 URL 转换为 base64 格式 - * 用于将 S3 URL 的图片转换为 base64 发送给 Gemini API - */ -async function convertImageUrlToBase64(url: string): Promise { - try { - console.log("[Image Convert] Fetching image from:", url); - const response = await fetch(url); - if (!response.ok) { - throw new Error(`Failed to fetch image: ${response.statusText}`); - } - - const arrayBuffer = await response.arrayBuffer(); - const buffer = Buffer.from(arrayBuffer); - const base64 = buffer.toString("base64"); - - // 获取 MIME 类型 - const contentType = response.headers.get("content-type") || "image/jpeg"; - const dataUrl = `data:${contentType};base64,${base64}`; - - console.log("[Image Convert] Converted to base64, size:", base64.length); - return dataUrl; - } catch (error) { - console.error("[Image Convert] Failed:", error); - throw error; - } -} - -/** - * 检测消息中是否包含用户标注过的图片（文件名以 annotated- 开头） - */ -function hasAnnotatedImage(messages: ChatMessage[]): boolean { - for (const message of messages) { - for (const part of message.parts) { - if (part.type === "file" && "name" in part && typeof part.name === "string") { - if (part.name.startsWith("annotated-")) { - return true; - } - } - } - } - return false; -} - -/** - * 标注图片的系统提示词（用户不可见，后台自动添加） - */ -const ANNOTATION_SYSTEM_PROMPT = "【重要】用户提供的图片上有标注线条（如箭头、圆圈、画笔痕迹或文字说明），这些标注是用户用来指示需要修改的区域或方向。请仔细理解标注的含义，按照标注指示进行编辑，但在生成的新图片中不要保留这些标注线条，输出干净的编辑结果。"; - -function applyHiddenPromptToMessages(messages: ChatMessage[], hiddenPrompt?: string): ChatMessage[] { - if (!hiddenPrompt || !hiddenPrompt.trim()) return messages; - const cloned = messages.map((msg) => ({ - ...msg, - parts: msg.parts.map((part) => ({ ...part })), - })); - for (let i = cloned.length - 1; i >= 0; i--) { - const msg = cloned[i]; - if (msg.role !== "user") continue; - const textPart = msg.parts.find((part) => part.type === "text") as { type: "text"; text: string } | undefined; - if (textPart) { - textPart.text = `${hiddenPrompt}\n\n${textPart.text}`.trim(); - } else { - msg.parts.unshift({ type: "text", text: hiddenPrompt }); - } - break; - } - return cloned; -} - -function applyOptimizedPromptToLatestUserMessage( - messages: ChatMessage[], - optimizedPrompt?: string, - hiddenPromptPrefix = "", -): ChatMessage[] { - const prompt = optimizedPrompt?.trim(); - if (!prompt) return messages; - - const cloned = messages.map((msg) => ({ - ...msg, - parts: msg.parts.map((part) => ({ ...part })), - })); - - for (let i = cloned.length - 1; i >= 0; i -= 1) { - const msg = cloned[i]; - if (msg.role !== "user") continue; - - const textPart = msg.parts.find( - (part): part is { type: "text"; text: string } => part.type === "text", - ); - const originalText = textPart?.text?.trim() || ""; - const { normalizedOriginal, hasOriginalDemand, hasOriginalInOptimized } = - analyzeOptimizedPromptMerge(originalText, prompt, hiddenPromptPrefix); - const finalPrompt = hasOriginalDemand && !hasOriginalInOptimized - ? [ - hiddenPromptPrefix.trim(), - `【核心目标（必须优先满足）】${normalizedOriginal}`, - `【执行优化细节】${prompt}`, - ] - .filter(Boolean) - .join("\n\n") - .trim() - : `${hiddenPromptPrefix}${prompt}`.trim(); - console.log("[Prompt Merge] Final image prompt prepared:", { - original: normalizedOriginal.substring(0, 120), - optimized: prompt.substring(0, 120), - merged: finalPrompt.substring(0, 180), - forcePreserveOriginal: hasOriginalDemand && !hasOriginalInOptimized, - }); - - if (textPart) { - textPart.text = finalPrompt; - } else { - msg.parts.unshift({ type: "text", text: finalPrompt }); - } - break; - } - - return cloned; -} - -function injectOptimizedPromptSection(params: { - message?: string; - originalText: string; - optimizedPrompt?: string; - hiddenPromptPrefix?: string; -}): string | undefined { - const { message, originalText, optimizedPrompt, hiddenPromptPrefix } = params; - if (!message?.trim()) return message; - - const preview = buildOptimizedPromptPreview({ - originalText, - optimizedPrompt, - hiddenPromptPrefix, - }); - if (!preview) return message; - - const optimizedPromptSection = `【优化后提示词】\n${preview}`; - const optimizedSectionRegex = - /(?:###\s*优化后提示词|【\s*优化后提示词\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/; - if (optimizedSectionRegex.test(message)) { - return message.replace(optimizedSectionRegex, optimizedPromptSection); - } - - const executionSectionRegex = - /(?:###\s*执行计划|【\s*执行计划\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/; - if (executionSectionRegex.test(message)) { - return message.replace(executionSectionRegex, optimizedPromptSection); - } - - const outputSectionRegex = /(?:###\s*输出设置|【\s*输出设置\s*】)/; - if (outputSectionRegex.test(message)) { - return message.replace(outputSectionRegex, `${optimizedPromptSection}\n$&`); - } - - return `${message.trim()}\n\n${optimizedPromptSection}`; -} - -type VideoModeType = "text-to-video" | "first-frame" | "first-last-frame" | "reference-images"; - -function isVideoProviderType( - providerType?: string | null, - model?: { - modelId?: string | null; - capabilities?: unknown; - supportsImageGen?: boolean | null; - } -): boolean { - return isVideoModelDescriptor({ - providerType, - modelId: model?.modelId, - capabilities: model?.capabilities, - supportsImageGen: model?.supportsImageGen, - }); -} - -function isImageGenerationModelType( - providerType?: string | null, - model?: { - modelId?: string | null; - capabilities?: unknown; - supportsImageGen?: boolean | null; - } -): boolean { - return isImageGenerationModelDescriptor({ - providerType, - modelId: model?.modelId, - capabilities: model?.capabilities, - supportsImageGen: model?.supportsImageGen, - }); -} - -function inferSupportedVideoModesByModelId(modelId: string): VideoModeType[] { - const normalized = modelId.toLowerCase(); - - if ( - normalized === "gen4.5" || - normalized === "gen4_turbo" || - normalized === "gen4_aleph" || - normalized === "gen3a_turbo" || - normalized === "veo3" || - normalized === "veo3.1" || - normalized === "veo3.1_fast" - ) { - return ["text-to-video", "first-frame"]; - } - - if (normalized.includes("lite-t2v")) { - return ["text-to-video"]; - } - if (normalized.includes("lite-i2v")) { - return ["first-frame", "first-last-frame", "reference-images"]; - } - if (normalized.includes("pro-fast")) { - return ["text-to-video", "first-frame"]; - } - if (normalized.includes("-pro-") || normalized.endsWith("-pro")) { - return ["text-to-video", "first-frame", "first-last-frame"]; - } - - return ["text-to-video", "first-frame", "first-last-frame", "reference-images"]; -} - -function getSupportedVideoModesForModel(model: any): VideoModeType[] { - const capabilityModes = (((model?.capabilities as any)?.supportedModes ?? []) as string[]) - .filter((mode): mode is VideoModeType => - mode === "text-to-video" || - mode === "first-frame" || - mode === "first-last-frame" || - mode === "reference-images" - ); - - if (capabilityModes.length > 0) { - return capabilityModes; - } - - const modelId = String(model?.modelId || ""); - return inferSupportedVideoModesByModelId(modelId); -} - -function scoreVideoSubModelForMode(modelId: string, mode: VideoModeType): number { - const normalized = modelId.toLowerCase(); - const isProFast = normalized.includes("pro-fast"); - const isLiteT2V = normalized.includes("lite-t2v"); - const isLiteI2V = normalized.includes("lite-i2v"); - const isPro = (normalized.includes("-pro-") || normalized.endsWith("-pro")) && !isProFast; - const isV15Pro = normalized.includes("seedance-1-5-pro"); - - switch (mode) { - case "first-last-frame": - if (isV15Pro) return 150; - if (isPro) return 120; - if (isLiteI2V) return 90; - if (isProFast) return -1000; - if (isLiteT2V) return -1000; - return 10; - case "first-frame": - if (isV15Pro) return 140; - if (isProFast) return 130; - if (isPro) return 120; - if (isLiteI2V) return 100; - if (isLiteT2V) return 20; - return 10; - case "reference-images": - if (isV15Pro) return 150; - if (isLiteI2V) return 140; - if (isPro) return 110; - if (isProFast || isLiteT2V) return -1000; - return 10; - case "text-to-video": - default: - if (isV15Pro) return 140; - if (isLiteT2V) return 130; - if (isProFast) return 120; - if (isPro) return 100; - if (isLiteI2V) return 80; - return 10; - } -} - -function selectBestVideoSubModelForMode( - subModels: Array<{ model: any }>, - mode: VideoModeType, -): { model: any } | null { - const compatible = subModels.filter(({ model }) => { - const supportedModes = getSupportedVideoModesForModel(model); - return supportedModes.includes(mode); - }); - - const candidates = compatible.length > 0 ? compatible : subModels; - if (candidates.length === 0) return null; - - return [...candidates].sort((a, b) => { - const scoreA = scoreVideoSubModelForMode(String(a.model?.modelId || ""), mode); - const scoreB = scoreVideoSubModelForMode(String(b.model?.modelId || ""), mode); - return scoreB - scoreA; - })[0]; -} - -async function resolveStoryboardModelCandidates( - selectedModelRef: string -): Promise { - const selectedModel = await getAiModelById({ id: selectedModelRef }).catch(() => undefined); - if (!selectedModel) return []; - - const configuredFallbackIds = [ - selectedModel.fallbackModelId1, - selectedModel.fallbackModelId2, - ].filter((item): item is string => Boolean(String(item || "").trim())); - - const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []); - const sameProvider = allModels - .filter( - ({ model, provider }: any) => - provider?.id === selectedModel.providerId && - model?.id !== selectedModel.id && - model?.isEnabled - ) - .map(({ model }: any) => String(model?.id || "").trim()) - .filter(Boolean); - - const candidateIds = Array.from( - new Set([selectedModel.id, ...configuredFallbackIds, ...sameProvider]) - ); - const byModelRef = new Map(); - allModels - .map(({ model }: any) => model) - .filter(Boolean) - .forEach((model: any) => { - byModelRef.set(String(model.id), model); - if (model.modelId) { - byModelRef.set(String(model.modelId), model); - } - }); - - return candidateIds - .map((id) => byModelRef.get(id)) - .filter(Boolean) - .map((model: any) => ({ - id: String(model.id), - modelId: String(model.modelId || "").trim(), - displayName: String(model.displayName || model.modelId || "").trim(), - })) - .filter((item) => item.modelId); -} - -function extractVideoUrlFromText(text: string): string | null { - const normalizedText = String(text || ""); - if (!normalizedText) return null; - - const htmlMatch = normalizedText.match(/]+src="([^"]+)"/i); - if (htmlMatch?.[1]) return htmlMatch[1]; - - const markdownMatch = normalizedText.match( - /\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i, - ); - if (markdownMatch?.[1]) return markdownMatch[1]; - - const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i); - return plainMatch?.[1] || null; -} - -function isAlreadyArchivedVideoUrl(url: string): boolean { - return /\/uploads\//i.test(url); -} - -function scheduleVideoArchive(params: { - chatId: string; - messageId: string; - text: string; - temporaryVideoUrl: string; - taskId?: string; - metadata?: Record | null; -}) { - const { chatId, messageId, temporaryVideoUrl, taskId, metadata } = params; - if (!temporaryVideoUrl || isAlreadyArchivedVideoUrl(temporaryVideoUrl)) { - return; - } - - const delayHoursRaw = Number(process.env.VIDEO_ARCHIVE_DELAY_HOURS || 48); - const delayHours = Number.isFinite(delayHoursRaw) - ? Math.max(1, Math.min(24 * 30, Math.round(delayHoursRaw))) - : 48; - const archiveAfterAtMs = Date.now() + delayHours * 60 * 60 * 1000; - - after(async () => { - try { - const nextMetadata = { - ...(metadata || {}), - temporaryVideoUrl, - storageStatus: "temporary", - archiveMode: "delayed_job", - archiveAfterAt: archiveAfterAtMs, - archiveDelayHours: delayHours, - archiveTaskId: taskId || messageId, - updatedAt: Date.now(), - }; - - await updateMessageById({ - messageId, - chatId, - videoStorageStatus: "temporary", - temporaryVideoUrl, - permanentVideoUrl: null, - archivedAt: null, - videoMetadata: nextMetadata as any, - }); - - console.log("[Chat API] 🕒 已登记延迟归档视频:", { - messageId, - archiveAfterAt: new Date(archiveAfterAtMs).toISOString(), - }); - } catch (error) { - console.error("[Chat API] ❌ 登记视频延迟归档失败（保留临时地址）:", { - messageId, - temporaryVideoUrl, - error, - }); - } - }); -} - async function replayAssistantMessagesForDuplicateRequest({ chatId, userMessageId, @@ -799,418 +438,6 @@ type AgentVideoPlan = { recommendedVideoParams?: AgentRecommendedVideoParams; }; -async function createAgentImageThumbnailBuffer(buffer: Buffer): Promise { - const sharp = (await import("sharp")).default; - return sharp(buffer, { failOn: "none" }) - .rotate() - .resize(500, null, { - fit: "inside", - withoutEnlargement: true, - }) - .jpeg({ quality: 82 }) - .toBuffer(); -} - -const REPLACEMENT_OUTPUT_ASPECT_RATIOS = [ - "1:1", - "1:4", - "1:8", - "2:3", - "3:2", - "3:4", - "4:1", - "4:3", - "4:5", - "5:4", - "8:1", - "9:16", - "16:9", - "21:9", -] as const; - -type ReplacementReferenceGeometry = { - width: number; - height: number; - aspectRatio: string; -}; - -function parseReplacementAspectRatioValue(ratio: string) { - const [width, height] = ratio.split(":").map((item) => Number(item)); - if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { - return 1; - } - return width / height; -} - -function inferClosestReplacementAspectRatio(width: number, height: number) { - const aspectValue = width / Math.max(1, height); - let closest = "1:1"; - let minDiff = Number.POSITIVE_INFINITY; - - for (const candidate of REPLACEMENT_OUTPUT_ASPECT_RATIOS) { - const diff = Math.abs(parseReplacementAspectRatioValue(candidate) - aspectValue); - if (diff < minDiff) { - minDiff = diff; - closest = candidate; - } - } - - return closest; -} - -async function resolveReplacementReferenceGeometry( - referenceImageUrl: string, -): Promise { - try { - const sharp = (await import("sharp")).default; - const referenceBuffer = await fetchImageBufferFromUrl(referenceImageUrl); - const metadata = await sharp(referenceBuffer, { failOn: "none" }) - .rotate() - .metadata(); - const width = metadata.width; - const height = metadata.height; - - if (!width || !height) { - return null; - } - - return { - width, - height, - aspectRatio: inferClosestReplacementAspectRatio(width, height), - }; - } catch (error) { - console.warn("[Chat API] Failed to resolve replacement reference geometry:", { - referenceImageUrl, - error: error instanceof Error ? error.message : "unknown", - }); - return null; - } -} - -async function normalizeReplacementResultImage(params: { - generatedImageUrl: string; - generatedThumbnailUrl?: string; - referenceImageUrl: string; - referenceGeometry?: ReplacementReferenceGeometry | null; - uploadPrefix: string; -}): Promise<{ - originalUrl: string; - thumbnailUrl: string; - imageSize: string; -}> { - const sharp = (await import("sharp")).default; - const generatedBuffer = await fetchImageBufferFromUrl(params.generatedImageUrl); - let targetWidth = params.referenceGeometry?.width; - let targetHeight = params.referenceGeometry?.height; - - if (!targetWidth || !targetHeight) { - const referenceBuffer = await fetchImageBufferFromUrl(params.referenceImageUrl); - const referenceMetadata = await sharp(referenceBuffer, { failOn: "none" }) - .rotate() - .metadata(); - targetWidth = referenceMetadata.width; - targetHeight = referenceMetadata.height; - } - - if (!targetWidth || !targetHeight) { - throw new Error("无法读取参考图尺寸"); - } - - const generatedMetadata = await sharp(generatedBuffer, { failOn: "none" }) - .rotate() - .metadata(); - if (generatedMetadata.width === targetWidth && generatedMetadata.height === targetHeight) { - return { - originalUrl: params.generatedImageUrl, - thumbnailUrl: params.generatedThumbnailUrl || params.generatedImageUrl, - imageSize: `${targetWidth}x${targetHeight}`, - }; - } - - const normalizedBuffer = await sharp(generatedBuffer, { failOn: "none" }) - .rotate() - .resize(targetWidth, targetHeight, { - // Keep the model output proportional when adapting it back to the reference page. - // `fill` stretches the picture and can squash products; `cover` behaves like a screenshot crop. - fit: "cover", - position: "centre", - withoutEnlargement: false, - }) - .png() - .toBuffer(); - const thumbnailBuffer = await createAgentImageThumbnailBuffer(normalizedBuffer); - - const uniqueSuffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; - const originalFilename = `${params.uploadPrefix}-orig-${uniqueSuffix}.png`; - const thumbnailFilename = `${params.uploadPrefix}-thumb-${uniqueSuffix}.jpg`; - const [originalUpload, thumbnailUpload] = await Promise.all([ - uploadFile(originalFilename, bufferToArrayBuffer(normalizedBuffer)), - uploadFile(thumbnailFilename, bufferToArrayBuffer(thumbnailBuffer)), - ]); - - return { - originalUrl: originalUpload.url, - thumbnailUrl: thumbnailUpload.url, - imageSize: `${targetWidth}x${targetHeight}`, - }; -} - -const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([ - "alibaba-image", - "bfl-image", - "ruxa-image", - "fal-image", - "openai-image", - "stepfun-image", - "runway", - "krea", - "grsai-gpt-image", - "volcano-vision", - "gemini", - "huanke-gemini", -]); - -const IMAGE_FAILOVER_MODEL_COOLDOWN_MS = 30 * 60 * 1000; -const imageFailoverModelCooldown = new Map(); - -function normalizeFailoverErrorMessage(error: unknown): string { - if (!error) return ""; - if (typeof error === "string") return error; - if (error instanceof Error) return error.message || String(error); - try { - return JSON.stringify(error); - } catch { - return String(error); - } -} - -function shouldCooldownImageFallbackModel(errorMessage: string): boolean { - const msg = errorMessage.toLowerCase(); - return ( - msg.includes("model not found") || - msg.includes("model_not_found") || - msg.includes("no available channel for model") || - msg.includes("unknown model") || - msg.includes("invalid model") || - msg.includes("api key is missing") || - msg.includes("invalid api key") || - msg.includes("unauthorized") || - msg.includes("cannot convert argument to a bytestring") - ); -} - -function isImageFallbackModelCoolingDown(modelId: string): boolean { - const snapshot = imageFailoverModelCooldown.get(modelId); - if (!snapshot) return false; - if (snapshot.until <= Date.now()) { - imageFailoverModelCooldown.delete(modelId); - return false; - } - return true; -} - -function setImageFallbackModelCooldown(modelId: string, reason: string): void { - imageFailoverModelCooldown.set(modelId, { - until: Date.now() + IMAGE_FAILOVER_MODEL_COOLDOWN_MS, - reason: reason.slice(0, 500), - }); -} - -function isImageFailoverExcludedModel(model: { excludeFromFailover?: boolean } | null | undefined) { - return Boolean(model?.excludeFromFailover); -} - -type ImageFallbackCandidate = { - model: any; - provider: any; - configured: boolean; - sameProvider: boolean; -}; - -function getImageFailoverConvertProvider(providerType?: string | null) { - switch (providerType) { - case "bfl-image": - case "ruxa-image": - case "fal-image": - case "openai-image": - case "stepfun-image": - case "midjourney": - case "runway": - case "krea": - case "gemini": - case "huanke-gemini": - return providerType; - case "grsai-gpt-image": - return "gemini"; - case "volcano-vision": - return "volcano"; - default: - return "alibaba-image"; - } -} - -function buildImageFallbackCandidates( - allModels: Array<{ model: any; provider: any }>, - options: { - currentProviderId?: string; - currentModelId?: string; - currentModelKey?: string; - configuredFallbackIds?: string[]; - } -) { - const { currentProviderId, currentModelId, currentModelKey, configuredFallbackIds = [] } = options; - const configuredRefs = configuredFallbackIds - .map((item) => String(item || "").trim()) - .filter(Boolean); - const byModelRef = new Map(); - - const isEligible = (item: { model: any; provider: any }, sameProvider: boolean) => { - if ( - !item?.model?.supportsImageGen || - !item?.model?.isEnabled || - item?.model?.isSubModel || - isImageFailoverExcludedModel(item?.model) || - item?.model?.modelId === currentModelKey || - item?.model?.id === currentModelId - ) { - return false; - } - - const providerType = String(item?.provider?.type || ""); - if (!sameProvider && !IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(providerType)) { - return false; - } - if (isImageFallbackModelCoolingDown(item.model.id)) { - const cooldown = imageFailoverModelCooldown.get(item.model.id); - console.log( - `[Chat API] ⏭ Skip configured image fallback on cooldown: ${item.model.displayName} (${providerType}) - ${cooldown?.reason || "cooldown"}` - ); - return false; - } - return true; - }; - - allModels.forEach((item) => { - if (!item?.model) return; - byModelRef.set(String(item.model.id), item); - if (item.model.modelId) { - byModelRef.set(String(item.model.modelId), item); - } - }); - - const candidates: ImageFallbackCandidate[] = []; - const seenModelIds = new Set(); - const pushCandidate = (item: { model: any; provider: any } | undefined, configured: boolean) => { - if (!item?.model) return; - const sameProvider = Boolean(currentProviderId && item.provider?.id === currentProviderId); - if (!isEligible(item, sameProvider)) return; - const modelId = String(item.model.id || ""); - if (!modelId || seenModelIds.has(modelId)) return; - seenModelIds.add(modelId); - candidates.push({ - model: item.model, - provider: item.provider, - configured, - sameProvider, - }); - }; - - configuredRefs.forEach((modelRef) => { - pushCandidate(byModelRef.get(modelRef), true); - }); - - if (currentProviderId) { - allModels - .filter((item) => item.provider?.id === currentProviderId) - .forEach((item) => pushCandidate(item, false)); - } - - return candidates; -} - -async function generateImageWithFailoverCandidate(params: { - candidate: ImageFallbackCandidate; - messages: ChatMessage[]; - imageParams: any; - fallbackImageParams: any; -}) { - const { candidate, messages, imageParams, fallbackImageParams } = params; - const providerType = String(candidate.provider?.type || ""); - const candidateRuntimeModelId = String(candidate.model?.modelId || candidate.model?.id || ""); - if ( - providerType === "fal-image" && - isFalGptImage2EditModelId(candidateRuntimeModelId) - ) { - throw new Error("FAL GPT Image 2 Edit 仅支持异步队列调用，已跳过同步故障迁移"); - } - const candidateLanguageModel = await getLanguageModelById(candidate.model.id); - if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") { - throw new Error(`备用模型不支持当前生图故障迁移调用：${candidate.model.displayName || candidate.model.id}`); - } - - const candidateMessages = await convertMessagesWithImages( - messages, - getImageFailoverConvertProvider(providerType) - ); - const candidateResult = await (candidateLanguageModel as any).doGenerate({ - messages: candidateMessages, - imageParams: imageParams || fallbackImageParams, - modelDisplayName: candidate.model.displayName, - overrideModelId: candidate.model.modelId, - }); - - if (candidateResult?.finishReason === "error") { - throw new Error(candidateResult.text || "图片生成失败"); - } - - return { - text: candidateResult.text || "图片生成失败", - usedModelId: String(candidate.model.modelId || candidate.model.id || ""), - displayName: String(candidate.model.displayName || candidate.model.modelId || candidate.model.id || ""), - }; -} - -function getImageCrossProviderCandidates( - allModels: any[], - options: { - currentModelId: string; - currentProviderId?: string; - providerPriority: Record; - } -) { - return allModels - .filter((m) => { - if ( - !m?.model?.supportsImageGen || - !m?.model?.isEnabled || - m?.model?.isSubModel || - isImageFailoverExcludedModel(m?.model) - ) { - return false; - } - if (!IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(String(m?.provider?.type || ""))) { - return false; - } - if (m.model.id === options.currentModelId) return false; - if (options.currentProviderId && m.provider.id === options.currentProviderId) return false; - if (isImageFallbackModelCoolingDown(m.model.id)) { - const cooldown = imageFailoverModelCooldown.get(m.model.id); - console.log( - `[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "cooldown"}` - ); - return false; - } - return true; - }) - .sort((a, b) => { - const left = options.providerPriority[a.provider.type as string] ?? 99; - const right = options.providerPriority[b.provider.type as string] ?? 99; - if (left !== right) return left - right; - return (a.model.sortOrder ?? 0) - (b.model.sortOrder ?? 0); - }); -} - function hasUserCustomizedImageParams(params?: PostRequestBody["imageParams"]): boolean { if (!params) return false; const customizedAspectRatio = @@ -1328,2735 +555,6 @@ function mergeVideoParamsWithAgentRecommendation(params: { return Object.keys(merged).length > 0 ? (merged as ChatVideoParams) : undefined; } -function sanitizeAgentOutputSettingsSection(params: { - message?: string; - userImageParams?: PostRequestBody["imageParams"]; - hasUserCustomImageParams: boolean; -}): string | undefined { - const { message, userImageParams, hasUserCustomImageParams } = params; - if (!message?.trim()) return message; - - const outputSectionRegex = - /(?:###\s*输出设置|【\s*输出设置\s*】)[\s\S]*?(?=\n(?:###\s*|【\s*[^】]+\s*】)|$)/; - if (!outputSectionRegex.test(message)) return message; - - if (!hasUserCustomImageParams) { - return message.replace( - outputSectionRegex, - "【输出设置】\n- 参数：按用户面板当前设置执行（未自动改写）" - ); - } - - const lines: string[] = []; - const aspectRatio = - typeof userImageParams?.aspectRatio === "string" ? userImageParams.aspectRatio.trim() : ""; - const imageSize = - typeof userImageParams?.imageSize === "string" ? userImageParams.imageSize.trim() : ""; - const imageCount = - typeof userImageParams?.imageCount === "number" ? userImageParams.imageCount : undefined; - - if (aspectRatio && aspectRatio !== "auto") { - lines.push(`- 宽高比：${aspectRatio}（来自用户面板设置）`); - } - if (imageSize && imageSize.toUpperCase() !== "2K") { - lines.push(`- 清晰度档位：${imageSize.toUpperCase()}（来自用户面板设置）`); - } - if (typeof imageCount === "number" && Number.isFinite(imageCount) && imageCount !== 1) { - lines.push(`- 出图数量：${Math.max(1, Math.round(imageCount))} 张（来自用户面板设置）`); - } - lines.push("- 其余参数：保持用户面板当前设置"); - - return message.replace(outputSectionRegex, `【输出设置】\n${lines.join("\n")}`); -} - -function normalizeAgentReportHeadingLine(line: string): string { - const trimmed = line.trim(); - if (!trimmed) return line; - - const markdownHeading = trimmed.match(/^#{1,6}\s*(.+?)\s*$/); - const plainTitle = markdownHeading?.[1]?.trim() || trimmed; - const normalizedTitle = plainTitle - .replace(/^[^\u4e00-\u9fa5A-Za-z0-9]+/, "") - .trim(); - - const knownTitles = [ - "需求理解", - "图片分析", - "执行计划", - "优化后提示词", - "输出设置", - "脚本规划", - "分镜规划", - "执行说明", - "视频生成完成", - ]; - - if (knownTitles.some((title) => normalizedTitle === title)) { - return `【${normalizedTitle}】`; - } - - if (markdownHeading) { - return `【${normalizedTitle || plainTitle}】`; - } - return line; -} - -function stripAgentFriendlyMediaEmbeds(message: string): string { - if (!message) return message; - - const withoutInlineMedia = message - .replace(/!\[[^\]]*]\((https?:\/\/[^\s)]+)\)/gi, "") - .replace(//gi, "") - .replace(/]*>/gi, ""); - - const cleanedLines = withoutInlineMedia - .split("\n") - .filter((line) => { - const trimmed = line.trim(); - if (!trimmed) return true; - if (/^https?:\/\/\S+$/i.test(trimmed)) return false; - if (/^\[[^\]]*]\(https?:\/\/[^\s)]+\)$/i.test(trimmed)) return false; - return true; - }) - .join("\n"); - - return cleanedLines.replace(/\n{3,}/g, "\n\n").trim(); -} - -function normalizeAgentReportForDisplay(message?: string): string | undefined { - if (!message?.trim()) return message; - const stripped = stripAgentFriendlyMediaEmbeds(message); - if (!stripped) return undefined; - return stripped - .split("\n") - .map((line) => normalizeAgentReportHeadingLine(line)) - .join("\n"); -} - -function guessImageMediaType(url: string): string { - const cleanUrl = url.split("?")[0]?.toLowerCase() || ""; - if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "image/jpeg"; - if (cleanUrl.endsWith(".webp")) return "image/webp"; - if (cleanUrl.endsWith(".gif")) return "image/gif"; - if (cleanUrl.endsWith(".bmp")) return "image/bmp"; - if (cleanUrl.endsWith(".svg")) return "image/svg+xml"; - return "image/png"; -} - -function injectHistoryImagesToLastUserMessage( - messages: ChatMessage[], - historyImages: string[] -): ChatMessage[] { - if (!historyImages || historyImages.length === 0) return messages; - - let lastUserIndex = -1; - for (let i = messages.length - 1; i >= 0; i--) { - if (messages[i]?.role === "user") { - lastUserIndex = i; - break; - } - } - - if (lastUserIndex < 0) return messages; - - const lastUserMessage = messages[lastUserIndex]; - const hasExistingImage = lastUserMessage.parts?.some( - (p: any) => p.type === "file" || p.type === "image" || p.type === "image_url" - ); - if (hasExistingImage) return messages; - - const imageParts = historyImages.map((url, index) => ({ - type: "file" as const, - url, - name: `history-image-${index + 1}`, - mediaType: guessImageMediaType(url), - mimeType: guessImageMediaType(url), - })); - - const updatedMessage: ChatMessage = { - ...lastUserMessage, - parts: [...lastUserMessage.parts, ...imageParts], - }; - - return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg)); -} - -function isImageInputPart(part: any) { - return part?.type === "file" || part?.type === "image" || part?.type === "image_url"; -} - -function normalizeResolvedTargetImageUrls(imageUrls?: string[] | null) { - return Array.from( - new Set( - (imageUrls || []) - .map((url) => String(url || "").trim()) - .filter((url) => /^https?:\/\//i.test(url)) - ) - ).slice(0, 8); -} - -function summarizeTargetImageUrl(url?: string) { - const normalized = String(url || "").trim(); - if (!normalized) return undefined; - try { - const parsed = new URL(normalized); - return `${parsed.origin}${parsed.pathname}`.slice(0, 160); - } catch { - return normalized.slice(0, 160); - } -} - -function applyResolvedTargetImagesToLastUserMessage( - messages: ChatMessage[], - targetImageUrls: string[] -): ChatMessage[] { - const normalizedTargetImageUrls = normalizeResolvedTargetImageUrls(targetImageUrls); - if (normalizedTargetImageUrls.length === 0) return messages; - - let lastUserIndex = -1; - for (let i = messages.length - 1; i >= 0; i--) { - if (messages[i]?.role === "user") { - lastUserIndex = i; - break; - } - } - - if (lastUserIndex < 0) return messages; - const lastUserMessage = messages[lastUserIndex]; - const nonImageParts = (lastUserMessage.parts || []).filter((part: any) => !isImageInputPart(part)); - const imageParts = normalizedTargetImageUrls.map((url, index) => ({ - type: "file" as const, - url, - name: `target-image-${index + 1}`, - mediaType: guessImageMediaType(url), - mimeType: guessImageMediaType(url), - })); - - const updatedMessage: ChatMessage = { - ...lastUserMessage, - parts: [...imageParts, ...nonImageParts], - }; - - return messages.map((msg, idx) => (idx === lastUserIndex ? updatedMessage : msg)); -} - -function compactAutoDemand(text: string, maxLength = 56): string { - const normalized = text.replace(/\s+/g, " ").trim(); - if (!normalized) return "按你刚才的描述执行"; - if (normalized.length <= maxLength) return normalized; - return `${normalized.slice(0, maxLength)}...`; -} - -function buildAutoModeFallbackMessage(params: { - text: string; - hasImage: boolean; - fromHistory: boolean; - imageSummary?: string; -}): string { - const demand = compactAutoDemand(params.text); - - if (params.hasImage) { - const sourceLine = params.fromHistory - ? "基于上一轮图片继续处理。" - : "基于当前上传图片处理。"; - - return [ - "### 需求理解", - `- ${demand}`, - "### 图片分析", - params.imageSummary - ? `- ${params.imageSummary.replace(/\s+/g, " ").trim()}` - : `- ${sourceLine}`, - "- 我会优先保持主体、构图和整体风格一致，只处理你指定的改动。", - "### 执行计划", - "- 先完成本轮核心改动，再检查边缘与光影一致性。", - "- 若结果偏差，我会按“保留A，修改B”的方式继续迭代。", - "### 输出设置", - "- 默认单图输出，优先稳定命中。", - ].join("\n"); - } - - return [ - "### 需求理解", - `- ${demand}`, - "### 执行计划", - "- 先锁定主体与场景，再细化风格、材质与光影。", - "### 输出设置", - "- 默认单图输出，优先稳定命中。", - ].join("\n"); -} - -function sanitizePromptOnlyUserIntent(text: string): string { - return String(text || "") - .replace(/先?分析(?:一下)?(?:这张)?(?:图片|图像|这张图)?/g, "") - .replace(/帮我|给我|先给我|只给我|仅给我/g, "") - .replace(/优化(?:一下)?提示词|提示词优化|提示词|prompt/gi, "") - .replace(/不要生图|先别生图|不需要生图|不要生成|别生成|不直接生成/g, "") - .replace(/[，。；：,.!！?？]/g, " ") - .replace(/\s+/g, " ") - .trim(); -} - -function buildPromptOnlyOptimizationSeed(params: { - text: string; - imageSummary?: string; -}): string { - const cleanedIntent = sanitizePromptOnlyUserIntent(params.text); - const summary = String(params.imageSummary || "").replace(/\s+/g, " ").trim(); - - if (cleanedIntent && summary) { - return `${cleanedIntent}。参考图关键特征：${summary}`; - } - if (summary) { - return `基于参考图生成一段可直接复现核心主体、场景、风格与光线特征的中文提示词。参考图关键特征：${summary}`; - } - if (cleanedIntent) { - return cleanedIntent; - } - return "基于当前参考图整理一段可直接使用的中文提示词。"; -} - -function buildPromptOnlyReply(params: { - userText: string; - imageSummary?: string; - optimizedPrompt: string; -}): string { - const demand = sanitizePromptOnlyUserIntent(params.userText) || "你希望我先分析这张图并整理可直接使用的提示词"; - const summaryLines = String(params.imageSummary || "") - .split(/\r?\n/) - .map((line) => line.replace(/\s+/g, " ").trim()) - .filter(Boolean); - const formattedSummary = - summaryLines.length > 0 - ? summaryLines.map((line) => `- ${line}`).join("\n") - : "- 我已基于当前图片梳理主体、场景、风格与光线特征。"; - - return [ - "【需求理解】", - `- ${demand}`, - "【图片分析】", - formattedSummary, - "【优化后提示词】", - `- ${params.optimizedPrompt.replace(/\s+/g, " ").trim()}`, - "【使用建议】", - "- 本轮仅输出优化后的提示词，不直接执行生图。", - ].join("\n"); -} - -function hasStructuredAgentSections(message?: string): boolean { - if (!message) return false; - return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test( - message, - ); -} - -function ensureStructuredAgentFriendlyMessage(params: { - message?: string; - text: string; - hasImage: boolean; - fromHistory: boolean; -}): string { - const existingMessage = String(params.message || "").trim(); - if (hasStructuredAgentSections(existingMessage)) { - return existingMessage; - } - - const fallback = buildAutoModeFallbackMessage({ - text: params.text, - hasImage: params.hasImage, - fromHistory: params.fromHistory, - }); - - if (!existingMessage) { - return fallback; - } - - const normalizedSupplement = existingMessage.replace(/\s+/g, " ").trim(); - if (!normalizedSupplement) { - return fallback; - } - - return fallback.replace( - "### 执行计划\n", - `### 执行计划\n- 补充说明：${normalizedSupplement}\n`, - ); -} - -function extractMessageImageUrls(parts: any[]): string[] { - return (parts || []) - .map((part: any) => { - if (!part) return null; - if (part.type === "file") return part.url || part.data || null; - if (part.type === "image") return part.url || part.image || null; - if (part.type === "image_url") return part.image_url?.url || null; - return null; - }) - .filter((url: any): url is string => typeof url === "string" && /^https?:\/\//i.test(url)); -} - -function extractMessageImageAssetIds(parts: any[]): string[] { - return Array.from( - new Set( - (parts || []) - .map((part: any) => { - if (!part || typeof part !== "object") return null; - return typeof part.assetId === "string" && part.assetId.trim() - ? part.assetId.trim() - : null; - }) - .filter((assetId: any): assetId is string => typeof assetId === "string" && assetId.length > 0), - ), - ); -} - -function shouldEnrichImageAnalysisText(message?: string): boolean { - if (!message) return false; - if (!/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) return false; - return /(基于当前上传图片处理|基于你选中的原图处理|基于上一轮图片继续处理)/.test(message); -} - -function injectImageAnalysisSummary(message: string, summary: string): string { - const normalizedSummaryLines = summary - .split(/\r?\n/) - .map((line) => line.replace(/\s+/g, " ").trim()) - .filter(Boolean); - if (normalizedSummaryLines.length === 0) return message; - const lines = message.split("\n"); - const analysisHeaderIndex = lines.findIndex((line) => - /^(###\s*图片分析|【\s*图片分析\s*】)/.test(line.trim()) - ); - if (analysisHeaderIndex < 0) return message; - - for (let i = analysisHeaderIndex + 1; i < lines.length; i += 1) { - const current = lines[i]?.trim() || ""; - if (/^(###\s*|【\s*[^】]+\s*】)/.test(current)) break; - if (/^- /.test(current)) { - lines.splice(i, 1, ...normalizedSummaryLines.map((line) => `- ${line}`)); - return lines.join("\n"); - } - } - - const insertIndex = analysisHeaderIndex + 1; - lines.splice(insertIndex, 0, ...normalizedSummaryLines.map((line) => `- ${line}`)); - return lines.join("\n"); -} - -function ensureImageAnalysisSummarySection(message: string, summary: string): string { - const normalizedSummaryLines = summary - .split(/\r?\n/) - .map((line) => line.replace(/\s+/g, " ").trim()) - .filter(Boolean); - if (normalizedSummaryLines.length === 0) return message; - - if (/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) { - return injectImageAnalysisSummary(message, normalizedSummaryLines.join("\n")); - } - - const trimmed = message.trim(); - const summaryBlock = normalizedSummaryLines.map((line) => `- ${line}`).join("\n"); - if (!trimmed) return `【图片分析】\n${summaryBlock}`; - return `${trimmed}\n【图片分析】\n${summaryBlock}`; -} - -/** - * 自定义消息转换函数 - * 注意：gemini-image 和 huanke-gemini 都需要 URL 格式（不是 Base64） - * 因为这些 provider 内部会自行处理图片上传到 S3 - */ -async function convertMessagesWithImages( - messages: ChatMessage[], - providerType: - | string - | { - providerType?: string; - providerBaseUrl?: string | null; - modelId?: string | null; - displayName?: string | null; - forceInlineImageUrls?: boolean; - loggerPrefix?: string; - }, -) { - type ImageConversionTarget = { - providerType?: string; - providerBaseUrl?: string | null; - modelId?: string | null; - displayName?: string | null; - forceInlineImageUrls?: boolean; - loggerPrefix?: string; - }; - - const normalizeTarget = ( - target: string | ImageConversionTarget, - ): Required> & - Omit => { - if (typeof target === "string") { - return { - providerType: target, - providerBaseUrl: null, - modelId: null, - displayName: null, - forceInlineImageUrls: false, - loggerPrefix: "[Chat API] Vision inline", - }; - } - - return { - providerType: String(target.providerType || ""), - providerBaseUrl: target.providerBaseUrl || null, - modelId: target.modelId || null, - displayName: target.displayName || null, - forceInlineImageUrls: target.forceInlineImageUrls === true, - loggerPrefix: target.loggerPrefix || "[Chat API] Vision inline", - }; - }; - - const extractImagePartUrl = (part: unknown): string | null => { - const partAny = part as any; - const candidateUrl = - partAny?.type === "file" - ? partAny.url || partAny.data - : partAny?.type === "image" - ? partAny.image || partAny.url - : partAny?.type === "image_url" - ? partAny.image_url?.url - : undefined; - - if (typeof candidateUrl !== "string") return null; - const normalized = candidateUrl.trim(); - if (!/^https?:\/\//i.test(normalized)) return null; - return normalized; - }; - - const replaceImagePartUrl = (part: unknown, nextUrl: string): unknown => { - const partAny = part as any; - if (partAny?.type === "file") { - return { - ...partAny, - url: nextUrl, - data: nextUrl, - mediaType: String(partAny.mediaType || "").startsWith("image/") - ? partAny.mediaType - : "image/jpeg", - mimeType: String(partAny.mimeType || "").startsWith("image/") - ? partAny.mimeType - : "image/jpeg", - }; - } - if (partAny?.type === "image") { - return { - ...partAny, - image: nextUrl, - url: nextUrl, - }; - } - if (partAny?.type === "image_url") { - return { - ...partAny, - image_url: { - ...(partAny.image_url || {}), - url: nextUrl, - }, - }; - } - return part; - }; - - const inlineImageUrlsInMessages = async ( - sourceMessages: ChatMessage[], - target: ReturnType, - ) => { - const uniqueUrls = Array.from( - new Set( - sourceMessages - .flatMap((message) => message.parts || []) - .map((part) => extractImagePartUrl(part)) - .filter((url): url is string => Boolean(url)), - ), - ); - - if (uniqueUrls.length === 0) { - return sourceMessages; - } - - const { inlineDataUrls } = await inlineVisionImages(uniqueUrls, { - maxImages: Math.min(8, uniqueUrls.length), - maxSide: 1200, - timeoutMs: 12000, - retries: 2, - loggerPrefix: target.loggerPrefix, - }).catch((error) => { - console.warn(`${target.loggerPrefix} failed to inline message images:`, error); - return { inlineDataUrls: [], skipped: [] }; - }); - - if (inlineDataUrls.length === 0) { - return sourceMessages; - } - - const inlineMap = new Map(); - for (let index = 0; index < inlineDataUrls.length; index += 1) { - const originalUrl = uniqueUrls[index]; - const inlineUrl = inlineDataUrls[index]; - if (originalUrl && inlineUrl) { - inlineMap.set(originalUrl, inlineUrl); - } - } - - if (inlineMap.size === 0) { - return sourceMessages; - } - - console.log(`${target.loggerPrefix} inlined chat message images:`, { - total: uniqueUrls.length, - inlined: inlineMap.size, - providerType: target.providerType, - modelId: target.modelId, - displayName: target.displayName, - }); - - return sourceMessages.map((message) => ({ - ...message, - parts: (message.parts || []).map((part) => { - const originalUrl = extractImagePartUrl(part); - if (!originalUrl) return part; - const inlineUrl = inlineMap.get(originalUrl); - return inlineUrl ? replaceImagePartUrl(part, inlineUrl) : part; - }) as typeof message.parts, - })) as ChatMessage[]; - }; - - const target = normalizeTarget(providerType); - let preparedMessages = messages; - - if (target.forceInlineImageUrls) { - preparedMessages = await inlineImageUrlsInMessages(preparedMessages, target); - } - - const normalizedProviderType = target.providerType; - - if ( - normalizedProviderType === "bfl-image" || - normalizedProviderType === "ruxa-image" || - normalizedProviderType === "fal-image" || - normalizedProviderType === "openai-image" || - normalizedProviderType === "stepfun-image" || - normalizedProviderType === "midjourney" || - normalizedProviderType === "runway" || - normalizedProviderType === "krea" - ) { - const convertedMessages = []; - - for (const message of preparedMessages) { - const content: any[] = []; - - for (const part of message.parts) { - const partAny = part as any; - - if (partAny.type === "text" && partAny.text) { - content.push({ - type: "text", - text: partAny.text, - }); - continue; - } - - const candidateUrl = - partAny.type === "file" - ? (partAny.url || partAny.data) - : partAny.type === "image" - ? (partAny.image || partAny.url) - : partAny.type === "image_url" - ? partAny.image_url?.url - : undefined; - - if (!candidateUrl || typeof candidateUrl !== "string") continue; - if (candidateUrl.startsWith("data:")) { - throw new Error("当前模型不支持 Base64 图片传输，请先上传图片后再编辑"); - } - if (!candidateUrl.startsWith("http")) continue; - - content.push({ - type: "image_url", - image_url: { - url: candidateUrl, - }, - }); - } - - convertedMessages.push({ - role: message.role, - content: content.length === 1 && content[0].type === "text" ? content[0].text : content, - }); - } - - return convertedMessages; - } - - // 所有 provider 都使用 URL 格式传输图片 - // gemini-image 和 huanke-gemini 内部会将 Base64 转换为 S3 URL - // 这里不再进行 Base64 转换，直接返回标准格式 - if (normalizedProviderType !== "gemini") { - return convertToModelMessages(preparedMessages); - } - - // 🆕 检测是否有标注图片 - const hasAnnotation = hasAnnotatedImage(messages); - if (hasAnnotation) { - console.log("[Message Convert] 🎨 检测到标注图片，将自动添加系统提示"); - } - - // 对于 Gemini，需要特殊处理图片 - const convertedMessages = []; - - for (const message of preparedMessages) { - const content: any[] = []; - - // 🆕 如果是用户消息且有标注图片，在文本前添加系统提示 - let hasAddedAnnotationPrompt = false; - - // 处理消息的各个部分 - for (const part of message.parts) { - if (part.type === "text" && part.text) { - let textContent = part.text; - - // 🆕 如果有标注图片且是用户消息，在用户文本前添加系统提示 - if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt) { - textContent = ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + textContent; - hasAddedAnnotationPrompt = true; - console.log("[Message Convert] 🎨 已注入标注提示词"); - } - - content.push({ - type: "text", - text: textContent, - }); - } else if (part.type === "file" && part.url) { - // 直接使用 URL 格式（不再转换为 Base64） - // gemini-image 和 huanke-gemini provider 内部会处理图片上传 - content.push({ - type: "image_url", - image_url: { - url: part.url, - }, - }); - console.log("[Message Convert] Image kept as URL format:", part.url.substring(0, 80) + "..."); - } - } - - // 🆕 如果有标注图片但消息中没有文本部分，添加一个纯提示文本 - if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt && content.length > 0) { - content.unshift({ - type: "text", - text: ANNOTATION_SYSTEM_PROMPT, - }); - console.log("[Message Convert] 🎨 已在消息开头注入标注提示词"); - } - - convertedMessages.push({ - role: message.role, - content: content.length === 1 && content[0].type === "text" - ? content[0].text - : content, - }); - } - - return convertedMessages; -} - -function extractOriginalImageUrlsFromText(text: string): string[] { - return extractImageUrlsFromText(text); -} - -function normalizeImageInputRef(rawValue: unknown, requestOrigin?: string): string | null { - if (typeof rawValue !== "string") return null; - const trimmed = rawValue.trim(); - if (!trimmed) return null; - - if (/^data:image\/[a-z0-9+.-]+;base64,/i.test(trimmed)) { - return trimmed.replace(/\s+/g, ""); - } - - if (/^https?:\/\//i.test(trimmed)) { - return trimmed; - } - - if (requestOrigin && trimmed.startsWith("/")) { - try { - return new URL(trimmed, requestOrigin).toString(); - } catch { - return null; - } - } - - return null; -} - -function collectImageInputsFromMessageParts( - parts: Array> | undefined, - requestOrigin?: string, -): string[] { - if (!Array.isArray(parts) || parts.length === 0) return []; - - const result: string[] = []; - const pushInput = (rawValue: unknown) => { - const normalized = normalizeImageInputRef(rawValue, requestOrigin); - if (!normalized) return; - if (!result.includes(normalized)) { - result.push(normalized); - } - }; - - for (const part of parts) { - if (!part) continue; - - if (part.type === "file") { - pushInput(part.url); - pushInput(part.data); - continue; - } - - if (part.type === "image") { - pushInput(part.url); - pushInput(part.image); - pushInput(part.data); - continue; - } - - if (part.type === "image_url") { - pushInput(part.image_url?.url); - pushInput(part.url); - continue; - } - - if (part.type === "text" && typeof part.text === "string") { - const parsedUrls = extractOriginalImageUrlsFromText(part.text); - for (const parsedUrl of parsedUrls) { - pushInput(parsedUrl); - } - } - } - - return result; -} - -function mimeTypeToExtension(mimeType: string) { - const normalized = mimeType.toLowerCase(); - if (normalized.includes("png")) return "png"; - if (normalized.includes("webp")) return "webp"; - if (normalized.includes("gif")) return "gif"; - if (normalized.includes("bmp")) return "bmp"; - return "jpg"; -} - -function getExtensionFromUrl(url: string): string { - try { - const pathname = new URL(url).pathname; - const ext = pathname.split(".").pop()?.toLowerCase(); - if (!ext) return "jpg"; - if (["jpg", "jpeg", "png", "webp", "gif", "bmp"].includes(ext)) return ext; - } catch { - // ignore - } - return "jpg"; -} - -async function uploadRemoteImageToStorage(imageUrl: string): Promise { - const response = await fetch(imageUrl, { - method: "GET", - signal: AbortSignal.timeout(30_000), - }); - - if (!response.ok) { - throw new Error(`Remote image fetch failed: ${response.status} ${response.statusText}`); - } - - const contentType = response.headers.get("content-type")?.toLowerCase() || ""; - if (contentType && !contentType.startsWith("image/")) { - throw new Error(`Remote url is not an image: ${contentType}`); - } - - const arrayBuffer = await response.arrayBuffer(); - if (!arrayBuffer || arrayBuffer.byteLength === 0) { - throw new Error("Remote image content is empty"); - } - - const ext = contentType ? mimeTypeToExtension(contentType) : getExtensionFromUrl(imageUrl); - const fileName = `remote-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`; - const uploaded = await uploadFile(fileName, arrayBuffer); - return uploaded.url; -} - -async function isRemoteImageAccessible(imageUrl: string): Promise { - try { - const headResponse = await fetch(imageUrl, { - method: "HEAD", - signal: AbortSignal.timeout(10_000), - }); - - if (headResponse.ok) { - return true; - } - - if (headResponse.status !== 405 && headResponse.status !== 403) { - return false; - } - - const probeResponse = await fetch(imageUrl, { - method: "GET", - headers: { - Range: "bytes=0-0", - }, - signal: AbortSignal.timeout(10_000), - }); - return probeResponse.ok; - } catch { - return false; - } -} - -async function ensureHostedImageUrl(imageUrl: string): Promise { - const normalizedUrl = imageUrl.trim(); - if (!normalizedUrl) { - throw new Error("Image URL is empty"); - } - - if (normalizedUrl.startsWith("data:image")) { - return uploadDataUrlImage(normalizedUrl); - } - - if (!/^https?:\/\//i.test(normalizedUrl)) { - throw new Error(`Unsupported image URL: ${normalizedUrl}`); - } - - if (/\/uploads\//i.test(normalizedUrl)) { - const accessible = await isRemoteImageAccessible(normalizedUrl); - if (accessible) { - return normalizedUrl; - } - } - - return uploadRemoteImageToStorage(normalizedUrl); -} - -async function uploadDataUrlImage(dataUrl: string): Promise { - const match = dataUrl.match(/^data:([^;]+);base64,([\s\S]+)$/i); - if (!match) { - throw new Error("Invalid data URL image format"); - } - - const mimeType = match[1] || "image/jpeg"; - const base64Data = (match[2] || "").replace(/\s+/g, ""); - if (!base64Data) { - throw new Error("Empty base64 image content"); - } - - const buffer = Buffer.from(base64Data, "base64"); - const fileExt = mimeTypeToExtension(mimeType); - const fileName = `inline-image-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${fileExt}`; - const arrayBuffer = buffer.buffer.slice( - buffer.byteOffset, - buffer.byteOffset + buffer.byteLength - ) as ArrayBuffer; - const uploaded = await uploadFile(fileName, arrayBuffer); - return uploaded.url; -} - -async function normalizeAssistantImageMarkdown( - text: string, - imageSize = "2K" -): Promise { - if (!text) return text; - - const markdownDataUrlRegex = /!\[([^\]]*)\]\s*\(\s*(data:image\/[a-z0-9+.-]+;base64,[^)]+)\s*\)/gim; - const markdownMatches = [...text.matchAll(markdownDataUrlRegex)]; - let normalizedText = text; - const uploadedUrls: string[] = []; - - if (markdownMatches.length > 0) { - let rebuilt = ""; - let cursor = 0; - - for (const match of markdownMatches) { - const matchText = match[0]; - const dataUrl = match[2]; - const start = match.index ?? -1; - if (start < 0) continue; - - rebuilt += normalizedText.slice(cursor, start); - - try { - const uploadedUrl = await uploadDataUrlImage(dataUrl); - uploadedUrls.push(uploadedUrl); - rebuilt += `![thumb|||${uploadedUrl}|||${imageSize}](${uploadedUrl})`; - } catch (error) { - console.error("[Chat API] Failed to upload inline data URL image:", error); - rebuilt += matchText; - } - - cursor = start + matchText.length; - } - - rebuilt += normalizedText.slice(cursor); - normalizedText = rebuilt; - } - - if (normalizedText.includes("data:image")) { - const rawDataUrlRegex = /data:image\/[a-z0-9+.-]+;base64,[a-z0-9+/=\s]+/gim; - const rawMatches = normalizedText.match(rawDataUrlRegex) ?? []; - - for (const rawDataUrl of rawMatches) { - try { - const uploadedUrl = await uploadDataUrlImage(rawDataUrl); - uploadedUrls.push(uploadedUrl); - normalizedText = normalizedText.replace(rawDataUrl, uploadedUrl); - } catch (error) { - console.error("[Chat API] Failed to normalize raw data URL image:", error); - } - } - } - - const markdownHttpImageRegex = /!\[([^\]]*)\]\s*\(\s*(https?:\/\/[^)\s]+)\s*\)/gim; - const markdownHttpMatches = [...normalizedText.matchAll(markdownHttpImageRegex)]; - - if (markdownHttpMatches.length > 0) { - let rebuilt = ""; - let cursor = 0; - - for (const match of markdownHttpMatches) { - const matchText = match[0]; - const altText = match[1] || ""; - const thumbnailUrl = (match[2] || "").trim(); - const start = match.index ?? -1; - if (start < 0) continue; - - rebuilt += normalizedText.slice(cursor, start); - - try { - const { originalUrl, imageSize: parsedImageSize } = parseImageAltMetadata( - altText, - thumbnailUrl - ); - - let hostedThumbnailUrl = thumbnailUrl; - let hostedOriginalUrl = - originalUrl && originalUrl !== thumbnailUrl ? originalUrl : thumbnailUrl; - - try { - hostedThumbnailUrl = await ensureHostedImageUrl(thumbnailUrl); - } catch (error) { - console.error("[Chat API] Failed to normalize remote thumbnail URL:", error); - } - - if (originalUrl && originalUrl !== thumbnailUrl) { - try { - hostedOriginalUrl = await ensureHostedImageUrl(originalUrl); - } catch (error) { - console.error("[Chat API] Failed to normalize remote original URL:", error); - hostedOriginalUrl = hostedThumbnailUrl || originalUrl; - } - } else { - hostedOriginalUrl = hostedThumbnailUrl; - } - - const finalThumbnailUrl = hostedThumbnailUrl || thumbnailUrl; - const finalOriginalUrl = hostedOriginalUrl || finalThumbnailUrl; - - uploadedUrls.push(finalThumbnailUrl); - uploadedUrls.push(finalOriginalUrl); - rebuilt += `![thumb|||${finalOriginalUrl}|||${parsedImageSize || imageSize}](${finalThumbnailUrl})`; - } catch (error) { - console.error("[Chat API] Failed to parse remote image markdown metadata:", error); - rebuilt += matchText; - } - - cursor = start + matchText.length; - } - - rebuilt += normalizedText.slice(cursor); - normalizedText = rebuilt; - } - - const hasMarkdownImage = /!\[[^\]]*\]\((https?:\/\/[^)]+)\)/i.test(normalizedText); - if (!hasMarkdownImage && uploadedUrls.length > 0) { - const uniqueUrls = [...new Set(uploadedUrls)]; - console.log( - `[Chat API] Converted ${uniqueUrls.length} image link(s) to hosted link(s)` - ); - return uniqueUrls - .map((url) => `![thumb|||${url}|||${imageSize}](${url})`) - .join("\n"); - } - - return normalizedText; -} - -function resolveModelDisplayNameByModelId( - models: Awaited> | undefined, - modelId: string | undefined, -): string { - if (!modelId || !models?.length) return ""; - const match = models.find( - (item) => item.model.modelId === modelId || item.model.id === modelId, - ); - return String(match?.model?.displayName || ""); -} - -function resolveModelEntryByRuntimeModelId( - models: Awaited> | undefined, - modelId: string | undefined, - fallbackRef?: string, -) { - const candidates = [modelId, fallbackRef] - .map((item) => String(item || "").trim()) - .filter(Boolean); - if (!candidates.length || !models?.length) return null; - - return ( - models.find((item) => - candidates.some( - (candidate) => - item.model.modelId === candidate || item.model.id === candidate, - ), - ) || null - ); -} - -function resolveModelRefByRuntimeModelId( - models: Awaited> | undefined, - modelId: string | undefined, - fallbackRef?: string, -): string { - const normalizedModelId = String(modelId || "").trim(); - if (!normalizedModelId) return fallbackRef || ""; - - const match = models?.find( - (item) => item.model.modelId === normalizedModelId || item.model.id === normalizedModelId, - ); - return String(match?.model?.id || fallbackRef || normalizedModelId || ""); -} - -type AgentPresetMessageImagePart = { - type: "file"; - url: string; - name: string; - mediaType: string; - assetId?: string; -}; - -function extractCurrentMessageImageParts( - parts: Array>, -): AgentPresetMessageImagePart[] { - return parts - .filter( - (part): part is AgentPresetMessageImagePart => - part?.type === "file" && - typeof part?.url === "string" && - /^https?:\/\//i.test(part.url) && - typeof part?.mediaType === "string" && - part.mediaType.startsWith("image/") - ) - .map((part) => ({ - type: "file", - url: String(part.url).trim(), - name: String(part.name || "图片").trim() || "图片", - mediaType: String(part.mediaType || "image/jpeg").trim() || "image/jpeg", - assetId: - typeof part.assetId === "string" && part.assetId.trim() ? part.assetId.trim() : undefined, - })); -} - -function getSceneUploadedImageBreakdown(params: { - uploadedImageParts: AgentPresetMessageImagePart[]; - fallbackPrimaryCount: number; - fallbackIncludeModelImage: boolean; -}) { - const normalizedNames = params.uploadedImageParts.map((part) => String(part.name || "").trim()); - const primaryCountFromNames = normalizedNames.filter((name) => name.startsWith("主SKU参考图")).length; - const secondaryCountFromNames = normalizedNames.filter((name) => name.startsWith("搭配SKU参考图")).length; - const hasModelFromNames = normalizedNames.some((name) => name.startsWith("品牌模特图")); - const hasNamedBreakdown = - primaryCountFromNames > 0 || secondaryCountFromNames > 0 || hasModelFromNames; - - if (hasNamedBreakdown) { - return { - primaryProductImageCount: primaryCountFromNames, - bundleProductImageCount: secondaryCountFromNames, - includeModelImage: hasModelFromNames, - }; - } - - const uploadedImageCount = params.uploadedImageParts.length; - const includeModelImage = - params.fallbackIncludeModelImage && uploadedImageCount > params.fallbackPrimaryCount; - const bundleProductImageCount = Math.max( - 0, - uploadedImageCount - params.fallbackPrimaryCount - (includeModelImage ? 1 : 0) - ); - const primaryProductImageCount = Math.max( - 0, - Math.min( - params.fallbackPrimaryCount, - uploadedImageCount - bundleProductImageCount - (includeModelImage ? 1 : 0) - ) - ); - - return { - primaryProductImageCount, - bundleProductImageCount, - includeModelImage, - }; -} - -function resolveScenePromptOptimizationImageUrls(params: { - uploadedImageParts: AgentPresetMessageImagePart[]; - primaryProductImageCount: number; - bundleProductImageCount: number; - includeModelImage: boolean; -}) { - const namedPrimaryParts = params.uploadedImageParts.filter((part) => - String(part.name || "").trim().startsWith("主SKU参考图") - ); - const namedModelPart = params.uploadedImageParts.find((part) => - String(part.name || "").trim().startsWith("品牌模特图") - ); - - if (namedPrimaryParts.length > 0 || namedModelPart) { - return { - primaryImageUrl: namedPrimaryParts[0]?.url || "", - modelImageUrl: namedModelPart?.url || null, - }; - } - - const primaryImageUrl = - params.uploadedImageParts.slice(0, Math.max(0, params.primaryProductImageCount))[0]?.url || ""; - const modelIndex = params.primaryProductImageCount + params.bundleProductImageCount; - - return { - primaryImageUrl, - modelImageUrl: - params.includeModelImage && modelIndex < params.uploadedImageParts.length - ? params.uploadedImageParts[modelIndex]?.url || null - : null, - }; -} - -function normalizeAgentSupplementPrompt(text: string, presetName?: string | null) { - const normalized = String(text || "").trim(); - if (!normalized) return ""; - const escapedPresetName = String(presetName || "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - const genericPattern = escapedPresetName - ? new RegExp(`^请按「${escapedPresetName}」风格化配置顺序处理这\\d+张图片。?$`) - : /^请按「.+」风格化配置顺序处理这\d+张图片。?$/; - if (genericPattern.test(normalized)) { - return ""; - } - return normalized; -} - -function normalizeAgentWorkflowPrompt(text: string) { - return String(text || "") - .split(/\n+/) - .map((line) => line.replace(/\s+/g, " ").trim()) - .filter(Boolean) - .join("\n") - .trim(); -} - -function parseAgentWorkflowPromptSections(text: string) { - const sectionMap = new Map(); - let activeSection = "__root__"; - const normalizedPrompt = normalizeAgentWorkflowPrompt(text); - if (!normalizedPrompt) { - return sectionMap; - } - - normalizedPrompt.split("\n").forEach((line) => { - const trimmedLine = line.trim(); - if (!trimmedLine) { - return; - } - const headerMatch = trimmedLine.match(/^【([^】]{1,30})】$/); - if (headerMatch) { - activeSection = headerMatch[1].trim(); - if (!sectionMap.has(activeSection)) { - sectionMap.set(activeSection, []); - } - return; - } - - const currentLines = sectionMap.get(activeSection) || []; - currentLines.push(trimmedLine); - sectionMap.set(activeSection, currentLines); - }); - - return sectionMap; -} - -function buildAgentPromptSectionText( - title: string, - lines: string[], - options?: { bullet?: boolean } -) { - const normalizedLines = lines - .map((line) => String(line || "").trim()) - .filter(Boolean); - if (normalizedLines.length === 0) { - return ""; - } - - const sectionBody = options?.bullet - ? normalizedLines.map((line) => (line.startsWith("- ") ? line : `- ${line}`)).join("\n") - : normalizedLines.join("\n"); - - return `【${title}】\n${sectionBody}`; -} - -function buildOptimizedAgentSceneWorkflowPrompt(params: { - workflowPrompt: string; - optimizationResult?: { - sceneGoal: string; - environmentSummary: string; - propSummary: string; - cameraLightingSummary: string; - characterSummary: string; - negativeConstraints: string[]; - conflicts: string[]; - } | null; -}) { - const normalizedWorkflowPrompt = normalizeAgentWorkflowPrompt(params.workflowPrompt); - if (!normalizedWorkflowPrompt) { - return ""; - } - - const optimizationResult = params.optimizationResult || null; - if (!optimizationResult) { - return normalizedWorkflowPrompt; - } - - const sectionMap = parseAgentWorkflowPromptSections(normalizedWorkflowPrompt); - const sceneGoalLines = [optimizationResult.sceneGoal, optimizationResult.environmentSummary].filter( - Boolean - ); - const propLines = [optimizationResult.propSummary].filter(Boolean); - const cameraLightingLines = [optimizationResult.cameraLightingSummary].filter(Boolean); - const characterLines = [optimizationResult.characterSummary].filter(Boolean); - const negativeLines = Array.from( - new Set([ - ...optimizationResult.negativeConstraints, - ...optimizationResult.conflicts.map((item) => `冲突校正：${item}`), - ]) - ); - - const optimizedSections = [ - buildAgentPromptSectionText("项目背景", sectionMap.get("项目背景") || sectionMap.get("__root__") || []), - buildAgentPromptSectionText("主SKU锁定", sectionMap.get("主SKU锁定") || []), - buildAgentPromptSectionText("搭配SKU锁定", sectionMap.get("搭配SKU锁定") || []), - buildAgentPromptSectionText("参考图", sectionMap.get("参考图") || []), - buildAgentPromptSectionText("场景创作总纲", sceneGoalLines), - buildAgentPromptSectionText("陈列与道具", propLines), - buildAgentPromptSectionText("机位与光线", cameraLightingLines), - buildAgentPromptSectionText("人物规则", characterLines), - buildAgentPromptSectionText("负面约束", negativeLines, { bullet: true }), - ].filter(Boolean); - - return optimizedSections.join("\n\n").trim(); -} - -const AGENT_PROMPT_STRUCTURAL_LABELS = new Set([ - "角色", - "技能", - "目标", - "约束条件", - "流程", - "输出", -]); - -const AGENT_PROMPT_LABEL_SEMANTIC_ALIASES = new Map([ - ["角色", "role"], - ["技能", "skills"], - ["目标", "goal"], - ["约束条件", "constraints"], - ["流程", "process"], - ["输出", "output"], - ["人群选择", "audience"], - ["人物", "audience"], - ["主题风格", "style"], - ["整体风格", "style"], - ["空间选择", "space"], - ["空间布局", "space"], - ["主空间", "space"], - ["子空间选择", "subspace"], - ["子空间", "subspace"], - ["硬装方案", "hardDecoration"], - ["硬装", "hardDecoration"], - ["软装方案", "softDecoration"], - ["软装", "softDecoration"], - ["光氛选择", "lighting"], - ["光源布置", "lighting"], - ["灯具", "lighting"], - ["机位选择", "camera"], - ["拍摄角度", "camera"], - ["配饰陈列", "accessories"], - ["放在产品内的小件配饰", "accessories"], - ["前景", "sceneLayout"], - ["中景", "sceneLayout"], - ["背景", "sceneLayout"], -]); - -const AGENT_SCENE_WORKFLOW_OWNED_SEMANTICS = new Set([ - "role", - "skills", - "goal", - "constraints", - "process", - "output", - "audience", - "style", - "space", - "subspace", - "hardDecoration", - "softDecoration", - "lighting", - "camera", - "accessories", - "sceneLayout", -]); - -function normalizePromptLineForCompare(line: string) { - return String(line || "") - .replace(/\s+/g, " ") - .replace(/[：:]/g, ":") - .replace(/[。；;，,、!！?？]+$/g, "") - .trim() - .toLowerCase(); -} - -function parsePromptLineLabel(line: string) { - const normalized = String(line || "").replace(/\s+/g, " ").trim(); - if (!normalized) return { label: "", value: "" }; - - const headerOnlyMatch = normalized.match(/^([^:：]{1,20})[:：]\s*$/); - if (headerOnlyMatch) { - return { - label: headerOnlyMatch[1].trim(), - value: "", - }; - } - - const colonMatch = normalized.match(/^([^:：]{1,20})[:：]\s*(.+)$/); - if (colonMatch) { - return { - label: colonMatch[1].trim(), - value: colonMatch[2].trim(), - }; - } - - const whitespaceMatch = normalized.match(/^([^\s]{1,20})\s+(.+)$/); - if (whitespaceMatch) { - return { - label: whitespaceMatch[1].trim(), - value: whitespaceMatch[2].trim(), - }; - } - - return { label: "", value: normalized }; -} - -function normalizePromptLabel(label: string) { - return String(label || "") - .replace(/\s+/g, "") - .replace(/[：:]/g, "") - .trim(); -} - -function getPromptSemanticLabel(label: string) { - const normalized = normalizePromptLabel(label); - return AGENT_PROMPT_LABEL_SEMANTIC_ALIASES.get(normalized) || normalized; -} - -function collectPromptSemanticLabelsFromText(text: string) { - const semanticLabels = new Set(); - const normalizedText = String(text || "").replace(/\s+/g, " ").trim(); - if (!normalizedText) { - return semanticLabels; - } - - for (const [label, semantic] of AGENT_PROMPT_LABEL_SEMANTIC_ALIASES.entries()) { - if (!label) { - continue; - } - if (normalizedText.includes(`${label}：`) || normalizedText.includes(`${label}:`)) { - semanticLabels.add(semantic); - continue; - } - if (normalizedText.includes(`${label} `) || normalizedText.includes(label)) { - semanticLabels.add(semantic); - } - } - - return semanticLabels; -} - -function shouldDropSceneSlotSemanticLine( - slotSemanticLabel: string, - workflowSemanticLabels: Set, -) { - if (!slotSemanticLabel || !AGENT_SCENE_WORKFLOW_OWNED_SEMANTICS.has(slotSemanticLabel)) { - return false; - } - - if (slotSemanticLabel === "sceneLayout") { - return [ - "style", - "space", - "subspace", - "hardDecoration", - "softDecoration", - "accessories", - "camera", - "lighting", - ].some((semantic) => workflowSemanticLabels.has(semantic)); - } - - return workflowSemanticLabels.has(slotSemanticLabel); -} - -function dedupeAgentSlotPromptAgainstWorkflow(params: { - workflowPrompt?: string | null; - slotPrompt?: string | null; - modeType?: "scene_generation" | "detail_replacement" | "sku_replacement" | "generic"; -}) { - const dedupedSlotLines: string[] = []; - const seenExactLines = new Set(); - const workflowStructuralValues = new Map(); - const workflowSemanticLabels = new Set(); - - const rememberLine = (line: string) => { - const normalizedLine = normalizePromptLineForCompare(line); - if (!normalizedLine) { - return; - } - seenExactLines.add(normalizedLine); - }; - - const normalizedWorkflowLines = normalizeAgentWorkflowPrompt(params.workflowPrompt || "") - .split("\n") - .map((line) => line.trim()) - .filter(Boolean); - normalizedWorkflowLines.forEach((line) => { - const parsed = parsePromptLineLabel(line); - if (parsed.label) { - workflowSemanticLabels.add(getPromptSemanticLabel(parsed.label)); - } - collectPromptSemanticLabelsFromText(line).forEach((semantic) => { - workflowSemanticLabels.add(semantic); - }); - collectPromptSemanticLabelsFromText(parsed.value || "").forEach((semantic) => { - workflowSemanticLabels.add(semantic); - }); - if (parsed.label && AGENT_PROMPT_STRUCTURAL_LABELS.has(parsed.label)) { - workflowStructuralValues.set( - parsed.label, - normalizePromptLineForCompare(parsed.value || "") - ); - } - rememberLine(line); - }); - - const normalizedSlotLines = String(params.slotPrompt || "") - .split(/\n+/) - .map((line) => line.replace(/\s+/g, " ").trim()) - .filter(Boolean); - let activeSlotSemanticLabel = ""; - normalizedSlotLines.forEach((line) => { - const normalizedLine = normalizePromptLineForCompare(line); - if (!normalizedLine || seenExactLines.has(normalizedLine)) { - return; - } - const parsed = parsePromptLineLabel(line); - const slotSemanticLabel = parsed.label ? getPromptSemanticLabel(parsed.label) : ""; - if (slotSemanticLabel) { - activeSlotSemanticLabel = slotSemanticLabel; - } - if ( - params.modeType === "scene_generation" && - shouldDropSceneSlotSemanticLine( - slotSemanticLabel || activeSlotSemanticLabel, - workflowSemanticLabels, - ) - ) { - return; - } - if (parsed.label && AGENT_PROMPT_STRUCTURAL_LABELS.has(parsed.label)) { - const normalizedValue = normalizePromptLineForCompare(parsed.value || ""); - const workflowValue = workflowStructuralValues.get(parsed.label); - if (workflowValue && workflowValue === normalizedValue) { - return; - } - } - seenExactLines.add(normalizedLine); - dedupedSlotLines.push(line); - }); - - return dedupedSlotLines.join("\n").trim(); -} - -function buildAgentSlotPrompt(params: { - workflowPrompt?: string | null; - slotPrompt: string; - slotTitle?: string | null; - userSupplementPrompt?: string | null; - includeSlotTitle?: boolean; - modeType?: "scene_generation" | "detail_replacement" | "sku_replacement" | "generic"; -}) { - const lines: string[] = []; - const workflowPrompt = normalizeAgentWorkflowPrompt(params.workflowPrompt || ""); - if (workflowPrompt) { - lines.push(workflowPrompt); - } - const slotTitle = String(params.slotTitle || "").trim(); - if (params.includeSlotTitle !== false && slotTitle) { - lines.push(`【槽位】${slotTitle}`); - } - const dedupedSlotPrompt = dedupeAgentSlotPromptAgainstWorkflow({ - workflowPrompt, - slotPrompt: params.slotPrompt, - modeType: params.modeType || "generic", - }); - if (dedupedSlotPrompt) { - lines.push(dedupedSlotPrompt); - } - const supplement = String(params.userSupplementPrompt || "").trim(); - if (supplement) { - lines.push(`【补充要求】${supplement}`); - } - return lines.join("\n\n").trim(); -} - -function buildAgentSceneWorkflowPrompt(params: { - workflowPrompt?: string | null; - slotPrompt?: string | null; - userSupplementPrompt?: string | null; - imageRolePrompt?: string | null; -}) { - const lines: string[] = []; - const workflowPrompt = normalizeAgentWorkflowPrompt(params.workflowPrompt || ""); - if (workflowPrompt) { - lines.push(workflowPrompt); - } - const slotPrompt = String(params.slotPrompt || "") - .split(/\n+/) - .map((line) => line.replace(/\s+/g, " ").trim()) - .filter(Boolean) - .join("\n") - .trim(); - if (slotPrompt) { - lines.push(slotPrompt); - } - let imageRolePrompt = String(params.imageRolePrompt || "").trim(); - if (workflowPrompt.includes("品牌模特图：已选") || workflowPrompt.includes("固定模特图")) { - imageRolePrompt = imageRolePrompt - .replace( - /^如当前请求未附带固定模特图，仅在工作流明确要求人物时再安排辅助人物，且人物不能喧宾夺主。$/m, - "" - ) - .replace(/^当前没有固定模特图，不要凭空生成强人物主体。$/m, "") - .split(/\n+/) - .map((line) => line.trim()) - .filter(Boolean) - .join("\n"); - } - if (imageRolePrompt) { - lines.push(imageRolePrompt); - } - const supplement = String(params.userSupplementPrompt || "").trim(); - if (supplement) { - lines.push(`【补充要求】${supplement}`); - } - return lines.join("\n\n").trim(); -} - -type AgentReplacementModePayload = NonNullable; -type AgentModeSourceImagePayload = NonNullable; -type AgentDetailReplacementStrategy = "scene_replace" | "product_only"; - -type NormalizedAgentModeSourceImage = { - url: string; - label: string; - name: string; - assetId: string | null; -}; - -type ReplacementReferenceDecision = { - shouldReplace: boolean; - containsFurnitureProduct: boolean; - reason: string; - detectionModelName?: string | null; - detectionSource: - | "local_moonshot" - | "configured_vision" - | "configured_qwen3_vl" - | "fallback_qwen3_vl" - | "safe_passthrough"; -}; - -type ReplacementScenePlan = { - useModelImage: boolean; - modelImageReason: string; - sceneSummary: string; - cameraSummary: string; - placementSummary: string; - preserveSummary: string; - styleSummary: string; - avoidSummary: string; - generationPrompt: string; - selectedProductImageIndexes: number[]; - productSelectionReason: string; - analysisModelName?: string | null; - analysisSource: "local_moonshot" | "configured_vision" | "heuristic"; -}; - -type ReplacementExecutionPlan = ReplacementReferenceDecision & { - scenePlan: ReplacementScenePlan; -}; - -type ReplacementScenePlanningRuntime = - | { - kind: "local_moonshot"; - model: any; - modelName: string; - } - | { - kind: "configured_vision"; - model: any; - modelName: string; - } - | null; - -const QWEN_VL_MODEL_MATCHER = /(qwen|千问|通义).*(vl|vision)|qwen3-vl|qwen-vl/i; - -function normalizeDetailReplacementStrategy( - value: unknown, -): AgentDetailReplacementStrategy { - return value === "product_only" ? "product_only" : "scene_replace"; -} - -function truncateReplacementAnalysisText(value: unknown, maxLength = 240) { - return String(value || "").trim().replace(/\s+/g, " ").slice(0, maxLength); -} - -function parseReplacementScenePlan( - text: string, - params: { - productImageCount: number; - hasModelImage: boolean; - }, -): ReplacementScenePlan | null { - const jsonText = extractJsonObjectText(text); - if (!jsonText) { - return null; - } - - try { - const parsed = JSON.parse(jsonText) as Record; - const useModelImageRaw = - normalizeBooleanLike(parsed.useModelImage) ?? - normalizeBooleanLike(parsed.needModelImage) ?? - normalizeBooleanLike(parsed.referenceNeedsModel) ?? - false; - - const generationPrompt = truncateReplacementAnalysisText( - parsed.generationPrompt || parsed.recommendedScenePrompt || parsed.scenePrompt, - 1200, - ); - - const scenePlan: ReplacementScenePlan = { - useModelImage: params.hasModelImage ? Boolean(useModelImageRaw) : false, - modelImageReason: truncateReplacementAnalysisText( - parsed.modelImageReason || parsed.modelReason, - 160, - ), - sceneSummary: truncateReplacementAnalysisText( - parsed.sceneSummary || parsed.sceneDescription, - 220, - ), - cameraSummary: truncateReplacementAnalysisText( - parsed.cameraSummary || parsed.cameraDescription, - 180, - ), - placementSummary: truncateReplacementAnalysisText( - parsed.placementSummary || parsed.productPlacement, - 180, - ), - preserveSummary: truncateReplacementAnalysisText( - parsed.preserveSummary || parsed.mustKeep || parsed.preserveInstruction, - 220, - ), - styleSummary: truncateReplacementAnalysisText( - parsed.styleSummary || parsed.styleInstruction || parsed.styleKeywords, - 220, - ), - avoidSummary: truncateReplacementAnalysisText( - parsed.avoidSummary || parsed.doNotInclude || parsed.avoidInstruction, - 220, - ), - generationPrompt, - selectedProductImageIndexes: Array.from( - { length: Math.min(Math.max(params.productImageCount, 1), 2) }, - (_, index) => index + 1, - ), - productSelectionReason: - params.productImageCount > 1 - ? "默认优先使用前两张产品图作为主视角与补充细节，其余角度不强制同时融入。" - : "默认使用当前唯一产品图作为主体参考。", - analysisModelName: null, - analysisSource: "heuristic", - }; - - if (!scenePlan.generationPrompt) { - scenePlan.generationPrompt = [ - scenePlan.sceneSummary, - scenePlan.cameraSummary, - scenePlan.placementSummary, - scenePlan.styleSummary, - ] - .filter(Boolean) - .join("；") - .slice(0, 1200); - } - - return scenePlan; - } catch (error) { - console.warn("[Chat API] Failed to parse replacement scene plan JSON:", error); - return null; - } -} - -function buildHeuristicReplacementScenePlan(params: { - productImageCount: number; - hasModelImage: boolean; - shouldReplace: boolean; - reason: string; - replacementStrategy?: AgentDetailReplacementStrategy; -}): ReplacementScenePlan { - const isProductOnly = params.replacementStrategy === "product_only"; - return { - selectedProductImageIndexes: Array.from( - { length: Math.min(Math.max(params.productImageCount, 1), 2) }, - (_, index) => index + 1, - ), - productSelectionReason: - params.productImageCount > 1 - ? "默认优先使用前两张产品图作为主视角与补充细节，其余角度不强制同时融入。" - : "默认使用当前唯一产品图作为主体参考。", - useModelImage: false, - modelImageReason: params.hasModelImage - ? isProductOnly - ? "仅替换产品模式不使用模特图，默认锁定参考图原有人物与场景。" - : "前置分析未明确当前参考图存在人物展示需求，兜底阶段默认不使用模特图。" - : "当前未提供模特图。", - sceneSummary: params.shouldReplace - ? isProductOnly - ? "严格保留参考图的场景、文案和版式，只把产品展示区中的旧产品替换成输入产品。" - : "保留参考图版式骨架、构图和主体位置，但把旧场景视觉内容按工作流目标风格重建。" - : "当前参考图不满足替换条件。", - cameraSummary: "保持参考图原有机位、构图和景别，不做整体重画。", - placementSummary: isProductOnly - ? "产品必须落在参考图原有产品所在区域，保持原位置、透视和占比。" - : "产品应落在参考图当前主体所在区域，并围绕该区域同步重建新的场景内容。", - preserveSummary: isProductOnly - ? "除产品主体外，文字排版、标题区、分割线、留白、背景、灯光和装饰都必须保留。" - : "文字排版、标题区、分割线、留白和信息层级必须保留。", - styleSummary: isProductOnly - ? "延续参考图原有风格、背景和氛围，不引入新的场景改造要求。" - : "除版式骨架外，墙面、地面、背景家具、软装、灯光、摆件和空间氛围都要按工作流风格重建。", - avoidSummary: isProductOnly - ? "不要改动背景、文案、按钮、图标、排版和装饰，不要重建整个场景，不要把多个产品角度同时塞进一张图。" - : "不要只换产品不换场景，不要把参考图原有背景物件和空间氛围原样保留下来，不要把所有输入产品角度图同时强行出现在一张图中。", - generationPrompt: params.shouldReplace - ? isProductOnly - ? `仅替换参考图中的产品主体，严格保留其余背景、文案、排版与光影关系。${params.reason}` - : `以工作流目标风格重建当前这一屏的场景，只保留版式、机位、文字区与留白结构。${params.reason}` - : params.reason, - analysisModelName: null, - analysisSource: "heuristic", - }; -} - -function normalizeAgentModeReferences( - payload?: PostRequestBody["agentModePayload"], -): AgentReplacementModePayload["references"] { - if (!payload?.references?.length) return []; - return payload.references - .map((item) => ({ - id: String(item.id || "").trim(), - label: String(item.label || "").trim() || "参考图", - imageUrl: String(item.imageUrl || "").trim(), - candidateId: String(item.candidateId || "").trim() || null, - sortOrder: Number.isFinite(Number(item.sortOrder)) - ? Math.max(0, Math.min(999, Math.round(Number(item.sortOrder)))) - : 0, - passthrough: item.passthrough === true, - assignedProductIndex: Number.isFinite(Number(item.assignedProductIndex)) - ? Math.max(0, Math.min(9, Math.round(Number(item.assignedProductIndex)))) - : null, - })) - .filter((item) => item.id && /^https?:\/\//i.test(item.imageUrl)) - .sort((left, right) => left.sortOrder - right.sortOrder); -} - -function normalizeAgentModeSourceImage(value: unknown): NormalizedAgentModeSourceImage | null { - const record = - value && typeof value === "object" && !Array.isArray(value) - ? (value as Record) - : {}; - const url = String(record.url || "").trim(); - if (!/^https?:\/\//i.test(url)) { - return null; - } - - const label = String(record.label || record.name || "参考图").trim().slice(0, 255); - const name = String(record.name || record.label || label || "参考图").trim().slice(0, 255); - const assetId = String(record.assetId || "").trim() || null; - - return { - url, - label: label || name || "参考图", - name: name || label || "参考图", - assetId, - }; -} - -function normalizeAgentModeSourceImages( - payload?: PostRequestBody["agentModePayload"], -): { - productImages: NormalizedAgentModeSourceImage[]; - modelImage: NormalizedAgentModeSourceImage | null; -} { - const sourceImages = payload?.sourceImages as AgentModeSourceImagePayload | undefined; - const productImages = Array.isArray(sourceImages?.productImages) - ? sourceImages.productImages - .map((item) => normalizeAgentModeSourceImage(item)) - .filter((item): item is NormalizedAgentModeSourceImage => Boolean(item)) - .slice(0, 16) - : []; - - return { - productImages, - modelImage: normalizeAgentModeSourceImage(sourceImages?.modelImage), - }; -} - -function normalizeAgentModeDetailReplacementStrategy( - payload?: PostRequestBody["agentModePayload"], -): AgentDetailReplacementStrategy { - return normalizeDetailReplacementStrategy(payload?.replacementStrategy); -} - -function resolveImagePartsByReferenceExclusion(params: { - uploadedImageParts: AgentPresetMessageImagePart[]; - referenceImageUrls: string[]; -}) { - const referenceUrlSet = new Set(params.referenceImageUrls.map((url) => String(url || "").trim())); - const modelPart = params.uploadedImageParts.find((part) => /模特|model/i.test(String(part.name || ""))); - const productParts = params.uploadedImageParts.filter((part) => { - const name = String(part.name || "").trim(); - if (referenceUrlSet.has(part.url)) return false; - if (/^详情参考图|^SKU参考图/i.test(name)) return false; - if (modelPart && part.url === modelPart.url) return false; - return true; - }); - - return { productParts, modelPart: modelPart || null }; -} - -async function resolveDirectReplacementModelRef(params: { - selectedChatModel?: string | null; - finalModelId?: string | null; - models?: Awaited>; -}) { - const explicitModel = String(params.selectedChatModel || "").trim(); - if (explicitModel && explicitModel !== "auto") { - return explicitModel; - } - const routedModel = String(params.finalModelId || "").trim(); - if (routedModel && routedModel !== "auto") { - return routedModel; - } - - const models = params.models || (await getEnabledAiModelsWithProvider(false)); - const imageModels = models.filter( - ({ model, provider }) => - model.supportsImageGen && - !model.isSubModel && - !isVideoProviderType(provider?.type, model), - ); - const preferred = - imageModels.find(({ model }) => model.isDefault) || - imageModels.find(({ provider }) => String(provider?.type || "").includes("image")) || - imageModels[0]; - - return preferred?.model.id || explicitModel || routedModel || "auto"; -} - -function normalizeBooleanLike(value: unknown): boolean | null { - if (typeof value === "boolean") { - return value; - } - - const normalized = String(value || "") - .trim() - .toLowerCase(); - if (!normalized) { - return null; - } - - if ( - normalized === "true" || - normalized === "yes" || - normalized === "y" || - normalized === "1" || - normalized === "是" - ) { - return true; - } - - if ( - normalized === "false" || - normalized === "no" || - normalized === "n" || - normalized === "0" || - normalized === "否" - ) { - return false; - } - - return null; -} - -function extractJsonObjectText(text: string): string | null { - const normalized = String(text || "").trim(); - if (!normalized) { - return null; - } - - const fencedMatch = normalized.match(/```(?:json)?\s*([\s\S]*?)```/i); - if (fencedMatch?.[1]) { - const fencedContent = fencedMatch[1].trim(); - if (fencedContent.startsWith("{") && fencedContent.endsWith("}")) { - return fencedContent; - } - } - - if (normalized.startsWith("{") && normalized.endsWith("}")) { - return normalized; - } - - const firstBraceIndex = normalized.indexOf("{"); - const lastBraceIndex = normalized.lastIndexOf("}"); - if (firstBraceIndex >= 0 && lastBraceIndex > firstBraceIndex) { - return normalized.slice(firstBraceIndex, lastBraceIndex + 1); - } - - return null; -} - -function parseReplacementReferenceDecision( - text: string, -): Omit | null { - const jsonText = extractJsonObjectText(text); - if (jsonText) { - try { - const parsed = JSON.parse(jsonText) as Record; - const shouldReplace = - normalizeBooleanLike(parsed.shouldReplace) ?? - normalizeBooleanLike(parsed.replace) ?? - normalizeBooleanLike(parsed.isReplaceable); - const containsFurnitureProduct = - normalizeBooleanLike(parsed.containsFurnitureProduct) ?? - normalizeBooleanLike(parsed.hasFurnitureProduct) ?? - shouldReplace; - const reason = String(parsed.reason || parsed.summary || parsed.note || "") - .trim() - .slice(0, 80); - - if (typeof shouldReplace === "boolean") { - return { - shouldReplace, - containsFurnitureProduct: Boolean(containsFurnitureProduct), - reason: - reason || - (shouldReplace - ? "检测到可替换的家具主体，继续执行替换" - : "未检测到可替换的家具主体，直接沿用参考图"), - }; - } - } catch (error) { - console.warn("[Chat API] Failed to parse replacement reference decision JSON:", error); - } - } - - const normalized = String(text || "") - .replace(/\s+/g, " ") - .trim(); - if (!normalized) { - return null; - } - - const shouldReplace = /shouldreplace["']?\s*[:=]\s*true|可替换|应该替换|存在可替换|检测到.*家具主体|yes/i.test( - normalized, - ); - const shouldKeep = /shouldreplace["']?\s*[:=]\s*false|不应替换|不要替换|沿用参考图|未检测到.*家具主体|无可替换|not replace|no/i.test( - normalized, - ); - - if (!shouldReplace && !shouldKeep) { - return null; - } - - return { - shouldReplace, - containsFurnitureProduct: shouldReplace, - reason: normalized.slice(0, 80), - }; -} - -async function analyzeReplacementReferenceImage(params: { - imageUrl: string; - models?: Awaited>; - modeType?: "detail_replacement" | "sku_replacement"; - replacementStrategy?: AgentDetailReplacementStrategy; -}): Promise { - const isDetailReplacement = params.modeType === "detail_replacement"; - const isProductOnlyDetailReplacement = - isDetailReplacement && params.replacementStrategy === "product_only"; - const detectionPrompt = isProductOnlyDetailReplacement - ? [ - "你是电商详情图产品主体替换判定器。", - "请判断这张详情参考图里，是否存在清晰、主要、适合被新产品替换的商品主体、产品展示区或产品照片区域。", - "你的任务不是重建整张场景，也不是替换文字排版；只判断是否有明确的产品替换目标。", - "文字、标题、卖点标签、按钮、UI图标、分割线、品牌标识、留白和背景装饰不属于替换对象。", - "只有当图片几乎是纯文字页、参数表、品牌介绍页、没有明确产品展示区时，shouldReplace 才能为 false。", - '只输出 JSON，不要输出额外解释。格式：{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字"}', - ].join("\n") - : isDetailReplacement - ? [ - "你是电商详情参考图背景替换判定器。", - "请判断这张图片里，是否存在适合被重建/替换的背景图、产品场景图、商品主体图或大面积视觉图片区。", - "这不是只找家具主体：只要存在照片、插画、场景背景、产品展示图区、人物展示图区，都可以 shouldReplace=true。", - "必须保留的文字、标题、卖点标签、UI图标、按钮、分割线、品牌标识、留白和排版不属于替换对象。", - "只有当图片几乎是纯文字页、参数表、品牌介绍页、无明显背景图/场景图区/商品图片区时，shouldReplace 才能为 false。", - '只输出 JSON，不要输出额外解释。格式：{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字"}', - ].join("\n") - : [ - "你是家具商品参考图替换判定器。", - "请判断这张图片里，是否存在清晰、主要、适合被产品替换流程替换的家具商品主体。", - "家具主体包含：床、床垫、沙发、椅子、餐桌、茶几、柜子、电视柜、斗柜、床头柜、书桌等。", - "只有当家具商品是主要视觉主体、轮廓清晰、不是单纯环境陪衬时，shouldReplace 才能为 true。", - "以下情况必须判定 shouldReplace=false：纯文字页、参数说明页、品牌介绍页、材质/五金局部特写、只有模特没有家具、只有空间环境没有明确家具主体、家具很小只是背景、主体过多无法明确替换对象。", - '只输出 JSON，不要输出额外解释。格式：{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字"}', - ].join("\n"); - - const configuredModels = params.models || (await getEnabledAiModelsWithProvider()); - const configuredQwenVisionModel = configuredModels.find( - ({ model }) => - model.supportsVision && - !model.isSubModel && - QWEN_VL_MODEL_MATCHER.test(`${model.displayName} ${model.modelId}`), - ); - - if (configuredQwenVisionModel) { - try { - const languageModel = await getLanguageModelById(configuredQwenVisionModel.model.id); - if (languageModel) { - const { text } = await generateText({ - model: languageModel, - messages: [ - { - role: "user", - content: [ - { - type: "text", - text: detectionPrompt, - }, - { - type: "image", - image: params.imageUrl, - }, - ], - }, - ], - temperature: 0, - maxTokens: 160, - abortSignal: AbortSignal.timeout(45_000), - } as any); - - const parsed = parseReplacementReferenceDecision(text); - if (parsed) { - return { - ...parsed, - detectionModelName: - configuredQwenVisionModel.model.displayName || - configuredQwenVisionModel.model.modelId, - detectionSource: "configured_qwen3_vl", - }; - } - } - } catch (error) { - console.warn("[Chat API] Configured Qwen-VL reference analysis failed:", error); - } - } - - try { - const { callQwen3Vl } = await import("@/lib/ai/qwen3-vl"); - const fallbackResult = await callQwen3Vl({ - imageUrl: params.imageUrl, - prompt: detectionPrompt, - temperature: 0, - maxTokens: 160, - timeoutMs: 45_000, - retries: 1, - }); - const parsed = parseReplacementReferenceDecision(fallbackResult.text); - if (parsed) { - return { - ...parsed, - detectionModelName: fallbackResult.model, - detectionSource: "fallback_qwen3_vl", - }; - } - } catch (error) { - console.warn("[Chat API] Fallback Qwen3-VL reference analysis failed:", error); - } - - return { - shouldReplace: false, - containsFurnitureProduct: false, - reason: "视觉判断失败，为避免误替换，直接沿用参考图", - detectionModelName: null, - detectionSource: "safe_passthrough", - }; -} - -async function resolveReplacementScenePlanningRuntime(params: { - models?: Awaited>; -}): Promise { - try { - const siteSettings = await getSiteSettings().catch(() => null); - const localDetailAgentResolved = await resolveDetailAgentLocalMoonshotModel({ - siteSettingModelId: siteSettings?.canvasDetailAgentModelId, - }); - if (localDetailAgentResolved) { - return { - kind: "local_moonshot", - model: localDetailAgentResolved.model, - modelName: - localDetailAgentResolved.modelRef.displayName || - localDetailAgentResolved.modelRef.modelId, - }; - } - } catch (error) { - console.warn("[Chat API] Failed to resolve local Moonshot scene planning model:", error); - } - - const configuredModels = params.models || (await getEnabledAiModelsWithProvider()); - const configuredVisionModel = - configuredModels.find( - ({ model }) => - model.supportsVision && - !model.isSubModel && - QWEN_VL_MODEL_MATCHER.test(`${model.displayName} ${model.modelId}`), - ) || - configuredModels.find(({ model }) => model.supportsVision && !model.isSubModel); - - if (!configuredVisionModel) { - return null; - } - - try { - const languageModel = await getLanguageModelById(configuredVisionModel.model.id); - if (!languageModel) { - return null; - } - return { - kind: "configured_vision", - model: languageModel, - modelName: - configuredVisionModel.model.displayName || - configuredVisionModel.model.modelId, - }; - } catch (error) { - console.warn("[Chat API] Failed to resolve configured replacement scene planning model:", { - modelId: configuredVisionModel.model.id, - error, - }); - return null; - } -} - -async function analyzeReplacementExecutionPlan(params: { - modeType?: "detail_replacement" | "sku_replacement"; - replacementStrategy?: AgentDetailReplacementStrategy; - referenceImageUrl: string; - referenceLabel: string; - productImageUrls: string[]; - modelImageUrl?: string | null; - workflowPrompt?: string | null; - planningRuntime: ReplacementScenePlanningRuntime; - models?: Awaited>; -}): Promise { - const isDetailReplacement = params.modeType === "detail_replacement"; - const isProductOnlyDetailReplacement = - isDetailReplacement && params.replacementStrategy === "product_only"; - if (params.planningRuntime) { - try { - const replacementScenePlanningTimeoutMs = - params.planningRuntime.kind === "local_moonshot" ? 90_000 : 45_000; - const analysisImageUrls = [params.referenceImageUrl]; - const { inlineDataUrls } = await inlineVisionImages(analysisImageUrls, { - maxImages: 1, - maxSide: 900, - timeoutMs: 12000, - retries: 2, - loggerPrefix: "[Chat API] Replacement scene plan", - }).catch((error) => { - console.warn("[Chat API] Failed to inline replacement scene planning images:", error); - return { inlineDataUrls: [], skipped: [] }; - }); - const effectiveVisionImages = - inlineDataUrls.length > 0 ? inlineDataUrls : analysisImageUrls; - const workflowPromptExcerpt = isProductOnlyDetailReplacement - ? "" - : String(params.workflowPrompt || "") - .trim() - .replace(/\s+/g, " ") - .slice(0, 2200); - const analysisPrompt = [ - isProductOnlyDetailReplacement - ? "你是电商详情参考图中的“产品主体替换规划器”。" - : isDetailReplacement - ? "你是电商详情参考图背景替换流程中的“版式与背景区域规划器”。" - : "你是电商详情替换流程中的“场景理解规划器”。", - `当前要分析的参考图：${params.referenceLabel}`, - "你现在只分析这一张参考图本身，不分析产品图，不分析模特图。", - "产品图和模特图会在后续生图阶段单独提供给生图模型使用，你这里不要尝试理解它们的具体外观，也不要要求必须使用所有输入图。", - params.modelImageUrl && !isProductOnlyDetailReplacement - ? "你只需要根据当前参考图判断：这一屏是否存在真实的人物展示需求；如果没有人物需求，useModelImage 必须为 false。" - : "本轮没有模特图，useModelImage 必须为 false。", - params.modelImageUrl && !isProductOnlyDetailReplacement - ? "如果当前参考图存在人物展示需求，后续生图必须尽量复用提供模特图中的同一人物身份，不要明显换脸、换发型、换体型。" - : "", - "这张图片是唯一的详情参考图底板，它决定当前这一屏的版式、文字排版、UI图标、按钮、机位、构图、留白和信息层级。", - isProductOnlyDetailReplacement - ? "这张参考图要求最大程度原样保留。后续生图只允许替换其中的产品主体/产品展示区域，背景、文案、按钮、图标、分割线、品牌标识、留白、装饰和光影氛围都必须保留。" - : isDetailReplacement - ? "这张参考图不是要求原样保留的场景照片。后续生图只允许替换/重建背景图、产品场景图、商品主体图或人物展示图区域；文字、标题、卖点标签、UI图标、按钮、分割线、品牌标识、留白和排版必须保留。" - : "这张参考图不是要求原样保留的场景照片。原图里的沙发、床、柜体、墙面、地面、窗帘、灯具、摆件、桌面道具、背景家具和环境光氛围，都可以在后续生图阶段按前置工作流风格重建。", - "请完成以下任务：", - isProductOnlyDetailReplacement - ? "1. 判断 shouldReplace：当前参考图里是否存在清晰、主要、适合被新产品替换的商品主体、产品照片区域或产品展示区。纯文字/纯参数/纯品牌介绍且没有产品展示区时才为 false。" - : isDetailReplacement - ? "1. 判断 shouldReplace：当前参考图里是否存在适合被替换/重建的背景图、产品场景图、商品主体图、人物展示图或大面积视觉图片区。纯文字/纯参数/纯品牌介绍且没有视觉图片区时才为 false。" - : "1. 判断 shouldReplace：当前参考图里是否存在清晰、主要、适合被替换的家具商品主体。", - "2. 判断 useModelImage，并说明原因。", - "3. 提炼 sceneSummary、cameraSummary、placementSummary、preserveSummary、styleSummary、avoidSummary。", - isProductOnlyDetailReplacement - ? "4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，要求只替换产品主体，保留其余背景、文案、排版、装饰和光影完全不变。" - : isDetailReplacement - ? "4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，描述需要替换的背景/场景图片区、产品放置方式、模特使用规则，以及必须保留的文字/UI/图标/排版。" - : "4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，描述这一屏需要的场景、机位、主体位置、周边环境重建方向以及必须保留的内容。", - isProductOnlyDetailReplacement - ? "5. sceneSummary、styleSummary 和 generationPrompt 必须强调‘只替换产品，不动其他内容’，不要引导重建场景。" - : "5. sceneSummary、styleSummary 和 generationPrompt 必须优先描述“按前置工作流重建后的目标场景”，而不是只复述参考图当前已有场景。", - isProductOnlyDetailReplacement - ? "6. avoidSummary 里要明确避免改背景、改文字、改按钮、改图标、改版式、改留白、改装饰、改光影氛围。" - : "6. 如果前置工作流风格与当前参考图原场景冲突，必须保留版式骨架，但放弃原场景物件本身，以前置工作流指定的空间风格、软装、灯光和氛围为准。", - isProductOnlyDetailReplacement - ? "7. 不要根据用户提示词或工作流风格改造场景，只服务于产品主体替换。" - : "7. avoidSummary 里要明确避免“只替换产品、不重建场景”这类结果。", - workflowPromptExcerpt - ? `当前风格与场景限制（来自前置工作流与模式模板）:\n${workflowPromptExcerpt}` - : "", - "只返回严格 JSON，不要任何额外解释。", - `JSON 格式： -{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字","useModelImage":false,"modelImageReason":"不超过40字","sceneSummary":"不超过80字","cameraSummary":"不超过60字","placementSummary":"不超过60字","preserveSummary":"不超过80字","styleSummary":"不超过80字","avoidSummary":"不超过80字","generationPrompt":"不超过300字"}`, - ] - .filter(Boolean) - .join("\n"); - - const { text } = await generateText({ - model: params.planningRuntime.model, - messages: [ - { - role: "user", - content: [ - { - type: "text", - text: analysisPrompt, - }, - ...effectiveVisionImages.map((image) => ({ - type: "image" as const, - image, - })), - ], - }, - ], - system: - "你是高精度商品详情图场景规划器。你只分析当前参考图本身，负责理解这一屏的场景、机位、主体位置和人物需求，不要臆测或依赖其他输入图片。", - temperature: params.planningRuntime.kind === "local_moonshot" ? 1 : 0.1, - maxTokens: 1200, - abortSignal: AbortSignal.timeout(replacementScenePlanningTimeoutMs), - } as any); - - const parsedDecision = parseReplacementReferenceDecision(text); - const parsedScenePlan = parseReplacementScenePlan(text, { - productImageCount: params.productImageUrls.length, - hasModelImage: isProductOnlyDetailReplacement - ? false - : Boolean(params.modelImageUrl), - }); - - if (parsedDecision && parsedScenePlan) { - return { - ...parsedDecision, - detectionModelName: params.planningRuntime.modelName, - detectionSource: - params.planningRuntime.kind === "local_moonshot" - ? "local_moonshot" - : "configured_vision", - scenePlan: { - ...parsedScenePlan, - analysisModelName: params.planningRuntime.modelName, - analysisSource: - params.planningRuntime.kind === "local_moonshot" - ? "local_moonshot" - : "configured_vision", - }, - }; - } - } catch (error) { - console.warn("[Chat API] Replacement scene planning analysis failed:", { - referenceLabel: params.referenceLabel, - referenceImageUrl: params.referenceImageUrl, - error, - }); - } - } - - const fallbackDecision = await analyzeReplacementReferenceImage({ - imageUrl: params.referenceImageUrl, - models: params.models, - modeType: params.modeType, - replacementStrategy: params.replacementStrategy, - }); - return { - ...fallbackDecision, - scenePlan: buildHeuristicReplacementScenePlan({ - productImageCount: params.productImageUrls.length, - hasModelImage: isProductOnlyDetailReplacement - ? false - : Boolean(params.modelImageUrl), - shouldReplace: fallbackDecision.shouldReplace, - reason: fallbackDecision.reason, - replacementStrategy: params.replacementStrategy, - }), - }; -} - -function buildAgentReplacementPrompt(params: { - basePrompt: string; - modeType: "detail_replacement" | "sku_replacement"; - replacementStrategy?: AgentDetailReplacementStrategy; - referenceLabel: string; - referenceIndex: number; - totalReferences: number; - hasModelImage: boolean; - productImageCount: number; - originalProductImageCount: number; - templateReferenceCount: number; - pairedProductIndex?: number; - scenePlan?: ReplacementScenePlan | null; - referenceGeometry?: ReplacementReferenceGeometry | null; -}) { - const imageOrderLines: string[] = []; - let nextImageIndex = 1; - - if (params.productImageCount > 0) { - const productRangeLabel = - params.productImageCount === 1 - ? `图片${nextImageIndex}` - : `图片${nextImageIndex}-${nextImageIndex + params.productImageCount - 1}`; - imageOrderLines.push( - `${productRangeLabel}：同一产品的白底图/细节图，必须综合理解所有产品图，不可只看第一张。` - ); - nextImageIndex += params.productImageCount; - } - - if (params.hasModelImage) { - imageOrderLines.push( - `图片${nextImageIndex}：模特图，仅在当前参考图本身包含人物展示时才允许参考。` - ); - nextImageIndex += 1; - } - - if (params.templateReferenceCount > 0) { - const templateRangeLabel = - params.templateReferenceCount === 1 - ? `图片${nextImageIndex}` - : `图片${nextImageIndex}-${nextImageIndex + params.templateReferenceCount - 1}`; - imageOrderLines.push( - `${templateRangeLabel}：模板补充参考图，只能补充风格/材质氛围，不能覆盖最终参考图版式。` - ); - nextImageIndex += params.templateReferenceCount; - } - - imageOrderLines.push( - `图片${nextImageIndex}：当前参考图底板（唯一版式基准图），必须保留它的文字排版、信息区、分割线、留白、机位和整体结构。` - ); - - const scenePlanLines = params.scenePlan - ? [ - "【本屏场景预分析】", - params.scenePlan.sceneSummary - ? `场景理解：${params.scenePlan.sceneSummary}` - : "", - params.scenePlan.cameraSummary - ? `机位构图：${params.scenePlan.cameraSummary}` - : "", - params.scenePlan.placementSummary - ? `主体位置：${params.scenePlan.placementSummary}` - : "", - params.scenePlan.preserveSummary - ? `必须保留：${params.scenePlan.preserveSummary}` - : "", - params.scenePlan.styleSummary - ? `风格约束：${params.scenePlan.styleSummary}` - : "", - params.scenePlan.avoidSummary - ? `禁止事项：${params.scenePlan.avoidSummary}` - : "", - params.originalProductImageCount > params.productImageCount - ? `原始共有 ${params.originalProductImageCount} 张产品角度图，本屏已筛选出最相关的 ${params.productImageCount} 张输入图；其余未入选产品图只是补充理解，不需要也不允许强行全部融入同一张结果图。` - : "", - params.scenePlan.productSelectionReason - ? `产品图筛选理由：${params.scenePlan.productSelectionReason}` - : "", - params.scenePlan.useModelImage - ? params.scenePlan.modelImageReason - ? `本屏允许使用模特图：${params.scenePlan.modelImageReason}` - : "本屏允许使用模特图，但仅在参考图当前这一屏确实存在人物展示需求时使用。" - : params.hasModelImage - ? params.scenePlan.modelImageReason - ? `本屏禁止使用模特图：${params.scenePlan.modelImageReason}` - : "本屏不需要模特图，不允许把模特图强行融入结果。" - : "", - params.scenePlan.generationPrompt - ? `本屏生成方向：${params.scenePlan.generationPrompt}` - : "", - ].filter(Boolean) - : []; - - const productDetailInstructionLines = - params.replacementStrategy === "product_only" - ? [ - "【执行模式】详情替换 / 仅替换产品", - `当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张详情参考图：${params.referenceLabel}`, - "【图片顺序规则】", - ...imageOrderLines, - ...scenePlanLines, - "你现在做的是‘仅替换产品’，不是重建场景，也不是重新设计整张详情图。", - "忽略用户补充提示词里的场景风格、氛围、空间改造要求；这一次只服务于产品主体替换。", - "最后一张参考图是唯一底板，除产品主体/产品展示区外，其余所有内容都必须保持不变。", - "必须严格保留：背景、人物、手部姿态、桌面摆件、灯光氛围、阴影关系、文字排版、标题、卖点标签、按钮、UI图标、分割线、品牌标识、留白、画布比例和信息层级。", - params.referenceGeometry - ? `当前参考图画布尺寸为 ${params.referenceGeometry.width}x${params.referenceGeometry.height}，输出必须保持这个页面画布比例。替换商品时只能等比缩放、透视匹配和自然融合，绝对禁止把商品、人物、文字、图标横向或纵向拉伸变形。` - : "输出必须保持当前参考图页面画布比例。替换商品时只能等比缩放、透视匹配和自然融合，绝对禁止把商品、人物、文字、图标横向或纵向拉伸变形。", - "镜头角度、透视关系、视平线、主体朝向和主体占比必须尽量跟随最后一张参考图，方便无缝替换当前产品区域。", - params.productImageCount > 1 - ? `前面的 ${params.productImageCount} 张商品输入图都属于同一产品的多角度/多细节白底图，请综合这些图片理解产品结构、材质、颜色、比例、边角、开孔、五金与细节，不可只参考第一张。` - : "前面的商品输入图就是当前要替换进去的产品白底图，请准确保持该产品的结构、材质、颜色、比例和细节。", - "你的任务只有一个：识别参考图中的旧产品主体或产品展示区，用输入产品替换它，并让新产品与原场景自然融合。", - "不要重建背景，不要重画空间，不要调整文案，不要修改按钮、图标、标签、装饰元素，也不要改变人物、道具和环境光氛围。", - "如果参考图中没有清晰可替换的产品主体或产品展示区，应直接沿用参考图，不要硬生成。", - "不要把多个产品角度同时塞进一张结果图，不要新增无关主体，不要删除任何关键版式或 UI 元素。", - ] - : [ - "【执行模式】详情替换", - `当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张详情参考图：${params.referenceLabel}`, - "【图片顺序规则】", - ...imageOrderLines, - ...scenePlanLines, - "你现在做的是“详情参考图背景/场景图替换”，不是重新设计整张详情图。", - "要把最后一张参考图理解为“页面版式底板 + 背景图/场景图区参考”，不是“旧场景照片必须原样保留”。", - "前面“工作流设定”里的场景组合、空间风格、硬装方案、软装方案、灯光氛围、人群设定，必须作为当前这一屏的强制场景改造要求生效。", - "场景风格必须优先执行前面用户提示词与工作流设定中给出的空间风格、材质、灯光和氛围描述，不要被参考图原有旧场景绑死。", - "执行时要基于这些场景组合要求，重建参考图中的背景图/产品场景图区域，使当前这一屏呈现出与第一步已选风格一致的新背景画面。", - "允许替换/重建的范围是参考图里的背景图、产品场景图、商品主体图和必要的人物展示图区；不允许改动版式层。", - "如果前面商品图、模特图与最后一张参考图底板存在冲突，始终以最后一张参考图底板的版式、机位、构图和信息层级为最高优先级。", - "最后一张参考图是唯一底板，必须完整保留这张图的文字排版、品牌字样、标题区、卖点区、UI图标、按钮、标签、装饰元素、分割线、留白、画布比例和信息层级。", - params.referenceGeometry - ? `当前参考图画布尺寸为 ${params.referenceGeometry.width}x${params.referenceGeometry.height}，输出必须保持这个页面画布比例。替换背景/商品时只能等比缩放后裁切或延展背景，绝对禁止把商品、人物、文字、图标横向或纵向拉伸变形。` - : "输出必须保持当前参考图页面画布比例。替换背景/商品时只能等比缩放后裁切或延展背景，绝对禁止把商品、人物、文字、图标横向或纵向拉伸变形。", - "镜头角度、透视关系、视平线、主体朝向和主体占比必须尽量跟随最后一张参考图，方便无缝替换当前版式，不要擅自改成完全不同的视角。", - "前面的商品图和模特图用于生成新的背景图/产品场景图内容，并按工作流设定替换参考图中的旧背景画面，但不能改变参考图的版式设计。", - "可被替换重建的内容包括但不限于：背景照片、产品展示照片、墙面、地面材质、沙发、床、背景柜、窗帘、灯具、地毯、挂画、摆件、桌面道具、空间色调、环境光氛围。", - params.productImageCount > 1 - ? `前面的 ${params.productImageCount} 张商品输入图都属于同一产品的多角度/多细节白底图，请综合这些图片理解产品结构、材质、颜色、比例、边角、开孔、五金与细节，不可只参考第一张。` - : "前面的商品输入图就是当前要替换进去的产品白底图，请准确保持该产品的结构、材质、颜色、比例和细节。", - "请先识别最后一张参考图中需要被替换的背景图/场景图区域，并在同一版式骨架下生成新的背景画面；不要只做局部抠图式换产品。", - "请严格保留最后一张参考图的版式结构、镜头关系、构图、文字排版、UI图标、标签、按钮、留白位置和信息层级；但旧背景图里的家具、背景物件、材质、灯光氛围、软装小物、空间质感需要按第一步工作流设定进行明显重建。", - "如果第一步工作流风格与参考图原场景冲突，必须保留版式骨架和UI层，但放弃原背景图物件本身，以第一步工作流风格为准。", - "前面的输入商品图用于提供商品主体、材质、颜色和细节特征；如有模特图，请只在参考图本身包含人物展示时再参考模特图。", - "如果提供了模特图且这一屏确实有人物展示需求，必须尽量复用同一人物身份与体态关系，不要明显换脸、换发型、换体型，服装风格也尽量保持一致。", - "绝对不要忽略最后一张参考图去生成一张无关的新场景图。", - "如果生成结果仍然保留了参考图原有的大部分背景图物件，只是把产品换掉，则视为执行失败；你必须让背景图/场景图发生清晰可见的变化。", - "绝对不要移动、删除、改写、重排参考图里的任何文字、UI图标、按钮、标签、装饰元素与排版元素。", - "输出结果必须保持最后一张参考图底板的原始纵横比与画布尺寸，不允许擅自改成长图、横图或裁掉任何版面区域。", - "如果参考图中没有可替换/重建的背景图、产品场景图或视觉图片区，应直接沿用参考图，不要硬生成。", - "不要改动参考图的整体布局，不要新增无关主体，不要删除关键版式或UI元素。", - ]; - - const skuInstructionLines = [ - "【执行模式】SKU替换", - `当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张 SKU 参考图：${params.referenceLabel}`, - "【图片顺序规则】", - ...imageOrderLines, - ...scenePlanLines, - params.pairedProductIndex !== undefined - ? `请优先使用第 ${params.pairedProductIndex + 1} 张输入商品图作为本张参考图的一对一替换主体。` - : "请使用输入商品图作为本张参考图的一对一替换主体。", - params.productImageCount > 1 - ? "其余商品输入图同样是这个产品的补充角度/细节参考，请一并综合理解产品信息，不可忽略。" - : "输入商品图就是当前产品主体，请保持产品信息准确一致。", - "如果前面商品图与最后一张参考图存在冲突，始终以最后一张参考图的机位、场景、光影和构图为准。", - "请严格保留最后一张参考图的场景、机位、透视、光影和构图关系，只替换商品主体本身。", - params.referenceGeometry - ? `当前参考图画布尺寸为 ${params.referenceGeometry.width}x${params.referenceGeometry.height}，输出必须保持这个画布比例。替换商品时只能等比缩放、透视匹配和自然融合，禁止把商品横向或纵向拉伸变形。` - : "输出必须保持当前参考图画布比例。替换商品时只能等比缩放、透视匹配和自然融合，禁止把商品横向或纵向拉伸变形。", - "输出结果必须保持最后一张参考图的原始纵横比与画布尺寸。", - "输出要自然融合，不要改变参考图原有风格和景别。", - ]; - - const instructionLines = - params.modeType === "detail_replacement" - ? productDetailInstructionLines - : skuInstructionLines; - - if (params.hasModelImage) { - instructionLines.push("如有模特图，请在需要人物展示时保持模特体态、穿搭和出镜关系一致。"); - } - - return [ - params.modeType === "detail_replacement" && - params.replacementStrategy === "product_only" - ? "" - : params.basePrompt, - instructionLines.join("\n"), - ] - .filter(Boolean) - .join("\n\n") - .trim(); -} - -const MODEL_UUID_REGEX = - /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; - -function getModelCreditCost( - modelConfig: any, - target: "image" | "video", - options?: { - duration?: number | null; - imageSize?: string | null; - resolution?: string | null; - } -): number { - return getModelGenerationCreditCost(modelConfig, target, options); -} - -async function resolveBillingModelByReference(modelRef: string): Promise { - if (!modelRef) return undefined; - - if (MODEL_UUID_REGEX.test(modelRef)) { - return await getAiModelById({ id: modelRef }).catch(() => undefined); - } - - const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []); - const matched = allModels.find( - ({ model }: any) => model.id === modelRef || model.modelId === modelRef - ); - return matched?.model; -} - -async function consumeGeneratedVideoPoints(params: { - userId: string; - chatId: string; - messageId: string; - modelRefs: Array; - duration?: number | null; - resolution?: string | null; -}) { - try { - const pointsConfig = await getPointsSystemConfig(); - if (!pointsConfig.enabled) return; - - const userData = await getUserById({ id: params.userId }).catch(() => undefined); - if (userData?.isHighVip) { - console.log(`[Points] Skip video points for VIP user=${params.userId}`); - return; - } - - let billingModel: any | undefined; - for (const ref of params.modelRefs) { - if (!ref) continue; - const resolved = await resolveBillingModelByReference(ref); - if (!resolved) continue; - billingModel = resolved; - if ( - getModelCreditCost(resolved, "video", { - duration: params.duration, - resolution: params.resolution, - }) > 0 - ) { - break; - } - } - - if (!billingModel) return; - - const unitCost = getModelCreditCost(billingModel, "video", { - duration: params.duration, - resolution: params.resolution, - }); - if (unitCost <= 0) return; - - const consumeResult = await consumeUserPoints({ - userId: params.userId, - amount: unitCost, - reason: "video_generation", - modelId: billingModel.id, - chatId: params.chatId, - messageId: params.messageId, - note: `模型(${billingModel.displayName || billingModel.modelId || billingModel.id}) 生视频 1 条${params.duration ? `（${params.duration} 秒）` : ""}${params.resolution ? `（${params.resolution}）` : ""}`, - }); - - if (!consumeResult.ok) { - console.warn( - `[Points] Failed to consume video points. user=${params.userId} model=${billingModel.id} cost=${unitCost}` - ); - } else if (consumeResult.skipped) { - console.log( - `[Points] Skipped ${unitCost} video points for VIP user=${params.userId} model=${billingModel.id}` - ); - } else { - console.log( - `[Points] Consumed ${unitCost} points for video. balance=${consumeResult.balance}` - ); - } - } catch (error) { - console.error("[Points] Failed to consume points after video generation:", error); - } -} - export const maxDuration = 600; // 10 分钟，用于支持图片编辑等耗时任务 let globalStreamContext: ResumableStreamContext | null = null;