diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 16b322eda..37a3c9864 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -44,6 +44,8 @@ import { RmbgServiceError, } from "@/lib/ai/rmbg/service"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; +import { resolveDetailAgentLocalMoonshotModel } from "@/lib/ai/detail-agent-model-runtime"; +import { inlineVisionImages } from "@/lib/ai/vision-inline"; import { createDocument } from "@/lib/ai/tools/create-document"; import { getWeather } from "@/lib/ai/tools/get-weather"; import { requestSuggestions } from "@/lib/ai/tools/request-suggestions"; @@ -83,6 +85,7 @@ import { getAccessibleAgentPresetById, getConversationWorkingStateByChatId, getMediaAssetsByUserAndUrls, + getSiteSettings, upsertMediaAsset, } from "@/lib/db/queries"; import { db } from "@/lib/db"; @@ -2897,13 +2900,188 @@ type ReplacementReferenceDecision = { reason: string; detectionModelName?: string | null; detectionSource: + | "local_moonshot" + | "configured_vision" | "configured_qwen3_vl" | "fallback_qwen3_vl" | "safe_passthrough"; }; +type ReplacementScenePlan = { + selectedProductImageIndexes: number[]; + productSelectionReason: string; + useModelImage: boolean; + modelImageReason: string; + sceneSummary: string; + cameraSummary: string; + placementSummary: string; + preserveSummary: string; + styleSummary: string; + avoidSummary: string; + generationPrompt: string; + analysisModelName?: string | null; + analysisSource: "local_moonshot" | "configured_vision" | "heuristic"; +}; + +type ReplacementExecutionPlan = ReplacementReferenceDecision & { + scenePlan: ReplacementScenePlan; +}; + +type ReplacementScenePlanningRuntime = + | { + kind: "local_moonshot"; + model: any; + modelName: string; + } + | { + kind: "configured_vision"; + model: any; + modelName: string; + } + | null; + const QWEN_VL_MODEL_MATCHER = /(qwen|千问|通义).*(vl|vision)|qwen3-vl|qwen-vl/i; +function truncateReplacementAnalysisText(value: unknown, maxLength = 240) { + return String(value || "").trim().replace(/\s+/g, " ").slice(0, maxLength); +} + +function parseReplacementScenePlan( + text: string, + params: { + productImageCount: number; + hasModelImage: boolean; + }, +): ReplacementScenePlan | null { + const jsonText = extractJsonObjectText(text); + if (!jsonText) { + return null; + } + + try { + const parsed = JSON.parse(jsonText) as Record; + const rawIndexes = Array.isArray(parsed.selectedProductImageIndexes) + ? parsed.selectedProductImageIndexes + : Array.isArray(parsed.productImageIndexes) + ? parsed.productImageIndexes + : []; + const selectedProductImageIndexes = Array.from( + new Set( + rawIndexes + .map((value) => Number(value)) + .filter((value) => Number.isFinite(value)) + .map((value) => + Math.max(1, Math.min(params.productImageCount, Math.round(value))), + ), + ), + ).slice(0, 3); + + const useModelImageRaw = + normalizeBooleanLike(parsed.useModelImage) ?? + normalizeBooleanLike(parsed.needModelImage) ?? + normalizeBooleanLike(parsed.referenceNeedsModel) ?? + false; + + const generationPrompt = truncateReplacementAnalysisText( + parsed.generationPrompt || parsed.recommendedScenePrompt || parsed.scenePrompt, + 1200, + ); + + const scenePlan: ReplacementScenePlan = { + selectedProductImageIndexes: + selectedProductImageIndexes.length > 0 + ? selectedProductImageIndexes + : [1].filter((value) => value <= params.productImageCount), + productSelectionReason: truncateReplacementAnalysisText( + parsed.productSelectionReason || parsed.selectionReason || parsed.productReason, + 160, + ), + useModelImage: params.hasModelImage ? Boolean(useModelImageRaw) : false, + modelImageReason: truncateReplacementAnalysisText( + parsed.modelImageReason || parsed.modelReason, + 160, + ), + sceneSummary: truncateReplacementAnalysisText( + parsed.sceneSummary || parsed.sceneDescription, + 220, + ), + cameraSummary: truncateReplacementAnalysisText( + parsed.cameraSummary || parsed.cameraDescription, + 180, + ), + placementSummary: truncateReplacementAnalysisText( + parsed.placementSummary || parsed.productPlacement, + 180, + ), + preserveSummary: truncateReplacementAnalysisText( + parsed.preserveSummary || parsed.mustKeep || parsed.preserveInstruction, + 220, + ), + styleSummary: truncateReplacementAnalysisText( + parsed.styleSummary || parsed.styleInstruction || parsed.styleKeywords, + 220, + ), + avoidSummary: truncateReplacementAnalysisText( + parsed.avoidSummary || parsed.doNotInclude || parsed.avoidInstruction, + 220, + ), + generationPrompt, + analysisModelName: null, + analysisSource: "heuristic", + }; + + if (!scenePlan.generationPrompt) { + scenePlan.generationPrompt = [ + scenePlan.sceneSummary, + scenePlan.cameraSummary, + scenePlan.placementSummary, + scenePlan.styleSummary, + ] + .filter(Boolean) + .join("；") + .slice(0, 1200); + } + + return scenePlan; + } catch (error) { + console.warn("[Chat API] Failed to parse replacement scene plan JSON:", error); + return null; + } +} + +function buildHeuristicReplacementScenePlan(params: { + productImageCount: number; + hasModelImage: boolean; + shouldReplace: boolean; + reason: string; +}): ReplacementScenePlan { + return { + selectedProductImageIndexes: Array.from( + { length: Math.min(Math.max(params.productImageCount, 1), 2) }, + (_, index) => index + 1, + ), + productSelectionReason: + params.productImageCount > 1 + ? "默认优先使用前两张产品图作为主视角与补充细节，其余角度不强制同时融入。" + : "默认使用当前唯一产品图作为主体参考。", + useModelImage: params.hasModelImage, + modelImageReason: params.hasModelImage + ? "未完成场景级人物需求分析，先保留模特图作为可选补充参考。" + : "当前未提供模特图。", + sceneSummary: params.shouldReplace + ? "保持当前参考图版式与场景结构，只替换参考图中的产品主体。" + : "当前参考图不满足替换条件。", + cameraSummary: "保持参考图原有机位、构图和景别，不做整体重画。", + placementSummary: "产品应落在参考图当前主体所在区域，不允许新增无关主体。", + preserveSummary: "文字排版、标题区、分割线、留白和信息层级必须保留。", + styleSummary: "沿用当前工作流风格设定，但只作用于产品相关场景内容。", + avoidSummary: "不要把所有输入产品角度图同时强行出现在一张图中。", + generationPrompt: params.reason, + analysisModelName: null, + analysisSource: "heuristic", + }; +} + function normalizeAgentModeReferences( payload?: PostRequestBody["agentModePayload"], ): AgentReplacementModePayload["references"] { @@ -3141,6 +3319,192 @@ async function analyzeReplacementReferenceImage(params: { }; } +async function resolveReplacementScenePlanningRuntime(params: { + models?: Awaited>; +}): Promise { + try { + const siteSettings = await getSiteSettings().catch(() => null); + const localDetailAgentResolved = await resolveDetailAgentLocalMoonshotModel({ + siteSettingModelId: siteSettings?.canvasDetailAgentModelId, + }); + if (localDetailAgentResolved) { + return { + kind: "local_moonshot", + model: localDetailAgentResolved.model, + modelName: + localDetailAgentResolved.modelRef.displayName || + localDetailAgentResolved.modelRef.modelId, + }; + } + } catch (error) { + console.warn("[Chat API] Failed to resolve local Moonshot scene planning model:", error); + } + + const configuredModels = params.models || (await getEnabledAiModelsWithProvider()); + const configuredVisionModel = + configuredModels.find( + ({ model }) => + model.supportsVision && + !model.isSubModel && + QWEN_VL_MODEL_MATCHER.test(`${model.displayName} ${model.modelId}`), + ) || + configuredModels.find(({ model }) => model.supportsVision && !model.isSubModel); + + if (!configuredVisionModel) { + return null; + } + + try { + const languageModel = await getLanguageModelById(configuredVisionModel.model.id); + if (!languageModel) { + return null; + } + return { + kind: "configured_vision", + model: languageModel, + modelName: + configuredVisionModel.model.displayName || + configuredVisionModel.model.modelId, + }; + } catch (error) { + console.warn("[Chat API] Failed to resolve configured replacement scene planning model:", { + modelId: configuredVisionModel.model.id, + error, + }); + return null; + } +} + +async function analyzeReplacementExecutionPlan(params: { + referenceImageUrl: string; + referenceLabel: string; + productImageUrls: string[]; + modelImageUrl?: string | null; + workflowPrompt?: string | null; + planningRuntime: ReplacementScenePlanningRuntime; + models?: Awaited>; +}): Promise { + if (params.planningRuntime) { + try { + const analysisImageUrls = [ + ...params.productImageUrls, + ...(params.modelImageUrl ? [params.modelImageUrl] : []), + params.referenceImageUrl, + ].filter(Boolean) as string[]; + const { inlineDataUrls } = await inlineVisionImages(analysisImageUrls, { + maxImages: 8, + maxSide: 900, + timeoutMs: 12000, + retries: 2, + loggerPrefix: "[Chat API] Replacement scene plan", + }).catch((error) => { + console.warn("[Chat API] Failed to inline replacement scene planning images:", error); + return { inlineDataUrls: [], skipped: [] }; + }); + const effectiveVisionImages = + inlineDataUrls.length > 0 ? inlineDataUrls : analysisImageUrls; + const workflowPromptExcerpt = String(params.workflowPrompt || "") + .trim() + .replace(/\s+/g, " ") + .slice(0, 2200); + const analysisPrompt = [ + "你是电商详情替换流程中的“场景理解规划器”。", + `当前要分析的参考图：${params.referenceLabel}`, + `产品图数量：${params.productImageUrls.length} 张。产品图全部属于同一产品的多角度/多细节白底图。`, + "这些产品图只是帮助你理解同一产品的结构、材质、颜色、比例和细节，不代表必须把所有角度都同时体现在这一张结果图里。", + "你必须根据最后一张参考图底板当前这一屏的机位、主体位置和露出范围，只选择最需要的 1-2 张产品图作为本屏主参考；只有在确实必要时才允许选择 3 张，禁止无理由把所有产品图都选上。", + params.modelImageUrl + ? "还提供了 1 张模特图。只有当最后一张参考图底板当前这一屏本身存在人物展示需求时，才允许使用模特图；如果这一屏不需要人物，必须明确 useModelImage=false。" + : "本轮没有模特图，useModelImage 必须为 false。", + "最后一张图片是唯一的详情参考图底板，它决定当前这一屏的版式、文字排版、机位、构图、留白和信息层级。", + "请完成以下任务：", + "1. 判断 shouldReplace：当前参考图里是否存在清晰、主要、适合被替换的家具商品主体。", + "2. 给出 selectedProductImageIndexes：从产品图中挑出最适合当前这一屏的产品角度索引（从1开始计数）。", + "3. 判断 useModelImage，并说明原因。", + "4. 提炼 sceneSummary、cameraSummary、placementSummary、preserveSummary、styleSummary、avoidSummary。", + "5. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，必须明确“只按当前参考图这一屏需要使用产品角度图，不需要把所有产品图全部融进结果”。", + workflowPromptExcerpt + ? `当前风格与场景限制（来自前置工作流与模式模板）:\n${workflowPromptExcerpt}` + : "", + "只返回严格 JSON，不要任何额外解释。", + `JSON 格式： +{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字","selectedProductImageIndexes":[1,2],"productSelectionReason":"不超过50字","useModelImage":false,"modelImageReason":"不超过40字","sceneSummary":"不超过80字","cameraSummary":"不超过60字","placementSummary":"不超过60字","preserveSummary":"不超过80字","styleSummary":"不超过80字","avoidSummary":"不超过80字","generationPrompt":"不超过300字"}`, + ] + .filter(Boolean) + .join("\n"); + + const { text } = await generateText({ + model: params.planningRuntime.model, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: analysisPrompt, + }, + ...effectiveVisionImages.map((image) => ({ + type: "image" as const, + image, + })), + ], + }, + ], + system: + "你是高精度商品详情图场景规划器。重点是：多张产品图只是同一产品的多视角信息源，不是必须全部同时使用。你必须根据参考图当前这一屏的真实需求筛选少量最合适的产品图。", + temperature: params.planningRuntime.kind === "local_moonshot" ? 1 : 0.1, + maxTokens: 1200, + abortSignal: AbortSignal.timeout(45_000), + } as any); + + const parsedDecision = parseReplacementReferenceDecision(text); + const parsedScenePlan = parseReplacementScenePlan(text, { + productImageCount: params.productImageUrls.length, + hasModelImage: Boolean(params.modelImageUrl), + }); + + if (parsedDecision && parsedScenePlan) { + return { + ...parsedDecision, + detectionModelName: params.planningRuntime.modelName, + detectionSource: + params.planningRuntime.kind === "local_moonshot" + ? "local_moonshot" + : "configured_vision", + scenePlan: { + ...parsedScenePlan, + analysisModelName: params.planningRuntime.modelName, + analysisSource: + params.planningRuntime.kind === "local_moonshot" + ? "local_moonshot" + : "configured_vision", + }, + }; + } + } catch (error) { + console.warn("[Chat API] Replacement scene planning analysis failed:", { + referenceLabel: params.referenceLabel, + referenceImageUrl: params.referenceImageUrl, + error, + }); + } + } + + const fallbackDecision = await analyzeReplacementReferenceImage({ + imageUrl: params.referenceImageUrl, + models: params.models, + }); + return { + ...fallbackDecision, + scenePlan: buildHeuristicReplacementScenePlan({ + productImageCount: params.productImageUrls.length, + hasModelImage: Boolean(params.modelImageUrl), + shouldReplace: fallbackDecision.shouldReplace, + reason: fallbackDecision.reason, + }), + }; +} + function buildAgentReplacementPrompt(params: { basePrompt: string; modeType: "detail_replacement" | "sku_replacement"; @@ -3149,8 +3513,10 @@ function buildAgentReplacementPrompt(params: { totalReferences: number; hasModelImage: boolean; productImageCount: number; + originalProductImageCount: number; templateReferenceCount: number; pairedProductIndex?: number; + scenePlan?: ReplacementScenePlan | null; }) { const imageOrderLines: string[] = []; let nextImageIndex = 1; @@ -3188,6 +3554,48 @@ function buildAgentReplacementPrompt(params: { `图片${nextImageIndex}：当前参考图底板（唯一版式基准图），必须保留它的文字排版、信息区、分割线、留白、机位和整体结构。` ); + const scenePlanLines = params.scenePlan + ? [ + "【本屏场景预分析】", + params.scenePlan.sceneSummary + ? `场景理解：${params.scenePlan.sceneSummary}` + : "", + params.scenePlan.cameraSummary + ? `机位构图：${params.scenePlan.cameraSummary}` + : "", + params.scenePlan.placementSummary + ? `主体位置：${params.scenePlan.placementSummary}` + : "", + params.scenePlan.preserveSummary + ? `必须保留：${params.scenePlan.preserveSummary}` + : "", + params.scenePlan.styleSummary + ? `风格约束：${params.scenePlan.styleSummary}` + : "", + params.scenePlan.avoidSummary + ? `禁止事项：${params.scenePlan.avoidSummary}` + : "", + params.originalProductImageCount > params.productImageCount + ? `原始共有 ${params.originalProductImageCount} 张产品角度图，本屏已筛选出最相关的 ${params.productImageCount} 张输入图；其余未入选产品图只是补充理解，不需要也不允许强行全部融入同一张结果图。` + : "", + params.scenePlan.productSelectionReason + ? `产品图筛选理由：${params.scenePlan.productSelectionReason}` + : "", + params.scenePlan.useModelImage + ? params.scenePlan.modelImageReason + ? `本屏允许使用模特图：${params.scenePlan.modelImageReason}` + : "本屏允许使用模特图，但仅在参考图当前这一屏确实存在人物展示需求时使用。" + : params.hasModelImage + ? params.scenePlan.modelImageReason + ? `本屏禁止使用模特图：${params.scenePlan.modelImageReason}` + : "本屏不需要模特图，不允许把模特图强行融入结果。" + : "", + params.scenePlan.generationPrompt + ? `本屏生成方向：${params.scenePlan.generationPrompt}` + : "", + ].filter(Boolean) + : []; + const instructionLines = params.modeType === "detail_replacement" ? [ @@ -3195,6 +3603,7 @@ function buildAgentReplacementPrompt(params: { `当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张详情参考图：${params.referenceLabel}`, "【图片顺序规则】", ...imageOrderLines, + ...scenePlanLines, "你现在做的不是重新生成一张新图，而是编辑最后一张参考图本身。", "如果前面商品图、模特图与最后一张参考图底板存在冲突，始终以最后一张参考图底板的版式、机位、构图和信息层级为最高优先级。", "最后一张参考图是唯一底板，必须保留这张图的文字排版、品牌字样、标题区、卖点区、分割线、留白、机位、构图和信息层级。", @@ -3216,6 +3625,7 @@ function buildAgentReplacementPrompt(params: { `当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张 SKU 参考图：${params.referenceLabel}`, "【图片顺序规则】", ...imageOrderLines, + ...scenePlanLines, params.pairedProductIndex !== undefined ? `请优先使用第 ${params.pairedProductIndex + 1} 张输入商品图作为本张参考图的一对一替换主体。` : "请使用输入商品图作为本张参考图的一对一替换主体。", @@ -6150,6 +6560,14 @@ export async function POST(request: Request) { return undefined; }, ); + const replacementScenePlanningRuntime = + await resolveReplacementScenePlanningRuntime({ + models: replacementVisionModels, + }); + console.info("[Chat API] Replacement scene planning runtime:", { + kind: replacementScenePlanningRuntime?.kind || "none", + modelName: replacementScenePlanningRuntime?.modelName || null, + }); const stopKeepalive = startStreamKeepalive( writer, "agent-preset:replacement", @@ -6276,8 +6694,13 @@ export async function POST(request: Request) { ), ] : productInputImageUrls; - const referenceDecision = await analyzeReplacementReferenceImage({ - imageUrl: reference.imageUrl, + const replacementExecutionPlan = await analyzeReplacementExecutionPlan({ + referenceImageUrl: reference.imageUrl, + referenceLabel: reference.label, + productImageUrls: orderedProductContextImageUrls, + modelImageUrl: modelInputImageUrl, + workflowPrompt: baseReplacementPrompt || hiddenPromptText || null, + planningRuntime: replacementScenePlanningRuntime, models: replacementVisionModels, }); const replacementSourceType = isDetailReplacementPreset @@ -6291,10 +6714,27 @@ export async function POST(request: Request) { referenceImageUrl: reference.imageUrl, pairedProductIndex: typeof pairedProductIndex === "number" ? pairedProductIndex + 1 : null, - shouldReplace: referenceDecision.shouldReplace, - reason: referenceDecision.reason, - detectionModelName: referenceDecision.detectionModelName, - detectionSource: referenceDecision.detectionSource, + shouldReplace: replacementExecutionPlan.shouldReplace, + reason: replacementExecutionPlan.reason, + detectionModelName: replacementExecutionPlan.detectionModelName, + detectionSource: replacementExecutionPlan.detectionSource, + scenePlanSource: replacementExecutionPlan.scenePlan.analysisSource, + selectedProductImageIndexes: + replacementExecutionPlan.scenePlan.selectedProductImageIndexes, + useModelImage: replacementExecutionPlan.scenePlan.useModelImage, + }); + console.info("[Chat API] Replacement scene plan summary:", { + mode: replacementSourceType, + referenceIndex: referenceIndex + 1, + referenceLabel: reference.label, + analysisModelName: replacementExecutionPlan.scenePlan.analysisModelName, + analysisSource: replacementExecutionPlan.scenePlan.analysisSource, + selectedProductImageIndexes: + replacementExecutionPlan.scenePlan.selectedProductImageIndexes, + productSelectionReason: + replacementExecutionPlan.scenePlan.productSelectionReason, + useModelImage: replacementExecutionPlan.scenePlan.useModelImage, + modelImageReason: replacementExecutionPlan.scenePlan.modelImageReason, }); const referenceSourceImageUrls = Array.from( new Set([ @@ -6304,17 +6744,17 @@ export async function POST(request: Request) { ]), ); - if (!referenceDecision.shouldReplace) { + if (!replacementExecutionPlan.shouldReplace) { console.info("[Chat API] Replacement reference skipped and reused:", { mode: replacementSourceType, referenceIndex: referenceIndex + 1, totalReferences: replacementReferences.length, referenceLabel: reference.label, - reason: referenceDecision.reason, + reason: replacementExecutionPlan.reason, }); await emitReplacementResultMessage({ reference, - bodyText: referenceDecision.reason, + bodyText: replacementExecutionPlan.reason, image: { originalUrl: reference.imageUrl, thumbnailUrl: reference.imageUrl, @@ -6330,11 +6770,29 @@ export async function POST(request: Request) { continue; } + const selectedProductIndexes = replacementExecutionPlan.scenePlan + .selectedProductImageIndexes.map((index) => index - 1) + .filter( + (index) => + index >= 0 && index < orderedProductContextImageUrls.length, + ); + const selectedProductContextImageUrls = + selectedProductIndexes.length > 0 + ? selectedProductIndexes.map( + (index) => orderedProductContextImageUrls[index]!, + ) + : orderedProductContextImageUrls.slice( + 0, + Math.min(orderedProductContextImageUrls.length, 2), + ); const sourceImageUrls = Array.from( new Set( [ - ...orderedProductContextImageUrls, - ...(modelInputImageUrl ? [modelInputImageUrl] : []), + ...selectedProductContextImageUrls, + ...(replacementExecutionPlan.scenePlan.useModelImage && + modelInputImageUrl + ? [modelInputImageUrl] + : []), ...replacementTemplateReferenceImageUrls, reference.imageUrl, ].filter(Boolean), @@ -6350,10 +6808,14 @@ export async function POST(request: Request) { referenceLabel: reference.label, referenceIndex, totalReferences: replacementReferences.length, - hasModelImage: Boolean(modelInputImageUrl), - productImageCount: productInputImageUrls.length, + hasModelImage: + replacementExecutionPlan.scenePlan.useModelImage && + Boolean(modelInputImageUrl), + productImageCount: selectedProductContextImageUrls.length, + originalProductImageCount: productInputImageUrls.length, templateReferenceCount: replacementTemplateReferenceImageUrls.length, pairedProductIndex, + scenePlan: replacementExecutionPlan.scenePlan, }); console.info("[Chat API] Replacement image generation started:", { mode: replacementSourceType, @@ -6364,8 +6826,11 @@ export async function POST(request: Request) { typeof pairedProductIndex === "number" ? pairedProductIndex + 1 : null, sourceImageCount: sourceImageUrls.length, imageOrder: { - productImages: productInputImageUrls.length, - hasModelImage: Boolean(modelInputImageUrl), + productImages: selectedProductContextImageUrls.length, + originalProductImages: productInputImageUrls.length, + hasModelImage: + replacementExecutionPlan.scenePlan.useModelImage && + Boolean(modelInputImageUrl), templateReferenceImages: replacementTemplateReferenceImageUrls.length, finalReferenceImageIndex: sourceImageUrls.length, },