diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 5fb3a5295..ac7c4cb0d 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -3346,6 +3346,14 @@ function buildAgentSceneWorkflowPrompt(params: { } type AgentReplacementModePayload = NonNullable; +type AgentModeSourceImagePayload = NonNullable; + +type NormalizedAgentModeSourceImage = { + url: string; + label: string; + name: string; + assetId: string | null; +}; type ReplacementReferenceDecision = { shouldReplace: boolean; @@ -3543,6 +3551,94 @@ function normalizeAgentModeReferences( .sort((left, right) => left.sortOrder - right.sortOrder); } +function normalizeAgentModeSourceImage(value: unknown): NormalizedAgentModeSourceImage | null { + const record = + value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; + const url = String(record.url || "").trim(); + if (!/^https?:\/\//i.test(url)) { + return null; + } + + const label = String(record.label || record.name || "参考图").trim().slice(0, 255); + const name = String(record.name || record.label || label || "参考图").trim().slice(0, 255); + const assetId = String(record.assetId || "").trim() || null; + + return { + url, + label: label || name || "参考图", + name: name || label || "参考图", + assetId, + }; +} + +function normalizeAgentModeSourceImages( + payload?: PostRequestBody["agentModePayload"], +): { + productImages: NormalizedAgentModeSourceImage[]; + modelImage: NormalizedAgentModeSourceImage | null; +} { + const sourceImages = payload?.sourceImages as AgentModeSourceImagePayload | undefined; + const productImages = Array.isArray(sourceImages?.productImages) + ? sourceImages.productImages + .map((item) => normalizeAgentModeSourceImage(item)) + .filter((item): item is NormalizedAgentModeSourceImage => Boolean(item)) + .slice(0, 16) + : []; + + return { + productImages, + modelImage: normalizeAgentModeSourceImage(sourceImages?.modelImage), + }; +} + +function resolveImagePartsByReferenceExclusion(params: { + uploadedImageParts: AgentPresetMessageImagePart[]; + referenceImageUrls: string[]; +}) { + const referenceUrlSet = new Set(params.referenceImageUrls.map((url) => String(url || "").trim())); + const modelPart = params.uploadedImageParts.find((part) => /模特|model/i.test(String(part.name || ""))); + const productParts = params.uploadedImageParts.filter((part) => { + const name = String(part.name || "").trim(); + if (referenceUrlSet.has(part.url)) return false; + if (/^详情参考图|^SKU参考图/i.test(name)) return false; + if (modelPart && part.url === modelPart.url) return false; + return true; + }); + + return { productParts, modelPart: modelPart || null }; +} + +async function resolveDirectReplacementModelRef(params: { + selectedChatModel?: string | null; + finalModelId?: string | null; + models?: Awaited>; +}) { + const explicitModel = String(params.selectedChatModel || "").trim(); + if (explicitModel && explicitModel !== "auto") { + return explicitModel; + } + const routedModel = String(params.finalModelId || "").trim(); + if (routedModel && routedModel !== "auto") { + return routedModel; + } + + const models = params.models || (await getEnabledAiModelsWithProvider(false)); + const imageModels = models.filter( + ({ model, provider }) => + model.supportsImageGen && + !model.isSubModel && + !isVideoProviderType(provider?.type, model), + ); + const preferred = + imageModels.find(({ model }) => model.isDefault) || + imageModels.find(({ provider }) => String(provider?.type || "").includes("image")) || + imageModels[0]; + + return preferred?.model.id || explicitModel || routedModel || "auto"; +} + function normalizeBooleanLike(value: unknown): boolean | null { if (typeof value === "boolean") { return value; @@ -3668,15 +3764,26 @@ function parseReplacementReferenceDecision( async function analyzeReplacementReferenceImage(params: { imageUrl: string; models?: Awaited>; + modeType?: "detail_replacement" | "sku_replacement"; }): Promise { - const detectionPrompt = [ - "你是家具商品参考图替换判定器。", - "请判断这张图片里，是否存在清晰、主要、适合被产品替换流程替换的家具商品主体。", - "家具主体包含：床、床垫、沙发、椅子、餐桌、茶几、柜子、电视柜、斗柜、床头柜、书桌等。", - "只有当家具商品是主要视觉主体、轮廓清晰、不是单纯环境陪衬时，shouldReplace 才能为 true。", - "以下情况必须判定 shouldReplace=false：纯文字页、参数说明页、品牌介绍页、材质/五金局部特写、只有模特没有家具、只有空间环境没有明确家具主体、家具很小只是背景、主体过多无法明确替换对象。", - '只输出 JSON，不要输出额外解释。格式：{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字"}', - ].join("\n"); + const isDetailReplacement = params.modeType === "detail_replacement"; + const detectionPrompt = isDetailReplacement + ? [ + "你是电商详情参考图背景替换判定器。", + "请判断这张图片里，是否存在适合被重建/替换的背景图、产品场景图、商品主体图或大面积视觉图片区。", + "这不是只找家具主体：只要存在照片、插画、场景背景、产品展示图区、人物展示图区，都可以 shouldReplace=true。", + "必须保留的文字、标题、卖点标签、UI图标、按钮、分割线、品牌标识、留白和排版不属于替换对象。", + "只有当图片几乎是纯文字页、参数表、品牌介绍页、无明显背景图/场景图区/商品图片区时，shouldReplace 才能为 false。", + '只输出 JSON，不要输出额外解释。格式：{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字"}', + ].join("\n") + : [ + "你是家具商品参考图替换判定器。", + "请判断这张图片里，是否存在清晰、主要、适合被产品替换流程替换的家具商品主体。", + "家具主体包含：床、床垫、沙发、椅子、餐桌、茶几、柜子、电视柜、斗柜、床头柜、书桌等。", + "只有当家具商品是主要视觉主体、轮廓清晰、不是单纯环境陪衬时，shouldReplace 才能为 true。", + "以下情况必须判定 shouldReplace=false：纯文字页、参数说明页、品牌介绍页、材质/五金局部特写、只有模特没有家具、只有空间环境没有明确家具主体、家具很小只是背景、主体过多无法明确替换对象。", + '只输出 JSON，不要输出额外解释。格式：{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字"}', + ].join("\n"); const configuredModels = params.models || (await getEnabledAiModelsWithProvider()); const configuredQwenVisionModel = configuredModels.find( @@ -3816,6 +3923,7 @@ async function resolveReplacementScenePlanningRuntime(params: { } async function analyzeReplacementExecutionPlan(params: { + modeType?: "detail_replacement" | "sku_replacement"; referenceImageUrl: string; referenceLabel: string; productImageUrls: string[]; @@ -3824,6 +3932,7 @@ async function analyzeReplacementExecutionPlan(params: { planningRuntime: ReplacementScenePlanningRuntime; models?: Awaited>; }): Promise { + const isDetailReplacement = params.modeType === "detail_replacement"; if (params.planningRuntime) { try { const replacementScenePlanningTimeoutMs = @@ -3846,20 +3955,28 @@ async function analyzeReplacementExecutionPlan(params: { .replace(/\s+/g, " ") .slice(0, 2200); const analysisPrompt = [ - "你是电商详情替换流程中的“场景理解规划器”。", + isDetailReplacement + ? "你是电商详情参考图背景替换流程中的“版式与背景区域规划器”。" + : "你是电商详情替换流程中的“场景理解规划器”。", `当前要分析的参考图：${params.referenceLabel}`, "你现在只分析这一张参考图本身，不分析产品图，不分析模特图。", "产品图和模特图会在后续生图阶段单独提供给生图模型使用，你这里不要尝试理解它们的具体外观，也不要要求必须使用所有输入图。", params.modelImageUrl ? "你只需要根据当前参考图判断：这一屏是否存在真实的人物展示需求；如果没有人物需求，useModelImage 必须为 false。" : "本轮没有模特图，useModelImage 必须为 false。", - "这张图片是唯一的详情参考图底板，它决定当前这一屏的版式、文字排版、机位、构图、留白和信息层级。", - "这张参考图不是要求原样保留的场景照片。原图里的沙发、床、柜体、墙面、地面、窗帘、灯具、摆件、桌面道具、背景家具和环境光氛围，都可以在后续生图阶段按前置工作流风格重建。", + "这张图片是唯一的详情参考图底板，它决定当前这一屏的版式、文字排版、UI图标、按钮、机位、构图、留白和信息层级。", + isDetailReplacement + ? "这张参考图不是要求原样保留的场景照片。后续生图只允许替换/重建背景图、产品场景图、商品主体图或人物展示图区域；文字、标题、卖点标签、UI图标、按钮、分割线、品牌标识、留白和排版必须保留。" + : "这张参考图不是要求原样保留的场景照片。原图里的沙发、床、柜体、墙面、地面、窗帘、灯具、摆件、桌面道具、背景家具和环境光氛围，都可以在后续生图阶段按前置工作流风格重建。", "请完成以下任务：", - "1. 判断 shouldReplace：当前参考图里是否存在清晰、主要、适合被替换的家具商品主体。", + isDetailReplacement + ? "1. 判断 shouldReplace：当前参考图里是否存在适合被替换/重建的背景图、产品场景图、商品主体图、人物展示图或大面积视觉图片区。纯文字/纯参数/纯品牌介绍且没有视觉图片区时才为 false。" + : "1. 判断 shouldReplace：当前参考图里是否存在清晰、主要、适合被替换的家具商品主体。", "2. 判断 useModelImage，并说明原因。", "3. 提炼 sceneSummary、cameraSummary、placementSummary、preserveSummary、styleSummary、avoidSummary。", - "4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，描述这一屏需要的场景、机位、主体位置、周边环境重建方向以及必须保留的内容。", + isDetailReplacement + ? "4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，描述需要替换的背景/场景图片区、产品放置方式、模特使用规则，以及必须保留的文字/UI/图标/排版。" + : "4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，描述这一屏需要的场景、机位、主体位置、周边环境重建方向以及必须保留的内容。", "5. sceneSummary、styleSummary 和 generationPrompt 必须优先描述“按前置工作流重建后的目标场景”，而不是只复述参考图当前已有场景。", "6. 如果前置工作流风格与当前参考图原场景冲突，必须保留版式骨架，但放弃原场景物件本身，以前置工作流指定的空间风格、软装、灯光和氛围为准。", "7. avoidSummary 里要明确避免“只替换产品、不重建场景”这类结果。", @@ -3933,6 +4050,7 @@ async function analyzeReplacementExecutionPlan(params: { const fallbackDecision = await analyzeReplacementReferenceImage({ imageUrl: params.referenceImageUrl, models: params.models, + modeType: params.modeType, }); return { ...fallbackDecision, @@ -4044,28 +4162,28 @@ function buildAgentReplacementPrompt(params: { "【图片顺序规则】", ...imageOrderLines, ...scenePlanLines, - "你现在做的不是重新生成一张新图，而是编辑最后一张参考图本身。", - "要把最后一张参考图理解为“页面版式底板 + 构图参考”，不是“旧场景照片必须原样保留”。", + "你现在做的是“详情参考图背景/场景图替换”，不是重新设计整张详情图。", + "要把最后一张参考图理解为“页面版式底板 + 背景图/场景图区参考”，不是“旧场景照片必须原样保留”。", "前面“工作流设定”里的场景组合、空间风格、硬装方案、软装方案、灯光氛围、人群设定，必须作为当前这一屏的强制场景改造要求生效。", - "执行时要基于这些场景组合要求，重建产品主体周边乃至整屏视觉中的场景元素、材质氛围、软装陈列、灯光与色调，使当前这一屏呈现出与第一步已选风格一致的场景效果。", - "允许改造的不只是产品主体周边的一小块区域；除文字排版、标题区、信息区、分割线、留白和版式结构外，原参考图里的旧场景视觉内容都可以按目标风格整体重建。", + "执行时要基于这些场景组合要求，重建参考图中的背景图/产品场景图区域，使当前这一屏呈现出与第一步已选风格一致的新背景画面。", + "允许替换/重建的范围是参考图里的背景图、产品场景图、商品主体图和必要的人物展示图区；不允许改动版式层。", "如果前面商品图、模特图与最后一张参考图底板存在冲突，始终以最后一张参考图底板的版式、机位、构图和信息层级为最高优先级。", - "最后一张参考图是唯一底板，必须保留这张图的文字排版、品牌字样、标题区、卖点区、分割线、留白、机位、构图和信息层级。", - "前面的商品图和模特图用于替换参考图中的产品主体，并按工作流设定去改造与产品直接相关的场景内容，但不能改变参考图的版式设计。", - "可被替换重建的内容包括但不限于：背景墙面、地面材质、沙发、床、背景柜、窗帘、灯具、地毯、挂画、摆件、桌面道具、空间色调、环境光氛围。", + "最后一张参考图是唯一底板，必须完整保留这张图的文字排版、品牌字样、标题区、卖点区、UI图标、按钮、标签、装饰元素、分割线、留白、画布比例和信息层级。", + "前面的商品图和模特图用于生成新的背景图/产品场景图内容，并按工作流设定替换参考图中的旧背景画面，但不能改变参考图的版式设计。", + "可被替换重建的内容包括但不限于：背景照片、产品展示照片、墙面、地面材质、沙发、床、背景柜、窗帘、灯具、地毯、挂画、摆件、桌面道具、空间色调、环境光氛围。", params.productImageCount > 1 ? `前面的 ${params.productImageCount} 张商品输入图都属于同一产品的多角度/多细节白底图，请综合这些图片理解产品结构、材质、颜色、比例、边角、开孔、五金与细节，不可只参考第一张。` : "前面的商品输入图就是当前要替换进去的产品白底图，请准确保持该产品的结构、材质、颜色、比例和细节。", - "请先识别最后一张参考图中需要被替换的产品主体所在区域，并围绕这个区域在同一版式骨架下重建新的场景画面；不要只做局部抠图式换产品。", - "请严格保留最后一张参考图的版式结构、镜头关系、构图、文字排版、留白位置和信息层级；但画面中的旧场景家具、背景物件、材质、灯光氛围、软装小物、空间质感需要按第一步工作流设定进行明显重建。", - "如果第一步工作流风格与参考图原场景冲突，必须保留版式骨架，但放弃原场景物件本身，以第一步工作流风格为准。", + "请先识别最后一张参考图中需要被替换的背景图/场景图区域，并在同一版式骨架下生成新的背景画面；不要只做局部抠图式换产品。", + "请严格保留最后一张参考图的版式结构、镜头关系、构图、文字排版、UI图标、标签、按钮、留白位置和信息层级；但旧背景图里的家具、背景物件、材质、灯光氛围、软装小物、空间质感需要按第一步工作流设定进行明显重建。", + "如果第一步工作流风格与参考图原场景冲突，必须保留版式骨架和UI层，但放弃原背景图物件本身，以第一步工作流风格为准。", "前面的输入商品图用于提供商品主体、材质、颜色和细节特征；如有模特图，请只在参考图本身包含人物展示时再参考模特图。", "绝对不要忽略最后一张参考图去生成一张无关的新场景图。", - "如果生成结果仍然保留了参考图原有的大部分场景物件，只是把产品换掉，则视为执行失败；你必须让场景风格发生清晰可见的变化。", - "绝对不要移动、删除、改写、重排参考图里的任何文字与排版元素。", + "如果生成结果仍然保留了参考图原有的大部分背景图物件，只是把产品换掉，则视为执行失败；你必须让背景图/场景图发生清晰可见的变化。", + "绝对不要移动、删除、改写、重排参考图里的任何文字、UI图标、按钮、标签、装饰元素与排版元素。", "输出结果必须保持最后一张参考图底板的原始纵横比与画布尺寸，不允许擅自改成长图、横图或裁掉任何版面区域。", - "如果参考图中没有可替换的产品主体，应直接沿用参考图，不要硬生成。", - "不要改动参考图的整体布局，不要新增无关主体，不要删除关键版式元素。", + "如果参考图中没有可替换/重建的背景图、产品场景图或视觉图片区，应直接沿用参考图，不要硬生成。", + "不要改动参考图的整体布局，不要新增无关主体，不要删除关键版式或UI元素。", ] : [ "【执行模式】SKU替换", @@ -6851,6 +6969,425 @@ export async function POST(request: Request) { return createSseResponse(confirmedStoryboardStream); } + if (!agentPresetId && agentModePayload?.modeType === "detail_replacement") { + const replacementReferences = normalizeAgentModeReferences(agentModePayload); + if (replacementReferences.length === 0) { + return NextResponse.json( + { + code: "bad_request:api", + message: "详情参考图替换至少需要 1 张详情参考图", + }, + { status: 400 }, + ); + } + + const uploadedImageParts = extractCurrentMessageImageParts(message.parts as any[]); + const sourceImages = normalizeAgentModeSourceImages(agentModePayload); + const inferredImageParts = resolveImagePartsByReferenceExclusion({ + uploadedImageParts, + referenceImageUrls: replacementReferences.map((item) => item.imageUrl), + }); + const productSourceImages = + sourceImages.productImages.length > 0 + ? sourceImages.productImages + : inferredImageParts.productParts.map((part) => ({ + url: part.url, + label: part.name || "产品图", + name: part.name || "产品图", + assetId: part.assetId || null, + })); + const modelSourceImage = + sourceImages.modelImage || + (inferredImageParts.modelPart + ? { + url: inferredImageParts.modelPart.url, + label: inferredImageParts.modelPart.name || "模特图", + name: inferredImageParts.modelPart.name || "模特图", + assetId: inferredImageParts.modelPart.assetId || null, + } + : null); + const replacementReferenceUrlSet = new Set( + replacementReferences.map((item) => item.imageUrl), + ); + const productInputImageUrls = Array.from( + new Set( + productSourceImages + .map((item) => item.url) + .filter((url) => /^https?:\/\//i.test(url) && !replacementReferenceUrlSet.has(url)), + ), + ).slice(0, 16); + const modelInputImageUrl = + modelSourceImage?.url && !replacementReferenceUrlSet.has(modelSourceImage.url) + ? modelSourceImage.url + : null; + + if (productInputImageUrls.length === 0) { + return NextResponse.json( + { + code: "bad_request:api", + message: "详情参考图替换至少需要 1 张产品图", + }, + { status: 400 }, + ); + } + + const directReplacementModels = await getEnabledAiModelsWithProvider().catch((error) => { + console.warn("[Chat API] Failed to load models for direct detail replacement:", error); + return undefined; + }); + const resolvedModelRef = await resolveDirectReplacementModelRef({ + selectedChatModel, + finalModelId, + models: directReplacementModels, + }); + if (!resolvedModelRef || resolvedModelRef === "auto") { + return NextResponse.json( + { + code: "bad_request:api", + message: "当前风格化未配置可用的生图模型", + }, + { status: 400 }, + ); + } + + const sanitizedUserSupplementPrompt = normalizeAgentSupplementPrompt( + requestTextContent, + "详情参考图替换", + ); + const baseReplacementPrompt = + buildAgentSlotPrompt({ + workflowPrompt: hiddenPromptText || null, + slotPrompt: "", + userSupplementPrompt: sanitizedUserSupplementPrompt || null, + includeSlotTitle: false, + modeType: "detail_replacement", + }) || + hiddenPromptText || + sanitizedUserSupplementPrompt || + "执行详情参考图背景替换"; + const resolvedAspectRatio = "auto"; + const resolvedImageSize = + effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize; + const sourceAssetIds = Array.from( + new Set( + [ + ...productSourceImages.map((item) => item.assetId), + modelSourceImage?.assetId, + ...uploadedImageParts.map((part) => part.assetId), + ].filter((assetId): assetId is string => Boolean(assetId)), + ), + ); + + const directDetailReplacementStream = createUIMessageStream({ + execute: async ({ writer }) => { + writeAssistantTextChunk( + writer, + "", + "agent-mode:detail-replacement:start", + ); + if (contextPayload?.source === "canvas") { + writeImageGenerationStatusChunk( + writer, + { + phase: "processing", + modelName: resolvedModelRef, + imageCount: replacementReferences.length, + }, + "agent-mode:detail-replacement:status-processing", + ); + } + + let successCount = 0; + let resolvedModelDisplayName = resolvedModelRef; + const replacementVisionModels = + directReplacementModels || + (await getEnabledAiModelsWithProvider().catch((error) => { + console.warn( + "[Chat API] Failed to load configured vision models for direct detail replacement:", + error, + ); + return undefined; + })); + const replacementScenePlanningRuntime = + await resolveReplacementScenePlanningRuntime({ + models: replacementVisionModels, + }); + const stopKeepalive = startStreamKeepalive( + writer, + "agent-mode:detail-replacement", + 12_000, + ); + + const emitReplacementResultMessage = async (params: { + reference: (typeof replacementReferences)[number]; + bodyText?: string | null; + image: { + originalUrl: string; + thumbnailUrl: string; + }; + imageSizeLabel?: string | null; + modelDisplayName?: string | null; + sourceImageUrls: string[]; + streamLabel: string; + promptForPersistence?: string | null; + }) => { + const persistedImageSize = + String(params.imageSizeLabel || "").trim() || resolvedImageSize; + const assistantText = prependImageModelBadge( + [ + `### ${params.reference.label}`, + String(params.bodyText || "").trim(), + buildAgentToolImagesMarkdown([ + { + originalUrl: params.image.originalUrl, + thumbnailUrl: params.image.thumbnailUrl, + imageSize: persistedImageSize, + }, + ]), + ] + .filter(Boolean) + .join("\n\n"), + params.modelDisplayName || null, + ); + const assistantMessageId = generateUUID(); + + await saveMessages({ + messages: [ + { + id: assistantMessageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: assistantText }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + + await saveGeneratedImages( + assistantText, + session.user.id, + id, + assistantMessageId, + params.promptForPersistence || baseReplacementPrompt, + resolvedModelRef, + { + sourceAssetIds, + sourceImageUrls: params.sourceImageUrls, + relationType: "edited_from", + sourceType: "detail_replacement", + projectId: contextPayload?.projectId || null, + userPrompt: requestTextContent, + optimizedPrompt: params.promptForPersistence || baseReplacementPrompt, + requestedImageCount: 1, + aspectRatio: resolvedAspectRatio, + imageSize: persistedImageSize, + }, + ); + + writeAssistantTextChunk( + writer, + assistantText, + params.streamLabel, + assistantMessageId, + ); + + if (contextPayload?.source === "canvas") { + writeCanvasMediaChunk( + writer, + { + messageId: assistantMessageId, + imageUrls: [params.image.originalUrl], + prompt: params.promptForPersistence || baseReplacementPrompt, + sourceImageUrls: params.sourceImageUrls, + }, + `${params.streamLabel}:canvas-media`, + ); + } + }; + + try { + for (const [referenceIndex, reference] of replacementReferences.entries()) { + const replacementExecutionPlan = await analyzeReplacementExecutionPlan({ + modeType: "detail_replacement", + referenceImageUrl: reference.imageUrl, + referenceLabel: reference.label, + productImageUrls: productInputImageUrls, + modelImageUrl: modelInputImageUrl, + workflowPrompt: baseReplacementPrompt || hiddenPromptText || null, + planningRuntime: replacementScenePlanningRuntime, + models: replacementVisionModels, + }); + const referenceSourceImageUrls = Array.from( + new Set([ + ...productInputImageUrls, + ...(modelInputImageUrl ? [modelInputImageUrl] : []), + reference.imageUrl, + ]), + ); + + if (!replacementExecutionPlan.shouldReplace) { + await emitReplacementResultMessage({ + reference, + bodyText: replacementExecutionPlan.reason, + image: { + originalUrl: reference.imageUrl, + thumbnailUrl: reference.imageUrl, + }, + modelDisplayName: null, + sourceImageUrls: referenceSourceImageUrls, + streamLabel: "agent-mode:image:detail-replacement:item", + }); + successCount += 1; + continue; + } + + const selectedProductIndexes = replacementExecutionPlan.scenePlan + .selectedProductImageIndexes.map((index) => index - 1) + .filter((index) => index >= 0 && index < productInputImageUrls.length); + const selectedProductContextImageUrls = + selectedProductIndexes.length > 0 + ? selectedProductIndexes.map((index) => productInputImageUrls[index]!) + : productInputImageUrls.slice(0, Math.min(productInputImageUrls.length, 2)); + const sourceImageUrls = Array.from( + new Set([ + ...selectedProductContextImageUrls, + ...(replacementExecutionPlan.scenePlan.useModelImage && modelInputImageUrl + ? [modelInputImageUrl] + : []), + reference.imageUrl, + ]), + ); + const replacementPrompt = buildAgentReplacementPrompt({ + basePrompt: baseReplacementPrompt, + modeType: "detail_replacement", + referenceLabel: reference.label, + referenceIndex, + totalReferences: replacementReferences.length, + hasModelImage: + replacementExecutionPlan.scenePlan.useModelImage && + Boolean(modelInputImageUrl), + productImageCount: selectedProductContextImageUrls.length, + originalProductImageCount: productInputImageUrls.length, + templateReferenceCount: 0, + scenePlan: replacementExecutionPlan.scenePlan, + }); + + try { + const result = await generateCanvasImage({ + prompt: replacementPrompt, + negativePrompt: effectiveImageParams?.negativePrompt, + model: resolvedModelRef, + aspectRatio: resolvedAspectRatio, + imageSize: resolvedImageSize, + imageCount: 1, + referenceImages: sourceImageUrls, + }); + resolvedModelDisplayName = result.model || resolvedModelDisplayName; + const firstImage = result.images[0]; + if (!firstImage) { + throw new Error("未返回结果图片"); + } + + let finalImage = firstImage; + let finalImageSizeLabel = resolvedImageSize; + try { + const normalizedImage = await normalizeReplacementResultImage({ + generatedImageUrl: firstImage.originalUrl, + referenceImageUrl: reference.imageUrl, + uploadPrefix: "agent-detail-replacement", + }); + finalImage = { + originalUrl: normalizedImage.originalUrl, + thumbnailUrl: normalizedImage.thumbnailUrl || normalizedImage.originalUrl, + }; + finalImageSizeLabel = normalizedImage.imageSize; + } catch (normalizeError) { + console.warn( + "[Chat API] Failed to normalize direct detail replacement output:", + { + referenceLabel: reference.label, + referenceImageUrl: reference.imageUrl, + resultImageUrl: firstImage.originalUrl, + error: + normalizeError instanceof Error + ? normalizeError.message + : "unknown", + }, + ); + } + + await emitReplacementResultMessage({ + reference, + image: finalImage, + imageSizeLabel: finalImageSizeLabel, + modelDisplayName: resolvedModelDisplayName, + sourceImageUrls: Array.from( + new Set([ + ...productInputImageUrls, + ...(modelInputImageUrl ? [modelInputImageUrl] : []), + ...replacementReferences.map((item) => item.imageUrl), + ]), + ), + streamLabel: "agent-mode:image:detail-replacement:item", + promptForPersistence: replacementPrompt, + }); + successCount += 1; + } catch (replacementItemError) { + console.error("[Chat API] Direct detail replacement item failed:", { + referenceIndex, + referenceLabel: reference.label, + error: replacementItemError, + }); + await emitReplacementResultMessage({ + reference, + bodyText: `生成失败，当前先沿用参考图：${ + replacementItemError instanceof Error + ? replacementItemError.message + : "请稍后重试" + }`, + image: { + originalUrl: reference.imageUrl, + thumbnailUrl: reference.imageUrl, + }, + modelDisplayName: null, + sourceImageUrls: referenceSourceImageUrls, + streamLabel: "agent-mode:image:detail-replacement:item-fallback", + }); + successCount += 1; + } + } + } finally { + stopKeepalive(); + } + + if (contextPayload?.source === "canvas") { + writeImageGenerationStatusChunk( + writer, + { + phase: successCount > 0 ? "completed" : "failed", + modelName: resolvedModelDisplayName, + imageCount: successCount, + }, + "agent-mode:detail-replacement:status-finished", + ); + } + writeAssistantTextChunk( + writer, + "", + "agent-mode:detail-replacement:end", + ); + }, + }); + + return createSseResponse(directDetailReplacementStream); + } + if (agentPresetId) { const selectedAgentPreset = await getAccessibleAgentPresetById({ id: agentPresetId, @@ -7297,6 +7834,7 @@ export async function POST(request: Request) { ] : productInputImageUrls; const replacementExecutionPlan = await analyzeReplacementExecutionPlan({ + modeType: isDetailReplacementPreset ? "detail_replacement" : "sku_replacement", referenceImageUrl: reference.imageUrl, referenceLabel: reference.label, productImageUrls: orderedProductContextImageUrls, diff --git a/app/(chat)/api/chat/schema.ts b/app/(chat)/api/chat/schema.ts index 1da50a2db..465db6ad5 100644 --- a/app/(chat)/api/chat/schema.ts +++ b/app/(chat)/api/chat/schema.ts @@ -80,6 +80,13 @@ const agentModeReferenceSchema = z.object({ assignedProductIndex: z.number().min(0).max(9).nullable().optional(), }); +const agentModeSourceImageSchema = z.object({ + url: z.string().url(), + label: z.string().max(255).optional(), + name: z.string().max(255).optional(), + assetId: z.string().min(1).max(128).nullable().optional(), +}); + const agentModePayloadSchema = z .object({ modeType: z.enum(["detail_replacement", "sku_replacement"]), @@ -87,6 +94,12 @@ const agentModePayloadSchema = z queryType: z.enum(["sku", "spu"]).optional(), selectedCandidateId: z.string().min(1).max(128).nullable().optional(), references: z.array(agentModeReferenceSchema).max(40), + sourceImages: z + .object({ + productImages: z.array(agentModeSourceImageSchema).max(16).optional(), + modelImage: agentModeSourceImageSchema.nullable().optional(), + }) + .optional(), }) .optional(); diff --git a/components/agents/AgentGenerationWorkspaceDarkV2.tsx b/components/agents/AgentGenerationWorkspaceDarkV2.tsx index 4fea9e333..435a39dcf 100644 --- a/components/agents/AgentGenerationWorkspaceDarkV2.tsx +++ b/components/agents/AgentGenerationWorkspaceDarkV2.tsx @@ -1385,6 +1385,60 @@ export function AgentGenerationWorkspaceDarkV2({ ); const selectedModelAttachment = modelSlotIndex >= 0 && attachments[modelSlotIndex] ? attachments[modelSlotIndex] : null; + const detailReplacementProductSourceImages = useMemo(() => { + const primaryImages = referenceImages.filter((image) => image.source === "primary"); + const fallbackProductImages = referenceImages.filter( + (image) => image.source === "primary" || image.source === "secondary" + ); + const productImages = primaryImages.length > 0 ? primaryImages : fallbackProductImages; + + return productImages + .filter((image) => isValidChatHttpUrl(image.url)) + .slice(0, 16) + .map((image) => ({ + url: image.url, + label: image.label || image.name || "产品图", + name: image.name || image.label || "产品图", + assetId: image.assetId || null, + })); + }, [referenceImages]); + const detailReplacementModelSourceImage = useMemo(() => { + const referenceModelImage = referenceImages.find( + (image) => image.source === "brandModel" && isValidChatHttpUrl(image.url) + ); + if (referenceModelImage) { + return { + url: referenceModelImage.url, + label: referenceModelImage.label || referenceModelImage.name || "模特图", + name: referenceModelImage.name || referenceModelImage.label || "模特图", + assetId: referenceModelImage.assetId || null, + }; + } + if (selectedModelAttachment && isValidChatHttpUrl(selectedModelAttachment.url)) { + return { + url: selectedModelAttachment.url, + label: selectedModelAttachment.name || "模特图", + name: selectedModelAttachment.name || "模特图", + assetId: selectedModelAttachment.assetId || null, + }; + } + if (selectedBrandModel?.imageUrl && isValidChatHttpUrl(selectedBrandModel.imageUrl)) { + return { + url: selectedBrandModel.imageUrl, + label: selectedBrandModel.displayName || `${selectedBrandModel.brandName || "品牌"}模特图`, + name: selectedBrandModel.displayName || `${selectedBrandModel.brandName || "品牌"}模特图`, + assetId: selectedBrandModel.assetId || null, + }; + } + return null; + }, [ + referenceImages, + selectedBrandModel?.assetId, + selectedBrandModel?.brandName, + selectedBrandModel?.displayName, + selectedBrandModel?.imageUrl, + selectedModelAttachment, + ]); const scenePrimaryReferenceCount = useMemo( () => referenceImages.filter((image) => image.source === "primary").length, [referenceImages] @@ -1674,6 +1728,8 @@ export function AgentGenerationWorkspaceDarkV2({ detailReplacementQueryInput: string; detailReplacementSelectedCandidateId: string | null; detailReplacementReferenceCards: typeof detailReplacementReferenceCards; + detailReplacementProductSourceImages: typeof detailReplacementProductSourceImages; + detailReplacementModelSourceImage: typeof detailReplacementModelSourceImage; skuReplacementQueryInput: string; skuReplacementQueryType: typeof skuReplacementQueryType; skuReplacementSelectedCandidateId: string | null; @@ -1693,6 +1749,8 @@ export function AgentGenerationWorkspaceDarkV2({ detailReplacementQueryInput: "", detailReplacementSelectedCandidateId: null, detailReplacementReferenceCards: [], + detailReplacementProductSourceImages: [], + detailReplacementModelSourceImage: null, skuReplacementQueryInput: "", skuReplacementQueryType: "sku", skuReplacementSelectedCandidateId: null, @@ -1709,11 +1767,16 @@ export function AgentGenerationWorkspaceDarkV2({ detailReplacementQueryInput, detailReplacementSelectedCandidateId: detailReplacementSelectedCandidateId || null, detailReplacementReferenceCards, + detailReplacementProductSourceImages, + detailReplacementModelSourceImage, skuReplacementQueryInput, skuReplacementQueryType, skuReplacementSelectedCandidateId: skuReplacementSelectedCandidateId || null, skuReplacementReferenceCards, - selectedProductAttachmentCount: selectedProductAttachments.length, + selectedProductAttachmentCount: + currentResultModeType === "detail_replacement" + ? detailReplacementProductSourceImages.length + : selectedProductAttachments.length, }; const selectableModels = useMemo(() => { @@ -1830,6 +1893,12 @@ export function AgentGenerationWorkspaceDarkV2({ ) : null, })), + sourceImages: transportState.isDetailReplacementPreset + ? { + productImages: transportState.detailReplacementProductSourceImages, + modelImage: transportState.detailReplacementModelSourceImage, + } + : undefined, } : undefined; return {