From c486a2e81b795e83df465da041f16388b3d32f56 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 9 Sep 2026 19:48:59 +0800 Subject: [PATCH] fix: preserve detail replacement product geometry --- .../api/chat/_lib/detail-replacement.ts | 399 +++++++++++++++++- app/(chat)/api/chat/route.ts | 30 +- .../agents/AgentGenerationWorkspaceDarkV2.tsx | 84 ++-- .../detail-replacement-image-model-config.ts | 22 + lib/ai/replacement-layout-lock.ts | 371 ++++++++++++++++ lib/ai/replacement-product-identity.ts | 181 ++++++++ ...099_set_detail_replacement_qwen_image3.sql | 10 + lib/db/migrations/meta/_journal.json | 7 + playwright.helpers.config.ts | 2 + .../agent-main-ai-template-entry.test.ts | 19 + tests/routes/replacement-layout-lock.test.ts | 154 +++++++ .../replacement-product-identity.test.ts | 79 ++++ 12 files changed, 1302 insertions(+), 56 deletions(-) create mode 100644 lib/ai/detail-replacement-image-model-config.ts create mode 100644 lib/ai/replacement-layout-lock.ts create mode 100644 lib/ai/replacement-product-identity.ts create mode 100644 lib/db/migrations/0099_set_detail_replacement_qwen_image3.sql create mode 100644 tests/routes/replacement-layout-lock.test.ts create mode 100644 tests/routes/replacement-product-identity.test.ts diff --git a/app/(chat)/api/chat/_lib/detail-replacement.ts b/app/(chat)/api/chat/_lib/detail-replacement.ts index c17b68e19..a8aab1970 100644 --- a/app/(chat)/api/chat/_lib/detail-replacement.ts +++ b/app/(chat)/api/chat/_lib/detail-replacement.ts @@ -2,9 +2,30 @@ import { createLogger } from "@/lib/observability/logger"; import { generateText } from "ai"; import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; +import { + DETAIL_REPLACEMENT_IMAGE_MODEL_CONFIG_KEY, + normalizeDetailReplacementImageModelConfig, +} from "@/lib/ai/detail-replacement-image-model-config"; +import { + compositeReplacementWithReferenceLocks, + parseReplacementLayoutLockRegions, + type NormalizedReplacementBox, +} from "@/lib/ai/replacement-layout-lock"; +import { + buildReplacementProductIdentityAnalysisPrompt, + buildReplacementProductStructureLockLines, + formatReplacementProductIdentityProfile, + normalizeReplacementProductImageIndexes, + parseReplacementProductIdentityProfile, + type ReplacementProductIdentityProfile, +} from "@/lib/ai/replacement-product-identity"; import { resolveDetailAgentLocalMoonshotModel } from "@/lib/ai/detail-agent-model-runtime"; import { inlineVisionImages } from "@/lib/ai/vision-inline"; -import { getEnabledAiModelsWithProvider, getSiteSettings } from "@/lib/db/queries"; +import { + getEnabledAiModelsWithProvider, + getSiteSettings, + getSystemSetting, +} from "@/lib/db/queries"; import { uploadFile } from "@/lib/storage/service"; import { bufferToArrayBuffer } from "./persistence"; import { isVideoProviderType } from "./video-models"; @@ -234,6 +255,8 @@ export async function normalizeReplacementResultImage(params: { referenceImageUrl: string; referenceGeometry?: ReplacementReferenceGeometry | null; scenePlan?: ReplacementScenePlan | null; + preserveReferenceOutsideSubject?: boolean; + preserveReferenceText?: boolean; uploadPrefix: string; }): Promise<{ originalUrl: string; @@ -242,6 +265,11 @@ export async function normalizeReplacementResultImage(params: { }> { const sharp = (await import("sharp")).default; const generatedBuffer = await fetchImageBufferFromUrl(params.generatedImageUrl); + let referenceBuffer: Buffer | null = null; + const getReferenceBuffer = async () => { + referenceBuffer ||= await fetchImageBufferFromUrl(params.referenceImageUrl); + return referenceBuffer; + }; let targetWidth = params.referenceGeometry?.width; let targetHeight = params.referenceGeometry?.height; const generatedMetadata = await sharp(generatedBuffer, { failOn: "none" }) @@ -250,8 +278,8 @@ export async function normalizeReplacementResultImage(params: { if (!targetWidth || !targetHeight) { try { - const referenceBuffer = await fetchImageBufferFromUrl(params.referenceImageUrl); - const referenceMetadata = await sharp(referenceBuffer, { failOn: "none" }) + const resolvedReferenceBuffer = await getReferenceBuffer(); + const referenceMetadata = await sharp(resolvedReferenceBuffer, { failOn: "none" }) .rotate() .metadata(); targetWidth = referenceMetadata.width; @@ -298,7 +326,7 @@ export async function normalizeReplacementResultImage(params: { : shouldPreserveFullGeneratedCanvas ? "contain" : "cover"; - const normalizedBuffer = + let normalizedBuffer = targetWidth && targetHeight ? await imagePipeline .resize(targetWidth, targetHeight, { @@ -312,6 +340,82 @@ export async function normalizeReplacementResultImage(params: { .png() .toBuffer() : await imagePipeline.png().toBuffer(); + + if ( + targetWidth && + targetHeight && + (params.preserveReferenceOutsideSubject || params.preserveReferenceText) + ) { + let layoutLocks = { + subjectBoundingBox: params.scenePlan?.subjectBoundingBox || null, + textBoundingBoxes: params.scenePlan?.textBoundingBoxes || [], + analyzed: params.scenePlan?.layoutLockAnalyzed === true, + }; + const needsLayoutLockFallback = + !layoutLocks.analyzed || + (params.preserveReferenceOutsideSubject && !layoutLocks.subjectBoundingBox); + + if (needsLayoutLockFallback) { + const fallbackLocks = await resolveReplacementLayoutLockRegions({ + referenceImageUrl: params.referenceImageUrl, + referenceBuffer: await getReferenceBuffer(), + }); + layoutLocks = { + subjectBoundingBox: + layoutLocks.subjectBoundingBox || fallbackLocks.subjectBoundingBox, + textBoundingBoxes: + layoutLocks.textBoundingBoxes.length > 0 + ? layoutLocks.textBoundingBoxes + : fallbackLocks.textBoundingBoxes, + analyzed: layoutLocks.analyzed || fallbackLocks.analyzed, + }; + } + + const shouldUseSafeReferenceFallback = Boolean( + (params.preserveReferenceOutsideSubject && !layoutLocks.subjectBoundingBox) || + (params.preserveReferenceText && !layoutLocks.analyzed), + ); + normalizedBuffer = shouldUseSafeReferenceFallback + ? await sharp(await getReferenceBuffer(), { failOn: "none" }) + .rotate() + .resize(targetWidth, targetHeight, { fit: "fill" }) + .png() + .toBuffer() + : await compositeReplacementWithReferenceLocks({ + generatedBuffer: normalizedBuffer, + referenceBuffer: await getReferenceBuffer(), + width: targetWidth, + height: targetHeight, + preserveReferenceOutsideSubject: Boolean( + params.preserveReferenceOutsideSubject && layoutLocks.subjectBoundingBox, + ), + subjectBoundingBox: layoutLocks.subjectBoundingBox, + textBoundingBoxes: params.preserveReferenceText + ? layoutLocks.textBoundingBoxes + : [], + }); + if (shouldUseSafeReferenceFallback) { + chatApiLogger.warn( + "[Chat API] Replacement layout lock analysis unavailable; preserved the reference image instead of returning a layout-damaged result", + { + preserveReferenceOutsideSubject: params.preserveReferenceOutsideSubject, + preserveReferenceText: params.preserveReferenceText, + hasSubjectBoundingBox: Boolean(layoutLocks.subjectBoundingBox), + layoutLockAnalyzed: layoutLocks.analyzed, + }, + ); + } + chatApiLogger.info("[Chat API] Replacement reference layout locks applied:", { + preserveReferenceOutsideSubject: Boolean( + params.preserveReferenceOutsideSubject && layoutLocks.subjectBoundingBox, + ), + preservedTextRegionCount: params.preserveReferenceText + ? layoutLocks.textBoundingBoxes.length + : 0, + usedFallbackAnalysis: needsLayoutLockFallback, + usedSafeReferenceFallback: shouldUseSafeReferenceFallback, + }); + } const thumbnailBuffer = await createAgentImageThumbnailBuffer(normalizedBuffer); const uniqueSuffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; @@ -399,6 +503,12 @@ export type ReplacementScenePlan = { generationPrompt: string; selectedProductImageIndexes: number[]; productSelectionReason: string; + productIdentitySummary?: string; + productStructureSummary?: string; + productStructuralRisks?: string; + subjectBoundingBox?: NormalizedReplacementBox | null; + textBoundingBoxes?: NormalizedReplacementBox[]; + layoutLockAnalyzed?: boolean; analysisModelName?: string | null; analysisSource: "local_moonshot" | "configured_vision" | "heuristic"; }; @@ -439,6 +549,187 @@ export type ReplacementScenePlanningRuntime = const QWEN_VL_MODEL_MATCHER = /(qwen|千问|通义).*(vl|vision)|qwen3-vl|qwen-vl/i; +const replacementProductIdentityProfileCache = new Map< + string, + { + expiresAt: number; + promise: Promise; + } +>(); + +async function resolveReplacementProductIdentityProfile(params: { + productImageUrls: string[]; + planningRuntime: Exclude; +}) { + const productImageUrls = Array.from( + new Set( + params.productImageUrls + .map((url) => String(url || "").trim()) + .filter((url) => /^https?:\/\//i.test(url)), + ), + ).slice(0, 8); + if (productImageUrls.length === 0) return null; + + const cacheKey = [ + params.planningRuntime.kind, + params.planningRuntime.modelName, + ...productImageUrls, + ].join("|"); + const now = Date.now(); + const cached = replacementProductIdentityProfileCache.get(cacheKey); + if (cached && cached.expiresAt > now) { + return cached.promise; + } + + if (replacementProductIdentityProfileCache.size >= 100) { + const oldestKey = replacementProductIdentityProfileCache.keys().next().value; + if (oldestKey) replacementProductIdentityProfileCache.delete(oldestKey); + } + + const promise = (async () => { + try { + const { inlineDataUrls } = await inlineVisionImages(productImageUrls, { + maxImages: productImageUrls.length, + maxSide: 800, + timeoutMs: 12_000, + retries: 2, + loggerPrefix: "[Chat API] Replacement product identity", + }); + const effectiveImages = + inlineDataUrls.length === productImageUrls.length + ? inlineDataUrls + : productImageUrls; + const { text } = await generateText({ + model: params.planningRuntime.model, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: buildReplacementProductIdentityAnalysisPrompt( + productImageUrls.length, + ), + }, + ...effectiveImages.map((image) => ({ + type: "image" as const, + image, + })), + ], + }, + ], + system: + "你是高精度商品结构分析器。多张图片属于同一商品；请跨视角还原唯一结构身份，并明确每个视角最清楚的构件。", + temperature: params.planningRuntime.kind === "local_moonshot" ? 1 : 0.1, + maxTokens: 1800, + abortSignal: AbortSignal.timeout( + params.planningRuntime.kind === "local_moonshot" ? 90_000 : 60_000, + ), + } as any); + const profile = parseReplacementProductIdentityProfile( + text, + productImageUrls.length, + ); + if (!profile) { + throw new Error("商品结构画像缺少必需字段"); + } + return profile; + } catch (error) { + chatApiLogger.warn("[Chat API] Replacement product identity analysis failed:", { + productImageCount: productImageUrls.length, + modelName: params.planningRuntime.modelName, + error: error instanceof Error ? error.message : "unknown", + }); + return null; + } + })(); + + replacementProductIdentityProfileCache.set(cacheKey, { + expiresAt: now + 30 * 60 * 1000, + promise, + }); + return promise; +} + +const replacementLayoutLockCache = new Map< + string, + { + expiresAt: number; + promise: Promise>; + } +>(); + +async function resolveReplacementLayoutLockRegions(params: { + referenceImageUrl: string; + referenceBuffer: Buffer; +}) { + const now = Date.now(); + const cached = replacementLayoutLockCache.get(params.referenceImageUrl); + if (cached && cached.expiresAt > now) { + return cached.promise; + } + + if (replacementLayoutLockCache.size >= 200) { + const oldestKey = replacementLayoutLockCache.keys().next().value; + if (oldestKey) { + replacementLayoutLockCache.delete(oldestKey); + } + } + + const promise = (async () => { + try { + const sharp = (await import("sharp")).default; + const inlineBuffer = await sharp(params.referenceBuffer, { failOn: "none" }) + .rotate() + .resize(1600, 1600, { + fit: "inside", + withoutEnlargement: true, + }) + .flatten({ background: "#ffffff" }) + .jpeg({ quality: 88 }) + .toBuffer(); + const { callQwen3Vl } = await import("@/lib/ai/qwen3-vl"); + const result = await callQwen3Vl({ + imageUrl: `data:image/jpeg;base64,${inlineBuffer.toString("base64")}`, + prompt: [ + "你是电商详情图的布局区域定位器。", + "只分析当前图像，返回严格 JSON，不要解释。", + "所有坐标使用 0到1000 的归一化坐标，格式必须是 [x1,y1,x2,y2]，绝对不是 x/y/宽/高。", + "subjectBoundingBox：只框出图中当前旧商品主体的可视外轮廓区域，包含完整商品和它的自然阴影，不要包含文字；没有商品则为 null。", + "textBoundingBoxes：框出所有可见文案块，包括中英文标题、卖点、参数、标签、数字、品牌字样、按钮文字和紧邻的图标；没有则返回空数组。", + '{"subjectBoundingBox":[0,0,1000,1000],"textBoundingBoxes":[[0,0,1000,1000]]}', + ].join("\n"), + temperature: 0, + maxTokens: 1600, + timeoutMs: 60_000, + retries: 1, + }); + const parsedLocks = parseReplacementLayoutLockRegions(result.text); + if (!parsedLocks.analyzed) { + throw new Error("布局区域定位结果缺少必需字段"); + } + return parsedLocks; + } catch (error) { + replacementLayoutLockCache.delete(params.referenceImageUrl); + chatApiLogger.warn("[Chat API] Replacement layout lock fallback analysis failed:", { + referenceImageUrl: params.referenceImageUrl, + error: error instanceof Error ? error.message : "unknown", + }); + return { + subjectBoundingBox: null, + textBoundingBoxes: [], + analyzed: false, + }; + } + })(); + + replacementLayoutLockCache.set(params.referenceImageUrl, { + expiresAt: now + 30 * 60 * 1000, + promise, + }); + return promise; +} + export function normalizeDetailReplacementStrategy( value: unknown, ): AgentDetailReplacementStrategy { @@ -1441,7 +1732,27 @@ export function parseReplacementScenePlan( visualProfile: params.visualProfile, }); - const productImageSelection = buildDefaultProductImageSelection(params.productImageCount); + const defaultProductImageSelection = buildDefaultProductImageSelection( + params.productImageCount, + ); + const selectedProductImageIndexes = normalizeReplacementProductImageIndexes( + parsed.selectedProductImageIndexes ?? + parsed.selectedProductIndexes ?? + parsed.productImageIndexes, + params.productImageCount, + ); + const productImageSelection = + selectedProductImageIndexes.length > 0 + ? { + selectedProductImageIndexes, + productSelectionReason: + truncateReplacementAnalysisText( + parsed.productSelectionReason ?? parsed.selectionReason, + 240, + ) || "按当前详情底板机位选择最匹配的产品视角。", + } + : defaultProductImageSelection; + const layoutLocks = parseReplacementLayoutLockRegions(parsed); let scenePlan: ReplacementScenePlan = { pageKind, generationRoute, @@ -1486,6 +1797,9 @@ export function parseReplacementScenePlan( generationPrompt, selectedProductImageIndexes: productImageSelection.selectedProductImageIndexes, productSelectionReason: productImageSelection.productSelectionReason, + subjectBoundingBox: layoutLocks.subjectBoundingBox, + textBoundingBoxes: layoutLocks.textBoundingBoxes, + layoutLockAnalyzed: layoutLocks.analyzed, analysisModelName: null, analysisSource: "heuristic", }; @@ -1844,6 +2158,27 @@ export async function resolveDirectReplacementModelRef(params: { finalModelId?: string | null; models?: Awaited>; }) { + const models = params.models || (await getEnabledAiModelsWithProvider(false)); + const imageModels = models.filter( + ({ model, provider }) => + model.supportsImageGen && + !model.isSubModel && + !isVideoProviderType(provider?.type, model), + ); + const configuredModelSetting = await getSystemSetting({ + key: DETAIL_REPLACEMENT_IMAGE_MODEL_CONFIG_KEY, + }).catch(() => null); + const configuredModelId = normalizeDetailReplacementImageModelConfig( + configuredModelSetting?.value, + ).modelId; + const configuredModel = imageModels.find( + ({ model }) => + model.id === configuredModelId || model.modelId === configuredModelId, + ); + if (configuredModel) { + return configuredModel.model.id; + } + const explicitModel = String(params.selectedChatModel || "").trim(); if (explicitModel && explicitModel !== "auto") { return explicitModel; @@ -1853,13 +2188,6 @@ export async function resolveDirectReplacementModelRef(params: { return routedModel; } - const models = params.models || (await getEnabledAiModelsWithProvider(false)); - const imageModels = models.filter( - ({ model, provider }) => - model.supportsImageGen && - !model.isSubModel && - !isVideoProviderType(provider?.type, model), - ); const preferred = imageModels.find(({ model }) => model.isDefault) || imageModels.find(({ provider }) => String(provider?.type || "").includes("image")) || @@ -2191,6 +2519,13 @@ export async function analyzeReplacementExecutionPlan(params: { const referenceVisualProfile = isDetailReplacement ? await resolveReplacementReferenceVisualProfile(params.referenceImageUrl) : null; + const productIdentityProfile = + params.planningRuntime && params.productImageUrls.length > 0 + ? await resolveReplacementProductIdentityProfile({ + productImageUrls: params.productImageUrls, + planningRuntime: params.planningRuntime, + }) + : null; if (params.planningRuntime) { try { const replacementScenePlanningTimeoutMs = @@ -2221,7 +2556,10 @@ export async function analyzeReplacementExecutionPlan(params: { ? "你是电商详情参考图背景替换流程中的“版式与背景区域规划器”。" : "你是电商详情替换流程中的“场景理解规划器”。", `当前要分析的参考图：${params.referenceLabel}`, - "你现在只分析这一张参考图本身，不分析产品图，不分析模特图。", + "当前视觉输入只包含这一张详情参考图；产品图已经单独做过跨视角结构分析，请根据下面的产品结构画像选择最匹配当前机位的产品视角，不要臆测产品结构。", + productIdentityProfile + ? `【产品结构画像】\n${formatReplacementProductIdentityProfile(productIdentityProfile)}` + : "产品结构画像暂不可用：仍需按当前参考图机位选择产品图，但不得把旧产品的腿型、支架或构件拓扑当成新产品结构。", referenceVisualProfile ? `像素初筛：nearWhite=${referenceVisualProfile.nearWhitePixelRatio.toFixed(2)}, lightLowSaturation=${referenceVisualProfile.lightLowSaturationPixelRatio.toFixed(2)}, topLight=${referenceVisualProfile.topLightLowSaturationPixelRatio.toFixed(2)}, borderLight=${referenceVisualProfile.borderLightLowSaturationPixelRatio.toFixed(2)}, dark=${referenceVisualProfile.darkPixelRatio.toFixed(2)}, likelyWhiteLayout=${referenceVisualProfile.likelyWhiteLayout ? "true" : "false"}, likelyWhiteScene=${referenceVisualProfile.likelyWhiteScene ? "true" : "false"}。${referenceVisualProfile.reason}` : "", @@ -2231,7 +2569,7 @@ export async function analyzeReplacementExecutionPlan(params: { !referenceVisualProfile?.likelyWhiteLayout && referenceVisualProfile?.likelyWhiteScene ? "像素初筛已经命中白/浅色低饱和真实场景：这不是白底说明页。若画面存在真实空间、人物或产品照片，promptPackPolicy 应为 soft 或 allow，必须使用前置工作流词包重建图片框内产品、空间、软装和灯光，使差异清晰可见；但最终仍保持白/浅色低饱和基调，不要改造成深色厨房、黑墙、复杂吊灯或强软装。" : "", - "产品图和模特图会在后续生图阶段单独提供给生图模型使用，你这里不要尝试理解它们的具体外观，也不要要求必须使用所有输入图。", + "产品图和模特图会在后续生图阶段单独提供给生图模型。你必须依据产品结构画像输出 selectedProductImageIndexes，选择最多两张最能匹配当前参考图机位并共同说明完整结构的产品图；不得固定选择前两张，也不要要求强行使用全部产品图。", params.modelImageUrl && !isProductOnlyDetailReplacement ? "你只需要根据当前参考图判断：这一屏是否存在真实的人物展示需求；如果没有人物需求，useModelImage 必须为 false。" : "本轮没有模特图，useModelImage 必须为 false。", @@ -2278,12 +2616,18 @@ export async function analyzeReplacementExecutionPlan(params: { isDetailReplacement ? "5. 判断 textPolicy：画面中存在营销标题、卖点、编号、参数、标签且与图匹配时必须 lock 或 preserve，避免后续生图改字、混色或重排。" : "", + isDetailReplacement + ? "5a. 必须输出 subjectBoundingBox 和 textBoundingBoxes。所有坐标为 0到1000 归一化坐标 [x1,y1,x2,y2]，不是 x/y/宽/高。subjectBoundingBox 紧密框住旧商品主体及自然阴影，不含文字，无商品时为 null；textBoundingBoxes 覆盖所有中英文文案、标题、卖点、参数、标签、数字、品牌字样、按钮文字及紧邻图标，无文字时为 []。" + : "", isDetailReplacement ? "6. 判断 peoplePolicy：有人物/多人/自拍/聚会/围坐/聚餐/模特互动/拿手机/用餐/试坐/承重动作时必须 preserve_relation；如果文字明确写了“多人围坐、聚餐、合照、用餐、单侧承重、横向受力测试、拿手机”等动作，generationPrompt 必须把这个动作写成硬约束；如果只提供 1 张模特图但参考图多人，仍要保留参考图人数和关系，额外人物按参考图生成相近人群。" : "", isDetailReplacement ? "7. 提炼 sceneSummary、cameraSummary、placementSummary、preserveSummary、styleSummary、avoidSummary；cameraSummary 必须写出产品主体的视角特征，例如正侧视/三分之四/俯视/平视、倾斜方向、可见面数量、主体朝向和透视消失方向。" : "", + isDetailReplacement + ? "7a. 输出 selectedProductImageIndexes：根据当前详情图机位，从产品结构画像的图片编号中选择最多两张最相关的产品图；第一张应最接近当前机位，第二张补足腿、支架、横梁、侧框或其他容易变形的结构。输出 productSelectionReason 说明选择依据。" + : "", isProductOnlyDetailReplacement ? "4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，要求只替换产品主体，保留其余背景、文案、排版、装饰和光影完全不变。" : isDetailReplacement @@ -2309,7 +2653,7 @@ export async function analyzeReplacementExecutionPlan(params: { : "", "只返回严格 JSON，不要任何额外解释。", `JSON 格式： -{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字","pageKind":"prompt_pack_scene|white_scene|white_layout|copy_locked|people_scene|action_poster|dimension_spec|material_detail|unknown","generationRoute":"${isDetailReplacement ? "prompt_pack_scene|layout_preserve_replace|copy_locked_regenerate|people_relation_locked" : "prompt_pack_scene|layout_preserve_replace|copy_locked_regenerate|people_relation_locked|direct_passthrough"}","promptPackPolicy":"allow|soft|forbid","textPolicy":"free|preserve|lock","peoplePolicy":"none|preserve_relation|require_user_model","whiteBackgroundLikely":false,"copyImageCoupled":false,"containsPeople":false,"containsMultiplePeople":false,"routeReason":"不超过50字","useModelImage":false,"modelImageReason":"不超过40字","sceneSummary":"不超过80字","cameraSummary":"不超过60字","placementSummary":"不超过60字","preserveSummary":"不超过80字","styleSummary":"不超过80字","avoidSummary":"不超过80字","generationPrompt":"不超过300字"}`, +{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字","pageKind":"prompt_pack_scene|white_scene|white_layout|copy_locked|people_scene|action_poster|dimension_spec|material_detail|unknown","generationRoute":"${isDetailReplacement ? "prompt_pack_scene|layout_preserve_replace|copy_locked_regenerate|people_relation_locked" : "prompt_pack_scene|layout_preserve_replace|copy_locked_regenerate|people_relation_locked|direct_passthrough"}","promptPackPolicy":"allow|soft|forbid","textPolicy":"free|preserve|lock","peoplePolicy":"none|preserve_relation|require_user_model","whiteBackgroundLikely":false,"copyImageCoupled":false,"containsPeople":false,"containsMultiplePeople":false,"subjectBoundingBox":[0,0,1000,1000],"textBoundingBoxes":[[0,0,1000,1000]],"selectedProductImageIndexes":[1,2],"productSelectionReason":"不超过80字","routeReason":"不超过50字","useModelImage":false,"modelImageReason":"不超过40字","sceneSummary":"不超过80字","cameraSummary":"不超过60字","placementSummary":"不超过60字","preserveSummary":"不超过80字","styleSummary":"不超过80字","avoidSummary":"不超过80字","generationPrompt":"不超过300字"}`, ] .filter(Boolean) .join("\n"); @@ -2334,7 +2678,7 @@ export async function analyzeReplacementExecutionPlan(params: { system: "你是高精度商品详情图场景规划器。你只分析当前参考图本身，负责理解这一屏的场景、机位、主体位置和人物需求，不要臆测或依赖其他输入图片。", temperature: params.planningRuntime.kind === "local_moonshot" ? 1 : 0.1, - maxTokens: 1200, + maxTokens: 1600, abortSignal: AbortSignal.timeout(replacementScenePlanningTimeoutMs), } as any); @@ -2368,6 +2712,9 @@ export async function analyzeReplacementExecutionPlan(params: { : "configured_vision", scenePlan: { ...normalizedPlan.scenePlan, + productIdentitySummary: productIdentityProfile?.identitySummary || "", + productStructureSummary: productIdentityProfile?.structureSummary || "", + productStructuralRisks: productIdentityProfile?.structuralRisks || "", analysisModelName: params.planningRuntime.modelName, analysisSource: params.planningRuntime.kind === "local_moonshot" @@ -2413,7 +2760,12 @@ export async function analyzeReplacementExecutionPlan(params: { ); return { ...normalizedFallbackPlan.decision, - scenePlan: normalizedFallbackPlan.scenePlan, + scenePlan: { + ...normalizedFallbackPlan.scenePlan, + productIdentitySummary: productIdentityProfile?.identitySummary || "", + productStructureSummary: productIdentityProfile?.structureSummary || "", + productStructuralRisks: productIdentityProfile?.structuralRisks || "", + }, }; } @@ -2537,11 +2889,17 @@ export function buildAgentReplacementPrompt(params: { ] : []; const productAngleLockLines = [ - "【产品角度锁定规则】最终商品/主体的拍摄角度以最后一张参考图为准，而不是以前面的产品输入图为准；前面的产品图只提供款式、材质、颜色、结构和细节。", - "必须逐项对齐最后一张参考图中的产品姿态：朝向、正侧视/三分之四/俯视/平视关系、桌面或面板倾斜方向、可见侧面数量、支架/腿部/把手/转轴位置、主体外轮廓、占比、重心和透视消失方向。", + "【产品角度锁定规则】最终商品/主体的拍摄角度、朝向、占比、重心和透视消失方向以最后一张参考图为准；商品身份、外轮廓、腿型、支架拓扑、构件数量和连接方式以前面的产品输入图为准。", + "必须对齐最后一张参考图的正侧视/三分之四/俯视/平视关系、桌面或面板倾斜方向和可见侧面数量，但不得照搬旧商品的腿部、横梁、侧框、把手、转轴或内部构件设计。", "如果最后一张参考图的产品是侧视或低视角，不允许改成俯视三分之四；如果是俯视三分之四，不允许改成正侧视；不允许镜像、旋转、翻转或把产品重画成另一个角度。", "若参考图含功能标签、箭头、尺寸线、编号或引导线，它们指向的实际结构部位必须保持一致，不得因产品转角改变而让标签指向错误位置。", ]; + const productStructureLockLines = buildReplacementProductStructureLockLines({ + productImageCount: params.productImageCount, + identitySummary: params.scenePlan?.productIdentitySummary, + structureSummary: params.scenePlan?.productStructureSummary, + structuralRisks: params.scenePlan?.productStructuralRisks, + }); const normalizedBasePrompt = String(params.basePrompt || "").trim(); const promptPackInstructionLines = params.modeType === "detail_replacement" && @@ -2641,6 +2999,7 @@ export function buildAgentReplacementPrompt(params: { ...promptPackInstructionLines, ...scenePlanLines, ...marketplaceBrandRemovalLines, + ...productStructureLockLines, ...productAngleLockLines, "你现在做的是‘仅替换产品’，不是重建场景，也不是重新设计整张详情图。", "忽略用户补充提示词里的场景风格、氛围、空间改造要求；这一次只服务于产品主体替换。", @@ -2670,6 +3029,7 @@ export function buildAgentReplacementPrompt(params: { ...promptPackInstructionLines, ...scenePlanLines, ...marketplaceBrandRemovalLines, + ...productStructureLockLines, ...productAngleLockLines, "你现在做的是“详情参考图背景/场景图替换”，不是重新设计整张详情图。", "【版式锁层规则】把最后一张参考图当成 PSD 模板：文字、标题、编号、卖点标签、按钮、图标、白底/浅色底、留白、分割线、原有圆角照片区域和原有裁切范围都属于锁定版式层，不允许移动、缩放、重排或重新设计；不要把这些版式边界画成新的白色线框、描边或卡片。", @@ -2785,6 +3145,7 @@ export function buildAgentReplacementPrompt(params: { "【图片顺序规则】", ...imageOrderLines, ...scenePlanLines, + ...productStructureLockLines, ...productAngleLockLines, params.pairedProductIndex !== undefined ? `请优先使用第 ${params.pairedProductIndex + 1} 张输入商品图作为本张参考图的一对一替换主体。` diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 0f06fac7b..5b36060c8 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -6054,6 +6054,9 @@ export async function POST(request: Request) { referenceImageUrl: reference.imageUrl, referenceGeometry, scenePlan: replacementExecutionPlan.scenePlan, + preserveReferenceOutsideSubject: + detailReplacementStrategy === "product_only", + preserveReferenceText: true, uploadPrefix: "agent-detail-replacement", }); finalImage = { @@ -6489,9 +6492,20 @@ export async function POST(request: Request) { ...(selectedAgentPreset.defaultParams || {}), ...(replacementSlot?.defaultParams || {}), } as Record; - const resolvedModelRef = - String(selectedChatModel || "").trim() && - String(selectedChatModel || "").trim() !== "auto" + const replacementVisionModels = await getEnabledAiModelsWithProvider().catch( + (error) => { + chatApiLogger.warn("[Chat API] Failed to load configured vision models for replacement analysis:", error); + return undefined; + }, + ); + const resolvedModelRef = isDetailReplacementPreset + ? await resolveDirectReplacementModelRef({ + selectedChatModel, + finalModelId: presetModelId, + models: replacementVisionModels, + }) + : String(selectedChatModel || "").trim() && + String(selectedChatModel || "").trim() !== "auto" ? String(selectedChatModel || "").trim() : presetModelId; const configuredReplacementAspectRatio = @@ -6513,12 +6527,6 @@ export async function POST(request: Request) { configuredAspectRatio: configuredReplacementAspectRatio, }); } - const replacementVisionModels = await getEnabledAiModelsWithProvider().catch( - (error) => { - chatApiLogger.warn("[Chat API] Failed to load configured vision models for replacement analysis:", error); - return undefined; - }, - ); const replacementScenePlanningRuntime = await resolveReplacementScenePlanningRuntime({ models: replacementVisionModels, @@ -6943,6 +6951,10 @@ export async function POST(request: Request) { referenceImageUrl: reference.imageUrl, referenceGeometry, scenePlan: replacementExecutionPlan.scenePlan, + preserveReferenceOutsideSubject: + isDetailReplacementPreset && + detailReplacementStrategy === "product_only", + preserveReferenceText: isDetailReplacementPreset, uploadPrefix: isDetailReplacementPreset ? "agent-detail-replacement" : "agent-sku-replacement", diff --git a/components/agents/AgentGenerationWorkspaceDarkV2.tsx b/components/agents/AgentGenerationWorkspaceDarkV2.tsx index 7d671d154..98e40f04d 100644 --- a/components/agents/AgentGenerationWorkspaceDarkV2.tsx +++ b/components/agents/AgentGenerationWorkspaceDarkV2.tsx @@ -671,9 +671,10 @@ const AGENT_DEFERRED_SECTION_STYLE = { containIntrinsicSize: "1px 760px", } as CSSProperties; -const AGENT_DEFERRED_RESULTS_STYLE = { - contentVisibility: "auto", - containIntrinsicSize: "1px 900px", +// Dynamic result content must stay painted while scrolling. Deferring this subtree +// makes Chromium briefly discard image and animation layers at viewport boundaries. +const AGENT_STABLE_RESULTS_STYLE = { + contentVisibility: "visible", } as CSSProperties; const AGENT_DEFERRED_LIST_CARD_STYLE = { @@ -681,6 +682,10 @@ const AGENT_DEFERRED_LIST_CARD_STYLE = { containIntrinsicSize: "1px 320px", } as CSSProperties; +const AGENT_STABLE_RESULT_CARD_STYLE = { + contentVisibility: "visible", +} as CSSProperties; + function handlePreviewImageLoadError( event: SyntheticEvent, originalUrl: string @@ -2712,29 +2717,40 @@ function DetailResultProcessingScanOverlay({ return (
{isHttpImageUrl(imageUrl) ? ( - ) : (
+ > +
+
@@ -9856,7 +9884,7 @@ export function AgentGenerationWorkspaceDarkV2({ selectedProductAttachments.map((attachment, index) => (
@@ -9972,7 +10000,7 @@ export function AgentGenerationWorkspaceDarkV2({ return (
(
@@ -10580,7 +10608,7 @@ export function AgentGenerationWorkspaceDarkV2({ key={candidate.id} type="button" onClick={() => handleSelectSkuReplacementCandidate(candidate.id)} - style={AGENT_DEFERRED_LIST_CARD_STYLE} + style={AGENT_STABLE_RESULT_CARD_STYLE} className={cn( "min-w-[280px] overflow-hidden rounded-[20px] border text-left transition-colors", isSelected @@ -10647,7 +10675,7 @@ export function AgentGenerationWorkspaceDarkV2({ skuReplacementReferenceCards.map((card, index) => (
{ if (!card.url) return; setSelectedResultUrl(card.url); diff --git a/lib/ai/detail-replacement-image-model-config.ts b/lib/ai/detail-replacement-image-model-config.ts new file mode 100644 index 000000000..d7c2d0d35 --- /dev/null +++ b/lib/ai/detail-replacement-image-model-config.ts @@ -0,0 +1,22 @@ +export const DETAIL_REPLACEMENT_IMAGE_MODEL_CONFIG_KEY = + "detail_replacement_image_model_config"; + +export const DEFAULT_DETAIL_REPLACEMENT_IMAGE_MODEL_ID = "qwen-image-3.0"; + +export type DetailReplacementImageModelConfig = { + modelId: string; +}; + +export function normalizeDetailReplacementImageModelConfig( + value: unknown, +): DetailReplacementImageModelConfig { + const record = + value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; + const modelId = String(record.modelId || "").trim(); + + return { + modelId: modelId || DEFAULT_DETAIL_REPLACEMENT_IMAGE_MODEL_ID, + }; +} diff --git a/lib/ai/replacement-layout-lock.ts b/lib/ai/replacement-layout-lock.ts new file mode 100644 index 000000000..d5898e4f1 --- /dev/null +++ b/lib/ai/replacement-layout-lock.ts @@ -0,0 +1,371 @@ +export type NormalizedReplacementBox = [number, number, number, number]; + +export type ReplacementLayoutLockRegions = { + subjectBoundingBox: NormalizedReplacementBox | null; + textBoundingBoxes: NormalizedReplacementBox[]; + analyzed: boolean; +}; + +type PixelBox = { + left: number; + top: number; + right: number; + bottom: number; +}; + +function extractJsonRecord(text: string): Record | null { + const trimmed = String(text || "") + .trim() + .replace(/^```(?:json)?\s*/i, "") + .replace(/\s*```$/i, ""); + const start = trimmed.indexOf("{"); + const end = trimmed.lastIndexOf("}"); + if (start < 0 || end <= start) { + return null; + } + + try { + const parsed = JSON.parse(trimmed.slice(start, end + 1)); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as Record) + : null; + } catch { + return null; + } +} + +function clampNormalizedCoordinate(value: unknown) { + const numberValue = Number(value); + if (!Number.isFinite(numberValue)) { + return null; + } + return Math.max(0, Math.min(1000, numberValue)); +} + +export function normalizeReplacementBoundingBox( + value: unknown, +): NormalizedReplacementBox | null { + let coordinates: unknown[] | null = null; + + if (Array.isArray(value)) { + coordinates = value; + } else if (value && typeof value === "object") { + const record = value as Record; + if (Array.isArray(record.bbox)) { + coordinates = record.bbox; + } else if ( + record.x1 !== undefined && + record.y1 !== undefined && + record.x2 !== undefined && + record.y2 !== undefined + ) { + coordinates = [record.x1, record.y1, record.x2, record.y2]; + } else if ( + record.left !== undefined && + record.top !== undefined && + record.right !== undefined && + record.bottom !== undefined + ) { + coordinates = [record.left, record.top, record.right, record.bottom]; + } + } + + if (!coordinates || coordinates.length !== 4) { + return null; + } + + const normalized = coordinates.map(clampNormalizedCoordinate); + if (normalized.some((coordinate) => coordinate === null)) { + return null; + } + + const [x1, y1, x2, y2] = normalized as NormalizedReplacementBox; + if (x2 - x1 < 2 || y2 - y1 < 2) { + return null; + } + + return [x1, y1, x2, y2]; +} + +export function expandReplacementSubjectBoundingBoxForGeneration( + value: unknown, +): NormalizedReplacementBox | null { + const box = normalizeReplacementBoundingBox(value); + if (!box) return null; + + const [x1, y1, x2, y2] = box; + const horizontalPadding = (x2 - x1) * 0.1; + const verticalPadding = (y2 - y1) * 0.015; + return [ + Math.max(0, Math.round(x1 - horizontalPadding)), + Math.max(0, Math.round(y1 - verticalPadding)), + Math.min(1000, Math.round(x2 + horizontalPadding)), + Math.min(1000, Math.round(y2 + verticalPadding)), + ]; +} + +export function buildReplacementInputBoundingBoxList(params: { + imageUrls: string[]; + referenceImageUrl: string; + subjectBoundingBox: unknown; +}): NormalizedReplacementBox[][] { + const referenceImageUrl = String(params.referenceImageUrl || "").trim(); + const expandedBox = expandReplacementSubjectBoundingBoxForGeneration( + params.subjectBoundingBox, + ); + + return params.imageUrls.map((imageUrl) => + expandedBox && String(imageUrl || "").trim() === referenceImageUrl + ? [expandedBox] + : [], + ); +} + +function findFirstOwnValue(record: Record, keys: string[]) { + for (const key of keys) { + if (Object.prototype.hasOwnProperty.call(record, key)) { + return { found: true, value: record[key] }; + } + } + return { found: false, value: undefined }; +} + +export function parseReplacementLayoutLockRegions( + textOrRecord: string | Record, +): ReplacementLayoutLockRegions { + const record = + typeof textOrRecord === "string" ? extractJsonRecord(textOrRecord) : textOrRecord; + if (!record) { + return { + subjectBoundingBox: null, + textBoundingBoxes: [], + analyzed: false, + }; + } + + const subjectValue = findFirstOwnValue(record, [ + "subjectBoundingBox", + "productBoundingBox", + "replacementBoundingBox", + ]); + const textValue = findFirstOwnValue(record, [ + "textBoundingBoxes", + "protectedTextBoundingBoxes", + "copyBoundingBoxes", + ]); + const rawTextBoxes = Array.isArray(textValue.value) ? textValue.value : []; + const textBoundingBoxes = rawTextBoxes + .map(normalizeReplacementBoundingBox) + .filter((box): box is NormalizedReplacementBox => Boolean(box)) + .slice(0, 60); + + return { + subjectBoundingBox: normalizeReplacementBoundingBox(subjectValue.value), + textBoundingBoxes, + analyzed: subjectValue.found && textValue.found, + }; +} + +function normalizedToPixelBox( + box: NormalizedReplacementBox, + width: number, + height: number, +): PixelBox { + return { + left: (box[0] / 1000) * width, + top: (box[1] / 1000) * height, + right: (box[2] / 1000) * width, + bottom: (box[3] / 1000) * height, + }; +} + +function expandPixelBox( + box: PixelBox, + expansion: { left: number; top: number; right: number; bottom: number }, + width: number, + height: number, +): PixelBox { + return { + left: Math.max(0, box.left - expansion.left), + top: Math.max(0, box.top - expansion.top), + right: Math.min(width, box.right + expansion.right), + bottom: Math.min(height, box.bottom + expansion.bottom), + }; +} + +function paintFeatheredRectangle( + mask: Buffer, + width: number, + height: number, + box: PixelBox, + feather: number, +) { + const startX = Math.max(0, Math.floor(box.left)); + const endX = Math.min(width - 1, Math.ceil(box.right)); + const startY = Math.max(0, Math.floor(box.top)); + const endY = Math.min(height - 1, Math.ceil(box.bottom)); + + for (let y = startY; y <= endY; y += 1) { + const verticalDistance = Math.min(y - box.top, box.bottom - y); + if (verticalDistance < 0) { + continue; + } + for (let x = startX; x <= endX; x += 1) { + const horizontalDistance = Math.min(x - box.left, box.right - x); + if (horizontalDistance < 0) { + continue; + } + const edgeDistance = Math.min(horizontalDistance, verticalDistance); + const alpha = Math.round(255 * Math.min(1, edgeDistance / Math.max(1, feather))); + const offset = y * width + x; + if (alpha > mask[offset]!) { + mask[offset] = alpha; + } + } + } +} + +function buildTextLockMask(params: { + boxes: NormalizedReplacementBox[]; + width: number; + height: number; + feather: number; +}) { + const mask = Buffer.alloc(params.width * params.height); + const padding = Math.max(4, Math.round(Math.min(params.width, params.height) * 0.006)); + + for (const normalizedBox of params.boxes) { + const box = normalizedToPixelBox(normalizedBox, params.width, params.height); + paintFeatheredRectangle( + mask, + params.width, + params.height, + expandPixelBox( + box, + { + left: padding + params.feather, + top: padding + params.feather, + right: padding + params.feather, + bottom: padding + params.feather, + }, + params.width, + params.height, + ), + params.feather, + ); + } + + return mask; +} + +async function addAlphaMask(params: { + buffer: Buffer; + mask: Buffer; + width: number; + height: number; +}) { + const sharp = (await import("sharp")).default; + const opaqueBuffer = await sharp(params.buffer, { failOn: "none" }) + .removeAlpha() + .png() + .toBuffer(); + return sharp(opaqueBuffer, { failOn: "none" }) + .joinChannel(params.mask, { + raw: { width: params.width, height: params.height, channels: 1 }, + }) + .png() + .toBuffer(); +} + +export async function compositeReplacementWithReferenceLocks(params: { + generatedBuffer: Buffer; + referenceBuffer: Buffer; + width: number; + height: number; + preserveReferenceOutsideSubject: boolean; + subjectBoundingBox?: NormalizedReplacementBox | null; + textBoundingBoxes?: NormalizedReplacementBox[]; +}) { + const sharp = (await import("sharp")).default; + const width = Math.max(1, Math.round(params.width)); + const height = Math.max(1, Math.round(params.height)); + const textBoundingBoxes = params.textBoundingBoxes || []; + const feather = Math.max(3, Math.min(12, Math.round(Math.min(width, height) * 0.006))); + const [generatedBuffer, referenceBuffer] = await Promise.all([ + sharp(params.generatedBuffer, { failOn: "none" }) + .rotate() + .resize(width, height, { fit: "fill" }) + .png() + .toBuffer(), + sharp(params.referenceBuffer, { failOn: "none" }) + .rotate() + .resize(width, height, { fit: "fill" }) + .png() + .toBuffer(), + ]); + + if (params.preserveReferenceOutsideSubject && params.subjectBoundingBox) { + const subjectBox = normalizedToPixelBox(params.subjectBoundingBox, width, height); + const subjectWidth = subjectBox.right - subjectBox.left; + const subjectHeight = subjectBox.bottom - subjectBox.top; + const expandedSubjectBox = expandPixelBox( + subjectBox, + { + left: subjectWidth * 0.22, + right: subjectWidth * 0.22, + top: subjectHeight * 0.035, + bottom: subjectHeight * 0.025, + }, + width, + height, + ); + const subjectMask = Buffer.alloc(width * height); + paintFeatheredRectangle(subjectMask, width, height, expandedSubjectBox, feather); + + if (textBoundingBoxes.length > 0) { + const textMask = buildTextLockMask({ + boxes: textBoundingBoxes, + width, + height, + feather, + }); + for (let index = 0; index < subjectMask.length; index += 1) { + subjectMask[index] = Math.round( + (subjectMask[index]! * (255 - textMask[index]!)) / 255, + ); + } + } + + const generatedLayer = await addAlphaMask({ + buffer: generatedBuffer, + mask: subjectMask, + width, + height, + }); + return sharp(referenceBuffer, { failOn: "none" }) + .composite([{ input: generatedLayer }]) + .png() + .toBuffer(); + } + + if (textBoundingBoxes.length === 0) { + return generatedBuffer; + } + + const textMask = buildTextLockMask({ + boxes: textBoundingBoxes, + width, + height, + feather, + }); + const referenceTextLayer = await addAlphaMask({ + buffer: referenceBuffer, + mask: textMask, + width, + height, + }); + return sharp(generatedBuffer, { failOn: "none" }) + .composite([{ input: referenceTextLayer }]) + .png() + .toBuffer(); +} diff --git a/lib/ai/replacement-product-identity.ts b/lib/ai/replacement-product-identity.ts new file mode 100644 index 000000000..178f3d207 --- /dev/null +++ b/lib/ai/replacement-product-identity.ts @@ -0,0 +1,181 @@ +export type ReplacementProductViewSummary = { + imageIndex: number; + cameraSummary: string; + visibleStructure: string; +}; + +export type ReplacementProductIdentityProfile = { + identitySummary: string; + structureSummary: string; + structuralRisks: string; + viewSummaries: ReplacementProductViewSummary[]; +}; + +function extractJsonObject(value: string) { + const source = String(value || "").trim(); + const fenced = source.match(/```(?:json)?\s*([\s\S]*?)```/i)?.[1]?.trim(); + const candidate = fenced || source; + const start = candidate.indexOf("{"); + const end = candidate.lastIndexOf("}"); + return start >= 0 && end > start ? candidate.slice(start, end + 1) : null; +} + +function normalizeText(value: unknown, maxLength: number) { + return String(value || "") + .trim() + .replace(/\s+/g, " ") + .slice(0, maxLength); +} + +export function normalizeReplacementProductImageIndexes( + value: unknown, + productImageCount: number, + maxSelected = 2, +) { + if (!Array.isArray(value)) return []; + + const safeCount = Math.max(0, Math.floor(productImageCount)); + const safeLimit = Math.max(1, Math.floor(maxSelected)); + const selected: number[] = []; + + value.forEach((item) => { + const index = Math.round(Number(item)); + if ( + Number.isFinite(index) && + index >= 1 && + index <= safeCount && + !selected.includes(index) + ) { + selected.push(index); + } + }); + + return selected.slice(0, safeLimit); +} + +export function parseReplacementProductIdentityProfile( + value: string, + productImageCount: number, +): ReplacementProductIdentityProfile | null { + const jsonText = extractJsonObject(value); + if (!jsonText) return null; + + try { + const parsed = JSON.parse(jsonText) as Record; + const rawViews = Array.isArray(parsed.viewSummaries) + ? parsed.viewSummaries + : Array.isArray(parsed.views) + ? parsed.views + : []; + const viewSummaries = rawViews + .map((value) => { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + const record = value as Record; + const [imageIndex] = normalizeReplacementProductImageIndexes( + [record.imageIndex ?? record.index], + productImageCount, + 1, + ); + if (!imageIndex) return null; + return { + imageIndex, + cameraSummary: normalizeText( + record.cameraSummary ?? record.camera ?? record.view, + 120, + ), + visibleStructure: normalizeText( + record.visibleStructure ?? record.visibleFeatures ?? record.structure, + 180, + ), + } satisfies ReplacementProductViewSummary; + }) + .filter((value): value is ReplacementProductViewSummary => Boolean(value)) + .filter( + (value, index, values) => + values.findIndex((candidate) => candidate.imageIndex === value.imageIndex) === index, + ) + .slice(0, Math.max(0, Math.floor(productImageCount))); + + const identitySummary = normalizeText( + parsed.identitySummary ?? parsed.productSummary ?? parsed.identity, + 260, + ); + const structureSummary = normalizeText( + parsed.structureSummary ?? parsed.structuralInvariants ?? parsed.structure, + 520, + ); + const structuralRisks = normalizeText( + parsed.structuralRisks ?? parsed.forbiddenMutations ?? parsed.risks, + 320, + ); + + if (!identitySummary && !structureSummary && viewSummaries.length === 0) { + return null; + } + + return { + identitySummary, + structureSummary, + structuralRisks, + viewSummaries, + }; + } catch { + return null; + } +} + +export function buildReplacementProductIdentityAnalysisPrompt(productImageCount: number) { + const safeCount = Math.max(1, Math.floor(productImageCount)); + return [ + "你是商品结构身份分析器。下面按顺序提供同一商品的多角度产品图。", + `共有 ${safeCount} 张图，图片编号从 1 到 ${safeCount}。`, + "只分析商品本身，不分析透明背景、白边、阴影或后期抠图瑕疵。", + "identitySummary：概括商品类别、整体轮廓、材质、颜色和稳定识别特征。", + "structureSummary：列出不可改变的几何拓扑，包括构件数量、左右对应关系、腿/支架/横梁/侧框/层板/把手/转轴的形状、粗细、连接点、开口方向、排列和比例。不要把不同视角下的同一构件误判为不同设计。", + "structuralRisks：列出最容易被生成模型误画的结构，以及必须禁止的新增、删除、弯曲、分叉、熔合、悬空、错接、镜像和左右不一致。", + "viewSummaries：逐图说明拍摄角度、可见面和最清楚展示的结构，imageIndex 必须使用 1 开始的原始图片编号。", + "对于桌、椅、柜架、置物架等家具，必须明确腿型和框架拓扑，例如口字型、K/X/A/V 型或直腿；不得含糊写成‘保持结构一致’。", + "只返回严格 JSON，不要解释。", + '{"identitySummary":"不超过100字","structureSummary":"不超过220字","structuralRisks":"不超过140字","viewSummaries":[{"imageIndex":1,"cameraSummary":"不超过50字","visibleStructure":"不超过80字"}]}', + ].join("\n"); +} + +export function formatReplacementProductIdentityProfile( + profile: ReplacementProductIdentityProfile, +) { + return [ + profile.identitySummary ? `商品身份：${profile.identitySummary}` : "", + profile.structureSummary ? `结构拓扑：${profile.structureSummary}` : "", + profile.structuralRisks ? `高风险误画：${profile.structuralRisks}` : "", + ...profile.viewSummaries.map( + (view) => + `产品图${view.imageIndex}：${[view.cameraSummary, view.visibleStructure] + .filter(Boolean) + .join("；")}`, + ), + ] + .filter(Boolean) + .join("\n"); +} + +export function buildReplacementProductStructureLockLines(params: { + productImageCount: number; + identitySummary?: string | null; + structureSummary?: string | null; + structuralRisks?: string | null; +}) { + if (params.productImageCount <= 0) return []; + + return [ + "【产品结构身份锁（高于旧详情图中的旧产品结构）】", + params.identitySummary ? `目标商品身份：${params.identitySummary}` : "", + params.structureSummary ? `目标商品结构拓扑：${params.structureSummary}` : "", + params.structuralRisks ? `重点防错：${params.structuralRisks}` : "", + "前面的产品输入图共同定义要换入的新产品；最后一张详情底板只决定新产品在页面中的拍摄角度、朝向、占比、位置和透视，绝不决定新产品的腿型、支撑杆拓扑、构件数量或连接方式。", + "必须逐项核对商品主体外轮廓、构件数量、左右对应关系、粗细、直线/曲线方向、开口方向，以及腿、支架、横梁、侧框、层板、把手和转轴的真实连接点；所有可见构件必须连续、合理承重且前后一致。", + "对于桌、椅、柜架和置物架，严禁把口字型框架改成 K/X/A/V 型或相反，严禁新增、删除、弯曲、分叉、熔合、悬空、错接、镜像任何腿和横梁，严禁左右两侧生成不同结构。", + "旧详情图里的旧商品结构必须完整移除，不得与新产品结构混合。结构准确优先于风格化和无痕融合；无法兼顾时宁可减少改动，也不能改造产品结构。", + ].filter(Boolean); +} diff --git a/lib/db/migrations/0099_set_detail_replacement_qwen_image3.sql b/lib/db/migrations/0099_set_detail_replacement_qwen_image3.sql new file mode 100644 index 000000000..40e4ba8fb --- /dev/null +++ b/lib/db/migrations/0099_set_detail_replacement_qwen_image3.sql @@ -0,0 +1,10 @@ +INSERT INTO "SystemSettings" ("key", "value", "updatedAt") +VALUES ( + 'detail_replacement_image_model_config', + '{"modelId":"qwen-image-3.0"}'::jsonb, + now() +) +ON CONFLICT ("key") DO UPDATE +SET + "value" = EXCLUDED."value", + "updatedAt" = EXCLUDED."updatedAt"; diff --git a/lib/db/migrations/meta/_journal.json b/lib/db/migrations/meta/_journal.json index 0aba30ca2..06c56f056 100644 --- a/lib/db/migrations/meta/_journal.json +++ b/lib/db/migrations/meta/_journal.json @@ -596,6 +596,13 @@ "when": 1786671736000, "tag": "0098_add_canvas_layout_replace_batches", "breakpoints": true + }, + { + "idx": 83, + "version": "7", + "when": 1788220800000, + "tag": "0099_set_detail_replacement_qwen_image3", + "breakpoints": true } ] } diff --git a/playwright.helpers.config.ts b/playwright.helpers.config.ts index bfb112431..6b26eb94d 100644 --- a/playwright.helpers.config.ts +++ b/playwright.helpers.config.ts @@ -16,6 +16,8 @@ export default defineConfig({ /routes\/canvas-runtime-internal-endpoint\.test\.ts/, /routes\/canvas-layout-replace-entry\.test\.ts/, /routes\/model-classification\.test\.ts/, + /routes\/replacement-layout-lock\.test\.ts/, + /routes\/replacement-product-identity\.test\.ts/, /routes\/model3d-style-pro-regressions\.test\.ts/, /routes\/clawd-request-lifecycle\.test\.ts/, /routes\/canvas-skill-tag-order\.test\.ts/, diff --git a/tests/routes/agent-main-ai-template-entry.test.ts b/tests/routes/agent-main-ai-template-entry.test.ts index 04b46480a..2fd0c5286 100644 --- a/tests/routes/agent-main-ai-template-entry.test.ts +++ b/tests/routes/agent-main-ai-template-entry.test.ts @@ -9,6 +9,25 @@ function readSource(path: string) { } test.describe("agent main-image AI template entry", () => { + test("keeps dynamic result cards painted while the workspace scrolls", () => { + const workspace = readSource("components/agents/AgentGenerationWorkspaceDarkV2.tsx"); + const resultSectionStart = workspace.indexOf('id="agent-anchor-results"'); + const resultSectionEnd = workspace.indexOf(" { const workspace = readSource("components/agents/AgentGenerationWorkspaceDarkV2.tsx"); const dialog = readSource("components/agents/AgentAiTemplateDialog.tsx"); diff --git a/tests/routes/replacement-layout-lock.test.ts b/tests/routes/replacement-layout-lock.test.ts new file mode 100644 index 000000000..d318594e8 --- /dev/null +++ b/tests/routes/replacement-layout-lock.test.ts @@ -0,0 +1,154 @@ +import { expect, test } from "@playwright/test"; +import sharp from "sharp"; +import { + buildReplacementInputBoundingBoxList, + compositeReplacementWithReferenceLocks, + expandReplacementSubjectBoundingBoxForGeneration, + parseReplacementLayoutLockRegions, +} from "@/lib/ai/replacement-layout-lock"; + +test("parses normalized subject and protected text regions", () => { + const result = parseReplacementLayoutLockRegions(` + \`\`\`json + { + "subjectBoundingBox": [200, 250, 800, 900], + "textBoundingBoxes": [[100, 80, 700, 160], [20, 20, 20, 40], [-20, 900, 1100, 980]] + } + \`\`\` + `); + + expect(result).toEqual({ + subjectBoundingBox: [200, 250, 800, 900], + textBoundingBoxes: [ + [100, 80, 700, 160], + [0, 900, 1000, 980], + ], + analyzed: true, + }); +}); + +test("marks incomplete layout analysis for a safe fallback", () => { + expect( + parseReplacementLayoutLockRegions({ + subjectBoundingBox: [200, 250, 800, 900], + }), + ).toEqual({ + subjectBoundingBox: [200, 250, 800, 900], + textBoundingBoxes: [], + analyzed: false, + }); +}); + +test("expands the replacement subject horizontally and aligns it with the layout input", () => { + expect( + expandReplacementSubjectBoundingBoxForGeneration([379, 240, 628, 917]), + ).toEqual([354, 230, 653, 927]); + expect( + buildReplacementInputBoundingBoxList({ + imageUrls: [ + "https://example.com/product.png", + "https://example.com/layout.png", + ], + referenceImageUrl: "https://example.com/layout.png", + subjectBoundingBox: [379, 240, 628, 917], + }), + ).toEqual([[], [[354, 230, 653, 927]]]); +}); + +test("keeps the reference outside the subject and inside protected text", async () => { + const width = 100; + const height = 100; + const reference = await sharp({ + create: { + width, + height, + channels: 3, + background: { r: 255, g: 0, b: 0 }, + }, + }) + .png() + .toBuffer(); + const generated = await sharp({ + create: { + width, + height, + channels: 3, + background: { r: 0, g: 0, b: 255 }, + }, + }) + .png() + .toBuffer(); + + const result = await compositeReplacementWithReferenceLocks({ + generatedBuffer: generated, + referenceBuffer: reference, + width, + height, + preserveReferenceOutsideSubject: true, + subjectBoundingBox: [300, 300, 700, 700], + textBoundingBoxes: [[450, 450, 550, 550]], + }); + const { data, info } = await sharp(result) + .removeAlpha() + .raw() + .toBuffer({ resolveWithObject: true }); + const pixel = (x: number, y: number) => + Array.from( + data.subarray( + (y * width + x) * info.channels, + (y * width + x) * info.channels + 3, + ), + ); + + expect(pixel(5, 5)).toEqual([255, 0, 0]); + expect(pixel(35, 50)).toEqual([0, 0, 255]); + expect(pixel(50, 50)).toEqual([255, 0, 0]); +}); + +test("restores original text pixels while retaining a regenerated scene", async () => { + const width = 100; + const height = 100; + const reference = await sharp({ + create: { + width, + height, + channels: 3, + background: { r: 255, g: 0, b: 0 }, + }, + }) + .png() + .toBuffer(); + const generated = await sharp({ + create: { + width, + height, + channels: 3, + background: { r: 0, g: 0, b: 255 }, + }, + }) + .png() + .toBuffer(); + + const result = await compositeReplacementWithReferenceLocks({ + generatedBuffer: generated, + referenceBuffer: reference, + width, + height, + preserveReferenceOutsideSubject: false, + textBoundingBoxes: [[400, 400, 600, 600]], + }); + const { data, info } = await sharp(result) + .removeAlpha() + .raw() + .toBuffer({ resolveWithObject: true }); + const pixel = (x: number, y: number) => + Array.from( + data.subarray( + (y * width + x) * info.channels, + (y * width + x) * info.channels + 3, + ), + ); + + expect(pixel(5, 5)).toEqual([0, 0, 255]); + expect(pixel(50, 50)).toEqual([255, 0, 0]); +}); diff --git a/tests/routes/replacement-product-identity.test.ts b/tests/routes/replacement-product-identity.test.ts new file mode 100644 index 000000000..85aa2bbd5 --- /dev/null +++ b/tests/routes/replacement-product-identity.test.ts @@ -0,0 +1,79 @@ +import { expect, test } from "@playwright/test"; +import { + DEFAULT_DETAIL_REPLACEMENT_IMAGE_MODEL_ID, + normalizeDetailReplacementImageModelConfig, +} from "@/lib/ai/detail-replacement-image-model-config"; +import { + buildReplacementProductStructureLockLines, + normalizeReplacementProductImageIndexes, + parseReplacementProductIdentityProfile, +} from "@/lib/ai/replacement-product-identity"; + +test("parses a reusable multi-view product structure profile", () => { + const profile = parseReplacementProductIdentityProfile( + ` + \`\`\`json + { + "identitySummary": "胡桃木桌面与黑色钢架电脑桌", + "structureSummary": "左右均为闭合口字型侧框，桌面下方由三组矩形横梁连接", + "structuralRisks": "禁止改成 K 型或 X 型腿，禁止横梁断裂或错接", + "viewSummaries": [ + {"imageIndex": 4, "cameraSummary": "右前方三分之四视角", "visibleStructure": "右侧框和全部纵向横梁清楚"}, + {"imageIndex": 1, "cameraSummary": "左前方三分之四视角", "visibleStructure": "左侧框和桌面比例清楚"}, + {"imageIndex": 9, "cameraSummary": "无效编号", "visibleStructure": "应被过滤"} + ] + } + \`\`\` + `, + 4, + ); + + expect(profile).toEqual({ + identitySummary: "胡桃木桌面与黑色钢架电脑桌", + structureSummary: "左右均为闭合口字型侧框，桌面下方由三组矩形横梁连接", + structuralRisks: "禁止改成 K 型或 X 型腿，禁止横梁断裂或错接", + viewSummaries: [ + { + imageIndex: 4, + cameraSummary: "右前方三分之四视角", + visibleStructure: "右侧框和全部纵向横梁清楚", + }, + { + imageIndex: 1, + cameraSummary: "左前方三分之四视角", + visibleStructure: "左侧框和桌面比例清楚", + }, + ], + }); +}); + +test("keeps only two valid, unique, one-based product image indexes", () => { + expect(normalizeReplacementProductImageIndexes([4, "1", 4, 0, 7], 4)).toEqual([ + 4, + 1, + ]); +}); + +test("makes product topology authoritative over the old detail product", () => { + const prompt = buildReplacementProductStructureLockLines({ + productImageCount: 2, + structureSummary: "左右均为口字型矩形侧框", + structuralRisks: "禁止变成 K 型或 X 型腿", + }).join("\n"); + + expect(prompt).toContain("产品结构身份锁"); + expect(prompt).toContain("最后一张详情底板只决定"); + expect(prompt).toContain("绝不决定新产品的腿型"); + expect(prompt).toContain("严禁把口字型框架改成 K/X/A/V 型"); + expect(prompt).toContain("旧商品结构必须完整移除"); +}); + +test("keeps non-Pro detail replacement bound to Qwen Image 3 in database config", () => { + expect(DEFAULT_DETAIL_REPLACEMENT_IMAGE_MODEL_ID).toBe("qwen-image-3.0"); + expect(normalizeDetailReplacementImageModelConfig(null)).toEqual({ + modelId: "qwen-image-3.0", + }); + expect( + normalizeDetailReplacementImageModelConfig({ modelId: "custom-image-model" }), + ).toEqual({ modelId: "custom-image-model" }); +});