From 19dc43049d77f6a997e5a952b1f04836aba6f560 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 2 Jun 2026 16:34:05 +0800 Subject: [PATCH] fix: guard detail replacement prompt routing --- .../api/chat/_lib/detail-replacement.ts | 318 +++++++++++++++++- 1 file changed, 303 insertions(+), 15 deletions(-) diff --git a/app/(chat)/api/chat/_lib/detail-replacement.ts b/app/(chat)/api/chat/_lib/detail-replacement.ts index b88049875..d52bcaecd 100644 --- a/app/(chat)/api/chat/_lib/detail-replacement.ts +++ b/app/(chat)/api/chat/_lib/detail-replacement.ts @@ -49,6 +49,16 @@ export type ReplacementReferenceGeometry = { aspectRatio: string; }; +export type ReplacementReferenceVisualProfile = { + width: number; + height: number; + nearWhitePixelRatio: number; + lightLowSaturationPixelRatio: number; + darkPixelRatio: number; + likelyWhiteLayout: boolean; + reason: string; +}; + export function parseReplacementAspectRatioValue(ratio: string) { const [width, height] = ratio.split(":").map((item) => Number(item)); if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { @@ -103,6 +113,75 @@ export async function resolveReplacementReferenceGeometry( } } +export async function resolveReplacementReferenceVisualProfile( + referenceImageUrl: string, +): Promise { + try { + const sharp = (await import("sharp")).default; + const referenceBuffer = await fetchImageBufferFromUrl(referenceImageUrl); + const { data, info } = await sharp(referenceBuffer, { failOn: "none" }) + .rotate() + .resize(160, 160, { + fit: "inside", + withoutEnlargement: true, + }) + .removeAlpha() + .raw() + .toBuffer({ resolveWithObject: true }); + const pixelCount = Math.max(1, info.width * info.height); + let nearWhiteCount = 0; + let lightLowSaturationCount = 0; + let darkCount = 0; + + for (let offset = 0; offset < data.length; offset += info.channels) { + const red = data[offset] || 0; + const green = data[offset + 1] || 0; + const blue = data[offset + 2] || 0; + const maxChannel = Math.max(red, green, blue); + const minChannel = Math.min(red, green, blue); + const luminance = (0.2126 * red + 0.7152 * green + 0.0722 * blue) / 255; + const saturationSpan = maxChannel - minChannel; + + if (red >= 238 && green >= 238 && blue >= 238) { + nearWhiteCount += 1; + } + if (luminance >= 0.82 && saturationSpan <= 42) { + lightLowSaturationCount += 1; + } + if (luminance <= 0.2) { + darkCount += 1; + } + } + + const nearWhitePixelRatio = nearWhiteCount / pixelCount; + const lightLowSaturationPixelRatio = lightLowSaturationCount / pixelCount; + const darkPixelRatio = darkCount / pixelCount; + const likelyWhiteLayout = + nearWhitePixelRatio >= 0.42 || + (nearWhitePixelRatio >= 0.3 && + lightLowSaturationPixelRatio >= 0.58 && + darkPixelRatio <= 0.18); + + return { + width: info.width, + height: info.height, + nearWhitePixelRatio, + lightLowSaturationPixelRatio, + darkPixelRatio, + likelyWhiteLayout, + reason: likelyWhiteLayout + ? "像素初筛显示大面积白底/浅色低饱和区域，按白底说明图保护" + : "像素初筛未达到白底说明图阈值", + }; + } catch (error) { + chatApiLogger.warn("[Chat API] Failed to resolve replacement reference visual profile:", { + referenceImageUrl, + error: error instanceof Error ? error.message : "unknown", + }); + return null; + } +} + export async function normalizeReplacementResultImage(params: { generatedImageUrl: string; generatedThumbnailUrl?: string; @@ -244,6 +323,11 @@ type ReplacementReferenceDecisionCore = Omit< "detectionModelName" | "detectionSource" >; +const DETAIL_WHITE_LAYOUT_HINT_PATTERN = + /白底|浅色底|留白|产品说明|参数|规格|卖点|标签|说明图|信息图|海报|版式|排版|文字|标题|编号|材质|工艺|桌腿|木蜡油|代名词|典雅|高贵/i; +const DETAIL_PEOPLE_ACTION_HINT_PATTERN = + /自拍|合照|拍照|拿手机|手机|聚会|好友|朋友|闺蜜|夫妻|情侣|家庭|亲子|陪伴|吃饭|用餐|举杯|互动|模特|人物|人像|多人|坐|站|走|逛|使用|体验/i; + export type ReplacementScenePlanningRuntime = | { kind: "local_moonshot"; @@ -544,7 +628,161 @@ function inferPeoplePolicy(params: { if (!params.containsPeople && !params.containsMultiplePeople) { return "none"; } - return params.hasModelImage ? "preserve_relation" : "require_user_model"; + return "preserve_relation"; +} + +function planTextForPolicyGuards(scenePlan: ReplacementScenePlan) { + return [ + scenePlan.routeReason, + scenePlan.sceneSummary, + scenePlan.cameraSummary, + scenePlan.placementSummary, + scenePlan.preserveSummary, + scenePlan.styleSummary, + scenePlan.avoidSummary, + scenePlan.generationPrompt, + ] + .filter(Boolean) + .join(" ") + .trim(); +} + +function appendUniqueInstruction(base: string, instruction: string, maxLength = 1200) { + const normalizedBase = String(base || "").trim(); + const normalizedInstruction = String(instruction || "").trim(); + if (!normalizedInstruction) { + return normalizedBase.slice(0, maxLength); + } + if (normalizedBase.includes(normalizedInstruction)) { + return normalizedBase.slice(0, maxLength); + } + return [normalizedBase, normalizedInstruction].filter(Boolean).join("；").slice(0, maxLength); +} + +function applyDetailReplacementScenePlanGuards( + scenePlan: ReplacementScenePlan, + params: { + modeType?: "detail_replacement" | "sku_replacement"; + hasModelImage: boolean; + hasImportantText: boolean; + containsPeople: boolean; + containsMultiplePeople: boolean; + visualProfile?: ReplacementReferenceVisualProfile | null; + replacementStrategy?: AgentDetailReplacementStrategy; + }, +): ReplacementScenePlan { + if (params.modeType !== "detail_replacement") { + return scenePlan; + } + + const isProductOnly = params.replacementStrategy === "product_only"; + const planText = planTextForPolicyGuards(scenePlan); + const hasWhiteLayoutHint = DETAIL_WHITE_LAYOUT_HINT_PATTERN.test(planText); + const hasPeopleActionHint = DETAIL_PEOPLE_ACTION_HINT_PATTERN.test(planText); + const shouldProtectWhiteLayout = + !isProductOnly && + (scenePlan.whiteBackgroundLikely || + params.visualProfile?.likelyWhiteLayout === true || + hasWhiteLayoutHint); + const shouldProtectPeople = + !isProductOnly && + (scenePlan.peoplePolicy !== "none" || + params.containsPeople || + params.containsMultiplePeople || + hasPeopleActionHint); + const shouldLockCopy = + !isProductOnly && + (scenePlan.copyImageCoupled || + params.hasImportantText || + scenePlan.textPolicy === "lock" || + hasWhiteLayoutHint); + + let next: ReplacementScenePlan = { ...scenePlan }; + + if (shouldProtectPeople) { + next = { + ...next, + generationRoute: "people_relation_locked", + promptPackPolicy: "forbid", + textPolicy: next.textPolicy === "free" ? "preserve" : next.textPolicy, + peoplePolicy: "preserve_relation", + useModelImage: params.hasModelImage, + modelImageReason: params.hasModelImage + ? next.modelImageReason || + "当前参考页存在人物/自拍/互动需求，必须使用用户模特图保持人物身份与动作关系。" + : next.modelImageReason || + "当前参考页存在人物/自拍/互动需求；未提供用户模特图时仍需保留参考页人物数量、动作和关系，但不要复刻特定身份。", + routeReason: appendUniqueInstruction( + next.routeReason, + "命中人物/自拍/互动语义，禁止按普通场景词包重建。", + 220, + ), + generationPrompt: appendUniqueInstruction( + next.generationPrompt, + "如果参考图文字或画面包含自拍、合照、聚会、用餐、互动、拿手机等动作语义，人物数量、相对位置、动作方向和出镜比例是硬约束；结果必须出现对应人物动作，不允许生成空场景。", + ), + avoidSummary: appendUniqueInstruction( + next.avoidSummary, + "不要把有人物/自拍动作的参考页生成成无人物空间，不要把多人压成单人，不要忽略参考页文字描述的人物动作。", + 260, + ), + }; + } else if (shouldLockCopy) { + next = { + ...next, + generationRoute: shouldProtectWhiteLayout + ? "layout_preserve_replace" + : "copy_locked_regenerate", + promptPackPolicy: "forbid", + textPolicy: "lock", + peoplePolicy: "none", + useModelImage: false, + routeReason: appendUniqueInstruction( + next.routeReason, + shouldProtectWhiteLayout + ? "命中白底/说明图或强文案版式，禁止套用场景词包。" + : "命中强图文绑定，禁止套用场景词包。", + 220, + ), + generationPrompt: appendUniqueInstruction( + next.generationPrompt, + shouldProtectWhiteLayout + ? "本屏按白底产品说明/参数排版处理：严格保留白底、留白、标题、标签、参数、文字颜色层级和产品位置，只做必要的产品/材质/视觉区替换，不允许生成室内场景、厨房、客厅、人物或软装背景。" + : "本屏文案与画面强绑定：文字内容、颜色、位置、层级和对应画面语义必须一致，只做必要局部替换，不允许按前置词包改造成另一套场景。", + ), + avoidSummary: appendUniqueInstruction( + next.avoidSummary, + "不要把白底说明页场景化，不要新增厨房/客厅/人物/软装，不要让文案和图片失配，不要把文字拆成多种颜色。", + 260, + ), + whiteBackgroundLikely: next.whiteBackgroundLikely || shouldProtectWhiteLayout, + copyImageCoupled: true, + }; + } else if (shouldProtectWhiteLayout) { + next = { + ...next, + generationRoute: "layout_preserve_replace", + promptPackPolicy: "forbid", + textPolicy: next.textPolicy === "free" ? "preserve" : next.textPolicy, + useModelImage: false, + routeReason: appendUniqueInstruction( + next.routeReason, + "命中白底/浅色说明图，按版式保护处理。", + 220, + ), + generationPrompt: appendUniqueInstruction( + next.generationPrompt, + "本屏是白底/浅色产品说明版式：不要套用空间、人群、灯光、机位等前置词包，只保留参考图白底排版并做必要局部替换。", + ), + whiteBackgroundLikely: true, + }; + } + + if (params.visualProfile?.likelyWhiteLayout && !next.routeReason.includes("像素初筛")) { + next.routeReason = appendUniqueInstruction(next.routeReason, params.visualProfile.reason, 220); + } + + return next; } export function parseReplacementScenePlan( @@ -552,7 +790,9 @@ export function parseReplacementScenePlan( params: { productImageCount: number; hasModelImage: boolean; + modeType?: "detail_replacement" | "sku_replacement"; replacementStrategy?: AgentDetailReplacementStrategy; + visualProfile?: ReplacementReferenceVisualProfile | null; }, ): ReplacementScenePlan | null { const jsonText = extractJsonObjectText(text); @@ -637,7 +877,7 @@ export function parseReplacementScenePlan( ); const productImageSelection = buildDefaultProductImageSelection(params.productImageCount); - const scenePlan: ReplacementScenePlan = { + let scenePlan: ReplacementScenePlan = { generationRoute, promptPackPolicy, textPolicy, @@ -697,6 +937,16 @@ export function parseReplacementScenePlan( .slice(0, 1200); } + scenePlan = applyDetailReplacementScenePlanGuards(scenePlan, { + modeType: params.modeType, + hasModelImage: params.hasModelImage, + hasImportantText, + containsPeople, + containsMultiplePeople, + visualProfile: params.visualProfile, + replacementStrategy: params.replacementStrategy, + }); + return scenePlan; } catch (error) { chatApiLogger.warn("[Chat API] Failed to parse replacement scene plan JSON:", error); @@ -705,30 +955,40 @@ export function parseReplacementScenePlan( } export function buildHeuristicReplacementScenePlan(params: { + modeType?: "detail_replacement" | "sku_replacement"; productImageCount: number; hasModelImage: boolean; shouldReplace: boolean; reason: string; replacementStrategy?: AgentDetailReplacementStrategy; + visualProfile?: ReplacementReferenceVisualProfile | null; }): ReplacementScenePlan { + const isDetailReplacement = params.modeType === "detail_replacement"; const isProductOnly = params.replacementStrategy === "product_only"; const productImageSelection = buildDefaultProductImageSelection(params.productImageCount); const generationRoute = params.shouldReplace ? isProductOnly ? "layout_preserve_replace" + : isDetailReplacement + ? "layout_preserve_replace" : "prompt_pack_scene" : "direct_passthrough"; - return { + const scenePlan: ReplacementScenePlan = { generationRoute, promptPackPolicy: inferPromptPackPolicy(generationRoute), - textPolicy: generationRoute === "direct_passthrough" ? "preserve" : "free", + textPolicy: + generationRoute === "direct_passthrough" || isDetailReplacement + ? "preserve" + : "free", peoplePolicy: "none", - whiteBackgroundLikely: false, + whiteBackgroundLikely: Boolean(params.visualProfile?.likelyWhiteLayout), copyImageCoupled: false, routeReason: params.shouldReplace ? isProductOnly ? "兜底按仅替换产品执行，不使用词包。" - : "兜底未检测到强约束，允许使用词包生成场景。" + : isDetailReplacement + ? "兜底按详情版式保护执行，不使用词包，避免白底/强文案/人物页被误场景化。" + : "兜底未检测到强约束，允许使用词包生成场景。" : params.reason, selectedProductImageIndexes: productImageSelection.selectedProductImageIndexes, productSelectionReason: productImageSelection.productSelectionReason, @@ -759,11 +1019,23 @@ export function buildHeuristicReplacementScenePlan(params: { generationPrompt: params.shouldReplace ? isProductOnly ? `仅替换参考图中的产品主体，严格保留其余背景、文案、排版与光影关系。${params.reason}` + : isDetailReplacement + ? `保留参考图的版式、文案、人物关系、白底/留白和图文语义，不使用前置词包；只在产品、材质、板材、原材料或视觉图片区做必要替换。${params.reason}` : `以工作流目标风格重建当前这一屏的场景，只保留版式、机位、文字区与留白结构。${params.reason}` : params.reason, analysisModelName: null, analysisSource: "heuristic", }; + + return applyDetailReplacementScenePlanGuards(scenePlan, { + modeType: params.modeType, + hasModelImage: params.hasModelImage, + hasImportantText: isDetailReplacement, + containsPeople: false, + containsMultiplePeople: false, + visualProfile: params.visualProfile, + replacementStrategy: params.replacementStrategy, + }); } export function normalizeAgentModeReferences( @@ -1202,6 +1474,9 @@ export async function analyzeReplacementExecutionPlan(params: { const isDetailReplacement = params.modeType === "detail_replacement"; const isProductOnlyDetailReplacement = isDetailReplacement && params.replacementStrategy === "product_only"; + const referenceVisualProfile = isDetailReplacement + ? await resolveReplacementReferenceVisualProfile(params.referenceImageUrl) + : null; if (params.planningRuntime) { try { const replacementScenePlanningTimeoutMs = @@ -1233,6 +1508,12 @@ export async function analyzeReplacementExecutionPlan(params: { : "你是电商详情替换流程中的“场景理解规划器”。", `当前要分析的参考图：${params.referenceLabel}`, "你现在只分析这一张参考图本身，不分析产品图，不分析模特图。", + referenceVisualProfile + ? `像素初筛：nearWhite=${referenceVisualProfile.nearWhitePixelRatio.toFixed(2)}, lightLowSaturation=${referenceVisualProfile.lightLowSaturationPixelRatio.toFixed(2)}, dark=${referenceVisualProfile.darkPixelRatio.toFixed(2)}, likelyWhiteLayout=${referenceVisualProfile.likelyWhiteLayout ? "true" : "false"}。${referenceVisualProfile.reason}` + : "", + referenceVisualProfile?.likelyWhiteLayout + ? "像素初筛已经命中白底/浅色说明页：除非你在画面中看到明确的大面积真实生活场景，否则 generationRoute 必须是 layout_preserve_replace 或 copy_locked_regenerate，promptPackPolicy 必须 forbid。" + : "", "产品图和模特图会在后续生图阶段单独提供给生图模型使用，你这里不要尝试理解它们的具体外观，也不要要求必须使用所有输入图。", params.modelImageUrl && !isProductOnlyDetailReplacement ? "你只需要根据当前参考图判断：这一屏是否存在真实的人物展示需求；如果没有人物需求，useModelImage 必须为 false。" @@ -1263,13 +1544,13 @@ export async function analyzeReplacementExecutionPlan(params: { ? "3. 判断 generationRoute：prompt_pack_scene=通用场景图且可吃词包；layout_preserve_replace=锁定版式/白底/产品说明/材质说明，只做有限替换；copy_locked_regenerate=文案与画面强绑定，必须锁文案；people_relation_locked=多人/模特/互动关系图，必须锁人物关系。详情替换不允许 direct_passthrough。" : "3. 提炼 sceneSummary、cameraSummary、placementSummary、preserveSummary、styleSummary、avoidSummary。", isDetailReplacement - ? "4. 判断 promptPackPolicy：只有通用空间场景且没有强文案、白底、多人关系、词图强绑定限制时才 allow；白底图、强文案图、多人关系图、图文匹配强的详情页必须 forbid。" + ? "4. 判断 promptPackPolicy：只有通用空间场景且没有强文案、白底、多人关系、自拍/动作文字、词图强绑定限制时才 allow；白底图、产品说明图、强文案图、多人关系图、自拍/人物动作图、图文匹配强的详情页必须 forbid。" : "", isDetailReplacement ? "5. 判断 textPolicy：画面中存在营销标题、卖点、编号、参数、标签且与图匹配时必须 lock 或 preserve，避免后续生图改字、混色或重排。" : "", isDetailReplacement - ? "6. 判断 peoplePolicy：有人物/多人/自拍/聚会/模特互动时必须 preserve_relation；如果只提供 1 张模特图但参考图多人，仍要保留参考图人数和关系，额外人物按参考图生成相近人群；如果没有提供用户模特图但参考图有人物身份，不允许把参考人物身份当成用户模特直接迁移。" + ? "6. 判断 peoplePolicy：有人物/多人/自拍/聚会/模特互动/拿手机/用餐动作时必须 preserve_relation；如果文字明确写了“来张自拍、合照、聚会、用餐、互动”等动作，generationPrompt 必须把这个动作写成硬约束；如果只提供 1 张模特图但参考图多人，仍要保留参考图人数和关系，额外人物按参考图生成相近人群。" : "", isDetailReplacement ? "7. 提炼 sceneSummary、cameraSummary、placementSummary、preserveSummary、styleSummary、avoidSummary。" @@ -1330,12 +1611,14 @@ export async function analyzeReplacementExecutionPlan(params: { const parsedDecision = parseReplacementReferenceDecision(text); const parsedScenePlan = parseReplacementScenePlan(text, { - productImageCount: params.productImageUrls.length, - hasModelImage: isProductOnlyDetailReplacement - ? false - : Boolean(params.modelImageUrl), - replacementStrategy: params.replacementStrategy, - }); + productImageCount: params.productImageUrls.length, + hasModelImage: isProductOnlyDetailReplacement + ? false + : Boolean(params.modelImageUrl), + modeType: params.modeType, + replacementStrategy: params.replacementStrategy, + visualProfile: referenceVisualProfile, + }); if (parsedDecision && parsedScenePlan) { const normalizedPlan = ensureDetailReferenceGenerates( @@ -1380,6 +1663,7 @@ export async function analyzeReplacementExecutionPlan(params: { replacementStrategy: params.replacementStrategy, }); const fallbackScenePlan = buildHeuristicReplacementScenePlan({ + modeType: params.modeType, productImageCount: params.productImageUrls.length, hasModelImage: isProductOnlyDetailReplacement ? false @@ -1387,6 +1671,7 @@ export async function analyzeReplacementExecutionPlan(params: { shouldReplace: fallbackDecision.shouldReplace, reason: fallbackDecision.reason, replacementStrategy: params.replacementStrategy, + visualProfile: referenceVisualProfile, }); const normalizedFallbackPlan = ensureDetailReferenceGenerates( fallbackDecision, @@ -1537,6 +1822,9 @@ export function buildAgentReplacementPrompt(params: { params.scenePlan.peoplePolicy === "preserve_relation" && params.hasModelImage ? "若当前参考图为多人/群体/互动关系图，即使只提供 1 张模特图，也必须保留参考图人数、相对位置、动作和关系；模特图作为主人物身份/气质参考，其余人物按参考图生成相近人群。" : "", + params.scenePlan.peoplePolicy === "preserve_relation" + ? "人物动作硬约束：如果当前参考图文字或画面包含自拍、合照、聚会、用餐、拿手机、使用产品、人与人互动等语义，生成结果必须出现对应人物数量、动作、朝向和互动关系；不允许生成无人空场景。" + : "", params.scenePlan.generationPrompt ? `本屏生成方向：${params.scenePlan.generationPrompt}` : "", @@ -1597,7 +1885,7 @@ export function buildAgentReplacementPrompt(params: { params.scenePlan?.peoplePolicy === "require_user_model" ? "本屏包含人物但没有提供用户模特图：禁止把参考图中的人物身份当作用户模特迁移；只能保留人物数量、关系、动作和构图语义，不要换成不相关强人物主体。" : params.scenePlan?.peoplePolicy === "preserve_relation" - ? "本屏包含人物关系：必须保留人物数量、相对位置、互动关系、动作方向和出镜比例，不要把多人图改成单人图或无关模特图。" + ? "本屏包含人物关系：必须保留人物数量、相对位置、互动关系、动作方向和出镜比例；如果参考图文字写了自拍/合照/聚会/用餐/拿手机等动作，必须让人物执行该动作，不要把多人图改成单人图、无人图或无关模特图。" : "如果参考图没有明确人物展示需求，不要凭空新增人物主体。", params.scenePlan?.whiteBackgroundLikely ? "本屏疑似白底/说明图：禁止场景化、生活化或套空间词包，只允许保留白底/说明图语义并做必要的局部替换。"