feat: add scene-aware detail replacement planning
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled

This commit is contained in:
Your Name 2026-03-23 21:13:20 +08:00
parent 53f8cd7052
commit d2aa27b1c2

View File

@ -44,6 +44,8 @@ import {
RmbgServiceError,
} from "@/lib/ai/rmbg/service";
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
import { resolveDetailAgentLocalMoonshotModel } from "@/lib/ai/detail-agent-model-runtime";
import { inlineVisionImages } from "@/lib/ai/vision-inline";
import { createDocument } from "@/lib/ai/tools/create-document";
import { getWeather } from "@/lib/ai/tools/get-weather";
import { requestSuggestions } from "@/lib/ai/tools/request-suggestions";
@ -83,6 +85,7 @@ import {
getAccessibleAgentPresetById,
getConversationWorkingStateByChatId,
getMediaAssetsByUserAndUrls,
getSiteSettings,
upsertMediaAsset,
} from "@/lib/db/queries";
import { db } from "@/lib/db";
@ -2897,13 +2900,188 @@ type ReplacementReferenceDecision = {
reason: string;
detectionModelName?: string | null;
detectionSource:
| "local_moonshot"
| "configured_vision"
| "configured_qwen3_vl"
| "fallback_qwen3_vl"
| "safe_passthrough";
};
type ReplacementScenePlan = {
selectedProductImageIndexes: number[];
productSelectionReason: string;
useModelImage: boolean;
modelImageReason: string;
sceneSummary: string;
cameraSummary: string;
placementSummary: string;
preserveSummary: string;
styleSummary: string;
avoidSummary: string;
generationPrompt: string;
analysisModelName?: string | null;
analysisSource: "local_moonshot" | "configured_vision" | "heuristic";
};
type ReplacementExecutionPlan = ReplacementReferenceDecision & {
scenePlan: ReplacementScenePlan;
};
type ReplacementScenePlanningRuntime =
| {
kind: "local_moonshot";
model: any;
modelName: string;
}
| {
kind: "configured_vision";
model: any;
modelName: string;
}
| null;
const QWEN_VL_MODEL_MATCHER = /(qwen|千问|通义).*(vl|vision)|qwen3-vl|qwen-vl/i;
function truncateReplacementAnalysisText(value: unknown, maxLength = 240) {
return String(value || "").trim().replace(/\s+/g, " ").slice(0, maxLength);
}
function parseReplacementScenePlan(
text: string,
params: {
productImageCount: number;
hasModelImage: boolean;
},
): ReplacementScenePlan | null {
const jsonText = extractJsonObjectText(text);
if (!jsonText) {
return null;
}
try {
const parsed = JSON.parse(jsonText) as Record<string, unknown>;
const rawIndexes = Array.isArray(parsed.selectedProductImageIndexes)
? parsed.selectedProductImageIndexes
: Array.isArray(parsed.productImageIndexes)
? parsed.productImageIndexes
: [];
const selectedProductImageIndexes = Array.from(
new Set(
rawIndexes
.map((value) => Number(value))
.filter((value) => Number.isFinite(value))
.map((value) =>
Math.max(1, Math.min(params.productImageCount, Math.round(value))),
),
),
).slice(0, 3);
const useModelImageRaw =
normalizeBooleanLike(parsed.useModelImage) ??
normalizeBooleanLike(parsed.needModelImage) ??
normalizeBooleanLike(parsed.referenceNeedsModel) ??
false;
const generationPrompt = truncateReplacementAnalysisText(
parsed.generationPrompt || parsed.recommendedScenePrompt || parsed.scenePrompt,
1200,
);
const scenePlan: ReplacementScenePlan = {
selectedProductImageIndexes:
selectedProductImageIndexes.length > 0
? selectedProductImageIndexes
: [1].filter((value) => value <= params.productImageCount),
productSelectionReason: truncateReplacementAnalysisText(
parsed.productSelectionReason || parsed.selectionReason || parsed.productReason,
160,
),
useModelImage: params.hasModelImage ? Boolean(useModelImageRaw) : false,
modelImageReason: truncateReplacementAnalysisText(
parsed.modelImageReason || parsed.modelReason,
160,
),
sceneSummary: truncateReplacementAnalysisText(
parsed.sceneSummary || parsed.sceneDescription,
220,
),
cameraSummary: truncateReplacementAnalysisText(
parsed.cameraSummary || parsed.cameraDescription,
180,
),
placementSummary: truncateReplacementAnalysisText(
parsed.placementSummary || parsed.productPlacement,
180,
),
preserveSummary: truncateReplacementAnalysisText(
parsed.preserveSummary || parsed.mustKeep || parsed.preserveInstruction,
220,
),
styleSummary: truncateReplacementAnalysisText(
parsed.styleSummary || parsed.styleInstruction || parsed.styleKeywords,
220,
),
avoidSummary: truncateReplacementAnalysisText(
parsed.avoidSummary || parsed.doNotInclude || parsed.avoidInstruction,
220,
),
generationPrompt,
analysisModelName: null,
analysisSource: "heuristic",
};
if (!scenePlan.generationPrompt) {
scenePlan.generationPrompt = [
scenePlan.sceneSummary,
scenePlan.cameraSummary,
scenePlan.placementSummary,
scenePlan.styleSummary,
]
.filter(Boolean)
.join(";")
.slice(0, 1200);
}
return scenePlan;
} catch (error) {
console.warn("[Chat API] Failed to parse replacement scene plan JSON:", error);
return null;
}
}
function buildHeuristicReplacementScenePlan(params: {
productImageCount: number;
hasModelImage: boolean;
shouldReplace: boolean;
reason: string;
}): ReplacementScenePlan {
return {
selectedProductImageIndexes: Array.from(
{ length: Math.min(Math.max(params.productImageCount, 1), 2) },
(_, index) => index + 1,
),
productSelectionReason:
params.productImageCount > 1
? "默认优先使用前两张产品图作为主视角与补充细节,其余角度不强制同时融入。"
: "默认使用当前唯一产品图作为主体参考。",
useModelImage: params.hasModelImage,
modelImageReason: params.hasModelImage
? "未完成场景级人物需求分析,先保留模特图作为可选补充参考。"
: "当前未提供模特图。",
sceneSummary: params.shouldReplace
? "保持当前参考图版式与场景结构,只替换参考图中的产品主体。"
: "当前参考图不满足替换条件。",
cameraSummary: "保持参考图原有机位、构图和景别,不做整体重画。",
placementSummary: "产品应落在参考图当前主体所在区域,不允许新增无关主体。",
preserveSummary: "文字排版、标题区、分割线、留白和信息层级必须保留。",
styleSummary: "沿用当前工作流风格设定,但只作用于产品相关场景内容。",
avoidSummary: "不要把所有输入产品角度图同时强行出现在一张图中。",
generationPrompt: params.reason,
analysisModelName: null,
analysisSource: "heuristic",
};
}
function normalizeAgentModeReferences(
payload?: PostRequestBody["agentModePayload"],
): AgentReplacementModePayload["references"] {
@ -3141,6 +3319,192 @@ async function analyzeReplacementReferenceImage(params: {
};
}
async function resolveReplacementScenePlanningRuntime(params: {
models?: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>;
}): Promise<ReplacementScenePlanningRuntime> {
try {
const siteSettings = await getSiteSettings().catch(() => null);
const localDetailAgentResolved = await resolveDetailAgentLocalMoonshotModel({
siteSettingModelId: siteSettings?.canvasDetailAgentModelId,
});
if (localDetailAgentResolved) {
return {
kind: "local_moonshot",
model: localDetailAgentResolved.model,
modelName:
localDetailAgentResolved.modelRef.displayName ||
localDetailAgentResolved.modelRef.modelId,
};
}
} catch (error) {
console.warn("[Chat API] Failed to resolve local Moonshot scene planning model:", error);
}
const configuredModels = params.models || (await getEnabledAiModelsWithProvider());
const configuredVisionModel =
configuredModels.find(
({ model }) =>
model.supportsVision &&
!model.isSubModel &&
QWEN_VL_MODEL_MATCHER.test(`${model.displayName} ${model.modelId}`),
) ||
configuredModels.find(({ model }) => model.supportsVision && !model.isSubModel);
if (!configuredVisionModel) {
return null;
}
try {
const languageModel = await getLanguageModelById(configuredVisionModel.model.id);
if (!languageModel) {
return null;
}
return {
kind: "configured_vision",
model: languageModel,
modelName:
configuredVisionModel.model.displayName ||
configuredVisionModel.model.modelId,
};
} catch (error) {
console.warn("[Chat API] Failed to resolve configured replacement scene planning model:", {
modelId: configuredVisionModel.model.id,
error,
});
return null;
}
}
async function analyzeReplacementExecutionPlan(params: {
referenceImageUrl: string;
referenceLabel: string;
productImageUrls: string[];
modelImageUrl?: string | null;
workflowPrompt?: string | null;
planningRuntime: ReplacementScenePlanningRuntime;
models?: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>;
}): Promise<ReplacementExecutionPlan> {
if (params.planningRuntime) {
try {
const analysisImageUrls = [
...params.productImageUrls,
...(params.modelImageUrl ? [params.modelImageUrl] : []),
params.referenceImageUrl,
].filter(Boolean) as string[];
const { inlineDataUrls } = await inlineVisionImages(analysisImageUrls, {
maxImages: 8,
maxSide: 900,
timeoutMs: 12000,
retries: 2,
loggerPrefix: "[Chat API] Replacement scene plan",
}).catch((error) => {
console.warn("[Chat API] Failed to inline replacement scene planning images:", error);
return { inlineDataUrls: [], skipped: [] };
});
const effectiveVisionImages =
inlineDataUrls.length > 0 ? inlineDataUrls : analysisImageUrls;
const workflowPromptExcerpt = String(params.workflowPrompt || "")
.trim()
.replace(/\s+/g, " ")
.slice(0, 2200);
const analysisPrompt = [
"你是电商详情替换流程中的“场景理解规划器”。",
`当前要分析的参考图:${params.referenceLabel}`,
`产品图数量:${params.productImageUrls.length} 张。产品图全部属于同一产品的多角度/多细节白底图。`,
"这些产品图只是帮助你理解同一产品的结构、材质、颜色、比例和细节,不代表必须把所有角度都同时体现在这一张结果图里。",
"你必须根据最后一张参考图底板当前这一屏的机位、主体位置和露出范围,只选择最需要的 1-2 张产品图作为本屏主参考;只有在确实必要时才允许选择 3 张,禁止无理由把所有产品图都选上。",
params.modelImageUrl
? "还提供了 1 张模特图。只有当最后一张参考图底板当前这一屏本身存在人物展示需求时,才允许使用模特图;如果这一屏不需要人物,必须明确 useModelImage=false。"
: "本轮没有模特图,useModelImage 必须为 false。",
"最后一张图片是唯一的详情参考图底板,它决定当前这一屏的版式、文字排版、机位、构图、留白和信息层级。",
"请完成以下任务:",
"1. 判断 shouldReplace:当前参考图里是否存在清晰、主要、适合被替换的家具商品主体。",
"2. 给出 selectedProductImageIndexes:从产品图中挑出最适合当前这一屏的产品角度索引(从1开始计数)。",
"3. 判断 useModelImage,并说明原因。",
"4. 提炼 sceneSummary、cameraSummary、placementSummary、preserveSummary、styleSummary、avoidSummary。",
"5. 输出 generationPrompt:这是一段给后续生图模型的本屏生成指令,必须明确“只按当前参考图这一屏需要使用产品角度图,不需要把所有产品图全部融进结果”。",
workflowPromptExcerpt
? `当前风格与场景限制(来自前置工作流与模式模板):\n${workflowPromptExcerpt}`
: "",
"只返回严格 JSON,不要任何额外解释。",
`JSON 格式:
{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字","selectedProductImageIndexes":[1,2],"productSelectionReason":"不超过50字","useModelImage":false,"modelImageReason":"不超过40字","sceneSummary":"不超过80字","cameraSummary":"不超过60字","placementSummary":"不超过60字","preserveSummary":"不超过80字","styleSummary":"不超过80字","avoidSummary":"不超过80字","generationPrompt":"不超过300字"}`,
]
.filter(Boolean)
.join("\n");
const { text } = await generateText({
model: params.planningRuntime.model,
messages: [
{
role: "user",
content: [
{
type: "text",
text: analysisPrompt,
},
...effectiveVisionImages.map((image) => ({
type: "image" as const,
image,
})),
],
},
],
system:
"你是高精度商品详情图场景规划器。重点是:多张产品图只是同一产品的多视角信息源,不是必须全部同时使用。你必须根据参考图当前这一屏的真实需求筛选少量最合适的产品图。",
temperature: params.planningRuntime.kind === "local_moonshot" ? 1 : 0.1,
maxTokens: 1200,
abortSignal: AbortSignal.timeout(45_000),
} as any);
const parsedDecision = parseReplacementReferenceDecision(text);
const parsedScenePlan = parseReplacementScenePlan(text, {
productImageCount: params.productImageUrls.length,
hasModelImage: Boolean(params.modelImageUrl),
});
if (parsedDecision && parsedScenePlan) {
return {
...parsedDecision,
detectionModelName: params.planningRuntime.modelName,
detectionSource:
params.planningRuntime.kind === "local_moonshot"
? "local_moonshot"
: "configured_vision",
scenePlan: {
...parsedScenePlan,
analysisModelName: params.planningRuntime.modelName,
analysisSource:
params.planningRuntime.kind === "local_moonshot"
? "local_moonshot"
: "configured_vision",
},
};
}
} catch (error) {
console.warn("[Chat API] Replacement scene planning analysis failed:", {
referenceLabel: params.referenceLabel,
referenceImageUrl: params.referenceImageUrl,
error,
});
}
}
const fallbackDecision = await analyzeReplacementReferenceImage({
imageUrl: params.referenceImageUrl,
models: params.models,
});
return {
...fallbackDecision,
scenePlan: buildHeuristicReplacementScenePlan({
productImageCount: params.productImageUrls.length,
hasModelImage: Boolean(params.modelImageUrl),
shouldReplace: fallbackDecision.shouldReplace,
reason: fallbackDecision.reason,
}),
};
}
function buildAgentReplacementPrompt(params: {
basePrompt: string;
modeType: "detail_replacement" | "sku_replacement";
@ -3149,8 +3513,10 @@ function buildAgentReplacementPrompt(params: {
totalReferences: number;
hasModelImage: boolean;
productImageCount: number;
originalProductImageCount: number;
templateReferenceCount: number;
pairedProductIndex?: number;
scenePlan?: ReplacementScenePlan | null;
}) {
const imageOrderLines: string[] = [];
let nextImageIndex = 1;
@ -3188,6 +3554,48 @@ function buildAgentReplacementPrompt(params: {
`图片${nextImageIndex}:当前参考图底板(唯一版式基准图),必须保留它的文字排版、信息区、分割线、留白、机位和整体结构。`
);
const scenePlanLines = params.scenePlan
? [
"【本屏场景预分析】",
params.scenePlan.sceneSummary
? `场景理解:${params.scenePlan.sceneSummary}`
: "",
params.scenePlan.cameraSummary
? `机位构图:${params.scenePlan.cameraSummary}`
: "",
params.scenePlan.placementSummary
? `主体位置:${params.scenePlan.placementSummary}`
: "",
params.scenePlan.preserveSummary
? `必须保留:${params.scenePlan.preserveSummary}`
: "",
params.scenePlan.styleSummary
? `风格约束:${params.scenePlan.styleSummary}`
: "",
params.scenePlan.avoidSummary
? `禁止事项:${params.scenePlan.avoidSummary}`
: "",
params.originalProductImageCount > params.productImageCount
? `原始共有 ${params.originalProductImageCount} 张产品角度图,本屏已筛选出最相关的 ${params.productImageCount} 张输入图;其余未入选产品图只是补充理解,不需要也不允许强行全部融入同一张结果图。`
: "",
params.scenePlan.productSelectionReason
? `产品图筛选理由:${params.scenePlan.productSelectionReason}`
: "",
params.scenePlan.useModelImage
? params.scenePlan.modelImageReason
? `本屏允许使用模特图:${params.scenePlan.modelImageReason}`
: "本屏允许使用模特图,但仅在参考图当前这一屏确实存在人物展示需求时使用。"
: params.hasModelImage
? params.scenePlan.modelImageReason
? `本屏禁止使用模特图:${params.scenePlan.modelImageReason}`
: "本屏不需要模特图,不允许把模特图强行融入结果。"
: "",
params.scenePlan.generationPrompt
? `本屏生成方向:${params.scenePlan.generationPrompt}`
: "",
].filter(Boolean)
: [];
const instructionLines =
params.modeType === "detail_replacement"
? [
@ -3195,6 +3603,7 @@ function buildAgentReplacementPrompt(params: {
`当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张详情参考图:${params.referenceLabel}`,
"【图片顺序规则】",
...imageOrderLines,
...scenePlanLines,
"你现在做的不是重新生成一张新图,而是编辑最后一张参考图本身。",
"如果前面商品图、模特图与最后一张参考图底板存在冲突,始终以最后一张参考图底板的版式、机位、构图和信息层级为最高优先级。",
"最后一张参考图是唯一底板,必须保留这张图的文字排版、品牌字样、标题区、卖点区、分割线、留白、机位、构图和信息层级。",
@ -3216,6 +3625,7 @@ function buildAgentReplacementPrompt(params: {
`当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张 SKU 参考图:${params.referenceLabel}`,
"【图片顺序规则】",
...imageOrderLines,
...scenePlanLines,
params.pairedProductIndex !== undefined
? `请优先使用第 ${params.pairedProductIndex + 1} 张输入商品图作为本张参考图的一对一替换主体。`
: "请使用输入商品图作为本张参考图的一对一替换主体。",
@ -6150,6 +6560,14 @@ export async function POST(request: Request) {
return undefined;
},
);
const replacementScenePlanningRuntime =
await resolveReplacementScenePlanningRuntime({
models: replacementVisionModels,
});
console.info("[Chat API] Replacement scene planning runtime:", {
kind: replacementScenePlanningRuntime?.kind || "none",
modelName: replacementScenePlanningRuntime?.modelName || null,
});
const stopKeepalive = startStreamKeepalive(
writer,
"agent-preset:replacement",
@ -6276,8 +6694,13 @@ export async function POST(request: Request) {
),
]
: productInputImageUrls;
const referenceDecision = await analyzeReplacementReferenceImage({
imageUrl: reference.imageUrl,
const replacementExecutionPlan = await analyzeReplacementExecutionPlan({
referenceImageUrl: reference.imageUrl,
referenceLabel: reference.label,
productImageUrls: orderedProductContextImageUrls,
modelImageUrl: modelInputImageUrl,
workflowPrompt: baseReplacementPrompt || hiddenPromptText || null,
planningRuntime: replacementScenePlanningRuntime,
models: replacementVisionModels,
});
const replacementSourceType = isDetailReplacementPreset
@ -6291,10 +6714,27 @@ export async function POST(request: Request) {
referenceImageUrl: reference.imageUrl,
pairedProductIndex:
typeof pairedProductIndex === "number" ? pairedProductIndex + 1 : null,
shouldReplace: referenceDecision.shouldReplace,
reason: referenceDecision.reason,
detectionModelName: referenceDecision.detectionModelName,
detectionSource: referenceDecision.detectionSource,
shouldReplace: replacementExecutionPlan.shouldReplace,
reason: replacementExecutionPlan.reason,
detectionModelName: replacementExecutionPlan.detectionModelName,
detectionSource: replacementExecutionPlan.detectionSource,
scenePlanSource: replacementExecutionPlan.scenePlan.analysisSource,
selectedProductImageIndexes:
replacementExecutionPlan.scenePlan.selectedProductImageIndexes,
useModelImage: replacementExecutionPlan.scenePlan.useModelImage,
});
console.info("[Chat API] Replacement scene plan summary:", {
mode: replacementSourceType,
referenceIndex: referenceIndex + 1,
referenceLabel: reference.label,
analysisModelName: replacementExecutionPlan.scenePlan.analysisModelName,
analysisSource: replacementExecutionPlan.scenePlan.analysisSource,
selectedProductImageIndexes:
replacementExecutionPlan.scenePlan.selectedProductImageIndexes,
productSelectionReason:
replacementExecutionPlan.scenePlan.productSelectionReason,
useModelImage: replacementExecutionPlan.scenePlan.useModelImage,
modelImageReason: replacementExecutionPlan.scenePlan.modelImageReason,
});
const referenceSourceImageUrls = Array.from(
new Set([
@ -6304,17 +6744,17 @@ export async function POST(request: Request) {
]),
);
if (!referenceDecision.shouldReplace) {
if (!replacementExecutionPlan.shouldReplace) {
console.info("[Chat API] Replacement reference skipped and reused:", {
mode: replacementSourceType,
referenceIndex: referenceIndex + 1,
totalReferences: replacementReferences.length,
referenceLabel: reference.label,
reason: referenceDecision.reason,
reason: replacementExecutionPlan.reason,
});
await emitReplacementResultMessage({
reference,
bodyText: referenceDecision.reason,
bodyText: replacementExecutionPlan.reason,
image: {
originalUrl: reference.imageUrl,
thumbnailUrl: reference.imageUrl,
@ -6330,11 +6770,29 @@ export async function POST(request: Request) {
continue;
}
const selectedProductIndexes = replacementExecutionPlan.scenePlan
.selectedProductImageIndexes.map((index) => index - 1)
.filter(
(index) =>
index >= 0 && index < orderedProductContextImageUrls.length,
);
const selectedProductContextImageUrls =
selectedProductIndexes.length > 0
? selectedProductIndexes.map(
(index) => orderedProductContextImageUrls[index]!,
)
: orderedProductContextImageUrls.slice(
0,
Math.min(orderedProductContextImageUrls.length, 2),
);
const sourceImageUrls = Array.from(
new Set(
[
...orderedProductContextImageUrls,
...(modelInputImageUrl ? [modelInputImageUrl] : []),
...selectedProductContextImageUrls,
...(replacementExecutionPlan.scenePlan.useModelImage &&
modelInputImageUrl
? [modelInputImageUrl]
: []),
...replacementTemplateReferenceImageUrls,
reference.imageUrl,
].filter(Boolean),
@ -6350,10 +6808,14 @@ export async function POST(request: Request) {
referenceLabel: reference.label,
referenceIndex,
totalReferences: replacementReferences.length,
hasModelImage: Boolean(modelInputImageUrl),
productImageCount: productInputImageUrls.length,
hasModelImage:
replacementExecutionPlan.scenePlan.useModelImage &&
Boolean(modelInputImageUrl),
productImageCount: selectedProductContextImageUrls.length,
originalProductImageCount: productInputImageUrls.length,
templateReferenceCount: replacementTemplateReferenceImageUrls.length,
pairedProductIndex,
scenePlan: replacementExecutionPlan.scenePlan,
});
console.info("[Chat API] Replacement image generation started:", {
mode: replacementSourceType,
@ -6364,8 +6826,11 @@ export async function POST(request: Request) {
typeof pairedProductIndex === "number" ? pairedProductIndex + 1 : null,
sourceImageCount: sourceImageUrls.length,
imageOrder: {
productImages: productInputImageUrls.length,
hasModelImage: Boolean(modelInputImageUrl),
productImages: selectedProductContextImageUrls.length,
originalProductImages: productInputImageUrls.length,
hasModelImage:
replacementExecutionPlan.scenePlan.useModelImage &&
Boolean(modelInputImageUrl),
templateReferenceImages: replacementTemplateReferenceImageUrls.length,
finalReferenceImageIndex: sourceImageUrls.length,
},