lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 优化详情参考图替换链路
Some checks failed
Lint / build (20) (push) Has been cancelled
Details
Playwright Tests / test (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Your Name 2026-04-25 18:12:55 +08:00
parent 7222c5d6dd
commit 653525cfda
3 changed files with 648 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

592
app/(chat)/api/chat/route.ts
View File

@ -3346,6 +3346,14 @@ function buildAgentSceneWorkflowPrompt(params: {
}
type AgentReplacementModePayload = NonNullable<PostRequestBody["agentModePayload"]>;
type AgentModeSourceImagePayload = NonNullable<AgentReplacementModePayload["sourceImages"]>;
type NormalizedAgentModeSourceImage = {
url: string;
label: string;
name: string;
assetId: string | null;
};
type ReplacementReferenceDecision = {
shouldReplace: boolean;
@ -3543,6 +3551,94 @@ function normalizeAgentModeReferences(
.sort((left, right) => left.sortOrder - right.sortOrder);
}
function normalizeAgentModeSourceImage(value: unknown): NormalizedAgentModeSourceImage | null {
const record =
value && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
const url = String(record.url || "").trim();
if (!/^https?:\/\//i.test(url)) {
return null;
}
const label = String(record.label || record.name || "参考图").trim().slice(0, 255);
const name = String(record.name || record.label || label || "参考图").trim().slice(0, 255);
const assetId = String(record.assetId || "").trim() || null;
return {
url,
label: label || name || "参考图",
name: name || label || "参考图",
assetId,
};
}
function normalizeAgentModeSourceImages(
payload?: PostRequestBody["agentModePayload"],
): {
productImages: NormalizedAgentModeSourceImage[];
modelImage: NormalizedAgentModeSourceImage | null;
} {
const sourceImages = payload?.sourceImages as AgentModeSourceImagePayload | undefined;
const productImages = Array.isArray(sourceImages?.productImages)
? sourceImages.productImages
.map((item) => normalizeAgentModeSourceImage(item))
.filter((item): item is NormalizedAgentModeSourceImage => Boolean(item))
.slice(0, 16)
: [];
return {
productImages,
modelImage: normalizeAgentModeSourceImage(sourceImages?.modelImage),
};
}
function resolveImagePartsByReferenceExclusion(params: {
uploadedImageParts: AgentPresetMessageImagePart[];
referenceImageUrls: string[];
}) {
const referenceUrlSet = new Set(params.referenceImageUrls.map((url) => String(url || "").trim()));
const modelPart = params.uploadedImageParts.find((part) => /模特|model/i.test(String(part.name || "")));
const productParts = params.uploadedImageParts.filter((part) => {
const name = String(part.name || "").trim();
if (referenceUrlSet.has(part.url)) return false;
if (/^详情参考图|^SKU参考图/i.test(name)) return false;
if (modelPart && part.url === modelPart.url) return false;
return true;
});
return { productParts, modelPart: modelPart || null };
}
async function resolveDirectReplacementModelRef(params: {
selectedChatModel?: string | null;
finalModelId?: string | null;
models?: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>;
}) {
const explicitModel = String(params.selectedChatModel || "").trim();
if (explicitModel && explicitModel !== "auto") {
return explicitModel;
}
const routedModel = String(params.finalModelId || "").trim();
if (routedModel && routedModel !== "auto") {
return routedModel;
}
const models = params.models || (await getEnabledAiModelsWithProvider(false));
const imageModels = models.filter(
({ model, provider }) =>
model.supportsImageGen &&
!model.isSubModel &&
!isVideoProviderType(provider?.type, model),
);
const preferred =
imageModels.find(({ model }) => model.isDefault) ||
imageModels.find(({ provider }) => String(provider?.type || "").includes("image")) ||
imageModels[0];
return preferred?.model.id || explicitModel || routedModel || "auto";
}
function normalizeBooleanLike(value: unknown): boolean | null {
if (typeof value === "boolean") {
return value;
@ -3668,15 +3764,26 @@ function parseReplacementReferenceDecision(
async function analyzeReplacementReferenceImage(params: {
imageUrl: string;
models?: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>;
modeType?: "detail_replacement" | "sku_replacement";
}): Promise<ReplacementReferenceDecision> {
const detectionPrompt = [
"你是家具商品参考图替换判定器。",
"请判断这张图片里，是否存在清晰、主要、适合被产品替换流程替换的家具商品主体。",
"家具主体包含：床、床垫、沙发、椅子、餐桌、茶几、柜子、电视柜、斗柜、床头柜、书桌等。",
"只有当家具商品是主要视觉主体、轮廓清晰、不是单纯环境陪衬时，shouldReplace 才能为 true。",
"以下情况必须判定 shouldReplace=false：纯文字页、参数说明页、品牌介绍页、材质/五金局部特写、只有模特没有家具、只有空间环境没有明确家具主体、家具很小只是背景、主体过多无法明确替换对象。",
'只输出 JSON，不要输出额外解释。格式：{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字"}',
].join("\n");
const isDetailReplacement = params.modeType === "detail_replacement";
const detectionPrompt = isDetailReplacement
? [
"你是电商详情参考图背景替换判定器。",
"请判断这张图片里，是否存在适合被重建/替换的背景图、产品场景图、商品主体图或大面积视觉图片区。",
"这不是只找家具主体：只要存在照片、插画、场景背景、产品展示图区、人物展示图区，都可以 shouldReplace=true。",
"必须保留的文字、标题、卖点标签、UI图标、按钮、分割线、品牌标识、留白和排版不属于替换对象。",
"只有当图片几乎是纯文字页、参数表、品牌介绍页、无明显背景图/场景图区/商品图片区时，shouldReplace 才能为 false。",
'只输出 JSON，不要输出额外解释。格式：{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字"}',
].join("\n")
: [
"你是家具商品参考图替换判定器。",
"请判断这张图片里，是否存在清晰、主要、适合被产品替换流程替换的家具商品主体。",
"家具主体包含：床、床垫、沙发、椅子、餐桌、茶几、柜子、电视柜、斗柜、床头柜、书桌等。",
"只有当家具商品是主要视觉主体、轮廓清晰、不是单纯环境陪衬时，shouldReplace 才能为 true。",
"以下情况必须判定 shouldReplace=false：纯文字页、参数说明页、品牌介绍页、材质/五金局部特写、只有模特没有家具、只有空间环境没有明确家具主体、家具很小只是背景、主体过多无法明确替换对象。",
'只输出 JSON，不要输出额外解释。格式：{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字"}',
].join("\n");
const configuredModels = params.models || (await getEnabledAiModelsWithProvider());
const configuredQwenVisionModel = configuredModels.find(
@ -3816,6 +3923,7 @@ async function resolveReplacementScenePlanningRuntime(params: {
}
async function analyzeReplacementExecutionPlan(params: {
modeType?: "detail_replacement" | "sku_replacement";
referenceImageUrl: string;
referenceLabel: string;
productImageUrls: string[];
@ -3824,6 +3932,7 @@ async function analyzeReplacementExecutionPlan(params: {
planningRuntime: ReplacementScenePlanningRuntime;
models?: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>;
}): Promise<ReplacementExecutionPlan> {
const isDetailReplacement = params.modeType === "detail_replacement";
if (params.planningRuntime) {
try {
const replacementScenePlanningTimeoutMs =
@ -3846,20 +3955,28 @@ async function analyzeReplacementExecutionPlan(params: {
.replace(/\s+/g, " ")
.slice(0, 2200);
const analysisPrompt = [
"你是电商详情替换流程中的“场景理解规划器”。",
isDetailReplacement
? "你是电商详情参考图背景替换流程中的“版式与背景区域规划器”。"
: "你是电商详情替换流程中的“场景理解规划器”。",
`当前要分析的参考图：${params.referenceLabel}`,
"你现在只分析这一张参考图本身，不分析产品图，不分析模特图。",
"产品图和模特图会在后续生图阶段单独提供给生图模型使用，你这里不要尝试理解它们的具体外观，也不要要求必须使用所有输入图。",
params.modelImageUrl
? "你只需要根据当前参考图判断：这一屏是否存在真实的人物展示需求；如果没有人物需求，useModelImage 必须为 false。"
: "本轮没有模特图，useModelImage 必须为 false。",
"这张图片是唯一的详情参考图底板，它决定当前这一屏的版式、文字排版、机位、构图、留白和信息层级。",
"这张参考图不是要求原样保留的场景照片。原图里的沙发、床、柜体、墙面、地面、窗帘、灯具、摆件、桌面道具、背景家具和环境光氛围，都可以在后续生图阶段按前置工作流风格重建。",
"这张图片是唯一的详情参考图底板，它决定当前这一屏的版式、文字排版、UI图标、按钮、机位、构图、留白和信息层级。",
isDetailReplacement
? "这张参考图不是要求原样保留的场景照片。后续生图只允许替换/重建背景图、产品场景图、商品主体图或人物展示图区域；文字、标题、卖点标签、UI图标、按钮、分割线、品牌标识、留白和排版必须保留。"
: "这张参考图不是要求原样保留的场景照片。原图里的沙发、床、柜体、墙面、地面、窗帘、灯具、摆件、桌面道具、背景家具和环境光氛围，都可以在后续生图阶段按前置工作流风格重建。",
"请完成以下任务：",
"1. 判断 shouldReplace：当前参考图里是否存在清晰、主要、适合被替换的家具商品主体。",
isDetailReplacement
? "1. 判断 shouldReplace：当前参考图里是否存在适合被替换/重建的背景图、产品场景图、商品主体图、人物展示图或大面积视觉图片区。纯文字/纯参数/纯品牌介绍且没有视觉图片区时才为 false。"
: "1. 判断 shouldReplace：当前参考图里是否存在清晰、主要、适合被替换的家具商品主体。",
"2. 判断 useModelImage，并说明原因。",
"3. 提炼 sceneSummary、cameraSummary、placementSummary、preserveSummary、styleSummary、avoidSummary。",
"4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，描述这一屏需要的场景、机位、主体位置、周边环境重建方向以及必须保留的内容。",
isDetailReplacement
? "4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，描述需要替换的背景/场景图片区、产品放置方式、模特使用规则，以及必须保留的文字/UI/图标/排版。"
: "4. 输出 generationPrompt：这是一段给后续生图模型的本屏生成指令，描述这一屏需要的场景、机位、主体位置、周边环境重建方向以及必须保留的内容。",
"5. sceneSummary、styleSummary 和 generationPrompt 必须优先描述“按前置工作流重建后的目标场景”，而不是只复述参考图当前已有场景。",
"6. 如果前置工作流风格与当前参考图原场景冲突，必须保留版式骨架，但放弃原场景物件本身，以前置工作流指定的空间风格、软装、灯光和氛围为准。",
"7. avoidSummary 里要明确避免“只替换产品、不重建场景”这类结果。",
@ -3933,6 +4050,7 @@ async function analyzeReplacementExecutionPlan(params: {
const fallbackDecision = await analyzeReplacementReferenceImage({
imageUrl: params.referenceImageUrl,
models: params.models,
modeType: params.modeType,
});
return {
...fallbackDecision,
@ -4044,28 +4162,28 @@ function buildAgentReplacementPrompt(params: {
"【图片顺序规则】",
...imageOrderLines,
...scenePlanLines,
"你现在做的不是重新生成一张新图，而是编辑最后一张参考图本身。",
"要把最后一张参考图理解为“页面版式底板 + 构图参考”，不是“旧场景照片必须原样保留”。",
"你现在做的是“详情参考图背景/场景图替换”，不是重新设计整张详情图。",
"要把最后一张参考图理解为“页面版式底板 + 背景图/场景图区参考”，不是“旧场景照片必须原样保留”。",
"前面“工作流设定”里的场景组合、空间风格、硬装方案、软装方案、灯光氛围、人群设定，必须作为当前这一屏的强制场景改造要求生效。",
"执行时要基于这些场景组合要求，重建产品主体周边乃至整屏视觉中的场景元素、材质氛围、软装陈列、灯光与色调，使当前这一屏呈现出与第一步已选风格一致的场景效果。",
"允许改造的不只是产品主体周边的一小块区域；除文字排版、标题区、信息区、分割线、留白和版式结构外，原参考图里的旧场景视觉内容都可以按目标风格整体重建。",
"执行时要基于这些场景组合要求，重建参考图中的背景图/产品场景图区域，使当前这一屏呈现出与第一步已选风格一致的新背景画面。",
"允许替换/重建的范围是参考图里的背景图、产品场景图、商品主体图和必要的人物展示图区；不允许改动版式层。",
"如果前面商品图、模特图与最后一张参考图底板存在冲突，始终以最后一张参考图底板的版式、机位、构图和信息层级为最高优先级。",
"最后一张参考图是唯一底板，必须保留这张图的文字排版、品牌字样、标题区、卖点区、分割线、留白、机位、构图和信息层级。",
"前面的商品图和模特图用于替换参考图中的产品主体，并按工作流设定去改造与产品直接相关的场景内容，但不能改变参考图的版式设计。",
"可被替换重建的内容包括但不限于：背景墙面、地面材质、沙发、床、背景柜、窗帘、灯具、地毯、挂画、摆件、桌面道具、空间色调、环境光氛围。",
"最后一张参考图是唯一底板，必须完整保留这张图的文字排版、品牌字样、标题区、卖点区、UI图标、按钮、标签、装饰元素、分割线、留白、画布比例和信息层级。",
"前面的商品图和模特图用于生成新的背景图/产品场景图内容，并按工作流设定替换参考图中的旧背景画面，但不能改变参考图的版式设计。",
"可被替换重建的内容包括但不限于：背景照片、产品展示照片、墙面、地面材质、沙发、床、背景柜、窗帘、灯具、地毯、挂画、摆件、桌面道具、空间色调、环境光氛围。",
params.productImageCount > 1
? `前面的 ${params.productImageCount} 张商品输入图都属于同一产品的多角度/多细节白底图，请综合这些图片理解产品结构、材质、颜色、比例、边角、开孔、五金与细节，不可只参考第一张。`
: "前面的商品输入图就是当前要替换进去的产品白底图，请准确保持该产品的结构、材质、颜色、比例和细节。",
"请先识别最后一张参考图中需要被替换的产品主体所在区域，并围绕这个区域在同一版式骨架下重建新的场景画面；不要只做局部抠图式换产品。",
"请严格保留最后一张参考图的版式结构、镜头关系、构图、文字排版、留白位置和信息层级；但画面中的旧场景家具、背景物件、材质、灯光氛围、软装小物、空间质感需要按第一步工作流设定进行明显重建。",
"如果第一步工作流风格与参考图原场景冲突，必须保留版式骨架，但放弃原场景物件本身，以第一步工作流风格为准。",
"请先识别最后一张参考图中需要被替换的背景图/场景图区域，并在同一版式骨架下生成新的背景画面；不要只做局部抠图式换产品。",
"请严格保留最后一张参考图的版式结构、镜头关系、构图、文字排版、UI图标、标签、按钮、留白位置和信息层级；但旧背景图里的家具、背景物件、材质、灯光氛围、软装小物、空间质感需要按第一步工作流设定进行明显重建。",
"如果第一步工作流风格与参考图原场景冲突，必须保留版式骨架和UI层，但放弃原背景图物件本身，以第一步工作流风格为准。",
"前面的输入商品图用于提供商品主体、材质、颜色和细节特征；如有模特图，请只在参考图本身包含人物展示时再参考模特图。",
"绝对不要忽略最后一张参考图去生成一张无关的新场景图。",
"如果生成结果仍然保留了参考图原有的大部分场景物件，只是把产品换掉，则视为执行失败；你必须让场景风格发生清晰可见的变化。",
"绝对不要移动、删除、改写、重排参考图里的任何文字与排版元素。",
"如果生成结果仍然保留了参考图原有的大部分背景图物件，只是把产品换掉，则视为执行失败；你必须让背景图/场景图发生清晰可见的变化。",
"绝对不要移动、删除、改写、重排参考图里的任何文字、UI图标、按钮、标签、装饰元素与排版元素。",
"输出结果必须保持最后一张参考图底板的原始纵横比与画布尺寸，不允许擅自改成长图、横图或裁掉任何版面区域。",
"如果参考图中没有可替换的产品主体，应直接沿用参考图，不要硬生成。",
"不要改动参考图的整体布局，不要新增无关主体，不要删除关键版式元素。",
"如果参考图中没有可替换/重建的背景图、产品场景图或视觉图片区，应直接沿用参考图，不要硬生成。",
"不要改动参考图的整体布局，不要新增无关主体，不要删除关键版式或UI元素。",
]
: [
"【执行模式】SKU替换",
@ -6851,6 +6969,425 @@ export async function POST(request: Request) {
return createSseResponse(confirmedStoryboardStream);
}
if (!agentPresetId && agentModePayload?.modeType === "detail_replacement") {
const replacementReferences = normalizeAgentModeReferences(agentModePayload);
if (replacementReferences.length === 0) {
return NextResponse.json(
{
code: "bad_request:api",
message: "详情参考图替换至少需要 1 张详情参考图",
},
{ status: 400 },
);
}
const uploadedImageParts = extractCurrentMessageImageParts(message.parts as any[]);
const sourceImages = normalizeAgentModeSourceImages(agentModePayload);
const inferredImageParts = resolveImagePartsByReferenceExclusion({
uploadedImageParts,
referenceImageUrls: replacementReferences.map((item) => item.imageUrl),
});
const productSourceImages =
sourceImages.productImages.length > 0
? sourceImages.productImages
: inferredImageParts.productParts.map((part) => ({
url: part.url,
label: part.name || "产品图",
name: part.name || "产品图",
assetId: part.assetId || null,
}));
const modelSourceImage =
sourceImages.modelImage ||
(inferredImageParts.modelPart
? {
url: inferredImageParts.modelPart.url,
label: inferredImageParts.modelPart.name || "模特图",
name: inferredImageParts.modelPart.name || "模特图",
assetId: inferredImageParts.modelPart.assetId || null,
}
: null);
const replacementReferenceUrlSet = new Set(
replacementReferences.map((item) => item.imageUrl),
);
const productInputImageUrls = Array.from(
new Set(
productSourceImages
.map((item) => item.url)
.filter((url) => /^https?:\/\//i.test(url) && !replacementReferenceUrlSet.has(url)),
),
).slice(0, 16);
const modelInputImageUrl =
modelSourceImage?.url && !replacementReferenceUrlSet.has(modelSourceImage.url)
? modelSourceImage.url
: null;
if (productInputImageUrls.length === 0) {
return NextResponse.json(
{
code: "bad_request:api",
message: "详情参考图替换至少需要 1 张产品图",
},
{ status: 400 },
);
}
const directReplacementModels = await getEnabledAiModelsWithProvider().catch((error) => {
console.warn("[Chat API] Failed to load models for direct detail replacement:", error);
return undefined;
});
const resolvedModelRef = await resolveDirectReplacementModelRef({
selectedChatModel,
finalModelId,
models: directReplacementModels,
});
if (!resolvedModelRef || resolvedModelRef === "auto") {
return NextResponse.json(
{
code: "bad_request:api",
message: "当前风格化未配置可用的生图模型",
},
{ status: 400 },
);
}
const sanitizedUserSupplementPrompt = normalizeAgentSupplementPrompt(
requestTextContent,
"详情参考图替换",
);
const baseReplacementPrompt =
buildAgentSlotPrompt({
workflowPrompt: hiddenPromptText || null,
slotPrompt: "",
userSupplementPrompt: sanitizedUserSupplementPrompt || null,
includeSlotTitle: false,
modeType: "detail_replacement",
}) ||
hiddenPromptText ||
sanitizedUserSupplementPrompt ||
"执行详情参考图背景替换";
const resolvedAspectRatio = "auto";
const resolvedImageSize =
effectiveImageParams?.imageSize || defaultImageQualityParams.imageSize;
const sourceAssetIds = Array.from(
new Set(
[
...productSourceImages.map((item) => item.assetId),
modelSourceImage?.assetId,
...uploadedImageParts.map((part) => part.assetId),
].filter((assetId): assetId is string => Boolean(assetId)),
),
);
const directDetailReplacementStream = createUIMessageStream({
execute: async ({ writer }) => {
writeAssistantTextChunk(
writer,
"<!-- IMAGE_GENERATION_START -->",
"agent-mode:detail-replacement:start",
);
if (contextPayload?.source === "canvas") {
writeImageGenerationStatusChunk(
writer,
{
phase: "processing",
modelName: resolvedModelRef,
imageCount: replacementReferences.length,
},
"agent-mode:detail-replacement:status-processing",
);
}
let successCount = 0;
let resolvedModelDisplayName = resolvedModelRef;
const replacementVisionModels =
directReplacementModels ||
(await getEnabledAiModelsWithProvider().catch((error) => {
console.warn(
"[Chat API] Failed to load configured vision models for direct detail replacement:",
error,
);
return undefined;
}));
const replacementScenePlanningRuntime =
await resolveReplacementScenePlanningRuntime({
models: replacementVisionModels,
});
const stopKeepalive = startStreamKeepalive(
writer,
"agent-mode:detail-replacement",
12_000,
);
const emitReplacementResultMessage = async (params: {
reference: (typeof replacementReferences)[number];
bodyText?: string | null;
image: {
originalUrl: string;
thumbnailUrl: string;
};
imageSizeLabel?: string | null;
modelDisplayName?: string | null;
sourceImageUrls: string[];
streamLabel: string;
promptForPersistence?: string | null;
}) => {
const persistedImageSize =
String(params.imageSizeLabel || "").trim() || resolvedImageSize;
const assistantText = prependImageModelBadge(
[
`### ${params.reference.label}`,
String(params.bodyText || "").trim(),
buildAgentToolImagesMarkdown([
{
originalUrl: params.image.originalUrl,
thumbnailUrl: params.image.thumbnailUrl,
imageSize: persistedImageSize,
},
]),
]
.filter(Boolean)
.join("\n\n"),
params.modelDisplayName || null,
);
const assistantMessageId = generateUUID();
await saveMessages({
messages: [
{
id: assistantMessageId,
chatId: id,
role: "assistant",
parts: [{ type: "text", text: assistantText }],
attachments: [],
createdAt: new Date(),
videoStorageStatus: null,
temporaryVideoUrl: null,
permanentVideoUrl: null,
archivedAt: null,
videoMetadata: null,
},
],
});
await saveGeneratedImages(
assistantText,
session.user.id,
id,
assistantMessageId,
params.promptForPersistence || baseReplacementPrompt,
resolvedModelRef,
{
sourceAssetIds,
sourceImageUrls: params.sourceImageUrls,
relationType: "edited_from",
sourceType: "detail_replacement",
projectId: contextPayload?.projectId || null,
userPrompt: requestTextContent,
optimizedPrompt: params.promptForPersistence || baseReplacementPrompt,
requestedImageCount: 1,
aspectRatio: resolvedAspectRatio,
imageSize: persistedImageSize,
},
);
writeAssistantTextChunk(
writer,
assistantText,
params.streamLabel,
assistantMessageId,
);
if (contextPayload?.source === "canvas") {
writeCanvasMediaChunk(
writer,
{
messageId: assistantMessageId,
imageUrls: [params.image.originalUrl],
prompt: params.promptForPersistence || baseReplacementPrompt,
sourceImageUrls: params.sourceImageUrls,
},
`${params.streamLabel}:canvas-media`,
);
}
};
try {
for (const [referenceIndex, reference] of replacementReferences.entries()) {
const replacementExecutionPlan = await analyzeReplacementExecutionPlan({
modeType: "detail_replacement",
referenceImageUrl: reference.imageUrl,
referenceLabel: reference.label,
productImageUrls: productInputImageUrls,
modelImageUrl: modelInputImageUrl,
workflowPrompt: baseReplacementPrompt || hiddenPromptText || null,
planningRuntime: replacementScenePlanningRuntime,
models: replacementVisionModels,
});
const referenceSourceImageUrls = Array.from(
new Set([
...productInputImageUrls,
...(modelInputImageUrl ? [modelInputImageUrl] : []),
reference.imageUrl,
]),
);
if (!replacementExecutionPlan.shouldReplace) {
await emitReplacementResultMessage({
reference,
bodyText: replacementExecutionPlan.reason,
image: {
originalUrl: reference.imageUrl,
thumbnailUrl: reference.imageUrl,
},
modelDisplayName: null,
sourceImageUrls: referenceSourceImageUrls,
streamLabel: "agent-mode:image:detail-replacement:item",
});
successCount += 1;
continue;
}
const selectedProductIndexes = replacementExecutionPlan.scenePlan
.selectedProductImageIndexes.map((index) => index - 1)
.filter((index) => index >= 0 && index < productInputImageUrls.length);
const selectedProductContextImageUrls =
selectedProductIndexes.length > 0
? selectedProductIndexes.map((index) => productInputImageUrls[index]!)
: productInputImageUrls.slice(0, Math.min(productInputImageUrls.length, 2));
const sourceImageUrls = Array.from(
new Set([
...selectedProductContextImageUrls,
...(replacementExecutionPlan.scenePlan.useModelImage && modelInputImageUrl
? [modelInputImageUrl]
: []),
reference.imageUrl,
]),
);
const replacementPrompt = buildAgentReplacementPrompt({
basePrompt: baseReplacementPrompt,
modeType: "detail_replacement",
referenceLabel: reference.label,
referenceIndex,
totalReferences: replacementReferences.length,
hasModelImage:
replacementExecutionPlan.scenePlan.useModelImage &&
Boolean(modelInputImageUrl),
productImageCount: selectedProductContextImageUrls.length,
originalProductImageCount: productInputImageUrls.length,
templateReferenceCount: 0,
scenePlan: replacementExecutionPlan.scenePlan,
});
try {
const result = await generateCanvasImage({
prompt: replacementPrompt,
negativePrompt: effectiveImageParams?.negativePrompt,
model: resolvedModelRef,
aspectRatio: resolvedAspectRatio,
imageSize: resolvedImageSize,
imageCount: 1,
referenceImages: sourceImageUrls,
});
resolvedModelDisplayName = result.model || resolvedModelDisplayName;
const firstImage = result.images[0];
if (!firstImage) {
throw new Error("未返回结果图片");
}
let finalImage = firstImage;
let finalImageSizeLabel = resolvedImageSize;
try {
const normalizedImage = await normalizeReplacementResultImage({
generatedImageUrl: firstImage.originalUrl,
referenceImageUrl: reference.imageUrl,
uploadPrefix: "agent-detail-replacement",
});
finalImage = {
originalUrl: normalizedImage.originalUrl,
thumbnailUrl: normalizedImage.thumbnailUrl || normalizedImage.originalUrl,
};
finalImageSizeLabel = normalizedImage.imageSize;
} catch (normalizeError) {
console.warn(
"[Chat API] Failed to normalize direct detail replacement output:",
{
referenceLabel: reference.label,
referenceImageUrl: reference.imageUrl,
resultImageUrl: firstImage.originalUrl,
error:
normalizeError instanceof Error
? normalizeError.message
: "unknown",
},
);
}
await emitReplacementResultMessage({
reference,
image: finalImage,
imageSizeLabel: finalImageSizeLabel,
modelDisplayName: resolvedModelDisplayName,
sourceImageUrls: Array.from(
new Set([
...productInputImageUrls,
...(modelInputImageUrl ? [modelInputImageUrl] : []),
...replacementReferences.map((item) => item.imageUrl),
]),
),
streamLabel: "agent-mode:image:detail-replacement:item",
promptForPersistence: replacementPrompt,
});
successCount += 1;
} catch (replacementItemError) {
console.error("[Chat API] Direct detail replacement item failed:", {
referenceIndex,
referenceLabel: reference.label,
error: replacementItemError,
});
await emitReplacementResultMessage({
reference,
bodyText: `生成失败，当前先沿用参考图：${
replacementItemError instanceof Error
? replacementItemError.message
: "请稍后重试"
}`,
image: {
originalUrl: reference.imageUrl,
thumbnailUrl: reference.imageUrl,
},
modelDisplayName: null,
sourceImageUrls: referenceSourceImageUrls,
streamLabel: "agent-mode:image:detail-replacement:item-fallback",
});
successCount += 1;
}
}
} finally {
stopKeepalive();
}
if (contextPayload?.source === "canvas") {
writeImageGenerationStatusChunk(
writer,
{
phase: successCount > 0 ? "completed" : "failed",
modelName: resolvedModelDisplayName,
imageCount: successCount,
},
"agent-mode:detail-replacement:status-finished",
);
}
writeAssistantTextChunk(
writer,
"<!-- IMAGE_GENERATION_END -->",
"agent-mode:detail-replacement:end",
);
},
});
return createSseResponse(directDetailReplacementStream);
}
if (agentPresetId) {
const selectedAgentPreset = await getAccessibleAgentPresetById({
id: agentPresetId,
@ -7297,6 +7834,7 @@ export async function POST(request: Request) {
]
: productInputImageUrls;
const replacementExecutionPlan = await analyzeReplacementExecutionPlan({
modeType: isDetailReplacementPreset ? "detail_replacement" : "sku_replacement",
referenceImageUrl: reference.imageUrl,
referenceLabel: reference.label,
productImageUrls: orderedProductContextImageUrls,

13
app/(chat)/api/chat/schema.ts
View File

@ -80,6 +80,13 @@ const agentModeReferenceSchema = z.object({
assignedProductIndex: z.number().min(0).max(9).nullable().optional(),
});
const agentModeSourceImageSchema = z.object({
url: z.string().url(),
label: z.string().max(255).optional(),
name: z.string().max(255).optional(),
assetId: z.string().min(1).max(128).nullable().optional(),
});
const agentModePayloadSchema = z
.object({
modeType: z.enum(["detail_replacement", "sku_replacement"]),
@ -87,6 +94,12 @@ const agentModePayloadSchema = z
queryType: z.enum(["sku", "spu"]).optional(),
selectedCandidateId: z.string().min(1).max(128).nullable().optional(),
references: z.array(agentModeReferenceSchema).max(40),
sourceImages: z
.object({
productImages: z.array(agentModeSourceImageSchema).max(16).optional(),
modelImage: agentModeSourceImageSchema.nullable().optional(),
})
.optional(),
})
.optional();

71
components/agents/AgentGenerationWorkspaceDarkV2.tsx
View File

@ -1385,6 +1385,60 @@ export function AgentGenerationWorkspaceDarkV2({
);
const selectedModelAttachment =
modelSlotIndex >= 0 && attachments[modelSlotIndex] ? attachments[modelSlotIndex] : null;
const detailReplacementProductSourceImages = useMemo(() => {
const primaryImages = referenceImages.filter((image) => image.source === "primary");
const fallbackProductImages = referenceImages.filter(
(image) => image.source === "primary" || image.source === "secondary"
);
const productImages = primaryImages.length > 0 ? primaryImages : fallbackProductImages;
return productImages
.filter((image) => isValidChatHttpUrl(image.url))
.slice(0, 16)
.map((image) => ({
url: image.url,
label: image.label || image.name || "产品图",
name: image.name || image.label || "产品图",
assetId: image.assetId || null,
}));
}, [referenceImages]);
const detailReplacementModelSourceImage = useMemo(() => {
const referenceModelImage = referenceImages.find(
(image) => image.source === "brandModel" && isValidChatHttpUrl(image.url)
);
if (referenceModelImage) {
return {
url: referenceModelImage.url,
label: referenceModelImage.label || referenceModelImage.name || "模特图",
name: referenceModelImage.name || referenceModelImage.label || "模特图",
assetId: referenceModelImage.assetId || null,
};
}
if (selectedModelAttachment && isValidChatHttpUrl(selectedModelAttachment.url)) {
return {
url: selectedModelAttachment.url,
label: selectedModelAttachment.name || "模特图",
name: selectedModelAttachment.name || "模特图",
assetId: selectedModelAttachment.assetId || null,
};
}
if (selectedBrandModel?.imageUrl && isValidChatHttpUrl(selectedBrandModel.imageUrl)) {
return {
url: selectedBrandModel.imageUrl,
label: selectedBrandModel.displayName || `${selectedBrandModel.brandName || "品牌"}模特图`,
name: selectedBrandModel.displayName || `${selectedBrandModel.brandName || "品牌"}模特图`,
assetId: selectedBrandModel.assetId || null,
};
}
return null;
}, [
referenceImages,
selectedBrandModel?.assetId,
selectedBrandModel?.brandName,
selectedBrandModel?.displayName,
selectedBrandModel?.imageUrl,
selectedModelAttachment,
]);
const scenePrimaryReferenceCount = useMemo(
() => referenceImages.filter((image) => image.source === "primary").length,
[referenceImages]
@ -1674,6 +1728,8 @@ export function AgentGenerationWorkspaceDarkV2({
detailReplacementQueryInput: string;
detailReplacementSelectedCandidateId: string | null;
detailReplacementReferenceCards: typeof detailReplacementReferenceCards;
detailReplacementProductSourceImages: typeof detailReplacementProductSourceImages;
detailReplacementModelSourceImage: typeof detailReplacementModelSourceImage;
skuReplacementQueryInput: string;
skuReplacementQueryType: typeof skuReplacementQueryType;
skuReplacementSelectedCandidateId: string | null;
@ -1693,6 +1749,8 @@ export function AgentGenerationWorkspaceDarkV2({
detailReplacementQueryInput: "",
detailReplacementSelectedCandidateId: null,
detailReplacementReferenceCards: [],
detailReplacementProductSourceImages: [],
detailReplacementModelSourceImage: null,
skuReplacementQueryInput: "",
skuReplacementQueryType: "sku",
skuReplacementSelectedCandidateId: null,
@ -1709,11 +1767,16 @@ export function AgentGenerationWorkspaceDarkV2({
detailReplacementQueryInput,
detailReplacementSelectedCandidateId: detailReplacementSelectedCandidateId || null,
detailReplacementReferenceCards,
detailReplacementProductSourceImages,
detailReplacementModelSourceImage,
skuReplacementQueryInput,
skuReplacementQueryType,
skuReplacementSelectedCandidateId: skuReplacementSelectedCandidateId || null,
skuReplacementReferenceCards,
selectedProductAttachmentCount: selectedProductAttachments.length,
selectedProductAttachmentCount:
currentResultModeType === "detail_replacement"
? detailReplacementProductSourceImages.length
: selectedProductAttachments.length,
};
const selectableModels = useMemo(() => {
@ -1830,6 +1893,12 @@ export function AgentGenerationWorkspaceDarkV2({
)
: null,
})),
sourceImages: transportState.isDetailReplacementPreset
? {
productImages: transportState.detailReplacementProductSourceImages,
modelImage: transportState.detailReplacementModelSourceImage,
}
: undefined,
}
: undefined;
return {