feat: add scene-aware detail replacement planning
This commit is contained in:
parent
53f8cd7052
commit
d2aa27b1c2
@ -44,6 +44,8 @@ import {
|
|||||||
RmbgServiceError,
|
RmbgServiceError,
|
||||||
} from "@/lib/ai/rmbg/service";
|
} from "@/lib/ai/rmbg/service";
|
||||||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||||||
|
import { resolveDetailAgentLocalMoonshotModel } from "@/lib/ai/detail-agent-model-runtime";
|
||||||
|
import { inlineVisionImages } from "@/lib/ai/vision-inline";
|
||||||
import { createDocument } from "@/lib/ai/tools/create-document";
|
import { createDocument } from "@/lib/ai/tools/create-document";
|
||||||
import { getWeather } from "@/lib/ai/tools/get-weather";
|
import { getWeather } from "@/lib/ai/tools/get-weather";
|
||||||
import { requestSuggestions } from "@/lib/ai/tools/request-suggestions";
|
import { requestSuggestions } from "@/lib/ai/tools/request-suggestions";
|
||||||
@ -83,6 +85,7 @@ import {
|
|||||||
getAccessibleAgentPresetById,
|
getAccessibleAgentPresetById,
|
||||||
getConversationWorkingStateByChatId,
|
getConversationWorkingStateByChatId,
|
||||||
getMediaAssetsByUserAndUrls,
|
getMediaAssetsByUserAndUrls,
|
||||||
|
getSiteSettings,
|
||||||
upsertMediaAsset,
|
upsertMediaAsset,
|
||||||
} from "@/lib/db/queries";
|
} from "@/lib/db/queries";
|
||||||
import { db } from "@/lib/db";
|
import { db } from "@/lib/db";
|
||||||
@ -2897,13 +2900,188 @@ type ReplacementReferenceDecision = {
|
|||||||
reason: string;
|
reason: string;
|
||||||
detectionModelName?: string | null;
|
detectionModelName?: string | null;
|
||||||
detectionSource:
|
detectionSource:
|
||||||
|
| "local_moonshot"
|
||||||
|
| "configured_vision"
|
||||||
| "configured_qwen3_vl"
|
| "configured_qwen3_vl"
|
||||||
| "fallback_qwen3_vl"
|
| "fallback_qwen3_vl"
|
||||||
| "safe_passthrough";
|
| "safe_passthrough";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type ReplacementScenePlan = {
|
||||||
|
selectedProductImageIndexes: number[];
|
||||||
|
productSelectionReason: string;
|
||||||
|
useModelImage: boolean;
|
||||||
|
modelImageReason: string;
|
||||||
|
sceneSummary: string;
|
||||||
|
cameraSummary: string;
|
||||||
|
placementSummary: string;
|
||||||
|
preserveSummary: string;
|
||||||
|
styleSummary: string;
|
||||||
|
avoidSummary: string;
|
||||||
|
generationPrompt: string;
|
||||||
|
analysisModelName?: string | null;
|
||||||
|
analysisSource: "local_moonshot" | "configured_vision" | "heuristic";
|
||||||
|
};
|
||||||
|
|
||||||
|
type ReplacementExecutionPlan = ReplacementReferenceDecision & {
|
||||||
|
scenePlan: ReplacementScenePlan;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ReplacementScenePlanningRuntime =
|
||||||
|
| {
|
||||||
|
kind: "local_moonshot";
|
||||||
|
model: any;
|
||||||
|
modelName: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
kind: "configured_vision";
|
||||||
|
model: any;
|
||||||
|
modelName: string;
|
||||||
|
}
|
||||||
|
| null;
|
||||||
|
|
||||||
const QWEN_VL_MODEL_MATCHER = /(qwen|千问|通义).*(vl|vision)|qwen3-vl|qwen-vl/i;
|
const QWEN_VL_MODEL_MATCHER = /(qwen|千问|通义).*(vl|vision)|qwen3-vl|qwen-vl/i;
|
||||||
|
|
||||||
|
function truncateReplacementAnalysisText(value: unknown, maxLength = 240) {
|
||||||
|
return String(value || "").trim().replace(/\s+/g, " ").slice(0, maxLength);
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseReplacementScenePlan(
|
||||||
|
text: string,
|
||||||
|
params: {
|
||||||
|
productImageCount: number;
|
||||||
|
hasModelImage: boolean;
|
||||||
|
},
|
||||||
|
): ReplacementScenePlan | null {
|
||||||
|
const jsonText = extractJsonObjectText(text);
|
||||||
|
if (!jsonText) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(jsonText) as Record<string, unknown>;
|
||||||
|
const rawIndexes = Array.isArray(parsed.selectedProductImageIndexes)
|
||||||
|
? parsed.selectedProductImageIndexes
|
||||||
|
: Array.isArray(parsed.productImageIndexes)
|
||||||
|
? parsed.productImageIndexes
|
||||||
|
: [];
|
||||||
|
const selectedProductImageIndexes = Array.from(
|
||||||
|
new Set(
|
||||||
|
rawIndexes
|
||||||
|
.map((value) => Number(value))
|
||||||
|
.filter((value) => Number.isFinite(value))
|
||||||
|
.map((value) =>
|
||||||
|
Math.max(1, Math.min(params.productImageCount, Math.round(value))),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
).slice(0, 3);
|
||||||
|
|
||||||
|
const useModelImageRaw =
|
||||||
|
normalizeBooleanLike(parsed.useModelImage) ??
|
||||||
|
normalizeBooleanLike(parsed.needModelImage) ??
|
||||||
|
normalizeBooleanLike(parsed.referenceNeedsModel) ??
|
||||||
|
false;
|
||||||
|
|
||||||
|
const generationPrompt = truncateReplacementAnalysisText(
|
||||||
|
parsed.generationPrompt || parsed.recommendedScenePrompt || parsed.scenePrompt,
|
||||||
|
1200,
|
||||||
|
);
|
||||||
|
|
||||||
|
const scenePlan: ReplacementScenePlan = {
|
||||||
|
selectedProductImageIndexes:
|
||||||
|
selectedProductImageIndexes.length > 0
|
||||||
|
? selectedProductImageIndexes
|
||||||
|
: [1].filter((value) => value <= params.productImageCount),
|
||||||
|
productSelectionReason: truncateReplacementAnalysisText(
|
||||||
|
parsed.productSelectionReason || parsed.selectionReason || parsed.productReason,
|
||||||
|
160,
|
||||||
|
),
|
||||||
|
useModelImage: params.hasModelImage ? Boolean(useModelImageRaw) : false,
|
||||||
|
modelImageReason: truncateReplacementAnalysisText(
|
||||||
|
parsed.modelImageReason || parsed.modelReason,
|
||||||
|
160,
|
||||||
|
),
|
||||||
|
sceneSummary: truncateReplacementAnalysisText(
|
||||||
|
parsed.sceneSummary || parsed.sceneDescription,
|
||||||
|
220,
|
||||||
|
),
|
||||||
|
cameraSummary: truncateReplacementAnalysisText(
|
||||||
|
parsed.cameraSummary || parsed.cameraDescription,
|
||||||
|
180,
|
||||||
|
),
|
||||||
|
placementSummary: truncateReplacementAnalysisText(
|
||||||
|
parsed.placementSummary || parsed.productPlacement,
|
||||||
|
180,
|
||||||
|
),
|
||||||
|
preserveSummary: truncateReplacementAnalysisText(
|
||||||
|
parsed.preserveSummary || parsed.mustKeep || parsed.preserveInstruction,
|
||||||
|
220,
|
||||||
|
),
|
||||||
|
styleSummary: truncateReplacementAnalysisText(
|
||||||
|
parsed.styleSummary || parsed.styleInstruction || parsed.styleKeywords,
|
||||||
|
220,
|
||||||
|
),
|
||||||
|
avoidSummary: truncateReplacementAnalysisText(
|
||||||
|
parsed.avoidSummary || parsed.doNotInclude || parsed.avoidInstruction,
|
||||||
|
220,
|
||||||
|
),
|
||||||
|
generationPrompt,
|
||||||
|
analysisModelName: null,
|
||||||
|
analysisSource: "heuristic",
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!scenePlan.generationPrompt) {
|
||||||
|
scenePlan.generationPrompt = [
|
||||||
|
scenePlan.sceneSummary,
|
||||||
|
scenePlan.cameraSummary,
|
||||||
|
scenePlan.placementSummary,
|
||||||
|
scenePlan.styleSummary,
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(";")
|
||||||
|
.slice(0, 1200);
|
||||||
|
}
|
||||||
|
|
||||||
|
return scenePlan;
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("[Chat API] Failed to parse replacement scene plan JSON:", error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildHeuristicReplacementScenePlan(params: {
|
||||||
|
productImageCount: number;
|
||||||
|
hasModelImage: boolean;
|
||||||
|
shouldReplace: boolean;
|
||||||
|
reason: string;
|
||||||
|
}): ReplacementScenePlan {
|
||||||
|
return {
|
||||||
|
selectedProductImageIndexes: Array.from(
|
||||||
|
{ length: Math.min(Math.max(params.productImageCount, 1), 2) },
|
||||||
|
(_, index) => index + 1,
|
||||||
|
),
|
||||||
|
productSelectionReason:
|
||||||
|
params.productImageCount > 1
|
||||||
|
? "默认优先使用前两张产品图作为主视角与补充细节,其余角度不强制同时融入。"
|
||||||
|
: "默认使用当前唯一产品图作为主体参考。",
|
||||||
|
useModelImage: params.hasModelImage,
|
||||||
|
modelImageReason: params.hasModelImage
|
||||||
|
? "未完成场景级人物需求分析,先保留模特图作为可选补充参考。"
|
||||||
|
: "当前未提供模特图。",
|
||||||
|
sceneSummary: params.shouldReplace
|
||||||
|
? "保持当前参考图版式与场景结构,只替换参考图中的产品主体。"
|
||||||
|
: "当前参考图不满足替换条件。",
|
||||||
|
cameraSummary: "保持参考图原有机位、构图和景别,不做整体重画。",
|
||||||
|
placementSummary: "产品应落在参考图当前主体所在区域,不允许新增无关主体。",
|
||||||
|
preserveSummary: "文字排版、标题区、分割线、留白和信息层级必须保留。",
|
||||||
|
styleSummary: "沿用当前工作流风格设定,但只作用于产品相关场景内容。",
|
||||||
|
avoidSummary: "不要把所有输入产品角度图同时强行出现在一张图中。",
|
||||||
|
generationPrompt: params.reason,
|
||||||
|
analysisModelName: null,
|
||||||
|
analysisSource: "heuristic",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function normalizeAgentModeReferences(
|
function normalizeAgentModeReferences(
|
||||||
payload?: PostRequestBody["agentModePayload"],
|
payload?: PostRequestBody["agentModePayload"],
|
||||||
): AgentReplacementModePayload["references"] {
|
): AgentReplacementModePayload["references"] {
|
||||||
@ -3141,6 +3319,192 @@ async function analyzeReplacementReferenceImage(params: {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function resolveReplacementScenePlanningRuntime(params: {
|
||||||
|
models?: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>;
|
||||||
|
}): Promise<ReplacementScenePlanningRuntime> {
|
||||||
|
try {
|
||||||
|
const siteSettings = await getSiteSettings().catch(() => null);
|
||||||
|
const localDetailAgentResolved = await resolveDetailAgentLocalMoonshotModel({
|
||||||
|
siteSettingModelId: siteSettings?.canvasDetailAgentModelId,
|
||||||
|
});
|
||||||
|
if (localDetailAgentResolved) {
|
||||||
|
return {
|
||||||
|
kind: "local_moonshot",
|
||||||
|
model: localDetailAgentResolved.model,
|
||||||
|
modelName:
|
||||||
|
localDetailAgentResolved.modelRef.displayName ||
|
||||||
|
localDetailAgentResolved.modelRef.modelId,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("[Chat API] Failed to resolve local Moonshot scene planning model:", error);
|
||||||
|
}
|
||||||
|
|
||||||
|
const configuredModels = params.models || (await getEnabledAiModelsWithProvider());
|
||||||
|
const configuredVisionModel =
|
||||||
|
configuredModels.find(
|
||||||
|
({ model }) =>
|
||||||
|
model.supportsVision &&
|
||||||
|
!model.isSubModel &&
|
||||||
|
QWEN_VL_MODEL_MATCHER.test(`${model.displayName} ${model.modelId}`),
|
||||||
|
) ||
|
||||||
|
configuredModels.find(({ model }) => model.supportsVision && !model.isSubModel);
|
||||||
|
|
||||||
|
if (!configuredVisionModel) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const languageModel = await getLanguageModelById(configuredVisionModel.model.id);
|
||||||
|
if (!languageModel) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
kind: "configured_vision",
|
||||||
|
model: languageModel,
|
||||||
|
modelName:
|
||||||
|
configuredVisionModel.model.displayName ||
|
||||||
|
configuredVisionModel.model.modelId,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("[Chat API] Failed to resolve configured replacement scene planning model:", {
|
||||||
|
modelId: configuredVisionModel.model.id,
|
||||||
|
error,
|
||||||
|
});
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function analyzeReplacementExecutionPlan(params: {
|
||||||
|
referenceImageUrl: string;
|
||||||
|
referenceLabel: string;
|
||||||
|
productImageUrls: string[];
|
||||||
|
modelImageUrl?: string | null;
|
||||||
|
workflowPrompt?: string | null;
|
||||||
|
planningRuntime: ReplacementScenePlanningRuntime;
|
||||||
|
models?: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>;
|
||||||
|
}): Promise<ReplacementExecutionPlan> {
|
||||||
|
if (params.planningRuntime) {
|
||||||
|
try {
|
||||||
|
const analysisImageUrls = [
|
||||||
|
...params.productImageUrls,
|
||||||
|
...(params.modelImageUrl ? [params.modelImageUrl] : []),
|
||||||
|
params.referenceImageUrl,
|
||||||
|
].filter(Boolean) as string[];
|
||||||
|
const { inlineDataUrls } = await inlineVisionImages(analysisImageUrls, {
|
||||||
|
maxImages: 8,
|
||||||
|
maxSide: 900,
|
||||||
|
timeoutMs: 12000,
|
||||||
|
retries: 2,
|
||||||
|
loggerPrefix: "[Chat API] Replacement scene plan",
|
||||||
|
}).catch((error) => {
|
||||||
|
console.warn("[Chat API] Failed to inline replacement scene planning images:", error);
|
||||||
|
return { inlineDataUrls: [], skipped: [] };
|
||||||
|
});
|
||||||
|
const effectiveVisionImages =
|
||||||
|
inlineDataUrls.length > 0 ? inlineDataUrls : analysisImageUrls;
|
||||||
|
const workflowPromptExcerpt = String(params.workflowPrompt || "")
|
||||||
|
.trim()
|
||||||
|
.replace(/\s+/g, " ")
|
||||||
|
.slice(0, 2200);
|
||||||
|
const analysisPrompt = [
|
||||||
|
"你是电商详情替换流程中的“场景理解规划器”。",
|
||||||
|
`当前要分析的参考图:${params.referenceLabel}`,
|
||||||
|
`产品图数量:${params.productImageUrls.length} 张。产品图全部属于同一产品的多角度/多细节白底图。`,
|
||||||
|
"这些产品图只是帮助你理解同一产品的结构、材质、颜色、比例和细节,不代表必须把所有角度都同时体现在这一张结果图里。",
|
||||||
|
"你必须根据最后一张参考图底板当前这一屏的机位、主体位置和露出范围,只选择最需要的 1-2 张产品图作为本屏主参考;只有在确实必要时才允许选择 3 张,禁止无理由把所有产品图都选上。",
|
||||||
|
params.modelImageUrl
|
||||||
|
? "还提供了 1 张模特图。只有当最后一张参考图底板当前这一屏本身存在人物展示需求时,才允许使用模特图;如果这一屏不需要人物,必须明确 useModelImage=false。"
|
||||||
|
: "本轮没有模特图,useModelImage 必须为 false。",
|
||||||
|
"最后一张图片是唯一的详情参考图底板,它决定当前这一屏的版式、文字排版、机位、构图、留白和信息层级。",
|
||||||
|
"请完成以下任务:",
|
||||||
|
"1. 判断 shouldReplace:当前参考图里是否存在清晰、主要、适合被替换的家具商品主体。",
|
||||||
|
"2. 给出 selectedProductImageIndexes:从产品图中挑出最适合当前这一屏的产品角度索引(从1开始计数)。",
|
||||||
|
"3. 判断 useModelImage,并说明原因。",
|
||||||
|
"4. 提炼 sceneSummary、cameraSummary、placementSummary、preserveSummary、styleSummary、avoidSummary。",
|
||||||
|
"5. 输出 generationPrompt:这是一段给后续生图模型的本屏生成指令,必须明确“只按当前参考图这一屏需要使用产品角度图,不需要把所有产品图全部融进结果”。",
|
||||||
|
workflowPromptExcerpt
|
||||||
|
? `当前风格与场景限制(来自前置工作流与模式模板):\n${workflowPromptExcerpt}`
|
||||||
|
: "",
|
||||||
|
"只返回严格 JSON,不要任何额外解释。",
|
||||||
|
`JSON 格式:
|
||||||
|
{"shouldReplace":true,"containsFurnitureProduct":true,"reason":"不超过30字","selectedProductImageIndexes":[1,2],"productSelectionReason":"不超过50字","useModelImage":false,"modelImageReason":"不超过40字","sceneSummary":"不超过80字","cameraSummary":"不超过60字","placementSummary":"不超过60字","preserveSummary":"不超过80字","styleSummary":"不超过80字","avoidSummary":"不超过80字","generationPrompt":"不超过300字"}`,
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join("\n");
|
||||||
|
|
||||||
|
const { text } = await generateText({
|
||||||
|
model: params.planningRuntime.model,
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content: [
|
||||||
|
{
|
||||||
|
type: "text",
|
||||||
|
text: analysisPrompt,
|
||||||
|
},
|
||||||
|
...effectiveVisionImages.map((image) => ({
|
||||||
|
type: "image" as const,
|
||||||
|
image,
|
||||||
|
})),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
system:
|
||||||
|
"你是高精度商品详情图场景规划器。重点是:多张产品图只是同一产品的多视角信息源,不是必须全部同时使用。你必须根据参考图当前这一屏的真实需求筛选少量最合适的产品图。",
|
||||||
|
temperature: params.planningRuntime.kind === "local_moonshot" ? 1 : 0.1,
|
||||||
|
maxTokens: 1200,
|
||||||
|
abortSignal: AbortSignal.timeout(45_000),
|
||||||
|
} as any);
|
||||||
|
|
||||||
|
const parsedDecision = parseReplacementReferenceDecision(text);
|
||||||
|
const parsedScenePlan = parseReplacementScenePlan(text, {
|
||||||
|
productImageCount: params.productImageUrls.length,
|
||||||
|
hasModelImage: Boolean(params.modelImageUrl),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (parsedDecision && parsedScenePlan) {
|
||||||
|
return {
|
||||||
|
...parsedDecision,
|
||||||
|
detectionModelName: params.planningRuntime.modelName,
|
||||||
|
detectionSource:
|
||||||
|
params.planningRuntime.kind === "local_moonshot"
|
||||||
|
? "local_moonshot"
|
||||||
|
: "configured_vision",
|
||||||
|
scenePlan: {
|
||||||
|
...parsedScenePlan,
|
||||||
|
analysisModelName: params.planningRuntime.modelName,
|
||||||
|
analysisSource:
|
||||||
|
params.planningRuntime.kind === "local_moonshot"
|
||||||
|
? "local_moonshot"
|
||||||
|
: "configured_vision",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("[Chat API] Replacement scene planning analysis failed:", {
|
||||||
|
referenceLabel: params.referenceLabel,
|
||||||
|
referenceImageUrl: params.referenceImageUrl,
|
||||||
|
error,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const fallbackDecision = await analyzeReplacementReferenceImage({
|
||||||
|
imageUrl: params.referenceImageUrl,
|
||||||
|
models: params.models,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
...fallbackDecision,
|
||||||
|
scenePlan: buildHeuristicReplacementScenePlan({
|
||||||
|
productImageCount: params.productImageUrls.length,
|
||||||
|
hasModelImage: Boolean(params.modelImageUrl),
|
||||||
|
shouldReplace: fallbackDecision.shouldReplace,
|
||||||
|
reason: fallbackDecision.reason,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function buildAgentReplacementPrompt(params: {
|
function buildAgentReplacementPrompt(params: {
|
||||||
basePrompt: string;
|
basePrompt: string;
|
||||||
modeType: "detail_replacement" | "sku_replacement";
|
modeType: "detail_replacement" | "sku_replacement";
|
||||||
@ -3149,8 +3513,10 @@ function buildAgentReplacementPrompt(params: {
|
|||||||
totalReferences: number;
|
totalReferences: number;
|
||||||
hasModelImage: boolean;
|
hasModelImage: boolean;
|
||||||
productImageCount: number;
|
productImageCount: number;
|
||||||
|
originalProductImageCount: number;
|
||||||
templateReferenceCount: number;
|
templateReferenceCount: number;
|
||||||
pairedProductIndex?: number;
|
pairedProductIndex?: number;
|
||||||
|
scenePlan?: ReplacementScenePlan | null;
|
||||||
}) {
|
}) {
|
||||||
const imageOrderLines: string[] = [];
|
const imageOrderLines: string[] = [];
|
||||||
let nextImageIndex = 1;
|
let nextImageIndex = 1;
|
||||||
@ -3188,6 +3554,48 @@ function buildAgentReplacementPrompt(params: {
|
|||||||
`图片${nextImageIndex}:当前参考图底板(唯一版式基准图),必须保留它的文字排版、信息区、分割线、留白、机位和整体结构。`
|
`图片${nextImageIndex}:当前参考图底板(唯一版式基准图),必须保留它的文字排版、信息区、分割线、留白、机位和整体结构。`
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const scenePlanLines = params.scenePlan
|
||||||
|
? [
|
||||||
|
"【本屏场景预分析】",
|
||||||
|
params.scenePlan.sceneSummary
|
||||||
|
? `场景理解:${params.scenePlan.sceneSummary}`
|
||||||
|
: "",
|
||||||
|
params.scenePlan.cameraSummary
|
||||||
|
? `机位构图:${params.scenePlan.cameraSummary}`
|
||||||
|
: "",
|
||||||
|
params.scenePlan.placementSummary
|
||||||
|
? `主体位置:${params.scenePlan.placementSummary}`
|
||||||
|
: "",
|
||||||
|
params.scenePlan.preserveSummary
|
||||||
|
? `必须保留:${params.scenePlan.preserveSummary}`
|
||||||
|
: "",
|
||||||
|
params.scenePlan.styleSummary
|
||||||
|
? `风格约束:${params.scenePlan.styleSummary}`
|
||||||
|
: "",
|
||||||
|
params.scenePlan.avoidSummary
|
||||||
|
? `禁止事项:${params.scenePlan.avoidSummary}`
|
||||||
|
: "",
|
||||||
|
params.originalProductImageCount > params.productImageCount
|
||||||
|
? `原始共有 ${params.originalProductImageCount} 张产品角度图,本屏已筛选出最相关的 ${params.productImageCount} 张输入图;其余未入选产品图只是补充理解,不需要也不允许强行全部融入同一张结果图。`
|
||||||
|
: "",
|
||||||
|
params.scenePlan.productSelectionReason
|
||||||
|
? `产品图筛选理由:${params.scenePlan.productSelectionReason}`
|
||||||
|
: "",
|
||||||
|
params.scenePlan.useModelImage
|
||||||
|
? params.scenePlan.modelImageReason
|
||||||
|
? `本屏允许使用模特图:${params.scenePlan.modelImageReason}`
|
||||||
|
: "本屏允许使用模特图,但仅在参考图当前这一屏确实存在人物展示需求时使用。"
|
||||||
|
: params.hasModelImage
|
||||||
|
? params.scenePlan.modelImageReason
|
||||||
|
? `本屏禁止使用模特图:${params.scenePlan.modelImageReason}`
|
||||||
|
: "本屏不需要模特图,不允许把模特图强行融入结果。"
|
||||||
|
: "",
|
||||||
|
params.scenePlan.generationPrompt
|
||||||
|
? `本屏生成方向:${params.scenePlan.generationPrompt}`
|
||||||
|
: "",
|
||||||
|
].filter(Boolean)
|
||||||
|
: [];
|
||||||
|
|
||||||
const instructionLines =
|
const instructionLines =
|
||||||
params.modeType === "detail_replacement"
|
params.modeType === "detail_replacement"
|
||||||
? [
|
? [
|
||||||
@ -3195,6 +3603,7 @@ function buildAgentReplacementPrompt(params: {
|
|||||||
`当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张详情参考图:${params.referenceLabel}`,
|
`当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张详情参考图:${params.referenceLabel}`,
|
||||||
"【图片顺序规则】",
|
"【图片顺序规则】",
|
||||||
...imageOrderLines,
|
...imageOrderLines,
|
||||||
|
...scenePlanLines,
|
||||||
"你现在做的不是重新生成一张新图,而是编辑最后一张参考图本身。",
|
"你现在做的不是重新生成一张新图,而是编辑最后一张参考图本身。",
|
||||||
"如果前面商品图、模特图与最后一张参考图底板存在冲突,始终以最后一张参考图底板的版式、机位、构图和信息层级为最高优先级。",
|
"如果前面商品图、模特图与最后一张参考图底板存在冲突,始终以最后一张参考图底板的版式、机位、构图和信息层级为最高优先级。",
|
||||||
"最后一张参考图是唯一底板,必须保留这张图的文字排版、品牌字样、标题区、卖点区、分割线、留白、机位、构图和信息层级。",
|
"最后一张参考图是唯一底板,必须保留这张图的文字排版、品牌字样、标题区、卖点区、分割线、留白、机位、构图和信息层级。",
|
||||||
@ -3216,6 +3625,7 @@ function buildAgentReplacementPrompt(params: {
|
|||||||
`当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张 SKU 参考图:${params.referenceLabel}`,
|
`当前要处理第 ${params.referenceIndex + 1} / ${params.totalReferences} 张 SKU 参考图:${params.referenceLabel}`,
|
||||||
"【图片顺序规则】",
|
"【图片顺序规则】",
|
||||||
...imageOrderLines,
|
...imageOrderLines,
|
||||||
|
...scenePlanLines,
|
||||||
params.pairedProductIndex !== undefined
|
params.pairedProductIndex !== undefined
|
||||||
? `请优先使用第 ${params.pairedProductIndex + 1} 张输入商品图作为本张参考图的一对一替换主体。`
|
? `请优先使用第 ${params.pairedProductIndex + 1} 张输入商品图作为本张参考图的一对一替换主体。`
|
||||||
: "请使用输入商品图作为本张参考图的一对一替换主体。",
|
: "请使用输入商品图作为本张参考图的一对一替换主体。",
|
||||||
@ -6150,6 +6560,14 @@ export async function POST(request: Request) {
|
|||||||
return undefined;
|
return undefined;
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
const replacementScenePlanningRuntime =
|
||||||
|
await resolveReplacementScenePlanningRuntime({
|
||||||
|
models: replacementVisionModels,
|
||||||
|
});
|
||||||
|
console.info("[Chat API] Replacement scene planning runtime:", {
|
||||||
|
kind: replacementScenePlanningRuntime?.kind || "none",
|
||||||
|
modelName: replacementScenePlanningRuntime?.modelName || null,
|
||||||
|
});
|
||||||
const stopKeepalive = startStreamKeepalive(
|
const stopKeepalive = startStreamKeepalive(
|
||||||
writer,
|
writer,
|
||||||
"agent-preset:replacement",
|
"agent-preset:replacement",
|
||||||
@ -6276,8 +6694,13 @@ export async function POST(request: Request) {
|
|||||||
),
|
),
|
||||||
]
|
]
|
||||||
: productInputImageUrls;
|
: productInputImageUrls;
|
||||||
const referenceDecision = await analyzeReplacementReferenceImage({
|
const replacementExecutionPlan = await analyzeReplacementExecutionPlan({
|
||||||
imageUrl: reference.imageUrl,
|
referenceImageUrl: reference.imageUrl,
|
||||||
|
referenceLabel: reference.label,
|
||||||
|
productImageUrls: orderedProductContextImageUrls,
|
||||||
|
modelImageUrl: modelInputImageUrl,
|
||||||
|
workflowPrompt: baseReplacementPrompt || hiddenPromptText || null,
|
||||||
|
planningRuntime: replacementScenePlanningRuntime,
|
||||||
models: replacementVisionModels,
|
models: replacementVisionModels,
|
||||||
});
|
});
|
||||||
const replacementSourceType = isDetailReplacementPreset
|
const replacementSourceType = isDetailReplacementPreset
|
||||||
@ -6291,10 +6714,27 @@ export async function POST(request: Request) {
|
|||||||
referenceImageUrl: reference.imageUrl,
|
referenceImageUrl: reference.imageUrl,
|
||||||
pairedProductIndex:
|
pairedProductIndex:
|
||||||
typeof pairedProductIndex === "number" ? pairedProductIndex + 1 : null,
|
typeof pairedProductIndex === "number" ? pairedProductIndex + 1 : null,
|
||||||
shouldReplace: referenceDecision.shouldReplace,
|
shouldReplace: replacementExecutionPlan.shouldReplace,
|
||||||
reason: referenceDecision.reason,
|
reason: replacementExecutionPlan.reason,
|
||||||
detectionModelName: referenceDecision.detectionModelName,
|
detectionModelName: replacementExecutionPlan.detectionModelName,
|
||||||
detectionSource: referenceDecision.detectionSource,
|
detectionSource: replacementExecutionPlan.detectionSource,
|
||||||
|
scenePlanSource: replacementExecutionPlan.scenePlan.analysisSource,
|
||||||
|
selectedProductImageIndexes:
|
||||||
|
replacementExecutionPlan.scenePlan.selectedProductImageIndexes,
|
||||||
|
useModelImage: replacementExecutionPlan.scenePlan.useModelImage,
|
||||||
|
});
|
||||||
|
console.info("[Chat API] Replacement scene plan summary:", {
|
||||||
|
mode: replacementSourceType,
|
||||||
|
referenceIndex: referenceIndex + 1,
|
||||||
|
referenceLabel: reference.label,
|
||||||
|
analysisModelName: replacementExecutionPlan.scenePlan.analysisModelName,
|
||||||
|
analysisSource: replacementExecutionPlan.scenePlan.analysisSource,
|
||||||
|
selectedProductImageIndexes:
|
||||||
|
replacementExecutionPlan.scenePlan.selectedProductImageIndexes,
|
||||||
|
productSelectionReason:
|
||||||
|
replacementExecutionPlan.scenePlan.productSelectionReason,
|
||||||
|
useModelImage: replacementExecutionPlan.scenePlan.useModelImage,
|
||||||
|
modelImageReason: replacementExecutionPlan.scenePlan.modelImageReason,
|
||||||
});
|
});
|
||||||
const referenceSourceImageUrls = Array.from(
|
const referenceSourceImageUrls = Array.from(
|
||||||
new Set([
|
new Set([
|
||||||
@ -6304,17 +6744,17 @@ export async function POST(request: Request) {
|
|||||||
]),
|
]),
|
||||||
);
|
);
|
||||||
|
|
||||||
if (!referenceDecision.shouldReplace) {
|
if (!replacementExecutionPlan.shouldReplace) {
|
||||||
console.info("[Chat API] Replacement reference skipped and reused:", {
|
console.info("[Chat API] Replacement reference skipped and reused:", {
|
||||||
mode: replacementSourceType,
|
mode: replacementSourceType,
|
||||||
referenceIndex: referenceIndex + 1,
|
referenceIndex: referenceIndex + 1,
|
||||||
totalReferences: replacementReferences.length,
|
totalReferences: replacementReferences.length,
|
||||||
referenceLabel: reference.label,
|
referenceLabel: reference.label,
|
||||||
reason: referenceDecision.reason,
|
reason: replacementExecutionPlan.reason,
|
||||||
});
|
});
|
||||||
await emitReplacementResultMessage({
|
await emitReplacementResultMessage({
|
||||||
reference,
|
reference,
|
||||||
bodyText: referenceDecision.reason,
|
bodyText: replacementExecutionPlan.reason,
|
||||||
image: {
|
image: {
|
||||||
originalUrl: reference.imageUrl,
|
originalUrl: reference.imageUrl,
|
||||||
thumbnailUrl: reference.imageUrl,
|
thumbnailUrl: reference.imageUrl,
|
||||||
@ -6330,11 +6770,29 @@ export async function POST(request: Request) {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const selectedProductIndexes = replacementExecutionPlan.scenePlan
|
||||||
|
.selectedProductImageIndexes.map((index) => index - 1)
|
||||||
|
.filter(
|
||||||
|
(index) =>
|
||||||
|
index >= 0 && index < orderedProductContextImageUrls.length,
|
||||||
|
);
|
||||||
|
const selectedProductContextImageUrls =
|
||||||
|
selectedProductIndexes.length > 0
|
||||||
|
? selectedProductIndexes.map(
|
||||||
|
(index) => orderedProductContextImageUrls[index]!,
|
||||||
|
)
|
||||||
|
: orderedProductContextImageUrls.slice(
|
||||||
|
0,
|
||||||
|
Math.min(orderedProductContextImageUrls.length, 2),
|
||||||
|
);
|
||||||
const sourceImageUrls = Array.from(
|
const sourceImageUrls = Array.from(
|
||||||
new Set(
|
new Set(
|
||||||
[
|
[
|
||||||
...orderedProductContextImageUrls,
|
...selectedProductContextImageUrls,
|
||||||
...(modelInputImageUrl ? [modelInputImageUrl] : []),
|
...(replacementExecutionPlan.scenePlan.useModelImage &&
|
||||||
|
modelInputImageUrl
|
||||||
|
? [modelInputImageUrl]
|
||||||
|
: []),
|
||||||
...replacementTemplateReferenceImageUrls,
|
...replacementTemplateReferenceImageUrls,
|
||||||
reference.imageUrl,
|
reference.imageUrl,
|
||||||
].filter(Boolean),
|
].filter(Boolean),
|
||||||
@ -6350,10 +6808,14 @@ export async function POST(request: Request) {
|
|||||||
referenceLabel: reference.label,
|
referenceLabel: reference.label,
|
||||||
referenceIndex,
|
referenceIndex,
|
||||||
totalReferences: replacementReferences.length,
|
totalReferences: replacementReferences.length,
|
||||||
hasModelImage: Boolean(modelInputImageUrl),
|
hasModelImage:
|
||||||
productImageCount: productInputImageUrls.length,
|
replacementExecutionPlan.scenePlan.useModelImage &&
|
||||||
|
Boolean(modelInputImageUrl),
|
||||||
|
productImageCount: selectedProductContextImageUrls.length,
|
||||||
|
originalProductImageCount: productInputImageUrls.length,
|
||||||
templateReferenceCount: replacementTemplateReferenceImageUrls.length,
|
templateReferenceCount: replacementTemplateReferenceImageUrls.length,
|
||||||
pairedProductIndex,
|
pairedProductIndex,
|
||||||
|
scenePlan: replacementExecutionPlan.scenePlan,
|
||||||
});
|
});
|
||||||
console.info("[Chat API] Replacement image generation started:", {
|
console.info("[Chat API] Replacement image generation started:", {
|
||||||
mode: replacementSourceType,
|
mode: replacementSourceType,
|
||||||
@ -6364,8 +6826,11 @@ export async function POST(request: Request) {
|
|||||||
typeof pairedProductIndex === "number" ? pairedProductIndex + 1 : null,
|
typeof pairedProductIndex === "number" ? pairedProductIndex + 1 : null,
|
||||||
sourceImageCount: sourceImageUrls.length,
|
sourceImageCount: sourceImageUrls.length,
|
||||||
imageOrder: {
|
imageOrder: {
|
||||||
productImages: productInputImageUrls.length,
|
productImages: selectedProductContextImageUrls.length,
|
||||||
hasModelImage: Boolean(modelInputImageUrl),
|
originalProductImages: productInputImageUrls.length,
|
||||||
|
hasModelImage:
|
||||||
|
replacementExecutionPlan.scenePlan.useModelImage &&
|
||||||
|
Boolean(modelInputImageUrl),
|
||||||
templateReferenceImages: replacementTemplateReferenceImageUrls.length,
|
templateReferenceImages: replacementTemplateReferenceImageUrls.length,
|
||||||
finalReferenceImageIndex: sourceImageUrls.length,
|
finalReferenceImageIndex: sourceImageUrls.length,
|
||||||
},
|
},
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user