feat(scene-fusion): enforce pre-rmbg for random mode and add agent background prompt enhancement
This commit is contained in:
parent
8688cac064
commit
7a32e6ef65
@ -23,6 +23,7 @@ import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
|
|||||||
const sceneFusionRequestSchema = z.object({
|
const sceneFusionRequestSchema = z.object({
|
||||||
productImage: z.string().min(1, "产品图不能为空"),
|
productImage: z.string().min(1, "产品图不能为空"),
|
||||||
productName: z.string().optional(), // 产品名称(如"沙发")
|
productName: z.string().optional(), // 产品名称(如"沙发")
|
||||||
|
sceneMode: z.enum(["scene-fission", "multi-product-scene", "random-bg"]).optional(),
|
||||||
aspectRatio: z.string().optional(), // 图片比例(如"1:1", "4:3", "auto")
|
aspectRatio: z.string().optional(), // 图片比例(如"1:1", "4:3", "auto")
|
||||||
imageSize: z.string().optional(), // 图片分辨率(如"1K", "2K", "4K")
|
imageSize: z.string().optional(), // 图片分辨率(如"1K", "2K", "4K")
|
||||||
scenes: z.array(z.object({
|
scenes: z.array(z.object({
|
||||||
@ -395,6 +396,133 @@ async function detectTargetNameWithAgent(
|
|||||||
return { name, confidence, model };
|
return { name, confidence, model };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function extractStyleHintFromPrompt(customPrompt: string | undefined): string | undefined {
|
||||||
|
const content = String(customPrompt || "").trim();
|
||||||
|
if (!content) return undefined;
|
||||||
|
const quoted = content.match(/[“"]([^”"]+?)(?:风格|场景)/);
|
||||||
|
if (quoted?.[1]) {
|
||||||
|
return normalizeSceneFusionName(quoted[1]) || quoted[1].trim();
|
||||||
|
}
|
||||||
|
const styleOnly = content.match(/(?:替换背景为|背景为|风格为)\s*([^,。;]+)/);
|
||||||
|
if (styleOnly?.[1]) {
|
||||||
|
return normalizeSceneFusionName(styleOnly[1]) || styleOnly[1].trim();
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function mergeCustomPrompt(basePrompt: string | undefined, addition: string | undefined): string | undefined {
|
||||||
|
const base = String(basePrompt || "").trim();
|
||||||
|
const extra = String(addition || "").trim();
|
||||||
|
if (!base && !extra) return undefined;
|
||||||
|
if (!base) return extra.slice(0, 280);
|
||||||
|
if (!extra) return base.slice(0, 280);
|
||||||
|
if (base.includes(extra)) return base.slice(0, 280);
|
||||||
|
if (extra.includes(base)) return extra.slice(0, 280);
|
||||||
|
return `${base} ${extra}`.slice(0, 280);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function generateRandomBackgroundPromptWithAgent(params: {
|
||||||
|
productImageUrl: string;
|
||||||
|
productNameHint?: string;
|
||||||
|
styleHint?: string;
|
||||||
|
userPrompt?: string;
|
||||||
|
}): Promise<string | null> {
|
||||||
|
const prompt = [
|
||||||
|
"你是电商场景策划助手。",
|
||||||
|
`产品名称:${params.productNameHint || "该产品"}`,
|
||||||
|
params.styleHint ? `期望风格:${params.styleHint}` : "",
|
||||||
|
params.userPrompt ? `用户补充要求:${params.userPrompt}` : "",
|
||||||
|
"请基于产品图生成一条“仅描述背景场景”的中文提示词。",
|
||||||
|
"要求:",
|
||||||
|
"- 场景必须与产品品类匹配,适合商品展示",
|
||||||
|
"- 只描述背景、灯光、材质氛围,不改动产品主体外观",
|
||||||
|
"- 不要输出流程指令(例如“替换背景/保持主体”)",
|
||||||
|
"- 20~60字,语义具体可执行",
|
||||||
|
"仅返回 JSON:",
|
||||||
|
"{\"backgroundPrompt\":\"...\"}",
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join("\n");
|
||||||
|
|
||||||
|
const { text } = await callQwen3Vl({
|
||||||
|
imageUrl: params.productImageUrl,
|
||||||
|
prompt,
|
||||||
|
temperature: 0.2,
|
||||||
|
maxTokens: 240,
|
||||||
|
timeoutMs: SCENE_FUSION_AGENT_TIMEOUT_MS,
|
||||||
|
retries: 1,
|
||||||
|
});
|
||||||
|
|
||||||
|
const parsed = extractJsonObject(text);
|
||||||
|
const backgroundPromptRaw = String(parsed?.backgroundPrompt || "").replace(/\s+/g, " ").trim();
|
||||||
|
if (!backgroundPromptRaw || backgroundPromptRaw.length < 6) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return backgroundPromptRaw.slice(0, 120);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function enrichRandomBackgroundScenesWithAgent(params: {
|
||||||
|
requestId: string;
|
||||||
|
productImageUrl: string;
|
||||||
|
productName?: string;
|
||||||
|
scenes: Array<{
|
||||||
|
id: string;
|
||||||
|
image: string;
|
||||||
|
targetName?: string;
|
||||||
|
customPrompt?: string;
|
||||||
|
imageUrl: string;
|
||||||
|
}>;
|
||||||
|
}): Promise<Array<{
|
||||||
|
id: string;
|
||||||
|
image: string;
|
||||||
|
targetName?: string;
|
||||||
|
customPrompt?: string;
|
||||||
|
imageUrl: string;
|
||||||
|
}>> {
|
||||||
|
const nextScenes = params.scenes.map((scene) => ({ ...scene }));
|
||||||
|
const workerTargets = nextScenes;
|
||||||
|
if (workerTargets.length === 0) return nextScenes;
|
||||||
|
|
||||||
|
let cursor = 0;
|
||||||
|
const workerCount = Math.max(1, Math.min(SCENE_FUSION_AGENT_MAX_WORKERS, workerTargets.length));
|
||||||
|
const workers = Array.from({ length: workerCount }, async () => {
|
||||||
|
while (true) {
|
||||||
|
const current = cursor;
|
||||||
|
cursor += 1;
|
||||||
|
if (current >= workerTargets.length) break;
|
||||||
|
|
||||||
|
const scene = workerTargets[current];
|
||||||
|
if (!scene) continue;
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (!normalizeSceneFusionName(scene.targetName) && params.productName) {
|
||||||
|
scene.targetName = params.productName;
|
||||||
|
}
|
||||||
|
|
||||||
|
const styleHint = extractStyleHintFromPrompt(scene.customPrompt);
|
||||||
|
const generatedPrompt = await generateRandomBackgroundPromptWithAgent({
|
||||||
|
productImageUrl: params.productImageUrl,
|
||||||
|
productNameHint: params.productName,
|
||||||
|
styleHint,
|
||||||
|
userPrompt: scene.customPrompt,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (generatedPrompt) {
|
||||||
|
scene.customPrompt = mergeCustomPrompt(scene.customPrompt, `场景建议:${generatedPrompt}`);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.warn(
|
||||||
|
`[SceneFusion][${params.requestId}] Random scene prompt enhance skipped: scene=${scene.id}`,
|
||||||
|
error
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
await Promise.all(workers);
|
||||||
|
return nextScenes;
|
||||||
|
}
|
||||||
|
|
||||||
async function enrichSceneLabelsWithAgent(params: {
|
async function enrichSceneLabelsWithAgent(params: {
|
||||||
requestId: string;
|
requestId: string;
|
||||||
productImageUrl: string;
|
productImageUrl: string;
|
||||||
@ -1426,8 +1554,9 @@ export async function POST(request: Request) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const { productImage, productName, aspectRatio, imageSize, scenes } = parseResult.data;
|
const { productImage, productName, sceneMode, aspectRatio, imageSize, scenes } = parseResult.data;
|
||||||
const requestId = `sf-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
const requestId = `sf-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||||
|
const resolvedSceneMode = sceneMode || "scene-fission";
|
||||||
|
|
||||||
// 计算输出尺寸
|
// 计算输出尺寸
|
||||||
const size = calculateSize(aspectRatio || "auto", imageSize || "2K");
|
const size = calculateSize(aspectRatio || "auto", imageSize || "2K");
|
||||||
@ -1444,6 +1573,7 @@ export async function POST(request: Request) {
|
|||||||
|
|
||||||
console.log(`[SceneFusion] Processing ${scenes.length} scenes for user ${session.user.id}`);
|
console.log(`[SceneFusion] Processing ${scenes.length} scenes for user ${session.user.id}`);
|
||||||
console.log(`[SceneFusion] Product name: ${productName || '(not specified)'}`);
|
console.log(`[SceneFusion] Product name: ${productName || '(not specified)'}`);
|
||||||
|
console.log(`[SceneFusion] Scene mode: ${resolvedSceneMode}`);
|
||||||
console.log(`[SceneFusion] Using default prompt: ${config.defaultPrompt.substring(0, 50)}...`);
|
console.log(`[SceneFusion] Using default prompt: ${config.defaultPrompt.substring(0, 50)}...`);
|
||||||
console.log(`[SceneFusion] Preferred model ID: ${config.modelId || '(none)'}`);
|
console.log(`[SceneFusion] Preferred model ID: ${config.modelId || '(none)'}`);
|
||||||
|
|
||||||
@ -1500,11 +1630,12 @@ export async function POST(request: Request) {
|
|||||||
if (productImage.startsWith("data:")) {
|
if (productImage.startsWith("data:")) {
|
||||||
productImageUrl = await uploadBase64ToS3(productImage, "scene-fusion-product");
|
productImageUrl = await uploadBase64ToS3(productImage, "scene-fusion-product");
|
||||||
}
|
}
|
||||||
|
const shouldForcePreRmbg = resolvedSceneMode === "random-bg";
|
||||||
const preprocessedProduct = await preprocessProductImageWithRmbg(
|
const preprocessedProduct = await preprocessProductImageWithRmbg(
|
||||||
request,
|
request,
|
||||||
requestId,
|
requestId,
|
||||||
productImageUrl,
|
productImageUrl,
|
||||||
config.preRmbgEnabled
|
shouldForcePreRmbg ? true : config.preRmbgEnabled
|
||||||
);
|
);
|
||||||
const fusionProductImageUrl = preprocessedProduct.productImageUrl;
|
const fusionProductImageUrl = preprocessedProduct.productImageUrl;
|
||||||
console.log(
|
console.log(
|
||||||
@ -1531,7 +1662,15 @@ export async function POST(request: Request) {
|
|||||||
scenes: processedScenes,
|
scenes: processedScenes,
|
||||||
});
|
});
|
||||||
const resolvedProductName = enriched.productName;
|
const resolvedProductName = enriched.productName;
|
||||||
const resolvedScenes = enriched.scenes;
|
let resolvedScenes = enriched.scenes;
|
||||||
|
if (resolvedSceneMode === "random-bg") {
|
||||||
|
resolvedScenes = await enrichRandomBackgroundScenesWithAgent({
|
||||||
|
requestId,
|
||||||
|
productImageUrl: fusionProductImageUrl,
|
||||||
|
productName: resolvedProductName,
|
||||||
|
scenes: resolvedScenes,
|
||||||
|
});
|
||||||
|
}
|
||||||
const resolvedSceneMap = new Map(resolvedScenes.map((scene) => [scene.id, scene]));
|
const resolvedSceneMap = new Map(resolvedScenes.map((scene) => [scene.id, scene]));
|
||||||
const imageHashCache = new Map<string, string>();
|
const imageHashCache = new Map<string, string>();
|
||||||
|
|
||||||
@ -1540,6 +1679,13 @@ export async function POST(request: Request) {
|
|||||||
.map((scene) => `${scene.id}:${scene.targetName || "-"}`)
|
.map((scene) => `${scene.id}:${scene.targetName || "-"}`)
|
||||||
.join(", ")}`
|
.join(", ")}`
|
||||||
);
|
);
|
||||||
|
if (resolvedSceneMode === "random-bg") {
|
||||||
|
console.log(
|
||||||
|
`[SceneFusion][${requestId}] Random-bg prompts: ${resolvedScenes
|
||||||
|
.map((scene) => `${scene.id}:${(scene.customPrompt || "").slice(0, 80) || "-"}`)
|
||||||
|
.join(" | ")}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// 6. 创建流式响应,每完成一张图就立即发送给前端
|
// 6. 创建流式响应,每完成一张图就立即发送给前端
|
||||||
const stream = new ReadableStream({
|
const stream = new ReadableStream({
|
||||||
|
|||||||
@ -417,6 +417,7 @@ export function SceneFusionView({
|
|||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
productImage,
|
productImage,
|
||||||
productName: productName || undefined,
|
productName: productName || undefined,
|
||||||
|
sceneMode: mode,
|
||||||
aspectRatio,
|
aspectRatio,
|
||||||
imageSize,
|
imageSize,
|
||||||
scenes: scenesPayload,
|
scenes: scenesPayload,
|
||||||
@ -632,6 +633,7 @@ export function SceneFusionView({
|
|||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
productImage: params.productImageInput,
|
productImage: params.productImageInput,
|
||||||
productName: productName || undefined,
|
productName: productName || undefined,
|
||||||
|
sceneMode: mode,
|
||||||
aspectRatio,
|
aspectRatio,
|
||||||
imageSize,
|
imageSize,
|
||||||
scenes: [
|
scenes: [
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user