feat(scene-fusion): enforce pre-rmbg for random mode and add agent background prompt enhancement
This commit is contained in:
Your Name
parent
8688cac064
commit
7a32e6ef65
@ -23,6 +23,7 @@ import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
|
||||
const sceneFusionRequestSchema = z.object({
|
||||
productImage: z.string().min(1, "产品图不能为空"),
|
||||
productName: z.string().optional(), // 产品名称（如"沙发"）
|
||||
sceneMode: z.enum(["scene-fission", "multi-product-scene", "random-bg"]).optional(),
|
||||
aspectRatio: z.string().optional(), // 图片比例（如"1:1", "4:3", "auto"）
|
||||
imageSize: z.string().optional(), // 图片分辨率（如"1K", "2K", "4K"）
|
||||
scenes: z.array(z.object({
|
||||
@ -395,6 +396,133 @@ async function detectTargetNameWithAgent(
|
||||
return { name, confidence, model };
|
||||
}
|
||||
|
||||
function extractStyleHintFromPrompt(customPrompt: string | undefined): string | undefined {
|
||||
const content = String(customPrompt || "").trim();
|
||||
if (!content) return undefined;
|
||||
const quoted = content.match(/[“"]([^”"]+?)(?:风格|场景)/);
|
||||
if (quoted?.[1]) {
|
||||
return normalizeSceneFusionName(quoted[1]) || quoted[1].trim();
|
||||
}
|
||||
const styleOnly = content.match(/(?:替换背景为|背景为|风格为)\s*([^，。；]+)/);
|
||||
if (styleOnly?.[1]) {
|
||||
return normalizeSceneFusionName(styleOnly[1]) || styleOnly[1].trim();
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function mergeCustomPrompt(basePrompt: string | undefined, addition: string | undefined): string | undefined {
|
||||
const base = String(basePrompt || "").trim();
|
||||
const extra = String(addition || "").trim();
|
||||
if (!base && !extra) return undefined;
|
||||
if (!base) return extra.slice(0, 280);
|
||||
if (!extra) return base.slice(0, 280);
|
||||
if (base.includes(extra)) return base.slice(0, 280);
|
||||
if (extra.includes(base)) return extra.slice(0, 280);
|
||||
return `${base} ${extra}`.slice(0, 280);
|
||||
}
|
||||
|
||||
async function generateRandomBackgroundPromptWithAgent(params: {
|
||||
productImageUrl: string;
|
||||
productNameHint?: string;
|
||||
styleHint?: string;
|
||||
userPrompt?: string;
|
||||
}): Promise<string | null> {
|
||||
const prompt = [
|
||||
"你是电商场景策划助手。",
|
||||
`产品名称：${params.productNameHint || "该产品"}`,
|
||||
params.styleHint ? `期望风格：${params.styleHint}` : "",
|
||||
params.userPrompt ? `用户补充要求：${params.userPrompt}` : "",
|
||||
"请基于产品图生成一条“仅描述背景场景”的中文提示词。",
|
||||
"要求：",
|
||||
"- 场景必须与产品品类匹配，适合商品展示",
|
||||
"- 只描述背景、灯光、材质氛围，不改动产品主体外观",
|
||||
"- 不要输出流程指令（例如“替换背景/保持主体”）",
|
||||
"- 20~60字，语义具体可执行",
|
||||
"仅返回 JSON：",
|
||||
"{\"backgroundPrompt\":\"...\"}",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
|
||||
const { text } = await callQwen3Vl({
|
||||
imageUrl: params.productImageUrl,
|
||||
prompt,
|
||||
temperature: 0.2,
|
||||
maxTokens: 240,
|
||||
timeoutMs: SCENE_FUSION_AGENT_TIMEOUT_MS,
|
||||
retries: 1,
|
||||
});
|
||||
|
||||
const parsed = extractJsonObject(text);
|
||||
const backgroundPromptRaw = String(parsed?.backgroundPrompt || "").replace(/\s+/g, " ").trim();
|
||||
if (!backgroundPromptRaw || backgroundPromptRaw.length < 6) {
|
||||
return null;
|
||||
}
|
||||
return backgroundPromptRaw.slice(0, 120);
|
||||
}
|
||||
|
||||
async function enrichRandomBackgroundScenesWithAgent(params: {
|
||||
requestId: string;
|
||||
productImageUrl: string;
|
||||
productName?: string;
|
||||
scenes: Array<{
|
||||
id: string;
|
||||
image: string;
|
||||
targetName?: string;
|
||||
customPrompt?: string;
|
||||
imageUrl: string;
|
||||
}>;
|
||||
}): Promise<Array<{
|
||||
id: string;
|
||||
image: string;
|
||||
targetName?: string;
|
||||
customPrompt?: string;
|
||||
imageUrl: string;
|
||||
}>> {
|
||||
const nextScenes = params.scenes.map((scene) => ({ ...scene }));
|
||||
const workerTargets = nextScenes;
|
||||
if (workerTargets.length === 0) return nextScenes;
|
||||
|
||||
let cursor = 0;
|
||||
const workerCount = Math.max(1, Math.min(SCENE_FUSION_AGENT_MAX_WORKERS, workerTargets.length));
|
||||
const workers = Array.from({ length: workerCount }, async () => {
|
||||
while (true) {
|
||||
const current = cursor;
|
||||
cursor += 1;
|
||||
if (current >= workerTargets.length) break;
|
||||
|
||||
const scene = workerTargets[current];
|
||||
if (!scene) continue;
|
||||
|
||||
try {
|
||||
if (!normalizeSceneFusionName(scene.targetName) && params.productName) {
|
||||
scene.targetName = params.productName;
|
||||
}
|
||||
|
||||
const styleHint = extractStyleHintFromPrompt(scene.customPrompt);
|
||||
const generatedPrompt = await generateRandomBackgroundPromptWithAgent({
|
||||
productImageUrl: params.productImageUrl,
|
||||
productNameHint: params.productName,
|
||||
styleHint,
|
||||
userPrompt: scene.customPrompt,
|
||||
});
|
||||
|
||||
if (generatedPrompt) {
|
||||
scene.customPrompt = mergeCustomPrompt(scene.customPrompt, `场景建议：${generatedPrompt}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`[SceneFusion][${params.requestId}] Random scene prompt enhance skipped: scene=${scene.id}`,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(workers);
|
||||
return nextScenes;
|
||||
}
|
||||
|
||||
async function enrichSceneLabelsWithAgent(params: {
|
||||
requestId: string;
|
||||
productImageUrl: string;
|
||||
@ -1426,8 +1554,9 @@ export async function POST(request: Request) {
|
||||
);
|
||||
}
|
||||
|
||||
const { productImage, productName, aspectRatio, imageSize, scenes } = parseResult.data;
|
||||
const { productImage, productName, sceneMode, aspectRatio, imageSize, scenes } = parseResult.data;
|
||||
const requestId = `sf-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
const resolvedSceneMode = sceneMode || "scene-fission";
|
||||
|
||||
// 计算输出尺寸
|
||||
const size = calculateSize(aspectRatio || "auto", imageSize || "2K");
|
||||
@ -1444,6 +1573,7 @@ export async function POST(request: Request) {
|
||||
|
||||
console.log(`[SceneFusion] Processing ${scenes.length} scenes for user ${session.user.id}`);
|
||||
console.log(`[SceneFusion] Product name: ${productName || '(not specified)'}`);
|
||||
console.log(`[SceneFusion] Scene mode: ${resolvedSceneMode}`);
|
||||
console.log(`[SceneFusion] Using default prompt: ${config.defaultPrompt.substring(0, 50)}...`);
|
||||
console.log(`[SceneFusion] Preferred model ID: ${config.modelId || '(none)'}`);
|
||||
|
||||
@ -1500,11 +1630,12 @@ export async function POST(request: Request) {
|
||||
if (productImage.startsWith("data:")) {
|
||||
productImageUrl = await uploadBase64ToS3(productImage, "scene-fusion-product");
|
||||
}
|
||||
const shouldForcePreRmbg = resolvedSceneMode === "random-bg";
|
||||
const preprocessedProduct = await preprocessProductImageWithRmbg(
|
||||
request,
|
||||
requestId,
|
||||
productImageUrl,
|
||||
config.preRmbgEnabled
|
||||
shouldForcePreRmbg ? true : config.preRmbgEnabled
|
||||
);
|
||||
const fusionProductImageUrl = preprocessedProduct.productImageUrl;
|
||||
console.log(
|
||||
@ -1531,7 +1662,15 @@ export async function POST(request: Request) {
|
||||
scenes: processedScenes,
|
||||
});
|
||||
const resolvedProductName = enriched.productName;
|
||||
const resolvedScenes = enriched.scenes;
|
||||
let resolvedScenes = enriched.scenes;
|
||||
if (resolvedSceneMode === "random-bg") {
|
||||
resolvedScenes = await enrichRandomBackgroundScenesWithAgent({
|
||||
requestId,
|
||||
productImageUrl: fusionProductImageUrl,
|
||||
productName: resolvedProductName,
|
||||
scenes: resolvedScenes,
|
||||
});
|
||||
}
|
||||
const resolvedSceneMap = new Map(resolvedScenes.map((scene) => [scene.id, scene]));
|
||||
const imageHashCache = new Map<string, string>();
|
||||
|
||||
@ -1540,6 +1679,13 @@ export async function POST(request: Request) {
|
||||
.map((scene) => `${scene.id}:${scene.targetName || "-"}`)
|
||||
.join(", ")}`
|
||||
);
|
||||
if (resolvedSceneMode === "random-bg") {
|
||||
console.log(
|
||||
`[SceneFusion][${requestId}] Random-bg prompts: ${resolvedScenes
|
||||
.map((scene) => `${scene.id}:${(scene.customPrompt || "").slice(0, 80) || "-"}`)
|
||||
.join(" | ")}`
|
||||
);
|
||||
}
|
||||
|
||||
// 6. 创建流式响应，每完成一张图就立即发送给前端
|
||||
const stream = new ReadableStream({
|
||||
|
||||
@ -417,6 +417,7 @@ export function SceneFusionView({
|
||||
body: JSON.stringify({
|
||||
productImage,
|
||||
productName: productName || undefined,
|
||||
sceneMode: mode,
|
||||
aspectRatio,
|
||||
imageSize,
|
||||
scenes: scenesPayload,
|
||||
@ -632,6 +633,7 @@ export function SceneFusionView({
|
||||
body: JSON.stringify({
|
||||
productImage: params.productImageInput,
|
||||
productName: productName || undefined,
|
||||
sceneMode: mode,
|
||||
aspectRatio,
|
||||
imageSize,
|
||||
scenes: [
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user