From cd451c8f9ad510165de90717a463d38b5a7e97a3 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 5 Mar 2026 21:21:40 +0800 Subject: [PATCH] feat(scene-fusion): add agent-based product and target name detection --- app/api/scene-fusion/route.ts | 271 ++++++++++++++++++++++++++++++- components/scene-fusion-view.tsx | 50 ++++++ 2 files changed, 315 insertions(+), 6 deletions(-) diff --git a/app/api/scene-fusion/route.ts b/app/api/scene-fusion/route.ts index f76a79208..11b34fac9 100644 --- a/app/api/scene-fusion/route.ts +++ b/app/api/scene-fusion/route.ts @@ -17,6 +17,7 @@ import { createGeminiImage } from "@/lib/ai/providers/gemini-image"; import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision"; import { createRuxaImage } from "@/lib/ai/providers/ruxa-image"; import { createFalImage } from "@/lib/ai/providers/fal-image"; +import { callQwen3Vl } from "@/lib/ai/qwen3-vl"; // 请求体 Schema const sceneFusionRequestSchema = z.object({ @@ -38,6 +39,8 @@ const FALLBACK_PROMPT = "请参考第一张图片中的产品，将其自然地 const SCENE_FUSION_MODEL_COOLDOWN_MS = 2 * 60 * 1000; const SCENE_FUSION_MAX_ITEMS = 10; const SCENE_FUSION_REQUEST_STAGGER_MS = 350; +const SCENE_FUSION_AGENT_TIMEOUT_MS = 20000; +const SCENE_FUSION_AGENT_MAX_WORKERS = 2; const sceneFusionModelCooldown = new Map(); let sceneFusionRoundRobinCursor = 0; @@ -55,6 +58,20 @@ interface SceneFusionModelCandidate { }; } +interface SceneFusionAgentMeta { + productName?: string; + productSource?: "user" | "agent"; + productConfidence?: number; + productModel?: string; + sceneTargets: Array<{ + id: string; + targetName?: string; + source?: "user" | "agent"; + confidence?: number; + model?: string; + }>; +} + function normalizeSceneFusionProviderType(providerType: string): string { return providerType === "gemini-image" ? "gemini" : providerType; } @@ -133,6 +150,220 @@ function waitSceneFusion(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } +function normalizeSceneFusionName(value: unknown): string { + const raw = String(value ?? "").trim(); + if (!raw) return ""; + const normalized = raw + .replace(/[：:，,。.!！?？、；;（）()[\]{}"'`]/g, " ") + .replace(/\s+/g, " ") + .trim(); + return normalized.slice(0, 16); +} + +function extractJsonObject(text: string): Record | null { + const content = String(text || "").trim(); + if (!content) return null; + const match = content.match(/\{[\s\S]*\}/); + if (!match) return null; + try { + const parsed = JSON.parse(match[0]); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + return parsed as Record; + } + return null; + } catch { + return null; + } +} + +async function detectProductNameWithAgent(imageUrl: string): Promise<{ + name: string; + confidence: number; + model: string; +} | null> { + const prompt = [ + "你是电商图片分析助手。", + "请识别图片中最核心、最主要的产品主体名称。", + "要求：", + "- 名称精简，2-8个字，尽量包含材质/品类（如：皮质沙发、木质餐椅、陶瓷花瓶）", + "- 不要返回颜色描述句子，不要返回背景和场景词", + "仅返回JSON：", + '{\"productName\":\"名称\",\"confidence\":0.0}', + ].join("\n"); + + const { text, model } = await callQwen3Vl({ + imageUrl, + prompt, + temperature: 0.1, + maxTokens: 220, + timeoutMs: SCENE_FUSION_AGENT_TIMEOUT_MS, + retries: 1, + }); + + const parsed = extractJsonObject(text); + const name = normalizeSceneFusionName(parsed?.productName); + const confidenceRaw = Number(parsed?.confidence); + const confidence = Number.isFinite(confidenceRaw) ? Math.max(0, Math.min(1, confidenceRaw)) : 0.7; + + if (!name || confidence < 0.45) { + return null; + } + + return { name, confidence, model }; +} + +async function detectTargetNameWithAgent( + sceneImageUrl: string, + productNameHint: string +): Promise<{ + name: string; + confidence: number; + model: string; +} | null> { + const prompt = [ + "你是场景替换识别助手。", + `已知要替换进场景的产品是：${productNameHint || "该产品"}`, + "请识别当前场景图里最应该被替换的目标物体名称（与该产品品类最接近的场景主体）。", + "要求：", + "- 返回“完整物体名”，不要返回局部部件名", + "- 名称精简，2-8个字（如：餐椅、沙发、茶几、花瓶）", + "- 如果无法确定，返回最主要主体物体名", + "仅返回JSON：", + '{\"targetName\":\"名称\",\"confidence\":0.0}', + ].join("\n"); + + const { text, model } = await callQwen3Vl({ + imageUrl: sceneImageUrl, + prompt, + temperature: 0.1, + maxTokens: 220, + timeoutMs: SCENE_FUSION_AGENT_TIMEOUT_MS, + retries: 1, + }); + + const parsed = extractJsonObject(text); + const name = normalizeSceneFusionName(parsed?.targetName); + const confidenceRaw = Number(parsed?.confidence); + const confidence = Number.isFinite(confidenceRaw) ? Math.max(0, Math.min(1, confidenceRaw)) : 0.7; + + if (!name || confidence < 0.4) { + return null; + } + + return { name, confidence, model }; +} + +async function enrichSceneLabelsWithAgent(params: { + requestId: string; + productImageUrl: string; + productName?: string; + scenes: Array<{ + id: string; + image: string; + targetName?: string; + customPrompt?: string; + imageUrl: string; + }>; +}): Promise<{ + productName?: string; + scenes: Array<{ + id: string; + image: string; + targetName?: string; + customPrompt?: string; + imageUrl: string; + }>; + meta: SceneFusionAgentMeta; +}> { + const clonedScenes = params.scenes.map((scene) => ({ ...scene })); + const userProductName = normalizeSceneFusionName(params.productName); + let resolvedProductName = userProductName; + + const meta: SceneFusionAgentMeta = { + productName: userProductName || undefined, + productSource: userProductName ? "user" : undefined, + sceneTargets: clonedScenes.map((scene) => ({ + id: scene.id, + targetName: normalizeSceneFusionName(scene.targetName) || undefined, + source: normalizeSceneFusionName(scene.targetName) ? "user" : undefined, + })), + }; + + if (!resolvedProductName) { + try { + const productDetected = await detectProductNameWithAgent(params.productImageUrl); + if (productDetected?.name) { + resolvedProductName = productDetected.name; + meta.productName = productDetected.name; + meta.productSource = "agent"; + meta.productConfidence = productDetected.confidence; + meta.productModel = productDetected.model; + } + } catch (error) { + console.warn(`[SceneFusion][${params.requestId}] Product detect skipped:`, error); + } + } + + const unresolvedIndexes = clonedScenes + .map((scene, index) => ({ + index, + hasTarget: Boolean(normalizeSceneFusionName(scene.targetName)), + sameAsProduct: scene.imageUrl === params.productImageUrl, + })) + .filter((item) => !item.hasTarget && !item.sameAsProduct) + .map((item) => item.index); + + if (unresolvedIndexes.length === 0) { + return { + productName: resolvedProductName || undefined, + scenes: clonedScenes, + meta, + }; + } + + let cursor = 0; + const workerCount = Math.max(1, Math.min(SCENE_FUSION_AGENT_MAX_WORKERS, unresolvedIndexes.length)); + + const workers = Array.from({ length: workerCount }, async () => { + while (true) { + const current = cursor; + cursor += 1; + if (current >= unresolvedIndexes.length) break; + + const sceneIndex = unresolvedIndexes[current]!; + const scene = clonedScenes[sceneIndex]!; + try { + const targetDetected = await detectTargetNameWithAgent( + scene.imageUrl, + resolvedProductName || "产品" + ); + if (!targetDetected?.name) continue; + scene.targetName = targetDetected.name; + const metaScene = meta.sceneTargets.find((item) => item.id === scene.id); + if (metaScene) { + metaScene.targetName = targetDetected.name; + metaScene.source = "agent"; + metaScene.confidence = targetDetected.confidence; + metaScene.model = targetDetected.model; + } + } catch (error) { + console.warn( + `[SceneFusion][${params.requestId}] Scene target detect skipped: scene=${scene.id}`, + error + ); + } + } + }); + + await Promise.all(workers); + + return { + productName: resolvedProductName || undefined, + scenes: clonedScenes, + meta, + }; +} + function buildSceneFusionCandidatePool( allModels: Awaited>, preferredModelId: string @@ -1021,6 +1252,22 @@ export async function POST(request: Request) { }) ); + const enriched = await enrichSceneLabelsWithAgent({ + requestId, + productImageUrl, + productName, + scenes: processedScenes, + }); + const resolvedProductName = enriched.productName; + const resolvedScenes = enriched.scenes; + const resolvedSceneMap = new Map(resolvedScenes.map((scene) => [scene.id, scene])); + + console.log( + `[SceneFusion][${requestId}] Agent labels: product=${resolvedProductName || "(none)"}; targets=${resolvedScenes + .map((scene) => `${scene.id}:${scene.targetName || "-"}`) + .join(", ")}` + ); + // 6. 创建流式响应，每完成一张图就立即发送给前端 const stream = new ReadableStream({ async start(controller) { @@ -1052,6 +1299,14 @@ export async function POST(request: Request) { if (!enqueue({ type: "start", total: processedScenes.length })) { return; } + enqueue({ + type: "agent_meta", + data: { + productName: enriched.meta.productName || resolvedProductName || "", + productSource: enriched.meta.productSource || (resolvedProductName ? "agent" : "none"), + sceneTargets: enriched.meta.sceneTargets, + }, + }); let successCount = 0; let failCount = 0; @@ -1075,6 +1330,8 @@ export async function POST(request: Request) { } const scene = processedScenes[currentIndex]; + if (!scene) break; + const resolvedScene = resolvedSceneMap.get(scene.id) || scene; console.log( `[SceneFusion] ========== Processing scene ${currentIndex + 1}/${processedScenes.length} ==========` @@ -1087,19 +1344,19 @@ export async function POST(request: Request) { `[SceneFusion] Scene Image URL: ${scene.imageUrl.substring(0, 80)}...` ); console.log( - `[SceneFusion] Target Name: ${scene.targetName || "(not specified)"}` + `[SceneFusion] Target Name: ${resolvedScene.targetName || "(not specified)"}` ); console.log( - `[SceneFusion] Custom Prompt: ${scene.customPrompt || "(none)"}` + `[SceneFusion] Custom Prompt: ${resolvedScene.customPrompt || "(none)"}` ); try { const { result, candidate, attempts } = await processSceneWithDiversity({ productImageUrl, - sceneImageUrl: scene.imageUrl, - productName, - targetName: scene.targetName, - customPrompt: scene.customPrompt, + sceneImageUrl: resolvedScene.imageUrl, + productName: resolvedProductName, + targetName: resolvedScene.targetName, + customPrompt: resolvedScene.customPrompt, defaultPrompt: config.defaultPrompt, aspectRatio, imageSize, @@ -1141,6 +1398,8 @@ export async function POST(request: Request) { modelId: candidate.model.modelId, providerType: candidate.provider.type, attempts, + recognizedProductName: resolvedProductName, + recognizedTargetName: resolvedScene.targetName, }, })) { break; diff --git a/components/scene-fusion-view.tsx b/components/scene-fusion-view.tsx index ec86d202e..3b6c8dacc 100644 --- a/components/scene-fusion-view.tsx +++ b/components/scene-fusion-view.tsx @@ -428,9 +428,41 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam result?: string; originalUrl?: string; error?: string; + productName?: string; + productSource?: "user" | "agent" | "none"; + sceneTargets?: Array<{ + id: string; + targetName?: string; + source?: "user" | "agent"; + confidence?: number; + model?: string; + }>; }; error?: string; }) => { + if (msg.type === "agent_meta" && msg.data) { + const detectedProductName = + typeof msg.data.productName === "string" ? msg.data.productName.trim() : ""; + if (!productName.trim() && detectedProductName) { + setProductName(detectedProductName); + } + if (Array.isArray(msg.data.sceneTargets) && msg.data.sceneTargets.length > 0) { + setter((prev) => + prev.map((item) => { + const detected = msg.data?.sceneTargets?.find((entry) => entry.id === item.id); + if (!detected?.targetName) return item; + const currentTargetName = (item.targetName || "").trim(); + if (currentTargetName) return item; + return { + ...item, + targetName: detected.targetName, + }; + }) + ); + } + return; + } + if (msg.type === "result" && msg.data) { const data = msg.data; if (data.status === "completed") { @@ -490,6 +522,15 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam result?: string; originalUrl?: string; error?: string; + productName?: string; + productSource?: "user" | "agent" | "none"; + sceneTargets?: Array<{ + id: string; + targetName?: string; + source?: "user" | "agent"; + confidence?: number; + model?: string; + }>; }; error?: string; }; @@ -510,6 +551,15 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam result?: string; originalUrl?: string; error?: string; + productName?: string; + productSource?: "user" | "agent" | "none"; + sceneTargets?: Array<{ + id: string; + targetName?: string; + source?: "user" | "agent"; + confidence?: number; + model?: string; + }>; }; error?: string; };