feat(scene-fusion): add agent-based product and target name detection

app/api/scene-fusion/route.ts

@@ -17,6 +17,7 @@ import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
 import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision";
 import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
 import { createFalImage } from "@/lib/ai/providers/fal-image";
+import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
 
 // 请求体 Schema
 const sceneFusionRequestSchema = z.object({
@@ -38,6 +39,8 @@ const FALLBACK_PROMPT = "请参考第一张图片中的产品，将其自然地
 const SCENE_FUSION_MODEL_COOLDOWN_MS = 2 * 60 * 1000;
 const SCENE_FUSION_MAX_ITEMS = 10;
 const SCENE_FUSION_REQUEST_STAGGER_MS = 350;
+const SCENE_FUSION_AGENT_TIMEOUT_MS = 20000;
+const SCENE_FUSION_AGENT_MAX_WORKERS = 2;
 const sceneFusionModelCooldown = new Map<string, number>();
 let sceneFusionRoundRobinCursor = 0;
 
@@ -55,6 +58,20 @@ interface SceneFusionModelCandidate {
   };
 }
 
+interface SceneFusionAgentMeta {
+  productName?: string;
+  productSource?: "user" | "agent";
+  productConfidence?: number;
+  productModel?: string;
+  sceneTargets: Array<{
+    id: string;
+    targetName?: string;
+    source?: "user" | "agent";
+    confidence?: number;
+    model?: string;
+  }>;
+}
+
 function normalizeSceneFusionProviderType(providerType: string): string {
   return providerType === "gemini-image" ? "gemini" : providerType;
 }
@@ -133,6 +150,220 @@ function waitSceneFusion(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+function normalizeSceneFusionName(value: unknown): string {
+  const raw = String(value ?? "").trim();
+  if (!raw) return "";
+  const normalized = raw
+    .replace(/[：:，,。.!！?？、；;（）()[\]{}"'`]/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+  return normalized.slice(0, 16);
+}
+
+function extractJsonObject(text: string): Record<string, unknown> | null {
+  const content = String(text || "").trim();
+  if (!content) return null;
+  const match = content.match(/\{[\s\S]*\}/);
+  if (!match) return null;
+  try {
+    const parsed = JSON.parse(match[0]);
+    if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+      return parsed as Record<string, unknown>;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+async function detectProductNameWithAgent(imageUrl: string): Promise<{
+  name: string;
+  confidence: number;
+  model: string;
+} | null> {
+  const prompt = [
+    "你是电商图片分析助手。",
+    "请识别图片中最核心、最主要的产品主体名称。",
+    "要求：",
+    "- 名称精简，2-8个字，尽量包含材质/品类（如：皮质沙发、木质餐椅、陶瓷花瓶）",
+    "- 不要返回颜色描述句子，不要返回背景和场景词",
+    "仅返回JSON：",
+    '{\"productName\":\"名称\",\"confidence\":0.0}',
+  ].join("\n");
+
+  const { text, model } = await callQwen3Vl({
+    imageUrl,
+    prompt,
+    temperature: 0.1,
+    maxTokens: 220,
+    timeoutMs: SCENE_FUSION_AGENT_TIMEOUT_MS,
+    retries: 1,
+  });
+
+  const parsed = extractJsonObject(text);
+  const name = normalizeSceneFusionName(parsed?.productName);
+  const confidenceRaw = Number(parsed?.confidence);
+  const confidence = Number.isFinite(confidenceRaw) ? Math.max(0, Math.min(1, confidenceRaw)) : 0.7;
+
+  if (!name || confidence < 0.45) {
+    return null;
+  }
+
+  return { name, confidence, model };
+}
+
+async function detectTargetNameWithAgent(
+  sceneImageUrl: string,
+  productNameHint: string
+): Promise<{
+  name: string;
+  confidence: number;
+  model: string;
+} | null> {
+  const prompt = [
+    "你是场景替换识别助手。",
+    `已知要替换进场景的产品是：${productNameHint || "该产品"}`,
+    "请识别当前场景图里最应该被替换的目标物体名称（与该产品品类最接近的场景主体）。",
+    "要求：",
+    "- 返回“完整物体名”，不要返回局部部件名",
+    "- 名称精简，2-8个字（如：餐椅、沙发、茶几、花瓶）",
+    "- 如果无法确定，返回最主要主体物体名",
+    "仅返回JSON：",
+    '{\"targetName\":\"名称\",\"confidence\":0.0}',
+  ].join("\n");
+
+  const { text, model } = await callQwen3Vl({
+    imageUrl: sceneImageUrl,
+    prompt,
+    temperature: 0.1,
+    maxTokens: 220,
+    timeoutMs: SCENE_FUSION_AGENT_TIMEOUT_MS,
+    retries: 1,
+  });
+
+  const parsed = extractJsonObject(text);
+  const name = normalizeSceneFusionName(parsed?.targetName);
+  const confidenceRaw = Number(parsed?.confidence);
+  const confidence = Number.isFinite(confidenceRaw) ? Math.max(0, Math.min(1, confidenceRaw)) : 0.7;
+
+  if (!name || confidence < 0.4) {
+    return null;
+  }
+
+  return { name, confidence, model };
+}
+
+async function enrichSceneLabelsWithAgent(params: {
+  requestId: string;
+  productImageUrl: string;
+  productName?: string;
+  scenes: Array<{
+    id: string;
+    image: string;
+    targetName?: string;
+    customPrompt?: string;
+    imageUrl: string;
+  }>;
+}): Promise<{
+  productName?: string;
+  scenes: Array<{
+    id: string;
+    image: string;
+    targetName?: string;
+    customPrompt?: string;
+    imageUrl: string;
+  }>;
+  meta: SceneFusionAgentMeta;
+}> {
+  const clonedScenes = params.scenes.map((scene) => ({ ...scene }));
+  const userProductName = normalizeSceneFusionName(params.productName);
+  let resolvedProductName = userProductName;
+
+  const meta: SceneFusionAgentMeta = {
+    productName: userProductName || undefined,
+    productSource: userProductName ? "user" : undefined,
+    sceneTargets: clonedScenes.map((scene) => ({
+      id: scene.id,
+      targetName: normalizeSceneFusionName(scene.targetName) || undefined,
+      source: normalizeSceneFusionName(scene.targetName) ? "user" : undefined,
+    })),
+  };
+
+  if (!resolvedProductName) {
+    try {
+      const productDetected = await detectProductNameWithAgent(params.productImageUrl);
+      if (productDetected?.name) {
+        resolvedProductName = productDetected.name;
+        meta.productName = productDetected.name;
+        meta.productSource = "agent";
+        meta.productConfidence = productDetected.confidence;
+        meta.productModel = productDetected.model;
+      }
+    } catch (error) {
+      console.warn(`[SceneFusion][${params.requestId}] Product detect skipped:`, error);
+    }
+  }
+
+  const unresolvedIndexes = clonedScenes
+    .map((scene, index) => ({
+      index,
+      hasTarget: Boolean(normalizeSceneFusionName(scene.targetName)),
+      sameAsProduct: scene.imageUrl === params.productImageUrl,
+    }))
+    .filter((item) => !item.hasTarget && !item.sameAsProduct)
+    .map((item) => item.index);
+
+  if (unresolvedIndexes.length === 0) {
+    return {
+      productName: resolvedProductName || undefined,
+      scenes: clonedScenes,
+      meta,
+    };
+  }
+
+  let cursor = 0;
+  const workerCount = Math.max(1, Math.min(SCENE_FUSION_AGENT_MAX_WORKERS, unresolvedIndexes.length));
+
+  const workers = Array.from({ length: workerCount }, async () => {
+    while (true) {
+      const current = cursor;
+      cursor += 1;
+      if (current >= unresolvedIndexes.length) break;
+
+      const sceneIndex = unresolvedIndexes[current]!;
+      const scene = clonedScenes[sceneIndex]!;
+      try {
+        const targetDetected = await detectTargetNameWithAgent(
+          scene.imageUrl,
+          resolvedProductName || "产品"
+        );
+        if (!targetDetected?.name) continue;
+        scene.targetName = targetDetected.name;
+        const metaScene = meta.sceneTargets.find((item) => item.id === scene.id);
+        if (metaScene) {
+          metaScene.targetName = targetDetected.name;
+          metaScene.source = "agent";
+          metaScene.confidence = targetDetected.confidence;
+          metaScene.model = targetDetected.model;
+        }
+      } catch (error) {
+        console.warn(
+          `[SceneFusion][${params.requestId}] Scene target detect skipped: scene=${scene.id}`,
+          error
+        );
+      }
+    }
+  });
+
+  await Promise.all(workers);
+
+  return {
+    productName: resolvedProductName || undefined,
+    scenes: clonedScenes,
+    meta,
+  };
+}
+
 function buildSceneFusionCandidatePool(
   allModels: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>,
   preferredModelId: string
@@ -1021,6 +1252,22 @@ export async function POST(request: Request) {
       })
     );
 
+    const enriched = await enrichSceneLabelsWithAgent({
+      requestId,
+      productImageUrl,
+      productName,
+      scenes: processedScenes,
+    });
+    const resolvedProductName = enriched.productName;
+    const resolvedScenes = enriched.scenes;
+    const resolvedSceneMap = new Map(resolvedScenes.map((scene) => [scene.id, scene]));
+
+    console.log(
+      `[SceneFusion][${requestId}] Agent labels: product=${resolvedProductName || "(none)"}; targets=${resolvedScenes
+        .map((scene) => `${scene.id}:${scene.targetName || "-"}`)
+        .join(", ")}`
+    );
+
     // 6. 创建流式响应，每完成一张图就立即发送给前端
     const stream = new ReadableStream({
       async start(controller) {
@@ -1052,6 +1299,14 @@ export async function POST(request: Request) {
           if (!enqueue({ type: "start", total: processedScenes.length })) {
             return;
           }
+          enqueue({
+            type: "agent_meta",
+            data: {
+              productName: enriched.meta.productName || resolvedProductName || "",
+              productSource: enriched.meta.productSource || (resolvedProductName ? "agent" : "none"),
+              sceneTargets: enriched.meta.sceneTargets,
+            },
+          });
           
           let successCount = 0;
           let failCount = 0;
@@ -1075,6 +1330,8 @@ export async function POST(request: Request) {
               }
 
               const scene = processedScenes[currentIndex];
+              if (!scene) break;
+              const resolvedScene = resolvedSceneMap.get(scene.id) || scene;
 
               console.log(
                 `[SceneFusion] ========== Processing scene ${currentIndex + 1}/${processedScenes.length} ==========`
@@ -1087,19 +1344,19 @@ export async function POST(request: Request) {
                 `[SceneFusion] Scene Image URL: ${scene.imageUrl.substring(0, 80)}...`
               );
               console.log(
-                `[SceneFusion] Target Name: ${scene.targetName || "(not specified)"}`
+                `[SceneFusion] Target Name: ${resolvedScene.targetName || "(not specified)"}`
               );
               console.log(
-                `[SceneFusion] Custom Prompt: ${scene.customPrompt || "(none)"}`
+                `[SceneFusion] Custom Prompt: ${resolvedScene.customPrompt || "(none)"}`
               );
 
               try {
                 const { result, candidate, attempts } = await processSceneWithDiversity({
                   productImageUrl,
-                  sceneImageUrl: scene.imageUrl,
+                  sceneImageUrl: resolvedScene.imageUrl,
-                  productName,
+                  productName: resolvedProductName,
-                  targetName: scene.targetName,
+                  targetName: resolvedScene.targetName,
-                  customPrompt: scene.customPrompt,
+                  customPrompt: resolvedScene.customPrompt,
                   defaultPrompt: config.defaultPrompt,
                   aspectRatio,
                   imageSize,
@@ -1141,6 +1398,8 @@ export async function POST(request: Request) {
                     modelId: candidate.model.modelId,
                     providerType: candidate.provider.type,
                     attempts,
+                    recognizedProductName: resolvedProductName,
+                    recognizedTargetName: resolvedScene.targetName,
                   },
                 })) {
                   break;

components/scene-fusion-view.tsx

@@ -428,9 +428,41 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
           result?: string;
           originalUrl?: string;
           error?: string;
+          productName?: string;
+          productSource?: "user" | "agent" | "none";
+          sceneTargets?: Array<{
+            id: string;
+            targetName?: string;
+            source?: "user" | "agent";
+            confidence?: number;
+            model?: string;
+          }>;
         };
         error?: string;
       }) => {
+        if (msg.type === "agent_meta" && msg.data) {
+          const detectedProductName =
+            typeof msg.data.productName === "string" ? msg.data.productName.trim() : "";
+          if (!productName.trim() && detectedProductName) {
+            setProductName(detectedProductName);
+          }
+          if (Array.isArray(msg.data.sceneTargets) && msg.data.sceneTargets.length > 0) {
+            setter((prev) =>
+              prev.map((item) => {
+                const detected = msg.data?.sceneTargets?.find((entry) => entry.id === item.id);
+                if (!detected?.targetName) return item;
+                const currentTargetName = (item.targetName || "").trim();
+                if (currentTargetName) return item;
+                return {
+                  ...item,
+                  targetName: detected.targetName,
+                };
+              })
+            );
+          }
+          return;
+        }
+
         if (msg.type === "result" && msg.data) {
           const data = msg.data;
           if (data.status === "completed") {
@@ -490,6 +522,15 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
                 result?: string;
                 originalUrl?: string;
                 error?: string;
+                productName?: string;
+                productSource?: "user" | "agent" | "none";
+                sceneTargets?: Array<{
+                  id: string;
+                  targetName?: string;
+                  source?: "user" | "agent";
+                  confidence?: number;
+                  model?: string;
+                }>;
               };
               error?: string;
             };
@@ -510,6 +551,15 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
               result?: string;
               originalUrl?: string;
               error?: string;
+              productName?: string;
+              productSource?: "user" | "agent" | "none";
+              sceneTargets?: Array<{
+                id: string;
+                targetName?: string;
+                source?: "user" | "agent";
+                confidence?: number;
+                model?: string;
+              }>;
             };
             error?: string;
           };