lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(scene-fusion): add agent-based product and target name detection
Some checks failed
Lint / build (20) (push) Has been cancelled
Details
Playwright Tests / test (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Your Name 2026-03-05 21:21:40 +08:00
parent 4104b68f23
commit cd451c8f9a
2 changed files with 315 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

271
app/api/scene-fusion/route.ts
View File

@ -17,6 +17,7 @@ import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision";
import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
import { createFalImage } from "@/lib/ai/providers/fal-image";
import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
// 请求体 Schema
const sceneFusionRequestSchema = z.object({
@ -38,6 +39,8 @@ const FALLBACK_PROMPT = "请参考第一张图片中的产品，将其自然地
const SCENE_FUSION_MODEL_COOLDOWN_MS = 2 * 60 * 1000;
const SCENE_FUSION_MAX_ITEMS = 10;
const SCENE_FUSION_REQUEST_STAGGER_MS = 350;
const SCENE_FUSION_AGENT_TIMEOUT_MS = 20000;
const SCENE_FUSION_AGENT_MAX_WORKERS = 2;
const sceneFusionModelCooldown = new Map<string, number>();
let sceneFusionRoundRobinCursor = 0;
@ -55,6 +58,20 @@ interface SceneFusionModelCandidate {
};
}
interface SceneFusionAgentMeta {
productName?: string;
productSource?: "user" | "agent";
productConfidence?: number;
productModel?: string;
sceneTargets: Array<{
id: string;
targetName?: string;
source?: "user" | "agent";
confidence?: number;
model?: string;
}>;
}
function normalizeSceneFusionProviderType(providerType: string): string {
return providerType === "gemini-image" ? "gemini" : providerType;
}
@ -133,6 +150,220 @@ function waitSceneFusion(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function normalizeSceneFusionName(value: unknown): string {
const raw = String(value ?? "").trim();
if (!raw) return "";
const normalized = raw
.replace(/[：:，,。.!！?？、；;（）()[\]{}"'`]/g, " ")
.replace(/\s+/g, " ")
.trim();
return normalized.slice(0, 16);
}
function extractJsonObject(text: string): Record<string, unknown> | null {
const content = String(text || "").trim();
if (!content) return null;
const match = content.match(/\{[\s\S]*\}/);
if (!match) return null;
try {
const parsed = JSON.parse(match[0]);
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
return parsed as Record<string, unknown>;
}
return null;
} catch {
return null;
}
}
async function detectProductNameWithAgent(imageUrl: string): Promise<{
name: string;
confidence: number;
model: string;
} | null> {
const prompt = [
"你是电商图片分析助手。",
"请识别图片中最核心、最主要的产品主体名称。",
"要求：",
"- 名称精简，2-8个字，尽量包含材质/品类（如：皮质沙发、木质餐椅、陶瓷花瓶）",
"- 不要返回颜色描述句子，不要返回背景和场景词",
"仅返回JSON：",
'{\"productName\":\"名称\",\"confidence\":0.0}',
].join("\n");
const { text, model } = await callQwen3Vl({
imageUrl,
prompt,
temperature: 0.1,
maxTokens: 220,
timeoutMs: SCENE_FUSION_AGENT_TIMEOUT_MS,
retries: 1,
});
const parsed = extractJsonObject(text);
const name = normalizeSceneFusionName(parsed?.productName);
const confidenceRaw = Number(parsed?.confidence);
const confidence = Number.isFinite(confidenceRaw) ? Math.max(0, Math.min(1, confidenceRaw)) : 0.7;
if (!name || confidence < 0.45) {
return null;
}
return { name, confidence, model };
}
async function detectTargetNameWithAgent(
sceneImageUrl: string,
productNameHint: string
): Promise<{
name: string;
confidence: number;
model: string;
} | null> {
const prompt = [
"你是场景替换识别助手。",
`已知要替换进场景的产品是：${productNameHint || "该产品"}`,
"请识别当前场景图里最应该被替换的目标物体名称（与该产品品类最接近的场景主体）。",
"要求：",
"- 返回“完整物体名”，不要返回局部部件名",
"- 名称精简，2-8个字（如：餐椅、沙发、茶几、花瓶）",
"- 如果无法确定，返回最主要主体物体名",
"仅返回JSON：",
'{\"targetName\":\"名称\",\"confidence\":0.0}',
].join("\n");
const { text, model } = await callQwen3Vl({
imageUrl: sceneImageUrl,
prompt,
temperature: 0.1,
maxTokens: 220,
timeoutMs: SCENE_FUSION_AGENT_TIMEOUT_MS,
retries: 1,
});
const parsed = extractJsonObject(text);
const name = normalizeSceneFusionName(parsed?.targetName);
const confidenceRaw = Number(parsed?.confidence);
const confidence = Number.isFinite(confidenceRaw) ? Math.max(0, Math.min(1, confidenceRaw)) : 0.7;
if (!name || confidence < 0.4) {
return null;
}
return { name, confidence, model };
}
async function enrichSceneLabelsWithAgent(params: {
requestId: string;
productImageUrl: string;
productName?: string;
scenes: Array<{
id: string;
image: string;
targetName?: string;
customPrompt?: string;
imageUrl: string;
}>;
}): Promise<{
productName?: string;
scenes: Array<{
id: string;
image: string;
targetName?: string;
customPrompt?: string;
imageUrl: string;
}>;
meta: SceneFusionAgentMeta;
}> {
const clonedScenes = params.scenes.map((scene) => ({ ...scene }));
const userProductName = normalizeSceneFusionName(params.productName);
let resolvedProductName = userProductName;
const meta: SceneFusionAgentMeta = {
productName: userProductName || undefined,
productSource: userProductName ? "user" : undefined,
sceneTargets: clonedScenes.map((scene) => ({
id: scene.id,
targetName: normalizeSceneFusionName(scene.targetName) || undefined,
source: normalizeSceneFusionName(scene.targetName) ? "user" : undefined,
})),
};
if (!resolvedProductName) {
try {
const productDetected = await detectProductNameWithAgent(params.productImageUrl);
if (productDetected?.name) {
resolvedProductName = productDetected.name;
meta.productName = productDetected.name;
meta.productSource = "agent";
meta.productConfidence = productDetected.confidence;
meta.productModel = productDetected.model;
}
} catch (error) {
console.warn(`[SceneFusion][${params.requestId}] Product detect skipped:`, error);
}
}
const unresolvedIndexes = clonedScenes
.map((scene, index) => ({
index,
hasTarget: Boolean(normalizeSceneFusionName(scene.targetName)),
sameAsProduct: scene.imageUrl === params.productImageUrl,
}))
.filter((item) => !item.hasTarget && !item.sameAsProduct)
.map((item) => item.index);
if (unresolvedIndexes.length === 0) {
return {
productName: resolvedProductName || undefined,
scenes: clonedScenes,
meta,
};
}
let cursor = 0;
const workerCount = Math.max(1, Math.min(SCENE_FUSION_AGENT_MAX_WORKERS, unresolvedIndexes.length));
const workers = Array.from({ length: workerCount }, async () => {
while (true) {
const current = cursor;
cursor += 1;
if (current >= unresolvedIndexes.length) break;
const sceneIndex = unresolvedIndexes[current]!;
const scene = clonedScenes[sceneIndex]!;
try {
const targetDetected = await detectTargetNameWithAgent(
scene.imageUrl,
resolvedProductName || "产品"
);
if (!targetDetected?.name) continue;
scene.targetName = targetDetected.name;
const metaScene = meta.sceneTargets.find((item) => item.id === scene.id);
if (metaScene) {
metaScene.targetName = targetDetected.name;
metaScene.source = "agent";
metaScene.confidence = targetDetected.confidence;
metaScene.model = targetDetected.model;
}
} catch (error) {
console.warn(
`[SceneFusion][${params.requestId}] Scene target detect skipped: scene=${scene.id}`,
error
);
}
}
});
await Promise.all(workers);
return {
productName: resolvedProductName || undefined,
scenes: clonedScenes,
meta,
};
}
function buildSceneFusionCandidatePool(
allModels: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>,
preferredModelId: string
@ -1021,6 +1252,22 @@ export async function POST(request: Request) {
})
);
const enriched = await enrichSceneLabelsWithAgent({
requestId,
productImageUrl,
productName,
scenes: processedScenes,
});
const resolvedProductName = enriched.productName;
const resolvedScenes = enriched.scenes;
const resolvedSceneMap = new Map(resolvedScenes.map((scene) => [scene.id, scene]));
console.log(
`[SceneFusion][${requestId}] Agent labels: product=${resolvedProductName || "(none)"}; targets=${resolvedScenes
.map((scene) => `${scene.id}:${scene.targetName || "-"}`)
.join(", ")}`
);
// 6. 创建流式响应，每完成一张图就立即发送给前端
const stream = new ReadableStream({
async start(controller) {
@ -1052,6 +1299,14 @@ export async function POST(request: Request) {
if (!enqueue({ type: "start", total: processedScenes.length })) {
return;
}
enqueue({
type: "agent_meta",
data: {
productName: enriched.meta.productName || resolvedProductName || "",
productSource: enriched.meta.productSource || (resolvedProductName ? "agent" : "none"),
sceneTargets: enriched.meta.sceneTargets,
},
});
let successCount = 0;
let failCount = 0;
@ -1075,6 +1330,8 @@ export async function POST(request: Request) {
}
const scene = processedScenes[currentIndex];
if (!scene) break;
const resolvedScene = resolvedSceneMap.get(scene.id) || scene;
console.log(
`[SceneFusion] ========== Processing scene ${currentIndex + 1}/${processedScenes.length} ==========`
@ -1087,19 +1344,19 @@ export async function POST(request: Request) {
`[SceneFusion] Scene Image URL: ${scene.imageUrl.substring(0, 80)}...`
);
console.log(
`[SceneFusion] Target Name: ${scene.targetName || "(not specified)"}`
`[SceneFusion] Target Name: ${resolvedScene.targetName || "(not specified)"}`
);
console.log(
`[SceneFusion] Custom Prompt: ${scene.customPrompt || "(none)"}`
`[SceneFusion] Custom Prompt: ${resolvedScene.customPrompt || "(none)"}`
);
try {
const { result, candidate, attempts } = await processSceneWithDiversity({
productImageUrl,
sceneImageUrl: scene.imageUrl,
productName,
targetName: scene.targetName,
customPrompt: scene.customPrompt,
sceneImageUrl: resolvedScene.imageUrl,
productName: resolvedProductName,
targetName: resolvedScene.targetName,
customPrompt: resolvedScene.customPrompt,
defaultPrompt: config.defaultPrompt,
aspectRatio,
imageSize,
@ -1141,6 +1398,8 @@ export async function POST(request: Request) {
modelId: candidate.model.modelId,
providerType: candidate.provider.type,
attempts,
recognizedProductName: resolvedProductName,
recognizedTargetName: resolvedScene.targetName,
},
})) {
break;

50
components/scene-fusion-view.tsx
View File

@ -428,9 +428,41 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
result?: string;
originalUrl?: string;
error?: string;
productName?: string;
productSource?: "user" | "agent" | "none";
sceneTargets?: Array<{
id: string;
targetName?: string;
source?: "user" | "agent";
confidence?: number;
model?: string;
}>;
};
error?: string;
}) => {
if (msg.type === "agent_meta" && msg.data) {
const detectedProductName =
typeof msg.data.productName === "string" ? msg.data.productName.trim() : "";
if (!productName.trim() && detectedProductName) {
setProductName(detectedProductName);
}
if (Array.isArray(msg.data.sceneTargets) && msg.data.sceneTargets.length > 0) {
setter((prev) =>
prev.map((item) => {
const detected = msg.data?.sceneTargets?.find((entry) => entry.id === item.id);
if (!detected?.targetName) return item;
const currentTargetName = (item.targetName || "").trim();
if (currentTargetName) return item;
return {
...item,
targetName: detected.targetName,
};
})
);
}
return;
}
if (msg.type === "result" && msg.data) {
const data = msg.data;
if (data.status === "completed") {
@ -490,6 +522,15 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
result?: string;
originalUrl?: string;
error?: string;
productName?: string;
productSource?: "user" | "agent" | "none";
sceneTargets?: Array<{
id: string;
targetName?: string;
source?: "user" | "agent";
confidence?: number;
model?: string;
}>;
};
error?: string;
};
@ -510,6 +551,15 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
result?: string;
originalUrl?: string;
error?: string;
productName?: string;
productSource?: "user" | "agent" | "none";
sceneTargets?: Array<{
id: string;
targetName?: string;
source?: "user" | "agent";
confidence?: number;
model?: string;
}>;
};
error?: string;
};