From cd09eb90044b37b17ebeb44d221cb3e91966f6bf Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 4 Mar 2026 20:42:38 +0800 Subject: [PATCH] feat(scene-fusion): enforce diversity mode from configured tagged models --- .../settings/admin-settings-client.tsx | 19 +- app/api/scene-fusion/config/route.ts | 2 + app/api/scene-fusion/route.ts | 457 +++++++++++++----- 3 files changed, 355 insertions(+), 123 deletions(-) diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx index a96158183..bb9734651 100644 --- a/app/(admin)/settings/admin-settings-client.tsx +++ b/app/(admin)/settings/admin-settings-client.tsx @@ -775,7 +775,7 @@ export function AdminSettingsClient() { enabled: true, defaultPrompt: "将图1的产品替换到图2的场景中，保持图1产品的外观、颜色和细节不变，自然融合到图2的环境中。", maxScenes: 10, - modelId: "", // 使用的生图模型ID + modelId: "", // 首选生图模型ID（多样性模式下优先尝试） }); // 加载抠图API配置 @@ -8281,19 +8281,26 @@ export function AdminSettingsClient() { +
+
生成模式：多样性模式（固定）
+
+ 系统会在所有可用生图模型中轮换分配任务，并在单模型失败时自动迁移到其他模型。 +
+
+ {/* 生图模型选择 */}
- +
- 选择用于详情替换的生图模型，支持阿里生图、Gemini、既梦等图片模型 + 仅用于设置“优先尝试”的模型；实际生成会自动在多模型间轮换并故障迁移
@@ -8334,7 +8341,7 @@ export function AdminSettingsClient() {

• 用户上传1张产品图和多张场景参考图

• 系统自动将产品图融合到每个场景中

-

• 支持选择阿里生图或Gemini图片模型

+

• 固定采用多样性模式：多模型轮换 + 自动故障迁移

• 生成的图片会自动保存到用户相册

diff --git a/app/api/scene-fusion/config/route.ts b/app/api/scene-fusion/config/route.ts index 37b14ba87..632001a00 100644 --- a/app/api/scene-fusion/config/route.ts +++ b/app/api/scene-fusion/config/route.ts @@ -36,6 +36,7 @@ export async function GET() { enabled: config.enabled ?? true, defaultPrompt: config.defaultPrompt || FALLBACK_PROMPT, maxScenes: config.maxScenes ?? 10, + mode: "diversity", }); } @@ -44,6 +45,7 @@ export async function GET() { enabled: true, defaultPrompt: FALLBACK_PROMPT, maxScenes: 10, + mode: "diversity", }); } catch (error) { console.error("[SceneFusion Config] Error:", error); diff --git a/app/api/scene-fusion/route.ts b/app/api/scene-fusion/route.ts index a08074e85..11395fc11 100644 --- a/app/api/scene-fusion/route.ts +++ b/app/api/scene-fusion/route.ts @@ -13,7 +13,6 @@ import { generatedImage } from "@/lib/db/schema"; import { getEnabledAiModelsWithProvider, getSystemSetting } from "@/lib/db/queries"; import { uploadFile } from "@/lib/storage/service"; import sharp from "sharp"; -import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image"; import { createGeminiImage } from "@/lib/ai/providers/gemini-image"; import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision"; import { createRuxaImage } from "@/lib/ai/providers/ruxa-image"; @@ -36,6 +35,138 @@ const sceneFusionRequestSchema = z.object({ // 默认提示词（当数据库未配置时使用）- 使用"参考"的表述方式 const FALLBACK_PROMPT = "请参考第一张图片中的产品，将其自然地融合到第二张图片的场景中。要求：1)保持产品的整体外观、颜色、形状和设计风格与第一张图一致；2)调整产品的大小和角度以适应场景；3)注意光影和透视关系使融合自然。"; +const SCENE_FUSION_MODEL_COOLDOWN_MS = 2 * 60 * 1000; +const sceneFusionModelCooldown = new Map(); + +interface SceneFusionModelCandidate { + model: { + id: string; + modelId: string; + displayName: string; + }; + provider: { + type: string; + baseUrl: string; + apiKey: string; + }; +} + +function normalizeSceneFusionProviderType(providerType: string): string { + return providerType === "gemini-image" ? "gemini" : providerType; +} + +function canProcessSceneFusionProviderType(providerType: string): boolean { + const normalized = normalizeSceneFusionProviderType(providerType); + switch (normalized) { + case "alibaba-image": + case "ruxa-image": + case "fal-image": + case "gemini": + case "volcano-vision": + return true; + default: + return false; + } +} + +function pruneSceneFusionCooldown(now: number = Date.now()) { + for (const [key, expireAt] of sceneFusionModelCooldown.entries()) { + if (expireAt <= now) { + sceneFusionModelCooldown.delete(key); + } + } +} + +function isSceneFusionCandidateCoolingDown( + candidate: SceneFusionModelCandidate, + now: number = Date.now() +): boolean { + const expireAt = sceneFusionModelCooldown.get(candidate.model.id); + if (!expireAt) return false; + if (expireAt <= now) { + sceneFusionModelCooldown.delete(candidate.model.id); + return false; + } + return true; +} + +function markSceneFusionCandidateFailed(candidate: SceneFusionModelCandidate, now: number = Date.now()) { + sceneFusionModelCooldown.set(candidate.model.id, now + SCENE_FUSION_MODEL_COOLDOWN_MS); +} + +function markSceneFusionCandidateHealthy(candidate: SceneFusionModelCandidate) { + sceneFusionModelCooldown.delete(candidate.model.id); +} + +function rotateCandidates(items: T[], startOffset: number): T[] { + if (items.length <= 1) { + return [...items]; + } + const normalizedOffset = ((startOffset % items.length) + items.length) % items.length; + return [ + ...items.slice(normalizedOffset), + ...items.slice(0, normalizedOffset), + ]; +} + +function shuffleCandidates(items: T[]): T[] { + const shuffled = [...items]; + for (let i = shuffled.length - 1; i > 0; i -= 1) { + const j = Math.floor(Math.random() * (i + 1)); + [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]]; + } + return shuffled; +} + +function buildSceneFusionCandidatePool( + allModels: Awaited>, + preferredModelId: string +): SceneFusionModelCandidate[] { + const backendConfiguredImageModels = allModels + .filter((entry) => { + if (!entry?.model?.isEnabled) return false; + if (!entry?.model?.supportsImageGen) return false; + if (!entry?.model?.supportsVision) return false; + if (entry?.model?.isSubModel) return false; + return true; + }) + .map((entry) => ({ + model: { + id: String(entry.model.id), + modelId: String(entry.model.modelId || ""), + displayName: String(entry.model.displayName || entry.model.modelId || entry.model.id), + }, + provider: { + type: normalizeSceneFusionProviderType(String(entry.provider.type || "")), + baseUrl: String(entry.provider.baseUrl || ""), + apiKey: String(entry.provider.apiKey || ""), + }, + })) + .filter( + (candidate) => + Boolean(candidate.model.modelId) && + Boolean(candidate.provider.type) && + Boolean(candidate.provider.apiKey) + ); + + if (backendConfiguredImageModels.length <= 1) { + return backendConfiguredImageModels; + } + + const preferred = preferredModelId + ? backendConfiguredImageModels.find((candidate) => candidate.model.id === preferredModelId) + : undefined; + + if (!preferred) { + return shuffleCandidates(backendConfiguredImageModels); + } + + const rest = backendConfiguredImageModels.filter( + (candidate) => candidate.model.id !== preferred.model.id + ); + return [preferred, ...shuffleCandidates(rest)]; +} + /** * 获取场景裂变配置 */ @@ -373,6 +504,7 @@ async function processSceneWithProvider( return processWithFalImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize); case "gemini": + case "gemini-image": return processWithGemini(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize); case "volcano-vision": @@ -659,6 +791,87 @@ async function processScene( return { ...uploaded, prompt }; } +async function processSceneWithDiversity( + params: { + productImageUrl: string; + sceneImageUrl: string; + productName?: string; + targetName?: string; + customPrompt?: string; + defaultPrompt: string; + aspectRatio?: string; + imageSize?: string; + sceneIndex: number; + candidates: SceneFusionModelCandidate[]; + } +): Promise<{ + result: { thumbnail: string; original: string; prompt: string }; + candidate: SceneFusionModelCandidate; + attempts: number; +}> { + const supportedCandidates = params.candidates.filter((candidate) => + canProcessSceneFusionProviderType(candidate.provider.type) + ); + if (supportedCandidates.length === 0) { + throw new Error("后台已配置生图模型中暂无可用于场景裂变的供应商，请检查模型供应商类型"); + } + + const rotatedCandidates = rotateCandidates(supportedCandidates, params.sceneIndex); + pruneSceneFusionCooldown(); + + const healthyCandidates = rotatedCandidates.filter( + (candidate) => !isSceneFusionCandidateCoolingDown(candidate) + ); + const attemptCandidates = + healthyCandidates.length > 0 ? healthyCandidates : rotatedCandidates; + + const failureReasons: string[] = []; + + for (let attemptIndex = 0; attemptIndex < attemptCandidates.length; attemptIndex += 1) { + const candidate = attemptCandidates[attemptIndex]!; + try { + const result = await processSceneWithProvider( + params.productImageUrl, + params.sceneImageUrl, + params.productName, + params.targetName, + params.customPrompt, + params.defaultPrompt, + { + type: candidate.provider.type, + baseUrl: candidate.provider.baseUrl, + apiKey: candidate.provider.apiKey, + }, + candidate.model.modelId, + params.aspectRatio, + params.imageSize + ); + markSceneFusionCandidateHealthy(candidate); + return { + result, + candidate, + attempts: attemptIndex + 1, + }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : "未知错误"; + failureReasons.push(`${candidate.model.displayName}: ${errorMessage}`); + markSceneFusionCandidateFailed(candidate); + console.error("[SceneFusion] Candidate failed, try next:", { + model: candidate.model.modelId, + provider: candidate.provider.type, + error: errorMessage, + }); + } + } + + const reason = failureReasons.join("；"); + throw new Error( + reason + ? `多样性模式下所有可用模型均失败：${reason}` + : "多样性模式下所有可用模型均失败" + ); +} + export async function POST(request: Request) { try { // 1. 认证 @@ -708,46 +921,41 @@ export async function POST(request: Request) { console.log(`[SceneFusion] Processing ${scenes.length} scenes for user ${session.user.id}`); console.log(`[SceneFusion] Product name: ${productName || '(not specified)'}`); console.log(`[SceneFusion] Using default prompt: ${config.defaultPrompt.substring(0, 50)}...`); - console.log(`[SceneFusion] Configured model ID: ${config.modelId || '(auto - default alibaba)'}`); + console.log(`[SceneFusion] Preferred model ID: ${config.modelId || '(none)'}`); - // 4. 获取生图模型配置 + // 4. 获取生图模型池（固定多样性模式） const allModels = await getEnabledAiModelsWithProvider(); - - let selectedModel: typeof allModels[0] | undefined; - - // 如果配置了指定的模型ID，优先使用 - if (config.modelId) { - selectedModel = allModels.find( - (m) => m.model.id === config.modelId && m.model.isEnabled + const candidateModels = buildSceneFusionCandidatePool(allModels, config.modelId); + if (config.modelId && !candidateModels.some((candidate) => candidate.model.id === config.modelId)) { + console.warn( + `[SceneFusion] Preferred model ${config.modelId} not available, fallback to full diversity pool` ); - if (!selectedModel) { - console.warn(`[SceneFusion] Configured model ${config.modelId} not found or disabled, falling back to default`); - } } - - // 如果没有配置或配置的模型不可用，使用默认的阿里生图模型 - if (!selectedModel) { - // 按优先级查找：alibaba-image > ruxa-image > fal-image > gemini > volcano-vision - const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "gemini", "volcano-vision"]; - for (const providerType of imageProviderTypes) { - selectedModel = allModels.find( - (m) => m.provider.type === providerType && m.model.isEnabled - ); - if (selectedModel) break; - } - } - - if (!selectedModel) { + + if (candidateModels.length === 0) { return NextResponse.json( - { error: "未配置可用的生图模型，请联系管理员" }, + { error: "未找到可用模型，请在后台将模型同时标记为“支持生图”和“支持图片输入(vision)”并启用" }, { status: 500 } ); } + + console.log( + `[SceneFusion] Diversity candidates: ${candidateModels + .map((candidate) => `${candidate.model.displayName}(${candidate.provider.type})`) + .join(", ")}` + ); + const unsupportedCandidates = candidateModels.filter( + (candidate) => !canProcessSceneFusionProviderType(candidate.provider.type) + ); + if (unsupportedCandidates.length > 0) { + console.warn( + `[SceneFusion] Skip unsupported providers for scene fusion: ${unsupportedCandidates + .map((candidate) => `${candidate.model.displayName}(${candidate.provider.type})`) + .join(", ")}` + ); + } - const { provider, model } = selectedModel; - console.log(`[SceneFusion] Using model: ${model.displayName} (${model.modelId}) - Provider: ${provider.type}`); - - // 4. 预处理：将 Base64 图片上传到 S3 + // 5. 预处理：将 Base64 图片上传到 S3 let productImageUrl = productImage; if (productImage.startsWith("data:")) { productImageUrl = await uploadBase64ToS3(productImage, "scene-fusion-product"); @@ -765,121 +973,136 @@ export async function POST(request: Request) { }; }) ); - - // 6. 串行处理所有场景，每次间隔5秒 - const results = []; - + // 6. 创建流式响应，每完成一张图就立即发送给前端 const stream = new ReadableStream({ async start(controller) { const encoder = new TextEncoder(); + const enqueue = (payload: Record) => { + controller.enqueue(encoder.encode(`${JSON.stringify(payload)}\n`)); + }; try { // 发送初始消息 - controller.enqueue(encoder.encode(JSON.stringify({ type: "start", total: processedScenes.length }) + "\n")); + enqueue({ type: "start", total: processedScenes.length }); let successCount = 0; let failCount = 0; - - for (let i = 0; i < processedScenes.length; i++) { - const scene = processedScenes[i]; - - console.log(`[SceneFusion] ========== Processing scene ${i + 1}/${processedScenes.length} ==========`); - console.log(`[SceneFusion] Scene ID: ${scene.id}`); - console.log(`[SceneFusion] Product Image URL: ${productImageUrl.substring(0, 80)}...`); - console.log(`[SceneFusion] Scene Image URL: ${scene.imageUrl.substring(0, 80)}...`); - console.log(`[SceneFusion] Target Name: ${scene.targetName || '(not specified)'}`); - console.log(`[SceneFusion] Custom Prompt: ${scene.customPrompt || '(none)'}`); - console.log(`[SceneFusion] Provider Type: ${provider.type}`); - - // 如果不是第一张图，等待5秒 - if (i > 0) { - console.log(`[SceneFusion] Waiting 5 seconds before processing...`); - await new Promise((resolve) => setTimeout(resolve, 5000)); - } - - try { - // 使用新的通用处理函数，支持多种模型类型 - const result = await processSceneWithProvider( - productImageUrl, - scene.imageUrl, - productName, - scene.targetName, - scene.customPrompt, - config.defaultPrompt, - { type: provider.type, baseUrl: provider.baseUrl, apiKey: provider.apiKey }, - model.modelId, - aspectRatio, - imageSize + + let nextSceneIndex = 0; + const workerCount = Math.max( + 1, + Math.min(4, candidateModels.length, processedScenes.length) + ); + + const workers = Array.from({ length: workerCount }, async () => { + while (true) { + const currentIndex = nextSceneIndex; + nextSceneIndex += 1; + if (currentIndex >= processedScenes.length) break; + + const scene = processedScenes[currentIndex]; + + console.log( + `[SceneFusion] ========== Processing scene ${currentIndex + 1}/${processedScenes.length} ==========` ); - - // 将分辨率等级用于数据库存储 - const dbImageSize = imageSize || "2K"; - - // 保存到数据库（chatId 和 messageId 设为 null，因为场景裂变不关联聊天） + console.log(`[SceneFusion] Scene ID: ${scene.id}`); + console.log( + `[SceneFusion] Product Image URL: ${productImageUrl.substring(0, 80)}...` + ); + console.log( + `[SceneFusion] Scene Image URL: ${scene.imageUrl.substring(0, 80)}...` + ); + console.log( + `[SceneFusion] Target Name: ${scene.targetName || "(not specified)"}` + ); + console.log( + `[SceneFusion] Custom Prompt: ${scene.customPrompt || "(none)"}` + ); + try { - await db.insert(generatedImage).values({ - userId: session.user.id, - chatId: null, // 场景裂变不关联聊天 - messageId: null, // 场景裂变不关联消息 - thumbnailUrl: result.thumbnail, - originalUrl: result.original, - imageSize: dbImageSize, - prompt: `[场景裂变] ${result.prompt}`, - modelId: model.modelId, + const { result, candidate, attempts } = await processSceneWithDiversity({ + productImageUrl, + sceneImageUrl: scene.imageUrl, + productName, + targetName: scene.targetName, + customPrompt: scene.customPrompt, + defaultPrompt: config.defaultPrompt, + aspectRatio, + imageSize, + sceneIndex: currentIndex, + candidates: candidateModels, + }); + + const dbImageSize = imageSize || "2K"; + + try { + await db.insert(generatedImage).values({ + userId: session.user.id, + chatId: null, + messageId: null, + thumbnailUrl: result.thumbnail, + originalUrl: result.original, + imageSize: dbImageSize, + prompt: `[场景裂变] ${result.prompt}`, + modelId: candidate.model.modelId, + }); + console.log( + `[SceneFusion] ✅ Saved image to gallery: ${result.original.substring(0, 80)}...` + ); + } catch (dbError) { + console.error("[SceneFusion] ❌ Failed to save to gallery:", dbError); + } + + successCount += 1; + enqueue({ + type: "result", + data: { + id: scene.id, + status: "completed" as const, + result: result.thumbnail, + originalUrl: result.original, + model: candidate.model.displayName, + modelId: candidate.model.modelId, + providerType: candidate.provider.type, + attempts, + }, + }); + } catch (error) { + console.error(`[SceneFusion] Failed to process scene ${scene.id}:`, error); + failCount += 1; + enqueue({ + type: "result", + data: { + id: scene.id, + status: "failed" as const, + error: error instanceof Error ? error.message : "处理失败", + }, }); - console.log(`[SceneFusion] ✅ Saved image to gallery: ${result.original.substring(0, 80)}...`); - } catch (dbError) { - console.error("[SceneFusion] ❌ Failed to save to gallery:", dbError); } - - successCount++; - - // 立即发送完成的结果给前端 - controller.enqueue(encoder.encode(JSON.stringify({ - type: "result", - data: { - id: scene.id, - status: "completed" as const, - result: result.thumbnail, // 返回缩略图用于预览 - originalUrl: result.original, // 原图用于下载 - }, - }) + "\n")); - - } catch (error) { - console.error(`[SceneFusion] Failed to process scene ${scene.id}:`, error); - failCount++; - - // 立即发送失败的结果给前端 - controller.enqueue(encoder.encode(JSON.stringify({ - type: "result", - data: { - id: scene.id, - status: "failed" as const, - error: error instanceof Error ? error.message : "处理失败", - }, - }) + "\n")); } - } + }); + + await Promise.all(workers); // 发送完成消息 console.log(`[SceneFusion] Completed: ${successCount} success, ${failCount} failed`); - controller.enqueue(encoder.encode(JSON.stringify({ + enqueue({ type: "done", summary: { total: processedScenes.length, success: successCount, failed: failCount, }, - }) + "\n")); + }); } catch (error) { console.error("[SceneFusion] Stream error:", error); // 发送错误消息 - controller.enqueue(encoder.encode(JSON.stringify({ + enqueue({ type: "error", error: error instanceof Error ? error.message : "处理失败", - }) + "\n")); + }); } finally { controller.close(); }