lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(scene-fusion): enforce diversity mode from configured tagged models
Some checks failed
Lint / build (20) (push) Has been cancelled
Details
Playwright Tests / test (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Your Name 2026-03-04 20:42:38 +08:00
parent 5e155ecf3c
commit cd09eb9004
3 changed files with 355 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
app/(admin)/settings/admin-settings-client.tsx
View File

@ -775,7 +775,7 @@ export function AdminSettingsClient() {
enabled: true,
defaultPrompt: "将图1的产品替换到图2的场景中，保持图1产品的外观、颜色和细节不变，自然融合到图2的环境中。",
maxScenes: 10,
modelId: "", // 使用的生图模型ID
modelId: "", // 首选生图模型ID（多样性模式下优先尝试）
});
// 加载抠图API配置
@ -8281,19 +8281,26 @@ export function AdminSettingsClient() {
</div>
</div>
<div className="rounded-lg border border-blue-200 bg-blue-50 p-3">
<div className="text-sm font-medium text-blue-900">生成模式：多样性模式（固定）</div>
<div className="text-xs text-blue-800 mt-1">
系统会在所有可用生图模型中轮换分配任务，并在单模型失败时自动迁移到其他模型。
</div>
</div>
{/* 生图模型选择 */}
<div className="space-y-2">
<Label htmlFor="sceneFusionModel">生图模型</Label>
<Label htmlFor="sceneFusionModel">首选生图模型（可选）</Label>
<Select
value={sceneFusionConfig.modelId || "auto"}
onValueChange={(value) => setSceneFusionConfig({ ...sceneFusionConfig, modelId: value === "auto" ? "" : value })}
disabled={!sceneFusionConfig.enabled}
>
<SelectTrigger>
<SelectValue placeholder="选择生图模型（留空使用默认阿里模型）" />
<SelectValue placeholder="选择首选模型（不选则随机轮换）" />
</SelectTrigger>
<SelectContent>
<SelectItem value="auto">自动选择（默认阿里生图）</SelectItem>
<SelectItem value="auto">不指定首选（完全随机轮换）</SelectItem>
{(modelsData?.models || [])
.filter((m: AiModel) => {
// 过滤出支持图片生成/编辑的模型
@ -8325,7 +8332,7 @@ export function AdminSettingsClient() {
</SelectContent>
</Select>
<div className="text-xs text-muted-foreground">
选择用于详情替换的生图模型，支持阿里生图、Gemini、既梦等图片模型
仅用于设置“优先尝试”的模型；实际生成会自动在多模型间轮换并故障迁移
</div>
</div>
@ -8334,7 +8341,7 @@ export function AdminSettingsClient() {
<div className="text-xs text-blue-800 space-y-1">
<p>• 用户上传1张产品图和多张场景参考图</p>
<p>• 系统自动将产品图融合到每个场景中</p>
<p>• 支持选择阿里生图或Gemini图片模型</p>
<p>• 固定采用多样性模式：多模型轮换 + 自动故障迁移</p>
<p>• 生成的图片会自动保存到用户相册</p>
</div>
</div>

2
app/api/scene-fusion/config/route.ts
View File

@ -36,6 +36,7 @@ export async function GET() {
enabled: config.enabled ?? true,
defaultPrompt: config.defaultPrompt || FALLBACK_PROMPT,
maxScenes: config.maxScenes ?? 10,
mode: "diversity",
});
}
@ -44,6 +45,7 @@ export async function GET() {
enabled: true,
defaultPrompt: FALLBACK_PROMPT,
maxScenes: 10,
mode: "diversity",
});
} catch (error) {
console.error("[SceneFusion Config] Error:", error);

457
app/api/scene-fusion/route.ts
View File

@ -13,7 +13,6 @@ import { generatedImage } from "@/lib/db/schema";
import { getEnabledAiModelsWithProvider, getSystemSetting } from "@/lib/db/queries";
import { uploadFile } from "@/lib/storage/service";
import sharp from "sharp";
import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image";
import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision";
import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
@ -36,6 +35,138 @@ const sceneFusionRequestSchema = z.object({
// 默认提示词（当数据库未配置时使用）- 使用"参考"的表述方式
const FALLBACK_PROMPT = "请参考第一张图片中的产品，将其自然地融合到第二张图片的场景中。要求：1)保持产品的整体外观、颜色、形状和设计风格与第一张图一致；2)调整产品的大小和角度以适应场景；3)注意光影和透视关系使融合自然。";
const SCENE_FUSION_MODEL_COOLDOWN_MS = 2 * 60 * 1000;
const sceneFusionModelCooldown = new Map<string, number>();
interface SceneFusionModelCandidate {
model: {
id: string;
modelId: string;
displayName: string;
};
provider: {
type: string;
baseUrl: string;
apiKey: string;
};
}
function normalizeSceneFusionProviderType(providerType: string): string {
return providerType === "gemini-image" ? "gemini" : providerType;
}
function canProcessSceneFusionProviderType(providerType: string): boolean {
const normalized = normalizeSceneFusionProviderType(providerType);
switch (normalized) {
case "alibaba-image":
case "ruxa-image":
case "fal-image":
case "gemini":
case "volcano-vision":
return true;
default:
return false;
}
}
function pruneSceneFusionCooldown(now: number = Date.now()) {
for (const [key, expireAt] of sceneFusionModelCooldown.entries()) {
if (expireAt <= now) {
sceneFusionModelCooldown.delete(key);
}
}
}
function isSceneFusionCandidateCoolingDown(
candidate: SceneFusionModelCandidate,
now: number = Date.now()
): boolean {
const expireAt = sceneFusionModelCooldown.get(candidate.model.id);
if (!expireAt) return false;
if (expireAt <= now) {
sceneFusionModelCooldown.delete(candidate.model.id);
return false;
}
return true;
}
function markSceneFusionCandidateFailed(candidate: SceneFusionModelCandidate, now: number = Date.now()) {
sceneFusionModelCooldown.set(candidate.model.id, now + SCENE_FUSION_MODEL_COOLDOWN_MS);
}
function markSceneFusionCandidateHealthy(candidate: SceneFusionModelCandidate) {
sceneFusionModelCooldown.delete(candidate.model.id);
}
function rotateCandidates<T>(items: T[], startOffset: number): T[] {
if (items.length <= 1) {
return [...items];
}
const normalizedOffset = ((startOffset % items.length) + items.length) % items.length;
return [
...items.slice(normalizedOffset),
...items.slice(0, normalizedOffset),
];
}
function shuffleCandidates<T>(items: T[]): T[] {
const shuffled = [...items];
for (let i = shuffled.length - 1; i > 0; i -= 1) {
const j = Math.floor(Math.random() * (i + 1));
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
}
return shuffled;
}
function buildSceneFusionCandidatePool(
allModels: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>,
preferredModelId: string
): SceneFusionModelCandidate[] {
const backendConfiguredImageModels = allModels
.filter((entry) => {
if (!entry?.model?.isEnabled) return false;
if (!entry?.model?.supportsImageGen) return false;
if (!entry?.model?.supportsVision) return false;
if (entry?.model?.isSubModel) return false;
return true;
})
.map((entry) => ({
model: {
id: String(entry.model.id),
modelId: String(entry.model.modelId || ""),
displayName: String(entry.model.displayName || entry.model.modelId || entry.model.id),
},
provider: {
type: normalizeSceneFusionProviderType(String(entry.provider.type || "")),
baseUrl: String(entry.provider.baseUrl || ""),
apiKey: String(entry.provider.apiKey || ""),
},
}))
.filter(
(candidate) =>
Boolean(candidate.model.modelId) &&
Boolean(candidate.provider.type) &&
Boolean(candidate.provider.apiKey)
);
if (backendConfiguredImageModels.length <= 1) {
return backendConfiguredImageModels;
}
const preferred = preferredModelId
? backendConfiguredImageModels.find((candidate) => candidate.model.id === preferredModelId)
: undefined;
if (!preferred) {
return shuffleCandidates(backendConfiguredImageModels);
}
const rest = backendConfiguredImageModels.filter(
(candidate) => candidate.model.id !== preferred.model.id
);
return [preferred, ...shuffleCandidates(rest)];
}
/**
* 获取场景裂变配置
*/
@ -373,6 +504,7 @@ async function processSceneWithProvider(
return processWithFalImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
case "gemini":
case "gemini-image":
return processWithGemini(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
case "volcano-vision":
@ -659,6 +791,87 @@ async function processScene(
return { ...uploaded, prompt };
}
async function processSceneWithDiversity(
params: {
productImageUrl: string;
sceneImageUrl: string;
productName?: string;
targetName?: string;
customPrompt?: string;
defaultPrompt: string;
aspectRatio?: string;
imageSize?: string;
sceneIndex: number;
candidates: SceneFusionModelCandidate[];
}
): Promise<{
result: { thumbnail: string; original: string; prompt: string };
candidate: SceneFusionModelCandidate;
attempts: number;
}> {
const supportedCandidates = params.candidates.filter((candidate) =>
canProcessSceneFusionProviderType(candidate.provider.type)
);
if (supportedCandidates.length === 0) {
throw new Error("后台已配置生图模型中暂无可用于场景裂变的供应商，请检查模型供应商类型");
}
const rotatedCandidates = rotateCandidates(supportedCandidates, params.sceneIndex);
pruneSceneFusionCooldown();
const healthyCandidates = rotatedCandidates.filter(
(candidate) => !isSceneFusionCandidateCoolingDown(candidate)
);
const attemptCandidates =
healthyCandidates.length > 0 ? healthyCandidates : rotatedCandidates;
const failureReasons: string[] = [];
for (let attemptIndex = 0; attemptIndex < attemptCandidates.length; attemptIndex += 1) {
const candidate = attemptCandidates[attemptIndex]!;
try {
const result = await processSceneWithProvider(
params.productImageUrl,
params.sceneImageUrl,
params.productName,
params.targetName,
params.customPrompt,
params.defaultPrompt,
{
type: candidate.provider.type,
baseUrl: candidate.provider.baseUrl,
apiKey: candidate.provider.apiKey,
},
candidate.model.modelId,
params.aspectRatio,
params.imageSize
);
markSceneFusionCandidateHealthy(candidate);
return {
result,
candidate,
attempts: attemptIndex + 1,
};
} catch (error) {
const errorMessage = error instanceof Error ? error.message : "未知错误";
failureReasons.push(`${candidate.model.displayName}: ${errorMessage}`);
markSceneFusionCandidateFailed(candidate);
console.error("[SceneFusion] Candidate failed, try next:", {
model: candidate.model.modelId,
provider: candidate.provider.type,
error: errorMessage,
});
}
}
const reason = failureReasons.join("；");
throw new Error(
reason
? `多样性模式下所有可用模型均失败：${reason}`
: "多样性模式下所有可用模型均失败"
);
}
export async function POST(request: Request) {
try {
// 1. 认证
@ -708,46 +921,41 @@ export async function POST(request: Request) {
console.log(`[SceneFusion] Processing ${scenes.length} scenes for user ${session.user.id}`);
console.log(`[SceneFusion] Product name: ${productName || '(not specified)'}`);
console.log(`[SceneFusion] Using default prompt: ${config.defaultPrompt.substring(0, 50)}...`);
console.log(`[SceneFusion] Configured model ID: ${config.modelId || '(auto - default alibaba)'}`);
console.log(`[SceneFusion] Preferred model ID: ${config.modelId || '(none)'}`);
// 4. 获取生图模型配置
// 4. 获取生图模型池（固定多样性模式）
const allModels = await getEnabledAiModelsWithProvider();
let selectedModel: typeof allModels[0] | undefined;
// 如果配置了指定的模型ID，优先使用
if (config.modelId) {
selectedModel = allModels.find(
(m) => m.model.id === config.modelId && m.model.isEnabled
const candidateModels = buildSceneFusionCandidatePool(allModels, config.modelId);
if (config.modelId && !candidateModels.some((candidate) => candidate.model.id === config.modelId)) {
console.warn(
`[SceneFusion] Preferred model ${config.modelId} not available, fallback to full diversity pool`
);
if (!selectedModel) {
console.warn(`[SceneFusion] Configured model ${config.modelId} not found or disabled, falling back to default`);
}
}
// 如果没有配置或配置的模型不可用，使用默认的阿里生图模型
if (!selectedModel) {
// 按优先级查找：alibaba-image > ruxa-image > fal-image > gemini > volcano-vision
const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "gemini", "volcano-vision"];
for (const providerType of imageProviderTypes) {
selectedModel = allModels.find(
(m) => m.provider.type === providerType && m.model.isEnabled
);
if (selectedModel) break;
}
}
if (!selectedModel) {
if (candidateModels.length === 0) {
return NextResponse.json(
{ error: "未配置可用的生图模型，请联系管理员" },
{ error: "未找到可用模型，请在后台将模型同时标记为“支持生图”和“支持图片输入(vision)”并启用" },
{ status: 500 }
);
}
console.log(
`[SceneFusion] Diversity candidates: ${candidateModels
.map((candidate) => `${candidate.model.displayName}(${candidate.provider.type})`)
.join(", ")}`
);
const unsupportedCandidates = candidateModels.filter(
(candidate) => !canProcessSceneFusionProviderType(candidate.provider.type)
);
if (unsupportedCandidates.length > 0) {
console.warn(
`[SceneFusion] Skip unsupported providers for scene fusion: ${unsupportedCandidates
.map((candidate) => `${candidate.model.displayName}(${candidate.provider.type})`)
.join(", ")}`
);
}
const { provider, model } = selectedModel;
console.log(`[SceneFusion] Using model: ${model.displayName} (${model.modelId}) - Provider: ${provider.type}`);
// 4. 预处理：将 Base64 图片上传到 S3
// 5. 预处理：将 Base64 图片上传到 S3
let productImageUrl = productImage;
if (productImage.startsWith("data:")) {
productImageUrl = await uploadBase64ToS3(productImage, "scene-fusion-product");
@ -765,121 +973,136 @@ export async function POST(request: Request) {
};
})
);
// 6. 串行处理所有场景，每次间隔5秒
const results = [];
// 6. 创建流式响应，每完成一张图就立即发送给前端
const stream = new ReadableStream({
async start(controller) {
const encoder = new TextEncoder();
const enqueue = (payload: Record<string, unknown>) => {
controller.enqueue(encoder.encode(`${JSON.stringify(payload)}\n`));
};
try {
// 发送初始消息
controller.enqueue(encoder.encode(JSON.stringify({ type: "start", total: processedScenes.length }) + "\n"));
enqueue({ type: "start", total: processedScenes.length });
let successCount = 0;
let failCount = 0;
for (let i = 0; i < processedScenes.length; i++) {
const scene = processedScenes[i];
console.log(`[SceneFusion] ========== Processing scene ${i + 1}/${processedScenes.length} ==========`);
console.log(`[SceneFusion] Scene ID: ${scene.id}`);
console.log(`[SceneFusion] Product Image URL: ${productImageUrl.substring(0, 80)}...`);
console.log(`[SceneFusion] Scene Image URL: ${scene.imageUrl.substring(0, 80)}...`);
console.log(`[SceneFusion] Target Name: ${scene.targetName || '(not specified)'}`);
console.log(`[SceneFusion] Custom Prompt: ${scene.customPrompt || '(none)'}`);
console.log(`[SceneFusion] Provider Type: ${provider.type}`);
// 如果不是第一张图，等待5秒
if (i > 0) {
console.log(`[SceneFusion] Waiting 5 seconds before processing...`);
await new Promise((resolve) => setTimeout(resolve, 5000));
}
try {
// 使用新的通用处理函数，支持多种模型类型
const result = await processSceneWithProvider(
productImageUrl,
scene.imageUrl,
productName,
scene.targetName,
scene.customPrompt,
config.defaultPrompt,
{ type: provider.type, baseUrl: provider.baseUrl, apiKey: provider.apiKey },
model.modelId,
aspectRatio,
imageSize
let nextSceneIndex = 0;
const workerCount = Math.max(
1,
Math.min(4, candidateModels.length, processedScenes.length)
);
const workers = Array.from({ length: workerCount }, async () => {
while (true) {
const currentIndex = nextSceneIndex;
nextSceneIndex += 1;
if (currentIndex >= processedScenes.length) break;
const scene = processedScenes[currentIndex];
console.log(
`[SceneFusion] ========== Processing scene ${currentIndex + 1}/${processedScenes.length} ==========`
);
// 将分辨率等级用于数据库存储
const dbImageSize = imageSize || "2K";
// 保存到数据库（chatId 和 messageId 设为 null，因为场景裂变不关联聊天）
console.log(`[SceneFusion] Scene ID: ${scene.id}`);
console.log(
`[SceneFusion] Product Image URL: ${productImageUrl.substring(0, 80)}...`
);
console.log(
`[SceneFusion] Scene Image URL: ${scene.imageUrl.substring(0, 80)}...`
);
console.log(
`[SceneFusion] Target Name: ${scene.targetName || "(not specified)"}`
);
console.log(
`[SceneFusion] Custom Prompt: ${scene.customPrompt || "(none)"}`
);
try {
await db.insert(generatedImage).values({
userId: session.user.id,
chatId: null, // 场景裂变不关联聊天
messageId: null, // 场景裂变不关联消息
thumbnailUrl: result.thumbnail,
originalUrl: result.original,
imageSize: dbImageSize,
prompt: `[场景裂变] ${result.prompt}`,
modelId: model.modelId,
const { result, candidate, attempts } = await processSceneWithDiversity({
productImageUrl,
sceneImageUrl: scene.imageUrl,
productName,
targetName: scene.targetName,
customPrompt: scene.customPrompt,
defaultPrompt: config.defaultPrompt,
aspectRatio,
imageSize,
sceneIndex: currentIndex,
candidates: candidateModels,
});
const dbImageSize = imageSize || "2K";
try {
await db.insert(generatedImage).values({
userId: session.user.id,
chatId: null,
messageId: null,
thumbnailUrl: result.thumbnail,
originalUrl: result.original,
imageSize: dbImageSize,
prompt: `[场景裂变] ${result.prompt}`,
modelId: candidate.model.modelId,
});
console.log(
`[SceneFusion] ✅ Saved image to gallery: ${result.original.substring(0, 80)}...`
);
} catch (dbError) {
console.error("[SceneFusion] ❌ Failed to save to gallery:", dbError);
}
successCount += 1;
enqueue({
type: "result",
data: {
id: scene.id,
status: "completed" as const,
result: result.thumbnail,
originalUrl: result.original,
model: candidate.model.displayName,
modelId: candidate.model.modelId,
providerType: candidate.provider.type,
attempts,
},
});
} catch (error) {
console.error(`[SceneFusion] Failed to process scene ${scene.id}:`, error);
failCount += 1;
enqueue({
type: "result",
data: {
id: scene.id,
status: "failed" as const,
error: error instanceof Error ? error.message : "处理失败",
},
});
console.log(`[SceneFusion] ✅ Saved image to gallery: ${result.original.substring(0, 80)}...`);
} catch (dbError) {
console.error("[SceneFusion] ❌ Failed to save to gallery:", dbError);
}
successCount++;
// 立即发送完成的结果给前端
controller.enqueue(encoder.encode(JSON.stringify({
type: "result",
data: {
id: scene.id,
status: "completed" as const,
result: result.thumbnail, // 返回缩略图用于预览
originalUrl: result.original, // 原图用于下载
},
}) + "\n"));
} catch (error) {
console.error(`[SceneFusion] Failed to process scene ${scene.id}:`, error);
failCount++;
// 立即发送失败的结果给前端
controller.enqueue(encoder.encode(JSON.stringify({
type: "result",
data: {
id: scene.id,
status: "failed" as const,
error: error instanceof Error ? error.message : "处理失败",
},
}) + "\n"));
}
}
});
await Promise.all(workers);
// 发送完成消息
console.log(`[SceneFusion] Completed: ${successCount} success, ${failCount} failed`);
controller.enqueue(encoder.encode(JSON.stringify({
enqueue({
type: "done",
summary: {
total: processedScenes.length,
success: successCount,
failed: failCount,
},
}) + "\n"));
});
} catch (error) {
console.error("[SceneFusion] Stream error:", error);
// 发送错误消息
controller.enqueue(encoder.encode(JSON.stringify({
enqueue({
type: "error",
error: error instanceof Error ? error.message : "处理失败",
}) + "\n"));
});
} finally {
controller.close();
}