2735 lines
88 KiB
TypeScript
2735 lines
88 KiB
TypeScript
/**
|
||
* 场景裂变 API
|
||
*
|
||
* 将一张产品图批量融合到多个场景图中
|
||
* 支持多种生图模型：阿里生图、RUXA、SnapGen、Gemini、既梦
|
||
*/
|
||
|
||
import { NextResponse } from "next/server";
|
||
import { z } from "zod";
|
||
import { auth } from "@/app/(auth)/auth";
|
||
import { db } from "@/lib/db";
|
||
import { generatedImage } from "@/lib/db/schema";
|
||
import {
|
||
createMediaAssetRelation,
|
||
getEnabledAiModelsWithProvider,
|
||
getMediaAssetsByUserAndUrls,
|
||
getSystemSetting,
|
||
upsertMediaAsset,
|
||
} from "@/lib/db/queries";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
import sharp from "sharp";
|
||
import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
|
||
import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision";
|
||
import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
|
||
import { createFalImage } from "@/lib/ai/providers/fal-image";
|
||
import { createSnapgenImage } from "@/lib/ai/providers/snapgen-image";
|
||
import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
|
||
import { createBananaImage } from "@/lib/ai/providers/bananaimage";
|
||
import { createVisionary } from "@/lib/ai/providers/visionary";
|
||
import { createAtlasCloud } from "@/lib/ai/providers/atlascloud";
|
||
import { createVip98k } from "@/lib/ai/providers/vip98k";
|
||
import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
|
||
import {
|
||
canProcessSceneFusionProviderType,
|
||
normalizeSceneFusionOutputParamsForCandidate,
|
||
normalizeSceneFusionProviderType,
|
||
type SceneFusionImageGenConfig,
|
||
} from "@/lib/scene-fusion/output-capabilities";
|
||
import { acquireCanvasImageModelSlot } from "@/lib/canvas/image-concurrency";
|
||
import { createLogger } from "@/lib/observability/logger";
|
||
import { resolveModelRuntimeId } from "@/lib/ai/model-runtime";
|
||
import { modelSupportsReferenceImageInput } from "@/lib/ai/reference-image-capability";
|
||
import { resolveLongestEdgeDimensions } from "@/lib/ai/image-size-constraints";
|
||
|
||
// 请求体 Schema
|
||
const sceneFusionRequestSchema = z.object({
|
||
productImage: z.string().min(1, "产品图不能为空"),
|
||
productName: z.string().optional(), // 产品名称（如"沙发"）
|
||
sceneMode: z.enum(["scene-fission", "multi-product-scene", "random-bg"]).optional(),
|
||
aspectRatio: z.string().optional(), // 图片比例（如"1:1", "4:3", "auto"）
|
||
imageSize: z.string().optional(), // 图片分辨率（如"1K", "2K", "4K"）
|
||
scenes: z.array(z.object({
|
||
id: z.string(),
|
||
image: z.string().min(1, "场景图不能为空"),
|
||
targetName: z.string().optional(), // 背景中需要被替换的物体名称（如"椅子"）
|
||
customPrompt: z.string().optional(),
|
||
})).min(1, "至少需要一个场景图").max(10, "最多支持10个场景图"),
|
||
});
|
||
|
||
type SceneFusionMode = "scene-fission" | "multi-product-scene" | "random-bg";
|
||
|
||
// 默认提示词（当数据库未配置时使用）- 强化主体锁定，避免把产品图里的摆件/文字/背景当作产品替入。
|
||
const FALLBACK_PROMPT =
|
||
"请参考第一张图片中的主产品主体，将其自然地融合到第二张图片的场景中。要求：1)只提取第一张图的主产品，忽略产品图中的杂志、杯子、摆件、文字、Logo、背景和装饰附件；2)必须保持主产品的核心结构、轮廓比例、部件数量、材质颜色和关键设计细节与第一张图一致；3)允许为了匹配参考场景调整产品角度、朝向、大小和透视，不要把角度变化当成结构改变；4)先移除第二张图里需要被替换的完整目标物，再放入主产品，禁止直接叠加在原目标物上；5)保留第二张图的大部分场景、机位、构图、光影方向和背景关系，使融合自然。";
|
||
|
||
const SCENE_FUSION_MODEL_COOLDOWN_MS = 2 * 60 * 1000;
|
||
const SCENE_FUSION_MAX_ITEMS = 10;
|
||
const SCENE_FUSION_REQUEST_STAGGER_MS = 350;
|
||
const SCENE_FUSION_AGENT_TIMEOUT_MS = 20000;
|
||
const SCENE_FUSION_AGENT_MAX_WORKERS = 3;
|
||
const SCENE_FUSION_PROCESS_MAX_WORKERS = 4;
|
||
const SCENE_FUSION_PRE_RMBG_TIMEOUT_MS = 60_000;
|
||
const SCENE_FUSION_ECHO_HASH_TIMEOUT_MS = 20_000;
|
||
const SCENE_FUSION_ECHO_HASH_THRESHOLD = 8;
|
||
const SCENE_FUSION_REVIEW_TIMEOUT_MS = 25_000;
|
||
const SCENE_FUSION_STRICT_PROMPT_TAIL =
|
||
"硬性要求：严禁直接返回第一张参考图（产品图）原图；严禁把产品图中的杂志、杯子、摆件、文字、Logo、背景或装饰附件当成要替换的产品；必须保留第二张图的场景与构图，仅替换目标主体；替换前必须移除原目标主体，结果中不得同时完整保留原目标主体与新产品，不得把新产品直接叠加在原物体表面；产品可因场景透视调整角度、朝向和大小，但核心结构、轮廓比例、部件关系、材质颜色不得被改造；若识别不确定，请优先保持第二张图背景不变并完成主体替换。";
|
||
const SCENE_FUSION_RANDOM_BG_PROMPT_TAIL =
|
||
"硬性要求：背景必须与原图明显不同（颜色、结构、元素至少两项显著变化），禁止保留原背景纹理或直接回传原图；仅保留产品主体并保证主体结构、轮廓比例、部件关系和边缘完整自然，允许为新背景调整朝向、角度和透视。";
|
||
let sceneFusionRoundRobinCursor = 0;
|
||
type SceneFusionModelCooldown = Map<string, number>;
|
||
const logger = createLogger("scene-fusion");
|
||
|
||
interface SceneFusionModelCandidate {
|
||
model: {
|
||
id: string;
|
||
modelId: string;
|
||
displayName: string;
|
||
supportsVision: boolean;
|
||
imageGenConfig?: SceneFusionImageGenConfig | null;
|
||
excludeFromFailover?: boolean;
|
||
defaultParams?: Record<string, any> | null;
|
||
};
|
||
provider: {
|
||
id?: string;
|
||
name?: string;
|
||
type: string;
|
||
baseUrl: string;
|
||
apiKey: string;
|
||
};
|
||
}
|
||
|
||
interface SceneFusionAgentMeta {
|
||
productName?: string;
|
||
productSource?: "user" | "agent";
|
||
productConfidence?: number;
|
||
productModel?: string;
|
||
sceneTargets: Array<{
|
||
id: string;
|
||
targetName?: string;
|
||
positionHint?: string;
|
||
replaceCue?: string;
|
||
source?: "user" | "agent";
|
||
confidence?: number;
|
||
model?: string;
|
||
}>;
|
||
}
|
||
|
||
function pruneSceneFusionCooldown(
|
||
cooldown: SceneFusionModelCooldown,
|
||
now: number = Date.now()
|
||
) {
|
||
for (const [key, expireAt] of cooldown.entries()) {
|
||
if (expireAt <= now) {
|
||
cooldown.delete(key);
|
||
}
|
||
}
|
||
}
|
||
|
||
function isSceneFusionCandidateCoolingDown(
|
||
candidate: SceneFusionModelCandidate,
|
||
cooldown: SceneFusionModelCooldown,
|
||
now: number = Date.now()
|
||
): boolean {
|
||
const expireAt = cooldown.get(candidate.model.id);
|
||
if (!expireAt) return false;
|
||
if (expireAt <= now) {
|
||
cooldown.delete(candidate.model.id);
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
function markSceneFusionCandidateFailed(
|
||
candidate: SceneFusionModelCandidate,
|
||
cooldown: SceneFusionModelCooldown,
|
||
now: number = Date.now()
|
||
) {
|
||
cooldown.set(candidate.model.id, now + SCENE_FUSION_MODEL_COOLDOWN_MS);
|
||
}
|
||
|
||
function markSceneFusionCandidateHealthy(
|
||
candidate: SceneFusionModelCandidate,
|
||
cooldown: SceneFusionModelCooldown
|
||
) {
|
||
cooldown.delete(candidate.model.id);
|
||
}
|
||
|
||
function rotateCandidates<T>(items: T[], startOffset: number): T[] {
|
||
if (items.length <= 1) {
|
||
return [...items];
|
||
}
|
||
const normalizedOffset = ((startOffset % items.length) + items.length) % items.length;
|
||
return [
|
||
...items.slice(normalizedOffset),
|
||
...items.slice(0, normalizedOffset),
|
||
];
|
||
}
|
||
|
||
function getNextSceneFusionRoundRobinOffset(poolSize: number): number {
|
||
if (poolSize <= 0) return 0;
|
||
const offset = sceneFusionRoundRobinCursor % poolSize;
|
||
sceneFusionRoundRobinCursor = (sceneFusionRoundRobinCursor + 1) % Number.MAX_SAFE_INTEGER;
|
||
return offset;
|
||
}
|
||
|
||
function getSceneFusionCandidatePriority(candidate: SceneFusionModelCandidate): number {
|
||
const providerType = normalizeSceneFusionProviderType(candidate.provider.type);
|
||
const modelText = `${candidate.model.displayName} ${candidate.model.modelId}`.toLowerCase();
|
||
|
||
let score = 0;
|
||
|
||
switch (providerType) {
|
||
case "openai-image":
|
||
score += 130;
|
||
break;
|
||
case "fal-image":
|
||
score += 120;
|
||
break;
|
||
case "snapgen-image":
|
||
score += 116;
|
||
break;
|
||
case "alibaba-image":
|
||
score += 112;
|
||
break;
|
||
case "gemini":
|
||
score += 104;
|
||
break;
|
||
case "volcano-vision":
|
||
score += 96;
|
||
break;
|
||
case "ruxa-image":
|
||
score += 72;
|
||
break;
|
||
default:
|
||
score += 60;
|
||
break;
|
||
}
|
||
|
||
score += candidate.model.supportsVision ? 28 : -18;
|
||
|
||
if (candidate.model.imageGenConfig?.supports2K !== false) {
|
||
score += 4;
|
||
}
|
||
if (candidate.model.imageGenConfig?.supports4K !== false) {
|
||
score += 2;
|
||
}
|
||
|
||
if (/nano-banana|ruxa/i.test(modelText)) {
|
||
score -= 18;
|
||
}
|
||
if (/seedream-5\.0/i.test(modelText)) {
|
||
score -= 4;
|
||
}
|
||
if (/edit|image[-\s]?edit|gpt[-\s]?image/i.test(modelText)) {
|
||
score += 6;
|
||
}
|
||
|
||
return score;
|
||
}
|
||
|
||
function isSceneFusionReviewRejectError(message: string): boolean {
|
||
return (
|
||
message.includes("生成结果未通过替换质检") ||
|
||
message.includes("生成结果疑似回传产品图")
|
||
);
|
||
}
|
||
|
||
function buildPreRmbgCandidateUrls(request: Request): string[] {
|
||
const urls: string[] = [];
|
||
const requestUrl = new URL(request.url);
|
||
const requestHost = requestUrl.hostname.toLowerCase();
|
||
const requestProtocol = requestUrl.protocol.replace(":", "");
|
||
if (!(requestProtocol === "https" && (requestHost === "localhost" || requestHost === "127.0.0.1"))) {
|
||
urls.push(new URL("/api/rmbg", request.url).toString());
|
||
}
|
||
|
||
const host = request.headers.get("host")?.trim();
|
||
const protoHint = request.headers
|
||
.get("x-forwarded-proto")
|
||
?.split(",")[0]
|
||
?.trim()
|
||
.toLowerCase();
|
||
const protocol = protoHint === "https" || protoHint === "http" ? protoHint : requestUrl.protocol.replace(":", "");
|
||
|
||
if (host) {
|
||
const normalizedHost = host.split(":")[0]?.toLowerCase() || "";
|
||
const isLocalHost = normalizedHost === "localhost" || normalizedHost === "127.0.0.1";
|
||
if (!(protocol === "https" && isLocalHost)) {
|
||
urls.push(`${protocol}://${host}/api/rmbg`);
|
||
}
|
||
if (isLocalHost) {
|
||
urls.push(`http://${host}/api/rmbg`);
|
||
}
|
||
}
|
||
|
||
const localPort = requestUrl.port || process.env.PORT || "3125";
|
||
urls.push(`http://127.0.0.1:${localPort}/api/rmbg`);
|
||
urls.push(`http://localhost:${localPort}/api/rmbg`);
|
||
|
||
return Array.from(new Set(urls));
|
||
}
|
||
|
||
function normalizeUrlForCompare(input: string): string {
|
||
try {
|
||
const parsed = new URL(input);
|
||
return `${parsed.origin}${parsed.pathname}`;
|
||
} catch {
|
||
return input;
|
||
}
|
||
}
|
||
|
||
async function fetchImageBufferWithTimeout(url: string, timeoutMs: number): Promise<Buffer> {
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||
try {
|
||
const response = await fetch(url, { signal: controller.signal });
|
||
if (!response.ok) {
|
||
throw new Error(`fetch_failed_${response.status}`);
|
||
}
|
||
return Buffer.from(await response.arrayBuffer());
|
||
} finally {
|
||
clearTimeout(timeoutId);
|
||
}
|
||
}
|
||
|
||
async function getAverageHashBits(
|
||
imageUrl: string,
|
||
hashCache: Map<string, string>
|
||
): Promise<string | null> {
|
||
if (hashCache.has(imageUrl)) {
|
||
return hashCache.get(imageUrl)!;
|
||
}
|
||
|
||
try {
|
||
const buffer = await fetchImageBufferWithTimeout(imageUrl, SCENE_FUSION_ECHO_HASH_TIMEOUT_MS);
|
||
const pixels = await sharp(buffer)
|
||
.resize(8, 8, { fit: "fill" })
|
||
.grayscale()
|
||
.raw()
|
||
.toBuffer();
|
||
if (!pixels.length) return null;
|
||
const total = pixels.reduce((sum, value) => sum + value, 0);
|
||
const avg = total / pixels.length;
|
||
const hashBits = Array.from(pixels)
|
||
.map((value) => (value >= avg ? "1" : "0"))
|
||
.join("");
|
||
hashCache.set(imageUrl, hashBits);
|
||
return hashBits;
|
||
} catch (error) {
|
||
logger.warn("echo_hash_compute_skipped", {
|
||
error: error instanceof Error ? error.message : "unknown",
|
||
});
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function hammingDistanceBits(a: string, b: string): number {
|
||
if (a.length !== b.length) {
|
||
return Math.max(a.length, b.length);
|
||
}
|
||
let distance = 0;
|
||
for (let index = 0; index < a.length; index += 1) {
|
||
if (a[index] !== b[index]) {
|
||
distance += 1;
|
||
}
|
||
}
|
||
return distance;
|
||
}
|
||
|
||
async function detectEchoedProductResult(
|
||
params: {
|
||
requestId: string;
|
||
sceneIndex: number;
|
||
productImageUrl: string;
|
||
generatedImageUrl: string;
|
||
modelName: string;
|
||
hashCache: Map<string, string>;
|
||
}
|
||
): Promise<{
|
||
tooSimilar: boolean;
|
||
reason?: string;
|
||
hammingDistance?: number;
|
||
}> {
|
||
const normalizedProductUrl = normalizeUrlForCompare(params.productImageUrl);
|
||
const normalizedGeneratedUrl = normalizeUrlForCompare(params.generatedImageUrl);
|
||
|
||
if (normalizedProductUrl === normalizedGeneratedUrl) {
|
||
return { tooSimilar: true, reason: "same_url" };
|
||
}
|
||
|
||
const [productHash, generatedHash] = await Promise.all([
|
||
getAverageHashBits(params.productImageUrl, params.hashCache),
|
||
getAverageHashBits(params.generatedImageUrl, params.hashCache),
|
||
]);
|
||
|
||
if (!productHash || !generatedHash) {
|
||
return { tooSimilar: false };
|
||
}
|
||
|
||
const distance = hammingDistanceBits(productHash, generatedHash);
|
||
if (distance <= SCENE_FUSION_ECHO_HASH_THRESHOLD) {
|
||
logger.warn("echo_guard_hit", {
|
||
requestId: params.requestId,
|
||
sceneIndex: params.sceneIndex + 1,
|
||
modelName: params.modelName,
|
||
distance,
|
||
});
|
||
return {
|
||
tooSimilar: true,
|
||
reason: "hash_too_close",
|
||
hammingDistance: distance,
|
||
};
|
||
}
|
||
|
||
return { tooSimilar: false, hammingDistance: distance };
|
||
}
|
||
|
||
function waitSceneFusion(ms: number): Promise<void> {
|
||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||
}
|
||
|
||
function normalizeSceneFusionName(value: unknown): string {
|
||
const raw = String(value ?? "").trim();
|
||
if (!raw) return "";
|
||
const normalized = raw
|
||
.replace(/[：:，,。.!！?？、；;（）()[\]{}"'`]/g, " ")
|
||
.replace(/\s+/g, " ")
|
||
.trim();
|
||
return normalized.slice(0, 16);
|
||
}
|
||
|
||
function normalizeSceneFusionHint(value: unknown, maxLength: number = 28): string {
|
||
const raw = String(value ?? "").trim();
|
||
if (!raw) return "";
|
||
const normalized = raw
|
||
.replace(/[\r\n\t]+/g, " ")
|
||
.replace(/[“”"']/g, "")
|
||
.replace(/\s+/g, " ")
|
||
.trim();
|
||
return normalized.slice(0, maxLength);
|
||
}
|
||
|
||
function normalizeSceneFusionBoolean(value: unknown): boolean | undefined {
|
||
if (typeof value === "boolean") return value;
|
||
const normalized = String(value ?? "").trim().toLowerCase();
|
||
if (!normalized) return undefined;
|
||
if (["true", "yes", "y", "1", "是"].includes(normalized)) return true;
|
||
if (["false", "no", "n", "0", "否"].includes(normalized)) return false;
|
||
return undefined;
|
||
}
|
||
|
||
function extractJsonObject(text: string): Record<string, unknown> | null {
|
||
const content = String(text || "").trim();
|
||
if (!content) return null;
|
||
const match = content.match(/\{[\s\S]*\}/);
|
||
if (!match) return null;
|
||
try {
|
||
const parsed = JSON.parse(match[0]);
|
||
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
||
return parsed as Record<string, unknown>;
|
||
}
|
||
return null;
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
async function detectProductNameWithAgent(imageUrl: string): Promise<{
|
||
name: string;
|
||
confidence: number;
|
||
model: string;
|
||
} | null> {
|
||
const prompt = [
|
||
"你是电商图片分析助手。",
|
||
"请识别图片中最核心、最主要、应被用于替换的产品主体名称。",
|
||
"要求：",
|
||
"- 名称精简，2-8个字，尽量包含材质/品类（如：皮质沙发、木质餐椅、陶瓷花瓶）",
|
||
"- 如果画面里有杂志、杯子、蜡烛、摆件、文字、Logo、包装、背景或装饰附件，只把它们当作干扰项，不要作为产品主体返回",
|
||
"- 如果产品图里有多个物体，优先选择占画面最大、最像商品主卖点、可独立售卖的主体",
|
||
"- 不要返回颜色描述句子，不要返回背景和场景词，不要返回附件名",
|
||
"仅返回JSON：",
|
||
'{\"productName\":\"名称\",\"confidence\":0.0}',
|
||
].join("\n");
|
||
|
||
const { text, model } = await callQwen3Vl({
|
||
imageUrl,
|
||
prompt,
|
||
temperature: 0.1,
|
||
maxTokens: 220,
|
||
timeoutMs: SCENE_FUSION_AGENT_TIMEOUT_MS,
|
||
retries: 1,
|
||
});
|
||
|
||
const parsed = extractJsonObject(text);
|
||
const name = normalizeSceneFusionName(parsed?.productName);
|
||
const confidenceRaw = Number(parsed?.confidence);
|
||
const confidence = Number.isFinite(confidenceRaw) ? Math.max(0, Math.min(1, confidenceRaw)) : 0.7;
|
||
|
||
if (!name || confidence < 0.45) {
|
||
return null;
|
||
}
|
||
|
||
return { name, confidence, model };
|
||
}
|
||
|
||
async function detectTargetNameWithAgent(params: {
|
||
productImageUrl: string;
|
||
sceneImageUrl: string;
|
||
productNameHint: string;
|
||
}): Promise<{
|
||
name: string;
|
||
confidence: number;
|
||
model: string;
|
||
positionHint?: string;
|
||
replaceCue?: string;
|
||
} | null> {
|
||
const prompt = [
|
||
"你是场景替换识别助手。",
|
||
`图1是待融合的产品参考图，产品名称提示：${params.productNameHint || "该产品"}`,
|
||
"图2是待处理的场景图。",
|
||
"请识别图2里最应该被图1主产品替换的完整目标物体，并给出简短位置描述，帮助后续模型准确定位。",
|
||
"要求：",
|
||
"- 返回“完整物体名”，不要返回局部部件名",
|
||
"- 名称精简，2-8个字（如：餐椅、沙发、茶几、花瓶）",
|
||
"- positionHint 用 4-16 个字描述目标大致位置（如：画面中央偏左、前景靠窗处、餐桌区域）",
|
||
"- replaceCue 用 4-20 个字描述该目标最醒目的识别特征（如：深色办公椅、木质圆餐桌）",
|
||
"- 不要把产品图里的附件、文字、Logo、装饰物当成目标；目标必须是图2中需要被移除并替换的完整原物体",
|
||
"- 如果图2里有多个同类物体，优先选择与图1主产品用途/体量最接近、画面中最醒目的那个",
|
||
"- 如果无法确定，返回最主要主体物体名，并尽量给出位置",
|
||
"仅返回JSON：",
|
||
'{\"targetName\":\"名称\",\"positionHint\":\"位置\",\"replaceCue\":\"识别特征\",\"confidence\":0.0}',
|
||
].join("\n");
|
||
|
||
const { text, model } = await callQwen3Vl({
|
||
imageUrls: [params.productImageUrl, params.sceneImageUrl],
|
||
prompt,
|
||
temperature: 0.1,
|
||
maxTokens: 320,
|
||
timeoutMs: SCENE_FUSION_AGENT_TIMEOUT_MS,
|
||
retries: 1,
|
||
});
|
||
|
||
const parsed = extractJsonObject(text);
|
||
const name = normalizeSceneFusionName(parsed?.targetName);
|
||
const positionHint = normalizeSceneFusionHint(parsed?.positionHint, 20) || undefined;
|
||
const replaceCue = normalizeSceneFusionHint(parsed?.replaceCue, 26) || undefined;
|
||
const confidenceRaw = Number(parsed?.confidence);
|
||
const confidence = Number.isFinite(confidenceRaw) ? Math.max(0, Math.min(1, confidenceRaw)) : 0.7;
|
||
|
||
if (!name || confidence < 0.4) {
|
||
return null;
|
||
}
|
||
|
||
return { name, confidence, model, positionHint, replaceCue };
|
||
}
|
||
|
||
async function reviewSceneFusionResult(params: {
|
||
requestId: string;
|
||
sceneIndex: number;
|
||
sceneMode: SceneFusionMode;
|
||
productImageUrl: string;
|
||
sceneImageUrl: string;
|
||
generatedImageUrl: string;
|
||
productName?: string;
|
||
targetName?: string;
|
||
positionHint?: string;
|
||
replaceCue?: string;
|
||
modelName: string;
|
||
}): Promise<{
|
||
passed: boolean;
|
||
score: number;
|
||
reason: string;
|
||
model: string;
|
||
} | null> {
|
||
if (params.sceneMode === "random-bg") {
|
||
return null;
|
||
}
|
||
|
||
const targetLabel = params.targetName || "目标主体";
|
||
const productLabel = params.productName || "产品";
|
||
const locationHints = [params.positionHint, params.replaceCue].filter(Boolean).join("；");
|
||
const prompt = [
|
||
"你是电商场景替换质检助手。",
|
||
"图1是产品参考图，图2是原始场景图，图3是生成结果图。",
|
||
`目标任务：把图1中的${productLabel}替换进图2中${targetLabel}所在的位置。`,
|
||
locationHints ? `定位提示：${locationHints}` : "",
|
||
"请严格判断图3是否同时满足以下条件：",
|
||
`1. 图3里已经出现了与图1同主体的${productLabel}`,
|
||
`2. 图3中的${productLabel}核心结构、轮廓比例、部件数量、材质颜色和主要设计细节与图1一致`,
|
||
`3. 图2原本需要被替换的${targetLabel}没有被完整保留下来`,
|
||
"4. 图2的大部分场景构图、机位和背景仍被保留，只做主体替换",
|
||
"5. 图3不是图1产品原图、裁切图或仅做轻微背景变化的回传图",
|
||
"注意：为了匹配图2的机位和参考图角度，产品朝向、旋转、透视、大小变化是允许的；不要仅因为角度不同判失败。",
|
||
"若任一条件不满足，就判定为 failed。",
|
||
"仅返回 JSON：",
|
||
'{"passed":true,"score":0.0,"productMatched":true,"productStructurePreserved":true,"angleAcceptable":true,"targetRemoved":true,"scenePreserved":true,"suspectedEcho":false,"reason":"一句话说明","confidence":0.0}',
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
|
||
try {
|
||
const { text, model } = await callQwen3Vl({
|
||
imageUrls: [params.productImageUrl, params.sceneImageUrl, params.generatedImageUrl],
|
||
prompt,
|
||
temperature: 0.1,
|
||
maxTokens: 320,
|
||
timeoutMs: SCENE_FUSION_REVIEW_TIMEOUT_MS,
|
||
retries: 1,
|
||
});
|
||
|
||
const parsed = extractJsonObject(text);
|
||
if (!parsed) {
|
||
return null;
|
||
}
|
||
|
||
const scoreRaw = Number(parsed.score);
|
||
const score = Number.isFinite(scoreRaw) ? Math.max(0, Math.min(1, scoreRaw)) : 0.5;
|
||
const passedFlag = normalizeSceneFusionBoolean(parsed.passed);
|
||
const productMatched = normalizeSceneFusionBoolean(parsed.productMatched);
|
||
const productStructurePreserved = normalizeSceneFusionBoolean(
|
||
parsed.productStructurePreserved ?? parsed.productMatched
|
||
);
|
||
const angleAcceptable = normalizeSceneFusionBoolean(parsed.angleAcceptable);
|
||
const targetRemoved = normalizeSceneFusionBoolean(parsed.targetRemoved);
|
||
const scenePreserved = normalizeSceneFusionBoolean(parsed.scenePreserved);
|
||
const suspectedEcho = normalizeSceneFusionBoolean(parsed.suspectedEcho);
|
||
const reason =
|
||
normalizeSceneFusionHint(parsed.reason, 80) ||
|
||
[
|
||
productMatched === false ? "未识别出参考产品主体" : "",
|
||
productStructurePreserved === false ? "产品核心结构被改变" : "",
|
||
angleAcceptable === false ? "产品角度透视不适合场景" : "",
|
||
targetRemoved === false ? "原目标主体仍明显保留" : "",
|
||
scenePreserved === false ? "原场景结构改动过大" : "",
|
||
suspectedEcho === true ? "结果疑似回传产品图" : "",
|
||
]
|
||
.filter(Boolean)
|
||
.join("；") ||
|
||
"质检结果不明确";
|
||
|
||
const passed =
|
||
passedFlag === true &&
|
||
productMatched !== false &&
|
||
productStructurePreserved !== false &&
|
||
angleAcceptable !== false &&
|
||
targetRemoved !== false &&
|
||
scenePreserved !== false &&
|
||
suspectedEcho !== true &&
|
||
score >= 0.5;
|
||
|
||
logger.info("review_result", {
|
||
requestId: params.requestId,
|
||
sceneIndex: params.sceneIndex + 1,
|
||
candidate: params.modelName,
|
||
passed,
|
||
score,
|
||
reason,
|
||
productStructurePreserved,
|
||
angleAcceptable,
|
||
reviewModel: model,
|
||
});
|
||
|
||
return {
|
||
passed,
|
||
score,
|
||
reason,
|
||
model,
|
||
};
|
||
} catch (error) {
|
||
logger.warn("review_skipped", {
|
||
requestId: params.requestId,
|
||
sceneIndex: params.sceneIndex + 1,
|
||
candidate: params.modelName,
|
||
error: error instanceof Error ? error.message : "unknown",
|
||
});
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function extractStyleHintFromPrompt(customPrompt: string | undefined): string | undefined {
|
||
const content = String(customPrompt || "").trim();
|
||
if (!content) return undefined;
|
||
const quoted = content.match(/[“"]([^”"]+?)(?:风格|场景)/);
|
||
if (quoted?.[1]) {
|
||
return normalizeSceneFusionName(quoted[1]) || quoted[1].trim();
|
||
}
|
||
const styleOnly = content.match(/(?:替换背景为|背景为|风格为)\s*([^，。；]+)/);
|
||
if (styleOnly?.[1]) {
|
||
return normalizeSceneFusionName(styleOnly[1]) || styleOnly[1].trim();
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
function mergeCustomPrompt(basePrompt: string | undefined, addition: string | undefined): string | undefined {
|
||
const base = String(basePrompt || "").trim();
|
||
const extra = String(addition || "").trim();
|
||
if (!base && !extra) return undefined;
|
||
if (!base) return extra.slice(0, 280);
|
||
if (!extra) return base.slice(0, 280);
|
||
if (base.includes(extra)) return base.slice(0, 280);
|
||
if (extra.includes(base)) return extra.slice(0, 280);
|
||
return `${base} ${extra}`.slice(0, 280);
|
||
}
|
||
|
||
async function generateRandomBackgroundPromptWithAgent(params: {
|
||
productImageUrl: string;
|
||
productNameHint?: string;
|
||
styleHint?: string;
|
||
userPrompt?: string;
|
||
}): Promise<string | null> {
|
||
const prompt = [
|
||
"你是电商场景策划助手。",
|
||
`产品名称：${params.productNameHint || "该产品"}`,
|
||
params.styleHint ? `期望风格：${params.styleHint}` : "",
|
||
params.userPrompt ? `用户补充要求：${params.userPrompt}` : "",
|
||
"请基于产品图生成一条“仅描述背景场景”的中文提示词。",
|
||
"要求：",
|
||
"- 场景必须与产品品类匹配，适合商品展示",
|
||
"- 只描述背景、灯光、材质氛围，不改动产品主体外观",
|
||
"- 不要输出流程指令（例如“替换背景/保持主体”）",
|
||
"- 20~60字，语义具体可执行",
|
||
"仅返回 JSON：",
|
||
"{\"backgroundPrompt\":\"...\"}",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
|
||
const { text } = await callQwen3Vl({
|
||
imageUrl: params.productImageUrl,
|
||
prompt,
|
||
temperature: 0.2,
|
||
maxTokens: 240,
|
||
timeoutMs: SCENE_FUSION_AGENT_TIMEOUT_MS,
|
||
retries: 1,
|
||
});
|
||
|
||
const parsed = extractJsonObject(text);
|
||
const backgroundPromptRaw = String(parsed?.backgroundPrompt || "").replace(/\s+/g, " ").trim();
|
||
if (!backgroundPromptRaw || backgroundPromptRaw.length < 6) {
|
||
return null;
|
||
}
|
||
return backgroundPromptRaw.slice(0, 120);
|
||
}
|
||
|
||
async function enrichRandomBackgroundScenesWithAgent(params: {
|
||
requestId: string;
|
||
productImageUrl: string;
|
||
productName?: string;
|
||
scenes: Array<{
|
||
id: string;
|
||
image: string;
|
||
targetName?: string;
|
||
customPrompt?: string;
|
||
imageUrl: string;
|
||
}>;
|
||
}): Promise<Array<{
|
||
id: string;
|
||
image: string;
|
||
targetName?: string;
|
||
customPrompt?: string;
|
||
imageUrl: string;
|
||
}>> {
|
||
const nextScenes = params.scenes.map((scene) => ({ ...scene }));
|
||
const workerTargets = nextScenes;
|
||
if (workerTargets.length === 0) return nextScenes;
|
||
|
||
let cursor = 0;
|
||
const workerCount = Math.max(1, Math.min(SCENE_FUSION_AGENT_MAX_WORKERS, workerTargets.length));
|
||
const workers = Array.from({ length: workerCount }, async () => {
|
||
while (true) {
|
||
const current = cursor;
|
||
cursor += 1;
|
||
if (current >= workerTargets.length) break;
|
||
|
||
const scene = workerTargets[current];
|
||
if (!scene) continue;
|
||
|
||
try {
|
||
if (!normalizeSceneFusionName(scene.targetName) && params.productName) {
|
||
scene.targetName = params.productName;
|
||
}
|
||
|
||
const styleHint = extractStyleHintFromPrompt(scene.customPrompt);
|
||
const generatedPrompt = await generateRandomBackgroundPromptWithAgent({
|
||
productImageUrl: params.productImageUrl,
|
||
productNameHint: params.productName,
|
||
styleHint,
|
||
userPrompt: scene.customPrompt,
|
||
});
|
||
|
||
if (generatedPrompt) {
|
||
scene.customPrompt = mergeCustomPrompt(scene.customPrompt, `场景建议：${generatedPrompt}`);
|
||
}
|
||
} catch (error) {
|
||
logger.warn("random_prompt_enhance_skipped", error, {
|
||
requestId: params.requestId,
|
||
sceneId: scene.id,
|
||
});
|
||
}
|
||
}
|
||
});
|
||
|
||
await Promise.all(workers);
|
||
return nextScenes;
|
||
}
|
||
|
||
async function enrichSceneLabelsWithAgent(params: {
|
||
requestId: string;
|
||
productImageUrl: string;
|
||
productName?: string;
|
||
scenes: Array<{
|
||
id: string;
|
||
image: string;
|
||
targetName?: string;
|
||
customPrompt?: string;
|
||
imageUrl: string;
|
||
}>;
|
||
}): Promise<{
|
||
productName?: string;
|
||
scenes: Array<{
|
||
id: string;
|
||
image: string;
|
||
targetName?: string;
|
||
customPrompt?: string;
|
||
imageUrl: string;
|
||
}>;
|
||
meta: SceneFusionAgentMeta;
|
||
}> {
|
||
const clonedScenes = params.scenes.map((scene) => ({ ...scene }));
|
||
const userProductName = normalizeSceneFusionName(params.productName);
|
||
let resolvedProductName = userProductName;
|
||
|
||
const meta: SceneFusionAgentMeta = {
|
||
productName: userProductName || undefined,
|
||
productSource: userProductName ? "user" : undefined,
|
||
sceneTargets: clonedScenes.map((scene) => ({
|
||
id: scene.id,
|
||
targetName: normalizeSceneFusionName(scene.targetName) || undefined,
|
||
positionHint: undefined,
|
||
replaceCue: undefined,
|
||
source: normalizeSceneFusionName(scene.targetName) ? "user" : undefined,
|
||
})),
|
||
};
|
||
|
||
try {
|
||
const productDetected = await detectProductNameWithAgent(params.productImageUrl);
|
||
if (productDetected?.name) {
|
||
if (!resolvedProductName) {
|
||
resolvedProductName = productDetected.name;
|
||
meta.productSource = "agent";
|
||
} else {
|
||
meta.productSource = "user";
|
||
}
|
||
meta.productName = resolvedProductName || productDetected.name;
|
||
meta.productConfidence = productDetected.confidence;
|
||
meta.productModel = productDetected.model;
|
||
}
|
||
} catch (error) {
|
||
logger.warn("product_detect_skipped", error, { requestId: params.requestId });
|
||
}
|
||
|
||
const unresolvedIndexes = clonedScenes
|
||
.map((scene, index) => ({
|
||
index,
|
||
hasTarget: Boolean(normalizeSceneFusionName(scene.targetName)),
|
||
sameAsProduct: scene.imageUrl === params.productImageUrl,
|
||
}))
|
||
.filter((item) => !item.hasTarget && !item.sameAsProduct)
|
||
.map((item) => item.index);
|
||
|
||
if (unresolvedIndexes.length === 0) {
|
||
return {
|
||
productName: resolvedProductName || undefined,
|
||
scenes: clonedScenes,
|
||
meta,
|
||
};
|
||
}
|
||
|
||
let cursor = 0;
|
||
const workerCount = Math.max(1, Math.min(SCENE_FUSION_AGENT_MAX_WORKERS, unresolvedIndexes.length));
|
||
|
||
const workers = Array.from({ length: workerCount }, async () => {
|
||
while (true) {
|
||
const current = cursor;
|
||
cursor += 1;
|
||
if (current >= unresolvedIndexes.length) break;
|
||
|
||
const sceneIndex = unresolvedIndexes[current]!;
|
||
const scene = clonedScenes[sceneIndex]!;
|
||
try {
|
||
const targetDetected = await detectTargetNameWithAgent({
|
||
productImageUrl: params.productImageUrl,
|
||
sceneImageUrl: scene.imageUrl,
|
||
productNameHint: resolvedProductName || "产品",
|
||
});
|
||
if (!targetDetected?.name) continue;
|
||
scene.targetName = targetDetected.name;
|
||
const metaScene = meta.sceneTargets.find((item) => item.id === scene.id);
|
||
if (metaScene) {
|
||
metaScene.targetName = targetDetected.name;
|
||
metaScene.positionHint = targetDetected.positionHint;
|
||
metaScene.replaceCue = targetDetected.replaceCue;
|
||
metaScene.source = "agent";
|
||
metaScene.confidence = targetDetected.confidence;
|
||
metaScene.model = targetDetected.model;
|
||
}
|
||
} catch (error) {
|
||
logger.warn("scene_target_detect_skipped", error, {
|
||
requestId: params.requestId,
|
||
sceneId: scene.id,
|
||
});
|
||
}
|
||
}
|
||
});
|
||
|
||
await Promise.all(workers);
|
||
|
||
return {
|
||
productName: resolvedProductName || undefined,
|
||
scenes: clonedScenes,
|
||
meta,
|
||
};
|
||
}
|
||
|
||
function buildSceneFusionCandidatePool(
|
||
allModels: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>,
|
||
preferredModelId: string
|
||
): SceneFusionModelCandidate[] {
|
||
const backendConfiguredImageModels = allModels
|
||
.filter((entry) => {
|
||
if (!entry?.model?.isEnabled) return false;
|
||
if (!entry?.model?.supportsImageGen) return false;
|
||
if (entry?.model?.isSubModel) return false;
|
||
if (entry?.model?.excludeFromFailover) return false;
|
||
return true;
|
||
})
|
||
.map((entry) => ({
|
||
model: {
|
||
id: String(entry.model.id),
|
||
modelId: String(entry.model.modelId || ""),
|
||
displayName: String(entry.model.displayName || entry.model.modelId || entry.model.id),
|
||
supportsVision: modelSupportsReferenceImageInput(entry.model, entry.provider),
|
||
imageGenConfig: (entry.model.imageGenConfig || null) as SceneFusionImageGenConfig | null,
|
||
excludeFromFailover: Boolean(entry.model.excludeFromFailover),
|
||
defaultParams: (entry.model.defaultParams as Record<string, any> | null) || null,
|
||
},
|
||
provider: {
|
||
id: String(entry.provider.id || ""),
|
||
name: String(entry.provider.name || ""),
|
||
type: normalizeSceneFusionProviderType(String(entry.provider.type || "")),
|
||
baseUrl: String(entry.provider.baseUrl || ""),
|
||
apiKey: String(entry.provider.apiKey || ""),
|
||
},
|
||
}))
|
||
.filter(
|
||
(candidate) =>
|
||
Boolean(candidate.model.modelId) &&
|
||
Boolean(candidate.provider.type) &&
|
||
Boolean(candidate.provider.apiKey)
|
||
);
|
||
|
||
if (backendConfiguredImageModels.length <= 1) {
|
||
return backendConfiguredImageModels;
|
||
}
|
||
|
||
const prioritySorted = [...backendConfiguredImageModels].sort((left, right) => {
|
||
const scoreDiff =
|
||
getSceneFusionCandidatePriority(right) - getSceneFusionCandidatePriority(left);
|
||
if (scoreDiff !== 0) return scoreDiff;
|
||
return left.model.displayName.localeCompare(right.model.displayName, "zh-Hans-CN");
|
||
});
|
||
|
||
const preferred = preferredModelId
|
||
? prioritySorted.find((candidate) => candidate.model.id === preferredModelId)
|
||
: undefined;
|
||
|
||
if (!preferred) {
|
||
return prioritySorted;
|
||
}
|
||
|
||
const rest = prioritySorted.filter((candidate) => candidate.model.id !== preferred.model.id);
|
||
return [preferred, ...rest];
|
||
}
|
||
|
||
/**
|
||
* 获取场景裂变配置
|
||
*/
|
||
async function getSceneFusionConfig() {
|
||
try {
|
||
const setting = await getSystemSetting({ key: "scene_fusion_config" });
|
||
if (setting?.value) {
|
||
const config = setting.value as {
|
||
enabled?: boolean;
|
||
defaultPrompt?: string;
|
||
maxScenes?: number;
|
||
modelId?: string;
|
||
preRmbgEnabled?: boolean;
|
||
};
|
||
return {
|
||
enabled: config.enabled ?? true,
|
||
defaultPrompt: config.defaultPrompt || FALLBACK_PROMPT,
|
||
maxScenes: Math.min(config.maxScenes ?? SCENE_FUSION_MAX_ITEMS, SCENE_FUSION_MAX_ITEMS),
|
||
modelId: config.modelId || "",
|
||
preRmbgEnabled: config.preRmbgEnabled ?? true,
|
||
};
|
||
}
|
||
} catch (error) {
|
||
logger.error("config_load_failed", error);
|
||
}
|
||
|
||
return {
|
||
enabled: true,
|
||
defaultPrompt: FALLBACK_PROMPT,
|
||
maxScenes: SCENE_FUSION_MAX_ITEMS,
|
||
modelId: "",
|
||
preRmbgEnabled: true,
|
||
};
|
||
}
|
||
|
||
async function preprocessProductImageWithRmbg(
|
||
request: Request,
|
||
requestId: string,
|
||
productImageUrl: string,
|
||
preRmbgEnabled: boolean
|
||
): Promise<{
|
||
productImageUrl: string;
|
||
usedPreRmbg: boolean;
|
||
provider?: string;
|
||
reason?: string;
|
||
}> {
|
||
if (!preRmbgEnabled) {
|
||
return {
|
||
productImageUrl,
|
||
usedPreRmbg: false,
|
||
reason: "disabled_by_config",
|
||
};
|
||
}
|
||
|
||
const cookie = request.headers.get("cookie");
|
||
if (!cookie) {
|
||
return {
|
||
productImageUrl,
|
||
usedPreRmbg: false,
|
||
reason: "missing_cookie",
|
||
};
|
||
}
|
||
|
||
const candidateUrls = buildPreRmbgCandidateUrls(request);
|
||
let lastReason = "no_candidate_url";
|
||
|
||
logger.info("pre_rmbg_started", {
|
||
requestId,
|
||
candidateCount: candidateUrls.length,
|
||
});
|
||
|
||
for (const rmbgUrl of candidateUrls) {
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), SCENE_FUSION_PRE_RMBG_TIMEOUT_MS);
|
||
try {
|
||
const response = await fetch(rmbgUrl, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
cookie,
|
||
},
|
||
body: JSON.stringify({
|
||
imageUrl: productImageUrl,
|
||
prompt: "场景裂变预处理-抠图",
|
||
skipGallerySave: true,
|
||
}),
|
||
signal: controller.signal,
|
||
});
|
||
|
||
const payload = await response.json().catch(() => null) as Record<string, unknown> | null;
|
||
|
||
if (!response.ok) {
|
||
lastReason = `request_failed_${response.status}`;
|
||
logger.warn("pre_rmbg_candidate_failed", {
|
||
requestId,
|
||
status: response.status,
|
||
detail: payload?.detail || payload?.error || "unknown",
|
||
});
|
||
continue;
|
||
}
|
||
|
||
const cleanedUrl = String(payload?.original_url || payload?.thumbnail_url || "").trim();
|
||
if (!cleanedUrl || cleanedUrl === productImageUrl) {
|
||
lastReason = cleanedUrl ? "same_as_source" : "empty_result";
|
||
logger.warn("pre_rmbg_candidate_invalid", {
|
||
requestId,
|
||
reason: lastReason,
|
||
});
|
||
continue;
|
||
}
|
||
|
||
return {
|
||
productImageUrl: cleanedUrl,
|
||
usedPreRmbg: true,
|
||
provider: String(payload?.provider || "unknown"),
|
||
};
|
||
} catch (error) {
|
||
lastReason = error instanceof Error ? error.message : "exception";
|
||
logger.warn("pre_rmbg_candidate_exception", error, { requestId });
|
||
} finally {
|
||
clearTimeout(timeoutId);
|
||
}
|
||
}
|
||
|
||
return {
|
||
productImageUrl,
|
||
usedPreRmbg: false,
|
||
reason: lastReason,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 创建缩略图
|
||
*/
|
||
async function createThumbnail(buffer: Buffer): Promise<Buffer> {
|
||
try {
|
||
const thumbnailBuffer = await sharp(buffer)
|
||
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
|
||
.jpeg({ quality: 80 })
|
||
.toBuffer();
|
||
return thumbnailBuffer;
|
||
} catch (error) {
|
||
logger.error("thumbnail_create_failed", error);
|
||
return buffer;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 生成唯一文件名
|
||
*/
|
||
function generateFilename(prefix: string, extension: string): string {
|
||
const timestamp = Date.now();
|
||
const random = Math.random().toString(36).substring(2, 8);
|
||
return `${prefix}-${timestamp}-${random}.${extension}`;
|
||
}
|
||
|
||
/**
|
||
* 将 Base64 图片上传到 S3 获取 URL
|
||
*/
|
||
async function uploadBase64ToS3(base64Data: string, prefix: string = "scene-fusion"): Promise<string> {
|
||
try {
|
||
// 移除 data URL 前缀
|
||
const base64Clean = base64Data.replace(/^data:image\/\w+;base64,/, "");
|
||
const buffer = Buffer.from(base64Clean, "base64");
|
||
|
||
const filename = generateFilename(`${prefix}-ref`, "png");
|
||
const result = await uploadFile(filename, buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength));
|
||
logger.info("reference_image_uploaded", { prefix, bytes: buffer.byteLength });
|
||
return result.url;
|
||
} catch (error) {
|
||
logger.error("reference_image_upload_failed", error, { prefix });
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 下载图片并上传到 S3
|
||
*/
|
||
async function downloadAndUploadToS3(
|
||
imageUrl: string,
|
||
prefix: string = "scene-fusion"
|
||
): Promise<{ thumbnail: string; original: string }> {
|
||
try {
|
||
logger.debug("download_upload_started", { prefix });
|
||
|
||
const response = await fetch(imageUrl);
|
||
if (!response.ok) {
|
||
throw new Error(`Failed to download image: ${response.status}`);
|
||
}
|
||
|
||
const buffer = Buffer.from(await response.arrayBuffer());
|
||
logger.debug("image_downloaded", { prefix, bytes: buffer.length });
|
||
|
||
// 创建缩略图
|
||
const thumbnailBuffer = await createThumbnail(buffer);
|
||
|
||
// 上传原图
|
||
const originalFilename = generateFilename(`${prefix}-orig`, "png");
|
||
const originalResult = await uploadFile(originalFilename, buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength));
|
||
logger.debug("original_uploaded", { prefix, bytes: buffer.byteLength });
|
||
|
||
// 上传缩略图
|
||
const thumbnailFilename = generateFilename(`${prefix}-thumb`, "jpg");
|
||
const thumbnailArrayBuffer = new Uint8Array(thumbnailBuffer).buffer;
|
||
const thumbnailResult = await uploadFile(thumbnailFilename, thumbnailArrayBuffer);
|
||
logger.debug("thumbnail_uploaded", { prefix, bytes: thumbnailBuffer.byteLength });
|
||
|
||
return {
|
||
original: originalResult.url,
|
||
thumbnail: thumbnailResult.url,
|
||
};
|
||
} catch (error) {
|
||
logger.error("download_upload_failed", error, { prefix });
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
async function saveSceneFusionImageRecord(params: {
|
||
userId: string;
|
||
thumbnailUrl: string;
|
||
originalUrl: string;
|
||
imageSize: string;
|
||
prompt: string;
|
||
modelId: string;
|
||
sourceImageUrls?: string[];
|
||
}): Promise<void> {
|
||
let lastError: unknown = null;
|
||
let savedImage:
|
||
| {
|
||
id: string;
|
||
}
|
||
| null = null;
|
||
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
||
try {
|
||
([savedImage] = await db.insert(generatedImage).values({
|
||
userId: params.userId,
|
||
chatId: null,
|
||
messageId: null,
|
||
thumbnailUrl: params.thumbnailUrl,
|
||
originalUrl: params.originalUrl,
|
||
imageSize: params.imageSize,
|
||
prompt: params.prompt,
|
||
modelId: params.modelId,
|
||
}).returning());
|
||
break;
|
||
} catch (error) {
|
||
lastError = error;
|
||
if (attempt < 3) {
|
||
await waitSceneFusion(120 * attempt);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!savedImage) {
|
||
throw lastError instanceof Error ? lastError : new Error("保存到相册失败");
|
||
}
|
||
|
||
try {
|
||
const sourceAssets =
|
||
Array.isArray(params.sourceImageUrls) && params.sourceImageUrls.length > 0
|
||
? await getMediaAssetsByUserAndUrls({
|
||
userId: params.userId,
|
||
urls: params.sourceImageUrls,
|
||
})
|
||
: [];
|
||
const media = await upsertMediaAsset({
|
||
userId: params.userId,
|
||
mediaType: "image",
|
||
sourceType: "edited",
|
||
status: "ready",
|
||
originalUrl: params.originalUrl,
|
||
previewUrl: params.originalUrl,
|
||
thumbnailUrl: params.thumbnailUrl,
|
||
displayName: params.prompt.slice(0, 80),
|
||
prompt: params.prompt,
|
||
modelId: params.modelId,
|
||
sourceTable: "GeneratedImage",
|
||
sourceRowId: savedImage.id,
|
||
primaryParentAssetId: sourceAssets[0]?.id ?? null,
|
||
metadata: {
|
||
imageSize: params.imageSize,
|
||
sourceImageUrls: params.sourceImageUrls ?? [],
|
||
},
|
||
});
|
||
if (media) {
|
||
for (const sourceAsset of sourceAssets) {
|
||
await createMediaAssetRelation({
|
||
userId: params.userId,
|
||
fromAssetId: sourceAsset.id,
|
||
toAssetId: media.id,
|
||
relationType: "edited_from",
|
||
metadata: {
|
||
prompt: params.prompt,
|
||
},
|
||
});
|
||
}
|
||
}
|
||
} catch (assetError) {
|
||
logger.error("media_asset_register_failed", assetError);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 根据 aspectRatio 和 imageSize 计算实际尺寸
|
||
* @param aspectRatio - 宽高比，如 "1:1", "4:3", "16:9", "auto"
|
||
* @param imageSize - 分辨率等级，如 "1K", "2K", "4K"
|
||
* @returns 尺寸字符串，如 "1024*1024"，或 undefined（auto 模式）
|
||
*/
|
||
function calculateSize(aspectRatio: string, imageSize: string): string | undefined {
|
||
// auto 模式：不设置尺寸，让 API 根据输入图片自动决定
|
||
if (aspectRatio === "auto") {
|
||
return undefined;
|
||
}
|
||
|
||
// 根据分辨率等级确定基准像素
|
||
const baseSizeMap: Record<string, number> = {
|
||
"1K": 1024,
|
||
"2K": 1536,
|
||
"4K": 2048,
|
||
};
|
||
const baseSize = baseSizeMap[imageSize] || 1536;
|
||
if (!/^(\d+(?:\.\d+)?):(\d+(?:\.\d+)?)$/.test(aspectRatio)) {
|
||
return undefined;
|
||
}
|
||
|
||
const { width, height } = resolveLongestEdgeDimensions({
|
||
aspectRatio,
|
||
longestEdge: baseSize,
|
||
constraints: {
|
||
maxEdge: 2048,
|
||
maxPixels: 2048 * 2048,
|
||
minEdge: 512,
|
||
multiple: 2,
|
||
},
|
||
});
|
||
|
||
return `${width}*${height}`;
|
||
}
|
||
|
||
/**
|
||
* 调用阿里生图 API
|
||
*/
|
||
async function callAlibabaImageAPI(
|
||
baseURL: string,
|
||
apiKey: string,
|
||
modelId: string,
|
||
productImageUrl: string,
|
||
sceneImageUrl: string,
|
||
prompt: string,
|
||
size?: string
|
||
): Promise<string> {
|
||
// 构建消息内容：产品图 + 场景图 + 提示词
|
||
const content = [
|
||
{ image: productImageUrl },
|
||
{ image: sceneImageUrl },
|
||
{ text: prompt },
|
||
];
|
||
|
||
const requestBody = {
|
||
model: modelId,
|
||
input: {
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content,
|
||
},
|
||
],
|
||
},
|
||
parameters: {
|
||
n: 1,
|
||
...(size && { size }),
|
||
prompt_extend: true,
|
||
watermark: false,
|
||
},
|
||
};
|
||
|
||
logger.debug("alibaba_request_prepared", {
|
||
modelId,
|
||
size: size || "auto",
|
||
productImageCount: productImageUrl ? 1 : 0,
|
||
sceneImageCount: sceneImageUrl ? 1 : 0,
|
||
promptLength: prompt.length,
|
||
});
|
||
|
||
const response = await fetch(baseURL, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${apiKey}`,
|
||
},
|
||
body: JSON.stringify(requestBody),
|
||
});
|
||
|
||
const data = await response.json();
|
||
|
||
logger.debug("alibaba_response_received", {
|
||
modelId,
|
||
ok: response.ok,
|
||
code: data?.code ?? null,
|
||
hasOutput: Boolean(data?.output),
|
||
});
|
||
|
||
if (!response.ok || data.code) {
|
||
logger.error("alibaba_response_error", {
|
||
modelId,
|
||
status: response.status,
|
||
code: data?.code ?? null,
|
||
message: data?.message ?? null,
|
||
});
|
||
throw new Error(data.message || (data.code ? `Error ${data.code}: ${data.message}` : `API request failed: ${response.status}`));
|
||
}
|
||
|
||
// 提取生成的图片 URL
|
||
const choices = data.output?.choices;
|
||
if (!choices || choices.length === 0) {
|
||
logger.error("alibaba_no_choices", { modelId, hasOutput: Boolean(data?.output) });
|
||
throw new Error("No images in response");
|
||
}
|
||
|
||
const imageContent = choices[0]?.message?.content;
|
||
logger.debug("alibaba_image_content_received", {
|
||
modelId,
|
||
contentCount: Array.isArray(imageContent) ? imageContent.length : 0,
|
||
});
|
||
|
||
if (!imageContent || imageContent.length === 0) {
|
||
throw new Error("No image content in response");
|
||
}
|
||
|
||
// 遍历所有内容项，找到 image 类型的
|
||
let generatedImageUrl: string | undefined;
|
||
for (const item of imageContent) {
|
||
if (item.image) {
|
||
generatedImageUrl = item.image;
|
||
logger.debug("alibaba_image_url_found", { modelId });
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (!generatedImageUrl) {
|
||
logger.error("alibaba_image_url_missing", {
|
||
modelId,
|
||
contentCount: Array.isArray(imageContent) ? imageContent.length : 0,
|
||
});
|
||
throw new Error("No image URL in response");
|
||
}
|
||
|
||
// 检查返回的图片 URL 是否与输入相同（可能是 API 返回了原图）
|
||
if (generatedImageUrl === productImageUrl || generatedImageUrl === sceneImageUrl) {
|
||
logger.warn("alibaba_returned_input_url", { modelId });
|
||
}
|
||
|
||
logger.debug("alibaba_image_generated", { modelId });
|
||
return generatedImageUrl;
|
||
}
|
||
|
||
/**
|
||
* 构建智能提示词
|
||
* @param defaultPrompt - 默认提示词模板
|
||
* @param productName - 产品名称（如"沙发"）
|
||
* @param targetName - 背景中需要被替换的物体名称（如"椅子"）
|
||
* @param customPrompt - 用户补充的关键词
|
||
*/
|
||
function buildPrompt(
|
||
defaultPrompt: string,
|
||
productName?: string,
|
||
targetName?: string,
|
||
positionHint?: string,
|
||
replaceCue?: string,
|
||
customPrompt?: string,
|
||
sceneMode: SceneFusionMode = "scene-fission"
|
||
): string {
|
||
let prompt = defaultPrompt;
|
||
|
||
if (sceneMode === "random-bg") {
|
||
const subject = productName || "产品";
|
||
prompt = `请以第一张图片中的${subject}为唯一商品主体，对第二张图片进行背景重绘。要求：1)只保留${subject}，忽略第一张图中的杂志、杯子、蜡烛、摆件、文字、Logo、包装、背景和装饰附件；2)必须保持${subject}核心结构、轮廓比例、部件数量、材质颜色和主要设计细节与第一张图一致，不要改造成另一款产品；3)允许调整主体位置、大小、角度、朝向和透视以匹配新背景，不要把角度变化当成结构改变；4)背景需要完整重建并与原图明显不同；5)光影透视自然，输出商业摄影质感。`;
|
||
} else {
|
||
const subject = productName || "产品";
|
||
const targetSubject = targetName || "场景主主体";
|
||
const locationHints = [positionHint, replaceCue].filter(Boolean).join("，");
|
||
const subjectLock =
|
||
`主产品锁定：第一张图中只有${subject}是要替换进场景的商品主体，必须忽略杂志、杯子、蜡烛、摆件、文字、Logo、包装、背景和其他装饰附件，不要把这些附件复制到结果图。`;
|
||
if (targetName) {
|
||
prompt = `${subjectLock} 请参考第一张图片中的${subject}，仅替换第二张图片中的${targetSubject}，不要改成全新场景。要求：1)必须保持${subject}核心结构、轮廓比例、部件数量、材质颜色和主要设计细节与第一张图一致，不要增删关键部件或改造成另一款产品；2)先完整移除第二张图中原有的${targetSubject}，再将${subject}放入其原位置，禁止在原${targetSubject}表面直接叠加；3)允许调整${subject}的大小、角度、朝向、光影和透视，使其与第二张图自然融合，角度变化本身是允许的；4)保留第二张图的主要背景、机位、透视线、空间关系和光源方向；5)结果中不得同时完整保留原${targetSubject}和新的${subject}。`;
|
||
} else {
|
||
prompt = `${subjectLock} 请参考第一张图片中的${subject}，将其自然地融合到第二张图片的场景主主体位置。要求：1)必须保持${subject}核心结构、轮廓比例、部件数量、材质颜色和主要设计细节与第一张图一致，不要改造成另一款产品；2)优先替换第二张图中最主要、用途和体量最接近的同类完整主体，不要只做风格微调；3)先移除原主体再放入${subject}，禁止叠加原物体；4)允许为了匹配场景调整${subject}角度、朝向、大小和透视，角度变化不代表结构变化；5)保留第二张图主要背景、机位、透视线、空间关系和光源方向，注意光影和透视关系使融合自然。`;
|
||
}
|
||
if (locationHints) {
|
||
prompt = `${prompt} 待替换目标定位：${locationHints}。`;
|
||
}
|
||
}
|
||
|
||
// 添加用户补充的关键词
|
||
if (customPrompt) {
|
||
prompt = `${prompt} 补充要求：${customPrompt}`;
|
||
}
|
||
|
||
if (sceneMode === "random-bg") {
|
||
if (!prompt.includes("背景必须与原图明显不同")) {
|
||
prompt = `${prompt} ${SCENE_FUSION_RANDOM_BG_PROMPT_TAIL}`;
|
||
}
|
||
} else if (!prompt.includes("严禁直接返回第一张参考图")) {
|
||
prompt = `${prompt} ${SCENE_FUSION_STRICT_PROMPT_TAIL}`;
|
||
}
|
||
|
||
return prompt;
|
||
}
|
||
|
||
/**
|
||
* 通用场景处理函数 - 根据 provider 类型调用不同的实现
|
||
*/
|
||
async function processSceneWithProvider(
|
||
productImageUrl: string,
|
||
sceneImageUrl: string,
|
||
productName: string | undefined,
|
||
targetName: string | undefined,
|
||
positionHint: string | undefined,
|
||
replaceCue: string | undefined,
|
||
customPrompt: string | undefined,
|
||
defaultPrompt: string,
|
||
sceneMode: SceneFusionMode,
|
||
provider: { type: string; baseUrl: string; apiKey: string },
|
||
modelId: string,
|
||
runtimeModelId: string,
|
||
aspectRatio?: string,
|
||
imageSize?: string,
|
||
imageGenConfig?: Record<string, any> | null
|
||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||
// 构建智能提示词
|
||
const prompt = buildPrompt(
|
||
defaultPrompt,
|
||
productName,
|
||
targetName,
|
||
positionHint,
|
||
replaceCue,
|
||
customPrompt,
|
||
sceneMode
|
||
);
|
||
|
||
logger.debug("provider_prompt_built", {
|
||
providerType: provider.type,
|
||
promptLength: prompt.length,
|
||
sceneMode,
|
||
});
|
||
|
||
// 根据 provider 类型选择不同的处理逻辑
|
||
switch (provider.type) {
|
||
case "alibaba-image":
|
||
return processWithAlibabaImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
|
||
|
||
case "ruxa-image":
|
||
return processWithRuxaImage(
|
||
productImageUrl,
|
||
sceneImageUrl,
|
||
prompt,
|
||
provider,
|
||
runtimeModelId || modelId,
|
||
aspectRatio,
|
||
imageSize,
|
||
);
|
||
|
||
case "fal-image":
|
||
return processWithFalImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
|
||
|
||
case "snapgen-image":
|
||
return processWithSnapgenImage(
|
||
productImageUrl,
|
||
sceneImageUrl,
|
||
prompt,
|
||
provider,
|
||
runtimeModelId || modelId,
|
||
aspectRatio,
|
||
imageSize,
|
||
);
|
||
|
||
case "openai-image":
|
||
return processWithOpenAIImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
|
||
|
||
case "bananaimage":
|
||
return processWithBananaImage(
|
||
productImageUrl,
|
||
sceneImageUrl,
|
||
prompt,
|
||
provider,
|
||
runtimeModelId || modelId,
|
||
aspectRatio,
|
||
imageSize,
|
||
);
|
||
|
||
case "visionary":
|
||
return processWithVisionary(
|
||
productImageUrl,
|
||
sceneImageUrl,
|
||
prompt,
|
||
provider,
|
||
runtimeModelId || modelId,
|
||
aspectRatio,
|
||
imageSize,
|
||
);
|
||
|
||
case "atlascloud":
|
||
return processWithAtlasCloud(
|
||
productImageUrl,
|
||
sceneImageUrl,
|
||
prompt,
|
||
provider,
|
||
runtimeModelId || modelId,
|
||
aspectRatio,
|
||
imageSize,
|
||
imageGenConfig,
|
||
);
|
||
|
||
case "vip98k":
|
||
return processWithVip98k(
|
||
productImageUrl,
|
||
sceneImageUrl,
|
||
prompt,
|
||
provider,
|
||
runtimeModelId || modelId,
|
||
aspectRatio,
|
||
imageSize,
|
||
imageGenConfig,
|
||
);
|
||
|
||
case "gemini":
|
||
case "gemini-image":
|
||
return processWithGemini(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
|
||
|
||
case "volcano-vision":
|
||
return processWithVolcanoVision(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
|
||
|
||
default:
|
||
throw new Error(`不支持的模型类型: ${provider.type}`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 使用阿里生图模型处理
|
||
*/
|
||
async function processWithAlibabaImage(
|
||
productImageUrl: string,
|
||
sceneImageUrl: string,
|
||
prompt: string,
|
||
provider: { baseUrl: string; apiKey: string },
|
||
modelId: string,
|
||
aspectRatio?: string,
|
||
imageSize?: string
|
||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||
// 计算尺寸
|
||
const size = calculateSize(aspectRatio || "auto", imageSize || "2K");
|
||
|
||
// 调用阿里生图 API
|
||
const generatedImageUrl = await callAlibabaImageAPI(
|
||
provider.baseUrl,
|
||
provider.apiKey,
|
||
modelId,
|
||
productImageUrl,
|
||
sceneImageUrl,
|
||
prompt,
|
||
size
|
||
);
|
||
|
||
// 下载并上传到 S3
|
||
const uploaded = await downloadAndUploadToS3(generatedImageUrl, "scene-fusion-alibaba");
|
||
|
||
return { ...uploaded, prompt };
|
||
}
|
||
|
||
/**
|
||
* 使用 Gemini 模型处理
|
||
*/
|
||
async function processWithGemini(
|
||
productImageUrl: string,
|
||
sceneImageUrl: string,
|
||
prompt: string,
|
||
provider: { baseUrl: string; apiKey: string },
|
||
modelId: string,
|
||
aspectRatio?: string,
|
||
imageSize?: string
|
||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||
logger.info("provider_processing", { providerType: "gemini", modelId });
|
||
|
||
// 创建 Gemini provider
|
||
const geminiProvider = createGeminiImage({
|
||
baseURL: provider.baseUrl,
|
||
apiKey: provider.apiKey,
|
||
});
|
||
|
||
// 构建消息格式（Gemini 使用 OpenAI 兼容格式）
|
||
const messages = [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{ type: "image_url", image_url: { url: productImageUrl } },
|
||
{ type: "image_url", image_url: { url: sceneImageUrl } },
|
||
{ type: "text", text: prompt },
|
||
],
|
||
},
|
||
];
|
||
|
||
// 调用 Gemini API
|
||
const result = await geminiProvider.chat(modelId).doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
aspectRatio: aspectRatio || "auto",
|
||
imageSize: imageSize || "2K",
|
||
imageCount: 1,
|
||
},
|
||
});
|
||
|
||
// 解析返回的 Markdown 格式图片
|
||
const { thumbnail, original } = parseImageFromMarkdown(result.text);
|
||
|
||
return { thumbnail, original, prompt };
|
||
}
|
||
|
||
/**
|
||
* 使用 RUXA 模型处理
|
||
*/
|
||
async function processWithRuxaImage(
|
||
productImageUrl: string,
|
||
sceneImageUrl: string,
|
||
prompt: string,
|
||
provider: { baseUrl: string; apiKey: string },
|
||
modelId: string,
|
||
aspectRatio?: string,
|
||
imageSize?: string
|
||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||
logger.info("provider_processing", { providerType: "ruxa-image", modelId });
|
||
|
||
const ruxaProvider = createRuxaImage({
|
||
baseURL: provider.baseUrl,
|
||
apiKey: provider.apiKey,
|
||
});
|
||
|
||
const messages = [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{ type: "image_url", image_url: { url: productImageUrl } },
|
||
{ type: "image_url", image_url: { url: sceneImageUrl } },
|
||
{ type: "text", text: prompt },
|
||
],
|
||
},
|
||
];
|
||
|
||
const result = await ruxaProvider.chat(modelId).doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
aspectRatio: aspectRatio || "auto",
|
||
imageSize: imageSize || "2K",
|
||
imageCount: 1,
|
||
},
|
||
});
|
||
|
||
const { thumbnail, original } = parseImageFromMarkdown(result.text);
|
||
return { thumbnail, original, prompt };
|
||
}
|
||
|
||
async function processWithFalImage(
|
||
productImageUrl: string,
|
||
sceneImageUrl: string,
|
||
prompt: string,
|
||
provider: { baseUrl: string; apiKey: string },
|
||
modelId: string,
|
||
aspectRatio?: string,
|
||
imageSize?: string
|
||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||
logger.info("provider_processing", { providerType: "fal-image", modelId });
|
||
|
||
const falProvider = createFalImage({
|
||
baseURL: provider.baseUrl,
|
||
apiKey: provider.apiKey,
|
||
});
|
||
|
||
const messages = [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{ type: "image_url", image_url: { url: productImageUrl } },
|
||
{ type: "image_url", image_url: { url: sceneImageUrl } },
|
||
{ type: "text", text: prompt },
|
||
],
|
||
},
|
||
];
|
||
|
||
const result = await falProvider.chat(modelId).doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
aspectRatio: aspectRatio || "auto",
|
||
imageSize: imageSize || "2K",
|
||
imageCount: 1,
|
||
},
|
||
});
|
||
|
||
const { thumbnail, original } = parseImageFromMarkdown(result.text);
|
||
return { thumbnail, original, prompt };
|
||
}
|
||
|
||
async function processWithSnapgenImage(
|
||
productImageUrl: string,
|
||
sceneImageUrl: string,
|
||
prompt: string,
|
||
provider: { baseUrl: string; apiKey: string },
|
||
modelId: string,
|
||
aspectRatio?: string,
|
||
imageSize?: string
|
||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||
logger.info("provider_processing", { providerType: "snapgen-image", modelId });
|
||
|
||
const snapgenProvider = createSnapgenImage({
|
||
baseURL: provider.baseUrl,
|
||
apiKey: provider.apiKey,
|
||
});
|
||
|
||
const messages = [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{ type: "image_url", image_url: { url: productImageUrl } },
|
||
{ type: "image_url", image_url: { url: sceneImageUrl } },
|
||
{ type: "text", text: prompt },
|
||
],
|
||
},
|
||
];
|
||
|
||
const result = await snapgenProvider.chat(modelId).doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
aspectRatio: aspectRatio || "1:1",
|
||
imageSize: imageSize || "2K",
|
||
imageCount: 1,
|
||
},
|
||
});
|
||
|
||
const { thumbnail, original } = parseImageFromMarkdown(result.text);
|
||
return { thumbnail, original, prompt };
|
||
}
|
||
|
||
async function processWithOpenAIImage(
|
||
productImageUrl: string,
|
||
sceneImageUrl: string,
|
||
prompt: string,
|
||
provider: { baseUrl: string; apiKey: string },
|
||
modelId: string,
|
||
aspectRatio?: string,
|
||
imageSize?: string
|
||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||
logger.info("provider_processing", { providerType: "openai-image", modelId });
|
||
|
||
const openAIImageProvider = createOpenAIImage({
|
||
baseURL: provider.baseUrl,
|
||
apiKey: provider.apiKey,
|
||
});
|
||
|
||
const messages = [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{ type: "image_url", image_url: { url: productImageUrl } },
|
||
{ type: "image_url", image_url: { url: sceneImageUrl } },
|
||
{ type: "text", text: prompt },
|
||
],
|
||
},
|
||
];
|
||
|
||
const result = await openAIImageProvider.chat(modelId).doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
aspectRatio: aspectRatio || "auto",
|
||
imageSize: imageSize || "2K",
|
||
imageCount: 1,
|
||
},
|
||
});
|
||
|
||
const { thumbnail, original } = parseImageFromMarkdown(result.text);
|
||
return { thumbnail, original, prompt };
|
||
}
|
||
|
||
async function processWithBananaImage(
|
||
productImageUrl: string,
|
||
sceneImageUrl: string,
|
||
prompt: string,
|
||
provider: { baseUrl: string; apiKey: string },
|
||
modelId: string,
|
||
aspectRatio?: string,
|
||
imageSize?: string
|
||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||
logger.info("provider_processing", { providerType: "bananaimage", modelId });
|
||
|
||
const bananaImageProvider = createBananaImage({
|
||
baseURL: provider.baseUrl,
|
||
apiKey: provider.apiKey,
|
||
});
|
||
const result = await bananaImageProvider.chat(modelId).doGenerate({
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{ type: "image_url", image_url: { url: productImageUrl } },
|
||
{ type: "image_url", image_url: { url: sceneImageUrl } },
|
||
{ type: "text", text: prompt },
|
||
],
|
||
},
|
||
],
|
||
imageParams: {
|
||
aspectRatio: aspectRatio || "auto",
|
||
imageSize: imageSize || "2K",
|
||
imageCount: 1,
|
||
},
|
||
});
|
||
|
||
const { thumbnail, original } = parseImageFromMarkdown(result.text);
|
||
return { thumbnail, original, prompt };
|
||
}
|
||
|
||
async function processWithVisionary(
|
||
productImageUrl: string,
|
||
sceneImageUrl: string,
|
||
prompt: string,
|
||
provider: { baseUrl: string; apiKey: string },
|
||
modelId: string,
|
||
aspectRatio?: string,
|
||
imageSize?: string
|
||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||
logger.info("provider_processing", { providerType: "visionary", modelId });
|
||
|
||
const visionaryProvider = createVisionary({
|
||
baseURL: provider.baseUrl,
|
||
apiKey: provider.apiKey,
|
||
});
|
||
const result = await visionaryProvider.chat(modelId).doGenerate({
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{ type: "image_url", image_url: { url: productImageUrl } },
|
||
{ type: "image_url", image_url: { url: sceneImageUrl } },
|
||
{ type: "text", text: prompt },
|
||
],
|
||
},
|
||
],
|
||
imageParams: {
|
||
aspectRatio: aspectRatio || "auto",
|
||
imageSize: imageSize || "2K",
|
||
imageCount: 1,
|
||
},
|
||
});
|
||
|
||
const { thumbnail, original } = parseImageFromMarkdown(result.text);
|
||
return { thumbnail, original, prompt };
|
||
}
|
||
|
||
async function processWithAtlasCloud(
|
||
productImageUrl: string,
|
||
sceneImageUrl: string,
|
||
prompt: string,
|
||
provider: { baseUrl: string; apiKey: string },
|
||
modelId: string,
|
||
aspectRatio?: string,
|
||
imageSize?: string,
|
||
imageGenConfig?: Record<string, any> | null
|
||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||
logger.info("provider_processing", { providerType: "atlascloud", modelId });
|
||
|
||
const atlasCloudProvider = createAtlasCloud({
|
||
baseURL: provider.baseUrl,
|
||
apiKey: provider.apiKey,
|
||
});
|
||
const result = await atlasCloudProvider.chat(modelId).doGenerate({
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{ type: "image_url", image_url: { url: productImageUrl } },
|
||
{ type: "image_url", image_url: { url: sceneImageUrl } },
|
||
{ type: "text", text: prompt },
|
||
],
|
||
},
|
||
],
|
||
imageParams: {
|
||
aspectRatio: aspectRatio || "auto",
|
||
imageSize: imageSize || "2K",
|
||
imageCount: 1,
|
||
},
|
||
imageGenConfig,
|
||
});
|
||
|
||
const { thumbnail, original } = parseImageFromMarkdown(result.text);
|
||
return { thumbnail, original, prompt };
|
||
}
|
||
|
||
async function processWithVip98k(
|
||
productImageUrl: string,
|
||
sceneImageUrl: string,
|
||
prompt: string,
|
||
provider: { baseUrl: string; apiKey: string },
|
||
modelId: string,
|
||
aspectRatio?: string,
|
||
imageSize?: string,
|
||
imageGenConfig?: Record<string, any> | null
|
||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||
logger.info("provider_processing", { providerType: "vip98k", modelId });
|
||
|
||
const vip98kProvider = createVip98k({
|
||
baseURL: provider.baseUrl,
|
||
apiKey: provider.apiKey,
|
||
});
|
||
const result = await vip98kProvider.chat(modelId, imageGenConfig || null).doGenerate({
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{ type: "image_url", image_url: { url: productImageUrl } },
|
||
{ type: "image_url", image_url: { url: sceneImageUrl } },
|
||
{ type: "text", text: prompt },
|
||
],
|
||
},
|
||
],
|
||
imageParams: {
|
||
aspectRatio: aspectRatio || "auto",
|
||
imageSize: imageSize || "2K",
|
||
imageCount: 1,
|
||
},
|
||
imageGenConfig,
|
||
});
|
||
|
||
const { thumbnail, original } = parseImageFromMarkdown(result.text);
|
||
return { thumbnail, original, prompt };
|
||
}
|
||
|
||
/**
|
||
* 使用既梦/火山模型处理
|
||
*/
|
||
async function processWithVolcanoVision(
|
||
productImageUrl: string,
|
||
sceneImageUrl: string,
|
||
prompt: string,
|
||
provider: { baseUrl: string; apiKey: string },
|
||
modelId: string,
|
||
aspectRatio?: string,
|
||
imageSize?: string
|
||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||
logger.info("provider_processing", { providerType: "volcano-vision", modelId });
|
||
|
||
// 创建 Volcano Vision provider
|
||
const volcanoProvider = createVolcanoVision({
|
||
baseURL: provider.baseUrl,
|
||
apiKey: provider.apiKey,
|
||
});
|
||
|
||
// 构建消息格式
|
||
const messages = [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{ type: "image_url", image_url: { url: productImageUrl } },
|
||
{ type: "image_url", image_url: { url: sceneImageUrl } },
|
||
{ type: "text", text: prompt },
|
||
],
|
||
},
|
||
];
|
||
|
||
// 调用 Volcano Vision API
|
||
const result = await volcanoProvider.chat(modelId).doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
aspectRatio: aspectRatio || "auto",
|
||
imageSize: imageSize || "2K",
|
||
imageCount: 1,
|
||
},
|
||
});
|
||
|
||
// 解析返回的 Markdown 格式图片
|
||
const { thumbnail, original } = parseImageFromMarkdown(result.text);
|
||
|
||
return { thumbnail, original, prompt };
|
||
}
|
||
|
||
/**
|
||
* 从 Markdown 格式解析图片 URL
|
||
* 格式: 
|
||
*/
|
||
function parseImageFromMarkdown(markdown: string): { thumbnail: string; original: string } {
|
||
logger.debug("markdown_parse_started", { markdownLength: markdown.length });
|
||
|
||
// 匹配  格式
|
||
const match = markdown.match(/!\[thumb\|\|\|([^|]+)\|\|\|[^\]]*\]\(([^)]+)\)/);
|
||
if (match) {
|
||
return {
|
||
original: match[1],
|
||
thumbnail: match[2],
|
||
};
|
||
}
|
||
|
||
// 尝试匹配普通图片格式 
|
||
const simpleMatch = markdown.match(/!\[[^\]]*\]\(([^)]+)\)/);
|
||
if (simpleMatch) {
|
||
return {
|
||
original: simpleMatch[1],
|
||
thumbnail: simpleMatch[1],
|
||
};
|
||
}
|
||
|
||
throw new Error("无法从响应中解析图片 URL");
|
||
}
|
||
|
||
/**
|
||
* 处理单个场景 (保留旧函数用于兼容)
|
||
*/
|
||
async function processScene(
|
||
productImageUrl: string,
|
||
sceneImageUrl: string,
|
||
productName: string | undefined,
|
||
targetName: string | undefined,
|
||
positionHint: string | undefined,
|
||
replaceCue: string | undefined,
|
||
customPrompt: string | undefined,
|
||
defaultPrompt: string,
|
||
sceneMode: SceneFusionMode,
|
||
baseURL: string,
|
||
apiKey: string,
|
||
modelId: string,
|
||
size?: string
|
||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||
// 构建智能提示词
|
||
const prompt = buildPrompt(
|
||
defaultPrompt,
|
||
productName,
|
||
targetName,
|
||
positionHint,
|
||
replaceCue,
|
||
customPrompt,
|
||
sceneMode
|
||
);
|
||
|
||
logger.debug("legacy_prompt_built", { promptLength: prompt.length, sceneMode });
|
||
|
||
// 调用阿里生图 API
|
||
const generatedImageUrl = await callAlibabaImageAPI(
|
||
baseURL,
|
||
apiKey,
|
||
modelId,
|
||
productImageUrl,
|
||
sceneImageUrl,
|
||
prompt,
|
||
size
|
||
);
|
||
|
||
// 下载并上传到 S3
|
||
const uploaded = await downloadAndUploadToS3(generatedImageUrl, "scene-fusion");
|
||
|
||
return { ...uploaded, prompt };
|
||
}
|
||
|
||
async function processSceneWithDiversity(
|
||
params: {
|
||
productImageUrl: string;
|
||
sceneImageUrl: string;
|
||
productName?: string;
|
||
targetName?: string;
|
||
positionHint?: string;
|
||
replaceCue?: string;
|
||
customPrompt?: string;
|
||
defaultPrompt: string;
|
||
sceneMode: SceneFusionMode;
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
sceneIndex: number;
|
||
requestOffset: number;
|
||
requestId: string;
|
||
ownerKey?: string;
|
||
candidates: SceneFusionModelCandidate[];
|
||
imageHashCache: Map<string, string>;
|
||
modelCooldown: SceneFusionModelCooldown;
|
||
onQueueStatus?: (queueStatus: {
|
||
queuePosition: number;
|
||
sameModelPendingCount: number;
|
||
sameModelRunningCount: number;
|
||
concurrencyLimit: number;
|
||
dispatchIntervalMs: number;
|
||
estimatedWaitMs: number;
|
||
source: "model_queue";
|
||
}) => void | Promise<void>;
|
||
onProcessingStart?: () => void | Promise<void>;
|
||
}
|
||
): Promise<{
|
||
result: { thumbnail: string; original: string; prompt: string };
|
||
candidate: SceneFusionModelCandidate;
|
||
attempts: number;
|
||
}> {
|
||
const supportedCandidates = params.candidates.filter((candidate) =>
|
||
canProcessSceneFusionProviderType(candidate.provider.type)
|
||
);
|
||
if (supportedCandidates.length === 0) {
|
||
throw new Error("后台已配置生图模型中暂无可用于场景裂变的供应商，请检查模型供应商类型");
|
||
}
|
||
|
||
const rotatedCandidates = rotateCandidates(
|
||
supportedCandidates,
|
||
params.requestOffset + params.sceneIndex
|
||
);
|
||
pruneSceneFusionCooldown(params.modelCooldown);
|
||
|
||
const healthyCandidates = rotatedCandidates.filter(
|
||
(candidate) => !isSceneFusionCandidateCoolingDown(candidate, params.modelCooldown)
|
||
);
|
||
const orderedHealthyCandidates = [...healthyCandidates].sort((left, right) => {
|
||
const scoreDiff =
|
||
getSceneFusionCandidatePriority(right) - getSceneFusionCandidatePriority(left);
|
||
if (scoreDiff !== 0) return scoreDiff;
|
||
return 0;
|
||
});
|
||
const orderedFallbackCandidates = [...rotatedCandidates].sort((left, right) => {
|
||
const scoreDiff =
|
||
getSceneFusionCandidatePriority(right) - getSceneFusionCandidatePriority(left);
|
||
if (scoreDiff !== 0) return scoreDiff;
|
||
return 0;
|
||
});
|
||
const attemptCandidates =
|
||
orderedHealthyCandidates.length > 0 ? orderedHealthyCandidates : orderedFallbackCandidates;
|
||
|
||
logger.debug("candidate_order_resolved", {
|
||
requestId: params.requestId,
|
||
sceneIndex: params.sceneIndex + 1,
|
||
candidates: attemptCandidates.map((candidate) => ({
|
||
modelId: candidate.model.modelId,
|
||
providerType: candidate.provider.type,
|
||
score: getSceneFusionCandidatePriority(candidate),
|
||
})),
|
||
});
|
||
|
||
const failureReasons: string[] = [];
|
||
|
||
for (let attemptIndex = 0; attemptIndex < attemptCandidates.length; attemptIndex += 1) {
|
||
const candidate = attemptCandidates[attemptIndex]!;
|
||
try {
|
||
const normalizedOutputParams = normalizeSceneFusionOutputParamsForCandidate(candidate, {
|
||
aspectRatio: params.aspectRatio,
|
||
imageSize: params.imageSize,
|
||
});
|
||
if (normalizedOutputParams.changed) {
|
||
logger.info("candidate_output_params_normalized", {
|
||
requestId: params.requestId,
|
||
sceneIndex: params.sceneIndex + 1,
|
||
modelId: candidate.model.modelId,
|
||
changes: normalizedOutputParams.changes,
|
||
});
|
||
}
|
||
|
||
const modelSlot = await acquireCanvasImageModelSlot({
|
||
modelKey: candidate.model.modelId || candidate.model.id,
|
||
ownerKey: params.ownerKey,
|
||
debugLabel: "scene-fusion",
|
||
debugMeta: {
|
||
requestId: params.requestId,
|
||
sceneIndex: params.sceneIndex,
|
||
modelId: candidate.model.modelId,
|
||
modelRef: candidate.model.id,
|
||
providerType: candidate.provider.type,
|
||
},
|
||
onQueued: params.onQueueStatus,
|
||
});
|
||
let result: Awaited<ReturnType<typeof processSceneWithProvider>>;
|
||
const runtimeModelId = resolveModelRuntimeId(candidate.model, {
|
||
providerType: candidate.provider.type,
|
||
});
|
||
await params.onProcessingStart?.();
|
||
let execution: ReturnType<typeof processSceneWithProvider> | null = null;
|
||
try {
|
||
execution = processSceneWithProvider(
|
||
params.productImageUrl,
|
||
params.sceneImageUrl,
|
||
params.productName,
|
||
params.targetName,
|
||
params.positionHint,
|
||
params.replaceCue,
|
||
params.customPrompt,
|
||
params.defaultPrompt,
|
||
params.sceneMode,
|
||
{
|
||
type: candidate.provider.type,
|
||
baseUrl: candidate.provider.baseUrl,
|
||
apiKey: candidate.provider.apiKey,
|
||
},
|
||
candidate.model.modelId,
|
||
runtimeModelId,
|
||
normalizedOutputParams.aspectRatio,
|
||
normalizedOutputParams.imageSize,
|
||
(candidate.model.imageGenConfig as Record<string, any> | null) || null
|
||
);
|
||
} finally {
|
||
await modelSlot.release();
|
||
}
|
||
if (!execution) {
|
||
throw new Error("场景裂变请求未能提交到模型服务");
|
||
}
|
||
result = await execution;
|
||
|
||
const echoCheck = await detectEchoedProductResult({
|
||
requestId: params.requestId,
|
||
sceneIndex: params.sceneIndex,
|
||
productImageUrl: params.productImageUrl,
|
||
generatedImageUrl: result.original,
|
||
modelName: candidate.model.displayName,
|
||
hashCache: params.imageHashCache,
|
||
});
|
||
if (echoCheck.tooSimilar) {
|
||
const detail =
|
||
typeof echoCheck.hammingDistance === "number"
|
||
? `hamming=${echoCheck.hammingDistance}`
|
||
: echoCheck.reason || "unknown";
|
||
throw new Error(`生成结果疑似回传产品图（${detail}）`);
|
||
}
|
||
|
||
const reviewResult = await reviewSceneFusionResult({
|
||
requestId: params.requestId,
|
||
sceneIndex: params.sceneIndex,
|
||
sceneMode: params.sceneMode,
|
||
productImageUrl: params.productImageUrl,
|
||
sceneImageUrl: params.sceneImageUrl,
|
||
generatedImageUrl: result.original,
|
||
productName: params.productName,
|
||
targetName: params.targetName,
|
||
positionHint: params.positionHint,
|
||
replaceCue: params.replaceCue,
|
||
modelName: candidate.model.displayName,
|
||
});
|
||
if (reviewResult && !reviewResult.passed) {
|
||
logger.warn("candidate_review_warning", {
|
||
requestId: params.requestId,
|
||
sceneIndex: params.sceneIndex,
|
||
attempt: attemptIndex + 1,
|
||
model: candidate.model.modelId,
|
||
provider: candidate.provider.type,
|
||
score: Number(reviewResult.score.toFixed(2)),
|
||
reason: reviewResult.reason,
|
||
});
|
||
}
|
||
|
||
markSceneFusionCandidateHealthy(candidate, params.modelCooldown);
|
||
return {
|
||
result,
|
||
candidate,
|
||
attempts: attemptIndex + 1,
|
||
};
|
||
} catch (error) {
|
||
const errorMessage = error instanceof Error ? error.message : "未知错误";
|
||
failureReasons.push(`${candidate.model.displayName}: ${errorMessage}`);
|
||
markSceneFusionCandidateFailed(candidate, params.modelCooldown);
|
||
// Batch processing can fail because of per-image inputs, review gates, or echo checks.
|
||
// Keep these failures local to this request so they do not auto-disable shared models.
|
||
const failureMeta = {
|
||
requestId: params.requestId,
|
||
sceneIndex: params.sceneIndex,
|
||
attempt: attemptIndex + 1,
|
||
model: candidate.model.modelId,
|
||
provider: candidate.provider.type,
|
||
error: errorMessage,
|
||
};
|
||
if (isSceneFusionReviewRejectError(errorMessage)) {
|
||
logger.warn("candidate_review_rejected", failureMeta);
|
||
} else {
|
||
logger.error("candidate_failed", failureMeta);
|
||
}
|
||
}
|
||
}
|
||
|
||
const reason = failureReasons.join("；");
|
||
throw new Error(
|
||
reason
|
||
? `多样性模式下所有可用模型均失败：${reason}`
|
||
: "多样性模式下所有可用模型均失败"
|
||
);
|
||
}
|
||
|
||
export async function POST(request: Request) {
|
||
try {
|
||
// 1. 认证
|
||
const session = await auth();
|
||
if (!session?.user?.id) {
|
||
return NextResponse.json(
|
||
{ error: "未登录" },
|
||
{ status: 401 }
|
||
);
|
||
}
|
||
|
||
// 2. 获取场景裂变配置
|
||
const config = await getSceneFusionConfig();
|
||
|
||
if (!config.enabled) {
|
||
return NextResponse.json(
|
||
{ error: "场景裂变功能已被管理员禁用" },
|
||
{ status: 403 }
|
||
);
|
||
}
|
||
|
||
// 3. 解析请求体
|
||
const json = await request.json();
|
||
const parseResult = sceneFusionRequestSchema.safeParse(json);
|
||
|
||
if (!parseResult.success) {
|
||
return NextResponse.json(
|
||
{ error: parseResult.error.errors[0].message },
|
||
{ status: 400 }
|
||
);
|
||
}
|
||
|
||
const { productImage, productName, sceneMode, aspectRatio, imageSize, scenes } = parseResult.data;
|
||
const requestId = `sf-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||
const resolvedSceneMode = sceneMode || "scene-fission";
|
||
|
||
// 计算输出尺寸
|
||
const size = calculateSize(aspectRatio || "auto", imageSize || "2K");
|
||
logger.info("request_received", {
|
||
requestId,
|
||
aspectRatio: aspectRatio || "auto",
|
||
imageSize: imageSize || "2K",
|
||
size: size || "auto",
|
||
sceneCount: scenes.length,
|
||
sceneMode: resolvedSceneMode,
|
||
hasProductName: Boolean(productName),
|
||
preferredModelId: config.modelId || null,
|
||
});
|
||
|
||
// 检查场景数量限制
|
||
const effectiveMaxScenes = Math.min(config.maxScenes, SCENE_FUSION_MAX_ITEMS);
|
||
if (scenes.length > effectiveMaxScenes) {
|
||
return NextResponse.json(
|
||
{ error: `最多只能添加 ${effectiveMaxScenes} 个场景` },
|
||
{ status: 400 }
|
||
);
|
||
}
|
||
|
||
// 4. 获取生图模型池（固定多样性模式）
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const candidateModels = buildSceneFusionCandidatePool(allModels, config.modelId);
|
||
if (config.modelId && !candidateModels.some((candidate) => candidate.model.id === config.modelId)) {
|
||
logger.warn("preferred_model_unavailable", { requestId, modelId: config.modelId });
|
||
}
|
||
|
||
if (candidateModels.length === 0) {
|
||
return NextResponse.json(
|
||
{ error: "未找到可用模型，请在后台启用至少一个支持生图的模型并配置供应商 API Key" },
|
||
{ status: 500 }
|
||
);
|
||
}
|
||
|
||
logger.info("candidate_pool_resolved", {
|
||
requestId,
|
||
candidateCount: candidateModels.length,
|
||
providers: [...new Set(candidateModels.map((candidate) => candidate.provider.type))],
|
||
});
|
||
const unsupportedCandidates = candidateModels.filter(
|
||
(candidate) => !canProcessSceneFusionProviderType(candidate.provider.type)
|
||
);
|
||
if (unsupportedCandidates.length > 0) {
|
||
logger.warn("unsupported_candidates_present", {
|
||
requestId,
|
||
count: unsupportedCandidates.length,
|
||
providers: [...new Set(unsupportedCandidates.map((candidate) => candidate.provider.type))],
|
||
});
|
||
}
|
||
const nonVisionCandidates = candidateModels.filter(
|
||
(candidate) => !candidate.model.supportsVision
|
||
);
|
||
if (nonVisionCandidates.length > 0) {
|
||
logger.warn("non_vision_candidates_present", {
|
||
requestId,
|
||
count: nonVisionCandidates.length,
|
||
});
|
||
}
|
||
const requestOffset = getNextSceneFusionRoundRobinOffset(
|
||
Math.max(1, candidateModels.length)
|
||
);
|
||
logger.debug("round_robin_offset_resolved", {
|
||
requestId,
|
||
requestOffset,
|
||
sceneCount: scenes.length,
|
||
candidateCount: candidateModels.length,
|
||
});
|
||
|
||
// 5. 预处理：将 Base64 图片上传到 S3
|
||
let productImageUrl = productImage;
|
||
if (productImage.startsWith("data:")) {
|
||
productImageUrl = await uploadBase64ToS3(productImage, "scene-fusion-product");
|
||
}
|
||
const shouldForcePreRmbg = resolvedSceneMode === "random-bg";
|
||
const preprocessedProduct = await preprocessProductImageWithRmbg(
|
||
request,
|
||
requestId,
|
||
productImageUrl,
|
||
shouldForcePreRmbg ? true : config.preRmbgEnabled
|
||
);
|
||
const fusionProductImageUrl = preprocessedProduct.productImageUrl;
|
||
logger.info("pre_rmbg_resolved", {
|
||
requestId,
|
||
used: preprocessedProduct.usedPreRmbg,
|
||
provider: preprocessedProduct.provider || null,
|
||
reason: preprocessedProduct.reason || null,
|
||
});
|
||
|
||
const processedScenes = await Promise.all(
|
||
scenes.map(async (scene) => {
|
||
let sceneImageUrl = scene.image;
|
||
if (scene.image.startsWith("data:")) {
|
||
sceneImageUrl = await uploadBase64ToS3(scene.image, "scene-fusion-scene");
|
||
}
|
||
return {
|
||
...scene,
|
||
imageUrl: sceneImageUrl,
|
||
};
|
||
})
|
||
);
|
||
|
||
const enriched = await enrichSceneLabelsWithAgent({
|
||
requestId,
|
||
productImageUrl: fusionProductImageUrl,
|
||
productName,
|
||
scenes: processedScenes,
|
||
});
|
||
const resolvedProductName = enriched.productName;
|
||
let resolvedScenes = enriched.scenes;
|
||
if (resolvedSceneMode === "random-bg") {
|
||
resolvedScenes = await enrichRandomBackgroundScenesWithAgent({
|
||
requestId,
|
||
productImageUrl: fusionProductImageUrl,
|
||
productName: resolvedProductName,
|
||
scenes: resolvedScenes,
|
||
});
|
||
}
|
||
const resolvedSceneMap = new Map(resolvedScenes.map((scene) => [scene.id, scene]));
|
||
const imageHashCache = new Map<string, string>();
|
||
const modelCooldown: SceneFusionModelCooldown = new Map();
|
||
|
||
logger.info("agent_labels_resolved", {
|
||
requestId,
|
||
hasProductName: Boolean(resolvedProductName),
|
||
targetCount: resolvedScenes.filter((scene) => Boolean(scene.targetName)).length,
|
||
sceneCount: resolvedScenes.length,
|
||
});
|
||
if (resolvedSceneMode === "random-bg") {
|
||
logger.debug("random_bg_prompts_resolved", {
|
||
requestId,
|
||
sceneCount: resolvedScenes.length,
|
||
promptCount: resolvedScenes.filter((scene) => Boolean(scene.customPrompt)).length,
|
||
});
|
||
}
|
||
|
||
// 6. 创建流式响应，每完成一张图就立即发送给前端
|
||
const stream = new ReadableStream({
|
||
async start(controller) {
|
||
const encoder = new TextEncoder();
|
||
let streamClosed = false;
|
||
const safeClose = () => {
|
||
if (streamClosed) return;
|
||
streamClosed = true;
|
||
try {
|
||
controller.close();
|
||
} catch (closeError) {
|
||
logger.warn("stream_close_skipped", closeError, { requestId });
|
||
}
|
||
};
|
||
const enqueue = (payload: Record<string, unknown>) => {
|
||
if (streamClosed) return false;
|
||
try {
|
||
controller.enqueue(encoder.encode(`${JSON.stringify(payload)}\n`));
|
||
return true;
|
||
} catch (enqueueError) {
|
||
streamClosed = true;
|
||
logger.warn("stream_enqueue_skipped", enqueueError, { requestId });
|
||
return false;
|
||
}
|
||
};
|
||
|
||
try {
|
||
// 发送初始消息
|
||
if (!enqueue({ type: "start", total: processedScenes.length })) {
|
||
return;
|
||
}
|
||
enqueue({
|
||
type: "agent_meta",
|
||
data: {
|
||
productName: enriched.meta.productName || resolvedProductName || "",
|
||
productSource: enriched.meta.productSource || (resolvedProductName ? "agent" : "none"),
|
||
sceneTargets: enriched.meta.sceneTargets,
|
||
},
|
||
});
|
||
|
||
let successCount = 0;
|
||
let failCount = 0;
|
||
|
||
let nextSceneIndex = 0;
|
||
const workerCount = Math.max(
|
||
1,
|
||
Math.min(
|
||
SCENE_FUSION_PROCESS_MAX_WORKERS,
|
||
processedScenes.length
|
||
)
|
||
);
|
||
|
||
const workers = Array.from({ length: workerCount }, async () => {
|
||
while (true) {
|
||
if (streamClosed) break;
|
||
const currentIndex = nextSceneIndex;
|
||
nextSceneIndex += 1;
|
||
if (currentIndex >= processedScenes.length) break;
|
||
|
||
const staggerDelay = (currentIndex % workerCount) * SCENE_FUSION_REQUEST_STAGGER_MS;
|
||
if (staggerDelay > 0) {
|
||
await waitSceneFusion(staggerDelay);
|
||
}
|
||
|
||
const scene = processedScenes[currentIndex];
|
||
if (!scene) break;
|
||
const resolvedScene = resolvedSceneMap.get(scene.id) || scene;
|
||
const resolvedMetaScene =
|
||
enriched.meta.sceneTargets.find((item) => item.id === resolvedScene.id) || null;
|
||
|
||
logger.info("scene_processing_started", {
|
||
requestId,
|
||
sceneId: scene.id,
|
||
sceneIndex: currentIndex + 1,
|
||
total: processedScenes.length,
|
||
hasTargetName: Boolean(resolvedScene.targetName),
|
||
hasPositionHint: Boolean(resolvedMetaScene?.positionHint),
|
||
hasReplaceCue: Boolean(resolvedMetaScene?.replaceCue),
|
||
hasCustomPrompt: Boolean(resolvedScene.customPrompt),
|
||
});
|
||
|
||
try {
|
||
const { result, candidate, attempts } = await processSceneWithDiversity({
|
||
productImageUrl: fusionProductImageUrl,
|
||
sceneImageUrl: resolvedScene.imageUrl,
|
||
productName: resolvedProductName,
|
||
targetName: resolvedScene.targetName,
|
||
positionHint: resolvedMetaScene?.positionHint,
|
||
replaceCue: resolvedMetaScene?.replaceCue,
|
||
customPrompt: resolvedScene.customPrompt,
|
||
defaultPrompt: config.defaultPrompt,
|
||
sceneMode: resolvedSceneMode,
|
||
aspectRatio,
|
||
imageSize,
|
||
sceneIndex: currentIndex,
|
||
requestOffset,
|
||
requestId,
|
||
ownerKey: session.user.id,
|
||
candidates: candidateModels,
|
||
imageHashCache,
|
||
modelCooldown,
|
||
onQueueStatus: (queueStatus) => {
|
||
enqueue({
|
||
type: "queue",
|
||
data: {
|
||
id: scene.id,
|
||
status: "pending" as const,
|
||
queueStatus,
|
||
},
|
||
});
|
||
},
|
||
onProcessingStart: () => {
|
||
enqueue({
|
||
type: "queue",
|
||
data: {
|
||
id: scene.id,
|
||
status: "processing" as const,
|
||
queueStatus: null,
|
||
},
|
||
});
|
||
},
|
||
});
|
||
|
||
const dbImageSize = imageSize || "2K";
|
||
|
||
await saveSceneFusionImageRecord({
|
||
userId: session.user.id,
|
||
thumbnailUrl: result.thumbnail,
|
||
originalUrl: result.original,
|
||
imageSize: dbImageSize,
|
||
prompt: `[场景裂变] ${result.prompt}`,
|
||
modelId: candidate.model.modelId,
|
||
sourceImageUrls: [fusionProductImageUrl, resolvedScene.imageUrl],
|
||
});
|
||
logger.info("scene_saved", {
|
||
requestId,
|
||
sceneId: scene.id,
|
||
modelId: candidate.model.modelId,
|
||
providerType: candidate.provider.type,
|
||
attempts,
|
||
});
|
||
|
||
successCount += 1;
|
||
if (!enqueue({
|
||
type: "result",
|
||
data: {
|
||
id: scene.id,
|
||
status: "completed" as const,
|
||
result: result.thumbnail,
|
||
originalUrl: result.original,
|
||
model: candidate.model.displayName,
|
||
modelId: candidate.model.modelId,
|
||
providerType: candidate.provider.type,
|
||
attempts,
|
||
recognizedProductName: resolvedProductName,
|
||
recognizedTargetName: resolvedScene.targetName,
|
||
},
|
||
})) {
|
||
break;
|
||
}
|
||
} catch (error) {
|
||
logger.error("scene_processing_failed", error, {
|
||
requestId,
|
||
sceneId: scene.id,
|
||
sceneIndex: currentIndex + 1,
|
||
});
|
||
failCount += 1;
|
||
if (!enqueue({
|
||
type: "result",
|
||
data: {
|
||
id: scene.id,
|
||
status: "failed" as const,
|
||
error: error instanceof Error ? error.message : "处理失败",
|
||
},
|
||
})) {
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
});
|
||
|
||
await Promise.all(workers);
|
||
|
||
// 发送完成消息
|
||
logger.info("request_completed", {
|
||
requestId,
|
||
total: processedScenes.length,
|
||
success: successCount,
|
||
failed: failCount,
|
||
});
|
||
enqueue({
|
||
type: "done",
|
||
summary: {
|
||
total: processedScenes.length,
|
||
success: successCount,
|
||
failed: failCount,
|
||
},
|
||
});
|
||
|
||
} catch (error) {
|
||
logger.error("stream_failed", error, { requestId });
|
||
// 发送错误消息
|
||
if (!streamClosed) {
|
||
enqueue({
|
||
type: "error",
|
||
error: error instanceof Error ? error.message : "处理失败",
|
||
});
|
||
}
|
||
} finally {
|
||
safeClose();
|
||
}
|
||
},
|
||
});
|
||
|
||
return new Response(stream, {
|
||
headers: {
|
||
"Content-Type": "text/event-stream",
|
||
"Cache-Control": "no-cache",
|
||
"Connection": "keep-alive",
|
||
},
|
||
});
|
||
} catch (error) {
|
||
logger.error("request_failed", error);
|
||
return NextResponse.json(
|
||
{ error: error instanceof Error ? error.message : "处理失败" },
|
||
{ status: 500 }
|
||
);
|
||
}
|
||
}
|