feat: support multiple model types for scene fusion (Alibaba, Gemini, Volcano Vision)
This commit is contained in:
Your Name
parent
1a0dad7895
commit
ade71701e8
@ -2,7 +2,7 @@
|
||||
* 场景裂变 API
|
||||
*
|
||||
* 将一张产品图批量融合到多个场景图中
|
||||
* 使用阿里云千问图片编辑模型
|
||||
* 支持多种生图模型：阿里生图、Gemini、既梦
|
||||
*/
|
||||
|
||||
import { NextResponse } from "next/server";
|
||||
@ -13,6 +13,9 @@ import { generatedImage } from "@/lib/db/schema";
|
||||
import { getEnabledAiModelsWithProvider, getSystemSetting } from "@/lib/db/queries";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
import sharp from "sharp";
|
||||
import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image";
|
||||
import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
|
||||
import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision";
|
||||
|
||||
// 请求体 Schema
|
||||
const sceneFusionRequestSchema = z.object({
|
||||
@ -336,7 +339,200 @@ function buildPrompt(
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理单个场景
|
||||
* 通用场景处理函数 - 根据 provider 类型调用不同的实现
|
||||
*/
|
||||
async function processSceneWithProvider(
|
||||
productImageUrl: string,
|
||||
sceneImageUrl: string,
|
||||
productName: string | undefined,
|
||||
targetName: string | undefined,
|
||||
customPrompt: string | undefined,
|
||||
defaultPrompt: string,
|
||||
provider: { type: string; baseUrl: string; apiKey: string },
|
||||
modelId: string,
|
||||
aspectRatio?: string,
|
||||
imageSize?: string
|
||||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||||
// 构建智能提示词
|
||||
const prompt = buildPrompt(defaultPrompt, productName, targetName, customPrompt);
|
||||
|
||||
console.log(`[SceneFusion] Built prompt: ${prompt}`);
|
||||
console.log(`[SceneFusion] Provider type: ${provider.type}`);
|
||||
|
||||
// 根据 provider 类型选择不同的处理逻辑
|
||||
switch (provider.type) {
|
||||
case "alibaba-image":
|
||||
return processWithAlibabaImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
|
||||
|
||||
case "gemini":
|
||||
return processWithGemini(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
|
||||
|
||||
case "volcano-vision":
|
||||
return processWithVolcanoVision(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
|
||||
|
||||
default:
|
||||
throw new Error(`不支持的模型类型: ${provider.type}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用阿里生图模型处理
|
||||
*/
|
||||
async function processWithAlibabaImage(
|
||||
productImageUrl: string,
|
||||
sceneImageUrl: string,
|
||||
prompt: string,
|
||||
provider: { baseUrl: string; apiKey: string },
|
||||
modelId: string,
|
||||
aspectRatio?: string,
|
||||
imageSize?: string
|
||||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||||
// 计算尺寸
|
||||
const size = calculateSize(aspectRatio || "auto", imageSize || "2K");
|
||||
|
||||
// 调用阿里生图 API
|
||||
const generatedImageUrl = await callAlibabaImageAPI(
|
||||
provider.baseUrl,
|
||||
provider.apiKey,
|
||||
modelId,
|
||||
productImageUrl,
|
||||
sceneImageUrl,
|
||||
prompt,
|
||||
size
|
||||
);
|
||||
|
||||
// 下载并上传到 S3
|
||||
const uploaded = await downloadAndUploadToS3(generatedImageUrl, "scene-fusion-alibaba");
|
||||
|
||||
return { ...uploaded, prompt };
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用 Gemini 模型处理
|
||||
*/
|
||||
async function processWithGemini(
|
||||
productImageUrl: string,
|
||||
sceneImageUrl: string,
|
||||
prompt: string,
|
||||
provider: { baseUrl: string; apiKey: string },
|
||||
modelId: string,
|
||||
aspectRatio?: string,
|
||||
imageSize?: string
|
||||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||||
console.log(`[SceneFusion][Gemini] Processing with model: ${modelId}`);
|
||||
|
||||
// 创建 Gemini provider
|
||||
const geminiProvider = createGeminiImage({
|
||||
baseURL: provider.baseUrl,
|
||||
apiKey: provider.apiKey,
|
||||
});
|
||||
|
||||
// 构建消息格式（Gemini 使用 OpenAI 兼容格式）
|
||||
const messages = [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "image_url", image_url: { url: productImageUrl } },
|
||||
{ type: "image_url", image_url: { url: sceneImageUrl } },
|
||||
{ type: "text", text: prompt },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
// 调用 Gemini API
|
||||
const result = await geminiProvider.chat(modelId).doGenerate({
|
||||
messages,
|
||||
imageParams: {
|
||||
aspectRatio: aspectRatio || "auto",
|
||||
imageSize: imageSize || "2K",
|
||||
imageCount: 1,
|
||||
},
|
||||
});
|
||||
|
||||
// 解析返回的 Markdown 格式图片
|
||||
const { thumbnail, original } = parseImageFromMarkdown(result.text);
|
||||
|
||||
return { thumbnail, original, prompt };
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用既梦/火山模型处理
|
||||
*/
|
||||
async function processWithVolcanoVision(
|
||||
productImageUrl: string,
|
||||
sceneImageUrl: string,
|
||||
prompt: string,
|
||||
provider: { baseUrl: string; apiKey: string },
|
||||
modelId: string,
|
||||
aspectRatio?: string,
|
||||
imageSize?: string
|
||||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||||
console.log(`[SceneFusion][Volcano] Processing with model: ${modelId}`);
|
||||
|
||||
// 创建 Volcano Vision provider
|
||||
const volcanoProvider = createVolcanoVision({
|
||||
baseURL: provider.baseUrl,
|
||||
apiKey: provider.apiKey,
|
||||
});
|
||||
|
||||
// 构建消息格式
|
||||
const messages = [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "image_url", image_url: { url: productImageUrl } },
|
||||
{ type: "image_url", image_url: { url: sceneImageUrl } },
|
||||
{ type: "text", text: prompt },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
// 调用 Volcano Vision API
|
||||
const result = await volcanoProvider.chat(modelId).doGenerate({
|
||||
messages,
|
||||
imageParams: {
|
||||
aspectRatio: aspectRatio || "auto",
|
||||
imageSize: imageSize || "2K",
|
||||
imageCount: 1,
|
||||
},
|
||||
});
|
||||
|
||||
// 解析返回的 Markdown 格式图片
|
||||
const { thumbnail, original } = parseImageFromMarkdown(result.text);
|
||||
|
||||
return { thumbnail, original, prompt };
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 Markdown 格式解析图片 URL
|
||||
* 格式: 
|
||||
*/
|
||||
function parseImageFromMarkdown(markdown: string): { thumbnail: string; original: string } {
|
||||
console.log(`[SceneFusion] Parsing markdown: ${markdown.substring(0, 200)}...`);
|
||||
|
||||
// 匹配  格式
|
||||
const match = markdown.match(/!\[thumb\|\|\|([^|]+)\|\|\|[^\]]*\]\(([^)]+)\)/);
|
||||
if (match) {
|
||||
return {
|
||||
original: match[1],
|
||||
thumbnail: match[2],
|
||||
};
|
||||
}
|
||||
|
||||
// 尝试匹配普通图片格式 
|
||||
const simpleMatch = markdown.match(/!\[[^\]]*\]\(([^)]+)\)/);
|
||||
if (simpleMatch) {
|
||||
return {
|
||||
original: simpleMatch[1],
|
||||
thumbnail: simpleMatch[1],
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error("无法从响应中解析图片 URL");
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理单个场景 (保留旧函数用于兼容)
|
||||
*/
|
||||
async function processScene(
|
||||
productImageUrl: string,
|
||||
@ -491,6 +687,7 @@ export async function POST(request: Request) {
|
||||
console.log(`[SceneFusion] Scene Image URL: ${scene.imageUrl.substring(0, 80)}...`);
|
||||
console.log(`[SceneFusion] Target Name: ${scene.targetName || '(not specified)'}`);
|
||||
console.log(`[SceneFusion] Custom Prompt: ${scene.customPrompt || '(none)'}`);
|
||||
console.log(`[SceneFusion] Provider Type: ${provider.type}`);
|
||||
|
||||
// 如果不是第一张图，等待5秒
|
||||
if (i > 0) {
|
||||
@ -499,17 +696,18 @@ export async function POST(request: Request) {
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await processScene(
|
||||
// 使用新的通用处理函数，支持多种模型类型
|
||||
const result = await processSceneWithProvider(
|
||||
productImageUrl,
|
||||
scene.imageUrl,
|
||||
productName,
|
||||
scene.targetName,
|
||||
scene.customPrompt,
|
||||
config.defaultPrompt,
|
||||
provider.baseUrl,
|
||||
provider.apiKey,
|
||||
{ type: provider.type, baseUrl: provider.baseUrl, apiKey: provider.apiKey },
|
||||
model.modelId,
|
||||
size
|
||||
aspectRatio,
|
||||
imageSize
|
||||
);
|
||||
|
||||
// 将分辨率等级用于数据库存储
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user