From cc320f6334ca802bd9aa70b7a4aa244826bc601d Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 18 May 2026 16:29:46 +0800 Subject: [PATCH] feat: default external image api to img2img failover --- app/api/external/image/generate/route.ts | 175 +++++++++-- docs/第三方生图API调用说明.md | 63 ++-- lib/external-image-api/service.ts | 384 +++++++++++++++++++++-- 3 files changed, 556 insertions(+), 66 deletions(-) diff --git a/app/api/external/image/generate/route.ts b/app/api/external/image/generate/route.ts index 267b95371..dc955445c 100644 --- a/app/api/external/image/generate/route.ts +++ b/app/api/external/image/generate/route.ts @@ -6,29 +6,145 @@ import { externalImageApiRequestLog } from "@/lib/db/schema"; import { saveGeneratedImages } from "@/lib/ai/runtime/generated-image-persistence"; import { authenticateExternalImageApiRequest, + buildExternalImageApiModelCandidates, buildExternalImageApiKeyFingerprint, ensureHostedImageMarkdown, + ensureHostedReferenceImageUrls, extractGeneratedImagesFromMarkdown, - generateImagesWithModel, + generateImagesWithFailover, getRequesterIp, - resolveExternalImageApiModel, resolveExternalImageApiOwnerUserId, serializeExternalImageApiModel, + uploadExternalImageApiFile, } from "@/lib/external-image-api/service"; export const runtime = "nodejs"; export const maxDuration = 120; const requestSchema = z.object({ - model: z.string().trim().min(1).optional(), prompt: z.string().trim().min(1, "prompt 不能为空"), negativePrompt: z.string().trim().optional(), imageCount: z.coerce.number().int().min(1).max(8).optional(), aspectRatio: z.string().trim().max(32).optional(), imageSize: z.string().trim().max(32).optional(), metadata: z.record(z.string(), z.any()).optional(), + referenceImages: z.array(z.string().trim().min(1)).max(14).optional(), }); +function normalizeString(value: FormDataEntryValue | string | null | undefined) { + return String(value || "").trim(); +} + +function parseOptionalInteger(value: FormDataEntryValue | null) { + const normalized = normalizeString(value); + if (!normalized) return undefined; + const parsed = Number(normalized); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function parseOptionalJsonRecord(value: FormDataEntryValue | null) { + const normalized = normalizeString(value); + if (!normalized) return undefined; + try { + const parsed = JSON.parse(normalized); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + return parsed as Record; + } + } catch { + // ignore invalid metadata JSON + } + return undefined; +} + +function normalizeReferenceImageInputs(value: unknown): string[] { + if (!value) return []; + if (Array.isArray(value)) { + return value + .flatMap((item) => normalizeReferenceImageInputs(item)) + .filter(Boolean); + } + if (typeof value === "string") { + return value + .split(",") + .map((item) => item.trim()) + .filter(Boolean); + } + return []; +} + +async function parseRequestPayload(request: Request) { + const contentType = request.headers.get("content-type") || ""; + if (contentType.toLowerCase().includes("multipart/form-data")) { + const formData = await request.formData(); + const fileEntries = [ + ...formData.getAll("images"), + ...formData.getAll("image"), + ...formData.getAll("referenceImages"), + ...formData.getAll("referenceImage"), + ...formData.getAll("files"), + ]; + const files = fileEntries.filter((item): item is File => item instanceof File); + const uploadedReferenceImages = await Promise.all( + files.map((file) => uploadExternalImageApiFile(file)), + ); + + const inlineReferenceImages = [ + ...formData.getAll("imageUrls"), + ...formData.getAll("imageUrl"), + ...formData.getAll("referenceImageUrls"), + ...formData.getAll("referenceImages"), + ...formData.getAll("referenceImage"), + ] + .filter((item): item is string => typeof item === "string") + .flatMap((item) => normalizeReferenceImageInputs(item)); + + const hostedInlineImages = await ensureHostedReferenceImageUrls( + inlineReferenceImages, + ); + + const parsed = requestSchema.parse({ + prompt: normalizeString(formData.get("prompt")), + negativePrompt: normalizeString(formData.get("negativePrompt")) || undefined, + imageCount: parseOptionalInteger(formData.get("imageCount")), + aspectRatio: normalizeString(formData.get("aspectRatio")) || undefined, + imageSize: normalizeString(formData.get("imageSize")) || undefined, + metadata: parseOptionalJsonRecord(formData.get("metadata")), + referenceImages: [...uploadedReferenceImages, ...hostedInlineImages], + }); + if (!parsed.referenceImages?.length) { + throw new z.ZodError([ + { + code: "custom", + path: ["referenceImages"], + message: "至少上传 1 张参考图", + }, + ]); + } + return parsed; + } + + const json = await request.json(); + const parsed = requestSchema.parse({ + ...json, + referenceImages: await ensureHostedReferenceImageUrls([ + ...normalizeReferenceImageInputs((json as any)?.referenceImages), + ...normalizeReferenceImageInputs((json as any)?.imageUrls), + ...normalizeReferenceImageInputs((json as any)?.images), + ...normalizeReferenceImageInputs((json as any)?.image), + ]), + }); + if (!parsed.referenceImages?.length) { + throw new z.ZodError([ + { + code: "custom", + path: ["referenceImages"], + message: "至少上传 1 张参考图", + }, + ]); + } + return parsed; +} + export async function POST(request: Request) { const authResult = await authenticateExternalImageApiRequest(request); if (!authResult.accessKey) { @@ -45,20 +161,11 @@ export async function POST(request: Request) { let logId: string | null = null; try { - const json = await request.json(); - const parsed = requestSchema.safeParse(json); - if (!parsed.success) { - return NextResponse.json( - { - success: false, - error: parsed.error.errors[0]?.message || "参数错误", - }, - { status: 400 }, - ); - } - - const payload = parsed.data; - const selectedModel = await resolveExternalImageApiModel(payload.model); + const payload = await parseRequestPayload(request); + const candidateModels = await buildExternalImageApiModelCandidates({ + referenceImageCount: payload.referenceImages?.length || 0, + }); + const selectedModel = candidateModels[0]; const ownerUserId = await resolveExternalImageApiOwnerUserId( authResult.config.defaultOwnerEmail, ); @@ -84,6 +191,7 @@ export async function POST(request: Request) { requesterUserAgent, requestPayload: { ...payload, + candidateModels: candidateModels.map(serializeExternalImageApiModel), apiKeyFingerprint: buildExternalImageApiKeyFingerprint( authResult.accessKey, ), @@ -93,14 +201,17 @@ export async function POST(request: Request) { logId = createdLog?.id || null; - const rawMarkdown = await generateImagesWithModel({ - modelDbId: selectedModel.model.id, + const generationResult = await generateImagesWithFailover({ + candidates: candidateModels, prompt: payload.prompt, negativePrompt: payload.negativePrompt || null, imageCount: payload.imageCount || 1, aspectRatio: payload.aspectRatio || null, imageSize: payload.imageSize || null, + referenceImageUrls: payload.referenceImages || [], }); + const actualModel = generationResult.usedModel; + const rawMarkdown = generationResult.markdown; const hostedMarkdown = await ensureHostedImageMarkdown( rawMarkdown, @@ -117,7 +228,7 @@ export async function POST(request: Request) { null, null, payload.prompt, - selectedModel.model.id, + actualModel.model.id, { sourceType: "external_api_generated", userPrompt: payload.prompt, @@ -125,6 +236,8 @@ export async function POST(request: Request) { requestedImageCount: payload.imageCount || images.length, aspectRatio: payload.aspectRatio, imageSize: payload.imageSize, + sourceImageUrls: payload.referenceImages || [], + relationType: "edited_from", skipBilling: true, }, ); @@ -133,11 +246,15 @@ export async function POST(request: Request) { await db .update(externalImageApiRequestLog) .set({ + modelDbId: actualModel.model.id, + modelRuntimeId: actualModel.model.modelId, + modelDisplayName: actualModel.model.displayName, status: "succeeded", responsePayload: { - model: serializeExternalImageApiModel(selectedModel), + model: serializeExternalImageApiModel(actualModel), markdown: hostedMarkdown, images, + attempts: generationResult.attempts, }, generatedImageIds: persistResult.savedImageIds, generatedAssetIds: persistResult.savedAssetIds, @@ -152,21 +269,35 @@ export async function POST(request: Request) { success: true, data: { requestId: logId, - model: serializeExternalImageApiModel(selectedModel), + model: serializeExternalImageApiModel(actualModel), images, generatedImageIds: persistResult.savedImageIds, generatedAssetIds: persistResult.savedAssetIds, + attempts: generationResult.attempts, }, }); } catch (error) { console.error("[External Image API] Generate failed:", error); + if (error instanceof z.ZodError) { + return NextResponse.json( + { + success: false, + error: error.errors[0]?.message || "参数错误", + }, + { status: 400 }, + ); + } + if (logId) { await db .update(externalImageApiRequestLog) .set({ status: "failed", errorMessage: error instanceof Error ? error.message : "图片生成失败", + responsePayload: { + error: error instanceof Error ? error.message : "图片生成失败", + }, durationMs: Date.now() - startedAt, completedAt: new Date(), }) diff --git a/docs/第三方生图API调用说明.md b/docs/第三方生图API调用说明.md index c7c5f3a75..e0989c541 100644 --- a/docs/第三方生图API调用说明.md +++ b/docs/第三方生图API调用说明.md @@ -12,7 +12,8 @@ 2. 发起生图 `POST /api/external/image/generate` -这套接口直接复用我们系统后台已启用的生图模型与模型名称。 +这套接口是 **图生图接口**。第三方调用方只需要上传参考图和提示词，**不需要也不允许指定内部模型名**。 +我们服务端会按后台默认优先级自动选择模型，优先尝试 `GPT-image-2`，失败后自动故障迁移到其他可用图生图模型，尽量返回图片。 生成结果图片会先上传到我们的储存桶，再写入我们的数据库与媒体资产表。 ## 2. 鉴权方式 @@ -33,12 +34,12 @@ x-api-key: YOUR_API_KEY 如果 API Key 无效、已停用，或后台未启用第三方生图 API，接口会返回 `401` 或 `403`。 -## 3. 获取模型列表 +## 3. 获取模型列表（可选） ### 请求 ```bash -curl -X GET "https://你的域名/api/external/image/models" \ +curl -X GET "https://ai.yydd.ai/api/external/image/models" \ -H "Authorization: Bearer YOUR_API_KEY" ``` @@ -56,7 +57,7 @@ curl -X GET "https://你的域名/api/external/image/models" \ "description": "轻量版商品生图模型", "providerType": "volcano-vision", "providerName": "火山视觉", - "avatarUrl": "https://你的域名/uploads/xxx.png", + "avatarUrl": "https://ai.yydd.ai/uploads/xxx.png", "imageGenConfig": { "creditCost": 20 } @@ -68,20 +69,25 @@ curl -X GET "https://你的域名/api/external/image/models" \ ### 字段说明 -- `id`：系统内部模型 UUID，推荐第三方优先使用这个值调用生成接口 +- `id`：系统内部模型 UUID - `modelId`：上游模型 ID - `displayName`：系统中展示给用户的模型名称 - `providerType`：模型服务商类型 - `providerName`：模型服务商显示名称 - `imageGenConfig`：当前模型的生图配置，包含积分等后台配置 +说明： + +- 这个接口主要用于联调排查或查看当前可用能力 +- 正式调用生图接口时，第三方 **无需传模型名** + ## 4. 发起生图 ### 请求地址 ```http -POST /api/external/image/generate -Content-Type: application/json +POST https://ai.yydd.ai/api/external/image/generate +Content-Type: application/json 或 multipart/form-data Authorization: Bearer YOUR_API_KEY ``` @@ -89,12 +95,15 @@ Authorization: Bearer YOUR_API_KEY ```json { - "model": "9b8f7c10-1111-4d22-9abc-1234567890ab", - "prompt": "请生成一张高端现代客厅空间图，暖光，真实摄影感，适合商业首页展示", + "prompt": "保留原空间结构，将这张衣帽间草图改成真实摄影效果，木色更高级，灯光柔和", "negativePrompt": "不要人物，不要文字水印，不要畸变", "imageCount": 2, "aspectRatio": "16:9", "imageSize": "2K", + "referenceImages": [ + "https://ai.yydd.ai/uploads/source-1.jpg", + "https://ai.yydd.ai/uploads/source-2.jpg" + ], "metadata": { "bizId": "partner-order-20260518-001" } @@ -103,17 +112,22 @@ Authorization: Bearer YOUR_API_KEY ### 参数说明 -- `model`：可传以下任意一种 - - 模型 UUID `id` - - 模型运行 ID `modelId` - - 模型显示名 `displayName` - `prompt`：正向提示词，必填 - `negativePrompt`：反向提示词，可选 - `imageCount`：生成张数，可选，默认 `1`，最大 `8` - `aspectRatio`：宽高比，可选，例如 `1:1`、`16:9` - `imageSize`：图片规格，可选，例如 `1K`、`2K` +- `referenceImages`：参考图地址数组，必填，至少 `1` 张，最多 `14` 张 - `metadata`：业务扩展字段，可选，会原样记录到请求日志里 +### multipart/form-data 调用方式 + +可直接上传图片文件，不必先把图片上传到公网： + +- 文本字段：`prompt`、`negativePrompt`、`imageCount`、`aspectRatio`、`imageSize` +- 文件字段：`images`（可重复上传多张） +- 也支持混合传参：文件字段 `images` + URL 字段 `imageUrls` + ## 5. 生图成功响应 ```json @@ -136,14 +150,14 @@ Authorization: Bearer YOUR_API_KEY "images": [ { "id": 1, - "originalUrl": "https://你的域名/uploads/2026xxxx-orig.jpg", - "thumbnailUrl": "https://你的域名/uploads/2026xxxx-orig.jpg", + "originalUrl": "https://ai.yydd.ai/uploads/2026xxxx-orig.jpg", + "thumbnailUrl": "https://ai.yydd.ai/uploads/2026xxxx-orig.jpg", "imageSize": "2K" }, { "id": 2, - "originalUrl": "https://你的域名/uploads/2026yyyy-orig.jpg", - "thumbnailUrl": "https://你的域名/uploads/2026yyyy-orig.jpg", + "originalUrl": "https://ai.yydd.ai/uploads/2026yyyy-orig.jpg", + "thumbnailUrl": "https://ai.yydd.ai/uploads/2026yyyy-orig.jpg", "imageSize": "2K" } ], @@ -152,6 +166,14 @@ Authorization: Bearer YOUR_API_KEY ], "generatedAssetIds": [ "b7b19e10-1111-4db8-aeb2-1234567890ab" + ], + "attempts": [ + { + "modelId": "07e2304a-a63a-4fd3-9c76-63ba98de464d", + "modelRuntimeId": "openai/gpt-image-2/edit", + "displayName": "GPT-image-2", + "success": true + } ] } } @@ -166,6 +188,7 @@ Authorization: Bearer YOUR_API_KEY - `imageSize`：记录的图片规格 - `generatedImageIds`：写入 `GeneratedImage` 表后的记录 ID - `generatedAssetIds`：写入媒体资产表后的记录 ID +- `attempts`：本次自动选模 / 故障迁移尝试明细 ## 6. 失败响应示例 @@ -180,14 +203,14 @@ Authorization: Bearer YOUR_API_KEY - API Key 缺失或无效 - 第三方生图 API 后台未开启 -- 请求的模型不存在或未启用 +- 未上传参考图 - 上游模型生成失败 - 上游返回内容中未解析出图片 ## 7. 调用建议 -1. 先调用模型列表接口，缓存可用模型 -2. 生成时优先传模型 UUID `id`，最稳定 +1. 这条接口按“图生图”设计，请始终上传至少 1 张参考图 +2. 第三方不要传模型名，交给服务端按默认优先级和故障迁移处理 3. 将返回的 `requestId` 保存到业务日志，便于双方排查问题 4. 返回的图片地址已经是我们系统储存桶地址，可以直接展示或下载 diff --git a/lib/external-image-api/service.ts b/lib/external-image-api/service.ts index 9b9abbc9c..54922f4f2 100644 --- a/lib/external-image-api/service.ts +++ b/lib/external-image-api/service.ts @@ -16,6 +16,13 @@ import { extractImageUrlsFromText } from "@/lib/image-url"; import { uploadFile } from "@/lib/storage/service"; type ImageModelEntry = Awaited>[number]; +type ExternalImageApiGenerateAttempt = { + modelId: string; + modelRuntimeId: string; + displayName: string; + success: boolean; + error?: string; +}; function normalizeText(value: unknown) { return String(value || "").trim(); @@ -30,6 +37,16 @@ function mimeTypeToExtension(mimeType: string) { return "jpg"; } +function sanitizeUploadFilename(filename: string, fallbackPrefix: string) { + const normalized = String(filename || "") + .trim() + .replace(/[^\w.\-]+/g, "-") + .replace(/-+/g, "-") + .replace(/^-|-$/g, ""); + if (normalized) return normalized; + return `${fallbackPrefix}-${Date.now()}.png`; +} + function getExtensionFromUrl(url: string): string { try { const pathname = new URL(url).pathname; @@ -136,6 +153,146 @@ async function ensureHostedImageUrl(imageUrl: string): Promise { return uploadRemoteImageToStorage(normalizedUrl); } +function toRecord(value: unknown): Record | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + return value as Record; +} + +function getSupportedImageModes(capabilities: unknown) { + const raw = toRecord(capabilities)?.supportedModes; + if (!Array.isArray(raw)) return new Set(); + return new Set( + raw + .map((item) => normalizeText(item).toLowerCase()) + .filter(Boolean), + ); +} + +function getMaxReferenceImages(capabilities: unknown): number | null { + const raw = toRecord(capabilities)?.maxReferenceImages; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) return null; + return Math.round(parsed); +} + +function supportsImageEditForReferenceCount( + entry: ImageModelEntry, + referenceImageCount: number, +) { + if (referenceImageCount <= 0) { + return true; + } + + const capabilities = toRecord(entry.model.capabilities); + const kind = normalizeText(capabilities?.kind).toLowerCase(); + const supportedModes = getSupportedImageModes(entry.model.capabilities); + const maxReferenceImages = getMaxReferenceImages(entry.model.capabilities); + const autoRouteEditEndpoint = capabilities?.autoRouteEditEndpoint === true; + const supportsMultiImageReference = + capabilities?.supportsMultiImageReference === true; + const runtimeModelId = normalizeText(entry.model.modelId).toLowerCase(); + + const supportsReferenceEdit = + kind === "image-edit" || + supportedModes.has("image-edit") || + supportedModes.has("reference-images") || + autoRouteEditEndpoint || + runtimeModelId.endsWith("/edit"); + + if (!supportsReferenceEdit) { + return false; + } + + if ( + maxReferenceImages !== null && + referenceImageCount > maxReferenceImages + ) { + return false; + } + + if (referenceImageCount > 1) { + if (maxReferenceImages !== null) { + return referenceImageCount <= maxReferenceImages; + } + return supportsMultiImageReference || autoRouteEditEndpoint; + } + + return true; +} + +function isPreferredGptImage2Model(entry: ImageModelEntry) { + const candidates = [ + entry.model.displayName, + entry.model.modelId, + entry.model.id, + ] + .map((item) => + normalizeText(item) + .toLowerCase() + .replace(/[\s_]+/g, "-"), + ) + .filter(Boolean); + + return candidates.some( + (item) => + item.includes("gpt-image-2") || + item.includes("gptimage2") || + item.includes("gpt-image2"), + ); +} + +function sortExternalImageApiModels( + models: ImageModelEntry[], + referenceImageCount: number, +) { + return [...models].sort((left, right) => { + const leftPreferred = isPreferredGptImage2Model(left) ? 1 : 0; + const rightPreferred = isPreferredGptImage2Model(right) ? 1 : 0; + if (leftPreferred !== rightPreferred) { + return rightPreferred - leftPreferred; + } + + const leftRefSupport = supportsImageEditForReferenceCount( + left, + referenceImageCount, + ) + ? 1 + : 0; + const rightRefSupport = supportsImageEditForReferenceCount( + right, + referenceImageCount, + ) + ? 1 + : 0; + if (leftRefSupport !== rightRefSupport) { + return rightRefSupport - leftRefSupport; + } + + return (left.model.sortOrder ?? 0) - (right.model.sortOrder ?? 0); + }); +} + +function findExternalImageApiModelMatch( + models: ImageModelEntry[], + modelRef?: string | null, +) { + const normalizedRef = normalizeText(modelRef).toLowerCase(); + if (!normalizedRef) return null; + + return ( + models.find((entry) => { + const candidates = [ + entry.model.id, + entry.model.modelId, + entry.model.displayName, + ] + .map((item) => normalizeText(item).toLowerCase()) + .filter(Boolean); + return candidates.includes(normalizedRef); + }) || null + ); +} + export async function ensureHostedImageMarkdown( markdown: string, imageSize = "2K", @@ -257,17 +414,23 @@ export async function authenticateExternalImageApiRequest(request: Request) { export async function getExternalImageApiModels() { const models = await getEnabledAiModelsWithProvider(false, false); - return models.filter((entry) => - isImageGenerationModelDescriptor({ - providerType: entry.provider.type, - modelId: entry.model.modelId, - capabilities: entry.model.capabilities, - supportsImageGen: entry.model.supportsImageGen, - }), + return sortExternalImageApiModels( + models.filter( + (entry) => + isImageGenerationModelDescriptor({ + providerType: entry.provider.type, + modelId: entry.model.modelId, + capabilities: entry.model.capabilities, + supportsImageGen: entry.model.supportsImageGen, + }) && supportsImageEditForReferenceCount(entry, 1), + ), + 1, ); } export function serializeExternalImageApiModel(entry: ImageModelEntry) { + const capabilities = toRecord(entry.model.capabilities); + const maxReferenceImages = getMaxReferenceImages(entry.model.capabilities); return { id: entry.model.id, modelId: entry.model.modelId, @@ -277,38 +440,127 @@ export function serializeExternalImageApiModel(entry: ImageModelEntry) { providerName: entry.provider.name, avatarUrl: entry.model.avatarUrl, imageGenConfig: entry.model.imageGenConfig, + supportsReferenceImages: supportsImageEditForReferenceCount(entry, 1), + supportsMultiImageReference: + capabilities?.supportsMultiImageReference === true || + (maxReferenceImages !== null && maxReferenceImages > 1), + maxReferenceImages, + preferred: isPreferredGptImage2Model(entry), }; } -export async function resolveExternalImageApiModel(modelRef?: string | null) { - const models = await getExternalImageApiModels(); +export async function resolveExternalImageApiModel( + modelRef?: string | null, + options?: { + referenceImageCount?: number; + }, +) { + const referenceImageCount = Math.max(0, options?.referenceImageCount ?? 1); + const models = sortExternalImageApiModels( + ( + await getEnabledAiModelsWithProvider(false, false) + ).filter((entry) => + isImageGenerationModelDescriptor({ + providerType: entry.provider.type, + modelId: entry.model.modelId, + capabilities: entry.model.capabilities, + supportsImageGen: entry.model.supportsImageGen, + }), + ), + referenceImageCount, + ); if (!models.length) { throw new Error("当前没有启用可用于生图的模型"); } - const normalizedRef = normalizeText(modelRef).toLowerCase(); - if (!normalizedRef) { - return models[0]; + const matched = findExternalImageApiModelMatch(models, modelRef); + if (!modelRef) { + const preferred = models.find((entry) => + supportsImageEditForReferenceCount(entry, referenceImageCount), + ); + if (!preferred) { + throw new Error("当前没有启用可用于图生图的模型"); + } + return preferred; } - const matched = models.find((entry) => { - const candidates = [ - entry.model.id, - entry.model.modelId, - entry.model.displayName, - ] - .map((item) => normalizeText(item).toLowerCase()) - .filter(Boolean); - return candidates.includes(normalizedRef); - }); - if (!matched) { throw new Error("未找到匹配的生图模型"); } + if (!supportsImageEditForReferenceCount(matched, referenceImageCount)) { + const maxReferenceImages = getMaxReferenceImages(matched.model.capabilities); + if ( + referenceImageCount > 1 && + maxReferenceImages !== null && + referenceImageCount > maxReferenceImages + ) { + throw new Error( + `模型「${matched.model.displayName}」最多支持 ${maxReferenceImages} 张参考图`, + ); + } + throw new Error(`模型「${matched.model.displayName}」不支持当前图生图调用`); + } + return matched; } +export async function buildExternalImageApiModelCandidates(params?: { + modelRef?: string | null; + referenceImageCount?: number; +}) { + const referenceImageCount = Math.max(0, params?.referenceImageCount ?? 1); + const allModels = sortExternalImageApiModels( + ( + await getEnabledAiModelsWithProvider(false, false) + ).filter((entry) => + isImageGenerationModelDescriptor({ + providerType: entry.provider.type, + modelId: entry.model.modelId, + capabilities: entry.model.capabilities, + supportsImageGen: entry.model.supportsImageGen, + }), + ), + referenceImageCount, + ); + + const supportedModels = allModels.filter((entry) => + supportsImageEditForReferenceCount(entry, referenceImageCount), + ); + + if (!supportedModels.length) { + throw new Error("当前没有启用可用于图生图的模型"); + } + + const explicitMatch = findExternalImageApiModelMatch(allModels, params?.modelRef); + if (!params?.modelRef) { + return supportedModels; + } + + if (!explicitMatch) { + throw new Error("未找到匹配的生图模型"); + } + + if (!supportsImageEditForReferenceCount(explicitMatch, referenceImageCount)) { + const maxReferenceImages = getMaxReferenceImages(explicitMatch.model.capabilities); + if ( + referenceImageCount > 1 && + maxReferenceImages !== null && + referenceImageCount > maxReferenceImages + ) { + throw new Error( + `模型「${explicitMatch.model.displayName}」最多支持 ${maxReferenceImages} 张参考图`, + ); + } + throw new Error(`模型「${explicitMatch.model.displayName}」不支持当前图生图调用`); + } + + return [ + explicitMatch, + ...supportedModels.filter((entry) => entry.model.id !== explicitMatch.model.id), + ]; +} + export async function resolveExternalImageApiOwnerUserId(ownerEmail?: string | null) { const normalizedEmail = normalizeText(ownerEmail).toLowerCase() || DEFAULT_EXTERNAL_IMAGE_API_OWNER_EMAIL; @@ -348,6 +600,7 @@ export async function generateImagesWithModel(params: { imageCount: number; aspectRatio?: string | null; imageSize?: string | null; + referenceImageUrls?: string[]; }) { const languageModel = await getLanguageModelById(params.modelDbId); if (!languageModel || typeof (languageModel as any).doGenerate !== "function") { @@ -367,7 +620,13 @@ export async function generateImagesWithModel(params: { messages: [ { role: "user", - content: [{ type: "text", text: composedPrompt }], + content: [ + { type: "text", text: composedPrompt }, + ...(params.referenceImageUrls || []).map((imageUrl) => ({ + type: "image", + image: imageUrl, + })), + ], }, ], imageParams: { @@ -389,6 +648,83 @@ export async function generateImagesWithModel(params: { return text; } +export async function generateImagesWithFailover(params: { + candidates: ImageModelEntry[]; + prompt: string; + negativePrompt?: string | null; + imageCount: number; + aspectRatio?: string | null; + imageSize?: string | null; + referenceImageUrls?: string[]; +}) { + const attempts: ExternalImageApiGenerateAttempt[] = []; + + for (const candidate of params.candidates) { + try { + const markdown = await generateImagesWithModel({ + modelDbId: candidate.model.id, + prompt: params.prompt, + negativePrompt: params.negativePrompt || null, + imageCount: params.imageCount, + aspectRatio: params.aspectRatio || null, + imageSize: params.imageSize || null, + referenceImageUrls: params.referenceImageUrls || [], + }); + + attempts.push({ + modelId: candidate.model.id, + modelRuntimeId: candidate.model.modelId, + displayName: candidate.model.displayName, + success: true, + }); + + return { + markdown, + usedModel: candidate, + attempts, + }; + } catch (error) { + attempts.push({ + modelId: candidate.model.id, + modelRuntimeId: candidate.model.modelId, + displayName: candidate.model.displayName, + success: false, + error: error instanceof Error ? error.message : "图片生成失败", + }); + } + } + + const summary = attempts + .filter((item) => !item.success) + .map((item) => `${item.displayName}: ${item.error || "图片生成失败"}`) + .join("；"); + throw new Error(summary || "所有候选生图模型均调用失败"); +} + +export async function ensureHostedReferenceImageUrls(imageUrls: string[]) { + const deduped = imageUrls + .map((item) => normalizeText(item)) + .filter(Boolean) + .filter((item, index, list) => list.indexOf(item) === index); + + return Promise.all(deduped.map((url) => ensureHostedImageUrl(url))); +} + +export async function uploadExternalImageApiFile(file: File) { + if (!(file instanceof File)) { + throw new Error("无效的图片文件"); + } + + const arrayBuffer = await file.arrayBuffer(); + if (!arrayBuffer.byteLength) { + throw new Error("上传的图片文件为空"); + } + + const filename = sanitizeUploadFilename(file.name, "external-reference"); + const uploaded = await uploadFile(filename, arrayBuffer); + return uploaded.url; +} + export function buildExternalImageApiKeyFingerprint( accessKey: ExternalImageApiAccessKey | null, ) {