import { createHash, timingSafeEqual } from "node:crypto"; import { sql } from "drizzle-orm"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { isImageGenerationModelDescriptor } from "@/lib/ai/model-classification"; import { db } from "@/lib/db"; import { getEnabledAiModelsWithProvider, getSystemSetting } from "@/lib/db/queries"; import { user } from "@/lib/db/schema"; import { generateDummyPassword } from "@/lib/db/utils"; import { DEFAULT_EXTERNAL_IMAGE_API_OWNER_EMAIL, EXTERNAL_IMAGE_API_CONFIG_KEY, normalizeExternalImageApiConfig, type ExternalImageApiAccessKey, } from "@/lib/external-image-api/config"; import { extractImageUrlsFromText } from "@/lib/image-url"; import { uploadFile } from "@/lib/storage/service"; type ImageModelEntry = Awaited>[number]; type ExternalImageApiGenerateAttempt = { modelId: string; modelRuntimeId: string; displayName: string; success: boolean; error?: string; }; function normalizeText(value: unknown) { return String(value || "").trim(); } function mimeTypeToExtension(mimeType: string) { const normalized = mimeType.toLowerCase(); if (normalized.includes("png")) return "png"; if (normalized.includes("webp")) return "webp"; if (normalized.includes("gif")) return "gif"; if (normalized.includes("bmp")) return "bmp"; return "jpg"; } function sanitizeUploadFilename(filename: string, fallbackPrefix: string) { const normalized = String(filename || "") .trim() .replace(/[^\w.\-]+/g, "-") .replace(/-+/g, "-") .replace(/^-|-$/g, ""); if (normalized) return normalized; return `${fallbackPrefix}-${Date.now()}.png`; } function getExtensionFromUrl(url: string): string { try { const pathname = new URL(url).pathname; const ext = pathname.split(".").pop()?.toLowerCase(); if (!ext) return "jpg"; if (["jpg", "jpeg", "png", "webp", "gif", "bmp"].includes(ext)) { return ext; } } catch { // ignore } return "jpg"; } function safeTokenCompare(left: string, right: string) { const a = Buffer.from(left); const b = Buffer.from(right); if (a.length !== b.length) return false; return timingSafeEqual(a, b); } function extractBearerToken(request: Request) { const authorization = normalizeText(request.headers.get("authorization")); if (/^bearer\s+/i.test(authorization)) { return authorization.replace(/^bearer\s+/i, "").trim(); } return normalizeText(request.headers.get("x-api-key")); } async function uploadRemoteImageToStorage(imageUrl: string): Promise { const response = await fetch(imageUrl, { method: "GET", signal: AbortSignal.timeout(30_000), }); if (!response.ok) { throw new Error( `Remote image fetch failed: ${response.status} ${response.statusText}`, ); } const contentType = response.headers.get("content-type")?.toLowerCase() || ""; if (contentType && !contentType.startsWith("image/")) { throw new Error(`Remote url is not an image: ${contentType}`); } const arrayBuffer = await response.arrayBuffer(); if (!arrayBuffer.byteLength) { throw new Error("Remote image content is empty"); } const ext = contentType ? mimeTypeToExtension(contentType) : getExtensionFromUrl(imageUrl); const fileName = `external-image-api-${Date.now()}-${Math.random() .toString(36) .slice(2, 8)}.${ext}`; const uploaded = await uploadFile(fileName, arrayBuffer); return uploaded.url; } async function uploadDataUrlImage(dataUrl: string): Promise { const match = dataUrl.match(/^data:([^;]+);base64,([\s\S]+)$/i); if (!match) { throw new Error("Invalid data URL image format"); } const mimeType = match[1] || "image/jpeg"; const base64Data = normalizeText(match[2]).replace(/\s+/g, ""); if (!base64Data) { throw new Error("Empty base64 image content"); } const buffer = Buffer.from(base64Data, "base64"); const fileName = `external-inline-${Date.now()}-${Math.random() .toString(36) .slice(2, 8)}.${mimeTypeToExtension(mimeType)}`; const arrayBuffer = buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength, ) as ArrayBuffer; const uploaded = await uploadFile(fileName, arrayBuffer); return uploaded.url; } async function ensureHostedImageUrl(imageUrl: string): Promise { const normalizedUrl = normalizeText(imageUrl); if (!normalizedUrl) { throw new Error("Image URL is empty"); } if (normalizedUrl.startsWith("data:image")) { return uploadDataUrlImage(normalizedUrl); } if (!/^https?:\/\//i.test(normalizedUrl)) { throw new Error(`Unsupported image URL: ${normalizedUrl}`); } if (/\/uploads\//i.test(normalizedUrl)) { return normalizedUrl; } return uploadRemoteImageToStorage(normalizedUrl); } function toRecord(value: unknown): Record | null { if (!value || typeof value !== "object" || Array.isArray(value)) return null; return value as Record; } function getSupportedImageModes(capabilities: unknown) { const raw = toRecord(capabilities)?.supportedModes; if (!Array.isArray(raw)) return new Set(); return new Set( raw .map((item) => normalizeText(item).toLowerCase()) .filter(Boolean), ); } function getMaxReferenceImages(capabilities: unknown): number | null { const raw = toRecord(capabilities)?.maxReferenceImages; const parsed = Number(raw); if (!Number.isFinite(parsed) || parsed <= 0) return null; return Math.round(parsed); } function supportsImageEditForReferenceCount( entry: ImageModelEntry, referenceImageCount: number, ) { if (referenceImageCount <= 0) { return true; } const capabilities = toRecord(entry.model.capabilities); const providerType = normalizeText(entry.provider.type).toLowerCase(); const kind = normalizeText(capabilities?.kind).toLowerCase(); const supportedModes = getSupportedImageModes(entry.model.capabilities); const maxReferenceImages = getMaxReferenceImages(entry.model.capabilities); const autoRouteEditEndpoint = capabilities?.autoRouteEditEndpoint === true; const supportsMultiImageReference = capabilities?.supportsMultiImageReference === true; const runtimeModelId = normalizeText(entry.model.modelId).toLowerCase(); const providerImpliesReferenceEdit = providerType === "openai-image" || providerType === "grsai-gpt-image"; const supportsReferenceEdit = providerImpliesReferenceEdit || kind === "image-edit" || supportedModes.has("image-edit") || supportedModes.has("reference-images") || autoRouteEditEndpoint || runtimeModelId.endsWith("/edit"); if (!supportsReferenceEdit) { return false; } if ( maxReferenceImages !== null && referenceImageCount > maxReferenceImages ) { return false; } if (referenceImageCount > 1) { if (maxReferenceImages !== null) { return referenceImageCount <= maxReferenceImages; } return ( supportsMultiImageReference || autoRouteEditEndpoint || providerImpliesReferenceEdit ); } return true; } function getNormalizedModelCandidates(entry: ImageModelEntry) { return [ entry.model.displayName, entry.model.modelId, entry.model.id, ] .map((item) => normalizeText(item) .toLowerCase() .replace(/[\s_]+/g, "-"), ) .filter(Boolean); } function isPreferredGptImage2Model(entry: ImageModelEntry) { const candidates = getNormalizedModelCandidates(entry); return candidates.some( (item) => item.includes("gpt-image-2") || item.includes("gptimage2") || item.includes("gpt-image2"), ); } function getExternalImageApiPriority(entry: ImageModelEntry) { const providerType = normalizeText(entry.provider.type).toLowerCase(); const candidates = getNormalizedModelCandidates(entry); const isGptImage2 = isPreferredGptImage2Model(entry); const isVip = candidates.some((item) => item.includes("vip")); if (isGptImage2 && providerType === "openai-image") { return 400; } if (isGptImage2 && providerType === "grsai-gpt-image") { return isVip ? 350 : 340; } if (isGptImage2 && providerType === "fal-image") { return 200; } if (isGptImage2) { return 100; } return 0; } function isExternalImageApiPreferredProvider(entry: ImageModelEntry) { return getExternalImageApiPriority(entry) >= 340; } function sortExternalImageApiModels( models: ImageModelEntry[], referenceImageCount: number, ) { return [...models].sort((left, right) => { const leftPriority = getExternalImageApiPriority(left); const rightPriority = getExternalImageApiPriority(right); if (leftPriority !== rightPriority) { return rightPriority - leftPriority; } const leftRefSupport = supportsImageEditForReferenceCount( left, referenceImageCount, ) ? 1 : 0; const rightRefSupport = supportsImageEditForReferenceCount( right, referenceImageCount, ) ? 1 : 0; if (leftRefSupport !== rightRefSupport) { return rightRefSupport - leftRefSupport; } return (left.model.sortOrder ?? 0) - (right.model.sortOrder ?? 0); }); } function findExternalImageApiModelMatch( models: ImageModelEntry[], modelRef?: string | null, ) { const normalizedRef = normalizeText(modelRef).toLowerCase(); if (!normalizedRef) return null; return ( models.find((entry) => { const candidates = [ entry.model.id, entry.model.modelId, entry.model.displayName, ] .map((item) => normalizeText(item).toLowerCase()) .filter(Boolean); return candidates.includes(normalizedRef); }) || null ); } export async function ensureHostedImageMarkdown( markdown: string, imageSize = "2K", ) { const pattern = /!\[([^\]]*)\]\(([^)]+)\)/gim; const matches = [...markdown.matchAll(pattern)]; if (!matches.length) { const plainUrls = extractImageUrlsFromText(markdown); if (!plainUrls.length) { return markdown; } const hostedUrls = await Promise.all( plainUrls.map((url) => ensureHostedImageUrl(url)), ); return hostedUrls .map((url) => `![thumb|||${url}|||${imageSize}](${url})`) .join("\n"); } let rebuilt = ""; let cursor = 0; for (const match of matches) { const raw = match[0]; const sourceUrl = match[2] || ""; const start = match.index ?? -1; if (start < 0) continue; rebuilt += markdown.slice(cursor, start); const hostedUrl = await ensureHostedImageUrl(sourceUrl); rebuilt += `![thumb|||${hostedUrl}|||${imageSize}](${hostedUrl})`; cursor = start + raw.length; } rebuilt += markdown.slice(cursor); return rebuilt; } export function extractGeneratedImagesFromMarkdown(markdown: string) { const pattern = /!\[([^\]]*)\]\(([^)]+)\)/gim; const matches = [...markdown.matchAll(pattern)]; return matches.map((match, index) => { const alt = match[1] || ""; const thumbnailUrl = match[2] || ""; let originalUrl = thumbnailUrl; let imageSize = "2K"; const metaMatch = alt.match(/thumb\|\|\|([^|]+)\|\|\|(.+)$/i); if (metaMatch) { originalUrl = normalizeText(metaMatch[1]) || thumbnailUrl; imageSize = normalizeText(metaMatch[2]) || imageSize; } return { id: index + 1, originalUrl, thumbnailUrl, imageSize, }; }); } export async function getExternalImageApiConfig() { const setting = await getSystemSetting({ key: EXTERNAL_IMAGE_API_CONFIG_KEY, }).catch(() => null); return normalizeExternalImageApiConfig(setting?.value); } export async function authenticateExternalImageApiRequest(request: Request) { const config = await getExternalImageApiConfig(); const token = extractBearerToken(request); if (!config.enabled) { return { config, accessKey: null, token, error: "第三方生图 API 未启用", status: 403 as const, }; } if (!token) { return { config, accessKey: null, token, error: "缺少 API Key", status: 401 as const, }; } const accessKey = config.apiKeys.find( (item) => item.enabled && safeTokenCompare(item.apiKey, token), ) || null; if (!accessKey) { return { config, accessKey, token, error: "API Key 无效或已禁用", status: 401 as const, }; } return { config, accessKey, token, error: null, status: 200 as const, }; } export async function getExternalImageApiModels() { const models = await getEnabledAiModelsWithProvider(false, false); return sortExternalImageApiModels( models.filter( (entry) => isImageGenerationModelDescriptor({ providerType: entry.provider.type, modelId: entry.model.modelId, capabilities: entry.model.capabilities, supportsImageGen: entry.model.supportsImageGen, }) && supportsImageEditForReferenceCount(entry, 1), ), 1, ); } export function serializeExternalImageApiModel(entry: ImageModelEntry) { const capabilities = toRecord(entry.model.capabilities); const maxReferenceImages = getMaxReferenceImages(entry.model.capabilities); return { id: entry.model.id, modelId: entry.model.modelId, displayName: entry.model.displayName, description: entry.model.description, providerType: entry.provider.type, providerName: entry.provider.name, avatarUrl: entry.model.avatarUrl, imageGenConfig: entry.model.imageGenConfig, supportsReferenceImages: supportsImageEditForReferenceCount(entry, 1), supportsMultiImageReference: capabilities?.supportsMultiImageReference === true || (maxReferenceImages !== null && maxReferenceImages > 1), maxReferenceImages, preferred: isExternalImageApiPreferredProvider(entry), }; } export async function resolveExternalImageApiModel( modelRef?: string | null, options?: { referenceImageCount?: number; }, ) { const referenceImageCount = Math.max(0, options?.referenceImageCount ?? 1); const models = sortExternalImageApiModels( ( await getEnabledAiModelsWithProvider(false, false) ).filter((entry) => isImageGenerationModelDescriptor({ providerType: entry.provider.type, modelId: entry.model.modelId, capabilities: entry.model.capabilities, supportsImageGen: entry.model.supportsImageGen, }), ), referenceImageCount, ); if (!models.length) { throw new Error("当前没有启用可用于生图的模型"); } const matched = findExternalImageApiModelMatch(models, modelRef); if (!modelRef) { const preferred = models.find((entry) => supportsImageEditForReferenceCount(entry, referenceImageCount), ); if (!preferred) { throw new Error("当前没有启用可用于图生图的模型"); } return preferred; } if (!matched) { throw new Error("未找到匹配的生图模型"); } if (!supportsImageEditForReferenceCount(matched, referenceImageCount)) { const maxReferenceImages = getMaxReferenceImages(matched.model.capabilities); if ( referenceImageCount > 1 && maxReferenceImages !== null && referenceImageCount > maxReferenceImages ) { throw new Error( `模型「${matched.model.displayName}」最多支持 ${maxReferenceImages} 张参考图`, ); } throw new Error(`模型「${matched.model.displayName}」不支持当前图生图调用`); } return matched; } export async function buildExternalImageApiModelCandidates(params?: { modelRef?: string | null; referenceImageCount?: number; }) { const referenceImageCount = Math.max(0, params?.referenceImageCount ?? 1); const allModels = sortExternalImageApiModels( ( await getEnabledAiModelsWithProvider(false, false) ).filter((entry) => isImageGenerationModelDescriptor({ providerType: entry.provider.type, modelId: entry.model.modelId, capabilities: entry.model.capabilities, supportsImageGen: entry.model.supportsImageGen, }), ), referenceImageCount, ); const supportedModels = allModels.filter((entry) => supportsImageEditForReferenceCount(entry, referenceImageCount), ); if (!supportedModels.length) { throw new Error("当前没有启用可用于图生图的模型"); } const explicitMatch = findExternalImageApiModelMatch(allModels, params?.modelRef); if (!params?.modelRef) { return supportedModels; } if (!explicitMatch) { throw new Error("未找到匹配的生图模型"); } if (!supportsImageEditForReferenceCount(explicitMatch, referenceImageCount)) { const maxReferenceImages = getMaxReferenceImages(explicitMatch.model.capabilities); if ( referenceImageCount > 1 && maxReferenceImages !== null && referenceImageCount > maxReferenceImages ) { throw new Error( `模型「${explicitMatch.model.displayName}」最多支持 ${maxReferenceImages} 张参考图`, ); } throw new Error(`模型「${explicitMatch.model.displayName}」不支持当前图生图调用`); } return [ explicitMatch, ...supportedModels.filter((entry) => entry.model.id !== explicitMatch.model.id), ]; } export async function resolveExternalImageApiOwnerUserId(ownerEmail?: string | null) { const normalizedEmail = normalizeText(ownerEmail).toLowerCase() || DEFAULT_EXTERNAL_IMAGE_API_OWNER_EMAIL; const existing = await db .select({ id: user.id }) .from(user) .where(sql`lower(${user.email}) = ${normalizedEmail}`) .limit(1); if (existing[0]?.id) { return existing[0].id; } const [created] = await db .insert(user) .values({ email: normalizedEmail, password: generateDummyPassword(), role: "guest", name: "第三方生图 API", isActive: true, }) .returning({ id: user.id }); if (!created?.id) { throw new Error("创建第三方 API 挂载用户失败"); } return created.id; } export async function generateImagesWithModel(params: { modelDbId: string; prompt: string; negativePrompt?: string | null; imageCount: number; aspectRatio?: string | null; imageSize?: string | null; referenceImageUrls?: string[]; }) { const languageModel = await getLanguageModelById(params.modelDbId); if (!languageModel || typeof (languageModel as any).doGenerate !== "function") { throw new Error("当前模型暂不支持第三方生图调用"); } const composedPrompt = [ normalizeText(params.prompt), normalizeText(params.negativePrompt) ? `Negative prompt:\n${normalizeText(params.negativePrompt)}` : "", ] .filter(Boolean) .join("\n\n"); const result = await (languageModel as any).doGenerate({ messages: [ { role: "user", content: [ { type: "text", text: composedPrompt }, ...(params.referenceImageUrls || []).map((imageUrl) => ({ type: "image", image: imageUrl, })), ], }, ], imageParams: { imageCount: params.imageCount, aspectRatio: normalizeText(params.aspectRatio) || undefined, imageSize: normalizeText(params.imageSize) || undefined, }, }); if (result?.finishReason === "error") { throw new Error(result.text || "图片生成失败"); } const text = normalizeText(result?.text); if (!text) { throw new Error("模型未返回生成图片"); } return text; } export async function generateImagesWithFailover(params: { candidates: ImageModelEntry[]; prompt: string; negativePrompt?: string | null; imageCount: number; aspectRatio?: string | null; imageSize?: string | null; referenceImageUrls?: string[]; }) { const attempts: ExternalImageApiGenerateAttempt[] = []; for (const candidate of params.candidates) { try { const markdown = await generateImagesWithModel({ modelDbId: candidate.model.id, prompt: params.prompt, negativePrompt: params.negativePrompt || null, imageCount: params.imageCount, aspectRatio: params.aspectRatio || null, imageSize: params.imageSize || null, referenceImageUrls: params.referenceImageUrls || [], }); attempts.push({ modelId: candidate.model.id, modelRuntimeId: candidate.model.modelId, displayName: candidate.model.displayName, success: true, }); return { markdown, usedModel: candidate, attempts, }; } catch (error) { attempts.push({ modelId: candidate.model.id, modelRuntimeId: candidate.model.modelId, displayName: candidate.model.displayName, success: false, error: error instanceof Error ? error.message : "图片生成失败", }); } } const summary = attempts .filter((item) => !item.success) .map((item) => `${item.displayName}: ${item.error || "图片生成失败"}`) .join(";"); throw new Error(summary || "所有候选生图模型均调用失败"); } export async function ensureHostedReferenceImageUrls(imageUrls: string[]) { const deduped = imageUrls .map((item) => normalizeText(item)) .filter(Boolean) .filter((item, index, list) => list.indexOf(item) === index); return Promise.all(deduped.map((url) => ensureHostedImageUrl(url))); } export async function uploadExternalImageApiFile(file: File) { if (!(file instanceof File)) { throw new Error("无效的图片文件"); } const arrayBuffer = await file.arrayBuffer(); if (!arrayBuffer.byteLength) { throw new Error("上传的图片文件为空"); } const filename = sanitizeUploadFilename(file.name, "external-reference"); const uploaded = await uploadFile(filename, arrayBuffer); return uploaded.url; } export function buildExternalImageApiKeyFingerprint( accessKey: ExternalImageApiAccessKey | null, ) { if (!accessKey?.apiKey) return null; return createHash("sha256").update(accessKey.apiKey).digest("hex").slice(0, 16); } export function getRequesterIp(request: Request) { const forwarded = normalizeText(request.headers.get("x-forwarded-for")); if (forwarded) { return forwarded.split(",")[0]?.trim() || null; } return normalizeText(request.headers.get("x-real-ip")) || null; }