import { createLogger } from "@/lib/observability/logger"; import { isFalGptImage2EditModelId } from "@/lib/ai/providers/fal-image"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { modelLacksReferenceImageInputSupport } from "@/lib/ai/reference-image-capability"; import type { ChatMessage } from "@/lib/types"; import { convertMessagesWithImages } from "./image-media"; const chatApiLogger = createLogger("chat.imageFailover"); export const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([ "alibaba-image", "bfl-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "stepfun-image", "runway", "krea", "grsai-gpt-image", "volcano-vision", "gemini", "huanke-gemini", ]); const IMAGE_FAILOVER_MODEL_COOLDOWN_MS = 30 * 60 * 1000; const imageFailoverModelCooldown = new Map(); export function resolveImageModelRuntimeId( model: { defaultParams?: unknown; modelId?: unknown; id?: unknown } | null | undefined, ) { const defaultParams = model?.defaultParams; const defaultEndpoint = typeof defaultParams === "object" && defaultParams !== null && "endpoint" in defaultParams ? String((defaultParams as Record).endpoint || "").trim() : ""; if (defaultEndpoint) { return defaultEndpoint; } const modelId = String(model?.modelId || "").trim(); if (modelId) { return modelId; } return String(model?.id || "").trim(); } export function normalizeFailoverErrorMessage(error: unknown): string { if (!error) return ""; if (typeof error === "string") return error; if (error instanceof Error) return error.message || String(error); try { return JSON.stringify(error); } catch { return String(error); } } export function shouldCooldownImageFallbackModel(errorMessage: string): boolean { const msg = errorMessage.toLowerCase(); return ( msg.includes("model not found") || msg.includes("model_not_found") || msg.includes("no available channel for model") || msg.includes("unknown model") || msg.includes("invalid model") || msg.includes("api key is missing") || msg.includes("invalid api key") || msg.includes("unauthorized") || msg.includes("cannot convert argument to a bytestring") || msg.includes("model maintenance") || msg.includes("maintenance") || msg.includes("维护中") || msg.includes("维护") || msg.includes("service unavailable") || msg.includes("temporarily unavailable") || msg.includes("系统繁忙") || msg.includes("无可用渠道") || /(?:^|\D)(?:429|500|502|503|504)(?:\D|$)/.test(msg) ); } export function isImageFallbackModelCoolingDown(modelId: string): boolean { const snapshot = imageFailoverModelCooldown.get(modelId); if (!snapshot) return false; if (snapshot.until <= Date.now()) { imageFailoverModelCooldown.delete(modelId); return false; } return true; } export function setImageFallbackModelCooldown(modelId: string, reason: string): void { imageFailoverModelCooldown.set(modelId, { until: Date.now() + IMAGE_FAILOVER_MODEL_COOLDOWN_MS, reason: reason.slice(0, 500), }); } export function isImageFailoverExcludedModel(model: { excludeFromFailover?: boolean } | null | undefined) { return Boolean(model?.excludeFromFailover); } export type ImageFallbackCandidate = { model: any; provider: any; configured: boolean; sameProvider: boolean; nameSimilarity: number; sameName: boolean; }; const IMAGE_FAILOVER_SAME_NAME_THRESHOLD = 0.5; function getModelRefKeys(item: { model?: any } | null | undefined) { return [ String(item?.model?.id || "").trim(), String(item?.model?.modelId || "").trim(), ].filter(Boolean); } function normalizeModelNameForSimilarity(value: unknown) { return String(value || "") .trim() .toLowerCase() .replace(/[「」『』【】[\]()(){}]/g, " ") .replace(/[_/|::,,.。+]+/g, "-") .replace(/\s+/g, " "); } function tokenizeModelNameForSimilarity(value: unknown) { const normalized = normalizeModelNameForSimilarity(value); const compact = normalized.replace(/[^a-z0-9\u4e00-\u9fa5]+/g, ""); const tokens = new Set( normalized .split(/[^a-z0-9\u4e00-\u9fa5]+/g) .map((item) => item.trim()) .filter((item) => item.length >= 2) ); if (compact.length >= 2) { tokens.add(compact); for (let index = 0; index < compact.length - 1; index += 1) { tokens.add(compact.slice(index, index + 2)); } } return tokens; } function scoreModelNameSimilarity( source: { model?: any } | null | undefined, candidate: { model?: any } | null | undefined ) { const sourceNames = [ source?.model?.displayName, source?.model?.modelId, ].filter(Boolean); const candidateNames = [ candidate?.model?.displayName, candidate?.model?.modelId, ].filter(Boolean); let bestScore = 0; for (const sourceName of sourceNames) { const sourceTokens = tokenizeModelNameForSimilarity(sourceName); if (sourceTokens.size === 0) continue; for (const candidateName of candidateNames) { const candidateTokens = tokenizeModelNameForSimilarity(candidateName); if (candidateTokens.size === 0) continue; let overlap = 0; sourceTokens.forEach((token) => { if (candidateTokens.has(token)) overlap += 1; }); const score = overlap / Math.max(sourceTokens.size, candidateTokens.size); bestScore = Math.max(bestScore, score); } } return bestScore; } function resolveCurrentImageModelEntry( allModels: Array<{ model: any; provider: any }>, refs: string[] ) { const normalizedRefs = new Set(refs.map((item) => String(item || "").trim()).filter(Boolean)); if (normalizedRefs.size === 0) return null; return ( allModels.find((item) => getModelRefKeys(item).some((key) => normalizedRefs.has(key)) ) || null ); } function messagePartHasImageInput(part: any) { if (!part || typeof part !== "object") return false; if (part.type === "file" || part.type === "image" || part.type === "image_url") { return true; } if (typeof part.url === "string" && /^https?:\/\//i.test(part.url)) { return true; } if (typeof part.image === "string" && part.image.trim()) { return true; } if (typeof part.image_url?.url === "string" && part.image_url.url.trim()) { return true; } return false; } function countImageInputPartsInMessage(message: any) { const countParts = (parts: any[] | undefined) => Array.isArray(parts) ? parts.filter(messagePartHasImageInput).length : 0; const contentCount = countParts(Array.isArray(message?.content) ? message.content : undefined); if (contentCount > 0) return contentCount; return countParts(Array.isArray(message?.parts) ? message.parts : undefined); } export function countChatMessageImageInputs(messages: ChatMessage[]) { return messages.reduce( (count, message) => count + countImageInputPartsInMessage(message), 0 ); } export function chatMessagesHaveImageInput(messages: ChatMessage[]) { return countChatMessageImageInputs(messages) > 0; } export function getImageFailoverConvertProvider(providerType?: string | null) { switch (providerType) { case "bfl-image": case "ruxa-image": case "fal-image": case "snapgen-image": case "openai-image": case "bananaimage": case "stepfun-image": case "midjourney": case "runway": case "krea": case "gemini": case "huanke-gemini": return providerType; case "grsai-gpt-image": return "gemini"; case "volcano-vision": return "volcano"; default: return "alibaba-image"; } } export function buildImageFallbackCandidates( allModels: Array<{ model: any; provider: any }>, options: { currentProviderId?: string; currentModelId?: string; currentModelKey?: string; configuredFallbackIds?: string[]; } ) { const { currentProviderId, currentModelId, currentModelKey, configuredFallbackIds = [] } = options; const currentEntry = resolveCurrentImageModelEntry(allModels, [ currentModelId || "", currentModelKey || "", ]); const configuredRefs = configuredFallbackIds .map((item) => String(item || "").trim()) .filter(Boolean); const byModelRef = new Map(); const isEligible = (item: { model: any; provider: any }, sameProvider: boolean) => { if ( !item?.model?.supportsImageGen || !item?.model?.isEnabled || item?.model?.isSubModel || isImageFailoverExcludedModel(item?.model) || item?.model?.modelId === currentModelKey || item?.model?.id === currentModelId ) { return false; } const providerType = String(item?.provider?.type || ""); if (!sameProvider && !IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(providerType)) { return false; } if (isImageFallbackModelCoolingDown(item.model.id)) { const cooldown = imageFailoverModelCooldown.get(item.model.id); chatApiLogger.info(`[Chat API] ⏭ Skip configured image fallback on cooldown: ${item.model.displayName} (${providerType}) - ${cooldown?.reason || "cooldown"}`); return false; } return true; }; allModels.forEach((item) => { if (!item?.model) return; byModelRef.set(String(item.model.id), item); if (item.model.modelId) { byModelRef.set(String(item.model.modelId), item); } }); const candidates: ImageFallbackCandidate[] = []; const seenModelIds = new Set(); const candidateOrder = new Map(); const pushCandidate = (item: { model: any; provider: any } | undefined, configured: boolean) => { if (!item?.model) return; const sameProvider = Boolean(currentProviderId && item.provider?.id === currentProviderId); if (!isEligible(item, sameProvider)) return; const modelId = String(item.model.id || ""); if (!modelId || seenModelIds.has(modelId)) return; seenModelIds.add(modelId); candidateOrder.set(modelId, candidateOrder.size); const nameSimilarity = scoreModelNameSimilarity(currentEntry, item); candidates.push({ model: item.model, provider: item.provider, configured, sameProvider, nameSimilarity, sameName: nameSimilarity >= IMAGE_FAILOVER_SAME_NAME_THRESHOLD, }); }; configuredRefs.forEach((modelRef) => { pushCandidate(byModelRef.get(modelRef), true); }); if (currentProviderId) { allModels .filter((item) => item.provider?.id === currentProviderId) .forEach((item) => pushCandidate(item, false)); } if (currentEntry) { allModels .filter((item) => scoreModelNameSimilarity(currentEntry, item) >= IMAGE_FAILOVER_SAME_NAME_THRESHOLD) .forEach((item) => pushCandidate(item, false)); } return candidates.sort((left, right) => { if (left.sameName !== right.sameName) return left.sameName ? -1 : 1; if (left.configured !== right.configured) return left.configured ? -1 : 1; if (left.sameProvider !== right.sameProvider) return left.sameProvider ? -1 : 1; if (right.nameSimilarity !== left.nameSimilarity) { return right.nameSimilarity - left.nameSimilarity; } return ( (candidateOrder.get(String(left.model.id || "")) ?? 0) - (candidateOrder.get(String(right.model.id || "")) ?? 0) ); }); } export async function generateImageWithFailoverCandidate(params: { candidate: ImageFallbackCandidate; messages: ChatMessage[]; imageParams: any; fallbackImageParams: any; }) { const { candidate, messages, imageParams, fallbackImageParams } = params; const providerType = String(candidate.provider?.type || ""); const candidateRuntimeModelId = resolveImageModelRuntimeId(candidate.model); if ( chatMessagesHaveImageInput(messages) && modelLacksReferenceImageInputSupport(candidate.model, candidate.provider) ) { throw new Error( `备用模型不支持参考图/图生图输入:${candidate.model.displayName || candidate.model.id}` ); } if ( providerType === "fal-image" && isFalGptImage2EditModelId(candidateRuntimeModelId) ) { throw new Error("FAL GPT Image 2 Edit 仅支持异步队列调用,已跳过同步故障迁移"); } const candidateLanguageModel = await getLanguageModelById( candidate.model.id, candidateRuntimeModelId ); if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") { throw new Error(`备用模型不支持当前生图故障迁移调用:${candidate.model.displayName || candidate.model.id}`); } const candidateMessages = await convertMessagesWithImages( messages, getImageFailoverConvertProvider(providerType) ); const effectiveImageParams = imageParams || fallbackImageParams; const imageInputCount = countChatMessageImageInputs(messages); chatApiLogger.info("[Chat API] Image failover candidate params:", { model: candidate.model?.modelId || candidate.model?.id, providerType, aspectRatio: effectiveImageParams?.aspectRatio, imageSize: effectiveImageParams?.imageSize, imageCount: effectiveImageParams?.imageCount, hasImageInput: imageInputCount > 0, imageInputCount, multiImageInput: imageInputCount > 1, }); const candidateResult = await (candidateLanguageModel as any).doGenerate({ messages: candidateMessages, imageParams: effectiveImageParams, imageGenConfig: candidate.model.imageGenConfig || null, modelDisplayName: candidate.model.displayName, overrideModelId: candidateRuntimeModelId || candidate.model.modelId, }); if (candidateResult?.finishReason === "error") { throw new Error(candidateResult.text || "图片生成失败"); } return { text: candidateResult.text || "", usedModelId: String(candidate.model.modelId || candidate.model.id || ""), displayName: String(candidate.model.displayName || candidate.model.modelId || candidate.model.id || ""), }; } export function getImageCrossProviderCandidates( allModels: any[], options: { currentModelId: string; currentModelKey?: string; currentProviderId?: string; providerPriority: Record; } ) { const currentEntry = resolveCurrentImageModelEntry(allModels, [ options.currentModelId, options.currentModelKey || "", ]); return allModels .filter((m) => { if ( !m?.model?.supportsImageGen || !m?.model?.isEnabled || m?.model?.isSubModel || isImageFailoverExcludedModel(m?.model) ) { return false; } if (!IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(String(m?.provider?.type || ""))) { return false; } if (m.model.id === options.currentModelId) return false; if (options.currentProviderId && m.provider.id === options.currentProviderId) return false; if (isImageFallbackModelCoolingDown(m.model.id)) { const cooldown = imageFailoverModelCooldown.get(m.model.id); chatApiLogger.info(`[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "cooldown"}`); return false; } return true; }) .map((item, index) => { const nameSimilarity = scoreModelNameSimilarity(currentEntry, item); return { ...item, __imageFailoverIndex: index, __imageFailoverNameSimilarity: nameSimilarity, __imageFailoverSameName: nameSimilarity >= IMAGE_FAILOVER_SAME_NAME_THRESHOLD, }; }) .sort((a, b) => { if (a.__imageFailoverSameName !== b.__imageFailoverSameName) { return a.__imageFailoverSameName ? -1 : 1; } if (b.__imageFailoverNameSimilarity !== a.__imageFailoverNameSimilarity) { return b.__imageFailoverNameSimilarity - a.__imageFailoverNameSimilarity; } const left = options.providerPriority[a.provider.type as string] ?? 99; const right = options.providerPriority[b.provider.type as string] ?? 99; if (left !== right) return left - right; const sortOrderDiff = (a.model.sortOrder ?? 0) - (b.model.sortOrder ?? 0); if (sortOrderDiff !== 0) return sortOrderDiff; return (a.__imageFailoverIndex ?? 0) - (b.__imageFailoverIndex ?? 0); }) .map((item) => { const { __imageFailoverIndex, __imageFailoverNameSimilarity, __imageFailoverSameName, ...candidate } = item; return candidate; }); }