506 lines
16 KiB
TypeScript
506 lines
16 KiB
TypeScript
import { createLogger } from "@/lib/observability/logger";
|
||
import { isFalGptImage2EditModelId } from "@/lib/ai/providers/fal-image";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { modelLacksReferenceImageInputSupport } from "@/lib/ai/reference-image-capability";
|
||
import type { ChatMessage } from "@/lib/types";
|
||
import { convertMessagesWithImages } from "./image-media";
|
||
|
||
const chatApiLogger = createLogger("chat.imageFailover");
|
||
|
||
|
||
export const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([
|
||
"alibaba-image",
|
||
"bfl-image",
|
||
"ruxa-image",
|
||
"fal-image",
|
||
"snapgen-image",
|
||
"openai-image",
|
||
"bananaimage",
|
||
"visionary",
|
||
"atlascloud",
|
||
"stepfun-image",
|
||
"runway",
|
||
"krea",
|
||
"grsai-gpt-image",
|
||
"volcano-vision",
|
||
"gemini",
|
||
"huanke-gemini",
|
||
]);
|
||
|
||
const IMAGE_FAILOVER_MODEL_COOLDOWN_MS = 30 * 60 * 1000;
|
||
const imageFailoverModelCooldown = new Map<string, { until: number; reason: string }>();
|
||
|
||
export function resolveImageModelRuntimeId(
|
||
model: { defaultParams?: unknown; modelId?: unknown; id?: unknown } | null | undefined,
|
||
) {
|
||
const defaultParams = model?.defaultParams;
|
||
const defaultEndpoint =
|
||
typeof defaultParams === "object" &&
|
||
defaultParams !== null &&
|
||
"endpoint" in defaultParams
|
||
? String((defaultParams as Record<string, unknown>).endpoint || "").trim()
|
||
: "";
|
||
|
||
if (defaultEndpoint) {
|
||
return defaultEndpoint;
|
||
}
|
||
|
||
const modelId = String(model?.modelId || "").trim();
|
||
if (modelId) {
|
||
return modelId;
|
||
}
|
||
|
||
return String(model?.id || "").trim();
|
||
}
|
||
|
||
export function normalizeFailoverErrorMessage(error: unknown): string {
|
||
if (!error) return "";
|
||
if (typeof error === "string") return error;
|
||
if (error instanceof Error) return error.message || String(error);
|
||
try {
|
||
return JSON.stringify(error);
|
||
} catch {
|
||
return String(error);
|
||
}
|
||
}
|
||
|
||
export function shouldCooldownImageFallbackModel(errorMessage: string): boolean {
|
||
const msg = errorMessage.toLowerCase();
|
||
return (
|
||
msg.includes("model not found") ||
|
||
msg.includes("model_not_found") ||
|
||
msg.includes("no available channel for model") ||
|
||
msg.includes("unknown model") ||
|
||
msg.includes("invalid model") ||
|
||
msg.includes("api key is missing") ||
|
||
msg.includes("invalid api key") ||
|
||
msg.includes("unauthorized") ||
|
||
msg.includes("cannot convert argument to a bytestring") ||
|
||
msg.includes("model maintenance") ||
|
||
msg.includes("maintenance") ||
|
||
msg.includes("维护中") ||
|
||
msg.includes("维护") ||
|
||
msg.includes("service unavailable") ||
|
||
msg.includes("temporarily unavailable") ||
|
||
msg.includes("系统繁忙") ||
|
||
msg.includes("无可用渠道") ||
|
||
/(?:^|\D)(?:429|500|502|503|504)(?:\D|$)/.test(msg)
|
||
);
|
||
}
|
||
|
||
export function isImageFallbackModelCoolingDown(modelId: string): boolean {
|
||
const snapshot = imageFailoverModelCooldown.get(modelId);
|
||
if (!snapshot) return false;
|
||
if (snapshot.until <= Date.now()) {
|
||
imageFailoverModelCooldown.delete(modelId);
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
export function setImageFallbackModelCooldown(modelId: string, reason: string): void {
|
||
imageFailoverModelCooldown.set(modelId, {
|
||
until: Date.now() + IMAGE_FAILOVER_MODEL_COOLDOWN_MS,
|
||
reason: reason.slice(0, 500),
|
||
});
|
||
}
|
||
|
||
export function isImageFailoverExcludedModel(model: { excludeFromFailover?: boolean } | null | undefined) {
|
||
return Boolean(model?.excludeFromFailover);
|
||
}
|
||
|
||
export type ImageFallbackCandidate = {
|
||
model: any;
|
||
provider: any;
|
||
configured: boolean;
|
||
sameProvider: boolean;
|
||
nameSimilarity: number;
|
||
sameName: boolean;
|
||
};
|
||
|
||
const IMAGE_FAILOVER_SAME_NAME_THRESHOLD = 0.5;
|
||
|
||
function getModelRefKeys(item: { model?: any } | null | undefined) {
|
||
return [
|
||
String(item?.model?.id || "").trim(),
|
||
String(item?.model?.modelId || "").trim(),
|
||
].filter(Boolean);
|
||
}
|
||
|
||
function normalizeModelNameForSimilarity(value: unknown) {
|
||
return String(value || "")
|
||
.trim()
|
||
.toLowerCase()
|
||
.replace(/[「」『』【】[\]()（）{}]/g, " ")
|
||
.replace(/[_/|:：,，.。+]+/g, "-")
|
||
.replace(/\s+/g, " ");
|
||
}
|
||
|
||
function tokenizeModelNameForSimilarity(value: unknown) {
|
||
const normalized = normalizeModelNameForSimilarity(value);
|
||
const compact = normalized.replace(/[^a-z0-9\u4e00-\u9fa5]+/g, "");
|
||
const tokens = new Set(
|
||
normalized
|
||
.split(/[^a-z0-9\u4e00-\u9fa5]+/g)
|
||
.map((item) => item.trim())
|
||
.filter((item) => item.length >= 2)
|
||
);
|
||
|
||
if (compact.length >= 2) {
|
||
tokens.add(compact);
|
||
for (let index = 0; index < compact.length - 1; index += 1) {
|
||
tokens.add(compact.slice(index, index + 2));
|
||
}
|
||
}
|
||
|
||
return tokens;
|
||
}
|
||
|
||
function scoreModelNameSimilarity(
|
||
source: { model?: any } | null | undefined,
|
||
candidate: { model?: any } | null | undefined
|
||
) {
|
||
const sourceNames = [
|
||
source?.model?.displayName,
|
||
source?.model?.modelId,
|
||
].filter(Boolean);
|
||
const candidateNames = [
|
||
candidate?.model?.displayName,
|
||
candidate?.model?.modelId,
|
||
].filter(Boolean);
|
||
|
||
let bestScore = 0;
|
||
for (const sourceName of sourceNames) {
|
||
const sourceTokens = tokenizeModelNameForSimilarity(sourceName);
|
||
if (sourceTokens.size === 0) continue;
|
||
for (const candidateName of candidateNames) {
|
||
const candidateTokens = tokenizeModelNameForSimilarity(candidateName);
|
||
if (candidateTokens.size === 0) continue;
|
||
let overlap = 0;
|
||
sourceTokens.forEach((token) => {
|
||
if (candidateTokens.has(token)) overlap += 1;
|
||
});
|
||
const score = overlap / Math.max(sourceTokens.size, candidateTokens.size);
|
||
bestScore = Math.max(bestScore, score);
|
||
}
|
||
}
|
||
|
||
return bestScore;
|
||
}
|
||
|
||
function resolveCurrentImageModelEntry(
|
||
allModels: Array<{ model: any; provider: any }>,
|
||
refs: string[]
|
||
) {
|
||
const normalizedRefs = new Set(refs.map((item) => String(item || "").trim()).filter(Boolean));
|
||
if (normalizedRefs.size === 0) return null;
|
||
return (
|
||
allModels.find((item) =>
|
||
getModelRefKeys(item).some((key) => normalizedRefs.has(key))
|
||
) || null
|
||
);
|
||
}
|
||
|
||
function messagePartHasImageInput(part: any) {
|
||
if (!part || typeof part !== "object") return false;
|
||
if (part.type === "file" || part.type === "image" || part.type === "image_url") {
|
||
return true;
|
||
}
|
||
if (typeof part.url === "string" && /^https?:\/\//i.test(part.url)) {
|
||
return true;
|
||
}
|
||
if (typeof part.image === "string" && part.image.trim()) {
|
||
return true;
|
||
}
|
||
if (typeof part.image_url?.url === "string" && part.image_url.url.trim()) {
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
function countImageInputPartsInMessage(message: any) {
|
||
const countParts = (parts: any[] | undefined) =>
|
||
Array.isArray(parts) ? parts.filter(messagePartHasImageInput).length : 0;
|
||
|
||
const contentCount = countParts(Array.isArray(message?.content) ? message.content : undefined);
|
||
if (contentCount > 0) return contentCount;
|
||
|
||
return countParts(Array.isArray(message?.parts) ? message.parts : undefined);
|
||
}
|
||
|
||
export function countChatMessageImageInputs(messages: ChatMessage[]) {
|
||
return messages.reduce(
|
||
(count, message) => count + countImageInputPartsInMessage(message),
|
||
0
|
||
);
|
||
}
|
||
|
||
export function chatMessagesHaveImageInput(messages: ChatMessage[]) {
|
||
return countChatMessageImageInputs(messages) > 0;
|
||
}
|
||
|
||
export function getImageFailoverConvertProvider(providerType?: string | null) {
|
||
switch (providerType) {
|
||
case "bfl-image":
|
||
case "ruxa-image":
|
||
case "fal-image":
|
||
case "snapgen-image":
|
||
case "openai-image":
|
||
case "bananaimage":
|
||
case "visionary":
|
||
case "atlascloud":
|
||
case "stepfun-image":
|
||
case "midjourney":
|
||
case "runway":
|
||
case "krea":
|
||
case "gemini":
|
||
case "huanke-gemini":
|
||
return providerType;
|
||
case "grsai-gpt-image":
|
||
return "gemini";
|
||
case "volcano-vision":
|
||
return "volcano";
|
||
default:
|
||
return "alibaba-image";
|
||
}
|
||
}
|
||
|
||
export function buildImageFallbackCandidates(
|
||
allModels: Array<{ model: any; provider: any }>,
|
||
options: {
|
||
currentProviderId?: string;
|
||
currentModelId?: string;
|
||
currentModelKey?: string;
|
||
configuredFallbackIds?: string[];
|
||
}
|
||
) {
|
||
const { currentProviderId, currentModelId, currentModelKey, configuredFallbackIds = [] } = options;
|
||
const currentEntry = resolveCurrentImageModelEntry(allModels, [
|
||
currentModelId || "",
|
||
currentModelKey || "",
|
||
]);
|
||
const configuredRefs = configuredFallbackIds
|
||
.map((item) => String(item || "").trim())
|
||
.filter(Boolean);
|
||
const byModelRef = new Map<string, { model: any; provider: any }>();
|
||
|
||
const isEligible = (item: { model: any; provider: any }, sameProvider: boolean) => {
|
||
if (
|
||
!item?.model?.supportsImageGen ||
|
||
!item?.model?.isEnabled ||
|
||
item?.model?.isSubModel ||
|
||
isImageFailoverExcludedModel(item?.model) ||
|
||
item?.model?.modelId === currentModelKey ||
|
||
item?.model?.id === currentModelId
|
||
) {
|
||
return false;
|
||
}
|
||
|
||
const providerType = String(item?.provider?.type || "");
|
||
if (!sameProvider && !IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(providerType)) {
|
||
return false;
|
||
}
|
||
if (isImageFallbackModelCoolingDown(item.model.id)) {
|
||
const cooldown = imageFailoverModelCooldown.get(item.model.id);
|
||
chatApiLogger.info(`[Chat API] ⏭ Skip configured image fallback on cooldown: ${item.model.displayName} (${providerType}) - ${cooldown?.reason || "cooldown"}`);
|
||
return false;
|
||
}
|
||
return true;
|
||
};
|
||
|
||
allModels.forEach((item) => {
|
||
if (!item?.model) return;
|
||
byModelRef.set(String(item.model.id), item);
|
||
if (item.model.modelId) {
|
||
byModelRef.set(String(item.model.modelId), item);
|
||
}
|
||
});
|
||
|
||
const candidates: ImageFallbackCandidate[] = [];
|
||
const seenModelIds = new Set<string>();
|
||
const candidateOrder = new Map<string, number>();
|
||
const pushCandidate = (item: { model: any; provider: any } | undefined, configured: boolean) => {
|
||
if (!item?.model) return;
|
||
const sameProvider = Boolean(currentProviderId && item.provider?.id === currentProviderId);
|
||
if (!isEligible(item, sameProvider)) return;
|
||
const modelId = String(item.model.id || "");
|
||
if (!modelId || seenModelIds.has(modelId)) return;
|
||
seenModelIds.add(modelId);
|
||
candidateOrder.set(modelId, candidateOrder.size);
|
||
const nameSimilarity = scoreModelNameSimilarity(currentEntry, item);
|
||
candidates.push({
|
||
model: item.model,
|
||
provider: item.provider,
|
||
configured,
|
||
sameProvider,
|
||
nameSimilarity,
|
||
sameName: nameSimilarity >= IMAGE_FAILOVER_SAME_NAME_THRESHOLD,
|
||
});
|
||
};
|
||
|
||
configuredRefs.forEach((modelRef) => {
|
||
pushCandidate(byModelRef.get(modelRef), true);
|
||
});
|
||
|
||
if (currentProviderId) {
|
||
allModels
|
||
.filter((item) => item.provider?.id === currentProviderId)
|
||
.forEach((item) => pushCandidate(item, false));
|
||
}
|
||
|
||
if (currentEntry) {
|
||
allModels
|
||
.filter((item) => scoreModelNameSimilarity(currentEntry, item) >= IMAGE_FAILOVER_SAME_NAME_THRESHOLD)
|
||
.forEach((item) => pushCandidate(item, false));
|
||
}
|
||
|
||
return candidates.sort((left, right) => {
|
||
if (left.sameName !== right.sameName) return left.sameName ? -1 : 1;
|
||
if (left.configured !== right.configured) return left.configured ? -1 : 1;
|
||
if (left.sameProvider !== right.sameProvider) return left.sameProvider ? -1 : 1;
|
||
if (right.nameSimilarity !== left.nameSimilarity) {
|
||
return right.nameSimilarity - left.nameSimilarity;
|
||
}
|
||
return (
|
||
(candidateOrder.get(String(left.model.id || "")) ?? 0) -
|
||
(candidateOrder.get(String(right.model.id || "")) ?? 0)
|
||
);
|
||
});
|
||
}
|
||
|
||
export async function generateImageWithFailoverCandidate(params: {
|
||
candidate: ImageFallbackCandidate;
|
||
messages: ChatMessage[];
|
||
imageParams: any;
|
||
fallbackImageParams: any;
|
||
}) {
|
||
const { candidate, messages, imageParams, fallbackImageParams } = params;
|
||
const providerType = String(candidate.provider?.type || "");
|
||
const candidateRuntimeModelId = resolveImageModelRuntimeId(candidate.model);
|
||
if (
|
||
chatMessagesHaveImageInput(messages) &&
|
||
modelLacksReferenceImageInputSupport(candidate.model, candidate.provider)
|
||
) {
|
||
throw new Error(
|
||
`备用模型不支持参考图/图生图输入：${candidate.model.displayName || candidate.model.id}`
|
||
);
|
||
}
|
||
if (
|
||
providerType === "fal-image" &&
|
||
isFalGptImage2EditModelId(candidateRuntimeModelId)
|
||
) {
|
||
throw new Error("FAL GPT Image 2 Edit 仅支持异步队列调用，已跳过同步故障迁移");
|
||
}
|
||
const candidateLanguageModel = await getLanguageModelById(
|
||
candidate.model.id,
|
||
candidateRuntimeModelId
|
||
);
|
||
if (!candidateLanguageModel || typeof (candidateLanguageModel as any).doGenerate !== "function") {
|
||
throw new Error(`备用模型不支持当前生图故障迁移调用：${candidate.model.displayName || candidate.model.id}`);
|
||
}
|
||
|
||
const candidateMessages = await convertMessagesWithImages(
|
||
messages,
|
||
getImageFailoverConvertProvider(providerType)
|
||
);
|
||
const effectiveImageParams = imageParams || fallbackImageParams;
|
||
const imageInputCount = countChatMessageImageInputs(messages);
|
||
chatApiLogger.info("[Chat API] Image failover candidate params:", {
|
||
model: candidate.model?.modelId || candidate.model?.id,
|
||
providerType,
|
||
aspectRatio: effectiveImageParams?.aspectRatio,
|
||
imageSize: effectiveImageParams?.imageSize,
|
||
imageCount: effectiveImageParams?.imageCount,
|
||
hasImageInput: imageInputCount > 0,
|
||
imageInputCount,
|
||
multiImageInput: imageInputCount > 1,
|
||
});
|
||
const candidateResult = await (candidateLanguageModel as any).doGenerate({
|
||
messages: candidateMessages,
|
||
imageParams: effectiveImageParams,
|
||
imageGenConfig: candidate.model.imageGenConfig || null,
|
||
modelDisplayName: candidate.model.displayName,
|
||
overrideModelId: candidateRuntimeModelId || candidate.model.modelId,
|
||
});
|
||
|
||
if (candidateResult?.finishReason === "error") {
|
||
throw new Error(candidateResult.text || "图片生成失败");
|
||
}
|
||
|
||
return {
|
||
text: candidateResult.text || "",
|
||
usedModelId: String(candidate.model.modelId || candidate.model.id || ""),
|
||
displayName: String(candidate.model.displayName || candidate.model.modelId || candidate.model.id || ""),
|
||
};
|
||
}
|
||
|
||
export function getImageCrossProviderCandidates(
|
||
allModels: any[],
|
||
options: {
|
||
currentModelId: string;
|
||
currentModelKey?: string;
|
||
currentProviderId?: string;
|
||
providerPriority: Record<string, number>;
|
||
}
|
||
) {
|
||
const currentEntry = resolveCurrentImageModelEntry(allModels, [
|
||
options.currentModelId,
|
||
options.currentModelKey || "",
|
||
]);
|
||
return allModels
|
||
.filter((m) => {
|
||
if (
|
||
!m?.model?.supportsImageGen ||
|
||
!m?.model?.isEnabled ||
|
||
m?.model?.isSubModel ||
|
||
isImageFailoverExcludedModel(m?.model)
|
||
) {
|
||
return false;
|
||
}
|
||
if (!IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(String(m?.provider?.type || ""))) {
|
||
return false;
|
||
}
|
||
if (m.model.id === options.currentModelId) return false;
|
||
if (options.currentProviderId && m.provider.id === options.currentProviderId) return false;
|
||
if (isImageFallbackModelCoolingDown(m.model.id)) {
|
||
const cooldown = imageFailoverModelCooldown.get(m.model.id);
|
||
chatApiLogger.info(`[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "cooldown"}`);
|
||
return false;
|
||
}
|
||
return true;
|
||
})
|
||
.map((item, index) => {
|
||
const nameSimilarity = scoreModelNameSimilarity(currentEntry, item);
|
||
return {
|
||
...item,
|
||
__imageFailoverIndex: index,
|
||
__imageFailoverNameSimilarity: nameSimilarity,
|
||
__imageFailoverSameName:
|
||
nameSimilarity >= IMAGE_FAILOVER_SAME_NAME_THRESHOLD,
|
||
};
|
||
})
|
||
.sort((a, b) => {
|
||
if (a.__imageFailoverSameName !== b.__imageFailoverSameName) {
|
||
return a.__imageFailoverSameName ? -1 : 1;
|
||
}
|
||
if (b.__imageFailoverNameSimilarity !== a.__imageFailoverNameSimilarity) {
|
||
return b.__imageFailoverNameSimilarity - a.__imageFailoverNameSimilarity;
|
||
}
|
||
const left = options.providerPriority[a.provider.type as string] ?? 99;
|
||
const right = options.providerPriority[b.provider.type as string] ?? 99;
|
||
if (left !== right) return left - right;
|
||
const sortOrderDiff = (a.model.sortOrder ?? 0) - (b.model.sortOrder ?? 0);
|
||
if (sortOrderDiff !== 0) return sortOrderDiff;
|
||
return (a.__imageFailoverIndex ?? 0) - (b.__imageFailoverIndex ?? 0);
|
||
})
|
||
.map((item) => {
|
||
const {
|
||
__imageFailoverIndex,
|
||
__imageFailoverNameSimilarity,
|
||
__imageFailoverSameName,
|
||
...candidate
|
||
} = item;
|
||
return candidate;
|
||
});
|
||
}
|