import "server-only"; import { and, desc, eq, inArray, sql } from "drizzle-orm"; import { db } from "@/lib/db"; import { aiModel, aiModelHealthEvent, type AiModel, } from "@/lib/db/schema"; export type AiModelHealthEventType = | "success" | "timeout" | "quota" | "unavailable" | "invalid_model" | "download_error" | "error"; export type AiModelHealthSeverity = "info" | "warning" | "error" | "critical"; const AUTO_DISABLE_FAILURE_THRESHOLD = 3; const AUTO_DISABLE_COOLDOWN_MS = 2 * 60 * 60 * 1000; const OBSERVATION_RECOVERY_SUCCESS_THRESHOLD = 3; const RECENT_EVENT_LIMIT = 80; const RECOVERABLE_FAILURE_EVENT_TYPES = new Set([ "timeout", "quota", "unavailable", "invalid_model", "download_error", "error", ]); function normalizeText(value: unknown) { return String(value || "").trim(); } function getErrorMessage(error: unknown) { if (error instanceof Error) { const parts = [normalizeText(error.message)]; const cause = (error as Error & { cause?: unknown }).cause; if (cause instanceof Error) { parts.push(normalizeText(cause.message)); } else if (typeof cause === "string") { parts.push(normalizeText(cause)); } return parts.filter(Boolean).join(": "); } return normalizeText(error); } export function classifyAiModelHealthFailure(error: unknown): { eventType: AiModelHealthEventType; severity: AiModelHealthSeverity; counted: boolean; reason: string; } { const message = getErrorMessage(error); const normalized = message.toLowerCase(); if (!message) { return { eventType: "error", severity: "error", counted: true, reason: "unknown_error", }; } if ( normalized.includes("生成结果未通过替换质检") || normalized.includes("quality check") || normalized.includes("review result") || normalized.includes("content policy") || normalized.includes("safety") || normalized.includes("user abort") || normalized.includes("aborted") || normalized.includes("cancelled") ) { return { eventType: "error", severity: "warning", counted: false, reason: "non_provider_failure", }; } if ( normalized.includes("model not exist") || normalized.includes("model_not_found") || normalized.includes("model not found") || normalized.includes("unknown model") ) { return { eventType: "invalid_model", severity: "critical", counted: true, reason: "invalid_model", }; } if ( normalized.includes("insufficient balance") || normalized.includes("quota") || normalized.includes("exceeded_current_quota") || normalized.includes("pre_consume_token_quota_failed") || normalized.includes("token quota is not enough") ) { return { eventType: "quota", severity: "critical", counted: true, reason: "quota_or_balance", }; } if ( normalized.includes("timeout") || normalized.includes("timed out") || normalized.includes("body timeout") || normalized.includes("headers timeout") ) { return { eventType: "timeout", severity: "critical", counted: true, reason: "timeout", }; } if ( normalized.includes("503") || normalized.includes("502") || normalized.includes("504") || normalized.includes("model maintenance") || normalized.includes("maintenance") || normalized.includes("修复中") || normalized.includes("系统繁忙") || normalized.includes("no available channel") || normalized.includes("no supported channel") || normalized.includes("service unavailable") || normalized.includes("fetch failed") ) { return { eventType: "unavailable", severity: "critical", counted: true, reason: "provider_unavailable", }; } if ( normalized.includes("download") || normalized.includes("no image url") || normalized.includes("no image") || normalized.includes("图片生成失败") ) { return { eventType: "download_error", severity: "error", counted: true, reason: "result_download_or_parse_failed", }; } return { eventType: "error", severity: "error", counted: true, reason: "provider_error", }; } export function shouldFailoverAiModelError(error: unknown) { return classifyAiModelHealthFailure(error).counted; } function buildMetadata(value?: Record) { if (!value || typeof value !== "object") return null; try { return JSON.parse(JSON.stringify(value)); } catch { return null; } } function isObservationPoolModel( model: | Pick< AiModel, | "healthStatus" | "healthObservationStartedAt" > | null | undefined ) { return ( model?.healthStatus === "degraded" && Boolean(model.healthObservationStartedAt) ); } function getAutoDisableCooldownCutoff(now = new Date()) { return new Date(now.getTime() - AUTO_DISABLE_COOLDOWN_MS); } export async function reviveAiModelsFromCooldown(params?: { modelId?: string }) { const now = new Date(); const cooldownCutoff = getAutoDisableCooldownCutoff(now); const conditions = [ eq(aiModel.isEnabled, false), eq(aiModel.excludeFromFailover, false), eq(aiModel.excludeFromHealthBan, false), ]; if (params?.modelId) { conditions.push(eq(aiModel.id, normalizeText(params.modelId))); } const candidates = await db .select({ id: aiModel.id, modelId: aiModel.modelId, displayName: aiModel.displayName, healthStatus: aiModel.healthStatus, healthConsecutiveFailures: aiModel.healthConsecutiveFailures, healthLastEventType: aiModel.healthLastEventType, healthAutoDisabledAt: aiModel.healthAutoDisabledAt, healthLastFailureAt: aiModel.healthLastFailureAt, healthUpdatedAt: aiModel.healthUpdatedAt, }) .from(aiModel) .where(and(...conditions)); const dueModels = candidates.filter((model) => { const cooldownStartedAt = model.healthAutoDisabledAt || model.healthLastFailureAt || model.healthUpdatedAt; if (!cooldownStartedAt || cooldownStartedAt > cooldownCutoff) { return false; } if (model.healthStatus === "auto_disabled") { return true; } return ( model.healthStatus === "degraded" && Boolean(model.healthLastFailureAt) && (model.healthConsecutiveFailures || 0) > 0 && RECOVERABLE_FAILURE_EVENT_TYPES.has(model.healthLastEventType as AiModelHealthEventType) ); }); if (dueModels.length === 0) { return []; } const revived = await db .update(aiModel) .set({ isEnabled: true, healthStatus: "degraded", healthConsecutiveFailures: 0, healthRecoverySuccessStreak: 0, healthObservationStartedAt: now, healthUpdatedAt: now, updatedAt: now, }) .where( inArray( aiModel.id, dueModels.map((item) => item.id) ) ) .returning(); if (revived.length > 0) { console.info("[AI Model Health] Revived cooled-down models into observation pool:", { count: revived.length, models: revived.map((item) => ({ id: item.id, modelId: item.modelId, displayName: item.displayName, previousStatus: item.healthStatus, })), }); } return revived; } export async function recordAiModelHealthSuccess(params: { model?: | Pick< AiModel, | "id" | "healthStatus" | "healthRecoverySuccessStreak" | "healthObservationStartedAt" > | null; }) { const modelId = normalizeText(params.model?.id); if (!modelId) return; const [existing] = await db .select({ id: aiModel.id, healthStatus: aiModel.healthStatus, healthRecoverySuccessStreak: aiModel.healthRecoverySuccessStreak, healthObservationStartedAt: aiModel.healthObservationStartedAt, }) .from(aiModel) .where(eq(aiModel.id, modelId)) .limit(1); if (!existing) return; const now = new Date(); const isObservation = isObservationPoolModel(existing); const nextRecoverySuccessStreak = isObservation ? (existing.healthRecoverySuccessStreak || 0) + 1 : 0; const recoveredToHealthy = isObservation && nextRecoverySuccessStreak >= OBSERVATION_RECOVERY_SUCCESS_THRESHOLD; const updateData: Partial = { healthStatus: existing.healthStatus === "auto_disabled" ? "auto_disabled" : recoveredToHealthy ? "healthy" : isObservation ? "degraded" : "healthy", healthConsecutiveFailures: 0, healthLastEventType: "success", healthLastError: null, healthLastSuccessAt: now, healthRecoverySuccessStreak: recoveredToHealthy ? 0 : isObservation ? nextRecoverySuccessStreak : 0, healthObservationStartedAt: recoveredToHealthy ? null : isObservation ? existing.healthObservationStartedAt : null, healthUpdatedAt: now, updatedAt: now, }; if (recoveredToHealthy) { updateData.healthAutoDisabledAt = null; updateData.healthAutoDisabledReason = null; } await db .update(aiModel) .set(updateData) .where(eq(aiModel.id, modelId)); } export async function recordAiModelHealthFailure(params: { model?: (Partial & Pick) | null; provider?: { id?: string | null; name?: string | null; type?: string | null; } | null; error: unknown; context?: string; elapsedMs?: number; requestId?: string; metadata?: Record; }) { const modelId = normalizeText(params.model?.id); if (!modelId) return; const classification = classifyAiModelHealthFailure(params.error); const errorMessage = getErrorMessage(params.error).slice(0, 1800); const now = new Date(); const [existing] = await db .select({ id: aiModel.id, modelId: aiModel.modelId, displayName: aiModel.displayName, isEnabled: aiModel.isEnabled, excludeFromFailover: aiModel.excludeFromFailover, excludeFromHealthBan: aiModel.excludeFromHealthBan, healthStatus: aiModel.healthStatus, healthConsecutiveFailures: aiModel.healthConsecutiveFailures, healthRecoverySuccessStreak: aiModel.healthRecoverySuccessStreak, healthObservationStartedAt: aiModel.healthObservationStartedAt, healthAutoDisabledAt: aiModel.healthAutoDisabledAt, }) .from(aiModel) .where(eq(aiModel.id, modelId)) .limit(1); if (!existing) return; const isObservation = isObservationPoolModel(existing); const nextConsecutiveFailures = classification.counted ? Math.max( 1, (isObservation ? 0 : existing.healthConsecutiveFailures || 0) + 1 ) : existing.healthConsecutiveFailures || 0; const shouldAutoDisable = classification.counted && !existing.excludeFromFailover && !existing.excludeFromHealthBan && ((isObservation && existing.isEnabled) || (existing.healthStatus !== "auto_disabled" && nextConsecutiveFailures >= AUTO_DISABLE_FAILURE_THRESHOLD)); const nextStatus = shouldAutoDisable ? "auto_disabled" : classification.counted ? "degraded" : existing.healthStatus; const autoDisabledReason = shouldAutoDisable ? `${classification.reason}: ${errorMessage}`.slice(0, 1800) : undefined; await db .insert(aiModelHealthEvent) .values({ modelId, providerId: normalizeText(params.provider?.id) || null, modelRuntimeId: normalizeText(params.model?.modelId) || null, modelDisplayName: normalizeText(params.model?.displayName) || null, providerType: normalizeText(params.provider?.type) || null, providerName: normalizeText(params.provider?.name) || null, eventType: classification.eventType, severity: classification.severity, context: normalizeText(params.context).slice(0, 96) || null, counted: classification.counted, consecutiveFailures: nextConsecutiveFailures, autoDisabled: shouldAutoDisable, elapsedMs: typeof params.elapsedMs === "number" && Number.isFinite(params.elapsedMs) ? Math.max(0, Math.round(params.elapsedMs)) : null, requestId: normalizeText(params.requestId).slice(0, 128) || null, errorMessage, metadata: buildMetadata(params.metadata), }); const updateData: Partial = { isEnabled: shouldAutoDisable ? false : existing.isEnabled, healthStatus: nextStatus, healthConsecutiveFailures: nextConsecutiveFailures, healthLastEventType: classification.eventType, healthLastError: errorMessage, healthLastFailureAt: now, healthRecoverySuccessStreak: shouldAutoDisable ? 0 : classification.counted ? 0 : existing.healthRecoverySuccessStreak || 0, healthObservationStartedAt: shouldAutoDisable ? null : isObservation ? existing.healthObservationStartedAt : null, healthUpdatedAt: now, updatedAt: now, }; if (shouldAutoDisable) { updateData.healthAutoDisabledAt = now; updateData.healthAutoDisabledReason = autoDisabledReason || errorMessage; } else if (existing.healthStatus !== "auto_disabled" && !isObservation) { updateData.healthAutoDisabledReason = null; } await db.update(aiModel).set(updateData).where(eq(aiModel.id, modelId)); if (shouldAutoDisable) { console.error("[AI Model Health] Auto disabled unhealthy model:", { modelId, runtimeModelId: params.model?.modelId, displayName: params.model?.displayName, providerType: params.provider?.type, consecutiveFailures: nextConsecutiveFailures, eventType: classification.eventType, error: errorMessage, }); } } export async function restoreAiModelHealth(modelId: string) { const normalizedModelId = normalizeText(modelId); if (!normalizedModelId) return []; return await db .update(aiModel) .set({ isEnabled: true, healthStatus: "healthy", healthConsecutiveFailures: 0, healthLastEventType: "success", healthLastError: null, healthLastFailureAt: null, healthAutoDisabledAt: null, healthAutoDisabledReason: null, healthRecoverySuccessStreak: 0, healthObservationStartedAt: null, healthUpdatedAt: new Date(), updatedAt: new Date(), }) .where(eq(aiModel.id, normalizedModelId)) .returning(); } export async function listAiModelHealthEvents(params?: { modelId?: string; limit?: number }) { const limit = Math.max(1, Math.min(params?.limit || RECENT_EVENT_LIMIT, 200)); const modelId = normalizeText(params?.modelId); const conditions = modelId ? [eq(aiModelHealthEvent.modelId, modelId)] : []; return await db .select() .from(aiModelHealthEvent) .where(conditions.length ? and(...conditions) : undefined) .orderBy(desc(aiModelHealthEvent.createdAt)) .limit(limit); } export async function getAiModelHealthSummary() { const rows = await db .select({ modelId: aiModel.id, eventCount: sql`count(${aiModelHealthEvent.id})::int`, countedEventCount: sql`sum(case when ${aiModelHealthEvent.counted} then 1 else 0 end)::int`, lastEventAt: sql`max(${aiModelHealthEvent.createdAt})`, }) .from(aiModel) .leftJoin(aiModelHealthEvent, eq(aiModel.id, aiModelHealthEvent.modelId)) .groupBy(aiModel.id); return rows; }