import "server-only"; import { and, desc, eq, sql } from "drizzle-orm"; import { db } from "@/lib/db"; import { aiModel, aiModelHealthEvent, type AiModel, } from "@/lib/db/schema"; export type AiModelHealthEventType = | "success" | "timeout" | "quota" | "unavailable" | "invalid_model" | "download_error" | "error"; export type AiModelHealthSeverity = "info" | "warning" | "error" | "critical"; const AUTO_DISABLE_FAILURE_THRESHOLD = 3; const RECENT_EVENT_LIMIT = 80; function normalizeText(value: unknown) { return String(value || "").trim(); } function getErrorMessage(error: unknown) { if (error instanceof Error) return normalizeText(error.message); return normalizeText(error); } export function classifyAiModelHealthFailure(error: unknown): { eventType: AiModelHealthEventType; severity: AiModelHealthSeverity; counted: boolean; reason: string; } { const message = getErrorMessage(error); const normalized = message.toLowerCase(); if (!message) { return { eventType: "error", severity: "error", counted: true, reason: "unknown_error", }; } if ( normalized.includes("生成结果未通过替换质检") || normalized.includes("quality check") || normalized.includes("review result") || normalized.includes("content policy") || normalized.includes("safety") || normalized.includes("user abort") || normalized.includes("aborted") || normalized.includes("cancelled") ) { return { eventType: "error", severity: "warning", counted: false, reason: "non_provider_failure", }; } if ( normalized.includes("model not exist") || normalized.includes("model_not_found") || normalized.includes("model not found") || normalized.includes("unknown model") ) { return { eventType: "invalid_model", severity: "critical", counted: true, reason: "invalid_model", }; } if ( normalized.includes("insufficient balance") || normalized.includes("quota") || normalized.includes("exceeded_current_quota") || normalized.includes("pre_consume_token_quota_failed") || normalized.includes("token quota is not enough") ) { return { eventType: "quota", severity: "critical", counted: true, reason: "quota_or_balance", }; } if ( normalized.includes("timeout") || normalized.includes("timed out") || normalized.includes("body timeout") || normalized.includes("headers timeout") ) { return { eventType: "timeout", severity: "critical", counted: true, reason: "timeout", }; } if ( normalized.includes("503") || normalized.includes("502") || normalized.includes("504") || normalized.includes("no available channel") || normalized.includes("no supported channel") || normalized.includes("service unavailable") || normalized.includes("fetch failed") ) { return { eventType: "unavailable", severity: "critical", counted: true, reason: "provider_unavailable", }; } if ( normalized.includes("download") || normalized.includes("no image url") || normalized.includes("no image") || normalized.includes("图片生成失败") ) { return { eventType: "download_error", severity: "error", counted: true, reason: "result_download_or_parse_failed", }; } return { eventType: "error", severity: "error", counted: true, reason: "provider_error", }; } export function shouldFailoverAiModelError(error: unknown) { return classifyAiModelHealthFailure(error).counted; } function buildMetadata(value?: Record) { if (!value || typeof value !== "object") return null; try { return JSON.parse(JSON.stringify(value)); } catch { return null; } } export async function recordAiModelHealthSuccess(params: { model?: Pick | null; }) { const modelId = normalizeText(params.model?.id); if (!modelId) return; await db .update(aiModel) .set({ healthStatus: params.model?.healthStatus === "auto_disabled" ? "auto_disabled" : "healthy", healthConsecutiveFailures: 0, healthLastEventType: "success", healthLastError: null, healthLastSuccessAt: new Date(), healthUpdatedAt: new Date(), }) .where(eq(aiModel.id, modelId)); } export async function recordAiModelHealthFailure(params: { model?: (Partial & Pick) | null; provider?: { id?: string | null; name?: string | null; type?: string | null; } | null; error: unknown; context?: string; elapsedMs?: number; requestId?: string; metadata?: Record; }) { const modelId = normalizeText(params.model?.id); if (!modelId) return; const classification = classifyAiModelHealthFailure(params.error); const errorMessage = getErrorMessage(params.error).slice(0, 1800); const now = new Date(); const [existing] = await db .select({ id: aiModel.id, isEnabled: aiModel.isEnabled, excludeFromFailover: aiModel.excludeFromFailover, healthStatus: aiModel.healthStatus, healthConsecutiveFailures: aiModel.healthConsecutiveFailures, }) .from(aiModel) .where(eq(aiModel.id, modelId)) .limit(1); if (!existing) return; const nextConsecutiveFailures = classification.counted ? Math.max(1, (existing.healthConsecutiveFailures || 0) + 1) : existing.healthConsecutiveFailures || 0; const shouldAutoDisable = classification.counted && !existing.excludeFromFailover && existing.healthStatus !== "auto_disabled" && nextConsecutiveFailures >= AUTO_DISABLE_FAILURE_THRESHOLD; const nextStatus = shouldAutoDisable ? "auto_disabled" : classification.counted ? "degraded" : existing.healthStatus; const autoDisabledReason = shouldAutoDisable ? `${classification.reason}: ${errorMessage}`.slice(0, 1800) : undefined; await db .insert(aiModelHealthEvent) .values({ modelId, providerId: normalizeText(params.provider?.id) || null, modelRuntimeId: normalizeText(params.model?.modelId) || null, modelDisplayName: normalizeText(params.model?.displayName) || null, providerType: normalizeText(params.provider?.type) || null, providerName: normalizeText(params.provider?.name) || null, eventType: classification.eventType, severity: classification.severity, context: normalizeText(params.context).slice(0, 96) || null, counted: classification.counted, consecutiveFailures: nextConsecutiveFailures, autoDisabled: shouldAutoDisable, elapsedMs: typeof params.elapsedMs === "number" && Number.isFinite(params.elapsedMs) ? Math.max(0, Math.round(params.elapsedMs)) : null, requestId: normalizeText(params.requestId).slice(0, 128) || null, errorMessage, metadata: buildMetadata(params.metadata), }); const updateData: Partial = { isEnabled: shouldAutoDisable ? false : existing.isEnabled, healthStatus: nextStatus, healthConsecutiveFailures: nextConsecutiveFailures, healthLastEventType: classification.eventType, healthLastError: errorMessage, healthLastFailureAt: now, healthUpdatedAt: now, updatedAt: now, }; if (shouldAutoDisable) { updateData.healthAutoDisabledAt = now; updateData.healthAutoDisabledReason = autoDisabledReason || errorMessage; } else if (existing.healthStatus !== "auto_disabled") { updateData.healthAutoDisabledReason = null; } await db.update(aiModel).set(updateData).where(eq(aiModel.id, modelId)); if (shouldAutoDisable) { console.error("[AI Model Health] Auto disabled unhealthy model:", { modelId, runtimeModelId: params.model?.modelId, displayName: params.model?.displayName, providerType: params.provider?.type, consecutiveFailures: nextConsecutiveFailures, eventType: classification.eventType, error: errorMessage, }); } } export async function restoreAiModelHealth(modelId: string) { const normalizedModelId = normalizeText(modelId); if (!normalizedModelId) return []; return await db .update(aiModel) .set({ isEnabled: true, healthStatus: "healthy", healthConsecutiveFailures: 0, healthLastEventType: "success", healthLastError: null, healthLastFailureAt: null, healthAutoDisabledAt: null, healthAutoDisabledReason: null, healthUpdatedAt: new Date(), updatedAt: new Date(), }) .where(eq(aiModel.id, normalizedModelId)) .returning(); } export async function listAiModelHealthEvents(params?: { modelId?: string; limit?: number }) { const limit = Math.max(1, Math.min(params?.limit || RECENT_EVENT_LIMIT, 200)); const modelId = normalizeText(params?.modelId); const conditions = modelId ? [eq(aiModelHealthEvent.modelId, modelId)] : []; return await db .select() .from(aiModelHealthEvent) .where(conditions.length ? and(...conditions) : undefined) .orderBy(desc(aiModelHealthEvent.createdAt)) .limit(limit); } export async function getAiModelHealthSummary() { const rows = await db .select({ modelId: aiModel.id, eventCount: sql`count(${aiModelHealthEvent.id})::int`, countedEventCount: sql`sum(case when ${aiModelHealthEvent.counted} then 1 else 0 end)::int`, lastEventAt: sql`max(${aiModelHealthEvent.createdAt})`, }) .from(aiModel) .leftJoin(aiModelHealthEvent, eq(aiModel.id, aiModelHealthEvent.modelId)) .groupBy(aiModel.id); return rows; }