feat: add ai model health failover

app/(admin)/settings/admin-settings-client.tsx

@@ -855,6 +855,14 @@ type AiModel = {
   fallbackModelId1?: string;
   fallbackModelId2?: string;
   excludeFromFailover?: boolean;
+  healthStatus?: "healthy" | "degraded" | "auto_disabled";
+  healthConsecutiveFailures?: number;
+  healthLastEventType?: string | null;
+  healthLastError?: string | null;
+  healthLastFailureAt?: string | null;
+  healthLastSuccessAt?: string | null;
+  healthAutoDisabledAt?: string | null;
+  healthAutoDisabledReason?: string | null;
   isAgentModel?: boolean;
   agentCodingModelId?: string;
   isMultimodal?: boolean;
@@ -862,6 +870,26 @@ type AiModel = {
   imageGenConfig?: ImageGenConfig; // 🆕 图片生成特性配置
 };
 
+type AiModelHealthEvent = {
+  id: string;
+  modelId?: string | null;
+  providerId?: string | null;
+  modelRuntimeId?: string | null;
+  modelDisplayName?: string | null;
+  providerType?: string | null;
+  providerName?: string | null;
+  eventType: string;
+  severity: string;
+  context?: string | null;
+  counted: boolean;
+  consecutiveFailures: number;
+  autoDisabled: boolean;
+  elapsedMs?: number | null;
+  requestId?: string | null;
+  errorMessage?: string | null;
+  createdAt: string;
+};
+
 type StorageConfig = {
   id: string;
   name: string;
@@ -1552,6 +1580,9 @@ export function AdminSettingsClient() {
   const { data: modelsData, mutate: mutateModels } = useSWR<{
     models: AiModel[];
   }>("/api/admin/ai-models", fetcher);
+  const { data: modelHealthData, mutate: mutateModelHealth } = useSWR<{
+    events: AiModelHealthEvent[];
+  }>("/api/admin/ai-model-health?limit=120", fetcher);
   const { data: storageData, mutate: mutateStorage } = useSWR<{
     configs: StorageConfig[];
   }>("/api/admin/storage", fetcher);
@@ -1682,6 +1713,26 @@ export function AdminSettingsClient() {
     });
     return map;
   }, [providersData?.providers]);
+  const modelHealthEventsByModelId = useMemo(() => {
+    const map = new Map<string, AiModelHealthEvent[]>();
+    (modelHealthData?.events || []).forEach((event) => {
+      const modelId = String(event.modelId || "");
+      if (!modelId) return;
+      const list = map.get(modelId) || [];
+      if (list.length < 5) {
+        list.push(event);
+      }
+      map.set(modelId, list);
+    });
+    return map;
+  }, [modelHealthData?.events]);
+  const recentCriticalModelHealthEvents = useMemo(
+    () =>
+      (modelHealthData?.events || [])
+        .filter((event) => event.counted || event.autoDisabled)
+        .slice(0, 12),
+    [modelHealthData?.events]
+  );
 
   // 网站设置状态
   const [siteSettings, setSiteSettings] = useState<SiteSettings>({
@@ -7018,6 +7069,25 @@ export function AdminSettingsClient() {
     }
   };
 
+  const handleRestoreModelHealth = async (model: AiModel) => {
+    try {
+      const response = await fetch("/api/admin/ai-model-health", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ modelId: model.id }),
+      });
+      const data = await response.json().catch(() => ({}));
+      if (!response.ok) {
+        throw new Error(data?.error || "恢复模型失败");
+      }
+      toast.success(`已恢复 ${model.displayName}，失败计数已清零`);
+      mutateModels();
+      mutateModelHealth();
+    } catch (error: any) {
+      toast.error(error?.message || "恢复模型失败");
+    }
+  };
+
   // --- 存储配置逻辑 ---
   const resetStorageForm = () => {
     setStorageForm({
@@ -15519,6 +15589,77 @@ export function AdminSettingsClient() {
             </CardHeader>
             <CardContent>
               <div className="space-y-4">
+                <div className="rounded-lg border bg-muted/20 p-4">
+                  <div className="flex items-center justify-between gap-3">
+                    <div>
+                      <div className="text-sm font-medium">模型故障日志</div>
+                      <p className="text-xs text-muted-foreground">
+                        只记录真实模型故障和严重超时；质检不通过不会触发自动禁用。
+                      </p>
+                    </div>
+                    <Button
+                      type="button"
+                      variant="outline"
+                      size="sm"
+                      onClick={() => {
+                        mutateModels();
+                        mutateModelHealth();
+                      }}
+                    >
+                      <RefreshCw className="mr-2 h-4 w-4" />
+                      刷新
+                    </Button>
+                  </div>
+                  <div className="mt-3 space-y-2">
+                    {recentCriticalModelHealthEvents.length > 0 ? (
+                      recentCriticalModelHealthEvents.map((event) => (
+                        <div
+                          key={event.id}
+                          className="rounded-md border bg-background px-3 py-2 text-xs"
+                        >
+                          <div className="flex flex-wrap items-center gap-2">
+                            <span className="font-medium">
+                              {event.modelDisplayName || event.modelRuntimeId || "未知模型"}
+                            </span>
+                            <span className="rounded bg-red-100 px-2 py-0.5 text-red-700">
+                              {event.eventType}
+                            </span>
+                            {event.autoDisabled && (
+                              <span className="rounded bg-zinc-900 px-2 py-0.5 text-white">
+                                已自动禁用
+                              </span>
+                            )}
+                            <span className="text-muted-foreground">
+                              连续失败 {event.consecutiveFailures} 次
+                            </span>
+                            {event.elapsedMs ? (
+                              <span className="text-muted-foreground">
+                                耗时 {Math.round(event.elapsedMs / 1000)} 秒
+                              </span>
+                            ) : null}
+                            <span className="text-muted-foreground">
+                              {formatBeijingTime(event.createdAt, {
+                                month: "2-digit",
+                                day: "2-digit",
+                                hour: "2-digit",
+                                minute: "2-digit",
+                              })}
+                            </span>
+                          </div>
+                          {event.errorMessage && (
+                            <div className="mt-1 line-clamp-2 text-muted-foreground">
+                              {event.errorMessage}
+                            </div>
+                          )}
+                        </div>
+                      ))
+                    ) : (
+                      <div className="rounded-md border border-dashed px-3 py-4 text-center text-xs text-muted-foreground">
+                        暂无模型故障日志。
+                      </div>
+                    )}
+                  </div>
+                </div>
                 {modelsData?.models?.map((model) => {
                   const provider = providersData?.providers?.find(
                     (p) => p.id === model.providerId
@@ -15535,6 +15676,10 @@ export function AdminSettingsClient() {
                     capabilities: (model as any).capabilities,
                     supportsImageGen: model.supportsImageGen,
                   });
+                  const healthStatus = model.healthStatus || "healthy";
+                  const modelHealthEvents =
+                    modelHealthEventsByModelId.get(model.id) || [];
+                  const latestHealthEvent = modelHealthEvents[0];
                   return (
                     <div
                       key={model.id}
@@ -15604,14 +15749,62 @@ export function AdminSettingsClient() {
                                 不参与故障迁移
                               </span>
                             )}
+                            {healthStatus === "auto_disabled" && (
+                              <span className="text-xs bg-red-100 text-red-700 px-2 py-0.5 rounded">
+                                系统自动禁用
+                              </span>
+                            )}
+                            {healthStatus === "degraded" && (
+                              <span className="text-xs bg-orange-100 text-orange-700 px-2 py-0.5 rounded">
+                                健康观察中
+                              </span>
+                            )}
                           </div>
                           <div className="text-sm text-muted-foreground truncate">
                             ID: {model.modelId} | 提供商:{" "}
                             {provider?.name ?? "未知"}
                           </div>
+                          {(healthStatus !== "healthy" || latestHealthEvent) && (
+                            <div className="mt-1 space-y-1 text-xs text-muted-foreground">
+                              <div className="flex flex-wrap items-center gap-2">
+                                <span>
+                                  连续故障：{model.healthConsecutiveFailures || 0} 次
+                                </span>
+                                {model.healthLastEventType && (
+                                  <span>类型：{model.healthLastEventType}</span>
+                                )}
+                                {model.healthAutoDisabledAt && (
+                                  <span>
+                                    禁用时间：
+                                    {formatBeijingTime(model.healthAutoDisabledAt, {
+                                      month: "2-digit",
+                                      day: "2-digit",
+                                      hour: "2-digit",
+                                      minute: "2-digit",
+                                    })}
+                                  </span>
+                                )}
+                              </div>
+                              {(model.healthLastError || latestHealthEvent?.errorMessage) && (
+                                <div className="line-clamp-2 max-w-[760px]">
+                                  最近原因：
+                                  {model.healthLastError || latestHealthEvent?.errorMessage}
+                                </div>
+                              )}
+                            </div>
+                          )}
                         </div>
                       </div>
                       <div className="flex items-center gap-2">
+                        {healthStatus === "auto_disabled" && (
+                          <Button
+                            variant="outline"
+                            size="sm"
+                            onClick={() => handleRestoreModelHealth(model)}
+                          >
+                            恢复
+                          </Button>
+                        )}
                         <Button
                           variant="outline"
                           size="sm"

app/api/admin/ai-model-health/route.ts

@@ -0,0 +1,55 @@
+import { z } from "zod";
+import { requireAdmin } from "@/lib/middleware/admin";
+import { ChatSDKError } from "@/lib/errors";
+import {
+  listAiModelHealthEvents,
+  restoreAiModelHealth,
+} from "@/lib/ai/runtime/model-health";
+
+const restoreSchema = z.object({
+  modelId: z.string().uuid(),
+});
+
+export async function GET(request: Request) {
+  try {
+    await requireAdmin();
+
+    const { searchParams } = new URL(request.url);
+    const modelId = searchParams.get("modelId") || undefined;
+    const limit = Number(searchParams.get("limit") || 80);
+    const events = await listAiModelHealthEvents({
+      modelId,
+      limit: Number.isFinite(limit) ? limit : 80,
+    });
+
+    return Response.json({ events });
+  } catch (error) {
+    if (error instanceof ChatSDKError) {
+      return error.toResponse();
+    }
+    return new ChatSDKError("offline:admin").toResponse();
+  }
+}
+
+export async function POST(request: Request) {
+  try {
+    await requireAdmin();
+
+    const body = await request.json();
+    const data = restoreSchema.parse(body);
+    const models = await restoreAiModelHealth(data.modelId);
+    if (models.length === 0) {
+      return new ChatSDKError("bad_request:api", "Model not found").toResponse();
+    }
+
+    return Response.json({ model: models[0] });
+  } catch (error) {
+    if (error instanceof ChatSDKError) {
+      return error.toResponse();
+    }
+    if (error instanceof z.ZodError) {
+      return new ChatSDKError("bad_request:api", error.message).toResponse();
+    }
+    return new ChatSDKError("offline:admin").toResponse();
+  }
+}

app/api/scene-fusion/route.ts

@@ -32,6 +32,10 @@ import {
   type SceneFusionImageGenConfig,
 } from "@/lib/scene-fusion/output-capabilities";
 import { acquireCanvasImageModelSlot } from "@/lib/canvas/image-concurrency";
+import {
+  recordAiModelHealthFailure,
+  recordAiModelHealthSuccess,
+} from "@/lib/ai/runtime/model-health";
 
 // 请求体 Schema
 const sceneFusionRequestSchema = z.object({
@@ -77,8 +81,11 @@ interface SceneFusionModelCandidate {
     displayName: string;
     supportsVision: boolean;
     imageGenConfig?: SceneFusionImageGenConfig | null;
+    excludeFromFailover?: boolean;
   };
   provider: {
+    id?: string;
+    name?: string;
     type: string;
     baseUrl: string;
     apiKey: string;
@@ -866,8 +873,11 @@ function buildSceneFusionCandidatePool(
         displayName: String(entry.model.displayName || entry.model.modelId || entry.model.id),
         supportsVision: Boolean(entry.model.supportsVision),
         imageGenConfig: (entry.model.imageGenConfig || null) as SceneFusionImageGenConfig | null,
+        excludeFromFailover: Boolean(entry.model.excludeFromFailover),
       },
       provider: {
+        id: String(entry.provider.id || ""),
+        name: String(entry.provider.name || ""),
         type: normalizeSceneFusionProviderType(String(entry.provider.type || "")),
         baseUrl: String(entry.provider.baseUrl || ""),
         apiKey: String(entry.provider.apiKey || ""),
@@ -1860,6 +1870,7 @@ async function processSceneWithDiversity(
 
   for (let attemptIndex = 0; attemptIndex < attemptCandidates.length; attemptIndex += 1) {
     const candidate = attemptCandidates[attemptIndex]!;
+    const attemptStartedAt = Date.now();
     try {
       const normalizedOutputParams = normalizeSceneFusionOutputParamsForCandidate(candidate, {
         aspectRatio: params.aspectRatio,
@@ -1943,6 +1954,18 @@ async function processSceneWithDiversity(
       }
 
       markSceneFusionCandidateHealthy(candidate, params.modelCooldown);
+      await recordAiModelHealthSuccess({
+        model: {
+          id: candidate.model.id,
+          healthStatus: "healthy",
+        },
+      }).catch((healthError) => {
+        console.warn("[SceneFusion] Record model health success failed:", {
+          requestId: params.requestId,
+          model: candidate.model.modelId,
+          error: healthError instanceof Error ? healthError.message : String(healthError),
+        });
+      });
       return {
         result,
         candidate,
@@ -1952,6 +1975,34 @@ async function processSceneWithDiversity(
       const errorMessage = error instanceof Error ? error.message : "未知错误";
       failureReasons.push(`${candidate.model.displayName}: ${errorMessage}`);
       markSceneFusionCandidateFailed(candidate, params.modelCooldown);
+      await recordAiModelHealthFailure({
+        model: {
+          id: candidate.model.id,
+          modelId: candidate.model.modelId,
+          displayName: candidate.model.displayName,
+          excludeFromFailover: Boolean(candidate.model.excludeFromFailover),
+        },
+        provider: {
+          id: candidate.provider.id,
+          name: candidate.provider.name,
+          type: candidate.provider.type,
+        },
+        error,
+        context: "scene-fusion",
+        elapsedMs: Date.now() - attemptStartedAt,
+        requestId: params.requestId,
+        metadata: {
+          sceneIndex: params.sceneIndex,
+          attempt: attemptIndex + 1,
+          sceneMode: params.sceneMode,
+        },
+      }).catch((healthError) => {
+        console.warn("[SceneFusion] Record model health failure failed:", {
+          requestId: params.requestId,
+          model: candidate.model.modelId,
+          error: healthError instanceof Error ? healthError.message : String(healthError),
+        });
+      });
       console.error("[SceneFusion] Candidate failed, try next:", {
         requestId: params.requestId,
         sceneIndex: params.sceneIndex,

lib/ai/runtime/model-health.ts

@@ -0,0 +1,340 @@
+import "server-only";
+
+import { and, desc, eq, sql } from "drizzle-orm";
+import { db } from "@/lib/db";
+import {
+  aiModel,
+  aiModelHealthEvent,
+  type AiModel,
+} from "@/lib/db/schema";
+
+export type AiModelHealthEventType =
+  | "success"
+  | "timeout"
+  | "quota"
+  | "unavailable"
+  | "invalid_model"
+  | "download_error"
+  | "error";
+
+export type AiModelHealthSeverity = "info" | "warning" | "error" | "critical";
+
+const AUTO_DISABLE_FAILURE_THRESHOLD = 3;
+const RECENT_EVENT_LIMIT = 80;
+
+function normalizeText(value: unknown) {
+  return String(value || "").trim();
+}
+
+function getErrorMessage(error: unknown) {
+  if (error instanceof Error) return normalizeText(error.message);
+  return normalizeText(error);
+}
+
+export function classifyAiModelHealthFailure(error: unknown): {
+  eventType: AiModelHealthEventType;
+  severity: AiModelHealthSeverity;
+  counted: boolean;
+  reason: string;
+} {
+  const message = getErrorMessage(error);
+  const normalized = message.toLowerCase();
+
+  if (!message) {
+    return {
+      eventType: "error",
+      severity: "error",
+      counted: true,
+      reason: "unknown_error",
+    };
+  }
+
+  if (
+    normalized.includes("生成结果未通过替换质检") ||
+    normalized.includes("quality check") ||
+    normalized.includes("review result") ||
+    normalized.includes("content policy") ||
+    normalized.includes("safety") ||
+    normalized.includes("user abort") ||
+    normalized.includes("aborted") ||
+    normalized.includes("cancelled")
+  ) {
+    return {
+      eventType: "error",
+      severity: "warning",
+      counted: false,
+      reason: "non_provider_failure",
+    };
+  }
+
+  if (
+    normalized.includes("model not exist") ||
+    normalized.includes("model_not_found") ||
+    normalized.includes("model not found") ||
+    normalized.includes("unknown model")
+  ) {
+    return {
+      eventType: "invalid_model",
+      severity: "critical",
+      counted: true,
+      reason: "invalid_model",
+    };
+  }
+
+  if (
+    normalized.includes("insufficient balance") ||
+    normalized.includes("quota") ||
+    normalized.includes("exceeded_current_quota") ||
+    normalized.includes("pre_consume_token_quota_failed") ||
+    normalized.includes("token quota is not enough")
+  ) {
+    return {
+      eventType: "quota",
+      severity: "critical",
+      counted: true,
+      reason: "quota_or_balance",
+    };
+  }
+
+  if (
+    normalized.includes("timeout") ||
+    normalized.includes("timed out") ||
+    normalized.includes("body timeout") ||
+    normalized.includes("headers timeout")
+  ) {
+    return {
+      eventType: "timeout",
+      severity: "critical",
+      counted: true,
+      reason: "timeout",
+    };
+  }
+
+  if (
+    normalized.includes("503") ||
+    normalized.includes("502") ||
+    normalized.includes("504") ||
+    normalized.includes("no available channel") ||
+    normalized.includes("no supported channel") ||
+    normalized.includes("service unavailable") ||
+    normalized.includes("fetch failed")
+  ) {
+    return {
+      eventType: "unavailable",
+      severity: "critical",
+      counted: true,
+      reason: "provider_unavailable",
+    };
+  }
+
+  if (
+    normalized.includes("download") ||
+    normalized.includes("no image url") ||
+    normalized.includes("no image") ||
+    normalized.includes("图片生成失败")
+  ) {
+    return {
+      eventType: "download_error",
+      severity: "error",
+      counted: true,
+      reason: "result_download_or_parse_failed",
+    };
+  }
+
+  return {
+    eventType: "error",
+    severity: "error",
+    counted: true,
+    reason: "provider_error",
+  };
+}
+
+export function shouldFailoverAiModelError(error: unknown) {
+  return classifyAiModelHealthFailure(error).counted;
+}
+
+function buildMetadata(value?: Record<string, unknown>) {
+  if (!value || typeof value !== "object") return null;
+  try {
+    return JSON.parse(JSON.stringify(value));
+  } catch {
+    return null;
+  }
+}
+
+export async function recordAiModelHealthSuccess(params: {
+  model?: Pick<AiModel, "id" | "healthStatus"> | null;
+}) {
+  const modelId = normalizeText(params.model?.id);
+  if (!modelId) return;
+
+  await db
+    .update(aiModel)
+    .set({
+      healthStatus:
+        params.model?.healthStatus === "auto_disabled" ? "auto_disabled" : "healthy",
+      healthConsecutiveFailures: 0,
+      healthLastEventType: "success",
+      healthLastError: null,
+      healthLastSuccessAt: new Date(),
+      healthUpdatedAt: new Date(),
+    })
+    .where(eq(aiModel.id, modelId));
+}
+
+export async function recordAiModelHealthFailure(params: {
+  model?: (Partial<AiModel> & Pick<AiModel, "id">) | null;
+  provider?: {
+    id?: string | null;
+    name?: string | null;
+    type?: string | null;
+  } | null;
+  error: unknown;
+  context?: string;
+  elapsedMs?: number;
+  requestId?: string;
+  metadata?: Record<string, unknown>;
+}) {
+  const modelId = normalizeText(params.model?.id);
+  if (!modelId) return;
+
+  const classification = classifyAiModelHealthFailure(params.error);
+  const errorMessage = getErrorMessage(params.error).slice(0, 1800);
+  const now = new Date();
+
+  const [existing] = await db
+    .select({
+      id: aiModel.id,
+      isEnabled: aiModel.isEnabled,
+      excludeFromFailover: aiModel.excludeFromFailover,
+      healthStatus: aiModel.healthStatus,
+      healthConsecutiveFailures: aiModel.healthConsecutiveFailures,
+    })
+    .from(aiModel)
+    .where(eq(aiModel.id, modelId))
+    .limit(1);
+
+  if (!existing) return;
+
+  const nextConsecutiveFailures = classification.counted
+    ? Math.max(1, (existing.healthConsecutiveFailures || 0) + 1)
+    : existing.healthConsecutiveFailures || 0;
+  const shouldAutoDisable =
+    classification.counted &&
+    !existing.excludeFromFailover &&
+    existing.healthStatus !== "auto_disabled" &&
+    nextConsecutiveFailures >= AUTO_DISABLE_FAILURE_THRESHOLD;
+  const nextStatus = shouldAutoDisable
+    ? "auto_disabled"
+    : classification.counted
+      ? "degraded"
+      : existing.healthStatus;
+  const autoDisabledReason = shouldAutoDisable
+    ? `${classification.reason}: ${errorMessage}`.slice(0, 1800)
+    : undefined;
+
+  await db
+    .insert(aiModelHealthEvent)
+    .values({
+      modelId,
+      providerId: normalizeText(params.provider?.id) || null,
+      modelRuntimeId: normalizeText(params.model?.modelId) || null,
+      modelDisplayName: normalizeText(params.model?.displayName) || null,
+      providerType: normalizeText(params.provider?.type) || null,
+      providerName: normalizeText(params.provider?.name) || null,
+      eventType: classification.eventType,
+      severity: classification.severity,
+      context: normalizeText(params.context).slice(0, 96) || null,
+      counted: classification.counted,
+      consecutiveFailures: nextConsecutiveFailures,
+      autoDisabled: shouldAutoDisable,
+      elapsedMs:
+        typeof params.elapsedMs === "number" && Number.isFinite(params.elapsedMs)
+          ? Math.max(0, Math.round(params.elapsedMs))
+          : null,
+      requestId: normalizeText(params.requestId).slice(0, 128) || null,
+      errorMessage,
+      metadata: buildMetadata(params.metadata),
+    });
+
+  const updateData: Partial<AiModel> = {
+    isEnabled: shouldAutoDisable ? false : existing.isEnabled,
+    healthStatus: nextStatus,
+    healthConsecutiveFailures: nextConsecutiveFailures,
+    healthLastEventType: classification.eventType,
+    healthLastError: errorMessage,
+    healthLastFailureAt: now,
+    healthUpdatedAt: now,
+    updatedAt: now,
+  };
+  if (shouldAutoDisable) {
+    updateData.healthAutoDisabledAt = now;
+    updateData.healthAutoDisabledReason = autoDisabledReason || errorMessage;
+  } else if (existing.healthStatus !== "auto_disabled") {
+    updateData.healthAutoDisabledReason = null;
+  }
+
+  await db.update(aiModel).set(updateData).where(eq(aiModel.id, modelId));
+
+  if (shouldAutoDisable) {
+    console.error("[AI Model Health] Auto disabled unhealthy model:", {
+      modelId,
+      runtimeModelId: params.model?.modelId,
+      displayName: params.model?.displayName,
+      providerType: params.provider?.type,
+      consecutiveFailures: nextConsecutiveFailures,
+      eventType: classification.eventType,
+      error: errorMessage,
+    });
+  }
+}
+
+export async function restoreAiModelHealth(modelId: string) {
+  const normalizedModelId = normalizeText(modelId);
+  if (!normalizedModelId) return [];
+
+  return await db
+    .update(aiModel)
+    .set({
+      isEnabled: true,
+      healthStatus: "healthy",
+      healthConsecutiveFailures: 0,
+      healthLastEventType: "success",
+      healthLastError: null,
+      healthLastFailureAt: null,
+      healthAutoDisabledAt: null,
+      healthAutoDisabledReason: null,
+      healthUpdatedAt: new Date(),
+      updatedAt: new Date(),
+    })
+    .where(eq(aiModel.id, normalizedModelId))
+    .returning();
+}
+
+export async function listAiModelHealthEvents(params?: { modelId?: string; limit?: number }) {
+  const limit = Math.max(1, Math.min(params?.limit || RECENT_EVENT_LIMIT, 200));
+  const modelId = normalizeText(params?.modelId);
+  const conditions = modelId ? [eq(aiModelHealthEvent.modelId, modelId)] : [];
+
+  return await db
+    .select()
+    .from(aiModelHealthEvent)
+    .where(conditions.length ? and(...conditions) : undefined)
+    .orderBy(desc(aiModelHealthEvent.createdAt))
+    .limit(limit);
+}
+
+export async function getAiModelHealthSummary() {
+  const rows = await db
+    .select({
+      modelId: aiModel.id,
+      eventCount: sql<number>`count(${aiModelHealthEvent.id})::int`,
+      countedEventCount: sql<number>`sum(case when ${aiModelHealthEvent.counted} then 1 else 0 end)::int`,
+      lastEventAt: sql<Date | null>`max(${aiModelHealthEvent.createdAt})`,
+    })
+    .from(aiModel)
+    .leftJoin(aiModelHealthEvent, eq(aiModel.id, aiModelHealthEvent.modelId))
+    .groupBy(aiModel.id);
+
+  return rows;
+}

lib/canvas/generate-image.ts

@@ -1,6 +1,6 @@
 import "server-only";
 
-import { getEnabledAiModelsWithProvider, getAiProviderById } from "@/lib/db/queries";
+import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
 import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
 import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image";
 import { createBflImage } from "@/lib/ai/providers/bfl-image";
@@ -16,6 +16,11 @@ import {
   acquireCanvasImageModelSlot,
   normalizeCanvasImageModelKey,
 } from "./image-concurrency";
+import {
+  recordAiModelHealthFailure,
+  recordAiModelHealthSuccess,
+  shouldFailoverAiModelError,
+} from "@/lib/ai/runtime/model-health";
 
 export interface CanvasGenerateImageRequest {
   prompt: string;
@@ -31,6 +36,7 @@ export interface CanvasGenerateImageRequest {
   ownerKey?: string;
   debugLabel?: string;
   debugMeta?: Record<string, unknown>;
+  internalFailoverTriedModelRefs?: string[];
 }
 
 export interface ParsedImage {
@@ -127,6 +133,60 @@ function getProviderBatchLimit(providerType: string) {
   }
 }
 
+type CanvasModelWithProvider = Awaited<
+  ReturnType<typeof getEnabledAiModelsWithProvider>
+>[number];
+
+function getCanvasModelRefKeys(item: CanvasModelWithProvider) {
+  return [
+    String(item.model.id || "").trim(),
+    String(item.model.modelId || "").trim(),
+  ].filter(Boolean);
+}
+
+function findCanvasFailoverModel(params: {
+  selectedModel: CanvasModelWithProvider;
+  imageModels: CanvasModelWithProvider[];
+  failedModelRefs: string[];
+}) {
+  const failedRefs = new Set(
+    params.failedModelRefs.map((item) => String(item || "").trim()).filter(Boolean)
+  );
+  const selectedFallbackRefs = [
+    params.selectedModel.model.fallbackModelId1,
+    params.selectedModel.model.fallbackModelId2,
+  ]
+    .map((item) => String(item || "").trim())
+    .filter(Boolean);
+
+  const isUsableFallback = (item: CanvasModelWithProvider) => {
+    if (!item.model.isEnabled || item.model.isSubModel || item.model.excludeFromFailover) {
+      return false;
+    }
+    if (!item.model.supportsImageGen) return false;
+    if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(item.provider.type)) return false;
+    return getCanvasModelRefKeys(item).every((key) => !failedRefs.has(key));
+  };
+
+  for (const fallbackRef of selectedFallbackRefs) {
+    const configured = params.imageModels.find(
+      (item) =>
+        isUsableFallback(item) &&
+        (item.model.id === fallbackRef || item.model.modelId === fallbackRef)
+    );
+    if (configured) return configured;
+  }
+
+  const sameProviderType = params.imageModels.find(
+    (item) =>
+      isUsableFallback(item) &&
+      item.provider.type === params.selectedModel.provider.type
+  );
+  if (sameProviderType) return sameProviderType;
+
+  return params.imageModels.find(isUsableFallback) || null;
+}
+
 export async function generateCanvasImage(
   request: CanvasGenerateImageRequest
 ): Promise<CanvasGenerateImageResult> {
@@ -183,19 +243,13 @@ export async function generateCanvasImage(
     selectedModel = imageModels[0];
   }
 
-  let provider = await getAiProviderById({ id: selectedModel.model.providerId });
-  if (!provider) {
-    throw new Error("模型提供商配置错误");
-  }
+  let provider = selectedModel.provider;
 
   if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(provider.type)) {
     let fallbackModel = selectedModel;
     let fallbackProvider = provider;
     for (const candidate of imageModels) {
-      const candidateProvider = await getAiProviderById({
-        id: candidate.model.providerId,
-      });
-      if (!candidateProvider) continue;
+      const candidateProvider = candidate.provider;
       if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(candidateProvider.type)) continue;
       fallbackModel = candidate;
       fallbackProvider = candidateProvider;
@@ -469,6 +523,8 @@ export async function generateCanvasImage(
     },
   });
 
+  let caughtError: unknown = null;
+
   try {
     for (let batchIndex = 0; batchIndex < maxBatches; batchIndex += 1) {
       const remaining = targetImageCount - collectedImages.length;
@@ -522,6 +578,16 @@ export async function generateCanvasImage(
       ...debugMeta,
     });
 
+    await recordAiModelHealthSuccess({
+      model: selectedModel.model,
+    }).catch((healthError) => {
+      console.warn("[Canvas Generate Image] Record model health success failed:", {
+        selectedModelRef: selectedModel.model.id,
+        selectedModelId: selectedModel.model.modelId,
+        error: healthError instanceof Error ? healthError.message : String(healthError),
+      });
+    });
+
     return {
       images: finalImages,
       model: selectedModel.model.displayName,
@@ -540,8 +606,72 @@ export async function generateCanvasImage(
       error: error instanceof Error ? error.message : String(error),
       ...debugMeta,
     });
-    throw error;
+    caughtError = error;
   } finally {
     await slot.release();
   }
+
+  await recordAiModelHealthFailure({
+    model: selectedModel.model,
+    provider,
+    error: caughtError,
+    context: debugLabel,
+    elapsedMs: Date.now() - startedAt,
+    metadata: {
+      aspectRatio,
+      imageSize,
+      imageCount,
+      requestedReferenceImageCount: normalizedReferenceImages.length,
+      ...debugMeta,
+    },
+  }).catch((healthError) => {
+    console.warn("[Canvas Generate Image] Record model health failure failed:", {
+      selectedModelRef: selectedModel.model.id,
+      selectedModelId: selectedModel.model.modelId,
+      error: healthError instanceof Error ? healthError.message : String(healthError),
+    });
+  });
+
+  const failedModelRefs = Array.from(
+    new Set([
+      ...(Array.isArray(request.internalFailoverTriedModelRefs)
+        ? request.internalFailoverTriedModelRefs
+        : []),
+      selectedModel.model.id,
+      selectedModel.model.modelId,
+    ].filter(Boolean))
+  );
+  const fallbackModel = shouldFailoverAiModelError(caughtError)
+    ? findCanvasFailoverModel({
+        selectedModel,
+        imageModels,
+        failedModelRefs,
+      })
+    : null;
+
+  if (fallbackModel) {
+    console.warn("[Canvas Generate Image] Failover to alternate image model:", {
+      debugLabel,
+      failedModelRef: selectedModel.model.id,
+      failedModelId: selectedModel.model.modelId,
+      fallbackModelRef: fallbackModel.model.id,
+      fallbackModelId: fallbackModel.model.modelId,
+      fallbackProviderType: fallbackModel.provider.type,
+      error: caughtError instanceof Error ? caughtError.message : String(caughtError),
+      ...debugMeta,
+    });
+
+    return await generateCanvasImage({
+      ...request,
+      model: fallbackModel.model.id,
+      internalFailoverTriedModelRefs: failedModelRefs,
+      debugMeta: {
+        ...debugMeta,
+        failoverFromModelId: selectedModel.model.modelId,
+        failoverFromModelRef: selectedModel.model.id,
+      },
+    });
+  }
+
+  throw caughtError;
 }

lib/db/migrations/0079_add_ai_model_health.sql

@@ -0,0 +1,40 @@
+ALTER TABLE "AiModel"
+  ADD COLUMN IF NOT EXISTS "healthStatus" varchar DEFAULT 'healthy' NOT NULL,
+  ADD COLUMN IF NOT EXISTS "healthConsecutiveFailures" integer DEFAULT 0 NOT NULL,
+  ADD COLUMN IF NOT EXISTS "healthLastEventType" varchar(32),
+  ADD COLUMN IF NOT EXISTS "healthLastError" text,
+  ADD COLUMN IF NOT EXISTS "healthLastFailureAt" timestamp,
+  ADD COLUMN IF NOT EXISTS "healthLastSuccessAt" timestamp,
+  ADD COLUMN IF NOT EXISTS "healthAutoDisabledAt" timestamp,
+  ADD COLUMN IF NOT EXISTS "healthAutoDisabledReason" text,
+  ADD COLUMN IF NOT EXISTS "healthUpdatedAt" timestamp DEFAULT now() NOT NULL;
+
+CREATE TABLE IF NOT EXISTS "AiModelHealthEvent" (
+  "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+  "modelId" uuid REFERENCES "AiModel"("id") ON DELETE SET NULL,
+  "providerId" uuid REFERENCES "AiProvider"("id") ON DELETE SET NULL,
+  "modelRuntimeId" varchar(128),
+  "modelDisplayName" varchar(128),
+  "providerType" varchar(64),
+  "providerName" varchar(128),
+  "eventType" varchar(32) DEFAULT 'error' NOT NULL,
+  "severity" varchar(24) DEFAULT 'error' NOT NULL,
+  "context" varchar(96),
+  "counted" boolean DEFAULT true NOT NULL,
+  "consecutiveFailures" integer DEFAULT 0 NOT NULL,
+  "autoDisabled" boolean DEFAULT false NOT NULL,
+  "elapsedMs" integer,
+  "requestId" varchar(128),
+  "errorMessage" text,
+  "metadata" jsonb,
+  "createdAt" timestamp DEFAULT now() NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS "AiModelHealthEvent_modelId_createdAt_idx"
+  ON "AiModelHealthEvent" ("modelId", "createdAt" DESC);
+
+CREATE INDEX IF NOT EXISTS "AiModelHealthEvent_createdAt_idx"
+  ON "AiModelHealthEvent" ("createdAt" DESC);
+
+CREATE INDEX IF NOT EXISTS "AiModel_healthStatus_idx"
+  ON "AiModel" ("healthStatus");

lib/db/migrations/meta/_journal.json

@@ -456,6 +456,13 @@
       "when": 1777569600000,
       "tag": "0078_add_dingtalk_notification_tasks",
       "breakpoints": true
+    },
+    {
+      "idx": 65,
+      "version": "7",
+      "when": 1778252400000,
+      "tag": "0079_add_ai_model_health",
+      "breakpoints": true
     }
   ]
 }

lib/db/queries.ts

@@ -4184,7 +4184,19 @@ export async function updateAiModel({
     if (supportsVision !== undefined) updateData.supportsVision = supportsVision;
     if (supportsImageGen !== undefined)
       updateData.supportsImageGen = supportsImageGen;
-    if (isEnabled !== undefined) updateData.isEnabled = isEnabled;
+    if (isEnabled !== undefined) {
+      updateData.isEnabled = isEnabled;
+      if (isEnabled) {
+        updateData.healthStatus = "healthy";
+        updateData.healthConsecutiveFailures = 0;
+        updateData.healthLastEventType = "success";
+        updateData.healthLastError = null;
+        updateData.healthLastFailureAt = null;
+        updateData.healthAutoDisabledAt = null;
+        updateData.healthAutoDisabledReason = null;
+        updateData.healthUpdatedAt = new Date();
+      }
+    }
     if (isHidden !== undefined) updateData.isHidden = isHidden;
     if (isDefault !== undefined) updateData.isDefault = isDefault;
     if (isDefaultTextEditModel !== undefined) {

lib/db/schema.ts

@@ -335,6 +335,21 @@ export const aiModel = pgTable("AiModel", {
   fallbackModelId1: varchar("fallbackModelId1", { length: 128 }), // 备用模型 ID 1
   fallbackModelId2: varchar("fallbackModelId2", { length: 128 }), // 备用模型 ID 2
   excludeFromFailover: boolean("excludeFromFailover").notNull().default(false), // 不参与故障迁移
+  healthStatus: varchar("healthStatus", {
+    enum: ["healthy", "degraded", "auto_disabled"],
+  })
+    .notNull()
+    .default("healthy"),
+  healthConsecutiveFailures: integer("healthConsecutiveFailures")
+    .notNull()
+    .default(0),
+  healthLastEventType: varchar("healthLastEventType", { length: 32 }),
+  healthLastError: text("healthLastError"),
+  healthLastFailureAt: timestamp("healthLastFailureAt"),
+  healthLastSuccessAt: timestamp("healthLastSuccessAt"),
+  healthAutoDisabledAt: timestamp("healthAutoDisabledAt"),
+  healthAutoDisabledReason: text("healthAutoDisabledReason"),
+  healthUpdatedAt: timestamp("healthUpdatedAt").notNull().defaultNow(),
   // 🆕 图片生成模型特性配置（用于阿里生图、Gemini等）
   imageGenConfig: jsonb("imageGenConfig"), // { maxImages: 6, supportsSize: true, supportsPromptExtend: true, supportsNegativePrompt: true, enableLoadingPlaceholder: true }
   createdAt: timestamp("createdAt").notNull().defaultNow(),
@@ -343,6 +358,31 @@ export const aiModel = pgTable("AiModel", {
 
 export type AiModel = InferSelectModel<typeof aiModel>;
 
+export const aiModelHealthEvent = pgTable("AiModelHealthEvent", {
+  id: uuid("id").primaryKey().notNull().defaultRandom(),
+  modelId: uuid("modelId").references(() => aiModel.id, { onDelete: "set null" }),
+  providerId: uuid("providerId").references(() => aiProvider.id, {
+    onDelete: "set null",
+  }),
+  modelRuntimeId: varchar("modelRuntimeId", { length: 128 }),
+  modelDisplayName: varchar("modelDisplayName", { length: 128 }),
+  providerType: varchar("providerType", { length: 64 }),
+  providerName: varchar("providerName", { length: 128 }),
+  eventType: varchar("eventType", { length: 32 }).notNull().default("error"),
+  severity: varchar("severity", { length: 24 }).notNull().default("error"),
+  context: varchar("context", { length: 96 }),
+  counted: boolean("counted").notNull().default(true),
+  consecutiveFailures: integer("consecutiveFailures").notNull().default(0),
+  autoDisabled: boolean("autoDisabled").notNull().default(false),
+  elapsedMs: integer("elapsedMs"),
+  requestId: varchar("requestId", { length: 128 }),
+  errorMessage: text("errorMessage"),
+  metadata: jsonb("metadata"),
+  createdAt: timestamp("createdAt").notNull().defaultNow(),
+});
+
+export type AiModelHealthEvent = InferSelectModel<typeof aiModelHealthEvent>;
+
 // S3 存储配置表
 export const storageConfig = pgTable("StorageConfig", {
   id: uuid("id").primaryKey().notNull().defaultRandom(),