feat: add ai model health recovery pool

app/(admin)/settings/admin-settings-client.tsx

@@ -863,6 +863,8 @@ type AiModel = {
   healthLastSuccessAt?: string | null;
   healthAutoDisabledAt?: string | null;
   healthAutoDisabledReason?: string | null;
+  healthRecoverySuccessStreak?: number;
+  healthObservationStartedAt?: string | null;
   isAgentModel?: boolean;
   agentCodingModelId?: string;
   isMultimodal?: boolean;
@@ -15677,6 +15679,19 @@ export function AdminSettingsClient() {
                     supportsImageGen: model.supportsImageGen,
                   });
                   const healthStatus = model.healthStatus || "healthy";
+                  const isObservationPool =
+                    healthStatus === "degraded" &&
+                    Boolean(model.healthObservationStartedAt);
+                  const recoverySuccessStreak = Math.max(
+                    0,
+                    Number(model.healthRecoverySuccessStreak || 0)
+                  );
+                  const autoDisableCooldownUntil = model.healthAutoDisabledAt
+                    ? new Date(
+                        new Date(model.healthAutoDisabledAt).getTime() +
+                          2 * 60 * 60 * 1000
+                      )
+                    : null;
                   const modelHealthEvents =
                     modelHealthEventsByModelId.get(model.id) || [];
                   const latestHealthEvent = modelHealthEvents[0];
@@ -15756,7 +15771,9 @@ export function AdminSettingsClient() {
                             )}
                             {healthStatus === "degraded" && (
                               <span className="text-xs bg-orange-100 text-orange-700 px-2 py-0.5 rounded">
-                                健康观察中
+                                {isObservationPool
+                                  ? `观察池 ${recoverySuccessStreak}/3`
+                                  : "健康观察中"}
                               </span>
                             )}
                           </div>
@@ -15784,6 +15801,38 @@ export function AdminSettingsClient() {
                                     })}
                                   </span>
                                 )}
+                                {healthStatus === "auto_disabled" &&
+                                  autoDisableCooldownUntil && (
+                                    <span>
+                                      解冻时间：
+                                      {formatBeijingTime(autoDisableCooldownUntil, {
+                                        month: "2-digit",
+                                        day: "2-digit",
+                                        hour: "2-digit",
+                                        minute: "2-digit",
+                                      })}
+                                    </span>
+                                  )}
+                                {isObservationPool && (
+                                  <span>
+                                    观察成功：{recoverySuccessStreak}/3
+                                  </span>
+                                )}
+                                {isObservationPool &&
+                                  model.healthObservationStartedAt && (
+                                    <span>
+                                      观察开始：
+                                      {formatBeijingTime(
+                                        model.healthObservationStartedAt,
+                                        {
+                                          month: "2-digit",
+                                          day: "2-digit",
+                                          hour: "2-digit",
+                                          minute: "2-digit",
+                                        }
+                                      )}
+                                    </span>
+                                  )}
                               </div>
                               {(model.healthLastError || latestHealthEvent?.errorMessage) && (
                                 <div className="line-clamp-2 max-w-[760px]">

lib/ai/runtime/model-health.ts

@@ -1,6 +1,6 @@
 import "server-only";
 
-import { and, desc, eq, sql } from "drizzle-orm";
+import { and, desc, eq, inArray, lte, sql } from "drizzle-orm";
 import { db } from "@/lib/db";
 import {
   aiModel,
@@ -20,6 +20,8 @@ export type AiModelHealthEventType =
 export type AiModelHealthSeverity = "info" | "warning" | "error" | "critical";
 
 const AUTO_DISABLE_FAILURE_THRESHOLD = 3;
+const AUTO_DISABLE_COOLDOWN_MS = 2 * 60 * 60 * 1000;
+const OBSERVATION_RECOVERY_SUCCESS_THRESHOLD = 3;
 const RECENT_EVENT_LIMIT = 80;
 
 function normalizeText(value: unknown) {
@@ -162,23 +164,157 @@ function buildMetadata(value?: Record<string, unknown>) {
   }
 }
 
+function isObservationPoolModel(
+  model:
+    | Pick<
+        AiModel,
+        | "healthStatus"
+        | "healthObservationStartedAt"
+      >
+    | null
+    | undefined
+) {
+  return (
+    model?.healthStatus === "degraded" &&
+    Boolean(model.healthObservationStartedAt)
+  );
+}
+
+function getAutoDisableCooldownCutoff(now = new Date()) {
+  return new Date(now.getTime() - AUTO_DISABLE_COOLDOWN_MS);
+}
+
+export async function reviveAiModelsFromCooldown(params?: { modelId?: string }) {
+  const now = new Date();
+  const cooldownCutoff = getAutoDisableCooldownCutoff(now);
+  const conditions = [
+    eq(aiModel.healthStatus, "auto_disabled"),
+    eq(aiModel.isEnabled, false),
+    eq(aiModel.excludeFromFailover, false),
+    lte(aiModel.healthAutoDisabledAt, cooldownCutoff),
+  ];
+
+  if (params?.modelId) {
+    conditions.push(eq(aiModel.id, normalizeText(params.modelId)));
+  }
+
+  const dueModels = await db
+    .select({
+      id: aiModel.id,
+      modelId: aiModel.modelId,
+      displayName: aiModel.displayName,
+    })
+    .from(aiModel)
+    .where(and(...conditions));
+
+  if (dueModels.length === 0) {
+    return [];
+  }
+
+  const revived = await db
+    .update(aiModel)
+    .set({
+      isEnabled: true,
+      healthStatus: "degraded",
+      healthConsecutiveFailures: 0,
+      healthRecoverySuccessStreak: 0,
+      healthObservationStartedAt: now,
+      healthUpdatedAt: now,
+      updatedAt: now,
+    })
+    .where(
+      inArray(
+        aiModel.id,
+        dueModels.map((item) => item.id)
+      )
+    )
+    .returning();
+
+  if (revived.length > 0) {
+    console.info("[AI Model Health] Revived cooled-down models into observation pool:", {
+      count: revived.length,
+      models: revived.map((item) => ({
+        id: item.id,
+        modelId: item.modelId,
+        displayName: item.displayName,
+      })),
+    });
+  }
+
+  return revived;
+}
+
 export async function recordAiModelHealthSuccess(params: {
-  model?: Pick<AiModel, "id" | "healthStatus"> | null;
+  model?:
+    | Pick<
+        AiModel,
+        | "id"
+        | "healthStatus"
+        | "healthRecoverySuccessStreak"
+        | "healthObservationStartedAt"
+      >
+    | null;
 }) {
   const modelId = normalizeText(params.model?.id);
   if (!modelId) return;
 
+  const [existing] = await db
+    .select({
+      id: aiModel.id,
+      healthStatus: aiModel.healthStatus,
+      healthRecoverySuccessStreak: aiModel.healthRecoverySuccessStreak,
+      healthObservationStartedAt: aiModel.healthObservationStartedAt,
+    })
+    .from(aiModel)
+    .where(eq(aiModel.id, modelId))
+    .limit(1);
+
+  if (!existing) return;
+
+  const now = new Date();
+  const isObservation = isObservationPoolModel(existing);
+  const nextRecoverySuccessStreak = isObservation
+    ? (existing.healthRecoverySuccessStreak || 0) + 1
+    : 0;
+  const recoveredToHealthy =
+    isObservation &&
+    nextRecoverySuccessStreak >= OBSERVATION_RECOVERY_SUCCESS_THRESHOLD;
+
+  const updateData: Partial<AiModel> = {
+    healthStatus:
+      existing.healthStatus === "auto_disabled"
+        ? "auto_disabled"
+        : recoveredToHealthy
+          ? "healthy"
+          : isObservation
+            ? "degraded"
+            : "healthy",
+    healthConsecutiveFailures: 0,
+    healthLastEventType: "success",
+    healthLastError: null,
+    healthLastSuccessAt: now,
+    healthRecoverySuccessStreak: recoveredToHealthy
+      ? 0
+      : isObservation
+        ? nextRecoverySuccessStreak
+        : 0,
+    healthObservationStartedAt: recoveredToHealthy
+      ? null
+      : isObservation
+        ? existing.healthObservationStartedAt
+        : null,
+    healthUpdatedAt: now,
+    updatedAt: now,
+  };
+
+  if (recoveredToHealthy) {
+    updateData.healthAutoDisabledAt = null;
+    updateData.healthAutoDisabledReason = null;
+  }
+
   await db
     .update(aiModel)
-    .set({
+    .set(updateData)
-      healthStatus:
-        params.model?.healthStatus === "auto_disabled" ? "auto_disabled" : "healthy",
-      healthConsecutiveFailures: 0,
-      healthLastEventType: "success",
-      healthLastError: null,
-      healthLastSuccessAt: new Date(),
-      healthUpdatedAt: new Date(),
-    })
     .where(eq(aiModel.id, modelId));
 }
 
@@ -205,10 +341,15 @@ export async function recordAiModelHealthFailure(params: {
   const [existing] = await db
     .select({
       id: aiModel.id,
+      modelId: aiModel.modelId,
+      displayName: aiModel.displayName,
       isEnabled: aiModel.isEnabled,
       excludeFromFailover: aiModel.excludeFromFailover,
       healthStatus: aiModel.healthStatus,
       healthConsecutiveFailures: aiModel.healthConsecutiveFailures,
+      healthRecoverySuccessStreak: aiModel.healthRecoverySuccessStreak,
+      healthObservationStartedAt: aiModel.healthObservationStartedAt,
+      healthAutoDisabledAt: aiModel.healthAutoDisabledAt,
     })
     .from(aiModel)
     .where(eq(aiModel.id, modelId))
@@ -216,14 +357,19 @@ export async function recordAiModelHealthFailure(params: {
 
   if (!existing) return;
 
+  const isObservation = isObservationPoolModel(existing);
   const nextConsecutiveFailures = classification.counted
-    ? Math.max(1, (existing.healthConsecutiveFailures || 0) + 1)
+    ? Math.max(
+        1,
+        (isObservation ? 0 : existing.healthConsecutiveFailures || 0) + 1
+      )
     : existing.healthConsecutiveFailures || 0;
   const shouldAutoDisable =
     classification.counted &&
     !existing.excludeFromFailover &&
-    existing.healthStatus !== "auto_disabled" &&
+    ((isObservation && existing.isEnabled) ||
-    nextConsecutiveFailures >= AUTO_DISABLE_FAILURE_THRESHOLD;
+      (existing.healthStatus !== "auto_disabled" &&
+        nextConsecutiveFailures >= AUTO_DISABLE_FAILURE_THRESHOLD));
   const nextStatus = shouldAutoDisable
     ? "auto_disabled"
     : classification.counted
@@ -264,13 +410,23 @@ export async function recordAiModelHealthFailure(params: {
     healthLastEventType: classification.eventType,
     healthLastError: errorMessage,
     healthLastFailureAt: now,
+    healthRecoverySuccessStreak: shouldAutoDisable
+      ? 0
+      : classification.counted
+        ? 0
+        : existing.healthRecoverySuccessStreak || 0,
+    healthObservationStartedAt: shouldAutoDisable
+      ? null
+      : isObservation
+        ? existing.healthObservationStartedAt
+        : null,
     healthUpdatedAt: now,
     updatedAt: now,
   };
   if (shouldAutoDisable) {
     updateData.healthAutoDisabledAt = now;
     updateData.healthAutoDisabledReason = autoDisabledReason || errorMessage;
-  } else if (existing.healthStatus !== "auto_disabled") {
+  } else if (existing.healthStatus !== "auto_disabled" && !isObservation) {
     updateData.healthAutoDisabledReason = null;
   }
 
@@ -304,6 +460,8 @@ export async function restoreAiModelHealth(modelId: string) {
       healthLastFailureAt: null,
       healthAutoDisabledAt: null,
       healthAutoDisabledReason: null,
+      healthRecoverySuccessStreak: 0,
+      healthObservationStartedAt: null,
       healthUpdatedAt: new Date(),
       updatedAt: new Date(),
     })

lib/db/migrate.ts

@@ -110,6 +110,14 @@ const runMigrate = async () => {
     ALTER TABLE "AiModel"
     ADD COLUMN IF NOT EXISTS "isDefaultTextEditModel" boolean NOT NULL DEFAULT false;
   `);
+  await db.execute(sql`
+    ALTER TABLE "AiModel"
+    ADD COLUMN IF NOT EXISTS "healthRecoverySuccessStreak" integer NOT NULL DEFAULT 0;
+  `);
+  await db.execute(sql`
+    ALTER TABLE "AiModel"
+    ADD COLUMN IF NOT EXISTS "healthObservationStartedAt" timestamp;
+  `);
   for (const item of providerBaseUrlDefaults) {
     await db.execute(sql`
       UPDATE "AiProvider"

lib/db/migrations/0080_add_ai_model_health_recovery.sql

@@ -0,0 +1,5 @@
+ALTER TABLE "AiModel"
+ADD COLUMN IF NOT EXISTS "healthRecoverySuccessStreak" integer NOT NULL DEFAULT 0;
+
+ALTER TABLE "AiModel"
+ADD COLUMN IF NOT EXISTS "healthObservationStartedAt" timestamp;

lib/db/migrations/meta/_journal.json

@@ -463,6 +463,13 @@
       "when": 1778252400000,
       "tag": "0079_add_ai_model_health",
       "breakpoints": true
+    },
+    {
+      "idx": 66,
+      "version": "7",
+      "when": 1778461200000,
+      "tag": "0080_add_ai_model_health_recovery",
+      "breakpoints": true
     }
   ]
 }

lib/db/queries.ts

@@ -21,6 +21,7 @@ import { drizzle } from "drizzle-orm/postgres-js";
 import postgres from "postgres";
 import type { ArtifactKind } from "@/components/artifact";
 import type { VisibilityType } from "@/components/visibility-selector";
+import { reviveAiModelsFromCooldown } from "@/lib/ai/runtime/model-health";
 import { ChatSDKError } from "../errors";
 import type { AppUsage } from "../usage";
 import { generateUUID } from "../utils";
@@ -3912,6 +3913,7 @@ export async function deleteAiProvider({ id }: { id: string }) {
 
 export async function getAllAiModels(): Promise<AiModel[]> {
   try {
+    await reviveAiModelsFromCooldown();
     // 暂时移除排序以避免 sortOrder 类型问题
     return await db.select().from(aiModel);
   } catch (_error) {
@@ -3925,6 +3927,7 @@ export async function getAllAiModels(): Promise<AiModel[]> {
 
 export async function getEnabledAiModels(): Promise<AiModel[]> {
   try {
+    await reviveAiModelsFromCooldown();
     return await db
       .select()
       .from(aiModel)
@@ -3943,6 +3946,7 @@ export async function getAiModelsByProviderId({
   providerId: string;
 }): Promise<AiModel[]> {
   try {
+    await reviveAiModelsFromCooldown();
     return await db
       .select()
       .from(aiModel)
@@ -3961,6 +3965,7 @@ export async function getAiModelById({
   id: string;
 }): Promise<AiModel | undefined> {
   try {
+    await reviveAiModelsFromCooldown({ modelId: id });
     const models = await db
       .select()
       .from(aiModel)
@@ -3977,6 +3982,7 @@ export async function getAiModelById({
 
 export async function getDefaultAiModel(): Promise<AiModel | undefined> {
   try {
+    await reviveAiModelsFromCooldown();
     const models = await db
       .select()
       .from(aiModel)
@@ -4194,6 +4200,8 @@ export async function updateAiModel({
         updateData.healthLastFailureAt = null;
         updateData.healthAutoDisabledAt = null;
         updateData.healthAutoDisabledReason = null;
+        updateData.healthRecoverySuccessStreak = 0;
+        updateData.healthObservationStartedAt = null;
         updateData.healthUpdatedAt = new Date();
       }
     }
@@ -4477,6 +4485,7 @@ export async function getEnabledAiModelsWithProvider(
   includeHidden: boolean = true
 ) {
   try {
+    await reviveAiModelsFromCooldown();
     const conditions = [
       eq(aiModel.isEnabled, true), 
       eq(aiProvider.isEnabled, true),
@@ -4499,7 +4508,23 @@ export async function getEnabledAiModelsWithProvider(
       .innerJoin(aiProvider, eq(aiModel.providerId, aiProvider.id))
       .where(and(...conditions));
 
-    return models;
+    const healthPriority = (status?: string | null) => {
+      if (status === "healthy") return 0;
+      if (status === "degraded") return 1;
+      if (status === "auto_disabled") return 2;
+      return 3;
+    };
+
+    return [...models].sort((left, right) => {
+      const byHealth =
+        healthPriority(left.model.healthStatus) -
+        healthPriority(right.model.healthStatus);
+      if (byHealth !== 0) return byHealth;
+      if (left.model.isDefault !== right.model.isDefault) {
+        return Number(right.model.isDefault) - Number(left.model.isDefault);
+      }
+      return (left.model.sortOrder ?? 0) - (right.model.sortOrder ?? 0);
+    });
   } catch (_error) {
     throw new ChatSDKError(
       "bad_request:database",
@@ -4528,6 +4553,7 @@ export async function getAgentAiModelWithProvider() {
  */
 export async function getSubModelsByParentModelId(parentModelId: string) {
   try {
+    await reviveAiModelsFromCooldown();
     const subModels = await db
       .select({
         model: aiModel,

lib/db/schema.ts

@@ -349,6 +349,10 @@ export const aiModel = pgTable("AiModel", {
   healthLastSuccessAt: timestamp("healthLastSuccessAt"),
   healthAutoDisabledAt: timestamp("healthAutoDisabledAt"),
   healthAutoDisabledReason: text("healthAutoDisabledReason"),
+  healthRecoverySuccessStreak: integer("healthRecoverySuccessStreak")
+    .notNull()
+    .default(0),
+  healthObservationStartedAt: timestamp("healthObservationStartedAt"),
   healthUpdatedAt: timestamp("healthUpdatedAt").notNull().defaultNow(),
   // 🆕 图片生成模型特性配置（用于阿里生图、Gemini等）
   imageGenConfig: jsonb("imageGenConfig"), // { maxImages: 6, supportsSize: true, supportsPromptExtend: true, supportsNegativePrompt: true, enableLoadingPlaceholder: true }