diff --git a/lib/ai/runtime/model-health.ts b/lib/ai/runtime/model-health.ts
index a6d4a44f6..72c0e9f32 100644
--- a/lib/ai/runtime/model-health.ts
+++ b/lib/ai/runtime/model-health.ts
@@ -1,6 +1,6 @@
import "server-only";
-import { and, desc, eq, sql } from "drizzle-orm";
+import { and, desc, eq, inArray, lte, sql } from "drizzle-orm";
import { db } from "@/lib/db";
import {
aiModel,
@@ -20,6 +20,8 @@ export type AiModelHealthEventType =
export type AiModelHealthSeverity = "info" | "warning" | "error" | "critical";
const AUTO_DISABLE_FAILURE_THRESHOLD = 3;
+const AUTO_DISABLE_COOLDOWN_MS = 2 * 60 * 60 * 1000;
+const OBSERVATION_RECOVERY_SUCCESS_THRESHOLD = 3;
const RECENT_EVENT_LIMIT = 80;
function normalizeText(value: unknown) {
@@ -162,23 +164,157 @@ function buildMetadata(value?: Record
) {
}
}
+function isObservationPoolModel(
+ model:
+ | Pick<
+ AiModel,
+ | "healthStatus"
+ | "healthObservationStartedAt"
+ >
+ | null
+ | undefined
+) {
+ return (
+ model?.healthStatus === "degraded" &&
+ Boolean(model.healthObservationStartedAt)
+ );
+}
+
+function getAutoDisableCooldownCutoff(now = new Date()) {
+ return new Date(now.getTime() - AUTO_DISABLE_COOLDOWN_MS);
+}
+
+export async function reviveAiModelsFromCooldown(params?: { modelId?: string }) {
+ const now = new Date();
+ const cooldownCutoff = getAutoDisableCooldownCutoff(now);
+ const conditions = [
+ eq(aiModel.healthStatus, "auto_disabled"),
+ eq(aiModel.isEnabled, false),
+ eq(aiModel.excludeFromFailover, false),
+ lte(aiModel.healthAutoDisabledAt, cooldownCutoff),
+ ];
+
+ if (params?.modelId) {
+ conditions.push(eq(aiModel.id, normalizeText(params.modelId)));
+ }
+
+ const dueModels = await db
+ .select({
+ id: aiModel.id,
+ modelId: aiModel.modelId,
+ displayName: aiModel.displayName,
+ })
+ .from(aiModel)
+ .where(and(...conditions));
+
+ if (dueModels.length === 0) {
+ return [];
+ }
+
+ const revived = await db
+ .update(aiModel)
+ .set({
+ isEnabled: true,
+ healthStatus: "degraded",
+ healthConsecutiveFailures: 0,
+ healthRecoverySuccessStreak: 0,
+ healthObservationStartedAt: now,
+ healthUpdatedAt: now,
+ updatedAt: now,
+ })
+ .where(
+ inArray(
+ aiModel.id,
+ dueModels.map((item) => item.id)
+ )
+ )
+ .returning();
+
+ if (revived.length > 0) {
+ console.info("[AI Model Health] Revived cooled-down models into observation pool:", {
+ count: revived.length,
+ models: revived.map((item) => ({
+ id: item.id,
+ modelId: item.modelId,
+ displayName: item.displayName,
+ })),
+ });
+ }
+
+ return revived;
+}
+
export async function recordAiModelHealthSuccess(params: {
- model?: Pick | null;
+ model?:
+ | Pick<
+ AiModel,
+ | "id"
+ | "healthStatus"
+ | "healthRecoverySuccessStreak"
+ | "healthObservationStartedAt"
+ >
+ | null;
}) {
const modelId = normalizeText(params.model?.id);
if (!modelId) return;
+ const [existing] = await db
+ .select({
+ id: aiModel.id,
+ healthStatus: aiModel.healthStatus,
+ healthRecoverySuccessStreak: aiModel.healthRecoverySuccessStreak,
+ healthObservationStartedAt: aiModel.healthObservationStartedAt,
+ })
+ .from(aiModel)
+ .where(eq(aiModel.id, modelId))
+ .limit(1);
+
+ if (!existing) return;
+
+ const now = new Date();
+ const isObservation = isObservationPoolModel(existing);
+ const nextRecoverySuccessStreak = isObservation
+ ? (existing.healthRecoverySuccessStreak || 0) + 1
+ : 0;
+ const recoveredToHealthy =
+ isObservation &&
+ nextRecoverySuccessStreak >= OBSERVATION_RECOVERY_SUCCESS_THRESHOLD;
+
+ const updateData: Partial = {
+ healthStatus:
+ existing.healthStatus === "auto_disabled"
+ ? "auto_disabled"
+ : recoveredToHealthy
+ ? "healthy"
+ : isObservation
+ ? "degraded"
+ : "healthy",
+ healthConsecutiveFailures: 0,
+ healthLastEventType: "success",
+ healthLastError: null,
+ healthLastSuccessAt: now,
+ healthRecoverySuccessStreak: recoveredToHealthy
+ ? 0
+ : isObservation
+ ? nextRecoverySuccessStreak
+ : 0,
+ healthObservationStartedAt: recoveredToHealthy
+ ? null
+ : isObservation
+ ? existing.healthObservationStartedAt
+ : null,
+ healthUpdatedAt: now,
+ updatedAt: now,
+ };
+
+ if (recoveredToHealthy) {
+ updateData.healthAutoDisabledAt = null;
+ updateData.healthAutoDisabledReason = null;
+ }
+
await db
.update(aiModel)
- .set({
- healthStatus:
- params.model?.healthStatus === "auto_disabled" ? "auto_disabled" : "healthy",
- healthConsecutiveFailures: 0,
- healthLastEventType: "success",
- healthLastError: null,
- healthLastSuccessAt: new Date(),
- healthUpdatedAt: new Date(),
- })
+ .set(updateData)
.where(eq(aiModel.id, modelId));
}
@@ -205,10 +341,15 @@ export async function recordAiModelHealthFailure(params: {
const [existing] = await db
.select({
id: aiModel.id,
+ modelId: aiModel.modelId,
+ displayName: aiModel.displayName,
isEnabled: aiModel.isEnabled,
excludeFromFailover: aiModel.excludeFromFailover,
healthStatus: aiModel.healthStatus,
healthConsecutiveFailures: aiModel.healthConsecutiveFailures,
+ healthRecoverySuccessStreak: aiModel.healthRecoverySuccessStreak,
+ healthObservationStartedAt: aiModel.healthObservationStartedAt,
+ healthAutoDisabledAt: aiModel.healthAutoDisabledAt,
})
.from(aiModel)
.where(eq(aiModel.id, modelId))
@@ -216,14 +357,19 @@ export async function recordAiModelHealthFailure(params: {
if (!existing) return;
+ const isObservation = isObservationPoolModel(existing);
const nextConsecutiveFailures = classification.counted
- ? Math.max(1, (existing.healthConsecutiveFailures || 0) + 1)
+ ? Math.max(
+ 1,
+ (isObservation ? 0 : existing.healthConsecutiveFailures || 0) + 1
+ )
: existing.healthConsecutiveFailures || 0;
const shouldAutoDisable =
classification.counted &&
!existing.excludeFromFailover &&
- existing.healthStatus !== "auto_disabled" &&
- nextConsecutiveFailures >= AUTO_DISABLE_FAILURE_THRESHOLD;
+ ((isObservation && existing.isEnabled) ||
+ (existing.healthStatus !== "auto_disabled" &&
+ nextConsecutiveFailures >= AUTO_DISABLE_FAILURE_THRESHOLD));
const nextStatus = shouldAutoDisable
? "auto_disabled"
: classification.counted
@@ -264,13 +410,23 @@ export async function recordAiModelHealthFailure(params: {
healthLastEventType: classification.eventType,
healthLastError: errorMessage,
healthLastFailureAt: now,
+ healthRecoverySuccessStreak: shouldAutoDisable
+ ? 0
+ : classification.counted
+ ? 0
+ : existing.healthRecoverySuccessStreak || 0,
+ healthObservationStartedAt: shouldAutoDisable
+ ? null
+ : isObservation
+ ? existing.healthObservationStartedAt
+ : null,
healthUpdatedAt: now,
updatedAt: now,
};
if (shouldAutoDisable) {
updateData.healthAutoDisabledAt = now;
updateData.healthAutoDisabledReason = autoDisabledReason || errorMessage;
- } else if (existing.healthStatus !== "auto_disabled") {
+ } else if (existing.healthStatus !== "auto_disabled" && !isObservation) {
updateData.healthAutoDisabledReason = null;
}
@@ -304,6 +460,8 @@ export async function restoreAiModelHealth(modelId: string) {
healthLastFailureAt: null,
healthAutoDisabledAt: null,
healthAutoDisabledReason: null,
+ healthRecoverySuccessStreak: 0,
+ healthObservationStartedAt: null,
healthUpdatedAt: new Date(),
updatedAt: new Date(),
})
diff --git a/lib/db/migrate.ts b/lib/db/migrate.ts
index deb3307a2..d4bcbb7bc 100644
--- a/lib/db/migrate.ts
+++ b/lib/db/migrate.ts
@@ -110,6 +110,14 @@ const runMigrate = async () => {
ALTER TABLE "AiModel"
ADD COLUMN IF NOT EXISTS "isDefaultTextEditModel" boolean NOT NULL DEFAULT false;
`);
+ await db.execute(sql`
+ ALTER TABLE "AiModel"
+ ADD COLUMN IF NOT EXISTS "healthRecoverySuccessStreak" integer NOT NULL DEFAULT 0;
+ `);
+ await db.execute(sql`
+ ALTER TABLE "AiModel"
+ ADD COLUMN IF NOT EXISTS "healthObservationStartedAt" timestamp;
+ `);
for (const item of providerBaseUrlDefaults) {
await db.execute(sql`
UPDATE "AiProvider"
diff --git a/lib/db/migrations/0080_add_ai_model_health_recovery.sql b/lib/db/migrations/0080_add_ai_model_health_recovery.sql
new file mode 100644
index 000000000..db9266981
--- /dev/null
+++ b/lib/db/migrations/0080_add_ai_model_health_recovery.sql
@@ -0,0 +1,5 @@
+ALTER TABLE "AiModel"
+ADD COLUMN IF NOT EXISTS "healthRecoverySuccessStreak" integer NOT NULL DEFAULT 0;
+
+ALTER TABLE "AiModel"
+ADD COLUMN IF NOT EXISTS "healthObservationStartedAt" timestamp;
diff --git a/lib/db/migrations/meta/_journal.json b/lib/db/migrations/meta/_journal.json
index bc98fadef..17d137ff0 100644
--- a/lib/db/migrations/meta/_journal.json
+++ b/lib/db/migrations/meta/_journal.json
@@ -463,6 +463,13 @@
"when": 1778252400000,
"tag": "0079_add_ai_model_health",
"breakpoints": true
+ },
+ {
+ "idx": 66,
+ "version": "7",
+ "when": 1778461200000,
+ "tag": "0080_add_ai_model_health_recovery",
+ "breakpoints": true
}
]
}
diff --git a/lib/db/queries.ts b/lib/db/queries.ts
index 6a87c2af5..68fc9d71c 100644
--- a/lib/db/queries.ts
+++ b/lib/db/queries.ts
@@ -21,6 +21,7 @@ import { drizzle } from "drizzle-orm/postgres-js";
import postgres from "postgres";
import type { ArtifactKind } from "@/components/artifact";
import type { VisibilityType } from "@/components/visibility-selector";
+import { reviveAiModelsFromCooldown } from "@/lib/ai/runtime/model-health";
import { ChatSDKError } from "../errors";
import type { AppUsage } from "../usage";
import { generateUUID } from "../utils";
@@ -3912,6 +3913,7 @@ export async function deleteAiProvider({ id }: { id: string }) {
export async function getAllAiModels(): Promise {
try {
+ await reviveAiModelsFromCooldown();
// 暂时移除排序以避免 sortOrder 类型问题
return await db.select().from(aiModel);
} catch (_error) {
@@ -3925,6 +3927,7 @@ export async function getAllAiModels(): Promise {
export async function getEnabledAiModels(): Promise {
try {
+ await reviveAiModelsFromCooldown();
return await db
.select()
.from(aiModel)
@@ -3943,6 +3946,7 @@ export async function getAiModelsByProviderId({
providerId: string;
}): Promise {
try {
+ await reviveAiModelsFromCooldown();
return await db
.select()
.from(aiModel)
@@ -3961,6 +3965,7 @@ export async function getAiModelById({
id: string;
}): Promise {
try {
+ await reviveAiModelsFromCooldown({ modelId: id });
const models = await db
.select()
.from(aiModel)
@@ -3977,6 +3982,7 @@ export async function getAiModelById({
export async function getDefaultAiModel(): Promise {
try {
+ await reviveAiModelsFromCooldown();
const models = await db
.select()
.from(aiModel)
@@ -4194,6 +4200,8 @@ export async function updateAiModel({
updateData.healthLastFailureAt = null;
updateData.healthAutoDisabledAt = null;
updateData.healthAutoDisabledReason = null;
+ updateData.healthRecoverySuccessStreak = 0;
+ updateData.healthObservationStartedAt = null;
updateData.healthUpdatedAt = new Date();
}
}
@@ -4477,6 +4485,7 @@ export async function getEnabledAiModelsWithProvider(
includeHidden: boolean = true
) {
try {
+ await reviveAiModelsFromCooldown();
const conditions = [
eq(aiModel.isEnabled, true),
eq(aiProvider.isEnabled, true),
@@ -4499,7 +4508,23 @@ export async function getEnabledAiModelsWithProvider(
.innerJoin(aiProvider, eq(aiModel.providerId, aiProvider.id))
.where(and(...conditions));
- return models;
+ const healthPriority = (status?: string | null) => {
+ if (status === "healthy") return 0;
+ if (status === "degraded") return 1;
+ if (status === "auto_disabled") return 2;
+ return 3;
+ };
+
+ return [...models].sort((left, right) => {
+ const byHealth =
+ healthPriority(left.model.healthStatus) -
+ healthPriority(right.model.healthStatus);
+ if (byHealth !== 0) return byHealth;
+ if (left.model.isDefault !== right.model.isDefault) {
+ return Number(right.model.isDefault) - Number(left.model.isDefault);
+ }
+ return (left.model.sortOrder ?? 0) - (right.model.sortOrder ?? 0);
+ });
} catch (_error) {
throw new ChatSDKError(
"bad_request:database",
@@ -4528,6 +4553,7 @@ export async function getAgentAiModelWithProvider() {
*/
export async function getSubModelsByParentModelId(parentModelId: string) {
try {
+ await reviveAiModelsFromCooldown();
const subModels = await db
.select({
model: aiModel,
diff --git a/lib/db/schema.ts b/lib/db/schema.ts
index 9482e61a8..8d42a99d9 100644
--- a/lib/db/schema.ts
+++ b/lib/db/schema.ts
@@ -349,6 +349,10 @@ export const aiModel = pgTable("AiModel", {
healthLastSuccessAt: timestamp("healthLastSuccessAt"),
healthAutoDisabledAt: timestamp("healthAutoDisabledAt"),
healthAutoDisabledReason: text("healthAutoDisabledReason"),
+ healthRecoverySuccessStreak: integer("healthRecoverySuccessStreak")
+ .notNull()
+ .default(0),
+ healthObservationStartedAt: timestamp("healthObservationStartedAt"),
healthUpdatedAt: timestamp("healthUpdatedAt").notNull().defaultNow(),
// 🆕 图片生成模型特性配置（用于阿里生图、Gemini等）
imageGenConfig: jsonb("imageGenConfig"), // { maxImages: 6, supportsSize: true, supportsPromptExtend: true, supportsNegativePrompt: true, enableLoadingPlaceholder: true }