lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add ai model health recovery pool

Browse Source
This commit is contained in:
Your Name 2026-05-11 20:27:21 +08:00
parent 1ed3184000
commit 3ebe7db37d
7 changed files with 274 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
app/(admin)/settings/admin-settings-client.tsx
View File

@ -863,6 +863,8 @@ type AiModel = {
healthLastSuccessAt?: string | null;
healthAutoDisabledAt?: string | null;
healthAutoDisabledReason?: string | null;
healthRecoverySuccessStreak?: number;
healthObservationStartedAt?: string | null;
isAgentModel?: boolean;
agentCodingModelId?: string;
isMultimodal?: boolean;
@ -15677,6 +15679,19 @@ export function AdminSettingsClient() {
supportsImageGen: model.supportsImageGen,
});
const healthStatus = model.healthStatus || "healthy";
const isObservationPool =
healthStatus === "degraded" &&
Boolean(model.healthObservationStartedAt);
const recoverySuccessStreak = Math.max(
0,
Number(model.healthRecoverySuccessStreak || 0)
);
const autoDisableCooldownUntil = model.healthAutoDisabledAt
? new Date(
new Date(model.healthAutoDisabledAt).getTime() +
2 * 60 * 60 * 1000
)
: null;
const modelHealthEvents =
modelHealthEventsByModelId.get(model.id) || [];
const latestHealthEvent = modelHealthEvents[0];
@ -15756,7 +15771,9 @@ export function AdminSettingsClient() {
)}
{healthStatus === "degraded" && (
<span className="text-xs bg-orange-100 text-orange-700 px-2 py-0.5 rounded">
健康观察中
{isObservationPool
? `观察池 ${recoverySuccessStreak}/3`
: "健康观察中"}
</span>
)}
</div>
@ -15784,6 +15801,38 @@ export function AdminSettingsClient() {
})}
</span>
)}
{healthStatus === "auto_disabled" &&
autoDisableCooldownUntil && (
<span>
解冻时间：
{formatBeijingTime(autoDisableCooldownUntil, {
month: "2-digit",
day: "2-digit",
hour: "2-digit",
minute: "2-digit",
})}
</span>
)}
{isObservationPool && (
<span>
观察成功：{recoverySuccessStreak}/3
</span>
)}
{isObservationPool &&
model.healthObservationStartedAt && (
<span>
观察开始：
{formatBeijingTime(
model.healthObservationStartedAt,
{
month: "2-digit",
day: "2-digit",
hour: "2-digit",
minute: "2-digit",
}
)}
</span>
)}
</div>
{(model.healthLastError || latestHealthEvent?.errorMessage) && (
<div className="line-clamp-2 max-w-[760px]">

188
lib/ai/runtime/model-health.ts
View File

@ -1,6 +1,6 @@
import "server-only";
import { and, desc, eq, sql } from "drizzle-orm";
import { and, desc, eq, inArray, lte, sql } from "drizzle-orm";
import { db } from "@/lib/db";
import {
aiModel,
@ -20,6 +20,8 @@ export type AiModelHealthEventType =
export type AiModelHealthSeverity = "info" | "warning" | "error" | "critical";
const AUTO_DISABLE_FAILURE_THRESHOLD = 3;
const AUTO_DISABLE_COOLDOWN_MS = 2 * 60 * 60 * 1000;
const OBSERVATION_RECOVERY_SUCCESS_THRESHOLD = 3;
const RECENT_EVENT_LIMIT = 80;
function normalizeText(value: unknown) {
@ -162,23 +164,157 @@ function buildMetadata(value?: Record<string, unknown>) {
}
}
function isObservationPoolModel(
model:
| Pick<
AiModel,
| "healthStatus"
| "healthObservationStartedAt"
>
| null
| undefined
) {
return (
model?.healthStatus === "degraded" &&
Boolean(model.healthObservationStartedAt)
);
}
function getAutoDisableCooldownCutoff(now = new Date()) {
return new Date(now.getTime() - AUTO_DISABLE_COOLDOWN_MS);
}
export async function reviveAiModelsFromCooldown(params?: { modelId?: string }) {
const now = new Date();
const cooldownCutoff = getAutoDisableCooldownCutoff(now);
const conditions = [
eq(aiModel.healthStatus, "auto_disabled"),
eq(aiModel.isEnabled, false),
eq(aiModel.excludeFromFailover, false),
lte(aiModel.healthAutoDisabledAt, cooldownCutoff),
];
if (params?.modelId) {
conditions.push(eq(aiModel.id, normalizeText(params.modelId)));
}
const dueModels = await db
.select({
id: aiModel.id,
modelId: aiModel.modelId,
displayName: aiModel.displayName,
})
.from(aiModel)
.where(and(...conditions));
if (dueModels.length === 0) {
return [];
}
const revived = await db
.update(aiModel)
.set({
isEnabled: true,
healthStatus: "degraded",
healthConsecutiveFailures: 0,
healthRecoverySuccessStreak: 0,
healthObservationStartedAt: now,
healthUpdatedAt: now,
updatedAt: now,
})
.where(
inArray(
aiModel.id,
dueModels.map((item) => item.id)
)
)
.returning();
if (revived.length > 0) {
console.info("[AI Model Health] Revived cooled-down models into observation pool:", {
count: revived.length,
models: revived.map((item) => ({
id: item.id,
modelId: item.modelId,
displayName: item.displayName,
})),
});
}
return revived;
}
export async function recordAiModelHealthSuccess(params: {
model?: Pick<AiModel, "id" | "healthStatus"> | null;
model?:
| Pick<
AiModel,
| "id"
| "healthStatus"
| "healthRecoverySuccessStreak"
| "healthObservationStartedAt"
>
| null;
}) {
const modelId = normalizeText(params.model?.id);
if (!modelId) return;
const [existing] = await db
.select({
id: aiModel.id,
healthStatus: aiModel.healthStatus,
healthRecoverySuccessStreak: aiModel.healthRecoverySuccessStreak,
healthObservationStartedAt: aiModel.healthObservationStartedAt,
})
.from(aiModel)
.where(eq(aiModel.id, modelId))
.limit(1);
if (!existing) return;
const now = new Date();
const isObservation = isObservationPoolModel(existing);
const nextRecoverySuccessStreak = isObservation
? (existing.healthRecoverySuccessStreak || 0) + 1
: 0;
const recoveredToHealthy =
isObservation &&
nextRecoverySuccessStreak >= OBSERVATION_RECOVERY_SUCCESS_THRESHOLD;
const updateData: Partial<AiModel> = {
healthStatus:
existing.healthStatus === "auto_disabled"
? "auto_disabled"
: recoveredToHealthy
? "healthy"
: isObservation
? "degraded"
: "healthy",
healthConsecutiveFailures: 0,
healthLastEventType: "success",
healthLastError: null,
healthLastSuccessAt: now,
healthRecoverySuccessStreak: recoveredToHealthy
? 0
: isObservation
? nextRecoverySuccessStreak
: 0,
healthObservationStartedAt: recoveredToHealthy
? null
: isObservation
? existing.healthObservationStartedAt
: null,
healthUpdatedAt: now,
updatedAt: now,
};
if (recoveredToHealthy) {
updateData.healthAutoDisabledAt = null;
updateData.healthAutoDisabledReason = null;
}
await db
.update(aiModel)
.set({
healthStatus:
params.model?.healthStatus === "auto_disabled" ? "auto_disabled" : "healthy",
healthConsecutiveFailures: 0,
healthLastEventType: "success",
healthLastError: null,
healthLastSuccessAt: new Date(),
healthUpdatedAt: new Date(),
})
.set(updateData)
.where(eq(aiModel.id, modelId));
}
@ -205,10 +341,15 @@ export async function recordAiModelHealthFailure(params: {
const [existing] = await db
.select({
id: aiModel.id,
modelId: aiModel.modelId,
displayName: aiModel.displayName,
isEnabled: aiModel.isEnabled,
excludeFromFailover: aiModel.excludeFromFailover,
healthStatus: aiModel.healthStatus,
healthConsecutiveFailures: aiModel.healthConsecutiveFailures,
healthRecoverySuccessStreak: aiModel.healthRecoverySuccessStreak,
healthObservationStartedAt: aiModel.healthObservationStartedAt,
healthAutoDisabledAt: aiModel.healthAutoDisabledAt,
})
.from(aiModel)
.where(eq(aiModel.id, modelId))
@ -216,14 +357,19 @@ export async function recordAiModelHealthFailure(params: {
if (!existing) return;
const isObservation = isObservationPoolModel(existing);
const nextConsecutiveFailures = classification.counted
? Math.max(1, (existing.healthConsecutiveFailures || 0) + 1)
? Math.max(
1,
(isObservation ? 0 : existing.healthConsecutiveFailures || 0) + 1
)
: existing.healthConsecutiveFailures || 0;
const shouldAutoDisable =
classification.counted &&
!existing.excludeFromFailover &&
existing.healthStatus !== "auto_disabled" &&
nextConsecutiveFailures >= AUTO_DISABLE_FAILURE_THRESHOLD;
((isObservation && existing.isEnabled) ||
(existing.healthStatus !== "auto_disabled" &&
nextConsecutiveFailures >= AUTO_DISABLE_FAILURE_THRESHOLD));
const nextStatus = shouldAutoDisable
? "auto_disabled"
: classification.counted
@ -264,13 +410,23 @@ export async function recordAiModelHealthFailure(params: {
healthLastEventType: classification.eventType,
healthLastError: errorMessage,
healthLastFailureAt: now,
healthRecoverySuccessStreak: shouldAutoDisable
? 0
: classification.counted
? 0
: existing.healthRecoverySuccessStreak || 0,
healthObservationStartedAt: shouldAutoDisable
? null
: isObservation
? existing.healthObservationStartedAt
: null,
healthUpdatedAt: now,
updatedAt: now,
};
if (shouldAutoDisable) {
updateData.healthAutoDisabledAt = now;
updateData.healthAutoDisabledReason = autoDisabledReason || errorMessage;
} else if (existing.healthStatus !== "auto_disabled") {
} else if (existing.healthStatus !== "auto_disabled" && !isObservation) {
updateData.healthAutoDisabledReason = null;
}
@ -304,6 +460,8 @@ export async function restoreAiModelHealth(modelId: string) {
healthLastFailureAt: null,
healthAutoDisabledAt: null,
healthAutoDisabledReason: null,
healthRecoverySuccessStreak: 0,
healthObservationStartedAt: null,
healthUpdatedAt: new Date(),
updatedAt: new Date(),
})

8
lib/db/migrate.ts
View File

@ -110,6 +110,14 @@ const runMigrate = async () => {
ALTER TABLE "AiModel"
ADD COLUMN IF NOT EXISTS "isDefaultTextEditModel" boolean NOT NULL DEFAULT false;
`);
await db.execute(sql`
ALTER TABLE "AiModel"
ADD COLUMN IF NOT EXISTS "healthRecoverySuccessStreak" integer NOT NULL DEFAULT 0;
`);
await db.execute(sql`
ALTER TABLE "AiModel"
ADD COLUMN IF NOT EXISTS "healthObservationStartedAt" timestamp;
`);
for (const item of providerBaseUrlDefaults) {
await db.execute(sql`
UPDATE "AiProvider"

5
lib/db/migrations/0080_add_ai_model_health_recovery.sql Normal file
View File

@ -0,0 +1,5 @@
ALTER TABLE "AiModel"
ADD COLUMN IF NOT EXISTS "healthRecoverySuccessStreak" integer NOT NULL DEFAULT 0;
ALTER TABLE "AiModel"
ADD COLUMN IF NOT EXISTS "healthObservationStartedAt" timestamp;

7
lib/db/migrations/meta/_journal.json
View File

@ -463,6 +463,13 @@
"when": 1778252400000,
"tag": "0079_add_ai_model_health",
"breakpoints": true
},
{
"idx": 66,
"version": "7",
"when": 1778461200000,
"tag": "0080_add_ai_model_health_recovery",
"breakpoints": true
}
]
}

28
lib/db/queries.ts
View File

@ -21,6 +21,7 @@ import { drizzle } from "drizzle-orm/postgres-js";
import postgres from "postgres";
import type { ArtifactKind } from "@/components/artifact";
import type { VisibilityType } from "@/components/visibility-selector";
import { reviveAiModelsFromCooldown } from "@/lib/ai/runtime/model-health";
import { ChatSDKError } from "../errors";
import type { AppUsage } from "../usage";
import { generateUUID } from "../utils";
@ -3912,6 +3913,7 @@ export async function deleteAiProvider({ id }: { id: string }) {
export async function getAllAiModels(): Promise<AiModel[]> {
try {
await reviveAiModelsFromCooldown();
// 暂时移除排序以避免 sortOrder 类型问题
return await db.select().from(aiModel);
} catch (_error) {
@ -3925,6 +3927,7 @@ export async function getAllAiModels(): Promise<AiModel[]> {
export async function getEnabledAiModels(): Promise<AiModel[]> {
try {
await reviveAiModelsFromCooldown();
return await db
.select()
.from(aiModel)
@ -3943,6 +3946,7 @@ export async function getAiModelsByProviderId({
providerId: string;
}): Promise<AiModel[]> {
try {
await reviveAiModelsFromCooldown();
return await db
.select()
.from(aiModel)
@ -3961,6 +3965,7 @@ export async function getAiModelById({
id: string;
}): Promise<AiModel | undefined> {
try {
await reviveAiModelsFromCooldown({ modelId: id });
const models = await db
.select()
.from(aiModel)
@ -3977,6 +3982,7 @@ export async function getAiModelById({
export async function getDefaultAiModel(): Promise<AiModel | undefined> {
try {
await reviveAiModelsFromCooldown();
const models = await db
.select()
.from(aiModel)
@ -4194,6 +4200,8 @@ export async function updateAiModel({
updateData.healthLastFailureAt = null;
updateData.healthAutoDisabledAt = null;
updateData.healthAutoDisabledReason = null;
updateData.healthRecoverySuccessStreak = 0;
updateData.healthObservationStartedAt = null;
updateData.healthUpdatedAt = new Date();
}
}
@ -4477,6 +4485,7 @@ export async function getEnabledAiModelsWithProvider(
includeHidden: boolean = true
) {
try {
await reviveAiModelsFromCooldown();
const conditions = [
eq(aiModel.isEnabled, true),
eq(aiProvider.isEnabled, true),
@ -4499,7 +4508,23 @@ export async function getEnabledAiModelsWithProvider(
.innerJoin(aiProvider, eq(aiModel.providerId, aiProvider.id))
.where(and(...conditions));
return models;
const healthPriority = (status?: string | null) => {
if (status === "healthy") return 0;
if (status === "degraded") return 1;
if (status === "auto_disabled") return 2;
return 3;
};
return [...models].sort((left, right) => {
const byHealth =
healthPriority(left.model.healthStatus) -
healthPriority(right.model.healthStatus);
if (byHealth !== 0) return byHealth;
if (left.model.isDefault !== right.model.isDefault) {
return Number(right.model.isDefault) - Number(left.model.isDefault);
}
return (left.model.sortOrder ?? 0) - (right.model.sortOrder ?? 0);
});
} catch (_error) {
throw new ChatSDKError(
"bad_request:database",
@ -4528,6 +4553,7 @@ export async function getAgentAiModelWithProvider() {
*/
export async function getSubModelsByParentModelId(parentModelId: string) {
try {
await reviveAiModelsFromCooldown();
const subModels = await db
.select({
model: aiModel,

4
lib/db/schema.ts
View File

@ -349,6 +349,10 @@ export const aiModel = pgTable("AiModel", {
healthLastSuccessAt: timestamp("healthLastSuccessAt"),
healthAutoDisabledAt: timestamp("healthAutoDisabledAt"),
healthAutoDisabledReason: text("healthAutoDisabledReason"),
healthRecoverySuccessStreak: integer("healthRecoverySuccessStreak")
.notNull()
.default(0),
healthObservationStartedAt: timestamp("healthObservationStartedAt"),
healthUpdatedAt: timestamp("healthUpdatedAt").notNull().defaultNow(),
// 🆕 图片生成模型特性配置（用于阿里生图、Gemini等）
imageGenConfig: jsonb("imageGenConfig"), // { maxImages: 6, supportsSize: true, supportsPromptExtend: true, supportsNegativePrompt: true, enableLoadingPlaceholder: true }