518 lines
14 KiB
TypeScript
518 lines
14 KiB
TypeScript
import "server-only";
|
|
|
|
import { and, desc, eq, inArray, sql } from "drizzle-orm";
|
|
import { db } from "@/lib/db";
|
|
import {
|
|
aiModel,
|
|
aiModelHealthEvent,
|
|
type AiModel,
|
|
} from "@/lib/db/schema";
|
|
|
|
export type AiModelHealthEventType =
|
|
| "success"
|
|
| "timeout"
|
|
| "quota"
|
|
| "unavailable"
|
|
| "invalid_model"
|
|
| "download_error"
|
|
| "error";
|
|
|
|
export type AiModelHealthSeverity = "info" | "warning" | "error" | "critical";
|
|
|
|
const AUTO_DISABLE_FAILURE_THRESHOLD = 3;
|
|
const AUTO_DISABLE_COOLDOWN_MS = 2 * 60 * 60 * 1000;
|
|
const OBSERVATION_RECOVERY_SUCCESS_THRESHOLD = 3;
|
|
const RECENT_EVENT_LIMIT = 80;
|
|
|
|
function normalizeText(value: unknown) {
|
|
return String(value || "").trim();
|
|
}
|
|
|
|
function getErrorMessage(error: unknown) {
|
|
if (error instanceof Error) return normalizeText(error.message);
|
|
return normalizeText(error);
|
|
}
|
|
|
|
export function classifyAiModelHealthFailure(error: unknown): {
|
|
eventType: AiModelHealthEventType;
|
|
severity: AiModelHealthSeverity;
|
|
counted: boolean;
|
|
reason: string;
|
|
} {
|
|
const message = getErrorMessage(error);
|
|
const normalized = message.toLowerCase();
|
|
|
|
if (!message) {
|
|
return {
|
|
eventType: "error",
|
|
severity: "error",
|
|
counted: true,
|
|
reason: "unknown_error",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("生成结果未通过替换质检") ||
|
|
normalized.includes("quality check") ||
|
|
normalized.includes("review result") ||
|
|
normalized.includes("content policy") ||
|
|
normalized.includes("safety") ||
|
|
normalized.includes("user abort") ||
|
|
normalized.includes("aborted") ||
|
|
normalized.includes("cancelled")
|
|
) {
|
|
return {
|
|
eventType: "error",
|
|
severity: "warning",
|
|
counted: false,
|
|
reason: "non_provider_failure",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("model not exist") ||
|
|
normalized.includes("model_not_found") ||
|
|
normalized.includes("model not found") ||
|
|
normalized.includes("unknown model")
|
|
) {
|
|
return {
|
|
eventType: "invalid_model",
|
|
severity: "critical",
|
|
counted: true,
|
|
reason: "invalid_model",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("insufficient balance") ||
|
|
normalized.includes("quota") ||
|
|
normalized.includes("exceeded_current_quota") ||
|
|
normalized.includes("pre_consume_token_quota_failed") ||
|
|
normalized.includes("token quota is not enough")
|
|
) {
|
|
return {
|
|
eventType: "quota",
|
|
severity: "critical",
|
|
counted: true,
|
|
reason: "quota_or_balance",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("timeout") ||
|
|
normalized.includes("timed out") ||
|
|
normalized.includes("body timeout") ||
|
|
normalized.includes("headers timeout")
|
|
) {
|
|
return {
|
|
eventType: "timeout",
|
|
severity: "critical",
|
|
counted: true,
|
|
reason: "timeout",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("503") ||
|
|
normalized.includes("502") ||
|
|
normalized.includes("504") ||
|
|
normalized.includes("no available channel") ||
|
|
normalized.includes("no supported channel") ||
|
|
normalized.includes("service unavailable") ||
|
|
normalized.includes("fetch failed")
|
|
) {
|
|
return {
|
|
eventType: "unavailable",
|
|
severity: "critical",
|
|
counted: true,
|
|
reason: "provider_unavailable",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("download") ||
|
|
normalized.includes("no image url") ||
|
|
normalized.includes("no image") ||
|
|
normalized.includes("图片生成失败")
|
|
) {
|
|
return {
|
|
eventType: "download_error",
|
|
severity: "error",
|
|
counted: true,
|
|
reason: "result_download_or_parse_failed",
|
|
};
|
|
}
|
|
|
|
return {
|
|
eventType: "error",
|
|
severity: "error",
|
|
counted: true,
|
|
reason: "provider_error",
|
|
};
|
|
}
|
|
|
|
export function shouldFailoverAiModelError(error: unknown) {
|
|
return classifyAiModelHealthFailure(error).counted;
|
|
}
|
|
|
|
function buildMetadata(value?: Record<string, unknown>) {
|
|
if (!value || typeof value !== "object") return null;
|
|
try {
|
|
return JSON.parse(JSON.stringify(value));
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function isObservationPoolModel(
|
|
model:
|
|
| Pick<
|
|
AiModel,
|
|
| "healthStatus"
|
|
| "healthObservationStartedAt"
|
|
>
|
|
| null
|
|
| undefined
|
|
) {
|
|
return (
|
|
model?.healthStatus === "degraded" &&
|
|
Boolean(model.healthObservationStartedAt)
|
|
);
|
|
}
|
|
|
|
function getAutoDisableCooldownCutoff(now = new Date()) {
|
|
return new Date(now.getTime() - AUTO_DISABLE_COOLDOWN_MS);
|
|
}
|
|
|
|
export async function reviveAiModelsFromCooldown(params?: { modelId?: string }) {
|
|
const now = new Date();
|
|
const cooldownCutoff = getAutoDisableCooldownCutoff(now);
|
|
const conditions = [
|
|
eq(aiModel.isEnabled, false),
|
|
eq(aiModel.excludeFromFailover, false),
|
|
sql`COALESCE(${aiModel.healthAutoDisabledAt}, ${aiModel.healthLastFailureAt}, ${aiModel.healthUpdatedAt}) <= ${cooldownCutoff}`,
|
|
sql`(
|
|
${aiModel.healthStatus} = 'auto_disabled'
|
|
OR (
|
|
${aiModel.healthStatus} = 'degraded'
|
|
AND ${aiModel.healthLastFailureAt} IS NOT NULL
|
|
AND ${aiModel.healthConsecutiveFailures} > 0
|
|
AND ${aiModel.healthLastEventType} IN (
|
|
'timeout',
|
|
'quota',
|
|
'unavailable',
|
|
'invalid_model',
|
|
'download_error',
|
|
'error'
|
|
)
|
|
)
|
|
)`,
|
|
];
|
|
|
|
if (params?.modelId) {
|
|
conditions.push(eq(aiModel.id, normalizeText(params.modelId)));
|
|
}
|
|
|
|
const dueModels = await db
|
|
.select({
|
|
id: aiModel.id,
|
|
modelId: aiModel.modelId,
|
|
displayName: aiModel.displayName,
|
|
healthStatus: aiModel.healthStatus,
|
|
healthAutoDisabledAt: aiModel.healthAutoDisabledAt,
|
|
healthLastFailureAt: aiModel.healthLastFailureAt,
|
|
})
|
|
.from(aiModel)
|
|
.where(and(...conditions));
|
|
|
|
if (dueModels.length === 0) {
|
|
return [];
|
|
}
|
|
|
|
const revived = await db
|
|
.update(aiModel)
|
|
.set({
|
|
isEnabled: true,
|
|
healthStatus: "degraded",
|
|
healthConsecutiveFailures: 0,
|
|
healthRecoverySuccessStreak: 0,
|
|
healthObservationStartedAt: now,
|
|
healthUpdatedAt: now,
|
|
updatedAt: now,
|
|
})
|
|
.where(
|
|
inArray(
|
|
aiModel.id,
|
|
dueModels.map((item) => item.id)
|
|
)
|
|
)
|
|
.returning();
|
|
|
|
if (revived.length > 0) {
|
|
console.info("[AI Model Health] Revived cooled-down models into observation pool:", {
|
|
count: revived.length,
|
|
models: revived.map((item) => ({
|
|
id: item.id,
|
|
modelId: item.modelId,
|
|
displayName: item.displayName,
|
|
previousStatus: item.healthStatus,
|
|
})),
|
|
});
|
|
}
|
|
|
|
return revived;
|
|
}
|
|
|
|
export async function recordAiModelHealthSuccess(params: {
|
|
model?:
|
|
| Pick<
|
|
AiModel,
|
|
| "id"
|
|
| "healthStatus"
|
|
| "healthRecoverySuccessStreak"
|
|
| "healthObservationStartedAt"
|
|
>
|
|
| null;
|
|
}) {
|
|
const modelId = normalizeText(params.model?.id);
|
|
if (!modelId) return;
|
|
|
|
const [existing] = await db
|
|
.select({
|
|
id: aiModel.id,
|
|
healthStatus: aiModel.healthStatus,
|
|
healthRecoverySuccessStreak: aiModel.healthRecoverySuccessStreak,
|
|
healthObservationStartedAt: aiModel.healthObservationStartedAt,
|
|
})
|
|
.from(aiModel)
|
|
.where(eq(aiModel.id, modelId))
|
|
.limit(1);
|
|
|
|
if (!existing) return;
|
|
|
|
const now = new Date();
|
|
const isObservation = isObservationPoolModel(existing);
|
|
const nextRecoverySuccessStreak = isObservation
|
|
? (existing.healthRecoverySuccessStreak || 0) + 1
|
|
: 0;
|
|
const recoveredToHealthy =
|
|
isObservation &&
|
|
nextRecoverySuccessStreak >= OBSERVATION_RECOVERY_SUCCESS_THRESHOLD;
|
|
|
|
const updateData: Partial<AiModel> = {
|
|
healthStatus:
|
|
existing.healthStatus === "auto_disabled"
|
|
? "auto_disabled"
|
|
: recoveredToHealthy
|
|
? "healthy"
|
|
: isObservation
|
|
? "degraded"
|
|
: "healthy",
|
|
healthConsecutiveFailures: 0,
|
|
healthLastEventType: "success",
|
|
healthLastError: null,
|
|
healthLastSuccessAt: now,
|
|
healthRecoverySuccessStreak: recoveredToHealthy
|
|
? 0
|
|
: isObservation
|
|
? nextRecoverySuccessStreak
|
|
: 0,
|
|
healthObservationStartedAt: recoveredToHealthy
|
|
? null
|
|
: isObservation
|
|
? existing.healthObservationStartedAt
|
|
: null,
|
|
healthUpdatedAt: now,
|
|
updatedAt: now,
|
|
};
|
|
|
|
if (recoveredToHealthy) {
|
|
updateData.healthAutoDisabledAt = null;
|
|
updateData.healthAutoDisabledReason = null;
|
|
}
|
|
|
|
await db
|
|
.update(aiModel)
|
|
.set(updateData)
|
|
.where(eq(aiModel.id, modelId));
|
|
}
|
|
|
|
export async function recordAiModelHealthFailure(params: {
|
|
model?: (Partial<AiModel> & Pick<AiModel, "id">) | null;
|
|
provider?: {
|
|
id?: string | null;
|
|
name?: string | null;
|
|
type?: string | null;
|
|
} | null;
|
|
error: unknown;
|
|
context?: string;
|
|
elapsedMs?: number;
|
|
requestId?: string;
|
|
metadata?: Record<string, unknown>;
|
|
}) {
|
|
const modelId = normalizeText(params.model?.id);
|
|
if (!modelId) return;
|
|
|
|
const classification = classifyAiModelHealthFailure(params.error);
|
|
const errorMessage = getErrorMessage(params.error).slice(0, 1800);
|
|
const now = new Date();
|
|
|
|
const [existing] = await db
|
|
.select({
|
|
id: aiModel.id,
|
|
modelId: aiModel.modelId,
|
|
displayName: aiModel.displayName,
|
|
isEnabled: aiModel.isEnabled,
|
|
excludeFromFailover: aiModel.excludeFromFailover,
|
|
healthStatus: aiModel.healthStatus,
|
|
healthConsecutiveFailures: aiModel.healthConsecutiveFailures,
|
|
healthRecoverySuccessStreak: aiModel.healthRecoverySuccessStreak,
|
|
healthObservationStartedAt: aiModel.healthObservationStartedAt,
|
|
healthAutoDisabledAt: aiModel.healthAutoDisabledAt,
|
|
})
|
|
.from(aiModel)
|
|
.where(eq(aiModel.id, modelId))
|
|
.limit(1);
|
|
|
|
if (!existing) return;
|
|
|
|
const isObservation = isObservationPoolModel(existing);
|
|
const nextConsecutiveFailures = classification.counted
|
|
? Math.max(
|
|
1,
|
|
(isObservation ? 0 : existing.healthConsecutiveFailures || 0) + 1
|
|
)
|
|
: existing.healthConsecutiveFailures || 0;
|
|
const shouldAutoDisable =
|
|
classification.counted &&
|
|
!existing.excludeFromFailover &&
|
|
((isObservation && existing.isEnabled) ||
|
|
(existing.healthStatus !== "auto_disabled" &&
|
|
nextConsecutiveFailures >= AUTO_DISABLE_FAILURE_THRESHOLD));
|
|
const nextStatus = shouldAutoDisable
|
|
? "auto_disabled"
|
|
: classification.counted
|
|
? "degraded"
|
|
: existing.healthStatus;
|
|
const autoDisabledReason = shouldAutoDisable
|
|
? `${classification.reason}: ${errorMessage}`.slice(0, 1800)
|
|
: undefined;
|
|
|
|
await db
|
|
.insert(aiModelHealthEvent)
|
|
.values({
|
|
modelId,
|
|
providerId: normalizeText(params.provider?.id) || null,
|
|
modelRuntimeId: normalizeText(params.model?.modelId) || null,
|
|
modelDisplayName: normalizeText(params.model?.displayName) || null,
|
|
providerType: normalizeText(params.provider?.type) || null,
|
|
providerName: normalizeText(params.provider?.name) || null,
|
|
eventType: classification.eventType,
|
|
severity: classification.severity,
|
|
context: normalizeText(params.context).slice(0, 96) || null,
|
|
counted: classification.counted,
|
|
consecutiveFailures: nextConsecutiveFailures,
|
|
autoDisabled: shouldAutoDisable,
|
|
elapsedMs:
|
|
typeof params.elapsedMs === "number" && Number.isFinite(params.elapsedMs)
|
|
? Math.max(0, Math.round(params.elapsedMs))
|
|
: null,
|
|
requestId: normalizeText(params.requestId).slice(0, 128) || null,
|
|
errorMessage,
|
|
metadata: buildMetadata(params.metadata),
|
|
});
|
|
|
|
const updateData: Partial<AiModel> = {
|
|
isEnabled: shouldAutoDisable ? false : existing.isEnabled,
|
|
healthStatus: nextStatus,
|
|
healthConsecutiveFailures: nextConsecutiveFailures,
|
|
healthLastEventType: classification.eventType,
|
|
healthLastError: errorMessage,
|
|
healthLastFailureAt: now,
|
|
healthRecoverySuccessStreak: shouldAutoDisable
|
|
? 0
|
|
: classification.counted
|
|
? 0
|
|
: existing.healthRecoverySuccessStreak || 0,
|
|
healthObservationStartedAt: shouldAutoDisable
|
|
? null
|
|
: isObservation
|
|
? existing.healthObservationStartedAt
|
|
: null,
|
|
healthUpdatedAt: now,
|
|
updatedAt: now,
|
|
};
|
|
if (shouldAutoDisable) {
|
|
updateData.healthAutoDisabledAt = now;
|
|
updateData.healthAutoDisabledReason = autoDisabledReason || errorMessage;
|
|
} else if (existing.healthStatus !== "auto_disabled" && !isObservation) {
|
|
updateData.healthAutoDisabledReason = null;
|
|
}
|
|
|
|
await db.update(aiModel).set(updateData).where(eq(aiModel.id, modelId));
|
|
|
|
if (shouldAutoDisable) {
|
|
console.error("[AI Model Health] Auto disabled unhealthy model:", {
|
|
modelId,
|
|
runtimeModelId: params.model?.modelId,
|
|
displayName: params.model?.displayName,
|
|
providerType: params.provider?.type,
|
|
consecutiveFailures: nextConsecutiveFailures,
|
|
eventType: classification.eventType,
|
|
error: errorMessage,
|
|
});
|
|
}
|
|
}
|
|
|
|
export async function restoreAiModelHealth(modelId: string) {
|
|
const normalizedModelId = normalizeText(modelId);
|
|
if (!normalizedModelId) return [];
|
|
|
|
return await db
|
|
.update(aiModel)
|
|
.set({
|
|
isEnabled: true,
|
|
healthStatus: "healthy",
|
|
healthConsecutiveFailures: 0,
|
|
healthLastEventType: "success",
|
|
healthLastError: null,
|
|
healthLastFailureAt: null,
|
|
healthAutoDisabledAt: null,
|
|
healthAutoDisabledReason: null,
|
|
healthRecoverySuccessStreak: 0,
|
|
healthObservationStartedAt: null,
|
|
healthUpdatedAt: new Date(),
|
|
updatedAt: new Date(),
|
|
})
|
|
.where(eq(aiModel.id, normalizedModelId))
|
|
.returning();
|
|
}
|
|
|
|
export async function listAiModelHealthEvents(params?: { modelId?: string; limit?: number }) {
|
|
const limit = Math.max(1, Math.min(params?.limit || RECENT_EVENT_LIMIT, 200));
|
|
const modelId = normalizeText(params?.modelId);
|
|
const conditions = modelId ? [eq(aiModelHealthEvent.modelId, modelId)] : [];
|
|
|
|
return await db
|
|
.select()
|
|
.from(aiModelHealthEvent)
|
|
.where(conditions.length ? and(...conditions) : undefined)
|
|
.orderBy(desc(aiModelHealthEvent.createdAt))
|
|
.limit(limit);
|
|
}
|
|
|
|
export async function getAiModelHealthSummary() {
|
|
const rows = await db
|
|
.select({
|
|
modelId: aiModel.id,
|
|
eventCount: sql<number>`count(${aiModelHealthEvent.id})::int`,
|
|
countedEventCount: sql<number>`sum(case when ${aiModelHealthEvent.counted} then 1 else 0 end)::int`,
|
|
lastEventAt: sql<Date | null>`max(${aiModelHealthEvent.createdAt})`,
|
|
})
|
|
.from(aiModel)
|
|
.leftJoin(aiModelHealthEvent, eq(aiModel.id, aiModelHealthEvent.modelId))
|
|
.groupBy(aiModel.id);
|
|
|
|
return rows;
|
|
}
|