341 lines
9.5 KiB
TypeScript
341 lines
9.5 KiB
TypeScript
import "server-only";
|
|
|
|
import { and, desc, eq, sql } from "drizzle-orm";
|
|
import { db } from "@/lib/db";
|
|
import {
|
|
aiModel,
|
|
aiModelHealthEvent,
|
|
type AiModel,
|
|
} from "@/lib/db/schema";
|
|
|
|
export type AiModelHealthEventType =
|
|
| "success"
|
|
| "timeout"
|
|
| "quota"
|
|
| "unavailable"
|
|
| "invalid_model"
|
|
| "download_error"
|
|
| "error";
|
|
|
|
export type AiModelHealthSeverity = "info" | "warning" | "error" | "critical";
|
|
|
|
const AUTO_DISABLE_FAILURE_THRESHOLD = 3;
|
|
const RECENT_EVENT_LIMIT = 80;
|
|
|
|
function normalizeText(value: unknown) {
|
|
return String(value || "").trim();
|
|
}
|
|
|
|
function getErrorMessage(error: unknown) {
|
|
if (error instanceof Error) return normalizeText(error.message);
|
|
return normalizeText(error);
|
|
}
|
|
|
|
export function classifyAiModelHealthFailure(error: unknown): {
|
|
eventType: AiModelHealthEventType;
|
|
severity: AiModelHealthSeverity;
|
|
counted: boolean;
|
|
reason: string;
|
|
} {
|
|
const message = getErrorMessage(error);
|
|
const normalized = message.toLowerCase();
|
|
|
|
if (!message) {
|
|
return {
|
|
eventType: "error",
|
|
severity: "error",
|
|
counted: true,
|
|
reason: "unknown_error",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("生成结果未通过替换质检") ||
|
|
normalized.includes("quality check") ||
|
|
normalized.includes("review result") ||
|
|
normalized.includes("content policy") ||
|
|
normalized.includes("safety") ||
|
|
normalized.includes("user abort") ||
|
|
normalized.includes("aborted") ||
|
|
normalized.includes("cancelled")
|
|
) {
|
|
return {
|
|
eventType: "error",
|
|
severity: "warning",
|
|
counted: false,
|
|
reason: "non_provider_failure",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("model not exist") ||
|
|
normalized.includes("model_not_found") ||
|
|
normalized.includes("model not found") ||
|
|
normalized.includes("unknown model")
|
|
) {
|
|
return {
|
|
eventType: "invalid_model",
|
|
severity: "critical",
|
|
counted: true,
|
|
reason: "invalid_model",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("insufficient balance") ||
|
|
normalized.includes("quota") ||
|
|
normalized.includes("exceeded_current_quota") ||
|
|
normalized.includes("pre_consume_token_quota_failed") ||
|
|
normalized.includes("token quota is not enough")
|
|
) {
|
|
return {
|
|
eventType: "quota",
|
|
severity: "critical",
|
|
counted: true,
|
|
reason: "quota_or_balance",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("timeout") ||
|
|
normalized.includes("timed out") ||
|
|
normalized.includes("body timeout") ||
|
|
normalized.includes("headers timeout")
|
|
) {
|
|
return {
|
|
eventType: "timeout",
|
|
severity: "critical",
|
|
counted: true,
|
|
reason: "timeout",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("503") ||
|
|
normalized.includes("502") ||
|
|
normalized.includes("504") ||
|
|
normalized.includes("no available channel") ||
|
|
normalized.includes("no supported channel") ||
|
|
normalized.includes("service unavailable") ||
|
|
normalized.includes("fetch failed")
|
|
) {
|
|
return {
|
|
eventType: "unavailable",
|
|
severity: "critical",
|
|
counted: true,
|
|
reason: "provider_unavailable",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("download") ||
|
|
normalized.includes("no image url") ||
|
|
normalized.includes("no image") ||
|
|
normalized.includes("图片生成失败")
|
|
) {
|
|
return {
|
|
eventType: "download_error",
|
|
severity: "error",
|
|
counted: true,
|
|
reason: "result_download_or_parse_failed",
|
|
};
|
|
}
|
|
|
|
return {
|
|
eventType: "error",
|
|
severity: "error",
|
|
counted: true,
|
|
reason: "provider_error",
|
|
};
|
|
}
|
|
|
|
export function shouldFailoverAiModelError(error: unknown) {
|
|
return classifyAiModelHealthFailure(error).counted;
|
|
}
|
|
|
|
function buildMetadata(value?: Record<string, unknown>) {
|
|
if (!value || typeof value !== "object") return null;
|
|
try {
|
|
return JSON.parse(JSON.stringify(value));
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export async function recordAiModelHealthSuccess(params: {
|
|
model?: Pick<AiModel, "id" | "healthStatus"> | null;
|
|
}) {
|
|
const modelId = normalizeText(params.model?.id);
|
|
if (!modelId) return;
|
|
|
|
await db
|
|
.update(aiModel)
|
|
.set({
|
|
healthStatus:
|
|
params.model?.healthStatus === "auto_disabled" ? "auto_disabled" : "healthy",
|
|
healthConsecutiveFailures: 0,
|
|
healthLastEventType: "success",
|
|
healthLastError: null,
|
|
healthLastSuccessAt: new Date(),
|
|
healthUpdatedAt: new Date(),
|
|
})
|
|
.where(eq(aiModel.id, modelId));
|
|
}
|
|
|
|
export async function recordAiModelHealthFailure(params: {
|
|
model?: (Partial<AiModel> & Pick<AiModel, "id">) | null;
|
|
provider?: {
|
|
id?: string | null;
|
|
name?: string | null;
|
|
type?: string | null;
|
|
} | null;
|
|
error: unknown;
|
|
context?: string;
|
|
elapsedMs?: number;
|
|
requestId?: string;
|
|
metadata?: Record<string, unknown>;
|
|
}) {
|
|
const modelId = normalizeText(params.model?.id);
|
|
if (!modelId) return;
|
|
|
|
const classification = classifyAiModelHealthFailure(params.error);
|
|
const errorMessage = getErrorMessage(params.error).slice(0, 1800);
|
|
const now = new Date();
|
|
|
|
const [existing] = await db
|
|
.select({
|
|
id: aiModel.id,
|
|
isEnabled: aiModel.isEnabled,
|
|
excludeFromFailover: aiModel.excludeFromFailover,
|
|
healthStatus: aiModel.healthStatus,
|
|
healthConsecutiveFailures: aiModel.healthConsecutiveFailures,
|
|
})
|
|
.from(aiModel)
|
|
.where(eq(aiModel.id, modelId))
|
|
.limit(1);
|
|
|
|
if (!existing) return;
|
|
|
|
const nextConsecutiveFailures = classification.counted
|
|
? Math.max(1, (existing.healthConsecutiveFailures || 0) + 1)
|
|
: existing.healthConsecutiveFailures || 0;
|
|
const shouldAutoDisable =
|
|
classification.counted &&
|
|
!existing.excludeFromFailover &&
|
|
existing.healthStatus !== "auto_disabled" &&
|
|
nextConsecutiveFailures >= AUTO_DISABLE_FAILURE_THRESHOLD;
|
|
const nextStatus = shouldAutoDisable
|
|
? "auto_disabled"
|
|
: classification.counted
|
|
? "degraded"
|
|
: existing.healthStatus;
|
|
const autoDisabledReason = shouldAutoDisable
|
|
? `${classification.reason}: ${errorMessage}`.slice(0, 1800)
|
|
: undefined;
|
|
|
|
await db
|
|
.insert(aiModelHealthEvent)
|
|
.values({
|
|
modelId,
|
|
providerId: normalizeText(params.provider?.id) || null,
|
|
modelRuntimeId: normalizeText(params.model?.modelId) || null,
|
|
modelDisplayName: normalizeText(params.model?.displayName) || null,
|
|
providerType: normalizeText(params.provider?.type) || null,
|
|
providerName: normalizeText(params.provider?.name) || null,
|
|
eventType: classification.eventType,
|
|
severity: classification.severity,
|
|
context: normalizeText(params.context).slice(0, 96) || null,
|
|
counted: classification.counted,
|
|
consecutiveFailures: nextConsecutiveFailures,
|
|
autoDisabled: shouldAutoDisable,
|
|
elapsedMs:
|
|
typeof params.elapsedMs === "number" && Number.isFinite(params.elapsedMs)
|
|
? Math.max(0, Math.round(params.elapsedMs))
|
|
: null,
|
|
requestId: normalizeText(params.requestId).slice(0, 128) || null,
|
|
errorMessage,
|
|
metadata: buildMetadata(params.metadata),
|
|
});
|
|
|
|
const updateData: Partial<AiModel> = {
|
|
isEnabled: shouldAutoDisable ? false : existing.isEnabled,
|
|
healthStatus: nextStatus,
|
|
healthConsecutiveFailures: nextConsecutiveFailures,
|
|
healthLastEventType: classification.eventType,
|
|
healthLastError: errorMessage,
|
|
healthLastFailureAt: now,
|
|
healthUpdatedAt: now,
|
|
updatedAt: now,
|
|
};
|
|
if (shouldAutoDisable) {
|
|
updateData.healthAutoDisabledAt = now;
|
|
updateData.healthAutoDisabledReason = autoDisabledReason || errorMessage;
|
|
} else if (existing.healthStatus !== "auto_disabled") {
|
|
updateData.healthAutoDisabledReason = null;
|
|
}
|
|
|
|
await db.update(aiModel).set(updateData).where(eq(aiModel.id, modelId));
|
|
|
|
if (shouldAutoDisable) {
|
|
console.error("[AI Model Health] Auto disabled unhealthy model:", {
|
|
modelId,
|
|
runtimeModelId: params.model?.modelId,
|
|
displayName: params.model?.displayName,
|
|
providerType: params.provider?.type,
|
|
consecutiveFailures: nextConsecutiveFailures,
|
|
eventType: classification.eventType,
|
|
error: errorMessage,
|
|
});
|
|
}
|
|
}
|
|
|
|
export async function restoreAiModelHealth(modelId: string) {
|
|
const normalizedModelId = normalizeText(modelId);
|
|
if (!normalizedModelId) return [];
|
|
|
|
return await db
|
|
.update(aiModel)
|
|
.set({
|
|
isEnabled: true,
|
|
healthStatus: "healthy",
|
|
healthConsecutiveFailures: 0,
|
|
healthLastEventType: "success",
|
|
healthLastError: null,
|
|
healthLastFailureAt: null,
|
|
healthAutoDisabledAt: null,
|
|
healthAutoDisabledReason: null,
|
|
healthUpdatedAt: new Date(),
|
|
updatedAt: new Date(),
|
|
})
|
|
.where(eq(aiModel.id, normalizedModelId))
|
|
.returning();
|
|
}
|
|
|
|
export async function listAiModelHealthEvents(params?: { modelId?: string; limit?: number }) {
|
|
const limit = Math.max(1, Math.min(params?.limit || RECENT_EVENT_LIMIT, 200));
|
|
const modelId = normalizeText(params?.modelId);
|
|
const conditions = modelId ? [eq(aiModelHealthEvent.modelId, modelId)] : [];
|
|
|
|
return await db
|
|
.select()
|
|
.from(aiModelHealthEvent)
|
|
.where(conditions.length ? and(...conditions) : undefined)
|
|
.orderBy(desc(aiModelHealthEvent.createdAt))
|
|
.limit(limit);
|
|
}
|
|
|
|
export async function getAiModelHealthSummary() {
|
|
const rows = await db
|
|
.select({
|
|
modelId: aiModel.id,
|
|
eventCount: sql<number>`count(${aiModelHealthEvent.id})::int`,
|
|
countedEventCount: sql<number>`sum(case when ${aiModelHealthEvent.counted} then 1 else 0 end)::int`,
|
|
lastEventAt: sql<Date | null>`max(${aiModelHealthEvent.createdAt})`,
|
|
})
|
|
.from(aiModel)
|
|
.leftJoin(aiModelHealthEvent, eq(aiModel.id, aiModelHealthEvent.modelId))
|
|
.groupBy(aiModel.id);
|
|
|
|
return rows;
|
|
}
|