809 lines
24 KiB
TypeScript
809 lines
24 KiB
TypeScript
import "server-only";
|
|
|
|
import { and, desc, eq, inArray, sql } from "drizzle-orm";
|
|
import { db } from "@/lib/db";
|
|
import {
|
|
aiModel,
|
|
aiModelHealthEvent,
|
|
type AiModel,
|
|
} from "@/lib/db/schema";
|
|
import {
|
|
isMediaInputFailure,
|
|
isNonProviderRequestStatus,
|
|
} from "@/lib/ai/runtime/request-failure-classification";
|
|
import { isProviderBalanceError, isRateLimitError } from "@/lib/request-error";
|
|
|
|
export type AiModelHealthEventType =
|
|
| "success"
|
|
| "timeout"
|
|
| "quota"
|
|
| "unavailable"
|
|
| "invalid_model"
|
|
| "download_error"
|
|
| "error";
|
|
|
|
export type AiModelHealthSeverity = "info" | "warning" | "error" | "critical";
|
|
|
|
const AUTO_DISABLE_FAILURE_THRESHOLD = 3;
|
|
const AUTO_DISABLE_COOLDOWN_MS = 2 * 60 * 60 * 1000;
|
|
const OBSERVATION_RECOVERY_SUCCESS_THRESHOLD = 3;
|
|
const RECENT_EVENT_LIMIT = 80;
|
|
function normalizeText(value: unknown) {
|
|
return String(value || "").trim();
|
|
}
|
|
|
|
function getErrorMessage(error: unknown) {
|
|
if (error instanceof Error) {
|
|
const parts = [normalizeText(error.message)];
|
|
const cause = (error as Error & { cause?: unknown }).cause;
|
|
if (cause instanceof Error) {
|
|
parts.push(normalizeText(cause.message));
|
|
} else if (typeof cause === "string") {
|
|
parts.push(normalizeText(cause));
|
|
}
|
|
return parts.filter(Boolean).join(": ");
|
|
}
|
|
return normalizeText(error);
|
|
}
|
|
|
|
function parseHttpStatus(value: unknown) {
|
|
if (typeof value === "number" && Number.isInteger(value)) {
|
|
return value >= 100 && value <= 599 ? value : null;
|
|
}
|
|
|
|
if (typeof value === "string" && /^\d{3}$/.test(value.trim())) {
|
|
const parsed = Number(value.trim());
|
|
return parsed >= 100 && parsed <= 599 ? parsed : null;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function getErrorHttpStatus(error: unknown, depth = 0): number | null {
|
|
if (!error || typeof error !== "object" || depth > 2) return null;
|
|
|
|
const record = error as Record<string, unknown>;
|
|
for (const key of ["status", "statusCode", "httpStatus"]) {
|
|
const status = parseHttpStatus(record[key]);
|
|
if (status !== null) return status;
|
|
}
|
|
|
|
for (const key of ["response", "cause"]) {
|
|
const nestedStatus = getErrorHttpStatus(record[key], depth + 1);
|
|
if (nestedStatus !== null) return nestedStatus;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function isExplicitUserCancellation(normalized: string) {
|
|
return (
|
|
normalized.includes("user abort") ||
|
|
normalized.includes("user aborted") ||
|
|
normalized.includes("user cancelled") ||
|
|
normalized.includes("user canceled") ||
|
|
normalized.includes("client aborted") ||
|
|
normalized.includes("request cancelled by user") ||
|
|
normalized.includes("request canceled by user")
|
|
);
|
|
}
|
|
|
|
function isOperationalAbort(error: unknown, normalized: string) {
|
|
if (isExplicitUserCancellation(normalized)) return false;
|
|
if (error instanceof Error && error.name === "AbortError") return true;
|
|
return (
|
|
normalized.includes("this operation was aborted") ||
|
|
normalized.includes("the operation was aborted") ||
|
|
normalized.includes("operation was aborted") ||
|
|
normalized.includes("request aborted")
|
|
);
|
|
}
|
|
|
|
export function classifyAiModelHealthFailure(error: unknown): {
|
|
eventType: AiModelHealthEventType;
|
|
severity: AiModelHealthSeverity;
|
|
counted: boolean;
|
|
reason: string;
|
|
} {
|
|
const message = getErrorMessage(error);
|
|
const normalized = message.toLowerCase();
|
|
const httpStatus = getErrorHttpStatus(error);
|
|
|
|
if (!message) {
|
|
return {
|
|
eventType: "error",
|
|
severity: "error",
|
|
counted: true,
|
|
reason: "unknown_error",
|
|
};
|
|
}
|
|
|
|
if (isExplicitUserCancellation(normalized)) {
|
|
return {
|
|
eventType: "error",
|
|
severity: "warning",
|
|
counted: false,
|
|
reason: "user_cancelled",
|
|
};
|
|
}
|
|
|
|
if (
|
|
httpStatus === 401 ||
|
|
/(?:^|\D)401(?:\D|$)/.test(normalized) ||
|
|
normalized.includes("unauthorized") ||
|
|
normalized.includes("invalid api key") ||
|
|
normalized.includes("invalid_api_key") ||
|
|
normalized.includes("authentication failed") ||
|
|
normalized.includes("authentication_error")
|
|
) {
|
|
return {
|
|
eventType: "unavailable",
|
|
severity: "critical",
|
|
counted: true,
|
|
reason: "provider_authentication_failed",
|
|
};
|
|
}
|
|
|
|
if (
|
|
httpStatus === 403 ||
|
|
/(?:^|\D)403(?:\D|$)/.test(normalized) ||
|
|
normalized === "forbidden" ||
|
|
normalized.includes("access forbidden") ||
|
|
normalized.includes("access denied") ||
|
|
normalized.includes("permission denied")
|
|
) {
|
|
return {
|
|
eventType: "unavailable",
|
|
severity: "critical",
|
|
counted: true,
|
|
reason: "provider_access_forbidden",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("生成结果未通过替换质检") ||
|
|
normalized.includes("output_moderation") ||
|
|
normalized.includes("quality check") ||
|
|
normalized.includes("review result") ||
|
|
normalized.includes("content policy") ||
|
|
normalized.includes("safety") ||
|
|
normalized.includes("generate image failed") ||
|
|
normalized.includes("could not generate images with the given prompts and images")
|
|
) {
|
|
return {
|
|
eventType: "error",
|
|
severity: "warning",
|
|
counted: false,
|
|
reason: "non_provider_failure",
|
|
};
|
|
}
|
|
|
|
if (isMediaInputFailure(normalized)) {
|
|
return {
|
|
eventType: "error",
|
|
severity: "warning",
|
|
counted: false,
|
|
reason: "request_media_rejected",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("fetch failed") ||
|
|
normalized.includes("econnreset") ||
|
|
normalized.includes("econnrefused") ||
|
|
normalized.includes("socket hang up") ||
|
|
normalized.includes("networkerror") ||
|
|
normalized.includes("network error")
|
|
) {
|
|
return {
|
|
eventType: "unavailable",
|
|
severity: "warning",
|
|
counted: true,
|
|
reason: "transport_failure_no_provider_response",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("model not exist") ||
|
|
normalized.includes("model_not_found") ||
|
|
normalized.includes("model not found") ||
|
|
normalized.includes("unknown model") ||
|
|
normalized.includes("model name") && normalized.includes("not supported")
|
|
) {
|
|
return {
|
|
eventType: "invalid_model",
|
|
severity: "critical",
|
|
counted: true,
|
|
reason: "invalid_model",
|
|
};
|
|
}
|
|
|
|
if (isProviderBalanceError(error)) {
|
|
return {
|
|
eventType: "quota",
|
|
severity: "critical",
|
|
counted: true,
|
|
reason: "provider_balance_or_billing_quota",
|
|
};
|
|
}
|
|
|
|
if (isRateLimitError(error)) {
|
|
return {
|
|
eventType: "unavailable",
|
|
severity: "warning",
|
|
counted: false,
|
|
reason: "provider_rate_limited",
|
|
};
|
|
}
|
|
|
|
if (isNonProviderRequestStatus(httpStatus, normalized)) {
|
|
return {
|
|
eventType: "error",
|
|
severity: "warning",
|
|
counted: false,
|
|
reason: "request_input_rejected",
|
|
};
|
|
}
|
|
|
|
if (
|
|
isOperationalAbort(error, normalized) ||
|
|
normalized.includes("timeout") ||
|
|
normalized.includes("超时") ||
|
|
normalized.includes("524") ||
|
|
normalized.includes("terminated") ||
|
|
normalized.includes("timed out") ||
|
|
normalized.includes("body timeout") ||
|
|
normalized.includes("headers timeout")
|
|
) {
|
|
return {
|
|
eventType: "timeout",
|
|
severity: "warning",
|
|
counted: true,
|
|
reason: "timeout_without_provider_response",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("503") ||
|
|
normalized.includes("502") ||
|
|
normalized.includes("504") ||
|
|
normalized.includes("model maintenance") ||
|
|
normalized.includes("maintenance") ||
|
|
normalized.includes("维护中") ||
|
|
normalized.includes("维护") ||
|
|
normalized.includes("修复中") ||
|
|
normalized.includes("系统繁忙") ||
|
|
normalized.includes("no available channel") ||
|
|
normalized.includes("no supported channel") ||
|
|
normalized.includes("service unavailable") ||
|
|
normalized.includes("temporarily unavailable")
|
|
) {
|
|
return {
|
|
eventType: "unavailable",
|
|
severity: "critical",
|
|
counted: true,
|
|
reason: "provider_unavailable",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("未返回任务 id") ||
|
|
normalized.includes("未返回任务id") ||
|
|
normalized.includes("no task id") ||
|
|
normalized.includes("missing task id") ||
|
|
normalized.includes("provider error") ||
|
|
normalized.includes("provider_error") ||
|
|
normalized.includes("provider request failed") ||
|
|
normalized.includes("provider task failed") ||
|
|
normalized.includes("vip98k task failed") ||
|
|
normalized.includes("upstream error") ||
|
|
normalized.includes("upstream_error") ||
|
|
normalized.includes("upstream request failed") ||
|
|
normalized.includes("上游服务错误") ||
|
|
normalized.includes("上游接口错误")
|
|
) {
|
|
return {
|
|
eventType: "unavailable",
|
|
severity: "critical",
|
|
counted: true,
|
|
reason: "provider_execution_failed",
|
|
};
|
|
}
|
|
|
|
if (
|
|
normalized.includes("download") ||
|
|
normalized.includes("no image url") ||
|
|
normalized.includes("no image") ||
|
|
normalized.includes("empty result") ||
|
|
normalized.includes("empty response") ||
|
|
normalized.includes("空结果") ||
|
|
normalized.includes("返回结果为空") ||
|
|
normalized.includes("响应为空") ||
|
|
normalized.includes("未返回可用图片") ||
|
|
normalized.includes("未包含可用图片") ||
|
|
normalized.includes("未解析到图片") ||
|
|
normalized.includes("未找到图片数据") ||
|
|
normalized.includes("图片生成失败")
|
|
) {
|
|
return {
|
|
eventType: "download_error",
|
|
severity: "error",
|
|
counted: true,
|
|
reason: "result_download_or_parse_failed",
|
|
};
|
|
}
|
|
|
|
return {
|
|
eventType: "error",
|
|
severity: "error",
|
|
counted: true,
|
|
reason: "provider_error",
|
|
};
|
|
}
|
|
|
|
export function shouldFailoverAiModelError(error: unknown) {
|
|
const classification = classifyAiModelHealthFailure(error);
|
|
const normalized = getErrorMessage(error).toLowerCase();
|
|
const imageProviderTaskFailed =
|
|
normalized.includes("image generation failed") ||
|
|
normalized.includes("snapgen 任务执行失败") ||
|
|
normalized.includes("snapgen 创建任务成功") ||
|
|
normalized.includes("snapgen 任务已完成但未返回图片地址") ||
|
|
normalized.includes("snapgen 未返回可用图片");
|
|
|
|
const providerExecutionFailed =
|
|
classification.reason === "provider_execution_failed" ||
|
|
normalized.includes("未返回任务 id") ||
|
|
normalized.includes("未返回任务id") ||
|
|
normalized.includes("no task id") ||
|
|
normalized.includes("missing task id");
|
|
|
|
if (
|
|
classification.reason === "provider_authentication_failed" ||
|
|
classification.reason === "provider_access_forbidden" ||
|
|
classification.reason === "provider_rate_limited"
|
|
) {
|
|
return true;
|
|
}
|
|
|
|
if (classification.reason === "user_cancelled" || classification.reason === "non_provider_failure") {
|
|
return false;
|
|
}
|
|
|
|
if (imageProviderTaskFailed || providerExecutionFailed) {
|
|
return true;
|
|
}
|
|
|
|
if (
|
|
classification.eventType === "timeout" ||
|
|
classification.reason === "transport_failure_no_provider_response"
|
|
) {
|
|
return true;
|
|
}
|
|
|
|
if (classification.eventType === "download_error") {
|
|
return (
|
|
normalized.includes("no image") ||
|
|
normalized.includes("empty result") ||
|
|
normalized.includes("empty response") ||
|
|
normalized.includes("空结果") ||
|
|
normalized.includes("返回结果为空") ||
|
|
normalized.includes("响应为空") ||
|
|
normalized.includes("未返回可用图片") ||
|
|
normalized.includes("未包含可用图片") ||
|
|
normalized.includes("未解析到图片") ||
|
|
normalized.includes("未找到图片数据") ||
|
|
normalized.includes("图片生成失败")
|
|
);
|
|
}
|
|
|
|
if (!classification.counted) return false;
|
|
|
|
if (classification.eventType === "invalid_model" || classification.eventType === "quota") {
|
|
return true;
|
|
}
|
|
if (classification.eventType === "unavailable") {
|
|
return (
|
|
/(?:^|\D)(?:429|500|502|503|504)(?:\D|$)/.test(normalized) ||
|
|
normalized.includes("no available channel") ||
|
|
normalized.includes("no supported channel") ||
|
|
normalized.includes("service unavailable") ||
|
|
normalized.includes("temporarily unavailable") ||
|
|
normalized.includes("model maintenance") ||
|
|
normalized.includes("maintenance") ||
|
|
normalized.includes("维护中") ||
|
|
normalized.includes("维护") ||
|
|
normalized.includes("修复中") ||
|
|
normalized.includes("系统繁忙") ||
|
|
normalized.includes("无可用渠道")
|
|
);
|
|
}
|
|
|
|
return /(?:^|\D)[45]\d\d(?:\D|$)/.test(normalized);
|
|
}
|
|
|
|
export function shouldFailoverVip98kRequestRejection(
|
|
error: unknown,
|
|
providerType: unknown,
|
|
) {
|
|
if (String(providerType || "").trim().toLowerCase() !== "vip98k") {
|
|
return false;
|
|
}
|
|
|
|
const normalized = getErrorMessage(error).trim().toLowerCase();
|
|
return (
|
|
/(?:^|\D)400(?:\D|$)/.test(normalized) &&
|
|
(normalized.includes("the request is invalid") ||
|
|
normalized.includes("invalid request"))
|
|
);
|
|
}
|
|
|
|
function buildMetadata(value?: Record<string, unknown>) {
|
|
if (!value || typeof value !== "object") return null;
|
|
try {
|
|
return JSON.parse(JSON.stringify(value));
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function isObservationPoolModel(
|
|
model:
|
|
| Pick<
|
|
AiModel,
|
|
| "healthStatus"
|
|
| "healthObservationStartedAt"
|
|
>
|
|
| null
|
|
| undefined
|
|
) {
|
|
return (
|
|
model?.healthStatus === "degraded" &&
|
|
Boolean(model.healthObservationStartedAt)
|
|
);
|
|
}
|
|
|
|
function getAutoDisableCooldownCutoff(now = new Date()) {
|
|
return new Date(now.getTime() - AUTO_DISABLE_COOLDOWN_MS);
|
|
}
|
|
|
|
export async function reviveAiModelsFromCooldown(params?: { modelId?: string }) {
|
|
const now = new Date();
|
|
const cooldownCutoff = getAutoDisableCooldownCutoff(now);
|
|
const conditions = [
|
|
eq(aiModel.isEnabled, false),
|
|
eq(aiModel.excludeFromFailover, false),
|
|
eq(aiModel.excludeFromHealthBan, false),
|
|
];
|
|
|
|
if (params?.modelId) {
|
|
conditions.push(eq(aiModel.id, normalizeText(params.modelId)));
|
|
}
|
|
|
|
const candidates = await db
|
|
.select({
|
|
id: aiModel.id,
|
|
modelId: aiModel.modelId,
|
|
displayName: aiModel.displayName,
|
|
healthStatus: aiModel.healthStatus,
|
|
healthConsecutiveFailures: aiModel.healthConsecutiveFailures,
|
|
healthLastEventType: aiModel.healthLastEventType,
|
|
healthAutoDisabledAt: aiModel.healthAutoDisabledAt,
|
|
healthLastFailureAt: aiModel.healthLastFailureAt,
|
|
healthUpdatedAt: aiModel.healthUpdatedAt,
|
|
})
|
|
.from(aiModel)
|
|
.where(and(...conditions));
|
|
|
|
const dueModels = candidates.filter((model) => {
|
|
const cooldownStartedAt =
|
|
model.healthAutoDisabledAt ||
|
|
model.healthLastFailureAt ||
|
|
model.healthUpdatedAt;
|
|
if (!cooldownStartedAt || cooldownStartedAt > cooldownCutoff) {
|
|
return false;
|
|
}
|
|
|
|
return model.healthStatus === "auto_disabled";
|
|
});
|
|
|
|
if (dueModels.length === 0) {
|
|
return [];
|
|
}
|
|
|
|
const revived = await db
|
|
.update(aiModel)
|
|
.set({
|
|
isEnabled: true,
|
|
healthStatus: "degraded",
|
|
healthConsecutiveFailures: 0,
|
|
healthRecoverySuccessStreak: 0,
|
|
healthObservationStartedAt: now,
|
|
healthUpdatedAt: now,
|
|
updatedAt: now,
|
|
})
|
|
.where(
|
|
inArray(
|
|
aiModel.id,
|
|
dueModels.map((item) => item.id)
|
|
)
|
|
)
|
|
.returning();
|
|
|
|
if (revived.length > 0) {
|
|
console.info("[AI Model Health] Revived cooled-down models into observation pool:", {
|
|
count: revived.length,
|
|
models: revived.map((item) => ({
|
|
id: item.id,
|
|
modelId: item.modelId,
|
|
displayName: item.displayName,
|
|
previousStatus: item.healthStatus,
|
|
})),
|
|
});
|
|
}
|
|
|
|
return revived;
|
|
}
|
|
|
|
export async function recordAiModelHealthSuccess(params: {
|
|
model?:
|
|
| Pick<
|
|
AiModel,
|
|
| "id"
|
|
| "healthStatus"
|
|
| "healthRecoverySuccessStreak"
|
|
| "healthObservationStartedAt"
|
|
>
|
|
| null;
|
|
}) {
|
|
const modelId = normalizeText(params.model?.id);
|
|
if (!modelId) return;
|
|
|
|
const [existing] = await db
|
|
.select({
|
|
id: aiModel.id,
|
|
healthStatus: aiModel.healthStatus,
|
|
healthRecoverySuccessStreak: aiModel.healthRecoverySuccessStreak,
|
|
healthObservationStartedAt: aiModel.healthObservationStartedAt,
|
|
})
|
|
.from(aiModel)
|
|
.where(eq(aiModel.id, modelId))
|
|
.limit(1);
|
|
|
|
if (!existing) return;
|
|
|
|
const now = new Date();
|
|
// A request that started before the circuit opened can still prove the
|
|
// provider is reachable. Treat that success as the circuit's half-open
|
|
// probe instead of leaving the model disabled until the full cooldown.
|
|
const recoveredFromAutoDisable = existing.healthStatus === "auto_disabled";
|
|
const isObservation =
|
|
isObservationPoolModel(existing) || recoveredFromAutoDisable;
|
|
const nextRecoverySuccessStreak = isObservation
|
|
? (existing.healthRecoverySuccessStreak || 0) + 1
|
|
: 0;
|
|
const recoveredToHealthy =
|
|
isObservation &&
|
|
nextRecoverySuccessStreak >= OBSERVATION_RECOVERY_SUCCESS_THRESHOLD;
|
|
|
|
const updateData: Partial<AiModel> = {
|
|
healthStatus:
|
|
recoveredToHealthy
|
|
? "healthy"
|
|
: isObservation
|
|
? "degraded"
|
|
: "healthy",
|
|
healthConsecutiveFailures: 0,
|
|
healthLastEventType: "success",
|
|
healthLastError: null,
|
|
healthLastSuccessAt: now,
|
|
healthRecoverySuccessStreak: recoveredToHealthy
|
|
? 0
|
|
: isObservation
|
|
? nextRecoverySuccessStreak
|
|
: 0,
|
|
healthObservationStartedAt: recoveredToHealthy
|
|
? null
|
|
: isObservation
|
|
? existing.healthObservationStartedAt || now
|
|
: null,
|
|
healthUpdatedAt: now,
|
|
updatedAt: now,
|
|
};
|
|
|
|
if (recoveredFromAutoDisable) {
|
|
updateData.isEnabled = true;
|
|
}
|
|
|
|
if (recoveredToHealthy) {
|
|
updateData.healthAutoDisabledAt = null;
|
|
updateData.healthAutoDisabledReason = null;
|
|
}
|
|
|
|
await db
|
|
.update(aiModel)
|
|
.set(updateData)
|
|
.where(eq(aiModel.id, modelId));
|
|
}
|
|
|
|
export async function recordAiModelHealthFailure(params: {
|
|
model?: (Partial<AiModel> & Pick<AiModel, "id">) | null;
|
|
provider?: {
|
|
id?: string | null;
|
|
name?: string | null;
|
|
type?: string | null;
|
|
} | null;
|
|
error: unknown;
|
|
context?: string;
|
|
elapsedMs?: number;
|
|
requestId?: string;
|
|
metadata?: Record<string, unknown>;
|
|
}) {
|
|
const modelId = normalizeText(params.model?.id);
|
|
if (!modelId) return;
|
|
|
|
const classification = classifyAiModelHealthFailure(params.error);
|
|
const errorMessage = getErrorMessage(params.error).slice(0, 1800);
|
|
const now = new Date();
|
|
|
|
const [existing] = await db
|
|
.select({
|
|
id: aiModel.id,
|
|
modelId: aiModel.modelId,
|
|
displayName: aiModel.displayName,
|
|
isEnabled: aiModel.isEnabled,
|
|
excludeFromFailover: aiModel.excludeFromFailover,
|
|
excludeFromHealthBan: aiModel.excludeFromHealthBan,
|
|
healthStatus: aiModel.healthStatus,
|
|
healthConsecutiveFailures: aiModel.healthConsecutiveFailures,
|
|
healthRecoverySuccessStreak: aiModel.healthRecoverySuccessStreak,
|
|
healthObservationStartedAt: aiModel.healthObservationStartedAt,
|
|
healthAutoDisabledAt: aiModel.healthAutoDisabledAt,
|
|
})
|
|
.from(aiModel)
|
|
.where(eq(aiModel.id, modelId))
|
|
.limit(1);
|
|
|
|
if (!existing) return;
|
|
|
|
const isObservation = isObservationPoolModel(existing);
|
|
const nextConsecutiveFailures = classification.counted
|
|
? Math.max(
|
|
1,
|
|
(isObservation ? 0 : existing.healthConsecutiveFailures || 0) + 1
|
|
)
|
|
: existing.healthConsecutiveFailures || 0;
|
|
const shouldAutoDisable =
|
|
classification.counted &&
|
|
!existing.excludeFromFailover &&
|
|
!existing.excludeFromHealthBan &&
|
|
((isObservation && existing.isEnabled) ||
|
|
(existing.healthStatus !== "auto_disabled" &&
|
|
nextConsecutiveFailures >= AUTO_DISABLE_FAILURE_THRESHOLD));
|
|
const nextStatus = shouldAutoDisable
|
|
? "auto_disabled"
|
|
: classification.counted
|
|
? "degraded"
|
|
: existing.healthStatus;
|
|
const autoDisabledReason = shouldAutoDisable
|
|
? `${classification.reason}: ${errorMessage}`.slice(0, 1800)
|
|
: undefined;
|
|
|
|
await db
|
|
.insert(aiModelHealthEvent)
|
|
.values({
|
|
modelId,
|
|
providerId: normalizeText(params.provider?.id) || null,
|
|
modelRuntimeId: normalizeText(params.model?.modelId) || null,
|
|
modelDisplayName: normalizeText(params.model?.displayName) || null,
|
|
providerType: normalizeText(params.provider?.type) || null,
|
|
providerName: normalizeText(params.provider?.name) || null,
|
|
eventType: classification.eventType,
|
|
severity: classification.severity,
|
|
context: normalizeText(params.context).slice(0, 96) || null,
|
|
counted: classification.counted,
|
|
consecutiveFailures: nextConsecutiveFailures,
|
|
autoDisabled: shouldAutoDisable,
|
|
elapsedMs:
|
|
typeof params.elapsedMs === "number" && Number.isFinite(params.elapsedMs)
|
|
? Math.max(0, Math.round(params.elapsedMs))
|
|
: null,
|
|
requestId: normalizeText(params.requestId).slice(0, 128) || null,
|
|
errorMessage,
|
|
metadata: buildMetadata(params.metadata),
|
|
});
|
|
|
|
const updateData: Partial<AiModel> = {
|
|
isEnabled: shouldAutoDisable ? false : existing.isEnabled,
|
|
healthStatus: nextStatus,
|
|
healthConsecutiveFailures: nextConsecutiveFailures,
|
|
healthLastEventType: classification.eventType,
|
|
healthLastError: errorMessage,
|
|
healthLastFailureAt: now,
|
|
healthRecoverySuccessStreak: shouldAutoDisable
|
|
? 0
|
|
: classification.counted
|
|
? 0
|
|
: existing.healthRecoverySuccessStreak || 0,
|
|
healthObservationStartedAt: shouldAutoDisable
|
|
? null
|
|
: isObservation
|
|
? existing.healthObservationStartedAt
|
|
: null,
|
|
healthUpdatedAt: now,
|
|
updatedAt: now,
|
|
};
|
|
if (shouldAutoDisable) {
|
|
updateData.healthAutoDisabledAt = now;
|
|
updateData.healthAutoDisabledReason = autoDisabledReason || errorMessage;
|
|
} else if (existing.healthStatus !== "auto_disabled" && !isObservation) {
|
|
updateData.healthAutoDisabledReason = null;
|
|
}
|
|
|
|
await db.update(aiModel).set(updateData).where(eq(aiModel.id, modelId));
|
|
|
|
if (shouldAutoDisable) {
|
|
console.error("[AI Model Health] Auto disabled unhealthy model:", {
|
|
modelId,
|
|
runtimeModelId: params.model?.modelId,
|
|
displayName: params.model?.displayName,
|
|
providerType: params.provider?.type,
|
|
consecutiveFailures: nextConsecutiveFailures,
|
|
eventType: classification.eventType,
|
|
error: errorMessage,
|
|
});
|
|
}
|
|
}
|
|
|
|
export async function restoreAiModelHealth(modelId: string) {
|
|
const normalizedModelId = normalizeText(modelId);
|
|
if (!normalizedModelId) return [];
|
|
|
|
return await db
|
|
.update(aiModel)
|
|
.set({
|
|
isEnabled: true,
|
|
healthStatus: "healthy",
|
|
healthConsecutiveFailures: 0,
|
|
healthLastEventType: "success",
|
|
healthLastError: null,
|
|
healthLastFailureAt: null,
|
|
healthAutoDisabledAt: null,
|
|
healthAutoDisabledReason: null,
|
|
healthRecoverySuccessStreak: 0,
|
|
healthObservationStartedAt: null,
|
|
healthUpdatedAt: new Date(),
|
|
updatedAt: new Date(),
|
|
})
|
|
.where(eq(aiModel.id, normalizedModelId))
|
|
.returning();
|
|
}
|
|
|
|
export async function listAiModelHealthEvents(params?: { modelId?: string; limit?: number }) {
|
|
const limit = Math.max(1, Math.min(params?.limit || RECENT_EVENT_LIMIT, 200));
|
|
const modelId = normalizeText(params?.modelId);
|
|
const conditions = modelId ? [eq(aiModelHealthEvent.modelId, modelId)] : [];
|
|
|
|
return await db
|
|
.select()
|
|
.from(aiModelHealthEvent)
|
|
.where(conditions.length ? and(...conditions) : undefined)
|
|
.orderBy(desc(aiModelHealthEvent.createdAt))
|
|
.limit(limit);
|
|
}
|
|
|
|
export async function getAiModelHealthSummary() {
|
|
const rows = await db
|
|
.select({
|
|
modelId: aiModel.id,
|
|
eventCount: sql<number>`count(${aiModelHealthEvent.id})::int`,
|
|
countedEventCount: sql<number>`sum(case when ${aiModelHealthEvent.counted} then 1 else 0 end)::int`,
|
|
lastEventAt: sql<Date | null>`max(${aiModelHealthEvent.createdAt})`,
|
|
})
|
|
.from(aiModel)
|
|
.leftJoin(aiModelHealthEvent, eq(aiModel.id, aiModelHealthEvent.modelId))
|
|
.groupBy(aiModel.id);
|
|
|
|
return rows;
|
|
}
|