fix: make model cooldown recovery query safe
This commit is contained in:
parent
8fd5eae96a
commit
132af95df9
@ -1,6 +1,6 @@
|
|||||||
import "server-only";
|
import "server-only";
|
||||||
|
|
||||||
import { and, desc, eq, inArray, sql } from "drizzle-orm";
|
import { and, desc, eq, inArray } from "drizzle-orm";
|
||||||
import { db } from "@/lib/db";
|
import { db } from "@/lib/db";
|
||||||
import {
|
import {
|
||||||
aiModel,
|
aiModel,
|
||||||
@ -23,6 +23,14 @@ const AUTO_DISABLE_FAILURE_THRESHOLD = 3;
|
|||||||
const AUTO_DISABLE_COOLDOWN_MS = 2 * 60 * 60 * 1000;
|
const AUTO_DISABLE_COOLDOWN_MS = 2 * 60 * 60 * 1000;
|
||||||
const OBSERVATION_RECOVERY_SUCCESS_THRESHOLD = 3;
|
const OBSERVATION_RECOVERY_SUCCESS_THRESHOLD = 3;
|
||||||
const RECENT_EVENT_LIMIT = 80;
|
const RECENT_EVENT_LIMIT = 80;
|
||||||
|
const RECOVERABLE_FAILURE_EVENT_TYPES = new Set<AiModelHealthEventType>([
|
||||||
|
"timeout",
|
||||||
|
"quota",
|
||||||
|
"unavailable",
|
||||||
|
"invalid_model",
|
||||||
|
"download_error",
|
||||||
|
"error",
|
||||||
|
]);
|
||||||
|
|
||||||
function normalizeText(value: unknown) {
|
function normalizeText(value: unknown) {
|
||||||
return String(value || "").trim();
|
return String(value || "").trim();
|
||||||
@ -190,41 +198,48 @@ export async function reviveAiModelsFromCooldown(params?: { modelId?: string })
|
|||||||
const conditions = [
|
const conditions = [
|
||||||
eq(aiModel.isEnabled, false),
|
eq(aiModel.isEnabled, false),
|
||||||
eq(aiModel.excludeFromFailover, false),
|
eq(aiModel.excludeFromFailover, false),
|
||||||
sql`COALESCE(${aiModel.healthAutoDisabledAt}, ${aiModel.healthLastFailureAt}, ${aiModel.healthUpdatedAt}) <= ${cooldownCutoff}`,
|
|
||||||
sql`(
|
|
||||||
${aiModel.healthStatus} = 'auto_disabled'
|
|
||||||
OR (
|
|
||||||
${aiModel.healthStatus} = 'degraded'
|
|
||||||
AND ${aiModel.healthLastFailureAt} IS NOT NULL
|
|
||||||
AND ${aiModel.healthConsecutiveFailures} > 0
|
|
||||||
AND ${aiModel.healthLastEventType} IN (
|
|
||||||
'timeout',
|
|
||||||
'quota',
|
|
||||||
'unavailable',
|
|
||||||
'invalid_model',
|
|
||||||
'download_error',
|
|
||||||
'error'
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)`,
|
|
||||||
];
|
];
|
||||||
|
|
||||||
if (params?.modelId) {
|
if (params?.modelId) {
|
||||||
conditions.push(eq(aiModel.id, normalizeText(params.modelId)));
|
conditions.push(eq(aiModel.id, normalizeText(params.modelId)));
|
||||||
}
|
}
|
||||||
|
|
||||||
const dueModels = await db
|
const candidates = await db
|
||||||
.select({
|
.select({
|
||||||
id: aiModel.id,
|
id: aiModel.id,
|
||||||
modelId: aiModel.modelId,
|
modelId: aiModel.modelId,
|
||||||
displayName: aiModel.displayName,
|
displayName: aiModel.displayName,
|
||||||
healthStatus: aiModel.healthStatus,
|
healthStatus: aiModel.healthStatus,
|
||||||
|
healthConsecutiveFailures: aiModel.healthConsecutiveFailures,
|
||||||
|
healthLastEventType: aiModel.healthLastEventType,
|
||||||
healthAutoDisabledAt: aiModel.healthAutoDisabledAt,
|
healthAutoDisabledAt: aiModel.healthAutoDisabledAt,
|
||||||
healthLastFailureAt: aiModel.healthLastFailureAt,
|
healthLastFailureAt: aiModel.healthLastFailureAt,
|
||||||
|
healthUpdatedAt: aiModel.healthUpdatedAt,
|
||||||
})
|
})
|
||||||
.from(aiModel)
|
.from(aiModel)
|
||||||
.where(and(...conditions));
|
.where(and(...conditions));
|
||||||
|
|
||||||
|
const dueModels = candidates.filter((model) => {
|
||||||
|
const cooldownStartedAt =
|
||||||
|
model.healthAutoDisabledAt ||
|
||||||
|
model.healthLastFailureAt ||
|
||||||
|
model.healthUpdatedAt;
|
||||||
|
if (!cooldownStartedAt || cooldownStartedAt > cooldownCutoff) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (model.healthStatus === "auto_disabled") {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
model.healthStatus === "degraded" &&
|
||||||
|
Boolean(model.healthLastFailureAt) &&
|
||||||
|
(model.healthConsecutiveFailures || 0) > 0 &&
|
||||||
|
RECOVERABLE_FAILURE_EVENT_TYPES.has(model.healthLastEventType as AiModelHealthEventType)
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
if (dueModels.length === 0) {
|
if (dueModels.length === 0) {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user