37 lines
2.3 KiB
TypeScript
37 lines
2.3 KiB
TypeScript
import "server-only";
|
|
import { eq, sql } from "drizzle-orm";
|
|
import { db } from "@/lib/db";
|
|
import { aiModelHealthEvent, systemSettings } from "@/lib/db/schema";
|
|
import { AGENT_MODEL_COOLDOWN_MS, type AgentCircuitStore } from "./agent-model-failover";
|
|
|
|
// PostgreSQL is shared by every process and release. Do not reuse the general
|
|
// image-model health ban (which has a different, two-hour recovery policy).
|
|
export function createAgentCircuitStore(prefix: string, context: string): AgentCircuitStore { return {
|
|
async isPaused(entry) {
|
|
const [row] = await db.select({ value: systemSettings.value, now: sql<string>`clock_timestamp()` })
|
|
.from(systemSettings).where(eq(systemSettings.key, prefix + entry.model.id)).limit(1);
|
|
const value = row?.value as { pausedUntil?: string } | undefined;
|
|
return Boolean(value?.pausedUntil && Date.parse(value.pausedUntil) > Date.parse(row.now));
|
|
},
|
|
async fail(entry, fault) {
|
|
await db.transaction(async tx => {
|
|
const [{ now }] = await tx.execute<{ now: string }>(sql`select clock_timestamp() as now`);
|
|
const failedAt = new Date(now);
|
|
const pausedUntil = new Date(failedAt.getTime() + AGENT_MODEL_COOLDOWN_MS).toISOString();
|
|
const value = { modelId: entry.model.id, reason: fault.reason, status: fault.status, failedAt: failedAt.toISOString(), pausedUntil };
|
|
await tx.insert(systemSettings).values({ key: prefix + entry.model.id, value, updatedAt: failedAt })
|
|
.onConflictDoUpdate({ target: systemSettings.key, set: { value, updatedAt: failedAt },
|
|
setWhere: sql`${systemSettings.updatedAt} <= ${failedAt.toISOString()}::timestamp` });
|
|
await tx.insert(aiModelHealthEvent).values({
|
|
modelId: entry.model.id, providerId: entry.provider.id, modelRuntimeId: entry.model.modelId,
|
|
modelDisplayName: entry.model.displayName, providerType: entry.provider.type, providerName: entry.provider.name,
|
|
eventType: "agent_circuit_open", severity: "warning", context,
|
|
counted: false, errorMessage: fault.reason, metadata: { ...value, cooldownMs: AGENT_MODEL_COOLDOWN_MS },
|
|
});
|
|
});
|
|
},
|
|
}; }
|
|
|
|
export const agentModelCircuitStore = createAgentCircuitStore("agent_multimodal_circuit:", "canvas-agent-multimodal");
|
|
export const agentImageCircuitStore = createAgentCircuitStore("agent_image_circuit:", "canvas-agent-image");
|