119 lines
8.8 KiB
TypeScript
119 lines
8.8 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { test } from "node:test";
|
|
import { AGENT_MODEL_COOLDOWN_MS, AgentModelProtocolError, agentModelCandidates, agentModelFault, createAgentModelFailover, type AgentCircuitStore, type AgentModelEntry } from "@/lib/ai/runtime/agent-model-failover";
|
|
import { completionResponse, responsesCompletion, toResponsesRequest } from "@/lib/ai/zcode-agent/model-gateway-protocol";
|
|
|
|
function entry(id: string, model = {}, provider = {}): AgentModelEntry {
|
|
return { model: { id, modelId: id, supportsVision: true, supportsImageGen: false, isEnabled: true, healthStatus: "healthy", ...model }, provider: { id: "provider", name: "provider", type: "vip98k", isEnabled: true, ...provider } } as AgentModelEntry;
|
|
}
|
|
function store() {
|
|
let now = 10_000;
|
|
const faults = new Map<string, { pausedUntil: number; reason: string }>();
|
|
const circuit: AgentCircuitStore = {
|
|
isPaused: async e => (faults.get(e.model.id)?.pausedUntil || 0) > now,
|
|
fail: async (e, fault) => { faults.set(e.model.id, { pausedUntil: now + AGENT_MODEL_COOLDOWN_MS, reason: fault.reason }); },
|
|
};
|
|
return { circuit, faults, advance: (ms: number) => { now += ms; } };
|
|
}
|
|
|
|
test("all enabled conversational vision models join the pool; generators and explicitly excluded tools do not", () => {
|
|
const primary = entry("primary", { fallbackModelId1: "preferred" });
|
|
const entries = [entry("other"), entry("image", { supportsImageGen: true }), entry("video", { capabilities: { kind: "video" } }), entry("tool", { capabilities: { kind: "image-tool" } }), entry("3d", { capabilities: { kind: "3d" } }), entry("disabled", { isEnabled: false }), entry("no-vision", { supportsVision: false }), entry("no-tools", { capabilities: { supportsTools: false } }), entry("excluded", { excludeFromFailover: true }), entry("preferred"), primary];
|
|
assert.deepEqual(agentModelCandidates(primary, entries).map(e => e.model.id), ["primary", "preferred", "other"]);
|
|
});
|
|
|
|
test("401, 403, 429, 5xx, network and invalid replies fail over and pause for exactly thirty minutes", async () => {
|
|
for (const error of [Object.assign(new Error("rejected"), { status: 401 }), Object.assign(new Error("rejected"), { status: 403 }), Object.assign(new Error("quota"), { status: 429 }), Object.assign(new Error("unavailable"), { status: 503 }), new Error("fetch failed"), new AgentModelProtocolError()]) {
|
|
const s = store(); const calls: string[] = [];
|
|
const execute = async (e: AgentModelEntry) => { calls.push(e.model.id); if (e.model.id === "primary") throw error; return "ok"; };
|
|
const entries = [entry("primary"), entry("backup")];
|
|
assert.equal(await createAgentModelFailover(entries, s.circuit)(execute), "ok");
|
|
assert.deepEqual(calls, ["primary", "backup"]);
|
|
calls.length = 0;
|
|
// A separate task/process sharing persisted state skips the failed primary.
|
|
await createAgentModelFailover(entries, s.circuit)(execute);
|
|
assert.deepEqual(calls, ["backup"]);
|
|
s.advance(AGENT_MODEL_COOLDOWN_MS - 1);
|
|
assert.equal(await s.circuit.isPaused(entries[0]), true);
|
|
s.advance(1);
|
|
assert.equal(await s.circuit.isPaused(entries[0]), false);
|
|
calls.length = 0;
|
|
await createAgentModelFailover(entries, s.circuit)(async e => { calls.push(e.model.id); return "recovered"; });
|
|
assert.deepEqual(calls, ["primary"]);
|
|
}
|
|
});
|
|
|
|
test("invalid input, content policy and cancellation neither quarantine nor retry", async () => {
|
|
for (const error of [Object.assign(new Error("invalid input"), { status: 400 }), Object.assign(new Error("content policy violation"), { status: 403 }), new Error("unsupported media type"), new Error("user cancelled")]) {
|
|
const s = store(); let calls = 0;
|
|
await assert.rejects(createAgentModelFailover([entry("one"), entry("two")], s.circuit)(async () => { calls++; throw error; }));
|
|
assert.equal(calls, 1); assert.equal(s.faults.size, 0);
|
|
}
|
|
const s = store(); const controller = new AbortController(); controller.abort();
|
|
await assert.rejects(createAgentModelFailover([entry("one")], s.circuit)(async () => "unused", controller.signal));
|
|
assert.equal(s.faults.size, 0);
|
|
});
|
|
|
|
test("every eligible backup is considered once; a fixed small retry count cannot strand a healthy later model", async () => {
|
|
const s = store(); const calls: string[] = [];
|
|
await assert.rejects(createAgentModelFailover(["a", "b", "c", "d", "e"].map(id => entry(id)), s.circuit)(async e => { calls.push(e.model.id); throw Object.assign(new Error("busy"), { status: 503 }); }));
|
|
assert.deepEqual(calls, ["a", "b", "c", "d", "e"]);
|
|
assert.equal(s.faults.size, 5);
|
|
calls.length = 0;
|
|
const result = await createAgentModelFailover(["a", "b", "c", "d", "e", "healthy"].map(id => entry(id)), s.circuit)(async e => { calls.push(e.model.id); return "ok"; });
|
|
assert.equal(result, "ok"); assert.deepEqual(calls, ["healthy"]);
|
|
});
|
|
|
|
test("late success cannot clear a fault recorded by another in-flight request", async () => {
|
|
const s = store(); const primary = entry("a");
|
|
let resolve!: (value: string) => void;
|
|
const pending = createAgentModelFailover([primary], s.circuit)(() => new Promise<string>(r => { resolve = r; }));
|
|
await new Promise(r => setTimeout(r, 0));
|
|
await s.circuit.fail(primary, { reason: "authentication", status: 401 });
|
|
resolve("late success"); await pending;
|
|
assert.equal(await s.circuit.isPaused(primary), true);
|
|
});
|
|
|
|
test("provider models rejecting tool calls are paused without blaming user input", () => {
|
|
assert.equal(agentModelFault(Object.assign(new Error("model does not support tools"), { status: 400 }))?.reason, "tools_unsupported");
|
|
assert.equal(agentModelFault(Object.assign(new Error("context too long"), { status: 400 })), null);
|
|
});
|
|
|
|
test("Responses conversion preserves references, tool history, parameters and function IDs", () => {
|
|
const request = { messages: [{ role: "system", content: "instructions" }, { role: "user", content: [{ type: "text", text: "edit" }, { type: "image_url", image_url: { url: "https://example.test/reference.png" } }] }, { role: "assistant", tool_calls: [{ id: "call1", function: { name: "readCanvas", arguments: '{"nodeIds":["n"]}' } }] }, { role: "tool", tool_call_id: "call1", content: '{"ok":true}' }], tools: [{ type: "function", function: { name: "readCanvas", parameters: { type: "object" } } }], max_tokens: 100, temperature: 0.5 };
|
|
const converted = toResponsesRequest(request);
|
|
assert.equal(converted.input[1].content[1].image_url, "https://example.test/reference.png");
|
|
assert.deepEqual(converted.input[2], { type: "function_call", call_id: "call1", name: "readCanvas", arguments: '{"nodeIds":["n"]}' });
|
|
assert.equal(converted.input[3].call_id, "call1"); assert.equal(converted.max_output_tokens, 100);
|
|
assert.equal(converted.tools[0].name, "readCanvas"); assert.equal(converted.store, false);
|
|
});
|
|
|
|
test("complete valid replies become native SSE; truncated/invalid tool replies are never exposed", async () => {
|
|
const completion = responsesCompletion({ status: "completed", output: [{ type: "function_call", call_id: "call1", name: "readCanvas", arguments: "{}" }] });
|
|
const request = { stream: true, tools: [{ function: { name: "readCanvas" } }] };
|
|
const sse = await completionResponse(completion, request).text();
|
|
assert.match(sse, /chat.completion.chunk/); assert.match(sse, /tool_calls/); assert.match(sse, /\[DONE\]/);
|
|
assert.throws(() => responsesCompletion({ status: "incomplete", output: [] }), AgentModelProtocolError);
|
|
completion.choices[0].message.tool_calls[0].function.arguments = '{"broken"';
|
|
assert.throws(() => completionResponse(completion, request), AgentModelProtocolError);
|
|
});
|
|
|
|
|
|
test("successful HTTP policy refusals are not retried as provider faults", () => {
|
|
let rejection: unknown;
|
|
try { completionResponse({ choices: [{ finish_reason: "content_filter", message: { content: "" } }] }, {}); } catch (error) { rejection = error; }
|
|
assert(rejection); assert.equal(agentModelFault(rejection), null);
|
|
assert.throws(() => responsesCompletion({ status: "completed", output: [{ type: "message", content: [{ type: "refusal" }] }] }), /content policy/);
|
|
assert.equal(agentModelFault(Object.assign(new Error("No output generated"), { name: "AI_NoOutputGeneratedError" }))?.reason, "invalid_response");
|
|
});
|
|
|
|
|
|
test("malformed protocol envelopes and duplicate tool IDs are classified before native execution", () => {
|
|
for (const value of [null, [], { status: "completed", output: {} }]) {
|
|
assert.throws(() => responsesCompletion(value as any), AgentModelProtocolError);
|
|
}
|
|
assert.throws(() => completionResponse({ choices: [{ finish_reason: "stop", message: { content: {} } }] }, {}), AgentModelProtocolError);
|
|
const call = { id: "duplicate", type: "function", function: { name: "readCanvas", arguments: "{}" } };
|
|
assert.throws(() => completionResponse({ choices: [{ finish_reason: "tool_calls", message: { tool_calls: [call, call] } }] }, { tools: [{ function: { name: "readCanvas" } }] }), AgentModelProtocolError);
|
|
});
|