import assert from "node:assert/strict"; import { test } from "node:test"; import { AGENT_MODEL_COOLDOWN_MS, AgentModelProtocolError, agentModelCandidates, agentModelFault, createAgentModelFailover, type AgentCircuitStore, type AgentModelEntry } from "@/lib/ai/runtime/agent-model-failover"; import { completionResponse, responsesCompletion, toResponsesRequest } from "@/lib/ai/zcode-agent/model-gateway-protocol"; function entry(id: string, model = {}, provider = {}): AgentModelEntry { return { model: { id, modelId: id, supportsVision: true, supportsImageGen: false, isEnabled: true, healthStatus: "healthy", ...model }, provider: { id: "provider", name: "provider", type: "vip98k", isEnabled: true, ...provider } } as AgentModelEntry; } function store() { let now = 10_000; const faults = new Map(); const circuit: AgentCircuitStore = { isPaused: async e => (faults.get(e.model.id)?.pausedUntil || 0) > now, fail: async (e, fault) => { faults.set(e.model.id, { pausedUntil: now + AGENT_MODEL_COOLDOWN_MS, reason: fault.reason }); }, }; return { circuit, faults, advance: (ms: number) => { now += ms; } }; } test("all enabled conversational vision models join the pool; generators and explicitly excluded tools do not", () => { const primary = entry("primary", { fallbackModelId1: "preferred" }); const entries = [entry("other"), entry("image", { supportsImageGen: true }), entry("video", { capabilities: { kind: "video" } }), entry("tool", { capabilities: { kind: "image-tool" } }), entry("3d", { capabilities: { kind: "3d" } }), entry("disabled", { isEnabled: false }), entry("no-vision", { supportsVision: false }), entry("no-tools", { capabilities: { supportsTools: false } }), entry("excluded", { excludeFromFailover: true }), entry("preferred"), primary]; assert.deepEqual(agentModelCandidates(primary, entries).map(e => e.model.id), ["primary", "preferred", "other"]); }); test("401, 403, 429, 5xx, network and invalid replies fail over and pause for exactly thirty minutes", async () => { for (const error of [Object.assign(new Error("rejected"), { status: 401 }), Object.assign(new Error("rejected"), { status: 403 }), Object.assign(new Error("quota"), { status: 429 }), Object.assign(new Error("unavailable"), { status: 503 }), new Error("fetch failed"), new AgentModelProtocolError()]) { const s = store(); const calls: string[] = []; const execute = async (e: AgentModelEntry) => { calls.push(e.model.id); if (e.model.id === "primary") throw error; return "ok"; }; const entries = [entry("primary"), entry("backup")]; assert.equal(await createAgentModelFailover(entries, s.circuit)(execute), "ok"); assert.deepEqual(calls, ["primary", "backup"]); calls.length = 0; // A separate task/process sharing persisted state skips the failed primary. await createAgentModelFailover(entries, s.circuit)(execute); assert.deepEqual(calls, ["backup"]); s.advance(AGENT_MODEL_COOLDOWN_MS - 1); assert.equal(await s.circuit.isPaused(entries[0]), true); s.advance(1); assert.equal(await s.circuit.isPaused(entries[0]), false); calls.length = 0; await createAgentModelFailover(entries, s.circuit)(async e => { calls.push(e.model.id); return "recovered"; }); assert.deepEqual(calls, ["primary"]); } }); test("invalid input, content policy and cancellation neither quarantine nor retry", async () => { for (const error of [Object.assign(new Error("invalid input"), { status: 400 }), Object.assign(new Error("content policy violation"), { status: 403 }), new Error("unsupported media type"), new Error("user cancelled")]) { const s = store(); let calls = 0; await assert.rejects(createAgentModelFailover([entry("one"), entry("two")], s.circuit)(async () => { calls++; throw error; })); assert.equal(calls, 1); assert.equal(s.faults.size, 0); } const s = store(); const controller = new AbortController(); controller.abort(); await assert.rejects(createAgentModelFailover([entry("one")], s.circuit)(async () => "unused", controller.signal)); assert.equal(s.faults.size, 0); }); test("multiple failed backups are skipped; no model is tried twice and attempts are bounded", async () => { const s = store(); const calls: string[] = []; await assert.rejects(createAgentModelFailover(["a", "b", "c", "d", "e"].map(id => entry(id)), s.circuit)(async e => { calls.push(e.model.id); throw Object.assign(new Error("busy"), { status: 503 }); })); assert.deepEqual(calls, ["a", "b", "c", "d"]); assert.equal(s.faults.size, 4); }); test("late success cannot clear a fault recorded by another in-flight request", async () => { const s = store(); const primary = entry("a"); let resolve!: (value: string) => void; const pending = createAgentModelFailover([primary], s.circuit)(() => new Promise(r => { resolve = r; })); await new Promise(r => setTimeout(r, 0)); await s.circuit.fail(primary, { reason: "authentication", status: 401 }); resolve("late success"); await pending; assert.equal(await s.circuit.isPaused(primary), true); }); test("provider models rejecting tool calls are paused without blaming user input", () => { assert.equal(agentModelFault(Object.assign(new Error("model does not support tools"), { status: 400 }))?.reason, "tools_unsupported"); assert.equal(agentModelFault(Object.assign(new Error("context too long"), { status: 400 })), null); }); test("Responses conversion preserves references, tool history, parameters and function IDs", () => { const request = { messages: [{ role: "system", content: "instructions" }, { role: "user", content: [{ type: "text", text: "edit" }, { type: "image_url", image_url: { url: "https://example.test/reference.png" } }] }, { role: "assistant", tool_calls: [{ id: "call1", function: { name: "readCanvas", arguments: '{"nodeIds":["n"]}' } }] }, { role: "tool", tool_call_id: "call1", content: '{"ok":true}' }], tools: [{ type: "function", function: { name: "readCanvas", parameters: { type: "object" } } }], max_tokens: 100, temperature: 0.5 }; const converted = toResponsesRequest(request); assert.equal(converted.input[1].content[1].image_url, "https://example.test/reference.png"); assert.deepEqual(converted.input[2], { type: "function_call", call_id: "call1", name: "readCanvas", arguments: '{"nodeIds":["n"]}' }); assert.equal(converted.input[3].call_id, "call1"); assert.equal(converted.max_output_tokens, 100); assert.equal(converted.tools[0].name, "readCanvas"); assert.equal(converted.store, false); }); test("complete valid replies become native SSE; truncated/invalid tool replies are never exposed", async () => { const completion = responsesCompletion({ status: "completed", output: [{ type: "function_call", call_id: "call1", name: "readCanvas", arguments: "{}" }] }); const request = { stream: true, tools: [{ function: { name: "readCanvas" } }] }; const sse = await completionResponse(completion, request).text(); assert.match(sse, /chat.completion.chunk/); assert.match(sse, /tool_calls/); assert.match(sse, /\[DONE\]/); assert.throws(() => responsesCompletion({ status: "incomplete", output: [] }), AgentModelProtocolError); completion.choices[0].message.tool_calls[0].function.arguments = '{"broken"'; assert.throws(() => completionResponse(completion, request), AgentModelProtocolError); }); test("successful HTTP policy refusals are not retried as provider faults", () => { let rejection: unknown; try { completionResponse({ choices: [{ finish_reason: "content_filter", message: { content: "" } }] }, {}); } catch (error) { rejection = error; } assert(rejection); assert.equal(agentModelFault(rejection), null); assert.throws(() => responsesCompletion({ status: "completed", output: [{ type: "message", content: [{ type: "refusal" }] }] }), /content policy/); assert.equal(agentModelFault(Object.assign(new Error("No output generated"), { name: "AI_NoOutputGeneratedError" }))?.reason, "invalid_response"); });