import { expect, test } from "@playwright/test"; import { APICallError, type LanguageModelV2CallOptions } from "@ai-sdk/provider"; import { generateText } from "ai"; import { createVip98kChatModel } from "@/lib/ai/providers/vip98k-chat"; import { selectVip98kChatInvocation, vip98kChatSupportsVision } from "@/lib/ai/vip98k-chat-contract"; import { buildVip98kModelPresets, fetchVip98kCatalog } from "@/lib/ai/vip98k-models"; const compatible = { kind: "chat", method: "POST", path: "/v1/chat/completions", headers: { "X-Relay-Route-Option": "route_chat", Authorization: "untrusted" }, }; const native = { kind: "chat", method: "POST", path: "/v1/messages", headers: { "X-Relay-Route-Option": "route_native" } }; const catalogModel = { id: "test-vision", type: "model", modality: "multimodal", default_route_option_id: "route_native", invocation: native, invocations: [native, { ...compatible, headers: {} }], service_offers: [{ public_operation_id: "openai.chat_completions.create", route_options: [{ id: "route_native", invocation: { ...compatible, headers: {} } }], }], }; const options: LanguageModelV2CallOptions = { prompt: [{ role: "user", content: [ { type: "text", text: "查看图片并报告颜色" }, { type: "file", mediaType: "image/png", data: new Uint8Array([1, 2, 3]) }, ] }], tools: [{ type: "function", name: "report_colors", inputSchema: { type: "object", properties: { color: { type: "string" } }, required: ["color"], } }], toolChoice: { type: "required" }, }; function response(message: Record) { return Response.json({ id: "chat_test", model: "test-vision", created: 1, choices: [{ index: 0, finish_reason: message.tool_calls ? "tool_calls" : "stop", message: { role: "assistant", ...message } }], usage: { prompt_tokens: 10, completion_tokens: 4, total_tokens: 14 }, }); } function toolResponse() { return response({ content: null, tool_calls: [{ id: "call_1", type: "function", function: { name: "report_colors", arguments: '{"color":"red"}' } }] }); } function model(baseFetch: typeof fetch, metadata: unknown = { invocation: compatible }) { return createVip98kChatModel({ apiKey: "test-only-key", baseURL: "https://relay.example/v1/", modelId: "test-vision", metadata, fetch: baseFetch }); } function streamResponse(chunks: unknown[]) { return new Response(chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n", { headers: { "Content-Type": "text/event-stream" }, }); } async function readStream(stream: ReadableStream) { const reader = stream.getReader(); const parts: T[] = []; try { while (true) { const next = await reader.read(); if (next.done) return parts; parts.push(next.value); } } finally { reader.releaseLock(); } } const streamChunk = (delta: Record, finishReason: string | null = null) => ({ id: "chat_test", model: "test-vision", created: 1, choices: [{ index: 0, delta, finish_reason: finishReason }], }); test("uses the documented compatible protocol with the matching default route", () => { expect(selectVip98kChatInvocation(catalogModel)).toEqual({ protocol: "openai-chat", path: "/v1/chat/completions", headers: { "X-Relay-Route-Option": "route_native" }, }); expect(selectVip98kChatInvocation({ invocation: compatible })?.headers).toEqual({ "X-Relay-Route-Option": "route_chat" }); expect(selectVip98kChatInvocation({ invocation: { ...compatible, path: "//attacker.example/v1/chat/completions" } })).toBeNull(); expect(selectVip98kChatInvocation({ id: "model_a", invocation: { ...compatible, request_example: { model: "model_b" } } })).toBeNull(); }); test("retains protocol and route during catalog-to-runtime conversion", () => { const preset = buildVip98kModelPresets([catalogModel])[0]; const metadata = (preset.capabilities as any).vip98k; expect(selectVip98kChatInvocation(metadata)).toEqual(selectVip98kChatInvocation(catalogModel)); expect(preset.supportsVision).toBe(true); expect(preset.defaultParams.apiFormat).toBe("openai-chat"); }); test("classifies vision from the purchased capability, not a family name", () => { expect(vip98kChatSupportsVision({ id: "gpt-vision-looking-name", modality: "text" })).toBe(false); expect(vip98kChatSupportsVision({ id: "gemini-test", input_modes: ["image_url"] })).toBe(true); expect(vip98kChatSupportsVision({ capabilities: ["image-understanding"] })).toBe(true); const presets = buildVip98kModelPresets([ { id: "minimax-h3", type: "video", input_modes: ["image_url"], output_modes: ["video"] }, { id: "world-generation", type: "model", capabilities: ["world_generation"], output_modes: ["interactive_3d"] }, ]); expect(presets.map((p) => p.type)).toEqual(["video", "world"]); expect(() => model(fetch, { invocation: { kind: "task", path: "/extensions/v1/tasks" } })).toThrow("未公布兼容的对话接口"); }); test("preserves images, tools, cancellation and route-scoped authentication in the actual SDK request", async () => { const abort = new AbortController(); let calls = 0; const mock: typeof fetch = async (input, init) => { calls++; expect(String(input)).toBe("https://relay.example/v1/chat/completions"); const headers = new Headers(init?.headers); expect(headers.get("Authorization")).toBe("Bearer test-only-key"); expect(headers.get("X-Relay-Route-Option")).toBe("route_native"); expect(init?.signal).toBe(abort.signal); expect(init?.redirect).toBe("error"); const body = JSON.parse(String(init?.body)); expect(body.messages[0].content[1]).toEqual({ type: "image_url", image_url: { url: "data:image/png;base64,AQID" } }); expect(body.tools[0].function.name).toBe("report_colors"); expect(body.tool_choice).toBe("required"); return toolResponse(); }; const result = await model(mock, catalogModel).doGenerate({ ...options, abortSignal: abort.signal }); expect(calls).toBe(1); expect(result.content).toContainEqual(expect.objectContaining({ type: "tool-call", toolName: "report_colors", input: '{"color":"red"}' })); }); test("keeps route headers isolated across two models on the same provider", async () => { const routes: string[] = []; const mock: typeof fetch = async (_url, init) => { routes.push(new Headers(init?.headers).get("X-Relay-Route-Option") || ""); return toolResponse(); }; await Promise.all([ model(mock).doGenerate(options), model(mock, { invocation: { ...compatible, headers: { "X-Relay-Route-Option": "other_route" } } }).doGenerate(options), ]); expect(routes.sort()).toEqual(["other_route", "route_chat"]); }); test("uses Responses only when that protocol is actually published", async () => { const mock: typeof fetch = async (url, init) => { expect(String(url)).toBe("https://relay.example/v1/responses"); expect(JSON.parse(String(init?.body))).toHaveProperty("input"); return Response.json({ id: "resp_test", created_at: 1, model: "test-vision", incomplete_details: null, output: [ { type: "message", id: "msg_test", role: "assistant", content: [{ type: "output_text", text: "已完成", annotations: [] }] }, ], usage: { input_tokens: 1, output_tokens: 1 } }); }; const result = await model(mock, { invocation: { ...compatible, path: "/v1/responses" } }).doGenerate({ prompt: options.prompt }); expect(result.content).toContainEqual(expect.objectContaining({ type: "text", text: "已完成" })); }); test("does not automatically repeat a paid request whose outcome is unknown", async () => { let calls = 0; const mock: typeof fetch = async () => { calls++; return new Response(JSON.stringify({ error: { code: "request_outcome_unknown", message: "The request was sent, but its final outcome could not be confirmed." } }), { status: 503 }); }; await expect(generateText({ model: model(mock), prompt: "hello", maxRetries: 2 })).rejects.toThrow("避免重复调用和扣费"); expect(calls).toBe(1); }); test("rejects silent empty success without retrying and redacts upstream errors", async () => { let calls = 0; const empty: typeof fetch = async () => { calls++; return response({ content: "" }); }; await expect(generateText({ model: model(empty), prompt: "hello", maxRetries: 2 })).rejects.toThrow("未返回可用"); expect(calls).toBe(1); const failing: typeof fetch = async () => new Response('secret-key prompt private', { status: 524 }); try { await model(failing).doGenerate(options); throw new Error("expected failure"); } catch (error) { expect(APICallError.isInstance(error)).toBe(true); expect(String(error)).not.toMatch(/secret-key|prompt private|/); expect((error as APICallError).isRetryable).toBe(false); } }); test("streams complete tool calls without waiting for a text answer", async () => { const mock: typeof fetch = async () => streamResponse([ streamChunk({ role: "assistant", tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "report_colors", arguments: '{"color":' } }] }), streamChunk({ tool_calls: [{ index: 0, function: { arguments: '"red"}' } }] }), streamChunk({}, "tool_calls"), ]); const { stream } = await model(mock).doStream(options); const parts = await readStream(stream); expect(parts).toContainEqual(expect.objectContaining({ type: "tool-call", toolName: "report_colors", input: '{"color":"red"}' })); expect(parts.at(-1)?.type).toBe("finish"); }); test("reports reasoning-only and disconnected streams as failures instead of success", async () => { const mock: typeof fetch = async () => streamResponse([streamChunk({}, "stop")]); const empty = await model(mock).doStream({ ...options, toolChoice: { type: "auto" } }); await expect(readStream(empty.stream)).rejects.toThrow("未返回可用"); const cut: typeof fetch = async () => streamResponse([streamChunk({ content: "部分回答" })]); const interrupted = await model(cut).doStream(options); await expect(readStream(interrupted.stream)).rejects.toThrow("提前中断"); }); test("does not claim success when a route silently ignores required tools", async () => { const mock: typeof fetch = async () => response({ content: "当前环境中未提供工具" }); await expect(model(mock).doGenerate(options)).rejects.toThrow("未返回所要求的工具调用"); const streamMock: typeof fetch = async () => streamResponse([ streamChunk({ content: "当前环境中未提供工具" }), streamChunk({}, "stop"), ]); const { stream } = await model(streamMock).doStream(options); await expect(readStream(stream)).rejects.toThrow("未返回所要求的工具调用"); }); test("cancelling a streamed response cancels the underlying connection", async () => { let cancelled = false; const mock: typeof fetch = async () => new Response(new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(streamChunk({ content: "hello" }))}\n\n`)); }, cancel() { cancelled = true; }, }), { headers: { "Content-Type": "text/event-stream" } }); const { stream } = await model(mock).doStream(options); const reader = stream.getReader(); await reader.read(); await reader.cancel(); await expect.poll(() => cancelled).toBe(true); }); test("public executable catalog wins over generic authorized type placeholders", async () => { const original = globalThis.fetch; const seen: string[] = []; globalThis.fetch = async (url, init) => { seen.push(String(url)); if (String(url).endsWith("/api/v2/catalog/models")) { expect(new Headers(init?.headers).has("Authorization")).toBe(false); return Response.json({ data: { discovery: { authorized_models_path: "//attacker.example/models" }, sections: [{ models: [ { id: "minimax-h3", type: "video", input_modes: ["image_url"], output_modes: ["video"] }, catalogModel, ] }] } }); } expect(new Headers(init?.headers).get("Authorization")).toBe("Bearer test-only-key"); return Response.json({ data: [{ id: "minimax-h3", type: "model" }, { id: "test-vision" }] }); }; try { const catalog = await fetchVip98kCatalog({ baseURL: "https://relay.example/v1/", apiKey: "test-only-key" }); expect(buildVip98kModelPresets(catalog).map((p) => p.type)).toEqual(["video", "model"]); expect(seen).toEqual(["https://relay.example/api/v2/catalog/models", "https://relay.example/v1/models"]); expect((buildVip98kModelPresets(catalog)[1].capabilities as any).vip98k.chatInvocation.headers).toEqual({ "X-Relay-Route-Option": "route_native" }); } finally { globalThis.fetch = original; } }); test("legacy models discover the public contract once for concurrent calls without disclosing the key", async () => { let catalogs = 0; let posts = 0; const mock: typeof fetch = async (url, init) => { const headers = new Headers(init?.headers); if (String(url).endsWith("/catalog/models")) { catalogs++; expect(headers.has("Authorization")).toBe(false); return Response.json({ data: { sections: [{ models: [catalogModel] }] } }); } posts++; expect(headers.get("Authorization")).toBe("Bearer test-only-key"); expect(headers.get("X-Relay-Route-Option")).toBe("route_native"); return toolResponse(); }; const legacy = () => createVip98kChatModel({ apiKey: "test-only-key", baseURL: "https://legacy-test.example", modelId: "test-vision", fetch: mock, discoverLegacyContract: true, }); await Promise.all([legacy().doGenerate(options), legacy().doGenerate(options)]); await legacy().doGenerate(options); expect(catalogs).toBe(1); expect(posts).toBe(3); }); test("legacy discovery prevents an incorrectly classified video model from being charged as chat", async () => { let posts = 0; const mock: typeof fetch = async (url) => { if (!String(url).endsWith("/catalog/models")) posts++; return Response.json({ data: { sections: [{ models: [{ id: "minimax-h3", type: "video", invocation: { kind: "task", path: "/extensions/v1/tasks" }, }] }] } }); }; const legacy = createVip98kChatModel({ apiKey: "test-only-key", baseURL: "https://legacy-video.example", modelId: "minimax-h3", fetch: mock, discoverLegacyContract: true, }); await expect(legacy.doGenerate(options)).rejects.toThrow("未公布兼容的对话接口"); expect(posts).toBe(0); }); test("a legacy native-only snapshot can discover its newly advertised compatible route", async () => { let posts = 0; const mock: typeof fetch = async (url, init) => { if (String(url).endsWith("/catalog/models")) { return Response.json({ data: { sections: [{ models: [catalogModel] }] } }); } expect(String(url)).toBe("https://legacy-native.example/v1/chat/completions"); expect(new Headers(init?.headers).get("X-Relay-Route-Option")).toBe("route_native"); posts++; return toolResponse(); }; const legacy = createVip98kChatModel({ apiKey: "test-only-key", baseURL: "https://legacy-native.example", modelId: "test-vision", metadata: { invocation: native }, fetch: mock, discoverLegacyContract: true, }); await legacy.doGenerate(options); expect(posts).toBe(1); }); test("a cancelled request can leave shared discovery without submitting a paid POST", async () => { const abort = new AbortController(); let finishCatalog: (response: Response) => void = () => {}; let posts = 0; const mock: typeof fetch = async (url) => { if (!String(url).endsWith("/catalog/models")) { posts++; return toolResponse(); } return new Promise((resolve) => { finishCatalog = resolve; }); }; const legacy = createVip98kChatModel({ apiKey: "test-only-key", baseURL: "https://legacy-cancel.example", modelId: "test-vision", fetch: mock, discoverLegacyContract: true, }); const result = Promise.resolve(legacy.doGenerate({ ...options, abortSignal: abort.signal })); abort.abort(new Error("cancelled by user")); await expect(result).rejects.toThrow("cancelled by user"); finishCatalog(Response.json({ data: { sections: [{ models: [catalogModel] }] } })); expect(posts).toBe(0); });