ai-chatbot/tests/routes/provider-call-compatibility.test.ts

import { expect, test } from "@playwright/test";
import { APICallError, type LanguageModelV2CallOptions } from "@ai-sdk/provider";
import { generateText } from "ai";
import { createVip98kChatModel } from "@/lib/ai/providers/vip98k-chat";
import { selectVip98kChatInvocation, vip98kChatSupportsVision } from "@/lib/ai/vip98k-chat-contract";
import { buildVip98kModelPresets, fetchVip98kCatalog } from "@/lib/ai/vip98k-models";

const compatible = {
  kind: "chat", method: "POST", path: "/v1/chat/completions",
  headers: { "X-Relay-Route-Option": "route_chat", Authorization: "untrusted" },
};
const native = { kind: "chat", method: "POST", path: "/v1/messages", headers: { "X-Relay-Route-Option": "route_native" } };
const catalogModel = {
  id: "test-vision", type: "model", modality: "multimodal",
  default_route_option_id: "route_native", invocation: native,
  invocations: [native, { ...compatible, headers: {} }],
  service_offers: [{
    public_operation_id: "openai.chat_completions.create",
    route_options: [{ id: "route_native", invocation: { ...compatible, headers: {} } }],
  }],
};
const options: LanguageModelV2CallOptions = {
  prompt: [{ role: "user", content: [
    { type: "text", text: "查看图片并报告颜色" },
    { type: "file", mediaType: "image/png", data: new Uint8Array([1, 2, 3]) },
  ] }],
  tools: [{ type: "function", name: "report_colors", inputSchema: {
    type: "object", properties: { color: { type: "string" } }, required: ["color"],
  } }],
  toolChoice: { type: "required" },
};
function response(message: Record<string, unknown>) {
  return Response.json({
    id: "chat_test", model: "test-vision", created: 1,
    choices: [{ index: 0, finish_reason: message.tool_calls ? "tool_calls" : "stop", message: { role: "assistant", ...message } }],
    usage: { prompt_tokens: 10, completion_tokens: 4, total_tokens: 14 },
  });
}
function toolResponse() {
  return response({ content: null, tool_calls: [{ id: "call_1", type: "function", function: { name: "report_colors", arguments: '{"color":"red"}' } }] });
}
function model(baseFetch: typeof fetch, metadata: unknown = { invocation: compatible }) {
  return createVip98kChatModel({ apiKey: "test-only-key", baseURL: "https://relay.example/v1/", modelId: "test-vision", metadata, fetch: baseFetch });
}
function streamResponse(chunks: unknown[]) {
  return new Response(chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n", {
    headers: { "Content-Type": "text/event-stream" },
  });
}
async function readStream<T>(stream: ReadableStream<T>) {
  const reader = stream.getReader();
  const parts: T[] = [];
  try {
    while (true) {
      const next = await reader.read();
      if (next.done) return parts;
      parts.push(next.value);
    }
  } finally { reader.releaseLock(); }
}
const streamChunk = (delta: Record<string, unknown>, finishReason: string | null = null) => ({
  id: "chat_test", model: "test-vision", created: 1,
  choices: [{ index: 0, delta, finish_reason: finishReason }],
});

test("uses the documented compatible protocol with the matching default route", () => {
  expect(selectVip98kChatInvocation(catalogModel)).toEqual({
    protocol: "openai-chat", path: "/v1/chat/completions", headers: { "X-Relay-Route-Option": "route_native" },
  });
  expect(selectVip98kChatInvocation({ invocation: compatible })?.headers).toEqual({ "X-Relay-Route-Option": "route_chat" });
  expect(selectVip98kChatInvocation({ invocation: { ...compatible, path: "//attacker.example/v1/chat/completions" } })).toBeNull();
  expect(selectVip98kChatInvocation({ id: "model_a", invocation: { ...compatible, request_example: { model: "model_b" } } })).toBeNull();
});

test("retains protocol and route during catalog-to-runtime conversion", () => {
  const preset = buildVip98kModelPresets([catalogModel])[0];
  const metadata = (preset.capabilities as any).vip98k;
  expect(selectVip98kChatInvocation(metadata)).toEqual(selectVip98kChatInvocation(catalogModel));
  expect(preset.supportsVision).toBe(true);
  expect(preset.defaultParams.apiFormat).toBe("openai-chat");
});

test("classifies vision from the purchased capability, not a family name", () => {
  expect(vip98kChatSupportsVision({ id: "gpt-vision-looking-name", modality: "text" })).toBe(false);
  expect(vip98kChatSupportsVision({ id: "gemini-test", input_modes: ["image_url"] })).toBe(true);
  expect(vip98kChatSupportsVision({ capabilities: ["image-understanding"] })).toBe(true);
  const presets = buildVip98kModelPresets([
    { id: "minimax-h3", type: "video", input_modes: ["image_url"], output_modes: ["video"] },
    { id: "world-generation", type: "model", capabilities: ["world_generation"], output_modes: ["interactive_3d"] },
  ]);
  expect(presets.map((p) => p.type)).toEqual(["video", "world"]);
  expect(() => model(fetch, { invocation: { kind: "task", path: "/extensions/v1/tasks" } })).toThrow("未公布兼容的对话接口");
});

test("preserves images, tools, cancellation and route-scoped authentication in the actual SDK request", async () => {
  const abort = new AbortController();
  let calls = 0;
  const mock: typeof fetch = async (input, init) => {
    calls++;
    expect(String(input)).toBe("https://relay.example/v1/chat/completions");
    const headers = new Headers(init?.headers);
    expect(headers.get("Authorization")).toBe("Bearer test-only-key");
    expect(headers.get("X-Relay-Route-Option")).toBe("route_native");
    expect(init?.signal).toBe(abort.signal);
    expect(init?.redirect).toBe("error");
    const body = JSON.parse(String(init?.body));
    expect(body.messages[0].content[1]).toEqual({ type: "image_url", image_url: { url: "data:image/png;base64,AQID" } });
    expect(body.tools[0].function.name).toBe("report_colors");
    expect(body.tool_choice).toBe("required");
    return toolResponse();
  };
  const result = await model(mock, catalogModel).doGenerate({ ...options, abortSignal: abort.signal });
  expect(calls).toBe(1);
  expect(result.content).toContainEqual(expect.objectContaining({ type: "tool-call", toolName: "report_colors", input: '{"color":"red"}' }));
});

test("keeps route headers isolated across two models on the same provider", async () => {
  const routes: string[] = [];
  const mock: typeof fetch = async (_url, init) => {
    routes.push(new Headers(init?.headers).get("X-Relay-Route-Option") || "");
    return toolResponse();
  };
  await Promise.all([
    model(mock).doGenerate(options),
    model(mock, { invocation: { ...compatible, headers: { "X-Relay-Route-Option": "other_route" } } }).doGenerate(options),
  ]);
  expect(routes.sort()).toEqual(["other_route", "route_chat"]);
});

test("uses Responses only when that protocol is actually published", async () => {
  const mock: typeof fetch = async (url, init) => {
    expect(String(url)).toBe("https://relay.example/v1/responses");
    expect(JSON.parse(String(init?.body))).toHaveProperty("input");
    return Response.json({ id: "resp_test", created_at: 1, model: "test-vision", incomplete_details: null, output: [
      { type: "message", id: "msg_test", role: "assistant", content: [{ type: "output_text", text: "已完成", annotations: [] }] },
    ], usage: { input_tokens: 1, output_tokens: 1 } });
  };
  const result = await model(mock, { invocation: { ...compatible, path: "/v1/responses" } }).doGenerate({ prompt: options.prompt });
  expect(result.content).toContainEqual(expect.objectContaining({ type: "text", text: "已完成" }));
});

test("does not automatically repeat a paid request whose outcome is unknown", async () => {
  let calls = 0;
  const mock: typeof fetch = async () => {
    calls++;
    return new Response(JSON.stringify({ error: { code: "request_outcome_unknown", message: "The request was sent, but its final outcome could not be confirmed." } }), { status: 503 });
  };
  await expect(generateText({ model: model(mock), prompt: "hello", maxRetries: 2 })).rejects.toThrow("避免重复调用和扣费");
  expect(calls).toBe(1);
});

test("rejects silent empty success without retrying and redacts upstream errors", async () => {
  let calls = 0;
  const empty: typeof fetch = async () => { calls++; return response({ content: "" }); };
  await expect(generateText({ model: model(empty), prompt: "hello", maxRetries: 2 })).rejects.toThrow("未返回可用");
  expect(calls).toBe(1);
  const failing: typeof fetch = async () => new Response('<html>secret-key prompt private</html>', { status: 524 });
  try { await model(failing).doGenerate(options); throw new Error("expected failure"); }
  catch (error) {
    expect(APICallError.isInstance(error)).toBe(true);
    expect(String(error)).not.toMatch(/secret-key|prompt private|<html>/);
    expect((error as APICallError).isRetryable).toBe(false);
  }
});

test("streams complete tool calls without waiting for a text answer", async () => {
  const mock: typeof fetch = async () => streamResponse([
    streamChunk({ role: "assistant", tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "report_colors", arguments: '{"color":' } }] }),
    streamChunk({ tool_calls: [{ index: 0, function: { arguments: '"red"}' } }] }),
    streamChunk({}, "tool_calls"),
  ]);
  const { stream } = await model(mock).doStream(options);
  const parts = await readStream(stream);
  expect(parts).toContainEqual(expect.objectContaining({ type: "tool-call", toolName: "report_colors", input: '{"color":"red"}' }));
  expect(parts.at(-1)?.type).toBe("finish");
});

test("reports reasoning-only and disconnected streams as failures instead of success", async () => {
  const mock: typeof fetch = async () => streamResponse([streamChunk({}, "stop")]);
  const empty = await model(mock).doStream({ ...options, toolChoice: { type: "auto" } });
  await expect(readStream(empty.stream)).rejects.toThrow("未返回可用");
  const cut: typeof fetch = async () => streamResponse([streamChunk({ content: "部分回答" })]);
  const interrupted = await model(cut).doStream(options);
  await expect(readStream(interrupted.stream)).rejects.toThrow("提前中断");
});

test("does not claim success when a route silently ignores required tools", async () => {
  const mock: typeof fetch = async () => response({ content: "当前环境中未提供工具" });
  await expect(model(mock).doGenerate(options)).rejects.toThrow("未返回所要求的工具调用");
  const streamMock: typeof fetch = async () => streamResponse([
    streamChunk({ content: "当前环境中未提供工具" }), streamChunk({}, "stop"),
  ]);
  const { stream } = await model(streamMock).doStream(options);
  await expect(readStream(stream)).rejects.toThrow("未返回所要求的工具调用");
});

test("cancelling a streamed response cancels the underlying connection", async () => {
  let cancelled = false;
  const mock: typeof fetch = async () => new Response(new ReadableStream({
    start(controller) {
      controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(streamChunk({ content: "hello" }))}\n\n`));
    },
    cancel() { cancelled = true; },
  }), { headers: { "Content-Type": "text/event-stream" } });
  const { stream } = await model(mock).doStream(options);
  const reader = stream.getReader();
  await reader.read();
  await reader.cancel();
  await expect.poll(() => cancelled).toBe(true);
});

test("public executable catalog wins over generic authorized type placeholders", async () => {
  const original = globalThis.fetch;
  const seen: string[] = [];
  globalThis.fetch = async (url, init) => {
    seen.push(String(url));
    if (String(url).endsWith("/api/v2/catalog/models")) {
      expect(new Headers(init?.headers).has("Authorization")).toBe(false);
      return Response.json({ data: { discovery: { authorized_models_path: "//attacker.example/models" }, sections: [{ models: [
        { id: "minimax-h3", type: "video", input_modes: ["image_url"], output_modes: ["video"] }, catalogModel,
      ] }] } });
    }
    expect(new Headers(init?.headers).get("Authorization")).toBe("Bearer test-only-key");
    return Response.json({ data: [{ id: "minimax-h3", type: "model" }, { id: "test-vision" }] });
  };
  try {
    const catalog = await fetchVip98kCatalog({ baseURL: "https://relay.example/v1/", apiKey: "test-only-key" });
    expect(buildVip98kModelPresets(catalog).map((p) => p.type)).toEqual(["video", "model"]);
    expect(seen).toEqual(["https://relay.example/api/v2/catalog/models", "https://relay.example/v1/models"]);
    expect((buildVip98kModelPresets(catalog)[1].capabilities as any).vip98k.chatInvocation.headers).toEqual({ "X-Relay-Route-Option": "route_native" });
  } finally { globalThis.fetch = original; }
});

test("legacy models discover the public contract once for concurrent calls without disclosing the key", async () => {
  let catalogs = 0;
  let posts = 0;
  const mock: typeof fetch = async (url, init) => {
    const headers = new Headers(init?.headers);
    if (String(url).endsWith("/catalog/models")) {
      catalogs++;
      expect(headers.has("Authorization")).toBe(false);
      return Response.json({ data: { sections: [{ models: [catalogModel] }] } });
    }
    posts++;
    expect(headers.get("Authorization")).toBe("Bearer test-only-key");
    expect(headers.get("X-Relay-Route-Option")).toBe("route_native");
    return toolResponse();
  };
  const legacy = () => createVip98kChatModel({
    apiKey: "test-only-key", baseURL: "https://legacy-test.example", modelId: "test-vision",
    fetch: mock, discoverLegacyContract: true,
  });
  await Promise.all([legacy().doGenerate(options), legacy().doGenerate(options)]);
  await legacy().doGenerate(options);
  expect(catalogs).toBe(1);
  expect(posts).toBe(3);
});

test("legacy discovery prevents an incorrectly classified video model from being charged as chat", async () => {
  let posts = 0;
  const mock: typeof fetch = async (url) => {
    if (!String(url).endsWith("/catalog/models")) posts++;
    return Response.json({ data: { sections: [{ models: [{
      id: "minimax-h3", type: "video", invocation: { kind: "task", path: "/extensions/v1/tasks" },
    }] }] } });
  };
  const legacy = createVip98kChatModel({
    apiKey: "test-only-key", baseURL: "https://legacy-video.example", modelId: "minimax-h3",
    fetch: mock, discoverLegacyContract: true,
  });
  await expect(legacy.doGenerate(options)).rejects.toThrow("未公布兼容的对话接口");
  expect(posts).toBe(0);
});

test("a legacy native-only snapshot can discover its newly advertised compatible route", async () => {
  let posts = 0;
  const mock: typeof fetch = async (url, init) => {
    if (String(url).endsWith("/catalog/models")) {
      return Response.json({ data: { sections: [{ models: [catalogModel] }] } });
    }
    expect(String(url)).toBe("https://legacy-native.example/v1/chat/completions");
    expect(new Headers(init?.headers).get("X-Relay-Route-Option")).toBe("route_native");
    posts++;
    return toolResponse();
  };
  const legacy = createVip98kChatModel({
    apiKey: "test-only-key", baseURL: "https://legacy-native.example", modelId: "test-vision",
    metadata: { invocation: native }, fetch: mock, discoverLegacyContract: true,
  });
  await legacy.doGenerate(options);
  expect(posts).toBe(1);
});

test("a cancelled request can leave shared discovery without submitting a paid POST", async () => {
  const abort = new AbortController();
  let finishCatalog: (response: Response) => void = () => {};
  let posts = 0;
  const mock: typeof fetch = async (url) => {
    if (!String(url).endsWith("/catalog/models")) { posts++; return toolResponse(); }
    return new Promise<Response>((resolve) => { finishCatalog = resolve; });
  };
  const legacy = createVip98kChatModel({
    apiKey: "test-only-key", baseURL: "https://legacy-cancel.example", modelId: "test-vision",
    fetch: mock, discoverLegacyContract: true,
  });
  const result = Promise.resolve(legacy.doGenerate({ ...options, abortSignal: abort.signal }));
  abort.abort(new Error("cancelled by user"));
  await expect(result).rejects.toThrow("cancelled by user");
  finishCatalog(Response.json({ data: { sections: [{ models: [catalogModel] }] } }));
  expect(posts).toBe(0);
});