import { expect, test } from "@playwright/test";
import {
getUserFacingImageGenerationErrorMessage,
getUserFacingRequestErrorMessage,
getGatewayErrorPageMessage,
isGatewayResponseTimeout,
isProviderBalanceError,
isRateLimitError,
} from "@/lib/request-error";
import { isImageModelBalanceBlocked, resolveConfiguredImageBalanceFallback } from "@/lib/ai/runtime/image-balance";
import { serverLoader } from "../helpers/canvas-download-server-loader";
test("a bare HTTP 429 is rate limiting, not insufficient balance", () => {
const error = Object.assign(new Error("Too many requests (HTTP 429)"), { status: 429 });
expect(isProviderBalanceError(error)).toBe(false);
expect(isRateLimitError(error)).toBe(true);
expect(getUserFacingImageGenerationErrorMessage(error)).toContain("请求过于频繁");
expect(getUserFacingRequestErrorMessage(error)).toContain("请求过于频繁");
});
const gatewayHtml = '
vip98k.com | 524: A timeout occurredError code 524
(HTTP 524)';
test("gateway timeout pages become a concise unconfirmed-result message", () => {
for (const error of [new Error(gatewayHtml), { status: 524, message: "Gateway failed" }, { error: { message: "Request failed (HTTP 524)" } }]) {
expect(isGatewayResponseTimeout(error)).toBe(true);
for (const text of [getUserFacingImageGenerationErrorMessage(error), getUserFacingRequestErrorMessage(error)]) {
expect(text).toContain("HTTP 524");
expect(text).toContain("结果尚未确认");
expect(text).not.toMatch(/ {
for (const text of ['My pageHosted on Cloudflare
', `\`\`\`html\n${gatewayHtml}\n\`\`\``, "Cloudflare 文档示例", "普通图片错误"]) {
expect(getGatewayErrorPageMessage(text)).toBeNull();
}
});
test("HTTP 524 does not start another model while the original result is unconfirmed", () => {
const { health } = healthHarness();
expect(health.shouldFailoverAiModelError(new Error(gatewayHtml))).toBe(false);
expect(health.shouldFailoverAiModelError({ status: 524, message: "Gateway timeout" })).toBe(false);
expect(health.shouldFailoverAiModelError(new Error("Service unavailable (HTTP 503)"))).toBe(true);
});
test("the provider submits once when an HTTP 524 request cannot be located", async () => {
let submissions = 0;
const provider = serverLoader({
"@/lib/db/queries": { getSystemSetting: async () => null },
"@/lib/storage/image-archive": {},
"@/lib/storage/service": {},
fetch: async (_url: string, init: RequestInit) => {
if (init?.method !== "POST") return Response.json({}, { status: 404 });
submissions++;
return new Response(gatewayHtml, { status: 524, headers: { "content-type": "text/html" } });
},
})("lib/ai/providers/vip98k.ts");
const model = provider.createVip98k({ apiKey: "test", baseURL: "https://provider.test" }).chat("gpt-image-2.5");
await expect(model.doGenerate({ messages: [{ role: "user", content: "A desk" }] })).rejects.toThrow(/结果未确认.*HTTP 524/);
expect(submissions).toBe(1);
});
function imageRecoveryHarness(
handler: (path: string, method: string, call: number) => Response | Promise,
maxPollSeconds = 3,
) {
let now = 1_000;
const calls: { path: string; method: string; headers: Headers }[] = [];
const archived: string[] = [];
const load = serverLoader({
"@/lib/db": { db: {} },
"@/lib/db/queries": { getSystemSetting: async () => null },
"@/lib/storage/image-archive": { createDeferredUploadedImage: async (url: string) => {
archived.push(url);
return { thumbnail: url, original: url };
} },
"@/lib/storage/service": {},
Date: class extends Date { static now() { return now; } },
setTimeout: (fn: () => void, ms: number) => { now += ms; return setTimeout(fn, 0); },
fetch: async (url: string, init: RequestInit) => {
const path = new URL(url).pathname;
const method = init?.method || "GET";
calls.push({ path, method, headers: new Headers(init?.headers) });
return handler(path, method, calls.length);
},
});
const provider = load("lib/ai/providers/vip98k.ts");
return {
calls, load, archived,
generate: () => provider.createVip98k({ apiKey: "test-only-key", baseURL: "https://provider.test" })
.chat("gpt-image-2.5", { capabilities: { vip98k: { pollIntervalSeconds: 1, maxPollSeconds } } })
.doGenerate({ messages: [{ role: "user", content: "A desk" }] }),
run: () => provider.executeVip98kExternalTask({
apiKey: "test-only-key", baseURL: "https://provider.test", modelId: "gpt-image-2.5",
kind: "image", prompt: "A desk", imageUrls: [],
modelConfig: { capabilities: { vip98k: { pollIntervalSeconds: 1, maxPollSeconds } } },
}),
};
}
const recoveredImage = { created: 1, data: [{ url: "https://image.example/original.png" }] };
const originalCompleted = { request_id: "original-request", status: "succeeded", result_available: true };
test("an edge timeout waits for and delivers the original image without another POST", async () => {
let lookups = 0;
const h = imageRecoveryHarness((path, method) => {
if (method === "POST") return new Response(gatewayHtml, { status: 524 });
if (path.endsWith("by-idempotency-key")) {
return Response.json(++lookups === 1 ? { status: "processing" } : originalCompleted);
}
return Response.json(recoveredImage);
});
expect((await h.generate()).text).toContain(recoveredImage.data[0].url);
expect(h.archived).toEqual([recoveredImage.data[0].url]);
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
expect(lookups).toBe(2);
const originalKey = h.calls[0].headers.get("Idempotency-Key");
expect(originalKey).toBeTruthy();
for (const call of h.calls.filter(c => c.path.endsWith("by-idempotency-key"))) {
expect(call.headers.get("Idempotency-Key")).toBe(originalKey);
expect(call.headers.get("Authorization")).toBe("Bearer test-only-key");
}
expect(h.calls.at(-1)?.path).toBe("/api/external/requests/original-request/result");
});
test("a lost POST response recovers by its original key without resubmitting", async () => {
const h = imageRecoveryHarness((path, method) => {
if (method === "POST") throw new TypeError("fetch failed");
return Response.json(path.endsWith("by-idempotency-key") ? originalCompleted : recoveredImage);
});
expect((await h.run()).outputUrls).toHaveLength(1);
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
});
test("temporary lookup and result-read failures retry only GET requests", async () => {
let lookups = 0;
let reads = 0;
const h = imageRecoveryHarness((path, method) => {
if (method === "POST") return new Response("Gateway timeout", { status: 504 });
if (path.endsWith("by-idempotency-key")) {
return ++lookups === 1 ? new Response("Unavailable", { status: 503 }) : Response.json(originalCompleted);
}
return ++reads === 1 ? Response.json({}, { status: 409 }) : Response.json(recoveredImage);
}, 5);
expect((await h.run()).outputUrls).toHaveLength(1);
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
expect(reads).toBe(2);
});
test("missing, unconfirmed and unretained originals never create a replacement", async () => {
for (const state of [null, { status: "failed" }, { status: "unresolved" }, { ...originalCompleted, result_available: false }]) {
const h = imageRecoveryHarness((_path, method) => method === "POST"
? new Response(gatewayHtml, { status: 524 })
: Response.json(state || {}, { status: state ? 200 : 404 }));
await expect(h.run()).rejects.toThrow(/结果未确认.*HTTP 524/);
expect(h.calls.map(c => c.method)).toEqual(["POST", "GET"]);
}
});
test("a still-processing original stops at the polling deadline without another POST", async () => {
const h = imageRecoveryHarness((_path, method) => method === "POST"
? new Response(gatewayHtml, { status: 524 }) : Response.json({ status: "processing" }), 2);
await expect(h.run()).rejects.toThrow(/结果未确认.*HTTP 524/);
expect(h.calls.map(c => c.method)).toEqual(["POST", "GET", "GET"]);
});
test("HTTP 502 and 504 without a recoverable original do not repeat the POST", async () => {
for (const status of [502, 504]) {
const h = imageRecoveryHarness((_path, method) => method === "POST"
? new Response("Gateway timeout", { status }) : Response.json({}, { status: 404 }));
await expect(h.run()).rejects.toThrow(/结果未确认/);
expect(h.calls.map(c => c.method)).toEqual(["POST", "GET"]);
}
});
test("a completed original with no image stays unconfirmed and cannot fail over", async () => {
const h = imageRecoveryHarness((path, method) => {
if (method === "POST") return new Response(gatewayHtml, { status: 524 });
return Response.json(path.endsWith("by-idempotency-key") ? originalCompleted : { data: [] });
});
const error = await h.run().then(() => null, (e: Error) => e);
expect(error).toBeTruthy();
expect(h.load("lib/ai/runtime/model-health.ts").shouldFailoverAiModelError(error)).toBe(false);
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
});
test("ordinary image success uses the original response directly", async () => {
const h = imageRecoveryHarness(() => Response.json(recoveredImage));
expect((await h.run()).outputUrls).toHaveLength(1);
expect(h.calls.map(c => c.method)).toEqual(["POST"]);
});
test("an explicit provider balance error takes precedence over HTTP 429", () => {
const error = Object.assign(new Error("余额不足,请点击充值页面续费(HTTP 429)"), { status: 429 });
expect(isProviderBalanceError(error)).toBe(true);
expect(isRateLimitError(error)).toBe(false);
expect(getUserFacingImageGenerationErrorMessage(error)).toContain("服务商线路余额");
expect(getUserFacingImageGenerationErrorMessage(error)).toContain("与您的积分余额无关");
expect(getUserFacingImageGenerationErrorMessage(error)).not.toMatch(/^积分不足/);
});
test("nested billing details and generic quota limits are kept distinct", () => {
const nested = { status: 429, message: "request failed", data: { error: { message: "insufficient_balance" } } };
expect(isProviderBalanceError(nested)).toBe(true);
expect(isRateLimitError(nested)).toBe(false);
const usageLimit = new Error("Rate quota exceeded (HTTP 429)");
expect(isProviderBalanceError(usageLimit)).toBe(false);
expect(isRateLimitError(usageLimit)).toBe(true);
expect(isProviderBalanceError(new Error("该模型额度不足(HTTP 429)"))).toBe(true);
});
test("confirmed balance failures remain isolated until explicitly restored", () => {
const model = {
healthLastEventType: "quota",
healthLastError: "余额不足(HTTP 429)",
};
expect(isImageModelBalanceBlocked(model)).toBe(true);
expect(isImageModelBalanceBlocked({ healthLastEventType: "error", healthAutoDisabledReason: "provider_balance_or_billing_quota: 余额不足" })).toBe(true);
expect(isImageModelBalanceBlocked({ ...model, healthLastEventType: "success", healthLastError: null })).toBe(false);
});
test("old selections use only configured, enabled replacements that preserve the requested inputs", () => {
const requested = { healthLastEventType: "quota", healthLastError: "余额不足", fallbackModelId1: "verified" };
const verified = { model: { id: "verified", modelId: "same-family", isEnabled: true, supportsImageGen: true,
imageGenConfig: { supportedImageSizes: ["2K"], supportedAspectRatios: ["1:1"], maxReferenceImages: 8 } }, provider: { isEnabled: true } };
const input = { imageSize: "2K", aspectRatio: "1:1", referenceImageCount: 2 };
expect(resolveConfiguredImageBalanceFallback(requested, [verified], input)).toBe(verified);
expect(resolveConfiguredImageBalanceFallback(requested, [verified], { ...input, referenceImageCount: 9 })).toBeUndefined();
expect(resolveConfiguredImageBalanceFallback(requested, [verified], { ...input, imageSize: "4K" })).toBeUndefined();
expect(resolveConfiguredImageBalanceFallback(requested, [verified], { ...input, aspectRatio: "9:16" })).toBeUndefined();
expect(resolveConfiguredImageBalanceFallback(requested, [{ ...verified, provider: { isEnabled: false } }], input)).toBeUndefined();
expect(resolveConfiguredImageBalanceFallback({ ...requested, fallbackModelId1: null }, [verified], input)).toBeUndefined();
});
function healthHarness() {
let model: any = { id: "test", modelId: "test", isEnabled: true, excludeFromFailover: true,
excludeFromHealthBan: true, healthStatus: "healthy", healthConsecutiveFailures: 0 };
const events: any[] = [];
const query = (rows: () => any[]) => Object.assign(Promise.resolve().then(rows), {
from() { return this; }, where() { return this; },
limit: async () => rows(), returning: async () => rows(),
});
const db = {
select: () => query(() => [model]),
insert: () => ({ values: async (value: any) => { events.push(value); } }),
update: () => ({ set: (value: any) => { model = { ...model, ...value }; return query(() => [model]); } }),
};
const health = serverLoader({ "@/lib/db": { db } })("lib/ai/runtime/model-health.ts");
return { health, events, get model() { return model; }, age() { model.healthAutoDisabledAt = new Date(Date.now() - 4 * 60 * 60_000); } };
}
test("quota pauses an exempt model, survives cooldown and late completion, and can be restored", async () => {
const h = healthHarness();
await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("余额不足(HTTP 429)") });
expect(h.model.isEnabled).toBe(false);
expect(h.model.healthStatus).toBe("auto_disabled");
expect(h.events[0].eventType).toBe("quota");
await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("HTTP 503 unavailable") });
await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("Too many requests (HTTP 429)") });
expect(h.model.healthStatus).toBe("auto_disabled");
expect(isImageModelBalanceBlocked(h.model)).toBe(true);
h.age();
expect(await h.health.reviveAiModelsFromCooldown()).toEqual([]);
await h.health.recordAiModelHealthSuccess({ model: h.model });
expect(h.model.isEnabled).toBe(false);
await h.health.restoreAiModelHealth(h.model.id);
expect(h.model.isEnabled).toBe(true);
expect(isImageModelBalanceBlocked(h.model)).toBe(false);
});
test("ordinary rate limiting does not trigger billing isolation", async () => {
const h = healthHarness();
await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("Too many requests (HTTP 429)") });
expect(h.model.isEnabled).toBe(true);
expect(isImageModelBalanceBlocked(h.model)).toBe(false);
});