import { expect, test } from "@playwright/test"; import { getUserFacingImageGenerationErrorMessage, getUserFacingRequestErrorMessage, getGatewayErrorPageMessage, isGatewayResponseTimeout, isProviderBalanceError, isRateLimitError, } from "@/lib/request-error"; import { isImageModelBalanceBlocked, resolveConfiguredImageBalanceFallback } from "@/lib/ai/runtime/image-balance"; import { serverLoader } from "../helpers/canvas-download-server-loader"; test("a bare HTTP 429 is rate limiting, not insufficient balance", () => { const error = Object.assign(new Error("Too many requests (HTTP 429)"), { status: 429 }); expect(isProviderBalanceError(error)).toBe(false); expect(isRateLimitError(error)).toBe(true); expect(getUserFacingImageGenerationErrorMessage(error)).toContain("请求过于频繁"); expect(getUserFacingRequestErrorMessage(error)).toContain("请求过于频繁"); }); const gatewayHtml = 'vip98k.com | 524: A timeout occurred
Error code 524
(HTTP 524)'; test("gateway timeout pages become a concise unconfirmed-result message", () => { for (const error of [new Error(gatewayHtml), { status: 524, message: "Gateway failed" }, { error: { message: "Request failed (HTTP 524)" } }]) { expect(isGatewayResponseTimeout(error)).toBe(true); for (const text of [getUserFacingImageGenerationErrorMessage(error), getUserFacingRequestErrorMessage(error)]) { expect(text).toContain("HTTP 524"); expect(text).toContain("结果尚未确认"); expect(text).not.toMatch(/ { for (const text of ['My page

Hosted on Cloudflare

', `\`\`\`html\n${gatewayHtml}\n\`\`\``, "Cloudflare 文档示例", "普通图片错误"]) { expect(getGatewayErrorPageMessage(text)).toBeNull(); } }); test("HTTP 524 does not start another model while the original result is unconfirmed", () => { const { health } = healthHarness(); expect(health.shouldFailoverAiModelError(new Error(gatewayHtml))).toBe(false); expect(health.shouldFailoverAiModelError({ status: 524, message: "Gateway timeout" })).toBe(false); expect(health.shouldFailoverAiModelError(new Error("Service unavailable (HTTP 503)"))).toBe(true); }); test("the provider submits once when an HTTP 524 request cannot be located", async () => { let submissions = 0; const provider = serverLoader({ "@/lib/db/queries": { getSystemSetting: async () => null }, "@/lib/storage/image-archive": {}, "@/lib/storage/service": {}, fetch: async (_url: string, init: RequestInit) => { if (init?.method !== "POST") return Response.json({}, { status: 404 }); submissions++; return new Response(gatewayHtml, { status: 524, headers: { "content-type": "text/html" } }); }, })("lib/ai/providers/vip98k.ts"); const model = provider.createVip98k({ apiKey: "test", baseURL: "https://provider.test" }).chat("gpt-image-2.5"); await expect(model.doGenerate({ messages: [{ role: "user", content: "A desk" }] })).rejects.toThrow(/结果未确认.*HTTP 524/); expect(submissions).toBe(1); }); function imageRecoveryHarness( handler: (path: string, method: string, call: number) => Response | Promise, maxPollSeconds = 3, ) { let now = 1_000; const calls: { path: string; method: string; headers: Headers }[] = []; const archived: string[] = []; const load = serverLoader({ "@/lib/db": { db: {} }, "@/lib/db/queries": { getSystemSetting: async () => null }, "@/lib/storage/image-archive": { createDeferredUploadedImage: async (url: string) => { archived.push(url); return { thumbnail: url, original: url }; } }, "@/lib/storage/service": {}, Date: class extends Date { static now() { return now; } }, setTimeout: (fn: () => void, ms: number) => { now += ms; return setTimeout(fn, 0); }, fetch: async (url: string, init: RequestInit) => { const path = new URL(url).pathname; const method = init?.method || "GET"; calls.push({ path, method, headers: new Headers(init?.headers) }); return handler(path, method, calls.length); }, }); const provider = load("lib/ai/providers/vip98k.ts"); return { calls, load, archived, generate: () => provider.createVip98k({ apiKey: "test-only-key", baseURL: "https://provider.test" }) .chat("gpt-image-2.5", { capabilities: { vip98k: { pollIntervalSeconds: 1, maxPollSeconds } } }) .doGenerate({ messages: [{ role: "user", content: "A desk" }] }), run: () => provider.executeVip98kExternalTask({ apiKey: "test-only-key", baseURL: "https://provider.test", modelId: "gpt-image-2.5", kind: "image", prompt: "A desk", imageUrls: [], modelConfig: { capabilities: { vip98k: { pollIntervalSeconds: 1, maxPollSeconds } } }, }), }; } const recoveredImage = { created: 1, data: [{ url: "https://image.example/original.png" }] }; const originalCompleted = { request_id: "original-request", status: "succeeded", result_available: true }; test("an edge timeout waits for and delivers the original image without another POST", async () => { let lookups = 0; const h = imageRecoveryHarness((path, method) => { if (method === "POST") return new Response(gatewayHtml, { status: 524 }); if (path.endsWith("by-idempotency-key")) { return Response.json(++lookups === 1 ? { status: "processing" } : originalCompleted); } return Response.json(recoveredImage); }); expect((await h.generate()).text).toContain(recoveredImage.data[0].url); expect(h.archived).toEqual([recoveredImage.data[0].url]); expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1); expect(lookups).toBe(2); const originalKey = h.calls[0].headers.get("Idempotency-Key"); expect(originalKey).toBeTruthy(); for (const call of h.calls.filter(c => c.path.endsWith("by-idempotency-key"))) { expect(call.headers.get("Idempotency-Key")).toBe(originalKey); expect(call.headers.get("Authorization")).toBe("Bearer test-only-key"); } expect(h.calls.at(-1)?.path).toBe("/api/external/requests/original-request/result"); }); test("a lost POST response recovers by its original key without resubmitting", async () => { const h = imageRecoveryHarness((path, method) => { if (method === "POST") throw new TypeError("fetch failed"); return Response.json(path.endsWith("by-idempotency-key") ? originalCompleted : recoveredImage); }); expect((await h.run()).outputUrls).toHaveLength(1); expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1); }); test("temporary lookup and result-read failures retry only GET requests", async () => { let lookups = 0; let reads = 0; const h = imageRecoveryHarness((path, method) => { if (method === "POST") return new Response("Gateway timeout", { status: 504 }); if (path.endsWith("by-idempotency-key")) { return ++lookups === 1 ? new Response("Unavailable", { status: 503 }) : Response.json(originalCompleted); } return ++reads === 1 ? Response.json({}, { status: 409 }) : Response.json(recoveredImage); }, 5); expect((await h.run()).outputUrls).toHaveLength(1); expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1); expect(reads).toBe(2); }); test("missing, unconfirmed and unretained originals never create a replacement", async () => { for (const state of [null, { status: "failed" }, { status: "unresolved" }, { ...originalCompleted, result_available: false }]) { const h = imageRecoveryHarness((_path, method) => method === "POST" ? new Response(gatewayHtml, { status: 524 }) : Response.json(state || {}, { status: state ? 200 : 404 })); await expect(h.run()).rejects.toThrow(/结果未确认.*HTTP 524/); expect(h.calls.map(c => c.method)).toEqual(["POST", "GET"]); } }); test("a still-processing original stops at the polling deadline without another POST", async () => { const h = imageRecoveryHarness((_path, method) => method === "POST" ? new Response(gatewayHtml, { status: 524 }) : Response.json({ status: "processing" }), 2); await expect(h.run()).rejects.toThrow(/结果未确认.*HTTP 524/); expect(h.calls.map(c => c.method)).toEqual(["POST", "GET", "GET"]); }); test("HTTP 502 and 504 without a recoverable original do not repeat the POST", async () => { for (const status of [502, 504]) { const h = imageRecoveryHarness((_path, method) => method === "POST" ? new Response("Gateway timeout", { status }) : Response.json({}, { status: 404 })); await expect(h.run()).rejects.toThrow(/结果未确认/); expect(h.calls.map(c => c.method)).toEqual(["POST", "GET"]); } }); test("a completed original with no image stays unconfirmed and cannot fail over", async () => { const h = imageRecoveryHarness((path, method) => { if (method === "POST") return new Response(gatewayHtml, { status: 524 }); return Response.json(path.endsWith("by-idempotency-key") ? originalCompleted : { data: [] }); }); const error = await h.run().then(() => null, (e: Error) => e); expect(error).toBeTruthy(); expect(h.load("lib/ai/runtime/model-health.ts").shouldFailoverAiModelError(error)).toBe(false); expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1); }); test("ordinary image success uses the original response directly", async () => { const h = imageRecoveryHarness(() => Response.json(recoveredImage)); expect((await h.run()).outputUrls).toHaveLength(1); expect(h.calls.map(c => c.method)).toEqual(["POST"]); }); test("an explicit provider balance error takes precedence over HTTP 429", () => { const error = Object.assign(new Error("余额不足,请点击充值页面续费(HTTP 429)"), { status: 429 }); expect(isProviderBalanceError(error)).toBe(true); expect(isRateLimitError(error)).toBe(false); expect(getUserFacingImageGenerationErrorMessage(error)).toContain("服务商线路余额"); expect(getUserFacingImageGenerationErrorMessage(error)).toContain("与您的积分余额无关"); expect(getUserFacingImageGenerationErrorMessage(error)).not.toMatch(/^积分不足/); }); test("nested billing details and generic quota limits are kept distinct", () => { const nested = { status: 429, message: "request failed", data: { error: { message: "insufficient_balance" } } }; expect(isProviderBalanceError(nested)).toBe(true); expect(isRateLimitError(nested)).toBe(false); const usageLimit = new Error("Rate quota exceeded (HTTP 429)"); expect(isProviderBalanceError(usageLimit)).toBe(false); expect(isRateLimitError(usageLimit)).toBe(true); expect(isProviderBalanceError(new Error("该模型额度不足(HTTP 429)"))).toBe(true); }); test("confirmed balance failures remain isolated until explicitly restored", () => { const model = { healthLastEventType: "quota", healthLastError: "余额不足(HTTP 429)", }; expect(isImageModelBalanceBlocked(model)).toBe(true); expect(isImageModelBalanceBlocked({ healthLastEventType: "error", healthAutoDisabledReason: "provider_balance_or_billing_quota: 余额不足" })).toBe(true); expect(isImageModelBalanceBlocked({ ...model, healthLastEventType: "success", healthLastError: null })).toBe(false); }); test("old selections use only configured, enabled replacements that preserve the requested inputs", () => { const requested = { healthLastEventType: "quota", healthLastError: "余额不足", fallbackModelId1: "verified" }; const verified = { model: { id: "verified", modelId: "same-family", isEnabled: true, supportsImageGen: true, imageGenConfig: { supportedImageSizes: ["2K"], supportedAspectRatios: ["1:1"], maxReferenceImages: 8 } }, provider: { isEnabled: true } }; const input = { imageSize: "2K", aspectRatio: "1:1", referenceImageCount: 2 }; expect(resolveConfiguredImageBalanceFallback(requested, [verified], input)).toBe(verified); expect(resolveConfiguredImageBalanceFallback(requested, [verified], { ...input, referenceImageCount: 9 })).toBeUndefined(); expect(resolveConfiguredImageBalanceFallback(requested, [verified], { ...input, imageSize: "4K" })).toBeUndefined(); expect(resolveConfiguredImageBalanceFallback(requested, [verified], { ...input, aspectRatio: "9:16" })).toBeUndefined(); expect(resolveConfiguredImageBalanceFallback(requested, [{ ...verified, provider: { isEnabled: false } }], input)).toBeUndefined(); expect(resolveConfiguredImageBalanceFallback({ ...requested, fallbackModelId1: null }, [verified], input)).toBeUndefined(); }); function healthHarness() { let model: any = { id: "test", modelId: "test", isEnabled: true, excludeFromFailover: true, excludeFromHealthBan: true, healthStatus: "healthy", healthConsecutiveFailures: 0 }; const events: any[] = []; const query = (rows: () => any[]) => Object.assign(Promise.resolve().then(rows), { from() { return this; }, where() { return this; }, limit: async () => rows(), returning: async () => rows(), }); const db = { select: () => query(() => [model]), insert: () => ({ values: async (value: any) => { events.push(value); } }), update: () => ({ set: (value: any) => { model = { ...model, ...value }; return query(() => [model]); } }), }; const health = serverLoader({ "@/lib/db": { db } })("lib/ai/runtime/model-health.ts"); return { health, events, get model() { return model; }, age() { model.healthAutoDisabledAt = new Date(Date.now() - 4 * 60 * 60_000); } }; } test("quota pauses an exempt model, survives cooldown and late completion, and can be restored", async () => { const h = healthHarness(); await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("余额不足(HTTP 429)") }); expect(h.model.isEnabled).toBe(false); expect(h.model.healthStatus).toBe("auto_disabled"); expect(h.events[0].eventType).toBe("quota"); await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("HTTP 503 unavailable") }); await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("Too many requests (HTTP 429)") }); expect(h.model.healthStatus).toBe("auto_disabled"); expect(isImageModelBalanceBlocked(h.model)).toBe(true); h.age(); expect(await h.health.reviveAiModelsFromCooldown()).toEqual([]); await h.health.recordAiModelHealthSuccess({ model: h.model }); expect(h.model.isEnabled).toBe(false); await h.health.restoreAiModelHealth(h.model.id); expect(h.model.isEnabled).toBe(true); expect(isImageModelBalanceBlocked(h.model)).toBe(false); }); test("ordinary rate limiting does not trigger billing isolation", async () => { const h = healthHarness(); await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("Too many requests (HTTP 429)") }); expect(h.model.isEnabled).toBe(true); expect(isImageModelBalanceBlocked(h.model)).toBe(false); });