WuYongJie
ac7928fe2d
同步生图超过边缘等待期限时，生成仍可能在源站继续执行。实际记录中，Cloudflare 在 125.006 秒断开，原请求在约 172 秒完成；应用已经显示 524，成功图片没有送达。原结果 GET 和原图片只读验证均返回 200。 同步图片生成／编辑收到 502、504、524 或提交响应丢失后，现在使用原 Key 和原 Idempotency-Key 查询既有请求。处理中继续查询，成功后 GET 同一请求的保留结果，进入原有图片归档与展示流程。查询复用已有轮询间隔和期限；缺失、失败、未确认、过期或未保留结果不会变成新的生成提交。普通成功响应保持直接返回，原生任务沿用现有协议。ProviderTaskUncertainError 也不触发自动模型切换。 验证：645 项辅助测试通过、1 项原有跳过，包含 524 后处理中到成功取回并进入展示流程、提交响应丢失、查询暂时失败、结果 GET 暂时失败、过期／缺失结果、轮询期限及一次 POST。TypeScript、改动文件 lint、日志脱敏、凭据、测试与制品检查通过。生产构建与上线验收另行记录。 已知问题：全量 lint 有 1004 项历史诊断。Cloudflare 本身的读取期限保持；本次通过取回已受理请求的原结果完成交付，不提高边缘期限。
279 lines
15 KiB
TypeScript
279 lines
15 KiB
TypeScript
import { expect, test } from "@playwright/test";
|
||
import {
|
||
getUserFacingImageGenerationErrorMessage,
|
||
getUserFacingRequestErrorMessage,
|
||
getGatewayErrorPageMessage,
|
||
isGatewayResponseTimeout,
|
||
isProviderBalanceError,
|
||
isRateLimitError,
|
||
} from "@/lib/request-error";
|
||
import { isImageModelBalanceBlocked, resolveConfiguredImageBalanceFallback } from "@/lib/ai/runtime/image-balance";
|
||
import { serverLoader } from "../helpers/canvas-download-server-loader";
|
||
|
||
test("a bare HTTP 429 is rate limiting, not insufficient balance", () => {
|
||
const error = Object.assign(new Error("Too many requests (HTTP 429)"), { status: 429 });
|
||
expect(isProviderBalanceError(error)).toBe(false);
|
||
expect(isRateLimitError(error)).toBe(true);
|
||
expect(getUserFacingImageGenerationErrorMessage(error)).toContain("请求过于频繁");
|
||
expect(getUserFacingRequestErrorMessage(error)).toContain("请求过于频繁");
|
||
});
|
||
|
||
const gatewayHtml = '<!DOCTYPE html><html><head><title>vip98k.com | 524: A timeout occurred</title></head><body><div id="cf-error-details">Error code 524</div><script>window.gatewayDebug = true</script></body></html> (HTTP 524)';
|
||
|
||
test("gateway timeout pages become a concise unconfirmed-result message", () => {
|
||
for (const error of [new Error(gatewayHtml), { status: 524, message: "Gateway failed" }, { error: { message: "Request failed (HTTP 524)" } }]) {
|
||
expect(isGatewayResponseTimeout(error)).toBe(true);
|
||
for (const text of [getUserFacingImageGenerationErrorMessage(error), getUserFacingRequestErrorMessage(error)]) {
|
||
expect(text).toContain("HTTP 524");
|
||
expect(text).toContain("结果尚未确认");
|
||
expect(text).not.toMatch(/<html|<script|cf-error-details/);
|
||
}
|
||
}
|
||
expect(getGatewayErrorPageMessage(gatewayHtml)).toContain("避免重复提交");
|
||
});
|
||
|
||
test("ordinary HTML documents and fenced examples retain their content", () => {
|
||
for (const text of ['<!DOCTYPE html><html><title>My page</title><p>Hosted on Cloudflare</p></html>', `\`\`\`html\n${gatewayHtml}\n\`\`\``, "Cloudflare 文档示例", "普通图片错误"]) {
|
||
expect(getGatewayErrorPageMessage(text)).toBeNull();
|
||
}
|
||
});
|
||
|
||
test("HTTP 524 does not start another model while the original result is unconfirmed", () => {
|
||
const { health } = healthHarness();
|
||
expect(health.shouldFailoverAiModelError(new Error(gatewayHtml))).toBe(false);
|
||
expect(health.shouldFailoverAiModelError({ status: 524, message: "Gateway timeout" })).toBe(false);
|
||
expect(health.shouldFailoverAiModelError(new Error("Service unavailable (HTTP 503)"))).toBe(true);
|
||
});
|
||
|
||
test("the provider submits once when an HTTP 524 request cannot be located", async () => {
|
||
let submissions = 0;
|
||
const provider = serverLoader({
|
||
"@/lib/db/queries": { getSystemSetting: async () => null },
|
||
"@/lib/storage/image-archive": {},
|
||
"@/lib/storage/service": {},
|
||
fetch: async (_url: string, init: RequestInit) => {
|
||
if (init?.method !== "POST") return Response.json({}, { status: 404 });
|
||
submissions++;
|
||
return new Response(gatewayHtml, { status: 524, headers: { "content-type": "text/html" } });
|
||
},
|
||
})("lib/ai/providers/vip98k.ts");
|
||
const model = provider.createVip98k({ apiKey: "test", baseURL: "https://provider.test" }).chat("gpt-image-2.5");
|
||
await expect(model.doGenerate({ messages: [{ role: "user", content: "A desk" }] })).rejects.toThrow(/结果未确认.*HTTP 524/);
|
||
expect(submissions).toBe(1);
|
||
});
|
||
|
||
function imageRecoveryHarness(
|
||
handler: (path: string, method: string, call: number) => Response | Promise<Response>,
|
||
maxPollSeconds = 3,
|
||
) {
|
||
let now = 1_000;
|
||
const calls: { path: string; method: string; headers: Headers }[] = [];
|
||
const archived: string[] = [];
|
||
const load = serverLoader({
|
||
"@/lib/db": { db: {} },
|
||
"@/lib/db/queries": { getSystemSetting: async () => null },
|
||
"@/lib/storage/image-archive": { createDeferredUploadedImage: async (url: string) => {
|
||
archived.push(url);
|
||
return { thumbnail: url, original: url };
|
||
} },
|
||
"@/lib/storage/service": {},
|
||
Date: class extends Date { static now() { return now; } },
|
||
setTimeout: (fn: () => void, ms: number) => { now += ms; return setTimeout(fn, 0); },
|
||
fetch: async (url: string, init: RequestInit) => {
|
||
const path = new URL(url).pathname;
|
||
const method = init?.method || "GET";
|
||
calls.push({ path, method, headers: new Headers(init?.headers) });
|
||
return handler(path, method, calls.length);
|
||
},
|
||
});
|
||
const provider = load("lib/ai/providers/vip98k.ts");
|
||
return {
|
||
calls, load, archived,
|
||
generate: () => provider.createVip98k({ apiKey: "test-only-key", baseURL: "https://provider.test" })
|
||
.chat("gpt-image-2.5", { capabilities: { vip98k: { pollIntervalSeconds: 1, maxPollSeconds } } })
|
||
.doGenerate({ messages: [{ role: "user", content: "A desk" }] }),
|
||
run: () => provider.executeVip98kExternalTask({
|
||
apiKey: "test-only-key", baseURL: "https://provider.test", modelId: "gpt-image-2.5",
|
||
kind: "image", prompt: "A desk", imageUrls: [],
|
||
modelConfig: { capabilities: { vip98k: { pollIntervalSeconds: 1, maxPollSeconds } } },
|
||
}),
|
||
};
|
||
}
|
||
|
||
const recoveredImage = { created: 1, data: [{ url: "https://image.example/original.png" }] };
|
||
const originalCompleted = { request_id: "original-request", status: "succeeded", result_available: true };
|
||
|
||
test("an edge timeout waits for and delivers the original image without another POST", async () => {
|
||
let lookups = 0;
|
||
const h = imageRecoveryHarness((path, method) => {
|
||
if (method === "POST") return new Response(gatewayHtml, { status: 524 });
|
||
if (path.endsWith("by-idempotency-key")) {
|
||
return Response.json(++lookups === 1 ? { status: "processing" } : originalCompleted);
|
||
}
|
||
return Response.json(recoveredImage);
|
||
});
|
||
expect((await h.generate()).text).toContain(recoveredImage.data[0].url);
|
||
expect(h.archived).toEqual([recoveredImage.data[0].url]);
|
||
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
|
||
expect(lookups).toBe(2);
|
||
const originalKey = h.calls[0].headers.get("Idempotency-Key");
|
||
expect(originalKey).toBeTruthy();
|
||
for (const call of h.calls.filter(c => c.path.endsWith("by-idempotency-key"))) {
|
||
expect(call.headers.get("Idempotency-Key")).toBe(originalKey);
|
||
expect(call.headers.get("Authorization")).toBe("Bearer test-only-key");
|
||
}
|
||
expect(h.calls.at(-1)?.path).toBe("/api/external/requests/original-request/result");
|
||
});
|
||
|
||
test("a lost POST response recovers by its original key without resubmitting", async () => {
|
||
const h = imageRecoveryHarness((path, method) => {
|
||
if (method === "POST") throw new TypeError("fetch failed");
|
||
return Response.json(path.endsWith("by-idempotency-key") ? originalCompleted : recoveredImage);
|
||
});
|
||
expect((await h.run()).outputUrls).toHaveLength(1);
|
||
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
|
||
});
|
||
|
||
test("temporary lookup and result-read failures retry only GET requests", async () => {
|
||
let lookups = 0;
|
||
let reads = 0;
|
||
const h = imageRecoveryHarness((path, method) => {
|
||
if (method === "POST") return new Response("Gateway timeout", { status: 504 });
|
||
if (path.endsWith("by-idempotency-key")) {
|
||
return ++lookups === 1 ? new Response("Unavailable", { status: 503 }) : Response.json(originalCompleted);
|
||
}
|
||
return ++reads === 1 ? Response.json({}, { status: 409 }) : Response.json(recoveredImage);
|
||
}, 5);
|
||
expect((await h.run()).outputUrls).toHaveLength(1);
|
||
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
|
||
expect(reads).toBe(2);
|
||
});
|
||
|
||
test("missing, unconfirmed and unretained originals never create a replacement", async () => {
|
||
for (const state of [null, { status: "failed" }, { status: "unresolved" }, { ...originalCompleted, result_available: false }]) {
|
||
const h = imageRecoveryHarness((_path, method) => method === "POST"
|
||
? new Response(gatewayHtml, { status: 524 })
|
||
: Response.json(state || {}, { status: state ? 200 : 404 }));
|
||
await expect(h.run()).rejects.toThrow(/结果未确认.*HTTP 524/);
|
||
expect(h.calls.map(c => c.method)).toEqual(["POST", "GET"]);
|
||
}
|
||
});
|
||
|
||
test("a still-processing original stops at the polling deadline without another POST", async () => {
|
||
const h = imageRecoveryHarness((_path, method) => method === "POST"
|
||
? new Response(gatewayHtml, { status: 524 }) : Response.json({ status: "processing" }), 2);
|
||
await expect(h.run()).rejects.toThrow(/结果未确认.*HTTP 524/);
|
||
expect(h.calls.map(c => c.method)).toEqual(["POST", "GET", "GET"]);
|
||
});
|
||
|
||
test("HTTP 502 and 504 without a recoverable original do not repeat the POST", async () => {
|
||
for (const status of [502, 504]) {
|
||
const h = imageRecoveryHarness((_path, method) => method === "POST"
|
||
? new Response("Gateway timeout", { status }) : Response.json({}, { status: 404 }));
|
||
await expect(h.run()).rejects.toThrow(/结果未确认/);
|
||
expect(h.calls.map(c => c.method)).toEqual(["POST", "GET"]);
|
||
}
|
||
});
|
||
|
||
test("a completed original with no image stays unconfirmed and cannot fail over", async () => {
|
||
const h = imageRecoveryHarness((path, method) => {
|
||
if (method === "POST") return new Response(gatewayHtml, { status: 524 });
|
||
return Response.json(path.endsWith("by-idempotency-key") ? originalCompleted : { data: [] });
|
||
});
|
||
const error = await h.run().then(() => null, (e: Error) => e);
|
||
expect(error).toBeTruthy();
|
||
expect(h.load("lib/ai/runtime/model-health.ts").shouldFailoverAiModelError(error)).toBe(false);
|
||
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
|
||
});
|
||
|
||
test("ordinary image success uses the original response directly", async () => {
|
||
const h = imageRecoveryHarness(() => Response.json(recoveredImage));
|
||
expect((await h.run()).outputUrls).toHaveLength(1);
|
||
expect(h.calls.map(c => c.method)).toEqual(["POST"]);
|
||
});
|
||
|
||
test("an explicit provider balance error takes precedence over HTTP 429", () => {
|
||
const error = Object.assign(new Error("余额不足，请点击充值页面续费（HTTP 429）"), { status: 429 });
|
||
expect(isProviderBalanceError(error)).toBe(true);
|
||
expect(isRateLimitError(error)).toBe(false);
|
||
expect(getUserFacingImageGenerationErrorMessage(error)).toContain("服务商线路余额");
|
||
expect(getUserFacingImageGenerationErrorMessage(error)).toContain("与您的积分余额无关");
|
||
expect(getUserFacingImageGenerationErrorMessage(error)).not.toMatch(/^积分不足/);
|
||
});
|
||
|
||
test("nested billing details and generic quota limits are kept distinct", () => {
|
||
const nested = { status: 429, message: "request failed", data: { error: { message: "insufficient_balance" } } };
|
||
expect(isProviderBalanceError(nested)).toBe(true);
|
||
expect(isRateLimitError(nested)).toBe(false);
|
||
const usageLimit = new Error("Rate quota exceeded (HTTP 429)");
|
||
expect(isProviderBalanceError(usageLimit)).toBe(false);
|
||
expect(isRateLimitError(usageLimit)).toBe(true);
|
||
expect(isProviderBalanceError(new Error("该模型额度不足（HTTP 429）"))).toBe(true);
|
||
});
|
||
|
||
test("confirmed balance failures remain isolated until explicitly restored", () => {
|
||
const model = {
|
||
healthLastEventType: "quota",
|
||
healthLastError: "余额不足（HTTP 429）",
|
||
};
|
||
expect(isImageModelBalanceBlocked(model)).toBe(true);
|
||
expect(isImageModelBalanceBlocked({ healthLastEventType: "error", healthAutoDisabledReason: "provider_balance_or_billing_quota: 余额不足" })).toBe(true);
|
||
expect(isImageModelBalanceBlocked({ ...model, healthLastEventType: "success", healthLastError: null })).toBe(false);
|
||
});
|
||
|
||
test("old selections use only configured, enabled replacements that preserve the requested inputs", () => {
|
||
const requested = { healthLastEventType: "quota", healthLastError: "余额不足", fallbackModelId1: "verified" };
|
||
const verified = { model: { id: "verified", modelId: "same-family", isEnabled: true, supportsImageGen: true,
|
||
imageGenConfig: { supportedImageSizes: ["2K"], supportedAspectRatios: ["1:1"], maxReferenceImages: 8 } }, provider: { isEnabled: true } };
|
||
const input = { imageSize: "2K", aspectRatio: "1:1", referenceImageCount: 2 };
|
||
expect(resolveConfiguredImageBalanceFallback(requested, [verified], input)).toBe(verified);
|
||
expect(resolveConfiguredImageBalanceFallback(requested, [verified], { ...input, referenceImageCount: 9 })).toBeUndefined();
|
||
expect(resolveConfiguredImageBalanceFallback(requested, [verified], { ...input, imageSize: "4K" })).toBeUndefined();
|
||
expect(resolveConfiguredImageBalanceFallback(requested, [verified], { ...input, aspectRatio: "9:16" })).toBeUndefined();
|
||
expect(resolveConfiguredImageBalanceFallback(requested, [{ ...verified, provider: { isEnabled: false } }], input)).toBeUndefined();
|
||
expect(resolveConfiguredImageBalanceFallback({ ...requested, fallbackModelId1: null }, [verified], input)).toBeUndefined();
|
||
});
|
||
|
||
function healthHarness() {
|
||
let model: any = { id: "test", modelId: "test", isEnabled: true, excludeFromFailover: true,
|
||
excludeFromHealthBan: true, healthStatus: "healthy", healthConsecutiveFailures: 0 };
|
||
const events: any[] = [];
|
||
const query = (rows: () => any[]) => Object.assign(Promise.resolve().then(rows), {
|
||
from() { return this; }, where() { return this; },
|
||
limit: async () => rows(), returning: async () => rows(),
|
||
});
|
||
const db = {
|
||
select: () => query(() => [model]),
|
||
insert: () => ({ values: async (value: any) => { events.push(value); } }),
|
||
update: () => ({ set: (value: any) => { model = { ...model, ...value }; return query(() => [model]); } }),
|
||
};
|
||
const health = serverLoader({ "@/lib/db": { db } })("lib/ai/runtime/model-health.ts");
|
||
return { health, events, get model() { return model; }, age() { model.healthAutoDisabledAt = new Date(Date.now() - 4 * 60 * 60_000); } };
|
||
}
|
||
|
||
test("quota pauses an exempt model, survives cooldown and late completion, and can be restored", async () => {
|
||
const h = healthHarness();
|
||
await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("余额不足（HTTP 429）") });
|
||
expect(h.model.isEnabled).toBe(false);
|
||
expect(h.model.healthStatus).toBe("auto_disabled");
|
||
expect(h.events[0].eventType).toBe("quota");
|
||
await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("HTTP 503 unavailable") });
|
||
await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("Too many requests (HTTP 429)") });
|
||
expect(h.model.healthStatus).toBe("auto_disabled");
|
||
expect(isImageModelBalanceBlocked(h.model)).toBe(true);
|
||
h.age();
|
||
expect(await h.health.reviveAiModelsFromCooldown()).toEqual([]);
|
||
await h.health.recordAiModelHealthSuccess({ model: h.model });
|
||
expect(h.model.isEnabled).toBe(false);
|
||
await h.health.restoreAiModelHealth(h.model.id);
|
||
expect(h.model.isEnabled).toBe(true);
|
||
expect(isImageModelBalanceBlocked(h.model)).toBe(false);
|
||
});
|
||
|
||
test("ordinary rate limiting does not trigger billing isolation", async () => {
|
||
const h = healthHarness();
|
||
await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("Too many requests (HTTP 429)") });
|
||
expect(h.model.isEnabled).toBe(true);
|
||
expect(isImageModelBalanceBlocked(h.model)).toBe(false);
|
||
});
|