恢复同步生图在网关超时后的原请求结果
同步生图超过边缘等待期限时，生成仍可能在源站继续执行。实际记录中，Cloudflare 在 125.006 秒断开，原请求在约 172 秒完成；应用已经显示 524，成功图片没有送达。原结果 GET 和原图片只读验证均返回 200。 同步图片生成／编辑收到 502、504、524 或提交响应丢失后，现在使用原 Key 和原 Idempotency-Key 查询既有请求。处理中继续查询，成功后 GET 同一请求的保留结果，进入原有图片归档与展示流程。查询复用已有轮询间隔和期限；缺失、失败、未确认、过期或未保留结果不会变成新的生成提交。普通成功响应保持直接返回，原生任务沿用现有协议。ProviderTaskUncertainError 也不触发自动模型切换。 验证：645 项辅助测试通过、1 项原有跳过，包含 524 后处理中到成功取回并进入展示流程、提交响应丢失、查询暂时失败、结果 GET 暂时失败、过期／缺失结果、轮询期限及一次 POST。TypeScript、改动文件 lint、日志脱敏、凭据、测试与制品检查通过。生产构建与上线验收另行记录。 已知问题：全量 lint 有 1004 项历史诊断。Cloudflare 本身的读取期限保持；本次通过取回已受理请求的原结果完成交付，不提高边缘期限。
This commit is contained in:
parent
b06fdbb38b
commit
ac7928fe2d
@ -1024,6 +1024,77 @@ function getConfiguredPollSeconds(config: Vip98kModelRuntimeConfig, key: string,
|
||||
return Math.min(Math.max(value, 1), 3600);
|
||||
}
|
||||
|
||||
function isSynchronousImagePath(path: string) {
|
||||
return /^\/v1\/images\/(?:generations|edits)\/?$/.test(path);
|
||||
}
|
||||
|
||||
async function recoverImageSubmission(params: {
|
||||
apiKey: string;
|
||||
baseURL: string;
|
||||
modelId: string;
|
||||
path: string;
|
||||
idempotencyKey: string;
|
||||
pollIntervalMs: number;
|
||||
maxPollMs: number;
|
||||
}) {
|
||||
if (!isSynchronousImagePath(params.path)) return null;
|
||||
const startedAt = Date.now();
|
||||
const deadline = startedAt + params.maxPollMs;
|
||||
logger.info("recovering original image request", { modelId: params.modelId });
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const lookup = await fetch(`${params.baseURL}/api/external/requests/by-idempotency-key`, {
|
||||
cache: "no-store",
|
||||
headers: {
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
"Idempotency-Key": params.idempotencyKey,
|
||||
},
|
||||
signal: AbortSignal.timeout(Math.min(60_000, deadline - Date.now())),
|
||||
});
|
||||
if ([401, 403, 404, 410].includes(lookup.status)) return null;
|
||||
if (lookup.ok) {
|
||||
const state = toRecord(await parseJsonResponse(lookup));
|
||||
if (!state || !["processing", "succeeded"].includes(String(state.status))) return null;
|
||||
if (state.status === "succeeded") {
|
||||
if (state.result_available !== true || !state.request_id) return null;
|
||||
const result = await fetch(
|
||||
`${params.baseURL}/api/external/requests/${encodeURIComponent(String(state.request_id))}/result`,
|
||||
{
|
||||
cache: "no-store",
|
||||
headers: { Authorization: `Bearer ${params.apiKey}` },
|
||||
signal: AbortSignal.timeout(Math.max(1, Math.min(60_000, deadline - Date.now()))),
|
||||
},
|
||||
);
|
||||
if ([401, 403, 404, 410].includes(result.status)) return null;
|
||||
if (result.ok) {
|
||||
const payload = await parseJsonResponse(result);
|
||||
const outputs = collectOutputUrls(payload);
|
||||
if (!outputs.length) {
|
||||
throw new ProviderTaskUncertainError("原生图请求已完成，但未取回图片结果，请勿重复生成");
|
||||
}
|
||||
logger.info("original image request recovered", {
|
||||
modelId: params.modelId,
|
||||
providerRequestId: state.request_id,
|
||||
outputCount: outputs.length,
|
||||
durationMs: Date.now() - startedAt,
|
||||
});
|
||||
return { payload, taskId: extractTaskId(payload), resultPath: extractResultPath(payload), outputs };
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof ProviderTaskUncertainError) throw error;
|
||||
}
|
||||
await delay(Math.max(0, Math.min(params.pollIntervalMs, deadline - Date.now())));
|
||||
}
|
||||
logger.warn("original image request recovery timed out", {
|
||||
modelId: params.modelId,
|
||||
durationMs: Date.now() - startedAt,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
async function submitTask(params: {
|
||||
apiKey: string;
|
||||
baseURL?: string | null;
|
||||
@ -1036,6 +1107,8 @@ async function submitTask(params: {
|
||||
submissionTimeoutMs: number;
|
||||
maxSubmissionAttempts: number;
|
||||
submissionRetryBaseDelayMs: number;
|
||||
pollIntervalMs: number;
|
||||
maxPollMs: number;
|
||||
}) {
|
||||
const baseURL = normalizeVip98kBaseUrl(params.baseURL);
|
||||
const path = String(params.path || "/api/external/task/generate").trim();
|
||||
@ -1099,7 +1172,9 @@ async function submitTask(params: {
|
||||
signal: AbortSignal.timeout(params.submissionTimeoutMs),
|
||||
});
|
||||
} catch (error) {
|
||||
if (attempt >= maxAttempts) {
|
||||
const recovered = await recoverImageSubmission({ ...params, baseURL, path, idempotencyKey });
|
||||
if (recovered) return recovered;
|
||||
if (isSynchronousImagePath(path) || attempt >= maxAttempts) {
|
||||
logger.error("provider submission transport failed", {
|
||||
modelId: params.modelId,
|
||||
path,
|
||||
@ -1145,6 +1220,14 @@ async function submitTask(params: {
|
||||
};
|
||||
}
|
||||
|
||||
if ([502, 504, 524].includes(response.status)) {
|
||||
const recovered = await recoverImageSubmission({ ...params, baseURL, path, idempotencyKey });
|
||||
if (recovered) return recovered;
|
||||
if (isSynchronousImagePath(path)) {
|
||||
throw new ProviderTaskUncertainError(`供应商提交结果未确认，可能仍在处理，请勿重复生成：${getUserFacingImageGenerationErrorMessage({ message: extractErrorMessage(payload), status: response.status })} (HTTP ${response.status})`);
|
||||
}
|
||||
}
|
||||
|
||||
const errorMessage = extractErrorMessage(payload);
|
||||
const stagedMediaRetryDelayMs = getVip98kStagedMediaRetryDelay({
|
||||
status: response.status,
|
||||
@ -1311,6 +1394,12 @@ export async function submitVip98kExternalTask(params: {
|
||||
),
|
||||
maxSubmissionAttempts: params.durable ? 1 : runtimeConfig.maxSubmissionAttempts,
|
||||
submissionRetryBaseDelayMs: runtimeConfig.submissionRetryBaseDelayMs,
|
||||
pollIntervalMs: getConfiguredPollSeconds(
|
||||
taskParams.modelConfig, "pollIntervalSeconds", DEFAULT_POLL_INTERVAL_SECONDS,
|
||||
) * 1000,
|
||||
maxPollMs: getConfiguredPollSeconds(
|
||||
taskParams.modelConfig, "maxPollSeconds", DEFAULT_MAX_POLL_SECONDS,
|
||||
) * 1000,
|
||||
});
|
||||
const resultPath = submitted.resultPath || invocation?.resultPath || "";
|
||||
return {
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import "server-only";
|
||||
import { ProviderTaskUncertainError } from "@/lib/video-canvas/video-task-lifecycle";
|
||||
import { isProviderAuthenticationFailure, isCredentialFailureBan } from "./authentication-failover";
|
||||
|
||||
import { and, desc, eq, inArray, sql } from "drizzle-orm";
|
||||
@ -343,7 +344,7 @@ export function classifyAiModelHealthFailure(error: unknown): {
|
||||
}
|
||||
|
||||
export function shouldFailoverAiModelError(error: unknown) {
|
||||
if (isGatewayResponseTimeout(error)) return false;
|
||||
if (error instanceof ProviderTaskUncertainError || isGatewayResponseTimeout(error)) return false;
|
||||
const classification = classifyAiModelHealthFailure(error);
|
||||
const normalized = getErrorMessage(error).toLowerCase();
|
||||
const imageProviderTaskFailed =
|
||||
|
||||
@ -45,13 +45,14 @@ test("HTTP 524 does not start another model while the original result is unconfi
|
||||
expect(health.shouldFailoverAiModelError(new Error("Service unavailable (HTTP 503)"))).toBe(true);
|
||||
});
|
||||
|
||||
test("the provider submits once and keeps an HTTP 524 response unconfirmed", async () => {
|
||||
test("the provider submits once when an HTTP 524 request cannot be located", async () => {
|
||||
let submissions = 0;
|
||||
const provider = serverLoader({
|
||||
"@/lib/db/queries": { getSystemSetting: async () => null },
|
||||
"@/lib/storage/image-archive": {},
|
||||
"@/lib/storage/service": {},
|
||||
fetch: async () => {
|
||||
fetch: async (_url: string, init: RequestInit) => {
|
||||
if (init?.method !== "POST") return Response.json({}, { status: 404 });
|
||||
submissions++;
|
||||
return new Response(gatewayHtml, { status: 524, headers: { "content-type": "text/html" } });
|
||||
},
|
||||
@ -61,6 +62,136 @@ test("the provider submits once and keeps an HTTP 524 response unconfirmed", asy
|
||||
expect(submissions).toBe(1);
|
||||
});
|
||||
|
||||
function imageRecoveryHarness(
|
||||
handler: (path: string, method: string, call: number) => Response | Promise<Response>,
|
||||
maxPollSeconds = 3,
|
||||
) {
|
||||
let now = 1_000;
|
||||
const calls: { path: string; method: string; headers: Headers }[] = [];
|
||||
const archived: string[] = [];
|
||||
const load = serverLoader({
|
||||
"@/lib/db": { db: {} },
|
||||
"@/lib/db/queries": { getSystemSetting: async () => null },
|
||||
"@/lib/storage/image-archive": { createDeferredUploadedImage: async (url: string) => {
|
||||
archived.push(url);
|
||||
return { thumbnail: url, original: url };
|
||||
} },
|
||||
"@/lib/storage/service": {},
|
||||
Date: class extends Date { static now() { return now; } },
|
||||
setTimeout: (fn: () => void, ms: number) => { now += ms; return setTimeout(fn, 0); },
|
||||
fetch: async (url: string, init: RequestInit) => {
|
||||
const path = new URL(url).pathname;
|
||||
const method = init?.method || "GET";
|
||||
calls.push({ path, method, headers: new Headers(init?.headers) });
|
||||
return handler(path, method, calls.length);
|
||||
},
|
||||
});
|
||||
const provider = load("lib/ai/providers/vip98k.ts");
|
||||
return {
|
||||
calls, load, archived,
|
||||
generate: () => provider.createVip98k({ apiKey: "test-only-key", baseURL: "https://provider.test" })
|
||||
.chat("gpt-image-2.5", { capabilities: { vip98k: { pollIntervalSeconds: 1, maxPollSeconds } } })
|
||||
.doGenerate({ messages: [{ role: "user", content: "A desk" }] }),
|
||||
run: () => provider.executeVip98kExternalTask({
|
||||
apiKey: "test-only-key", baseURL: "https://provider.test", modelId: "gpt-image-2.5",
|
||||
kind: "image", prompt: "A desk", imageUrls: [],
|
||||
modelConfig: { capabilities: { vip98k: { pollIntervalSeconds: 1, maxPollSeconds } } },
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
const recoveredImage = { created: 1, data: [{ url: "https://image.example/original.png" }] };
|
||||
const originalCompleted = { request_id: "original-request", status: "succeeded", result_available: true };
|
||||
|
||||
test("an edge timeout waits for and delivers the original image without another POST", async () => {
|
||||
let lookups = 0;
|
||||
const h = imageRecoveryHarness((path, method) => {
|
||||
if (method === "POST") return new Response(gatewayHtml, { status: 524 });
|
||||
if (path.endsWith("by-idempotency-key")) {
|
||||
return Response.json(++lookups === 1 ? { status: "processing" } : originalCompleted);
|
||||
}
|
||||
return Response.json(recoveredImage);
|
||||
});
|
||||
expect((await h.generate()).text).toContain(recoveredImage.data[0].url);
|
||||
expect(h.archived).toEqual([recoveredImage.data[0].url]);
|
||||
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
|
||||
expect(lookups).toBe(2);
|
||||
const originalKey = h.calls[0].headers.get("Idempotency-Key");
|
||||
expect(originalKey).toBeTruthy();
|
||||
for (const call of h.calls.filter(c => c.path.endsWith("by-idempotency-key"))) {
|
||||
expect(call.headers.get("Idempotency-Key")).toBe(originalKey);
|
||||
expect(call.headers.get("Authorization")).toBe("Bearer test-only-key");
|
||||
}
|
||||
expect(h.calls.at(-1)?.path).toBe("/api/external/requests/original-request/result");
|
||||
});
|
||||
|
||||
test("a lost POST response recovers by its original key without resubmitting", async () => {
|
||||
const h = imageRecoveryHarness((path, method) => {
|
||||
if (method === "POST") throw new TypeError("fetch failed");
|
||||
return Response.json(path.endsWith("by-idempotency-key") ? originalCompleted : recoveredImage);
|
||||
});
|
||||
expect((await h.run()).outputUrls).toHaveLength(1);
|
||||
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("temporary lookup and result-read failures retry only GET requests", async () => {
|
||||
let lookups = 0;
|
||||
let reads = 0;
|
||||
const h = imageRecoveryHarness((path, method) => {
|
||||
if (method === "POST") return new Response("Gateway timeout", { status: 504 });
|
||||
if (path.endsWith("by-idempotency-key")) {
|
||||
return ++lookups === 1 ? new Response("Unavailable", { status: 503 }) : Response.json(originalCompleted);
|
||||
}
|
||||
return ++reads === 1 ? Response.json({}, { status: 409 }) : Response.json(recoveredImage);
|
||||
}, 5);
|
||||
expect((await h.run()).outputUrls).toHaveLength(1);
|
||||
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
|
||||
expect(reads).toBe(2);
|
||||
});
|
||||
|
||||
test("missing, unconfirmed and unretained originals never create a replacement", async () => {
|
||||
for (const state of [null, { status: "failed" }, { status: "unresolved" }, { ...originalCompleted, result_available: false }]) {
|
||||
const h = imageRecoveryHarness((_path, method) => method === "POST"
|
||||
? new Response(gatewayHtml, { status: 524 })
|
||||
: Response.json(state || {}, { status: state ? 200 : 404 }));
|
||||
await expect(h.run()).rejects.toThrow(/结果未确认.*HTTP 524/);
|
||||
expect(h.calls.map(c => c.method)).toEqual(["POST", "GET"]);
|
||||
}
|
||||
});
|
||||
|
||||
test("a still-processing original stops at the polling deadline without another POST", async () => {
|
||||
const h = imageRecoveryHarness((_path, method) => method === "POST"
|
||||
? new Response(gatewayHtml, { status: 524 }) : Response.json({ status: "processing" }), 2);
|
||||
await expect(h.run()).rejects.toThrow(/结果未确认.*HTTP 524/);
|
||||
expect(h.calls.map(c => c.method)).toEqual(["POST", "GET", "GET"]);
|
||||
});
|
||||
|
||||
test("HTTP 502 and 504 without a recoverable original do not repeat the POST", async () => {
|
||||
for (const status of [502, 504]) {
|
||||
const h = imageRecoveryHarness((_path, method) => method === "POST"
|
||||
? new Response("Gateway timeout", { status }) : Response.json({}, { status: 404 }));
|
||||
await expect(h.run()).rejects.toThrow(/结果未确认/);
|
||||
expect(h.calls.map(c => c.method)).toEqual(["POST", "GET"]);
|
||||
}
|
||||
});
|
||||
|
||||
test("a completed original with no image stays unconfirmed and cannot fail over", async () => {
|
||||
const h = imageRecoveryHarness((path, method) => {
|
||||
if (method === "POST") return new Response(gatewayHtml, { status: 524 });
|
||||
return Response.json(path.endsWith("by-idempotency-key") ? originalCompleted : { data: [] });
|
||||
});
|
||||
const error = await h.run().then(() => null, (e: Error) => e);
|
||||
expect(error).toBeTruthy();
|
||||
expect(h.load("lib/ai/runtime/model-health.ts").shouldFailoverAiModelError(error)).toBe(false);
|
||||
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("ordinary image success uses the original response directly", async () => {
|
||||
const h = imageRecoveryHarness(() => Response.json(recoveredImage));
|
||||
expect((await h.run()).outputUrls).toHaveLength(1);
|
||||
expect(h.calls.map(c => c.method)).toEqual(["POST"]);
|
||||
});
|
||||
|
||||
test("an explicit provider balance error takes precedence over HTTP 429", () => {
|
||||
const error = Object.assign(new Error("余额不足，请点击充值页面续费（HTTP 429）"), { status: 429 });
|
||||
expect(isProviderBalanceError(error)).toBe(true);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user