恢复同步生图在网关超时后的原请求结果
同步生图超过边缘等待期限时,生成仍可能在源站继续执行。实际记录中,Cloudflare 在 125.006 秒断开,原请求在约 172 秒完成;应用已经显示 524,成功图片没有送达。原结果 GET 和原图片只读验证均返回 200。 同步图片生成/编辑收到 502、504、524 或提交响应丢失后,现在使用原 Key 和原 Idempotency-Key 查询既有请求。处理中继续查询,成功后 GET 同一请求的保留结果,进入原有图片归档与展示流程。查询复用已有轮询间隔和期限;缺失、失败、未确认、过期或未保留结果不会变成新的生成提交。普通成功响应保持直接返回,原生任务沿用现有协议。ProviderTaskUncertainError 也不触发自动模型切换。 验证:645 项辅助测试通过、1 项原有跳过,包含 524 后处理中到成功取回并进入展示流程、提交响应丢失、查询暂时失败、结果 GET 暂时失败、过期/缺失结果、轮询期限及一次 POST。TypeScript、改动文件 lint、日志脱敏、凭据、测试与制品检查通过。生产构建与上线验收另行记录。 已知问题:全量 lint 有 1004 项历史诊断。Cloudflare 本身的读取期限保持;本次通过取回已受理请求的原结果完成交付,不提高边缘期限。
This commit is contained in:
parent
b06fdbb38b
commit
ac7928fe2d
@ -1024,6 +1024,77 @@ function getConfiguredPollSeconds(config: Vip98kModelRuntimeConfig, key: string,
|
|||||||
return Math.min(Math.max(value, 1), 3600);
|
return Math.min(Math.max(value, 1), 3600);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isSynchronousImagePath(path: string) {
|
||||||
|
return /^\/v1\/images\/(?:generations|edits)\/?$/.test(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function recoverImageSubmission(params: {
|
||||||
|
apiKey: string;
|
||||||
|
baseURL: string;
|
||||||
|
modelId: string;
|
||||||
|
path: string;
|
||||||
|
idempotencyKey: string;
|
||||||
|
pollIntervalMs: number;
|
||||||
|
maxPollMs: number;
|
||||||
|
}) {
|
||||||
|
if (!isSynchronousImagePath(params.path)) return null;
|
||||||
|
const startedAt = Date.now();
|
||||||
|
const deadline = startedAt + params.maxPollMs;
|
||||||
|
logger.info("recovering original image request", { modelId: params.modelId });
|
||||||
|
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
try {
|
||||||
|
const lookup = await fetch(`${params.baseURL}/api/external/requests/by-idempotency-key`, {
|
||||||
|
cache: "no-store",
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${params.apiKey}`,
|
||||||
|
"Idempotency-Key": params.idempotencyKey,
|
||||||
|
},
|
||||||
|
signal: AbortSignal.timeout(Math.min(60_000, deadline - Date.now())),
|
||||||
|
});
|
||||||
|
if ([401, 403, 404, 410].includes(lookup.status)) return null;
|
||||||
|
if (lookup.ok) {
|
||||||
|
const state = toRecord(await parseJsonResponse(lookup));
|
||||||
|
if (!state || !["processing", "succeeded"].includes(String(state.status))) return null;
|
||||||
|
if (state.status === "succeeded") {
|
||||||
|
if (state.result_available !== true || !state.request_id) return null;
|
||||||
|
const result = await fetch(
|
||||||
|
`${params.baseURL}/api/external/requests/${encodeURIComponent(String(state.request_id))}/result`,
|
||||||
|
{
|
||||||
|
cache: "no-store",
|
||||||
|
headers: { Authorization: `Bearer ${params.apiKey}` },
|
||||||
|
signal: AbortSignal.timeout(Math.max(1, Math.min(60_000, deadline - Date.now()))),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
if ([401, 403, 404, 410].includes(result.status)) return null;
|
||||||
|
if (result.ok) {
|
||||||
|
const payload = await parseJsonResponse(result);
|
||||||
|
const outputs = collectOutputUrls(payload);
|
||||||
|
if (!outputs.length) {
|
||||||
|
throw new ProviderTaskUncertainError("原生图请求已完成,但未取回图片结果,请勿重复生成");
|
||||||
|
}
|
||||||
|
logger.info("original image request recovered", {
|
||||||
|
modelId: params.modelId,
|
||||||
|
providerRequestId: state.request_id,
|
||||||
|
outputCount: outputs.length,
|
||||||
|
durationMs: Date.now() - startedAt,
|
||||||
|
});
|
||||||
|
return { payload, taskId: extractTaskId(payload), resultPath: extractResultPath(payload), outputs };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof ProviderTaskUncertainError) throw error;
|
||||||
|
}
|
||||||
|
await delay(Math.max(0, Math.min(params.pollIntervalMs, deadline - Date.now())));
|
||||||
|
}
|
||||||
|
logger.warn("original image request recovery timed out", {
|
||||||
|
modelId: params.modelId,
|
||||||
|
durationMs: Date.now() - startedAt,
|
||||||
|
});
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
async function submitTask(params: {
|
async function submitTask(params: {
|
||||||
apiKey: string;
|
apiKey: string;
|
||||||
baseURL?: string | null;
|
baseURL?: string | null;
|
||||||
@ -1036,6 +1107,8 @@ async function submitTask(params: {
|
|||||||
submissionTimeoutMs: number;
|
submissionTimeoutMs: number;
|
||||||
maxSubmissionAttempts: number;
|
maxSubmissionAttempts: number;
|
||||||
submissionRetryBaseDelayMs: number;
|
submissionRetryBaseDelayMs: number;
|
||||||
|
pollIntervalMs: number;
|
||||||
|
maxPollMs: number;
|
||||||
}) {
|
}) {
|
||||||
const baseURL = normalizeVip98kBaseUrl(params.baseURL);
|
const baseURL = normalizeVip98kBaseUrl(params.baseURL);
|
||||||
const path = String(params.path || "/api/external/task/generate").trim();
|
const path = String(params.path || "/api/external/task/generate").trim();
|
||||||
@ -1099,7 +1172,9 @@ async function submitTask(params: {
|
|||||||
signal: AbortSignal.timeout(params.submissionTimeoutMs),
|
signal: AbortSignal.timeout(params.submissionTimeoutMs),
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (attempt >= maxAttempts) {
|
const recovered = await recoverImageSubmission({ ...params, baseURL, path, idempotencyKey });
|
||||||
|
if (recovered) return recovered;
|
||||||
|
if (isSynchronousImagePath(path) || attempt >= maxAttempts) {
|
||||||
logger.error("provider submission transport failed", {
|
logger.error("provider submission transport failed", {
|
||||||
modelId: params.modelId,
|
modelId: params.modelId,
|
||||||
path,
|
path,
|
||||||
@ -1145,6 +1220,14 @@ async function submitTask(params: {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ([502, 504, 524].includes(response.status)) {
|
||||||
|
const recovered = await recoverImageSubmission({ ...params, baseURL, path, idempotencyKey });
|
||||||
|
if (recovered) return recovered;
|
||||||
|
if (isSynchronousImagePath(path)) {
|
||||||
|
throw new ProviderTaskUncertainError(`供应商提交结果未确认,可能仍在处理,请勿重复生成:${getUserFacingImageGenerationErrorMessage({ message: extractErrorMessage(payload), status: response.status })} (HTTP ${response.status})`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const errorMessage = extractErrorMessage(payload);
|
const errorMessage = extractErrorMessage(payload);
|
||||||
const stagedMediaRetryDelayMs = getVip98kStagedMediaRetryDelay({
|
const stagedMediaRetryDelayMs = getVip98kStagedMediaRetryDelay({
|
||||||
status: response.status,
|
status: response.status,
|
||||||
@ -1311,6 +1394,12 @@ export async function submitVip98kExternalTask(params: {
|
|||||||
),
|
),
|
||||||
maxSubmissionAttempts: params.durable ? 1 : runtimeConfig.maxSubmissionAttempts,
|
maxSubmissionAttempts: params.durable ? 1 : runtimeConfig.maxSubmissionAttempts,
|
||||||
submissionRetryBaseDelayMs: runtimeConfig.submissionRetryBaseDelayMs,
|
submissionRetryBaseDelayMs: runtimeConfig.submissionRetryBaseDelayMs,
|
||||||
|
pollIntervalMs: getConfiguredPollSeconds(
|
||||||
|
taskParams.modelConfig, "pollIntervalSeconds", DEFAULT_POLL_INTERVAL_SECONDS,
|
||||||
|
) * 1000,
|
||||||
|
maxPollMs: getConfiguredPollSeconds(
|
||||||
|
taskParams.modelConfig, "maxPollSeconds", DEFAULT_MAX_POLL_SECONDS,
|
||||||
|
) * 1000,
|
||||||
});
|
});
|
||||||
const resultPath = submitted.resultPath || invocation?.resultPath || "";
|
const resultPath = submitted.resultPath || invocation?.resultPath || "";
|
||||||
return {
|
return {
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
import "server-only";
|
import "server-only";
|
||||||
|
import { ProviderTaskUncertainError } from "@/lib/video-canvas/video-task-lifecycle";
|
||||||
import { isProviderAuthenticationFailure, isCredentialFailureBan } from "./authentication-failover";
|
import { isProviderAuthenticationFailure, isCredentialFailureBan } from "./authentication-failover";
|
||||||
|
|
||||||
import { and, desc, eq, inArray, sql } from "drizzle-orm";
|
import { and, desc, eq, inArray, sql } from "drizzle-orm";
|
||||||
@ -343,7 +344,7 @@ export function classifyAiModelHealthFailure(error: unknown): {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function shouldFailoverAiModelError(error: unknown) {
|
export function shouldFailoverAiModelError(error: unknown) {
|
||||||
if (isGatewayResponseTimeout(error)) return false;
|
if (error instanceof ProviderTaskUncertainError || isGatewayResponseTimeout(error)) return false;
|
||||||
const classification = classifyAiModelHealthFailure(error);
|
const classification = classifyAiModelHealthFailure(error);
|
||||||
const normalized = getErrorMessage(error).toLowerCase();
|
const normalized = getErrorMessage(error).toLowerCase();
|
||||||
const imageProviderTaskFailed =
|
const imageProviderTaskFailed =
|
||||||
|
|||||||
@ -45,13 +45,14 @@ test("HTTP 524 does not start another model while the original result is unconfi
|
|||||||
expect(health.shouldFailoverAiModelError(new Error("Service unavailable (HTTP 503)"))).toBe(true);
|
expect(health.shouldFailoverAiModelError(new Error("Service unavailable (HTTP 503)"))).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("the provider submits once and keeps an HTTP 524 response unconfirmed", async () => {
|
test("the provider submits once when an HTTP 524 request cannot be located", async () => {
|
||||||
let submissions = 0;
|
let submissions = 0;
|
||||||
const provider = serverLoader({
|
const provider = serverLoader({
|
||||||
"@/lib/db/queries": { getSystemSetting: async () => null },
|
"@/lib/db/queries": { getSystemSetting: async () => null },
|
||||||
"@/lib/storage/image-archive": {},
|
"@/lib/storage/image-archive": {},
|
||||||
"@/lib/storage/service": {},
|
"@/lib/storage/service": {},
|
||||||
fetch: async () => {
|
fetch: async (_url: string, init: RequestInit) => {
|
||||||
|
if (init?.method !== "POST") return Response.json({}, { status: 404 });
|
||||||
submissions++;
|
submissions++;
|
||||||
return new Response(gatewayHtml, { status: 524, headers: { "content-type": "text/html" } });
|
return new Response(gatewayHtml, { status: 524, headers: { "content-type": "text/html" } });
|
||||||
},
|
},
|
||||||
@ -61,6 +62,136 @@ test("the provider submits once and keeps an HTTP 524 response unconfirmed", asy
|
|||||||
expect(submissions).toBe(1);
|
expect(submissions).toBe(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
function imageRecoveryHarness(
|
||||||
|
handler: (path: string, method: string, call: number) => Response | Promise<Response>,
|
||||||
|
maxPollSeconds = 3,
|
||||||
|
) {
|
||||||
|
let now = 1_000;
|
||||||
|
const calls: { path: string; method: string; headers: Headers }[] = [];
|
||||||
|
const archived: string[] = [];
|
||||||
|
const load = serverLoader({
|
||||||
|
"@/lib/db": { db: {} },
|
||||||
|
"@/lib/db/queries": { getSystemSetting: async () => null },
|
||||||
|
"@/lib/storage/image-archive": { createDeferredUploadedImage: async (url: string) => {
|
||||||
|
archived.push(url);
|
||||||
|
return { thumbnail: url, original: url };
|
||||||
|
} },
|
||||||
|
"@/lib/storage/service": {},
|
||||||
|
Date: class extends Date { static now() { return now; } },
|
||||||
|
setTimeout: (fn: () => void, ms: number) => { now += ms; return setTimeout(fn, 0); },
|
||||||
|
fetch: async (url: string, init: RequestInit) => {
|
||||||
|
const path = new URL(url).pathname;
|
||||||
|
const method = init?.method || "GET";
|
||||||
|
calls.push({ path, method, headers: new Headers(init?.headers) });
|
||||||
|
return handler(path, method, calls.length);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const provider = load("lib/ai/providers/vip98k.ts");
|
||||||
|
return {
|
||||||
|
calls, load, archived,
|
||||||
|
generate: () => provider.createVip98k({ apiKey: "test-only-key", baseURL: "https://provider.test" })
|
||||||
|
.chat("gpt-image-2.5", { capabilities: { vip98k: { pollIntervalSeconds: 1, maxPollSeconds } } })
|
||||||
|
.doGenerate({ messages: [{ role: "user", content: "A desk" }] }),
|
||||||
|
run: () => provider.executeVip98kExternalTask({
|
||||||
|
apiKey: "test-only-key", baseURL: "https://provider.test", modelId: "gpt-image-2.5",
|
||||||
|
kind: "image", prompt: "A desk", imageUrls: [],
|
||||||
|
modelConfig: { capabilities: { vip98k: { pollIntervalSeconds: 1, maxPollSeconds } } },
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const recoveredImage = { created: 1, data: [{ url: "https://image.example/original.png" }] };
|
||||||
|
const originalCompleted = { request_id: "original-request", status: "succeeded", result_available: true };
|
||||||
|
|
||||||
|
test("an edge timeout waits for and delivers the original image without another POST", async () => {
|
||||||
|
let lookups = 0;
|
||||||
|
const h = imageRecoveryHarness((path, method) => {
|
||||||
|
if (method === "POST") return new Response(gatewayHtml, { status: 524 });
|
||||||
|
if (path.endsWith("by-idempotency-key")) {
|
||||||
|
return Response.json(++lookups === 1 ? { status: "processing" } : originalCompleted);
|
||||||
|
}
|
||||||
|
return Response.json(recoveredImage);
|
||||||
|
});
|
||||||
|
expect((await h.generate()).text).toContain(recoveredImage.data[0].url);
|
||||||
|
expect(h.archived).toEqual([recoveredImage.data[0].url]);
|
||||||
|
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
|
||||||
|
expect(lookups).toBe(2);
|
||||||
|
const originalKey = h.calls[0].headers.get("Idempotency-Key");
|
||||||
|
expect(originalKey).toBeTruthy();
|
||||||
|
for (const call of h.calls.filter(c => c.path.endsWith("by-idempotency-key"))) {
|
||||||
|
expect(call.headers.get("Idempotency-Key")).toBe(originalKey);
|
||||||
|
expect(call.headers.get("Authorization")).toBe("Bearer test-only-key");
|
||||||
|
}
|
||||||
|
expect(h.calls.at(-1)?.path).toBe("/api/external/requests/original-request/result");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a lost POST response recovers by its original key without resubmitting", async () => {
|
||||||
|
const h = imageRecoveryHarness((path, method) => {
|
||||||
|
if (method === "POST") throw new TypeError("fetch failed");
|
||||||
|
return Response.json(path.endsWith("by-idempotency-key") ? originalCompleted : recoveredImage);
|
||||||
|
});
|
||||||
|
expect((await h.run()).outputUrls).toHaveLength(1);
|
||||||
|
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("temporary lookup and result-read failures retry only GET requests", async () => {
|
||||||
|
let lookups = 0;
|
||||||
|
let reads = 0;
|
||||||
|
const h = imageRecoveryHarness((path, method) => {
|
||||||
|
if (method === "POST") return new Response("Gateway timeout", { status: 504 });
|
||||||
|
if (path.endsWith("by-idempotency-key")) {
|
||||||
|
return ++lookups === 1 ? new Response("Unavailable", { status: 503 }) : Response.json(originalCompleted);
|
||||||
|
}
|
||||||
|
return ++reads === 1 ? Response.json({}, { status: 409 }) : Response.json(recoveredImage);
|
||||||
|
}, 5);
|
||||||
|
expect((await h.run()).outputUrls).toHaveLength(1);
|
||||||
|
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
|
||||||
|
expect(reads).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("missing, unconfirmed and unretained originals never create a replacement", async () => {
|
||||||
|
for (const state of [null, { status: "failed" }, { status: "unresolved" }, { ...originalCompleted, result_available: false }]) {
|
||||||
|
const h = imageRecoveryHarness((_path, method) => method === "POST"
|
||||||
|
? new Response(gatewayHtml, { status: 524 })
|
||||||
|
: Response.json(state || {}, { status: state ? 200 : 404 }));
|
||||||
|
await expect(h.run()).rejects.toThrow(/结果未确认.*HTTP 524/);
|
||||||
|
expect(h.calls.map(c => c.method)).toEqual(["POST", "GET"]);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a still-processing original stops at the polling deadline without another POST", async () => {
|
||||||
|
const h = imageRecoveryHarness((_path, method) => method === "POST"
|
||||||
|
? new Response(gatewayHtml, { status: 524 }) : Response.json({ status: "processing" }), 2);
|
||||||
|
await expect(h.run()).rejects.toThrow(/结果未确认.*HTTP 524/);
|
||||||
|
expect(h.calls.map(c => c.method)).toEqual(["POST", "GET", "GET"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("HTTP 502 and 504 without a recoverable original do not repeat the POST", async () => {
|
||||||
|
for (const status of [502, 504]) {
|
||||||
|
const h = imageRecoveryHarness((_path, method) => method === "POST"
|
||||||
|
? new Response("Gateway timeout", { status }) : Response.json({}, { status: 404 }));
|
||||||
|
await expect(h.run()).rejects.toThrow(/结果未确认/);
|
||||||
|
expect(h.calls.map(c => c.method)).toEqual(["POST", "GET"]);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a completed original with no image stays unconfirmed and cannot fail over", async () => {
|
||||||
|
const h = imageRecoveryHarness((path, method) => {
|
||||||
|
if (method === "POST") return new Response(gatewayHtml, { status: 524 });
|
||||||
|
return Response.json(path.endsWith("by-idempotency-key") ? originalCompleted : { data: [] });
|
||||||
|
});
|
||||||
|
const error = await h.run().then(() => null, (e: Error) => e);
|
||||||
|
expect(error).toBeTruthy();
|
||||||
|
expect(h.load("lib/ai/runtime/model-health.ts").shouldFailoverAiModelError(error)).toBe(false);
|
||||||
|
expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("ordinary image success uses the original response directly", async () => {
|
||||||
|
const h = imageRecoveryHarness(() => Response.json(recoveredImage));
|
||||||
|
expect((await h.run()).outputUrls).toHaveLength(1);
|
||||||
|
expect(h.calls.map(c => c.method)).toEqual(["POST"]);
|
||||||
|
});
|
||||||
|
|
||||||
test("an explicit provider balance error takes precedence over HTTP 429", () => {
|
test("an explicit provider balance error takes precedence over HTTP 429", () => {
|
||||||
const error = Object.assign(new Error("余额不足,请点击充值页面续费(HTTP 429)"), { status: 429 });
|
const error = Object.assign(new Error("余额不足,请点击充值页面续费(HTTP 429)"), { status: 429 });
|
||||||
expect(isProviderBalanceError(error)).toBe(true);
|
expect(isProviderBalanceError(error)).toBe(true);
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user