diff --git a/lib/ai/providers/vip98k.ts b/lib/ai/providers/vip98k.ts index b123fb6ab..d3e2d88cd 100644 --- a/lib/ai/providers/vip98k.ts +++ b/lib/ai/providers/vip98k.ts @@ -1024,6 +1024,77 @@ function getConfiguredPollSeconds(config: Vip98kModelRuntimeConfig, key: string, return Math.min(Math.max(value, 1), 3600); } +function isSynchronousImagePath(path: string) { + return /^\/v1\/images\/(?:generations|edits)\/?$/.test(path); +} + +async function recoverImageSubmission(params: { + apiKey: string; + baseURL: string; + modelId: string; + path: string; + idempotencyKey: string; + pollIntervalMs: number; + maxPollMs: number; +}) { + if (!isSynchronousImagePath(params.path)) return null; + const startedAt = Date.now(); + const deadline = startedAt + params.maxPollMs; + logger.info("recovering original image request", { modelId: params.modelId }); + + while (Date.now() < deadline) { + try { + const lookup = await fetch(`${params.baseURL}/api/external/requests/by-idempotency-key`, { + cache: "no-store", + headers: { + Authorization: `Bearer ${params.apiKey}`, + "Idempotency-Key": params.idempotencyKey, + }, + signal: AbortSignal.timeout(Math.min(60_000, deadline - Date.now())), + }); + if ([401, 403, 404, 410].includes(lookup.status)) return null; + if (lookup.ok) { + const state = toRecord(await parseJsonResponse(lookup)); + if (!state || !["processing", "succeeded"].includes(String(state.status))) return null; + if (state.status === "succeeded") { + if (state.result_available !== true || !state.request_id) return null; + const result = await fetch( + `${params.baseURL}/api/external/requests/${encodeURIComponent(String(state.request_id))}/result`, + { + cache: "no-store", + headers: { Authorization: `Bearer ${params.apiKey}` }, + signal: AbortSignal.timeout(Math.max(1, Math.min(60_000, deadline - Date.now()))), + }, + ); + if ([401, 403, 404, 410].includes(result.status)) return null; + if (result.ok) { + const payload = await parseJsonResponse(result); + const outputs = collectOutputUrls(payload); + if (!outputs.length) { + throw new ProviderTaskUncertainError("原生图请求已完成,但未取回图片结果,请勿重复生成"); + } + logger.info("original image request recovered", { + modelId: params.modelId, + providerRequestId: state.request_id, + outputCount: outputs.length, + durationMs: Date.now() - startedAt, + }); + return { payload, taskId: extractTaskId(payload), resultPath: extractResultPath(payload), outputs }; + } + } + } + } catch (error) { + if (error instanceof ProviderTaskUncertainError) throw error; + } + await delay(Math.max(0, Math.min(params.pollIntervalMs, deadline - Date.now()))); + } + logger.warn("original image request recovery timed out", { + modelId: params.modelId, + durationMs: Date.now() - startedAt, + }); + return null; +} + async function submitTask(params: { apiKey: string; baseURL?: string | null; @@ -1036,6 +1107,8 @@ async function submitTask(params: { submissionTimeoutMs: number; maxSubmissionAttempts: number; submissionRetryBaseDelayMs: number; + pollIntervalMs: number; + maxPollMs: number; }) { const baseURL = normalizeVip98kBaseUrl(params.baseURL); const path = String(params.path || "/api/external/task/generate").trim(); @@ -1099,7 +1172,9 @@ async function submitTask(params: { signal: AbortSignal.timeout(params.submissionTimeoutMs), }); } catch (error) { - if (attempt >= maxAttempts) { + const recovered = await recoverImageSubmission({ ...params, baseURL, path, idempotencyKey }); + if (recovered) return recovered; + if (isSynchronousImagePath(path) || attempt >= maxAttempts) { logger.error("provider submission transport failed", { modelId: params.modelId, path, @@ -1145,6 +1220,14 @@ async function submitTask(params: { }; } + if ([502, 504, 524].includes(response.status)) { + const recovered = await recoverImageSubmission({ ...params, baseURL, path, idempotencyKey }); + if (recovered) return recovered; + if (isSynchronousImagePath(path)) { + throw new ProviderTaskUncertainError(`供应商提交结果未确认,可能仍在处理,请勿重复生成:${getUserFacingImageGenerationErrorMessage({ message: extractErrorMessage(payload), status: response.status })} (HTTP ${response.status})`); + } + } + const errorMessage = extractErrorMessage(payload); const stagedMediaRetryDelayMs = getVip98kStagedMediaRetryDelay({ status: response.status, @@ -1311,6 +1394,12 @@ export async function submitVip98kExternalTask(params: { ), maxSubmissionAttempts: params.durable ? 1 : runtimeConfig.maxSubmissionAttempts, submissionRetryBaseDelayMs: runtimeConfig.submissionRetryBaseDelayMs, + pollIntervalMs: getConfiguredPollSeconds( + taskParams.modelConfig, "pollIntervalSeconds", DEFAULT_POLL_INTERVAL_SECONDS, + ) * 1000, + maxPollMs: getConfiguredPollSeconds( + taskParams.modelConfig, "maxPollSeconds", DEFAULT_MAX_POLL_SECONDS, + ) * 1000, }); const resultPath = submitted.resultPath || invocation?.resultPath || ""; return { diff --git a/lib/ai/runtime/model-health.ts b/lib/ai/runtime/model-health.ts index b1403b5a0..ebb71a8dd 100644 --- a/lib/ai/runtime/model-health.ts +++ b/lib/ai/runtime/model-health.ts @@ -1,4 +1,5 @@ import "server-only"; +import { ProviderTaskUncertainError } from "@/lib/video-canvas/video-task-lifecycle"; import { isProviderAuthenticationFailure, isCredentialFailureBan } from "./authentication-failover"; import { and, desc, eq, inArray, sql } from "drizzle-orm"; @@ -343,7 +344,7 @@ export function classifyAiModelHealthFailure(error: unknown): { } export function shouldFailoverAiModelError(error: unknown) { - if (isGatewayResponseTimeout(error)) return false; + if (error instanceof ProviderTaskUncertainError || isGatewayResponseTimeout(error)) return false; const classification = classifyAiModelHealthFailure(error); const normalized = getErrorMessage(error).toLowerCase(); const imageProviderTaskFailed = diff --git a/tests/routes/image-provider-error-classification.test.ts b/tests/routes/image-provider-error-classification.test.ts index fca519434..5e3d93e71 100644 --- a/tests/routes/image-provider-error-classification.test.ts +++ b/tests/routes/image-provider-error-classification.test.ts @@ -45,13 +45,14 @@ test("HTTP 524 does not start another model while the original result is unconfi expect(health.shouldFailoverAiModelError(new Error("Service unavailable (HTTP 503)"))).toBe(true); }); -test("the provider submits once and keeps an HTTP 524 response unconfirmed", async () => { +test("the provider submits once when an HTTP 524 request cannot be located", async () => { let submissions = 0; const provider = serverLoader({ "@/lib/db/queries": { getSystemSetting: async () => null }, "@/lib/storage/image-archive": {}, "@/lib/storage/service": {}, - fetch: async () => { + fetch: async (_url: string, init: RequestInit) => { + if (init?.method !== "POST") return Response.json({}, { status: 404 }); submissions++; return new Response(gatewayHtml, { status: 524, headers: { "content-type": "text/html" } }); }, @@ -61,6 +62,136 @@ test("the provider submits once and keeps an HTTP 524 response unconfirmed", asy expect(submissions).toBe(1); }); +function imageRecoveryHarness( + handler: (path: string, method: string, call: number) => Response | Promise, + maxPollSeconds = 3, +) { + let now = 1_000; + const calls: { path: string; method: string; headers: Headers }[] = []; + const archived: string[] = []; + const load = serverLoader({ + "@/lib/db": { db: {} }, + "@/lib/db/queries": { getSystemSetting: async () => null }, + "@/lib/storage/image-archive": { createDeferredUploadedImage: async (url: string) => { + archived.push(url); + return { thumbnail: url, original: url }; + } }, + "@/lib/storage/service": {}, + Date: class extends Date { static now() { return now; } }, + setTimeout: (fn: () => void, ms: number) => { now += ms; return setTimeout(fn, 0); }, + fetch: async (url: string, init: RequestInit) => { + const path = new URL(url).pathname; + const method = init?.method || "GET"; + calls.push({ path, method, headers: new Headers(init?.headers) }); + return handler(path, method, calls.length); + }, + }); + const provider = load("lib/ai/providers/vip98k.ts"); + return { + calls, load, archived, + generate: () => provider.createVip98k({ apiKey: "test-only-key", baseURL: "https://provider.test" }) + .chat("gpt-image-2.5", { capabilities: { vip98k: { pollIntervalSeconds: 1, maxPollSeconds } } }) + .doGenerate({ messages: [{ role: "user", content: "A desk" }] }), + run: () => provider.executeVip98kExternalTask({ + apiKey: "test-only-key", baseURL: "https://provider.test", modelId: "gpt-image-2.5", + kind: "image", prompt: "A desk", imageUrls: [], + modelConfig: { capabilities: { vip98k: { pollIntervalSeconds: 1, maxPollSeconds } } }, + }), + }; +} + +const recoveredImage = { created: 1, data: [{ url: "https://image.example/original.png" }] }; +const originalCompleted = { request_id: "original-request", status: "succeeded", result_available: true }; + +test("an edge timeout waits for and delivers the original image without another POST", async () => { + let lookups = 0; + const h = imageRecoveryHarness((path, method) => { + if (method === "POST") return new Response(gatewayHtml, { status: 524 }); + if (path.endsWith("by-idempotency-key")) { + return Response.json(++lookups === 1 ? { status: "processing" } : originalCompleted); + } + return Response.json(recoveredImage); + }); + expect((await h.generate()).text).toContain(recoveredImage.data[0].url); + expect(h.archived).toEqual([recoveredImage.data[0].url]); + expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1); + expect(lookups).toBe(2); + const originalKey = h.calls[0].headers.get("Idempotency-Key"); + expect(originalKey).toBeTruthy(); + for (const call of h.calls.filter(c => c.path.endsWith("by-idempotency-key"))) { + expect(call.headers.get("Idempotency-Key")).toBe(originalKey); + expect(call.headers.get("Authorization")).toBe("Bearer test-only-key"); + } + expect(h.calls.at(-1)?.path).toBe("/api/external/requests/original-request/result"); +}); + +test("a lost POST response recovers by its original key without resubmitting", async () => { + const h = imageRecoveryHarness((path, method) => { + if (method === "POST") throw new TypeError("fetch failed"); + return Response.json(path.endsWith("by-idempotency-key") ? originalCompleted : recoveredImage); + }); + expect((await h.run()).outputUrls).toHaveLength(1); + expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1); +}); + +test("temporary lookup and result-read failures retry only GET requests", async () => { + let lookups = 0; + let reads = 0; + const h = imageRecoveryHarness((path, method) => { + if (method === "POST") return new Response("Gateway timeout", { status: 504 }); + if (path.endsWith("by-idempotency-key")) { + return ++lookups === 1 ? new Response("Unavailable", { status: 503 }) : Response.json(originalCompleted); + } + return ++reads === 1 ? Response.json({}, { status: 409 }) : Response.json(recoveredImage); + }, 5); + expect((await h.run()).outputUrls).toHaveLength(1); + expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1); + expect(reads).toBe(2); +}); + +test("missing, unconfirmed and unretained originals never create a replacement", async () => { + for (const state of [null, { status: "failed" }, { status: "unresolved" }, { ...originalCompleted, result_available: false }]) { + const h = imageRecoveryHarness((_path, method) => method === "POST" + ? new Response(gatewayHtml, { status: 524 }) + : Response.json(state || {}, { status: state ? 200 : 404 })); + await expect(h.run()).rejects.toThrow(/结果未确认.*HTTP 524/); + expect(h.calls.map(c => c.method)).toEqual(["POST", "GET"]); + } +}); + +test("a still-processing original stops at the polling deadline without another POST", async () => { + const h = imageRecoveryHarness((_path, method) => method === "POST" + ? new Response(gatewayHtml, { status: 524 }) : Response.json({ status: "processing" }), 2); + await expect(h.run()).rejects.toThrow(/结果未确认.*HTTP 524/); + expect(h.calls.map(c => c.method)).toEqual(["POST", "GET", "GET"]); +}); + +test("HTTP 502 and 504 without a recoverable original do not repeat the POST", async () => { + for (const status of [502, 504]) { + const h = imageRecoveryHarness((_path, method) => method === "POST" + ? new Response("Gateway timeout", { status }) : Response.json({}, { status: 404 })); + await expect(h.run()).rejects.toThrow(/结果未确认/); + expect(h.calls.map(c => c.method)).toEqual(["POST", "GET"]); + } +}); + +test("a completed original with no image stays unconfirmed and cannot fail over", async () => { + const h = imageRecoveryHarness((path, method) => { + if (method === "POST") return new Response(gatewayHtml, { status: 524 }); + return Response.json(path.endsWith("by-idempotency-key") ? originalCompleted : { data: [] }); + }); + const error = await h.run().then(() => null, (e: Error) => e); + expect(error).toBeTruthy(); + expect(h.load("lib/ai/runtime/model-health.ts").shouldFailoverAiModelError(error)).toBe(false); + expect(h.calls.filter(c => c.method === "POST")).toHaveLength(1); +}); + +test("ordinary image success uses the original response directly", async () => { + const h = imageRecoveryHarness(() => Response.json(recoveredImage)); + expect((await h.run()).outputUrls).toHaveLength(1); + expect(h.calls.map(c => c.method)).toEqual(["POST"]); +}); + test("an explicit provider balance error takes precedence over HTTP 429", () => { const error = Object.assign(new Error("余额不足,请点击充值页面续费(HTTP 429)"), { status: 429 }); expect(isProviderBalanceError(error)).toBe(true);