From e5dc650443e1a4372b0893156d298fcd16e2e6cb Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 25 Aug 2026 17:00:38 +0800 Subject: [PATCH] fix(ai): fail over failed VIP98K image tasks --- app/(chat)/api/chat/_lib/image-failover.ts | 18 ++++- lib/ai/providers/vip98k.ts | 85 +++++++++++++++++++--- lib/ai/runtime/model-health.ts | 53 ++++++++++++++ 3 files changed, 143 insertions(+), 13 deletions(-) diff --git a/app/(chat)/api/chat/_lib/image-failover.ts b/app/(chat)/api/chat/_lib/image-failover.ts index 8d1dc9e92..81d7a7cfc 100644 --- a/app/(chat)/api/chat/_lib/image-failover.ts +++ b/app/(chat)/api/chat/_lib/image-failover.ts @@ -57,7 +57,21 @@ export function resolveImageModelRuntimeId( export function normalizeFailoverErrorMessage(error: unknown): string { if (!error) return ""; if (typeof error === "string") return error; - if (error instanceof Error) return error.message || String(error); + if (error instanceof Error) { + const record = error as Error & { + status?: unknown; + statusCode?: unknown; + httpStatus?: unknown; + response?: { status?: unknown }; + }; + const status = + record.status ?? + record.statusCode ?? + record.httpStatus ?? + record.response?.status; + const message = error.message || String(error); + return status ? `[HTTP ${String(status)}] ${message}` : message; + } try { return JSON.stringify(error); } catch { @@ -76,6 +90,8 @@ export function shouldCooldownImageFallbackModel(errorMessage: string): boolean msg.includes("api key is missing") || msg.includes("invalid api key") || msg.includes("unauthorized") || + /(?:^|\D)401(?:\D|$)/.test(msg) || + msg.includes("vip98k task failed") || msg.includes("cannot convert argument to a bytestring") || msg.includes("model maintenance") || msg.includes("maintenance") || diff --git a/lib/ai/providers/vip98k.ts b/lib/ai/providers/vip98k.ts index a7d81a1b6..1f091b83e 100644 --- a/lib/ai/providers/vip98k.ts +++ b/lib/ai/providers/vip98k.ts @@ -147,20 +147,68 @@ async function parseJsonResponse(response: Response) { } } +const ERROR_MESSAGE_FIELDS = [ + "failureReason", + "failure_reason", + "errorMessage", + "error_message", + "errorDescription", + "error_description", + "reason", + "detail", + "description", + "message", + "error", +] as const; + +const ERROR_CONTAINER_FIELDS = [ + "data", + "result", + "task", + "output", + "response", +] as const; + +function findNestedErrorMessage(value: unknown, depth = 0): string { + if (depth > 4 || value == null) return ""; + if (typeof value === "string") return value.trim(); + if (Array.isArray(value)) { + for (const item of value) { + const message = findNestedErrorMessage(item, depth + 1); + if (message) return message; + } + return ""; + } + + const record = toRecord(value); + if (!record) return ""; + + for (const key of ERROR_MESSAGE_FIELDS) { + if (!(key in record)) continue; + const message = findNestedErrorMessage(record[key], depth + 1); + if (message) return message; + } + for (const key of ERROR_CONTAINER_FIELDS) { + if (!(key in record)) continue; + const message = findNestedErrorMessage(record[key], depth + 1); + if (message) return message; + } + return ""; +} + function extractErrorMessage(payload: unknown, fallback = "VIP98K request failed") { - if (typeof payload === "string" && payload.trim()) return payload.trim(); + return findNestedErrorMessage(payload) || fallback; +} + +function extractPayloadShape(payload: unknown) { const record = toRecord(payload); const data = toRecord(record?.data); - const error = toRecord(record?.error); - const value = - data?.message || - data?.errorMessage || - data?.error || - error?.message || - record?.message || - record?.error || - fallback; - return typeof value === "string" && value.trim() ? value.trim() : fallback; + const task = toRecord(data?.task) || toRecord(record?.task); + return { + payloadKeys: record ? Object.keys(record).sort() : [], + dataKeys: data ? Object.keys(data).sort() : [], + taskKeys: task ? Object.keys(task).sort() : [], + }; } function unwrapData(payload: unknown) { @@ -555,6 +603,7 @@ async function waitForTask(params: { apiKey: string; baseURL?: string | null; taskId: string; + modelId: string; resultPath?: string; modelConfig: Vip98kModelRuntimeConfig; }) { @@ -597,7 +646,18 @@ async function waitForTask(params: { return { payload, outputs }; } if (["failed", "error", "timeout", "canceled", "cancelled"].includes(status)) { - throw new Error(extractErrorMessage(payload, "VIP98K task failed")); + const providerMessage = extractErrorMessage(payload, ""); + logger.error("task failed", { + taskId: params.taskId, + modelId: params.modelId, + status, + providerMessage: providerMessage || null, + ...extractPayloadShape(payload), + }); + const detail = providerMessage ? `: ${providerMessage}` : ""; + throw new Error( + `VIP98K task failed${detail} (task_id: ${params.taskId}, model: ${params.modelId})`, + ); } } @@ -771,6 +831,7 @@ export async function executeVip98kExternalTask(params: { apiKey: params.apiKey, baseURL: params.baseURL, taskId: submitted.taskId, + modelId: params.modelId, resultPath: submitted.resultPath, modelConfig: params.modelConfig || {}, }); diff --git a/lib/ai/runtime/model-health.ts b/lib/ai/runtime/model-health.ts index f5c4a7087..809983673 100644 --- a/lib/ai/runtime/model-health.ts +++ b/lib/ai/runtime/model-health.ts @@ -50,6 +50,36 @@ function getErrorMessage(error: unknown) { return normalizeText(error); } +function parseHttpStatus(value: unknown) { + if (typeof value === "number" && Number.isInteger(value)) { + return value >= 100 && value <= 599 ? value : null; + } + + if (typeof value === "string" && /^\d{3}$/.test(value.trim())) { + const parsed = Number(value.trim()); + return parsed >= 100 && parsed <= 599 ? parsed : null; + } + + return null; +} + +function getErrorHttpStatus(error: unknown, depth = 0): number | null { + if (!error || typeof error !== "object" || depth > 2) return null; + + const record = error as Record; + for (const key of ["status", "statusCode", "httpStatus"]) { + const status = parseHttpStatus(record[key]); + if (status !== null) return status; + } + + for (const key of ["response", "cause"]) { + const nestedStatus = getErrorHttpStatus(record[key], depth + 1); + if (nestedStatus !== null) return nestedStatus; + } + + return null; +} + function isExplicitUserCancellation(normalized: string) { return ( normalized.includes("user abort") || @@ -81,6 +111,7 @@ export function classifyAiModelHealthFailure(error: unknown): { } { const message = getErrorMessage(error); const normalized = message.toLowerCase(); + const httpStatus = getErrorHttpStatus(error); if (!message) { return { @@ -100,6 +131,23 @@ export function classifyAiModelHealthFailure(error: unknown): { }; } + if ( + httpStatus === 401 || + /(?:^|\D)401(?:\D|$)/.test(normalized) || + normalized.includes("unauthorized") || + normalized.includes("invalid api key") || + normalized.includes("invalid_api_key") || + normalized.includes("authentication failed") || + normalized.includes("authentication_error") + ) { + return { + eventType: "unavailable", + severity: "critical", + counted: true, + reason: "provider_authentication_failed", + }; + } + if ( normalized.includes("生成结果未通过替换质检") || normalized.includes("output_moderation") || @@ -221,6 +269,7 @@ export function classifyAiModelHealthFailure(error: unknown): { normalized.includes("provider_error") || normalized.includes("provider request failed") || normalized.includes("provider task failed") || + normalized.includes("vip98k task failed") || normalized.includes("upstream error") || normalized.includes("upstream_error") || normalized.includes("upstream request failed") || @@ -283,6 +332,10 @@ export function shouldFailoverAiModelError(error: unknown) { normalized.includes("no task id") || normalized.includes("missing task id"); + if (classification.reason === "provider_authentication_failed") { + return true; + } + if (classification.reason === "user_cancelled" || classification.reason === "non_provider_failure") { return false; }