fix(ai): fail over failed VIP98K image tasks
This commit is contained in:
parent
86c8e4995d
commit
07a9a61743
@ -57,7 +57,21 @@ export function resolveImageModelRuntimeId(
|
||||
export function normalizeFailoverErrorMessage(error: unknown): string {
|
||||
if (!error) return "";
|
||||
if (typeof error === "string") return error;
|
||||
if (error instanceof Error) return error.message || String(error);
|
||||
if (error instanceof Error) {
|
||||
const record = error as Error & {
|
||||
status?: unknown;
|
||||
statusCode?: unknown;
|
||||
httpStatus?: unknown;
|
||||
response?: { status?: unknown };
|
||||
};
|
||||
const status =
|
||||
record.status ??
|
||||
record.statusCode ??
|
||||
record.httpStatus ??
|
||||
record.response?.status;
|
||||
const message = error.message || String(error);
|
||||
return status ? `[HTTP ${String(status)}] ${message}` : message;
|
||||
}
|
||||
try {
|
||||
return JSON.stringify(error);
|
||||
} catch {
|
||||
@ -76,6 +90,8 @@ export function shouldCooldownImageFallbackModel(errorMessage: string): boolean
|
||||
msg.includes("api key is missing") ||
|
||||
msg.includes("invalid api key") ||
|
||||
msg.includes("unauthorized") ||
|
||||
/(?:^|\D)401(?:\D|$)/.test(msg) ||
|
||||
msg.includes("vip98k task failed") ||
|
||||
msg.includes("cannot convert argument to a bytestring") ||
|
||||
msg.includes("model maintenance") ||
|
||||
msg.includes("maintenance") ||
|
||||
|
||||
@ -147,20 +147,68 @@ async function parseJsonResponse(response: Response) {
|
||||
}
|
||||
}
|
||||
|
||||
const ERROR_MESSAGE_FIELDS = [
|
||||
"failureReason",
|
||||
"failure_reason",
|
||||
"errorMessage",
|
||||
"error_message",
|
||||
"errorDescription",
|
||||
"error_description",
|
||||
"reason",
|
||||
"detail",
|
||||
"description",
|
||||
"message",
|
||||
"error",
|
||||
] as const;
|
||||
|
||||
const ERROR_CONTAINER_FIELDS = [
|
||||
"data",
|
||||
"result",
|
||||
"task",
|
||||
"output",
|
||||
"response",
|
||||
] as const;
|
||||
|
||||
function findNestedErrorMessage(value: unknown, depth = 0): string {
|
||||
if (depth > 4 || value == null) return "";
|
||||
if (typeof value === "string") return value.trim();
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) {
|
||||
const message = findNestedErrorMessage(item, depth + 1);
|
||||
if (message) return message;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
const record = toRecord(value);
|
||||
if (!record) return "";
|
||||
|
||||
for (const key of ERROR_MESSAGE_FIELDS) {
|
||||
if (!(key in record)) continue;
|
||||
const message = findNestedErrorMessage(record[key], depth + 1);
|
||||
if (message) return message;
|
||||
}
|
||||
for (const key of ERROR_CONTAINER_FIELDS) {
|
||||
if (!(key in record)) continue;
|
||||
const message = findNestedErrorMessage(record[key], depth + 1);
|
||||
if (message) return message;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function extractErrorMessage(payload: unknown, fallback = "VIP98K request failed") {
|
||||
if (typeof payload === "string" && payload.trim()) return payload.trim();
|
||||
return findNestedErrorMessage(payload) || fallback;
|
||||
}
|
||||
|
||||
function extractPayloadShape(payload: unknown) {
|
||||
const record = toRecord(payload);
|
||||
const data = toRecord(record?.data);
|
||||
const error = toRecord(record?.error);
|
||||
const value =
|
||||
data?.message ||
|
||||
data?.errorMessage ||
|
||||
data?.error ||
|
||||
error?.message ||
|
||||
record?.message ||
|
||||
record?.error ||
|
||||
fallback;
|
||||
return typeof value === "string" && value.trim() ? value.trim() : fallback;
|
||||
const task = toRecord(data?.task) || toRecord(record?.task);
|
||||
return {
|
||||
payloadKeys: record ? Object.keys(record).sort() : [],
|
||||
dataKeys: data ? Object.keys(data).sort() : [],
|
||||
taskKeys: task ? Object.keys(task).sort() : [],
|
||||
};
|
||||
}
|
||||
|
||||
function unwrapData(payload: unknown) {
|
||||
@ -555,6 +603,7 @@ async function waitForTask(params: {
|
||||
apiKey: string;
|
||||
baseURL?: string | null;
|
||||
taskId: string;
|
||||
modelId: string;
|
||||
resultPath?: string;
|
||||
modelConfig: Vip98kModelRuntimeConfig;
|
||||
}) {
|
||||
@ -597,7 +646,18 @@ async function waitForTask(params: {
|
||||
return { payload, outputs };
|
||||
}
|
||||
if (["failed", "error", "timeout", "canceled", "cancelled"].includes(status)) {
|
||||
throw new Error(extractErrorMessage(payload, "VIP98K task failed"));
|
||||
const providerMessage = extractErrorMessage(payload, "");
|
||||
logger.error("task failed", {
|
||||
taskId: params.taskId,
|
||||
modelId: params.modelId,
|
||||
status,
|
||||
providerMessage: providerMessage || null,
|
||||
...extractPayloadShape(payload),
|
||||
});
|
||||
const detail = providerMessage ? `: ${providerMessage}` : "";
|
||||
throw new Error(
|
||||
`VIP98K task failed${detail} (task_id: ${params.taskId}, model: ${params.modelId})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -771,6 +831,7 @@ export async function executeVip98kExternalTask(params: {
|
||||
apiKey: params.apiKey,
|
||||
baseURL: params.baseURL,
|
||||
taskId: submitted.taskId,
|
||||
modelId: params.modelId,
|
||||
resultPath: submitted.resultPath,
|
||||
modelConfig: params.modelConfig || {},
|
||||
});
|
||||
|
||||
@ -50,6 +50,36 @@ function getErrorMessage(error: unknown) {
|
||||
return normalizeText(error);
|
||||
}
|
||||
|
||||
function parseHttpStatus(value: unknown) {
|
||||
if (typeof value === "number" && Number.isInteger(value)) {
|
||||
return value >= 100 && value <= 599 ? value : null;
|
||||
}
|
||||
|
||||
if (typeof value === "string" && /^\d{3}$/.test(value.trim())) {
|
||||
const parsed = Number(value.trim());
|
||||
return parsed >= 100 && parsed <= 599 ? parsed : null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function getErrorHttpStatus(error: unknown, depth = 0): number | null {
|
||||
if (!error || typeof error !== "object" || depth > 2) return null;
|
||||
|
||||
const record = error as Record<string, unknown>;
|
||||
for (const key of ["status", "statusCode", "httpStatus"]) {
|
||||
const status = parseHttpStatus(record[key]);
|
||||
if (status !== null) return status;
|
||||
}
|
||||
|
||||
for (const key of ["response", "cause"]) {
|
||||
const nestedStatus = getErrorHttpStatus(record[key], depth + 1);
|
||||
if (nestedStatus !== null) return nestedStatus;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function isExplicitUserCancellation(normalized: string) {
|
||||
return (
|
||||
normalized.includes("user abort") ||
|
||||
@ -81,6 +111,7 @@ export function classifyAiModelHealthFailure(error: unknown): {
|
||||
} {
|
||||
const message = getErrorMessage(error);
|
||||
const normalized = message.toLowerCase();
|
||||
const httpStatus = getErrorHttpStatus(error);
|
||||
|
||||
if (!message) {
|
||||
return {
|
||||
@ -100,6 +131,23 @@ export function classifyAiModelHealthFailure(error: unknown): {
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
httpStatus === 401 ||
|
||||
/(?:^|\D)401(?:\D|$)/.test(normalized) ||
|
||||
normalized.includes("unauthorized") ||
|
||||
normalized.includes("invalid api key") ||
|
||||
normalized.includes("invalid_api_key") ||
|
||||
normalized.includes("authentication failed") ||
|
||||
normalized.includes("authentication_error")
|
||||
) {
|
||||
return {
|
||||
eventType: "unavailable",
|
||||
severity: "critical",
|
||||
counted: true,
|
||||
reason: "provider_authentication_failed",
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
normalized.includes("生成结果未通过替换质检") ||
|
||||
normalized.includes("output_moderation") ||
|
||||
@ -221,6 +269,7 @@ export function classifyAiModelHealthFailure(error: unknown): {
|
||||
normalized.includes("provider_error") ||
|
||||
normalized.includes("provider request failed") ||
|
||||
normalized.includes("provider task failed") ||
|
||||
normalized.includes("vip98k task failed") ||
|
||||
normalized.includes("upstream error") ||
|
||||
normalized.includes("upstream_error") ||
|
||||
normalized.includes("upstream request failed") ||
|
||||
@ -283,6 +332,10 @@ export function shouldFailoverAiModelError(error: unknown) {
|
||||
normalized.includes("no task id") ||
|
||||
normalized.includes("missing task id");
|
||||
|
||||
if (classification.reason === "provider_authentication_failed") {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (classification.reason === "user_cancelled" || classification.reason === "non_provider_failure") {
|
||||
return false;
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user