fix(canvas): distinguish provider balance from image rate limits

This commit is contained in:
134 2026-10-01 07:44:29 +08:00
parent 85d678eaf4
commit 39d58d0f2c
15 changed files with 189 additions and 41 deletions

View File

@ -1,6 +1,6 @@
import { createLogger } from "@/lib/observability/logger"; import { createLogger } from "@/lib/observability/logger";
import { NextResponse } from "next/server"; import { NextResponse } from "next/server";
import { getRequestErrorMessage } from "@/lib/request-error"; import { getRequestErrorMessage, isProviderBalanceError, isRateLimitError } from "@/lib/request-error";
import { generateUUID } from "@/lib/utils"; import { generateUUID } from "@/lib/utils";
const chatApiLogger = createLogger("chat.errors"); const chatApiLogger = createLogger("chat.errors");
@ -20,6 +20,14 @@ export function getChatStreamErrorMessage(error: unknown): string {
error instanceof Error ? String(error.message || "").trim() : String(error || "").trim(); error instanceof Error ? String(error.message || "").trim() : String(error || "").trim();
const normalized = rawMessage.toLowerCase(); const normalized = rawMessage.toLowerCase();
if (isProviderBalanceError(error)) {
return "当前模型服务商线路余额或额度不足,请切换模型或联系管理员;与您的积分余额无关。";
}
if (isRateLimitError(error)) {
return "当前模型服务请求过于频繁,请稍后重试或切换模型。";
}
if ( if (
normalized.includes("token quota is not enough") || normalized.includes("token quota is not enough") ||
normalized.includes("pre_consume_token_quota_failed") || normalized.includes("pre_consume_token_quota_failed") ||
@ -47,6 +55,16 @@ function resolveChatApiFailure(error: unknown) {
const detailMessage = getRequestErrorMessage(error); const detailMessage = getRequestErrorMessage(error);
const normalized = String(detailMessage || "").toLowerCase(); const normalized = String(detailMessage || "").toLowerCase();
if (isProviderBalanceError(error)) {
return {
code: "server_error:chat",
message: "模型服务商线路余额或额度不足,请切换模型或联系管理员。",
status: 503,
category: "provider_balance",
detailMessage,
};
}
if ( if (
normalized.includes("payload too large") || normalized.includes("payload too large") ||
normalized.includes("request entity too large") || normalized.includes("request entity too large") ||

View File

@ -4,6 +4,8 @@ import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
import { modelLacksReferenceImageInputSupport } from "@/lib/ai/reference-image-capability"; import { modelLacksReferenceImageInputSupport } from "@/lib/ai/reference-image-capability";
import type { ChatMessage } from "@/lib/types"; import type { ChatMessage } from "@/lib/types";
import { convertMessagesWithImages } from "./image-media"; import { convertMessagesWithImages } from "./image-media";
import { isProviderBalanceError } from "@/lib/request-error";
import { isImageFallbackModelBalanceCoolingDown } from "@/lib/ai/runtime/image-fallback-balance-cooldown";
const chatApiLogger = createLogger("chat.imageFailover"); const chatApiLogger = createLogger("chat.imageFailover");
@ -82,6 +84,7 @@ export function normalizeFailoverErrorMessage(error: unknown): string {
export function shouldCooldownImageFallbackModel(errorMessage: string): boolean { export function shouldCooldownImageFallbackModel(errorMessage: string): boolean {
const msg = errorMessage.toLowerCase(); const msg = errorMessage.toLowerCase();
return ( return (
isProviderBalanceError(errorMessage) ||
msg.includes("model not found") || msg.includes("model not found") ||
msg.includes("model_not_found") || msg.includes("model_not_found") ||
msg.includes("no available channel for model") || msg.includes("no available channel for model") ||
@ -323,9 +326,9 @@ export function buildImageFallbackCandidates(
if (!sameProvider && !IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(providerType)) { if (!sameProvider && !IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(providerType)) {
return false; return false;
} }
if (isImageFallbackModelCoolingDown(item.model.id)) { if (isImageFallbackModelCoolingDown(item.model.id) || isImageFallbackModelBalanceCoolingDown(item.model)) {
const cooldown = imageFailoverModelCooldown.get(item.model.id); const cooldown = imageFailoverModelCooldown.get(item.model.id);
chatApiLogger.info(`[Chat API] ⏭ Skip configured image fallback on cooldown: ${item.model.displayName} (${providerType}) - ${cooldown?.reason || "cooldown"}`); chatApiLogger.info(`[Chat API] ⏭ Skip configured image fallback on cooldown: ${item.model.displayName} (${providerType}) - ${cooldown?.reason || "provider balance cooldown"}`);
return false; return false;
} }
return true; return true;
@ -485,9 +488,9 @@ export function getImageCrossProviderCandidates(
} }
if (m.model.id === options.currentModelId) return false; if (m.model.id === options.currentModelId) return false;
if (options.currentProviderId && m.provider.id === options.currentProviderId) return false; if (options.currentProviderId && m.provider.id === options.currentProviderId) return false;
if (isImageFallbackModelCoolingDown(m.model.id)) { if (isImageFallbackModelCoolingDown(m.model.id) || isImageFallbackModelBalanceCoolingDown(m.model)) {
const cooldown = imageFailoverModelCooldown.get(m.model.id); const cooldown = imageFailoverModelCooldown.get(m.model.id);
chatApiLogger.info(`[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "cooldown"}`); chatApiLogger.info(`[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "provider balance cooldown"}`);
return false; return false;
} }
return true; return true;

View File

@ -179,6 +179,7 @@ import { ChatSDKError } from "@/lib/errors";
import { import {
getFriendlyRateLimitMessage, getFriendlyRateLimitMessage,
getRequestErrorMessage, getRequestErrorMessage,
getUserFacingImageGenerationErrorMessage,
isRateLimitError, isRateLimitError,
} from "@/lib/request-error"; } from "@/lib/request-error";
import { import {
@ -437,10 +438,11 @@ function assertGeneratedImageMarkdownHasImage(markdown: string, modelLabel: stri
function getPublicImageGenerationErrorText(error: unknown) { function getPublicImageGenerationErrorText(error: unknown) {
const message = error instanceof Error ? error.message : String(error || ""); const message = error instanceof Error ? error.message : String(error || "");
if (!message || INTERNAL_IMAGE_GENERATION_FAILURE_PATTERN.test(message)) { const publicMessage = getUserFacingImageGenerationErrorMessage(error);
if (!message || (publicMessage === message && INTERNAL_IMAGE_GENERATION_FAILURE_PATTERN.test(message))) {
return "图片生成失败,请稍后重试"; return "图片生成失败,请稍后重试";
} }
return message; return publicMessage;
} }
function replacementScenePlanShouldIncludeModelImage(scenePlan: any) { function replacementScenePlanShouldIncludeModelImage(scenePlan: any) {

View File

@ -8774,7 +8774,7 @@ export function CanvasChatPanel({
const responseData = (error as any)?.data; const responseData = (error as any)?.data;
const isInsufficientPoints = const isInsufficientPoints =
responseData?.error === "insufficient_points" || responseData?.error === "insufficient_points" ||
/积分不足/.test(fallbackMessage); /^\s*积分不足(?:[,。::\s]|$)/.test(fallbackMessage);
if (isInsufficientPoints) { if (isInsufficientPoints) {
const balance = Math.max(0, Number(responseData?.balance ?? 0)); const balance = Math.max(0, Number(responseData?.balance ?? 0));

View File

@ -0,0 +1,18 @@
import { isProviderBalanceError } from "@/lib/request-error";
const IMAGE_FAILOVER_BALANCE_COOLDOWN_MS = 2 * 60 * 60 * 1000;
export function isImageFallbackModelBalanceCoolingDown(
model: {
healthLastEventType?: unknown;
healthLastError?: unknown;
healthLastFailureAt?: unknown;
} | null | undefined,
now = Date.now(),
) {
if (model?.healthLastEventType !== "quota" || !isProviderBalanceError(model.healthLastError)) {
return false;
}
const failedAt = new Date(String(model.healthLastFailureAt || "")).getTime();
return Number.isFinite(failedAt) && now - failedAt < IMAGE_FAILOVER_BALANCE_COOLDOWN_MS && now >= failedAt;
}

View File

@ -11,6 +11,7 @@ import {
isMediaInputFailure, isMediaInputFailure,
isNonProviderRequestStatus, isNonProviderRequestStatus,
} from "@/lib/ai/runtime/request-failure-classification"; } from "@/lib/ai/runtime/request-failure-classification";
import { isProviderBalanceError, isRateLimitError } from "@/lib/request-error";
export type AiModelHealthEventType = export type AiModelHealthEventType =
| "success" | "success"
@ -217,25 +218,21 @@ export function classifyAiModelHealthFailure(error: unknown): {
}; };
} }
if ( if (isProviderBalanceError(error)) {
normalized.includes("insufficient balance") ||
normalized.includes("insufficient credit") ||
normalized.includes("credits insufficient") ||
normalized.includes("credit insufficient") ||
normalized.includes("not enough credit") ||
normalized.includes("quota") ||
normalized.includes("余额") ||
normalized.includes("额度") ||
normalized.includes("预扣费额度") ||
normalized.includes("exceeded_current_quota") ||
normalized.includes("pre_consume_token_quota_failed") ||
normalized.includes("token quota is not enough")
) {
return { return {
eventType: "quota", eventType: "quota",
severity: "critical", severity: "critical",
counted: true, counted: true,
reason: "quota_or_balance", reason: "provider_balance_or_billing_quota",
};
}
if (isRateLimitError(error)) {
return {
eventType: "unavailable",
severity: "warning",
counted: false,
reason: "provider_rate_limited",
}; };
} }
@ -363,7 +360,8 @@ export function shouldFailoverAiModelError(error: unknown) {
if ( if (
classification.reason === "provider_authentication_failed" || classification.reason === "provider_authentication_failed" ||
classification.reason === "provider_access_forbidden" classification.reason === "provider_access_forbidden" ||
classification.reason === "provider_rate_limited"
) { ) {
return true; return true;
} }

View File

@ -1,4 +1,5 @@
import "server-only"; import "server-only";
import { isProviderBalanceError } from "@/lib/request-error";
export type ProviderHealthEventType = "success" | "timeout" | "quota" | "error"; export type ProviderHealthEventType = "success" | "timeout" | "quota" | "error";
@ -22,6 +23,7 @@ export function classifyProviderHealthEvent(error: unknown): ProviderHealthEvent
const normalized = normalizeErrorMessage(error).toLowerCase(); const normalized = normalizeErrorMessage(error).toLowerCase();
if ( if (
isProviderBalanceError(error) ||
normalized.includes("token quota is not enough") || normalized.includes("token quota is not enough") ||
normalized.includes("pre_consume_token_quota_failed") || normalized.includes("pre_consume_token_quota_failed") ||
normalized.includes("quota is not enough") normalized.includes("quota is not enough")

View File

@ -9,6 +9,7 @@ import { generateCanvasImage, type CanvasGenerateImageRequest } from "@/lib/canv
import { getPointsSystemConfig } from "@/lib/db/queries"; import { getPointsSystemConfig } from "@/lib/db/queries";
import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
import { getImageToolCreditCost, type ImageToolCreditFeature } from "@/lib/points/config"; import { getImageToolCreditCost, type ImageToolCreditFeature } from "@/lib/points/config";
import { getUserFacingImageGenerationErrorMessage } from "@/lib/request-error";
interface GenerateImageRequest extends CanvasGenerateImageRequest { interface GenerateImageRequest extends CanvasGenerateImageRequest {
async?: boolean; async?: boolean;
@ -119,7 +120,7 @@ export async function executeCanvasImageResponse(body: GenerateImageRequest, use
} catch (error) { } catch (error) {
console.error("[Canvas Generate Image] Error:", error); console.error("[Canvas Generate Image] Error:", error);
return NextResponse.json( return NextResponse.json(
{ success: false, error: error instanceof Error ? error.message : "生成失败" }, { success: false, error: getUserFacingImageGenerationErrorMessage(error) },
{ status: 500 }, { status: 500 },
); );
} }

View File

@ -37,6 +37,7 @@ import {
shouldFailoverAiModelError, shouldFailoverAiModelError,
} from "@/lib/ai/runtime/model-health"; } from "@/lib/ai/runtime/model-health";
import { uploadFile } from "@/lib/storage/service"; import { uploadFile } from "@/lib/storage/service";
import { isImageFallbackModelBalanceCoolingDown } from "@/lib/ai/runtime/image-fallback-balance-cooldown";
export interface CanvasLocalEditCompositeRegion { export interface CanvasLocalEditCompositeRegion {
x: number; x: number;
@ -4506,6 +4507,7 @@ function findCanvasFailoverModel(params: {
if ( if (
!item.model.isEnabled || !item.model.isEnabled ||
item.model.isSubModel || item.model.isSubModel ||
isImageFallbackModelBalanceCoolingDown(item.model) ||
(item.model.excludeFromFailover && !isSameNameScaleAdjustFallback) (item.model.excludeFromFailover && !isSameNameScaleAdjustFallback)
) { ) {
return false; return false;

View File

@ -31,6 +31,7 @@ import {
markCanvasLayoutReplaceBatchItemRunning, markCanvasLayoutReplaceBatchItemRunning,
} from "@/lib/canvas/layout-replace-batch"; } from "@/lib/canvas/layout-replace-batch";
import { persistAgentProjectGeneratedResult } from "@/lib/agents/agent-project-result-persistence"; import { persistAgentProjectGeneratedResult } from "@/lib/agents/agent-project-result-persistence";
import { getUserFacingImageGenerationErrorMessage } from "@/lib/request-error";
type CanvasImageTaskStatus = type CanvasImageTaskStatus =
| "waiting" | "waiting"
@ -562,12 +563,16 @@ async function runSingleTask(entry: PendingTaskEntry) {
and(eq(canvasImageTask.id, entry.taskId), eq(canvasImageTask.status, "running")) and(eq(canvasImageTask.id, entry.taskId), eq(canvasImageTask.status, "running"))
); );
} catch (error) { } catch (error) {
completedBatchError = error instanceof Error ? error.message : String(error); console.error("[Canvas Image Task Queue] Image task failed:", {
taskId: entry.taskId,
error,
});
completedBatchError = getUserFacingImageGenerationErrorMessage(error);
await db await db
.update(canvasImageTask) .update(canvasImageTask)
.set({ .set({
status: "failed", status: "failed",
error: error instanceof Error ? error.message : String(error), error: completedBatchError,
completedAt: new Date(), completedAt: new Date(),
updatedAt: new Date(), updatedAt: new Date(),
}) })

View File

@ -186,7 +186,7 @@ export async function executeExtendPOST(request: Request, trustedUserId: string)
message: error.message, message: error.message,
detail: error.detail, detail: error.detail,
}); });
if (error.status === 429 || isRateLimitError(error)) { if (isRateLimitError(error)) {
return NextResponse.json( return NextResponse.json(
{ {
error: getFriendlyRateLimitMessage("当前操作过于频繁,请稍后再试"), error: getFriendlyRateLimitMessage("当前操作过于频繁,请稍后再试"),

View File

@ -9,6 +9,18 @@ const RATE_LIMIT_PATTERNS = [
/达到速率限制/, /达到速率限制/,
/限流/, /限流/,
]; ];
const PROVIDER_BALANCE_PATTERNS = [
/余额(?:不足|不够|已?耗尽)/i,
/额度(?:不足|不够|已?耗尽)/i,
/预扣费额度(?:不足|失败)/i,
/\binsufficient[ _-]?(?:balance|credits?|funds?|quota)\b/i,
/\b(?:balance|credits?|funds?)\s+(?:is\s+)?(?:not enough|insufficient|exhausted)\b/i,
/\b(?:exceeded_current_quota|pre_consume_token_quota_failed)\b/i,
/\btoken quota is not enough\b/i,
/\bquota is not enough\b/i,
/\b(?:not enough credits?|credits? is not enough)\b/i,
/\b(?:billing_hard_limit_reached|payment_required)\b/i,
];
const NETWORK_TRANSPORT_PATTERNS = [ const NETWORK_TRANSPORT_PATTERNS = [
/\bfailed to fetch\b/i, /\bfailed to fetch\b/i,
/\bnetworkerror\b/i, /\bnetworkerror\b/i,
@ -153,17 +165,51 @@ export function getRequestErrorMessage(error: unknown): string | null {
return null; return null;
} }
function getRequestErrorText(error: unknown) {
const candidates: string[] = [];
if (error instanceof Error) {
collectTextCandidates({ ...error, message: error.message, cause: error.cause }, candidates);
} else {
collectTextCandidates(error, candidates);
}
return candidates.join(" ").slice(0, 4000);
}
export function isProviderBalanceError(error: unknown): boolean {
const message = getRequestErrorText(error);
return PROVIDER_BALANCE_PATTERNS.some((pattern) => pattern.test(message));
}
export function isRateLimitError(error: unknown): boolean { export function isRateLimitError(error: unknown): boolean {
// Some providers return HTTP 429 for an actual unpaid balance. The status
// alone must not override an explicit billing error in the response body.
if (isProviderBalanceError(error)) {
return false;
}
if (getRequestErrorStatus(error) === 429) { if (getRequestErrorStatus(error) === 429) {
return true; return true;
} }
const message = getRequestErrorMessage(error); const message = getRequestErrorText(error);
if (!message) { if (!message) {
return false; return false;
} }
return RATE_LIMIT_PATTERNS.some((pattern) => pattern.test(message)); return /(?:^|\D)429(?:\D|$)/.test(message) ||
RATE_LIMIT_PATTERNS.some((pattern) => pattern.test(message));
}
export function getUserFacingImageGenerationErrorMessage(
error: unknown,
fallbackMessage = "图片生成失败,请稍后重试",
) {
if (isProviderBalanceError(error)) {
return "图片服务商线路余额或额度不足,请切换模型或联系管理员;与您的积分余额无关。";
}
if (isRateLimitError(error)) {
return "图片服务请求过于频繁,请稍后重试或切换模型。";
}
return getRequestErrorMessage(error) || fallbackMessage;
} }
export function getFriendlyRateLimitMessage( export function getFriendlyRateLimitMessage(
@ -181,6 +227,9 @@ export function getUserFacingRequestErrorMessage(
rateLimitMessage?: string; rateLimitMessage?: string;
} }
) { ) {
if (isProviderBalanceError(error)) {
return "模型服务商线路余额或额度不足,请切换模型或联系管理员。";
}
if (isRateLimitError(error)) { if (isRateLimitError(error)) {
return getFriendlyRateLimitMessage(options?.rateLimitMessage); return getFriendlyRateLimitMessage(options?.rateLimitMessage);
} }

View File

@ -70,17 +70,19 @@ export async function fetchWithErrorHandlers(
} }
const fallbackMessage = const fallbackMessage =
getUserFacingRequestErrorMessage( payload?.error === "quota_exceeded" && typeof payload?.message === "string"
{ ? payload.message
status: response.status, : getUserFacingRequestErrorMessage(
data: payload, {
message: responseText, status: response.status,
}, data: payload,
`Request failed with status ${response.status}`, message: responseText,
{ },
rateLimitMessage: getFriendlyRateLimitMessage(), `Request failed with status ${response.status}`,
} {
); rateLimitMessage: getFriendlyRateLimitMessage(),
}
);
const requestError = new Error(fallbackMessage) as Error & { const requestError = new Error(fallbackMessage) as Error & {
status?: number; status?: number;
data?: any; data?: any;

View File

@ -5,6 +5,7 @@ export default defineConfig({
testMatch: [ testMatch: [
/routes\/video-canvas-media-import\.test\.ts/, /routes\/video-canvas-media-import\.test\.ts/,
/routes\/video-canvas-toolbar-jobs\.test\.ts/, /routes\/video-canvas-toolbar-jobs\.test\.ts/,
/routes\/image-provider-error-classification\.test\.ts/,
/routes\/video-canvas-audio-remote\.test\.ts/, /routes\/video-canvas-audio-remote\.test\.ts/,
/routes\/video-canvas-groups-upscale\.test\.ts/, /routes\/video-canvas-groups-upscale\.test\.ts/,
/routes\/video-canvas-connections\.test\.ts/, /routes\/video-canvas-connections\.test\.ts/,

View File

@ -0,0 +1,47 @@
import { expect, test } from "@playwright/test";
import {
getUserFacingImageGenerationErrorMessage,
getUserFacingRequestErrorMessage,
isProviderBalanceError,
isRateLimitError,
} from "@/lib/request-error";
import { isImageFallbackModelBalanceCoolingDown } from "@/lib/ai/runtime/image-fallback-balance-cooldown";
test("a bare HTTP 429 is rate limiting, not insufficient balance", () => {
const error = Object.assign(new Error("Too many requests (HTTP 429)"), { status: 429 });
expect(isProviderBalanceError(error)).toBe(false);
expect(isRateLimitError(error)).toBe(true);
expect(getUserFacingImageGenerationErrorMessage(error)).toContain("请求过于频繁");
expect(getUserFacingRequestErrorMessage(error)).toContain("请求过于频繁");
});
test("an explicit provider balance error takes precedence over HTTP 429", () => {
const error = Object.assign(new Error("余额不足,请点击充值页面续费(HTTP 429)"), { status: 429 });
expect(isProviderBalanceError(error)).toBe(true);
expect(isRateLimitError(error)).toBe(false);
expect(getUserFacingImageGenerationErrorMessage(error)).toContain("服务商线路余额");
expect(getUserFacingImageGenerationErrorMessage(error)).toContain("与您的积分余额无关");
expect(getUserFacingImageGenerationErrorMessage(error)).not.toMatch(/^积分不足/);
});
test("nested billing details and generic quota limits are kept distinct", () => {
const nested = { status: 429, message: "request failed", data: { error: { message: "insufficient_balance" } } };
expect(isProviderBalanceError(nested)).toBe(true);
expect(isRateLimitError(nested)).toBe(false);
const usageLimit = new Error("Rate quota exceeded (HTTP 429)");
expect(isProviderBalanceError(usageLimit)).toBe(false);
expect(isRateLimitError(usageLimit)).toBe(true);
expect(isProviderBalanceError(new Error("该模型额度不足(HTTP 429)"))).toBe(true);
});
test("a recent confirmed provider balance failure skips only fallback selection", () => {
const now = Date.now();
const model = {
healthLastEventType: "quota",
healthLastError: "余额不足(HTTP 429)",
healthLastFailureAt: new Date(now - 30_000),
};
expect(isImageFallbackModelBalanceCoolingDown(model, now)).toBe(true);
expect(isImageFallbackModelBalanceCoolingDown({ ...model, healthLastFailureAt: new Date(now - 3 * 60 * 60 * 1000) }, now)).toBe(false);
expect(isImageFallbackModelBalanceCoolingDown({ ...model, healthLastEventType: "success" }, now)).toBe(false);
});