fix(canvas): distinguish provider balance from image rate limits

This commit is contained in:
134 2026-10-01 07:44:29 +08:00
parent 85d678eaf4
commit 39d58d0f2c
15 changed files with 189 additions and 41 deletions

View File

@ -1,6 +1,6 @@
import { createLogger } from "@/lib/observability/logger";
import { NextResponse } from "next/server";
import { getRequestErrorMessage } from "@/lib/request-error";
import { getRequestErrorMessage, isProviderBalanceError, isRateLimitError } from "@/lib/request-error";
import { generateUUID } from "@/lib/utils";
const chatApiLogger = createLogger("chat.errors");
@ -20,6 +20,14 @@ export function getChatStreamErrorMessage(error: unknown): string {
error instanceof Error ? String(error.message || "").trim() : String(error || "").trim();
const normalized = rawMessage.toLowerCase();
if (isProviderBalanceError(error)) {
return "当前模型服务商线路余额或额度不足,请切换模型或联系管理员;与您的积分余额无关。";
}
if (isRateLimitError(error)) {
return "当前模型服务请求过于频繁,请稍后重试或切换模型。";
}
if (
normalized.includes("token quota is not enough") ||
normalized.includes("pre_consume_token_quota_failed") ||
@ -47,6 +55,16 @@ function resolveChatApiFailure(error: unknown) {
const detailMessage = getRequestErrorMessage(error);
const normalized = String(detailMessage || "").toLowerCase();
if (isProviderBalanceError(error)) {
return {
code: "server_error:chat",
message: "模型服务商线路余额或额度不足,请切换模型或联系管理员。",
status: 503,
category: "provider_balance",
detailMessage,
};
}
if (
normalized.includes("payload too large") ||
normalized.includes("request entity too large") ||

View File

@ -4,6 +4,8 @@ import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
import { modelLacksReferenceImageInputSupport } from "@/lib/ai/reference-image-capability";
import type { ChatMessage } from "@/lib/types";
import { convertMessagesWithImages } from "./image-media";
import { isProviderBalanceError } from "@/lib/request-error";
import { isImageFallbackModelBalanceCoolingDown } from "@/lib/ai/runtime/image-fallback-balance-cooldown";
const chatApiLogger = createLogger("chat.imageFailover");
@ -82,6 +84,7 @@ export function normalizeFailoverErrorMessage(error: unknown): string {
export function shouldCooldownImageFallbackModel(errorMessage: string): boolean {
const msg = errorMessage.toLowerCase();
return (
isProviderBalanceError(errorMessage) ||
msg.includes("model not found") ||
msg.includes("model_not_found") ||
msg.includes("no available channel for model") ||
@ -323,9 +326,9 @@ export function buildImageFallbackCandidates(
if (!sameProvider && !IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(providerType)) {
return false;
}
if (isImageFallbackModelCoolingDown(item.model.id)) {
if (isImageFallbackModelCoolingDown(item.model.id) || isImageFallbackModelBalanceCoolingDown(item.model)) {
const cooldown = imageFailoverModelCooldown.get(item.model.id);
chatApiLogger.info(`[Chat API] ⏭ Skip configured image fallback on cooldown: ${item.model.displayName} (${providerType}) - ${cooldown?.reason || "cooldown"}`);
chatApiLogger.info(`[Chat API] ⏭ Skip configured image fallback on cooldown: ${item.model.displayName} (${providerType}) - ${cooldown?.reason || "provider balance cooldown"}`);
return false;
}
return true;
@ -485,9 +488,9 @@ export function getImageCrossProviderCandidates(
}
if (m.model.id === options.currentModelId) return false;
if (options.currentProviderId && m.provider.id === options.currentProviderId) return false;
if (isImageFallbackModelCoolingDown(m.model.id)) {
if (isImageFallbackModelCoolingDown(m.model.id) || isImageFallbackModelBalanceCoolingDown(m.model)) {
const cooldown = imageFailoverModelCooldown.get(m.model.id);
chatApiLogger.info(`[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "cooldown"}`);
chatApiLogger.info(`[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "provider balance cooldown"}`);
return false;
}
return true;

View File

@ -179,6 +179,7 @@ import { ChatSDKError } from "@/lib/errors";
import {
getFriendlyRateLimitMessage,
getRequestErrorMessage,
getUserFacingImageGenerationErrorMessage,
isRateLimitError,
} from "@/lib/request-error";
import {
@ -437,10 +438,11 @@ function assertGeneratedImageMarkdownHasImage(markdown: string, modelLabel: stri
function getPublicImageGenerationErrorText(error: unknown) {
const message = error instanceof Error ? error.message : String(error || "");
if (!message || INTERNAL_IMAGE_GENERATION_FAILURE_PATTERN.test(message)) {
const publicMessage = getUserFacingImageGenerationErrorMessage(error);
if (!message || (publicMessage === message && INTERNAL_IMAGE_GENERATION_FAILURE_PATTERN.test(message))) {
return "图片生成失败,请稍后重试";
}
return message;
return publicMessage;
}
function replacementScenePlanShouldIncludeModelImage(scenePlan: any) {

View File

@ -8774,7 +8774,7 @@ export function CanvasChatPanel({
const responseData = (error as any)?.data;
const isInsufficientPoints =
responseData?.error === "insufficient_points" ||
/积分不足/.test(fallbackMessage);
/^\s*积分不足(?:[,。::\s]|$)/.test(fallbackMessage);
if (isInsufficientPoints) {
const balance = Math.max(0, Number(responseData?.balance ?? 0));

View File

@ -0,0 +1,18 @@
import { isProviderBalanceError } from "@/lib/request-error";
const IMAGE_FAILOVER_BALANCE_COOLDOWN_MS = 2 * 60 * 60 * 1000;
export function isImageFallbackModelBalanceCoolingDown(
model: {
healthLastEventType?: unknown;
healthLastError?: unknown;
healthLastFailureAt?: unknown;
} | null | undefined,
now = Date.now(),
) {
if (model?.healthLastEventType !== "quota" || !isProviderBalanceError(model.healthLastError)) {
return false;
}
const failedAt = new Date(String(model.healthLastFailureAt || "")).getTime();
return Number.isFinite(failedAt) && now - failedAt < IMAGE_FAILOVER_BALANCE_COOLDOWN_MS && now >= failedAt;
}

View File

@ -11,6 +11,7 @@ import {
isMediaInputFailure,
isNonProviderRequestStatus,
} from "@/lib/ai/runtime/request-failure-classification";
import { isProviderBalanceError, isRateLimitError } from "@/lib/request-error";
export type AiModelHealthEventType =
| "success"
@ -217,25 +218,21 @@ export function classifyAiModelHealthFailure(error: unknown): {
};
}
if (
normalized.includes("insufficient balance") ||
normalized.includes("insufficient credit") ||
normalized.includes("credits insufficient") ||
normalized.includes("credit insufficient") ||
normalized.includes("not enough credit") ||
normalized.includes("quota") ||
normalized.includes("余额") ||
normalized.includes("额度") ||
normalized.includes("预扣费额度") ||
normalized.includes("exceeded_current_quota") ||
normalized.includes("pre_consume_token_quota_failed") ||
normalized.includes("token quota is not enough")
) {
if (isProviderBalanceError(error)) {
return {
eventType: "quota",
severity: "critical",
counted: true,
reason: "quota_or_balance",
reason: "provider_balance_or_billing_quota",
};
}
if (isRateLimitError(error)) {
return {
eventType: "unavailable",
severity: "warning",
counted: false,
reason: "provider_rate_limited",
};
}
@ -363,7 +360,8 @@ export function shouldFailoverAiModelError(error: unknown) {
if (
classification.reason === "provider_authentication_failed" ||
classification.reason === "provider_access_forbidden"
classification.reason === "provider_access_forbidden" ||
classification.reason === "provider_rate_limited"
) {
return true;
}

View File

@ -1,4 +1,5 @@
import "server-only";
import { isProviderBalanceError } from "@/lib/request-error";
export type ProviderHealthEventType = "success" | "timeout" | "quota" | "error";
@ -22,6 +23,7 @@ export function classifyProviderHealthEvent(error: unknown): ProviderHealthEvent
const normalized = normalizeErrorMessage(error).toLowerCase();
if (
isProviderBalanceError(error) ||
normalized.includes("token quota is not enough") ||
normalized.includes("pre_consume_token_quota_failed") ||
normalized.includes("quota is not enough")

View File

@ -9,6 +9,7 @@ import { generateCanvasImage, type CanvasGenerateImageRequest } from "@/lib/canv
import { getPointsSystemConfig } from "@/lib/db/queries";
import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
import { getImageToolCreditCost, type ImageToolCreditFeature } from "@/lib/points/config";
import { getUserFacingImageGenerationErrorMessage } from "@/lib/request-error";
interface GenerateImageRequest extends CanvasGenerateImageRequest {
async?: boolean;
@ -119,7 +120,7 @@ export async function executeCanvasImageResponse(body: GenerateImageRequest, use
} catch (error) {
console.error("[Canvas Generate Image] Error:", error);
return NextResponse.json(
{ success: false, error: error instanceof Error ? error.message : "生成失败" },
{ success: false, error: getUserFacingImageGenerationErrorMessage(error) },
{ status: 500 },
);
}

View File

@ -37,6 +37,7 @@ import {
shouldFailoverAiModelError,
} from "@/lib/ai/runtime/model-health";
import { uploadFile } from "@/lib/storage/service";
import { isImageFallbackModelBalanceCoolingDown } from "@/lib/ai/runtime/image-fallback-balance-cooldown";
export interface CanvasLocalEditCompositeRegion {
x: number;
@ -4506,6 +4507,7 @@ function findCanvasFailoverModel(params: {
if (
!item.model.isEnabled ||
item.model.isSubModel ||
isImageFallbackModelBalanceCoolingDown(item.model) ||
(item.model.excludeFromFailover && !isSameNameScaleAdjustFallback)
) {
return false;

View File

@ -31,6 +31,7 @@ import {
markCanvasLayoutReplaceBatchItemRunning,
} from "@/lib/canvas/layout-replace-batch";
import { persistAgentProjectGeneratedResult } from "@/lib/agents/agent-project-result-persistence";
import { getUserFacingImageGenerationErrorMessage } from "@/lib/request-error";
type CanvasImageTaskStatus =
| "waiting"
@ -562,12 +563,16 @@ async function runSingleTask(entry: PendingTaskEntry) {
and(eq(canvasImageTask.id, entry.taskId), eq(canvasImageTask.status, "running"))
);
} catch (error) {
completedBatchError = error instanceof Error ? error.message : String(error);
console.error("[Canvas Image Task Queue] Image task failed:", {
taskId: entry.taskId,
error,
});
completedBatchError = getUserFacingImageGenerationErrorMessage(error);
await db
.update(canvasImageTask)
.set({
status: "failed",
error: error instanceof Error ? error.message : String(error),
error: completedBatchError,
completedAt: new Date(),
updatedAt: new Date(),
})

View File

@ -186,7 +186,7 @@ export async function executeExtendPOST(request: Request, trustedUserId: string)
message: error.message,
detail: error.detail,
});
if (error.status === 429 || isRateLimitError(error)) {
if (isRateLimitError(error)) {
return NextResponse.json(
{
error: getFriendlyRateLimitMessage("当前操作过于频繁,请稍后再试"),

View File

@ -9,6 +9,18 @@ const RATE_LIMIT_PATTERNS = [
/达到速率限制/,
/限流/,
];
const PROVIDER_BALANCE_PATTERNS = [
/余额(?:不足|不够|已?耗尽)/i,
/额度(?:不足|不够|已?耗尽)/i,
/预扣费额度(?:不足|失败)/i,
/\binsufficient[ _-]?(?:balance|credits?|funds?|quota)\b/i,
/\b(?:balance|credits?|funds?)\s+(?:is\s+)?(?:not enough|insufficient|exhausted)\b/i,
/\b(?:exceeded_current_quota|pre_consume_token_quota_failed)\b/i,
/\btoken quota is not enough\b/i,
/\bquota is not enough\b/i,
/\b(?:not enough credits?|credits? is not enough)\b/i,
/\b(?:billing_hard_limit_reached|payment_required)\b/i,
];
const NETWORK_TRANSPORT_PATTERNS = [
/\bfailed to fetch\b/i,
/\bnetworkerror\b/i,
@ -153,17 +165,51 @@ export function getRequestErrorMessage(error: unknown): string | null {
return null;
}
function getRequestErrorText(error: unknown) {
const candidates: string[] = [];
if (error instanceof Error) {
collectTextCandidates({ ...error, message: error.message, cause: error.cause }, candidates);
} else {
collectTextCandidates(error, candidates);
}
return candidates.join(" ").slice(0, 4000);
}
export function isProviderBalanceError(error: unknown): boolean {
const message = getRequestErrorText(error);
return PROVIDER_BALANCE_PATTERNS.some((pattern) => pattern.test(message));
}
export function isRateLimitError(error: unknown): boolean {
// Some providers return HTTP 429 for an actual unpaid balance. The status
// alone must not override an explicit billing error in the response body.
if (isProviderBalanceError(error)) {
return false;
}
if (getRequestErrorStatus(error) === 429) {
return true;
}
const message = getRequestErrorMessage(error);
const message = getRequestErrorText(error);
if (!message) {
return false;
}
return RATE_LIMIT_PATTERNS.some((pattern) => pattern.test(message));
return /(?:^|\D)429(?:\D|$)/.test(message) ||
RATE_LIMIT_PATTERNS.some((pattern) => pattern.test(message));
}
export function getUserFacingImageGenerationErrorMessage(
error: unknown,
fallbackMessage = "图片生成失败,请稍后重试",
) {
if (isProviderBalanceError(error)) {
return "图片服务商线路余额或额度不足,请切换模型或联系管理员;与您的积分余额无关。";
}
if (isRateLimitError(error)) {
return "图片服务请求过于频繁,请稍后重试或切换模型。";
}
return getRequestErrorMessage(error) || fallbackMessage;
}
export function getFriendlyRateLimitMessage(
@ -181,6 +227,9 @@ export function getUserFacingRequestErrorMessage(
rateLimitMessage?: string;
}
) {
if (isProviderBalanceError(error)) {
return "模型服务商线路余额或额度不足,请切换模型或联系管理员。";
}
if (isRateLimitError(error)) {
return getFriendlyRateLimitMessage(options?.rateLimitMessage);
}

View File

@ -70,17 +70,19 @@ export async function fetchWithErrorHandlers(
}
const fallbackMessage =
getUserFacingRequestErrorMessage(
{
status: response.status,
data: payload,
message: responseText,
},
`Request failed with status ${response.status}`,
{
rateLimitMessage: getFriendlyRateLimitMessage(),
}
);
payload?.error === "quota_exceeded" && typeof payload?.message === "string"
? payload.message
: getUserFacingRequestErrorMessage(
{
status: response.status,
data: payload,
message: responseText,
},
`Request failed with status ${response.status}`,
{
rateLimitMessage: getFriendlyRateLimitMessage(),
}
);
const requestError = new Error(fallbackMessage) as Error & {
status?: number;
data?: any;

View File

@ -5,6 +5,7 @@ export default defineConfig({
testMatch: [
/routes\/video-canvas-media-import\.test\.ts/,
/routes\/video-canvas-toolbar-jobs\.test\.ts/,
/routes\/image-provider-error-classification\.test\.ts/,
/routes\/video-canvas-audio-remote\.test\.ts/,
/routes\/video-canvas-groups-upscale\.test\.ts/,
/routes\/video-canvas-connections\.test\.ts/,

View File

@ -0,0 +1,47 @@
import { expect, test } from "@playwright/test";
import {
getUserFacingImageGenerationErrorMessage,
getUserFacingRequestErrorMessage,
isProviderBalanceError,
isRateLimitError,
} from "@/lib/request-error";
import { isImageFallbackModelBalanceCoolingDown } from "@/lib/ai/runtime/image-fallback-balance-cooldown";
test("a bare HTTP 429 is rate limiting, not insufficient balance", () => {
const error = Object.assign(new Error("Too many requests (HTTP 429)"), { status: 429 });
expect(isProviderBalanceError(error)).toBe(false);
expect(isRateLimitError(error)).toBe(true);
expect(getUserFacingImageGenerationErrorMessage(error)).toContain("请求过于频繁");
expect(getUserFacingRequestErrorMessage(error)).toContain("请求过于频繁");
});
test("an explicit provider balance error takes precedence over HTTP 429", () => {
const error = Object.assign(new Error("余额不足,请点击充值页面续费(HTTP 429)"), { status: 429 });
expect(isProviderBalanceError(error)).toBe(true);
expect(isRateLimitError(error)).toBe(false);
expect(getUserFacingImageGenerationErrorMessage(error)).toContain("服务商线路余额");
expect(getUserFacingImageGenerationErrorMessage(error)).toContain("与您的积分余额无关");
expect(getUserFacingImageGenerationErrorMessage(error)).not.toMatch(/^积分不足/);
});
test("nested billing details and generic quota limits are kept distinct", () => {
const nested = { status: 429, message: "request failed", data: { error: { message: "insufficient_balance" } } };
expect(isProviderBalanceError(nested)).toBe(true);
expect(isRateLimitError(nested)).toBe(false);
const usageLimit = new Error("Rate quota exceeded (HTTP 429)");
expect(isProviderBalanceError(usageLimit)).toBe(false);
expect(isRateLimitError(usageLimit)).toBe(true);
expect(isProviderBalanceError(new Error("该模型额度不足(HTTP 429)"))).toBe(true);
});
test("a recent confirmed provider balance failure skips only fallback selection", () => {
const now = Date.now();
const model = {
healthLastEventType: "quota",
healthLastError: "余额不足(HTTP 429)",
healthLastFailureAt: new Date(now - 30_000),
};
expect(isImageFallbackModelBalanceCoolingDown(model, now)).toBe(true);
expect(isImageFallbackModelBalanceCoolingDown({ ...model, healthLastFailureAt: new Date(now - 3 * 60 * 60 * 1000) }, now)).toBe(false);
expect(isImageFallbackModelBalanceCoolingDown({ ...model, healthLastEventType: "success" }, now)).toBe(false);
});