fix(canvas): distinguish provider balance from image rate limits
This commit is contained in:
parent
08eecb0190
commit
365bad8539
@ -1,6 +1,6 @@
|
||||
import { createLogger } from "@/lib/observability/logger";
|
||||
import { NextResponse } from "next/server";
|
||||
import { getRequestErrorMessage } from "@/lib/request-error";
|
||||
import { getRequestErrorMessage, isProviderBalanceError, isRateLimitError } from "@/lib/request-error";
|
||||
import { generateUUID } from "@/lib/utils";
|
||||
|
||||
const chatApiLogger = createLogger("chat.errors");
|
||||
@ -20,6 +20,14 @@ export function getChatStreamErrorMessage(error: unknown): string {
|
||||
error instanceof Error ? String(error.message || "").trim() : String(error || "").trim();
|
||||
const normalized = rawMessage.toLowerCase();
|
||||
|
||||
if (isProviderBalanceError(error)) {
|
||||
return "当前模型服务商线路余额或额度不足,请切换模型或联系管理员;与您的积分余额无关。";
|
||||
}
|
||||
|
||||
if (isRateLimitError(error)) {
|
||||
return "当前模型服务请求过于频繁,请稍后重试或切换模型。";
|
||||
}
|
||||
|
||||
if (
|
||||
normalized.includes("token quota is not enough") ||
|
||||
normalized.includes("pre_consume_token_quota_failed") ||
|
||||
@ -47,6 +55,16 @@ function resolveChatApiFailure(error: unknown) {
|
||||
const detailMessage = getRequestErrorMessage(error);
|
||||
const normalized = String(detailMessage || "").toLowerCase();
|
||||
|
||||
if (isProviderBalanceError(error)) {
|
||||
return {
|
||||
code: "server_error:chat",
|
||||
message: "模型服务商线路余额或额度不足,请切换模型或联系管理员。",
|
||||
status: 503,
|
||||
category: "provider_balance",
|
||||
detailMessage,
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
normalized.includes("payload too large") ||
|
||||
normalized.includes("request entity too large") ||
|
||||
|
||||
@ -4,6 +4,8 @@ import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||||
import { modelLacksReferenceImageInputSupport } from "@/lib/ai/reference-image-capability";
|
||||
import type { ChatMessage } from "@/lib/types";
|
||||
import { convertMessagesWithImages } from "./image-media";
|
||||
import { isProviderBalanceError } from "@/lib/request-error";
|
||||
import { isImageFallbackModelBalanceCoolingDown } from "@/lib/ai/runtime/image-fallback-balance-cooldown";
|
||||
|
||||
const chatApiLogger = createLogger("chat.imageFailover");
|
||||
|
||||
@ -82,6 +84,7 @@ export function normalizeFailoverErrorMessage(error: unknown): string {
|
||||
export function shouldCooldownImageFallbackModel(errorMessage: string): boolean {
|
||||
const msg = errorMessage.toLowerCase();
|
||||
return (
|
||||
isProviderBalanceError(errorMessage) ||
|
||||
msg.includes("model not found") ||
|
||||
msg.includes("model_not_found") ||
|
||||
msg.includes("no available channel for model") ||
|
||||
@ -323,9 +326,9 @@ export function buildImageFallbackCandidates(
|
||||
if (!sameProvider && !IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(providerType)) {
|
||||
return false;
|
||||
}
|
||||
if (isImageFallbackModelCoolingDown(item.model.id)) {
|
||||
if (isImageFallbackModelCoolingDown(item.model.id) || isImageFallbackModelBalanceCoolingDown(item.model)) {
|
||||
const cooldown = imageFailoverModelCooldown.get(item.model.id);
|
||||
chatApiLogger.info(`[Chat API] ⏭ Skip configured image fallback on cooldown: ${item.model.displayName} (${providerType}) - ${cooldown?.reason || "cooldown"}`);
|
||||
chatApiLogger.info(`[Chat API] ⏭ Skip configured image fallback on cooldown: ${item.model.displayName} (${providerType}) - ${cooldown?.reason || "provider balance cooldown"}`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
@ -485,9 +488,9 @@ export function getImageCrossProviderCandidates(
|
||||
}
|
||||
if (m.model.id === options.currentModelId) return false;
|
||||
if (options.currentProviderId && m.provider.id === options.currentProviderId) return false;
|
||||
if (isImageFallbackModelCoolingDown(m.model.id)) {
|
||||
if (isImageFallbackModelCoolingDown(m.model.id) || isImageFallbackModelBalanceCoolingDown(m.model)) {
|
||||
const cooldown = imageFailoverModelCooldown.get(m.model.id);
|
||||
chatApiLogger.info(`[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "cooldown"}`);
|
||||
chatApiLogger.info(`[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "provider balance cooldown"}`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
||||
@ -179,6 +179,7 @@ import { ChatSDKError } from "@/lib/errors";
|
||||
import {
|
||||
getFriendlyRateLimitMessage,
|
||||
getRequestErrorMessage,
|
||||
getUserFacingImageGenerationErrorMessage,
|
||||
isRateLimitError,
|
||||
} from "@/lib/request-error";
|
||||
import {
|
||||
@ -438,10 +439,11 @@ function assertGeneratedImageMarkdownHasImage(markdown: string, modelLabel: stri
|
||||
|
||||
function getPublicImageGenerationErrorText(error: unknown) {
|
||||
const message = error instanceof Error ? error.message : String(error || "");
|
||||
if (!message || INTERNAL_IMAGE_GENERATION_FAILURE_PATTERN.test(message)) {
|
||||
const publicMessage = getUserFacingImageGenerationErrorMessage(error);
|
||||
if (!message || (publicMessage === message && INTERNAL_IMAGE_GENERATION_FAILURE_PATTERN.test(message))) {
|
||||
return "图片生成失败,请稍后重试";
|
||||
}
|
||||
return message;
|
||||
return publicMessage;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -8774,7 +8774,7 @@ export function CanvasChatPanel({
|
||||
const responseData = (error as any)?.data;
|
||||
const isInsufficientPoints =
|
||||
responseData?.error === "insufficient_points" ||
|
||||
/积分不足/.test(fallbackMessage);
|
||||
/^\s*积分不足(?:[,。::\s]|$)/.test(fallbackMessage);
|
||||
|
||||
if (isInsufficientPoints) {
|
||||
const balance = Math.max(0, Number(responseData?.balance ?? 0));
|
||||
|
||||
18
lib/ai/runtime/image-fallback-balance-cooldown.ts
Normal file
18
lib/ai/runtime/image-fallback-balance-cooldown.ts
Normal file
@ -0,0 +1,18 @@
|
||||
import { isProviderBalanceError } from "@/lib/request-error";
|
||||
|
||||
const IMAGE_FAILOVER_BALANCE_COOLDOWN_MS = 2 * 60 * 60 * 1000;
|
||||
|
||||
export function isImageFallbackModelBalanceCoolingDown(
|
||||
model: {
|
||||
healthLastEventType?: unknown;
|
||||
healthLastError?: unknown;
|
||||
healthLastFailureAt?: unknown;
|
||||
} | null | undefined,
|
||||
now = Date.now(),
|
||||
) {
|
||||
if (model?.healthLastEventType !== "quota" || !isProviderBalanceError(model.healthLastError)) {
|
||||
return false;
|
||||
}
|
||||
const failedAt = new Date(String(model.healthLastFailureAt || "")).getTime();
|
||||
return Number.isFinite(failedAt) && now - failedAt < IMAGE_FAILOVER_BALANCE_COOLDOWN_MS && now >= failedAt;
|
||||
}
|
||||
@ -11,6 +11,7 @@ import {
|
||||
isMediaInputFailure,
|
||||
isNonProviderRequestStatus,
|
||||
} from "@/lib/ai/runtime/request-failure-classification";
|
||||
import { isProviderBalanceError, isRateLimitError } from "@/lib/request-error";
|
||||
|
||||
export type AiModelHealthEventType =
|
||||
| "success"
|
||||
@ -217,25 +218,21 @@ export function classifyAiModelHealthFailure(error: unknown): {
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
normalized.includes("insufficient balance") ||
|
||||
normalized.includes("insufficient credit") ||
|
||||
normalized.includes("credits insufficient") ||
|
||||
normalized.includes("credit insufficient") ||
|
||||
normalized.includes("not enough credit") ||
|
||||
normalized.includes("quota") ||
|
||||
normalized.includes("余额") ||
|
||||
normalized.includes("额度") ||
|
||||
normalized.includes("预扣费额度") ||
|
||||
normalized.includes("exceeded_current_quota") ||
|
||||
normalized.includes("pre_consume_token_quota_failed") ||
|
||||
normalized.includes("token quota is not enough")
|
||||
) {
|
||||
if (isProviderBalanceError(error)) {
|
||||
return {
|
||||
eventType: "quota",
|
||||
severity: "critical",
|
||||
counted: true,
|
||||
reason: "quota_or_balance",
|
||||
reason: "provider_balance_or_billing_quota",
|
||||
};
|
||||
}
|
||||
|
||||
if (isRateLimitError(error)) {
|
||||
return {
|
||||
eventType: "unavailable",
|
||||
severity: "warning",
|
||||
counted: false,
|
||||
reason: "provider_rate_limited",
|
||||
};
|
||||
}
|
||||
|
||||
@ -363,7 +360,8 @@ export function shouldFailoverAiModelError(error: unknown) {
|
||||
|
||||
if (
|
||||
classification.reason === "provider_authentication_failed" ||
|
||||
classification.reason === "provider_access_forbidden"
|
||||
classification.reason === "provider_access_forbidden" ||
|
||||
classification.reason === "provider_rate_limited"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import "server-only";
|
||||
import { isProviderBalanceError } from "@/lib/request-error";
|
||||
|
||||
export type ProviderHealthEventType = "success" | "timeout" | "quota" | "error";
|
||||
|
||||
@ -22,6 +23,7 @@ export function classifyProviderHealthEvent(error: unknown): ProviderHealthEvent
|
||||
const normalized = normalizeErrorMessage(error).toLowerCase();
|
||||
|
||||
if (
|
||||
isProviderBalanceError(error) ||
|
||||
normalized.includes("token quota is not enough") ||
|
||||
normalized.includes("pre_consume_token_quota_failed") ||
|
||||
normalized.includes("quota is not enough")
|
||||
|
||||
@ -9,6 +9,7 @@ import { generateCanvasImage, type CanvasGenerateImageRequest } from "@/lib/canv
|
||||
import { getPointsSystemConfig } from "@/lib/db/queries";
|
||||
import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
|
||||
import { getImageToolCreditCost, type ImageToolCreditFeature } from "@/lib/points/config";
|
||||
import { getUserFacingImageGenerationErrorMessage } from "@/lib/request-error";
|
||||
|
||||
interface GenerateImageRequest extends CanvasGenerateImageRequest {
|
||||
async?: boolean;
|
||||
@ -119,7 +120,7 @@ export async function executeCanvasImageResponse(body: GenerateImageRequest, use
|
||||
} catch (error) {
|
||||
console.error("[Canvas Generate Image] Error:", error);
|
||||
return NextResponse.json(
|
||||
{ success: false, error: error instanceof Error ? error.message : "生成失败" },
|
||||
{ success: false, error: getUserFacingImageGenerationErrorMessage(error) },
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
|
||||
@ -37,6 +37,7 @@ import {
|
||||
shouldFailoverAiModelError,
|
||||
} from "@/lib/ai/runtime/model-health";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
import { isImageFallbackModelBalanceCoolingDown } from "@/lib/ai/runtime/image-fallback-balance-cooldown";
|
||||
|
||||
export interface CanvasLocalEditCompositeRegion {
|
||||
x: number;
|
||||
@ -4506,6 +4507,7 @@ function findCanvasFailoverModel(params: {
|
||||
if (
|
||||
!item.model.isEnabled ||
|
||||
item.model.isSubModel ||
|
||||
isImageFallbackModelBalanceCoolingDown(item.model) ||
|
||||
(item.model.excludeFromFailover && !isSameNameScaleAdjustFallback)
|
||||
) {
|
||||
return false;
|
||||
|
||||
@ -37,6 +37,7 @@ import {
|
||||
type DetailReplacementBatchResult,
|
||||
} from "@/lib/agents/detail-replacement-batch";
|
||||
import { executeDetailReplacementBatch } from "@/lib/agents/detail-replacement-batch-runner";
|
||||
import { getUserFacingImageGenerationErrorMessage } from "@/lib/request-error";
|
||||
|
||||
type CanvasImageTaskStatus =
|
||||
| "waiting"
|
||||
@ -593,12 +594,16 @@ async function runSingleTask(entry: PendingTaskEntry) {
|
||||
and(eq(canvasImageTask.id, entry.taskId), eq(canvasImageTask.status, "running"))
|
||||
);
|
||||
} catch (error) {
|
||||
completedBatchError = error instanceof Error ? error.message : String(error);
|
||||
console.error("[Canvas Image Task Queue] Image task failed:", {
|
||||
taskId: entry.taskId,
|
||||
error,
|
||||
});
|
||||
completedBatchError = getUserFacingImageGenerationErrorMessage(error);
|
||||
await db
|
||||
.update(canvasImageTask)
|
||||
.set({
|
||||
status: "failed",
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
error: completedBatchError,
|
||||
completedAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
|
||||
@ -186,7 +186,7 @@ export async function executeExtendPOST(request: Request, trustedUserId: string)
|
||||
message: error.message,
|
||||
detail: error.detail,
|
||||
});
|
||||
if (error.status === 429 || isRateLimitError(error)) {
|
||||
if (isRateLimitError(error)) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: getFriendlyRateLimitMessage("当前操作过于频繁,请稍后再试"),
|
||||
|
||||
@ -9,6 +9,18 @@ const RATE_LIMIT_PATTERNS = [
|
||||
/达到速率限制/,
|
||||
/限流/,
|
||||
];
|
||||
const PROVIDER_BALANCE_PATTERNS = [
|
||||
/余额(?:不足|不够|已?耗尽)/i,
|
||||
/额度(?:不足|不够|已?耗尽)/i,
|
||||
/预扣费额度(?:不足|失败)/i,
|
||||
/\binsufficient[ _-]?(?:balance|credits?|funds?|quota)\b/i,
|
||||
/\b(?:balance|credits?|funds?)\s+(?:is\s+)?(?:not enough|insufficient|exhausted)\b/i,
|
||||
/\b(?:exceeded_current_quota|pre_consume_token_quota_failed)\b/i,
|
||||
/\btoken quota is not enough\b/i,
|
||||
/\bquota is not enough\b/i,
|
||||
/\b(?:not enough credits?|credits? is not enough)\b/i,
|
||||
/\b(?:billing_hard_limit_reached|payment_required)\b/i,
|
||||
];
|
||||
const NETWORK_TRANSPORT_PATTERNS = [
|
||||
/\bfailed to fetch\b/i,
|
||||
/\bnetworkerror\b/i,
|
||||
@ -153,17 +165,51 @@ export function getRequestErrorMessage(error: unknown): string | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function getRequestErrorText(error: unknown) {
|
||||
const candidates: string[] = [];
|
||||
if (error instanceof Error) {
|
||||
collectTextCandidates({ ...error, message: error.message, cause: error.cause }, candidates);
|
||||
} else {
|
||||
collectTextCandidates(error, candidates);
|
||||
}
|
||||
return candidates.join(" ").slice(0, 4000);
|
||||
}
|
||||
|
||||
export function isProviderBalanceError(error: unknown): boolean {
|
||||
const message = getRequestErrorText(error);
|
||||
return PROVIDER_BALANCE_PATTERNS.some((pattern) => pattern.test(message));
|
||||
}
|
||||
|
||||
export function isRateLimitError(error: unknown): boolean {
|
||||
// Some providers return HTTP 429 for an actual unpaid balance. The status
|
||||
// alone must not override an explicit billing error in the response body.
|
||||
if (isProviderBalanceError(error)) {
|
||||
return false;
|
||||
}
|
||||
if (getRequestErrorStatus(error) === 429) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const message = getRequestErrorMessage(error);
|
||||
const message = getRequestErrorText(error);
|
||||
if (!message) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return RATE_LIMIT_PATTERNS.some((pattern) => pattern.test(message));
|
||||
return /(?:^|\D)429(?:\D|$)/.test(message) ||
|
||||
RATE_LIMIT_PATTERNS.some((pattern) => pattern.test(message));
|
||||
}
|
||||
|
||||
export function getUserFacingImageGenerationErrorMessage(
|
||||
error: unknown,
|
||||
fallbackMessage = "图片生成失败,请稍后重试",
|
||||
) {
|
||||
if (isProviderBalanceError(error)) {
|
||||
return "图片服务商线路余额或额度不足,请切换模型或联系管理员;与您的积分余额无关。";
|
||||
}
|
||||
if (isRateLimitError(error)) {
|
||||
return "图片服务请求过于频繁,请稍后重试或切换模型。";
|
||||
}
|
||||
return getRequestErrorMessage(error) || fallbackMessage;
|
||||
}
|
||||
|
||||
export function getFriendlyRateLimitMessage(
|
||||
@ -181,6 +227,9 @@ export function getUserFacingRequestErrorMessage(
|
||||
rateLimitMessage?: string;
|
||||
}
|
||||
) {
|
||||
if (isProviderBalanceError(error)) {
|
||||
return "模型服务商线路余额或额度不足,请切换模型或联系管理员。";
|
||||
}
|
||||
if (isRateLimitError(error)) {
|
||||
return getFriendlyRateLimitMessage(options?.rateLimitMessage);
|
||||
}
|
||||
|
||||
24
lib/utils.ts
24
lib/utils.ts
@ -70,17 +70,19 @@ export async function fetchWithErrorHandlers(
|
||||
}
|
||||
|
||||
const fallbackMessage =
|
||||
getUserFacingRequestErrorMessage(
|
||||
{
|
||||
status: response.status,
|
||||
data: payload,
|
||||
message: responseText,
|
||||
},
|
||||
`Request failed with status ${response.status}`,
|
||||
{
|
||||
rateLimitMessage: getFriendlyRateLimitMessage(),
|
||||
}
|
||||
);
|
||||
payload?.error === "quota_exceeded" && typeof payload?.message === "string"
|
||||
? payload.message
|
||||
: getUserFacingRequestErrorMessage(
|
||||
{
|
||||
status: response.status,
|
||||
data: payload,
|
||||
message: responseText,
|
||||
},
|
||||
`Request failed with status ${response.status}`,
|
||||
{
|
||||
rateLimitMessage: getFriendlyRateLimitMessage(),
|
||||
}
|
||||
);
|
||||
const requestError = new Error(fallbackMessage) as Error & {
|
||||
status?: number;
|
||||
data?: any;
|
||||
|
||||
@ -4,6 +4,8 @@ export default defineConfig({
|
||||
testDir: "./tests",
|
||||
testMatch: [
|
||||
/routes\/video-canvas-media-import\.test\.ts/,
|
||||
/routes\/video-canvas-toolbar-jobs\.test\.ts/,
|
||||
/routes\/image-provider-error-classification\.test\.ts/,
|
||||
/routes\/video-canvas-audio-remote\.test\.ts/,
|
||||
/routes\/video-canvas-groups-upscale\.test\.ts/,
|
||||
/routes\/video-canvas-connections\.test\.ts/,
|
||||
|
||||
47
tests/routes/image-provider-error-classification.test.ts
Normal file
47
tests/routes/image-provider-error-classification.test.ts
Normal file
@ -0,0 +1,47 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
import {
|
||||
getUserFacingImageGenerationErrorMessage,
|
||||
getUserFacingRequestErrorMessage,
|
||||
isProviderBalanceError,
|
||||
isRateLimitError,
|
||||
} from "@/lib/request-error";
|
||||
import { isImageFallbackModelBalanceCoolingDown } from "@/lib/ai/runtime/image-fallback-balance-cooldown";
|
||||
|
||||
test("a bare HTTP 429 is rate limiting, not insufficient balance", () => {
|
||||
const error = Object.assign(new Error("Too many requests (HTTP 429)"), { status: 429 });
|
||||
expect(isProviderBalanceError(error)).toBe(false);
|
||||
expect(isRateLimitError(error)).toBe(true);
|
||||
expect(getUserFacingImageGenerationErrorMessage(error)).toContain("请求过于频繁");
|
||||
expect(getUserFacingRequestErrorMessage(error)).toContain("请求过于频繁");
|
||||
});
|
||||
|
||||
test("an explicit provider balance error takes precedence over HTTP 429", () => {
|
||||
const error = Object.assign(new Error("余额不足,请点击充值页面续费(HTTP 429)"), { status: 429 });
|
||||
expect(isProviderBalanceError(error)).toBe(true);
|
||||
expect(isRateLimitError(error)).toBe(false);
|
||||
expect(getUserFacingImageGenerationErrorMessage(error)).toContain("服务商线路余额");
|
||||
expect(getUserFacingImageGenerationErrorMessage(error)).toContain("与您的积分余额无关");
|
||||
expect(getUserFacingImageGenerationErrorMessage(error)).not.toMatch(/^积分不足/);
|
||||
});
|
||||
|
||||
test("nested billing details and generic quota limits are kept distinct", () => {
|
||||
const nested = { status: 429, message: "request failed", data: { error: { message: "insufficient_balance" } } };
|
||||
expect(isProviderBalanceError(nested)).toBe(true);
|
||||
expect(isRateLimitError(nested)).toBe(false);
|
||||
const usageLimit = new Error("Rate quota exceeded (HTTP 429)");
|
||||
expect(isProviderBalanceError(usageLimit)).toBe(false);
|
||||
expect(isRateLimitError(usageLimit)).toBe(true);
|
||||
expect(isProviderBalanceError(new Error("该模型额度不足(HTTP 429)"))).toBe(true);
|
||||
});
|
||||
|
||||
test("a recent confirmed provider balance failure skips only fallback selection", () => {
|
||||
const now = Date.now();
|
||||
const model = {
|
||||
healthLastEventType: "quota",
|
||||
healthLastError: "余额不足(HTTP 429)",
|
||||
healthLastFailureAt: new Date(now - 30_000),
|
||||
};
|
||||
expect(isImageFallbackModelBalanceCoolingDown(model, now)).toBe(true);
|
||||
expect(isImageFallbackModelBalanceCoolingDown({ ...model, healthLastFailureAt: new Date(now - 3 * 60 * 60 * 1000) }, now)).toBe(false);
|
||||
expect(isImageFallbackModelBalanceCoolingDown({ ...model, healthLastEventType: "success" }, now)).toBe(false);
|
||||
});
|
||||
Loading…
x
Reference in New Issue
Block a user