diff --git a/app/(chat)/api/chat/_lib/image-failover.ts b/app/(chat)/api/chat/_lib/image-failover.ts index e4b9d7ec0..c8c661634 100644 --- a/app/(chat)/api/chat/_lib/image-failover.ts +++ b/app/(chat)/api/chat/_lib/image-failover.ts @@ -5,7 +5,7 @@ import { modelLacksReferenceImageInputSupport } from "@/lib/ai/reference-image-c import type { ChatMessage } from "@/lib/types"; import { convertMessagesWithImages } from "./image-media"; import { isProviderBalanceError } from "@/lib/request-error"; -import { isImageFallbackModelBalanceCoolingDown } from "@/lib/ai/runtime/image-fallback-balance-cooldown"; +import { isImageModelBalanceBlocked } from "@/lib/ai/runtime/image-balance"; const chatApiLogger = createLogger("chat.imageFailover"); @@ -326,7 +326,7 @@ export function buildImageFallbackCandidates( if (!sameProvider && !IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(providerType)) { return false; } - if (isImageFallbackModelCoolingDown(item.model.id) || isImageFallbackModelBalanceCoolingDown(item.model)) { + if (isImageFallbackModelCoolingDown(item.model.id) || isImageModelBalanceBlocked(item.model)) { const cooldown = imageFailoverModelCooldown.get(item.model.id); chatApiLogger.info(`[Chat API] ⏭ Skip configured image fallback on cooldown: ${item.model.displayName} (${providerType}) - ${cooldown?.reason || "provider balance cooldown"}`); return false; @@ -488,7 +488,7 @@ export function getImageCrossProviderCandidates( } if (m.model.id === options.currentModelId) return false; if (options.currentProviderId && m.provider.id === options.currentProviderId) return false; - if (isImageFallbackModelCoolingDown(m.model.id) || isImageFallbackModelBalanceCoolingDown(m.model)) { + if (isImageFallbackModelCoolingDown(m.model.id) || isImageModelBalanceBlocked(m.model)) { const cooldown = imageFailoverModelCooldown.get(m.model.id); chatApiLogger.info(`[Chat API] ⏭ Skip image failover model on cooldown: ${m.model.displayName} (${m.provider.type}) - ${cooldown?.reason || "provider balance cooldown"}`); return false; diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 82138faf1..e1971a923 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -336,6 +336,7 @@ import { } from "./_lib/image-media"; import { buildImageFallbackCandidates, + countChatMessageImageInputs, generateImageWithFailoverCandidate, getImageCrossProviderCandidates, IMAGE_FAILOVER_PROVIDER_ALLOWLIST, @@ -344,6 +345,8 @@ import { setImageFallbackModelCooldown, shouldCooldownImageFallbackModel, } from "./_lib/image-failover"; +import { isImageModelBalanceBlocked, resolveConfiguredImageBalanceFallback } from "@/lib/ai/runtime/image-balance"; +import { isProviderBalanceError } from "@/lib/request-error"; import { buildAgentSceneWorkflowPrompt, buildAgentSlotPrompt, @@ -8769,6 +8772,19 @@ export async function POST(request: Request) { } } + if (/^[0-9a-f-]{36}$/i.test(finalModelId)) { + const requested = await getAiModelById({ id: finalModelId }); + if (requested?.supportsImageGen && isImageModelBalanceBlocked(requested)) { + const replacement = resolveConfiguredImageBalanceFallback(requested, await getEnabledAiModelsWithProvider(), { + imageSize: effectiveImageGenerationParams.imageSize, + aspectRatio: effectiveImageGenerationParams.aspectRatio, + referenceImageCount: countChatMessageImageInputs(uiMessagesForModel.filter(item => item.role === "user").slice(-1)), + }); + if (!replacement) return new ChatSDKError("bad_request:api", "当前生图线路余额不足,已暂停新请求;请切换可用模型,已有任务和图片会保留。").toResponse(); + finalModelId = replacement.model.id; + } + } + // 检查是否是 UUID 格式(数据库中的模型 ID) const isUUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( @@ -10002,7 +10018,10 @@ export async function POST(request: Request) { let fallbackSuccess = false; const triedFallbackModelIds = new Set(); + const balanceBlockedProviders = new Set(); + if (isProviderBalanceError(primaryError) && currentModelEntry?.provider.id) balanceBlockedProviders.add(currentModelEntry.provider.id); for (const candidate of fallbackCandidates) { + if (balanceBlockedProviders.has(candidate.provider.id)) continue; try { triedFallbackModelIds.add(String(candidate.model.id || "")); chatApiLogger.info(`[Chat API] 🔄 Alibaba fallback model: ${candidate.model.displayName} (${candidate.provider.type})`); @@ -10025,6 +10044,7 @@ export async function POST(request: Request) { chatApiLogger.info(`[Chat API] ✅ Alibaba fallback model ${fallbackResult.displayName} succeeded!`); break; } catch (fallbackError) { + if (isProviderBalanceError(fallbackError)) balanceBlockedProviders.add(candidate.provider.id); const fallbackErrorMessage = normalizeFailoverErrorMessage(fallbackError); if (shouldCooldownImageFallbackModel(fallbackErrorMessage)) { setImageFallbackModelCooldown(candidate.model.id, fallbackErrorMessage); @@ -10073,9 +10093,10 @@ export async function POST(request: Request) { currentModelKey: actualModelId, currentProviderId: alibabaModelConfig?.providerId, providerPriority, - }).filter((candidate) => !triedFallbackModelIds.has(String(candidate.model.id || ""))); + }).filter((candidate) => !triedFallbackModelIds.has(String(candidate.model.id || "")) && !balanceBlockedProviders.has(candidate.provider.id)); for (const candidate of crossProviderCandidates) { + if (balanceBlockedProviders.has(candidate.provider.id)) continue; try { triedFallbackModelIds.add(String(candidate.model.id || "")); chatApiLogger.info(`[Chat API] 🔁 Cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})`); @@ -10157,6 +10178,7 @@ export async function POST(request: Request) { chatApiLogger.info(`[Chat API] ✅ Cross-provider failover succeeded: ${candidate.model.displayName} (${candidate.provider.type})`); break; } catch (crossError) { + if (isProviderBalanceError(crossError)) balanceBlockedProviders.add(candidate.provider.id); const crossErrorMessage = normalizeFailoverErrorMessage(crossError); if (shouldCooldownImageFallbackModel(crossErrorMessage)) { setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage); diff --git a/lib/ai/dynamic-provider.ts b/lib/ai/dynamic-provider.ts index 8d561841b..87e881509 100644 --- a/lib/ai/dynamic-provider.ts +++ b/lib/ai/dynamic-provider.ts @@ -1,4 +1,5 @@ import "server-only"; +import { isImageModelBalanceBlocked } from "@/lib/ai/runtime/image-balance"; import { createOpenAI } from "@ai-sdk/openai"; import { @@ -418,7 +419,7 @@ export async function getDynamicModel( ): Promise { try { const model = await getAiModelById({ id: modelId }); - if (!model || !model.isEnabled) { + if (!model || !model.isEnabled || (model.supportsImageGen && isImageModelBalanceBlocked(model))) { logger.warn("model unavailable", { modelId }); return null; } diff --git a/lib/ai/providers/vip98k.ts b/lib/ai/providers/vip98k.ts index 2f085c675..386ad542b 100644 --- a/lib/ai/providers/vip98k.ts +++ b/lib/ai/providers/vip98k.ts @@ -1261,6 +1261,7 @@ export async function submitVip98kExternalTask(params: { const configuredInvocations = getVip98kInvocations(taskParams.modelConfig); const invocation = selectVip98kTaskInvocation({ modelId: taskParams.modelId, + kind: taskParams.kind, configuredInvocation: configuredInvocations[0] || null, configuredInvocations, imageCount: imageUrls.length, diff --git a/lib/ai/runtime/image-balance.ts b/lib/ai/runtime/image-balance.ts new file mode 100644 index 000000000..01aa9267a --- /dev/null +++ b/lib/ai/runtime/image-balance.ts @@ -0,0 +1,74 @@ +import { isProviderBalanceError } from "@/lib/request-error"; + +type BalanceState = { + healthLastEventType?: unknown; + healthLastError?: unknown; + healthAutoDisabledReason?: unknown; +}; + +export function isImageModelBalanceBlocked(model: BalanceState | null | undefined) { + return Boolean( + model && + ((model.healthLastEventType === "quota" && isProviderBalanceError(model.healthLastError)) || + isProviderBalanceError(model.healthAutoDisabledReason)), + ); +} + +export function resolveConfiguredImageBalanceFallback< + T extends { + model: BalanceState & { + id: string; + modelId: string; + isEnabled: boolean; + supportsImageGen: boolean; + imageGenConfig?: unknown; + }; + provider: { isEnabled?: boolean }; + }, +>( + requested: BalanceState & { fallbackModelId1?: string | null; fallbackModelId2?: string | null }, + entries: T[], + input: { + imageSize?: unknown; + aspectRatio?: unknown; + referenceImageCount?: number; + }, +) { + if (!isImageModelBalanceBlocked(requested)) return undefined; + for (const ref of [requested.fallbackModelId1, requested.fallbackModelId2]) { + if (!ref) continue; + const entry = entries.find(({ model }) => model.id === ref || model.modelId === ref); + if ( + !entry || + !entry.model.isEnabled || + entry.provider.isEnabled === false || + !entry.model.supportsImageGen || + isImageModelBalanceBlocked(entry.model) + ) + continue; + const config = entry.model.imageGenConfig as Record | null; + const size = String(input.imageSize || "").toUpperCase(); + const ratio = String(input.aspectRatio || ""); + if ( + size && + Array.isArray(config?.supportedImageSizes) && + !config.supportedImageSizes.includes(size) + ) + continue; + if ( + ratio && + ratio !== "auto" && + Array.isArray(config?.supportedAspectRatios) && + !config.supportedAspectRatios.includes(ratio) + ) + continue; + if ( + input.referenceImageCount && + typeof config?.maxReferenceImages === "number" && + input.referenceImageCount > config.maxReferenceImages + ) + continue; + return entry; + } + return undefined; +} diff --git a/lib/ai/runtime/image-fallback-balance-cooldown.ts b/lib/ai/runtime/image-fallback-balance-cooldown.ts deleted file mode 100644 index 57b2847bc..000000000 --- a/lib/ai/runtime/image-fallback-balance-cooldown.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { isProviderBalanceError } from "@/lib/request-error"; - -const IMAGE_FAILOVER_BALANCE_COOLDOWN_MS = 2 * 60 * 60 * 1000; - -export function isImageFallbackModelBalanceCoolingDown( - model: { - healthLastEventType?: unknown; - healthLastError?: unknown; - healthLastFailureAt?: unknown; - } | null | undefined, - now = Date.now(), -) { - if (model?.healthLastEventType !== "quota" || !isProviderBalanceError(model.healthLastError)) { - return false; - } - const failedAt = new Date(String(model.healthLastFailureAt || "")).getTime(); - return Number.isFinite(failedAt) && now - failedAt < IMAGE_FAILOVER_BALANCE_COOLDOWN_MS && now >= failedAt; -} diff --git a/lib/ai/runtime/model-health.ts b/lib/ai/runtime/model-health.ts index ce46d8f4e..22ab80038 100644 --- a/lib/ai/runtime/model-health.ts +++ b/lib/ai/runtime/model-health.ts @@ -499,6 +499,7 @@ export async function reviveAiModelsFromCooldown(params?: { modelId?: string }) const dueModels = candidates.filter((model) => { if (isCredentialFailureBan(model.healthAutoDisabledReason)) return false; + if (isProviderBalanceError(model.healthAutoDisabledReason)) return false; const cooldownStartedAt = model.healthAutoDisabledAt || model.healthLastFailureAt || @@ -568,6 +569,7 @@ export async function recordAiModelHealthSuccess(params: { healthStatus: aiModel.healthStatus, healthRecoverySuccessStreak: aiModel.healthRecoverySuccessStreak, healthObservationStartedAt: aiModel.healthObservationStartedAt, + healthAutoDisabledReason: aiModel.healthAutoDisabledReason, }) .from(aiModel) .where(eq(aiModel.id, modelId)) @@ -575,6 +577,9 @@ export async function recordAiModelHealthSuccess(params: { if (!existing) return; + // A previously accepted task can finish after the account runs out of credit. + if (isProviderBalanceError(existing.healthAutoDisabledReason)) return; + const now = new Date(); // A request that started before the circuit opened can still prove the // provider is reachable. Treat that success as the circuit's half-open @@ -662,6 +667,7 @@ export async function recordAiModelHealthFailure(params: { healthRecoverySuccessStreak: aiModel.healthRecoverySuccessStreak, healthObservationStartedAt: aiModel.healthObservationStartedAt, healthAutoDisabledAt: aiModel.healthAutoDisabledAt, + healthAutoDisabledReason: aiModel.healthAutoDisabledReason, }) .from(aiModel) .where(eq(aiModel.id, modelId)) @@ -678,13 +684,15 @@ export async function recordAiModelHealthFailure(params: { : existing.healthConsecutiveFailures || 0; const shouldAutoDisable = classification.counted && - !existing.excludeFromFailover && - !existing.excludeFromHealthBan && - (classification.reason === "provider_authentication_failed" || - (isObservation && existing.isEnabled) || - (existing.healthStatus !== "auto_disabled" && - nextConsecutiveFailures >= AUTO_DISABLE_FAILURE_THRESHOLD)); - const nextStatus = shouldAutoDisable + (classification.eventType === "quota" || + (!existing.excludeFromFailover && + !existing.excludeFromHealthBan && + (classification.reason === "provider_authentication_failed" || + (isObservation && existing.isEnabled) || + (existing.healthStatus !== "auto_disabled" && + nextConsecutiveFailures >= AUTO_DISABLE_FAILURE_THRESHOLD)))); + const balanceDisabled = isProviderBalanceError(existing.healthAutoDisabledReason); + const nextStatus = shouldAutoDisable || balanceDisabled ? "auto_disabled" : classification.counted ? "degraded" @@ -737,7 +745,7 @@ export async function recordAiModelHealthFailure(params: { healthUpdatedAt: now, updatedAt: now, }; - if (shouldAutoDisable) { + if (shouldAutoDisable && !balanceDisabled) { updateData.healthAutoDisabledAt = now; updateData.healthAutoDisabledReason = autoDisabledReason || errorMessage; } else if (existing.healthStatus !== "auto_disabled" && !isObservation) { diff --git a/lib/ai/vip98k-task-invocation.ts b/lib/ai/vip98k-task-invocation.ts index a6ed58be3..875847c3b 100644 --- a/lib/ai/vip98k-task-invocation.ts +++ b/lib/ai/vip98k-task-invocation.ts @@ -238,7 +238,10 @@ export function bindVip98kImages( imageUrls: string[], ) { const bindingKeys: string[] = []; - if (imageUrls.length === 0) return bindingKeys; + if (imageUrls.length === 0) { + for (const key of [...IMAGE_ARRAY_INPUT_KEYS, ...IMAGE_SINGLE_INPUT_KEYS, "video_clips"]) delete body[key]; + return bindingKeys; + } for (const key of IMAGE_ARRAY_INPUT_KEYS) { if (!(key in body)) continue; @@ -305,6 +308,7 @@ function scoreReferenceInvocation(invocation: Vip98kTaskInvocation) { export function selectVip98kTaskInvocation(params: { modelId: string; + kind?: string; configuredInvocation: Vip98kTaskInvocation | null; configuredInvocations?: Vip98kTaskInvocation[]; imageCount?: number; @@ -334,6 +338,17 @@ export function selectVip98kTaskInvocation(params: { } } + if (params.kind === "image" && !params.imageCount) { + const textImageTask = candidates.find(candidate => { + const properties = candidate.inputSchema?.properties as Record | undefined; + const required = candidate.inputSchema?.required as string[] | undefined; + return candidate.kind === "task" && candidate.path === "/extensions/v1/tasks" && + invocationTargetsModel(candidate, params.modelId) && properties?.resolution && properties?.aspect_ratio && + !required?.some(key => [...IMAGE_ARRAY_INPUT_KEYS, ...IMAGE_SINGLE_INPUT_KEYS, "video_clips"].includes(key)); + }); + if (textImageTask) return resolveVip98kTaskInvocation(params.modelId, textImageTask); + } + return resolveVip98kTaskInvocation( params.modelId, params.configuredInvocation || candidates[0] || null, diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 46ff05c63..6a9bb6a02 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -6,7 +6,7 @@ import { agentImageCircuitStore } from "@/lib/ai/runtime/agent-model-circuit-sto import { getModelGenerationCreditCost, type DiscountableImageGenConfigLike } from "@/lib/ai/image-model-pricing"; import { agentImageFailurePolicy } from "@/lib/ai/runtime/agent-image-failover"; import { agentImageGenerationError } from "@/lib/ai/runtime/image-generation-error"; -import { getEnabledAiModelsWithProvider } from "@/lib/db/queries"; +import { getAiModelById, getEnabledAiModelsWithProvider } from "@/lib/db/queries"; import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import { createGeminiImage } from "@/lib/ai/providers/gemini-image"; import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image"; @@ -43,7 +43,8 @@ import { shouldFailoverAiModelError, } from "@/lib/ai/runtime/model-health"; import { uploadFile } from "@/lib/storage/service"; -import { isImageFallbackModelBalanceCoolingDown } from "@/lib/ai/runtime/image-fallback-balance-cooldown"; +import { isImageModelBalanceBlocked, resolveConfiguredImageBalanceFallback } from "@/lib/ai/runtime/image-balance"; +import { isProviderBalanceError } from "@/lib/request-error"; export interface CanvasLocalEditCompositeRegion { x: number; @@ -198,6 +199,7 @@ export interface CanvasGenerateImageRequest { agentImageMaxCreditCost?: number; internalFailoverTriedModelRefs?: string[]; internalFailoverTriedProviderTypes?: string[]; + internalBalanceBlockedProviderIds?: string[]; onQueueStatusChange?: ( status: CanvasImageModelQueueStatus & { modelName?: string | null; @@ -4512,7 +4514,7 @@ function findCanvasFailoverModel(params: { !item.model.isEnabled || item.model.healthStatus === "auto_disabled" || item.model.isSubModel || - isImageFallbackModelBalanceCoolingDown(item.model) || + isImageModelBalanceBlocked(item.model) || (item.model.excludeFromFailover && !isSameNameScaleAdjustFallback) ) { return false; @@ -4904,7 +4906,20 @@ export async function generateCanvasImage( ); const allModels = await getEnabledAiModelsWithProvider(); - let imageModels = allModels.filter((m) => m.model.supportsImageGen); + const balanceBlockedProviderIds = new Set(request.internalBalanceBlockedProviderIds || []); + let imageModels = allModels.filter((m) => m.model.supportsImageGen && !isImageModelBalanceBlocked(m.model) && !balanceBlockedProviderIds.has(m.provider.id)); + const requestedModel = allModels.find(entry => entry.model.id === model || entry.model.modelId === model)?.model || + (/^[0-9a-f-]{36}$/i.test(String(model || "")) ? await getAiModelById({ id: model! }) : undefined); + let effectiveModel = model; + if (requestedModel?.supportsImageGen && isImageModelBalanceBlocked(requestedModel)) { + const replacement = request.allowFailover !== false && request.agentImageMode !== "pinned" + ? resolveConfiguredImageBalanceFallback(requestedModel, imageModels, { + imageSize: request.imageSize, aspectRatio: request.aspectRatio, + referenceImageCount: new Set([...(referenceImages || []), referenceImage].filter(Boolean)).size, + }) : undefined; + if (!replacement) throw new Error("当前生图线路余额不足,已暂停新请求;请切换可用模型,已有任务和图片会保留。"); + effectiveModel = replacement.model.id; + } if (request.agentImageMode === "auto") { const states = await Promise.all(imageModels.map(async entry => ({ entry, paused: await agentImageCircuitStore.isPaused(entry) }))); imageModels = states.filter(item => !item.paused && item.entry.model.healthStatus !== "auto_disabled" && @@ -4916,8 +4931,8 @@ export async function generateCanvasImage( throw new Error("暂无可用的图片生成模型"); } - let selectedModel = model - ? imageModels.find((m) => m.model.modelId === model || m.model.id === model) + let selectedModel = effectiveModel + ? imageModels.find((m) => m.model.modelId === effectiveModel || m.model.id === effectiveModel) : imageModels.find((m) => m.model.isDefault) || imageModels[0]; if (!selectedModel) { @@ -5654,7 +5669,7 @@ export async function generateCanvasImage( ? request.internalRetryVariantIndex + 1 : 0; const retryVariant = retryVariants[nextRetryVariantIndex]; - if (retryVariant && !request.agentImageMode) { + if (retryVariant && !request.agentImageMode && !isProviderBalanceError(caughtError)) { const { debugMeta: retryVariantDebugMeta, label: retryVariantLabel, @@ -5710,7 +5725,8 @@ export async function generateCanvasImage( } else if (request.agentImageMode) { const failure = agentImageFailurePolicy(caughtError); if (failure.pause) await agentImageCircuitStore.fail(selectedModel, failure.pause); - } else { + } + if (!scaleAdjustRejected && (!request.agentImageMode || isProviderBalanceError(caughtError))) { await recordAiModelHealthFailure({ model: selectedModel.model, provider, @@ -5756,17 +5772,18 @@ export async function generateCanvasImage( ...(Array.isArray(request.internalFailoverTriedProviderTypes) ? request.internalFailoverTriedProviderTypes : []), - ...(request.agentImageMode !== "auto" && shouldFailover && + ...(request.agentImageMode !== "auto" && !isProviderBalanceError(caughtError) && shouldFailover && failureClassification.reason !== "non_provider_failure" && failureClassification.reason !== "user_cancelled" ? [provider.type] : []), ].filter(Boolean)) ); + if (isProviderBalanceError(caughtError)) balanceBlockedProviderIds.add(provider.id); const fallbackModel = allowFailover && shouldFailover ? findCanvasFailoverModel({ selectedModel, - imageModels, + imageModels: imageModels.filter(entry => !balanceBlockedProviderIds.has(entry.provider.id)), failedModelRefs, failedProviderTypes, preferLowCostSameNameProviders: Boolean( @@ -5797,6 +5814,7 @@ export async function generateCanvasImage( model: fallbackModel.model.id, internalFailoverTriedModelRefs: failedModelRefs, internalFailoverTriedProviderTypes: failedProviderTypes, + internalBalanceBlockedProviderIds: [...balanceBlockedProviderIds], debugMeta: { ...debugMeta, failoverFromModelId: selectedModel.model.modelId, diff --git a/lib/db/queries.ts b/lib/db/queries.ts index 7700792aa..a17ce3514 100644 --- a/lib/db/queries.ts +++ b/lib/db/queries.ts @@ -1,4 +1,5 @@ import "server-only"; +import { isImageModelBalanceBlocked } from "@/lib/ai/runtime/image-balance"; import { resolveSiteBranding } from "@/lib/site-branding"; import { resolveArchivedImageUrls } from "@/lib/storage/image-archive-db"; @@ -4555,7 +4556,7 @@ export async function getEnabledAiModelsWithProvider( return 3; }; - return [...models].sort((left, right) => { + return models.filter(({ model }) => !model.supportsImageGen || !isImageModelBalanceBlocked(model)).sort((left, right) => { const byHealth = healthPriority(left.model.healthStatus) - healthPriority(right.model.healthStatus); diff --git a/playwright.helpers.config.ts b/playwright.helpers.config.ts index c1a33aad8..1b69fb7b7 100644 --- a/playwright.helpers.config.ts +++ b/playwright.helpers.config.ts @@ -8,6 +8,7 @@ export default defineConfig({ /routes\/video-canvas-media-import\.test\.ts/, /routes\/video-canvas-toolbar-jobs\.test\.ts/, /routes\/image-provider-error-classification\.test\.ts/, + /routes\/canvas-agent-runtime-image-failover\.test\.ts/, /routes\/video-canvas-audio-remote\.test\.ts/, /routes\/video-canvas-groups-upscale\.test\.ts/, /routes\/tvstar-creative-analysis\.test\.ts/, diff --git a/tests/helpers/agent-image-provider-harness.ts b/tests/helpers/agent-image-provider-harness.ts index 3f6805b4f..95c0b8233 100644 --- a/tests/helpers/agent-image-provider-harness.ts +++ b/tests/helpers/agent-image-provider-harness.ts @@ -6,7 +6,7 @@ export function agentImageProviderHarness(options: { generate?: (id: string, cou const models = options.models || ["primary", "backup"].map((id, i) => ({ model: { id, modelId: id, displayName: id, isEnabled: true, supportsImageGen: true, supportsVision: true, isDefault: !i, imageGenConfig: { maxImages: 12 }, capabilities: { kind: "image" } }, provider: { id: "same-provider", type: "openai-image", apiKey: "synthetic", isEnabled: true } })); const mocks: Record = { "@/lib/ai/runtime/image-generation-error": generationErrors, - "@/lib/db/queries": { getEnabledAiModelsWithProvider: async () => models }, + "@/lib/db/queries": { getEnabledAiModelsWithProvider: async () => models.filter(entry => entry.model.isEnabled), getAiModelById: async ({ id }: { id: string }) => models.find(entry => entry.model.id === id)?.model }, "@/lib/ai/rmbg/service": {}, "@/lib/net/url-access": { filterReachableRemoteUrls: async (urls: string[]) => ({ kept: urls, dropped: [] }) }, "@/lib/canvas/image-concurrency": { acquireCanvasImageModelSlot: async () => ({ waitMs: 0, maxConcurrent: 999, release: async () => {} }), normalizeCanvasImageModelKey: (v: string) => v, recordCanvasImageModelExecutionDuration() {} }, diff --git a/tests/routes/canvas-agent-runtime-image-failover.test.ts b/tests/routes/canvas-agent-runtime-image-failover.test.ts index 78530d06c..f17cd29a8 100644 --- a/tests/routes/canvas-agent-runtime-image-failover.test.ts +++ b/tests/routes/canvas-agent-runtime-image-failover.test.ts @@ -5,6 +5,33 @@ import { ImageSubmissionRejectedError, agentImageGenerationError } from "@/lib/a import * as generationErrors from "@/lib/ai/runtime/image-generation-error"; import { serverLoader } from "../helpers/canvas-download-server-loader"; +test("a balance rejection skips other models on the same account but permits another account of the same provider type", async () => { + const models = ["primary", "same-account", "backup"].map(id => ({ + model: { id, modelId: id, displayName: id, isEnabled: true, supportsImageGen: true, supportsVision: true, imageGenConfig: { maxImages: 1 }, capabilities: { kind: "image" } }, + provider: { id: id === "backup" ? "funded-account" : "exhausted-account", type: "openai-image", apiKey: "synthetic", isEnabled: true }, + })); + const h = harness({ models, generate: id => { + if (id !== "backup") throw new Error("余额不足,请充值 (HTTP 429)"); + return "![image](https://fixture.invalid/ok.png)"; + } }); + expect((await h.generate(h.request)).modelRef).toBe("backup"); + expect(h.calls.map(call => call.id)).toEqual(["primary", "backup"]); +}); + +test("an old disabled balance selection uses its configured compatible replacement without calling the exhausted account", async () => { + const oldId = "11111111-1111-4111-8111-111111111111"; + const models = [ + { model: { id: oldId, modelId: "old", displayName: "old", isEnabled: false, supportsImageGen: true, healthLastEventType: "quota", healthLastError: "insufficient balance", fallbackModelId1: "backup" }, provider: { id: "exhausted-account", type: "openai-image", isEnabled: true } }, + { model: { id: "backup", modelId: "backup", displayName: "backup", isEnabled: true, supportsImageGen: true, supportsVision: true, imageGenConfig: { maxImages: 1, maxReferenceImages: 8, supportedImageSizes: ["2K"], supportedAspectRatios: ["3:4"] } }, provider: { id: "funded-account", type: "openai-image", apiKey: "synthetic", isEnabled: true } }, + ]; + const h = harness({ models }); + expect((await h.generate({ ...h.request, model: oldId })).modelRef).toBe("backup"); + expect(h.calls.map(call => call.id)).toEqual(["backup"]); + await expect(h.generate({ ...h.request, model: oldId, agentImageMode: "pinned" })).rejects.toThrow("余额不足"); + await expect(h.generate({ ...h.request, model: oldId, referenceImages: Array.from({ length: 9 }, (_, i) => `https://fixture.invalid/${i}.png`) })).rejects.toThrow("余额不足"); + expect(h.calls).toHaveLength(1); +}); + test("automatic image generation switches within the same provider and skips a paused model on the next task", async () => { const h = harness({ generate: id => { if (id === "primary") throw Object.assign(new Error("channel unavailable"), { status: 503 }); return "![image](https://fixture.invalid/ok.png)"; } }); diff --git a/tests/routes/image-provider-error-classification.test.ts b/tests/routes/image-provider-error-classification.test.ts index 978f291fc..65a4480fb 100644 --- a/tests/routes/image-provider-error-classification.test.ts +++ b/tests/routes/image-provider-error-classification.test.ts @@ -5,7 +5,8 @@ import { isProviderBalanceError, isRateLimitError, } from "@/lib/request-error"; -import { isImageFallbackModelBalanceCoolingDown } from "@/lib/ai/runtime/image-fallback-balance-cooldown"; +import { isImageModelBalanceBlocked, resolveConfiguredImageBalanceFallback } from "@/lib/ai/runtime/image-balance"; +import { serverLoader } from "../helpers/canvas-download-server-loader"; test("a bare HTTP 429 is rate limiting, not insufficient balance", () => { const error = Object.assign(new Error("Too many requests (HTTP 429)"), { status: 429 }); @@ -34,14 +35,66 @@ test("nested billing details and generic quota limits are kept distinct", () => expect(isProviderBalanceError(new Error("该模型额度不足(HTTP 429)"))).toBe(true); }); -test("a recent confirmed provider balance failure skips only fallback selection", () => { - const now = Date.now(); +test("confirmed balance failures remain isolated until explicitly restored", () => { const model = { healthLastEventType: "quota", healthLastError: "余额不足(HTTP 429)", - healthLastFailureAt: new Date(now - 30_000), }; - expect(isImageFallbackModelBalanceCoolingDown(model, now)).toBe(true); - expect(isImageFallbackModelBalanceCoolingDown({ ...model, healthLastFailureAt: new Date(now - 3 * 60 * 60 * 1000) }, now)).toBe(false); - expect(isImageFallbackModelBalanceCoolingDown({ ...model, healthLastEventType: "success" }, now)).toBe(false); + expect(isImageModelBalanceBlocked(model)).toBe(true); + expect(isImageModelBalanceBlocked({ healthLastEventType: "error", healthAutoDisabledReason: "provider_balance_or_billing_quota: 余额不足" })).toBe(true); + expect(isImageModelBalanceBlocked({ ...model, healthLastEventType: "success", healthLastError: null })).toBe(false); +}); + +test("old selections use only configured, enabled replacements that preserve the requested inputs", () => { + const requested = { healthLastEventType: "quota", healthLastError: "余额不足", fallbackModelId1: "verified" }; + const verified = { model: { id: "verified", modelId: "same-family", isEnabled: true, supportsImageGen: true, + imageGenConfig: { supportedImageSizes: ["2K"], supportedAspectRatios: ["1:1"], maxReferenceImages: 8 } }, provider: { isEnabled: true } }; + const input = { imageSize: "2K", aspectRatio: "1:1", referenceImageCount: 2 }; + expect(resolveConfiguredImageBalanceFallback(requested, [verified], input)).toBe(verified); + expect(resolveConfiguredImageBalanceFallback(requested, [verified], { ...input, referenceImageCount: 9 })).toBeUndefined(); + expect(resolveConfiguredImageBalanceFallback(requested, [verified], { ...input, imageSize: "4K" })).toBeUndefined(); + expect(resolveConfiguredImageBalanceFallback(requested, [verified], { ...input, aspectRatio: "9:16" })).toBeUndefined(); + expect(resolveConfiguredImageBalanceFallback(requested, [{ ...verified, provider: { isEnabled: false } }], input)).toBeUndefined(); + expect(resolveConfiguredImageBalanceFallback({ ...requested, fallbackModelId1: null }, [verified], input)).toBeUndefined(); +}); + +function healthHarness() { + let model: any = { id: "test", modelId: "test", isEnabled: true, excludeFromFailover: true, + excludeFromHealthBan: true, healthStatus: "healthy", healthConsecutiveFailures: 0 }; + const events: any[] = []; + const query = (rows: () => any[]) => ({ from() { return this; }, where() { return this; }, + limit: async () => rows(), then: (resolve: any) => Promise.resolve(rows()).then(resolve), returning: async () => rows() }); + const db = { + select: () => query(() => [model]), + insert: () => ({ values: async (value: any) => { events.push(value); } }), + update: () => ({ set: (value: any) => { model = { ...model, ...value }; return query(() => [model]); } }), + }; + const health = serverLoader({ "@/lib/db": { db } })("lib/ai/runtime/model-health.ts"); + return { health, events, get model() { return model; }, age() { model.healthAutoDisabledAt = new Date(Date.now() - 4 * 60 * 60_000); } }; +} + +test("quota pauses an exempt model, survives cooldown and late completion, and can be restored", async () => { + const h = healthHarness(); + await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("余额不足(HTTP 429)") }); + expect(h.model.isEnabled).toBe(false); + expect(h.model.healthStatus).toBe("auto_disabled"); + expect(h.events[0].eventType).toBe("quota"); + await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("HTTP 503 unavailable") }); + await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("Too many requests (HTTP 429)") }); + expect(h.model.healthStatus).toBe("auto_disabled"); + expect(isImageModelBalanceBlocked(h.model)).toBe(true); + h.age(); + expect(await h.health.reviveAiModelsFromCooldown()).toEqual([]); + await h.health.recordAiModelHealthSuccess({ model: h.model }); + expect(h.model.isEnabled).toBe(false); + await h.health.restoreAiModelHealth(h.model.id); + expect(h.model.isEnabled).toBe(true); + expect(isImageModelBalanceBlocked(h.model)).toBe(false); +}); + +test("ordinary rate limiting does not trigger billing isolation", async () => { + const h = healthHarness(); + await h.health.recordAiModelHealthFailure({ model: h.model, error: new Error("Too many requests (HTTP 429)") }); + expect(h.model.isEnabled).toBe(true); + expect(isImageModelBalanceBlocked(h.model)).toBe(false); }); diff --git a/tests/routes/vip98k-reference-routing.test.ts b/tests/routes/vip98k-reference-routing.test.ts index f36076c54..6f807d1b9 100644 --- a/tests/routes/vip98k-reference-routing.test.ts +++ b/tests/routes/vip98k-reference-routing.test.ts @@ -54,6 +54,21 @@ const textToImageInvocation: Vip98kTaskInvocation = { "/api/external/tools/result?tool_id=google%2Fnano-banana-2%2Ftext-to-image&taskId={task_id}", }; +test("text-to-image uses the advertised task contract with native size and ratio, without example reference images", () => { + const configured: Vip98kTaskInvocation = { ...textToImageInvocation, path: "/v1/images/generations", requestExample: { model: "nano-banana-pro", prompt: "sample" } }; + const task: Vip98kTaskInvocation = { ...configured, kind: "task", path: "/extensions/v1/tasks", + requestExample: { model: "nano-banana-pro", prompt: "sample", file_urls: ["https://provider.test/logo.png"] }, + inputSchema: { required: ["model"], properties: { resolution: { type: "string" }, aspect_ratio: { type: "string" }, file_urls: { type: "array" } } }, + }; + const invocation = selectVip98kTaskInvocation({ modelId: "nano-banana-pro", kind: "image", configuredInvocation: configured, configuredInvocations: [configured, task], imageCount: 0 }); + expect(invocation?.path).toBe("/extensions/v1/tasks"); + const result = buildTaskBody({ apiKey: "test", modelId: "nano-banana-pro", kind: "image", prompt: "a blue mug", imageUrls: [], videoUrls: [], modelConfig: {}, imageParams: { imageSize: "2K", aspectRatio: "3:4" } }, invocation); + expect(result.body).toMatchObject({ prompt: "a blue mug", resolution: "2K", aspect_ratio: "3:4" }); + expect(result.body).not.toHaveProperty("file_urls"); + expect(result.outputConstraintFallbacks).toEqual([]); + expect(selectVip98kTaskInvocation({ modelId: "nano-banana-pro", kind: "image", configuredInvocation: configured, configuredInvocations: [configured, { ...task, inputSchema: { ...task.inputSchema, required: ["model", "file_urls"] } }], imageCount: 0 })?.path).toBe("/v1/images/generations"); +}); + test("forwards six requested outputs independently of reference count", () => { const invocation = selectVip98kTaskInvocation({ modelId: "gpt-image-2.5", configuredInvocation: null, imageCount: 2 }); const { body } = buildTaskBody({