diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 67eaff122..81b2eb029 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -27,6 +27,7 @@ import { recordCanvasImageModelExecutionDuration, } from "./image-concurrency"; import { + classifyAiModelHealthFailure, recordAiModelHealthFailure, recordAiModelHealthSuccess, shouldFailoverAiModelError, @@ -126,6 +127,7 @@ export interface CanvasGenerateImageRequest { internalRetryVariantIndex?: number; allowFailover?: boolean; internalFailoverTriedModelRefs?: string[]; + internalFailoverTriedProviderTypes?: string[]; onQueueStatusChange?: ( status: CanvasImageModelQueueStatus & { modelName?: string | null; @@ -3320,6 +3322,7 @@ function findCanvasFailoverModel(params: { selectedModel: CanvasModelWithProvider; imageModels: CanvasModelWithProvider[]; failedModelRefs: string[]; + failedProviderTypes?: string[]; preferLowCostSameNameProviders?: boolean; requiresImageInput?: boolean; requiresTextToImage?: boolean; @@ -3328,6 +3331,11 @@ function findCanvasFailoverModel(params: { const failedRefs = new Set( params.failedModelRefs.map((item) => String(item || "").trim()).filter(Boolean) ); + const failedProviderTypes = new Set( + (params.failedProviderTypes || []) + .map((item) => String(item || "").trim()) + .filter(Boolean) + ); const strictScaleAdjustModelFamily = params.preferLowCostSameNameProviders && typeof params.scaleAdjustModelFamily === "string" && @@ -3360,6 +3368,7 @@ function findCanvasFailoverModel(params: { ) { return false; } + if (failedProviderTypes.has(item.provider.type)) return false; if (!item.model.supportsImageGen) return false; if ( params.requiresImageInput && @@ -4210,11 +4219,25 @@ export async function generateCanvasImage( (!scaleAdjustRejected && Boolean(debugMeta?.detailFixedModelApplied) && Boolean(caughtError)); + const failureClassification = classifyAiModelHealthFailure(caughtError); + const failedProviderTypes = Array.from( + new Set([ + ...(Array.isArray(request.internalFailoverTriedProviderTypes) + ? request.internalFailoverTriedProviderTypes + : []), + ...(shouldFailover && + failureClassification.reason !== "non_provider_failure" && + failureClassification.reason !== "user_cancelled" + ? [provider.type] + : []), + ].filter(Boolean)) + ); const fallbackModel = allowFailover && shouldFailover ? findCanvasFailoverModel({ selectedModel, imageModels, failedModelRefs, + failedProviderTypes, preferLowCostSameNameProviders: Boolean( debugMeta?.scaleAdjustPreferSameNameProviderFailover ), @@ -4240,6 +4263,7 @@ export async function generateCanvasImage( ...request, model: fallbackModel.model.id, internalFailoverTriedModelRefs: failedModelRefs, + internalFailoverTriedProviderTypes: failedProviderTypes, debugMeta: { ...debugMeta, failoverFromModelId: selectedModel.model.modelId,