From e7cd3d6e73ca7ad27889e09fed08bb2b14e7a672 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 30 Jun 2026 14:38:37 +0800 Subject: [PATCH] fix: recognize BFL reference image support --- app/(chat)/api/chat/_lib/image-failover.ts | 6 +- app/api/canvas/local-edit/route.ts | 11 +++- app/api/scene-fusion/route.ts | 3 +- lib/ai/model-router.ts | 23 ++++--- lib/ai/reference-image-capability.ts | 72 ++++++++++++++++++++++ lib/canvas/generate-image.ts | 18 +++++- 6 files changed, 119 insertions(+), 14 deletions(-) create mode 100644 lib/ai/reference-image-capability.ts diff --git a/app/(chat)/api/chat/_lib/image-failover.ts b/app/(chat)/api/chat/_lib/image-failover.ts index c88298c24..a94303d04 100644 --- a/app/(chat)/api/chat/_lib/image-failover.ts +++ b/app/(chat)/api/chat/_lib/image-failover.ts @@ -1,6 +1,7 @@ import { createLogger } from "@/lib/observability/logger"; import { isFalGptImage2EditModelId } from "@/lib/ai/providers/fal-image"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; +import { modelLacksReferenceImageInputSupport } from "@/lib/ai/reference-image-capability"; import type { ChatMessage } from "@/lib/types"; import { convertMessagesWithImages } from "./image-media"; @@ -370,7 +371,10 @@ export async function generateImageWithFailoverCandidate(params: { const { candidate, messages, imageParams, fallbackImageParams } = params; const providerType = String(candidate.provider?.type || ""); const candidateRuntimeModelId = resolveImageModelRuntimeId(candidate.model); - if (chatMessagesHaveImageInput(messages) && candidate.model?.supportsVision === false) { + if ( + chatMessagesHaveImageInput(messages) && + modelLacksReferenceImageInputSupport(candidate.model, candidate.provider) + ) { throw new Error( `备用模型不支持参考图/图生图输入：${candidate.model.displayName || candidate.model.id}` ); diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index a9fb65c47..06184eba8 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -14,6 +14,10 @@ import { getPointsSystemConfig, getSiteSettings, } from "@/lib/db/queries"; +import { + modelLacksReferenceImageInputSupport, + modelSupportsReferenceImageInput, +} from "@/lib/ai/reference-image-capability"; import { getImageToolCreditCost } from "@/lib/points/config"; import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; import { uploadFile } from "@/lib/storage/service"; @@ -217,7 +221,10 @@ function findModelSelectionEntry( function isScaleAdjustDedicatedDetailModel( entry: AiModelWithProviderForSelection | null | undefined ) { - if (!entry?.model?.supportsImageGen || entry.model.supportsVision === false) { + if ( + !entry?.model?.supportsImageGen || + modelLacksReferenceImageInputSupport(entry.model, entry.provider) + ) { return false; } const label = `${entry.model.displayName || ""} ${entry.model.modelId || ""}` @@ -237,7 +244,7 @@ function isHealthyImageModelEntry( ) { return ( Boolean(entry?.model?.supportsImageGen) && - entry?.model?.supportsVision !== false && + modelSupportsReferenceImageInput(entry?.model, entry?.provider) && entry?.model?.healthStatus === "healthy" ); } diff --git a/app/api/scene-fusion/route.ts b/app/api/scene-fusion/route.ts index 9c0240f2b..d63a9c2b8 100644 --- a/app/api/scene-fusion/route.ts +++ b/app/api/scene-fusion/route.ts @@ -35,6 +35,7 @@ import { import { acquireCanvasImageModelSlot } from "@/lib/canvas/image-concurrency"; import { createLogger } from "@/lib/observability/logger"; import { resolveModelRuntimeId } from "@/lib/ai/model-runtime"; +import { modelSupportsReferenceImageInput } from "@/lib/ai/reference-image-capability"; import { resolveLongestEdgeDimensions } from "@/lib/ai/image-size-constraints"; // 请求体 Schema @@ -905,7 +906,7 @@ function buildSceneFusionCandidatePool( id: String(entry.model.id), modelId: String(entry.model.modelId || ""), displayName: String(entry.model.displayName || entry.model.modelId || entry.model.id), - supportsVision: Boolean(entry.model.supportsVision), + supportsVision: modelSupportsReferenceImageInput(entry.model, entry.provider), imageGenConfig: (entry.model.imageGenConfig || null) as SceneFusionImageGenConfig | null, excludeFromFailover: Boolean(entry.model.excludeFromFailover), defaultParams: (entry.model.defaultParams as Record | null) || null, diff --git a/lib/ai/model-router.ts b/lib/ai/model-router.ts index 7e68bdc80..89b6622e8 100644 --- a/lib/ai/model-router.ts +++ b/lib/ai/model-router.ts @@ -10,6 +10,7 @@ import { isImageGenerationModelDescriptor, isVideoModelDescriptor, } from "@/lib/ai/model-classification"; +import { isKnownReferenceImageModel } from "@/lib/ai/reference-image-capability"; type EnabledModelWithProvider = Awaited< ReturnType @@ -31,8 +32,13 @@ function hasVisionCapability(model: { supportsVision?: boolean | null; capabilities?: unknown; defaultParams?: unknown; -}) { - return Boolean(model.supportsVision) || isAiModelMultimodal(model); + modelId?: unknown; +}, providerType?: string | null) { + return ( + Boolean(model.supportsVision) || + isKnownReferenceImageModel(model, { type: providerType }) || + isAiModelMultimodal(model) + ); } const CHAT_PROVIDER_TYPES = [ @@ -86,7 +92,7 @@ function isConversationalCandidate( const matchesPreferredProvider = CHAT_PROVIDER_TYPES.includes( providerType as (typeof CHAT_PROVIDER_TYPES)[number], ); - const hasVision = hasVisionCapability(item.model); + const hasVision = hasVisionCapability(item.model, providerType); if (matchesPreferredProvider) { return requireVision ? hasVision : true; @@ -120,7 +126,7 @@ function matchesIntentRequirements( if (params.intent === "vision") { return ( !isVideoProviderType(String(item.provider.type || ""), item.model) && - hasVisionCapability(item.model) + hasVisionCapability(item.model, item.provider.type) ); } @@ -128,7 +134,7 @@ function matchesIntentRequirements( return isVideoProviderType(String(item.provider.type || ""), item.model); } - if (params.requireVision && !hasVisionCapability(item.model)) return false; + if (params.requireVision && !hasVisionCapability(item.model, item.provider.type)) return false; if ( params.requireImageGen && !isImageGenerationModelDescriptor({ @@ -271,7 +277,9 @@ export async function routeToModel(params: { if (configuredModel) { const isImageIntent = intent === "image_edit" || intent === "image_gen"; - const matchesVision = !config.requireVision || hasVisionCapability(configuredModel.model); + const matchesVision = + !config.requireVision || + hasVisionCapability(configuredModel.model, configuredModel.provider.type); const matchesImageGen = !config.requireImageGen || configuredModel.model.supportsImageGen; const allowImageIntentWithImageGenOnly = isImageIntent && configuredModel.model.supportsImageGen; @@ -347,7 +355,8 @@ export async function routeToModel(params: { !item.model.isSubModel && item.model.isEnabled && !isVideoProviderType(String(item.provider.type || ""), item.model) && - (item.model.supportsImageGen || hasVisionCapability(item.model)) + (item.model.supportsImageGen || + hasVisionCapability(item.model, item.provider.type)) ); }); diff --git a/lib/ai/reference-image-capability.ts b/lib/ai/reference-image-capability.ts new file mode 100644 index 000000000..d29d8b925 --- /dev/null +++ b/lib/ai/reference-image-capability.ts @@ -0,0 +1,72 @@ +type ModelDescriptor = { + modelId?: unknown; + supportsVision?: boolean | null; + capabilities?: unknown; + defaultParams?: unknown; +}; + +type ProviderDescriptor = { + type?: unknown; +}; + +const BFL_REFERENCE_IMAGE_MODEL_REGEX = + /^(?:bfl\/)?(?:flux-2-(?:max|flex|pro(?:-preview)?|klein-(?:4b|9b)(?:-preview)?)|flux-kontext-(?:pro|max))$/i; + +function readStringField(source: unknown, key: string) { + if (!source || typeof source !== "object" || !(key in source)) return ""; + return String((source as Record)[key] || "").trim(); +} + +function normalizeBflModelId(value: unknown) { + const trimmed = String(value || "").trim(); + if (!trimmed) return ""; + + return trimmed + .replace(/^https?:\/\/[^/]+/i, "") + .replace(/^\/?(?:bfl\/)?v1\//i, "") + .replace(/^\/+/, "") + .replace(/[?#].*$/, ""); +} + +export function isKnownReferenceImageModel( + model: ModelDescriptor | null | undefined, + provider?: ProviderDescriptor | null +) { + const providerType = String(provider?.type || "").trim(); + if (providerType !== "bfl-image") return false; + + const candidates = [ + model?.modelId, + readStringField(model?.defaultParams, "endpoint"), + readStringField(model?.defaultParams, "submitEndpoint"), + readStringField(model?.capabilities, "endpoint"), + ]; + + const capabilitiesEndpoints = + model?.capabilities && + typeof model.capabilities === "object" && + "endpoints" in model.capabilities + ? (model.capabilities as { endpoints?: unknown }).endpoints + : undefined; + + candidates.push(readStringField(capabilitiesEndpoints, "submit")); + + return candidates + .map(normalizeBflModelId) + .some((modelId) => BFL_REFERENCE_IMAGE_MODEL_REGEX.test(modelId)); +} + +export function modelLacksReferenceImageInputSupport( + model: ModelDescriptor | null | undefined, + provider?: ProviderDescriptor | null +) { + if (isKnownReferenceImageModel(model, provider)) return false; + return model?.supportsVision === false; +} + +export function modelSupportsReferenceImageInput( + model: ModelDescriptor | null | undefined, + provider?: ProviderDescriptor | null +) { + return !modelLacksReferenceImageInputSupport(model, provider); +} diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index a44d64ab8..ca06d0ecc 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -16,6 +16,7 @@ import { createRunway } from "@/lib/ai/providers/runway"; import { createKrea } from "@/lib/ai/providers/krea"; import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision"; import { resolveModelRuntimeId } from "@/lib/ai/model-runtime"; +import { modelLacksReferenceImageInputSupport } from "@/lib/ai/reference-image-capability"; import { filterReachableRemoteUrls } from "@/lib/net/url-access"; import { acquireCanvasImageModelSlot, @@ -1766,7 +1767,10 @@ function findCanvasFailoverModel(params: { return false; } if (!item.model.supportsImageGen) return false; - if (params.requiresImageInput && item.model.supportsVision === false) { + if ( + params.requiresImageInput && + modelLacksReferenceImageInputSupport(item.model, item.provider) + ) { return false; } if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(item.provider.type)) return false; @@ -1884,7 +1888,10 @@ export async function generateCanvasImage( let provider = selectedModel.provider; - if (requiresImageInput && selectedModel.model.supportsVision === false) { + if ( + requiresImageInput && + modelLacksReferenceImageInputSupport(selectedModel.model, selectedModel.provider) + ) { if (request.allowFailover === false) { throw new Error("当前图片模型不支持参考图/图生图，请选择支持图像输入的生图模型。"); } @@ -1925,7 +1932,12 @@ export async function generateCanvasImage( for (const candidate of imageModels) { const candidateProvider = candidate.provider; if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(candidateProvider.type)) continue; - if (requiresImageInput && candidate.model.supportsVision === false) continue; + if ( + requiresImageInput && + modelLacksReferenceImageInputSupport(candidate.model, candidate.provider) + ) { + continue; + } fallbackModel = candidate; fallbackProvider = candidateProvider; break;