diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 4070b7f0c..c94249a2f 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -2394,6 +2394,15 @@ export function CanvasChatPanel({ return next; }); }, []); + const clearSettledAgentProgressStateForChat = useCallback((chatSessionId: string) => { + setAgentProgressStateByChatId((prev) => { + const current = prev[chatSessionId]; + if (!current || current.stage === "clarify") return prev; + const next = { ...prev }; + delete next[chatSessionId]; + return next; + }); + }, []); const shouldBlockDuplicateSend = useCallback((chatSessionId: string, signature: string) => { const now = Date.now(); @@ -6813,6 +6822,7 @@ export function CanvasChatPanel({ } catch (error) { console.error("[CanvasChat] Failed to sync messages after finish:", error); } finally { + clearSettledAgentProgressStateForChat(finishedChatId); if (inFlightChatIdRef.current === finishedChatId) { inFlightChatIdRef.current = null; inFlightRawChatIdRef.current = null; @@ -12143,7 +12153,10 @@ export function CanvasChatPanel({ const isAgentTargetClarificationActive = shouldShowAgentTargetLockCard && activeAgentTargetLock?.stage === "clarify"; const shouldShowAgentProgressCard = - activeComposerMode === "agent" && Boolean(activeAgentProgress); + activeComposerMode === "agent" && + Boolean(activeAgentProgress) && + activeAgentProgress?.stage !== "completed" && + activeAgentProgress?.stage !== "failed"; const agentTargetLockPreviewItems = useMemo(() => { if (!activeAgentTargetLock) return [] as AgentTargetLockRefPreview[]; const targets = Array.isArray(activeAgentTargetLock.targets) ? activeAgentTargetLock.targets : []; diff --git a/lib/ai/model-classification.ts b/lib/ai/model-classification.ts index cd81668c8..75fd94722 100644 --- a/lib/ai/model-classification.ts +++ b/lib/ai/model-classification.ts @@ -49,10 +49,55 @@ const VIDEO_MODE_KEYS = new Set([ "reference-images", ]); +const TEXT_TO_IMAGE_MODE_KEYS = new Set([ + "text-to-image", + "text2image", + "txt2img", + "image-generation", + "image_generate", + "generate-image", +]); + +const IMAGE_EDIT_ONLY_KIND_KEYS = new Set([ + "image-edit", + "image-to-image", + "img2img", + "reference-image", + "reference-images", +]); + +const TEXT_TO_IMAGE_KIND_KEYS = new Set([ + "text-to-image", + "text2image", + "txt2img", + "image", + "image-generation", + "image_generate", +]); + function normalizeText(value?: string | null) { return String(value || "").trim().toLowerCase(); } +function toRecord(value: unknown) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + return value as Record; +} + +function getStringArrayField(source: unknown, key: string) { + const record = toRecord(source); + const value = record?.[key]; + if (!Array.isArray(value)) return [] as string[]; + return value.map((item) => normalizeText(String(item || ""))).filter(Boolean); +} + +function getStringField(source: unknown, key: string) { + const record = toRecord(source); + return normalizeText(String(record?.[key] || "")); +} + function hasVideoModes(capabilities: unknown) { if (!capabilities || typeof capabilities !== "object") return false; const supportedModes = (capabilities as { supportedModes?: unknown }).supportedModes; @@ -131,3 +176,63 @@ export function isImageGenerationModelDescriptor(params: { return STATIC_IMAGE_PROVIDER_TYPES.has(providerType); } + +export function modelSupportsTextToImageDescriptor(params: { + providerType?: string | null; + modelId?: string | null; + capabilities?: unknown; + defaultParams?: unknown; + supportsImageGen?: boolean | null; +}): boolean { + if ( + !isImageGenerationModelDescriptor({ + providerType: params.providerType, + modelId: params.modelId, + capabilities: params.capabilities, + supportsImageGen: params.supportsImageGen, + }) + ) { + return false; + } + + const providerType = normalizeText(params.providerType); + const modelId = normalizeText(params.modelId); + const capabilityModes = getStringArrayField(params.capabilities, "supportedModes"); + const defaultModes = getStringArrayField(params.defaultParams, "supportedModes"); + const modes = new Set([...capabilityModes, ...defaultModes]); + + if ([...modes].some((mode) => TEXT_TO_IMAGE_MODE_KEYS.has(mode))) { + return true; + } + + const kind = + getStringField(params.capabilities, "kind") || + getStringField(params.defaultParams, "kind"); + if (TEXT_TO_IMAGE_KIND_KEYS.has(kind)) { + return true; + } + + if (providerType === "stepfun-image") { + return true; + } + + const runtimeEndpoint = + getStringField(params.capabilities, "endpoint") || + getStringField(params.defaultParams, "endpoint"); + const runtimeModelText = `${modelId} ${runtimeEndpoint}`.trim(); + + if ( + IMAGE_EDIT_ONLY_KIND_KEYS.has(kind) || + [...modes].some((mode) => IMAGE_EDIT_ONLY_KIND_KEYS.has(mode)) || + runtimeModelText.endsWith("/edit") || + runtimeModelText.includes("image-edit") + ) { + return false; + } + + if (providerType === "alibaba-image") { + return false; + } + + return true; +} diff --git a/lib/ai/model-router.ts b/lib/ai/model-router.ts index 89b6622e8..3794956b3 100644 --- a/lib/ai/model-router.ts +++ b/lib/ai/model-router.ts @@ -8,6 +8,7 @@ import { isAiModelMultimodal } from "@/lib/db/ai-model-flags"; import { isImageGenerationModelDescriptor, + modelSupportsTextToImageDescriptor, isVideoModelDescriptor, } from "@/lib/ai/model-classification"; import { isKnownReferenceImageModel } from "@/lib/ai/reference-image-capability"; @@ -137,12 +138,20 @@ function matchesIntentRequirements( if (params.requireVision && !hasVisionCapability(item.model, item.provider.type)) return false; if ( params.requireImageGen && - !isImageGenerationModelDescriptor({ + !(params.intent === "image_gen" + ? modelSupportsTextToImageDescriptor({ + providerType: item.provider.type, + modelId: item.model.modelId, + capabilities: item.model.capabilities, + defaultParams: item.model.defaultParams, + supportsImageGen: item.model.supportsImageGen, + }) + : isImageGenerationModelDescriptor({ providerType: item.provider.type, modelId: item.model.modelId, capabilities: item.model.capabilities, supportsImageGen: item.model.supportsImageGen, - }) + })) ) { return false; } @@ -280,9 +289,20 @@ export async function routeToModel(params: { const matchesVision = !config.requireVision || hasVisionCapability(configuredModel.model, configuredModel.provider.type); - const matchesImageGen = !config.requireImageGen || configuredModel.model.supportsImageGen; + const configuredModelSupportsRequestedImageGeneration = + intent === "image_gen" + ? modelSupportsTextToImageDescriptor({ + providerType: configuredModel.provider.type, + modelId: configuredModel.model.modelId, + capabilities: configuredModel.model.capabilities, + defaultParams: configuredModel.model.defaultParams, + supportsImageGen: configuredModel.model.supportsImageGen, + }) + : configuredModel.model.supportsImageGen; + const matchesImageGen = + !config.requireImageGen || configuredModelSupportsRequestedImageGeneration; const allowImageIntentWithImageGenOnly = - isImageIntent && configuredModel.model.supportsImageGen; + isImageIntent && configuredModelSupportsRequestedImageGeneration; if ((matchesVision && matchesImageGen) || allowImageIntentWithImageGenOnly) { const reason = allowImageIntentWithImageGenOnly && !matchesVision @@ -355,7 +375,15 @@ export async function routeToModel(params: { !item.model.isSubModel && item.model.isEnabled && !isVideoProviderType(String(item.provider.type || ""), item.model) && - (item.model.supportsImageGen || + ((intent === "image_gen" + ? modelSupportsTextToImageDescriptor({ + providerType: item.provider.type, + modelId: item.model.modelId, + capabilities: item.model.capabilities, + defaultParams: item.model.defaultParams, + supportsImageGen: item.model.supportsImageGen, + }) + : item.model.supportsImageGen) || hasVisionCapability(item.model, item.provider.type)) ); }); diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index d3906f7ef..49cf5f0bd 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -16,6 +16,7 @@ import { createRunway } from "@/lib/ai/providers/runway"; import { createKrea } from "@/lib/ai/providers/krea"; import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision"; import { resolveModelRuntimeId } from "@/lib/ai/model-runtime"; +import { modelSupportsTextToImageDescriptor } from "@/lib/ai/model-classification"; import { modelLacksReferenceImageInputSupport } from "@/lib/ai/reference-image-capability"; import { filterReachableRemoteUrls } from "@/lib/net/url-access"; import { @@ -2647,6 +2648,7 @@ function findCanvasFailoverModel(params: { failedModelRefs: string[]; preferLowCostSameNameProviders?: boolean; requiresImageInput?: boolean; + requiresTextToImage?: boolean; scaleAdjustModelFamily?: string | null; }) { const failedRefs = new Set( @@ -2691,6 +2693,18 @@ function findCanvasFailoverModel(params: { ) { return false; } + if ( + params.requiresTextToImage && + !modelSupportsTextToImageDescriptor({ + providerType: item.provider.type, + modelId: item.model.modelId, + capabilities: item.model.capabilities, + defaultParams: item.model.defaultParams, + supportsImageGen: item.model.supportsImageGen, + }) + ) { + return false; + } if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(item.provider.type)) return false; return getCanvasModelRefKeys(item).every((key) => !failedRefs.has(key)); }; @@ -2844,6 +2858,7 @@ export async function generateCanvasImage( ) ).slice(0, 12); const requiresImageInput = normalizedReferenceImages.length > 0; + const requiresTextToImage = normalizedReferenceImages.length === 0; let provider = selectedModel.provider; @@ -2883,6 +2898,48 @@ export async function generateCanvasImage( provider = selectedModel.provider; } + if ( + requiresTextToImage && + !modelSupportsTextToImageDescriptor({ + providerType: selectedModel.provider.type, + modelId: selectedModel.model.modelId, + capabilities: selectedModel.model.capabilities, + defaultParams: selectedModel.model.defaultParams, + supportsImageGen: selectedModel.model.supportsImageGen, + }) + ) { + if (request.allowFailover === false) { + throw new Error("当前图片模型不支持纯文生图,请选择支持文生图的生图模型。"); + } + + const textToImageFallback = findCanvasFailoverModel({ + selectedModel, + imageModels, + failedModelRefs: getCanvasModelRefKeys(selectedModel), + preferLowCostSameNameProviders: Boolean( + debugMeta?.scaleAdjustPreferSameNameProviderFailover + ), + requiresTextToImage, + scaleAdjustModelFamily, + }); + + if (!textToImageFallback) { + throw new Error("当前任务需要纯文生图,但暂无可用的文生图模型。"); + } + + console.warn("[Canvas Generate Image] Fallback to text-to-image capable model:", { + debugLabel, + requestedModel: selectedModel.model.modelId || selectedModel.model.id, + fallbackModel: textToImageFallback.model.modelId || textToImageFallback.model.id, + requestedProviderType: provider.type, + fallbackProviderType: textToImageFallback.provider.type, + reason: "selected-model-lacks-text-to-image-support", + }); + + selectedModel = textToImageFallback; + provider = selectedModel.provider; + } + if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(provider.type)) { if (request.allowFailover === false) { throw new Error( @@ -2901,6 +2958,18 @@ export async function generateCanvasImage( ) { continue; } + if ( + requiresTextToImage && + !modelSupportsTextToImageDescriptor({ + providerType: candidate.provider.type, + modelId: candidate.model.modelId, + capabilities: candidate.model.capabilities, + defaultParams: candidate.model.defaultParams, + supportsImageGen: candidate.model.supportsImageGen, + }) + ) { + continue; + } fallbackModel = candidate; fallbackProvider = candidateProvider; break; @@ -3476,6 +3545,7 @@ export async function generateCanvasImage( debugMeta?.scaleAdjustPreferSameNameProviderFailover ), requiresImageInput: normalizedReferenceImages.length > 0, + requiresTextToImage: normalizedReferenceImages.length === 0, scaleAdjustModelFamily, }) : null; diff --git a/lib/external-image-api/service.ts b/lib/external-image-api/service.ts index 6e0cb936a..e8e436461 100644 --- a/lib/external-image-api/service.ts +++ b/lib/external-image-api/service.ts @@ -2,7 +2,10 @@ import { createHash, timingSafeEqual } from "node:crypto"; import { sql } from "drizzle-orm"; import sharp from "sharp"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; -import { isImageGenerationModelDescriptor } from "@/lib/ai/model-classification"; +import { + isImageGenerationModelDescriptor, + modelSupportsTextToImageDescriptor, +} from "@/lib/ai/model-classification"; import { db } from "@/lib/db"; import { getEnabledAiModelsWithProvider, getSystemSetting } from "@/lib/db/queries"; import { user } from "@/lib/db/schema"; @@ -322,7 +325,13 @@ function supportsImageEditForReferenceCount( referenceImageCount: number, ) { if (referenceImageCount <= 0) { - return true; + return modelSupportsTextToImageDescriptor({ + providerType: entry.provider.type, + modelId: entry.model.modelId, + capabilities: entry.model.capabilities, + defaultParams: entry.model.defaultParams, + supportsImageGen: entry.model.supportsImageGen, + }); } const capabilities = toRecord(entry.model.capabilities);