diff --git a/app/(chat)/api/models/route.ts b/app/(chat)/api/models/route.ts index 34ea4776c..6c7ad7d05 100644 --- a/app/(chat)/api/models/route.ts +++ b/app/(chat)/api/models/route.ts @@ -7,7 +7,7 @@ import { } from "@/lib/db/queries"; import { buildVideoToolCapability } from "@/lib/ai/video-tool-capability"; import { ChatSDKError } from "@/lib/errors"; -import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags"; +import { isAiModelAgentAuto, isAiModelMultimodal } from "@/lib/db/ai-model-flags"; const VIDEO_PROVIDER_TYPES = new Set([ "volcano-video", @@ -86,6 +86,7 @@ export async function GET() { description: model.description, supportsVision: model.supportsVision, supportsImageGen: model.supportsImageGen, + isMultimodal: isAiModelMultimodal(model), isDefault: model.isDefault, providerName: provider.name, providerType: provider.type, diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 08fe2ca32..8c0143fd8 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -1654,6 +1654,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF const [videoSlotUploading, setVideoSlotUploading] = useState(null); const [lastImageModelId, setLastImageModelId] = useState(null); const [lastVideoModelId, setLastVideoModelId] = useState(null); + const [lastConversationModelId, setLastConversationModelId] = useState(null); const [deepThinkingEnabled, setDeepThinkingEnabled] = useState(false); const [analysisCollapseState, setAnalysisCollapseState] = useState>({}); const [typedAssistantSections, setTypedAssistantSections] = useState>({}); @@ -2392,6 +2393,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF providerType: string; supportsVision?: boolean; supportsImageGen?: boolean; + isMultimodal?: boolean; capabilities?: any; videoToolCapability?: VideoToolCapabilityContract | null; imageGenConfig?: { @@ -2621,7 +2623,11 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF modelsData?.models?.filter( (model) => model.modelId !== "volcano-video-master" && - (model.supportsImageGen || VIDEO_PROVIDER_TYPES.has(model.providerType)) + ( + model.supportsImageGen || + model.isMultimodal || + VIDEO_PROVIDER_TYPES.has(model.providerType) + ) ) || []; return [...filtered].sort((a, b) => { @@ -2641,16 +2647,37 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF sortOrder: -999, providerType: "auto", supportsImageGen: true, + isMultimodal: false, videoToolCapability: null, imageGenConfig: modelsData?.agentModel?.imageGenConfig || undefined, }; // 所有可选模型（Agent模式 + 可视化模型） - const allModels = [SMART_AUTO_MODEL, ...visualModels]; + const allModels = useMemo(() => { + const merged = [SMART_AUTO_MODEL, ...visualModels]; + const seen = new Set(); + return merged.filter((model) => { + if (!model?.id || seen.has(model.id)) { + return false; + } + seen.add(model.id); + return true; + }); + }, [SMART_AUTO_MODEL, visualModels]); const imageOnlyModels = useMemo( () => visualModels.filter( - (model) => model.supportsImageGen && !VIDEO_PROVIDER_TYPES.has(model.providerType) + (model) => + model.supportsImageGen && + !model.isMultimodal && + !VIDEO_PROVIDER_TYPES.has(model.providerType) + ), + [visualModels] + ); + const conversationOnlyModels = useMemo( + () => + visualModels.filter( + (model) => model.isMultimodal && !VIDEO_PROVIDER_TYPES.has(model.providerType) ), [visualModels] ); @@ -2660,15 +2687,32 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF ); const currentModel = allModels.find((model) => model.id === selectedModelId); const isVideoModel = Boolean(currentModel && VIDEO_PROVIDER_TYPES.has(currentModel.providerType)); + const isConversationModel = Boolean( + currentModel && + currentModel.id !== "auto" && + currentModel.isMultimodal && + !VIDEO_PROVIDER_TYPES.has(currentModel.providerType) + ); const selectableModels = useMemo(() => { if (isDetailGenerationMode) { return [SMART_AUTO_MODEL]; } + if (isConversationModel) { + return conversationOnlyModels; + } if (isVideoModel) { return videoOnlyModels; } return [SMART_AUTO_MODEL, ...imageOnlyModels]; - }, [SMART_AUTO_MODEL, imageOnlyModels, isDetailGenerationMode, isVideoModel, videoOnlyModels]); + }, [ + SMART_AUTO_MODEL, + conversationOnlyModels, + imageOnlyModels, + isConversationModel, + isDetailGenerationMode, + isVideoModel, + videoOnlyModels, + ]); const modelSelectValue = selectableModels.some((model) => model.id === selectedModelId) ? selectedModelId : selectableModels[0]?.id || selectedModelId; @@ -2824,27 +2868,31 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF ]); const firstImageModelId = useMemo( - () => - allModels.find( - (model) => model.id !== "auto" && !VIDEO_PROVIDER_TYPES.has(model.providerType) - )?.id || null, - [allModels] + () => imageOnlyModels[0]?.id || null, + [imageOnlyModels] + ); + const firstConversationModelId = useMemo( + () => conversationOnlyModels[0]?.id || null, + [conversationOnlyModels] ); const firstVideoModelId = useMemo( - () => allModels.find((model) => VIDEO_PROVIDER_TYPES.has(model.providerType))?.id || null, - [allModels] + () => videoOnlyModels[0]?.id || null, + [videoOnlyModels] ); - const activeComposerMode = useMemo<"agent" | "image" | "video" | "detail">(() => { + const activeComposerMode = useMemo<"agent" | "chat" | "image" | "video" | "detail">(() => { if (isDetailGenerationMode) return "detail"; if (selectedModelId === "auto") return "agent"; if (isVideoModel) return "video"; + if (isConversationModel) return "chat"; return "image"; - }, [isDetailGenerationMode, isVideoModel, selectedModelId]); + }, [isConversationModel, isDetailGenerationMode, isVideoModel, selectedModelId]); const activeModeLabel = activeComposerMode === "video" ? "视频" : activeComposerMode === "detail" ? "AI详情" + : activeComposerMode === "chat" + ? "对话" : activeComposerMode === "image" ? "生图" : "Agent"; @@ -2937,13 +2985,17 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF setLastVideoModelId(currentModel.id); return; } + if (isConversationModel) { + setLastConversationModelId(currentModel.id); + return; + } if (currentModel.id !== "auto") { setLastImageModelId(currentModel.id); } - }, [currentModel, isDetailGenerationMode, isVideoModel]); + }, [currentModel, isConversationModel, isDetailGenerationMode, isVideoModel]); const handleSelectGenerationMode = useCallback( - (mode: "agent" | "image" | "video" | "detail") => { + (mode: "agent" | "chat" | "image" | "video" | "detail") => { if (mode === "agent") { currentModelIdRef.current = "auto"; setSelectedModelId("auto"); @@ -2963,12 +3015,32 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF return; } + if (mode === "chat") { + const preferredConversationModelId = + allModels.find( + (model) => + model.id === lastConversationModelId && + model.isMultimodal && + !VIDEO_PROVIDER_TYPES.has(model.providerType) + )?.id || firstConversationModelId; + + if (!preferredConversationModelId) { + toast.error("当前没有可用的多模态模型"); + return; + } + currentModelIdRef.current = preferredConversationModelId; + setSelectedModelId(preferredConversationModelId); + setIsDetailGenerationMode(false); + return; + } + if (mode === "image") { const preferredImageModelId = allModels.find( (model) => model.id === lastImageModelId && model.id !== "auto" && + !model.isMultimodal && !VIDEO_PROVIDER_TYPES.has(model.providerType) )?.id || firstImageModelId; @@ -2998,8 +3070,10 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF [ aiDetailGenerationEnabled, allModels, + firstConversationModelId, firstImageModelId, firstVideoModelId, + lastConversationModelId, lastImageModelId, lastVideoModelId, ] @@ -3191,6 +3265,16 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF const hiddenPrompt = hiddenPromptRef.current || undefined; const autoContext = autoContextRef.current || undefined; const contextPayload = buildContextPayload(); + const requestModel = allModels.find((model) => model.id === currentModelIdRef.current); + const requestIsVideoModel = Boolean( + requestModel && VIDEO_PROVIDER_TYPES.has(requestModel.providerType) + ); + const requestIsConversationModel = Boolean( + requestModel && + requestModel.id !== "auto" && + requestModel.isMultimodal && + !requestIsVideoModel + ); hiddenPromptRef.current = null; autoContextRef.current = null; return { @@ -3200,11 +3284,16 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF selectedChatModel: currentModelIdRef.current, selectedVisibilityType: "private", deepThinkingEnabled: - isVideoModelRef.current || currentModelIdRef.current === "auto" + requestIsVideoModel || + requestIsConversationModel || + currentModelIdRef.current === "auto" ? undefined : deepThinkingEnabledRef.current, - videoParams: isVideoModelRef.current ? videoParamsRef.current || undefined : undefined, - imageParams: isVideoModelRef.current ? undefined : imageParamsRef.current || undefined, + videoParams: requestIsVideoModel ? videoParamsRef.current || undefined : undefined, + imageParams: + requestIsVideoModel || requestIsConversationModel + ? undefined + : imageParamsRef.current || undefined, hiddenPrompt, autoContext, contextPayload, @@ -5832,7 +5921,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF ]); const ensureImagePointsBeforeSend = useCallback(() => { - if (isVideoModel) return true; + if (isVideoModel || activeComposerMode === "chat") return true; const pointsEnabled = membershipSummary?.pointsSystemConfig?.enabled === true; const blockOnInsufficient = @@ -5869,6 +5958,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF toast.error("积分不足，请先充值后再生图"); return false; }, [ + activeComposerMode, currentModel?.imageGenConfig?.creditCostPerImage, imageParams?.imageCount, isVideoModel, @@ -6042,7 +6132,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF text: textContent, attachments: normalizedAttachmentKeys, skillTagId: selectedSkillTag?.id || null, - mode: isVideoModel ? "video" : "image", + mode: activeComposerMode, }); if (shouldBlockDuplicateSend(sendSignature)) { @@ -6085,6 +6175,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF } }, [ activeChatId, + activeComposerMode, input, attachments, selectedSkillTag, @@ -8378,6 +8469,8 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF ? "border border-blue-500 bg-blue-50 text-blue-600 hover:bg-blue-100" : activeComposerMode === "detail" ? "border border-violet-500 bg-violet-50 text-violet-700 hover:bg-violet-100" + : activeComposerMode === "chat" + ? "border border-sky-500 bg-sky-50 text-sky-700 hover:bg-sky-100" : activeComposerMode === "image" ? "border border-purple-500 bg-purple-50 text-purple-700 hover:bg-purple-100" : "border border-[#9eb3ff] bg-white text-zinc-700 hover:bg-zinc-50" @@ -8388,6 +8481,8 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF