feat: add multimodal chat mode to canvas panel

app/(chat)/api/models/route.ts

@@ -7,7 +7,7 @@ import {
 } from "@/lib/db/queries";
 import { buildVideoToolCapability } from "@/lib/ai/video-tool-capability";
 import { ChatSDKError } from "@/lib/errors";
-import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags";
+import { isAiModelAgentAuto, isAiModelMultimodal } from "@/lib/db/ai-model-flags";
 
 const VIDEO_PROVIDER_TYPES = new Set([
   "volcano-video",
@@ -86,6 +86,7 @@ export async function GET() {
       description: model.description,
       supportsVision: model.supportsVision,
       supportsImageGen: model.supportsImageGen,
+      isMultimodal: isAiModelMultimodal(model),
       isDefault: model.isDefault,
       providerName: provider.name,
       providerType: provider.type,

components/canvas/CanvasChatPanel.tsx

@@ -1654,6 +1654,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
   const [videoSlotUploading, setVideoSlotUploading] = useState<number | null>(null);
   const [lastImageModelId, setLastImageModelId] = useState<string | null>(null);
   const [lastVideoModelId, setLastVideoModelId] = useState<string | null>(null);
+  const [lastConversationModelId, setLastConversationModelId] = useState<string | null>(null);
   const [deepThinkingEnabled, setDeepThinkingEnabled] = useState(false);
   const [analysisCollapseState, setAnalysisCollapseState] = useState<Record<string, boolean>>({});
   const [typedAssistantSections, setTypedAssistantSections] = useState<Record<string, boolean>>({});
@@ -2392,6 +2393,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
       providerType: string;
       supportsVision?: boolean;
       supportsImageGen?: boolean;
+      isMultimodal?: boolean;
       capabilities?: any;
       videoToolCapability?: VideoToolCapabilityContract | null;
       imageGenConfig?: {
@@ -2621,7 +2623,11 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
       modelsData?.models?.filter(
         (model) =>
           model.modelId !== "volcano-video-master" &&
-          (model.supportsImageGen || VIDEO_PROVIDER_TYPES.has(model.providerType))
+          (
+            model.supportsImageGen ||
+            model.isMultimodal ||
+            VIDEO_PROVIDER_TYPES.has(model.providerType)
+          )
       ) || [];
 
     return [...filtered].sort((a, b) => {
@@ -2641,16 +2647,37 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
     sortOrder: -999,
     providerType: "auto",
     supportsImageGen: true,
+    isMultimodal: false,
     videoToolCapability: null,
     imageGenConfig: modelsData?.agentModel?.imageGenConfig || undefined,
   };
   
   // 所有可选模型（Agent模式 + 可视化模型）
-  const allModels = [SMART_AUTO_MODEL, ...visualModels];
+  const allModels = useMemo(() => {
+    const merged = [SMART_AUTO_MODEL, ...visualModels];
+    const seen = new Set<string>();
+    return merged.filter((model) => {
+      if (!model?.id || seen.has(model.id)) {
+        return false;
+      }
+      seen.add(model.id);
+      return true;
+    });
+  }, [SMART_AUTO_MODEL, visualModels]);
   const imageOnlyModels = useMemo(
     () =>
       visualModels.filter(
-        (model) => model.supportsImageGen && !VIDEO_PROVIDER_TYPES.has(model.providerType)
+        (model) =>
+          model.supportsImageGen &&
+          !model.isMultimodal &&
+          !VIDEO_PROVIDER_TYPES.has(model.providerType)
+      ),
+    [visualModels]
+  );
+  const conversationOnlyModels = useMemo(
+    () =>
+      visualModels.filter(
+        (model) => model.isMultimodal && !VIDEO_PROVIDER_TYPES.has(model.providerType)
       ),
     [visualModels]
   );
@@ -2660,15 +2687,32 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
   );
   const currentModel = allModels.find((model) => model.id === selectedModelId);
   const isVideoModel = Boolean(currentModel && VIDEO_PROVIDER_TYPES.has(currentModel.providerType));
+  const isConversationModel = Boolean(
+    currentModel &&
+      currentModel.id !== "auto" &&
+      currentModel.isMultimodal &&
+      !VIDEO_PROVIDER_TYPES.has(currentModel.providerType)
+  );
   const selectableModels = useMemo(() => {
     if (isDetailGenerationMode) {
       return [SMART_AUTO_MODEL];
     }
+    if (isConversationModel) {
+      return conversationOnlyModels;
+    }
     if (isVideoModel) {
       return videoOnlyModels;
     }
     return [SMART_AUTO_MODEL, ...imageOnlyModels];
-  }, [SMART_AUTO_MODEL, imageOnlyModels, isDetailGenerationMode, isVideoModel, videoOnlyModels]);
+  }, [
+    SMART_AUTO_MODEL,
+    conversationOnlyModels,
+    imageOnlyModels,
+    isConversationModel,
+    isDetailGenerationMode,
+    isVideoModel,
+    videoOnlyModels,
+  ]);
   const modelSelectValue = selectableModels.some((model) => model.id === selectedModelId)
     ? selectedModelId
     : selectableModels[0]?.id || selectedModelId;
@@ -2824,27 +2868,31 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
   ]);
 
   const firstImageModelId = useMemo(
-    () =>
-      allModels.find(
-        (model) => model.id !== "auto" && !VIDEO_PROVIDER_TYPES.has(model.providerType)
-      )?.id || null,
-    [allModels]
+    () => imageOnlyModels[0]?.id || null,
+    [imageOnlyModels]
+  );
+  const firstConversationModelId = useMemo(
+    () => conversationOnlyModels[0]?.id || null,
+    [conversationOnlyModels]
   );
   const firstVideoModelId = useMemo(
-    () => allModels.find((model) => VIDEO_PROVIDER_TYPES.has(model.providerType))?.id || null,
-    [allModels]
+    () => videoOnlyModels[0]?.id || null,
+    [videoOnlyModels]
   );
-  const activeComposerMode = useMemo<"agent" | "image" | "video" | "detail">(() => {
+  const activeComposerMode = useMemo<"agent" | "chat" | "image" | "video" | "detail">(() => {
     if (isDetailGenerationMode) return "detail";
     if (selectedModelId === "auto") return "agent";
     if (isVideoModel) return "video";
+    if (isConversationModel) return "chat";
     return "image";
-  }, [isDetailGenerationMode, isVideoModel, selectedModelId]);
+  }, [isConversationModel, isDetailGenerationMode, isVideoModel, selectedModelId]);
   const activeModeLabel =
     activeComposerMode === "video"
       ? "视频"
       : activeComposerMode === "detail"
         ? "AI详情"
+      : activeComposerMode === "chat"
+        ? "对话"
       : activeComposerMode === "image"
         ? "生图"
         : "Agent";
@@ -2937,13 +2985,17 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
       setLastVideoModelId(currentModel.id);
       return;
     }
+    if (isConversationModel) {
+      setLastConversationModelId(currentModel.id);
+      return;
+    }
     if (currentModel.id !== "auto") {
       setLastImageModelId(currentModel.id);
     }
-  }, [currentModel, isDetailGenerationMode, isVideoModel]);
+  }, [currentModel, isConversationModel, isDetailGenerationMode, isVideoModel]);
 
   const handleSelectGenerationMode = useCallback(
-    (mode: "agent" | "image" | "video" | "detail") => {
+    (mode: "agent" | "chat" | "image" | "video" | "detail") => {
       if (mode === "agent") {
         currentModelIdRef.current = "auto";
         setSelectedModelId("auto");
@@ -2963,12 +3015,32 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
         return;
       }
 
+      if (mode === "chat") {
+        const preferredConversationModelId =
+          allModels.find(
+            (model) =>
+              model.id === lastConversationModelId &&
+              model.isMultimodal &&
+              !VIDEO_PROVIDER_TYPES.has(model.providerType)
+          )?.id || firstConversationModelId;
+
+        if (!preferredConversationModelId) {
+          toast.error("当前没有可用的多模态模型");
+          return;
+        }
+        currentModelIdRef.current = preferredConversationModelId;
+        setSelectedModelId(preferredConversationModelId);
+        setIsDetailGenerationMode(false);
+        return;
+      }
+
       if (mode === "image") {
         const preferredImageModelId =
           allModels.find(
             (model) =>
               model.id === lastImageModelId &&
               model.id !== "auto" &&
+              !model.isMultimodal &&
               !VIDEO_PROVIDER_TYPES.has(model.providerType)
           )?.id || firstImageModelId;
 
@@ -2998,8 +3070,10 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
     [
       aiDetailGenerationEnabled,
       allModels,
+      firstConversationModelId,
       firstImageModelId,
       firstVideoModelId,
+      lastConversationModelId,
       lastImageModelId,
       lastVideoModelId,
     ]
@@ -3191,6 +3265,16 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
         const hiddenPrompt = hiddenPromptRef.current || undefined;
         const autoContext = autoContextRef.current || undefined;
         const contextPayload = buildContextPayload();
+        const requestModel = allModels.find((model) => model.id === currentModelIdRef.current);
+        const requestIsVideoModel = Boolean(
+          requestModel && VIDEO_PROVIDER_TYPES.has(requestModel.providerType)
+        );
+        const requestIsConversationModel = Boolean(
+          requestModel &&
+            requestModel.id !== "auto" &&
+            requestModel.isMultimodal &&
+            !requestIsVideoModel
+        );
         hiddenPromptRef.current = null;
         autoContextRef.current = null;
         return {
@@ -3200,11 +3284,16 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
             selectedChatModel: currentModelIdRef.current,
             selectedVisibilityType: "private",
             deepThinkingEnabled:
-              isVideoModelRef.current || currentModelIdRef.current === "auto"
+              requestIsVideoModel ||
+              requestIsConversationModel ||
+              currentModelIdRef.current === "auto"
                 ? undefined
                 : deepThinkingEnabledRef.current,
-            videoParams: isVideoModelRef.current ? videoParamsRef.current || undefined : undefined,
-            imageParams: isVideoModelRef.current ? undefined : imageParamsRef.current || undefined,
+            videoParams: requestIsVideoModel ? videoParamsRef.current || undefined : undefined,
+            imageParams:
+              requestIsVideoModel || requestIsConversationModel
+                ? undefined
+                : imageParamsRef.current || undefined,
             hiddenPrompt,
             autoContext,
             contextPayload,
@@ -5832,7 +5921,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
   ]);
 
   const ensureImagePointsBeforeSend = useCallback(() => {
-    if (isVideoModel) return true;
+    if (isVideoModel || activeComposerMode === "chat") return true;
 
     const pointsEnabled = membershipSummary?.pointsSystemConfig?.enabled === true;
     const blockOnInsufficient =
@@ -5869,6 +5958,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
     toast.error("积分不足，请先充值后再生图");
     return false;
   }, [
+    activeComposerMode,
     currentModel?.imageGenConfig?.creditCostPerImage,
     imageParams?.imageCount,
     isVideoModel,
@@ -6042,7 +6132,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
       text: textContent,
       attachments: normalizedAttachmentKeys,
       skillTagId: selectedSkillTag?.id || null,
-      mode: isVideoModel ? "video" : "image",
+      mode: activeComposerMode,
     });
 
     if (shouldBlockDuplicateSend(sendSignature)) {
@@ -6085,6 +6175,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
     }
   }, [
     activeChatId,
+    activeComposerMode,
     input,
     attachments,
     selectedSkillTag,
@@ -8378,6 +8469,8 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
                         ? "border border-blue-500 bg-blue-50 text-blue-600 hover:bg-blue-100"
                         : activeComposerMode === "detail"
                           ? "border border-violet-500 bg-violet-50 text-violet-700 hover:bg-violet-100"
+                        : activeComposerMode === "chat"
+                          ? "border border-sky-500 bg-sky-50 text-sky-700 hover:bg-sky-100"
                         : activeComposerMode === "image"
                           ? "border border-purple-500 bg-purple-50 text-purple-700 hover:bg-purple-100"
                           : "border border-[#9eb3ff] bg-white text-zinc-700 hover:bg-zinc-50"
@@ -8388,6 +8481,8 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
                         <Video className="mr-1 h-4 w-4" />
                       ) : activeComposerMode === "detail" ? (
                         <LayoutGrid className="mr-1 h-4 w-4" />
+                      ) : activeComposerMode === "chat" ? (
+                        <MessageCircle className="mr-1 h-4 w-4" />
                       ) : activeComposerMode === "image" ? (
                         <ImagePlus className="mr-1 h-4 w-4" />
                       ) : (
@@ -8414,6 +8509,17 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
                     <span className="flex-1">Agent</span>
                     {activeComposerMode === "agent" ? <Check className="h-4 w-4 text-[#3b82f6]" /> : null}
                   </DropdownMenuItem>
+                  <DropdownMenuItem
+                    onClick={() => handleSelectGenerationMode("chat")}
+                    className={cn(
+                      "h-8 cursor-pointer rounded-lg px-2 text-[13px] font-medium text-[#2e3238]",
+                      activeComposerMode === "chat" ? "bg-white/80" : "hover:bg-white/70"
+                    )}
+                  >
+                    <MessageCircle className="h-4 w-4" />
+                    <span className="flex-1">对话模式</span>
+                    {activeComposerMode === "chat" ? <Check className="h-4 w-4 text-[#3b82f6]" /> : null}
+                  </DropdownMenuItem>
                   {aiDetailGenerationEnabled ? (
                     <DropdownMenuItem
                       onClick={() => handleSelectGenerationMode("detail")}
@@ -8477,7 +8583,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
                     />
                   </div>
                 </div>
-              ) : (
+              ) : activeComposerMode !== "chat" ? (
                 <div className="shrink-0">
                   <ImageParamsBar
                     modelName={currentModel?.displayName}
@@ -8489,7 +8595,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
                     hideAdvancedOptions={activeComposerMode === "image"}
                   />
                 </div>
-              )}
+              ) : null}
             </div>
 
             <div className="flex items-center gap-2 flex-shrink-0">