feat(ai): add volcano multimodal provider and agent multimodal mode

2026-02-18 19:58:31 +08:00 · 2026-02-18 19:58:31 +08:00 · cc26d2eb7f
commit cc26d2eb7f
parent 6bad214c3d
9 changed files with 400 additions and 16 deletions
--- a/app/(admin)/settings/admin-settings-client.tsx
+++ b/app/(admin)/settings/admin-settings-client.tsx
@ -121,7 +121,7 @@ type ToolConfig = {
 type AiProvider = {
  id: string;
  name: string;
-  type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu";
+  type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "volcano-multimodal";
  baseUrl: string;
  apiKey: string;
  isEnabled: boolean;
@ -154,6 +154,7 @@ type AiModel = {
  isDefault: boolean;
  sortOrder: number;
  isAgentModel?: boolean;
+  isMultimodal?: boolean;
  imageGenConfig?: ImageGenConfig; // 🆕 图片生成特性配置
 };

@ -365,6 +366,7 @@ const PROVIDER_TYPES = [
  { value: "ruxa-image", label: "RUXA 生图模型" },
  { value: "fal-image", label: "FAL 生图模型" },
  { value: "zhipu", label: "智普官方 4.7" },
+  { value: "volcano-multimodal", label: "火山引擎多模态" },
  { value: "openai", label: "OpenAI 兼容" },
  { value: "custom", label: "自定义 OpenAI 兼容" },
  { value: "local", label: "本地模型" },
@ -950,7 +952,7 @@ export function AdminSettingsClient() {
  const [providerForm, setProviderForm] = useState({
    id: "",
    name: "",
-    type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu",
+    type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "volcano-multimodal",
    baseUrl: "",
    apiKey: "",
    isEnabled: true,
@ -971,6 +973,7 @@ export function AdminSettingsClient() {
    isEnabled: true,
    isDefault: false,
    isAgentModel: false,
+    isMultimodal: false,
    sortOrder: 0,
    fallbackModelId1: "", // 备用模型 ID 1
    fallbackModelId2: "", // 备用模型 ID 2
@ -1816,6 +1819,7 @@ export function AdminSettingsClient() {
      isEnabled: true,
      isDefault: false,
      isAgentModel: false,
+      isMultimodal: false,
      sortOrder: 0,
      fallbackModelId1: "",
      fallbackModelId2: "",
@ -2981,6 +2985,7 @@ export function AdminSettingsClient() {
                        "fal-image": "bg-sky-500",
                        "openai": "bg-green-500",
                        "zhipu": "bg-lime-500",
+                        "volcano-multimodal": "bg-fuchsia-500",
                        "qianwen-video": "bg-cyan-500",
                        "custom": "bg-gray-500",
                        "local": "bg-yellow-500",
@ -5439,6 +5444,8 @@ export function AdminSettingsClient() {
                                  ? "https://fal.run"
                                : !prev.baseUrl && value === "zhipu"
                                  ? "https://open.bigmodel.cn/api/paas/v4"
+                                : !prev.baseUrl && value === "volcano-multimodal"
+                                  ? "https://ark.cn-beijing.volces.com/api/v3"
                                : prev.baseUrl,
                          }))
                        }
@ -5468,6 +5475,8 @@ export function AdminSettingsClient() {
                              ? "https://fal.run"
                            : providerForm.type === "zhipu"
                              ? "https://open.bigmodel.cn/api/paas/v4"
+                            : providerForm.type === "volcano-multimodal"
+                              ? "https://ark.cn-beijing.volces.com/api/v3"
                              : "https://api.openai.com/v1"
                        }
                      />
@ -5792,6 +5801,13 @@ export function AdminSettingsClient() {
                          />
                          <Label>Agent 自动模型（前端隐藏，仅后台 Agent 使用）</Label>
                        </div>
+                        <div className="flex items-center gap-2">
+                          <Switch
+                            checked={modelForm.isMultimodal || false}
+                            onCheckedChange={(checked) => setModelForm({ ...modelForm, isMultimodal: checked })}
+                          />
+                          <Label>多模态模型（Agent 识图与分析使用同一模型）</Label>
+                        </div>
                        
                        {/* 🆕 图片生成模型特性配置 */}
                        {modelForm.supportsImageGen && (
@ -5956,6 +5972,7 @@ export function AdminSettingsClient() {
                            <span className="truncate">{model.displayName}</span>
                            {model.isDefault && <span className="text-xs bg-blue-100 text-blue-800 px-2 py-0.5 rounded">默认</span>}
                            {model.isAgentModel && <span className="text-xs bg-violet-100 text-violet-800 px-2 py-0.5 rounded">Agent</span>}
+                            {model.isMultimodal && <span className="text-xs bg-emerald-100 text-emerald-800 px-2 py-0.5 rounded">多模态</span>}
                          </div>
                          <div className="text-sm text-muted-foreground truncate">
                            ID: {model.modelId} | 提供商: {provider?.name ?? "未知"}
@ -5976,6 +5993,7 @@ export function AdminSettingsClient() {
                            isEnabled: model.isEnabled,
                            isDefault: model.isDefault,
                            isAgentModel: model.isAgentModel || false,
+                            isMultimodal: model.isMultimodal || false,
                            sortOrder: model.sortOrder || 0,
                            fallbackModelId1: (model as any).fallbackModelId1 || "",
                            fallbackModelId2: (model as any).fallbackModelId2 || "",
--- a/app/api/admin/ai-models/route.ts
+++ b/app/api/admin/ai-models/route.ts
@ -10,7 +10,7 @@ import {
  getAiProviderById,
 } from "@/lib/db/queries";
 import { ChatSDKError } from "@/lib/errors";
-import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags";
+import { isAiModelAgentAuto, isAiModelMultimodal } from "@/lib/db/ai-model-flags";

 // 🆕 图片生成模型特性配置 Schema
 const imageGenConfigSchema = z.object({
@ -39,6 +39,7 @@ const createModelSchema = z.object({
  fallbackModelId1: z.string().max(128).optional(),
  fallbackModelId2: z.string().max(128).optional(),
  isAgentModel: z.boolean().optional(),
+  isMultimodal: z.boolean().optional(),
  imageGenConfig: imageGenConfigSchema, // 🆕 图片生成特性配置
 });

@ -57,6 +58,7 @@ const updateModelSchema = z.object({
  fallbackModelId1: z.string().max(128).optional(),
  fallbackModelId2: z.string().max(128).optional(),
  isAgentModel: z.boolean().optional(),
+  isMultimodal: z.boolean().optional(),
  imageGenConfig: imageGenConfigSchema, // 🆕 图片生成特性配置
 });

@ -64,6 +66,7 @@ function normalizeModel(model: any) {
  return {
    ...model,
    isAgentModel: isAiModelAgentAuto(model),
+    isMultimodal: isAiModelMultimodal(model),
  };
 }

--- a/app/api/admin/ai-providers/route.ts
+++ b/app/api/admin/ai-providers/route.ts
@ -24,9 +24,10 @@ const providerTypes = [
  "fal-image",
  "fal-video",
  "zhipu",
+  "volcano-multimodal",
 ] as const;
 type ProviderType = (typeof providerTypes)[number];
-type StoredProviderType = Exclude<ProviderType, "zhipu">;
+type StoredProviderType = Exclude<ProviderType, "zhipu" | "volcano-multimodal">;

 const createProviderSchema = z.object({
  name: z.string().min(1).max(128),
@ -46,22 +47,35 @@ const updateProviderSchema = z.object({
 });

 const ZHIPU_BASE_URL = "https://open.bigmodel.cn/api/paas/v4";
+const VOLCANO_MULTIMODAL_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3";

 function normalizeProviderTypeForStorage(
  type?: ProviderType
 ): StoredProviderType | undefined {
  if (!type) return undefined;
-  return type === "zhipu" ? "custom" : type;
+  if (type === "zhipu" || type === "volcano-multimodal") return "custom";
+  return type;
 }

 function isZhipuBaseUrl(baseUrl: string) {
  return baseUrl.includes("open.bigmodel.cn");
 }

+function isVolcanoMultimodalBaseUrl(baseUrl: string) {
+  return /ark\.cn-beijing\.volces\.com/i.test(baseUrl);
+}
+
 function normalizeProviderTypeForResponse(provider: Record<string, any>) {
  if (provider.type === "custom" && typeof provider.baseUrl === "string" && isZhipuBaseUrl(provider.baseUrl)) {
    return { ...provider, type: "zhipu" };
  }
+  if (
+    provider.type === "custom" &&
+    typeof provider.baseUrl === "string" &&
+    isVolcanoMultimodalBaseUrl(provider.baseUrl)
+  ) {
+    return { ...provider, type: "volcano-multimodal" };
+  }
  return provider;
 }

@ -133,7 +147,12 @@ export async function POST(request: Request) {
    const provider = await createAiProvider({
      ...data,
      type: storageType,
-      baseUrl: data.type === "zhipu" ? data.baseUrl || ZHIPU_BASE_URL : data.baseUrl,
+      baseUrl:
+        data.type === "zhipu"
+          ? data.baseUrl || ZHIPU_BASE_URL
+          : data.type === "volcano-multimodal"
+            ? data.baseUrl || VOLCANO_MULTIMODAL_BASE_URL
+            : data.baseUrl,
      updatedBy: session.user.id,
    });

--- a/lib/ai/auto-agent-graph.ts
+++ b/lib/ai/auto-agent-graph.ts
@ -2,6 +2,7 @@ import { Annotation, StateGraph, START, END } from "@langchain/langgraph";
 import { generateText } from "ai";
 import type { AnalysisResult } from "@/lib/ai/smart-analyzer";
 import type { RouteResult } from "@/lib/ai/model-router";
+import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";

 interface HistoryMessage {
  role: "user" | "assistant";
@ -410,6 +411,15 @@ type EnabledModelWithProvider = Awaited<

 function pickQwenVisionModel(models: EnabledModelWithProvider[]) {
  const visionModels = models.filter((item) => item.model.supportsVision && item.model.isEnabled);
+  const zhipuVision = visionModels.find(
+    (item) =>
+      item.provider.type === "zhipu" ||
+      /(glm[-_.]?4v|glm[-_.]?4\.?1v|glm[-_.]?4\.?5v|vision)/i.test(
+        `${item.model.displayName} ${item.model.modelId}`,
+      ),
+  );
+  if (zhipuVision) return zhipuVision;
+
  const qwenByName = visionModels.find(
    (item) =>
      item.provider.type === "alibaba" &&
@ -422,6 +432,22 @@ function pickQwenVisionModel(models: EnabledModelWithProvider[]) {
  return visionModels[0];
 }

+function pickAgentVisionModel(
+  agentModel: EnabledModelWithProvider | undefined,
+  models: EnabledModelWithProvider[],
+) {
+  if (
+    agentModel &&
+    agentModel.model.isEnabled &&
+    agentModel.model.supportsVision &&
+    isAiModelMultimodal(agentModel.model)
+  ) {
+    return agentModel;
+  }
+
+  return pickQwenVisionModel(models);
+}
+
 function pickAgentReasoningModel(
  agentModel: EnabledModelWithProvider | undefined,
  models: EnabledModelWithProvider[],
@ -429,6 +455,14 @@ function pickAgentReasoningModel(
  const isReasoningCompatibleProvider = (providerType: string) =>
    ["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType);

+  if (
+    agentModel &&
+    agentModel.model.isEnabled &&
+    isAiModelMultimodal(agentModel.model)
+  ) {
+    return agentModel;
+  }
+
  if (agentModel && isReasoningCompatibleProvider(agentModel.provider.type)) return agentModel;
  const zhipuModel = models.find(
    (item) =>
@ -493,7 +527,7 @@ async function generateAgentConversation(
    let visualAnalysis = "";
    if (state.hasImage && validImageUrls.length > 0) {
      try {
-        const visionModelConfig = pickQwenVisionModel(allModels);
+        const visionModelConfig = pickAgentVisionModel(agentModel, allModels);
        if (!visionModelConfig) {
          throw new Error("missing_vision_model");
        }
--- a/lib/ai/dynamic-provider.ts
+++ b/lib/ai/dynamic-provider.ts
@ -21,10 +21,16 @@ import { createAlibabaImage } from "./providers/alibaba-image";
 import { createRuxaImage } from "./providers/ruxa-image";
 import { createFalImage } from "./providers/fal-image";
 import { createFalVideo } from "./providers/fal-video";
+import { createVolcanoMultimodal } from "./providers/volcano-multimodal";

 // 缓存已创建的 provider 实例
 const providerCache = new Map<string, ReturnType<typeof createOpenAI>>();

+function isVolcanoMultimodalBaseUrl(baseUrl?: string | null) {
+  if (!baseUrl) return false;
+  return /ark\.cn-beijing\.volces\.com/i.test(baseUrl);
+}
+
 /**
 * 根据提供商类型创建对应的 AI Provider 实例
 * 不同供应商使用OpenAI兼容的API格式，但可能有细微差异
@ -178,6 +184,16 @@ function getOrCreateProvider(provider: Pick<AiProvider, "id" | "type" | "baseUrl

    case "custom":
    default: {
+      if (isVolcanoMultimodalBaseUrl(provider.baseUrl)) {
+        const volcanoMultimodalProvider = createVolcanoMultimodal({
+          baseURL: provider.baseUrl,
+          apiKey: provider.apiKey,
+        });
+        console.log(
+          `[Provider] Created Volcano Multimodal provider via Responses API: ${provider.baseUrl}`
+        );
+        return volcanoMultimodalProvider as any;
+      }
      // OpenAI 兼容的第三方 API（如 GPT-5 Nano、Claude via OpenAI proxy 等）
      // 这些服务提供与 OpenAI API 完全兼容的接口
      // 请求格式遵循 OpenAI Chat Completions 标准：/v1/chat/completions
--- a/lib/ai/providers/volcano-multimodal.ts
+++ b/lib/ai/providers/volcano-multimodal.ts
@ -0,0 +1,222 @@
+import "server-only";
+
+interface VolcanoMultimodalConfig {
+  apiKey: string;
+  baseURL: string;
+  timeout?: number;
+}
+
+function normalizeResponsesEndpoint(baseURL: string) {
+  const cleaned = baseURL.trim().replace(/\/+$/, "");
+  if (!cleaned) return "https://ark.cn-beijing.volces.com/api/v3/responses";
+  if (/\/responses$/i.test(cleaned)) return cleaned;
+  if (/\/api\/v3$/i.test(cleaned)) return `${cleaned}/responses`;
+  if (/ark\.cn-beijing\.volces\.com/i.test(cleaned)) {
+    return `${cleaned}/api/v3/responses`.replace(/\/api\/v3\/api\/v3\//, "/api/v3/");
+  }
+  return `${cleaned}/responses`;
+}
+
+function parseMessageParts(content: unknown) {
+  const textParts: string[] = [];
+  const imageUrls: string[] = [];
+
+  const pushImageUrl = (value: unknown) => {
+    if (typeof value !== "string") return;
+    const url = value.trim();
+    if (!url) return;
+    imageUrls.push(url);
+  };
+
+  const readPart = (part: any) => {
+    if (!part || typeof part !== "object") return;
+    if (part.type === "text" || part.type === "input_text") {
+      if (typeof part.text === "string" && part.text.trim()) textParts.push(part.text.trim());
+      return;
+    }
+    if (part.type === "image" || part.type === "input_image") {
+      pushImageUrl(part.image ?? part.image_url);
+      return;
+    }
+    if (part.type === "image_url") {
+      if (typeof part.image_url === "string") {
+        pushImageUrl(part.image_url);
+      } else {
+        pushImageUrl(part.image_url?.url);
+      }
+      return;
+    }
+    if (part.type === "file") {
+      pushImageUrl(part.url ?? part.data);
+    }
+  };
+
+  if (typeof content === "string") {
+    if (content.trim()) textParts.push(content.trim());
+    return { text: textParts.join("\n"), imageUrls };
+  }
+
+  if (Array.isArray(content)) {
+    for (const part of content) readPart(part);
+    return { text: textParts.join("\n"), imageUrls };
+  }
+
+  if (content && typeof content === "object") {
+    readPart(content);
+  }
+
+  return { text: textParts.join("\n"), imageUrls };
+}
+
+function extractPromptAndImages(options: any) {
+  if (typeof options?.prompt === "string") {
+    return { text: options.prompt.trim(), imageUrls: [] as string[] };
+  }
+
+  const messages = Array.isArray(options?.messages) ? options.messages : [];
+  for (let i = messages.length - 1; i >= 0; i -= 1) {
+    const message = messages[i];
+    if (message?.role !== "user") continue;
+    const parsed = parseMessageParts(message.content);
+    if (parsed.text || parsed.imageUrls.length > 0) {
+      return parsed;
+    }
+  }
+
+  const lastMessage = messages[messages.length - 1];
+  if (lastMessage) {
+    return parseMessageParts(lastMessage.content);
+  }
+
+  return { text: "", imageUrls: [] as string[] };
+}
+
+function extractOutputText(payload: any): string {
+  if (typeof payload?.output_text === "string" && payload.output_text.trim()) {
+    return payload.output_text.trim();
+  }
+
+  const chunks: string[] = [];
+  const output = Array.isArray(payload?.output) ? payload.output : [];
+  for (const item of output) {
+    const content = Array.isArray(item?.content) ? item.content : [];
+    for (const part of content) {
+      if (typeof part?.text === "string" && part.text.trim()) {
+        chunks.push(part.text.trim());
+      } else if (typeof part?.output_text === "string" && part.output_text.trim()) {
+        chunks.push(part.output_text.trim());
+      }
+    }
+  }
+
+  return chunks.join("\n").trim();
+}
+
+async function callResponsesApi(params: {
+  endpoint: string;
+  apiKey: string;
+  modelId: string;
+  text: string;
+  imageUrls: string[];
+  timeout: number;
+}) {
+  const content = [
+    ...params.imageUrls.map((url) => ({ type: "input_image", image_url: url })),
+    { type: "input_text", text: params.text || "请根据图片内容进行分析。" },
+  ];
+
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), params.timeout);
+
+  try {
+    const response = await fetch(params.endpoint, {
+      method: "POST",
+      signal: controller.signal,
+      headers: {
+        Authorization: `Bearer ${params.apiKey}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        model: params.modelId,
+        input: [
+          {
+            role: "user",
+            content,
+          },
+        ],
+      }),
+    });
+
+    const payload = await response.json().catch(() => ({}));
+    if (!response.ok || payload?.error) {
+      const message =
+        payload?.error?.message || payload?.message || `HTTP ${response.status}`;
+      throw new Error(message);
+    }
+
+    const text = extractOutputText(payload);
+    if (!text) {
+      throw new Error("火山多模态未返回文本结果");
+    }
+
+    const usage = payload?.usage || {};
+    return {
+      text,
+      usage: {
+        promptTokens:
+          Number(usage?.input_tokens) ||
+          Number(usage?.prompt_tokens) ||
+          0,
+        completionTokens:
+          Number(usage?.output_tokens) ||
+          Number(usage?.completion_tokens) ||
+          0,
+      },
+    };
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
+
+export function createVolcanoMultimodal(config: VolcanoMultimodalConfig) {
+  const endpoint = normalizeResponsesEndpoint(config.baseURL);
+  const timeout = config.timeout ?? 60000;
+
+  return {
+    provider: "volcano-multimodal",
+    chat: (modelId: string) => ({
+      provider: "volcano-multimodal",
+      modelId,
+      specificationVersion: "v1" as const,
+      defaultObjectGenerationMode: "json" as const,
+      doGenerate: async (options: any) => {
+        const { text, imageUrls } = extractPromptAndImages(options);
+        const result = await callResponsesApi({
+          endpoint,
+          apiKey: config.apiKey,
+          modelId,
+          text,
+          imageUrls,
+          timeout,
+        });
+        return {
+          text: result.text,
+          finishReason: "stop" as const,
+          usage: result.usage,
+        };
+      },
+      doStream: async function* (options: any) {
+        const result = await this.doGenerate(options);
+        yield {
+          type: "text-delta" as const,
+          textDelta: result.text,
+        };
+        yield {
+          type: "finish" as const,
+          finishReason: "stop" as const,
+          usage: result.usage,
+        };
+      },
+    }),
+  };
+}
--- a/lib/ai/smart-analyzer.ts
+++ b/lib/ai/smart-analyzer.ts
@ -7,6 +7,7 @@

 import { generateText } from "ai";
 import sharp from "sharp";
+import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
 // 使用动态导入避免循环依赖问题
 // import { getLanguageModelById } from "./dynamic-provider";
 // import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
@ -642,7 +643,9 @@ async function analyzWithLLM(
  } = await import("@/lib/db/queries");
  const { getLanguageModelById } = await import("./dynamic-provider");
  
-  // 图片识别固定走千问多模态；纯文本理解优先走 Agent 模型（建议智普）
+  // 优先规则：
+  // 1) 若 Agent 模型被标记为「多模态」，且当前请求包含图片，则识图+分析统一走该 Agent 模型
+  // 2) 否则按既有视觉模型优先级选择
  const allModels = await getEnabledAiModelsWithProvider();
  const agentModel = await getAgentAiModelWithProvider();
  const isReasoningCompatibleProvider = (providerType: string) =>
@ -681,8 +684,16 @@ async function analyzWithLLM(
    agentModel && isReasoningCompatibleProvider(agentModel.provider.type)
      ? agentModel
      : undefined;
+  const multimodalAgentVisionModel =
+    agentModel &&
+    agentModel.model.isEnabled &&
+    !agentModel.model.isSubModel &&
+    agentModel.model.supportsVision &&
+    isAiModelMultimodal(agentModel.model)
+      ? agentModel
+      : undefined;
  const selectedAnalyzerModel = hasImage
-    ? preferredVisionModel
+    ? multimodalAgentVisionModel || preferredVisionModel
    : usableAgentModel ||
      allModels.find((m) => m.provider.type === "zhipu" && !m.model.isSubModel) ||
      preferredVisionModel;
@ -746,8 +757,15 @@ ${historyText}
      (m.provider.type === "zhipu" || isLikelyZhipuEndpoint(m.provider.baseUrl)),
  )?.provider;
  const zhipuProvider = zhipuProviderFromAgent || zhipuProviderFallback;
+  const forceSingleMultimodalAgent = Boolean(hasImage && multimodalAgentVisionModel);

-  if (hasImage && visionImages.length > 0 && zhipuProvider?.apiKey && zhipuProvider?.baseUrl) {
+  if (
+    !forceSingleMultimodalAgent &&
+    hasImage &&
+    visionImages.length > 0 &&
+    zhipuProvider?.apiKey &&
+    zhipuProvider?.baseUrl
+  ) {
    const zhipuOfficialResult = await analyzeWithZhipuOfficialApi({
      apiKey: zhipuProvider.apiKey,
      baseUrl: zhipuProvider.baseUrl,
--- a/lib/db/ai-model-flags.ts
+++ b/lib/db/ai-model-flags.ts
@ -1,4 +1,5 @@
 const AGENT_AUTO_MODEL_KEY = "agentAutoModel";
+const MULTIMODAL_MODEL_KEY = "multimodalModel";

 function toRecord(value: unknown): Record<string, unknown> | null {
  if (!value || typeof value !== "object" || Array.isArray(value)) {
@ -60,3 +61,35 @@ export function withAgentAutoModelFlag(

  return Object.keys(next).length > 0 ? next : null;
 }
+
+export function isAiModelMultimodal(input: {
+  capabilities?: unknown;
+  defaultParams?: unknown;
+}): boolean {
+  const defaultParams = toRecord(input.defaultParams);
+  if (typeof defaultParams?.[MULTIMODAL_MODEL_KEY] === "boolean") {
+    return defaultParams[MULTIMODAL_MODEL_KEY] as boolean;
+  }
+
+  const capabilities = toRecord(input.capabilities);
+  if (typeof capabilities?.[MULTIMODAL_MODEL_KEY] === "boolean") {
+    return capabilities[MULTIMODAL_MODEL_KEY] as boolean;
+  }
+
+  return false;
+}
+
+export function withMultimodalModelFlag(
+  capabilities: unknown,
+  enabled: boolean
+): Record<string, unknown> | null {
+  const next = { ...(toRecord(capabilities) || {}) };
+
+  if (enabled) {
+    next[MULTIMODAL_MODEL_KEY] = true;
+  } else {
+    delete next[MULTIMODAL_MODEL_KEY];
+  }
+
+  return Object.keys(next).length > 0 ? next : null;
+}
--- a/lib/db/queries.ts
+++ b/lib/db/queries.ts
@ -55,6 +55,7 @@ import { generateHashedPassword } from "./utils";
 import {
  isAiModelAgentAuto,
  withAgentAutoModelFlag,
+  withMultimodalModelFlag,
 } from "./ai-model-flags";
 import {
  defaultPointsSystemConfig,
@ -1276,6 +1277,7 @@ export async function createAiModel({
  fallbackModelId2,
  imageGenConfig,
  isAgentModel,
+  isMultimodal,
 }: {
  providerId: string;
  modelId: string;
@ -1291,6 +1293,7 @@ export async function createAiModel({
  fallbackModelId2?: string;
  imageGenConfig?: ImageGenConfig;
  isAgentModel?: boolean;
+  isMultimodal?: boolean;
 }): Promise<AiModel[]> {
  try {
    // 如果设置为默认，先取消其他默认
@ -1321,6 +1324,10 @@ export async function createAiModel({
          isAgentModel === undefined
            ? null
            : withAgentAutoModelFlag(null, isAgentModel),
+        capabilities:
+          isMultimodal === undefined
+            ? null
+            : withMultimodalModelFlag(null, isMultimodal),
      })
      .returning();
  } catch (_error) {
@ -1347,6 +1354,7 @@ export async function updateAiModel({
  fallbackModelId2,
  imageGenConfig,
  isAgentModel,
+  isMultimodal,
 }: {
  id: string;
  providerId?: string;
@ -1363,6 +1371,7 @@ export async function updateAiModel({
  fallbackModelId2?: string;
  imageGenConfig?: ImageGenConfig;
  isAgentModel?: boolean;
+  isMultimodal?: boolean;
 }): Promise<AiModel[]> {
  try {
    // 如果设置为默认，先取消其他默认
@ -1390,9 +1399,12 @@ export async function updateAiModel({
    if (fallbackModelId2 !== undefined) updateData.fallbackModelId2 = fallbackModelId2 || null;
    // 🆕 图片生成配置
    if (imageGenConfig !== undefined) updateData.imageGenConfig = imageGenConfig || null;
-    if (isAgentModel !== undefined) {
+    if (isAgentModel !== undefined || isMultimodal !== undefined) {
      const [existingModel] = await db
-        .select({ defaultParams: aiModel.defaultParams })
+        .select({
+          defaultParams: aiModel.defaultParams,
+          capabilities: aiModel.capabilities,
+        })
        .from(aiModel)
        .where(eq(aiModel.id, id))
        .limit(1);
@ -1401,10 +1413,19 @@ export async function updateAiModel({
        return [];
      }

-      updateData.defaultParams = withAgentAutoModelFlag(
-        existingModel.defaultParams,
-        isAgentModel
-      );
+      if (isAgentModel !== undefined) {
+        updateData.defaultParams = withAgentAutoModelFlag(
+          existingModel.defaultParams,
+          isAgentModel
+        );
+      }
+
+      if (isMultimodal !== undefined) {
+        updateData.capabilities = withMultimodalModelFlag(
+          existingModel.capabilities,
+          isMultimodal
+        );
+      }
    }

    return await db