ID: {model.modelId} | 提供商: {provider?.name ?? "未知"}
@@ -5976,6 +5993,7 @@ export function AdminSettingsClient() {
isEnabled: model.isEnabled,
isDefault: model.isDefault,
isAgentModel: model.isAgentModel || false,
+ isMultimodal: model.isMultimodal || false,
sortOrder: model.sortOrder || 0,
fallbackModelId1: (model as any).fallbackModelId1 || "",
fallbackModelId2: (model as any).fallbackModelId2 || "",
diff --git a/app/api/admin/ai-models/route.ts b/app/api/admin/ai-models/route.ts
index 6e225697d..e58f89c93 100644
--- a/app/api/admin/ai-models/route.ts
+++ b/app/api/admin/ai-models/route.ts
@@ -10,7 +10,7 @@ import {
getAiProviderById,
} from "@/lib/db/queries";
import { ChatSDKError } from "@/lib/errors";
-import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags";
+import { isAiModelAgentAuto, isAiModelMultimodal } from "@/lib/db/ai-model-flags";
// 🆕 图片生成模型特性配置 Schema
const imageGenConfigSchema = z.object({
@@ -39,6 +39,7 @@ const createModelSchema = z.object({
fallbackModelId1: z.string().max(128).optional(),
fallbackModelId2: z.string().max(128).optional(),
isAgentModel: z.boolean().optional(),
+ isMultimodal: z.boolean().optional(),
imageGenConfig: imageGenConfigSchema, // 🆕 图片生成特性配置
});
@@ -57,6 +58,7 @@ const updateModelSchema = z.object({
fallbackModelId1: z.string().max(128).optional(),
fallbackModelId2: z.string().max(128).optional(),
isAgentModel: z.boolean().optional(),
+ isMultimodal: z.boolean().optional(),
imageGenConfig: imageGenConfigSchema, // 🆕 图片生成特性配置
});
@@ -64,6 +66,7 @@ function normalizeModel(model: any) {
return {
...model,
isAgentModel: isAiModelAgentAuto(model),
+ isMultimodal: isAiModelMultimodal(model),
};
}
diff --git a/app/api/admin/ai-providers/route.ts b/app/api/admin/ai-providers/route.ts
index 5fe78ff67..2d0759d4c 100644
--- a/app/api/admin/ai-providers/route.ts
+++ b/app/api/admin/ai-providers/route.ts
@@ -24,9 +24,10 @@ const providerTypes = [
"fal-image",
"fal-video",
"zhipu",
+ "volcano-multimodal",
] as const;
type ProviderType = (typeof providerTypes)[number];
-type StoredProviderType = Exclude
;
+type StoredProviderType = Exclude;
const createProviderSchema = z.object({
name: z.string().min(1).max(128),
@@ -46,22 +47,35 @@ const updateProviderSchema = z.object({
});
const ZHIPU_BASE_URL = "https://open.bigmodel.cn/api/paas/v4";
+const VOLCANO_MULTIMODAL_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3";
function normalizeProviderTypeForStorage(
type?: ProviderType
): StoredProviderType | undefined {
if (!type) return undefined;
- return type === "zhipu" ? "custom" : type;
+ if (type === "zhipu" || type === "volcano-multimodal") return "custom";
+ return type;
}
function isZhipuBaseUrl(baseUrl: string) {
return baseUrl.includes("open.bigmodel.cn");
}
+function isVolcanoMultimodalBaseUrl(baseUrl: string) {
+ return /ark\.cn-beijing\.volces\.com/i.test(baseUrl);
+}
+
function normalizeProviderTypeForResponse(provider: Record) {
if (provider.type === "custom" && typeof provider.baseUrl === "string" && isZhipuBaseUrl(provider.baseUrl)) {
return { ...provider, type: "zhipu" };
}
+ if (
+ provider.type === "custom" &&
+ typeof provider.baseUrl === "string" &&
+ isVolcanoMultimodalBaseUrl(provider.baseUrl)
+ ) {
+ return { ...provider, type: "volcano-multimodal" };
+ }
return provider;
}
@@ -133,7 +147,12 @@ export async function POST(request: Request) {
const provider = await createAiProvider({
...data,
type: storageType,
- baseUrl: data.type === "zhipu" ? data.baseUrl || ZHIPU_BASE_URL : data.baseUrl,
+ baseUrl:
+ data.type === "zhipu"
+ ? data.baseUrl || ZHIPU_BASE_URL
+ : data.type === "volcano-multimodal"
+ ? data.baseUrl || VOLCANO_MULTIMODAL_BASE_URL
+ : data.baseUrl,
updatedBy: session.user.id,
});
diff --git a/lib/ai/auto-agent-graph.ts b/lib/ai/auto-agent-graph.ts
index f5db25a86..2fae37353 100644
--- a/lib/ai/auto-agent-graph.ts
+++ b/lib/ai/auto-agent-graph.ts
@@ -2,6 +2,7 @@ import { Annotation, StateGraph, START, END } from "@langchain/langgraph";
import { generateText } from "ai";
import type { AnalysisResult } from "@/lib/ai/smart-analyzer";
import type { RouteResult } from "@/lib/ai/model-router";
+import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
interface HistoryMessage {
role: "user" | "assistant";
@@ -410,6 +411,15 @@ type EnabledModelWithProvider = Awaited<
function pickQwenVisionModel(models: EnabledModelWithProvider[]) {
const visionModels = models.filter((item) => item.model.supportsVision && item.model.isEnabled);
+ const zhipuVision = visionModels.find(
+ (item) =>
+ item.provider.type === "zhipu" ||
+ /(glm[-_.]?4v|glm[-_.]?4\.?1v|glm[-_.]?4\.?5v|vision)/i.test(
+ `${item.model.displayName} ${item.model.modelId}`,
+ ),
+ );
+ if (zhipuVision) return zhipuVision;
+
const qwenByName = visionModels.find(
(item) =>
item.provider.type === "alibaba" &&
@@ -422,6 +432,22 @@ function pickQwenVisionModel(models: EnabledModelWithProvider[]) {
return visionModels[0];
}
+function pickAgentVisionModel(
+ agentModel: EnabledModelWithProvider | undefined,
+ models: EnabledModelWithProvider[],
+) {
+ if (
+ agentModel &&
+ agentModel.model.isEnabled &&
+ agentModel.model.supportsVision &&
+ isAiModelMultimodal(agentModel.model)
+ ) {
+ return agentModel;
+ }
+
+ return pickQwenVisionModel(models);
+}
+
function pickAgentReasoningModel(
agentModel: EnabledModelWithProvider | undefined,
models: EnabledModelWithProvider[],
@@ -429,6 +455,14 @@ function pickAgentReasoningModel(
const isReasoningCompatibleProvider = (providerType: string) =>
["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType);
+ if (
+ agentModel &&
+ agentModel.model.isEnabled &&
+ isAiModelMultimodal(agentModel.model)
+ ) {
+ return agentModel;
+ }
+
if (agentModel && isReasoningCompatibleProvider(agentModel.provider.type)) return agentModel;
const zhipuModel = models.find(
(item) =>
@@ -493,7 +527,7 @@ async function generateAgentConversation(
let visualAnalysis = "";
if (state.hasImage && validImageUrls.length > 0) {
try {
- const visionModelConfig = pickQwenVisionModel(allModels);
+ const visionModelConfig = pickAgentVisionModel(agentModel, allModels);
if (!visionModelConfig) {
throw new Error("missing_vision_model");
}
diff --git a/lib/ai/dynamic-provider.ts b/lib/ai/dynamic-provider.ts
index e59f3758f..b31015876 100644
--- a/lib/ai/dynamic-provider.ts
+++ b/lib/ai/dynamic-provider.ts
@@ -21,10 +21,16 @@ import { createAlibabaImage } from "./providers/alibaba-image";
import { createRuxaImage } from "./providers/ruxa-image";
import { createFalImage } from "./providers/fal-image";
import { createFalVideo } from "./providers/fal-video";
+import { createVolcanoMultimodal } from "./providers/volcano-multimodal";
// 缓存已创建的 provider 实例
const providerCache = new Map>();
+function isVolcanoMultimodalBaseUrl(baseUrl?: string | null) {
+ if (!baseUrl) return false;
+ return /ark\.cn-beijing\.volces\.com/i.test(baseUrl);
+}
+
/**
* 根据提供商类型创建对应的 AI Provider 实例
* 不同供应商使用OpenAI兼容的API格式，但可能有细微差异
@@ -178,6 +184,16 @@ function getOrCreateProvider(provider: Pick {
+ if (typeof value !== "string") return;
+ const url = value.trim();
+ if (!url) return;
+ imageUrls.push(url);
+ };
+
+ const readPart = (part: any) => {
+ if (!part || typeof part !== "object") return;
+ if (part.type === "text" || part.type === "input_text") {
+ if (typeof part.text === "string" && part.text.trim()) textParts.push(part.text.trim());
+ return;
+ }
+ if (part.type === "image" || part.type === "input_image") {
+ pushImageUrl(part.image ?? part.image_url);
+ return;
+ }
+ if (part.type === "image_url") {
+ if (typeof part.image_url === "string") {
+ pushImageUrl(part.image_url);
+ } else {
+ pushImageUrl(part.image_url?.url);
+ }
+ return;
+ }
+ if (part.type === "file") {
+ pushImageUrl(part.url ?? part.data);
+ }
+ };
+
+ if (typeof content === "string") {
+ if (content.trim()) textParts.push(content.trim());
+ return { text: textParts.join("\n"), imageUrls };
+ }
+
+ if (Array.isArray(content)) {
+ for (const part of content) readPart(part);
+ return { text: textParts.join("\n"), imageUrls };
+ }
+
+ if (content && typeof content === "object") {
+ readPart(content);
+ }
+
+ return { text: textParts.join("\n"), imageUrls };
+}
+
+function extractPromptAndImages(options: any) {
+ if (typeof options?.prompt === "string") {
+ return { text: options.prompt.trim(), imageUrls: [] as string[] };
+ }
+
+ const messages = Array.isArray(options?.messages) ? options.messages : [];
+ for (let i = messages.length - 1; i >= 0; i -= 1) {
+ const message = messages[i];
+ if (message?.role !== "user") continue;
+ const parsed = parseMessageParts(message.content);
+ if (parsed.text || parsed.imageUrls.length > 0) {
+ return parsed;
+ }
+ }
+
+ const lastMessage = messages[messages.length - 1];
+ if (lastMessage) {
+ return parseMessageParts(lastMessage.content);
+ }
+
+ return { text: "", imageUrls: [] as string[] };
+}
+
+function extractOutputText(payload: any): string {
+ if (typeof payload?.output_text === "string" && payload.output_text.trim()) {
+ return payload.output_text.trim();
+ }
+
+ const chunks: string[] = [];
+ const output = Array.isArray(payload?.output) ? payload.output : [];
+ for (const item of output) {
+ const content = Array.isArray(item?.content) ? item.content : [];
+ for (const part of content) {
+ if (typeof part?.text === "string" && part.text.trim()) {
+ chunks.push(part.text.trim());
+ } else if (typeof part?.output_text === "string" && part.output_text.trim()) {
+ chunks.push(part.output_text.trim());
+ }
+ }
+ }
+
+ return chunks.join("\n").trim();
+}
+
+async function callResponsesApi(params: {
+ endpoint: string;
+ apiKey: string;
+ modelId: string;
+ text: string;
+ imageUrls: string[];
+ timeout: number;
+}) {
+ const content = [
+ ...params.imageUrls.map((url) => ({ type: "input_image", image_url: url })),
+ { type: "input_text", text: params.text || "请根据图片内容进行分析。" },
+ ];
+
+ const controller = new AbortController();
+ const timeoutId = setTimeout(() => controller.abort(), params.timeout);
+
+ try {
+ const response = await fetch(params.endpoint, {
+ method: "POST",
+ signal: controller.signal,
+ headers: {
+ Authorization: `Bearer ${params.apiKey}`,
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ model: params.modelId,
+ input: [
+ {
+ role: "user",
+ content,
+ },
+ ],
+ }),
+ });
+
+ const payload = await response.json().catch(() => ({}));
+ if (!response.ok || payload?.error) {
+ const message =
+ payload?.error?.message || payload?.message || `HTTP ${response.status}`;
+ throw new Error(message);
+ }
+
+ const text = extractOutputText(payload);
+ if (!text) {
+ throw new Error("火山多模态未返回文本结果");
+ }
+
+ const usage = payload?.usage || {};
+ return {
+ text,
+ usage: {
+ promptTokens:
+ Number(usage?.input_tokens) ||
+ Number(usage?.prompt_tokens) ||
+ 0,
+ completionTokens:
+ Number(usage?.output_tokens) ||
+ Number(usage?.completion_tokens) ||
+ 0,
+ },
+ };
+ } finally {
+ clearTimeout(timeoutId);
+ }
+}
+
+export function createVolcanoMultimodal(config: VolcanoMultimodalConfig) {
+ const endpoint = normalizeResponsesEndpoint(config.baseURL);
+ const timeout = config.timeout ?? 60000;
+
+ return {
+ provider: "volcano-multimodal",
+ chat: (modelId: string) => ({
+ provider: "volcano-multimodal",
+ modelId,
+ specificationVersion: "v1" as const,
+ defaultObjectGenerationMode: "json" as const,
+ doGenerate: async (options: any) => {
+ const { text, imageUrls } = extractPromptAndImages(options);
+ const result = await callResponsesApi({
+ endpoint,
+ apiKey: config.apiKey,
+ modelId,
+ text,
+ imageUrls,
+ timeout,
+ });
+ return {
+ text: result.text,
+ finishReason: "stop" as const,
+ usage: result.usage,
+ };
+ },
+ doStream: async function* (options: any) {
+ const result = await this.doGenerate(options);
+ yield {
+ type: "text-delta" as const,
+ textDelta: result.text,
+ };
+ yield {
+ type: "finish" as const,
+ finishReason: "stop" as const,
+ usage: result.usage,
+ };
+ },
+ }),
+ };
+}
diff --git a/lib/ai/smart-analyzer.ts b/lib/ai/smart-analyzer.ts
index 87f413e9f..0698cd557 100644
--- a/lib/ai/smart-analyzer.ts
+++ b/lib/ai/smart-analyzer.ts
@@ -7,6 +7,7 @@
import { generateText } from "ai";
import sharp from "sharp";
+import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
// 使用动态导入避免循环依赖问题
// import { getLanguageModelById } from "./dynamic-provider";
// import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
@@ -642,7 +643,9 @@ async function analyzWithLLM(
} = await import("@/lib/db/queries");
const { getLanguageModelById } = await import("./dynamic-provider");
- // 图片识别固定走千问多模态；纯文本理解优先走 Agent 模型（建议智普）
+ // 优先规则：
+ // 1) 若 Agent 模型被标记为「多模态」，且当前请求包含图片，则识图+分析统一走该 Agent 模型
+ // 2) 否则按既有视觉模型优先级选择
const allModels = await getEnabledAiModelsWithProvider();
const agentModel = await getAgentAiModelWithProvider();
const isReasoningCompatibleProvider = (providerType: string) =>
@@ -681,8 +684,16 @@ async function analyzWithLLM(
agentModel && isReasoningCompatibleProvider(agentModel.provider.type)
? agentModel
: undefined;
+ const multimodalAgentVisionModel =
+ agentModel &&
+ agentModel.model.isEnabled &&
+ !agentModel.model.isSubModel &&
+ agentModel.model.supportsVision &&
+ isAiModelMultimodal(agentModel.model)
+ ? agentModel
+ : undefined;
const selectedAnalyzerModel = hasImage
- ? preferredVisionModel
+ ? multimodalAgentVisionModel || preferredVisionModel
: usableAgentModel ||
allModels.find((m) => m.provider.type === "zhipu" && !m.model.isSubModel) ||
preferredVisionModel;
@@ -746,8 +757,15 @@ ${historyText}
(m.provider.type === "zhipu" || isLikelyZhipuEndpoint(m.provider.baseUrl)),
)?.provider;
const zhipuProvider = zhipuProviderFromAgent || zhipuProviderFallback;
+ const forceSingleMultimodalAgent = Boolean(hasImage && multimodalAgentVisionModel);
- if (hasImage && visionImages.length > 0 && zhipuProvider?.apiKey && zhipuProvider?.baseUrl) {
+ if (
+ !forceSingleMultimodalAgent &&
+ hasImage &&
+ visionImages.length > 0 &&
+ zhipuProvider?.apiKey &&
+ zhipuProvider?.baseUrl
+ ) {
const zhipuOfficialResult = await analyzeWithZhipuOfficialApi({
apiKey: zhipuProvider.apiKey,
baseUrl: zhipuProvider.baseUrl,
diff --git a/lib/db/ai-model-flags.ts b/lib/db/ai-model-flags.ts
index 1d9b3e0d2..bcdb4fea9 100644
--- a/lib/db/ai-model-flags.ts
+++ b/lib/db/ai-model-flags.ts
@@ -1,4 +1,5 @@
const AGENT_AUTO_MODEL_KEY = "agentAutoModel";
+const MULTIMODAL_MODEL_KEY = "multimodalModel";
function toRecord(value: unknown): Record | null {
if (!value || typeof value !== "object" || Array.isArray(value)) {
@@ -60,3 +61,35 @@ export function withAgentAutoModelFlag(
return Object.keys(next).length > 0 ? next : null;
}
+
+export function isAiModelMultimodal(input: {
+ capabilities?: unknown;
+ defaultParams?: unknown;
+}): boolean {
+ const defaultParams = toRecord(input.defaultParams);
+ if (typeof defaultParams?.[MULTIMODAL_MODEL_KEY] === "boolean") {
+ return defaultParams[MULTIMODAL_MODEL_KEY] as boolean;
+ }
+
+ const capabilities = toRecord(input.capabilities);
+ if (typeof capabilities?.[MULTIMODAL_MODEL_KEY] === "boolean") {
+ return capabilities[MULTIMODAL_MODEL_KEY] as boolean;
+ }
+
+ return false;
+}
+
+export function withMultimodalModelFlag(
+ capabilities: unknown,
+ enabled: boolean
+): Record | null {
+ const next = { ...(toRecord(capabilities) || {}) };
+
+ if (enabled) {
+ next[MULTIMODAL_MODEL_KEY] = true;
+ } else {
+ delete next[MULTIMODAL_MODEL_KEY];
+ }
+
+ return Object.keys(next).length > 0 ? next : null;
+}
diff --git a/lib/db/queries.ts b/lib/db/queries.ts
index f6c2d575f..d0b25d518 100644
--- a/lib/db/queries.ts
+++ b/lib/db/queries.ts
@@ -55,6 +55,7 @@ import { generateHashedPassword } from "./utils";
import {
isAiModelAgentAuto,
withAgentAutoModelFlag,
+ withMultimodalModelFlag,
} from "./ai-model-flags";
import {
defaultPointsSystemConfig,
@@ -1276,6 +1277,7 @@ export async function createAiModel({
fallbackModelId2,
imageGenConfig,
isAgentModel,
+ isMultimodal,
}: {
providerId: string;
modelId: string;
@@ -1291,6 +1293,7 @@ export async function createAiModel({
fallbackModelId2?: string;
imageGenConfig?: ImageGenConfig;
isAgentModel?: boolean;
+ isMultimodal?: boolean;
}): Promise {
try {
// 如果设置为默认，先取消其他默认
@@ -1321,6 +1324,10 @@ export async function createAiModel({
isAgentModel === undefined
? null
: withAgentAutoModelFlag(null, isAgentModel),
+ capabilities:
+ isMultimodal === undefined
+ ? null
+ : withMultimodalModelFlag(null, isMultimodal),
})
.returning();
} catch (_error) {
@@ -1347,6 +1354,7 @@ export async function updateAiModel({
fallbackModelId2,
imageGenConfig,
isAgentModel,
+ isMultimodal,
}: {
id: string;
providerId?: string;
@@ -1363,6 +1371,7 @@ export async function updateAiModel({
fallbackModelId2?: string;
imageGenConfig?: ImageGenConfig;
isAgentModel?: boolean;
+ isMultimodal?: boolean;
}): Promise {
try {
// 如果设置为默认，先取消其他默认
@@ -1390,9 +1399,12 @@ export async function updateAiModel({
if (fallbackModelId2 !== undefined) updateData.fallbackModelId2 = fallbackModelId2 || null;
// 🆕 图片生成配置
if (imageGenConfig !== undefined) updateData.imageGenConfig = imageGenConfig || null;
- if (isAgentModel !== undefined) {
+ if (isAgentModel !== undefined || isMultimodal !== undefined) {
const [existingModel] = await db
- .select({ defaultParams: aiModel.defaultParams })
+ .select({
+ defaultParams: aiModel.defaultParams,
+ capabilities: aiModel.capabilities,
+ })
.from(aiModel)
.where(eq(aiModel.id, id))
.limit(1);
@@ -1401,10 +1413,19 @@ export async function updateAiModel({
return [];
}
- updateData.defaultParams = withAgentAutoModelFlag(
- existingModel.defaultParams,
- isAgentModel
- );
+ if (isAgentModel !== undefined) {
+ updateData.defaultParams = withAgentAutoModelFlag(
+ existingModel.defaultParams,
+ isAgentModel
+ );
+ }
+
+ if (isMultimodal !== undefined) {
+ updateData.capabilities = withMultimodalModelFlag(
+ existingModel.capabilities,
+ isMultimodal
+ );
+ }
}
return await db