diff --git a/components/multimodal-input.tsx b/components/multimodal-input.tsx index 87de8b00d..f92b5c3b0 100644 --- a/components/multimodal-input.tsx +++ b/components/multimodal-input.tsx @@ -14,6 +14,7 @@ type DynamicModel = { providerType?: string; supportsVision: boolean; supportsImageGen?: boolean; + isMultimodal?: boolean; isDefault: boolean; providerName: string; isSubModel?: boolean; // 🆕 是否为子模型 @@ -1594,7 +1595,7 @@ function PureModelSelectorCompact({ // 优先选择支持视觉的模型(如果有附件) if (hasAttachments) { - const visionModel = dynamicModels.find((m) => m.supportsVision); + const visionModel = dynamicModels.find((m) => m.supportsVision || m.isMultimodal); if (visionModel) return visionModel; } diff --git a/lib/ai/model-router.ts b/lib/ai/model-router.ts index 85e61645d..8b9cdfb51 100644 --- a/lib/ai/model-router.ts +++ b/lib/ai/model-router.ts @@ -5,6 +5,8 @@ * 核心逻辑:图片编辑场景优先使用 alibaba-image (灵魂4.5) */ +import { isAiModelMultimodal } from "@/lib/db/ai-model-flags"; + // 使用动态导入避免循环依赖 // import { getEnabledAiModelsWithProvider } from "@/lib/db/queries"; @@ -17,6 +19,14 @@ export interface RouteResult { reason: string; } +function hasVisionCapability(model: { + supportsVision?: boolean | null; + capabilities?: unknown; + defaultParams?: unknown; +}) { + return Boolean(model.supportsVision) || isAiModelMultimodal(model); +} + // 意图到模型的映射配置 // providerTypes 按优先级排序,第一个是首选 const INTENT_MODEL_MAP: Record< @@ -144,7 +154,7 @@ export async function routeToModel(params: { if (configuredModel) { const isImageIntent = intent === "image_edit" || intent === "image_gen"; - const matchesVision = !config.requireVision || configuredModel.model.supportsVision; + const matchesVision = !config.requireVision || hasVisionCapability(configuredModel.model); const matchesImageGen = !config.requireImageGen || configuredModel.model.supportsImageGen; const allowImageIntentWithImageGenOnly = isImageIntent && configuredModel.model.supportsImageGen; @@ -181,7 +191,7 @@ export async function routeToModel(params: { if (m.provider.type !== providerType) return false; // 检查能力要求 - if (config.requireVision && !m.model.supportsVision) return false; + if (config.requireVision && !hasVisionCapability(m.model)) return false; if (config.requireImageGen && !m.model.supportsImageGen) return false; // 排除子模型(只选主模型) diff --git a/lib/ai/smart-analyzer.ts b/lib/ai/smart-analyzer.ts index 2ffcf35f5..bd1c0c01c 100644 --- a/lib/ai/smart-analyzer.ts +++ b/lib/ai/smart-analyzer.ts @@ -681,6 +681,7 @@ async function summarizeWithVolcanoOfficialApi(params: { baseUrl: string; promptText: string; imageUrls: string[]; + modelId?: string; }): Promise { const endpoint = normalizeVolcanoResponsesEndpoint(params.baseUrl); const rewrittenUrls = await Promise.all( @@ -695,7 +696,7 @@ async function summarizeWithVolcanoOfficialApi(params: { Authorization: `Bearer ${params.apiKey}`, }, body: JSON.stringify({ - model: "doubao-seed-2-0-pro-260215", + model: String(params.modelId || "").trim() || "doubao-seed-2-0-pro-260215", input: [ { role: "user", @@ -737,6 +738,91 @@ async function summarizeWithVolcanoOfficialApi(params: { return null; } +async function analyzeWithVolcanoOfficialApi(params: { + apiKey: string; + baseUrl: string; + userMessage: string; + imageUrls: string[]; + userText: string; + hasImage: boolean; + modelId?: string; +}): Promise { + const endpoint = normalizeVolcanoResponsesEndpoint(params.baseUrl); + const rewrittenUrls = ( + await Promise.all( + params.imageUrls.map((url) => rewriteUploadPublicUrlForVolcanoSummary(url)), + ) + ) + .filter((url) => isHttpUrl(url)) + .slice(0, ANALYZER_MAX_IMAGE_COUNT); + + if (rewrittenUrls.length === 0) return null; + + try { + const response = await fetch(endpoint, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${params.apiKey}`, + }, + body: JSON.stringify({ + model: String(params.modelId || "").trim() || "doubao-seed-2-0-pro-260215", + input: [ + { + role: "system", + content: [ + { + type: "input_text", + text: ANALYZER_SYSTEM_PROMPT, + }, + ], + }, + { + role: "user", + content: [ + { + type: "input_text", + text: params.userMessage, + }, + ...rewrittenUrls.map((url) => ({ type: "input_image", image_url: url })), + ], + }, + ], + }), + }); + + const payload = await response.json().catch(() => ({})); + if (!response.ok || payload?.error) { + return null; + } + + const directText = String(payload?.output_text || "").trim(); + const output = Array.isArray(payload?.output) ? payload.output : []; + const chunks: string[] = []; + for (const item of output) { + const content = Array.isArray(item?.content) ? item.content : []; + for (const part of content) { + if (typeof part?.text === "string" && part.text.trim()) { + chunks.push(part.text.trim()); + } else if (typeof part?.output_text === "string" && part.output_text.trim()) { + chunks.push(part.output_text.trim()); + } + } + } + + const responseText = directText || chunks.join("\n").trim(); + if (!responseText) return null; + + return parseAnalysisResultFromModelText( + responseText, + params.userText, + params.hasImage, + ); + } catch { + return null; + } +} + function normalizeZhipuChatEndpoint(baseUrl: string): string { const cleaned = baseUrl.replace(/\/+$/, ""); if (/\/chat\/completions$/i.test(cleaned)) return cleaned; @@ -1137,6 +1223,7 @@ export async function summarizeImageForExecutionReport(params: { baseUrl: volcanoProvider.baseUrl, promptText, imageUrls: (imageUrls || []).filter((url) => isHttpUrl(url)).slice(0, ANALYZER_MAX_IMAGE_COUNT), + modelId: agentModel?.model?.modelId, }); if (summary) return await finalizeVisualSummary(summary); } @@ -1339,9 +1426,11 @@ function isLikelyVisionCapableModel(modelWithProvider: { const model = modelWithProvider.model; const providerType = String(modelWithProvider.provider?.type || ""); if (!model?.isEnabled || model.isSubModel) return false; + if (isAiModelMultimodal(model)) return true; const modelText = `${model.displayName || ""} ${model.modelId || ""}`.toLowerCase(); const likelyImageGenerationOnly = !model.supportsVision && + !isAiModelMultimodal(model) && Boolean(model.supportsImageGen) && ((["custom", "openai", "local"].includes(providerType) && /(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0|seedream|gpt[-_.]?image|nano[-_.]?banana|flux|imagen|image[-_.]?edit)/i.test( @@ -1352,7 +1441,6 @@ function isLikelyVisionCapableModel(modelWithProvider: { )); if (likelyImageGenerationOnly) return false; if (model.supportsVision) return true; - if (isAiModelMultimodal(model)) return true; if (providerType === "volcano-vision" || providerType === "volcano-multimodal") return true; @@ -1382,6 +1470,18 @@ function isLikelyVisionCapableModel(modelWithProvider: { return false; } +function getPreferredVolcanoMultimodalAnalyzerCandidate( + ...candidates: Array +): VisionSummaryModel | null { + for (const candidate of candidates) { + if (!candidate?.provider?.apiKey || !candidate?.provider?.baseUrl) continue; + if (!isLikelyVolcanoEndpoint(candidate.provider.baseUrl)) continue; + if (!isAiModelMultimodal(candidate.model)) continue; + return candidate; + } + return null; +} + function shouldForceQwen3VlForImageAnalysis(modelWithProvider?: { model: { supportsVision?: boolean; @@ -1694,6 +1794,36 @@ ${historyText} ? `${contextMessage}用户上传了图片,请求内容:"${text}"` : `${contextMessage}用户请求(无图片):"${text}"`; + const volcanoAnalyzerCandidate = getPreferredVolcanoMultimodalAnalyzerCandidate( + selectedAnalyzerModel, + preferredModel, + agentModel, + ); + if ( + hasImage && + visionImages.length > 0 && + volcanoAnalyzerCandidate?.provider?.apiKey && + volcanoAnalyzerCandidate?.provider?.baseUrl + ) { + const volcanoOfficialResult = await analyzeWithVolcanoOfficialApi({ + apiKey: volcanoAnalyzerCandidate.provider.apiKey, + baseUrl: volcanoAnalyzerCandidate.provider.baseUrl, + modelId: volcanoAnalyzerCandidate.model.modelId, + userMessage, + userText: text, + hasImage, + imageUrls: (imageUrls || []) + .filter((url) => isHttpUrl(url)) + .slice(0, ANALYZER_MAX_IMAGE_COUNT), + }); + if (volcanoOfficialResult) { + console.log( + `[Smart Analyzer] Volcano official analyzer used: ${volcanoAnalyzerCandidate.model.displayName} (${volcanoAnalyzerCandidate.model.modelId})`, + ); + return volcanoOfficialResult; + } + } + const zhipuProviderFromAgent = usableAgentModel && usableAgentModel.provider &&