diff --git a/lib/ai/auto-agent-graph.ts b/lib/ai/auto-agent-graph.ts index 7eb4eec99..7b5ae4986 100644 --- a/lib/ai/auto-agent-graph.ts +++ b/lib/ai/auto-agent-graph.ts @@ -147,8 +147,19 @@ async function toAgentVisionDataUrl(imageUrl: string): Promise { } } -async function buildAgentVisionInputs(imageUrls: string[]) { +async function buildAgentVisionInputs( + imageUrls: string[], + options?: { + preferRemoteUrls?: boolean; + }, +) { if (imageUrls.length === 0) return [] as Array<{ type: "image"; image: string }>; + if (options?.preferRemoteUrls) { + return imageUrls.map((url) => ({ + type: "image" as const, + image: url, + })); + } const convertedResults = await Promise.all(imageUrls.map((url) => toAgentVisionDataUrl(url))); const inlineCount = convertedResults.filter(Boolean).length; @@ -492,7 +503,7 @@ function isLikelyVisionCapableModel(item: EnabledModelWithProvider) { if (item.model.supportsVision) return true; if (isAiModelMultimodal(item.model)) return true; - const providerType = item.provider.type || ""; + const providerType = String(item.provider.type || ""); const modelText = `${item.model.displayName || ""} ${item.model.modelId || ""}`.toLowerCase(); if (providerType === "volcano-multimodal") return true; @@ -523,6 +534,24 @@ function isLikelyVisionCapableModel(item: EnabledModelWithProvider) { return false; } +function shouldPreferRemoteVisionUrls(item?: EnabledModelWithProvider) { + if (!item) return false; + const providerType = String(item.provider.type || ""); + const baseUrl = String((item.provider as { baseUrl?: string | null })?.baseUrl || "").toLowerCase(); + const modelText = `${item.model.displayName || ""} ${item.model.modelId || ""}`.toLowerCase(); + + if (providerType === "volcano-multimodal" || providerType === "doubao-video" || providerType === "volcano-video") { + return true; + } + if (baseUrl.includes("ark.cn-beijing.volces.com")) { + return true; + } + if (/(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0)/i.test(modelText)) { + return true; + } + return false; +} + function pickQwenVisionModel(models: EnabledModelWithProvider[]) { const visionModels = models.filter((item) => isLikelyVisionCapableModel(item)); const zhipuVision = visionModels.find( @@ -651,7 +680,10 @@ async function generateAgentConversation( 2) 只写客观可见信息,不要编造品牌、尺寸、镜头参数。 3) 全部输出控制在 6 行内。`; - const visionInputs = await buildAgentVisionInputs(validImageUrls); + const preferRemoteVisionUrls = shouldPreferRemoteVisionUrls(visionModelConfig); + const visionInputs = await buildAgentVisionInputs(validImageUrls, { + preferRemoteUrls: preferRemoteVisionUrls, + }); const { text: visionText } = await generateText({ model: visionModel, messages: [ @@ -666,7 +698,7 @@ async function generateAgentConversation( visualAnalysis = visionText.trim(); console.log( - `[Auto Agent] Vision analysis model: ${visionModelConfig.model.displayName} (${visionModelConfig.provider.type})` + `[Auto Agent] Vision analysis model: ${visionModelConfig.model.displayName} (${visionModelConfig.provider.type}), preferRemote=${preferRemoteVisionUrls}` ); } catch (error) { console.warn("[Auto Agent] Vision analysis failed, continue without visual summary"); diff --git a/lib/ai/smart-analyzer.ts b/lib/ai/smart-analyzer.ts index 91fa20832..d892cd77c 100644 --- a/lib/ai/smart-analyzer.ts +++ b/lib/ai/smart-analyzer.ts @@ -324,19 +324,68 @@ async function buildAnalyzerVisionImages(imageUrls?: string[]) { }); } +function shouldPreferRemoteVisionUrls(modelWithProvider?: { + model?: { + displayName?: string | null; + modelId?: string | null; + } | null; + provider?: { + type?: string | null; + baseUrl?: string | null; + } | null; +}): boolean { + if (!modelWithProvider) return false; + const providerType = String(modelWithProvider.provider?.type || ""); + const baseUrl = String(modelWithProvider.provider?.baseUrl || "").toLowerCase(); + const modelText = `${modelWithProvider.model?.displayName || ""} ${modelWithProvider.model?.modelId || ""}`.toLowerCase(); + + if (providerType === "volcano-multimodal" || providerType === "doubao-video" || providerType === "volcano-video") { + return true; + } + if (baseUrl.includes("ark.cn-beijing.volces.com")) { + return true; + } + if (/(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0)/i.test(modelText)) { + return true; + } + return false; +} + +async function buildAnalyzerVisionImagesByMode(params: { + imageUrls?: string[]; + preferRemoteUrls: boolean; +}) { + const urls = (params.imageUrls || []) + .filter((url) => isHttpUrl(url)) + .slice(0, ANALYZER_MAX_IMAGE_COUNT); + if (urls.length === 0) return [] as Array<{ type: "image"; image: string }>; + if (params.preferRemoteUrls) { + return urls.map((url) => ({ type: "image" as const, image: url })); + } + return buildAnalyzerVisionImages(urls); +} + function parseAnalysisResultFromModelText( responseText: string, userText: string, hasImage: boolean, ): AnalysisResult | null { const jsonMatch = responseText.match(/\{[\s\S]*\}/); - if (!jsonMatch) return null; + if (!jsonMatch) { + console.warn("[Smart Analyzer] Model response missing JSON:", responseText.slice(0, 240)); + return null; + } try { const result = JSON.parse(jsonMatch[0]) as AnalysisResult; return normalizeIntentForImageContext(result, userText, hasImage); } catch (error) { - console.error("[Smart Analyzer] Failed to parse model response:", error); + console.error( + "[Smart Analyzer] Failed to parse model response:", + error, + "raw=", + jsonMatch[0].slice(0, 240), + ); return null; } } @@ -780,11 +829,17 @@ ${historyText} ? `${contextMessage}用户上传了图片,请求内容:"${text}"` : `${contextMessage}用户请求(无图片):"${text}"`; - const visionImages = hasImage ? await buildAnalyzerVisionImages(imageUrls) : []; + const preferRemoteVisionUrls = shouldPreferRemoteVisionUrls(selectedAnalyzerModel); + const visionImages = hasImage + ? await buildAnalyzerVisionImagesByMode({ + imageUrls, + preferRemoteUrls: preferRemoteVisionUrls, + }) + : []; if (visionImages.length > 0) { const inlineCount = visionImages.filter((item) => item.image.startsWith("data:image/")).length; console.log( - `[Smart Analyzer] Prepared analyzer images: total=${visionImages.length}, inline=${inlineCount}, remote=${visionImages.length - inlineCount}`, + `[Smart Analyzer] Prepared analyzer images: total=${visionImages.length}, inline=${inlineCount}, remote=${visionImages.length - inlineCount}, preferRemote=${preferRemoteVisionUrls}`, ); }