From 889cf33b63fb38808465fc975c25106d039cf645 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 6 Mar 2026 12:50:54 +0800 Subject: [PATCH] fix(agent): harden multimodal analysis routing and fallback --- lib/ai/auto-agent-graph.ts | 91 ++++++++++++---- lib/ai/smart-analyzer.ts | 210 ++++++++++++++++++++++++++++--------- 2 files changed, 234 insertions(+), 67 deletions(-) diff --git a/lib/ai/auto-agent-graph.ts b/lib/ai/auto-agent-graph.ts index 9d77d6489..a1dfd781d 100644 --- a/lib/ai/auto-agent-graph.ts +++ b/lib/ai/auto-agent-graph.ts @@ -90,6 +90,13 @@ function compactText(text: string, max = 48) { return `${normalized.slice(0, max)}...`; } +function getEnvInt(name: string, fallback: number, min: number, max: number): number { + const raw = process.env[name]; + const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN; + if (!Number.isFinite(parsed)) return fallback; + return Math.max(min, Math.min(max, parsed)); +} + async function withTimeoutFallback( promise: Promise, timeoutMs: number, @@ -110,8 +117,18 @@ async function withTimeoutFallback( const AGENT_VISION_MAX_IMAGES = 4; const AGENT_VISION_MAX_SIDE = 500; -const AGENT_OPERATION_TIMEOUT_MS = 120_000; -const AGENT_VISION_FETCH_TIMEOUT_MS = AGENT_OPERATION_TIMEOUT_MS; +const AGENT_OPERATION_TIMEOUT_MS = getEnvInt( + "AGENT_OPERATION_TIMEOUT_MS", + 18000, + 8000, + 120000, +); +const AGENT_VISION_FETCH_TIMEOUT_MS = getEnvInt( + "AGENT_VISION_FETCH_TIMEOUT_MS", + 12000, + 3000, + 60000, +); function isHttpUrl(value: string) { return /^https?:\/\//i.test(value); @@ -540,6 +557,10 @@ type EnabledModelWithProvider = Awaited< ReturnType >[number]; +function isAgentGenerateTextProviderType(providerType: string): boolean { + return ["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType); +} + function isLikelyVisionCapableModel(item: EnabledModelWithProvider) { if (!item.model.isEnabled || item.model.isSubModel) return false; if (item.model.supportsVision) return true; @@ -576,6 +597,30 @@ function isLikelyVisionCapableModel(item: EnabledModelWithProvider) { return false; } +function getAgentModelIncompatibleReason(params: { + item: EnabledModelWithProvider; + requireVision: boolean; +}): string | null { + const providerType = String(params.item.provider.type || ""); + if (!params.item.model.isEnabled || params.item.model.isSubModel) { + return "disabled_or_submodel"; + } + if (!isAgentGenerateTextProviderType(providerType)) { + return `unsupported_provider:${providerType || "unknown"}`; + } + if (params.requireVision && !isLikelyVisionCapableModel(params.item)) { + return "vision_capability_missing"; + } + return null; +} + +function isAgentGenerateTextCompatibleModel(params: { + item: EnabledModelWithProvider; + requireVision: boolean; +}): boolean { + return !getAgentModelIncompatibleReason(params); +} + function shouldPreferRemoteVisionUrls(item?: EnabledModelWithProvider) { if (!item) return false; const providerType = String(item.provider.type || ""); @@ -600,7 +645,9 @@ function shouldPreferRemoteVisionUrls(item?: EnabledModelWithProvider) { } function pickQwenVisionModel(models: EnabledModelWithProvider[]) { - const visionModels = models.filter((item) => isLikelyVisionCapableModel(item)); + const visionModels = models.filter((item) => + isAgentGenerateTextCompatibleModel({ item, requireVision: true }), + ); const zhipuVision = visionModels.find( (item) => item.provider.type === "zhipu" || @@ -626,7 +673,10 @@ function pickAgentVisionModel( agentModel: EnabledModelWithProvider | undefined, models: EnabledModelWithProvider[], ) { - if (agentModel && isLikelyVisionCapableModel(agentModel)) { + if ( + agentModel && + isAgentGenerateTextCompatibleModel({ item: agentModel, requireVision: true }) + ) { return agentModel; } @@ -658,32 +708,24 @@ function pickAgentReasoningModel( agentModel: EnabledModelWithProvider | undefined, models: EnabledModelWithProvider[], ) { - const isReasoningCompatibleProvider = (providerType: string) => - ["zhipu", "alibaba", "openai", "custom", "local", "volcano-multimodal", "volcano-vision"].includes(providerType); - if ( agentModel && - agentModel.model.isEnabled && - isAiModelMultimodal(agentModel.model) + isAgentGenerateTextCompatibleModel({ item: agentModel, requireVision: false }) ) { return agentModel; } - if (agentModel && isReasoningCompatibleProvider(agentModel.provider.type)) return agentModel; - const zhipuModel = models.find( + const candidates = models.filter((item) => + isAgentGenerateTextCompatibleModel({ item, requireVision: false }), + ); + const zhipuModel = candidates.find( (item) => item.provider.type === "zhipu" && item.model.isEnabled && - !item.model.isSubModel && - isReasoningCompatibleProvider(item.provider.type) + !item.model.isSubModel ); if (zhipuModel) return zhipuModel; - return models.find( - (item) => - item.model.isEnabled && - !item.model.isSubModel && - isReasoningCompatibleProvider(item.provider.type) - ); + return candidates[0]; } function parseJsonBlock>(text: string): T | null { @@ -738,6 +780,17 @@ async function generateAgentConversation( `[Auto Agent] Preferred analysis model: ${primaryAgentModel.model.displayName} (${primaryAgentModel.provider.type})`, ); } + if (primaryAgentModel) { + const reasoningIncompatibleReason = getAgentModelIncompatibleReason({ + item: primaryAgentModel, + requireVision: false, + }); + if (reasoningIncompatibleReason) { + console.warn( + `[Auto Agent] Preferred model incompatible for reasoning, skip direct use: ${primaryAgentModel.model.displayName} (${primaryAgentModel.provider.type}), reason=${reasoningIncompatibleReason}`, + ); + } + } const reasoningModelConfig = pickAgentReasoningModel(primaryAgentModel, allModels); if (!reasoningModelConfig) { return analysis.conversationalReply || null; diff --git a/lib/ai/smart-analyzer.ts b/lib/ai/smart-analyzer.ts index d6a43f323..d770e2558 100644 --- a/lib/ai/smart-analyzer.ts +++ b/lib/ai/smart-analyzer.ts @@ -247,7 +247,12 @@ function getEnvInt(name: string, fallback: number, min: number, max: number): nu const ANALYZER_MAX_IMAGE_COUNT = 2; const ANALYZER_IMAGE_MAX_SIDE = 500; -const ANALYZER_TIMEOUT_MS = 120000; +const ANALYZER_TIMEOUT_MS = getEnvInt( + "AGENT_ANALYZER_TIMEOUT_MS", + 18000, + 8000, + 120000, +); const ANALYZER_IMAGE_FETCH_TIMEOUT_MS = getEnvInt( "AGENT_IMAGE_FETCH_TIMEOUT_MS", 15000, @@ -256,8 +261,8 @@ const ANALYZER_IMAGE_FETCH_TIMEOUT_MS = getEnvInt( ); const ANALYZER_QWEN_FALLBACK_TIMEOUT_MS = getEnvInt( "AGENT_QWEN_FALLBACK_TIMEOUT_MS", - 35000, - 10000, + 12000, + 6000, 90000, ); const ANALYZER_QWEN_FALLBACK_RETRIES = getEnvInt( @@ -266,6 +271,12 @@ const ANALYZER_QWEN_FALLBACK_RETRIES = getEnvInt( 0, 1, ); +const ANALYZER_QWEN_FALLBACK_MAX_CANDIDATES = getEnvInt( + "AGENT_QWEN_FALLBACK_MAX_CANDIDATES", + 2, + 1, + 4, +); const ZHIPU_VISION_MODEL_CANDIDATES = [ "glm-4.6v-flash", "glm-4.6v", @@ -396,33 +407,50 @@ async function buildAnalyzerVisionImagesByMode(params: { return buildAnalyzerVisionImages(urls); } -async function getQwen3VlFallbackImageCandidates(imageUrls?: string[]): Promise { - const firstImageUrl = (imageUrls || []).find((url) => isHttpUrl(url)); - if (!firstImageUrl) return []; +async function getQwen3VlFallbackImageCandidates(params: { + imageUrls?: string[]; + visionImages?: Array<{ type: "image"; image: string }>; +}): Promise { + const allInputs = [ + ...(params.visionImages || []).map((item) => String(item.image || "").trim()), + ...((params.imageUrls || []).map((url) => String(url || "").trim())), + ].filter(Boolean); - const candidates: string[] = []; - const inlineDataUrl = await toAnalyzerDataUrl(firstImageUrl); - if (inlineDataUrl) { - candidates.push(inlineDataUrl); + const uniqueInputs = Array.from(new Set(allInputs)); + const inlineInputs = uniqueInputs.filter((value) => value.startsWith("data:image/")); + const remoteInputs = uniqueInputs.filter((value) => isHttpUrl(value)); + if (inlineInputs.length === 0 && remoteInputs.length === 0) return []; + + const candidates: string[] = [...inlineInputs]; + for (const remoteUrl of remoteInputs.slice(0, ANALYZER_MAX_IMAGE_COUNT)) { + const inlineDataUrl = await toAnalyzerDataUrl(remoteUrl); + if (inlineDataUrl) { + candidates.push(inlineDataUrl); + } + candidates.push(remoteUrl); } - candidates.push(firstImageUrl); - return Array.from(new Set(candidates)); + + return Array.from(new Set(candidates)).slice(0, ANALYZER_QWEN_FALLBACK_MAX_CANDIDATES); } async function analyzeWithHardcodedQwen3VlFallback(params: { text: string; hasImage: boolean; imageUrls?: string[]; + visionImages?: Array<{ type: "image"; image: string }>; historyMessages?: HistoryMessage[]; context?: string; reason: string; }): Promise { if (!params.hasImage) return null; - const imageCandidates = await getQwen3VlFallbackImageCandidates(params.imageUrls); + const imageCandidates = await getQwen3VlFallbackImageCandidates({ + imageUrls: params.imageUrls, + visionImages: params.visionImages, + }); if (imageCandidates.length === 0) { console.warn( - `[Smart Analyzer] qwen3-vl fallback skipped: no valid image url (${params.reason})`, + `[Smart Analyzer] qwen3-vl fallback skipped: no valid image input (${params.reason})`, ); return null; } @@ -450,7 +478,7 @@ async function analyzeWithHardcodedQwen3VlFallback(params: { try { const { callQwen3Vl } = await import("./qwen3-vl"); console.warn( - `[Smart Analyzer] Primary analysis failed, fallback to hardcoded qwen3-vl (${params.reason})`, + `[Smart Analyzer] Primary analysis failed, fallback to hardcoded qwen3-vl (${params.reason}, candidates=${imageCandidates.length})`, ); for (const imageInput of imageCandidates) { const inputMode = imageInput.startsWith("data:image/") ? "inline" : "remote"; @@ -488,8 +516,12 @@ async function summarizeWithHardcodedQwen3VlFallback(params: { text?: string; context?: string; imageUrls?: string[]; + visionImages?: Array<{ type: "image"; image: string }>; }): Promise { - const imageCandidates = await getQwen3VlFallbackImageCandidates(params.imageUrls); + const imageCandidates = await getQwen3VlFallbackImageCandidates({ + imageUrls: params.imageUrls, + visionImages: params.visionImages, + }); if (imageCandidates.length === 0) return null; const prompt = [ @@ -933,6 +965,7 @@ export async function summarizeImageForExecutionReport(params: { text, context, imageUrls, + visionImages, }); if (forcedSummary) return forcedSummary; } @@ -1027,6 +1060,7 @@ export async function summarizeImageForExecutionReport(params: { text, context, imageUrls, + visionImages, }); if (qwenFallbackSummary) return qwenFallbackSummary; @@ -1073,12 +1107,40 @@ type VisionSummaryModel = Awaited< ReturnType >[number]; -function isVisionSummaryCandidateModel(modelWithProvider: VisionSummaryModel): boolean { - const providerType = String(modelWithProvider.provider?.type || ""); - if (["fal-video", "volcano-video", "qianwen-video", "doubao-video", "ruxa-video"].includes(providerType)) { - return false; +function isAnalyzerGenerateTextProviderType(providerType: string): boolean { + return ["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType); +} + +function getAnalyzerModelIncompatibleReason(params: { + modelWithProvider: VisionSummaryModel; + requireVision: boolean; +}): string | null { + const providerType = String(params.modelWithProvider.provider?.type || ""); + const model = params.modelWithProvider.model; + if (!model?.isEnabled || model.isSubModel) { + return "disabled_or_submodel"; } - return isLikelyVisionCapableModel(modelWithProvider); + if (!isAnalyzerGenerateTextProviderType(providerType)) { + return `unsupported_provider:${providerType || "unknown"}`; + } + if (params.requireVision && !isLikelyVisionCapableModel(params.modelWithProvider)) { + return "vision_capability_missing"; + } + return null; +} + +function isAnalyzerGenerateTextCompatibleModel(params: { + modelWithProvider: VisionSummaryModel; + requireVision: boolean; +}): boolean { + return !getAnalyzerModelIncompatibleReason(params); +} + +function isVisionSummaryCandidateModel(modelWithProvider: VisionSummaryModel): boolean { + return isAnalyzerGenerateTextCompatibleModel({ + modelWithProvider, + requireVision: true, + }); } function collectVisionSummaryCandidates(params: { @@ -1282,27 +1344,32 @@ async function analyzWithLLM( ) : undefined; const agentModel = preferredModel || configuredAgentModel; - const isReasoningCompatibleProvider = (providerType: string) => - ["zhipu", "alibaba", "openai", "custom", "local", "volcano-multimodal", "volcano-vision"].includes(providerType); + const analyzerTextCandidates = allModels.filter((item) => + isAnalyzerGenerateTextCompatibleModel({ + modelWithProvider: item, + requireVision: false, + }), + ); + const analyzerVisionCandidates = analyzerTextCandidates.filter((item) => + isLikelyVisionCapableModel(item), + ); const isLikelyZhipuVisionModel = (displayName: string, modelId: string, providerType: string) => providerType === "zhipu" || /(glm[-_.]?4v|glm[-_.]?4\.?1v|glm[-_.]?4\.?5v|vision)/i.test( `${displayName} ${modelId}`, ); - const zhipuVisionModel = allModels.find( + const zhipuVisionModel = analyzerVisionCandidates.find( (m) => - isLikelyVisionCapableModel(m) && isLikelyZhipuVisionModel(m.model.displayName, m.model.modelId, m.provider.type), ); - const qwenVisionModel = allModels.find( + const qwenVisionModel = analyzerVisionCandidates.find( (m) => m.provider.type === "alibaba" && - isLikelyVisionCapableModel(m) && /(qwen|通义|vl)/i.test(`${m.model.displayName} ${m.model.modelId}`) ); const fallbackVisionModel = qwenVisionModel || - allModels.find( + analyzerVisionCandidates.find( (m) => m.provider.type === "alibaba" && isLikelyVisionCapableModel(m), @@ -1310,21 +1377,57 @@ async function analyzWithLLM( const preferredVisionModel = zhipuVisionModel || fallbackVisionModel || - allModels.find((m) => isLikelyVisionCapableModel(m)); + analyzerVisionCandidates[0]; const usableAgentModel = - agentModel && isReasoningCompatibleProvider(agentModel.provider.type) + agentModel && + isAnalyzerGenerateTextCompatibleModel({ + modelWithProvider: agentModel, + requireVision: false, + }) ? agentModel : undefined; const multimodalAgentVisionModel = agentModel && - isLikelyVisionCapableModel(agentModel) + isAnalyzerGenerateTextCompatibleModel({ + modelWithProvider: agentModel, + requireVision: true, + }) ? agentModel : undefined; + + if (preferredModel) { + const preferredIncompatibleReason = getAnalyzerModelIncompatibleReason({ + modelWithProvider: preferredModel, + requireVision: hasImage, + }); + if (preferredIncompatibleReason) { + console.warn( + `[Smart Analyzer] Preferred analyzer model incompatible, skip: ${preferredModel.model.displayName} (${preferredModel.provider.type}), reason=${preferredIncompatibleReason}`, + ); + } + } + + if (agentModel && !usableAgentModel) { + const agentIncompatibleReason = getAnalyzerModelIncompatibleReason({ + modelWithProvider: agentModel, + requireVision: false, + }); + if (agentIncompatibleReason) { + console.warn( + `[Smart Analyzer] Agent analyzer model incompatible, skip: ${agentModel.model.displayName} (${agentModel.provider.type}), reason=${agentIncompatibleReason}`, + ); + } + } + const selectedAnalyzerModel = hasImage - ? multimodalAgentVisionModel || preferredVisionModel || usableAgentModel + ? multimodalAgentVisionModel || + preferredVisionModel || + analyzerVisionCandidates[0] || + usableAgentModel || + analyzerTextCandidates[0] : usableAgentModel || - allModels.find((m) => m.provider.type === "zhipu" && !m.model.isSubModel) || - preferredVisionModel; + analyzerTextCandidates.find((m) => m.provider.type === "zhipu" && !m.model.isSubModel) || + analyzerTextCandidates[0]; if (preferredModel && selectedAnalyzerModel?.model.id === preferredModel.model.id) { console.log( @@ -1332,12 +1435,33 @@ async function analyzWithLLM( ); } + console.log( + `[Smart Analyzer] Analyzer candidate stats: text=${analyzerTextCandidates.length}, vision=${analyzerVisionCandidates.length}, selected=${selectedAnalyzerModel?.model.displayName || "none"} (${selectedAnalyzerModel?.provider?.type || "none"})`, + ); + + const preferRemoteVisionUrls = hasImage + ? shouldPreferRemoteVisionUrls(selectedAnalyzerModel) + : false; + const visionImages = hasImage + ? await buildAnalyzerVisionImagesByMode({ + imageUrls, + preferRemoteUrls: preferRemoteVisionUrls, + }) + : []; + if (visionImages.length > 0) { + const inlineCount = visionImages.filter((item) => item.image.startsWith("data:image/")).length; + console.log( + `[Smart Analyzer] Prepared analyzer images: total=${visionImages.length}, inline=${inlineCount}, remote=${visionImages.length - inlineCount}, preferRemote=${preferRemoteVisionUrls}`, + ); + } + if (!selectedAnalyzerModel) { console.log("[Smart Analyzer] No available analyzer model found"); return analyzeWithHardcodedQwen3VlFallback({ text, hasImage, imageUrls, + visionImages, historyMessages, context, reason: "no_available_model", @@ -1352,6 +1476,7 @@ async function analyzWithLLM( text, hasImage, imageUrls, + visionImages, historyMessages, context, reason: "force_qwen3vl_multimodal_flag_only", @@ -1365,6 +1490,7 @@ async function analyzWithLLM( text, hasImage, imageUrls, + visionImages, historyMessages, context, reason: "model_runtime_unavailable", @@ -1398,20 +1524,6 @@ ${historyText} ? `${contextMessage}用户上传了图片，请求内容："${text}"` : `${contextMessage}用户请求（无图片）："${text}"`; - const preferRemoteVisionUrls = shouldPreferRemoteVisionUrls(selectedAnalyzerModel); - const visionImages = hasImage - ? await buildAnalyzerVisionImagesByMode({ - imageUrls, - preferRemoteUrls: preferRemoteVisionUrls, - }) - : []; - if (visionImages.length > 0) { - const inlineCount = visionImages.filter((item) => item.image.startsWith("data:image/")).length; - console.log( - `[Smart Analyzer] Prepared analyzer images: total=${visionImages.length}, inline=${inlineCount}, remote=${visionImages.length - inlineCount}, preferRemote=${preferRemoteVisionUrls}`, - ); - } - const zhipuProviderFromAgent = usableAgentModel && usableAgentModel.provider && @@ -1493,6 +1605,7 @@ ${historyText} text, hasImage, imageUrls, + visionImages, historyMessages, context, reason: "primary_non_json_response", @@ -1510,6 +1623,7 @@ ${historyText} text, hasImage, imageUrls, + visionImages, historyMessages, context, reason: "primary_analysis_failed",