fix(agent): harden multimodal analysis routing and fallback

lib/ai/auto-agent-graph.ts

@@ -90,6 +90,13 @@ function compactText(text: string, max = 48) {
   return `${normalized.slice(0, max)}...`;
 }
 
+function getEnvInt(name: string, fallback: number, min: number, max: number): number {
+  const raw = process.env[name];
+  const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
+  if (!Number.isFinite(parsed)) return fallback;
+  return Math.max(min, Math.min(max, parsed));
+}
+
 async function withTimeoutFallback<T>(
   promise: Promise<T>,
   timeoutMs: number,
@@ -110,8 +117,18 @@ async function withTimeoutFallback<T>(
 
 const AGENT_VISION_MAX_IMAGES = 4;
 const AGENT_VISION_MAX_SIDE = 500;
-const AGENT_OPERATION_TIMEOUT_MS = 120_000;
-const AGENT_VISION_FETCH_TIMEOUT_MS = AGENT_OPERATION_TIMEOUT_MS;
+const AGENT_OPERATION_TIMEOUT_MS = getEnvInt(
+  "AGENT_OPERATION_TIMEOUT_MS",
+  18000,
+  8000,
+  120000,
+);
+const AGENT_VISION_FETCH_TIMEOUT_MS = getEnvInt(
+  "AGENT_VISION_FETCH_TIMEOUT_MS",
+  12000,
+  3000,
+  60000,
+);
 
 function isHttpUrl(value: string) {
   return /^https?:\/\//i.test(value);
@@ -540,6 +557,10 @@ type EnabledModelWithProvider = Awaited<
   ReturnType<typeof import("@/lib/db/queries")["getEnabledAiModelsWithProvider"]>
 >[number];
 
+function isAgentGenerateTextProviderType(providerType: string): boolean {
+  return ["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType);
+}
+
 function isLikelyVisionCapableModel(item: EnabledModelWithProvider) {
   if (!item.model.isEnabled || item.model.isSubModel) return false;
   if (item.model.supportsVision) return true;
@@ -576,6 +597,30 @@ function isLikelyVisionCapableModel(item: EnabledModelWithProvider) {
   return false;
 }
 
+function getAgentModelIncompatibleReason(params: {
+  item: EnabledModelWithProvider;
+  requireVision: boolean;
+}): string | null {
+  const providerType = String(params.item.provider.type || "");
+  if (!params.item.model.isEnabled || params.item.model.isSubModel) {
+    return "disabled_or_submodel";
+  }
+  if (!isAgentGenerateTextProviderType(providerType)) {
+    return `unsupported_provider:${providerType || "unknown"}`;
+  }
+  if (params.requireVision && !isLikelyVisionCapableModel(params.item)) {
+    return "vision_capability_missing";
+  }
+  return null;
+}
+
+function isAgentGenerateTextCompatibleModel(params: {
+  item: EnabledModelWithProvider;
+  requireVision: boolean;
+}): boolean {
+  return !getAgentModelIncompatibleReason(params);
+}
+
 function shouldPreferRemoteVisionUrls(item?: EnabledModelWithProvider) {
   if (!item) return false;
   const providerType = String(item.provider.type || "");
@@ -600,7 +645,9 @@ function shouldPreferRemoteVisionUrls(item?: EnabledModelWithProvider) {
 }
 
 function pickQwenVisionModel(models: EnabledModelWithProvider[]) {
-  const visionModels = models.filter((item) => isLikelyVisionCapableModel(item));
+  const visionModels = models.filter((item) =>
+    isAgentGenerateTextCompatibleModel({ item, requireVision: true }),
+  );
   const zhipuVision = visionModels.find(
     (item) =>
       item.provider.type === "zhipu" ||
@@ -626,7 +673,10 @@ function pickAgentVisionModel(
   agentModel: EnabledModelWithProvider | undefined,
   models: EnabledModelWithProvider[],
 ) {
-  if (agentModel && isLikelyVisionCapableModel(agentModel)) {
+  if (
+    agentModel &&
+    isAgentGenerateTextCompatibleModel({ item: agentModel, requireVision: true })
+  ) {
     return agentModel;
   }
 
@@ -658,32 +708,24 @@ function pickAgentReasoningModel(
   agentModel: EnabledModelWithProvider | undefined,
   models: EnabledModelWithProvider[],
 ) {
-  const isReasoningCompatibleProvider = (providerType: string) =>
-    ["zhipu", "alibaba", "openai", "custom", "local", "volcano-multimodal", "volcano-vision"].includes(providerType);
-
   if (
     agentModel &&
-    agentModel.model.isEnabled &&
-    isAiModelMultimodal(agentModel.model)
+    isAgentGenerateTextCompatibleModel({ item: agentModel, requireVision: false })
   ) {
     return agentModel;
   }
 
-  if (agentModel && isReasoningCompatibleProvider(agentModel.provider.type)) return agentModel;
-  const zhipuModel = models.find(
+  const candidates = models.filter((item) =>
+    isAgentGenerateTextCompatibleModel({ item, requireVision: false }),
+  );
+  const zhipuModel = candidates.find(
     (item) =>
       item.provider.type === "zhipu" &&
       item.model.isEnabled &&
-      !item.model.isSubModel &&
-      isReasoningCompatibleProvider(item.provider.type)
+      !item.model.isSubModel
   );
   if (zhipuModel) return zhipuModel;
-  return models.find(
-    (item) =>
-      item.model.isEnabled &&
-      !item.model.isSubModel &&
-      isReasoningCompatibleProvider(item.provider.type)
-  );
+  return candidates[0];
 }
 
 function parseJsonBlock<T = Record<string, unknown>>(text: string): T | null {
@@ -738,6 +780,17 @@ async function generateAgentConversation(
         `[Auto Agent] Preferred analysis model: ${primaryAgentModel.model.displayName} (${primaryAgentModel.provider.type})`,
       );
     }
+    if (primaryAgentModel) {
+      const reasoningIncompatibleReason = getAgentModelIncompatibleReason({
+        item: primaryAgentModel,
+        requireVision: false,
+      });
+      if (reasoningIncompatibleReason) {
+        console.warn(
+          `[Auto Agent] Preferred model incompatible for reasoning, skip direct use: ${primaryAgentModel.model.displayName} (${primaryAgentModel.provider.type}), reason=${reasoningIncompatibleReason}`,
+        );
+      }
+    }
     const reasoningModelConfig = pickAgentReasoningModel(primaryAgentModel, allModels);
     if (!reasoningModelConfig) {
       return analysis.conversationalReply || null;

lib/ai/smart-analyzer.ts

@@ -247,7 +247,12 @@ function getEnvInt(name: string, fallback: number, min: number, max: number): nu
 
 const ANALYZER_MAX_IMAGE_COUNT = 2;
 const ANALYZER_IMAGE_MAX_SIDE = 500;
-const ANALYZER_TIMEOUT_MS = 120000;
+const ANALYZER_TIMEOUT_MS = getEnvInt(
+  "AGENT_ANALYZER_TIMEOUT_MS",
+  18000,
+  8000,
+  120000,
+);
 const ANALYZER_IMAGE_FETCH_TIMEOUT_MS = getEnvInt(
   "AGENT_IMAGE_FETCH_TIMEOUT_MS",
   15000,
@@ -256,8 +261,8 @@ const ANALYZER_IMAGE_FETCH_TIMEOUT_MS = getEnvInt(
 );
 const ANALYZER_QWEN_FALLBACK_TIMEOUT_MS = getEnvInt(
   "AGENT_QWEN_FALLBACK_TIMEOUT_MS",
-  35000,
-  10000,
+  12000,
+  6000,
   90000,
 );
 const ANALYZER_QWEN_FALLBACK_RETRIES = getEnvInt(
@@ -266,6 +271,12 @@ const ANALYZER_QWEN_FALLBACK_RETRIES = getEnvInt(
   0,
   1,
 );
+const ANALYZER_QWEN_FALLBACK_MAX_CANDIDATES = getEnvInt(
+  "AGENT_QWEN_FALLBACK_MAX_CANDIDATES",
+  2,
+  1,
+  4,
+);
 const ZHIPU_VISION_MODEL_CANDIDATES = [
   "glm-4.6v-flash",
   "glm-4.6v",
@@ -396,33 +407,50 @@ async function buildAnalyzerVisionImagesByMode(params: {
   return buildAnalyzerVisionImages(urls);
 }
 
-async function getQwen3VlFallbackImageCandidates(imageUrls?: string[]): Promise<string[]> {
-  const firstImageUrl = (imageUrls || []).find((url) => isHttpUrl(url));
-  if (!firstImageUrl) return [];
+async function getQwen3VlFallbackImageCandidates(params: {
+  imageUrls?: string[];
+  visionImages?: Array<{ type: "image"; image: string }>;
+}): Promise<string[]> {
+  const allInputs = [
+    ...(params.visionImages || []).map((item) => String(item.image || "").trim()),
+    ...((params.imageUrls || []).map((url) => String(url || "").trim())),
+  ].filter(Boolean);
 
-  const candidates: string[] = [];
-  const inlineDataUrl = await toAnalyzerDataUrl(firstImageUrl);
-  if (inlineDataUrl) {
-    candidates.push(inlineDataUrl);
+  const uniqueInputs = Array.from(new Set(allInputs));
+  const inlineInputs = uniqueInputs.filter((value) => value.startsWith("data:image/"));
+  const remoteInputs = uniqueInputs.filter((value) => isHttpUrl(value));
+  if (inlineInputs.length === 0 && remoteInputs.length === 0) return [];
+
+  const candidates: string[] = [...inlineInputs];
+  for (const remoteUrl of remoteInputs.slice(0, ANALYZER_MAX_IMAGE_COUNT)) {
+    const inlineDataUrl = await toAnalyzerDataUrl(remoteUrl);
+    if (inlineDataUrl) {
+      candidates.push(inlineDataUrl);
+    }
+    candidates.push(remoteUrl);
   }
-  candidates.push(firstImageUrl);
-  return Array.from(new Set(candidates));
+
+  return Array.from(new Set(candidates)).slice(0, ANALYZER_QWEN_FALLBACK_MAX_CANDIDATES);
 }
 
 async function analyzeWithHardcodedQwen3VlFallback(params: {
   text: string;
   hasImage: boolean;
   imageUrls?: string[];
+  visionImages?: Array<{ type: "image"; image: string }>;
   historyMessages?: HistoryMessage[];
   context?: string;
   reason: string;
 }): Promise<AnalysisResult | null> {
   if (!params.hasImage) return null;
 
-  const imageCandidates = await getQwen3VlFallbackImageCandidates(params.imageUrls);
+  const imageCandidates = await getQwen3VlFallbackImageCandidates({
+    imageUrls: params.imageUrls,
+    visionImages: params.visionImages,
+  });
   if (imageCandidates.length === 0) {
     console.warn(
-      `[Smart Analyzer] qwen3-vl fallback skipped: no valid image url (${params.reason})`,
+      `[Smart Analyzer] qwen3-vl fallback skipped: no valid image input (${params.reason})`,
     );
     return null;
   }
@@ -450,7 +478,7 @@ async function analyzeWithHardcodedQwen3VlFallback(params: {
   try {
     const { callQwen3Vl } = await import("./qwen3-vl");
     console.warn(
-      `[Smart Analyzer] Primary analysis failed, fallback to hardcoded qwen3-vl (${params.reason})`,
+      `[Smart Analyzer] Primary analysis failed, fallback to hardcoded qwen3-vl (${params.reason}, candidates=${imageCandidates.length})`,
     );
     for (const imageInput of imageCandidates) {
       const inputMode = imageInput.startsWith("data:image/") ? "inline" : "remote";
@@ -488,8 +516,12 @@ async function summarizeWithHardcodedQwen3VlFallback(params: {
   text?: string;
   context?: string;
   imageUrls?: string[];
+  visionImages?: Array<{ type: "image"; image: string }>;
 }): Promise<string | null> {
-  const imageCandidates = await getQwen3VlFallbackImageCandidates(params.imageUrls);
+  const imageCandidates = await getQwen3VlFallbackImageCandidates({
+    imageUrls: params.imageUrls,
+    visionImages: params.visionImages,
+  });
   if (imageCandidates.length === 0) return null;
 
   const prompt = [
@@ -933,6 +965,7 @@ export async function summarizeImageForExecutionReport(params: {
       text,
       context,
       imageUrls,
+      visionImages,
     });
     if (forcedSummary) return forcedSummary;
   }
@@ -1027,6 +1060,7 @@ export async function summarizeImageForExecutionReport(params: {
     text,
     context,
     imageUrls,
+    visionImages,
   });
   if (qwenFallbackSummary) return qwenFallbackSummary;
 
@@ -1073,12 +1107,40 @@ type VisionSummaryModel = Awaited<
   ReturnType<typeof import("@/lib/db/queries")["getEnabledAiModelsWithProvider"]>
 >[number];
 
-function isVisionSummaryCandidateModel(modelWithProvider: VisionSummaryModel): boolean {
-  const providerType = String(modelWithProvider.provider?.type || "");
-  if (["fal-video", "volcano-video", "qianwen-video", "doubao-video", "ruxa-video"].includes(providerType)) {
-    return false;
+function isAnalyzerGenerateTextProviderType(providerType: string): boolean {
+  return ["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType);
+}
+
+function getAnalyzerModelIncompatibleReason(params: {
+  modelWithProvider: VisionSummaryModel;
+  requireVision: boolean;
+}): string | null {
+  const providerType = String(params.modelWithProvider.provider?.type || "");
+  const model = params.modelWithProvider.model;
+  if (!model?.isEnabled || model.isSubModel) {
+    return "disabled_or_submodel";
   }
-  return isLikelyVisionCapableModel(modelWithProvider);
+  if (!isAnalyzerGenerateTextProviderType(providerType)) {
+    return `unsupported_provider:${providerType || "unknown"}`;
+  }
+  if (params.requireVision && !isLikelyVisionCapableModel(params.modelWithProvider)) {
+    return "vision_capability_missing";
+  }
+  return null;
+}
+
+function isAnalyzerGenerateTextCompatibleModel(params: {
+  modelWithProvider: VisionSummaryModel;
+  requireVision: boolean;
+}): boolean {
+  return !getAnalyzerModelIncompatibleReason(params);
+}
+
+function isVisionSummaryCandidateModel(modelWithProvider: VisionSummaryModel): boolean {
+  return isAnalyzerGenerateTextCompatibleModel({
+    modelWithProvider,
+    requireVision: true,
+  });
 }
 
 function collectVisionSummaryCandidates(params: {
@@ -1282,27 +1344,32 @@ async function analyzWithLLM(
         )
       : undefined;
   const agentModel = preferredModel || configuredAgentModel;
-  const isReasoningCompatibleProvider = (providerType: string) =>
-    ["zhipu", "alibaba", "openai", "custom", "local", "volcano-multimodal", "volcano-vision"].includes(providerType);
+  const analyzerTextCandidates = allModels.filter((item) =>
+    isAnalyzerGenerateTextCompatibleModel({
+      modelWithProvider: item,
+      requireVision: false,
+    }),
+  );
+  const analyzerVisionCandidates = analyzerTextCandidates.filter((item) =>
+    isLikelyVisionCapableModel(item),
+  );
   const isLikelyZhipuVisionModel = (displayName: string, modelId: string, providerType: string) =>
     providerType === "zhipu" ||
     /(glm[-_.]?4v|glm[-_.]?4\.?1v|glm[-_.]?4\.?5v|vision)/i.test(
       `${displayName} ${modelId}`,
     );
-  const zhipuVisionModel = allModels.find(
+  const zhipuVisionModel = analyzerVisionCandidates.find(
     (m) =>
-      isLikelyVisionCapableModel(m) &&
       isLikelyZhipuVisionModel(m.model.displayName, m.model.modelId, m.provider.type),
   );
-  const qwenVisionModel = allModels.find(
+  const qwenVisionModel = analyzerVisionCandidates.find(
     (m) =>
       m.provider.type === "alibaba" &&
-      isLikelyVisionCapableModel(m) &&
       /(qwen|通义|vl)/i.test(`${m.model.displayName} ${m.model.modelId}`)
   );
   const fallbackVisionModel =
     qwenVisionModel ||
-    allModels.find(
+    analyzerVisionCandidates.find(
       (m) =>
         m.provider.type === "alibaba" &&
         isLikelyVisionCapableModel(m),
@@ -1310,21 +1377,57 @@ async function analyzWithLLM(
   const preferredVisionModel =
     zhipuVisionModel ||
     fallbackVisionModel ||
-    allModels.find((m) => isLikelyVisionCapableModel(m));
+    analyzerVisionCandidates[0];
   const usableAgentModel =
-    agentModel && isReasoningCompatibleProvider(agentModel.provider.type)
+    agentModel &&
+    isAnalyzerGenerateTextCompatibleModel({
+      modelWithProvider: agentModel,
+      requireVision: false,
+    })
       ? agentModel
       : undefined;
   const multimodalAgentVisionModel =
     agentModel &&
-    isLikelyVisionCapableModel(agentModel)
+    isAnalyzerGenerateTextCompatibleModel({
+      modelWithProvider: agentModel,
+      requireVision: true,
+    })
       ? agentModel
       : undefined;
+
+  if (preferredModel) {
+    const preferredIncompatibleReason = getAnalyzerModelIncompatibleReason({
+      modelWithProvider: preferredModel,
+      requireVision: hasImage,
+    });
+    if (preferredIncompatibleReason) {
+      console.warn(
+        `[Smart Analyzer] Preferred analyzer model incompatible, skip: ${preferredModel.model.displayName} (${preferredModel.provider.type}), reason=${preferredIncompatibleReason}`,
+      );
+    }
+  }
+
+  if (agentModel && !usableAgentModel) {
+    const agentIncompatibleReason = getAnalyzerModelIncompatibleReason({
+      modelWithProvider: agentModel,
+      requireVision: false,
+    });
+    if (agentIncompatibleReason) {
+      console.warn(
+        `[Smart Analyzer] Agent analyzer model incompatible, skip: ${agentModel.model.displayName} (${agentModel.provider.type}), reason=${agentIncompatibleReason}`,
+      );
+    }
+  }
+
   const selectedAnalyzerModel = hasImage
-    ? multimodalAgentVisionModel || preferredVisionModel || usableAgentModel
+    ? multimodalAgentVisionModel ||
+      preferredVisionModel ||
+      analyzerVisionCandidates[0] ||
+      usableAgentModel ||
+      analyzerTextCandidates[0]
     : usableAgentModel ||
-      allModels.find((m) => m.provider.type === "zhipu" && !m.model.isSubModel) ||
-      preferredVisionModel;
+      analyzerTextCandidates.find((m) => m.provider.type === "zhipu" && !m.model.isSubModel) ||
+      analyzerTextCandidates[0];
 
   if (preferredModel && selectedAnalyzerModel?.model.id === preferredModel.model.id) {
     console.log(
@@ -1332,12 +1435,33 @@ async function analyzWithLLM(
     );
   }
 
+  console.log(
+    `[Smart Analyzer] Analyzer candidate stats: text=${analyzerTextCandidates.length}, vision=${analyzerVisionCandidates.length}, selected=${selectedAnalyzerModel?.model.displayName || "none"} (${selectedAnalyzerModel?.provider?.type || "none"})`,
+  );
+
+  const preferRemoteVisionUrls = hasImage
+    ? shouldPreferRemoteVisionUrls(selectedAnalyzerModel)
+    : false;
+  const visionImages = hasImage
+    ? await buildAnalyzerVisionImagesByMode({
+        imageUrls,
+        preferRemoteUrls: preferRemoteVisionUrls,
+      })
+    : [];
+  if (visionImages.length > 0) {
+    const inlineCount = visionImages.filter((item) => item.image.startsWith("data:image/")).length;
+    console.log(
+      `[Smart Analyzer] Prepared analyzer images: total=${visionImages.length}, inline=${inlineCount}, remote=${visionImages.length - inlineCount}, preferRemote=${preferRemoteVisionUrls}`,
+    );
+  }
+
   if (!selectedAnalyzerModel) {
     console.log("[Smart Analyzer] No available analyzer model found");
     return analyzeWithHardcodedQwen3VlFallback({
       text,
       hasImage,
       imageUrls,
+      visionImages,
       historyMessages,
       context,
       reason: "no_available_model",
@@ -1352,6 +1476,7 @@ async function analyzWithLLM(
       text,
       hasImage,
       imageUrls,
+      visionImages,
       historyMessages,
       context,
       reason: "force_qwen3vl_multimodal_flag_only",
@@ -1365,6 +1490,7 @@ async function analyzWithLLM(
       text,
       hasImage,
       imageUrls,
+      visionImages,
       historyMessages,
       context,
       reason: "model_runtime_unavailable",
@@ -1398,20 +1524,6 @@ ${historyText}
     ? `${contextMessage}用户上传了图片，请求内容："${text}"`
     : `${contextMessage}用户请求（无图片）："${text}"`;
 
-  const preferRemoteVisionUrls = shouldPreferRemoteVisionUrls(selectedAnalyzerModel);
-  const visionImages = hasImage
-    ? await buildAnalyzerVisionImagesByMode({
-        imageUrls,
-        preferRemoteUrls: preferRemoteVisionUrls,
-      })
-    : [];
-  if (visionImages.length > 0) {
-    const inlineCount = visionImages.filter((item) => item.image.startsWith("data:image/")).length;
-    console.log(
-      `[Smart Analyzer] Prepared analyzer images: total=${visionImages.length}, inline=${inlineCount}, remote=${visionImages.length - inlineCount}, preferRemote=${preferRemoteVisionUrls}`,
-    );
-  }
-
   const zhipuProviderFromAgent =
     usableAgentModel &&
     usableAgentModel.provider &&
@@ -1493,6 +1605,7 @@ ${historyText}
         text,
         hasImage,
         imageUrls,
+        visionImages,
         historyMessages,
         context,
         reason: "primary_non_json_response",
@@ -1510,6 +1623,7 @@ ${historyText}
     text,
     hasImage,
     imageUrls,
+    visionImages,
     historyMessages,
     context,
     reason: "primary_analysis_failed",