diff --git a/lib/ai/smart-analyzer.ts b/lib/ai/smart-analyzer.ts index 8a76ec053..dca1d439c 100644 --- a/lib/ai/smart-analyzer.ts +++ b/lib/ai/smart-analyzer.ts @@ -624,6 +624,8 @@ export async function summarizeImageForExecutionReport(params: { "只根据图片中可见内容做客观观察,不要推测用户意图。", "禁止出现“选中/待处理/按需求/处理/修改/转换”等执行词。", "输出一行:画面:主体与关键元素;场景:环境;风格:视觉风格;光线:明暗与色温。", + text ? `用户目标(仅辅助理解,不要复述):${text.slice(0, 120)}` : "", + context ? `画布上下文(仅辅助理解,不要复述):${context.slice(0, 120)}` : "", ].join("\n"); if (zhipuProvider?.apiKey && zhipuProvider?.baseUrl) { @@ -637,47 +639,65 @@ export async function summarizeImageForExecutionReport(params: { } try { - const selectedVisionModel = - (agentModel && isLikelyVisionCapableModel(agentModel) ? agentModel : undefined) || - allModels.find((m) => isLikelyVisionCapableModel(m)); - if (!selectedVisionModel) return null; + const candidateModels = collectVisionSummaryCandidates({ + agentModel, + allModels, + }); + if (candidateModels.length === 0) return null; const { getLanguageModelById } = await import("./dynamic-provider"); - const model = await getLanguageModelById(selectedVisionModel.model.id); - if (!model) return null; + for (const candidate of candidateModels) { + const model = await getLanguageModelById(candidate.model.id); + if (!model) continue; - const preferRemote = shouldPreferRemoteVisionUrls(selectedVisionModel); - const modeAttempts = Array.from(new Set([preferRemote, !preferRemote])); + const preferRemote = shouldPreferRemoteVisionUrls(candidate); + const modeAttempts = Array.from(new Set([preferRemote, !preferRemote])); - for (const preferRemoteUrls of modeAttempts) { - const fallbackVisionImages = await buildAnalyzerVisionImagesByMode({ - imageUrls, - preferRemoteUrls, - }); - if (fallbackVisionImages.length === 0) continue; + for (const preferRemoteUrls of modeAttempts) { + const fallbackVisionImages = await buildAnalyzerVisionImagesByMode({ + imageUrls, + preferRemoteUrls, + }); + if (fallbackVisionImages.length === 0) continue; - const { text: summaryText } = await generateText({ - model, - messages: [ - { - role: "user", - content: [ + try { + const { text: summaryText } = await generateText({ + model, + messages: [ { - type: "text", - text: `${promptText}\n仅输出一行,不超过60字。`, + role: "user", + content: [ + { + type: "text", + text: `${promptText}\n仅输出一行,不超过60字。`, + }, + ...fallbackVisionImages, + ], }, - ...fallbackVisionImages, ], - }, - ], - temperature: 0.1, - maxTokens: 120, - abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS), - } as any); + temperature: 0.1, + maxTokens: 120, + abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS), + } as any); - const normalized = normalizeVisualSummaryText(summaryText); - if (normalized && !isWeakVisualSummary(normalized)) { - return normalized; + const normalized = normalizeVisualSummaryText(summaryText); + if (normalized && !isWeakVisualSummary(normalized)) { + console.log( + `[Smart Analyzer] Visual summary model used: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`, + ); + return normalized; + } + if (normalized) { + console.warn( + `[Smart Analyzer] Weak visual summary ignored: ${normalized.slice(0, 120)}`, + ); + } + } catch (attemptError) { + console.warn( + `[Smart Analyzer] Visual summary attempt failed: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`, + attemptError, + ); + } } } } catch (error) { @@ -701,6 +721,12 @@ function isWeakVisualSummary(summary: string): boolean { if (!normalized || normalized.length < 14) return true; const weakPatterns = [ + /未获取到待分析/, + /无可见内容可描述/, + /无法获取图片/, + /无法访问图片/, + /看不到图片/, + /图片不可用/, /待处理图/, /选中.*位置/, /当前上传图片/, @@ -717,6 +743,39 @@ function isWeakVisualSummary(summary: string): boolean { return false; } +type VisionSummaryModel = Awaited< + ReturnType +>[number]; + +function isVisionSummaryCandidateModel(modelWithProvider: VisionSummaryModel): boolean { + const providerType = String(modelWithProvider.provider?.type || ""); + if (["fal-video", "volcano-video", "qianwen-video", "doubao-video"].includes(providerType)) { + return false; + } + return isLikelyVisionCapableModel(modelWithProvider); +} + +function collectVisionSummaryCandidates(params: { + agentModel?: VisionSummaryModel | null; + allModels: VisionSummaryModel[]; +}): VisionSummaryModel[] { + const result: VisionSummaryModel[] = []; + const seen = new Set(); + + const push = (item?: VisionSummaryModel | null) => { + if (!item) return; + if (!isVisionSummaryCandidateModel(item)) return; + const id = String(item.model?.id || ""); + if (!id || seen.has(id)) return; + seen.add(id); + result.push(item); + }; + + push(params.agentModel || undefined); + for (const model of params.allModels) push(model); + return result; +} + // 历史消息类型 interface HistoryMessage { role: "user" | "assistant";