From 75e139c167ad51b4444056d461b74663cd6823d2 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 21 Feb 2026 04:19:25 +0800 Subject: [PATCH] fix(agent): fallback image analysis to hardcoded qwen3-vl --- lib/ai/smart-analyzer.ts | 274 +++++++++++++++++++++++++++++++-------- 1 file changed, 218 insertions(+), 56 deletions(-) diff --git a/lib/ai/smart-analyzer.ts b/lib/ai/smart-analyzer.ts index 4621112b8..e5efe0d25 100644 --- a/lib/ai/smart-analyzer.ts +++ b/lib/ai/smart-analyzer.ts @@ -340,7 +340,12 @@ function shouldPreferRemoteVisionUrls(modelWithProvider?: { const baseUrl = String(modelWithProvider.provider?.baseUrl || "").toLowerCase(); const modelText = `${modelWithProvider.model?.displayName || ""} ${modelWithProvider.model?.modelId || ""}`.toLowerCase(); - if (providerType === "volcano-multimodal" || providerType === "doubao-video" || providerType === "volcano-video") { + if ( + providerType === "volcano-vision" || + providerType === "volcano-multimodal" || + providerType === "doubao-video" || + providerType === "volcano-video" + ) { return true; } if (baseUrl.includes("ark.cn-beijing.volces.com")) { @@ -366,6 +371,121 @@ async function buildAnalyzerVisionImagesByMode(params: { return buildAnalyzerVisionImages(urls); } +async function getQwen3VlFallbackImageInput(imageUrls?: string[]): Promise { + const firstImageUrl = (imageUrls || []).find((url) => isHttpUrl(url)); + if (!firstImageUrl) return null; + const inlineDataUrl = await toAnalyzerDataUrl(firstImageUrl); + return inlineDataUrl || firstImageUrl; +} + +async function analyzeWithHardcodedQwen3VlFallback(params: { + text: string; + hasImage: boolean; + imageUrls?: string[]; + historyMessages?: HistoryMessage[]; + context?: string; + reason: string; +}): Promise { + if (!params.hasImage) return null; + + const imageInput = await getQwen3VlFallbackImageInput(params.imageUrls); + if (!imageInput) { + console.warn( + `[Smart Analyzer] qwen3-vl fallback skipped: no valid image url (${params.reason})`, + ); + return null; + } + + const historySnippet = (params.historyMessages || []) + .slice(-6) + .map((msg) => `${msg.role === "user" ? "用户" : "AI"}: ${msg.content.slice(0, 100)}`) + .join("\n"); + + const prompt = [ + "你是图片需求分析助手。请结合图片与用户意图，输出严格 JSON，不要输出其他内容。", + "intent 只能是 image_edit / vision / image_gen / video_gen / chat。", + "subIntent 可选：remove_background / replace_background / enhance / extend / inpaint / style_transfer / color_adjust / redo / resize。", + "如果用户提到视频、首帧、尾帧、分镜，intent=video_gen。", + "有图片且用户在修改图片时，优先 intent=image_edit。", + "JSON 字段：intent, subIntent, confidence, conversationalReply, needsClarification, clarifyingQuestions。", + "confidence 为 0-1 数字；needsClarification 默认 false；clarifyingQuestions 默认空数组。", + params.context ? `画布上下文（仅理解，不要复述）: ${params.context.slice(0, 180)}` : "", + historySnippet ? `近期对话（仅理解，不要复述）:\n${historySnippet}` : "", + `当前用户请求: ${params.text}`, + ] + .filter(Boolean) + .join("\n"); + + try { + const { callQwen3Vl } = await import("./qwen3-vl"); + console.warn( + `[Smart Analyzer] Primary analysis failed, fallback to hardcoded qwen3-vl (${params.reason})`, + ); + const { text: fallbackText, model } = await callQwen3Vl({ + imageUrl: imageInput, + prompt, + temperature: 0.1, + maxTokens: 360, + timeoutMs: ANALYZER_TIMEOUT_MS, + retries: 1, + }); + const parsed = parseAnalysisResultFromModelText( + fallbackText, + params.text, + params.hasImage, + ); + if (parsed) { + console.log(`[Smart Analyzer] qwen3-vl fallback success: ${model}`); + return parsed; + } + console.warn("[Smart Analyzer] qwen3-vl fallback returned unparsable content"); + return null; + } catch (error) { + console.warn("[Smart Analyzer] qwen3-vl fallback failed:", error); + return null; + } +} + +async function summarizeWithHardcodedQwen3VlFallback(params: { + text?: string; + context?: string; + imageUrls?: string[]; +}): Promise { + const imageInput = await getQwen3VlFallbackImageInput(params.imageUrls); + if (!imageInput) return null; + + const prompt = [ + "你是图像观察助手。只描述图片可见内容，不要写执行过程，不要推测用户意图。", + "输出严格一行中文：画面：主体与关键元素；场景：环境；风格：视觉风格；光线：明暗与色温。", + params.text ? `用户目标（仅理解，不要复述）: ${params.text.slice(0, 120)}` : "", + params.context ? `画布上下文（仅理解，不要复述）: ${params.context.slice(0, 120)}` : "", + ] + .filter(Boolean) + .join("\n"); + + try { + const { callQwen3Vl } = await import("./qwen3-vl"); + console.warn("[Smart Analyzer] Visual summary fallback to hardcoded qwen3-vl"); + const { text, model } = await callQwen3Vl({ + imageUrl: imageInput, + prompt, + temperature: 0.1, + maxTokens: 140, + timeoutMs: ANALYZER_TIMEOUT_MS, + retries: 1, + }); + const normalized = normalizeVisualSummaryText(text); + if (normalized && !isWeakVisualSummary(normalized)) { + console.log(`[Smart Analyzer] qwen3-vl summary fallback success: ${model}`); + return normalized; + } + return null; + } catch (error) { + console.warn("[Smart Analyzer] qwen3-vl summary fallback failed:", error); + return null; + } +} + function parseAnalysisResultFromModelText( responseText: string, userText: string, @@ -728,9 +848,17 @@ export async function summarizeImageForExecutionReport(params: { ? agentModel.provider : null; const volcanoProviderFallback = allModels.find( - (m) => - m.provider?.apiKey && - (m.provider.type === "volcano-multimodal" || isLikelyVolcanoEndpoint(m.provider.baseUrl)), + (m) => { + const providerType = String(m.provider?.type || ""); + return ( + m.provider?.apiKey && + ( + providerType === "volcano-vision" || + providerType === "volcano-multimodal" || + isLikelyVolcanoEndpoint(m.provider?.baseUrl) + ) + ); + }, )?.provider; const volcanoProvider = volcanoProviderFromAgent || volcanoProviderFallback; @@ -767,60 +895,60 @@ export async function summarizeImageForExecutionReport(params: { agentModel, allModels, }); - if (candidateModels.length === 0) return null; + if (candidateModels.length > 0) { + const { getLanguageModelById } = await import("./dynamic-provider"); + for (const candidate of candidateModels) { + const model = await getLanguageModelById(candidate.model.id); + if (!model) continue; - const { getLanguageModelById } = await import("./dynamic-provider"); - for (const candidate of candidateModels) { - const model = await getLanguageModelById(candidate.model.id); - if (!model) continue; + const preferRemote = shouldPreferRemoteVisionUrls(candidate); + const modeAttempts = Array.from(new Set([preferRemote, !preferRemote])); - const preferRemote = shouldPreferRemoteVisionUrls(candidate); - const modeAttempts = Array.from(new Set([preferRemote, !preferRemote])); + for (const preferRemoteUrls of modeAttempts) { + const fallbackVisionImages = await buildAnalyzerVisionImagesByMode({ + imageUrls, + preferRemoteUrls, + }); + if (fallbackVisionImages.length === 0) continue; - for (const preferRemoteUrls of modeAttempts) { - const fallbackVisionImages = await buildAnalyzerVisionImagesByMode({ - imageUrls, - preferRemoteUrls, - }); - if (fallbackVisionImages.length === 0) continue; + try { + const { text: summaryText } = await generateText({ + model, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: `${promptText}\n仅输出一行，不超过60字。`, + }, + ...fallbackVisionImages, + ], + }, + ], + temperature: 0.1, + maxTokens: 120, + abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS), + } as any); - try { - const { text: summaryText } = await generateText({ - model, - messages: [ - { - role: "user", - content: [ - { - type: "text", - text: `${promptText}\n仅输出一行，不超过60字。`, - }, - ...fallbackVisionImages, - ], - }, - ], - temperature: 0.1, - maxTokens: 120, - abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS), - } as any); - - const normalized = normalizeVisualSummaryText(summaryText); - if (normalized && !isWeakVisualSummary(normalized)) { - console.log( - `[Smart Analyzer] Visual summary model used: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`, - ); - return normalized; - } - if (normalized) { + const normalized = normalizeVisualSummaryText(summaryText); + if (normalized && !isWeakVisualSummary(normalized)) { + console.log( + `[Smart Analyzer] Visual summary model used: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`, + ); + return normalized; + } + if (normalized) { + console.warn( + `[Smart Analyzer] Weak visual summary ignored: ${normalized.slice(0, 120)}`, + ); + } + } catch (attemptError) { console.warn( - `[Smart Analyzer] Weak visual summary ignored: ${normalized.slice(0, 120)}`, + `[Smart Analyzer] Visual summary attempt failed: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`, + attemptError, ); } - } catch (attemptError) { - console.warn( - `[Smart Analyzer] Visual summary attempt failed: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`, - attemptError, - ); } } } @@ -828,6 +956,13 @@ export async function summarizeImageForExecutionReport(params: { console.warn("[Smart Analyzer] summarizeImageForExecutionReport fallback failed:", error); } + const qwenFallbackSummary = await summarizeWithHardcodedQwen3VlFallback({ + text, + context, + imageUrls, + }); + if (qwenFallbackSummary) return qwenFallbackSummary; + return null; } @@ -928,7 +1063,7 @@ function isLikelyVisionCapableModel(modelWithProvider: { const modelText = `${model.displayName || ""} ${model.modelId || ""}`.toLowerCase(); - if (providerType === "volcano-multimodal") return true; + if (providerType === "volcano-vision" || providerType === "volcano-multimodal") return true; if ( providerType === "zhipu" && @@ -1082,13 +1217,27 @@ async function analyzWithLLM( if (!selectedAnalyzerModel) { console.log("[Smart Analyzer] No available analyzer model found"); - return null; + return analyzeWithHardcodedQwen3VlFallback({ + text, + hasImage, + imageUrls, + historyMessages, + context, + reason: "no_available_model", + }); } const model = await getLanguageModelById(selectedAnalyzerModel.model.id); if (!model) { console.log("[Smart Analyzer] Failed to get language model"); - return null; + return analyzeWithHardcodedQwen3VlFallback({ + text, + hasImage, + imageUrls, + historyMessages, + context, + reason: "model_runtime_unavailable", + }); } // 🆕 构建包含历史上下文的消息 @@ -1202,9 +1351,22 @@ ${historyText} abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS), }; - const { text: responseText } = await generateText(requestPayload as any); + try { + const { text: responseText } = await generateText(requestPayload as any); + const parsed = parseAnalysisResultFromModelText(responseText, text, hasImage); + if (parsed) return parsed; + } catch (error) { + console.warn("[Smart Analyzer] Primary analyzer model failed:", error); + } - return parseAnalysisResultFromModelText(responseText, text, hasImage); + return analyzeWithHardcodedQwen3VlFallback({ + text, + hasImage, + imageUrls, + historyMessages, + context, + reason: "primary_analysis_failed", + }); } /**