/** * 智能意图分析器 * 分析用户输入,识别图片编辑意图 * * 主要应用场景:图片编辑(抠图、换背景、增强等) */ import { generateText } from "ai"; import sharp from "sharp"; import { isAiModelMultimodal } from "@/lib/db/ai-model-flags"; // 使用动态导入避免循环依赖问题 // import { getLanguageModelById } from "./dynamic-provider"; // import { getEnabledAiModelsWithProvider } from "@/lib/db/queries"; // 分析结果类型 export interface AnalysisResult { intent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat"; subIntent?: string; confidence: number; optimizedPrompt?: string; negativePrompt?: string; suggestedModel?: string; needsClarification?: boolean; clarifyingQuestions?: string[]; conversationalReply?: string; // 🆕 自然对话回复 } function inferDefaultImageEditSubIntent(text: string): AnalysisResult["subIntent"] { const lowerText = text.toLowerCase(); if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background"; if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background"; if (/风格|卡通|动漫|油画|像素风|写实|插画/.test(lowerText)) return "style_transfer"; if (/颜色|调色|亮度|对比度|饱和度|改成.*色|换成.*色/.test(lowerText)) return "color_adjust"; if (/去除|消除|擦掉|删除|移除|p掉|重绘/.test(lowerText)) return "inpaint"; return "enhance"; } function normalizeCompactText(text: string): string { return text.toLowerCase().replace(/\s+/g, "").trim(); } function isLikelyAdvisoryOrPlanningRequest(text: string): boolean { const normalizedText = text.toLowerCase(); return /怎么|如何|为什么|是否|可不可以|能不能|行不行|建议|思路|方案|流程|步骤|教程|推荐|对比|区别|差别|优缺点|评估|分析|判断|解释|讲讲|科普|适合吗|可行吗|值不值得|要不要|帮我看看|帮我分析|帮我判断/.test( normalizedText, ); } export function isExplicitPromptOptimizationRequest(text: string): boolean { const normalizedText = normalizeCompactText(text); if (!normalizedText) return false; const asksPrompt = /提示词|prompt|文生图提示|出图提示|描述词|咒语|关键词/.test(normalizedText); if (!asksPrompt) return false; const alsoRequestsExecution = /然后生图|再生图|并生成|再生成|顺便生成|直接生图|开始生图|出图|执行生成|生成出来|继续生成|做视频|生成视频/.test( normalizedText, ); if (alsoRequestsExecution) return false; const asksAnalysis = /分析|看图|看看|识别|拆解|提炼|整理|归纳|总结|描述|写出|给我|输出|优化|改写/.test( normalizedText, ); const asksPromptQuestion = /提示词应该是什么|什么提示词|提示词是什么|怎么写提示词|如何写提示词|怎么描述|如何描述|怎么表达|如何表达|给我一版提示词|给我一套提示词|适合.*提示词|用于生图的提示词|生成这样的图片.*提示词/.test( normalizedText, ); const blocksExecution = /不要生图|先别生图|不需要生图|不要生成|别生成|仅给|只给|先给我|先出提示词|先分析/.test( normalizedText, ); return asksAnalysis || asksPromptQuestion || blocksExecution; } function isExplicitVideoGenerationRequest(text: string): boolean { const normalizedText = normalizeCompactText(text); const hasStrongVideoMarkers = /图生视频|文生视频|首帧|尾帧|分镜|运镜|视频生成/.test(normalizedText); if (hasStrongVideoMarkers) return true; if (isLikelyAdvisoryOrPlanningRequest(text)) return false; return ( /让.+动起来/.test(normalizedText) || (/(生成|制作|做|创建|输出|渲染|合成|出|做成|变成|转成)/.test(normalizedText) && /(视频|动画|短片|片子|动起来|会动)/.test(normalizedText)) ); } function isExplicitImageGenerationRequest(text: string): boolean { const normalizedText = normalizeCompactText(text); if (isLikelyAdvisoryOrPlanningRequest(text)) return false; return /重新生成|再来一张|生成新图|新建图片|创作一张|生成一张|画一张|给我画|帮我画|出一张|做一张图|做一张海报|创建一张图片|参考.*生成|按这个风格生成|以这张图为灵感生成|仿照.*生成|生成一只|生成一个|生成一幅|生成一版|生成一套|生成一组|帮我生成.*(?:图|图片|海报|头像|插画|场景|小猫|猫咪|人物|产品图)|给我生成.*(?:图|图片|海报|头像|插画|场景|小猫|猫咪|人物|产品图)/.test( normalizedText, ); } function isLikelyImageEditRequest(text: string): boolean { const normalizedText = text.toLowerCase(); return /抠图|去背景|透明背景|去背|扣图|移除背景|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test( normalizedText, ); } function isLikelyVisionRequest(text: string): boolean { const normalizedText = text.toLowerCase(); return /这是什么|是什么|图里|图片里|画面里|照片里|有什么|有啥|看起来像|像什么|帮我看|看看|看下|识别|分析|描述|介绍|解读|讲讲|解释|评价|评估|适合|优缺点|问题|构图|什么风格|风格是什么|材质|元素|内容|文案|品牌感|商业化|电商|卖点|哪里不好|哪里有问题|能看出/.test( normalizedText, ); } function isLikelyGeneralChatTurn(text: string): boolean { const compactText = normalizeCompactText(text); if (!compactText) return false; return /^(你好|您好|hello|hi|嗨|哈喽|在吗|谢谢|感谢|收到|好的|ok|okay|算了|不用了|明白了|了解了)$/.test( compactText, ); } function applyDirectExecutionPolicy( result: AnalysisResult, params: { text: string; hasImage: boolean; imageUrls?: string[]; imageFromHistory?: boolean; } ): AnalysisResult { const hasUsableImage = params.hasImage || Boolean(params.imageFromHistory) || (params.imageUrls?.length || 0) > 0; const compactText = normalizeCompactText(params.text); const hasAnyRequestText = compactText.length > 0; const explicitVideoGenerationRequest = hasAnyRequestText && isExplicitVideoGenerationRequest(params.text); const explicitImageGenerationRequest = hasAnyRequestText && isExplicitImageGenerationRequest(params.text); const likelyAdvisoryRequest = hasAnyRequestText && isLikelyAdvisoryOrPlanningRequest(params.text); const likelyVisionRequest = hasAnyRequestText && isLikelyVisionRequest(params.text); const likelyGeneralChatTurn = hasAnyRequestText && isLikelyGeneralChatTurn(params.text); const promptOptimizationOnlyRequest = hasAnyRequestText && isExplicitPromptOptimizationRequest(params.text); if (explicitVideoGenerationRequest) { return { ...result, intent: "video_gen", subIntent: undefined, confidence: Math.max(result.confidence, 0.8), needsClarification: false, clarifyingQuestions: [], conversationalReply: result.conversationalReply || (params.hasImage ? "收到,我先按你这轮图片做视频需求分析,再自动优化提示词执行视频生成。" : "收到,我先做视频需求分析并自动优化提示词后执行生成。"), }; } if (explicitImageGenerationRequest) { return { ...result, intent: "image_gen", subIntent: undefined, confidence: Math.max(result.confidence, 0.84), needsClarification: false, clarifyingQuestions: [], conversationalReply: result.conversationalReply || (hasUsableImage ? "收到,我会结合你提供的参考图直接执行生图。" : "好的,我来直接帮你生成这张图。"), }; } if (!hasUsableImage) { if (promptOptimizationOnlyRequest) { return { ...result, intent: "chat", subIntent: undefined, confidence: Math.max(result.confidence, 0.82), needsClarification: false, clarifyingQuestions: [], conversationalReply: result.conversationalReply || "收到,我先按你的要求整理提示词,不直接执行生成。", }; } if (!explicitImageGenerationRequest && !explicitVideoGenerationRequest) { if (likelyAdvisoryRequest || likelyGeneralChatTurn || result.intent === "chat") { return { ...result, intent: "chat", subIntent: undefined, confidence: Math.max(result.confidence, 0.76), needsClarification: false, clarifyingQuestions: [], conversationalReply: result.conversationalReply || "收到,我先按你的问题理解需求,再直接回答你。", }; } } if (result.intent === "video_gen" && !explicitVideoGenerationRequest) { return { ...result, intent: "chat", subIntent: undefined, confidence: Math.max(result.confidence, 0.74), needsClarification: false, clarifyingQuestions: [], }; } if (result.intent === "image_gen" && !explicitImageGenerationRequest) { return { ...result, intent: "chat", subIntent: undefined, confidence: Math.max(result.confidence, 0.74), needsClarification: false, clarifyingQuestions: [], }; } return result; } const baseResult = normalizeIntentForImageContext(result, params.text, hasUsableImage); const keepClarification = Boolean(result.needsClarification); const explicitEditRequest = hasAnyRequestText && isLikelyImageEditRequest(params.text); if (promptOptimizationOnlyRequest) { return { ...baseResult, intent: "vision", subIntent: undefined, confidence: Math.max(baseResult.confidence, 0.88), needsClarification: false, clarifyingQuestions: [], conversationalReply: baseResult.conversationalReply || "收到,我先分析这张图并整理可直接使用的优化提示词,不会直接生图。", }; } if (!hasAnyRequestText) { return { ...baseResult, intent: "image_edit", subIntent: baseResult.subIntent || inferDefaultImageEditSubIntent(params.text), confidence: Math.max(baseResult.confidence, 0.72), needsClarification: keepClarification, clarifyingQuestions: keepClarification ? result.clarifyingQuestions || [] : [], conversationalReply: baseResult.conversationalReply || "收到图片,我先按保真优化策略直接执行生成。", }; } if (!explicitEditRequest && !explicitImageGenerationRequest && !explicitVideoGenerationRequest) { if (likelyVisionRequest || baseResult.intent === "vision") { return { ...baseResult, intent: "vision", subIntent: undefined, confidence: Math.max(baseResult.confidence, 0.78), needsClarification: false, clarifyingQuestions: [], conversationalReply: baseResult.conversationalReply || "收到,我先看看这张图,再直接告诉你我的判断。", }; } if (likelyAdvisoryRequest || likelyGeneralChatTurn || baseResult.intent === "chat") { return { ...baseResult, intent: "chat", subIntent: undefined, confidence: Math.max(baseResult.confidence, 0.72), needsClarification: keepClarification, clarifyingQuestions: keepClarification ? result.clarifyingQuestions || [] : [], conversationalReply: baseResult.conversationalReply || "收到,我们先围绕这张图继续聊。", }; } } const normalizedIntent = explicitEditRequest && baseResult.intent !== "image_gen" ? "image_edit" : baseResult.intent; const normalizedSubIntent = normalizedIntent === "image_edit" ? baseResult.subIntent || inferDefaultImageEditSubIntent(params.text) : baseResult.subIntent; return { ...baseResult, intent: normalizedIntent, subIntent: normalizedSubIntent, confidence: Math.max(baseResult.confidence, 0.72), needsClarification: keepClarification, clarifyingQuestions: keepClarification ? result.clarifyingQuestions || [] : [], conversationalReply: baseResult.conversationalReply || (hasAnyRequestText ? "收到,我会按你的描述直接执行生成。" : "收到图片,我先按保真优化策略直接执行生成。"), }; } function normalizeIntentForImageContext( result: AnalysisResult, text: string, hasImage: boolean ): AnalysisResult { if (!hasImage || result.intent !== "image_gen") { return result; } const normalizedText = text.toLowerCase(); const explicitGenerate = isExplicitImageGenerationRequest(normalizedText); const editSignal = /改|调整|优化|去掉|换成|改成|保留|继续|再.*一点|修|抠|替换|增强|清晰|重绘/.test(normalizedText); if (explicitGenerate && !editSignal) { return result; } return { ...result, intent: "image_edit", subIntent: result.subIntent, confidence: Math.max(result.confidence, 0.78), }; } // 系统提示词 - 针对图片编辑场景优化,生成自然对话 const ANALYZER_SYSTEM_PROMPT = `你是一个友好的AI多模态Agent。用户可能会让你聊天、看图分析、修图、生图或生视频,你需要: 1. 理解用户想要对图片做什么 2. 用自然、友好的语气回复用户,告诉他你理解了他的需求并会帮他处理 3. 同时输出结构化的意图分析结果 4. 只有用户明确要求执行图片/视频任务时,才输出 image_edit / image_gen / video_gen;如果用户是在问图、聊图、要建议或分析,就输出 vision 或 chat;只有“只发图没说话”时,才允许默认按保真优化执行 ## 意图分类 ### image_edit - 图片编辑(用户上传了图片并想编辑) 子意图: - remove_background: 抠图、去背景、透明背景 - replace_background: 换背景、改背景、白底 - enhance: 变清晰、高清、锐化、降噪 - extend: 扩展、拓展、延伸画布 - inpaint: 去除某物、消除、擦掉、P掉、局部修改 - style_transfer: 风格化、卡通化、动漫风、改风格 - color_adjust: 调色、调整颜色、亮度、对比度 - redo: 用户不满意之前的结果,想重新处理 ### vision - 图片理解(用户想让你看图说话) ### image_gen - 图片生成(用户想生成新图片) ### video_gen - 视频生成 ### chat - 普通对话 ## 特殊场景:连续对话(用户没有发送新图片,系统自动使用历史图片) 当用户继续对话但没有发送新图片时,系统会自动找到之前的图片。这时候: - 用户可能在继续描述想要的效果(如:"换成蓝色"、"再清晰一点"、"把背景去掉") - 用户可能在修改之前的结果(如:"不满意"、"再调整"、"换个风格") - 用户可能在追问(如:"还可以再亮一点吗"、"能不能换个颜色") 处理方式: - intent 通常是 image_edit - subIntent 根据用户的具体要求判断 - conversationalReply 要自然地回应用户,不需要特别强调"找到了之前的图片" ## 输出格式(严格JSON) { "intent": "image_edit", "subIntent": "replace_background", "confidence": 0.95, "conversationalReply": "好的,我来帮你把背景换成白色!这样产品会更突出~", "needsClarification": false, "clarifyingQuestions": [] } ## 回复风格要求 - conversationalReply 要自然、友好、简洁 - 可以加一些语气词让对话更亲切,如"好的"、"没问题"、"~"等 - 说明你理解了什么需求,让用户知道你在做什么 - 有图片时优先直接执行,不要提问 - 如果提供了画布上下文,仅用于理解,不要在 conversationalReply 中复述坐标或技术细节 - video_gen 场景要明确说明:会先分析视频需求/分镜,再自动优化提示词并执行生成 ## 示例 用户:"帮我把背景换成白色" { "intent": "image_edit", "subIntent": "replace_background", "confidence": 0.95, "conversationalReply": "好的,我来帮你把背景换成纯白色,这样图片会更干净专业~", "needsClarification": false } 用户:"把这张图变清晰一点" { "intent": "image_edit", "subIntent": "enhance", "confidence": 0.9, "conversationalReply": "收到!我来帮你提升图片清晰度,稍等一下~", "needsClarification": false } 用户发了图片但没说要做什么: { "intent": "image_edit", "subIntent": "enhance", "confidence": 0.72, "conversationalReply": "收到图片,我先做保真优化并输出结果,你可以再继续细化要求~", "needsClarification": false } 用户发了图片并问:"图片里是什么?" { "intent": "vision", "confidence": 0.92, "conversationalReply": "我先看一下这张图,然后直接告诉你内容和判断~", "needsClarification": false } 用户发了图片并问:"这个 logo 适合做品牌吗?" { "intent": "chat", "confidence": 0.86, "conversationalReply": "可以,我先结合这张图聊聊它的品牌感和适用场景~", "needsClarification": false } 连续对话场景 - 用户说"换成蓝色"(系统自动使用历史图片): { "intent": "image_edit", "subIntent": "color_adjust", "confidence": 0.9, "conversationalReply": "好的,我来帮你调整成蓝色~", "needsClarification": false } 连续对话场景 - 用户说"不满意,再调整一下": { "intent": "image_edit", "subIntent": "redo", "confidence": 0.8, "conversationalReply": "没问题,我先按上一轮目标继续优化并给你一个更稳的版本~", "needsClarification": false } 连续对话场景 - 用户说"再清晰一点": { "intent": "image_edit", "subIntent": "enhance", "confidence": 0.9, "conversationalReply": "好的,我再帮你提升一下清晰度~", "needsClarification": false } 用户:"把这张图做成5秒短视频,镜头慢慢推进" { "intent": "video_gen", "confidence": 0.9, "conversationalReply": "收到,我先按你的目标做视频需求分析和分镜规划,再自动优化提示词执行生成~", "needsClarification": false }`; function getEnvInt(name: string, fallback: number, min: number, max: number): number { const raw = process.env[name]; const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN; if (!Number.isFinite(parsed)) return fallback; return Math.max(min, Math.min(max, parsed)); } const ANALYZER_MAX_IMAGE_COUNT = 4; const ANALYZER_IMAGE_MAX_SIDE = 500; const ANALYZER_TIMEOUT_MS = getEnvInt( "AGENT_ANALYZER_TIMEOUT_MS", 12000, 8000, 120000, ); const ANALYZER_IMAGE_FETCH_TIMEOUT_MS = getEnvInt( "AGENT_IMAGE_FETCH_TIMEOUT_MS", 15000, 3000, 60000, ); const ANALYZER_QWEN_FALLBACK_TIMEOUT_MS = getEnvInt( "AGENT_QWEN_FALLBACK_TIMEOUT_MS", 12000, 6000, 90000, ); const ANALYZER_QWEN_FALLBACK_RETRIES = getEnvInt( "AGENT_QWEN_FALLBACK_RETRIES", 0, 0, 1, ); const ANALYZER_QWEN_FALLBACK_MAX_CANDIDATES = getEnvInt( "AGENT_QWEN_FALLBACK_MAX_CANDIDATES", 2, 1, 4, ); const ZHIPU_VISION_MODEL_CANDIDATES = [ "glm-4.6v-flash", "glm-4.6v", "glm-4v-plus-0111", "glm-4v-plus", "glm-4v", ]; function isHttpUrl(value: string) { return /^https?:\/\//i.test(value); } function isDataImageUrl(value: string) { return /^data:image\//i.test(value); } async function fetchWithTimeout(url: string, timeoutMs: number): Promise { const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), timeoutMs); try { return await fetch(url, { signal: controller.signal, cache: "no-store", headers: { "user-agent": "ai-chatbot-smart-analyzer/1.0", }, }); } finally { clearTimeout(timeoutId); } } async function toAnalyzerDataUrl(imageUrl: string): Promise { try { const response = await fetchWithTimeout(imageUrl, ANALYZER_IMAGE_FETCH_TIMEOUT_MS); if (!response.ok) { console.warn( `[Smart Analyzer] Failed to fetch image for inline payload (${response.status}): ${imageUrl}`, ); return null; } const contentType = response.headers.get("content-type") || ""; if (!contentType.startsWith("image/")) { console.warn(`[Smart Analyzer] Skip non-image content-type: ${contentType}`); return null; } const arrayBuffer = await response.arrayBuffer(); const inputBuffer = Buffer.from(arrayBuffer); const optimized = await sharp(inputBuffer) .rotate() .resize({ width: ANALYZER_IMAGE_MAX_SIDE, height: ANALYZER_IMAGE_MAX_SIDE, fit: "inside", withoutEnlargement: true, }) .jpeg({ quality: 80, mozjpeg: true, }) .toBuffer(); return `data:image/jpeg;base64,${optimized.toString("base64")}`; } catch (error) { console.warn("[Smart Analyzer] Failed to convert image URL to inline base64:", error); return null; } } async function buildAnalyzerVisionImages(imageUrls?: string[]) { const inputs = (imageUrls || []) .map((url) => String(url || "").trim()) .filter((url) => isHttpUrl(url) || isDataImageUrl(url)) .slice(0, ANALYZER_MAX_IMAGE_COUNT); if (inputs.length === 0) return [] as Array<{ type: "image"; image: string }>; const convertedResults = await Promise.all( inputs.map((input) => (isDataImageUrl(input) ? Promise.resolve(input) : toAnalyzerDataUrl(input))), ); return inputs.map((originalInput, index) => { const inlineDataUrl = convertedResults[index]; return { type: "image" as const, image: inlineDataUrl || originalInput, }; }); } function shouldPreferRemoteVisionUrls(modelWithProvider?: { model?: { displayName?: string | null; modelId?: string | null; } | null; provider?: { type?: string | null; baseUrl?: string | null; } | null; }): boolean { if (!modelWithProvider) return false; const providerType = String(modelWithProvider.provider?.type || ""); const baseUrl = String(modelWithProvider.provider?.baseUrl || "").toLowerCase(); const modelText = `${modelWithProvider.model?.displayName || ""} ${modelWithProvider.model?.modelId || ""}`.toLowerCase(); if ( providerType === "volcano-vision" || providerType === "volcano-multimodal" || providerType === "doubao-video" || providerType === "volcano-video" || providerType === "ruxa-video" ) { return true; } if (baseUrl.includes("ark.cn-beijing.volces.com")) { return true; } if (/(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0)/i.test(modelText)) { return true; } return false; } async function buildAnalyzerVisionImagesByMode(params: { imageUrls?: string[]; preferRemoteUrls: boolean; }) { const urls = (params.imageUrls || []) .map((url) => String(url || "").trim()) .filter((url) => isHttpUrl(url) || isDataImageUrl(url)) .slice(0, ANALYZER_MAX_IMAGE_COUNT); if (urls.length === 0) return [] as Array<{ type: "image"; image: string }>; if (params.preferRemoteUrls) { return urls.map((url) => ({ type: "image" as const, image: url })); } return buildAnalyzerVisionImages(urls); } async function getQwen3VlFallbackImageCandidates(params: { imageUrls?: string[]; visionImages?: Array<{ type: "image"; image: string }>; }): Promise { const allInputs = [ ...(params.visionImages || []).map((item) => String(item.image || "").trim()), ...((params.imageUrls || []).map((url) => String(url || "").trim())), ].filter(Boolean); const uniqueInputs = Array.from(new Set(allInputs)); const inlineInputs = uniqueInputs.filter((value) => value.startsWith("data:image/")); const remoteInputs = uniqueInputs.filter((value) => isHttpUrl(value)); if (inlineInputs.length === 0 && remoteInputs.length === 0) return []; const candidates: string[] = [...inlineInputs]; for (const remoteUrl of remoteInputs.slice(0, ANALYZER_MAX_IMAGE_COUNT)) { const inlineDataUrl = await toAnalyzerDataUrl(remoteUrl); if (inlineDataUrl) { candidates.push(inlineDataUrl); } candidates.push(remoteUrl); } return Array.from(new Set(candidates)).slice(0, ANALYZER_QWEN_FALLBACK_MAX_CANDIDATES); } async function analyzeWithHardcodedQwen3VlFallback(params: { text: string; hasImage: boolean; imageUrls?: string[]; assetIds?: string[]; visionImages?: Array<{ type: "image"; image: string }>; historyMessages?: HistoryMessage[]; context?: string; reason: string; }): Promise { if (!params.hasImage) return null; const imageCandidates = await getQwen3VlFallbackImageCandidates({ imageUrls: params.imageUrls, visionImages: params.visionImages, }); if (imageCandidates.length === 0) { console.warn( `[Smart Analyzer] qwen3-vl fallback skipped: no valid image input (${params.reason})`, ); return null; } const historySnippet = (params.historyMessages || []) .slice(-6) .map((msg) => `${msg.role === "user" ? "用户" : "AI"}: ${msg.content.slice(0, 100)}`) .join("\n"); const prompt = [ "你是图片需求分析助手。请结合图片与用户意图,输出严格 JSON,不要输出其他内容。", "intent 只能是 image_edit / vision / image_gen / video_gen / chat。", "subIntent 可选:remove_background / replace_background / enhance / extend / inpaint / style_transfer / color_adjust / redo / resize。", "如果用户提到视频、首帧、尾帧、分镜,intent=video_gen。", "有图片且用户在修改图片时,优先 intent=image_edit。", "JSON 字段:intent, subIntent, confidence, conversationalReply, needsClarification, clarifyingQuestions。", "confidence 为 0-1 数字;needsClarification 默认 false;clarifyingQuestions 默认空数组。", params.context ? `画布上下文(仅理解,不要复述): ${params.context.slice(0, 180)}` : "", historySnippet ? `近期对话(仅理解,不要复述):\n${historySnippet}` : "", `当前用户请求: ${params.text}`, ] .filter(Boolean) .join("\n"); try { const { callQwen3Vl } = await import("./qwen3-vl"); console.warn( `[Smart Analyzer] Primary analysis failed, fallback to hardcoded qwen3-vl (${params.reason}, candidates=${imageCandidates.length})`, ); for (const imageInput of imageCandidates) { const inputMode = imageInput.startsWith("data:image/") ? "inline" : "remote"; try { const { text: fallbackText, model } = await callQwen3Vl({ imageUrl: imageInput, prompt, temperature: 0.1, maxTokens: 360, timeoutMs: ANALYZER_QWEN_FALLBACK_TIMEOUT_MS, retries: ANALYZER_QWEN_FALLBACK_RETRIES, }); const parsed = parseAnalysisResultFromModelText( fallbackText, params.text, params.hasImage, ); if (parsed) { console.log(`[Smart Analyzer] qwen3-vl fallback success: ${model}, input=${inputMode}`); return parsed; } } catch (attemptError) { console.warn(`[Smart Analyzer] qwen3-vl fallback attempt failed: input=${inputMode}`, attemptError); } } console.warn("[Smart Analyzer] qwen3-vl fallback returned unparsable content"); return null; } catch (error) { console.warn("[Smart Analyzer] qwen3-vl fallback failed:", error); return null; } } async function summarizeWithHardcodedQwen3VlFallback(params: { text?: string; context?: string; imageUrls?: string[]; visionImages?: Array<{ type: "image"; image: string }>; }): Promise { const imageCandidates = await getQwen3VlFallbackImageCandidates({ imageUrls: params.imageUrls, visionImages: params.visionImages, }); if (imageCandidates.length === 0) return null; const prompt = [ "你是图像观察助手。只描述图片可见内容,不要写执行过程,不要推测用户意图。", "输出严格一行中文:画面:主体与关键元素;场景:环境;风格:视觉风格;光线:明暗与色温。", params.text ? `用户目标(仅理解,不要复述): ${params.text.slice(0, 120)}` : "", params.context ? `画布上下文(仅理解,不要复述): ${params.context.slice(0, 120)}` : "", ] .filter(Boolean) .join("\n"); try { const { callQwen3Vl } = await import("./qwen3-vl"); console.warn("[Smart Analyzer] Visual summary fallback to hardcoded qwen3-vl"); for (const imageInput of imageCandidates) { const inputMode = imageInput.startsWith("data:image/") ? "inline" : "remote"; try { const { text, model } = await callQwen3Vl({ imageUrl: imageInput, prompt, temperature: 0.1, maxTokens: 140, timeoutMs: ANALYZER_QWEN_FALLBACK_TIMEOUT_MS, retries: ANALYZER_QWEN_FALLBACK_RETRIES, }); const normalized = normalizeVisualSummaryText(text); if (normalized && !isWeakVisualSummary(normalized)) { console.log(`[Smart Analyzer] qwen3-vl summary fallback success: ${model}, input=${inputMode}`); return normalized; } } catch (attemptError) { console.warn(`[Smart Analyzer] qwen3-vl summary fallback attempt failed: input=${inputMode}`, attemptError); } } return null; } catch (error) { console.warn("[Smart Analyzer] qwen3-vl summary fallback failed:", error); return null; } } function parseAnalysisResultFromModelText( responseText: string, userText: string, hasImage: boolean, ): AnalysisResult | null { const jsonMatch = responseText.match(/\{[\s\S]*\}/); if (!jsonMatch) { console.warn("[Smart Analyzer] Model response missing JSON:", responseText.slice(0, 240)); const fallback = fallbackKeywordAnalysis(userText, hasImage); const conversationalReply = responseText.trim(); if (!conversationalReply) { return null; } return normalizeIntentForImageContext( { ...fallback, confidence: Math.max(fallback.confidence, 0.82), needsClarification: false, conversationalReply, }, userText, hasImage, ); } try { const result = JSON.parse(jsonMatch[0]) as AnalysisResult; return normalizeIntentForImageContext(result, userText, hasImage); } catch (error) { console.error( "[Smart Analyzer] Failed to parse model response:", error, "raw=", jsonMatch[0].slice(0, 240), ); const fallback = fallbackKeywordAnalysis(userText, hasImage); const conversationalReply = responseText.trim(); if (!conversationalReply) { return null; } return normalizeIntentForImageContext( { ...fallback, confidence: Math.max(fallback.confidence, 0.8), needsClarification: false, conversationalReply, }, userText, hasImage, ); } } function isLikelyZhipuEndpoint(baseUrl?: string | null): boolean { if (!baseUrl) return false; return /open\.bigmodel\.cn/i.test(baseUrl); } function isLikelyVolcanoEndpoint(baseUrl?: string | null): boolean { if (!baseUrl) return false; return /ark\.cn-beijing\.volces\.com/i.test(baseUrl); } function normalizeVolcanoResponsesEndpoint(baseUrl: string): string { const cleaned = baseUrl.replace(/\/+$/, ""); if (!cleaned) return "https://ark.cn-beijing.volces.com/api/v3/responses"; if (/\/responses$/i.test(cleaned)) return cleaned; if (/\/api\/v3$/i.test(cleaned)) return `${cleaned}/responses`; if (/ark\.cn-beijing\.volces\.com/i.test(cleaned)) { return `${cleaned}/api/v3/responses`.replace(/\/api\/v3\/api\/v3\//, "/api/v3/"); } return `${cleaned}/responses`; } async function rewriteUploadPublicUrlForVolcanoSummary(url: string): Promise { if (!/^https?:\/\//i.test(url)) return url; try { const { getDefaultStorageConfig } = await import("@/lib/db/queries"); const storageConfig = await getDefaultStorageConfig(); if ( !storageConfig || storageConfig.type !== "s3" || !storageConfig.endpoint || !storageConfig.bucket || !storageConfig.publicUrl ) { return url; } const parsed = new URL(url); const publicOrigin = new URL(storageConfig.publicUrl).origin; if (parsed.origin !== publicOrigin) return url; if (!parsed.pathname.startsWith("/uploads/")) return url; const endpointBase = storageConfig.endpoint.replace(/\/+$/, ""); return `${endpointBase}/${storageConfig.bucket}${parsed.pathname}${parsed.search}`; } catch { return url; } } async function summarizeWithVolcanoOfficialApi(params: { apiKey: string; baseUrl: string; promptText: string; imageUrls: string[]; modelId?: string; }): Promise { const endpoint = normalizeVolcanoResponsesEndpoint(params.baseUrl); const rewrittenUrls = await Promise.all( params.imageUrls.map((url) => rewriteUploadPublicUrlForVolcanoSummary(url)), ); try { const response = await fetch(endpoint, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${params.apiKey}`, }, body: JSON.stringify({ model: String(params.modelId || "").trim() || "doubao-seed-2-0-pro-260215", input: [ { role: "user", content: [ ...rewrittenUrls.map((url) => ({ type: "input_image", image_url: url })), { type: "input_text", text: params.promptText }, ], }, ], }), }); const payload = await response.json().catch(() => ({})); if (!response.ok || payload?.error) { return null; } const directText = String(payload?.output_text || "").trim(); if (directText) { const normalized = normalizeVisualSummaryText(directText); if (normalized && !isWeakVisualSummary(normalized)) return normalized; } const output = Array.isArray(payload?.output) ? payload.output : []; const chunks: string[] = []; for (const item of output) { const content = Array.isArray(item?.content) ? item.content : []; for (const part of content) { if (typeof part?.text === "string" && part.text.trim()) { chunks.push(part.text.trim()); } } } const merged = normalizeVisualSummaryText(chunks.join(" ").trim()); if (merged && !isWeakVisualSummary(merged)) return merged; } catch { return null; } return null; } async function analyzeWithVolcanoOfficialApi(params: { apiKey: string; baseUrl: string; userMessage: string; imageUrls: string[]; userText: string; hasImage: boolean; modelId?: string; }): Promise { const endpoint = normalizeVolcanoResponsesEndpoint(params.baseUrl); const rewrittenUrls = ( await Promise.all( params.imageUrls.map((url) => rewriteUploadPublicUrlForVolcanoSummary(url)), ) ) .filter((url) => isHttpUrl(url)) .slice(0, ANALYZER_MAX_IMAGE_COUNT); if (rewrittenUrls.length === 0) return null; try { const response = await fetch(endpoint, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${params.apiKey}`, }, body: JSON.stringify({ model: String(params.modelId || "").trim() || "doubao-seed-2-0-pro-260215", input: [ { role: "system", content: [ { type: "input_text", text: ANALYZER_SYSTEM_PROMPT, }, ], }, { role: "user", content: [ { type: "input_text", text: params.userMessage, }, ...rewrittenUrls.map((url) => ({ type: "input_image", image_url: url })), ], }, ], }), }); const payload = await response.json().catch(() => ({})); if (!response.ok || payload?.error) { return null; } const directText = String(payload?.output_text || "").trim(); const output = Array.isArray(payload?.output) ? payload.output : []; const chunks: string[] = []; for (const item of output) { const content = Array.isArray(item?.content) ? item.content : []; for (const part of content) { if (typeof part?.text === "string" && part.text.trim()) { chunks.push(part.text.trim()); } else if (typeof part?.output_text === "string" && part.output_text.trim()) { chunks.push(part.output_text.trim()); } } } const responseText = directText || chunks.join("\n").trim(); if (!responseText) return null; return parseAnalysisResultFromModelText( responseText, params.userText, params.hasImage, ); } catch { return null; } } function normalizeZhipuChatEndpoint(baseUrl: string): string { const cleaned = baseUrl.replace(/\/+$/, ""); if (/\/chat\/completions$/i.test(cleaned)) return cleaned; if (/\/api\/paas\/v4$/i.test(cleaned)) return `${cleaned}/chat/completions`; if (/\/api\/paas\/v4\//i.test(cleaned)) return cleaned; if (/open\.bigmodel\.cn/i.test(cleaned)) return `${cleaned}/api/paas/v4/chat/completions`; return `${cleaned}/chat/completions`; } function toZhipuImageUrl(image: string): string { const dataUrlPrefix = /^data:image\/[a-zA-Z0-9.+-]+;base64,/i; if (dataUrlPrefix.test(image)) { return image.replace(dataUrlPrefix, ""); } return image; } async function analyzeWithZhipuOfficialApi(params: { apiKey: string; baseUrl: string; userMessage: string; visionImages: Array<{ type: "image"; image: string }>; userText: string; hasImage: boolean; }): Promise { const endpoint = normalizeZhipuChatEndpoint(params.baseUrl); const content = [ { type: "text", text: params.userMessage }, ...params.visionImages.map((imagePart) => ({ type: "image_url", image_url: { url: toZhipuImageUrl(imagePart.image), }, })), ]; for (const modelId of ZHIPU_VISION_MODEL_CANDIDATES) { try { const response = await fetch(endpoint, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${params.apiKey}`, }, body: JSON.stringify({ model: modelId, messages: [ { role: "system", content: ANALYZER_SYSTEM_PROMPT, }, { role: "user", content, }, ], temperature: 0.1, max_tokens: 360, }), }); const payload = await response.json().catch(() => ({})); if (!response.ok || payload?.error) { const errorCode = String(payload?.error?.code || ""); const errorMessage = String(payload?.error?.message || ""); if (errorCode === "1211" || /模型不存在/.test(errorMessage)) { continue; } console.warn( `[Smart Analyzer] Zhipu official vision request failed (model=${modelId})`, payload?.error || payload, ); continue; } const message = payload?.choices?.[0]?.message; const responseText = String(message?.content || message?.reasoning_content || "").trim(); if (!responseText) { continue; } const parsed = parseAnalysisResultFromModelText( responseText, params.userText, params.hasImage, ); if (parsed) { console.log(`[Smart Analyzer] Zhipu official vision model used: ${modelId}`); return parsed; } } catch (error) { console.warn( `[Smart Analyzer] Zhipu official vision request exception (model=${modelId})`, error, ); } } return null; } async function summarizeWithZhipuOfficialApi(params: { apiKey: string; baseUrl: string; promptText: string; visionImages: Array<{ type: "image"; image: string }>; }): Promise { const endpoint = normalizeZhipuChatEndpoint(params.baseUrl); const content = [ { type: "text", text: params.promptText }, ...params.visionImages.map((imagePart) => ({ type: "image_url", image_url: { url: toZhipuImageUrl(imagePart.image), }, })), ]; for (const modelId of ZHIPU_VISION_MODEL_CANDIDATES) { try { const response = await fetch(endpoint, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${params.apiKey}`, }, body: JSON.stringify({ model: modelId, messages: [ { role: "system", content: "你是图像观察助手。只描述图片可见内容,不要提执行动作或用户意图。输出严格一行中文。", }, { role: "user", content, }, ], temperature: 0.2, max_tokens: 140, }), }); const payload = await response.json().catch(() => ({})); if (!response.ok || payload?.error) { const errorCode = String(payload?.error?.code || ""); const errorMessage = String(payload?.error?.message || ""); if (errorCode === "1211" || /模型不存在/.test(errorMessage)) { continue; } continue; } const message = payload?.choices?.[0]?.message; const responseText = String(message?.content || message?.reasoning_content || "").trim(); const normalized = normalizeVisualSummaryText(responseText); if (normalized && !isWeakVisualSummary(normalized)) { console.log(`[Smart Analyzer] Zhipu official summary model used: ${modelId}`); return normalized; } } catch { continue; } } return null; } function uniqueNonEmptyStrings(values?: string[] | null): string[] { return Array.from( new Set( (values || []) .map((value) => String(value || "").trim()) .filter(Boolean), ), ); } function extractVisualSummarySection(summary: string, label: string): string { const normalized = summary.replace(/\s+/g, " ").trim(); if (!normalized) return ""; const match = normalized.match(new RegExp(`${label}\\s*[::]\\s*([^;;]+)`, "i")); return match?.[1]?.trim() || ""; } function tokenizeVisualSummarySection(section: string, maxTags = 6): string[] { const normalized = section .replace(/[()()]/g, " ") .replace(/\s+/g, " ") .trim(); if (!normalized) return []; const parts = normalized .split(/[、,,/|]/) .map((item) => item.trim()) .filter(Boolean); if (parts.length === 0) { return normalized ? [normalized.slice(0, 80)] : []; } return Array.from(new Set(parts)).slice(0, maxTags); } function buildVisualSummaryMemoryPayload(summary: string) { return { subjectTags: tokenizeVisualSummarySection(extractVisualSummarySection(summary, "画面")), sceneTags: tokenizeVisualSummarySection(extractVisualSummarySection(summary, "场景")), styleTags: tokenizeVisualSummarySection(extractVisualSummarySection(summary, "风格")), lightingTags: tokenizeVisualSummarySection(extractVisualSummarySection(summary, "光线")), }; } async function getCachedVisualSummaryByAssetIds(assetIds?: string[]): Promise { const uniqueAssetIds = uniqueNonEmptyStrings(assetIds); if (uniqueAssetIds.length !== 1) return null; try { const { getAssetMemorySnapshotsByAssetIds } = await import("@/lib/db/queries"); const snapshots = await getAssetMemorySnapshotsByAssetIds({ assetIds: uniqueAssetIds, snapshotType: "visual_summary", status: "ready", }); const latest = snapshots.find( (snapshot) => typeof snapshot.summaryText === "string" && snapshot.summaryText.trim().length > 0, ); const normalized = normalizeVisualSummaryText(latest?.summaryText); if (normalized && !isWeakVisualSummary(normalized)) { console.log("[Smart Analyzer] Asset memory cache hit:", uniqueAssetIds[0]); return normalized; } } catch (error) { console.warn("[Smart Analyzer] Failed to read asset memory snapshot cache:", error); } return null; } async function persistVisualSummarySnapshot(params: { assetIds?: string[]; summary: string; preferredModelId?: string; text?: string; context?: string; imageUrls?: string[]; }) { const uniqueAssetIds = uniqueNonEmptyStrings(params.assetIds); if (uniqueAssetIds.length !== 1) return; try { const { getMediaAssetsByIds, upsertAssetMemorySnapshot } = await import("@/lib/db/queries"); const assets = await getMediaAssetsByIds({ ids: uniqueAssetIds }); const asset = assets.find((item) => item.id === uniqueAssetIds[0]); if (!asset?.userId || !asset.normalizedUrl) return; const normalizedSummary = normalizeVisualSummaryText(params.summary); if (!normalizedSummary || isWeakVisualSummary(normalizedSummary)) return; const memoryPayload = buildVisualSummaryMemoryPayload(normalizedSummary); await upsertAssetMemorySnapshot({ userId: asset.userId, assetId: asset.id, chatId: asset.chatId ?? null, projectId: asset.projectId ?? null, snapshotType: "visual_summary", status: "ready", normalizedUrl: asset.normalizedUrl, summaryText: normalizedSummary, subjectTags: memoryPayload.subjectTags, sceneTags: memoryPayload.sceneTags, styleTags: memoryPayload.styleTags, lightingTags: memoryPayload.lightingTags, confidence: 88, sourceModelId: params.preferredModelId || asset.modelId || null, metadata: { generator: "summarizeImageForExecutionReport", promptText: params.text || null, context: params.context || null, imageUrls: uniqueNonEmptyStrings(params.imageUrls), }, }); } catch (error) { console.warn("[Smart Analyzer] Failed to persist asset memory snapshot:", error); } } function stripVisualSummaryImageLabel(summary: string): string { return String(summary || "") .replace(/^\s*图\s*\d+\s*[::]\s*/i, "") .trim(); } export async function summarizeImageForExecutionReport(params: { text?: string; imageUrls?: string[]; context?: string; preferredModelId?: string; assetIds?: string[]; }): Promise { const { text, imageUrls, context, preferredModelId, assetIds } = params; const normalizedInputUrls = uniqueNonEmptyStrings(imageUrls).slice(0, ANALYZER_MAX_IMAGE_COUNT); if (normalizedInputUrls.length > 1) { const mergedLines: string[] = []; for (const [index, imageUrl] of normalizedInputUrls.entries()) { const singleSummary = await summarizeImageForExecutionReport({ text, imageUrls: [imageUrl], context, preferredModelId, assetIds: assetIds?.[index] ? [assetIds[index] as string] : [], }); if (singleSummary?.trim()) { mergedLines.push(`图${index + 1}:${stripVisualSummaryImageLabel(singleSummary)}`); } } return mergedLines.length > 0 ? mergedLines.join("\n") : null; } const uniqueAssetIds = uniqueNonEmptyStrings(assetIds); const cachedSummary = await getCachedVisualSummaryByAssetIds(uniqueAssetIds); if (cachedSummary) return cachedSummary; const visionImages = await buildAnalyzerVisionImages(normalizedInputUrls); if (visionImages.length === 0) return null; const finalizeVisualSummary = async (summary?: string | null): Promise => { const normalized = normalizeVisualSummaryText(summary); if (!normalized || isWeakVisualSummary(normalized)) return null; await persistVisualSummarySnapshot({ assetIds: uniqueAssetIds, summary: normalized, preferredModelId, text, context, imageUrls, }); return normalized; }; const { getEnabledAiModelsWithProvider, getAgentAiModelWithProvider, } = await import("@/lib/db/queries"); const allModels = await getEnabledAiModelsWithProvider(); const configuredAgentModel = await getAgentAiModelWithProvider(); const normalizedPreferredId = String(preferredModelId || "").trim(); const preferredModel = normalizedPreferredId.length > 0 ? allModels.find( (item) => item.model.id === normalizedPreferredId || item.model.modelId === normalizedPreferredId, ) : undefined; const agentModel = preferredModel || configuredAgentModel; const zhipuProviderFromAgent = agentModel?.provider?.apiKey && isLikelyZhipuEndpoint(agentModel.provider.baseUrl) ? agentModel.provider : null; const zhipuProviderFallback = allModels.find( (m) => m.provider?.apiKey && (m.provider.type === "zhipu" || isLikelyZhipuEndpoint(m.provider.baseUrl)), )?.provider; const zhipuProvider = zhipuProviderFromAgent || zhipuProviderFallback; const volcanoProviderFromAgent = agentModel?.provider?.apiKey && isLikelyVolcanoEndpoint(agentModel.provider.baseUrl) ? agentModel.provider : null; const volcanoProviderFallback = allModels.find( (m) => { const providerType = String(m.provider?.type || ""); return ( m.provider?.apiKey && ( providerType === "volcano-vision" || providerType === "volcano-multimodal" || isLikelyVolcanoEndpoint(m.provider?.baseUrl) ) ); }, )?.provider; const volcanoProvider = volcanoProviderFromAgent || volcanoProviderFallback; const forceQwenSummary = shouldForceQwen3VlForImageAnalysis(agentModel); const promptText = [ "只根据图片中可见内容做客观观察,不要推测用户意图。", "禁止出现“选中/待处理/按需求/处理/修改/转换”等执行词。", "如果只有1张图,输出一行:画面:主体与关键元素;场景:环境;风格:视觉风格;光线:明暗与色温。", "如果有多张图,逐张输出“图1:...”“图2:...”格式,每张各一行。", text ? `用户目标(仅辅助理解,不要复述):${text.slice(0, 120)}` : "", context ? `画布上下文(仅辅助理解,不要复述):${context.slice(0, 120)}` : "", ].join("\n"); if (forceQwenSummary) { console.warn( "[Smart Analyzer] Force qwen3-vl visual summary due multimodal-flag-only agent model", ); const forcedSummary = await summarizeWithHardcodedQwen3VlFallback({ text, context, imageUrls, visionImages, }); if (forcedSummary) return await finalizeVisualSummary(forcedSummary); } if (zhipuProvider?.apiKey && zhipuProvider?.baseUrl) { const summary = await summarizeWithZhipuOfficialApi({ apiKey: zhipuProvider.apiKey, baseUrl: zhipuProvider.baseUrl, promptText, visionImages, }); if (summary) return await finalizeVisualSummary(summary); } if (volcanoProvider?.apiKey && volcanoProvider?.baseUrl) { const summary = await summarizeWithVolcanoOfficialApi({ apiKey: volcanoProvider.apiKey, baseUrl: volcanoProvider.baseUrl, promptText, imageUrls: normalizedInputUrls.filter((url) => isHttpUrl(url)).slice(0, ANALYZER_MAX_IMAGE_COUNT), modelId: agentModel?.model?.modelId, }); if (summary) return await finalizeVisualSummary(summary); } try { const candidateModels = collectVisionSummaryCandidates({ agentModel, allModels, }); if (candidateModels.length > 0) { const { getLanguageModelById } = await import("./dynamic-provider"); for (const candidate of candidateModels) { const model = await getLanguageModelById(candidate.model.id); if (!model) continue; const preferRemote = shouldPreferRemoteVisionUrls(candidate); const modeAttempts = Array.from(new Set([preferRemote, !preferRemote])); for (const preferRemoteUrls of modeAttempts) { const fallbackVisionImages = await buildAnalyzerVisionImagesByMode({ imageUrls, preferRemoteUrls, }); if (fallbackVisionImages.length === 0) continue; try { const { text: summaryText } = await generateText({ model, messages: [ { role: "user", content: [ { type: "text", text: `${promptText}\n单图时仅输出一行,不超过60字;多图时每张一行,总计不超过6行。`, }, ...fallbackVisionImages, ], }, ], temperature: 0.1, maxTokens: 120, abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS), } as any); const normalized = normalizeVisualSummaryText(summaryText); if (normalized && !isWeakVisualSummary(normalized)) { console.log( `[Smart Analyzer] Visual summary model used: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`, ); return await finalizeVisualSummary(normalized); } if (normalized) { console.warn( `[Smart Analyzer] Weak visual summary ignored: ${normalized.slice(0, 120)}`, ); } } catch (attemptError) { console.warn( `[Smart Analyzer] Visual summary attempt failed: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`, attemptError, ); } } } } } catch (error) { console.warn("[Smart Analyzer] summarizeImageForExecutionReport fallback failed:", error); } const qwenFallbackSummary = await summarizeWithHardcodedQwen3VlFallback({ text, context, imageUrls, visionImages, }); if (qwenFallbackSummary) return await finalizeVisualSummary(qwenFallbackSummary); return null; } function normalizeVisualSummaryText(text?: string | null): string { if (!text) return ""; return text .split(/\r?\n/) .map((line) => line .replace(/\s+/g, " ") .replace(/^[-*•]\s*/, "") .replace(/^(画面要点|画面描述|图片分析|分析结果)[::]\s*/i, "") .trim(), ) .filter(Boolean) .join("\n") .trim(); } function isWeakVisualSummary(summary: string): boolean { const normalized = summary.replace(/\s+/g, " ").trim(); if (!normalized || normalized.length < 14) return true; const weakPatterns = [ /未获取到待分析/, /无可见内容可描述/, /无法获取图片/, /无法访问图片/, /看不到图片/, /图片不可用/, /待处理图/, /选中.*位置/, /当前上传图片/, /基于.*原图处理/, /按.*需求执行/, /仅做.*调整/, /不改动原有内容结构/, /转换为.*场景/, ]; if (weakPatterns.some((pattern) => pattern.test(normalized))) { return true; } return false; } type VisionSummaryModel = Awaited< ReturnType >[number]; function isAnalyzerGenerateTextProviderType(providerType: string): boolean { return ["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType); } function getAnalyzerModelIncompatibleReason(params: { modelWithProvider: VisionSummaryModel; requireVision: boolean; }): string | null { const providerType = String(params.modelWithProvider.provider?.type || ""); const model = params.modelWithProvider.model; if (!model?.isEnabled || model.isSubModel) { return "disabled_or_submodel"; } if (!isAnalyzerGenerateTextProviderType(providerType)) { return `unsupported_provider:${providerType || "unknown"}`; } if (params.requireVision && !isLikelyVisionCapableModel(params.modelWithProvider)) { return "vision_capability_missing"; } return null; } function isAnalyzerGenerateTextCompatibleModel(params: { modelWithProvider: VisionSummaryModel; requireVision: boolean; }): boolean { return !getAnalyzerModelIncompatibleReason(params); } function isVisionSummaryCandidateModel(modelWithProvider: VisionSummaryModel): boolean { return isAnalyzerGenerateTextCompatibleModel({ modelWithProvider, requireVision: true, }); } function collectVisionSummaryCandidates(params: { agentModel?: VisionSummaryModel | null; allModels: VisionSummaryModel[]; }): VisionSummaryModel[] { const result: VisionSummaryModel[] = []; const seen = new Set(); const push = (item?: VisionSummaryModel | null) => { if (!item) return; if (!isVisionSummaryCandidateModel(item)) return; const id = String(item.model?.id || ""); if (!id || seen.has(id)) return; seen.add(id); result.push(item); }; push(params.agentModel || undefined); for (const model of params.allModels) push(model); return result; } // 历史消息类型 interface HistoryMessage { role: "user" | "assistant"; content: string; } function isLikelyVisionCapableModel(modelWithProvider: { model: { isEnabled: boolean; isSubModel: boolean; supportsVision?: boolean; supportsImageGen?: boolean; displayName?: string | null; modelId?: string | null; capabilities?: unknown; defaultParams?: unknown; }; provider: { type?: string | null; }; }) { const model = modelWithProvider.model; if (!model?.isEnabled || model.isSubModel) return false; return Boolean(model.supportsVision) || isAiModelMultimodal(model); } function getPreferredVolcanoMultimodalAnalyzerCandidate( ...candidates: Array ): VisionSummaryModel | null { for (const candidate of candidates) { if (!candidate?.provider?.apiKey || !candidate?.provider?.baseUrl) continue; if (!isLikelyVolcanoEndpoint(candidate.provider.baseUrl)) continue; if (!isAiModelMultimodal(candidate.model)) continue; return candidate; } return null; } function shouldForceQwen3VlForImageAnalysis(modelWithProvider?: { model: { supportsVision?: boolean; displayName?: string | null; modelId?: string | null; capabilities?: unknown; }; provider: { type?: string | null; }; } | null): boolean { if (process.env.AGENT_FORCE_QWEN3VL_FALLBACK !== "1") { return false; } if (!modelWithProvider) return false; if (modelWithProvider.model.supportsVision) return false; if (!isAiModelMultimodal(modelWithProvider.model)) return false; const providerType = String(modelWithProvider.provider?.type || ""); if (!["custom", "openai", "local"].includes(providerType)) return false; const modelText = `${modelWithProvider.model.displayName || ""} ${modelWithProvider.model.modelId || ""}`.toLowerCase(); return /(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0)/i.test(modelText); } /** * 分析用户意图 */ export async function analyzeIntent(params: { text: string; hasImage: boolean; imageUrls?: string[]; assetIds?: string[]; imageFromHistory?: boolean; // 🆕 图片是否来自历史记录 historyMessages?: HistoryMessage[]; // 🆕 历史对话消息(用于上下文理解) context?: string; // 🆕 画布上下文(仅用于理解) preferredModelId?: string; }): Promise { const { text, hasImage, imageUrls, assetIds, imageFromHistory, historyMessages, context, preferredModelId } = params; console.log("[Smart Analyzer] Analyzing intent:", { text, hasImage, imageCount: imageUrls?.length, imageFromHistory, historyCount: historyMessages?.length || 0 }); // 先尝试关键词快速匹配(高置信度场景直接返回,节省LLM调用) const quickResult = quickKeywordMatch(text, hasImage); if (quickResult.confidence >= 0.9) { console.log("[Smart Analyzer] Quick match result:", quickResult); return applyDirectExecutionPolicy(quickResult, { text, hasImage, imageUrls, imageFromHistory }); } // 尝试使用多模态模型进行深度分析(传入历史对话) try { const llmResult = await analyzWithLLM( text, hasImage, imageUrls, assetIds, historyMessages, context, preferredModelId, ); if (llmResult) { console.log("[Smart Analyzer] LLM analysis result:", llmResult); return applyDirectExecutionPolicy(llmResult, { text, hasImage, imageUrls, imageFromHistory }); } } catch (error) { console.error("[Smart Analyzer] LLM analysis failed:", error); } // 回退到完整关键词分析 const fallbackResult = fallbackKeywordAnalysis(text, hasImage); console.log("[Smart Analyzer] Fallback result:", fallbackResult); return applyDirectExecutionPolicy(fallbackResult, { text, hasImage, imageUrls, imageFromHistory }); } /** * 快速关键词匹配(高置信度场景) * 🆕 现在返回低置信度,强制走LLM分析以获取自然对话回复 */ function quickKeywordMatch(text: string, hasImage: boolean): AnalysisResult { if (isExplicitVideoGenerationRequest(text)) { return { intent: "video_gen", confidence: 0.95, conversationalReply: hasImage ? "收到,我会结合你这轮图片先做视频需求分析,再执行生成。" : "收到,我先做视频需求分析并执行生成。", }; } if (isExplicitImageGenerationRequest(text)) { return { intent: "image_gen", confidence: 0.95, conversationalReply: hasImage ? "收到,我会结合参考图直接帮你生成。" : "好的,我来直接帮你生成这张图。", }; } return { intent: "chat", confidence: 0.3, }; } /** * 使用 LLM 进行深度分析 */ async function analyzWithLLM( text: string, hasImage: boolean, imageUrls?: string[], assetIds?: string[], historyMessages?: HistoryMessage[], context?: string, preferredModelId?: string ): Promise { // 使用动态导入避免循环依赖问题 const { getEnabledAiModelsWithProvider, getAgentAiModelWithProvider, } = await import("@/lib/db/queries"); const { getLanguageModelById } = await import("./dynamic-provider"); // 优先规则: // 1) 若 Agent 模型被标记为「多模态」,且当前请求包含图片,则识图+分析统一走该 Agent 模型 // 2) 否则按既有视觉模型优先级选择 const allModels = await getEnabledAiModelsWithProvider(); const configuredAgentModel = await getAgentAiModelWithProvider(); const normalizedPreferredId = String(preferredModelId || "").trim(); const preferredModel = normalizedPreferredId.length > 0 ? allModels.find( (item) => item.model.id === normalizedPreferredId || item.model.modelId === normalizedPreferredId, ) : undefined; const agentModel = preferredModel || configuredAgentModel; const analyzerTextCandidates = allModels.filter((item) => isAnalyzerGenerateTextCompatibleModel({ modelWithProvider: item, requireVision: false, }), ); const analyzerVisionCandidates = analyzerTextCandidates.filter((item) => isLikelyVisionCapableModel(item), ); const isLikelyZhipuVisionModel = (displayName: string, modelId: string, providerType: string) => providerType === "zhipu" || /(glm[-_.]?4v|glm[-_.]?4\.?1v|glm[-_.]?4\.?5v|vision)/i.test( `${displayName} ${modelId}`, ); const zhipuVisionModel = analyzerVisionCandidates.find( (m) => isLikelyZhipuVisionModel(m.model.displayName, m.model.modelId, m.provider.type), ); const qwenVisionModel = analyzerVisionCandidates.find( (m) => m.provider.type === "alibaba" && /(qwen|通义|vl)/i.test(`${m.model.displayName} ${m.model.modelId}`) ); const fallbackVisionModel = qwenVisionModel || analyzerVisionCandidates.find( (m) => m.provider.type === "alibaba" && isLikelyVisionCapableModel(m), ); const preferredVisionModel = zhipuVisionModel || fallbackVisionModel || analyzerVisionCandidates[0]; const usableAgentModel = agentModel && isAnalyzerGenerateTextCompatibleModel({ modelWithProvider: agentModel, requireVision: false, }) ? agentModel : undefined; const multimodalAgentVisionModel = agentModel && isAnalyzerGenerateTextCompatibleModel({ modelWithProvider: agentModel, requireVision: true, }) ? agentModel : undefined; if (preferredModel) { const preferredIncompatibleReason = getAnalyzerModelIncompatibleReason({ modelWithProvider: preferredModel, requireVision: hasImage, }); if (preferredIncompatibleReason) { console.warn( `[Smart Analyzer] Preferred analyzer model incompatible, skip: ${preferredModel.model.displayName} (${preferredModel.provider.type}), reason=${preferredIncompatibleReason}`, ); } } if (agentModel && !usableAgentModel) { const agentIncompatibleReason = getAnalyzerModelIncompatibleReason({ modelWithProvider: agentModel, requireVision: false, }); if (agentIncompatibleReason) { console.warn( `[Smart Analyzer] Agent analyzer model incompatible, skip: ${agentModel.model.displayName} (${agentModel.provider.type}), reason=${agentIncompatibleReason}`, ); } } const selectedAnalyzerModel = hasImage ? multimodalAgentVisionModel || preferredVisionModel || analyzerVisionCandidates[0] : usableAgentModel || analyzerTextCandidates.find((m) => m.provider.type === "zhipu" && !m.model.isSubModel) || analyzerTextCandidates[0]; if (preferredModel && selectedAnalyzerModel?.model.id === preferredModel.model.id) { console.log( `[Smart Analyzer] Preferred analyzer model selected: ${preferredModel.model.displayName} (${preferredModel.provider.type})`, ); } console.log( `[Smart Analyzer] Analyzer candidate stats: text=${analyzerTextCandidates.length}, vision=${analyzerVisionCandidates.length}, selected=${selectedAnalyzerModel?.model.displayName || "none"} (${selectedAnalyzerModel?.provider?.type || "none"})`, ); const preferRemoteVisionUrls = hasImage ? shouldPreferRemoteVisionUrls(selectedAnalyzerModel) : false; const visionImages = hasImage ? await buildAnalyzerVisionImagesByMode({ imageUrls, preferRemoteUrls: preferRemoteVisionUrls, }) : []; if (visionImages.length > 0) { const inlineCount = visionImages.filter((item) => item.image.startsWith("data:image/")).length; console.log( `[Smart Analyzer] Prepared analyzer images: total=${visionImages.length}, inline=${inlineCount}, remote=${visionImages.length - inlineCount}, preferRemote=${preferRemoteVisionUrls}`, ); } if (!selectedAnalyzerModel) { console.log("[Smart Analyzer] No available analyzer model found"); return analyzeWithHardcodedQwen3VlFallback({ text, hasImage, imageUrls, visionImages, historyMessages, context, reason: "no_available_model", }); } if (hasImage && shouldForceQwen3VlForImageAnalysis(selectedAnalyzerModel)) { console.warn( "[Smart Analyzer] Force hardcoded qwen3-vl for image analysis due multimodal-flag-only primary model", ); return analyzeWithHardcodedQwen3VlFallback({ text, hasImage, imageUrls, visionImages, historyMessages, context, reason: "force_qwen3vl_multimodal_flag_only", }); } const model = await getLanguageModelById(selectedAnalyzerModel.model.id); if (!model) { console.log("[Smart Analyzer] Failed to get language model"); return analyzeWithHardcodedQwen3VlFallback({ text, hasImage, imageUrls, assetIds, visionImages, historyMessages, context, reason: "model_runtime_unavailable", }); } // 🆕 构建包含历史上下文的消息 let contextMessage = ""; const cachedVisualSummary = hasImage ? await getCachedVisualSummaryByAssetIds(assetIds) : null; // 🆕 画布上下文(仅用于理解,不要复述) if (context && context.trim().length > 0) { contextMessage = `## 画布上下文(仅用于理解,不要在对话回复中复述):\n${context.trim()}\n\n`; } if (cachedVisualSummary) { contextMessage += `## 已缓存图片摘要(可直接作为视觉上下文):\n${cachedVisualSummary}\n\n`; } // 添加历史对话上下文(最多取最近5轮对话) if (historyMessages && historyMessages.length > 0) { const recentHistory = historyMessages.slice(-10); // 最多10条消息(约5轮对话) const historyText = recentHistory.map(msg => `${msg.role === 'user' ? '用户' : 'AI'}: ${msg.content.substring(0, 200)}` ).join('\n'); contextMessage += `## 对话历史(最近的对话上下文): ${historyText} ## 当前请求: `; } // 构建用户消息 const userMessage = hasImage ? `${contextMessage}用户上传了图片,请求内容:"${text}"` : `${contextMessage}用户请求(无图片):"${text}"`; const volcanoAnalyzerCandidate = getPreferredVolcanoMultimodalAnalyzerCandidate( selectedAnalyzerModel, preferredModel, agentModel, ); if ( hasImage && visionImages.length > 0 && volcanoAnalyzerCandidate?.provider?.apiKey && volcanoAnalyzerCandidate?.provider?.baseUrl ) { const volcanoOfficialResult = await analyzeWithVolcanoOfficialApi({ apiKey: volcanoAnalyzerCandidate.provider.apiKey, baseUrl: volcanoAnalyzerCandidate.provider.baseUrl, modelId: volcanoAnalyzerCandidate.model.modelId, userMessage, userText: text, hasImage, imageUrls: (imageUrls || []) .filter((url) => isHttpUrl(url)) .slice(0, ANALYZER_MAX_IMAGE_COUNT), }); if (volcanoOfficialResult) { console.log( `[Smart Analyzer] Volcano official analyzer used: ${volcanoAnalyzerCandidate.model.displayName} (${volcanoAnalyzerCandidate.model.modelId})`, ); return volcanoOfficialResult; } } const zhipuProviderFromAgent = usableAgentModel && usableAgentModel.provider && usableAgentModel.provider.apiKey && isLikelyZhipuEndpoint(usableAgentModel.provider.baseUrl) ? usableAgentModel.provider : null; const zhipuProviderFallback = allModels.find( (m) => m.provider?.apiKey && (m.provider.type === "zhipu" || isLikelyZhipuEndpoint(m.provider.baseUrl)), )?.provider; const zhipuProvider = zhipuProviderFromAgent || zhipuProviderFallback; const forceSingleMultimodalAgent = Boolean(hasImage && multimodalAgentVisionModel); if ( !cachedVisualSummary && !forceSingleMultimodalAgent && hasImage && visionImages.length > 0 && zhipuProvider?.apiKey && zhipuProvider?.baseUrl ) { const zhipuOfficialResult = await analyzeWithZhipuOfficialApi({ apiKey: zhipuProvider.apiKey, baseUrl: zhipuProvider.baseUrl, userMessage, visionImages, userText: text, hasImage, }); if (zhipuOfficialResult) { return zhipuOfficialResult; } } console.log( "[Smart Analyzer] Calling analyzer model:", selectedAnalyzerModel.model.displayName, `(${selectedAnalyzerModel.provider.type})`, "history=", historyMessages?.length || 0 ); // 调用模型分析(有图时携带图片,确保真实视觉分析) const requestPayload = hasImage && visionImages.length > 0 && !cachedVisualSummary ? { model, system: ANALYZER_SYSTEM_PROMPT, messages: [ { role: "user" as const, content: [ { type: "text" as const, text: userMessage }, ...visionImages, ], }, ], temperature: 0.1, maxTokens: 360, abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS), } : { model, system: ANALYZER_SYSTEM_PROMPT, prompt: userMessage, temperature: 0.1, maxTokens: 300, abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS), }; try { const { text: responseText } = await generateText(requestPayload as any); const hasStructuredJson = /\{[\s\S]*\}/.test(String(responseText || "")); if (hasImage && !hasStructuredJson) { console.warn( "[Smart Analyzer] Primary analyzer returned non-JSON for image input, fallback to hardcoded qwen3-vl", ); const qwenFallback = await analyzeWithHardcodedQwen3VlFallback({ text, hasImage, imageUrls, assetIds, visionImages, historyMessages, context, reason: "primary_non_json_response", }); if (qwenFallback) return qwenFallback; } const parsed = parseAnalysisResultFromModelText(responseText, text, hasImage); if (parsed) return parsed; } catch (error) { console.warn("[Smart Analyzer] Primary analyzer model failed:", error); } return analyzeWithHardcodedQwen3VlFallback({ text, hasImage, imageUrls, assetIds, visionImages, historyMessages, context, reason: "primary_analysis_failed", }); } /** * 回退方案:完整关键词匹配(当LLM不可用时使用) * 🆕 添加了自然对话回复 */ export function fallbackKeywordAnalysis( text: string, hasImage: boolean ): AnalysisResult { const lowerText = text.toLowerCase(); // === 图片编辑场景(需要有图片) === if (hasImage) { if (isExplicitPromptOptimizationRequest(text)) { return { intent: "vision", confidence: 0.92, conversationalReply: "收到,我先分析这张图并给你整理优化后的提示词,不直接生图。", }; } // 视频生成(图生视频/首尾帧等) if (isExplicitVideoGenerationRequest(text)) { return { intent: "video_gen", confidence: 0.9, conversationalReply: "收到,我先按你提供的图片做视频需求分析和分镜规划,再自动优化提示词执行生成~", }; } // 抠图/去背景 if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) { return { intent: "image_edit", subIntent: "remove_background", confidence: 0.9, conversationalReply: "好的,我来帮你抠图,去除背景保留主体~", }; } // 换背景 if (/换背景|白底|改背景|背景换成|纯色背景|换成.*背景|替换.*背景/.test(lowerText)) { // 提取背景颜色 const bgMatch = text.match(/(?:换成|改成|替换为|变成)(.+?)(?:背景|底|$)/); const bg = bgMatch ? bgMatch[1].trim() : "白色"; return { intent: "image_edit", subIntent: "replace_background", confidence: 0.9, conversationalReply: `好的,我来帮你把背景换成${bg}~`, }; } // 图片增强 if (/变清晰|高清|锐化|降噪|提升质量|修复|清晰化|变清楚|画质/.test(lowerText)) { return { intent: "image_edit", subIntent: "enhance", confidence: 0.9, conversationalReply: "收到,我来帮你提升图片清晰度~", }; } // 图像拓展 if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大/.test(lowerText)) { return { intent: "image_edit", subIntent: "extend", confidence: 0.9, conversationalReply: "好的,我来帮你扩展图片边界~", }; } // 局部修改/去除 if (/去除|消除|擦掉|P掉|p掉|删除|修改.*部分|移除.*(?:人|物|水印|logo|文字)/.test(lowerText)) { return { intent: "image_edit", subIntent: "inpaint", confidence: 0.9, conversationalReply: "明白,我来帮你处理这个区域~", }; } // 风格迁移 if (/风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|转换为.*风格/.test(lowerText)) { const styleMatch = text.match(/(?:转换为|变成|改成|转成)(.+?)(?:风格|$)/); const style = styleMatch ? styleMatch[1].trim() : "新"; return { intent: "image_edit", subIntent: "style_transfer", confidence: 0.9, conversationalReply: `好的,我来帮你转换成${style}风格~`, }; } // 调色/改颜色 if (/调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成/.test(lowerText)) { // 提取目标颜色 const colorMatch = text.match(/(?:改成|换成|变成)(.+?)(?:色|$)/); const color = colorMatch ? colorMatch[1].trim() + "色" : ""; return { intent: "image_edit", subIntent: "color_adjust", confidence: 0.9, conversationalReply: color ? `好的,我来帮你把颜色改成${color}~` : "收到,我来帮你调整图片色彩~", }; } // 尺寸调整 if (/缩放|放大|缩小|调整尺寸|裁剪|resize|crop/.test(lowerText)) { return { intent: "image_edit", subIntent: "resize", confidence: 0.9, conversationalReply: "好的,我来帮你调整图片尺寸~", }; } // 通用编辑关键词 if (/修图|P图|改图|调图|处理.*图|编辑.*图|优化.*图/.test(lowerText)) { return { intent: "image_edit", subIntent: inferDefaultImageEditSubIntent(text), confidence: 0.8, needsClarification: false, conversationalReply: "收到,我先按你的需求直接执行图片优化。", }; } // 视觉理解 if (/看|识别|分析|描述|这是什么|有什么|tell me|what is|看看|帮我看/.test(lowerText)) { return { intent: "vision", confidence: 0.85, conversationalReply: "让我来看看这张图片~", }; } if (isLikelyGeneralChatTurn(text)) { return { intent: "chat", confidence: 0.78, conversationalReply: "收到,我们先继续聊这张图~", }; } if (text.trim()) { return { intent: "vision", confidence: 0.72, conversationalReply: "收到,我先看看这张图,再根据你的问题继续分析~", }; } // 有图但没文本 return { intent: "image_edit", subIntent: inferDefaultImageEditSubIntent(text), confidence: 0.75, needsClarification: false, conversationalReply: "收到图片,我先按保真优化策略直接执行。", }; } // === 无图片场景 === // 图片生成 if (isExplicitImageGenerationRequest(text)) { return { intent: "image_gen", confidence: 0.85, conversationalReply: "好的,我来帮你生成这张图片~", }; } // 视频生成 if (isExplicitVideoGenerationRequest(text)) { return { intent: "video_gen", confidence: 0.85, conversationalReply: "好的,我先做视频需求分析并自动优化提示词后执行生成~", }; } // 需要图片但没有上传 if (/抠图|去背景|换背景|修图|P图|编辑图/.test(lowerText)) { return { intent: "image_edit", confidence: 0.7, needsClarification: true, conversationalReply: "你想编辑图片的话,需要先上传一张图片哦~", }; } // 默认对话 return { intent: "chat", confidence: 0.7 }; }