1690 lines
55 KiB
TypeScript
1690 lines
55 KiB
TypeScript
/**
|
||
* 智能意图分析器
|
||
* 分析用户输入，识别图片编辑意图
|
||
*
|
||
* 主要应用场景：图片编辑（抠图、换背景、增强等）
|
||
*/
|
||
|
||
import { generateText } from "ai";
|
||
import sharp from "sharp";
|
||
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||
// 使用动态导入避免循环依赖问题
|
||
// import { getLanguageModelById } from "./dynamic-provider";
|
||
// import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||
|
||
// 分析结果类型
|
||
export interface AnalysisResult {
|
||
intent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat";
|
||
subIntent?: string;
|
||
confidence: number;
|
||
optimizedPrompt?: string;
|
||
negativePrompt?: string;
|
||
suggestedModel?: string;
|
||
needsClarification?: boolean;
|
||
clarifyingQuestions?: string[];
|
||
conversationalReply?: string; // 🆕 自然对话回复
|
||
}
|
||
|
||
function inferDefaultImageEditSubIntent(text: string): AnalysisResult["subIntent"] {
|
||
const lowerText = text.toLowerCase();
|
||
if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background";
|
||
if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background";
|
||
if (/风格|卡通|动漫|油画|像素风|写实|插画/.test(lowerText)) return "style_transfer";
|
||
if (/颜色|调色|亮度|对比度|饱和度|改成.*色|换成.*色/.test(lowerText)) return "color_adjust";
|
||
if (/去除|消除|擦掉|删除|移除|p掉|重绘/.test(lowerText)) return "inpaint";
|
||
return "enhance";
|
||
}
|
||
|
||
function isVideoGenerationRequest(text: string): boolean {
|
||
return /视频|动画|动起来|生成视频|做视频|视频生成|图生视频|文生视频|镜头|分镜|运镜|首帧|尾帧/.test(
|
||
text.toLowerCase(),
|
||
);
|
||
}
|
||
|
||
function applyDirectExecutionPolicy(
|
||
result: AnalysisResult,
|
||
params: {
|
||
text: string;
|
||
hasImage: boolean;
|
||
imageUrls?: string[];
|
||
imageFromHistory?: boolean;
|
||
}
|
||
): AnalysisResult {
|
||
if (isVideoGenerationRequest(params.text)) {
|
||
return {
|
||
...result,
|
||
intent: "video_gen",
|
||
subIntent: undefined,
|
||
confidence: Math.max(result.confidence, 0.8),
|
||
needsClarification: false,
|
||
clarifyingQuestions: [],
|
||
conversationalReply:
|
||
result.conversationalReply ||
|
||
(params.hasImage
|
||
? "收到，我先按你这轮图片做视频需求分析，再自动优化提示词执行视频生成。"
|
||
: "收到，我先做视频需求分析并自动优化提示词后执行生成。"),
|
||
};
|
||
}
|
||
|
||
const hasUsableImage =
|
||
params.hasImage || Boolean(params.imageFromHistory) || (params.imageUrls?.length || 0) > 0;
|
||
if (!hasUsableImage) return result;
|
||
|
||
const compactText = params.text.replace(/\s+/g, "").trim();
|
||
const hasAnyRequestText = compactText.length > 0;
|
||
const normalizedIntent = result.intent === "chat" || result.intent === "vision" ? "image_edit" : result.intent;
|
||
const normalizedSubIntent =
|
||
normalizedIntent === "image_edit"
|
||
? result.subIntent || inferDefaultImageEditSubIntent(params.text)
|
||
: result.subIntent;
|
||
|
||
return {
|
||
...result,
|
||
intent: normalizedIntent,
|
||
subIntent: normalizedSubIntent,
|
||
confidence: Math.max(result.confidence, 0.72),
|
||
needsClarification: false,
|
||
clarifyingQuestions: [],
|
||
conversationalReply:
|
||
result.conversationalReply ||
|
||
(hasAnyRequestText ? "收到，我会按你的描述直接执行生成。" : "收到图片，我先按保真优化策略直接执行生成。"),
|
||
};
|
||
}
|
||
|
||
function normalizeIntentForImageContext(
|
||
result: AnalysisResult,
|
||
text: string,
|
||
hasImage: boolean
|
||
): AnalysisResult {
|
||
if (!hasImage || result.intent !== "image_gen") {
|
||
return result;
|
||
}
|
||
|
||
const normalizedText = text.toLowerCase();
|
||
const explicitGenerate =
|
||
/重新生成|再来一张|生成新图|新建图片|创作一张|参考.*生成|按这个风格生成|以这张图为灵感生成/.test(
|
||
normalizedText
|
||
);
|
||
const editSignal =
|
||
/改|调整|优化|去掉|换成|改成|保留|继续|再.*一点|修|抠|替换|增强|清晰|重绘/.test(normalizedText);
|
||
|
||
if (explicitGenerate && !editSignal) {
|
||
return result;
|
||
}
|
||
|
||
return {
|
||
...result,
|
||
intent: "image_edit",
|
||
subIntent: result.subIntent,
|
||
confidence: Math.max(result.confidence, 0.78),
|
||
};
|
||
}
|
||
|
||
// 系统提示词 - 针对图片编辑场景优化，生成自然对话
|
||
const ANALYZER_SYSTEM_PROMPT = `你是一个友好的AI图片编辑助手。用户会给你发送图片和编辑请求，你需要：
|
||
1. 理解用户想要对图片做什么
|
||
2. 用自然、友好的语气回复用户，告诉他你理解了他的需求并会帮他处理
|
||
3. 同时输出结构化的意图分析结果
|
||
4. 默认直接执行，不要反问用户；只有在完全无图且需求为空时才允许澄清
|
||
|
||
## 意图分类
|
||
|
||
### image_edit - 图片编辑（用户上传了图片并想编辑）
|
||
子意图：
|
||
- remove_background: 抠图、去背景、透明背景
|
||
- replace_background: 换背景、改背景、白底
|
||
- enhance: 变清晰、高清、锐化、降噪
|
||
- extend: 扩展、拓展、延伸画布
|
||
- inpaint: 去除某物、消除、擦掉、P掉、局部修改
|
||
- style_transfer: 风格化、卡通化、动漫风、改风格
|
||
- color_adjust: 调色、调整颜色、亮度、对比度
|
||
- redo: 用户不满意之前的结果，想重新处理
|
||
|
||
### vision - 图片理解（用户想让你看图说话）
|
||
### image_gen - 图片生成（用户想生成新图片）
|
||
### video_gen - 视频生成
|
||
### chat - 普通对话
|
||
|
||
## 特殊场景：连续对话（用户没有发送新图片，系统自动使用历史图片）
|
||
|
||
当用户继续对话但没有发送新图片时，系统会自动找到之前的图片。这时候：
|
||
- 用户可能在继续描述想要的效果（如："换成蓝色"、"再清晰一点"、"把背景去掉"）
|
||
- 用户可能在修改之前的结果（如："不满意"、"再调整"、"换个风格"）
|
||
- 用户可能在追问（如："还可以再亮一点吗"、"能不能换个颜色"）
|
||
|
||
处理方式：
|
||
- intent 通常是 image_edit
|
||
- subIntent 根据用户的具体要求判断
|
||
- conversationalReply 要自然地回应用户，不需要特别强调"找到了之前的图片"
|
||
|
||
## 输出格式（严格JSON）
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "replace_background",
|
||
"confidence": 0.95,
|
||
"conversationalReply": "好的，我来帮你把背景换成白色！这样产品会更突出~",
|
||
"needsClarification": false,
|
||
"clarifyingQuestions": []
|
||
}
|
||
|
||
## 回复风格要求
|
||
- conversationalReply 要自然、友好、简洁
|
||
- 可以加一些语气词让对话更亲切，如"好的"、"没问题"、"~"等
|
||
- 说明你理解了什么需求，让用户知道你在做什么
|
||
- 有图片时优先直接执行，不要提问
|
||
- 如果提供了画布上下文，仅用于理解，不要在 conversationalReply 中复述坐标或技术细节
|
||
- video_gen 场景要明确说明：会先分析视频需求/分镜，再自动优化提示词并执行生成
|
||
|
||
## 示例
|
||
用户："帮我把背景换成白色"
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "replace_background",
|
||
"confidence": 0.95,
|
||
"conversationalReply": "好的，我来帮你把背景换成纯白色，这样图片会更干净专业~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
用户："把这张图变清晰一点"
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "enhance",
|
||
"confidence": 0.9,
|
||
"conversationalReply": "收到！我来帮你提升图片清晰度，稍等一下~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
用户发了图片但没说要做什么：
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "enhance",
|
||
"confidence": 0.72,
|
||
"conversationalReply": "收到图片，我先做保真优化并输出结果，你可以再继续细化要求~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
连续对话场景 - 用户说"换成蓝色"（系统自动使用历史图片）：
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "color_adjust",
|
||
"confidence": 0.9,
|
||
"conversationalReply": "好的，我来帮你调整成蓝色~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
连续对话场景 - 用户说"不满意，再调整一下"：
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "redo",
|
||
"confidence": 0.8,
|
||
"conversationalReply": "没问题，我先按上一轮目标继续优化并给你一个更稳的版本~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
连续对话场景 - 用户说"再清晰一点"：
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "enhance",
|
||
"confidence": 0.9,
|
||
"conversationalReply": "好的，我再帮你提升一下清晰度~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
用户："把这张图做成5秒短视频，镜头慢慢推进"
|
||
{
|
||
"intent": "video_gen",
|
||
"confidence": 0.9,
|
||
"conversationalReply": "收到，我先按你的目标做视频需求分析和分镜规划，再自动优化提示词执行生成~",
|
||
"needsClarification": false
|
||
}`;
|
||
|
||
function getEnvInt(name: string, fallback: number, min: number, max: number): number {
|
||
const raw = process.env[name];
|
||
const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
|
||
if (!Number.isFinite(parsed)) return fallback;
|
||
return Math.max(min, Math.min(max, parsed));
|
||
}
|
||
|
||
const ANALYZER_MAX_IMAGE_COUNT = 2;
|
||
const ANALYZER_IMAGE_MAX_SIDE = 500;
|
||
const ANALYZER_TIMEOUT_MS = 120000;
|
||
const ANALYZER_IMAGE_FETCH_TIMEOUT_MS = getEnvInt(
|
||
"AGENT_IMAGE_FETCH_TIMEOUT_MS",
|
||
15000,
|
||
3000,
|
||
60000,
|
||
);
|
||
const ANALYZER_QWEN_FALLBACK_TIMEOUT_MS = getEnvInt(
|
||
"AGENT_QWEN_FALLBACK_TIMEOUT_MS",
|
||
35000,
|
||
10000,
|
||
90000,
|
||
);
|
||
const ANALYZER_QWEN_FALLBACK_RETRIES = getEnvInt(
|
||
"AGENT_QWEN_FALLBACK_RETRIES",
|
||
0,
|
||
0,
|
||
1,
|
||
);
|
||
const ZHIPU_VISION_MODEL_CANDIDATES = [
|
||
"glm-4.6v-flash",
|
||
"glm-4.6v",
|
||
"glm-4v-plus-0111",
|
||
"glm-4v-plus",
|
||
"glm-4v",
|
||
];
|
||
|
||
function isHttpUrl(value: string) {
|
||
return /^https?:\/\//i.test(value);
|
||
}
|
||
|
||
async function fetchWithTimeout(url: string, timeoutMs: number): Promise<Response> {
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||
try {
|
||
return await fetch(url, {
|
||
signal: controller.signal,
|
||
cache: "no-store",
|
||
headers: {
|
||
"user-agent": "ai-chatbot-smart-analyzer/1.0",
|
||
},
|
||
});
|
||
} finally {
|
||
clearTimeout(timeoutId);
|
||
}
|
||
}
|
||
|
||
async function toAnalyzerDataUrl(imageUrl: string): Promise<string | null> {
|
||
try {
|
||
const response = await fetchWithTimeout(imageUrl, ANALYZER_IMAGE_FETCH_TIMEOUT_MS);
|
||
if (!response.ok) {
|
||
console.warn(
|
||
`[Smart Analyzer] Failed to fetch image for inline payload (${response.status}): ${imageUrl}`,
|
||
);
|
||
return null;
|
||
}
|
||
|
||
const contentType = response.headers.get("content-type") || "";
|
||
if (!contentType.startsWith("image/")) {
|
||
console.warn(`[Smart Analyzer] Skip non-image content-type: ${contentType}`);
|
||
return null;
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
const inputBuffer = Buffer.from(arrayBuffer);
|
||
|
||
const optimized = await sharp(inputBuffer)
|
||
.rotate()
|
||
.resize({
|
||
width: ANALYZER_IMAGE_MAX_SIDE,
|
||
height: ANALYZER_IMAGE_MAX_SIDE,
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
})
|
||
.jpeg({
|
||
quality: 80,
|
||
mozjpeg: true,
|
||
})
|
||
.toBuffer();
|
||
|
||
return `data:image/jpeg;base64,${optimized.toString("base64")}`;
|
||
} catch (error) {
|
||
console.warn("[Smart Analyzer] Failed to convert image URL to inline base64:", error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
async function buildAnalyzerVisionImages(imageUrls?: string[]) {
|
||
const urls = (imageUrls || []).filter((url) => isHttpUrl(url)).slice(0, ANALYZER_MAX_IMAGE_COUNT);
|
||
if (urls.length === 0) return [] as Array<{ type: "image"; image: string }>;
|
||
|
||
const convertedResults = await Promise.all(urls.map((url) => toAnalyzerDataUrl(url)));
|
||
|
||
return urls.map((originalUrl, index) => {
|
||
const inlineDataUrl = convertedResults[index];
|
||
return {
|
||
type: "image" as const,
|
||
image: inlineDataUrl || originalUrl,
|
||
};
|
||
});
|
||
}
|
||
|
||
function shouldPreferRemoteVisionUrls(modelWithProvider?: {
|
||
model?: {
|
||
displayName?: string | null;
|
||
modelId?: string | null;
|
||
} | null;
|
||
provider?: {
|
||
type?: string | null;
|
||
baseUrl?: string | null;
|
||
} | null;
|
||
}): boolean {
|
||
if (!modelWithProvider) return false;
|
||
const providerType = String(modelWithProvider.provider?.type || "");
|
||
const baseUrl = String(modelWithProvider.provider?.baseUrl || "").toLowerCase();
|
||
const modelText = `${modelWithProvider.model?.displayName || ""} ${modelWithProvider.model?.modelId || ""}`.toLowerCase();
|
||
|
||
if (
|
||
providerType === "volcano-vision" ||
|
||
providerType === "volcano-multimodal" ||
|
||
providerType === "doubao-video" ||
|
||
providerType === "volcano-video"
|
||
) {
|
||
return true;
|
||
}
|
||
if (baseUrl.includes("ark.cn-beijing.volces.com")) {
|
||
return true;
|
||
}
|
||
if (/(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0)/i.test(modelText)) {
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
async function buildAnalyzerVisionImagesByMode(params: {
|
||
imageUrls?: string[];
|
||
preferRemoteUrls: boolean;
|
||
}) {
|
||
const urls = (params.imageUrls || [])
|
||
.filter((url) => isHttpUrl(url))
|
||
.slice(0, ANALYZER_MAX_IMAGE_COUNT);
|
||
if (urls.length === 0) return [] as Array<{ type: "image"; image: string }>;
|
||
if (params.preferRemoteUrls) {
|
||
return urls.map((url) => ({ type: "image" as const, image: url }));
|
||
}
|
||
return buildAnalyzerVisionImages(urls);
|
||
}
|
||
|
||
async function getQwen3VlFallbackImageCandidates(imageUrls?: string[]): Promise<string[]> {
|
||
const firstImageUrl = (imageUrls || []).find((url) => isHttpUrl(url));
|
||
if (!firstImageUrl) return [];
|
||
|
||
const candidates: string[] = [];
|
||
const inlineDataUrl = await toAnalyzerDataUrl(firstImageUrl);
|
||
if (inlineDataUrl) {
|
||
candidates.push(inlineDataUrl);
|
||
}
|
||
candidates.push(firstImageUrl);
|
||
return Array.from(new Set(candidates));
|
||
}
|
||
|
||
async function analyzeWithHardcodedQwen3VlFallback(params: {
|
||
text: string;
|
||
hasImage: boolean;
|
||
imageUrls?: string[];
|
||
historyMessages?: HistoryMessage[];
|
||
context?: string;
|
||
reason: string;
|
||
}): Promise<AnalysisResult | null> {
|
||
if (!params.hasImage) return null;
|
||
|
||
const imageCandidates = await getQwen3VlFallbackImageCandidates(params.imageUrls);
|
||
if (imageCandidates.length === 0) {
|
||
console.warn(
|
||
`[Smart Analyzer] qwen3-vl fallback skipped: no valid image url (${params.reason})`,
|
||
);
|
||
return null;
|
||
}
|
||
|
||
const historySnippet = (params.historyMessages || [])
|
||
.slice(-6)
|
||
.map((msg) => `${msg.role === "user" ? "用户" : "AI"}: ${msg.content.slice(0, 100)}`)
|
||
.join("\n");
|
||
|
||
const prompt = [
|
||
"你是图片需求分析助手。请结合图片与用户意图，输出严格 JSON，不要输出其他内容。",
|
||
"intent 只能是 image_edit / vision / image_gen / video_gen / chat。",
|
||
"subIntent 可选：remove_background / replace_background / enhance / extend / inpaint / style_transfer / color_adjust / redo / resize。",
|
||
"如果用户提到视频、首帧、尾帧、分镜，intent=video_gen。",
|
||
"有图片且用户在修改图片时，优先 intent=image_edit。",
|
||
"JSON 字段：intent, subIntent, confidence, conversationalReply, needsClarification, clarifyingQuestions。",
|
||
"confidence 为 0-1 数字；needsClarification 默认 false；clarifyingQuestions 默认空数组。",
|
||
params.context ? `画布上下文（仅理解，不要复述）: ${params.context.slice(0, 180)}` : "",
|
||
historySnippet ? `近期对话（仅理解，不要复述）:\n${historySnippet}` : "",
|
||
`当前用户请求: ${params.text}`,
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
|
||
try {
|
||
const { callQwen3Vl } = await import("./qwen3-vl");
|
||
console.warn(
|
||
`[Smart Analyzer] Primary analysis failed, fallback to hardcoded qwen3-vl (${params.reason})`,
|
||
);
|
||
for (const imageInput of imageCandidates) {
|
||
const inputMode = imageInput.startsWith("data:image/") ? "inline" : "remote";
|
||
try {
|
||
const { text: fallbackText, model } = await callQwen3Vl({
|
||
imageUrl: imageInput,
|
||
prompt,
|
||
temperature: 0.1,
|
||
maxTokens: 360,
|
||
timeoutMs: ANALYZER_QWEN_FALLBACK_TIMEOUT_MS,
|
||
retries: ANALYZER_QWEN_FALLBACK_RETRIES,
|
||
});
|
||
const parsed = parseAnalysisResultFromModelText(
|
||
fallbackText,
|
||
params.text,
|
||
params.hasImage,
|
||
);
|
||
if (parsed) {
|
||
console.log(`[Smart Analyzer] qwen3-vl fallback success: ${model}, input=${inputMode}`);
|
||
return parsed;
|
||
}
|
||
} catch (attemptError) {
|
||
console.warn(`[Smart Analyzer] qwen3-vl fallback attempt failed: input=${inputMode}`, attemptError);
|
||
}
|
||
}
|
||
console.warn("[Smart Analyzer] qwen3-vl fallback returned unparsable content");
|
||
return null;
|
||
} catch (error) {
|
||
console.warn("[Smart Analyzer] qwen3-vl fallback failed:", error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
async function summarizeWithHardcodedQwen3VlFallback(params: {
|
||
text?: string;
|
||
context?: string;
|
||
imageUrls?: string[];
|
||
}): Promise<string | null> {
|
||
const imageCandidates = await getQwen3VlFallbackImageCandidates(params.imageUrls);
|
||
if (imageCandidates.length === 0) return null;
|
||
|
||
const prompt = [
|
||
"你是图像观察助手。只描述图片可见内容，不要写执行过程，不要推测用户意图。",
|
||
"输出严格一行中文：画面：主体与关键元素；场景：环境；风格：视觉风格；光线：明暗与色温。",
|
||
params.text ? `用户目标（仅理解，不要复述）: ${params.text.slice(0, 120)}` : "",
|
||
params.context ? `画布上下文（仅理解，不要复述）: ${params.context.slice(0, 120)}` : "",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
|
||
try {
|
||
const { callQwen3Vl } = await import("./qwen3-vl");
|
||
console.warn("[Smart Analyzer] Visual summary fallback to hardcoded qwen3-vl");
|
||
for (const imageInput of imageCandidates) {
|
||
const inputMode = imageInput.startsWith("data:image/") ? "inline" : "remote";
|
||
try {
|
||
const { text, model } = await callQwen3Vl({
|
||
imageUrl: imageInput,
|
||
prompt,
|
||
temperature: 0.1,
|
||
maxTokens: 140,
|
||
timeoutMs: ANALYZER_QWEN_FALLBACK_TIMEOUT_MS,
|
||
retries: ANALYZER_QWEN_FALLBACK_RETRIES,
|
||
});
|
||
const normalized = normalizeVisualSummaryText(text);
|
||
if (normalized && !isWeakVisualSummary(normalized)) {
|
||
console.log(`[Smart Analyzer] qwen3-vl summary fallback success: ${model}, input=${inputMode}`);
|
||
return normalized;
|
||
}
|
||
} catch (attemptError) {
|
||
console.warn(`[Smart Analyzer] qwen3-vl summary fallback attempt failed: input=${inputMode}`, attemptError);
|
||
}
|
||
}
|
||
return null;
|
||
} catch (error) {
|
||
console.warn("[Smart Analyzer] qwen3-vl summary fallback failed:", error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function parseAnalysisResultFromModelText(
|
||
responseText: string,
|
||
userText: string,
|
||
hasImage: boolean,
|
||
): AnalysisResult | null {
|
||
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
||
if (!jsonMatch) {
|
||
console.warn("[Smart Analyzer] Model response missing JSON:", responseText.slice(0, 240));
|
||
const fallback = fallbackKeywordAnalysis(userText, hasImage);
|
||
const conversationalReply = responseText.trim();
|
||
if (!conversationalReply) {
|
||
return null;
|
||
}
|
||
return normalizeIntentForImageContext(
|
||
{
|
||
...fallback,
|
||
confidence: Math.max(fallback.confidence, 0.82),
|
||
needsClarification: false,
|
||
conversationalReply,
|
||
},
|
||
userText,
|
||
hasImage,
|
||
);
|
||
}
|
||
|
||
try {
|
||
const result = JSON.parse(jsonMatch[0]) as AnalysisResult;
|
||
return normalizeIntentForImageContext(result, userText, hasImage);
|
||
} catch (error) {
|
||
console.error(
|
||
"[Smart Analyzer] Failed to parse model response:",
|
||
error,
|
||
"raw=",
|
||
jsonMatch[0].slice(0, 240),
|
||
);
|
||
const fallback = fallbackKeywordAnalysis(userText, hasImage);
|
||
const conversationalReply = responseText.trim();
|
||
if (!conversationalReply) {
|
||
return null;
|
||
}
|
||
return normalizeIntentForImageContext(
|
||
{
|
||
...fallback,
|
||
confidence: Math.max(fallback.confidence, 0.8),
|
||
needsClarification: false,
|
||
conversationalReply,
|
||
},
|
||
userText,
|
||
hasImage,
|
||
);
|
||
}
|
||
}
|
||
|
||
function isLikelyZhipuEndpoint(baseUrl?: string | null): boolean {
|
||
if (!baseUrl) return false;
|
||
return /open\.bigmodel\.cn/i.test(baseUrl);
|
||
}
|
||
|
||
function isLikelyVolcanoEndpoint(baseUrl?: string | null): boolean {
|
||
if (!baseUrl) return false;
|
||
return /ark\.cn-beijing\.volces\.com/i.test(baseUrl);
|
||
}
|
||
|
||
function normalizeVolcanoResponsesEndpoint(baseUrl: string): string {
|
||
const cleaned = baseUrl.replace(/\/+$/, "");
|
||
if (!cleaned) return "https://ark.cn-beijing.volces.com/api/v3/responses";
|
||
if (/\/responses$/i.test(cleaned)) return cleaned;
|
||
if (/\/api\/v3$/i.test(cleaned)) return `${cleaned}/responses`;
|
||
if (/ark\.cn-beijing\.volces\.com/i.test(cleaned)) {
|
||
return `${cleaned}/api/v3/responses`.replace(/\/api\/v3\/api\/v3\//, "/api/v3/");
|
||
}
|
||
return `${cleaned}/responses`;
|
||
}
|
||
|
||
async function rewriteUploadPublicUrlForVolcanoSummary(url: string): Promise<string> {
|
||
if (!/^https?:\/\//i.test(url)) return url;
|
||
try {
|
||
const { getDefaultStorageConfig } = await import("@/lib/db/queries");
|
||
const storageConfig = await getDefaultStorageConfig();
|
||
if (
|
||
!storageConfig ||
|
||
storageConfig.type !== "s3" ||
|
||
!storageConfig.endpoint ||
|
||
!storageConfig.bucket ||
|
||
!storageConfig.publicUrl
|
||
) {
|
||
return url;
|
||
}
|
||
|
||
const parsed = new URL(url);
|
||
const publicOrigin = new URL(storageConfig.publicUrl).origin;
|
||
if (parsed.origin !== publicOrigin) return url;
|
||
if (!parsed.pathname.startsWith("/uploads/")) return url;
|
||
|
||
const endpointBase = storageConfig.endpoint.replace(/\/+$/, "");
|
||
return `${endpointBase}/${storageConfig.bucket}${parsed.pathname}${parsed.search}`;
|
||
} catch {
|
||
return url;
|
||
}
|
||
}
|
||
|
||
async function summarizeWithVolcanoOfficialApi(params: {
|
||
apiKey: string;
|
||
baseUrl: string;
|
||
promptText: string;
|
||
imageUrls: string[];
|
||
}): Promise<string | null> {
|
||
const endpoint = normalizeVolcanoResponsesEndpoint(params.baseUrl);
|
||
const rewrittenUrls = await Promise.all(
|
||
params.imageUrls.map((url) => rewriteUploadPublicUrlForVolcanoSummary(url)),
|
||
);
|
||
|
||
try {
|
||
const response = await fetch(endpoint, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${params.apiKey}`,
|
||
},
|
||
body: JSON.stringify({
|
||
model: "doubao-seed-2-0-pro-260215",
|
||
input: [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
...rewrittenUrls.map((url) => ({ type: "input_image", image_url: url })),
|
||
{ type: "input_text", text: params.promptText },
|
||
],
|
||
},
|
||
],
|
||
}),
|
||
});
|
||
|
||
const payload = await response.json().catch(() => ({}));
|
||
if (!response.ok || payload?.error) {
|
||
return null;
|
||
}
|
||
|
||
const directText = String(payload?.output_text || "").trim();
|
||
if (directText) {
|
||
const normalized = normalizeVisualSummaryText(directText);
|
||
if (normalized && !isWeakVisualSummary(normalized)) return normalized;
|
||
}
|
||
|
||
const output = Array.isArray(payload?.output) ? payload.output : [];
|
||
const chunks: string[] = [];
|
||
for (const item of output) {
|
||
const content = Array.isArray(item?.content) ? item.content : [];
|
||
for (const part of content) {
|
||
if (typeof part?.text === "string" && part.text.trim()) {
|
||
chunks.push(part.text.trim());
|
||
}
|
||
}
|
||
}
|
||
const merged = normalizeVisualSummaryText(chunks.join(" ").trim());
|
||
if (merged && !isWeakVisualSummary(merged)) return merged;
|
||
} catch {
|
||
return null;
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function normalizeZhipuChatEndpoint(baseUrl: string): string {
|
||
const cleaned = baseUrl.replace(/\/+$/, "");
|
||
if (/\/chat\/completions$/i.test(cleaned)) return cleaned;
|
||
if (/\/api\/paas\/v4$/i.test(cleaned)) return `${cleaned}/chat/completions`;
|
||
if (/\/api\/paas\/v4\//i.test(cleaned)) return cleaned;
|
||
if (/open\.bigmodel\.cn/i.test(cleaned)) return `${cleaned}/api/paas/v4/chat/completions`;
|
||
return `${cleaned}/chat/completions`;
|
||
}
|
||
|
||
function toZhipuImageUrl(image: string): string {
|
||
const dataUrlPrefix = /^data:image\/[a-zA-Z0-9.+-]+;base64,/i;
|
||
if (dataUrlPrefix.test(image)) {
|
||
return image.replace(dataUrlPrefix, "");
|
||
}
|
||
return image;
|
||
}
|
||
|
||
async function analyzeWithZhipuOfficialApi(params: {
|
||
apiKey: string;
|
||
baseUrl: string;
|
||
userMessage: string;
|
||
visionImages: Array<{ type: "image"; image: string }>;
|
||
userText: string;
|
||
hasImage: boolean;
|
||
}): Promise<AnalysisResult | null> {
|
||
const endpoint = normalizeZhipuChatEndpoint(params.baseUrl);
|
||
|
||
const content = [
|
||
{ type: "text", text: params.userMessage },
|
||
...params.visionImages.map((imagePart) => ({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: toZhipuImageUrl(imagePart.image),
|
||
},
|
||
})),
|
||
];
|
||
|
||
for (const modelId of ZHIPU_VISION_MODEL_CANDIDATES) {
|
||
try {
|
||
const response = await fetch(endpoint, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${params.apiKey}`,
|
||
},
|
||
body: JSON.stringify({
|
||
model: modelId,
|
||
messages: [
|
||
{
|
||
role: "system",
|
||
content: ANALYZER_SYSTEM_PROMPT,
|
||
},
|
||
{
|
||
role: "user",
|
||
content,
|
||
},
|
||
],
|
||
temperature: 0.1,
|
||
max_tokens: 360,
|
||
}),
|
||
});
|
||
|
||
const payload = await response.json().catch(() => ({}));
|
||
if (!response.ok || payload?.error) {
|
||
const errorCode = String(payload?.error?.code || "");
|
||
const errorMessage = String(payload?.error?.message || "");
|
||
if (errorCode === "1211" || /模型不存在/.test(errorMessage)) {
|
||
continue;
|
||
}
|
||
console.warn(
|
||
`[Smart Analyzer] Zhipu official vision request failed (model=${modelId})`,
|
||
payload?.error || payload,
|
||
);
|
||
continue;
|
||
}
|
||
|
||
const message = payload?.choices?.[0]?.message;
|
||
const responseText = String(message?.content || message?.reasoning_content || "").trim();
|
||
if (!responseText) {
|
||
continue;
|
||
}
|
||
|
||
const parsed = parseAnalysisResultFromModelText(
|
||
responseText,
|
||
params.userText,
|
||
params.hasImage,
|
||
);
|
||
if (parsed) {
|
||
console.log(`[Smart Analyzer] Zhipu official vision model used: ${modelId}`);
|
||
return parsed;
|
||
}
|
||
} catch (error) {
|
||
console.warn(
|
||
`[Smart Analyzer] Zhipu official vision request exception (model=${modelId})`,
|
||
error,
|
||
);
|
||
}
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
async function summarizeWithZhipuOfficialApi(params: {
|
||
apiKey: string;
|
||
baseUrl: string;
|
||
promptText: string;
|
||
visionImages: Array<{ type: "image"; image: string }>;
|
||
}): Promise<string | null> {
|
||
const endpoint = normalizeZhipuChatEndpoint(params.baseUrl);
|
||
const content = [
|
||
{ type: "text", text: params.promptText },
|
||
...params.visionImages.map((imagePart) => ({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: toZhipuImageUrl(imagePart.image),
|
||
},
|
||
})),
|
||
];
|
||
|
||
for (const modelId of ZHIPU_VISION_MODEL_CANDIDATES) {
|
||
try {
|
||
const response = await fetch(endpoint, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${params.apiKey}`,
|
||
},
|
||
body: JSON.stringify({
|
||
model: modelId,
|
||
messages: [
|
||
{
|
||
role: "system",
|
||
content:
|
||
"你是图像观察助手。只描述图片可见内容，不要提执行动作或用户意图。输出严格一行中文。",
|
||
},
|
||
{
|
||
role: "user",
|
||
content,
|
||
},
|
||
],
|
||
temperature: 0.2,
|
||
max_tokens: 140,
|
||
}),
|
||
});
|
||
|
||
const payload = await response.json().catch(() => ({}));
|
||
if (!response.ok || payload?.error) {
|
||
const errorCode = String(payload?.error?.code || "");
|
||
const errorMessage = String(payload?.error?.message || "");
|
||
if (errorCode === "1211" || /模型不存在/.test(errorMessage)) {
|
||
continue;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
const message = payload?.choices?.[0]?.message;
|
||
const responseText = String(message?.content || message?.reasoning_content || "").trim();
|
||
const normalized = normalizeVisualSummaryText(responseText);
|
||
if (normalized && !isWeakVisualSummary(normalized)) {
|
||
console.log(`[Smart Analyzer] Zhipu official summary model used: ${modelId}`);
|
||
return normalized;
|
||
}
|
||
} catch {
|
||
continue;
|
||
}
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
export async function summarizeImageForExecutionReport(params: {
|
||
text?: string;
|
||
imageUrls?: string[];
|
||
context?: string;
|
||
preferredModelId?: string;
|
||
}): Promise<string | null> {
|
||
const { text, imageUrls, context, preferredModelId } = params;
|
||
const visionImages = await buildAnalyzerVisionImages(imageUrls);
|
||
if (visionImages.length === 0) return null;
|
||
|
||
const {
|
||
getEnabledAiModelsWithProvider,
|
||
getAgentAiModelWithProvider,
|
||
} = await import("@/lib/db/queries");
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const configuredAgentModel = await getAgentAiModelWithProvider();
|
||
const normalizedPreferredId = String(preferredModelId || "").trim();
|
||
const preferredModel =
|
||
normalizedPreferredId.length > 0
|
||
? allModels.find(
|
||
(item) => item.model.id === normalizedPreferredId || item.model.modelId === normalizedPreferredId,
|
||
)
|
||
: undefined;
|
||
const agentModel = preferredModel || configuredAgentModel;
|
||
const zhipuProviderFromAgent =
|
||
agentModel?.provider?.apiKey && isLikelyZhipuEndpoint(agentModel.provider.baseUrl)
|
||
? agentModel.provider
|
||
: null;
|
||
const zhipuProviderFallback = allModels.find(
|
||
(m) =>
|
||
m.provider?.apiKey &&
|
||
(m.provider.type === "zhipu" || isLikelyZhipuEndpoint(m.provider.baseUrl)),
|
||
)?.provider;
|
||
const zhipuProvider = zhipuProviderFromAgent || zhipuProviderFallback;
|
||
const volcanoProviderFromAgent =
|
||
agentModel?.provider?.apiKey && isLikelyVolcanoEndpoint(agentModel.provider.baseUrl)
|
||
? agentModel.provider
|
||
: null;
|
||
const volcanoProviderFallback = allModels.find(
|
||
(m) => {
|
||
const providerType = String(m.provider?.type || "");
|
||
return (
|
||
m.provider?.apiKey &&
|
||
(
|
||
providerType === "volcano-vision" ||
|
||
providerType === "volcano-multimodal" ||
|
||
isLikelyVolcanoEndpoint(m.provider?.baseUrl)
|
||
)
|
||
);
|
||
},
|
||
)?.provider;
|
||
const volcanoProvider = volcanoProviderFromAgent || volcanoProviderFallback;
|
||
const forceQwenSummary = shouldForceQwen3VlForImageAnalysis(agentModel);
|
||
|
||
const promptText = [
|
||
"只根据图片中可见内容做客观观察，不要推测用户意图。",
|
||
"禁止出现“选中/待处理/按需求/处理/修改/转换”等执行词。",
|
||
"输出一行：画面：主体与关键元素；场景：环境；风格：视觉风格；光线：明暗与色温。",
|
||
text ? `用户目标（仅辅助理解，不要复述）：${text.slice(0, 120)}` : "",
|
||
context ? `画布上下文（仅辅助理解，不要复述）：${context.slice(0, 120)}` : "",
|
||
].join("\n");
|
||
|
||
if (forceQwenSummary) {
|
||
console.warn(
|
||
"[Smart Analyzer] Force qwen3-vl visual summary due multimodal-flag-only agent model",
|
||
);
|
||
const forcedSummary = await summarizeWithHardcodedQwen3VlFallback({
|
||
text,
|
||
context,
|
||
imageUrls,
|
||
});
|
||
if (forcedSummary) return forcedSummary;
|
||
}
|
||
|
||
if (zhipuProvider?.apiKey && zhipuProvider?.baseUrl) {
|
||
const summary = await summarizeWithZhipuOfficialApi({
|
||
apiKey: zhipuProvider.apiKey,
|
||
baseUrl: zhipuProvider.baseUrl,
|
||
promptText,
|
||
visionImages,
|
||
});
|
||
if (summary) return summary;
|
||
}
|
||
|
||
if (volcanoProvider?.apiKey && volcanoProvider?.baseUrl) {
|
||
const summary = await summarizeWithVolcanoOfficialApi({
|
||
apiKey: volcanoProvider.apiKey,
|
||
baseUrl: volcanoProvider.baseUrl,
|
||
promptText,
|
||
imageUrls: (imageUrls || []).filter((url) => isHttpUrl(url)).slice(0, ANALYZER_MAX_IMAGE_COUNT),
|
||
});
|
||
if (summary) return summary;
|
||
}
|
||
|
||
try {
|
||
const candidateModels = collectVisionSummaryCandidates({
|
||
agentModel,
|
||
allModels,
|
||
});
|
||
if (candidateModels.length > 0) {
|
||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||
for (const candidate of candidateModels) {
|
||
const model = await getLanguageModelById(candidate.model.id);
|
||
if (!model) continue;
|
||
|
||
const preferRemote = shouldPreferRemoteVisionUrls(candidate);
|
||
const modeAttempts = Array.from(new Set([preferRemote, !preferRemote]));
|
||
|
||
for (const preferRemoteUrls of modeAttempts) {
|
||
const fallbackVisionImages = await buildAnalyzerVisionImagesByMode({
|
||
imageUrls,
|
||
preferRemoteUrls,
|
||
});
|
||
if (fallbackVisionImages.length === 0) continue;
|
||
|
||
try {
|
||
const { text: summaryText } = await generateText({
|
||
model,
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{
|
||
type: "text",
|
||
text: `${promptText}\n仅输出一行，不超过60字。`,
|
||
},
|
||
...fallbackVisionImages,
|
||
],
|
||
},
|
||
],
|
||
temperature: 0.1,
|
||
maxTokens: 120,
|
||
abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS),
|
||
} as any);
|
||
|
||
const normalized = normalizeVisualSummaryText(summaryText);
|
||
if (normalized && !isWeakVisualSummary(normalized)) {
|
||
console.log(
|
||
`[Smart Analyzer] Visual summary model used: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`,
|
||
);
|
||
return normalized;
|
||
}
|
||
if (normalized) {
|
||
console.warn(
|
||
`[Smart Analyzer] Weak visual summary ignored: ${normalized.slice(0, 120)}`,
|
||
);
|
||
}
|
||
} catch (attemptError) {
|
||
console.warn(
|
||
`[Smart Analyzer] Visual summary attempt failed: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`,
|
||
attemptError,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} catch (error) {
|
||
console.warn("[Smart Analyzer] summarizeImageForExecutionReport fallback failed:", error);
|
||
}
|
||
|
||
const qwenFallbackSummary = await summarizeWithHardcodedQwen3VlFallback({
|
||
text,
|
||
context,
|
||
imageUrls,
|
||
});
|
||
if (qwenFallbackSummary) return qwenFallbackSummary;
|
||
|
||
return null;
|
||
}
|
||
|
||
function normalizeVisualSummaryText(text?: string | null): string {
|
||
if (!text) return "";
|
||
return text
|
||
.replace(/\s+/g, " ")
|
||
.replace(/^[-*•]\s*/, "")
|
||
.replace(/^(画面要点|画面描述|图片分析|分析结果)[:：]\s*/i, "")
|
||
.trim();
|
||
}
|
||
|
||
function isWeakVisualSummary(summary: string): boolean {
|
||
const normalized = summary.replace(/\s+/g, " ").trim();
|
||
if (!normalized || normalized.length < 14) return true;
|
||
|
||
const weakPatterns = [
|
||
/未获取到待分析/,
|
||
/无可见内容可描述/,
|
||
/无法获取图片/,
|
||
/无法访问图片/,
|
||
/看不到图片/,
|
||
/图片不可用/,
|
||
/待处理图/,
|
||
/选中.*位置/,
|
||
/当前上传图片/,
|
||
/基于.*原图处理/,
|
||
/按.*需求执行/,
|
||
/仅做.*调整/,
|
||
/不改动原有内容结构/,
|
||
/转换为.*场景/,
|
||
];
|
||
if (weakPatterns.some((pattern) => pattern.test(normalized))) {
|
||
return true;
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
type VisionSummaryModel = Awaited<
|
||
ReturnType<typeof import("@/lib/db/queries")["getEnabledAiModelsWithProvider"]>
|
||
>[number];
|
||
|
||
function isVisionSummaryCandidateModel(modelWithProvider: VisionSummaryModel): boolean {
|
||
const providerType = String(modelWithProvider.provider?.type || "");
|
||
if (["fal-video", "volcano-video", "qianwen-video", "doubao-video"].includes(providerType)) {
|
||
return false;
|
||
}
|
||
return isLikelyVisionCapableModel(modelWithProvider);
|
||
}
|
||
|
||
function collectVisionSummaryCandidates(params: {
|
||
agentModel?: VisionSummaryModel | null;
|
||
allModels: VisionSummaryModel[];
|
||
}): VisionSummaryModel[] {
|
||
const result: VisionSummaryModel[] = [];
|
||
const seen = new Set<string>();
|
||
|
||
const push = (item?: VisionSummaryModel | null) => {
|
||
if (!item) return;
|
||
if (!isVisionSummaryCandidateModel(item)) return;
|
||
const id = String(item.model?.id || "");
|
||
if (!id || seen.has(id)) return;
|
||
seen.add(id);
|
||
result.push(item);
|
||
};
|
||
|
||
push(params.agentModel || undefined);
|
||
for (const model of params.allModels) push(model);
|
||
return result;
|
||
}
|
||
|
||
// 历史消息类型
|
||
interface HistoryMessage {
|
||
role: "user" | "assistant";
|
||
content: string;
|
||
}
|
||
|
||
function isLikelyVisionCapableModel(modelWithProvider: {
|
||
model: {
|
||
isEnabled: boolean;
|
||
isSubModel: boolean;
|
||
supportsVision?: boolean;
|
||
displayName?: string | null;
|
||
modelId?: string | null;
|
||
capabilities?: unknown;
|
||
defaultParams?: unknown;
|
||
};
|
||
provider: {
|
||
type?: string | null;
|
||
};
|
||
}) {
|
||
const model = modelWithProvider.model;
|
||
const providerType = String(modelWithProvider.provider?.type || "");
|
||
if (!model?.isEnabled || model.isSubModel) return false;
|
||
if (model.supportsVision) return true;
|
||
if (isAiModelMultimodal(model)) return true;
|
||
|
||
const modelText = `${model.displayName || ""} ${model.modelId || ""}`.toLowerCase();
|
||
|
||
if (providerType === "volcano-vision" || providerType === "volcano-multimodal") return true;
|
||
|
||
if (
|
||
providerType === "zhipu" &&
|
||
/(glm[-_.]?4v|glm[-_.]?4\.?6v|vision|多模态)/i.test(modelText)
|
||
) {
|
||
return true;
|
||
}
|
||
|
||
if (
|
||
providerType === "alibaba" &&
|
||
/(qwen|vl|vision|通义|多模态)/i.test(modelText)
|
||
) {
|
||
return true;
|
||
}
|
||
|
||
if (
|
||
["custom", "openai", "local"].includes(providerType) &&
|
||
/(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0|qwen[-_.]?vl|glm[-_.]?4v|gpt[-_.]?4o|vision|vl|多模态)/i.test(
|
||
modelText,
|
||
)
|
||
) {
|
||
return true;
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
function shouldForceQwen3VlForImageAnalysis(modelWithProvider?: {
|
||
model: {
|
||
supportsVision?: boolean;
|
||
displayName?: string | null;
|
||
modelId?: string | null;
|
||
capabilities?: unknown;
|
||
};
|
||
provider: {
|
||
type?: string | null;
|
||
};
|
||
} | null): boolean {
|
||
if (process.env.AGENT_FORCE_QWEN3VL_FALLBACK !== "1") {
|
||
return false;
|
||
}
|
||
if (!modelWithProvider) return false;
|
||
if (modelWithProvider.model.supportsVision) return false;
|
||
if (!isAiModelMultimodal(modelWithProvider.model)) return false;
|
||
|
||
const providerType = String(modelWithProvider.provider?.type || "");
|
||
if (!["custom", "openai", "local"].includes(providerType)) return false;
|
||
|
||
const modelText =
|
||
`${modelWithProvider.model.displayName || ""} ${modelWithProvider.model.modelId || ""}`.toLowerCase();
|
||
|
||
return /(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0)/i.test(modelText);
|
||
}
|
||
|
||
/**
|
||
* 分析用户意图
|
||
*/
|
||
export async function analyzeIntent(params: {
|
||
text: string;
|
||
hasImage: boolean;
|
||
imageUrls?: string[];
|
||
imageFromHistory?: boolean; // 🆕 图片是否来自历史记录
|
||
historyMessages?: HistoryMessage[]; // 🆕 历史对话消息（用于上下文理解）
|
||
context?: string; // 🆕 画布上下文（仅用于理解）
|
||
preferredModelId?: string;
|
||
}): Promise<AnalysisResult> {
|
||
const { text, hasImage, imageUrls, imageFromHistory, historyMessages, context, preferredModelId } = params;
|
||
|
||
console.log("[Smart Analyzer] Analyzing intent:", {
|
||
text,
|
||
hasImage,
|
||
imageCount: imageUrls?.length,
|
||
imageFromHistory,
|
||
historyCount: historyMessages?.length || 0
|
||
});
|
||
|
||
// 先尝试关键词快速匹配（高置信度场景直接返回，节省LLM调用）
|
||
const quickResult = quickKeywordMatch(text, hasImage);
|
||
if (quickResult.confidence >= 0.9) {
|
||
console.log("[Smart Analyzer] Quick match result:", quickResult);
|
||
return applyDirectExecutionPolicy(quickResult, { text, hasImage, imageUrls, imageFromHistory });
|
||
}
|
||
|
||
// 尝试使用多模态模型进行深度分析（传入历史对话）
|
||
try {
|
||
const llmResult = await analyzWithLLM(
|
||
text,
|
||
hasImage,
|
||
imageUrls,
|
||
historyMessages,
|
||
context,
|
||
preferredModelId,
|
||
);
|
||
if (llmResult) {
|
||
console.log("[Smart Analyzer] LLM analysis result:", llmResult);
|
||
return applyDirectExecutionPolicy(llmResult, { text, hasImage, imageUrls, imageFromHistory });
|
||
}
|
||
} catch (error) {
|
||
console.error("[Smart Analyzer] LLM analysis failed:", error);
|
||
}
|
||
|
||
// 回退到完整关键词分析
|
||
const fallbackResult = fallbackKeywordAnalysis(text, hasImage);
|
||
console.log("[Smart Analyzer] Fallback result:", fallbackResult);
|
||
return applyDirectExecutionPolicy(fallbackResult, { text, hasImage, imageUrls, imageFromHistory });
|
||
}
|
||
|
||
/**
|
||
* 快速关键词匹配（高置信度场景）
|
||
* 🆕 现在返回低置信度，强制走LLM分析以获取自然对话回复
|
||
*/
|
||
function quickKeywordMatch(text: string, hasImage: boolean): AnalysisResult {
|
||
// 返回低置信度，强制走LLM分析以获取自然对话回复
|
||
// 只有LLM不可用时才会使用fallback
|
||
return {
|
||
intent: "chat",
|
||
confidence: 0.3,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 使用 LLM 进行深度分析
|
||
*/
|
||
async function analyzWithLLM(
|
||
text: string,
|
||
hasImage: boolean,
|
||
imageUrls?: string[],
|
||
historyMessages?: HistoryMessage[],
|
||
context?: string,
|
||
preferredModelId?: string
|
||
): Promise<AnalysisResult | null> {
|
||
// 使用动态导入避免循环依赖问题
|
||
const {
|
||
getEnabledAiModelsWithProvider,
|
||
getAgentAiModelWithProvider,
|
||
} = await import("@/lib/db/queries");
|
||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||
|
||
// 优先规则：
|
||
// 1) 若 Agent 模型被标记为「多模态」，且当前请求包含图片，则识图+分析统一走该 Agent 模型
|
||
// 2) 否则按既有视觉模型优先级选择
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const configuredAgentModel = await getAgentAiModelWithProvider();
|
||
const normalizedPreferredId = String(preferredModelId || "").trim();
|
||
const preferredModel =
|
||
normalizedPreferredId.length > 0
|
||
? allModels.find(
|
||
(item) => item.model.id === normalizedPreferredId || item.model.modelId === normalizedPreferredId,
|
||
)
|
||
: undefined;
|
||
const agentModel = preferredModel || configuredAgentModel;
|
||
const isReasoningCompatibleProvider = (providerType: string) =>
|
||
["zhipu", "alibaba", "openai", "custom", "local", "volcano-multimodal", "volcano-vision"].includes(providerType);
|
||
const isLikelyZhipuVisionModel = (displayName: string, modelId: string, providerType: string) =>
|
||
providerType === "zhipu" ||
|
||
/(glm[-_.]?4v|glm[-_.]?4\.?1v|glm[-_.]?4\.?5v|vision)/i.test(
|
||
`${displayName} ${modelId}`,
|
||
);
|
||
const zhipuVisionModel = allModels.find(
|
||
(m) =>
|
||
isLikelyVisionCapableModel(m) &&
|
||
isLikelyZhipuVisionModel(m.model.displayName, m.model.modelId, m.provider.type),
|
||
);
|
||
const qwenVisionModel = allModels.find(
|
||
(m) =>
|
||
m.provider.type === "alibaba" &&
|
||
isLikelyVisionCapableModel(m) &&
|
||
/(qwen|通义|vl)/i.test(`${m.model.displayName} ${m.model.modelId}`)
|
||
);
|
||
const fallbackVisionModel =
|
||
qwenVisionModel ||
|
||
allModels.find(
|
||
(m) =>
|
||
m.provider.type === "alibaba" &&
|
||
isLikelyVisionCapableModel(m),
|
||
);
|
||
const preferredVisionModel =
|
||
zhipuVisionModel ||
|
||
fallbackVisionModel ||
|
||
allModels.find((m) => isLikelyVisionCapableModel(m));
|
||
const usableAgentModel =
|
||
agentModel && isReasoningCompatibleProvider(agentModel.provider.type)
|
||
? agentModel
|
||
: undefined;
|
||
const multimodalAgentVisionModel =
|
||
agentModel &&
|
||
isLikelyVisionCapableModel(agentModel)
|
||
? agentModel
|
||
: undefined;
|
||
const selectedAnalyzerModel = hasImage
|
||
? multimodalAgentVisionModel || preferredVisionModel || usableAgentModel
|
||
: usableAgentModel ||
|
||
allModels.find((m) => m.provider.type === "zhipu" && !m.model.isSubModel) ||
|
||
preferredVisionModel;
|
||
|
||
if (preferredModel && selectedAnalyzerModel?.model.id === preferredModel.model.id) {
|
||
console.log(
|
||
`[Smart Analyzer] Preferred analyzer model selected: ${preferredModel.model.displayName} (${preferredModel.provider.type})`,
|
||
);
|
||
}
|
||
|
||
if (!selectedAnalyzerModel) {
|
||
console.log("[Smart Analyzer] No available analyzer model found");
|
||
return analyzeWithHardcodedQwen3VlFallback({
|
||
text,
|
||
hasImage,
|
||
imageUrls,
|
||
historyMessages,
|
||
context,
|
||
reason: "no_available_model",
|
||
});
|
||
}
|
||
|
||
if (hasImage && shouldForceQwen3VlForImageAnalysis(selectedAnalyzerModel)) {
|
||
console.warn(
|
||
"[Smart Analyzer] Force hardcoded qwen3-vl for image analysis due multimodal-flag-only primary model",
|
||
);
|
||
return analyzeWithHardcodedQwen3VlFallback({
|
||
text,
|
||
hasImage,
|
||
imageUrls,
|
||
historyMessages,
|
||
context,
|
||
reason: "force_qwen3vl_multimodal_flag_only",
|
||
});
|
||
}
|
||
|
||
const model = await getLanguageModelById(selectedAnalyzerModel.model.id);
|
||
if (!model) {
|
||
console.log("[Smart Analyzer] Failed to get language model");
|
||
return analyzeWithHardcodedQwen3VlFallback({
|
||
text,
|
||
hasImage,
|
||
imageUrls,
|
||
historyMessages,
|
||
context,
|
||
reason: "model_runtime_unavailable",
|
||
});
|
||
}
|
||
|
||
// 🆕 构建包含历史上下文的消息
|
||
let contextMessage = "";
|
||
|
||
// 🆕 画布上下文（仅用于理解，不要复述）
|
||
if (context && context.trim().length > 0) {
|
||
contextMessage = `## 画布上下文（仅用于理解，不要在对话回复中复述）：\n${context.trim()}\n\n`;
|
||
}
|
||
|
||
// 添加历史对话上下文（最多取最近5轮对话）
|
||
if (historyMessages && historyMessages.length > 0) {
|
||
const recentHistory = historyMessages.slice(-10); // 最多10条消息（约5轮对话）
|
||
const historyText = recentHistory.map(msg =>
|
||
`${msg.role === 'user' ? '用户' : 'AI'}: ${msg.content.substring(0, 200)}`
|
||
).join('\n');
|
||
|
||
contextMessage += `## 对话历史（最近的对话上下文）：
|
||
${historyText}
|
||
|
||
## 当前请求：
|
||
`;
|
||
}
|
||
|
||
// 构建用户消息
|
||
const userMessage = hasImage
|
||
? `${contextMessage}用户上传了图片，请求内容："${text}"`
|
||
: `${contextMessage}用户请求（无图片）："${text}"`;
|
||
|
||
const preferRemoteVisionUrls = shouldPreferRemoteVisionUrls(selectedAnalyzerModel);
|
||
const visionImages = hasImage
|
||
? await buildAnalyzerVisionImagesByMode({
|
||
imageUrls,
|
||
preferRemoteUrls: preferRemoteVisionUrls,
|
||
})
|
||
: [];
|
||
if (visionImages.length > 0) {
|
||
const inlineCount = visionImages.filter((item) => item.image.startsWith("data:image/")).length;
|
||
console.log(
|
||
`[Smart Analyzer] Prepared analyzer images: total=${visionImages.length}, inline=${inlineCount}, remote=${visionImages.length - inlineCount}, preferRemote=${preferRemoteVisionUrls}`,
|
||
);
|
||
}
|
||
|
||
const zhipuProviderFromAgent =
|
||
usableAgentModel &&
|
||
usableAgentModel.provider &&
|
||
usableAgentModel.provider.apiKey &&
|
||
isLikelyZhipuEndpoint(usableAgentModel.provider.baseUrl)
|
||
? usableAgentModel.provider
|
||
: null;
|
||
const zhipuProviderFallback = allModels.find(
|
||
(m) =>
|
||
m.provider?.apiKey &&
|
||
(m.provider.type === "zhipu" || isLikelyZhipuEndpoint(m.provider.baseUrl)),
|
||
)?.provider;
|
||
const zhipuProvider = zhipuProviderFromAgent || zhipuProviderFallback;
|
||
const forceSingleMultimodalAgent = Boolean(hasImage && multimodalAgentVisionModel);
|
||
|
||
if (
|
||
!forceSingleMultimodalAgent &&
|
||
hasImage &&
|
||
visionImages.length > 0 &&
|
||
zhipuProvider?.apiKey &&
|
||
zhipuProvider?.baseUrl
|
||
) {
|
||
const zhipuOfficialResult = await analyzeWithZhipuOfficialApi({
|
||
apiKey: zhipuProvider.apiKey,
|
||
baseUrl: zhipuProvider.baseUrl,
|
||
userMessage,
|
||
visionImages,
|
||
userText: text,
|
||
hasImage,
|
||
});
|
||
if (zhipuOfficialResult) {
|
||
return zhipuOfficialResult;
|
||
}
|
||
}
|
||
|
||
console.log(
|
||
"[Smart Analyzer] Calling analyzer model:",
|
||
selectedAnalyzerModel.model.displayName,
|
||
`(${selectedAnalyzerModel.provider.type})`,
|
||
"history=",
|
||
historyMessages?.length || 0
|
||
);
|
||
|
||
// 调用模型分析（有图时携带图片，确保真实视觉分析）
|
||
const requestPayload = hasImage && visionImages.length > 0
|
||
? {
|
||
model,
|
||
system: ANALYZER_SYSTEM_PROMPT,
|
||
messages: [
|
||
{
|
||
role: "user" as const,
|
||
content: [
|
||
{ type: "text" as const, text: userMessage },
|
||
...visionImages,
|
||
],
|
||
},
|
||
],
|
||
temperature: 0.1,
|
||
maxTokens: 360,
|
||
abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS),
|
||
}
|
||
: {
|
||
model,
|
||
system: ANALYZER_SYSTEM_PROMPT,
|
||
prompt: userMessage,
|
||
temperature: 0.1,
|
||
maxTokens: 300,
|
||
abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS),
|
||
};
|
||
|
||
try {
|
||
const { text: responseText } = await generateText(requestPayload as any);
|
||
const hasStructuredJson = /\{[\s\S]*\}/.test(String(responseText || ""));
|
||
if (hasImage && !hasStructuredJson) {
|
||
console.warn(
|
||
"[Smart Analyzer] Primary analyzer returned non-JSON for image input, fallback to hardcoded qwen3-vl",
|
||
);
|
||
const qwenFallback = await analyzeWithHardcodedQwen3VlFallback({
|
||
text,
|
||
hasImage,
|
||
imageUrls,
|
||
historyMessages,
|
||
context,
|
||
reason: "primary_non_json_response",
|
||
});
|
||
if (qwenFallback) return qwenFallback;
|
||
}
|
||
|
||
const parsed = parseAnalysisResultFromModelText(responseText, text, hasImage);
|
||
if (parsed) return parsed;
|
||
} catch (error) {
|
||
console.warn("[Smart Analyzer] Primary analyzer model failed:", error);
|
||
}
|
||
|
||
return analyzeWithHardcodedQwen3VlFallback({
|
||
text,
|
||
hasImage,
|
||
imageUrls,
|
||
historyMessages,
|
||
context,
|
||
reason: "primary_analysis_failed",
|
||
});
|
||
}
|
||
|
||
/**
|
||
* 回退方案：完整关键词匹配（当LLM不可用时使用）
|
||
* 🆕 添加了自然对话回复
|
||
*/
|
||
export function fallbackKeywordAnalysis(
|
||
text: string,
|
||
hasImage: boolean
|
||
): AnalysisResult {
|
||
const lowerText = text.toLowerCase();
|
||
|
||
// === 图片编辑场景（需要有图片） ===
|
||
if (hasImage) {
|
||
// 视频生成（图生视频/首尾帧等）
|
||
if (isVideoGenerationRequest(lowerText)) {
|
||
return {
|
||
intent: "video_gen",
|
||
confidence: 0.9,
|
||
conversationalReply: "收到，我先按你提供的图片做视频需求分析和分镜规划，再自动优化提示词执行生成~",
|
||
};
|
||
}
|
||
|
||
// 抠图/去背景
|
||
if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "remove_background",
|
||
confidence: 0.9,
|
||
conversationalReply: "好的，我来帮你抠图，去除背景保留主体~",
|
||
};
|
||
}
|
||
|
||
// 换背景
|
||
if (/换背景|白底|改背景|背景换成|纯色背景|换成.*背景|替换.*背景/.test(lowerText)) {
|
||
// 提取背景颜色
|
||
const bgMatch = text.match(/(?:换成|改成|替换为|变成)(.+?)(?:背景|底|$)/);
|
||
const bg = bgMatch ? bgMatch[1].trim() : "白色";
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "replace_background",
|
||
confidence: 0.9,
|
||
conversationalReply: `好的，我来帮你把背景换成${bg}~`,
|
||
};
|
||
}
|
||
|
||
// 图片增强
|
||
if (/变清晰|高清|锐化|降噪|提升质量|修复|清晰化|变清楚|画质/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "enhance",
|
||
confidence: 0.9,
|
||
conversationalReply: "收到，我来帮你提升图片清晰度~",
|
||
};
|
||
}
|
||
|
||
// 图像拓展
|
||
if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "extend",
|
||
confidence: 0.9,
|
||
conversationalReply: "好的，我来帮你扩展图片边界~",
|
||
};
|
||
}
|
||
|
||
// 局部修改/去除
|
||
if (/去除|消除|擦掉|P掉|p掉|删除|修改.*部分|移除.*(?:人|物|水印|logo|文字)/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "inpaint",
|
||
confidence: 0.9,
|
||
conversationalReply: "明白，我来帮你处理这个区域~",
|
||
};
|
||
}
|
||
|
||
// 风格迁移
|
||
if (/风格|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格/.test(lowerText)) {
|
||
const styleMatch = text.match(/(?:转换为|变成|改成|转成)(.+?)(?:风格|$)/);
|
||
const style = styleMatch ? styleMatch[1].trim() : "新";
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "style_transfer",
|
||
confidence: 0.9,
|
||
conversationalReply: `好的，我来帮你转换成${style}风格~`,
|
||
};
|
||
}
|
||
|
||
// 调色/改颜色
|
||
if (/调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成/.test(lowerText)) {
|
||
// 提取目标颜色
|
||
const colorMatch = text.match(/(?:改成|换成|变成)(.+?)(?:色|$)/);
|
||
const color = colorMatch ? colorMatch[1].trim() + "色" : "";
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "color_adjust",
|
||
confidence: 0.9,
|
||
conversationalReply: color ? `好的，我来帮你把颜色改成${color}~` : "收到，我来帮你调整图片色彩~",
|
||
};
|
||
}
|
||
|
||
// 尺寸调整
|
||
if (/缩放|放大|缩小|调整尺寸|裁剪|resize|crop/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "resize",
|
||
confidence: 0.9,
|
||
conversationalReply: "好的，我来帮你调整图片尺寸~",
|
||
};
|
||
}
|
||
|
||
// 通用编辑关键词
|
||
if (/修图|P图|改图|调图|处理.*图|编辑.*图|优化.*图/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: inferDefaultImageEditSubIntent(text),
|
||
confidence: 0.8,
|
||
needsClarification: false,
|
||
conversationalReply: "收到，我先按你的需求直接执行图片优化。",
|
||
};
|
||
}
|
||
|
||
// 视觉理解
|
||
if (/看|识别|分析|描述|这是什么|有什么|tell me|what is|看看|帮我看/.test(lowerText)) {
|
||
return {
|
||
intent: "vision",
|
||
confidence: 0.85,
|
||
conversationalReply: "让我来看看这张图片~",
|
||
};
|
||
}
|
||
|
||
// 有图但意图不明确
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: inferDefaultImageEditSubIntent(text),
|
||
confidence: 0.75,
|
||
needsClarification: false,
|
||
conversationalReply: "收到图片，我先按保真优化策略直接执行。",
|
||
};
|
||
}
|
||
|
||
// === 无图片场景 ===
|
||
|
||
// 图片生成
|
||
if (/画|绘制|生成图|创建.*图|设计.*图|画一个|生成一张|做一张/.test(lowerText)) {
|
||
return {
|
||
intent: "image_gen",
|
||
confidence: 0.85,
|
||
conversationalReply: "好的，我来帮你生成这张图片~",
|
||
};
|
||
}
|
||
|
||
// 视频生成
|
||
if (isVideoGenerationRequest(lowerText)) {
|
||
return {
|
||
intent: "video_gen",
|
||
confidence: 0.85,
|
||
conversationalReply: "好的，我先做视频需求分析并自动优化提示词后执行生成~",
|
||
};
|
||
}
|
||
|
||
// 需要图片但没有上传
|
||
if (/抠图|去背景|换背景|修图|P图|编辑图/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
confidence: 0.7,
|
||
needsClarification: true,
|
||
conversationalReply: "你想编辑图片的话，需要先上传一张图片哦~",
|
||
};
|
||
}
|
||
|
||
// 默认对话
|
||
return { intent: "chat", confidence: 0.7 };
|
||
}
|