568 lines
19 KiB
TypeScript
568 lines
19 KiB
TypeScript
/**
|
||
* 智能意图分析器
|
||
* 分析用户输入，识别图片编辑意图
|
||
*
|
||
* 主要应用场景：图片编辑（抠图、换背景、增强等）
|
||
*/
|
||
|
||
import { generateText } from "ai";
|
||
// 使用动态导入避免循环依赖问题
|
||
// import { getLanguageModelById } from "./dynamic-provider";
|
||
// import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||
|
||
// 分析结果类型
|
||
export interface AnalysisResult {
|
||
intent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat";
|
||
subIntent?: string;
|
||
confidence: number;
|
||
optimizedPrompt?: string;
|
||
negativePrompt?: string;
|
||
suggestedModel?: string;
|
||
needsClarification?: boolean;
|
||
clarifyingQuestions?: string[];
|
||
conversationalReply?: string; // 🆕 自然对话回复
|
||
}
|
||
|
||
function inferDefaultImageEditSubIntent(text: string): AnalysisResult["subIntent"] {
|
||
const lowerText = text.toLowerCase();
|
||
if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background";
|
||
if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background";
|
||
if (/风格|卡通|动漫|油画|像素风|写实|插画/.test(lowerText)) return "style_transfer";
|
||
if (/颜色|调色|亮度|对比度|饱和度|改成.*色|换成.*色/.test(lowerText)) return "color_adjust";
|
||
if (/去除|消除|擦掉|删除|移除|p掉|重绘/.test(lowerText)) return "inpaint";
|
||
return "enhance";
|
||
}
|
||
|
||
function applyDirectExecutionPolicy(
|
||
result: AnalysisResult,
|
||
params: {
|
||
text: string;
|
||
hasImage: boolean;
|
||
imageUrls?: string[];
|
||
imageFromHistory?: boolean;
|
||
}
|
||
): AnalysisResult {
|
||
const hasUsableImage =
|
||
params.hasImage || Boolean(params.imageFromHistory) || (params.imageUrls?.length || 0) > 0;
|
||
if (!hasUsableImage) return result;
|
||
|
||
const compactText = params.text.replace(/\s+/g, "").trim();
|
||
const hasAnyRequestText = compactText.length > 0;
|
||
const normalizedIntent = result.intent === "chat" || result.intent === "vision" ? "image_edit" : result.intent;
|
||
const normalizedSubIntent =
|
||
normalizedIntent === "image_edit"
|
||
? result.subIntent || inferDefaultImageEditSubIntent(params.text)
|
||
: result.subIntent;
|
||
|
||
return {
|
||
...result,
|
||
intent: normalizedIntent,
|
||
subIntent: normalizedSubIntent,
|
||
confidence: Math.max(result.confidence, 0.72),
|
||
needsClarification: false,
|
||
clarifyingQuestions: [],
|
||
conversationalReply:
|
||
result.conversationalReply ||
|
||
(hasAnyRequestText ? "收到，我会按你的描述直接执行生成。" : "收到图片，我先按保真优化策略直接执行生成。"),
|
||
};
|
||
}
|
||
|
||
function normalizeIntentForImageContext(
|
||
result: AnalysisResult,
|
||
text: string,
|
||
hasImage: boolean
|
||
): AnalysisResult {
|
||
if (!hasImage || result.intent !== "image_gen") {
|
||
return result;
|
||
}
|
||
|
||
const normalizedText = text.toLowerCase();
|
||
const explicitGenerate =
|
||
/重新生成|再来一张|生成新图|新建图片|创作一张|参考.*生成|按这个风格生成|以这张图为灵感生成/.test(
|
||
normalizedText
|
||
);
|
||
const editSignal =
|
||
/改|调整|优化|去掉|换成|改成|保留|继续|再.*一点|修|抠|替换|增强|清晰|重绘/.test(normalizedText);
|
||
|
||
if (explicitGenerate && !editSignal) {
|
||
return result;
|
||
}
|
||
|
||
return {
|
||
...result,
|
||
intent: "image_edit",
|
||
subIntent: result.subIntent,
|
||
confidence: Math.max(result.confidence, 0.78),
|
||
};
|
||
}
|
||
|
||
// 系统提示词 - 针对图片编辑场景优化，生成自然对话
|
||
const ANALYZER_SYSTEM_PROMPT = `你是一个友好的AI图片编辑助手。用户会给你发送图片和编辑请求，你需要：
|
||
1. 理解用户想要对图片做什么
|
||
2. 用自然、友好的语气回复用户，告诉他你理解了他的需求并会帮他处理
|
||
3. 同时输出结构化的意图分析结果
|
||
4. 默认直接执行，不要反问用户；只有在完全无图且需求为空时才允许澄清
|
||
|
||
## 意图分类
|
||
|
||
### image_edit - 图片编辑（用户上传了图片并想编辑）
|
||
子意图：
|
||
- remove_background: 抠图、去背景、透明背景
|
||
- replace_background: 换背景、改背景、白底
|
||
- enhance: 变清晰、高清、锐化、降噪
|
||
- extend: 扩展、拓展、延伸画布
|
||
- inpaint: 去除某物、消除、擦掉、P掉、局部修改
|
||
- style_transfer: 风格化、卡通化、动漫风、改风格
|
||
- color_adjust: 调色、调整颜色、亮度、对比度
|
||
- redo: 用户不满意之前的结果，想重新处理
|
||
|
||
### vision - 图片理解（用户想让你看图说话）
|
||
### image_gen - 图片生成（用户想生成新图片）
|
||
### video_gen - 视频生成
|
||
### chat - 普通对话
|
||
|
||
## 特殊场景：连续对话（用户没有发送新图片，系统自动使用历史图片）
|
||
|
||
当用户继续对话但没有发送新图片时，系统会自动找到之前的图片。这时候：
|
||
- 用户可能在继续描述想要的效果（如："换成蓝色"、"再清晰一点"、"把背景去掉"）
|
||
- 用户可能在修改之前的结果（如："不满意"、"再调整"、"换个风格"）
|
||
- 用户可能在追问（如："还可以再亮一点吗"、"能不能换个颜色"）
|
||
|
||
处理方式：
|
||
- intent 通常是 image_edit
|
||
- subIntent 根据用户的具体要求判断
|
||
- conversationalReply 要自然地回应用户，不需要特别强调"找到了之前的图片"
|
||
|
||
## 输出格式（严格JSON）
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "replace_background",
|
||
"confidence": 0.95,
|
||
"conversationalReply": "好的，我来帮你把背景换成白色！这样产品会更突出~",
|
||
"needsClarification": false,
|
||
"clarifyingQuestions": []
|
||
}
|
||
|
||
## 回复风格要求
|
||
- conversationalReply 要自然、友好、简洁
|
||
- 可以加一些语气词让对话更亲切，如"好的"、"没问题"、"~"等
|
||
- 说明你理解了什么需求，让用户知道你在做什么
|
||
- 有图片时优先直接执行，不要提问
|
||
- 如果提供了画布上下文，仅用于理解，不要在 conversationalReply 中复述坐标或技术细节
|
||
|
||
## 示例
|
||
用户："帮我把背景换成白色"
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "replace_background",
|
||
"confidence": 0.95,
|
||
"conversationalReply": "好的，我来帮你把背景换成纯白色，这样图片会更干净专业~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
用户："把这张图变清晰一点"
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "enhance",
|
||
"confidence": 0.9,
|
||
"conversationalReply": "收到！我来帮你提升图片清晰度，稍等一下~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
用户发了图片但没说要做什么：
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "enhance",
|
||
"confidence": 0.72,
|
||
"conversationalReply": "收到图片，我先做保真优化并输出结果，你可以再继续细化要求~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
连续对话场景 - 用户说"换成蓝色"（系统自动使用历史图片）：
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "color_adjust",
|
||
"confidence": 0.9,
|
||
"conversationalReply": "好的，我来帮你调整成蓝色~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
连续对话场景 - 用户说"不满意，再调整一下"：
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "redo",
|
||
"confidence": 0.8,
|
||
"conversationalReply": "没问题，我先按上一轮目标继续优化并给你一个更稳的版本~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
连续对话场景 - 用户说"再清晰一点"：
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "enhance",
|
||
"confidence": 0.9,
|
||
"conversationalReply": "好的，我再帮你提升一下清晰度~",
|
||
"needsClarification": false
|
||
}`;
|
||
|
||
// 历史消息类型
|
||
interface HistoryMessage {
|
||
role: "user" | "assistant";
|
||
content: string;
|
||
}
|
||
|
||
/**
|
||
* 分析用户意图
|
||
*/
|
||
export async function analyzeIntent(params: {
|
||
text: string;
|
||
hasImage: boolean;
|
||
imageUrls?: string[];
|
||
imageFromHistory?: boolean; // 🆕 图片是否来自历史记录
|
||
historyMessages?: HistoryMessage[]; // 🆕 历史对话消息（用于上下文理解）
|
||
context?: string; // 🆕 画布上下文（仅用于理解）
|
||
}): Promise<AnalysisResult> {
|
||
const { text, hasImage, imageUrls, imageFromHistory, historyMessages, context } = params;
|
||
|
||
console.log("[Smart Analyzer] Analyzing intent:", {
|
||
text,
|
||
hasImage,
|
||
imageCount: imageUrls?.length,
|
||
imageFromHistory,
|
||
historyCount: historyMessages?.length || 0
|
||
});
|
||
|
||
// 先尝试关键词快速匹配（高置信度场景直接返回，节省LLM调用）
|
||
const quickResult = quickKeywordMatch(text, hasImage);
|
||
if (quickResult.confidence >= 0.9) {
|
||
console.log("[Smart Analyzer] Quick match result:", quickResult);
|
||
return applyDirectExecutionPolicy(quickResult, { text, hasImage, imageUrls, imageFromHistory });
|
||
}
|
||
|
||
// 尝试使用 Qwen3 VL 进行深度分析（传入历史对话）
|
||
try {
|
||
const llmResult = await analyzWithLLM(text, hasImage, imageUrls, historyMessages, context);
|
||
if (llmResult) {
|
||
console.log("[Smart Analyzer] LLM analysis result:", llmResult);
|
||
return applyDirectExecutionPolicy(llmResult, { text, hasImage, imageUrls, imageFromHistory });
|
||
}
|
||
} catch (error) {
|
||
console.error("[Smart Analyzer] LLM analysis failed:", error);
|
||
}
|
||
|
||
// 回退到完整关键词分析
|
||
const fallbackResult = fallbackKeywordAnalysis(text, hasImage);
|
||
console.log("[Smart Analyzer] Fallback result:", fallbackResult);
|
||
return applyDirectExecutionPolicy(fallbackResult, { text, hasImage, imageUrls, imageFromHistory });
|
||
}
|
||
|
||
/**
|
||
* 快速关键词匹配（高置信度场景）
|
||
* 🆕 现在返回低置信度，强制走LLM分析以获取自然对话回复
|
||
*/
|
||
function quickKeywordMatch(text: string, hasImage: boolean): AnalysisResult {
|
||
// 返回低置信度，强制走LLM分析以获取自然对话回复
|
||
// 只有LLM不可用时才会使用fallback
|
||
return {
|
||
intent: "chat",
|
||
confidence: 0.3,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 使用 LLM 进行深度分析
|
||
*/
|
||
async function analyzWithLLM(
|
||
text: string,
|
||
hasImage: boolean,
|
||
imageUrls?: string[],
|
||
historyMessages?: HistoryMessage[],
|
||
context?: string
|
||
): Promise<AnalysisResult | null> {
|
||
// 使用动态导入避免循环依赖问题
|
||
const {
|
||
getEnabledAiModelsWithProvider,
|
||
getAgentAiModelWithProvider,
|
||
} = await import("@/lib/db/queries");
|
||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||
|
||
// 图片识别固定走千问多模态；纯文本理解优先走 Agent 模型（建议智普）
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const agentModel = await getAgentAiModelWithProvider();
|
||
const isReasoningCompatibleProvider = (providerType: string) =>
|
||
["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType);
|
||
const qwenVisionModel = allModels.find(
|
||
(m) =>
|
||
m.provider.type === "alibaba" &&
|
||
m.model.supportsVision &&
|
||
/(qwen|通义|vl)/i.test(`${m.model.displayName} ${m.model.modelId}`)
|
||
);
|
||
const fallbackVisionModel =
|
||
qwenVisionModel ||
|
||
allModels.find((m) => m.provider.type === "alibaba" && m.model.supportsVision);
|
||
const usableAgentModel =
|
||
agentModel && isReasoningCompatibleProvider(agentModel.provider.type)
|
||
? agentModel
|
||
: undefined;
|
||
const selectedAnalyzerModel = hasImage
|
||
? fallbackVisionModel
|
||
: usableAgentModel ||
|
||
allModels.find((m) => m.provider.type === "zhipu" && !m.model.isSubModel) ||
|
||
fallbackVisionModel;
|
||
|
||
if (!selectedAnalyzerModel) {
|
||
console.log("[Smart Analyzer] No available analyzer model found");
|
||
return null;
|
||
}
|
||
|
||
const model = await getLanguageModelById(selectedAnalyzerModel.model.id);
|
||
if (!model) {
|
||
console.log("[Smart Analyzer] Failed to get language model");
|
||
return null;
|
||
}
|
||
|
||
// 🆕 构建包含历史上下文的消息
|
||
let contextMessage = "";
|
||
|
||
// 🆕 画布上下文（仅用于理解，不要复述）
|
||
if (context && context.trim().length > 0) {
|
||
contextMessage = `## 画布上下文（仅用于理解，不要在对话回复中复述）：\n${context.trim()}\n\n`;
|
||
}
|
||
|
||
// 添加历史对话上下文（最多取最近5轮对话）
|
||
if (historyMessages && historyMessages.length > 0) {
|
||
const recentHistory = historyMessages.slice(-10); // 最多10条消息（约5轮对话）
|
||
const historyText = recentHistory.map(msg =>
|
||
`${msg.role === 'user' ? '用户' : 'AI'}: ${msg.content.substring(0, 200)}`
|
||
).join('\n');
|
||
|
||
contextMessage += `## 对话历史（最近的对话上下文）：
|
||
${historyText}
|
||
|
||
## 当前请求：
|
||
`;
|
||
}
|
||
|
||
// 构建用户消息
|
||
const userMessage = hasImage
|
||
? `${contextMessage}用户上传了图片，请求内容："${text}"`
|
||
: `${contextMessage}用户请求（无图片）："${text}"`;
|
||
|
||
console.log(
|
||
"[Smart Analyzer] Calling analyzer model:",
|
||
selectedAnalyzerModel.model.displayName,
|
||
`(${selectedAnalyzerModel.provider.type})`,
|
||
"history=",
|
||
historyMessages?.length || 0
|
||
);
|
||
|
||
const visionImages = (imageUrls || [])
|
||
.filter((url) => /^https?:\/\//i.test(url))
|
||
.slice(0, 4)
|
||
.map((url) => ({ type: "image" as const, image: url }));
|
||
|
||
// 调用模型分析（有图时携带图片，确保真实视觉分析）
|
||
const requestPayload = hasImage && visionImages.length > 0
|
||
? {
|
||
model,
|
||
system: ANALYZER_SYSTEM_PROMPT,
|
||
messages: [
|
||
{
|
||
role: "user" as const,
|
||
content: [
|
||
{ type: "text" as const, text: userMessage },
|
||
...visionImages,
|
||
],
|
||
},
|
||
],
|
||
temperature: 0.1,
|
||
maxTokens: 360,
|
||
}
|
||
: {
|
||
model,
|
||
system: ANALYZER_SYSTEM_PROMPT,
|
||
prompt: userMessage,
|
||
temperature: 0.1,
|
||
maxTokens: 300,
|
||
};
|
||
|
||
const { text: responseText } = await generateText(requestPayload as any);
|
||
|
||
// 解析JSON响应
|
||
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
||
if (jsonMatch) {
|
||
try {
|
||
const result = JSON.parse(jsonMatch[0]) as AnalysisResult;
|
||
return normalizeIntentForImageContext(result, text, hasImage);
|
||
} catch (e) {
|
||
console.error("[Smart Analyzer] Failed to parse LLM response:", e);
|
||
}
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
/**
|
||
* 回退方案：完整关键词匹配（当LLM不可用时使用）
|
||
* 🆕 添加了自然对话回复
|
||
*/
|
||
export function fallbackKeywordAnalysis(
|
||
text: string,
|
||
hasImage: boolean
|
||
): AnalysisResult {
|
||
const lowerText = text.toLowerCase();
|
||
|
||
// === 图片编辑场景（需要有图片） ===
|
||
if (hasImage) {
|
||
// 抠图/去背景
|
||
if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "remove_background",
|
||
confidence: 0.9,
|
||
conversationalReply: "好的，我来帮你抠图，去除背景保留主体~",
|
||
};
|
||
}
|
||
|
||
// 换背景
|
||
if (/换背景|白底|改背景|背景换成|纯色背景|换成.*背景|替换.*背景/.test(lowerText)) {
|
||
// 提取背景颜色
|
||
const bgMatch = text.match(/(?:换成|改成|替换为|变成)(.+?)(?:背景|底|$)/);
|
||
const bg = bgMatch ? bgMatch[1].trim() : "白色";
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "replace_background",
|
||
confidence: 0.9,
|
||
conversationalReply: `好的，我来帮你把背景换成${bg}~`,
|
||
};
|
||
}
|
||
|
||
// 图片增强
|
||
if (/变清晰|高清|锐化|降噪|提升质量|修复|清晰化|变清楚|画质/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "enhance",
|
||
confidence: 0.9,
|
||
conversationalReply: "收到，我来帮你提升图片清晰度~",
|
||
};
|
||
}
|
||
|
||
// 图像拓展
|
||
if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "extend",
|
||
confidence: 0.9,
|
||
conversationalReply: "好的，我来帮你扩展图片边界~",
|
||
};
|
||
}
|
||
|
||
// 局部修改/去除
|
||
if (/去除|消除|擦掉|P掉|p掉|删除|修改.*部分|移除.*(?:人|物|水印|logo|文字)/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "inpaint",
|
||
confidence: 0.9,
|
||
conversationalReply: "明白，我来帮你处理这个区域~",
|
||
};
|
||
}
|
||
|
||
// 风格迁移
|
||
if (/风格|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格/.test(lowerText)) {
|
||
const styleMatch = text.match(/(?:转换为|变成|改成|转成)(.+?)(?:风格|$)/);
|
||
const style = styleMatch ? styleMatch[1].trim() : "新";
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "style_transfer",
|
||
confidence: 0.9,
|
||
conversationalReply: `好的，我来帮你转换成${style}风格~`,
|
||
};
|
||
}
|
||
|
||
// 调色/改颜色
|
||
if (/调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成/.test(lowerText)) {
|
||
// 提取目标颜色
|
||
const colorMatch = text.match(/(?:改成|换成|变成)(.+?)(?:色|$)/);
|
||
const color = colorMatch ? colorMatch[1].trim() + "色" : "";
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "color_adjust",
|
||
confidence: 0.9,
|
||
conversationalReply: color ? `好的，我来帮你把颜色改成${color}~` : "收到，我来帮你调整图片色彩~",
|
||
};
|
||
}
|
||
|
||
// 尺寸调整
|
||
if (/缩放|放大|缩小|调整尺寸|裁剪|resize|crop/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "resize",
|
||
confidence: 0.9,
|
||
conversationalReply: "好的，我来帮你调整图片尺寸~",
|
||
};
|
||
}
|
||
|
||
// 通用编辑关键词
|
||
if (/修图|P图|改图|调图|处理.*图|编辑.*图|优化.*图/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: inferDefaultImageEditSubIntent(text),
|
||
confidence: 0.8,
|
||
needsClarification: false,
|
||
conversationalReply: "收到，我先按你的需求直接执行图片优化。",
|
||
};
|
||
}
|
||
|
||
// 视觉理解
|
||
if (/看|识别|分析|描述|这是什么|有什么|tell me|what is|看看|帮我看/.test(lowerText)) {
|
||
return {
|
||
intent: "vision",
|
||
confidence: 0.85,
|
||
conversationalReply: "让我来看看这张图片~",
|
||
};
|
||
}
|
||
|
||
// 有图但意图不明确
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: inferDefaultImageEditSubIntent(text),
|
||
confidence: 0.75,
|
||
needsClarification: false,
|
||
conversationalReply: "收到图片，我先按保真优化策略直接执行。",
|
||
};
|
||
}
|
||
|
||
// === 无图片场景 ===
|
||
|
||
// 图片生成
|
||
if (/画|绘制|生成图|创建.*图|设计.*图|画一个|生成一张|做一张/.test(lowerText)) {
|
||
return {
|
||
intent: "image_gen",
|
||
confidence: 0.85,
|
||
conversationalReply: "好的，我来帮你生成这张图片~",
|
||
};
|
||
}
|
||
|
||
// 视频生成
|
||
if (/视频|动画|动起来|生成视频|做视频|视频生成/.test(lowerText)) {
|
||
return {
|
||
intent: "video_gen",
|
||
confidence: 0.85,
|
||
conversationalReply: "好的，我来帮你生成视频~",
|
||
};
|
||
}
|
||
|
||
// 需要图片但没有上传
|
||
if (/抠图|去背景|换背景|修图|P图|编辑图/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
confidence: 0.7,
|
||
needsClarification: true,
|
||
conversationalReply: "你想编辑图片的话，需要先上传一张图片哦~",
|
||
};
|
||
}
|
||
|
||
// 默认对话
|
||
return { intent: "chat", confidence: 0.7 };
|
||
}
|