992 lines
32 KiB
TypeScript
992 lines
32 KiB
TypeScript
/**
|
||
* 智能意图分析器
|
||
* 分析用户输入，识别图片编辑意图
|
||
*
|
||
* 主要应用场景：图片编辑（抠图、换背景、增强等）
|
||
*/
|
||
|
||
import { generateText } from "ai";
|
||
import sharp from "sharp";
|
||
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||
// 使用动态导入避免循环依赖问题
|
||
// import { getLanguageModelById } from "./dynamic-provider";
|
||
// import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||
|
||
// 分析结果类型
|
||
export interface AnalysisResult {
|
||
intent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat";
|
||
subIntent?: string;
|
||
confidence: number;
|
||
optimizedPrompt?: string;
|
||
negativePrompt?: string;
|
||
suggestedModel?: string;
|
||
needsClarification?: boolean;
|
||
clarifyingQuestions?: string[];
|
||
conversationalReply?: string; // 🆕 自然对话回复
|
||
}
|
||
|
||
function inferDefaultImageEditSubIntent(text: string): AnalysisResult["subIntent"] {
|
||
const lowerText = text.toLowerCase();
|
||
if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background";
|
||
if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background";
|
||
if (/风格|卡通|动漫|油画|像素风|写实|插画/.test(lowerText)) return "style_transfer";
|
||
if (/颜色|调色|亮度|对比度|饱和度|改成.*色|换成.*色/.test(lowerText)) return "color_adjust";
|
||
if (/去除|消除|擦掉|删除|移除|p掉|重绘/.test(lowerText)) return "inpaint";
|
||
return "enhance";
|
||
}
|
||
|
||
function isVideoGenerationRequest(text: string): boolean {
|
||
return /视频|动画|动起来|生成视频|做视频|视频生成|图生视频|文生视频|镜头|分镜|运镜|首帧|尾帧/.test(
|
||
text.toLowerCase(),
|
||
);
|
||
}
|
||
|
||
function applyDirectExecutionPolicy(
|
||
result: AnalysisResult,
|
||
params: {
|
||
text: string;
|
||
hasImage: boolean;
|
||
imageUrls?: string[];
|
||
imageFromHistory?: boolean;
|
||
}
|
||
): AnalysisResult {
|
||
if (isVideoGenerationRequest(params.text)) {
|
||
return {
|
||
...result,
|
||
intent: "video_gen",
|
||
subIntent: undefined,
|
||
confidence: Math.max(result.confidence, 0.8),
|
||
needsClarification: false,
|
||
clarifyingQuestions: [],
|
||
conversationalReply:
|
||
result.conversationalReply ||
|
||
(params.hasImage
|
||
? "收到，我先按你这轮图片做视频需求分析，再自动优化提示词执行视频生成。"
|
||
: "收到，我先做视频需求分析并自动优化提示词后执行生成。"),
|
||
};
|
||
}
|
||
|
||
const hasUsableImage =
|
||
params.hasImage || Boolean(params.imageFromHistory) || (params.imageUrls?.length || 0) > 0;
|
||
if (!hasUsableImage) return result;
|
||
|
||
const compactText = params.text.replace(/\s+/g, "").trim();
|
||
const hasAnyRequestText = compactText.length > 0;
|
||
const normalizedIntent = result.intent === "chat" || result.intent === "vision" ? "image_edit" : result.intent;
|
||
const normalizedSubIntent =
|
||
normalizedIntent === "image_edit"
|
||
? result.subIntent || inferDefaultImageEditSubIntent(params.text)
|
||
: result.subIntent;
|
||
|
||
return {
|
||
...result,
|
||
intent: normalizedIntent,
|
||
subIntent: normalizedSubIntent,
|
||
confidence: Math.max(result.confidence, 0.72),
|
||
needsClarification: false,
|
||
clarifyingQuestions: [],
|
||
conversationalReply:
|
||
result.conversationalReply ||
|
||
(hasAnyRequestText ? "收到，我会按你的描述直接执行生成。" : "收到图片，我先按保真优化策略直接执行生成。"),
|
||
};
|
||
}
|
||
|
||
function normalizeIntentForImageContext(
|
||
result: AnalysisResult,
|
||
text: string,
|
||
hasImage: boolean
|
||
): AnalysisResult {
|
||
if (!hasImage || result.intent !== "image_gen") {
|
||
return result;
|
||
}
|
||
|
||
const normalizedText = text.toLowerCase();
|
||
const explicitGenerate =
|
||
/重新生成|再来一张|生成新图|新建图片|创作一张|参考.*生成|按这个风格生成|以这张图为灵感生成/.test(
|
||
normalizedText
|
||
);
|
||
const editSignal =
|
||
/改|调整|优化|去掉|换成|改成|保留|继续|再.*一点|修|抠|替换|增强|清晰|重绘/.test(normalizedText);
|
||
|
||
if (explicitGenerate && !editSignal) {
|
||
return result;
|
||
}
|
||
|
||
return {
|
||
...result,
|
||
intent: "image_edit",
|
||
subIntent: result.subIntent,
|
||
confidence: Math.max(result.confidence, 0.78),
|
||
};
|
||
}
|
||
|
||
// 系统提示词 - 针对图片编辑场景优化，生成自然对话
|
||
const ANALYZER_SYSTEM_PROMPT = `你是一个友好的AI图片编辑助手。用户会给你发送图片和编辑请求，你需要：
|
||
1. 理解用户想要对图片做什么
|
||
2. 用自然、友好的语气回复用户，告诉他你理解了他的需求并会帮他处理
|
||
3. 同时输出结构化的意图分析结果
|
||
4. 默认直接执行，不要反问用户；只有在完全无图且需求为空时才允许澄清
|
||
|
||
## 意图分类
|
||
|
||
### image_edit - 图片编辑（用户上传了图片并想编辑）
|
||
子意图：
|
||
- remove_background: 抠图、去背景、透明背景
|
||
- replace_background: 换背景、改背景、白底
|
||
- enhance: 变清晰、高清、锐化、降噪
|
||
- extend: 扩展、拓展、延伸画布
|
||
- inpaint: 去除某物、消除、擦掉、P掉、局部修改
|
||
- style_transfer: 风格化、卡通化、动漫风、改风格
|
||
- color_adjust: 调色、调整颜色、亮度、对比度
|
||
- redo: 用户不满意之前的结果，想重新处理
|
||
|
||
### vision - 图片理解（用户想让你看图说话）
|
||
### image_gen - 图片生成（用户想生成新图片）
|
||
### video_gen - 视频生成
|
||
### chat - 普通对话
|
||
|
||
## 特殊场景：连续对话（用户没有发送新图片，系统自动使用历史图片）
|
||
|
||
当用户继续对话但没有发送新图片时，系统会自动找到之前的图片。这时候：
|
||
- 用户可能在继续描述想要的效果（如："换成蓝色"、"再清晰一点"、"把背景去掉"）
|
||
- 用户可能在修改之前的结果（如："不满意"、"再调整"、"换个风格"）
|
||
- 用户可能在追问（如："还可以再亮一点吗"、"能不能换个颜色"）
|
||
|
||
处理方式：
|
||
- intent 通常是 image_edit
|
||
- subIntent 根据用户的具体要求判断
|
||
- conversationalReply 要自然地回应用户，不需要特别强调"找到了之前的图片"
|
||
|
||
## 输出格式（严格JSON）
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "replace_background",
|
||
"confidence": 0.95,
|
||
"conversationalReply": "好的，我来帮你把背景换成白色！这样产品会更突出~",
|
||
"needsClarification": false,
|
||
"clarifyingQuestions": []
|
||
}
|
||
|
||
## 回复风格要求
|
||
- conversationalReply 要自然、友好、简洁
|
||
- 可以加一些语气词让对话更亲切，如"好的"、"没问题"、"~"等
|
||
- 说明你理解了什么需求，让用户知道你在做什么
|
||
- 有图片时优先直接执行，不要提问
|
||
- 如果提供了画布上下文，仅用于理解，不要在 conversationalReply 中复述坐标或技术细节
|
||
- video_gen 场景要明确说明：会先分析视频需求/分镜，再自动优化提示词并执行生成
|
||
|
||
## 示例
|
||
用户："帮我把背景换成白色"
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "replace_background",
|
||
"confidence": 0.95,
|
||
"conversationalReply": "好的，我来帮你把背景换成纯白色，这样图片会更干净专业~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
用户："把这张图变清晰一点"
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "enhance",
|
||
"confidence": 0.9,
|
||
"conversationalReply": "收到！我来帮你提升图片清晰度，稍等一下~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
用户发了图片但没说要做什么：
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "enhance",
|
||
"confidence": 0.72,
|
||
"conversationalReply": "收到图片，我先做保真优化并输出结果，你可以再继续细化要求~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
连续对话场景 - 用户说"换成蓝色"（系统自动使用历史图片）：
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "color_adjust",
|
||
"confidence": 0.9,
|
||
"conversationalReply": "好的，我来帮你调整成蓝色~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
连续对话场景 - 用户说"不满意，再调整一下"：
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "redo",
|
||
"confidence": 0.8,
|
||
"conversationalReply": "没问题，我先按上一轮目标继续优化并给你一个更稳的版本~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
连续对话场景 - 用户说"再清晰一点"：
|
||
{
|
||
"intent": "image_edit",
|
||
"subIntent": "enhance",
|
||
"confidence": 0.9,
|
||
"conversationalReply": "好的，我再帮你提升一下清晰度~",
|
||
"needsClarification": false
|
||
}
|
||
|
||
用户："把这张图做成5秒短视频，镜头慢慢推进"
|
||
{
|
||
"intent": "video_gen",
|
||
"confidence": 0.9,
|
||
"conversationalReply": "收到，我先按你的目标做视频需求分析和分镜规划，再自动优化提示词执行生成~",
|
||
"needsClarification": false
|
||
}`;
|
||
|
||
const ANALYZER_MAX_IMAGE_COUNT = 2;
|
||
const ANALYZER_IMAGE_MAX_SIDE = 500;
|
||
const ANALYZER_IMAGE_FETCH_TIMEOUT_MS = 10000;
|
||
const ZHIPU_VISION_MODEL_CANDIDATES = [
|
||
"glm-4.6v-flash",
|
||
"glm-4.6v",
|
||
"glm-4v-plus-0111",
|
||
"glm-4v-plus",
|
||
"glm-4v",
|
||
];
|
||
|
||
function isHttpUrl(value: string) {
|
||
return /^https?:\/\//i.test(value);
|
||
}
|
||
|
||
async function fetchWithTimeout(url: string, timeoutMs: number): Promise<Response> {
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||
try {
|
||
return await fetch(url, {
|
||
signal: controller.signal,
|
||
cache: "no-store",
|
||
headers: {
|
||
"user-agent": "ai-chatbot-smart-analyzer/1.0",
|
||
},
|
||
});
|
||
} finally {
|
||
clearTimeout(timeoutId);
|
||
}
|
||
}
|
||
|
||
async function toAnalyzerDataUrl(imageUrl: string): Promise<string | null> {
|
||
try {
|
||
const response = await fetchWithTimeout(imageUrl, ANALYZER_IMAGE_FETCH_TIMEOUT_MS);
|
||
if (!response.ok) {
|
||
console.warn(
|
||
`[Smart Analyzer] Failed to fetch image for inline payload (${response.status}): ${imageUrl}`,
|
||
);
|
||
return null;
|
||
}
|
||
|
||
const contentType = response.headers.get("content-type") || "";
|
||
if (!contentType.startsWith("image/")) {
|
||
console.warn(`[Smart Analyzer] Skip non-image content-type: ${contentType}`);
|
||
return null;
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
const inputBuffer = Buffer.from(arrayBuffer);
|
||
|
||
const optimized = await sharp(inputBuffer)
|
||
.rotate()
|
||
.resize({
|
||
width: ANALYZER_IMAGE_MAX_SIDE,
|
||
height: ANALYZER_IMAGE_MAX_SIDE,
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
})
|
||
.jpeg({
|
||
quality: 80,
|
||
mozjpeg: true,
|
||
})
|
||
.toBuffer();
|
||
|
||
return `data:image/jpeg;base64,${optimized.toString("base64")}`;
|
||
} catch (error) {
|
||
console.warn("[Smart Analyzer] Failed to convert image URL to inline base64:", error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
async function buildAnalyzerVisionImages(imageUrls?: string[]) {
|
||
const urls = (imageUrls || []).filter((url) => isHttpUrl(url)).slice(0, ANALYZER_MAX_IMAGE_COUNT);
|
||
if (urls.length === 0) return [] as Array<{ type: "image"; image: string }>;
|
||
|
||
const convertedResults = await Promise.all(urls.map((url) => toAnalyzerDataUrl(url)));
|
||
|
||
return urls.map((originalUrl, index) => {
|
||
const inlineDataUrl = convertedResults[index];
|
||
return {
|
||
type: "image" as const,
|
||
image: inlineDataUrl || originalUrl,
|
||
};
|
||
});
|
||
}
|
||
|
||
function parseAnalysisResultFromModelText(
|
||
responseText: string,
|
||
userText: string,
|
||
hasImage: boolean,
|
||
): AnalysisResult | null {
|
||
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
||
if (!jsonMatch) return null;
|
||
|
||
try {
|
||
const result = JSON.parse(jsonMatch[0]) as AnalysisResult;
|
||
return normalizeIntentForImageContext(result, userText, hasImage);
|
||
} catch (error) {
|
||
console.error("[Smart Analyzer] Failed to parse model response:", error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function isLikelyZhipuEndpoint(baseUrl?: string | null): boolean {
|
||
if (!baseUrl) return false;
|
||
return /open\.bigmodel\.cn/i.test(baseUrl);
|
||
}
|
||
|
||
function normalizeZhipuChatEndpoint(baseUrl: string): string {
|
||
const cleaned = baseUrl.replace(/\/+$/, "");
|
||
if (/\/chat\/completions$/i.test(cleaned)) return cleaned;
|
||
if (/\/api\/paas\/v4$/i.test(cleaned)) return `${cleaned}/chat/completions`;
|
||
if (/\/api\/paas\/v4\//i.test(cleaned)) return cleaned;
|
||
if (/open\.bigmodel\.cn/i.test(cleaned)) return `${cleaned}/api/paas/v4/chat/completions`;
|
||
return `${cleaned}/chat/completions`;
|
||
}
|
||
|
||
function toZhipuImageUrl(image: string): string {
|
||
const dataUrlPrefix = /^data:image\/[a-zA-Z0-9.+-]+;base64,/i;
|
||
if (dataUrlPrefix.test(image)) {
|
||
return image.replace(dataUrlPrefix, "");
|
||
}
|
||
return image;
|
||
}
|
||
|
||
async function analyzeWithZhipuOfficialApi(params: {
|
||
apiKey: string;
|
||
baseUrl: string;
|
||
userMessage: string;
|
||
visionImages: Array<{ type: "image"; image: string }>;
|
||
userText: string;
|
||
hasImage: boolean;
|
||
}): Promise<AnalysisResult | null> {
|
||
const endpoint = normalizeZhipuChatEndpoint(params.baseUrl);
|
||
|
||
const content = [
|
||
{ type: "text", text: params.userMessage },
|
||
...params.visionImages.map((imagePart) => ({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: toZhipuImageUrl(imagePart.image),
|
||
},
|
||
})),
|
||
];
|
||
|
||
for (const modelId of ZHIPU_VISION_MODEL_CANDIDATES) {
|
||
try {
|
||
const response = await fetch(endpoint, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${params.apiKey}`,
|
||
},
|
||
body: JSON.stringify({
|
||
model: modelId,
|
||
messages: [
|
||
{
|
||
role: "system",
|
||
content: ANALYZER_SYSTEM_PROMPT,
|
||
},
|
||
{
|
||
role: "user",
|
||
content,
|
||
},
|
||
],
|
||
temperature: 0.1,
|
||
max_tokens: 360,
|
||
}),
|
||
});
|
||
|
||
const payload = await response.json().catch(() => ({}));
|
||
if (!response.ok || payload?.error) {
|
||
const errorCode = String(payload?.error?.code || "");
|
||
const errorMessage = String(payload?.error?.message || "");
|
||
if (errorCode === "1211" || /模型不存在/.test(errorMessage)) {
|
||
continue;
|
||
}
|
||
console.warn(
|
||
`[Smart Analyzer] Zhipu official vision request failed (model=${modelId})`,
|
||
payload?.error || payload,
|
||
);
|
||
continue;
|
||
}
|
||
|
||
const message = payload?.choices?.[0]?.message;
|
||
const responseText = String(message?.content || message?.reasoning_content || "").trim();
|
||
if (!responseText) {
|
||
continue;
|
||
}
|
||
|
||
const parsed = parseAnalysisResultFromModelText(
|
||
responseText,
|
||
params.userText,
|
||
params.hasImage,
|
||
);
|
||
if (parsed) {
|
||
console.log(`[Smart Analyzer] Zhipu official vision model used: ${modelId}`);
|
||
return parsed;
|
||
}
|
||
} catch (error) {
|
||
console.warn(
|
||
`[Smart Analyzer] Zhipu official vision request exception (model=${modelId})`,
|
||
error,
|
||
);
|
||
}
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
async function summarizeWithZhipuOfficialApi(params: {
|
||
apiKey: string;
|
||
baseUrl: string;
|
||
promptText: string;
|
||
visionImages: Array<{ type: "image"; image: string }>;
|
||
}): Promise<string | null> {
|
||
const endpoint = normalizeZhipuChatEndpoint(params.baseUrl);
|
||
const content = [
|
||
{ type: "text", text: params.promptText },
|
||
...params.visionImages.map((imagePart) => ({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: toZhipuImageUrl(imagePart.image),
|
||
},
|
||
})),
|
||
];
|
||
|
||
for (const modelId of ZHIPU_VISION_MODEL_CANDIDATES) {
|
||
try {
|
||
const response = await fetch(endpoint, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${params.apiKey}`,
|
||
},
|
||
body: JSON.stringify({
|
||
model: modelId,
|
||
messages: [
|
||
{
|
||
role: "system",
|
||
content:
|
||
"你是图像观察助手。请根据图片给出执行前说明，输出严格两行中文：第1行“画面：...”，第2行“关键要素：...”。不要输出其他内容。",
|
||
},
|
||
{
|
||
role: "user",
|
||
content,
|
||
},
|
||
],
|
||
temperature: 0.2,
|
||
max_tokens: 140,
|
||
}),
|
||
});
|
||
|
||
const payload = await response.json().catch(() => ({}));
|
||
if (!response.ok || payload?.error) {
|
||
const errorCode = String(payload?.error?.code || "");
|
||
const errorMessage = String(payload?.error?.message || "");
|
||
if (errorCode === "1211" || /模型不存在/.test(errorMessage)) {
|
||
continue;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
const message = payload?.choices?.[0]?.message;
|
||
const responseText = String(message?.content || message?.reasoning_content || "").trim();
|
||
if (responseText) {
|
||
console.log(`[Smart Analyzer] Zhipu official summary model used: ${modelId}`);
|
||
return responseText;
|
||
}
|
||
} catch {
|
||
continue;
|
||
}
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
export async function summarizeImageForExecutionReport(params: {
|
||
text?: string;
|
||
imageUrls?: string[];
|
||
context?: string;
|
||
}): Promise<string | null> {
|
||
const { text, imageUrls, context } = params;
|
||
const visionImages = await buildAnalyzerVisionImages(imageUrls);
|
||
if (visionImages.length === 0) return null;
|
||
|
||
const {
|
||
getEnabledAiModelsWithProvider,
|
||
getAgentAiModelWithProvider,
|
||
} = await import("@/lib/db/queries");
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const agentModel = await getAgentAiModelWithProvider();
|
||
const zhipuProviderFromAgent =
|
||
agentModel?.provider?.apiKey && isLikelyZhipuEndpoint(agentModel.provider.baseUrl)
|
||
? agentModel.provider
|
||
: null;
|
||
const zhipuProviderFallback = allModels.find(
|
||
(m) =>
|
||
m.provider?.apiKey &&
|
||
(m.provider.type === "zhipu" || isLikelyZhipuEndpoint(m.provider.baseUrl)),
|
||
)?.provider;
|
||
const zhipuProvider = zhipuProviderFromAgent || zhipuProviderFallback;
|
||
|
||
const promptText = [
|
||
text ? `用户需求：${text}` : "",
|
||
context ? `画布上下文：${context.slice(0, 220)}` : "",
|
||
"请给出当前画面的客观描述，便于执行前说明。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
|
||
if (zhipuProvider?.apiKey && zhipuProvider?.baseUrl) {
|
||
const summary = await summarizeWithZhipuOfficialApi({
|
||
apiKey: zhipuProvider.apiKey,
|
||
baseUrl: zhipuProvider.baseUrl,
|
||
promptText,
|
||
visionImages,
|
||
});
|
||
if (summary) return summary;
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
// 历史消息类型
|
||
interface HistoryMessage {
|
||
role: "user" | "assistant";
|
||
content: string;
|
||
}
|
||
|
||
/**
|
||
* 分析用户意图
|
||
*/
|
||
export async function analyzeIntent(params: {
|
||
text: string;
|
||
hasImage: boolean;
|
||
imageUrls?: string[];
|
||
imageFromHistory?: boolean; // 🆕 图片是否来自历史记录
|
||
historyMessages?: HistoryMessage[]; // 🆕 历史对话消息（用于上下文理解）
|
||
context?: string; // 🆕 画布上下文（仅用于理解）
|
||
}): Promise<AnalysisResult> {
|
||
const { text, hasImage, imageUrls, imageFromHistory, historyMessages, context } = params;
|
||
|
||
console.log("[Smart Analyzer] Analyzing intent:", {
|
||
text,
|
||
hasImage,
|
||
imageCount: imageUrls?.length,
|
||
imageFromHistory,
|
||
historyCount: historyMessages?.length || 0
|
||
});
|
||
|
||
// 先尝试关键词快速匹配（高置信度场景直接返回，节省LLM调用）
|
||
const quickResult = quickKeywordMatch(text, hasImage);
|
||
if (quickResult.confidence >= 0.9) {
|
||
console.log("[Smart Analyzer] Quick match result:", quickResult);
|
||
return applyDirectExecutionPolicy(quickResult, { text, hasImage, imageUrls, imageFromHistory });
|
||
}
|
||
|
||
// 尝试使用多模态模型进行深度分析（传入历史对话）
|
||
try {
|
||
const llmResult = await analyzWithLLM(text, hasImage, imageUrls, historyMessages, context);
|
||
if (llmResult) {
|
||
console.log("[Smart Analyzer] LLM analysis result:", llmResult);
|
||
return applyDirectExecutionPolicy(llmResult, { text, hasImage, imageUrls, imageFromHistory });
|
||
}
|
||
} catch (error) {
|
||
console.error("[Smart Analyzer] LLM analysis failed:", error);
|
||
}
|
||
|
||
// 回退到完整关键词分析
|
||
const fallbackResult = fallbackKeywordAnalysis(text, hasImage);
|
||
console.log("[Smart Analyzer] Fallback result:", fallbackResult);
|
||
return applyDirectExecutionPolicy(fallbackResult, { text, hasImage, imageUrls, imageFromHistory });
|
||
}
|
||
|
||
/**
|
||
* 快速关键词匹配（高置信度场景）
|
||
* 🆕 现在返回低置信度，强制走LLM分析以获取自然对话回复
|
||
*/
|
||
function quickKeywordMatch(text: string, hasImage: boolean): AnalysisResult {
|
||
// 返回低置信度，强制走LLM分析以获取自然对话回复
|
||
// 只有LLM不可用时才会使用fallback
|
||
return {
|
||
intent: "chat",
|
||
confidence: 0.3,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 使用 LLM 进行深度分析
|
||
*/
|
||
async function analyzWithLLM(
|
||
text: string,
|
||
hasImage: boolean,
|
||
imageUrls?: string[],
|
||
historyMessages?: HistoryMessage[],
|
||
context?: string
|
||
): Promise<AnalysisResult | null> {
|
||
// 使用动态导入避免循环依赖问题
|
||
const {
|
||
getEnabledAiModelsWithProvider,
|
||
getAgentAiModelWithProvider,
|
||
} = await import("@/lib/db/queries");
|
||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||
|
||
// 优先规则：
|
||
// 1) 若 Agent 模型被标记为「多模态」，且当前请求包含图片，则识图+分析统一走该 Agent 模型
|
||
// 2) 否则按既有视觉模型优先级选择
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const agentModel = await getAgentAiModelWithProvider();
|
||
const isReasoningCompatibleProvider = (providerType: string) =>
|
||
["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType);
|
||
const isLikelyZhipuVisionModel = (displayName: string, modelId: string, providerType: string) =>
|
||
providerType === "zhipu" ||
|
||
/(glm[-_.]?4v|glm[-_.]?4\.?1v|glm[-_.]?4\.?5v|vision)/i.test(
|
||
`${displayName} ${modelId}`,
|
||
);
|
||
const zhipuVisionModel = allModels.find(
|
||
(m) =>
|
||
m.model.supportsVision &&
|
||
m.model.isEnabled &&
|
||
!m.model.isSubModel &&
|
||
isLikelyZhipuVisionModel(m.model.displayName, m.model.modelId, m.provider.type),
|
||
);
|
||
const qwenVisionModel = allModels.find(
|
||
(m) =>
|
||
m.provider.type === "alibaba" &&
|
||
m.model.supportsVision &&
|
||
m.model.isEnabled &&
|
||
!m.model.isSubModel &&
|
||
/(qwen|通义|vl)/i.test(`${m.model.displayName} ${m.model.modelId}`)
|
||
);
|
||
const fallbackVisionModel =
|
||
qwenVisionModel ||
|
||
allModels.find(
|
||
(m) =>
|
||
m.provider.type === "alibaba" &&
|
||
m.model.supportsVision &&
|
||
m.model.isEnabled &&
|
||
!m.model.isSubModel,
|
||
);
|
||
const preferredVisionModel = zhipuVisionModel || fallbackVisionModel;
|
||
const usableAgentModel =
|
||
agentModel && isReasoningCompatibleProvider(agentModel.provider.type)
|
||
? agentModel
|
||
: undefined;
|
||
const multimodalAgentVisionModel =
|
||
agentModel &&
|
||
agentModel.model.isEnabled &&
|
||
!agentModel.model.isSubModel &&
|
||
agentModel.model.supportsVision &&
|
||
isAiModelMultimodal(agentModel.model)
|
||
? agentModel
|
||
: undefined;
|
||
const selectedAnalyzerModel = hasImage
|
||
? multimodalAgentVisionModel || preferredVisionModel
|
||
: usableAgentModel ||
|
||
allModels.find((m) => m.provider.type === "zhipu" && !m.model.isSubModel) ||
|
||
preferredVisionModel;
|
||
|
||
if (!selectedAnalyzerModel) {
|
||
console.log("[Smart Analyzer] No available analyzer model found");
|
||
return null;
|
||
}
|
||
|
||
const model = await getLanguageModelById(selectedAnalyzerModel.model.id);
|
||
if (!model) {
|
||
console.log("[Smart Analyzer] Failed to get language model");
|
||
return null;
|
||
}
|
||
|
||
// 🆕 构建包含历史上下文的消息
|
||
let contextMessage = "";
|
||
|
||
// 🆕 画布上下文（仅用于理解，不要复述）
|
||
if (context && context.trim().length > 0) {
|
||
contextMessage = `## 画布上下文（仅用于理解，不要在对话回复中复述）：\n${context.trim()}\n\n`;
|
||
}
|
||
|
||
// 添加历史对话上下文（最多取最近5轮对话）
|
||
if (historyMessages && historyMessages.length > 0) {
|
||
const recentHistory = historyMessages.slice(-10); // 最多10条消息（约5轮对话）
|
||
const historyText = recentHistory.map(msg =>
|
||
`${msg.role === 'user' ? '用户' : 'AI'}: ${msg.content.substring(0, 200)}`
|
||
).join('\n');
|
||
|
||
contextMessage += `## 对话历史（最近的对话上下文）：
|
||
${historyText}
|
||
|
||
## 当前请求：
|
||
`;
|
||
}
|
||
|
||
// 构建用户消息
|
||
const userMessage = hasImage
|
||
? `${contextMessage}用户上传了图片，请求内容："${text}"`
|
||
: `${contextMessage}用户请求（无图片）："${text}"`;
|
||
|
||
const visionImages = hasImage ? await buildAnalyzerVisionImages(imageUrls) : [];
|
||
if (visionImages.length > 0) {
|
||
const inlineCount = visionImages.filter((item) => item.image.startsWith("data:image/")).length;
|
||
console.log(
|
||
`[Smart Analyzer] Prepared analyzer images: total=${visionImages.length}, inline=${inlineCount}, remote=${visionImages.length - inlineCount}`,
|
||
);
|
||
}
|
||
|
||
const zhipuProviderFromAgent =
|
||
usableAgentModel &&
|
||
usableAgentModel.provider &&
|
||
usableAgentModel.provider.apiKey &&
|
||
isLikelyZhipuEndpoint(usableAgentModel.provider.baseUrl)
|
||
? usableAgentModel.provider
|
||
: null;
|
||
const zhipuProviderFallback = allModels.find(
|
||
(m) =>
|
||
m.provider?.apiKey &&
|
||
(m.provider.type === "zhipu" || isLikelyZhipuEndpoint(m.provider.baseUrl)),
|
||
)?.provider;
|
||
const zhipuProvider = zhipuProviderFromAgent || zhipuProviderFallback;
|
||
const forceSingleMultimodalAgent = Boolean(hasImage && multimodalAgentVisionModel);
|
||
|
||
if (
|
||
!forceSingleMultimodalAgent &&
|
||
hasImage &&
|
||
visionImages.length > 0 &&
|
||
zhipuProvider?.apiKey &&
|
||
zhipuProvider?.baseUrl
|
||
) {
|
||
const zhipuOfficialResult = await analyzeWithZhipuOfficialApi({
|
||
apiKey: zhipuProvider.apiKey,
|
||
baseUrl: zhipuProvider.baseUrl,
|
||
userMessage,
|
||
visionImages,
|
||
userText: text,
|
||
hasImage,
|
||
});
|
||
if (zhipuOfficialResult) {
|
||
return zhipuOfficialResult;
|
||
}
|
||
}
|
||
|
||
console.log(
|
||
"[Smart Analyzer] Calling analyzer model:",
|
||
selectedAnalyzerModel.model.displayName,
|
||
`(${selectedAnalyzerModel.provider.type})`,
|
||
"history=",
|
||
historyMessages?.length || 0
|
||
);
|
||
|
||
// 调用模型分析（有图时携带图片，确保真实视觉分析）
|
||
const requestPayload = hasImage && visionImages.length > 0
|
||
? {
|
||
model,
|
||
system: ANALYZER_SYSTEM_PROMPT,
|
||
messages: [
|
||
{
|
||
role: "user" as const,
|
||
content: [
|
||
{ type: "text" as const, text: userMessage },
|
||
...visionImages,
|
||
],
|
||
},
|
||
],
|
||
temperature: 0.1,
|
||
maxTokens: 360,
|
||
}
|
||
: {
|
||
model,
|
||
system: ANALYZER_SYSTEM_PROMPT,
|
||
prompt: userMessage,
|
||
temperature: 0.1,
|
||
maxTokens: 300,
|
||
};
|
||
|
||
const { text: responseText } = await generateText(requestPayload as any);
|
||
|
||
return parseAnalysisResultFromModelText(responseText, text, hasImage);
|
||
}
|
||
|
||
/**
|
||
* 回退方案：完整关键词匹配（当LLM不可用时使用）
|
||
* 🆕 添加了自然对话回复
|
||
*/
|
||
export function fallbackKeywordAnalysis(
|
||
text: string,
|
||
hasImage: boolean
|
||
): AnalysisResult {
|
||
const lowerText = text.toLowerCase();
|
||
|
||
// === 图片编辑场景（需要有图片） ===
|
||
if (hasImage) {
|
||
// 视频生成（图生视频/首尾帧等）
|
||
if (isVideoGenerationRequest(lowerText)) {
|
||
return {
|
||
intent: "video_gen",
|
||
confidence: 0.9,
|
||
conversationalReply: "收到，我先按你提供的图片做视频需求分析和分镜规划，再自动优化提示词执行生成~",
|
||
};
|
||
}
|
||
|
||
// 抠图/去背景
|
||
if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "remove_background",
|
||
confidence: 0.9,
|
||
conversationalReply: "好的，我来帮你抠图，去除背景保留主体~",
|
||
};
|
||
}
|
||
|
||
// 换背景
|
||
if (/换背景|白底|改背景|背景换成|纯色背景|换成.*背景|替换.*背景/.test(lowerText)) {
|
||
// 提取背景颜色
|
||
const bgMatch = text.match(/(?:换成|改成|替换为|变成)(.+?)(?:背景|底|$)/);
|
||
const bg = bgMatch ? bgMatch[1].trim() : "白色";
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "replace_background",
|
||
confidence: 0.9,
|
||
conversationalReply: `好的，我来帮你把背景换成${bg}~`,
|
||
};
|
||
}
|
||
|
||
// 图片增强
|
||
if (/变清晰|高清|锐化|降噪|提升质量|修复|清晰化|变清楚|画质/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "enhance",
|
||
confidence: 0.9,
|
||
conversationalReply: "收到，我来帮你提升图片清晰度~",
|
||
};
|
||
}
|
||
|
||
// 图像拓展
|
||
if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "extend",
|
||
confidence: 0.9,
|
||
conversationalReply: "好的，我来帮你扩展图片边界~",
|
||
};
|
||
}
|
||
|
||
// 局部修改/去除
|
||
if (/去除|消除|擦掉|P掉|p掉|删除|修改.*部分|移除.*(?:人|物|水印|logo|文字)/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "inpaint",
|
||
confidence: 0.9,
|
||
conversationalReply: "明白，我来帮你处理这个区域~",
|
||
};
|
||
}
|
||
|
||
// 风格迁移
|
||
if (/风格|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格/.test(lowerText)) {
|
||
const styleMatch = text.match(/(?:转换为|变成|改成|转成)(.+?)(?:风格|$)/);
|
||
const style = styleMatch ? styleMatch[1].trim() : "新";
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "style_transfer",
|
||
confidence: 0.9,
|
||
conversationalReply: `好的，我来帮你转换成${style}风格~`,
|
||
};
|
||
}
|
||
|
||
// 调色/改颜色
|
||
if (/调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成/.test(lowerText)) {
|
||
// 提取目标颜色
|
||
const colorMatch = text.match(/(?:改成|换成|变成)(.+?)(?:色|$)/);
|
||
const color = colorMatch ? colorMatch[1].trim() + "色" : "";
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "color_adjust",
|
||
confidence: 0.9,
|
||
conversationalReply: color ? `好的，我来帮你把颜色改成${color}~` : "收到，我来帮你调整图片色彩~",
|
||
};
|
||
}
|
||
|
||
// 尺寸调整
|
||
if (/缩放|放大|缩小|调整尺寸|裁剪|resize|crop/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: "resize",
|
||
confidence: 0.9,
|
||
conversationalReply: "好的，我来帮你调整图片尺寸~",
|
||
};
|
||
}
|
||
|
||
// 通用编辑关键词
|
||
if (/修图|P图|改图|调图|处理.*图|编辑.*图|优化.*图/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: inferDefaultImageEditSubIntent(text),
|
||
confidence: 0.8,
|
||
needsClarification: false,
|
||
conversationalReply: "收到，我先按你的需求直接执行图片优化。",
|
||
};
|
||
}
|
||
|
||
// 视觉理解
|
||
if (/看|识别|分析|描述|这是什么|有什么|tell me|what is|看看|帮我看/.test(lowerText)) {
|
||
return {
|
||
intent: "vision",
|
||
confidence: 0.85,
|
||
conversationalReply: "让我来看看这张图片~",
|
||
};
|
||
}
|
||
|
||
// 有图但意图不明确
|
||
return {
|
||
intent: "image_edit",
|
||
subIntent: inferDefaultImageEditSubIntent(text),
|
||
confidence: 0.75,
|
||
needsClarification: false,
|
||
conversationalReply: "收到图片，我先按保真优化策略直接执行。",
|
||
};
|
||
}
|
||
|
||
// === 无图片场景 ===
|
||
|
||
// 图片生成
|
||
if (/画|绘制|生成图|创建.*图|设计.*图|画一个|生成一张|做一张/.test(lowerText)) {
|
||
return {
|
||
intent: "image_gen",
|
||
confidence: 0.85,
|
||
conversationalReply: "好的，我来帮你生成这张图片~",
|
||
};
|
||
}
|
||
|
||
// 视频生成
|
||
if (isVideoGenerationRequest(lowerText)) {
|
||
return {
|
||
intent: "video_gen",
|
||
confidence: 0.85,
|
||
conversationalReply: "好的，我先做视频需求分析并自动优化提示词后执行生成~",
|
||
};
|
||
}
|
||
|
||
// 需要图片但没有上传
|
||
if (/抠图|去背景|换背景|修图|P图|编辑图/.test(lowerText)) {
|
||
return {
|
||
intent: "image_edit",
|
||
confidence: 0.7,
|
||
needsClarification: true,
|
||
conversationalReply: "你想编辑图片的话，需要先上传一张图片哦~",
|
||
};
|
||
}
|
||
|
||
// 默认对话
|
||
return { intent: "chat", confidence: 0.7 };
|
||
}
|