diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 413e28b..aa0ed56 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -331,6 +331,10 @@ export async function POST(request: Request) { let finalModelId = selectedChatModel; + // 🆕 存储智能模式的相关信息 + let smartAutoFriendlyMessage: string | undefined; // AI的自然对话回复 + let smartAutoOptimizedPrompt: string | undefined; // 优化后的提示词（不修改用户消息，只用于API调用） + // ========== Smart Auto Mode Logic ========== if (finalModelId === "auto") { console.log("[Smart Auto Mode] 🚀 Starting intelligent analysis..."); @@ -370,6 +374,12 @@ export async function POST(request: Request) { needsClarification: analysisResult.needsClarification, }); + // 🆕 保存AI的自然对话回复（来自千问3 VL的理解） + if (analysisResult.conversationalReply) { + smartAutoFriendlyMessage = analysisResult.conversationalReply; + console.log("[Smart Auto Mode] 💬 Conversational reply:", smartAutoFriendlyMessage); + } + // 3. 如果是图片编辑类意图，优化提示词 if (analysisResult.intent === "image_edit" && analysisResult.subIntent) { const { optimizePrompt } = await import("@/lib/ai/prompt-optimizer"); @@ -384,16 +394,9 @@ export async function POST(request: Request) { optimized: optimizedResult.optimizedPrompt.substring(0, 100), }); - // 替换消息中的文本为优化后的提示词 - message.parts = message.parts.map((part) => { - if (part.type === "text") { - return { - ...part, - text: optimizedResult.optimizedPrompt, - }; - } - return part; - }); + // 🆕 保存优化后的提示词，但不修改用户消息 + // 用户消息保持原样显示，优化后的提示词只用于API调用 + smartAutoOptimizedPrompt = optimizedResult.optimizedPrompt; } // 4. 路由到最佳模型 @@ -923,9 +926,23 @@ export async function POST(request: Request) { try { // 只传递最后一条用户消息 const lastUserMessage = uiMessages.filter((m) => m.role === "user").pop(); - const messagesForAlibaba = lastUserMessage ? [lastUserMessage] : uiMessages; + let messagesForAlibaba = lastUserMessage ? [lastUserMessage] : uiMessages; console.log("[Chat API] Alibaba: Using only last user message"); + // 🆕 如果有智能模式优化后的提示词，替换消息中的文本（仅用于API调用） + if (smartAutoOptimizedPrompt && messagesForAlibaba.length > 0) { + console.log("[Chat API] Using optimized prompt for API call"); + messagesForAlibaba = messagesForAlibaba.map((msg) => ({ + ...msg, + parts: msg.parts.map((part) => { + if (part.type === "text") { + return { ...part, text: smartAutoOptimizedPrompt! }; + } + return part; + }), + })); + } + // 转换消息（阿里生图支持 URL，会自动上传 base64） const modelMessages = await convertMessagesWithImages(messagesForAlibaba, "alibaba-image"); @@ -939,6 +956,13 @@ export async function POST(request: Request) { const imageMarkdown = result.text || "图片生成失败"; console.log("[Chat API] Alibaba generated, length:", imageMarkdown.length); + // 🆕 如果是智能模式，在图片前加上友好消息 + let finalMessage = imageMarkdown; + if (smartAutoFriendlyMessage) { + finalMessage = `${smartAutoFriendlyMessage}\n\n${imageMarkdown}`; + console.log("[Chat API] Added friendly message:", smartAutoFriendlyMessage); + } + // 生成消息 ID const messageId = generateUUID(); @@ -948,7 +972,7 @@ export async function POST(request: Request) { data: JSON.stringify({ id: messageId, role: "assistant", - parts: [{ type: "text", text: imageMarkdown }], + parts: [{ type: "text", text: finalMessage }], }), }); @@ -960,7 +984,7 @@ export async function POST(request: Request) { id: messageId, chatId: id, role: "assistant", - parts: [{ type: "text", text: imageMarkdown }], + parts: [{ type: "text", text: finalMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, diff --git a/lib/ai/prompt-optimizer.ts b/lib/ai/prompt-optimizer.ts index abe8370..55ca4d2 100644 --- a/lib/ai/prompt-optimizer.ts +++ b/lib/ai/prompt-optimizer.ts @@ -10,8 +10,45 @@ export interface OptimizeResult { originalPrompt: string; optimizedPrompt: string; negativePrompt: string; + friendlyMessage?: string; // 友好的对话消息 } +// 子意图的友好名称和回复模板 +const FRIENDLY_MESSAGES: Record = { + remove_background: { + name: "抠图", + template: "好的，我来帮你抠图，去除背景保留主体~", + }, + replace_background: { + name: "换背景", + template: "收到，我来帮你把背景换成{background}~", + }, + enhance: { + name: "图片增强", + template: "明白，我来帮你提升图片清晰度~", + }, + extend: { + name: "图像拓展", + template: "好的，我来帮你扩展图片边界~", + }, + inpaint: { + name: "局部修改", + template: "收到，我来帮你{action}{target}~", + }, + style_transfer: { + name: "风格迁移", + template: "好的，我来帮你转换成{style}风格~", + }, + resize: { + name: "尺寸调整", + template: "明白，我来帮你调整图片尺寸~", + }, + color_adjust: { + name: "调色", + template: "收到，我来帮你调整图片颜色~", + }, +}; + // 图片编辑子意图的提示词模板 const EDIT_TEMPLATES: Record< string, @@ -188,10 +225,34 @@ export function optimizePrompt(params: { optimizedPrompt = `${optimizedPrompt}\n\n用户补充要求：${originalPrompt}`; } + // 生成友好的对话消息 + let friendlyMessage = ""; + const friendlyTemplate = FRIENDLY_MESSAGES[subIntent]; + if (friendlyTemplate) { + friendlyMessage = friendlyTemplate.template; + + // 填充友好消息中的变量 + for (const [key, regex] of Object.entries(template.extractors)) { + const match = originalPrompt.match(regex); + let value = match ? match[1].trim() : template.defaults[key] || ""; + value = value.replace(/[，。！？,.!?]/g, "").trim(); + if (!value) { + value = template.defaults[key] || ""; + } + friendlyMessage = friendlyMessage.replace(`{${key}}`, value); + } + + // 填充剩余的默认值 + for (const [key, defaultValue] of Object.entries(template.defaults)) { + friendlyMessage = friendlyMessage.replace(`{${key}}`, defaultValue); + } + } + const result: OptimizeResult = { originalPrompt, optimizedPrompt, negativePrompt: template.negative, + friendlyMessage: friendlyMessage || `好的，我来帮你处理这张图片~`, }; console.log("[Prompt Optimizer] Output:", result); diff --git a/lib/ai/smart-analyzer.ts b/lib/ai/smart-analyzer.ts index f809c15..1158dec 100644 --- a/lib/ai/smart-analyzer.ts +++ b/lib/ai/smart-analyzer.ts @@ -19,49 +19,74 @@ export interface AnalysisResult { suggestedModel?: string; needsClarification?: boolean; clarifyingQuestions?: string[]; + conversationalReply?: string; // 🆕 自然对话回复 } -// 系统提示词 - 针对图片编辑场景优化 -const ANALYZER_SYSTEM_PROMPT = `你是一个专业的图片编辑意图分析专家。请分析用户的请求，判断其意图类型。 +// 系统提示词 - 针对图片编辑场景优化，生成自然对话 +const ANALYZER_SYSTEM_PROMPT = `你是一个友好的AI图片编辑助手。用户会给你发送图片和编辑请求，你需要： +1. 理解用户想要对图片做什么 +2. 用自然、友好的语气回复用户，告诉他你理解了他的需求并会帮他处理 +3. 同时输出结构化的意图分析结果 -## 意图分类（按优先级） +## 意图分类 -### 1. image_edit - 图片编辑（最高优先级，需要用户上传了图片） +### image_edit - 图片编辑（用户上传了图片并想编辑） 子意图： -- remove_background: 抠图、去背景、透明背景、去除背景、扣图 -- replace_background: 换背景、改背景、白底、场景替换、背景换成、纯色背景 -- enhance: 变清晰、高清、锐化、降噪、提升质量、修复、清晰化 -- extend: 扩展、拓展、延伸、outpainting、扩图、拓图 -- inpaint: 去除XX、消除、擦掉、修改XX部分、局部修改、P掉、删除某物 -- style_transfer: 风格化、卡通化、动漫风、油画风、改风格、转风格 -- resize: 缩放、放大、缩小、调整尺寸、裁剪 -- color_adjust: 调色、调整颜色、亮度、对比度、饱和度、色调 +- remove_background: 抠图、去背景、透明背景 +- replace_background: 换背景、改背景、白底 +- enhance: 变清晰、高清、锐化、降噪 +- extend: 扩展、拓展、延伸画布 +- inpaint: 去除某物、消除、擦掉、P掉、局部修改 +- style_transfer: 风格化、卡通化、动漫风、改风格 +- color_adjust: 调色、调整颜色、亮度、对比度 -### 2. vision - 图片理解（需要用户上传了图片） -触发词：看、识别、分析、描述、这是什么、有什么、tell me、what is +### vision - 图片理解（用户想让你看图说话） +### image_gen - 图片生成（用户想生成新图片） +### video_gen - 视频生成 +### chat - 普通对话 -### 3. image_gen - 图片生成（不需要上传图片） -触发词：画、绘制、生成图片、创建、设计、画一个、生成一张 - -### 4. video_gen - 视频生成 -触发词：视频、动画、动起来、让它动 - -### 5. chat - 普通对话（默认） - -## 输出格式（严格JSON，不要有其他内容） +## 输出格式（严格JSON） { "intent": "image_edit", - "subIntent": "replace_background", + "subIntent": "replace_background", "confidence": 0.95, + "conversationalReply": "好的，我来帮你把背景换成白色！这样产品会更突出~", "needsClarification": false, "clarifyingQuestions": [] } -## 特殊规则 -1. 如果用户上传了图片且有编辑类关键词 → image_edit -2. 如果用户上传了图片但意图不明确 → needsClarification: true，并提供澄清问题 -3. 如果用户要求编辑但没上传图片 → needsClarification: true，提示"请上传图片" -4. 只输出JSON，不要有任何解释文字`; +## 回复风格要求 +- conversationalReply 要自然、友好、简洁 +- 可以加一些语气词让对话更亲切，如"好的"、"没问题"、"~"等 +- 说明你理解了什么需求，让用户知道你在做什么 +- 如果意图不明确，礼貌地询问用户想要什么效果 + +## 示例 +用户："帮我把背景换成白色" +{ + "intent": "image_edit", + "subIntent": "replace_background", + "confidence": 0.95, + "conversationalReply": "好的，我来帮你把背景换成纯白色，这样图片会更干净专业~", + "needsClarification": false +} + +用户："把这张图变清晰一点" +{ + "intent": "image_edit", + "subIntent": "enhance", + "confidence": 0.9, + "conversationalReply": "收到！我来帮你提升图片清晰度，稍等一下~", + "needsClarification": false +} + +用户发了图片但没说要做什么： +{ + "intent": "image_edit", + "confidence": 0.5, + "conversationalReply": "我看到你发了一张图片，你想对它做什么处理呢？比如抠图、换背景、变清晰？", + "needsClarification": true +}`; /** * 分析用户意图 @@ -101,57 +126,11 @@ export async function analyzeIntent(params: { /** * 快速关键词匹配（高置信度场景） + * 🆕 现在返回低置信度，强制走LLM分析以获取自然对话回复 */ function quickKeywordMatch(text: string, hasImage: boolean): AnalysisResult { - const lowerText = text.toLowerCase(); - - if (hasImage) { - // 抠图/去背景 - 高频场景 - if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) { - return { - intent: "image_edit", - subIntent: "remove_background", - confidence: 0.95, - }; - } - - // 换背景 - 高频场景 - if (/换背景|白底|改背景|背景换成|纯色背景|白色背景/.test(lowerText)) { - return { - intent: "image_edit", - subIntent: "replace_background", - confidence: 0.95, - }; - } - - // 图片增强 - 高频场景 - if (/变清晰|高清|锐化|降噪|清晰一点|更清晰|提升质量/.test(lowerText)) { - return { - intent: "image_edit", - subIntent: "enhance", - confidence: 0.95, - }; - } - - // P图/去除某物 - if (/P掉|p掉|去除|消除|擦掉|删除.*(?:人|物|水印|logo)/.test(lowerText)) { - return { - intent: "image_edit", - subIntent: "inpaint", - confidence: 0.95, - }; - } - } - - // 视频生成 - if (/生成视频|做视频|视频生成|动起来|让.{0,3}动/.test(lowerText)) { - return { - intent: "video_gen", - confidence: 0.95, - }; - } - - // 返回低置信度结果，触发LLM分析 + // 返回低置信度，强制走LLM分析以获取自然对话回复 + // 只有LLM不可用时才会使用fallback return { intent: "chat", confidence: 0.3, @@ -210,7 +189,8 @@ async function analyzWithLLM(text: string, hasImage: boolean): Promise { doGenerate: (options: any) => Promise<{ text: string; finishReason: \"stop\"; usage: { promptTokens: number; completionTokens: number; }; }>; doStream: (options: any) => AsyncGenerator<...>; modelId: string; provider: string; specificationVersion: \"v1\"; defaultObjectGenerationMode: \"json...'."},{"start":3260,"length":10,"code":2339,"category":1,"messageText":"Property 'doGenerate' does not exist on type '{ provider: string; chat: (modelId: string) => { provider: string; doGenerate: (options: { messages: { role: string; content: string | { type: string; text?: string | undefined; image?: string | undefined; }[]; }[]; imageParams?: (AlibabaImageParams & { ...; }) | undefined; modelDisplayName?: string | undefined; }) ...'."}]],[2789,[{"start":783,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":1313,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":1915,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":2207,"length":22,"messageText":"'secondAssistantMessage' is possibly 'null'.","category":1,"code":18047}]],[2790,[{"start":523,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":849,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":1194,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":2311,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":2624,"length":23,"messageText":"'updatedAssistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":3557,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":3862,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":4192,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":4491,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":4787,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":4862,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":5305,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047}]],[2791,[{"start":579,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":646,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":1024,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":1138,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":1249,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":1624,"length":16,"messageText":"'assistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":2044,"length":23,"messageText":"'updatedAssistantMessage' is possibly 'null'.","category":1,"code":18047},{"start":2119,"length":23,"messageText":"'updatedAssistantMessage' is possibly 'null'.","category":1,"code":18047}]],[2793,[{"start":658,"length":7,"code":2322,"category":1,"messageText":{"messageText":"Type 'Request | null' is not assignable to type 'Request'.","category":1,"code":2322,"next":[{"messageText":"Type 'null' is not assignable to type 'Request'.","category":1,"code":2322}]}},{"start":1957,"length":7,"code":2322,"category":1,"messageText":{"messageText":"Type 'Request | null' is not assignable to type 'Request'.","category":1,"code":2322,"next":[{"messageText":"Type 'null' is not assignable to type 'Request'.","category":1,"code":2322}]}}]],[2817,[{"start":127,"length":11,"code":7016,"category":1,"messageText":{"messageText":"Could not find a declaration file for module 'matter-js'. '/Users/lanmu/Desktop/ai-chatbot-main/node_modules/.pnpm/matter-js@0.20.0/node_modules/matter-js/build/matter.js' implicitly has an 'any' type.","category":1,"code":7016,"next":[{"info":{"moduleReference":"matter-js","mode":99}}]}}]],[2839,[{"start":7908,"length":11,"code":2353,"category":1,"messageText":"Object literal may only specify known properties, and 'attachments' does not exist in type 'Omit & { id?: string | undefined; role?: \"system\" | \"user\" | \"assistant\" | undefined; } & { text?: undefined; files?: undefined; messageId?: string | undefined; }'."},{"start":14496,"length":11,"code":2339,"category":1,"messageText":"Property 'attachments' does not exist on type 'ChatMessage'."},{"start":14519,"length":11,"code":2339,"category":1,"messageText":"Property 'attachments' does not exist on type 'ChatMessage'."},{"start":14642,"length":11,"code":2339,"category":1,"messageText":"Property 'attachments' does not exist on type 'ChatMessage'."}]],[2841,[{"start":5057,"length":11,"code":2353,"category":1,"messageText":"Object literal may only specify known properties, and 'connections' does not exist in type 'CanvasState'."}]],[3370,[{"start":6163,"length":45,"messageText":"Cannot find module '../../app/(chat)/api/detect-object/route.js' or its corresponding type declarations.","category":1,"code":2307}]]],"affectedFilesPendingEmit":[3370,2807,2809,2810,2642,2643,2644,2640,2641,2818,2819,2622,2663,2664,2662,2658,2665,2666,2667,2668,2669,2670,2834,2835,2836,2837,2822,2833,2672,2673,2675,2674,2676,2677,2678,2679,2681,2680,2683,2694,2698,2695,2696,2697,2700,2701,2702,2699,2703,2705,2706,2708,2707,2709,2712,2713,2714,2715,2716,2718,2717,2719,2720,2721,2722,2723,2724,2838,2841,2804,2486,1908,1846,1911,1917,1847,2487,1848,3316,2474,2817,2821,2489,2490,2609,2636,2814,2839,3319,2840,2747,2748,2744,2745,2746,2824,2831,1334,1906,1907,2832,2493,2187,2495,2188,2494,2502,3320,2500,2624,3324,3325,3364,1856,2497,2625,2508,2480,3365,2626,3366,2501,3367,2801,2825,2829,1857,2477,1910,2828,2631,2478,2607,2503,2506,2509,2608,2830,3317,2632,2505,2826,2472,2743,2476,1916,2618,2619,2823,2820,3318,2815,2627,2481,2485,2800,2816,2634,2806,2612,2496,2471,1905,2808,3362,2498,2217,2610,3363,2216,2219,2630,2623,2475,2470,2613,2614,2617,2615,2218,2628,2473,2504,2488,2635,2629,2805,2479,2621,2827,2606,843,847,1335,2749,2620,2492,2616,2750,2491,1010,2773,1845,2653,2661,2774,2779,2652,2660,1351,1355,1844,1841,1356,1842,2655,1843,2659,2657,1850,2656,1851,1852,1853,2682,1849,1354,2782,2637,2781,2638,840,842,2484,2186,2483,3368,2482,1338,2711,2671,2783,1044,1045,1046,1840,2710,1854,839,1855,1011,1019,1047,2784,2742,2741,2789,2790,2791,2793,2787,2786,2788,2792,2785,2777,2794,2778,2795,2796,2654,2740],"version":"5.8.2"} \ No newline at end of file