From 48deda49f7cfef6fb7543820738244639e262782 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 8 Feb 2026 11:25:50 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=87=AA=E5=8A=A8=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F=E9=9C=80=E6=B1=82=E6=94=B6=E6=95=9B=E4=B8=8E=E6=BE=84?= =?UTF-8?q?=E6=B8=85=E6=B5=81=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/(chat)/api/chat/route.ts | 106 +++++++++++++++++++++++++++++++---- lib/ai/auto-agent-graph.ts | 58 ++++++++++++++++++- lib/ai/prompt-optimizer.ts | 83 ++++++++++++++++++++------- 3 files changed, 215 insertions(+), 32 deletions(-) diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index af6a54a..60879a0 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -124,6 +124,39 @@ function applyHiddenPromptToMessages(messages: ChatMessage[], hiddenPrompt?: str return cloned; } +function applyOptimizedPromptToLatestUserMessage( + messages: ChatMessage[], + optimizedPrompt?: string, + hiddenPromptPrefix = "", +): ChatMessage[] { + const prompt = optimizedPrompt?.trim(); + if (!prompt) return messages; + + const cloned = messages.map((msg) => ({ + ...msg, + parts: msg.parts.map((part) => ({ ...part })), + })); + + for (let i = cloned.length - 1; i >= 0; i -= 1) { + const msg = cloned[i]; + if (msg.role !== "user") continue; + + const textPart = msg.parts.find( + (part): part is { type: "text"; text: string } => part.type === "text", + ); + const finalPrompt = `${hiddenPromptPrefix}${prompt}`.trim(); + + if (textPart) { + textPart.text = finalPrompt; + } else { + msg.parts.unshift({ type: "text", text: finalPrompt }); + } + break; + } + + return cloned; +} + function guessImageMediaType(url: string): string { const cleanUrl = url.split("?")[0]?.toLowerCase() || ""; if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "image/jpeg"; @@ -453,6 +486,8 @@ export async function POST(request: Request) { let smartAutoFriendlyMessage: string | undefined; // AI的自然对话回复 let smartAutoOptimizedPrompt: string | undefined; // 优化后的提示词（不修改用户消息，只用于API调用） let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL（支持多张） + let smartAutoNeedsClarification = false; // 是否需要先追问用户 + let smartAutoClarificationQuestion: string | undefined; // ========== Smart Auto Mode Logic ========== if (finalModelId === "auto") { @@ -579,6 +614,9 @@ export async function POST(request: Request) { confidence: autoGraphResult.analysis.confidence, needsClarification: autoGraphResult.analysis.needsClarification, }); + + smartAutoNeedsClarification = Boolean(autoGraphResult.analysis.needsClarification); + smartAutoClarificationQuestion = autoGraphResult.analysis.clarifyingQuestions?.[0]; } if (autoGraphResult.friendlyMessage) { @@ -744,6 +782,47 @@ export async function POST(request: Request) { const streamId = generateUUID(); await createStreamId({ streamId, chatId: id }); + if (selectedChatModel === "auto" && smartAutoNeedsClarification) { + const clarificationMessage = + smartAutoClarificationQuestion || + smartAutoFriendlyMessage || + "我先确认一下，你希望我优先处理哪一部分？"; + + const clarificationStream = createUIMessageStream({ + execute: async ({ writer }) => { + const messageId = generateUUID(); + writer.write({ + type: "data-appendMessage", + data: JSON.stringify({ + id: messageId, + role: "assistant", + parts: [{ type: "text", text: clarificationMessage }], + }), + }); + + await saveMessages({ + messages: [ + { + id: messageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: clarificationMessage }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + }, + }); + + return new Response(clarificationStream.pipeThrough(new JsonToSseTransformStream())); + } + let finalMergedUsage: AppUsage | undefined; // 获取启用的工具列表 @@ -927,6 +1006,11 @@ export async function POST(request: Request) { // 只传递最后一条用户消息，避免上下文历史干扰生图效果 const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop(); let messagesForGemini = lastUserMessage ? [lastUserMessage] : uiMessagesForModel; + messagesForGemini = applyOptimizedPromptToLatestUserMessage( + messagesForGemini, + smartAutoOptimizedPrompt, + hiddenPromptPrefix, + ); messagesForGemini = injectHistoryImagesToLastUserMessage(messagesForGemini, smartAutoHistoryImages); console.log("[Chat API] Gemini: Using only last user message (no history context)"); @@ -1284,17 +1368,13 @@ export async function POST(request: Request) { console.log("[Chat API] Alibaba: Using only last user message"); // 🆕 如果有智能模式优化后的提示词，替换消息中的文本（仅用于API调用） - if (smartAutoOptimizedPrompt && messagesForAlibaba.length > 0) { + if (smartAutoOptimizedPrompt) { console.log("[Chat API] Using optimized prompt for API call"); - messagesForAlibaba = messagesForAlibaba.map((msg) => ({ - ...msg, - parts: msg.parts.map((part) => { - if (part.type === "text") { - return { ...part, text: `${hiddenPromptPrefix}${smartAutoOptimizedPrompt!}`.trim() }; - } - return part; - }), - })); + messagesForAlibaba = applyOptimizedPromptToLatestUserMessage( + messagesForAlibaba, + smartAutoOptimizedPrompt, + hiddenPromptPrefix, + ); } messagesForAlibaba = injectHistoryImagesToLastUserMessage(messagesForAlibaba, smartAutoHistoryImages); @@ -1479,7 +1559,11 @@ export async function POST(request: Request) { // 🆕 检测是否有标注图片，如果有则在消息中注入系统提示 const hasAnnotation = hasAnnotatedImage(uiMessages); - let messagesForVolcano = uiMessagesForModel; + let messagesForVolcano = applyOptimizedPromptToLatestUserMessage( + uiMessagesForModel, + smartAutoOptimizedPrompt, + hiddenPromptPrefix, + ); const volcanoModelConfig = await getAiModelById({ id: finalModelId }); const configuredFallbacks = [ volcanoModelConfig?.fallbackModelId1, diff --git a/lib/ai/auto-agent-graph.ts b/lib/ai/auto-agent-graph.ts index 97656ca..354ec0e 100644 --- a/lib/ai/auto-agent-graph.ts +++ b/lib/ai/auto-agent-graph.ts @@ -38,6 +38,49 @@ const AutoAgentState = Annotation.Root({ type AutoAgentStateType = typeof AutoAgentState.State; +function compactText(text: string, max = 48) { + const normalized = text.replace(/\s+/g, " ").trim(); + if (normalized.length <= max) return normalized; + return `${normalized.slice(0, max)}...`; +} + +function buildStrategicFriendlyMessage( + analysis: AnalysisResult, + state: AutoAgentStateType, +): string | null { + const fallbackQuestion = analysis.clarifyingQuestions?.[0]; + + if (analysis.needsClarification) { + return analysis.conversationalReply || fallbackQuestion || "我先确认一下，你希望我优先调整哪一部分？"; + } + + if (analysis.intent === "image_gen") { + const demand = compactText(state.text); + return [ + "收到，我来帮你走自动模式生成方案。", + `- 需求理解：${demand}`, + "- 执行策略：先锁定主体与场景，再控制风格/光影/构图，最后输出可继续迭代的版本。", + "- 继续优化：你可以直接说“保留A，改B”，我会按最新要求快速迭代。", + ].join("\n"); + } + + if (analysis.intent === "image_edit") { + const sourceHint = state.hasImage + ? state.imageFromHistory + ? "我会基于你上一轮图片继续编辑。" + : "我会基于你刚上传的图片编辑。" + : "我会先按当前上下文进行编辑。"; + return [ + "收到，我来帮你精准编辑这张图。", + `- 执行基准：${sourceHint}`, + "- 处理方式：只改你提到的目标区域，其他主体、构图与风格尽量保持不变。", + "- 交付方式：先出一版结果，你再告诉我细调方向（如颜色、材质、氛围、细节）。", + ].join("\n"); + } + + return analysis.conversationalReply || null; +} + const analyzeNode = async (state: AutoAgentStateType) => { const { analyzeIntent } = await import("./smart-analyzer"); const analysis = await analyzeIntent({ @@ -53,15 +96,20 @@ const analyzeNode = async (state: AutoAgentStateType) => { ? analysis.clarifyingQuestions?.[0] : undefined; + const strategicMessage = buildStrategicFriendlyMessage(analysis, state); + return { analysis, - friendlyMessage: analysis.conversationalReply || fallbackQuestion || null, + friendlyMessage: strategicMessage || analysis.conversationalReply || fallbackQuestion || null, }; }; const shouldOptimize = (state: AutoAgentStateType) => { if (!state.analysis) return "route"; if (state.analysis.needsClarification) return "route"; + if (state.analysis.intent === "image_gen") { + return "optimize"; + } if (state.analysis.intent === "image_edit" && state.analysis.subIntent) { return "optimize"; } @@ -69,7 +117,13 @@ const shouldOptimize = (state: AutoAgentStateType) => { }; const optimizeNode = async (state: AutoAgentStateType) => { - if (!state.analysis || state.analysis.intent !== "image_edit" || !state.analysis.subIntent) { + if (!state.analysis) { + return {}; + } + if (state.analysis.intent === "image_edit" && !state.analysis.subIntent) { + return {}; + } + if (state.analysis.intent !== "image_edit" && state.analysis.intent !== "image_gen") { return {}; } const { optimizePromptWithLLM } = await import("./prompt-optimizer"); diff --git a/lib/ai/prompt-optimizer.ts b/lib/ai/prompt-optimizer.ts index b145372..890ee6a 100644 --- a/lib/ai/prompt-optimizer.ts +++ b/lib/ai/prompt-optimizer.ts @@ -224,6 +224,47 @@ const LLM_OPTIMIZER_SYSTEM_PROMPT = `你是一个专业的AI图片编辑提示 - negativePrompt 必须包含"其他元素改变"相关描述 - 结合对话历史理解用户的连续编辑意图`; +const IMAGE_GEN_OPTIMIZER_SYSTEM_PROMPT = `你是一个专业的图片生成提示词优化专家，目标是把用户自然语言改写成更稳定、高质量、可迭代的生图提示词。 + +## 输出格式（严格 JSON） +{ + "optimizedPrompt": "适合生图模型的中文提示词", + "negativePrompt": "需要避免的效果" +} + +## 优化原则 +1. 保留用户核心诉求，不擅自改题。 +2. 自动补齐：主体、场景、构图、材质、光线、风格、镜头感。 +3. 对电商/家居场景优先强调：真实材质、空间比例合理、商业可用。 +4. negativePrompt 要包含：畸形、模糊、水印、文字错乱、额外主体、比例错误等。 +5. 输出简洁可执行，不要解释过程。 +`; + +function parseOptimizerJson(responseText: string): { optimizedPrompt?: string; negativePrompt?: string } | null { + const jsonMatch = responseText.match(/\{[\s\S]*\}/); + if (!jsonMatch) return null; + try { + return JSON.parse(jsonMatch[0]); + } catch (error) { + return null; + } +} + +function fallbackImageGenPrompt(originalPrompt: string): OptimizeResult { + const normalized = originalPrompt.replace(/\s+/g, " ").trim(); + const optimizedPrompt = [ + "高质量图像生成：", + normalized, + "要求：主体清晰，构图平衡，光影自然，材质真实，细节丰富，整体风格统一。", + ].join(" "); + + return { + originalPrompt, + optimizedPrompt, + negativePrompt: "低清晰度，构图混乱，比例畸形，重复主体，手部异常，文字错乱，水印，噪点，过曝，过度锐化", + }; +} + /** * 🆕 使用 LLM 深度优化提示词（基于历史对话上下文） */ @@ -243,8 +284,7 @@ export async function optimizePromptWithLLM(params: { historyCount: historyMessages?.length || 0 }); - // 只对图片编辑类意图进行 LLM 优化 - if (intent !== "image_edit") { + if (intent !== "image_edit" && intent !== "image_gen") { return { originalPrompt, optimizedPrompt: originalPrompt, @@ -287,33 +327,34 @@ export async function optimizePromptWithLLM(params: { contextMessage += `## 对话历史：\n${historyText}\n\n`; } - const userMessage = `${contextMessage}## 当前用户请求：\n"${originalPrompt}"\n\n## 识别的意图：${subIntent || '图片编辑'}\n\n请生成优化后的提示词：`; + const optimizerSystemPrompt = + intent === "image_gen" ? IMAGE_GEN_OPTIMIZER_SYSTEM_PROMPT : LLM_OPTIMIZER_SYSTEM_PROMPT; + + const intentLabel = + intent === "image_gen" ? "图片生成" : (subIntent || "图片编辑"); + + const finalUserMessage = `${contextMessage}## 当前用户请求：\n"${originalPrompt}"\n\n## 识别的意图：${intentLabel}\n\n请生成优化后的提示词：`; console.log("[Prompt Optimizer LLM] Calling Qwen3 VL for prompt optimization"); const { text: responseText } = await generateText({ model, - system: LLM_OPTIMIZER_SYSTEM_PROMPT, - prompt: userMessage, + system: optimizerSystemPrompt, + prompt: finalUserMessage, temperature: 0.2, maxTokens: 200, } as any); // 解析 JSON 响应 - const jsonMatch = responseText.match(/\{[\s\S]*\}/); - if (jsonMatch) { - try { - const result = JSON.parse(jsonMatch[0]); - console.log("[Prompt Optimizer LLM] Result:", result); - return { - originalPrompt, - optimizedPrompt: result.optimizedPrompt || originalPrompt, - negativePrompt: result.negativePrompt || "模糊，低质量，变形，失真", - friendlyMessage: FRIENDLY_MESSAGES[subIntent || ""]?.template, - }; - } catch (e) { - console.error("[Prompt Optimizer LLM] Failed to parse response:", e); - } + const parsedResult = parseOptimizerJson(responseText); + if (parsedResult) { + console.log("[Prompt Optimizer LLM] Result parsed"); + return { + originalPrompt, + optimizedPrompt: parsedResult.optimizedPrompt || originalPrompt, + negativePrompt: parsedResult.negativePrompt || "模糊，低质量，变形，失真", + friendlyMessage: intent === "image_edit" ? FRIENDLY_MESSAGES[subIntent || ""]?.template : undefined, + }; } // 解析失败，回退到模板优化 @@ -339,6 +380,10 @@ export function optimizePrompt(params: { console.log("[Prompt Optimizer] Input:", { originalPrompt, intent, subIntent }); + if (intent === "image_gen") { + return fallbackImageGenPrompt(originalPrompt); + } + // 只对图片编辑类意图进行优化 if (intent !== "image_edit") { return {