优化自动模式需求收敛与澄清流程
This commit is contained in:
Your Name
parent
a8b078b8ae
commit
48deda49f7
@ -124,6 +124,39 @@ function applyHiddenPromptToMessages(messages: ChatMessage[], hiddenPrompt?: str
|
||||
return cloned;
|
||||
}
|
||||
|
||||
function applyOptimizedPromptToLatestUserMessage(
|
||||
messages: ChatMessage[],
|
||||
optimizedPrompt?: string,
|
||||
hiddenPromptPrefix = "",
|
||||
): ChatMessage[] {
|
||||
const prompt = optimizedPrompt?.trim();
|
||||
if (!prompt) return messages;
|
||||
|
||||
const cloned = messages.map((msg) => ({
|
||||
...msg,
|
||||
parts: msg.parts.map((part) => ({ ...part })),
|
||||
}));
|
||||
|
||||
for (let i = cloned.length - 1; i >= 0; i -= 1) {
|
||||
const msg = cloned[i];
|
||||
if (msg.role !== "user") continue;
|
||||
|
||||
const textPart = msg.parts.find(
|
||||
(part): part is { type: "text"; text: string } => part.type === "text",
|
||||
);
|
||||
const finalPrompt = `${hiddenPromptPrefix}${prompt}`.trim();
|
||||
|
||||
if (textPart) {
|
||||
textPart.text = finalPrompt;
|
||||
} else {
|
||||
msg.parts.unshift({ type: "text", text: finalPrompt });
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
return cloned;
|
||||
}
|
||||
|
||||
function guessImageMediaType(url: string): string {
|
||||
const cleanUrl = url.split("?")[0]?.toLowerCase() || "";
|
||||
if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "image/jpeg";
|
||||
@ -453,6 +486,8 @@ export async function POST(request: Request) {
|
||||
let smartAutoFriendlyMessage: string | undefined; // AI的自然对话回复
|
||||
let smartAutoOptimizedPrompt: string | undefined; // 优化后的提示词（不修改用户消息，只用于API调用）
|
||||
let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL（支持多张）
|
||||
let smartAutoNeedsClarification = false; // 是否需要先追问用户
|
||||
let smartAutoClarificationQuestion: string | undefined;
|
||||
|
||||
// ========== Smart Auto Mode Logic ==========
|
||||
if (finalModelId === "auto") {
|
||||
@ -579,6 +614,9 @@ export async function POST(request: Request) {
|
||||
confidence: autoGraphResult.analysis.confidence,
|
||||
needsClarification: autoGraphResult.analysis.needsClarification,
|
||||
});
|
||||
|
||||
smartAutoNeedsClarification = Boolean(autoGraphResult.analysis.needsClarification);
|
||||
smartAutoClarificationQuestion = autoGraphResult.analysis.clarifyingQuestions?.[0];
|
||||
}
|
||||
|
||||
if (autoGraphResult.friendlyMessage) {
|
||||
@ -744,6 +782,47 @@ export async function POST(request: Request) {
|
||||
const streamId = generateUUID();
|
||||
await createStreamId({ streamId, chatId: id });
|
||||
|
||||
if (selectedChatModel === "auto" && smartAutoNeedsClarification) {
|
||||
const clarificationMessage =
|
||||
smartAutoClarificationQuestion ||
|
||||
smartAutoFriendlyMessage ||
|
||||
"我先确认一下，你希望我优先处理哪一部分？";
|
||||
|
||||
const clarificationStream = createUIMessageStream({
|
||||
execute: async ({ writer }) => {
|
||||
const messageId = generateUUID();
|
||||
writer.write({
|
||||
type: "data-appendMessage",
|
||||
data: JSON.stringify({
|
||||
id: messageId,
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: clarificationMessage }],
|
||||
}),
|
||||
});
|
||||
|
||||
await saveMessages({
|
||||
messages: [
|
||||
{
|
||||
id: messageId,
|
||||
chatId: id,
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: clarificationMessage }],
|
||||
attachments: [],
|
||||
createdAt: new Date(),
|
||||
videoStorageStatus: null,
|
||||
temporaryVideoUrl: null,
|
||||
permanentVideoUrl: null,
|
||||
archivedAt: null,
|
||||
videoMetadata: null,
|
||||
},
|
||||
],
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
return new Response(clarificationStream.pipeThrough(new JsonToSseTransformStream()));
|
||||
}
|
||||
|
||||
let finalMergedUsage: AppUsage | undefined;
|
||||
|
||||
// 获取启用的工具列表
|
||||
@ -927,6 +1006,11 @@ export async function POST(request: Request) {
|
||||
// 只传递最后一条用户消息，避免上下文历史干扰生图效果
|
||||
const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop();
|
||||
let messagesForGemini = lastUserMessage ? [lastUserMessage] : uiMessagesForModel;
|
||||
messagesForGemini = applyOptimizedPromptToLatestUserMessage(
|
||||
messagesForGemini,
|
||||
smartAutoOptimizedPrompt,
|
||||
hiddenPromptPrefix,
|
||||
);
|
||||
messagesForGemini = injectHistoryImagesToLastUserMessage(messagesForGemini, smartAutoHistoryImages);
|
||||
console.log("[Chat API] Gemini: Using only last user message (no history context)");
|
||||
|
||||
@ -1284,17 +1368,13 @@ export async function POST(request: Request) {
|
||||
console.log("[Chat API] Alibaba: Using only last user message");
|
||||
|
||||
// 🆕 如果有智能模式优化后的提示词，替换消息中的文本（仅用于API调用）
|
||||
if (smartAutoOptimizedPrompt && messagesForAlibaba.length > 0) {
|
||||
if (smartAutoOptimizedPrompt) {
|
||||
console.log("[Chat API] Using optimized prompt for API call");
|
||||
messagesForAlibaba = messagesForAlibaba.map((msg) => ({
|
||||
...msg,
|
||||
parts: msg.parts.map((part) => {
|
||||
if (part.type === "text") {
|
||||
return { ...part, text: `${hiddenPromptPrefix}${smartAutoOptimizedPrompt!}`.trim() };
|
||||
}
|
||||
return part;
|
||||
}),
|
||||
}));
|
||||
messagesForAlibaba = applyOptimizedPromptToLatestUserMessage(
|
||||
messagesForAlibaba,
|
||||
smartAutoOptimizedPrompt,
|
||||
hiddenPromptPrefix,
|
||||
);
|
||||
}
|
||||
|
||||
messagesForAlibaba = injectHistoryImagesToLastUserMessage(messagesForAlibaba, smartAutoHistoryImages);
|
||||
@ -1479,7 +1559,11 @@ export async function POST(request: Request) {
|
||||
|
||||
// 🆕 检测是否有标注图片，如果有则在消息中注入系统提示
|
||||
const hasAnnotation = hasAnnotatedImage(uiMessages);
|
||||
let messagesForVolcano = uiMessagesForModel;
|
||||
let messagesForVolcano = applyOptimizedPromptToLatestUserMessage(
|
||||
uiMessagesForModel,
|
||||
smartAutoOptimizedPrompt,
|
||||
hiddenPromptPrefix,
|
||||
);
|
||||
const volcanoModelConfig = await getAiModelById({ id: finalModelId });
|
||||
const configuredFallbacks = [
|
||||
volcanoModelConfig?.fallbackModelId1,
|
||||
|
||||
@ -38,6 +38,49 @@ const AutoAgentState = Annotation.Root({
|
||||
|
||||
type AutoAgentStateType = typeof AutoAgentState.State;
|
||||
|
||||
function compactText(text: string, max = 48) {
|
||||
const normalized = text.replace(/\s+/g, " ").trim();
|
||||
if (normalized.length <= max) return normalized;
|
||||
return `${normalized.slice(0, max)}...`;
|
||||
}
|
||||
|
||||
function buildStrategicFriendlyMessage(
|
||||
analysis: AnalysisResult,
|
||||
state: AutoAgentStateType,
|
||||
): string | null {
|
||||
const fallbackQuestion = analysis.clarifyingQuestions?.[0];
|
||||
|
||||
if (analysis.needsClarification) {
|
||||
return analysis.conversationalReply || fallbackQuestion || "我先确认一下，你希望我优先调整哪一部分？";
|
||||
}
|
||||
|
||||
if (analysis.intent === "image_gen") {
|
||||
const demand = compactText(state.text);
|
||||
return [
|
||||
"收到，我来帮你走自动模式生成方案。",
|
||||
`- 需求理解：${demand}`,
|
||||
"- 执行策略：先锁定主体与场景，再控制风格/光影/构图，最后输出可继续迭代的版本。",
|
||||
"- 继续优化：你可以直接说“保留A，改B”，我会按最新要求快速迭代。",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
if (analysis.intent === "image_edit") {
|
||||
const sourceHint = state.hasImage
|
||||
? state.imageFromHistory
|
||||
? "我会基于你上一轮图片继续编辑。"
|
||||
: "我会基于你刚上传的图片编辑。"
|
||||
: "我会先按当前上下文进行编辑。";
|
||||
return [
|
||||
"收到，我来帮你精准编辑这张图。",
|
||||
`- 执行基准：${sourceHint}`,
|
||||
"- 处理方式：只改你提到的目标区域，其他主体、构图与风格尽量保持不变。",
|
||||
"- 交付方式：先出一版结果，你再告诉我细调方向（如颜色、材质、氛围、细节）。",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
return analysis.conversationalReply || null;
|
||||
}
|
||||
|
||||
const analyzeNode = async (state: AutoAgentStateType) => {
|
||||
const { analyzeIntent } = await import("./smart-analyzer");
|
||||
const analysis = await analyzeIntent({
|
||||
@ -53,15 +96,20 @@ const analyzeNode = async (state: AutoAgentStateType) => {
|
||||
? analysis.clarifyingQuestions?.[0]
|
||||
: undefined;
|
||||
|
||||
const strategicMessage = buildStrategicFriendlyMessage(analysis, state);
|
||||
|
||||
return {
|
||||
analysis,
|
||||
friendlyMessage: analysis.conversationalReply || fallbackQuestion || null,
|
||||
friendlyMessage: strategicMessage || analysis.conversationalReply || fallbackQuestion || null,
|
||||
};
|
||||
};
|
||||
|
||||
const shouldOptimize = (state: AutoAgentStateType) => {
|
||||
if (!state.analysis) return "route";
|
||||
if (state.analysis.needsClarification) return "route";
|
||||
if (state.analysis.intent === "image_gen") {
|
||||
return "optimize";
|
||||
}
|
||||
if (state.analysis.intent === "image_edit" && state.analysis.subIntent) {
|
||||
return "optimize";
|
||||
}
|
||||
@ -69,7 +117,13 @@ const shouldOptimize = (state: AutoAgentStateType) => {
|
||||
};
|
||||
|
||||
const optimizeNode = async (state: AutoAgentStateType) => {
|
||||
if (!state.analysis || state.analysis.intent !== "image_edit" || !state.analysis.subIntent) {
|
||||
if (!state.analysis) {
|
||||
return {};
|
||||
}
|
||||
if (state.analysis.intent === "image_edit" && !state.analysis.subIntent) {
|
||||
return {};
|
||||
}
|
||||
if (state.analysis.intent !== "image_edit" && state.analysis.intent !== "image_gen") {
|
||||
return {};
|
||||
}
|
||||
const { optimizePromptWithLLM } = await import("./prompt-optimizer");
|
||||
|
||||
@ -224,6 +224,47 @@ const LLM_OPTIMIZER_SYSTEM_PROMPT = `你是一个专业的AI图片编辑提示
|
||||
- negativePrompt 必须包含"其他元素改变"相关描述
|
||||
- 结合对话历史理解用户的连续编辑意图`;
|
||||
|
||||
const IMAGE_GEN_OPTIMIZER_SYSTEM_PROMPT = `你是一个专业的图片生成提示词优化专家，目标是把用户自然语言改写成更稳定、高质量、可迭代的生图提示词。
|
||||
|
||||
## 输出格式（严格 JSON）
|
||||
{
|
||||
"optimizedPrompt": "适合生图模型的中文提示词",
|
||||
"negativePrompt": "需要避免的效果"
|
||||
}
|
||||
|
||||
## 优化原则
|
||||
1. 保留用户核心诉求，不擅自改题。
|
||||
2. 自动补齐：主体、场景、构图、材质、光线、风格、镜头感。
|
||||
3. 对电商/家居场景优先强调：真实材质、空间比例合理、商业可用。
|
||||
4. negativePrompt 要包含：畸形、模糊、水印、文字错乱、额外主体、比例错误等。
|
||||
5. 输出简洁可执行，不要解释过程。
|
||||
`;
|
||||
|
||||
function parseOptimizerJson(responseText: string): { optimizedPrompt?: string; negativePrompt?: string } | null {
|
||||
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
||||
if (!jsonMatch) return null;
|
||||
try {
|
||||
return JSON.parse(jsonMatch[0]);
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function fallbackImageGenPrompt(originalPrompt: string): OptimizeResult {
|
||||
const normalized = originalPrompt.replace(/\s+/g, " ").trim();
|
||||
const optimizedPrompt = [
|
||||
"高质量图像生成：",
|
||||
normalized,
|
||||
"要求：主体清晰，构图平衡，光影自然，材质真实，细节丰富，整体风格统一。",
|
||||
].join(" ");
|
||||
|
||||
return {
|
||||
originalPrompt,
|
||||
optimizedPrompt,
|
||||
negativePrompt: "低清晰度，构图混乱，比例畸形，重复主体，手部异常，文字错乱，水印，噪点，过曝，过度锐化",
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 🆕 使用 LLM 深度优化提示词（基于历史对话上下文）
|
||||
*/
|
||||
@ -243,8 +284,7 @@ export async function optimizePromptWithLLM(params: {
|
||||
historyCount: historyMessages?.length || 0
|
||||
});
|
||||
|
||||
// 只对图片编辑类意图进行 LLM 优化
|
||||
if (intent !== "image_edit") {
|
||||
if (intent !== "image_edit" && intent !== "image_gen") {
|
||||
return {
|
||||
originalPrompt,
|
||||
optimizedPrompt: originalPrompt,
|
||||
@ -287,33 +327,34 @@ export async function optimizePromptWithLLM(params: {
|
||||
contextMessage += `## 对话历史：\n${historyText}\n\n`;
|
||||
}
|
||||
|
||||
const userMessage = `${contextMessage}## 当前用户请求：\n"${originalPrompt}"\n\n## 识别的意图：${subIntent || '图片编辑'}\n\n请生成优化后的提示词：`;
|
||||
const optimizerSystemPrompt =
|
||||
intent === "image_gen" ? IMAGE_GEN_OPTIMIZER_SYSTEM_PROMPT : LLM_OPTIMIZER_SYSTEM_PROMPT;
|
||||
|
||||
const intentLabel =
|
||||
intent === "image_gen" ? "图片生成" : (subIntent || "图片编辑");
|
||||
|
||||
const finalUserMessage = `${contextMessage}## 当前用户请求：\n"${originalPrompt}"\n\n## 识别的意图：${intentLabel}\n\n请生成优化后的提示词：`;
|
||||
|
||||
console.log("[Prompt Optimizer LLM] Calling Qwen3 VL for prompt optimization");
|
||||
|
||||
const { text: responseText } = await generateText({
|
||||
model,
|
||||
system: LLM_OPTIMIZER_SYSTEM_PROMPT,
|
||||
prompt: userMessage,
|
||||
system: optimizerSystemPrompt,
|
||||
prompt: finalUserMessage,
|
||||
temperature: 0.2,
|
||||
maxTokens: 200,
|
||||
} as any);
|
||||
|
||||
// 解析 JSON 响应
|
||||
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
||||
if (jsonMatch) {
|
||||
try {
|
||||
const result = JSON.parse(jsonMatch[0]);
|
||||
console.log("[Prompt Optimizer LLM] Result:", result);
|
||||
return {
|
||||
originalPrompt,
|
||||
optimizedPrompt: result.optimizedPrompt || originalPrompt,
|
||||
negativePrompt: result.negativePrompt || "模糊，低质量，变形，失真",
|
||||
friendlyMessage: FRIENDLY_MESSAGES[subIntent || ""]?.template,
|
||||
};
|
||||
} catch (e) {
|
||||
console.error("[Prompt Optimizer LLM] Failed to parse response:", e);
|
||||
}
|
||||
const parsedResult = parseOptimizerJson(responseText);
|
||||
if (parsedResult) {
|
||||
console.log("[Prompt Optimizer LLM] Result parsed");
|
||||
return {
|
||||
originalPrompt,
|
||||
optimizedPrompt: parsedResult.optimizedPrompt || originalPrompt,
|
||||
negativePrompt: parsedResult.negativePrompt || "模糊，低质量，变形，失真",
|
||||
friendlyMessage: intent === "image_edit" ? FRIENDLY_MESSAGES[subIntent || ""]?.template : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
// 解析失败，回退到模板优化
|
||||
@ -339,6 +380,10 @@ export function optimizePrompt(params: {
|
||||
|
||||
console.log("[Prompt Optimizer] Input:", { originalPrompt, intent, subIntent });
|
||||
|
||||
if (intent === "image_gen") {
|
||||
return fallbackImageGenPrompt(originalPrompt);
|
||||
}
|
||||
|
||||
// 只对图片编辑类意图进行优化
|
||||
if (intent !== "image_edit") {
|
||||
return {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user