优化自动模式上下文
This commit is contained in:
Your Name
parent
21a6280db8
commit
2ea63d978f
@ -337,12 +337,14 @@ export async function POST(request: Request) {
|
||||
selectedChatModel,
|
||||
selectedVisibilityType,
|
||||
hiddenPrompt,
|
||||
autoContext,
|
||||
}: {
|
||||
id: string;
|
||||
message: ChatMessage;
|
||||
selectedChatModel: ChatModel["id"];
|
||||
selectedVisibilityType: VisibilityType;
|
||||
hiddenPrompt?: string;
|
||||
autoContext?: string;
|
||||
} = requestBody;
|
||||
|
||||
let finalModelId = selectedChatModel;
|
||||
@ -367,6 +369,7 @@ export async function POST(request: Request) {
|
||||
.filter((p): p is { type: "text"; text: string } => p.type === "text")
|
||||
.map((p) => p.text)
|
||||
.join(" ");
|
||||
const autoContextText = typeof autoContext === "string" ? autoContext.trim() : "";
|
||||
|
||||
// @ts-ignore - 检查各种可能的图片类型
|
||||
let hasImage = lastUserMessage.parts.some(
|
||||
@ -466,6 +469,7 @@ export async function POST(request: Request) {
|
||||
imageUrls,
|
||||
imageFromHistory: smartAutoHistoryImages.length > 0, // 🆕 标识图片是否来自历史
|
||||
historyMessages, // 🆕 传入历史对话用于上下文理解
|
||||
context: autoContextText || undefined,
|
||||
});
|
||||
|
||||
console.log("[Smart Auto Mode] Analysis result:", {
|
||||
@ -489,6 +493,7 @@ export async function POST(request: Request) {
|
||||
intent: analysisResult.intent,
|
||||
subIntent: analysisResult.subIntent,
|
||||
historyMessages, // 🆕 传入历史对话上下文
|
||||
context: autoContextText || undefined,
|
||||
});
|
||||
|
||||
console.log("[Smart Auto Mode] ✨ Prompt optimized (LLM):", {
|
||||
|
||||
@ -50,6 +50,7 @@ export const postRequestBodySchema = z.object({
|
||||
parts: z.array(partSchema),
|
||||
}),
|
||||
hiddenPrompt: z.string().max(4000).optional(),
|
||||
autoContext: z.string().max(4000).optional(),
|
||||
// 支持预定义模型名称或数据库中的模型 UUID
|
||||
selectedChatModel: z.string().min(1).max(128),
|
||||
selectedVisibilityType: z.enum(["public", "private"]),
|
||||
|
||||
@ -143,6 +143,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
const fileInputRef = useRef<HTMLInputElement>(null);
|
||||
const textareaRef = useRef<HTMLTextAreaElement>(null);
|
||||
const hiddenPromptRef = useRef<string | null>(null);
|
||||
const autoContextRef = useRef<string | null>(null);
|
||||
|
||||
// 🆕 标签悬停预览状态
|
||||
const [tagPreview, setTagPreview] = useState<TagPreviewInfo | null>(null);
|
||||
@ -557,7 +558,9 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
lastMessage: request.messages.at(-1),
|
||||
});
|
||||
const hiddenPrompt = hiddenPromptRef.current || undefined;
|
||||
const autoContext = autoContextRef.current || undefined;
|
||||
hiddenPromptRef.current = null;
|
||||
autoContextRef.current = null;
|
||||
return {
|
||||
body: {
|
||||
id: rawId,
|
||||
@ -567,6 +570,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
videoParams: undefined, // 必须传递，即使是 undefined
|
||||
imageParams: imageParamsRef.current || undefined, // 传递图片参数
|
||||
hiddenPrompt,
|
||||
autoContext,
|
||||
...request.body,
|
||||
},
|
||||
};
|
||||
@ -765,6 +769,50 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
return urls;
|
||||
}, [selectedImageNodes, nodes, messages]);
|
||||
|
||||
// 🆕 智能自动模式上下文（画布图片布局）
|
||||
const buildAutoContext = useCallback(() => {
|
||||
const imageNodes = nodes.filter(n => n.type === 'image' && (n.data as any)?.src);
|
||||
if (imageNodes.length === 0) return "";
|
||||
|
||||
const MAX_CONTEXT_IMAGES = 6;
|
||||
const xs = imageNodes.map(n => n.position.x);
|
||||
const ys = imageNodes.map(n => n.position.y);
|
||||
const minX = Math.min(...xs);
|
||||
const maxX = Math.max(...xs);
|
||||
const minY = Math.min(...ys);
|
||||
const maxY = Math.max(...ys);
|
||||
const spanX = Math.max(1, maxX - minX);
|
||||
const spanY = Math.max(1, maxY - minY);
|
||||
|
||||
const positionLabel = (node: (typeof imageNodes)[number]) => {
|
||||
const relX = (node.position.x - minX) / spanX;
|
||||
const relY = (node.position.y - minY) / spanY;
|
||||
const horizontal = relX <= 0.33 ? "左" : relX >= 0.66 ? "右" : "中";
|
||||
const vertical = relY <= 0.33 ? "上" : relY >= 0.66 ? "下" : "中";
|
||||
if (horizontal === "中" && vertical === "中") return "中间";
|
||||
return `${vertical}${horizontal}`;
|
||||
};
|
||||
|
||||
const selectedNodes = selectedImageNodes.length > 0 ? selectedImageNodes : [];
|
||||
const contextNodes = (selectedNodes.length > 0 ? selectedNodes : imageNodes.slice(-MAX_CONTEXT_IMAGES)).slice(0, MAX_CONTEXT_IMAGES);
|
||||
|
||||
const lines = [
|
||||
`画布图片总数: ${imageNodes.length}`,
|
||||
selectedNodes.length > 0
|
||||
? `用户已选中 ${selectedNodes.length} 张图片，优先处理这些`
|
||||
: `用户未选中图片，默认使用最近 ${contextNodes.length} 张`,
|
||||
...contextNodes.map((node, index) => {
|
||||
const label = positionLabel(node);
|
||||
const size = node.size ? `${Math.round(node.size.width)}x${Math.round(node.size.height)}` : "未知尺寸";
|
||||
const prompt = (node.data as any)?.prompt;
|
||||
return `图片${index + 1}: ${label}，尺寸 ${size}${prompt ? `，描述：${String(prompt).slice(0, 60)}` : ""}`;
|
||||
}),
|
||||
"若用户提到左/右/上/下/中等方位，请匹配对应位置的图片。",
|
||||
];
|
||||
|
||||
return lines.join("\n");
|
||||
}, [nodes, selectedImageNodes]);
|
||||
|
||||
// 处理发送（图片已在上传时添加到画布，这里只发送消息）
|
||||
const handleSend = useCallback(async () => {
|
||||
if (!input.trim() && attachments.length === 0) return;
|
||||
@ -840,6 +888,9 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
const hiddenPrompt = hiddenSegments.join('\n\n').trim();
|
||||
hiddenPromptRef.current = hiddenPrompt || null;
|
||||
|
||||
const autoContext = selectedModelId === "auto" ? buildAutoContext() : "";
|
||||
autoContextRef.current = autoContext ? autoContext : null;
|
||||
|
||||
if (textContent) {
|
||||
parts.push({ type: "text" as const, text: textContent });
|
||||
}
|
||||
@ -863,7 +914,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
editableDiv.innerHTML = '';
|
||||
console.log('[CanvasChat] 已清空输入框和选中状态');
|
||||
}
|
||||
}, [input, attachments, selectedModelId, sendMessage, getRecentImageUrls, clearSelection]);
|
||||
}, [input, attachments, selectedModelId, sendMessage, getRecentImageUrls, clearSelection, buildAutoContext]);
|
||||
|
||||
// 处理文件选择 - 上传后立即添加到画布
|
||||
const handleFileSelect = async (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
|
||||
@ -92,31 +92,29 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
|
||||
const [productName, setProductName] = useState<string>(initialProductName || ""); // 产品名称（如"沙发"）
|
||||
const [scenes, setScenes] = useState<SceneItem[]>([]);
|
||||
const [isProcessing, setIsProcessing] = useState(false);
|
||||
const [isProductUploading, setIsProductUploading] = useState(false);
|
||||
const [aspectRatio, setAspectRatio] = useState<string>("auto"); // 图片比例
|
||||
const [imageSize, setImageSize] = useState<string>("2K"); // 图片分辨率
|
||||
|
||||
// 文件上传 Input 引用
|
||||
const productInputRef = useRef<HTMLInputElement>(null);
|
||||
const sceneInputRef = useRef<HTMLInputElement>(null);
|
||||
const productUploadIdRef = useRef(0);
|
||||
const productPreviewObjectUrlRef = useRef<string | null>(null);
|
||||
|
||||
// 如果外部传入了初始产品图，则在首次挂载或传入值变化时自动带入
|
||||
const updateProductPreview = (previewUrl: string | null) => {
|
||||
const updateProductPreview = (previewUrl: string | null, isObjectUrl = false) => {
|
||||
if (productPreviewObjectUrlRef.current) {
|
||||
URL.revokeObjectURL(productPreviewObjectUrlRef.current);
|
||||
productPreviewObjectUrlRef.current = null;
|
||||
}
|
||||
setProductImagePreview(previewUrl);
|
||||
if (isObjectUrl && previewUrl) {
|
||||
productPreviewObjectUrlRef.current = previewUrl;
|
||||
}
|
||||
};
|
||||
|
||||
const clearProductImage = () => {
|
||||
setProductImage(null);
|
||||
updateProductPreview(null);
|
||||
setIsProductUploading(false);
|
||||
productUploadIdRef.current += 1;
|
||||
};
|
||||
|
||||
const handleProductPreviewError = () => {
|
||||
if (productImage && productImagePreview !== productImage) {
|
||||
updateProductPreview(productImage);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
@ -126,37 +124,18 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
|
||||
}
|
||||
}, [initialProductImage]);
|
||||
|
||||
// 处理产品图上传
|
||||
const uploadProductImageFile = async (file: File): Promise<string> => {
|
||||
const formData = new FormData();
|
||||
formData.append("file", file);
|
||||
const response = await fetch("/api/files/upload", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
let errorMessage = "上传失败";
|
||||
try {
|
||||
const errorData = await response.json();
|
||||
errorMessage = errorData?.error || errorData?.message || errorMessage;
|
||||
} catch {
|
||||
// ignore
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (productPreviewObjectUrlRef.current) {
|
||||
URL.revokeObjectURL(productPreviewObjectUrlRef.current);
|
||||
productPreviewObjectUrlRef.current = null;
|
||||
}
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const result = await response.json();
|
||||
if (!result?.url) {
|
||||
throw new Error("上传失败");
|
||||
}
|
||||
|
||||
return result.url as string;
|
||||
};
|
||||
|
||||
const handleProductUpload = async (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const input = e.target;
|
||||
const file = input.files?.[0];
|
||||
// 处理产品图上传
|
||||
const handleProductUpload = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const file = e.target.files?.[0];
|
||||
if (!file) return;
|
||||
|
||||
if (file.size > 10 * 1024 * 1024) {
|
||||
@ -165,16 +144,13 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
|
||||
}
|
||||
|
||||
setProductImage(null);
|
||||
updateProductPreview(null);
|
||||
setIsProductUploading(true);
|
||||
const uploadId = ++productUploadIdRef.current;
|
||||
const previewUrl = URL.createObjectURL(file);
|
||||
updateProductPreview(previewUrl, true);
|
||||
|
||||
const reader = new FileReader();
|
||||
reader.onload = (e) => {
|
||||
if (uploadId !== productUploadIdRef.current) return;
|
||||
const result = e.target?.result;
|
||||
if (typeof result === "string") {
|
||||
updateProductPreview(result);
|
||||
setProductImage(result);
|
||||
} else {
|
||||
toast.error("图片读取失败，请重试");
|
||||
@ -182,28 +158,12 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
|
||||
}
|
||||
};
|
||||
reader.onerror = () => {
|
||||
if (uploadId !== productUploadIdRef.current) return;
|
||||
toast.error("图片读取失败，请重试");
|
||||
clearProductImage();
|
||||
};
|
||||
reader.readAsDataURL(file);
|
||||
|
||||
try {
|
||||
toast.info("正在上传产品图...");
|
||||
const uploadedUrl = await uploadProductImageFile(file);
|
||||
if (uploadId === productUploadIdRef.current) {
|
||||
setProductImage(uploadedUrl);
|
||||
toast.success("产品图上传成功");
|
||||
}
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : "产品图上传失败");
|
||||
} finally {
|
||||
if (uploadId === productUploadIdRef.current) {
|
||||
setIsProductUploading(false);
|
||||
}
|
||||
}
|
||||
// 重置 input，允许重复选择同一文件
|
||||
input.value = "";
|
||||
e.target.value = "";
|
||||
};
|
||||
|
||||
// 处理场景图上传
|
||||
@ -672,19 +632,11 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
|
||||
<div className="space-y-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<Label className="text-base font-medium">产品图 A (必须)</Label>
|
||||
<div className="flex items-center gap-2">
|
||||
{isProductUploading && (
|
||||
<span className="flex items-center gap-1 text-xs text-muted-foreground">
|
||||
<Loader2 className="h-3 w-3 animate-spin" />
|
||||
上传中...
|
||||
</span>
|
||||
)}
|
||||
{(productImagePreview || productImage) && (
|
||||
<Button variant="ghost" size="sm" onClick={clearProductImage} className="h-6 text-xs text-muted-foreground hover:text-destructive">
|
||||
清除
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
{(productImagePreview || productImage) && (
|
||||
<Button variant="ghost" size="sm" onClick={clearProductImage} className="h-6 text-xs text-muted-foreground hover:text-destructive">
|
||||
清除
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div
|
||||
@ -703,13 +655,8 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
|
||||
/>
|
||||
|
||||
{(productImagePreview || productImage) ? (
|
||||
<div className="relative mx-auto h-48 w-48 rounded-md border bg-muted overflow-hidden">
|
||||
<img
|
||||
src={productImagePreview || productImage || ""}
|
||||
alt="产品图"
|
||||
className="w-full h-full object-contain"
|
||||
onError={handleProductPreviewError}
|
||||
/>
|
||||
<div className="relative aspect-square w-full max-w-[200px] mx-auto">
|
||||
<img src={productImagePreview || productImage || ""} alt="产品图" className="w-full h-full object-contain rounded-md" />
|
||||
<div className="absolute inset-0 bg-black/40 opacity-0 group-hover:opacity-100 transition-opacity flex items-center justify-center rounded-md">
|
||||
<Button variant="secondary" size="sm" onClick={(e) => { e.stopPropagation(); productInputRef.current?.click(); }}>
|
||||
更换图片
|
||||
@ -876,7 +823,7 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
|
||||
className="w-full"
|
||||
size="lg"
|
||||
onClick={startFusion}
|
||||
disabled={isProcessing || isProductUploading || !productImage || scenes.length === 0}
|
||||
disabled={isProcessing || !productImage || scenes.length === 0}
|
||||
>
|
||||
{isProcessing ? (
|
||||
<>
|
||||
|
||||
@ -232,8 +232,9 @@ export async function optimizePromptWithLLM(params: {
|
||||
intent: string;
|
||||
subIntent?: string;
|
||||
historyMessages?: HistoryMessage[];
|
||||
context?: string;
|
||||
}): Promise<OptimizeResult> {
|
||||
const { originalPrompt, intent, subIntent, historyMessages } = params;
|
||||
const { originalPrompt, intent, subIntent, historyMessages, context } = params;
|
||||
|
||||
console.log("[Prompt Optimizer LLM] Input:", {
|
||||
originalPrompt,
|
||||
@ -273,14 +274,17 @@ export async function optimizePromptWithLLM(params: {
|
||||
return optimizePrompt({ originalPrompt, intent, subIntent });
|
||||
}
|
||||
|
||||
// 构建包含历史上下文的消息
|
||||
// 构建包含上下文的消息
|
||||
let contextMessage = "";
|
||||
if (context && context.trim().length > 0) {
|
||||
contextMessage = `## 画布上下文（仅用于理解，不需要在最终提示词中复述坐标）：\n${context.trim()}\n\n`;
|
||||
}
|
||||
if (historyMessages && historyMessages.length > 0) {
|
||||
const recentHistory = historyMessages.slice(-6); // 最近3轮对话
|
||||
const historyText = recentHistory.map(msg =>
|
||||
`${msg.role === 'user' ? '用户' : 'AI'}: ${msg.content.substring(0, 150)}`
|
||||
).join('\n');
|
||||
contextMessage = `## 对话历史：\n${historyText}\n\n`;
|
||||
contextMessage += `## 对话历史：\n${historyText}\n\n`;
|
||||
}
|
||||
|
||||
const userMessage = `${contextMessage}## 当前用户请求：\n"${originalPrompt}"\n\n## 识别的意图：${subIntent || '图片编辑'}\n\n请生成优化后的提示词：`;
|
||||
|
||||
@ -74,6 +74,7 @@ const ANALYZER_SYSTEM_PROMPT = `你是一个友好的AI图片编辑助手。用
|
||||
- 可以加一些语气词让对话更亲切，如"好的"、"没问题"、"~"等
|
||||
- 说明你理解了什么需求，让用户知道你在做什么
|
||||
- 如果意图不明确，礼貌地询问用户想要什么效果
|
||||
- 如果提供了画布上下文，仅用于理解，不要在 conversationalReply 中复述坐标或技术细节
|
||||
|
||||
## 示例
|
||||
用户："帮我把背景换成白色"
|
||||
@ -145,8 +146,9 @@ export async function analyzeIntent(params: {
|
||||
imageUrls?: string[];
|
||||
imageFromHistory?: boolean; // 🆕 图片是否来自历史记录
|
||||
historyMessages?: HistoryMessage[]; // 🆕 历史对话消息（用于上下文理解）
|
||||
context?: string; // 🆕 画布上下文（仅用于理解）
|
||||
}): Promise<AnalysisResult> {
|
||||
const { text, hasImage, imageUrls, imageFromHistory, historyMessages } = params;
|
||||
const { text, hasImage, imageUrls, imageFromHistory, historyMessages, context } = params;
|
||||
|
||||
console.log("[Smart Analyzer] Analyzing intent:", {
|
||||
text,
|
||||
@ -165,7 +167,7 @@ export async function analyzeIntent(params: {
|
||||
|
||||
// 尝试使用 Qwen3 VL 进行深度分析（传入历史对话）
|
||||
try {
|
||||
const llmResult = await analyzWithLLM(text, hasImage, historyMessages);
|
||||
const llmResult = await analyzWithLLM(text, hasImage, historyMessages, context);
|
||||
if (llmResult) {
|
||||
console.log("[Smart Analyzer] LLM analysis result:", llmResult);
|
||||
return llmResult;
|
||||
@ -199,7 +201,8 @@ function quickKeywordMatch(text: string, hasImage: boolean): AnalysisResult {
|
||||
async function analyzWithLLM(
|
||||
text: string,
|
||||
hasImage: boolean,
|
||||
historyMessages?: HistoryMessage[]
|
||||
historyMessages?: HistoryMessage[],
|
||||
context?: string
|
||||
): Promise<AnalysisResult | null> {
|
||||
// 使用动态导入避免循环依赖问题
|
||||
const { getEnabledAiModelsWithProvider } = await import("@/lib/db/queries");
|
||||
@ -225,6 +228,11 @@ async function analyzWithLLM(
|
||||
// 🆕 构建包含历史上下文的消息
|
||||
let contextMessage = "";
|
||||
|
||||
// 🆕 画布上下文（仅用于理解，不要复述）
|
||||
if (context && context.trim().length > 0) {
|
||||
contextMessage = `## 画布上下文（仅用于理解，不要在对话回复中复述）：\n${context.trim()}\n\n`;
|
||||
}
|
||||
|
||||
// 添加历史对话上下文（最多取最近5轮对话）
|
||||
if (historyMessages && historyMessages.length > 0) {
|
||||
const recentHistory = historyMessages.slice(-10); // 最多10条消息（约5轮对话）
|
||||
@ -232,7 +240,7 @@ async function analyzWithLLM(
|
||||
`${msg.role === 'user' ? '用户' : 'AI'}: ${msg.content.substring(0, 200)}`
|
||||
).join('\n');
|
||||
|
||||
contextMessage = `## 对话历史（最近的对话上下文）：
|
||||
contextMessage += `## 对话历史（最近的对话上下文）：
|
||||
${historyText}
|
||||
|
||||
## 当前请求：
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user