优化自动模式上下文

2026-02-05 20:53:01 +08:00 · 2026-02-05 20:53:01 +08:00 · 2ea63d978f
commit 2ea63d978f
parent 21a6280db8
6 changed files with 107 additions and 91 deletions
--- a/app/(chat)/api/chat/route.ts
+++ b/app/(chat)/api/chat/route.ts
@ -337,12 +337,14 @@ export async function POST(request: Request) {
      selectedChatModel,
      selectedVisibilityType,
      hiddenPrompt,
+      autoContext,
    }: {
      id: string;
      message: ChatMessage;
      selectedChatModel: ChatModel["id"];
      selectedVisibilityType: VisibilityType;
      hiddenPrompt?: string;
+      autoContext?: string;
    } = requestBody;

    let finalModelId = selectedChatModel;
@ -367,6 +369,7 @@ export async function POST(request: Request) {
          .filter((p): p is { type: "text"; text: string } => p.type === "text")
          .map((p) => p.text)
          .join(" ");
+        const autoContextText = typeof autoContext === "string" ? autoContext.trim() : "";

        // @ts-ignore - 检查各种可能的图片类型
        let hasImage = lastUserMessage.parts.some(
@ -466,6 +469,7 @@ export async function POST(request: Request) {
          imageUrls,
          imageFromHistory: smartAutoHistoryImages.length > 0,  // 🆕 标识图片是否来自历史
          historyMessages,  // 🆕 传入历史对话用于上下文理解
+          context: autoContextText || undefined,
        });

        console.log("[Smart Auto Mode] Analysis result:", {
@ -489,6 +493,7 @@ export async function POST(request: Request) {
            intent: analysisResult.intent,
            subIntent: analysisResult.subIntent,
            historyMessages,  // 🆕 传入历史对话上下文
+            context: autoContextText || undefined,
          });

          console.log("[Smart Auto Mode] ✨ Prompt optimized (LLM):", {
--- a/app/(chat)/api/chat/schema.ts
+++ b/app/(chat)/api/chat/schema.ts
@ -50,6 +50,7 @@ export const postRequestBodySchema = z.object({
    parts: z.array(partSchema),
  }),
  hiddenPrompt: z.string().max(4000).optional(),
+  autoContext: z.string().max(4000).optional(),
  // 支持预定义模型名称或数据库中的模型 UUID
  selectedChatModel: z.string().min(1).max(128),
  selectedVisibilityType: z.enum(["public", "private"]),
--- a/components/canvas/CanvasChatPanel.tsx
+++ b/components/canvas/CanvasChatPanel.tsx
@ -143,6 +143,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
  const fileInputRef = useRef<HTMLInputElement>(null);
  const textareaRef = useRef<HTMLTextAreaElement>(null);
  const hiddenPromptRef = useRef<string | null>(null);
+  const autoContextRef = useRef<string | null>(null);
  
  // 🆕 标签悬停预览状态
  const [tagPreview, setTagPreview] = useState<TagPreviewInfo | null>(null);
@ -557,7 +558,9 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
          lastMessage: request.messages.at(-1),
        });
        const hiddenPrompt = hiddenPromptRef.current || undefined;
+        const autoContext = autoContextRef.current || undefined;
        hiddenPromptRef.current = null;
+        autoContextRef.current = null;
        return {
          body: {
            id: rawId,
@ -567,6 +570,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
            videoParams: undefined, // 必须传递，即使是 undefined
            imageParams: imageParamsRef.current || undefined, // 传递图片参数
            hiddenPrompt,
+            autoContext,
            ...request.body,
          },
        };
@ -765,6 +769,50 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
    return urls;
  }, [selectedImageNodes, nodes, messages]);

+  // 🆕 智能自动模式上下文（画布图片布局）
+  const buildAutoContext = useCallback(() => {
+    const imageNodes = nodes.filter(n => n.type === 'image' && (n.data as any)?.src);
+    if (imageNodes.length === 0) return "";
+
+    const MAX_CONTEXT_IMAGES = 6;
+    const xs = imageNodes.map(n => n.position.x);
+    const ys = imageNodes.map(n => n.position.y);
+    const minX = Math.min(...xs);
+    const maxX = Math.max(...xs);
+    const minY = Math.min(...ys);
+    const maxY = Math.max(...ys);
+    const spanX = Math.max(1, maxX - minX);
+    const spanY = Math.max(1, maxY - minY);
+
+    const positionLabel = (node: (typeof imageNodes)[number]) => {
+      const relX = (node.position.x - minX) / spanX;
+      const relY = (node.position.y - minY) / spanY;
+      const horizontal = relX <= 0.33 ? "左" : relX >= 0.66 ? "右" : "中";
+      const vertical = relY <= 0.33 ? "上" : relY >= 0.66 ? "下" : "中";
+      if (horizontal === "中" && vertical === "中") return "中间";
+      return `${vertical}${horizontal}`;
+    };
+
+    const selectedNodes = selectedImageNodes.length > 0 ? selectedImageNodes : [];
+    const contextNodes = (selectedNodes.length > 0 ? selectedNodes : imageNodes.slice(-MAX_CONTEXT_IMAGES)).slice(0, MAX_CONTEXT_IMAGES);
+
+    const lines = [
+      `画布图片总数: ${imageNodes.length}`,
+      selectedNodes.length > 0
+        ? `用户已选中 ${selectedNodes.length} 张图片，优先处理这些`
+        : `用户未选中图片，默认使用最近 ${contextNodes.length} 张`,
+      ...contextNodes.map((node, index) => {
+        const label = positionLabel(node);
+        const size = node.size ? `${Math.round(node.size.width)}x${Math.round(node.size.height)}` : "未知尺寸";
+        const prompt = (node.data as any)?.prompt;
+        return `图片${index + 1}: ${label}，尺寸 ${size}${prompt ? `，描述：${String(prompt).slice(0, 60)}` : ""}`;
+      }),
+      "若用户提到左/右/上/下/中等方位，请匹配对应位置的图片。",
+    ];
+
+    return lines.join("\n");
+  }, [nodes, selectedImageNodes]);
+
  // 处理发送（图片已在上传时添加到画布，这里只发送消息）
  const handleSend = useCallback(async () => {
    if (!input.trim() && attachments.length === 0) return;
@ -840,6 +888,9 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
    const hiddenPrompt = hiddenSegments.join('\n\n').trim();
    hiddenPromptRef.current = hiddenPrompt || null;
    
+    const autoContext = selectedModelId === "auto" ? buildAutoContext() : "";
+    autoContextRef.current = autoContext ? autoContext : null;
+    
    if (textContent) {
      parts.push({ type: "text" as const, text: textContent });
    }
@ -863,7 +914,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
      editableDiv.innerHTML = '';
      console.log('[CanvasChat] 已清空输入框和选中状态');
    }
-  }, [input, attachments, selectedModelId, sendMessage, getRecentImageUrls, clearSelection]);
+  }, [input, attachments, selectedModelId, sendMessage, getRecentImageUrls, clearSelection, buildAutoContext]);

  // 处理文件选择 - 上传后立即添加到画布
  const handleFileSelect = async (e: React.ChangeEvent<HTMLInputElement>) => {
--- a/components/scene-fusion-view.tsx
+++ b/components/scene-fusion-view.tsx
@ -92,31 +92,29 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
  const [productName, setProductName] = useState<string>(initialProductName || ""); // 产品名称（如"沙发"）
  const [scenes, setScenes] = useState<SceneItem[]>([]);
  const [isProcessing, setIsProcessing] = useState(false);
-  const [isProductUploading, setIsProductUploading] = useState(false);
  const [aspectRatio, setAspectRatio] = useState<string>("auto"); // 图片比例
  const [imageSize, setImageSize] = useState<string>("2K"); // 图片分辨率
  
  // 文件上传 Input 引用
  const productInputRef = useRef<HTMLInputElement>(null);
  const sceneInputRef = useRef<HTMLInputElement>(null);
-  const productUploadIdRef = useRef(0);
+  const productPreviewObjectUrlRef = useRef<string | null>(null);

  // 如果外部传入了初始产品图，则在首次挂载或传入值变化时自动带入
-  const updateProductPreview = (previewUrl: string | null) => {
+  const updateProductPreview = (previewUrl: string | null, isObjectUrl = false) => {
+    if (productPreviewObjectUrlRef.current) {
+      URL.revokeObjectURL(productPreviewObjectUrlRef.current);
+      productPreviewObjectUrlRef.current = null;
+    }
    setProductImagePreview(previewUrl);
+    if (isObjectUrl && previewUrl) {
+      productPreviewObjectUrlRef.current = previewUrl;
+    }
  };

  const clearProductImage = () => {
    setProductImage(null);
    updateProductPreview(null);
-    setIsProductUploading(false);
-    productUploadIdRef.current += 1;
-  };
-
-  const handleProductPreviewError = () => {
-    if (productImage && productImagePreview !== productImage) {
-      updateProductPreview(productImage);
-    }
  };

  useEffect(() => {
@ -126,37 +124,18 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
    }
  }, [initialProductImage]);

-  // 处理产品图上传
-  const uploadProductImageFile = async (file: File): Promise<string> => {
-    const formData = new FormData();
-    formData.append("file", file);
-    const response = await fetch("/api/files/upload", {
-      method: "POST",
-      body: formData,
-    });
-
-    if (!response.ok) {
-      let errorMessage = "上传失败";
-      try {
-        const errorData = await response.json();
-        errorMessage = errorData?.error || errorData?.message || errorMessage;
-      } catch {
-        // ignore
+  useEffect(() => {
+    return () => {
+      if (productPreviewObjectUrlRef.current) {
+        URL.revokeObjectURL(productPreviewObjectUrlRef.current);
+        productPreviewObjectUrlRef.current = null;
      }
-      throw new Error(errorMessage);
-    }
+    };
+  }, []);

-    const result = await response.json();
-    if (!result?.url) {
-      throw new Error("上传失败");
-    }
-
-    return result.url as string;
-  };
-
-  const handleProductUpload = async (e: React.ChangeEvent<HTMLInputElement>) => {
-    const input = e.target;
-    const file = input.files?.[0];
+  // 处理产品图上传
+  const handleProductUpload = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const file = e.target.files?.[0];
    if (!file) return;

    if (file.size > 10 * 1024 * 1024) {
@ -165,16 +144,13 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
    }

    setProductImage(null);
-    updateProductPreview(null);
-    setIsProductUploading(true);
-    const uploadId = ++productUploadIdRef.current;
+    const previewUrl = URL.createObjectURL(file);
+    updateProductPreview(previewUrl, true);

    const reader = new FileReader();
    reader.onload = (e) => {
-      if (uploadId !== productUploadIdRef.current) return;
      const result = e.target?.result;
      if (typeof result === "string") {
-        updateProductPreview(result);
        setProductImage(result);
      } else {
        toast.error("图片读取失败，请重试");
@ -182,28 +158,12 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
      }
    };
    reader.onerror = () => {
-      if (uploadId !== productUploadIdRef.current) return;
      toast.error("图片读取失败，请重试");
      clearProductImage();
    };
    reader.readAsDataURL(file);
-
-    try {
-      toast.info("正在上传产品图...");
-      const uploadedUrl = await uploadProductImageFile(file);
-      if (uploadId === productUploadIdRef.current) {
-        setProductImage(uploadedUrl);
-        toast.success("产品图上传成功");
-      }
-    } catch (error) {
-      toast.error(error instanceof Error ? error.message : "产品图上传失败");
-    } finally {
-      if (uploadId === productUploadIdRef.current) {
-        setIsProductUploading(false);
-      }
-    }
    // 重置 input，允许重复选择同一文件
-    input.value = "";
+    e.target.value = "";
  };

  // 处理场景图上传
@ -672,19 +632,11 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
              <div className="space-y-2">
                <div className="flex items-center justify-between">
                  <Label className="text-base font-medium">产品图 A (必须)</Label>
-                  <div className="flex items-center gap-2">
-                    {isProductUploading && (
-                      <span className="flex items-center gap-1 text-xs text-muted-foreground">
-                        <Loader2 className="h-3 w-3 animate-spin" />
-                        上传中...
-                      </span>
-                    )}
-                    {(productImagePreview || productImage) && (
-                      <Button variant="ghost" size="sm" onClick={clearProductImage} className="h-6 text-xs text-muted-foreground hover:text-destructive">
-                        清除
-                      </Button>
-                    )}
-                  </div>
+                  {(productImagePreview || productImage) && (
+                    <Button variant="ghost" size="sm" onClick={clearProductImage} className="h-6 text-xs text-muted-foreground hover:text-destructive">
+                      清除
+                    </Button>
+                  )}
                </div>
                
                <div 
@ -703,13 +655,8 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
                  />
                  
                  {(productImagePreview || productImage) ? (
-                    <div className="relative mx-auto h-48 w-48 rounded-md border bg-muted overflow-hidden">
-                      <img
-                        src={productImagePreview || productImage || ""}
-                        alt="产品图"
-                        className="w-full h-full object-contain"
-                        onError={handleProductPreviewError}
-                      />
+                    <div className="relative aspect-square w-full max-w-[200px] mx-auto">
+                      <img src={productImagePreview || productImage || ""} alt="产品图" className="w-full h-full object-contain rounded-md" />
                      <div className="absolute inset-0 bg-black/40 opacity-0 group-hover:opacity-100 transition-opacity flex items-center justify-center rounded-md">
                        <Button variant="secondary" size="sm" onClick={(e) => { e.stopPropagation(); productInputRef.current?.click(); }}>
                          更换图片
@ -876,7 +823,7 @@ export function SceneFusionView({ onBack, initialProductImage, initialProductNam
              className="w-full" 
              size="lg" 
              onClick={startFusion}
-              disabled={isProcessing || isProductUploading || !productImage || scenes.length === 0}
+              disabled={isProcessing || !productImage || scenes.length === 0}
            >
              {isProcessing ? (
                <>
--- a/lib/ai/prompt-optimizer.ts
+++ b/lib/ai/prompt-optimizer.ts
@ -232,8 +232,9 @@ export async function optimizePromptWithLLM(params: {
  intent: string;
  subIntent?: string;
  historyMessages?: HistoryMessage[];
+  context?: string;
 }): Promise<OptimizeResult> {
-  const { originalPrompt, intent, subIntent, historyMessages } = params;
+  const { originalPrompt, intent, subIntent, historyMessages, context } = params;

  console.log("[Prompt Optimizer LLM] Input:", { 
    originalPrompt, 
@ -273,14 +274,17 @@ export async function optimizePromptWithLLM(params: {
      return optimizePrompt({ originalPrompt, intent, subIntent });
    }

-    // 构建包含历史上下文的消息
+    // 构建包含上下文的消息
    let contextMessage = "";
+    if (context && context.trim().length > 0) {
+      contextMessage = `## 画布上下文（仅用于理解，不需要在最终提示词中复述坐标）：\n${context.trim()}\n\n`;
+    }
    if (historyMessages && historyMessages.length > 0) {
      const recentHistory = historyMessages.slice(-6);  // 最近3轮对话
      const historyText = recentHistory.map(msg => 
        `${msg.role === 'user' ? '用户' : 'AI'}: ${msg.content.substring(0, 150)}`
      ).join('\n');
-      contextMessage = `## 对话历史：\n${historyText}\n\n`;
+      contextMessage += `## 对话历史：\n${historyText}\n\n`;
    }

    const userMessage = `${contextMessage}## 当前用户请求：\n"${originalPrompt}"\n\n## 识别的意图：${subIntent || '图片编辑'}\n\n请生成优化后的提示词：`;
--- a/lib/ai/smart-analyzer.ts
+++ b/lib/ai/smart-analyzer.ts
@ -74,6 +74,7 @@ const ANALYZER_SYSTEM_PROMPT = `你是一个友好的AI图片编辑助手。用
 - 可以加一些语气词让对话更亲切，如"好的"、"没问题"、"~"等
 - 说明你理解了什么需求，让用户知道你在做什么
 - 如果意图不明确，礼貌地询问用户想要什么效果
+ - 如果提供了画布上下文，仅用于理解，不要在 conversationalReply 中复述坐标或技术细节

 ## 示例
 用户："帮我把背景换成白色"
@ -145,8 +146,9 @@ export async function analyzeIntent(params: {
  imageUrls?: string[];
  imageFromHistory?: boolean;  // 🆕 图片是否来自历史记录
  historyMessages?: HistoryMessage[];  // 🆕 历史对话消息（用于上下文理解）
+  context?: string; // 🆕 画布上下文（仅用于理解）
 }): Promise<AnalysisResult> {
-  const { text, hasImage, imageUrls, imageFromHistory, historyMessages } = params;
+  const { text, hasImage, imageUrls, imageFromHistory, historyMessages, context } = params;

  console.log("[Smart Analyzer] Analyzing intent:", { 
    text, 
@ -165,7 +167,7 @@ export async function analyzeIntent(params: {

  // 尝试使用 Qwen3 VL 进行深度分析（传入历史对话）
  try {
-    const llmResult = await analyzWithLLM(text, hasImage, historyMessages);
+    const llmResult = await analyzWithLLM(text, hasImage, historyMessages, context);
    if (llmResult) {
      console.log("[Smart Analyzer] LLM analysis result:", llmResult);
      return llmResult;
@ -199,7 +201,8 @@ function quickKeywordMatch(text: string, hasImage: boolean): AnalysisResult {
 async function analyzWithLLM(
  text: string, 
  hasImage: boolean,
-  historyMessages?: HistoryMessage[]
+  historyMessages?: HistoryMessage[],
+  context?: string
 ): Promise<AnalysisResult | null> {
  // 使用动态导入避免循环依赖问题
  const { getEnabledAiModelsWithProvider } = await import("@/lib/db/queries");
@ -225,6 +228,11 @@ async function analyzWithLLM(
  // 🆕 构建包含历史上下文的消息
  let contextMessage = "";
  
+  // 🆕 画布上下文（仅用于理解，不要复述）
+  if (context && context.trim().length > 0) {
+    contextMessage = `## 画布上下文（仅用于理解，不要在对话回复中复述）：\n${context.trim()}\n\n`;
+  }
+  
  // 添加历史对话上下文（最多取最近5轮对话）
  if (historyMessages && historyMessages.length > 0) {
    const recentHistory = historyMessages.slice(-10);  // 最多10条消息（约5轮对话）
@ -232,7 +240,7 @@ async function analyzWithLLM(
      `${msg.role === 'user' ? '用户' : 'AI'}: ${msg.content.substring(0, 200)}`
    ).join('\n');
    
-    contextMessage = `## 对话历史（最近的对话上下文）：
+    contextMessage += `## 对话历史（最近的对话上下文）：
 ${historyText}

 ## 当前请求：