From 4c25d433f8593111d76c2586db804d54fb203204 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 19 Dec 2025 01:12:08 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=9B=BE=E7=89=87=E9=A2=84?= =?UTF-8?q?=E8=A7=88=E5=92=8C=E6=A0=87=E6=B3=A8=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/(chat)/api/chat/route.ts | 77 ++++- components/chat.tsx | 19 ++ components/elements/response.tsx | 7 +- components/gemini-image-loading.tsx | 133 ++++---- components/image-annotator.tsx | 474 ++++++++++++++++++++++++++ components/image-preview.tsx | 52 ++- components/message.tsx | 4 +- components/messages.tsx | 3 + components/multimodal-input.tsx | 14 + components/preview-attachment.tsx | 36 +- components/volcano-vision-loading.tsx | 4 +- lib/ai/providers/gemini-image.ts | 54 ++- lib/ai/providers/volcano-vision.ts | 17 +- 13 files changed, 799 insertions(+), 95 deletions(-) create mode 100644 components/image-annotator.tsx diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index e1f1c4886..51f936301 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -83,6 +83,27 @@ async function convertImageUrlToBase64(url: string): Promise { } } +/** + * 检测消息中是否包含用户标注过的图片（文件名以 annotated- 开头） + */ +function hasAnnotatedImage(messages: ChatMessage[]): boolean { + for (const message of messages) { + for (const part of message.parts) { + if (part.type === "file" && "name" in part && typeof part.name === "string") { + if (part.name.startsWith("annotated-")) { + return true; + } + } + } + } + return false; +} + +/** + * 标注图片的系统提示词（用户不可见，后台自动添加） + */ +const ANNOTATION_SYSTEM_PROMPT = "【重要】用户提供的图片上有标注线条（如箭头、圆圈、画笔痕迹或文字说明），这些标注是用户用来指示需要修改的区域或方向。请仔细理解标注的含义，按照标注指示进行编辑，但在生成的新图片中不要保留这些标注线条，输出干净的编辑结果。"; + /** * 自定义消息转换函数，支持将图片 URL 转换为 base64（仅用于 Gemini） */ @@ -93,18 +114,36 @@ async function convertMessagesWithImages(messages: ChatMessage[], providerType: return convertToModelMessages(messages); } + // 🆕 检测是否有标注图片 + const hasAnnotation = hasAnnotatedImage(messages); + if (hasAnnotation) { + console.log("[Message Convert] 🎨 检测到标注图片，将自动添加系统提示"); + } + // 对于 Gemini，需要特殊处理图片 const convertedMessages = []; for (const message of messages) { const content: any[] = []; + // 🆕 如果是用户消息且有标注图片，在文本前添加系统提示 + let hasAddedAnnotationPrompt = false; + // 处理消息的各个部分 for (const part of message.parts) { if (part.type === "text" && part.text) { + let textContent = part.text; + + // 🆕 如果有标注图片且是用户消息，在用户文本前添加系统提示 + if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt) { + textContent = ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + textContent; + hasAddedAnnotationPrompt = true; + console.log("[Message Convert] 🎨 已注入标注提示词"); + } + content.push({ type: "text", - text: part.text, + text: textContent, }); } else if (part.type === "file" && part.url) { // 将图片 URL 转换为 base64 @@ -130,6 +169,15 @@ async function convertMessagesWithImages(messages: ChatMessage[], providerType: } } + // 🆕 如果有标注图片但消息中没有文本部分，添加一个纯提示文本 + if (hasAnnotation && message.role === "user" && !hasAddedAnnotationPrompt && content.length > 0) { + content.unshift({ + type: "text", + text: ANNOTATION_SYSTEM_PROMPT, + }); + console.log("[Message Convert] 🎨 已在消息开头注入标注提示词"); + } + convertedMessages.push({ role: message.role, content: content.length === 1 && content[0].type === "text" @@ -800,11 +848,36 @@ export async function POST(request: Request) { if ((languageModel as any).provider === "volcano-vision" && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using Volcano Vision image generation"); + // 🆕 检测是否有标注图片，如果有则在消息中注入系统提示 + const hasAnnotation = hasAnnotatedImage(uiMessages); + let messagesForVolcano = uiMessages; + + if (hasAnnotation) { + console.log("[Chat API] 🎨 Volcano Vision: 检测到标注图片，将注入系统提示"); + // 创建消息副本，修改最后一条用户消息的文本 + messagesForVolcano = uiMessages.map((msg, idx) => { + if (msg.role === "user" && idx === uiMessages.length - 1) { + // 修改最后一条用户消息 + const modifiedParts = msg.parts.map((part) => { + if (part.type === "text" && part.text) { + return { + ...part, + text: ANNOTATION_SYSTEM_PROMPT + "\n\n用户请求：" + part.text, + }; + } + return part; + }); + return { ...msg, parts: modifiedParts }; + } + return msg; + }); + } + const volcanoStream = createUIMessageStream({ execute: async ({ writer }) => { try { // 转换消息，火山引擎支持 URL 和 base64 - const modelMessages = await convertMessagesWithImages(uiMessages, "volcano"); + const modelMessages = await convertMessagesWithImages(messagesForVolcano, "volcano"); // 调用 doGenerate 获取图片 const result = await (languageModel as any).doGenerate({ diff --git a/components/chat.tsx b/components/chat.tsx index c1c9dc708..735e4e545 100644 --- a/components/chat.tsx +++ b/components/chat.tsx @@ -372,6 +372,24 @@ export function Chat({ setInput((prev) => prev ? prev : "请帮我编辑这张图片："); }; + // 处理画图标注完成 + const handleDrawingComplete = (annotatedImageUrl: string) => { + // annotatedImageUrl 是 base64 data URL 格式 + const fileName = `annotated-${Date.now()}.png`; + + setAttachments((prev) => [ + ...prev, + { + url: annotatedImageUrl, + name: fileName, + contentType: "image/png", + }, + ]); + + // 如果输入框为空，自动填入提示词 + setInput((prev) => prev ? prev : "请根据我的标注修改这张图片："); + }; + useAutoResume({ autoResume, initialMessages, @@ -400,6 +418,7 @@ export function Chat({ status={status} votes={votes} onImageEdit={handleImageEdit} + onDrawingComplete={handleDrawingComplete} />
diff --git a/components/elements/response.tsx b/components/elements/response.tsx index b8dc8b00b..803dd9d55 100644 --- a/components/elements/response.tsx +++ b/components/elements/response.tsx @@ -8,10 +8,11 @@ import { VideoPreview } from "@/components/video-preview"; type ResponseProps = ComponentProps & { onImageEdit?: (imageUrl: string) => void; + onDrawingComplete?: (annotatedImageUrl: string) => void; }; export const Response = memo( - ({ className, onImageEdit, ...props }: ResponseProps) => ( + ({ className, onImageEdit, onDrawingComplete, ...props }: ResponseProps) => ( *:first-child]:mt-0 [&>*:last-child]:mb-0 [&_code]:whitespace-pre-wrap [&_code]:break-words [&_pre]:max-w-full [&_pre]:overflow-x-auto", @@ -22,7 +23,7 @@ export const Response = memo( if (!src) { return null; } - return ; + return ; }, video: ({ src, alt }: { src?: string; alt?: string }) => { if (!src) { @@ -35,7 +36,7 @@ export const Response = memo( /> ), (prevProps: ResponseProps, nextProps: ResponseProps) => - prevProps.children === nextProps.children && prevProps.onImageEdit === nextProps.onImageEdit + prevProps.children === nextProps.children && prevProps.onImageEdit === nextProps.onImageEdit && prevProps.onDrawingComplete === nextProps.onDrawingComplete ); Response.displayName = "Response"; diff --git a/components/gemini-image-loading.tsx b/components/gemini-image-loading.tsx index 860285513..d5ff5fc72 100644 --- a/components/gemini-image-loading.tsx +++ b/components/gemini-image-loading.tsx @@ -5,7 +5,7 @@ import { SparklesIcon } from "./icons"; /** * Gemini 图片生成加载占位符组件 - * 显示一个 170x200 像素的模糊动态效果 + * 显示 2 个 170x200 像素的模糊动态效果 */ export const GeminiImageLoading = memo(() => { return ( @@ -30,7 +30,7 @@ export const GeminiImageLoading = memo(() => { {/* 文本提示 */}
- 正在生成图片 + 正在生成 2 张图片 {
- {/* 图片占位符 - 170x200 像素，带模糊动态效果 */} -
- {/* 底层渐变背景 */} -
- - {/* 动态模糊叠加层 1 */} -
- - {/* 动态模糊叠加层 2 */} -
- - {/* 噪点纹理效果 */} -
- - {/* 模糊叠加 */} -
- - {/* 中心图标 */} -
-
- {/* 外圈光晕 */} -
- {/* 图片图标 */} - - - + {/* 2 个图片占位符 - 横向排列，每个 170x200px */} +
+ {[0, 1].map((index) => ( +
+ {/* 底层渐变背景 */} +
+ + {/* 动态模糊叠加层 1 */} +
+ + {/* 动态模糊叠加层 2 */} +
+ + {/* 噪点纹理效果 */} +
+ + {/* 模糊叠加 */} +
+ + {/* 中心图标 */} +
+
+ {/* 外圈光晕 */} +
+ {/* 图片图标 */} + + + +
+
+ + {/* 流光效果 */} +
-
- - {/* 流光效果 */} -
+ ))}
{/* 提示文字 */} - AI 正在创作中，请稍候... + Gemini 正在创作 2 张图片，请稍候...
@@ -134,4 +140,3 @@ export const GeminiImageLoading = memo(() => { }); GeminiImageLoading.displayName = "GeminiImageLoading"; - diff --git a/components/image-annotator.tsx b/components/image-annotator.tsx new file mode 100644 index 000000000..d44f81123 --- /dev/null +++ b/components/image-annotator.tsx @@ -0,0 +1,474 @@ +"use client"; + +import React, { useRef, useState, useEffect, useCallback } from "react"; +import { X, Pencil, Circle, ArrowRight, Type, Check, Undo, Trash2 } from "lucide-react"; +import { Button } from "./ui/button"; + +type Tool = "pen" | "arrow" | "circle" | "text"; + +type DrawingAction = { + type: Tool; + color: string; + lineWidth: number; + points?: { x: number; y: number }[]; + startX?: number; + startY?: number; + endX?: number; + endY?: number; + radius?: number; + text?: string; + fontSize?: number; +}; + +interface ImageAnnotatorProps { + imageUrl: string; + onConfirm: (annotatedImageUrl: string) => void; + onCancel: () => void; +} + +const COLORS = [ + "#FF0000", // 红色 + "#FF6B00", // 橙色 + "#FFCC00", // 黄色 + "#00CC00", // 绿色 + "#0099FF", // 蓝色 + "#9933FF", // 紫色 + "#FF0099", // 粉色 + "#FFFFFF", // 白色 + "#000000", // 黑色 +]; + +const LINE_WIDTHS = [2, 4, 6, 8]; + +export function ImageAnnotator({ imageUrl, onConfirm, onCancel }: ImageAnnotatorProps) { + const canvasRef = useRef(null); + const containerRef = useRef(null); + const [ctx, setCtx] = useState(null); + const [image, setImage] = useState(null); + const [isDrawing, setIsDrawing] = useState(false); + const [tool, setTool] = useState("pen"); + const [color, setColor] = useState("#FF0000"); + const [lineWidth, setLineWidth] = useState(4); + const [actions, setActions] = useState([]); + const [currentAction, setCurrentAction] = useState(null); + const [textInput, setTextInput] = useState(""); + const [textPosition, setTextPosition] = useState<{ x: number; y: number } | null>(null); + const [scale, setScale] = useState(1); + const [offset, setOffset] = useState({ x: 0, y: 0 }); + + // 加载图片 + useEffect(() => { + const img = new window.Image(); + img.crossOrigin = "anonymous"; + img.onload = () => { + setImage(img); + }; + img.src = imageUrl; + }, [imageUrl]); + + // 初始化 canvas - 适配弹窗大小 + useEffect(() => { + if (!canvasRef.current || !image || !containerRef.current) return; + + const canvas = canvasRef.current; + const container = containerRef.current; + const context = canvas.getContext("2d"); + if (!context) return; + + // 弹窗内画布区域大小（减去工具栏和边距） + const containerWidth = 800; // 850 - padding + const containerHeight = 350; // 550 - 工具栏高度 + const scaleX = containerWidth / image.width; + const scaleY = containerHeight / image.height; + const newScale = Math.min(scaleX, scaleY, 1); // 不放大，只缩小 + + setScale(newScale); + + canvas.width = image.width * newScale; + canvas.height = image.height * newScale; + + // 计算居中偏移 + const offsetX = (containerWidth - canvas.width) / 2; + const offsetY = (containerHeight - canvas.height) / 2; + setOffset({ x: Math.max(0, offsetX), y: Math.max(0, offsetY) }); + + setCtx(context); + redrawCanvas(context, image, newScale, []); + }, [image]); + + // 重绘画布 + const redrawCanvas = useCallback( + (context: CanvasRenderingContext2D, img: HTMLImageElement, currentScale: number, drawActions: DrawingAction[]) => { + const canvas = context.canvas; + context.clearRect(0, 0, canvas.width, canvas.height); + + // 绘制图片 + context.drawImage(img, 0, 0, canvas.width, canvas.height); + + // 绘制所有动作 + for (const action of drawActions) { + drawAction(context, action, currentScale); + } + }, + [] + ); + + // 当 actions 变化时重绘 + useEffect(() => { + if (ctx && image) { + redrawCanvas(ctx, image, scale, actions); + } + }, [actions, ctx, image, scale, redrawCanvas]); + + // 绘制单个动作 + const drawAction = (context: CanvasRenderingContext2D, action: DrawingAction, currentScale: number) => { + context.strokeStyle = action.color; + context.fillStyle = action.color; + context.lineWidth = action.lineWidth; + context.lineCap = "round"; + context.lineJoin = "round"; + + switch (action.type) { + case "pen": + if (action.points && action.points.length > 0) { + context.beginPath(); + context.moveTo(action.points[0].x * currentScale, action.points[0].y * currentScale); + for (let i = 1; i < action.points.length; i++) { + context.lineTo(action.points[i].x * currentScale, action.points[i].y * currentScale); + } + context.stroke(); + } + break; + + case "arrow": + if (action.startX !== undefined && action.startY !== undefined && action.endX !== undefined && action.endY !== undefined) { + const sx = action.startX * currentScale; + const sy = action.startY * currentScale; + const ex = action.endX * currentScale; + const ey = action.endY * currentScale; + + // 绘制线条 + context.beginPath(); + context.moveTo(sx, sy); + context.lineTo(ex, ey); + context.stroke(); + + // 绘制箭头 + const angle = Math.atan2(ey - sy, ex - sx); + const arrowLength = 15 + action.lineWidth * 2; + context.beginPath(); + context.moveTo(ex, ey); + context.lineTo(ex - arrowLength * Math.cos(angle - Math.PI / 6), ey - arrowLength * Math.sin(angle - Math.PI / 6)); + context.lineTo(ex - arrowLength * Math.cos(angle + Math.PI / 6), ey - arrowLength * Math.sin(angle + Math.PI / 6)); + context.closePath(); + context.fill(); + } + break; + + case "circle": + if (action.startX !== undefined && action.startY !== undefined && action.radius !== undefined) { + context.beginPath(); + context.arc(action.startX * currentScale, action.startY * currentScale, action.radius * currentScale, 0, 2 * Math.PI); + context.stroke(); + } + break; + + case "text": + if (action.startX !== undefined && action.startY !== undefined && action.text) { + const fontSize = (action.fontSize || 20) * currentScale; + context.font = `bold ${fontSize}px sans-serif`; + context.fillText(action.text, action.startX * currentScale, action.startY * currentScale); + } + break; + } + }; + + // 获取鼠标/触摸位置（转换为原始图片坐标） + const getPosition = (e: React.MouseEvent | React.TouchEvent) => { + const canvas = canvasRef.current; + if (!canvas) return { x: 0, y: 0 }; + + const rect = canvas.getBoundingClientRect(); + let clientX: number; + let clientY: number; + + if ("touches" in e) { + clientX = e.touches[0].clientX; + clientY = e.touches[0].clientY; + } else { + clientX = e.clientX; + clientY = e.clientY; + } + + // 转换为原始图片坐标 + return { + x: (clientX - rect.left) / scale, + y: (clientY - rect.top) / scale, + }; + }; + + // 开始绘制 + const handleStart = (e: React.MouseEvent | React.TouchEvent) => { + if (tool === "text") { + const pos = getPosition(e); + setTextPosition(pos); + return; + } + + setIsDrawing(true); + const pos = getPosition(e); + + const newAction: DrawingAction = { + type: tool, + color, + lineWidth, + }; + + if (tool === "pen") { + newAction.points = [pos]; + } else { + newAction.startX = pos.x; + newAction.startY = pos.y; + newAction.endX = pos.x; + newAction.endY = pos.y; + if (tool === "circle") { + newAction.radius = 0; + } + } + + setCurrentAction(newAction); + }; + + // 绘制中 + const handleMove = (e: React.MouseEvent | React.TouchEvent) => { + if (!isDrawing || !currentAction || !ctx || !image) return; + + const pos = getPosition(e); + + const updatedAction = { ...currentAction }; + + if (tool === "pen" && updatedAction.points) { + updatedAction.points = [...updatedAction.points, pos]; + } else if (tool === "arrow") { + updatedAction.endX = pos.x; + updatedAction.endY = pos.y; + } else if (tool === "circle" && updatedAction.startX !== undefined && updatedAction.startY !== undefined) { + const dx = pos.x - updatedAction.startX; + const dy = pos.y - updatedAction.startY; + updatedAction.radius = Math.sqrt(dx * dx + dy * dy); + } + + setCurrentAction(updatedAction); + + // 实时重绘 + redrawCanvas(ctx, image, scale, [...actions, updatedAction]); + }; + + // 结束绘制 + const handleEnd = () => { + if (!isDrawing || !currentAction) return; + + setIsDrawing(false); + setActions((prev) => [...prev, currentAction]); + setCurrentAction(null); + }; + + // 添加文字 + const handleAddText = () => { + if (!textInput.trim() || !textPosition) return; + + const textAction: DrawingAction = { + type: "text", + color, + lineWidth, + startX: textPosition.x, + startY: textPosition.y, + text: textInput, + fontSize: 20 + lineWidth * 2, + }; + + setActions((prev) => [...prev, textAction]); + setTextInput(""); + setTextPosition(null); + }; + + // 撤销 + const handleUndo = () => { + setActions((prev) => prev.slice(0, -1)); + }; + + // 清空 + const handleClear = () => { + setActions([]); + }; + + // 确认 + const handleConfirm = () => { + const canvas = canvasRef.current; + if (!canvas || !image) return; + + // 创建一个新的 canvas 用于导出原始尺寸 + const exportCanvas = document.createElement("canvas"); + exportCanvas.width = image.width; + exportCanvas.height = image.height; + const exportCtx = exportCanvas.getContext("2d"); + if (!exportCtx) return; + + // 绘制原始图片 + exportCtx.drawImage(image, 0, 0); + + // 绘制所有动作（使用原始尺寸） + for (const action of actions) { + drawAction(exportCtx, action, 1); + } + + // 导出为 data URL + const dataUrl = exportCanvas.toDataURL("image/png"); + onConfirm(dataUrl); + }; + + return ( +
+ {/* 弹窗容器 */} +
e.stopPropagation()} + > + {/* 顶部标题栏 */} +
+ 图片标注 +
+ + + + +
+
+ + {/* 工具选择栏 */} +
+ {/* 工具按钮 */} +
+ + + + +
+ + {/* 颜色选择 */} +
+ {COLORS.map((c) => ( +
+ + {/* 线宽选择 */} +
+ {LINE_WIDTHS.map((w) => ( + + ))} +
+
+ + {/* 画布区域 */} +
+ +
+
+ + {/* 文字输入弹窗 */} + {textPosition && ( +
setTextPosition(null)}> +
e.stopPropagation()}> +
输入标注文字
+ setTextInput(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter") handleAddText(); + }} + className="mb-3 w-64 rounded-md border bg-muted px-3 py-2 text-foreground outline-none focus:border-primary" + placeholder="输入文字..." + autoFocus + /> +
+ + +
+
+
+ )} +
+ ); +} diff --git a/components/image-preview.tsx b/components/image-preview.tsx index 9b4aed5ba..f822cc47c 100644 --- a/components/image-preview.tsx +++ b/components/image-preview.tsx @@ -1,14 +1,16 @@ "use client"; import React, { useState } from "react"; -import { X, Download, Edit, Target } from "lucide-react"; +import { X, Download, Edit, Target, Paintbrush } from "lucide-react"; import { toast } from "sonner"; +import { ImageAnnotator } from "./image-annotator"; export function ImagePreview({ src, alt, onImageEdit, onAnnotate, + onDrawingComplete, imageIndex }: { src: string; @@ -19,11 +21,13 @@ export function ImagePreview({ description: string; formattedText: string; }) => void; + onDrawingComplete?: (annotatedImageUrl: string) => void; imageIndex?: number; }) { const [isOpen, setIsOpen] = useState(false); const [isAnnotationMode, setIsAnnotationMode] = useState(false); const [isDetecting, setIsDetecting] = useState(false); + const [showAnnotator, setShowAnnotator] = useState(false); // 解析 alt 文本，提取原图 URL 和宽度 // 格式优先级： @@ -197,6 +201,22 @@ export function ImagePreview({ {/* 悬停时显示的操作按钮 */}
+ {/* 画图按钮 */} + {onDrawingComplete && ( + + )} + {/* 编辑按钮 */} {onImageEdit && ( + )} + {/* 编辑按钮 */} {onImageEdit && (
)} + + {/* 图片标注器 */} + {showAnnotator && onDrawingComplete && ( + { + onDrawingComplete(annotatedImageUrl); + setShowAnnotator(false); + toast.success("标注图片已添加到聊天"); + }} + onCancel={() => setShowAnnotator(false)} + /> + )} ); } diff --git a/components/message.tsx b/components/message.tsx index 3d1a3e3b8..fc8884ac9 100644 --- a/components/message.tsx +++ b/components/message.tsx @@ -146,6 +146,7 @@ const PurePreviewMessage = ({ isReadonly, requiresScrollPadding: _requiresScrollPadding, onImageEdit, + onDrawingComplete, }: { chatId: string; message: ChatMessage; @@ -157,6 +158,7 @@ const PurePreviewMessage = ({ isReadonly: boolean; requiresScrollPadding: boolean; onImageEdit?: (imageUrl: string) => void; + onDrawingComplete?: (annotatedImageUrl: string) => void; }) => { const [mode, setMode] = useState<"view" | "edit">("view"); @@ -265,7 +267,7 @@ const PurePreviewMessage = ({ ); } return ( - + {sanitizeText(part.content)} ); diff --git a/components/messages.tsx b/components/messages.tsx index 41dd7723a..bb7f82f4b 100644 --- a/components/messages.tsx +++ b/components/messages.tsx @@ -35,6 +35,7 @@ type MessagesProps = { isArtifactVisible: boolean; selectedModelId: string; onImageEdit?: (imageUrl: string) => void; + onDrawingComplete?: (annotatedImageUrl: string) => void; }; function PureMessages({ @@ -48,6 +49,7 @@ function PureMessages({ isReadonly, selectedModelId, onImageEdit, + onDrawingComplete, }: MessagesProps) { const { containerRef: messagesContainerRef, @@ -212,6 +214,7 @@ function PureMessages({ : undefined } onImageEdit={onImageEdit} + onDrawingComplete={onDrawingComplete} /> ))} diff --git a/components/multimodal-input.tsx b/components/multimodal-input.tsx index 2351a3562..45e0c53fa 100644 --- a/components/multimodal-input.tsx +++ b/components/multimodal-input.tsx @@ -749,6 +749,20 @@ function PureMultimodalInput({ }); toast.success(`已添加标注：${annotation.description}`); }} + onDrawingComplete={(annotatedImageUrl) => { + // 将标注后的图片替换原图片（或添加为新附件） + const fileName = `annotated-${Date.now()}.png`; + setAttachments((prev) => [ + ...prev, + { + url: annotatedImageUrl, + name: fileName, + contentType: "image/png", + }, + ]); + // 如果输入框为空，自动填入提示词 + setInput((prev) => prev || "请根据我的标注修改这张图片："); + }} imageIndex={index + 1} /> ))} diff --git a/components/preview-attachment.tsx b/components/preview-attachment.tsx index baf8a95f9..84162da53 100644 --- a/components/preview-attachment.tsx +++ b/components/preview-attachment.tsx @@ -6,8 +6,9 @@ import type { Attachment } from "@/lib/types"; import { Loader } from "./elements/loader"; import { CrossSmallIcon } from "./icons"; import { Button } from "./ui/button"; -import { X, Download, Target } from "lucide-react"; +import { X, Download, Target, Paintbrush } from "lucide-react"; import { toast } from "sonner"; +import { ImageAnnotator } from "./image-annotator"; export const PreviewAttachment = ({ attachment, @@ -19,6 +20,7 @@ export const PreviewAttachment = ({ onDrop, draggable = false, onAnnotate, + onDrawingComplete, imageIndex, }: { attachment: Attachment; @@ -34,6 +36,7 @@ export const PreviewAttachment = ({ description: string; formattedText: string; }) => void; + onDrawingComplete?: (annotatedImageUrl: string) => void; imageIndex?: number; }) => { const { name, url, contentType } = attachment; @@ -41,6 +44,7 @@ export const PreviewAttachment = ({ const [isDragging, setIsDragging] = useState(false); const [isAnnotationMode, setIsAnnotationMode] = useState(false); const [isDetecting, setIsDetecting] = useState(false); + const [showAnnotator, setShowAnnotator] = useState(false); const [annotationPoints, setAnnotationPoints] = useState )} + {/* 画图标注按钮 */} + {onDrawingComplete && ( + + )} + {/* 下载按钮 */}
)} + + {/* 图片标注器 */} + {showAnnotator && onDrawingComplete && ( + { + onDrawingComplete(annotatedImageUrl); + setShowAnnotator(false); + toast.success("标注图片已添加到聊天"); + }} + onCancel={() => setShowAnnotator(false)} + /> + )} ); }; diff --git a/components/volcano-vision-loading.tsx b/components/volcano-vision-loading.tsx index 09e3e8245..0373e8726 100644 --- a/components/volcano-vision-loading.tsx +++ b/components/volcano-vision-loading.tsx @@ -48,13 +48,13 @@ export const VolcanoVisionLoading = memo(() => {
- {/* 3 个图片占位符 - 横向排列，每个最大宽度 250px */} + {/* 3 个图片占位符 - 横向排列，每个 170x200px */}
{[0, 1, 2].map((index) => (
{/* 底层渐变背景 */}
diff --git a/lib/ai/providers/gemini-image.ts b/lib/ai/providers/gemini-image.ts index dcc7e336d..52c9c3f1c 100644 --- a/lib/ai/providers/gemini-image.ts +++ b/lib/ai/providers/gemini-image.ts @@ -574,13 +574,30 @@ export function createGeminiImage({ apiKey, baseURL }: GeminiImageConfig) { imageSize: options.imageParams?.imageSize || "2K" }; console.log("[Gemini Image] Final imageParams after defaults:", imageParams); - const response = await sendGeminiRequest(baseURL, apiKey, options.messages, modelId, imageParams); - // 提取图片并上传 base64 到 S3 - const imageUrls = await extractAndUploadImages(response); - const markdownText = formatImageResponse(imageUrls, imageParams.imageSize); + + // 🆕 并发发起两个请求，生成2张图片（类似豆包模型） + console.log("[Gemini Image] Starting 2 concurrent requests..."); + const startTime = Date.now(); + + const [response1, response2] = await Promise.all([ + sendGeminiRequest(baseURL, apiKey, options.messages, modelId, imageParams), + sendGeminiRequest(baseURL, apiKey, options.messages, modelId, imageParams), + ]); + + console.log(`[Gemini Image] Both requests completed in ${Date.now() - startTime}ms`); + + // 提取图片并上传 base64 到 S3（并行处理） + const [imageUrls1, imageUrls2] = await Promise.all([ + extractAndUploadImages(response1), + extractAndUploadImages(response2), + ]); + + // 合并两次请求的图片结果 + const allImageUrls = [...imageUrls1, ...imageUrls2]; + const markdownText = formatImageResponse(allImageUrls, imageParams.imageSize); console.log( - `[Gemini Image] Generated ${imageUrls.length} images (uploaded to S3)` + `[Gemini Image] Generated ${allImageUrls.length} images from 2 concurrent requests (uploaded to S3)` ); // 返回 AI SDK v1 格式 @@ -614,13 +631,30 @@ export function createGeminiImage({ apiKey, baseURL }: GeminiImageConfig) { imageSize: options.imageParams?.imageSize || "2K" }; console.log("[Gemini Image] Final imageParams after defaults:", imageParams); - const response = await sendGeminiRequest(baseURL, apiKey, options.messages, modelId, imageParams); - // 提取图片并上传 base64 到 S3 - const imageUrls = await extractAndUploadImages(response); - const markdownText = formatImageResponse(imageUrls, imageParams.imageSize); + + // 🆕 并发发起两个请求，生成2张图片（类似豆包模型） + console.log("[Gemini Image Stream] Starting 2 concurrent requests..."); + const startTime = Date.now(); + + const [response1, response2] = await Promise.all([ + sendGeminiRequest(baseURL, apiKey, options.messages, modelId, imageParams), + sendGeminiRequest(baseURL, apiKey, options.messages, modelId, imageParams), + ]); + + console.log(`[Gemini Image Stream] Both requests completed in ${Date.now() - startTime}ms`); + + // 提取图片并上传 base64 到 S3（并行处理） + const [imageUrls1, imageUrls2] = await Promise.all([ + extractAndUploadImages(response1), + extractAndUploadImages(response2), + ]); + + // 合并两次请求的图片结果 + const allImageUrls = [...imageUrls1, ...imageUrls2]; + const markdownText = formatImageResponse(allImageUrls, imageParams.imageSize); console.log( - `[Gemini Image Stream] Generated ${imageUrls.length} images (uploaded to S3)` + `[Gemini Image Stream] Generated ${allImageUrls.length} images from 2 concurrent requests (uploaded to S3)` ); // 一次性发送整个图片 markdown（不再分块，因为 base64 图片很大） diff --git a/lib/ai/providers/volcano-vision.ts b/lib/ai/providers/volcano-vision.ts index 3b3d34c7e..e7632182c 100644 --- a/lib/ai/providers/volcano-vision.ts +++ b/lib/ai/providers/volcano-vision.ts @@ -183,17 +183,12 @@ function formatImageResponse(images: Array<{ thumbnail: string; original: string return "生成图片失败，未找到图片数据"; } - // 如果只有一张图片，使用 Markdown 格式 - if (images.length === 1) { - return `![thumb|${images[0].original}](${images[0].thumbnail})`; - } - - // 多张图片：每张图片用空格分隔，会自动横向排列 - const imagesMarkdown = images - .map((img) => `![thumb|${img.original}|250](${img.thumbnail})`) - .join(' '); - - return imagesMarkdown; + // 🆕 统一使用 ||| 分隔符格式（与 Gemini 保持一致） + // 格式: ![thumb|||原图URL|||尺寸](缩略图URL) + // 豆包固定生成 2K 图片 + return images + .map((img) => `![thumb|||${img.original}|||2K](${img.thumbnail})`) + .join("\n\n"); } /**