feat: 添加物品拾取功能（MVP版本）
- 新增 ObjectPickerOverlay 组件，支持点击图片拾取物品 - 新增 /api/pick-object API，调用 Qwen-VL 识别物品 - Canvas Store 扩展拾取模式状态管理 - ImageNode 工具栏新增拾取按钮 - CanvasChatPanel 集成物品标签，发送时自动构建精准prompt - 图片压缩至 600px 以内加速 AI 分析 - 物品名称限制 8 字，位置固定 4 字标准词库
This commit is contained in:
Your Name
parent
3fc00270c4
commit
47afddfd73
221
app/api/pick-object/route.ts
Normal file
221
app/api/pick-object/route.ts
Normal file
@ -0,0 +1,221 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { db } from "@/lib/db";
|
||||
import { aiProvider } from "@/lib/db/schema";
|
||||
import { eq } from "drizzle-orm";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 30;
|
||||
|
||||
// 位置词库
|
||||
const POSITION_OPTIONS = [
|
||||
'左上区域', '中央顶部', '右上区域',
|
||||
'左侧中部', '中央区域', '右侧中部',
|
||||
'左下区域', '中央底部', '右下区域'
|
||||
] as const;
|
||||
|
||||
interface PickObjectRequest {
|
||||
imageData: string; // Base64 压缩后的图片
|
||||
clickX: number; // 点击位置 X (0-1)
|
||||
clickY: number; // 点击位置 Y (0-1)
|
||||
originalImageUrl: string; // 原图 URL
|
||||
}
|
||||
|
||||
interface PickObjectResult {
|
||||
name: string;
|
||||
position: string;
|
||||
confidence: number;
|
||||
boundingBox: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
const body: PickObjectRequest = await request.json();
|
||||
const { imageData, clickX, clickY } = body;
|
||||
|
||||
// 验证参数
|
||||
if (!imageData) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: '缺少图片数据',
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
if (typeof clickX !== 'number' || typeof clickY !== 'number') {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: '缺少点击坐标',
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
// 获取 Qwen-VL 配置
|
||||
const qwenProvider = await db.query.aiProvider.findFirst({
|
||||
where: eq(aiProvider.providerId, 'qwen'),
|
||||
});
|
||||
|
||||
if (!qwenProvider || !qwenProvider.isEnabled || !qwenProvider.apiKey) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'Qwen 服务未配置',
|
||||
}, { status: 500 });
|
||||
}
|
||||
|
||||
// 构建 prompt（严格限制输出格式）
|
||||
const prompt = `
|
||||
识别图片中坐标 (${(clickX * 100).toFixed(0)}%, ${(clickY * 100).toFixed(0)}%) 位置的物品。
|
||||
|
||||
【输出要求 - 严格遵守】
|
||||
返回纯 JSON，无其他文字：
|
||||
{
|
||||
"name": "物品名称",
|
||||
"position": "位置描述",
|
||||
"confidence": 0.95,
|
||||
"boundingBox": { "x": 0.1, "y": 0.2, "width": 0.3, "height": 0.4 }
|
||||
}
|
||||
|
||||
【name 格式】≤8字，格式：[颜色/材质]+[物品]
|
||||
正确：黑色沙发、白色花瓶、木质书架、玻璃茶几
|
||||
错误：一个黑色的皮质沙发（太长）
|
||||
|
||||
【position 格式】固定4字，从以下选择：
|
||||
${POSITION_OPTIONS.join('、')}
|
||||
|
||||
【boundingBox】相对坐标 0-1，精确到小数点后2位
|
||||
`.trim();
|
||||
|
||||
// 调用 Qwen-VL API
|
||||
const response = await fetch(`${qwenProvider.apiBaseUrl || 'https://dashscope.aliyuncs.com/compatible-mode/v1'}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${qwenProvider.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'qwen-vl-max',
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: {
|
||||
url: imageData, // Base64 格式的图片
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'text',
|
||||
text: prompt,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
max_tokens: 200,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
console.error('[PickObject] Qwen API error:', errorText);
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'AI 识别服务错误',
|
||||
}, { status: 500 });
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const content = data.choices?.[0]?.message?.content;
|
||||
|
||||
if (!content) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'AI 未返回结果',
|
||||
}, { status: 500 });
|
||||
}
|
||||
|
||||
// 解析 JSON 结果
|
||||
let result: PickObjectResult;
|
||||
try {
|
||||
// 尝试从返回内容中提取 JSON
|
||||
const jsonMatch = content.match(/\{[\s\S]*\}/);
|
||||
if (!jsonMatch) {
|
||||
throw new Error('未找到 JSON');
|
||||
}
|
||||
result = JSON.parse(jsonMatch[0]);
|
||||
|
||||
// 验证和修正结果
|
||||
if (!result.name || result.name.length > 12) {
|
||||
result.name = result.name?.substring(0, 8) || '未知物品';
|
||||
}
|
||||
|
||||
// 验证位置是否在词库中
|
||||
if (!POSITION_OPTIONS.includes(result.position as any)) {
|
||||
// 根据点击位置推算
|
||||
const posX = clickX < 0.33 ? '左' : clickX > 0.66 ? '右' : '中央';
|
||||
const posY = clickY < 0.33 ? '上' : clickY > 0.66 ? '下' : '中';
|
||||
if (posX === '中央') {
|
||||
result.position = posY === '中' ? '中央区域' : posY === '上' ? '中央顶部' : '中央底部';
|
||||
} else {
|
||||
result.position = posY === '中' ? `${posX}侧中部` : `${posX}${posY}区域`;
|
||||
}
|
||||
}
|
||||
|
||||
// 确保 boundingBox 有效
|
||||
if (!result.boundingBox ||
|
||||
typeof result.boundingBox.x !== 'number' ||
|
||||
typeof result.boundingBox.y !== 'number') {
|
||||
// 根据点击位置生成默认边界框
|
||||
result.boundingBox = {
|
||||
x: Math.max(0, clickX - 0.15),
|
||||
y: Math.max(0, clickY - 0.15),
|
||||
width: 0.3,
|
||||
height: 0.3,
|
||||
};
|
||||
}
|
||||
|
||||
// 确保 confidence 有效
|
||||
if (typeof result.confidence !== 'number' || result.confidence < 0 || result.confidence > 1) {
|
||||
result.confidence = 0.85;
|
||||
}
|
||||
|
||||
} catch (parseError) {
|
||||
console.error('[PickObject] JSON parse error:', parseError, 'Content:', content);
|
||||
|
||||
// 返回默认结果
|
||||
const posX = clickX < 0.33 ? '左' : clickX > 0.66 ? '右' : '中央';
|
||||
const posY = clickY < 0.33 ? '上' : clickY > 0.66 ? '下' : '中';
|
||||
const position = posX === '中央'
|
||||
? (posY === '中' ? '中央区域' : posY === '上' ? '中央顶部' : '中央底部')
|
||||
: (posY === '中' ? `${posX}侧中部` : `${posX}${posY}区域`);
|
||||
|
||||
result = {
|
||||
name: '物品',
|
||||
position,
|
||||
confidence: 0.5,
|
||||
boundingBox: {
|
||||
x: Math.max(0, clickX - 0.15),
|
||||
y: Math.max(0, clickY - 0.15),
|
||||
width: 0.3,
|
||||
height: 0.3,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
console.log('[PickObject] 识别结果:', result);
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
object: result,
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('[PickObject] Error:', error);
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: `识别失败: ${error instanceof Error ? error.message : '未知错误'}`,
|
||||
}, { status: 500 });
|
||||
}
|
||||
}
|
||||
@ -63,6 +63,7 @@ import { canvasService, imageGenerationService } from "@/services/canvas";
|
||||
import { CanvasWorkspace } from "@/components/canvas";
|
||||
import { CanvasChatPanel } from "@/components/canvas/CanvasChatPanel";
|
||||
import { CanvasFileListPanel } from "@/components/canvas/CanvasFileListPanel";
|
||||
import { ObjectPickerOverlay } from "@/components/canvas/ObjectPickerOverlay";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { MessageSquare, FolderOpen, Grid2X2 } from "lucide-react";
|
||||
@ -713,6 +714,8 @@ export default function CanvasEditorPage({ params }: PageProps) {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* 6. 物品拾取模式覆盖层 */}
|
||||
<ObjectPickerOverlay />
|
||||
</div>
|
||||
</TooltipProvider>
|
||||
);
|
||||
|
||||
@ -99,10 +99,21 @@ function extractImageUrls(text: string): string[] {
|
||||
return urls;
|
||||
}
|
||||
|
||||
// 🆕 拾取物品的附件类型扩展
|
||||
interface PickedObjectAttachment extends Attachment {
|
||||
pickedObject?: {
|
||||
objectName: string;
|
||||
objectPosition: string;
|
||||
clickX: number;
|
||||
clickY: number;
|
||||
boundingBox: { x: number; y: number; width: number; height: number };
|
||||
};
|
||||
}
|
||||
|
||||
export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenFileList }: CanvasChatPanelProps) {
|
||||
const router = useRouter();
|
||||
const [input, setInput] = useState("");
|
||||
const [attachments, setAttachments] = useState<Attachment[]>([]);
|
||||
const [attachments, setAttachments] = useState<PickedObjectAttachment[]>([]);
|
||||
const [selectedModelId, setSelectedModelId] = useState<string>("");
|
||||
const messagesEndRef = useRef<HTMLDivElement>(null);
|
||||
const fileInputRef = useRef<HTMLInputElement>(null);
|
||||
@ -145,6 +156,9 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
const clearSelection = useCanvasStore((state) => state.clearSelection);
|
||||
const pendingChatAttachments = useCanvasStore((state) => state.pendingChatAttachments);
|
||||
const clearPendingChatAttachments = useCanvasStore((state) => state.clearPendingChatAttachments);
|
||||
// 🆕 物品拾取
|
||||
const pendingPickedObjects = useCanvasStore((state) => state.pendingPickedObjects);
|
||||
const clearPendingPickedObjects = useCanvasStore((state) => state.clearPendingPickedObjects);
|
||||
|
||||
// 获取纯 UUID（去掉 canvas- 前缀）用于 API 调用
|
||||
const rawChatId = chatId.startsWith('canvas-')
|
||||
@ -201,6 +215,57 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
console.log('[CanvasChat] 已插入图片标签:', url.substring(0, 50));
|
||||
}, []);
|
||||
|
||||
// 🆕 辅助函数：在 contenteditable 中插入物品标签（带🎯图标）
|
||||
const insertPickedObjectTagInEditor = useCallback((
|
||||
url: string,
|
||||
objectName: string,
|
||||
objectPosition: string,
|
||||
index: number
|
||||
) => {
|
||||
const editableDiv = textareaRef.current as HTMLDivElement | null;
|
||||
if (!editableDiv || !editableDiv.isContentEditable) return;
|
||||
|
||||
// 检查 DOM 中是否已存在该物品的标签
|
||||
const existingTags = editableDiv.querySelectorAll('[data-picked-object="true"]');
|
||||
for (const tag of existingTags) {
|
||||
if (tag.getAttribute('data-attachment-url') === url &&
|
||||
tag.getAttribute('data-object-name') === objectName) {
|
||||
console.log('[CanvasChat] 物品标签已存在，跳过');
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const tagSpan = document.createElement('span');
|
||||
tagSpan.contentEditable = 'false';
|
||||
tagSpan.setAttribute('data-attachment-index', String(index));
|
||||
tagSpan.setAttribute('data-attachment-url', url);
|
||||
tagSpan.setAttribute('data-picked-object', 'true');
|
||||
tagSpan.setAttribute('data-object-name', objectName);
|
||||
tagSpan.className = 'inline-flex items-center gap-1 bg-orange-50 hover:bg-orange-100 border border-orange-200 rounded-full pl-1.5 pr-2 py-0.5 mx-0.5 align-middle cursor-default select-none';
|
||||
tagSpan.innerHTML = `
|
||||
<span class="text-orange-500 text-xs">🎯</span>
|
||||
<span class="text-xs text-orange-700 font-medium">${objectName}</span>
|
||||
<span class="text-xs text-orange-500">·</span>
|
||||
<span class="text-xs text-orange-600">${objectPosition}</span>
|
||||
`;
|
||||
|
||||
// 在光标位置插入，或添加到末尾
|
||||
const selection = window.getSelection();
|
||||
if (selection && selection.rangeCount > 0 && editableDiv.contains(selection.anchorNode)) {
|
||||
const range = selection.getRangeAt(0);
|
||||
range.deleteContents();
|
||||
range.insertNode(tagSpan);
|
||||
range.setStartAfter(tagSpan);
|
||||
range.setEndAfter(tagSpan);
|
||||
selection.removeAllRanges();
|
||||
selection.addRange(range);
|
||||
} else {
|
||||
editableDiv.appendChild(tagSpan);
|
||||
}
|
||||
|
||||
console.log('[CanvasChat] 已插入物品标签:', objectName, objectPosition);
|
||||
}, []);
|
||||
|
||||
// 🆕 监听 pendingChatAttachments，当用户点击"添加到对话"按钮时添加标签
|
||||
useEffect(() => {
|
||||
if (pendingChatAttachments.length > 0) {
|
||||
@ -230,6 +295,55 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
clearPendingChatAttachments();
|
||||
}
|
||||
}, [pendingChatAttachments, attachments, insertImageTagInEditor, clearPendingChatAttachments]);
|
||||
|
||||
// 🆕 监听 pendingPickedObjects，当用户确认拾取物品时添加标签
|
||||
useEffect(() => {
|
||||
if (pendingPickedObjects.length > 0) {
|
||||
const currentAttachments = [...attachments];
|
||||
|
||||
pendingPickedObjects.forEach((picked, i) => {
|
||||
// 检查是否已存在相同的物品
|
||||
const exists = currentAttachments.some(a =>
|
||||
a.url === picked.imageUrl &&
|
||||
a.pickedObject?.objectName === picked.objectName
|
||||
);
|
||||
|
||||
if (!exists) {
|
||||
// 插入物品标签到编辑器
|
||||
insertPickedObjectTagInEditor(
|
||||
picked.imageUrl,
|
||||
picked.objectName,
|
||||
picked.objectPosition,
|
||||
currentAttachments.length + i
|
||||
);
|
||||
|
||||
// 添加到附件列表（带物品信息）
|
||||
currentAttachments.push({
|
||||
url: picked.imageUrl,
|
||||
name: `🎯 ${picked.objectName}`,
|
||||
contentType: 'image/jpeg',
|
||||
pickedObject: {
|
||||
objectName: picked.objectName,
|
||||
objectPosition: picked.objectPosition,
|
||||
clickX: picked.clickX,
|
||||
clickY: picked.clickY,
|
||||
boundingBox: picked.boundingBox,
|
||||
},
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
setAttachments(currentAttachments);
|
||||
|
||||
// 聚焦到输入框
|
||||
setTimeout(() => {
|
||||
(textareaRef.current as HTMLDivElement | null)?.focus();
|
||||
}, 0);
|
||||
|
||||
// 清空待添加列表
|
||||
clearPendingPickedObjects();
|
||||
}
|
||||
}, [pendingPickedObjects, attachments, insertPickedObjectTagInEditor, clearPendingPickedObjects]);
|
||||
|
||||
// 获取模型列表
|
||||
const { data: modelsData } = useSWR<{
|
||||
@ -557,19 +671,39 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
const hasAnnotatedImages = annotatedNodes.length > 0;
|
||||
|
||||
// 添加文件部分（作为 file 类型的 part）
|
||||
// 🆕 收集物品信息用于构建增强 prompt
|
||||
const pickedObjectInfos: string[] = [];
|
||||
|
||||
effectiveAttachments.forEach((a) => {
|
||||
const att = a as PickedObjectAttachment;
|
||||
parts.push({
|
||||
type: "file" as const,
|
||||
mediaType: a.contentType,
|
||||
name: a.name,
|
||||
url: a.url,
|
||||
hasAnnotations: hasAnnotatedImages, // 🆕 标记图片是否有标注
|
||||
pickedObject: att.pickedObject, // 🆕 包含物品信息
|
||||
});
|
||||
|
||||
// 收集物品信息
|
||||
if (att.pickedObject) {
|
||||
const { objectName, objectPosition, boundingBox } = att.pickedObject;
|
||||
pickedObjectInfos.push(
|
||||
`【物品：${objectName}】位置：${objectPosition}，边界框：(${Math.round(boundingBox.x * 100)}%, ${Math.round(boundingBox.y * 100)}%) - (${Math.round((boundingBox.x + boundingBox.width) * 100)}%, ${Math.round((boundingBox.y + boundingBox.height) * 100)}%)`
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// 添加文本部分
|
||||
let textContent = input.trim() || (effectiveAttachments.length > 0 ? "请处理这张图片" : "");
|
||||
|
||||
// 🆕 如果有拾取的物品，在文本前添加物品信息
|
||||
if (pickedObjectInfos.length > 0 && textContent) {
|
||||
const objectsPrompt = pickedObjectInfos.join('\n');
|
||||
textContent = `【用户指定的物品】\n${objectsPrompt}\n\n【用户指令】${textContent}\n\n请精准地只处理上述指定的物品区域，保持其他部分不变。`;
|
||||
console.log('[CanvasChat] 检测到拾取物品，已添加物品信息:', pickedObjectInfos.length, '个');
|
||||
}
|
||||
|
||||
// 🆕 如果有标注的图片，在文本中添加提示
|
||||
if (hasAnnotatedImages && textContent) {
|
||||
textContent = `【注意：图片中有标注（箭头、圆圈、文字等）标记的区域是需要重点关注或修改的部分，请根据标注进行处理】\n${textContent}`;
|
||||
|
||||
303
components/canvas/ObjectPickerOverlay.tsx
Normal file
303
components/canvas/ObjectPickerOverlay.tsx
Normal file
@ -0,0 +1,303 @@
|
||||
"use client";
|
||||
|
||||
import React, { useState, useRef, useCallback } from "react";
|
||||
import { createPortal } from "react-dom";
|
||||
import { X, Loader2, Check, RefreshCw, Crosshair } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { useCanvasStore, type PickedObject } from "@/store/canvas";
|
||||
import { toast } from "sonner";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
// 图片压缩配置
|
||||
const PICKER_IMAGE_CONFIG = {
|
||||
maxSize: 600,
|
||||
quality: 0.85,
|
||||
};
|
||||
|
||||
// 压缩图片
|
||||
async function compressImageForPicker(imageUrl: string): Promise<{
|
||||
compressedDataUrl: string;
|
||||
scale: number;
|
||||
originalWidth: number;
|
||||
originalHeight: number;
|
||||
}> {
|
||||
const img = new Image();
|
||||
img.crossOrigin = 'anonymous';
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
img.onload = () => {
|
||||
const { naturalWidth, naturalHeight } = img;
|
||||
let scale = 1;
|
||||
let targetWidth = naturalWidth;
|
||||
let targetHeight = naturalHeight;
|
||||
|
||||
// 计算缩放比例
|
||||
if (naturalWidth > PICKER_IMAGE_CONFIG.maxSize || naturalHeight > PICKER_IMAGE_CONFIG.maxSize) {
|
||||
scale = Math.min(
|
||||
PICKER_IMAGE_CONFIG.maxSize / naturalWidth,
|
||||
PICKER_IMAGE_CONFIG.maxSize / naturalHeight
|
||||
);
|
||||
targetWidth = Math.round(naturalWidth * scale);
|
||||
targetHeight = Math.round(naturalHeight * scale);
|
||||
}
|
||||
|
||||
// Canvas 压缩
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = targetWidth;
|
||||
canvas.height = targetHeight;
|
||||
const ctx = canvas.getContext('2d')!;
|
||||
ctx.drawImage(img, 0, 0, targetWidth, targetHeight);
|
||||
|
||||
const compressedDataUrl = canvas.toDataURL('image/jpeg', PICKER_IMAGE_CONFIG.quality);
|
||||
|
||||
resolve({
|
||||
compressedDataUrl,
|
||||
scale,
|
||||
originalWidth: naturalWidth,
|
||||
originalHeight: naturalHeight,
|
||||
});
|
||||
};
|
||||
img.onerror = () => reject(new Error('图片加载失败'));
|
||||
img.src = imageUrl;
|
||||
});
|
||||
}
|
||||
|
||||
export function ObjectPickerOverlay() {
|
||||
const isPickerMode = useCanvasStore((state) => state.isPickerMode);
|
||||
const pickerTargetNodeId = useCanvasStore((state) => state.pickerTargetNodeId);
|
||||
const isPickerLoading = useCanvasStore((state) => state.isPickerLoading);
|
||||
const pickedObject = useCanvasStore((state) => state.pickedObject);
|
||||
const setPickerMode = useCanvasStore((state) => state.setPickerMode);
|
||||
const setPickerLoading = useCanvasStore((state) => state.setPickerLoading);
|
||||
const setPickedObject = useCanvasStore((state) => state.setPickedObject);
|
||||
const addPendingPickedObject = useCanvasStore((state) => state.addPendingPickedObject);
|
||||
const getNode = useCanvasStore((state) => state.getNode);
|
||||
|
||||
const [clickPosition, setClickPosition] = useState<{ x: number; y: number } | null>(null);
|
||||
const overlayRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
// 获取目标节点
|
||||
const targetNode = pickerTargetNodeId ? getNode(pickerTargetNodeId) : null;
|
||||
const imageUrl = targetNode?.type === 'image' ? (targetNode.data as any)?.src as string | undefined : undefined;
|
||||
|
||||
// 处理图片点击
|
||||
const handleImageClick = useCallback(async (e: React.MouseEvent<HTMLImageElement>) => {
|
||||
if (isPickerLoading || !imageUrl || !pickerTargetNodeId) return;
|
||||
|
||||
const img = e.currentTarget;
|
||||
const rect = img.getBoundingClientRect();
|
||||
|
||||
// 计算相对坐标 (0-1)
|
||||
const clickX = (e.clientX - rect.left) / rect.width;
|
||||
const clickY = (e.clientY - rect.top) / rect.height;
|
||||
|
||||
// 显示点击位置
|
||||
setClickPosition({ x: clickX, y: clickY });
|
||||
setPickerLoading(true);
|
||||
|
||||
try {
|
||||
// 压缩图片
|
||||
const { compressedDataUrl } = await compressImageForPicker(imageUrl);
|
||||
|
||||
// 调用 API
|
||||
const response = await fetch('/api/pick-object', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
imageData: compressedDataUrl,
|
||||
clickX,
|
||||
clickY,
|
||||
originalImageUrl: imageUrl,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error('识别失败');
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
if (result.success && result.object) {
|
||||
// 设置识别结果
|
||||
const picked: PickedObject = {
|
||||
imageUrl,
|
||||
imageNodeId: pickerTargetNodeId,
|
||||
clickX,
|
||||
clickY,
|
||||
objectName: result.object.name,
|
||||
objectPosition: result.object.position,
|
||||
confidence: result.object.confidence,
|
||||
boundingBox: result.object.boundingBox,
|
||||
};
|
||||
setPickedObject(picked);
|
||||
toast.success(`识别成功: ${result.object.name}`);
|
||||
} else {
|
||||
toast.error(result.error || '未能识别物品');
|
||||
setClickPosition(null);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('物品拾取失败:', error);
|
||||
toast.error('识别失败，请重试');
|
||||
setClickPosition(null);
|
||||
} finally {
|
||||
setPickerLoading(false);
|
||||
}
|
||||
}, [imageUrl, isPickerLoading, pickerTargetNodeId, setPickerLoading, setPickedObject]);
|
||||
|
||||
// 确认添加到对话
|
||||
const handleConfirm = useCallback(() => {
|
||||
if (pickedObject) {
|
||||
addPendingPickedObject(pickedObject);
|
||||
toast.success('已添加到对话框');
|
||||
setPickerMode(false);
|
||||
setClickPosition(null);
|
||||
}
|
||||
}, [pickedObject, addPendingPickedObject, setPickerMode]);
|
||||
|
||||
// 重新选择
|
||||
const handleRetry = useCallback(() => {
|
||||
setPickedObject(null);
|
||||
setClickPosition(null);
|
||||
}, [setPickedObject]);
|
||||
|
||||
// 关闭拾取模式
|
||||
const handleClose = useCallback(() => {
|
||||
setPickerMode(false);
|
||||
setClickPosition(null);
|
||||
}, [setPickerMode]);
|
||||
|
||||
if (!isPickerMode || !imageUrl) return null;
|
||||
|
||||
return createPortal(
|
||||
<div
|
||||
ref={overlayRef}
|
||||
className="fixed inset-0 z-[9998] bg-black/60 flex items-center justify-center"
|
||||
onClick={(e) => {
|
||||
// 点击背景关闭
|
||||
if (e.target === overlayRef.current) {
|
||||
handleClose();
|
||||
}
|
||||
}}
|
||||
>
|
||||
{/* 顶部提示栏 */}
|
||||
<div className="absolute top-4 left-1/2 -translate-x-1/2 flex items-center gap-3 bg-white rounded-full px-4 py-2 shadow-lg">
|
||||
<Crosshair className="h-5 w-5 text-orange-500" />
|
||||
<span className="text-sm font-medium">
|
||||
{isPickerLoading ? '正在识别...' : pickedObject ? '识别完成' : '点击图片中的物品进行拾取'}
|
||||
</span>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-7 w-7 rounded-full"
|
||||
onClick={handleClose}
|
||||
>
|
||||
<X className="h-4 w-4" />
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{/* 图片容器 */}
|
||||
<div className="relative max-w-[80vw] max-h-[70vh]">
|
||||
{/* 图片 */}
|
||||
<img
|
||||
src={imageUrl}
|
||||
alt="拾取目标"
|
||||
className={cn(
|
||||
"max-w-full max-h-[70vh] object-contain rounded-lg border-4 transition-all",
|
||||
isPickerLoading ? "border-orange-400 opacity-80" : "border-orange-500",
|
||||
!pickedObject && !isPickerLoading && "cursor-crosshair"
|
||||
)}
|
||||
onClick={!pickedObject && !isPickerLoading ? handleImageClick : undefined}
|
||||
draggable={false}
|
||||
/>
|
||||
|
||||
{/* 点击位置标记（脉冲动画） */}
|
||||
{clickPosition && (
|
||||
<div
|
||||
className="absolute pointer-events-none"
|
||||
style={{
|
||||
left: `${clickPosition.x * 100}%`,
|
||||
top: `${clickPosition.y * 100}%`,
|
||||
transform: 'translate(-50%, -50%)',
|
||||
}}
|
||||
>
|
||||
{/* 脉冲圆点 */}
|
||||
<div className={cn(
|
||||
"relative w-8 h-8 rounded-full border-2 border-orange-500 bg-orange-500/30",
|
||||
isPickerLoading && "animate-pulse"
|
||||
)}>
|
||||
<div className="absolute inset-0 rounded-full bg-orange-500 animate-ping opacity-30" />
|
||||
<div className="absolute inset-2 rounded-full bg-orange-500" />
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* 边界框显示（识别成功后） */}
|
||||
{pickedObject && (
|
||||
<div
|
||||
className="absolute border-2 border-dashed border-green-500 rounded pointer-events-none"
|
||||
style={{
|
||||
left: `${pickedObject.boundingBox.x * 100}%`,
|
||||
top: `${pickedObject.boundingBox.y * 100}%`,
|
||||
width: `${pickedObject.boundingBox.width * 100}%`,
|
||||
height: `${pickedObject.boundingBox.height * 100}%`,
|
||||
}}
|
||||
>
|
||||
{/* 物品名称标签 */}
|
||||
<div className="absolute -top-7 left-0 bg-green-500 text-white text-xs px-2 py-1 rounded whitespace-nowrap">
|
||||
{pickedObject.objectName}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* 加载状态 */}
|
||||
{isPickerLoading && (
|
||||
<div className="absolute inset-0 flex items-center justify-center bg-black/20 rounded-lg">
|
||||
<div className="bg-white rounded-lg px-4 py-3 flex items-center gap-2 shadow-lg">
|
||||
<Loader2 className="h-5 w-5 animate-spin text-orange-500" />
|
||||
<span className="text-sm">正在识别...</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* 识别结果确认框 */}
|
||||
{pickedObject && !isPickerLoading && (
|
||||
<div className="absolute bottom-8 left-1/2 -translate-x-1/2 bg-white rounded-xl shadow-2xl p-4 min-w-[280px]">
|
||||
<div className="flex items-start gap-3 mb-4">
|
||||
<div className="w-10 h-10 rounded-lg bg-green-100 flex items-center justify-center flex-shrink-0">
|
||||
<Crosshair className="h-5 w-5 text-green-600" />
|
||||
</div>
|
||||
<div>
|
||||
<h4 className="font-medium text-gray-900">{pickedObject.objectName}</h4>
|
||||
<p className="text-sm text-gray-500">{pickedObject.objectPosition}</p>
|
||||
<p className="text-xs text-gray-400 mt-1">
|
||||
置信度: {Math.round(pickedObject.confidence * 100)}%
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="flex-1"
|
||||
onClick={handleRetry}
|
||||
>
|
||||
<RefreshCw className="h-4 w-4 mr-1" />
|
||||
重新选择
|
||||
</Button>
|
||||
<Button
|
||||
size="sm"
|
||||
className="flex-1 bg-green-600 hover:bg-green-700"
|
||||
onClick={handleConfirm}
|
||||
>
|
||||
<Check className="h-4 w-4 mr-1" />
|
||||
添加到对话
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>,
|
||||
document.body
|
||||
);
|
||||
}
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
import React, { useState, useRef, useEffect } from "react";
|
||||
import { createPortal } from "react-dom";
|
||||
import { ImagePlus, Loader2, AlertCircle, MoreVertical, Scissors, Sparkles, Expand, Maximize2, Move3D, Layers, Download, Trash2, RefreshCw, Pencil, MessageSquarePlus } from "lucide-react";
|
||||
import { ImagePlus, Loader2, AlertCircle, MoreVertical, Scissors, Sparkles, Expand, Maximize2, Move3D, Layers, Download, Trash2, RefreshCw, Pencil, MessageSquarePlus, Crosshair } from "lucide-react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import {
|
||||
DropdownMenu,
|
||||
@ -55,6 +55,7 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
const deleteNode = useCanvasStore((state) => state.deleteNode);
|
||||
const addNode = useCanvasStore((state) => state.addNode);
|
||||
const addPendingChatAttachment = useCanvasStore((state) => state.addPendingChatAttachment);
|
||||
const setPickerMode = useCanvasStore((state) => state.setPickerMode);
|
||||
|
||||
// 获取工具栏位置 - 在渲染时直接计算
|
||||
const getToolbarPosition = () => {
|
||||
@ -628,6 +629,24 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
<TooltipContent side="bottom" className="text-xs">添加到对话</TooltipContent>
|
||||
</Tooltip>
|
||||
|
||||
{/* 🆕 物品拾取 */}
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="h-8 w-8 rounded-full hover:bg-orange-50 text-orange-600 hover:text-orange-700"
|
||||
onClick={() => {
|
||||
setPickerMode(true, node.id);
|
||||
toast.info('点击图片中的物品进行拾取');
|
||||
}}
|
||||
>
|
||||
<Crosshair className="h-4 w-4" />
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="bottom" className="text-xs">拾取物品</TooltipContent>
|
||||
</Tooltip>
|
||||
|
||||
{/* 分隔线 */}
|
||||
<div className="h-5 w-px bg-gray-200 mx-0.5" />
|
||||
|
||||
|
||||
@ -29,6 +29,30 @@ interface PendingChatAttachment {
|
||||
contentType: string;
|
||||
}
|
||||
|
||||
// 🆕 拾取的物品信息
|
||||
export interface PickedObject {
|
||||
// 原图信息
|
||||
imageUrl: string;
|
||||
imageNodeId: string;
|
||||
|
||||
// 点击位置（相对坐标 0-1）
|
||||
clickX: number;
|
||||
clickY: number;
|
||||
|
||||
// AI 识别结果
|
||||
objectName: string; // ≤8字，格式：[颜色/材质]+[物品]
|
||||
objectPosition: string; // 固定4字，如"左上区域"
|
||||
confidence: number; // 置信度 0-1
|
||||
|
||||
// 物品边界框
|
||||
boundingBox: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
}
|
||||
|
||||
// Store 状态接口
|
||||
interface CanvasStore {
|
||||
// 项目信息
|
||||
@ -53,6 +77,13 @@ interface CanvasStore {
|
||||
// 🆕 待添加到对话框的图片（用于从画布添加图片到对话）
|
||||
pendingChatAttachments: PendingChatAttachment[];
|
||||
|
||||
// 🆕 物品拾取模式
|
||||
isPickerMode: boolean;
|
||||
pickerTargetNodeId: string | null;
|
||||
isPickerLoading: boolean;
|
||||
pickedObject: PickedObject | null;
|
||||
pendingPickedObjects: PickedObject[];
|
||||
|
||||
// 项目操作
|
||||
setProject: (id: string | null, title?: string) => void;
|
||||
setProjectTitle: (title: string) => void;
|
||||
@ -110,6 +141,13 @@ interface CanvasStore {
|
||||
// 🆕 对话框附件操作
|
||||
addPendingChatAttachment: (attachment: PendingChatAttachment) => void;
|
||||
clearPendingChatAttachments: () => void;
|
||||
|
||||
// 🆕 物品拾取操作
|
||||
setPickerMode: (enabled: boolean, nodeId?: string) => void;
|
||||
setPickerLoading: (loading: boolean) => void;
|
||||
setPickedObject: (obj: PickedObject | null) => void;
|
||||
addPendingPickedObject: (obj: PickedObject) => void;
|
||||
clearPendingPickedObjects: () => void;
|
||||
}
|
||||
|
||||
export const useCanvasStore = create<CanvasStore>()(
|
||||
@ -129,6 +167,13 @@ export const useCanvasStore = create<CanvasStore>()(
|
||||
isPanelOpen: true,
|
||||
isGenerating: false,
|
||||
pendingChatAttachments: [], // 🆕 待添加到对话框的图片
|
||||
|
||||
// 🆕 物品拾取模式初始状态
|
||||
isPickerMode: false,
|
||||
pickerTargetNodeId: null,
|
||||
isPickerLoading: false,
|
||||
pickedObject: null,
|
||||
pendingPickedObjects: [],
|
||||
|
||||
// 项目操作
|
||||
setProject: (id, title) => set((state) => {
|
||||
@ -396,6 +441,39 @@ export const useCanvasStore = create<CanvasStore>()(
|
||||
clearPendingChatAttachments: () => set((state) => {
|
||||
state.pendingChatAttachments = [];
|
||||
}),
|
||||
|
||||
// 🆕 物品拾取操作
|
||||
setPickerMode: (enabled, nodeId) => set((state) => {
|
||||
state.isPickerMode = enabled;
|
||||
state.pickerTargetNodeId = enabled ? (nodeId || null) : null;
|
||||
if (!enabled) {
|
||||
state.pickedObject = null;
|
||||
state.isPickerLoading = false;
|
||||
}
|
||||
}),
|
||||
|
||||
setPickerLoading: (loading) => set((state) => {
|
||||
state.isPickerLoading = loading;
|
||||
}),
|
||||
|
||||
setPickedObject: (obj) => set((state) => {
|
||||
state.pickedObject = obj;
|
||||
}),
|
||||
|
||||
addPendingPickedObject: (obj) => set((state) => {
|
||||
// 避免重复添加
|
||||
if (!state.pendingPickedObjects.some(o =>
|
||||
o.imageUrl === obj.imageUrl &&
|
||||
o.clickX === obj.clickX &&
|
||||
o.clickY === obj.clickY
|
||||
)) {
|
||||
state.pendingPickedObjects.push(obj);
|
||||
}
|
||||
}),
|
||||
|
||||
clearPendingPickedObjects: () => set((state) => {
|
||||
state.pendingPickedObjects = [];
|
||||
}),
|
||||
})),
|
||||
{
|
||||
name: 'canvas-storage',
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user