- 新增 ObjectPickerOverlay 组件,支持点击图片拾取物品 - 新增 /api/pick-object API,调用 Qwen-VL 识别物品 - Canvas Store 扩展拾取模式状态管理 - ImageNode 工具栏新增拾取按钮 - CanvasChatPanel 集成物品标签,发送时自动构建精准prompt - 图片压缩至 600px 以内加速 AI 分析 - 物品名称限制 8 字,位置固定 4 字标准词库
304 lines
10 KiB
TypeScript
304 lines
10 KiB
TypeScript
"use client";
|
|
|
|
import React, { useState, useRef, useCallback } from "react";
|
|
import { createPortal } from "react-dom";
|
|
import { X, Loader2, Check, RefreshCw, Crosshair } from "lucide-react";
|
|
import { Button } from "@/components/ui/button";
|
|
import { useCanvasStore, type PickedObject } from "@/store/canvas";
|
|
import { toast } from "sonner";
|
|
import { cn } from "@/lib/utils";
|
|
|
|
// 图片压缩配置
|
|
const PICKER_IMAGE_CONFIG = {
|
|
maxSize: 600,
|
|
quality: 0.85,
|
|
};
|
|
|
|
// 压缩图片
|
|
async function compressImageForPicker(imageUrl: string): Promise<{
|
|
compressedDataUrl: string;
|
|
scale: number;
|
|
originalWidth: number;
|
|
originalHeight: number;
|
|
}> {
|
|
const img = new Image();
|
|
img.crossOrigin = 'anonymous';
|
|
|
|
return new Promise((resolve, reject) => {
|
|
img.onload = () => {
|
|
const { naturalWidth, naturalHeight } = img;
|
|
let scale = 1;
|
|
let targetWidth = naturalWidth;
|
|
let targetHeight = naturalHeight;
|
|
|
|
// 计算缩放比例
|
|
if (naturalWidth > PICKER_IMAGE_CONFIG.maxSize || naturalHeight > PICKER_IMAGE_CONFIG.maxSize) {
|
|
scale = Math.min(
|
|
PICKER_IMAGE_CONFIG.maxSize / naturalWidth,
|
|
PICKER_IMAGE_CONFIG.maxSize / naturalHeight
|
|
);
|
|
targetWidth = Math.round(naturalWidth * scale);
|
|
targetHeight = Math.round(naturalHeight * scale);
|
|
}
|
|
|
|
// Canvas 压缩
|
|
const canvas = document.createElement('canvas');
|
|
canvas.width = targetWidth;
|
|
canvas.height = targetHeight;
|
|
const ctx = canvas.getContext('2d')!;
|
|
ctx.drawImage(img, 0, 0, targetWidth, targetHeight);
|
|
|
|
const compressedDataUrl = canvas.toDataURL('image/jpeg', PICKER_IMAGE_CONFIG.quality);
|
|
|
|
resolve({
|
|
compressedDataUrl,
|
|
scale,
|
|
originalWidth: naturalWidth,
|
|
originalHeight: naturalHeight,
|
|
});
|
|
};
|
|
img.onerror = () => reject(new Error('图片加载失败'));
|
|
img.src = imageUrl;
|
|
});
|
|
}
|
|
|
|
export function ObjectPickerOverlay() {
|
|
const isPickerMode = useCanvasStore((state) => state.isPickerMode);
|
|
const pickerTargetNodeId = useCanvasStore((state) => state.pickerTargetNodeId);
|
|
const isPickerLoading = useCanvasStore((state) => state.isPickerLoading);
|
|
const pickedObject = useCanvasStore((state) => state.pickedObject);
|
|
const setPickerMode = useCanvasStore((state) => state.setPickerMode);
|
|
const setPickerLoading = useCanvasStore((state) => state.setPickerLoading);
|
|
const setPickedObject = useCanvasStore((state) => state.setPickedObject);
|
|
const addPendingPickedObject = useCanvasStore((state) => state.addPendingPickedObject);
|
|
const getNode = useCanvasStore((state) => state.getNode);
|
|
|
|
const [clickPosition, setClickPosition] = useState<{ x: number; y: number } | null>(null);
|
|
const overlayRef = useRef<HTMLDivElement>(null);
|
|
|
|
// 获取目标节点
|
|
const targetNode = pickerTargetNodeId ? getNode(pickerTargetNodeId) : null;
|
|
const imageUrl = targetNode?.type === 'image' ? (targetNode.data as any)?.src as string | undefined : undefined;
|
|
|
|
// 处理图片点击
|
|
const handleImageClick = useCallback(async (e: React.MouseEvent<HTMLImageElement>) => {
|
|
if (isPickerLoading || !imageUrl || !pickerTargetNodeId) return;
|
|
|
|
const img = e.currentTarget;
|
|
const rect = img.getBoundingClientRect();
|
|
|
|
// 计算相对坐标 (0-1)
|
|
const clickX = (e.clientX - rect.left) / rect.width;
|
|
const clickY = (e.clientY - rect.top) / rect.height;
|
|
|
|
// 显示点击位置
|
|
setClickPosition({ x: clickX, y: clickY });
|
|
setPickerLoading(true);
|
|
|
|
try {
|
|
// 压缩图片
|
|
const { compressedDataUrl } = await compressImageForPicker(imageUrl);
|
|
|
|
// 调用 API
|
|
const response = await fetch('/api/pick-object', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({
|
|
imageData: compressedDataUrl,
|
|
clickX,
|
|
clickY,
|
|
originalImageUrl: imageUrl,
|
|
}),
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error('识别失败');
|
|
}
|
|
|
|
const result = await response.json();
|
|
|
|
if (result.success && result.object) {
|
|
// 设置识别结果
|
|
const picked: PickedObject = {
|
|
imageUrl,
|
|
imageNodeId: pickerTargetNodeId,
|
|
clickX,
|
|
clickY,
|
|
objectName: result.object.name,
|
|
objectPosition: result.object.position,
|
|
confidence: result.object.confidence,
|
|
boundingBox: result.object.boundingBox,
|
|
};
|
|
setPickedObject(picked);
|
|
toast.success(`识别成功: ${result.object.name}`);
|
|
} else {
|
|
toast.error(result.error || '未能识别物品');
|
|
setClickPosition(null);
|
|
}
|
|
} catch (error) {
|
|
console.error('物品拾取失败:', error);
|
|
toast.error('识别失败,请重试');
|
|
setClickPosition(null);
|
|
} finally {
|
|
setPickerLoading(false);
|
|
}
|
|
}, [imageUrl, isPickerLoading, pickerTargetNodeId, setPickerLoading, setPickedObject]);
|
|
|
|
// 确认添加到对话
|
|
const handleConfirm = useCallback(() => {
|
|
if (pickedObject) {
|
|
addPendingPickedObject(pickedObject);
|
|
toast.success('已添加到对话框');
|
|
setPickerMode(false);
|
|
setClickPosition(null);
|
|
}
|
|
}, [pickedObject, addPendingPickedObject, setPickerMode]);
|
|
|
|
// 重新选择
|
|
const handleRetry = useCallback(() => {
|
|
setPickedObject(null);
|
|
setClickPosition(null);
|
|
}, [setPickedObject]);
|
|
|
|
// 关闭拾取模式
|
|
const handleClose = useCallback(() => {
|
|
setPickerMode(false);
|
|
setClickPosition(null);
|
|
}, [setPickerMode]);
|
|
|
|
if (!isPickerMode || !imageUrl) return null;
|
|
|
|
return createPortal(
|
|
<div
|
|
ref={overlayRef}
|
|
className="fixed inset-0 z-[9998] bg-black/60 flex items-center justify-center"
|
|
onClick={(e) => {
|
|
// 点击背景关闭
|
|
if (e.target === overlayRef.current) {
|
|
handleClose();
|
|
}
|
|
}}
|
|
>
|
|
{/* 顶部提示栏 */}
|
|
<div className="absolute top-4 left-1/2 -translate-x-1/2 flex items-center gap-3 bg-white rounded-full px-4 py-2 shadow-lg">
|
|
<Crosshair className="h-5 w-5 text-orange-500" />
|
|
<span className="text-sm font-medium">
|
|
{isPickerLoading ? '正在识别...' : pickedObject ? '识别完成' : '点击图片中的物品进行拾取'}
|
|
</span>
|
|
<Button
|
|
variant="ghost"
|
|
size="icon"
|
|
className="h-7 w-7 rounded-full"
|
|
onClick={handleClose}
|
|
>
|
|
<X className="h-4 w-4" />
|
|
</Button>
|
|
</div>
|
|
|
|
{/* 图片容器 */}
|
|
<div className="relative max-w-[80vw] max-h-[70vh]">
|
|
{/* 图片 */}
|
|
<img
|
|
src={imageUrl}
|
|
alt="拾取目标"
|
|
className={cn(
|
|
"max-w-full max-h-[70vh] object-contain rounded-lg border-4 transition-all",
|
|
isPickerLoading ? "border-orange-400 opacity-80" : "border-orange-500",
|
|
!pickedObject && !isPickerLoading && "cursor-crosshair"
|
|
)}
|
|
onClick={!pickedObject && !isPickerLoading ? handleImageClick : undefined}
|
|
draggable={false}
|
|
/>
|
|
|
|
{/* 点击位置标记(脉冲动画) */}
|
|
{clickPosition && (
|
|
<div
|
|
className="absolute pointer-events-none"
|
|
style={{
|
|
left: `${clickPosition.x * 100}%`,
|
|
top: `${clickPosition.y * 100}%`,
|
|
transform: 'translate(-50%, -50%)',
|
|
}}
|
|
>
|
|
{/* 脉冲圆点 */}
|
|
<div className={cn(
|
|
"relative w-8 h-8 rounded-full border-2 border-orange-500 bg-orange-500/30",
|
|
isPickerLoading && "animate-pulse"
|
|
)}>
|
|
<div className="absolute inset-0 rounded-full bg-orange-500 animate-ping opacity-30" />
|
|
<div className="absolute inset-2 rounded-full bg-orange-500" />
|
|
</div>
|
|
</div>
|
|
)}
|
|
|
|
{/* 边界框显示(识别成功后) */}
|
|
{pickedObject && (
|
|
<div
|
|
className="absolute border-2 border-dashed border-green-500 rounded pointer-events-none"
|
|
style={{
|
|
left: `${pickedObject.boundingBox.x * 100}%`,
|
|
top: `${pickedObject.boundingBox.y * 100}%`,
|
|
width: `${pickedObject.boundingBox.width * 100}%`,
|
|
height: `${pickedObject.boundingBox.height * 100}%`,
|
|
}}
|
|
>
|
|
{/* 物品名称标签 */}
|
|
<div className="absolute -top-7 left-0 bg-green-500 text-white text-xs px-2 py-1 rounded whitespace-nowrap">
|
|
{pickedObject.objectName}
|
|
</div>
|
|
</div>
|
|
)}
|
|
|
|
{/* 加载状态 */}
|
|
{isPickerLoading && (
|
|
<div className="absolute inset-0 flex items-center justify-center bg-black/20 rounded-lg">
|
|
<div className="bg-white rounded-lg px-4 py-3 flex items-center gap-2 shadow-lg">
|
|
<Loader2 className="h-5 w-5 animate-spin text-orange-500" />
|
|
<span className="text-sm">正在识别...</span>
|
|
</div>
|
|
</div>
|
|
)}
|
|
</div>
|
|
|
|
{/* 识别结果确认框 */}
|
|
{pickedObject && !isPickerLoading && (
|
|
<div className="absolute bottom-8 left-1/2 -translate-x-1/2 bg-white rounded-xl shadow-2xl p-4 min-w-[280px]">
|
|
<div className="flex items-start gap-3 mb-4">
|
|
<div className="w-10 h-10 rounded-lg bg-green-100 flex items-center justify-center flex-shrink-0">
|
|
<Crosshair className="h-5 w-5 text-green-600" />
|
|
</div>
|
|
<div>
|
|
<h4 className="font-medium text-gray-900">{pickedObject.objectName}</h4>
|
|
<p className="text-sm text-gray-500">{pickedObject.objectPosition}</p>
|
|
<p className="text-xs text-gray-400 mt-1">
|
|
置信度: {Math.round(pickedObject.confidence * 100)}%
|
|
</p>
|
|
</div>
|
|
</div>
|
|
|
|
<div className="flex items-center gap-2">
|
|
<Button
|
|
variant="outline"
|
|
size="sm"
|
|
className="flex-1"
|
|
onClick={handleRetry}
|
|
>
|
|
<RefreshCw className="h-4 w-4 mr-1" />
|
|
重新选择
|
|
</Button>
|
|
<Button
|
|
size="sm"
|
|
className="flex-1 bg-green-600 hover:bg-green-700"
|
|
onClick={handleConfirm}
|
|
>
|
|
<Check className="h-4 w-4 mr-1" />
|
|
添加到对话
|
|
</Button>
|
|
</div>
|
|
</div>
|
|
)}
|
|
</div>,
|
|
document.body
|
|
);
|
|
}
|