ai-chatbot/components/canvas/chat-panel/OpenClawInteractiveModelViewer.tsx

605 lines
20 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"use client";
/* eslint-disable @next/next/no-img-element */
import { Environment, OrbitControls, useGLTF } from "@react-three/drei";
import { Canvas, useFrame, useThree } from "@react-three/fiber";
import { AlertCircle } from "lucide-react";
import React, { Suspense, useEffect, useMemo, useRef } from "react";
import * as THREE from "three";
import type { Mesh, Object3D } from "three";
import type { OrbitControls as OrbitControlsImpl } from "three-stdlib";
import type {
OpenClawModelInteractionManifest,
OpenClawModelInteractionPart,
} from "@/lib/openclaw-chat/model-interaction";
import {
getOpenClawExplodedOffsetMm,
mapOpenClawCadOffsetToGlb,
} from "@/lib/openclaw-chat/model-interaction";
export type OpenClawModelTransformRequest = {
actionId: string | null;
amount: number;
animate: boolean;
durationMs: number;
nonce: number;
};
type InteractiveEntry = {
node: Object3D;
part: OpenClawModelInteractionPart;
initialPosition: THREE.Vector3;
};
type AnimationState = {
startedAt: number;
durationMs: number;
starts: Map<string, THREE.Vector3>;
targets: Map<string, THREE.Vector3>;
easing: string;
cameraStart: THREE.Vector3;
cameraTarget: THREE.Vector3;
controlsTargetStart: THREE.Vector3;
controlsTargetEnd: THREE.Vector3;
};
type DragState = {
entry: InteractiveEntry;
plane: THREE.Plane;
startIntersection: THREE.Vector3;
startWorldPosition: THREE.Vector3;
startLocalPosition: THREE.Vector3;
pointerId: number;
};
class InteractiveViewerErrorBoundary extends React.Component<
{ children: React.ReactNode; fallback: React.ReactNode; resetKey: string },
{ hasError: boolean }
> {
state = { hasError: false };
static getDerivedStateFromError() {
return { hasError: true };
}
componentDidUpdate(previousProps: { resetKey: string }) {
if (
this.state.hasError &&
previousProps.resetKey !== this.props.resetKey
) {
this.setState({ hasError: false });
}
}
render() {
return this.state.hasError ? this.props.fallback : this.props.children;
}
}
function isMesh(node: Object3D): node is Mesh {
return (node as Mesh).isMesh === true;
}
function cloneScene(scene: Object3D) {
const clone = scene.clone(true);
clone.traverse((node) => {
if (!isMesh(node)) return;
node.castShadow = true;
node.receiveShadow = true;
const materials = Array.isArray(node.material)
? node.material
: [node.material];
const clones = materials.map((material) => material.clone());
node.material = Array.isArray(node.material)
? clones
: clones[0] || node.material;
});
return clone;
}
function getFitTransform(scene: Object3D) {
const bounds = new THREE.Box3().setFromObject(scene);
if (bounds.isEmpty()) {
return {
position: [0, 0, 0] as [number, number, number],
scale: 1,
};
}
const size = bounds.getSize(new THREE.Vector3());
const center = bounds.getCenter(new THREE.Vector3());
const largestDimension = Math.max(size.x, size.y, size.z);
if (!Number.isFinite(largestDimension) || largestDimension <= 0) {
return {
position: [0, 0, 0] as [number, number, number],
scale: 1,
};
}
const scale = 2 / largestDimension;
return {
position: [
-center.x * scale,
-center.y * scale,
-center.z * scale,
] as [number, number, number],
scale,
};
}
function getEasedProgress(progress: number, easing: string) {
if (easing.toLowerCase() === "linear") return progress;
return progress < 0.5
? 4 * progress * progress * progress
: 1 - Math.pow(-2 * progress + 2, 3) / 2;
}
function getPartTargetPosition(
entry: InteractiveEntry,
manifest: OpenClawModelInteractionManifest,
amount: number
) {
const offsetMm = getOpenClawExplodedOffsetMm(entry.part);
if (!offsetMm) return entry.initialPosition.clone();
const offset = mapOpenClawCadOffsetToGlb(offsetMm, manifest.scaleMmToGlb);
return entry.initialPosition
.clone()
.add(new THREE.Vector3(offset.x, offset.y, offset.z).multiplyScalar(amount));
}
function ModelInteractionScene({
allowDragging,
manifest,
modelUrl,
onMatchChange,
onSelectedPartChange,
resetAllToken,
resetSelectedToken,
selectedNodeName,
transformRequest,
}: {
allowDragging: boolean;
manifest: OpenClawModelInteractionManifest;
modelUrl: string;
onMatchChange: (matched: number, total: number) => void;
onSelectedPartChange: (part: OpenClawModelInteractionPart | null) => void;
resetAllToken: number;
resetSelectedToken: number;
selectedNodeName: string | null;
transformRequest: OpenClawModelTransformRequest;
}) {
const { camera, gl, scene: rootScene } = useThree();
const controlsRef = useRef<OrbitControlsImpl | null>(null);
const gltf = useGLTF(modelUrl, true);
const scene = useMemo(() => cloneScene(gltf.scene), [gltf.scene]);
const fitTransform = useMemo(() => getFitTransform(scene), [scene]);
const entriesRef = useRef<Map<string, InteractiveEntry>>(new Map());
const objectEntriesRef = useRef<Map<Object3D, InteractiveEntry>>(new Map());
const animationRef = useRef<AnimationState | null>(null);
const dragRef = useRef<DragState | null>(null);
const currentAmountRef = useRef(0);
const selectionHelperRef = useRef<THREE.BoxHelper | null>(null);
const getCameraFrame = (
starts: Map<string, THREE.Vector3>,
targets: Map<string, THREE.Vector3>
) => {
for (const [nodeName, target] of targets) {
entriesRef.current.get(nodeName)?.node.position.copy(target);
}
scene.updateMatrixWorld(true);
const bounds = new THREE.Box3().setFromObject(scene);
for (const [nodeName, start] of starts) {
entriesRef.current.get(nodeName)?.node.position.copy(start);
}
scene.updateMatrixWorld(true);
const controlsTargetStart = controlsRef.current?.target?.clone?.() ||
new THREE.Vector3();
if (bounds.isEmpty()) {
return {
cameraTarget: camera.position.clone(),
controlsTargetEnd: controlsTargetStart.clone(),
};
}
const sphere = bounds.getBoundingSphere(new THREE.Sphere());
const perspectiveCamera = camera as THREE.PerspectiveCamera;
const verticalHalfFov = THREE.MathUtils.degToRad(perspectiveCamera.fov) / 2;
const horizontalHalfFov = Math.atan(
Math.tan(verticalHalfFov) * perspectiveCamera.aspect
);
const limitingHalfFov = Math.max(
0.1,
Math.min(verticalHalfFov, horizontalHalfFov)
);
const distance = Math.max(
1.2,
(sphere.radius / Math.sin(limitingHalfFov)) * 1.12
);
const direction = camera.position.clone().sub(controlsTargetStart);
if (direction.lengthSq() < 0.0001) direction.set(1, 0.75, 1);
direction.normalize();
return {
cameraTarget: sphere.center.clone().addScaledVector(direction, distance),
controlsTargetEnd: sphere.center.clone(),
};
};
useEffect(() => {
const partsByNodeName = new Map(
manifest.parts.map((part) => [part.nodeName, part])
);
const entries = new Map<string, InteractiveEntry>();
const objectEntries = new Map<Object3D, InteractiveEntry>();
scene.traverse((node) => {
const part = partsByNodeName.get(node.name);
if (!part || entries.has(part.nodeName)) return;
const entry = {
node,
part,
initialPosition: node.position.clone(),
};
entries.set(part.nodeName, entry);
objectEntries.set(node, entry);
});
entriesRef.current = entries;
objectEntriesRef.current = objectEntries;
onMatchChange(entries.size, manifest.parts.length);
return () => {
entriesRef.current.clear();
objectEntriesRef.current.clear();
onMatchChange(0, manifest.parts.length);
};
}, [manifest, onMatchChange, scene]);
useEffect(() => {
const previous = selectionHelperRef.current;
if (previous) {
rootScene.remove(previous);
previous.geometry.dispose();
(previous.material as THREE.Material).dispose();
selectionHelperRef.current = null;
}
if (!selectedNodeName) return;
const entry = entriesRef.current.get(selectedNodeName);
if (!entry) return;
const helper = new THREE.BoxHelper(entry.node, 0x3ecf8e);
helper.raycast = () => undefined;
rootScene.add(helper);
selectionHelperRef.current = helper;
return () => {
rootScene.remove(helper);
helper.geometry.dispose();
(helper.material as THREE.Material).dispose();
if (selectionHelperRef.current === helper) {
selectionHelperRef.current = null;
}
};
}, [rootScene, selectedNodeName]);
useEffect(() => {
const amount = THREE.MathUtils.clamp(transformRequest.amount, 0, 1);
currentAmountRef.current = amount;
const starts = new Map<string, THREE.Vector3>();
const targets = new Map<string, THREE.Vector3>();
for (const [nodeName, entry] of entriesRef.current) {
starts.set(nodeName, entry.node.position.clone());
targets.set(nodeName, getPartTargetPosition(entry, manifest, amount));
}
const cameraFrame = getCameraFrame(starts, targets);
if (!transformRequest.animate) {
animationRef.current = null;
for (const [nodeName, target] of targets) {
entriesRef.current.get(nodeName)?.node.position.copy(target);
}
camera.position.copy(cameraFrame.cameraTarget);
if (controlsRef.current) {
controlsRef.current.target.copy(cameraFrame.controlsTargetEnd);
controlsRef.current.update();
} else {
camera.lookAt(cameraFrame.controlsTargetEnd);
}
return;
}
const targetState = amount >= 0.5 ? "exploded" : "assembled";
const action =
manifest.actions.find((item) => item.id === transformRequest.actionId) ||
manifest.actions.find((item) => item.toState === targetState);
animationRef.current = {
startedAt: performance.now(),
durationMs: action?.durationMs ?? transformRequest.durationMs,
starts,
targets,
easing: action?.easing || "easeInOutCubic",
cameraStart: camera.position.clone(),
cameraTarget: cameraFrame.cameraTarget,
controlsTargetStart:
controlsRef.current?.target?.clone?.() || new THREE.Vector3(),
controlsTargetEnd: cameraFrame.controlsTargetEnd,
};
}, [camera, manifest, scene, transformRequest]);
useEffect(() => {
if (resetAllToken <= 0) return;
animationRef.current = null;
for (const entry of entriesRef.current.values()) {
entry.node.position.copy(
getPartTargetPosition(entry, manifest, currentAmountRef.current)
);
}
}, [manifest, resetAllToken]);
useEffect(() => {
if (resetSelectedToken <= 0 || !selectedNodeName) return;
const entry = entriesRef.current.get(selectedNodeName);
if (!entry) return;
entry.node.position.copy(
getPartTargetPosition(entry, manifest, currentAmountRef.current)
);
}, [manifest, resetSelectedToken, selectedNodeName]);
useEffect(() => {
const element = gl.domElement;
const raycaster = new THREE.Raycaster();
const pointer = new THREE.Vector2();
const planeIntersection = new THREE.Vector3();
const worldPosition = new THREE.Vector3();
const updateRay = (event: PointerEvent) => {
const bounds = element.getBoundingClientRect();
if (bounds.width <= 0 || bounds.height <= 0) return false;
pointer.set(
((event.clientX - bounds.left) / bounds.width) * 2 - 1,
-((event.clientY - bounds.top) / bounds.height) * 2 + 1
);
raycaster.setFromCamera(pointer, camera);
return true;
};
const findEntry = (object: Object3D) => {
let current: Object3D | null = object;
while (current) {
const entry = objectEntriesRef.current.get(current);
if (entry) return entry;
current = current.parent;
}
return null;
};
const handlePointerDown = (event: PointerEvent) => {
if (event.button !== 0 || animationRef.current || !updateRay(event)) return;
const intersections = raycaster.intersectObjects(
[...entriesRef.current.values()].map((entry) => entry.node),
true
);
const entry = intersections
.map((intersection) => findEntry(intersection.object))
.find((candidate): candidate is InteractiveEntry => Boolean(candidate));
if (!entry) {
onSelectedPartChange(null);
return;
}
onSelectedPartChange(entry.part);
if (!allowDragging || !entry.part.draggable) return;
const normal = camera.getWorldDirection(new THREE.Vector3()).normalize();
const startWorldPosition = entry.node.getWorldPosition(new THREE.Vector3());
const plane = new THREE.Plane().setFromNormalAndCoplanarPoint(
normal,
startWorldPosition
);
if (!raycaster.ray.intersectPlane(plane, planeIntersection)) return;
dragRef.current = {
entry,
plane,
startIntersection: planeIntersection.clone(),
startWorldPosition,
startLocalPosition: entry.node.position.clone(),
pointerId: event.pointerId,
};
if (controlsRef.current) controlsRef.current.enabled = false;
element.setPointerCapture?.(event.pointerId);
event.preventDefault();
event.stopImmediatePropagation();
};
const handlePointerMove = (event: PointerEvent) => {
const drag = dragRef.current;
if (!drag || drag.pointerId !== event.pointerId || !updateRay(event)) return;
if (!raycaster.ray.intersectPlane(drag.plane, planeIntersection)) return;
const targetWorld = worldPosition
.copy(drag.startWorldPosition)
.add(planeIntersection.clone().sub(drag.startIntersection));
const targetLocal = drag.entry.node.parent
? drag.entry.node.parent.worldToLocal(targetWorld.clone())
: targetWorld;
const axes = manifest.drag.axes;
if (!axes.includes("x")) targetLocal.x = drag.startLocalPosition.x;
if (!axes.includes("z")) targetLocal.y = drag.startLocalPosition.y;
if (!axes.includes("y")) targetLocal.z = drag.startLocalPosition.z;
const maxDistance = manifest.drag.maxDistanceMm * manifest.scaleMmToGlb;
const fromInitial = targetLocal.clone().sub(drag.entry.initialPosition);
if (maxDistance > 0 && fromInitial.length() > maxDistance) {
fromInitial.setLength(maxDistance);
targetLocal.copy(drag.entry.initialPosition).add(fromInitial);
}
drag.entry.node.position.copy(targetLocal);
event.preventDefault();
event.stopImmediatePropagation();
};
const finishDrag = (event: PointerEvent) => {
const drag = dragRef.current;
if (!drag || drag.pointerId !== event.pointerId) return;
const snapDistance =
manifest.drag.snapDistanceMm * manifest.scaleMmToGlb;
if (snapDistance > 0) {
const delta = drag.entry.node.position
.clone()
.sub(drag.entry.initialPosition);
delta.set(
Math.round(delta.x / snapDistance) * snapDistance,
Math.round(delta.y / snapDistance) * snapDistance,
Math.round(delta.z / snapDistance) * snapDistance
);
drag.entry.node.position.copy(drag.entry.initialPosition).add(delta);
}
element.releasePointerCapture?.(event.pointerId);
dragRef.current = null;
if (controlsRef.current) controlsRef.current.enabled = true;
event.preventDefault();
event.stopImmediatePropagation();
};
element.addEventListener("pointerdown", handlePointerDown, true);
window.addEventListener("pointermove", handlePointerMove, true);
window.addEventListener("pointerup", finishDrag, true);
window.addEventListener("pointercancel", finishDrag, true);
return () => {
element.removeEventListener("pointerdown", handlePointerDown, true);
window.removeEventListener("pointermove", handlePointerMove, true);
window.removeEventListener("pointerup", finishDrag, true);
window.removeEventListener("pointercancel", finishDrag, true);
dragRef.current = null;
};
}, [allowDragging, camera, gl, manifest, onSelectedPartChange]);
useEffect(() => {
return () => {
scene.traverse((node) => {
if (!isMesh(node)) return;
const materials = Array.isArray(node.material)
? node.material
: [node.material];
for (const material of materials) material.dispose();
});
};
}, [scene]);
useFrame(() => {
selectionHelperRef.current?.update();
const animation = animationRef.current;
if (!animation) return;
const rawProgress =
animation.durationMs <= 0
? 1
: Math.min(1, (performance.now() - animation.startedAt) / animation.durationMs);
const progress = getEasedProgress(rawProgress, animation.easing);
for (const [nodeName, target] of animation.targets) {
const entry = entriesRef.current.get(nodeName);
const start = animation.starts.get(nodeName);
if (entry && start) entry.node.position.lerpVectors(start, target, progress);
}
camera.position.lerpVectors(
animation.cameraStart,
animation.cameraTarget,
progress
);
if (controlsRef.current) {
controlsRef.current.target.lerpVectors(
animation.controlsTargetStart,
animation.controlsTargetEnd,
progress
);
controlsRef.current.update();
} else {
camera.lookAt(animation.controlsTargetEnd);
}
if (rawProgress >= 1) animationRef.current = null;
});
return (
<>
<ambientLight intensity={0.75} />
<directionalLight castShadow intensity={1.6} position={[4, 7, 5]} />
<directionalLight intensity={0.45} position={[-4, 2, -3]} />
<primitive
object={scene}
position={fitTransform.position}
scale={fitTransform.scale}
/>
<Environment preset="studio" />
<OrbitControls
dampingFactor={0.08}
enableDamping
enablePan
enableRotate
enableZoom
makeDefault
maxPolarAngle={Math.PI / 1.65}
minDistance={1.2}
minPolarAngle={0}
ref={controlsRef}
/>
</>
);
}
export function OpenClawInteractiveModelViewer({
allowDragging,
background,
manifest,
modelUrl,
onMatchChange,
onSelectedPartChange,
previewImageUrl,
resetAllToken,
resetSelectedToken,
resetViewToken,
selectedNodeName,
transformRequest,
}: {
allowDragging: boolean;
background: string;
manifest: OpenClawModelInteractionManifest;
modelUrl: string;
onMatchChange: (matched: number, total: number) => void;
onSelectedPartChange: (part: OpenClawModelInteractionPart | null) => void;
previewImageUrl?: string | null;
resetAllToken: number;
resetSelectedToken: number;
resetViewToken: number;
selectedNodeName: string | null;
transformRequest: OpenClawModelTransformRequest;
}) {
return (
<InteractiveViewerErrorBoundary
resetKey={`${modelUrl}:${resetViewToken}`}
fallback={
<div className="relative flex h-full w-full items-center justify-center overflow-hidden bg-zinc-950 text-zinc-200">
{previewImageUrl ? (
<img
alt="3D 模型静态预览"
className="absolute inset-0 h-full w-full object-contain opacity-80"
src={previewImageUrl}
/>
) : null}
<div className="relative flex max-w-xs flex-col items-center gap-2 rounded-md bg-black/70 px-4 py-3 text-center backdrop-blur-sm">
<AlertCircle className="h-6 w-6 text-amber-300" />
<p className="text-xs">3D 交互预览加载失败,已显示静态预览</p>
</div>
</div>
}
>
<Canvas
camera={{ fov: 42, position: [2.4, 1.8, 2.4] }}
dpr={[1, 1.75]}
gl={{ antialias: true, powerPreference: "high-performance" }}
key={resetViewToken}
shadows
style={{ background }}
>
<Suspense fallback={null}>
<ModelInteractionScene
allowDragging={allowDragging}
manifest={manifest}
modelUrl={modelUrl}
onMatchChange={onMatchChange}
onSelectedPartChange={onSelectedPartChange}
resetAllToken={resetAllToken}
resetSelectedToken={resetSelectedToken}
selectedNodeName={selectedNodeName}
transformRequest={transformRequest}
/>
</Suspense>
</Canvas>
</InteractiveViewerErrorBoundary>
);
}