ai-chatbot/components/canvas/chat-panel/OpenClawInteractiveModelViewer.tsx

599 lines
20 KiB
TypeScript

"use client";
/* eslint-disable @next/next/no-img-element */
import { OrbitControls, useGLTF } from "@react-three/drei";
import { Canvas, useFrame, useThree } from "@react-three/fiber";
import { AlertCircle } from "lucide-react";
import React, { Suspense, useEffect, useMemo, useRef } from "react";
import * as THREE from "three";
import type { Mesh, Object3D } from "three";
import type { OrbitControls as OrbitControlsImpl } from "three-stdlib";
import { LocalStudioEnvironment } from "@/components/canvas/model3d/LocalStudioEnvironment";
import { ModelLoadingFallback } from "@/components/canvas/model3d/ModelLoadingFallback";
import type {
OpenClawModelInteractionManifest,
OpenClawModelInteractionPart,
} from "@/lib/openclaw-chat/model-interaction";
import {
getOpenClawExplodedOffsetMm,
getOpenClawPartForSceneNode,
getOpenClawSourceNodeOffset,
mapOpenClawCadOffsetToGlb,
} from "@/lib/openclaw-chat/model-interaction";
export type OpenClawModelTransformRequest = {
actionId: string | null;
amount: number;
animate: boolean;
durationMs: number;
nonce: number;
};
export type OpenClawModelCameraState = {
position: [number, number, number];
target: [number, number, number];
};
type InteractiveEntry = {
node: Object3D;
part: OpenClawModelInteractionPart;
initialPosition: THREE.Vector3;
};
type AnimationState = {
startedAt: number;
durationMs: number;
starts: Map<string, THREE.Vector3>;
targets: Map<string, THREE.Vector3>;
easing: string;
};
type DragState = {
entry: InteractiveEntry;
plane: THREE.Plane;
startIntersection: THREE.Vector3;
startWorldPosition: THREE.Vector3;
startLocalPosition: THREE.Vector3;
pointerId: number;
};
class InteractiveViewerErrorBoundary extends React.Component<
{ children: React.ReactNode; fallback: React.ReactNode; resetKey: string },
{ hasError: boolean }
> {
state = { hasError: false };
static getDerivedStateFromError() {
return { hasError: true };
}
componentDidUpdate(previousProps: { resetKey: string }) {
if (
this.state.hasError &&
previousProps.resetKey !== this.props.resetKey
) {
this.setState({ hasError: false });
}
}
render() {
return this.state.hasError ? this.props.fallback : this.props.children;
}
}
function isMesh(node: Object3D): node is Mesh {
return (node as Mesh).isMesh === true;
}
function cloneScene(scene: Object3D) {
const clone = scene.clone(true);
clone.traverse((node) => {
if (!isMesh(node)) return;
node.castShadow = true;
node.receiveShadow = true;
const materials = Array.isArray(node.material)
? node.material
: [node.material];
const clones = materials.map((material) => material.clone());
node.material = Array.isArray(node.material)
? clones
: clones[0] || node.material;
});
return clone;
}
function getFitTransform(scene: Object3D) {
const bounds = new THREE.Box3().setFromObject(scene);
if (bounds.isEmpty()) {
return {
position: [0, 0, 0] as [number, number, number],
scale: 1,
};
}
const size = bounds.getSize(new THREE.Vector3());
const center = bounds.getCenter(new THREE.Vector3());
const largestDimension = Math.max(size.x, size.y, size.z);
if (!Number.isFinite(largestDimension) || largestDimension <= 0) {
return {
position: [0, 0, 0] as [number, number, number],
scale: 1,
};
}
const scale = 2 / largestDimension;
return {
position: [
-center.x * scale,
-center.y * scale,
-center.z * scale,
] as [number, number, number],
scale,
};
}
function getEasedProgress(progress: number, easing: string) {
if (easing.toLowerCase() === "linear") return progress;
return progress < 0.5
? 4 * progress * progress * progress
: 1 - Math.pow(-2 * progress + 2, 3) / 2;
}
function getPartTargetPosition(
entry: InteractiveEntry,
manifest: OpenClawModelInteractionManifest,
amount: number
) {
const offsetMm = getOpenClawExplodedOffsetMm(entry.part);
if (!offsetMm) return entry.initialPosition.clone();
const offset = mapOpenClawCadOffsetToGlb(offsetMm, manifest.scaleMmToGlb);
return entry.initialPosition
.clone()
.add(new THREE.Vector3(offset.x, offset.y, offset.z).multiplyScalar(amount));
}
function ModelInteractionScene({
allowDragging,
fitExplodedSource,
initialCameraState,
interactionParts,
manifest,
modelUrl,
onMatchChange,
onCameraStateChange,
onSelectedPartChange,
resetAllToken,
resetSelectedToken,
selectedNodeName,
sourceVariant,
transformRequest,
}: {
allowDragging: boolean;
fitExplodedSource: boolean;
initialCameraState?: OpenClawModelCameraState | null;
interactionParts: OpenClawModelInteractionPart[];
manifest: OpenClawModelInteractionManifest;
modelUrl: string;
onMatchChange: (matched: number, total: number) => void;
onCameraStateChange?: (state: OpenClawModelCameraState) => void;
onSelectedPartChange: (part: OpenClawModelInteractionPart | null) => void;
resetAllToken: number;
resetSelectedToken: number;
selectedNodeName: string | null;
sourceVariant: "assembled" | "exploded";
transformRequest: OpenClawModelTransformRequest;
}) {
const { camera, gl, scene: rootScene } = useThree();
const controlsRef = useRef<OrbitControlsImpl | null>(null);
const gltf = useGLTF(modelUrl, true);
const scene = useMemo(() => {
const cloned = cloneScene(gltf.scene);
if (sourceVariant !== "exploded") return cloned;
const partsByNodeName = new Map(
interactionParts.map((part) => [part.nodeName, part])
);
cloned.traverse((node) => {
const part = getOpenClawPartForSceneNode(node, partsByNodeName);
if (!part) return;
const offset = getOpenClawSourceNodeOffset(
part,
sourceVariant,
manifest.scaleMmToGlb
);
node.position.add(new THREE.Vector3(offset.x, offset.y, offset.z));
});
cloned.updateMatrixWorld(true);
return cloned;
}, [gltf.scene, interactionParts, manifest.scaleMmToGlb, sourceVariant]);
const fitTransform = useMemo(
() => getFitTransform(fitExplodedSource ? gltf.scene : scene),
[fitExplodedSource, gltf.scene, scene]
);
const entriesRef = useRef<Map<string, InteractiveEntry>>(new Map());
const objectEntriesRef = useRef<Map<Object3D, InteractiveEntry>>(new Map());
const animationRef = useRef<AnimationState | null>(null);
const dragRef = useRef<DragState | null>(null);
const currentAmountRef = useRef(0);
const selectionHelperRef = useRef<THREE.BoxHelper | null>(null);
useEffect(() => {
const partsByNodeName = new Map(
interactionParts.map((part) => [part.nodeName, part])
);
const entries = new Map<string, InteractiveEntry>();
const objectEntries = new Map<Object3D, InteractiveEntry>();
scene.traverse((node) => {
const part = getOpenClawPartForSceneNode(node, partsByNodeName);
if (!part || entries.has(part.nodeName)) return;
const entry = {
node,
part,
initialPosition: node.position.clone(),
};
entries.set(part.nodeName, entry);
objectEntries.set(node, entry);
});
entriesRef.current = entries;
objectEntriesRef.current = objectEntries;
onMatchChange(entries.size, interactionParts.length);
return () => {
entriesRef.current.clear();
objectEntriesRef.current.clear();
onMatchChange(0, interactionParts.length);
};
}, [interactionParts, onMatchChange, scene]);
useEffect(() => {
if (!initialCameraState || !controlsRef.current) return;
camera.position.set(...initialCameraState.position);
controlsRef.current.target.set(...initialCameraState.target);
camera.updateProjectionMatrix();
controlsRef.current.update();
}, [camera, initialCameraState]);
useEffect(() => {
const previous = selectionHelperRef.current;
if (previous) {
rootScene.remove(previous);
previous.geometry.dispose();
(previous.material as THREE.Material).dispose();
selectionHelperRef.current = null;
}
if (!selectedNodeName) return;
const entry = entriesRef.current.get(selectedNodeName);
if (!entry) return;
const helper = new THREE.BoxHelper(entry.node, 0x3ecf8e);
helper.raycast = () => undefined;
rootScene.add(helper);
selectionHelperRef.current = helper;
return () => {
rootScene.remove(helper);
helper.geometry.dispose();
(helper.material as THREE.Material).dispose();
if (selectionHelperRef.current === helper) {
selectionHelperRef.current = null;
}
};
}, [rootScene, selectedNodeName]);
useEffect(() => {
const amount = THREE.MathUtils.clamp(transformRequest.amount, 0, 1);
currentAmountRef.current = amount;
const starts = new Map<string, THREE.Vector3>();
const targets = new Map<string, THREE.Vector3>();
for (const [nodeName, entry] of entriesRef.current) {
starts.set(nodeName, entry.node.position.clone());
targets.set(nodeName, getPartTargetPosition(entry, manifest, amount));
}
if (!transformRequest.animate) {
animationRef.current = null;
for (const [nodeName, target] of targets) {
entriesRef.current.get(nodeName)?.node.position.copy(target);
}
return;
}
const targetState = amount >= 0.5 ? "exploded" : "assembled";
const action =
manifest.actions.find((item) => item.id === transformRequest.actionId) ||
manifest.actions.find((item) => item.toState === targetState);
animationRef.current = {
startedAt: performance.now(),
durationMs: action?.durationMs ?? transformRequest.durationMs,
starts,
targets,
easing: action?.easing || "easeInOutCubic",
};
}, [manifest, transformRequest]);
useEffect(() => {
if (resetAllToken <= 0) return;
animationRef.current = null;
for (const entry of entriesRef.current.values()) {
entry.node.position.copy(
getPartTargetPosition(entry, manifest, currentAmountRef.current)
);
}
}, [manifest, resetAllToken]);
useEffect(() => {
if (resetSelectedToken <= 0 || !selectedNodeName) return;
const entry = entriesRef.current.get(selectedNodeName);
if (!entry) return;
entry.node.position.copy(
getPartTargetPosition(entry, manifest, currentAmountRef.current)
);
}, [manifest, resetSelectedToken, selectedNodeName]);
useEffect(() => {
const element = gl.domElement;
const raycaster = new THREE.Raycaster();
const pointer = new THREE.Vector2();
const planeIntersection = new THREE.Vector3();
const worldPosition = new THREE.Vector3();
const updateRay = (event: PointerEvent) => {
const bounds = element.getBoundingClientRect();
if (bounds.width <= 0 || bounds.height <= 0) return false;
pointer.set(
((event.clientX - bounds.left) / bounds.width) * 2 - 1,
-((event.clientY - bounds.top) / bounds.height) * 2 + 1
);
raycaster.setFromCamera(pointer, camera);
return true;
};
const findEntry = (object: Object3D) => {
let current: Object3D | null = object;
while (current) {
const entry = objectEntriesRef.current.get(current);
if (entry) return entry;
current = current.parent;
}
return null;
};
const handlePointerDown = (event: PointerEvent) => {
if (event.button !== 0 || animationRef.current || !updateRay(event)) return;
const intersections = raycaster.intersectObjects(
[...entriesRef.current.values()].map((entry) => entry.node),
true
);
const entry = intersections
.map((intersection) => findEntry(intersection.object))
.find((candidate): candidate is InteractiveEntry => Boolean(candidate));
if (!entry) {
onSelectedPartChange(null);
return;
}
onSelectedPartChange(entry.part);
if (!allowDragging || !entry.part.draggable) return;
const normal = camera.getWorldDirection(new THREE.Vector3()).normalize();
const startWorldPosition = entry.node.getWorldPosition(new THREE.Vector3());
const plane = new THREE.Plane().setFromNormalAndCoplanarPoint(
normal,
startWorldPosition
);
if (!raycaster.ray.intersectPlane(plane, planeIntersection)) return;
dragRef.current = {
entry,
plane,
startIntersection: planeIntersection.clone(),
startWorldPosition,
startLocalPosition: entry.node.position.clone(),
pointerId: event.pointerId,
};
if (controlsRef.current) controlsRef.current.enabled = false;
element.setPointerCapture?.(event.pointerId);
event.preventDefault();
event.stopImmediatePropagation();
};
const handlePointerMove = (event: PointerEvent) => {
const drag = dragRef.current;
if (!drag || drag.pointerId !== event.pointerId || !updateRay(event)) return;
if (!raycaster.ray.intersectPlane(drag.plane, planeIntersection)) return;
const targetWorld = worldPosition
.copy(drag.startWorldPosition)
.add(planeIntersection.clone().sub(drag.startIntersection));
const targetLocal = drag.entry.node.parent
? drag.entry.node.parent.worldToLocal(targetWorld.clone())
: targetWorld;
const axes = manifest.drag.axes;
if (!axes.includes("x")) targetLocal.x = drag.startLocalPosition.x;
if (!axes.includes("z")) targetLocal.y = drag.startLocalPosition.y;
if (!axes.includes("y")) targetLocal.z = drag.startLocalPosition.z;
const maxDistance = manifest.drag.maxDistanceMm * manifest.scaleMmToGlb;
const fromInitial = targetLocal.clone().sub(drag.entry.initialPosition);
if (maxDistance > 0 && fromInitial.length() > maxDistance) {
fromInitial.setLength(maxDistance);
targetLocal.copy(drag.entry.initialPosition).add(fromInitial);
}
drag.entry.node.position.copy(targetLocal);
event.preventDefault();
event.stopImmediatePropagation();
};
const finishDrag = (event: PointerEvent) => {
const drag = dragRef.current;
if (!drag || drag.pointerId !== event.pointerId) return;
const snapDistance =
manifest.drag.snapDistanceMm * manifest.scaleMmToGlb;
if (snapDistance > 0) {
const delta = drag.entry.node.position
.clone()
.sub(drag.entry.initialPosition);
delta.set(
Math.round(delta.x / snapDistance) * snapDistance,
Math.round(delta.y / snapDistance) * snapDistance,
Math.round(delta.z / snapDistance) * snapDistance
);
drag.entry.node.position.copy(drag.entry.initialPosition).add(delta);
}
element.releasePointerCapture?.(event.pointerId);
dragRef.current = null;
if (controlsRef.current) controlsRef.current.enabled = true;
event.preventDefault();
event.stopImmediatePropagation();
};
element.addEventListener("pointerdown", handlePointerDown, true);
window.addEventListener("pointermove", handlePointerMove, true);
window.addEventListener("pointerup", finishDrag, true);
window.addEventListener("pointercancel", finishDrag, true);
return () => {
element.removeEventListener("pointerdown", handlePointerDown, true);
window.removeEventListener("pointermove", handlePointerMove, true);
window.removeEventListener("pointerup", finishDrag, true);
window.removeEventListener("pointercancel", finishDrag, true);
dragRef.current = null;
};
}, [allowDragging, camera, gl, manifest, onSelectedPartChange]);
useEffect(() => {
return () => {
scene.traverse((node) => {
if (!isMesh(node)) return;
const materials = Array.isArray(node.material)
? node.material
: [node.material];
for (const material of materials) material.dispose();
});
};
}, [scene]);
useFrame(() => {
selectionHelperRef.current?.update();
const animation = animationRef.current;
if (!animation) return;
const rawProgress =
animation.durationMs <= 0
? 1
: Math.min(1, (performance.now() - animation.startedAt) / animation.durationMs);
const progress = getEasedProgress(rawProgress, animation.easing);
for (const [nodeName, target] of animation.targets) {
const entry = entriesRef.current.get(nodeName);
const start = animation.starts.get(nodeName);
if (entry && start) entry.node.position.lerpVectors(start, target, progress);
}
if (rawProgress >= 1) animationRef.current = null;
});
return (
<>
<ambientLight intensity={0.75} />
<directionalLight castShadow intensity={1.6} position={[4, 7, 5]} />
<directionalLight intensity={0.45} position={[-4, 2, -3]} />
<primitive
object={scene}
position={fitTransform.position}
scale={fitTransform.scale}
/>
<OrbitControls
dampingFactor={0.08}
enableDamping
enablePan
enableRotate
enableZoom
makeDefault
maxPolarAngle={Math.PI / 1.65}
minDistance={1.2}
minPolarAngle={0}
onChange={() => {
const controls = controlsRef.current;
if (!controls || !onCameraStateChange) return;
onCameraStateChange({
position: camera.position.toArray() as [number, number, number],
target: controls.target.toArray() as [number, number, number],
});
}}
ref={controlsRef}
/>
</>
);
}
export function OpenClawInteractiveModelViewer({
allowDragging,
background,
fitExplodedSource = false,
initialCameraState,
interactionParts,
manifest,
modelUrl,
onMatchChange,
onCameraStateChange,
onSelectedPartChange,
previewImageUrl,
resetAllToken,
resetSelectedToken,
resetViewToken,
selectedNodeName,
sourceVariant,
transformRequest,
}: {
allowDragging: boolean;
background: string;
fitExplodedSource?: boolean;
initialCameraState?: OpenClawModelCameraState | null;
interactionParts?: OpenClawModelInteractionPart[];
manifest: OpenClawModelInteractionManifest;
modelUrl: string;
onMatchChange: (matched: number, total: number) => void;
onCameraStateChange?: (state: OpenClawModelCameraState) => void;
onSelectedPartChange: (part: OpenClawModelInteractionPart | null) => void;
previewImageUrl?: string | null;
resetAllToken: number;
resetSelectedToken: number;
resetViewToken: number;
selectedNodeName: string | null;
sourceVariant: "assembled" | "exploded";
transformRequest: OpenClawModelTransformRequest;
}) {
return (
<InteractiveViewerErrorBoundary
resetKey={`${modelUrl}:${resetViewToken}`}
fallback={
<div className="relative flex h-full w-full items-center justify-center overflow-hidden bg-zinc-950 text-zinc-200">
{previewImageUrl ? (
<img
alt="3D 模型静态预览"
className="absolute inset-0 h-full w-full object-contain opacity-80"
src={previewImageUrl}
/>
) : null}
<div className="relative flex max-w-xs flex-col items-center gap-2 rounded-md bg-black/70 px-4 py-3 text-center backdrop-blur-sm">
<AlertCircle className="h-6 w-6 text-amber-300" />
<p className="text-xs">{previewImageUrl ? "3D 交互预览加载失败,可查看静态预览或下载模型" : "3D 交互预览加载失败,请退出预览后重试或下载模型"}</p>
</div>
</div>
}
>
<Canvas
camera={{ fov: 42, position: [2.4, 1.8, 2.4] }}
dpr={[1, 1.75]}
gl={{ antialias: true, powerPreference: "high-performance" }}
key={`${modelUrl}:${resetViewToken}`}
shadows
style={{ background }}
>
<LocalStudioEnvironment />
<Suspense fallback={<ModelLoadingFallback />}>
<ModelInteractionScene
allowDragging={allowDragging}
fitExplodedSource={fitExplodedSource}
initialCameraState={initialCameraState}
interactionParts={interactionParts || manifest.parts}
manifest={manifest}
modelUrl={modelUrl}
onMatchChange={onMatchChange}
onCameraStateChange={onCameraStateChange}
onSelectedPartChange={onSelectedPartChange}
resetAllToken={resetAllToken}
resetSelectedToken={resetSelectedToken}
selectedNodeName={selectedNodeName}
sourceVariant={sourceVariant}
transformRequest={transformRequest}
/>
</Suspense>
</Canvas>
</InteractiveViewerErrorBoundary>
);
}