From bbfece57200134f439d969ef8efeebd919d3cb58 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 2 Sep 2026 20:21:40 +0800 Subject: [PATCH] feat(claw): support interactive 3D previews --- .../OpenClawInteractiveModelViewer.tsx | 604 ++++++++++++++++++ .../chat-panel/OpenClawModel3DPreview.tsx | 390 +++++++++-- lib/openclaw-chat/artifacts.ts | 128 ++++ lib/openclaw-chat/model-interaction.ts | 289 +++++++++ tests/routes/openclaw-3d-artifacts.test.ts | 151 +++++ 5 files changed, 1513 insertions(+), 49 deletions(-) create mode 100644 components/canvas/chat-panel/OpenClawInteractiveModelViewer.tsx create mode 100644 lib/openclaw-chat/model-interaction.ts diff --git a/components/canvas/chat-panel/OpenClawInteractiveModelViewer.tsx b/components/canvas/chat-panel/OpenClawInteractiveModelViewer.tsx new file mode 100644 index 000000000..dae71ef76 --- /dev/null +++ b/components/canvas/chat-panel/OpenClawInteractiveModelViewer.tsx @@ -0,0 +1,604 @@ +"use client"; + +/* eslint-disable @next/next/no-img-element */ +import { Environment, OrbitControls, useGLTF } from "@react-three/drei"; +import { Canvas, useFrame, useThree } from "@react-three/fiber"; +import { AlertCircle } from "lucide-react"; +import React, { Suspense, useEffect, useMemo, useRef } from "react"; +import * as THREE from "three"; +import type { Mesh, Object3D } from "three"; +import type { OrbitControls as OrbitControlsImpl } from "three-stdlib"; +import type { + OpenClawModelInteractionManifest, + OpenClawModelInteractionPart, +} from "@/lib/openclaw-chat/model-interaction"; +import { + getOpenClawExplodedOffsetMm, + mapOpenClawCadOffsetToGlb, +} from "@/lib/openclaw-chat/model-interaction"; + +export type OpenClawModelTransformRequest = { + actionId: string | null; + amount: number; + animate: boolean; + durationMs: number; + nonce: number; +}; + +type InteractiveEntry = { + node: Object3D; + part: OpenClawModelInteractionPart; + initialPosition: THREE.Vector3; +}; + +type AnimationState = { + startedAt: number; + durationMs: number; + starts: Map; + targets: Map; + easing: string; + cameraStart: THREE.Vector3; + cameraTarget: THREE.Vector3; + controlsTargetStart: THREE.Vector3; + controlsTargetEnd: THREE.Vector3; +}; + +type DragState = { + entry: InteractiveEntry; + plane: THREE.Plane; + startIntersection: THREE.Vector3; + startWorldPosition: THREE.Vector3; + startLocalPosition: THREE.Vector3; + pointerId: number; +}; + +class InteractiveViewerErrorBoundary extends React.Component< + { children: React.ReactNode; fallback: React.ReactNode; resetKey: string }, + { hasError: boolean } +> { + state = { hasError: false }; + + static getDerivedStateFromError() { + return { hasError: true }; + } + + componentDidUpdate(previousProps: { resetKey: string }) { + if ( + this.state.hasError && + previousProps.resetKey !== this.props.resetKey + ) { + this.setState({ hasError: false }); + } + } + + render() { + return this.state.hasError ? this.props.fallback : this.props.children; + } +} + +function isMesh(node: Object3D): node is Mesh { + return (node as Mesh).isMesh === true; +} + +function cloneScene(scene: Object3D) { + const clone = scene.clone(true); + clone.traverse((node) => { + if (!isMesh(node)) return; + node.castShadow = true; + node.receiveShadow = true; + const materials = Array.isArray(node.material) + ? node.material + : [node.material]; + const clones = materials.map((material) => material.clone()); + node.material = Array.isArray(node.material) + ? clones + : clones[0] || node.material; + }); + return clone; +} + +function getFitTransform(scene: Object3D) { + const bounds = new THREE.Box3().setFromObject(scene); + if (bounds.isEmpty()) { + return { + position: [0, 0, 0] as [number, number, number], + scale: 1, + }; + } + const size = bounds.getSize(new THREE.Vector3()); + const center = bounds.getCenter(new THREE.Vector3()); + const largestDimension = Math.max(size.x, size.y, size.z); + if (!Number.isFinite(largestDimension) || largestDimension <= 0) { + return { + position: [0, 0, 0] as [number, number, number], + scale: 1, + }; + } + const scale = 2 / largestDimension; + return { + position: [ + -center.x * scale, + -center.y * scale, + -center.z * scale, + ] as [number, number, number], + scale, + }; +} + +function getEasedProgress(progress: number, easing: string) { + if (easing.toLowerCase() === "linear") return progress; + return progress < 0.5 + ? 4 * progress * progress * progress + : 1 - Math.pow(-2 * progress + 2, 3) / 2; +} + +function getPartTargetPosition( + entry: InteractiveEntry, + manifest: OpenClawModelInteractionManifest, + amount: number +) { + const offsetMm = getOpenClawExplodedOffsetMm(entry.part); + if (!offsetMm) return entry.initialPosition.clone(); + const offset = mapOpenClawCadOffsetToGlb(offsetMm, manifest.scaleMmToGlb); + return entry.initialPosition + .clone() + .add(new THREE.Vector3(offset.x, offset.y, offset.z).multiplyScalar(amount)); +} + +function ModelInteractionScene({ + allowDragging, + manifest, + modelUrl, + onMatchChange, + onSelectedPartChange, + resetAllToken, + resetSelectedToken, + selectedNodeName, + transformRequest, +}: { + allowDragging: boolean; + manifest: OpenClawModelInteractionManifest; + modelUrl: string; + onMatchChange: (matched: number, total: number) => void; + onSelectedPartChange: (part: OpenClawModelInteractionPart | null) => void; + resetAllToken: number; + resetSelectedToken: number; + selectedNodeName: string | null; + transformRequest: OpenClawModelTransformRequest; +}) { + const { camera, gl, scene: rootScene } = useThree(); + const controlsRef = useRef(null); + const gltf = useGLTF(modelUrl, true); + const scene = useMemo(() => cloneScene(gltf.scene), [gltf.scene]); + const fitTransform = useMemo(() => getFitTransform(scene), [scene]); + const entriesRef = useRef>(new Map()); + const objectEntriesRef = useRef>(new Map()); + const animationRef = useRef(null); + const dragRef = useRef(null); + const currentAmountRef = useRef(0); + const selectionHelperRef = useRef(null); + + const getCameraFrame = ( + starts: Map, + targets: Map + ) => { + for (const [nodeName, target] of targets) { + entriesRef.current.get(nodeName)?.node.position.copy(target); + } + scene.updateMatrixWorld(true); + const bounds = new THREE.Box3().setFromObject(scene); + for (const [nodeName, start] of starts) { + entriesRef.current.get(nodeName)?.node.position.copy(start); + } + scene.updateMatrixWorld(true); + + const controlsTargetStart = controlsRef.current?.target?.clone?.() || + new THREE.Vector3(); + if (bounds.isEmpty()) { + return { + cameraTarget: camera.position.clone(), + controlsTargetEnd: controlsTargetStart.clone(), + }; + } + const sphere = bounds.getBoundingSphere(new THREE.Sphere()); + const perspectiveCamera = camera as THREE.PerspectiveCamera; + const verticalHalfFov = THREE.MathUtils.degToRad(perspectiveCamera.fov) / 2; + const horizontalHalfFov = Math.atan( + Math.tan(verticalHalfFov) * perspectiveCamera.aspect + ); + const limitingHalfFov = Math.max( + 0.1, + Math.min(verticalHalfFov, horizontalHalfFov) + ); + const distance = Math.max( + 1.2, + (sphere.radius / Math.sin(limitingHalfFov)) * 1.12 + ); + const direction = camera.position.clone().sub(controlsTargetStart); + if (direction.lengthSq() < 0.0001) direction.set(1, 0.75, 1); + direction.normalize(); + return { + cameraTarget: sphere.center.clone().addScaledVector(direction, distance), + controlsTargetEnd: sphere.center.clone(), + }; + }; + + useEffect(() => { + const partsByNodeName = new Map( + manifest.parts.map((part) => [part.nodeName, part]) + ); + const entries = new Map(); + const objectEntries = new Map(); + scene.traverse((node) => { + const part = partsByNodeName.get(node.name); + if (!part || entries.has(part.nodeName)) return; + const entry = { + node, + part, + initialPosition: node.position.clone(), + }; + entries.set(part.nodeName, entry); + objectEntries.set(node, entry); + }); + entriesRef.current = entries; + objectEntriesRef.current = objectEntries; + onMatchChange(entries.size, manifest.parts.length); + + return () => { + entriesRef.current.clear(); + objectEntriesRef.current.clear(); + onMatchChange(0, manifest.parts.length); + }; + }, [manifest, onMatchChange, scene]); + + useEffect(() => { + const previous = selectionHelperRef.current; + if (previous) { + rootScene.remove(previous); + previous.geometry.dispose(); + (previous.material as THREE.Material).dispose(); + selectionHelperRef.current = null; + } + if (!selectedNodeName) return; + const entry = entriesRef.current.get(selectedNodeName); + if (!entry) return; + const helper = new THREE.BoxHelper(entry.node, 0x3ecf8e); + helper.raycast = () => undefined; + rootScene.add(helper); + selectionHelperRef.current = helper; + return () => { + rootScene.remove(helper); + helper.geometry.dispose(); + (helper.material as THREE.Material).dispose(); + if (selectionHelperRef.current === helper) { + selectionHelperRef.current = null; + } + }; + }, [rootScene, selectedNodeName]); + + useEffect(() => { + const amount = THREE.MathUtils.clamp(transformRequest.amount, 0, 1); + currentAmountRef.current = amount; + const starts = new Map(); + const targets = new Map(); + for (const [nodeName, entry] of entriesRef.current) { + starts.set(nodeName, entry.node.position.clone()); + targets.set(nodeName, getPartTargetPosition(entry, manifest, amount)); + } + const cameraFrame = getCameraFrame(starts, targets); + if (!transformRequest.animate) { + animationRef.current = null; + for (const [nodeName, target] of targets) { + entriesRef.current.get(nodeName)?.node.position.copy(target); + } + camera.position.copy(cameraFrame.cameraTarget); + if (controlsRef.current) { + controlsRef.current.target.copy(cameraFrame.controlsTargetEnd); + controlsRef.current.update(); + } else { + camera.lookAt(cameraFrame.controlsTargetEnd); + } + return; + } + const targetState = amount >= 0.5 ? "exploded" : "assembled"; + const action = + manifest.actions.find((item) => item.id === transformRequest.actionId) || + manifest.actions.find((item) => item.toState === targetState); + animationRef.current = { + startedAt: performance.now(), + durationMs: action?.durationMs ?? transformRequest.durationMs, + starts, + targets, + easing: action?.easing || "easeInOutCubic", + cameraStart: camera.position.clone(), + cameraTarget: cameraFrame.cameraTarget, + controlsTargetStart: + controlsRef.current?.target?.clone?.() || new THREE.Vector3(), + controlsTargetEnd: cameraFrame.controlsTargetEnd, + }; + }, [camera, manifest, scene, transformRequest]); + + useEffect(() => { + if (resetAllToken <= 0) return; + animationRef.current = null; + for (const entry of entriesRef.current.values()) { + entry.node.position.copy( + getPartTargetPosition(entry, manifest, currentAmountRef.current) + ); + } + }, [manifest, resetAllToken]); + + useEffect(() => { + if (resetSelectedToken <= 0 || !selectedNodeName) return; + const entry = entriesRef.current.get(selectedNodeName); + if (!entry) return; + entry.node.position.copy( + getPartTargetPosition(entry, manifest, currentAmountRef.current) + ); + }, [manifest, resetSelectedToken, selectedNodeName]); + + useEffect(() => { + const element = gl.domElement; + const raycaster = new THREE.Raycaster(); + const pointer = new THREE.Vector2(); + const planeIntersection = new THREE.Vector3(); + const worldPosition = new THREE.Vector3(); + + const updateRay = (event: PointerEvent) => { + const bounds = element.getBoundingClientRect(); + if (bounds.width <= 0 || bounds.height <= 0) return false; + pointer.set( + ((event.clientX - bounds.left) / bounds.width) * 2 - 1, + -((event.clientY - bounds.top) / bounds.height) * 2 + 1 + ); + raycaster.setFromCamera(pointer, camera); + return true; + }; + const findEntry = (object: Object3D) => { + let current: Object3D | null = object; + while (current) { + const entry = objectEntriesRef.current.get(current); + if (entry) return entry; + current = current.parent; + } + return null; + }; + const handlePointerDown = (event: PointerEvent) => { + if (event.button !== 0 || animationRef.current || !updateRay(event)) return; + const intersections = raycaster.intersectObjects( + [...entriesRef.current.values()].map((entry) => entry.node), + true + ); + const entry = intersections + .map((intersection) => findEntry(intersection.object)) + .find((candidate): candidate is InteractiveEntry => Boolean(candidate)); + if (!entry) { + onSelectedPartChange(null); + return; + } + onSelectedPartChange(entry.part); + if (!allowDragging || !entry.part.draggable) return; + + const normal = camera.getWorldDirection(new THREE.Vector3()).normalize(); + const startWorldPosition = entry.node.getWorldPosition(new THREE.Vector3()); + const plane = new THREE.Plane().setFromNormalAndCoplanarPoint( + normal, + startWorldPosition + ); + if (!raycaster.ray.intersectPlane(plane, planeIntersection)) return; + dragRef.current = { + entry, + plane, + startIntersection: planeIntersection.clone(), + startWorldPosition, + startLocalPosition: entry.node.position.clone(), + pointerId: event.pointerId, + }; + if (controlsRef.current) controlsRef.current.enabled = false; + element.setPointerCapture?.(event.pointerId); + event.preventDefault(); + event.stopImmediatePropagation(); + }; + const handlePointerMove = (event: PointerEvent) => { + const drag = dragRef.current; + if (!drag || drag.pointerId !== event.pointerId || !updateRay(event)) return; + if (!raycaster.ray.intersectPlane(drag.plane, planeIntersection)) return; + const targetWorld = worldPosition + .copy(drag.startWorldPosition) + .add(planeIntersection.clone().sub(drag.startIntersection)); + const targetLocal = drag.entry.node.parent + ? drag.entry.node.parent.worldToLocal(targetWorld.clone()) + : targetWorld; + const axes = manifest.drag.axes; + if (!axes.includes("x")) targetLocal.x = drag.startLocalPosition.x; + if (!axes.includes("z")) targetLocal.y = drag.startLocalPosition.y; + if (!axes.includes("y")) targetLocal.z = drag.startLocalPosition.z; + + const maxDistance = manifest.drag.maxDistanceMm * manifest.scaleMmToGlb; + const fromInitial = targetLocal.clone().sub(drag.entry.initialPosition); + if (maxDistance > 0 && fromInitial.length() > maxDistance) { + fromInitial.setLength(maxDistance); + targetLocal.copy(drag.entry.initialPosition).add(fromInitial); + } + drag.entry.node.position.copy(targetLocal); + event.preventDefault(); + event.stopImmediatePropagation(); + }; + const finishDrag = (event: PointerEvent) => { + const drag = dragRef.current; + if (!drag || drag.pointerId !== event.pointerId) return; + const snapDistance = + manifest.drag.snapDistanceMm * manifest.scaleMmToGlb; + if (snapDistance > 0) { + const delta = drag.entry.node.position + .clone() + .sub(drag.entry.initialPosition); + delta.set( + Math.round(delta.x / snapDistance) * snapDistance, + Math.round(delta.y / snapDistance) * snapDistance, + Math.round(delta.z / snapDistance) * snapDistance + ); + drag.entry.node.position.copy(drag.entry.initialPosition).add(delta); + } + element.releasePointerCapture?.(event.pointerId); + dragRef.current = null; + if (controlsRef.current) controlsRef.current.enabled = true; + event.preventDefault(); + event.stopImmediatePropagation(); + }; + + element.addEventListener("pointerdown", handlePointerDown, true); + window.addEventListener("pointermove", handlePointerMove, true); + window.addEventListener("pointerup", finishDrag, true); + window.addEventListener("pointercancel", finishDrag, true); + return () => { + element.removeEventListener("pointerdown", handlePointerDown, true); + window.removeEventListener("pointermove", handlePointerMove, true); + window.removeEventListener("pointerup", finishDrag, true); + window.removeEventListener("pointercancel", finishDrag, true); + dragRef.current = null; + }; + }, [allowDragging, camera, gl, manifest, onSelectedPartChange]); + + useEffect(() => { + return () => { + scene.traverse((node) => { + if (!isMesh(node)) return; + const materials = Array.isArray(node.material) + ? node.material + : [node.material]; + for (const material of materials) material.dispose(); + }); + }; + }, [scene]); + + useFrame(() => { + selectionHelperRef.current?.update(); + const animation = animationRef.current; + if (!animation) return; + const rawProgress = + animation.durationMs <= 0 + ? 1 + : Math.min(1, (performance.now() - animation.startedAt) / animation.durationMs); + const progress = getEasedProgress(rawProgress, animation.easing); + for (const [nodeName, target] of animation.targets) { + const entry = entriesRef.current.get(nodeName); + const start = animation.starts.get(nodeName); + if (entry && start) entry.node.position.lerpVectors(start, target, progress); + } + camera.position.lerpVectors( + animation.cameraStart, + animation.cameraTarget, + progress + ); + if (controlsRef.current) { + controlsRef.current.target.lerpVectors( + animation.controlsTargetStart, + animation.controlsTargetEnd, + progress + ); + controlsRef.current.update(); + } else { + camera.lookAt(animation.controlsTargetEnd); + } + if (rawProgress >= 1) animationRef.current = null; + }); + + return ( + <> + + + + + + + + ); +} + +export function OpenClawInteractiveModelViewer({ + allowDragging, + background, + manifest, + modelUrl, + onMatchChange, + onSelectedPartChange, + previewImageUrl, + resetAllToken, + resetSelectedToken, + resetViewToken, + selectedNodeName, + transformRequest, +}: { + allowDragging: boolean; + background: string; + manifest: OpenClawModelInteractionManifest; + modelUrl: string; + onMatchChange: (matched: number, total: number) => void; + onSelectedPartChange: (part: OpenClawModelInteractionPart | null) => void; + previewImageUrl?: string | null; + resetAllToken: number; + resetSelectedToken: number; + resetViewToken: number; + selectedNodeName: string | null; + transformRequest: OpenClawModelTransformRequest; +}) { + return ( + + {previewImageUrl ? ( + 3D 模型静态预览 + ) : null} +
+ +

3D 交互预览加载失败，已显示静态预览

+
+ + } + > + + + + + + + ); +} diff --git a/components/canvas/chat-panel/OpenClawModel3DPreview.tsx b/components/canvas/chat-panel/OpenClawModel3DPreview.tsx index 6a648c889..46177adf9 100644 --- a/components/canvas/chat-panel/OpenClawModel3DPreview.tsx +++ b/components/canvas/chat-panel/OpenClawModel3DPreview.tsx @@ -2,13 +2,27 @@ /* eslint-disable @next/next/no-img-element */ import dynamic from "next/dynamic"; -import { Box, Download, Loader2, RotateCcw } from "lucide-react"; -import { useMemo, useState } from "react"; +import { + Box, + Download, + Loader2, + Maximize2, + Minimize2, + RotateCcw, + Undo2, + X, +} from "lucide-react"; +import { useCallback, useEffect, useId, useMemo, useState } from "react"; import { buildOpenClawArtifactProxyUrl, - isOpenClawInteractive3DArtifact, + getOpenClaw3DArtifactBundle, type OpenClawArtifact, } from "@/lib/openclaw-chat/artifacts"; +import { + normalizeOpenClawModelInteractionManifest, + type OpenClawModelInteractionManifest, + type OpenClawModelInteractionPart, +} from "@/lib/openclaw-chat/model-interaction"; const Model3DViewer = dynamic( () => @@ -17,14 +31,31 @@ const Model3DViewer = dynamic( ), { ssr: false, - loading: () => ( -
- -
- ), + loading: () => , } ); +const OpenClawInteractiveModelViewer = dynamic( + () => + import( + "@/components/canvas/chat-panel/OpenClawInteractiveModelViewer" + ).then((module) => module.OpenClawInteractiveModelViewer), + { + ssr: false, + loading: () => , + } +); + +const ACTIVE_VIEWER_EVENT = "openclaw-active-3d-viewer"; + +function ViewerLoading() { + return ( +
+ +
+ ); +} + function supportsWebGl() { try { const canvas = document.createElement("canvas"); @@ -38,6 +69,12 @@ function supportsWebGl() { } } +function formatDimensions(part: OpenClawModelInteractionPart) { + const dimensions = part.dimensionsMm; + if (!dimensions) return null; + return `${dimensions.x} x ${dimensions.y} x ${dimensions.z} mm`; +} + export function OpenClawModel3DPreview({ artifacts, chatId, @@ -47,40 +84,146 @@ export function OpenClawModel3DPreview({ chatId: string; messageId: string; }) { - const modelIndex = useMemo( - () => artifacts.findIndex(isOpenClawInteractive3DArtifact), + const viewerId = useId(); + const bundle = useMemo( + () => getOpenClaw3DArtifactBundle(artifacts), [artifacts] ); - const model = modelIndex >= 0 ? artifacts[modelIndex] : null; - const fallbackRole = model?.viewer?.fallbackArtifactRole || "cad_preview"; - const previewIndex = artifacts.findIndex( - (artifact) => artifact.artifactRole === fallbackRole - ); + const model = bundle?.model || null; const [isActive, setIsActive] = useState(false); const [webGlUnavailable, setWebGlUnavailable] = useState(false); const [resetViewToken, setResetViewToken] = useState(0); + const [resetSelectedToken, setResetSelectedToken] = useState(0); + const [resetAllToken, setResetAllToken] = useState(0); + const [manifest, setManifest] = + useState(); + const [manifestError, setManifestError] = useState(null); + const [selectedPart, setSelectedPart] = + useState(null); + const [matchState, setMatchState] = useState({ matched: 0, total: 0 }); + const [explodeAmount, setExplodeAmount] = useState(0); + const [transformRequest, setTransformRequest] = useState({ + actionId: null as string | null, + amount: 0, + animate: false, + durationMs: 700, + nonce: 0, + }); - if (!model) return null; + const closeViewer = useCallback(() => { + setIsActive(false); + setSelectedPart(null); + setMatchState({ matched: 0, total: 0 }); + }, []); + const handleMatchChange = useCallback((matched: number, total: number) => { + setMatchState((current) => + current.matched === matched && current.total === total + ? current + : { matched, total } + ); + }, []); + + useEffect(() => { + const handleOtherViewerActivated = (event: Event) => { + const activeId = (event as CustomEvent).detail; + if (activeId !== viewerId) closeViewer(); + }; + window.addEventListener(ACTIVE_VIEWER_EVENT, handleOtherViewerActivated); + return () => { + window.removeEventListener(ACTIVE_VIEWER_EVENT, handleOtherViewerActivated); + }; + }, [closeViewer, viewerId]); + + useEffect(() => { + if (!isActive) return; + const handleKeyDown = (event: KeyboardEvent) => { + if (event.key !== "Escape") return; + event.preventDefault(); + closeViewer(); + }; + window.addEventListener("keydown", handleKeyDown); + return () => window.removeEventListener("keydown", handleKeyDown); + }, [closeViewer, isActive]); + + useEffect(() => { + if (!isActive || !bundle || manifest !== undefined) return; + if (bundle.manifestIndex < 0) { + setManifest(null); + return; + } + const controller = new AbortController(); + const manifestUrl = buildOpenClawArtifactProxyUrl({ + chatId, + messageId, + artifactIndex: bundle.manifestIndex, + }); + setManifestError(null); + fetch(manifestUrl, { cache: "no-store", signal: controller.signal }) + .then(async (response) => { + if (!response.ok) { + throw new Error(`交互清单读取失败（HTTP ${response.status}）`); + } + const normalized = normalizeOpenClawModelInteractionManifest( + await response.json() + ); + if (!normalized) throw new Error("交互清单格式不正确"); + setManifest(normalized); + }) + .catch((error) => { + if (controller.signal.aborted) return; + setManifest(null); + setManifestError( + error instanceof Error ? error.message : "交互清单读取失败" + ); + }); + return () => controller.abort(); + }, [bundle, chatId, isActive, manifest, messageId]); + + if (!bundle || !model) return null; const modelUrl = buildOpenClawArtifactProxyUrl({ chatId, messageId, - artifactIndex: modelIndex, + artifactIndex: bundle.modelIndex, }); const downloadUrl = buildOpenClawArtifactProxyUrl({ chatId, messageId, - artifactIndex: modelIndex, + artifactIndex: bundle.modelIndex, download: true, }); const previewUrl = - previewIndex >= 0 + bundle.previewIndex >= 0 ? buildOpenClawArtifactProxyUrl({ chatId, messageId, - artifactIndex: previewIndex, + artifactIndex: bundle.previewIndex, }) : null; + const isAssembledModel = + model.artifactRole.toLowerCase() === "assembled_3d" || + model.viewer?.variant === "assembled"; + const supportsNodeInteraction = Boolean(manifest && isAssembledModel); + const canDragParts = Boolean( + supportsNodeInteraction && + (model.viewer?.partDragging || manifest?.parts.some((part) => part.draggable)) + ); + const canExplode = Boolean( + supportsNodeInteraction && + (model.viewer?.oneClickExplode || manifest?.capabilities.oneClickExplode) + ); + const canAssemble = Boolean( + supportsNodeInteraction && + (model.viewer?.oneClickAssemble || manifest?.capabilities.oneClickAssemble) + ); + const showExplodeSlider = Boolean( + supportsNodeInteraction && + (model.viewer?.explodeSlider || manifest?.capabilities.explodeSlider) + ); + const canResetParts = Boolean( + supportsNodeInteraction && + (model.viewer?.resetParts || manifest?.parts.some((part) => part.draggable)) + ); const activateViewer = () => { if (!supportsWebGl()) { @@ -88,12 +231,31 @@ export function OpenClawModel3DPreview({ return; } setWebGlUnavailable(false); + window.dispatchEvent(new CustomEvent(ACTIVE_VIEWER_EVENT, { detail: viewerId })); setIsActive(true); }; + const updateExplodeAmount = ( + amount: number, + animate: boolean, + actionId: string | null = null + ) => { + const normalized = Math.max(0, Math.min(1, amount)); + setExplodeAmount(normalized); + setTransformRequest((current) => ({ + actionId, + amount: normalized, + animate, + durationMs: model.viewer?.transitionDurationMs || 700, + nonce: current.nonce + 1, + })); + }; + const selectedDimensions = selectedPart + ? formatDimensions(selectedPart) + : null; return (
-
+
@@ -105,15 +267,27 @@ export function OpenClawModel3DPreview({
{isActive ? ( - + <> + + + ) : null}
{isActive ? ( - + manifest === undefined && bundle.manifestIndex >= 0 ? ( + + ) : supportsNodeInteraction && manifest ? ( + <> + + + {selectedPart ? ( +
+

+ {selectedPart.partId} · {selectedPart.partName} +

+ {selectedDimensions ? ( +

+ {selectedDimensions} +

+ ) : null} +
+ ) : null} + +
+
+ {canExplode ? ( + + ) : null} + {canAssemble ? ( + + ) : null} + {selectedPart && canResetParts ? ( + + ) : null} + {canResetParts ? ( + + ) : null} + {matchState.total > 0 && matchState.matched < matchState.total ? ( + + 已识别 {matchState.matched}/{matchState.total} 个配件 + + ) : null} +
+ {showExplodeSlider ? ( + + ) : null} +
+ + ) : ( + + ) ) : (
{previewUrl ? ( @@ -160,7 +450,7 @@ export function OpenClawModel3DPreview({ )}
)}
- {webGlUnavailable ? ( + {webGlUnavailable || manifestError ? (

- 当前浏览器无法启动3D预览，已保留静态预览和模型下载。 + {webGlUnavailable + ? "当前浏览器无法启动3D预览，已保留静态预览和模型下载。" + : `${manifestError}，已切换为基础3D预览。`}

) : null}
diff --git a/lib/openclaw-chat/artifacts.ts b/lib/openclaw-chat/artifacts.ts index 7cbb88f7c..4eeb845ef 100644 --- a/lib/openclaw-chat/artifacts.ts +++ b/lib/openclaw-chat/artifacts.ts @@ -1,10 +1,23 @@ export type OpenClawArtifactViewer = { type: "model_3d"; format: "glb" | "gltf"; + variant: "assembled" | "exploded" | null; + isDefault: boolean; cameraControls: boolean; autoRotate: boolean; background: string | null; fallbackArtifactRole: string | null; + alternateArtifactRole: string | null; + interactionManifestArtifactRole: string | null; + partSelection: boolean; + partDragging: boolean; + resetParts: boolean; + oneClickExplode: boolean; + oneClickAssemble: boolean; + explodeActionId: string | null; + assembleActionId: string | null; + transitionDurationMs: number; + explodeSlider: boolean; }; export type OpenClawArtifact = { @@ -40,6 +53,18 @@ function readBoolean(value: unknown, fallback: boolean) { return typeof value === "boolean" ? value : fallback; } +function readNumber( + value: unknown, + fallback: number, + limits?: { min?: number; max?: number } +) { + const parsed = Number(value); + if (!Number.isFinite(parsed)) return fallback; + if (limits?.min !== undefined && parsed < limits.min) return fallback; + if (limits?.max !== undefined && parsed > limits.max) return fallback; + return parsed; +} + function normalizeArtifactUrl(value: unknown, baseUrl?: string) { const raw = readText(value, MAX_URL_LENGTH); if (!raw) return ""; @@ -90,6 +115,7 @@ function normalizeViewer( if (type && type !== "model_3d") return null; const background = readText(raw.background, 32); + const variant = readText(raw.variant, 32).toLowerCase(); return { type: "model_3d", format: inferViewerFormat({ @@ -97,11 +123,48 @@ function normalizeViewer( mimeType: artifact.mimeType, value: raw.format, }), + variant: + variant === "assembled" || variant === "exploded" ? variant : null, + isDefault: readBoolean(raw.default ?? raw.isDefault, false), cameraControls: readBoolean(raw.camera_controls ?? raw.cameraControls, true), autoRotate: readBoolean(raw.auto_rotate ?? raw.autoRotate, false), background: background || null, fallbackArtifactRole: readText(raw.fallback_artifact_role ?? raw.fallbackArtifactRole) || null, + alternateArtifactRole: + readText(raw.alternate_artifact_role ?? raw.alternateArtifactRole) || null, + interactionManifestArtifactRole: + readText( + raw.interaction_manifest_artifact_role ?? + raw.interactionManifestArtifactRole + ) || null, + partSelection: readBoolean( + raw.part_selection ?? raw.partSelection, + false + ), + partDragging: readBoolean(raw.part_dragging ?? raw.partDragging, false), + resetParts: readBoolean(raw.reset_parts ?? raw.resetParts, false), + oneClickExplode: readBoolean( + raw.one_click_explode ?? raw.oneClickExplode, + false + ), + oneClickAssemble: readBoolean( + raw.one_click_assemble ?? raw.oneClickAssemble, + false + ), + explodeActionId: + readText(raw.explode_action_id ?? raw.explodeActionId, 160) || null, + assembleActionId: + readText(raw.assemble_action_id ?? raw.assembleActionId, 160) || null, + transitionDurationMs: readNumber( + raw.transition_duration_ms ?? raw.transitionDurationMs, + 700, + { min: 0, max: 30_000 } + ), + explodeSlider: readBoolean( + raw.explode_slider ?? raw.explodeSlider, + false + ), }; } @@ -194,6 +257,7 @@ export function isOpenClawInteractive3DArtifact( const mimeType = artifact.mimeType.toLowerCase(); const filename = artifact.filename.toLowerCase(); return ( + role === "assembled_3d" || role === "interactive_3d" || mimeType === "model/gltf-binary" || mimeType === "model/gltf+json" || @@ -202,6 +266,70 @@ export function isOpenClawInteractive3DArtifact( ); } +export type OpenClaw3DArtifactBundle = { + modelIndex: number; + model: OpenClawArtifact; + alternateModelIndex: number; + manifestIndex: number; + previewIndex: number; +}; + +export function getOpenClaw3DArtifactBundle( + artifacts: OpenClawArtifact[] +): OpenClaw3DArtifactBundle | null { + const modelIndexes = artifacts + .map((artifact, index) => + isOpenClawInteractive3DArtifact(artifact) ? index : -1 + ) + .filter((index) => index >= 0); + if (modelIndexes.length === 0) return null; + + const modelIndex = + modelIndexes.find((index) => artifacts[index]?.viewer?.isDefault) ?? + modelIndexes.find( + (index) => artifacts[index]?.artifactRole.toLowerCase() === "assembled_3d" + ) ?? + modelIndexes[0]; + const model = artifacts[modelIndex]; + if (!model) return null; + + const alternateRole = model.viewer?.alternateArtifactRole?.toLowerCase(); + const alternateModelIndex = alternateRole + ? artifacts.findIndex( + (artifact, index) => + index !== modelIndex && + artifact.artifactRole.toLowerCase() === alternateRole && + isOpenClawInteractive3DArtifact(artifact) + ) + : modelIndexes.find((index) => index !== modelIndex) ?? -1; + const manifestRole = + model.viewer?.interactionManifestArtifactRole?.toLowerCase() || + "model_interaction_manifest"; + const manifestIndex = artifacts.findIndex( + (artifact) => artifact.artifactRole.toLowerCase() === manifestRole + ); + const fallbackRole = + model.viewer?.fallbackArtifactRole?.toLowerCase() || + (model.viewer?.variant === "exploded" ? "cad_preview" : "assembled_preview"); + let previewIndex = artifacts.findIndex( + (artifact) => artifact.artifactRole.toLowerCase() === fallbackRole + ); + if (previewIndex < 0) { + previewIndex = artifacts.findIndex((artifact) => { + const role = artifact.artifactRole.toLowerCase(); + return role === "assembled_preview" || role === "cad_preview"; + }); + } + + return { + modelIndex, + model, + alternateModelIndex, + manifestIndex, + previewIndex, + }; +} + export function buildOpenClawArtifactProxyUrl(params: { chatId: string; messageId: string; diff --git a/lib/openclaw-chat/model-interaction.ts b/lib/openclaw-chat/model-interaction.ts new file mode 100644 index 000000000..870506196 --- /dev/null +++ b/lib/openclaw-chat/model-interaction.ts @@ -0,0 +1,289 @@ +export type OpenClawModelVector = { + x: number; + y: number; + z: number; +}; + +export type OpenClawModelInteractionPart = { + nodeName: string; + partId: string; + partName: string; + instanceIndex: number | null; + dimensionsMm: OpenClawModelVector | null; + assembledPositionMm: OpenClawModelVector | null; + explodedPositionMm: OpenClawModelVector | null; + explodedOffsetMm: OpenClawModelVector | null; + color: string | null; + draggable: boolean; +}; + +export type OpenClawModelInteractionAction = { + id: string; + label: string; + toState: "assembled" | "exploded" | "custom"; + offsetField: string | null; + durationMs: number; + easing: string; + interruptible: boolean; +}; + +export type OpenClawModelInteractionManifest = { + schema: string; + scaleMmToGlb: number; + defaultVariant: "assembled" | "exploded"; + parts: OpenClawModelInteractionPart[]; + actions: OpenClawModelInteractionAction[]; + drag: { + axes: Array<"x" | "y" | "z">; + snapDistanceMm: number; + maxDistanceMm: number; + }; + capabilities: { + oneClickExplode: boolean; + oneClickAssemble: boolean; + animatedTransition: boolean; + explodeSlider: boolean; + }; +}; + +const MAX_PARTS = 5000; +const MAX_ACTIONS = 32; +const MAX_TEXT_LENGTH = 500; +const MAX_ABSOLUTE_COORDINATE_MM = 10_000_000; + +function readRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function readText(value: unknown, maxLength = MAX_TEXT_LENGTH) { + return typeof value === "string" ? value.trim().slice(0, maxLength) : ""; +} + +function readBoolean(value: unknown, fallback: boolean) { + return typeof value === "boolean" ? value : fallback; +} + +function readFiniteNumber( + value: unknown, + fallback: number, + limits?: { min?: number; max?: number } +) { + const parsed = Number(value); + if (!Number.isFinite(parsed)) return fallback; + if (limits?.min !== undefined && parsed < limits.min) return fallback; + if (limits?.max !== undefined && parsed > limits.max) return fallback; + return parsed; +} + +function normalizeVector(value: unknown): OpenClawModelVector | null { + const raw = readRecord(value); + const vector = { + x: Number(raw.x), + y: Number(raw.y), + z: Number(raw.z), + }; + if ( + Object.values(vector).some( + (coordinate) => + !Number.isFinite(coordinate) || + Math.abs(coordinate) > MAX_ABSOLUTE_COORDINATE_MM + ) + ) { + return null; + } + return vector; +} + +function normalizePart(value: unknown): OpenClawModelInteractionPart | null { + const raw = readRecord(value); + const nodeName = readText(raw.node_name ?? raw.nodeName); + if (!nodeName) return null; + const instanceIndex = Number(raw.instance_index ?? raw.instanceIndex); + const color = readText(raw.color, 32); + + return { + nodeName, + partId: readText(raw.part_id ?? raw.partId) || nodeName, + partName: readText(raw.part_name ?? raw.partName) || "未命名配件", + instanceIndex: + Number.isInteger(instanceIndex) && instanceIndex >= 0 + ? instanceIndex + : null, + dimensionsMm: normalizeVector(raw.dimensions_mm ?? raw.dimensionsMm), + assembledPositionMm: normalizeVector( + raw.assembled_position_mm ?? raw.assembledPositionMm + ), + explodedPositionMm: normalizeVector( + raw.exploded_position_mm ?? raw.explodedPositionMm + ), + explodedOffsetMm: normalizeVector( + raw.exploded_offset_mm ?? raw.explodedOffsetMm + ), + color: color || null, + draggable: readBoolean(raw.draggable, true), + }; +} + +function normalizeAction( + value: unknown +): OpenClawModelInteractionAction | null { + const raw = readRecord(value); + const id = readText(raw.id, 160); + if (!id) return null; + const toState = readText(raw.to_state ?? raw.toState, 32).toLowerCase(); + return { + id, + label: readText(raw.label, 160) || id, + toState: + toState === "exploded" || toState === "custom" + ? toState + : "assembled", + offsetField: readText(raw.offset_field ?? raw.offsetField, 160) || null, + durationMs: readFiniteNumber( + raw.duration_ms ?? raw.durationMs, + 700, + { min: 0, max: 30_000 } + ), + easing: readText(raw.easing, 80) || "easeInOutCubic", + interruptible: readBoolean(raw.interruptible, true), + }; +} + +function hasExplodedTransform(part: OpenClawModelInteractionPart) { + return Boolean( + part.explodedOffsetMm || + (part.assembledPositionMm && part.explodedPositionMm) + ); +} + +function defaultActions(): OpenClawModelInteractionAction[] { + return [ + { + id: "explode_all", + label: "一键炸开", + toState: "exploded", + offsetField: "exploded_offset_mm", + durationMs: 700, + easing: "easeInOutCubic", + interruptible: true, + }, + { + id: "assemble_all", + label: "一键还原", + toState: "assembled", + offsetField: null, + durationMs: 700, + easing: "easeInOutCubic", + interruptible: true, + }, + ]; +} + +export function normalizeOpenClawModelInteractionManifest( + value: unknown +): OpenClawModelInteractionManifest | null { + const raw = readRecord(value); + if (Object.keys(raw).length === 0) return null; + + const seenNodeNames = new Set(); + const parts = (Array.isArray(raw.parts) ? raw.parts : []) + .slice(0, MAX_PARTS) + .map(normalizePart) + .filter((part): part is OpenClawModelInteractionPart => { + if (!part || seenNodeNames.has(part.nodeName)) return false; + seenNodeNames.add(part.nodeName); + return true; + }); + if (parts.length === 0) return null; + + const parsedActions = (Array.isArray(raw.actions) ? raw.actions : []) + .slice(0, MAX_ACTIONS) + .map(normalizeAction) + .filter((action): action is OpenClawModelInteractionAction => Boolean(action)); + const supportsTransforms = parts.some(hasExplodedTransform); + const actions = + parsedActions.length > 0 + ? parsedActions + : supportsTransforms + ? defaultActions() + : []; + const coordinates = readRecord(raw.coordinates); + const drag = readRecord(raw.drag); + const capabilities = readRecord(raw.capabilities); + const axes = (Array.isArray(drag.axes) ? drag.axes : ["x", "y", "z"]).filter( + (axis): axis is "x" | "y" | "z" => + axis === "x" || axis === "y" || axis === "z" + ); + const defaultVariant = readText( + raw.default_variant ?? raw.defaultVariant, + 32 + ).toLowerCase(); + + return { + schema: readText(raw.schema) || "dalongxia.furniture-model-interaction.v1", + scaleMmToGlb: readFiniteNumber( + coordinates.scale_mm_to_glb ?? coordinates.scaleMmToGlb, + 0.001, + { min: 0.000_000_1, max: 1 } + ), + defaultVariant: defaultVariant === "exploded" ? "exploded" : "assembled", + parts, + actions, + drag: { + axes: axes.length > 0 ? axes : ["x", "y", "z"], + snapDistanceMm: readFiniteNumber( + drag.snap_distance_mm ?? drag.snapDistanceMm, + 25, + { min: 0, max: 100_000 } + ), + maxDistanceMm: readFiniteNumber( + drag.max_distance_mm ?? drag.maxDistanceMm, + 1800, + { min: 0, max: MAX_ABSOLUTE_COORDINATE_MM } + ), + }, + capabilities: { + oneClickExplode: readBoolean( + capabilities.one_click_explode ?? capabilities.oneClickExplode, + supportsTransforms + ), + oneClickAssemble: readBoolean( + capabilities.one_click_assemble ?? capabilities.oneClickAssemble, + supportsTransforms + ), + animatedTransition: readBoolean( + capabilities.animated_transition ?? capabilities.animatedTransition, + supportsTransforms + ), + explodeSlider: readBoolean( + capabilities.explode_slider ?? capabilities.explodeSlider, + supportsTransforms + ), + }, + }; +} + +export function getOpenClawExplodedOffsetMm( + part: OpenClawModelInteractionPart +): OpenClawModelVector | null { + if (part.explodedOffsetMm) return part.explodedOffsetMm; + if (!part.assembledPositionMm || !part.explodedPositionMm) return null; + return { + x: part.explodedPositionMm.x - part.assembledPositionMm.x, + y: part.explodedPositionMm.y - part.assembledPositionMm.y, + z: part.explodedPositionMm.z - part.assembledPositionMm.z, + }; +} + +export function mapOpenClawCadOffsetToGlb( + offset: OpenClawModelVector, + scaleMmToGlb: number +): OpenClawModelVector { + return { + x: offset.x * scaleMmToGlb, + y: offset.z * scaleMmToGlb, + z: -offset.y * scaleMmToGlb, + }; +} diff --git a/tests/routes/openclaw-3d-artifacts.test.ts b/tests/routes/openclaw-3d-artifacts.test.ts index 7818584d9..751007981 100644 --- a/tests/routes/openclaw-3d-artifacts.test.ts +++ b/tests/routes/openclaw-3d-artifacts.test.ts @@ -2,9 +2,15 @@ import { expect, test } from "@playwright/test"; import { buildOpenClawArtifactProxyUrl, extractOpenClawArtifacts, + getOpenClaw3DArtifactBundle, isOpenClawInteractive3DArtifact, normalizeOpenClawArtifacts, } from "@/lib/openclaw-chat/artifacts"; +import { + getOpenClawExplodedOffsetMm, + mapOpenClawCadOffsetToGlb, + normalizeOpenClawModelInteractionManifest, +} from "@/lib/openclaw-chat/model-interaction"; import { normalizeClawdTaskMetadata } from "@/lib/openclaw-chat/task-state"; const modelFile = { @@ -34,6 +40,39 @@ const previewFile = { download_url: "https://files.example.com/wood-demo-preview.png", }; +const assembledModelFile = { + ...modelFile, + filename: "wood-demo-assembled.glb", + display_name: "可旋转3D整装成品模型", + artifact_role: "assembled_3d", + viewer: { + ...modelFile.viewer, + variant: "assembled", + default: true, + fallback_artifact_role: "assembled_preview", + alternate_artifact_role: "interactive_3d", + interaction_manifest_artifact_role: "model_interaction_manifest", + part_selection: true, + part_dragging: true, + reset_parts: true, + one_click_explode: true, + one_click_assemble: true, + explode_action_id: "explode_all", + assemble_action_id: "assemble_all", + transition_duration_ms: 850, + explode_slider: true, + }, +}; + +const manifestFile = { + type: "file", + filename: "wood-demo-model-interaction.json", + artifact_role: "model_interaction_manifest", + display_mode: "inline_only", + mime_type: "application/json", + download_url: "https://files.example.com/wood-demo-model-interaction.json", +}; + test.describe("OpenClaw 3D artifacts", () => { test("normalizes the documented GLB viewer contract", () => { const artifacts = normalizeOpenClawArtifacts([modelFile, previewFile], { @@ -60,6 +99,118 @@ test.describe("OpenClaw 3D artifacts", () => { expect(isOpenClawInteractive3DArtifact(artifacts[1])).toBe(false); }); + test("keeps the new viewer controls and selects the default assembled bundle", () => { + const artifacts = normalizeOpenClawArtifacts( + [ + modelFile, + manifestFile, + assembledModelFile, + { ...previewFile, artifact_role: "assembled_preview" }, + ], + { baseUrl: "https://claw.example.com" } + ); + const bundle = getOpenClaw3DArtifactBundle(artifacts); + + expect(bundle?.model.artifactRole).toBe("assembled_3d"); + expect(bundle?.model.viewer).toMatchObject({ + variant: "assembled", + isDefault: true, + alternateArtifactRole: "interactive_3d", + interactionManifestArtifactRole: "model_interaction_manifest", + partSelection: true, + partDragging: true, + resetParts: true, + oneClickExplode: true, + oneClickAssemble: true, + explodeActionId: "explode_all", + assembleActionId: "assemble_all", + transitionDurationMs: 850, + explodeSlider: true, + }); + expect(bundle?.manifestIndex).toBe(1); + expect(bundle?.alternateModelIndex).toBe(0); + expect(bundle?.previewIndex).toBe(3); + }); + + test("normalizes v2 interaction transforms and CAD-to-GLB coordinates", () => { + const manifest = normalizeOpenClawModelInteractionManifest({ + schema: "dalongxia.furniture-model-interaction.v2", + coordinates: { scale_mm_to_glb: 0.001 }, + capabilities: { + one_click_explode: true, + one_click_assemble: true, + animated_transition: true, + explode_slider: true, + }, + actions: [ + { + id: "explode_all", + label: "一键炸开", + to_state: "exploded", + offset_field: "exploded_offset_mm", + duration_ms: 700, + easing: "easeInOutCubic", + interruptible: true, + }, + ], + drag: { + axes: ["x", "y", "z"], + snap_distance_mm: 25, + max_distance_mm: 1800, + }, + parts: [ + { + node_name: "DOOR-001:1", + part_id: "DOOR-001", + part_name: "门板", + dimensions_mm: { x: 578, y: 20, z: 720 }, + exploded_offset_mm: { x: 2726, y: 120, z: -40 }, + draggable: true, + }, + ], + }); + + expect(manifest).not.toBeNull(); + expect(manifest?.parts[0]).toMatchObject({ + nodeName: "DOOR-001:1", + partId: "DOOR-001", + partName: "门板", + draggable: true, + }); + const offset = getOpenClawExplodedOffsetMm(manifest!.parts[0]); + expect(offset).toEqual({ x: 2726, y: 120, z: -40 }); + expect(mapOpenClawCadOffsetToGlb(offset!, manifest!.scaleMmToGlb)).toEqual({ + x: 2.726, + y: -0.04, + z: -0.12, + }); + }); + + test("derives exploded offsets and default actions for historical v1 manifests", () => { + const manifest = normalizeOpenClawModelInteractionManifest({ + schema: "dalongxia.furniture-model-interaction.v1", + parts: [ + { + node_name: "TOP-001:1", + part_id: "TOP-001", + assembled_position_mm: { x: 20, y: 10, z: 40 }, + exploded_position_mm: { x: 220, y: -40, z: 140 }, + }, + ], + }); + + expect(manifest?.actions.map((action) => action.id)).toEqual([ + "explode_all", + "assemble_all", + ]); + expect(getOpenClawExplodedOffsetMm(manifest!.parts[0])).toEqual({ + x: 200, + y: -50, + z: 100, + }); + expect(manifest?.capabilities.explodeSlider).toBe(true); + }); + test("extracts files from durable task results without duplicating them", () => { const artifacts = extractOpenClawArtifacts( {