568 lines
21 KiB
TypeScript
568 lines
21 KiB
TypeScript
"use client";
|
|
import { directorPoses } from "@/lib/video-canvas/director-poses";
|
|
import { forwardRef, useEffect, useImperativeHandle, useRef, useState } from "react";
|
|
import * as THREE from "three";
|
|
import { buildThreeAssetUrl } from "@/lib/canvas/three-asset-url";
|
|
import { OrbitControls } from "three/examples/jsm/controls/OrbitControls.js";
|
|
import { TransformControls } from "three/examples/jsm/controls/TransformControls.js";
|
|
import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
|
|
import {
|
|
directorImageSize,
|
|
type DirectorScene,
|
|
type DirectorObject,
|
|
} from "@/lib/video-canvas/director-scene";
|
|
import { sampleDirectorMotion, type DirectorPose } from "@/lib/video-canvas/director-motion";
|
|
import { recordDirectorVideo } from "@/lib/video-canvas/director-recording";
|
|
export type DirectorViewportHandle = {
|
|
showPath: () => void;
|
|
record: (signal: AbortSignal, onProgress: (time: number) => void) => Promise<Blob>;
|
|
capture: (pose?: DirectorPose) => Promise<{ blob: Blob; width: number; height: number }>;
|
|
};
|
|
type Props = {
|
|
scene: DirectorScene;
|
|
selected: string;
|
|
mode: "translate" | "rotate" | "scale";
|
|
cameraView: boolean;
|
|
previewTime: number | null;
|
|
locked: boolean;
|
|
onTransform: (id: string, patch: Partial<DirectorObject>) => void;
|
|
onCamera: (position: [number, number, number], target: [number, number, number]) => void;
|
|
};
|
|
function dispose(root: THREE.Object3D) {
|
|
root.traverse((o) => {
|
|
if (o instanceof THREE.Mesh || o instanceof THREE.Line) {
|
|
o.geometry.dispose();
|
|
for (const m of Array.isArray(o.material) ? o.material : [o.material]) {
|
|
for (const value of Object.values(m)) if (value instanceof THREE.Texture) value.dispose();
|
|
m.dispose();
|
|
}
|
|
}
|
|
});
|
|
}
|
|
export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
|
function DirectorViewport(props, ref) {
|
|
const host = useRef<HTMLDivElement>(null);
|
|
const latest = useRef(props);
|
|
latest.current = props;
|
|
const runtime = useRef<{
|
|
renderer: THREE.WebGLRenderer;
|
|
world: THREE.Scene;
|
|
camera: THREE.PerspectiveCamera;
|
|
orbit: OrbitControls;
|
|
transform: TransformControls;
|
|
helper: THREE.Object3D;
|
|
grid: THREE.GridHelper;
|
|
path: THREE.Group;
|
|
objects: THREE.Group;
|
|
pending: number;
|
|
resize: () => void;
|
|
} | null>(null);
|
|
const [ready, setReady] = useState(false);
|
|
const [error, setError] = useState("");
|
|
const [loading, setLoading] = useState(false);
|
|
useEffect(() => {
|
|
if (!host.current) return;
|
|
let renderer: THREE.WebGLRenderer;
|
|
try {
|
|
renderer = new THREE.WebGLRenderer({
|
|
antialias: true,
|
|
preserveDrawingBuffer: true,
|
|
});
|
|
} catch {
|
|
setError("无法初始化 3D 视图，请检查浏览器 WebGL 支持");
|
|
return;
|
|
}
|
|
renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
|
|
renderer.outputColorSpace = THREE.SRGBColorSpace;
|
|
renderer.setClearColor("#c5cbd3");
|
|
host.current.appendChild(renderer.domElement);
|
|
renderer.domElement.setAttribute("aria-label", "3D 导演台场景");
|
|
Object.assign(renderer.domElement.style, {
|
|
position: "absolute",
|
|
left: "50%",
|
|
top: "50%",
|
|
transform: "translate(-50%, -50%)",
|
|
});
|
|
const world = new THREE.Scene();
|
|
world.background = new THREE.Color("#c5cbd3");
|
|
world.add(new THREE.HemisphereLight(0xffffff, 0x647080, 2.4));
|
|
const light = new THREE.DirectionalLight(0xffffff, 3);
|
|
light.position.set(4, 8, 5);
|
|
world.add(light);
|
|
const floor = new THREE.Mesh(
|
|
new THREE.PlaneGeometry(100, 100),
|
|
new THREE.MeshStandardMaterial({ color: 0xb1b6be, roughness: 1 }),
|
|
);
|
|
floor.rotation.x = -Math.PI / 2;
|
|
floor.position.y = -0.02;
|
|
world.add(floor);
|
|
const grid = new THREE.GridHelper(30, 30, 0x596573, 0x929daa);
|
|
world.add(grid);
|
|
const path = new THREE.Group();
|
|
world.add(path);
|
|
const camera = new THREE.PerspectiveCamera(50, 1, 0.05, 500);
|
|
camera.position.set(4, 3, 7);
|
|
const orbit = new OrbitControls(camera, renderer.domElement);
|
|
orbit.target.set(0, 1, 0);
|
|
orbit.minDistance = 0.3;
|
|
orbit.maxDistance = 80;
|
|
const transform = new TransformControls(camera, renderer.domElement);
|
|
const helper = transform.getHelper();
|
|
world.add(helper);
|
|
const objects = new THREE.Group();
|
|
world.add(objects);
|
|
transform.addEventListener("dragging-changed", (e) => {
|
|
orbit.enabled =
|
|
!e.value && !latest.current.cameraView && latest.current.previewTime === null;
|
|
});
|
|
transform.addEventListener("mouseUp", () => {
|
|
const o = transform.object;
|
|
if (o)
|
|
latest.current.onTransform(o.name, {
|
|
position: o.position.toArray(),
|
|
rotation: [o.rotation.x, o.rotation.y, o.rotation.z],
|
|
scale: o.scale.toArray(),
|
|
});
|
|
});
|
|
orbit.addEventListener("end", () => {
|
|
if (latest.current.previewTime !== null || latest.current.cameraView) return;
|
|
latest.current.onCamera(camera.position.toArray(), orbit.target.toArray());
|
|
});
|
|
|
|
const resize = () => {
|
|
if (!host.current) return;
|
|
const { width, height } = host.current.getBoundingClientRect();
|
|
const size = directorImageSize(latest.current.scene.ratio);
|
|
const aspect = size.width / size.height;
|
|
const fitWidth = Math.max(1, Math.min(width, height * aspect));
|
|
renderer.setSize(fitWidth, fitWidth / aspect);
|
|
camera.aspect = aspect;
|
|
camera.updateProjectionMatrix();
|
|
};
|
|
const state = {
|
|
renderer,
|
|
world,
|
|
camera,
|
|
orbit,
|
|
transform,
|
|
helper,
|
|
grid,
|
|
path,
|
|
objects,
|
|
pending: 0,
|
|
resize,
|
|
};
|
|
runtime.current = state;
|
|
const observer = new ResizeObserver(resize);
|
|
observer.observe(host.current);
|
|
resize();
|
|
renderer.setAnimationLoop(() => {
|
|
if (orbit.enabled) orbit.update();
|
|
renderer.render(world, camera);
|
|
});
|
|
setReady(true);
|
|
return () => {
|
|
runtime.current = null;
|
|
observer.disconnect();
|
|
renderer.setAnimationLoop(null);
|
|
transform.dispose();
|
|
orbit.dispose();
|
|
dispose(world);
|
|
renderer.dispose();
|
|
renderer.domElement.remove();
|
|
};
|
|
}, []);
|
|
useEffect(() => {
|
|
const r = runtime.current;
|
|
if (!r || !ready) return;
|
|
let active = true;
|
|
r.transform.detach();
|
|
const desired = new Set(props.scene.objects.map((o) => o.id));
|
|
r.objects.children.slice().forEach((o) => {
|
|
if (!desired.has(o.name)) {
|
|
dispose(o);
|
|
r.objects.remove(o);
|
|
}
|
|
});
|
|
setError("");
|
|
r.pending = 0;
|
|
const attach = (object: THREE.Object3D, item: DirectorObject) => {
|
|
if (!active) {
|
|
dispose(object);
|
|
return;
|
|
}
|
|
object.name = item.id;
|
|
object.visible = item.visible !== false;
|
|
object.userData.directorSignature = JSON.stringify([
|
|
item.kind,
|
|
item.url,
|
|
item.pose,
|
|
item.color,
|
|
]);
|
|
object.position.fromArray(item.position);
|
|
object.rotation.set(...item.rotation);
|
|
object.scale.fromArray(item.scale);
|
|
r.objects.add(object);
|
|
if (
|
|
item.id === latest.current.selected &&
|
|
!item.locked &&
|
|
item.visible !== false &&
|
|
!latest.current.cameraView &&
|
|
latest.current.previewTime === null
|
|
)
|
|
r.transform.attach(object);
|
|
};
|
|
for (const item of props.scene.objects) {
|
|
const existing = r.objects.children.find((o) => o.name === item.id);
|
|
if (
|
|
existing &&
|
|
existing.userData.directorSignature ===
|
|
JSON.stringify([item.kind, item.url, item.pose, item.color])
|
|
) {
|
|
attach(existing, item);
|
|
continue;
|
|
}
|
|
if (existing) {
|
|
dispose(existing);
|
|
r.objects.remove(existing);
|
|
}
|
|
if (item.kind === "model" && item.url) {
|
|
r.pending++;
|
|
new GLTFLoader().load(
|
|
buildThreeAssetUrl(item.url),
|
|
(g) => {
|
|
if (active) {
|
|
const box = new THREE.Box3().setFromObject(g.scene);
|
|
const size = box.getSize(new THREE.Vector3());
|
|
const scale = 2 / Math.max(size.x, size.y, size.z, 0.001);
|
|
g.scene.scale.setScalar(scale);
|
|
const normalized = new THREE.Group();
|
|
normalized.add(g.scene);
|
|
const fitted = new THREE.Box3().setFromObject(normalized);
|
|
const center = fitted.getCenter(new THREE.Vector3());
|
|
g.scene.position.set(-center.x, -fitted.min.y, -center.z);
|
|
attach(normalized, item);
|
|
r.pending--;
|
|
setLoading(r.pending > 0);
|
|
} else dispose(g.scene);
|
|
},
|
|
undefined,
|
|
() => {
|
|
if (active) {
|
|
r.pending--;
|
|
setLoading(r.pending > 0);
|
|
setError("模型加载失败，请检查 GLB 文件及资源访问权限");
|
|
}
|
|
},
|
|
);
|
|
continue;
|
|
}
|
|
const mat = new THREE.MeshStandardMaterial({
|
|
color: item.color || (item.kind === "actor" ? "#687bb0" : "#c3664c"),
|
|
roughness: 0.6,
|
|
});
|
|
let object: THREE.Object3D;
|
|
if (item.kind === "actor") {
|
|
const group = new THREE.Group();
|
|
const pose = directorPoses[item.pose || "站立"] || directorPoses["站立"];
|
|
const radians = THREE.MathUtils.degToRad;
|
|
const hips = new THREE.Group();
|
|
hips.position.y = 1 + ("height" in pose ? pose.height : 0);
|
|
group.add(hips);
|
|
const torso = new THREE.Group();
|
|
torso.rotation.x = radians(pose.lean);
|
|
hips.add(torso);
|
|
const body = new THREE.Mesh(new THREE.BoxGeometry(0.52, 0.65, 0.28), mat);
|
|
body.position.y = 0.38;
|
|
torso.add(body);
|
|
const head = new THREE.Mesh(new THREE.SphereGeometry(0.2, 20, 16), mat.clone());
|
|
head.position.y = 0.95;
|
|
torso.add(head);
|
|
const limb = (
|
|
parent: THREE.Object3D,
|
|
position: [number, number, number],
|
|
angle: number,
|
|
bend: number,
|
|
length: number,
|
|
radius: number,
|
|
spread = 0,
|
|
) => {
|
|
const root = new THREE.Group();
|
|
root.position.set(...position);
|
|
root.rotation.x = radians(angle);
|
|
root.rotation.z = radians(spread);
|
|
parent.add(root);
|
|
const upper = new THREE.Mesh(
|
|
new THREE.CapsuleGeometry(radius, length - radius * 2, 4, 12),
|
|
mat.clone(),
|
|
);
|
|
upper.position.y = -length / 2;
|
|
root.add(upper);
|
|
const joint = new THREE.Group();
|
|
joint.position.y = -length;
|
|
joint.rotation.x = radians(bend);
|
|
root.add(joint);
|
|
const lower = new THREE.Mesh(
|
|
new THREE.CapsuleGeometry(radius * 0.85, length - radius * 2, 4, 12),
|
|
mat.clone(),
|
|
);
|
|
lower.position.y = -length / 2;
|
|
joint.add(lower);
|
|
};
|
|
for (let i = 0; i < 2; i++) {
|
|
const side = i === 0 ? -1 : 1;
|
|
limb(
|
|
torso,
|
|
[side * 0.33, 0.65, 0],
|
|
pose.arms[i],
|
|
-pose.elbows[i],
|
|
0.34,
|
|
0.07,
|
|
("spread" in pose ? pose.spread : 5) * side,
|
|
);
|
|
limb(hips, [side * 0.15, 0, 0], pose.legs[i], pose.knees[i], 0.46, 0.1);
|
|
}
|
|
object = group;
|
|
} else {
|
|
object = new THREE.Mesh(
|
|
item.kind === "sphere"
|
|
? new THREE.SphereGeometry(0.5, 32, 20)
|
|
: new THREE.BoxGeometry(1, 1, 1),
|
|
mat,
|
|
);
|
|
}
|
|
attach(object, item);
|
|
}
|
|
setLoading(r.pending > 0);
|
|
return () => {
|
|
active = false;
|
|
};
|
|
}, [props.scene.objects, ready]);
|
|
useEffect(() => {
|
|
const r = runtime.current;
|
|
if (!r) return;
|
|
if (props.locked) {
|
|
r.orbit.enabled = false;
|
|
r.transform.detach();
|
|
return;
|
|
}
|
|
const camera =
|
|
props.scene.cameras.find((c) => c.id === props.scene.activeCameraId) ||
|
|
props.scene.cameras[0];
|
|
if (props.previewTime === null) {
|
|
r.camera.position.fromArray(camera.position);
|
|
r.camera.fov = camera.fov;
|
|
r.camera.updateProjectionMatrix();
|
|
r.orbit.target.fromArray(camera.target);
|
|
r.orbit.update();
|
|
}
|
|
r.orbit.enabled = !props.cameraView && props.previewTime === null;
|
|
r.transform.setMode(props.mode);
|
|
const selected = r.objects.getObjectByName(props.selected);
|
|
if (
|
|
selected &&
|
|
selected.visible &&
|
|
!props.scene.objects.find((o) => o.id === props.selected)?.locked &&
|
|
!props.cameraView &&
|
|
props.previewTime === null
|
|
)
|
|
r.transform.attach(selected);
|
|
else r.transform.detach();
|
|
}, [
|
|
props.scene.objects,
|
|
props.scene.cameras,
|
|
props.scene.activeCameraId,
|
|
props.selected,
|
|
props.mode,
|
|
props.cameraView,
|
|
props.previewTime === null,
|
|
props.locked,
|
|
ready,
|
|
]);
|
|
useEffect(() => {
|
|
const r = runtime.current;
|
|
if (!r) return;
|
|
dispose(r.path);
|
|
r.path.clear();
|
|
const motion = props.scene.motion;
|
|
if (motion.keyframes.length < 2) return;
|
|
const points = Array.from(
|
|
{ length: 121 },
|
|
(_, i) =>
|
|
new THREE.Vector3(...sampleDirectorMotion(motion, (motion.duration * i) / 120)!.position),
|
|
);
|
|
r.path.add(
|
|
new THREE.Line(
|
|
new THREE.BufferGeometry().setFromPoints(points),
|
|
new THREE.LineBasicMaterial({ color: 0xfbbf24 }),
|
|
),
|
|
);
|
|
for (const k of motion.keyframes) {
|
|
const dot = new THREE.Mesh(
|
|
new THREE.SphereGeometry(0.08, 10, 8),
|
|
new THREE.MeshBasicMaterial({ color: 0xfbbf24 }),
|
|
);
|
|
dot.position.fromArray(k.position);
|
|
r.path.add(dot);
|
|
}
|
|
}, [props.scene.motion, ready]);
|
|
useEffect(() => {
|
|
const r = runtime.current;
|
|
if (!r) return;
|
|
r.path.visible = !props.locked && !props.cameraView && props.previewTime === null;
|
|
if (props.locked || props.previewTime === null) return;
|
|
const pose = sampleDirectorMotion(props.scene.motion, props.previewTime);
|
|
if (!pose) return;
|
|
r.camera.position.fromArray(pose.position);
|
|
r.orbit.target.fromArray(pose.target);
|
|
r.camera.fov = pose.fov;
|
|
r.camera.updateProjectionMatrix();
|
|
r.camera.lookAt(r.orbit.target);
|
|
r.transform.detach();
|
|
}, [props.previewTime, props.scene.motion, props.cameraView, props.locked, ready]);
|
|
useEffect(() => {
|
|
runtime.current?.resize();
|
|
}, [props.scene.ratio, ready]);
|
|
useImperativeHandle(
|
|
ref,
|
|
() => ({
|
|
showPath: () => {
|
|
const r = runtime.current;
|
|
if (!r) return;
|
|
const points = latest.current.scene.motion.keyframes.map(
|
|
(k) => new THREE.Vector3(...k.position),
|
|
);
|
|
if (!points.length) return;
|
|
points.push(new THREE.Vector3(0, 1, 0));
|
|
const box = new THREE.Box3().setFromPoints(points),
|
|
center = box.getCenter(new THREE.Vector3());
|
|
const radius = Math.max(3, box.getSize(new THREE.Vector3()).length());
|
|
const position = center
|
|
.clone()
|
|
.add(new THREE.Vector3(1, 0.8, 1).normalize().multiplyScalar(radius * 1.4));
|
|
latest.current.onCamera(position.toArray(), center.toArray());
|
|
},
|
|
record: async (signal, onProgress) => {
|
|
const r = runtime.current;
|
|
if (!r || !ready || r.pending || error) throw Error(error || "3D 场景尚未准备好");
|
|
if (latest.current.scene.motion.keyframes.length < 2) throw Error("至少需要两个关键帧");
|
|
const motion = latest.current.scene.motion;
|
|
const size = r.renderer.getSize(new THREE.Vector2()),
|
|
pixelRatio = r.renderer.getPixelRatio();
|
|
const position = r.camera.position.clone(),
|
|
target = r.orbit.target.clone(),
|
|
fov = r.camera.fov,
|
|
aspect = r.camera.aspect;
|
|
const visibility = [r.helper.visible, r.grid.visible, r.path.visible],
|
|
enabled = r.orbit.enabled;
|
|
const dimensions = directorImageSize(latest.current.scene.ratio);
|
|
r.helper.visible = false;
|
|
r.grid.visible = false;
|
|
r.path.visible = false;
|
|
r.orbit.enabled = false;
|
|
r.renderer.setPixelRatio(1);
|
|
r.renderer.setSize(dimensions.width, dimensions.height, false);
|
|
r.camera.aspect = dimensions.width / dimensions.height;
|
|
try {
|
|
return await recordDirectorVideo({
|
|
canvas: r.renderer.domElement,
|
|
duration: motion.duration,
|
|
signal,
|
|
onProgress,
|
|
render: (time) => {
|
|
const pose = sampleDirectorMotion(motion, time)!;
|
|
r.camera.position.fromArray(pose.position);
|
|
r.orbit.target.fromArray(pose.target);
|
|
r.camera.fov = pose.fov;
|
|
r.camera.updateProjectionMatrix();
|
|
r.camera.lookAt(r.orbit.target);
|
|
r.renderer.render(r.world, r.camera);
|
|
},
|
|
});
|
|
} finally {
|
|
if (runtime.current === r) {
|
|
r.renderer.setPixelRatio(pixelRatio);
|
|
r.renderer.setSize(size.x, size.y);
|
|
r.camera.position.copy(position);
|
|
r.orbit.target.copy(target);
|
|
r.camera.fov = fov;
|
|
r.camera.aspect = aspect;
|
|
r.camera.updateProjectionMatrix();
|
|
r.orbit.enabled = enabled;
|
|
r.orbit.update();
|
|
[r.helper.visible, r.grid.visible, r.path.visible] = visibility;
|
|
}
|
|
}
|
|
},
|
|
capture: async (pose) => {
|
|
const r = runtime.current;
|
|
if (!r || !ready || r.pending || error) throw Error(error || "3D 场景尚未准备好");
|
|
const { width, height } = directorImageSize(latest.current.scene.ratio);
|
|
const previousSize = r.renderer.getSize(new THREE.Vector2());
|
|
const pixelRatio = r.renderer.getPixelRatio();
|
|
const aspect = r.camera.aspect;
|
|
const position = r.camera.position.clone(),
|
|
target = r.orbit.target.clone(),
|
|
fov = r.camera.fov;
|
|
const visibility = [r.helper.visible, r.grid.visible, r.path.visible];
|
|
const enabled = r.orbit.enabled;
|
|
r.orbit.enabled = false;
|
|
if (pose) {
|
|
r.camera.position.fromArray(pose.position);
|
|
r.orbit.target.fromArray(pose.target);
|
|
r.camera.fov = pose.fov;
|
|
r.camera.lookAt(r.orbit.target);
|
|
}
|
|
r.path.visible = false;
|
|
r.helper.visible = false;
|
|
r.grid.visible = false;
|
|
r.renderer.setPixelRatio(1);
|
|
r.renderer.setSize(width, height, false);
|
|
r.camera.aspect = width / height;
|
|
r.camera.updateProjectionMatrix();
|
|
r.renderer.render(r.world, r.camera);
|
|
try {
|
|
const blob = await new Promise<Blob>((resolve, reject) =>
|
|
r.renderer.domElement.toBlob(
|
|
(b) => (b ? resolve(b) : reject(Error("截图失败"))),
|
|
"image/png",
|
|
),
|
|
);
|
|
return { blob, width, height };
|
|
} finally {
|
|
r.renderer.setPixelRatio(pixelRatio);
|
|
r.renderer.setSize(previousSize.x, previousSize.y);
|
|
r.camera.aspect = aspect;
|
|
r.camera.updateProjectionMatrix();
|
|
r.camera.position.copy(position);
|
|
r.orbit.target.copy(target);
|
|
r.camera.fov = fov;
|
|
r.camera.updateProjectionMatrix();
|
|
r.orbit.update();
|
|
[r.helper.visible, r.grid.visible, r.path.visible] = visibility;
|
|
r.orbit.enabled = enabled;
|
|
}
|
|
},
|
|
}),
|
|
[ready, error],
|
|
);
|
|
return (
|
|
<div className="relative h-full w-full" data-testid="director-viewport">
|
|
<div ref={host} className="h-full w-full" />
|
|
{error ? (
|
|
<div
|
|
role="alert"
|
|
className="absolute inset-x-4 top-4 rounded bg-red-950/90 p-3 text-sm text-white"
|
|
>
|
|
{error}
|
|
</div>
|
|
) : loading ? (
|
|
<div className="absolute left-4 top-4 rounded bg-black/60 p-2 text-white">
|
|
正在加载模型...
|
|
</div>
|
|
) : null}
|
|
</div>
|
|
);
|
|
},
|
|
);
|