From d5593ac3f0af9a6eb088704f3a492dc620b05da2 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 17 Sep 2026 00:31:36 +0800 Subject: [PATCH] feat(video-canvas): implement camera motion timeline and real MP4 export --- .../canvas/LiblibVideoCanvasReplica.tsx | 37 ++ .../canvas/director/DirectorMotionPanel.tsx | 439 +++++++++++++++++ .../canvas/director/DirectorStudioDialog.tsx | 457 +++++++++++++----- .../canvas/director/DirectorViewport.tsx | 216 ++++++++- ...video-canvas-director-motion-2026-09-17.md | 50 ++ lib/video-canvas/director-motion.ts | 228 +++++++++ lib/video-canvas/director-recording.ts | 95 ++++ lib/video-canvas/director-scene.ts | 12 +- tests/routes/video-canvas-director.test.ts | 218 +++++++++ 9 files changed, 1609 insertions(+), 143 deletions(-) create mode 100644 components/canvas/director/DirectorMotionPanel.tsx create mode 100644 docs/video-canvas-director-motion-2026-09-17.md create mode 100644 lib/video-canvas/director-motion.ts create mode 100644 lib/video-canvas/director-recording.ts diff --git a/components/canvas/LiblibVideoCanvasReplica.tsx b/components/canvas/LiblibVideoCanvasReplica.tsx index 0b76b3e5e..167d8f435 100644 --- a/components/canvas/LiblibVideoCanvasReplica.tsx +++ b/components/canvas/LiblibVideoCanvasReplica.tsx @@ -4,6 +4,7 @@ const DirectorStudioDialog = dynamic( () => import("@/components/canvas/director/DirectorStudioDialog").then(module => module.DirectorStudioDialog), { ssr: false }, ); +import { directorMotionPrompt, resizeDirectorMotion } from "@/lib/video-canvas/director-motion"; import type { DirectorSnapshot } from "@/lib/video-canvas/director-scene"; import { filterCanvasModels, getCanvasVideoModes, getCanvasVideoParameters, type CanvasModelKind } from "@/lib/video-canvas/model-matching"; @@ -11498,6 +11499,42 @@ export function LiblibVideoCanvasReplica({ initialScene={directorConsoleNode.draftValues?.directorScene} onChange={scene => setNodes(current => current.map(node => node.id === directorConsoleNode.id ? {...node, draftValues: {...node.draftValues, directorScene: scene}} : node))} onSend={snapshots => sendDirectorSnapshotsToCanvas(directorConsoleNode, snapshots)} + onSendMotion={(snapshot, motion) => { + const source = createMediaResourceNode({ + mediaType: "image", mediaUrl: snapshot.url, title: "运镜首帧", + width: 350, height: Math.round(350 * snapshot.height / snapshot.width), + x: directorConsoleNode.x + 500, y: directorConsoleNode.y, + }); + const video = createTargetModuleNode("video", source.x + 500, source.y, nodes.length + 2); + const duration = Math.round(motion.duration); + const prompt = directorMotionPrompt(resizeDirectorMotion(motion, duration)); + video.title = "导演台运镜视频"; + video.draftValues = { + ...video.draftValues, directorMotion: motion, + sourceDirectorNodeId: directorConsoleNode.id, prompt, + params: { + mode: "first-frame", prompt, duration, ratio: "adaptive", + settings: { model: "auto", duration: `${duration}s`, resolution: "720P", ratio: "自适应", audio: "关闭" }, + }, + }; + addNodes([source, video], [ + { id: makeId("edge"), source: directorConsoleNode.id, target: source.id }, + { id: makeId("edge"), source: source.id, target: video.id }, + ]); + }} + onSendVideo={video => { + const result = createMediaResourceNode({ + mediaType: "video", mediaUrl: video.url, title: "导演台真实运镜", + width: 420, height: Math.round(420 * video.height / video.width), + x: directorConsoleNode.x + 500, y: directorConsoleNode.y, + }); + result.draftValues = { + ...result.draftValues, generated: true, durationSeconds: video.duration, + outputSizeLabel: `${video.width} × ${video.height}`, + sourceDirectorNodeId: directorConsoleNode.id, + }; + addNodes([result], [{ id: makeId("edge"), source: directorConsoleNode.id, target: result.id }]); + }} onClose={() => setDirectorConsoleNodeId(null)} /> ) : null} diff --git a/components/canvas/director/DirectorMotionPanel.tsx b/components/canvas/director/DirectorMotionPanel.tsx new file mode 100644 index 000000000..1e0676524 --- /dev/null +++ b/components/canvas/director/DirectorMotionPanel.tsx @@ -0,0 +1,439 @@ +"use client"; +import { useEffect, useRef, useState } from "react"; +import { toast } from "sonner"; +import { + presetDirectorMotion, + sampleDirectorMotion, + readDirectorMotion, + resizeDirectorMotion, + validDirectorPose, + type DirectorMotion, + type DirectorPose, +} from "@/lib/video-canvas/director-motion"; +export function DirectorMotionPanel({ + motion, + pose, + disabled, + onChange, + onPose, + onPreview, + onExport, + onSend, + exportProgress, + onCancel, + onOverview, +}: { + motion: DirectorMotion; + pose: DirectorPose; + disabled: boolean; + onChange: (motion: DirectorMotion) => void; + onPose: (pose: DirectorPose) => void; + onPreview: (time: number | null) => void; + onExport: () => void; + onSend: () => void; + exportProgress: number | null; + onCancel: () => void; + onOverview: () => void; +}) { + const [time, setTime] = useState(0), + [playing, setPlaying] = useState(false), + [preview, setPreview] = useState(false); + const timeRef = useRef(time), + latest = useRef({ motion, onPreview }); + latest.current = { motion, onPreview }; + timeRef.current = time; + const input = useRef(null); + const button = + "rounded border border-white/15 px-2 py-1.5 text-xs hover:bg-white/10 disabled:opacity-40"; + const seek = (t: number) => { + const next = Math.max(0, Math.min(motion.duration, t)); + setTime(next); + timeRef.current = next; + setPreview(true); + onPreview(next); + }; + const edit = () => { + if (preview) { + const sampled = sampleDirectorMotion(motion, timeRef.current); + if (sampled) onPose(sampled); + } + setPlaying(false); + setPreview(false); + onPreview(null); + }; + useEffect(() => { + if (!playing) return; + let frame = 0, + previous = 0; + const tick = (now: number) => { + const m = latest.current.motion; + const next = timeRef.current + (previous ? (now - previous) / 1000 : 0); + previous = now; + const t = + next >= m.duration ? (m.loop ? next % m.duration : m.duration) : next; + timeRef.current = t; + setTime(t); + latest.current.onPreview(t); + if (next >= m.duration && !m.loop) { + setPlaying(false); + return; + } + frame = requestAnimationFrame(tick); + }; + const hide = () => { + if (document.hidden) { + setPlaying(false); + toast.info("页面已切换，运镜预览已暂停"); + } + }; + frame = requestAnimationFrame(tick); + document.addEventListener("visibilitychange", hide); + return () => { + cancelAnimationFrame(frame); + document.removeEventListener("visibilitychange", hide); + }; + }, [playing]); + useEffect(() => { + if (disabled) setPlaying(false); + }, [disabled]); + useEffect(() => { + if (timeRef.current > motion.duration) { + setTime(motion.duration); + onPreview(preview ? motion.duration : null); + } + }, [motion.duration, onPreview, preview]); + const add = () => { + if (!validDirectorPose(pose)) { + toast.error("相机不能与注视点重合"); + return; + } + const existing = motion.keyframes.find( + (k) => Math.abs(k.time - time) < 0.01, + ); + if (!existing && motion.keyframes.length >= 100) { + toast.error("最多支持 100 个关键帧"); + return; + } + const key = { + id: existing?.id || crypto.randomUUID(), + time: existing?.time ?? Number(time.toFixed(2)), + position: [...pose.position] as DirectorPose["position"], + target: [...pose.target] as DirectorPose["target"], + fov: pose.fov, + }; + onChange({ + ...motion, + keyframes: [...motion.keyframes.filter((k) => k.id !== key.id), key].sort( + (a, b) => a.time - b.time, + ), + }); + }; + const downloadJson = () => { + const url = URL.createObjectURL( + new Blob([JSON.stringify({ version: 1, motion }, null, 2)], { + type: "application/json", + }), + ); + const a = document.createElement("a"); + a.href = url; + a.download = "director-motion.json"; + a.click(); + setTimeout(() => URL.revokeObjectURL(url), 1000); + }; + return ( +
+
镜头轨迹
+
+
+ + +
+
+ {(["push", "pull", "orbit", "pan"] as const).map((kind, i) => ( + + ))} +
+ +

+ 预设会替换当前轨迹；黄色线条为相机路径。编辑机位后，在指定时间记录关键帧。 +

+ +
+ + + + +
+ + {preview ? ( +

+ 正在预览轨迹；点击“编辑机位”后可调整并记录机位。 +

+ ) : null} +
+ {motion.keyframes.map((k, i) => ( +
+ + { + const n = Number(e.target.value); + if (n < 0 || n > motion.duration || !Number.isFinite(n)) + return; + if ( + motion.keyframes.some( + (other) => + other.id !== k.id && Math.abs(other.time - n) < 0.01, + ) + ) { + toast.error("关键帧时间不能重复"); + return; + } + onChange({ + ...motion, + keyframes: motion.keyframes + .map((other) => + other.id === k.id ? { ...k, time: n } : other, + ) + .sort((a, b) => a.time - b.time), + }); + }} + /> + +
+ ))} +
+
+ + +
+ + +

+ 真实导出严格使用本场景轨迹，不含音频。AI + 生视频使用首帧与运镜描述作参考，不保证精确复现轨迹。 +

+
+ {exportProgress !== null ? ( +
+ 正在导出 {exportProgress.toFixed(1)} / {motion.duration} 秒{" "} + +
+ ) : null} + { + const file = e.target.files?.[0]; + e.target.value = ""; + if (!file) return; + try { + if (file.size > 256 * 1024) throw Error("轨迹文件不能超过 256 KiB"); + const data = JSON.parse(await file.text()); + if (data.version !== 1) throw Error("不支持的轨迹文件版本"); + const m = readDirectorMotion(data.motion); + if ( + m.keyframes.length < 2 || + m.keyframes.length !== data.motion?.keyframes?.length + ) + throw Error("轨迹数据无效或包含重复时间"); + edit(); + onChange(m); + setTime(0); + toast.success("轨迹已导入"); + } catch (e) { + toast.error(e instanceof Error ? e.message : "轨迹导入失败"); + } + }} + /> +
+ ); +} diff --git a/components/canvas/director/DirectorStudioDialog.tsx b/components/canvas/director/DirectorStudioDialog.tsx index 49219146d..95fb22e36 100644 --- a/components/canvas/director/DirectorStudioDialog.tsx +++ b/components/canvas/director/DirectorStudioDialog.tsx @@ -1,5 +1,5 @@ "use client"; -import { useRef, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { Camera, Plus, Upload, X, Trash2 } from "lucide-react"; import { toast } from "sonner"; import { @@ -9,15 +9,25 @@ import { import { readDirectorScene, directorRatios, + directorImageSize, type DirectorScene, type DirectorSnapshot, type DirectorObject, type DirectorVector, } from "@/lib/video-canvas/director-scene"; +import { DirectorMotionPanel } from "./DirectorMotionPanel"; +import { + sampleDirectorMotion, + mergeDirectorCameraPose, + type DirectorMotion, +} from "@/lib/video-canvas/director-motion"; async function upload(file: File) { const form = new FormData(); form.append("file", file); - form.append("purpose", "director-studio"); + form.append( + "purpose", + file.type.startsWith("video/") ? "canvas-video-process" : "director-studio", + ); const response = await fetch( /\.glb$/i.test(file.name) ? "/api/canvas/director/upload-model" @@ -38,11 +48,20 @@ export function DirectorStudioDialog({ onChange, onClose, onSend, + onSendMotion, + onSendVideo, }: { initialScene: unknown; onChange: (scene: DirectorScene) => void; onClose: () => void; onSend: (snapshots: DirectorSnapshot[]) => void; + onSendMotion: (snapshot: DirectorSnapshot, motion: DirectorMotion) => void; + onSendVideo: (video: { + url: string; + width: number; + height: number; + duration: number; + }) => void; }) { const [scene, setScene] = useState(() => readDirectorScene(initialScene)); const current = useRef(scene); @@ -59,6 +78,85 @@ export function DirectorStudioDialog({ const [cameraView, setCameraView] = useState(false); const [busy, setBusy] = useState(false); const viewport = useRef(null); + const [previewTime, setPreviewTime] = useState(null); + const [exportProgress, setExportProgress] = useState(null); + const recording = useRef(null); + useEffect(() => () => recording.current?.abort(), []); + const [exportedVideo, setExportedVideo] = useState<{ + blob: Blob; + url: string; + width: number; + height: number; + duration: number; + } | null>(null); + useEffect( + () => () => { + if (exportedVideo) URL.revokeObjectURL(exportedVideo.url); + }, + [exportedVideo], + ); + const exportMotion = async () => { + if (busy) return; + setBusy(true); + setExportProgress(0); + const controller = new AbortController(); + recording.current = controller; + try { + const motion = current.current.motion; + const dimensions = directorImageSize(current.current.ratio); + const blob = await viewport.current!.record( + controller.signal, + setExportProgress, + ); + setExportedVideo({ + blob, + url: URL.createObjectURL(blob), + ...dimensions, + duration: motion.duration, + }); + toast.success("真实运镜视频已导出，可下载或发送到画布"); + } catch (e) { + toast.error(e instanceof Error ? e.message : "轨迹导出失败"); + } finally { + recording.current = null; + setBusy(false); + setExportProgress(null); + } + }; + const sendMotion = async () => { + if (busy) return; + setBusy(true); + try { + const motion = current.current.motion; + const pose = sampleDirectorMotion(motion, 0); + if (!pose || motion.keyframes.length < 2) + throw Error("至少需要两个关键帧"); + const image = await viewport.current!.capture(pose); + const url = await upload( + new File([image.blob], `director-motion-${Date.now()}.png`, { + type: "image/png", + }), + ); + const snapshot = { + id: crypto.randomUUID(), + title: "运镜首帧", + url, + width: image.width, + height: image.height, + cameraId: current.current.activeCameraId, + }; + commit({ + ...current.current, + snapshots: [...current.current.snapshots, snapshot], + }); + onSendMotion(snapshot, motion); + toast.success("首帧和运镜描述已发送，请在视频节点选择模型"); + } catch (e) { + toast.error(e instanceof Error ? e.message : "发送失败"); + } finally { + setBusy(false); + } + }; const fileInput = useRef(null); const camera = scene.cameras.find((c) => c.id === scene.activeCameraId) || @@ -75,7 +173,9 @@ export function DirectorStudioDialog({ commit({ ...current.current, cameras: current.current.cameras.map((c) => - c.id === current.current.activeCameraId ? { ...c, ...patch } : c, + c.id === current.current.activeCameraId + ? mergeDirectorCameraPose(c, patch) + : c, ), }); const addObject = ( @@ -150,6 +250,7 @@ export function DirectorStudioDialog({ -
-
+ + + {(["position", "target"] as const).map((key, j) => ( + + ))} + {object ? ( +
+
+ {object.name} + +
+ {(["position", "rotation", "scale"] as const).map((key, j) => ( +
+
+ {["位置", "旋转（弧度）", "缩放"][j]} +
+
+ {object[key].map((v, i) => ( + { + const n = Number(e.target.value); + if ( + !Number.isFinite(n) || + (key === "scale" && n <= 0) + ) + return; + const values = [...object[key]] as DirectorVector; + values[i] = n; + patchObject(object.id, { [key]: values }); + }} + /> + ))} +
+
+ ))} +
+ ) : null} + + commit({ ...current.current, motion })} + onPose={patchCamera} + onPreview={setPreviewTime} + onExport={exportMotion} + onSend={sendMotion} + exportProgress={exportProgress} + onCancel={() => recording.current?.abort()} + onOverview={() => { + setCameraView(false); + viewport.current?.showPath(); + }} + /> + {exportedVideo ? ( +
+
) : null}
diff --git a/components/canvas/director/DirectorViewport.tsx b/components/canvas/director/DirectorViewport.tsx index 53778eee5..ec20cdc26 100644 --- a/components/canvas/director/DirectorViewport.tsx +++ b/components/canvas/director/DirectorViewport.tsx @@ -16,14 +16,28 @@ import { type DirectorScene, type DirectorObject, } from "@/lib/video-canvas/director-scene"; +import { + sampleDirectorMotion, + type DirectorPose, +} from "@/lib/video-canvas/director-motion"; +import { recordDirectorVideo } from "@/lib/video-canvas/director-recording"; export type DirectorViewportHandle = { - capture: () => Promise<{ blob: Blob; width: number; height: number }>; + showPath: () => void; + record: ( + signal: AbortSignal, + onProgress: (time: number) => void, + ) => Promise; + capture: ( + pose?: DirectorPose, + ) => Promise<{ blob: Blob; width: number; height: number }>; }; type Props = { scene: DirectorScene; selected: string; mode: "translate" | "rotate" | "scale"; cameraView: boolean; + previewTime: number | null; + locked: boolean; onTransform: (id: string, patch: Partial) => void; onCamera: ( position: [number, number, number], @@ -32,7 +46,7 @@ type Props = { }; function dispose(root: THREE.Object3D) { root.traverse((o) => { - if (o instanceof THREE.Mesh) { + if (o instanceof THREE.Mesh || o instanceof THREE.Line) { o.geometry.dispose(); for (const m of Array.isArray(o.material) ? o.material : [o.material]) { for (const value of Object.values(m)) @@ -55,6 +69,7 @@ export const DirectorViewport = forwardRef( transform: TransformControls; helper: THREE.Object3D; grid: THREE.GridHelper; + path: THREE.Group; objects: THREE.Group; pending: number; resize: () => void; @@ -100,6 +115,8 @@ export const DirectorViewport = forwardRef( world.add(floor); const grid = new THREE.GridHelper(30, 30, 0x596573, 0x929daa); world.add(grid); + const path = new THREE.Group(); + world.add(path); const camera = new THREE.PerspectiveCamera(50, 1, 0.05, 500); camera.position.set(4, 3, 7); const orbit = new OrbitControls(camera, renderer.domElement); @@ -112,7 +129,10 @@ export const DirectorViewport = forwardRef( const objects = new THREE.Group(); world.add(objects); transform.addEventListener("dragging-changed", (e) => { - orbit.enabled = !e.value && !latest.current.cameraView; + orbit.enabled = + !e.value && + !latest.current.cameraView && + latest.current.previewTime === null; }); transform.addEventListener("mouseUp", () => { const o = transform.object; @@ -123,12 +143,14 @@ export const DirectorViewport = forwardRef( scale: o.scale.toArray(), }); }); - orbit.addEventListener("end", () => + orbit.addEventListener("end", () => { + if (latest.current.previewTime !== null || latest.current.cameraView) + return; latest.current.onCamera( camera.position.toArray(), orbit.target.toArray(), - ), - ); + ); + }); const resize = () => { if (!host.current) return; @@ -148,6 +170,7 @@ export const DirectorViewport = forwardRef( transform, helper, grid, + path, objects, pending: 0, resize, @@ -157,7 +180,7 @@ export const DirectorViewport = forwardRef( observer.observe(host.current); resize(); renderer.setAnimationLoop(() => { - orbit.update(); + if (orbit.enabled) orbit.update(); renderer.render(world, camera); }); setReady(true); @@ -193,7 +216,11 @@ export const DirectorViewport = forwardRef( object.rotation.set(...item.rotation); object.scale.fromArray(item.scale); r.objects.add(object); - if (item.id === latest.current.selected && !latest.current.cameraView) + if ( + item.id === latest.current.selected && + !latest.current.cameraView && + latest.current.previewTime === null + ) r.transform.attach(object); }; for (const item of props.scene.objects) { @@ -274,18 +301,26 @@ export const DirectorViewport = forwardRef( useEffect(() => { const r = runtime.current; if (!r) return; + if (props.locked) { + r.orbit.enabled = false; + r.transform.detach(); + return; + } const camera = props.scene.cameras.find((c) => c.id === props.scene.activeCameraId) || props.scene.cameras[0]; - r.camera.position.fromArray(camera.position); - r.camera.fov = camera.fov; - r.camera.updateProjectionMatrix(); - r.orbit.target.fromArray(camera.target); - r.orbit.enabled = !props.cameraView; - r.orbit.update(); + if (props.previewTime === null) { + r.camera.position.fromArray(camera.position); + r.camera.fov = camera.fov; + r.camera.updateProjectionMatrix(); + r.orbit.target.fromArray(camera.target); + r.orbit.update(); + } + r.orbit.enabled = !props.cameraView && props.previewTime === null; r.transform.setMode(props.mode); const selected = r.objects.getObjectByName(props.selected); - if (selected && !props.cameraView) r.transform.attach(selected); + if (selected && !props.cameraView && props.previewTime === null) + r.transform.attach(selected); else r.transform.detach(); }, [ props.scene.cameras, @@ -293,6 +328,59 @@ export const DirectorViewport = forwardRef( props.selected, props.mode, props.cameraView, + props.previewTime === null, + props.locked, + ready, + ]); + useEffect(() => { + const r = runtime.current; + if (!r) return; + dispose(r.path); + r.path.clear(); + const motion = props.scene.motion; + if (motion.keyframes.length < 2) return; + const points = Array.from( + { length: 121 }, + (_, i) => + new THREE.Vector3( + ...sampleDirectorMotion(motion, (motion.duration * i) / 120)! + .position, + ), + ); + r.path.add( + new THREE.Line( + new THREE.BufferGeometry().setFromPoints(points), + new THREE.LineBasicMaterial({ color: 0xfbbf24 }), + ), + ); + for (const k of motion.keyframes) { + const dot = new THREE.Mesh( + new THREE.SphereGeometry(0.08, 10, 8), + new THREE.MeshBasicMaterial({ color: 0xfbbf24 }), + ); + dot.position.fromArray(k.position); + r.path.add(dot); + } + }, [props.scene.motion, ready]); + useEffect(() => { + const r = runtime.current; + if (!r) return; + r.path.visible = + !props.locked && !props.cameraView && props.previewTime === null; + if (props.locked || props.previewTime === null) return; + const pose = sampleDirectorMotion(props.scene.motion, props.previewTime); + if (!pose) return; + r.camera.position.fromArray(pose.position); + r.orbit.target.fromArray(pose.target); + r.camera.fov = pose.fov; + r.camera.updateProjectionMatrix(); + r.camera.lookAt(r.orbit.target); + r.transform.detach(); + }, [ + props.previewTime, + props.scene.motion, + props.cameraView, + props.locked, ready, ]); useEffect(() => { @@ -301,7 +389,81 @@ export const DirectorViewport = forwardRef( useImperativeHandle( ref, () => ({ - capture: async () => { + showPath: () => { + const r = runtime.current; + if (!r) return; + const points = latest.current.scene.motion.keyframes.map( + (k) => new THREE.Vector3(...k.position), + ); + if (!points.length) return; + points.push(new THREE.Vector3(0, 1, 0)); + const box = new THREE.Box3().setFromPoints(points), + center = box.getCenter(new THREE.Vector3()); + const radius = Math.max(3, box.getSize(new THREE.Vector3()).length()); + const position = center + .clone() + .add( + new THREE.Vector3(1, 0.8, 1) + .normalize() + .multiplyScalar(radius * 1.4), + ); + latest.current.onCamera(position.toArray(), center.toArray()); + }, + record: async (signal, onProgress) => { + const r = runtime.current; + if (!r || !ready || r.pending || error) + throw Error(error || "3D 场景尚未准备好"); + if (latest.current.scene.motion.keyframes.length < 2) + throw Error("至少需要两个关键帧"); + const motion = latest.current.scene.motion; + const size = r.renderer.getSize(new THREE.Vector2()), + pixelRatio = r.renderer.getPixelRatio(); + const position = r.camera.position.clone(), + target = r.orbit.target.clone(), + fov = r.camera.fov, + aspect = r.camera.aspect; + const visibility = [r.helper.visible, r.grid.visible, r.path.visible], + enabled = r.orbit.enabled; + const dimensions = directorImageSize(latest.current.scene.ratio); + r.helper.visible = false; + r.grid.visible = false; + r.path.visible = false; + r.orbit.enabled = false; + r.renderer.setPixelRatio(1); + r.renderer.setSize(dimensions.width, dimensions.height, false); + r.camera.aspect = dimensions.width / dimensions.height; + try { + return await recordDirectorVideo({ + canvas: r.renderer.domElement, + duration: motion.duration, + signal, + onProgress, + render: (time) => { + const pose = sampleDirectorMotion(motion, time)!; + r.camera.position.fromArray(pose.position); + r.orbit.target.fromArray(pose.target); + r.camera.fov = pose.fov; + r.camera.updateProjectionMatrix(); + r.camera.lookAt(r.orbit.target); + r.renderer.render(r.world, r.camera); + }, + }); + } finally { + if (runtime.current === r) { + r.renderer.setPixelRatio(pixelRatio); + r.renderer.setSize(size.x, size.y); + r.camera.position.copy(position); + r.orbit.target.copy(target); + r.camera.fov = fov; + r.camera.aspect = aspect; + r.camera.updateProjectionMatrix(); + r.orbit.enabled = enabled; + r.orbit.update(); + [r.helper.visible, r.grid.visible, r.path.visible] = visibility; + } + } + }, + capture: async (pose) => { const r = runtime.current; if (!r || !ready || r.pending || error) throw Error(error || "3D 场景尚未准备好"); @@ -311,6 +473,19 @@ export const DirectorViewport = forwardRef( const previousSize = r.renderer.getSize(new THREE.Vector2()); const pixelRatio = r.renderer.getPixelRatio(); const aspect = r.camera.aspect; + const position = r.camera.position.clone(), + target = r.orbit.target.clone(), + fov = r.camera.fov; + const visibility = [r.helper.visible, r.grid.visible, r.path.visible]; + const enabled = r.orbit.enabled; + r.orbit.enabled = false; + if (pose) { + r.camera.position.fromArray(pose.position); + r.orbit.target.fromArray(pose.target); + r.camera.fov = pose.fov; + r.camera.lookAt(r.orbit.target); + } + r.path.visible = false; r.helper.visible = false; r.grid.visible = false; r.renderer.setPixelRatio(1); @@ -331,8 +506,13 @@ export const DirectorViewport = forwardRef( r.renderer.setSize(previousSize.x, previousSize.y); r.camera.aspect = aspect; r.camera.updateProjectionMatrix(); - r.grid.visible = true; - r.helper.visible = true; + r.camera.position.copy(position); + r.orbit.target.copy(target); + r.camera.fov = fov; + r.camera.updateProjectionMatrix(); + r.orbit.update(); + [r.helper.visible, r.grid.visible, r.path.visible] = visibility; + r.orbit.enabled = enabled; } }, }), diff --git a/docs/video-canvas-director-motion-2026-09-17.md b/docs/video-canvas-director-motion-2026-09-17.md new file mode 100644 index 000000000..093fe4c72 --- /dev/null +++ b/docs/video-canvas-director-motion-2026-09-17.md @@ -0,0 +1,50 @@ +# 3D 导演台镜头轨迹修复与验收（2026-09-17） + +此前导演台只保存静态机位，运镜素材库只向提示词追加文字。静态机位截图生成视频的验收不代表连续镜头轨迹可用。本次补齐时间轴、真实 3D 播放和视频导出，未部署生产环境。 + +## 完成的功能 + +- 场景增加 `motion` 数据：2–15 秒、最多 100 个关键帧、相机位置、注视点、FOV、线性/缓入缓出插值和循环设置。兼容没有轨迹的旧场景；轨迹随原有画布保存链路持久化，不需要数据库迁移。 +- 支持记录、覆盖、编辑、删除关键帧及修改时间；重复时间和无效数据拦截。调整时长按比例缩放关键帧时间。 +- 推进、拉远、90° 环绕、横移预设；黄色轨迹与关键帧标记、完整轨迹总览。 +- 播放、暂停、循环、时间轴定位；预览时关闭手动相机操控。进入编辑时从当前预览机位继续，避免跳回旧机位。页面进入后台时暂停预览。 +- 导入/导出带版本号的 JSON 轨迹文件；文件限制 256 KiB，坏文件不会覆盖现有轨迹。 +- 按真实相机轨迹录制无声 MP4，隐藏网格、辅助轴和路径，支持取消；浏览器不支持 MP4 编码时明确提示更新浏览器。切到后台、关闭导演台、渲染错误或超时会停止录制并释放媒体轨道，不把部分文件当成功结果。 +- 导出结果可预览、下载、上传为真实视频节点；保存实际时长，供剪辑使用。视频上传使用现有 `canvas-video-process` 用途参数。 +- “发送首帧与运镜到视频节点”真实渲染 0 秒机位，建立导演台→首帧→视频节点连线，携带完整轨迹元数据及精简的自然语言运镜描述。模型、尺寸等继续由视频节点选择；不自动发起计费生成。 + +## 实测中修复的回归 + +- 使用整个关键帧覆盖机位会把关键帧 ID 写成相机 ID，导致后续 FOV 编辑失效。现在仅合并 position、target、fov，保留相机身份，并有专门回归测试。 +- WebM 原始录制缺失 duration，影响剪辑；改为 MP4。实际产物为 H.264，包含可读取的时长。 +- 预览/录制时 OrbitControls 仍可能约束相机距离；改为显式设置相机朝向，录制期间锁定交互，避免改变轨迹。 +- 从预览切换到录制时，React 更新可能重新显示轨迹辅助线；加入录制锁，确保辅助线不进入导出画面。 + +## 验收证据 + +浏览器使用生产组件进行本地独立验收；测试页仅将画布保存与对象存储替换成本地适配器，不伪造 WebGL、时间轴或编码结果。本轮没有调用收费 AI 模型，也未重新验证生产数据库/对象存储的整条链路；原画布持久化和真实万相链路见 2026-09-16 验收记录。 + +| 检查 | 结果 | +|---|---| +| 环绕、推进与时间轴 | 浏览器真实播放，循环后仍持续播放；暂停及定位到终点有效 | +| 轨迹总览 | 相机切换到轨迹包围盒总览视角 | +| 自定义 JSON 导入 | 3 秒、2 个关键帧，位置与 FOV 参数生效 | +| 坏文件导入 | 显示无效数据提示，保留原轨迹 | +| 暂停时选择场景对象 | 预览不跳回编辑机位；场景区域像素对比无变化 | +| 关键帧更新 | 第二帧 FOV 改为 65° 并记录，完整刷新后恢复 | +| 首帧传递 | 从 0 秒机位生成 1280×720 PNG，回调收到首帧及 2 帧轨迹 | +| 横屏 MP4 | 1280×720，H.264，150 帧，4.998 秒，无音轨 | +| 播放中切换导出 | 1280×720，90 帧，3.0005 秒；从轨迹起点完整导出，画面无辅助线 | +| 竖屏 MP4 | 720×1280，H.264，90 帧，3.002 秒，无音轨 | +| 逐帧检查 | 对比初始、中间、结尾帧，视角连续变化，非静态图片录制 | +| 取消导出 | 浏览器中取消后恢复按钮与编辑状态，未出现成功产物 | +| 录制生命周期 | 自动化验证正常结束、取消、后台事件及渲染失败均释放轨道与帧回调 | +| 代码验证 | TypeScript 源码检查通过；49 条定向回归测试通过 | + +浏览器自动化的多标签切换未可靠触发页面可见性事件，因此后台处理依靠事件级回归验证，不将该次标签切换算作真实后台场景通过。下载链接已接入原生下载；本轮媒体文件通过组件上传回调留档，并用 FFprobe 核验。 + +本地 `.qa/director-motion/` 留存测试 MP4、逐帧截图与探测结果，不提交生成文件和本地测试适配器。 + +## AI 生成与真实轨迹的边界 + +真实 3D 导出使用配置的轨迹。当前阿里云首帧模型接入接收首帧与运镜文字，不接收可精确执行的相机轨迹参数，因此 AI 生成仍可能偏离预设，界面已明确说明。需要精确复现时使用“导出真实运镜视频”。AI 识图重建仍是独立的未接通能力，不属于本次运镜修复。 diff --git a/lib/video-canvas/director-motion.ts b/lib/video-canvas/director-motion.ts new file mode 100644 index 000000000..af6e8496b --- /dev/null +++ b/lib/video-canvas/director-motion.ts @@ -0,0 +1,228 @@ +import type { DirectorCamera, DirectorVector } from "./director-scene"; +export type DirectorPose = Pick; +export function mergeDirectorCameraPose( + camera: DirectorCamera, + pose: Partial, +): DirectorCamera { + return { + ...camera, + position: pose.position ?? camera.position, + target: pose.target ?? camera.target, + fov: pose.fov ?? camera.fov, + }; +} +export type DirectorKeyframe = DirectorPose & { id: string; time: number }; +export type DirectorMotion = { + duration: number; + easing: "linear" | "smooth"; + loop: boolean; + keyframes: DirectorKeyframe[]; +}; +export const defaultDirectorMotion = (): DirectorMotion => ({ + duration: 5, + easing: "smooth", + loop: false, + keyframes: [], +}); +const vector = (v: unknown): v is DirectorVector => + Array.isArray(v) && + v.length === 3 && + v.every( + (n) => typeof n === "number" && Number.isFinite(n) && Math.abs(n) <= 10000, + ); +export function validDirectorPose(p: DirectorPose) { + return ( + !!p && + vector(p.position) && + vector(p.target) && + Number.isFinite(p.fov) && + p.fov >= 15 && + p.fov <= 90 && + Math.hypot(...p.position.map((n, i) => n - p.target[i])) > 0.001 + ); +} +export function readDirectorMotion(input: unknown): DirectorMotion { + const v = (input || {}) as Partial; + const duration = Number.isFinite(v.duration) + ? Math.max(2, Math.min(15, Number(v.duration))) + : 5; + const seen = new Set(); + const times = new Set(); + const keyframes = (Array.isArray(v.keyframes) ? v.keyframes : []) + .filter((k) => { + if ( + !k || + typeof k.id !== "string" || + seen.has(k.id) || + !Number.isFinite(k.time) || + k.time < 0 || + k.time > duration || + times.has(k.time) || + !validDirectorPose(k) + ) + return false; + seen.add(k.id); + times.add(k.time); + return true; + }) + .slice(0, 100) + .map((k) => ({ + id: k.id, + time: k.time, + position: [...k.position] as DirectorVector, + target: [...k.target] as DirectorVector, + fov: k.fov, + })) + .sort((a, b) => a.time - b.time); + return { + duration, + easing: v.easing === "linear" ? "linear" : "smooth", + loop: v.loop === true, + keyframes, + }; +} +const lerpVector = ( + a: DirectorVector, + b: DirectorVector, + t: number, +): DirectorVector => a.map((n, i) => n + (b[i] - n) * t) as DirectorVector; +export function sampleDirectorMotion( + m: DirectorMotion, + time: number, +): DirectorPose | null { + const keys = m.keyframes; + if (!keys.length) return null; + const t = Number.isFinite(time) ? Math.max(0, Math.min(m.duration, time)) : 0; + const right = keys.findIndex((k) => k.time > t); + const a = right < 0 ? keys[keys.length - 1] : keys[Math.max(0, right - 1)]; + const b = right < 0 ? a : keys[right]; + let u = + a === b ? 0 : Math.max(0, Math.min(1, (t - a.time) / (b.time - a.time))); + if (m.easing === "smooth") u = u * u * (3 - 2 * u); + const pose = { + position: lerpVector(a.position, b.position, u), + target: lerpVector(a.target, b.target, u), + fov: a.fov + (b.fov - a.fov) * u, + }; + // A path crossing its look-at target has no defined viewing direction. + if (!validDirectorPose(pose)) { + const direction = a.position.map((n, i) => n - a.target[i]); + const length = Math.hypot(...direction); + pose.target = pose.position.map( + (n, i) => n - (direction[i] / length) * 0.01, + ) as DirectorVector; + } + return pose; +} +export function resizeDirectorMotion( + m: DirectorMotion, + duration: number, +): DirectorMotion { + const next = Math.max(2, Math.min(15, duration)); + return { + ...m, + duration: next, + keyframes: m.keyframes.map((k) => ({ + ...k, + time: (k.time / m.duration) * next, + })), + }; +} +export function presetDirectorMotion( + pose: DirectorPose, + kind: "push" | "pull" | "orbit" | "pan", + duration = 5, +): DirectorMotion { + const m = { ...defaultDirectorMotion(), duration }; + const offset = pose.position.map( + (n, i) => n - pose.target[i], + ) as DirectorVector; + const count = kind === "orbit" ? 9 : 2; + m.keyframes = Array.from({ length: count }, (_, i) => { + const u = i / (count - 1); + let position: DirectorVector = [...pose.position], + target: DirectorVector = [...pose.target]; + if (kind === "push" || kind === "pull") + position = pose.target.map( + (n, j) => n + offset[j] * (1 + u * (kind === "push" ? -0.5 : 0.5)), + ) as DirectorVector; + if (kind === "orbit") { + const angle = (u * Math.PI) / 2; + position = [ + pose.target[0] + + offset[0] * Math.cos(angle) + + offset[2] * Math.sin(angle), + pose.position[1], + pose.target[2] - + offset[0] * Math.sin(angle) + + offset[2] * Math.cos(angle), + ]; + } + if (kind === "pan") { + const length = Math.hypot(offset[0], offset[2]) || 1; + const shift: DirectorVector = [ + (offset[2] / length) * 2 * u, + 0, + (-offset[0] / length) * 2 * u, + ]; + position = position.map((n, j) => n + shift[j]) as DirectorVector; + target = target.map((n, j) => n + shift[j]) as DirectorVector; + } + return { + id: `${kind}-${i}`, + time: u * duration, + position, + target, + fov: pose.fov, + }; + }); + if (kind === "orbit") m.easing = "linear"; + return m; +} +export function directorMotionPrompt(m: DirectorMotion) { + if (m.keyframes.length < 2) return ""; + // Keep the model prompt compact; the full trajectory remains in the canvas data. + const keys = + m.keyframes.length <= 9 + ? m.keyframes + : Array.from({ length: 9 }, (_, i) => ({ + time: (m.duration * i) / 8, + ...sampleDirectorMotion(m, (m.duration * i) / 8)!, + })); + const lines = keys.slice(1).map((b, i) => { + const a = keys[i], + av = a.position.map((n, j) => n - a.target[j]), + bv = b.position.map((n, j) => n - b.target[j]); + const ar = Math.hypot(...av), + br = Math.hypot(...bv), + angle = + (Math.acos( + Math.max( + -1, + Math.min( + 1, + av.reduce((sum, n, j) => sum + n * bv[j], 0) / (ar * br), + ), + ), + ) * + 180) / + Math.PI; + const descriptions: string[] = []; + if (angle > 2) + descriptions.push(`围绕注视主体转动约 ${Math.round(angle)} 度`); + if (br < ar * 0.98) descriptions.push("向主体推进"); + else if (br > ar * 1.02) descriptions.push("远离主体拉远"); + if (Math.hypot(...b.target.map((n, j) => n - a.target[j])) > 0.05) + descriptions.push("注视点随镜头平移"); + if (Math.abs(b.position[1] - a.position[1]) > 0.05) + descriptions.push( + b.position[1] > a.position[1] ? "相机升高" : "相机降低", + ); + if (Math.abs(b.fov - a.fov) > 0.5) + descriptions.push( + `视角从 ${a.fov.toFixed(0)}° 变至 ${b.fov.toFixed(0)}°`, + ); + return `${a.time.toFixed(2)}–${b.time.toFixed(2)}秒：${descriptions.join("，") || "保持机位和构图"}。`; + }); + return `单一连续镜头，不切镜，总时长 ${m.duration} 秒；镜头${m.easing === "smooth" ? "平滑缓入缓出" : "匀速"}运动。\n${lines.join("\n")}\n保持参考图中的主体和场景一致，镜头连续运动。`; +} diff --git a/lib/video-canvas/director-recording.ts b/lib/video-canvas/director-recording.ts new file mode 100644 index 000000000..9cef6e707 --- /dev/null +++ b/lib/video-canvas/director-recording.ts @@ -0,0 +1,95 @@ +/** Record the rendered camera path, never an AI approximation. */ +export async function recordDirectorVideo({ + canvas, + duration, + render, + signal, + onProgress, +}: { + canvas: HTMLCanvasElement; + duration: number; + render: (time: number) => void; + signal: AbortSignal; + onProgress?: (time: number) => void; +}): Promise { + const mimeType = ["video/mp4"].find( + (t) => + typeof MediaRecorder !== "undefined" && MediaRecorder.isTypeSupported(t), + ); + if (!mimeType || !canvas.captureStream) + throw Error("当前浏览器不支持 MP4 轨迹导出，请使用最新版 Chrome 或 Edge"); + if (signal.aborted || document.hidden) + throw Error("请保持当前页面可见后导出"); + const stream = canvas.captureStream(30); + try { + return await new Promise((resolve, reject) => { + const recorder = new MediaRecorder(stream, { + mimeType, + videoBitsPerSecond: 6_000_000, + }); + const chunks: Blob[] = []; + let frame = 0, + started = 0, + failure: Error | undefined, + finished = false; + const cleanup = () => { + cancelAnimationFrame(frame); + clearTimeout(timeout); + signal.removeEventListener("abort", cancel); + document.removeEventListener("visibilitychange", visibility); + }; + const stop = (error?: Error) => { + if (finished) return; + finished = true; + failure = error; + cleanup(); + if (recorder.state !== "inactive") recorder.stop(); + else reject(error || Error("视频录制未启动")); + }; + const cancel = () => stop(Error("已取消轨迹视频导出")); + const visibility = () => { + if (document.hidden) + stop(Error("导出时切换了页面，请回到当前页面重新导出")); + }; + const timeout = setTimeout( + () => stop(Error("轨迹导出超时，请重试")), + duration * 1000 + 15000, + ); + recorder.ondataavailable = (e) => { + if (e.data.size) chunks.push(e.data); + }; + recorder.onerror = () => stop(Error("浏览器视频编码失败")); + recorder.onstop = () => { + cleanup(); + if (failure) reject(failure); + else if (!chunks.length) reject(Error("未生成有效视频")); + else resolve(new Blob(chunks, { type: recorder.mimeType })); + }; + signal.addEventListener("abort", cancel, { once: true }); + document.addEventListener("visibilitychange", visibility); + const tick = (now: number) => { + if (finished) return; + if (!started) started = now; + const t = Math.min(duration, (now - started) / 1000); + try { + render(t); + onProgress?.(t); + } catch (e) { + stop(e instanceof Error ? e : Error("轨迹渲染失败")); + return; + } + if (t >= duration) stop(); + else frame = requestAnimationFrame(tick); + }; + try { + render(0); + recorder.start(100); + frame = requestAnimationFrame(tick); + } catch (e) { + stop(e instanceof Error ? e : Error("视频编码失败")); + } + }); + } finally { + stream.getTracks().forEach((t) => t.stop()); + } +} diff --git a/lib/video-canvas/director-scene.ts b/lib/video-canvas/director-scene.ts index 64b9d2720..39b8c679d 100644 --- a/lib/video-canvas/director-scene.ts +++ b/lib/video-canvas/director-scene.ts @@ -1,3 +1,9 @@ +import { + defaultDirectorMotion, + readDirectorMotion, + validDirectorPose, + type DirectorMotion, +} from "./director-motion"; export type DirectorVector = [number, number, number]; export type DirectorObject = { id: string; @@ -29,6 +35,7 @@ export type DirectorScene = { activeCameraId: string; ratio: string; snapshots: DirectorSnapshot[]; + motion: DirectorMotion; }; export const directorRatios = ["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"]; export function directorImageSize(ratio: string) { @@ -63,6 +70,7 @@ export function defaultDirectorScene(): DirectorScene { activeCameraId: "camera-1", ratio: "16:9", snapshots: [], + motion: defaultDirectorMotion(), }; } export function readDirectorScene(value: unknown): DirectorScene { @@ -83,7 +91,8 @@ export function readDirectorScene(value: unknown): DirectorScene { vector(c.target) && Number.isFinite(c.fov) && c.fov >= 15 && - c.fov <= 90, + c.fov <= 90 && + validDirectorPose(c), ) : []; const objects = Array.isArray(v.objects) @@ -120,5 +129,6 @@ export function readDirectorScene(value: unknown): DirectorScene { : validCameras[0].id, ratio: directorRatios.includes(v.ratio) ? v.ratio : "16:9", snapshots, + motion: readDirectorMotion(v.motion), }; } diff --git a/tests/routes/video-canvas-director.test.ts b/tests/routes/video-canvas-director.test.ts index 48b91778d..5e82f07fa 100644 --- a/tests/routes/video-canvas-director.test.ts +++ b/tests/routes/video-canvas-director.test.ts @@ -104,3 +104,221 @@ test("models stored on the active media CDN use the authenticated same-origin pr "/api/canvas/3d/model-proxy?url=" + encodeURIComponent(url), ); }); + +import { + defaultDirectorMotion, + readDirectorMotion, + sampleDirectorMotion, + resizeDirectorMotion, + presetDirectorMotion, + directorMotionPrompt, + validDirectorPose, +} from "@/lib/video-canvas/director-motion"; +const camera = { position: [0, 2, 8], target: [0, 1, 0], fov: 50 } as const; +const pose = () => + ({ + position: [...camera.position], + target: [...camera.target], + fov: 50, + }) as import("@/lib/video-canvas/director-motion").DirectorPose; +test("legacy scenes acquire an empty timeline without altering cameras", () => { + const { motion, ...legacy } = defaultDirectorScene(); + expect(readDirectorScene(legacy).motion).toEqual(defaultDirectorMotion()); + expect(readDirectorScene(legacy).cameras).toEqual(legacy.cameras); +}); +test("motion survives serialization and duration changes preserve relative keyframe timing", () => { + const scene = defaultDirectorScene(); + scene.motion = presetDirectorMotion(pose(), "orbit", 5); + scene.motion.loop = true; + expect(readDirectorScene(JSON.parse(JSON.stringify(scene)))).toEqual(scene); + const resized = resizeDirectorMotion(scene.motion, 10); + expect(resized.keyframes[4].time).toBe(5); + expect(resized.keyframes.at(-1)?.time).toBe(10); + expect(resized.keyframes[4].position).toEqual( + scene.motion.keyframes[4].position, + ); +}); +test("timeline sampling clamps endpoints, holds outside keyframes and differentiates easing", () => { + const motion = presetDirectorMotion(pose(), "push", 5); + motion.easing = "linear"; + expect(sampleDirectorMotion(motion, -20)?.position).toEqual( + motion.keyframes[0].position, + ); + expect(sampleDirectorMotion(motion, 30)?.position).toEqual( + motion.keyframes[1].position, + ); + expect(sampleDirectorMotion(motion, 2.5)?.position[2]).toBe(6); + const linear = sampleDirectorMotion(motion, 1)?.position[2]!; + motion.easing = "smooth"; + expect(sampleDirectorMotion(motion, 1)?.position[2]).toBeGreaterThan(linear); + motion.keyframes[0].time = 1; + motion.keyframes[1].time = 4; + expect(sampleDirectorMotion(motion, 0)?.position).toEqual( + motion.keyframes[0].position, + ); + expect(sampleDirectorMotion(motion, 5)?.position).toEqual( + motion.keyframes[1].position, + ); + expect(sampleDirectorMotion(defaultDirectorMotion(), 0)).toBeNull(); +}); +test("invalid, repeated, out of range or coincident keyframes are rejected", () => { + const m = presetDirectorMotion(pose(), "pull"); + const first = m.keyframes[0]; + m.keyframes.push( + { ...first, id: "duplicate-time" }, + { ...first, id: "nan", time: NaN }, + { ...first, id: "out", time: 16 }, + { ...first, id: "broken", time: 2, position: [Infinity, 0, 1] }, + { ...first, id: "coincident", time: 3, position: [...first.target] }, + ); + const read = readDirectorMotion(m); + expect(read.keyframes).toHaveLength(2); + expect(read.keyframes[0].id).toBe(first.id); +}); +test("all motion presets remain finite and differ from their initial camera", () => { + for (const kind of ["push", "pull", "orbit", "pan"] as const) { + const m = presetDirectorMotion(pose(), kind, 5); + for (let t = 0; t <= 5; t += 0.025) + expect(validDirectorPose(sampleDirectorMotion(m, t)!)).toBe(true); + expect(sampleDirectorMotion(m, 5)?.position).not.toEqual( + sampleDirectorMotion(m, 0)?.position, + ); + expect(directorMotionPrompt(m)).toContain("5 秒"); + expect(directorMotionPrompt(m)).toContain("5.00秒"); + } +}); +test("look-at remains defined when opposing keyframes cross their target", () => { + const m = presetDirectorMotion(pose(), "push"); + m.easing = "linear"; + m.keyframes[0].position = [-1, 0, 0]; + m.keyframes[0].target = [0, 0, 0]; + m.keyframes[1].position = [1, 0, 0]; + m.keyframes[1].target = [0, 0, 0]; + expect(validDirectorPose(sampleDirectorMotion(m, 2.5)!)).toBe(true); +}); + +import { recordDirectorVideo } from "@/lib/video-canvas/director-recording"; +test("recording stops tracks and rejects partial exports on cancel or background switch", async () => { + for (const outcome of [ + "complete", + "cancel", + "hidden", + "render-error", + ] as const) { + const saved = new Map(); + const install = (key: string, value: unknown) => { + saved.set(key, Object.getOwnPropertyDescriptor(globalThis, key)); + Object.defineProperty(globalThis, key, { + configurable: true, + writable: true, + value, + }); + }; + const doc = new EventTarget() as EventTarget & { hidden: boolean }; + doc.hidden = false; + const callbacks = new Map(); + let id = 0, + stopped = 0; + class Recorder { + static isTypeSupported(type: string) { + return type === "video/mp4"; + } + state = "inactive"; + mimeType = "video/mp4"; + onstop?: () => void; + ondataavailable?: (e: { data: Blob }) => void; + start() { + this.state = "recording"; + } + stop() { + this.state = "inactive"; + this.ondataavailable?.({ data: new Blob(["recorded"]) }); + queueMicrotask(() => this.onstop?.()); + } + } + install("document", doc); + install("MediaRecorder", Recorder); + install("requestAnimationFrame", (fn: FrameRequestCallback) => { + callbacks.set(++id, fn); + return id; + }); + install("cancelAnimationFrame", (n: number) => callbacks.delete(n)); + try { + const controller = new AbortController(); + const rendered: number[] = []; + const canvas = { + captureStream: () => ({ getTracks: () => [{ stop: () => stopped++ }] }), + } as unknown as HTMLCanvasElement; + const result = recordDirectorVideo({ + canvas, + duration: 2, + signal: controller.signal, + render: (t) => { + if (outcome === "render-error" && t > 0) throw Error("render fault"); + rendered.push(t); + }, + }); + const step = (time: number) => { + const pending = [...callbacks.values()]; + callbacks.clear(); + pending.forEach((fn) => fn(time)); + }; + if (outcome === "complete") { + step(1000); + step(3000); + expect((await result).type).toBe("video/mp4"); + expect(rendered.at(-1)).toBe(2); + } else { + const rejected = expect(result).rejects.toThrow(); + if (outcome === "cancel") controller.abort(); + else if (outcome === "hidden") { + doc.hidden = true; + doc.dispatchEvent(new Event("visibilitychange")); + } else { + step(1000); + step(3000); + } + await rejected; + } + expect(stopped).toBe(1); + expect(callbacks.size).toBe(0); + } finally { + for (const [key, descriptor] of saved) { + if (descriptor) Object.defineProperty(globalThis, key, descriptor); + else Reflect.deleteProperty(globalThis, key); + } + } + } +}); +test("AI motion prompt describes the movement and stays bounded for large timelines", () => { + expect(directorMotionPrompt(presetDirectorMotion(pose(), "push"))).toContain( + "推进", + ); + expect(directorMotionPrompt(presetDirectorMotion(pose(), "pull"))).toContain( + "拉远", + ); + expect(directorMotionPrompt(presetDirectorMotion(pose(), "orbit"))).toContain( + "围绕", + ); + expect(directorMotionPrompt(presetDirectorMotion(pose(), "pan"))).toContain( + "平移", + ); + const m = presetDirectorMotion(pose(), "orbit", 15); + m.keyframes = Array.from({ length: 100 }, (_, i) => ({ + id: String(i), + time: (i / 99) * 15, + ...sampleDirectorMotion(m, (i / 99) * 15)!, + })); + expect(directorMotionPrompt(m).length).toBeLessThan(1500); +}); + +import { mergeDirectorCameraPose } from "@/lib/video-canvas/director-motion"; +test("editing a keyframe cannot replace the active camera identity", () => { + const c = defaultDirectorScene().cameras[0]; + const key = presetDirectorMotion(pose(), "push").keyframes[1]; + const edited = mergeDirectorCameraPose(c, key); + expect(edited.id).toBe(c.id); + expect(edited.name).toBe(c.name); + expect(edited.position).toEqual(key.position); + expect(mergeDirectorCameraPose(edited, { fov: 65 }).fov).toBe(65); +});