feat(video-canvas): implement camera motion timeline and real MP4 export
This commit is contained in:
134
parent
38be09de9c
commit
d5593ac3f0
@ -4,6 +4,7 @@ const DirectorStudioDialog = dynamic(
|
||||
() => import("@/components/canvas/director/DirectorStudioDialog").then(module => module.DirectorStudioDialog),
|
||||
{ ssr: false },
|
||||
);
|
||||
import { directorMotionPrompt, resizeDirectorMotion } from "@/lib/video-canvas/director-motion";
|
||||
import type { DirectorSnapshot } from "@/lib/video-canvas/director-scene";
|
||||
import { filterCanvasModels, getCanvasVideoModes, getCanvasVideoParameters, type CanvasModelKind } from "@/lib/video-canvas/model-matching";
|
||||
|
||||
@ -11498,6 +11499,42 @@ export function LiblibVideoCanvasReplica({
|
||||
initialScene={directorConsoleNode.draftValues?.directorScene}
|
||||
onChange={scene => setNodes(current => current.map(node => node.id === directorConsoleNode.id ? {...node, draftValues: {...node.draftValues, directorScene: scene}} : node))}
|
||||
onSend={snapshots => sendDirectorSnapshotsToCanvas(directorConsoleNode, snapshots)}
|
||||
onSendMotion={(snapshot, motion) => {
|
||||
const source = createMediaResourceNode({
|
||||
mediaType: "image", mediaUrl: snapshot.url, title: "运镜首帧",
|
||||
width: 350, height: Math.round(350 * snapshot.height / snapshot.width),
|
||||
x: directorConsoleNode.x + 500, y: directorConsoleNode.y,
|
||||
});
|
||||
const video = createTargetModuleNode("video", source.x + 500, source.y, nodes.length + 2);
|
||||
const duration = Math.round(motion.duration);
|
||||
const prompt = directorMotionPrompt(resizeDirectorMotion(motion, duration));
|
||||
video.title = "导演台运镜视频";
|
||||
video.draftValues = {
|
||||
...video.draftValues, directorMotion: motion,
|
||||
sourceDirectorNodeId: directorConsoleNode.id, prompt,
|
||||
params: {
|
||||
mode: "first-frame", prompt, duration, ratio: "adaptive",
|
||||
settings: { model: "auto", duration: `${duration}s`, resolution: "720P", ratio: "自适应", audio: "关闭" },
|
||||
},
|
||||
};
|
||||
addNodes([source, video], [
|
||||
{ id: makeId("edge"), source: directorConsoleNode.id, target: source.id },
|
||||
{ id: makeId("edge"), source: source.id, target: video.id },
|
||||
]);
|
||||
}}
|
||||
onSendVideo={video => {
|
||||
const result = createMediaResourceNode({
|
||||
mediaType: "video", mediaUrl: video.url, title: "导演台真实运镜",
|
||||
width: 420, height: Math.round(420 * video.height / video.width),
|
||||
x: directorConsoleNode.x + 500, y: directorConsoleNode.y,
|
||||
});
|
||||
result.draftValues = {
|
||||
...result.draftValues, generated: true, durationSeconds: video.duration,
|
||||
outputSizeLabel: `${video.width} × ${video.height}`,
|
||||
sourceDirectorNodeId: directorConsoleNode.id,
|
||||
};
|
||||
addNodes([result], [{ id: makeId("edge"), source: directorConsoleNode.id, target: result.id }]);
|
||||
}}
|
||||
onClose={() => setDirectorConsoleNodeId(null)}
|
||||
/>
|
||||
) : null}
|
||||
|
||||
439
components/canvas/director/DirectorMotionPanel.tsx
Normal file
439
components/canvas/director/DirectorMotionPanel.tsx
Normal file
@ -0,0 +1,439 @@
|
||||
"use client";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
presetDirectorMotion,
|
||||
sampleDirectorMotion,
|
||||
readDirectorMotion,
|
||||
resizeDirectorMotion,
|
||||
validDirectorPose,
|
||||
type DirectorMotion,
|
||||
type DirectorPose,
|
||||
} from "@/lib/video-canvas/director-motion";
|
||||
export function DirectorMotionPanel({
|
||||
motion,
|
||||
pose,
|
||||
disabled,
|
||||
onChange,
|
||||
onPose,
|
||||
onPreview,
|
||||
onExport,
|
||||
onSend,
|
||||
exportProgress,
|
||||
onCancel,
|
||||
onOverview,
|
||||
}: {
|
||||
motion: DirectorMotion;
|
||||
pose: DirectorPose;
|
||||
disabled: boolean;
|
||||
onChange: (motion: DirectorMotion) => void;
|
||||
onPose: (pose: DirectorPose) => void;
|
||||
onPreview: (time: number | null) => void;
|
||||
onExport: () => void;
|
||||
onSend: () => void;
|
||||
exportProgress: number | null;
|
||||
onCancel: () => void;
|
||||
onOverview: () => void;
|
||||
}) {
|
||||
const [time, setTime] = useState(0),
|
||||
[playing, setPlaying] = useState(false),
|
||||
[preview, setPreview] = useState(false);
|
||||
const timeRef = useRef(time),
|
||||
latest = useRef({ motion, onPreview });
|
||||
latest.current = { motion, onPreview };
|
||||
timeRef.current = time;
|
||||
const input = useRef<HTMLInputElement>(null);
|
||||
const button =
|
||||
"rounded border border-white/15 px-2 py-1.5 text-xs hover:bg-white/10 disabled:opacity-40";
|
||||
const seek = (t: number) => {
|
||||
const next = Math.max(0, Math.min(motion.duration, t));
|
||||
setTime(next);
|
||||
timeRef.current = next;
|
||||
setPreview(true);
|
||||
onPreview(next);
|
||||
};
|
||||
const edit = () => {
|
||||
if (preview) {
|
||||
const sampled = sampleDirectorMotion(motion, timeRef.current);
|
||||
if (sampled) onPose(sampled);
|
||||
}
|
||||
setPlaying(false);
|
||||
setPreview(false);
|
||||
onPreview(null);
|
||||
};
|
||||
useEffect(() => {
|
||||
if (!playing) return;
|
||||
let frame = 0,
|
||||
previous = 0;
|
||||
const tick = (now: number) => {
|
||||
const m = latest.current.motion;
|
||||
const next = timeRef.current + (previous ? (now - previous) / 1000 : 0);
|
||||
previous = now;
|
||||
const t =
|
||||
next >= m.duration ? (m.loop ? next % m.duration : m.duration) : next;
|
||||
timeRef.current = t;
|
||||
setTime(t);
|
||||
latest.current.onPreview(t);
|
||||
if (next >= m.duration && !m.loop) {
|
||||
setPlaying(false);
|
||||
return;
|
||||
}
|
||||
frame = requestAnimationFrame(tick);
|
||||
};
|
||||
const hide = () => {
|
||||
if (document.hidden) {
|
||||
setPlaying(false);
|
||||
toast.info("页面已切换，运镜预览已暂停");
|
||||
}
|
||||
};
|
||||
frame = requestAnimationFrame(tick);
|
||||
document.addEventListener("visibilitychange", hide);
|
||||
return () => {
|
||||
cancelAnimationFrame(frame);
|
||||
document.removeEventListener("visibilitychange", hide);
|
||||
};
|
||||
}, [playing]);
|
||||
useEffect(() => {
|
||||
if (disabled) setPlaying(false);
|
||||
}, [disabled]);
|
||||
useEffect(() => {
|
||||
if (timeRef.current > motion.duration) {
|
||||
setTime(motion.duration);
|
||||
onPreview(preview ? motion.duration : null);
|
||||
}
|
||||
}, [motion.duration, onPreview, preview]);
|
||||
const add = () => {
|
||||
if (!validDirectorPose(pose)) {
|
||||
toast.error("相机不能与注视点重合");
|
||||
return;
|
||||
}
|
||||
const existing = motion.keyframes.find(
|
||||
(k) => Math.abs(k.time - time) < 0.01,
|
||||
);
|
||||
if (!existing && motion.keyframes.length >= 100) {
|
||||
toast.error("最多支持 100 个关键帧");
|
||||
return;
|
||||
}
|
||||
const key = {
|
||||
id: existing?.id || crypto.randomUUID(),
|
||||
time: existing?.time ?? Number(time.toFixed(2)),
|
||||
position: [...pose.position] as DirectorPose["position"],
|
||||
target: [...pose.target] as DirectorPose["target"],
|
||||
fov: pose.fov,
|
||||
};
|
||||
onChange({
|
||||
...motion,
|
||||
keyframes: [...motion.keyframes.filter((k) => k.id !== key.id), key].sort(
|
||||
(a, b) => a.time - b.time,
|
||||
),
|
||||
});
|
||||
};
|
||||
const downloadJson = () => {
|
||||
const url = URL.createObjectURL(
|
||||
new Blob([JSON.stringify({ version: 1, motion }, null, 2)], {
|
||||
type: "application/json",
|
||||
}),
|
||||
);
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
a.download = "director-motion.json";
|
||||
a.click();
|
||||
setTimeout(() => URL.revokeObjectURL(url), 1000);
|
||||
};
|
||||
return (
|
||||
<section
|
||||
className="space-y-3 border-t border-white/15 pt-3"
|
||||
aria-label="镜头轨迹"
|
||||
>
|
||||
<div className="text-sm font-semibold">镜头轨迹</div>
|
||||
<fieldset disabled={disabled} className="space-y-3 disabled:opacity-50">
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
<label className="text-xs">
|
||||
时长（秒）
|
||||
<input
|
||||
aria-label="运镜时长"
|
||||
type="number"
|
||||
min={2}
|
||||
max={15}
|
||||
step={1}
|
||||
value={motion.duration}
|
||||
className="mt-1 w-full rounded bg-white/10 p-1"
|
||||
onChange={(e) => {
|
||||
const n = Number(e.target.value);
|
||||
if (n >= 2 && n <= 15) {
|
||||
edit();
|
||||
onChange(resizeDirectorMotion(motion, n));
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
<label className="text-xs">
|
||||
速度
|
||||
<select
|
||||
aria-label="运镜插值"
|
||||
value={motion.easing}
|
||||
className="mt-1 w-full rounded bg-[#303030] p-1"
|
||||
onChange={(e) =>
|
||||
onChange({
|
||||
...motion,
|
||||
easing: e.target.value as DirectorMotion["easing"],
|
||||
})
|
||||
}
|
||||
>
|
||||
<option value="smooth">缓入缓出</option>
|
||||
<option value="linear">匀速</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{(["push", "pull", "orbit", "pan"] as const).map((kind, i) => (
|
||||
<button
|
||||
type="button"
|
||||
className={button}
|
||||
key={kind}
|
||||
onClick={() => {
|
||||
const base =
|
||||
(preview
|
||||
? sampleDirectorMotion(motion, timeRef.current)
|
||||
: pose) || pose;
|
||||
if (!validDirectorPose(base)) {
|
||||
toast.error("相机不能与注视点重合");
|
||||
return;
|
||||
}
|
||||
edit();
|
||||
setTime(0);
|
||||
onChange(presetDirectorMotion(base, kind, motion.duration));
|
||||
}}
|
||||
>
|
||||
{["推进轨迹", "拉远轨迹", "环绕轨迹", "横移轨迹"][i]}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
className={button}
|
||||
disabled={motion.keyframes.length < 2}
|
||||
onClick={() => {
|
||||
edit();
|
||||
onOverview();
|
||||
}}
|
||||
>
|
||||
查看完整轨迹
|
||||
</button>
|
||||
<p className="text-xs text-white/50">
|
||||
预设会替换当前轨迹；黄色线条为相机路径。编辑机位后，在指定时间记录关键帧。
|
||||
</p>
|
||||
<label className="block text-xs">
|
||||
时间 {time.toFixed(2)} / {motion.duration.toFixed(2)} 秒
|
||||
<input
|
||||
aria-label="运镜时间轴"
|
||||
className="w-full"
|
||||
type="range"
|
||||
min={0}
|
||||
max={motion.duration}
|
||||
step={0.01}
|
||||
value={time}
|
||||
onChange={(e) => {
|
||||
setPlaying(false);
|
||||
seek(Number(e.target.value));
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
<button
|
||||
type="button"
|
||||
className={button}
|
||||
disabled={motion.keyframes.length < 2}
|
||||
onClick={() => {
|
||||
if (playing) {
|
||||
setPlaying(false);
|
||||
return;
|
||||
}
|
||||
seek(time >= motion.duration ? 0 : time);
|
||||
setPlaying(true);
|
||||
}}
|
||||
>
|
||||
{playing ? "暂停运镜" : "播放运镜"}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={button}
|
||||
onClick={() => {
|
||||
setPlaying(false);
|
||||
seek(0);
|
||||
}}
|
||||
>
|
||||
回到起点
|
||||
</button>
|
||||
<button type="button" className={button} onClick={edit}>
|
||||
编辑机位
|
||||
</button>
|
||||
<label className="flex items-center gap-1 text-xs">
|
||||
<input
|
||||
type="checkbox"
|
||||
aria-label="循环运镜"
|
||||
checked={motion.loop}
|
||||
onChange={(e) => onChange({ ...motion, loop: e.target.checked })}
|
||||
/>
|
||||
循环
|
||||
</label>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
className={button}
|
||||
disabled={preview}
|
||||
onClick={add}
|
||||
>
|
||||
记录/更新当前机位
|
||||
</button>
|
||||
{preview ? (
|
||||
<p className="text-xs text-amber-200">
|
||||
正在预览轨迹；点击“编辑机位”后可调整并记录机位。
|
||||
</p>
|
||||
) : null}
|
||||
<div className="max-h-48 space-y-1 overflow-y-auto">
|
||||
{motion.keyframes.map((k, i) => (
|
||||
<div
|
||||
key={k.id}
|
||||
className="flex items-center gap-1 rounded bg-white/5 p-1"
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
className={button}
|
||||
aria-label={`编辑关键帧 ${i + 1}`}
|
||||
onClick={() => {
|
||||
edit();
|
||||
setTime(k.time);
|
||||
onPose(k);
|
||||
}}
|
||||
>
|
||||
帧 {i + 1}
|
||||
</button>
|
||||
<input
|
||||
aria-label={`关键帧 ${i + 1} 时间`}
|
||||
type="number"
|
||||
step={0.1}
|
||||
min={0}
|
||||
max={motion.duration}
|
||||
value={Number(k.time.toFixed(3))}
|
||||
className="min-w-0 flex-1 rounded bg-white/10 p-1 text-xs"
|
||||
onChange={(e) => {
|
||||
const n = Number(e.target.value);
|
||||
if (n < 0 || n > motion.duration || !Number.isFinite(n))
|
||||
return;
|
||||
if (
|
||||
motion.keyframes.some(
|
||||
(other) =>
|
||||
other.id !== k.id && Math.abs(other.time - n) < 0.01,
|
||||
)
|
||||
) {
|
||||
toast.error("关键帧时间不能重复");
|
||||
return;
|
||||
}
|
||||
onChange({
|
||||
...motion,
|
||||
keyframes: motion.keyframes
|
||||
.map((other) =>
|
||||
other.id === k.id ? { ...k, time: n } : other,
|
||||
)
|
||||
.sort((a, b) => a.time - b.time),
|
||||
});
|
||||
}}
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
className={button}
|
||||
aria-label={`删除关键帧 ${i + 1}`}
|
||||
onClick={() => {
|
||||
edit();
|
||||
onChange({
|
||||
...motion,
|
||||
keyframes: motion.keyframes.filter(
|
||||
(other) => other.id !== k.id,
|
||||
),
|
||||
});
|
||||
}}
|
||||
>
|
||||
删除
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
<button type="button" className={button} onClick={downloadJson}>
|
||||
保存轨迹文件
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={button}
|
||||
onClick={() => input.current?.click()}
|
||||
>
|
||||
导入轨迹文件
|
||||
</button>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
className={`${button} w-full`}
|
||||
disabled={motion.keyframes.length < 2}
|
||||
onClick={() => {
|
||||
edit();
|
||||
onExport();
|
||||
}}
|
||||
>
|
||||
导出真实运镜视频
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={`${button} w-full`}
|
||||
disabled={motion.keyframes.length < 2}
|
||||
onClick={() => {
|
||||
edit();
|
||||
onSend();
|
||||
}}
|
||||
>
|
||||
发送首帧与运镜到视频节点
|
||||
</button>
|
||||
<p className="text-xs leading-5 text-white/50">
|
||||
真实导出严格使用本场景轨迹，不含音频。AI
|
||||
生视频使用首帧与运镜描述作参考，不保证精确复现轨迹。
|
||||
</p>
|
||||
</fieldset>
|
||||
{exportProgress !== null ? (
|
||||
<div role="status" className="text-xs">
|
||||
正在导出 {exportProgress.toFixed(1)} / {motion.duration} 秒{" "}
|
||||
<button className={button} onClick={onCancel}>
|
||||
取消导出
|
||||
</button>
|
||||
</div>
|
||||
) : null}
|
||||
<input
|
||||
ref={input}
|
||||
type="file"
|
||||
accept=".json,application/json"
|
||||
className="hidden"
|
||||
onChange={async (e) => {
|
||||
const file = e.target.files?.[0];
|
||||
e.target.value = "";
|
||||
if (!file) return;
|
||||
try {
|
||||
if (file.size > 256 * 1024) throw Error("轨迹文件不能超过 256 KiB");
|
||||
const data = JSON.parse(await file.text());
|
||||
if (data.version !== 1) throw Error("不支持的轨迹文件版本");
|
||||
const m = readDirectorMotion(data.motion);
|
||||
if (
|
||||
m.keyframes.length < 2 ||
|
||||
m.keyframes.length !== data.motion?.keyframes?.length
|
||||
)
|
||||
throw Error("轨迹数据无效或包含重复时间");
|
||||
edit();
|
||||
onChange(m);
|
||||
setTime(0);
|
||||
toast.success("轨迹已导入");
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "轨迹导入失败");
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@ -1,5 +1,5 @@
|
||||
"use client";
|
||||
import { useRef, useState } from "react";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { Camera, Plus, Upload, X, Trash2 } from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
@ -9,15 +9,25 @@ import {
|
||||
import {
|
||||
readDirectorScene,
|
||||
directorRatios,
|
||||
directorImageSize,
|
||||
type DirectorScene,
|
||||
type DirectorSnapshot,
|
||||
type DirectorObject,
|
||||
type DirectorVector,
|
||||
} from "@/lib/video-canvas/director-scene";
|
||||
import { DirectorMotionPanel } from "./DirectorMotionPanel";
|
||||
import {
|
||||
sampleDirectorMotion,
|
||||
mergeDirectorCameraPose,
|
||||
type DirectorMotion,
|
||||
} from "@/lib/video-canvas/director-motion";
|
||||
async function upload(file: File) {
|
||||
const form = new FormData();
|
||||
form.append("file", file);
|
||||
form.append("purpose", "director-studio");
|
||||
form.append(
|
||||
"purpose",
|
||||
file.type.startsWith("video/") ? "canvas-video-process" : "director-studio",
|
||||
);
|
||||
const response = await fetch(
|
||||
/\.glb$/i.test(file.name)
|
||||
? "/api/canvas/director/upload-model"
|
||||
@ -38,11 +48,20 @@ export function DirectorStudioDialog({
|
||||
onChange,
|
||||
onClose,
|
||||
onSend,
|
||||
onSendMotion,
|
||||
onSendVideo,
|
||||
}: {
|
||||
initialScene: unknown;
|
||||
onChange: (scene: DirectorScene) => void;
|
||||
onClose: () => void;
|
||||
onSend: (snapshots: DirectorSnapshot[]) => void;
|
||||
onSendMotion: (snapshot: DirectorSnapshot, motion: DirectorMotion) => void;
|
||||
onSendVideo: (video: {
|
||||
url: string;
|
||||
width: number;
|
||||
height: number;
|
||||
duration: number;
|
||||
}) => void;
|
||||
}) {
|
||||
const [scene, setScene] = useState(() => readDirectorScene(initialScene));
|
||||
const current = useRef(scene);
|
||||
@ -59,6 +78,85 @@ export function DirectorStudioDialog({
|
||||
const [cameraView, setCameraView] = useState(false);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const viewport = useRef<DirectorViewportHandle>(null);
|
||||
const [previewTime, setPreviewTime] = useState<number | null>(null);
|
||||
const [exportProgress, setExportProgress] = useState<number | null>(null);
|
||||
const recording = useRef<AbortController | null>(null);
|
||||
useEffect(() => () => recording.current?.abort(), []);
|
||||
const [exportedVideo, setExportedVideo] = useState<{
|
||||
blob: Blob;
|
||||
url: string;
|
||||
width: number;
|
||||
height: number;
|
||||
duration: number;
|
||||
} | null>(null);
|
||||
useEffect(
|
||||
() => () => {
|
||||
if (exportedVideo) URL.revokeObjectURL(exportedVideo.url);
|
||||
},
|
||||
[exportedVideo],
|
||||
);
|
||||
const exportMotion = async () => {
|
||||
if (busy) return;
|
||||
setBusy(true);
|
||||
setExportProgress(0);
|
||||
const controller = new AbortController();
|
||||
recording.current = controller;
|
||||
try {
|
||||
const motion = current.current.motion;
|
||||
const dimensions = directorImageSize(current.current.ratio);
|
||||
const blob = await viewport.current!.record(
|
||||
controller.signal,
|
||||
setExportProgress,
|
||||
);
|
||||
setExportedVideo({
|
||||
blob,
|
||||
url: URL.createObjectURL(blob),
|
||||
...dimensions,
|
||||
duration: motion.duration,
|
||||
});
|
||||
toast.success("真实运镜视频已导出，可下载或发送到画布");
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "轨迹导出失败");
|
||||
} finally {
|
||||
recording.current = null;
|
||||
setBusy(false);
|
||||
setExportProgress(null);
|
||||
}
|
||||
};
|
||||
const sendMotion = async () => {
|
||||
if (busy) return;
|
||||
setBusy(true);
|
||||
try {
|
||||
const motion = current.current.motion;
|
||||
const pose = sampleDirectorMotion(motion, 0);
|
||||
if (!pose || motion.keyframes.length < 2)
|
||||
throw Error("至少需要两个关键帧");
|
||||
const image = await viewport.current!.capture(pose);
|
||||
const url = await upload(
|
||||
new File([image.blob], `director-motion-${Date.now()}.png`, {
|
||||
type: "image/png",
|
||||
}),
|
||||
);
|
||||
const snapshot = {
|
||||
id: crypto.randomUUID(),
|
||||
title: "运镜首帧",
|
||||
url,
|
||||
width: image.width,
|
||||
height: image.height,
|
||||
cameraId: current.current.activeCameraId,
|
||||
};
|
||||
commit({
|
||||
...current.current,
|
||||
snapshots: [...current.current.snapshots, snapshot],
|
||||
});
|
||||
onSendMotion(snapshot, motion);
|
||||
toast.success("首帧和运镜描述已发送，请在视频节点选择模型");
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "发送失败");
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
};
|
||||
const fileInput = useRef<HTMLInputElement>(null);
|
||||
const camera =
|
||||
scene.cameras.find((c) => c.id === scene.activeCameraId) ||
|
||||
@ -75,7 +173,9 @@ export function DirectorStudioDialog({
|
||||
commit({
|
||||
...current.current,
|
||||
cameras: current.current.cameras.map((c) =>
|
||||
c.id === current.current.activeCameraId ? { ...c, ...patch } : c,
|
||||
c.id === current.current.activeCameraId
|
||||
? mergeDirectorCameraPose(c, patch)
|
||||
: c,
|
||||
),
|
||||
});
|
||||
const addObject = (
|
||||
@ -150,6 +250,7 @@ export function DirectorStudioDialog({
|
||||
<button
|
||||
className={button}
|
||||
aria-pressed={!cameraView}
|
||||
disabled={busy}
|
||||
onClick={() => setCameraView(false)}
|
||||
>
|
||||
导演视角
|
||||
@ -157,17 +258,28 @@ export function DirectorStudioDialog({
|
||||
<button
|
||||
className={button}
|
||||
aria-pressed={cameraView}
|
||||
disabled={busy}
|
||||
onClick={() => setCameraView(true)}
|
||||
>
|
||||
机位视角
|
||||
</button>
|
||||
</div>
|
||||
<button aria-label="关闭导演台" className={button} onClick={onClose}>
|
||||
<button
|
||||
aria-label="关闭导演台"
|
||||
className={button}
|
||||
onClick={() => {
|
||||
recording.current?.abort();
|
||||
onClose();
|
||||
}}
|
||||
>
|
||||
<X size={18} />
|
||||
</button>
|
||||
</header>
|
||||
<div className="grid min-h-0 flex-1 grid-cols-[210px_minmax(0,1fr)_260px]">
|
||||
<aside className="overflow-y-auto border-r border-white/10 p-4">
|
||||
<aside
|
||||
inert={busy}
|
||||
className="overflow-y-auto border-r border-white/10 p-4"
|
||||
>
|
||||
<div className="mb-3 text-sm text-white/60">场景对象</div>
|
||||
<div className="space-y-2">
|
||||
{scene.objects.map((o) => (
|
||||
@ -219,7 +331,10 @@ export function DirectorStudioDialog({
|
||||
左键旋转视角，右键平移，滚轮缩放。选择对象后使用变换轴调整。
|
||||
</p>
|
||||
</aside>
|
||||
<main className="relative flex min-h-0 items-center justify-center overflow-hidden bg-[#080808]">
|
||||
<main
|
||||
inert={busy}
|
||||
className="relative flex min-h-0 items-center justify-center overflow-hidden bg-[#080808]"
|
||||
>
|
||||
<div
|
||||
className="h-full max-h-full w-full"
|
||||
style={{ aspectRatio: scene.ratio.replace(":", " / ") }}
|
||||
@ -230,6 +345,8 @@ export function DirectorStudioDialog({
|
||||
selected={selected}
|
||||
mode={mode}
|
||||
cameraView={cameraView}
|
||||
previewTime={previewTime}
|
||||
locked={busy}
|
||||
onTransform={patchObject}
|
||||
onCamera={(position, target) => patchCamera({ position, target })}
|
||||
/>
|
||||
@ -251,134 +368,226 @@ export function DirectorStudioDialog({
|
||||
</div>
|
||||
</main>
|
||||
<aside className="space-y-4 overflow-y-auto border-l border-white/10 p-4">
|
||||
<label className="block text-sm">
|
||||
机位
|
||||
<select
|
||||
aria-label="机位"
|
||||
className={`${field} mt-2`}
|
||||
value={scene.activeCameraId}
|
||||
onChange={(e) =>
|
||||
commit({ ...current.current, activeCameraId: e.target.value })
|
||||
}
|
||||
>
|
||||
{scene.cameras.map((c) => (
|
||||
<option key={c.id} value={c.id}>
|
||||
{c.name}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{[
|
||||
["正面", [0, 2, 7]],
|
||||
["侧面", [7, 2, 0]],
|
||||
["俯视", [0, 8, 0.1]],
|
||||
].map(([name, position]) => (
|
||||
<button
|
||||
className={button}
|
||||
key={String(name)}
|
||||
onClick={() => {
|
||||
const id = crypto.randomUUID();
|
||||
commit({
|
||||
...current.current,
|
||||
activeCameraId: id,
|
||||
cameras: [
|
||||
...current.current.cameras,
|
||||
{
|
||||
id,
|
||||
name: `${name}机位${current.current.cameras.length + 1}`,
|
||||
position: position as DirectorVector,
|
||||
target: [0, 1, 0],
|
||||
fov: 50,
|
||||
},
|
||||
],
|
||||
});
|
||||
}}
|
||||
<fieldset
|
||||
disabled={busy || previewTime !== null}
|
||||
className="space-y-4"
|
||||
>
|
||||
<label className="block text-sm">
|
||||
机位
|
||||
<select
|
||||
aria-label="机位"
|
||||
className={`${field} mt-2`}
|
||||
value={scene.activeCameraId}
|
||||
onChange={(e) =>
|
||||
commit({ ...current.current, activeCameraId: e.target.value })
|
||||
}
|
||||
>
|
||||
<Plus size={12} className="inline" />
|
||||
{String(name)}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<label className="block text-sm">
|
||||
视角 FOV
|
||||
<input
|
||||
aria-label="视角 FOV"
|
||||
className={`${field} mt-2`}
|
||||
type="number"
|
||||
min={15}
|
||||
max={90}
|
||||
value={camera.fov}
|
||||
onChange={(e) => {
|
||||
const n = Number(e.target.value);
|
||||
if (n >= 15 && n <= 90) patchCamera({ fov: n });
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
<label className="block text-sm">
|
||||
截图比例
|
||||
<select
|
||||
aria-label="截图比例"
|
||||
className={`${field} mt-2`}
|
||||
value={scene.ratio}
|
||||
onChange={(e) =>
|
||||
commit({ ...current.current, ratio: e.target.value })
|
||||
}
|
||||
>
|
||||
{directorRatios.map((r) => (
|
||||
<option key={r}>{r}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
{object ? (
|
||||
<section className="space-y-2 border-t border-white/10 pt-3">
|
||||
<div className="flex items-center justify-between text-sm">
|
||||
<span>{object.name}</span>
|
||||
{scene.cameras.map((c) => (
|
||||
<option key={c.id} value={c.id}>
|
||||
{c.name}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{[
|
||||
["正面", [0, 2, 7]],
|
||||
["侧面", [7, 2, 0]],
|
||||
["俯视", [0, 8, 0.1]],
|
||||
].map(([name, position]) => (
|
||||
<button
|
||||
aria-label="删除所选场景对象"
|
||||
className={button}
|
||||
key={String(name)}
|
||||
onClick={() => {
|
||||
const id = crypto.randomUUID();
|
||||
commit({
|
||||
...current.current,
|
||||
objects: current.current.objects.filter(
|
||||
(o) => o.id !== object.id,
|
||||
),
|
||||
activeCameraId: id,
|
||||
cameras: [
|
||||
...current.current.cameras,
|
||||
{
|
||||
id,
|
||||
name: `${name}机位${current.current.cameras.length + 1}`,
|
||||
position: position as DirectorVector,
|
||||
target: [0, 1, 0],
|
||||
fov: 50,
|
||||
},
|
||||
],
|
||||
});
|
||||
setSelected("");
|
||||
}}
|
||||
>
|
||||
<Trash2 size={14} />
|
||||
<Plus size={12} className="inline" />
|
||||
{String(name)}
|
||||
</button>
|
||||
</div>
|
||||
{(["position", "rotation", "scale"] as const).map((key, j) => (
|
||||
<div key={key}>
|
||||
<div className="mb-1 text-xs text-white/50">
|
||||
{["位置", "旋转（弧度）", "缩放"][j]}
|
||||
</div>
|
||||
<div className="grid grid-cols-3 gap-1">
|
||||
{object[key].map((v, i) => (
|
||||
<input
|
||||
key={i}
|
||||
aria-label={`${object.name}-${key}-${"XYZ"[i]}`}
|
||||
className={field}
|
||||
type="number"
|
||||
step="0.1"
|
||||
value={Number(v.toFixed(2))}
|
||||
onChange={(e) => {
|
||||
const n = Number(e.target.value);
|
||||
if (
|
||||
!Number.isFinite(n) ||
|
||||
(key === "scale" && n <= 0)
|
||||
)
|
||||
return;
|
||||
const values = [...object[key]] as DirectorVector;
|
||||
values[i] = n;
|
||||
patchObject(object.id, { [key]: values });
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
<label className="block text-sm">
|
||||
视角 FOV
|
||||
<input
|
||||
aria-label="视角 FOV"
|
||||
className={`${field} mt-2`}
|
||||
type="number"
|
||||
min={15}
|
||||
max={90}
|
||||
value={camera.fov}
|
||||
onChange={(e) => {
|
||||
const n = Number(e.target.value);
|
||||
if (n >= 15 && n <= 90) patchCamera({ fov: n });
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
<label className="block text-sm">
|
||||
截图比例
|
||||
<select
|
||||
aria-label="截图比例"
|
||||
className={`${field} mt-2`}
|
||||
value={scene.ratio}
|
||||
onChange={(e) =>
|
||||
commit({ ...current.current, ratio: e.target.value })
|
||||
}
|
||||
>
|
||||
{directorRatios.map((r) => (
|
||||
<option key={r}>{r}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
{(["position", "target"] as const).map((key, j) => (
|
||||
<label className="block text-xs" key={key}>
|
||||
{j === 0 ? "相机位置" : "相机注视点"}
|
||||
<div className="mt-1 grid grid-cols-3 gap-1">
|
||||
{camera[key].map((n, i) => (
|
||||
<input
|
||||
key={i}
|
||||
aria-label={`相机-${key}-${"XYZ"[i]}`}
|
||||
className={field}
|
||||
type="number"
|
||||
step="0.1"
|
||||
value={Number(n.toFixed(2))}
|
||||
onChange={(e) => {
|
||||
const n = Number(e.target.value);
|
||||
if (!Number.isFinite(n) || Math.abs(n) > 10000) return;
|
||||
const values = [...camera[key]] as DirectorVector;
|
||||
values[i] = n;
|
||||
patchCamera({ [key]: values });
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</label>
|
||||
))}
|
||||
{object ? (
|
||||
<section className="space-y-2 border-t border-white/10 pt-3">
|
||||
<div className="flex items-center justify-between text-sm">
|
||||
<span>{object.name}</span>
|
||||
<button
|
||||
aria-label="删除所选场景对象"
|
||||
onClick={() => {
|
||||
commit({
|
||||
...current.current,
|
||||
objects: current.current.objects.filter(
|
||||
(o) => o.id !== object.id,
|
||||
),
|
||||
});
|
||||
setSelected("");
|
||||
}}
|
||||
>
|
||||
<Trash2 size={14} />
|
||||
</button>
|
||||
</div>
|
||||
{(["position", "rotation", "scale"] as const).map((key, j) => (
|
||||
<div key={key}>
|
||||
<div className="mb-1 text-xs text-white/50">
|
||||
{["位置", "旋转（弧度）", "缩放"][j]}
|
||||
</div>
|
||||
<div className="grid grid-cols-3 gap-1">
|
||||
{object[key].map((v, i) => (
|
||||
<input
|
||||
key={i}
|
||||
aria-label={`${object.name}-${key}-${"XYZ"[i]}`}
|
||||
className={field}
|
||||
type="number"
|
||||
step="0.1"
|
||||
value={Number(v.toFixed(2))}
|
||||
onChange={(e) => {
|
||||
const n = Number(e.target.value);
|
||||
if (
|
||||
!Number.isFinite(n) ||
|
||||
(key === "scale" && n <= 0)
|
||||
)
|
||||
return;
|
||||
const values = [...object[key]] as DirectorVector;
|
||||
values[i] = n;
|
||||
patchObject(object.id, { [key]: values });
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</section>
|
||||
) : null}
|
||||
</fieldset>
|
||||
<DirectorMotionPanel
|
||||
motion={scene.motion}
|
||||
pose={camera}
|
||||
disabled={busy}
|
||||
onChange={(motion) => commit({ ...current.current, motion })}
|
||||
onPose={patchCamera}
|
||||
onPreview={setPreviewTime}
|
||||
onExport={exportMotion}
|
||||
onSend={sendMotion}
|
||||
exportProgress={exportProgress}
|
||||
onCancel={() => recording.current?.abort()}
|
||||
onOverview={() => {
|
||||
setCameraView(false);
|
||||
viewport.current?.showPath();
|
||||
}}
|
||||
/>
|
||||
{exportedVideo ? (
|
||||
<section
|
||||
className="space-y-2 border-t border-white/10 pt-3"
|
||||
aria-label="运镜导出结果"
|
||||
>
|
||||
<video
|
||||
controls
|
||||
src={exportedVideo.url}
|
||||
className="w-full"
|
||||
aria-label="真实运镜导出预览"
|
||||
/>
|
||||
<a
|
||||
className={button}
|
||||
href={exportedVideo.url}
|
||||
download={`director-motion.${exportedVideo.blob.type.includes("mp4") ? "mp4" : "webm"}`}
|
||||
>
|
||||
下载运镜视频
|
||||
</a>
|
||||
<button
|
||||
disabled={busy}
|
||||
className={`${button} w-full`}
|
||||
onClick={async () => {
|
||||
setBusy(true);
|
||||
try {
|
||||
const ext = exportedVideo.blob.type.includes("mp4")
|
||||
? "mp4"
|
||||
: "webm";
|
||||
const url = await upload(
|
||||
new File(
|
||||
[exportedVideo.blob],
|
||||
`director-motion-${Date.now()}.${ext}`,
|
||||
{ type: exportedVideo.blob.type.split(";")[0] },
|
||||
),
|
||||
);
|
||||
onSendVideo({ ...exportedVideo, url });
|
||||
toast.success("运镜视频已发送到画布");
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "上传失败");
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}}
|
||||
>
|
||||
发送真实运镜视频到画布
|
||||
</button>
|
||||
</section>
|
||||
) : null}
|
||||
<section className="space-y-2 border-t border-white/10 pt-3">
|
||||
|
||||
@ -16,14 +16,28 @@ import {
|
||||
type DirectorScene,
|
||||
type DirectorObject,
|
||||
} from "@/lib/video-canvas/director-scene";
|
||||
import {
|
||||
sampleDirectorMotion,
|
||||
type DirectorPose,
|
||||
} from "@/lib/video-canvas/director-motion";
|
||||
import { recordDirectorVideo } from "@/lib/video-canvas/director-recording";
|
||||
export type DirectorViewportHandle = {
|
||||
capture: () => Promise<{ blob: Blob; width: number; height: number }>;
|
||||
showPath: () => void;
|
||||
record: (
|
||||
signal: AbortSignal,
|
||||
onProgress: (time: number) => void,
|
||||
) => Promise<Blob>;
|
||||
capture: (
|
||||
pose?: DirectorPose,
|
||||
) => Promise<{ blob: Blob; width: number; height: number }>;
|
||||
};
|
||||
type Props = {
|
||||
scene: DirectorScene;
|
||||
selected: string;
|
||||
mode: "translate" | "rotate" | "scale";
|
||||
cameraView: boolean;
|
||||
previewTime: number | null;
|
||||
locked: boolean;
|
||||
onTransform: (id: string, patch: Partial<DirectorObject>) => void;
|
||||
onCamera: (
|
||||
position: [number, number, number],
|
||||
@ -32,7 +46,7 @@ type Props = {
|
||||
};
|
||||
function dispose(root: THREE.Object3D) {
|
||||
root.traverse((o) => {
|
||||
if (o instanceof THREE.Mesh) {
|
||||
if (o instanceof THREE.Mesh || o instanceof THREE.Line) {
|
||||
o.geometry.dispose();
|
||||
for (const m of Array.isArray(o.material) ? o.material : [o.material]) {
|
||||
for (const value of Object.values(m))
|
||||
@ -55,6 +69,7 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
transform: TransformControls;
|
||||
helper: THREE.Object3D;
|
||||
grid: THREE.GridHelper;
|
||||
path: THREE.Group;
|
||||
objects: THREE.Group;
|
||||
pending: number;
|
||||
resize: () => void;
|
||||
@ -100,6 +115,8 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
world.add(floor);
|
||||
const grid = new THREE.GridHelper(30, 30, 0x596573, 0x929daa);
|
||||
world.add(grid);
|
||||
const path = new THREE.Group();
|
||||
world.add(path);
|
||||
const camera = new THREE.PerspectiveCamera(50, 1, 0.05, 500);
|
||||
camera.position.set(4, 3, 7);
|
||||
const orbit = new OrbitControls(camera, renderer.domElement);
|
||||
@ -112,7 +129,10 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
const objects = new THREE.Group();
|
||||
world.add(objects);
|
||||
transform.addEventListener("dragging-changed", (e) => {
|
||||
orbit.enabled = !e.value && !latest.current.cameraView;
|
||||
orbit.enabled =
|
||||
!e.value &&
|
||||
!latest.current.cameraView &&
|
||||
latest.current.previewTime === null;
|
||||
});
|
||||
transform.addEventListener("mouseUp", () => {
|
||||
const o = transform.object;
|
||||
@ -123,12 +143,14 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
scale: o.scale.toArray(),
|
||||
});
|
||||
});
|
||||
orbit.addEventListener("end", () =>
|
||||
orbit.addEventListener("end", () => {
|
||||
if (latest.current.previewTime !== null || latest.current.cameraView)
|
||||
return;
|
||||
latest.current.onCamera(
|
||||
camera.position.toArray(),
|
||||
orbit.target.toArray(),
|
||||
),
|
||||
);
|
||||
);
|
||||
});
|
||||
|
||||
const resize = () => {
|
||||
if (!host.current) return;
|
||||
@ -148,6 +170,7 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
transform,
|
||||
helper,
|
||||
grid,
|
||||
path,
|
||||
objects,
|
||||
pending: 0,
|
||||
resize,
|
||||
@ -157,7 +180,7 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
observer.observe(host.current);
|
||||
resize();
|
||||
renderer.setAnimationLoop(() => {
|
||||
orbit.update();
|
||||
if (orbit.enabled) orbit.update();
|
||||
renderer.render(world, camera);
|
||||
});
|
||||
setReady(true);
|
||||
@ -193,7 +216,11 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
object.rotation.set(...item.rotation);
|
||||
object.scale.fromArray(item.scale);
|
||||
r.objects.add(object);
|
||||
if (item.id === latest.current.selected && !latest.current.cameraView)
|
||||
if (
|
||||
item.id === latest.current.selected &&
|
||||
!latest.current.cameraView &&
|
||||
latest.current.previewTime === null
|
||||
)
|
||||
r.transform.attach(object);
|
||||
};
|
||||
for (const item of props.scene.objects) {
|
||||
@ -274,18 +301,26 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
useEffect(() => {
|
||||
const r = runtime.current;
|
||||
if (!r) return;
|
||||
if (props.locked) {
|
||||
r.orbit.enabled = false;
|
||||
r.transform.detach();
|
||||
return;
|
||||
}
|
||||
const camera =
|
||||
props.scene.cameras.find((c) => c.id === props.scene.activeCameraId) ||
|
||||
props.scene.cameras[0];
|
||||
r.camera.position.fromArray(camera.position);
|
||||
r.camera.fov = camera.fov;
|
||||
r.camera.updateProjectionMatrix();
|
||||
r.orbit.target.fromArray(camera.target);
|
||||
r.orbit.enabled = !props.cameraView;
|
||||
r.orbit.update();
|
||||
if (props.previewTime === null) {
|
||||
r.camera.position.fromArray(camera.position);
|
||||
r.camera.fov = camera.fov;
|
||||
r.camera.updateProjectionMatrix();
|
||||
r.orbit.target.fromArray(camera.target);
|
||||
r.orbit.update();
|
||||
}
|
||||
r.orbit.enabled = !props.cameraView && props.previewTime === null;
|
||||
r.transform.setMode(props.mode);
|
||||
const selected = r.objects.getObjectByName(props.selected);
|
||||
if (selected && !props.cameraView) r.transform.attach(selected);
|
||||
if (selected && !props.cameraView && props.previewTime === null)
|
||||
r.transform.attach(selected);
|
||||
else r.transform.detach();
|
||||
}, [
|
||||
props.scene.cameras,
|
||||
@ -293,6 +328,59 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
props.selected,
|
||||
props.mode,
|
||||
props.cameraView,
|
||||
props.previewTime === null,
|
||||
props.locked,
|
||||
ready,
|
||||
]);
|
||||
useEffect(() => {
|
||||
const r = runtime.current;
|
||||
if (!r) return;
|
||||
dispose(r.path);
|
||||
r.path.clear();
|
||||
const motion = props.scene.motion;
|
||||
if (motion.keyframes.length < 2) return;
|
||||
const points = Array.from(
|
||||
{ length: 121 },
|
||||
(_, i) =>
|
||||
new THREE.Vector3(
|
||||
...sampleDirectorMotion(motion, (motion.duration * i) / 120)!
|
||||
.position,
|
||||
),
|
||||
);
|
||||
r.path.add(
|
||||
new THREE.Line(
|
||||
new THREE.BufferGeometry().setFromPoints(points),
|
||||
new THREE.LineBasicMaterial({ color: 0xfbbf24 }),
|
||||
),
|
||||
);
|
||||
for (const k of motion.keyframes) {
|
||||
const dot = new THREE.Mesh(
|
||||
new THREE.SphereGeometry(0.08, 10, 8),
|
||||
new THREE.MeshBasicMaterial({ color: 0xfbbf24 }),
|
||||
);
|
||||
dot.position.fromArray(k.position);
|
||||
r.path.add(dot);
|
||||
}
|
||||
}, [props.scene.motion, ready]);
|
||||
useEffect(() => {
|
||||
const r = runtime.current;
|
||||
if (!r) return;
|
||||
r.path.visible =
|
||||
!props.locked && !props.cameraView && props.previewTime === null;
|
||||
if (props.locked || props.previewTime === null) return;
|
||||
const pose = sampleDirectorMotion(props.scene.motion, props.previewTime);
|
||||
if (!pose) return;
|
||||
r.camera.position.fromArray(pose.position);
|
||||
r.orbit.target.fromArray(pose.target);
|
||||
r.camera.fov = pose.fov;
|
||||
r.camera.updateProjectionMatrix();
|
||||
r.camera.lookAt(r.orbit.target);
|
||||
r.transform.detach();
|
||||
}, [
|
||||
props.previewTime,
|
||||
props.scene.motion,
|
||||
props.cameraView,
|
||||
props.locked,
|
||||
ready,
|
||||
]);
|
||||
useEffect(() => {
|
||||
@ -301,7 +389,81 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
useImperativeHandle(
|
||||
ref,
|
||||
() => ({
|
||||
capture: async () => {
|
||||
showPath: () => {
|
||||
const r = runtime.current;
|
||||
if (!r) return;
|
||||
const points = latest.current.scene.motion.keyframes.map(
|
||||
(k) => new THREE.Vector3(...k.position),
|
||||
);
|
||||
if (!points.length) return;
|
||||
points.push(new THREE.Vector3(0, 1, 0));
|
||||
const box = new THREE.Box3().setFromPoints(points),
|
||||
center = box.getCenter(new THREE.Vector3());
|
||||
const radius = Math.max(3, box.getSize(new THREE.Vector3()).length());
|
||||
const position = center
|
||||
.clone()
|
||||
.add(
|
||||
new THREE.Vector3(1, 0.8, 1)
|
||||
.normalize()
|
||||
.multiplyScalar(radius * 1.4),
|
||||
);
|
||||
latest.current.onCamera(position.toArray(), center.toArray());
|
||||
},
|
||||
record: async (signal, onProgress) => {
|
||||
const r = runtime.current;
|
||||
if (!r || !ready || r.pending || error)
|
||||
throw Error(error || "3D 场景尚未准备好");
|
||||
if (latest.current.scene.motion.keyframes.length < 2)
|
||||
throw Error("至少需要两个关键帧");
|
||||
const motion = latest.current.scene.motion;
|
||||
const size = r.renderer.getSize(new THREE.Vector2()),
|
||||
pixelRatio = r.renderer.getPixelRatio();
|
||||
const position = r.camera.position.clone(),
|
||||
target = r.orbit.target.clone(),
|
||||
fov = r.camera.fov,
|
||||
aspect = r.camera.aspect;
|
||||
const visibility = [r.helper.visible, r.grid.visible, r.path.visible],
|
||||
enabled = r.orbit.enabled;
|
||||
const dimensions = directorImageSize(latest.current.scene.ratio);
|
||||
r.helper.visible = false;
|
||||
r.grid.visible = false;
|
||||
r.path.visible = false;
|
||||
r.orbit.enabled = false;
|
||||
r.renderer.setPixelRatio(1);
|
||||
r.renderer.setSize(dimensions.width, dimensions.height, false);
|
||||
r.camera.aspect = dimensions.width / dimensions.height;
|
||||
try {
|
||||
return await recordDirectorVideo({
|
||||
canvas: r.renderer.domElement,
|
||||
duration: motion.duration,
|
||||
signal,
|
||||
onProgress,
|
||||
render: (time) => {
|
||||
const pose = sampleDirectorMotion(motion, time)!;
|
||||
r.camera.position.fromArray(pose.position);
|
||||
r.orbit.target.fromArray(pose.target);
|
||||
r.camera.fov = pose.fov;
|
||||
r.camera.updateProjectionMatrix();
|
||||
r.camera.lookAt(r.orbit.target);
|
||||
r.renderer.render(r.world, r.camera);
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
if (runtime.current === r) {
|
||||
r.renderer.setPixelRatio(pixelRatio);
|
||||
r.renderer.setSize(size.x, size.y);
|
||||
r.camera.position.copy(position);
|
||||
r.orbit.target.copy(target);
|
||||
r.camera.fov = fov;
|
||||
r.camera.aspect = aspect;
|
||||
r.camera.updateProjectionMatrix();
|
||||
r.orbit.enabled = enabled;
|
||||
r.orbit.update();
|
||||
[r.helper.visible, r.grid.visible, r.path.visible] = visibility;
|
||||
}
|
||||
}
|
||||
},
|
||||
capture: async (pose) => {
|
||||
const r = runtime.current;
|
||||
if (!r || !ready || r.pending || error)
|
||||
throw Error(error || "3D 场景尚未准备好");
|
||||
@ -311,6 +473,19 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
const previousSize = r.renderer.getSize(new THREE.Vector2());
|
||||
const pixelRatio = r.renderer.getPixelRatio();
|
||||
const aspect = r.camera.aspect;
|
||||
const position = r.camera.position.clone(),
|
||||
target = r.orbit.target.clone(),
|
||||
fov = r.camera.fov;
|
||||
const visibility = [r.helper.visible, r.grid.visible, r.path.visible];
|
||||
const enabled = r.orbit.enabled;
|
||||
r.orbit.enabled = false;
|
||||
if (pose) {
|
||||
r.camera.position.fromArray(pose.position);
|
||||
r.orbit.target.fromArray(pose.target);
|
||||
r.camera.fov = pose.fov;
|
||||
r.camera.lookAt(r.orbit.target);
|
||||
}
|
||||
r.path.visible = false;
|
||||
r.helper.visible = false;
|
||||
r.grid.visible = false;
|
||||
r.renderer.setPixelRatio(1);
|
||||
@ -331,8 +506,13 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
r.renderer.setSize(previousSize.x, previousSize.y);
|
||||
r.camera.aspect = aspect;
|
||||
r.camera.updateProjectionMatrix();
|
||||
r.grid.visible = true;
|
||||
r.helper.visible = true;
|
||||
r.camera.position.copy(position);
|
||||
r.orbit.target.copy(target);
|
||||
r.camera.fov = fov;
|
||||
r.camera.updateProjectionMatrix();
|
||||
r.orbit.update();
|
||||
[r.helper.visible, r.grid.visible, r.path.visible] = visibility;
|
||||
r.orbit.enabled = enabled;
|
||||
}
|
||||
},
|
||||
}),
|
||||
|
||||
50
docs/video-canvas-director-motion-2026-09-17.md
Normal file
50
docs/video-canvas-director-motion-2026-09-17.md
Normal file
@ -0,0 +1,50 @@
|
||||
# 3D 导演台镜头轨迹修复与验收（2026-09-17）
|
||||
|
||||
此前导演台只保存静态机位，运镜素材库只向提示词追加文字。静态机位截图生成视频的验收不代表连续镜头轨迹可用。本次补齐时间轴、真实 3D 播放和视频导出，未部署生产环境。
|
||||
|
||||
## 完成的功能
|
||||
|
||||
- 场景增加 `motion` 数据：2–15 秒、最多 100 个关键帧、相机位置、注视点、FOV、线性/缓入缓出插值和循环设置。兼容没有轨迹的旧场景；轨迹随原有画布保存链路持久化，不需要数据库迁移。
|
||||
- 支持记录、覆盖、编辑、删除关键帧及修改时间；重复时间和无效数据拦截。调整时长按比例缩放关键帧时间。
|
||||
- 推进、拉远、90° 环绕、横移预设；黄色轨迹与关键帧标记、完整轨迹总览。
|
||||
- 播放、暂停、循环、时间轴定位；预览时关闭手动相机操控。进入编辑时从当前预览机位继续，避免跳回旧机位。页面进入后台时暂停预览。
|
||||
- 导入/导出带版本号的 JSON 轨迹文件；文件限制 256 KiB，坏文件不会覆盖现有轨迹。
|
||||
- 按真实相机轨迹录制无声 MP4，隐藏网格、辅助轴和路径，支持取消；浏览器不支持 MP4 编码时明确提示更新浏览器。切到后台、关闭导演台、渲染错误或超时会停止录制并释放媒体轨道，不把部分文件当成功结果。
|
||||
- 导出结果可预览、下载、上传为真实视频节点；保存实际时长，供剪辑使用。视频上传使用现有 `canvas-video-process` 用途参数。
|
||||
- “发送首帧与运镜到视频节点”真实渲染 0 秒机位，建立导演台→首帧→视频节点连线，携带完整轨迹元数据及精简的自然语言运镜描述。模型、尺寸等继续由视频节点选择；不自动发起计费生成。
|
||||
|
||||
## 实测中修复的回归
|
||||
|
||||
- 使用整个关键帧覆盖机位会把关键帧 ID 写成相机 ID，导致后续 FOV 编辑失效。现在仅合并 position、target、fov，保留相机身份，并有专门回归测试。
|
||||
- WebM 原始录制缺失 duration，影响剪辑；改为 MP4。实际产物为 H.264，包含可读取的时长。
|
||||
- 预览/录制时 OrbitControls 仍可能约束相机距离；改为显式设置相机朝向，录制期间锁定交互，避免改变轨迹。
|
||||
- 从预览切换到录制时，React 更新可能重新显示轨迹辅助线；加入录制锁，确保辅助线不进入导出画面。
|
||||
|
||||
## 验收证据
|
||||
|
||||
浏览器使用生产组件进行本地独立验收；测试页仅将画布保存与对象存储替换成本地适配器，不伪造 WebGL、时间轴或编码结果。本轮没有调用收费 AI 模型，也未重新验证生产数据库/对象存储的整条链路；原画布持久化和真实万相链路见 2026-09-16 验收记录。
|
||||
|
||||
| 检查 | 结果 |
|
||||
|---|---|
|
||||
| 环绕、推进与时间轴 | 浏览器真实播放，循环后仍持续播放；暂停及定位到终点有效 |
|
||||
| 轨迹总览 | 相机切换到轨迹包围盒总览视角 |
|
||||
| 自定义 JSON 导入 | 3 秒、2 个关键帧，位置与 FOV 参数生效 |
|
||||
| 坏文件导入 | 显示无效数据提示，保留原轨迹 |
|
||||
| 暂停时选择场景对象 | 预览不跳回编辑机位；场景区域像素对比无变化 |
|
||||
| 关键帧更新 | 第二帧 FOV 改为 65° 并记录，完整刷新后恢复 |
|
||||
| 首帧传递 | 从 0 秒机位生成 1280×720 PNG，回调收到首帧及 2 帧轨迹 |
|
||||
| 横屏 MP4 | 1280×720，H.264，150 帧，4.998 秒，无音轨 |
|
||||
| 播放中切换导出 | 1280×720，90 帧，3.0005 秒；从轨迹起点完整导出，画面无辅助线 |
|
||||
| 竖屏 MP4 | 720×1280，H.264，90 帧，3.002 秒，无音轨 |
|
||||
| 逐帧检查 | 对比初始、中间、结尾帧，视角连续变化，非静态图片录制 |
|
||||
| 取消导出 | 浏览器中取消后恢复按钮与编辑状态，未出现成功产物 |
|
||||
| 录制生命周期 | 自动化验证正常结束、取消、后台事件及渲染失败均释放轨道与帧回调 |
|
||||
| 代码验证 | TypeScript 源码检查通过；49 条定向回归测试通过 |
|
||||
|
||||
浏览器自动化的多标签切换未可靠触发页面可见性事件，因此后台处理依靠事件级回归验证，不将该次标签切换算作真实后台场景通过。下载链接已接入原生下载；本轮媒体文件通过组件上传回调留档，并用 FFprobe 核验。
|
||||
|
||||
本地 `.qa/director-motion/` 留存测试 MP4、逐帧截图与探测结果，不提交生成文件和本地测试适配器。
|
||||
|
||||
## AI 生成与真实轨迹的边界
|
||||
|
||||
真实 3D 导出使用配置的轨迹。当前阿里云首帧模型接入接收首帧与运镜文字，不接收可精确执行的相机轨迹参数，因此 AI 生成仍可能偏离预设，界面已明确说明。需要精确复现时使用“导出真实运镜视频”。AI 识图重建仍是独立的未接通能力，不属于本次运镜修复。
|
||||
228
lib/video-canvas/director-motion.ts
Normal file
228
lib/video-canvas/director-motion.ts
Normal file
@ -0,0 +1,228 @@
|
||||
import type { DirectorCamera, DirectorVector } from "./director-scene";
|
||||
export type DirectorPose = Pick<DirectorCamera, "position" | "target" | "fov">;
|
||||
export function mergeDirectorCameraPose(
|
||||
camera: DirectorCamera,
|
||||
pose: Partial<DirectorPose>,
|
||||
): DirectorCamera {
|
||||
return {
|
||||
...camera,
|
||||
position: pose.position ?? camera.position,
|
||||
target: pose.target ?? camera.target,
|
||||
fov: pose.fov ?? camera.fov,
|
||||
};
|
||||
}
|
||||
export type DirectorKeyframe = DirectorPose & { id: string; time: number };
|
||||
export type DirectorMotion = {
|
||||
duration: number;
|
||||
easing: "linear" | "smooth";
|
||||
loop: boolean;
|
||||
keyframes: DirectorKeyframe[];
|
||||
};
|
||||
export const defaultDirectorMotion = (): DirectorMotion => ({
|
||||
duration: 5,
|
||||
easing: "smooth",
|
||||
loop: false,
|
||||
keyframes: [],
|
||||
});
|
||||
const vector = (v: unknown): v is DirectorVector =>
|
||||
Array.isArray(v) &&
|
||||
v.length === 3 &&
|
||||
v.every(
|
||||
(n) => typeof n === "number" && Number.isFinite(n) && Math.abs(n) <= 10000,
|
||||
);
|
||||
export function validDirectorPose(p: DirectorPose) {
|
||||
return (
|
||||
!!p &&
|
||||
vector(p.position) &&
|
||||
vector(p.target) &&
|
||||
Number.isFinite(p.fov) &&
|
||||
p.fov >= 15 &&
|
||||
p.fov <= 90 &&
|
||||
Math.hypot(...p.position.map((n, i) => n - p.target[i])) > 0.001
|
||||
);
|
||||
}
|
||||
export function readDirectorMotion(input: unknown): DirectorMotion {
|
||||
const v = (input || {}) as Partial<DirectorMotion>;
|
||||
const duration = Number.isFinite(v.duration)
|
||||
? Math.max(2, Math.min(15, Number(v.duration)))
|
||||
: 5;
|
||||
const seen = new Set<string>();
|
||||
const times = new Set<number>();
|
||||
const keyframes = (Array.isArray(v.keyframes) ? v.keyframes : [])
|
||||
.filter((k) => {
|
||||
if (
|
||||
!k ||
|
||||
typeof k.id !== "string" ||
|
||||
seen.has(k.id) ||
|
||||
!Number.isFinite(k.time) ||
|
||||
k.time < 0 ||
|
||||
k.time > duration ||
|
||||
times.has(k.time) ||
|
||||
!validDirectorPose(k)
|
||||
)
|
||||
return false;
|
||||
seen.add(k.id);
|
||||
times.add(k.time);
|
||||
return true;
|
||||
})
|
||||
.slice(0, 100)
|
||||
.map((k) => ({
|
||||
id: k.id,
|
||||
time: k.time,
|
||||
position: [...k.position] as DirectorVector,
|
||||
target: [...k.target] as DirectorVector,
|
||||
fov: k.fov,
|
||||
}))
|
||||
.sort((a, b) => a.time - b.time);
|
||||
return {
|
||||
duration,
|
||||
easing: v.easing === "linear" ? "linear" : "smooth",
|
||||
loop: v.loop === true,
|
||||
keyframes,
|
||||
};
|
||||
}
|
||||
const lerpVector = (
|
||||
a: DirectorVector,
|
||||
b: DirectorVector,
|
||||
t: number,
|
||||
): DirectorVector => a.map((n, i) => n + (b[i] - n) * t) as DirectorVector;
|
||||
export function sampleDirectorMotion(
|
||||
m: DirectorMotion,
|
||||
time: number,
|
||||
): DirectorPose | null {
|
||||
const keys = m.keyframes;
|
||||
if (!keys.length) return null;
|
||||
const t = Number.isFinite(time) ? Math.max(0, Math.min(m.duration, time)) : 0;
|
||||
const right = keys.findIndex((k) => k.time > t);
|
||||
const a = right < 0 ? keys[keys.length - 1] : keys[Math.max(0, right - 1)];
|
||||
const b = right < 0 ? a : keys[right];
|
||||
let u =
|
||||
a === b ? 0 : Math.max(0, Math.min(1, (t - a.time) / (b.time - a.time)));
|
||||
if (m.easing === "smooth") u = u * u * (3 - 2 * u);
|
||||
const pose = {
|
||||
position: lerpVector(a.position, b.position, u),
|
||||
target: lerpVector(a.target, b.target, u),
|
||||
fov: a.fov + (b.fov - a.fov) * u,
|
||||
};
|
||||
// A path crossing its look-at target has no defined viewing direction.
|
||||
if (!validDirectorPose(pose)) {
|
||||
const direction = a.position.map((n, i) => n - a.target[i]);
|
||||
const length = Math.hypot(...direction);
|
||||
pose.target = pose.position.map(
|
||||
(n, i) => n - (direction[i] / length) * 0.01,
|
||||
) as DirectorVector;
|
||||
}
|
||||
return pose;
|
||||
}
|
||||
export function resizeDirectorMotion(
|
||||
m: DirectorMotion,
|
||||
duration: number,
|
||||
): DirectorMotion {
|
||||
const next = Math.max(2, Math.min(15, duration));
|
||||
return {
|
||||
...m,
|
||||
duration: next,
|
||||
keyframes: m.keyframes.map((k) => ({
|
||||
...k,
|
||||
time: (k.time / m.duration) * next,
|
||||
})),
|
||||
};
|
||||
}
|
||||
export function presetDirectorMotion(
|
||||
pose: DirectorPose,
|
||||
kind: "push" | "pull" | "orbit" | "pan",
|
||||
duration = 5,
|
||||
): DirectorMotion {
|
||||
const m = { ...defaultDirectorMotion(), duration };
|
||||
const offset = pose.position.map(
|
||||
(n, i) => n - pose.target[i],
|
||||
) as DirectorVector;
|
||||
const count = kind === "orbit" ? 9 : 2;
|
||||
m.keyframes = Array.from({ length: count }, (_, i) => {
|
||||
const u = i / (count - 1);
|
||||
let position: DirectorVector = [...pose.position],
|
||||
target: DirectorVector = [...pose.target];
|
||||
if (kind === "push" || kind === "pull")
|
||||
position = pose.target.map(
|
||||
(n, j) => n + offset[j] * (1 + u * (kind === "push" ? -0.5 : 0.5)),
|
||||
) as DirectorVector;
|
||||
if (kind === "orbit") {
|
||||
const angle = (u * Math.PI) / 2;
|
||||
position = [
|
||||
pose.target[0] +
|
||||
offset[0] * Math.cos(angle) +
|
||||
offset[2] * Math.sin(angle),
|
||||
pose.position[1],
|
||||
pose.target[2] -
|
||||
offset[0] * Math.sin(angle) +
|
||||
offset[2] * Math.cos(angle),
|
||||
];
|
||||
}
|
||||
if (kind === "pan") {
|
||||
const length = Math.hypot(offset[0], offset[2]) || 1;
|
||||
const shift: DirectorVector = [
|
||||
(offset[2] / length) * 2 * u,
|
||||
0,
|
||||
(-offset[0] / length) * 2 * u,
|
||||
];
|
||||
position = position.map((n, j) => n + shift[j]) as DirectorVector;
|
||||
target = target.map((n, j) => n + shift[j]) as DirectorVector;
|
||||
}
|
||||
return {
|
||||
id: `${kind}-${i}`,
|
||||
time: u * duration,
|
||||
position,
|
||||
target,
|
||||
fov: pose.fov,
|
||||
};
|
||||
});
|
||||
if (kind === "orbit") m.easing = "linear";
|
||||
return m;
|
||||
}
|
||||
export function directorMotionPrompt(m: DirectorMotion) {
|
||||
if (m.keyframes.length < 2) return "";
|
||||
// Keep the model prompt compact; the full trajectory remains in the canvas data.
|
||||
const keys =
|
||||
m.keyframes.length <= 9
|
||||
? m.keyframes
|
||||
: Array.from({ length: 9 }, (_, i) => ({
|
||||
time: (m.duration * i) / 8,
|
||||
...sampleDirectorMotion(m, (m.duration * i) / 8)!,
|
||||
}));
|
||||
const lines = keys.slice(1).map((b, i) => {
|
||||
const a = keys[i],
|
||||
av = a.position.map((n, j) => n - a.target[j]),
|
||||
bv = b.position.map((n, j) => n - b.target[j]);
|
||||
const ar = Math.hypot(...av),
|
||||
br = Math.hypot(...bv),
|
||||
angle =
|
||||
(Math.acos(
|
||||
Math.max(
|
||||
-1,
|
||||
Math.min(
|
||||
1,
|
||||
av.reduce((sum, n, j) => sum + n * bv[j], 0) / (ar * br),
|
||||
),
|
||||
),
|
||||
) *
|
||||
180) /
|
||||
Math.PI;
|
||||
const descriptions: string[] = [];
|
||||
if (angle > 2)
|
||||
descriptions.push(`围绕注视主体转动约 ${Math.round(angle)} 度`);
|
||||
if (br < ar * 0.98) descriptions.push("向主体推进");
|
||||
else if (br > ar * 1.02) descriptions.push("远离主体拉远");
|
||||
if (Math.hypot(...b.target.map((n, j) => n - a.target[j])) > 0.05)
|
||||
descriptions.push("注视点随镜头平移");
|
||||
if (Math.abs(b.position[1] - a.position[1]) > 0.05)
|
||||
descriptions.push(
|
||||
b.position[1] > a.position[1] ? "相机升高" : "相机降低",
|
||||
);
|
||||
if (Math.abs(b.fov - a.fov) > 0.5)
|
||||
descriptions.push(
|
||||
`视角从 ${a.fov.toFixed(0)}° 变至 ${b.fov.toFixed(0)}°`,
|
||||
);
|
||||
return `${a.time.toFixed(2)}–${b.time.toFixed(2)}秒：${descriptions.join("，") || "保持机位和构图"}。`;
|
||||
});
|
||||
return `单一连续镜头，不切镜，总时长 ${m.duration} 秒；镜头${m.easing === "smooth" ? "平滑缓入缓出" : "匀速"}运动。\n${lines.join("\n")}\n保持参考图中的主体和场景一致，镜头连续运动。`;
|
||||
}
|
||||
95
lib/video-canvas/director-recording.ts
Normal file
95
lib/video-canvas/director-recording.ts
Normal file
@ -0,0 +1,95 @@
|
||||
/** Record the rendered camera path, never an AI approximation. */
|
||||
export async function recordDirectorVideo({
|
||||
canvas,
|
||||
duration,
|
||||
render,
|
||||
signal,
|
||||
onProgress,
|
||||
}: {
|
||||
canvas: HTMLCanvasElement;
|
||||
duration: number;
|
||||
render: (time: number) => void;
|
||||
signal: AbortSignal;
|
||||
onProgress?: (time: number) => void;
|
||||
}): Promise<Blob> {
|
||||
const mimeType = ["video/mp4"].find(
|
||||
(t) =>
|
||||
typeof MediaRecorder !== "undefined" && MediaRecorder.isTypeSupported(t),
|
||||
);
|
||||
if (!mimeType || !canvas.captureStream)
|
||||
throw Error("当前浏览器不支持 MP4 轨迹导出，请使用最新版 Chrome 或 Edge");
|
||||
if (signal.aborted || document.hidden)
|
||||
throw Error("请保持当前页面可见后导出");
|
||||
const stream = canvas.captureStream(30);
|
||||
try {
|
||||
return await new Promise<Blob>((resolve, reject) => {
|
||||
const recorder = new MediaRecorder(stream, {
|
||||
mimeType,
|
||||
videoBitsPerSecond: 6_000_000,
|
||||
});
|
||||
const chunks: Blob[] = [];
|
||||
let frame = 0,
|
||||
started = 0,
|
||||
failure: Error | undefined,
|
||||
finished = false;
|
||||
const cleanup = () => {
|
||||
cancelAnimationFrame(frame);
|
||||
clearTimeout(timeout);
|
||||
signal.removeEventListener("abort", cancel);
|
||||
document.removeEventListener("visibilitychange", visibility);
|
||||
};
|
||||
const stop = (error?: Error) => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
failure = error;
|
||||
cleanup();
|
||||
if (recorder.state !== "inactive") recorder.stop();
|
||||
else reject(error || Error("视频录制未启动"));
|
||||
};
|
||||
const cancel = () => stop(Error("已取消轨迹视频导出"));
|
||||
const visibility = () => {
|
||||
if (document.hidden)
|
||||
stop(Error("导出时切换了页面，请回到当前页面重新导出"));
|
||||
};
|
||||
const timeout = setTimeout(
|
||||
() => stop(Error("轨迹导出超时，请重试")),
|
||||
duration * 1000 + 15000,
|
||||
);
|
||||
recorder.ondataavailable = (e) => {
|
||||
if (e.data.size) chunks.push(e.data);
|
||||
};
|
||||
recorder.onerror = () => stop(Error("浏览器视频编码失败"));
|
||||
recorder.onstop = () => {
|
||||
cleanup();
|
||||
if (failure) reject(failure);
|
||||
else if (!chunks.length) reject(Error("未生成有效视频"));
|
||||
else resolve(new Blob(chunks, { type: recorder.mimeType }));
|
||||
};
|
||||
signal.addEventListener("abort", cancel, { once: true });
|
||||
document.addEventListener("visibilitychange", visibility);
|
||||
const tick = (now: number) => {
|
||||
if (finished) return;
|
||||
if (!started) started = now;
|
||||
const t = Math.min(duration, (now - started) / 1000);
|
||||
try {
|
||||
render(t);
|
||||
onProgress?.(t);
|
||||
} catch (e) {
|
||||
stop(e instanceof Error ? e : Error("轨迹渲染失败"));
|
||||
return;
|
||||
}
|
||||
if (t >= duration) stop();
|
||||
else frame = requestAnimationFrame(tick);
|
||||
};
|
||||
try {
|
||||
render(0);
|
||||
recorder.start(100);
|
||||
frame = requestAnimationFrame(tick);
|
||||
} catch (e) {
|
||||
stop(e instanceof Error ? e : Error("视频编码失败"));
|
||||
}
|
||||
});
|
||||
} finally {
|
||||
stream.getTracks().forEach((t) => t.stop());
|
||||
}
|
||||
}
|
||||
@ -1,3 +1,9 @@
|
||||
import {
|
||||
defaultDirectorMotion,
|
||||
readDirectorMotion,
|
||||
validDirectorPose,
|
||||
type DirectorMotion,
|
||||
} from "./director-motion";
|
||||
export type DirectorVector = [number, number, number];
|
||||
export type DirectorObject = {
|
||||
id: string;
|
||||
@ -29,6 +35,7 @@ export type DirectorScene = {
|
||||
activeCameraId: string;
|
||||
ratio: string;
|
||||
snapshots: DirectorSnapshot[];
|
||||
motion: DirectorMotion;
|
||||
};
|
||||
export const directorRatios = ["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"];
|
||||
export function directorImageSize(ratio: string) {
|
||||
@ -63,6 +70,7 @@ export function defaultDirectorScene(): DirectorScene {
|
||||
activeCameraId: "camera-1",
|
||||
ratio: "16:9",
|
||||
snapshots: [],
|
||||
motion: defaultDirectorMotion(),
|
||||
};
|
||||
}
|
||||
export function readDirectorScene(value: unknown): DirectorScene {
|
||||
@ -83,7 +91,8 @@ export function readDirectorScene(value: unknown): DirectorScene {
|
||||
vector(c.target) &&
|
||||
Number.isFinite(c.fov) &&
|
||||
c.fov >= 15 &&
|
||||
c.fov <= 90,
|
||||
c.fov <= 90 &&
|
||||
validDirectorPose(c),
|
||||
)
|
||||
: [];
|
||||
const objects = Array.isArray(v.objects)
|
||||
@ -120,5 +129,6 @@ export function readDirectorScene(value: unknown): DirectorScene {
|
||||
: validCameras[0].id,
|
||||
ratio: directorRatios.includes(v.ratio) ? v.ratio : "16:9",
|
||||
snapshots,
|
||||
motion: readDirectorMotion(v.motion),
|
||||
};
|
||||
}
|
||||
|
||||
@ -104,3 +104,221 @@ test("models stored on the active media CDN use the authenticated same-origin pr
|
||||
"/api/canvas/3d/model-proxy?url=" + encodeURIComponent(url),
|
||||
);
|
||||
});
|
||||
|
||||
import {
|
||||
defaultDirectorMotion,
|
||||
readDirectorMotion,
|
||||
sampleDirectorMotion,
|
||||
resizeDirectorMotion,
|
||||
presetDirectorMotion,
|
||||
directorMotionPrompt,
|
||||
validDirectorPose,
|
||||
} from "@/lib/video-canvas/director-motion";
|
||||
const camera = { position: [0, 2, 8], target: [0, 1, 0], fov: 50 } as const;
|
||||
const pose = () =>
|
||||
({
|
||||
position: [...camera.position],
|
||||
target: [...camera.target],
|
||||
fov: 50,
|
||||
}) as import("@/lib/video-canvas/director-motion").DirectorPose;
|
||||
test("legacy scenes acquire an empty timeline without altering cameras", () => {
|
||||
const { motion, ...legacy } = defaultDirectorScene();
|
||||
expect(readDirectorScene(legacy).motion).toEqual(defaultDirectorMotion());
|
||||
expect(readDirectorScene(legacy).cameras).toEqual(legacy.cameras);
|
||||
});
|
||||
test("motion survives serialization and duration changes preserve relative keyframe timing", () => {
|
||||
const scene = defaultDirectorScene();
|
||||
scene.motion = presetDirectorMotion(pose(), "orbit", 5);
|
||||
scene.motion.loop = true;
|
||||
expect(readDirectorScene(JSON.parse(JSON.stringify(scene)))).toEqual(scene);
|
||||
const resized = resizeDirectorMotion(scene.motion, 10);
|
||||
expect(resized.keyframes[4].time).toBe(5);
|
||||
expect(resized.keyframes.at(-1)?.time).toBe(10);
|
||||
expect(resized.keyframes[4].position).toEqual(
|
||||
scene.motion.keyframes[4].position,
|
||||
);
|
||||
});
|
||||
test("timeline sampling clamps endpoints, holds outside keyframes and differentiates easing", () => {
|
||||
const motion = presetDirectorMotion(pose(), "push", 5);
|
||||
motion.easing = "linear";
|
||||
expect(sampleDirectorMotion(motion, -20)?.position).toEqual(
|
||||
motion.keyframes[0].position,
|
||||
);
|
||||
expect(sampleDirectorMotion(motion, 30)?.position).toEqual(
|
||||
motion.keyframes[1].position,
|
||||
);
|
||||
expect(sampleDirectorMotion(motion, 2.5)?.position[2]).toBe(6);
|
||||
const linear = sampleDirectorMotion(motion, 1)?.position[2]!;
|
||||
motion.easing = "smooth";
|
||||
expect(sampleDirectorMotion(motion, 1)?.position[2]).toBeGreaterThan(linear);
|
||||
motion.keyframes[0].time = 1;
|
||||
motion.keyframes[1].time = 4;
|
||||
expect(sampleDirectorMotion(motion, 0)?.position).toEqual(
|
||||
motion.keyframes[0].position,
|
||||
);
|
||||
expect(sampleDirectorMotion(motion, 5)?.position).toEqual(
|
||||
motion.keyframes[1].position,
|
||||
);
|
||||
expect(sampleDirectorMotion(defaultDirectorMotion(), 0)).toBeNull();
|
||||
});
|
||||
test("invalid, repeated, out of range or coincident keyframes are rejected", () => {
|
||||
const m = presetDirectorMotion(pose(), "pull");
|
||||
const first = m.keyframes[0];
|
||||
m.keyframes.push(
|
||||
{ ...first, id: "duplicate-time" },
|
||||
{ ...first, id: "nan", time: NaN },
|
||||
{ ...first, id: "out", time: 16 },
|
||||
{ ...first, id: "broken", time: 2, position: [Infinity, 0, 1] },
|
||||
{ ...first, id: "coincident", time: 3, position: [...first.target] },
|
||||
);
|
||||
const read = readDirectorMotion(m);
|
||||
expect(read.keyframes).toHaveLength(2);
|
||||
expect(read.keyframes[0].id).toBe(first.id);
|
||||
});
|
||||
test("all motion presets remain finite and differ from their initial camera", () => {
|
||||
for (const kind of ["push", "pull", "orbit", "pan"] as const) {
|
||||
const m = presetDirectorMotion(pose(), kind, 5);
|
||||
for (let t = 0; t <= 5; t += 0.025)
|
||||
expect(validDirectorPose(sampleDirectorMotion(m, t)!)).toBe(true);
|
||||
expect(sampleDirectorMotion(m, 5)?.position).not.toEqual(
|
||||
sampleDirectorMotion(m, 0)?.position,
|
||||
);
|
||||
expect(directorMotionPrompt(m)).toContain("5 秒");
|
||||
expect(directorMotionPrompt(m)).toContain("5.00秒");
|
||||
}
|
||||
});
|
||||
test("look-at remains defined when opposing keyframes cross their target", () => {
|
||||
const m = presetDirectorMotion(pose(), "push");
|
||||
m.easing = "linear";
|
||||
m.keyframes[0].position = [-1, 0, 0];
|
||||
m.keyframes[0].target = [0, 0, 0];
|
||||
m.keyframes[1].position = [1, 0, 0];
|
||||
m.keyframes[1].target = [0, 0, 0];
|
||||
expect(validDirectorPose(sampleDirectorMotion(m, 2.5)!)).toBe(true);
|
||||
});
|
||||
|
||||
import { recordDirectorVideo } from "@/lib/video-canvas/director-recording";
|
||||
test("recording stops tracks and rejects partial exports on cancel or background switch", async () => {
|
||||
for (const outcome of [
|
||||
"complete",
|
||||
"cancel",
|
||||
"hidden",
|
||||
"render-error",
|
||||
] as const) {
|
||||
const saved = new Map<string, PropertyDescriptor | undefined>();
|
||||
const install = (key: string, value: unknown) => {
|
||||
saved.set(key, Object.getOwnPropertyDescriptor(globalThis, key));
|
||||
Object.defineProperty(globalThis, key, {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value,
|
||||
});
|
||||
};
|
||||
const doc = new EventTarget() as EventTarget & { hidden: boolean };
|
||||
doc.hidden = false;
|
||||
const callbacks = new Map<number, FrameRequestCallback>();
|
||||
let id = 0,
|
||||
stopped = 0;
|
||||
class Recorder {
|
||||
static isTypeSupported(type: string) {
|
||||
return type === "video/mp4";
|
||||
}
|
||||
state = "inactive";
|
||||
mimeType = "video/mp4";
|
||||
onstop?: () => void;
|
||||
ondataavailable?: (e: { data: Blob }) => void;
|
||||
start() {
|
||||
this.state = "recording";
|
||||
}
|
||||
stop() {
|
||||
this.state = "inactive";
|
||||
this.ondataavailable?.({ data: new Blob(["recorded"]) });
|
||||
queueMicrotask(() => this.onstop?.());
|
||||
}
|
||||
}
|
||||
install("document", doc);
|
||||
install("MediaRecorder", Recorder);
|
||||
install("requestAnimationFrame", (fn: FrameRequestCallback) => {
|
||||
callbacks.set(++id, fn);
|
||||
return id;
|
||||
});
|
||||
install("cancelAnimationFrame", (n: number) => callbacks.delete(n));
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const rendered: number[] = [];
|
||||
const canvas = {
|
||||
captureStream: () => ({ getTracks: () => [{ stop: () => stopped++ }] }),
|
||||
} as unknown as HTMLCanvasElement;
|
||||
const result = recordDirectorVideo({
|
||||
canvas,
|
||||
duration: 2,
|
||||
signal: controller.signal,
|
||||
render: (t) => {
|
||||
if (outcome === "render-error" && t > 0) throw Error("render fault");
|
||||
rendered.push(t);
|
||||
},
|
||||
});
|
||||
const step = (time: number) => {
|
||||
const pending = [...callbacks.values()];
|
||||
callbacks.clear();
|
||||
pending.forEach((fn) => fn(time));
|
||||
};
|
||||
if (outcome === "complete") {
|
||||
step(1000);
|
||||
step(3000);
|
||||
expect((await result).type).toBe("video/mp4");
|
||||
expect(rendered.at(-1)).toBe(2);
|
||||
} else {
|
||||
const rejected = expect(result).rejects.toThrow();
|
||||
if (outcome === "cancel") controller.abort();
|
||||
else if (outcome === "hidden") {
|
||||
doc.hidden = true;
|
||||
doc.dispatchEvent(new Event("visibilitychange"));
|
||||
} else {
|
||||
step(1000);
|
||||
step(3000);
|
||||
}
|
||||
await rejected;
|
||||
}
|
||||
expect(stopped).toBe(1);
|
||||
expect(callbacks.size).toBe(0);
|
||||
} finally {
|
||||
for (const [key, descriptor] of saved) {
|
||||
if (descriptor) Object.defineProperty(globalThis, key, descriptor);
|
||||
else Reflect.deleteProperty(globalThis, key);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
test("AI motion prompt describes the movement and stays bounded for large timelines", () => {
|
||||
expect(directorMotionPrompt(presetDirectorMotion(pose(), "push"))).toContain(
|
||||
"推进",
|
||||
);
|
||||
expect(directorMotionPrompt(presetDirectorMotion(pose(), "pull"))).toContain(
|
||||
"拉远",
|
||||
);
|
||||
expect(directorMotionPrompt(presetDirectorMotion(pose(), "orbit"))).toContain(
|
||||
"围绕",
|
||||
);
|
||||
expect(directorMotionPrompt(presetDirectorMotion(pose(), "pan"))).toContain(
|
||||
"平移",
|
||||
);
|
||||
const m = presetDirectorMotion(pose(), "orbit", 15);
|
||||
m.keyframes = Array.from({ length: 100 }, (_, i) => ({
|
||||
id: String(i),
|
||||
time: (i / 99) * 15,
|
||||
...sampleDirectorMotion(m, (i / 99) * 15)!,
|
||||
}));
|
||||
expect(directorMotionPrompt(m).length).toBeLessThan(1500);
|
||||
});
|
||||
|
||||
import { mergeDirectorCameraPose } from "@/lib/video-canvas/director-motion";
|
||||
test("editing a keyframe cannot replace the active camera identity", () => {
|
||||
const c = defaultDirectorScene().cameras[0];
|
||||
const key = presetDirectorMotion(pose(), "push").keyframes[1];
|
||||
const edited = mergeDirectorCameraPose(c, key);
|
||||
expect(edited.id).toBe(c.id);
|
||||
expect(edited.name).toBe(c.name);
|
||||
expect(edited.position).toEqual(key.position);
|
||||
expect(mergeDirectorCameraPose(edited, { fov: 65 }).fov).toBe(65);
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user