fix(video-canvas): connect generation controls and repair script media director workflows
This commit is contained in:
134
parent
e5b2de3c1b
commit
f10e6b7c8d
@ -1,42 +1,96 @@
|
||||
import {auth} from '@/app/(auth)/auth';
|
||||
import {NextResponse} from 'next/server';
|
||||
import {mkdtemp,readFile,rm} from 'node:fs/promises';
|
||||
import {tmpdir} from 'node:os';
|
||||
import {join} from 'node:path';
|
||||
import {randomUUID} from 'node:crypto';
|
||||
import {uploadFile} from '@/lib/storage/service';
|
||||
import {downloadCanvasMedia} from '@/lib/video-canvas/download-media';
|
||||
import {processVideoFile} from '@/lib/video-canvas/media-processing';
|
||||
import {videoProcessSchema} from '@/lib/video-canvas/media-contract';
|
||||
export const runtime='nodejs';
|
||||
export const maxDuration=300;
|
||||
const running=new Set<string>();
|
||||
export async function POST(request:Request) {
|
||||
const session=await auth();
|
||||
if(!session?.user?.id) return NextResponse.json({success:false,error:'未登录'},{status:401});
|
||||
const parsed=videoProcessSchema.safeParse(await request.json().catch(()=>null));
|
||||
if(!parsed.success) return NextResponse.json({success:false,error:'视频处理参数无效'},{status:400});
|
||||
if(running.has(session.user.id)||running.size>=2) return NextResponse.json({success:false,error:'视频处理中，请稍后再提交'},{status:429});
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { NextResponse } from "next/server";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
import { downloadCanvasMedia } from "@/lib/video-canvas/download-media";
|
||||
import { processVideoFile } from "@/lib/video-canvas/media-processing";
|
||||
import { videoProcessSchema } from "@/lib/video-canvas/media-contract";
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 300;
|
||||
const running = new Set<string>();
|
||||
export async function POST(request: Request) {
|
||||
const session = await auth();
|
||||
if (!session?.user?.id)
|
||||
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
|
||||
const parsed = videoProcessSchema.safeParse(await request.json().catch(() => null));
|
||||
if (!parsed.success)
|
||||
return NextResponse.json({ success: false, error: "视频处理参数无效" }, { status: 400 });
|
||||
if (running.has(session.user.id) || running.size >= 2)
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "视频处理中，请稍后再提交" },
|
||||
{ status: 429 },
|
||||
);
|
||||
running.add(session.user.id);
|
||||
let dir='';
|
||||
let dir = "";
|
||||
try {
|
||||
dir=await mkdtemp(join(tmpdir(),'canvas-video-'));
|
||||
const input=join(dir,'input');
|
||||
await downloadCanvasMedia(parsed.data.url,input);
|
||||
const save=async(name:string)=>{const buffer=await readFile(join(dir,name));return (await uploadFile(`canvas-video-${randomUUID()}-${name}`,new Uint8Array(buffer).buffer)).url;};
|
||||
if(parsed.data.operation==='trim') {
|
||||
const metadata=await processVideoFile(input,join(dir,'clip.mp4'),'trim',undefined,parsed.data.range);
|
||||
return NextResponse.json({success:true,data:{url:await save('clip.mp4'),...metadata}});
|
||||
dir = await mkdtemp(join(tmpdir(), "canvas-video-"));
|
||||
const input = join(dir, "input");
|
||||
await downloadCanvasMedia(parsed.data.url, input);
|
||||
const save = async (name: string) => {
|
||||
const buffer = await readFile(join(dir, name));
|
||||
return (
|
||||
await uploadFile(`canvas-video-${randomUUID()}-${name}`, new Uint8Array(buffer).buffer)
|
||||
).url;
|
||||
};
|
||||
if (parsed.data.operation === "frame" || parsed.data.operation === "slow") {
|
||||
const name = parsed.data.operation === "frame" ? "frame.png" : "slow.mp4";
|
||||
const metadata = await processVideoFile(
|
||||
input,
|
||||
join(dir, name),
|
||||
parsed.data.operation,
|
||||
undefined,
|
||||
undefined,
|
||||
parsed.data,
|
||||
);
|
||||
return NextResponse.json({ success: true, data: { url: await save(name), ...metadata } });
|
||||
}
|
||||
if(parsed.data.operation==='crop') {
|
||||
const metadata=await processVideoFile(input,join(dir,'crop.mp4'),'crop',parsed.data.rect);
|
||||
return NextResponse.json({success:true,data:{url:await save('crop.mp4'),...metadata}});
|
||||
if (parsed.data.operation === "trim") {
|
||||
const metadata = await processVideoFile(
|
||||
input,
|
||||
join(dir, "clip.mp4"),
|
||||
"trim",
|
||||
undefined,
|
||||
parsed.data.range,
|
||||
);
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: { url: await save("clip.mp4"), ...metadata },
|
||||
});
|
||||
}
|
||||
const metadata=await processVideoFile(input,join(dir,'audio.m4a'),'extract-audio');
|
||||
await processVideoFile(input,join(dir,'silent.mp4'),'silent-video');
|
||||
return NextResponse.json({success:true,data:{audioUrl:await save('audio.m4a'),url:await save('silent.mp4'),...metadata}});
|
||||
} catch(error) {
|
||||
const message=error instanceof Error ? error.message : '视频处理失败';
|
||||
return NextResponse.json({success:false,error:message.startsWith('Command failed')?'视频处理失败，请确认文件完整且格式受支持':message},{status:400});
|
||||
} finally {running.delete(session.user.id);if(dir)await rm(dir,{recursive:true,force:true});}
|
||||
if (parsed.data.operation === "crop") {
|
||||
const metadata = await processVideoFile(
|
||||
input,
|
||||
join(dir, "crop.mp4"),
|
||||
"crop",
|
||||
parsed.data.rect,
|
||||
);
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: { url: await save("crop.mp4"), ...metadata },
|
||||
});
|
||||
}
|
||||
const metadata = await processVideoFile(input, join(dir, "audio.m4a"), "extract-audio");
|
||||
await processVideoFile(input, join(dir, "silent.mp4"), "silent-video");
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: { audioUrl: await save("audio.m4a"), url: await save("silent.mp4"), ...metadata },
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "视频处理失败";
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: message.startsWith("Command failed")
|
||||
? "视频处理失败，请确认文件完整且格式受支持"
|
||||
: message,
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
} finally {
|
||||
running.delete(session.user.id);
|
||||
if (dir) await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -12,6 +12,7 @@ import {
|
||||
} from "@/lib/video-canvas/director-motion";
|
||||
export function DirectorMotionPanel({
|
||||
motion,
|
||||
compact = false,
|
||||
pose,
|
||||
disabled,
|
||||
onChange,
|
||||
@ -23,6 +24,7 @@ export function DirectorMotionPanel({
|
||||
onCancel,
|
||||
onOverview,
|
||||
}: {
|
||||
compact?: boolean;
|
||||
motion: DirectorMotion;
|
||||
pose: DirectorPose;
|
||||
disabled: boolean;
|
||||
@ -69,8 +71,7 @@ export function DirectorMotionPanel({
|
||||
const m = latest.current.motion;
|
||||
const next = timeRef.current + (previous ? (now - previous) / 1000 : 0);
|
||||
previous = now;
|
||||
const t =
|
||||
next >= m.duration ? (m.loop ? next % m.duration : m.duration) : next;
|
||||
const t = next >= m.duration ? (m.loop ? next % m.duration : m.duration) : next;
|
||||
timeRef.current = t;
|
||||
setTime(t);
|
||||
latest.current.onPreview(t);
|
||||
@ -107,9 +108,7 @@ export function DirectorMotionPanel({
|
||||
toast.error("相机不能与注视点重合");
|
||||
return;
|
||||
}
|
||||
const existing = motion.keyframes.find(
|
||||
(k) => Math.abs(k.time - time) < 0.01,
|
||||
);
|
||||
const existing = motion.keyframes.find((k) => Math.abs(k.time - time) < 0.01);
|
||||
if (!existing && motion.keyframes.length >= 100) {
|
||||
toast.error("最多支持 100 个关键帧");
|
||||
return;
|
||||
@ -141,12 +140,16 @@ export function DirectorMotionPanel({
|
||||
setTimeout(() => URL.revokeObjectURL(url), 1000);
|
||||
};
|
||||
return (
|
||||
<section
|
||||
className="space-y-3 border-t border-white/15 pt-3"
|
||||
aria-label="镜头轨迹"
|
||||
>
|
||||
<section className="space-y-3 border-t border-white/15 pt-3" aria-label="镜头轨迹">
|
||||
<div className="text-sm font-semibold">镜头轨迹</div>
|
||||
<fieldset disabled={disabled} className="space-y-3 disabled:opacity-50">
|
||||
<fieldset
|
||||
disabled={disabled}
|
||||
className={
|
||||
compact
|
||||
? "grid grid-cols-1 gap-3 sm:grid-cols-3 lg:grid-cols-4 disabled:opacity-50"
|
||||
: "space-y-3 disabled:opacity-50"
|
||||
}
|
||||
>
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
<label className="text-xs">
|
||||
时长（秒）
|
||||
@ -193,9 +196,7 @@ export function DirectorMotionPanel({
|
||||
key={kind}
|
||||
onClick={() => {
|
||||
const base =
|
||||
(preview
|
||||
? sampleDirectorMotion(motion, timeRef.current)
|
||||
: pose) || pose;
|
||||
(preview ? sampleDirectorMotion(motion, timeRef.current) : pose) || pose;
|
||||
if (!validDirectorPose(base)) {
|
||||
toast.error("相机不能与注视点重合");
|
||||
return;
|
||||
@ -278,25 +279,15 @@ export function DirectorMotionPanel({
|
||||
循环
|
||||
</label>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
className={button}
|
||||
disabled={preview}
|
||||
onClick={add}
|
||||
>
|
||||
<button type="button" className={button} disabled={preview} onClick={add}>
|
||||
记录/更新当前机位
|
||||
</button>
|
||||
{preview ? (
|
||||
<p className="text-xs text-amber-200">
|
||||
正在预览轨迹；点击“编辑机位”后可调整并记录机位。
|
||||
</p>
|
||||
<p className="text-xs text-amber-200">正在预览轨迹；点击“编辑机位”后可调整并记录机位。</p>
|
||||
) : null}
|
||||
<div className="max-h-48 space-y-1 overflow-y-auto">
|
||||
{motion.keyframes.map((k, i) => (
|
||||
<div
|
||||
key={k.id}
|
||||
className="flex items-center gap-1 rounded bg-white/5 p-1"
|
||||
>
|
||||
<div key={k.id} className="flex items-center gap-1 rounded bg-white/5 p-1">
|
||||
<button
|
||||
type="button"
|
||||
className={button}
|
||||
@ -319,12 +310,10 @@ export function DirectorMotionPanel({
|
||||
className="min-w-0 flex-1 rounded bg-white/10 p-1 text-xs"
|
||||
onChange={(e) => {
|
||||
const n = Number(e.target.value);
|
||||
if (n < 0 || n > motion.duration || !Number.isFinite(n))
|
||||
return;
|
||||
if (n < 0 || n > motion.duration || !Number.isFinite(n)) return;
|
||||
if (
|
||||
motion.keyframes.some(
|
||||
(other) =>
|
||||
other.id !== k.id && Math.abs(other.time - n) < 0.01,
|
||||
(other) => other.id !== k.id && Math.abs(other.time - n) < 0.01,
|
||||
)
|
||||
) {
|
||||
toast.error("关键帧时间不能重复");
|
||||
@ -333,9 +322,7 @@ export function DirectorMotionPanel({
|
||||
onChange({
|
||||
...motion,
|
||||
keyframes: motion.keyframes
|
||||
.map((other) =>
|
||||
other.id === k.id ? { ...k, time: n } : other,
|
||||
)
|
||||
.map((other) => (other.id === k.id ? { ...k, time: n } : other))
|
||||
.sort((a, b) => a.time - b.time),
|
||||
});
|
||||
}}
|
||||
@ -348,9 +335,7 @@ export function DirectorMotionPanel({
|
||||
edit();
|
||||
onChange({
|
||||
...motion,
|
||||
keyframes: motion.keyframes.filter(
|
||||
(other) => other.id !== k.id,
|
||||
),
|
||||
keyframes: motion.keyframes.filter((other) => other.id !== k.id),
|
||||
});
|
||||
}}
|
||||
>
|
||||
@ -363,11 +348,7 @@ export function DirectorMotionPanel({
|
||||
<button type="button" className={button} onClick={downloadJson}>
|
||||
保存轨迹文件
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className={button}
|
||||
onClick={() => input.current?.click()}
|
||||
>
|
||||
<button type="button" className={button} onClick={() => input.current?.click()}>
|
||||
导入轨迹文件
|
||||
</button>
|
||||
</div>
|
||||
@ -420,10 +401,7 @@ export function DirectorMotionPanel({
|
||||
const data = JSON.parse(await file.text());
|
||||
if (data.version !== 1) throw Error("不支持的轨迹文件版本");
|
||||
const m = readDirectorMotion(data.motion);
|
||||
if (
|
||||
m.keyframes.length < 2 ||
|
||||
m.keyframes.length !== data.motion?.keyframes?.length
|
||||
)
|
||||
if (m.keyframes.length < 2 || m.keyframes.length !== data.motion?.keyframes?.length)
|
||||
throw Error("轨迹数据无效或包含重复时间");
|
||||
edit();
|
||||
onChange(m);
|
||||
|
||||
@ -1,11 +1,9 @@
|
||||
"use client";
|
||||
import { directorPoses, type DirectorPoseName } from "@/lib/video-canvas/director-poses";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { Camera, Plus, Upload, X, Trash2 } from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
import {
|
||||
DirectorViewport,
|
||||
type DirectorViewportHandle,
|
||||
} from "./DirectorViewport";
|
||||
import { DirectorViewport, type DirectorViewportHandle } from "./DirectorViewport";
|
||||
import {
|
||||
readDirectorScene,
|
||||
directorRatios,
|
||||
@ -29,9 +27,7 @@ async function upload(file: File) {
|
||||
file.type.startsWith("video/") ? "canvas-video-process" : "director-studio",
|
||||
);
|
||||
const response = await fetch(
|
||||
/\.glb$/i.test(file.name)
|
||||
? "/api/canvas/director/upload-model"
|
||||
: "/api/files/upload",
|
||||
/\.glb$/i.test(file.name) ? "/api/canvas/director/upload-model" : "/api/files/upload",
|
||||
{
|
||||
method: "POST",
|
||||
body: form,
|
||||
@ -56,12 +52,7 @@ export function DirectorStudioDialog({
|
||||
onClose: () => void;
|
||||
onSend: (snapshots: DirectorSnapshot[]) => void;
|
||||
onSendMotion: (snapshot: DirectorSnapshot, motion: DirectorMotion) => void;
|
||||
onSendVideo: (video: {
|
||||
url: string;
|
||||
width: number;
|
||||
height: number;
|
||||
duration: number;
|
||||
}) => void;
|
||||
onSendVideo: (video: { url: string; width: number; height: number; duration: number }) => void;
|
||||
}) {
|
||||
const [scene, setScene] = useState(() => readDirectorScene(initialScene));
|
||||
const current = useRef(scene);
|
||||
@ -71,10 +62,9 @@ export function DirectorStudioDialog({
|
||||
setScene(next);
|
||||
onChange(next);
|
||||
};
|
||||
const [objectSearch, setObjectSearch] = useState("");
|
||||
const [selected, setSelected] = useState(scene.objects[0]?.id || "");
|
||||
const [mode, setMode] = useState<"translate" | "rotate" | "scale">(
|
||||
"translate",
|
||||
);
|
||||
const [mode, setMode] = useState<"translate" | "rotate" | "scale">("translate");
|
||||
const [cameraView, setCameraView] = useState(false);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const viewport = useRef<DirectorViewportHandle>(null);
|
||||
@ -104,10 +94,7 @@ export function DirectorStudioDialog({
|
||||
try {
|
||||
const motion = current.current.motion;
|
||||
const dimensions = directorImageSize(current.current.ratio);
|
||||
const blob = await viewport.current!.record(
|
||||
controller.signal,
|
||||
setExportProgress,
|
||||
);
|
||||
const blob = await viewport.current!.record(controller.signal, setExportProgress);
|
||||
setExportedVideo({
|
||||
blob,
|
||||
url: URL.createObjectURL(blob),
|
||||
@ -129,8 +116,7 @@ export function DirectorStudioDialog({
|
||||
try {
|
||||
const motion = current.current.motion;
|
||||
const pose = sampleDirectorMotion(motion, 0);
|
||||
if (!pose || motion.keyframes.length < 2)
|
||||
throw Error("至少需要两个关键帧");
|
||||
if (!pose || motion.keyframes.length < 2) throw Error("至少需要两个关键帧");
|
||||
const image = await viewport.current!.capture(pose);
|
||||
const url = await upload(
|
||||
new File([image.blob], `director-motion-${Date.now()}.png`, {
|
||||
@ -158,31 +144,21 @@ export function DirectorStudioDialog({
|
||||
}
|
||||
};
|
||||
const fileInput = useRef<HTMLInputElement>(null);
|
||||
const camera =
|
||||
scene.cameras.find((c) => c.id === scene.activeCameraId) ||
|
||||
scene.cameras[0];
|
||||
const camera = scene.cameras.find((c) => c.id === scene.activeCameraId) || scene.cameras[0];
|
||||
const object = scene.objects.find((o) => o.id === selected);
|
||||
const patchObject = (id: string, patch: Partial<DirectorObject>) =>
|
||||
commit({
|
||||
...current.current,
|
||||
objects: current.current.objects.map((o) =>
|
||||
o.id === id ? { ...o, ...patch } : o,
|
||||
),
|
||||
objects: current.current.objects.map((o) => (o.id === id ? { ...o, ...patch } : o)),
|
||||
});
|
||||
const patchCamera = (patch: Partial<typeof camera>) =>
|
||||
commit({
|
||||
...current.current,
|
||||
cameras: current.current.cameras.map((c) =>
|
||||
c.id === current.current.activeCameraId
|
||||
? mergeDirectorCameraPose(c, patch)
|
||||
: c,
|
||||
c.id === current.current.activeCameraId ? mergeDirectorCameraPose(c, patch) : c,
|
||||
),
|
||||
});
|
||||
const addObject = (
|
||||
kind: DirectorObject["kind"],
|
||||
name: string,
|
||||
url?: string,
|
||||
) => {
|
||||
const addObject = (kind: DirectorObject["kind"], name: string, url?: string) => {
|
||||
const id = crypto.randomUUID();
|
||||
commit({
|
||||
...current.current,
|
||||
@ -211,9 +187,7 @@ export function DirectorStudioDialog({
|
||||
type: "image/png",
|
||||
}),
|
||||
);
|
||||
const active = current.current.cameras.find(
|
||||
(c) => c.id === current.current.activeCameraId,
|
||||
)!;
|
||||
const active = current.current.cameras.find((c) => c.id === current.current.activeCameraId)!;
|
||||
const snapshot = {
|
||||
id: crypto.randomUUID(),
|
||||
title: `${active.name}-截图${current.current.snapshots.length + 1}`,
|
||||
@ -235,7 +209,7 @@ export function DirectorStudioDialog({
|
||||
};
|
||||
const field = "w-full rounded bg-white/10 px-2 py-1.5 text-sm outline-none";
|
||||
const button =
|
||||
"rounded-lg border border-white/10 px-3 py-2 text-sm hover:bg-white/10 disabled:opacity-40";
|
||||
"whitespace-nowrap rounded-lg border border-white/10 px-2 py-2 text-sm hover:bg-white/10 disabled:opacity-40";
|
||||
return (
|
||||
<div
|
||||
role="dialog"
|
||||
@ -276,54 +250,43 @@ export function DirectorStudioDialog({
|
||||
</button>
|
||||
</header>
|
||||
<div className="grid min-h-0 flex-1 grid-cols-[210px_minmax(0,1fr)_260px]">
|
||||
<aside
|
||||
inert={busy}
|
||||
className="overflow-y-auto border-r border-white/10 p-4"
|
||||
>
|
||||
<aside inert={busy} className="overflow-y-auto border-r border-white/10 p-4">
|
||||
<div className="mb-3 text-sm text-white/60">场景对象</div>
|
||||
<input
|
||||
aria-label="搜索场景对象"
|
||||
className={`${field} mb-3`}
|
||||
placeholder="搜索场景对象"
|
||||
value={objectSearch}
|
||||
onChange={(e) => setObjectSearch(e.target.value)}
|
||||
/>
|
||||
<div className="space-y-2">
|
||||
{scene.objects.map((o) => (
|
||||
<button
|
||||
className={`${button} w-full text-left ${selected === o.id ? "bg-white/15" : ""}`}
|
||||
key={o.id}
|
||||
onClick={() => setSelected(o.id)}
|
||||
>
|
||||
{o.name}
|
||||
</button>
|
||||
))}
|
||||
{scene.objects
|
||||
.filter((o) => o.name.toLowerCase().includes(objectSearch.trim().toLowerCase()))
|
||||
.map((o) => (
|
||||
<button
|
||||
className={`${button} w-full text-left ${selected === o.id ? "bg-white/15" : ""}`}
|
||||
key={o.id}
|
||||
onClick={() => setSelected(o.id)}
|
||||
>
|
||||
{o.name}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<div className="mt-5 flex flex-col gap-2">
|
||||
<button
|
||||
className={button}
|
||||
onClick={() => addObject("actor", "基础人偶")}
|
||||
>
|
||||
<button className={button} onClick={() => addObject("actor", "基础人偶")}>
|
||||
添加人偶
|
||||
</button>
|
||||
<button
|
||||
className={button}
|
||||
onClick={() => addObject("box", "立方体")}
|
||||
>
|
||||
<button className={button} onClick={() => addObject("box", "立方体")}>
|
||||
添加立方体
|
||||
</button>
|
||||
<button
|
||||
className={button}
|
||||
onClick={() => addObject("sphere", "球体")}
|
||||
>
|
||||
<button className={button} onClick={() => addObject("sphere", "球体")}>
|
||||
添加球体
|
||||
</button>
|
||||
<button
|
||||
className={button}
|
||||
disabled={busy}
|
||||
onClick={() => fileInput.current?.click()}
|
||||
>
|
||||
<button className={button} disabled={busy} onClick={() => fileInput.current?.click()}>
|
||||
<Upload size={14} className="mr-2 inline" />
|
||||
上传 GLB 模型
|
||||
</button>
|
||||
<button
|
||||
className={button}
|
||||
disabled
|
||||
title="尚未接入图片识别与场景重建"
|
||||
>
|
||||
<button className={button} disabled title="尚未接入图片识别与场景重建">
|
||||
AI 识图导入（未接通）
|
||||
</button>
|
||||
</div>
|
||||
@ -368,19 +331,14 @@ export function DirectorStudioDialog({
|
||||
</div>
|
||||
</main>
|
||||
<aside className="space-y-4 overflow-y-auto border-l border-white/10 p-4">
|
||||
<fieldset
|
||||
disabled={busy || previewTime !== null}
|
||||
className="space-y-4"
|
||||
>
|
||||
<fieldset disabled={busy || previewTime !== null} className="space-y-4">
|
||||
<label className="block text-sm">
|
||||
机位
|
||||
<select
|
||||
aria-label="机位"
|
||||
className={`${field} mt-2`}
|
||||
value={scene.activeCameraId}
|
||||
onChange={(e) =>
|
||||
commit({ ...current.current, activeCameraId: e.target.value })
|
||||
}
|
||||
onChange={(e) => commit({ ...current.current, activeCameraId: e.target.value })}
|
||||
>
|
||||
{scene.cameras.map((c) => (
|
||||
<option key={c.id} value={c.id}>
|
||||
@ -394,6 +352,15 @@ export function DirectorStudioDialog({
|
||||
["正面", [0, 2, 7]],
|
||||
["侧面", [7, 2, 0]],
|
||||
["俯视", [0, 8, 0.1]],
|
||||
["正面特写", [0, 1.7, 2]],
|
||||
["正面全景", [0, 2, 10]],
|
||||
["侧面近景", [2, 1.5, 0]],
|
||||
["背面", [0, 2, -7]],
|
||||
["45°俯拍", [4, 6, 4]],
|
||||
["低机位", [0, 0.3, 4]],
|
||||
["左过肩", [-1, 1.8, -2]],
|
||||
["右过肩", [1, 1.8, -2]],
|
||||
["鸟瞰", [0, 14, 0.1]],
|
||||
].map(([name, position]) => (
|
||||
<button
|
||||
className={button}
|
||||
@ -429,7 +396,7 @@ export function DirectorStudioDialog({
|
||||
type="number"
|
||||
min={15}
|
||||
max={90}
|
||||
value={camera.fov}
|
||||
value={Number(camera.fov.toFixed(2))}
|
||||
onChange={(e) => {
|
||||
const n = Number(e.target.value);
|
||||
if (n >= 15 && n <= 90) patchCamera({ fov: n });
|
||||
@ -442,9 +409,7 @@ export function DirectorStudioDialog({
|
||||
aria-label="截图比例"
|
||||
className={`${field} mt-2`}
|
||||
value={scene.ratio}
|
||||
onChange={(e) =>
|
||||
commit({ ...current.current, ratio: e.target.value })
|
||||
}
|
||||
onChange={(e) => commit({ ...current.current, ratio: e.target.value })}
|
||||
>
|
||||
{directorRatios.map((r) => (
|
||||
<option key={r}>{r}</option>
|
||||
@ -478,15 +443,19 @@ export function DirectorStudioDialog({
|
||||
{object ? (
|
||||
<section className="space-y-2 border-t border-white/10 pt-3">
|
||||
<div className="flex items-center justify-between text-sm">
|
||||
<span>{object.name}</span>
|
||||
<input
|
||||
aria-label="对象名称"
|
||||
className={field}
|
||||
value={object.name}
|
||||
maxLength={80}
|
||||
onChange={(e) => patchObject(object.id, { name: e.target.value })}
|
||||
/>
|
||||
<button
|
||||
aria-label="删除所选场景对象"
|
||||
onClick={() => {
|
||||
commit({
|
||||
...current.current,
|
||||
objects: current.current.objects.filter(
|
||||
(o) => o.id !== object.id,
|
||||
),
|
||||
objects: current.current.objects.filter((o) => o.id !== object.id),
|
||||
});
|
||||
setSelected("");
|
||||
}}
|
||||
@ -494,10 +463,104 @@ export function DirectorStudioDialog({
|
||||
<Trash2 size={14} />
|
||||
</button>
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-3 text-xs">
|
||||
<label>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={object.visible !== false}
|
||||
onChange={(e) => patchObject(object.id, { visible: e.target.checked })}
|
||||
/>{" "}
|
||||
显示
|
||||
</label>
|
||||
<label>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={object.locked === true}
|
||||
onChange={(e) => patchObject(object.id, { locked: e.target.checked })}
|
||||
/>{" "}
|
||||
锁定变换轴
|
||||
</label>
|
||||
{object.kind !== "model" && (
|
||||
<label>
|
||||
颜色{" "}
|
||||
<input
|
||||
aria-label="对象颜色"
|
||||
type="color"
|
||||
value={object.color || (object.kind === "actor" ? "#687bb0" : "#c3664c")}
|
||||
onChange={(e) => patchObject(object.id, { color: e.target.value })}
|
||||
/>
|
||||
</label>
|
||||
)}
|
||||
</div>
|
||||
{object.kind === "actor" && (
|
||||
<label className="block text-xs">
|
||||
人偶姿势
|
||||
<select
|
||||
aria-label="人偶姿势"
|
||||
className={`${field} mt-1`}
|
||||
value={object.pose || "站立"}
|
||||
onChange={(e) =>
|
||||
patchObject(object.id, { pose: e.target.value as DirectorPoseName })
|
||||
}
|
||||
>
|
||||
{Object.keys(directorPoses).map((name) => (
|
||||
<option key={name}>{name}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
)}
|
||||
<button
|
||||
className={button}
|
||||
onClick={() => {
|
||||
const copy = {
|
||||
...object,
|
||||
id: crypto.randomUUID(),
|
||||
name: `${object.name} 副本`,
|
||||
position: [
|
||||
object.position[0] + 1,
|
||||
...object.position.slice(1),
|
||||
] as DirectorVector,
|
||||
};
|
||||
commit({ ...current.current, objects: [...current.current.objects, copy] });
|
||||
setSelected(copy.id);
|
||||
}}
|
||||
>
|
||||
复制对象
|
||||
</button>
|
||||
<button
|
||||
className={button}
|
||||
onClick={() =>
|
||||
patchCamera({
|
||||
target: [object.position[0], object.position[1] + 1, object.position[2]],
|
||||
})
|
||||
}
|
||||
>
|
||||
相机对准所选对象
|
||||
</button>
|
||||
<label className="block text-xs">
|
||||
等比缩放
|
||||
<input
|
||||
aria-label="等比缩放"
|
||||
className={`${field} mt-1`}
|
||||
type="number"
|
||||
step="0.1"
|
||||
min="0.01"
|
||||
value={Number(object.scale[0].toFixed(2))}
|
||||
onChange={(e) => {
|
||||
const n = Number(e.target.value);
|
||||
if (n > 0 && Number.isFinite(n))
|
||||
patchObject(object.id, {
|
||||
scale: object.scale.map(
|
||||
(v) => (v * n) / object.scale[0],
|
||||
) as DirectorVector,
|
||||
});
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
{(["position", "rotation", "scale"] as const).map((key, j) => (
|
||||
<div key={key}>
|
||||
<div className="mb-1 text-xs text-white/50">
|
||||
{["位置", "旋转（弧度）", "缩放"][j]}
|
||||
{["位置", "旋转（度）", "缩放"][j]}
|
||||
</div>
|
||||
<div className="grid grid-cols-3 gap-1">
|
||||
{object[key].map((v, i) => (
|
||||
@ -507,16 +570,12 @@ export function DirectorStudioDialog({
|
||||
className={field}
|
||||
type="number"
|
||||
step="0.1"
|
||||
value={Number(v.toFixed(2))}
|
||||
value={Number((key === "rotation" ? (v * 180) / Math.PI : v).toFixed(2))}
|
||||
onChange={(e) => {
|
||||
const n = Number(e.target.value);
|
||||
if (
|
||||
!Number.isFinite(n) ||
|
||||
(key === "scale" && n <= 0)
|
||||
)
|
||||
return;
|
||||
if (!Number.isFinite(n) || (key === "scale" && n <= 0)) return;
|
||||
const values = [...object[key]] as DirectorVector;
|
||||
values[i] = n;
|
||||
values[i] = key === "rotation" ? (n * Math.PI) / 180 : n;
|
||||
patchObject(object.id, { [key]: values });
|
||||
}}
|
||||
/>
|
||||
@ -527,27 +586,8 @@ export function DirectorStudioDialog({
|
||||
</section>
|
||||
) : null}
|
||||
</fieldset>
|
||||
<DirectorMotionPanel
|
||||
motion={scene.motion}
|
||||
pose={camera}
|
||||
disabled={busy}
|
||||
onChange={(motion) => commit({ ...current.current, motion })}
|
||||
onPose={patchCamera}
|
||||
onPreview={setPreviewTime}
|
||||
onExport={exportMotion}
|
||||
onSend={sendMotion}
|
||||
exportProgress={exportProgress}
|
||||
onCancel={() => recording.current?.abort()}
|
||||
onOverview={() => {
|
||||
setCameraView(false);
|
||||
viewport.current?.showPath();
|
||||
}}
|
||||
/>
|
||||
{exportedVideo ? (
|
||||
<section
|
||||
className="space-y-2 border-t border-white/10 pt-3"
|
||||
aria-label="运镜导出结果"
|
||||
>
|
||||
<section className="space-y-2 border-t border-white/10 pt-3" aria-label="运镜导出结果">
|
||||
<video
|
||||
controls
|
||||
src={exportedVideo.url}
|
||||
@ -567,15 +607,11 @@ export function DirectorStudioDialog({
|
||||
onClick={async () => {
|
||||
setBusy(true);
|
||||
try {
|
||||
const ext = exportedVideo.blob.type.includes("mp4")
|
||||
? "mp4"
|
||||
: "webm";
|
||||
const ext = exportedVideo.blob.type.includes("mp4") ? "mp4" : "webm";
|
||||
const url = await upload(
|
||||
new File(
|
||||
[exportedVideo.blob],
|
||||
`director-motion-${Date.now()}.${ext}`,
|
||||
{ type: exportedVideo.blob.type.split(";")[0] },
|
||||
),
|
||||
new File([exportedVideo.blob], `director-motion-${Date.now()}.${ext}`, {
|
||||
type: exportedVideo.blob.type.split(";")[0],
|
||||
}),
|
||||
);
|
||||
onSendVideo({ ...exportedVideo, url });
|
||||
toast.success("运镜视频已发送到画布");
|
||||
@ -612,6 +648,29 @@ export function DirectorStudioDialog({
|
||||
</section>
|
||||
</aside>
|
||||
</div>
|
||||
<details
|
||||
open
|
||||
className="max-h-[32vh] shrink-0 overflow-auto border-t border-white/15 bg-[#1c1c1c] px-4 pb-3"
|
||||
>
|
||||
<summary className="cursor-pointer py-2 text-sm">相机时间轴与运镜</summary>
|
||||
<DirectorMotionPanel
|
||||
compact
|
||||
motion={scene.motion}
|
||||
pose={camera}
|
||||
disabled={busy}
|
||||
onChange={(motion) => commit({ ...current.current, motion })}
|
||||
onPose={patchCamera}
|
||||
onPreview={setPreviewTime}
|
||||
onExport={exportMotion}
|
||||
onSend={sendMotion}
|
||||
exportProgress={exportProgress}
|
||||
onCancel={() => recording.current?.abort()}
|
||||
onOverview={() => {
|
||||
setCameraView(false);
|
||||
viewport.current?.showPath();
|
||||
}}
|
||||
/>
|
||||
</details>
|
||||
<input
|
||||
ref={fileInput}
|
||||
className="hidden"
|
||||
@ -628,10 +687,7 @@ export function DirectorStudioDialog({
|
||||
setBusy(true);
|
||||
try {
|
||||
const header = new DataView(await file.slice(0, 12).arrayBuffer());
|
||||
if (
|
||||
header.byteLength < 12 ||
|
||||
header.getUint32(0, true) !== 0x46546c67
|
||||
)
|
||||
if (header.byteLength < 12 || header.getUint32(0, true) !== 0x46546c67)
|
||||
throw Error("文件不是有效 GLB 模型");
|
||||
const url = await upload(file);
|
||||
addObject("model", file.name, url);
|
||||
|
||||
@ -1,11 +1,6 @@
|
||||
"use client";
|
||||
import {
|
||||
forwardRef,
|
||||
useEffect,
|
||||
useImperativeHandle,
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import { directorPoses } from "@/lib/video-canvas/director-poses";
|
||||
import { forwardRef, useEffect, useImperativeHandle, useRef, useState } from "react";
|
||||
import * as THREE from "three";
|
||||
import { buildThreeAssetUrl } from "@/lib/canvas/three-asset-url";
|
||||
import { OrbitControls } from "three/examples/jsm/controls/OrbitControls.js";
|
||||
@ -16,20 +11,12 @@ import {
|
||||
type DirectorScene,
|
||||
type DirectorObject,
|
||||
} from "@/lib/video-canvas/director-scene";
|
||||
import {
|
||||
sampleDirectorMotion,
|
||||
type DirectorPose,
|
||||
} from "@/lib/video-canvas/director-motion";
|
||||
import { sampleDirectorMotion, type DirectorPose } from "@/lib/video-canvas/director-motion";
|
||||
import { recordDirectorVideo } from "@/lib/video-canvas/director-recording";
|
||||
export type DirectorViewportHandle = {
|
||||
showPath: () => void;
|
||||
record: (
|
||||
signal: AbortSignal,
|
||||
onProgress: (time: number) => void,
|
||||
) => Promise<Blob>;
|
||||
capture: (
|
||||
pose?: DirectorPose,
|
||||
) => Promise<{ blob: Blob; width: number; height: number }>;
|
||||
record: (signal: AbortSignal, onProgress: (time: number) => void) => Promise<Blob>;
|
||||
capture: (pose?: DirectorPose) => Promise<{ blob: Blob; width: number; height: number }>;
|
||||
};
|
||||
type Props = {
|
||||
scene: DirectorScene;
|
||||
@ -39,18 +26,14 @@ type Props = {
|
||||
previewTime: number | null;
|
||||
locked: boolean;
|
||||
onTransform: (id: string, patch: Partial<DirectorObject>) => void;
|
||||
onCamera: (
|
||||
position: [number, number, number],
|
||||
target: [number, number, number],
|
||||
) => void;
|
||||
onCamera: (position: [number, number, number], target: [number, number, number]) => void;
|
||||
};
|
||||
function dispose(root: THREE.Object3D) {
|
||||
root.traverse((o) => {
|
||||
if (o instanceof THREE.Mesh || o instanceof THREE.Line) {
|
||||
o.geometry.dispose();
|
||||
for (const m of Array.isArray(o.material) ? o.material : [o.material]) {
|
||||
for (const value of Object.values(m))
|
||||
if (value instanceof THREE.Texture) value.dispose();
|
||||
for (const value of Object.values(m)) if (value instanceof THREE.Texture) value.dispose();
|
||||
m.dispose();
|
||||
}
|
||||
}
|
||||
@ -130,9 +113,7 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
world.add(objects);
|
||||
transform.addEventListener("dragging-changed", (e) => {
|
||||
orbit.enabled =
|
||||
!e.value &&
|
||||
!latest.current.cameraView &&
|
||||
latest.current.previewTime === null;
|
||||
!e.value && !latest.current.cameraView && latest.current.previewTime === null;
|
||||
});
|
||||
transform.addEventListener("mouseUp", () => {
|
||||
const o = transform.object;
|
||||
@ -144,12 +125,8 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
});
|
||||
});
|
||||
orbit.addEventListener("end", () => {
|
||||
if (latest.current.previewTime !== null || latest.current.cameraView)
|
||||
return;
|
||||
latest.current.onCamera(
|
||||
camera.position.toArray(),
|
||||
orbit.target.toArray(),
|
||||
);
|
||||
if (latest.current.previewTime !== null || latest.current.cameraView) return;
|
||||
latest.current.onCamera(camera.position.toArray(), orbit.target.toArray());
|
||||
});
|
||||
|
||||
const resize = () => {
|
||||
@ -200,9 +177,12 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
if (!r || !ready) return;
|
||||
let active = true;
|
||||
r.transform.detach();
|
||||
const desired = new Set(props.scene.objects.map((o) => o.id));
|
||||
r.objects.children.slice().forEach((o) => {
|
||||
dispose(o);
|
||||
r.objects.remove(o);
|
||||
if (!desired.has(o.name)) {
|
||||
dispose(o);
|
||||
r.objects.remove(o);
|
||||
}
|
||||
});
|
||||
setError("");
|
||||
r.pending = 0;
|
||||
@ -212,18 +192,40 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
return;
|
||||
}
|
||||
object.name = item.id;
|
||||
object.visible = item.visible !== false;
|
||||
object.userData.directorSignature = JSON.stringify([
|
||||
item.kind,
|
||||
item.url,
|
||||
item.pose,
|
||||
item.color,
|
||||
]);
|
||||
object.position.fromArray(item.position);
|
||||
object.rotation.set(...item.rotation);
|
||||
object.scale.fromArray(item.scale);
|
||||
r.objects.add(object);
|
||||
if (
|
||||
item.id === latest.current.selected &&
|
||||
!item.locked &&
|
||||
item.visible !== false &&
|
||||
!latest.current.cameraView &&
|
||||
latest.current.previewTime === null
|
||||
)
|
||||
r.transform.attach(object);
|
||||
};
|
||||
for (const item of props.scene.objects) {
|
||||
const existing = r.objects.children.find((o) => o.name === item.id);
|
||||
if (
|
||||
existing &&
|
||||
existing.userData.directorSignature ===
|
||||
JSON.stringify([item.kind, item.url, item.pose, item.color])
|
||||
) {
|
||||
attach(existing, item);
|
||||
continue;
|
||||
}
|
||||
if (existing) {
|
||||
dispose(existing);
|
||||
r.objects.remove(existing);
|
||||
}
|
||||
if (item.kind === "model" && item.url) {
|
||||
r.pending++;
|
||||
new GLTFLoader().load(
|
||||
@ -256,31 +258,69 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
continue;
|
||||
}
|
||||
const mat = new THREE.MeshStandardMaterial({
|
||||
color: item.kind === "actor" ? 0x687bb0 : 0xc3664c,
|
||||
color: item.color || (item.kind === "actor" ? "#687bb0" : "#c3664c"),
|
||||
roughness: 0.6,
|
||||
});
|
||||
let object: THREE.Object3D;
|
||||
if (item.kind === "actor") {
|
||||
const group = new THREE.Group();
|
||||
const body = new THREE.Mesh(
|
||||
new THREE.CapsuleGeometry(0.32, 0.7, 6, 16),
|
||||
mat,
|
||||
);
|
||||
body.position.y = 1.05;
|
||||
group.add(body);
|
||||
const head = new THREE.Mesh(
|
||||
new THREE.SphereGeometry(0.24, 24, 16),
|
||||
mat.clone(),
|
||||
);
|
||||
head.position.y = 1.95;
|
||||
group.add(head);
|
||||
for (const x of [-0.18, 0.18]) {
|
||||
const leg = new THREE.Mesh(
|
||||
new THREE.CapsuleGeometry(0.12, 0.55, 4, 12),
|
||||
const pose = directorPoses[item.pose || "站立"] || directorPoses["站立"];
|
||||
const radians = THREE.MathUtils.degToRad;
|
||||
const hips = new THREE.Group();
|
||||
hips.position.y = 1 + ("height" in pose ? pose.height : 0);
|
||||
group.add(hips);
|
||||
const torso = new THREE.Group();
|
||||
torso.rotation.x = radians(pose.lean);
|
||||
hips.add(torso);
|
||||
const body = new THREE.Mesh(new THREE.BoxGeometry(0.52, 0.65, 0.28), mat);
|
||||
body.position.y = 0.38;
|
||||
torso.add(body);
|
||||
const head = new THREE.Mesh(new THREE.SphereGeometry(0.2, 20, 16), mat.clone());
|
||||
head.position.y = 0.95;
|
||||
torso.add(head);
|
||||
const limb = (
|
||||
parent: THREE.Object3D,
|
||||
position: [number, number, number],
|
||||
angle: number,
|
||||
bend: number,
|
||||
length: number,
|
||||
radius: number,
|
||||
spread = 0,
|
||||
) => {
|
||||
const root = new THREE.Group();
|
||||
root.position.set(...position);
|
||||
root.rotation.x = radians(angle);
|
||||
root.rotation.z = radians(spread);
|
||||
parent.add(root);
|
||||
const upper = new THREE.Mesh(
|
||||
new THREE.CapsuleGeometry(radius, length - radius * 2, 4, 12),
|
||||
mat.clone(),
|
||||
);
|
||||
leg.position.set(x, 0.42, 0);
|
||||
group.add(leg);
|
||||
upper.position.y = -length / 2;
|
||||
root.add(upper);
|
||||
const joint = new THREE.Group();
|
||||
joint.position.y = -length;
|
||||
joint.rotation.x = radians(bend);
|
||||
root.add(joint);
|
||||
const lower = new THREE.Mesh(
|
||||
new THREE.CapsuleGeometry(radius * 0.85, length - radius * 2, 4, 12),
|
||||
mat.clone(),
|
||||
);
|
||||
lower.position.y = -length / 2;
|
||||
joint.add(lower);
|
||||
};
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const side = i === 0 ? -1 : 1;
|
||||
limb(
|
||||
torso,
|
||||
[side * 0.33, 0.65, 0],
|
||||
pose.arms[i],
|
||||
-pose.elbows[i],
|
||||
0.34,
|
||||
0.07,
|
||||
("spread" in pose ? pose.spread : 5) * side,
|
||||
);
|
||||
limb(hips, [side * 0.15, 0, 0], pose.legs[i], pose.knees[i], 0.46, 0.1);
|
||||
}
|
||||
object = group;
|
||||
} else {
|
||||
@ -319,10 +359,17 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
r.orbit.enabled = !props.cameraView && props.previewTime === null;
|
||||
r.transform.setMode(props.mode);
|
||||
const selected = r.objects.getObjectByName(props.selected);
|
||||
if (selected && !props.cameraView && props.previewTime === null)
|
||||
if (
|
||||
selected &&
|
||||
selected.visible &&
|
||||
!props.scene.objects.find((o) => o.id === props.selected)?.locked &&
|
||||
!props.cameraView &&
|
||||
props.previewTime === null
|
||||
)
|
||||
r.transform.attach(selected);
|
||||
else r.transform.detach();
|
||||
}, [
|
||||
props.scene.objects,
|
||||
props.scene.cameras,
|
||||
props.scene.activeCameraId,
|
||||
props.selected,
|
||||
@ -342,10 +389,7 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
const points = Array.from(
|
||||
{ length: 121 },
|
||||
(_, i) =>
|
||||
new THREE.Vector3(
|
||||
...sampleDirectorMotion(motion, (motion.duration * i) / 120)!
|
||||
.position,
|
||||
),
|
||||
new THREE.Vector3(...sampleDirectorMotion(motion, (motion.duration * i) / 120)!.position),
|
||||
);
|
||||
r.path.add(
|
||||
new THREE.Line(
|
||||
@ -365,8 +409,7 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
useEffect(() => {
|
||||
const r = runtime.current;
|
||||
if (!r) return;
|
||||
r.path.visible =
|
||||
!props.locked && !props.cameraView && props.previewTime === null;
|
||||
r.path.visible = !props.locked && !props.cameraView && props.previewTime === null;
|
||||
if (props.locked || props.previewTime === null) return;
|
||||
const pose = sampleDirectorMotion(props.scene.motion, props.previewTime);
|
||||
if (!pose) return;
|
||||
@ -376,13 +419,7 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
r.camera.updateProjectionMatrix();
|
||||
r.camera.lookAt(r.orbit.target);
|
||||
r.transform.detach();
|
||||
}, [
|
||||
props.previewTime,
|
||||
props.scene.motion,
|
||||
props.cameraView,
|
||||
props.locked,
|
||||
ready,
|
||||
]);
|
||||
}, [props.previewTime, props.scene.motion, props.cameraView, props.locked, ready]);
|
||||
useEffect(() => {
|
||||
runtime.current?.resize();
|
||||
}, [props.scene.ratio, ready]);
|
||||
@ -402,19 +439,13 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
const radius = Math.max(3, box.getSize(new THREE.Vector3()).length());
|
||||
const position = center
|
||||
.clone()
|
||||
.add(
|
||||
new THREE.Vector3(1, 0.8, 1)
|
||||
.normalize()
|
||||
.multiplyScalar(radius * 1.4),
|
||||
);
|
||||
.add(new THREE.Vector3(1, 0.8, 1).normalize().multiplyScalar(radius * 1.4));
|
||||
latest.current.onCamera(position.toArray(), center.toArray());
|
||||
},
|
||||
record: async (signal, onProgress) => {
|
||||
const r = runtime.current;
|
||||
if (!r || !ready || r.pending || error)
|
||||
throw Error(error || "3D 场景尚未准备好");
|
||||
if (latest.current.scene.motion.keyframes.length < 2)
|
||||
throw Error("至少需要两个关键帧");
|
||||
if (!r || !ready || r.pending || error) throw Error(error || "3D 场景尚未准备好");
|
||||
if (latest.current.scene.motion.keyframes.length < 2) throw Error("至少需要两个关键帧");
|
||||
const motion = latest.current.scene.motion;
|
||||
const size = r.renderer.getSize(new THREE.Vector2()),
|
||||
pixelRatio = r.renderer.getPixelRatio();
|
||||
@ -465,11 +496,8 @@ export const DirectorViewport = forwardRef<DirectorViewportHandle, Props>(
|
||||
},
|
||||
capture: async (pose) => {
|
||||
const r = runtime.current;
|
||||
if (!r || !ready || r.pending || error)
|
||||
throw Error(error || "3D 场景尚未准备好");
|
||||
const { width, height } = directorImageSize(
|
||||
latest.current.scene.ratio,
|
||||
);
|
||||
if (!r || !ready || r.pending || error) throw Error(error || "3D 场景尚未准备好");
|
||||
const { width, height } = directorImageSize(latest.current.scene.ratio);
|
||||
const previousSize = r.renderer.getSize(new THREE.Vector2());
|
||||
const pixelRatio = r.renderer.getPixelRatio();
|
||||
const aspect = r.camera.aspect;
|
||||
|
||||
@ -1929,5 +1929,10 @@
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
],
|
||||
"followup": {
|
||||
"date": "2026-09-17",
|
||||
"report": "video-canvas-repairs-2026-09-17.md",
|
||||
"scope": "基线保留；后续已修复、部分补齐和明确未接通项目见报告，未部署"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
# 视频画布逐层功能对照记录 — 2026-09-17
|
||||
|
||||
> 后续修复见 [逐项修复和验证记录](video-canvas-repairs-2026-09-17.md)。下方为修复前基线，尤其 05-15 的截帧判断已在修复记录中纠正；不能将基线结论作为最新状态。
|
||||
|
||||
本轮按页面 → 模块 → 菜单 → 子菜单/参数 → 实际处理代码逐层核对，共记录 **231 个检查点、10 个分组**。结论：当前本地版本的基础生成、视频处理和相机运镜已有真实能力，但离参考站完整功能与像素级对齐仍有明显差距；旧文档中“已完成/已对齐”的泛化表述不能作为验收结论。
|
||||
|
||||
## 范围和证据口径
|
||||
|
||||
57
docs/video-canvas-repairs-2026-09-17.md
Normal file
57
docs/video-canvas-repairs-2026-09-17.md
Normal file
@ -0,0 +1,57 @@
|
||||
# 视频画布修复记录 — 2026-09-17
|
||||
|
||||
基于 `e5b2de3c` 的 231 项对照检查继续修复。本文记录已修改的具体行为和验证边界；**不表示 231 项全部完成，也不表示与 LibTV 像素级一致**。代码在 `codex/video-canvas-parity-0916`；本轮没有部署、重启 48 服务器或修改生产数据库。
|
||||
|
||||
## 生成与参数
|
||||
|
||||
| 对照 ID | 修复 | 验证 |
|
||||
|---|---|---|
|
||||
| 03-02、03-06、03-33 | 主图片生成传递所选清晰度；展示每张返回图片；空参考不再注入人物示例 | 请求契约单测覆盖 1K/2K/4K、数量、空参考、多张结果；不表示各模型都支持全部规格 |
|
||||
| 03-22 | 禁用的宫格功能在创建结果前拦截，不再把示例图标记成功 | 代码检查 |
|
||||
| 04-14 | 视频 1/2/4 个对应逐个请求及独立结果节点，单项失败保留其余成功结果；重复点击拦截 | 类型与执行链检查；未付费调用 4 个视频进行验收 |
|
||||
| 02-09、02-10 | 移除未执行的联网搜索、合规素材校验开关；明确保留基础输入校验 | 代码检查 |
|
||||
| 03-12、03-13、03-32 | 多角度预设更新并提交参数；环绕角转换到 0–360、缩放 0–100 转 0–10；鱼眼和倾斜通过配置的图片模型重绘 | 单测；浏览器选择背面实际提交 180°/0°/50%；未调用模型评估角度效果 |
|
||||
| 03-14 | 明确中央图片是参考原图；去掉无执行作用的提示词开关和固定价格 | 代码检查；不宣称实时生成预览 |
|
||||
| 06-19 | 分镜视频使用配置模型及镜头时长、真实参考；未提交时显示待生成，移除 Seedance 写死与示例封面 | 代码检查；仍是创建生成器节点，需用户提交生成 |
|
||||
|
||||
## 脚本工作流
|
||||
|
||||
- `06-04/05/07`：表格可编辑、添加、删除和上移镜头，清空内容不会立即被默认值填回；采用画布现有保存流程。
|
||||
- `06-08/09/10/11/13`：不再注入唐朝示例角色；从脚本引用识别候选资产，类型可人工修正；支持新增、上传、引用当前画布、清除、删除、调用配置图片模型生成。
|
||||
- `06-14/15/16/17`：完成数量按真实图片和提示词计算；全局风格可编辑并参与合成；单镜和批量合成调用现有分析服务，验证镜头数量与输出字段，失败保留原内容；去掉固定第三方模型和价格。
|
||||
- `06-18`：分镜图片节点按 `@资产名` 携带准备好的参考图片；按钮文案区分创建节点与已生成图片。
|
||||
- 浏览器组件验收：修改对白、新增镜头、引用资产、修改风格，刷新后恢复；空提示词仍显示 `0/2`，资产引用后显示 `1/3`。验收页采用 localStorage 保存适配器，**不是本轮数据库端到端验收**。AI 合成、资产 AI 生成与上传仅完成执行链检查，本轮没有重复付费请求。
|
||||
- 尚未实现拖动排序、资产逐项模型/比例/数量选择、完整自动批量生成闭环。
|
||||
|
||||
## 素材、下载和音频
|
||||
|
||||
- `10-09/10/11/12`：面板明确范围为“当前画布素材”，只列有真实文件地址的节点，不补示例，不截断为 8/10 条，去掉虚构日期，按节点添加顺序排序。空列表已在浏览器验证。
|
||||
- `10-13`：单项、预览和批量下载调用已有下载/ZIP 接口，不再用 15 秒动画代替下载；显示 ZIP 部分失败；音频下载保留原文件，不交给图片解码器。跨域文件经现有认证下载接口处理。单测验证 WAV 字节与扩展名。
|
||||
- `10-14`：动作明确为从素材列表移除，状态随画布保存；不冒充删除源文件或跨项目历史记录。
|
||||
- `10-18`：原空资产页改为当前画布媒体列表，搜索、类型过滤、定位和下载有实际执行；工作区显示项目标题。个人跨画布资产库和 Agent 资产仍未实现。
|
||||
- `07-01…18`：未找到已实现的 TTS/音乐后端适配器，移除固定模型、假音色、空生成按钮与假调节控件。提供 `/api/canvas/media/upload` 的真实音频上传入口，保存时长，继续使用原有播放器。**这是纠正不可用入口，不是接通配音/音乐生成**。
|
||||
- `01-26/10-25`：快捷键说明只显示已绑定操作，不再列出不存在的撤销、分组等快捷键。
|
||||
|
||||
## 视频后期
|
||||
|
||||
- `05-15/16`：纠正上一轮对截帧的判断。旧活动处理函数实际复制封面，并硬写 1280×720；现在使用 FFmpeg 按首帧、尾帧、播放器当前时间截取真实 PNG，并探测实际输出尺寸。
|
||||
- `05-06`：新增 2/3/5 倍慢放，视频时间戳与音轨同步变速；保留音高，输出最长 10 分钟。属于普通变速，**不是 AI 补帧**。
|
||||
- `05-04/09/10/12/13`：超分、去字幕、人声/背景音分离未接专用服务，在活动工具栏明确不可用；仍保留真实音视频分离。
|
||||
- 真实本地 FFmpeg 验证：320×180、24fps、2 秒合成视频（含音轨），首帧/1 秒帧/尾帧输出不同；2/3/5 倍慢放分别约 **4.013 / 6.005 / 9.990 秒**，三份都保留音轨。集成测试用临时目录并自动清理。
|
||||
|
||||
## 3D 导演台
|
||||
|
||||
- `08-03/04/05/07/08/09`：对象搜索、改名、复制、显隐、锁定变换轴、等比缩放、基础对象颜色；旋转输入改为度，保存仍用弧度。
|
||||
- `08-11/13`：基础人偶改为连接肩肘髋膝的关节层级，支持 12 种姿势：站立、T 型、行走、跑步、坐姿、蹲下、招手、伸手、鞠躬、格斗、踢球、看手机。**不是参考站完整的 20 姿势、8 体型或专业骨骼编辑器**。
|
||||
- `08-15`：已加载对象只在资源、几何或外观变化时重建；单纯位置/旋转/缩放不再销毁并重新下载全部 GLB。
|
||||
- `09-01/06/07/09`：扩展至 12 个机位预设；可对准所选对象；FOV 保留两位小数；相机时间轴移至底部可折叠区域。对准是一次性定位，不是自动跟随。
|
||||
- 浏览器实际显示 T 型、招手姿势，创建正面机位与环绕轨迹，导出并通过本地上传适配器接收 **1280×720、约 2.997 秒 MP4（VP9）**。旧场景与新增姿势/显隐等数据 JSON 往返单测通过。
|
||||
- 对象多轨、自动帧、目标跟随、自由路径、全景和 AI 识图重建仍未实现。
|
||||
|
||||
## 验证与未完成范围
|
||||
|
||||
- TypeScript：`pnpm exec tsc --noEmit --pretty false` 通过。
|
||||
- 5 组相关回归：**50 项通过，无跳过**，包括新加入的本地真实 FFmpeg 测试。
|
||||
- 浏览器：脚本编辑与恢复、资产引用计数、空素材列表、多角度预设提交、关节人偶渲染、相机轨迹导出。
|
||||
- 本轮使用生成参数契约和组件验收，没有重复测试全部供应商；不宣称新模型线路成功。
|
||||
- 尚需继续：生成中任务断线/刷新恢复、全画布撤销分组、多画布/故事板/Agent、跨项目资产、完整智能剪辑、专用后期服务、3D 高级轨迹和逐状态像素验收。底层未接通能力不能只靠复制页面宣称完成。
|
||||
@ -157,7 +157,7 @@ function looksLikeErrorDocument(buffer: Buffer, contentType: string): boolean {
|
||||
return /^(?:<!doctype\s+html|<html[\s>]|<\?xml|<error[\s>]|[{[])/i.test(text);
|
||||
}
|
||||
|
||||
function detectAsset(bytes: Buffer, contentType: string): { kind: "image" | "video"; mimeType: string; ext: string } | null {
|
||||
function detectAsset(bytes: Buffer, contentType: string): { kind: "image" | "video" | "audio"; mimeType: string; ext: string } | null {
|
||||
if (bytes.length < 4) return null;
|
||||
|
||||
if (bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
|
||||
@ -184,8 +184,14 @@ function detectAsset(bytes: Buffer, contentType: string): { kind: "image" | "vid
|
||||
if (looksLikeSvg(bytes, contentType)) {
|
||||
return { kind: "image", mimeType: "image/svg+xml", ext: "svg" };
|
||||
}
|
||||
if (hasAsciiAt(bytes, 0, "RIFF") && hasAsciiAt(bytes, 8, "WAVE")) return {kind:"audio",mimeType:"audio/wav",ext:"wav"};
|
||||
if (hasAsciiAt(bytes, 0, "ID3") || (bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0 && (bytes[1] & 0x06) !== 0)) return {kind:"audio",mimeType:"audio/mpeg",ext:"mp3"};
|
||||
if (hasAsciiAt(bytes, 0, "OggS")) return {kind:"audio",mimeType:"audio/ogg",ext:"ogg"};
|
||||
if (hasAsciiAt(bytes, 0, "fLaC")) return {kind:"audio",mimeType:"audio/flac",ext:"flac"};
|
||||
if (getContentTypeMime(contentType).startsWith("audio/")) return {kind:"audio",mimeType:getContentTypeMime(contentType),ext:getContentTypeMime(contentType).includes("mp4") ? "m4a" : getContentTypeMime(contentType).split("/")[1]};
|
||||
if (hasAsciiAt(bytes, 4, "ftyp")) {
|
||||
const brand = bytes.subarray(8, Math.min(bytes.length, 32)).toString("latin1").toLowerCase();
|
||||
if (brand.includes("m4a")) return {kind:"audio",mimeType:"audio/mp4",ext:"m4a"};
|
||||
if (brand.includes("avif")) return { kind: "image", mimeType: "image/avif", ext: "avif" };
|
||||
if (brand.includes("heic") || brand.includes("heif") || brand.includes("mif1")) {
|
||||
return { kind: "image", mimeType: "image/heic", ext: "heic" };
|
||||
@ -258,7 +264,7 @@ async function normalizeCanvasDownloadAsset(params: {
|
||||
throw new Error("无法识别图片格式");
|
||||
}
|
||||
|
||||
if (detected.kind === "video") {
|
||||
if (detected.kind !== "image") {
|
||||
return {
|
||||
ok: true,
|
||||
filename: buildCanvasAssetFilename({
|
||||
|
||||
35
lib/video-canvas/angle-settings.ts
Normal file
35
lib/video-canvas/angle-settings.ts
Normal file
@ -0,0 +1,35 @@
|
||||
export const anglePresets: Record<
|
||||
string,
|
||||
{ horizontalAngle: number; verticalAngle: number; zoom: number; prompt?: string }
|
||||
> = {
|
||||
自定义: { horizontalAngle: 0, verticalAngle: 0, zoom: 50 },
|
||||
鱼眼视角: {
|
||||
horizontalAngle: 0,
|
||||
verticalAngle: 0,
|
||||
zoom: 20,
|
||||
prompt: "保持主体身份与场景，以鱼眼超广角镜头重新构图，明显桶形畸变",
|
||||
},
|
||||
倾斜视角: {
|
||||
horizontalAngle: 30,
|
||||
verticalAngle: 0,
|
||||
zoom: 50,
|
||||
prompt: "保持主体身份与场景，以荷兰角镜头拍摄，画面地平线倾斜 25 度",
|
||||
},
|
||||
正面俯拍: { horizontalAngle: 0, verticalAngle: 45, zoom: 50 },
|
||||
正面仰拍: { horizontalAngle: 0, verticalAngle: -30, zoom: 50 },
|
||||
全景俯拍: { horizontalAngle: 0, verticalAngle: 90, zoom: 0 },
|
||||
背面视角: { horizontalAngle: 180, verticalAngle: 0, zoom: 50 },
|
||||
};
|
||||
export function angleRequestParameters(settings: {
|
||||
horizontalAngle: number;
|
||||
verticalAngle: number;
|
||||
zoom: number;
|
||||
}) {
|
||||
if (![settings.horizontalAngle, settings.verticalAngle, settings.zoom].every(Number.isFinite))
|
||||
throw Error("视角参数必须是有限数值");
|
||||
return {
|
||||
horizontalAngle: ((settings.horizontalAngle % 360) + 360) % 360,
|
||||
verticalAngle: Math.max(-30, Math.min(90, settings.verticalAngle)),
|
||||
zoom: Math.max(0, Math.min(10, settings.zoom / 10)),
|
||||
};
|
||||
}
|
||||
41
lib/video-canvas/director-poses.ts
Normal file
41
lib/video-canvas/director-poses.ts
Normal file
@ -0,0 +1,41 @@
|
||||
// Angles in degrees. Each joint is local to its parent, so limbs remain connected.
|
||||
export const directorPoses = {
|
||||
站立: { arms: [0, 0], elbows: [0, 0], legs: [0, 0], knees: [0, 0], lean: 0 },
|
||||
"T 型": { arms: [0, 0], elbows: [0, 0], legs: [0, 0], knees: [0, 0], lean: 0, spread: 90 },
|
||||
行走: { arms: [-30, 30], elbows: [15, 15], legs: [30, -30], knees: [10, 35], lean: 0 },
|
||||
跑步: { arms: [-55, 55], elbows: [80, 80], legs: [60, -45], knees: [70, 100], lean: 12 },
|
||||
坐姿: {
|
||||
arms: [10, 10],
|
||||
elbows: [75, 75],
|
||||
legs: [-90, -90],
|
||||
knees: [90, 90],
|
||||
lean: 0,
|
||||
height: -0.45,
|
||||
},
|
||||
蹲下: {
|
||||
arms: [-65, -65],
|
||||
elbows: [15, 15],
|
||||
legs: [-95, -95],
|
||||
knees: [125, 125],
|
||||
lean: 25,
|
||||
height: -0.65,
|
||||
},
|
||||
招手: { arms: [0, -165], elbows: [0, 30], legs: [0, 0], knees: [0, 0], lean: 0 },
|
||||
伸手: { arms: [-90, -90], elbows: [0, 0], legs: [0, 0], knees: [0, 0], lean: 0 },
|
||||
鞠躬: { arms: [-20, -20], elbows: [0, 0], legs: [0, 0], knees: [0, 0], lean: 40 },
|
||||
格斗: { arms: [-50, -75], elbows: [100, 90], legs: [-20, 20], knees: [20, 35], lean: 5 },
|
||||
踢球: { arms: [-15, 30], elbows: [20, 20], legs: [-70, 5], knees: [10, 10], lean: -10 },
|
||||
看手机: { arms: [-30, -30], elbows: [105, 105], legs: [0, 0], knees: [0, 0], lean: 8 },
|
||||
} satisfies Record<
|
||||
string,
|
||||
{
|
||||
arms: number[];
|
||||
elbows: number[];
|
||||
legs: number[];
|
||||
knees: number[];
|
||||
lean: number;
|
||||
spread?: number;
|
||||
height?: number;
|
||||
}
|
||||
>;
|
||||
export type DirectorPoseName = keyof typeof directorPoses;
|
||||
@ -1,3 +1,4 @@
|
||||
import { directorPoses, type DirectorPoseName } from "./director-poses";
|
||||
import {
|
||||
defaultDirectorMotion,
|
||||
readDirectorMotion,
|
||||
@ -10,6 +11,10 @@ export type DirectorObject = {
|
||||
name: string;
|
||||
kind: "actor" | "box" | "sphere" | "model";
|
||||
url?: string;
|
||||
color?: string;
|
||||
visible?: boolean;
|
||||
locked?: boolean;
|
||||
pose?: DirectorPoseName;
|
||||
position: DirectorVector;
|
||||
rotation: DirectorVector;
|
||||
scale: DirectorVector;
|
||||
@ -39,9 +44,7 @@ export type DirectorScene = {
|
||||
};
|
||||
export const directorRatios = ["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"];
|
||||
export function directorImageSize(ratio: string) {
|
||||
const [w, h] = (directorRatios.includes(ratio) ? ratio : "16:9")
|
||||
.split(":")
|
||||
.map(Number);
|
||||
const [w, h] = (directorRatios.includes(ratio) ? ratio : "16:9").split(":").map(Number);
|
||||
return w >= h
|
||||
? { width: 1280, height: Math.round((1280 * h) / w) }
|
||||
: { width: Math.round((1280 * w) / h), height: 1280 };
|
||||
@ -122,7 +125,18 @@ export function readDirectorScene(value: unknown): DirectorScene {
|
||||
: [];
|
||||
const validCameras = cameras.length ? cameras : fallback.cameras;
|
||||
return {
|
||||
objects,
|
||||
objects: objects.map((o) => {
|
||||
const clean = { ...o };
|
||||
if (
|
||||
o.color !== undefined &&
|
||||
(typeof o.color !== "string" || !/^#[0-9a-f]{6}$/i.test(o.color))
|
||||
)
|
||||
delete clean.color;
|
||||
if (o.visible !== undefined) clean.visible = o.visible !== false;
|
||||
if (o.locked !== undefined) clean.locked = o.locked === true;
|
||||
if (o.pose !== undefined && !Object.hasOwn(directorPoses, o.pose)) delete clean.pose;
|
||||
return clean;
|
||||
}),
|
||||
cameras: validCameras,
|
||||
activeCameraId: validCameras.some((c) => c.id === v.activeCameraId)
|
||||
? v.activeCameraId
|
||||
|
||||
35
lib/video-canvas/download.ts
Normal file
35
lib/video-canvas/download.ts
Normal file
@ -0,0 +1,35 @@
|
||||
/** Downloads through the authenticated canvas export route, including cross-origin media. */
|
||||
export async function downloadCanvasMedia(files: Array<{ url: string; name: string }>) {
|
||||
if (!files.length) throw Error("没有可下载的素材");
|
||||
if (files.length > 200) throw Error("每次最多下载 200 项，请分批选择");
|
||||
const batch = files.length > 1;
|
||||
const response = await fetch(batch ? "/api/canvas/download-zip" : "/api/canvas/download-image", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(batch ? { files, folderName: "画布素材" } : files[0]),
|
||||
signal: AbortSignal.timeout(300000),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const data = await response.json().catch(() => ({}));
|
||||
throw Error(data.error || "素材下载失败");
|
||||
}
|
||||
const blob = await response.blob();
|
||||
if (!blob.size) throw Error("返回的下载文件为空");
|
||||
const encodedName = response.headers
|
||||
.get("content-disposition")
|
||||
?.match(/filename\*=UTF-8''([^;]+)/i)?.[1];
|
||||
const filename = encodedName
|
||||
? decodeURIComponent(encodedName)
|
||||
: batch
|
||||
? "画布素材.zip"
|
||||
: files[0].name;
|
||||
const url = URL.createObjectURL(blob);
|
||||
const link = document.createElement("a");
|
||||
link.href = url;
|
||||
link.download = filename;
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
link.remove();
|
||||
window.setTimeout(() => URL.revokeObjectURL(url), 60000);
|
||||
return { failed: Number(response.headers.get("X-Download-Failed-Count") || 0) };
|
||||
}
|
||||
29
lib/video-canvas/generation-contract.ts
Normal file
29
lib/video-canvas/generation-contract.ts
Normal file
@ -0,0 +1,29 @@
|
||||
export function canvasGenerationCount(value: unknown) {
|
||||
const count = Number.parseInt(String(value), 10);
|
||||
return [1, 2, 4].includes(count) ? count : 1;
|
||||
}
|
||||
export function canvasImageRequest(
|
||||
settings: { model?: string; ratio: string; resolution: string; count: string },
|
||||
references: string[],
|
||||
wide = false,
|
||||
) {
|
||||
const images = [...new Set(references.map((url) => url.trim()).filter(Boolean))].slice(0, 12);
|
||||
return {
|
||||
model: settings.model,
|
||||
aspectRatio: wide ? "16:9" : settings.ratio,
|
||||
imageSize: settings.resolution,
|
||||
imageCount: wide ? 1 : canvasGenerationCount(settings.count),
|
||||
referenceImage: images[0],
|
||||
referenceImages: images.slice(1),
|
||||
};
|
||||
}
|
||||
export function canvasImageResultUrls(response: {
|
||||
success: boolean;
|
||||
data?: { images?: Array<{ originalUrl?: string; thumbnailUrl?: string }> };
|
||||
}) {
|
||||
return response.success
|
||||
? (response.data?.images || [])
|
||||
.map((image) => image.originalUrl || image.thumbnailUrl || "")
|
||||
.filter(Boolean)
|
||||
: [];
|
||||
}
|
||||
@ -1,20 +1,45 @@
|
||||
import { z } from "zod";
|
||||
|
||||
const fraction = z.number().finite().min(0).max(1);
|
||||
export const videoTimeRangeSchema = z.object({ start: fraction, end: fraction })
|
||||
.refine(range => range.end > range.start, "结束位置必须晚于开始位置");
|
||||
export const videoTimeRangeSchema = z
|
||||
.object({ start: fraction, end: fraction })
|
||||
.refine((range) => range.end > range.start, "结束位置必须晚于开始位置");
|
||||
export type VideoTimeRange = z.infer<typeof videoTimeRangeSchema>;
|
||||
|
||||
export const videoProcessSchema = z.discriminatedUnion("operation", [
|
||||
z.object({ operation: z.literal("crop"), url: z.string().url(), rect: z.object({
|
||||
x: fraction, y: fraction, width: fraction.positive(), height: fraction.positive(),
|
||||
}).refine(rect => rect.x + rect.width <= 1.001 && rect.y + rect.height <= 1.001, "裁剪区域超出画面") }),
|
||||
z.object({
|
||||
operation: z.literal("crop"),
|
||||
url: z.string().url(),
|
||||
rect: z
|
||||
.object({
|
||||
x: fraction,
|
||||
y: fraction,
|
||||
width: fraction.positive(),
|
||||
height: fraction.positive(),
|
||||
})
|
||||
.refine(
|
||||
(rect) => rect.x + rect.width <= 1.001 && rect.y + rect.height <= 1.001,
|
||||
"裁剪区域超出画面",
|
||||
),
|
||||
}),
|
||||
z.object({ operation: z.literal("audio-video"), url: z.string().url() }),
|
||||
z.object({
|
||||
operation: z.literal("frame"),
|
||||
url: z.string().url(),
|
||||
kind: z.enum(["first", "last", "current"]),
|
||||
time: z.number().finite().min(0).default(0),
|
||||
}),
|
||||
z.object({
|
||||
operation: z.literal("slow"),
|
||||
url: z.string().url(),
|
||||
factor: z.union([z.literal(2), z.literal(3), z.literal(5)]),
|
||||
}),
|
||||
z.object({ operation: z.literal("trim"), url: z.string().url(), range: videoTimeRangeSchema }),
|
||||
]);
|
||||
|
||||
export function toVideoTimeRange(range: VideoTimeRange, duration: number) {
|
||||
if (!videoTimeRangeSchema.safeParse(range).success || !Number.isFinite(duration) || duration <= 0) throw new Error("剪辑范围无效");
|
||||
if (!videoTimeRangeSchema.safeParse(range).success || !Number.isFinite(duration) || duration <= 0)
|
||||
throw new Error("剪辑范围无效");
|
||||
const start = range.start * duration;
|
||||
const end = range.end * duration;
|
||||
if (end - start < 0.1) throw new Error("剪辑时长不能少于 0.1 秒");
|
||||
|
||||
@ -1,47 +1,213 @@
|
||||
import {execFile} from 'node:child_process';
|
||||
import {promisify} from 'node:util';
|
||||
import { toVideoTimeRange, type VideoTimeRange } from './media-contract';
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { toVideoTimeRange, type VideoTimeRange } from "./media-contract";
|
||||
|
||||
export type MediaRect = {x:number;y:number;width:number;height:number};
|
||||
export function toVideoPixelRect(rect:MediaRect,width:number,height:number) {
|
||||
if (![rect.x,rect.y,rect.width,rect.height,width,height].every(Number.isFinite) || rect.x<0 || rect.y<0 || rect.width<=0 || rect.height<=0 || rect.x+rect.width>1.001 || rect.y+rect.height>1.001) throw new Error('裁剪区域无效');
|
||||
const even=(n:number)=>Math.floor(n/2)*2;
|
||||
const x=even(rect.x*width), y=even(rect.y*height);
|
||||
const w=Math.min(even(rect.width*width),even(width-x)), h=Math.min(even(rect.height*height),even(height-y));
|
||||
if(w<2||h<2) throw new Error('裁剪区域太小');
|
||||
return {x,y,width:w,height:h};
|
||||
export type MediaRect = { x: number; y: number; width: number; height: number };
|
||||
export function toVideoPixelRect(rect: MediaRect, width: number, height: number) {
|
||||
if (
|
||||
![rect.x, rect.y, rect.width, rect.height, width, height].every(Number.isFinite) ||
|
||||
rect.x < 0 ||
|
||||
rect.y < 0 ||
|
||||
rect.width <= 0 ||
|
||||
rect.height <= 0 ||
|
||||
rect.x + rect.width > 1.001 ||
|
||||
rect.y + rect.height > 1.001
|
||||
)
|
||||
throw new Error("裁剪区域无效");
|
||||
const even = (n: number) => Math.floor(n / 2) * 2;
|
||||
const x = even(rect.x * width),
|
||||
y = even(rect.y * height);
|
||||
const w = Math.min(even(rect.width * width), even(width - x)),
|
||||
h = Math.min(even(rect.height * height), even(height - y));
|
||||
if (w < 2 || h < 2) throw new Error("裁剪区域太小");
|
||||
return { x, y, width: w, height: h };
|
||||
}
|
||||
const run=promisify(execFile);
|
||||
const inputSafety=['-protocol_whitelist','file','-format_whitelist','mov,matroska,webm,avi,mpeg,mpegts,flv,ogg'];
|
||||
export async function processVideoFile(input:string,output:string,operation:'crop'|'extract-audio'|'silent-video'|'trim',rect?:MediaRect,range?:VideoTimeRange) {
|
||||
const {stdout}=await run('ffprobe',['-v','error',...inputSafety,'-show_streams','-show_format','-of','json',input],{timeout:15000,maxBuffer:1024*1024});
|
||||
const info=JSON.parse(stdout), video=info.streams?.find((s:any)=>s.codec_type==='video');
|
||||
if(!video) throw new Error('素材中没有视频画面');
|
||||
if (!(Number(video.width)>0 && Number(video.height)>0) || Number(video.width)*Number(video.height)>33554432) throw new Error('视频尺寸不受支持，请使用 8K 以内的视频');
|
||||
const duration=Number(info.format?.duration || 0);
|
||||
if(!Number.isFinite(duration)||duration<=0||duration>600) throw new Error('本地视频处理支持 10 分钟以内的视频');
|
||||
let args:string[];
|
||||
let dimensions={width:Number(video.width),height:Number(video.height)};
|
||||
const run = promisify(execFile);
|
||||
const inputSafety = [
|
||||
"-protocol_whitelist",
|
||||
"file",
|
||||
"-format_whitelist",
|
||||
"mov,matroska,webm,avi,mpeg,mpegts,flv,ogg",
|
||||
];
|
||||
export async function processVideoFile(
|
||||
input: string,
|
||||
output: string,
|
||||
operation: "crop" | "extract-audio" | "silent-video" | "trim" | "frame" | "slow",
|
||||
rect?: MediaRect,
|
||||
range?: VideoTimeRange,
|
||||
options?: { kind?: "first" | "last" | "current"; time?: number; factor?: number },
|
||||
) {
|
||||
const { stdout } = await run(
|
||||
"ffprobe",
|
||||
["-v", "error", ...inputSafety, "-show_streams", "-show_format", "-of", "json", input],
|
||||
{ timeout: 15000, maxBuffer: 1024 * 1024 },
|
||||
);
|
||||
const info = JSON.parse(stdout),
|
||||
video = info.streams?.find((s: any) => s.codec_type === "video");
|
||||
if (!video) throw new Error("素材中没有视频画面");
|
||||
if (
|
||||
!(Number(video.width) > 0 && Number(video.height) > 0) ||
|
||||
Number(video.width) * Number(video.height) > 33554432
|
||||
)
|
||||
throw new Error("视频尺寸不受支持，请使用 8K 以内的视频");
|
||||
const duration = Number(video.duration || info.format?.duration || 0);
|
||||
if (!Number.isFinite(duration) || duration <= 0 || duration > 600)
|
||||
throw new Error("本地视频处理支持 10 分钟以内的视频");
|
||||
let args: string[];
|
||||
let dimensions = { width: Number(video.width), height: Number(video.height) };
|
||||
let outputDuration = duration;
|
||||
if(operation==='extract-audio') {
|
||||
if(!info.streams.some((s:any)=>s.codec_type==='audio')) throw new Error('原视频不包含音轨');
|
||||
args=['-map','0:a:0','-vn','-c:a','aac','-b:a','192k'];
|
||||
} else if(operation==='silent-video') {
|
||||
args=['-map','0:v:0','-an','-c:v','copy','-movflags','+faststart'];
|
||||
} else if(operation==='trim') {
|
||||
if (!range) throw new Error('请选择剪辑范围');
|
||||
if (operation === "frame") {
|
||||
const [numerator, denominator] = String(video.avg_frame_rate || "30/1")
|
||||
.split("/")
|
||||
.map(Number);
|
||||
const fps = numerator / denominator;
|
||||
const last = Math.max(0, duration - 1 / (Number.isFinite(fps) && fps > 0 ? fps : 30));
|
||||
const time =
|
||||
options?.kind === "last"
|
||||
? last
|
||||
: options?.kind === "current"
|
||||
? Math.min(last, Math.max(0, Number(options.time) || 0))
|
||||
: 0;
|
||||
args = ["-ss", String(time), "-map", "0:v:0", "-frames:v", "1", "-an", "-c:v", "png"];
|
||||
outputDuration = 0;
|
||||
} else if (operation === "slow") {
|
||||
const factor = options?.factor;
|
||||
if (!factor || ![2, 3, 5].includes(factor)) throw Error("慢放倍数无效");
|
||||
outputDuration = duration * factor;
|
||||
if (outputDuration > 600) throw Error("慢放后视频不能超过 10 分钟");
|
||||
let speed = 1 / factor;
|
||||
const filters: string[] = [];
|
||||
while (speed < 0.5) {
|
||||
filters.push("atempo=0.5");
|
||||
speed /= 0.5;
|
||||
}
|
||||
filters.push(`atempo=${speed}`);
|
||||
args = [
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"0:a?",
|
||||
"-vf",
|
||||
`setpts=${factor}*(PTS-STARTPTS),scale=trunc(iw/2)*2:trunc(ih/2)*2`,
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"20",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
];
|
||||
if (info.streams.some((stream: any) => stream.codec_type === "audio"))
|
||||
args.push("-af", filters.join(","), "-c:a", "aac");
|
||||
args.push("-movflags", "+faststart");
|
||||
dimensions = {
|
||||
width: Math.floor(dimensions.width / 2) * 2,
|
||||
height: Math.floor(dimensions.height / 2) * 2,
|
||||
};
|
||||
} else if (operation === "extract-audio") {
|
||||
if (!info.streams.some((s: any) => s.codec_type === "audio"))
|
||||
throw new Error("原视频不包含音轨");
|
||||
args = ["-map", "0:a:0", "-vn", "-c:a", "aac", "-b:a", "192k"];
|
||||
} else if (operation === "silent-video") {
|
||||
args = ["-map", "0:v:0", "-an", "-c:v", "copy", "-movflags", "+faststart"];
|
||||
} else if (operation === "trim") {
|
||||
if (!range) throw new Error("请选择剪辑范围");
|
||||
const selected = toVideoTimeRange(range, duration);
|
||||
outputDuration = selected.duration;
|
||||
dimensions = {width:Math.floor(dimensions.width/2)*2,height:Math.floor(dimensions.height/2)*2};
|
||||
args=['-ss',String(selected.start),'-t',String(selected.duration),'-map','0:v:0','-map','0:a?',
|
||||
'-vf','scale=trunc(iw/2)*2:trunc(ih/2)*2','-c:v','libx264','-preset','veryfast','-crf','20',
|
||||
'-pix_fmt','yuv420p','-c:a','aac','-movflags','+faststart'];
|
||||
dimensions = {
|
||||
width: Math.floor(dimensions.width / 2) * 2,
|
||||
height: Math.floor(dimensions.height / 2) * 2,
|
||||
};
|
||||
args = [
|
||||
"-ss",
|
||||
String(selected.start),
|
||||
"-t",
|
||||
String(selected.duration),
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"0:a?",
|
||||
"-vf",
|
||||
"scale=trunc(iw/2)*2:trunc(ih/2)*2",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"20",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
];
|
||||
} else {
|
||||
if(!rect) throw new Error('请选择裁剪区域');
|
||||
const crop=toVideoPixelRect(rect,dimensions.width,dimensions.height);
|
||||
dimensions=crop;
|
||||
args=['-map','0:v:0','-map','0:a?','-vf',`crop=${crop.width}:${crop.height}:${crop.x}:${crop.y}`,'-c:v','libx264','-preset','veryfast','-crf','20','-pix_fmt','yuv420p','-c:a','aac','-movflags','+faststart'];
|
||||
if (!rect) throw new Error("请选择裁剪区域");
|
||||
const crop = toVideoPixelRect(rect, dimensions.width, dimensions.height);
|
||||
dimensions = crop;
|
||||
args = [
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"0:a?",
|
||||
"-vf",
|
||||
`crop=${crop.width}:${crop.height}:${crop.x}:${crop.y}`,
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"20",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
];
|
||||
}
|
||||
await run('ffmpeg',['-nostdin','-y','-v','error','-threads','2','-filter_threads','1',...inputSafety,'-i',input,...args,'-threads','2',output],{timeout:180000,maxBuffer:1024*1024});
|
||||
return {duration:outputDuration,...dimensions};
|
||||
await run(
|
||||
"ffmpeg",
|
||||
[
|
||||
"-nostdin",
|
||||
"-y",
|
||||
"-v",
|
||||
"error",
|
||||
"-threads",
|
||||
"2",
|
||||
"-filter_threads",
|
||||
"1",
|
||||
...inputSafety,
|
||||
"-i",
|
||||
input,
|
||||
...args,
|
||||
"-threads",
|
||||
"2",
|
||||
output,
|
||||
],
|
||||
{ timeout: 180000, maxBuffer: 1024 * 1024 },
|
||||
);
|
||||
if (operation === "frame") {
|
||||
const probe = await run(
|
||||
"ffprobe",
|
||||
[
|
||||
"-v",
|
||||
"error",
|
||||
"-select_streams",
|
||||
"v:0",
|
||||
"-show_entries",
|
||||
"stream=width,height",
|
||||
"-of",
|
||||
"json",
|
||||
output,
|
||||
],
|
||||
{ timeout: 15000, maxBuffer: 1024 * 1024 },
|
||||
);
|
||||
const frame = JSON.parse(probe.stdout).streams?.[0];
|
||||
if (!frame?.width || !frame?.height) throw Error("未截取到有效视频帧");
|
||||
dimensions = { width: Number(frame.width), height: Number(frame.height) };
|
||||
}
|
||||
return { duration: outputDuration, ...dimensions };
|
||||
}
|
||||
|
||||
@ -322,3 +322,13 @@ test("editing a keyframe cannot replace the active camera identity", () => {
|
||||
expect(edited.position).toEqual(key.position);
|
||||
expect(mergeDirectorCameraPose(edited, { fov: 65 }).fov).toBe(65);
|
||||
});
|
||||
|
||||
test('director appearance, pose, visibility and locking survive save/reload without changing transforms',()=>{
|
||||
const scene=defaultDirectorScene();
|
||||
Object.assign(scene.objects[0],{pose:'跑步',color:'#123abc',visible:false,locked:true});
|
||||
expect(readDirectorScene(JSON.parse(JSON.stringify(scene)))).toEqual(scene);
|
||||
const invalid=JSON.parse(JSON.stringify(scene));invalid.objects[0].pose='unknown';invalid.objects[0].color='url(evil)';
|
||||
const loaded=readDirectorScene(invalid);
|
||||
expect(loaded.objects[0].pose).toBeUndefined();expect(loaded.objects[0].color).toBeUndefined();
|
||||
expect(loaded.objects[0].position).toEqual(scene.objects[0].position);
|
||||
});
|
||||
|
||||
@ -45,3 +45,38 @@ test("archive failure preserves the generated output and cannot mark it permanen
|
||||
expect(out.metadata.permanentVideoUrl).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
import { canvasGenerationCount, canvasImageRequest, canvasImageResultUrls } from '@/lib/video-canvas/generation-contract';
|
||||
import { angleRequestParameters, anglePresets } from '@/lib/video-canvas/angle-settings';
|
||||
import { fetchCanvasDownloadAsset } from '@/lib/canvas/photoshop-safe-download';
|
||||
|
||||
test('image composer transmits resolution and all references without injecting demo content',()=>{
|
||||
for(const resolution of ['1K','2K','4K']) {
|
||||
const request=canvasImageRequest({model:'configured-image',ratio:'9:16',resolution,count:'4张'},[]);
|
||||
expect(request).toMatchObject({model:'configured-image',aspectRatio:'9:16',imageSize:resolution,imageCount:4,referenceImages:[]});
|
||||
expect(request.referenceImage).toBeUndefined();
|
||||
}
|
||||
expect(canvasImageRequest({ratio:'1:1',resolution:'2K',count:'2'},['https://a.test/1','https://a.test/1','https://a.test/2'])).toMatchObject({imageCount:2,referenceImage:'https://a.test/1',referenceImages:['https://a.test/2']});
|
||||
});
|
||||
test('all returned image results survive, while failures never expose fallback images',()=>{
|
||||
const data={images:[{originalUrl:'a',thumbnailUrl:'thumb'},{originalUrl:'b'},{thumbnailUrl:'c'},{}]};
|
||||
expect(canvasImageResultUrls({success:true,data})).toEqual(['a','b','c']);
|
||||
expect(canvasImageResultUrls({success:false,data})).toEqual([]);
|
||||
expect(['1个','2个','4个','Infinity','100','0'].map(canvasGenerationCount)).toEqual([1,2,4,1,1,1]);
|
||||
});
|
||||
test('angle controls convert degrees and percent zoom into provider units',()=>{
|
||||
expect(angleRequestParameters({horizontalAngle:-90,verticalAngle:-45,zoom:50})).toEqual({horizontalAngle:270,verticalAngle:-30,zoom:5});
|
||||
expect(angleRequestParameters(anglePresets['全景俯拍'])).toEqual({horizontalAngle:0,verticalAngle:90,zoom:0});
|
||||
expect(anglePresets['鱼眼视角'].prompt).toContain('鱼眼');
|
||||
expect(()=>angleRequestParameters({horizontalAngle:NaN,verticalAngle:0,zoom:50})).toThrow();
|
||||
});
|
||||
test('audio downloads preserve original bytes and extension instead of passing through an image decoder',async()=>{
|
||||
const original=globalThis.fetch;
|
||||
const bytes=Buffer.from('RIFF00000000WAVEfmt synthetic QA audio');
|
||||
globalThis.fetch=async()=>new Response(bytes,{headers:{'content-type':'audio/wav'}});
|
||||
try {
|
||||
const result=await fetchCanvasDownloadAsset({file:{url:'https://cdn.example/audio.wav',name:'配音'},index:0,requestOrigin:'https://canvas.example',cookie:'',retryCount:1});
|
||||
expect(result.ok).toBe(true);
|
||||
if(result.ok){expect(result.bytes).toEqual(bytes);expect(result.filename).toMatch(/\.wav$/);expect(result.contentType).toBe('audio/wav');}
|
||||
} finally {globalThis.fetch=original;}
|
||||
});
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import { processVideoFile } from "@/lib/video-canvas/media-processing";
|
||||
import { test, expect } from "@playwright/test";
|
||||
import { promptToolSchema, selectPromptToolModel, promptToolErrorMessage } from "@/lib/video-canvas/prompt-tools";
|
||||
import { toVideoTimeRange, videoProcessSchema } from "@/lib/video-canvas/media-contract";
|
||||
@ -51,3 +52,23 @@ test("invalid trim and crop requests are rejected before downloading", () => {
|
||||
expect(() => toVideoTimeRange({ start: 0, end: 1 }, NaN)).toThrow();
|
||||
expect(videoProcessSchema.safeParse({ operation: "audio-video", url }).success).toBe(true);
|
||||
});
|
||||
|
||||
test('frame and slow-motion inputs reject invalid timestamps and unbounded multipliers',()=>{
|
||||
expect(videoProcessSchema.safeParse({operation:'frame',url:'https://example.com/a.mp4',kind:'current',time:-1}).success).toBe(false);
|
||||
expect(videoProcessSchema.safeParse({operation:'frame',url:'https://example.com/a.mp4',kind:'last'}).success).toBe(true);
|
||||
for(const factor of [0,1,4,100,Infinity]) expect(videoProcessSchema.safeParse({operation:'slow',url:'https://example.com/a.mp4',factor}).success).toBe(false);
|
||||
for(const factor of [2,3,5]) expect(videoProcessSchema.safeParse({operation:'slow',url:'https://example.com/a.mp4',factor}).success).toBe(true);
|
||||
});
|
||||
|
||||
test('real FFmpeg frames differ by time and slow motion keeps synchronized audio',async()=>{
|
||||
const {execFileSync}=await import('node:child_process');
|
||||
try{execFileSync('ffmpeg',['-version'],{stdio:'ignore'});execFileSync('ffprobe',['-version'],{stdio:'ignore'});}catch{test.skip(true,'FFmpeg and ffprobe required for media integration');return;}
|
||||
const {mkdtemp,readFile,rm}=await import('node:fs/promises');const {tmpdir}=await import('node:os');const {join}=await import('node:path');
|
||||
const dir=await mkdtemp(join(tmpdir(),'canvas-media-test-'));
|
||||
try{
|
||||
const source=join(dir,'source.mp4');execFileSync('ffmpeg',['-y','-v','error','-f','lavfi','-i','testsrc2=size=320x180:rate=24:duration=2','-f','lavfi','-i','sine=frequency=440:duration=2','-c:v','libx264','-pix_fmt','yuv420p','-c:a','aac',source]);
|
||||
const frames=[];for(const kind of ['first','current','last'] as const){const file=join(dir,`${kind}.png`);const result=await processVideoFile(source,file,'frame',undefined,undefined,{kind,time:1});expect(result.width).toBe(320);expect(result.height).toBe(180);frames.push((await readFile(file)).toString('base64'));}
|
||||
expect(new Set(frames).size).toBe(3);
|
||||
for(const factor of [2,3,5]){const file=join(dir,`slow${factor}.mp4`);await processVideoFile(source,file,'slow',undefined,undefined,{factor});const info=JSON.parse(execFileSync('ffprobe',['-v','error','-show_format','-show_streams','-of','json',file],{encoding:'utf8'}));expect(Math.abs(Number(info.format.duration)-2*factor)).toBeLessThan(0.2);expect(info.streams.some((s:{codec_type:string})=>s.codec_type==='audio')).toBe(true);}
|
||||
}finally{await rm(dir,{recursive:true,force:true});}
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user