Support local preview transforms and detached smart edit audio tracks
This commit is contained in:
134
parent
6869bd8c18
commit
84bfd6b9f4
@ -1,12 +1,15 @@
|
||||
"use client";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import {
|
||||
compiledSubtitles,
|
||||
editableCaptions,
|
||||
defaultCaptionStyle,
|
||||
type CaptionStyle,
|
||||
planTimeline,
|
||||
SECOND,
|
||||
type EditPlan,
|
||||
defaultVideoTransform,
|
||||
type VideoTransform,
|
||||
type TimelineCaption,
|
||||
} from "@/lib/video-canvas/smart-edit/contract";
|
||||
export function EditPreview({
|
||||
plan,
|
||||
@ -16,8 +19,20 @@ export function EditPreview({
|
||||
onPlaying,
|
||||
onError,
|
||||
fit = false,
|
||||
selectedClipId,
|
||||
selectedCaptionId,
|
||||
onSelectClip,
|
||||
onSelectCaption,
|
||||
onTransform,
|
||||
onCaptionPosition,
|
||||
}: {
|
||||
fit?: boolean;
|
||||
selectedClipId?: string;
|
||||
selectedCaptionId?: string;
|
||||
onSelectClip?: (id: string) => void;
|
||||
onSelectCaption?: (id: string) => void;
|
||||
onTransform?: (id: string, value: VideoTransform) => void;
|
||||
onCaptionPosition?: (id: string, value: { x: number; y: number }) => void;
|
||||
plan: EditPlan;
|
||||
time: number;
|
||||
playing: boolean;
|
||||
@ -27,6 +42,7 @@ export function EditPreview({
|
||||
}) {
|
||||
const videos = useRef(new Map<string, HTMLVideoElement>()),
|
||||
music = useRef<HTMLAudioElement | null>(null),
|
||||
audios = useRef(new Map<string, HTMLAudioElement>()),
|
||||
latest = useRef({ time, plan, onTime, onPlaying });
|
||||
latest.current = { time, plan, onTime, onPlaying };
|
||||
const frameRef = useRef<HTMLDivElement>(null);
|
||||
@ -43,7 +59,9 @@ export function EditPreview({
|
||||
const timeline = planTimeline(plan),
|
||||
visibleTime = Math.min(time * SECOND, timeline.duration - 1),
|
||||
current = timeline.clips.filter((c) => visibleTime >= c.start && visibleTime < c.end),
|
||||
subs = compiledSubtitles(plan).filter((s) => time * SECOND >= s.start && time * SECOND < s.end);
|
||||
subs = plan.subtitles
|
||||
? editableCaptions(plan).filter((s) => time * SECOND >= s.start && time * SECOND < s.end)
|
||||
: [];
|
||||
useEffect(() => {
|
||||
if (!playing) return;
|
||||
let frame = 0,
|
||||
@ -94,7 +112,7 @@ export function EditPreview({
|
||||
gain = (time * SECOND - clip.start) / clip.overlap;
|
||||
if (next?.overlap && time * SECOND > next.start)
|
||||
gain = Math.min(gain, (clip.end - time * SECOND) / next.overlap);
|
||||
video.volume = Math.min(1, Math.max(0, clip.gain * gain));
|
||||
video.volume = clip.audioDetached ? 0 : Math.min(1, Math.max(0, clip.gain * gain));
|
||||
if (playing)
|
||||
void video.play().catch(() => {
|
||||
onPlaying(false);
|
||||
@ -102,6 +120,31 @@ export function EditPreview({
|
||||
});
|
||||
else video.pause();
|
||||
}
|
||||
for (const clip of timeline.audioClips) {
|
||||
const audio = audios.current.get(clip.id);
|
||||
if (!audio) continue;
|
||||
const relative = time - clip.start / SECOND;
|
||||
if (relative < 0 || time * SECOND >= clip.end) {
|
||||
audio.pause();
|
||||
continue;
|
||||
}
|
||||
const target = clip.sourceIn / SECOND + relative * clip.speed;
|
||||
if (Math.abs(audio.currentTime - target) > 0.1) audio.currentTime = target;
|
||||
audio.playbackRate = clip.speed;
|
||||
const length = clip.duration / SECOND,
|
||||
fadeIn = Math.min(clip.fadeIn, length),
|
||||
fadeOut = Math.min(clip.fadeOut, length);
|
||||
const gain =
|
||||
Math.min(1, fadeIn ? relative / fadeIn : 1) *
|
||||
Math.min(1, fadeOut ? (length - relative) / fadeOut : 1);
|
||||
audio.volume = Math.max(0, Math.min(1, clip.gain * gain));
|
||||
if (playing)
|
||||
void audio.play().catch(() => {
|
||||
onPlaying(false);
|
||||
onError("独立音轨无法播放");
|
||||
});
|
||||
else audio.pause();
|
||||
}
|
||||
const audio = music.current;
|
||||
if (audio && plan.music) {
|
||||
const target = time + plan.music.start / SECOND;
|
||||
@ -125,6 +168,116 @@ export function EditPreview({
|
||||
const scale = height / 720;
|
||||
const incomingClip = current.find((c) => c.overlap && visibleTime < c.start + c.overlap);
|
||||
const progress = incomingClip ? (visibleTime - incomingClip.start) / incomingClip.overlap : 1;
|
||||
const [gesture, setGesture] = useState<{
|
||||
clipId?: string;
|
||||
captionId?: string;
|
||||
transform?: VideoTransform;
|
||||
x?: number;
|
||||
y?: number;
|
||||
}>();
|
||||
const gestureCleanup = useRef<() => void>();
|
||||
useEffect(() => () => gestureCleanup.current?.(), []);
|
||||
const moveTransform = (e: React.PointerEvent, id: string, corner?: [number, number]) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
onPlaying(false);
|
||||
onSelectClip?.(id);
|
||||
const c = plan.clips.find((c) => c.id === id)!,
|
||||
a = plan.assets.find((a) => a.id === c.assetId)!,
|
||||
original = c.transform || defaultVideoTransform;
|
||||
const frameWidth = height * ratio,
|
||||
baseWidth = Math.min(frameWidth, (height * a.width) / a.height),
|
||||
baseHeight = (baseWidth * a.height) / a.width,
|
||||
startX = e.clientX,
|
||||
startY = e.clientY;
|
||||
let value = original;
|
||||
const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v));
|
||||
const move = (ev: PointerEvent) => {
|
||||
const dx = ev.clientX - startX,
|
||||
dy = ev.clientY - startY;
|
||||
if (corner) {
|
||||
const nextScale = clamp(
|
||||
original.scale + ((dx * corner[0]) / baseWidth + (dy * corner[1]) / baseHeight) / 2,
|
||||
0.1,
|
||||
3,
|
||||
),
|
||||
delta = nextScale - original.scale;
|
||||
value = {
|
||||
scale: nextScale,
|
||||
x: clamp(original.x + (delta * baseWidth * corner[0]) / 2 / frameWidth, -1, 1),
|
||||
y: clamp(original.y + (delta * baseHeight * corner[1]) / 2 / height, -1, 1),
|
||||
};
|
||||
} else
|
||||
value = {
|
||||
...original,
|
||||
x: clamp(original.x + dx / frameWidth, -1, 1),
|
||||
y: clamp(original.y + dy / height, -1, 1),
|
||||
};
|
||||
setGesture({ clipId: id, transform: value });
|
||||
};
|
||||
const cleanup = () => {
|
||||
window.removeEventListener("pointermove", move);
|
||||
window.removeEventListener("pointerup", end);
|
||||
window.removeEventListener("pointercancel", cancel);
|
||||
gestureCleanup.current = undefined;
|
||||
};
|
||||
const end = () => {
|
||||
cleanup();
|
||||
setGesture(undefined);
|
||||
onTransform?.(id, value);
|
||||
},
|
||||
cancel = () => {
|
||||
cleanup();
|
||||
setGesture(undefined);
|
||||
};
|
||||
gestureCleanup.current?.();
|
||||
gestureCleanup.current = cleanup;
|
||||
window.addEventListener("pointermove", move);
|
||||
window.addEventListener("pointerup", end);
|
||||
window.addEventListener("pointercancel", cancel);
|
||||
};
|
||||
const moveCaption = (e: React.PointerEvent, id: string, style: CaptionStyle) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
onPlaying(false);
|
||||
onSelectCaption?.(id);
|
||||
const startX = e.clientX,
|
||||
startY = e.clientY;
|
||||
let value = { x: style.x ?? 0.5, y: style.y ?? 0.95 };
|
||||
const move = (ev: PointerEvent) => {
|
||||
value = {
|
||||
x: Math.max(0, Math.min(1, (style.x ?? 0.5) + (ev.clientX - startX) / (height * ratio))),
|
||||
y: Math.max(0, Math.min(1, (style.y ?? 0.95) + (ev.clientY - startY) / height)),
|
||||
};
|
||||
setGesture({ captionId: id, ...value });
|
||||
};
|
||||
const cleanup = () => {
|
||||
window.removeEventListener("pointermove", move);
|
||||
window.removeEventListener("pointerup", end);
|
||||
window.removeEventListener("pointercancel", cancel);
|
||||
gestureCleanup.current = undefined;
|
||||
};
|
||||
const end = () => {
|
||||
cleanup();
|
||||
setGesture(undefined);
|
||||
onCaptionPosition?.(id, value);
|
||||
},
|
||||
cancel = () => {
|
||||
cleanup();
|
||||
setGesture(undefined);
|
||||
};
|
||||
gestureCleanup.current?.();
|
||||
gestureCleanup.current = cleanup;
|
||||
window.addEventListener("pointermove", move);
|
||||
window.addEventListener("pointerup", end);
|
||||
window.addEventListener("pointercancel", cancel);
|
||||
};
|
||||
const selectedClip = current.find((c) => c.id === selectedClipId);
|
||||
const selectedAsset = selectedClip && plan.assets.find((a) => a.id === selectedClip.assetId);
|
||||
const selectedTransform =
|
||||
gesture && gesture.clipId === selectedClipId
|
||||
? gesture.transform
|
||||
: selectedClip?.transform || defaultVideoTransform;
|
||||
return (
|
||||
<div
|
||||
ref={frameRef}
|
||||
@ -169,6 +322,13 @@ export function EditPreview({
|
||||
className="absolute inset-0 h-full w-full object-contain"
|
||||
style={{
|
||||
visibility: active ? "visible" : "hidden",
|
||||
transform: (() => {
|
||||
const t =
|
||||
gesture?.clipId === clip.id
|
||||
? gesture.transform!
|
||||
: clip.transform || defaultVideoTransform;
|
||||
return `translate(${t.x * 100}%,${t.y * 100}%) scale(${t.scale})`;
|
||||
})(),
|
||||
opacity: incoming
|
||||
? clip.transitionType === "fade"
|
||||
? progress
|
||||
@ -190,6 +350,17 @@ export function EditPreview({
|
||||
/>
|
||||
);
|
||||
})}
|
||||
{timeline.audioClips.map((c) => (
|
||||
<audio
|
||||
key={c.id}
|
||||
ref={(el) => {
|
||||
if (el) audios.current.set(c.id, el);
|
||||
else audios.current.delete(c.id);
|
||||
}}
|
||||
src={plan.assets.find((a) => a.id === c.assetId)?.url}
|
||||
preload="metadata"
|
||||
/>
|
||||
))}
|
||||
{plan.music ? <audio ref={music} src={plan.music.url} preload="metadata" /> : null}
|
||||
{incomingClip && incomingClip.transitionType !== "fade" && (
|
||||
<div
|
||||
@ -202,13 +373,78 @@ export function EditPreview({
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
<div className="pointer-events-none absolute inset-x-4 bottom-[5%] text-center">
|
||||
{selectedClip && selectedAsset && selectedTransform && onTransform && (
|
||||
<div
|
||||
aria-label="调整画面位置"
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
onPointerDown={(e) => moveTransform(e, selectedClip.id)}
|
||||
style={{
|
||||
position: "absolute",
|
||||
left: `${(selectedTransform.x + 0.5) * 100}%`,
|
||||
top: `${(selectedTransform.y + 0.5) * 100}%`,
|
||||
width:
|
||||
Math.min(height * ratio, (height * selectedAsset.width) / selectedAsset.height) *
|
||||
selectedTransform.scale,
|
||||
height:
|
||||
Math.min(height, (height * ratio * selectedAsset.height) / selectedAsset.width) *
|
||||
selectedTransform.scale,
|
||||
transform: "translate(-50%,-50%)",
|
||||
border: "2px solid #87c9ff",
|
||||
cursor: "move",
|
||||
zIndex: 3,
|
||||
}}
|
||||
>
|
||||
{(
|
||||
[
|
||||
[-1, -1],
|
||||
[1, -1],
|
||||
[1, 1],
|
||||
[-1, 1],
|
||||
] as [number, number][]
|
||||
).map(([x, y]) => (
|
||||
<button
|
||||
key={`${x}:${y}`}
|
||||
type="button"
|
||||
aria-label="调整视频尺寸"
|
||||
onPointerDown={(e) => moveTransform(e, selectedClip.id, [x, y])}
|
||||
style={{
|
||||
position: "absolute",
|
||||
width: 12,
|
||||
height: 12,
|
||||
left: x < 0 ? 0 : "100%",
|
||||
top: y < 0 ? 0 : "100%",
|
||||
transform: "translate(-50%,-50%)",
|
||||
border: "1px solid white",
|
||||
background: "#87c9ff",
|
||||
borderRadius: 2,
|
||||
cursor: x === y ? "nwse-resize" : "nesw-resize",
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<div className="pointer-events-none absolute inset-0 text-center" style={{ zIndex: 4 }}>
|
||||
{subs.map((s, i) => {
|
||||
const style = (s as { style?: CaptionStyle }).style || defaultCaptionStyle;
|
||||
return (
|
||||
<div
|
||||
key={i}
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
onPointerDown={(e) => {
|
||||
if (onCaptionPosition && "id" in s) moveCaption(e, s.id as string, style);
|
||||
}}
|
||||
style={{
|
||||
position: "absolute",
|
||||
left: `${((gesture && gesture.captionId === (s as TimelineCaption).id ? gesture.x : style.x) ?? 0.5) * 100}%`,
|
||||
bottom: `${(1 - ((gesture && gesture.captionId === (s as TimelineCaption).id ? gesture.y : style.y) ?? 0.95)) * 100}%`,
|
||||
transform: "translateX(-50%)",
|
||||
maxWidth: "95%",
|
||||
pointerEvents: onCaptionPosition ? "auto" : "none",
|
||||
cursor: onCaptionPosition ? "move" : undefined,
|
||||
outline:
|
||||
selectedCaptionId === (s as TimelineCaption).id
|
||||
? "1px solid #87c9ff"
|
||||
: undefined,
|
||||
fontFamily: '"Noto Sans CJK SC", "Source Han Sans CN", sans-serif',
|
||||
whiteSpace: "pre-wrap",
|
||||
fontSize: style.fontSize * scale,
|
||||
|
||||
@ -910,3 +910,44 @@
|
||||
left: auto;
|
||||
right: 0;
|
||||
}
|
||||
|
||||
.root :global(.se-reset-transform) {
|
||||
width: 100%;
|
||||
height: 30px;
|
||||
border: 1px solid #444;
|
||||
border-radius: 8px;
|
||||
font-size: 12px;
|
||||
margin: 0 0 12px;
|
||||
}
|
||||
.root :global(.se-audio-track) {
|
||||
height: 44px;
|
||||
position: relative;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
.root :global(.se-audio-clip) {
|
||||
height: 44px;
|
||||
position: absolute;
|
||||
background: #285f61;
|
||||
border: 1px solid #397d7d;
|
||||
border-radius: 7px;
|
||||
overflow: hidden;
|
||||
cursor: grab;
|
||||
padding: 5px 8px;
|
||||
font-size: 11px;
|
||||
white-space: nowrap;
|
||||
touch-action: none;
|
||||
user-select: none;
|
||||
}
|
||||
.root :global(.se-audio-clip.selected) {
|
||||
outline: 2px solid #fff;
|
||||
outline-offset: -2px;
|
||||
}
|
||||
.root :global(.se-track-label.audio) {
|
||||
height: 52px;
|
||||
}
|
||||
.root :global(.se-track-labels) {
|
||||
overflow: hidden;
|
||||
}
|
||||
.root :global(.se-track-label:not(.audio):not(.caption)) {
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
@ -33,6 +33,8 @@ import {
|
||||
type EditPlan,
|
||||
type TimelineCaption,
|
||||
type CaptionStyle,
|
||||
type AudioClip,
|
||||
defaultVideoTransform,
|
||||
} from "@/lib/video-canvas/smart-edit/contract";
|
||||
import type { CanvasGenerationTask } from "@/lib/video-canvas/task-contract";
|
||||
import { downloadCanvasMedia } from "@/lib/video-canvas/download";
|
||||
@ -114,7 +116,8 @@ export function SmartEditEditor({
|
||||
[playing, setPlaying] = useState(false),
|
||||
[selected, setSelected] = useState<string>(),
|
||||
[draggedClip, setDraggedClip] = useState<{ id: string; offset: number }>(),
|
||||
[panel, setPanel] = useState<"video" | "caption" | "transition" | null>(null),
|
||||
[panel, setPanel] = useState<"video" | "audio" | "caption" | "transition" | null>(null),
|
||||
[audioId, setAudioId] = useState<string>(),
|
||||
[panelWidth, setPanelWidth] = useState(300),
|
||||
[libraryWidth, setLibraryWidth] = useState(372),
|
||||
[captionsOpen, setCaptionsOpen] = useState(false),
|
||||
@ -138,11 +141,12 @@ export function SmartEditEditor({
|
||||
[context, setContext] = useState<{
|
||||
x: number;
|
||||
y: number;
|
||||
kind: "clip" | "caption" | "timeline";
|
||||
kind: "clip" | "audio" | "caption" | "timeline";
|
||||
}>();
|
||||
const root = useRef<HTMLDivElement>(null),
|
||||
preview = useRef<HTMLDivElement>(null),
|
||||
scroll = useRef<HTMLDivElement>(null);
|
||||
scroll = useRef<HTMLDivElement>(null),
|
||||
trackLabels = useRef<HTMLDivElement>(null);
|
||||
const resized = useRef(false);
|
||||
useEffect(() => {
|
||||
const resize = () => {
|
||||
@ -162,6 +166,7 @@ export function SmartEditEditor({
|
||||
pps = 25 * 2 ** (zoom / 10),
|
||||
clipIndex = plan.clips.findIndex((c) => c.id === selected),
|
||||
clip = plan.clips[clipIndex],
|
||||
audioClip = plan.audioClips?.find((c) => c.id === audioId),
|
||||
captions = editableCaptions(plan),
|
||||
caption = captions.find((s) => s.id === captionId),
|
||||
captionStyle = caption?.style || defaultCaptionStyle;
|
||||
@ -190,6 +195,43 @@ export function SmartEditEditor({
|
||||
};
|
||||
const changeClip = (index: number, patch: Partial<EditPlan["clips"][number]>) =>
|
||||
apply({ ...plan, clips: plan.clips.map((c, i) => (i === index ? { ...c, ...patch } : c)) });
|
||||
const changeAudio = (id: string, patch: Partial<AudioClip>) =>
|
||||
apply({
|
||||
...plan,
|
||||
audioClips: (plan.audioClips || []).map((c) => (c.id === id ? { ...c, ...patch } : c)),
|
||||
});
|
||||
const detachAudio = () => {
|
||||
if (!clip || clip.audioDetached || !plan.assets.find((a) => a.id === clip.assetId)?.hasAudio)
|
||||
return;
|
||||
if ((plan.audioClips?.length || 0) >= 24) {
|
||||
setError("最多24个独立音频片段");
|
||||
return;
|
||||
}
|
||||
const id = crypto.randomUUID(),
|
||||
t = timeline.clips[clipIndex];
|
||||
apply({
|
||||
...plan,
|
||||
clips: plan.clips.map((c) => (c.id === clip.id ? { ...c, audioDetached: true } : c)),
|
||||
audioClips: [
|
||||
...(plan.audioClips || []),
|
||||
{
|
||||
id,
|
||||
assetId: clip.assetId,
|
||||
sourceIn: clip.sourceIn,
|
||||
sourceOut: clip.sourceOut,
|
||||
speed: clip.speed,
|
||||
gain: clip.gain,
|
||||
start: t.start,
|
||||
fadeIn: t.overlap / SECOND,
|
||||
fadeOut: (timeline.clips[clipIndex + 1]?.overlap || 0) / SECOND,
|
||||
},
|
||||
],
|
||||
});
|
||||
setSelected(undefined);
|
||||
setAudioId(id);
|
||||
setPanel("audio");
|
||||
setContext(undefined);
|
||||
};
|
||||
const changeCaption = (id: string, patch: Partial<TimelineCaption>) =>
|
||||
apply({ ...plan, captions: captions.map((s) => (s.id === id ? { ...s, ...patch } : s)) });
|
||||
const changeStyle = (patch: Partial<CaptionStyle>) => {
|
||||
@ -212,10 +254,13 @@ export function SmartEditEditor({
|
||||
time,
|
||||
duration,
|
||||
...timeline.clips.flatMap((c) => [c.start / SECOND, c.end / SECOND]),
|
||||
...timeline.audioClips.flatMap((c) => [c.start / SECOND, c.end / SECOND]),
|
||||
];
|
||||
return values.find((v) => Math.abs(v - seconds) * pps < 8) ?? seconds;
|
||||
};
|
||||
const [, setHistoryRevision] = useState(0);
|
||||
const dragCleanup = useRef<() => void>();
|
||||
useEffect(() => () => dragCleanup.current?.(), []);
|
||||
const drag = (
|
||||
e: PointerEvent,
|
||||
move: (event: globalThis.PointerEvent) => void,
|
||||
@ -223,23 +268,27 @@ export function SmartEditEditor({
|
||||
) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
dragCleanup.current?.();
|
||||
const original = latest.current;
|
||||
const el = e.currentTarget as HTMLElement;
|
||||
el.setPointerCapture(e.pointerId);
|
||||
const end = () => {
|
||||
const cleanup = () => {
|
||||
window.removeEventListener("pointermove", move);
|
||||
window.removeEventListener("pointerup", end);
|
||||
window.removeEventListener("pointercancel", cancel);
|
||||
dragCleanup.current = undefined;
|
||||
};
|
||||
const end = () => {
|
||||
cleanup();
|
||||
finish?.();
|
||||
setHistoryRevision((value) => value + 1);
|
||||
};
|
||||
const cancel = () => {
|
||||
setDraggedClip(undefined);
|
||||
if (latest.current !== original) onChange(original);
|
||||
window.removeEventListener("pointermove", move);
|
||||
window.removeEventListener("pointerup", end);
|
||||
window.removeEventListener("pointercancel", cancel);
|
||||
cleanup();
|
||||
};
|
||||
dragCleanup.current = cleanup;
|
||||
window.addEventListener("pointermove", move);
|
||||
window.addEventListener("pointerup", end);
|
||||
window.addEventListener("pointercancel", cancel);
|
||||
@ -350,13 +399,111 @@ export function SmartEditEditor({
|
||||
},
|
||||
);
|
||||
};
|
||||
const dragAudio = (e: PointerEvent, c: AudioClip, side?: "left" | "right") => {
|
||||
const original = plan,
|
||||
x = e.clientX,
|
||||
asset = plan.assets.find((a) => a.id === c.assetId)!;
|
||||
setAudioId(c.id);
|
||||
setSelected(undefined);
|
||||
setPanel("audio");
|
||||
setPlaying(false);
|
||||
drag(
|
||||
e,
|
||||
(event) => {
|
||||
const delta = ((event.clientX - x) / pps) * SECOND;
|
||||
let patch: Partial<AudioClip>;
|
||||
if (side === "left") {
|
||||
const sourceIn = Math.round(
|
||||
clamp(
|
||||
c.sourceIn + delta * c.speed,
|
||||
Math.max(0, c.sourceIn - c.start * c.speed),
|
||||
c.sourceOut - 200000,
|
||||
),
|
||||
);
|
||||
patch = { sourceIn, start: Math.round(c.start + (sourceIn - c.sourceIn) / c.speed) };
|
||||
} else if (side === "right")
|
||||
patch = {
|
||||
sourceOut: Math.round(
|
||||
clamp(
|
||||
c.sourceOut + delta * c.speed,
|
||||
c.sourceIn + 200000,
|
||||
Math.min(asset.duration, c.sourceIn + (300 * SECOND - c.start) * c.speed),
|
||||
),
|
||||
),
|
||||
};
|
||||
else
|
||||
patch = {
|
||||
start: Math.round(
|
||||
clamp(
|
||||
snapped((c.start + delta) / SECOND) * SECOND,
|
||||
0,
|
||||
300 * SECOND - (c.sourceOut - c.sourceIn) / c.speed,
|
||||
),
|
||||
),
|
||||
};
|
||||
const next = {
|
||||
...original,
|
||||
audioClips: original.audioClips!.map((a) => (a.id === c.id ? { ...a, ...patch } : a)),
|
||||
};
|
||||
try {
|
||||
planTimeline(next);
|
||||
onChange(next);
|
||||
} catch {}
|
||||
},
|
||||
() => {
|
||||
undo.current.push(original);
|
||||
redo.current = [];
|
||||
},
|
||||
);
|
||||
};
|
||||
const audioCutPoint = audioClip
|
||||
? Math.round(audioClip.sourceIn + (time * SECOND - audioClip.start) * audioClip.speed)
|
||||
: 0;
|
||||
const canCutAudio = Boolean(
|
||||
panel === "audio" &&
|
||||
audioClip &&
|
||||
audioCutPoint - audioClip.sourceIn >= 200000 &&
|
||||
audioClip.sourceOut - audioCutPoint >= 200000,
|
||||
);
|
||||
const cutPoint = clip
|
||||
? Math.round(clip.sourceIn + (time * SECOND - timeline.clips[clipIndex].start) * clip.speed)
|
||||
: 0;
|
||||
const canCut = Boolean(
|
||||
clip && cutPoint - clip.sourceIn >= 200000 && clip.sourceOut - cutPoint >= 200000,
|
||||
);
|
||||
const canCut =
|
||||
canCutAudio ||
|
||||
Boolean(clip && cutPoint - clip.sourceIn >= 200000 && clip.sourceOut - cutPoint >= 200000);
|
||||
const cut = (kind: "split" | "left" | "right") => {
|
||||
if (panel === "audio" && audioClip && canCutAudio) {
|
||||
if (kind === "split") {
|
||||
if ((plan.audioClips?.length || 0) >= 24) {
|
||||
setError("最多24个独立音频片段");
|
||||
return;
|
||||
}
|
||||
apply({
|
||||
...plan,
|
||||
audioClips: plan.audioClips!.flatMap((c) =>
|
||||
c.id !== audioClip.id
|
||||
? [c]
|
||||
: [
|
||||
{ ...c, sourceOut: audioCutPoint, fadeOut: 0 },
|
||||
{
|
||||
...c,
|
||||
id: crypto.randomUUID(),
|
||||
sourceIn: audioCutPoint,
|
||||
start: Math.round(time * SECOND),
|
||||
fadeIn: 0,
|
||||
},
|
||||
],
|
||||
),
|
||||
});
|
||||
} else
|
||||
changeAudio(
|
||||
audioClip.id,
|
||||
kind === "left"
|
||||
? { sourceIn: audioCutPoint, start: Math.round(time * SECOND) }
|
||||
: { sourceOut: audioCutPoint },
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!clip || !canCut) return;
|
||||
if (kind === "split") {
|
||||
if (plan.clips.length >= 12) {
|
||||
@ -381,7 +528,11 @@ export function SmartEditEditor({
|
||||
changeClip(clipIndex, kind === "left" ? { sourceIn: cutPoint } : { sourceOut: cutPoint });
|
||||
};
|
||||
const remove = () => {
|
||||
if (panel === "caption" && caption) {
|
||||
if (panel === "audio" && audioClip) {
|
||||
apply({ ...plan, audioClips: plan.audioClips!.filter((c) => c.id !== audioClip.id) });
|
||||
setAudioId(undefined);
|
||||
setPanel(null);
|
||||
} else if (panel === "caption" && caption) {
|
||||
apply({ ...plan, captions: captions.filter((s) => s.id !== caption.id) });
|
||||
setCaptionId(undefined);
|
||||
} else if (clip) {
|
||||
@ -498,7 +649,7 @@ export function SmartEditEditor({
|
||||
}
|
||||
changeClip(transitionIndex, { transition: { type, duration: value } });
|
||||
};
|
||||
const openContext = (e: React.MouseEvent, kind: "clip" | "caption" | "timeline") => {
|
||||
const openContext = (e: React.MouseEvent, kind: "clip" | "audio" | "caption" | "timeline") => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
setContext({
|
||||
@ -639,6 +790,33 @@ export function SmartEditEditor({
|
||||
onPlaying={setPlaying}
|
||||
onError={setError}
|
||||
fit
|
||||
selectedClipId={selected}
|
||||
selectedCaptionId={captionId}
|
||||
onSelectClip={(id) => {
|
||||
setSelected(id);
|
||||
setPanel("video");
|
||||
setPlaying(false);
|
||||
}}
|
||||
onSelectCaption={(id) => {
|
||||
setCaptionId(id);
|
||||
setPanel("caption");
|
||||
setSubTab("text");
|
||||
setCaptionsOpen(true);
|
||||
setPlaying(false);
|
||||
}}
|
||||
onTransform={(id, transform) =>
|
||||
changeClip(
|
||||
plan.clips.findIndex((c) => c.id === id),
|
||||
{ transform },
|
||||
)
|
||||
}
|
||||
onCaptionPosition={(id, position) => {
|
||||
const c = captions.find((c) => c.id === id);
|
||||
if (c)
|
||||
changeCaption(id, {
|
||||
style: { ...(c.style || defaultCaptionStyle), ...position },
|
||||
});
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
{exportWaiting && (
|
||||
@ -663,6 +841,14 @@ export function SmartEditEditor({
|
||||
</Icon>
|
||||
</div>
|
||||
<div className="se-panel-scroll">
|
||||
{clip.transform && (
|
||||
<button
|
||||
className="se-reset-transform"
|
||||
onClick={() => changeClip(clipIndex, { transform: defaultVideoTransform })}
|
||||
>
|
||||
重置画面位置与缩放
|
||||
</button>
|
||||
)}
|
||||
<h3>变速</h3>
|
||||
<ValueField
|
||||
label="变速倍数"
|
||||
@ -704,6 +890,67 @@ export function SmartEditEditor({
|
||||
</div>
|
||||
</aside>
|
||||
)}
|
||||
{panel === "audio" && audioClip && (
|
||||
<aside className="se-panel" aria-label="音频参数" style={{ width: panelWidth }}>
|
||||
<div className="se-panel-head">
|
||||
音频
|
||||
<Icon title="关闭音频参数" onClick={() => setPanel(null)}>
|
||||
<X />
|
||||
</Icon>
|
||||
</div>
|
||||
<div className="se-panel-scroll">
|
||||
<h3>变速</h3>
|
||||
<ValueField
|
||||
label="音频变速倍数"
|
||||
suffix="ax"
|
||||
value={audioClip.speed}
|
||||
min={0.5}
|
||||
max={2}
|
||||
precision={2}
|
||||
onChange={(speed) => changeAudio(audioClip.id, { speed })}
|
||||
/>
|
||||
<div className="se-divider" />
|
||||
<h3>音量</h3>
|
||||
<ValueField
|
||||
label="音频音量增益"
|
||||
suffix="adB"
|
||||
value={audioClip.gain ? 20 * Math.log10(audioClip.gain) : -60}
|
||||
min={-60}
|
||||
max={0}
|
||||
precision={1}
|
||||
onChange={(v) =>
|
||||
changeAudio(audioClip.id, { gain: v <= -60 ? 0 : 10 ** (v / 20) })
|
||||
}
|
||||
/>
|
||||
<div className="se-divider" />
|
||||
<h3>淡入淡出</h3>
|
||||
<ValueField
|
||||
label="淡入时长"
|
||||
suffix="in"
|
||||
value={audioClip.fadeIn}
|
||||
min={0}
|
||||
max={Math.min(
|
||||
5,
|
||||
(audioClip.sourceOut - audioClip.sourceIn) / audioClip.speed / SECOND,
|
||||
)}
|
||||
precision={1}
|
||||
onChange={(fadeIn) => changeAudio(audioClip.id, { fadeIn })}
|
||||
/>
|
||||
<ValueField
|
||||
label="淡出时长"
|
||||
suffix="out"
|
||||
value={audioClip.fadeOut}
|
||||
min={0}
|
||||
max={Math.min(
|
||||
5,
|
||||
(audioClip.sourceOut - audioClip.sourceIn) / audioClip.speed / SECOND,
|
||||
)}
|
||||
precision={1}
|
||||
onChange={(fadeOut) => changeAudio(audioClip.id, { fadeOut })}
|
||||
/>
|
||||
</div>
|
||||
</aside>
|
||||
)}
|
||||
{panel === "transition" && (
|
||||
<aside className="se-panel" aria-label="转场参数" style={{ width: panelWidth }}>
|
||||
<div
|
||||
@ -1009,7 +1256,10 @@ export function SmartEditEditor({
|
||||
</Icon>
|
||||
<Icon
|
||||
title="分割"
|
||||
disabled={!canCut || plan.clips.length >= 12}
|
||||
disabled={
|
||||
!canCut ||
|
||||
(panel === "audio" ? (plan.audioClips?.length || 0) >= 24 : plan.clips.length >= 12)
|
||||
}
|
||||
onClick={() => cut("split")}
|
||||
>
|
||||
<CutIcon />
|
||||
@ -1093,7 +1343,7 @@ export function SmartEditEditor({
|
||||
</div>
|
||||
</div>
|
||||
<div className="se-tracks">
|
||||
<div className="se-track-labels">
|
||||
<div className="se-track-labels" ref={trackLabels}>
|
||||
{captionsOpen && (
|
||||
<div className="se-track-label caption">
|
||||
<Type size={14} />
|
||||
@ -1124,8 +1374,25 @@ export function SmartEditEditor({
|
||||
{plan.clips.some((c) => c.gain > 0) ? <Volume2 /> : <VolumeX />}
|
||||
</Icon>
|
||||
</div>
|
||||
{(plan.audioClips || []).map((c) => (
|
||||
<div className="se-track-label audio" key={c.id}>
|
||||
<Volume2 size={14} />
|
||||
<Icon
|
||||
title={c.gain ? "静音独立音轨" : "恢复独立音轨"}
|
||||
onClick={() => changeAudio(c.id, { gain: c.gain ? 0 : 1 })}
|
||||
>
|
||||
{c.gain ? <Volume2 /> : <VolumeX />}
|
||||
</Icon>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
<div className="se-scroll" ref={scroll}>
|
||||
<div
|
||||
className="se-scroll"
|
||||
ref={scroll}
|
||||
onScroll={(e) => {
|
||||
if (trackLabels.current) trackLabels.current.scrollTop = e.currentTarget.scrollTop;
|
||||
}}
|
||||
>
|
||||
<div
|
||||
className="se-track-content"
|
||||
style={{ width: Math.max(duration + 1, (viewportWidth - 112) / pps) * pps }}
|
||||
@ -1326,6 +1593,50 @@ export function SmartEditEditor({
|
||||
),
|
||||
)}
|
||||
</div>
|
||||
{timeline.audioClips.map((c) => (
|
||||
<div className="se-audio-track" key={c.id} aria-label="独立音频时间轴">
|
||||
<div
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
aria-label={`独立音频：${plan.assets.find((a) => a.id === c.assetId)?.name}`}
|
||||
className={`se-audio-clip ${audioId === c.id ? "selected" : ""}`}
|
||||
style={{ left: (c.start / SECOND) * pps, width: (c.duration / SECOND) * pps }}
|
||||
onPointerDown={(e) => dragAudio(e, c)}
|
||||
onClick={() => {
|
||||
setAudioId(c.id);
|
||||
setSelected(undefined);
|
||||
setPanel("audio");
|
||||
}}
|
||||
onContextMenu={(e) => {
|
||||
setAudioId(c.id);
|
||||
setSelected(undefined);
|
||||
setPanel("audio");
|
||||
openContext(e, "audio");
|
||||
}}
|
||||
>
|
||||
<span>
|
||||
{plan.assets.find((a) => a.id === c.assetId)?.name}{" "}
|
||||
{frameStamp(c.duration / SECOND)}
|
||||
</span>
|
||||
{audioId === c.id && (
|
||||
<>
|
||||
<div
|
||||
role="slider"
|
||||
aria-label="音频起点"
|
||||
className="se-handle left"
|
||||
onPointerDown={(e) => dragAudio(e, c, "left")}
|
||||
/>
|
||||
<div
|
||||
role="slider"
|
||||
aria-label="音频终点"
|
||||
className="se-handle right"
|
||||
onPointerDown={(e) => dragAudio(e, c, "right")}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div className="se-playhead" aria-label="播放头" style={{ left: time * pps }} />
|
||||
</div>
|
||||
</div>
|
||||
@ -1338,9 +1649,27 @@ export function SmartEditEditor({
|
||||
style={{ left: context.x, top: context.y }}
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
>
|
||||
{context.kind === "clip" && (
|
||||
<button
|
||||
role="menuitem"
|
||||
disabled={
|
||||
!clip ||
|
||||
clip.audioDetached ||
|
||||
!plan.assets.find((a) => a.id === clip.assetId)?.hasAudio
|
||||
}
|
||||
title={
|
||||
!plan.assets.find((a) => a.id === clip?.assetId)?.hasAudio
|
||||
? "当前视频无音轨，无法分离音视频"
|
||||
: undefined
|
||||
}
|
||||
onClick={detachAudio}
|
||||
>
|
||||
音视频分离
|
||||
</button>
|
||||
)}
|
||||
{context.kind !== "timeline" && (
|
||||
<button role="menuitem" onClick={remove}>
|
||||
删除{context.kind === "clip" ? "片段" : "字幕"}
|
||||
删除{context.kind === "clip" ? "片段" : context.kind === "audio" ? "音频" : "字幕"}
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
@ -1490,13 +1819,27 @@ function ValueField({
|
||||
if (e.key === "Enter") e.currentTarget.blur();
|
||||
}}
|
||||
/>
|
||||
<span>{suffix}</span>
|
||||
<span>
|
||||
{suffix === "ax"
|
||||
? "x"
|
||||
: suffix === "adB"
|
||||
? "dB"
|
||||
: suffix === "in" || suffix === "out"
|
||||
? "s"
|
||||
: suffix}
|
||||
</span>
|
||||
<div className="se-number-steps">
|
||||
<button
|
||||
type="button"
|
||||
aria-label={`增加${label}`}
|
||||
onClick={() =>
|
||||
onChange(clamp(value + (suffix === "dB" ? 1 : suffix === "x" ? 0.05 : 0.1), min, max))
|
||||
onChange(
|
||||
clamp(
|
||||
value + (suffix.endsWith("dB") ? 1 : suffix.endsWith("x") ? 0.05 : 0.1),
|
||||
min,
|
||||
max,
|
||||
),
|
||||
)
|
||||
}
|
||||
>
|
||||
⌃
|
||||
@ -1505,7 +1848,13 @@ function ValueField({
|
||||
type="button"
|
||||
aria-label={`减少${label}`}
|
||||
onClick={() =>
|
||||
onChange(clamp(value - (suffix === "dB" ? 1 : suffix === "x" ? 0.05 : 0.1), min, max))
|
||||
onChange(
|
||||
clamp(
|
||||
value - (suffix.endsWith("dB") ? 1 : suffix.endsWith("x") ? 0.05 : 0.1),
|
||||
min,
|
||||
max,
|
||||
),
|
||||
)
|
||||
}
|
||||
>
|
||||
⌄
|
||||
|
||||
@ -30,3 +30,19 @@
|
||||
- FunASR权重仍未安装，自动转写保持不可用；可导入SRT、手工编辑字幕。完整复杂对象多轨、音频等其他模块的状态不由这份记录扩大承诺。
|
||||
|
||||
部署状态以 `video-render-service-2026-09-17.md` 最后更新为准。
|
||||
|
||||
## 本地补齐与发布冻结
|
||||
|
||||
用户要求“在本地测试确保功能都全了再更新到服务器”后，停止服务器更新。以下为此后仅在本地实现、验证的增量，不能视为已经上线。
|
||||
|
||||
| 子功能 | 当前实现 | 本地验证 |
|
||||
| --- | --- | --- |
|
||||
| 预览画面变换 | 拖动画面、四角等比例缩放、重置、撤销；导出保留位置/缩放 | 浏览器实操；真实成片采样确认移动后的颜色区域和黑色留白 |
|
||||
| 字幕画面位置 | 在预览中拖动字幕；受控坐标生成ASS位置，用户文字继续转义 | 浏览器拖动；真实字幕输出像素区域检查；ASS注入回归通过 |
|
||||
| 音视频分离 | 有音轨视频右键分离；原视频静音，音频独立保存，不随后续视频排序移动 | 浏览器分离、拖动、分割、撤销；真实音频RMS确认音量未翻倍 |
|
||||
| 独立音轨编辑 | 起止裁剪、变速、音量、淡入淡出、静音、删除、最多24片段 | 合同边界和真实2倍速/延迟/增益/淡入淡出验证；超过视频尾部输出黑场 |
|
||||
| 转场库联动 | 三种基础转场，接点选择、时长和删除；与字幕/画面变换/独立音轨共用时间轴 | 三种转场再次通过FFmpeg像素检验；浏览器黑场转场操作通过 |
|
||||
|
||||
本轮类型检查通过，智能剪辑合同测试15项通过；真实FFmpeg覆盖既有两种全局转场、三种逐接点转场、六种比例及新增画面/音频/字幕编辑。浏览器测试仍使用真实组件的独立本地入口，不把模拟任务回调当作线上端到端验证。转场库与完整特效库是不同范围：参考站目前展开可见的基础转场为三项，未确认的其他特效不能登记为已完成。
|
||||
|
||||
证据保存在本地 `.qa/smart-edit/local-extension-helpers.log`、`local-extension-tsc.log`、`local-extension-ffmpeg.log` 和 `transition-library-local.png`。这些测试产物不随源码提交。
|
||||
|
||||
@ -32,3 +32,9 @@ FunASR适配与安装脚本已实现，未装权重、未做真实ASR验收，
|
||||
用户再次要求按参考站像素级对齐。新增独立编辑器和画布节点输入卡片，逐项记录见 `smart-edit-ui-parity-2026-09-17.md`。新增手动字幕样式、逐片段三种转场、六种比例、480P和300秒上限，主项目与算力服务共用合同；修正远程返回结果旧90秒/1920限制。真实300秒输出发现AAC封装尾差，输出探测及校验保留0.2/0.15秒容差。
|
||||
|
||||
48候选应用仍运行独立目录 `/var/www/ai-chatbot-render-0917` 和回环3135，公网应用3125尚未切换。候选旧版已完成真实Next API鉴权、排队、选片、节拍、远程渲染和归档测试；新UI必须完成候选重建与验证后再更新此状态。不要将算力服务已部署理解成主站UI已上线。
|
||||
|
||||
## 最新状态：本地验收优先，停止服务器更新
|
||||
|
||||
上述候选随后已在 `6869bd8c` 完成构建及启动；七个素材、手工字幕、21:9/480P、重复提交幂等、真实Next API到远程渲染和对象存储回传通过。此时公网仍为3125旧应用，未切换。算力服务当时的包为 `0f7d1c0eeb42971c`。
|
||||
|
||||
用户进一步要求先在本地补齐和测试特效等缺项，再更新服务器。因此此后的画面变换、字幕坐标和独立音轨等增量只在本地开发验证，尚未同步至候选或算力服务；需要将主项目与算力服务的共享合同同步发布后才能在线使用。完成已确认功能不代表全部参考站子功能已核对完毕，继续对照的记录见智能剪辑界面验收文档。
|
||||
|
||||
@ -30,6 +30,8 @@ export const captionStyleSchema = z.object({
|
||||
outlineWidth: z.number().min(0).max(10).default(0),
|
||||
opacity: z.number().min(0).max(1).default(1),
|
||||
spacing: z.number().min(-5).max(30).default(0),
|
||||
x: z.number().min(0).max(1).optional(),
|
||||
y: z.number().min(0).max(1).optional(),
|
||||
});
|
||||
export const timelineCaptionSchema = z
|
||||
.object({
|
||||
@ -63,6 +65,13 @@ export const editAssetSchema = assetInputSchema.extend({
|
||||
export const candidateSchema = z
|
||||
.object({ id, assetId: id, start: time, end: time, text: z.string().max(1500).default("") })
|
||||
.refine((x) => x.end - x.start >= 200000);
|
||||
export const videoTransformSchema = z.object({
|
||||
x: z.number().min(-1).max(1).default(0),
|
||||
y: z.number().min(-1).max(1).default(0),
|
||||
scale: z.number().min(0.1).max(3).default(1),
|
||||
});
|
||||
export type VideoTransform = z.infer<typeof videoTransformSchema>;
|
||||
export const defaultVideoTransform: VideoTransform = { x: 0, y: 0, scale: 1 };
|
||||
export const editClipSchema = z
|
||||
.object({
|
||||
id,
|
||||
@ -74,8 +83,24 @@ export const editClipSchema = z
|
||||
label: z.string().max(300).default(""),
|
||||
// The transition belongs to the incoming clip, so it follows that clip when reordered.
|
||||
transition: transitionSchema.optional(),
|
||||
transform: videoTransformSchema.optional(),
|
||||
audioDetached: z.boolean().optional(),
|
||||
})
|
||||
.refine((x) => x.sourceOut - x.sourceIn >= 200000);
|
||||
export const audioClipSchema = z
|
||||
.object({
|
||||
id,
|
||||
assetId: id,
|
||||
start: time,
|
||||
sourceIn: time,
|
||||
sourceOut: time,
|
||||
speed: z.number().min(0.5).max(2).default(1),
|
||||
gain: z.number().min(0).max(1).default(1),
|
||||
fadeIn: z.number().min(0).max(5).default(0),
|
||||
fadeOut: z.number().min(0).max(5).default(0),
|
||||
})
|
||||
.refine((c) => c.sourceOut - c.sourceIn >= 200000);
|
||||
export type AudioClip = z.infer<typeof audioClipSchema>;
|
||||
export const editPlanSchema = z.object({
|
||||
version: z.literal(1),
|
||||
title: z.string().max(200),
|
||||
@ -86,6 +111,7 @@ export const editPlanSchema = z.object({
|
||||
transition: z.number().min(0).max(0.5).default(0),
|
||||
subtitles: z.boolean().default(true),
|
||||
captions: z.array(timelineCaptionSchema).max(500).optional(),
|
||||
audioClips: z.array(audioClipSchema).max(24).optional(),
|
||||
music: z
|
||||
.object({ url, gain: z.number().min(0).max(1).default(0.2), start: time.default(0) })
|
||||
.optional(),
|
||||
@ -167,7 +193,17 @@ export function planTimeline(input: EditPlan) {
|
||||
throw Error("字幕 ID 重复");
|
||||
if (cursor > MAX_EDIT_SECONDS * SECOND || cursor < 200000)
|
||||
throw Error("成片时长需在 0.2–300 秒内");
|
||||
return { clips, duration: cursor };
|
||||
const audioClips = (plan.audioClips || []).map((c) => {
|
||||
const asset = plan.assets.find((a) => a.id === c.assetId);
|
||||
if (!asset?.hasAudio || c.sourceOut > asset.duration) throw Error("独立音轨素材无效或超出范围");
|
||||
const duration = Math.round((c.sourceOut - c.sourceIn) / c.speed);
|
||||
return { ...c, duration, end: c.start + duration };
|
||||
});
|
||||
if (new Set(audioClips.map((c) => c.id)).size !== audioClips.length)
|
||||
throw Error("独立音轨 ID 重复");
|
||||
cursor = Math.max(cursor, ...audioClips.map((c) => c.end));
|
||||
if (cursor > MAX_EDIT_SECONDS * SECOND) throw Error("成片时长需在 0.2–300 秒内");
|
||||
return { clips, audioClips, duration: cursor };
|
||||
}
|
||||
export function compiledSubtitles(plan: EditPlan) {
|
||||
if (!plan.subtitles) return [];
|
||||
@ -230,7 +266,12 @@ export function subtitlesToAss(plan: EditPlan, width: number, height: number) {
|
||||
.replace(/\}/g, "｝")
|
||||
.replace(/\r/g, "")
|
||||
.replace(/\n/g, "\\N");
|
||||
return `Dialogue: 0,${stamp(s.start)},${stamp(s.end)},c${i},,0,0,0,,${text}`;
|
||||
const style = s.style || defaultCaptionStyle;
|
||||
const position =
|
||||
style.x !== undefined || style.y !== undefined
|
||||
? `{\\pos(${Math.round((style.x ?? 0.5) * width)},${Math.round((style.y ?? 0.95) * height)})}`
|
||||
: "";
|
||||
return `Dialogue: 0,${stamp(s.start)},${stamp(s.end)},c${i},,0,0,0,,${position}${text}`;
|
||||
});
|
||||
return `[Script Info]\nScriptType: v4.00+\nPlayResX: ${width}\nPlayResY: ${height}\nWrapStyle: 0\nScaledBorderAndShadow: yes\n\n[V4+ Styles]\nFormat: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding\n${styles.join("\n")}\n\n[Events]\nFormat: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n${events.join("\n")}\n`;
|
||||
}
|
||||
|
||||
@ -8,6 +8,7 @@ import {
|
||||
subtitlesToSrt,
|
||||
subtitlesToAss,
|
||||
SECOND,
|
||||
defaultVideoTransform,
|
||||
} from "./contract";
|
||||
import { ffmpeg, mediaSafety } from "./local";
|
||||
const renderFfmpeg = (args: string[], cwd?: string) => ffmpeg(args, cwd, 600000);
|
||||
@ -38,16 +39,27 @@ export async function renderEditPlan(
|
||||
files[clip.assetId],
|
||||
];
|
||||
if (!asset.hasAudio) inputs.push("-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo");
|
||||
const t = clip.transform || defaultVideoTransform;
|
||||
const transformWidth = Math.max(2, Math.round((width * t.scale) / 2) * 2),
|
||||
transformHeight = Math.max(2, Math.round((height * t.scale) / 2) * 2);
|
||||
const transformX = Math.round((width - transformWidth) / 2 + t.x * width),
|
||||
transformY = Math.round((height - transformHeight) / 2 + t.y * height);
|
||||
const normalize = `setpts=(PTS-STARTPTS)/${clip.speed},scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2,setsar=1,fps=30,format=yuv420p`;
|
||||
const transformed = t.scale !== 1 || t.x !== 0 || t.y !== 0;
|
||||
await renderFfmpeg([
|
||||
...inputs,
|
||||
"-map",
|
||||
"0:v:0",
|
||||
transformed ? "[v]" : "0:v:0",
|
||||
"-map",
|
||||
asset.hasAudio ? "0:a:0" : "1:a:0",
|
||||
"-vf",
|
||||
`setpts=(PTS-STARTPTS)/${clip.speed},scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2,setsar=1,fps=30,format=yuv420p`,
|
||||
...(transformed
|
||||
? [
|
||||
"-filter_complex",
|
||||
`[0:v]${normalize},scale=${transformWidth}:${transformHeight}[fg];color=c=black:s=${width}x${height}:r=30[bg];[bg][fg]overlay=x=${transformX}:y=${transformY}:shortest=1[v]`,
|
||||
]
|
||||
: ["-vf", normalize]),
|
||||
"-af",
|
||||
`asetpts=PTS-STARTPTS,atempo=${clip.speed},aresample=48000,aformat=channel_layouts=stereo,volume=${clip.gain},apad`,
|
||||
`asetpts=PTS-STARTPTS,atempo=${clip.speed},aresample=48000,aformat=channel_layouts=stereo,volume=${clip.audioDetached ? 0 : clip.gain},apad`,
|
||||
"-t",
|
||||
String(clip.duration / SECOND),
|
||||
"-c:v",
|
||||
@ -135,26 +147,58 @@ export async function renderEditPlan(
|
||||
const output = path.join(dir, "result.mp4");
|
||||
const inputs = ["-i", joined];
|
||||
const filters: string[] = [];
|
||||
if (subtitles.length) filters.push("[0:v]ass=filename=subtitles.ass[v]");
|
||||
const videoTail = Math.max(0, duration - timeline.clips[timeline.clips.length - 1].end / SECOND);
|
||||
const videoFilters = [
|
||||
...(videoTail ? [`tpad=stop_mode=add:stop_duration=${videoTail}`] : []),
|
||||
...(subtitles.length ? ["ass=filename=subtitles.ass"] : []),
|
||||
];
|
||||
if (videoFilters.length) filters.push(`[0:v]${videoFilters.join(",")}[v]`);
|
||||
filters.push(`[0:a]apad,atrim=duration=${duration}[baseaudio]`);
|
||||
const mixLabels = ["[baseaudio]"];
|
||||
let audioInput = 1;
|
||||
if (musicFile && plan.music) {
|
||||
inputs.push("-ss", String(plan.music.start / SECOND), "-i", musicFile);
|
||||
filters.push(
|
||||
`[1:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=channel_layouts=stereo,volume=${plan.music.gain},apad,atrim=duration=${duration}[music]`,
|
||||
`[0:a][music]amix=inputs=2:duration=first:normalize=0,alimiter=limit=0.95:level=0[a]`,
|
||||
`[${audioInput}:a]asetpts=PTS-STARTPTS,aresample=48000,aformat=channel_layouts=stereo,volume=${plan.music.gain},apad,atrim=duration=${duration}[music]`,
|
||||
);
|
||||
mixLabels.push("[music]");
|
||||
audioInput++;
|
||||
}
|
||||
for (const [index, clip] of timeline.audioClips.entries()) {
|
||||
inputs.push(
|
||||
...mediaSafety,
|
||||
"-ss",
|
||||
String(clip.sourceIn / SECOND),
|
||||
"-t",
|
||||
String((clip.sourceOut - clip.sourceIn) / SECOND),
|
||||
"-i",
|
||||
files[clip.assetId],
|
||||
);
|
||||
const length = clip.duration / SECOND,
|
||||
fadeIn = Math.min(clip.fadeIn, length),
|
||||
fadeOut = Math.min(clip.fadeOut, length);
|
||||
filters.push(
|
||||
`[${audioInput}:a]asetpts=PTS-STARTPTS,atempo=${clip.speed},aresample=48000,aformat=channel_layouts=stereo,volume=${clip.gain},atrim=duration=${length}${fadeIn ? `,afade=t=in:d=${fadeIn}` : ""}${fadeOut ? `,afade=t=out:st=${length - fadeOut}:d=${fadeOut}` : ""},adelay=${Math.round(clip.start / 1000)}:all=1[track${index}]`,
|
||||
);
|
||||
mixLabels.push(`[track${index}]`);
|
||||
audioInput++;
|
||||
}
|
||||
if (mixLabels.length > 1)
|
||||
filters.push(
|
||||
`${mixLabels.join("")}amix=inputs=${mixLabels.length}:duration=first:normalize=0,alimiter=limit=0.95:level=0[a]`,
|
||||
);
|
||||
await progress("输出成片");
|
||||
await renderFfmpeg(
|
||||
[
|
||||
...inputs,
|
||||
...(filters.length ? ["-filter_complex", filters.join(";")] : []),
|
||||
"-map",
|
||||
subtitles.length ? "[v]" : "0:v:0",
|
||||
videoFilters.length ? "[v]" : "0:v:0",
|
||||
"-map",
|
||||
musicFile && plan.music ? "[a]" : "0:a:0",
|
||||
mixLabels.length > 1 ? "[a]" : "[baseaudio]",
|
||||
"-c:v",
|
||||
subtitles.length ? "libx264" : "copy",
|
||||
...(subtitles.length
|
||||
videoFilters.length ? "libx264" : "copy",
|
||||
...(videoFilters.length
|
||||
? [
|
||||
"-preset",
|
||||
"veryfast",
|
||||
|
||||
@ -211,6 +211,193 @@ const fs = require("node:fs"),
|
||||
assert.equal(probe.height % 2, 0);
|
||||
console.log("PASS: real 480P " + ratio + " " + probe.width + "×" + probe.height);
|
||||
}
|
||||
// Verify edited geometry, detached audio, gain, fades and black tail in actual encoded media.
|
||||
const sine = path.join(dir, "steady-tone.mp4");
|
||||
execFileSync("ffmpeg", [
|
||||
"-v",
|
||||
"error",
|
||||
"-y",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"color=c=red:s=320x180:r=30:d=4",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=440:sample_rate=48000:duration=4",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-threads",
|
||||
"2",
|
||||
"-c:a",
|
||||
"aac",
|
||||
sine,
|
||||
]);
|
||||
const tone = { ...b, id: "tone", subtitles: [], ...(await probeMedia(sine)) };
|
||||
const extensionFiles = { tone: sine };
|
||||
const base = {
|
||||
...plan,
|
||||
resolution: "480p",
|
||||
assets: [tone],
|
||||
clips: [
|
||||
{
|
||||
...plan.clips[0],
|
||||
assetId: "tone",
|
||||
sourceOut: 2e6,
|
||||
},
|
||||
],
|
||||
};
|
||||
const actual = async (name, edited) => {
|
||||
const out = path.join(dir, name);
|
||||
fs.mkdirSync(out, { recursive: true });
|
||||
return renderEditPlan(edited, extensionFiles, undefined, out);
|
||||
};
|
||||
const pixelAt = (file, time, x, y) =>
|
||||
execFileSync("ffmpeg", [
|
||||
"-v",
|
||||
"error",
|
||||
"-ss",
|
||||
String(time),
|
||||
"-i",
|
||||
file,
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-vf",
|
||||
`crop=2:2:${x}:${y}`,
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pix_fmt",
|
||||
"rgb24",
|
||||
"-",
|
||||
]);
|
||||
const rms = (file, time, duration = 0.2) => {
|
||||
const data = execFileSync("ffmpeg", [
|
||||
"-v",
|
||||
"error",
|
||||
"-ss",
|
||||
String(time),
|
||||
"-i",
|
||||
file,
|
||||
"-t",
|
||||
String(duration),
|
||||
"-vn",
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
"48000",
|
||||
"-f",
|
||||
"f32le",
|
||||
"-",
|
||||
]);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < data.length; i += 4) sum += data.readFloatLE(i) ** 2;
|
||||
return Math.sqrt(sum / (data.length / 4));
|
||||
};
|
||||
const baseline = await actual("audio-baseline", base);
|
||||
const detached = await actual("audio-detached", {
|
||||
...base,
|
||||
clips: [{ ...base.clips[0], audioDetached: true, transform: { scale: 0.5, x: 0.2, y: -0.1 } }],
|
||||
audioClips: [
|
||||
{
|
||||
id: "track",
|
||||
assetId: "tone",
|
||||
start: 0,
|
||||
sourceIn: 0,
|
||||
sourceOut: 2e6,
|
||||
speed: 1,
|
||||
gain: 1,
|
||||
fadeIn: 0,
|
||||
fadeOut: 0,
|
||||
},
|
||||
],
|
||||
});
|
||||
const ratio = rms(detached.file, 0.8) / rms(baseline.file, 0.8);
|
||||
assert.ok(ratio > 0.9 && ratio < 1.1, `detached sound must not double: ${ratio}`);
|
||||
assert.ok(pixelAt(detached.file, 0.8, 600, 150)[0] > 220, "shifted scaled video should be red");
|
||||
assert.ok(
|
||||
pixelAt(detached.file, 0.8, 150, 150)[0] < 10,
|
||||
"outside transformed video should be black",
|
||||
);
|
||||
const delayedPlan = {
|
||||
...base,
|
||||
clips: [{ ...base.clips[0], audioDetached: true }],
|
||||
audioClips: [
|
||||
{
|
||||
id: "delayed",
|
||||
assetId: "tone",
|
||||
start: 1e6,
|
||||
sourceIn: 0,
|
||||
sourceOut: 4e6,
|
||||
speed: 2,
|
||||
gain: 0.5,
|
||||
fadeIn: 0.5,
|
||||
fadeOut: 0.5,
|
||||
},
|
||||
],
|
||||
};
|
||||
const delayed = await actual("audio-delayed-tail", delayedPlan);
|
||||
const delayedProbe = await probeMedia(delayed.file);
|
||||
assert.ok(Math.abs(delayedProbe.duration - 3e6) < 150000, JSON.stringify(delayedProbe));
|
||||
assert.ok(rms(delayed.file, 0.4) < 0.0001, "detached original must stay silent");
|
||||
const middleRms = rms(delayed.file, 1.7);
|
||||
assert.ok(
|
||||
middleRms > rms(baseline.file, 0.8) * 0.4 && middleRms < rms(baseline.file, 0.8) * 0.6,
|
||||
"independent gain should be applied once",
|
||||
);
|
||||
assert.ok(rms(delayed.file, 1.05, 0.1) < middleRms * 0.4, "audio fade in");
|
||||
assert.ok(rms(delayed.file, 2.8, 0.1) < middleRms * 0.5, "audio fade out");
|
||||
assert.ok(pixelAt(delayed.file, 2.6, 200, 200)[0] < 10, "audio beyond video uses black tail");
|
||||
const captioned = await actual("caption-position", {
|
||||
...base,
|
||||
subtitles: true,
|
||||
captions: [
|
||||
{
|
||||
id: "pos",
|
||||
text: "位置测试",
|
||||
start: 0,
|
||||
end: 2e6,
|
||||
style: {
|
||||
fontSize: 80,
|
||||
color: "#ffffff",
|
||||
outlineColor: "#000000",
|
||||
outlineWidth: 0,
|
||||
opacity: 1,
|
||||
spacing: 0,
|
||||
x: 0.25,
|
||||
y: 0.5,
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
const frame = execFileSync(
|
||||
"ffmpeg",
|
||||
[
|
||||
"-v",
|
||||
"error",
|
||||
"-ss",
|
||||
"0.8",
|
||||
"-i",
|
||||
captioned.file,
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pix_fmt",
|
||||
"rgb24",
|
||||
"-",
|
||||
],
|
||||
{ maxBuffer: 4 * 1024 * 1024 },
|
||||
);
|
||||
let whitePixels = 0;
|
||||
for (let y = 170; y < 240; y++)
|
||||
for (let x = 90; x < 340; x++) {
|
||||
const offset = (y * 854 + x) * 3;
|
||||
if (frame[offset] > 200 && frame[offset + 1] > 200 && frame[offset + 2] > 200) whitePixels++;
|
||||
}
|
||||
assert.ok(whitePixels > 150, `caption must render at moved position: ${whitePixels}`);
|
||||
console.log(
|
||||
"PASS: real transform pixels, caption position, detached audio without doubling, 2x audio, gain, fades, delay and black tail",
|
||||
);
|
||||
if (process.env.SMART_EDIT_PYTHON) {
|
||||
const prepared = await prepareAnalysis(files.b, "fixture", dir, true);
|
||||
const analysis = await pythonAnalysis(
|
||||
|
||||
@ -54,6 +54,53 @@ const plan = (): EditPlan => ({
|
||||
},
|
||||
],
|
||||
});
|
||||
test("detached audio retains its own timing after video edits and extends the composition", () => {
|
||||
const p = plan();
|
||||
p.clips[0].audioDetached = true;
|
||||
p.audioClips = [
|
||||
{
|
||||
id: "audio",
|
||||
assetId: "a",
|
||||
start: 3 * SECOND,
|
||||
sourceIn: SECOND,
|
||||
sourceOut: 5 * SECOND,
|
||||
speed: 2,
|
||||
gain: 0.5,
|
||||
fadeIn: 0.2,
|
||||
fadeOut: 0.3,
|
||||
},
|
||||
];
|
||||
expect(planTimeline(p).duration).toBe(5 * SECOND);
|
||||
expect(planTimeline(p).audioClips[0].duration).toBe(2 * SECOND);
|
||||
p.clips.reverse();
|
||||
expect(planTimeline(p).audioClips[0].start).toBe(3 * SECOND);
|
||||
p.audioClips[0].sourceOut = 7 * SECOND;
|
||||
expect(() => planTimeline(p)).toThrow("范围");
|
||||
p.audioClips[0].sourceOut = 5 * SECOND;
|
||||
p.audioClips[0].start = 300 * SECOND;
|
||||
expect(() => planTimeline(p)).toThrow("300");
|
||||
p.audioClips[0].start = 0;
|
||||
p.assets = [{ ...asset, hasAudio: false }];
|
||||
expect(() => planTimeline(p)).toThrow("音轨");
|
||||
});
|
||||
test("positioned captions produce trusted ASS positions and still escape caption text", () => {
|
||||
const p = plan();
|
||||
p.captions = [
|
||||
{
|
||||
id: "positioned",
|
||||
start: 0,
|
||||
end: SECOND,
|
||||
text: "移动{\\pos(1,1)}",
|
||||
style: { ...defaultCaptionStyle, x: 0.25, y: 0.5 },
|
||||
},
|
||||
];
|
||||
const ass = subtitlesToAss(p, 1280, 720);
|
||||
expect(ass).toContain("{\\pos(320,360)}移动｛＼pos(1,1)｝");
|
||||
p.clips[0].transform = { x: 0.2, y: -0.2, scale: 0.5 };
|
||||
expect(planTimeline(p).clips[0].transform).toEqual(p.clips[0].transform);
|
||||
p.clips[0].transform.scale = 4;
|
||||
expect(() => planTimeline(p)).toThrow();
|
||||
});
|
||||
test("EDL computes overlap, speed and remapped subtitle timestamps from a shared timeline", () => {
|
||||
const p = plan();
|
||||
p.transition = 0.25;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user