632 lines
18 KiB
JavaScript
632 lines
18 KiB
JavaScript
// Offline integration: real PySceneDetect/librosa and FFmpeg, no provider generation requests.
|
||
const fs = require("node:fs"),
|
||
path = require("node:path"),
|
||
assert = require("node:assert/strict"),
|
||
{ execFileSync } = require("node:child_process");
|
||
(async () => {
|
||
const root = process.cwd(),
|
||
dir = path.resolve(".qa/smart-edit");
|
||
fs.mkdirSync(dir, { recursive: true });
|
||
execFileSync("python3", ["scripts/qa/fixtures/smart-edit.py"]);
|
||
execFileSync("ffmpeg", [
|
||
"-v",
|
||
"error",
|
||
"-y",
|
||
...["red", "blue", "green"].flatMap((c) => [
|
||
"-f",
|
||
"lavfi",
|
||
"-i",
|
||
`color=c=${c}:s=320x180:r=30:d=2`,
|
||
]),
|
||
"-filter_complex",
|
||
"[0:v][1:v][2:v]concat=n=3:v=1:a=0[v]",
|
||
"-map",
|
||
"[v]",
|
||
"-c:v",
|
||
"libx264",
|
||
"-threads",
|
||
"2",
|
||
path.join(dir, "colors.mp4"),
|
||
]);
|
||
execFileSync("ffmpeg", [
|
||
"-v",
|
||
"error",
|
||
"-y",
|
||
"-i",
|
||
path.join(dir, "colors.mp4"),
|
||
"-i",
|
||
path.join(dir, "speech.wav"),
|
||
"-c:v",
|
||
"copy",
|
||
"-c:a",
|
||
"aac",
|
||
path.join(dir, "with-audio.mp4"),
|
||
]);
|
||
const esbuild = require(
|
||
path.resolve(
|
||
"node_modules/.pnpm",
|
||
fs.readdirSync("node_modules/.pnpm").find((n) => n.startsWith("esbuild@0.25.")),
|
||
"node_modules/esbuild",
|
||
),
|
||
);
|
||
await esbuild.build({
|
||
entryPoints: [
|
||
"lib/video-canvas/smart-edit/render.ts",
|
||
"lib/video-canvas/smart-edit/local.ts",
|
||
],
|
||
outdir: ".qa/smart-edit/build",
|
||
bundle: true,
|
||
platform: "node",
|
||
format: "cjs",
|
||
packages: "external",
|
||
alias: { "@": root },
|
||
});
|
||
const { renderEditPlan } = require(path.join(dir, "build/render.js")),
|
||
{ probeMedia, pythonAnalysis, prepareAnalysis } = require(path.join(dir, "build/local.js"));
|
||
const a = {
|
||
id: "a",
|
||
name: "无音轨",
|
||
url: "https://example.com/colors.mp4",
|
||
hash: "a".repeat(64),
|
||
...(await probeMedia(path.join(dir, "colors.mp4"))),
|
||
subtitles: [{ start: 0, end: 1000000, text: "红色测试" }],
|
||
},
|
||
b = {
|
||
id: "b",
|
||
name: "有音轨",
|
||
url: "https://example.com/with-audio.mp4",
|
||
hash: "b".repeat(64),
|
||
...(await probeMedia(path.join(dir, "with-audio.mp4"))),
|
||
subtitles: [{ start: 3000000, end: 4000000, text: "第二句" }],
|
||
};
|
||
const plan = {
|
||
version: 1,
|
||
title: "QA",
|
||
assets: [a, b],
|
||
ratio: "16:9",
|
||
resolution: "720p",
|
||
transition: 0,
|
||
subtitles: false,
|
||
warnings: [],
|
||
clips: [
|
||
{
|
||
id: "1",
|
||
assetId: "a",
|
||
sourceIn: 0,
|
||
sourceOut: 2000000,
|
||
speed: 1,
|
||
gain: 1,
|
||
label: "red",
|
||
},
|
||
{
|
||
id: "2",
|
||
assetId: "b",
|
||
sourceIn: 2000000,
|
||
sourceOut: 6000000,
|
||
speed: 2,
|
||
gain: 1,
|
||
label: "blue-green",
|
||
},
|
||
],
|
||
};
|
||
const files = { a: path.join(dir, "colors.mp4"), b: path.join(dir, "with-audio.mp4") };
|
||
for (const transition of [0, 0.5]) {
|
||
const out = path.join(dir, "render-" + transition);
|
||
fs.mkdirSync(out, { recursive: true });
|
||
const result = await renderEditPlan(
|
||
{
|
||
...plan,
|
||
transition,
|
||
subtitles: true,
|
||
music: { url: "https://example.com/beats.wav", gain: 0.2, start: 0 },
|
||
},
|
||
files,
|
||
path.join(dir, "beats.wav"),
|
||
out,
|
||
);
|
||
const probe = await probeMedia(result.file);
|
||
assert.equal(probe.width, 1280);
|
||
assert.equal(probe.height, 720);
|
||
assert.equal(probe.hasAudio, true);
|
||
assert.ok(Math.abs(probe.duration / 1e6 - (4 - transition)) < 0.15, JSON.stringify(probe));
|
||
const srt = fs.readFileSync(result.srt, "utf8");
|
||
assert.match(srt, /红色测试/);
|
||
assert.match(srt, /第二句/);
|
||
console.log(
|
||
"PASS: actual FFmpeg " +
|
||
transition +
|
||
"s transition, speed 2x, silent source, music, subtitles, 720p and duration " +
|
||
probe.duration / 1e6,
|
||
);
|
||
}
|
||
for (const type of ["fade", "fadeblack", "fadewhite"]) {
|
||
const out = path.join(dir, "manual-" + type);
|
||
fs.mkdirSync(out, { recursive: true });
|
||
const mixed = {
|
||
...plan,
|
||
subtitles: true,
|
||
clips: Array.from({ length: 4 }, (_, i) => ({
|
||
...plan.clips[0],
|
||
id: "mixed-" + i,
|
||
sourceIn: (i % 3) * 1000000,
|
||
sourceOut: ((i % 3) + 1) * 1000000,
|
||
transition: { type, duration: i === 2 ? 0 : 0.3 },
|
||
})),
|
||
captions: [
|
||
{
|
||
id: "manual",
|
||
start: 0,
|
||
end: 2000000,
|
||
text: "手动字幕 测试",
|
||
style: {
|
||
fontSize: 36,
|
||
color: "#ffff00",
|
||
outlineColor: "#000000",
|
||
outlineWidth: 2,
|
||
opacity: 1,
|
||
spacing: 2,
|
||
},
|
||
},
|
||
],
|
||
};
|
||
const result = await renderEditPlan(mixed, files, undefined, out);
|
||
const probe = await probeMedia(result.file);
|
||
assert.ok(Math.abs(probe.duration / 1e6 - 3.4) < 0.15, JSON.stringify(probe));
|
||
assert.match(
|
||
fs.readFileSync(path.join(out, "subtitles.ass"), "utf8"),
|
||
/Noto Sans CJK SC,36/,
|
||
);
|
||
assert.match(fs.readFileSync(result.srt, "utf8"), /手动字幕/);
|
||
const pixel = execFileSync("ffmpeg", [
|
||
"-v",
|
||
"error",
|
||
"-ss",
|
||
"0.833333",
|
||
"-i",
|
||
result.file,
|
||
"-frames:v",
|
||
"1",
|
||
"-vf",
|
||
"crop=2:2:400:200",
|
||
"-f",
|
||
"rawvideo",
|
||
"-pix_fmt",
|
||
"rgb24",
|
||
"-",
|
||
]);
|
||
if (type === "fadeblack")
|
||
assert.ok(pixel[0] < 70, "black transition must darken the midpoint");
|
||
if (type === "fadewhite")
|
||
assert.ok(pixel[1] > 180, "white transition must brighten the midpoint");
|
||
if (type === "fade")
|
||
assert.ok(pixel[0] > 220 && pixel[1] < 20, "crossfade must preserve equal red sources");
|
||
console.log(
|
||
"PASS: real " +
|
||
type +
|
||
" → hard cut → " +
|
||
type +
|
||
", styled manual caption, duration " +
|
||
probe.duration / 1e6,
|
||
);
|
||
}
|
||
for (const ratio of ["21:9", "16:9", "9:16", "1:1", "4:3", "3:4"]) {
|
||
const out = path.join(dir, "ratio-" + ratio.replace(":", "-"));
|
||
fs.mkdirSync(out, { recursive: true });
|
||
const result = await renderEditPlan(
|
||
{ ...plan, ratio, resolution: "480p", clips: [{ ...plan.clips[0], sourceOut: 1000000 }] },
|
||
files,
|
||
undefined,
|
||
out,
|
||
);
|
||
const probe = await probeMedia(result.file);
|
||
const [w, h] = ratio.split(":").map(Number);
|
||
assert.equal(Math.min(probe.width, probe.height), 480);
|
||
assert.ok(Math.abs(probe.width / probe.height - w / h) < 0.005, JSON.stringify(probe));
|
||
assert.equal(probe.width % 2, 0);
|
||
assert.equal(probe.height % 2, 0);
|
||
console.log("PASS: real 480P " + ratio + " " + probe.width + "×" + probe.height);
|
||
}
|
||
for (const type of ["fade", "fadeblack", "fadewhite"]) {
|
||
for (const duration of [2.2, 4]) {
|
||
const out = path.join(dir, `center-${type}-${duration}`);
|
||
fs.mkdirSync(out, { recursive: true });
|
||
const centered = {
|
||
...plan,
|
||
resolution: "480p",
|
||
clips: [
|
||
{ ...plan.clips[0], sourceOut: 2e6 },
|
||
{
|
||
...plan.clips[0],
|
||
id: "incoming",
|
||
sourceIn: 2e6,
|
||
sourceOut: 4e6,
|
||
transition: { type, duration, alignment: "center" },
|
||
},
|
||
],
|
||
};
|
||
const result = await renderEditPlan(centered, files, undefined, out),
|
||
probe = await probeMedia(result.file);
|
||
assert.ok(
|
||
Math.abs(probe.duration - 4e6) < 150000,
|
||
`centered transition must keep 4 seconds: ${JSON.stringify(probe)}`,
|
||
);
|
||
const pixel = execFileSync("ffmpeg", [
|
||
"-v",
|
||
"error",
|
||
"-ss",
|
||
"2",
|
||
"-i",
|
||
result.file,
|
||
"-frames:v",
|
||
"1",
|
||
"-vf",
|
||
"crop=2:2:400:200",
|
||
"-f",
|
||
"rawvideo",
|
||
"-pix_fmt",
|
||
"rgb24",
|
||
"-",
|
||
]);
|
||
if (type === "fadeblack")
|
||
assert.ok(pixel[0] < 30 && pixel[2] < 30, `black midpoint ${pixel.slice(0, 3)}`);
|
||
if (type === "fadewhite")
|
||
assert.ok(
|
||
pixel[0] > 220 && pixel[1] > 220 && pixel[2] > 220,
|
||
`white midpoint ${pixel.slice(0, 3)}`,
|
||
);
|
||
if (type === "fade")
|
||
assert.ok(pixel[0] > 80 && pixel[2] > 80, `crossfade midpoint ${pixel.slice(0, 3)}`);
|
||
console.log(
|
||
`PASS: centered ${type} ${duration}s preserves 4s edit; midpoint pixel verified`,
|
||
);
|
||
}
|
||
}
|
||
// Verify edited geometry, detached audio, gain, fades and black tail in actual encoded media.
|
||
const sine = path.join(dir, "steady-tone.mp4");
|
||
execFileSync("ffmpeg", [
|
||
"-v",
|
||
"error",
|
||
"-y",
|
||
"-f",
|
||
"lavfi",
|
||
"-i",
|
||
"color=c=red:s=320x180:r=30:d=4",
|
||
"-f",
|
||
"lavfi",
|
||
"-i",
|
||
"sine=frequency=440:sample_rate=48000:duration=4",
|
||
"-c:v",
|
||
"libx264",
|
||
"-threads",
|
||
"2",
|
||
"-c:a",
|
||
"aac",
|
||
sine,
|
||
]);
|
||
const tone = { ...b, id: "tone", subtitles: [], ...(await probeMedia(sine)) };
|
||
const extensionFiles = { tone: sine };
|
||
const base = {
|
||
...plan,
|
||
resolution: "480p",
|
||
assets: [tone],
|
||
clips: [
|
||
{
|
||
...plan.clips[0],
|
||
assetId: "tone",
|
||
sourceOut: 2e6,
|
||
},
|
||
],
|
||
};
|
||
const actual = async (name, edited) => {
|
||
const out = path.join(dir, name);
|
||
fs.mkdirSync(out, { recursive: true });
|
||
return renderEditPlan(edited, extensionFiles, undefined, out);
|
||
};
|
||
const pixelAt = (file, time, x, y) =>
|
||
execFileSync("ffmpeg", [
|
||
"-v",
|
||
"error",
|
||
"-ss",
|
||
String(time),
|
||
"-i",
|
||
file,
|
||
"-frames:v",
|
||
"1",
|
||
"-vf",
|
||
`crop=2:2:${x}:${y}`,
|
||
"-f",
|
||
"rawvideo",
|
||
"-pix_fmt",
|
||
"rgb24",
|
||
"-",
|
||
]);
|
||
const rms = (file, time, duration = 0.2) => {
|
||
const data = execFileSync("ffmpeg", [
|
||
"-v",
|
||
"error",
|
||
"-ss",
|
||
String(time),
|
||
"-i",
|
||
file,
|
||
"-t",
|
||
String(duration),
|
||
"-vn",
|
||
"-ac",
|
||
"1",
|
||
"-ar",
|
||
"48000",
|
||
"-f",
|
||
"f32le",
|
||
"-",
|
||
]);
|
||
let sum = 0;
|
||
for (let i = 0; i < data.length; i += 4) sum += data.readFloatLE(i) ** 2;
|
||
return Math.sqrt(sum / (data.length / 4));
|
||
};
|
||
const baseline = await actual("audio-baseline", base);
|
||
const detached = await actual("audio-detached", {
|
||
...base,
|
||
clips: [
|
||
{ ...base.clips[0], audioDetached: true, transform: { scale: 0.5, x: 0.2, y: -0.1 } },
|
||
],
|
||
audioClips: [
|
||
{
|
||
id: "track",
|
||
assetId: "tone",
|
||
start: 0,
|
||
sourceIn: 0,
|
||
sourceOut: 2e6,
|
||
speed: 1,
|
||
gain: 1,
|
||
fadeIn: 0,
|
||
fadeOut: 0,
|
||
},
|
||
],
|
||
});
|
||
const ratio = rms(detached.file, 0.8) / rms(baseline.file, 0.8);
|
||
assert.ok(ratio > 0.9 && ratio < 1.1, `detached sound must not double: ${ratio}`);
|
||
assert.ok(
|
||
pixelAt(detached.file, 0.8, 600, 150)[0] > 220,
|
||
"shifted scaled video should be red",
|
||
);
|
||
assert.ok(
|
||
pixelAt(detached.file, 0.8, 150, 150)[0] < 10,
|
||
"outside transformed video should be black",
|
||
);
|
||
const delayedPlan = {
|
||
...base,
|
||
clips: [{ ...base.clips[0], audioDetached: true }],
|
||
audioClips: [
|
||
{
|
||
id: "delayed",
|
||
assetId: "tone",
|
||
start: 1e6,
|
||
sourceIn: 0,
|
||
sourceOut: 4e6,
|
||
speed: 2,
|
||
gain: 0.5,
|
||
fadeIn: 0.5,
|
||
fadeOut: 0.5,
|
||
},
|
||
],
|
||
};
|
||
const delayed = await actual("audio-delayed-tail", delayedPlan);
|
||
const delayedProbe = await probeMedia(delayed.file);
|
||
assert.ok(Math.abs(delayedProbe.duration - 3e6) < 150000, JSON.stringify(delayedProbe));
|
||
assert.ok(rms(delayed.file, 0.4) < 0.0001, "detached original must stay silent");
|
||
const middleRms = rms(delayed.file, 1.7);
|
||
assert.ok(
|
||
middleRms > rms(baseline.file, 0.8) * 0.4 && middleRms < rms(baseline.file, 0.8) * 0.6,
|
||
"independent gain should be applied once",
|
||
);
|
||
assert.ok(rms(delayed.file, 1.05, 0.1) < middleRms * 0.4, "audio fade in");
|
||
assert.ok(rms(delayed.file, 2.8, 0.1) < middleRms * 0.5, "audio fade out");
|
||
assert.ok(pixelAt(delayed.file, 2.6, 200, 200)[0] < 10, "audio beyond video uses black tail");
|
||
const withCenter = {
|
||
...base,
|
||
clips: [
|
||
base.clips[0],
|
||
{
|
||
...base.clips[0],
|
||
id: "center-audio",
|
||
sourceIn: 2e6,
|
||
sourceOut: 4e6,
|
||
transition: { type: "fadeblack", duration: 2.2, alignment: "center" },
|
||
},
|
||
],
|
||
};
|
||
const centerSound = await actual("center-original-audio", withCenter);
|
||
const centerDetached = await actual("center-detached-audio", {
|
||
...withCenter,
|
||
clips: [withCenter.clips[0], { ...withCenter.clips[1], audioDetached: true }],
|
||
audioClips: [
|
||
{
|
||
id: "center-detached",
|
||
assetId: "tone",
|
||
start: 2e6,
|
||
sourceIn: 2e6,
|
||
sourceOut: 4e6,
|
||
speed: 1,
|
||
gain: 1,
|
||
fadeIn: 1.1,
|
||
fadeOut: 0,
|
||
fadeInStart: 0.5,
|
||
},
|
||
],
|
||
});
|
||
for (const time of [1.3, 2.1, 2.6, 3.5]) {
|
||
const ratio = rms(centerDetached.file, time, 0.1) / rms(centerSound.file, time, 0.1);
|
||
assert.ok(
|
||
ratio > 0.85 && ratio < 1.15,
|
||
`detaching centered-transition audio must preserve its envelope at ${time}s: ${ratio}`,
|
||
);
|
||
}
|
||
console.log("PASS: centered transition audio envelope preserved after detachment");
|
||
const captioned = await actual("caption-position", {
|
||
...base,
|
||
subtitles: true,
|
||
captions: [
|
||
{
|
||
id: "pos",
|
||
text: "位置测试",
|
||
start: 0,
|
||
end: 2e6,
|
||
style: {
|
||
fontSize: 80,
|
||
color: "#ffffff",
|
||
outlineColor: "#000000",
|
||
outlineWidth: 8,
|
||
outlineOpacity: 0,
|
||
opacity: 1,
|
||
spacing: 0,
|
||
x: 0.25,
|
||
y: 0.5,
|
||
},
|
||
},
|
||
],
|
||
});
|
||
const frame = execFileSync(
|
||
"ffmpeg",
|
||
[
|
||
"-v",
|
||
"error",
|
||
"-ss",
|
||
"0.8",
|
||
"-i",
|
||
captioned.file,
|
||
"-frames:v",
|
||
"1",
|
||
"-f",
|
||
"rawvideo",
|
||
"-pix_fmt",
|
||
"rgb24",
|
||
"-",
|
||
],
|
||
{ maxBuffer: 4 * 1024 * 1024 },
|
||
);
|
||
let whitePixels = 0,
|
||
blackPixels = 0;
|
||
for (let y = 170; y < 240; y++)
|
||
for (let x = 90; x < 340; x++) {
|
||
const offset = (y * 854 + x) * 3;
|
||
if (frame[offset] > 200 && frame[offset + 1] > 200 && frame[offset + 2] > 200)
|
||
whitePixels++;
|
||
if (frame[offset] < 30 && frame[offset + 1] < 30 && frame[offset + 2] < 30) blackPixels++;
|
||
}
|
||
assert.ok(whitePixels > 150, `caption must render at moved position: ${whitePixels}`);
|
||
assert.ok(
|
||
blackPixels < 10,
|
||
`zero outline opacity must remove only the black outline: ${blackPixels}`,
|
||
);
|
||
console.log(
|
||
"PASS: real transform pixels, caption position, detached audio without doubling, 2x audio, gain, fades, delay and black tail",
|
||
);
|
||
const boosted = await actual("positive-gain", {
|
||
...base,
|
||
clips: [{ ...base.clips[0], gain: 2 }],
|
||
});
|
||
const boostRatio = rms(boosted.file, 0.8) / rms(baseline.file, 0.8);
|
||
assert.ok(
|
||
boostRatio > 1.85 && boostRatio < 2.15,
|
||
`+6 dB should double amplitude: ${boostRatio}`,
|
||
);
|
||
const muted = await actual("muted-gain", {
|
||
...base,
|
||
clips: [{ ...base.clips[0], gain: 2, muted: true }],
|
||
});
|
||
assert.ok(
|
||
rms(muted.file, 0.8) < 0.0001,
|
||
"mute must silence original sound without overwriting gain",
|
||
);
|
||
const independentMuted = await actual("muted-independent-gain", {
|
||
...delayedPlan,
|
||
audioClips: delayedPlan.audioClips.map((clip) => ({ ...clip, muted: true })),
|
||
});
|
||
assert.ok(rms(independentMuted.file, 1.7) < 0.0001, "mute must silence detached sound");
|
||
const fontHashes = new Set();
|
||
for (const fontFamily of ["source-han", "noto-sans-sc", "noto-serif-sc", "roboto", "inter"]) {
|
||
const output = await actual("font-" + fontFamily, {
|
||
...base,
|
||
subtitles: true,
|
||
captions: [
|
||
{
|
||
id: "font",
|
||
text: "字幕字体 AaGg 123",
|
||
start: 0,
|
||
end: 2e6,
|
||
style: {
|
||
fontFamily,
|
||
referenceHeight: 1080,
|
||
fontSize: 56,
|
||
color: "#ffffff",
|
||
outlineColor: "#ffffff",
|
||
outlineWidth: 0,
|
||
opacity: 1,
|
||
spacing: 0,
|
||
x: 0.5,
|
||
y: 0.5,
|
||
},
|
||
},
|
||
],
|
||
});
|
||
const pixels = execFileSync(
|
||
"ffmpeg",
|
||
[
|
||
"-v",
|
||
"error",
|
||
"-ss",
|
||
"0.8",
|
||
"-i",
|
||
output.file,
|
||
"-frames:v",
|
||
"1",
|
||
"-f",
|
||
"rawvideo",
|
||
"-pix_fmt",
|
||
"rgb24",
|
||
"-",
|
||
],
|
||
{ maxBuffer: 4 * 1024 * 1024 },
|
||
);
|
||
let light = 0;
|
||
for (let i = 0; i < pixels.length; i += 3)
|
||
if (pixels[i] > 200 && pixels[i + 1] > 200 && pixels[i + 2] > 200) light++;
|
||
assert.ok(light > 150, `${fontFamily} must draw glyphs: ${light}`);
|
||
fontHashes.add(require("node:crypto").createHash("sha256").update(pixels).digest("hex"));
|
||
}
|
||
assert.ok(
|
||
fontHashes.size >= 4,
|
||
"selected fonts must change actual glyph pixels, not all fall back to one face",
|
||
);
|
||
console.log(
|
||
"PASS: positive gain amplitude, original/detached mute and five packaged caption fonts",
|
||
);
|
||
if (process.env.SMART_EDIT_PYTHON) {
|
||
const prepared = await prepareAnalysis(files.b, "fixture", dir, true);
|
||
const analysis = await pythonAnalysis(
|
||
{ version: 1, video: prepared.proxy, audio: prepared.audio, duration: 6000000 },
|
||
dir,
|
||
);
|
||
assert.ok(analysis.scenes.length >= 3, JSON.stringify(analysis));
|
||
assert.ok(analysis.active.length >= 2, JSON.stringify(analysis));
|
||
const beats = await pythonAnalysis(
|
||
{ version: 1, audio: path.join(dir, "beats.wav"), beats: true },
|
||
dir,
|
||
);
|
||
assert.ok(beats.beats.length > 10, JSON.stringify(beats));
|
||
console.log(
|
||
"PASS: actual PySceneDetect found " +
|
||
analysis.scenes.length +
|
||
" scenes; RMS gate " +
|
||
analysis.active.length +
|
||
" active intervals; librosa " +
|
||
beats.beats.length +
|
||
" beats",
|
||
);
|
||
}
|
||
fs.writeFileSync(path.join(dir, "fixture-plan.json"), JSON.stringify(plan, null, 2));
|
||
})().catch((e) => {
|
||
console.error(e);
|
||
process.exitCode = 1;
|
||
});
|