// Offline integration: real PySceneDetect/librosa and FFmpeg, no provider generation requests. const fs = require("node:fs"), path = require("node:path"), assert = require("node:assert/strict"), { execFileSync } = require("node:child_process"); (async () => { const root = process.cwd(), dir = path.resolve(".qa/smart-edit"); fs.mkdirSync(dir, { recursive: true }); execFileSync("python3", ["scripts/qa/fixtures/smart-edit.py"]); execFileSync("ffmpeg", [ "-v", "error", "-y", ...["red", "blue", "green"].flatMap((c) => [ "-f", "lavfi", "-i", `color=c=${c}:s=320x180:r=30:d=2`, ]), "-filter_complex", "[0:v][1:v][2:v]concat=n=3:v=1:a=0[v]", "-map", "[v]", "-c:v", "libx264", "-threads", "2", path.join(dir, "colors.mp4"), ]); execFileSync("ffmpeg", [ "-v", "error", "-y", "-i", path.join(dir, "colors.mp4"), "-i", path.join(dir, "speech.wav"), "-c:v", "copy", "-c:a", "aac", path.join(dir, "with-audio.mp4"), ]); const esbuild = require( path.resolve( "node_modules/.pnpm", fs.readdirSync("node_modules/.pnpm").find((n) => n.startsWith("esbuild@0.25.")), "node_modules/esbuild", ), ); await esbuild.build({ entryPoints: [ "lib/video-canvas/smart-edit/render.ts", "lib/video-canvas/smart-edit/local.ts", ], outdir: ".qa/smart-edit/build", bundle: true, platform: "node", format: "cjs", packages: "external", alias: { "@": root }, }); const { renderEditPlan } = require(path.join(dir, "build/render.js")), { probeMedia, pythonAnalysis, prepareAnalysis } = require(path.join(dir, "build/local.js")); const a = { id: "a", name: "无音轨", url: "https://example.com/colors.mp4", hash: "a".repeat(64), ...(await probeMedia(path.join(dir, "colors.mp4"))), subtitles: [{ start: 0, end: 1000000, text: "红色测试" }], }, b = { id: "b", name: "有音轨", url: "https://example.com/with-audio.mp4", hash: "b".repeat(64), ...(await probeMedia(path.join(dir, "with-audio.mp4"))), subtitles: [{ start: 3000000, end: 4000000, text: "第二句" }], }; const plan = { version: 1, title: "QA", assets: [a, b], ratio: "16:9", resolution: "720p", transition: 0, subtitles: false, warnings: [], clips: [ { id: "1", assetId: "a", sourceIn: 0, sourceOut: 2000000, speed: 1, gain: 1, label: "red", }, { id: "2", assetId: "b", sourceIn: 2000000, sourceOut: 6000000, speed: 2, gain: 1, label: "blue-green", }, ], }; const files = { a: path.join(dir, "colors.mp4"), b: path.join(dir, "with-audio.mp4") }; for (const transition of [0, 0.5]) { const out = path.join(dir, "render-" + transition); fs.mkdirSync(out, { recursive: true }); const result = await renderEditPlan( { ...plan, transition, subtitles: true, music: { url: "https://example.com/beats.wav", gain: 0.2, start: 0 }, }, files, path.join(dir, "beats.wav"), out, ); const probe = await probeMedia(result.file); assert.equal(probe.width, 1280); assert.equal(probe.height, 720); assert.equal(probe.hasAudio, true); assert.ok(Math.abs(probe.duration / 1e6 - (4 - transition)) < 0.15, JSON.stringify(probe)); const srt = fs.readFileSync(result.srt, "utf8"); assert.match(srt, /红色测试/); assert.match(srt, /第二句/); console.log( "PASS: actual FFmpeg " + transition + "s transition, speed 2x, silent source, music, subtitles, 720p and duration " + probe.duration / 1e6, ); } for (const type of ["fade", "fadeblack", "fadewhite"]) { const out = path.join(dir, "manual-" + type); fs.mkdirSync(out, { recursive: true }); const mixed = { ...plan, subtitles: true, clips: Array.from({ length: 4 }, (_, i) => ({ ...plan.clips[0], id: "mixed-" + i, sourceIn: (i % 3) * 1000000, sourceOut: ((i % 3) + 1) * 1000000, transition: { type, duration: i === 2 ? 0 : 0.3 }, })), captions: [ { id: "manual", start: 0, end: 2000000, text: "手动字幕 测试", style: { fontSize: 36, color: "#ffff00", outlineColor: "#000000", outlineWidth: 2, opacity: 1, spacing: 2, }, }, ], }; const result = await renderEditPlan(mixed, files, undefined, out); const probe = await probeMedia(result.file); assert.ok(Math.abs(probe.duration / 1e6 - 3.4) < 0.15, JSON.stringify(probe)); assert.match( fs.readFileSync(path.join(out, "subtitles.ass"), "utf8"), /Noto Sans CJK SC,36/, ); assert.match(fs.readFileSync(result.srt, "utf8"), /手动字幕/); const pixel = execFileSync("ffmpeg", [ "-v", "error", "-ss", "0.833333", "-i", result.file, "-frames:v", "1", "-vf", "crop=2:2:400:200", "-f", "rawvideo", "-pix_fmt", "rgb24", "-", ]); if (type === "fadeblack") assert.ok(pixel[0] < 70, "black transition must darken the midpoint"); if (type === "fadewhite") assert.ok(pixel[1] > 180, "white transition must brighten the midpoint"); if (type === "fade") assert.ok(pixel[0] > 220 && pixel[1] < 20, "crossfade must preserve equal red sources"); console.log( "PASS: real " + type + " → hard cut → " + type + ", styled manual caption, duration " + probe.duration / 1e6, ); } for (const ratio of ["21:9", "16:9", "9:16", "1:1", "4:3", "3:4"]) { const out = path.join(dir, "ratio-" + ratio.replace(":", "-")); fs.mkdirSync(out, { recursive: true }); const result = await renderEditPlan( { ...plan, ratio, resolution: "480p", clips: [{ ...plan.clips[0], sourceOut: 1000000 }] }, files, undefined, out, ); const probe = await probeMedia(result.file); const [w, h] = ratio.split(":").map(Number); assert.equal(Math.min(probe.width, probe.height), 480); assert.ok(Math.abs(probe.width / probe.height - w / h) < 0.005, JSON.stringify(probe)); assert.equal(probe.width % 2, 0); assert.equal(probe.height % 2, 0); console.log("PASS: real 480P " + ratio + " " + probe.width + "×" + probe.height); } for (const type of ["fade", "fadeblack", "fadewhite"]) { for (const duration of [2.2, 4]) { const out = path.join(dir, `center-${type}-${duration}`); fs.mkdirSync(out, { recursive: true }); const centered = { ...plan, resolution: "480p", clips: [ { ...plan.clips[0], sourceOut: 2e6 }, { ...plan.clips[0], id: "incoming", sourceIn: 2e6, sourceOut: 4e6, transition: { type, duration, alignment: "center" }, }, ], }; const result = await renderEditPlan(centered, files, undefined, out), probe = await probeMedia(result.file); assert.ok( Math.abs(probe.duration - 4e6) < 150000, `centered transition must keep 4 seconds: ${JSON.stringify(probe)}`, ); const pixel = execFileSync("ffmpeg", [ "-v", "error", "-ss", "2", "-i", result.file, "-frames:v", "1", "-vf", "crop=2:2:400:200", "-f", "rawvideo", "-pix_fmt", "rgb24", "-", ]); if (type === "fadeblack") assert.ok(pixel[0] < 30 && pixel[2] < 30, `black midpoint ${pixel.slice(0, 3)}`); if (type === "fadewhite") assert.ok( pixel[0] > 220 && pixel[1] > 220 && pixel[2] > 220, `white midpoint ${pixel.slice(0, 3)}`, ); if (type === "fade") assert.ok(pixel[0] > 80 && pixel[2] > 80, `crossfade midpoint ${pixel.slice(0, 3)}`); console.log( `PASS: centered ${type} ${duration}s preserves 4s edit; midpoint pixel verified`, ); } } // Verify edited geometry, detached audio, gain, fades and black tail in actual encoded media. const sine = path.join(dir, "steady-tone.mp4"); execFileSync("ffmpeg", [ "-v", "error", "-y", "-f", "lavfi", "-i", "color=c=red:s=320x180:r=30:d=4", "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=4", "-c:v", "libx264", "-threads", "2", "-c:a", "aac", sine, ]); const tone = { ...b, id: "tone", subtitles: [], ...(await probeMedia(sine)) }; const extensionFiles = { tone: sine }; const base = { ...plan, resolution: "480p", assets: [tone], clips: [ { ...plan.clips[0], assetId: "tone", sourceOut: 2e6, }, ], }; const actual = async (name, edited) => { const out = path.join(dir, name); fs.mkdirSync(out, { recursive: true }); return renderEditPlan(edited, extensionFiles, undefined, out); }; const pixelAt = (file, time, x, y) => execFileSync("ffmpeg", [ "-v", "error", "-ss", String(time), "-i", file, "-frames:v", "1", "-vf", `crop=2:2:${x}:${y}`, "-f", "rawvideo", "-pix_fmt", "rgb24", "-", ]); const rms = (file, time, duration = 0.2) => { const data = execFileSync("ffmpeg", [ "-v", "error", "-ss", String(time), "-i", file, "-t", String(duration), "-vn", "-ac", "1", "-ar", "48000", "-f", "f32le", "-", ]); let sum = 0; for (let i = 0; i < data.length; i += 4) sum += data.readFloatLE(i) ** 2; return Math.sqrt(sum / (data.length / 4)); }; const baseline = await actual("audio-baseline", base); const detached = await actual("audio-detached", { ...base, clips: [ { ...base.clips[0], audioDetached: true, transform: { scale: 0.5, x: 0.2, y: -0.1 } }, ], audioClips: [ { id: "track", assetId: "tone", start: 0, sourceIn: 0, sourceOut: 2e6, speed: 1, gain: 1, fadeIn: 0, fadeOut: 0, }, ], }); const ratio = rms(detached.file, 0.8) / rms(baseline.file, 0.8); assert.ok(ratio > 0.9 && ratio < 1.1, `detached sound must not double: ${ratio}`); assert.ok( pixelAt(detached.file, 0.8, 600, 150)[0] > 220, "shifted scaled video should be red", ); assert.ok( pixelAt(detached.file, 0.8, 150, 150)[0] < 10, "outside transformed video should be black", ); const delayedPlan = { ...base, clips: [{ ...base.clips[0], audioDetached: true }], audioClips: [ { id: "delayed", assetId: "tone", start: 1e6, sourceIn: 0, sourceOut: 4e6, speed: 2, gain: 0.5, fadeIn: 0.5, fadeOut: 0.5, }, ], }; const delayed = await actual("audio-delayed-tail", delayedPlan); const delayedProbe = await probeMedia(delayed.file); assert.ok(Math.abs(delayedProbe.duration - 3e6) < 150000, JSON.stringify(delayedProbe)); assert.ok(rms(delayed.file, 0.4) < 0.0001, "detached original must stay silent"); const middleRms = rms(delayed.file, 1.7); assert.ok( middleRms > rms(baseline.file, 0.8) * 0.4 && middleRms < rms(baseline.file, 0.8) * 0.6, "independent gain should be applied once", ); assert.ok(rms(delayed.file, 1.05, 0.1) < middleRms * 0.4, "audio fade in"); assert.ok(rms(delayed.file, 2.8, 0.1) < middleRms * 0.5, "audio fade out"); assert.ok(pixelAt(delayed.file, 2.6, 200, 200)[0] < 10, "audio beyond video uses black tail"); const withCenter = { ...base, clips: [ base.clips[0], { ...base.clips[0], id: "center-audio", sourceIn: 2e6, sourceOut: 4e6, transition: { type: "fadeblack", duration: 2.2, alignment: "center" }, }, ], }; const centerSound = await actual("center-original-audio", withCenter); const centerDetached = await actual("center-detached-audio", { ...withCenter, clips: [withCenter.clips[0], { ...withCenter.clips[1], audioDetached: true }], audioClips: [ { id: "center-detached", assetId: "tone", start: 2e6, sourceIn: 2e6, sourceOut: 4e6, speed: 1, gain: 1, fadeIn: 1.1, fadeOut: 0, fadeInStart: 0.5, }, ], }); for (const time of [1.3, 2.1, 2.6, 3.5]) { const ratio = rms(centerDetached.file, time, 0.1) / rms(centerSound.file, time, 0.1); assert.ok( ratio > 0.85 && ratio < 1.15, `detaching centered-transition audio must preserve its envelope at ${time}s: ${ratio}`, ); } console.log("PASS: centered transition audio envelope preserved after detachment"); const captioned = await actual("caption-position", { ...base, subtitles: true, captions: [ { id: "pos", text: "位置测试", start: 0, end: 2e6, style: { fontSize: 80, color: "#ffffff", outlineColor: "#000000", outlineWidth: 8, outlineOpacity: 0, opacity: 1, spacing: 0, x: 0.25, y: 0.5, }, }, ], }); const frame = execFileSync( "ffmpeg", [ "-v", "error", "-ss", "0.8", "-i", captioned.file, "-frames:v", "1", "-f", "rawvideo", "-pix_fmt", "rgb24", "-", ], { maxBuffer: 4 * 1024 * 1024 }, ); let whitePixels = 0, blackPixels = 0; for (let y = 170; y < 240; y++) for (let x = 90; x < 340; x++) { const offset = (y * 854 + x) * 3; if (frame[offset] > 200 && frame[offset + 1] > 200 && frame[offset + 2] > 200) whitePixels++; if (frame[offset] < 30 && frame[offset + 1] < 30 && frame[offset + 2] < 30) blackPixels++; } assert.ok(whitePixels > 150, `caption must render at moved position: ${whitePixels}`); assert.ok( blackPixels < 10, `zero outline opacity must remove only the black outline: ${blackPixels}`, ); console.log( "PASS: real transform pixels, caption position, detached audio without doubling, 2x audio, gain, fades, delay and black tail", ); const boosted = await actual("positive-gain", { ...base, clips: [{ ...base.clips[0], gain: 2 }], }); const boostRatio = rms(boosted.file, 0.8) / rms(baseline.file, 0.8); assert.ok( boostRatio > 1.85 && boostRatio < 2.15, `+6 dB should double amplitude: ${boostRatio}`, ); const muted = await actual("muted-gain", { ...base, clips: [{ ...base.clips[0], gain: 2, muted: true }], }); assert.ok( rms(muted.file, 0.8) < 0.0001, "mute must silence original sound without overwriting gain", ); const independentMuted = await actual("muted-independent-gain", { ...delayedPlan, audioClips: delayedPlan.audioClips.map((clip) => ({ ...clip, muted: true })), }); assert.ok(rms(independentMuted.file, 1.7) < 0.0001, "mute must silence detached sound"); const fontHashes = new Set(); for (const fontFamily of ["source-han", "noto-sans-sc", "noto-serif-sc", "roboto", "inter", ...(process.env.SMART_EDIT_TEST_CATALOG_FONTS ? ["catalog-0006-font", "catalog-0007-abeezee", "catalog-0014-abril-fatface", "catalog-0365-cjgdgmh", "catalog-1021-molle", "catalog-1685-scjssht"] : [])]) { const output = await actual("font-" + fontFamily, { ...base, subtitles: true, captions: [ { id: "font", text: "字幕字体 AaGg 123", start: 0, end: 2e6, style: { fontFamily, referenceHeight: 1080, fontSize: 56, color: "#ffffff", outlineColor: "#ffffff", outlineWidth: 0, opacity: 1, spacing: 0, x: 0.5, y: 0.5, }, }, ], }); const pixels = execFileSync( "ffmpeg", [ "-v", "error", "-ss", "0.8", "-i", output.file, "-frames:v", "1", "-f", "rawvideo", "-pix_fmt", "rgb24", "-", ], { maxBuffer: 4 * 1024 * 1024 }, ); let light = 0; for (let i = 0; i < pixels.length; i += 3) if (pixels[i] > 200 && pixels[i + 1] > 200 && pixels[i + 2] > 200) light++; assert.ok(light > 150, `${fontFamily} must draw glyphs: ${light}`); fontHashes.add(require("node:crypto").createHash("sha256").update(pixels).digest("hex")); } assert.ok( fontHashes.size >= 4, "selected fonts must change actual glyph pixels, not all fall back to one face", ); console.log( `PASS: positive gain amplitude, original/detached mute, five packaged fonts${process.env.SMART_EDIT_TEST_CATALOG_FONTS ? " and six catalog fonts" : ""}`, ); if (process.env.SMART_EDIT_PYTHON) { const prepared = await prepareAnalysis(files.b, "fixture", dir, true); const analysis = await pythonAnalysis( { version: 1, video: prepared.proxy, audio: prepared.audio, duration: 6000000 }, dir, ); assert.ok(analysis.scenes.length >= 3, JSON.stringify(analysis)); assert.ok(analysis.active.length >= 2, JSON.stringify(analysis)); const beats = await pythonAnalysis( { version: 1, audio: path.join(dir, "beats.wav"), beats: true }, dir, ); assert.ok(beats.beats.length > 10, JSON.stringify(beats)); console.log( "PASS: actual PySceneDetect found " + analysis.scenes.length + " scenes; RMS gate " + analysis.active.length + " active intervals; librosa " + beats.beats.length + " beats", ); } fs.writeFileSync(path.join(dir, "fixture-plan.json"), JSON.stringify(plan, null, 2)); })().catch((e) => { console.error(e); process.exitCode = 1; });