179 lines
6.6 KiB
TypeScript
179 lines
6.6 KiB
TypeScript
import { execFile } from "node:child_process";
|
|
import { promisify } from "node:util";
|
|
import { createHash, randomUUID } from "node:crypto";
|
|
import { readFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import {
|
|
SECOND,
|
|
editPlanSchema,
|
|
planTimeline,
|
|
editableCaptions,
|
|
newCaptionStyle,
|
|
type EditPlan,
|
|
type TimelineCaption,
|
|
} from "./contract";
|
|
import {
|
|
materialize,
|
|
probeMedia,
|
|
ffmpeg,
|
|
mediaSafety,
|
|
prepareAudio,
|
|
pythonAnalysis,
|
|
} from "./local";
|
|
|
|
export async function processEditAudio(
|
|
operation: "separate" | "transcribe",
|
|
plan: EditPlan,
|
|
clipId: string | undefined,
|
|
userId: string,
|
|
dir: string,
|
|
progress: (stage: string) => Promise<void>,
|
|
upload: (file: string, suffix: string) => Promise<string>,
|
|
track: "video" | "audio" = "video",
|
|
keep: "both" | "vocals" | "background" = "both",
|
|
): Promise<EditPlan> {
|
|
const timeline = planTimeline(plan);
|
|
if (operation === "separate") {
|
|
const index = plan.clips.findIndex((c) => c.id === clipId);
|
|
const audio = track === "audio" ? plan.audioClips?.find((c) => c.id === clipId) : undefined;
|
|
const clip = track === "audio" ? audio : plan.clips[index];
|
|
if (!clip || (track === "video" && plan.clips[index].audioDetached))
|
|
throw Error("请选择可分离的音视频片段");
|
|
if ((plan.audioClips?.length || 0) + (keep === "both" ? 2 : 1) - (audio ? 1 : 0) > 24)
|
|
throw Error("分离需要空闲音轨，最多支持 24 个音频片段");
|
|
const asset = plan.assets.find((a) => a.id === clip.assetId)!;
|
|
if (!asset.hasAudio) throw Error("当前视频没有音轨");
|
|
await progress("准备人声分离素材");
|
|
const media = await materialize(asset.url, userId);
|
|
if (media.hash !== asset.hash) throw Error("原素材内容已变化，请重新解析");
|
|
const actual = await probeMedia(media.file);
|
|
if (!actual.hasAudio || Math.abs(actual.duration - asset.duration) > 100000)
|
|
throw Error("原音轨参数已变化，请重新解析");
|
|
const inputAudio = path.join(dir, "separation-input.wav");
|
|
await ffmpeg([
|
|
...mediaSafety,
|
|
"-ss",
|
|
String(clip.sourceIn / SECOND),
|
|
"-i",
|
|
media.file,
|
|
"-t",
|
|
String((clip.sourceOut - clip.sourceIn) / SECOND),
|
|
"-vn",
|
|
"-af",
|
|
"apad",
|
|
"-ac",
|
|
"2",
|
|
"-ar",
|
|
"44100",
|
|
"-c:a",
|
|
"pcm_f32le",
|
|
inputAudio,
|
|
]);
|
|
await progress("正在分离人声与背景声");
|
|
try {
|
|
await promisify(execFile)(
|
|
process.env.SMART_EDIT_PYTHON || "python3",
|
|
[path.resolve("services/smart-edit/separate.py"), inputAudio, dir],
|
|
{ timeout: 600000, maxBuffer: 2 * 1024 * 1024 },
|
|
);
|
|
} catch (e) {
|
|
const detail = String((e as { stderr?: string }).stderr || "");
|
|
throw Error(
|
|
detail.includes("尚未配置")
|
|
? "人声分离模型尚未配置"
|
|
: "人声分离失败或超时，原音轨已保留",
|
|
);
|
|
}
|
|
await progress("保存人声与背景声音轨");
|
|
const audioClips = (plan.audioClips || []).filter((c) => !audio || c.id !== audio.id);
|
|
const used = new Set([...plan.clips, ...audioClips].map((c) => c.assetId));
|
|
const assets = plan.assets.filter((a) => used.has(a.id));
|
|
const t = timeline.clips[index],
|
|
next = timeline.clips[index + 1];
|
|
for (const [stem, title] of [
|
|
["vocals", "人声"],
|
|
["background", "背景声"],
|
|
]) {
|
|
if (keep !== "both" && keep !== stem) continue;
|
|
const file = path.join(dir, stem + ".wav"),
|
|
metadata = await probeMedia(file);
|
|
if (Math.abs(metadata.duration - (clip.sourceOut - clip.sourceIn)) > 50000)
|
|
throw Error("分离音轨时长异常，原音轨已保留");
|
|
const id = randomUUID(),
|
|
bytes = await readFile(file);
|
|
assets.push({
|
|
id,
|
|
name: `${asset.name.slice(0, 180)} · ${title}`,
|
|
url: await upload(file, "wav"),
|
|
hash: createHash("sha256").update(bytes).digest("hex"),
|
|
subtitles: [],
|
|
...metadata,
|
|
});
|
|
audioClips.push({
|
|
id: randomUUID(),
|
|
assetId: id,
|
|
start: audio ? audio.start : t.start,
|
|
sourceIn: 0,
|
|
sourceOut: clip.sourceOut - clip.sourceIn,
|
|
speed: clip.speed,
|
|
gain: clip.gain,
|
|
muted: clip.muted,
|
|
fadeIn: audio ? audio.fadeIn : (t.overlap - t.padBefore) / SECOND,
|
|
fadeInStart: audio ? audio.fadeInStart : t.overlap ? t.padBefore / t.overlap : 0,
|
|
fadeOut: audio ? audio.fadeOut : next ? (next.overlap - t.padAfter) / SECOND : 0,
|
|
fadeOutEnd: audio ? audio.fadeOutEnd : next?.overlap ? t.padAfter / next.overlap : 0,
|
|
});
|
|
}
|
|
return editPlanSchema.parse({
|
|
...plan,
|
|
assets,
|
|
audioClips,
|
|
clips: plan.clips.map((c) =>
|
|
!audio && c.id === clip.id ? { ...c, audioDetached: true } : c,
|
|
),
|
|
});
|
|
}
|
|
const captions: TimelineCaption[] = editableCaptions(plan).filter(
|
|
(c) => !c.id.startsWith("asr-"),
|
|
);
|
|
let recognized = 0;
|
|
for (const [index, clip] of timeline.clips.entries()) {
|
|
const asset = plan.assets.find((a) => a.id === clip.assetId)!;
|
|
if (!asset.hasAudio) continue;
|
|
await progress(`自动转写 ${index + 1}/${plan.clips.length}`);
|
|
const media = await materialize(asset.url, userId);
|
|
if (media.hash !== asset.hash) throw Error("原素材内容已变化，请重新解析");
|
|
const actual = await probeMedia(media.file);
|
|
if (!actual.hasAudio || Math.abs(actual.duration - asset.duration) > 100000)
|
|
throw Error("原音轨参数已变化，请重新解析");
|
|
const audio = await prepareAudio(media.file, path.join(media.dir, media.hash + ".wav"));
|
|
const analysis = await pythonAnalysis({ version: 1, audio, transcribe: true }, media.dir);
|
|
for (const [j, s] of analysis.subtitles.entries()) {
|
|
const from = Math.max(s.start, clip.sourceIn),
|
|
to = Math.min(s.end, clip.sourceOut);
|
|
if (to <= from) continue;
|
|
const start = Math.round(clip.start + (from - clip.sourceIn) / clip.speed);
|
|
const end = Math.min(
|
|
timeline.duration,
|
|
Math.round(clip.start + (to - clip.sourceIn) / clip.speed),
|
|
);
|
|
if (end <= start) continue;
|
|
captions.push({
|
|
id: `asr-${clip.id}-${j}`,
|
|
start,
|
|
end,
|
|
text: s.text,
|
|
style: { ...newCaptionStyle },
|
|
});
|
|
recognized++;
|
|
}
|
|
}
|
|
if (!recognized) throw Error("未识别到有效人声，原字幕已保留");
|
|
if (captions.length > 500) throw Error("转写后字幕超过 500 条，请缩短素材");
|
|
return editPlanSchema.parse({
|
|
...plan,
|
|
subtitles: true,
|
|
captions: captions.sort((a, b) => a.start - b.start),
|
|
});
|
|
}
|