import { remoteAudio, remoteAudioConfig } from "./remote-audio"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { createHash, randomUUID } from "node:crypto"; import { readFile } from "node:fs/promises"; import path from "node:path"; import { SECOND, editPlanSchema, planTimeline, editableCaptions, newCaptionStyle, type EditPlan, type TimelineCaption, } from "./contract"; import { materialize, probeMedia, ffmpeg, mediaSafety, prepareAudio, pythonAnalysis, } from "./local"; export async function processEditAudio( operation: "separate" | "transcribe", plan: EditPlan, clipId: string | undefined, userId: string, dir: string, progress: (stage: string) => Promise, upload: (file: string, suffix: string) => Promise, track: "video" | "audio" = "video", keep: "both" | "vocals" | "background" = "both", requestKey: string = userId ): Promise { const timeline = planTimeline(plan); if (operation === "separate") { const index = plan.clips.findIndex((c) => c.id === clipId); const audio = track === "audio" ? plan.audioClips?.find((c) => c.id === clipId) : undefined; const clip = track === "audio" ? audio : plan.clips[index]; if (!clip || (track === "video" && plan.clips[index].audioDetached)) throw Error("请选择可分离的音视频片段"); if ( (plan.audioClips?.length || 0) + (keep === "both" ? 2 : 1) - (audio ? 1 : 0) > 24 ) throw Error("分离需要空闲音轨,最多支持 24 个音频片段"); const asset = plan.assets.find((a) => a.id === clip.assetId)!; if (!asset.hasAudio) throw Error("当前视频没有音轨"); await progress("准备人声分离素材"); const media = await materialize(asset.url, userId); if (media.hash !== asset.hash) throw Error("原素材内容已变化,请重新解析"); const actual = await probeMedia(media.file); if (!actual.hasAudio || Math.abs(actual.duration - asset.duration) > 100000) throw Error("原音轨参数已变化,请重新解析"); const inputAudio = path.join(dir, "separation-input.wav"); await ffmpeg([ ...mediaSafety, "-ss", String(clip.sourceIn / SECOND), "-i", media.file, "-t", String((clip.sourceOut - clip.sourceIn) / SECOND), "-vn", "-af", "apad", "-ac", "2", "-ar", "44100", "-c:a", "pcm_f32le", inputAudio, ]); await progress("正在分离人声与背景声"); if (remoteAudioConfig()) { await remoteAudio( "separate", inputAudio, userId, requestKey, dir, progress ); } else try { await promisify(execFile)( process.env.SMART_EDIT_PYTHON || "python3", [path.resolve("services/smart-edit/separate.py"), inputAudio, dir], { timeout: 600000, maxBuffer: 2 * 1024 * 1024 } ); } catch (e) { const detail = String((e as { stderr?: string }).stderr || ""); throw Error( detail.includes("尚未配置") ? "人声分离模型尚未配置" : "人声分离失败或超时,原音轨已保留" ); } await progress("保存人声与背景声音轨"); const audioClips = (plan.audioClips || []).filter( (c) => !audio || c.id !== audio.id ); const used = new Set([...plan.clips, ...audioClips].map((c) => c.assetId)); const assets = plan.assets.filter((a) => used.has(a.id)); const t = timeline.clips[index], next = timeline.clips[index + 1]; for (const [stem, title] of [ ["vocals", "人声"], ["background", "背景声"], ]) { if (keep !== "both" && keep !== stem) continue; const file = path.join(dir, stem + ".wav"), metadata = await probeMedia(file); if ( Math.abs(metadata.duration - (clip.sourceOut - clip.sourceIn)) > 50000 ) throw Error("分离音轨时长异常,原音轨已保留"); const id = randomUUID(), bytes = await readFile(file); assets.push({ id, name: `${asset.name.slice(0, 180)} · ${title}`, url: await upload(file, "wav"), hash: createHash("sha256").update(bytes).digest("hex"), subtitles: [], ...metadata, }); audioClips.push({ id: randomUUID(), assetId: id, start: audio ? audio.start : t.start, sourceIn: 0, sourceOut: clip.sourceOut - clip.sourceIn, speed: clip.speed, gain: clip.gain, muted: clip.muted, fadeIn: audio ? audio.fadeIn : (t.overlap - t.padBefore) / SECOND, fadeInStart: audio ? audio.fadeInStart : t.overlap ? t.padBefore / t.overlap : 0, fadeOut: audio ? audio.fadeOut : next ? (next.overlap - t.padAfter) / SECOND : 0, fadeOutEnd: audio ? audio.fadeOutEnd : next?.overlap ? t.padAfter / next.overlap : 0, }); } return editPlanSchema.parse({ ...plan, assets, audioClips, clips: plan.clips.map((c) => !audio && c.id === clip.id ? { ...c, audioDetached: true } : c ), }); } const captions: TimelineCaption[] = editableCaptions(plan).filter( (c) => !c.id.startsWith("asr-") ); let recognized = 0; for (const [index, clip] of timeline.clips.entries()) { const asset = plan.assets.find((a) => a.id === clip.assetId)!; if (!asset.hasAudio) continue; await progress(`自动转写 ${index + 1}/${plan.clips.length}`); const media = await materialize(asset.url, userId); if (media.hash !== asset.hash) throw Error("原素材内容已变化,请重新解析"); const actual = await probeMedia(media.file); if (!actual.hasAudio || Math.abs(actual.duration - asset.duration) > 100000) throw Error("原音轨参数已变化,请重新解析"); const audio = await prepareAudio( media.file, path.join(media.dir, media.hash + ".wav") ); const analysis = await pythonAnalysis( { version: 1, audio, transcribe: true }, media.dir, requestKey, progress ); for (const [j, s] of analysis.subtitles.entries()) { const from = Math.max(s.start, clip.sourceIn), to = Math.min(s.end, clip.sourceOut); if (to <= from) continue; const start = Math.round( clip.start + (from - clip.sourceIn) / clip.speed ); const end = Math.min( timeline.duration, Math.round(clip.start + (to - clip.sourceIn) / clip.speed) ); if (end <= start) continue; captions.push({ id: `asr-${clip.id}-${j}`, start, end, text: s.text, style: { ...newCaptionStyle }, }); recognized++; } } if (!recognized) throw Error("未识别到有效人声,原字幕已保留"); if (captions.length > 500) throw Error("转写后字幕超过 500 条,请缩短素材"); return editPlanSchema.parse({ ...plan, subtitles: true, captions: captions.sort((a, b) => a.start - b.start), }); }