Wire canvas voice/background separation to remote HTDemucs service
The private FunASR/HTDemucs audio service was already live and used by the smart edit pipeline, but the canvas video toolbar still claimed voice/background separation was unavailable. Add a dedicated /api/canvas/audio-separate route (download, ffmpeg stem extraction, remote separation, storage upload) and route the toolbar's voice and background modes through it, replacing the stale not-connected notice.
This commit is contained in:
134
parent
c1975942ac
commit
6ed8879b28
111
app/api/canvas/audio-separate/route.ts
Normal file
111
app/api/canvas/audio-separate/route.ts
Normal file
@ -0,0 +1,111 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { NextResponse } from "next/server";
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
import { ffmpeg, mediaSafety, probeMedia } from "@/lib/video-canvas/smart-edit/local";
|
||||
import {
|
||||
remoteAudio,
|
||||
remoteAudioConfig,
|
||||
} from "@/lib/video-canvas/smart-edit/remote-audio";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 600;
|
||||
|
||||
const running = new Set<string>();
|
||||
const allowedStems = ["vocals", "background"] as const;
|
||||
|
||||
export async function POST(request: Request) {
|
||||
const session = await auth();
|
||||
const userId = session?.user?.id;
|
||||
if (!userId)
|
||||
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
|
||||
const body = (await request.json().catch(() => null)) as {
|
||||
url?: unknown;
|
||||
stem?: unknown;
|
||||
} | null;
|
||||
const url = typeof body?.url === "string" ? body.url.trim() : "";
|
||||
const stem = typeof body?.stem === "string" ? body.stem : "";
|
||||
if (!/^https?:\/\//i.test(url) || !allowedStems.includes(stem as (typeof allowedStems)[number]))
|
||||
return NextResponse.json({ success: false, error: "参数不完整" }, { status: 400 });
|
||||
if (!remoteAudioConfig())
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "音频分离服务未配置，请联系管理员" },
|
||||
{ status: 503 }
|
||||
);
|
||||
const lockKey = `${userId}:${url}:${stem}`;
|
||||
if (running.has(lockKey) || running.size >= 3)
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "已有分离任务在进行，请稍候" },
|
||||
{ status: 429 }
|
||||
);
|
||||
running.add(lockKey);
|
||||
let dir = "";
|
||||
try {
|
||||
dir = await mkdtemp(path.join(tmpdir(), "canvas-separate-"));
|
||||
const input = path.join(dir, "input");
|
||||
const response = await fetch(url, {
|
||||
signal: AbortSignal.timeout(180000),
|
||||
cache: "no-store",
|
||||
});
|
||||
if (!response.ok) throw new Error("素材下载失败，请确认素材仍可访问");
|
||||
const bytes = Buffer.from(await response.arrayBuffer());
|
||||
if (!bytes.length || bytes.length > 150 * 1024 * 1024)
|
||||
throw new Error("素材必须非空且不超过 125 MiB");
|
||||
await writeFile(input, bytes);
|
||||
const info = await probeMedia(input, 600);
|
||||
if (!info.hasAudio) throw new Error("当前视频没有音轨");
|
||||
const inputAudio = path.join(dir, "separation-input.wav");
|
||||
await ffmpeg(
|
||||
[
|
||||
...mediaSafety,
|
||||
"-i",
|
||||
input,
|
||||
"-vn",
|
||||
"-af",
|
||||
"apad",
|
||||
"-ac",
|
||||
"2",
|
||||
"-ar",
|
||||
"44100",
|
||||
"-c:a",
|
||||
"pcm_f32le",
|
||||
inputAudio,
|
||||
],
|
||||
undefined,
|
||||
240000
|
||||
);
|
||||
const requestKey = createHash("sha256")
|
||||
.update(`${url}:${stem}:${(await stat(inputAudio)).size}`)
|
||||
.digest("hex");
|
||||
await remoteAudio("separate", inputAudio, userId, requestKey, dir);
|
||||
const stemFile = path.join(dir, `${stem}.wav`);
|
||||
const wav = await readFile(stemFile);
|
||||
if (wav.length < 1024) throw new Error("分离结果异常，请稍后重试");
|
||||
const uploaded = await uploadFile(
|
||||
`canvas-separate-${randomUUID()}.wav`,
|
||||
await new Blob([wav]).arrayBuffer()
|
||||
);
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
url: uploaded.url,
|
||||
duration: Math.round(info.duration / 1e6),
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error:
|
||||
error instanceof Error ? error.message : "人声分离失败，原音轨已保留",
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
} finally {
|
||||
running.delete(lockKey);
|
||||
if (dir) await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@ -46,6 +46,7 @@ import {
|
||||
FolderOpen,
|
||||
GitBranch,
|
||||
Headphones,
|
||||
Mic,
|
||||
HelpCircle,
|
||||
Image as ImageIcon,
|
||||
Keyboard,
|
||||
@ -8244,7 +8245,6 @@ export function LiblibVideoCanvasReplica({
|
||||
|
||||
const splitVideoAudioTrack = useCallback(
|
||||
async (nodeId: string, mode: VideoAudioSplitMode) => {
|
||||
if (mode !== "audio-video") { toast.error("人声与背景音乐分离需要专用音频分离模型，当前尚未接通。"); return; }
|
||||
const videoNode = nodes.find((item) => item.id === nodeId);
|
||||
if (!videoNode || videoAudioSplitStates[nodeId]?.status === "loading") return;
|
||||
const splitCopy: Record<
|
||||
@ -8313,14 +8313,31 @@ export function LiblibVideoCanvasReplica({
|
||||
toast.info(`${copy.processingLabel}处理中，请稍候…`);
|
||||
|
||||
try {
|
||||
const response = await fetch("/api/canvas/video/process", {method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({operation:"audio-video",url:sourceUrl})});
|
||||
const result = await response.json();
|
||||
if (!response.ok || !result.success || !result.data?.audioUrl || !result.data?.url) throw new Error(result.error || "音视频分离失败");
|
||||
let audioUrl: string;
|
||||
let silentUrl: string | null = null;
|
||||
if (mode === "audio-video") {
|
||||
const response = await fetch("/api/canvas/video/process", {method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({operation:"audio-video",url:sourceUrl})});
|
||||
const result = await response.json();
|
||||
if (!response.ok || !result.success || !result.data?.audioUrl || !result.data?.url) throw new Error(result.error || "音视频分离失败");
|
||||
audioUrl = result.data.audioUrl;
|
||||
silentUrl = result.data.url;
|
||||
} else {
|
||||
const response = await fetch("/api/canvas/audio-separate", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ url: sourceUrl, stem: mode === "voice" ? "vocals" : "background" }),
|
||||
signal: AbortSignal.timeout(600_000),
|
||||
});
|
||||
const result = await response.json();
|
||||
if (!response.ok || !result.success || !result.data?.url)
|
||||
throw new Error(result.error || "人声分离失败，原音轨已保留");
|
||||
audioUrl = result.data.url;
|
||||
}
|
||||
const audioCreated = createMediaResourceNode({
|
||||
action: copy.audioAction,
|
||||
height: 148,
|
||||
mediaType: "audio",
|
||||
mediaUrl: result.data.audioUrl,
|
||||
mediaUrl: audioUrl,
|
||||
title: `${baseTitle}_${copy.audioSuffix}`,
|
||||
width: 350,
|
||||
x: rightX,
|
||||
@ -8338,41 +8355,52 @@ export function LiblibVideoCanvasReplica({
|
||||
splitMode: mode,
|
||||
},
|
||||
};
|
||||
const silentCreated = createMediaResourceNode({
|
||||
action: copy.silentAction,
|
||||
height: 350,
|
||||
mediaType: "video",
|
||||
mediaUrl: result.data.url,
|
||||
title: `${baseTitle}_无声`,
|
||||
width: 350,
|
||||
x: rightX,
|
||||
y: splitResultY,
|
||||
});
|
||||
const silentVideoNode: CloneNode = {
|
||||
...silentCreated,
|
||||
subtitle: "无声视频",
|
||||
draftValues: {
|
||||
...(silentCreated.draftValues || {}),
|
||||
durationSeconds,
|
||||
generated: true,
|
||||
mediaType: "video",
|
||||
nodeType: "video",
|
||||
outputSizeLabel,
|
||||
silentVideo: true,
|
||||
sourceUrl,
|
||||
sourceVideoNodeId: videoNode.id,
|
||||
sourceVideoTitle: videoNode.title,
|
||||
splitMode: mode,
|
||||
},
|
||||
};
|
||||
const silentCreated = silentUrl
|
||||
? createMediaResourceNode({
|
||||
action: copy.silentAction,
|
||||
height: 350,
|
||||
mediaType: "video",
|
||||
mediaUrl: silentUrl,
|
||||
title: `${baseTitle}_无声`,
|
||||
width: 350,
|
||||
x: rightX,
|
||||
y: splitResultY,
|
||||
})
|
||||
: null;
|
||||
const silentVideoNode: CloneNode | null = silentCreated
|
||||
? {
|
||||
...silentCreated,
|
||||
subtitle: "无声视频",
|
||||
draftValues: {
|
||||
...(silentCreated.draftValues || {}),
|
||||
durationSeconds,
|
||||
generated: true,
|
||||
mediaType: "video",
|
||||
nodeType: "video",
|
||||
outputSizeLabel,
|
||||
silentVideo: true,
|
||||
sourceUrl,
|
||||
sourceVideoNodeId: videoNode.id,
|
||||
sourceVideoTitle: videoNode.title,
|
||||
splitMode: mode,
|
||||
},
|
||||
}
|
||||
: null;
|
||||
|
||||
addNodes(
|
||||
[silentVideoNode, audioNode],
|
||||
[
|
||||
{ id: makeId("edge"), source: videoNode.id, target: silentVideoNode.id },
|
||||
if (silentVideoNode) {
|
||||
addNodes(
|
||||
[silentVideoNode, audioNode],
|
||||
[
|
||||
{ id: makeId("edge"), source: videoNode.id, target: silentVideoNode.id },
|
||||
{ id: makeId("edge"), source: videoNode.id, target: audioNode.id },
|
||||
]
|
||||
);
|
||||
} else {
|
||||
addNodes([audioNode], [
|
||||
{ id: makeId("edge"), source: videoNode.id, target: audioNode.id },
|
||||
]
|
||||
);
|
||||
]);
|
||||
}
|
||||
toast.success(mode === "audio-video" ? "音视频分离完成" : `${copy.processingLabel}完成，已生成音轨`);
|
||||
setSelectedNodeId(null);
|
||||
setSelectedEdgeId(null);
|
||||
setVideoAudioSplitStates((current) => {
|
||||
@ -12426,7 +12454,24 @@ function OfficialVideoMediaToolbar({
|
||||
/>
|
||||
{menu === "audio" ? (
|
||||
<VideoToolbarMenuPopover width={168}>
|
||||
<p className="px-3 py-2 text-xs text-white/40">人声/背景音分离尚未接通</p>
|
||||
<VideoToolbarMenuItem
|
||||
description="仅保留人声，输出人声音轨"
|
||||
icon={<Mic className="h-4 w-4" strokeWidth={2.1} />}
|
||||
label="人声分离"
|
||||
onClick={() => {
|
||||
setMenu(null);
|
||||
onVoiceSplit("voice");
|
||||
}}
|
||||
/>
|
||||
<VideoToolbarMenuItem
|
||||
description="仅保留背景音，输出背景音轨"
|
||||
icon={<Music2 className="h-4 w-4" strokeWidth={2.1} />}
|
||||
label="背景音分离"
|
||||
onClick={() => {
|
||||
setMenu(null);
|
||||
onVoiceSplit("background");
|
||||
}}
|
||||
/>
|
||||
<VideoToolbarMenuItem
|
||||
description="分离内嵌音轨为独立音频节点"
|
||||
icon={<GitBranch className="h-4 w-4" strokeWidth={2.1} />}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user