diff --git a/app/api/canvas/audio-separate/route.ts b/app/api/canvas/audio-separate/route.ts new file mode 100644 index 000000000..0ab3de8d9 --- /dev/null +++ b/app/api/canvas/audio-separate/route.ts @@ -0,0 +1,111 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { uploadFile } from "@/lib/storage/service"; +import { ffmpeg, mediaSafety, probeMedia } from "@/lib/video-canvas/smart-edit/local"; +import { + remoteAudio, + remoteAudioConfig, +} from "@/lib/video-canvas/smart-edit/remote-audio"; + +export const runtime = "nodejs"; +export const maxDuration = 600; + +const running = new Set(); +const allowedStems = ["vocals", "background"] as const; + +export async function POST(request: Request) { + const session = await auth(); + const userId = session?.user?.id; + if (!userId) + return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + const body = (await request.json().catch(() => null)) as { + url?: unknown; + stem?: unknown; + } | null; + const url = typeof body?.url === "string" ? body.url.trim() : ""; + const stem = typeof body?.stem === "string" ? body.stem : ""; + if (!/^https?:\/\//i.test(url) || !allowedStems.includes(stem as (typeof allowedStems)[number])) + return NextResponse.json({ success: false, error: "参数不完整" }, { status: 400 }); + if (!remoteAudioConfig()) + return NextResponse.json( + { success: false, error: "音频分离服务未配置，请联系管理员" }, + { status: 503 } + ); + const lockKey = `${userId}:${url}:${stem}`; + if (running.has(lockKey) || running.size >= 3) + return NextResponse.json( + { success: false, error: "已有分离任务在进行，请稍候" }, + { status: 429 } + ); + running.add(lockKey); + let dir = ""; + try { + dir = await mkdtemp(path.join(tmpdir(), "canvas-separate-")); + const input = path.join(dir, "input"); + const response = await fetch(url, { + signal: AbortSignal.timeout(180000), + cache: "no-store", + }); + if (!response.ok) throw new Error("素材下载失败，请确认素材仍可访问"); + const bytes = Buffer.from(await response.arrayBuffer()); + if (!bytes.length || bytes.length > 150 * 1024 * 1024) + throw new Error("素材必须非空且不超过 125 MiB"); + await writeFile(input, bytes); + const info = await probeMedia(input, 600); + if (!info.hasAudio) throw new Error("当前视频没有音轨"); + const inputAudio = path.join(dir, "separation-input.wav"); + await ffmpeg( + [ + ...mediaSafety, + "-i", + input, + "-vn", + "-af", + "apad", + "-ac", + "2", + "-ar", + "44100", + "-c:a", + "pcm_f32le", + inputAudio, + ], + undefined, + 240000 + ); + const requestKey = createHash("sha256") + .update(`${url}:${stem}:${(await stat(inputAudio)).size}`) + .digest("hex"); + await remoteAudio("separate", inputAudio, userId, requestKey, dir); + const stemFile = path.join(dir, `${stem}.wav`); + const wav = await readFile(stemFile); + if (wav.length < 1024) throw new Error("分离结果异常，请稍后重试"); + const uploaded = await uploadFile( + `canvas-separate-${randomUUID()}.wav`, + await new Blob([wav]).arrayBuffer() + ); + return NextResponse.json({ + success: true, + data: { + url: uploaded.url, + duration: Math.round(info.duration / 1e6), + }, + }); + } catch (error) { + return NextResponse.json( + { + success: false, + error: + error instanceof Error ? error.message : "人声分离失败，原音轨已保留", + }, + { status: 500 } + ); + } finally { + running.delete(lockKey); + if (dir) await rm(dir, { recursive: true, force: true }); + } +} diff --git a/components/canvas/LiblibVideoCanvasReplica.tsx b/components/canvas/LiblibVideoCanvasReplica.tsx index dceed2e82..d8ab6aa45 100644 --- a/components/canvas/LiblibVideoCanvasReplica.tsx +++ b/components/canvas/LiblibVideoCanvasReplica.tsx @@ -46,6 +46,7 @@ import { FolderOpen, GitBranch, Headphones, + Mic, HelpCircle, Image as ImageIcon, Keyboard, @@ -8244,7 +8245,6 @@ export function LiblibVideoCanvasReplica({ const splitVideoAudioTrack = useCallback( async (nodeId: string, mode: VideoAudioSplitMode) => { - if (mode !== "audio-video") { toast.error("人声与背景音乐分离需要专用音频分离模型，当前尚未接通。"); return; } const videoNode = nodes.find((item) => item.id === nodeId); if (!videoNode || videoAudioSplitStates[nodeId]?.status === "loading") return; const splitCopy: Record< @@ -8313,14 +8313,31 @@ export function LiblibVideoCanvasReplica({ toast.info(`${copy.processingLabel}处理中，请稍候…`); try { - const response = await fetch("/api/canvas/video/process", {method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({operation:"audio-video",url:sourceUrl})}); - const result = await response.json(); - if (!response.ok || !result.success || !result.data?.audioUrl || !result.data?.url) throw new Error(result.error || "音视频分离失败"); + let audioUrl: string; + let silentUrl: string | null = null; + if (mode === "audio-video") { + const response = await fetch("/api/canvas/video/process", {method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({operation:"audio-video",url:sourceUrl})}); + const result = await response.json(); + if (!response.ok || !result.success || !result.data?.audioUrl || !result.data?.url) throw new Error(result.error || "音视频分离失败"); + audioUrl = result.data.audioUrl; + silentUrl = result.data.url; + } else { + const response = await fetch("/api/canvas/audio-separate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ url: sourceUrl, stem: mode === "voice" ? "vocals" : "background" }), + signal: AbortSignal.timeout(600_000), + }); + const result = await response.json(); + if (!response.ok || !result.success || !result.data?.url) + throw new Error(result.error || "人声分离失败，原音轨已保留"); + audioUrl = result.data.url; + } const audioCreated = createMediaResourceNode({ action: copy.audioAction, height: 148, mediaType: "audio", - mediaUrl: result.data.audioUrl, + mediaUrl: audioUrl, title: `${baseTitle}_${copy.audioSuffix}`, width: 350, x: rightX, @@ -8338,41 +8355,52 @@ export function LiblibVideoCanvasReplica({ splitMode: mode, }, }; - const silentCreated = createMediaResourceNode({ - action: copy.silentAction, - height: 350, - mediaType: "video", - mediaUrl: result.data.url, - title: `${baseTitle}_无声`, - width: 350, - x: rightX, - y: splitResultY, - }); - const silentVideoNode: CloneNode = { - ...silentCreated, - subtitle: "无声视频", - draftValues: { - ...(silentCreated.draftValues || {}), - durationSeconds, - generated: true, - mediaType: "video", - nodeType: "video", - outputSizeLabel, - silentVideo: true, - sourceUrl, - sourceVideoNodeId: videoNode.id, - sourceVideoTitle: videoNode.title, - splitMode: mode, - }, - }; + const silentCreated = silentUrl + ? createMediaResourceNode({ + action: copy.silentAction, + height: 350, + mediaType: "video", + mediaUrl: silentUrl, + title: `${baseTitle}_无声`, + width: 350, + x: rightX, + y: splitResultY, + }) + : null; + const silentVideoNode: CloneNode | null = silentCreated + ? { + ...silentCreated, + subtitle: "无声视频", + draftValues: { + ...(silentCreated.draftValues || {}), + durationSeconds, + generated: true, + mediaType: "video", + nodeType: "video", + outputSizeLabel, + silentVideo: true, + sourceUrl, + sourceVideoNodeId: videoNode.id, + sourceVideoTitle: videoNode.title, + splitMode: mode, + }, + } + : null; - addNodes( - [silentVideoNode, audioNode], - [ - { id: makeId("edge"), source: videoNode.id, target: silentVideoNode.id }, + if (silentVideoNode) { + addNodes( + [silentVideoNode, audioNode], + [ + { id: makeId("edge"), source: videoNode.id, target: silentVideoNode.id }, + { id: makeId("edge"), source: videoNode.id, target: audioNode.id }, + ] + ); + } else { + addNodes([audioNode], [ { id: makeId("edge"), source: videoNode.id, target: audioNode.id }, - ] - ); + ]); + } + toast.success(mode === "audio-video" ? "音视频分离完成" : `${copy.processingLabel}完成，已生成音轨`); setSelectedNodeId(null); setSelectedEdgeId(null); setVideoAudioSplitStates((current) => { @@ -12426,7 +12454,24 @@ function OfficialVideoMediaToolbar({ /> {menu === "audio" ? ( -

人声/背景音分离尚未接通

+ } + label="人声分离" + onClick={() => { + setMenu(null); + onVoiceSplit("voice"); + }} + /> + } + label="背景音分离" + onClick={() => { + setMenu(null); + onVoiceSplit("background"); + }} + /> }