773 lines
29 KiB
TypeScript
773 lines
29 KiB
TypeScript
import type { VideoClipRange } from "@/lib/video-canvas/video-clip";
|
|
import { getVideoClipDuration, normalizeVideoClipRange } from "@/lib/video-canvas/video-clip";
|
|
import type { VideoCropRect } from "@/lib/video-canvas/video-crop";
|
|
import { normalizeVideoCropRect } from "@/lib/video-canvas/video-crop";
|
|
import {
|
|
getVideoFrameExtractionTarget,
|
|
type VideoFrameExtractionKind,
|
|
} from "@/lib/video-canvas/video-frame-extraction";
|
|
import {
|
|
hasVideoAudioTrack,
|
|
normalizeVideoCanvasAudioSeparationConfig,
|
|
type VideoAudioSplitMode,
|
|
type VideoCanvasAudioSeparationConfig,
|
|
} from "@/lib/video-canvas/video-audio-separation";
|
|
import {
|
|
detectSubtitleEraseRegions,
|
|
normalizeSubtitleEraseRegion,
|
|
normalizeVideoCanvasSubtitleEraseConfig,
|
|
type SubtitleEraseMode,
|
|
type SubtitleEraseRegion,
|
|
type VideoCanvasSubtitleEraseConfig,
|
|
} from "@/lib/video-canvas/subtitle-erase";
|
|
|
|
type CapturableVideoElement = HTMLVideoElement & {
|
|
captureStream?: () => MediaStream;
|
|
mozCaptureStream?: () => MediaStream;
|
|
};
|
|
|
|
function waitForMediaEvent(
|
|
target: HTMLMediaElement,
|
|
eventName: "loadeddata" | "loadedmetadata" | "seeked",
|
|
) {
|
|
return new Promise<void>((resolve, reject) => {
|
|
const timeout = window.setTimeout(() => reject(new Error(`Video ${eventName} timeout`)), 15_000);
|
|
const cleanup = () => {
|
|
window.clearTimeout(timeout);
|
|
target.removeEventListener(eventName, handleSuccess);
|
|
target.removeEventListener("error", handleError);
|
|
};
|
|
const handleSuccess = () => {
|
|
cleanup();
|
|
resolve();
|
|
};
|
|
const handleError = () => {
|
|
cleanup();
|
|
reject(new Error("Video source could not be decoded"));
|
|
};
|
|
target.addEventListener(eventName, handleSuccess, { once: true });
|
|
target.addEventListener("error", handleError, { once: true });
|
|
});
|
|
}
|
|
|
|
function canvasToPngBlob(canvas: HTMLCanvasElement) {
|
|
return new Promise<Blob>((resolve, reject) => {
|
|
canvas.toBlob((blob) => {
|
|
if (blob) resolve(blob);
|
|
else reject(new Error("视频帧编码失败"));
|
|
}, "image/png");
|
|
});
|
|
}
|
|
|
|
function getRecorderMimeType() {
|
|
const candidates = [
|
|
"video/webm;codecs=vp9,opus",
|
|
"video/webm;codecs=vp8,opus",
|
|
"video/webm",
|
|
];
|
|
return candidates.find((candidate) => MediaRecorder.isTypeSupported(candidate)) || "";
|
|
}
|
|
|
|
function getAudioRecorderMimeType() {
|
|
const candidates = [
|
|
"audio/webm;codecs=opus",
|
|
"audio/webm",
|
|
"audio/ogg;codecs=opus",
|
|
];
|
|
return candidates.find((candidate) => MediaRecorder.isTypeSupported(candidate)) || "";
|
|
}
|
|
|
|
function getCanvasVideoSourceUrl(mediaUrl: string) {
|
|
const parsedUrl = new URL(mediaUrl, window.location.origin);
|
|
return parsedUrl.origin === window.location.origin
|
|
? parsedUrl.toString()
|
|
: `/api/canvas/video-source?url=${encodeURIComponent(parsedUrl.toString())}`;
|
|
}
|
|
|
|
async function fetchCanvasVideoBlob(mediaUrl: string) {
|
|
const response = await fetch(getCanvasVideoSourceUrl(mediaUrl), { credentials: "include" });
|
|
if (!response.ok) throw new Error("视频源加载失败");
|
|
return response.blob();
|
|
}
|
|
|
|
export async function inspectCanvasVideoAudioTrack(mediaUrl: string) {
|
|
try {
|
|
return hasVideoAudioTrack(await (await fetchCanvasVideoBlob(mediaUrl)).arrayBuffer());
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
export async function loadCanvasVideoClipFrames(mediaUrl: string, frameCount = 8) {
|
|
const response = await fetch(getCanvasVideoSourceUrl(mediaUrl), { credentials: "include" });
|
|
if (!response.ok) throw new Error("视频预览加载失败");
|
|
const objectUrl = URL.createObjectURL(await response.blob());
|
|
const video = document.createElement("video");
|
|
video.muted = true;
|
|
video.playsInline = true;
|
|
video.preload = "auto";
|
|
video.src = objectUrl;
|
|
|
|
try {
|
|
if (video.readyState < HTMLMediaElement.HAVE_METADATA) {
|
|
await waitForMediaEvent(video, "loadedmetadata");
|
|
}
|
|
const canvas = document.createElement("canvas");
|
|
const width = Math.max(160, Math.min(480, video.videoWidth || 320));
|
|
const height = Math.max(90, Math.round(width * ((video.videoHeight || 180) / (video.videoWidth || 320))));
|
|
canvas.width = width;
|
|
canvas.height = height;
|
|
const context = canvas.getContext("2d");
|
|
if (!context) return [];
|
|
const frames: string[] = [];
|
|
for (let index = 0; index < Math.max(1, frameCount); index += 1) {
|
|
const target = Math.min(
|
|
Math.max(0, video.duration - 0.05),
|
|
(video.duration * (index + 0.5)) / Math.max(1, frameCount),
|
|
);
|
|
video.currentTime = target;
|
|
await waitForMediaEvent(video, "seeked");
|
|
context.drawImage(video, 0, 0, width, height);
|
|
frames.push(canvas.toDataURL("image/jpeg", 0.72));
|
|
}
|
|
return frames;
|
|
} finally {
|
|
video.removeAttribute("src");
|
|
video.load();
|
|
URL.revokeObjectURL(objectUrl);
|
|
}
|
|
}
|
|
|
|
export async function extractCanvasVideoFrame(
|
|
mediaUrl: string,
|
|
kind: VideoFrameExtractionKind,
|
|
currentTimeSeconds: number,
|
|
displayDurationSeconds: number,
|
|
) {
|
|
const sourceBlob = await fetchCanvasVideoBlob(mediaUrl);
|
|
const sourceObjectUrl = URL.createObjectURL(sourceBlob);
|
|
const video = document.createElement("video");
|
|
video.muted = true;
|
|
video.playsInline = true;
|
|
video.preload = "auto";
|
|
video.src = sourceObjectUrl;
|
|
|
|
try {
|
|
if (video.readyState < HTMLMediaElement.HAVE_METADATA) {
|
|
await waitForMediaEvent(video, "loadedmetadata");
|
|
}
|
|
if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
|
|
await waitForMediaEvent(video, "loadeddata");
|
|
}
|
|
const durationSeconds = Number.isFinite(video.duration) && video.duration > 0
|
|
? video.duration
|
|
: displayDurationSeconds;
|
|
const targetTimeSeconds = getVideoFrameExtractionTarget(
|
|
kind,
|
|
currentTimeSeconds,
|
|
durationSeconds,
|
|
);
|
|
if (Math.abs(video.currentTime - targetTimeSeconds) > 0.001) {
|
|
video.currentTime = targetTimeSeconds;
|
|
await waitForMediaEvent(video, "seeked");
|
|
}
|
|
|
|
const width = Math.max(1, Math.round(video.videoWidth));
|
|
const height = Math.max(1, Math.round(video.videoHeight));
|
|
if (width <= 1 || height <= 1) throw new Error("视频帧尺寸无效");
|
|
const canvas = document.createElement("canvas");
|
|
canvas.width = width;
|
|
canvas.height = height;
|
|
const context = canvas.getContext("2d");
|
|
if (!context) throw new Error("视频帧画布创建失败");
|
|
context.drawImage(video, 0, 0, width, height);
|
|
return {
|
|
blob: await canvasToPngBlob(canvas),
|
|
height,
|
|
targetTimeSeconds,
|
|
width,
|
|
};
|
|
} finally {
|
|
video.pause();
|
|
video.removeAttribute("src");
|
|
video.load();
|
|
URL.revokeObjectURL(sourceObjectUrl);
|
|
}
|
|
}
|
|
|
|
export async function processCanvasVideoClip(
|
|
mediaUrl: string,
|
|
range: VideoClipRange,
|
|
displayDurationSeconds: number,
|
|
) {
|
|
const response = await fetch(getCanvasVideoSourceUrl(mediaUrl), { credentials: "include" });
|
|
if (!response.ok) throw new Error("视频源加载失败");
|
|
|
|
const sourceObjectUrl = URL.createObjectURL(await response.blob());
|
|
const video = document.createElement("video") as CapturableVideoElement;
|
|
video.muted = true;
|
|
video.playsInline = true;
|
|
video.preload = "auto";
|
|
video.src = sourceObjectUrl;
|
|
|
|
try {
|
|
if (video.readyState < HTMLMediaElement.HAVE_METADATA) {
|
|
await waitForMediaEvent(video, "loadedmetadata");
|
|
}
|
|
const normalizedRange = normalizeVideoClipRange(range);
|
|
const requestedDuration = getVideoClipDuration(normalizedRange, displayDurationSeconds);
|
|
const startSeconds = Math.min(
|
|
Math.max(0, normalizedRange.start * displayDurationSeconds),
|
|
Math.max(0, video.duration - 0.05),
|
|
);
|
|
const endSeconds = Math.min(video.duration, startSeconds + requestedDuration);
|
|
video.currentTime = startSeconds;
|
|
if (Math.abs(video.currentTime - startSeconds) > 0.01 || startSeconds > 0) {
|
|
await waitForMediaEvent(video, "seeked");
|
|
}
|
|
|
|
const captureStream = video.captureStream || video.mozCaptureStream;
|
|
if (!captureStream || typeof MediaRecorder === "undefined") {
|
|
throw new Error("当前浏览器不支持视频剪辑");
|
|
}
|
|
const stream = captureStream.call(video);
|
|
const mimeType = getRecorderMimeType();
|
|
const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined);
|
|
const chunks: BlobPart[] = [];
|
|
recorder.addEventListener("dataavailable", (event) => {
|
|
if (event.data.size > 0) chunks.push(event.data);
|
|
});
|
|
|
|
const completed = new Promise<Blob>((resolve, reject) => {
|
|
const timeout = window.setTimeout(() => {
|
|
if (recorder.state !== "inactive") recorder.stop();
|
|
}, Math.ceil((endSeconds - startSeconds + 2) * 1_000));
|
|
recorder.addEventListener("error", () => {
|
|
window.clearTimeout(timeout);
|
|
reject(new Error("视频剪辑失败"));
|
|
}, { once: true });
|
|
recorder.addEventListener("stop", () => {
|
|
window.clearTimeout(timeout);
|
|
resolve(new Blob(chunks, { type: recorder.mimeType || "video/webm" }));
|
|
}, { once: true });
|
|
});
|
|
|
|
const stopAtSelectionEnd = () => {
|
|
if (video.currentTime >= endSeconds - 0.02 && recorder.state !== "inactive") {
|
|
recorder.stop();
|
|
}
|
|
};
|
|
video.addEventListener("timeupdate", stopAtSelectionEnd);
|
|
recorder.start(100);
|
|
await video.play();
|
|
const blob = await completed;
|
|
video.pause();
|
|
video.removeEventListener("timeupdate", stopAtSelectionEnd);
|
|
stream.getTracks().forEach((track) => track.stop());
|
|
if (blob.size === 0) throw new Error("视频剪辑结果为空");
|
|
return { blob, durationSeconds: Math.max(0.05, endSeconds - startSeconds) };
|
|
} finally {
|
|
video.pause();
|
|
video.removeAttribute("src");
|
|
video.load();
|
|
URL.revokeObjectURL(sourceObjectUrl);
|
|
}
|
|
}
|
|
|
|
export async function processCanvasVideoCrop(
|
|
mediaUrl: string,
|
|
cropRect: VideoCropRect,
|
|
durationSeconds: number,
|
|
outputWidth: number,
|
|
outputHeight: number,
|
|
) {
|
|
const response = await fetch(getCanvasVideoSourceUrl(mediaUrl), { credentials: "include" });
|
|
if (!response.ok) throw new Error("视频源加载失败");
|
|
const sourceObjectUrl = URL.createObjectURL(await response.blob());
|
|
const video = document.createElement("video") as CapturableVideoElement;
|
|
video.muted = true;
|
|
video.playsInline = true;
|
|
video.preload = "auto";
|
|
video.src = sourceObjectUrl;
|
|
|
|
try {
|
|
if (video.readyState < HTMLMediaElement.HAVE_METADATA) {
|
|
await waitForMediaEvent(video, "loadedmetadata");
|
|
}
|
|
const captureStream = video.captureStream || video.mozCaptureStream;
|
|
if (!captureStream || typeof MediaRecorder === "undefined") {
|
|
throw new Error("当前浏览器不支持视频裁剪");
|
|
}
|
|
|
|
const normalizedRect = normalizeVideoCropRect(cropRect);
|
|
const canvas = document.createElement("canvas");
|
|
canvas.width = Math.max(2, Math.round(outputWidth));
|
|
canvas.height = Math.max(2, Math.round(outputHeight));
|
|
const context = canvas.getContext("2d");
|
|
if (!context) throw new Error("视频裁剪画布创建失败");
|
|
const canvasStream = canvas.captureStream(30);
|
|
const sourceStream = captureStream.call(video);
|
|
const outputStream = new MediaStream([
|
|
...canvasStream.getVideoTracks(),
|
|
...sourceStream.getAudioTracks(),
|
|
]);
|
|
const mimeType = getRecorderMimeType();
|
|
const recorder = new MediaRecorder(outputStream, mimeType ? { mimeType } : undefined);
|
|
const chunks: BlobPart[] = [];
|
|
recorder.addEventListener("dataavailable", (event) => {
|
|
if (event.data.size > 0) chunks.push(event.data);
|
|
});
|
|
|
|
let animationFrame = 0;
|
|
const drawFrame = () => {
|
|
context.drawImage(
|
|
video,
|
|
video.videoWidth * normalizedRect.x,
|
|
video.videoHeight * normalizedRect.y,
|
|
video.videoWidth * normalizedRect.width,
|
|
video.videoHeight * normalizedRect.height,
|
|
0,
|
|
0,
|
|
canvas.width,
|
|
canvas.height,
|
|
);
|
|
animationFrame = window.requestAnimationFrame(drawFrame);
|
|
};
|
|
const stopAt = Math.min(video.duration, Math.max(0.1, durationSeconds));
|
|
const completed = new Promise<Blob>((resolve, reject) => {
|
|
const timeout = window.setTimeout(() => {
|
|
if (recorder.state !== "inactive") recorder.stop();
|
|
}, Math.ceil((stopAt + 2) * 1_000));
|
|
recorder.addEventListener("error", () => {
|
|
window.clearTimeout(timeout);
|
|
reject(new Error("视频裁剪失败"));
|
|
}, { once: true });
|
|
recorder.addEventListener("stop", () => {
|
|
window.clearTimeout(timeout);
|
|
resolve(new Blob(chunks, { type: recorder.mimeType || "video/webm" }));
|
|
}, { once: true });
|
|
});
|
|
const stopAtDuration = () => {
|
|
if (video.currentTime >= stopAt - 0.02 && recorder.state !== "inactive") recorder.stop();
|
|
};
|
|
video.addEventListener("timeupdate", stopAtDuration);
|
|
recorder.start(100);
|
|
drawFrame();
|
|
await video.play();
|
|
const blob = await completed;
|
|
video.pause();
|
|
window.cancelAnimationFrame(animationFrame);
|
|
video.removeEventListener("timeupdate", stopAtDuration);
|
|
outputStream.getTracks().forEach((track) => track.stop());
|
|
sourceStream.getTracks().forEach((track) => track.stop());
|
|
if (blob.size === 0) throw new Error("视频裁剪结果为空");
|
|
return { blob, durationSeconds: stopAt };
|
|
} finally {
|
|
video.pause();
|
|
video.removeAttribute("src");
|
|
video.load();
|
|
URL.revokeObjectURL(sourceObjectUrl);
|
|
}
|
|
}
|
|
|
|
export async function processCanvasVideoSubtitleErase(
|
|
mediaUrl: string,
|
|
mode: SubtitleEraseMode,
|
|
requestedRegions: SubtitleEraseRegion[],
|
|
durationSeconds: number,
|
|
configValue: VideoCanvasSubtitleEraseConfig,
|
|
) {
|
|
const config = normalizeVideoCanvasSubtitleEraseConfig(configValue);
|
|
if (!config.enabled) throw new Error("智能去字幕功能已在后台关闭");
|
|
const response = await fetch(getCanvasVideoSourceUrl(mediaUrl), { credentials: "include" });
|
|
if (!response.ok) throw new Error("视频源加载失败");
|
|
const sourceObjectUrl = URL.createObjectURL(await response.blob());
|
|
const video = document.createElement("video") as CapturableVideoElement;
|
|
video.muted = true;
|
|
video.playsInline = true;
|
|
video.preload = "auto";
|
|
video.src = sourceObjectUrl;
|
|
|
|
try {
|
|
if (video.readyState < HTMLMediaElement.HAVE_METADATA) {
|
|
await waitForMediaEvent(video, "loadedmetadata");
|
|
}
|
|
const captureStream = video.captureStream || video.mozCaptureStream;
|
|
if (!captureStream || typeof MediaRecorder === "undefined") {
|
|
throw new Error("当前浏览器不支持视频去字幕");
|
|
}
|
|
const width = Math.max(2, Math.round(video.videoWidth));
|
|
const height = Math.max(2, Math.round(video.videoHeight));
|
|
let regions = requestedRegions.map(normalizeSubtitleEraseRegion);
|
|
|
|
if (mode === "smart") {
|
|
const sampleWidth = Math.min(width, 960);
|
|
const sampleHeight = Math.max(2, Math.round(sampleWidth * height / width));
|
|
const sampleCanvas = document.createElement("canvas");
|
|
sampleCanvas.width = sampleWidth;
|
|
sampleCanvas.height = sampleHeight;
|
|
const sampleContext = sampleCanvas.getContext("2d", { willReadFrequently: true });
|
|
if (!sampleContext) throw new Error("字幕检测画布创建失败");
|
|
const sampleAt = Math.min(
|
|
Math.max(0, video.duration - 0.05),
|
|
Math.max(0, Math.min(video.duration * 0.6, durationSeconds * 0.6)),
|
|
);
|
|
if (sampleAt > 0) {
|
|
video.currentTime = sampleAt;
|
|
await waitForMediaEvent(video, "seeked");
|
|
}
|
|
sampleContext.drawImage(video, 0, 0, sampleWidth, sampleHeight);
|
|
regions = detectSubtitleEraseRegions(
|
|
sampleContext.getImageData(0, 0, sampleWidth, sampleHeight),
|
|
config,
|
|
);
|
|
video.currentTime = 0;
|
|
if (sampleAt > 0) await waitForMediaEvent(video, "seeked");
|
|
}
|
|
|
|
const canvas = document.createElement("canvas");
|
|
canvas.width = width;
|
|
canvas.height = height;
|
|
const context = canvas.getContext("2d");
|
|
if (!context) throw new Error("视频去字幕画布创建失败");
|
|
const canvasStream = canvas.captureStream(config.frameRate);
|
|
const sourceStream = captureStream.call(video);
|
|
const outputStream = new MediaStream([
|
|
...canvasStream.getVideoTracks(),
|
|
...sourceStream.getAudioTracks(),
|
|
]);
|
|
const mimeType = getRecorderMimeType();
|
|
const recorder = new MediaRecorder(outputStream, mimeType ? { mimeType } : undefined);
|
|
const chunks: BlobPart[] = [];
|
|
recorder.addEventListener("dataavailable", (event) => {
|
|
if (event.data.size > 0) chunks.push(event.data);
|
|
});
|
|
|
|
let animationFrame = 0;
|
|
const drawFrame = () => {
|
|
context.filter = "none";
|
|
context.drawImage(video, 0, 0, width, height);
|
|
for (const rawRegion of regions) {
|
|
const region = normalizeSubtitleEraseRegion(rawRegion);
|
|
const paddingX = Math.round(width * config.paddingRatio);
|
|
const paddingY = Math.round(height * config.paddingRatio);
|
|
const destinationX = Math.max(0, Math.floor(region.x * width) - paddingX);
|
|
const destinationY = Math.max(0, Math.floor(region.y * height) - paddingY);
|
|
const destinationWidth = Math.min(
|
|
width - destinationX,
|
|
Math.ceil(region.width * width) + paddingX * 2,
|
|
);
|
|
const destinationHeight = Math.min(
|
|
height - destinationY,
|
|
Math.ceil(region.height * height) + paddingY * 2,
|
|
);
|
|
if (destinationWidth < 1 || destinationHeight < 1) continue;
|
|
const sourceY = destinationY >= destinationHeight
|
|
? destinationY - destinationHeight
|
|
: Math.min(height - destinationHeight, destinationY + destinationHeight);
|
|
context.save();
|
|
context.beginPath();
|
|
context.rect(destinationX, destinationY, destinationWidth, destinationHeight);
|
|
context.clip();
|
|
context.filter = config.blurRadius > 0 ? `blur(${config.blurRadius}px)` : "none";
|
|
context.drawImage(
|
|
video,
|
|
destinationX,
|
|
sourceY,
|
|
destinationWidth,
|
|
destinationHeight,
|
|
destinationX,
|
|
destinationY,
|
|
destinationWidth,
|
|
destinationHeight,
|
|
);
|
|
context.restore();
|
|
}
|
|
animationFrame = window.requestAnimationFrame(drawFrame);
|
|
};
|
|
const stopAt = Math.min(video.duration, Math.max(0.1, durationSeconds));
|
|
const completed = new Promise<Blob>((resolve, reject) => {
|
|
const timeout = window.setTimeout(() => {
|
|
if (recorder.state !== "inactive") recorder.stop();
|
|
}, Math.ceil((stopAt + 3) * 1_000));
|
|
recorder.addEventListener("error", () => {
|
|
window.clearTimeout(timeout);
|
|
reject(new Error("视频去字幕失败"));
|
|
}, { once: true });
|
|
recorder.addEventListener("stop", () => {
|
|
window.clearTimeout(timeout);
|
|
resolve(new Blob(chunks, { type: recorder.mimeType || "video/webm" }));
|
|
}, { once: true });
|
|
});
|
|
const stopAtDuration = () => {
|
|
if (video.currentTime >= stopAt - 0.02 && recorder.state !== "inactive") recorder.stop();
|
|
};
|
|
video.addEventListener("timeupdate", stopAtDuration);
|
|
recorder.start(100);
|
|
drawFrame();
|
|
await video.play();
|
|
const blob = await completed;
|
|
video.pause();
|
|
window.cancelAnimationFrame(animationFrame);
|
|
video.removeEventListener("timeupdate", stopAtDuration);
|
|
outputStream.getTracks().forEach((track) => track.stop());
|
|
sourceStream.getTracks().forEach((track) => track.stop());
|
|
if (blob.size === 0) throw new Error("视频去字幕结果为空");
|
|
return { blob, durationSeconds: stopAt, height, regions, width };
|
|
} finally {
|
|
video.pause();
|
|
video.removeAttribute("src");
|
|
video.load();
|
|
URL.revokeObjectURL(sourceObjectUrl);
|
|
}
|
|
}
|
|
|
|
export async function processCanvasVideoAudioSplit(
|
|
mediaUrl: string,
|
|
mode: VideoAudioSplitMode,
|
|
durationSeconds: number,
|
|
configValue: VideoCanvasAudioSeparationConfig,
|
|
) {
|
|
const config = normalizeVideoCanvasAudioSeparationConfig(configValue);
|
|
if (!config.enabled) throw new Error("视频音频分离功能已在后台关闭");
|
|
const sourceBlob = await fetchCanvasVideoBlob(mediaUrl);
|
|
const sourceObjectUrl = URL.createObjectURL(sourceBlob);
|
|
const video = document.createElement("video") as CapturableVideoElement;
|
|
video.muted = false;
|
|
video.playsInline = true;
|
|
video.preload = "auto";
|
|
video.src = sourceObjectUrl;
|
|
try {
|
|
if (video.readyState < HTMLMediaElement.HAVE_METADATA) {
|
|
await waitForMediaEvent(video, "loadedmetadata");
|
|
}
|
|
const AudioContextCtor = window.AudioContext ||
|
|
(window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext;
|
|
if (typeof MediaRecorder === "undefined" || !AudioContextCtor) {
|
|
throw new Error("当前浏览器不支持视频音频分离");
|
|
}
|
|
const width = Math.max(2, Math.round(video.videoWidth));
|
|
const height = Math.max(2, Math.round(video.videoHeight));
|
|
const canvas = document.createElement("canvas");
|
|
canvas.width = width;
|
|
canvas.height = height;
|
|
const context = canvas.getContext("2d");
|
|
if (!context) throw new Error("无声视频画布创建失败");
|
|
const outputStream = canvas.captureStream(config.frameRate);
|
|
const audioContext = new AudioContextCtor();
|
|
const sampleRate = audioContext.sampleRate;
|
|
const audioSource = audioContext.createMediaElementSource(video);
|
|
const audioDestination = audioContext.createMediaStreamDestination();
|
|
if (mode === "audio-video") {
|
|
audioSource.connect(audioDestination);
|
|
} else {
|
|
const splitter = audioContext.createChannelSplitter(2);
|
|
const merger = audioContext.createChannelMerger(2);
|
|
const signalBus = audioContext.createGain();
|
|
const leftGain = audioContext.createGain();
|
|
const rightGain = audioContext.createGain();
|
|
audioSource.connect(splitter);
|
|
if (mode === "voice") {
|
|
leftGain.gain.value = config.voiceGain / 2;
|
|
rightGain.gain.value = config.voiceGain / 2;
|
|
splitter.connect(leftGain, 0);
|
|
splitter.connect(rightGain, 1);
|
|
leftGain.connect(signalBus);
|
|
rightGain.connect(signalBus);
|
|
signalBus.connect(merger, 0, 0);
|
|
signalBus.connect(merger, 0, 1);
|
|
} else {
|
|
leftGain.gain.value = config.backgroundGain / 2;
|
|
rightGain.gain.value = -config.backgroundGain / 2;
|
|
splitter.connect(leftGain, 0);
|
|
splitter.connect(rightGain, 1);
|
|
leftGain.connect(signalBus);
|
|
rightGain.connect(signalBus);
|
|
const invertedSide = audioContext.createGain();
|
|
invertedSide.gain.value = -1;
|
|
signalBus.connect(merger, 0, 0);
|
|
signalBus.connect(invertedSide);
|
|
invertedSide.connect(merger, 0, 1);
|
|
}
|
|
merger.connect(audioDestination);
|
|
}
|
|
const mimeType = getRecorderMimeType();
|
|
const recorder = new MediaRecorder(outputStream, mimeType ? { mimeType } : undefined);
|
|
const audioMimeType = getAudioRecorderMimeType();
|
|
const audioRecorder = new MediaRecorder(
|
|
audioDestination.stream,
|
|
audioMimeType ? { mimeType: audioMimeType } : undefined,
|
|
);
|
|
const chunks: BlobPart[] = [];
|
|
const audioChunks: BlobPart[] = [];
|
|
recorder.addEventListener("dataavailable", (event) => {
|
|
if (event.data.size > 0) chunks.push(event.data);
|
|
});
|
|
audioRecorder.addEventListener("dataavailable", (event) => {
|
|
if (event.data.size > 0) audioChunks.push(event.data);
|
|
});
|
|
|
|
let animationFrame = 0;
|
|
const drawFrame = () => {
|
|
context.drawImage(video, 0, 0, width, height);
|
|
animationFrame = window.requestAnimationFrame(drawFrame);
|
|
};
|
|
const stopAt = Math.min(video.duration, Math.max(0.1, durationSeconds));
|
|
const completed = new Promise<{ audioBlob: Blob; silentVideoBlob: Blob }>((resolve, reject) => {
|
|
const timeout = window.setTimeout(() => {
|
|
if (recorder.state !== "inactive") recorder.stop();
|
|
if (audioRecorder.state !== "inactive") audioRecorder.stop();
|
|
}, Math.ceil((stopAt + 3) * 1_000));
|
|
let audioStopped = false;
|
|
let videoStopped = false;
|
|
const finish = () => {
|
|
if (!audioStopped || !videoStopped) return;
|
|
window.clearTimeout(timeout);
|
|
resolve({
|
|
audioBlob: new Blob(audioChunks, { type: audioRecorder.mimeType || "audio/webm" }),
|
|
silentVideoBlob: new Blob(chunks, { type: recorder.mimeType || "video/webm" }),
|
|
});
|
|
};
|
|
const handleError = () => {
|
|
window.clearTimeout(timeout);
|
|
reject(new Error("视频音频分离失败"));
|
|
};
|
|
recorder.addEventListener("error", handleError, { once: true });
|
|
audioRecorder.addEventListener("error", handleError, { once: true });
|
|
recorder.addEventListener("stop", () => {
|
|
videoStopped = true;
|
|
finish();
|
|
}, { once: true });
|
|
audioRecorder.addEventListener("stop", () => {
|
|
audioStopped = true;
|
|
finish();
|
|
}, { once: true });
|
|
});
|
|
const stopAtDuration = () => {
|
|
if (video.currentTime < stopAt - 0.02) return;
|
|
if (recorder.state !== "inactive") recorder.stop();
|
|
if (audioRecorder.state !== "inactive") audioRecorder.stop();
|
|
};
|
|
video.addEventListener("timeupdate", stopAtDuration);
|
|
recorder.start(100);
|
|
audioRecorder.start(100);
|
|
drawFrame();
|
|
await audioContext.resume();
|
|
await video.play();
|
|
const { audioBlob, silentVideoBlob } = await completed;
|
|
video.pause();
|
|
window.cancelAnimationFrame(animationFrame);
|
|
video.removeEventListener("timeupdate", stopAtDuration);
|
|
outputStream.getTracks().forEach((track) => track.stop());
|
|
audioDestination.stream.getTracks().forEach((track) => track.stop());
|
|
await audioContext.close().catch(() => undefined);
|
|
if (audioBlob.size === 0) throw new Error("分离音频结果为空");
|
|
if (silentVideoBlob.size === 0) throw new Error("无声视频结果为空");
|
|
return {
|
|
audioBlob,
|
|
durationSeconds: stopAt,
|
|
height,
|
|
sampleRate,
|
|
silentVideoBlob,
|
|
width,
|
|
};
|
|
} finally {
|
|
video.pause();
|
|
video.removeAttribute("src");
|
|
video.load();
|
|
URL.revokeObjectURL(sourceObjectUrl);
|
|
}
|
|
}
|
|
|
|
export async function processCanvasVideoUpscale(
|
|
mediaUrl: string,
|
|
durationSeconds: number,
|
|
outputWidth: number,
|
|
outputHeight: number,
|
|
frameRate: number,
|
|
slowMultiplier: number,
|
|
) {
|
|
const response = await fetch(getCanvasVideoSourceUrl(mediaUrl), { credentials: "include" });
|
|
if (!response.ok) throw new Error("视频源加载失败");
|
|
const sourceObjectUrl = URL.createObjectURL(await response.blob());
|
|
const video = document.createElement("video") as CapturableVideoElement;
|
|
video.muted = true;
|
|
video.playsInline = true;
|
|
video.preload = "auto";
|
|
video.src = sourceObjectUrl;
|
|
|
|
try {
|
|
if (video.readyState < HTMLMediaElement.HAVE_METADATA) {
|
|
await waitForMediaEvent(video, "loadedmetadata");
|
|
}
|
|
const captureStream = video.captureStream || video.mozCaptureStream;
|
|
if (!captureStream || typeof MediaRecorder === "undefined") {
|
|
throw new Error("当前浏览器不支持视频高清处理");
|
|
}
|
|
const canvas = document.createElement("canvas");
|
|
canvas.width = Math.max(2, Math.round(outputWidth));
|
|
canvas.height = Math.max(2, Math.round(outputHeight));
|
|
const context = canvas.getContext("2d");
|
|
if (!context) throw new Error("视频高清画布创建失败");
|
|
context.imageSmoothingEnabled = true;
|
|
context.imageSmoothingQuality = "high";
|
|
|
|
const normalizedFrameRate = Math.max(1, Math.min(120, Math.round(frameRate)));
|
|
const normalizedSlowMultiplier = Math.max(1, Math.min(5, Number(slowMultiplier) || 1));
|
|
const canvasStream = canvas.captureStream(normalizedFrameRate);
|
|
const sourceStream = captureStream.call(video);
|
|
const outputStream = new MediaStream([
|
|
...canvasStream.getVideoTracks(),
|
|
...sourceStream.getAudioTracks(),
|
|
]);
|
|
const mimeType = getRecorderMimeType();
|
|
const recorder = new MediaRecorder(outputStream, mimeType ? { mimeType } : undefined);
|
|
const chunks: BlobPart[] = [];
|
|
recorder.addEventListener("dataavailable", (event) => {
|
|
if (event.data.size > 0) chunks.push(event.data);
|
|
});
|
|
|
|
let animationFrame = 0;
|
|
const drawFrame = () => {
|
|
context.drawImage(video, 0, 0, canvas.width, canvas.height);
|
|
animationFrame = window.requestAnimationFrame(drawFrame);
|
|
};
|
|
const sourceStopAt = Math.min(video.duration, Math.max(0.1, durationSeconds));
|
|
const outputDuration = sourceStopAt * normalizedSlowMultiplier;
|
|
const completed = new Promise<Blob>((resolve, reject) => {
|
|
const timeout = window.setTimeout(() => {
|
|
if (recorder.state !== "inactive") recorder.stop();
|
|
}, Math.ceil((outputDuration + 2) * 1_000));
|
|
recorder.addEventListener("error", () => {
|
|
window.clearTimeout(timeout);
|
|
reject(new Error("视频高清处理失败"));
|
|
}, { once: true });
|
|
recorder.addEventListener("stop", () => {
|
|
window.clearTimeout(timeout);
|
|
resolve(new Blob(chunks, { type: recorder.mimeType || "video/webm" }));
|
|
}, { once: true });
|
|
});
|
|
const stopAtDuration = () => {
|
|
if (video.currentTime >= sourceStopAt - 0.02 && recorder.state !== "inactive") recorder.stop();
|
|
};
|
|
video.addEventListener("timeupdate", stopAtDuration);
|
|
video.playbackRate = 1 / normalizedSlowMultiplier;
|
|
recorder.start(100);
|
|
drawFrame();
|
|
await video.play();
|
|
const blob = await completed;
|
|
video.pause();
|
|
window.cancelAnimationFrame(animationFrame);
|
|
video.removeEventListener("timeupdate", stopAtDuration);
|
|
outputStream.getTracks().forEach((track) => track.stop());
|
|
sourceStream.getTracks().forEach((track) => track.stop());
|
|
if (blob.size === 0) throw new Error("视频高清结果为空");
|
|
return { blob, durationSeconds: outputDuration };
|
|
} finally {
|
|
video.pause();
|
|
video.removeAttribute("src");
|
|
video.load();
|
|
URL.revokeObjectURL(sourceObjectUrl);
|
|
}
|
|
}
|