478 lines
19 KiB
TypeScript
478 lines
19 KiB
TypeScript
import { z } from "zod";
|
||
import { captionFontIds, captionFont } from "./fonts";
|
||
export const SECOND = 1_000_000;
|
||
export const MAX_EDIT_SECONDS = 300;
|
||
export const editRatios = ["21:9", "16:9", "9:16", "1:1", "4:3", "3:4"] as const;
|
||
export const editResolutions = ["480p", "720p", "1080p"] as const;
|
||
const time = z
|
||
.number()
|
||
.int()
|
||
.min(0)
|
||
.max(600 * SECOND);
|
||
const id = z.string().min(1).max(180);
|
||
const url = z
|
||
.string()
|
||
.url()
|
||
.max(4096)
|
||
.refine((value) => /^https?:\/\//i.test(value));
|
||
export const subtitleSchema = z
|
||
.object({ start: time, end: time, text: z.string().max(500) })
|
||
.refine((x) => x.end > x.start);
|
||
export const captionStyleSchema = z.object({
|
||
fontFamily: z.enum(captionFontIds).optional(),
|
||
referenceHeight: z.union([z.literal(720), z.literal(1080)]).optional(),
|
||
fontSize: z.number().min(8).max(160).default(28),
|
||
color: z
|
||
.string()
|
||
.regex(/^#[0-9a-f]{6}$/i)
|
||
.default("#ffffff"),
|
||
outlineColor: z
|
||
.string()
|
||
.regex(/^#[0-9a-f]{6}$/i)
|
||
.default("#000000"),
|
||
outlineWidth: z.number().min(0).max(20).default(0),
|
||
opacity: z.number().min(0).max(1).default(1),
|
||
// Optional so existing edits keep their whole-caption opacity unchanged.
|
||
outlineOpacity: z.number().min(0).max(1).optional(),
|
||
spacing: z.number().min(-5).max(100).default(0),
|
||
x: z.number().min(0).max(1).optional(),
|
||
y: z.number().min(0).max(1).optional(),
|
||
});
|
||
export const timelineCaptionSchema = z
|
||
.object({
|
||
id,
|
||
start: time,
|
||
end: time,
|
||
text: z.string().max(500),
|
||
style: captionStyleSchema.optional(),
|
||
})
|
||
.refine((x) => x.end > x.start);
|
||
export type TimelineCaption = z.infer<typeof timelineCaptionSchema>;
|
||
export type CaptionStyle = z.infer<typeof captionStyleSchema>;
|
||
export const defaultCaptionStyle: CaptionStyle = captionStyleSchema.parse({});
|
||
export const newCaptionStyle: CaptionStyle = {
|
||
...defaultCaptionStyle,
|
||
fontFamily: "source-han",
|
||
referenceHeight: 1080,
|
||
fontSize: 56,
|
||
outlineColor: "#ffffff",
|
||
};
|
||
const transitionSchema = z.object({
|
||
duration: z.number().min(0).max(4),
|
||
// Centered transitions preserve the edit duration; absent keeps saved legacy overlaps.
|
||
alignment: z.enum(["overlap", "center"]).optional(),
|
||
type: z.enum(["fade", "fadeblack", "fadewhite"]),
|
||
});
|
||
export const assetInputSchema = z.object({
|
||
id,
|
||
url,
|
||
name: z.string().max(200),
|
||
subtitles: z.array(subtitleSchema).max(500).default([]),
|
||
});
|
||
export const editAssetSchema = assetInputSchema.extend({
|
||
duration: time,
|
||
width: z.number().int().positive().max(8192),
|
||
height: z.number().int().positive().max(8192),
|
||
hasAudio: z.boolean(),
|
||
hash: z.string().regex(/^[a-f0-9]{64}$/),
|
||
});
|
||
export const candidateSchema = z
|
||
.object({ id, assetId: id, start: time, end: time, text: z.string().max(1500).default("") })
|
||
.refine((x) => x.end - x.start >= 200000);
|
||
export const videoTransformSchema = z.object({
|
||
x: z.number().min(-1).max(1).default(0),
|
||
y: z.number().min(-1).max(1).default(0),
|
||
scale: z.number().min(0.1).max(3).default(1),
|
||
});
|
||
export type VideoTransform = z.infer<typeof videoTransformSchema>;
|
||
export const defaultVideoTransform: VideoTransform = { x: 0, y: 0, scale: 1 };
|
||
export const editClipSchema = z
|
||
.object({
|
||
id,
|
||
assetId: id,
|
||
sourceIn: time,
|
||
sourceOut: time,
|
||
speed: z.number().min(0.5).max(2).default(1),
|
||
gain: z.number().min(0).max(4).default(1),
|
||
muted: z.boolean().optional(),
|
||
label: z.string().max(300).default(""),
|
||
// The transition belongs to the incoming clip, so it follows that clip when reordered.
|
||
transition: transitionSchema.optional(),
|
||
transform: videoTransformSchema.optional(),
|
||
audioDetached: z.boolean().optional(),
|
||
})
|
||
.refine((x) => x.sourceOut - x.sourceIn >= 200000);
|
||
export const audioClipSchema = z
|
||
.object({
|
||
id,
|
||
assetId: id,
|
||
start: time,
|
||
sourceIn: time,
|
||
sourceOut: time,
|
||
speed: z.number().min(0.5).max(2).default(1),
|
||
gain: z.number().min(0).max(4).default(1),
|
||
muted: z.boolean().optional(),
|
||
fadeIn: z.number().min(0).max(5).default(0),
|
||
fadeOut: z.number().min(0).max(5).default(0),
|
||
fadeInStart: z.number().min(0).max(1).optional(),
|
||
fadeOutEnd: z.number().min(0).max(1).optional(),
|
||
})
|
||
.refine((c) => c.sourceOut - c.sourceIn >= 200000);
|
||
export type AudioClip = z.infer<typeof audioClipSchema>;
|
||
export const editPlanSchema = z.object({
|
||
version: z.literal(1),
|
||
title: z.string().max(200),
|
||
assets: z.array(editAssetSchema).min(1).max(12),
|
||
clips: z.array(editClipSchema).min(1).max(12),
|
||
ratio: z.enum(editRatios),
|
||
resolution: z.enum(editResolutions).default("720p"),
|
||
transition: z.number().min(0).max(0.5).default(0),
|
||
subtitles: z.boolean().default(true),
|
||
captions: z.array(timelineCaptionSchema).max(500).optional(),
|
||
audioClips: z.array(audioClipSchema).max(24).optional(),
|
||
music: z
|
||
.object({ url, gain: z.number().min(0).max(1).default(0.2), start: time.default(0) })
|
||
.optional(),
|
||
warnings: z.array(z.string().max(500)).max(30).default([]),
|
||
});
|
||
export type EditPlan = z.infer<typeof editPlanSchema>;
|
||
export type EditAsset = z.infer<typeof editAssetSchema>;
|
||
export type EditCandidate = z.infer<typeof candidateSchema>;
|
||
export const smartEditRequestSchema = z.discriminatedUnion("operation", [
|
||
z.object({
|
||
operation: z.literal("plan"),
|
||
assets: z.array(assetInputSchema).min(1).max(12),
|
||
mode: z.enum(["sequence", "auto", "montage", "talk", "explain", "ads"]).default("montage"),
|
||
prompt: z.string().max(3000).default(""),
|
||
targetDuration: z.number().int().min(0).max(MAX_EDIT_SECONDS).default(20),
|
||
ratio: z.enum(editRatios).default("16:9"),
|
||
resolution: z.enum(editResolutions).default("720p"),
|
||
removeSilence: z.boolean().default(false),
|
||
transcribe: z.boolean().default(false),
|
||
semantic: z.boolean().default(true),
|
||
preferredModelRef: z.string().max(180).optional(),
|
||
music: z
|
||
.object({ url, gain: z.number().min(0).max(1).default(0.2), start: time.default(0) })
|
||
.optional(),
|
||
beatSync: z.boolean().default(false),
|
||
beatsPerCut: z.enum(["2", "4"]).default("4"),
|
||
variants: z.number().int().min(1).max(3).default(1),
|
||
}),
|
||
z.object({
|
||
operation: z.literal("render"),
|
||
plan: editPlanSchema,
|
||
sourceNodeId: z.string().max(180).optional(),
|
||
}),
|
||
]);
|
||
export type SmartEditRequest = z.infer<typeof smartEditRequestSchema>;
|
||
export function editOutputSize(plan: Pick<EditPlan, "ratio" | "resolution">) {
|
||
const size = Number.parseInt(plan.resolution, 10),
|
||
[w, h] = plan.ratio.split(":").map(Number);
|
||
const even = (v: number) => Math.round(v / 2) * 2;
|
||
return w >= h
|
||
? { width: even((size * w) / h), height: size }
|
||
: { width: size, height: even((size * h) / w) };
|
||
}
|
||
export function planTimeline(input: EditPlan) {
|
||
const plan = editPlanSchema.parse(input),
|
||
ids = new Set<string>();
|
||
const durations = plan.clips.map((c) =>
|
||
Math.round(
|
||
(Math.round(((c.sourceOut - c.sourceIn) / c.speed / SECOND) * 30) * SECOND) / 30,
|
||
),
|
||
);
|
||
const centered = plan.clips.map((c) => c.transition?.alignment === "center");
|
||
const overlaps = plan.clips.map((c, i) =>
|
||
i
|
||
? Math.min(
|
||
Math.round((c.transition?.duration ?? plan.transition) * SECOND),
|
||
centered[i] ? durations[i] * 2 : Math.floor(durations[i] / 3),
|
||
centered[i] ? durations[i - 1] * 2 : Math.floor(durations[i - 1] / 3),
|
||
)
|
||
: 0,
|
||
);
|
||
// Neighboring transitions cannot consume the same frames in a short middle clip.
|
||
for (let i = 0; i < durations.length; i++) {
|
||
const incoming = overlaps[i] / (centered[i] ? 2 : 1),
|
||
outgoing = (overlaps[i + 1] || 0) / (centered[i + 1] ? 2 : 1);
|
||
if (incoming + outgoing > durations[i]) {
|
||
const factor = durations[i] / (incoming + outgoing);
|
||
overlaps[i] = Math.floor(overlaps[i] * factor);
|
||
if (i + 1 < overlaps.length) overlaps[i + 1] = Math.floor(overlaps[i + 1] * factor);
|
||
}
|
||
}
|
||
const before = overlaps.map((d, i) =>
|
||
centered[i] ? Math.round((Math.floor(((d / SECOND) * 30) / 2) * SECOND) / 30) : 0,
|
||
);
|
||
const after = overlaps.map((d, i) => (centered[i] ? d - before[i] : 0));
|
||
let cursor = 0;
|
||
const clips = plan.clips.map((clip, index) => {
|
||
if (ids.has(clip.id)) throw Error("片段 ID 重复");
|
||
ids.add(clip.id);
|
||
const asset = plan.assets.find((a) => a.id === clip.assetId);
|
||
if (!asset || clip.sourceOut > asset.duration) throw Error("片段超出原始素材范围");
|
||
const duration = durations[index],
|
||
overlap = overlaps[index];
|
||
if (!centered[index]) cursor -= overlap;
|
||
const start = cursor;
|
||
cursor += duration;
|
||
const padBefore = before[index],
|
||
padAfter = after[index + 1] || 0;
|
||
return {
|
||
...clip,
|
||
start,
|
||
end: cursor,
|
||
duration,
|
||
overlap,
|
||
padBefore,
|
||
padAfter,
|
||
renderStart: start - padBefore,
|
||
renderEnd: cursor + padAfter,
|
||
transitionStart: start - padBefore,
|
||
transitionType: clip.transition?.type || "fade",
|
||
};
|
||
});
|
||
if (new Set(plan.assets.map((a) => a.id)).size !== plan.assets.length)
|
||
throw Error("素材 ID 重复");
|
||
if (plan.captions && new Set(plan.captions.map((s) => s.id)).size !== plan.captions.length)
|
||
throw Error("字幕 ID 重复");
|
||
if (cursor > MAX_EDIT_SECONDS * SECOND || cursor < 200000)
|
||
throw Error("成片时长需在 0.2–300 秒内");
|
||
const audioClips = (plan.audioClips || []).map((c) => {
|
||
const asset = plan.assets.find((a) => a.id === c.assetId);
|
||
if (!asset?.hasAudio || c.sourceOut > asset.duration)
|
||
throw Error("独立音轨素材无效或超出范围");
|
||
const duration = Math.round((c.sourceOut - c.sourceIn) / c.speed);
|
||
return { ...c, duration, end: c.start + duration };
|
||
});
|
||
if (new Set(audioClips.map((c) => c.id)).size !== audioClips.length)
|
||
throw Error("独立音轨 ID 重复");
|
||
cursor = Math.max(cursor, ...audioClips.map((c) => c.end));
|
||
if (cursor > MAX_EDIT_SECONDS * SECOND) throw Error("成片时长需在 0.2–300 秒内");
|
||
return { clips, audioClips, duration: cursor };
|
||
}
|
||
export function compiledSubtitles(plan: EditPlan) {
|
||
if (!plan.subtitles) return [];
|
||
if (plan.captions) {
|
||
const duration = planTimeline(plan).duration;
|
||
return plan.captions.flatMap((s) => {
|
||
const end = Math.min(s.end, duration);
|
||
return end > s.start ? [{ ...s, end }] : [];
|
||
});
|
||
}
|
||
return planTimeline(plan).clips.flatMap((clip) =>
|
||
plan.assets
|
||
.find((a) => a.id === clip.assetId)!
|
||
.subtitles.flatMap((s) => {
|
||
const a = Math.max(s.start, clip.sourceIn),
|
||
b = Math.min(s.end, clip.sourceOut);
|
||
if (b <= a) return [];
|
||
// Only legacy overlapping edits delay incoming captions until the outgoing clip ends.
|
||
const start = Math.max(
|
||
clip.start + (clip.transition?.alignment === "center" ? 0 : clip.overlap),
|
||
clip.start + Math.round((a - clip.sourceIn) / clip.speed),
|
||
);
|
||
const end = Math.min(
|
||
clip.end,
|
||
clip.start + Math.round((b - clip.sourceIn) / clip.speed),
|
||
);
|
||
return end > start ? [{ start, end, text: s.text }] : [];
|
||
}),
|
||
);
|
||
}
|
||
export function editableCaptions(plan: EditPlan): TimelineCaption[] {
|
||
if (plan.captions) return plan.captions;
|
||
return compiledSubtitles({ ...plan, subtitles: true }).map((s, i) => ({
|
||
...s,
|
||
id: `caption-${i}`,
|
||
}));
|
||
}
|
||
export function subtitlesToAss(plan: EditPlan, width: number, height: number) {
|
||
const stamp = (v: number) => {
|
||
const cs = Math.round(v / 10000);
|
||
return `${Math.floor(cs / 360000)}:${String(Math.floor(cs / 6000) % 60).padStart(2, "0")}:${String(Math.floor(cs / 100) % 60).padStart(2, "0")}.${String(cs % 100).padStart(2, "0")}`;
|
||
};
|
||
const color = (hex: string, opacity: number) =>
|
||
`&H${Math.round((1 - opacity) * 255)
|
||
.toString(16)
|
||
.padStart(2, "0")}${hex.slice(5, 7)}${hex.slice(3, 5)}${hex.slice(1, 3)}`;
|
||
// Saved legacy styles use 720px; new reference-aligned styles explicitly use 1080px.
|
||
const rows = compiledSubtitles(plan) as Array<{
|
||
start: number;
|
||
end: number;
|
||
text: string;
|
||
style?: CaptionStyle;
|
||
}>;
|
||
const styles = rows.map((s, i) => {
|
||
const style = s.style || defaultCaptionStyle;
|
||
const scale = height / (style.referenceHeight || 720);
|
||
return `Style: c${i},${captionFont(style.fontFamily)?.family || "Noto Sans CJK SC"},${style.fontSize * scale},${color(style.color, style.opacity)},&H00FFFFFF,${color(style.outlineColor, style.opacity * (style.outlineOpacity ?? 1))},&HFF000000,0,0,0,0,100,100,${style.spacing * scale},0,1,${style.outlineWidth * scale},0,2,20,20,${Math.round(height * 0.05)},1`;
|
||
});
|
||
const events = rows.map((s, i) => {
|
||
const text = s.text
|
||
.replace(/\\/g, "＼")
|
||
.replace(/\{/g, "｛")
|
||
.replace(/\}/g, "｝")
|
||
.replace(/\r/g, "")
|
||
.replace(/\n/g, "\\N");
|
||
const style = s.style || defaultCaptionStyle;
|
||
const position =
|
||
style.x !== undefined || style.y !== undefined
|
||
? `{\\pos(${Math.round((style.x ?? 0.5) * width)},${Math.round((style.y ?? 0.95) * height)})}`
|
||
: "";
|
||
return `Dialogue: 0,${stamp(s.start)},${stamp(s.end)},c${i},,0,0,0,,${position}${text}`;
|
||
});
|
||
return `[Script Info]\nScriptType: v4.00+\nPlayResX: ${width}\nPlayResY: ${height}\nWrapStyle: 0\nScaledBorderAndShadow: yes\n\n[V4+ Styles]\nFormat: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding\n${styles.join("\n")}\n\n[Events]\nFormat: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n${events.join("\n")}\n`;
|
||
}
|
||
export function subtitlesToSrt(subtitles: z.infer<typeof subtitleSchema>[]) {
|
||
const stamp = (v: number) => {
|
||
const ms = Math.round(v / 1000);
|
||
return `${String(Math.floor(ms / 3600000)).padStart(2, "0")}:${String(Math.floor(ms / 60000) % 60).padStart(2, "0")}:${String(Math.floor(ms / 1000) % 60).padStart(2, "0")},${String(ms % 1000).padStart(3, "0")}`;
|
||
};
|
||
return subtitles
|
||
.map(
|
||
(s, i) =>
|
||
`${i + 1}\n${stamp(s.start)} --> ${stamp(s.end)}\n${s.text.replace(/\r/g, "").replace(/\n\s*\n/g, "\n")}`,
|
||
)
|
||
.join("\n\n");
|
||
}
|
||
export function parseSrt(text: string) {
|
||
if (text.length > 200000) throw Error("字幕文件过大");
|
||
const stamp = (v: string) => {
|
||
const [h, m, s] = v.replace(",", ".").split(":").map(Number);
|
||
return Math.round((h * 3600 + m * 60 + s) * SECOND);
|
||
};
|
||
const rows = text
|
||
.replace(/^\uFEFF/, "")
|
||
.replace(/\r/g, "")
|
||
.trim()
|
||
.split(/\n\s*\n/)
|
||
.filter(Boolean)
|
||
.map((block) => {
|
||
const lines = block.split("\n"),
|
||
i = lines.findIndex((l) => l.includes("-->"));
|
||
const match = lines[i]?.match(
|
||
/(\d{2}:\d{2}:\d{2}[,.]\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2}[,.]\d{3})/,
|
||
);
|
||
if (!match) throw Error("SRT 时间格式无效");
|
||
return subtitleSchema.parse({
|
||
start: stamp(match[1]),
|
||
end: stamp(match[2]),
|
||
text: lines.slice(i + 1).join("\n"),
|
||
});
|
||
});
|
||
if (rows.length > 500) throw Error("字幕最多 500 条");
|
||
return rows;
|
||
}
|
||
export function buildEditPlans(
|
||
params: Extract<SmartEditRequest, { operation: "plan" }>,
|
||
assets: EditAsset[],
|
||
candidates: EditCandidate[],
|
||
order: string[],
|
||
beats: number[],
|
||
warnings: string[],
|
||
): EditPlan[] {
|
||
if (!candidates.length) throw Error("没有可剪辑片段，请关闭去静音或更换素材");
|
||
const valid = candidates.map((c) => candidateSchema.parse(c));
|
||
const seen = new Set<string>();
|
||
const ordered = order.map((id) => {
|
||
const c = valid.find((c) => c.id === id);
|
||
if (!c || seen.has(id)) throw Error("选片模型返回了无效或重复片段 ID");
|
||
seen.add(id);
|
||
return c;
|
||
});
|
||
if (!ordered.length) throw Error("没有选中可用片段");
|
||
return Array.from({ length: params.mode === "ads" ? params.variants : 1 }, (_, variant) => {
|
||
const list = variant
|
||
? [
|
||
...ordered.slice(variant % ordered.length),
|
||
...ordered.slice(0, variant % ordered.length),
|
||
]
|
||
: ordered;
|
||
const clips: EditPlan["clips"] = [];
|
||
let duration = 0;
|
||
const target =
|
||
params.mode === "sequence"
|
||
? MAX_EDIT_SECONDS
|
||
: params.targetDuration ||
|
||
Math.min(
|
||
MAX_EDIT_SECONDS,
|
||
assets.reduce((sum, a) => sum + a.duration / SECOND, 0),
|
||
);
|
||
const visualLength = Math.max(5, Math.min(30, target / 12)) * SECOND;
|
||
for (const candidate of list) {
|
||
if (duration >= target * SECOND || clips.length >= 12) break;
|
||
const available = candidate.end - candidate.start,
|
||
remaining = target * SECOND - duration;
|
||
const speech = params.mode === "talk" || params.mode === "explain";
|
||
// Speech selections keep their sentence boundary even if it slightly misses requested length.
|
||
let length =
|
||
params.mode === "sequence"
|
||
? Math.min(available, remaining)
|
||
: speech
|
||
? available
|
||
: Math.min(available, remaining, visualLength);
|
||
if (speech && duration + length > MAX_EDIT_SECONDS * SECOND) continue;
|
||
if (params.mode !== "sequence" && params.beatSync && !speech && beats.length) {
|
||
const stride = Number(params.beatsPerCut);
|
||
const desired = Math.min(
|
||
visualLength,
|
||
available,
|
||
remaining / Math.max(1, Math.min(12 - clips.length, list.length - clips.length)),
|
||
);
|
||
const next = beats
|
||
.filter((_, i) => i % stride === 0)
|
||
.map((t) => t - params.music!.start)
|
||
.filter(
|
||
(t) =>
|
||
t - duration >= 0.5 * SECOND && t - duration <= Math.min(available, remaining),
|
||
)
|
||
.sort(
|
||
(a, b) => Math.abs(a - duration - desired) - Math.abs(b - duration - desired),
|
||
)[0];
|
||
if (next !== undefined) length = next - duration;
|
||
}
|
||
if (length < 200000) continue;
|
||
clips.push({
|
||
id: `clip-${variant}-${clips.length}`,
|
||
assetId: candidate.assetId,
|
||
sourceIn: candidate.start,
|
||
sourceOut: candidate.start + length,
|
||
speed: 1,
|
||
gain: 1,
|
||
label: candidate.text.slice(0, 300) || `镜头 ${clips.length + 1}`,
|
||
});
|
||
duration += length;
|
||
}
|
||
const plan: EditPlan = {
|
||
version: 1,
|
||
title: `${{ sequence: "智能剪辑", auto: "默认模式", montage: "素材混剪", talk: "口播视频", explain: "讲解视频", ads: "批量广告" }[params.mode]}${variant ? ` ${variant + 1}` : ""}`,
|
||
assets,
|
||
clips,
|
||
ratio: params.ratio,
|
||
resolution: params.resolution,
|
||
transition: 0,
|
||
subtitles: true,
|
||
music: params.music,
|
||
warnings: [...warnings],
|
||
};
|
||
const actual = planTimeline(plan).duration / SECOND;
|
||
if (
|
||
params.mode === "sequence" &&
|
||
assets.reduce((total, a) => total + a.duration, 0) > duration
|
||
)
|
||
plan.warnings.push("时间轴已载入前 300 秒，超出的素材未加入；可在设置中另建剪辑方案。");
|
||
if (
|
||
params.mode !== "sequence" &&
|
||
params.targetDuration > 0 &&
|
||
Math.abs(actual - params.targetDuration) > 0.5
|
||
)
|
||
plan.warnings.push(
|
||
`目标 ${params.targetDuration} 秒，实际 ${actual.toFixed(2)} 秒；保留完整语句或受素材长度限制。`,
|
||
);
|
||
if (params.mode === "ads" && params.variants > ordered.length)
|
||
plan.warnings.push("候选镜头不足，部分广告版本可能相同。");
|
||
return plan;
|
||
});
|
||
}
|