import { z } from "zod"; import { captionFont } from "./fonts"; export const SECOND = 1_000_000; export const MAX_EDIT_SECONDS = 300; export const editRatios = ["21:9", "16:9", "9:16", "1:1", "4:3", "3:4"] as const; export const editResolutions = ["480p", "720p", "1080p"] as const; const time = z .number() .int() .min(0) .max(600 * SECOND); const id = z.string().min(1).max(180); const url = z .string() .url() .max(4096) .refine((value) => /^https?:\/\//i.test(value)); export const subtitleSchema = z .object({ start: time, end: time, text: z.string().max(500) }) .refine((x) => x.end > x.start); export const captionStyleSchema = z.object({ fontFamily: z .string() .max(180) .refine((id) => Boolean(captionFont(id)), "未知字幕字体") .optional(), referenceHeight: z.union([z.literal(720), z.literal(1080)]).optional(), fontSize: z.number().min(8).max(160).default(28), color: z .string() .regex(/^#[0-9a-f]{6}$/i) .default("#ffffff"), outlineColor: z .string() .regex(/^#[0-9a-f]{6}$/i) .default("#000000"), outlineWidth: z.number().min(0).max(20).default(0), opacity: z.number().min(0).max(1).default(1), // Optional so existing edits keep their whole-caption opacity unchanged. outlineOpacity: z.number().min(0).max(1).optional(), spacing: z.number().min(-5).max(100).default(0), x: z.number().min(0).max(1).optional(), y: z.number().min(0).max(1).optional(), }); export const timelineCaptionSchema = z .object({ id, start: time, end: time, text: z.string().max(500), style: captionStyleSchema.optional(), }) .refine((x) => x.end > x.start); export type TimelineCaption = z.infer; export type CaptionStyle = z.infer; export const defaultCaptionStyle: CaptionStyle = captionStyleSchema.parse({}); export const newCaptionStyle: CaptionStyle = { ...defaultCaptionStyle, fontFamily: "source-han", referenceHeight: 1080, fontSize: 56, outlineColor: "#ffffff", }; const transitionSchema = z.object({ duration: z.number().min(0).max(4), // Centered transitions preserve the edit duration; absent keeps saved legacy overlaps. alignment: z.enum(["overlap", "center"]).optional(), type: z.enum(["fade", "fadeblack", "fadewhite"]), }); export const assetInputSchema = z.object({ id, url, name: z.string().max(200), subtitles: z.array(subtitleSchema).max(500).default([]), }); export const editAssetSchema = assetInputSchema.extend({ duration: time, width: z.number().int().min(0).max(8192), height: z.number().int().min(0).max(8192), hasAudio: z.boolean(), hash: z.string().regex(/^[a-f0-9]{64}$/), }); export const candidateSchema = z .object({ id, assetId: id, start: time, end: time, text: z.string().max(1500).default("") }) .refine((x) => x.end - x.start >= 200000); export const videoTransformSchema = z.object({ x: z.number().min(-1).max(1).default(0), y: z.number().min(-1).max(1).default(0), scale: z.number().min(0.1).max(3).default(1), }); export type VideoTransform = z.infer; export const defaultVideoTransform: VideoTransform = { x: 0, y: 0, scale: 1 }; export const editClipSchema = z .object({ id, assetId: id, sourceIn: time, sourceOut: time, speed: z.number().min(0.5).max(2).default(1), gain: z.number().min(0).max(4).default(1), muted: z.boolean().optional(), label: z.string().max(300).default(""), // The transition belongs to the incoming clip, so it follows that clip when reordered. transition: transitionSchema.optional(), transform: videoTransformSchema.optional(), audioDetached: z.boolean().optional(), }) .refine((x) => x.sourceOut - x.sourceIn >= 200000); export const audioClipSchema = z .object({ id, assetId: id, start: time, sourceIn: time, sourceOut: time, speed: z.number().min(0.5).max(2).default(1), gain: z.number().min(0).max(4).default(1), muted: z.boolean().optional(), fadeIn: z.number().min(0).max(5).default(0), fadeOut: z.number().min(0).max(5).default(0), fadeInStart: z.number().min(0).max(1).optional(), fadeOutEnd: z.number().min(0).max(1).optional(), }) .refine((c) => c.sourceOut - c.sourceIn >= 200000); export type AudioClip = z.infer; export const editPlanSchema = z.object({ version: z.literal(1), title: z.string().max(200), assets: z.array(editAssetSchema).min(1).max(36), clips: z.array(editClipSchema).min(1).max(12), ratio: z.enum(editRatios), resolution: z.enum(editResolutions).default("720p"), transition: z.number().min(0).max(0.5).default(0), subtitles: z.boolean().default(true), captions: z.array(timelineCaptionSchema).max(500).optional(), audioClips: z.array(audioClipSchema).max(24).optional(), music: z .object({ url, gain: z.number().min(0).max(1).default(0.2), start: time.default(0) }) .optional(), warnings: z.array(z.string().max(500)).max(30).default([]), }); export type EditPlan = z.infer; export type EditAsset = z.infer; export type EditCandidate = z.infer; export const smartEditRequestSchema = z.discriminatedUnion("operation", [ z.object({ operation: z.literal("plan"), assets: z.array(assetInputSchema).min(1).max(12), mode: z.enum(["sequence", "auto", "montage", "talk", "explain", "ads"]).default("montage"), prompt: z.string().max(3000).default(""), targetDuration: z.number().min(0).max(MAX_EDIT_SECONDS).default(20), shotDurations: z.array(z.number().min(0.2).max(MAX_EDIT_SECONDS)).max(12).optional(), shotDescriptions: z.array(z.string().max(600)).max(12).optional(), ratio: z.enum(editRatios).default("16:9"), resolution: z.enum(editResolutions).default("720p"), removeSilence: z.boolean().default(false), transcribe: z.boolean().default(false), semantic: z.boolean().default(true), muteAudio: z.boolean().default(false), preferredModelRef: z.string().max(180).optional(), music: z .object({ url, gain: z.number().min(0).max(1).default(0.2), start: time.default(0) }) .optional(), beatSync: z.boolean().default(false), beatsPerCut: z.enum(["2", "4"]).default("4"), variants: z.number().int().min(1).max(3).default(1), }), z.object({ operation: z.literal("separate"), plan: editPlanSchema, clipId: id, track: z.enum(["video", "audio"]).default("video"), keep: z.enum(["both", "vocals", "background"]).default("both"), }), z.object({ operation: z.literal("transcribe"), plan: editPlanSchema, }), z.object({ operation: z.literal("render"), plan: editPlanSchema, sourceNodeId: z.string().max(180).optional(), }), ]); export type SmartEditRequest = z.infer; export function editOutputSize(plan: Pick) { const size = Number.parseInt(plan.resolution, 10), [w, h] = plan.ratio.split(":").map(Number); const even = (v: number) => Math.round(v / 2) * 2; return w >= h ? { width: even((size * w) / h), height: size } : { width: size, height: even((size * h) / w) }; } export function planTimeline(input: EditPlan) { const plan = editPlanSchema.parse(input), ids = new Set(); const durations = plan.clips.map((c) => Math.round( (Math.round(((c.sourceOut - c.sourceIn) / c.speed / SECOND) * 30) * SECOND) / 30, ), ); const centered = plan.clips.map((c) => c.transition?.alignment === "center"); const overlaps = plan.clips.map((c, i) => i ? Math.min( Math.round((c.transition?.duration ?? plan.transition) * SECOND), centered[i] ? durations[i] * 2 : Math.floor(durations[i] / 3), centered[i] ? durations[i - 1] * 2 : Math.floor(durations[i - 1] / 3), ) : 0, ); // Neighboring transitions cannot consume the same frames in a short middle clip. for (let i = 0; i < durations.length; i++) { const incoming = overlaps[i] / (centered[i] ? 2 : 1), outgoing = (overlaps[i + 1] || 0) / (centered[i + 1] ? 2 : 1); if (incoming + outgoing > durations[i]) { const factor = durations[i] / (incoming + outgoing); overlaps[i] = Math.floor(overlaps[i] * factor); if (i + 1 < overlaps.length) overlaps[i + 1] = Math.floor(overlaps[i + 1] * factor); } } const before = overlaps.map((d, i) => centered[i] ? Math.round((Math.floor(((d / SECOND) * 30) / 2) * SECOND) / 30) : 0, ); const after = overlaps.map((d, i) => (centered[i] ? d - before[i] : 0)); let cursor = 0; const clips = plan.clips.map((clip, index) => { if (ids.has(clip.id)) throw Error("片段 ID 重复"); ids.add(clip.id); const asset = plan.assets.find((a) => a.id === clip.assetId); if (!asset || !asset.width || !asset.height || clip.sourceOut > asset.duration) throw Error("片段超出原始素材范围"); const duration = durations[index], overlap = overlaps[index]; if (!centered[index]) cursor -= overlap; const start = cursor; cursor += duration; const padBefore = before[index], padAfter = after[index + 1] || 0; return { ...clip, start, end: cursor, duration, overlap, padBefore, padAfter, renderStart: start - padBefore, renderEnd: cursor + padAfter, transitionStart: start - padBefore, transitionType: clip.transition?.type || "fade", }; }); if (new Set(plan.assets.map((a) => a.id)).size !== plan.assets.length) throw Error("素材 ID 重复"); if (plan.captions && new Set(plan.captions.map((s) => s.id)).size !== plan.captions.length) throw Error("字幕 ID 重复"); if (cursor > MAX_EDIT_SECONDS * SECOND || cursor < 200000) throw Error("成片时长需在 0.2–300 秒内"); const audioClips = (plan.audioClips || []).map((c) => { const asset = plan.assets.find((a) => a.id === c.assetId); if (!asset?.hasAudio || c.sourceOut > asset.duration) throw Error("独立音轨素材无效或超出范围"); const duration = Math.round((c.sourceOut - c.sourceIn) / c.speed); return { ...c, duration, end: c.start + duration }; }); if (new Set(audioClips.map((c) => c.id)).size !== audioClips.length) throw Error("独立音轨 ID 重复"); cursor = Math.max(cursor, ...audioClips.map((c) => c.end)); if (cursor > MAX_EDIT_SECONDS * SECOND) throw Error("成片时长需在 0.2–300 秒内"); return { clips, audioClips, duration: cursor }; } export function compiledSubtitles(plan: EditPlan) { if (!plan.subtitles) return []; if (plan.captions) { const duration = planTimeline(plan).duration; return plan.captions.flatMap((s) => { const end = Math.min(s.end, duration); return end > s.start ? [{ ...s, end }] : []; }); } return planTimeline(plan).clips.flatMap((clip) => plan.assets .find((a) => a.id === clip.assetId)! .subtitles.flatMap((s) => { const a = Math.max(s.start, clip.sourceIn), b = Math.min(s.end, clip.sourceOut); if (b <= a) return []; // Only legacy overlapping edits delay incoming captions until the outgoing clip ends. const start = Math.max( clip.start + (clip.transition?.alignment === "center" ? 0 : clip.overlap), clip.start + Math.round((a - clip.sourceIn) / clip.speed), ); const end = Math.min( clip.end, clip.start + Math.round((b - clip.sourceIn) / clip.speed), ); return end > start ? [{ start, end, text: s.text }] : []; }), ); } export function editableCaptions(plan: EditPlan): TimelineCaption[] { if (plan.captions) return plan.captions; return compiledSubtitles({ ...plan, subtitles: true }).map((s, i) => ({ ...s, id: `caption-${i}`, })); } export function subtitlesToAss(plan: EditPlan, width: number, height: number) { const stamp = (v: number) => { const cs = Math.round(v / 10000); return `${Math.floor(cs / 360000)}:${String(Math.floor(cs / 6000) % 60).padStart(2, "0")}:${String(Math.floor(cs / 100) % 60).padStart(2, "0")}.${String(cs % 100).padStart(2, "0")}`; }; const color = (hex: string, opacity: number) => `&H${Math.round((1 - opacity) * 255) .toString(16) .padStart(2, "0")}${hex.slice(5, 7)}${hex.slice(3, 5)}${hex.slice(1, 3)}`; // Saved legacy styles use 720px; new reference-aligned styles explicitly use 1080px. const rows = compiledSubtitles(plan) as Array<{ start: number; end: number; text: string; style?: CaptionStyle; }>; const styles = rows.map((s, i) => { const style = s.style || defaultCaptionStyle; const scale = height / (style.referenceHeight || 720); return `Style: c${i},${captionFont(style.fontFamily)?.family || "Noto Sans CJK SC"},${style.fontSize * scale},${color(style.color, style.opacity)},&H00FFFFFF,${color(style.outlineColor, style.opacity * (style.outlineOpacity ?? 1))},&HFF000000,0,0,0,0,100,100,${style.spacing * scale},0,1,${style.outlineWidth * scale},0,2,20,20,${Math.round(height * 0.05)},1`; }); const events = rows.map((s, i) => { const text = s.text .replace(/\\/g, "\") .replace(/\{/g, "{") .replace(/\}/g, "}") .replace(/\r/g, "") .replace(/\n/g, "\\N"); const style = s.style || defaultCaptionStyle; const position = style.x !== undefined || style.y !== undefined ? `{\\pos(${Math.round((style.x ?? 0.5) * width)},${Math.round((style.y ?? 0.95) * height)})}` : ""; return `Dialogue: 0,${stamp(s.start)},${stamp(s.end)},c${i},,0,0,0,,${position}${text}`; }); return `[Script Info]\nScriptType: v4.00+\nPlayResX: ${width}\nPlayResY: ${height}\nWrapStyle: 0\nScaledBorderAndShadow: yes\n\n[V4+ Styles]\nFormat: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding\n${styles.join("\n")}\n\n[Events]\nFormat: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n${events.join("\n")}\n`; } export function subtitlesToSrt(subtitles: z.infer[]) { const stamp = (v: number) => { const ms = Math.round(v / 1000); return `${String(Math.floor(ms / 3600000)).padStart(2, "0")}:${String(Math.floor(ms / 60000) % 60).padStart(2, "0")}:${String(Math.floor(ms / 1000) % 60).padStart(2, "0")},${String(ms % 1000).padStart(3, "0")}`; }; return subtitles .map( (s, i) => `${i + 1}\n${stamp(s.start)} --> ${stamp(s.end)}\n${s.text.replace(/\r/g, "").replace(/\n\s*\n/g, "\n")}`, ) .join("\n\n"); } export function parseSrt(text: string) { if (text.length > 200000) throw Error("字幕文件过大"); const stamp = (v: string) => { const [h, m, s] = v.replace(",", ".").split(":").map(Number); return Math.round((h * 3600 + m * 60 + s) * SECOND); }; const rows = text .replace(/^\uFEFF/, "") .replace(/\r/g, "") .trim() .split(/\n\s*\n/) .filter(Boolean) .map((block) => { const lines = block.split("\n"), i = lines.findIndex((l) => l.includes("-->")); const match = lines[i]?.match( /(\d{2}:\d{2}:\d{2}[,.]\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2}[,.]\d{3})/, ); if (!match) throw Error("SRT 时间格式无效"); return subtitleSchema.parse({ start: stamp(match[1]), end: stamp(match[2]), text: lines.slice(i + 1).join("\n"), }); }); if (rows.length > 500) throw Error("字幕最多 500 条"); return rows; } export function buildEditPlans( params: Extract, assets: EditAsset[], candidates: EditCandidate[], order: string[], beats: number[], warnings: string[], sequenceTransitions: ("cut" | "fade")[] = [], ): EditPlan[] { if (!candidates.length) throw Error("没有可剪辑片段,请关闭去静音或更换素材"); const valid = candidates.map((c) => candidateSchema.parse(c)); const seen = new Set(); const ordered = order.map((id) => { const c = valid.find((c) => c.id === id); if (!c || seen.has(id)) throw Error("选片模型返回了无效或重复片段 ID"); seen.add(id); return c; }); if (!ordered.length) throw Error("没有选中可用片段"); if (params.mode === "sequence" && params.shotDurations && params.shotDurations.length !== ordered.length) throw Error("分镜时长与视频素材数量不匹配"); return Array.from({ length: params.mode === "ads" ? params.variants : 1 }, (_, variant) => { const list = variant ? [ ...ordered.slice(variant % ordered.length), ...ordered.slice(0, variant % ordered.length), ] : ordered; const clips: EditPlan["clips"] = []; let duration = 0; const target = params.mode === "sequence" ? params.shotDurations?.reduce((sum, seconds) => sum + seconds, 0) || MAX_EDIT_SECONDS : params.targetDuration || Math.min( MAX_EDIT_SECONDS, assets.reduce((sum, a) => sum + a.duration / SECOND, 0), ); const visualLength = Math.max(5, Math.min(30, target / 12)) * SECOND; for (const candidate of list) { if (duration >= target * SECOND || clips.length >= 12) break; const available = candidate.end - candidate.start, remaining = target * SECOND - duration; const speech = params.mode === "talk" || params.mode === "explain"; // Speech selections keep their sentence boundary even if it slightly misses requested length. let length = params.mode === "sequence" ? Math.min(available, remaining, (params.shotDurations?.[clips.length] ?? MAX_EDIT_SECONDS) * SECOND) : speech ? available : Math.min(available, remaining, visualLength); if (speech && duration + length > MAX_EDIT_SECONDS * SECOND) continue; if (params.mode !== "sequence" && params.beatSync && !speech && beats.length) { const stride = Number(params.beatsPerCut); const desired = Math.min( visualLength, available, remaining / Math.max(1, Math.min(12 - clips.length, list.length - clips.length)), ); const next = beats .filter((_, i) => i % stride === 0) .map((t) => t - params.music!.start) .filter( (t) => t - duration >= 0.5 * SECOND && t - duration <= Math.min(available, remaining), ) .sort( (a, b) => Math.abs(a - duration - desired) - Math.abs(b - duration - desired), )[0]; if (next !== undefined) length = next - duration; } if (length < 200000) continue; clips.push({ id: `clip-${variant}-${clips.length}`, assetId: candidate.assetId, sourceIn: candidate.start, sourceOut: candidate.start + length, speed: 1, gain: 1, muted: params.muteAudio, label: candidate.text.slice(0, 300) || `镜头 ${clips.length + 1}`, ...(params.mode === "sequence" && sequenceTransitions[clips.length] === "fade" && clips.length > 0 ? { transition: { type: "fade" as const, duration: 0.2, alignment: "center" as const } } : {}), }); duration += length; } const plan: EditPlan = { version: 1, title: `${{ sequence: "智能剪辑", auto: "默认模式", montage: "素材混剪", talk: "口播视频", explain: "讲解视频", ads: "批量广告" }[params.mode]}${variant ? ` ${variant + 1}` : ""}`, assets, clips, ratio: params.ratio, resolution: params.resolution, transition: 0, subtitles: true, music: params.muteAudio ? undefined : params.music, warnings: [...warnings], }; const actual = planTimeline(plan).duration / SECOND; if ( params.mode === "sequence" && !params.shotDurations && assets.reduce((total, a) => total + a.duration, 0) > duration ) plan.warnings.push("时间轴已载入前 300 秒,超出的素材未加入;可在设置中另建剪辑方案。"); if (params.targetDuration > 0 && Math.abs(actual - params.targetDuration) > 0.5 && (params.mode !== "sequence" || params.shotDurations)) plan.warnings.push( `目标 ${params.targetDuration} 秒,实际 ${actual.toFixed(2)} 秒;保留完整语句或受素材长度限制。`, ); if (params.mode === "ads" && params.variants > ordered.length) plan.warnings.push("候选镜头不足,部分广告版本可能相同。"); return plan; }); }