From c6533e92f328779b8c0621a8edab738ed7bed06e Mon Sep 17 00:00:00 2001 From: 134 Date: Fri, 18 Sep 2026 10:28:09 +0800 Subject: [PATCH] Complete smart edit audio processing and font catalogue with main app validation --- .gitignore | 8 + app/api/canvas/[id]/route.ts | 2 + app/api/canvas/route.ts | 2 + .../canvas/smart-edit/capabilities/route.ts | 13 +- app/api/canvas/smart-edit/fonts/[id]/route.ts | 22 + .../canvas/LiblibVideoCanvasReplica.tsx | 12 +- components/canvas/smart-edit/CaptionFonts.tsx | 183 +- .../canvas/smart-edit/SmartEditDialog.tsx | 40 +- .../smart-edit/SmartEditEditor.module.css | 17 +- .../canvas/smart-edit/SmartEditEditor.tsx | 110 +- docs/smart-edit-completion-2026-09-18.md | 63 + docs/smart-edit-fixes-2026-09-18.json | 41 +- docs/smart-edit-fixes-2026-09-18.md | 32 +- lib/video-canvas/smart-edit/audio-process.ts | 178 + lib/video-canvas/smart-edit/contract.ts | 28 +- lib/video-canvas/smart-edit/execute.ts | 44 +- lib/video-canvas/smart-edit/font-catalog.json | 29957 ++++++++++++++++ lib/video-canvas/smart-edit/font-files.ts | 88 + lib/video-canvas/smart-edit/fonts.ts | 28 +- lib/video-canvas/smart-edit/local.ts | 12 +- lib/video-canvas/smart-edit/render-job.ts | 7 +- lib/video-canvas/smart-edit/render.ts | 12 +- lib/video-canvas/task-contract.ts | 38 +- next.config.ts | 5 +- scripts/qa/smart-edit-main-chain.cjs | 214 + scripts/qa/smart-edit-models.py | 30 + scripts/qa/video-canvas-smart-edit.cjs | 4 +- services/smart-edit/README.md | 18 +- services/smart-edit/analyze.py | 12 +- services/smart-edit/font.py | 35 + services/smart-edit/prepare_fonts.py | 23 + services/smart-edit/prepare_separation.py | 18 + services/smart-edit/requirements-asr.txt | 2 + services/smart-edit/requirements-fonts.txt | 2 + .../smart-edit/requirements-separation.txt | 4 + services/smart-edit/separate.py | 59 + services/smart-edit/setup.sh | 8 +- services/video-render/build.cjs | 5 +- services/video-render/server.ts | 1 + tests/routes/video-canvas-smart-edit.test.ts | 53 + 40 files changed, 31285 insertions(+), 145 deletions(-) create mode 100644 app/api/canvas/smart-edit/fonts/[id]/route.ts create mode 100644 docs/smart-edit-completion-2026-09-18.md create mode 100644 lib/video-canvas/smart-edit/audio-process.ts create mode 100644 lib/video-canvas/smart-edit/font-catalog.json create mode 100644 lib/video-canvas/smart-edit/font-files.ts create mode 100644 scripts/qa/smart-edit-main-chain.cjs create mode 100644 scripts/qa/smart-edit-models.py create mode 100644 services/smart-edit/font.py create mode 100644 services/smart-edit/prepare_fonts.py create mode 100644 services/smart-edit/prepare_separation.py create mode 100644 services/smart-edit/requirements-fonts.txt create mode 100644 services/smart-edit/requirements-separation.txt create mode 100644 services/smart-edit/separate.py diff --git a/.gitignore b/.gitignore index bdc5ded58..a0e8f5c0d 100644 --- a/.gitignore +++ b/.gitignore @@ -112,3 +112,11 @@ public/vendor/zoo-modeling-app/ .cache/smart-edit/ services/video-render/dist/ + +# Downloaded and normalized subtitle fonts are runtime assets. +.cache/smart-edit-fonts/ +services/smart-edit/__pycache__/ + +# Local verification artefacts (also excluded from Tailwind's automatic source scan). +.qa/ +.next-video-*/ diff --git a/app/api/canvas/[id]/route.ts b/app/api/canvas/[id]/route.ts index 3d8063edf..a6aac174e 100644 --- a/app/api/canvas/[id]/route.ts +++ b/app/api/canvas/[id]/route.ts @@ -163,6 +163,8 @@ export async function PUT( .values({ id, userId: session.user.id, + createdAt: new Date(), + updatedAt: new Date(), title: title || getDefaultWorkspaceTitle(workspaceType || "canvas"), workspaceType: workspaceType || "canvas", canvasData: { diff --git a/app/api/canvas/route.ts b/app/api/canvas/route.ts index 621e14182..b51ce00a8 100644 --- a/app/api/canvas/route.ts +++ b/app/api/canvas/route.ts @@ -100,6 +100,8 @@ export async function POST(request: NextRequest) { .insert(canvasProject) .values({ userId: session.user.id, + createdAt: new Date(), + updatedAt: new Date(), title: title || getDefaultWorkspaceTitle(workspaceType), description, workspaceType, diff --git a/app/api/canvas/smart-edit/capabilities/route.ts b/app/api/canvas/smart-edit/capabilities/route.ts index b67185b4b..7f3899d42 100644 --- a/app/api/canvas/smart-edit/capabilities/route.ts +++ b/app/api/canvas/smart-edit/capabilities/route.ts @@ -10,7 +10,14 @@ export async function GET() { const session = await auth(); if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); try { - let capabilities = { scenes: false, beats: false, asr: false, ffmpeg: false, ffprobe: false }; + let capabilities = { + separation: false, + scenes: false, + beats: false, + asr: false, + ffmpeg: false, + ffprobe: false, + }; try { const { stdout } = await promisify(execFile)( process.env.SMART_EDIT_PYTHON || "python3", @@ -35,7 +42,9 @@ export async function GET() { } } const models = (await getEnabledAiModelsWithProvider(false)) - .filter((item) => isCanvasVisionModel({ ...item.model, providerType: item.provider.type })) + .filter((item) => + isCanvasVisionModel({ ...item.model, providerType: item.provider.type }), + ) .map(({ model }) => ({ id: model.id, name: model.displayName, modelId: model.modelId })); return NextResponse.json( { capabilities: { ...capabilities, render: renderAvailable }, models }, diff --git a/app/api/canvas/smart-edit/fonts/[id]/route.ts b/app/api/canvas/smart-edit/fonts/[id]/route.ts new file mode 100644 index 000000000..1e2e33be1 --- /dev/null +++ b/app/api/canvas/smart-edit/fonts/[id]/route.ts @@ -0,0 +1,22 @@ +import { auth } from "@/app/(auth)/auth"; +import { captionFont } from "@/lib/video-canvas/smart-edit/fonts"; +import { captionFontFile } from "@/lib/video-canvas/smart-edit/font-files"; +import { readFile } from "node:fs/promises"; +export const runtime = "nodejs"; +export const maxDuration = 180; +export async function GET(_request: Request, { params }: { params: Promise<{ id: string }> }) { + if (!(await auth())?.user?.id) return Response.json({ error: "未登录" }, { status: 401 }); + const { id } = await params; + if (!captionFont(id)) return Response.json({ error: "字体不存在" }, { status: 404 }); + try { + const data = await readFile(await captionFontFile(id)); + return new Response(new Uint8Array(data), { + headers: { + "Content-Type": "font/ttf", + "Cache-Control": "private, max-age=31536000, immutable", + }, + }); + } catch { + return Response.json({ error: "字体加载失败,请检查字体服务配置后重试" }, { status: 503 }); + } +} diff --git a/components/canvas/LiblibVideoCanvasReplica.tsx b/components/canvas/LiblibVideoCanvasReplica.tsx index a8a95756f..5861122a8 100644 --- a/components/canvas/LiblibVideoCanvasReplica.tsx +++ b/components/canvas/LiblibVideoCanvasReplica.tsx @@ -2226,7 +2226,8 @@ function getVideoNodePreviewUrl(node: CloneNode | null | undefined) { } function getVideoNodeDurationSeconds(node: CloneNode | null | undefined) { - const recordedDuration = Number(node?.draftValues?.durationSeconds); + const meta = node?.draftValues?.videoMeta as Record | undefined; + const recordedDuration = Number(meta?.durationSeconds || node?.draftValues?.durationSeconds || meta?.duration); if (Number.isFinite(recordedDuration) && recordedDuration > 0) return recordedDuration; const params = getNodeParams(node); const paramsSettings = @@ -20688,7 +20689,9 @@ function OfficialVideoResultCard({ const [currentTime, setCurrentTime] = useState(0); const [frameMenuOpen, setFrameMenuOpen] = useState(false); const [hovered, setHovered] = useState(false); - const safeDuration = Math.max(0.1, durationSeconds || 5); + const [decodedDuration, setDecodedDuration] = useState(0); + useEffect(() => { setDecodedDuration(0); setCurrentTime(0); setPlaying(false); }, [mediaUrl]); + const safeDuration = Math.max(0.1, decodedDuration || durationSeconds || 5); const progress = Math.max(0, Math.min(currentTime / safeDuration, 1)); const controlsVisible = hovered || selected || frameMenuOpen; const frameActions: Array<{ kind: VideoFrameExtractionKind; label: string }> = [ @@ -20701,8 +20704,7 @@ function OfficialVideoResultCard({ const video = videoRef.current; if (!video) return; if (video.paused) { - void video.play(); - setPlaying(true); + void video.play().catch(() => { setPlaying(false); toast.error("视频无法播放,请检查素材地址"); }); return; } video.pause(); @@ -20730,6 +20732,8 @@ function OfficialVideoResultCard({