From f25e78e570bbdafa0bbc2f28c813524d741e3139 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 17 Sep 2026 18:05:17 +0800 Subject: [PATCH] Align smart edit canvas and timeline UI with reference workflow --- .../canvas/LiblibVideoCanvasReplica.tsx | 67 +- components/canvas/smart-edit/EditPreview.tsx | 152 +- .../canvas/smart-edit/SmartEditDialog.tsx | 104 +- .../smart-edit/SmartEditEditor.module.css | 912 +++++++++ .../canvas/smart-edit/SmartEditEditor.tsx | 1664 +++++++++++++++++ .../smart-edit/SmartEditNode.module.css | 377 ++++ .../canvas/smart-edit/SmartEditNode.tsx | 301 +++ docs/smart-edit-ui-parity-2026-09-17.md | 32 + docs/video-render-service-2026-09-17.md | 6 + hooks/use-video-canvas-tasks.ts | 10 +- lib/video-canvas/smart-edit/contract.ts | 158 +- lib/video-canvas/smart-edit/execute.ts | 57 +- lib/video-canvas/smart-edit/local.ts | 10 +- lib/video-canvas/smart-edit/remote-render.ts | 11 +- lib/video-canvas/smart-edit/render-job.ts | 2 +- lib/video-canvas/smart-edit/render.ts | 77 +- scripts/qa/video-canvas-smart-edit.cjs | 83 + services/video-render/README.md | 2 +- tests/routes/video-canvas-smart-edit.test.ts | 152 +- 19 files changed, 4043 insertions(+), 134 deletions(-) create mode 100644 components/canvas/smart-edit/SmartEditEditor.module.css create mode 100644 components/canvas/smart-edit/SmartEditEditor.tsx create mode 100644 components/canvas/smart-edit/SmartEditNode.module.css create mode 100644 components/canvas/smart-edit/SmartEditNode.tsx create mode 100644 docs/smart-edit-ui-parity-2026-09-17.md diff --git a/components/canvas/LiblibVideoCanvasReplica.tsx b/components/canvas/LiblibVideoCanvasReplica.tsx index a64a8915c..b81b4f808 100644 --- a/components/canvas/LiblibVideoCanvasReplica.tsx +++ b/components/canvas/LiblibVideoCanvasReplica.tsx @@ -1,6 +1,7 @@ "use client"; import { SmartEditDialog } from "./smart-edit/SmartEditDialog"; -import { type EditPlan } from "@/lib/video-canvas/smart-edit/contract"; +import { SmartEditNodeCard, SmartEditNodeComposer } from "./smart-edit/SmartEditNode"; +import { smartEditRequestSchema, type EditPlan } from "@/lib/video-canvas/smart-edit/contract"; import { AudioGenerationComposer } from "./AudioGenerationComposer"; import { VideoCanvasConflictNotice } from "./VideoCanvasConflictNotice"; import { normalizeVideoAnalysisRows } from "@/lib/video-canvas/video-story"; @@ -1599,7 +1600,7 @@ function createTargetModuleNode(kind: AddModuleKind, x: number, y: number, index } if (kind === "smart-edit") { - return {id:makeId("smart-edit"),kind:"model",title:`智能剪辑 ${index}`,subtitle:"智能剪辑",inputType:"video",x,y,width:350,height:350,accent:"#ffb8c8",draftValues:{action:"SMART_EDIT_WORKFLOW",nodeType:"smart-edit"}}; + return {id:makeId("smart-edit"),kind:"model",title:`智能剪辑 ${index}`,subtitle:"智能剪辑",inputType:"video",x,y,width:566,height:350,accent:"#ffb8c8",draftValues:{action:"SMART_EDIT_WORKFLOW",nodeType:"smart-edit"}}; } if (kind === "video-clip") { return { @@ -5421,7 +5422,7 @@ export function LiblibVideoCanvasReplica({ ); const addUploadedFilesAtPoint = useCallback( - (files: File[], center: { x: number; y: number }) => { + (files: File[], center: { x: number; y: number }, connectTargetId?: string) => { if (files.length === 0) return; const batch = files.slice(0, 10); const newNodes = batch.map((file, index) => { @@ -5430,7 +5431,7 @@ export function LiblibVideoCanvasReplica({ y: Math.round(center.y - created.height / 2), draftValues: { ...created.draftValues, uploadStatus: "uploading" } }; }); - addNodes(newNodes, []); + addNodes(newNodes, connectTargetId ? newNodes.map(node => ({ id: makeId("edge"), source: node.id, target: connectTargetId })) : []); setActivePanel(null); setCanvasCommand(null); setConnectionMenu(null); @@ -5906,6 +5907,16 @@ export function LiblibVideoCanvasReplica({ toast.error("未找到可连接的节点"); return false; } + if (getTargetModuleKind(target) === "smart-edit") { + if (source.inputType !== "video" || !getNodeReferenceMediaUrl(source)) { + toast.info("智能剪辑需要已上传或已生成的视频素材"); + return false; + } + if (edges.filter(edge => edge.target === targetId).length >= 12) { + toast.info("智能剪辑最多连接 12 份视频素材"); + return false; + } + } const exists = edges.some((edge) => edge.source === sourceId && edge.target === targetId); if (exists) { toast.info("这两个节点已经连接"); @@ -5963,6 +5974,7 @@ export function LiblibVideoCanvasReplica({ switch (actionLabel) { case "打开智能剪辑": { + setNodes(current => current.map(node => node.id === nodeId ? {...node,width:566,height:350} : node)); setSmartEditNodeId(nodeId);setSelectedNodeId(nodeId);setCanvasCommand(null);setNodeContextMenu(null);setConnectionMenu(null);return; } case "打开导演台": { @@ -9510,6 +9522,7 @@ export function LiblibVideoCanvasReplica({ useEffect(() => { const onKeyDown = (event: KeyboardEvent) => { + if (smartEditNodeId) return; const target = event.target as HTMLElement | null; if (target?.closest("input, textarea, select, [contenteditable='true']")) return; if (event.code === "Space") { event.preventDefault(); spacePanRef.current = true; return; } @@ -9562,7 +9575,7 @@ export function LiblibVideoCanvasReplica({ window.addEventListener("keyup", onKeyUp); window.addEventListener("blur", releasePan); return () => { window.removeEventListener("keydown", onKeyDown); window.removeEventListener("keyup", onKeyUp); window.removeEventListener("blur", releasePan); }; - }, [createLineConnectState, deleteNode, deleteSelectedEdge, nodes, selectedEdgeId, selectedNodeId]); + }, [createLineConnectState, deleteNode, deleteSelectedEdge, nodes, selectedEdgeId, selectedNodeId, smartEditNodeId]); const onPointerMove = useCallback( (event: ReactPointerEvent) => { @@ -10053,6 +10066,7 @@ export function LiblibVideoCanvasReplica({ connectionHoverActive={node.id === lineConnectState?.hoverNodeId} key={node.id} node={node} + smartEditPreviewUrl={getTargetModuleKind(node) === "smart-edit" ? getNodeReferenceMediaUrl(nodes.find(source => edges.some(edge => edge.target === node.id && edge.source === source.id) && source.inputType === "video") || node) : undefined} selected={node.id === selectedNodeId} onContextMenu={(event) => openNodeContext(event, node)} onCancelGeneration={() => cancelGenerationNode(node.id)} @@ -10359,6 +10373,34 @@ export function LiblibVideoCanvasReplica({ ) : null} + {selectedNode && selectedTargetModuleKind === "smart-edit" && selectedTargetAnchor && !smartEditNodeId ? ( + } + sources={selectedTargetSourceNodes.filter(node => node.inputType === "video" && getNodeReferenceMediaUrl(node)).map(node => ({id:node.id,name:node.title,url:getNodeReferenceMediaUrl(node),duration:getVideoNodeDurationSeconds(node)}))} + availableSources={nodes.filter(node => node.id !== selectedNode.id && node.inputType === "video" && getNodeReferenceMediaUrl(node)).map(node => ({id:node.id,name:node.title,url:getNodeReferenceMediaUrl(node),duration:getVideoNodeDurationSeconds(node)}))} + busy={generationTasks.tasks.some(task => task.request.node.id === selectedNode.id && (task.status === "pending" || task.status === "running"))} + onDraft={patch => setNodes(current => current.map(node => node.id === selectedNode.id ? {...node,draftValues:{...node.draftValues,smartEditDraft:{...(node.draftValues?.smartEditDraft as Record || {}),...patch}}} : node))} + onAddSource={source => setEdges(current => current.some(edge => edge.source === source && edge.target === selectedNode.id) ? current : [...current,{id:makeId("edge"),source,target:selectedNode.id}])} + onRemoveSource={source => { + setEdges(current => current.filter(edge => !(edge.source === source && edge.target === selectedNode.id))); + setNodes(current => current.map(node => node.id === selectedNode.id ? {...node,draftValues:{...node.draftValues,smartEditDraft:{...(node.draftValues?.smartEditDraft as Record || {}),assets:((node.draftValues?.smartEditDraft as Record)?.assets || []).filter((asset:{id:string}) => asset.id !== source)}}} : node)); + }} + onReference={() => setLineConnectState({sourceId:null,targetId:selectedNode.id,cursorCanvasX:selectedNode.x-80,cursorCanvasY:selectedNode.y+selectedNode.height/2})} + onUpload={files => addUploadedFilesAtPoint(files,{x:selectedNode.x-600,y:selectedNode.y},selectedNode.id)} + onSubmit={async () => { + const draft = (selectedNode.draftValues?.smartEditDraft || {}) as Record; + const assets = selectedTargetSourceNodes.filter(node => node.inputType === "video" && getNodeReferenceMediaUrl(node)).map(node => ({id:node.id,name:node.title,url:getNodeReferenceMediaUrl(node),subtitles:[]})); + const params = smartEditRequestSchema.parse({ratio:"16:9",resolution:"1080p",targetDuration:20,...draft,mode:["auto","montage","talk","explain","ads"].includes(draft.mode)?draft.mode:"auto",semantic:true,operation:"plan",assets,preferredModelRef:videoCanvasFunctionConfig.defaultMultimodalModel}); + const requestId = await generationTasks.submit({...selectedNode,draftValues:{...selectedNode.draftValues,action:"SMART_EDIT_WORKFLOW",nodeType:"smart-edit"}},"smart-edit",params,edges.filter(edge => edge.target === selectedNode.id)); + setNodes(current => current.map(node => node.id === selectedNode.id ? {...node,draftValues:{...node.draftValues,smartEditDraft:{...draft,assets,preparationRequestId:requestId}}} : node)); + setSmartEditNodeId(selectedNode.id); + return requestId; + }} + /> + ) : null} + {selectedNode && selectedTargetModuleKind === "text" && selectedTargetAnchor ? ( edge.target===smartEditNode.id).map(edge=>edge.source)} sources={nodes.filter(node=>node.inputType==="video"&&getNodeReferenceMediaUrl(node)).map(node=>({id:node.id,name:node.title,url:getNodeReferenceMediaUrl(node),subtitles:[]}))} audios={nodes.filter(node=>node.inputType==="audio"&&getNodeReferenceMediaUrl(node)).map(node=>({name:node.title,url:getNodeReferenceMediaUrl(node)}))} tasks={generationTasks.tasks.filter(task=>task.request.kind==="smart-edit"&&(task.request.node.id===smartEditNode.id||task.request.params.sourceNodeId===smartEditNode.id))} onClose={()=>setSmartEditNodeId(null)} onDraft={smartEditDraft=>setNodes(current=>current.map(node=>node.id===smartEditNode.id?{...node,draftValues:{...node.draftValues,smartEditDraft}}:node))} - onPlan={params=>generationTasks.submit({...smartEditNode,draftValues:{action:"SMART_EDIT_WORKFLOW",nodeType:"smart-edit"}},"smart-edit",params,edges.filter(edge=>edge.target===smartEditNode.id))} + onPlan={params=>generationTasks.submit({...smartEditNode,draftValues:{...smartEditNode.draftValues,action:"SMART_EDIT_WORKFLOW",nodeType:"smart-edit"}},"smart-edit",params,edges.filter(edge=>edge.target===smartEditNode.id))} onRender={async plan=>{ const result=createTargetModuleNode("video",smartEditNode.x+550,smartEditNode.y+generationTasks.tasks.filter(task=>task.request.params.sourceNodeId===smartEditNode.id).length*80,nodes.length+1); result.title=`${plan.title} · 成片`; - await generationTasks.submit(result,"smart-edit",{operation:"render",plan,sourceNodeId:smartEditNode.id},[{id:makeId("edge"),source:smartEditNode.id,target:result.id}]); + return generationTasks.submit(result,"smart-edit",{operation:"render",plan,sourceNodeId:smartEditNode.id},[{id:makeId("edge"),source:smartEditNode.id,target:result.id}]); }} />:null} {directorConsoleNode ? ( @@ -18298,6 +18341,7 @@ function CloneCanvasNode({ onUpdateLocalEditRegions, onTextResourceChange, selected, + smartEditPreviewUrl, }: { connectionsHidden: boolean; connectionHoverActive: boolean; @@ -18305,6 +18349,7 @@ function CloneCanvasNode({ onClearGridSplitSelection: () => void; onClearLocalEditRegions: () => void; node: CloneNode; + smartEditPreviewUrl?: string; onCancelGeneration: () => void; onContextMenu: (event: ReactMouseEvent) => void; onExtractVideoFrame: (kind: VideoFrameExtractionKind,time?:number) => void; @@ -18360,7 +18405,7 @@ function CloneCanvasNode({ if (targetModuleKind) { return ( - + ); } @@ -18598,12 +18643,15 @@ function TargetModuleCanvasNode({ node, onAction, selected, + previewUrl, }: { kind: AddModuleKind; + previewUrl?: string; node: CloneNode; onAction: (actionLabel: TargetModuleActionLabel) => void; selected: boolean; }) { + if (kind === "smart-edit") return onAction("打开智能剪辑")} />; const HeaderIcon = kind === "video-clip" ? Scissors : kind === "director" ? Settings2 : kind === "audio" ? Music2 : kind === "video" ? Video : kind === "image" ? ImageIcon : FileText; const actions: Record> = { "smart-edit": [{icon:Scissors,label:"打开智能剪辑"}], @@ -18639,7 +18687,6 @@ function TargetModuleCanvasNode({ "video-clip": [], }; const isCompactTargetModule = - kind === "smart-edit" || kind === "text" || kind === "image" || kind === "video" || @@ -18686,7 +18733,7 @@ function TargetModuleCanvasNode({ )} - {kind === "smart-edit" ?

自动选片 · 音乐卡点 · 字幕 · 多素材时间轴

: kind === "video-clip" ? ( + {kind === "video-clip" ? (
空空如也，请连接视频节点后操作
diff --git a/components/canvas/smart-edit/EditPreview.tsx b/components/canvas/smart-edit/EditPreview.tsx index b74c757e1..3f07b72b0 100644 --- a/components/canvas/smart-edit/EditPreview.tsx +++ b/components/canvas/smart-edit/EditPreview.tsx @@ -1,7 +1,9 @@ "use client"; -import { useEffect, useRef } from "react"; +import { useEffect, useRef, useState } from "react"; import { compiledSubtitles, + defaultCaptionStyle, + type CaptionStyle, planTimeline, SECOND, type EditPlan, @@ -13,7 +15,9 @@ export function EditPreview({ onTime, onPlaying, onError, + fit = false, }: { + fit?: boolean; plan: EditPlan; time: number; playing: boolean; @@ -25,8 +29,20 @@ export function EditPreview({ music = useRef(null), latest = useRef({ time, plan, onTime, onPlaying }); latest.current = { time, plan, onTime, onPlaying }; + const frameRef = useRef(null); + const [frameSize, setFrameSize] = useState({ width: 0, height: 0 }); + useEffect(() => { + const el = frameRef.current; + if (!el) return; + const observer = new ResizeObserver(([entry]) => + setFrameSize({ width: entry.contentRect.width, height: entry.contentRect.height }), + ); + observer.observe(el); + return () => observer.disconnect(); + }, []); const timeline = planTimeline(plan), - current = timeline.clips.filter((c) => time * SECOND >= c.start && time * SECOND < c.end), + visibleTime = Math.min(time * SECOND, timeline.duration - 1), + current = timeline.clips.filter((c) => visibleTime >= c.start && visibleTime < c.end), subs = compiledSubtitles(plan).filter((s) => time * SECOND >= s.start && time * SECOND < s.end); useEffect(() => { if (!playing) return; @@ -103,46 +119,112 @@ export function EditPreview({ else audio.pause(); } }, [time, playing, plan]); + const [ratioWidth, ratioHeight] = plan.ratio.split(":").map(Number); + const ratio = ratioWidth / ratioHeight; + const height = Math.min(frameSize.height || 720, (frameSize.width || 1280) / ratio); + const scale = height / 720; + const incomingClip = current.find((c) => c.overlap && visibleTime < c.start + c.overlap); + const progress = incomingClip ? (visibleTime - incomingClip.start) / incomingClip.overlap : 1; return (
- {timeline.clips.map((clip) => { - const asset = plan.assets.find((a) => a.id === clip.assetId)!; - const active = current.some((c) => c.id === clip.id), - incoming = clip.overlap && time * SECOND < clip.start + clip.overlap; - return ( -