From 85d678eaf4f15e55dc13d2505bea280c14b21258 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 24 Sep 2026 16:06:16 +0800 Subject: [PATCH] Connect video canvas media toolbar actions to tasks --- app/api/canvas/generation-tasks/route.ts | 4 + .../canvas/LiblibVideoCanvasReplica.tsx | 387 ++++++++++++------ .../video-canvas-toolbar-parity-2026-09-24.md | 21 + lib/video-canvas/aliyun-video-edit.ts | 122 ++++++ lib/video-canvas/layer-split-contract.ts | 6 + lib/video-canvas/task-contract.ts | 29 +- lib/video-canvas/task-recovery.ts | 11 + lib/video-canvas/task-worker.ts | 16 + lib/video-canvas/video-edit-contract.ts | 31 ++ lib/video-canvas/video-edit-mask.ts | 18 + lib/video-canvas/video-edit-request.ts | 24 ++ playwright.helpers.config.ts | 1 + .../routes/video-canvas-toolbar-jobs.test.ts | 98 +++++ 13 files changed, 648 insertions(+), 120 deletions(-) create mode 100644 docs/video-canvas-toolbar-parity-2026-09-24.md create mode 100644 lib/video-canvas/aliyun-video-edit.ts create mode 100644 lib/video-canvas/layer-split-contract.ts create mode 100644 lib/video-canvas/video-edit-contract.ts create mode 100644 lib/video-canvas/video-edit-mask.ts create mode 100644 lib/video-canvas/video-edit-request.ts create mode 100644 tests/routes/video-canvas-toolbar-jobs.test.ts diff --git a/app/api/canvas/generation-tasks/route.ts b/app/api/canvas/generation-tasks/route.ts index 588f6e8ee..dd08b4bcc 100644 --- a/app/api/canvas/generation-tasks/route.ts +++ b/app/api/canvas/generation-tasks/route.ts @@ -1,4 +1,6 @@ import { videoUpscaleSchema } from "@/lib/video-canvas/video-upscale-contract"; +import { videoEditRequestSchema } from "@/lib/video-canvas/video-edit-contract"; +import { canvasLayerSplitSchema } from "@/lib/video-canvas/layer-split-contract"; import { smartEditRequestSchema } from "@/lib/video-canvas/smart-edit/contract"; import { audioRequestSchema } from "@/lib/video-canvas/audio-contract"; import { canResumeCanvasTask } from "@/lib/video-canvas/task-recovery"; @@ -64,6 +66,8 @@ export async function POST(request: Request) { ) throw Object.assign(Error("剪辑参数无效"), { status: 400 }); if (parsed.data.kind === "video-upscale" && !videoUpscaleSchema.safeParse(parsed.data.params).success) throw Object.assign(Error("视频高清参数无效"), {status:400}); + if (parsed.data.kind === "video-edit" && !videoEditRequestSchema.safeParse(parsed.data.params).success) throw Object.assign(Error("视频编辑参数无效"), {status:400}); + if (parsed.data.kind === "image-layer-split" && !canvasLayerSplitSchema.safeParse(parsed.data.params).success) throw Object.assign(Error("图层分离参数无效"), {status:400}); const userId = await owner(parsed.data.projectId); const task = await createCanvasGenerationTask(userId, parsed.data); kick(userId, parsed.data.projectId); diff --git a/components/canvas/LiblibVideoCanvasReplica.tsx b/components/canvas/LiblibVideoCanvasReplica.tsx index 914c08966..56d8c48af 100644 --- a/components/canvas/LiblibVideoCanvasReplica.tsx +++ b/components/canvas/LiblibVideoCanvasReplica.tsx @@ -202,8 +202,8 @@ type ComposerMenu = | "slash" | null; type MemberStoreTab = "vip" | "history" | "credits"; -type VideoToolbarMode = "clip" | "crop" | "subtitleErase" | "analysis" | null; -type VideoToolbarMenu = "hd" | "subtitle" | "audio" | null; +type VideoToolbarMode = "clip" | "crop" | "subtitleErase" | "subjectErase" | "analysis" | null; +type VideoToolbarMenu = "hd" | "subtitle" | "audio" | "subject" | "more" | null; type VideoFrameExtractionKind = "first" | "last" | "current"; type SubtitleEraseMode = "smart" | "region"; type VideoAudioSplitMode = "voice" | "background" | "audio-video"; @@ -495,6 +495,7 @@ type TargetModuleActionLabel = | "拉片音乐"; type ImageToolbarAction = + | "portrait" | "panorama" | "orbit" | "lighting" @@ -502,6 +503,7 @@ type ImageToolbarAction = | "hd" | "crop" | "localEdit" + | "layerSplit" | "annotate" | "rotate" | "download" @@ -4529,6 +4531,10 @@ export function LiblibVideoCanvasReplica({ "/api/site-settings?scope=branding", fetcher ); + const { data: canvasFeatureSettings } = useSWR<{ layerSplitEnabled?: boolean }>( + "/api/site-settings?scope=agent-canvas", + fetcher + ); const { data: videoCanvasFunctionSettings } = useSWR<{ config?: VideoCanvasFunctionConfig }>( "/api/video-canvas/config", fetcher @@ -4623,6 +4629,9 @@ export function LiblibVideoCanvasReplica({ const [nodeContextMenu, setNodeContextMenu] = useState(null); const [connectionMenu, setConnectionMenu] = useState(null); const [previewNode, setPreviewNode] = useState(null); + const [layerSplitBusyNodeId, setLayerSplitBusyNodeId] = useState(null); + const [repaintClipNodeId, setRepaintClipNodeId] = useState(null); + const [videoEditPrompt, setVideoEditPrompt] = useState<{ nodeId: string; operation: "repaint" | "intro"; prompt: string } | null>(null); const [cancelGenerationConfirmNodeId, setCancelGenerationConfirmNodeId] = useState(null); const [storyWorkflowNodeId, setStoryWorkflowNodeId] = useState(null); const [storyWorkflowStep, setStoryWorkflowStep] = useState("shots"); @@ -6591,6 +6600,41 @@ export function LiblibVideoCanvasReplica({ return; } + if (action === "layerSplit") { + if (layerSplitBusyNodeId) return; + if (!canvasFeatureSettings) { + toast.info("正在读取图层分离配置,请稍后重试"); + return; + } + if (!canvasFeatureSettings?.layerSplitEnabled) { + toast.error("图层分离模型尚未启用,请先在系统设置中配置并开启"); + return; + } + if (!anchorSourceUrl) { + toast.error("请先上传图片,再拆分图层"); + return; + } + setLayerSplitBusyNodeId(anchor.id); + try { + const created = createMediaResourceNode({ + action: "IMAGE_RESOURCE", mediaType: "image", mediaUrl: "", + title: `${anchor.title} · 图层分离`, + width: anchor.width, height: anchor.height, + x: Math.round(anchor.x + anchor.width + OFFICIAL_NODE_GAP), y: anchor.y, + }); + created.draftValues = { ...created.draftValues, sourceNodeId: anchor.id, generationStatus: "pending", pendingLabel: "图层分离中" }; + const edge = { id: makeId("edge"), source: anchor.id, target: created.id }; + addNodes([created], [edge]); + setSelectedNodeId(created.id); + await generationTasks.submit(created, "image-layer-split", { imageUrl: anchorSourceUrl, sourceNodeId: anchor.id }, [edge]); + } catch (error) { + toast.error(error instanceof Error ? error.message : "图层分离失败,原图已保留"); + } finally { + setLayerSplitBusyNodeId(null); + } + return; + } + if (action === "orbit") { setSelectedNodeId(anchor.id); setAngleEditorNodeId(anchor.id); @@ -6667,7 +6711,7 @@ export function LiblibVideoCanvasReplica({ const labels: Record< Exclude< ImageToolbarAction, - "annotate" | "rotate" | "download" | "preview" | "crop" | "orbit" | "localEdit" + "annotate" | "rotate" | "download" | "preview" | "crop" | "orbit" | "localEdit" | "layerSplit" >, string > = { @@ -6675,6 +6719,7 @@ export function LiblibVideoCanvasReplica({ hd: "高清", lighting: "打光", panorama: "720 全景", + portrait: "人像质感调节", }; const title = labels[action]; if (action === "grid") { @@ -6853,7 +6898,9 @@ export function LiblibVideoCanvasReplica({ } const functionKey = - action === "lighting" + action === "portrait" + ? "image-repaint" + : action === "lighting" ? "lighting" : action === "panorama" ? "panorama-720" @@ -6869,7 +6916,13 @@ export function LiblibVideoCanvasReplica({ ? "remove-bg" : "hd-upscale"; const definition = getConfiguredFunction(videoCanvasFunctionConfig, functionKey); - const prompt = composeFunctionPrompt(definition, option?.trim() || title); + if (!videoCanvasFunctionConfig.enabled || !definition?.enabled) { + toast.error(`${title}尚未配置可用模型`); + return; + } + const prompt = action === "portrait" + ? "以输入照片为唯一参考,保持人物身份、五官比例、发型、肤色、衣着、姿势和原始构图不变;自然修复皮肤细节、面部光影与发丝质感,抑制塑料磨皮和过度锐化,不新增人物或物体,不改变背景和画面比例。" + : composeFunctionPrompt(definition, option?.trim() || title); if (videoCanvasFunctionConfig.enabled && definition?.enabled) { const pendingResultNode = createMediaResourceNode({ @@ -6976,40 +7029,9 @@ export function LiblibVideoCanvasReplica({ return; } - const created = createTargetModuleNode("image", rightX, 0, nodes.length + 1); - const resultNode: CloneNode = { - ...created, - title, - y: centerY(created.height), - draftValues: { - ...(created.draftValues || {}), - generatorType: action === "hd" ? "ENHANCE" : "DEFAULT", - params: { - mode: action, - prompt, - referenceImages: [sourceUrl, ...extraReferenceImages], - sourceNodeId: anchor.id, - sourceUrl, - }, - sourceNodeId: anchor.id, - sourceUrl, - toolbarOption: option || "", - }, - }; - - setNodes((current) => [...current, resultNode]); - setEdges((current) => [ - ...current, - { id: makeId("edge"), source: anchor.id, target: resultNode.id }, - ]); - setSelectedNodeId(anchor.id); - setCanvasCommand(null); - setNodeContextMenu(null); - setConnectionMenu(null); - setActivePanel(null); - toast.success(`已创建「${title}」工作流节点`); + toast.error(`${title}尚未配置可执行的处理接口`); }, - [nodes, videoCanvasFunctionConfig, generationTasks.submit], + [addNodes, canvasFeatureSettings?.layerSplitEnabled, layerSplitBusyNodeId, nodes, videoCanvasFunctionConfig, generationTasks.submit], ); const updateTextResourceContent = useCallback((nodeId: string, content: string) => { @@ -8272,6 +8294,7 @@ export function LiblibVideoCanvasReplica({ setSelectedNodeId(clipNode.id); setActiveVideoToolbarMode(null); toast.success("视频剪辑完成,已生成新视频"); + return clipNode; } catch (error) { toast.error(error instanceof Error ? error.message : "视频剪辑失败,原视频已保留"); } finally { @@ -8385,10 +8408,12 @@ export function LiblibVideoCanvasReplica({ ); const createVideoHdUpscaleFromVideo = useCallback( - (nodeId: string, option: string) => { + async (nodeId: string, option: string) => { const videoNode = nodes.find((item) => item.id === nodeId); if (!videoNode) return; const resolution = normalizeVideoHdResolutionOption(option); + const url = getNodeMediaUrl(videoNode); + if (!url) { toast.error("请先上传视频"); return; } const existingHdNode = nodes.find( (item) => getDraftString(item, "action") === "VIDEO_HD_UPSCALE" && @@ -8400,17 +8425,19 @@ export function LiblibVideoCanvasReplica({ option, sourceNode: videoNode, }); + const edge = { id: makeId("edge"), source: videoNode.id, target: hdNode.id }; if (existingHdNode) { setNodes((current) => current.map((item) => (item.id === hdNode.id ? hdNode : item))); } else { - addNodes([hdNode], [{ id: makeId("edge"), source: videoNode.id, target: hdNode.id }]); + addNodes([hdNode], [edge]); } setSelectedNodeId(hdNode.id); setSelectedEdgeId(null); setActiveVideoToolbarMode(null); + await generationTasks.submit(hdNode, "video-upscale", { url, resolution }, [edge]); }, - [addNodes, nodes] + [addNodes, nodes, generationTasks.submit] ); const generateVideoHdUpscaleNode = useCallback(async (nodeId: string) => { @@ -8504,9 +8531,62 @@ export function LiblibVideoCanvasReplica({ [subtitleEraseRegionHistory, subtitleEraseRegions] ); - const applySubtitleEraseSelection = useCallback((_nodeId: string, _mode: SubtitleEraseMode = subtitleEraseMode) => { - toast.error("当前没有已接通的视频字幕擦除模型,暂不能处理。所选区域会保留。"); - },[subtitleEraseMode]); + const submitVideoEdit = useCallback(async (nodeId: string, operation: "repaint" | "erase" | "subtitle" | "intro", regions?: SubtitleEraseRegion[], prompt?: string) => { + const source = nodes.find((item) => item.id === nodeId); + if (!source) return; + const sourceUrl = getNodePlayableVideoUrl(source); + if (!sourceUrl) { toast.error("请先上传视频"); return; } + const dimensions = getDraftString(source, "outputSizeLabel").match(/(\d+)\s*[×x]\s*(\d+)/); + const videoMeta = source.draftValues?.videoMeta as { width?: number; height?: number } | undefined; + const width = Number(videoMeta?.width || dimensions?.[1]); + const height = Number(videoMeta?.height || dimensions?.[2]); + if (!(width > 0 && height > 0)) { toast.error("无法读取视频原始尺寸,请重新上传视频"); return; } + const durationSeconds = getVideoNodeDurationSeconds(source); + if (durationSeconds > (operation === "repaint" ? 10 : 5)) { toast.error(operation === "repaint" ? "片段重拍最多处理 10 秒,请先截取目标片段" : "该视频编辑模型最多处理 5 秒,请先截取目标片段"); return; } + if (operation === "intro" && durationSeconds > 3) { toast.error("请先将尾段剪辑至 3 秒以内,再生成创意片头"); return; } + if (operation === "erase" && !regions?.length) { toast.error("请先框选要消除的主体"); return; } + if ((operation === "repaint" || operation === "intro") && !prompt?.trim()) { + toast.error("请先填写视频编辑提示词"); + return; + } + const titles = { repaint: "片段重拍", erase: "主体消除", subtitle: "智能去字幕", intro: "创意片头" }; + const node = createMediaResourceNode({ + action: "VIDEO_RESOURCE", mediaType: "video", mediaUrl: "", + title: `${source.title} · ${titles[operation]}`, + width: source.width, height: source.height, + x: source.x + source.width + OFFICIAL_NODE_GAP, y: source.y, + }); + node.draftValues = { ...node.draftValues, generated: false, generationStatus: "pending", sourceNodeId: source.id, outputSizeLabel: `${width} × ${height}`, durationSeconds }; + const edge = { id: makeId("edge"), source: source.id, target: node.id }; + addNodes([node], [edge]); + setSelectedNodeId(node.id); + setActiveVideoToolbarMode(null); + await generationTasks.submit(node, "video-edit", { + operation, sourceUrl, sourceNodeId: source.id, prompt: prompt?.trim() || undefined, + durationSeconds, width, height, + regions: regions?.map(({ x, y, width, height }) => ({ x, y, width, height })), + }, [edge]); + }, [addNodes, generationTasks.submit, nodes]); + + const applyVideoRepaintSelection = useCallback(async (nodeId: string) => { + const source = nodes.find((item) => item.id === nodeId); + if (!source) return; + const range = normalizeVideoClipRange(videoClipRanges[nodeId] || source.draftValues?.clipRange); + if (getVideoNodeDurationSeconds(source) * (range.end - range.start) > 10) { + toast.error("请选择不超过 10 秒的重拍片段"); + return; + } + const clip = await applyVideoClipSelection(nodeId); + if (!clip) return; + setRepaintClipNodeId(null); + setVideoEditPrompt({ nodeId: clip.id, operation: "repaint", prompt: "保持原片人物和动作,提升镜头质感" }); + }, [applyVideoClipSelection, nodes, videoClipRanges]); + + const applySubtitleEraseSelection = useCallback((nodeId: string, mode: SubtitleEraseMode = subtitleEraseMode) => { + const regions = mode === "region" ? subtitleEraseRegions[nodeId] : undefined; + if (mode === "region" && !regions?.length) { toast.error("请先框选字幕区域"); return; } + void submitVideoEdit(nodeId, "subtitle", regions); + }, [subtitleEraseMode, subtitleEraseRegions, submitVideoEdit]); const splitVideoAudioTrack = useCallback( async (nodeId: string, mode: VideoAudioSplitMode) => { @@ -10967,6 +11047,23 @@ export function LiblibVideoCanvasReplica({ void requestVideoAnalysis(selectedVideoMediaNode.id); }} onAudioVideoSplit={() => splitVideoAudioTrack(selectedVideoMediaNode.id, "audio-video")} + onDownload={() => void downloadVideoNode(selectedVideoMediaNode)} + onPreview={() => setPreviewNode(selectedVideoMediaNode)} + onRepaint={() => { + if (getVideoNodeDurationSeconds(selectedVideoMediaNode) > 10) { + setRepaintClipNodeId(selectedVideoMediaNode.id); + setActiveVideoToolbarMode("clip"); + toast.info("请选择不超过 10 秒的重拍片段"); + } else setVideoEditPrompt({ nodeId: selectedVideoMediaNode.id, operation: "repaint", prompt: "保持原片人物和动作,提升镜头质感" }); + }} + onIntro={() => { + if (getVideoNodeDurationSeconds(selectedVideoMediaNode) > 3) { + toast.error("请先截取不超过 3 秒的视频尾段,再生成创意片头"); + return; + } + setVideoEditPrompt({ nodeId: selectedVideoMediaNode.id, operation: "intro", prompt: "生成与尾段自然衔接的创意片头" }); + }} + onSubjectErase={() => setActiveVideoToolbarMode((current) => current === "subjectErase" ? null : "subjectErase")} onSlow={factor=>void slowVideoNode(selectedVideoMediaNode.id,factor)} onHdOption={(option) => createVideoHdUpscaleFromVideo(selectedVideoMediaNode.id, option)} onSubtitleErase={(mode) => { @@ -10974,7 +11071,7 @@ export function LiblibVideoCanvasReplica({ setActiveVideoToolbarMode("subtitleErase"); }} onToggleCrop={() => setActiveVideoToolbarMode((current) => current === "crop" ? null : "crop")} - onToggleClip={() => setActiveVideoToolbarMode((current) => current === "clip" ? null : "clip")} + onToggleClip={() => { setRepaintClipNodeId(null); setActiveVideoToolbarMode((current) => current === "clip" ? null : "clip"); }} onVoiceSplit={(mode) => splitVideoAudioTrack(selectedVideoMediaNode.id, mode)} /> {activeVideoToolbarMode === "crop" ? ( @@ -11002,13 +11099,12 @@ export function LiblibVideoCanvasReplica({ actionLabel="智能擦除" anchor={selectedVideoAnchor} applyLabel="生成擦除" - costLabel="0" onApply={() => applySubtitleEraseSelection(selectedVideoMediaNode.id, "smart")} onClose={() => setActiveVideoToolbarMode(null)} title="关闭去字幕生成器" /> ) : null} - {activeVideoToolbarMode === "subtitleErase" && subtitleEraseMode === "region" ? ( + {(activeVideoToolbarMode === "subtitleErase" && subtitleEraseMode === "region") || activeVideoToolbarMode === "subjectErase" ? ( <> 0} canReset={selectedSubtitleEraseRegions.length > 0} canUndo={selectedSubtitleEraseHistory.past.length > 0} - costLabel={selectedSubtitleEraseRegions.length > 0 ? "0" : "-"} - onApply={() => applySubtitleEraseSelection(selectedVideoMediaNode.id, "region")} + costLabel="-" + onApply={() => activeVideoToolbarMode === "subjectErase" + ? void submitVideoEdit(selectedVideoMediaNode.id, "erase", selectedSubtitleEraseRegions) + : applySubtitleEraseSelection(selectedVideoMediaNode.id, "region")} onClose={() => setActiveVideoToolbarMode(null)} onRedo={() => redoSubtitleEraseRegionChange(selectedVideoMediaNode.id)} onReset={() => commitSubtitleEraseRegions(selectedVideoMediaNode.id, [])} onUndo={() => undoSubtitleEraseRegionChange(selectedVideoMediaNode.id)} regionCount={selectedSubtitleEraseRegions.length} + titleLabel={activeVideoToolbarMode === "subjectErase" ? "主体消除" : "框选擦除"} /> ) : null} @@ -11045,12 +11144,28 @@ export function LiblibVideoCanvasReplica({ durationSeconds={getVideoNodeDurationSeconds(selectedVideoMediaNode)} previewUrl={getVideoNodePreviewUrl(selectedVideoMediaNode) || getNodeMediaUrl(selectedVideoMediaNode)} title={selectedVideoMediaNode.title} - onApply={() => applyVideoClipSelection(selectedVideoMediaNode.id)} + onApply={() => repaintClipNodeId === selectedVideoMediaNode.id + ? void applyVideoRepaintSelection(selectedVideoMediaNode.id) + : void applyVideoClipSelection(selectedVideoMediaNode.id)} + applyLabel={repaintClipNodeId === selectedVideoMediaNode.id ? "重拍所选片段" : "导出所选片段"} onChange={(range) => updateVideoClipRange(selectedVideoMediaNode.id, range)} onExtractAudio={() => splitVideoAudioTrack(selectedVideoMediaNode.id, "audio-video")} onReset={() => updateVideoClipRange(selectedVideoMediaNode.id, { start: 0.18, end: 0.52 })} /> ) : null} + {videoEditPrompt?.nodeId === selectedVideoMediaNode.id ? ( + setVideoEditPrompt((current) => current ? { ...current, prompt: value } : null)} + onClose={() => setVideoEditPrompt(null)} + onSubmit={() => { + void submitVideoEdit(selectedVideoMediaNode.id, videoEditPrompt.operation, undefined, videoEditPrompt.prompt); + setVideoEditPrompt(null); + }} + /> + ) : null} ) : null} @@ -13037,9 +13152,9 @@ function VideoMediaToolbar({ anchor ? { left: anchor.centerX, - top: anchor.topY - (24 * anchor.zoom + 10), - transform: `translateX(-50%) translateY(-100%) scale(${OFFICIAL_IMAGE_TOOLBAR_SCALE})`, - transformOrigin: "bottom center", + top: Math.max(68, anchor.topY - (24 * anchor.zoom + 10) - 50 * OFFICIAL_IMAGE_TOOLBAR_SCALE), + transform: `translateX(-50%) scale(${OFFICIAL_IMAGE_TOOLBAR_SCALE})`, + transformOrigin: "top center", } : { left: "50%", @@ -13125,6 +13240,48 @@ function VideoToolbarActionButton({ ); } +function VideoEditPromptPanel({ + anchor, + operation, + prompt, + onChange, + onClose, + onSubmit, +}: { + anchor: PresetControlAnchor; + operation: "repaint" | "intro"; + prompt: string; + onChange: (value: string) => void; + onClose: () => void; + onSubmit: () => void; +}) { + const viewportWidth = typeof window === "undefined" ? 1280 : window.innerWidth; + const viewportHeight = typeof window === "undefined" ? 800 : window.innerHeight; + return ( +
+
+ {operation === "repaint" ? "片段重拍" : "创意片头"} + +
+