lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Connect video canvas media toolbar actions to tasks

Browse Source
This commit is contained in:
134 2026-09-24 16:06:16 +08:00
parent 1c2acee547
commit 85d678eaf4
13 changed files with 648 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/api/canvas/generation-tasks/route.ts
View File

@ -1,4 +1,6 @@
import { videoUpscaleSchema } from "@/lib/video-canvas/video-upscale-contract";
import { videoEditRequestSchema } from "@/lib/video-canvas/video-edit-contract";
import { canvasLayerSplitSchema } from "@/lib/video-canvas/layer-split-contract";
import { smartEditRequestSchema } from "@/lib/video-canvas/smart-edit/contract";
import { audioRequestSchema } from "@/lib/video-canvas/audio-contract";
import { canResumeCanvasTask } from "@/lib/video-canvas/task-recovery";
@ -64,6 +66,8 @@ export async function POST(request: Request) {
)
throw Object.assign(Error("剪辑参数无效"), { status: 400 });
if (parsed.data.kind === "video-upscale" && !videoUpscaleSchema.safeParse(parsed.data.params).success) throw Object.assign(Error("视频高清参数无效"), {status:400});
if (parsed.data.kind === "video-edit" && !videoEditRequestSchema.safeParse(parsed.data.params).success) throw Object.assign(Error("视频编辑参数无效"), {status:400});
if (parsed.data.kind === "image-layer-split" && !canvasLayerSplitSchema.safeParse(parsed.data.params).success) throw Object.assign(Error("图层分离参数无效"), {status:400});
const userId = await owner(parsed.data.projectId);
const task = await createCanvasGenerationTask(userId, parsed.data);
kick(userId, parsed.data.projectId);

387
components/canvas/LiblibVideoCanvasReplica.tsx
View File

@ -202,8 +202,8 @@ type ComposerMenu =
| "slash"
| null;
type MemberStoreTab = "vip" | "history" | "credits";
type VideoToolbarMode = "clip" | "crop" | "subtitleErase" | "analysis" | null;
type VideoToolbarMenu = "hd" | "subtitle" | "audio" | null;
type VideoToolbarMode = "clip" | "crop" | "subtitleErase" | "subjectErase" | "analysis" | null;
type VideoToolbarMenu = "hd" | "subtitle" | "audio" | "subject" | "more" | null;
type VideoFrameExtractionKind = "first" | "last" | "current";
type SubtitleEraseMode = "smart" | "region";
type VideoAudioSplitMode = "voice" | "background" | "audio-video";
@ -495,6 +495,7 @@ type TargetModuleActionLabel =
| "拉片音乐";
type ImageToolbarAction =
| "portrait"
| "panorama"
| "orbit"
| "lighting"
@ -502,6 +503,7 @@ type ImageToolbarAction =
| "hd"
| "crop"
| "localEdit"
| "layerSplit"
| "annotate"
| "rotate"
| "download"
@ -4529,6 +4531,10 @@ export function LiblibVideoCanvasReplica({
"/api/site-settings?scope=branding",
fetcher
);
const { data: canvasFeatureSettings } = useSWR<{ layerSplitEnabled?: boolean }>(
"/api/site-settings?scope=agent-canvas",
fetcher
);
const { data: videoCanvasFunctionSettings } = useSWR<{ config?: VideoCanvasFunctionConfig }>(
"/api/video-canvas/config",
fetcher
@ -4623,6 +4629,9 @@ export function LiblibVideoCanvasReplica({
const [nodeContextMenu, setNodeContextMenu] = useState<NodeContextState>(null);
const [connectionMenu, setConnectionMenu] = useState<ConnectionMenuState>(null);
const [previewNode, setPreviewNode] = useState<CloneNode | null>(null);
const [layerSplitBusyNodeId, setLayerSplitBusyNodeId] = useState<string | null>(null);
const [repaintClipNodeId, setRepaintClipNodeId] = useState<string | null>(null);
const [videoEditPrompt, setVideoEditPrompt] = useState<{ nodeId: string; operation: "repaint" | "intro"; prompt: string } | null>(null);
const [cancelGenerationConfirmNodeId, setCancelGenerationConfirmNodeId] = useState<string | null>(null);
const [storyWorkflowNodeId, setStoryWorkflowNodeId] = useState<string | null>(null);
const [storyWorkflowStep, setStoryWorkflowStep] = useState<StoryWorkflowStep>("shots");
@ -6591,6 +6600,41 @@ export function LiblibVideoCanvasReplica({
return;
}
if (action === "layerSplit") {
if (layerSplitBusyNodeId) return;
if (!canvasFeatureSettings) {
toast.info("正在读取图层分离配置，请稍后重试");
return;
}
if (!canvasFeatureSettings?.layerSplitEnabled) {
toast.error("图层分离模型尚未启用，请先在系统设置中配置并开启");
return;
}
if (!anchorSourceUrl) {
toast.error("请先上传图片，再拆分图层");
return;
}
setLayerSplitBusyNodeId(anchor.id);
try {
const created = createMediaResourceNode({
action: "IMAGE_RESOURCE", mediaType: "image", mediaUrl: "",
title: `${anchor.title} · 图层分离`,
width: anchor.width, height: anchor.height,
x: Math.round(anchor.x + anchor.width + OFFICIAL_NODE_GAP), y: anchor.y,
});
created.draftValues = { ...created.draftValues, sourceNodeId: anchor.id, generationStatus: "pending", pendingLabel: "图层分离中" };
const edge = { id: makeId("edge"), source: anchor.id, target: created.id };
addNodes([created], [edge]);
setSelectedNodeId(created.id);
await generationTasks.submit(created, "image-layer-split", { imageUrl: anchorSourceUrl, sourceNodeId: anchor.id }, [edge]);
} catch (error) {
toast.error(error instanceof Error ? error.message : "图层分离失败，原图已保留");
} finally {
setLayerSplitBusyNodeId(null);
}
return;
}
if (action === "orbit") {
setSelectedNodeId(anchor.id);
setAngleEditorNodeId(anchor.id);
@ -6667,7 +6711,7 @@ export function LiblibVideoCanvasReplica({
const labels: Record<
Exclude<
ImageToolbarAction,
"annotate" | "rotate" | "download" | "preview" | "crop" | "orbit" | "localEdit"
"annotate" | "rotate" | "download" | "preview" | "crop" | "orbit" | "localEdit" | "layerSplit"
>,
string
> = {
@ -6675,6 +6719,7 @@ export function LiblibVideoCanvasReplica({
hd: "高清",
lighting: "打光",
panorama: "720 全景",
portrait: "人像质感调节",
};
const title = labels[action];
if (action === "grid") {
@ -6853,7 +6898,9 @@ export function LiblibVideoCanvasReplica({
}
const functionKey =
action === "lighting"
action === "portrait"
? "image-repaint"
: action === "lighting"
? "lighting"
: action === "panorama"
? "panorama-720"
@ -6869,7 +6916,13 @@ export function LiblibVideoCanvasReplica({
? "remove-bg"
: "hd-upscale";
const definition = getConfiguredFunction(videoCanvasFunctionConfig, functionKey);
const prompt = composeFunctionPrompt(definition, option?.trim() || title);
if (!videoCanvasFunctionConfig.enabled || !definition?.enabled) {
toast.error(`${title}尚未配置可用模型`);
return;
}
const prompt = action === "portrait"
? "以输入照片为唯一参考，保持人物身份、五官比例、发型、肤色、衣着、姿势和原始构图不变；自然修复皮肤细节、面部光影与发丝质感，抑制塑料磨皮和过度锐化，不新增人物或物体，不改变背景和画面比例。"
: composeFunctionPrompt(definition, option?.trim() || title);
if (videoCanvasFunctionConfig.enabled && definition?.enabled) {
const pendingResultNode = createMediaResourceNode({
@ -6976,40 +7029,9 @@ export function LiblibVideoCanvasReplica({
return;
}
const created = createTargetModuleNode("image", rightX, 0, nodes.length + 1);
const resultNode: CloneNode = {
...created,
title,
y: centerY(created.height),
draftValues: {
...(created.draftValues || {}),
generatorType: action === "hd" ? "ENHANCE" : "DEFAULT",
params: {
mode: action,
prompt,
referenceImages: [sourceUrl, ...extraReferenceImages],
sourceNodeId: anchor.id,
sourceUrl,
},
sourceNodeId: anchor.id,
sourceUrl,
toolbarOption: option || "",
},
};
setNodes((current) => [...current, resultNode]);
setEdges((current) => [
...current,
{ id: makeId("edge"), source: anchor.id, target: resultNode.id },
]);
setSelectedNodeId(anchor.id);
setCanvasCommand(null);
setNodeContextMenu(null);
setConnectionMenu(null);
setActivePanel(null);
toast.success(`已创建「${title}」工作流节点`);
toast.error(`${title}尚未配置可执行的处理接口`);
},
[nodes, videoCanvasFunctionConfig, generationTasks.submit],
[addNodes, canvasFeatureSettings?.layerSplitEnabled, layerSplitBusyNodeId, nodes, videoCanvasFunctionConfig, generationTasks.submit],
);
const updateTextResourceContent = useCallback((nodeId: string, content: string) => {
@ -8272,6 +8294,7 @@ export function LiblibVideoCanvasReplica({
setSelectedNodeId(clipNode.id);
setActiveVideoToolbarMode(null);
toast.success("视频剪辑完成，已生成新视频");
return clipNode;
} catch (error) {
toast.error(error instanceof Error ? error.message : "视频剪辑失败，原视频已保留");
} finally {
@ -8385,10 +8408,12 @@ export function LiblibVideoCanvasReplica({
);
const createVideoHdUpscaleFromVideo = useCallback(
(nodeId: string, option: string) => {
async (nodeId: string, option: string) => {
const videoNode = nodes.find((item) => item.id === nodeId);
if (!videoNode) return;
const resolution = normalizeVideoHdResolutionOption(option);
const url = getNodeMediaUrl(videoNode);
if (!url) { toast.error("请先上传视频"); return; }
const existingHdNode = nodes.find(
(item) =>
getDraftString(item, "action") === "VIDEO_HD_UPSCALE" &&
@ -8400,17 +8425,19 @@ export function LiblibVideoCanvasReplica({
option,
sourceNode: videoNode,
});
const edge = { id: makeId("edge"), source: videoNode.id, target: hdNode.id };
if (existingHdNode) {
setNodes((current) => current.map((item) => (item.id === hdNode.id ? hdNode : item)));
} else {
addNodes([hdNode], [{ id: makeId("edge"), source: videoNode.id, target: hdNode.id }]);
addNodes([hdNode], [edge]);
}
setSelectedNodeId(hdNode.id);
setSelectedEdgeId(null);
setActiveVideoToolbarMode(null);
await generationTasks.submit(hdNode, "video-upscale", { url, resolution }, [edge]);
},
[addNodes, nodes]
[addNodes, nodes, generationTasks.submit]
);
const generateVideoHdUpscaleNode = useCallback(async (nodeId: string) => {
@ -8504,9 +8531,62 @@ export function LiblibVideoCanvasReplica({
[subtitleEraseRegionHistory, subtitleEraseRegions]
);
const applySubtitleEraseSelection = useCallback((_nodeId: string, _mode: SubtitleEraseMode = subtitleEraseMode) => {
toast.error("当前没有已接通的视频字幕擦除模型，暂不能处理。所选区域会保留。");
},[subtitleEraseMode]);
const submitVideoEdit = useCallback(async (nodeId: string, operation: "repaint" | "erase" | "subtitle" | "intro", regions?: SubtitleEraseRegion[], prompt?: string) => {
const source = nodes.find((item) => item.id === nodeId);
if (!source) return;
const sourceUrl = getNodePlayableVideoUrl(source);
if (!sourceUrl) { toast.error("请先上传视频"); return; }
const dimensions = getDraftString(source, "outputSizeLabel").match(/(\d+)\s*[×x]\s*(\d+)/);
const videoMeta = source.draftValues?.videoMeta as { width?: number; height?: number } | undefined;
const width = Number(videoMeta?.width || dimensions?.[1]);
const height = Number(videoMeta?.height || dimensions?.[2]);
if (!(width > 0 && height > 0)) { toast.error("无法读取视频原始尺寸，请重新上传视频"); return; }
const durationSeconds = getVideoNodeDurationSeconds(source);
if (durationSeconds > (operation === "repaint" ? 10 : 5)) { toast.error(operation === "repaint" ? "片段重拍最多处理 10 秒，请先截取目标片段" : "该视频编辑模型最多处理 5 秒，请先截取目标片段"); return; }
if (operation === "intro" && durationSeconds > 3) { toast.error("请先将尾段剪辑至 3 秒以内，再生成创意片头"); return; }
if (operation === "erase" && !regions?.length) { toast.error("请先框选要消除的主体"); return; }
if ((operation === "repaint" || operation === "intro") && !prompt?.trim()) {
toast.error("请先填写视频编辑提示词");
return;
}
const titles = { repaint: "片段重拍", erase: "主体消除", subtitle: "智能去字幕", intro: "创意片头" };
const node = createMediaResourceNode({
action: "VIDEO_RESOURCE", mediaType: "video", mediaUrl: "",
title: `${source.title} · ${titles[operation]}`,
width: source.width, height: source.height,
x: source.x + source.width + OFFICIAL_NODE_GAP, y: source.y,
});
node.draftValues = { ...node.draftValues, generated: false, generationStatus: "pending", sourceNodeId: source.id, outputSizeLabel: `${width} × ${height}`, durationSeconds };
const edge = { id: makeId("edge"), source: source.id, target: node.id };
addNodes([node], [edge]);
setSelectedNodeId(node.id);
setActiveVideoToolbarMode(null);
await generationTasks.submit(node, "video-edit", {
operation, sourceUrl, sourceNodeId: source.id, prompt: prompt?.trim() || undefined,
durationSeconds, width, height,
regions: regions?.map(({ x, y, width, height }) => ({ x, y, width, height })),
}, [edge]);
}, [addNodes, generationTasks.submit, nodes]);
const applyVideoRepaintSelection = useCallback(async (nodeId: string) => {
const source = nodes.find((item) => item.id === nodeId);
if (!source) return;
const range = normalizeVideoClipRange(videoClipRanges[nodeId] || source.draftValues?.clipRange);
if (getVideoNodeDurationSeconds(source) * (range.end - range.start) > 10) {
toast.error("请选择不超过 10 秒的重拍片段");
return;
}
const clip = await applyVideoClipSelection(nodeId);
if (!clip) return;
setRepaintClipNodeId(null);
setVideoEditPrompt({ nodeId: clip.id, operation: "repaint", prompt: "保持原片人物和动作，提升镜头质感" });
}, [applyVideoClipSelection, nodes, videoClipRanges]);
const applySubtitleEraseSelection = useCallback((nodeId: string, mode: SubtitleEraseMode = subtitleEraseMode) => {
const regions = mode === "region" ? subtitleEraseRegions[nodeId] : undefined;
if (mode === "region" && !regions?.length) { toast.error("请先框选字幕区域"); return; }
void submitVideoEdit(nodeId, "subtitle", regions);
}, [subtitleEraseMode, subtitleEraseRegions, submitVideoEdit]);
const splitVideoAudioTrack = useCallback(
async (nodeId: string, mode: VideoAudioSplitMode) => {
@ -10967,6 +11047,23 @@ export function LiblibVideoCanvasReplica({
void requestVideoAnalysis(selectedVideoMediaNode.id);
}}
onAudioVideoSplit={() => splitVideoAudioTrack(selectedVideoMediaNode.id, "audio-video")}
onDownload={() => void downloadVideoNode(selectedVideoMediaNode)}
onPreview={() => setPreviewNode(selectedVideoMediaNode)}
onRepaint={() => {
if (getVideoNodeDurationSeconds(selectedVideoMediaNode) > 10) {
setRepaintClipNodeId(selectedVideoMediaNode.id);
setActiveVideoToolbarMode("clip");
toast.info("请选择不超过 10 秒的重拍片段");
} else setVideoEditPrompt({ nodeId: selectedVideoMediaNode.id, operation: "repaint", prompt: "保持原片人物和动作，提升镜头质感" });
}}
onIntro={() => {
if (getVideoNodeDurationSeconds(selectedVideoMediaNode) > 3) {
toast.error("请先截取不超过 3 秒的视频尾段，再生成创意片头");
return;
}
setVideoEditPrompt({ nodeId: selectedVideoMediaNode.id, operation: "intro", prompt: "生成与尾段自然衔接的创意片头" });
}}
onSubjectErase={() => setActiveVideoToolbarMode((current) => current === "subjectErase" ? null : "subjectErase")}
onSlow={factor=>void slowVideoNode(selectedVideoMediaNode.id,factor)}
onHdOption={(option) => createVideoHdUpscaleFromVideo(selectedVideoMediaNode.id, option)}
onSubtitleErase={(mode) => {
@ -10974,7 +11071,7 @@ export function LiblibVideoCanvasReplica({
setActiveVideoToolbarMode("subtitleErase");
}}
onToggleCrop={() => setActiveVideoToolbarMode((current) => current === "crop" ? null : "crop")}
onToggleClip={() => setActiveVideoToolbarMode((current) => current === "clip" ? null : "clip")}
onToggleClip={() => { setRepaintClipNodeId(null); setActiveVideoToolbarMode((current) => current === "clip" ? null : "clip"); }}
onVoiceSplit={(mode) => splitVideoAudioTrack(selectedVideoMediaNode.id, mode)}
/>
{activeVideoToolbarMode === "crop" ? (
@ -11002,13 +11099,12 @@ export function LiblibVideoCanvasReplica({
actionLabel="智能擦除"
anchor={selectedVideoAnchor}
applyLabel="生成擦除"
costLabel="0"
onApply={() => applySubtitleEraseSelection(selectedVideoMediaNode.id, "smart")}
onClose={() => setActiveVideoToolbarMode(null)}
title="关闭去字幕生成器"
/>
) : null}
{activeVideoToolbarMode === "subtitleErase" && subtitleEraseMode === "region" ? (
{(activeVideoToolbarMode === "subtitleErase" && subtitleEraseMode === "region") || activeVideoToolbarMode === "subjectErase" ? (
<>
<VideoSubtitleEraseRegionOverlay
anchor={selectedVideoAnchor}
@ -11023,13 +11119,16 @@ export function LiblibVideoCanvasReplica({
canRedo={selectedSubtitleEraseHistory.future.length > 0}
canReset={selectedSubtitleEraseRegions.length > 0}
canUndo={selectedSubtitleEraseHistory.past.length > 0}
costLabel={selectedSubtitleEraseRegions.length > 0 ? "0" : "-"}
onApply={() => applySubtitleEraseSelection(selectedVideoMediaNode.id, "region")}
costLabel="-"
onApply={() => activeVideoToolbarMode === "subjectErase"
? void submitVideoEdit(selectedVideoMediaNode.id, "erase", selectedSubtitleEraseRegions)
: applySubtitleEraseSelection(selectedVideoMediaNode.id, "region")}
onClose={() => setActiveVideoToolbarMode(null)}
onRedo={() => redoSubtitleEraseRegionChange(selectedVideoMediaNode.id)}
onReset={() => commitSubtitleEraseRegions(selectedVideoMediaNode.id, [])}
onUndo={() => undoSubtitleEraseRegionChange(selectedVideoMediaNode.id)}
regionCount={selectedSubtitleEraseRegions.length}
titleLabel={activeVideoToolbarMode === "subjectErase" ? "主体消除" : "框选擦除"}
/>
</>
) : null}
@ -11045,12 +11144,28 @@ export function LiblibVideoCanvasReplica({
durationSeconds={getVideoNodeDurationSeconds(selectedVideoMediaNode)}
previewUrl={getVideoNodePreviewUrl(selectedVideoMediaNode) || getNodeMediaUrl(selectedVideoMediaNode)}
title={selectedVideoMediaNode.title}
onApply={() => applyVideoClipSelection(selectedVideoMediaNode.id)}
onApply={() => repaintClipNodeId === selectedVideoMediaNode.id
? void applyVideoRepaintSelection(selectedVideoMediaNode.id)
: void applyVideoClipSelection(selectedVideoMediaNode.id)}
applyLabel={repaintClipNodeId === selectedVideoMediaNode.id ? "重拍所选片段" : "导出所选片段"}
onChange={(range) => updateVideoClipRange(selectedVideoMediaNode.id, range)}
onExtractAudio={() => splitVideoAudioTrack(selectedVideoMediaNode.id, "audio-video")}
onReset={() => updateVideoClipRange(selectedVideoMediaNode.id, { start: 0.18, end: 0.52 })}
/>
) : null}
{videoEditPrompt?.nodeId === selectedVideoMediaNode.id ? (
<VideoEditPromptPanel
anchor={selectedVideoAnchor}
operation={videoEditPrompt.operation}
prompt={videoEditPrompt.prompt}
onChange={(value) => setVideoEditPrompt((current) => current ? { ...current, prompt: value } : null)}
onClose={() => setVideoEditPrompt(null)}
onSubmit={() => {
void submitVideoEdit(selectedVideoMediaNode.id, videoEditPrompt.operation, undefined, videoEditPrompt.prompt);
setVideoEditPrompt(null);
}}
/>
) : null}
</div>
) : null}
@ -13037,9 +13152,9 @@ function VideoMediaToolbar({
anchor
? {
left: anchor.centerX,
top: anchor.topY - (24 * anchor.zoom + 10),
transform: `translateX(-50%) translateY(-100%) scale(${OFFICIAL_IMAGE_TOOLBAR_SCALE})`,
transformOrigin: "bottom center",
top: Math.max(68, anchor.topY - (24 * anchor.zoom + 10) - 50 * OFFICIAL_IMAGE_TOOLBAR_SCALE),
transform: `translateX(-50%) scale(${OFFICIAL_IMAGE_TOOLBAR_SCALE})`,
transformOrigin: "top center",
}
: {
left: "50%",
@ -13125,6 +13240,48 @@ function VideoToolbarActionButton({
);
}
function VideoEditPromptPanel({
anchor,
operation,
prompt,
onChange,
onClose,
onSubmit,
}: {
anchor: PresetControlAnchor;
operation: "repaint" | "intro";
prompt: string;
onChange: (value: string) => void;
onClose: () => void;
onSubmit: () => void;
}) {
const viewportWidth = typeof window === "undefined" ? 1280 : window.innerWidth;
const viewportHeight = typeof window === "undefined" ? 800 : window.innerHeight;
return (
<div
className="pointer-events-auto absolute z-50 w-[420px] rounded-xl border border-white/10 bg-[#262626] p-3 text-white shadow-[0_20px_64px_rgba(0,0,0,0.5)]"
style={{ left: Math.max(12, Math.min(anchor.centerX - 210, viewportWidth - 432)), top: Math.max(68, Math.min(anchor.bottomY + 14, viewportHeight - 210)) }}
>
<div className="mb-2 flex items-center justify-between text-[13px] font-medium">
<span>{operation === "repaint" ? "片段重拍" : "创意片头"}</span>
<button aria-label="关闭编辑面板" className="rounded p-1 text-white/50 hover:bg-white/10 hover:text-white" onClick={onClose} type="button"><X className="size-4" /></button>
</div>
<textarea
aria-label="视频编辑提示词"
className="h-[92px] w-full resize-none rounded-lg border border-white/10 bg-black/20 p-2 text-[13px] text-white outline-none placeholder:text-white/35 focus:border-white/30"
maxLength={800}
onChange={(event) => onChange(event.target.value)}
placeholder="描述希望生成的画面"
value={prompt}
/>
<div className="mt-2 flex items-center justify-between">
<span className="text-[11px] text-white/40">{operation === "repaint" ? "最长 10 秒 · 阿里云 Wan 2.7" : "尾段最长 3 秒 · 阿里云 VACE"}</span>
<button className="rounded-lg bg-white px-3 py-1.5 text-[12px] font-medium text-black hover:bg-white/85 disabled:opacity-40" disabled={!prompt.trim()} onClick={onSubmit} type="button">生成</button>
</div>
</div>
);
}
function OfficialVideoMediaToolbar({
archiveControl,
activeMode,
@ -13134,7 +13291,12 @@ function OfficialVideoMediaToolbar({
onAnalyze,
onSlow,
onAudioVideoSplit,
onDownload,
onHdOption,
onIntro,
onPreview,
onRepaint,
onSubjectErase,
onSubtitleErase,
onToggleCrop,
onToggleClip,
@ -13148,7 +13310,12 @@ function OfficialVideoMediaToolbar({
onAnalyze: () => void;
onSlow: (factor:number)=>void;
onAudioVideoSplit: () => void;
onDownload: () => void;
onHdOption: (option: string) => void;
onIntro: () => void;
onPreview: () => void;
onRepaint: () => void;
onSubjectErase: () => void;
onSubtitleErase: (mode: SubtitleEraseMode) => void;
onToggleCrop: () => void;
onToggleClip: () => void;
@ -13158,15 +13325,15 @@ function OfficialVideoMediaToolbar({
return (
<div
className="pointer-events-auto absolute flex h-[58px] items-center gap-1 rounded-[14px] border border-white/8 bg-[#262626]/96 px-2 text-[16px] text-white/82 shadow-[0_20px_64px_rgba(0,0,0,0.42)] backdrop-blur-[10px]"
className="pointer-events-auto absolute flex h-[50px] max-w-[min(96vw,1200px)] items-center gap-0.5 rounded-[12px] border border-white/8 bg-[#262626]/96 px-2 text-[13px] text-white/82 shadow-[0_20px_64px_rgba(0,0,0,0.42)] backdrop-blur-[10px]"
data-canvas-overlay="true"
style={
anchor
? {
left: anchor.centerX,
top: anchor.topY - (24 * anchor.zoom + 10),
transform: `translateX(-50%) translateY(-100%) scale(${OFFICIAL_IMAGE_TOOLBAR_SCALE})`,
transformOrigin: "bottom center",
top: Math.max(68, anchor.topY - (24 * anchor.zoom + 10) - 50 * OFFICIAL_IMAGE_TOOLBAR_SCALE),
transform: `translateX(-50%) scale(${OFFICIAL_IMAGE_TOOLBAR_SCALE})`,
transformOrigin: "top center",
}
: {
left: "50%",
@ -13177,25 +13344,6 @@ function OfficialVideoMediaToolbar({
}
>
{archiveControl}
<select aria-label="视频慢放" title="调整播放速度，不进行 AI 补帧" className="rounded bg-[#262626] p-2 text-sm" value="" onChange={e=>{if(e.target.value)onSlow(Number(e.target.value));}}><option value="">慢放</option>{[2,3,5].map(factor=><option key={factor} value={factor}>{factor} 倍慢放</option>)}</select>
<OfficialVideoToolbarButton
active={activeMode === "clip"}
icon={<Scissors className="h-[18px] w-[18px]" strokeWidth={2.1} />}
label="剪辑"
onClick={() => {
setMenu(null);
onToggleClip();
}}
/>
<OfficialVideoToolbarButton
active={activeMode === "crop"}
icon={<SceneToolIcon kind="crop" />}
label="裁剪"
onClick={() => {
setMenu(null);
onToggleCrop();
}}
/>
<div className="relative">
<OfficialVideoToolbarButton
active={menu === "hd"}
@ -13213,11 +13361,12 @@ function OfficialVideoMediaToolbar({
/>
) : null}
</div>
<OfficialVideoToolbarButton icon={<Clapperboard className="size-[18px]" />} label="片段重拍" onClick={onRepaint} />
<OfficialVideoToolbarButton
active={activeMode === "analysis" || analysisLoading}
disabled={analysisLoading}
icon={<VideoAnalysisGlyph />}
label={analysisLoading ? "解析中" : "解析"}
label={analysisLoading ? "拉片中" : "逐帧拉片"}
onClick={() => {
setMenu(null);
onAnalyze();
@ -13228,18 +13377,17 @@ function OfficialVideoMediaToolbar({
active={menu === "subtitle" || activeMode === "subtitleErase"}
dropdown
icon={<Wand2 className="h-[18px] w-[18px]" strokeWidth={2.1} />}
disabled
label="去字幕（未接通）"
label="智能去字幕"
onClick={() => setMenu((current) => current === "subtitle" ? null : "subtitle")}
/>
{menu === "subtitle" ? (
<VideoToolbarMenuPopover width={220}>
<div className="px-2.5 pb-1.5 pt-1 text-[11px] leading-4 text-white/38">
AI一键去除视频字幕，仅支持中英文字幕
默认处理画面底部字幕；也可框选准确区域
</div>
<VideoToolbarMenuItem
icon={<Wand2 className="h-4 w-4" strokeWidth={2.1} />}
label="智能擦除"
label="底部字幕擦除"
onClick={() => {
setMenu(null);
onSubtitleErase("smart");
@ -13262,7 +13410,7 @@ function OfficialVideoMediaToolbar({
disabled={audioSplitLoading}
dropdown
icon={<Headphones className="h-[18px] w-[18px]" strokeWidth={2.1} />}
label={audioSplitLoading ? "分离中" : "音频分离"}
label={audioSplitLoading ? "分离中" : "音视频分离"}
onClick={() => setMenu((current) => current === "audio" ? null : "audio")}
/>
{menu === "audio" ? (
@ -13297,6 +13445,23 @@ function OfficialVideoMediaToolbar({
</VideoToolbarMenuPopover>
) : null}
</div>
<div className="relative">
<OfficialVideoToolbarButton active={menu === "subject" || activeMode === "subjectErase"} dropdown icon={<BoxSelect className="size-[18px]" />} label="主体消除" onClick={() => setMenu((current) => current === "subject" ? null : "subject")} />
{menu === "subject" ? <VideoToolbarMenuPopover width={180}><VideoToolbarMenuItem icon={<Wand2 className="size-4" />} label="框选主体并消除" onClick={() => { setMenu(null); onSubjectErase(); }} /></VideoToolbarMenuPopover> : null}
</div>
<OfficialVideoToolbarButton icon={<Clapperboard className="size-[18px]" />} label="创意片头" onClick={onIntro} />
<div className="h-6 w-px bg-white/10" />
<button className="flex size-9 items-center justify-center rounded-lg hover:bg-white/8" onClick={() => { setMenu(null); onToggleCrop(); }} title="裁剪画面" type="button"><BoxSelect className="size-[18px]" /></button>
<button className="flex size-9 items-center justify-center rounded-lg hover:bg-white/8" onClick={onDownload} title="下载视频" type="button"><Download className="size-[18px]" /></button>
<button className="flex size-9 items-center justify-center rounded-lg hover:bg-white/8" onClick={onPreview} title="全屏预览" type="button"><Maximize2 className="size-[18px]" /></button>
<div className="relative">
<button className="flex size-9 items-center justify-center rounded-lg hover:bg-white/8" onClick={() => setMenu((current) => current === "more" ? null : "more")} title="更多视频工具" type="button"><Ellipsis className="size-[18px]" /></button>
{menu === "more" ? <VideoToolbarMenuPopover width={180}>
<VideoToolbarMenuItem icon={<Scissors className="size-4" />} label="剪辑片段" onClick={() => { setMenu(null); onToggleClip(); }} />
<VideoToolbarMenuItem icon={<SceneToolIcon kind="crop" />} label="裁剪画面" onClick={() => { setMenu(null); onToggleCrop(); }} />
{[2, 3, 5].map((factor) => <VideoToolbarMenuItem key={factor} icon={<Clock3 className="size-4" />} label={`${factor} 倍慢放`} onClick={() => { setMenu(null); onSlow(factor); }} />)}
</VideoToolbarMenuPopover> : null}
</div>
</div>
);
}
@ -13337,14 +13502,7 @@ function OfficialVideoToolbarButton({
function VideoHdSettingsPopover({ onSelect }: { onSelect: (option: string) => void }) {
return (
<div className="absolute left-0 top-[calc(100%+10px)] z-50 w-[267px] rounded-[12px] border border-white/10 bg-[#202020]/98 p-2 text-white shadow-[0_18px_54px_rgba(0,0,0,0.48)] backdrop-blur-[16px]">
<button
className="mb-2 flex h-8 w-full items-center justify-between rounded-[8px] border border-white/10 bg-white/[0.03] px-3 text-[13px] text-white/82 hover:bg-white/8"
onClick={() => onSelect("Real-ESRGAN")}
type="button"
>
<span>Real-ESRGAN</span>
<ChevronDown className="h-3.5 w-3.5 text-white/46" />
</button>
<div className="mb-2 px-2 text-[12px] text-white/55">Real-ESRGAN · 不补帧</div>
<div className="mb-2 flex gap-1">
{["1080P", "2K", "4K"].map((option, index) => (
<button
@ -13360,22 +13518,7 @@ function VideoHdSettingsPopover({ onSelect }: { onSelect: (option: string) => vo
</button>
))}
</div>
<button
className="mb-2 flex h-8 w-full items-center justify-between rounded-[8px] border border-white/10 bg-white/[0.03] px-3 text-[13px] text-white/82 hover:bg-white/8"
onClick={() => onSelect("不补帧")}
type="button"
>
<span>不补帧</span>
<ChevronDown className="h-3.5 w-3.5 text-white/46" />
</button>
<button
className="flex h-8 w-full items-center justify-between rounded-[8px] border border-white/10 bg-white/[0.03] px-3 text-[13px] text-white/82 hover:bg-white/8"
onClick={() => onSelect("1x")}
type="button"
>
<span>1x</span>
<ChevronDown className="h-3.5 w-3.5 text-white/46" />
</button>
<div className="px-2 text-[11px] text-white/40">选择分辨率后立即提交高清视频任务</div>
</div>
);
}
@ -13718,6 +13861,7 @@ function VideoSubtitleEraseRegionActionBar({
onReset,
onUndo,
regionCount,
titleLabel = "框选擦除",
}: {
anchor?: PresetControlAnchor;
canRedo: boolean;
@ -13730,6 +13874,7 @@ function VideoSubtitleEraseRegionActionBar({
onReset: () => void;
onUndo: () => void;
regionCount: number;
titleLabel?: string;
}) {
return (
<div
@ -13751,7 +13896,7 @@ function VideoSubtitleEraseRegionActionBar({
>
<div className="flex h-[49px] w-max items-center gap-3 rounded-[12px] border border-white/10 bg-[#202020]/98 px-2 py-2 text-white shadow-[0_18px_54px_rgba(0,0,0,0.48)] backdrop-blur-[16px]">
<button
aria-label="关闭去字幕生成器"
aria-label={`关闭${titleLabel}`}
className="flex h-8 w-8 shrink-0 items-center justify-center rounded-[8px] text-white/46 transition hover:bg-white/8 hover:text-white"
onClick={onClose}
type="button"
@ -13760,14 +13905,14 @@ function VideoSubtitleEraseRegionActionBar({
</button>
<div className="h-6 w-px shrink-0 bg-white/10" />
<div className="flex h-8 shrink-0 items-center gap-1 px-2 text-[13px] text-white/86">
<span className="whitespace-nowrap">框选擦除</span>
<span className="whitespace-nowrap">{titleLabel}</span>
<span className="group/help relative flex h-4 w-4 cursor-help items-center justify-center text-white/42 transition hover:text-white/80">
<HelpCircle className="h-3.5 w-3.5" />
<span className="pointer-events-none absolute bottom-[calc(100%+10px)] left-1/2 hidden w-[253px] -translate-x-1/2 rounded-[8px] border border-white/10 bg-[#202020]/98 px-3 py-2 text-left text-[12px] leading-5 text-white/72 shadow-[0_14px_40px_rgba(0,0,0,0.42)] group-hover/help:block">
<span className="mb-1 block text-white/88">使用方法</span>
<span className="block">在画面上拖拽鼠标,框选要擦除的区域</span>
<span className="block">支持框选多个区域,所有框内文字将被擦除</span>
<span className="block">擦除作用于整段视频,不仅是当前帧</span>
<span className="block">在画面上拖拽鼠标，框选要擦除的区域</span>
<span className="block">{titleLabel === "主体消除" ? "模型将跟踪框选主体并修复背景" : "支持框选多个字幕区域"}</span>
<span className="block">擦除作用于所选视频片段</span>
</span>
</span>
</div>
@ -14102,6 +14247,7 @@ function VideoStoryCanvasNode({ node, selected }: { node: CloneNode; selected: b
function VideoClipEditorBar({
anchor,
applyLabel = "导出所选片段",
busy = false,
clipRange,
durationSeconds,
@ -14113,6 +14259,7 @@ function VideoClipEditorBar({
title,
}: {
anchor?: PresetControlAnchor;
applyLabel?: string;
busy?: boolean;
clipRange: VideoClipRange;
durationSeconds: number;
@ -14245,8 +14392,8 @@ function VideoClipEditorBar({
<button
className="flex h-11 w-11 items-center justify-center rounded-full bg-white text-black shadow-[0_10px_24px_rgba(255,255,255,0.18)] transition hover:bg-white/90"
disabled={busy}
aria-label={busy ? "正在导出剪辑" : "导出所选片段"}
title={busy ? "正在导出剪辑" : "导出所选片段"}
aria-label={busy ? "正在处理片段" : applyLabel}
title={busy ? "正在处理片段" : applyLabel}
onClick={onApply}
type="button"
>
@ -15690,9 +15837,9 @@ function CharacterPresetToolbar({
anchor
? {
left: anchor.centerX,
top: anchor.topY - (24 * anchor.zoom + 10),
transform: `translateX(-50%) translateY(-100%) scale(${OFFICIAL_IMAGE_TOOLBAR_SCALE})`,
transformOrigin: "bottom center",
top: Math.max(68, anchor.topY - (24 * anchor.zoom + 10) - 58 * OFFICIAL_IMAGE_TOOLBAR_SCALE),
transform: `translateX(-50%) scale(${OFFICIAL_IMAGE_TOOLBAR_SCALE})`,
transformOrigin: "top center",
}
: {
left: "50%",
@ -15702,7 +15849,10 @@ function CharacterPresetToolbar({
}
}
>
<SceneTool active={toolbarMenu === "panorama"} kind="panorama" label="全景" badge="NEW" onClick={() => setToolbarMenu((current) => current === "panorama" ? null : "panorama")} />
<button className="flex h-10 items-center gap-1.5 whitespace-nowrap rounded-[8px] px-1 text-white/82 hover:bg-white/8" onClick={() => onToolAction("portrait")} type="button">
<UserRound className="size-[18px]" /><span>人像质感调节</span><span className="rounded bg-[#0e6371] px-1.5 text-[11px] font-bold text-[#42eaff]">NEW</span>
</button>
<SceneTool active={toolbarMenu === "panorama"} kind="panorama" label="全景" onClick={() => setToolbarMenu((current) => current === "panorama" ? null : "panorama")} />
<SceneTool active={false} kind="orbit" label="多角度" onClick={() => {
setToolbarMenu(null);
onToolAction("orbit");
@ -15711,7 +15861,8 @@ function CharacterPresetToolbar({
<SceneTool active={toolbarMenu === "grid"} kind="grid" label="九宫格" dropdown onClick={() => setToolbarMenu((current) => current === "grid" ? null : "grid")} />
<SceneTool active={toolbarMenu === "hd"} kind="hd" label="高清" dropdown onClick={() => setToolbarMenu((current) => current === "hd" ? null : "hd")} />
<SceneTool active={toolbarMenu === "crop"} kind="crop" label="宫格切分" dropdown onClick={() => setToolbarMenu((current) => current === "crop" ? null : "crop")} />
<SceneTool active={false} kind="localEdit" label="局部编辑" onClick={() => onToolAction("localEdit")} />
<SceneTool active={false} kind="localEdit" label="元素编辑" onClick={() => onToolAction("localEdit")} />
<button className="flex h-10 items-center gap-1.5 whitespace-nowrap rounded-[8px] px-1 text-white/82 hover:bg-white/8" onClick={() => onToolAction("layerSplit")} type="button"><Layers className="size-[18px]" /><span>图层分离</span></button>
<div className="h-9 w-px bg-white/8" />
{iconActions.map((item) => (
<button

21
docs/video-canvas-toolbar-parity-2026-09-24.md Normal file
View File

@ -0,0 +1,21 @@
# 视频画布选中素材工具条核对（2026-09-24）
参考：用户提供的 Liblib 选中视频、图片截图，以及已登录参考画布。以下「已接线」只表示按钮有真实任务路径，不表示真实素材输出和像素一致性已验收。
| 素材 | 参考功能 | 本地路径 | 本轮验证 | 发布状态 |
| --- | --- | --- | --- | --- |
| 视频 | 高清 | Real-ESRGAN 算力服务；仅展示可执行的 1080P、2K、4K，选定即提交持久任务 | 尺寸与任务恢复测试通过 | 真实结果未验收 |
| 视频 | 片段重拍 | 画布内提示词面板；Wan 2.7 `wan2.7-videoedit`；超过 10 秒先截片段 | 请求契约、恢复、节点结果测试通过；模型目录可查 | 真实结果未验收 |
| 视频 | 逐帧拉片 | 既有视频分析任务 | 既有分析测试通过 | 本轮未完成真实素材验收 |
| 视频 | 智能去字幕 | 框选/底部区域掩码；Wan 2.1 VACE `wanx2.1-vace-plus` | 掩码像素、请求契约测试通过；模型目录可查 | 单次限 5 秒，整片识别擦除与参考站不同 |
| 视频 | 音视频分离 | 既有音轨/人声/背景声分离路径 | 既有音频测试通过 | 本轮未完成真实素材验收 |
| 视频 | 主体消除 | 框选动态跟踪掩码；VACE 视频局部编辑 | 区域掩码与请求契约测试通过 | 单次限 5 秒，真实结果未验收 |
| 视频 | 创意片头 | 画布内提示词面板；VACE 尾片段延展 | 请求契约测试通过 | 尾片段限 3 秒，真实结果未验收 |
| 视频 | 裁剪、下载、全屏、剪辑、慢放 | 既有媒体处理及预览/下载 | FFmpeg 剪辑与音频同步测试通过 | 本轮未完成逐项视觉验收 |
| 图片 | 人像质感调节 | 配置的图片重绘模型与身份保持提示词 | 工具条入口已在本地页面显示 | 真实结果未验收 |
| 图片 | 全景、多角度、打光、九宫格、高清、宫格切分、元素编辑 | 既有模型或本地编辑路径；未配置时明确报错，不再创建空壳节点 | 本轮核对按钮及部分既有测试 | 本轮未完成逐项视觉/真实结果验收 |
| 图片 | 图层分离 | 现有 Replicate `qwen/qwen-image-layered` 服务，返回多图层并恢复到画布 | 多图层恢复测试通过 | 线上已有 Token 但开关关闭；本地无配置，未验收 |
本地对象存储未配置，上传真实图片或视频会失败。本轮另在临时本地画布引用阿里云公开示例视频，实测选中视频工具条、高清/字幕/音频/主体/更多菜单、重拍提示词面板、主体框选及字幕操作栏；框选后「生成擦除」从禁用变为可用。临时项目已删除。这验证了本地交互，不代表模型出图。阿里云账号的模型目录查询同时列出 `wan2.7-videoedit`、`wanx2.1-vace-plus`，但没有发起付费生成，也不能由此推断输出质量或速度。官方 VACE 还要求输入为 MP4、帧率至少 16 FPS、文件不超过 50 MB，局部编辑最多处理 5 秒，片头参考片段最多 3 秒；当前仅按时长拦截，格式、帧率、体积与完整长片处理仍需补齐。生产构建、TypeScript 检查和本轮工具条契约测试通过，不等于像素级和真实媒体验收。发布前仍需在隔离测试环境配置可用媒体存储、启用图层分离模型，并逐个对真实素材运行、留存结果与参考站截图比较。
模型接口依据：[Wan 2.7 视频编辑](https://help.aliyun.com/zh/model-studio/wan-video-editing-api-reference)、[Wan 2.1 VACE 视频编辑](https://help.aliyun.com/zh/model-studio/legacy-wanx-vace-api-reference)、[Qwen-Image-Layered](https://github.com/QwenLM/Qwen-Image-Layered)。

122
lib/video-canvas/aliyun-video-edit.ts Normal file
View File

@ -0,0 +1,122 @@
import "server-only";
import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
import { uploadFile } from "@/lib/storage/service";
import { archiveCanvasVideo } from "./archive-video";
import { preserveGeneratedVideo } from "./archive-result";
import { ProviderTaskFailedError, ProviderTaskUncertainError } from "./video-task-lifecycle";
import { videoEditRequestSchema, type VideoEditRequest } from "./video-edit-contract";
import { buildAliyunVideoEditRequest } from "./video-edit-request";
import { createVideoEditMask } from "./video-edit-mask";
type VideoEditCheckpoint = {
operation: "video-edit";
action: VideoEditRequest["operation"];
sourceUrl: string;
taskId: string;
maskUrl?: string;
};
async function resolveProvider(modelId: string) {
const models = await getEnabledAiModelsWithProvider().catch(() => []);
const officialProvider = (item: (typeof models)[number]) =>
["alibaba", "qianwen-video"].includes(item.provider.type) &&
/^https:\/\/(?:dashscope(?:-intl)?\.aliyuncs\.com|[-a-z0-9]+\.[-a-z0-9]+\.maas\.aliyuncs\.com)(?:\/|$)/i.test(item.provider.baseUrl || "");
const matching = models.find((item) => item.model.modelId === modelId && officialProvider(item)) ||
models.find(officialProvider);
const apiKey = matching?.provider.apiKey || process.env.DASHSCOPE_API_KEY || "";
const baseUrl = matching?.provider.baseUrl || "https://dashscope.aliyuncs.com";
if (!apiKey) throw new Error("阿里云视频编辑 Key 未配置");
const url = new URL(baseUrl);
if (url.protocol !== "https:" || !(
url.hostname === "dashscope.aliyuncs.com" ||
url.hostname === "dashscope-intl.aliyuncs.com" ||
/^[-a-z0-9]+\.[-a-z0-9]+\.maas\.aliyuncs\.com$/.test(url.hostname)
)) throw new Error("视频编辑只允许调用阿里云百炼官方地址");
url.pathname = "/api/v1";
url.search = "";
return { apiKey, endpoint: url.toString().replace(/\/$/, "") };
}
export async function executeAliyunVideoEdit(
raw: Record<string, unknown>,
lifecycle: {
resume?: VideoEditCheckpoint | null;
active: () => Promise<void>;
save: (checkpoint: VideoEditCheckpoint) => Promise<void>;
},
runtime: { fetch: typeof fetch; sleep: (ms: number) => Promise<void> } = {
fetch: globalThis.fetch,
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
},
) {
const input = videoEditRequestSchema.parse(raw);
const modelId = input.operation === "repaint" ? "wan2.7-videoedit" : "wanx2.1-vace-plus";
const { apiKey, endpoint } = await resolveProvider(modelId);
const headers = {
Authorization: `Bearer ${apiKey}`,
"Content-Type": "application/json",
"X-DashScope-Async": "enable",
};
const deadline = Date.now() + 650_000;
const checkpoint = lifecycle.resume;
if (checkpoint && (checkpoint.action !== input.operation || checkpoint.sourceUrl !== input.sourceUrl)) {
throw new ProviderTaskUncertainError("视频编辑源素材已变化，不能自动重提任务");
}
let taskId = checkpoint?.taskId || "";
const request = async (path: string, init: RequestInit = {}) => {
const response = await runtime.fetch(`${endpoint}${path}`, {
...init,
headers,
signal: AbortSignal.timeout(30_000),
});
const body = await response.json();
if (!response.ok) throw new Error(`阿里云视频编辑失败（${response.status}）：${body.message || body.code || "未知错误"}`);
return body;
};
await lifecycle.active();
if (!taskId) {
let maskUrl: string | undefined;
if (input.operation === "erase" || input.operation === "subtitle") {
const regions = input.operation === "subtitle" && !input.regions?.length
? [{ x: 0.04, y: 0.78, width: 0.92, height: 0.2 }]
: input.regions || [];
const mask = await createVideoEditMask(regions, input.width, input.height);
const bytes = Uint8Array.from(mask);
const uploaded = await uploadFile(`video-edit-mask-${crypto.randomUUID()}.png`, bytes.buffer);
maskUrl = uploaded.url;
}
const submitted = await request("/services/aigc/video-generation/video-synthesis", {
method: "POST",
body: JSON.stringify(buildAliyunVideoEditRequest(input, maskUrl)),
});
taskId = String(submitted.output?.task_id || "");
if (!taskId) throw new Error("阿里云未返回视频编辑任务 ID");
await lifecycle.save({ operation: "video-edit", action: input.operation, sourceUrl: input.sourceUrl, taskId, maskUrl });
}
while (Date.now() < deadline) {
await runtime.sleep(5000);
await lifecycle.active();
let result;
try { result = await request(`/tasks/${encodeURIComponent(taskId)}`); }
catch (error) { throw new ProviderTaskUncertainError(`上游任务 ${taskId} 查询失败，将按原任务编号恢复：${error instanceof Error ? error.message : "网络错误"}`); }
const output = result.output || {};
if (output.task_status === "SUCCEEDED") {
const url = String(output.video_url || "");
if (!/^https?:\/\//i.test(url)) throw new Error("视频编辑成功但未返回视频地址");
const stored = await preserveGeneratedVideo(url, {
provider: "aliyun",
model: modelId,
taskId,
operation: input.operation,
durationSeconds: input.operation === "intro" ? 5 : Math.min(input.durationSeconds, input.operation === "repaint" ? 10 : 5),
}, archiveCanvasVideo);
return stored;
}
if (["FAILED", "CANCELED", "UNKNOWN"].includes(output.task_status)) {
throw new ProviderTaskFailedError(`阿里云视频编辑失败：${output.message || output.code || output.task_status}，任务 ${taskId}`);
}
}
throw new ProviderTaskUncertainError(`上游任务 ${taskId} 仍在处理，将按原任务编号继续查询`);
}
export type { VideoEditCheckpoint };

6
lib/video-canvas/layer-split-contract.ts Normal file
View File

@ -0,0 +1,6 @@
import { z } from "zod";
export const canvasLayerSplitSchema = z.object({
imageUrl: z.string().url().refine((url) => /^https?:\/\//i.test(url)),
sourceNodeId: z.string().min(1).max(180),
});

29
lib/video-canvas/task-contract.ts
View File

@ -17,6 +17,8 @@ export const canvasTaskRequestSchema = z.object({
"audio",
"smart-edit",
"video-upscale",
"video-edit",
"image-layer-split",
]),
node: z
.object({
@ -235,7 +237,7 @@ export function reconcileCanvasTasks<
audioParams:
task.request.node.draftValues?.audioParams || task.request.params,
});
if (["video", "video-upscale"].includes(task.request.kind))
if (["video", "video-upscale", "video-edit"].includes(task.request.kind))
Object.assign(nextDraft, {
mediaUrl: data.url,
url: [data.url],
@ -275,6 +277,29 @@ export function reconcileCanvasTasks<
];
}
}
if (task.request.kind === "image-layer-split") {
const layers = Array.isArray(data.layers) ? data.layers : [];
const first = layers[0];
if (first?.originalUrl) Object.assign(nextDraft, {
mediaUrl: first.originalUrl,
url: [first.originalUrl],
action: "IMAGE_RESOURCE",
layerIndex: first.layerIndex || 1,
});
for (let i = 1; i < layers.length; i++) {
const layer = layers[i];
if (!layer?.originalUrl) continue;
const id = `${source.id}-layer-${task.id}-${i}`;
if (hidden.has(id) || result.some((item) => item.id === id)) continue;
result = [...result, {
...node,
id,
title: `${source.title} · ${layer.title || `图层${i + 1}`}`,
y: node.y + i * (node.height + 20),
draftValues: { ...nextDraft, mediaUrl: layer.originalUrl, url: [layer.originalUrl], layerIndex: layer.layerIndex || i + 1 },
} as T];
}
}
if (
task.request.kind === "analysis" &&
task.request.params.mode === "director-reconstruct"
@ -353,7 +378,7 @@ export function reconcileCanvasTasks<
? {
kind: "media",
inputType:
["video", "video-upscale"].includes(task.request.kind) ||
["video", "video-upscale", "video-edit"].includes(task.request.kind) ||
task.request.kind === "smart-edit"
? "video"
: task.request.kind === "audio"

11
lib/video-canvas/task-recovery.ts
View File

@ -45,6 +45,16 @@ export function getVideoCheckpoint(
return null;
return parsed.data;
}
export function getVideoEditCheckpoint(task: Pick<RecoveryTask, "request" | "result">) {
const checkpoint = task.result?.videoEdit;
return task.request.kind === "video-edit" &&
checkpoint?.operation === "video-edit" &&
checkpoint.action === task.request.params.operation &&
checkpoint.sourceUrl === task.request.params.sourceUrl &&
typeof checkpoint.taskId === "string" && checkpoint.taskId.trim()
? checkpoint as import("./aliyun-video-edit").VideoEditCheckpoint
: null;
}
export function getAudioCheckpoint(
task: Pick<RecoveryTask, "request" | "result">
) {
@ -80,6 +90,7 @@ export function hasRecoverableCanvasTask(task: RecoveryTask, now = Date.now()) {
Boolean(
getEnhanceCheckpoint(task) ||
getVideoCheckpoint(task) ||
getVideoEditCheckpoint(task) ||
getAudioCheckpoint(task) ||
getSmartEditCheckpoint(task) ||
getUpscaleCheckpoint(task)

16
lib/video-canvas/task-worker.ts
View File

@ -1,6 +1,9 @@
import { resolveGroupInputs } from "./group-execution";
import { groupTaskInputs } from "./task-store";
import { executeVideoUpscale } from "./video-upscale";
import { executeAliyunVideoEdit } from "./aliyun-video-edit";
import { executeLayerSplitJob } from "@/lib/ai/layer-split/service";
import { canvasLayerSplitSchema } from "./layer-split-contract";
import { getUpscaleCheckpoint } from "./task-recovery";
import { executeSmartEdit } from "./smart-edit/execute";
import { executeCanvasAudio } from "./audio-generation";
@ -8,6 +11,7 @@ import { ProviderTaskUncertainError } from "./video-task-lifecycle";
import {
getEnhanceCheckpoint,
getVideoCheckpoint,
getVideoEditCheckpoint,
getAudioCheckpoint,
getSmartEditCheckpoint,
} from "./task-recovery";
@ -54,6 +58,7 @@ async function execute(task: CanvasGenerationTask, userId: string) {
await assertLease();
const resume = getEnhanceCheckpoint(task);
const videoResume = getVideoCheckpoint(task);
const videoEditResume = getVideoEditCheckpoint(task);
const audioResume = getAudioCheckpoint(task);
const editResume = getSmartEditCheckpoint(task);
const upscaleResume = getUpscaleCheckpoint(task);
@ -61,6 +66,7 @@ async function execute(task: CanvasGenerationTask, userId: string) {
(task.attempts || 1) > 1 &&
!resume &&
!videoResume &&
!videoEditResume &&
!audioResume &&
!editResume &&
!upscaleResume
@ -76,6 +82,16 @@ async function execute(task: CanvasGenerationTask, userId: string) {
save: (value) =>
checkpointCanvasGenerationTask(task.id, value, task.attempts),
});
if (request.kind === "video-edit")
return executeAliyunVideoEdit(params, {
resume: videoEditResume,
active: assertLease,
save: (checkpoint) => checkpointCanvasGenerationTask(task.id, { videoEdit: checkpoint }, task.attempts),
});
if (request.kind === "image-layer-split") {
const input = canvasLayerSplitSchema.parse(params);
return executeLayerSplitJob({ userId, imageUrl: input.imageUrl, sourceNodeId: input.sourceNodeId });
}
if (request.kind === "smart-edit")
return executeSmartEdit(params, userId, {
taskId: task.id,

31
lib/video-canvas/video-edit-contract.ts Normal file
View File

@ -0,0 +1,31 @@
import { z } from "zod";
export const videoEditRequestSchema = z.object({
operation: z.enum(["repaint", "erase", "subtitle", "intro"]),
sourceUrl: z.string().url().refine((value) => /^https?:\/\//i.test(value)),
prompt: z.string().trim().max(800).optional(),
sourceNodeId: z.string().min(1).max(180),
durationSeconds: z.number().positive().max(600),
width: z.number().int().min(64).max(8192),
height: z.number().int().min(64).max(8192),
regions: z.array(z.object({
x: z.number().min(0).max(1),
y: z.number().min(0).max(1),
width: z.number().positive().max(1),
height: z.number().positive().max(1),
})).max(12).optional(),
}).superRefine((value, context) => {
if (value.durationSeconds > (value.operation === "repaint" ? 10 : 5)) {
context.addIssue({ code: "custom", message: value.operation === "repaint" ? "片段重拍最多处理 10 秒，请先剪辑目标片段" : "区域编辑和片头最多处理 5 秒，请先剪辑目标片段", path: ["durationSeconds"] });
}
if (value.width * value.height > 8_388_608) {
context.addIssue({ code: "custom", message: "视频分辨率过高，请先缩小素材", path: ["width"] });
}
if (value.operation === "erase" && !value.regions?.length) {
context.addIssue({ code: "custom", message: "请先框选要消除的主体", path: ["regions"] });
}
if (value.operation === "intro" && value.durationSeconds > 3) {
context.addIssue({ code: "custom", message: "创意片头需要不超过 3 秒的尾段视频，请先剪辑尾段", path: ["durationSeconds"] });
}
});
export type VideoEditRequest = z.infer<typeof videoEditRequestSchema>;

18
lib/video-canvas/video-edit-mask.ts Normal file
View File

@ -0,0 +1,18 @@
import sharp from "sharp";
import type { VideoEditRequest } from "./video-edit-contract";
export async function createVideoEditMask(
regions: NonNullable<VideoEditRequest["regions"]>,
width: number,
height: number,
) {
const pixels = Buffer.alloc(width * height * 3, 0);
for (const region of regions) {
const left = Math.max(0, Math.floor(region.x * width));
const top = Math.max(0, Math.floor(region.y * height));
const right = Math.min(width, Math.ceil((region.x + region.width) * width));
const bottom = Math.min(height, Math.ceil((region.y + region.height) * height));
for (let y = top; y < bottom; y++) pixels.fill(255, (y * width + left) * 3, (y * width + right) * 3);
}
return sharp(pixels, { raw: { width, height, channels: 3 } }).png().toBuffer();
}

24
lib/video-canvas/video-edit-request.ts Normal file
View File

@ -0,0 +1,24 @@
import type { VideoEditRequest } from "./video-edit-contract";
export function buildAliyunVideoEditRequest(input: VideoEditRequest, maskUrl?: string) {
const parameters: Record<string, unknown> = { prompt_extend: false, watermark: false };
let modelInput: Record<string, unknown>;
if (input.operation === "intro") {
modelInput = { function: "video_extension", prompt: input.prompt || "生成与尾段自然衔接的创意片头", last_clip_url: input.sourceUrl };
} else if (input.operation === "repaint") {
modelInput = { prompt: input.prompt || "保持原视频主体身份、动作与场景，提升镜头质感和光影，其他保持不变", media: [{ type: "video", url: input.sourceUrl }] };
parameters.resolution = "720P";
return { model: "wan2.7-videoedit", input: modelInput, parameters };
} else {
if (!maskUrl) throw new Error("区域编辑需要掩码图像");
modelInput = {
function: "video_edit",
prompt: input.prompt || (input.operation === "subtitle" ? "移除画面底部字幕和文字，按周围画面自然修复背景" : "移除框选主体，按周围画面自然修复背景"),
video_url: input.sourceUrl,
mask_image_url: maskUrl,
mask_frame_id: 1,
};
parameters.mask_type = input.operation === "erase" ? "tracking" : "fixed";
}
return { model: "wanx2.1-vace-plus", input: modelInput, parameters };
}

1
playwright.helpers.config.ts
View File

@ -4,6 +4,7 @@ export default defineConfig({
testDir: "./tests",
testMatch: [
/routes\/video-canvas-media-import\.test\.ts/,
/routes\/video-canvas-toolbar-jobs\.test\.ts/,
/routes\/video-canvas-audio-remote\.test\.ts/,
/routes\/video-canvas-groups-upscale\.test\.ts/,
/routes\/video-canvas-connections\.test\.ts/,

98
tests/routes/video-canvas-toolbar-jobs.test.ts Normal file
View File

@ -0,0 +1,98 @@
import { expect, test } from "@playwright/test";
import { videoEditRequestSchema } from "@/lib/video-canvas/video-edit-contract";
import { buildAliyunVideoEditRequest } from "@/lib/video-canvas/video-edit-request";
import { createVideoEditMask } from "@/lib/video-canvas/video-edit-mask";
import sharp from "sharp";
import { canvasLayerSplitSchema } from "@/lib/video-canvas/layer-split-contract";
import { reconcileCanvasTasks, type CanvasGenerationTask } from "@/lib/video-canvas/task-contract";
import { getVideoEditCheckpoint, hasRecoverableCanvasTask } from "@/lib/video-canvas/task-recovery";
const edit = {
operation: "erase",
sourceUrl: "https://example.com/source.mp4",
sourceNodeId: "source",
durationSeconds: 4,
width: 1080,
height: 1440,
regions: [{ x: 0.2, y: 0.1, width: 0.4, height: 0.5 }],
};
test("video editing rejects a long clip and an unselected removal area before provider submission", () => {
expect(videoEditRequestSchema.safeParse(edit).success).toBe(true);
expect(videoEditRequestSchema.safeParse({ ...edit, durationSeconds: 37 }).success).toBe(false);
expect(videoEditRequestSchema.safeParse({ ...edit, operation: "repaint", durationSeconds: 10 }).success).toBe(true);
expect(videoEditRequestSchema.safeParse({ ...edit, regions: [] }).success).toBe(false);
expect(videoEditRequestSchema.safeParse({ ...edit, operation: "intro", durationSeconds: 4 }).success).toBe(false);
expect(canvasLayerSplitSchema.safeParse({ imageUrl: "file:///private/image.png", sourceNodeId: "source" }).success).toBe(false);
});
test("video toolbar operations send the corresponding VACE function and required mask", () => {
const base = videoEditRequestSchema.parse(edit);
const erase = buildAliyunVideoEditRequest(base, "https://example.com/mask.png");
expect(erase.input).toMatchObject({ function: "video_edit", video_url: edit.sourceUrl, mask_image_url: "https://example.com/mask.png" });
expect(erase.parameters.mask_type).toBe("tracking");
expect(() => buildAliyunVideoEditRequest(base)).toThrow("掩码");
const subtitle = buildAliyunVideoEditRequest({ ...base, operation: "subtitle" }, "https://example.com/subtitles.png");
expect(subtitle.parameters.mask_type).toBe("fixed");
expect(buildAliyunVideoEditRequest({ ...base, operation: "repaint" })).toMatchObject({ model: "wan2.7-videoedit", input: { media: [{ type: "video", url: edit.sourceUrl }] } });
expect(buildAliyunVideoEditRequest({ ...base, operation: "intro" }).input).toMatchObject({ function: "video_extension", last_clip_url: edit.sourceUrl });
});
test("region masks preserve exact video dimensions and pure black and white pixels", async () => {
const png = await createVideoEditMask([{ x: 0.25, y: 0.25, width: 0.5, height: 0.5 }], 8, 8);
const image = sharp(png);
expect(await image.metadata()).toMatchObject({ format: "png", width: 8, height: 8 });
const { data } = await image.raw().toBuffer({ resolveWithObject: true });
const pixel = (x: number, y: number) => [...data.subarray((y * 8 + x) * 3, (y * 8 + x + 1) * 3)];
expect(pixel(0, 0)).toEqual([0, 0, 0]);
expect(pixel(3, 3)).toEqual([255, 255, 255]);
expect(pixel(7, 7)).toEqual([0, 0, 0]);
});
test("an accepted video edit resumes only the same upstream task and source", () => {
const base = {
status: "interrupted",
createdAt: new Date().toISOString(),
updatedAt: new Date(Date.now() - 60_000).toISOString(),
attempts: 1,
request: { kind: "video-edit", params: edit },
result: { videoEdit: { operation: "video-edit", action: "erase", sourceUrl: edit.sourceUrl, taskId: "upstream-1" } },
};
expect(getVideoEditCheckpoint(base)?.taskId).toBe("upstream-1");
expect(hasRecoverableCanvasTask(base)).toBe(true);
expect(getVideoEditCheckpoint({ ...base, request: { ...base.request, params: { ...edit, sourceUrl: "https://example.com/other.mp4" } } })).toBeNull();
});
test("completed video editing stays a playable video node after task reconciliation", () => {
const task: CanvasGenerationTask = {
id: "video-edit-task", requestId: "11111111-1111-4111-8111-111111111111", status: "succeeded", createdAt: "2026-09-24T01:01:00Z",
request: {
projectId: "22222222-2222-4222-8222-222222222222", requestId: "11111111-1111-4111-8111-111111111111", kind: "video-edit",
node: { id: "edit-result", title: "片段重拍", kind: "media", inputType: "video", x: 0, y: 0, width: 420, height: 300, draftValues: { action: "VIDEO_RESOURCE" } },
params: edit, edges: [],
},
result: { url: "https://example.com/edited.mp4", metadata: { provider: "aliyun", storageStatus: "temporary" } },
};
const [result] = reconcileCanvasTasks<{ id: string; inputType?: string; draftValues?: Record<string, any>; [key: string]: any }>([], [task], "2026-09-24T01:00:00Z");
expect(result.inputType).toBe("video");
expect(result.draftValues?.mediaUrl).toBe("https://example.com/edited.mp4");
expect(result.draftValues?.generationStatus).toBe("complete");
});
test("layer split restores every returned layer once after leaving and reopening the canvas", () => {
const source = { id: "split-result", title: "图层分离", kind: "media", inputType: "image", x: 400, y: 200, width: 420, height: 312, draftValues: { action: "IMAGE_RESOURCE" } };
const task: CanvasGenerationTask = {
id: "split-task", requestId: "11111111-1111-4111-8111-111111111111", status: "succeeded", createdAt: "2026-09-24T01:01:00Z",
request: { projectId: "22222222-2222-4222-8222-222222222222", requestId: "11111111-1111-4111-8111-111111111111", kind: "image-layer-split", node: source, params: { imageUrl: "https://example.com/image.png", sourceNodeId: "source" }, edges: [] },
result: { layers: [
{ originalUrl: "https://example.com/layer-1.png", layerIndex: 1, title: "图层1" },
{ originalUrl: "https://example.com/layer-2.png", layerIndex: 2, title: "图层2" },
{ originalUrl: "https://example.com/layer-3.png", layerIndex: 3, title: "图层3" },
] },
};
const restored = reconcileCanvasTasks<{ id: string; draftValues?: Record<string, any>; [key: string]: any }>([], [task], "2026-09-24T01:00:00Z");
expect(restored.map((node) => node.draftValues?.mediaUrl)).toEqual([
"https://example.com/layer-1.png", "https://example.com/layer-2.png", "https://example.com/layer-3.png",
]);
expect(reconcileCanvasTasks(restored, [task], "2026-09-24T01:00:00Z")).toHaveLength(3);
});