import { tool, type ToolSet } from "ai"; import { z } from "zod"; import { defaultVideoCanvasFunctionConfig, type VideoCanvasFunctionConfig, } from "@/lib/video-canvas/config"; import { getCanvasAgentAccessManifest, isExplicitDestructiveIntent, summarizeAgentCapabilities, VIDEO_CANVAS_EXTRA_ACTIONS, VIDEO_CANVAS_INTERACTIVE_ACTIONS, } from "./capability-access"; import { resolveVideoMode, videoNodeType, videoNodeUrl, type VideoRuntimeNode, } from "./video-protocol"; import type { VideoRuntimePorts } from "./video-tools"; import type { CanvasAgentTool } from "./loop"; import type { AgentRunSnapshot } from "./protocol"; export function registerVideoCanvasAccessTools(params: { runId: string; projectId?: string; selectedNodeIds: string[]; snapshot: AgentRunSnapshot; ports: VideoRuntimePorts; definitions: ToolSet; executable: Record; makeNode: ( callId: string, title: string, type: "text" | "image" | "video" | "audio", draft: Record, ) => VideoRuntimeNode; }) { function register( name: string, title: string, schema: T, execute: (input: z.infer, id: string) => Promise>, readOnly = false, ) { params.definitions[name] = tool({ description: title, inputSchema: schema }); params.executable[name] = { title, readOnly, concurrentSafe: readOnly, execute: (input, id) => execute(schema.parse(input), id), }; } const config = params.ports.featureConfig || defaultVideoCanvasFunctionConfig; const nodeIds = z.array(z.string().min(1).max(180)).max(50).default([]); async function sources(ids: string[]) { const { nodes } = await params.ports.read(); return ids.map((id) => { const node = nodes.find((item) => item.id === id); if (!node) throw new Error("素材节点不属于当前视频画布"); return node; }); } register( "listVideoCanvasCapabilities", "列出视频画布全部已授权能力、执行方式及尚缺真实后端的入口。", z.object({}), async () => ({ capabilities: summarizeAgentCapabilities( getCanvasAgentAccessManifest(config).filter((item) => item.workspace === "video"), ), }), true, ); register( "editVideoCanvas", "改名、移动、复制、删除、连线、移除连线、批注、编辑或添加文本、排列和缩放定位视频画布节点。删除需要用户明确要求并选中具体目标。", z.object({ action: z.enum( VIDEO_CANVAS_EXTRA_ACTIONS.map(([id]) => id) as [ (typeof VIDEO_CANVAS_EXTRA_ACTIONS)[number][0], ...(typeof VIDEO_CANVAS_EXTRA_ACTIONS)[number][0][], ], ), nodeIds, text: z.string().max(8000).optional(), x: z.number().finite().optional(), y: z.number().finite().optional(), zoom: z.number().min(0.05).max(8).optional(), }), async (input, id) => { await sources(input.nodeIds); const text = [ params.snapshot.objective, ...params.snapshot.inputs.map((item) => item.text), ].join("\n"); if ( input.action === "delete" && (!isExplicitDestructiveIntent(text) || !input.nodeIds.length || input.nodeIds.some((nodeId) => !params.selectedNodeIds.includes(nodeId))) ) throw new Error("删除需要用户明确要求,并选中具体目标;模型不能自行授权"); if (!params.ports.mutate) throw new Error("当前执行环境未连接视频画布编辑服务"); return params.ports.mutate(`${params.runId}:${id}`, input); }, ); register( "openVideoCanvasFeature", "打开视频画布真实工具面板或设置入口。上传、导入、发布、付费提交仍需用户在面板完成;占位功能不能当作真实处理完成。", z.object({ action: z.enum( VIDEO_CANVAS_INTERACTIVE_ACTIONS.map(([id]) => id) as [ (typeof VIDEO_CANVAS_INTERACTIVE_ACTIONS)[number][0], ...(typeof VIDEO_CANVAS_INTERACTIVE_ACTIONS)[number][0][], ], ), nodeIds, }), async (input, id) => { await sources(input.nodeIds); params.ports.emit?.("data-canvasAgentUiCommand", { projectId: params.projectId, requestId: `${params.runId}:${id}`, workspace: "video", featureId: `video.ui.${input.action}`, nodeIds: input.nodeIds, }); return { mode: "interactive", action: input.action, message: "等待用户完成界面操作;不是媒体处理完成", }; }, ); register( "processVideoMedia", "真实提取视频首帧、尾帧或指定时间帧,按秒剪辑、按像素裁剪、提取完整音轨或生成无声视频。不会伪造人声/背景音分离。保留原素材并写入新节点。", z.object({ operation: z.enum(["extract-frame", "trim", "crop", "extract-audio", "mute"]), nodeId: z.string().min(1).max(180), time: z.number().min(0).max(120).optional(), end: z.number().positive().max(120).optional(), frame: z.enum(["first", "last", "time"]).optional(), x: z.number().int().nonnegative().optional(), y: z.number().int().nonnegative().optional(), width: z.number().int().positive().optional(), height: z.number().int().positive().optional(), }), async (input, id) => { const [source] = await sources([input.nodeId]); if (videoNodeType(source) !== "video" || !videoNodeUrl(source)) throw new Error("请选择当前画布中真实的视频节点"); if (!params.ports.processMedia) throw new Error("媒体处理服务尚未连接"); const result = await params.ports.processMedia({ ...input, url: videoNodeUrl(source) }); const title = { "extract-frame": "视频截帧", trim: "视频剪辑", crop: "视频裁剪", "extract-audio": "视频音轨", mute: "无声视频", }[input.operation]; const node = params.makeNode(id, title, result.type, { mediaUrl: result.url, url: [result.url], mediaOperation: input.operation, sourceVideoNodeId: source.id, duration: result.duration, }); return { status: "succeeded", type: result.type, createdNodeIds: [await params.ports.append(node, [source.id])], }; }, ); register( "runVideoCanvasFunction", "执行视频画布配置清单中的预设或编辑功能。先查询能力清单。参数缺失时打开对应真实面板;缺少后端的功能只授权入口,不产生模拟结果。", z.object({ functionKey: z.string().min(1).max(120), nodeIds, prompt: z.string().max(8000).default(""), args: z.record(z.unknown()).default({}), }), async (input, id) => { const definition = config.functions[input.functionKey]; if (!definition) throw new Error("视频画布没有这个功能"); if (!config.enabled || !definition.enabled) throw new Error("功能已授权,但管理员关闭了对应后端开关"); const nodes = await sources(input.nodeIds); if (definition.key === "audio-to-video") { params.ports.emit?.("data-canvasAgentUiCommand", { projectId: params.projectId, requestId: `${params.runId}:${id}`, workspace: "video", featureId: `video.function.${definition.key}`, nodeIds: input.nodeIds, }); return { mode: "interactive", backendRequired: true, message: "音频驱动视频尚缺真实后端,已打开设置入口;没有生成音频驱动视频", }; } const images = nodes.filter((node) => videoNodeType(node) === "image"); const imageUrls = images.map(videoNodeUrl); if (images.some((node) => !videoNodeUrl(node))) throw new Error("参考图片还未上传完成"); const prompt = [definition.prompt, input.prompt].filter(Boolean).join("\n\n"); if (definition.route === "canvas-generate-video") { const mode = definition.key === "first-frame-video" ? "first-frame" : definition.key === "first-last-frame-video" ? "first-last-frame" : definition.key === "text-to-video" ? "text-to-video" : resolveVideoMode(params.snapshot.videoSettings!.mode, images.length); resolveVideoMode(mode, images.length); return params.executable.generateVideoShot.execute( { prompt, referenceNodeIds: images.map((node) => node.id), mode }, id, ); } if (definition.route === "canvas-workflow-analyze") { const data = await params.ports.analyze({ mode: definition.key === "story-script" ? "story-script" : nodes.some((node) => videoNodeType(node) === "video") ? "video-analysis" : "agent-plan", prompt, imageUrls, videoUrls: nodes.filter((node) => videoNodeType(node) === "video").map(videoNodeUrl), }); const node = params.makeNode(id, definition.label, "text", { content: [String(data.raw || data.summary || "")], rows: data.analysisRows || data.shotIdeas || [], functionKey: definition.key, }); return { status: "succeeded", createdNodeIds: [await params.ports.append(node, input.nodeIds)], type: "script", }; } if (definition.route === "canvas-generate-image") { if (definition.kind === "image-edit" && !imageUrls.length) throw new Error("这个预设需要先连接一张真实参考图片"); const result = await params.ports.generateImage({ prompt, references: imageUrls, callId: id, functionKey: definition.key, modelRef: definition.model, aspectRatio: /grid|three-view|prediction|lighting-correction/.test(definition.key) ? "16:9" : undefined, }); const grid = definition.key === "story-beat-grid-4" ? 2 : definition.key === "multi-view-nine-grid" ? 3 : definition.key === "continuous-storyboard-grid-25" ? 5 : undefined; const node = params.makeNode(id, definition.label, "image", { mediaUrl: result.url, url: [result.url], prompt, functionKey: definition.key, model: result.model, ...(grid ? { gridRows: grid, gridColumns: grid, nineGridComposite: true } : {}), }); return { status: "succeeded", createdNodeIds: [await params.ports.append(node, input.nodeIds)], type: "image", }; } const requiresPanel = definition.route === "pick-object" || (definition.key === "image-crop" && !["x", "y", "width", "height"].every((key) => Number.isFinite(input.args[key]))) || (definition.key === "grid-split" && !Number(input.args.rows)); if (requiresPanel || !params.ports.processImage) { params.ports.emit?.("data-canvasAgentUiCommand", { projectId: params.projectId, requestId: `${params.runId}:${id}`, workspace: "video", featureId: `video.function.${definition.key}`, nodeIds: input.nodeIds, }); return { mode: "interactive", message: "已请求打开参数面板,需要完成参数设置后执行" }; } if (images.length !== 1) throw new Error("请为此编辑功能选择一张真实图片"); const results = await params.ports.processImage({ definition, imageUrl: imageUrls[0], args: input.args, prompt, callId: id, }); const created: string[] = []; for (const [index, result] of results.entries()) created.push( await params.ports.append( params.makeNode(`${id}-${index}`, `${definition.label}-${index + 1}`, "image", { mediaUrl: result.url, url: [result.url], functionKey: definition.key, prompt, }), input.nodeIds, ), ); return { status: "succeeded", createdNodeIds: created, type: "image" }; }, ); }