323 lines
12 KiB
TypeScript
323 lines
12 KiB
TypeScript
import { tool, type ToolSet } from "ai";
|
|
import { z } from "zod";
|
|
import {
|
|
defaultVideoCanvasFunctionConfig,
|
|
type VideoCanvasFunctionConfig,
|
|
} from "@/lib/video-canvas/config";
|
|
import {
|
|
getCanvasAgentAccessManifest,
|
|
isExplicitDestructiveIntent,
|
|
summarizeAgentCapabilities,
|
|
VIDEO_CANVAS_EXTRA_ACTIONS,
|
|
VIDEO_CANVAS_INTERACTIVE_ACTIONS,
|
|
} from "./capability-access";
|
|
import {
|
|
resolveVideoMode,
|
|
videoNodeType,
|
|
videoNodeUrl,
|
|
type VideoRuntimeNode,
|
|
} from "./video-protocol";
|
|
import type { VideoRuntimePorts } from "./video-tools";
|
|
import type { CanvasAgentTool } from "./loop";
|
|
import type { AgentRunSnapshot } from "./protocol";
|
|
|
|
export function registerVideoCanvasAccessTools(params: {
|
|
runId: string;
|
|
projectId?: string;
|
|
selectedNodeIds: string[];
|
|
snapshot: AgentRunSnapshot;
|
|
ports: VideoRuntimePorts;
|
|
definitions: ToolSet;
|
|
executable: Record<string, CanvasAgentTool>;
|
|
makeNode: (
|
|
callId: string,
|
|
title: string,
|
|
type: "text" | "image" | "video" | "audio",
|
|
draft: Record<string, unknown>,
|
|
) => VideoRuntimeNode;
|
|
}) {
|
|
function register<T extends z.ZodTypeAny>(
|
|
name: string,
|
|
title: string,
|
|
schema: T,
|
|
execute: (input: z.infer<T>, id: string) => Promise<Record<string, unknown>>,
|
|
readOnly = false,
|
|
) {
|
|
params.definitions[name] = tool({ description: title, inputSchema: schema });
|
|
params.executable[name] = {
|
|
title,
|
|
readOnly,
|
|
concurrentSafe: readOnly,
|
|
execute: (input, id) => execute(schema.parse(input), id),
|
|
};
|
|
}
|
|
const config = params.ports.featureConfig || defaultVideoCanvasFunctionConfig;
|
|
const nodeIds = z.array(z.string().min(1).max(180)).max(50).default([]);
|
|
async function sources(ids: string[]) {
|
|
const { nodes } = await params.ports.read();
|
|
return ids.map((id) => {
|
|
const node = nodes.find((item) => item.id === id);
|
|
if (!node) throw new Error("素材节点不属于当前视频画布");
|
|
return node;
|
|
});
|
|
}
|
|
register(
|
|
"listVideoCanvasCapabilities",
|
|
"列出视频画布全部已授权能力、执行方式及尚缺真实后端的入口。",
|
|
z.object({}),
|
|
async () => ({
|
|
capabilities: summarizeAgentCapabilities(
|
|
getCanvasAgentAccessManifest(config).filter((item) => item.workspace === "video"),
|
|
),
|
|
}),
|
|
true,
|
|
);
|
|
register(
|
|
"editVideoCanvas",
|
|
"改名、移动、复制、删除、连线、移除连线、批注、编辑或添加文本、排列和缩放定位视频画布节点。删除需要用户明确要求并选中具体目标。",
|
|
z.object({
|
|
action: z.enum(
|
|
VIDEO_CANVAS_EXTRA_ACTIONS.map(([id]) => id) as [
|
|
(typeof VIDEO_CANVAS_EXTRA_ACTIONS)[number][0],
|
|
...(typeof VIDEO_CANVAS_EXTRA_ACTIONS)[number][0][],
|
|
],
|
|
),
|
|
nodeIds,
|
|
text: z.string().max(8000).optional(),
|
|
x: z.number().finite().optional(),
|
|
y: z.number().finite().optional(),
|
|
zoom: z.number().min(0.05).max(8).optional(),
|
|
}),
|
|
async (input, id) => {
|
|
await sources(input.nodeIds);
|
|
const text = [
|
|
params.snapshot.objective,
|
|
...params.snapshot.inputs.map((item) => item.text),
|
|
].join("\n");
|
|
if (
|
|
input.action === "delete" &&
|
|
(!isExplicitDestructiveIntent(text) ||
|
|
!input.nodeIds.length ||
|
|
input.nodeIds.some((nodeId) => !params.selectedNodeIds.includes(nodeId)))
|
|
)
|
|
throw new Error("删除需要用户明确要求，并选中具体目标；模型不能自行授权");
|
|
if (!params.ports.mutate) throw new Error("当前执行环境未连接视频画布编辑服务");
|
|
return params.ports.mutate(`${params.runId}:${id}`, input);
|
|
},
|
|
);
|
|
register(
|
|
"openVideoCanvasFeature",
|
|
"打开视频画布真实工具面板或设置入口。上传、导入、发布、付费提交仍需用户在面板完成；占位功能不能当作真实处理完成。",
|
|
z.object({
|
|
action: z.enum(
|
|
VIDEO_CANVAS_INTERACTIVE_ACTIONS.map(([id]) => id) as [
|
|
(typeof VIDEO_CANVAS_INTERACTIVE_ACTIONS)[number][0],
|
|
...(typeof VIDEO_CANVAS_INTERACTIVE_ACTIONS)[number][0][],
|
|
],
|
|
),
|
|
nodeIds,
|
|
}),
|
|
async (input, id) => {
|
|
await sources(input.nodeIds);
|
|
params.ports.emit?.("data-canvasAgentUiCommand", {
|
|
projectId: params.projectId,
|
|
requestId: `${params.runId}:${id}`,
|
|
workspace: "video",
|
|
featureId: `video.ui.${input.action}`,
|
|
nodeIds: input.nodeIds,
|
|
});
|
|
return {
|
|
mode: "interactive",
|
|
action: input.action,
|
|
message: "等待用户完成界面操作；不是媒体处理完成",
|
|
};
|
|
},
|
|
);
|
|
register(
|
|
"processVideoMedia",
|
|
"真实提取视频首帧、尾帧或指定时间帧，按秒剪辑、按像素裁剪、提取完整音轨或生成无声视频。不会伪造人声/背景音分离。保留原素材并写入新节点。",
|
|
z.object({
|
|
operation: z.enum(["extract-frame", "trim", "crop", "extract-audio", "mute"]),
|
|
nodeId: z.string().min(1).max(180),
|
|
time: z.number().min(0).max(120).optional(),
|
|
end: z.number().positive().max(120).optional(),
|
|
frame: z.enum(["first", "last", "time"]).optional(),
|
|
x: z.number().int().nonnegative().optional(),
|
|
y: z.number().int().nonnegative().optional(),
|
|
width: z.number().int().positive().optional(),
|
|
height: z.number().int().positive().optional(),
|
|
}),
|
|
async (input, id) => {
|
|
const [source] = await sources([input.nodeId]);
|
|
if (videoNodeType(source) !== "video" || !videoNodeUrl(source))
|
|
throw new Error("请选择当前画布中真实的视频节点");
|
|
if (!params.ports.processMedia) throw new Error("媒体处理服务尚未连接");
|
|
const result = await params.ports.processMedia({ ...input, url: videoNodeUrl(source) });
|
|
const title = {
|
|
"extract-frame": "视频截帧",
|
|
trim: "视频剪辑",
|
|
crop: "视频裁剪",
|
|
"extract-audio": "视频音轨",
|
|
mute: "无声视频",
|
|
}[input.operation];
|
|
const node = params.makeNode(id, title, result.type, {
|
|
mediaUrl: result.url,
|
|
url: [result.url],
|
|
mediaOperation: input.operation,
|
|
sourceVideoNodeId: source.id,
|
|
duration: result.duration,
|
|
});
|
|
return {
|
|
status: "succeeded",
|
|
type: result.type,
|
|
createdNodeIds: [await params.ports.append(node, [source.id])],
|
|
};
|
|
},
|
|
);
|
|
register(
|
|
"runVideoCanvasFunction",
|
|
"执行视频画布配置清单中的预设或编辑功能。先查询能力清单。参数缺失时打开对应真实面板；缺少后端的功能只授权入口，不产生模拟结果。",
|
|
z.object({
|
|
functionKey: z.string().min(1).max(120),
|
|
nodeIds,
|
|
prompt: z.string().max(8000).default(""),
|
|
args: z.record(z.unknown()).default({}),
|
|
}),
|
|
async (input, id) => {
|
|
const definition = config.functions[input.functionKey];
|
|
if (!definition) throw new Error("视频画布没有这个功能");
|
|
if (!config.enabled || !definition.enabled)
|
|
throw new Error("功能已授权，但管理员关闭了对应后端开关");
|
|
const nodes = await sources(input.nodeIds);
|
|
if (definition.key === "audio-to-video") {
|
|
params.ports.emit?.("data-canvasAgentUiCommand", {
|
|
projectId: params.projectId,
|
|
requestId: `${params.runId}:${id}`,
|
|
workspace: "video",
|
|
featureId: `video.function.${definition.key}`,
|
|
nodeIds: input.nodeIds,
|
|
});
|
|
return {
|
|
mode: "interactive",
|
|
backendRequired: true,
|
|
message: "音频驱动视频尚缺真实后端，已打开设置入口；没有生成音频驱动视频",
|
|
};
|
|
}
|
|
const images = nodes.filter((node) => videoNodeType(node) === "image");
|
|
const imageUrls = images.map(videoNodeUrl);
|
|
if (images.some((node) => !videoNodeUrl(node))) throw new Error("参考图片还未上传完成");
|
|
const prompt = [definition.prompt, input.prompt].filter(Boolean).join("\n\n");
|
|
if (definition.route === "canvas-generate-video") {
|
|
const mode =
|
|
definition.key === "first-frame-video"
|
|
? "first-frame"
|
|
: definition.key === "first-last-frame-video"
|
|
? "first-last-frame"
|
|
: definition.key === "text-to-video"
|
|
? "text-to-video"
|
|
: resolveVideoMode(params.snapshot.videoSettings!.mode, images.length);
|
|
resolveVideoMode(mode, images.length);
|
|
return params.executable.generateVideoShot.execute(
|
|
{ prompt, referenceNodeIds: images.map((node) => node.id), mode },
|
|
id,
|
|
);
|
|
}
|
|
if (definition.route === "canvas-workflow-analyze") {
|
|
const data = await params.ports.analyze({
|
|
mode:
|
|
definition.key === "story-script"
|
|
? "story-script"
|
|
: nodes.some((node) => videoNodeType(node) === "video")
|
|
? "video-analysis"
|
|
: "agent-plan",
|
|
prompt,
|
|
imageUrls,
|
|
videoUrls: nodes.filter((node) => videoNodeType(node) === "video").map(videoNodeUrl),
|
|
});
|
|
const node = params.makeNode(id, definition.label, "text", {
|
|
content: [String(data.raw || data.summary || "")],
|
|
rows: data.analysisRows || data.shotIdeas || [],
|
|
functionKey: definition.key,
|
|
});
|
|
return {
|
|
status: "succeeded",
|
|
createdNodeIds: [await params.ports.append(node, input.nodeIds)],
|
|
type: "script",
|
|
};
|
|
}
|
|
if (definition.route === "canvas-generate-image") {
|
|
if (definition.kind === "image-edit" && !imageUrls.length)
|
|
throw new Error("这个预设需要先连接一张真实参考图片");
|
|
const result = await params.ports.generateImage({
|
|
prompt,
|
|
references: imageUrls,
|
|
callId: id,
|
|
functionKey: definition.key,
|
|
modelRef: definition.model,
|
|
aspectRatio: /grid|three-view|prediction|lighting-correction/.test(definition.key)
|
|
? "16:9"
|
|
: undefined,
|
|
});
|
|
const grid =
|
|
definition.key === "story-beat-grid-4"
|
|
? 2
|
|
: definition.key === "multi-view-nine-grid"
|
|
? 3
|
|
: definition.key === "continuous-storyboard-grid-25"
|
|
? 5
|
|
: undefined;
|
|
const node = params.makeNode(id, definition.label, "image", {
|
|
mediaUrl: result.url,
|
|
url: [result.url],
|
|
prompt,
|
|
functionKey: definition.key,
|
|
model: result.model,
|
|
...(grid ? { gridRows: grid, gridColumns: grid, nineGridComposite: true } : {}),
|
|
});
|
|
return {
|
|
status: "succeeded",
|
|
createdNodeIds: [await params.ports.append(node, input.nodeIds)],
|
|
type: "image",
|
|
};
|
|
}
|
|
const requiresPanel =
|
|
definition.route === "pick-object" ||
|
|
(definition.key === "image-crop" &&
|
|
!["x", "y", "width", "height"].every((key) => Number.isFinite(input.args[key]))) ||
|
|
(definition.key === "grid-split" && !Number(input.args.rows));
|
|
if (requiresPanel || !params.ports.processImage) {
|
|
params.ports.emit?.("data-canvasAgentUiCommand", {
|
|
projectId: params.projectId,
|
|
requestId: `${params.runId}:${id}`,
|
|
workspace: "video",
|
|
featureId: `video.function.${definition.key}`,
|
|
nodeIds: input.nodeIds,
|
|
});
|
|
return { mode: "interactive", message: "已请求打开参数面板，需要完成参数设置后执行" };
|
|
}
|
|
if (images.length !== 1) throw new Error("请为此编辑功能选择一张真实图片");
|
|
const results = await params.ports.processImage({
|
|
definition,
|
|
imageUrl: imageUrls[0],
|
|
args: input.args,
|
|
prompt,
|
|
callId: id,
|
|
});
|
|
const created: string[] = [];
|
|
for (const [index, result] of results.entries())
|
|
created.push(
|
|
await params.ports.append(
|
|
params.makeNode(`${id}-${index}`, `${definition.label}-${index + 1}`, "image", {
|
|
mediaUrl: result.url,
|
|
url: [result.url],
|
|
functionKey: definition.key,
|
|
prompt,
|
|
}),
|
|
input.nodeIds,
|
|
),
|
|
);
|
|
return { status: "succeeded", createdNodeIds: created, type: "image" };
|
|
},
|
|
);
|
|
}
|