226 lines
8.4 KiB
TypeScript
226 lines
8.4 KiB
TypeScript
import { tool, type ToolSet } from "ai";
|
|
import { z } from "zod";
|
|
import { applyCanvasOperationPlan } from "@/lib/canvas/agent-mutations";
|
|
import { readOwnedCanvas } from "./canvas";
|
|
import {
|
|
getCanvasAgentAccessManifest,
|
|
IMAGE_CANVAS_OPERATIONS,
|
|
isExplicitDestructiveIntent,
|
|
summarizeAgentCapabilities,
|
|
} from "./capability-access";
|
|
import type { CanvasAgentTool } from "./loop";
|
|
import type { CanvasOperationPlan } from "@/types/canvas/agent-protocol";
|
|
|
|
export function registerImageCanvasAccessTools(params: {
|
|
userId: string;
|
|
projectId: string;
|
|
runId: string;
|
|
selectedNodeIds: string[];
|
|
objective?: string;
|
|
definitions: ToolSet;
|
|
executable: Record<string, CanvasAgentTool>;
|
|
emit: (type: string, data: unknown) => void;
|
|
}) {
|
|
function register<T extends z.ZodTypeAny>(
|
|
name: string,
|
|
title: string,
|
|
schema: T,
|
|
execute: (input: z.infer<T>, id: string) => Promise<Record<string, unknown>>,
|
|
readOnly = false,
|
|
) {
|
|
params.definitions[name] = tool({ description: title, inputSchema: schema });
|
|
params.executable[name] = {
|
|
title,
|
|
readOnly,
|
|
concurrentSafe: readOnly,
|
|
execute: (input, id) => execute(schema.parse(input), id),
|
|
};
|
|
}
|
|
register(
|
|
"listCanvasCapabilities",
|
|
"列出图片画布全部已授权的功能、交互入口和限制。权限与工具栏是否显示独立。",
|
|
z.object({}),
|
|
async () => ({
|
|
capabilities: summarizeAgentCapabilities(
|
|
getCanvasAgentAccessManifest().filter((item) => item.workspace === "canvas"),
|
|
),
|
|
}),
|
|
true,
|
|
);
|
|
register(
|
|
"openCanvasFeature",
|
|
"打开图片画布真实工具栏或专业工作区入口。只代表打开入口；交互、选文件和收费提交由用户完成，不代表媒体处理完成。",
|
|
z.object({
|
|
featureId: z.string().min(1).max(180),
|
|
nodeIds: z.array(z.string().min(1).max(180)).max(50).default([]),
|
|
}),
|
|
async (input, id) => {
|
|
const feature = getCanvasAgentAccessManifest().find(
|
|
(item) => item.workspace === "canvas" && item.id === input.featureId,
|
|
);
|
|
if (!feature?.authorized || !feature.id.startsWith("canvas.ui."))
|
|
throw new Error("请使用清单中的图片画布界面入口，媒体能力请使用对应专用工具");
|
|
const { nodes } = await readOwnedCanvas(params.userId, params.projectId);
|
|
if (input.nodeIds.some((nodeId) => !nodes.some((node) => node.id === nodeId)))
|
|
throw new Error("目标节点不属于当前画布");
|
|
params.emit("data-canvasAgentUiCommand", {
|
|
projectId: params.projectId,
|
|
requestId: `${params.runId}:${id}`,
|
|
workspace: "canvas",
|
|
featureId: feature.id,
|
|
nodeIds: input.nodeIds,
|
|
});
|
|
return {
|
|
mode: "interactive",
|
|
featureId: feature.id,
|
|
message: "已请求打开界面入口，等待用户操作；不能声称处理完成",
|
|
};
|
|
},
|
|
);
|
|
register(
|
|
"editCanvasNodes",
|
|
"创建文本或图片槽位、填槽、连线、移动、改名、批注、改文字、追加版本及删除节点。删除或覆盖只在用户明确要求、目标已选中时执行。保留原图优先使用追加版本。",
|
|
z.object({
|
|
operation: z.enum(IMAGE_CANVAS_OPERATIONS),
|
|
nodeIds: z.array(z.string().min(1).max(180)).max(50).default([]),
|
|
sourceNodeIds: z.array(z.string().min(1).max(180)).max(20).default([]),
|
|
targetSlotId: z.string().optional(),
|
|
text: z.string().max(8000).optional(),
|
|
position: z.object({ x: z.number().finite(), y: z.number().finite() }).optional(),
|
|
slot: z
|
|
.object({
|
|
slotType: z.enum(["image", "video", "ui-screen"]),
|
|
aspectRatio: z.string().max(24).optional(),
|
|
label: z.string().max(120).optional(),
|
|
fillPolicy: z.enum(["replace-empty", "append-version", "replace-existing"]).optional(),
|
|
})
|
|
.optional(),
|
|
annotations: z
|
|
.array(
|
|
z.object({
|
|
type: z.enum(["text-note", "arrow-note", "box", "freehand", "crop-box"]),
|
|
text: z.string().max(1000).optional(),
|
|
bounds: z
|
|
.object({
|
|
x: z.number(),
|
|
y: z.number(),
|
|
width: z.number().positive(),
|
|
height: z.number().positive(),
|
|
})
|
|
.optional(),
|
|
}),
|
|
)
|
|
.max(20)
|
|
.optional(),
|
|
}),
|
|
async (input, id) => {
|
|
const { nodes } = await readOwnedCanvas(params.userId, params.projectId);
|
|
const all = [
|
|
...input.nodeIds,
|
|
...input.sourceNodeIds,
|
|
...(input.targetSlotId ? [input.targetSlotId] : []),
|
|
];
|
|
if (all.some((nodeId) => !nodes.some((node) => node.id === nodeId)))
|
|
throw new Error("目标或来源节点不属于当前画布");
|
|
const destructive =
|
|
input.operation === "delete_nodes" ||
|
|
input.operation === "replace_node_media" ||
|
|
input.slot?.fillPolicy === "replace-existing";
|
|
if (
|
|
destructive &&
|
|
(!isExplicitDestructiveIntent(params.objective || "") ||
|
|
!input.nodeIds.length ||
|
|
input.nodeIds.some((nodeId) => !params.selectedNodeIds.includes(nodeId)))
|
|
)
|
|
throw new Error("删除或覆盖需要用户明确提出，并选中具体目标；模型参数不能代替用户确认");
|
|
const plan: CanvasOperationPlan = {
|
|
version: 1,
|
|
projectId: params.projectId,
|
|
operationId: `${params.runId}:${id}`,
|
|
operation: input.operation,
|
|
targetNodeIds: input.nodeIds,
|
|
sourceNodeIds: input.sourceNodeIds,
|
|
targetSlotId: input.targetSlotId,
|
|
textContent: input.text,
|
|
targetPosition: input.position,
|
|
slot: input.slot,
|
|
annotations: input.annotations,
|
|
preserveOriginal: !destructive,
|
|
confirmed: destructive,
|
|
placement: "right",
|
|
};
|
|
if (
|
|
["insert_beside", "append_revision", "replace_node_media", "fill_slot"].includes(
|
|
input.operation,
|
|
)
|
|
) {
|
|
if (!input.sourceNodeIds.length) throw new Error("请提供真实已存在的媒体来源节点");
|
|
plan.mediaItems = input.sourceNodeIds.map((nodeId, index) => {
|
|
const source = nodes.find((node) => node.id === nodeId)!;
|
|
if (
|
|
!["image", "video", "model3d", "world3d"].includes(source.type) ||
|
|
!/^https?:\/\//i.test(source.data.src || source.data.url || "")
|
|
)
|
|
throw new Error("来源节点缺少有效媒体，不能编造链接");
|
|
return {
|
|
mediaItemId: `${id}:${index}`,
|
|
type: source.type as "image" | "video" | "model3d" | "world3d",
|
|
url: source.data.src || source.data.url!,
|
|
sourceNodeIds: [nodeId],
|
|
};
|
|
});
|
|
}
|
|
const result = await applyCanvasOperationPlan({
|
|
userId: params.userId,
|
|
projectId: params.projectId,
|
|
plan,
|
|
});
|
|
params.emit("data-canvasOperation", {
|
|
projectId: params.projectId,
|
|
operationId: plan.operationId,
|
|
operation: input.operation,
|
|
status: "applied",
|
|
...result,
|
|
});
|
|
const { canvasData: _canvasData, ...summary } = result;
|
|
return { status: "succeeded", operation: input.operation, ...summary };
|
|
},
|
|
);
|
|
register(
|
|
"controlCanvasView",
|
|
"缩放、平移、定位节点或显示所有元素，不修改图片内容。",
|
|
z.object({
|
|
operation: z.enum([
|
|
"focus_node",
|
|
"focus_nodes",
|
|
"focus_selection",
|
|
"focus_latest_result",
|
|
"fit_all",
|
|
"zoom_in",
|
|
"zoom_out",
|
|
"zoom_to",
|
|
"pan_to",
|
|
]),
|
|
nodeIds: z.array(z.string().min(1).max(180)).max(50).default([]),
|
|
zoom: z.number().min(0.05).max(8).optional(),
|
|
x: z.number().optional(),
|
|
y: z.number().optional(),
|
|
}),
|
|
async (input, id) => {
|
|
const { nodes } = await readOwnedCanvas(params.userId, params.projectId);
|
|
if (input.nodeIds.some((nodeId) => !nodes.some((node) => node.id === nodeId)))
|
|
throw new Error("视图目标不属于当前画布");
|
|
params.emit("data-canvasViewOperation", {
|
|
version: 1,
|
|
projectId: params.projectId,
|
|
viewOperationId: `${params.runId}:${id}`,
|
|
operation: input.operation,
|
|
nodeIds: input.nodeIds,
|
|
zoom: input.zoom,
|
|
center: { x: input.x || 0, y: input.y || 0 },
|
|
});
|
|
return { status: "succeeded", mode: "view", message: "已发送画布视图操作" };
|
|
},
|
|
);
|
|
}
|