547 lines
17 KiB
TypeScript
547 lines
17 KiB
TypeScript
import { generateObject, type LanguageModel } from "ai";
|
||
import { z } from "zod";
|
||
import type {
|
||
CanvasAgentContext,
|
||
CanvasOperationPlan,
|
||
CanvasViewOperationPlan,
|
||
} from "@/types/canvas/agent-protocol";
|
||
|
||
const plannerDecisionSchema = z.object({
|
||
action: z.enum([
|
||
"none",
|
||
"focus_nodes",
|
||
"fit_all",
|
||
"zoom_in",
|
||
"zoom_out",
|
||
"link_nodes",
|
||
"update_text",
|
||
"confirmation_required",
|
||
]),
|
||
nodeIds: z.array(z.string().min(1).max(160)).max(20).optional(),
|
||
textContent: z.string().max(4000).optional(),
|
||
riskOperation: z
|
||
.enum([
|
||
"replace_node_media",
|
||
"delete_nodes",
|
||
"move_nodes",
|
||
"overwrite_slot",
|
||
"batch_move",
|
||
"other",
|
||
])
|
||
.optional(),
|
||
reply: z.string().max(280).optional(),
|
||
reason: z.string().max(280).optional(),
|
||
});
|
||
|
||
type PlannerDecision = z.infer<typeof plannerDecisionSchema>;
|
||
|
||
export type CanvasAgentModelPlannerOutcome =
|
||
| {
|
||
kind: "view";
|
||
viewPlan: CanvasViewOperationPlan;
|
||
reply: string;
|
||
reason?: string;
|
||
}
|
||
| {
|
||
kind: "operation";
|
||
operationPlan: CanvasOperationPlan;
|
||
viewPlan?: CanvasViewOperationPlan;
|
||
reply: string;
|
||
reason?: string;
|
||
}
|
||
| {
|
||
kind: "confirmation_required";
|
||
riskOperation: string;
|
||
targetNodeIds: string[];
|
||
operationPlan?: CanvasOperationPlan;
|
||
viewPlan?: CanvasViewOperationPlan;
|
||
reply: string;
|
||
reason?: string;
|
||
}
|
||
| {
|
||
kind: "none";
|
||
reason?: string;
|
||
};
|
||
|
||
function trimString(value: unknown, maxLength: number) {
|
||
return typeof value === "string" ? value.trim().slice(0, maxLength) : "";
|
||
}
|
||
|
||
function uniqueStrings(values: Array<string | undefined | null>, maxItems = 20) {
|
||
return Array.from(
|
||
new Set(values.map((value) => trimString(value, 160)).filter(Boolean))
|
||
).slice(0, maxItems);
|
||
}
|
||
|
||
function buildKnownNodeMap(context: CanvasAgentContext) {
|
||
const nodeMap = new Map<string, { nodeId: string; type: string; isSelected?: boolean }>();
|
||
[...context.candidateNodes, ...context.selectedNodes].forEach((node) => {
|
||
nodeMap.set(node.nodeId, {
|
||
nodeId: node.nodeId,
|
||
type: node.type,
|
||
isSelected: node.isSelected,
|
||
});
|
||
});
|
||
context.slots.forEach((slot) => {
|
||
nodeMap.set(slot.nodeId, {
|
||
nodeId: slot.nodeId,
|
||
type: "agent-slot",
|
||
isSelected: slot.isSelected,
|
||
});
|
||
});
|
||
return nodeMap;
|
||
}
|
||
|
||
function getSelectedKnownNodeIds(context: CanvasAgentContext, maxItems = 20) {
|
||
const selectedFromRefs = [
|
||
...context.selectedNodes.map((node) => node.nodeId),
|
||
...context.slots
|
||
.filter((slot) => slot.isSelected || context.selectedNodeIds.includes(slot.nodeId))
|
||
.map((slot) => slot.nodeId),
|
||
];
|
||
return uniqueStrings([...context.selectedNodeIds, ...selectedFromRefs], maxItems);
|
||
}
|
||
|
||
function getDecisionNodeIds(params: {
|
||
decision: PlannerDecision;
|
||
context: CanvasAgentContext;
|
||
allowSelectedFallback: boolean;
|
||
maxItems?: number;
|
||
}) {
|
||
const knownNodes = buildKnownNodeMap(params.context);
|
||
const requestedIds = uniqueStrings(params.decision.nodeIds || [], params.maxItems || 20).filter(
|
||
(nodeId) => knownNodes.has(nodeId)
|
||
);
|
||
if (requestedIds.length > 0 || !params.allowSelectedFallback) {
|
||
return requestedIds;
|
||
}
|
||
return getSelectedKnownNodeIds(params.context, params.maxItems || 20).filter((nodeId) =>
|
||
knownNodes.has(nodeId)
|
||
);
|
||
}
|
||
|
||
function inferHighRiskOperation(userText: string) {
|
||
const text = userText.toLowerCase();
|
||
if (/删除|删掉|移除/.test(text)) return "delete_nodes";
|
||
if (/覆盖|替换当前|替换原图|直接替换|覆盖原图/.test(text)) {
|
||
return "replace_node_media";
|
||
}
|
||
if (/批量移动|整体移动|移动到|移到|整理|排列|排版/.test(text)) {
|
||
return "move_nodes";
|
||
}
|
||
return "";
|
||
}
|
||
|
||
function defaultConfirmationReply(riskOperation: string) {
|
||
if (riskOperation === "delete_nodes") {
|
||
return "这个动作会删除画布节点，需要先确认后才能执行。";
|
||
}
|
||
if (riskOperation === "replace_node_media") {
|
||
return "这个动作会覆盖或替换现有图片，需要先确认后才能执行。";
|
||
}
|
||
if (riskOperation === "move_nodes" || riskOperation === "batch_move") {
|
||
return "这个动作会移动或重排画布节点，需要先确认后才能执行。";
|
||
}
|
||
return "这个画布动作风险较高，需要先确认后才能执行。";
|
||
}
|
||
|
||
function inferMoveLayoutStrategy(userText: string): "row" | "column" | "grid" {
|
||
const text = userText.toLowerCase();
|
||
if (/网格|宫格|矩阵|两列|三列|多列|grid/.test(text)) return "grid";
|
||
if (/竖|纵|列|上下|从上到下|column/.test(text)) return "column";
|
||
return "row";
|
||
}
|
||
|
||
function buildViewPlan(params: {
|
||
action: PlannerDecision["action"];
|
||
context: CanvasAgentContext;
|
||
projectId: string;
|
||
operationId: string;
|
||
nodeIds?: string[];
|
||
reason?: string;
|
||
}): CanvasViewOperationPlan | null {
|
||
if (params.action === "fit_all") {
|
||
return {
|
||
version: 1,
|
||
viewOperationId: `${params.operationId}:view`,
|
||
operation: "fit_all",
|
||
projectId: params.projectId,
|
||
padding: 120,
|
||
persist: false,
|
||
reason: params.reason || "model_planned_fit_all",
|
||
};
|
||
}
|
||
if (params.action === "zoom_in" || params.action === "zoom_out") {
|
||
return {
|
||
version: 1,
|
||
viewOperationId: `${params.operationId}:view`,
|
||
operation: params.action,
|
||
projectId: params.projectId,
|
||
persist: false,
|
||
reason: params.reason || `model_planned_${params.action}`,
|
||
};
|
||
}
|
||
|
||
const nodeIds =
|
||
params.nodeIds && params.nodeIds.length > 0
|
||
? params.nodeIds
|
||
: getSelectedKnownNodeIds(params.context, 10);
|
||
if (nodeIds.length === 0) return null;
|
||
return {
|
||
version: 1,
|
||
viewOperationId: `${params.operationId}:view`,
|
||
operation: nodeIds.length > 1 ? "focus_nodes" : "focus_selection",
|
||
projectId: params.projectId,
|
||
nodeIds,
|
||
anchorNodeId: nodeIds[0],
|
||
padding: 120,
|
||
persist: false,
|
||
reason: params.reason || "model_planned_focus",
|
||
};
|
||
}
|
||
|
||
function buildContextSummary(context: CanvasAgentContext) {
|
||
return {
|
||
projectId: context.projectId,
|
||
canvasRevision: context.canvasRevision,
|
||
selectedNodeIds: context.selectedNodeIds,
|
||
viewport: context.viewport,
|
||
selectedNodes: context.selectedNodes.slice(0, 12).map((node) => ({
|
||
nodeId: node.nodeId,
|
||
type: node.type,
|
||
title: node.title,
|
||
x: node.x,
|
||
y: node.y,
|
||
width: node.width,
|
||
height: node.height,
|
||
})),
|
||
candidateNodes: context.candidateNodes.slice(0, 12).map((node) => ({
|
||
nodeId: node.nodeId,
|
||
type: node.type,
|
||
title: node.title,
|
||
x: node.x,
|
||
y: node.y,
|
||
width: node.width,
|
||
height: node.height,
|
||
})),
|
||
slots: context.slots.slice(0, 12).map((slot) => ({
|
||
nodeId: slot.nodeId,
|
||
slotType: slot.slotType,
|
||
label: slot.label,
|
||
isSelected: slot.isSelected,
|
||
fillPolicy: slot.fillPolicy,
|
||
})),
|
||
annotations: context.annotations.slice(0, 12).map((annotation) => ({
|
||
id: annotation.id,
|
||
sourceNodeId: annotation.sourceNodeId,
|
||
type: annotation.type,
|
||
text: annotation.text,
|
||
})),
|
||
};
|
||
}
|
||
|
||
export function shouldConsiderCanvasAgentModelPlanner(params: {
|
||
userText: string;
|
||
context?: CanvasAgentContext | null;
|
||
}) {
|
||
if (!params.context?.projectId) return false;
|
||
const text = params.userText.trim().toLowerCase();
|
||
if (!text) return false;
|
||
|
||
const hasViewIntent =
|
||
/视角|视野|画布|聚焦|定位|居中|看全局|看整体|看全部|缩小|放大|拉近|拉远|zoom/.test(
|
||
text
|
||
);
|
||
const hasLinkIntent = /连起来|连线|连接|关联|参考关系|版本关系|关系/.test(text);
|
||
const hasTextIntent =
|
||
/文本|文字|文案|标题|标签|改写|润色|优化|改成|改为|设置为|设为|替换为/.test(
|
||
text
|
||
);
|
||
const hasHighRiskIntent = Boolean(inferHighRiskOperation(text));
|
||
const hasMediaGenerationIntent =
|
||
/生成|出图|做一版|再出|换背景|背景|抠图|去水印|高清|扩图|重绘|视频|变体|图层|多角度|洗图/.test(
|
||
text
|
||
);
|
||
|
||
if (
|
||
hasMediaGenerationIntent &&
|
||
!hasViewIntent &&
|
||
!hasLinkIntent &&
|
||
!hasTextIntent &&
|
||
!hasHighRiskIntent
|
||
) {
|
||
return false;
|
||
}
|
||
|
||
return hasViewIntent || hasLinkIntent || hasTextIntent || hasHighRiskIntent;
|
||
}
|
||
|
||
export function normalizeCanvasAgentModelPlannerDecision(params: {
|
||
decision: unknown;
|
||
context: CanvasAgentContext;
|
||
projectId: string;
|
||
messageId: string;
|
||
chatId: string;
|
||
userText: string;
|
||
}): CanvasAgentModelPlannerOutcome {
|
||
const parsed = plannerDecisionSchema.safeParse(params.decision);
|
||
if (!parsed.success) {
|
||
return { kind: "none", reason: "invalid_model_planner_decision" };
|
||
}
|
||
|
||
const decision = parsed.data;
|
||
const reason = trimString(decision.reason, 280) || undefined;
|
||
const operationId = `${params.messageId}:canvas-model-${decision.action}`;
|
||
const explicitRisk = trimString(decision.riskOperation, 80);
|
||
const inferredRisk = inferHighRiskOperation(params.userText);
|
||
const riskOperation =
|
||
decision.action === "confirmation_required"
|
||
? explicitRisk || inferredRisk || "other"
|
||
: inferredRisk;
|
||
if (riskOperation) {
|
||
const targetNodeIds = getDecisionNodeIds({
|
||
decision,
|
||
context: params.context,
|
||
allowSelectedFallback: true,
|
||
maxItems: 20,
|
||
});
|
||
const canBuildMovePlan =
|
||
(riskOperation === "move_nodes" || riskOperation === "batch_move") &&
|
||
targetNodeIds.length > 0;
|
||
const operationPlan: CanvasOperationPlan | undefined = canBuildMovePlan
|
||
? {
|
||
version: 1,
|
||
operationId: operationId.replace(`canvas-model-${decision.action}`, "canvas-move"),
|
||
operation: "move_nodes",
|
||
projectId: params.projectId,
|
||
baseCanvasRevision: params.context.canvasRevision,
|
||
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
|
||
targetNodeIds,
|
||
layoutStrategy: inferMoveLayoutStrategy(params.userText),
|
||
spacing: 40,
|
||
preserveOriginal: true,
|
||
metadata: {
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
planner: "model",
|
||
prompt: params.userText,
|
||
pendingConfirmation: true,
|
||
riskOperation,
|
||
},
|
||
}
|
||
: undefined;
|
||
return {
|
||
kind: "confirmation_required",
|
||
riskOperation,
|
||
targetNodeIds,
|
||
operationPlan,
|
||
viewPlan: operationPlan
|
||
? buildViewPlan({
|
||
action: "focus_nodes",
|
||
context: params.context,
|
||
projectId: params.projectId,
|
||
operationId: operationPlan.operationId,
|
||
nodeIds: targetNodeIds,
|
||
reason: "focus_pending_move_nodes",
|
||
}) || undefined
|
||
: undefined,
|
||
reply: trimString(decision.reply, 280) || defaultConfirmationReply(riskOperation),
|
||
reason,
|
||
};
|
||
}
|
||
|
||
if (decision.action === "none") {
|
||
return { kind: "none", reason };
|
||
}
|
||
|
||
if (
|
||
decision.action === "focus_nodes" ||
|
||
decision.action === "fit_all" ||
|
||
decision.action === "zoom_in" ||
|
||
decision.action === "zoom_out"
|
||
) {
|
||
const nodeIds = getDecisionNodeIds({
|
||
decision,
|
||
context: params.context,
|
||
allowSelectedFallback: decision.action === "focus_nodes",
|
||
maxItems: 10,
|
||
});
|
||
const viewPlan = buildViewPlan({
|
||
action: decision.action,
|
||
context: params.context,
|
||
projectId: params.projectId,
|
||
operationId,
|
||
nodeIds,
|
||
reason,
|
||
});
|
||
if (!viewPlan) return { kind: "none", reason: "missing_view_target" };
|
||
return {
|
||
kind: "view",
|
||
viewPlan,
|
||
reply:
|
||
trimString(decision.reply, 280) ||
|
||
(decision.action === "fit_all"
|
||
? "已切到画布全局视图。"
|
||
: decision.action === "zoom_in"
|
||
? "已放大画布视角。"
|
||
: decision.action === "zoom_out"
|
||
? "已缩小画布视角。"
|
||
: "已聚焦到当前画布对象。"),
|
||
reason,
|
||
};
|
||
}
|
||
|
||
if (decision.action === "link_nodes") {
|
||
const nodeIds = getDecisionNodeIds({
|
||
decision,
|
||
context: params.context,
|
||
allowSelectedFallback: true,
|
||
maxItems: 20,
|
||
});
|
||
if (nodeIds.length < 2) {
|
||
return { kind: "none", reason: "link_nodes_requires_two_targets" };
|
||
}
|
||
const operationPlan: CanvasOperationPlan = {
|
||
version: 1,
|
||
operationId,
|
||
operation: "link_nodes",
|
||
projectId: params.projectId,
|
||
baseCanvasRevision: params.context.canvasRevision,
|
||
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
|
||
sourceNodeIds: nodeIds,
|
||
edgeType: "reference",
|
||
preserveOriginal: true,
|
||
metadata: {
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
planner: "model",
|
||
prompt: params.userText,
|
||
},
|
||
};
|
||
return {
|
||
kind: "operation",
|
||
operationPlan,
|
||
viewPlan: buildViewPlan({
|
||
action: "focus_nodes",
|
||
context: params.context,
|
||
projectId: params.projectId,
|
||
operationId,
|
||
nodeIds,
|
||
reason: "focus_model_linked_nodes",
|
||
}) || undefined,
|
||
reply: trimString(decision.reply, 280) || "已为选中的画布节点建立参考连线。",
|
||
reason,
|
||
};
|
||
}
|
||
|
||
if (decision.action === "update_text") {
|
||
const knownNodes = buildKnownNodeMap(params.context);
|
||
const requestedIds = getDecisionNodeIds({
|
||
decision,
|
||
context: params.context,
|
||
allowSelectedFallback: true,
|
||
maxItems: 10,
|
||
});
|
||
const targetNodeIds = requestedIds.filter(
|
||
(nodeId) => knownNodes.get(nodeId)?.type === "text"
|
||
);
|
||
const textContent = trimString(decision.textContent, 4000);
|
||
if (targetNodeIds.length === 0 || !textContent) {
|
||
return { kind: "none", reason: "update_text_missing_target_or_content" };
|
||
}
|
||
const operationPlan: CanvasOperationPlan = {
|
||
version: 1,
|
||
operationId,
|
||
operation: "update_text",
|
||
projectId: params.projectId,
|
||
baseCanvasRevision: params.context.canvasRevision,
|
||
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
|
||
targetNodeIds,
|
||
textContent,
|
||
preserveOriginal: false,
|
||
metadata: {
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
planner: "model",
|
||
prompt: params.userText,
|
||
},
|
||
};
|
||
return {
|
||
kind: "operation",
|
||
operationPlan,
|
||
viewPlan: buildViewPlan({
|
||
action: "focus_nodes",
|
||
context: params.context,
|
||
projectId: params.projectId,
|
||
operationId,
|
||
nodeIds: targetNodeIds,
|
||
reason: "focus_model_updated_text_nodes",
|
||
}) || undefined,
|
||
reply: trimString(decision.reply, 280) || "已更新选中的文本节点。",
|
||
reason,
|
||
};
|
||
}
|
||
|
||
return { kind: "none", reason: "unsupported_model_planner_action" };
|
||
}
|
||
|
||
export async function planCanvasAgentOperationWithModel(params: {
|
||
model: LanguageModel;
|
||
userText: string;
|
||
context: CanvasAgentContext;
|
||
projectId: string;
|
||
messageId: string;
|
||
chatId: string;
|
||
abortSignal?: AbortSignal;
|
||
}): Promise<CanvasAgentModelPlannerOutcome> {
|
||
if (
|
||
!shouldConsiderCanvasAgentModelPlanner({
|
||
userText: params.userText,
|
||
context: params.context,
|
||
})
|
||
) {
|
||
return { kind: "none", reason: "planner_not_relevant" };
|
||
}
|
||
|
||
const { object } = await generateObject({
|
||
model: params.model,
|
||
schema: plannerDecisionSchema,
|
||
temperature: 0,
|
||
abortSignal: params.abortSignal || AbortSignal.timeout(18_000),
|
||
system: [
|
||
"你是无限画布 Agent 的受控操作规划器。",
|
||
"你只能规划低风险画布操作：视角聚焦/缩放/看全局、连接节点、更新文本节点。",
|
||
"图片/视频生成、换背景、抠图、高清、扩图、重绘等媒体任务必须返回 action=none，交给其他执行器。",
|
||
"删除、覆盖、替换现有图片、移动/重排节点等高风险动作必须返回 action=confirmation_required。",
|
||
"只能使用上下文中出现的 nodeId。不要编造节点 ID。",
|
||
"如果用户只是聊天或目标不明确，返回 action=none。",
|
||
].join("\n"),
|
||
prompt: JSON.stringify(
|
||
{
|
||
userText: params.userText,
|
||
context: buildContextSummary(params.context),
|
||
outputContract: {
|
||
action:
|
||
"none | focus_nodes | fit_all | zoom_in | zoom_out | link_nodes | update_text | confirmation_required",
|
||
nodeIds: "只填上下文存在的 nodeId",
|
||
textContent: "仅 update_text 时填写最终文本",
|
||
riskOperation:
|
||
"confirmation_required 时填写 replace_node_media/delete_nodes/move_nodes/overwrite_slot/batch_move/other",
|
||
reply: "一句给用户看的中文短回复",
|
||
},
|
||
},
|
||
null,
|
||
2
|
||
),
|
||
});
|
||
|
||
return normalizeCanvasAgentModelPlannerDecision({
|
||
decision: object,
|
||
context: params.context,
|
||
projectId: params.projectId,
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
userText: params.userText,
|
||
});
|
||
}
|