1043 lines
33 KiB
TypeScript
1043 lines
33 KiB
TypeScript
import { generateObject, type LanguageModel } from "ai";
|
||
import { z } from "zod";
|
||
import type {
|
||
CanvasAgentContext,
|
||
CanvasOperationPlan,
|
||
CanvasViewOperationPlan,
|
||
} from "@/types/canvas/agent-protocol";
|
||
import { hasCanvasAgentComparisonViewIntent } from "@/lib/canvas/agent-view-intent";
|
||
import { hasCanvasAgentPanViewIntent } from "@/lib/canvas/agent-view-pan";
|
||
|
||
const plannerDecisionSchema = z.object({
|
||
action: z.enum([
|
||
"none",
|
||
"focus_nodes",
|
||
"fit_all",
|
||
"zoom_in",
|
||
"zoom_out",
|
||
"link_nodes",
|
||
"create_text_node",
|
||
"update_text",
|
||
"rename_node",
|
||
"update_note",
|
||
"add_annotation",
|
||
"create_agent_slot",
|
||
"confirmation_required",
|
||
]),
|
||
nodeIds: z.array(z.string().min(1).max(160)).max(20).optional(),
|
||
textContent: z.string().max(4000).optional(),
|
||
slotType: z.enum(["image", "video", "ui-screen"]).optional(),
|
||
slotLabel: z.string().max(120).optional(),
|
||
aspectRatio: z.string().max(32).optional(),
|
||
riskOperation: z
|
||
.enum([
|
||
"replace_node_media",
|
||
"delete_nodes",
|
||
"move_nodes",
|
||
"overwrite_slot",
|
||
"batch_move",
|
||
"other",
|
||
])
|
||
.optional(),
|
||
reply: z.string().max(280).optional(),
|
||
reason: z.string().max(280).optional(),
|
||
});
|
||
|
||
type PlannerDecision = z.infer<typeof plannerDecisionSchema>;
|
||
|
||
export type CanvasAgentSlotCreationIntent = {
|
||
shouldCreate: boolean;
|
||
combinedWithMediaExecution: boolean;
|
||
slotType: "image" | "video" | "ui-screen";
|
||
label: string;
|
||
aspectRatio: string;
|
||
};
|
||
|
||
export type CanvasAgentModelPlannerOutcome =
|
||
| {
|
||
kind: "view";
|
||
viewPlan: CanvasViewOperationPlan;
|
||
reply: string;
|
||
reason?: string;
|
||
}
|
||
| {
|
||
kind: "operation";
|
||
operationPlan: CanvasOperationPlan;
|
||
viewPlan?: CanvasViewOperationPlan;
|
||
reply: string;
|
||
reason?: string;
|
||
}
|
||
| {
|
||
kind: "confirmation_required";
|
||
riskOperation: string;
|
||
targetNodeIds: string[];
|
||
operationPlan?: CanvasOperationPlan;
|
||
viewPlan?: CanvasViewOperationPlan;
|
||
reply: string;
|
||
reason?: string;
|
||
}
|
||
| {
|
||
kind: "none";
|
||
reason?: string;
|
||
};
|
||
|
||
function trimString(value: unknown, maxLength: number) {
|
||
return typeof value === "string" ? value.trim().slice(0, maxLength) : "";
|
||
}
|
||
|
||
function uniqueStrings(values: Array<string | undefined | null>, maxItems = 20) {
|
||
return Array.from(
|
||
new Set(values.map((value) => trimString(value, 160)).filter(Boolean))
|
||
).slice(0, maxItems);
|
||
}
|
||
|
||
function buildKnownNodeMap(context: CanvasAgentContext) {
|
||
const nodeMap = new Map<string, { nodeId: string; type: string; isSelected?: boolean }>();
|
||
[...context.candidateNodes, ...context.selectedNodes].forEach((node) => {
|
||
nodeMap.set(node.nodeId, {
|
||
nodeId: node.nodeId,
|
||
type: node.type,
|
||
isSelected: node.isSelected,
|
||
});
|
||
});
|
||
context.slots.forEach((slot) => {
|
||
nodeMap.set(slot.nodeId, {
|
||
nodeId: slot.nodeId,
|
||
type: "agent-slot",
|
||
isSelected: slot.isSelected,
|
||
});
|
||
});
|
||
return nodeMap;
|
||
}
|
||
|
||
function getSelectedKnownNodeIds(context: CanvasAgentContext, maxItems = 20) {
|
||
const selectedFromRefs = [
|
||
...context.selectedNodes.map((node) => node.nodeId),
|
||
...context.slots
|
||
.filter((slot) => slot.isSelected || context.selectedNodeIds.includes(slot.nodeId))
|
||
.map((slot) => slot.nodeId),
|
||
];
|
||
return uniqueStrings([...context.selectedNodeIds, ...selectedFromRefs], maxItems);
|
||
}
|
||
|
||
function getDecisionNodeIds(params: {
|
||
decision: PlannerDecision;
|
||
context: CanvasAgentContext;
|
||
allowSelectedFallback: boolean;
|
||
maxItems?: number;
|
||
}) {
|
||
const knownNodes = buildKnownNodeMap(params.context);
|
||
const requestedIds = uniqueStrings(params.decision.nodeIds || [], params.maxItems || 20).filter(
|
||
(nodeId) => knownNodes.has(nodeId)
|
||
);
|
||
if (requestedIds.length > 0 || !params.allowSelectedFallback) {
|
||
return requestedIds;
|
||
}
|
||
return getSelectedKnownNodeIds(params.context, params.maxItems || 20).filter((nodeId) =>
|
||
knownNodes.has(nodeId)
|
||
);
|
||
}
|
||
|
||
function inferHighRiskOperation(userText: string) {
|
||
const text = userText.toLowerCase();
|
||
if (
|
||
/删除背景|删掉背景|去掉背景|去背景|移除背景|抠图|去背/.test(text)
|
||
) {
|
||
return "";
|
||
}
|
||
if (
|
||
/删除|删掉|删了|删去|移除/.test(text) ||
|
||
/清空(?:画布|选中|当前|节点|对象|元素|这些|全部)/.test(text) ||
|
||
/清除(?:选中|当前|节点|对象|元素|这些|全部)/.test(text)
|
||
) {
|
||
return "delete_nodes";
|
||
}
|
||
if (/覆盖|替换当前|替换原图|直接替换|覆盖原图/.test(text)) {
|
||
return "replace_node_media";
|
||
}
|
||
if (hasCanvasViewMovementIntent(text)) {
|
||
return "";
|
||
}
|
||
if (/批量移动|整体移动|移动到|移到|整理|排列|排版/.test(text)) {
|
||
return "move_nodes";
|
||
}
|
||
return "";
|
||
}
|
||
|
||
function hasCanvasViewMovementIntent(userText: string) {
|
||
const text = userText.toLowerCase();
|
||
return (
|
||
/(?:移动|平移|拖动|拖拽|挪动)(?:一下|一点)?(?:画布|视角|视野|镜头|视图|视口)/.test(
|
||
text
|
||
) ||
|
||
/(?:视野|视口|屏幕|画面|镜头|视图|窗口).{0,8}(?:中央|中心|中间)/.test(
|
||
text
|
||
) ||
|
||
/(?:居中显示|居中查看|居中看|聚焦到|定位到|拉近到|带我看)/.test(text)
|
||
);
|
||
}
|
||
|
||
function defaultConfirmationReply(riskOperation: string) {
|
||
if (riskOperation === "delete_nodes") {
|
||
return "这个动作会删除画布节点，需要先确认后才能执行。";
|
||
}
|
||
if (riskOperation === "replace_node_media") {
|
||
return "这个动作会覆盖或替换现有图片，需要先确认后才能执行。";
|
||
}
|
||
if (riskOperation === "move_nodes" || riskOperation === "batch_move") {
|
||
return "这个动作会移动或重排画布节点，需要先确认后才能执行。";
|
||
}
|
||
return "这个画布动作风险较高，需要先确认后才能执行。";
|
||
}
|
||
|
||
function inferMoveLayoutStrategy(userText: string): "row" | "column" | "grid" {
|
||
const text = userText.toLowerCase();
|
||
if (/网格|宫格|矩阵|两列|三列|多列|grid/.test(text)) return "grid";
|
||
if (/竖|纵|列|上下|从上到下|column/.test(text)) return "column";
|
||
return "row";
|
||
}
|
||
|
||
function hasCombinedMediaExecutionIntent(userText: string) {
|
||
return /并生成|然后生成|填充|出图|做一版|再出|换背景|抠图|去水印|高清|扩图|重绘|变体|图层|多角度|洗图/.test(
|
||
userText.toLowerCase()
|
||
);
|
||
}
|
||
|
||
function inferAgentSlotType(
|
||
userText: string,
|
||
modelSlotType?: "image" | "video" | "ui-screen"
|
||
): "image" | "video" | "ui-screen" {
|
||
if (modelSlotType === "video" || modelSlotType === "ui-screen") {
|
||
return modelSlotType;
|
||
}
|
||
const text = userText.toLowerCase();
|
||
if (/视频|video/.test(text)) return "video";
|
||
if (/ui|界面|页面|画面|screen/.test(text)) return "ui-screen";
|
||
return "image";
|
||
}
|
||
|
||
function getAgentSlotDefaults(slotType: "image" | "video" | "ui-screen") {
|
||
if (slotType === "video") {
|
||
return {
|
||
label: "视频槽位",
|
||
aspectRatio: "16:9",
|
||
size: { width: 360, height: 220 },
|
||
};
|
||
}
|
||
if (slotType === "ui-screen") {
|
||
return {
|
||
label: "UI 槽位",
|
||
aspectRatio: "16:9",
|
||
size: { width: 360, height: 260 },
|
||
};
|
||
}
|
||
return {
|
||
label: "图片槽位",
|
||
aspectRatio: "1:1",
|
||
size: { width: 320, height: 320 },
|
||
};
|
||
}
|
||
|
||
function inferAspectRatioFromText(userText: string) {
|
||
const match = userText.match(/\b(?:1:1|4:3|3:4|16:9|9:16)\b/);
|
||
return match?.[0] || "";
|
||
}
|
||
|
||
export function resolveCanvasAgentSlotCreationIntent(params: {
|
||
userText: string;
|
||
slotType?: "image" | "video" | "ui-screen";
|
||
slotLabel?: string;
|
||
aspectRatio?: string;
|
||
}): CanvasAgentSlotCreationIntent {
|
||
const text = params.userText.trim();
|
||
const lowerText = text.toLowerCase();
|
||
const hasCreationVerb =
|
||
/创建|新增|新建|添加|放一个|加一个|建一个|开一个|create|add|new/.test(
|
||
lowerText
|
||
);
|
||
const hasSlotWord = /slot|槽位|空框|占位|holder/.test(lowerText);
|
||
const slotType = inferAgentSlotType(text, params.slotType);
|
||
const defaults = getAgentSlotDefaults(slotType);
|
||
return {
|
||
shouldCreate: hasCreationVerb && hasSlotWord,
|
||
combinedWithMediaExecution: hasCombinedMediaExecutionIntent(text),
|
||
slotType,
|
||
label: trimString(params.slotLabel, 120) || defaults.label,
|
||
aspectRatio:
|
||
trimString(params.aspectRatio, 32) ||
|
||
inferAspectRatioFromText(text) ||
|
||
defaults.aspectRatio,
|
||
};
|
||
}
|
||
|
||
function getViewportCenteredSlotPosition(params: {
|
||
context: CanvasAgentContext;
|
||
size: { width: number; height: number };
|
||
}) {
|
||
const viewport = params.context.viewport || { x: 0, y: 0, zoom: 1 };
|
||
const zoom = Math.max(0.1, Number(viewport.zoom || 1));
|
||
const viewportWidth = Math.max(320, Number(viewport.width || 1200));
|
||
const viewportHeight = Math.max(320, Number(viewport.height || 800));
|
||
return {
|
||
x: (viewportWidth / 2 - Number(viewport.x || 0)) / zoom - params.size.width / 2,
|
||
y: (viewportHeight / 2 - Number(viewport.y || 0)) / zoom - params.size.height / 2,
|
||
};
|
||
}
|
||
|
||
export function buildCanvasAgentSlotCreationPlan(params: {
|
||
context: CanvasAgentContext;
|
||
projectId: string;
|
||
messageId: string;
|
||
chatId: string;
|
||
userText: string;
|
||
operationId?: string;
|
||
slotType?: "image" | "video" | "ui-screen";
|
||
slotLabel?: string;
|
||
aspectRatio?: string;
|
||
}): CanvasOperationPlan | null {
|
||
const intent = resolveCanvasAgentSlotCreationIntent({
|
||
userText: params.userText,
|
||
slotType: params.slotType,
|
||
slotLabel: params.slotLabel,
|
||
aspectRatio: params.aspectRatio,
|
||
});
|
||
if (!intent.shouldCreate) return null;
|
||
|
||
const defaults = getAgentSlotDefaults(intent.slotType);
|
||
return {
|
||
version: 1,
|
||
operationId: params.operationId || `${params.messageId}:canvas-create-agent-slot`,
|
||
operation: "create_agent_slot",
|
||
projectId: params.projectId,
|
||
baseCanvasRevision: params.context.canvasRevision,
|
||
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
|
||
targetPosition: getViewportCenteredSlotPosition({
|
||
context: params.context,
|
||
size: defaults.size,
|
||
}),
|
||
slot: {
|
||
slotType: intent.slotType,
|
||
label: intent.label,
|
||
aspectRatio: intent.aspectRatio,
|
||
fillPolicy: "replace-empty",
|
||
acceptedSources: ["text", "image", "selection", "annotation"],
|
||
},
|
||
preserveOriginal: true,
|
||
metadata: {
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
planner: "model",
|
||
prompt: params.userText,
|
||
},
|
||
};
|
||
}
|
||
|
||
function hasCanvasAgentSlotCreationIntent(userText: string) {
|
||
return resolveCanvasAgentSlotCreationIntent({ userText }).shouldCreate;
|
||
}
|
||
|
||
function hasCanvasAgentTextCreationIntent(userText: string) {
|
||
const text = userText.trim().toLowerCase();
|
||
if (!text) return false;
|
||
const hasCreateVerb =
|
||
/创建|新增|新建|添加|加一段|加一句|写一段|写一句|放一段|放一句|create|add|new/.test(
|
||
text
|
||
) || /在(?:画布|这里|当前|旁边).*写|写到画布|写在画布|写上/.test(text);
|
||
const hasTextWord = /文本|文字|文案|标题|标语|说明|caption|copy|text/.test(text);
|
||
return hasCreateVerb && hasTextWord;
|
||
}
|
||
|
||
function buildViewPlan(params: {
|
||
action: PlannerDecision["action"];
|
||
context: CanvasAgentContext;
|
||
projectId: string;
|
||
operationId: string;
|
||
nodeIds?: string[];
|
||
reason?: string;
|
||
}): CanvasViewOperationPlan | null {
|
||
if (params.action === "fit_all") {
|
||
return {
|
||
version: 1,
|
||
viewOperationId: `${params.operationId}:view`,
|
||
operation: "fit_all",
|
||
projectId: params.projectId,
|
||
padding: 120,
|
||
persist: false,
|
||
reason: params.reason || "model_planned_fit_all",
|
||
};
|
||
}
|
||
if (params.action === "zoom_in" || params.action === "zoom_out") {
|
||
return {
|
||
version: 1,
|
||
viewOperationId: `${params.operationId}:view`,
|
||
operation: params.action,
|
||
projectId: params.projectId,
|
||
persist: false,
|
||
reason: params.reason || `model_planned_${params.action}`,
|
||
};
|
||
}
|
||
|
||
const nodeIds =
|
||
params.nodeIds && params.nodeIds.length > 0
|
||
? params.nodeIds
|
||
: getSelectedKnownNodeIds(params.context, 10);
|
||
if (nodeIds.length === 0) return null;
|
||
return {
|
||
version: 1,
|
||
viewOperationId: `${params.operationId}:view`,
|
||
operation: nodeIds.length > 1 ? "focus_nodes" : "focus_selection",
|
||
projectId: params.projectId,
|
||
nodeIds,
|
||
anchorNodeId: nodeIds[0],
|
||
padding: 120,
|
||
persist: false,
|
||
reason: params.reason || "model_planned_focus",
|
||
};
|
||
}
|
||
|
||
function buildContextSummary(context: CanvasAgentContext) {
|
||
return {
|
||
projectId: context.projectId,
|
||
canvasRevision: context.canvasRevision,
|
||
selectedNodeIds: context.selectedNodeIds,
|
||
viewport: context.viewport,
|
||
selectedNodes: context.selectedNodes.slice(0, 12).map((node) => ({
|
||
nodeId: node.nodeId,
|
||
type: node.type,
|
||
title: node.title,
|
||
x: node.x,
|
||
y: node.y,
|
||
width: node.width,
|
||
height: node.height,
|
||
})),
|
||
candidateNodes: context.candidateNodes.slice(0, 12).map((node) => ({
|
||
nodeId: node.nodeId,
|
||
type: node.type,
|
||
title: node.title,
|
||
x: node.x,
|
||
y: node.y,
|
||
width: node.width,
|
||
height: node.height,
|
||
})),
|
||
slots: context.slots.slice(0, 12).map((slot) => ({
|
||
nodeId: slot.nodeId,
|
||
slotType: slot.slotType,
|
||
label: slot.label,
|
||
isSelected: slot.isSelected,
|
||
fillPolicy: slot.fillPolicy,
|
||
})),
|
||
annotations: context.annotations.slice(0, 12).map((annotation) => ({
|
||
id: annotation.id,
|
||
sourceNodeId: annotation.sourceNodeId,
|
||
type: annotation.type,
|
||
text: annotation.text,
|
||
})),
|
||
};
|
||
}
|
||
|
||
export function shouldConsiderCanvasAgentModelPlanner(params: {
|
||
userText: string;
|
||
context?: CanvasAgentContext | null;
|
||
}) {
|
||
if (!params.context?.projectId) return false;
|
||
const text = params.userText.trim().toLowerCase();
|
||
if (!text) return false;
|
||
|
||
const hasViewIntent =
|
||
hasCanvasAgentComparisonViewIntent(text) ||
|
||
hasCanvasAgentPanViewIntent(text) ||
|
||
/视角|视野|画布|聚焦|定位|居中|看全局|看整体|看全部|缩小|放大|拉近|拉远|zoom/.test(
|
||
text
|
||
);
|
||
const hasLinkIntent = /连起来|连线|连接|关联|参考关系|版本关系|关系/.test(text);
|
||
const hasTextIntent =
|
||
/文本|文字|文案|标题|标签|改写|润色|优化|改成|改为|设置为|设为|替换为/.test(
|
||
text
|
||
);
|
||
const hasCreateTextIntent = hasCanvasAgentTextCreationIntent(text);
|
||
const hasRenameIntent = /改名|重命名|命名|标题/.test(text);
|
||
const hasNoteIntent = /备注|note/.test(text);
|
||
const hasAnnotationIntent = /批注|标注|备注|note|annotation/.test(text);
|
||
const hasSlotIntent = hasCanvasAgentSlotCreationIntent(text);
|
||
const hasHighRiskIntent = Boolean(inferHighRiskOperation(text));
|
||
const hasMediaGenerationIntent =
|
||
/生成|出图|做一版|再出|换背景|背景|抠图|去水印|高清|扩图|重绘|变体|图层|多角度|洗图/.test(
|
||
text
|
||
);
|
||
|
||
if (
|
||
hasMediaGenerationIntent &&
|
||
!hasViewIntent &&
|
||
!hasLinkIntent &&
|
||
!hasTextIntent &&
|
||
!hasCreateTextIntent &&
|
||
!hasRenameIntent &&
|
||
!hasNoteIntent &&
|
||
!hasAnnotationIntent &&
|
||
!hasSlotIntent &&
|
||
!hasHighRiskIntent
|
||
) {
|
||
return false;
|
||
}
|
||
|
||
return (
|
||
hasViewIntent ||
|
||
hasLinkIntent ||
|
||
hasTextIntent ||
|
||
hasCreateTextIntent ||
|
||
hasRenameIntent ||
|
||
hasNoteIntent ||
|
||
hasAnnotationIntent ||
|
||
hasSlotIntent ||
|
||
hasHighRiskIntent
|
||
);
|
||
}
|
||
|
||
export function normalizeCanvasAgentModelPlannerDecision(params: {
|
||
decision: unknown;
|
||
context: CanvasAgentContext;
|
||
projectId: string;
|
||
messageId: string;
|
||
chatId: string;
|
||
userText: string;
|
||
}): CanvasAgentModelPlannerOutcome {
|
||
const parsed = plannerDecisionSchema.safeParse(params.decision);
|
||
if (!parsed.success) {
|
||
return { kind: "none", reason: "invalid_model_planner_decision" };
|
||
}
|
||
|
||
const decision = parsed.data;
|
||
const reason = trimString(decision.reason, 280) || undefined;
|
||
const operationId = `${params.messageId}:canvas-model-${decision.action}`;
|
||
const explicitRisk = trimString(decision.riskOperation, 80);
|
||
const inferredRisk = inferHighRiskOperation(params.userText);
|
||
const riskOperation =
|
||
decision.action === "confirmation_required"
|
||
? explicitRisk || inferredRisk || "other"
|
||
: inferredRisk;
|
||
if (riskOperation) {
|
||
const targetNodeIds = getDecisionNodeIds({
|
||
decision,
|
||
context: params.context,
|
||
allowSelectedFallback: true,
|
||
maxItems: 20,
|
||
});
|
||
const canBuildMovePlan =
|
||
(riskOperation === "move_nodes" || riskOperation === "batch_move") &&
|
||
targetNodeIds.length > 0;
|
||
const canBuildDeletePlan = riskOperation === "delete_nodes" && targetNodeIds.length > 0;
|
||
const operationPlan: CanvasOperationPlan | undefined = canBuildMovePlan
|
||
? {
|
||
version: 1,
|
||
operationId: operationId.replace(`canvas-model-${decision.action}`, "canvas-move"),
|
||
operation: "move_nodes",
|
||
projectId: params.projectId,
|
||
baseCanvasRevision: params.context.canvasRevision,
|
||
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
|
||
targetNodeIds,
|
||
layoutStrategy: inferMoveLayoutStrategy(params.userText),
|
||
spacing: 40,
|
||
preserveOriginal: true,
|
||
metadata: {
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
planner: "model",
|
||
prompt: params.userText,
|
||
pendingConfirmation: true,
|
||
riskOperation,
|
||
},
|
||
}
|
||
: canBuildDeletePlan
|
||
? {
|
||
version: 1,
|
||
operationId: operationId.replace(
|
||
`canvas-model-${decision.action}`,
|
||
"canvas-delete"
|
||
),
|
||
operation: "delete_nodes",
|
||
projectId: params.projectId,
|
||
baseCanvasRevision: params.context.canvasRevision,
|
||
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
|
||
targetNodeIds,
|
||
preserveOriginal: false,
|
||
metadata: {
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
planner: "model",
|
||
prompt: params.userText,
|
||
pendingConfirmation: true,
|
||
riskOperation,
|
||
},
|
||
}
|
||
: undefined;
|
||
return {
|
||
kind: "confirmation_required",
|
||
riskOperation,
|
||
targetNodeIds,
|
||
operationPlan,
|
||
viewPlan: operationPlan && operationPlan.operation !== "delete_nodes"
|
||
? buildViewPlan({
|
||
action: "focus_nodes",
|
||
context: params.context,
|
||
projectId: params.projectId,
|
||
operationId: operationPlan.operationId,
|
||
nodeIds: targetNodeIds,
|
||
reason: "focus_pending_move_nodes",
|
||
}) || undefined
|
||
: undefined,
|
||
reply: trimString(decision.reply, 280) || defaultConfirmationReply(riskOperation),
|
||
reason,
|
||
};
|
||
}
|
||
|
||
if (decision.action === "none") {
|
||
return { kind: "none", reason };
|
||
}
|
||
|
||
if (decision.action === "create_agent_slot") {
|
||
if (hasCombinedMediaExecutionIntent(params.userText)) {
|
||
return { kind: "none", reason: "slot_creation_combined_with_media_execution" };
|
||
}
|
||
|
||
const operationPlan = buildCanvasAgentSlotCreationPlan({
|
||
context: params.context,
|
||
projectId: params.projectId,
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
userText: params.userText,
|
||
operationId,
|
||
slotType: decision.slotType,
|
||
slotLabel: decision.slotLabel,
|
||
aspectRatio: decision.aspectRatio,
|
||
});
|
||
if (!operationPlan) return { kind: "none", reason: "missing_slot_creation_intent" };
|
||
|
||
const slotLabel = operationPlan.slot?.label || "Agent 槽位";
|
||
return {
|
||
kind: "operation",
|
||
operationPlan,
|
||
reply:
|
||
trimString(decision.reply, 280) ||
|
||
`已在当前视口创建${slotLabel}。`,
|
||
reason,
|
||
};
|
||
}
|
||
|
||
if (
|
||
decision.action === "focus_nodes" ||
|
||
decision.action === "fit_all" ||
|
||
decision.action === "zoom_in" ||
|
||
decision.action === "zoom_out"
|
||
) {
|
||
const nodeIds = getDecisionNodeIds({
|
||
decision,
|
||
context: params.context,
|
||
allowSelectedFallback: decision.action === "focus_nodes",
|
||
maxItems: 10,
|
||
});
|
||
const viewPlan = buildViewPlan({
|
||
action: decision.action,
|
||
context: params.context,
|
||
projectId: params.projectId,
|
||
operationId,
|
||
nodeIds,
|
||
reason,
|
||
});
|
||
if (!viewPlan) return { kind: "none", reason: "missing_view_target" };
|
||
return {
|
||
kind: "view",
|
||
viewPlan,
|
||
reply:
|
||
trimString(decision.reply, 280) ||
|
||
(decision.action === "fit_all"
|
||
? "已切到画布全局视图。"
|
||
: decision.action === "zoom_in"
|
||
? "已放大画布视角。"
|
||
: decision.action === "zoom_out"
|
||
? "已缩小画布视角。"
|
||
: "已聚焦到当前画布对象。"),
|
||
reason,
|
||
};
|
||
}
|
||
|
||
if (decision.action === "link_nodes") {
|
||
const nodeIds = getDecisionNodeIds({
|
||
decision,
|
||
context: params.context,
|
||
allowSelectedFallback: true,
|
||
maxItems: 20,
|
||
});
|
||
if (nodeIds.length < 2) {
|
||
return { kind: "none", reason: "link_nodes_requires_two_targets" };
|
||
}
|
||
const operationPlan: CanvasOperationPlan = {
|
||
version: 1,
|
||
operationId,
|
||
operation: "link_nodes",
|
||
projectId: params.projectId,
|
||
baseCanvasRevision: params.context.canvasRevision,
|
||
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
|
||
sourceNodeIds: nodeIds,
|
||
edgeType: "reference",
|
||
preserveOriginal: true,
|
||
metadata: {
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
planner: "model",
|
||
prompt: params.userText,
|
||
},
|
||
};
|
||
return {
|
||
kind: "operation",
|
||
operationPlan,
|
||
viewPlan: buildViewPlan({
|
||
action: "focus_nodes",
|
||
context: params.context,
|
||
projectId: params.projectId,
|
||
operationId,
|
||
nodeIds,
|
||
reason: "focus_model_linked_nodes",
|
||
}) || undefined,
|
||
reply: trimString(decision.reply, 280) || "已为选中的画布节点建立参考连线。",
|
||
reason,
|
||
};
|
||
}
|
||
|
||
if (decision.action === "add_annotation") {
|
||
const knownNodes = buildKnownNodeMap(params.context);
|
||
const targetNodeIds = getDecisionNodeIds({
|
||
decision,
|
||
context: params.context,
|
||
allowSelectedFallback: true,
|
||
maxItems: 10,
|
||
}).filter((nodeId) => {
|
||
const type = knownNodes.get(nodeId)?.type;
|
||
return type === "image" || type === "video";
|
||
});
|
||
const textContent = trimString(decision.textContent, 1000);
|
||
if (targetNodeIds.length === 0 || !textContent) {
|
||
return { kind: "none", reason: "add_annotation_missing_target_or_content" };
|
||
}
|
||
|
||
const operationPlan: CanvasOperationPlan = {
|
||
version: 1,
|
||
operationId,
|
||
operation: "add_annotation",
|
||
projectId: params.projectId,
|
||
baseCanvasRevision: params.context.canvasRevision,
|
||
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
|
||
targetNodeIds,
|
||
textContent,
|
||
annotations: [
|
||
{
|
||
type: "text-note",
|
||
text: textContent,
|
||
color: "#f59e0b",
|
||
},
|
||
],
|
||
preserveOriginal: true,
|
||
metadata: {
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
planner: "model",
|
||
prompt: params.userText,
|
||
},
|
||
};
|
||
return {
|
||
kind: "operation",
|
||
operationPlan,
|
||
viewPlan: buildViewPlan({
|
||
action: "focus_nodes",
|
||
context: params.context,
|
||
projectId: params.projectId,
|
||
operationId,
|
||
nodeIds: targetNodeIds,
|
||
reason: "focus_model_annotated_nodes",
|
||
}) || undefined,
|
||
reply: trimString(decision.reply, 280) || "已给选中的画布节点添加批注。",
|
||
reason,
|
||
};
|
||
}
|
||
|
||
if (decision.action === "update_note") {
|
||
const knownNodes = buildKnownNodeMap(params.context);
|
||
const targetNodeIds = getDecisionNodeIds({
|
||
decision,
|
||
context: params.context,
|
||
allowSelectedFallback: true,
|
||
maxItems: 10,
|
||
}).filter((nodeId) => {
|
||
const type = knownNodes.get(nodeId)?.type;
|
||
return type === "image" || type === "video";
|
||
});
|
||
const note = trimString(decision.textContent, 1000);
|
||
if (targetNodeIds.length === 0 || !note) {
|
||
return { kind: "none", reason: "update_note_missing_target_or_content" };
|
||
}
|
||
|
||
const operationPlan: CanvasOperationPlan = {
|
||
version: 1,
|
||
operationId,
|
||
operation: "update_note",
|
||
projectId: params.projectId,
|
||
baseCanvasRevision: params.context.canvasRevision,
|
||
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
|
||
targetNodeIds,
|
||
textContent: note,
|
||
preserveOriginal: true,
|
||
metadata: {
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
planner: "model",
|
||
prompt: params.userText,
|
||
},
|
||
};
|
||
return {
|
||
kind: "operation",
|
||
operationPlan,
|
||
viewPlan: buildViewPlan({
|
||
action: "focus_nodes",
|
||
context: params.context,
|
||
projectId: params.projectId,
|
||
operationId,
|
||
nodeIds: targetNodeIds,
|
||
reason: "focus_model_note_nodes",
|
||
}) || undefined,
|
||
reply: trimString(decision.reply, 280) || "已更新选中图片备注。",
|
||
reason,
|
||
};
|
||
}
|
||
|
||
if (decision.action === "create_text_node") {
|
||
const textContent = trimString(decision.textContent, 4000);
|
||
if (!textContent) {
|
||
return { kind: "none", reason: "create_text_node_missing_content" };
|
||
}
|
||
const textSize = { width: 320, height: 120 };
|
||
const operationPlan: CanvasOperationPlan = {
|
||
version: 1,
|
||
operationId,
|
||
operation: "create_text_node",
|
||
projectId: params.projectId,
|
||
baseCanvasRevision: params.context.canvasRevision,
|
||
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
|
||
targetPosition: getViewportCenteredSlotPosition({
|
||
context: params.context,
|
||
size: textSize,
|
||
}),
|
||
textContent,
|
||
preserveOriginal: true,
|
||
metadata: {
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
planner: "model",
|
||
prompt: params.userText,
|
||
canvasAgentViewport: params.context.viewport,
|
||
},
|
||
};
|
||
return {
|
||
kind: "operation",
|
||
operationPlan,
|
||
reply: trimString(decision.reply, 280) || "已在当前画布添加文本节点。",
|
||
reason,
|
||
};
|
||
}
|
||
|
||
if (decision.action === "update_text") {
|
||
const knownNodes = buildKnownNodeMap(params.context);
|
||
const requestedIds = getDecisionNodeIds({
|
||
decision,
|
||
context: params.context,
|
||
allowSelectedFallback: true,
|
||
maxItems: 10,
|
||
});
|
||
const targetNodeIds = requestedIds.filter(
|
||
(nodeId) => knownNodes.get(nodeId)?.type === "text"
|
||
);
|
||
const textContent = trimString(decision.textContent, 4000);
|
||
if (
|
||
targetNodeIds.length === 0 &&
|
||
textContent &&
|
||
hasCanvasAgentTextCreationIntent(params.userText)
|
||
) {
|
||
const textSize = { width: 320, height: 120 };
|
||
const operationPlan: CanvasOperationPlan = {
|
||
version: 1,
|
||
operationId: operationId.replace("canvas-model-update_text", "canvas-model-create_text_node"),
|
||
operation: "create_text_node",
|
||
projectId: params.projectId,
|
||
baseCanvasRevision: params.context.canvasRevision,
|
||
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
|
||
targetPosition: getViewportCenteredSlotPosition({
|
||
context: params.context,
|
||
size: textSize,
|
||
}),
|
||
textContent,
|
||
preserveOriginal: true,
|
||
metadata: {
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
planner: "model",
|
||
prompt: params.userText,
|
||
normalizedFromAction: "update_text",
|
||
canvasAgentViewport: params.context.viewport,
|
||
},
|
||
};
|
||
return {
|
||
kind: "operation",
|
||
operationPlan,
|
||
reply: trimString(decision.reply, 280) || "已在当前画布添加文本节点。",
|
||
reason,
|
||
};
|
||
}
|
||
if (targetNodeIds.length === 0 || !textContent) {
|
||
return { kind: "none", reason: "update_text_missing_target_or_content" };
|
||
}
|
||
const operationPlan: CanvasOperationPlan = {
|
||
version: 1,
|
||
operationId,
|
||
operation: "update_text",
|
||
projectId: params.projectId,
|
||
baseCanvasRevision: params.context.canvasRevision,
|
||
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
|
||
targetNodeIds,
|
||
textContent,
|
||
preserveOriginal: false,
|
||
metadata: {
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
planner: "model",
|
||
prompt: params.userText,
|
||
},
|
||
};
|
||
return {
|
||
kind: "operation",
|
||
operationPlan,
|
||
viewPlan: buildViewPlan({
|
||
action: "focus_nodes",
|
||
context: params.context,
|
||
projectId: params.projectId,
|
||
operationId,
|
||
nodeIds: targetNodeIds,
|
||
reason: "focus_model_updated_text_nodes",
|
||
}) || undefined,
|
||
reply: trimString(decision.reply, 280) || "已更新选中的文本节点。",
|
||
reason,
|
||
};
|
||
}
|
||
|
||
if (decision.action === "rename_node") {
|
||
const targetNodeIds = getDecisionNodeIds({
|
||
decision,
|
||
context: params.context,
|
||
allowSelectedFallback: true,
|
||
maxItems: 10,
|
||
});
|
||
const title = trimString(decision.textContent, 120);
|
||
if (targetNodeIds.length === 0 || !title) {
|
||
return { kind: "none", reason: "rename_node_missing_target_or_title" };
|
||
}
|
||
const operationPlan: CanvasOperationPlan = {
|
||
version: 1,
|
||
operationId,
|
||
operation: "rename_node",
|
||
projectId: params.projectId,
|
||
baseCanvasRevision: params.context.canvasRevision,
|
||
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
|
||
targetNodeIds,
|
||
textContent: title,
|
||
preserveOriginal: true,
|
||
metadata: {
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
planner: "model",
|
||
prompt: params.userText,
|
||
},
|
||
};
|
||
return {
|
||
kind: "operation",
|
||
operationPlan,
|
||
viewPlan: buildViewPlan({
|
||
action: "focus_nodes",
|
||
context: params.context,
|
||
projectId: params.projectId,
|
||
operationId,
|
||
nodeIds: targetNodeIds,
|
||
reason: "focus_model_renamed_nodes",
|
||
}) || undefined,
|
||
reply: trimString(decision.reply, 280) || "已更新选中节点标题。",
|
||
reason,
|
||
};
|
||
}
|
||
|
||
return { kind: "none", reason: "unsupported_model_planner_action" };
|
||
}
|
||
|
||
export async function planCanvasAgentOperationWithModel(params: {
|
||
model: LanguageModel;
|
||
userText: string;
|
||
context: CanvasAgentContext;
|
||
projectId: string;
|
||
messageId: string;
|
||
chatId: string;
|
||
abortSignal?: AbortSignal;
|
||
}): Promise<CanvasAgentModelPlannerOutcome> {
|
||
if (
|
||
!shouldConsiderCanvasAgentModelPlanner({
|
||
userText: params.userText,
|
||
context: params.context,
|
||
})
|
||
) {
|
||
return { kind: "none", reason: "planner_not_relevant" };
|
||
}
|
||
|
||
const { object } = await generateObject({
|
||
model: params.model,
|
||
schema: plannerDecisionSchema,
|
||
temperature: 0,
|
||
abortSignal: params.abortSignal || AbortSignal.timeout(18_000),
|
||
system: [
|
||
"你是无限画布 Agent 的受控操作规划器。",
|
||
"你只能规划低风险画布操作：视角聚焦/缩放/看全局、连接节点、新建文本节点、更新文本节点、修改节点标题、更新图片或视频备注、给图片或视频节点添加批注、创建空 Agent 槽位。",
|
||
"图片/视频生成、换背景、抠图、高清、扩图、重绘等媒体任务必须返回 action=none，交给其他执行器。",
|
||
"当用户明确要求新建/添加空图片槽位、视频槽位、UI槽位、空框或占位时，可以返回 action=create_agent_slot；如果同时要求生成或填充媒体，返回 action=none。",
|
||
"当用户明确要求在画布中新建/添加/写入文本节点、文案、说明、标题或标语时，返回 action=create_text_node，并把最终文本写入 textContent。",
|
||
"当用户要求给图片/视频/slot/节点改名、重命名或修改标题时，返回 action=rename_node；不要把图片节点标题改名误判为 update_text。",
|
||
"当用户要求给图片或视频添加/更新备注时，返回 action=update_note；批注、标注、箭头、框选说明才返回 action=add_annotation。",
|
||
"删除、覆盖、替换现有图片、移动/重排节点等高风险动作必须返回 action=confirmation_required。",
|
||
"只能使用上下文中出现的 nodeId。不要编造节点 ID。",
|
||
"如果用户只是聊天或目标不明确，返回 action=none。",
|
||
].join("\n"),
|
||
prompt: JSON.stringify(
|
||
{
|
||
userText: params.userText,
|
||
context: buildContextSummary(params.context),
|
||
outputContract: {
|
||
action:
|
||
"none | focus_nodes | fit_all | zoom_in | zoom_out | link_nodes | create_text_node | update_text | rename_node | update_note | add_annotation | create_agent_slot | confirmation_required",
|
||
nodeIds: "只填上下文存在的 nodeId",
|
||
textContent:
|
||
"create_text_node/update_text 时填写最终文本；rename_node 时填写新标题；update_note 时填写备注内容；add_annotation 时填写批注内容",
|
||
slotType: "create_agent_slot 时填写 image/video/ui-screen",
|
||
slotLabel: "create_agent_slot 时可填写槽位名称",
|
||
aspectRatio: "create_agent_slot 时可填写 1:1/16:9 等比例",
|
||
riskOperation:
|
||
"confirmation_required 时填写 replace_node_media/delete_nodes/move_nodes/overwrite_slot/batch_move/other",
|
||
reply: "一句给用户看的中文短回复",
|
||
},
|
||
},
|
||
null,
|
||
2
|
||
),
|
||
});
|
||
|
||
return normalizeCanvasAgentModelPlannerDecision({
|
||
decision: object,
|
||
context: params.context,
|
||
projectId: params.projectId,
|
||
messageId: params.messageId,
|
||
chatId: params.chatId,
|
||
userText: params.userText,
|
||
});
|
||
}
|