fix: route explicit canvas agent operations

This commit is contained in:
134 2026-06-23 18:15:08 +08:00
parent 6e1d512021
commit 06834699be
3 changed files with 571 additions and 0 deletions

View File

@ -73,6 +73,10 @@ import {
resolveCanvasAgentSlotCreationIntent,
type CanvasAgentModelPlannerOutcome,
} from "@/lib/canvas/agent-model-planner";
import {
buildCanvasAgentExplicitOperationPlan,
type CanvasAgentExplicitOperationPlan,
} from "@/lib/canvas/agent-explicit-operations";
import { buildCanvasAgentResultViewOperation } from "@/lib/canvas/agent-result-view";
import {
buildCanvasAgentComparisonViewPlan,
@ -3425,6 +3429,25 @@ export async function POST(request: Request) {
};
};
const resolveCanvasExplicitOperationPlan =
(): CanvasAgentExplicitOperationPlan | null => {
if (!canApplyCanvasAgentMutation) return null;
if (composerMode !== "agent") return null;
if (contextPayload?.source !== "canvas" || !contextPayload.projectId) return null;
if (!requestCanvasAgentContext) return null;
return buildCanvasAgentExplicitOperationPlan({
context: requestCanvasAgentContext,
projectId: contextPayload.projectId,
messageId: message.id,
chatId: id,
userText: requestTextContent,
requestedNodeIds: (requestTargetResolution?.targetRefs || [])
.map((ref) => ref.nodeId)
.filter((nodeId): nodeId is string => Boolean(nodeId)),
});
};
const resolveCanvasModelPlannerOutcome =
async (): Promise<CanvasAgentModelPlannerOutcome | null> => {
if (composerMode !== "agent") return null;
@ -4038,6 +4061,152 @@ export async function POST(request: Request) {
return createSseResponse(canvasTextStream);
}
const canvasExplicitOperationPlan = resolveCanvasExplicitOperationPlan();
if (canvasExplicitOperationPlan) {
const { operationPlan } = canvasExplicitOperationPlan;
const canvasExplicitStream = createUIMessageStream({
execute: async ({ writer }) => {
const messageId = generateUUID();
writeAgentProgressChunk(
writer,
{
stage: "executing",
currentStep: "execute",
title: canvasExplicitOperationPlan.progressTitle,
message: canvasExplicitOperationPlan.progressMessage,
toolName: "canvasOperation",
capability: operationPlan.operation,
},
"canvas-explicit:progress-executing",
);
try {
const result = await applyCanvasOperationPlan({
userId: session.user.id,
projectId: operationPlan.projectId,
plan: operationPlan,
});
const anchorNodeId =
result.createdNodeIds[0] ||
operationPlan.targetNodeIds?.[0] ||
operationPlan.anchorNodeId ||
null;
writeCanvasOperationChunk(
writer,
{
projectId: operationPlan.projectId,
operationId: operationPlan.operationId,
operation: operationPlan.operation,
createdNodeIds: result.createdNodeIds,
createdNodeIdsByMediaItemId: result.createdNodeIdsByMediaItemId,
updatedNodeIds: result.updatedNodeIds,
createdEdgeIds: result.createdEdgeIds,
canvasRevision: result.canvasRevision,
anchorNodeId,
handledCanvasMediaItemIds: [],
message: canvasExplicitOperationPlan.reply,
},
"canvas-explicit:operation",
);
const focusNodeIds = Array.from(
new Set([
...result.createdNodeIds,
...(operationPlan.targetNodeIds || []),
])
)
.filter(Boolean)
.slice(0, 20);
if (canvasAgentFeatures.viewOperationEnabled && focusNodeIds.length > 0) {
writeCanvasViewOperationChunk(
writer,
{
version: 1,
viewOperationId: `${operationPlan.operationId}:view`,
operation: focusNodeIds.length > 1 ? "focus_nodes" : "focus_selection",
projectId: operationPlan.projectId,
nodeIds: focusNodeIds,
anchorNodeId: focusNodeIds[0],
padding: operationPlan.operation === "create_text_node" ? 160 : 120,
persist: false,
reason: "focus_explicit_canvas_operation",
},
"canvas-explicit:view-operation",
);
}
writeAgentProgressChunk(
writer,
{
stage: "completed",
currentStep: "finalize",
title: canvasExplicitOperationPlan.completedTitle,
message: canvasExplicitOperationPlan.reply,
toolName: "canvasOperation",
capability: operationPlan.operation,
},
"canvas-explicit:progress-completed",
);
writeAssistantTextChunk(
writer,
canvasExplicitOperationPlan.reply,
"canvas-explicit:reply",
messageId,
);
await saveMessages({
messages: [
{
id: messageId,
chatId: id,
role: "assistant",
parts: [{ type: "text", text: canvasExplicitOperationPlan.reply }],
attachments: [],
createdAt: new Date(),
videoStorageStatus: null,
temporaryVideoUrl: null,
permanentVideoUrl: null,
archivedAt: null,
videoMetadata: null,
},
],
});
} catch (error) {
const failure = writeCanvasOperationFailureChunk({
writer,
label: "canvas-explicit:operation-failed",
projectId: operationPlan.projectId,
operationId: operationPlan.operationId,
operation: operationPlan.operation,
error,
anchorNodeId:
operationPlan.targetNodeIds?.[0] ||
operationPlan.anchorNodeId ||
null,
});
writeAgentProgressChunk(
writer,
{
stage: "failed",
currentStep: "finalize",
title: canvasExplicitOperationPlan.failedTitle,
message: failure.errorMessage,
toolName: "canvasOperation",
capability: operationPlan.operation,
},
"canvas-explicit:progress-failed",
);
writeAssistantTextChunk(
writer,
failure.errorMessage,
"canvas-explicit:error",
messageId,
);
}
},
});
return createSseResponse(canvasExplicitStream);
}
const canvasModelPlannerOutcome = await resolveCanvasModelPlannerOutcome();
if (canvasModelPlannerOutcome && canvasModelPlannerOutcome.kind !== "none") {
const canvasModelPlannerStream = createUIMessageStream({

View File

@ -0,0 +1,307 @@
import type {
CanvasAgentContext,
CanvasOperationPlan,
} from "@/types/canvas/agent-protocol";
type ExplicitOperationKind = "create_text_node" | "rename_node" | "update_note";
export type CanvasAgentExplicitOperationPlan = {
operationPlan: CanvasOperationPlan;
reply: string;
progressTitle: string;
progressMessage: string;
completedTitle: string;
failedTitle: string;
};
function trimString(value: unknown, maxLength: number) {
return typeof value === "string" ? value.trim().slice(0, maxLength) : "";
}
function normalizeNodeIds(values: Array<string | undefined | null>, maxItems = 20) {
return Array.from(
new Set(values.map((value) => trimString(value, 160)).filter(Boolean))
).slice(0, maxItems);
}
function stripCommandContent(value: unknown, maxLength: number) {
return trimString(value, maxLength)
.replace(/^["'“”‘’]+|["'“”‘’。;;]+$/g, "")
.trim()
.slice(0, maxLength);
}
function getKnownNodeTypes(context: CanvasAgentContext) {
const nodeMap = new Map<string, string>();
[...context.candidateNodes, ...context.selectedNodes].forEach((node) => {
if (node.nodeId) nodeMap.set(node.nodeId, node.type);
});
context.slots.forEach((slot) => {
if (slot.nodeId) nodeMap.set(slot.nodeId, "agent-slot");
});
return nodeMap;
}
function getSelectedNodeIds(context: CanvasAgentContext) {
return normalizeNodeIds([
...context.selectedNodeIds,
...context.selectedNodes.map((node) => node.nodeId),
...context.slots
.filter((slot) => slot.isSelected || context.selectedNodeIds.includes(slot.nodeId))
.map((slot) => slot.nodeId),
]);
}
function getTargetNodeIds(params: {
context: CanvasAgentContext;
requestedNodeIds?: string[];
allowedTypes?: string[];
}) {
const knownNodeTypes = getKnownNodeTypes(params.context);
const allowedTypes = params.allowedTypes ? new Set(params.allowedTypes) : null;
return normalizeNodeIds([
...(params.requestedNodeIds || []),
...getSelectedNodeIds(params.context),
]).filter((nodeId) => {
const type = knownNodeTypes.get(nodeId);
if (!type) return false;
return allowedTypes ? allowedTypes.has(type) : true;
});
}
function getRenameAllowedTypes(userText: string) {
const lowerText = userText.toLowerCase();
if (/槽位|slot/.test(lowerText)) return ["agent-slot"];
if (/视频|video/.test(lowerText)) return ["video"];
if (/图片|图像|照片|主图|参考图|image/.test(lowerText)) return ["image"];
if (/文本|文字|文案|text/.test(lowerText)) return ["text"];
return undefined;
}
function hasMediaMutationIntent(userText: string) {
return /生成|出图|做一版|再出|换|改图|编辑图片|编辑图|抠图|去水印|高清|清晰|扩图|裁剪|删除|替换|背景|模型|洗图|重绘|尺寸|像素|分辨率/.test(
userText.toLowerCase()
);
}
function extractContentFromPatterns(userText: string, patterns: RegExp[], maxLength: number) {
for (const pattern of patterns) {
const match = userText.match(pattern);
const content = stripCommandContent(match?.[1] || "", maxLength);
if (content) return content;
}
return "";
}
function getViewportCenteredPosition(params: {
context: CanvasAgentContext;
size: { width: number; height: number };
}) {
const viewport = params.context.viewport || { x: 0, y: 0, zoom: 1 };
const zoom = Math.max(0.1, Number(viewport.zoom || 1));
const viewportWidth = Math.max(320, Number(viewport.width || 1200));
const viewportHeight = Math.max(320, Number(viewport.height || 800));
return {
x: (viewportWidth / 2 - Number(viewport.x || 0)) / zoom - params.size.width / 2,
y: (viewportHeight / 2 - Number(viewport.y || 0)) / zoom - params.size.height / 2,
};
}
function buildPlanBase(params: {
context: CanvasAgentContext;
projectId: string;
messageId: string;
chatId: string;
userText: string;
operation: ExplicitOperationKind;
}): Pick<
CanvasOperationPlan,
| "version"
| "operationId"
| "operation"
| "projectId"
| "baseCanvasRevision"
| "baseCanvasUpdatedAt"
| "preserveOriginal"
| "metadata"
> {
return {
version: 1,
operationId: `${params.messageId}:canvas-${params.operation}`,
operation: params.operation,
projectId: params.projectId,
baseCanvasRevision: params.context.canvasRevision,
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
preserveOriginal: true,
metadata: {
messageId: params.messageId,
chatId: params.chatId,
planner: "deterministic",
prompt: params.userText,
},
};
}
function buildCreateTextOperation(params: {
context: CanvasAgentContext;
projectId: string;
messageId: string;
chatId: string;
userText: string;
}): CanvasAgentExplicitOperationPlan | null {
const text = params.userText.trim();
const hasCreateTextIntent =
/(?:创建|新增|新建|添加|加一段|加一句|写一段|写一句|放一段|放一句|写入|写上|放上).*(?:文本|文字|文案|标题|标语|说明|caption|copy|text)|(?:文本|文字|文案|标题|标语|说明).*(?:创建|新增|新建|添加|写入|写上)/i.test(
text
);
if (!hasCreateTextIntent || hasMediaMutationIntent(text)) return null;
const textContent = extractContentFromPatterns(
text,
[
/(?:文本|文字|文案|标题|标语|说明|caption|copy|text)[^::]*[::]\s*([\s\S]+)$/i,
/(?:写入|写上|写成|内容是|设为|设置为)\s*[::]?\s*([\s\S]+)$/i,
/[::]\s*([\s\S]+)$/i,
],
4000
);
if (!textContent) return null;
const textSize = { width: 320, height: 120 };
const basePlan = buildPlanBase({ ...params, operation: "create_text_node" });
return {
operationPlan: {
...basePlan,
targetPosition: getViewportCenteredPosition({
context: params.context,
size: textSize,
}),
textContent,
metadata: {
...basePlan.metadata,
canvasAgentViewport: params.context.viewport,
},
},
reply: "已在当前画布添加文本节点。",
progressTitle: "正在添加画布文本",
progressMessage: "我会在当前视口中心创建一个新的文本节点。",
completedTitle: "画布文本已添加",
failedTitle: "画布文本添加失败",
};
}
function buildRenameOperation(params: {
context: CanvasAgentContext;
projectId: string;
messageId: string;
chatId: string;
userText: string;
requestedNodeIds?: string[];
}): CanvasAgentExplicitOperationPlan | null {
const text = params.userText.trim();
const hasRenameIntent = /改名|重命名|命名为|标题(?:改成|改为|设为|设置为)|改标题|修改标题/.test(
text
);
if (!hasRenameIntent || hasMediaMutationIntent(text)) return null;
const title = extractContentFromPatterns(
text,
[
/(?:改名为|重命名为|命名为|标题(?:改成|改为|设为|设置为)|改标题为|修改标题为)\s*[::]?\s*([\s\S]+)$/i,
/[::]\s*([\s\S]+)$/i,
],
120
);
if (!title) return null;
const targetNodeIds = getTargetNodeIds({
context: params.context,
requestedNodeIds: params.requestedNodeIds,
allowedTypes: getRenameAllowedTypes(text),
}).slice(0, 10);
if (targetNodeIds.length === 0) return null;
return {
operationPlan: {
...buildPlanBase({ ...params, operation: "rename_node" }),
targetNodeIds,
textContent: title,
},
reply: "已更新选中节点标题。",
progressTitle: "正在更新节点标题",
progressMessage: "我会只修改选中节点的标题,不改变媒体内容。",
completedTitle: "节点标题已更新",
failedTitle: "节点标题更新失败",
};
}
function buildUpdateNoteOperation(params: {
context: CanvasAgentContext;
projectId: string;
messageId: string;
chatId: string;
userText: string;
requestedNodeIds?: string[];
}): CanvasAgentExplicitOperationPlan | null {
const text = params.userText.trim();
if (!/备注|note/i.test(text)) return null;
if (/按.*(?:备注|note).*(?:修改|编辑|改图|改主图|生成|出图)/i.test(text)) {
return null;
}
if (hasMediaMutationIntent(text)) return null;
const note = extractContentFromPatterns(
text,
[
/(?:备注为|备注成|添加备注|加备注|写备注|备注|note)\s*[::]?\s*([\s\S]+)$/i,
/[::]\s*([\s\S]+)$/i,
],
1000
);
if (!note) return null;
const targetNodeIds = getTargetNodeIds({
context: params.context,
requestedNodeIds: params.requestedNodeIds,
allowedTypes: ["image", "video"],
}).slice(0, 10);
if (targetNodeIds.length === 0) return null;
return {
operationPlan: {
...buildPlanBase({ ...params, operation: "update_note" }),
targetNodeIds,
textContent: note,
},
reply: "已更新选中图片备注。",
progressTitle: "正在更新图片备注",
progressMessage: "我会把备注保存到选中的图片或视频节点。",
completedTitle: "图片备注已更新",
failedTitle: "图片备注更新失败",
};
}
export function buildCanvasAgentExplicitOperationPlan(params: {
context: CanvasAgentContext;
projectId: string;
messageId: string;
chatId: string;
userText: string;
requestedNodeIds?: string[];
}): CanvasAgentExplicitOperationPlan | null {
const commonParams = {
...params,
projectId: trimString(params.projectId, 160),
messageId: trimString(params.messageId, 160),
chatId: trimString(params.chatId, 160),
};
if (!commonParams.projectId || !commonParams.messageId || !commonParams.chatId) {
return null;
}
return (
buildCreateTextOperation(commonParams) ||
buildRenameOperation(commonParams) ||
buildUpdateNoteOperation(commonParams)
);
}

View File

@ -68,6 +68,7 @@ import {
resolveCanvasAgentSlotCreationIntent,
shouldConsiderCanvasAgentModelPlanner,
} from "@/lib/canvas/agent-model-planner";
import { buildCanvasAgentExplicitOperationPlan } from "@/lib/canvas/agent-explicit-operations";
import { buildCanvasAgentDirectWritebackPlan } from "@/lib/canvas/agent-direct-writeback";
import { resolveCanvasAgentMediaPlacements } from "@/lib/canvas/agent-placement";
import { buildCanvasAgentResultViewOperation } from "@/lib/canvas/agent-result-view";
@ -913,6 +914,100 @@ test.describe("canvas chat pure helpers", () => {
});
});
test("builds deterministic text creation plans from explicit canvas prompts", () => {
const context: CanvasAgentContext = {
...plannerContext,
selectedNodeIds: [],
selectedNodes: [],
viewport: { x: 0, y: 0, zoom: 1, width: 1200, height: 800 },
focusPolicy: "auto",
};
const outcome = buildCanvasAgentExplicitOperationPlan({
context,
projectId: plannerContext.projectId || "",
messageId: "message-1",
chatId: "chat-1",
userText: "在画布上添加文字:新品上市",
});
expect(outcome?.operationPlan).toMatchObject({
operation: "create_text_node",
targetPosition: { x: 440, y: 340 },
textContent: "新品上市",
preserveOriginal: true,
metadata: {
messageId: "message-1",
chatId: "chat-1",
planner: "deterministic",
canvasAgentViewport: { x: 0, y: 0, zoom: 1, width: 1200, height: 800 },
},
});
});
test("builds deterministic node rename plans without treating titles as image edits", () => {
const outcome = buildCanvasAgentExplicitOperationPlan({
context: plannerContext,
projectId: plannerContext.projectId || "",
messageId: "message-1",
chatId: "chat-1",
userText: "把选中图片改名为白底主图 A",
});
expect(outcome?.operationPlan).toMatchObject({
operation: "rename_node",
targetNodeIds: ["image-1"],
textContent: "白底主图 A",
metadata: {
messageId: "message-1",
chatId: "chat-1",
planner: "deterministic",
},
});
});
test("builds deterministic note updates for selected media nodes only", () => {
const outcome = buildCanvasAgentExplicitOperationPlan({
context: plannerContext,
projectId: plannerContext.projectId || "",
messageId: "message-1",
chatId: "chat-1",
userText: "给选中图片备注:保留主体轮廓",
});
expect(outcome?.operationPlan).toMatchObject({
operation: "update_note",
targetNodeIds: ["image-1"],
textContent: "保留主体轮廓",
metadata: {
messageId: "message-1",
chatId: "chat-1",
planner: "deterministic",
},
});
});
test("does not route media generation or note-driven image edits as explicit canvas operations", () => {
expect(
buildCanvasAgentExplicitOperationPlan({
context: plannerContext,
projectId: plannerContext.projectId || "",
messageId: "message-1",
chatId: "chat-1",
userText: "按这个备注修改主图",
})
).toBeNull();
expect(
buildCanvasAgentExplicitOperationPlan({
context: plannerContext,
projectId: plannerContext.projectId || "",
messageId: "message-1",
chatId: "chat-1",
userText: "生成一张图,标题文字:新品上市",
})
).toBeNull();
});
test("normalizes model-planned annotations to selected media nodes", () => {
const outcome = normalizeCanvasAgentModelPlannerDecision({
decision: {