From 38dba2b1bc9b9c556a6b7731d4025a927134787d Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 23 Jun 2026 16:34:52 +0800 Subject: [PATCH] feat: let canvas agent update image notes --- ...框Agent模式升级开发文档-借鉴Cowart-2026-06-22.md | 5 ++ lib/canvas/agent-model-planner.ts | 60 ++++++++++++- lib/canvas/agent-mutations.ts | 85 +++++++++++++++++++ lib/canvas/agent-operation-request.ts | 11 +++ tests/routes/canvas-agent-operations.test.ts | 58 +++++++++++++ tests/routes/canvas-chat-helpers.test.ts | 27 ++++++ types/canvas/agent-protocol.ts | 1 + 7 files changed, 244 insertions(+), 3 deletions(-) diff --git a/docs/无限画布对话框Agent模式升级开发文档-借鉴Cowart-2026-06-22.md b/docs/无限画布对话框Agent模式升级开发文档-借鉴Cowart-2026-06-22.md index 35cc19f28..a73f7e2e0 100644 --- a/docs/无限画布对话框Agent模式升级开发文档-借鉴Cowart-2026-06-22.md +++ b/docs/无限画布对话框Agent模式升级开发文档-借鉴Cowart-2026-06-22.md @@ -616,6 +616,7 @@ export type CanvasOperationKind = | "link_nodes" | "move_nodes" | "rename_node" + | "update_note" | "update_text"; export interface CanvasMediaItem { @@ -740,6 +741,7 @@ Agent 首期只能请求以下操作，后端 mutation 层也只接受这些操 | `append_revision` | 基于源节点追加一个新版本 | 低 | 否 | | `link_nodes` | 给两个节点建立参考/版本关系 | 低 | 否 | | `rename_node` | 修改图片/视频/slot/节点标题 | 低 | 否 | +| `update_note` | 更新图片/视频节点备注并同步结构化备注标注 | 低 | 否 | | `update_text` | 修改选中文本节点内容 | 中 | 视情况 | | `replace_node_media` | 替换已有图片/视频内容 | 高 | 是 | | `move_nodes` | 批量移动或整理节点 | 中 | 视情况 | @@ -751,6 +753,7 @@ Agent 首期只能请求以下操作，后端 mutation 层也只接受这些操 - `append_revision` - `link_nodes` - `rename_node` +- `update_note` 这样可以先把“对话框驱动画布”跑通，同时把误操作风险降到最低。 @@ -1066,6 +1069,7 @@ autosave 冲突要求： - `append_revision` - `link_nodes` - `rename_node` +- `update_note` 首期 revision 不一致时必须拒绝或要求确认的操作： @@ -1169,6 +1173,7 @@ hasSelectedAgentSlot?: boolean; | “按批注修改” | `append_revision` | selected image + annotations | | “把这两张连起来” | `link_nodes` | selected nodes | | “把选中图片改名为白底主图 A” | `rename_node` | selected image/video/slot node | +| “给选中图片加备注：保留主体轮廓” | `update_note` | selected image/video node | | “把这段文字改成更高级” | `update_text` | selected text node | | “替换掉原图” | `replace_node_media` | selected image，需要确认 | | “把这些整理成三列” | `move_nodes` | selected nodes，建议确认 | diff --git a/lib/canvas/agent-model-planner.ts b/lib/canvas/agent-model-planner.ts index d1da481b4..a32e27df4 100644 --- a/lib/canvas/agent-model-planner.ts +++ b/lib/canvas/agent-model-planner.ts @@ -18,6 +18,7 @@ const plannerDecisionSchema = z.object({ "link_nodes", "update_text", "rename_node", + "update_note", "add_annotation", "create_agent_slot", "confirmation_required", @@ -421,6 +422,7 @@ export function shouldConsiderCanvasAgentModelPlanner(params: { text ); const hasRenameIntent = /改名|重命名|命名|标题/.test(text); + const hasNoteIntent = /备注|note/.test(text); const hasAnnotationIntent = /批注|标注|备注|note|annotation/.test(text); const hasSlotIntent = hasCanvasAgentSlotCreationIntent(text); const hasHighRiskIntent = Boolean(inferHighRiskOperation(text)); @@ -435,6 +437,7 @@ export function shouldConsiderCanvasAgentModelPlanner(params: { !hasLinkIntent && !hasTextIntent && !hasRenameIntent && + !hasNoteIntent && !hasAnnotationIntent && !hasSlotIntent && !hasHighRiskIntent @@ -447,6 +450,7 @@ export function shouldConsiderCanvasAgentModelPlanner(params: { hasLinkIntent || hasTextIntent || hasRenameIntent || + hasNoteIntent || hasAnnotationIntent || hasSlotIntent || hasHighRiskIntent @@ -719,6 +723,55 @@ export function normalizeCanvasAgentModelPlannerDecision(params: { }; } + if (decision.action === "update_note") { + const knownNodes = buildKnownNodeMap(params.context); + const targetNodeIds = getDecisionNodeIds({ + decision, + context: params.context, + allowSelectedFallback: true, + maxItems: 10, + }).filter((nodeId) => { + const type = knownNodes.get(nodeId)?.type; + return type === "image" || type === "video"; + }); + const note = trimString(decision.textContent, 1000); + if (targetNodeIds.length === 0 || !note) { + return { kind: "none", reason: "update_note_missing_target_or_content" }; + } + + const operationPlan: CanvasOperationPlan = { + version: 1, + operationId, + operation: "update_note", + projectId: params.projectId, + baseCanvasRevision: params.context.canvasRevision, + baseCanvasUpdatedAt: params.context.canvasUpdatedAt, + targetNodeIds, + textContent: note, + preserveOriginal: true, + metadata: { + messageId: params.messageId, + chatId: params.chatId, + planner: "model", + prompt: params.userText, + }, + }; + return { + kind: "operation", + operationPlan, + viewPlan: buildViewPlan({ + action: "focus_nodes", + context: params.context, + projectId: params.projectId, + operationId, + nodeIds: targetNodeIds, + reason: "focus_model_note_nodes", + }) || undefined, + reply: trimString(decision.reply, 280) || "已更新选中图片备注。", + reason, + }; + } + if (decision.action === "update_text") { const knownNodes = buildKnownNodeMap(params.context); const requestedIds = getDecisionNodeIds({ @@ -839,10 +892,11 @@ export async function planCanvasAgentOperationWithModel(params: { abortSignal: params.abortSignal || AbortSignal.timeout(18_000), system: [ "你是无限画布 Agent 的受控操作规划器。", - "你只能规划低风险画布操作：视角聚焦/缩放/看全局、连接节点、更新文本节点、修改节点标题、给图片或视频节点添加批注、创建空 Agent 槽位。", + "你只能规划低风险画布操作：视角聚焦/缩放/看全局、连接节点、更新文本节点、修改节点标题、更新图片或视频备注、给图片或视频节点添加批注、创建空 Agent 槽位。", "图片/视频生成、换背景、抠图、高清、扩图、重绘等媒体任务必须返回 action=none，交给其他执行器。", "当用户明确要求新建/添加空图片槽位、视频槽位、UI槽位、空框或占位时，可以返回 action=create_agent_slot；如果同时要求生成或填充媒体，返回 action=none。", "当用户要求给图片/视频/slot/节点改名、重命名或修改标题时，返回 action=rename_node；不要把图片节点标题改名误判为 update_text。", + "当用户要求给图片或视频添加/更新备注时，返回 action=update_note；批注、标注、箭头、框选说明才返回 action=add_annotation。", "删除、覆盖、替换现有图片、移动/重排节点等高风险动作必须返回 action=confirmation_required。", "只能使用上下文中出现的 nodeId。不要编造节点 ID。", "如果用户只是聊天或目标不明确，返回 action=none。", @@ -853,10 +907,10 @@ export async function planCanvasAgentOperationWithModel(params: { context: buildContextSummary(params.context), outputContract: { action: - "none | focus_nodes | fit_all | zoom_in | zoom_out | link_nodes | update_text | rename_node | add_annotation | create_agent_slot | confirmation_required", + "none | focus_nodes | fit_all | zoom_in | zoom_out | link_nodes | update_text | rename_node | update_note | add_annotation | create_agent_slot | confirmation_required", nodeIds: "只填上下文存在的 nodeId", textContent: - "update_text 时填写最终文本；rename_node 时填写新标题；add_annotation 时填写批注内容", + "update_text 时填写最终文本；rename_node 时填写新标题；update_note 时填写备注内容；add_annotation 时填写批注内容", slotType: "create_agent_slot 时填写 image/video/ui-screen", slotLabel: "create_agent_slot 时可填写槽位名称", aspectRatio: "create_agent_slot 时可填写 1:1/16:9 等比例", diff --git a/lib/canvas/agent-mutations.ts b/lib/canvas/agent-mutations.ts index 174d941f8..e5b35f165 100644 --- a/lib/canvas/agent-mutations.ts +++ b/lib/canvas/agent-mutations.ts @@ -10,6 +10,7 @@ import { findCanvasAgentOperationAudit, normalizeCanvasAgentAuditStringList, } from "@/lib/canvas/agent-operation-audit"; +import { syncCanvasNodeNoteAnnotationMetadata } from "@/lib/canvas/agent-annotation-metadata"; import type { CanvasMediaItem, CanvasOperationPlan, @@ -939,6 +940,7 @@ function applyOperationToCanvasData(params: { params.plan.operation !== "replace_node_media" && params.plan.operation !== "delete_nodes" && params.plan.operation !== "rename_node" && + params.plan.operation !== "update_note" && params.plan.operation !== "update_text" ) { throw new CanvasAgentMutationError("当前仅支持非破坏式画布写回操作"); @@ -1105,6 +1107,89 @@ function applyOperationToCanvasData(params: { }; } + if (params.plan.operation === "update_note") { + const note = String(params.plan.textContent || "").trim().slice(0, 1000); + if (!note) { + throw new CanvasAgentMutationError("缺少备注内容"); + } + + const targetNodeIds = getOperationTargetNodeIds(params.plan); + if (targetNodeIds.length === 0) { + throw new CanvasAgentMutationError("缺少要更新备注的节点"); + } + + const now = new Date().toISOString(); + const updatedNodeIds: string[] = []; + targetNodeIds.forEach((nodeId) => { + const node = findNode(nodes, nodeId); + if (!node) { + throw new CanvasAgentMutationError(`备注节点不存在：${nodeId}`); + } + if (node.type !== "image" && node.type !== "video") { + throw new CanvasAgentMutationError("update_note 只能更新图片或视频节点备注"); + } + + const data = (node.data || {}) as Record; + const previousNote = + typeof data.note === "string" ? String(data.note).slice(0, 1000) : undefined; + const noteSync = syncCanvasNodeNoteAnnotationMetadata({ + metadata: data.metadata, + nodeId: node.id, + note, + now, + }); + const syncedMetadata = normalizeCanvasMetadata(noteSync.metadata); + const annotations = Array.isArray(syncedMetadata.annotations) + ? syncedMetadata.annotations.map((annotation) => { + if ( + annotation && + typeof annotation === "object" && + (annotation as Record).id === noteSync.noteAnnotationId + ) { + return { + ...(annotation as Record), + createdByAgent: true, + updatedByAgent: true, + canvasAgentOperationId: params.plan.operationId, + }; + } + return annotation; + }) + : []; + + node.data = { + ...data, + note, + hasAnnotations: noteSync.hasStructuredAnnotations || data.hasAnnotations, + metadata: { + ...syncedMetadata, + ...(params.plan.metadata || {}), + annotations, + noteUpdatedByAgent: true, + canvasAgentOperationId: params.plan.operationId, + canvasAgentOperation: params.plan.operation, + previousNote, + agentNoteUpdatedAt: now, + }, + }; + node.updatedAt = now; + updatedNodeIds.push(node.id); + }); + + return { + canvasData: { + ...params.canvasData, + nodes, + edges, + selectedNodeIds: updatedNodeIds, + }, + createdNodeIds: [], + createdNodeIdsByMediaItemId: {}, + updatedNodeIds, + createdEdgeIds: [], + }; + } + if (params.plan.operation === "update_text") { const textContent = String(params.plan.textContent || "").trim(); if (!textContent) { diff --git a/lib/canvas/agent-operation-request.ts b/lib/canvas/agent-operation-request.ts index c1b71042c..485c3f61d 100644 --- a/lib/canvas/agent-operation-request.ts +++ b/lib/canvas/agent-operation-request.ts @@ -16,6 +16,7 @@ export const canvasAgentOperationPlanSchema = z "move_nodes", "delete_nodes", "rename_node", + "update_note", "update_text", ]), projectId: z.string().uuid().optional(), @@ -125,6 +126,7 @@ export const canvasAgentOperationPlanSchema = z value.operation === "move_nodes" || value.operation === "delete_nodes" || value.operation === "rename_node" || + value.operation === "update_note" || value.media || (value.mediaItems?.length ?? 0) > 0, { @@ -159,6 +161,15 @@ export const canvasAgentOperationPlanSchema = z message: "缺少节点标题", path: ["textContent"], } + ) + .refine( + (value) => + value.operation !== "update_note" || + Boolean(String(value.textContent || "").trim()), + { + message: "缺少备注内容", + path: ["textContent"], + } ); export const canvasAgentOperationRequestSchema = z.object({ diff --git a/tests/routes/canvas-agent-operations.test.ts b/tests/routes/canvas-agent-operations.test.ts index 74db41c33..f0faa9bcf 100644 --- a/tests/routes/canvas-agent-operations.test.ts +++ b/tests/routes/canvas-agent-operations.test.ts @@ -288,6 +288,13 @@ test.describe("canvas agent operation request schema", () => { targetNodeIds: ["image-1"], }) ).toContain("operationPlan.textContent"); + + expect( + issuePathsFor({ + operation: "update_note", + targetNodeIds: ["image-1"], + }) + ).toContain("operationPlan.textContent"); }); test("rejects invalid media urls", () => { @@ -797,6 +804,57 @@ test.describe("canvas agent operation mutation contract", () => { }); }); + test("updates image notes and syncs note annotations with trace metadata", () => { + const target = buildImageNode("image-note"); + (target.data as any).note = "旧备注"; + const result = applyCanvasOperationPlanToCanvasData({ + canvasData: getCanvasData([target]), + plan: { + version: 1, + operationId: "op-update-note", + operation: "update_note", + projectId, + targetNodeIds: ["image-note"], + textContent: "保留主体轮廓，背景可以更干净", + metadata: { + messageId: "assistant-message-note", + chatId: "chat-1", + }, + }, + }); + + const notedNode = getResultNodes(result.canvasData).find( + (node) => node.id === "image-note" + ); + + expect(result.createdNodeIds).toEqual([]); + expect(result.updatedNodeIds).toEqual(["image-note"]); + expect(notedNode).toMatchObject({ + data: { + note: "保留主体轮廓，背景可以更干净", + hasAnnotations: true, + metadata: { + noteUpdatedByAgent: true, + canvasAgentOperationId: "op-update-note", + canvasAgentOperation: "update_note", + previousNote: "旧备注", + messageId: "assistant-message-note", + annotations: [ + expect.objectContaining({ + id: "image-node-note:image-note", + sourceNodeId: "image-note", + type: "text-note", + text: "保留主体轮廓，背景可以更干净", + createdByAgent: true, + updatedByAgent: true, + canvasAgentOperationId: "op-update-note", + }), + ], + }, + }, + }); + }); + test("keeps batch media writes idempotent by operation id", () => { const anchor = buildImageNode("image-anchor"); const plan = { diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index 2a28d3ba5..bb1f4db73 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -761,6 +761,33 @@ test.describe("canvas chat pure helpers", () => { }); }); + test("normalizes model-planned image notes as note updates", () => { + const outcome = normalizeCanvasAgentModelPlannerDecision({ + decision: { + action: "update_note", + nodeIds: ["image-1", "text-1"], + textContent: "保留主体轮廓，背景可以更干净", + }, + context: plannerContext, + projectId: plannerContext.projectId || "", + messageId: "message-1", + chatId: "chat-1", + userText: "给选中图片备注：保留主体轮廓，背景可以更干净", + }); + + expect(outcome.kind).toBe("operation"); + expect(outcome.kind === "operation" ? outcome.operationPlan : null).toMatchObject({ + operation: "update_note", + targetNodeIds: ["image-1"], + textContent: "保留主体轮廓，背景可以更干净", + metadata: { + messageId: "message-1", + chatId: "chat-1", + planner: "model", + }, + }); + }); + test("normalizes model-planned annotations to selected media nodes", () => { const outcome = normalizeCanvasAgentModelPlannerDecision({ decision: { diff --git a/types/canvas/agent-protocol.ts b/types/canvas/agent-protocol.ts index 91379c166..fca6b11c8 100644 --- a/types/canvas/agent-protocol.ts +++ b/types/canvas/agent-protocol.ts @@ -120,6 +120,7 @@ export type CanvasOperationKind = | "move_nodes" | "delete_nodes" | "rename_node" + | "update_note" | "update_text"; export interface CanvasMediaItem {