lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: let canvas agent update image notes

Browse Source
This commit is contained in:
134 2026-06-23 16:34:52 +08:00
parent e319b16f48
commit 38dba2b1bc
7 changed files with 244 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/无限画布对话框Agent模式升级开发文档-借鉴Cowart-2026-06-22.md
View File

@ -616,6 +616,7 @@ export type CanvasOperationKind =
| "link_nodes"
| "move_nodes"
| "rename_node"
| "update_note"
| "update_text";
export interface CanvasMediaItem {
@ -740,6 +741,7 @@ Agent 首期只能请求以下操作，后端 mutation 层也只接受这些操
| `append_revision` | 基于源节点追加一个新版本 | 低 | 否 |
| `link_nodes` | 给两个节点建立参考/版本关系 | 低 | 否 |
| `rename_node` | 修改图片/视频/slot/节点标题 | 低 | 否 |
| `update_note` | 更新图片/视频节点备注并同步结构化备注标注 | 低 | 否 |
| `update_text` | 修改选中文本节点内容 | 中 | 视情况 |
| `replace_node_media` | 替换已有图片/视频内容 | 高 | 是 |
| `move_nodes` | 批量移动或整理节点 | 中 | 视情况 |
@ -751,6 +753,7 @@ Agent 首期只能请求以下操作，后端 mutation 层也只接受这些操
- `append_revision`
- `link_nodes`
- `rename_node`
- `update_note`
这样可以先把“对话框驱动画布”跑通，同时把误操作风险降到最低。
@ -1066,6 +1069,7 @@ autosave 冲突要求：
- `append_revision`
- `link_nodes`
- `rename_node`
- `update_note`
首期 revision 不一致时必须拒绝或要求确认的操作：
@ -1169,6 +1173,7 @@ hasSelectedAgentSlot?: boolean;
| “按批注修改” | `append_revision` | selected image + annotations |
| “把这两张连起来” | `link_nodes` | selected nodes |
| “把选中图片改名为白底主图 A” | `rename_node` | selected image/video/slot node |
| “给选中图片加备注：保留主体轮廓” | `update_note` | selected image/video node |
| “把这段文字改成更高级” | `update_text` | selected text node |
| “替换掉原图” | `replace_node_media` | selected image，需要确认 |
| “把这些整理成三列” | `move_nodes` | selected nodes，建议确认 |

60
lib/canvas/agent-model-planner.ts
View File

@ -18,6 +18,7 @@ const plannerDecisionSchema = z.object({
"link_nodes",
"update_text",
"rename_node",
"update_note",
"add_annotation",
"create_agent_slot",
"confirmation_required",
@ -421,6 +422,7 @@ export function shouldConsiderCanvasAgentModelPlanner(params: {
text
);
const hasRenameIntent = /改名|重命名|命名|标题/.test(text);
const hasNoteIntent = /备注|note/.test(text);
const hasAnnotationIntent = /批注|标注|备注|note|annotation/.test(text);
const hasSlotIntent = hasCanvasAgentSlotCreationIntent(text);
const hasHighRiskIntent = Boolean(inferHighRiskOperation(text));
@ -435,6 +437,7 @@ export function shouldConsiderCanvasAgentModelPlanner(params: {
!hasLinkIntent &&
!hasTextIntent &&
!hasRenameIntent &&
!hasNoteIntent &&
!hasAnnotationIntent &&
!hasSlotIntent &&
!hasHighRiskIntent
@ -447,6 +450,7 @@ export function shouldConsiderCanvasAgentModelPlanner(params: {
hasLinkIntent ||
hasTextIntent ||
hasRenameIntent ||
hasNoteIntent ||
hasAnnotationIntent ||
hasSlotIntent ||
hasHighRiskIntent
@ -719,6 +723,55 @@ export function normalizeCanvasAgentModelPlannerDecision(params: {
};
}
if (decision.action === "update_note") {
const knownNodes = buildKnownNodeMap(params.context);
const targetNodeIds = getDecisionNodeIds({
decision,
context: params.context,
allowSelectedFallback: true,
maxItems: 10,
}).filter((nodeId) => {
const type = knownNodes.get(nodeId)?.type;
return type === "image" || type === "video";
});
const note = trimString(decision.textContent, 1000);
if (targetNodeIds.length === 0 || !note) {
return { kind: "none", reason: "update_note_missing_target_or_content" };
}
const operationPlan: CanvasOperationPlan = {
version: 1,
operationId,
operation: "update_note",
projectId: params.projectId,
baseCanvasRevision: params.context.canvasRevision,
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
targetNodeIds,
textContent: note,
preserveOriginal: true,
metadata: {
messageId: params.messageId,
chatId: params.chatId,
planner: "model",
prompt: params.userText,
},
};
return {
kind: "operation",
operationPlan,
viewPlan: buildViewPlan({
action: "focus_nodes",
context: params.context,
projectId: params.projectId,
operationId,
nodeIds: targetNodeIds,
reason: "focus_model_note_nodes",
}) || undefined,
reply: trimString(decision.reply, 280) || "已更新选中图片备注。",
reason,
};
}
if (decision.action === "update_text") {
const knownNodes = buildKnownNodeMap(params.context);
const requestedIds = getDecisionNodeIds({
@ -839,10 +892,11 @@ export async function planCanvasAgentOperationWithModel(params: {
abortSignal: params.abortSignal || AbortSignal.timeout(18_000),
system: [
"你是无限画布 Agent 的受控操作规划器。",
"你只能规划低风险画布操作：视角聚焦/缩放/看全局、连接节点、更新文本节点、修改节点标题、给图片或视频节点添加批注、创建空 Agent 槽位。",
"你只能规划低风险画布操作：视角聚焦/缩放/看全局、连接节点、更新文本节点、修改节点标题、更新图片或视频备注、给图片或视频节点添加批注、创建空 Agent 槽位。",
"图片/视频生成、换背景、抠图、高清、扩图、重绘等媒体任务必须返回 action=none，交给其他执行器。",
"当用户明确要求新建/添加空图片槽位、视频槽位、UI槽位、空框或占位时，可以返回 action=create_agent_slot；如果同时要求生成或填充媒体，返回 action=none。",
"当用户要求给图片/视频/slot/节点改名、重命名或修改标题时，返回 action=rename_node；不要把图片节点标题改名误判为 update_text。",
"当用户要求给图片或视频添加/更新备注时，返回 action=update_note；批注、标注、箭头、框选说明才返回 action=add_annotation。",
"删除、覆盖、替换现有图片、移动/重排节点等高风险动作必须返回 action=confirmation_required。",
"只能使用上下文中出现的 nodeId。不要编造节点 ID。",
"如果用户只是聊天或目标不明确，返回 action=none。",
@ -853,10 +907,10 @@ export async function planCanvasAgentOperationWithModel(params: {
context: buildContextSummary(params.context),
outputContract: {
action:
"none | focus_nodes | fit_all | zoom_in | zoom_out | link_nodes | update_text | rename_node | add_annotation | create_agent_slot | confirmation_required",
"none | focus_nodes | fit_all | zoom_in | zoom_out | link_nodes | update_text | rename_node | update_note | add_annotation | create_agent_slot | confirmation_required",
nodeIds: "只填上下文存在的 nodeId",
textContent:
"update_text 时填写最终文本；rename_node 时填写新标题；add_annotation 时填写批注内容",
"update_text 时填写最终文本；rename_node 时填写新标题；update_note 时填写备注内容；add_annotation 时填写批注内容",
slotType: "create_agent_slot 时填写 image/video/ui-screen",
slotLabel: "create_agent_slot 时可填写槽位名称",
aspectRatio: "create_agent_slot 时可填写 1:1/16:9 等比例",

85
lib/canvas/agent-mutations.ts
View File

@ -10,6 +10,7 @@ import {
findCanvasAgentOperationAudit,
normalizeCanvasAgentAuditStringList,
} from "@/lib/canvas/agent-operation-audit";
import { syncCanvasNodeNoteAnnotationMetadata } from "@/lib/canvas/agent-annotation-metadata";
import type {
CanvasMediaItem,
CanvasOperationPlan,
@ -939,6 +940,7 @@ function applyOperationToCanvasData(params: {
params.plan.operation !== "replace_node_media" &&
params.plan.operation !== "delete_nodes" &&
params.plan.operation !== "rename_node" &&
params.plan.operation !== "update_note" &&
params.plan.operation !== "update_text"
) {
throw new CanvasAgentMutationError("当前仅支持非破坏式画布写回操作");
@ -1105,6 +1107,89 @@ function applyOperationToCanvasData(params: {
};
}
if (params.plan.operation === "update_note") {
const note = String(params.plan.textContent || "").trim().slice(0, 1000);
if (!note) {
throw new CanvasAgentMutationError("缺少备注内容");
}
const targetNodeIds = getOperationTargetNodeIds(params.plan);
if (targetNodeIds.length === 0) {
throw new CanvasAgentMutationError("缺少要更新备注的节点");
}
const now = new Date().toISOString();
const updatedNodeIds: string[] = [];
targetNodeIds.forEach((nodeId) => {
const node = findNode(nodes, nodeId);
if (!node) {
throw new CanvasAgentMutationError(`备注节点不存在：${nodeId}`);
}
if (node.type !== "image" && node.type !== "video") {
throw new CanvasAgentMutationError("update_note 只能更新图片或视频节点备注");
}
const data = (node.data || {}) as Record<string, any>;
const previousNote =
typeof data.note === "string" ? String(data.note).slice(0, 1000) : undefined;
const noteSync = syncCanvasNodeNoteAnnotationMetadata({
metadata: data.metadata,
nodeId: node.id,
note,
now,
});
const syncedMetadata = normalizeCanvasMetadata(noteSync.metadata);
const annotations = Array.isArray(syncedMetadata.annotations)
? syncedMetadata.annotations.map((annotation) => {
if (
annotation &&
typeof annotation === "object" &&
(annotation as Record<string, unknown>).id === noteSync.noteAnnotationId
) {
return {
...(annotation as Record<string, unknown>),
createdByAgent: true,
updatedByAgent: true,
canvasAgentOperationId: params.plan.operationId,
};
}
return annotation;
})
: [];
node.data = {
...data,
note,
hasAnnotations: noteSync.hasStructuredAnnotations || data.hasAnnotations,
metadata: {
...syncedMetadata,
...(params.plan.metadata || {}),
annotations,
noteUpdatedByAgent: true,
canvasAgentOperationId: params.plan.operationId,
canvasAgentOperation: params.plan.operation,
previousNote,
agentNoteUpdatedAt: now,
},
};
node.updatedAt = now;
updatedNodeIds.push(node.id);
});
return {
canvasData: {
...params.canvasData,
nodes,
edges,
selectedNodeIds: updatedNodeIds,
},
createdNodeIds: [],
createdNodeIdsByMediaItemId: {},
updatedNodeIds,
createdEdgeIds: [],
};
}
if (params.plan.operation === "update_text") {
const textContent = String(params.plan.textContent || "").trim();
if (!textContent) {

11
lib/canvas/agent-operation-request.ts
View File

@ -16,6 +16,7 @@ export const canvasAgentOperationPlanSchema = z
"move_nodes",
"delete_nodes",
"rename_node",
"update_note",
"update_text",
]),
projectId: z.string().uuid().optional(),
@ -125,6 +126,7 @@ export const canvasAgentOperationPlanSchema = z
value.operation === "move_nodes" ||
value.operation === "delete_nodes" ||
value.operation === "rename_node" ||
value.operation === "update_note" ||
value.media ||
(value.mediaItems?.length ?? 0) > 0,
{
@ -159,6 +161,15 @@ export const canvasAgentOperationPlanSchema = z
message: "缺少节点标题",
path: ["textContent"],
}
)
.refine(
(value) =>
value.operation !== "update_note" ||
Boolean(String(value.textContent || "").trim()),
{
message: "缺少备注内容",
path: ["textContent"],
}
);
export const canvasAgentOperationRequestSchema = z.object({

58
tests/routes/canvas-agent-operations.test.ts
View File

@ -288,6 +288,13 @@ test.describe("canvas agent operation request schema", () => {
targetNodeIds: ["image-1"],
})
).toContain("operationPlan.textContent");
expect(
issuePathsFor({
operation: "update_note",
targetNodeIds: ["image-1"],
})
).toContain("operationPlan.textContent");
});
test("rejects invalid media urls", () => {
@ -797,6 +804,57 @@ test.describe("canvas agent operation mutation contract", () => {
});
});
test("updates image notes and syncs note annotations with trace metadata", () => {
const target = buildImageNode("image-note");
(target.data as any).note = "旧备注";
const result = applyCanvasOperationPlanToCanvasData({
canvasData: getCanvasData([target]),
plan: {
version: 1,
operationId: "op-update-note",
operation: "update_note",
projectId,
targetNodeIds: ["image-note"],
textContent: "保留主体轮廓，背景可以更干净",
metadata: {
messageId: "assistant-message-note",
chatId: "chat-1",
},
},
});
const notedNode = getResultNodes(result.canvasData).find(
(node) => node.id === "image-note"
);
expect(result.createdNodeIds).toEqual([]);
expect(result.updatedNodeIds).toEqual(["image-note"]);
expect(notedNode).toMatchObject({
data: {
note: "保留主体轮廓，背景可以更干净",
hasAnnotations: true,
metadata: {
noteUpdatedByAgent: true,
canvasAgentOperationId: "op-update-note",
canvasAgentOperation: "update_note",
previousNote: "旧备注",
messageId: "assistant-message-note",
annotations: [
expect.objectContaining({
id: "image-node-note:image-note",
sourceNodeId: "image-note",
type: "text-note",
text: "保留主体轮廓，背景可以更干净",
createdByAgent: true,
updatedByAgent: true,
canvasAgentOperationId: "op-update-note",
}),
],
},
},
});
});
test("keeps batch media writes idempotent by operation id", () => {
const anchor = buildImageNode("image-anchor");
const plan = {

27
tests/routes/canvas-chat-helpers.test.ts
View File

@ -761,6 +761,33 @@ test.describe("canvas chat pure helpers", () => {
});
});
test("normalizes model-planned image notes as note updates", () => {
const outcome = normalizeCanvasAgentModelPlannerDecision({
decision: {
action: "update_note",
nodeIds: ["image-1", "text-1"],
textContent: "保留主体轮廓，背景可以更干净",
},
context: plannerContext,
projectId: plannerContext.projectId || "",
messageId: "message-1",
chatId: "chat-1",
userText: "给选中图片备注：保留主体轮廓，背景可以更干净",
});
expect(outcome.kind).toBe("operation");
expect(outcome.kind === "operation" ? outcome.operationPlan : null).toMatchObject({
operation: "update_note",
targetNodeIds: ["image-1"],
textContent: "保留主体轮廓，背景可以更干净",
metadata: {
messageId: "message-1",
chatId: "chat-1",
planner: "model",
},
});
});
test("normalizes model-planned annotations to selected media nodes", () => {
const outcome = normalizeCanvasAgentModelPlannerDecision({
decision: {

1
types/canvas/agent-protocol.ts
View File

@ -120,6 +120,7 @@ export type CanvasOperationKind =
| "move_nodes"
| "delete_nodes"
| "rename_node"
| "update_note"
| "update_text";
export interface CanvasMediaItem {