From 06834699befeba0b63a50bf54e474f9177cef2cc Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 23 Jun 2026 18:15:08 +0800 Subject: [PATCH] fix: route explicit canvas agent operations --- app/(chat)/api/chat/route.ts | 169 +++++++++++++ lib/canvas/agent-explicit-operations.ts | 307 +++++++++++++++++++++++ tests/routes/canvas-chat-helpers.test.ts | 95 +++++++ 3 files changed, 571 insertions(+) create mode 100644 lib/canvas/agent-explicit-operations.ts diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index dd39dfdb6..f7c255a12 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -73,6 +73,10 @@ import { resolveCanvasAgentSlotCreationIntent, type CanvasAgentModelPlannerOutcome, } from "@/lib/canvas/agent-model-planner"; +import { + buildCanvasAgentExplicitOperationPlan, + type CanvasAgentExplicitOperationPlan, +} from "@/lib/canvas/agent-explicit-operations"; import { buildCanvasAgentResultViewOperation } from "@/lib/canvas/agent-result-view"; import { buildCanvasAgentComparisonViewPlan, @@ -3425,6 +3429,25 @@ export async function POST(request: Request) { }; }; + const resolveCanvasExplicitOperationPlan = + (): CanvasAgentExplicitOperationPlan | null => { + if (!canApplyCanvasAgentMutation) return null; + if (composerMode !== "agent") return null; + if (contextPayload?.source !== "canvas" || !contextPayload.projectId) return null; + if (!requestCanvasAgentContext) return null; + + return buildCanvasAgentExplicitOperationPlan({ + context: requestCanvasAgentContext, + projectId: contextPayload.projectId, + messageId: message.id, + chatId: id, + userText: requestTextContent, + requestedNodeIds: (requestTargetResolution?.targetRefs || []) + .map((ref) => ref.nodeId) + .filter((nodeId): nodeId is string => Boolean(nodeId)), + }); + }; + const resolveCanvasModelPlannerOutcome = async (): Promise => { if (composerMode !== "agent") return null; @@ -4038,6 +4061,152 @@ export async function POST(request: Request) { return createSseResponse(canvasTextStream); } + const canvasExplicitOperationPlan = resolveCanvasExplicitOperationPlan(); + if (canvasExplicitOperationPlan) { + const { operationPlan } = canvasExplicitOperationPlan; + const canvasExplicitStream = createUIMessageStream({ + execute: async ({ writer }) => { + const messageId = generateUUID(); + writeAgentProgressChunk( + writer, + { + stage: "executing", + currentStep: "execute", + title: canvasExplicitOperationPlan.progressTitle, + message: canvasExplicitOperationPlan.progressMessage, + toolName: "canvasOperation", + capability: operationPlan.operation, + }, + "canvas-explicit:progress-executing", + ); + try { + const result = await applyCanvasOperationPlan({ + userId: session.user.id, + projectId: operationPlan.projectId, + plan: operationPlan, + }); + const anchorNodeId = + result.createdNodeIds[0] || + operationPlan.targetNodeIds?.[0] || + operationPlan.anchorNodeId || + null; + writeCanvasOperationChunk( + writer, + { + projectId: operationPlan.projectId, + operationId: operationPlan.operationId, + operation: operationPlan.operation, + createdNodeIds: result.createdNodeIds, + createdNodeIdsByMediaItemId: result.createdNodeIdsByMediaItemId, + updatedNodeIds: result.updatedNodeIds, + createdEdgeIds: result.createdEdgeIds, + canvasRevision: result.canvasRevision, + anchorNodeId, + handledCanvasMediaItemIds: [], + message: canvasExplicitOperationPlan.reply, + }, + "canvas-explicit:operation", + ); + + const focusNodeIds = Array.from( + new Set([ + ...result.createdNodeIds, + ...(operationPlan.targetNodeIds || []), + ]) + ) + .filter(Boolean) + .slice(0, 20); + if (canvasAgentFeatures.viewOperationEnabled && focusNodeIds.length > 0) { + writeCanvasViewOperationChunk( + writer, + { + version: 1, + viewOperationId: `${operationPlan.operationId}:view`, + operation: focusNodeIds.length > 1 ? "focus_nodes" : "focus_selection", + projectId: operationPlan.projectId, + nodeIds: focusNodeIds, + anchorNodeId: focusNodeIds[0], + padding: operationPlan.operation === "create_text_node" ? 160 : 120, + persist: false, + reason: "focus_explicit_canvas_operation", + }, + "canvas-explicit:view-operation", + ); + } + writeAgentProgressChunk( + writer, + { + stage: "completed", + currentStep: "finalize", + title: canvasExplicitOperationPlan.completedTitle, + message: canvasExplicitOperationPlan.reply, + toolName: "canvasOperation", + capability: operationPlan.operation, + }, + "canvas-explicit:progress-completed", + ); + writeAssistantTextChunk( + writer, + canvasExplicitOperationPlan.reply, + "canvas-explicit:reply", + messageId, + ); + + await saveMessages({ + messages: [ + { + id: messageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: canvasExplicitOperationPlan.reply }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + } catch (error) { + const failure = writeCanvasOperationFailureChunk({ + writer, + label: "canvas-explicit:operation-failed", + projectId: operationPlan.projectId, + operationId: operationPlan.operationId, + operation: operationPlan.operation, + error, + anchorNodeId: + operationPlan.targetNodeIds?.[0] || + operationPlan.anchorNodeId || + null, + }); + writeAgentProgressChunk( + writer, + { + stage: "failed", + currentStep: "finalize", + title: canvasExplicitOperationPlan.failedTitle, + message: failure.errorMessage, + toolName: "canvasOperation", + capability: operationPlan.operation, + }, + "canvas-explicit:progress-failed", + ); + writeAssistantTextChunk( + writer, + failure.errorMessage, + "canvas-explicit:error", + messageId, + ); + } + }, + }); + + return createSseResponse(canvasExplicitStream); + } + const canvasModelPlannerOutcome = await resolveCanvasModelPlannerOutcome(); if (canvasModelPlannerOutcome && canvasModelPlannerOutcome.kind !== "none") { const canvasModelPlannerStream = createUIMessageStream({ diff --git a/lib/canvas/agent-explicit-operations.ts b/lib/canvas/agent-explicit-operations.ts new file mode 100644 index 000000000..7a8432b47 --- /dev/null +++ b/lib/canvas/agent-explicit-operations.ts @@ -0,0 +1,307 @@ +import type { + CanvasAgentContext, + CanvasOperationPlan, +} from "@/types/canvas/agent-protocol"; + +type ExplicitOperationKind = "create_text_node" | "rename_node" | "update_note"; + +export type CanvasAgentExplicitOperationPlan = { + operationPlan: CanvasOperationPlan; + reply: string; + progressTitle: string; + progressMessage: string; + completedTitle: string; + failedTitle: string; +}; + +function trimString(value: unknown, maxLength: number) { + return typeof value === "string" ? value.trim().slice(0, maxLength) : ""; +} + +function normalizeNodeIds(values: Array, maxItems = 20) { + return Array.from( + new Set(values.map((value) => trimString(value, 160)).filter(Boolean)) + ).slice(0, maxItems); +} + +function stripCommandContent(value: unknown, maxLength: number) { + return trimString(value, maxLength) + .replace(/^["'“”‘’]+|["'“”‘’。；;]+$/g, "") + .trim() + .slice(0, maxLength); +} + +function getKnownNodeTypes(context: CanvasAgentContext) { + const nodeMap = new Map(); + [...context.candidateNodes, ...context.selectedNodes].forEach((node) => { + if (node.nodeId) nodeMap.set(node.nodeId, node.type); + }); + context.slots.forEach((slot) => { + if (slot.nodeId) nodeMap.set(slot.nodeId, "agent-slot"); + }); + return nodeMap; +} + +function getSelectedNodeIds(context: CanvasAgentContext) { + return normalizeNodeIds([ + ...context.selectedNodeIds, + ...context.selectedNodes.map((node) => node.nodeId), + ...context.slots + .filter((slot) => slot.isSelected || context.selectedNodeIds.includes(slot.nodeId)) + .map((slot) => slot.nodeId), + ]); +} + +function getTargetNodeIds(params: { + context: CanvasAgentContext; + requestedNodeIds?: string[]; + allowedTypes?: string[]; +}) { + const knownNodeTypes = getKnownNodeTypes(params.context); + const allowedTypes = params.allowedTypes ? new Set(params.allowedTypes) : null; + return normalizeNodeIds([ + ...(params.requestedNodeIds || []), + ...getSelectedNodeIds(params.context), + ]).filter((nodeId) => { + const type = knownNodeTypes.get(nodeId); + if (!type) return false; + return allowedTypes ? allowedTypes.has(type) : true; + }); +} + +function getRenameAllowedTypes(userText: string) { + const lowerText = userText.toLowerCase(); + if (/槽位|slot/.test(lowerText)) return ["agent-slot"]; + if (/视频|video/.test(lowerText)) return ["video"]; + if (/图片|图像|照片|主图|参考图|image/.test(lowerText)) return ["image"]; + if (/文本|文字|文案|text/.test(lowerText)) return ["text"]; + return undefined; +} + +function hasMediaMutationIntent(userText: string) { + return /生成|出图|做一版|再出|换|改图|编辑图片|编辑图|抠图|去水印|高清|清晰|扩图|裁剪|删除|替换|背景|模型|洗图|重绘|尺寸|像素|分辨率/.test( + userText.toLowerCase() + ); +} + +function extractContentFromPatterns(userText: string, patterns: RegExp[], maxLength: number) { + for (const pattern of patterns) { + const match = userText.match(pattern); + const content = stripCommandContent(match?.[1] || "", maxLength); + if (content) return content; + } + return ""; +} + +function getViewportCenteredPosition(params: { + context: CanvasAgentContext; + size: { width: number; height: number }; +}) { + const viewport = params.context.viewport || { x: 0, y: 0, zoom: 1 }; + const zoom = Math.max(0.1, Number(viewport.zoom || 1)); + const viewportWidth = Math.max(320, Number(viewport.width || 1200)); + const viewportHeight = Math.max(320, Number(viewport.height || 800)); + return { + x: (viewportWidth / 2 - Number(viewport.x || 0)) / zoom - params.size.width / 2, + y: (viewportHeight / 2 - Number(viewport.y || 0)) / zoom - params.size.height / 2, + }; +} + +function buildPlanBase(params: { + context: CanvasAgentContext; + projectId: string; + messageId: string; + chatId: string; + userText: string; + operation: ExplicitOperationKind; +}): Pick< + CanvasOperationPlan, + | "version" + | "operationId" + | "operation" + | "projectId" + | "baseCanvasRevision" + | "baseCanvasUpdatedAt" + | "preserveOriginal" + | "metadata" +> { + return { + version: 1, + operationId: `${params.messageId}:canvas-${params.operation}`, + operation: params.operation, + projectId: params.projectId, + baseCanvasRevision: params.context.canvasRevision, + baseCanvasUpdatedAt: params.context.canvasUpdatedAt, + preserveOriginal: true, + metadata: { + messageId: params.messageId, + chatId: params.chatId, + planner: "deterministic", + prompt: params.userText, + }, + }; +} + +function buildCreateTextOperation(params: { + context: CanvasAgentContext; + projectId: string; + messageId: string; + chatId: string; + userText: string; +}): CanvasAgentExplicitOperationPlan | null { + const text = params.userText.trim(); + const hasCreateTextIntent = + /(?:创建|新增|新建|添加|加一段|加一句|写一段|写一句|放一段|放一句|写入|写上|放上).*(?:文本|文字|文案|标题|标语|说明|caption|copy|text)|(?:文本|文字|文案|标题|标语|说明).*(?:创建|新增|新建|添加|写入|写上)/i.test( + text + ); + if (!hasCreateTextIntent || hasMediaMutationIntent(text)) return null; + + const textContent = extractContentFromPatterns( + text, + [ + /(?:文本|文字|文案|标题|标语|说明|caption|copy|text)[^:：]*[:：]\s*([\s\S]+)$/i, + /(?:写入|写上|写成|内容是|设为|设置为)\s*[:：]?\s*([\s\S]+)$/i, + /[:：]\s*([\s\S]+)$/i, + ], + 4000 + ); + if (!textContent) return null; + + const textSize = { width: 320, height: 120 }; + const basePlan = buildPlanBase({ ...params, operation: "create_text_node" }); + return { + operationPlan: { + ...basePlan, + targetPosition: getViewportCenteredPosition({ + context: params.context, + size: textSize, + }), + textContent, + metadata: { + ...basePlan.metadata, + canvasAgentViewport: params.context.viewport, + }, + }, + reply: "已在当前画布添加文本节点。", + progressTitle: "正在添加画布文本", + progressMessage: "我会在当前视口中心创建一个新的文本节点。", + completedTitle: "画布文本已添加", + failedTitle: "画布文本添加失败", + }; +} + +function buildRenameOperation(params: { + context: CanvasAgentContext; + projectId: string; + messageId: string; + chatId: string; + userText: string; + requestedNodeIds?: string[]; +}): CanvasAgentExplicitOperationPlan | null { + const text = params.userText.trim(); + const hasRenameIntent = /改名|重命名|命名为|标题(?:改成|改为|设为|设置为)|改标题|修改标题/.test( + text + ); + if (!hasRenameIntent || hasMediaMutationIntent(text)) return null; + + const title = extractContentFromPatterns( + text, + [ + /(?:改名为|重命名为|命名为|标题(?:改成|改为|设为|设置为)|改标题为|修改标题为)\s*[:：]?\s*([\s\S]+)$/i, + /[:：]\s*([\s\S]+)$/i, + ], + 120 + ); + if (!title) return null; + + const targetNodeIds = getTargetNodeIds({ + context: params.context, + requestedNodeIds: params.requestedNodeIds, + allowedTypes: getRenameAllowedTypes(text), + }).slice(0, 10); + if (targetNodeIds.length === 0) return null; + + return { + operationPlan: { + ...buildPlanBase({ ...params, operation: "rename_node" }), + targetNodeIds, + textContent: title, + }, + reply: "已更新选中节点标题。", + progressTitle: "正在更新节点标题", + progressMessage: "我会只修改选中节点的标题，不改变媒体内容。", + completedTitle: "节点标题已更新", + failedTitle: "节点标题更新失败", + }; +} + +function buildUpdateNoteOperation(params: { + context: CanvasAgentContext; + projectId: string; + messageId: string; + chatId: string; + userText: string; + requestedNodeIds?: string[]; +}): CanvasAgentExplicitOperationPlan | null { + const text = params.userText.trim(); + if (!/备注|note/i.test(text)) return null; + if (/按.*(?:备注|note).*(?:修改|编辑|改图|改主图|生成|出图)/i.test(text)) { + return null; + } + if (hasMediaMutationIntent(text)) return null; + + const note = extractContentFromPatterns( + text, + [ + /(?:备注为|备注成|添加备注|加备注|写备注|备注|note)\s*[:：]?\s*([\s\S]+)$/i, + /[:：]\s*([\s\S]+)$/i, + ], + 1000 + ); + if (!note) return null; + + const targetNodeIds = getTargetNodeIds({ + context: params.context, + requestedNodeIds: params.requestedNodeIds, + allowedTypes: ["image", "video"], + }).slice(0, 10); + if (targetNodeIds.length === 0) return null; + + return { + operationPlan: { + ...buildPlanBase({ ...params, operation: "update_note" }), + targetNodeIds, + textContent: note, + }, + reply: "已更新选中图片备注。", + progressTitle: "正在更新图片备注", + progressMessage: "我会把备注保存到选中的图片或视频节点。", + completedTitle: "图片备注已更新", + failedTitle: "图片备注更新失败", + }; +} + +export function buildCanvasAgentExplicitOperationPlan(params: { + context: CanvasAgentContext; + projectId: string; + messageId: string; + chatId: string; + userText: string; + requestedNodeIds?: string[]; +}): CanvasAgentExplicitOperationPlan | null { + const commonParams = { + ...params, + projectId: trimString(params.projectId, 160), + messageId: trimString(params.messageId, 160), + chatId: trimString(params.chatId, 160), + }; + if (!commonParams.projectId || !commonParams.messageId || !commonParams.chatId) { + return null; + } + + return ( + buildCreateTextOperation(commonParams) || + buildRenameOperation(commonParams) || + buildUpdateNoteOperation(commonParams) + ); +} diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index 9154ef6b6..505f2ca6f 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -68,6 +68,7 @@ import { resolveCanvasAgentSlotCreationIntent, shouldConsiderCanvasAgentModelPlanner, } from "@/lib/canvas/agent-model-planner"; +import { buildCanvasAgentExplicitOperationPlan } from "@/lib/canvas/agent-explicit-operations"; import { buildCanvasAgentDirectWritebackPlan } from "@/lib/canvas/agent-direct-writeback"; import { resolveCanvasAgentMediaPlacements } from "@/lib/canvas/agent-placement"; import { buildCanvasAgentResultViewOperation } from "@/lib/canvas/agent-result-view"; @@ -913,6 +914,100 @@ test.describe("canvas chat pure helpers", () => { }); }); + test("builds deterministic text creation plans from explicit canvas prompts", () => { + const context: CanvasAgentContext = { + ...plannerContext, + selectedNodeIds: [], + selectedNodes: [], + viewport: { x: 0, y: 0, zoom: 1, width: 1200, height: 800 }, + focusPolicy: "auto", + }; + const outcome = buildCanvasAgentExplicitOperationPlan({ + context, + projectId: plannerContext.projectId || "", + messageId: "message-1", + chatId: "chat-1", + userText: "在画布上添加文字：新品上市", + }); + + expect(outcome?.operationPlan).toMatchObject({ + operation: "create_text_node", + targetPosition: { x: 440, y: 340 }, + textContent: "新品上市", + preserveOriginal: true, + metadata: { + messageId: "message-1", + chatId: "chat-1", + planner: "deterministic", + canvasAgentViewport: { x: 0, y: 0, zoom: 1, width: 1200, height: 800 }, + }, + }); + }); + + test("builds deterministic node rename plans without treating titles as image edits", () => { + const outcome = buildCanvasAgentExplicitOperationPlan({ + context: plannerContext, + projectId: plannerContext.projectId || "", + messageId: "message-1", + chatId: "chat-1", + userText: "把选中图片改名为白底主图 A", + }); + + expect(outcome?.operationPlan).toMatchObject({ + operation: "rename_node", + targetNodeIds: ["image-1"], + textContent: "白底主图 A", + metadata: { + messageId: "message-1", + chatId: "chat-1", + planner: "deterministic", + }, + }); + }); + + test("builds deterministic note updates for selected media nodes only", () => { + const outcome = buildCanvasAgentExplicitOperationPlan({ + context: plannerContext, + projectId: plannerContext.projectId || "", + messageId: "message-1", + chatId: "chat-1", + userText: "给选中图片备注：保留主体轮廓", + }); + + expect(outcome?.operationPlan).toMatchObject({ + operation: "update_note", + targetNodeIds: ["image-1"], + textContent: "保留主体轮廓", + metadata: { + messageId: "message-1", + chatId: "chat-1", + planner: "deterministic", + }, + }); + }); + + test("does not route media generation or note-driven image edits as explicit canvas operations", () => { + expect( + buildCanvasAgentExplicitOperationPlan({ + context: plannerContext, + projectId: plannerContext.projectId || "", + messageId: "message-1", + chatId: "chat-1", + userText: "按这个备注修改主图", + }) + ).toBeNull(); + + expect( + buildCanvasAgentExplicitOperationPlan({ + context: plannerContext, + projectId: plannerContext.projectId || "", + messageId: "message-1", + chatId: "chat-1", + userText: "生成一张图，标题文字：新品上市", + }) + ).toBeNull(); + }); + test("normalizes model-planned annotations to selected media nodes", () => { const outcome = normalizeCanvasAgentModelPlannerDecision({ decision: {