diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 18eca6087..07b519ced 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -78,6 +78,7 @@ import { buildCanvasAgentComparisonViewPlan, hasCanvasAgentComparisonViewIntent, } from "@/lib/canvas/agent-view-intent"; +import { buildCanvasAgentDirectWritebackPlan } from "@/lib/canvas/agent-direct-writeback"; import { getCanvasAgentOperationErrorMessage, normalizeCanvasAgentOperationErrorCode, @@ -85,8 +86,6 @@ import { import { applyCanvasOperationPlan } from "@/lib/canvas/agent-mutations"; import { isProductionEnvironment } from "@/lib/constants"; import type { - CanvasMediaItem, - CanvasOperationKind, CanvasOperationPlan, CanvasViewOperationPlan, } from "@/types/canvas/agent-protocol"; @@ -2843,101 +2842,36 @@ export async function POST(request: Request) { return { mediaItemIds: [], handledCanvasMediaItemIds: [] }; } - const resultUrls = params.resultUrls - .map((url) => String(url || "").trim()) - .filter((url) => /^https?:\/\//i.test(url)); - if (resultUrls.length === 0) { + const writebackPlan = buildCanvasAgentDirectWritebackPlan({ + projectId, + chatId: id, + assistantMessageId: params.assistantMessageId, + operationLabel: params.operationLabel, + resultType: params.resultType, + resultUrls: params.resultUrls, + prompt: params.prompt, + mode: params.mode, + context: requestCanvasAgentContext, + requestTextContent, + targetRefs: requestTargetResolution?.targetRefs || [], + sourceImageUrls: params.sourceImageUrls, + assetIdsByUrl: params.assetIdsByUrl, + model: params.model, + targetSlotId: params.targetSlotId, + }); + if (!writebackPlan) { return { mediaItemIds: [], handledCanvasMediaItemIds: [] }; } - const preferredTargetSlotId = String(params.targetSlotId || "").trim(); - const selectedSlot = getSelectedSlots(requestCanvasAgentContext)[0] || null; - const targetSlotId = preferredTargetSlotId || selectedSlot?.nodeId || ""; - const shouldFillSlot = Boolean(targetSlotId); - const selectedMediaNodes = getSelectedMediaNodes(requestCanvasAgentContext); - const anchorNode = - selectedSlot || selectedMediaNodes[0] || getBestAnchorNode(requestCanvasAgentContext); - const targetRefs = requestTargetResolution?.targetRefs || []; - const targetNodeIds = Array.from( - new Set( - targetRefs - .map((ref) => ref.nodeId) - .filter((value): value is string => Boolean(value)) - ) - ); - const selectedMediaNodeIds = selectedMediaNodes - .map((node) => node.nodeId) - .filter((value): value is string => Boolean(value)); - const perResultSourceNodeIds = - !shouldFillSlot && resultUrls.length > 1 - ? [targetNodeIds, selectedMediaNodeIds].find( - (nodeIds) => nodeIds.length === resultUrls.length - ) || [] - : []; - const sourceNodeIds = Array.from( - new Set( - [ - ...targetNodeIds, - ...selectedMediaNodeIds, - anchorNode?.nodeId, - ].filter((value): value is string => Boolean(value)) - ) - ); - const operation: CanvasOperationKind = selectedSlot - ? "fill_slot" - : shouldFillSlot - ? "fill_slot" - : params.mode === "edit" - ? "append_revision" - : "insert_beside"; - const mediaItems: CanvasMediaItem[] = resultUrls.map((url, index) => { - const mediaItemId = `${params.assistantMessageId}:${params.operationLabel}:${index + 1}`; - return { - mediaItemId, - type: params.resultType, - url, - assetId: params.assetIdsByUrl?.[url], - title: params.resultType === "video" ? "Agent 视频结果" : "Agent 图片结果", - prompt: params.prompt, - sourceNodeIds: perResultSourceNodeIds[index] - ? [perResultSourceNodeIds[index]] - : sourceNodeIds, - metadata: { - messageId: params.assistantMessageId, - chatId: id, - model: params.model || undefined, - sourceImageUrls: params.sourceImageUrls || [], - }, - }; - }); - const operationId = `${params.assistantMessageId}:${params.operationLabel}`; - const plan: CanvasOperationPlan = { - version: 1, - operationId, - operation, - projectId, - baseCanvasRevision: requestCanvasAgentContext.canvasRevision, - baseCanvasUpdatedAt: requestCanvasAgentContext.canvasUpdatedAt, - sourceNodeIds, - targetSlotId: targetSlotId || undefined, - anchorNodeId: shouldFillSlot ? undefined : anchorNode?.nodeId, - placement: "right", - placementStrategy: - perResultSourceNodeIds.length === mediaItems.length - ? "beside_each_source" - : mediaItems.length > 1 - ? "grid" - : "single", - preserveOriginal: true, + const { + mediaItemIds, mediaItems, - metadata: { - messageId: params.assistantMessageId, - chatId: id, - mode: params.mode, - prompt: params.prompt || requestTextContent, - }, - }; - const mediaItemIds = mediaItems.map((item) => item.mediaItemId); + operation, + operationId, + plan, + selectedMediaNodes, + shouldFillSlot, + } = writebackPlan; if (!canApplyCanvasAgentMutation) { if (shouldStreamCanvasAgentShadowOperations(canvasAgentFeatures)) { writeCanvasOperationShadowChunk({ diff --git a/lib/canvas/agent-direct-writeback.ts b/lib/canvas/agent-direct-writeback.ts new file mode 100644 index 000000000..d1e4a0000 --- /dev/null +++ b/lib/canvas/agent-direct-writeback.ts @@ -0,0 +1,159 @@ +import { + getBestAnchorNode, + getSelectedMediaNodes, + getSelectedSlots, +} from "@/lib/canvas/agent-context"; +import type { + CanvasAgentContext, + CanvasMediaItem, + CanvasOperationKind, + CanvasOperationPlan, +} from "@/types/canvas/agent-protocol"; + +export type CanvasAgentDirectWritebackMode = "generate" | "edit" | "video"; +export type CanvasAgentDirectWritebackResultType = "image" | "video"; + +export interface CanvasAgentDirectWritebackTargetRef { + nodeId?: string | null; +} + +export interface BuildCanvasAgentDirectWritebackPlanParams { + projectId: string; + chatId: string; + assistantMessageId: string; + operationLabel: string; + resultType: CanvasAgentDirectWritebackResultType; + resultUrls: string[]; + prompt: string; + mode: CanvasAgentDirectWritebackMode; + context: CanvasAgentContext; + requestTextContent?: string | null; + targetRefs?: CanvasAgentDirectWritebackTargetRef[]; + sourceImageUrls?: string[]; + assetIdsByUrl?: Record; + model?: string | null; + targetSlotId?: string | null; +} + +export interface CanvasAgentDirectWritebackPlanResult { + plan: CanvasOperationPlan; + mediaItems: CanvasMediaItem[]; + mediaItemIds: string[]; + operationId: string; + operation: CanvasOperationKind; + targetSlotId: string; + shouldFillSlot: boolean; + selectedMediaNodes: ReturnType; +} + +export function buildCanvasAgentDirectWritebackPlan( + params: BuildCanvasAgentDirectWritebackPlanParams +): CanvasAgentDirectWritebackPlanResult | null { + const projectId = String(params.projectId || "").trim(); + if (!projectId) return null; + + const resultUrls = params.resultUrls + .map((url) => String(url || "").trim()) + .filter((url) => /^https?:\/\//i.test(url)); + if (resultUrls.length === 0) return null; + + const preferredTargetSlotId = String(params.targetSlotId || "").trim(); + const selectedSlot = getSelectedSlots(params.context)[0] || null; + const targetSlotId = preferredTargetSlotId || selectedSlot?.nodeId || ""; + const shouldFillSlot = Boolean(targetSlotId); + const selectedMediaNodes = getSelectedMediaNodes(params.context); + const anchorNode = + selectedSlot || selectedMediaNodes[0] || getBestAnchorNode(params.context); + const targetRefs = params.targetRefs || []; + const targetNodeIds = Array.from( + new Set( + targetRefs + .map((ref) => ref.nodeId) + .filter((value): value is string => Boolean(value)) + ) + ); + const selectedMediaNodeIds = selectedMediaNodes + .map((node) => node.nodeId) + .filter((value): value is string => Boolean(value)); + const perResultSourceNodeIds = + !shouldFillSlot && resultUrls.length > 1 + ? [targetNodeIds, selectedMediaNodeIds].find( + (nodeIds) => nodeIds.length === resultUrls.length + ) || [] + : []; + const sourceNodeIds = Array.from( + new Set( + [ + ...targetNodeIds, + ...selectedMediaNodeIds, + anchorNode?.nodeId, + ].filter((value): value is string => Boolean(value)) + ) + ); + const operation: CanvasOperationKind = selectedSlot + ? "fill_slot" + : shouldFillSlot + ? "fill_slot" + : params.mode === "edit" + ? "append_revision" + : "insert_beside"; + const mediaItems: CanvasMediaItem[] = resultUrls.map((url, index) => { + const mediaItemId = `${params.assistantMessageId}:${params.operationLabel}:${index + 1}`; + return { + mediaItemId, + type: params.resultType, + url, + assetId: params.assetIdsByUrl?.[url], + title: + params.resultType === "video" ? "Agent 视频结果" : "Agent 图片结果", + prompt: params.prompt, + sourceNodeIds: perResultSourceNodeIds[index] + ? [perResultSourceNodeIds[index]] + : sourceNodeIds, + metadata: { + messageId: params.assistantMessageId, + chatId: params.chatId, + model: params.model || undefined, + sourceImageUrls: params.sourceImageUrls || [], + }, + }; + }); + const operationId = `${params.assistantMessageId}:${params.operationLabel}`; + const plan: CanvasOperationPlan = { + version: 1, + operationId, + operation, + projectId, + baseCanvasRevision: params.context.canvasRevision, + baseCanvasUpdatedAt: params.context.canvasUpdatedAt, + sourceNodeIds, + targetSlotId: targetSlotId || undefined, + anchorNodeId: shouldFillSlot ? undefined : anchorNode?.nodeId, + placement: "right", + placementStrategy: + perResultSourceNodeIds.length === mediaItems.length + ? "beside_each_source" + : mediaItems.length > 1 + ? "grid" + : "single", + preserveOriginal: true, + mediaItems, + metadata: { + messageId: params.assistantMessageId, + chatId: params.chatId, + mode: params.mode, + prompt: params.prompt || params.requestTextContent || "", + }, + }; + + return { + plan, + mediaItems, + mediaItemIds: mediaItems.map((item) => item.mediaItemId), + operationId, + operation, + targetSlotId, + shouldFillSlot, + selectedMediaNodes, + }; +} diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index 18d4700cf..ed8623cf3 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -55,6 +55,7 @@ import { resolveCanvasAgentSlotCreationIntent, shouldConsiderCanvasAgentModelPlanner, } from "@/lib/canvas/agent-model-planner"; +import { buildCanvasAgentDirectWritebackPlan } from "@/lib/canvas/agent-direct-writeback"; import { resolveCanvasAgentMediaPlacements } from "@/lib/canvas/agent-placement"; import { buildCanvasAgentResultViewOperation } from "@/lib/canvas/agent-result-view"; import { @@ -371,6 +372,199 @@ test.describe("canvas chat pure helpers", () => { }); }); + test("plans direct writeback as a slot fill when an agent slot is selected", () => { + const context: CanvasAgentContext = { + ...plannerContext, + selectedNodeIds: ["slot-1"], + selectedNodes: [ + { + nodeId: "slot-1", + type: "agent-slot", + title: "主图槽位", + x: 100, + y: 120, + width: 320, + height: 240, + isSelected: true, + }, + ], + slots: [ + { + nodeId: "slot-1", + slotType: "image", + label: "主图槽位", + x: 100, + y: 120, + width: 320, + height: 240, + aspectRatio: "4:3", + isSelected: true, + fillPolicy: "replace-empty", + }, + ], + }; + + const result = buildCanvasAgentDirectWritebackPlan({ + projectId: plannerContext.projectId || "", + chatId: "chat-1", + assistantMessageId: "assistant-1", + operationLabel: "agent-generate-image", + resultType: "image", + resultUrls: ["https://cdn.example.com/slot.png"], + prompt: "生成主图", + mode: "generate", + context, + model: "image-model", + }); + + expect(result).toMatchObject({ + operationId: "assistant-1:agent-generate-image", + operation: "fill_slot", + targetSlotId: "slot-1", + shouldFillSlot: true, + mediaItemIds: ["assistant-1:agent-generate-image:1"], + plan: { + operation: "fill_slot", + targetSlotId: "slot-1", + anchorNodeId: undefined, + placementStrategy: "single", + sourceNodeIds: ["slot-1"], + }, + }); + expect(result?.mediaItems[0]).toMatchObject({ + mediaItemId: "assistant-1:agent-generate-image:1", + type: "image", + url: "https://cdn.example.com/slot.png", + prompt: "生成主图", + sourceNodeIds: ["slot-1"], + metadata: { + messageId: "assistant-1", + chatId: "chat-1", + model: "image-model", + }, + }); + }); + + test("plans direct image edits as append revisions anchored to selected media", () => { + const result = buildCanvasAgentDirectWritebackPlan({ + projectId: plannerContext.projectId || "", + chatId: "chat-1", + assistantMessageId: "assistant-2", + operationLabel: "agent-edit-image", + resultType: "image", + resultUrls: [ + "relative.png", + "https://cdn.example.com/revision.png", + ], + prompt: "", + requestTextContent: "把选中图片改成黄昏光线", + mode: "edit", + context: plannerContext, + targetRefs: [{ nodeId: "image-1" }], + sourceImageUrls: ["https://cdn.example.com/source.png"], + assetIdsByUrl: { + "https://cdn.example.com/revision.png": "asset-1", + }, + }); + + expect(result).toMatchObject({ + operation: "append_revision", + shouldFillSlot: false, + mediaItemIds: ["assistant-2:agent-edit-image:1"], + plan: { + operation: "append_revision", + anchorNodeId: "image-1", + placementStrategy: "single", + metadata: { + prompt: "把选中图片改成黄昏光线", + }, + }, + }); + expect(result?.mediaItems[0]).toMatchObject({ + assetId: "asset-1", + sourceNodeIds: ["image-1"], + metadata: { + sourceImageUrls: ["https://cdn.example.com/source.png"], + }, + }); + }); + + test("plans direct batch generations beside each matched source", () => { + const context: CanvasAgentContext = { + ...plannerContext, + selectedNodeIds: ["image-a", "image-b"], + selectedNodes: [ + { + nodeId: "image-a", + type: "image", + title: "A", + x: 0, + y: 0, + width: 200, + height: 160, + isSelected: true, + }, + { + nodeId: "image-b", + type: "image", + title: "B", + x: 260, + y: 0, + width: 200, + height: 160, + isSelected: true, + }, + ], + }; + + const result = buildCanvasAgentDirectWritebackPlan({ + projectId: plannerContext.projectId || "", + chatId: "chat-1", + assistantMessageId: "assistant-3", + operationLabel: "agent-generate-image", + resultType: "image", + resultUrls: [ + "https://cdn.example.com/a.png", + "https://cdn.example.com/b.png", + ], + prompt: "分别生成两个变体", + mode: "generate", + context, + targetRefs: [{ nodeId: "image-a" }, { nodeId: "image-b" }], + }); + + expect(result).toMatchObject({ + operation: "insert_beside", + shouldFillSlot: false, + plan: { + operation: "insert_beside", + anchorNodeId: "image-a", + placementStrategy: "beside_each_source", + sourceNodeIds: ["image-a", "image-b"], + }, + }); + expect(result?.mediaItems.map((item) => item.sourceNodeIds)).toEqual([ + ["image-a"], + ["image-b"], + ]); + }); + + test("does not build direct writeback plans without valid result urls", () => { + expect( + buildCanvasAgentDirectWritebackPlan({ + projectId: plannerContext.projectId || "", + chatId: "chat-1", + assistantMessageId: "assistant-4", + operationLabel: "agent-generate-image", + resultType: "image", + resultUrls: ["", "blob:local", "/relative.png"], + prompt: "生成图片", + mode: "generate", + context: plannerContext, + }) + ).toBeNull(); + }); + test("normalizes model-planned focus to known canvas nodes only", () => { const outcome = normalizeCanvasAgentModelPlannerDecision({ decision: {