refactor: extract canvas agent writeback planning
This commit is contained in:
134
parent
16ebbeb6ac
commit
909f1c506c
@ -78,6 +78,7 @@ import {
|
||||
buildCanvasAgentComparisonViewPlan,
|
||||
hasCanvasAgentComparisonViewIntent,
|
||||
} from "@/lib/canvas/agent-view-intent";
|
||||
import { buildCanvasAgentDirectWritebackPlan } from "@/lib/canvas/agent-direct-writeback";
|
||||
import {
|
||||
getCanvasAgentOperationErrorMessage,
|
||||
normalizeCanvasAgentOperationErrorCode,
|
||||
@ -85,8 +86,6 @@ import {
|
||||
import { applyCanvasOperationPlan } from "@/lib/canvas/agent-mutations";
|
||||
import { isProductionEnvironment } from "@/lib/constants";
|
||||
import type {
|
||||
CanvasMediaItem,
|
||||
CanvasOperationKind,
|
||||
CanvasOperationPlan,
|
||||
CanvasViewOperationPlan,
|
||||
} from "@/types/canvas/agent-protocol";
|
||||
@ -2843,101 +2842,36 @@ export async function POST(request: Request) {
|
||||
return { mediaItemIds: [], handledCanvasMediaItemIds: [] };
|
||||
}
|
||||
|
||||
const resultUrls = params.resultUrls
|
||||
.map((url) => String(url || "").trim())
|
||||
.filter((url) => /^https?:\/\//i.test(url));
|
||||
if (resultUrls.length === 0) {
|
||||
const writebackPlan = buildCanvasAgentDirectWritebackPlan({
|
||||
projectId,
|
||||
chatId: id,
|
||||
assistantMessageId: params.assistantMessageId,
|
||||
operationLabel: params.operationLabel,
|
||||
resultType: params.resultType,
|
||||
resultUrls: params.resultUrls,
|
||||
prompt: params.prompt,
|
||||
mode: params.mode,
|
||||
context: requestCanvasAgentContext,
|
||||
requestTextContent,
|
||||
targetRefs: requestTargetResolution?.targetRefs || [],
|
||||
sourceImageUrls: params.sourceImageUrls,
|
||||
assetIdsByUrl: params.assetIdsByUrl,
|
||||
model: params.model,
|
||||
targetSlotId: params.targetSlotId,
|
||||
});
|
||||
if (!writebackPlan) {
|
||||
return { mediaItemIds: [], handledCanvasMediaItemIds: [] };
|
||||
}
|
||||
|
||||
const preferredTargetSlotId = String(params.targetSlotId || "").trim();
|
||||
const selectedSlot = getSelectedSlots(requestCanvasAgentContext)[0] || null;
|
||||
const targetSlotId = preferredTargetSlotId || selectedSlot?.nodeId || "";
|
||||
const shouldFillSlot = Boolean(targetSlotId);
|
||||
const selectedMediaNodes = getSelectedMediaNodes(requestCanvasAgentContext);
|
||||
const anchorNode =
|
||||
selectedSlot || selectedMediaNodes[0] || getBestAnchorNode(requestCanvasAgentContext);
|
||||
const targetRefs = requestTargetResolution?.targetRefs || [];
|
||||
const targetNodeIds = Array.from(
|
||||
new Set(
|
||||
targetRefs
|
||||
.map((ref) => ref.nodeId)
|
||||
.filter((value): value is string => Boolean(value))
|
||||
)
|
||||
);
|
||||
const selectedMediaNodeIds = selectedMediaNodes
|
||||
.map((node) => node.nodeId)
|
||||
.filter((value): value is string => Boolean(value));
|
||||
const perResultSourceNodeIds =
|
||||
!shouldFillSlot && resultUrls.length > 1
|
||||
? [targetNodeIds, selectedMediaNodeIds].find(
|
||||
(nodeIds) => nodeIds.length === resultUrls.length
|
||||
) || []
|
||||
: [];
|
||||
const sourceNodeIds = Array.from(
|
||||
new Set(
|
||||
[
|
||||
...targetNodeIds,
|
||||
...selectedMediaNodeIds,
|
||||
anchorNode?.nodeId,
|
||||
].filter((value): value is string => Boolean(value))
|
||||
)
|
||||
);
|
||||
const operation: CanvasOperationKind = selectedSlot
|
||||
? "fill_slot"
|
||||
: shouldFillSlot
|
||||
? "fill_slot"
|
||||
: params.mode === "edit"
|
||||
? "append_revision"
|
||||
: "insert_beside";
|
||||
const mediaItems: CanvasMediaItem[] = resultUrls.map((url, index) => {
|
||||
const mediaItemId = `${params.assistantMessageId}:${params.operationLabel}:${index + 1}`;
|
||||
return {
|
||||
mediaItemId,
|
||||
type: params.resultType,
|
||||
url,
|
||||
assetId: params.assetIdsByUrl?.[url],
|
||||
title: params.resultType === "video" ? "Agent 视频结果" : "Agent 图片结果",
|
||||
prompt: params.prompt,
|
||||
sourceNodeIds: perResultSourceNodeIds[index]
|
||||
? [perResultSourceNodeIds[index]]
|
||||
: sourceNodeIds,
|
||||
metadata: {
|
||||
messageId: params.assistantMessageId,
|
||||
chatId: id,
|
||||
model: params.model || undefined,
|
||||
sourceImageUrls: params.sourceImageUrls || [],
|
||||
},
|
||||
};
|
||||
});
|
||||
const operationId = `${params.assistantMessageId}:${params.operationLabel}`;
|
||||
const plan: CanvasOperationPlan = {
|
||||
version: 1,
|
||||
operationId,
|
||||
operation,
|
||||
projectId,
|
||||
baseCanvasRevision: requestCanvasAgentContext.canvasRevision,
|
||||
baseCanvasUpdatedAt: requestCanvasAgentContext.canvasUpdatedAt,
|
||||
sourceNodeIds,
|
||||
targetSlotId: targetSlotId || undefined,
|
||||
anchorNodeId: shouldFillSlot ? undefined : anchorNode?.nodeId,
|
||||
placement: "right",
|
||||
placementStrategy:
|
||||
perResultSourceNodeIds.length === mediaItems.length
|
||||
? "beside_each_source"
|
||||
: mediaItems.length > 1
|
||||
? "grid"
|
||||
: "single",
|
||||
preserveOriginal: true,
|
||||
const {
|
||||
mediaItemIds,
|
||||
mediaItems,
|
||||
metadata: {
|
||||
messageId: params.assistantMessageId,
|
||||
chatId: id,
|
||||
mode: params.mode,
|
||||
prompt: params.prompt || requestTextContent,
|
||||
},
|
||||
};
|
||||
const mediaItemIds = mediaItems.map((item) => item.mediaItemId);
|
||||
operation,
|
||||
operationId,
|
||||
plan,
|
||||
selectedMediaNodes,
|
||||
shouldFillSlot,
|
||||
} = writebackPlan;
|
||||
if (!canApplyCanvasAgentMutation) {
|
||||
if (shouldStreamCanvasAgentShadowOperations(canvasAgentFeatures)) {
|
||||
writeCanvasOperationShadowChunk({
|
||||
|
||||
159
lib/canvas/agent-direct-writeback.ts
Normal file
159
lib/canvas/agent-direct-writeback.ts
Normal file
@ -0,0 +1,159 @@
|
||||
import {
|
||||
getBestAnchorNode,
|
||||
getSelectedMediaNodes,
|
||||
getSelectedSlots,
|
||||
} from "@/lib/canvas/agent-context";
|
||||
import type {
|
||||
CanvasAgentContext,
|
||||
CanvasMediaItem,
|
||||
CanvasOperationKind,
|
||||
CanvasOperationPlan,
|
||||
} from "@/types/canvas/agent-protocol";
|
||||
|
||||
export type CanvasAgentDirectWritebackMode = "generate" | "edit" | "video";
|
||||
export type CanvasAgentDirectWritebackResultType = "image" | "video";
|
||||
|
||||
export interface CanvasAgentDirectWritebackTargetRef {
|
||||
nodeId?: string | null;
|
||||
}
|
||||
|
||||
export interface BuildCanvasAgentDirectWritebackPlanParams {
|
||||
projectId: string;
|
||||
chatId: string;
|
||||
assistantMessageId: string;
|
||||
operationLabel: string;
|
||||
resultType: CanvasAgentDirectWritebackResultType;
|
||||
resultUrls: string[];
|
||||
prompt: string;
|
||||
mode: CanvasAgentDirectWritebackMode;
|
||||
context: CanvasAgentContext;
|
||||
requestTextContent?: string | null;
|
||||
targetRefs?: CanvasAgentDirectWritebackTargetRef[];
|
||||
sourceImageUrls?: string[];
|
||||
assetIdsByUrl?: Record<string, string | undefined>;
|
||||
model?: string | null;
|
||||
targetSlotId?: string | null;
|
||||
}
|
||||
|
||||
export interface CanvasAgentDirectWritebackPlanResult {
|
||||
plan: CanvasOperationPlan;
|
||||
mediaItems: CanvasMediaItem[];
|
||||
mediaItemIds: string[];
|
||||
operationId: string;
|
||||
operation: CanvasOperationKind;
|
||||
targetSlotId: string;
|
||||
shouldFillSlot: boolean;
|
||||
selectedMediaNodes: ReturnType<typeof getSelectedMediaNodes>;
|
||||
}
|
||||
|
||||
export function buildCanvasAgentDirectWritebackPlan(
|
||||
params: BuildCanvasAgentDirectWritebackPlanParams
|
||||
): CanvasAgentDirectWritebackPlanResult | null {
|
||||
const projectId = String(params.projectId || "").trim();
|
||||
if (!projectId) return null;
|
||||
|
||||
const resultUrls = params.resultUrls
|
||||
.map((url) => String(url || "").trim())
|
||||
.filter((url) => /^https?:\/\//i.test(url));
|
||||
if (resultUrls.length === 0) return null;
|
||||
|
||||
const preferredTargetSlotId = String(params.targetSlotId || "").trim();
|
||||
const selectedSlot = getSelectedSlots(params.context)[0] || null;
|
||||
const targetSlotId = preferredTargetSlotId || selectedSlot?.nodeId || "";
|
||||
const shouldFillSlot = Boolean(targetSlotId);
|
||||
const selectedMediaNodes = getSelectedMediaNodes(params.context);
|
||||
const anchorNode =
|
||||
selectedSlot || selectedMediaNodes[0] || getBestAnchorNode(params.context);
|
||||
const targetRefs = params.targetRefs || [];
|
||||
const targetNodeIds = Array.from(
|
||||
new Set(
|
||||
targetRefs
|
||||
.map((ref) => ref.nodeId)
|
||||
.filter((value): value is string => Boolean(value))
|
||||
)
|
||||
);
|
||||
const selectedMediaNodeIds = selectedMediaNodes
|
||||
.map((node) => node.nodeId)
|
||||
.filter((value): value is string => Boolean(value));
|
||||
const perResultSourceNodeIds =
|
||||
!shouldFillSlot && resultUrls.length > 1
|
||||
? [targetNodeIds, selectedMediaNodeIds].find(
|
||||
(nodeIds) => nodeIds.length === resultUrls.length
|
||||
) || []
|
||||
: [];
|
||||
const sourceNodeIds = Array.from(
|
||||
new Set(
|
||||
[
|
||||
...targetNodeIds,
|
||||
...selectedMediaNodeIds,
|
||||
anchorNode?.nodeId,
|
||||
].filter((value): value is string => Boolean(value))
|
||||
)
|
||||
);
|
||||
const operation: CanvasOperationKind = selectedSlot
|
||||
? "fill_slot"
|
||||
: shouldFillSlot
|
||||
? "fill_slot"
|
||||
: params.mode === "edit"
|
||||
? "append_revision"
|
||||
: "insert_beside";
|
||||
const mediaItems: CanvasMediaItem[] = resultUrls.map((url, index) => {
|
||||
const mediaItemId = `${params.assistantMessageId}:${params.operationLabel}:${index + 1}`;
|
||||
return {
|
||||
mediaItemId,
|
||||
type: params.resultType,
|
||||
url,
|
||||
assetId: params.assetIdsByUrl?.[url],
|
||||
title:
|
||||
params.resultType === "video" ? "Agent 视频结果" : "Agent 图片结果",
|
||||
prompt: params.prompt,
|
||||
sourceNodeIds: perResultSourceNodeIds[index]
|
||||
? [perResultSourceNodeIds[index]]
|
||||
: sourceNodeIds,
|
||||
metadata: {
|
||||
messageId: params.assistantMessageId,
|
||||
chatId: params.chatId,
|
||||
model: params.model || undefined,
|
||||
sourceImageUrls: params.sourceImageUrls || [],
|
||||
},
|
||||
};
|
||||
});
|
||||
const operationId = `${params.assistantMessageId}:${params.operationLabel}`;
|
||||
const plan: CanvasOperationPlan = {
|
||||
version: 1,
|
||||
operationId,
|
||||
operation,
|
||||
projectId,
|
||||
baseCanvasRevision: params.context.canvasRevision,
|
||||
baseCanvasUpdatedAt: params.context.canvasUpdatedAt,
|
||||
sourceNodeIds,
|
||||
targetSlotId: targetSlotId || undefined,
|
||||
anchorNodeId: shouldFillSlot ? undefined : anchorNode?.nodeId,
|
||||
placement: "right",
|
||||
placementStrategy:
|
||||
perResultSourceNodeIds.length === mediaItems.length
|
||||
? "beside_each_source"
|
||||
: mediaItems.length > 1
|
||||
? "grid"
|
||||
: "single",
|
||||
preserveOriginal: true,
|
||||
mediaItems,
|
||||
metadata: {
|
||||
messageId: params.assistantMessageId,
|
||||
chatId: params.chatId,
|
||||
mode: params.mode,
|
||||
prompt: params.prompt || params.requestTextContent || "",
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
plan,
|
||||
mediaItems,
|
||||
mediaItemIds: mediaItems.map((item) => item.mediaItemId),
|
||||
operationId,
|
||||
operation,
|
||||
targetSlotId,
|
||||
shouldFillSlot,
|
||||
selectedMediaNodes,
|
||||
};
|
||||
}
|
||||
@ -55,6 +55,7 @@ import {
|
||||
resolveCanvasAgentSlotCreationIntent,
|
||||
shouldConsiderCanvasAgentModelPlanner,
|
||||
} from "@/lib/canvas/agent-model-planner";
|
||||
import { buildCanvasAgentDirectWritebackPlan } from "@/lib/canvas/agent-direct-writeback";
|
||||
import { resolveCanvasAgentMediaPlacements } from "@/lib/canvas/agent-placement";
|
||||
import { buildCanvasAgentResultViewOperation } from "@/lib/canvas/agent-result-view";
|
||||
import {
|
||||
@ -371,6 +372,199 @@ test.describe("canvas chat pure helpers", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("plans direct writeback as a slot fill when an agent slot is selected", () => {
|
||||
const context: CanvasAgentContext = {
|
||||
...plannerContext,
|
||||
selectedNodeIds: ["slot-1"],
|
||||
selectedNodes: [
|
||||
{
|
||||
nodeId: "slot-1",
|
||||
type: "agent-slot",
|
||||
title: "主图槽位",
|
||||
x: 100,
|
||||
y: 120,
|
||||
width: 320,
|
||||
height: 240,
|
||||
isSelected: true,
|
||||
},
|
||||
],
|
||||
slots: [
|
||||
{
|
||||
nodeId: "slot-1",
|
||||
slotType: "image",
|
||||
label: "主图槽位",
|
||||
x: 100,
|
||||
y: 120,
|
||||
width: 320,
|
||||
height: 240,
|
||||
aspectRatio: "4:3",
|
||||
isSelected: true,
|
||||
fillPolicy: "replace-empty",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = buildCanvasAgentDirectWritebackPlan({
|
||||
projectId: plannerContext.projectId || "",
|
||||
chatId: "chat-1",
|
||||
assistantMessageId: "assistant-1",
|
||||
operationLabel: "agent-generate-image",
|
||||
resultType: "image",
|
||||
resultUrls: ["https://cdn.example.com/slot.png"],
|
||||
prompt: "生成主图",
|
||||
mode: "generate",
|
||||
context,
|
||||
model: "image-model",
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
operationId: "assistant-1:agent-generate-image",
|
||||
operation: "fill_slot",
|
||||
targetSlotId: "slot-1",
|
||||
shouldFillSlot: true,
|
||||
mediaItemIds: ["assistant-1:agent-generate-image:1"],
|
||||
plan: {
|
||||
operation: "fill_slot",
|
||||
targetSlotId: "slot-1",
|
||||
anchorNodeId: undefined,
|
||||
placementStrategy: "single",
|
||||
sourceNodeIds: ["slot-1"],
|
||||
},
|
||||
});
|
||||
expect(result?.mediaItems[0]).toMatchObject({
|
||||
mediaItemId: "assistant-1:agent-generate-image:1",
|
||||
type: "image",
|
||||
url: "https://cdn.example.com/slot.png",
|
||||
prompt: "生成主图",
|
||||
sourceNodeIds: ["slot-1"],
|
||||
metadata: {
|
||||
messageId: "assistant-1",
|
||||
chatId: "chat-1",
|
||||
model: "image-model",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("plans direct image edits as append revisions anchored to selected media", () => {
|
||||
const result = buildCanvasAgentDirectWritebackPlan({
|
||||
projectId: plannerContext.projectId || "",
|
||||
chatId: "chat-1",
|
||||
assistantMessageId: "assistant-2",
|
||||
operationLabel: "agent-edit-image",
|
||||
resultType: "image",
|
||||
resultUrls: [
|
||||
"relative.png",
|
||||
"https://cdn.example.com/revision.png",
|
||||
],
|
||||
prompt: "",
|
||||
requestTextContent: "把选中图片改成黄昏光线",
|
||||
mode: "edit",
|
||||
context: plannerContext,
|
||||
targetRefs: [{ nodeId: "image-1" }],
|
||||
sourceImageUrls: ["https://cdn.example.com/source.png"],
|
||||
assetIdsByUrl: {
|
||||
"https://cdn.example.com/revision.png": "asset-1",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
operation: "append_revision",
|
||||
shouldFillSlot: false,
|
||||
mediaItemIds: ["assistant-2:agent-edit-image:1"],
|
||||
plan: {
|
||||
operation: "append_revision",
|
||||
anchorNodeId: "image-1",
|
||||
placementStrategy: "single",
|
||||
metadata: {
|
||||
prompt: "把选中图片改成黄昏光线",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(result?.mediaItems[0]).toMatchObject({
|
||||
assetId: "asset-1",
|
||||
sourceNodeIds: ["image-1"],
|
||||
metadata: {
|
||||
sourceImageUrls: ["https://cdn.example.com/source.png"],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("plans direct batch generations beside each matched source", () => {
|
||||
const context: CanvasAgentContext = {
|
||||
...plannerContext,
|
||||
selectedNodeIds: ["image-a", "image-b"],
|
||||
selectedNodes: [
|
||||
{
|
||||
nodeId: "image-a",
|
||||
type: "image",
|
||||
title: "A",
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 200,
|
||||
height: 160,
|
||||
isSelected: true,
|
||||
},
|
||||
{
|
||||
nodeId: "image-b",
|
||||
type: "image",
|
||||
title: "B",
|
||||
x: 260,
|
||||
y: 0,
|
||||
width: 200,
|
||||
height: 160,
|
||||
isSelected: true,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = buildCanvasAgentDirectWritebackPlan({
|
||||
projectId: plannerContext.projectId || "",
|
||||
chatId: "chat-1",
|
||||
assistantMessageId: "assistant-3",
|
||||
operationLabel: "agent-generate-image",
|
||||
resultType: "image",
|
||||
resultUrls: [
|
||||
"https://cdn.example.com/a.png",
|
||||
"https://cdn.example.com/b.png",
|
||||
],
|
||||
prompt: "分别生成两个变体",
|
||||
mode: "generate",
|
||||
context,
|
||||
targetRefs: [{ nodeId: "image-a" }, { nodeId: "image-b" }],
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
operation: "insert_beside",
|
||||
shouldFillSlot: false,
|
||||
plan: {
|
||||
operation: "insert_beside",
|
||||
anchorNodeId: "image-a",
|
||||
placementStrategy: "beside_each_source",
|
||||
sourceNodeIds: ["image-a", "image-b"],
|
||||
},
|
||||
});
|
||||
expect(result?.mediaItems.map((item) => item.sourceNodeIds)).toEqual([
|
||||
["image-a"],
|
||||
["image-b"],
|
||||
]);
|
||||
});
|
||||
|
||||
test("does not build direct writeback plans without valid result urls", () => {
|
||||
expect(
|
||||
buildCanvasAgentDirectWritebackPlan({
|
||||
projectId: plannerContext.projectId || "",
|
||||
chatId: "chat-1",
|
||||
assistantMessageId: "assistant-4",
|
||||
operationLabel: "agent-generate-image",
|
||||
resultType: "image",
|
||||
resultUrls: ["", "blob:local", "/relative.png"],
|
||||
prompt: "生成图片",
|
||||
mode: "generate",
|
||||
context: plannerContext,
|
||||
})
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
test("normalizes model-planned focus to known canvas nodes only", () => {
|
||||
const outcome = normalizeCanvasAgentModelPlannerDecision({
|
||||
decision: {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user