From df0daf4d27fe0cbb72dc67c5d57095cd0292fb74 Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 22 Jun 2026 21:45:23 +0800 Subject: [PATCH] feat: add canvas agent operation protocol --- app/(chat)/api/chat/_lib/stream.ts | 35 +- app/(chat)/api/chat/route.ts | 788 +++++++++- app/api/canvas/[id]/agent-operations/route.ts | 81 +- .../[id]/CanvasEditorLegacyPageClient.tsx | 125 +- components/canvas/CanvasChatPanel.tsx | 170 +- components/canvas/CanvasWorkspace.tsx | 243 ++- .../canvas/chat-panel/canvas-agent-stream.ts | 33 + components/canvas/nodes/AgentSlotNode.tsx | 283 +++- components/canvas/nodes/ImageNode.tsx | 292 +++- components/image-annotator.tsx | 34 +- lib/canvas/agent-annotation-metadata.ts | 491 ++++++ lib/canvas/agent-annotation-prompt.ts | 63 + lib/canvas/agent-feature-flags.ts | 82 + lib/canvas/agent-model-planner.ts | 287 +++- lib/canvas/agent-mutations.ts | 674 +++++++- lib/canvas/agent-operation-audit.ts | 118 ++ lib/canvas/agent-operation-errors.ts | 45 + lib/canvas/agent-risk.ts | 22 + lib/canvas/agent-view-operations.ts | 122 ++ store/canvas/index.ts | 23 + tests/routes/canvas-chat-helpers.test.ts | 1385 +++++++++++++++++ types/canvas/agent-protocol.ts | 33 +- 22 files changed, 5195 insertions(+), 234 deletions(-) create mode 100644 lib/canvas/agent-annotation-metadata.ts create mode 100644 lib/canvas/agent-annotation-prompt.ts create mode 100644 lib/canvas/agent-feature-flags.ts create mode 100644 lib/canvas/agent-operation-audit.ts create mode 100644 lib/canvas/agent-operation-errors.ts create mode 100644 lib/canvas/agent-risk.ts create mode 100644 lib/canvas/agent-view-operations.ts diff --git a/app/(chat)/api/chat/_lib/stream.ts b/app/(chat)/api/chat/_lib/stream.ts index 33c390494..c2c431d79 100644 --- a/app/(chat)/api/chat/_lib/stream.ts +++ b/app/(chat)/api/chat/_lib/stream.ts @@ -67,6 +67,12 @@ export function writeStructuredDataChunk( ); } +function normalizeStreamIdList(value: unknown) { + return Array.isArray(value) + ? value.map((id) => String(id || "").trim()).filter(Boolean) + : []; +} + export function writeProtocolDebugChunk( writer: StreamWriter, enabled: boolean, @@ -245,16 +251,10 @@ export function writeCanvasMediaChunk( agentCategoryName: String(params.agentCategoryName || "").trim() || undefined, agentExecutionSnapshotId: String(params.agentExecutionSnapshotId || "").trim() || undefined, - mediaItemIds: Array.isArray(params.mediaItemIds) - ? params.mediaItemIds - .map((id) => String(id || "").trim()) - .filter(Boolean) - : [], - handledCanvasMediaItemIds: Array.isArray(params.handledCanvasMediaItemIds) - ? params.handledCanvasMediaItemIds - .map((id) => String(id || "").trim()) - .filter(Boolean) - : [], + mediaItemIds: normalizeStreamIdList(params.mediaItemIds), + handledCanvasMediaItemIds: normalizeStreamIdList( + params.handledCanvasMediaItemIds + ), }, label ); @@ -266,6 +266,7 @@ export function writeCanvasOperationChunk( projectId: string; operationId: string; operation: string; + status?: "applied" | "failed" | "shadowed"; createdNodeIds: string[]; createdNodeIdsByMediaItemId?: Record; updatedNodeIds?: string[]; @@ -273,6 +274,10 @@ export function writeCanvasOperationChunk( canvasRevision?: string; anchorNodeId?: string | null; handledCanvasMediaItemIds?: string[]; + mediaItemIds?: string[]; + errorCode?: string; + errorMessage?: string; + fallbackToCanvasMedia?: boolean; message?: string; }, label: string @@ -284,13 +289,21 @@ export function writeCanvasOperationChunk( projectId: params.projectId, operationId: params.operationId, operation: params.operation, + status: params.status || "applied", createdNodeIds: params.createdNodeIds || [], createdNodeIdsByMediaItemId: params.createdNodeIdsByMediaItemId || {}, updatedNodeIds: params.updatedNodeIds || [], createdEdgeIds: params.createdEdgeIds || [], canvasRevision: params.canvasRevision, anchorNodeId: params.anchorNodeId || undefined, - handledCanvasMediaItemIds: params.handledCanvasMediaItemIds || [], + handledCanvasMediaItemIds: + params.status && params.status !== "applied" + ? [] + : normalizeStreamIdList(params.handledCanvasMediaItemIds), + mediaItemIds: normalizeStreamIdList(params.mediaItemIds), + errorCode: params.errorCode, + errorMessage: params.errorMessage, + fallbackToCanvasMedia: params.fallbackToCanvasMedia, message: params.message, }, label diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index e134f7d99..f907756e8 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -45,12 +45,21 @@ import { import { buildAgentSceneImageRolePrompt } from "@/lib/agents/scene-image-role-prompt"; import { optimizeAgentSceneWorkflowPrompt } from "@/lib/agents/scene-workflow-prompt-optimizer"; import { getEnabledTools } from "@/lib/ai/tools-registry"; +import { + buildCanvasAgentAnnotationEditPrompt, + hasCanvasAgentAnnotationEditIntent, +} from "@/lib/canvas/agent-annotation-prompt"; import { getBestAnchorNode, getSelectedMediaNodes, getSelectedSlots, normalizeCanvasAgentContext, } from "@/lib/canvas/agent-context"; +import { + canApplyCanvasAgentMutations, + getCanvasAgentFeatureFlags, + shouldStreamCanvasAgentShadowOperations, +} from "@/lib/canvas/agent-feature-flags"; import { buildPendingCanvasOperation, extractPendingCanvasOperationFromWorkingState, @@ -59,9 +68,15 @@ import { withConfirmedCanvasOperationPlan, } from "@/lib/canvas/agent-confirmation"; import { + buildCanvasAgentSlotCreationPlan, planCanvasAgentOperationWithModel, + resolveCanvasAgentSlotCreationIntent, type CanvasAgentModelPlannerOutcome, } from "@/lib/canvas/agent-model-planner"; +import { + getCanvasAgentOperationErrorMessage, + normalizeCanvasAgentOperationErrorCode, +} from "@/lib/canvas/agent-operation-errors"; import { applyCanvasOperationPlan } from "@/lib/canvas/agent-mutations"; import { isProductionEnvironment } from "@/lib/constants"; import type { @@ -429,6 +444,82 @@ const AGENT_REPLACEMENT_PARALLEL_WORKER_COUNT = Math.max( Math.min(8, Number(process.env.AGENT_REPLACEMENT_PARALLEL_WORKER_COUNT || 6) || 6), ); +function writeCanvasOperationFailureChunk(params: { + writer: StreamWriter; + label: string; + projectId: string; + operationId: string; + operation: string; + error: unknown; + anchorNodeId?: string | null; + mediaItemIds?: string[]; + fallbackToCanvasMedia?: boolean; +}) { + const errorCode = normalizeCanvasAgentOperationErrorCode( + (params.error as { code?: unknown } | null)?.code + ); + const errorMessage = getCanvasAgentOperationErrorMessage({ + code: errorCode, + fallbackMessage: params.error instanceof Error ? params.error.message : undefined, + }); + + writeCanvasOperationChunk( + params.writer, + { + projectId: params.projectId, + operationId: params.operationId, + operation: params.operation, + status: "failed", + createdNodeIds: [], + updatedNodeIds: [], + createdEdgeIds: [], + anchorNodeId: params.anchorNodeId || null, + handledCanvasMediaItemIds: [], + mediaItemIds: params.mediaItemIds || [], + errorCode, + errorMessage, + fallbackToCanvasMedia: params.fallbackToCanvasMedia, + message: errorMessage, + }, + params.label + ); + + return { + errorCode, + errorMessage, + }; +} + +function writeCanvasOperationShadowChunk(params: { + writer: StreamWriter; + label: string; + projectId: string; + operationId: string; + operation: string; + anchorNodeId?: string | null; + mediaItemIds?: string[]; + message?: string; +}) { + writeCanvasOperationChunk( + params.writer, + { + projectId: params.projectId, + operationId: params.operationId, + operation: params.operation, + status: "shadowed", + createdNodeIds: [], + updatedNodeIds: [], + createdEdgeIds: [], + anchorNodeId: params.anchorNodeId || null, + handledCanvasMediaItemIds: [], + mediaItemIds: params.mediaItemIds || [], + fallbackToCanvasMedia: true, + message: params.message || "已记录画布操作计划（灰度模式，未修改画布）", + }, + params.label + ); +} + async function replayAssistantMessagesForDuplicateRequest({ chatId, userMessageId, @@ -782,6 +873,9 @@ export async function POST(request: Request) { const requestCanvasAgentContext = normalizeCanvasAgentContext( contextPayload || undefined ); + const canvasAgentFeatures = getCanvasAgentFeatureFlags(); + const canApplyCanvasAgentMutation = + canApplyCanvasAgentMutations(canvasAgentFeatures); if (requestCanvasAgentContext) { chatApiLogger.info("[CanvasAgentContext] Normalized request context:", { source: requestCanvasAgentContext.source, @@ -791,6 +885,7 @@ export async function POST(request: Request) { slotCount: requestCanvasAgentContext.slots.length, annotationCount: requestCanvasAgentContext.annotations.length, focusPolicy: requestCanvasAgentContext.focusPolicy, + writebackMode: canvasAgentFeatures.writebackMode, }); } @@ -2513,9 +2608,22 @@ export async function POST(request: Request) { const uiMessages = [...convertToUIMessages(messagesFromDb), message]; const hiddenPromptText = typeof hiddenPrompt === "string" ? hiddenPrompt.trim() : ""; - const hiddenPromptPrefix = hiddenPromptText ? `${hiddenPromptText}\n\n` : ""; - const uiMessagesForModel = hiddenPromptText - ? applyHiddenPromptToMessages(uiMessages, hiddenPromptText) + const canvasAnnotationHiddenPrompt = canvasAgentFeatures.annotationEditEnabled + ? buildCanvasAgentAnnotationEditPrompt({ + context: requestCanvasAgentContext, + targetNodeIds: (requestTargetResolution?.targetRefs || []) + .map((ref) => ref.nodeId) + .filter((nodeId): nodeId is string => Boolean(nodeId)), + userText: requestTextContent, + }) + : ""; + const hiddenPromptPrefixText = [ + hiddenPromptText, + canvasAnnotationHiddenPrompt, + ].filter(Boolean).join("\n\n"); + const hiddenPromptPrefix = hiddenPromptPrefixText ? `${hiddenPromptPrefixText}\n\n` : ""; + const uiMessagesForModel = hiddenPromptPrefixText + ? applyHiddenPromptToMessages(uiMessages, hiddenPromptPrefixText) : uiMessages; if (hiddenPromptText) { @@ -2528,6 +2636,15 @@ export async function POST(request: Request) { preview: hiddenPromptLines.slice(0, 12), }); } + if (canvasAnnotationHiddenPrompt) { + chatApiLogger.info("[Chat API] Canvas annotation edit prompt enabled:", { + targetNodeCount: (requestTargetResolution?.targetRefs || []).filter((ref) => + Boolean(ref.nodeId) + ).length, + hasAnnotationIntent: hasCanvasAgentAnnotationEditIntent(requestTextContent), + preview: canvasAnnotationHiddenPrompt.split("\n").slice(0, 6), + }); + } const { longitude, latitude, city, country } = geolocation(request); @@ -2702,6 +2819,7 @@ export async function POST(request: Request) { sourceImageUrls?: string[]; assetIdsByUrl?: Record; model?: string | null; + targetSlotId?: string | null; }): Promise<{ mediaItemIds: string[]; handledCanvasMediaItemIds: string[]; @@ -2722,7 +2840,10 @@ export async function POST(request: Request) { return { mediaItemIds: [], handledCanvasMediaItemIds: [] }; } + const preferredTargetSlotId = String(params.targetSlotId || "").trim(); const selectedSlot = getSelectedSlots(requestCanvasAgentContext)[0] || null; + const targetSlotId = preferredTargetSlotId || selectedSlot?.nodeId || ""; + const shouldFillSlot = Boolean(targetSlotId); const selectedMediaNodes = getSelectedMediaNodes(requestCanvasAgentContext); const anchorNode = selectedSlot || selectedMediaNodes[0] || getBestAnchorNode(requestCanvasAgentContext); @@ -2738,7 +2859,7 @@ export async function POST(request: Request) { .map((node) => node.nodeId) .filter((value): value is string => Boolean(value)); const perResultSourceNodeIds = - !selectedSlot && resultUrls.length > 1 + !shouldFillSlot && resultUrls.length > 1 ? [targetNodeIds, selectedMediaNodeIds].find( (nodeIds) => nodeIds.length === resultUrls.length ) || [] @@ -2754,7 +2875,9 @@ export async function POST(request: Request) { ); const operation: CanvasOperationKind = selectedSlot ? "fill_slot" - : params.mode === "edit" + : shouldFillSlot + ? "fill_slot" + : params.mode === "edit" ? "append_revision" : "insert_beside"; const mediaItems: CanvasMediaItem[] = resultUrls.map((url, index) => { @@ -2786,8 +2909,8 @@ export async function POST(request: Request) { baseCanvasRevision: requestCanvasAgentContext.canvasRevision, baseCanvasUpdatedAt: requestCanvasAgentContext.canvasUpdatedAt, sourceNodeIds, - targetSlotId: selectedSlot?.nodeId, - anchorNodeId: selectedSlot ? undefined : anchorNode?.nodeId, + targetSlotId: targetSlotId || undefined, + anchorNodeId: shouldFillSlot ? undefined : anchorNode?.nodeId, placement: "right", placementStrategy: perResultSourceNodeIds.length === mediaItems.length @@ -2804,6 +2927,109 @@ export async function POST(request: Request) { prompt: params.prompt || requestTextContent, }, }; + const mediaItemIds = mediaItems.map((item) => item.mediaItemId); + if (!canApplyCanvasAgentMutation) { + if (shouldStreamCanvasAgentShadowOperations(canvasAgentFeatures)) { + writeCanvasOperationShadowChunk({ + writer: params.writer, + label: `${params.operationLabel}:canvas-operation-shadow`, + projectId, + operationId, + operation, + anchorNodeId: plan.anchorNodeId || plan.targetSlotId || null, + mediaItemIds, + }); + } + chatApiLogger.info("[CanvasAgentOperation] Skipped direct canvas mutation:", { + operationId, + operation, + writebackMode: canvasAgentFeatures.writebackMode, + operationProtocolEnabled: canvasAgentFeatures.operationProtocolEnabled, + mediaItemCount: mediaItems.length, + }); + return { + mediaItemIds, + handledCanvasMediaItemIds: [], + }; + } + + const wantsReplaceExistingMedia = + params.mode === "edit" && + params.resultType === "image" && + /覆盖|替换原图|替换当前|直接替换|替换选中|替换这张|覆盖原图/.test( + requestTextContent + ); + const replaceTargetNodeIds = selectedMediaNodes + .filter((node) => node.type === params.resultType) + .map((node) => node.nodeId) + .filter(Boolean) + .slice(0, mediaItems.length); + if ( + wantsReplaceExistingMedia && + replaceTargetNodeIds.length === mediaItems.length && + replaceTargetNodeIds.length > 0 + ) { + const replaceOperationId = `${operationId}:replace`; + const replacePlan: CanvasOperationPlan = { + ...plan, + operationId: replaceOperationId, + operation: "replace_node_media", + sourceNodeIds: replaceTargetNodeIds, + targetNodeIds: replaceTargetNodeIds, + anchorNodeId: undefined, + targetSlotId: undefined, + confirmed: false, + mediaItems: mediaItems.map((item, index) => ({ + ...item, + sourceNodeIds: [replaceTargetNodeIds[index]], + })), + metadata: { + ...(plan.metadata || {}), + pendingConfirmation: true, + riskOperation: "replace_node_media", + }, + }; + const pendingReply = + "新结果已生成。替换原图会覆盖现有节点，请回复“确认执行”后我再替换。"; + await persistPendingCanvasOperation( + buildPendingCanvasOperation({ + projectId, + messageId: message.id, + userText: requestTextContent, + riskOperation: "replace_node_media", + targetNodeIds: replaceTargetNodeIds, + reply: pendingReply, + operationPlan: replacePlan, + viewPlan: { + version: 1, + viewOperationId: `${replaceOperationId}:view`, + operation: replaceTargetNodeIds.length > 1 ? "focus_nodes" : "focus_selection", + projectId, + nodeIds: replaceTargetNodeIds, + anchorNodeId: replaceTargetNodeIds[0], + padding: 120, + persist: false, + reason: "focus_pending_replace_node_media", + }, + }) + ); + writeAgentProgressChunk( + params.writer, + { + stage: "clarify", + currentStep: "finalize", + title: "替换原图需要确认", + message: pendingReply, + toolName: "canvasOperation", + capability: "replace_node_media", + }, + `${params.operationLabel}:replace-confirmation`, + ); + return { + mediaItemIds, + handledCanvasMediaItemIds: [], + }; + } try { const result = await applyCanvasOperationPlan({ @@ -2825,7 +3051,7 @@ export async function POST(request: Request) { canvasRevision: result.canvasRevision, anchorNodeId: plan.anchorNodeId || plan.targetSlotId || null, handledCanvasMediaItemIds, - message: selectedSlot ? "已填充画布槽位" : "已写回画布", + message: shouldFillSlot ? "已填充画布槽位" : "已写回画布", }, `${params.operationLabel}:canvas-operation`, ); @@ -2834,18 +3060,156 @@ export async function POST(request: Request) { handledCanvasMediaItemIds, }; } catch (error) { + const failure = writeCanvasOperationFailureChunk({ + writer: params.writer, + label: `${params.operationLabel}:canvas-operation-failed`, + projectId, + operationId, + operation, + error, + anchorNodeId: plan.anchorNodeId || plan.targetSlotId || null, + mediaItemIds, + fallbackToCanvasMedia: true, + }); chatApiLogger.warn("[CanvasAgentOperation] Writeback failed, fallback to canvasMedia:", { operationId, + errorCode: failure.errorCode, error, }); return { - mediaItemIds: mediaItems.map((item) => item.mediaItemId), + mediaItemIds, handledCanvasMediaItemIds: [], }; } }; + const createDirectAgentTargetSlotIfNeeded = async (params: { + writer: StreamWriter; + assistantMessageId: string; + operationLabel: string; + resultType: "image" | "video"; + }) => { + const projectId = contextPayload?.projectId; + if ( + contextPayload?.source !== "canvas" || + !projectId || + !requestCanvasAgentContext || + !canvasAgentFeatures.slotEnabled + ) { + return null; + } + + if (getSelectedSlots(requestCanvasAgentContext).length > 0) { + return null; + } + + const slotIntent = resolveCanvasAgentSlotCreationIntent({ + userText: requestTextContent, + }); + if (!slotIntent.shouldCreate || !slotIntent.combinedWithMediaExecution) { + return null; + } + + const operationId = `${params.assistantMessageId}:${params.operationLabel}:create-slot`; + const plan = buildCanvasAgentSlotCreationPlan({ + context: requestCanvasAgentContext, + projectId, + messageId: params.assistantMessageId, + chatId: id, + userText: requestTextContent, + operationId, + slotType: slotIntent.slotType, + slotLabel: slotIntent.label, + aspectRatio: slotIntent.aspectRatio, + }); + if (!plan) return null; + + if (!canApplyCanvasAgentMutation) { + if (shouldStreamCanvasAgentShadowOperations(canvasAgentFeatures)) { + writeCanvasOperationShadowChunk({ + writer: params.writer, + label: `${params.operationLabel}:create-slot-operation-shadow`, + projectId, + operationId, + operation: plan.operation, + anchorNodeId: plan.anchorNodeId || null, + message: "已记录创建画布槽位计划（灰度模式，未修改画布）", + }); + } + chatApiLogger.info( + "[CanvasAgentOperation] Skipped target slot creation mutation:", + { + operationId, + resultType: params.resultType, + writebackMode: canvasAgentFeatures.writebackMode, + operationProtocolEnabled: canvasAgentFeatures.operationProtocolEnabled, + } + ); + return null; + } + + try { + writeAgentProgressChunk( + params.writer, + { + stage: "executing", + currentStep: "execute", + title: "正在创建画布槽位", + message: "我会先在当前视口创建一个空槽位，再把本次结果填进去。", + toolName: "canvasOperation", + capability: "create_agent_slot", + }, + `${params.operationLabel}:create-slot-progress`, + ); + const result = await applyCanvasOperationPlan({ + userId: session.user.id, + projectId, + plan, + }); + const createdSlotId = result.createdNodeIds[0] || null; + writeCanvasOperationChunk( + params.writer, + { + projectId, + operationId: plan.operationId, + operation: plan.operation, + createdNodeIds: result.createdNodeIds, + createdNodeIdsByMediaItemId: result.createdNodeIdsByMediaItemId, + updatedNodeIds: result.updatedNodeIds, + createdEdgeIds: result.createdEdgeIds, + canvasRevision: result.canvasRevision, + anchorNodeId: createdSlotId, + handledCanvasMediaItemIds: [], + message: "已创建画布槽位", + }, + `${params.operationLabel}:create-slot-operation`, + ); + return createdSlotId; + } catch (error) { + const failure = writeCanvasOperationFailureChunk({ + writer: params.writer, + label: `${params.operationLabel}:create-slot-operation-failed`, + projectId, + operationId, + operation: plan.operation, + error, + fallbackToCanvasMedia: false, + }); + chatApiLogger.warn( + "[CanvasAgentOperation] Failed to create target slot before media writeback:", + { + operationId, + resultType: params.resultType, + errorCode: failure.errorCode, + error, + } + ); + return null; + } + }; + const resolveCanvasViewOperationPlan = (): CanvasViewOperationPlan | null => { + if (!canvasAgentFeatures.viewOperationEnabled) return null; if (composerMode !== "agent") return null; if (contextPayload?.source !== "canvas" || !contextPayload.projectId) return null; if (!requestCanvasAgentContext) return null; @@ -2946,6 +3310,7 @@ export async function POST(request: Request) { }; const resolveCanvasLinkOperationPlan = (): CanvasOperationPlan | null => { + if (!canApplyCanvasAgentMutation) return null; if (composerMode !== "agent") return null; if (contextPayload?.source !== "canvas" || !contextPayload.projectId) return null; if (!requestCanvasAgentContext) return null; @@ -3015,6 +3380,7 @@ export async function POST(request: Request) { }; const resolveCanvasTextOperationPlan = (): CanvasOperationPlan | null => { + if (!canApplyCanvasAgentMutation) return null; if (composerMode !== "agent") return null; if (contextPayload?.source !== "canvas" || !contextPayload.projectId) return null; if (!requestCanvasAgentContext) return null; @@ -3062,6 +3428,12 @@ export async function POST(request: Request) { if (composerMode !== "agent") return null; if (contextPayload?.source !== "canvas" || !contextPayload.projectId) return null; if (!requestCanvasAgentContext) return null; + if ( + !canvasAgentFeatures.viewOperationEnabled && + !canApplyCanvasAgentMutation + ) { + return null; + } const plannerModelId = configuredAgentConversationModelId || @@ -3194,6 +3566,56 @@ export async function POST(request: Request) { return; } + if (!canApplyCanvasAgentMutation) { + const reply = + "已收到确认，但当前 Agent 画布写回处于灰度模式，未修改画布。"; + await persistPendingCanvasOperation(null); + chatApiLogger.info( + "[CanvasAgentOperation] Skipped confirmed pending operation in non-authoritative mode:", + { + writebackMode: canvasAgentFeatures.writebackMode, + operationId: confirmedPlan.operationId, + operation: confirmedPlan.operation, + } + ); + writeAgentProgressChunk( + writer, + { + stage: "completed", + currentStep: "finalize", + title: "画布操作已进入灰度保护", + message: reply, + toolName: "canvasOperation", + capability: confirmedPlan.operation, + }, + "canvas-pending-operation:shadow", + ); + writeAssistantTextChunk( + writer, + reply, + "canvas-pending-operation:shadow-reply", + messageId, + ); + await saveMessages({ + messages: [ + { + id: messageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: reply }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + return; + } + const reply = "已确认，画布操作已执行。"; writeAgentProgressChunk( writer, @@ -3236,7 +3658,10 @@ export async function POST(request: Request) { }, "canvas-pending-operation:operation", ); - if (pendingCanvasOperation.viewPlan) { + if ( + pendingCanvasOperation.viewPlan && + canvasAgentFeatures.viewOperationEnabled + ) { writeCanvasViewOperationChunk( writer, pendingCanvasOperation.viewPlan, @@ -3279,15 +3704,26 @@ export async function POST(request: Request) { ], }); } catch (error) { - const errorMessage = - error instanceof Error ? error.message : "执行已确认的画布操作失败"; + const failure = writeCanvasOperationFailureChunk({ + writer, + label: "canvas-pending-operation:operation-failed", + projectId: confirmedPlan.projectId, + operationId: confirmedPlan.operationId, + operation: confirmedPlan.operation, + error, + anchorNodeId: + confirmedPlan.targetNodeIds?.[0] || + confirmedPlan.sourceNodeIds?.[0] || + confirmedPlan.anchorNodeId || + null, + }); writeAgentProgressChunk( writer, { stage: "failed", currentStep: "finalize", title: "画布操作失败", - message: errorMessage, + message: failure.errorMessage, toolName: "canvasOperation", capability: confirmedPlan.operation, }, @@ -3295,7 +3731,7 @@ export async function POST(request: Request) { ); writeAssistantTextChunk( writer, - errorMessage, + failure.errorMessage, "canvas-pending-operation:error", messageId, ); @@ -3403,20 +3839,22 @@ export async function POST(request: Request) { }, "canvas-link:operation", ); - writeCanvasViewOperationChunk( - writer, - { - version: 1, - viewOperationId: `${canvasLinkOperationPlan.operationId}:view`, - operation: "focus_nodes", - projectId: canvasLinkOperationPlan.projectId, - nodeIds: canvasLinkOperationPlan.sourceNodeIds, - padding: 140, - persist: false, - reason: "focus_linked_nodes", - }, - "canvas-link:view-operation", - ); + if (canvasAgentFeatures.viewOperationEnabled) { + writeCanvasViewOperationChunk( + writer, + { + version: 1, + viewOperationId: `${canvasLinkOperationPlan.operationId}:view`, + operation: "focus_nodes", + projectId: canvasLinkOperationPlan.projectId, + nodeIds: canvasLinkOperationPlan.sourceNodeIds, + padding: 140, + persist: false, + reason: "focus_linked_nodes", + }, + "canvas-link:view-operation", + ); + } writeAgentProgressChunk( writer, { @@ -3449,21 +3887,28 @@ export async function POST(request: Request) { ], }); } catch (error) { - const errorMessage = - error instanceof Error ? error.message : "连接画布节点失败"; + const failure = writeCanvasOperationFailureChunk({ + writer, + label: "canvas-link:operation-failed", + projectId: canvasLinkOperationPlan.projectId, + operationId: canvasLinkOperationPlan.operationId, + operation: canvasLinkOperationPlan.operation, + error, + anchorNodeId: canvasLinkOperationPlan.sourceNodeIds?.[0] || null, + }); writeAgentProgressChunk( writer, { stage: "failed", currentStep: "finalize", title: "画布连线失败", - message: errorMessage, + message: failure.errorMessage, toolName: "canvasOperation", capability: "link_nodes", }, "canvas-link:progress-failed", ); - writeAssistantTextChunk(writer, errorMessage, "canvas-link:error", messageId); + writeAssistantTextChunk(writer, failure.errorMessage, "canvas-link:error", messageId); } }, }); @@ -3512,20 +3957,22 @@ export async function POST(request: Request) { }, "canvas-text:operation", ); - writeCanvasViewOperationChunk( - writer, - { - version: 1, - viewOperationId: `${canvasTextOperationPlan.operationId}:view`, - operation: "focus_nodes", - projectId: canvasTextOperationPlan.projectId, - nodeIds: canvasTextOperationPlan.targetNodeIds, - padding: 120, - persist: false, - reason: "focus_updated_text_nodes", - }, - "canvas-text:view-operation", - ); + if (canvasAgentFeatures.viewOperationEnabled) { + writeCanvasViewOperationChunk( + writer, + { + version: 1, + viewOperationId: `${canvasTextOperationPlan.operationId}:view`, + operation: "focus_nodes", + projectId: canvasTextOperationPlan.projectId, + nodeIds: canvasTextOperationPlan.targetNodeIds, + padding: 120, + persist: false, + reason: "focus_updated_text_nodes", + }, + "canvas-text:view-operation", + ); + } writeAgentProgressChunk( writer, { @@ -3558,21 +4005,28 @@ export async function POST(request: Request) { ], }); } catch (error) { - const errorMessage = - error instanceof Error ? error.message : "更新画布文本失败"; + const failure = writeCanvasOperationFailureChunk({ + writer, + label: "canvas-text:operation-failed", + projectId: canvasTextOperationPlan.projectId, + operationId: canvasTextOperationPlan.operationId, + operation: canvasTextOperationPlan.operation, + error, + anchorNodeId: canvasTextOperationPlan.targetNodeIds?.[0] || null, + }); writeAgentProgressChunk( writer, { stage: "failed", currentStep: "finalize", title: "画布文本更新失败", - message: errorMessage, + message: failure.errorMessage, toolName: "canvasOperation", capability: "update_text", }, "canvas-text:progress-failed", ); - writeAssistantTextChunk(writer, errorMessage, "canvas-text:error", messageId); + writeAssistantTextChunk(writer, failure.errorMessage, "canvas-text:error", messageId); } }, }); @@ -3587,6 +4041,55 @@ export async function POST(request: Request) { const messageId = generateUUID(); if (canvasModelPlannerOutcome.kind === "confirmation_required") { + if (!canApplyCanvasAgentMutation) { + const shadowReply = + "我已识别到这是高风险画布操作，但当前 Agent 画布写回处于灰度模式，未修改画布。"; + chatApiLogger.info( + "[CanvasAgentOperation] Skipped model-planned confirmation in non-authoritative mode:", + { + writebackMode: canvasAgentFeatures.writebackMode, + riskOperation: canvasModelPlannerOutcome.riskOperation, + targetNodeIds: canvasModelPlannerOutcome.targetNodeIds, + } + ); + writeAgentProgressChunk( + writer, + { + stage: "completed", + currentStep: "finalize", + title: "画布操作已进入灰度保护", + message: shadowReply, + toolName: "canvasOperation", + capability: canvasModelPlannerOutcome.riskOperation, + }, + "canvas-model-planner:confirmation-shadow", + ); + writeAssistantTextChunk( + writer, + shadowReply, + "canvas-model-planner:shadow-reply", + messageId, + ); + await saveMessages({ + messages: [ + { + id: messageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: shadowReply }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + return; + } + const pendingOperation = buildPendingCanvasOperation({ projectId: contextPayload?.projectId || canvasModelPlannerOutcome.operationPlan?.projectId || "", messageId: message.id, @@ -3638,6 +4141,34 @@ export async function POST(request: Request) { } if (canvasModelPlannerOutcome.kind === "view") { + if (!canvasAgentFeatures.viewOperationEnabled) { + const disabledReply = "当前画布视角控制暂未开启。"; + writeAssistantTextChunk( + writer, + disabledReply, + "canvas-model-planner:view-disabled", + messageId, + ); + await saveMessages({ + messages: [ + { + id: messageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: disabledReply }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + return; + } + writeAgentProgressChunk( writer, { @@ -3683,6 +4214,118 @@ export async function POST(request: Request) { } const operationPlan = canvasModelPlannerOutcome.operationPlan; + if ( + operationPlan.operation === "create_agent_slot" && + !canvasAgentFeatures.slotEnabled + ) { + const slotDisabledReply = "当前画布 Agent Slot 能力暂未开启。"; + writeAgentProgressChunk( + writer, + { + stage: "completed", + currentStep: "finalize", + title: "画布槽位未创建", + message: slotDisabledReply, + toolName: "canvasOperation", + capability: operationPlan.operation, + }, + "canvas-model-planner:slot-disabled", + ); + writeAssistantTextChunk( + writer, + slotDisabledReply, + "canvas-model-planner:slot-disabled-reply", + messageId, + ); + await saveMessages({ + messages: [ + { + id: messageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: slotDisabledReply }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + return; + } + + if (!canApplyCanvasAgentMutation) { + const shadowReply = + "我已生成画布操作计划，但当前 Agent 画布写回处于灰度模式，未修改画布。"; + if (shouldStreamCanvasAgentShadowOperations(canvasAgentFeatures)) { + writeCanvasOperationShadowChunk({ + writer, + label: "canvas-model-planner:operation-shadow-event", + projectId: operationPlan.projectId, + operationId: operationPlan.operationId, + operation: operationPlan.operation, + anchorNodeId: + operationPlan.targetNodeIds?.[0] || + operationPlan.sourceNodeIds?.[0] || + operationPlan.anchorNodeId || + operationPlan.targetSlotId || + null, + mediaItemIds: (operationPlan.mediaItems || []) + .map((item) => item.mediaItemId) + .filter(Boolean), + message: shadowReply, + }); + } + chatApiLogger.info( + "[CanvasAgentOperation] Skipped model-planned mutation in non-authoritative mode:", + { + writebackMode: canvasAgentFeatures.writebackMode, + operationId: operationPlan.operationId, + operation: operationPlan.operation, + projectId: operationPlan.projectId, + } + ); + writeAgentProgressChunk( + writer, + { + stage: "completed", + currentStep: "finalize", + title: "画布操作已进入灰度保护", + message: shadowReply, + toolName: "canvasOperation", + capability: operationPlan.operation, + }, + "canvas-model-planner:operation-shadow", + ); + writeAssistantTextChunk( + writer, + shadowReply, + "canvas-model-planner:shadow-reply", + messageId, + ); + await saveMessages({ + messages: [ + { + id: messageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: shadowReply }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + return; + } + writeAgentProgressChunk( writer, { @@ -3724,7 +4367,10 @@ export async function POST(request: Request) { }, "canvas-model-planner:operation", ); - if (canvasModelPlannerOutcome.viewPlan) { + if ( + canvasModelPlannerOutcome.viewPlan && + canvasAgentFeatures.viewOperationEnabled + ) { writeCanvasViewOperationChunk( writer, canvasModelPlannerOutcome.viewPlan, @@ -3768,15 +4414,26 @@ export async function POST(request: Request) { ], }); } catch (error) { - const errorMessage = - error instanceof Error ? error.message : "模型规划的画布操作执行失败"; + const failure = writeCanvasOperationFailureChunk({ + writer, + label: "canvas-model-planner:operation-failed", + projectId: operationPlan.projectId, + operationId: operationPlan.operationId, + operation: operationPlan.operation, + error, + anchorNodeId: + operationPlan.targetNodeIds?.[0] || + operationPlan.sourceNodeIds?.[0] || + operationPlan.anchorNodeId || + null, + }); writeAgentProgressChunk( writer, { stage: "failed", currentStep: "finalize", title: "画布操作失败", - message: errorMessage, + message: failure.errorMessage, toolName: "canvasOperation", capability: operationPlan.operation, }, @@ -3784,7 +4441,7 @@ export async function POST(request: Request) { ); writeAssistantTextChunk( writer, - errorMessage, + failure.errorMessage, "canvas-model-planner:error", messageId, ); @@ -6685,6 +7342,12 @@ export async function POST(request: Request) { } const assistantMessageId = generateUUID(); + const createdTargetSlotId = await createDirectAgentTargetSlotIfNeeded({ + writer, + assistantMessageId, + operationLabel: "agent-generate-image", + resultType: "image", + }); try { const directImageResult = await executeDirectAgentImageGeneration({ requestTextContent, @@ -6753,6 +7416,7 @@ export async function POST(request: Request) { mode: "generate", sourceImageUrls: directAgentToolTargetImageUrls, model: directImageResult.modelRef || finalModelId, + targetSlotId: createdTargetSlotId, }); writeCanvasMediaChunk( writer, @@ -6854,6 +7518,12 @@ export async function POST(request: Request) { "agent:generate_video:progress-select-tool", ); writeCurrentAgentTargetLockChunk(writer, "agent:generate_video:target-lock"); + const createdTargetSlotId = await createDirectAgentTargetSlotIfNeeded({ + writer, + assistantMessageId, + operationLabel: "agent-generate-video", + resultType: "video", + }); try { writeAgentProgressChunk( writer, @@ -6941,6 +7611,7 @@ export async function POST(request: Request) { mode: "video", sourceImageUrls: directAgentToolTargetImageUrls, model: result.modelRef || finalModelId, + targetSlotId: createdTargetSlotId, }); writeCanvasMediaChunk( writer, @@ -7070,6 +7741,12 @@ export async function POST(request: Request) { ); try { + const createdTargetSlotId = await createDirectAgentTargetSlotIfNeeded({ + writer, + assistantMessageId, + operationLabel: `agent-${directAgentToolCapability}`, + resultType: "image", + }); const directImageEditResult = await executeDirectAgentImageEditTool({ capability: directAgentToolCapability, requestUrl: request.url, @@ -7148,6 +7825,7 @@ export async function POST(request: Request) { ? { [resultOriginalUrl]: resultAssetId || undefined } : undefined, model: finalModelId, + targetSlotId: createdTargetSlotId, }); writeCanvasMediaChunk( writer, diff --git a/app/api/canvas/[id]/agent-operations/route.ts b/app/api/canvas/[id]/agent-operations/route.ts index 6f39f15cb..306488588 100644 --- a/app/api/canvas/[id]/agent-operations/route.ts +++ b/app/api/canvas/[id]/agent-operations/route.ts @@ -5,6 +5,10 @@ import { applyCanvasOperationPlan, CanvasAgentMutationError, } from "@/lib/canvas/agent-mutations"; +import { + canApplyCanvasAgentMutations, + getCanvasAgentFeatureFlags, +} from "@/lib/canvas/agent-feature-flags"; import type { CanvasOperationPlan } from "@/types/canvas/agent-protocol"; const canvasOperationPlanSchema = z @@ -12,11 +16,15 @@ const canvasOperationPlanSchema = z version: z.literal(1), operationId: z.string().min(1).max(160), operation: z.enum([ + "create_agent_slot", "fill_slot", "insert_beside", "append_revision", + "replace_node_media", + "add_annotation", "link_nodes", "move_nodes", + "delete_nodes", "update_text", ]), projectId: z.string().uuid().optional(), @@ -37,6 +45,47 @@ const canvasOperationPlanSchema = z y: z.number(), }) .optional(), + slot: z + .object({ + slotType: z.enum(["image", "video", "ui-screen"]), + label: z.string().max(120).optional(), + aspectRatio: z.string().max(32).optional(), + fillPolicy: z + .enum(["replace-empty", "append-version", "replace-existing"]) + .optional(), + acceptedSources: z + .array(z.enum(["text", "image", "selection", "annotation"])) + .max(4) + .optional(), + }) + .optional(), + annotations: z + .array( + z.object({ + id: z.string().min(1).max(128).optional(), + sourceNodeId: z.string().min(1).max(160).optional(), + type: z.enum(["text-note", "arrow-note", "box", "freehand", "crop-box"]), + text: z.string().max(1000).optional(), + bounds: z + .object({ + x: z.number(), + y: z.number(), + width: z.number(), + height: z.number(), + }) + .optional(), + targetPoint: z + .object({ + x: z.number(), + y: z.number(), + }) + .optional(), + color: z.string().max(64).optional(), + createdAt: z.string().max(80).optional(), + }) + ) + .max(20) + .optional(), placement: z .enum(["right", "left", "below", "above", "viewport-center"]) .optional(), @@ -78,9 +127,12 @@ const canvasOperationPlanSchema = z }) .refine( (value) => + value.operation === "create_agent_slot" || value.operation === "link_nodes" || value.operation === "update_text" || + value.operation === "add_annotation" || value.operation === "move_nodes" || + value.operation === "delete_nodes" || value.media || (value.mediaItems?.length ?? 0) > 0, { @@ -88,6 +140,16 @@ const canvasOperationPlanSchema = z path: ["mediaItems"], } ) + .refine( + (value) => + value.operation !== "add_annotation" || + Boolean(String(value.textContent || "").trim()) || + (value.annotations?.length ?? 0) > 0, + { + message: "缺少批注内容", + path: ["annotations"], + } + ) .refine( (value) => value.operation !== "update_text" || @@ -116,6 +178,19 @@ export async function POST( } const { id } = await params; + const canvasAgentFeatures = getCanvasAgentFeatureFlags(); + if (!canApplyCanvasAgentMutations(canvasAgentFeatures)) { + return NextResponse.json( + { + success: false, + error: "当前画布 Agent 写回处于灰度保护，未修改画布", + code: "CANVAS_AGENT_WRITEBACK_DISABLED", + writebackMode: canvasAgentFeatures.writebackMode, + }, + { status: 409 } + ); + } + const body = requestSchema.parse(await request.json()); const result = await applyCanvasOperationPlan({ userId: session.user.id, @@ -130,7 +205,11 @@ export async function POST( } catch (error) { if (error instanceof CanvasAgentMutationError) { return NextResponse.json( - { success: false, error: error.message }, + { + success: false, + error: error.message, + code: error.code, + }, { status: error.status } ); } diff --git a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx index da932f6cc..9a382e306 100644 --- a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx +++ b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx @@ -29,6 +29,7 @@ import { Loader2, type LucideIcon, MessageCircle, + MonitorSmartphone, MousePointer, Music2, Palette, @@ -234,6 +235,7 @@ import type { AgentWorkflowUiConfig, } from "@/types/agent"; import type { + AgentSlotNodeData, CanvasEdge, CanvasNode, CanvasState, @@ -1785,6 +1787,9 @@ export default function CanvasEditorLegacyPage({ params }: PageProps) { const updateNode = useCanvasStore((state) => state.updateNode); const pushHistory = useCanvasStore((state) => state.pushHistory); const setGenerating = useCanvasStore((state) => state.setGenerating); + const isAgentOperationRefreshActive = useCanvasStore( + (state) => state.isAgentOperationRefreshActive + ); // Local state const [isLoading, setIsLoading] = useState(true); @@ -4477,7 +4482,7 @@ export default function CanvasEditorLegacyPage({ params }: PageProps) { // 自动保存逻辑 useEffect(() => { - if (isLoading || isAgentMode) return; + if (isLoading || isAgentMode || isAgentOperationRefreshActive()) return; const saveTimer = setTimeout(async () => { setIsSaving(true); @@ -4490,11 +4495,11 @@ export default function CanvasEditorLegacyPage({ params }: PageProps) { }, 3000); // 3秒自动保存 return () => clearTimeout(saveTimer); - }, [isAgentMode, nodes, isLoading, persistCanvasState]); + }, [isAgentMode, isAgentOperationRefreshActive, nodes, isLoading, persistCanvasState]); // 删除节点时立即保存（按节点ID差集检测），避免删除后快速刷新被回滚 useEffect(() => { - if (isLoading || isAgentMode) { + if (isLoading || isAgentMode || isAgentOperationRefreshActive()) { previousNodeIdsRef.current = new Set(nodes.map((node) => node.id)); return; } @@ -4521,7 +4526,7 @@ export default function CanvasEditorLegacyPage({ params }: PageProps) { }; void persistAfterDelete(); } - }, [isAgentMode, nodes, isLoading, persistCanvasState]); + }, [isAgentMode, isAgentOperationRefreshActive, nodes, isLoading, persistCanvasState]); // 页面关闭/刷新前尝试做一次 keepalive 保存，降低“刚删就刷新”回滚概率 useEffect(() => { @@ -4529,6 +4534,7 @@ export default function CanvasEditorLegacyPage({ params }: PageProps) { const handleBeforeUnload = () => { try { + if (isAgentOperationRefreshActive()) return; const canvasData = buildPersistedCanvasData(); const { canvasRevision, canvasUpdatedAt } = useCanvasStore.getState(); void fetch(`/api/canvas/${id}`, { @@ -4557,6 +4563,7 @@ export default function CanvasEditorLegacyPage({ params }: PageProps) { activeCanvasWorkspaceType, buildPersistedCanvasData, id, + isAgentOperationRefreshActive, isAgentMode, isLoading, ]); @@ -6773,9 +6780,34 @@ export default function CanvasEditorLegacyPage({ params }: PageProps) { }); }, [createCanvasNode, handleAddWorkflowNode, isVideoMode]); - const handleAddAgentImageSlotNode = useCallback(() => { + const handleAddAgentSlotNode = useCallback((slotType: AgentSlotNodeData["slotType"]) => { if (isVideoMode) return; + const slotDefaults: Record< + AgentSlotNodeData["slotType"], + { + label: string; + size: CanvasNode["size"]; + aspectRatio: string; + } + > = { + image: { + label: "图片槽位", + size: { width: 320, height: 320 }, + aspectRatio: "1:1", + }, + video: { + label: "视频槽位", + size: { width: 360, height: 220 }, + aspectRatio: "16:9", + }, + "ui-screen": { + label: "UI 槽位", + size: { width: 360, height: 260 }, + aspectRatio: "16:9", + }, + }; + const defaults = slotDefaults[slotType]; const origin = getVisibleCanvasOrigin(); createCanvasNode({ type: "agent-slot", @@ -6783,17 +6815,18 @@ export default function CanvasEditorLegacyPage({ params }: PageProps) { x: origin.x + 40, y: origin.y + 40, }, - size: { width: 320, height: 320 }, + size: defaults.size, data: { - slotType: "image", - label: "图片槽位", + slotType, + label: defaults.label, status: "empty", - aspectRatio: "1:1", + aspectRatio: defaults.aspectRatio, fillPolicy: "replace-empty", acceptedSources: ["text", "image", "selection", "annotation"], filledNodeIds: [], metadata: { createdBy: "toolbar", + createdSlotType: slotType, createdAt: new Date().toISOString(), }, }, @@ -6801,6 +6834,10 @@ export default function CanvasEditorLegacyPage({ params }: PageProps) { }); }, [createCanvasNode, getVisibleCanvasOrigin, isVideoMode]); + const handleAddAgentImageSlotNode = useCallback(() => { + handleAddAgentSlotNode("image"); + }, [handleAddAgentSlotNode]); + const handleAddVideoNode = useCallback(() => { if (isVideoMode) { handleAddWorkflowNode("video-input"); @@ -10403,33 +10440,53 @@ export default function CanvasEditorLegacyPage({ params }: PageProps) { canvasToolbarConfig, "addAgentImageSlot" ) ? ( - - - + + + + { + getCanvasToolbarItem( canvasToolbarConfig, "addAgentImageSlot" - )} - /> - - - - { - getCanvasToolbarItem( - canvasToolbarConfig, - "addAgentImageSlot" - ).label - } - - + ).label + } + + + + handleAddAgentImageSlotNode()} + > + + 图片槽位 + + handleAddAgentSlotNode("video")} + > + + + handleAddAgentSlotNode("ui-screen")} + > + + UI 槽位 + + + ) : null} diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 80695e026..e4d050040 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -96,6 +96,11 @@ import { isCanvasImageReadyForAi, isCanvasImageUploadPending, } from "@/lib/canvas/local-upload-state"; +import { + computeCanvasAgentPanViewport, + resolveCanvasViewOperationTargetNodeIds, +} from "@/lib/canvas/agent-view-operations"; +import { getCanvasAgentOperationErrorMessage } from "@/lib/canvas/agent-operation-errors"; import { getCanvasSkillIconComponent } from "@/components/canvas/canvas-skill-icons"; import { ImageParamsBar, type ImageGenerationParams } from "@/components/image-params-bar"; import { VideoParamsBar, type VideoGenerationParams } from "@/components/video-params-bar"; @@ -122,7 +127,10 @@ import { getAgentProgressFallbackTitle, getAgentTargetLockTitle, getAgentTargetResolutionLabel, + getCanvasMediaPayloadHandledItemIds, + getCanvasOperationHandledMediaItemIds, type ImageGenerationStatusStreamPayload, + shouldSkipCanvasMediaPayloadForHandledItems, } from "@/components/canvas/chat-panel/canvas-agent-stream"; import { type DetailArchive, @@ -1825,9 +1833,6 @@ export function CanvasChatPanel({ const store = useCanvasStore.getState(); const currentNodes = store.nodes; - const nodeIds = Array.isArray(payload.nodeIds) - ? payload.nodeIds.map((id) => String(id || "").trim()).filter(Boolean) - : []; const selectedIds = store.selectedNodeIds.filter(Boolean); const zoomByFactor = (factor: number) => { if (typeof window === "undefined") return; @@ -1859,15 +1864,31 @@ export function CanvasChatPanel({ }); return; } + if ( + payload.operation === "pan_to" && + payload.center && + typeof payload.center === "object" && + typeof window !== "undefined" + ) { + setTransientViewport( + computeCanvasAgentPanViewport({ + currentViewport: useCanvasStore.getState().viewport, + center: payload.center, + stage: { + width: window.innerWidth, + height: window.innerHeight, + }, + zoom: payload.zoom, + }) + ); + return; + } - const targetIds = - payload.operation === "focus_selection" && nodeIds.length === 0 - ? selectedIds - : nodeIds.length > 0 - ? nodeIds - : payload.anchorNodeId - ? [payload.anchorNodeId] - : []; + const targetIds = resolveCanvasViewOperationTargetNodeIds({ + plan: payload, + nodes: currentNodes, + selectedNodeIds: selectedIds, + }); const targetNodes = payload.operation === "fit_all" ? currentNodes.filter((node) => @@ -3593,6 +3614,48 @@ export function CanvasChatPanel({ ] ); + useEffect(() => { + if (!effectiveAgentEnabled || typeof window === "undefined") return; + + const handleAgentSlotFillRequest = (event: Event) => { + const detail = (event as CustomEvent<{ nodeId?: string; label?: string }>).detail; + const label = String(detail?.label || "这个槽位").trim(); + const prompt = `在${label === "这个槽位" ? label : `「${label}」`}生成一张图`; + + handleSelectGenerationMode("agent"); + setInput(prompt); + + window.setTimeout(() => { + const editableDiv = textareaRef.current as HTMLDivElement | null; + if (!editableDiv) return; + editableDiv.innerHTML = ""; + editableDiv.appendChild(document.createTextNode(prompt)); + editableDiv.focus(); + + const selection = window.getSelection(); + if (!selection) return; + const range = document.createRange(); + range.selectNodeContents(editableDiv); + range.collapse(false); + selection.removeAllRanges(); + selection.addRange(range); + }, 0); + + toast.message("已选中槽位，可补充生成要求后发送"); + }; + + window.addEventListener( + "canvas-agent-slot-fill-request", + handleAgentSlotFillRequest + ); + return () => { + window.removeEventListener( + "canvas-agent-slot-fill-request", + handleAgentSlotFillRequest + ); + }; + }, [effectiveAgentEnabled, handleSelectGenerationMode]); + const applyVideoWorkspacePreset = useCallback( (mode: VideoGenerationParams["mode"]) => { if (!preferredVideoWorkspaceModel) { @@ -4002,19 +4065,19 @@ export function CanvasChatPanel({ try { const result = await canvasService.getProject(targetProjectId); if (!result.success || !result.data?.canvasData) { + useCanvasStore.getState().markAgentOperationRefresh(); void mutate(`/api/canvas/${targetProjectId}`); return false; } - useCanvasStore.getState().loadCanvas(result.data.canvasData); - useCanvasStore - .getState() - .setCanvasRevision(String(result.data.updatedAt || ""), String(result.data.updatedAt || "")); + const store = useCanvasStore.getState(); + store.markAgentOperationRefresh(); + store.loadCanvas(result.data.canvasData); + store.setCanvasRevision(String(result.data.updatedAt || ""), String(result.data.updatedAt || "")); if (createdNodeIds.length > 0) { - useCanvasStore.getState().selectNodes(createdNodeIds); - const focusNode = useCanvasStore - .getState() - .nodes.find((node) => node.id === createdNodeIds[0]); + const refreshedStore = useCanvasStore.getState(); + refreshedStore.selectNodes(createdNodeIds); + const focusNode = refreshedStore.nodes.find((node) => node.id === createdNodeIds[0]); if (focusNode) { focusCanvasOnNode(focusNode); } @@ -4023,6 +4086,7 @@ export function CanvasChatPanel({ return true; } catch (error) { console.error("[CanvasChat] Failed to refresh canvas after agent operation:", error); + useCanvasStore.getState().markAgentOperationRefresh(); void mutate(`/api/canvas/${targetProjectId}`); return false; } @@ -4156,16 +4220,29 @@ export function CanvasChatPanel({ if (dataPart.type === "data-canvasOperation") { try { - const payload: CanvasOperationStreamPayload = - typeof dataPart.data === "string" ? JSON.parse(dataPart.data) : dataPart.data; - if (!payload || typeof payload !== "object") return; - const handledIds = Array.isArray(payload.handledCanvasMediaItemIds) - ? payload.handledCanvasMediaItemIds - .map((id) => String(id || "").trim()) - .filter(Boolean) - : []; - handledIds.forEach((id) => handledCanvasMediaItemIdsRef.current.add(id)); - const targetProjectId = String(payload.projectId || "").trim(); + const payload: CanvasOperationStreamPayload = + typeof dataPart.data === "string" ? JSON.parse(dataPart.data) : dataPart.data; + if (!payload || typeof payload !== "object") return; + if (payload.status === "failed") { + toast.warning( + getCanvasAgentOperationErrorMessage({ + code: payload.errorCode, + fallbackMessage: payload.errorMessage || payload.message, + }) + ); + return; + } + if (payload.status === "shadowed") { + console.info("[CanvasChat] Canvas operation shadowed:", { + operationId: payload.operationId, + operation: payload.operation, + mediaItemIds: payload.mediaItemIds, + }); + return; + } + const handledIds = getCanvasOperationHandledMediaItemIds(payload); + handledIds.forEach((id) => handledCanvasMediaItemIdsRef.current.add(id)); + const targetProjectId = String(payload.projectId || "").trim(); if (targetProjectId) { void refreshCanvasAfterAgentOperation({ projectId: targetProjectId, @@ -4194,27 +4271,20 @@ export function CanvasChatPanel({ if (dataPart.type === "data-canvasMedia") { try { - const payload: CanvasMediaStreamPayload = - typeof dataPart.data === "string" ? JSON.parse(dataPart.data) : dataPart.data; - if (!payload || typeof payload !== "object") return; - const handledIds = Array.isArray(payload.handledCanvasMediaItemIds) - ? payload.handledCanvasMediaItemIds - .map((id) => String(id || "").trim()) - .filter(Boolean) - : []; - handledIds.forEach((id) => handledCanvasMediaItemIdsRef.current.add(id)); - const mediaItemIds = Array.isArray(payload.mediaItemIds) - ? payload.mediaItemIds - .map((id) => String(id || "").trim()) - .filter(Boolean) - : []; - if ( - mediaItemIds.length > 0 && - mediaItemIds.every((id) => handledCanvasMediaItemIdsRef.current.has(id)) - ) { - clearStreamImageGenerationStateForChat(eventChatId); - return; - } + const payload: CanvasMediaStreamPayload = + typeof dataPart.data === "string" ? JSON.parse(dataPart.data) : dataPart.data; + if (!payload || typeof payload !== "object") return; + const handledIds = getCanvasMediaPayloadHandledItemIds(payload); + handledIds.forEach((id) => handledCanvasMediaItemIdsRef.current.add(id)); + if ( + shouldSkipCanvasMediaPayloadForHandledItems({ + payload, + handledCanvasMediaItemIds: handledCanvasMediaItemIdsRef.current, + }) + ) { + clearStreamImageGenerationStateForChat(eventChatId); + return; + } const imageUrls = Array.isArray(payload.imageUrls) ? payload.imageUrls.filter( (url): url is string => diff --git a/components/canvas/CanvasWorkspace.tsx b/components/canvas/CanvasWorkspace.tsx index b5ac29448..a9476e3db 100644 --- a/components/canvas/CanvasWorkspace.tsx +++ b/components/canvas/CanvasWorkspace.tsx @@ -4,7 +4,7 @@ import React, { useRef, useState, useCallback, useMemo } from "react"; import { Rnd } from "react-rnd"; import { motion } from "framer-motion"; import { useCanvasStore, canvasSelectors } from "@/store/canvas"; -import type { CanvasEdge, CanvasNode, Position } from "@/types/canvas"; +import type { AgentSlotNodeData, CanvasEdge, CanvasNode, Position } from "@/types/canvas"; import { ImageNode } from "./nodes/ImageNode"; import { AgentSlotNode } from "./nodes/AgentSlotNode"; import { Model3DNode } from "./nodes/Model3DNode"; @@ -26,14 +26,18 @@ import { GitBranch, Heart, HelpCircle, + ImagePlus, + LayoutGrid, Lightbulb, Minus, Pin, Plus, + RefreshCw, Star, Tag, type LucideIcon, UserRound, + Video, Wand2, Zap, } from "lucide-react"; @@ -299,6 +303,12 @@ export function CanvasWorkspace({ vertical?: { x: number; top: number; bottom: number }; horizontal?: { y: number; left: number; right: number }; }; + type AgentSlotContextMenuState = { + nodeId: string; + label: string; + x: number; + y: number; + } | null; const containerRef = useRef(null); const contentLayerRef = useRef(null); const [isPanning, setIsPanning] = useState(false); @@ -310,6 +320,8 @@ export function CanvasWorkspace({ const [isDownloadSubmitting, setIsDownloadSubmitting] = useState(false); const [snapGuides, setSnapGuides] = useState(null); const [selectedEdgeId, setSelectedEdgeId] = useState(null); + const [agentSlotContextMenu, setAgentSlotContextMenu] = + useState(null); const [connectionDraft, setConnectionDraft] = useState<{ sourceId: string; current: Position; @@ -578,6 +590,115 @@ export function CanvasWorkspace({ }); }, [updateNode]); + const handleAgentSlotDataChange = useCallback( + (node: CanvasNode, updates: Partial) => { + const currentData = (node.data || {}) as AgentSlotNodeData; + updateNode(node.id, { + data: { + ...currentData, + ...updates, + metadata: { + ...(currentData.metadata || {}), + ...(updates.metadata || {}), + }, + }, + }); + }, + [updateNode] + ); + + const openAgentSlotContextMenu = useCallback( + (event: React.MouseEvent, node: CanvasNode) => { + if (node.type !== "agent-slot") return; + event.preventDefault(); + event.stopPropagation(); + + const rect = containerRef.current?.getBoundingClientRect(); + const menuWidth = 236; + const menuHeight = 240; + const rawX = rect ? event.clientX - rect.left : event.clientX; + const rawY = rect ? event.clientY - rect.top : event.clientY; + const label = + String((node.data as AgentSlotNodeData | undefined)?.label || "").trim() || + "Agent 槽位"; + + clearLatestGeneratedImageHighlights(); + setSelectedEdgeId(null); + selectNode(node.id, false); + setAgentSlotContextMenu({ + nodeId: node.id, + label, + x: Math.max(8, Math.min(rawX, Math.max(8, containerSize.width - menuWidth - 8))), + y: Math.max(8, Math.min(rawY, Math.max(8, containerSize.height - menuHeight - 8))), + }); + }, + [ + clearLatestGeneratedImageHighlights, + containerSize.height, + containerSize.width, + selectNode, + ] + ); + + const updateAgentSlotFromContextMenu = useCallback( + (updates: Partial) => { + if (!agentSlotContextMenu) return; + const node = nodes.find((item) => item.id === agentSlotContextMenu.nodeId); + if (!node || node.type !== "agent-slot") return; + handleAgentSlotDataChange(node, updates); + if (updates.label) { + setAgentSlotContextMenu((current) => + current ? { ...current, label: String(updates.label) } : current + ); + } + }, + [agentSlotContextMenu, handleAgentSlotDataChange, nodes] + ); + + const updateAgentSlotTypeFromContextMenu = useCallback( + (slotType: NonNullable) => { + if (!agentSlotContextMenu) return; + const node = nodes.find((item) => item.id === agentSlotContextMenu.nodeId); + if (!node || node.type !== "agent-slot") return; + + const data = (node.data || {}) as AgentSlotNodeData; + const currentLabel = String(data.label || "").trim(); + const defaultLabels = new Set(["图片槽位", "视频槽位", "UI 槽位", "Agent 槽位"]); + const defaults = + slotType === "video" + ? { label: "视频槽位", aspectRatio: "16:9" } + : slotType === "ui-screen" + ? { label: "UI 槽位", aspectRatio: "16:9" } + : { label: "图片槽位", aspectRatio: "1:1" }; + const nextLabel = + !currentLabel || defaultLabels.has(currentLabel) ? defaults.label : currentLabel; + + handleAgentSlotDataChange(node, { + slotType, + label: nextLabel, + aspectRatio: defaults.aspectRatio, + }); + setAgentSlotContextMenu((current) => + current ? { ...current, label: nextLabel } : current + ); + }, + [agentSlotContextMenu, handleAgentSlotDataChange, nodes] + ); + + const requestFillAgentSlotFromContextMenu = useCallback(() => { + if (!agentSlotContextMenu || typeof window === "undefined") return; + selectNode(agentSlotContextMenu.nodeId, false); + window.dispatchEvent( + new CustomEvent("canvas-agent-slot-fill-request", { + detail: { + nodeId: agentSlotContextMenu.nodeId, + label: agentSlotContextMenu.label, + }, + }) + ); + setAgentSlotContextMenu(null); + }, [agentSlotContextMenu, selectNode]); + const isMacPlatform = React.useMemo(() => { if (typeof navigator === "undefined") return false; return /Mac|iPhone|iPad|iPod/i.test(navigator.platform); @@ -947,6 +1068,7 @@ export function CanvasWorkspace({ // 处理画布点击（取消选择） const handleCanvasClick = useCallback((e: React.MouseEvent) => { if (hasBlockingDialogOpen()) return; + setAgentSlotContextMenu(null); if (!isEventFromCanvasContainer(e.target)) return; if (suppressCanvasClickRef.current) { suppressCanvasClickRef.current = false; @@ -1738,8 +1860,14 @@ export function CanvasWorkspace({ )} data-canvas-node-id={node.id} style={{ overflow: 'visible' }} // 允许工具栏显示在节点外部 + onContextMenu={(event: React.MouseEvent) => { + if (node.type === "agent-slot") { + openAgentSlotContextMenu(event, node); + } + }} onClick={(e: React.MouseEvent) => { e.stopPropagation(); + setAgentSlotContextMenu(null); clearLatestGeneratedImageHighlights(); setSelectedEdgeId(null); selectNode(node.id, e.shiftKey); @@ -1822,6 +1950,7 @@ export function CanvasWorkspace({ handleAgentSlotDataChange(node, updates)} /> )} {node.type === 'text' && ( @@ -1876,6 +2005,17 @@ export function CanvasWorkspace({ ); }; + const agentSlotContextNode = agentSlotContextMenu + ? nodes.find((node) => node.id === agentSlotContextMenu.nodeId) + : null; + const agentSlotContextData = + agentSlotContextNode?.type === "agent-slot" + ? ((agentSlotContextNode.data || {}) as AgentSlotNodeData) + : null; + const agentSlotContextType = agentSlotContextData?.slotType || "image"; + const agentSlotContextFillPolicy = + agentSlotContextData?.fillPolicy || "replace-empty"; + return (
)} + {agentSlotContextMenu ? ( +
event.stopPropagation()} + onContextMenu={(event) => { + event.preventDefault(); + event.stopPropagation(); + }} + onMouseDown={(event) => event.stopPropagation()} + > +
+ {agentSlotContextMenu.label} +
+ +
+ + + +
+ + +
+ ) : null} + {showZoomControls ? (
; updatedNodeIds?: string[]; @@ -26,11 +27,43 @@ export type CanvasOperationStreamPayload = { canvasRevision?: string; anchorNodeId?: string; handledCanvasMediaItemIds?: string[]; + mediaItemIds?: string[]; + errorCode?: string; + errorMessage?: string; + fallbackToCanvasMedia?: boolean; message?: string; }; export type CanvasViewOperationStreamPayload = CanvasViewOperationPlan; +export function normalizeCanvasStreamIdList(value: unknown) { + return Array.isArray(value) + ? value.map((id) => String(id || "").trim()).filter(Boolean) + : []; +} + +export function getCanvasOperationHandledMediaItemIds( + payload: CanvasOperationStreamPayload +) { + if (payload.status === "failed" || payload.status === "shadowed") return []; + return normalizeCanvasStreamIdList(payload.handledCanvasMediaItemIds); +} + +export function getCanvasMediaPayloadHandledItemIds(payload: CanvasMediaStreamPayload) { + return normalizeCanvasStreamIdList(payload.handledCanvasMediaItemIds); +} + +export function shouldSkipCanvasMediaPayloadForHandledItems(params: { + payload: CanvasMediaStreamPayload; + handledCanvasMediaItemIds: ReadonlySet; +}) { + const mediaItemIds = normalizeCanvasStreamIdList(params.payload.mediaItemIds); + return ( + mediaItemIds.length > 0 && + mediaItemIds.every((id) => params.handledCanvasMediaItemIds.has(id)) + ); +} + export type ImageGenerationStatusStreamPayload = { phase?: "queued" | "processing" | "finalizing" | "failed" | "completed"; providerType?: string; diff --git a/components/canvas/nodes/AgentSlotNode.tsx b/components/canvas/nodes/AgentSlotNode.tsx index 4e7a0eb98..2f2054e3b 100644 --- a/components/canvas/nodes/AgentSlotNode.tsx +++ b/components/canvas/nodes/AgentSlotNode.tsx @@ -1,14 +1,72 @@ "use client"; -import { ImagePlus, MonitorSmartphone, Sparkles, Video } from "lucide-react"; +import { + FileText, + ImagePlus, + Layers, + MonitorSmartphone, + MousePointer2, + PencilLine, + Settings2, + Sparkles, + Video, +} from "lucide-react"; +import { + DropdownMenu, + DropdownMenuCheckboxItem, + DropdownMenuContent, + DropdownMenuLabel, + DropdownMenuRadioGroup, + DropdownMenuRadioItem, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { Input } from "@/components/ui/input"; import type { AgentSlotNodeData, CanvasNode } from "@/types/canvas"; import { cn } from "@/lib/utils"; interface AgentSlotNodeProps { node: CanvasNode; isSelected: boolean; + onDataChange?: (updates: Partial) => void; } +type SlotType = AgentSlotNodeData["slotType"]; +type SlotFillPolicy = NonNullable; +type SlotAcceptedSource = NonNullable[number]; + +const SLOT_TYPE_OPTIONS: Array<{ + value: SlotType; + label: string; + icon: typeof ImagePlus; +}> = [ + { value: "image", label: "图片槽位", icon: ImagePlus }, + { value: "video", label: "视频槽位", icon: Video }, + { value: "ui-screen", label: "UI 槽位", icon: MonitorSmartphone }, +]; + +const FILL_POLICY_OPTIONS: Array<{ + value: SlotFillPolicy; + label: string; +}> = [ + { value: "replace-empty", label: "填充空槽" }, + { value: "append-version", label: "追加版本" }, + { value: "replace-existing", label: "允许替换" }, +]; + +const ACCEPTED_SOURCE_OPTIONS: Array<{ + value: SlotAcceptedSource; + label: string; + icon: typeof FileText; +}> = [ + { value: "text", label: "文本", icon: FileText }, + { value: "image", label: "图片", icon: ImagePlus }, + { value: "selection", label: "选中对象", icon: MousePointer2 }, + { value: "annotation", label: "标注", icon: PencilLine }, +]; + +const ASPECT_RATIO_OPTIONS = ["1:1", "4:3", "3:4", "16:9", "9:16"]; + function getSlotIcon(slotType: AgentSlotNodeData["slotType"]) { if (slotType === "video") return Video; if (slotType === "ui-screen") return MonitorSmartphone; @@ -28,18 +86,87 @@ function getStatusLabel(status: AgentSlotNodeData["status"]) { return "等待生成"; } -export function AgentSlotNode({ node, isSelected }: AgentSlotNodeProps) { +function getFillPolicyLabel(fillPolicy: AgentSlotNodeData["fillPolicy"]) { + if (fillPolicy === "append-version") return "追加版本"; + if (fillPolicy === "replace-existing") return "允许替换"; + return "填充空槽"; +} + +function normalizeAcceptedSources( + acceptedSources: AgentSlotNodeData["acceptedSources"] +): SlotAcceptedSource[] { + const normalized = (Array.isArray(acceptedSources) ? acceptedSources : []) + .filter((value): value is SlotAcceptedSource => + ACCEPTED_SOURCE_OPTIONS.some((option) => option.value === value) + ); + + if (normalized.length > 0) { + return Array.from(new Set(normalized)); + } + + return ACCEPTED_SOURCE_OPTIONS.map((option) => option.value); +} + +function getAcceptedSourcesLabel(acceptedSources: SlotAcceptedSource[]) { + return ACCEPTED_SOURCE_OPTIONS.filter((option) => + acceptedSources.includes(option.value) + ) + .map((option) => option.label) + .join("、"); +} + +function isSlotType(value: string): value is SlotType { + return SLOT_TYPE_OPTIONS.some((option) => option.value === value); +} + +function isSlotFillPolicy(value: string): value is SlotFillPolicy { + return FILL_POLICY_OPTIONS.some((option) => option.value === value); +} + +export function AgentSlotNode({ + node, + isSelected, + onDataChange, +}: AgentSlotNodeProps) { const data = node.data as AgentSlotNodeData; const slotType = data.slotType || "image"; + const acceptedSources = normalizeAcceptedSources(data.acceptedSources); const SlotIcon = getSlotIcon(slotType); const filledCount = Array.isArray(data.filledNodeIds) ? data.filledNodeIds.length : 0; + const fillPolicy = data.fillPolicy || "replace-empty"; + const aspectRatio = data.aspectRatio || "1:1"; + const canEdit = Boolean(onDataChange); + + const updateSlotData = (updates: Partial) => { + if (!onDataChange) return; + onDataChange({ + ...updates, + metadata: { + ...(data.metadata || {}), + ...(updates.metadata || {}), + agentSlotConfiguredAt: new Date().toISOString(), + }, + }); + }; + + const toggleAcceptedSource = ( + source: SlotAcceptedSource, + checked: boolean + ) => { + const next = checked + ? Array.from(new Set([...acceptedSources, source])) + : acceptedSources.filter((value) => value !== source); + + if (next.length === 0) return; + updateSlotData({ acceptedSources: next }); + }; return (
+ {canEdit ? ( + + + + + event.stopPropagation()} + onMouseDown={(event) => event.stopPropagation()} + sideOffset={8} + > +
+ + + updateSlotData({ label: event.target.value }) + } + onClick={(event) => event.stopPropagation()} + onKeyDown={(event) => event.stopPropagation()} + onMouseDown={(event) => event.stopPropagation()} + /> +
+ + + + + 槽位类型 + + { + if (!isSlotType(value)) return; + updateSlotData({ slotType: value }); + }} + > + {SLOT_TYPE_OPTIONS.map((option) => { + const OptionIcon = option.icon; + return ( + + + {option.label} + + ); + })} + + + + + + 画幅 + + { + if (!ASPECT_RATIO_OPTIONS.includes(value)) return; + updateSlotData({ aspectRatio: value }); + }} + > + {ASPECT_RATIO_OPTIONS.map((option) => ( + + + {option} + + ))} + + + + + + 填充策略 + + { + if (!isSlotFillPolicy(value)) return; + updateSlotData({ fillPolicy: value }); + }} + > + {FILL_POLICY_OPTIONS.map((option) => ( + + {option.label} + + ))} + + + + + + 可用来源 + + {ACCEPTED_SOURCE_OPTIONS.map((option) => { + const OptionIcon = option.icon; + const checked = acceptedSources.includes(option.value); + return ( + + toggleAcceptedSource(option.value, Boolean(nextChecked)) + } + onSelect={(event) => event.preventDefault()} + > + + {option.label} + + ); + })} + + + ) : null} +
- {data.aspectRatio || "1:1"} - - {data.fillPolicy === "append-version" - ? "追加版本" - : data.fillPolicy === "replace-existing" - ? "允许替换" - : "填充空槽"} + {aspectRatio} + + {getAcceptedSourcesLabel(acceptedSources)} + {getFillPolicyLabel(fillPolicy)}
); diff --git a/components/canvas/nodes/ImageNode.tsx b/components/canvas/nodes/ImageNode.tsx index b2b957b17..e6adca9d5 100644 --- a/components/canvas/nodes/ImageNode.tsx +++ b/components/canvas/nodes/ImageNode.tsx @@ -115,8 +115,16 @@ import { ThreeDGenerationDialog, } from "@/components/canvas/nodes/image-node/image-node-dialogs"; import { ImageNodeSelectedTitle } from "@/components/canvas/nodes/image-node/image-node-title"; -import { ImageAnnotator } from "@/components/image-annotator"; +import { + ImageAnnotator, + type ImageAnnotatorConfirmPayload, +} from "@/components/image-annotator"; import { SceneFusionDialog } from "@/components/scene-fusion-dialog"; +import { + appendCanvasVisualAnnotationsMetadata, + syncCanvasLocalEditRegionAnnotationsMetadata, + syncCanvasNodeNoteAnnotationMetadata, +} from "@/lib/canvas/agent-annotation-metadata"; import { Button } from "@/components/ui/button"; import { Dialog, @@ -245,6 +253,25 @@ interface ImageOverlayRectStyle { height: number; } +interface CanvasNodeAnnotationDisplayItem { + id: string; + type: "text-note" | "arrow-note" | "box" | "freehand" | "crop-box"; + text?: string; + bounds?: { + x: number; + y: number; + width: number; + height: number; + }; + targetPoint?: { + x: number; + y: number; + }; + color?: string; + createdAt?: string; + createdByAgent?: boolean; +} + interface LayerSplitBounds { x: number; y: number; @@ -1127,6 +1154,120 @@ async function drawMarkerOnImage( }); } +function normalizeCanvasNodeAnnotations( + metadata: unknown +): CanvasNodeAnnotationDisplayItem[] { + if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) { + return []; + } + const rawAnnotations = (metadata as Record).annotations; + if (!Array.isArray(rawAnnotations)) return []; + + return rawAnnotations + .map((annotation, index): CanvasNodeAnnotationDisplayItem | null => { + if (!annotation || typeof annotation !== "object" || Array.isArray(annotation)) { + return null; + } + const raw = annotation as Record; + const type = String(raw.type || "text-note"); + if ( + type !== "text-note" && + type !== "arrow-note" && + type !== "box" && + type !== "freehand" && + type !== "crop-box" + ) { + return null; + } + const normalizePoint = (value: unknown) => { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return undefined; + } + const point = value as Record; + const x = Number(point.x); + const y = Number(point.y); + if (!Number.isFinite(x) || !Number.isFinite(y)) return undefined; + return { x, y }; + }; + const normalizeBounds = (value: unknown) => { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return undefined; + } + const bounds = value as Record; + const x = Number(bounds.x); + const y = Number(bounds.y); + const width = Number(bounds.width); + const height = Number(bounds.height); + if ( + !Number.isFinite(x) || + !Number.isFinite(y) || + !Number.isFinite(width) || + !Number.isFinite(height) || + width <= 0 || + height <= 0 + ) { + return undefined; + } + return { x, y, width, height }; + }; + + return { + id: String(raw.id || `annotation-${index + 1}`).slice(0, 128), + type: type as CanvasNodeAnnotationDisplayItem["type"], + text: + typeof raw.text === "string" + ? String(raw.text).trim().slice(0, 180) + : undefined, + bounds: normalizeBounds(raw.bounds), + targetPoint: normalizePoint(raw.targetPoint), + color: + typeof raw.color === "string" ? String(raw.color).slice(0, 64) : undefined, + createdAt: + typeof raw.createdAt === "string" + ? String(raw.createdAt).slice(0, 64) + : undefined, + createdByAgent: raw.createdByAgent === true, + }; + }) + .filter((annotation): annotation is CanvasNodeAnnotationDisplayItem => + Boolean(annotation) + ) + .slice(-12); +} + +function getCanvasAnnotationBoundsStyle( + annotation: CanvasNodeAnnotationDisplayItem, + metrics: ImageDisplayMetrics | null +): React.CSSProperties | null { + if (!metrics || !annotation.bounds) return null; + const { bounds } = annotation; + const normalized = + Math.abs(bounds.x) <= 1 && + Math.abs(bounds.y) <= 1 && + Math.abs(bounds.width) <= 1 && + Math.abs(bounds.height) <= 1; + + if (normalized) { + return { + left: metrics.offsetX + bounds.x * metrics.renderedWidth, + top: metrics.offsetY + bounds.y * metrics.renderedHeight, + width: Math.max(bounds.width * metrics.renderedWidth, 1), + height: Math.max(bounds.height * metrics.renderedHeight, 1), + }; + } + + const scaleX = + metrics.naturalWidth > 0 ? metrics.renderedWidth / metrics.naturalWidth : 1; + const scaleY = + metrics.naturalHeight > 0 ? metrics.renderedHeight / metrics.naturalHeight : 1; + return { + left: metrics.offsetX + bounds.x * scaleX, + top: metrics.offsetY + bounds.y * scaleY, + width: Math.max(bounds.width * scaleX, 1), + height: Math.max(bounds.height * scaleY, 1), + }; +} + interface ImageNodeProps { node: CanvasNode; isSelected: boolean; @@ -1184,6 +1325,14 @@ export function ImageNode({ "/api/user/membership-summary", fetcher ); + const canvasAgentAnnotations = useMemo( + () => normalizeCanvasNodeAnnotations(data.metadata), + [data.metadata] + ); + const visibleCanvasAgentAnnotations = useMemo( + () => canvasAgentAnnotations.slice(-3), + [canvasAgentAnnotations] + ); const imageNodeMoreItem = getCanvasToolbarItem(canvasToolbarConfig, "imageNodeMore"); const imageAddToChatItem = getCanvasToolbarItem(canvasToolbarConfig, "imageAddToChat"); const imagePickObjectItem = getCanvasToolbarItem(canvasToolbarConfig, "imagePickObject"); @@ -2579,10 +2728,20 @@ export function ImageNode({ delete nextMetadata[LOCAL_EDIT_METADATA_KEY]; } + const syncedAnnotationMetadata = + syncCanvasLocalEditRegionAnnotationsMetadata({ + metadata: nextMetadata, + nodeId: node.id, + regions: nextRegions, + }); + updateNode(node.id, { data: { ...data, - metadata: Object.keys(nextMetadata).length > 0 ? nextMetadata : undefined, + metadata: + Object.keys(syncedAnnotationMetadata.metadata).length > 0 + ? syncedAnnotationMetadata.metadata + : undefined, }, }); }, @@ -5557,10 +5716,19 @@ export function ImageNode({ const handleSaveNote = useCallback(() => { const nextNote = draftNote.trim(); + const noteAnnotationMetadata = syncCanvasNodeNoteAnnotationMetadata({ + metadata: data.metadata, + nodeId: node.id, + note: nextNote, + }); updateNode(node.id, { data: { ...data, note: nextNote || undefined, + metadata: + Object.keys(noteAnnotationMetadata.metadata).length > 0 + ? noteAnnotationMetadata.metadata + : undefined, }, }); setDraftNote(nextNote); @@ -6700,7 +6868,10 @@ export function ImageNode({ ); // 图片标注完成回调 - const handleAnnotateComplete = async (annotatedImageUrl: string) => { + const handleAnnotateComplete = async ( + annotatedImageUrl: string, + payload?: ImageAnnotatorConfirmPayload + ) => { setAnnotatorOpen(false); // 如果仍然是 base64 格式，先上传到服务器（ImageAnnotator 应该已经处理了，这里作为保护） @@ -6728,6 +6899,14 @@ export function ImageNode({ } } + const visualAnnotationMetadata = appendCanvasVisualAnnotationsMetadata({ + metadata: data.metadata, + nodeId: node.id, + actions: payload?.actions || [], + imageWidth: payload?.imageWidth, + imageHeight: payload?.imageHeight, + }); + // 更新当前节点，标记为已标注图片 // 标注后的图片会带有视觉标记（箭头、圆圈、文字等） // 这些标记指示了用户希望 AI 重点关注/修改的区域 @@ -6738,6 +6917,10 @@ export function ImageNode({ hasAnnotations: true, // 🆕 标记为已标注 originalSrc: data.originalSrc || data.src, // 保留原始图片 prompt: data.prompt ? `${data.prompt} (已标注)` : "已标注图片", + metadata: + Object.keys(visualAnnotationMetadata.metadata).length > 0 + ? visualAnnotationMetadata.metadata + : undefined, }, }); @@ -8161,6 +8344,64 @@ export function ImageNode({ /> ) ) : null} + {canvasAgentAnnotations.length > 0 && + !boxCutoutMode && + !localEditMode && + !cropPanelOpen && + !isThisNodePicking ? ( +
+ {boxCutoutMetrics + ? canvasAgentAnnotations.map((annotation) => { + const boundsStyle = getCanvasAnnotationBoundsStyle( + annotation, + boxCutoutMetrics + ); + if (!boundsStyle) return null; + const color = annotation.color || "#f59e0b"; + return ( +
+ ); + }) + : null} +
+
annotation.text) + .filter(Boolean) + .join(" / ")} + > + + 批注 + {canvasAgentAnnotations.length} +
+ {isSelected + ? visibleCanvasAgentAnnotations + .filter((annotation) => annotation.text) + .map((annotation) => ( +
+ {annotation.text} +
+ )) + : null} +
+
+ ) : null} )} @@ -8495,7 +8736,9 @@ export function ImageNode({
); })} - {typeof document !== "undefined" && localEditPromptPortalFrame + {(localEditOperationMode as CanvasLocalEditOperationMode) !== "scale_adjust" && + typeof document !== "undefined" && + localEditPromptPortalFrame ? createPortal(
- {isEditing && localEditOperationMode !== "scale_adjust" ? ( + {isEditing ? ( { event.preventDefault(); @@ -8655,36 +8896,12 @@ export function ImageNode({ event.preventDefault(); event.nativeEvent.stopImmediatePropagation?.(); event.stopPropagation(); - if (localEditOperationMode !== "scale_adjust") { - setEditingLocalEditRegionId(region.id); - } + setEditingLocalEditRegionId(region.id); }} - title={ - localEditOperationMode === "scale_adjust" - ? region.targetRect - ? getCanvasScaleAdjustLabel( - deriveCanvasScalePercentFromRects( - region.rect, - region.targetRect - ) - ) - : "请框选青色目标框" - : region.prompt - } + title={region.prompt} type="button" > - {localEditOperationMode === "scale_adjust" - ? `${region.index}. ${ - region.targetRect - ? getCanvasScaleAdjustLabel( - deriveCanvasScalePercentFromRects( - region.rect, - region.targetRect - ) - ) - : "请框选目标框" - }` - : `${region.index}. ${region.prompt}`} + {`${region.index}. ${region.prompt}`} )}