From bcea56c0d9e39907a19a11fb05e3a89f07a77d9d Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 22 Jun 2026 22:14:14 +0800 Subject: [PATCH] fix: bypass clarification for selected canvas slots --- app/(chat)/api/chat/route.ts | 22 ++++++ lib/ai/runtime/canvas-agent-clarification.ts | 67 ++++++++++++++++ .../canvas-agent-clarification-gate.ts | 12 ++- tests/routes/canvas-chat-helpers.test.ts | 79 +++++++++++++++++++ 4 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 lib/ai/runtime/canvas-agent-clarification.ts diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index f907756e8..023bc43dc 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -164,6 +164,7 @@ import { resolveClarificationBypass, resolveClarificationMessage, } from "@/lib/ai/runtime/clarification-interruption"; +import { shouldBypassCanvasAgentMissingTargetClarification } from "@/lib/ai/runtime/canvas-agent-clarification"; import { buildAgentToolImagesMarkdown, buildAgentToolVideoMarkdown, @@ -4625,6 +4626,27 @@ export async function POST(request: Request) { return createSseResponse(promptOnlyStream); } + if (shouldRunAgentConversation && smartAutoNeedsClarification) { + const canvasAgentSlotClarificationBypass = + shouldBypassCanvasAgentMissingTargetClarification({ + text: requestTextContent, + contextPayload: effectiveContextPayload, + executionMode: smartAutoExecution?.mode, + }); + if (canvasAgentSlotClarificationBypass.shouldBypass) { + smartAutoNeedsClarification = false; + smartAutoClarificationQuestion = undefined; + smartAutoFriendlyMessage = "收到，我会把结果填入当前选中的画布槽位。"; + chatApiLogger.info( + "[Smart Auto Mode] ✅ Skip clarification for selected canvas Agent slot", + { + reason: canvasAgentSlotClarificationBypass.reason, + executionMode: smartAutoExecution?.mode, + } + ); + } + } + if (shouldRunAgentConversation && smartAutoNeedsClarification) { const clarificationBypass = resolveClarificationBypass({ text: message.parts diff --git a/lib/ai/runtime/canvas-agent-clarification.ts b/lib/ai/runtime/canvas-agent-clarification.ts new file mode 100644 index 000000000..0e13fe4bf --- /dev/null +++ b/lib/ai/runtime/canvas-agent-clarification.ts @@ -0,0 +1,67 @@ +import type { ContextPayload } from "@/lib/ai/context-memory/types"; + +type CanvasAgentExecutionMode = "edit_image" | "generate_image" | "generate_video"; + +const CANVAS_AGENT_SLOT_GENERATION_REGEX = + /(生成|生图|出图|画一张|来一张|做一张|做图|做视频|视频生成|生成视频|填充|放到|放进|在这里|这个空框|这个槽位|槽位|slot)/i; + +function isCanvasSource(contextPayload?: ContextPayload | null) { + return ( + contextPayload?.source === "canvas" || contextPayload?.source === "realtime-canvas" + ); +} + +function hasSelectedCanvasAgentSlot(contextPayload?: ContextPayload | null) { + const context = contextPayload?.canvasAgentContext; + if (!context) return false; + const selectedNodeIds = new Set( + (context.selectedNodeIds || contextPayload?.selectedNodeIds || []) + .map((id) => String(id || "").trim()) + .filter(Boolean) + ); + + return (context.slots || []).some((slot) => { + const slotId = String(slot.nodeId || "").trim(); + return Boolean(slotId && (slot.isSelected || selectedNodeIds.has(slotId))); + }); +} + +export function shouldBypassCanvasAgentMissingTargetClarification(params: { + text: string; + contextPayload?: ContextPayload | null; + executionMode?: string | null; +}) { + const executionMode = String(params.executionMode || "") as CanvasAgentExecutionMode; + if (executionMode !== "generate_image" && executionMode !== "generate_video") { + return { + shouldBypass: false, + reason: undefined, + }; + } + + if (!isCanvasSource(params.contextPayload)) { + return { + shouldBypass: false, + reason: undefined, + }; + } + + if (!hasSelectedCanvasAgentSlot(params.contextPayload)) { + return { + shouldBypass: false, + reason: undefined, + }; + } + + if (!CANVAS_AGENT_SLOT_GENERATION_REGEX.test(String(params.text || ""))) { + return { + shouldBypass: false, + reason: undefined, + }; + } + + return { + shouldBypass: true, + reason: "selected_canvas_agent_slot_generation_intent", + }; +} diff --git a/lib/ai/runtime/middlewares/canvas-agent-clarification-gate.ts b/lib/ai/runtime/middlewares/canvas-agent-clarification-gate.ts index 7264485a3..232480c0a 100644 --- a/lib/ai/runtime/middlewares/canvas-agent-clarification-gate.ts +++ b/lib/ai/runtime/middlewares/canvas-agent-clarification-gate.ts @@ -9,6 +9,7 @@ import type { } from "@/lib/ai/context-memory/types"; import type { IntentArbiterMode } from "@/lib/ai/runtime/intent-arbiter"; import type { TaskBoundaryDecision } from "@/lib/ai/runtime/task-boundary-classifier"; +import { shouldBypassCanvasAgentMissingTargetClarification } from "@/lib/ai/runtime/canvas-agent-clarification"; export interface CanvasAgentClarificationGateDecision { shouldClarify: boolean; @@ -205,6 +206,12 @@ export function resolveCanvasAgentClarificationGate(params: { const targetResolution = params.targetResolution; const selectedRefs = mediaInput?.selectedImageRefs || []; const candidateRefs = targetResolution?.candidateRefs || mediaInput?.imageRefs || []; + const missingTargetClarificationBypass = + shouldBypassCanvasAgentMissingTargetClarification({ + text, + contextPayload: params.contextPayload, + executionMode: params.executionMode, + }); if ( params.executionMode === "edit_image" && @@ -219,7 +226,10 @@ export function resolveCanvasAgentClarificationGate(params: { } satisfies CanvasAgentClarificationGateDecision; } - if (targetResolution?.needsClarification) { + if ( + targetResolution?.needsClarification && + !missingTargetClarificationBypass.shouldBypass + ) { return { shouldClarify: true, clarificationMessage: buildMissingTargetClarificationMessage({ diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index 721ec9422..d3918b315 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -20,6 +20,7 @@ import { buildCanvasAgentAnnotationEditPrompt, hasCanvasAgentAnnotationEditIntent, } from "@/lib/canvas/agent-annotation-prompt"; +import { shouldBypassCanvasAgentMissingTargetClarification } from "@/lib/ai/runtime/canvas-agent-clarification"; import { CANVAS_LOCAL_EDIT_REGION_ANNOTATION_SOURCE, CANVAS_NODE_NOTE_ANNOTATION_SOURCE, @@ -1545,6 +1546,84 @@ test.describe("canvas chat pure helpers", () => { expect(context?.focusPolicy).toBe("slot"); }); + test("bypasses missing-target clarification for selected agent slot generation", () => { + const slotOnlyContext = { + source: "canvas" as const, + projectId: "11111111-1111-4111-8111-111111111111", + selectedNodeIds: ["slot-1"], + canvasImageNodes: [], + canvasAgentContext: { + version: 1 as const, + source: "canvas" as const, + projectId: "11111111-1111-4111-8111-111111111111", + selectedNodeIds: ["slot-1"], + selectedNodes: [ + { + nodeId: "slot-1", + type: "agent-slot" as const, + x: 120, + y: 80, + width: 320, + height: 240, + isSelected: true, + }, + ], + candidateNodes: [], + slots: [ + { + nodeId: "slot-1", + slotType: "image" as const, + x: 120, + y: 80, + width: 320, + height: 240, + aspectRatio: "4:3", + isSelected: true, + fillPolicy: "replace-empty" as const, + }, + ], + annotations: [], + viewport: { x: 0, y: 0, zoom: 1 }, + focusPolicy: "slot" as const, + }, + }; + + expect( + shouldBypassCanvasAgentMissingTargetClarification({ + text: "在这里生成一张电商主图", + contextPayload: slotOnlyContext, + executionMode: "generate_image", + }) + ).toMatchObject({ + shouldBypass: true, + reason: "selected_canvas_agent_slot_generation_intent", + }); + + expect( + shouldBypassCanvasAgentMissingTargetClarification({ + text: "把这张图背景换白", + contextPayload: slotOnlyContext, + executionMode: "edit_image", + }).shouldBypass + ).toBe(false); + + expect( + shouldBypassCanvasAgentMissingTargetClarification({ + text: "在这里生成一张电商主图", + contextPayload: { + ...slotOnlyContext, + selectedNodeIds: [], + canvasAgentContext: { + ...slotOnlyContext.canvasAgentContext, + selectedNodeIds: [], + slots: [{ ...slotOnlyContext.canvasAgentContext.slots[0], isSelected: false }], + }, + }, + executionMode: "generate_image", + }).shouldBypass + ).toBe(false); + }); + test("keeps view-only canvas agent context without nodes or media", () => { const context = normalizeCanvasAgentContext({ source: "canvas",