From de457b5dc325965f647c7dea090bc251661c2538 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 23 Jun 2026 14:03:02 +0800 Subject: [PATCH] feat: pass canvas slot context to agent planner --- app/(chat)/api/chat/route.ts | 6 ++- lib/ai/auto-agent-graph.ts | 48 +++++++++++++++++++++++- lib/ai/canvas-agent-routing.ts | 23 ++++++++++++ lib/ai/skill-router.ts | 12 +----- lib/ai/specialists/index.ts | 2 + tests/routes/canvas-chat-helpers.test.ts | 25 ++++++++++++ 6 files changed, 103 insertions(+), 13 deletions(-) create mode 100644 lib/ai/canvas-agent-routing.ts diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 1386281e3..18eca6087 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -1480,8 +1480,12 @@ export async function POST(request: Request) { mediaSource: requestResolvedMediaInput.source, hasCanvasSelection: Boolean(effectiveContextPayload?.selectedNodeIds?.length) || - requestResolvedMediaInput.selectedImageUrls.length > 0, + requestResolvedMediaInput.selectedImageUrls.length > 0 || + getSelectedSlots(requestCanvasAgentContext).length > 0, selectedImageCount: requestResolvedMediaInput.selectedImageUrls.length, + canvasAgentContext: requestCanvasAgentContext, + selectedSlotCount: requestCanvasAgentContext?.slots.length || 0, + hasSelectedAgentSlot: getSelectedSlots(requestCanvasAgentContext).length > 0, historyImageCount: requestResolvedMediaInput.historyAssistantImageUrls.length + requestResolvedMediaInput.historyUserImageUrls.length, diff --git a/lib/ai/auto-agent-graph.ts b/lib/ai/auto-agent-graph.ts index 13d875138..a2d367ede 100644 --- a/lib/ai/auto-agent-graph.ts +++ b/lib/ai/auto-agent-graph.ts @@ -5,6 +5,7 @@ import { getCompatibleTextGenerationTemperature } from "@/lib/ai/model-compat"; import type { AnalysisResult } from "@/lib/ai/smart-analyzer"; import type { RouteResult } from "@/lib/ai/model-router"; import type { AgentSkillRouteResult } from "@/lib/ai/skill-router"; +import type { CanvasAgentContext } from "@/types/canvas/agent-protocol"; import { getEnabledAgentToolDefinitionsForContext, type AgentProjectToolDefinition, @@ -144,6 +145,9 @@ export interface AutoAgentInput { mediaSource?: "current_message" | "canvas_selection" | "history" | "none"; hasCanvasSelection?: boolean; selectedImageCount?: number; + canvasAgentContext?: CanvasAgentContext | null; + selectedSlotCount?: number; + hasSelectedAgentSlot?: boolean; historyImageCount?: number; currentUploadImageCount?: number; historyMessages?: HistoryMessage[]; @@ -175,6 +179,9 @@ const AutoAgentState = Annotation.Root({ mediaSource: Annotation<"current_message" | "canvas_selection" | "history" | "none">, hasCanvasSelection: Annotation, selectedImageCount: Annotation, + canvasAgentContext: Annotation, + selectedSlotCount: Annotation, + hasSelectedAgentSlot: Annotation, historyImageCount: Annotation, currentUploadImageCount: Annotation, historyMessages: Annotation, @@ -1431,6 +1438,8 @@ async function refinePlanWithSkillRoute(params: { targetResolutionMode: AgentTargetResolutionMode; mediaSource: "current_message" | "canvas_selection" | "history" | "none"; selectedImageCount: number; + selectedSlotCount: number; + hasSelectedAgentSlot: boolean; threadState: AgentThreadState | null; }): Promise { if (!params.skillRoute) return params.plan; @@ -1448,6 +1457,8 @@ async function refinePlanWithSkillRoute(params: { targetResolutionMode: params.targetResolutionMode, mediaSource: params.mediaSource, selectedImageCount: params.selectedImageCount, + selectedSlotCount: params.selectedSlotCount, + hasSelectedAgentSlot: params.hasSelectedAgentSlot, threadState: params.threadState, }); } @@ -1500,6 +1511,8 @@ async function buildExecutionPlanWithAgent( targetResolutionMode: state.targetResolutionMode, mediaSource: state.mediaSource, selectedImageCount: state.selectedImageCount, + selectedSlotCount: state.selectedSlotCount, + hasSelectedAgentSlot: state.hasSelectedAgentSlot, threadState: state.threadState, }); console.log("[Auto Agent] Lightweight planner used:", { @@ -1604,6 +1617,26 @@ async function buildExecutionPlanWithAgent( `当前上传图数量：${state.currentUploadImageCount}`, `历史图数量：${state.historyImageCount}`, `画布选中图数量：${state.selectedImageCount}`, + `画布选中 Agent 槽位数量：${state.selectedSlotCount}`, + state.hasSelectedAgentSlot + ? "画布中当前选中的空 Agent Slot 是有效生成目标；如果用户要求在这里生成/填充，不要因为没有图片而澄清。" + : "", + state.canvasAgentContext?.slots?.length + ? `画布槽位摘要：${state.canvasAgentContext.slots + .slice(0, 6) + .map((slot) => + [ + slot.nodeId, + slot.slotType, + slot.label || "未命名槽位", + slot.isSelected ? "selected" : "", + slot.fillPolicy, + ] + .filter(Boolean) + .join("/") + ) + .join("；")}` + : "", `基础分析：intent=${analysis.intent}; subIntent=${analysis.subIntent || "none"}; confidence=${analysis.confidence}`, skillRoute ? `专家路由：${skillRoute.primary.specialistId}｜原因：${skillRoute.reason}` @@ -1638,6 +1671,8 @@ async function buildExecutionPlanWithAgent( targetResolutionMode: state.targetResolutionMode, mediaSource: state.mediaSource, selectedImageCount: state.selectedImageCount, + selectedSlotCount: state.selectedSlotCount, + hasSelectedAgentSlot: state.hasSelectedAgentSlot, threadState: state.threadState, }); console.log( @@ -2115,9 +2150,13 @@ const shouldBypassPlanner = (state: AutoAgentStateType) => { const planNode = async (state: AutoAgentStateType) => { if (!state.analysis) return {}; + const hasCanvasOperationTarget = + state.hasCanvasSelection || + state.selectedImageCount > 0 || + state.hasSelectedAgentSlot; const baseAvailableTools = await getEnabledAgentToolDefinitionsForContext({ hasImage: state.hasImage, - hasCanvasSelection: state.hasCanvasSelection || state.selectedImageCount > 0, + hasCanvasSelection: hasCanvasOperationTarget, hasHistoryImage: state.historyImageCount > 0 || state.imageFromHistory, hasCurrentUpload: state.currentUploadImageCount > 0, }); @@ -2126,10 +2165,12 @@ const planNode = async (state: AutoAgentStateType) => { text: state.text, analysis: state.analysis, hasImage: state.hasImage, - hasCanvasSelection: state.hasCanvasSelection || state.selectedImageCount > 0, + hasCanvasSelection: hasCanvasOperationTarget, targetResolutionMode: state.targetResolutionMode, mediaSource: state.mediaSource, selectedImageCount: state.selectedImageCount, + selectedSlotCount: state.selectedSlotCount, + hasSelectedAgentSlot: state.hasSelectedAgentSlot, threadState: state.threadState, }); const availableTools = filterToolsForSkillRoute(baseAvailableTools, skillRoute); @@ -2343,6 +2384,9 @@ export async function runAutoAgentGraph(params: AutoAgentInput): Promise 0 || + (context.selectedSlotCount || 0) > 0 || + Boolean(context.hasSelectedAgentSlot) || + context.mediaSource === "canvas_selection" || + context.targetResolutionMode === "explicit_selection" + ); +} diff --git a/lib/ai/skill-router.ts b/lib/ai/skill-router.ts index 218b1e773..69f9e8dfe 100644 --- a/lib/ai/skill-router.ts +++ b/lib/ai/skill-router.ts @@ -6,6 +6,7 @@ import type { AgentSpecialistContext, AgentSpecialistRoute, } from "@/lib/ai/specialists"; +import { hasCanvasAgentRoutingContext } from "@/lib/ai/canvas-agent-routing"; import { buildImageEditSpecialistRoute } from "@/lib/ai/image-edit-subagent"; import { buildImageGenerationSpecialistRoute } from "@/lib/ai/image-generation-subagent"; import { buildStoryboardSpecialistRoute } from "@/lib/video-agent/storyboard-subagent"; @@ -34,15 +35,6 @@ function isPureGreeting(text: string) { ); } -function hasCanvasContext(context: AgentSpecialistContext) { - return ( - context.hasCanvasSelection || - (context.selectedImageCount || 0) > 0 || - context.mediaSource === "canvas_selection" || - context.targetResolutionMode === "explicit_selection" - ); -} - function mentionsCanvas(text: string) { return /画布|发布到画布|同步到画布|发送到画布|放到画布/.test( String(text || "").toLowerCase(), @@ -220,7 +212,7 @@ export function routeAgentSkill( const overlays = primary.domain === "conversation" ? [] - : hasCanvasContext(context) || mentionsCanvas(context.text) + : hasCanvasAgentRoutingContext(context) || mentionsCanvas(context.text) ? [canvasOverlay] : []; diff --git a/lib/ai/specialists/index.ts b/lib/ai/specialists/index.ts index dfd77672c..fc3c5e649 100644 --- a/lib/ai/specialists/index.ts +++ b/lib/ai/specialists/index.ts @@ -54,6 +54,8 @@ export interface AgentSpecialistContext { targetResolutionMode: AgentTargetResolutionMode; mediaSource?: "current_message" | "canvas_selection" | "history" | "none"; selectedImageCount?: number; + selectedSlotCount?: number; + hasSelectedAgentSlot?: boolean; threadState?: AgentThreadState | null; } diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index ae8185464..799345ab0 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -26,6 +26,7 @@ import { getCanvasToolbarItem, } from "@/lib/canvas-toolbar-config"; import { shouldBypassCanvasAgentMissingTargetClarification } from "@/lib/ai/runtime/canvas-agent-clarification"; +import { hasCanvasAgentRoutingContext } from "@/lib/ai/canvas-agent-routing"; import { CANVAS_LOCAL_EDIT_REGION_ANNOTATION_SOURCE, CANVAS_NODE_NOTE_ANNOTATION_SOURCE, @@ -227,6 +228,30 @@ test.describe("canvas chat pure helpers", () => { ).toBe(true); }); + test("treats selected agent slots as canvas routing context", () => { + expect( + hasCanvasAgentRoutingContext({ + hasCanvasSelection: false, + targetResolutionMode: "none", + mediaSource: "none", + selectedImageCount: 0, + selectedSlotCount: 1, + hasSelectedAgentSlot: true, + }) + ).toBe(true); + + expect( + hasCanvasAgentRoutingContext({ + hasCanvasSelection: false, + targetResolutionMode: "none", + mediaSource: "none", + selectedImageCount: 0, + selectedSlotCount: 0, + hasSelectedAgentSlot: false, + }) + ).toBe(false); + }); + test("normalizes model-planned empty slot creation", () => { const outcome = normalizeCanvasAgentModelPlannerDecision({ decision: {