From d34e66da25b1e4f9088e0c851c507c4a0f73a58c Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 5 Apr 2026 01:30:40 +0800 Subject: [PATCH] fix: add missing canvas clarification gate --- .../canvas-agent-clarification-gate.ts | 280 ++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 lib/ai/runtime/middlewares/canvas-agent-clarification-gate.ts diff --git a/lib/ai/runtime/middlewares/canvas-agent-clarification-gate.ts b/lib/ai/runtime/middlewares/canvas-agent-clarification-gate.ts new file mode 100644 index 000000000..7264485a3 --- /dev/null +++ b/lib/ai/runtime/middlewares/canvas-agent-clarification-gate.ts @@ -0,0 +1,280 @@ +import "server-only"; + +import type { ConversationActionKind } from "@/lib/ai/conversation-action-resolver"; +import type { + ContextPayload, + ResolvedChatMediaInput, + ResolvedImageRef, + ResolvedTargetAssets, +} from "@/lib/ai/context-memory/types"; +import type { IntentArbiterMode } from "@/lib/ai/runtime/intent-arbiter"; +import type { TaskBoundaryDecision } from "@/lib/ai/runtime/task-boundary-classifier"; + +export interface CanvasAgentClarificationGateDecision { + shouldClarify: boolean; + clarificationMessage?: string; + reason?: string; +} + +const BATCH_EDIT_REGEX = + /(全部|都|一起|批量|这几张|这些图|所有|统一处理|都改|全部改|一起改|一起处理)/i; +const EXPLICIT_TARGET_HINT_REGEX = + /(第\s*[一二三四五六七八九十两\d]+\s*(?:张|个|幅|图)|左边|左侧|左上|左下|右边|右侧|右上|右下|上面|上方|下面|下方|中间|居中|中央|选中的|当前这张|原图|上传图|上一张|上一版|结果图|刚生成|最新生成)/i; +const EXPLICIT_CONTINUATION_TARGET_REGEX = + /(继续|上一张|上一版|结果图|刚生成|最新生成|原图|上传图|选中的|当前这张)/i; +const ACTIONABLE_EDIT_REGEX = + /(改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按|保留|统一|背景|清晰|风格|亮一点|暗一点|白底|透明底|去背|扩图|放大|缩小|裁剪|调色)/i; +const ACTIONABLE_GENERATION_REGEX = + /(生图|生成|画一张|出图|来一张|再来一张|做一张|做图|做视频|视频生成|生成视频|动起来)/i; +const ANALYSIS_ONLY_REGEX = /(分析|看图|看看|识别|描述|总结|解释|判断|评价)/i; +const VAGUE_EXECUTION_REGEX = + /^(按刚才那个做|按刚才的做|按这个做|按这个来|照这个来|照刚才那个|就这个|这个|来吧|开始吧|搞一下|弄一下|继续吧|就这样|按上一个来|按上一版来|按刚才那个来)$/i; + +function normalizeCompactText(text: string) { + return String(text || "").replace(/\s+/g, "").trim(); +} + +function sortRefsForCanvasOrdering(refs: ResolvedImageRef[]) { + return [...refs].sort((left, right) => { + const leftY = Number(left.y ?? 0); + const rightY = Number(right.y ?? 0); + if (Math.abs(leftY - rightY) > 12) { + return leftY - rightY; + } + return Number(left.x ?? 0) - Number(right.x ?? 0); + }); +} + +function getPositionLabel(ref: ResolvedImageRef, refs: ResolvedImageRef[]): string { + if (refs.length <= 1) return "中间"; + const xs = refs.map((item) => Number(item.x ?? 0)); + const ys = refs.map((item) => Number(item.y ?? 0)); + const minX = Math.min(...xs); + const maxX = Math.max(...xs); + const minY = Math.min(...ys); + const maxY = Math.max(...ys); + const spanX = Math.max(1, maxX - minX); + const spanY = Math.max(1, maxY - minY); + const relX = (Number(ref.x ?? 0) - minX) / spanX; + const relY = (Number(ref.y ?? 0) - minY) / spanY; + + const horizontal = relX <= 0.33 ? "左" : relX >= 0.66 ? "右" : "中"; + const vertical = relY <= 0.33 ? "上" : relY >= 0.66 ? "下" : "中"; + + if (horizontal === "中" && vertical === "中") return "中间"; + return `${vertical}${horizontal}`; +} + +function buildRefPreviewList(refs: ResolvedImageRef[]) { + const orderedRefs = sortRefsForCanvasOrdering(refs).slice(0, 4); + return orderedRefs.map((ref, index) => { + const position = getPositionLabel(ref, orderedRefs); + const title = String(ref.title || "").trim(); + return `- 第${index + 1}张（${position}）${title ? `：${title}` : ""}`; + }); +} + +function buildMultiSelectionClarificationMessage(selectedRefs: ResolvedImageRef[]) { + const previewLines = buildRefPreviewList(selectedRefs); + return [ + `你当前选中了 ${selectedRefs.length} 张图，我还不能确定这一步是只处理其中一张，还是批量处理全部。`, + previewLines.length > 0 ? "当前选中候选：" : "", + ...previewLines, + "请直接回复：", + "- 处理全部", + "- 只处理第2张 / 左边那张", + "", + "也可以先在画布里只保留一张选中，再让我继续。", + ] + .filter(Boolean) + .join("\n"); +} + +function buildMissingTargetClarificationMessage(params: { + selectedRefs: ResolvedImageRef[]; + candidateRefs: ResolvedImageRef[]; +}) { + if (params.selectedRefs.length > 1) { + return buildMultiSelectionClarificationMessage(params.selectedRefs); + } + + if (params.candidateRefs.length > 1) { + const previewLines = buildRefPreviewList(params.candidateRefs); + return [ + "我这次还没锁定要处理哪张图。", + `当前画布里有 ${params.candidateRefs.length} 张候选图：`, + ...previewLines, + "", + "请直接回复“处理第2张 / 左边那张 / 选中的这张 / 上一版结果图 / 原图”，或者先在画布里选中一张再继续。", + ].join("\n"); + } + + return [ + "我还没拿到可处理的目标图。", + "你可以先上传图片，或者先在画布里选中一张图，再告诉我要怎么处理。", + ].join("\n"); +} + +function buildLowConfidenceTargetClarificationMessage(params: { + candidateRefs: ResolvedImageRef[]; + targetResolution: ResolvedTargetAssets; +}) { + const previewLines = buildRefPreviewList(params.candidateRefs); + return [ + "这一步我能继续执行，但当前目标图只是在低置信度下被系统猜中的，我不想替你处理错图。", + previewLines.length > 0 ? `当前画布里可参考的候选图有 ${params.candidateRefs.length} 张：` : "", + ...previewLines, + "", + `系统当前偏向的是「${params.targetResolution.reason}」这一路径，但我建议你先明确回复“第2张 / 左边那张 / 上一版结果图 / 原图”。`, + ] + .filter(Boolean) + .join("\n"); +} + +function buildActionClarificationMessage(mode: IntentArbiterMode) { + if (mode === "generate_image") { + return [ + "我已经锁定目标图了，但还不清楚你这一步想让我怎么做。", + "你可以直接回复类似：", + "- 基于这张再生成 3 张", + "- 按这个主体重新生成一版海报", + "- 保留主体，换成白底产品图", + "- 先分析这张图", + ].join("\n"); + } + + if (mode === "generate_video") { + return [ + "我已经锁定参考图了，但这一步的视频目标还不够明确。", + "你可以直接回复类似：", + "- 让产品缓慢旋转 5 秒", + "- 做成首尾帧视频", + "- 先按这张图规划分镜，不直接生成", + ].join("\n"); + } + + return [ + "我已经锁定目标图了，但还不清楚你这一步想让我做什么。", + "你可以直接回复类似：", + "- 把背景改成白色", + "- 去掉桌上的杯子", + "- 提高清晰度并保留原构图", + "- 先分析这张图", + ].join("\n"); +} + +function isCanvasSource(contextPayload?: ContextPayload | null) { + return ( + contextPayload?.source === "canvas" || contextPayload?.source === "realtime-canvas" + ); +} + +function isExecutionMode(mode?: IntentArbiterMode) { + return mode === "edit_image" || mode === "generate_image" || mode === "generate_video"; +} + +export function resolveCanvasAgentClarificationGate(params: { + text: string; + composerMode?: "agent" | "image" | "video" | "detail"; + contextPayload?: ContextPayload | null; + mediaInput?: ResolvedChatMediaInput | null; + targetResolution?: ResolvedTargetAssets | null; + taskBoundaryDecision: TaskBoundaryDecision; + executionMode?: IntentArbiterMode; + actionKind?: ConversationActionKind; +}) { + if (params.composerMode !== "agent") { + return { shouldClarify: false } satisfies CanvasAgentClarificationGateDecision; + } + + if (!isCanvasSource(params.contextPayload)) { + return { shouldClarify: false } satisfies CanvasAgentClarificationGateDecision; + } + + if (!isExecutionMode(params.executionMode)) { + return { shouldClarify: false } satisfies CanvasAgentClarificationGateDecision; + } + + if (!params.taskBoundaryDecision.shouldAllowExecution) { + return { shouldClarify: false } satisfies CanvasAgentClarificationGateDecision; + } + + const text = String(params.text || "").trim(); + const compactText = normalizeCompactText(text); + const mediaInput = params.mediaInput; + const targetResolution = params.targetResolution; + const selectedRefs = mediaInput?.selectedImageRefs || []; + const candidateRefs = targetResolution?.candidateRefs || mediaInput?.imageRefs || []; + + if ( + params.executionMode === "edit_image" && + selectedRefs.length > 1 && + !BATCH_EDIT_REGEX.test(compactText) && + !EXPLICIT_TARGET_HINT_REGEX.test(compactText) + ) { + return { + shouldClarify: true, + clarificationMessage: buildMultiSelectionClarificationMessage(selectedRefs), + reason: "multiple_selected_canvas_targets_without_scope", + } satisfies CanvasAgentClarificationGateDecision; + } + + if (targetResolution?.needsClarification) { + return { + shouldClarify: true, + clarificationMessage: buildMissingTargetClarificationMessage({ + selectedRefs, + candidateRefs, + }), + reason: "target_resolution_requires_clarification", + } satisfies CanvasAgentClarificationGateDecision; + } + + const shouldClarifyLowConfidenceCanvasTarget = + params.executionMode === "edit_image" && + selectedRefs.length === 0 && + (targetResolution?.resolutionMode === "working_memory" || + targetResolution?.resolutionMode === "latest_result") && + Number(targetResolution?.confidence || 0) < 85 && + candidateRefs.length > 1 && + !EXPLICIT_TARGET_HINT_REGEX.test(compactText) && + !EXPLICIT_CONTINUATION_TARGET_REGEX.test(compactText); + + if (shouldClarifyLowConfidenceCanvasTarget && targetResolution) { + return { + shouldClarify: true, + clarificationMessage: buildLowConfidenceTargetClarificationMessage({ + candidateRefs, + targetResolution, + }), + reason: "low_confidence_canvas_target_guess", + } satisfies CanvasAgentClarificationGateDecision; + } + + const lacksExplicitAction = + !ACTIONABLE_EDIT_REGEX.test(compactText) && + !ACTIONABLE_GENERATION_REGEX.test(compactText) && + !ANALYSIS_ONLY_REGEX.test(compactText); + const looksLikeVagueExecution = + VAGUE_EXECUTION_REGEX.test(compactText) || + (compactText.length <= 8 && + /^(这个|就这个|按这个|来吧|开始吧|继续吧|就这样)$/.test(compactText)); + + if ( + targetResolution && + targetResolution.targetRefs.length > 0 && + params.actionKind === "default" && + lacksExplicitAction && + looksLikeVagueExecution + ) { + return { + shouldClarify: true, + clarificationMessage: buildActionClarificationMessage(params.executionMode), + reason: "target_locked_but_action_is_too_vague", + } satisfies CanvasAgentClarificationGateDecision; + } + + return { + shouldClarify: false, + } satisfies CanvasAgentClarificationGateDecision; +}