fix: add missing canvas clarification gate
This commit is contained in:
Your Name
parent
3a087d3392
commit
d34e66da25
280
lib/ai/runtime/middlewares/canvas-agent-clarification-gate.ts
Normal file
280
lib/ai/runtime/middlewares/canvas-agent-clarification-gate.ts
Normal file
@ -0,0 +1,280 @@
|
||||
import "server-only";
|
||||
|
||||
import type { ConversationActionKind } from "@/lib/ai/conversation-action-resolver";
|
||||
import type {
|
||||
ContextPayload,
|
||||
ResolvedChatMediaInput,
|
||||
ResolvedImageRef,
|
||||
ResolvedTargetAssets,
|
||||
} from "@/lib/ai/context-memory/types";
|
||||
import type { IntentArbiterMode } from "@/lib/ai/runtime/intent-arbiter";
|
||||
import type { TaskBoundaryDecision } from "@/lib/ai/runtime/task-boundary-classifier";
|
||||
|
||||
export interface CanvasAgentClarificationGateDecision {
|
||||
shouldClarify: boolean;
|
||||
clarificationMessage?: string;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
const BATCH_EDIT_REGEX =
|
||||
/(全部|都|一起|批量|这几张|这些图|所有|统一处理|都改|全部改|一起改|一起处理)/i;
|
||||
const EXPLICIT_TARGET_HINT_REGEX =
|
||||
/(第\s*[一二三四五六七八九十两\d]+\s*(?:张|个|幅|图)|左边|左侧|左上|左下|右边|右侧|右上|右下|上面|上方|下面|下方|中间|居中|中央|选中的|当前这张|原图|上传图|上一张|上一版|结果图|刚生成|最新生成)/i;
|
||||
const EXPLICIT_CONTINUATION_TARGET_REGEX =
|
||||
/(继续|上一张|上一版|结果图|刚生成|最新生成|原图|上传图|选中的|当前这张)/i;
|
||||
const ACTIONABLE_EDIT_REGEX =
|
||||
/(改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按|保留|统一|背景|清晰|风格|亮一点|暗一点|白底|透明底|去背|扩图|放大|缩小|裁剪|调色)/i;
|
||||
const ACTIONABLE_GENERATION_REGEX =
|
||||
/(生图|生成|画一张|出图|来一张|再来一张|做一张|做图|做视频|视频生成|生成视频|动起来)/i;
|
||||
const ANALYSIS_ONLY_REGEX = /(分析|看图|看看|识别|描述|总结|解释|判断|评价)/i;
|
||||
const VAGUE_EXECUTION_REGEX =
|
||||
/^(按刚才那个做|按刚才的做|按这个做|按这个来|照这个来|照刚才那个|就这个|这个|来吧|开始吧|搞一下|弄一下|继续吧|就这样|按上一个来|按上一版来|按刚才那个来)$/i;
|
||||
|
||||
function normalizeCompactText(text: string) {
|
||||
return String(text || "").replace(/\s+/g, "").trim();
|
||||
}
|
||||
|
||||
function sortRefsForCanvasOrdering(refs: ResolvedImageRef[]) {
|
||||
return [...refs].sort((left, right) => {
|
||||
const leftY = Number(left.y ?? 0);
|
||||
const rightY = Number(right.y ?? 0);
|
||||
if (Math.abs(leftY - rightY) > 12) {
|
||||
return leftY - rightY;
|
||||
}
|
||||
return Number(left.x ?? 0) - Number(right.x ?? 0);
|
||||
});
|
||||
}
|
||||
|
||||
function getPositionLabel(ref: ResolvedImageRef, refs: ResolvedImageRef[]): string {
|
||||
if (refs.length <= 1) return "中间";
|
||||
const xs = refs.map((item) => Number(item.x ?? 0));
|
||||
const ys = refs.map((item) => Number(item.y ?? 0));
|
||||
const minX = Math.min(...xs);
|
||||
const maxX = Math.max(...xs);
|
||||
const minY = Math.min(...ys);
|
||||
const maxY = Math.max(...ys);
|
||||
const spanX = Math.max(1, maxX - minX);
|
||||
const spanY = Math.max(1, maxY - minY);
|
||||
const relX = (Number(ref.x ?? 0) - minX) / spanX;
|
||||
const relY = (Number(ref.y ?? 0) - minY) / spanY;
|
||||
|
||||
const horizontal = relX <= 0.33 ? "左" : relX >= 0.66 ? "右" : "中";
|
||||
const vertical = relY <= 0.33 ? "上" : relY >= 0.66 ? "下" : "中";
|
||||
|
||||
if (horizontal === "中" && vertical === "中") return "中间";
|
||||
return `${vertical}${horizontal}`;
|
||||
}
|
||||
|
||||
function buildRefPreviewList(refs: ResolvedImageRef[]) {
|
||||
const orderedRefs = sortRefsForCanvasOrdering(refs).slice(0, 4);
|
||||
return orderedRefs.map((ref, index) => {
|
||||
const position = getPositionLabel(ref, orderedRefs);
|
||||
const title = String(ref.title || "").trim();
|
||||
return `- 第${index + 1}张（${position}）${title ? `：${title}` : ""}`;
|
||||
});
|
||||
}
|
||||
|
||||
function buildMultiSelectionClarificationMessage(selectedRefs: ResolvedImageRef[]) {
|
||||
const previewLines = buildRefPreviewList(selectedRefs);
|
||||
return [
|
||||
`你当前选中了 ${selectedRefs.length} 张图，我还不能确定这一步是只处理其中一张，还是批量处理全部。`,
|
||||
previewLines.length > 0 ? "当前选中候选：" : "",
|
||||
...previewLines,
|
||||
"请直接回复：",
|
||||
"- 处理全部",
|
||||
"- 只处理第2张 / 左边那张",
|
||||
"",
|
||||
"也可以先在画布里只保留一张选中，再让我继续。",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function buildMissingTargetClarificationMessage(params: {
|
||||
selectedRefs: ResolvedImageRef[];
|
||||
candidateRefs: ResolvedImageRef[];
|
||||
}) {
|
||||
if (params.selectedRefs.length > 1) {
|
||||
return buildMultiSelectionClarificationMessage(params.selectedRefs);
|
||||
}
|
||||
|
||||
if (params.candidateRefs.length > 1) {
|
||||
const previewLines = buildRefPreviewList(params.candidateRefs);
|
||||
return [
|
||||
"我这次还没锁定要处理哪张图。",
|
||||
`当前画布里有 ${params.candidateRefs.length} 张候选图：`,
|
||||
...previewLines,
|
||||
"",
|
||||
"请直接回复“处理第2张 / 左边那张 / 选中的这张 / 上一版结果图 / 原图”，或者先在画布里选中一张再继续。",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
return [
|
||||
"我还没拿到可处理的目标图。",
|
||||
"你可以先上传图片，或者先在画布里选中一张图，再告诉我要怎么处理。",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function buildLowConfidenceTargetClarificationMessage(params: {
|
||||
candidateRefs: ResolvedImageRef[];
|
||||
targetResolution: ResolvedTargetAssets;
|
||||
}) {
|
||||
const previewLines = buildRefPreviewList(params.candidateRefs);
|
||||
return [
|
||||
"这一步我能继续执行，但当前目标图只是在低置信度下被系统猜中的，我不想替你处理错图。",
|
||||
previewLines.length > 0 ? `当前画布里可参考的候选图有 ${params.candidateRefs.length} 张：` : "",
|
||||
...previewLines,
|
||||
"",
|
||||
`系统当前偏向的是「${params.targetResolution.reason}」这一路径，但我建议你先明确回复“第2张 / 左边那张 / 上一版结果图 / 原图”。`,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function buildActionClarificationMessage(mode: IntentArbiterMode) {
|
||||
if (mode === "generate_image") {
|
||||
return [
|
||||
"我已经锁定目标图了，但还不清楚你这一步想让我怎么做。",
|
||||
"你可以直接回复类似：",
|
||||
"- 基于这张再生成 3 张",
|
||||
"- 按这个主体重新生成一版海报",
|
||||
"- 保留主体，换成白底产品图",
|
||||
"- 先分析这张图",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
if (mode === "generate_video") {
|
||||
return [
|
||||
"我已经锁定参考图了，但这一步的视频目标还不够明确。",
|
||||
"你可以直接回复类似：",
|
||||
"- 让产品缓慢旋转 5 秒",
|
||||
"- 做成首尾帧视频",
|
||||
"- 先按这张图规划分镜，不直接生成",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
return [
|
||||
"我已经锁定目标图了，但还不清楚你这一步想让我做什么。",
|
||||
"你可以直接回复类似：",
|
||||
"- 把背景改成白色",
|
||||
"- 去掉桌上的杯子",
|
||||
"- 提高清晰度并保留原构图",
|
||||
"- 先分析这张图",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function isCanvasSource(contextPayload?: ContextPayload | null) {
|
||||
return (
|
||||
contextPayload?.source === "canvas" || contextPayload?.source === "realtime-canvas"
|
||||
);
|
||||
}
|
||||
|
||||
function isExecutionMode(mode?: IntentArbiterMode) {
|
||||
return mode === "edit_image" || mode === "generate_image" || mode === "generate_video";
|
||||
}
|
||||
|
||||
export function resolveCanvasAgentClarificationGate(params: {
|
||||
text: string;
|
||||
composerMode?: "agent" | "image" | "video" | "detail";
|
||||
contextPayload?: ContextPayload | null;
|
||||
mediaInput?: ResolvedChatMediaInput | null;
|
||||
targetResolution?: ResolvedTargetAssets | null;
|
||||
taskBoundaryDecision: TaskBoundaryDecision;
|
||||
executionMode?: IntentArbiterMode;
|
||||
actionKind?: ConversationActionKind;
|
||||
}) {
|
||||
if (params.composerMode !== "agent") {
|
||||
return { shouldClarify: false } satisfies CanvasAgentClarificationGateDecision;
|
||||
}
|
||||
|
||||
if (!isCanvasSource(params.contextPayload)) {
|
||||
return { shouldClarify: false } satisfies CanvasAgentClarificationGateDecision;
|
||||
}
|
||||
|
||||
if (!isExecutionMode(params.executionMode)) {
|
||||
return { shouldClarify: false } satisfies CanvasAgentClarificationGateDecision;
|
||||
}
|
||||
|
||||
if (!params.taskBoundaryDecision.shouldAllowExecution) {
|
||||
return { shouldClarify: false } satisfies CanvasAgentClarificationGateDecision;
|
||||
}
|
||||
|
||||
const text = String(params.text || "").trim();
|
||||
const compactText = normalizeCompactText(text);
|
||||
const mediaInput = params.mediaInput;
|
||||
const targetResolution = params.targetResolution;
|
||||
const selectedRefs = mediaInput?.selectedImageRefs || [];
|
||||
const candidateRefs = targetResolution?.candidateRefs || mediaInput?.imageRefs || [];
|
||||
|
||||
if (
|
||||
params.executionMode === "edit_image" &&
|
||||
selectedRefs.length > 1 &&
|
||||
!BATCH_EDIT_REGEX.test(compactText) &&
|
||||
!EXPLICIT_TARGET_HINT_REGEX.test(compactText)
|
||||
) {
|
||||
return {
|
||||
shouldClarify: true,
|
||||
clarificationMessage: buildMultiSelectionClarificationMessage(selectedRefs),
|
||||
reason: "multiple_selected_canvas_targets_without_scope",
|
||||
} satisfies CanvasAgentClarificationGateDecision;
|
||||
}
|
||||
|
||||
if (targetResolution?.needsClarification) {
|
||||
return {
|
||||
shouldClarify: true,
|
||||
clarificationMessage: buildMissingTargetClarificationMessage({
|
||||
selectedRefs,
|
||||
candidateRefs,
|
||||
}),
|
||||
reason: "target_resolution_requires_clarification",
|
||||
} satisfies CanvasAgentClarificationGateDecision;
|
||||
}
|
||||
|
||||
const shouldClarifyLowConfidenceCanvasTarget =
|
||||
params.executionMode === "edit_image" &&
|
||||
selectedRefs.length === 0 &&
|
||||
(targetResolution?.resolutionMode === "working_memory" ||
|
||||
targetResolution?.resolutionMode === "latest_result") &&
|
||||
Number(targetResolution?.confidence || 0) < 85 &&
|
||||
candidateRefs.length > 1 &&
|
||||
!EXPLICIT_TARGET_HINT_REGEX.test(compactText) &&
|
||||
!EXPLICIT_CONTINUATION_TARGET_REGEX.test(compactText);
|
||||
|
||||
if (shouldClarifyLowConfidenceCanvasTarget && targetResolution) {
|
||||
return {
|
||||
shouldClarify: true,
|
||||
clarificationMessage: buildLowConfidenceTargetClarificationMessage({
|
||||
candidateRefs,
|
||||
targetResolution,
|
||||
}),
|
||||
reason: "low_confidence_canvas_target_guess",
|
||||
} satisfies CanvasAgentClarificationGateDecision;
|
||||
}
|
||||
|
||||
const lacksExplicitAction =
|
||||
!ACTIONABLE_EDIT_REGEX.test(compactText) &&
|
||||
!ACTIONABLE_GENERATION_REGEX.test(compactText) &&
|
||||
!ANALYSIS_ONLY_REGEX.test(compactText);
|
||||
const looksLikeVagueExecution =
|
||||
VAGUE_EXECUTION_REGEX.test(compactText) ||
|
||||
(compactText.length <= 8 &&
|
||||
/^(这个|就这个|按这个|来吧|开始吧|继续吧|就这样)$/.test(compactText));
|
||||
|
||||
if (
|
||||
targetResolution &&
|
||||
targetResolution.targetRefs.length > 0 &&
|
||||
params.actionKind === "default" &&
|
||||
lacksExplicitAction &&
|
||||
looksLikeVagueExecution
|
||||
) {
|
||||
return {
|
||||
shouldClarify: true,
|
||||
clarificationMessage: buildActionClarificationMessage(params.executionMode),
|
||||
reason: "target_locked_but_action_is_too_vague",
|
||||
} satisfies CanvasAgentClarificationGateDecision;
|
||||
}
|
||||
|
||||
return {
|
||||
shouldClarify: false,
|
||||
} satisfies CanvasAgentClarificationGateDecision;
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user