281 lines
10 KiB
TypeScript
281 lines
10 KiB
TypeScript
import "server-only";
|
||
|
||
import type { ConversationActionKind } from "@/lib/ai/conversation-action-resolver";
|
||
import type {
|
||
ContextPayload,
|
||
ResolvedChatMediaInput,
|
||
ResolvedImageRef,
|
||
ResolvedTargetAssets,
|
||
} from "@/lib/ai/context-memory/types";
|
||
import type { IntentArbiterMode } from "@/lib/ai/runtime/intent-arbiter";
|
||
import type { TaskBoundaryDecision } from "@/lib/ai/runtime/task-boundary-classifier";
|
||
|
||
export interface CanvasAgentClarificationGateDecision {
|
||
shouldClarify: boolean;
|
||
clarificationMessage?: string;
|
||
reason?: string;
|
||
}
|
||
|
||
const BATCH_EDIT_REGEX =
|
||
/(全部|都|一起|批量|这几张|这些图|所有|统一处理|都改|全部改|一起改|一起处理)/i;
|
||
const EXPLICIT_TARGET_HINT_REGEX =
|
||
/(第\s*[一二三四五六七八九十两\d]+\s*(?:张|个|幅|图)|左边|左侧|左上|左下|右边|右侧|右上|右下|上面|上方|下面|下方|中间|居中|中央|选中的|当前这张|原图|上传图|上一张|上一版|结果图|刚生成|最新生成)/i;
|
||
const EXPLICIT_CONTINUATION_TARGET_REGEX =
|
||
/(继续|上一张|上一版|结果图|刚生成|最新生成|原图|上传图|选中的|当前这张)/i;
|
||
const ACTIONABLE_EDIT_REGEX =
|
||
/(改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按|保留|统一|背景|清晰|风格|亮一点|暗一点|白底|透明底|去背|扩图|放大|缩小|裁剪|调色)/i;
|
||
const ACTIONABLE_GENERATION_REGEX =
|
||
/(生图|生成|画一张|出图|来一张|再来一张|做一张|做图|做视频|视频生成|生成视频|动起来)/i;
|
||
const ANALYSIS_ONLY_REGEX = /(分析|看图|看看|识别|描述|总结|解释|判断|评价)/i;
|
||
const VAGUE_EXECUTION_REGEX =
|
||
/^(按刚才那个做|按刚才的做|按这个做|按这个来|照这个来|照刚才那个|就这个|这个|来吧|开始吧|搞一下|弄一下|继续吧|就这样|按上一个来|按上一版来|按刚才那个来)$/i;
|
||
|
||
function normalizeCompactText(text: string) {
|
||
return String(text || "").replace(/\s+/g, "").trim();
|
||
}
|
||
|
||
function sortRefsForCanvasOrdering(refs: ResolvedImageRef[]) {
|
||
return [...refs].sort((left, right) => {
|
||
const leftY = Number(left.y ?? 0);
|
||
const rightY = Number(right.y ?? 0);
|
||
if (Math.abs(leftY - rightY) > 12) {
|
||
return leftY - rightY;
|
||
}
|
||
return Number(left.x ?? 0) - Number(right.x ?? 0);
|
||
});
|
||
}
|
||
|
||
function getPositionLabel(ref: ResolvedImageRef, refs: ResolvedImageRef[]): string {
|
||
if (refs.length <= 1) return "中间";
|
||
const xs = refs.map((item) => Number(item.x ?? 0));
|
||
const ys = refs.map((item) => Number(item.y ?? 0));
|
||
const minX = Math.min(...xs);
|
||
const maxX = Math.max(...xs);
|
||
const minY = Math.min(...ys);
|
||
const maxY = Math.max(...ys);
|
||
const spanX = Math.max(1, maxX - minX);
|
||
const spanY = Math.max(1, maxY - minY);
|
||
const relX = (Number(ref.x ?? 0) - minX) / spanX;
|
||
const relY = (Number(ref.y ?? 0) - minY) / spanY;
|
||
|
||
const horizontal = relX <= 0.33 ? "左" : relX >= 0.66 ? "右" : "中";
|
||
const vertical = relY <= 0.33 ? "上" : relY >= 0.66 ? "下" : "中";
|
||
|
||
if (horizontal === "中" && vertical === "中") return "中间";
|
||
return `${vertical}${horizontal}`;
|
||
}
|
||
|
||
function buildRefPreviewList(refs: ResolvedImageRef[]) {
|
||
const orderedRefs = sortRefsForCanvasOrdering(refs).slice(0, 4);
|
||
return orderedRefs.map((ref, index) => {
|
||
const position = getPositionLabel(ref, orderedRefs);
|
||
const title = String(ref.title || "").trim();
|
||
return `- 第${index + 1}张（${position}）${title ? `：${title}` : ""}`;
|
||
});
|
||
}
|
||
|
||
function buildMultiSelectionClarificationMessage(selectedRefs: ResolvedImageRef[]) {
|
||
const previewLines = buildRefPreviewList(selectedRefs);
|
||
return [
|
||
`你当前选中了 ${selectedRefs.length} 张图，我还不能确定这一步是只处理其中一张，还是批量处理全部。`,
|
||
previewLines.length > 0 ? "当前选中候选：" : "",
|
||
...previewLines,
|
||
"请直接回复：",
|
||
"- 处理全部",
|
||
"- 只处理第2张 / 左边那张",
|
||
"",
|
||
"也可以先在画布里只保留一张选中，再让我继续。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
}
|
||
|
||
function buildMissingTargetClarificationMessage(params: {
|
||
selectedRefs: ResolvedImageRef[];
|
||
candidateRefs: ResolvedImageRef[];
|
||
}) {
|
||
if (params.selectedRefs.length > 1) {
|
||
return buildMultiSelectionClarificationMessage(params.selectedRefs);
|
||
}
|
||
|
||
if (params.candidateRefs.length > 1) {
|
||
const previewLines = buildRefPreviewList(params.candidateRefs);
|
||
return [
|
||
"我这次还没锁定要处理哪张图。",
|
||
`当前画布里有 ${params.candidateRefs.length} 张候选图：`,
|
||
...previewLines,
|
||
"",
|
||
"请直接回复“处理第2张 / 左边那张 / 选中的这张 / 上一版结果图 / 原图”，或者先在画布里选中一张再继续。",
|
||
].join("\n");
|
||
}
|
||
|
||
return [
|
||
"我还没拿到可处理的目标图。",
|
||
"你可以先上传图片，或者先在画布里选中一张图，再告诉我要怎么处理。",
|
||
].join("\n");
|
||
}
|
||
|
||
function buildLowConfidenceTargetClarificationMessage(params: {
|
||
candidateRefs: ResolvedImageRef[];
|
||
targetResolution: ResolvedTargetAssets;
|
||
}) {
|
||
const previewLines = buildRefPreviewList(params.candidateRefs);
|
||
return [
|
||
"这一步我能继续执行，但当前目标图只是在低置信度下被系统猜中的，我不想替你处理错图。",
|
||
previewLines.length > 0 ? `当前画布里可参考的候选图有 ${params.candidateRefs.length} 张：` : "",
|
||
...previewLines,
|
||
"",
|
||
`系统当前偏向的是「${params.targetResolution.reason}」这一路径，但我建议你先明确回复“第2张 / 左边那张 / 上一版结果图 / 原图”。`,
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
}
|
||
|
||
function buildActionClarificationMessage(mode: IntentArbiterMode) {
|
||
if (mode === "generate_image") {
|
||
return [
|
||
"我已经锁定目标图了，但还不清楚你这一步想让我怎么做。",
|
||
"你可以直接回复类似：",
|
||
"- 基于这张再生成 3 张",
|
||
"- 按这个主体重新生成一版海报",
|
||
"- 保留主体，换成白底产品图",
|
||
"- 先分析这张图",
|
||
].join("\n");
|
||
}
|
||
|
||
if (mode === "generate_video") {
|
||
return [
|
||
"我已经锁定参考图了，但这一步的视频目标还不够明确。",
|
||
"你可以直接回复类似：",
|
||
"- 让产品缓慢旋转 5 秒",
|
||
"- 做成首尾帧视频",
|
||
"- 先按这张图规划分镜，不直接生成",
|
||
].join("\n");
|
||
}
|
||
|
||
return [
|
||
"我已经锁定目标图了，但还不清楚你这一步想让我做什么。",
|
||
"你可以直接回复类似：",
|
||
"- 把背景改成白色",
|
||
"- 去掉桌上的杯子",
|
||
"- 提高清晰度并保留原构图",
|
||
"- 先分析这张图",
|
||
].join("\n");
|
||
}
|
||
|
||
function isCanvasSource(contextPayload?: ContextPayload | null) {
|
||
return (
|
||
contextPayload?.source === "canvas" || contextPayload?.source === "realtime-canvas"
|
||
);
|
||
}
|
||
|
||
function isExecutionMode(mode?: IntentArbiterMode) {
|
||
return mode === "edit_image" || mode === "generate_image" || mode === "generate_video";
|
||
}
|
||
|
||
export function resolveCanvasAgentClarificationGate(params: {
|
||
text: string;
|
||
composerMode?: "agent" | "image" | "video" | "detail";
|
||
contextPayload?: ContextPayload | null;
|
||
mediaInput?: ResolvedChatMediaInput | null;
|
||
targetResolution?: ResolvedTargetAssets | null;
|
||
taskBoundaryDecision: TaskBoundaryDecision;
|
||
executionMode?: IntentArbiterMode;
|
||
actionKind?: ConversationActionKind;
|
||
}) {
|
||
if (params.composerMode !== "agent") {
|
||
return { shouldClarify: false } satisfies CanvasAgentClarificationGateDecision;
|
||
}
|
||
|
||
if (!isCanvasSource(params.contextPayload)) {
|
||
return { shouldClarify: false } satisfies CanvasAgentClarificationGateDecision;
|
||
}
|
||
|
||
if (!isExecutionMode(params.executionMode)) {
|
||
return { shouldClarify: false } satisfies CanvasAgentClarificationGateDecision;
|
||
}
|
||
|
||
if (!params.taskBoundaryDecision.shouldAllowExecution) {
|
||
return { shouldClarify: false } satisfies CanvasAgentClarificationGateDecision;
|
||
}
|
||
|
||
const text = String(params.text || "").trim();
|
||
const compactText = normalizeCompactText(text);
|
||
const mediaInput = params.mediaInput;
|
||
const targetResolution = params.targetResolution;
|
||
const selectedRefs = mediaInput?.selectedImageRefs || [];
|
||
const candidateRefs = targetResolution?.candidateRefs || mediaInput?.imageRefs || [];
|
||
|
||
if (
|
||
params.executionMode === "edit_image" &&
|
||
selectedRefs.length > 1 &&
|
||
!BATCH_EDIT_REGEX.test(compactText) &&
|
||
!EXPLICIT_TARGET_HINT_REGEX.test(compactText)
|
||
) {
|
||
return {
|
||
shouldClarify: true,
|
||
clarificationMessage: buildMultiSelectionClarificationMessage(selectedRefs),
|
||
reason: "multiple_selected_canvas_targets_without_scope",
|
||
} satisfies CanvasAgentClarificationGateDecision;
|
||
}
|
||
|
||
if (targetResolution?.needsClarification) {
|
||
return {
|
||
shouldClarify: true,
|
||
clarificationMessage: buildMissingTargetClarificationMessage({
|
||
selectedRefs,
|
||
candidateRefs,
|
||
}),
|
||
reason: "target_resolution_requires_clarification",
|
||
} satisfies CanvasAgentClarificationGateDecision;
|
||
}
|
||
|
||
const shouldClarifyLowConfidenceCanvasTarget =
|
||
params.executionMode === "edit_image" &&
|
||
selectedRefs.length === 0 &&
|
||
(targetResolution?.resolutionMode === "working_memory" ||
|
||
targetResolution?.resolutionMode === "latest_result") &&
|
||
Number(targetResolution?.confidence || 0) < 85 &&
|
||
candidateRefs.length > 1 &&
|
||
!EXPLICIT_TARGET_HINT_REGEX.test(compactText) &&
|
||
!EXPLICIT_CONTINUATION_TARGET_REGEX.test(compactText);
|
||
|
||
if (shouldClarifyLowConfidenceCanvasTarget && targetResolution) {
|
||
return {
|
||
shouldClarify: true,
|
||
clarificationMessage: buildLowConfidenceTargetClarificationMessage({
|
||
candidateRefs,
|
||
targetResolution,
|
||
}),
|
||
reason: "low_confidence_canvas_target_guess",
|
||
} satisfies CanvasAgentClarificationGateDecision;
|
||
}
|
||
|
||
const lacksExplicitAction =
|
||
!ACTIONABLE_EDIT_REGEX.test(compactText) &&
|
||
!ACTIONABLE_GENERATION_REGEX.test(compactText) &&
|
||
!ANALYSIS_ONLY_REGEX.test(compactText);
|
||
const looksLikeVagueExecution =
|
||
VAGUE_EXECUTION_REGEX.test(compactText) ||
|
||
(compactText.length <= 8 &&
|
||
/^(这个|就这个|按这个|来吧|开始吧|继续吧|就这样)$/.test(compactText));
|
||
|
||
if (
|
||
targetResolution &&
|
||
targetResolution.targetRefs.length > 0 &&
|
||
params.actionKind === "default" &&
|
||
lacksExplicitAction &&
|
||
looksLikeVagueExecution
|
||
) {
|
||
return {
|
||
shouldClarify: true,
|
||
clarificationMessage: buildActionClarificationMessage(params.executionMode),
|
||
reason: "target_locked_but_action_is_too_vague",
|
||
} satisfies CanvasAgentClarificationGateDecision;
|
||
}
|
||
|
||
return {
|
||
shouldClarify: false,
|
||
} satisfies CanvasAgentClarificationGateDecision;
|
||
}
|