395 lines
15 KiB
TypeScript
395 lines
15 KiB
TypeScript
import "server-only";
|
|
|
|
import type { ConversationActionResolution } from "@/lib/ai/conversation-action-resolver";
|
|
import type { ResolvedTargetAssets } from "@/lib/ai/context-memory/types";
|
|
import type { AnalysisResult } from "@/lib/ai/smart-analyzer";
|
|
import type { TaskBoundaryDecision } from "@/lib/ai/runtime/task-boundary-classifier";
|
|
|
|
export type IntentArbiterMode =
|
|
| "respond"
|
|
| "clarify"
|
|
| "analyze_image"
|
|
| "generate_image"
|
|
| "edit_image"
|
|
| "generate_video";
|
|
|
|
export type IntentArbiterPreferredCapability =
|
|
| "respond"
|
|
| "analyze"
|
|
| "generic_edit"
|
|
| "remove_background"
|
|
| "enhance"
|
|
| "extend"
|
|
| "resize"
|
|
| "box_cutout"
|
|
| "generate_image"
|
|
| "generate_video"
|
|
| "publish_canvas";
|
|
|
|
export interface IntentArbiterExecution {
|
|
mode: IntentArbiterMode;
|
|
routeIntent: AnalysisResult["intent"];
|
|
routeSubIntent?: string;
|
|
targetSource: ResolvedTargetAssets["resolutionMode"];
|
|
preferredCapability: IntentArbiterPreferredCapability;
|
|
shouldUseStructuredReply: boolean;
|
|
shouldOptimizePrompt: boolean;
|
|
shouldApplyImagePipeline: boolean;
|
|
shouldApplyVideoPipeline: boolean;
|
|
shouldStayConversational: boolean;
|
|
shouldPublishToCanvas: boolean;
|
|
toolCalls: Array<{ name: string; reason?: string; args?: Record<string, unknown> }>;
|
|
availableTools: string[];
|
|
}
|
|
|
|
export interface IntentArbiterResult {
|
|
analysis: AnalysisResult;
|
|
execution: IntentArbiterExecution;
|
|
overridden: boolean;
|
|
reason: string;
|
|
friendlyMessage?: string;
|
|
}
|
|
|
|
function normalizeCompactText(text: string): string {
|
|
return String(text || "").toLowerCase().replace(/\s+/g, "").trim();
|
|
}
|
|
|
|
function isLikelyAdvisoryOrPlanningRequest(text: string): boolean {
|
|
const normalizedText = String(text || "").toLowerCase();
|
|
return /怎么|如何|为什么|是否|可不可以|能不能|行不行|建议|思路|方案|流程|步骤|教程|推荐|对比|区别|差别|优缺点|评估|分析|判断|解释|讲讲|科普|适合吗|可行吗|值不值得|要不要|帮我看看|帮我分析|帮我判断/.test(
|
|
normalizedText,
|
|
);
|
|
}
|
|
|
|
function isExplicitVideoGenerationRequest(text: string): boolean {
|
|
const normalizedText = normalizeCompactText(text);
|
|
if (!normalizedText) return false;
|
|
const hasStrongVideoMarkers =
|
|
/图生视频|文生视频|首帧|尾帧|分镜|运镜|视频生成/.test(normalizedText);
|
|
if (hasStrongVideoMarkers) return true;
|
|
if (isLikelyAdvisoryOrPlanningRequest(text)) return false;
|
|
return (
|
|
/让.+动起来/.test(normalizedText) ||
|
|
(/(生成|制作|做|创建|输出|渲染|合成|出|做成|变成|转成)/.test(normalizedText) &&
|
|
/(视频|动画|短片|片子|动起来|会动)/.test(normalizedText))
|
|
);
|
|
}
|
|
|
|
function isExplicitImageGenerationRequest(text: string): boolean {
|
|
const normalizedText = normalizeCompactText(text);
|
|
if (!normalizedText || isLikelyAdvisoryOrPlanningRequest(text)) return false;
|
|
if (/不要生图|先别生图|不需要生图|不要生成|别生成|只要提示词|仅给提示词|先给提示词/.test(normalizedText)) {
|
|
return false;
|
|
}
|
|
|
|
const countToken = "(?:一|两|二|三|四|五|六|七|八|九|十|[1-9]\\d?)";
|
|
const visualTarget =
|
|
"(?:图|图片|海报|头像|插画|场景|主图|产品图|商品图|电商图|壁纸|封面|logo|图标)";
|
|
const createVisualTarget = new RegExp(
|
|
`(?:生成|创作|创建|制作|设计|做|出|画|绘制)${countToken}?张?.{0,32}${visualTarget}`,
|
|
);
|
|
const drawSubject = new RegExp(
|
|
`(?:画|绘制|生成|创作|创建)${countToken}?(?:只|个|位|幅|张)?.{0,24}(?:小猫|猫咪|猫|狗|人物|角色|ip|头像|logo|图标|插画)`,
|
|
);
|
|
const continueVariants = new RegExp(
|
|
`(?:再来|再出|再做|再给我|补|多来|继续生成|继续出)${countToken}?张`,
|
|
);
|
|
|
|
return (
|
|
/重新生成|生成新图|新建图片|创作一张|生成一张|画一张|给我画|帮我画|出一张|做一张图|做一张海报|创建一张图片|生成一只|生成一个|生成一幅|生成一版|生成一套|生成一组|生图|出图/.test(
|
|
normalizedText,
|
|
) ||
|
|
createVisualTarget.test(normalizedText) ||
|
|
drawSubject.test(normalizedText) ||
|
|
continueVariants.test(normalizedText) ||
|
|
/参考.*生成|按这个风格生成|以这张图为灵感生成|仿照.*生成|同风格再来|同样风格再来/.test(
|
|
normalizedText,
|
|
) ||
|
|
/帮我生成.*(?:图|图片|海报|头像|插画|场景|小猫|猫咪|人物|产品图)|给我生成.*(?:图|图片|海报|头像|插画|场景|小猫|猫咪|人物|产品图)/.test(
|
|
normalizedText,
|
|
)
|
|
);
|
|
}
|
|
|
|
function isLikelyImageEditRequest(text: string): boolean {
|
|
const normalizedText = String(text || "").toLowerCase();
|
|
return /抠图|去背景|透明背景|去背|扣图|移除背景|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test(
|
|
normalizedText,
|
|
);
|
|
}
|
|
|
|
function inferDefaultImageEditSubIntent(text: string): string {
|
|
const lowerText = String(text || "").toLowerCase();
|
|
if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background";
|
|
if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background";
|
|
if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|比例|画幅|竖版|横版|竖图|横图|9:16|16:9|1:1/.test(lowerText)) return "extend";
|
|
if (/风格|卡通|动漫|油画|像素风|写实|插画/.test(lowerText)) return "style_transfer";
|
|
if (/颜色|调色|亮度|对比度|饱和度|改成.*色|换成.*色/.test(lowerText)) return "color_adjust";
|
|
if (/去除|消除|擦掉|删除|移除|p掉|重绘/.test(lowerText)) return "inpaint";
|
|
return "enhance";
|
|
}
|
|
|
|
function mapIntentToMode(
|
|
intent: AnalysisResult["intent"],
|
|
needsClarification?: boolean,
|
|
): IntentArbiterMode {
|
|
if (needsClarification) return "clarify";
|
|
switch (intent) {
|
|
case "image_edit":
|
|
return "edit_image";
|
|
case "image_gen":
|
|
return "generate_image";
|
|
case "video_gen":
|
|
return "generate_video";
|
|
case "vision":
|
|
return "analyze_image";
|
|
case "chat":
|
|
default:
|
|
return "respond";
|
|
}
|
|
}
|
|
|
|
function mapPreferredCapability(
|
|
intent: AnalysisResult["intent"],
|
|
subIntent?: string,
|
|
): IntentArbiterPreferredCapability {
|
|
if (intent === "video_gen") return "generate_video";
|
|
if (intent === "image_gen") return "generate_image";
|
|
if (intent === "vision") return "analyze";
|
|
if (intent === "chat") return "respond";
|
|
|
|
switch (subIntent) {
|
|
case "remove_background":
|
|
return "remove_background";
|
|
case "extend":
|
|
return "extend";
|
|
case "resize":
|
|
return "resize";
|
|
case "box_cutout":
|
|
return "box_cutout";
|
|
case "enhance":
|
|
return "enhance";
|
|
default:
|
|
return "generic_edit";
|
|
}
|
|
}
|
|
|
|
function buildExecutionFromAnalysis(params: {
|
|
analysis: AnalysisResult;
|
|
execution?: IntentArbiterExecution | null;
|
|
targetSource: ResolvedTargetAssets["resolutionMode"];
|
|
}): IntentArbiterExecution {
|
|
const mode = mapIntentToMode(params.analysis.intent, params.analysis.needsClarification);
|
|
|
|
return {
|
|
mode,
|
|
routeIntent: params.analysis.intent,
|
|
routeSubIntent: params.analysis.subIntent,
|
|
targetSource: params.targetSource,
|
|
preferredCapability: mapPreferredCapability(params.analysis.intent, params.analysis.subIntent),
|
|
shouldUseStructuredReply:
|
|
params.analysis.intent === "image_edit" ||
|
|
params.analysis.intent === "image_gen" ||
|
|
params.analysis.intent === "video_gen",
|
|
shouldOptimizePrompt:
|
|
params.analysis.intent === "image_edit" ||
|
|
params.analysis.intent === "image_gen" ||
|
|
params.analysis.intent === "video_gen",
|
|
shouldApplyImagePipeline:
|
|
params.analysis.intent === "image_edit" || params.analysis.intent === "image_gen",
|
|
shouldApplyVideoPipeline: params.analysis.intent === "video_gen",
|
|
shouldStayConversational:
|
|
mode === "respond" || mode === "analyze_image" || mode === "clarify",
|
|
shouldPublishToCanvas: params.execution?.shouldPublishToCanvas || false,
|
|
toolCalls: params.execution?.toolCalls || [],
|
|
availableTools: params.execution?.availableTools || [],
|
|
};
|
|
}
|
|
|
|
function buildDefaultFriendlyMessage(
|
|
intent: AnalysisResult["intent"],
|
|
params: { hasImage: boolean; text: string },
|
|
) {
|
|
switch (intent) {
|
|
case "image_gen":
|
|
return params.hasImage ? "收到，我会结合你提供的参考图直接执行生图。" : "好的，我来直接帮你生成这张图。";
|
|
case "image_edit":
|
|
return params.hasImage ? "收到，我会按你的目标直接处理这张图。" : "收到，我来继续执行这轮图片处理。";
|
|
case "video_gen":
|
|
return params.hasImage ? "收到，我会结合这轮图片先规划并执行视频生成。" : "收到，我先按你的需求规划并执行视频生成。";
|
|
case "vision":
|
|
return params.hasImage ? "收到，我先分析这张图。" : "收到，我先按你的问题做分析。";
|
|
case "chat":
|
|
default:
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
function resolvePolicyIntent(params: {
|
|
text: string;
|
|
hasImage: boolean;
|
|
hasHistoryImage: boolean;
|
|
taskBoundaryDecision: TaskBoundaryDecision;
|
|
actionResolution: ConversationActionResolution;
|
|
analysis: AnalysisResult;
|
|
}): { intent: AnalysisResult["intent"]; subIntent?: string; reason: string } {
|
|
const explicitImageGen = isExplicitImageGenerationRequest(params.text);
|
|
const explicitVideoGen = isExplicitVideoGenerationRequest(params.text);
|
|
const likelyEdit = isLikelyImageEditRequest(params.text);
|
|
const hasUsableImage = params.hasImage || params.hasHistoryImage;
|
|
|
|
if (explicitVideoGen) {
|
|
return {
|
|
intent: "video_gen",
|
|
reason: "explicit_video_generation_request",
|
|
};
|
|
}
|
|
|
|
switch (params.actionResolution.kind) {
|
|
case "image_edit_followup":
|
|
return {
|
|
intent: "image_edit",
|
|
subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text),
|
|
reason: "conversation_action_image_edit_followup",
|
|
};
|
|
case "image_generation_followup":
|
|
case "prompt_carry_forward":
|
|
return {
|
|
intent: "image_gen",
|
|
reason: "conversation_action_image_generation_followup",
|
|
};
|
|
default:
|
|
break;
|
|
}
|
|
|
|
switch (params.taskBoundaryDecision.taskType) {
|
|
case "prompt_only":
|
|
case "analysis_only":
|
|
return {
|
|
intent: "vision",
|
|
reason: `task_boundary_${params.taskBoundaryDecision.taskType}`,
|
|
};
|
|
case "execution_video":
|
|
return {
|
|
intent: "video_gen",
|
|
reason: "task_boundary_execution_video",
|
|
};
|
|
case "execution_image":
|
|
if (hasUsableImage && likelyEdit && !explicitImageGen) {
|
|
return {
|
|
intent: "image_edit",
|
|
subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text),
|
|
reason: "task_boundary_execution_image_edit",
|
|
};
|
|
}
|
|
return {
|
|
intent: "image_gen",
|
|
reason: "task_boundary_execution_image_generate",
|
|
};
|
|
case "workflow_followup":
|
|
if (hasUsableImage && likelyEdit) {
|
|
return {
|
|
intent: "image_edit",
|
|
subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text),
|
|
reason: "task_boundary_workflow_followup_edit",
|
|
};
|
|
}
|
|
if (explicitImageGen) {
|
|
return {
|
|
intent: "image_gen",
|
|
reason: "task_boundary_workflow_followup_generate",
|
|
};
|
|
}
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
if (explicitImageGen) {
|
|
return {
|
|
intent: "image_gen",
|
|
reason: "explicit_image_generation_request",
|
|
};
|
|
}
|
|
|
|
if (hasUsableImage && likelyEdit && params.analysis.intent === "chat") {
|
|
return {
|
|
intent: "image_edit",
|
|
subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text),
|
|
reason: "image_edit_policy_override",
|
|
};
|
|
}
|
|
|
|
return {
|
|
intent: params.analysis.intent,
|
|
subIntent:
|
|
params.analysis.intent === "image_edit"
|
|
? params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text)
|
|
: params.analysis.subIntent,
|
|
reason: "keep_ai_analysis",
|
|
};
|
|
}
|
|
|
|
export function arbitrateAgentIntent(params: {
|
|
text: string;
|
|
hasImage: boolean;
|
|
hasHistoryImage: boolean;
|
|
actionResolution: ConversationActionResolution;
|
|
taskBoundaryDecision: TaskBoundaryDecision;
|
|
analysis: AnalysisResult;
|
|
execution?: IntentArbiterExecution | null;
|
|
targetSource: ResolvedTargetAssets["resolutionMode"];
|
|
}): IntentArbiterResult {
|
|
const resolved = resolvePolicyIntent(params);
|
|
const nextAnalysis: AnalysisResult = {
|
|
...params.analysis,
|
|
intent: resolved.intent,
|
|
subIntent:
|
|
resolved.intent === "image_edit"
|
|
? resolved.subIntent || inferDefaultImageEditSubIntent(params.text)
|
|
: undefined,
|
|
confidence:
|
|
resolved.reason === "keep_ai_analysis"
|
|
? params.analysis.confidence
|
|
: Math.max(params.analysis.confidence, 0.84),
|
|
needsClarification:
|
|
resolved.reason === "keep_ai_analysis"
|
|
? Boolean(params.analysis.needsClarification)
|
|
: false,
|
|
clarifyingQuestions:
|
|
resolved.reason === "keep_ai_analysis" && params.analysis.needsClarification
|
|
? params.analysis.clarifyingQuestions || []
|
|
: [],
|
|
};
|
|
const nextExecution = buildExecutionFromAnalysis({
|
|
analysis: nextAnalysis,
|
|
execution: params.execution,
|
|
targetSource: params.targetSource,
|
|
});
|
|
const overridden =
|
|
nextAnalysis.intent !== params.analysis.intent ||
|
|
nextAnalysis.subIntent !== params.analysis.subIntent ||
|
|
nextExecution.mode !== params.execution?.mode ||
|
|
nextExecution.routeIntent !== params.execution?.routeIntent;
|
|
const friendlyMessage =
|
|
overridden || !params.analysis.conversationalReply?.trim()
|
|
? buildDefaultFriendlyMessage(nextAnalysis.intent, {
|
|
hasImage: params.hasImage || params.hasHistoryImage,
|
|
text: params.text,
|
|
})
|
|
: undefined;
|
|
|
|
return {
|
|
analysis: friendlyMessage
|
|
? {
|
|
...nextAnalysis,
|
|
conversationalReply: friendlyMessage,
|
|
}
|
|
: nextAnalysis,
|
|
execution: nextExecution,
|
|
overridden,
|
|
reason: resolved.reason,
|
|
friendlyMessage,
|
|
};
|
|
}
|