feat: add unified intent arbiter
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled

This commit is contained in:
Your Name 2026-03-17 17:34:00 +08:00
parent 4f1745b865
commit 0b6a794097
2 changed files with 454 additions and 117 deletions

View File

@ -143,6 +143,7 @@ import {
recordProviderHealthSuccess,
shouldAvoidProvider,
} from "@/lib/ai/runtime/provider-health";
import { arbitrateAgentIntent } from "@/lib/ai/runtime/intent-arbiter";
import { buildAgentRoleIdentityInstruction } from "@/lib/ai/runtime/agent-role-profile";
import { uploadFile } from "@/lib/storage/service";
import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
@ -3398,6 +3399,7 @@ export async function POST(request: Request) {
let autoCurrentImageUrlsForFallback: string[] = [];
let confirmedPromptForGeneration: string | undefined;
let forcePromptDrivenImageGeneration = false;
let shouldPreferArbitratedFriendlyMessage = false;
const smartAutoInputText =
resolvedConversationAction.effectiveInputText || requestTextContent;
@ -3699,12 +3701,36 @@ export async function POST(request: Request) {
needsClarification: autoGraphResult.analysis.needsClarification,
});
smartAutoDetectedIntent =
autoGraphResult.execution?.routeIntent || autoGraphResult.analysis.intent;
const arbitratedIntent = arbitrateAgentIntent({
text: requestTextContent,
hasImage,
hasHistoryImage: smartAutoHistoryImages.length > 0,
actionResolution: resolvedConversationAction,
taskBoundaryDecision,
analysis: autoGraphResult.analysis,
execution: smartAutoExecution,
targetSource: requestTargetResolution.resolutionMode,
});
if (arbitratedIntent.overridden) {
console.log("[IntentArbiter] Override auto-agent intent:", {
fromIntent: autoGraphResult.analysis.intent,
fromMode: autoGraphResult.execution?.mode,
toIntent: arbitratedIntent.analysis.intent,
toMode: arbitratedIntent.execution.mode,
reason: arbitratedIntent.reason,
});
}
smartAutoExecution = arbitratedIntent.execution as AgentPreparedExecution;
smartAutoDetectedIntent = arbitratedIntent.analysis.intent;
smartAutoNeedsClarification =
smartAutoPlan?.nextAction === "clarify" ||
Boolean(autoGraphResult.analysis.needsClarification);
smartAutoClarificationQuestion = autoGraphResult.analysis.clarifyingQuestions?.[0];
Boolean(arbitratedIntent.analysis.needsClarification) ||
(!arbitratedIntent.overridden && smartAutoPlan?.nextAction === "clarify");
smartAutoClarificationQuestion = arbitratedIntent.analysis.clarifyingQuestions?.[0];
if (arbitratedIntent.friendlyMessage) {
smartAutoFriendlyMessage = arbitratedIntent.friendlyMessage;
shouldPreferArbitratedFriendlyMessage = true;
}
const targetExecutionOverride = resolveTargetExecutionOverride({
hasImage,
@ -3739,7 +3765,7 @@ export async function POST(request: Request) {
requestedGrid,
hasActiveStoryboardTask: Boolean(activeStoryboardTaskForChat?.task?.id),
executionMode: smartAutoExecution?.mode,
detectedIntent: autoGraphResult.analysis?.intent,
detectedIntent: smartAutoDetectedIntent,
requestText: requestTextContent,
effectiveInputText: smartAutoInputText,
});
@ -3755,7 +3781,7 @@ export async function POST(request: Request) {
});
}
if (autoGraphResult.friendlyMessage) {
if (autoGraphResult.friendlyMessage && !shouldPreferArbitratedFriendlyMessage) {
smartAutoFriendlyMessage = autoGraphResult.friendlyMessage;
console.log("[Smart Auto Mode] 💬 Conversational reply:", smartAutoFriendlyMessage);
}
@ -3777,7 +3803,7 @@ export async function POST(request: Request) {
console.log("[Smart Auto Mode] ✅ Replace clarification-like friendly reply");
}
const autoIntent = autoGraphResult.analysis?.intent;
const autoIntent = smartAutoDetectedIntent || autoGraphResult.analysis?.intent;
const autoExecutionMode = smartAutoExecution?.mode;
const isImageTask =
autoExecutionMode === "generate_image" ||
@ -3954,62 +3980,34 @@ export async function POST(request: Request) {
rescueTimeoutMs,
);
smartAutoDetectedIntent = rescueAnalysis.intent;
smartAutoNeedsClarification = Boolean(rescueAnalysis.needsClarification);
smartAutoClarificationQuestion = rescueAnalysis.clarifyingQuestions?.[0];
smartAutoExecution = {
mode:
rescueAnalysis.intent === "video_gen"
? "generate_video"
: rescueAnalysis.intent === "image_gen"
? "generate_image"
: rescueAnalysis.intent === "image_edit"
? "edit_image"
: rescueAnalysis.intent === "vision"
? "analyze_image"
: rescueAnalysis.needsClarification
? "clarify"
: "respond",
routeIntent: rescueAnalysis.intent,
routeSubIntent: rescueAnalysis.subIntent,
const arbitratedIntent = arbitrateAgentIntent({
text: autoTextForFallback,
hasImage: rescueHasImage,
hasHistoryImage: rescueFromHistory || autoFromHistoryForFallback,
actionResolution: resolvedConversationAction,
taskBoundaryDecision,
analysis: rescueAnalysis,
execution: undefined,
targetSource: requestTargetResolution?.resolutionMode || "none",
preferredCapability:
rescueAnalysis.intent === "video_gen"
? "generate_video"
: rescueAnalysis.intent === "image_gen"
? "generate_image"
: rescueAnalysis.intent === "image_edit"
? rescueAnalysis.subIntent === "remove_background"
? "remove_background"
: rescueAnalysis.subIntent === "enhance"
? "enhance"
: rescueAnalysis.subIntent === "extend"
? "extend"
: rescueAnalysis.subIntent === "resize"
? "resize"
: "generic_edit"
: rescueAnalysis.intent === "vision"
? "analyze"
: "respond",
shouldUseStructuredReply:
rescueAnalysis.intent === "video_gen" ||
rescueAnalysis.intent === "image_gen" ||
rescueAnalysis.intent === "image_edit",
shouldOptimizePrompt:
rescueAnalysis.intent === "video_gen" ||
rescueAnalysis.intent === "image_gen" ||
rescueAnalysis.intent === "image_edit",
shouldApplyImagePipeline:
rescueAnalysis.intent === "image_gen" ||
rescueAnalysis.intent === "image_edit",
shouldApplyVideoPipeline: rescueAnalysis.intent === "video_gen",
shouldStayConversational:
rescueAnalysis.intent === "chat" || rescueAnalysis.intent === "vision",
shouldPublishToCanvas: false,
toolCalls: [],
availableTools: [],
};
if (rescueAnalysis.conversationalReply?.trim()) {
smartAutoFriendlyMessage = rescueAnalysis.conversationalReply.trim();
});
if (arbitratedIntent.overridden) {
console.log("[IntentArbiter] Override rescue intent:", {
fromIntent: rescueAnalysis.intent,
toIntent: arbitratedIntent.analysis.intent,
toMode: arbitratedIntent.execution.mode,
reason: arbitratedIntent.reason,
});
}
smartAutoDetectedIntent = arbitratedIntent.analysis.intent;
smartAutoNeedsClarification = Boolean(arbitratedIntent.analysis.needsClarification);
smartAutoClarificationQuestion = arbitratedIntent.analysis.clarifyingQuestions?.[0];
smartAutoExecution = arbitratedIntent.execution as AgentPreparedExecution;
if (arbitratedIntent.friendlyMessage || rescueAnalysis.conversationalReply?.trim()) {
smartAutoFriendlyMessage = (
arbitratedIntent.friendlyMessage ||
rescueAnalysis.conversationalReply ||
""
).trim();
}
} else {
console.warn("[Smart Auto Mode] Skip rescue analysis due to pre-response budget");
@ -4022,61 +4020,35 @@ export async function POST(request: Request) {
autoTextForFallback,
autoCurrentImageUrlsForFallback.length > 0 || smartAutoHistoryImages.length > 0,
);
smartAutoDetectedIntent = keywordFallback.intent;
smartAutoExecution = {
mode:
keywordFallback.intent === "video_gen"
? "generate_video"
: keywordFallback.intent === "image_gen"
? "generate_image"
: keywordFallback.intent === "image_edit"
? "edit_image"
: keywordFallback.intent === "vision"
? "analyze_image"
: keywordFallback.needsClarification
? "clarify"
: "respond",
routeIntent: keywordFallback.intent,
routeSubIntent: keywordFallback.subIntent,
const arbitratedIntent = arbitrateAgentIntent({
text: autoTextForFallback,
hasImage:
autoCurrentImageUrlsForFallback.length > 0 || smartAutoHistoryImages.length > 0,
hasHistoryImage: smartAutoHistoryImages.length > 0 || autoFromHistoryForFallback,
actionResolution: resolvedConversationAction,
taskBoundaryDecision,
analysis: keywordFallback,
execution: undefined,
targetSource: requestTargetResolution?.resolutionMode || "none",
preferredCapability:
keywordFallback.intent === "video_gen"
? "generate_video"
: keywordFallback.intent === "image_gen"
? "generate_image"
: keywordFallback.intent === "image_edit"
? keywordFallback.subIntent === "remove_background"
? "remove_background"
: keywordFallback.subIntent === "enhance"
? "enhance"
: keywordFallback.subIntent === "extend"
? "extend"
: keywordFallback.subIntent === "resize"
? "resize"
: "generic_edit"
: keywordFallback.intent === "vision"
? "analyze"
: "respond",
shouldUseStructuredReply:
keywordFallback.intent === "video_gen" ||
keywordFallback.intent === "image_gen" ||
keywordFallback.intent === "image_edit",
shouldOptimizePrompt:
keywordFallback.intent === "video_gen" ||
keywordFallback.intent === "image_gen" ||
keywordFallback.intent === "image_edit",
shouldApplyImagePipeline:
keywordFallback.intent === "image_gen" ||
keywordFallback.intent === "image_edit",
shouldApplyVideoPipeline: keywordFallback.intent === "video_gen",
shouldStayConversational:
keywordFallback.intent === "chat" || keywordFallback.intent === "vision",
shouldPublishToCanvas: false,
toolCalls: [],
availableTools: [],
};
if (keywordFallback.conversationalReply?.trim()) {
smartAutoFriendlyMessage = keywordFallback.conversationalReply.trim();
});
if (arbitratedIntent.overridden) {
console.log("[IntentArbiter] Override keyword-fallback intent:", {
fromIntent: keywordFallback.intent,
toIntent: arbitratedIntent.analysis.intent,
toMode: arbitratedIntent.execution.mode,
reason: arbitratedIntent.reason,
});
}
smartAutoDetectedIntent = arbitratedIntent.analysis.intent;
smartAutoNeedsClarification = Boolean(arbitratedIntent.analysis.needsClarification);
smartAutoClarificationQuestion = arbitratedIntent.analysis.clarifyingQuestions?.[0];
smartAutoExecution = arbitratedIntent.execution as AgentPreparedExecution;
if (arbitratedIntent.friendlyMessage || keywordFallback.conversationalReply?.trim()) {
smartAutoFriendlyMessage = (
arbitratedIntent.friendlyMessage ||
keywordFallback.conversationalReply ||
""
).trim();
console.warn("[Smart Auto Mode] Applied keyword fallback after rescue failure");
}
} catch (keywordFallbackError) {

View File

@ -0,0 +1,365 @@
import "server-only";
import type { ConversationActionResolution } from "@/lib/ai/conversation-action-resolver";
import type { ResolvedTargetAssets } from "@/lib/ai/context-memory/types";
import type { AnalysisResult } from "@/lib/ai/smart-analyzer";
import type { TaskBoundaryDecision } from "@/lib/ai/runtime/task-boundary-classifier";
export type IntentArbiterMode =
| "respond"
| "clarify"
| "analyze_image"
| "generate_image"
| "edit_image"
| "generate_video";
export type IntentArbiterPreferredCapability =
| "respond"
| "analyze"
| "generic_edit"
| "remove_background"
| "enhance"
| "extend"
| "resize"
| "box_cutout"
| "generate_image"
| "generate_video"
| "publish_canvas";
export interface IntentArbiterExecution {
mode: IntentArbiterMode;
routeIntent: AnalysisResult["intent"];
routeSubIntent?: string;
targetSource: ResolvedTargetAssets["resolutionMode"];
preferredCapability: IntentArbiterPreferredCapability;
shouldUseStructuredReply: boolean;
shouldOptimizePrompt: boolean;
shouldApplyImagePipeline: boolean;
shouldApplyVideoPipeline: boolean;
shouldStayConversational: boolean;
shouldPublishToCanvas: boolean;
toolCalls: Array<{ name: string; reason?: string; args?: Record<string, unknown> }>;
availableTools: string[];
}
export interface IntentArbiterResult {
analysis: AnalysisResult;
execution: IntentArbiterExecution;
overridden: boolean;
reason: string;
friendlyMessage?: string;
}
function normalizeCompactText(text: string): string {
return String(text || "").toLowerCase().replace(/\s+/g, "").trim();
}
function isLikelyAdvisoryOrPlanningRequest(text: string): boolean {
const normalizedText = String(text || "").toLowerCase();
return /怎么|如何|为什么|是否|可不可以|能不能|行不行|建议|思路|方案|流程|步骤|教程|推荐|对比|区别|差别|优缺点|评估|分析|判断|解释|讲讲|科普|适合吗|可行吗|值不值得|要不要|帮我看看|帮我分析|帮我判断/.test(
normalizedText,
);
}
function isExplicitVideoGenerationRequest(text: string): boolean {
const normalizedText = normalizeCompactText(text);
if (!normalizedText) return false;
const hasStrongVideoMarkers =
/图生视频|文生视频|首帧|尾帧|分镜|运镜|视频生成/.test(normalizedText);
if (hasStrongVideoMarkers) return true;
if (isLikelyAdvisoryOrPlanningRequest(text)) return false;
return (
/让.+动起来/.test(normalizedText) ||
(/(生成|制作|做|创建|输出|渲染|合成|出|做成|变成|转成)/.test(normalizedText) &&
/(视频|动画|短片|片子|动起来|会动)/.test(normalizedText))
);
}
function isExplicitImageGenerationRequest(text: string): boolean {
const normalizedText = normalizeCompactText(text);
if (!normalizedText || isLikelyAdvisoryOrPlanningRequest(text)) return false;
return /重新生成|再来一张|生成新图|新建图片|创作一张|生成一张|画一张|给我画|帮我画|出一张|做一张图|做一张海报|创建一张图片|参考.*生成|按这个风格生成|以这张图为灵感生成|仿照.*生成|生成一只|生成一个|生成一幅|生成一版|生成一套|生成一组|帮我生成.*(?:图|图片|海报|头像|插画|场景|小猫|猫咪|人物|产品图)|给我生成.*(?:图|图片|海报|头像|插画|场景|小猫|猫咪|人物|产品图)/.test(
normalizedText,
);
}
function isLikelyImageEditRequest(text: string): boolean {
const normalizedText = String(text || "").toLowerCase();
return /抠图|去背景|透明背景|去背|扣图|移除背景|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test(
normalizedText,
);
}
function inferDefaultImageEditSubIntent(text: string): string {
const lowerText = String(text || "").toLowerCase();
if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background";
if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background";
if (/风格|卡通|动漫|油画|像素风|写实|插画/.test(lowerText)) return "style_transfer";
if (/颜色|调色|亮度|对比度|饱和度|改成.*色|换成.*色/.test(lowerText)) return "color_adjust";
if (/去除|消除|擦掉|删除|移除|p掉|重绘/.test(lowerText)) return "inpaint";
return "enhance";
}
function mapIntentToMode(
intent: AnalysisResult["intent"],
needsClarification?: boolean,
): IntentArbiterMode {
if (needsClarification) return "clarify";
switch (intent) {
case "image_edit":
return "edit_image";
case "image_gen":
return "generate_image";
case "video_gen":
return "generate_video";
case "vision":
return "analyze_image";
case "chat":
default:
return "respond";
}
}
function mapPreferredCapability(
intent: AnalysisResult["intent"],
subIntent?: string,
): IntentArbiterPreferredCapability {
if (intent === "video_gen") return "generate_video";
if (intent === "image_gen") return "generate_image";
if (intent === "vision") return "analyze";
if (intent === "chat") return "respond";
switch (subIntent) {
case "remove_background":
return "remove_background";
case "extend":
return "extend";
case "resize":
return "resize";
case "box_cutout":
return "box_cutout";
case "enhance":
return "enhance";
default:
return "generic_edit";
}
}
function buildExecutionFromAnalysis(params: {
analysis: AnalysisResult;
execution?: IntentArbiterExecution | null;
targetSource: ResolvedTargetAssets["resolutionMode"];
}): IntentArbiterExecution {
const mode = mapIntentToMode(params.analysis.intent, params.analysis.needsClarification);
return {
mode,
routeIntent: params.analysis.intent,
routeSubIntent: params.analysis.subIntent,
targetSource: params.targetSource,
preferredCapability: mapPreferredCapability(params.analysis.intent, params.analysis.subIntent),
shouldUseStructuredReply:
params.analysis.intent === "image_edit" ||
params.analysis.intent === "image_gen" ||
params.analysis.intent === "video_gen",
shouldOptimizePrompt:
params.analysis.intent === "image_edit" ||
params.analysis.intent === "image_gen" ||
params.analysis.intent === "video_gen",
shouldApplyImagePipeline:
params.analysis.intent === "image_edit" || params.analysis.intent === "image_gen",
shouldApplyVideoPipeline: params.analysis.intent === "video_gen",
shouldStayConversational:
mode === "respond" || mode === "analyze_image" || mode === "clarify",
shouldPublishToCanvas: params.execution?.shouldPublishToCanvas || false,
toolCalls: params.execution?.toolCalls || [],
availableTools: params.execution?.availableTools || [],
};
}
function buildDefaultFriendlyMessage(
intent: AnalysisResult["intent"],
params: { hasImage: boolean; text: string },
) {
switch (intent) {
case "image_gen":
return params.hasImage ? "收到,我会结合你提供的参考图直接执行生图。" : "好的,我来直接帮你生成这张图。";
case "image_edit":
return params.hasImage ? "收到,我会按你的目标直接处理这张图。" : "收到,我来继续执行这轮图片处理。";
case "video_gen":
return params.hasImage ? "收到,我会结合这轮图片先规划并执行视频生成。" : "收到,我先按你的需求规划并执行视频生成。";
case "vision":
return params.hasImage ? "收到,我先分析这张图。" : "收到,我先按你的问题做分析。";
case "chat":
default:
return undefined;
}
}
function resolvePolicyIntent(params: {
text: string;
hasImage: boolean;
hasHistoryImage: boolean;
taskBoundaryDecision: TaskBoundaryDecision;
actionResolution: ConversationActionResolution;
analysis: AnalysisResult;
}): { intent: AnalysisResult["intent"]; subIntent?: string; reason: string } {
const explicitImageGen = isExplicitImageGenerationRequest(params.text);
const explicitVideoGen = isExplicitVideoGenerationRequest(params.text);
const likelyEdit = isLikelyImageEditRequest(params.text);
const hasUsableImage = params.hasImage || params.hasHistoryImage;
if (explicitVideoGen) {
return {
intent: "video_gen",
reason: "explicit_video_generation_request",
};
}
switch (params.actionResolution.kind) {
case "image_edit_followup":
return {
intent: "image_edit",
subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text),
reason: "conversation_action_image_edit_followup",
};
case "image_generation_followup":
case "prompt_carry_forward":
return {
intent: "image_gen",
reason: "conversation_action_image_generation_followup",
};
default:
break;
}
switch (params.taskBoundaryDecision.taskType) {
case "prompt_only":
case "analysis_only":
return {
intent: "vision",
reason: `task_boundary_${params.taskBoundaryDecision.taskType}`,
};
case "execution_video":
return {
intent: "video_gen",
reason: "task_boundary_execution_video",
};
case "execution_image":
if (hasUsableImage && likelyEdit && !explicitImageGen) {
return {
intent: "image_edit",
subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text),
reason: "task_boundary_execution_image_edit",
};
}
return {
intent: "image_gen",
reason: "task_boundary_execution_image_generate",
};
case "workflow_followup":
if (hasUsableImage && likelyEdit) {
return {
intent: "image_edit",
subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text),
reason: "task_boundary_workflow_followup_edit",
};
}
if (explicitImageGen) {
return {
intent: "image_gen",
reason: "task_boundary_workflow_followup_generate",
};
}
break;
default:
break;
}
if (explicitImageGen) {
return {
intent: "image_gen",
reason: "explicit_image_generation_request",
};
}
if (hasUsableImage && likelyEdit && params.analysis.intent === "chat") {
return {
intent: "image_edit",
subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text),
reason: "image_edit_policy_override",
};
}
return {
intent: params.analysis.intent,
subIntent:
params.analysis.intent === "image_edit"
? params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text)
: params.analysis.subIntent,
reason: "keep_ai_analysis",
};
}
export function arbitrateAgentIntent(params: {
text: string;
hasImage: boolean;
hasHistoryImage: boolean;
actionResolution: ConversationActionResolution;
taskBoundaryDecision: TaskBoundaryDecision;
analysis: AnalysisResult;
execution?: IntentArbiterExecution | null;
targetSource: ResolvedTargetAssets["resolutionMode"];
}): IntentArbiterResult {
const resolved = resolvePolicyIntent(params);
const nextAnalysis: AnalysisResult = {
...params.analysis,
intent: resolved.intent,
subIntent:
resolved.intent === "image_edit"
? resolved.subIntent || inferDefaultImageEditSubIntent(params.text)
: undefined,
confidence:
resolved.reason === "keep_ai_analysis"
? params.analysis.confidence
: Math.max(params.analysis.confidence, 0.84),
needsClarification:
resolved.reason === "keep_ai_analysis"
? Boolean(params.analysis.needsClarification)
: false,
clarifyingQuestions:
resolved.reason === "keep_ai_analysis" && params.analysis.needsClarification
? params.analysis.clarifyingQuestions || []
: [],
};
const nextExecution = buildExecutionFromAnalysis({
analysis: nextAnalysis,
execution: params.execution,
targetSource: params.targetSource,
});
const overridden =
nextAnalysis.intent !== params.analysis.intent ||
nextAnalysis.subIntent !== params.analysis.subIntent ||
nextExecution.mode !== params.execution?.mode ||
nextExecution.routeIntent !== params.execution?.routeIntent;
const friendlyMessage =
overridden || !params.analysis.conversationalReply?.trim()
? buildDefaultFriendlyMessage(nextAnalysis.intent, {
hasImage: params.hasImage || params.hasHistoryImage,
text: params.text,
})
: undefined;
return {
analysis: friendlyMessage
? {
...nextAnalysis,
conversationalReply: friendlyMessage,
}
: nextAnalysis,
execution: nextExecution,
overridden,
reason: resolved.reason,
friendlyMessage,
};
}