diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 32c63c7d6..0b9e57f2f 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -143,6 +143,7 @@ import { recordProviderHealthSuccess, shouldAvoidProvider, } from "@/lib/ai/runtime/provider-health"; +import { arbitrateAgentIntent } from "@/lib/ai/runtime/intent-arbiter"; import { buildAgentRoleIdentityInstruction } from "@/lib/ai/runtime/agent-role-profile"; import { uploadFile } from "@/lib/storage/service"; import { getVideoStoryboardConfig } from "@/lib/video-agent/config"; @@ -3398,6 +3399,7 @@ export async function POST(request: Request) { let autoCurrentImageUrlsForFallback: string[] = []; let confirmedPromptForGeneration: string | undefined; let forcePromptDrivenImageGeneration = false; + let shouldPreferArbitratedFriendlyMessage = false; const smartAutoInputText = resolvedConversationAction.effectiveInputText || requestTextContent; @@ -3699,12 +3701,36 @@ export async function POST(request: Request) { needsClarification: autoGraphResult.analysis.needsClarification, }); - smartAutoDetectedIntent = - autoGraphResult.execution?.routeIntent || autoGraphResult.analysis.intent; + const arbitratedIntent = arbitrateAgentIntent({ + text: requestTextContent, + hasImage, + hasHistoryImage: smartAutoHistoryImages.length > 0, + actionResolution: resolvedConversationAction, + taskBoundaryDecision, + analysis: autoGraphResult.analysis, + execution: smartAutoExecution, + targetSource: requestTargetResolution.resolutionMode, + }); + if (arbitratedIntent.overridden) { + console.log("[IntentArbiter] Override auto-agent intent:", { + fromIntent: autoGraphResult.analysis.intent, + fromMode: autoGraphResult.execution?.mode, + toIntent: arbitratedIntent.analysis.intent, + toMode: arbitratedIntent.execution.mode, + reason: arbitratedIntent.reason, + }); + } + + smartAutoExecution = arbitratedIntent.execution as AgentPreparedExecution; + smartAutoDetectedIntent = arbitratedIntent.analysis.intent; smartAutoNeedsClarification = - smartAutoPlan?.nextAction === "clarify" || - Boolean(autoGraphResult.analysis.needsClarification); - smartAutoClarificationQuestion = autoGraphResult.analysis.clarifyingQuestions?.[0]; + Boolean(arbitratedIntent.analysis.needsClarification) || + (!arbitratedIntent.overridden && smartAutoPlan?.nextAction === "clarify"); + smartAutoClarificationQuestion = arbitratedIntent.analysis.clarifyingQuestions?.[0]; + if (arbitratedIntent.friendlyMessage) { + smartAutoFriendlyMessage = arbitratedIntent.friendlyMessage; + shouldPreferArbitratedFriendlyMessage = true; + } const targetExecutionOverride = resolveTargetExecutionOverride({ hasImage, @@ -3739,7 +3765,7 @@ export async function POST(request: Request) { requestedGrid, hasActiveStoryboardTask: Boolean(activeStoryboardTaskForChat?.task?.id), executionMode: smartAutoExecution?.mode, - detectedIntent: autoGraphResult.analysis?.intent, + detectedIntent: smartAutoDetectedIntent, requestText: requestTextContent, effectiveInputText: smartAutoInputText, }); @@ -3755,7 +3781,7 @@ export async function POST(request: Request) { }); } - if (autoGraphResult.friendlyMessage) { + if (autoGraphResult.friendlyMessage && !shouldPreferArbitratedFriendlyMessage) { smartAutoFriendlyMessage = autoGraphResult.friendlyMessage; console.log("[Smart Auto Mode] 💬 Conversational reply:", smartAutoFriendlyMessage); } @@ -3777,7 +3803,7 @@ export async function POST(request: Request) { console.log("[Smart Auto Mode] ✅ Replace clarification-like friendly reply"); } - const autoIntent = autoGraphResult.analysis?.intent; + const autoIntent = smartAutoDetectedIntent || autoGraphResult.analysis?.intent; const autoExecutionMode = smartAutoExecution?.mode; const isImageTask = autoExecutionMode === "generate_image" || @@ -3954,62 +3980,34 @@ export async function POST(request: Request) { rescueTimeoutMs, ); smartAutoDetectedIntent = rescueAnalysis.intent; - smartAutoNeedsClarification = Boolean(rescueAnalysis.needsClarification); - smartAutoClarificationQuestion = rescueAnalysis.clarifyingQuestions?.[0]; - smartAutoExecution = { - mode: - rescueAnalysis.intent === "video_gen" - ? "generate_video" - : rescueAnalysis.intent === "image_gen" - ? "generate_image" - : rescueAnalysis.intent === "image_edit" - ? "edit_image" - : rescueAnalysis.intent === "vision" - ? "analyze_image" - : rescueAnalysis.needsClarification - ? "clarify" - : "respond", - routeIntent: rescueAnalysis.intent, - routeSubIntent: rescueAnalysis.subIntent, + const arbitratedIntent = arbitrateAgentIntent({ + text: autoTextForFallback, + hasImage: rescueHasImage, + hasHistoryImage: rescueFromHistory || autoFromHistoryForFallback, + actionResolution: resolvedConversationAction, + taskBoundaryDecision, + analysis: rescueAnalysis, + execution: undefined, targetSource: requestTargetResolution?.resolutionMode || "none", - preferredCapability: - rescueAnalysis.intent === "video_gen" - ? "generate_video" - : rescueAnalysis.intent === "image_gen" - ? "generate_image" - : rescueAnalysis.intent === "image_edit" - ? rescueAnalysis.subIntent === "remove_background" - ? "remove_background" - : rescueAnalysis.subIntent === "enhance" - ? "enhance" - : rescueAnalysis.subIntent === "extend" - ? "extend" - : rescueAnalysis.subIntent === "resize" - ? "resize" - : "generic_edit" - : rescueAnalysis.intent === "vision" - ? "analyze" - : "respond", - shouldUseStructuredReply: - rescueAnalysis.intent === "video_gen" || - rescueAnalysis.intent === "image_gen" || - rescueAnalysis.intent === "image_edit", - shouldOptimizePrompt: - rescueAnalysis.intent === "video_gen" || - rescueAnalysis.intent === "image_gen" || - rescueAnalysis.intent === "image_edit", - shouldApplyImagePipeline: - rescueAnalysis.intent === "image_gen" || - rescueAnalysis.intent === "image_edit", - shouldApplyVideoPipeline: rescueAnalysis.intent === "video_gen", - shouldStayConversational: - rescueAnalysis.intent === "chat" || rescueAnalysis.intent === "vision", - shouldPublishToCanvas: false, - toolCalls: [], - availableTools: [], - }; - if (rescueAnalysis.conversationalReply?.trim()) { - smartAutoFriendlyMessage = rescueAnalysis.conversationalReply.trim(); + }); + if (arbitratedIntent.overridden) { + console.log("[IntentArbiter] Override rescue intent:", { + fromIntent: rescueAnalysis.intent, + toIntent: arbitratedIntent.analysis.intent, + toMode: arbitratedIntent.execution.mode, + reason: arbitratedIntent.reason, + }); + } + smartAutoDetectedIntent = arbitratedIntent.analysis.intent; + smartAutoNeedsClarification = Boolean(arbitratedIntent.analysis.needsClarification); + smartAutoClarificationQuestion = arbitratedIntent.analysis.clarifyingQuestions?.[0]; + smartAutoExecution = arbitratedIntent.execution as AgentPreparedExecution; + if (arbitratedIntent.friendlyMessage || rescueAnalysis.conversationalReply?.trim()) { + smartAutoFriendlyMessage = ( + arbitratedIntent.friendlyMessage || + rescueAnalysis.conversationalReply || + "" + ).trim(); } } else { console.warn("[Smart Auto Mode] Skip rescue analysis due to pre-response budget"); @@ -4022,61 +4020,35 @@ export async function POST(request: Request) { autoTextForFallback, autoCurrentImageUrlsForFallback.length > 0 || smartAutoHistoryImages.length > 0, ); - smartAutoDetectedIntent = keywordFallback.intent; - smartAutoExecution = { - mode: - keywordFallback.intent === "video_gen" - ? "generate_video" - : keywordFallback.intent === "image_gen" - ? "generate_image" - : keywordFallback.intent === "image_edit" - ? "edit_image" - : keywordFallback.intent === "vision" - ? "analyze_image" - : keywordFallback.needsClarification - ? "clarify" - : "respond", - routeIntent: keywordFallback.intent, - routeSubIntent: keywordFallback.subIntent, + const arbitratedIntent = arbitrateAgentIntent({ + text: autoTextForFallback, + hasImage: + autoCurrentImageUrlsForFallback.length > 0 || smartAutoHistoryImages.length > 0, + hasHistoryImage: smartAutoHistoryImages.length > 0 || autoFromHistoryForFallback, + actionResolution: resolvedConversationAction, + taskBoundaryDecision, + analysis: keywordFallback, + execution: undefined, targetSource: requestTargetResolution?.resolutionMode || "none", - preferredCapability: - keywordFallback.intent === "video_gen" - ? "generate_video" - : keywordFallback.intent === "image_gen" - ? "generate_image" - : keywordFallback.intent === "image_edit" - ? keywordFallback.subIntent === "remove_background" - ? "remove_background" - : keywordFallback.subIntent === "enhance" - ? "enhance" - : keywordFallback.subIntent === "extend" - ? "extend" - : keywordFallback.subIntent === "resize" - ? "resize" - : "generic_edit" - : keywordFallback.intent === "vision" - ? "analyze" - : "respond", - shouldUseStructuredReply: - keywordFallback.intent === "video_gen" || - keywordFallback.intent === "image_gen" || - keywordFallback.intent === "image_edit", - shouldOptimizePrompt: - keywordFallback.intent === "video_gen" || - keywordFallback.intent === "image_gen" || - keywordFallback.intent === "image_edit", - shouldApplyImagePipeline: - keywordFallback.intent === "image_gen" || - keywordFallback.intent === "image_edit", - shouldApplyVideoPipeline: keywordFallback.intent === "video_gen", - shouldStayConversational: - keywordFallback.intent === "chat" || keywordFallback.intent === "vision", - shouldPublishToCanvas: false, - toolCalls: [], - availableTools: [], - }; - if (keywordFallback.conversationalReply?.trim()) { - smartAutoFriendlyMessage = keywordFallback.conversationalReply.trim(); + }); + if (arbitratedIntent.overridden) { + console.log("[IntentArbiter] Override keyword-fallback intent:", { + fromIntent: keywordFallback.intent, + toIntent: arbitratedIntent.analysis.intent, + toMode: arbitratedIntent.execution.mode, + reason: arbitratedIntent.reason, + }); + } + smartAutoDetectedIntent = arbitratedIntent.analysis.intent; + smartAutoNeedsClarification = Boolean(arbitratedIntent.analysis.needsClarification); + smartAutoClarificationQuestion = arbitratedIntent.analysis.clarifyingQuestions?.[0]; + smartAutoExecution = arbitratedIntent.execution as AgentPreparedExecution; + if (arbitratedIntent.friendlyMessage || keywordFallback.conversationalReply?.trim()) { + smartAutoFriendlyMessage = ( + arbitratedIntent.friendlyMessage || + keywordFallback.conversationalReply || + "" + ).trim(); console.warn("[Smart Auto Mode] Applied keyword fallback after rescue failure"); } } catch (keywordFallbackError) { diff --git a/lib/ai/runtime/intent-arbiter.ts b/lib/ai/runtime/intent-arbiter.ts new file mode 100644 index 000000000..a1e257222 --- /dev/null +++ b/lib/ai/runtime/intent-arbiter.ts @@ -0,0 +1,365 @@ +import "server-only"; + +import type { ConversationActionResolution } from "@/lib/ai/conversation-action-resolver"; +import type { ResolvedTargetAssets } from "@/lib/ai/context-memory/types"; +import type { AnalysisResult } from "@/lib/ai/smart-analyzer"; +import type { TaskBoundaryDecision } from "@/lib/ai/runtime/task-boundary-classifier"; + +export type IntentArbiterMode = + | "respond" + | "clarify" + | "analyze_image" + | "generate_image" + | "edit_image" + | "generate_video"; + +export type IntentArbiterPreferredCapability = + | "respond" + | "analyze" + | "generic_edit" + | "remove_background" + | "enhance" + | "extend" + | "resize" + | "box_cutout" + | "generate_image" + | "generate_video" + | "publish_canvas"; + +export interface IntentArbiterExecution { + mode: IntentArbiterMode; + routeIntent: AnalysisResult["intent"]; + routeSubIntent?: string; + targetSource: ResolvedTargetAssets["resolutionMode"]; + preferredCapability: IntentArbiterPreferredCapability; + shouldUseStructuredReply: boolean; + shouldOptimizePrompt: boolean; + shouldApplyImagePipeline: boolean; + shouldApplyVideoPipeline: boolean; + shouldStayConversational: boolean; + shouldPublishToCanvas: boolean; + toolCalls: Array<{ name: string; reason?: string; args?: Record }>; + availableTools: string[]; +} + +export interface IntentArbiterResult { + analysis: AnalysisResult; + execution: IntentArbiterExecution; + overridden: boolean; + reason: string; + friendlyMessage?: string; +} + +function normalizeCompactText(text: string): string { + return String(text || "").toLowerCase().replace(/\s+/g, "").trim(); +} + +function isLikelyAdvisoryOrPlanningRequest(text: string): boolean { + const normalizedText = String(text || "").toLowerCase(); + return /怎么|如何|为什么|是否|可不可以|能不能|行不行|建议|思路|方案|流程|步骤|教程|推荐|对比|区别|差别|优缺点|评估|分析|判断|解释|讲讲|科普|适合吗|可行吗|值不值得|要不要|帮我看看|帮我分析|帮我判断/.test( + normalizedText, + ); +} + +function isExplicitVideoGenerationRequest(text: string): boolean { + const normalizedText = normalizeCompactText(text); + if (!normalizedText) return false; + const hasStrongVideoMarkers = + /图生视频|文生视频|首帧|尾帧|分镜|运镜|视频生成/.test(normalizedText); + if (hasStrongVideoMarkers) return true; + if (isLikelyAdvisoryOrPlanningRequest(text)) return false; + return ( + /让.+动起来/.test(normalizedText) || + (/(生成|制作|做|创建|输出|渲染|合成|出|做成|变成|转成)/.test(normalizedText) && + /(视频|动画|短片|片子|动起来|会动)/.test(normalizedText)) + ); +} + +function isExplicitImageGenerationRequest(text: string): boolean { + const normalizedText = normalizeCompactText(text); + if (!normalizedText || isLikelyAdvisoryOrPlanningRequest(text)) return false; + return /重新生成|再来一张|生成新图|新建图片|创作一张|生成一张|画一张|给我画|帮我画|出一张|做一张图|做一张海报|创建一张图片|参考.*生成|按这个风格生成|以这张图为灵感生成|仿照.*生成|生成一只|生成一个|生成一幅|生成一版|生成一套|生成一组|帮我生成.*(?:图|图片|海报|头像|插画|场景|小猫|猫咪|人物|产品图)|给我生成.*(?:图|图片|海报|头像|插画|场景|小猫|猫咪|人物|产品图)/.test( + normalizedText, + ); +} + +function isLikelyImageEditRequest(text: string): boolean { + const normalizedText = String(text || "").toLowerCase(); + return /抠图|去背景|透明背景|去背|扣图|移除背景|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test( + normalizedText, + ); +} + +function inferDefaultImageEditSubIntent(text: string): string { + const lowerText = String(text || "").toLowerCase(); + if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background"; + if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background"; + if (/风格|卡通|动漫|油画|像素风|写实|插画/.test(lowerText)) return "style_transfer"; + if (/颜色|调色|亮度|对比度|饱和度|改成.*色|换成.*色/.test(lowerText)) return "color_adjust"; + if (/去除|消除|擦掉|删除|移除|p掉|重绘/.test(lowerText)) return "inpaint"; + return "enhance"; +} + +function mapIntentToMode( + intent: AnalysisResult["intent"], + needsClarification?: boolean, +): IntentArbiterMode { + if (needsClarification) return "clarify"; + switch (intent) { + case "image_edit": + return "edit_image"; + case "image_gen": + return "generate_image"; + case "video_gen": + return "generate_video"; + case "vision": + return "analyze_image"; + case "chat": + default: + return "respond"; + } +} + +function mapPreferredCapability( + intent: AnalysisResult["intent"], + subIntent?: string, +): IntentArbiterPreferredCapability { + if (intent === "video_gen") return "generate_video"; + if (intent === "image_gen") return "generate_image"; + if (intent === "vision") return "analyze"; + if (intent === "chat") return "respond"; + + switch (subIntent) { + case "remove_background": + return "remove_background"; + case "extend": + return "extend"; + case "resize": + return "resize"; + case "box_cutout": + return "box_cutout"; + case "enhance": + return "enhance"; + default: + return "generic_edit"; + } +} + +function buildExecutionFromAnalysis(params: { + analysis: AnalysisResult; + execution?: IntentArbiterExecution | null; + targetSource: ResolvedTargetAssets["resolutionMode"]; +}): IntentArbiterExecution { + const mode = mapIntentToMode(params.analysis.intent, params.analysis.needsClarification); + + return { + mode, + routeIntent: params.analysis.intent, + routeSubIntent: params.analysis.subIntent, + targetSource: params.targetSource, + preferredCapability: mapPreferredCapability(params.analysis.intent, params.analysis.subIntent), + shouldUseStructuredReply: + params.analysis.intent === "image_edit" || + params.analysis.intent === "image_gen" || + params.analysis.intent === "video_gen", + shouldOptimizePrompt: + params.analysis.intent === "image_edit" || + params.analysis.intent === "image_gen" || + params.analysis.intent === "video_gen", + shouldApplyImagePipeline: + params.analysis.intent === "image_edit" || params.analysis.intent === "image_gen", + shouldApplyVideoPipeline: params.analysis.intent === "video_gen", + shouldStayConversational: + mode === "respond" || mode === "analyze_image" || mode === "clarify", + shouldPublishToCanvas: params.execution?.shouldPublishToCanvas || false, + toolCalls: params.execution?.toolCalls || [], + availableTools: params.execution?.availableTools || [], + }; +} + +function buildDefaultFriendlyMessage( + intent: AnalysisResult["intent"], + params: { hasImage: boolean; text: string }, +) { + switch (intent) { + case "image_gen": + return params.hasImage ? "收到，我会结合你提供的参考图直接执行生图。" : "好的，我来直接帮你生成这张图。"; + case "image_edit": + return params.hasImage ? "收到，我会按你的目标直接处理这张图。" : "收到，我来继续执行这轮图片处理。"; + case "video_gen": + return params.hasImage ? "收到，我会结合这轮图片先规划并执行视频生成。" : "收到，我先按你的需求规划并执行视频生成。"; + case "vision": + return params.hasImage ? "收到，我先分析这张图。" : "收到，我先按你的问题做分析。"; + case "chat": + default: + return undefined; + } +} + +function resolvePolicyIntent(params: { + text: string; + hasImage: boolean; + hasHistoryImage: boolean; + taskBoundaryDecision: TaskBoundaryDecision; + actionResolution: ConversationActionResolution; + analysis: AnalysisResult; +}): { intent: AnalysisResult["intent"]; subIntent?: string; reason: string } { + const explicitImageGen = isExplicitImageGenerationRequest(params.text); + const explicitVideoGen = isExplicitVideoGenerationRequest(params.text); + const likelyEdit = isLikelyImageEditRequest(params.text); + const hasUsableImage = params.hasImage || params.hasHistoryImage; + + if (explicitVideoGen) { + return { + intent: "video_gen", + reason: "explicit_video_generation_request", + }; + } + + switch (params.actionResolution.kind) { + case "image_edit_followup": + return { + intent: "image_edit", + subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text), + reason: "conversation_action_image_edit_followup", + }; + case "image_generation_followup": + case "prompt_carry_forward": + return { + intent: "image_gen", + reason: "conversation_action_image_generation_followup", + }; + default: + break; + } + + switch (params.taskBoundaryDecision.taskType) { + case "prompt_only": + case "analysis_only": + return { + intent: "vision", + reason: `task_boundary_${params.taskBoundaryDecision.taskType}`, + }; + case "execution_video": + return { + intent: "video_gen", + reason: "task_boundary_execution_video", + }; + case "execution_image": + if (hasUsableImage && likelyEdit && !explicitImageGen) { + return { + intent: "image_edit", + subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text), + reason: "task_boundary_execution_image_edit", + }; + } + return { + intent: "image_gen", + reason: "task_boundary_execution_image_generate", + }; + case "workflow_followup": + if (hasUsableImage && likelyEdit) { + return { + intent: "image_edit", + subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text), + reason: "task_boundary_workflow_followup_edit", + }; + } + if (explicitImageGen) { + return { + intent: "image_gen", + reason: "task_boundary_workflow_followup_generate", + }; + } + break; + default: + break; + } + + if (explicitImageGen) { + return { + intent: "image_gen", + reason: "explicit_image_generation_request", + }; + } + + if (hasUsableImage && likelyEdit && params.analysis.intent === "chat") { + return { + intent: "image_edit", + subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text), + reason: "image_edit_policy_override", + }; + } + + return { + intent: params.analysis.intent, + subIntent: + params.analysis.intent === "image_edit" + ? params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text) + : params.analysis.subIntent, + reason: "keep_ai_analysis", + }; +} + +export function arbitrateAgentIntent(params: { + text: string; + hasImage: boolean; + hasHistoryImage: boolean; + actionResolution: ConversationActionResolution; + taskBoundaryDecision: TaskBoundaryDecision; + analysis: AnalysisResult; + execution?: IntentArbiterExecution | null; + targetSource: ResolvedTargetAssets["resolutionMode"]; +}): IntentArbiterResult { + const resolved = resolvePolicyIntent(params); + const nextAnalysis: AnalysisResult = { + ...params.analysis, + intent: resolved.intent, + subIntent: + resolved.intent === "image_edit" + ? resolved.subIntent || inferDefaultImageEditSubIntent(params.text) + : undefined, + confidence: + resolved.reason === "keep_ai_analysis" + ? params.analysis.confidence + : Math.max(params.analysis.confidence, 0.84), + needsClarification: + resolved.reason === "keep_ai_analysis" + ? Boolean(params.analysis.needsClarification) + : false, + clarifyingQuestions: + resolved.reason === "keep_ai_analysis" && params.analysis.needsClarification + ? params.analysis.clarifyingQuestions || [] + : [], + }; + const nextExecution = buildExecutionFromAnalysis({ + analysis: nextAnalysis, + execution: params.execution, + targetSource: params.targetSource, + }); + const overridden = + nextAnalysis.intent !== params.analysis.intent || + nextAnalysis.subIntent !== params.analysis.subIntent || + nextExecution.mode !== params.execution?.mode || + nextExecution.routeIntent !== params.execution?.routeIntent; + const friendlyMessage = + overridden || !params.analysis.conversationalReply?.trim() + ? buildDefaultFriendlyMessage(nextAnalysis.intent, { + hasImage: params.hasImage || params.hasHistoryImage, + text: params.text, + }) + : undefined; + + return { + analysis: friendlyMessage + ? { + ...nextAnalysis, + conversationalReply: friendlyMessage, + } + : nextAnalysis, + execution: nextExecution, + overridden, + reason: resolved.reason, + friendlyMessage, + }; +}