diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index df38dbc43..c3be7a608 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -262,6 +262,7 @@ import { recordProviderHealthSuccess, shouldAvoidProvider, } from "@/lib/ai/runtime/provider-health"; +import { resolveCanvasAgentRequestPolicy } from "@/lib/ai/runtime/canvas-agent-kernel"; import { arbitrateAgentIntent } from "@/lib/ai/runtime/intent-arbiter"; import { buildAgentRoleIdentityInstruction } from "@/lib/ai/runtime/agent-role-profile"; import { getVideoStoryboardConfig } from "@/lib/video-agent/config"; @@ -1095,8 +1096,19 @@ export async function POST(request: Request) { const shouldRespectDirectComposerMode = composerMode === "image" || composerMode === "video" || composerMode === "detail"; const shouldIsolateDirectComposerContext = shouldRespectDirectComposerMode; - let shouldRunAgentConversation = finalModelId === "auto"; + const requestPolicy = resolveCanvasAgentRequestPolicy({ + composerMode, + agentEntryMode: agentPresetId ? "preset" : "auto", + selectedChatModel: finalModelId, + hasCanvasControlIntentFastPath, + }); + finalModelId = requestPolicy.effectiveSelectedChatModel; + let shouldRunAgentConversation = + finalModelId === "auto" || requestPolicy.shouldForceToolOrchestration; let shouldKeepSelectedModel = false; + if (requestPolicy.reason === "agent_auto_composer") { + chatApiLogger.info("[Smart Auto Mode] 🧭 Agent composer forces tool orchestration, ignoring stale selected model"); + } if (shouldRespectDirectComposerMode && shouldRunAgentConversation) { shouldRunAgentConversation = false; chatApiLogger.info(`[Smart Auto Mode] ⛳ Direct composer mode (${composerMode}) disables initial auto-agent routing`); diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index e4516c616..8fef5c262 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -77,6 +77,7 @@ import { getImageCreditCostFromConfig, getImageModelCreditDiscountLabel, } from "@/lib/ai/image-model-pricing"; +import { resolveCanvasAgentRequestPolicy } from "@/lib/ai/runtime/canvas-agent-kernel"; import { getVideoToolOptionsForMode } from "@/lib/ai/video-tool-capability"; import type { ChatMessage } from "@/lib/types"; import type { @@ -5352,14 +5353,8 @@ export function CanvasChatPanel({ } if (mode === "agent") { - const preferredAgentModelId = - allModels.find( - (model) => - model.id === lastConversationModelId && - isCanvasConversationOnlyModel(model) - )?.id || "auto"; - currentModelIdRef.current = preferredAgentModelId; - setSelectedModelId(preferredAgentModelId); + currentModelIdRef.current = "auto"; + setSelectedModelId("auto"); setIsDetailGenerationMode(false); return targetChatSessionId; } @@ -6160,7 +6155,15 @@ export function CanvasChatPanel({ const hiddenPrompt = hiddenPromptRef.current || undefined; const autoContext = autoContextRef.current || undefined; const contextPayload = capturedContextPayload; - const requestModel = allModels.find((model) => model.id === currentModelIdRef.current); + const requestPolicy = resolveCanvasAgentRequestPolicy({ + composerMode: effectiveComposerMode, + agentEntryMode: effectiveAgentEntryMode, + selectedChatModel: currentModelIdRef.current, + }); + const effectiveSelectedChatModel = requestPolicy.effectiveSelectedChatModel; + const requestModel = allModels.find( + (model) => model.id === effectiveSelectedChatModel + ); const requestIsVideoModel = isCanvasVideoModel(requestModel); const requestIsConversationModel = isCanvasConversationOnlyModel(requestModel); hiddenPromptRef.current = null; @@ -6177,10 +6180,13 @@ export function CanvasChatPanel({ selectedAgentPreset?.id ? selectedAgentPreset.id : undefined, - selectedChatModel: currentModelIdRef.current, + selectedChatModel: effectiveSelectedChatModel, composerMode: effectiveComposerMode, selectedVisibilityType: "private", - deepThinkingEnabled: requestIsConversationModel + deepThinkingEnabled: + effectiveComposerMode === "agent" + ? undefined + : requestIsConversationModel ? false : requestIsVideoModel || currentModelIdRef.current === "auto" ? undefined diff --git a/lib/ai/context-memory/target-resolution.ts b/lib/ai/context-memory/target-resolution.ts index e0d8f8064..6c69dd7ef 100644 --- a/lib/ai/context-memory/target-resolution.ts +++ b/lib/ai/context-memory/target-resolution.ts @@ -5,16 +5,17 @@ import type { ResolvedImageRef, ResolvedTargetAssets, } from "@/lib/ai/context-memory/types"; +import { + hasCanvasAgentActionableImageEditIntent, + hasCanvasAgentDeicticTarget, + resolveCanvasAgentTargetPreference, +} from "@/lib/ai/runtime/canvas-agent-kernel"; const ORIGINAL_IMAGE_REGEX = /(原图|上传图|参考图|最初那张|第一张图|素材图|源图|原始图片|原始那张|我发的那张|上传的那张|原来的图)/i; const LATEST_RESULT_REGEX = /(上一张|上一轮|刚生成|最新生成|这个结果|该结果|按这个结果|继续在这张|基于这张结果|延续上一次结果|上一版|刚才那张结果|生成的图|生成图|结果图|出图结果|刚出的图|上一张生成图|上一轮结果图|现在这张生成图)/i; -const DEICTIC_TARGET_REGEX = - /(这张图|这只|这个|这款|这套|当前这张|当前这个|当前这只|这一张|这一只|这个产品|这个物体|这只猫|这只狗)/i; const SELECTION_REGEX = /(选中的|当前这张|我选的|这几张|这些图|选中图)/i; -const ACTIONABLE_EDIT_REGEX = - /(改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格|亮一点|暗一点|白底|透明底)/i; const ACTIONABLE_GENERATION_REGEX = /(生图|生成|画一张|出图|来一张|再来一张|做一张|做图|做视频|视频生成|生成视频|动起来)/i; const IMPLICIT_HISTORY_TARGET_REGEX = @@ -255,6 +256,7 @@ export function resolveTargetAssets( const focusPolicy = params.contextPayload?.focusPolicy || "auto"; const workingStateGeneratedUrls = normalizeStringArray(workingState?.lastGeneratedImageUrls); const workingStateUploadedUrls = normalizeStringArray(workingState?.lastUploadedImageUrls); + const workingStateResolvedUrls = normalizeStringArray(workingState?.lastResolvedTargetImageUrls); const generatedRefs = workingStateGeneratedUrls.length > 0 ? toWorkingMemoryRefs(workingStateGeneratedUrls) : assistantRefs; const originalRefs = @@ -384,7 +386,39 @@ export function resolveTargetAssets( }); } - if (DEICTIC_TARGET_REGEX.test(text)) { + const canvasAgentTargetPreference = resolveCanvasAgentTargetPreference({ + text, + selectedImageCount: selectedRefs.length, + latestResultImageCount: generatedRefs.length, + workingMemoryTargetImageCount: workingStateResolvedUrls.length, + }); + if (canvasAgentTargetPreference.shouldPreferLatestResult) { + return buildResolvedTargetAssets({ + targetRefs: generatedRefs, + candidateRefs, + resolutionMode: + workingStateGeneratedUrls.length > 0 ? "working_memory" : "latest_result", + confidence: canvasAgentTargetPreference.confidence, + needsClarification: false, + reason: "用户使用指代表达并提出图片编辑意图,默认续接最近生成结果", + fromHistory: true, + }); + } + + if (canvasAgentTargetPreference.shouldPreferWorkingMemoryTarget) { + const refs = toWorkingMemoryRefs(workingStateResolvedUrls); + return buildResolvedTargetAssets({ + targetRefs: refs, + candidateRefs, + resolutionMode: "working_memory", + confidence: canvasAgentTargetPreference.confidence, + needsClarification: false, + reason: "用户使用指代表达并提出图片编辑意图,使用工作记忆中的上一轮目标图", + fromHistory: true, + }); + } + + if (hasCanvasAgentDeicticTarget(text)) { if (selectedRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: selectedRefs, @@ -421,7 +455,7 @@ export function resolveTargetAssets( } if ( - ACTIONABLE_EDIT_REGEX.test(text) && + hasCanvasAgentActionableImageEditIntent(text) && currentRefs.length === 0 && generatedRefs.length > 0 && selectedLooksLikeOriginal && @@ -463,14 +497,13 @@ export function resolveTargetAssets( }); } - const workingStateResolvedUrls = normalizeStringArray(workingState?.lastResolvedTargetImageUrls); const allowsImplicitHistoryTarget = IMPLICIT_HISTORY_TARGET_REGEX.test(text) && - (ACTIONABLE_EDIT_REGEX.test(text) || + (hasCanvasAgentActionableImageEditIntent(text) || ACTIONABLE_GENERATION_REGEX.test(text) || LATEST_RESULT_REGEX.test(text) || ORIGINAL_IMAGE_REGEX.test(text) || - DEICTIC_TARGET_REGEX.test(text) || + hasCanvasAgentDeicticTarget(text) || SELECTION_REGEX.test(text)); if (allowsImplicitHistoryTarget) { @@ -525,7 +558,7 @@ export function resolveTargetAssets( } } - if (ACTIONABLE_EDIT_REGEX.test(text)) { + if (hasCanvasAgentActionableImageEditIntent(text)) { return buildResolvedTargetAssets({ targetRefs: [], candidateRefs, diff --git a/lib/ai/runtime/canvas-agent-kernel.ts b/lib/ai/runtime/canvas-agent-kernel.ts new file mode 100644 index 000000000..68467384f --- /dev/null +++ b/lib/ai/runtime/canvas-agent-kernel.ts @@ -0,0 +1,156 @@ +import type { DirectAgentToolCapability } from "@/lib/ai/runtime/direct-agent-execution"; + +export type CanvasAgentComposerMode = "agent" | "image" | "video" | "detail" | "clawd"; +export type CanvasAgentEntryMode = "auto" | "preset"; + +export interface CanvasAgentRequestPolicy { + effectiveSelectedChatModel: string; + shouldForceToolOrchestration: boolean; + reason: string | null; +} + +export interface CanvasAgentTargetPreference { + shouldPreferLatestResult: boolean; + shouldPreferWorkingMemoryTarget: boolean; + confidence: number; + reason: string | null; +} + +const DEICTIC_TARGET_REGEX = + /(这张图|这只|这个|这款|这套|当前这张|当前这个|当前这只|这一张|这一只|这个产品|这个物体|这只猫|这只狗)/i; + +const ACTIONABLE_IMAGE_EDIT_REGEX = + /(抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|调色|调整颜色|亮度|对比度|饱和度|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调|处理)/i; + +export function normalizeCanvasAgentRequestText(text: string) { + return String(text || "").toLowerCase().replace(/\s+/g, "").trim(); +} + +export function hasCanvasAgentDeicticTarget(text: string) { + return DEICTIC_TARGET_REGEX.test(String(text || "")); +} + +export function hasCanvasAgentActionableImageEditIntent(text: string) { + return ACTIONABLE_IMAGE_EDIT_REGEX.test(String(text || "")); +} + +export function inferCanvasAgentImageEditCapability( + text: string, +): DirectAgentToolCapability | null { + const normalizedText = normalizeCanvasAgentRequestText(text); + if (!normalizedText) return null; + if (/去水印|水印|去logo|移除logo|去除logo/.test(normalizedText)) { + return "remove_watermark"; + } + if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(normalizedText)) { + return "remove_background"; + } + if (/多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位/.test(normalizedText)) { + return "angle_variant"; + } + if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|比例|画幅|竖版|横版|竖图|横图|9:16|16:9|1:1/.test(normalizedText)) { + return "extend"; + } + if (/调整尺寸|裁剪|resize|crop|缩放|放大|缩小/.test(normalizedText)) { + return "resize"; + } + if (/局部|框选|选区|区域|涂抹|蒙版/.test(normalizedText)) { + return "local_edit"; + } + if (/高清|清晰|锐化|降噪|提升质量|修复|清晰化/.test(normalizedText)) { + return "enhance"; + } + if (hasCanvasAgentActionableImageEditIntent(text)) { + return "generic_edit"; + } + return null; +} + +export function resolveCanvasAgentRequestPolicy(params: { + composerMode?: string | null; + agentEntryMode?: string | null; + selectedChatModel?: string | null; + hasCanvasControlIntentFastPath?: boolean; +}): CanvasAgentRequestPolicy { + const composerMode = String(params.composerMode || "").trim(); + const agentEntryMode = String(params.agentEntryMode || "").trim(); + const selectedChatModel = String(params.selectedChatModel || "").trim() || "auto"; + const isAgentAutoComposer = composerMode === "agent" && agentEntryMode !== "preset"; + const shouldForceToolOrchestration = + isAgentAutoComposer || Boolean(params.hasCanvasControlIntentFastPath); + + if (isAgentAutoComposer) { + return { + effectiveSelectedChatModel: "auto", + shouldForceToolOrchestration: true, + reason: "agent_auto_composer", + }; + } + + if (params.hasCanvasControlIntentFastPath) { + return { + effectiveSelectedChatModel: selectedChatModel, + shouldForceToolOrchestration: true, + reason: "canvas_control_fast_path", + }; + } + + return { + effectiveSelectedChatModel: selectedChatModel, + shouldForceToolOrchestration, + reason: null, + }; +} + +export function resolveCanvasAgentTargetPreference(params: { + text: string; + selectedImageCount: number; + latestResultImageCount: number; + workingMemoryTargetImageCount: number; +}): CanvasAgentTargetPreference { + const hasDeicticTarget = hasCanvasAgentDeicticTarget(params.text); + const hasEditIntent = hasCanvasAgentActionableImageEditIntent(params.text); + const selectedImageCount = Math.max(0, Math.round(params.selectedImageCount || 0)); + const latestResultImageCount = Math.max( + 0, + Math.round(params.latestResultImageCount || 0), + ); + const workingMemoryTargetImageCount = Math.max( + 0, + Math.round(params.workingMemoryTargetImageCount || 0), + ); + + if (!hasDeicticTarget || !hasEditIntent || selectedImageCount > 0) { + return { + shouldPreferLatestResult: false, + shouldPreferWorkingMemoryTarget: false, + confidence: 0, + reason: null, + }; + } + + if (latestResultImageCount > 0) { + return { + shouldPreferLatestResult: true, + shouldPreferWorkingMemoryTarget: false, + confidence: 90, + reason: "deictic_edit_prefers_latest_result", + }; + } + + if (workingMemoryTargetImageCount > 0) { + return { + shouldPreferLatestResult: false, + shouldPreferWorkingMemoryTarget: true, + confidence: 86, + reason: "deictic_edit_uses_working_memory_target", + }; + } + + return { + shouldPreferLatestResult: false, + shouldPreferWorkingMemoryTarget: false, + confidence: 0, + reason: null, + }; +} diff --git a/lib/ai/runtime/intent-arbiter.ts b/lib/ai/runtime/intent-arbiter.ts index cfd463d3b..ce1081ef1 100644 --- a/lib/ai/runtime/intent-arbiter.ts +++ b/lib/ai/runtime/intent-arbiter.ts @@ -5,6 +5,10 @@ import type { ResolvedTargetAssets } from "@/lib/ai/context-memory/types"; import type { AnalysisResult } from "@/lib/ai/smart-analyzer"; import type { TaskBoundaryDecision } from "@/lib/ai/runtime/task-boundary-classifier"; import { hasDirectAgentLocalEditIntent } from "@/lib/ai/runtime/local-edit-options"; +import { + hasCanvasAgentActionableImageEditIntent, + inferCanvasAgentImageEditCapability, +} from "@/lib/ai/runtime/canvas-agent-kernel"; export type IntentArbiterMode = | "respond" @@ -116,23 +120,21 @@ function isExplicitImageGenerationRequest(text: string): boolean { } function isLikelyImageEditRequest(text: string): boolean { - const normalizedText = String(text || "").toLowerCase(); - return hasDirectAgentLocalEditIntent(text) || /抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test( - normalizedText, - ); + return hasDirectAgentLocalEditIntent(text) || hasCanvasAgentActionableImageEditIntent(text); } function inferDefaultImageEditSubIntent(text: string): string { - const lowerText = String(text || "").toLowerCase(); - if (/去水印|水印|去logo|去\s*logo|移除logo|去除logo/.test(lowerText)) return "remove_watermark"; - if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background"; - if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background"; - if (/多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位/.test(lowerText)) return "angle_variant"; + const inferredCapability = inferCanvasAgentImageEditCapability(text); + if ( + inferredCapability && + inferredCapability !== "generic_edit" && + inferredCapability !== "generate_image" && + inferredCapability !== "generate_video" && + inferredCapability !== "analyze" + ) { + return inferredCapability; + } if (hasDirectAgentLocalEditIntent(text)) return "local_edit"; - if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|比例|画幅|竖版|横版|竖图|横图|9:16|16:9|1:1/.test(lowerText)) return "extend"; - if (/风格|卡通|动漫|油画|像素风|写实|插画/.test(lowerText)) return "style_transfer"; - if (/颜色|调色|亮度|对比度|饱和度|改成.*色|换成.*色/.test(lowerText)) return "color_adjust"; - if (/去除|消除|擦掉|删除|移除|p掉|重绘/.test(lowerText)) return "inpaint"; return "enhance"; } diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index c4d257b53..1ed8f6a5a 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -45,6 +45,10 @@ import { } from "@/lib/canvas-toolbar-config"; import { shouldBypassCanvasAgentMissingTargetClarification } from "@/lib/ai/runtime/canvas-agent-clarification"; import { hasCanvasAgentRoutingContext } from "@/lib/ai/canvas-agent-routing"; +import { + resolveCanvasAgentRequestPolicy, + resolveCanvasAgentTargetPreference, +} from "@/lib/ai/runtime/canvas-agent-kernel"; import { inferAgentImageCountFromText } from "@/lib/ai/runtime/image-count-intent"; import { isAgentModeScopedChatId, @@ -128,6 +132,7 @@ import { getDirectAgentToolSuccessMessage, hasExplicitDirectAgentBatchImageEditIntent, isDirectAgentBatchImageEditCapability, + resolveDirectAgentToolContext, resolveDirectAgentToolExecution, shouldRunDirectAgentBatchImageEdit, } from "@/lib/ai/runtime/direct-agent-execution"; @@ -1205,6 +1210,95 @@ test.describe("canvas chat pure helpers", () => { expect(editText).toContain("替换背景"); }); + test("resolves deictic canvas edit requests to latest results and direct tools", () => { + const latestResultUrl = "https://cdn.example.com/latest-cat-result.png"; + const requestPolicy = resolveCanvasAgentRequestPolicy({ + composerMode: "agent", + selectedChatModel: "conversation-model-id", + }); + + expect(requestPolicy.effectiveSelectedChatModel).toBe("auto"); + expect(requestPolicy.shouldForceToolOrchestration).toBe(true); + + const targetPreference = resolveCanvasAgentTargetPreference({ + text: "将这只猫从图中扣图扣出来", + selectedImageCount: 0, + latestResultImageCount: 1, + workingMemoryTargetImageCount: 1, + }); + + expect(targetPreference.shouldPreferLatestResult).toBe(true); + + const mediaInput = { + imageRefs: [], + currentMessageImageRefs: [], + selectedImageRefs: [], + canvasImageRefs: [], + historyUserImageRefs: [], + historyAssistantImageRefs: [ + { + url: latestResultUrl, + normalizedUrl: latestResultUrl, + source: "history_assistant_result", + role: "history_result", + }, + ], + resolvedImageRefs: [], + currentMessageImageUrls: [], + selectedImageUrls: [], + historyUserImageUrls: [], + historyAssistantImageUrls: [latestResultUrl], + resolvedImageUrls: [latestResultUrl], + selectedAssetIds: [], + candidateAssetIds: [], + uploadedAssetIds: [], + hasImage: true, + source: "history", + fromHistory: true, + } as any; + const workingState = { + lastGeneratedImageUrls: [latestResultUrl], + lastResolvedTargetImageUrls: [latestResultUrl], + }; + const targetResolution = { + targetRefs: [], + candidateRefs: [], + targetImageUrls: [latestResultUrl], + targetAssetIds: [], + candidateImageUrls: [], + candidateAssetIds: [], + resolutionMode: "working_memory", + confidence: targetPreference.confidence, + needsClarification: false, + reason: targetPreference.reason, + fromHistory: true, + } as any; + + const directContext = resolveDirectAgentToolContext({ + execution: { + mode: "edit_image", + routeIntent: "image_edit", + routeSubIntent: "remove_background", + targetSource: targetResolution.resolutionMode, + preferredCapability: "remove_background", + shouldUseStructuredReply: true, + shouldOptimizePrompt: true, + shouldApplyImagePipeline: true, + shouldApplyVideoPipeline: false, + shouldStayConversational: false, + shouldPublishToCanvas: false, + toolCalls: [], + availableTools: ["removeBackground"], + } as any, + requestResolvedMediaInput: mediaInput, + requestTargetResolution: targetResolution, + requestWorkingState: workingState, + }); + + expect(directContext?.execution.toolName).toBe("removeBackground"); + expect(directContext?.targetImageUrl).toBe(latestResultUrl); + }); + test("only runs direct image edit batches for explicit safe batch requests", () => { expect( shouldRunDirectAgentBatchImageEdit({