lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add canvas agent orchestration kernel

Browse Source
This commit is contained in:
134 2026-07-07 15:17:43 +08:00
parent 7e1fd15ebd
commit 8f679a100c
6 changed files with 338 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
app/(chat)/api/chat/route.ts
View File

@ -262,6 +262,7 @@ import {
recordProviderHealthSuccess,
shouldAvoidProvider,
} from "@/lib/ai/runtime/provider-health";
import { resolveCanvasAgentRequestPolicy } from "@/lib/ai/runtime/canvas-agent-kernel";
import { arbitrateAgentIntent } from "@/lib/ai/runtime/intent-arbiter";
import { buildAgentRoleIdentityInstruction } from "@/lib/ai/runtime/agent-role-profile";
import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
@ -1095,8 +1096,19 @@ export async function POST(request: Request) {
const shouldRespectDirectComposerMode =
composerMode === "image" || composerMode === "video" || composerMode === "detail";
const shouldIsolateDirectComposerContext = shouldRespectDirectComposerMode;
let shouldRunAgentConversation = finalModelId === "auto";
const requestPolicy = resolveCanvasAgentRequestPolicy({
composerMode,
agentEntryMode: agentPresetId ? "preset" : "auto",
selectedChatModel: finalModelId,
hasCanvasControlIntentFastPath,
});
finalModelId = requestPolicy.effectiveSelectedChatModel;
let shouldRunAgentConversation =
finalModelId === "auto" || requestPolicy.shouldForceToolOrchestration;
let shouldKeepSelectedModel = false;
if (requestPolicy.reason === "agent_auto_composer") {
chatApiLogger.info("[Smart Auto Mode] 🧭 Agent composer forces tool orchestration, ignoring stale selected model");
}
if (shouldRespectDirectComposerMode && shouldRunAgentConversation) {
shouldRunAgentConversation = false;
chatApiLogger.info(`[Smart Auto Mode] ⛳ Direct composer mode (${composerMode}) disables initial auto-agent routing`);

28
components/canvas/CanvasChatPanel.tsx
View File

@ -77,6 +77,7 @@ import {
getImageCreditCostFromConfig,
getImageModelCreditDiscountLabel,
} from "@/lib/ai/image-model-pricing";
import { resolveCanvasAgentRequestPolicy } from "@/lib/ai/runtime/canvas-agent-kernel";
import { getVideoToolOptionsForMode } from "@/lib/ai/video-tool-capability";
import type { ChatMessage } from "@/lib/types";
import type {
@ -5352,14 +5353,8 @@ export function CanvasChatPanel({
}
if (mode === "agent") {
const preferredAgentModelId =
allModels.find(
(model) =>
model.id === lastConversationModelId &&
isCanvasConversationOnlyModel(model)
)?.id || "auto";
currentModelIdRef.current = preferredAgentModelId;
setSelectedModelId(preferredAgentModelId);
currentModelIdRef.current = "auto";
setSelectedModelId("auto");
setIsDetailGenerationMode(false);
return targetChatSessionId;
}
@ -6160,7 +6155,15 @@ export function CanvasChatPanel({
const hiddenPrompt = hiddenPromptRef.current || undefined;
const autoContext = autoContextRef.current || undefined;
const contextPayload = capturedContextPayload;
const requestModel = allModels.find((model) => model.id === currentModelIdRef.current);
const requestPolicy = resolveCanvasAgentRequestPolicy({
composerMode: effectiveComposerMode,
agentEntryMode: effectiveAgentEntryMode,
selectedChatModel: currentModelIdRef.current,
});
const effectiveSelectedChatModel = requestPolicy.effectiveSelectedChatModel;
const requestModel = allModels.find(
(model) => model.id === effectiveSelectedChatModel
);
const requestIsVideoModel = isCanvasVideoModel(requestModel);
const requestIsConversationModel = isCanvasConversationOnlyModel(requestModel);
hiddenPromptRef.current = null;
@ -6177,10 +6180,13 @@ export function CanvasChatPanel({
selectedAgentPreset?.id
? selectedAgentPreset.id
: undefined,
selectedChatModel: currentModelIdRef.current,
selectedChatModel: effectiveSelectedChatModel,
composerMode: effectiveComposerMode,
selectedVisibilityType: "private",
deepThinkingEnabled: requestIsConversationModel
deepThinkingEnabled:
effectiveComposerMode === "agent"
? undefined
: requestIsConversationModel
? false
: requestIsVideoModel || currentModelIdRef.current === "auto"
? undefined

53
lib/ai/context-memory/target-resolution.ts
View File

@ -5,16 +5,17 @@ import type {
ResolvedImageRef,
ResolvedTargetAssets,
} from "@/lib/ai/context-memory/types";
import {
hasCanvasAgentActionableImageEditIntent,
hasCanvasAgentDeicticTarget,
resolveCanvasAgentTargetPreference,
} from "@/lib/ai/runtime/canvas-agent-kernel";
const ORIGINAL_IMAGE_REGEX =
/(原图|上传图|参考图|最初那张|第一张图|素材图|源图|原始图片|原始那张|我发的那张|上传的那张|原来的图)/i;
const LATEST_RESULT_REGEX =
/(上一张|上一轮|刚生成|最新生成|这个结果|该结果|按这个结果|继续在这张|基于这张结果|延续上一次结果|上一版|刚才那张结果|生成的图|生成图|结果图|出图结果|刚出的图|上一张生成图|上一轮结果图|现在这张生成图)/i;
const DEICTIC_TARGET_REGEX =
/(这张图|这只|这个|这款|这套|当前这张|当前这个|当前这只|这一张|这一只|这个产品|这个物体|这只猫|这只狗)/i;
const SELECTION_REGEX = /(选中的|当前这张|我选的|这几张|这些图|选中图)/i;
const ACTIONABLE_EDIT_REGEX =
/(改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格|亮一点|暗一点|白底|透明底)/i;
const ACTIONABLE_GENERATION_REGEX =
/(生图|生成|画一张|出图|来一张|再来一张|做一张|做图|做视频|视频生成|生成视频|动起来)/i;
const IMPLICIT_HISTORY_TARGET_REGEX =
@ -255,6 +256,7 @@ export function resolveTargetAssets(
const focusPolicy = params.contextPayload?.focusPolicy || "auto";
const workingStateGeneratedUrls = normalizeStringArray(workingState?.lastGeneratedImageUrls);
const workingStateUploadedUrls = normalizeStringArray(workingState?.lastUploadedImageUrls);
const workingStateResolvedUrls = normalizeStringArray(workingState?.lastResolvedTargetImageUrls);
const generatedRefs =
workingStateGeneratedUrls.length > 0 ? toWorkingMemoryRefs(workingStateGeneratedUrls) : assistantRefs;
const originalRefs =
@ -384,7 +386,39 @@ export function resolveTargetAssets(
});
}
if (DEICTIC_TARGET_REGEX.test(text)) {
const canvasAgentTargetPreference = resolveCanvasAgentTargetPreference({
text,
selectedImageCount: selectedRefs.length,
latestResultImageCount: generatedRefs.length,
workingMemoryTargetImageCount: workingStateResolvedUrls.length,
});
if (canvasAgentTargetPreference.shouldPreferLatestResult) {
return buildResolvedTargetAssets({
targetRefs: generatedRefs,
candidateRefs,
resolutionMode:
workingStateGeneratedUrls.length > 0 ? "working_memory" : "latest_result",
confidence: canvasAgentTargetPreference.confidence,
needsClarification: false,
reason: "用户使用指代表达并提出图片编辑意图，默认续接最近生成结果",
fromHistory: true,
});
}
if (canvasAgentTargetPreference.shouldPreferWorkingMemoryTarget) {
const refs = toWorkingMemoryRefs(workingStateResolvedUrls);
return buildResolvedTargetAssets({
targetRefs: refs,
candidateRefs,
resolutionMode: "working_memory",
confidence: canvasAgentTargetPreference.confidence,
needsClarification: false,
reason: "用户使用指代表达并提出图片编辑意图，使用工作记忆中的上一轮目标图",
fromHistory: true,
});
}
if (hasCanvasAgentDeicticTarget(text)) {
if (selectedRefs.length > 0) {
return buildResolvedTargetAssets({
targetRefs: selectedRefs,
@ -421,7 +455,7 @@ export function resolveTargetAssets(
}
if (
ACTIONABLE_EDIT_REGEX.test(text) &&
hasCanvasAgentActionableImageEditIntent(text) &&
currentRefs.length === 0 &&
generatedRefs.length > 0 &&
selectedLooksLikeOriginal &&
@ -463,14 +497,13 @@ export function resolveTargetAssets(
});
}
const workingStateResolvedUrls = normalizeStringArray(workingState?.lastResolvedTargetImageUrls);
const allowsImplicitHistoryTarget =
IMPLICIT_HISTORY_TARGET_REGEX.test(text) &&
(ACTIONABLE_EDIT_REGEX.test(text) ||
(hasCanvasAgentActionableImageEditIntent(text) ||
ACTIONABLE_GENERATION_REGEX.test(text) ||
LATEST_RESULT_REGEX.test(text) ||
ORIGINAL_IMAGE_REGEX.test(text) ||
DEICTIC_TARGET_REGEX.test(text) ||
hasCanvasAgentDeicticTarget(text) ||
SELECTION_REGEX.test(text));
if (allowsImplicitHistoryTarget) {
@ -525,7 +558,7 @@ export function resolveTargetAssets(
}
}
if (ACTIONABLE_EDIT_REGEX.test(text)) {
if (hasCanvasAgentActionableImageEditIntent(text)) {
return buildResolvedTargetAssets({
targetRefs: [],
candidateRefs,

156
lib/ai/runtime/canvas-agent-kernel.ts Normal file
View File

@ -0,0 +1,156 @@
import type { DirectAgentToolCapability } from "@/lib/ai/runtime/direct-agent-execution";
export type CanvasAgentComposerMode = "agent" | "image" | "video" | "detail" | "clawd";
export type CanvasAgentEntryMode = "auto" | "preset";
export interface CanvasAgentRequestPolicy {
effectiveSelectedChatModel: string;
shouldForceToolOrchestration: boolean;
reason: string | null;
}
export interface CanvasAgentTargetPreference {
shouldPreferLatestResult: boolean;
shouldPreferWorkingMemoryTarget: boolean;
confidence: number;
reason: string | null;
}
const DEICTIC_TARGET_REGEX =
/(这张图|这只|这个|这款|这套|当前这张|当前这个|当前这只|这一张|这一只|这个产品|这个物体|这只猫|这只狗)/i;
const ACTIONABLE_IMAGE_EDIT_REGEX =
/(抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|调色|调整颜色|亮度|对比度|饱和度|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调|处理)/i;
export function normalizeCanvasAgentRequestText(text: string) {
return String(text || "").toLowerCase().replace(/\s+/g, "").trim();
}
export function hasCanvasAgentDeicticTarget(text: string) {
return DEICTIC_TARGET_REGEX.test(String(text || ""));
}
export function hasCanvasAgentActionableImageEditIntent(text: string) {
return ACTIONABLE_IMAGE_EDIT_REGEX.test(String(text || ""));
}
export function inferCanvasAgentImageEditCapability(
text: string,
): DirectAgentToolCapability | null {
const normalizedText = normalizeCanvasAgentRequestText(text);
if (!normalizedText) return null;
if (/去水印|水印|去logo|移除logo|去除logo/.test(normalizedText)) {
return "remove_watermark";
}
if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(normalizedText)) {
return "remove_background";
}
if (/多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位/.test(normalizedText)) {
return "angle_variant";
}
if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|比例|画幅|竖版|横版|竖图|横图|9:16|16:9|1:1/.test(normalizedText)) {
return "extend";
}
if (/调整尺寸|裁剪|resize|crop|缩放|放大|缩小/.test(normalizedText)) {
return "resize";
}
if (/局部|框选|选区|区域|涂抹|蒙版/.test(normalizedText)) {
return "local_edit";
}
if (/高清|清晰|锐化|降噪|提升质量|修复|清晰化/.test(normalizedText)) {
return "enhance";
}
if (hasCanvasAgentActionableImageEditIntent(text)) {
return "generic_edit";
}
return null;
}
export function resolveCanvasAgentRequestPolicy(params: {
composerMode?: string | null;
agentEntryMode?: string | null;
selectedChatModel?: string | null;
hasCanvasControlIntentFastPath?: boolean;
}): CanvasAgentRequestPolicy {
const composerMode = String(params.composerMode || "").trim();
const agentEntryMode = String(params.agentEntryMode || "").trim();
const selectedChatModel = String(params.selectedChatModel || "").trim() || "auto";
const isAgentAutoComposer = composerMode === "agent" && agentEntryMode !== "preset";
const shouldForceToolOrchestration =
isAgentAutoComposer || Boolean(params.hasCanvasControlIntentFastPath);
if (isAgentAutoComposer) {
return {
effectiveSelectedChatModel: "auto",
shouldForceToolOrchestration: true,
reason: "agent_auto_composer",
};
}
if (params.hasCanvasControlIntentFastPath) {
return {
effectiveSelectedChatModel: selectedChatModel,
shouldForceToolOrchestration: true,
reason: "canvas_control_fast_path",
};
}
return {
effectiveSelectedChatModel: selectedChatModel,
shouldForceToolOrchestration,
reason: null,
};
}
export function resolveCanvasAgentTargetPreference(params: {
text: string;
selectedImageCount: number;
latestResultImageCount: number;
workingMemoryTargetImageCount: number;
}): CanvasAgentTargetPreference {
const hasDeicticTarget = hasCanvasAgentDeicticTarget(params.text);
const hasEditIntent = hasCanvasAgentActionableImageEditIntent(params.text);
const selectedImageCount = Math.max(0, Math.round(params.selectedImageCount || 0));
const latestResultImageCount = Math.max(
0,
Math.round(params.latestResultImageCount || 0),
);
const workingMemoryTargetImageCount = Math.max(
0,
Math.round(params.workingMemoryTargetImageCount || 0),
);
if (!hasDeicticTarget || !hasEditIntent || selectedImageCount > 0) {
return {
shouldPreferLatestResult: false,
shouldPreferWorkingMemoryTarget: false,
confidence: 0,
reason: null,
};
}
if (latestResultImageCount > 0) {
return {
shouldPreferLatestResult: true,
shouldPreferWorkingMemoryTarget: false,
confidence: 90,
reason: "deictic_edit_prefers_latest_result",
};
}
if (workingMemoryTargetImageCount > 0) {
return {
shouldPreferLatestResult: false,
shouldPreferWorkingMemoryTarget: true,
confidence: 86,
reason: "deictic_edit_uses_working_memory_target",
};
}
return {
shouldPreferLatestResult: false,
shouldPreferWorkingMemoryTarget: false,
confidence: 0,
reason: null,
};
}

28
lib/ai/runtime/intent-arbiter.ts
View File

@ -5,6 +5,10 @@ import type { ResolvedTargetAssets } from "@/lib/ai/context-memory/types";
import type { AnalysisResult } from "@/lib/ai/smart-analyzer";
import type { TaskBoundaryDecision } from "@/lib/ai/runtime/task-boundary-classifier";
import { hasDirectAgentLocalEditIntent } from "@/lib/ai/runtime/local-edit-options";
import {
hasCanvasAgentActionableImageEditIntent,
inferCanvasAgentImageEditCapability,
} from "@/lib/ai/runtime/canvas-agent-kernel";
export type IntentArbiterMode =
| "respond"
@ -116,23 +120,21 @@ function isExplicitImageGenerationRequest(text: string): boolean {
}
function isLikelyImageEditRequest(text: string): boolean {
const normalizedText = String(text || "").toLowerCase();
return hasDirectAgentLocalEditIntent(text) || /抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test(
normalizedText,
);
return hasDirectAgentLocalEditIntent(text) || hasCanvasAgentActionableImageEditIntent(text);
}
function inferDefaultImageEditSubIntent(text: string): string {
const lowerText = String(text || "").toLowerCase();
if (/去水印|水印|去logo|去\s*logo|移除logo|去除logo/.test(lowerText)) return "remove_watermark";
if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background";
if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background";
if (/多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位/.test(lowerText)) return "angle_variant";
const inferredCapability = inferCanvasAgentImageEditCapability(text);
if (
inferredCapability &&
inferredCapability !== "generic_edit" &&
inferredCapability !== "generate_image" &&
inferredCapability !== "generate_video" &&
inferredCapability !== "analyze"
) {
return inferredCapability;
}
if (hasDirectAgentLocalEditIntent(text)) return "local_edit";
if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|比例|画幅|竖版|横版|竖图|横图|9:16|16:9|1:1/.test(lowerText)) return "extend";
if (/风格|卡通|动漫|油画|像素风|写实|插画/.test(lowerText)) return "style_transfer";
if (/颜色|调色|亮度|对比度|饱和度|改成.*色|换成.*色/.test(lowerText)) return "color_adjust";
if (/去除|消除|擦掉|删除|移除|p掉|重绘/.test(lowerText)) return "inpaint";
return "enhance";
}

94
tests/routes/canvas-chat-helpers.test.ts
View File

@ -45,6 +45,10 @@ import {
} from "@/lib/canvas-toolbar-config";
import { shouldBypassCanvasAgentMissingTargetClarification } from "@/lib/ai/runtime/canvas-agent-clarification";
import { hasCanvasAgentRoutingContext } from "@/lib/ai/canvas-agent-routing";
import {
resolveCanvasAgentRequestPolicy,
resolveCanvasAgentTargetPreference,
} from "@/lib/ai/runtime/canvas-agent-kernel";
import { inferAgentImageCountFromText } from "@/lib/ai/runtime/image-count-intent";
import {
isAgentModeScopedChatId,
@ -128,6 +132,7 @@ import {
getDirectAgentToolSuccessMessage,
hasExplicitDirectAgentBatchImageEditIntent,
isDirectAgentBatchImageEditCapability,
resolveDirectAgentToolContext,
resolveDirectAgentToolExecution,
shouldRunDirectAgentBatchImageEdit,
} from "@/lib/ai/runtime/direct-agent-execution";
@ -1205,6 +1210,95 @@ test.describe("canvas chat pure helpers", () => {
expect(editText).toContain("替换背景");
});
test("resolves deictic canvas edit requests to latest results and direct tools", () => {
const latestResultUrl = "https://cdn.example.com/latest-cat-result.png";
const requestPolicy = resolveCanvasAgentRequestPolicy({
composerMode: "agent",
selectedChatModel: "conversation-model-id",
});
expect(requestPolicy.effectiveSelectedChatModel).toBe("auto");
expect(requestPolicy.shouldForceToolOrchestration).toBe(true);
const targetPreference = resolveCanvasAgentTargetPreference({
text: "将这只猫从图中扣图扣出来",
selectedImageCount: 0,
latestResultImageCount: 1,
workingMemoryTargetImageCount: 1,
});
expect(targetPreference.shouldPreferLatestResult).toBe(true);
const mediaInput = {
imageRefs: [],
currentMessageImageRefs: [],
selectedImageRefs: [],
canvasImageRefs: [],
historyUserImageRefs: [],
historyAssistantImageRefs: [
{
url: latestResultUrl,
normalizedUrl: latestResultUrl,
source: "history_assistant_result",
role: "history_result",
},
],
resolvedImageRefs: [],
currentMessageImageUrls: [],
selectedImageUrls: [],
historyUserImageUrls: [],
historyAssistantImageUrls: [latestResultUrl],
resolvedImageUrls: [latestResultUrl],
selectedAssetIds: [],
candidateAssetIds: [],
uploadedAssetIds: [],
hasImage: true,
source: "history",
fromHistory: true,
} as any;
const workingState = {
lastGeneratedImageUrls: [latestResultUrl],
lastResolvedTargetImageUrls: [latestResultUrl],
};
const targetResolution = {
targetRefs: [],
candidateRefs: [],
targetImageUrls: [latestResultUrl],
targetAssetIds: [],
candidateImageUrls: [],
candidateAssetIds: [],
resolutionMode: "working_memory",
confidence: targetPreference.confidence,
needsClarification: false,
reason: targetPreference.reason,
fromHistory: true,
} as any;
const directContext = resolveDirectAgentToolContext({
execution: {
mode: "edit_image",
routeIntent: "image_edit",
routeSubIntent: "remove_background",
targetSource: targetResolution.resolutionMode,
preferredCapability: "remove_background",
shouldUseStructuredReply: true,
shouldOptimizePrompt: true,
shouldApplyImagePipeline: true,
shouldApplyVideoPipeline: false,
shouldStayConversational: false,
shouldPublishToCanvas: false,
toolCalls: [],
availableTools: ["removeBackground"],
} as any,
requestResolvedMediaInput: mediaInput,
requestTargetResolution: targetResolution,
requestWorkingState: workingState,
});
expect(directContext?.execution.toolName).toBe("removeBackground");
expect(directContext?.targetImageUrl).toBe(latestResultUrl);
});
test("only runs direct image edit batches for explicit safe batch requests", () => {
expect(
shouldRunDirectAgentBatchImageEdit({