From 9ecd91376d16bc352d0e1430cb59e09d6174774a Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 16 Mar 2026 11:45:00 +0800 Subject: [PATCH] feat: strengthen agent runtime boundaries --- app/(chat)/api/chat/route.ts | 312 ++++++-- ...ent真正变聪明-系统级整改清单-2026-03-16.md | 667 ++++++++++++++++++ lib/ai/conversation-memory-composer.ts | 14 +- lib/ai/conversation-state-protocol.ts | 67 +- lib/ai/runtime/agent-model-slots.ts | 108 +++ lib/ai/runtime/memory-relevance.ts | 134 ++++ lib/ai/runtime/provider-health.ts | 109 +++ lib/ai/runtime/task-boundary-classifier.ts | 226 ++++++ .../runtime/task-boundary-context-pruner.ts | 169 +++++ 9 files changed, 1743 insertions(+), 63 deletions(-) create mode 100644 docs/Agent真正变聪明-系统级整改清单-2026-03-16.md create mode 100644 lib/ai/runtime/agent-model-slots.ts create mode 100644 lib/ai/runtime/memory-relevance.ts create mode 100644 lib/ai/runtime/provider-health.ts create mode 100644 lib/ai/runtime/task-boundary-classifier.ts create mode 100644 lib/ai/runtime/task-boundary-context-pruner.ts diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index f9e111e09..d74b34b11 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -123,11 +123,26 @@ import type { } from "@/lib/ai/context-memory/types"; import { ChatSDKError } from "@/lib/errors"; import { extractImageUrlsFromText } from "@/lib/image-url"; -import { isExplicitPromptOptimizationRequest } from "@/lib/ai/smart-analyzer"; import { resolveClarificationBypass, resolveClarificationMessage, } from "@/lib/ai/runtime/clarification-interruption"; +import { + pruneTaskBoundaryContext, + pruneTaskBoundaryHistoryMessages, +} from "@/lib/ai/runtime/task-boundary-context-pruner"; +import { + resolveAgentModelSlotDescriptor, + upsertAgentModelSlot, + type AgentModelSlots, +} from "@/lib/ai/runtime/agent-model-slots"; +import { isPureGreetingTurnRequest } from "@/lib/ai/runtime/task-boundary-classifier"; +import { + getProviderHealthSnapshot, + recordProviderHealthFailure, + recordProviderHealthSuccess, + shouldAvoidProvider, +} from "@/lib/ai/runtime/provider-health"; import { uploadFile } from "@/lib/storage/service"; import { getVideoStoryboardConfig } from "@/lib/video-agent/config"; import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline"; @@ -1234,15 +1249,6 @@ function normalizeAssistantPartsForPersistence(parts: Array> }); } -function isPureGreetingTurnRequest(text: string): boolean { - const normalized = String(text || "").toLowerCase().replace(/\s+/g, "").trim(); - if (!normalized) return false; - - return /^(你好|您好|hello|hi|嗨|哈喽|在吗|早上好|中午好|下午好|晚上好|深夜好|谢谢|感谢|收到|好的|ok|okay|明白了|了解了)$/.test( - normalized, - ); -} - function buildAgentToolImageMarkdown(params: { originalUrl: string; thumbnailUrl?: string | null; @@ -3044,8 +3050,10 @@ export async function POST(request: Request) { let reusablePromptMemory = null; let confirmedPromptForConversationState: string | undefined; let smartAutoHistoryMessagesFromDb: DBMessage[] = []; + let smartAutoMediaHistoryMessagesFromDb: DBMessage[] = []; let preferredAgentModelId: string | undefined; let configuredAgentConversationModelId: string | undefined; + let agentModelSlots: AgentModelSlots = {}; let pendingGridConfirmation: | PendingGridConfirmation | null = null; @@ -3153,6 +3161,24 @@ export async function POST(request: Request) { if (configuredAgentModel?.model?.id) { preferredAgentModelId = configuredAgentModel.model.id; configuredAgentConversationModelId = configuredAgentModel.model.id; + agentModelSlots = upsertAgentModelSlot(agentModelSlots, "planner", { + modelId: configuredAgentModel.model.id, + displayName: configuredAgentModel.model.displayName || configuredAgentModel.model.modelId, + providerId: configuredAgentModel.model.providerId, + providerType: configuredAgentModel.provider.type, + providerKey: configuredAgentModel.model.providerId, + source: "configured_agent_default", + reason: "agent_default_model", + }); + agentModelSlots = upsertAgentModelSlot(agentModelSlots, "analyzer", { + modelId: configuredAgentModel.model.id, + displayName: configuredAgentModel.model.displayName || configuredAgentModel.model.modelId, + providerId: configuredAgentModel.model.providerId, + providerType: configuredAgentModel.provider.type, + providerKey: configuredAgentModel.model.providerId, + source: "configured_agent_default", + reason: "agent_default_model", + }); console.log( `[Smart Auto Mode] 🧠 Loaded configured agent conversation model: ${configuredAgentModel.model.displayName}`, ); @@ -3239,9 +3265,12 @@ export async function POST(request: Request) { workingState: requestWorkingState, activeStoryboardTask: activeStoryboardTaskForChat, hasImageInputForPromptCarryForward: hasImageInputForConversationAction, + hasCanvasSelection: Boolean(contextPayload?.selectedNodeIds?.length), }); console.log("[ConversationStateProtocol] Bootstrap:", { action: conversationStateBootstrap.actionResolution.kind, + taskType: conversationStateBootstrap.taskBoundaryDecision.taskType, + taskMode: conversationStateBootstrap.taskBoundaryDecision.taskMode, hasPromptMemory: Boolean(conversationStateBootstrap.promptMemory?.optimizedPrompt), hasImageWorkflowMemory: Boolean(conversationStateBootstrap.imageWorkflowMemory), hasStoryboardPreferenceMemory: Boolean(conversationStateBootstrap.storyboardPreferenceMemory), @@ -3252,6 +3281,7 @@ export async function POST(request: Request) { phase: "bootstrap", payload: { action: conversationStateBootstrap.actionResolution.kind, + taskBoundary: conversationStateBootstrap.taskBoundaryDecision, pendingGridConfirmation: conversationStateBootstrap.pendingGridConfirmation, hasPromptMemory: Boolean(conversationStateBootstrap.promptMemory?.optimizedPrompt), hasImageWorkflowMemory: Boolean(conversationStateBootstrap.imageWorkflowMemory), @@ -3265,6 +3295,17 @@ export async function POST(request: Request) { pendingGridConfirmation = conversationStateBootstrap.pendingGridConfirmation; confirmedPromptForConversationState = conversationStateBootstrap.confirmedPrompt; const resolvedConversationAction = conversationStateBootstrap.actionResolution; + const taskBoundaryDecision = conversationStateBootstrap.taskBoundaryDecision; + const prunedTaskBoundaryContext = pruneTaskBoundaryContext({ + decision: taskBoundaryDecision, + contextPayload: contextPayload || undefined, + autoContextText: autoContextText || undefined, + workingState: requestWorkingState || undefined, + hasCurrentImageInput: hasImageInputForConversationAction, + }); + const effectiveContextPayload = prunedTaskBoundaryContext.contextPayload; + const effectiveAutoContextText = prunedTaskBoundaryContext.autoContextText || ""; + const effectiveWorkingState = prunedTaskBoundaryContext.workingState; if (resolvedConversationAction.forceAgentConversation && !shouldRunAgentConversation) { shouldRunAgentConversation = true; @@ -3288,13 +3329,13 @@ export async function POST(request: Request) { } const shouldFastTrackPromptOnly = - shouldRunAgentConversation && isExplicitPromptOptimizationRequest(requestTextContent); + shouldRunAgentConversation && taskBoundaryDecision.taskType === "prompt_only"; if (shouldFastTrackPromptOnly) { if (!requestResolvedMediaInput) { requestResolvedMediaInput = resolveChatMediaInput({ messageParts: message.parts as any[], historyMessages: [], - contextPayload: contextPayload || undefined, + contextPayload: effectiveContextPayload, requestOrigin, maxImages: 4, }); @@ -3304,8 +3345,8 @@ export async function POST(request: Request) { requestTargetResolution = resolveTargetAssets({ text: requestTextContent, mediaInput: requestResolvedMediaInput, - contextPayload: contextPayload || undefined, - workingState: requestWorkingState || undefined, + contextPayload: effectiveContextPayload, + workingState: effectiveWorkingState || undefined, }); } @@ -3329,8 +3370,26 @@ export async function POST(request: Request) { try { // 🆕 首先获取历史消息（在 Smart Auto Mode 内部单独获取，避免依赖后面的 messagesFromDb） - smartAutoHistoryMessagesFromDb = await getMessagesByChatId({ id }); - console.log("[Smart Auto Mode] Loaded", smartAutoHistoryMessagesFromDb.length, "messages from DB"); + const loadedHistoryMessages = await getMessagesByChatId({ id }); + smartAutoHistoryMessagesFromDb = pruneTaskBoundaryHistoryMessages({ + decision: taskBoundaryDecision, + messages: loadedHistoryMessages, + purpose: "model", + hasCurrentImageInput: hasImageInputForConversationAction, + }); + smartAutoMediaHistoryMessagesFromDb = pruneTaskBoundaryHistoryMessages({ + decision: taskBoundaryDecision, + messages: loadedHistoryMessages, + purpose: "media", + hasCurrentImageInput: hasImageInputForConversationAction, + }); + console.log("[Smart Auto Mode] Loaded", loadedHistoryMessages.length, "messages from DB"); + console.log("[Smart Auto Mode] Pruned history:", { + modelMessages: smartAutoHistoryMessagesFromDb.length, + mediaMessages: smartAutoMediaHistoryMessagesFromDb.length, + taskType: taskBoundaryDecision.taskType, + taskMode: taskBoundaryDecision.taskMode, + }); autoTextForFallback = smartAutoInputText; if (!requestWorkingState) { @@ -3347,8 +3406,8 @@ export async function POST(request: Request) { if (!requestResolvedMediaInput) { requestResolvedMediaInput = resolveChatMediaInput({ messageParts: message.parts as any[], - historyMessages: smartAutoHistoryMessagesFromDb, - contextPayload: contextPayload || undefined, + historyMessages: smartAutoMediaHistoryMessagesFromDb, + contextPayload: effectiveContextPayload, requestOrigin, maxImages: 4, }); @@ -3357,14 +3416,14 @@ export async function POST(request: Request) { requestTargetResolution = resolveTargetAssets({ text: requestTextContent, mediaInput: requestResolvedMediaInput, - contextPayload: contextPayload || undefined, - workingState: requestWorkingState || undefined, + contextPayload: effectiveContextPayload, + workingState: effectiveWorkingState || undefined, }); } const hasSelectedImageHint = - Boolean(contextPayload?.selectedNodeIds?.length) || - /用户已选中\s*\d+\s*张图片/.test(autoContextText); + Boolean(effectiveContextPayload?.selectedNodeIds?.length) || + /用户已选中\s*\d+\s*张图片/.test(effectiveAutoContextText); const hasExplicitTargetHint = requestTargetResolution.resolutionMode === "text_reference" || /第[一二三四五六七八九十\d]+张|左边|左侧|左上|左下|右边|右侧|右上|右下|上面|上方|下面|下方|中间|居中|中央|选中/.test(requestTextContent); @@ -3438,7 +3497,7 @@ export async function POST(request: Request) { } const composedConversationState = composeConversationStateProtocol({ bootstrap: conversationStateBootstrap, - autoContextText: autoContextText || undefined, + autoContextText: effectiveAutoContextText || undefined, resolvedMediaInput: requestResolvedMediaInput, targetResolution: requestTargetResolution, }); @@ -3462,16 +3521,16 @@ export async function POST(request: Request) { hasImage, imageUrls, assetIds: Array.from( - new Set([ - ...extractMessageImageAssetIds(message.parts as any[]), - ...((contextPayload?.selectedAssetIds || []).filter(Boolean) as string[]), - ]), - ), + new Set([ + ...extractMessageImageAssetIds(message.parts as any[]), + ...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]), + ]), + ), imageFromHistory: smartAutoHistoryImages.length > 0, targetResolutionMode: requestTargetResolution.resolutionMode, mediaSource: requestResolvedMediaInput.source, hasCanvasSelection: - Boolean(contextPayload?.selectedNodeIds?.length) || + Boolean(effectiveContextPayload?.selectedNodeIds?.length) || requestResolvedMediaInput.selectedImageUrls.length > 0, selectedImageCount: requestResolvedMediaInput.selectedImageUrls.length, historyImageCount: @@ -3481,7 +3540,7 @@ export async function POST(request: Request) { historyMessages, context: composedConversationState.composedConversationMemory.contextSummary || - autoContextText || + effectiveAutoContextText || undefined, preferredModelId: preferredAgentModelId, threadState: composedConversationState.threadState, @@ -3838,7 +3897,7 @@ export async function POST(request: Request) { const rescueAssetIds = Array.from( new Set([ ...extractMessageImageAssetIds(message.parts as any[]), - ...((contextPayload?.selectedAssetIds || []).filter(Boolean) as string[]), + ...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]), ]), ); const rescueTimeoutMs = getBudgetedTimeout({ @@ -4043,7 +4102,7 @@ export async function POST(request: Request) { const summaryAssetIds = Array.from( new Set([ ...extractMessageImageAssetIds(message.parts as any[]), - ...((contextPayload?.selectedAssetIds || []).filter(Boolean) as string[]), + ...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]), ]), ); const fallbackSummaryTimeoutMs = getBudgetedTimeout({ @@ -4056,7 +4115,7 @@ export async function POST(request: Request) { summarizeImageForExecutionReport({ text: autoTextForFallback, imageUrls: summaryImageUrls, - context: typeof autoContext === "string" ? autoContext : undefined, + context: effectiveAutoContextText || undefined, preferredModelId: preferredAgentModelId, assetIds: summaryAssetIds, }), @@ -4335,11 +4394,11 @@ export async function POST(request: Request) { .map((part) => part.text) .join(" "); const summaryAssetIds = Array.from( - new Set([ - ...extractMessageImageAssetIds(message.parts as any[]), - ...((contextPayload?.selectedAssetIds || []).filter(Boolean) as string[]), - ]), - ); + new Set([ + ...extractMessageImageAssetIds(message.parts as any[]), + ...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]), + ]), + ); let summary: string | null = null; const summaryTimeoutMs = getBudgetedTimeout({ @@ -4352,7 +4411,7 @@ export async function POST(request: Request) { summarizeImageForExecutionReport({ text: latestUserText, imageUrls: summaryImageUrls, - context: typeof autoContext === "string" ? autoContext : undefined, + context: effectiveAutoContextText || undefined, preferredModelId: preferredAgentModelId, assetIds: summaryAssetIds, }), @@ -4421,6 +4480,86 @@ export async function POST(request: Request) { } } + if (shouldRunAgentConversation && smartAutoExecution?.shouldStayConversational) { + const candidateReplyDescriptor = await resolveAgentModelSlotDescriptor(finalModelId).catch( + () => null, + ); + if (candidateReplyDescriptor) { + agentModelSlots = upsertAgentModelSlot(agentModelSlots, "reply", { + ...candidateReplyDescriptor, + source: "candidate_reply_model", + reason: "post_agent_routing", + locked: shouldKeepSelectedModel, + }); + + const providerHealth = getProviderHealthSnapshot(candidateReplyDescriptor.providerKey); + if ( + !shouldKeepSelectedModel && + shouldAvoidProvider(providerHealth) + ) { + let fallbackReplyModelId: string | undefined; + + if ( + preferredAgentModelId && + preferredAgentModelId !== finalModelId + ) { + const preferredDescriptor = await resolveAgentModelSlotDescriptor( + preferredAgentModelId, + ).catch(() => null); + if ( + preferredDescriptor && + !shouldAvoidProvider(getProviderHealthSnapshot(preferredDescriptor.providerKey)) + ) { + fallbackReplyModelId = preferredAgentModelId; + } + } + + if (!fallbackReplyModelId) { + try { + const { routeToModel } = await import("@/lib/ai/model-router"); + const replyFallbackRoute = await routeToModel({ + intent: "chat", + subIntent: undefined, + hasImage: false, + }); + const fallbackDescriptor = await resolveAgentModelSlotDescriptor( + replyFallbackRoute.id, + ).catch(() => null); + if ( + fallbackDescriptor && + !shouldAvoidProvider(getProviderHealthSnapshot(fallbackDescriptor.providerKey)) + ) { + fallbackReplyModelId = replyFallbackRoute.id; + } + } catch (replyRouteError) { + console.warn( + "[Smart Auto Mode] Failed to resolve reply fallback route after provider degradation:", + replyRouteError, + ); + } + } + + if (fallbackReplyModelId && fallbackReplyModelId !== finalModelId) { + finalModelId = fallbackReplyModelId; + const fallbackDescriptor = await resolveAgentModelSlotDescriptor(finalModelId).catch( + () => null, + ); + if (fallbackDescriptor) { + agentModelSlots = upsertAgentModelSlot(agentModelSlots, "reply", { + ...fallbackDescriptor, + source: "provider_health_fallback", + reason: providerHealth?.lastEventType || "degraded_provider", + locked: false, + }); + } + console.warn( + `[Smart Auto Mode] ♻️ Switched reply model due provider health: ${candidateReplyDescriptor.displayName || candidateReplyDescriptor.modelId} -> ${fallbackDescriptor?.displayName || fallbackReplyModelId}`, + ); + } + } + } + } + const session = await auth(); if (!session?.user) { @@ -4474,6 +4613,7 @@ export async function POST(request: Request) { let chat = await getChatById({ id }); let messagesFromDb: DBMessage[] = []; + let messagesForMediaInput: DBMessage[] = []; if (chat) { if (chat.userId !== session.user.id) { @@ -4522,6 +4662,20 @@ export async function POST(request: Request) { messagesFromDb = await getMessagesByChatId({ id }); } + const loadedMessagesForBoundary = messagesFromDb; + messagesFromDb = pruneTaskBoundaryHistoryMessages({ + decision: taskBoundaryDecision, + messages: loadedMessagesForBoundary, + purpose: "model", + hasCurrentImageInput: hasImageInputForConversationAction, + }); + messagesForMediaInput = pruneTaskBoundaryHistoryMessages({ + decision: taskBoundaryDecision, + messages: loadedMessagesForBoundary, + purpose: "media", + hasCurrentImageInput: hasImageInputForConversationAction, + }); + if (!requestWorkingState) { requestWorkingState = await getConversationWorkingStateByChatId({ chatId: id, @@ -4530,8 +4684,8 @@ export async function POST(request: Request) { if (!requestResolvedMediaInput) { requestResolvedMediaInput = resolveChatMediaInput({ messageParts: message.parts as any[], - historyMessages: messagesFromDb, - contextPayload: contextPayload || undefined, + historyMessages: messagesForMediaInput, + contextPayload: effectiveContextPayload, requestOrigin, maxImages: 4, }); @@ -4540,8 +4694,8 @@ export async function POST(request: Request) { requestTargetResolution = resolveTargetAssets({ text: requestTextContent, mediaInput: requestResolvedMediaInput, - contextPayload: contextPayload || undefined, - workingState: requestWorkingState || undefined, + contextPayload: effectiveContextPayload, + workingState: effectiveWorkingState || undefined, }); } if ( @@ -4688,7 +4842,7 @@ export async function POST(request: Request) { shouldRunAgentConversation && !smartAutoNeedsClarification && promptOnlySourceImageUrls.length > 0 && - isExplicitPromptOptimizationRequest(requestTextContent); + taskBoundaryDecision.taskType === "prompt_only"; if (wantsPromptOnlyFromImageContext) { let promptOnlyImageSummary = ""; @@ -4701,16 +4855,16 @@ export async function POST(request: Request) { }); if (summaryTimeoutMs) { const summaryAssetIds = Array.from( - new Set([ - ...extractMessageImageAssetIds(message.parts as any[]), - ...((contextPayload?.selectedAssetIds || []).filter(Boolean) as string[]), - ]), - ); + new Set([ + ...extractMessageImageAssetIds(message.parts as any[]), + ...((effectiveContextPayload?.selectedAssetIds || []).filter(Boolean) as string[]), + ]), + ); const summary = await withOperationTimeout( summarizeImageForExecutionReport({ text: requestTextContent, imageUrls: promptOnlySourceImageUrls, - context: typeof autoContext === "string" ? autoContext : undefined, + context: effectiveAutoContextText || undefined, preferredModelId: preferredAgentModelId, assetIds: promptOnlySourceAssetIds.length > 0 ? promptOnlySourceAssetIds : summaryAssetIds, @@ -4752,7 +4906,7 @@ export async function POST(request: Request) { }; }) .filter(Boolean) as Array<{ role: "user" | "assistant"; content: string }>, - context: typeof autoContext === "string" ? autoContext : undefined, + context: effectiveAutoContextText || undefined, }).catch(() => optimizePrompt({ originalPrompt: promptSeed, @@ -4832,7 +4986,7 @@ export async function POST(request: Request) { part?.type === "file" || part?.type === "image" || part?.type === "image_url" ), hasHistoryImageInput: smartAutoHistoryImages.length > 0, - hasCanvasSelection: Boolean(contextPayload?.selectedNodeIds?.length), + hasCanvasSelection: Boolean(effectiveContextPayload?.selectedNodeIds?.length), targetResolutionMode: requestTargetResolution?.resolutionMode, executionMode: smartAutoExecution?.mode, friendlyMessage: smartAutoFriendlyMessage, @@ -6495,6 +6649,7 @@ export async function POST(request: Request) { let actualModelId: string; let resolvedModelSupportsImageGen = false; let resolvedProviderType: string | undefined; + let resolvedRuntimeModelLookupId = finalModelId; if (isUUID) { // 从数据库获取动态配置的模型 @@ -6542,6 +6697,7 @@ export async function POST(request: Request) { ).toResponse(); } languageModel = dynamicModel; + resolvedRuntimeModelLookupId = targetModelId; // 对于动态模型，我们需要从数据库获取 modelId const finalModelData = await getAiModelById({ id: targetModelId }); actualModelId = finalModelData?.modelId ?? targetModelId; @@ -6556,6 +6712,38 @@ export async function POST(request: Request) { languageModel = myProvider.languageModel(finalModelId); actualModelId = languageModel.modelId; resolvedProviderType = (languageModel as any)?.provider; + resolvedRuntimeModelLookupId = actualModelId; + } + + const activeRuntimeModelDescriptor = await resolveAgentModelSlotDescriptor( + resolvedRuntimeModelLookupId, + ).catch(() => null); + if (shouldRunAgentConversation && activeRuntimeModelDescriptor) { + const slotName = + smartAutoExecution?.shouldApplyImagePipeline || + smartAutoExecution?.shouldApplyVideoPipeline + ? "execution" + : "reply"; + agentModelSlots = upsertAgentModelSlot(agentModelSlots, slotName, { + ...activeRuntimeModelDescriptor, + source: "runtime_selected_model", + reason: + slotName === "reply" + ? smartAutoExecution?.shouldStayConversational + ? "conversational_reply_runtime" + : "default_reply_runtime" + : "execution_runtime_model", + locked: slotName === "reply" ? shouldKeepSelectedModel : false, + }); + pendingProtocolDebugEvents.push({ + phase: "modelSlots", + payload: { + modelSlots: agentModelSlots, + providerHealth: activeRuntimeModelDescriptor.providerKey + ? getProviderHealthSnapshot(activeRuntimeModelDescriptor.providerKey) + : null, + }, + }); } // ========== 图片生成配额检查 ========== @@ -9301,6 +9489,12 @@ export async function POST(request: Request) { functionId: "stream-text", }, onFinish: async ({ usage }) => { + if (activeRuntimeModelDescriptor?.providerKey) { + recordProviderHealthSuccess({ + providerKey: activeRuntimeModelDescriptor.providerKey, + providerType: activeRuntimeModelDescriptor.providerType, + }); + } try { const providers = await getTokenlensCatalog(); if (!actualModelId) { @@ -9343,6 +9537,13 @@ export async function POST(request: Request) { result.consumeStream({ onError: (error) => { console.error("[Chat API] streamText consume error:", error); + if (activeRuntimeModelDescriptor?.providerKey) { + recordProviderHealthFailure({ + providerKey: activeRuntimeModelDescriptor.providerKey, + providerType: activeRuntimeModelDescriptor.providerType, + error, + }); + } }, }); @@ -9351,6 +9552,13 @@ export async function POST(request: Request) { sendReasoning: true, onError: (error) => { console.error("[Chat API] streamText ui stream error:", error); + if (activeRuntimeModelDescriptor?.providerKey) { + recordProviderHealthFailure({ + providerKey: activeRuntimeModelDescriptor.providerKey, + providerType: activeRuntimeModelDescriptor.providerType, + error, + }); + } return getChatStreamErrorMessage(error); }, }) diff --git a/docs/Agent真正变聪明-系统级整改清单-2026-03-16.md b/docs/Agent真正变聪明-系统级整改清单-2026-03-16.md new file mode 100644 index 000000000..f15005181 --- /dev/null +++ b/docs/Agent真正变聪明-系统级整改清单-2026-03-16.md @@ -0,0 +1,667 @@ +# Agent 真正变聪明：系统级整改清单（2026-03-16） + +> 目标：不是继续给 Agent 打零散补丁，而是把它从“很多聪明 patch 的组合体”，升级成“统一、稳定、可解释、可扩展的对话智能系统”。 +> +> 适用范围： +> - `app/(chat)/api/chat/route.ts` +> - `lib/ai/auto-agent-graph.ts` +> - `lib/ai/smart-analyzer.ts` +> - `lib/ai/conversation-state-protocol.ts` +> - `lib/ai/conversation-*` +> - `lib/ai/runtime/*` +> - `lib/video-agent/*` +> - 画布上下文、图片工作流、视频工作流 + +--- + +## 0. 先说结论 + +### 0.1 现在 Agent 为什么“看起来不聪明” + +不是单纯模型不够强，而是系统层面还有 5 个根问题： + +1. **任务边界不稳定** + - 新任务、继续任务、只聊天、只看图、只要提示词、直接执行，经常互相串。 + +2. **上下文污染严重** + - 当前输入、历史图片、画布图片、上一轮 workflow、长期记忆，经常一起灌给模型。 + - 结果就是用户说一句“你好”，系统也可能把它理解成“继续上一轮修图”。 + +3. **理解层和执行层没完全解耦** + - 一部分逻辑已经进 protocol / subagent / runtime。 + - 但还有不少判断仍然散在 `chat/route.ts`，导致行为不稳定、修改成本高。 + +4. **模型链路不够统一** + - planner、analyzer、最终回复、图片执行、视频执行并不是同一套稳定规则。 + - 之前就出现过：Agent 默认模型是豆包，但最终对话回复走了别的供应商。 + +5. **超时退化不够“保正确”** + - 一旦 `runAutoAgentGraph`、图像总结、Provider 流式返回超时，系统容易退化成“还能跑，但不对题”。 + +### 0.2 我们这次整改的目标 + +这次不是“继续修 10 个 bug”，而是把 Agent 升级成： + +- **先判断任务，再选择上下文** +- **先选正确路线，再决定模型和工具** +- **宁可简单，也不跑偏** +- **即使失败，也要明确告诉用户为什么失败** + +--- + +## 1. 这次整改的总原则 + +### 1.1 一个原则：先判任务，再喂上下文 + +以后不能再用“把所有上下文先塞给模型，再让模型自己判断”的方式。 + +要改成： + +1. 先判断这轮是： + - 新任务 + - 连续任务 + - 只分析 + - 只要提示词 + - 直接执行 + - 聊天寒暄 +2. 再按这个任务类型，裁剪要喂给模型的上下文 + +### 1.2 一个原则：执行型和对话型必须分开 + +以后要明确分两类： + +- **对话型回合** + - 聊天 + - 看图分析 + - 提示词整理 + - 澄清确认 + +- **执行型回合** + - 生图 + - 修图 + - 生视频 + - 分镜确认执行 + +对话型回合不能轻易被旧工作流拖进执行型链路。 + +### 1.3 一个原则：失败也要“可解释” + +以后不允许再出现这几种体验： + +- 用户发了消息，看起来“没反应” +- 数据库里只存了 keepalive +- 模型报错了，但前端没显示 +- 回退了，但用户不知道系统为什么回退 + +--- + +## 2. 总体架构目标 + +目标架构： + +1. **Task Boundary Layer** +2. **Context Pruning Layer** +3. **Conversation State Protocol** +4. **Planner / Subagent Layer** +5. **Execution Gate Layer** +6. **Persistence / Event Layer** +7. **Frontend Projection Layer** + +一句话解释： + +- **边界层**决定“你这轮到底要干嘛” +- **裁剪层**决定“该给模型看哪些上下文” +- **协议层**负责统一状态读写 +- **子代理层**负责专业领域判断 +- **执行门控层**负责确认、取消、修改、超时回退 +- **事件层**负责可追踪 +- **前端层**负责把系统理解结果明确展示给用户 + +--- + +## 3. 五大整改块 + +--- + +## 3.1 任务理解层 + +### 3.1.1 目标 + +让 Agent 先稳定判断： + +- 这是新任务还是继续任务 +- 这是对话型还是执行型 +- 这是只分析、只要提示词、还是直接执行 + +### 3.1.2 当前问题 + +当前系统虽然已经有： + +- `conversation-action-resolver` +- `smart-analyzer` +- `skill-router` +- `auto-agent-graph` + +但这几层对“任务边界”的定义还不完全统一。 + +典型问题： + +1. “你好”被旧图片 workflow 拖成连续修图 +2. “帮我写这张图的提示词”被拖成生图/修图流程 +3. 用户只是问“这个图怎么写提示词”，系统却提前进入执行链 + +### 3.1.3 必做改造 + +#### A. 新增统一任务边界分类器 + +建议新增： + +- `lib/ai/runtime/task-boundary-classifier.ts` + +输出统一结构： + +```ts +type TaskBoundaryDecision = { + taskType: + | "greeting" + | "chat" + | "analysis_only" + | "prompt_only" + | "execution_image" + | "execution_video" + | "workflow_followup"; + taskMode: "new" | "continue" | "revise" | "confirm" | "cancel"; + shouldUseCanvasContext: boolean; + shouldUseHistoryImages: boolean; + shouldUseWorkflowMemory: boolean; + shouldAllowExecution: boolean; + confidence: number; + reason: string; +}; +``` + +#### B. 明确任务优先级规则 + +优先级必须写死在系统里，不能全靠模型猜： + +1. `prompt_only` > `workflow_followup` +2. `analysis_only` > `execution_image` +3. `greeting` > 任意旧 workflow +4. `confirm/cancel/revise` > 普通分析 +5. 当前上传图 > 历史图 > 画布背景图 + +#### C. 给“非执行任务”开快路径 + +以下场景应该直接走快路径： + +- 纯打招呼 +- 只要提示词 +- 只看图分析 +- 只问建议 + +这些回合不应该再先跑完整 `runAutoAgentGraph`。 + +### 3.1.4 落点文件 + +- `app/(chat)/api/chat/route.ts` +- `lib/ai/conversation-action-resolver.ts` +- `lib/ai/smart-analyzer.ts` +- `lib/ai/runtime/task-boundary-classifier.ts`（新增） + +### 3.1.5 验收标准 + +- “你好”永远不被识别为连续修图/连续生图 +- “帮我写这张图的提示词”永远不直接触发执行 +- “按照刚才那个提示词，再生成 4 张”会稳定识别为连续生图执行 + +--- + +## 3.2 记忆层 + +### 3.2.1 目标 + +让 Agent 的记忆从“混在一起”变成“分层可控”。 + +### 3.2.2 当前问题 + +现在虽然已经有： + +- `promptMemory` +- `imageWorkflowMemory` +- `storyboardPreferenceMemory` +- `threadState` +- `workingState` + +但问题是： + +1. 历史 workflow 容易污染当前回合 +2. 画布上下文太大，容易把简单任务拖重 +3. “当前相关记忆”和“长期偏好记忆”还没完全分层 + +### 3.2.3 必做改造 + +#### A. 拆成 4 层记忆 + +1. **Turn Memory** + - 当前回合输入 + - 当前上传图片 + - 当前画布选中 + +2. **Task Memory** + - 当前任务工作流状态 + - 当前任务目标图 + - 当前任务优化提示词 + +3. **Session Memory** + - 当前会话最近 3~5 个高相关任务摘要 + +4. **Preference Memory** + - 用户长期偏好 + - 常用比例 + - 常用宫格 + - 常用确认习惯 + +#### B. 增加“记忆衰减/降权”机制 + +如果旧 workflow 满足任一条件，应自动降权： + +- 超过 N 轮未继续 +- 当前上传了新图 +- 当前任务类型和旧任务类型冲突 +- 用户显式说“重新来 / 新开一个 / 别按刚才那个” + +#### C. 建立 relevance score + +建议新增： + +- `lib/ai/runtime/memory-relevance.ts` + +对候选记忆打分： + +- 当前上传图命中：+100 +- 当前选中图命中：+90 +- 显式引用“上一张 / 第三张 / 这个提示词”：+80 +- 同类任务：+40 +- 同会话最近 3 轮：+20 +- 超过 10 轮未使用：-50 + +以后只把 top-k 记忆喂给 Planner。 + +### 3.2.4 落点文件 + +- `lib/ai/conversation-memory-composer.ts` +- `lib/ai/conversation-thread-state-builder.ts` +- `lib/ai/runtime/agent-thread-state.ts` +- `lib/ai/runtime/memory-relevance.ts`（新增） + +### 3.2.5 验收标准 + +- 同一会话里，旧图不再轻易干扰新上传图任务 +- 简单聊天不再带上大段无关画布上下文 +- “继续上一个提示词”仍然能稳定命中 + +--- + +## 3.3 路由层 + +### 3.3.1 目标 + +让 Planner、Analyzer、最终回复模型、执行模型之间的职责彻底清晰。 + +### 3.3.2 当前问题 + +现在的问题不是“没有路由”，而是“路由规则不够统一”： + +1. planner 模型和 reply 模型曾经不一致 +2. analyzer 有时会被别的 provider 抢走 +3. 生图/修图/视频执行模型与对话模型混杂 +4. provider 报错或额度不足时，回退不够统一 + +### 3.3.3 必做改造 + +#### A. 明确 4 类模型槽位 + +统一定义： + +1. `plannerModel` +2. `analyzerModel` +3. `replyModel` +4. `executionModel` + +而不是所有地方都混着叫“当前模型”。 + +#### B. Agent 模式默认规则 + +在 `selectedChatModel = auto` 或进入 Agent 模式时： + +- `plannerModel` = Agent 默认模型 +- `analyzerModel` = Agent 默认模型优先，多模态能力不足时再回退 +- `replyModel` = Agent 默认模型 +- `executionModel` = 按任务类型路由 + +#### C. Provider 健康与额度状态缓存 + +建议新增： + +- `lib/ai/runtime/provider-health.ts` + +缓存内容： + +- 最近错误类型 +- 最近额度不足 +- 最近超时率 +- 最近 5 分钟成功率 + +如果某 provider 连续报： + +- `quota not enough` +- `403` +- `timeout` + +则在短时间内直接降级，避免用户连续踩坑。 + +### 3.3.4 落点文件 + +- `app/(chat)/api/chat/route.ts` +- `lib/ai/model-router.ts` +- `lib/ai/smart-analyzer.ts` +- `lib/ai/auto-agent-graph.ts` +- `lib/ai/runtime/provider-health.ts`（新增） + +### 3.3.5 验收标准 + +- Agent 模式不会再出现“规划走豆包、回复走别家”的错位 +- 上游 provider 额度不足时，能快速回退或明确报错 +- 对话模型、执行模型职责清晰 + +--- + +## 3.4 执行层 + +### 3.4.1 目标 + +让执行系统遵守： + +- 该确认时必须确认 +- 该取消时必须停 +- 该提示时只提示，不执行 +- 该失败时明确失败 + +### 3.4.2 当前问题 + +虽然我们已经做了很多 protocol / workflow 事件： + +- clarification +- storyboard confirm +- revision +- cancel +- execution started/completed +- video generation started/completed/failed + +但执行层还存在 3 个结构问题： + +1. 快路径不够多 +2. timeout fallback 仍偏重 +3. “对话型输出”和“执行型输出”没有完全剥离 + +### 3.4.3 必做改造 + +#### A. 建立 execution policy gate + +建议新增： + +- `lib/ai/runtime/execution-policy-gate.ts` + +统一做这些判定： + +- 当前回合是否允许执行 +- 当前回合是否必须先确认 +- 当前回合是否应该只输出提示词 +- 当前回合是否应该只对话 + +#### B. 扩展 fast-path 列表 + +必须内建以下快路径： + +1. greeting fast-path +2. prompt-only fast-path +3. analysis-only fast-path +4. confirmation fast-path +5. cancel fast-path + +#### C. timeout 退化策略升级 + +以后 timeout 退化规则应改成： + +- `execute` 超时 → 不自动换任务类型 +- `planner` 超时 → 退到简单规则，但不跨任务边界 +- `provider` 超时 → 直接明确告诉用户该 provider 超时 + +一句话： + +**退化后可以更笨，但不能更错。** + +### 3.4.4 落点文件 + +- `lib/ai/runtime/middlewares/human-confirmation-gate.ts` +- `lib/ai/runtime/execution-policy-gate.ts`（新增） +- `app/(chat)/api/chat/route.ts` +- `lib/video-agent/storyboard-chat-followup.ts` + +### 3.4.5 验收标准 + +- “只要提示词”绝不执行生图 +- “确认开始生成”绝不只回复聊天 +- provider 报错时前端总能看到明确原因 + +--- + +## 3.5 前端交互层 + +### 3.5.1 目标 + +让用户知道 Agent 当前理解成了什么，而不是靠猜。 + +### 3.5.2 当前问题 + +现在前端可见性还不够： + +1. 用户不知道系统把这轮理解成聊天、分析还是执行 +2. 用户不知道系统到底引用了哪张图 +3. 用户不知道系统为什么突然继续上一轮任务 +4. 用户不知道系统为什么没执行 + +### 3.5.3 必做改造 + +#### A. 增加“本轮理解摘要” + +建议在输入框附近或 assistant 第一段里展示： + +- 本轮类型：`聊天 / 分析 / 提示词 / 生图 / 修图 / 视频` +- 目标来源：`当前上传 / 画布选中 / 历史结果 / 上一轮提示词` +- 是否会执行：`仅分析 / 待确认 / 直接执行` + +#### B. 增加“任务切换提示” + +当系统从旧 workflow 切到新任务时，前端明确显示： + +- “已识别为新任务，不继续沿用上一轮修图流程” + +当系统继续旧任务时，显示： + +- “已继续上一轮修图流程” + +#### C. 协议调试面板继续升级 + +当前已有 `ProtocolDebugPanel`，建议继续加： + +- 当前任务边界摘要 +- 当前记忆裁剪摘要 +- 当前模型槽位摘要（planner / reply / execution） +- 当前 provider 健康状态 + +### 3.5.4 落点文件 + +- `components/chat.tsx` +- `components/protocol-debug-panel.tsx` +- `components/canvas/CanvasChatPanel.tsx` + +### 3.5.5 验收标准 + +- 用户能看见“系统理解成什么” +- 用户能看见“为什么这轮不执行” +- debug 时能看见模型和 workflow 生命周期 + +--- + +## 4. 实施顺序（必须按这个顺序） + +### P0：先止血 + +1. greeting fast-path +2. prompt-only fast-path +3. stream provider 错误明确回显 +4. Agent 模式 replyModel 跟随 Agent 默认模型 + +> 这几项已经部分落地，但要继续收边并补测试。 + +### P1：统一任务边界 + +1. `task-boundary-classifier` +2. 统一 taskType / taskMode / allowExecution +3. 对 route 中旧 heuristics 收口 + +### P2：统一记忆裁剪 + +1. relevance scoring +2. session/task/preference 记忆分层 +3. 旧 workflow 自动降权 + +### P3：统一模型链路 + +1. planner / analyzer / reply / execution 四槽位 +2. provider health cache +3. 额度不足/超时自动降级 + +### P4：统一前端解释性 + +1. 本轮理解摘要 +2. 任务切换提示 +3. 调试面板生命周期摘要 + +--- + +## 5. 开发清单（可直接排期） + +### 5.1 任务理解层 + +- [ ] 新增 `lib/ai/runtime/task-boundary-classifier.ts` +- [ ] 把 `conversation-action-resolver` 和 `smart-analyzer` 结果统一映射到 `taskType` +- [ ] 增加 `taskMode`：`new/continue/revise/confirm/cancel` +- [ ] 增加 fast-path 优先级表 + +### 5.2 记忆层 + +- [ ] 新增 `memory-relevance.ts` +- [ ] `composeConversationMemory` 支持 top-k 裁剪 +- [ ] 为 workflow memory 增加过期/降权规则 +- [ ] 为 prompt-only / analysis-only 场景建立独立上下文模板 + +### 5.3 路由层 + +- [ ] 统一 planner / analyzer / reply / execution 槽位 +- [ ] Agent replyModel 跟随 Agent 默认模型 +- [ ] provider health 状态缓存 +- [ ] 上游 quota / timeout 短时熔断 + +### 5.4 执行层 + +- [ ] 新增 `execution-policy-gate.ts` +- [ ] clarification / prompt-only / analysis-only 全部走快路径 +- [ ] timeout fallback 改成“保正确优先” +- [ ] 错误统一映射为用户可理解消息 + +### 5.5 前端层 + +- [ ] 增加“本轮理解摘要” +- [ ] 增加“任务切换提示” +- [ ] Protocol Debug 增加模型槽位摘要 +- [ ] Protocol Debug 增加上下文裁剪摘要 + +--- + +## 6. 这次整改完成后，用户体验应该变成什么样 + +### 场景 1：用户说“你好” + +系统行为： + +- 识别为 `greeting` +- 不读取旧修图 workflow +- 不做大规模图像分析 +- 1 秒内稳定回复 + +### 场景 2：用户上传图片，说“帮我写这张图的提示词” + +系统行为： + +- 识别为 `prompt_only` +- 只分析当前图片 +- 不进入生图/修图执行链 +- 输出提示词 +- 把提示词写入 prompt memory + +### 场景 3：用户说“按刚才那个提示词再来 4 张” + +系统行为： + +- 识别为 `image_generation_followup` +- 复用上一轮提示词和参数 +- 明确出图数量为 4 +- 进入执行链 + +### 场景 4：用户说“修改第 3 镜头，再确认” + +系统行为： + +- 识别为 `workflow_followup + revise` +- 只进入 storyboard revision 子链路 +- 不误切普通生图 + +--- + +## 7. 最后一句话 + +我们现在已经有： + +- protocol +- subagent +- workflow event +- image/video memory +- debug stream + +所以真正差的不是“没有 Agent”，而是： + +> **还没有把任务边界、记忆裁剪、模型链路、执行门控做成统一系统。** + +这份整改清单的目标，就是把 Agent 模式从： + +**很多聪明 patch** + +升级成： + +**真正稳定的统一 Agent Runtime** + +--- + +## 8. 建议的下一步执行顺序 + +如果马上开工，建议按下面顺序推进： + +1. 先做 **`task-boundary-classifier`** +2. 再做 **memory relevance / context pruning** +3. 再做 **provider health + model slot unification** +4. 最后做 **前端理解摘要** + +因为只有先把任务边界做对，后面的“更聪明”才不会继续建立在错误理解之上。 + diff --git a/lib/ai/conversation-memory-composer.ts b/lib/ai/conversation-memory-composer.ts index 406c8f84c..5d5d34237 100644 --- a/lib/ai/conversation-memory-composer.ts +++ b/lib/ai/conversation-memory-composer.ts @@ -3,6 +3,11 @@ import "server-only"; import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state"; import type { ImageWorkflowMemory } from "@/lib/ai/image-workflow-memory"; import type { PromptMemoryEntry } from "@/lib/ai/prompt-memory"; +import { + applyContextLineLimit, + applyMemoryHintLimit, + type MemoryRelevancePolicy, +} from "@/lib/ai/runtime/memory-relevance"; import type { StoryboardPreferenceMemory } from "@/lib/video-agent/storyboard-preference-memory"; import type { ResolvedChatMediaInput, @@ -164,6 +169,7 @@ export function composeConversationMemory(params: { imageWorkflowMemory?: ImageWorkflowMemory | null; storyboardPreferenceMemory?: StoryboardPreferenceMemory | null; threadState?: AgentThreadState | null; + memoryRelevancePolicy?: MemoryRelevancePolicy; }) { const targetSummary = summarizeTargetResolution(params.targetResolution); const mediaSummary = summarizeResolvedMedia(params.resolvedMediaInput); @@ -197,8 +203,12 @@ export function composeConversationMemory(params: { }; return { - contextSummary, - memoryHints, + contextSummary: params.memoryRelevancePolicy + ? applyContextLineLimit(contextSummary, params.memoryRelevancePolicy.maxContextLines) + : contextSummary, + memoryHints: params.memoryRelevancePolicy + ? applyMemoryHintLimit(memoryHints, params.memoryRelevancePolicy.maxMemoryHints) + : memoryHints, metadata, } satisfies ComposedConversationMemory; } diff --git a/lib/ai/conversation-state-protocol.ts b/lib/ai/conversation-state-protocol.ts index 80b06b2ac..76f311541 100644 --- a/lib/ai/conversation-state-protocol.ts +++ b/lib/ai/conversation-state-protocol.ts @@ -22,6 +22,11 @@ import { resolveConversationAction, extractPendingGridConfirmationFromWorkingState, } from "@/lib/ai/conversation-action-resolver"; +import { + applyContextLineLimit, + applyMemoryHintLimit, + resolveMemoryRelevancePolicy, +} from "@/lib/ai/runtime/memory-relevance"; import { composeConversationMemory, type ComposedConversationMemory, @@ -37,6 +42,10 @@ import { buildPromptMemoryUpdate, } from "@/lib/ai/prompt-memory"; import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state"; +import { + classifyTaskBoundary, + type TaskBoundaryDecision, +} from "@/lib/ai/runtime/task-boundary-classifier"; import type { StoryboardPreferenceMemory } from "@/lib/video-agent/storyboard-preference-memory"; import { extractStoryboardPreferenceMemory, @@ -56,6 +65,7 @@ export interface ConversationStateProtocolBootstrap { pendingGridConfirmation: PendingGridConfirmation | null; confirmedPrompt: string | undefined; actionResolution: ReturnType; + taskBoundaryDecision: TaskBoundaryDecision; } export interface ConversationStateProtocolComposed { @@ -181,6 +191,7 @@ export function bootstrapConversationStateProtocol(params: { } | null; activeStoryboardTask?: StoryboardTaskDetail | null; hasImageInputForPromptCarryForward: boolean; + hasCanvasSelection?: boolean; }) { const storyboardPreferenceMemory = extractStoryboardPreferenceMemory(params.workingState); const imageWorkflowMemory = extractImageWorkflowMemory(params.workingState); @@ -197,6 +208,14 @@ export function bootstrapConversationStateProtocol(params: { confirmedPrompt, hasImageInputForPromptCarryForward: params.hasImageInputForPromptCarryForward, }); + const taskBoundaryDecision = classifyTaskBoundary({ + text: params.text, + actionResolution, + hasCurrentImageInput: params.hasImageInputForPromptCarryForward, + hasCanvasSelection: params.hasCanvasSelection === true, + pendingGridConfirmation, + hasActiveStoryboardTask: Boolean(params.activeStoryboardTask?.task?.id), + }); return { workingState: params.workingState || null, @@ -207,6 +226,7 @@ export function bootstrapConversationStateProtocol(params: { pendingGridConfirmation, confirmedPrompt, actionResolution, + taskBoundaryDecision, } satisfies ConversationStateProtocolBootstrap; } @@ -216,33 +236,62 @@ export function composeConversationStateProtocol(params: { resolvedMediaInput: ResolvedChatMediaInput; targetResolution: ResolvedTargetAssets; }) { + const memoryRelevancePolicy = resolveMemoryRelevancePolicy({ + taskBoundaryDecision: params.bootstrap.taskBoundaryDecision, + actionResolution: params.bootstrap.actionResolution, + hasActiveStoryboardTask: Boolean(params.bootstrap.activeStoryboardTask?.task?.id), + hasPendingGridConfirmation: Boolean(params.bootstrap.pendingGridConfirmation), + hasCurrentImageInput: params.resolvedMediaInput.currentMessageImageUrls.length > 0, + hasCanvasSelection: params.resolvedMediaInput.selectedImageUrls.length > 0, + }); + const baseThreadState = buildConversationThreadState({ - activeStoryboardTask: params.bootstrap.activeStoryboardTask, - storyboardPreferenceMemory: params.bootstrap.storyboardPreferenceMemory, - promptMemory: params.bootstrap.promptMemory, - imageWorkflowMemory: params.bootstrap.imageWorkflowMemory, + activeStoryboardTask: memoryRelevancePolicy.includeWorkflowState + ? params.bootstrap.activeStoryboardTask + : null, + storyboardPreferenceMemory: memoryRelevancePolicy.includeStoryboardPreferenceMemory + ? params.bootstrap.storyboardPreferenceMemory + : null, + promptMemory: memoryRelevancePolicy.includePromptMemory + ? params.bootstrap.promptMemory + : null, + imageWorkflowMemory: memoryRelevancePolicy.includeImageWorkflowMemory + ? params.bootstrap.imageWorkflowMemory + : null, }); const composedConversationMemory = composeConversationMemory({ autoContextText: params.autoContextText || undefined, resolvedMediaInput: params.resolvedMediaInput, targetResolution: params.targetResolution, - promptMemory: params.bootstrap.promptMemory, - imageWorkflowMemory: params.bootstrap.imageWorkflowMemory, - storyboardPreferenceMemory: params.bootstrap.storyboardPreferenceMemory, + promptMemory: memoryRelevancePolicy.includePromptMemory + ? params.bootstrap.promptMemory + : null, + imageWorkflowMemory: memoryRelevancePolicy.includeImageWorkflowMemory + ? params.bootstrap.imageWorkflowMemory + : null, + storyboardPreferenceMemory: memoryRelevancePolicy.includeStoryboardPreferenceMemory + ? params.bootstrap.storyboardPreferenceMemory + : null, threadState: baseThreadState, + memoryRelevancePolicy, }); const threadState: AgentThreadState = { workflowState: baseThreadState?.workflowState || null, - memoryHints: Array.from( + memoryHints: applyMemoryHintLimit( + Array.from( new Set([ ...(baseThreadState?.memoryHints || []), ...composedConversationMemory.memoryHints, ]) ), - contextSummary: + memoryRelevancePolicy.maxMemoryHints, + ), + contextSummary: applyContextLineLimit( composedConversationMemory.contextSummary || baseThreadState?.contextSummary, + memoryRelevancePolicy.maxContextLines, + ), metadata: { ...(baseThreadState?.metadata || {}), ...composedConversationMemory.metadata, diff --git a/lib/ai/runtime/agent-model-slots.ts b/lib/ai/runtime/agent-model-slots.ts new file mode 100644 index 000000000..f33e1c44f --- /dev/null +++ b/lib/ai/runtime/agent-model-slots.ts @@ -0,0 +1,108 @@ +import "server-only"; + +import { + getAiModelById, + getAiProviderById, + getEnabledAiModelsWithProvider, +} from "@/lib/db/queries"; + +export type AgentModelSlotName = "planner" | "analyzer" | "reply" | "execution"; + +export interface AgentModelSlotDescriptor { + modelId: string; + displayName?: string; + providerId?: string; + providerType?: string; + providerKey?: string; +} + +export interface AgentModelSlot extends AgentModelSlotDescriptor { + slot: AgentModelSlotName; + source: string; + reason?: string; + locked?: boolean; +} + +export interface AgentModelSlots { + planner?: AgentModelSlot; + analyzer?: AgentModelSlot; + reply?: AgentModelSlot; + execution?: AgentModelSlot; +} + +function buildProviderKey(params: { + providerId?: string; + providerType?: string; + modelId: string; +}) { + if (params.providerId) return params.providerId; + if (params.providerType) return `${params.providerType}:${params.modelId}`; + return `model:${params.modelId}`; +} + +export async function resolveAgentModelSlotDescriptor(modelId: string) { + const normalizedModelId = String(modelId || "").trim(); + if (!normalizedModelId) return null; + + const isUuid = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(normalizedModelId); + + if (isUuid) { + const model = await getAiModelById({ id: normalizedModelId }).catch(() => null); + if (!model) { + return { + modelId: normalizedModelId, + providerKey: `model:${normalizedModelId}`, + } satisfies AgentModelSlotDescriptor; + } + const provider = await getAiProviderById({ id: model.providerId }).catch(() => null); + return { + modelId: model.id, + displayName: model.displayName || model.modelId, + providerId: model.providerId, + providerType: provider?.type || undefined, + providerKey: buildProviderKey({ + providerId: model.providerId, + providerType: provider?.type || undefined, + modelId: model.id, + }), + } satisfies AgentModelSlotDescriptor; + } + + const enabledModels = await getEnabledAiModelsWithProvider(false).catch(() => []); + const matched = enabledModels.find( + ({ model }) => model.id === normalizedModelId || model.modelId === normalizedModelId, + ); + if (!matched) { + return { + modelId: normalizedModelId, + providerKey: `model:${normalizedModelId}`, + } satisfies AgentModelSlotDescriptor; + } + + return { + modelId: matched.model.id, + displayName: matched.model.displayName || matched.model.modelId, + providerId: matched.model.providerId, + providerType: matched.provider?.type || undefined, + providerKey: buildProviderKey({ + providerId: matched.model.providerId, + providerType: matched.provider?.type || undefined, + modelId: matched.model.id, + }), + } satisfies AgentModelSlotDescriptor; +} + +export function upsertAgentModelSlot( + slots: AgentModelSlots, + slot: AgentModelSlotName, + value: Omit, +) { + return { + ...slots, + [slot]: { + slot, + ...value, + }, + } satisfies AgentModelSlots; +} diff --git a/lib/ai/runtime/memory-relevance.ts b/lib/ai/runtime/memory-relevance.ts new file mode 100644 index 000000000..6745e26c0 --- /dev/null +++ b/lib/ai/runtime/memory-relevance.ts @@ -0,0 +1,134 @@ +import "server-only"; + +import type { ConversationActionResolution } from "@/lib/ai/conversation-action-resolver"; +import type { TaskBoundaryDecision } from "@/lib/ai/runtime/task-boundary-classifier"; + +export interface MemoryRelevancePolicy { + includePromptMemory: boolean; + includeImageWorkflowMemory: boolean; + includeStoryboardPreferenceMemory: boolean; + includeWorkflowState: boolean; + maxMemoryHints: number; + maxContextLines: number; + reason: string; +} + +export function resolveMemoryRelevancePolicy(params: { + taskBoundaryDecision: TaskBoundaryDecision; + actionResolution: ConversationActionResolution; + hasActiveStoryboardTask: boolean; + hasPendingGridConfirmation: boolean; + hasCurrentImageInput: boolean; + hasCanvasSelection: boolean; +}) { + const { taskBoundaryDecision, actionResolution } = params; + + switch (taskBoundaryDecision.taskType) { + case "greeting": + return { + includePromptMemory: false, + includeImageWorkflowMemory: false, + includeStoryboardPreferenceMemory: false, + includeWorkflowState: false, + maxMemoryHints: 0, + maxContextLines: 2, + reason: "greeting_turn_ignores_task_memory", + } satisfies MemoryRelevancePolicy; + case "chat": + return { + includePromptMemory: false, + includeImageWorkflowMemory: false, + includeStoryboardPreferenceMemory: false, + includeWorkflowState: false, + maxMemoryHints: 1, + maxContextLines: 3, + reason: "chat_turn_prefers_clean_context", + } satisfies MemoryRelevancePolicy; + case "prompt_only": + return { + includePromptMemory: false, + includeImageWorkflowMemory: false, + includeStoryboardPreferenceMemory: false, + includeWorkflowState: false, + maxMemoryHints: 2, + maxContextLines: 4, + reason: "prompt_only_turn_ignores_execution_memory", + } satisfies MemoryRelevancePolicy; + case "analysis_only": + return { + includePromptMemory: false, + includeImageWorkflowMemory: false, + includeStoryboardPreferenceMemory: false, + includeWorkflowState: false, + maxMemoryHints: 2, + maxContextLines: 4, + reason: "analysis_only_turn_ignores_workflow_memory", + } satisfies MemoryRelevancePolicy; + case "execution_video": + return { + includePromptMemory: false, + includeImageWorkflowMemory: false, + includeStoryboardPreferenceMemory: true, + includeWorkflowState: + params.hasActiveStoryboardTask || params.hasPendingGridConfirmation, + maxMemoryHints: 4, + maxContextLines: 6, + reason: "video_execution_prefers_storyboard_preferences", + } satisfies MemoryRelevancePolicy; + case "workflow_followup": + return { + includePromptMemory: + actionResolution.kind === "image_generation_followup" || + actionResolution.kind === "prompt_carry_forward", + includeImageWorkflowMemory: + actionResolution.kind === "image_edit_followup" || + actionResolution.kind === "image_generation_followup" || + actionResolution.kind === "prompt_carry_forward", + includeStoryboardPreferenceMemory: + actionResolution.kind === "grid_confirmation_followup" || + actionResolution.kind === "active_storyboard_followup", + includeWorkflowState: true, + maxMemoryHints: 6, + maxContextLines: 8, + reason: "workflow_followup_requires_workflow_memory", + } satisfies MemoryRelevancePolicy; + case "execution_image": + default: + return { + includePromptMemory: + actionResolution.usePromptCarryForward || + taskBoundaryDecision.taskMode === "continue", + includeImageWorkflowMemory: + taskBoundaryDecision.taskMode === "continue" || + (!params.hasCurrentImageInput && !params.hasCanvasSelection), + includeStoryboardPreferenceMemory: false, + includeWorkflowState: + taskBoundaryDecision.taskMode === "continue" && + (actionResolution.kind === "image_edit_followup" || + actionResolution.kind === "image_generation_followup" || + actionResolution.kind === "prompt_carry_forward"), + maxMemoryHints: taskBoundaryDecision.taskMode === "continue" ? 5 : 3, + maxContextLines: taskBoundaryDecision.taskMode === "continue" ? 7 : 5, + reason: + taskBoundaryDecision.taskMode === "continue" + ? "image_execution_continue_reuses_relevant_memory" + : "image_execution_new_task_limits_old_memory", + } satisfies MemoryRelevancePolicy; + } +} + +export function applyMemoryHintLimit(hints: string[], maxHints: number) { + if (!Array.isArray(hints) || maxHints <= 0) return [] as string[]; + return hints.slice(0, maxHints); +} + +export function applyContextLineLimit(summary: string | undefined, maxLines: number) { + const normalized = String(summary || "").trim(); + if (!normalized || maxLines <= 0) return ""; + return normalized + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .slice(0, maxLines) + .join("\n"); +} diff --git a/lib/ai/runtime/provider-health.ts b/lib/ai/runtime/provider-health.ts new file mode 100644 index 000000000..35a609483 --- /dev/null +++ b/lib/ai/runtime/provider-health.ts @@ -0,0 +1,109 @@ +import "server-only"; + +export type ProviderHealthEventType = "success" | "timeout" | "quota" | "error"; + +export interface ProviderHealthSnapshot { + providerKey: string; + providerType?: string; + lastEventType: ProviderHealthEventType; + lastError?: string; + consecutiveFailures: number; + lastSuccessAt?: number; + updatedAt: number; +} + +const providerHealthMap = new Map(); + +function normalizeErrorMessage(error: unknown) { + return error instanceof Error ? String(error.message || "").trim() : String(error || "").trim(); +} + +export function classifyProviderHealthEvent(error: unknown): ProviderHealthEventType { + const normalized = normalizeErrorMessage(error).toLowerCase(); + + if ( + normalized.includes("token quota is not enough") || + normalized.includes("pre_consume_token_quota_failed") || + normalized.includes("quota is not enough") + ) { + return "quota"; + } + + if ( + normalized.includes("timeout") || + normalized.includes("timed out") || + normalized.includes("body timeout") || + normalized.includes("headers timeout") + ) { + return "timeout"; + } + + return "error"; +} + +export function recordProviderHealthSuccess(params: { + providerKey?: string; + providerType?: string; +}) { + const providerKey = String(params.providerKey || "").trim(); + if (!providerKey) return; + + const existing = providerHealthMap.get(providerKey); + providerHealthMap.set(providerKey, { + providerKey, + providerType: params.providerType || existing?.providerType, + lastEventType: "success", + consecutiveFailures: 0, + lastError: undefined, + lastSuccessAt: Date.now(), + updatedAt: Date.now(), + }); +} + +export function recordProviderHealthFailure(params: { + providerKey?: string; + providerType?: string; + error: unknown; +}) { + const providerKey = String(params.providerKey || "").trim(); + if (!providerKey) return; + + const existing = providerHealthMap.get(providerKey); + providerHealthMap.set(providerKey, { + providerKey, + providerType: params.providerType || existing?.providerType, + lastEventType: classifyProviderHealthEvent(params.error), + lastError: normalizeErrorMessage(params.error).slice(0, 400), + consecutiveFailures: + existing?.lastEventType === "success" + ? 1 + : Math.max(1, (existing?.consecutiveFailures || 0) + 1), + lastSuccessAt: existing?.lastSuccessAt, + updatedAt: Date.now(), + }); +} + +export function getProviderHealthSnapshot(providerKey?: string) { + const normalized = String(providerKey || "").trim(); + if (!normalized) return null; + return providerHealthMap.get(normalized) || null; +} + +export function shouldAvoidProvider(snapshot?: ProviderHealthSnapshot | null) { + if (!snapshot) return false; + + const ageMs = Date.now() - snapshot.updatedAt; + if (snapshot.lastEventType === "quota") { + return ageMs <= 15 * 60 * 1000; + } + + if (snapshot.lastEventType === "timeout") { + return snapshot.consecutiveFailures >= 2 && ageMs <= 5 * 60 * 1000; + } + + if (snapshot.lastEventType === "error") { + return snapshot.consecutiveFailures >= 3 && ageMs <= 5 * 60 * 1000; + } + + return false; +} diff --git a/lib/ai/runtime/task-boundary-classifier.ts b/lib/ai/runtime/task-boundary-classifier.ts new file mode 100644 index 000000000..ffcb63379 --- /dev/null +++ b/lib/ai/runtime/task-boundary-classifier.ts @@ -0,0 +1,226 @@ +import "server-only"; + +import type { + ConversationActionResolution, + PendingGridConfirmation, +} from "@/lib/ai/conversation-action-resolver"; +import { isExplicitPromptOptimizationRequest } from "@/lib/ai/smart-analyzer"; +import { detectStoryboardCommandIntent } from "@/lib/video-agent/storyboard-command-interpreter"; + +export type TaskBoundaryType = + | "greeting" + | "chat" + | "analysis_only" + | "prompt_only" + | "execution_image" + | "execution_video" + | "workflow_followup"; + +export type TaskBoundaryMode = "new" | "continue" | "revise" | "confirm" | "cancel"; + +export interface TaskBoundaryDecision { + taskType: TaskBoundaryType; + taskMode: TaskBoundaryMode; + shouldUseCanvasContext: boolean; + shouldUseHistoryImages: boolean; + shouldUseWorkflowMemory: boolean; + shouldAllowExecution: boolean; + confidence: number; + reason: string; +} + +function normalizeCompactText(text: string) { + return String(text || "").toLowerCase().replace(/\s+/g, "").trim(); +} + +export function isPureGreetingTurnRequest(text: string): boolean { + const normalized = normalizeCompactText(text); + if (!normalized) return false; + + return /^(你好|您好|hello|hi|嗨|哈喽|在吗|早上好|中午好|下午好|晚上好|深夜好|谢谢|感谢|收到|好的|ok|okay|明白了|了解了)$/.test( + normalized, + ); +} + +function isLikelyVideoExecutionRequest(text: string): boolean { + const normalized = normalizeCompactText(text); + if (!normalized) return false; + + return /图生视频|文生视频|视频生成|生成视频|做视频|做成视频|开始生成视频|让.*动起来|变成视频|视频/.test( + normalized, + ); +} + +function isLikelyImageExecutionRequest(text: string): boolean { + const normalized = normalizeCompactText(text); + if (!normalized) return false; + + return /生图|生成图片|出图|画一张|帮我画|做图|做一张图|重新生成|再来一张|执行生成|开始生成|生成出来/.test( + normalized, + ); +} + +function isLikelyAnalysisOnlyRequest(text: string, hasImageContext: boolean): boolean { + if (!hasImageContext) return false; + const normalized = normalizeCompactText(text); + if (!normalized) return false; + + return /分析|看图|看看|识别|拆解|提炼|整理|归纳|总结|描述|解释|判断|帮我看看|帮我分析|帮我判断|是什么|适合吗|怎么样|评价/.test( + normalized, + ); +} + +function resolveStoryboardTaskMode(params: { + text: string; + hasPendingGridConfirmation: boolean; + hasActiveStoryboardTask: boolean; +}): TaskBoundaryMode { + if (!params.hasPendingGridConfirmation && !params.hasActiveStoryboardTask) { + return "continue"; + } + + const intent = detectStoryboardCommandIntent(params.text); + switch (intent) { + case "confirm": + return "confirm"; + case "revise": + return "revise"; + case "cancel": + return "cancel"; + case "unclear": + default: + return "continue"; + } +} + +export function classifyTaskBoundary(params: { + text: string; + actionResolution: ConversationActionResolution; + hasCurrentImageInput: boolean; + hasCanvasSelection: boolean; + pendingGridConfirmation?: PendingGridConfirmation | null; + hasActiveStoryboardTask: boolean; +}) { + const hasImageContext = params.hasCurrentImageInput || params.hasCanvasSelection; + + if (isPureGreetingTurnRequest(params.text) && !params.hasCurrentImageInput) { + return { + taskType: "greeting", + taskMode: "new", + shouldUseCanvasContext: false, + shouldUseHistoryImages: false, + shouldUseWorkflowMemory: false, + shouldAllowExecution: false, + confidence: 0.99, + reason: "pure_greeting_turn", + } satisfies TaskBoundaryDecision; + } + + if (isExplicitPromptOptimizationRequest(params.text)) { + return { + taskType: "prompt_only", + taskMode: "new", + shouldUseCanvasContext: !params.hasCurrentImageInput && params.hasCanvasSelection, + shouldUseHistoryImages: !params.hasCurrentImageInput && !params.hasCanvasSelection, + shouldUseWorkflowMemory: false, + shouldAllowExecution: false, + confidence: 0.97, + reason: "explicit_prompt_only_request", + } satisfies TaskBoundaryDecision; + } + + switch (params.actionResolution.kind) { + case "grid_confirmation_followup": + case "active_storyboard_followup": + return { + taskType: "workflow_followup", + taskMode: resolveStoryboardTaskMode({ + text: params.text, + hasPendingGridConfirmation: Boolean(params.pendingGridConfirmation), + hasActiveStoryboardTask: params.hasActiveStoryboardTask, + }), + shouldUseCanvasContext: params.hasCanvasSelection, + shouldUseHistoryImages: !params.hasCurrentImageInput, + shouldUseWorkflowMemory: true, + shouldAllowExecution: true, + confidence: 0.95, + reason: params.actionResolution.kind, + } satisfies TaskBoundaryDecision; + case "image_edit_followup": + case "image_generation_followup": + return { + taskType: "workflow_followup", + taskMode: "continue", + shouldUseCanvasContext: params.hasCanvasSelection, + shouldUseHistoryImages: !params.hasCurrentImageInput, + shouldUseWorkflowMemory: true, + shouldAllowExecution: true, + confidence: 0.92, + reason: params.actionResolution.kind, + } satisfies TaskBoundaryDecision; + case "prompt_carry_forward": + return { + taskType: "execution_image", + taskMode: "continue", + shouldUseCanvasContext: params.hasCanvasSelection, + shouldUseHistoryImages: !params.hasCurrentImageInput, + shouldUseWorkflowMemory: true, + shouldAllowExecution: true, + confidence: 0.93, + reason: "prompt_carry_forward_execution", + } satisfies TaskBoundaryDecision; + case "default": + default: + break; + } + + if (isLikelyVideoExecutionRequest(params.text)) { + return { + taskType: "execution_video", + taskMode: "new", + shouldUseCanvasContext: params.hasCanvasSelection, + shouldUseHistoryImages: false, + shouldUseWorkflowMemory: false, + shouldAllowExecution: true, + confidence: 0.88, + reason: "explicit_video_execution_request", + } satisfies TaskBoundaryDecision; + } + + if (isLikelyImageExecutionRequest(params.text)) { + return { + taskType: "execution_image", + taskMode: "new", + shouldUseCanvasContext: params.hasCanvasSelection, + shouldUseHistoryImages: false, + shouldUseWorkflowMemory: false, + shouldAllowExecution: true, + confidence: 0.87, + reason: "explicit_image_execution_request", + } satisfies TaskBoundaryDecision; + } + + if (isLikelyAnalysisOnlyRequest(params.text, hasImageContext)) { + return { + taskType: "analysis_only", + taskMode: "new", + shouldUseCanvasContext: params.hasCanvasSelection, + shouldUseHistoryImages: !params.hasCurrentImageInput && !params.hasCanvasSelection, + shouldUseWorkflowMemory: false, + shouldAllowExecution: false, + confidence: 0.9, + reason: "analysis_only_request", + } satisfies TaskBoundaryDecision; + } + + return { + taskType: "chat", + taskMode: "new", + shouldUseCanvasContext: false, + shouldUseHistoryImages: false, + shouldUseWorkflowMemory: false, + shouldAllowExecution: false, + confidence: 0.7, + reason: "default_chat_turn", + } satisfies TaskBoundaryDecision; +} diff --git a/lib/ai/runtime/task-boundary-context-pruner.ts b/lib/ai/runtime/task-boundary-context-pruner.ts new file mode 100644 index 000000000..17ea29680 --- /dev/null +++ b/lib/ai/runtime/task-boundary-context-pruner.ts @@ -0,0 +1,169 @@ +import "server-only"; + +import type { DBMessage } from "@/lib/db/schema"; +import type { + CanvasImageNodeContext, + ContextPayload, + WorkingStateImageMemory, +} from "@/lib/ai/context-memory/types"; +import type { TaskBoundaryDecision } from "@/lib/ai/runtime/task-boundary-classifier"; + +function buildSelectedCanvasSubset(contextPayload?: ContextPayload) { + const selectedNodeIds = new Set(contextPayload?.selectedNodeIds || []); + const selectedAssetIds = new Set(contextPayload?.selectedAssetIds || []); + const selectedNodes = (contextPayload?.canvasImageNodes || []).filter((node) => { + if (!node) return false; + if (node.isSelected) return true; + if (node.nodeId && selectedNodeIds.has(node.nodeId)) return true; + if (node.assetId && selectedAssetIds.has(node.assetId)) return true; + return false; + }); + + return { + selectedNodes, + selectedNodeIds: selectedNodes.map((node) => node.nodeId).filter(Boolean), + selectedAssetIds: selectedNodes.map((node) => node.assetId).filter(Boolean) as string[], + }; +} + +function buildSelectedCanvasSummary(selectedNodes: CanvasImageNodeContext[]) { + if (selectedNodes.length === 0) return undefined; + const previewLines = selectedNodes.slice(0, 3).map((node, index) => { + const title = String(node.title || node.prompt || node.nodeId || `图片${index + 1}`).trim(); + return `选中图片${index + 1}: ${title}`; + }); + return [`用户已选中 ${selectedNodes.length} 张图片，优先处理这些`, ...previewLines].join("\n"); +} + +export function pruneTaskBoundaryContext(params: { + decision: TaskBoundaryDecision; + contextPayload?: ContextPayload; + autoContextText?: string; + workingState?: WorkingStateImageMemory | null; + hasCurrentImageInput: boolean; +}) { + if (!params.contextPayload && !params.autoContextText) { + return { + contextPayload: params.contextPayload, + autoContextText: params.autoContextText, + workingState: params.decision.shouldUseWorkflowMemory ? params.workingState || null : null, + }; + } + + const selectedSubset = buildSelectedCanvasSubset(params.contextPayload); + const shouldPreferSelectedOnly = + selectedSubset.selectedNodes.length > 0 && + (params.decision.taskType === "prompt_only" || + params.decision.taskType === "analysis_only" || + params.decision.taskType === "execution_image" || + params.decision.taskType === "execution_video" || + params.hasCurrentImageInput); + + let nextContextPayload = params.contextPayload; + let nextAutoContextText = params.autoContextText; + + if (!params.decision.shouldUseCanvasContext) { + nextContextPayload = params.contextPayload + ? { + source: params.contextPayload.source, + projectId: params.contextPayload.projectId, + focusPolicy: params.contextPayload.focusPolicy, + selectedNodeIds: [], + selectedAssetIds: [], + canvasImageNodes: [], + } + : undefined; + nextAutoContextText = undefined; + } else if (shouldPreferSelectedOnly) { + nextContextPayload = params.contextPayload + ? { + ...params.contextPayload, + focusPolicy: "selected", + selectedNodeIds: selectedSubset.selectedNodeIds, + selectedAssetIds: selectedSubset.selectedAssetIds, + canvasImageNodes: selectedSubset.selectedNodes, + } + : undefined; + nextAutoContextText = buildSelectedCanvasSummary(selectedSubset.selectedNodes); + } + + return { + contextPayload: nextContextPayload, + autoContextText: nextAutoContextText, + workingState: params.decision.shouldUseWorkflowMemory ? params.workingState || null : null, + }; +} + +function sanitizeHistoryText(text: string) { + return String(text || "") + .replace(/!\[[^\]]*]\([^)]+\)/g, "") + .replace(/]*>[\s\S]*?<\/video>/gi, "") + .replace(/]+src="[^"]+"[^>]*>/gi, "") + .replace(//g, "") + .replace(/◈\s*[^\n]+\n*/g, "") + .replace(/\n{3,}/g, "\n\n") + .trim(); +} + +function toTextOnlyHistoryMessages(messages: DBMessage[], maxMessages: number): DBMessage[] { + const sliced = messages.slice(-Math.max(1, maxMessages)); + const result: DBMessage[] = []; + + for (const message of sliced) { + const parts = Array.isArray(message.parts) ? (message.parts as Array>) : []; + const textParts = parts + .filter((part) => part?.type === "text" && typeof part?.text === "string") + .map((part) => ({ + type: "text", + text: sanitizeHistoryText(String(part.text || "")), + })) + .filter((part) => part.text); + + if (textParts.length === 0) continue; + + result.push({ + ...message, + parts: textParts, + attachments: [], + } satisfies DBMessage); + } + + return result; +} + +export function pruneTaskBoundaryHistoryMessages(params: { + decision: TaskBoundaryDecision; + messages: DBMessage[]; + purpose: "media" | "model"; + hasCurrentImageInput: boolean; +}): DBMessage[] { + const messages = Array.isArray(params.messages) ? params.messages : []; + if (messages.length === 0) return [] as DBMessage[]; + + switch (params.decision.taskType) { + case "greeting": + return []; + case "prompt_only": + return params.purpose === "media" + ? [] + : toTextOnlyHistoryMessages(messages, params.hasCurrentImageInput ? 2 : 4); + case "analysis_only": + return params.purpose === "media" + ? [] + : toTextOnlyHistoryMessages(messages, 4); + case "chat": + return params.purpose === "media" + ? [] + : toTextOnlyHistoryMessages(messages, 6); + case "execution_image": + case "execution_video": + if (params.decision.shouldUseHistoryImages) { + return messages.slice(-8); + } + return params.purpose === "media" ? [] : toTextOnlyHistoryMessages(messages, 4); + case "workflow_followup": + return messages.slice(-(params.purpose === "media" ? 8 : 12)); + default: + return params.purpose === "media" ? [] : toTextOnlyHistoryMessages(messages, 4); + } +}