diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 581f34056..4cc080bb0 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -79,7 +79,6 @@ import { getAccessibleAgentPresetById, getConversationWorkingStateByChatId, getMediaAssetsByUserAndUrls, - upsertConversationWorkingState, upsertMediaAsset, } from "@/lib/db/queries"; import { db } from "@/lib/db"; @@ -93,13 +92,25 @@ import { generateCanvasImage } from "@/lib/canvas/generate-image"; import { buildGridConfirmationMessage, detectRequestedStoryboardGrid, - extractPendingGridConfirmationFromWorkingState, isGridConfirmationReply, type PendingGridConfirmation, - resolveConversationAction, } from "@/lib/ai/conversation-action-resolver"; -import { composeConversationMemory } from "@/lib/ai/conversation-memory-composer"; -import { buildConversationThreadState } from "@/lib/ai/conversation-thread-state-builder"; +import { + bootstrapConversationStateProtocol, + composeConversationStateProtocol, + persistAgentPresetTurn, + persistConversationStateProtocol, + setConversationStateProtocolEventObserver, + persistResolvedTargetTurn, + persistGridConfirmationTurn, + persistImageWorkflowTurn, + persistPromptOnlyTurn, + resolveGridConfirmationRequest, + resolveTargetExecutionOverride, + persistStoryboardConfirmedTurn, + persistStoryboardAssistantTurn, + persistVideoWorkflowTurn, +} from "@/lib/ai/conversation-state-protocol"; import { resolveChatMediaInput } from "@/lib/ai/context-memory/media-input"; import { resolveTargetAssets } from "@/lib/ai/context-memory/target-resolution"; import type { @@ -114,7 +125,6 @@ import { } from "@/lib/ai/image-workflow-memory"; import { buildPromptMemoryUpdate, - extractPromptMemoryFromWorkingState, } from "@/lib/ai/prompt-memory"; import { uploadFile } from "@/lib/storage/service"; import { getVideoStoryboardConfig } from "@/lib/video-agent/config"; @@ -403,6 +413,16 @@ function writeStructuredDataChunk( ); } +function writeProtocolDebugChunk( + writer: StreamWriter, + enabled: boolean, + data: unknown, + label: string, +) { + if (!enabled) return; + writeStructuredDataChunk(writer, "data-protocolDebug", data, label); +} + function normalizeImageLoadingProviderType(providerType?: string | null): string { const normalized = String(providerType || "").trim().toLowerCase(); if (normalized === "gemini-image" || normalized === "huanke-gemini") { @@ -1772,38 +1792,6 @@ function buildPromptOnlyReply(params: { ].join("\n"); } -function isConfirmedPromptGenerationRequest(text: string): boolean { - const normalized = String(text || "").toLowerCase().replace(/\s+/g, ""); - if (!normalized) return false; - - const refersPrompt = - /根据这个提示词|按这个提示词|就按这个提示词|按刚才的提示词|按上面的提示词|用这个提示词|用刚才的提示词|按该提示词/.test( - normalized, - ); - const wantsImageGeneration = - /生图|生成图片|出图|画出来|生成一张|来一张|开始生成|按此生成|按这个生成/.test(normalized); - - return refersPrompt && wantsImageGeneration; -} - -function extractConfirmedPromptFromWorkingState( - workingState?: { - lastTurnSummary?: unknown; - } | null, -): string | undefined { - const summary = workingState?.lastTurnSummary; - if (!summary || typeof summary !== "object") return undefined; - const summaryRecord = summary as Record; - - const resolutionMode = - typeof summaryRecord.resolutionMode === "string" ? summaryRecord.resolutionMode.trim() : ""; - const optimizedPrompt = - typeof summaryRecord.optimizedPrompt === "string" ? summaryRecord.optimizedPrompt.trim() : ""; - - if (resolutionMode !== "prompt_only" || !optimizedPrompt) return undefined; - return optimizedPrompt; -} - function hasStructuredAgentSections(message?: string): boolean { if (!message) return false; return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test( @@ -2571,16 +2559,11 @@ async function saveGeneratedImages( if (savedOriginalUrls.length > 0) { try { - const existingWorkingState = await getConversationWorkingStateByChatId({ - chatId, - }).catch(() => null); - const existingSummary = - existingWorkingState?.lastTurnSummary && - typeof existingWorkingState.lastTurnSummary === "object" - ? (existingWorkingState.lastTurnSummary as Record) - : {}; + const existingWorkingState = await getConversationWorkingStateByChatId({ chatId }).catch( + () => null + ); const existingImageWorkflowMemory = extractImageWorkflowMemory(existingWorkingState); - await upsertConversationWorkingState({ + await persistConversationStateProtocol({ userId, chatId, currentFocusAssetIds: savedAssetIds, @@ -2590,8 +2573,7 @@ async function saveGeneratedImages( lastResolvedTargetAssetIds: savedAssetIds, lastResolvedTargetImageUrls: savedOriginalUrls, lastMessageId: messageId, - lastTurnSummary: { - ...existingSummary, + lastTurnSummaryPatch: { imageWorkflowMemory: buildImageWorkflowMemoryUpdate({ existing: existingImageWorkflowMemory, lastIntent: options?.relationType === "edited_from" ? "image_edit" : "image_gen", @@ -2926,6 +2908,7 @@ export async function POST(request: Request) { autoContext, contextPayload, deepThinkingEnabled, + debugProtocol, }: { id: string; message: ChatMessage; @@ -2936,6 +2919,7 @@ export async function POST(request: Request) { autoContext?: string; contextPayload?: PostRequestBody["contextPayload"]; deepThinkingEnabled?: boolean; + debugProtocol?: boolean; } = requestBody; const requestStartedAt = Date.now(); const requestOrigin = (() => { @@ -3002,6 +2986,7 @@ export async function POST(request: Request) { let storyboardPreferenceMemory = null; let imageWorkflowMemory = null; let reusablePromptMemory = null; + let confirmedPromptForConversationState: string | undefined; let smartAutoHistoryMessagesFromDb: DBMessage[] = []; let preferredAgentModelId: string | undefined; let pendingGridConfirmation: @@ -3011,6 +2996,7 @@ export async function POST(request: Request) { const deepThinkingOn = deepThinkingEnabled !== false; let shouldRunAgentConversation = finalModelId === "auto"; let shouldKeepSelectedModel = false; + const pendingProtocolDebugEvents: Array<{ phase: string; payload: Record }> = []; const requestedModelId = selectedChatModel; let requestedModelProviderType: string | undefined; let requestedModelIsFixedVisual = false; @@ -3128,25 +3114,40 @@ export async function POST(request: Request) { ); return null; }); - storyboardPreferenceMemory = extractStoryboardPreferenceMemory(requestWorkingState); - imageWorkflowMemory = extractImageWorkflowMemory(requestWorkingState); - reusablePromptMemory = extractPromptMemoryFromWorkingState(requestWorkingState); - pendingGridConfirmation = - extractPendingGridConfirmationFromWorkingState(requestWorkingState); const hasImageInputForConversationAction = message.parts.some((part: any) => part?.type === "file" || part?.type === "image" || part?.type === "image_url" ); - const preloadedConfirmedPrompt = - extractConfirmedPromptFromWorkingState(requestWorkingState) || - reusablePromptMemory?.optimizedPrompt; - const resolvedConversationAction = resolveConversationAction({ - text: requestTextContent, - pendingGridConfirmation, - hasActiveStoryboardTask: Boolean(activeStoryboardTaskForChat?.task?.id), - imageWorkflowMemory, - confirmedPrompt: preloadedConfirmedPrompt, - hasImageInputForPromptCarryForward: hasImageInputForConversationAction, - }); + const conversationStateBootstrap = bootstrapConversationStateProtocol({ + text: requestTextContent, + workingState: requestWorkingState, + activeStoryboardTask: activeStoryboardTaskForChat, + hasImageInputForPromptCarryForward: hasImageInputForConversationAction, + }); + console.log("[ConversationStateProtocol] Bootstrap:", { + action: conversationStateBootstrap.actionResolution.kind, + hasPromptMemory: Boolean(conversationStateBootstrap.promptMemory?.optimizedPrompt), + hasImageWorkflowMemory: Boolean(conversationStateBootstrap.imageWorkflowMemory), + hasStoryboardPreferenceMemory: Boolean(conversationStateBootstrap.storyboardPreferenceMemory), + hasPendingGridConfirmation: Boolean(conversationStateBootstrap.pendingGridConfirmation), + hasActiveStoryboardTask: Boolean(conversationStateBootstrap.activeStoryboardTask?.task?.id), + }); + pendingProtocolDebugEvents.push({ + phase: "bootstrap", + payload: { + action: conversationStateBootstrap.actionResolution.kind, + pendingGridConfirmation: conversationStateBootstrap.pendingGridConfirmation, + hasPromptMemory: Boolean(conversationStateBootstrap.promptMemory?.optimizedPrompt), + hasImageWorkflowMemory: Boolean(conversationStateBootstrap.imageWorkflowMemory), + hasStoryboardPreferenceMemory: Boolean(conversationStateBootstrap.storyboardPreferenceMemory), + hasActiveStoryboardTask: Boolean(conversationStateBootstrap.activeStoryboardTask?.task?.id), + }, + }); + storyboardPreferenceMemory = conversationStateBootstrap.storyboardPreferenceMemory; + imageWorkflowMemory = conversationStateBootstrap.imageWorkflowMemory; + reusablePromptMemory = conversationStateBootstrap.promptMemory; + pendingGridConfirmation = conversationStateBootstrap.pendingGridConfirmation; + confirmedPromptForConversationState = conversationStateBootstrap.confirmedPrompt; + const resolvedConversationAction = conversationStateBootstrap.actionResolution; if (resolvedConversationAction.forceAgentConversation && !shouldRunAgentConversation) { shouldRunAgentConversation = true; @@ -3189,7 +3190,7 @@ export async function POST(request: Request) { chatId: id, }); } - confirmedPromptForGeneration = preloadedConfirmedPrompt; + confirmedPromptForGeneration = confirmedPromptForConversationState; const isPromptCarryForwardExecution = resolvedConversationAction.kind === "prompt_carry_forward"; forcePromptDrivenImageGeneration = @@ -3287,37 +3288,26 @@ export async function POST(request: Request) { CHAT_PRE_RESPONSE_SOFT_TIMEOUT_MS ); } - const baseThreadState = buildConversationThreadState({ - activeStoryboardTask: activeStoryboardTaskForChat, - storyboardPreferenceMemory, - promptMemory: reusablePromptMemory, - imageWorkflowMemory, - }); - const composedConversationMemory = composeConversationMemory({ + const composedConversationState = composeConversationStateProtocol({ + bootstrap: conversationStateBootstrap, autoContextText: autoContextText || undefined, resolvedMediaInput: requestResolvedMediaInput, targetResolution: requestTargetResolution, - promptMemory: reusablePromptMemory, - imageWorkflowMemory, - storyboardPreferenceMemory, - threadState: baseThreadState, }); - const composedThreadState = { - workflowState: baseThreadState?.workflowState || null, - memoryHints: Array.from( - new Set([ - ...(baseThreadState?.memoryHints || []), - ...composedConversationMemory.memoryHints, - ]) - ), - contextSummary: - composedConversationMemory.contextSummary || - baseThreadState?.contextSummary, - metadata: { - ...(baseThreadState?.metadata || {}), - ...composedConversationMemory.metadata, + console.log("[ConversationStateProtocol] Composed:", { + workflowType: composedConversationState.threadState.workflowState?.type || null, + workflowStatus: composedConversationState.threadState.workflowState?.status || null, + memoryHintsCount: composedConversationState.threadState.memoryHints.length, + contextSummaryPreview: composedConversationState.threadState.contextSummary?.slice(0, 180), + }); + pendingProtocolDebugEvents.push({ + phase: "composed", + payload: { + workflowState: composedConversationState.threadState.workflowState, + memoryHints: composedConversationState.threadState.memoryHints, + contextSummary: composedConversationState.threadState.contextSummary, }, - }; + }); const autoGraphResult = await withOperationTimeout( runAutoAgentGraph({ text: smartAutoInputText, @@ -3342,9 +3332,11 @@ export async function POST(request: Request) { currentUploadImageCount: requestResolvedMediaInput.currentMessageImageUrls.length, historyMessages, context: - composedConversationMemory.contextSummary || autoContextText || undefined, + composedConversationState.composedConversationMemory.contextSummary || + autoContextText || + undefined, preferredModelId: preferredAgentModelId, - threadState: composedThreadState, + threadState: composedConversationState.threadState, }), "runAutoAgentGraph", autoGraphTimeoutMs, @@ -3474,26 +3466,25 @@ export async function POST(request: Request) { Boolean(autoGraphResult.analysis.needsClarification); smartAutoClarificationQuestion = autoGraphResult.analysis.clarifyingQuestions?.[0]; + const targetExecutionOverride = resolveTargetExecutionOverride({ + hasImage, + hasSelectedImageHint, + hasExplicitTargetHint, + hasActionableEditIntent, + executionMode: smartAutoExecution?.mode, + friendlyMessage: smartAutoFriendlyMessage, + optimizedPrompt: smartAutoOptimizedPrompt, + requestText: requestTextContent, + }); + if ( smartAutoNeedsClarification && - hasImage && - hasActionableEditIntent && - (hasSelectedImageHint || hasExplicitTargetHint) && - (smartAutoExecution?.mode === "edit_image" || - smartAutoExecution?.mode === "generate_image") + targetExecutionOverride.shouldBypassClarification ) { smartAutoNeedsClarification = false; smartAutoClarificationQuestion = undefined; - if (!smartAutoOptimizedPrompt && requestTextContent.trim()) { - smartAutoOptimizedPrompt = requestTextContent.trim(); - } - if ( - !smartAutoFriendlyMessage || - /[?？]$/.test(smartAutoFriendlyMessage.trim()) || - /哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage) - ) { - smartAutoFriendlyMessage = "收到，我将按你指定的目标图直接执行。"; - } + smartAutoOptimizedPrompt = targetExecutionOverride.nextOptimizedPrompt; + smartAutoFriendlyMessage = targetExecutionOverride.nextFriendlyMessage; console.log( "[Smart Auto Mode] ✅ Skip clarification due to explicit target/selection context" ); @@ -3504,28 +3495,23 @@ export async function POST(request: Request) { !pendingGridConfirmation || smartAutoInputText === requestTextContent ? detectRequestedStoryboardGrid(requestTextContent) : pendingGridConfirmation; - const isGridTaskCandidate = - smartAutoExecution?.mode === "generate_image" || - smartAutoExecution?.mode === "generate_video" || - autoGraphResult.analysis?.intent === "image_gen" || - autoGraphResult.analysis?.intent === "video_gen"; - if ( - requestedGrid && - !isGridConfirmationReply(requestTextContent) && - !activeStoryboardTaskForChat?.task?.id && - isGridTaskCandidate - ) { + const gridConfirmationResolution = resolveGridConfirmationRequest({ + requestedGrid, + hasActiveStoryboardTask: Boolean(activeStoryboardTaskForChat?.task?.id), + executionMode: smartAutoExecution?.mode, + detectedIntent: autoGraphResult.analysis?.intent, + requestText: requestTextContent, + effectiveInputText: smartAutoInputText, + }); + if (gridConfirmationResolution.shouldRequestConfirmation) { shouldAskGridConfirmation = true; smartAutoNeedsClarification = true; - smartAutoClarificationQuestion = buildGridConfirmationMessage(requestedGrid); - smartAutoFriendlyMessage = smartAutoClarificationQuestion; - pendingGridConfirmation = { - ...requestedGrid, - originalPrompt: smartAutoInputText, - }; + smartAutoClarificationQuestion = gridConfirmationResolution.clarificationMessage; + smartAutoFriendlyMessage = gridConfirmationResolution.clarificationMessage; + pendingGridConfirmation = gridConfirmationResolution.pendingGridConfirmation; console.log("[Smart Auto Mode] 🧱 Require grid confirmation before generation:", { - gridType: requestedGrid.gridType, - shotCount: requestedGrid.shotCount, + gridType: gridConfirmationResolution.pendingGridConfirmation?.gridType, + shotCount: gridConfirmationResolution.pendingGridConfirmation?.shotCount, }); } @@ -4484,7 +4470,7 @@ export async function POST(request: Request) { }, }); - await upsertConversationWorkingState({ + await persistResolvedTargetTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, @@ -4515,13 +4501,10 @@ export async function POST(request: Request) { canvasImageNodes: contextPayload?.canvasImageNodes, } : null, - lastTurnSummary: { - text: requestTextContent.slice(0, 500), - resolutionMode: requestTargetResolution.resolutionMode, - confidence: requestTargetResolution.confidence, - }, lastMessageId: message.id, - incrementTurnCount: true, + text: requestTextContent, + resolutionMode: requestTargetResolution.resolutionMode, + confidence: requestTargetResolution.confidence, }); } catch (memoryError) { console.warn("[Chat API] Failed to persist context-memory state:", memoryError); @@ -4659,40 +4642,15 @@ export async function POST(request: Request) { }); try { - await upsertConversationWorkingState({ + await persistPromptOnlyTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, - currentIntent: "vision", - currentFocusAssetIds: - promptOnlySourceAssetIds.length > 0 ? promptOnlySourceAssetIds : null, - currentFocusImageUrls: promptOnlySourceImageUrls, - lastResolvedTargetAssetIds: - promptOnlySourceAssetIds.length > 0 ? promptOnlySourceAssetIds : null, - lastResolvedTargetImageUrls: promptOnlySourceImageUrls, - lastMessageId: messageId, - lastTurnSummary: { - text: requestTextContent.slice(0, 500), - resolutionMode: "prompt_only", - confidence: 90, - optimizedPrompt: promptOnlyOptimizedPrompt.slice(0, 1000), - promptMemory: buildPromptMemoryUpdate({ - optimizedPrompt: promptOnlyOptimizedPrompt.slice(0, 1000), - sourceIntent: "image_gen", - sourceText: requestTextContent.slice(0, 500), - reusable: true, - }), - imageWorkflowMemory: buildImageWorkflowMemoryUpdate({ - existing: imageWorkflowMemory, - lastIntent: "vision", - lastOptimizedPrompt: promptOnlyOptimizedPrompt.slice(0, 1000), - lastUserPrompt: requestTextContent.slice(0, 500), - lastRequestedImageCount: 1, - lastSourceImageUrls: promptOnlySourceImageUrls, - }), - imageSummary: promptOnlyImageSummary.slice(0, 1200), - sourceImageUrls: promptOnlySourceImageUrls, - }, + messageId, + text: requestTextContent, + optimizedPrompt: promptOnlyOptimizedPrompt, + imageSummary: promptOnlyImageSummary, + sourceImageUrls: promptOnlySourceImageUrls, }); } catch (promptOnlyMemoryError) { console.warn("[Chat API] Failed to persist prompt-only response state:", promptOnlyMemoryError); @@ -4782,31 +4740,13 @@ export async function POST(request: Request) { if (shouldAskGridConfirmation && pendingGridConfirmation) { try { - await upsertConversationWorkingState({ + await persistGridConfirmationTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, - currentIntent: "image_gen", - lastMessageId: messageId, - lastTurnSummary: { - text: requestTextContent.slice(0, 500), - resolutionMode: "grid_confirmation", - confidence: 90, - storyboardMemory: buildStoryboardPreferenceMemoryUpdate({ - existing: storyboardPreferenceMemory, - preferredGridType: pendingGridConfirmation.gridType, - preferredShotCount: pendingGridConfirmation.shotCount, - prefersConfirmation: true, - lastUsedGridType: pendingGridConfirmation.gridType, - lastUsedShotCount: pendingGridConfirmation.shotCount, - }), - pendingGridConfirmation: { - gridType: pendingGridConfirmation.gridType, - shotCount: pendingGridConfirmation.shotCount, - label: pendingGridConfirmation.label, - originalPrompt: pendingGridConfirmation.originalPrompt.slice(0, 1000), - }, - }, + messageId, + text: requestTextContent, + pendingGridConfirmation, }); } catch (gridConfirmationPersistError) { console.warn( @@ -4956,21 +4896,12 @@ export async function POST(request: Request) { }); try { - await upsertConversationWorkingState({ + await persistStoryboardConfirmedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, - currentIntent: "video_gen", - lastMessageId: acknowledgeMessageId, - lastTurnSummary: { - text: requestTextContent.slice(0, 500), - resolutionMode: "storyboard_confirmed", - confidence: 95, - storyboardMemory: buildStoryboardPreferenceMemoryUpdate({ - existing: storyboardPreferenceMemory, - prefersConfirmation: true, - }), - }, + messageId: acknowledgeMessageId, + text: requestTextContent, }); } catch (storyboardConfirmMemoryError) { console.warn( @@ -5440,15 +5371,16 @@ export async function POST(request: Request) { ], }); - try { - await upsertConversationWorkingState({ - userId: session.user.id, - chatId: id, - projectId: contextPayload?.projectId || null, - currentIntent: "video_gen", - lastMessageId: assistantMessageId, - }); - } catch (memoryError) { + try { + await persistVideoWorkflowTurn({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + lastMessageId: assistantMessageId, + text: requestTextContent, + status: "completed", + }); + } catch (memoryError) { console.warn( "[Chat API] Failed to update working state after agent preset video generation:", memoryError, @@ -5536,19 +5468,17 @@ export async function POST(request: Request) { } try { - await upsertConversationWorkingState({ + await persistAgentPresetTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, currentIntent: selectedAgentPreset.type === "video" ? "video_gen" : "image_gen", lastMessageId: message.id, - lastTurnSummary: { - text: requestTextContent.slice(0, 500), - agentPresetId: selectedAgentPreset.id, - agentPresetName: selectedAgentPreset.name, - slotCount: orderedSlots.length, - successCount, - }, + text: requestTextContent, + agentPresetId: selectedAgentPreset.id, + agentPresetName: selectedAgentPreset.name, + slotCount: orderedSlots.length, + successCount, }); } catch (agentPresetMemoryError) { console.warn("[Chat API] Failed to persist agent preset working state:", agentPresetMemoryError); @@ -5874,12 +5804,13 @@ export async function POST(request: Request) { }); try { - await upsertConversationWorkingState({ + await persistVideoWorkflowTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, - currentIntent: "video_gen", lastMessageId: assistantMessageId, + text: requestTextContent, + status: "completed", }); } catch (memoryError) { console.warn( @@ -6186,13 +6117,7 @@ export async function POST(request: Request) { try { const resultAssetIds = resultAssetId ? [resultAssetId] : null; const resultImageUrls = resultOriginalUrl ? [resultOriginalUrl] : null; - const existingImageWorkflowMemory = extractImageWorkflowMemory(requestWorkingState); - const existingSummary = - requestWorkingState?.lastTurnSummary && - typeof requestWorkingState.lastTurnSummary === "object" - ? (requestWorkingState.lastTurnSummary as Record) - : {}; - await upsertConversationWorkingState({ + await persistImageWorkflowTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, @@ -6205,17 +6130,10 @@ export async function POST(request: Request) { lastResolvedTargetAssetIds: resultAssetIds, lastResolvedTargetImageUrls: resultImageUrls, lastMessageId: assistantMessageId, - lastTurnSummary: { - ...existingSummary, - imageWorkflowMemory: buildImageWorkflowMemoryUpdate({ - existing: existingImageWorkflowMemory, - lastIntent: "image_edit", - lastUserPrompt: requestTextContent.slice(0, 500), - lastRequestedImageCount: resultImageUrls?.length || 1, - lastSourceImageUrls: directAgentToolTargetImageUrls, - lastResultImageUrls: resultImageUrls, - }), - }, + lastUserPrompt: requestTextContent, + lastRequestedImageCount: resultImageUrls?.length || 1, + lastSourceImageUrls: directAgentToolTargetImageUrls, + lastResultImageUrls: resultImageUrls, }); } catch (memoryError) { console.warn( @@ -7931,33 +7849,24 @@ export async function POST(request: Request) { try { const storyboardGridType = String((videoMetadata as any)?.gridType || "").trim(); const storyboardShotCount = Number((videoMetadata as any)?.shotCount || 0); - await upsertConversationWorkingState({ + await persistStoryboardAssistantTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, - currentIntent: "video_gen", lastMessageId: messageId, - lastTurnSummary: { - text: requestTextContent.slice(0, 500), - resolutionMode: "storyboard", - confidence: 95, - storyboardMemory: buildStoryboardPreferenceMemoryUpdate({ - existing: storyboardPreferenceMemory, - prefersConfirmation: true, - lastUsedGridType: - storyboardGridType === "grid_2x2" || - storyboardGridType === "grid_3x2" || - storyboardGridType === "grid_3x3" - ? (storyboardGridType as "grid_2x2" | "grid_3x2" | "grid_3x3") - : undefined, - lastUsedShotCount: - storyboardShotCount === 4 || - storyboardShotCount === 6 || - storyboardShotCount === 9 - ? storyboardShotCount - : undefined, - }), - }, + text: requestTextContent, + storyboardGridType: + storyboardGridType === "grid_2x2" || + storyboardGridType === "grid_3x2" || + storyboardGridType === "grid_3x3" + ? (storyboardGridType as "grid_2x2" | "grid_3x2" | "grid_3x3") + : undefined, + storyboardShotCount: + storyboardShotCount === 4 || + storyboardShotCount === 6 || + storyboardShotCount === 9 + ? storyboardShotCount + : undefined, }); } catch (storyboardMemoryPersistError) { console.warn( @@ -8020,6 +7929,18 @@ export async function POST(request: Request) { DEFAULT_STREAM_KEEPALIVE_INTERVAL_MS ); stopStreamKeepalive = stopDefaultKeepalive; + setConversationStateProtocolEventObserver((event) => { + writeProtocolDebugChunk( + dataStream, + Boolean(debugProtocol), + { + phase: "event", + type: event.type, + payload: event.payload, + }, + `protocol:event:${event.type}` + ); + }); safeWriteStreamChunk( dataStream, { @@ -8028,6 +7949,29 @@ export async function POST(request: Request) { }, "chat-default:init" ); + writeProtocolDebugChunk( + dataStream, + Boolean(debugProtocol), + { + phase: "request", + chatId: id, + selectedChatModel, + hasHiddenPrompt: Boolean(hiddenPrompt), + hasAutoContext: Boolean(autoContext), + }, + "protocol:request" + ); + for (const event of pendingProtocolDebugEvents) { + writeProtocolDebugChunk( + dataStream, + Boolean(debugProtocol), + { + phase: event.phase, + ...event.payload, + }, + `protocol:${event.phase}` + ); + } try { // 构建工具对象，只包含启用的工具 const tools: any = {}; @@ -8977,6 +8921,7 @@ export async function POST(request: Request) { }) ); } finally { + setConversationStateProtocolEventObserver(null); stopStreamKeepalive = stopDefaultKeepalive; } }, diff --git a/app/(chat)/api/chat/schema.ts b/app/(chat)/api/chat/schema.ts index ce362429a..eb57f24a2 100644 --- a/app/(chat)/api/chat/schema.ts +++ b/app/(chat)/api/chat/schema.ts @@ -85,6 +85,7 @@ export const postRequestBodySchema = z.object({ selectedChatModel: z.string().min(1).max(128), selectedVisibilityType: z.enum(["public", "private"]), deepThinkingEnabled: z.boolean().optional(), + debugProtocol: z.boolean().optional(), videoParams: videoParamsSchema, // 视频参数 imageParams: imageParamsSchema, // 🆕 图片生成参数 contextPayload: contextPayloadSchema, diff --git a/components/chat.tsx b/components/chat.tsx index ef94165b1..e6c19ff1a 100644 --- a/components/chat.tsx +++ b/components/chat.tsx @@ -30,6 +30,7 @@ import { requestLongTaskResult } from "@/lib/client/long-task"; import { fetchUploadWithRetry } from "@/lib/client/upload-with-retry"; import { fetcher, fetchWithErrorHandlers, generateUUID } from "@/lib/utils"; import { Artifact } from "@/components/artifact"; +import { ProtocolDebugPanel } from "@/components/protocol-debug-panel"; import { useDataStream } from "@/components/data-stream-provider"; import { Messages } from "@/components/messages"; import { MultimodalInput } from "@/components/multimodal-input"; @@ -318,6 +319,11 @@ export function Chat({ userType?: UserType; }) { const router = useRouter(); + const searchParams = useSearchParams(); + const protocolDebugEnabled = searchParams.get("debugProtocol") === "1"; + const [protocolDebugEvents, setProtocolDebugEvents] = useState< + Array<{ id: string; at: number; payload: Record }> + >([]); const { visibilityType } = useChatVisibility({ chatId: id, @@ -600,6 +606,7 @@ export function Chat({ message: request.messages.at(-1), selectedChatModel: currentModelIdRef.current, selectedVisibilityType: visibilityType, + debugProtocol: protocolDebugEnabled, videoParams: videoParamsRef.current || undefined, // 🆕 传递视频参数（使用ref获取最新值） imageParams: imageParamsRef.current || undefined, // 🆕 传递图片参数（使用ref获取最新值） ...request.body, @@ -753,6 +760,24 @@ export function Chat({ console.error("[Chat] Failed to parse data-appendMessage:", e); } } + + if (dataPart.type === "data-protocolDebug") { + try { + const payload = + typeof dataPart.data === "string" ? JSON.parse(dataPart.data) : dataPart.data; + if (!payload || typeof payload !== "object") return; + setProtocolDebugEvents((prev) => [ + ...prev.slice(-59), + { + id: generateUUID(), + at: Date.now(), + payload, + }, + ]); + } catch (e) { + console.error("[Chat] Failed to parse data-protocolDebug:", e); + } + } }, onFinish: async () => { mutate(unstable_serialize(getChatHistoryPaginationKey)); @@ -1074,7 +1099,6 @@ export function Chat({ } }, [status, currentModelId, messages, startGeneration, updateImageGenerationPhase]); - const searchParams = useSearchParams(); const query = searchParams.get("query"); const editImageUrl = searchParams.get("editImage"); @@ -2070,6 +2094,7 @@ export function Chat({ /> )} + ; +}; + +export function ProtocolDebugPanel({ + enabled, + events, +}: { + enabled: boolean; + events: ProtocolDebugEvent[]; +}) { + if (!enabled) return null; + + return ( +
+
+ + Protocol Debug + +
+ {events.length === 0 ? ( +
+ 暂无协议调试事件 +
+ ) : ( + events.map((event) => ( +
+
+ {new Date(event.at).toLocaleTimeString()} + {" · "} + {String(event.payload.phase || event.payload.type || "unknown")} +
+ 
+                  {JSON.stringify(event.payload, null, 2)}
+
+
+ )) + )} +
+
+
+ ); +} + diff --git a/docs/Agent模式与DeerFlow对比-大白话版-2026-03-13.md b/docs/Agent模式与DeerFlow对比-大白话版-2026-03-13.md new file mode 100644 index 000000000..1c158b00f --- /dev/null +++ b/docs/Agent模式与DeerFlow对比-大白话版-2026-03-13.md @@ -0,0 +1,776 @@ +# Agent 模式与 DeerFlow 对比说明（大白话版） + +> 写给谁看：不懂技术，但想真正看明白这个项目的人。 +> 更新时间：2026-03-13。 +> 本文已经按**今天实际落地的代码**重写，不再沿用旧判断。 + +--- + +## 先说结论 + +### 一句话结论 + +- **我们这个项目**，现在已经不是“普通聊天 + 一堆工具”的形态了。 + 它正在变成一个**面向视觉生产的统一 Agent Runtime**。 + +- **DeerFlow** 仍然更像一个**通用超级智能体底座**。 + 它更擅长：通用任务拆解、子代理协作、沙箱执行、技能体系、长期记忆、MCP 集成。 + +### 现在最准确的判断 + +过去说我们项目“还只是主脑 + 一堆补丁”，今天之后就不完全对了。 + +因为我们已经补上了这些关键层： + +1. `conversation action resolver` +2. `conversation memory composer` +3. `conversation thread state builder` +4. `conversation state protocol` +5. `conversation state protocol events` +6. `human-confirmation-gate` +7. `storyboard-subagent` +8. `image-generation-subagent` +9. `image-edit-subagent` + +也就是说： + +**我们还不是 DeerFlow 那种通用超级智能体平台，但已经明确进入“统一运行时 Agent”阶段。** + +--- + +## 1. 这个项目现在的 Agent，到底是什么 + +这个项目现在有两层 Agent： + +### 1.1 运行时 Agent + +这是核心大脑，负责： + +1. 理解用户这轮到底在说什么 +2. 判断用户到底指哪张图 +3. 判断这是普通聊天、看图、修图、生图、视频、还是分镜任务 +4. 恢复上一轮工作记忆 +5. 根据当前 workflow 决定是继续、修改、确认、取消，还是新开任务 +6. 选择工具、选择模型、执行 + +主要代码： + +- `app/(chat)/api/chat/route.ts` +- `lib/ai/auto-agent-graph.ts` +- `lib/ai/smart-analyzer.ts` +- `lib/ai/skill-router.ts` +- `lib/ai/runtime/*` +- `lib/ai/conversation-*` + +### 1.2 预设智能体层 + +这层更像“模板化工作流”： + +- 默认模型 +- 固定提示词 +- 固定槽位 +- 固定参数 + +主要代码： + +- `app/api/agents/*` +- `components/agents/*` +- `types/agent.ts` + +所以一定要分清： + +- **运行时 Agent** = 真正思考和调度的系统大脑 +- **预设智能体** = 预先配置好的业务模板 + +--- + +## 2. 今天升级后，Agent 底层结构已经变成什么样了 + +今天最重要的升级，不是多接了一个模型，而是把 Agent 的“脑子”和“记忆”开始统一起来。 + +### 2.1 现在已经有一套会话协议层 + +我们已经新增了以下模块： + +- `lib/ai/conversation-action-resolver.ts` +- `lib/ai/conversation-memory-composer.ts` +- `lib/ai/conversation-thread-state-builder.ts` +- `lib/ai/conversation-state-protocol.ts` +- `lib/ai/conversation-state-protocol-events.ts` + +你可以把它理解成： + +#### 以前 + +像很多项目一样，是： + +- 用户说一句 +- 系统临时判断一次 +- 执行一次 +- 结果存一点 + +每轮都更像“重新开始” + +#### 现在 + +变成： + +- **先恢复这段对话的会话状态** +- **再判断这句话属于什么动作** +- **再拼装统一上下文** +- **再让 planner 决定下一步** +- **执行完成后，再按统一协议写回状态** + +也就是说，我们开始有真正的： + +**会话状态协议** + +而不是单轮聊天补丁。 + +### 2.2 现在已经有统一的动作协议 + +现在系统会先判断，这轮对话属于哪种动作： + +- `default` +- `grid_confirmation_followup` +- `active_storyboard_followup` +- `image_edit_followup` +- `prompt_carry_forward` + +这一步在： + +- `lib/ai/conversation-action-resolver.ts` + +这件事很重要，因为它解决了一个以前特别容易乱的问题： + +#### 用户到底是在： + +- 开一个新任务 +- 继续上一轮任务 +- 修改上一轮任务 +- 确认一个待执行任务 +- 取消一个待执行任务 + +以前这些判断很多散在 `chat/route.ts` 里。 +今天开始，它们在往统一协议收。 + +### 2.3 现在已经有统一的 thread state + +现在运行时 Agent 已经不只是收到一段文本了。 + +它会收到一个 `threadState`，里面可能包含： + +- 当前 workflow +- 当前 workflow 状态 +- 当前 memory hints +- 当前上下文摘要 +- 当前 prompt memory +- 当前 image workflow memory +- 当前 storyboard preference memory + +这一步主要在： + +- `lib/ai/runtime/agent-thread-state.ts` +- `lib/ai/conversation-thread-state-builder.ts` +- `lib/video-agent/agent-thread-state.ts` + +这意味着： + +**Agent 已经开始像人一样“带着上下文继续理解”，不是每轮都失忆。** + +### 2.4 现在已经有统一的读写协议 + +今天另一个很关键的变化是： + +`app/(chat)/api/chat/route.ts` 里已经不再直接调用 `upsertConversationWorkingState(...)`。 + +写回现在统一走： + +- `persistConversationStateEvent(...)` +- `persistPromptOnlyTurn(...)` +- `persistGridConfirmationTurn(...)` +- `persistStoryboardConfirmedTurn(...)` +- `persistImageWorkflowTurn(...)` +- `persistResolvedTargetTurn(...)` +- `persistVideoWorkflowTurn(...)` +- `persistAgentPresetTurn(...)` +- `persistStoryboardAssistantTurn(...)` + +也就是说： + +**聊天主路由已经从“自己管理记忆结构”，升级为“只触发语义事件”。** + +这是一件很大的架构进步。 + +### 2.5 现在已经有协议级调试流 + +当请求里带 `debugProtocol: true` 时，前端会收到： + +- `request` +- `bootstrap` +- `composed` +- `event` + +这些 `data-protocolDebug` 事件。 + +前端也已经能显示协议调试面板： + +- `components/protocol-debug-panel.tsx` +- `components/chat.tsx` + +这意味着你们现在不是只能“看结果”，而是能直接看： + +- 这轮命中了什么动作 +- 这轮 workflow 是什么 +- Agent 拿到的 memory hints 是什么 +- 这轮写回了哪个协议事件 + +这对后面继续调优 Agent 非常重要。 + +--- + +## 3. 现在这套 Agent 实际怎么工作 + +### 3.1 先收素材 + +系统先统一整理： + +- 当前上传图 +- 画布选中图 +- 历史图 +- 上一轮结果图 +- 文本里贴的 URL 图 + +主要代码： + +- `lib/ai/context-memory/media-input.ts` +- `lib/ai/context-memory/target-resolution.ts` + +### 3.2 恢复会话状态 + +系统会先从数据库恢复： + +- prompt memory +- image workflow memory +- storyboard preference memory +- active storyboard task +- pending grid confirmation + +然后用协议层统一拼成： + +- `actionResolution` +- `threadState` +- `contextSummary` +- `memoryHints` + +主要代码： + +- `lib/ai/conversation-state-protocol.ts` +- `lib/ai/conversation-memory-composer.ts` +- `lib/ai/conversation-thread-state-builder.ts` + +### 3.3 先判断“这轮属于什么动作” + +比如： + +- “根据刚才那个提示词来做这张图” +- “把第三镜头改成近景” +- “先别生成了” +- “按 6 宫格来” +- “确认开始生成视频” + +系统会先给这句话归类。 + +这不是最终业务意图，而是： + +**当前回合的会话动作** + +这一步特别像一个“对话状态机入口”。 + +### 3.4 再进入 Auto Agent Graph + +当前 graph 主要节点是： + +1. `confirmationGate` +2. `analyze` +3. `plan` +4. `prepareExecution` +5. `optimize` +6. `routeModel` + +也就是： + +```mermaid +flowchart TD + A["用户输入"] --> B["恢复会话状态"] + B --> C["conversation action resolver"] + C --> D["threadState + memory composer"] + D --> E["confirmationGate"] + E --> F["analyze"] + F --> G["plan"] + G --> H["prepareExecution"] + H --> I{"要不要优化提示词/分镜"} + I -- "要" --> J["optimize"] + I -- "不要" --> K["routeModel"] + J --> K + K --> L["执行工具 / 模型"] + L --> M["协议化写回状态"] +``` + +### 3.5 专家和子代理会介入 + +现在已经有明确的子代理/专家层： + +- `storyboard-subagent` +- `image-generation-subagent` +- `image-edit-subagent` +- `canvas-specialist` + +对应文件： + +- `lib/video-agent/storyboard-subagent.ts` +- `lib/ai/image-generation-subagent.ts` +- `lib/ai/image-edit-subagent.ts` +- `lib/ai/specialists/index.ts` +- `lib/ai/skill-router.ts` + +虽然它们还没有发展成 DeerFlow 那种“很多通用子代理并发工作”的程度， +但已经不是简单的路由白名单了。 + +它们现在已经会： + +- 给 planner 注入强约束 +- 修正执行计划 +- 改变工具选择 +- 改变结果回画布策略 +- 改变是否继续执行 + +所以这里要纠正一个旧说法： + +> 旧判断：我们项目还只是“主脑 + 一堆专家修正”，不算真正子代理。 + +这个说法现在只算“部分正确”。 + +更准确的说法应该是： + +> 当前项目已经进入了“子代理化早期阶段”， +> 但还没有发展成 DeerFlow 那种真正的大规模通用子代理系统。 + +--- + +## 4. 普通生图对话现在怎么理解了 + +这是今天升级里非常重要的一块。 + +### 4.1 以前 + +用户说： + +1. “帮我根据这张图优化提示词” +2. 下一轮再说：“按上一个优化后的提示词来优化这张图” + +以前很多系统会丢上下文，只看到第二句话。 + +### 4.2 现在 + +系统已经有： + +- `promptMemory` +- `imageWorkflowMemory` +- `image_generation_followup workflow` + +这意味着 Agent 更可能理解成： + +> 使用上一轮优化后的提示词 +> + 当前这轮的新图片 +> + 当前用户的数量/画幅/清晰度需求 +> 来继续执行这轮生图任务 + +这一步主要在： + +- `lib/ai/prompt-memory.ts` +- `lib/ai/image-workflow-memory.ts` +- `lib/ai/image-generation-subagent.ts` + +### 4.3 多张图能力也已经升级 + +今天还补了一个很重要的底层能力： + +如果用户要 6 张、8 张、12 张图，而当前模型单次只能返回 1 张或 2 张， +系统会自动分批调用，直到凑够目标数量。 + +主要代码： + +- `lib/canvas/generate-image.ts` + +这说明现在系统对： + +- “连续生图” +- “多张图需求” +- “提示词接力” + +已经开始形成统一能力，不再是三个分开的 feature。 + +--- + +## 5. 连续修图对话现在怎么理解了 + +今天还补了： + +- `image_edit_followup workflow` +- `image-edit-subagent` + +这意味着如果用户说： + +- “继续改上一张结果图” +- “保留主体不变，只把背景换掉” +- “上一版再亮一点” +- “这张继续优化” + +系统更可能理解成： + +> 这是连续修图任务 +> 目标图优先取上一轮结果图或当前绑定目标图 +> 不要误判成重新生图 + +这一步主要在： + +- `lib/ai/image-edit-subagent.ts` +- `lib/ai/conversation-thread-state-builder.ts` +- `lib/ai/conversation-action-resolver.ts` + +--- + +## 6. 分镜视频链路现在是什么水平 + +这里也要纠正旧判断。 + +### 6.1 以前可以说“文档多、草图多、骨架多” + +### 6.2 现在已经不是纯方案阶段了 + +现在已经实际落地了这些模块： + +- `lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts` +- `lib/video-agent/execute-storyboard-task.ts` +- `lib/video-agent/revise-storyboard-task.ts` +- `lib/video-agent/storyboard-command-interpreter.ts` +- `lib/video-agent/storyboard-preference-memory.ts` +- `lib/video-agent/task-store.ts` +- `app/(chat)/api/video/storyboard/*` + +而且链路已经具备： + +1. 分镜拼图检测 +2. 4/6/9 宫格识别 +3. 镜头切片 +4. 高清预览图生成 +5. 用户确认门 +6. 单镜头修改 +7. 单镜头重生高清图 +8. 确认后逐镜头开始视频执行 + +所以这里也要纠正一个旧说法： + +> 旧判断：分镜视频还主要停留在方案文档阶段。 + +现在更准确的说法是： + +> 分镜视频已经进入“半落地工作流阶段”， +> 不是只有文档，已经有实际 runtime、task-store、API 和前端联动。 +> 但还没有完全进化成通用化的视频工作流平台。 + +--- + +## 7. 这个项目当前到底像什么 + +今天之后，再用一句话总结这个项目： + +**它已经不是单纯的多模态聊天应用，而是一个“面向视觉生产的 Agent 工作流系统”。** + +更准确一点说，它现在像： + +### 产品层 + +- 图片编辑 +- 图片生成 +- 视频生成 +- 分镜工作流 +- 无限画布 + +### Agent 层 + +- 统一动作解析 +- 统一记忆合成 +- 统一 thread state +- 统一 workflow state +- 统一写回协议 +- 统一协议事件 +- 可选调试面板 + +也就是说，它现在已经开始具备一个真正 Agent Runtime 的轮廓。 + +--- + +## 8. DeerFlow 到底是什么，今天要怎么正确看 + +根据 DeerFlow 官方仓库和架构说明，最准确的大白话是： + +**DeerFlow 是一个“通用超级智能体运行底座”，不是一个专门给视觉生产做深度业务适配的产品。** + +它的强项主要在： + +- 通用任务拆解 +- Sub-agents +- Skills +- Sandbox +- Middleware +- Long-term Memory +- MCP 集成 + +简单说： + +### DeerFlow 擅长 + +- 做一个什么都能接的“超级智能体平台” + +### 我们现在擅长 + +- 把图片、视频、画布、分镜这种视觉业务做深 + +### DeerFlow 强于我们的地方 + +1. 通用子代理体系更成熟 +2. 沙箱执行更完整 +3. MCP 和工具生态更平台化 +4. 长期记忆体系更偏平台级 + +### 我们强于 DeerFlow 的地方 + +1. 图片目标定位更细 +2. 画布联动更深 +3. 连续修图/连续生图更贴业务 +4. 分镜视频确认流程更贴实际视觉生产 + +--- + +## 9. 现在该怎么评价我们和 DeerFlow 的差距 + +现在最准确的说法不应该是： + +> 我们只是一个普通业务项目，DeerFlow 才是真 Agent。 + +这个说法已经不对了。 + +现在更准确的说法是： + +> 我们已经开始构建自己的统一 Agent Runtime， +> 但它仍然是**视觉业务优先**的 Agent Runtime， +> 不是 DeerFlow 那种**通用超级 Agent 底座**。 + +所以差距主要不在“有没有 Agent”，而在： + +### 我们还缺这些 + +1. 更彻底的 route 解耦 + `app/(chat)/api/chat/route.ts` 仍然是超大总控（现在 9065 行） + +2. 更强的通用子代理并发体系 + 当前已有 subagent，但还不算真正的多工人协作系统 + +3. 更成熟的长期记忆 + 现在更强的是会话工作记忆，不是跨会话长期人格/知识记忆 + +4. 更彻底的执行环境抽象 + 我们强在视觉业务执行，不强在通用沙箱型任务执行 + +### 但我们已经补上的 + +1. 统一会话协议 +2. 统一 workflow 语义 +3. 统一协议事件 +4. 统一前端调试入口 +5. 统一图片/分镜连续任务理解 + +这说明我们不是“还没开始”，而是已经跨过了最难的第一阶段。 + +--- + +## 10. 今天最该纠正的旧判断 + +下面这些是现在必须纠正的： + +### 旧判断 1 + +> 当前项目还是一堆业务补丁，没形成统一运行时。 + +#### 现在更准确 + +已经形成了统一运行时骨架： + +- action resolver +- memory composer +- thread state builder +- state protocol +- event schema +- debug stream + +还没彻底清完旧逻辑，但不能再说“没有统一运行时”了。 + +### 旧判断 2 + +> specialist 只是一些工具白名单，不算真子代理。 + +#### 现在更准确 + +已经进入早期子代理阶段： + +- `storyboard-subagent` +- `image-generation-subagent` +- `image-edit-subagent` + +它们已经能改变 planning hints、tool 选择、workflow 行为和对话理解。 + +### 旧判断 3 + +> 当前项目只擅长单轮修图/生图。 + +#### 现在更准确 + +多轮能力已经明显增强： + +- 连续生图 +- 连续修图 +- 提示词接力 +- 分镜确认 +- 单镜头修改 +- 单镜头重生 + +### 旧判断 4 + +> 这套系统不够像真人对话。 + +#### 现在更准确 + +它离“真人协作理解”还没完全到位，但已经开始具备几个核心特征： + +- 记得上一轮的提示词 +- 记得上一轮的目标图和结果图 +- 知道当前这句话是在继续任务还是新开任务 +- 知道贵活要不要先确认 +- 能理解“继续改上一版 / 按上一个 prompt 来 / 第三镜头改成……” + +这已经不是普通问答模型的水平了。 + +--- + +## 11. 我现在给这个项目的最准确定位 + +今天之后，我会把这个项目定位成： + +> **一个以视觉生产为核心、正在持续进化成统一 Agent Runtime 的内容创作系统。** + +不是： + +- 普通聊天机器人 +- 只会调几个工具的脚手架 +- 纯文档阶段的 Agent 方案 + +也还不是： + +- DeerFlow 那种通用超级 Agent 操作系统 + +而是处在中间的一个非常关键阶段： + +**业务已经很深，底座开始统一。** + +这其实是最有价值的阶段。 + +--- + +## 12. 现在最值得继续做的事 + +如果后面继续投入，最值得的方向不是乱加功能，而是继续收敛： + +### 12.1 继续把旧逻辑迁入统一协议 + +现在最该做的，是继续把 `chat/route.ts` 里剩下的零散判断收进： + +- `conversation state protocol` +- subagents +- workflow state + +### 12.2 把协议消费侧再做强 + +现在已经有协议调试事件和前端调试面板了，下一步可以继续做： + +- 后台协议事件查看 +- 任务追踪面板 +- Agent 解释路径可视化 + +### 12.3 长期记忆继续补强 + +下一步建议补： + +- 用户长期风格偏好 +- 常用画幅/清晰度/数量偏好 +- 品牌/行业上下文 +- 跨项目偏好 + +### 12.4 真正统一“图片/视频/画布/分镜”工作流状态 + +现在已经有： + +- `storyboard_video` +- `image_generation_followup` +- `image_edit_followup` + +后面可以继续扩成完整的统一 workflow catalog。 + +--- + +## 13. 最后一句人话总结 + +### DeerFlow + +像一个通用的超级智能体操作系统。 + +### 我们现在的项目 + +像一个已经非常懂视觉生产业务、而且今天开始真正长出统一 Agent 神经系统的创作引擎。 + +### 今天之后最准确的一句话 + +**不要再把我们看成“很多智能 patch 拼起来的聊天应用”， +它已经在往“统一会话协议驱动的视觉 Agent Runtime”演进了。** + +--- + +## 资料来源 + +### 本项目代码与文档 + +- `app/(chat)/api/chat/route.ts` +- `app/(chat)/api/chat/schema.ts` +- `components/chat.tsx` +- `components/protocol-debug-panel.tsx` +- `lib/ai/auto-agent-graph.ts` +- `lib/ai/smart-analyzer.ts` +- `lib/ai/skill-router.ts` +- `lib/ai/specialists/index.ts` +- `lib/ai/conversation-action-resolver.ts` +- `lib/ai/conversation-memory-composer.ts` +- `lib/ai/conversation-thread-state-builder.ts` +- `lib/ai/conversation-state-protocol.ts` +- `lib/ai/conversation-state-protocol-events.ts` +- `lib/ai/prompt-memory.ts` +- `lib/ai/image-workflow-memory.ts` +- `lib/ai/image-generation-subagent.ts` +- `lib/ai/image-edit-subagent.ts` +- `lib/video-agent/*` +- `docs/DeerFlow调研与Agent升级建议-2026-03-12.md` + +### DeerFlow 官方资料 + +- 官方仓库 README： +- 架构/运行说明：仓库内 `README`、`backend/README`、`backend/CLAUDE.md` diff --git a/lib/ai/conversation-state-protocol-events.ts b/lib/ai/conversation-state-protocol-events.ts new file mode 100644 index 000000000..b46c10648 --- /dev/null +++ b/lib/ai/conversation-state-protocol-events.ts @@ -0,0 +1,123 @@ +import "server-only"; + +import { z } from "zod"; + +const stringArraySchema = z.array(z.string().trim()).optional(); +const nullableStringArraySchema = stringArraySchema.nullable().optional(); + +export const conversationStateEventSchemas = { + prompt_only_ready: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + messageId: z.string().uuid(), + text: z.string(), + optimizedPrompt: z.string().min(1), + imageSummary: z.string().optional(), + sourceImageUrls: nullableStringArraySchema, + }), + grid_confirmation_requested: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + messageId: z.string().uuid(), + text: z.string(), + pendingGridConfirmation: z.object({ + gridType: z.enum(["grid_2x2", "grid_3x2", "grid_3x3"]), + shotCount: z.union([z.literal(4), z.literal(6), z.literal(9)]), + label: z.string().min(1), + originalPrompt: z.string().min(1), + }), + }), + storyboard_confirmed: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + messageId: z.string().uuid(), + text: z.string(), + }), + image_workflow_updated: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + currentIntent: z.enum(["image_gen", "image_edit", "vision"]), + currentSubIntent: z.string().nullable().optional(), + currentFocusAssetIds: nullableStringArraySchema, + currentFocusImageUrls: nullableStringArraySchema, + lastGeneratedAssetIds: nullableStringArraySchema, + lastGeneratedImageUrls: nullableStringArraySchema, + lastResolvedTargetAssetIds: nullableStringArraySchema, + lastResolvedTargetImageUrls: nullableStringArraySchema, + lastMessageId: z.string().uuid(), + lastUserPrompt: z.string(), + lastRequestedImageCount: z.number().int().min(1).max(12).optional(), + lastSourceImageUrls: nullableStringArraySchema, + lastResultImageUrls: nullableStringArraySchema, + }), + target_resolved: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + currentIntent: z.string().nullable().optional(), + currentFocusAssetIds: nullableStringArraySchema, + currentFocusImageUrls: nullableStringArraySchema, + selectedAssetIds: nullableStringArraySchema, + selectedImageUrls: nullableStringArraySchema, + lastUploadedAssetIds: nullableStringArraySchema, + lastUploadedImageUrls: nullableStringArraySchema, + lastReferencedAssetIds: nullableStringArraySchema, + lastReferencedImageUrls: nullableStringArraySchema, + lastResolvedTargetAssetIds: nullableStringArraySchema, + lastResolvedTargetImageUrls: nullableStringArraySchema, + activeCanvasNodeIds: nullableStringArraySchema, + lastCanvasSnapshot: z.unknown().optional(), + lastMessageId: z.string().uuid(), + text: z.string(), + resolutionMode: z.string(), + confidence: z.number(), + }), + video_workflow_updated: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + lastMessageId: z.string().uuid(), + text: z.string().optional(), + status: z.string().optional(), + }), + agent_preset_completed: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + currentIntent: z.enum(["image_gen", "video_gen"]), + lastMessageId: z.string().uuid(), + text: z.string(), + agentPresetId: z.string().uuid(), + agentPresetName: z.string().min(1), + slotCount: z.number().int().min(0), + successCount: z.number().int().min(0), + }), + storyboard_assistant_updated: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + lastMessageId: z.string().uuid(), + text: z.string(), + storyboardGridType: z.enum(["grid_2x2", "grid_3x2", "grid_3x3"]).optional(), + storyboardShotCount: z.union([z.literal(4), z.literal(6), z.literal(9)]).optional(), + }), +} as const; + +export type ConversationStateEventType = keyof typeof conversationStateEventSchemas; + +export type ConversationStateEventPayload = z.infer< + (typeof conversationStateEventSchemas)[T] +>; + +export type ConversationStateEvent = + { + [K in ConversationStateEventType]: { + type: K; + payload: ConversationStateEventPayload; + }; + }[ConversationStateEventType]; + diff --git a/lib/ai/conversation-state-protocol.ts b/lib/ai/conversation-state-protocol.ts new file mode 100644 index 000000000..1b1820bfd --- /dev/null +++ b/lib/ai/conversation-state-protocol.ts @@ -0,0 +1,712 @@ +import "server-only"; + +import { + getConversationWorkingStateByChatId, + upsertConversationWorkingState, +} from "@/lib/db/queries"; +import { + conversationStateEventSchemas, + type ConversationStateEvent, + type ConversationStateEventPayload, + type ConversationStateEventType, +} from "@/lib/ai/conversation-state-protocol-events"; +import { + buildImageWorkflowMemoryUpdate, + extractImageWorkflowMemory, +} from "@/lib/ai/image-workflow-memory"; +import type { ResolvedChatMediaInput, ResolvedTargetAssets } from "@/lib/ai/context-memory/types"; +import { + buildGridConfirmationMessage, + isGridConfirmationReply, + type PendingGridConfirmation, + resolveConversationAction, + extractPendingGridConfirmationFromWorkingState, +} from "@/lib/ai/conversation-action-resolver"; +import { + composeConversationMemory, + type ComposedConversationMemory, +} from "@/lib/ai/conversation-memory-composer"; +import { + buildConversationThreadState, +} from "@/lib/ai/conversation-thread-state-builder"; +import type { ImageWorkflowMemory } from "@/lib/ai/image-workflow-memory"; +import type { PromptMemoryEntry } from "@/lib/ai/prompt-memory"; +import { + extractConfirmedPromptFromWorkingState, + extractPromptMemoryFromWorkingState, + buildPromptMemoryUpdate, +} from "@/lib/ai/prompt-memory"; +import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state"; +import type { StoryboardPreferenceMemory } from "@/lib/video-agent/storyboard-preference-memory"; +import { + extractStoryboardPreferenceMemory, + buildStoryboardPreferenceMemoryUpdate, +} from "@/lib/video-agent/storyboard-preference-memory"; + +type StoryboardTaskDetail = Parameters[0]["activeStoryboardTask"]; + +export interface ConversationStateProtocolBootstrap { + workingState: { + lastTurnSummary?: unknown; + } | null; + activeStoryboardTask: StoryboardTaskDetail | null; + storyboardPreferenceMemory: StoryboardPreferenceMemory | null; + imageWorkflowMemory: ImageWorkflowMemory | null; + promptMemory: PromptMemoryEntry | null; + pendingGridConfirmation: PendingGridConfirmation | null; + confirmedPrompt: string | undefined; + actionResolution: ReturnType; +} + +export interface ConversationStateProtocolComposed { + baseThreadState: AgentThreadState | null; + composedConversationMemory: ComposedConversationMemory; + threadState: AgentThreadState; +} + +export function resolveTargetExecutionOverride(params: { + hasImage: boolean; + hasSelectedImageHint: boolean; + hasExplicitTargetHint: boolean; + hasActionableEditIntent: boolean; + executionMode?: string; + friendlyMessage?: string; + optimizedPrompt?: string; + requestText: string; +}) { + const isExecutionMode = + params.executionMode === "edit_image" || params.executionMode === "generate_image"; + const shouldBypassClarification = + params.hasImage && + params.hasActionableEditIntent && + (params.hasSelectedImageHint || params.hasExplicitTargetHint) && + isExecutionMode; + + if (!shouldBypassClarification) { + return { + shouldBypassClarification: false, + nextFriendlyMessage: params.friendlyMessage, + nextOptimizedPrompt: params.optimizedPrompt, + }; + } + + const shouldReplaceFriendlyMessage = + !params.friendlyMessage || + /[?？]$/.test(params.friendlyMessage.trim()) || + /哪一张|哪个|请先确认/.test(params.friendlyMessage); + + return { + shouldBypassClarification: true, + nextFriendlyMessage: shouldReplaceFriendlyMessage + ? "收到，我将按你指定的目标图直接执行。" + : params.friendlyMessage, + nextOptimizedPrompt: params.optimizedPrompt || params.requestText.trim() || undefined, + }; +} + +export function resolveGridConfirmationRequest(params: { + requestedGrid: PendingGridConfirmation | null; + hasActiveStoryboardTask: boolean; + executionMode?: string; + detectedIntent?: string; + requestText: string; + effectiveInputText: string; +}) { + const isGridTaskCandidate = + params.executionMode === "generate_image" || + params.executionMode === "generate_video" || + params.detectedIntent === "image_gen" || + params.detectedIntent === "video_gen"; + + const shouldRequestConfirmation = + Boolean(params.requestedGrid) && + !isGridConfirmationReply(params.requestText) && + !params.hasActiveStoryboardTask && + isGridTaskCandidate; + + if (!shouldRequestConfirmation || !params.requestedGrid) { + return { + shouldRequestConfirmation: false, + pendingGridConfirmation: null as PendingGridConfirmation | null, + clarificationMessage: undefined, + }; + } + + return { + shouldRequestConfirmation: true, + pendingGridConfirmation: { + ...params.requestedGrid, + originalPrompt: params.effectiveInputText, + }, + clarificationMessage: buildGridConfirmationMessage(params.requestedGrid), + }; +} + +type ConversationStateProtocolPersistParams = { + userId: string; + chatId: string; + projectId?: string | null; + currentIntent?: string | null; + currentSubIntent?: string | null; + currentFocusAssetIds?: string[] | null; + currentFocusImageUrls?: string[] | null; + selectedAssetIds?: string[] | null; + selectedImageUrls?: string[] | null; + lastUploadedAssetIds?: string[] | null; + lastUploadedImageUrls?: string[] | null; + lastGeneratedAssetIds?: string[] | null; + lastGeneratedImageUrls?: string[] | null; + lastReferencedAssetIds?: string[] | null; + lastReferencedImageUrls?: string[] | null; + lastResolvedTargetAssetIds?: string[] | null; + lastResolvedTargetImageUrls?: string[] | null; + activeCanvasNodeIds?: string[] | null; + lastCanvasSnapshot?: unknown; + lastMessageId?: string | null; + incrementTurnCount?: boolean; + lastTurnSummaryPatch?: Record; +}; + +type ConversationStateProtocolEventObserver = (event: { + type: ConversationStateEventType; + payload: Record; +}) => void; + +let protocolEventObserver: ConversationStateProtocolEventObserver | null = null; + +export function bootstrapConversationStateProtocol(params: { + text: string; + workingState?: { + lastTurnSummary?: unknown; + } | null; + activeStoryboardTask?: StoryboardTaskDetail | null; + hasImageInputForPromptCarryForward: boolean; +}) { + const storyboardPreferenceMemory = extractStoryboardPreferenceMemory(params.workingState); + const imageWorkflowMemory = extractImageWorkflowMemory(params.workingState); + const promptMemory = extractPromptMemoryFromWorkingState(params.workingState); + const pendingGridConfirmation = + extractPendingGridConfirmationFromWorkingState(params.workingState); + const confirmedPrompt = + extractConfirmedPromptFromWorkingState(params.workingState) || promptMemory?.optimizedPrompt; + const actionResolution = resolveConversationAction({ + text: params.text, + pendingGridConfirmation, + hasActiveStoryboardTask: Boolean(params.activeStoryboardTask?.task?.id), + imageWorkflowMemory, + confirmedPrompt, + hasImageInputForPromptCarryForward: params.hasImageInputForPromptCarryForward, + }); + + return { + workingState: params.workingState || null, + activeStoryboardTask: params.activeStoryboardTask || null, + storyboardPreferenceMemory, + imageWorkflowMemory, + promptMemory, + pendingGridConfirmation, + confirmedPrompt, + actionResolution, + } satisfies ConversationStateProtocolBootstrap; +} + +export function composeConversationStateProtocol(params: { + bootstrap: ConversationStateProtocolBootstrap; + autoContextText?: string; + resolvedMediaInput: ResolvedChatMediaInput; + targetResolution: ResolvedTargetAssets; +}) { + const baseThreadState = buildConversationThreadState({ + activeStoryboardTask: params.bootstrap.activeStoryboardTask, + storyboardPreferenceMemory: params.bootstrap.storyboardPreferenceMemory, + promptMemory: params.bootstrap.promptMemory, + imageWorkflowMemory: params.bootstrap.imageWorkflowMemory, + }); + + const composedConversationMemory = composeConversationMemory({ + autoContextText: params.autoContextText || undefined, + resolvedMediaInput: params.resolvedMediaInput, + targetResolution: params.targetResolution, + promptMemory: params.bootstrap.promptMemory, + imageWorkflowMemory: params.bootstrap.imageWorkflowMemory, + storyboardPreferenceMemory: params.bootstrap.storyboardPreferenceMemory, + threadState: baseThreadState, + }); + + const threadState: AgentThreadState = { + workflowState: baseThreadState?.workflowState || null, + memoryHints: Array.from( + new Set([ + ...(baseThreadState?.memoryHints || []), + ...composedConversationMemory.memoryHints, + ]) + ), + contextSummary: + composedConversationMemory.contextSummary || baseThreadState?.contextSummary, + metadata: { + ...(baseThreadState?.metadata || {}), + ...composedConversationMemory.metadata, + }, + }; + + return { + baseThreadState, + composedConversationMemory, + threadState, + } satisfies ConversationStateProtocolComposed; +} + +export async function persistConversationStateProtocol( + params: ConversationStateProtocolPersistParams, +) { + const existing = await getConversationWorkingStateByChatId({ + chatId: params.chatId, + }).catch(() => null); + + const existingSummary = + existing?.lastTurnSummary && typeof existing.lastTurnSummary === "object" + ? (existing.lastTurnSummary as Record) + : {}; + + return upsertConversationWorkingState({ + userId: params.userId, + chatId: params.chatId, + projectId: params.projectId, + currentIntent: params.currentIntent, + currentSubIntent: params.currentSubIntent, + currentFocusAssetIds: params.currentFocusAssetIds, + currentFocusImageUrls: params.currentFocusImageUrls, + selectedAssetIds: params.selectedAssetIds, + selectedImageUrls: params.selectedImageUrls, + lastUploadedAssetIds: params.lastUploadedAssetIds, + lastUploadedImageUrls: params.lastUploadedImageUrls, + lastGeneratedAssetIds: params.lastGeneratedAssetIds, + lastGeneratedImageUrls: params.lastGeneratedImageUrls, + lastReferencedAssetIds: params.lastReferencedAssetIds, + lastReferencedImageUrls: params.lastReferencedImageUrls, + lastResolvedTargetAssetIds: params.lastResolvedTargetAssetIds, + lastResolvedTargetImageUrls: params.lastResolvedTargetImageUrls, + activeCanvasNodeIds: params.activeCanvasNodeIds, + lastCanvasSnapshot: params.lastCanvasSnapshot, + lastMessageId: params.lastMessageId, + incrementTurnCount: params.incrementTurnCount, + lastTurnSummary: + params.lastTurnSummaryPatch !== undefined + ? { + ...existingSummary, + ...params.lastTurnSummaryPatch, + } + : undefined, + }); +} + +export async function persistConversationStateEvent( + type: T, + payload: ConversationStateEventPayload, +) { + const schema = conversationStateEventSchemas[type]; + const parsed = schema.parse(payload) as ConversationStateEvent["payload"]; + console.log("[ConversationStateProtocol] Event:", { + type, + chatId: (parsed as any).chatId, + userId: (parsed as any).userId, + messageId: (parsed as any).messageId, + }); + protocolEventObserver?.({ + type, + payload: parsed as Record, + }); + + switch (type) { + case "prompt_only_ready": { + const typed = parsed as ConversationStateEventPayload<"prompt_only_ready">; + const existing = await getConversationWorkingStateByChatId({ + chatId: typed.chatId, + }).catch(() => null); + const existingImageWorkflowMemory = extractImageWorkflowMemory(existing); + + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: "vision", + currentFocusImageUrls: typed.sourceImageUrls || null, + lastResolvedTargetImageUrls: typed.sourceImageUrls || null, + lastMessageId: typed.messageId, + lastTurnSummaryPatch: { + text: typed.text.slice(0, 500), + resolutionMode: "prompt_only", + confidence: 90, + optimizedPrompt: typed.optimizedPrompt.slice(0, 1000), + promptMemory: buildPromptMemoryUpdate({ + optimizedPrompt: typed.optimizedPrompt.slice(0, 1000), + sourceIntent: "image_gen", + sourceText: typed.text.slice(0, 500), + reusable: true, + }), + imageWorkflowMemory: buildImageWorkflowMemoryUpdate({ + existing: existingImageWorkflowMemory, + lastIntent: "vision", + lastOptimizedPrompt: typed.optimizedPrompt.slice(0, 1000), + lastUserPrompt: typed.text.slice(0, 500), + lastRequestedImageCount: 1, + lastSourceImageUrls: typed.sourceImageUrls || null, + }), + imageSummary: typed.imageSummary?.slice(0, 1200), + sourceImageUrls: typed.sourceImageUrls || [], + }, + }); + } + + case "grid_confirmation_requested": { + const typed = parsed as ConversationStateEventPayload<"grid_confirmation_requested">; + const existing = await getConversationWorkingStateByChatId({ + chatId: typed.chatId, + }).catch(() => null); + const existingStoryboardMemory = extractStoryboardPreferenceMemory(existing); + + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: "image_gen", + lastMessageId: typed.messageId, + lastTurnSummaryPatch: { + text: typed.text.slice(0, 500), + resolutionMode: "grid_confirmation", + confidence: 90, + storyboardMemory: buildStoryboardPreferenceMemoryUpdate({ + existing: existingStoryboardMemory, + preferredGridType: typed.pendingGridConfirmation.gridType, + preferredShotCount: typed.pendingGridConfirmation.shotCount, + prefersConfirmation: true, + lastUsedGridType: typed.pendingGridConfirmation.gridType, + lastUsedShotCount: typed.pendingGridConfirmation.shotCount, + }), + pendingGridConfirmation: { + gridType: typed.pendingGridConfirmation.gridType, + shotCount: typed.pendingGridConfirmation.shotCount, + label: typed.pendingGridConfirmation.label, + originalPrompt: typed.pendingGridConfirmation.originalPrompt.slice(0, 1000), + }, + }, + }); + } + + case "storyboard_confirmed": { + const typed = parsed as ConversationStateEventPayload<"storyboard_confirmed">; + const existing = await getConversationWorkingStateByChatId({ + chatId: typed.chatId, + }).catch(() => null); + const existingStoryboardMemory = extractStoryboardPreferenceMemory(existing); + + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: "video_gen", + lastMessageId: typed.messageId, + lastTurnSummaryPatch: { + text: typed.text.slice(0, 500), + resolutionMode: "storyboard_confirmed", + confidence: 95, + storyboardMemory: buildStoryboardPreferenceMemoryUpdate({ + existing: existingStoryboardMemory, + prefersConfirmation: true, + }), + }, + }); + } + + case "image_workflow_updated": { + const typed = parsed as ConversationStateEventPayload<"image_workflow_updated">; + const existing = await getConversationWorkingStateByChatId({ + chatId: typed.chatId, + }).catch(() => null); + const existingImageWorkflowMemory = extractImageWorkflowMemory(existing); + + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: typed.currentIntent, + currentSubIntent: typed.currentSubIntent, + currentFocusAssetIds: typed.currentFocusAssetIds, + currentFocusImageUrls: typed.currentFocusImageUrls, + lastGeneratedAssetIds: typed.lastGeneratedAssetIds, + lastGeneratedImageUrls: typed.lastGeneratedImageUrls, + lastResolvedTargetAssetIds: typed.lastResolvedTargetAssetIds, + lastResolvedTargetImageUrls: typed.lastResolvedTargetImageUrls, + lastMessageId: typed.lastMessageId, + lastTurnSummaryPatch: { + imageWorkflowMemory: buildImageWorkflowMemoryUpdate({ + existing: existingImageWorkflowMemory, + lastIntent: typed.currentIntent, + lastUserPrompt: typed.lastUserPrompt.slice(0, 500), + lastRequestedImageCount: typed.lastRequestedImageCount, + lastSourceImageUrls: typed.lastSourceImageUrls || null, + lastResultImageUrls: typed.lastResultImageUrls || null, + }), + }, + }); + } + + case "target_resolved": { + const typed = parsed as ConversationStateEventPayload<"target_resolved">; + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: typed.currentIntent, + currentFocusAssetIds: typed.currentFocusAssetIds, + currentFocusImageUrls: typed.currentFocusImageUrls, + selectedAssetIds: typed.selectedAssetIds, + selectedImageUrls: typed.selectedImageUrls, + lastUploadedAssetIds: typed.lastUploadedAssetIds, + lastUploadedImageUrls: typed.lastUploadedImageUrls, + lastReferencedAssetIds: typed.lastReferencedAssetIds, + lastReferencedImageUrls: typed.lastReferencedImageUrls, + lastResolvedTargetAssetIds: typed.lastResolvedTargetAssetIds, + lastResolvedTargetImageUrls: typed.lastResolvedTargetImageUrls, + activeCanvasNodeIds: typed.activeCanvasNodeIds, + lastCanvasSnapshot: typed.lastCanvasSnapshot, + lastMessageId: typed.lastMessageId, + incrementTurnCount: true, + lastTurnSummaryPatch: { + text: typed.text.slice(0, 500), + resolutionMode: typed.resolutionMode, + confidence: typed.confidence, + }, + }); + } + + case "video_workflow_updated": { + const typed = parsed as ConversationStateEventPayload<"video_workflow_updated">; + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: "video_gen", + lastMessageId: typed.lastMessageId, + lastTurnSummaryPatch: + typed.text || typed.status + ? { + text: typed.text?.slice(0, 500), + videoWorkflowStatus: typed.status, + } + : undefined, + }); + } + + case "agent_preset_completed": { + const typed = parsed as ConversationStateEventPayload<"agent_preset_completed">; + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: typed.currentIntent, + lastMessageId: typed.lastMessageId, + lastTurnSummaryPatch: { + text: typed.text.slice(0, 500), + agentPresetId: typed.agentPresetId, + agentPresetName: typed.agentPresetName, + slotCount: typed.slotCount, + successCount: typed.successCount, + }, + }); + } + + case "storyboard_assistant_updated": { + const typed = parsed as ConversationStateEventPayload<"storyboard_assistant_updated">; + const existing = await getConversationWorkingStateByChatId({ + chatId: typed.chatId, + }).catch(() => null); + const existingStoryboardMemory = extractStoryboardPreferenceMemory(existing); + + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: "video_gen", + lastMessageId: typed.lastMessageId, + lastTurnSummaryPatch: { + text: typed.text.slice(0, 500), + resolutionMode: "storyboard", + confidence: 95, + storyboardMemory: buildStoryboardPreferenceMemoryUpdate({ + existing: existingStoryboardMemory, + prefersConfirmation: true, + lastUsedGridType: typed.storyboardGridType, + lastUsedShotCount: typed.storyboardShotCount, + }), + }, + }); + } + + default: + return null; + } +} + +export function setConversationStateProtocolEventObserver( + observer: ConversationStateProtocolEventObserver | null, +) { + protocolEventObserver = observer; +} + +export async function persistPromptOnlyTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + messageId: string; + text: string; + optimizedPrompt: string; + imageSummary?: string; + sourceImageUrls?: string[] | null; +}) { + return persistConversationStateEvent("prompt_only_ready", { + userId: params.userId, + chatId: params.chatId, + projectId: params.projectId, + messageId: params.messageId, + text: params.text, + optimizedPrompt: params.optimizedPrompt, + imageSummary: params.imageSummary, + sourceImageUrls: params.sourceImageUrls || null, + }); +} + +export async function persistGridConfirmationTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + messageId: string; + text: string; + pendingGridConfirmation: PendingGridConfirmation; +}) { + return persistConversationStateEvent("grid_confirmation_requested", { + userId: params.userId, + chatId: params.chatId, + projectId: params.projectId, + messageId: params.messageId, + text: params.text, + pendingGridConfirmation: params.pendingGridConfirmation, + }); +} + +export async function persistStoryboardConfirmedTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + messageId: string; + text: string; +}) { + return persistConversationStateEvent("storyboard_confirmed", { + userId: params.userId, + chatId: params.chatId, + projectId: params.projectId, + messageId: params.messageId, + text: params.text, + }); +} + +export async function persistImageWorkflowTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + currentIntent: "image_gen" | "image_edit" | "vision"; + currentSubIntent?: string | null; + currentFocusAssetIds?: string[] | null; + currentFocusImageUrls?: string[] | null; + lastGeneratedAssetIds?: string[] | null; + lastGeneratedImageUrls?: string[] | null; + lastResolvedTargetAssetIds?: string[] | null; + lastResolvedTargetImageUrls?: string[] | null; + lastMessageId: string; + lastUserPrompt: string; + lastRequestedImageCount?: number; + lastSourceImageUrls?: string[] | null; + lastResultImageUrls?: string[] | null; +}) { + return persistConversationStateEvent("image_workflow_updated", { + userId: params.userId, + chatId: params.chatId, + projectId: params.projectId, + currentIntent: params.currentIntent, + currentSubIntent: params.currentSubIntent, + currentFocusAssetIds: params.currentFocusAssetIds, + currentFocusImageUrls: params.currentFocusImageUrls, + lastGeneratedAssetIds: params.lastGeneratedAssetIds, + lastGeneratedImageUrls: params.lastGeneratedImageUrls, + lastResolvedTargetAssetIds: params.lastResolvedTargetAssetIds, + lastResolvedTargetImageUrls: params.lastResolvedTargetImageUrls, + lastMessageId: params.lastMessageId, + lastUserPrompt: params.lastUserPrompt, + lastRequestedImageCount: params.lastRequestedImageCount, + lastSourceImageUrls: params.lastSourceImageUrls || null, + lastResultImageUrls: params.lastResultImageUrls || null, + }); +} + +export async function persistResolvedTargetTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + currentIntent?: string | null; + currentFocusAssetIds?: string[] | null; + currentFocusImageUrls?: string[] | null; + selectedAssetIds?: string[] | null; + selectedImageUrls?: string[] | null; + lastUploadedAssetIds?: string[] | null; + lastUploadedImageUrls?: string[] | null; + lastReferencedAssetIds?: string[] | null; + lastReferencedImageUrls?: string[] | null; + lastResolvedTargetAssetIds?: string[] | null; + lastResolvedTargetImageUrls?: string[] | null; + activeCanvasNodeIds?: string[] | null; + lastCanvasSnapshot?: unknown; + lastMessageId: string; + text: string; + resolutionMode: string; + confidence: number; +}) { + return persistConversationStateEvent("target_resolved", params); +} + +export async function persistVideoWorkflowTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + lastMessageId: string; + text?: string; + status?: string; +}) { + return persistConversationStateEvent("video_workflow_updated", params); +} + +export async function persistAgentPresetTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + currentIntent: "image_gen" | "video_gen"; + lastMessageId: string; + text: string; + agentPresetId: string; + agentPresetName: string; + slotCount: number; + successCount: number; +}) { + return persistConversationStateEvent("agent_preset_completed", params); +} + +export async function persistStoryboardAssistantTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + lastMessageId: string; + text: string; + storyboardGridType?: "grid_2x2" | "grid_3x2" | "grid_3x3"; + storyboardShotCount?: 4 | 6 | 9; +}) { + return persistConversationStateEvent("storyboard_assistant_updated", params); +}