import "server-only"; import { getConversationWorkingStateByChatId, upsertConversationWorkingState, } from "@/lib/db/queries"; import { conversationStateEventSchemas, type ConversationStateEvent, type ConversationStateEventPayload, type ConversationStateEventType, } from "@/lib/ai/conversation-state-protocol-events"; import { buildImageWorkflowMemoryUpdate, extractImageWorkflowMemory, } from "@/lib/ai/image-workflow-memory"; import type { ResolvedChatMediaInput, ResolvedTargetAssets } from "@/lib/ai/context-memory/types"; import { buildGridConfirmationMessage, isGridConfirmationReply, type PendingGridConfirmation, resolveConversationAction, extractPendingGridConfirmationFromWorkingState, } from "@/lib/ai/conversation-action-resolver"; import { applyContextLineLimit, applyMemoryHintLimit, resolveMemoryRelevancePolicy, } from "@/lib/ai/runtime/memory-relevance"; import { composeConversationMemory, type ComposedConversationMemory, } from "@/lib/ai/conversation-memory-composer"; import { buildConversationThreadState, } from "@/lib/ai/conversation-thread-state-builder"; import type { ImageWorkflowMemory } from "@/lib/ai/image-workflow-memory"; import type { PromptMemoryEntry } from "@/lib/ai/prompt-memory"; import { extractConfirmedPromptFromWorkingState, extractPromptMemoryFromWorkingState, buildPromptMemoryUpdate, } from "@/lib/ai/prompt-memory"; import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state"; import { classifyTaskBoundary, type TaskBoundaryDecision, } from "@/lib/ai/runtime/task-boundary-classifier"; import type { StoryboardPreferenceMemory } from "@/lib/video-agent/storyboard-preference-memory"; import { extractStoryboardPreferenceMemory, buildStoryboardPreferenceMemoryUpdate, } from "@/lib/video-agent/storyboard-preference-memory"; type StoryboardTaskDetail = Parameters[0]["activeStoryboardTask"]; export interface ConversationStateProtocolBootstrap { workingState: { lastTurnSummary?: unknown; } | null; activeStoryboardTask: StoryboardTaskDetail | null; storyboardPreferenceMemory: StoryboardPreferenceMemory | null; imageWorkflowMemory: ImageWorkflowMemory | null; promptMemory: PromptMemoryEntry | null; pendingGridConfirmation: PendingGridConfirmation | null; confirmedPrompt: string | undefined; actionResolution: ReturnType; taskBoundaryDecision: TaskBoundaryDecision; } export interface ConversationStateProtocolComposed { baseThreadState: AgentThreadState | null; composedConversationMemory: ComposedConversationMemory; threadState: AgentThreadState; } export function resolveTargetExecutionOverride(params: { hasImage: boolean; hasSelectedImageHint: boolean; hasExplicitTargetHint: boolean; hasActionableEditIntent: boolean; executionMode?: string; friendlyMessage?: string; optimizedPrompt?: string; requestText: string; }) { const isExecutionMode = params.executionMode === "edit_image" || params.executionMode === "generate_image"; const shouldBypassClarification = params.hasImage && params.hasActionableEditIntent && (params.hasSelectedImageHint || params.hasExplicitTargetHint) && isExecutionMode; if (!shouldBypassClarification) { return { shouldBypassClarification: false, nextFriendlyMessage: params.friendlyMessage, nextOptimizedPrompt: params.optimizedPrompt, }; } const shouldReplaceFriendlyMessage = !params.friendlyMessage || /[??]$/.test(params.friendlyMessage.trim()) || /哪一张|哪个|请先确认/.test(params.friendlyMessage); return { shouldBypassClarification: true, nextFriendlyMessage: shouldReplaceFriendlyMessage ? "收到,我将按你指定的目标图直接执行。" : params.friendlyMessage, nextOptimizedPrompt: params.optimizedPrompt || params.requestText.trim() || undefined, }; } export function resolveGridConfirmationRequest(params: { requestedGrid: PendingGridConfirmation | null; hasActiveStoryboardTask: boolean; executionMode?: string; detectedIntent?: string; requestText: string; effectiveInputText: string; }) { const isGridTaskCandidate = params.executionMode === "generate_image" || params.executionMode === "generate_video" || params.detectedIntent === "image_gen" || params.detectedIntent === "video_gen"; const shouldRequestConfirmation = Boolean(params.requestedGrid) && !isGridConfirmationReply(params.requestText) && !params.hasActiveStoryboardTask && isGridTaskCandidate; if (!shouldRequestConfirmation || !params.requestedGrid) { return { shouldRequestConfirmation: false, pendingGridConfirmation: null as PendingGridConfirmation | null, clarificationMessage: undefined, }; } return { shouldRequestConfirmation: true, pendingGridConfirmation: { ...params.requestedGrid, originalPrompt: params.effectiveInputText, }, clarificationMessage: buildGridConfirmationMessage(params.requestedGrid), }; } type ConversationStateProtocolPersistParams = { userId: string; chatId: string; projectId?: string | null; currentIntent?: string | null; currentSubIntent?: string | null; currentFocusAssetIds?: string[] | null; currentFocusImageUrls?: string[] | null; selectedAssetIds?: string[] | null; selectedImageUrls?: string[] | null; lastUploadedAssetIds?: string[] | null; lastUploadedImageUrls?: string[] | null; lastGeneratedAssetIds?: string[] | null; lastGeneratedImageUrls?: string[] | null; lastReferencedAssetIds?: string[] | null; lastReferencedImageUrls?: string[] | null; lastResolvedTargetAssetIds?: string[] | null; lastResolvedTargetImageUrls?: string[] | null; activeCanvasNodeIds?: string[] | null; lastCanvasSnapshot?: unknown; lastMessageId?: string | null; incrementTurnCount?: boolean; lastTurnSummaryPatch?: Record; }; type ConversationStateProtocolEventObserver = (event: { type: ConversationStateEventType; payload: Record; }) => void; function preferIncomingNonEmptyArray( incoming?: string[] | null, fallback?: string[] | null, ): string[] | null | undefined { if (incoming === undefined) return fallback ?? undefined; if (Array.isArray(incoming) && incoming.length > 0) return incoming; if (Array.isArray(fallback) && fallback.length > 0) return fallback; return incoming; } let protocolEventObserver: ConversationStateProtocolEventObserver | null = null; export function bootstrapConversationStateProtocol(params: { text: string; workingState?: { lastTurnSummary?: unknown; } | null; activeStoryboardTask?: StoryboardTaskDetail | null; hasImageInputForPromptCarryForward: boolean; hasCanvasSelection?: boolean; }) { const storyboardPreferenceMemory = extractStoryboardPreferenceMemory(params.workingState); const imageWorkflowMemory = extractImageWorkflowMemory(params.workingState); const promptMemory = extractPromptMemoryFromWorkingState(params.workingState); const pendingGridConfirmation = extractPendingGridConfirmationFromWorkingState(params.workingState); const confirmedPrompt = extractConfirmedPromptFromWorkingState(params.workingState) || promptMemory?.optimizedPrompt; const actionResolution = resolveConversationAction({ text: params.text, pendingGridConfirmation, hasActiveStoryboardTask: Boolean(params.activeStoryboardTask?.task?.id), imageWorkflowMemory, confirmedPrompt, hasImageInputForPromptCarryForward: params.hasImageInputForPromptCarryForward, }); const taskBoundaryDecision = classifyTaskBoundary({ text: params.text, actionResolution, hasCurrentImageInput: params.hasImageInputForPromptCarryForward, hasCanvasSelection: params.hasCanvasSelection === true, pendingGridConfirmation, hasActiveStoryboardTask: Boolean(params.activeStoryboardTask?.task?.id), }); return { workingState: params.workingState || null, activeStoryboardTask: params.activeStoryboardTask || null, storyboardPreferenceMemory, imageWorkflowMemory, promptMemory, pendingGridConfirmation, confirmedPrompt, actionResolution, taskBoundaryDecision, } satisfies ConversationStateProtocolBootstrap; } export function composeConversationStateProtocol(params: { bootstrap: ConversationStateProtocolBootstrap; autoContextText?: string; resolvedMediaInput: ResolvedChatMediaInput; targetResolution: ResolvedTargetAssets; }) { const memoryRelevancePolicy = resolveMemoryRelevancePolicy({ taskBoundaryDecision: params.bootstrap.taskBoundaryDecision, actionResolution: params.bootstrap.actionResolution, hasActiveStoryboardTask: Boolean(params.bootstrap.activeStoryboardTask?.task?.id), hasPendingGridConfirmation: Boolean(params.bootstrap.pendingGridConfirmation), hasCurrentImageInput: params.resolvedMediaInput.currentMessageImageUrls.length > 0, hasCanvasSelection: params.resolvedMediaInput.selectedImageUrls.length > 0, }); const baseThreadState = buildConversationThreadState({ activeStoryboardTask: memoryRelevancePolicy.includeWorkflowState ? params.bootstrap.activeStoryboardTask : null, storyboardPreferenceMemory: memoryRelevancePolicy.includeStoryboardPreferenceMemory ? params.bootstrap.storyboardPreferenceMemory : null, promptMemory: memoryRelevancePolicy.includePromptMemory ? params.bootstrap.promptMemory : null, imageWorkflowMemory: memoryRelevancePolicy.includeImageWorkflowMemory ? params.bootstrap.imageWorkflowMemory : null, }); const composedConversationMemory = composeConversationMemory({ autoContextText: params.autoContextText || undefined, resolvedMediaInput: params.resolvedMediaInput, targetResolution: params.targetResolution, promptMemory: memoryRelevancePolicy.includePromptMemory ? params.bootstrap.promptMemory : null, imageWorkflowMemory: memoryRelevancePolicy.includeImageWorkflowMemory ? params.bootstrap.imageWorkflowMemory : null, storyboardPreferenceMemory: memoryRelevancePolicy.includeStoryboardPreferenceMemory ? params.bootstrap.storyboardPreferenceMemory : null, threadState: baseThreadState, memoryRelevancePolicy, }); const threadState: AgentThreadState = { workflowState: baseThreadState?.workflowState || null, memoryHints: applyMemoryHintLimit( Array.from( new Set([ ...(baseThreadState?.memoryHints || []), ...composedConversationMemory.memoryHints, ]) ), memoryRelevancePolicy.maxMemoryHints, ), contextSummary: applyContextLineLimit( composedConversationMemory.contextSummary || baseThreadState?.contextSummary, memoryRelevancePolicy.maxContextLines, ), metadata: { ...(baseThreadState?.metadata || {}), ...composedConversationMemory.metadata, }, }; return { baseThreadState, composedConversationMemory, threadState, } satisfies ConversationStateProtocolComposed; } export async function persistConversationStateProtocol( params: ConversationStateProtocolPersistParams, ) { const existing = await getConversationWorkingStateByChatId({ chatId: params.chatId, }).catch(() => null); const existingSummary = existing?.lastTurnSummary && typeof existing.lastTurnSummary === "object" ? (existing.lastTurnSummary as Record) : {}; return upsertConversationWorkingState({ userId: params.userId, chatId: params.chatId, projectId: params.projectId, currentIntent: params.currentIntent, currentSubIntent: params.currentSubIntent, currentFocusAssetIds: params.currentFocusAssetIds, currentFocusImageUrls: params.currentFocusImageUrls, selectedAssetIds: params.selectedAssetIds, selectedImageUrls: params.selectedImageUrls, lastUploadedAssetIds: params.lastUploadedAssetIds, lastUploadedImageUrls: params.lastUploadedImageUrls, lastGeneratedAssetIds: params.lastGeneratedAssetIds, lastGeneratedImageUrls: params.lastGeneratedImageUrls, lastReferencedAssetIds: params.lastReferencedAssetIds, lastReferencedImageUrls: params.lastReferencedImageUrls, lastResolvedTargetAssetIds: params.lastResolvedTargetAssetIds, lastResolvedTargetImageUrls: params.lastResolvedTargetImageUrls, activeCanvasNodeIds: params.activeCanvasNodeIds, lastCanvasSnapshot: params.lastCanvasSnapshot, lastMessageId: params.lastMessageId, incrementTurnCount: params.incrementTurnCount, lastTurnSummary: params.lastTurnSummaryPatch !== undefined ? { ...existingSummary, ...params.lastTurnSummaryPatch, } : undefined, }); } export async function persistConversationStateEvent( type: T, payload: ConversationStateEventPayload, ) { const schema = conversationStateEventSchemas[type]; const parsed = schema.parse(payload) as ConversationStateEvent["payload"]; console.log("[ConversationStateProtocol] Event:", { type, chatId: (parsed as any).chatId, userId: (parsed as any).userId, messageId: (parsed as any).messageId, }); protocolEventObserver?.({ type, payload: parsed as Record, }); switch (type) { case "prompt_only_ready": { const typed = parsed as ConversationStateEventPayload<"prompt_only_ready">; const existing = await getConversationWorkingStateByChatId({ chatId: typed.chatId, }).catch(() => null); const existingImageWorkflowMemory = extractImageWorkflowMemory(existing); return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: "vision", currentFocusImageUrls: typed.sourceImageUrls || null, lastResolvedTargetImageUrls: typed.sourceImageUrls || null, lastMessageId: typed.messageId, lastTurnSummaryPatch: { text: typed.text.slice(0, 500), resolutionMode: "prompt_only", confidence: 90, optimizedPrompt: typed.optimizedPrompt.slice(0, 1000), promptMemory: buildPromptMemoryUpdate({ optimizedPrompt: typed.optimizedPrompt.slice(0, 1000), sourceIntent: "image_gen", sourceText: typed.text.slice(0, 500), reusable: true, }), imageWorkflowMemory: buildImageWorkflowMemoryUpdate({ existing: existingImageWorkflowMemory, lastIntent: "vision", lastOptimizedPrompt: typed.optimizedPrompt.slice(0, 1000), lastUserPrompt: typed.text.slice(0, 500), lastRequestedImageCount: 1, lastSourceImageUrls: typed.sourceImageUrls || null, }), imageSummary: typed.imageSummary?.slice(0, 1200), sourceImageUrls: typed.sourceImageUrls || [], }, }); } case "grid_confirmation_requested": { const typed = parsed as ConversationStateEventPayload<"grid_confirmation_requested">; const existing = await getConversationWorkingStateByChatId({ chatId: typed.chatId, }).catch(() => null); const existingStoryboardMemory = extractStoryboardPreferenceMemory(existing); return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: "image_gen", lastMessageId: typed.messageId, lastTurnSummaryPatch: { text: typed.text.slice(0, 500), resolutionMode: "grid_confirmation", confidence: 90, storyboardMemory: buildStoryboardPreferenceMemoryUpdate({ existing: existingStoryboardMemory, preferredGridType: typed.pendingGridConfirmation.gridType, preferredShotCount: typed.pendingGridConfirmation.shotCount, prefersConfirmation: true, lastUsedGridType: typed.pendingGridConfirmation.gridType, lastUsedShotCount: typed.pendingGridConfirmation.shotCount, }), pendingGridConfirmation: { gridType: typed.pendingGridConfirmation.gridType, shotCount: typed.pendingGridConfirmation.shotCount, label: typed.pendingGridConfirmation.label, originalPrompt: typed.pendingGridConfirmation.originalPrompt.slice(0, 1000), }, }, }); } case "clarification_requested": { const typed = parsed as ConversationStateEventPayload<"clarification_requested">; return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: typed.currentIntent, lastMessageId: typed.messageId, lastTurnSummaryPatch: { text: typed.text.slice(0, 500), resolutionMode: "clarification", confidence: 80, clarificationMessage: typed.clarificationMessage.slice(0, 1000), }, }); } case "storyboard_confirmed": { const typed = parsed as ConversationStateEventPayload<"storyboard_confirmed">; const existing = await getConversationWorkingStateByChatId({ chatId: typed.chatId, }).catch(() => null); const existingStoryboardMemory = extractStoryboardPreferenceMemory(existing); return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: "video_gen", lastMessageId: typed.messageId, lastTurnSummaryPatch: { text: typed.text.slice(0, 500), resolutionMode: "storyboard_confirmed", confidence: 95, videoWorkflowStatus: typed.status || "confirmed", storyboardTaskId: typed.taskId, storyboardWorkflowAction: "confirmed", storyboardMemory: buildStoryboardPreferenceMemoryUpdate({ existing: existingStoryboardMemory, prefersConfirmation: true, }), }, }); } case "storyboard_execution_started": { const typed = parsed as ConversationStateEventPayload<"storyboard_execution_started">; return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: "video_gen", lastMessageId: typed.messageId, lastTurnSummaryPatch: { text: typed.text.slice(0, 500), resolutionMode: "storyboard_execution_started", confidence: 95, videoWorkflowStatus: typed.status || "running", storyboardTaskId: typed.taskId, storyboardWorkflowAction: "execution_started", }, }); } case "image_workflow_updated": { const typed = parsed as ConversationStateEventPayload<"image_workflow_updated">; const existing = await getConversationWorkingStateByChatId({ chatId: typed.chatId, }).catch(() => null); const existingImageWorkflowMemory = extractImageWorkflowMemory(existing); const nextOptimizedPrompt = typed.lastOptimizedPrompt?.trim(); return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: typed.currentIntent, currentSubIntent: typed.currentSubIntent, currentFocusAssetIds: typed.currentFocusAssetIds, currentFocusImageUrls: typed.currentFocusImageUrls, lastGeneratedAssetIds: typed.lastGeneratedAssetIds, lastGeneratedImageUrls: typed.lastGeneratedImageUrls, lastResolvedTargetAssetIds: typed.lastResolvedTargetAssetIds, lastResolvedTargetImageUrls: typed.lastResolvedTargetImageUrls, lastMessageId: typed.lastMessageId, lastTurnSummaryPatch: { promptMemory: nextOptimizedPrompt ? buildPromptMemoryUpdate({ optimizedPrompt: nextOptimizedPrompt.slice(0, 1000), sourceIntent: typed.currentIntent, sourceText: typed.lastUserPrompt.slice(0, 500), reusable: true, }) : undefined, imageWorkflowMemory: buildImageWorkflowMemoryUpdate({ existing: existingImageWorkflowMemory, lastIntent: typed.currentIntent, lastOptimizedPrompt: nextOptimizedPrompt?.slice(0, 1000), lastUserPrompt: typed.lastUserPrompt.slice(0, 500), lastRequestedImageCount: typed.lastRequestedImageCount, lastAspectRatio: typed.lastAspectRatio?.slice(0, 32), lastImageSize: typed.lastImageSize?.slice(0, 32), lastSourceImageUrls: typed.lastSourceImageUrls || null, lastResultImageUrls: typed.lastResultImageUrls || null, }), }, }); } case "storyboard_revision_requested": { const typed = parsed as ConversationStateEventPayload<"storyboard_revision_requested">; const existing = await getConversationWorkingStateByChatId({ chatId: typed.chatId, }).catch(() => null); const existingStoryboardMemory = extractStoryboardPreferenceMemory(existing); return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: "video_gen", lastMessageId: typed.messageId, lastTurnSummaryPatch: { text: typed.text.slice(0, 500), resolutionMode: "storyboard_revision", confidence: 90, videoWorkflowStatus: typed.status || "awaiting_user_confirmation", storyboardTaskId: typed.taskId, storyboardWorkflowAction: "revision_requested", storyboardMemory: buildStoryboardPreferenceMemoryUpdate({ existing: existingStoryboardMemory, prefersConfirmation: true, }), }, }); } case "storyboard_execution_completed": { const typed = parsed as ConversationStateEventPayload<"storyboard_execution_completed">; return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: "video_gen", lastMessageId: typed.messageId, lastTurnSummaryPatch: { text: typed.text.slice(0, 500), resolutionMode: "storyboard_execution_completed", confidence: 95, videoWorkflowStatus: typed.status || "completed", storyboardTaskId: typed.taskId, storyboardWorkflowAction: "execution_completed", successCount: typed.successCount, failedCount: typed.failedCount, firstVideoUrl: typed.firstVideoUrl, }, }); } case "video_generation_started": { const typed = parsed as ConversationStateEventPayload<"video_generation_started">; return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: "video_gen", lastMessageId: typed.messageId, lastTurnSummaryPatch: { text: typed.text.slice(0, 500), resolutionMode: "video_generation_started", confidence: 90, videoWorkflowStatus: typed.status || "running", videoWorkflowSource: typed.source, videoProviderType: typed.providerType, videoModelRef: typed.modelRef, videoGenerationMode: typed.mode, videoTaskId: typed.taskId, }, }); } case "video_generation_completed": { const typed = parsed as ConversationStateEventPayload<"video_generation_completed">; return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: "video_gen", lastMessageId: typed.messageId, lastTurnSummaryPatch: { text: typed.text.slice(0, 500), resolutionMode: "video_generation_completed", confidence: 95, videoWorkflowStatus: typed.status || "completed", videoWorkflowSource: typed.source, videoProviderType: typed.providerType, videoModelRef: typed.modelRef, videoGenerationMode: typed.mode, videoTaskId: typed.taskId, videoTemporaryUrl: typed.temporaryVideoUrl, }, }); } case "video_generation_failed": { const typed = parsed as ConversationStateEventPayload<"video_generation_failed">; return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: "video_gen", lastMessageId: typed.messageId, lastTurnSummaryPatch: { text: typed.text.slice(0, 500), resolutionMode: "video_generation_failed", confidence: 85, videoWorkflowStatus: typed.status || "failed", videoWorkflowSource: typed.source, videoProviderType: typed.providerType, videoModelRef: typed.modelRef, videoGenerationMode: typed.mode, videoTaskId: typed.taskId, videoWorkflowError: typed.error?.slice(0, 1000), }, }); } case "target_resolved": { const typed = parsed as ConversationStateEventPayload<"target_resolved">; const existing = await getConversationWorkingStateByChatId({ chatId: typed.chatId, }).catch(() => null); const existingImageWorkflowMemory = extractImageWorkflowMemory(existing); const preservedCurrentFocusAssetIds = preferIncomingNonEmptyArray( typed.currentFocusAssetIds, existing?.currentFocusAssetIds, ); const preservedCurrentFocusImageUrls = preferIncomingNonEmptyArray( typed.currentFocusImageUrls, existing?.currentFocusImageUrls, ); const preservedLastUploadedAssetIds = preferIncomingNonEmptyArray( typed.lastUploadedAssetIds, existing?.lastUploadedAssetIds, ); const preservedLastUploadedImageUrls = preferIncomingNonEmptyArray( typed.lastUploadedImageUrls, existing?.lastUploadedImageUrls, ); const preservedLastReferencedAssetIds = preferIncomingNonEmptyArray( typed.lastReferencedAssetIds, existing?.lastReferencedAssetIds, ); const preservedLastReferencedImageUrls = preferIncomingNonEmptyArray( typed.lastReferencedImageUrls, existing?.lastReferencedImageUrls, ); const preservedLastResolvedTargetAssetIds = preferIncomingNonEmptyArray( typed.lastResolvedTargetAssetIds, existing?.lastResolvedTargetAssetIds, ); const preservedLastResolvedTargetImageUrls = preferIncomingNonEmptyArray( typed.lastResolvedTargetImageUrls, existing?.lastResolvedTargetImageUrls, ); const workflowSourceImageUrls = (Array.isArray(preservedLastResolvedTargetImageUrls) && preservedLastResolvedTargetImageUrls.length > 0 ? preservedLastResolvedTargetImageUrls : preservedLastUploadedImageUrls) || null; return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: typed.currentIntent, currentFocusAssetIds: preservedCurrentFocusAssetIds, currentFocusImageUrls: preservedCurrentFocusImageUrls, selectedAssetIds: typed.selectedAssetIds, selectedImageUrls: typed.selectedImageUrls, lastUploadedAssetIds: preservedLastUploadedAssetIds, lastUploadedImageUrls: preservedLastUploadedImageUrls, lastReferencedAssetIds: preservedLastReferencedAssetIds, lastReferencedImageUrls: preservedLastReferencedImageUrls, lastResolvedTargetAssetIds: preservedLastResolvedTargetAssetIds, lastResolvedTargetImageUrls: preservedLastResolvedTargetImageUrls, activeCanvasNodeIds: typed.activeCanvasNodeIds, lastCanvasSnapshot: typed.lastCanvasSnapshot, lastMessageId: typed.lastMessageId, incrementTurnCount: true, lastTurnSummaryPatch: { text: typed.text.slice(0, 500), resolutionMode: typed.resolutionMode, confidence: typed.confidence, imageWorkflowMemory: typed.currentIntent === "image_edit" || typed.currentIntent === "image_gen" || typed.currentIntent === "vision" ? buildImageWorkflowMemoryUpdate({ existing: existingImageWorkflowMemory, lastIntent: typed.currentIntent, lastUserPrompt: typed.text.slice(0, 500), lastSourceImageUrls: workflowSourceImageUrls, }) : undefined, }, }); } case "video_workflow_updated": { const typed = parsed as ConversationStateEventPayload<"video_workflow_updated">; return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: "video_gen", lastMessageId: typed.lastMessageId, lastTurnSummaryPatch: typed.text || typed.status ? { text: typed.text?.slice(0, 500), videoWorkflowStatus: typed.status, } : undefined, }); } case "workflow_cancelled": { const typed = parsed as ConversationStateEventPayload<"workflow_cancelled">; return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: typed.workflowType === "storyboard_video" ? "video_gen" : undefined, lastMessageId: typed.messageId, lastTurnSummaryPatch: { text: typed.text.slice(0, 500), resolutionMode: "workflow_cancelled", confidence: 95, videoWorkflowStatus: typed.status || "cancelled", cancelledWorkflowType: typed.workflowType, cancelledWorkflowId: typed.workflowId, workflowAction: "cancel_requested", }, }); } case "agent_preset_completed": { const typed = parsed as ConversationStateEventPayload<"agent_preset_completed">; return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: typed.currentIntent, lastMessageId: typed.lastMessageId, lastTurnSummaryPatch: { text: typed.text.slice(0, 500), agentPresetId: typed.agentPresetId, agentPresetName: typed.agentPresetName, slotCount: typed.slotCount, successCount: typed.successCount, }, }); } case "storyboard_assistant_updated": { const typed = parsed as ConversationStateEventPayload<"storyboard_assistant_updated">; const existing = await getConversationWorkingStateByChatId({ chatId: typed.chatId, }).catch(() => null); const existingStoryboardMemory = extractStoryboardPreferenceMemory(existing); return persistConversationStateProtocol({ userId: typed.userId, chatId: typed.chatId, projectId: typed.projectId, currentIntent: "video_gen", lastMessageId: typed.lastMessageId, lastTurnSummaryPatch: { text: typed.text.slice(0, 500), resolutionMode: "storyboard", confidence: 95, storyboardMemory: buildStoryboardPreferenceMemoryUpdate({ existing: existingStoryboardMemory, prefersConfirmation: true, lastUsedGridType: typed.storyboardGridType, lastUsedShotCount: typed.storyboardShotCount, }), }, }); } default: return null; } } export function setConversationStateProtocolEventObserver( observer: ConversationStateProtocolEventObserver | null, ) { protocolEventObserver = observer; } export async function persistPromptOnlyTurn(params: { userId: string; chatId: string; projectId?: string | null; messageId: string; text: string; optimizedPrompt: string; imageSummary?: string; sourceImageUrls?: string[] | null; }) { return persistConversationStateEvent("prompt_only_ready", { userId: params.userId, chatId: params.chatId, projectId: params.projectId, messageId: params.messageId, text: params.text, optimizedPrompt: params.optimizedPrompt, imageSummary: params.imageSummary, sourceImageUrls: params.sourceImageUrls || null, }); } export async function persistGridConfirmationTurn(params: { userId: string; chatId: string; projectId?: string | null; messageId: string; text: string; pendingGridConfirmation: PendingGridConfirmation; }) { return persistConversationStateEvent("grid_confirmation_requested", { userId: params.userId, chatId: params.chatId, projectId: params.projectId, messageId: params.messageId, text: params.text, pendingGridConfirmation: params.pendingGridConfirmation, }); } export async function persistClarificationTurn(params: { userId: string; chatId: string; projectId?: string | null; messageId: string; text: string; clarificationMessage: string; currentIntent?: string | null; pendingGridConfirmation?: PendingGridConfirmation | null; }) { if (params.pendingGridConfirmation) { return persistConversationStateEvent("grid_confirmation_requested", { userId: params.userId, chatId: params.chatId, projectId: params.projectId, messageId: params.messageId, text: params.text, pendingGridConfirmation: params.pendingGridConfirmation, }); } return persistConversationStateEvent("clarification_requested", { userId: params.userId, chatId: params.chatId, projectId: params.projectId, messageId: params.messageId, text: params.text, clarificationMessage: params.clarificationMessage, currentIntent: params.currentIntent, }); } export async function persistStoryboardConfirmedTurn(params: { userId: string; chatId: string; projectId?: string | null; messageId: string; text: string; taskId?: string; status?: string; }) { return persistConversationStateEvent("storyboard_confirmed", { userId: params.userId, chatId: params.chatId, projectId: params.projectId, messageId: params.messageId, text: params.text, taskId: params.taskId, status: params.status, }); } export async function persistStoryboardExecutionStartedTurn(params: { userId: string; chatId: string; projectId?: string | null; messageId: string; text: string; taskId: string; status?: string; }) { return persistConversationStateEvent("storyboard_execution_started", { userId: params.userId, chatId: params.chatId, projectId: params.projectId, messageId: params.messageId, text: params.text, taskId: params.taskId, status: params.status, }); } export async function persistStoryboardRevisionTurn(params: { userId: string; chatId: string; projectId?: string | null; messageId: string; text: string; taskId: string; status?: string; }) { return persistConversationStateEvent("storyboard_revision_requested", { userId: params.userId, chatId: params.chatId, projectId: params.projectId, messageId: params.messageId, text: params.text, taskId: params.taskId, status: params.status, }); } export async function persistStoryboardExecutionCompletedTurn(params: { userId: string; chatId: string; projectId?: string | null; messageId: string; text: string; taskId: string; status?: string; successCount?: number; failedCount?: number; firstVideoUrl?: string; }) { return persistConversationStateEvent("storyboard_execution_completed", { userId: params.userId, chatId: params.chatId, projectId: params.projectId, messageId: params.messageId, text: params.text, taskId: params.taskId, status: params.status, successCount: params.successCount, failedCount: params.failedCount, firstVideoUrl: params.firstVideoUrl, }); } export async function persistVideoGenerationStartedTurn(params: { userId: string; chatId: string; projectId?: string | null; messageId: string; text: string; source: string; providerType?: string; modelRef?: string; mode?: string; taskId?: string; status?: string; }) { return persistConversationStateEvent("video_generation_started", { userId: params.userId, chatId: params.chatId, projectId: params.projectId, messageId: params.messageId, text: params.text, source: params.source, providerType: params.providerType, modelRef: params.modelRef, mode: params.mode, taskId: params.taskId, status: params.status, }); } export async function persistVideoGenerationCompletedTurn(params: { userId: string; chatId: string; projectId?: string | null; messageId: string; text: string; source: string; providerType?: string; modelRef?: string; mode?: string; taskId?: string; status?: string; temporaryVideoUrl?: string; }) { return persistConversationStateEvent("video_generation_completed", { userId: params.userId, chatId: params.chatId, projectId: params.projectId, messageId: params.messageId, text: params.text, source: params.source, providerType: params.providerType, modelRef: params.modelRef, mode: params.mode, taskId: params.taskId, status: params.status, temporaryVideoUrl: params.temporaryVideoUrl, }); } export async function persistVideoGenerationFailedTurn(params: { userId: string; chatId: string; projectId?: string | null; messageId: string; text: string; source: string; providerType?: string; modelRef?: string; mode?: string; taskId?: string; status?: string; error?: string; }) { return persistConversationStateEvent("video_generation_failed", { userId: params.userId, chatId: params.chatId, projectId: params.projectId, messageId: params.messageId, text: params.text, source: params.source, providerType: params.providerType, modelRef: params.modelRef, mode: params.mode, taskId: params.taskId, status: params.status, error: params.error, }); } export async function persistImageWorkflowTurn(params: { userId: string; chatId: string; projectId?: string | null; currentIntent: "image_gen" | "image_edit" | "vision"; currentSubIntent?: string | null; currentFocusAssetIds?: string[] | null; currentFocusImageUrls?: string[] | null; lastGeneratedAssetIds?: string[] | null; lastGeneratedImageUrls?: string[] | null; lastResolvedTargetAssetIds?: string[] | null; lastResolvedTargetImageUrls?: string[] | null; lastMessageId: string; lastUserPrompt: string; lastOptimizedPrompt?: string; lastRequestedImageCount?: number; lastAspectRatio?: string; lastImageSize?: string; lastSourceImageUrls?: string[] | null; lastResultImageUrls?: string[] | null; }) { return persistConversationStateEvent("image_workflow_updated", { userId: params.userId, chatId: params.chatId, projectId: params.projectId, currentIntent: params.currentIntent, currentSubIntent: params.currentSubIntent, currentFocusAssetIds: params.currentFocusAssetIds, currentFocusImageUrls: params.currentFocusImageUrls, lastGeneratedAssetIds: params.lastGeneratedAssetIds, lastGeneratedImageUrls: params.lastGeneratedImageUrls, lastResolvedTargetAssetIds: params.lastResolvedTargetAssetIds, lastResolvedTargetImageUrls: params.lastResolvedTargetImageUrls, lastMessageId: params.lastMessageId, lastUserPrompt: params.lastUserPrompt, lastOptimizedPrompt: params.lastOptimizedPrompt, lastRequestedImageCount: params.lastRequestedImageCount, lastAspectRatio: params.lastAspectRatio, lastImageSize: params.lastImageSize, lastSourceImageUrls: params.lastSourceImageUrls || null, lastResultImageUrls: params.lastResultImageUrls || null, }); } export async function persistResolvedTargetTurn(params: { userId: string; chatId: string; projectId?: string | null; currentIntent?: string | null; currentFocusAssetIds?: string[] | null; currentFocusImageUrls?: string[] | null; selectedAssetIds?: string[] | null; selectedImageUrls?: string[] | null; lastUploadedAssetIds?: string[] | null; lastUploadedImageUrls?: string[] | null; lastReferencedAssetIds?: string[] | null; lastReferencedImageUrls?: string[] | null; lastResolvedTargetAssetIds?: string[] | null; lastResolvedTargetImageUrls?: string[] | null; activeCanvasNodeIds?: string[] | null; lastCanvasSnapshot?: unknown; lastMessageId: string; text: string; resolutionMode: string; confidence: number; }) { return persistConversationStateEvent("target_resolved", params); } export async function persistVideoWorkflowTurn(params: { userId: string; chatId: string; projectId?: string | null; lastMessageId: string; text?: string; status?: string; }) { return persistConversationStateEvent("video_workflow_updated", params); } export async function persistWorkflowCancelledTurn(params: { userId: string; chatId: string; projectId?: string | null; messageId: string; text: string; workflowType: string; workflowId?: string; status?: string; }) { return persistConversationStateEvent("workflow_cancelled", { userId: params.userId, chatId: params.chatId, projectId: params.projectId, messageId: params.messageId, text: params.text, workflowType: params.workflowType, workflowId: params.workflowId, status: params.status, }); } export async function persistAgentPresetTurn(params: { userId: string; chatId: string; projectId?: string | null; currentIntent: "image_gen" | "video_gen"; lastMessageId: string; text: string; agentPresetId: string; agentPresetName: string; slotCount: number; successCount: number; }) { return persistConversationStateEvent("agent_preset_completed", params); } export async function persistStoryboardAssistantTurn(params: { userId: string; chatId: string; projectId?: string | null; lastMessageId: string; text: string; storyboardGridType?: "grid_2x2" | "grid_3x2" | "grid_3x3"; storyboardShotCount?: 4 | 6 | 9; }) { return persistConversationStateEvent("storyboard_assistant_updated", params); }