1120 lines
37 KiB
TypeScript
1120 lines
37 KiB
TypeScript
import "server-only";
|
||
|
||
import {
|
||
getConversationWorkingStateByChatId,
|
||
upsertConversationWorkingState,
|
||
} from "@/lib/db/queries";
|
||
import {
|
||
conversationStateEventSchemas,
|
||
type ConversationStateEvent,
|
||
type ConversationStateEventPayload,
|
||
type ConversationStateEventType,
|
||
} from "@/lib/ai/conversation-state-protocol-events";
|
||
import {
|
||
buildImageWorkflowMemoryUpdate,
|
||
extractImageWorkflowMemory,
|
||
} from "@/lib/ai/image-workflow-memory";
|
||
import type { ResolvedChatMediaInput, ResolvedTargetAssets } from "@/lib/ai/context-memory/types";
|
||
import {
|
||
buildGridConfirmationMessage,
|
||
isGridConfirmationReply,
|
||
type PendingGridConfirmation,
|
||
resolveConversationAction,
|
||
extractPendingGridConfirmationFromWorkingState,
|
||
} from "@/lib/ai/conversation-action-resolver";
|
||
import {
|
||
composeConversationMemory,
|
||
type ComposedConversationMemory,
|
||
} from "@/lib/ai/conversation-memory-composer";
|
||
import {
|
||
buildConversationThreadState,
|
||
} from "@/lib/ai/conversation-thread-state-builder";
|
||
import type { ImageWorkflowMemory } from "@/lib/ai/image-workflow-memory";
|
||
import type { PromptMemoryEntry } from "@/lib/ai/prompt-memory";
|
||
import {
|
||
extractConfirmedPromptFromWorkingState,
|
||
extractPromptMemoryFromWorkingState,
|
||
buildPromptMemoryUpdate,
|
||
} from "@/lib/ai/prompt-memory";
|
||
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
|
||
import type { StoryboardPreferenceMemory } from "@/lib/video-agent/storyboard-preference-memory";
|
||
import {
|
||
extractStoryboardPreferenceMemory,
|
||
buildStoryboardPreferenceMemoryUpdate,
|
||
} from "@/lib/video-agent/storyboard-preference-memory";
|
||
|
||
type StoryboardTaskDetail = Parameters<typeof buildConversationThreadState>[0]["activeStoryboardTask"];
|
||
|
||
export interface ConversationStateProtocolBootstrap {
|
||
workingState: {
|
||
lastTurnSummary?: unknown;
|
||
} | null;
|
||
activeStoryboardTask: StoryboardTaskDetail | null;
|
||
storyboardPreferenceMemory: StoryboardPreferenceMemory | null;
|
||
imageWorkflowMemory: ImageWorkflowMemory | null;
|
||
promptMemory: PromptMemoryEntry | null;
|
||
pendingGridConfirmation: PendingGridConfirmation | null;
|
||
confirmedPrompt: string | undefined;
|
||
actionResolution: ReturnType<typeof resolveConversationAction>;
|
||
}
|
||
|
||
export interface ConversationStateProtocolComposed {
|
||
baseThreadState: AgentThreadState | null;
|
||
composedConversationMemory: ComposedConversationMemory;
|
||
threadState: AgentThreadState;
|
||
}
|
||
|
||
export function resolveTargetExecutionOverride(params: {
|
||
hasImage: boolean;
|
||
hasSelectedImageHint: boolean;
|
||
hasExplicitTargetHint: boolean;
|
||
hasActionableEditIntent: boolean;
|
||
executionMode?: string;
|
||
friendlyMessage?: string;
|
||
optimizedPrompt?: string;
|
||
requestText: string;
|
||
}) {
|
||
const isExecutionMode =
|
||
params.executionMode === "edit_image" || params.executionMode === "generate_image";
|
||
const shouldBypassClarification =
|
||
params.hasImage &&
|
||
params.hasActionableEditIntent &&
|
||
(params.hasSelectedImageHint || params.hasExplicitTargetHint) &&
|
||
isExecutionMode;
|
||
|
||
if (!shouldBypassClarification) {
|
||
return {
|
||
shouldBypassClarification: false,
|
||
nextFriendlyMessage: params.friendlyMessage,
|
||
nextOptimizedPrompt: params.optimizedPrompt,
|
||
};
|
||
}
|
||
|
||
const shouldReplaceFriendlyMessage =
|
||
!params.friendlyMessage ||
|
||
/[?？]$/.test(params.friendlyMessage.trim()) ||
|
||
/哪一张|哪个|请先确认/.test(params.friendlyMessage);
|
||
|
||
return {
|
||
shouldBypassClarification: true,
|
||
nextFriendlyMessage: shouldReplaceFriendlyMessage
|
||
? "收到，我将按你指定的目标图直接执行。"
|
||
: params.friendlyMessage,
|
||
nextOptimizedPrompt: params.optimizedPrompt || params.requestText.trim() || undefined,
|
||
};
|
||
}
|
||
|
||
export function resolveGridConfirmationRequest(params: {
|
||
requestedGrid: PendingGridConfirmation | null;
|
||
hasActiveStoryboardTask: boolean;
|
||
executionMode?: string;
|
||
detectedIntent?: string;
|
||
requestText: string;
|
||
effectiveInputText: string;
|
||
}) {
|
||
const isGridTaskCandidate =
|
||
params.executionMode === "generate_image" ||
|
||
params.executionMode === "generate_video" ||
|
||
params.detectedIntent === "image_gen" ||
|
||
params.detectedIntent === "video_gen";
|
||
|
||
const shouldRequestConfirmation =
|
||
Boolean(params.requestedGrid) &&
|
||
!isGridConfirmationReply(params.requestText) &&
|
||
!params.hasActiveStoryboardTask &&
|
||
isGridTaskCandidate;
|
||
|
||
if (!shouldRequestConfirmation || !params.requestedGrid) {
|
||
return {
|
||
shouldRequestConfirmation: false,
|
||
pendingGridConfirmation: null as PendingGridConfirmation | null,
|
||
clarificationMessage: undefined,
|
||
};
|
||
}
|
||
|
||
return {
|
||
shouldRequestConfirmation: true,
|
||
pendingGridConfirmation: {
|
||
...params.requestedGrid,
|
||
originalPrompt: params.effectiveInputText,
|
||
},
|
||
clarificationMessage: buildGridConfirmationMessage(params.requestedGrid),
|
||
};
|
||
}
|
||
|
||
type ConversationStateProtocolPersistParams = {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
currentIntent?: string | null;
|
||
currentSubIntent?: string | null;
|
||
currentFocusAssetIds?: string[] | null;
|
||
currentFocusImageUrls?: string[] | null;
|
||
selectedAssetIds?: string[] | null;
|
||
selectedImageUrls?: string[] | null;
|
||
lastUploadedAssetIds?: string[] | null;
|
||
lastUploadedImageUrls?: string[] | null;
|
||
lastGeneratedAssetIds?: string[] | null;
|
||
lastGeneratedImageUrls?: string[] | null;
|
||
lastReferencedAssetIds?: string[] | null;
|
||
lastReferencedImageUrls?: string[] | null;
|
||
lastResolvedTargetAssetIds?: string[] | null;
|
||
lastResolvedTargetImageUrls?: string[] | null;
|
||
activeCanvasNodeIds?: string[] | null;
|
||
lastCanvasSnapshot?: unknown;
|
||
lastMessageId?: string | null;
|
||
incrementTurnCount?: boolean;
|
||
lastTurnSummaryPatch?: Record<string, unknown>;
|
||
};
|
||
|
||
type ConversationStateProtocolEventObserver = (event: {
|
||
type: ConversationStateEventType;
|
||
payload: Record<string, unknown>;
|
||
}) => void;
|
||
|
||
let protocolEventObserver: ConversationStateProtocolEventObserver | null = null;
|
||
|
||
export function bootstrapConversationStateProtocol(params: {
|
||
text: string;
|
||
workingState?: {
|
||
lastTurnSummary?: unknown;
|
||
} | null;
|
||
activeStoryboardTask?: StoryboardTaskDetail | null;
|
||
hasImageInputForPromptCarryForward: boolean;
|
||
}) {
|
||
const storyboardPreferenceMemory = extractStoryboardPreferenceMemory(params.workingState);
|
||
const imageWorkflowMemory = extractImageWorkflowMemory(params.workingState);
|
||
const promptMemory = extractPromptMemoryFromWorkingState(params.workingState);
|
||
const pendingGridConfirmation =
|
||
extractPendingGridConfirmationFromWorkingState(params.workingState);
|
||
const confirmedPrompt =
|
||
extractConfirmedPromptFromWorkingState(params.workingState) || promptMemory?.optimizedPrompt;
|
||
const actionResolution = resolveConversationAction({
|
||
text: params.text,
|
||
pendingGridConfirmation,
|
||
hasActiveStoryboardTask: Boolean(params.activeStoryboardTask?.task?.id),
|
||
imageWorkflowMemory,
|
||
confirmedPrompt,
|
||
hasImageInputForPromptCarryForward: params.hasImageInputForPromptCarryForward,
|
||
});
|
||
|
||
return {
|
||
workingState: params.workingState || null,
|
||
activeStoryboardTask: params.activeStoryboardTask || null,
|
||
storyboardPreferenceMemory,
|
||
imageWorkflowMemory,
|
||
promptMemory,
|
||
pendingGridConfirmation,
|
||
confirmedPrompt,
|
||
actionResolution,
|
||
} satisfies ConversationStateProtocolBootstrap;
|
||
}
|
||
|
||
export function composeConversationStateProtocol(params: {
|
||
bootstrap: ConversationStateProtocolBootstrap;
|
||
autoContextText?: string;
|
||
resolvedMediaInput: ResolvedChatMediaInput;
|
||
targetResolution: ResolvedTargetAssets;
|
||
}) {
|
||
const baseThreadState = buildConversationThreadState({
|
||
activeStoryboardTask: params.bootstrap.activeStoryboardTask,
|
||
storyboardPreferenceMemory: params.bootstrap.storyboardPreferenceMemory,
|
||
promptMemory: params.bootstrap.promptMemory,
|
||
imageWorkflowMemory: params.bootstrap.imageWorkflowMemory,
|
||
});
|
||
|
||
const composedConversationMemory = composeConversationMemory({
|
||
autoContextText: params.autoContextText || undefined,
|
||
resolvedMediaInput: params.resolvedMediaInput,
|
||
targetResolution: params.targetResolution,
|
||
promptMemory: params.bootstrap.promptMemory,
|
||
imageWorkflowMemory: params.bootstrap.imageWorkflowMemory,
|
||
storyboardPreferenceMemory: params.bootstrap.storyboardPreferenceMemory,
|
||
threadState: baseThreadState,
|
||
});
|
||
|
||
const threadState: AgentThreadState = {
|
||
workflowState: baseThreadState?.workflowState || null,
|
||
memoryHints: Array.from(
|
||
new Set([
|
||
...(baseThreadState?.memoryHints || []),
|
||
...composedConversationMemory.memoryHints,
|
||
])
|
||
),
|
||
contextSummary:
|
||
composedConversationMemory.contextSummary || baseThreadState?.contextSummary,
|
||
metadata: {
|
||
...(baseThreadState?.metadata || {}),
|
||
...composedConversationMemory.metadata,
|
||
},
|
||
};
|
||
|
||
return {
|
||
baseThreadState,
|
||
composedConversationMemory,
|
||
threadState,
|
||
} satisfies ConversationStateProtocolComposed;
|
||
}
|
||
|
||
export async function persistConversationStateProtocol(
|
||
params: ConversationStateProtocolPersistParams,
|
||
) {
|
||
const existing = await getConversationWorkingStateByChatId({
|
||
chatId: params.chatId,
|
||
}).catch(() => null);
|
||
|
||
const existingSummary =
|
||
existing?.lastTurnSummary && typeof existing.lastTurnSummary === "object"
|
||
? (existing.lastTurnSummary as Record<string, unknown>)
|
||
: {};
|
||
|
||
return upsertConversationWorkingState({
|
||
userId: params.userId,
|
||
chatId: params.chatId,
|
||
projectId: params.projectId,
|
||
currentIntent: params.currentIntent,
|
||
currentSubIntent: params.currentSubIntent,
|
||
currentFocusAssetIds: params.currentFocusAssetIds,
|
||
currentFocusImageUrls: params.currentFocusImageUrls,
|
||
selectedAssetIds: params.selectedAssetIds,
|
||
selectedImageUrls: params.selectedImageUrls,
|
||
lastUploadedAssetIds: params.lastUploadedAssetIds,
|
||
lastUploadedImageUrls: params.lastUploadedImageUrls,
|
||
lastGeneratedAssetIds: params.lastGeneratedAssetIds,
|
||
lastGeneratedImageUrls: params.lastGeneratedImageUrls,
|
||
lastReferencedAssetIds: params.lastReferencedAssetIds,
|
||
lastReferencedImageUrls: params.lastReferencedImageUrls,
|
||
lastResolvedTargetAssetIds: params.lastResolvedTargetAssetIds,
|
||
lastResolvedTargetImageUrls: params.lastResolvedTargetImageUrls,
|
||
activeCanvasNodeIds: params.activeCanvasNodeIds,
|
||
lastCanvasSnapshot: params.lastCanvasSnapshot,
|
||
lastMessageId: params.lastMessageId,
|
||
incrementTurnCount: params.incrementTurnCount,
|
||
lastTurnSummary:
|
||
params.lastTurnSummaryPatch !== undefined
|
||
? {
|
||
...existingSummary,
|
||
...params.lastTurnSummaryPatch,
|
||
}
|
||
: undefined,
|
||
});
|
||
}
|
||
|
||
export async function persistConversationStateEvent<T extends ConversationStateEventType>(
|
||
type: T,
|
||
payload: ConversationStateEventPayload<T>,
|
||
) {
|
||
const schema = conversationStateEventSchemas[type];
|
||
const parsed = schema.parse(payload) as ConversationStateEvent["payload"];
|
||
console.log("[ConversationStateProtocol] Event:", {
|
||
type,
|
||
chatId: (parsed as any).chatId,
|
||
userId: (parsed as any).userId,
|
||
messageId: (parsed as any).messageId,
|
||
});
|
||
protocolEventObserver?.({
|
||
type,
|
||
payload: parsed as Record<string, unknown>,
|
||
});
|
||
|
||
switch (type) {
|
||
case "prompt_only_ready": {
|
||
const typed = parsed as ConversationStateEventPayload<"prompt_only_ready">;
|
||
const existing = await getConversationWorkingStateByChatId({
|
||
chatId: typed.chatId,
|
||
}).catch(() => null);
|
||
const existingImageWorkflowMemory = extractImageWorkflowMemory(existing);
|
||
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: "vision",
|
||
currentFocusImageUrls: typed.sourceImageUrls || null,
|
||
lastResolvedTargetImageUrls: typed.sourceImageUrls || null,
|
||
lastMessageId: typed.messageId,
|
||
lastTurnSummaryPatch: {
|
||
text: typed.text.slice(0, 500),
|
||
resolutionMode: "prompt_only",
|
||
confidence: 90,
|
||
optimizedPrompt: typed.optimizedPrompt.slice(0, 1000),
|
||
promptMemory: buildPromptMemoryUpdate({
|
||
optimizedPrompt: typed.optimizedPrompt.slice(0, 1000),
|
||
sourceIntent: "image_gen",
|
||
sourceText: typed.text.slice(0, 500),
|
||
reusable: true,
|
||
}),
|
||
imageWorkflowMemory: buildImageWorkflowMemoryUpdate({
|
||
existing: existingImageWorkflowMemory,
|
||
lastIntent: "vision",
|
||
lastOptimizedPrompt: typed.optimizedPrompt.slice(0, 1000),
|
||
lastUserPrompt: typed.text.slice(0, 500),
|
||
lastRequestedImageCount: 1,
|
||
lastSourceImageUrls: typed.sourceImageUrls || null,
|
||
}),
|
||
imageSummary: typed.imageSummary?.slice(0, 1200),
|
||
sourceImageUrls: typed.sourceImageUrls || [],
|
||
},
|
||
});
|
||
}
|
||
|
||
case "grid_confirmation_requested": {
|
||
const typed = parsed as ConversationStateEventPayload<"grid_confirmation_requested">;
|
||
const existing = await getConversationWorkingStateByChatId({
|
||
chatId: typed.chatId,
|
||
}).catch(() => null);
|
||
const existingStoryboardMemory = extractStoryboardPreferenceMemory(existing);
|
||
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: "image_gen",
|
||
lastMessageId: typed.messageId,
|
||
lastTurnSummaryPatch: {
|
||
text: typed.text.slice(0, 500),
|
||
resolutionMode: "grid_confirmation",
|
||
confidence: 90,
|
||
storyboardMemory: buildStoryboardPreferenceMemoryUpdate({
|
||
existing: existingStoryboardMemory,
|
||
preferredGridType: typed.pendingGridConfirmation.gridType,
|
||
preferredShotCount: typed.pendingGridConfirmation.shotCount,
|
||
prefersConfirmation: true,
|
||
lastUsedGridType: typed.pendingGridConfirmation.gridType,
|
||
lastUsedShotCount: typed.pendingGridConfirmation.shotCount,
|
||
}),
|
||
pendingGridConfirmation: {
|
||
gridType: typed.pendingGridConfirmation.gridType,
|
||
shotCount: typed.pendingGridConfirmation.shotCount,
|
||
label: typed.pendingGridConfirmation.label,
|
||
originalPrompt: typed.pendingGridConfirmation.originalPrompt.slice(0, 1000),
|
||
},
|
||
},
|
||
});
|
||
}
|
||
|
||
case "clarification_requested": {
|
||
const typed = parsed as ConversationStateEventPayload<"clarification_requested">;
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: typed.currentIntent,
|
||
lastMessageId: typed.messageId,
|
||
lastTurnSummaryPatch: {
|
||
text: typed.text.slice(0, 500),
|
||
resolutionMode: "clarification",
|
||
confidence: 80,
|
||
clarificationMessage: typed.clarificationMessage.slice(0, 1000),
|
||
},
|
||
});
|
||
}
|
||
|
||
case "storyboard_confirmed": {
|
||
const typed = parsed as ConversationStateEventPayload<"storyboard_confirmed">;
|
||
const existing = await getConversationWorkingStateByChatId({
|
||
chatId: typed.chatId,
|
||
}).catch(() => null);
|
||
const existingStoryboardMemory = extractStoryboardPreferenceMemory(existing);
|
||
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: "video_gen",
|
||
lastMessageId: typed.messageId,
|
||
lastTurnSummaryPatch: {
|
||
text: typed.text.slice(0, 500),
|
||
resolutionMode: "storyboard_confirmed",
|
||
confidence: 95,
|
||
videoWorkflowStatus: typed.status || "confirmed",
|
||
storyboardTaskId: typed.taskId,
|
||
storyboardWorkflowAction: "confirmed",
|
||
storyboardMemory: buildStoryboardPreferenceMemoryUpdate({
|
||
existing: existingStoryboardMemory,
|
||
prefersConfirmation: true,
|
||
}),
|
||
},
|
||
});
|
||
}
|
||
|
||
case "storyboard_execution_started": {
|
||
const typed = parsed as ConversationStateEventPayload<"storyboard_execution_started">;
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: "video_gen",
|
||
lastMessageId: typed.messageId,
|
||
lastTurnSummaryPatch: {
|
||
text: typed.text.slice(0, 500),
|
||
resolutionMode: "storyboard_execution_started",
|
||
confidence: 95,
|
||
videoWorkflowStatus: typed.status || "running",
|
||
storyboardTaskId: typed.taskId,
|
||
storyboardWorkflowAction: "execution_started",
|
||
},
|
||
});
|
||
}
|
||
|
||
case "image_workflow_updated": {
|
||
const typed = parsed as ConversationStateEventPayload<"image_workflow_updated">;
|
||
const existing = await getConversationWorkingStateByChatId({
|
||
chatId: typed.chatId,
|
||
}).catch(() => null);
|
||
const existingImageWorkflowMemory = extractImageWorkflowMemory(existing);
|
||
const nextOptimizedPrompt = typed.lastOptimizedPrompt?.trim();
|
||
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: typed.currentIntent,
|
||
currentSubIntent: typed.currentSubIntent,
|
||
currentFocusAssetIds: typed.currentFocusAssetIds,
|
||
currentFocusImageUrls: typed.currentFocusImageUrls,
|
||
lastGeneratedAssetIds: typed.lastGeneratedAssetIds,
|
||
lastGeneratedImageUrls: typed.lastGeneratedImageUrls,
|
||
lastResolvedTargetAssetIds: typed.lastResolvedTargetAssetIds,
|
||
lastResolvedTargetImageUrls: typed.lastResolvedTargetImageUrls,
|
||
lastMessageId: typed.lastMessageId,
|
||
lastTurnSummaryPatch: {
|
||
promptMemory: nextOptimizedPrompt
|
||
? buildPromptMemoryUpdate({
|
||
optimizedPrompt: nextOptimizedPrompt.slice(0, 1000),
|
||
sourceIntent: typed.currentIntent,
|
||
sourceText: typed.lastUserPrompt.slice(0, 500),
|
||
reusable: true,
|
||
})
|
||
: undefined,
|
||
imageWorkflowMemory: buildImageWorkflowMemoryUpdate({
|
||
existing: existingImageWorkflowMemory,
|
||
lastIntent: typed.currentIntent,
|
||
lastOptimizedPrompt: nextOptimizedPrompt?.slice(0, 1000),
|
||
lastUserPrompt: typed.lastUserPrompt.slice(0, 500),
|
||
lastRequestedImageCount: typed.lastRequestedImageCount,
|
||
lastAspectRatio: typed.lastAspectRatio?.slice(0, 32),
|
||
lastImageSize: typed.lastImageSize?.slice(0, 32),
|
||
lastSourceImageUrls: typed.lastSourceImageUrls || null,
|
||
lastResultImageUrls: typed.lastResultImageUrls || null,
|
||
}),
|
||
},
|
||
});
|
||
}
|
||
|
||
case "storyboard_revision_requested": {
|
||
const typed = parsed as ConversationStateEventPayload<"storyboard_revision_requested">;
|
||
const existing = await getConversationWorkingStateByChatId({
|
||
chatId: typed.chatId,
|
||
}).catch(() => null);
|
||
const existingStoryboardMemory = extractStoryboardPreferenceMemory(existing);
|
||
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: "video_gen",
|
||
lastMessageId: typed.messageId,
|
||
lastTurnSummaryPatch: {
|
||
text: typed.text.slice(0, 500),
|
||
resolutionMode: "storyboard_revision",
|
||
confidence: 90,
|
||
videoWorkflowStatus: typed.status || "awaiting_user_confirmation",
|
||
storyboardTaskId: typed.taskId,
|
||
storyboardWorkflowAction: "revision_requested",
|
||
storyboardMemory: buildStoryboardPreferenceMemoryUpdate({
|
||
existing: existingStoryboardMemory,
|
||
prefersConfirmation: true,
|
||
}),
|
||
},
|
||
});
|
||
}
|
||
|
||
case "storyboard_execution_completed": {
|
||
const typed = parsed as ConversationStateEventPayload<"storyboard_execution_completed">;
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: "video_gen",
|
||
lastMessageId: typed.messageId,
|
||
lastTurnSummaryPatch: {
|
||
text: typed.text.slice(0, 500),
|
||
resolutionMode: "storyboard_execution_completed",
|
||
confidence: 95,
|
||
videoWorkflowStatus: typed.status || "completed",
|
||
storyboardTaskId: typed.taskId,
|
||
storyboardWorkflowAction: "execution_completed",
|
||
successCount: typed.successCount,
|
||
failedCount: typed.failedCount,
|
||
firstVideoUrl: typed.firstVideoUrl,
|
||
},
|
||
});
|
||
}
|
||
|
||
case "video_generation_started": {
|
||
const typed = parsed as ConversationStateEventPayload<"video_generation_started">;
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: "video_gen",
|
||
lastMessageId: typed.messageId,
|
||
lastTurnSummaryPatch: {
|
||
text: typed.text.slice(0, 500),
|
||
resolutionMode: "video_generation_started",
|
||
confidence: 90,
|
||
videoWorkflowStatus: typed.status || "running",
|
||
videoWorkflowSource: typed.source,
|
||
videoProviderType: typed.providerType,
|
||
videoModelRef: typed.modelRef,
|
||
videoGenerationMode: typed.mode,
|
||
videoTaskId: typed.taskId,
|
||
},
|
||
});
|
||
}
|
||
|
||
case "video_generation_completed": {
|
||
const typed = parsed as ConversationStateEventPayload<"video_generation_completed">;
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: "video_gen",
|
||
lastMessageId: typed.messageId,
|
||
lastTurnSummaryPatch: {
|
||
text: typed.text.slice(0, 500),
|
||
resolutionMode: "video_generation_completed",
|
||
confidence: 95,
|
||
videoWorkflowStatus: typed.status || "completed",
|
||
videoWorkflowSource: typed.source,
|
||
videoProviderType: typed.providerType,
|
||
videoModelRef: typed.modelRef,
|
||
videoGenerationMode: typed.mode,
|
||
videoTaskId: typed.taskId,
|
||
videoTemporaryUrl: typed.temporaryVideoUrl,
|
||
},
|
||
});
|
||
}
|
||
|
||
case "video_generation_failed": {
|
||
const typed = parsed as ConversationStateEventPayload<"video_generation_failed">;
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: "video_gen",
|
||
lastMessageId: typed.messageId,
|
||
lastTurnSummaryPatch: {
|
||
text: typed.text.slice(0, 500),
|
||
resolutionMode: "video_generation_failed",
|
||
confidence: 85,
|
||
videoWorkflowStatus: typed.status || "failed",
|
||
videoWorkflowSource: typed.source,
|
||
videoProviderType: typed.providerType,
|
||
videoModelRef: typed.modelRef,
|
||
videoGenerationMode: typed.mode,
|
||
videoTaskId: typed.taskId,
|
||
videoWorkflowError: typed.error?.slice(0, 1000),
|
||
},
|
||
});
|
||
}
|
||
|
||
case "target_resolved": {
|
||
const typed = parsed as ConversationStateEventPayload<"target_resolved">;
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: typed.currentIntent,
|
||
currentFocusAssetIds: typed.currentFocusAssetIds,
|
||
currentFocusImageUrls: typed.currentFocusImageUrls,
|
||
selectedAssetIds: typed.selectedAssetIds,
|
||
selectedImageUrls: typed.selectedImageUrls,
|
||
lastUploadedAssetIds: typed.lastUploadedAssetIds,
|
||
lastUploadedImageUrls: typed.lastUploadedImageUrls,
|
||
lastReferencedAssetIds: typed.lastReferencedAssetIds,
|
||
lastReferencedImageUrls: typed.lastReferencedImageUrls,
|
||
lastResolvedTargetAssetIds: typed.lastResolvedTargetAssetIds,
|
||
lastResolvedTargetImageUrls: typed.lastResolvedTargetImageUrls,
|
||
activeCanvasNodeIds: typed.activeCanvasNodeIds,
|
||
lastCanvasSnapshot: typed.lastCanvasSnapshot,
|
||
lastMessageId: typed.lastMessageId,
|
||
incrementTurnCount: true,
|
||
lastTurnSummaryPatch: {
|
||
text: typed.text.slice(0, 500),
|
||
resolutionMode: typed.resolutionMode,
|
||
confidence: typed.confidence,
|
||
},
|
||
});
|
||
}
|
||
|
||
case "video_workflow_updated": {
|
||
const typed = parsed as ConversationStateEventPayload<"video_workflow_updated">;
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: "video_gen",
|
||
lastMessageId: typed.lastMessageId,
|
||
lastTurnSummaryPatch:
|
||
typed.text || typed.status
|
||
? {
|
||
text: typed.text?.slice(0, 500),
|
||
videoWorkflowStatus: typed.status,
|
||
}
|
||
: undefined,
|
||
});
|
||
}
|
||
|
||
case "workflow_cancelled": {
|
||
const typed = parsed as ConversationStateEventPayload<"workflow_cancelled">;
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: typed.workflowType === "storyboard_video" ? "video_gen" : undefined,
|
||
lastMessageId: typed.messageId,
|
||
lastTurnSummaryPatch: {
|
||
text: typed.text.slice(0, 500),
|
||
resolutionMode: "workflow_cancelled",
|
||
confidence: 95,
|
||
videoWorkflowStatus: typed.status || "cancelled",
|
||
cancelledWorkflowType: typed.workflowType,
|
||
cancelledWorkflowId: typed.workflowId,
|
||
workflowAction: "cancel_requested",
|
||
},
|
||
});
|
||
}
|
||
|
||
case "agent_preset_completed": {
|
||
const typed = parsed as ConversationStateEventPayload<"agent_preset_completed">;
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: typed.currentIntent,
|
||
lastMessageId: typed.lastMessageId,
|
||
lastTurnSummaryPatch: {
|
||
text: typed.text.slice(0, 500),
|
||
agentPresetId: typed.agentPresetId,
|
||
agentPresetName: typed.agentPresetName,
|
||
slotCount: typed.slotCount,
|
||
successCount: typed.successCount,
|
||
},
|
||
});
|
||
}
|
||
|
||
case "storyboard_assistant_updated": {
|
||
const typed = parsed as ConversationStateEventPayload<"storyboard_assistant_updated">;
|
||
const existing = await getConversationWorkingStateByChatId({
|
||
chatId: typed.chatId,
|
||
}).catch(() => null);
|
||
const existingStoryboardMemory = extractStoryboardPreferenceMemory(existing);
|
||
|
||
return persistConversationStateProtocol({
|
||
userId: typed.userId,
|
||
chatId: typed.chatId,
|
||
projectId: typed.projectId,
|
||
currentIntent: "video_gen",
|
||
lastMessageId: typed.lastMessageId,
|
||
lastTurnSummaryPatch: {
|
||
text: typed.text.slice(0, 500),
|
||
resolutionMode: "storyboard",
|
||
confidence: 95,
|
||
storyboardMemory: buildStoryboardPreferenceMemoryUpdate({
|
||
existing: existingStoryboardMemory,
|
||
prefersConfirmation: true,
|
||
lastUsedGridType: typed.storyboardGridType,
|
||
lastUsedShotCount: typed.storyboardShotCount,
|
||
}),
|
||
},
|
||
});
|
||
}
|
||
|
||
default:
|
||
return null;
|
||
}
|
||
}
|
||
|
||
export function setConversationStateProtocolEventObserver(
|
||
observer: ConversationStateProtocolEventObserver | null,
|
||
) {
|
||
protocolEventObserver = observer;
|
||
}
|
||
|
||
export async function persistPromptOnlyTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
messageId: string;
|
||
text: string;
|
||
optimizedPrompt: string;
|
||
imageSummary?: string;
|
||
sourceImageUrls?: string[] | null;
|
||
}) {
|
||
return persistConversationStateEvent("prompt_only_ready", {
|
||
userId: params.userId,
|
||
chatId: params.chatId,
|
||
projectId: params.projectId,
|
||
messageId: params.messageId,
|
||
text: params.text,
|
||
optimizedPrompt: params.optimizedPrompt,
|
||
imageSummary: params.imageSummary,
|
||
sourceImageUrls: params.sourceImageUrls || null,
|
||
});
|
||
}
|
||
|
||
export async function persistGridConfirmationTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
messageId: string;
|
||
text: string;
|
||
pendingGridConfirmation: PendingGridConfirmation;
|
||
}) {
|
||
return persistConversationStateEvent("grid_confirmation_requested", {
|
||
userId: params.userId,
|
||
chatId: params.chatId,
|
||
projectId: params.projectId,
|
||
messageId: params.messageId,
|
||
text: params.text,
|
||
pendingGridConfirmation: params.pendingGridConfirmation,
|
||
});
|
||
}
|
||
|
||
export async function persistClarificationTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
messageId: string;
|
||
text: string;
|
||
clarificationMessage: string;
|
||
currentIntent?: string | null;
|
||
pendingGridConfirmation?: PendingGridConfirmation | null;
|
||
}) {
|
||
if (params.pendingGridConfirmation) {
|
||
return persistConversationStateEvent("grid_confirmation_requested", {
|
||
userId: params.userId,
|
||
chatId: params.chatId,
|
||
projectId: params.projectId,
|
||
messageId: params.messageId,
|
||
text: params.text,
|
||
pendingGridConfirmation: params.pendingGridConfirmation,
|
||
});
|
||
}
|
||
|
||
return persistConversationStateEvent("clarification_requested", {
|
||
userId: params.userId,
|
||
chatId: params.chatId,
|
||
projectId: params.projectId,
|
||
messageId: params.messageId,
|
||
text: params.text,
|
||
clarificationMessage: params.clarificationMessage,
|
||
currentIntent: params.currentIntent,
|
||
});
|
||
}
|
||
|
||
export async function persistStoryboardConfirmedTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
messageId: string;
|
||
text: string;
|
||
taskId?: string;
|
||
status?: string;
|
||
}) {
|
||
return persistConversationStateEvent("storyboard_confirmed", {
|
||
userId: params.userId,
|
||
chatId: params.chatId,
|
||
projectId: params.projectId,
|
||
messageId: params.messageId,
|
||
text: params.text,
|
||
taskId: params.taskId,
|
||
status: params.status,
|
||
});
|
||
}
|
||
|
||
export async function persistStoryboardExecutionStartedTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
messageId: string;
|
||
text: string;
|
||
taskId: string;
|
||
status?: string;
|
||
}) {
|
||
return persistConversationStateEvent("storyboard_execution_started", {
|
||
userId: params.userId,
|
||
chatId: params.chatId,
|
||
projectId: params.projectId,
|
||
messageId: params.messageId,
|
||
text: params.text,
|
||
taskId: params.taskId,
|
||
status: params.status,
|
||
});
|
||
}
|
||
|
||
export async function persistStoryboardRevisionTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
messageId: string;
|
||
text: string;
|
||
taskId: string;
|
||
status?: string;
|
||
}) {
|
||
return persistConversationStateEvent("storyboard_revision_requested", {
|
||
userId: params.userId,
|
||
chatId: params.chatId,
|
||
projectId: params.projectId,
|
||
messageId: params.messageId,
|
||
text: params.text,
|
||
taskId: params.taskId,
|
||
status: params.status,
|
||
});
|
||
}
|
||
|
||
export async function persistStoryboardExecutionCompletedTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
messageId: string;
|
||
text: string;
|
||
taskId: string;
|
||
status?: string;
|
||
successCount?: number;
|
||
failedCount?: number;
|
||
firstVideoUrl?: string;
|
||
}) {
|
||
return persistConversationStateEvent("storyboard_execution_completed", {
|
||
userId: params.userId,
|
||
chatId: params.chatId,
|
||
projectId: params.projectId,
|
||
messageId: params.messageId,
|
||
text: params.text,
|
||
taskId: params.taskId,
|
||
status: params.status,
|
||
successCount: params.successCount,
|
||
failedCount: params.failedCount,
|
||
firstVideoUrl: params.firstVideoUrl,
|
||
});
|
||
}
|
||
|
||
export async function persistVideoGenerationStartedTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
messageId: string;
|
||
text: string;
|
||
source: string;
|
||
providerType?: string;
|
||
modelRef?: string;
|
||
mode?: string;
|
||
taskId?: string;
|
||
status?: string;
|
||
}) {
|
||
return persistConversationStateEvent("video_generation_started", {
|
||
userId: params.userId,
|
||
chatId: params.chatId,
|
||
projectId: params.projectId,
|
||
messageId: params.messageId,
|
||
text: params.text,
|
||
source: params.source,
|
||
providerType: params.providerType,
|
||
modelRef: params.modelRef,
|
||
mode: params.mode,
|
||
taskId: params.taskId,
|
||
status: params.status,
|
||
});
|
||
}
|
||
|
||
export async function persistVideoGenerationCompletedTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
messageId: string;
|
||
text: string;
|
||
source: string;
|
||
providerType?: string;
|
||
modelRef?: string;
|
||
mode?: string;
|
||
taskId?: string;
|
||
status?: string;
|
||
temporaryVideoUrl?: string;
|
||
}) {
|
||
return persistConversationStateEvent("video_generation_completed", {
|
||
userId: params.userId,
|
||
chatId: params.chatId,
|
||
projectId: params.projectId,
|
||
messageId: params.messageId,
|
||
text: params.text,
|
||
source: params.source,
|
||
providerType: params.providerType,
|
||
modelRef: params.modelRef,
|
||
mode: params.mode,
|
||
taskId: params.taskId,
|
||
status: params.status,
|
||
temporaryVideoUrl: params.temporaryVideoUrl,
|
||
});
|
||
}
|
||
|
||
export async function persistVideoGenerationFailedTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
messageId: string;
|
||
text: string;
|
||
source: string;
|
||
providerType?: string;
|
||
modelRef?: string;
|
||
mode?: string;
|
||
taskId?: string;
|
||
status?: string;
|
||
error?: string;
|
||
}) {
|
||
return persistConversationStateEvent("video_generation_failed", {
|
||
userId: params.userId,
|
||
chatId: params.chatId,
|
||
projectId: params.projectId,
|
||
messageId: params.messageId,
|
||
text: params.text,
|
||
source: params.source,
|
||
providerType: params.providerType,
|
||
modelRef: params.modelRef,
|
||
mode: params.mode,
|
||
taskId: params.taskId,
|
||
status: params.status,
|
||
error: params.error,
|
||
});
|
||
}
|
||
|
||
export async function persistImageWorkflowTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
currentIntent: "image_gen" | "image_edit" | "vision";
|
||
currentSubIntent?: string | null;
|
||
currentFocusAssetIds?: string[] | null;
|
||
currentFocusImageUrls?: string[] | null;
|
||
lastGeneratedAssetIds?: string[] | null;
|
||
lastGeneratedImageUrls?: string[] | null;
|
||
lastResolvedTargetAssetIds?: string[] | null;
|
||
lastResolvedTargetImageUrls?: string[] | null;
|
||
lastMessageId: string;
|
||
lastUserPrompt: string;
|
||
lastOptimizedPrompt?: string;
|
||
lastRequestedImageCount?: number;
|
||
lastAspectRatio?: string;
|
||
lastImageSize?: string;
|
||
lastSourceImageUrls?: string[] | null;
|
||
lastResultImageUrls?: string[] | null;
|
||
}) {
|
||
return persistConversationStateEvent("image_workflow_updated", {
|
||
userId: params.userId,
|
||
chatId: params.chatId,
|
||
projectId: params.projectId,
|
||
currentIntent: params.currentIntent,
|
||
currentSubIntent: params.currentSubIntent,
|
||
currentFocusAssetIds: params.currentFocusAssetIds,
|
||
currentFocusImageUrls: params.currentFocusImageUrls,
|
||
lastGeneratedAssetIds: params.lastGeneratedAssetIds,
|
||
lastGeneratedImageUrls: params.lastGeneratedImageUrls,
|
||
lastResolvedTargetAssetIds: params.lastResolvedTargetAssetIds,
|
||
lastResolvedTargetImageUrls: params.lastResolvedTargetImageUrls,
|
||
lastMessageId: params.lastMessageId,
|
||
lastUserPrompt: params.lastUserPrompt,
|
||
lastOptimizedPrompt: params.lastOptimizedPrompt,
|
||
lastRequestedImageCount: params.lastRequestedImageCount,
|
||
lastAspectRatio: params.lastAspectRatio,
|
||
lastImageSize: params.lastImageSize,
|
||
lastSourceImageUrls: params.lastSourceImageUrls || null,
|
||
lastResultImageUrls: params.lastResultImageUrls || null,
|
||
});
|
||
}
|
||
|
||
export async function persistResolvedTargetTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
currentIntent?: string | null;
|
||
currentFocusAssetIds?: string[] | null;
|
||
currentFocusImageUrls?: string[] | null;
|
||
selectedAssetIds?: string[] | null;
|
||
selectedImageUrls?: string[] | null;
|
||
lastUploadedAssetIds?: string[] | null;
|
||
lastUploadedImageUrls?: string[] | null;
|
||
lastReferencedAssetIds?: string[] | null;
|
||
lastReferencedImageUrls?: string[] | null;
|
||
lastResolvedTargetAssetIds?: string[] | null;
|
||
lastResolvedTargetImageUrls?: string[] | null;
|
||
activeCanvasNodeIds?: string[] | null;
|
||
lastCanvasSnapshot?: unknown;
|
||
lastMessageId: string;
|
||
text: string;
|
||
resolutionMode: string;
|
||
confidence: number;
|
||
}) {
|
||
return persistConversationStateEvent("target_resolved", params);
|
||
}
|
||
|
||
export async function persistVideoWorkflowTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
lastMessageId: string;
|
||
text?: string;
|
||
status?: string;
|
||
}) {
|
||
return persistConversationStateEvent("video_workflow_updated", params);
|
||
}
|
||
|
||
export async function persistWorkflowCancelledTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
messageId: string;
|
||
text: string;
|
||
workflowType: string;
|
||
workflowId?: string;
|
||
status?: string;
|
||
}) {
|
||
return persistConversationStateEvent("workflow_cancelled", {
|
||
userId: params.userId,
|
||
chatId: params.chatId,
|
||
projectId: params.projectId,
|
||
messageId: params.messageId,
|
||
text: params.text,
|
||
workflowType: params.workflowType,
|
||
workflowId: params.workflowId,
|
||
status: params.status,
|
||
});
|
||
}
|
||
|
||
export async function persistAgentPresetTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
currentIntent: "image_gen" | "video_gen";
|
||
lastMessageId: string;
|
||
text: string;
|
||
agentPresetId: string;
|
||
agentPresetName: string;
|
||
slotCount: number;
|
||
successCount: number;
|
||
}) {
|
||
return persistConversationStateEvent("agent_preset_completed", params);
|
||
}
|
||
|
||
export async function persistStoryboardAssistantTurn(params: {
|
||
userId: string;
|
||
chatId: string;
|
||
projectId?: string | null;
|
||
lastMessageId: string;
|
||
text: string;
|
||
storyboardGridType?: "grid_2x2" | "grid_3x2" | "grid_3x3";
|
||
storyboardShotCount?: 4 | 6 | 9;
|
||
}) {
|
||
return persistConversationStateEvent("storyboard_assistant_updated", params);
|
||
}
|