feat: unify agent workflow protocol
This commit is contained in:
parent
159137b3d7
commit
5a4e9fd582
@ -52,7 +52,6 @@ import {
|
|||||||
executeDirectVideoGeneration,
|
executeDirectVideoGeneration,
|
||||||
generateVideo,
|
generateVideo,
|
||||||
} from "@/lib/ai/tools/generate-video";
|
} from "@/lib/ai/tools/generate-video";
|
||||||
import { executeConfirmedStoryboardTask } from "@/lib/video-agent/execute-storyboard-task";
|
|
||||||
import { getEnabledTools } from "@/lib/ai/tools-registry";
|
import { getEnabledTools } from "@/lib/ai/tools-registry";
|
||||||
import { isProductionEnvironment } from "@/lib/constants";
|
import { isProductionEnvironment } from "@/lib/constants";
|
||||||
import {
|
import {
|
||||||
@ -90,7 +89,6 @@ import {
|
|||||||
} from "@/lib/db/schema";
|
} from "@/lib/db/schema";
|
||||||
import { generateCanvasImage } from "@/lib/canvas/generate-image";
|
import { generateCanvasImage } from "@/lib/canvas/generate-image";
|
||||||
import {
|
import {
|
||||||
buildGridConfirmationMessage,
|
|
||||||
detectRequestedStoryboardGrid,
|
detectRequestedStoryboardGrid,
|
||||||
isGridConfirmationReply,
|
isGridConfirmationReply,
|
||||||
type PendingGridConfirmation,
|
type PendingGridConfirmation,
|
||||||
@ -99,17 +97,22 @@ import {
|
|||||||
bootstrapConversationStateProtocol,
|
bootstrapConversationStateProtocol,
|
||||||
composeConversationStateProtocol,
|
composeConversationStateProtocol,
|
||||||
persistAgentPresetTurn,
|
persistAgentPresetTurn,
|
||||||
persistConversationStateProtocol,
|
persistClarificationTurn,
|
||||||
setConversationStateProtocolEventObserver,
|
setConversationStateProtocolEventObserver,
|
||||||
persistResolvedTargetTurn,
|
persistResolvedTargetTurn,
|
||||||
persistGridConfirmationTurn,
|
|
||||||
persistImageWorkflowTurn,
|
persistImageWorkflowTurn,
|
||||||
persistPromptOnlyTurn,
|
persistPromptOnlyTurn,
|
||||||
resolveGridConfirmationRequest,
|
resolveGridConfirmationRequest,
|
||||||
resolveTargetExecutionOverride,
|
resolveTargetExecutionOverride,
|
||||||
persistStoryboardConfirmedTurn,
|
persistStoryboardConfirmedTurn,
|
||||||
|
persistStoryboardExecutionCompletedTurn,
|
||||||
|
persistStoryboardExecutionStartedTurn,
|
||||||
|
persistStoryboardRevisionTurn,
|
||||||
persistStoryboardAssistantTurn,
|
persistStoryboardAssistantTurn,
|
||||||
persistVideoWorkflowTurn,
|
persistVideoGenerationCompletedTurn,
|
||||||
|
persistVideoGenerationFailedTurn,
|
||||||
|
persistVideoGenerationStartedTurn,
|
||||||
|
persistWorkflowCancelledTurn,
|
||||||
} from "@/lib/ai/conversation-state-protocol";
|
} from "@/lib/ai/conversation-state-protocol";
|
||||||
import { resolveChatMediaInput } from "@/lib/ai/context-memory/media-input";
|
import { resolveChatMediaInput } from "@/lib/ai/context-memory/media-input";
|
||||||
import { resolveTargetAssets } from "@/lib/ai/context-memory/target-resolution";
|
import { resolveTargetAssets } from "@/lib/ai/context-memory/target-resolution";
|
||||||
@ -120,26 +123,22 @@ import type {
|
|||||||
import { ChatSDKError } from "@/lib/errors";
|
import { ChatSDKError } from "@/lib/errors";
|
||||||
import { extractImageUrlsFromText } from "@/lib/image-url";
|
import { extractImageUrlsFromText } from "@/lib/image-url";
|
||||||
import {
|
import {
|
||||||
buildImageWorkflowMemoryUpdate,
|
resolveClarificationBypass,
|
||||||
extractImageWorkflowMemory,
|
resolveClarificationMessage,
|
||||||
} from "@/lib/ai/image-workflow-memory";
|
} from "@/lib/ai/runtime/clarification-interruption";
|
||||||
import {
|
|
||||||
buildPromptMemoryUpdate,
|
|
||||||
} from "@/lib/ai/prompt-memory";
|
|
||||||
import { uploadFile } from "@/lib/storage/service";
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
|
import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
|
||||||
import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline";
|
import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline";
|
||||||
import {
|
import {
|
||||||
reviseStoryboardTaskByCommand,
|
buildStoryboardConfirmedAck,
|
||||||
resolveStoryboardRevisionCommand,
|
executeStoryboardConfirmedWorkflow,
|
||||||
} from "@/lib/video-agent/revise-storyboard-task";
|
handleStoryboardWorkflowFollowup,
|
||||||
|
} from "@/lib/video-agent/storyboard-chat-followup";
|
||||||
import {
|
import {
|
||||||
buildStoryboardPreferenceMemoryUpdate,
|
buildStoryboardPreferenceMemoryUpdate,
|
||||||
extractStoryboardPreferenceMemory,
|
|
||||||
} from "@/lib/video-agent/storyboard-preference-memory";
|
} from "@/lib/video-agent/storyboard-preference-memory";
|
||||||
import {
|
import {
|
||||||
attachVideoStoryboardTaskMessage,
|
attachVideoStoryboardTaskMessage,
|
||||||
getVideoStoryboardTaskDetail,
|
|
||||||
getLatestActiveVideoStoryboardTaskByChatId,
|
getLatestActiveVideoStoryboardTaskByChatId,
|
||||||
markVideoStoryboardAwaitingConfirmation,
|
markVideoStoryboardAwaitingConfirmation,
|
||||||
markVideoStoryboardConfirmed,
|
markVideoStoryboardConfirmed,
|
||||||
@ -2459,6 +2458,11 @@ async function saveGeneratedImages(
|
|||||||
relationType?: "generated_from" | "edited_from";
|
relationType?: "generated_from" | "edited_from";
|
||||||
sourceType?: string;
|
sourceType?: string;
|
||||||
projectId?: string | null;
|
projectId?: string | null;
|
||||||
|
userPrompt?: string;
|
||||||
|
optimizedPrompt?: string;
|
||||||
|
requestedImageCount?: number;
|
||||||
|
aspectRatio?: string;
|
||||||
|
imageSize?: string;
|
||||||
},
|
},
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
// 匹配 Markdown 图片格式: 
|
// 匹配 Markdown 图片格式: 
|
||||||
@ -2559,30 +2563,31 @@ async function saveGeneratedImages(
|
|||||||
|
|
||||||
if (savedOriginalUrls.length > 0) {
|
if (savedOriginalUrls.length > 0) {
|
||||||
try {
|
try {
|
||||||
const existingWorkingState = await getConversationWorkingStateByChatId({ chatId }).catch(
|
const resultAssetIds = savedAssetIds.length > 0 ? savedAssetIds : null;
|
||||||
() => null
|
await persistImageWorkflowTurn({
|
||||||
);
|
|
||||||
const existingImageWorkflowMemory = extractImageWorkflowMemory(existingWorkingState);
|
|
||||||
await persistConversationStateProtocol({
|
|
||||||
userId,
|
userId,
|
||||||
chatId,
|
chatId,
|
||||||
currentFocusAssetIds: savedAssetIds,
|
projectId: options?.projectId || null,
|
||||||
|
currentIntent: options?.relationType === "edited_from" ? "image_edit" : "image_gen",
|
||||||
|
currentFocusAssetIds: resultAssetIds,
|
||||||
currentFocusImageUrls: savedOriginalUrls,
|
currentFocusImageUrls: savedOriginalUrls,
|
||||||
lastGeneratedAssetIds: savedAssetIds,
|
lastGeneratedAssetIds: resultAssetIds,
|
||||||
lastGeneratedImageUrls: savedOriginalUrls,
|
lastGeneratedImageUrls: savedOriginalUrls,
|
||||||
lastResolvedTargetAssetIds: savedAssetIds,
|
lastResolvedTargetAssetIds: resultAssetIds,
|
||||||
lastResolvedTargetImageUrls: savedOriginalUrls,
|
lastResolvedTargetImageUrls: savedOriginalUrls,
|
||||||
lastMessageId: messageId,
|
lastMessageId: messageId,
|
||||||
lastTurnSummaryPatch: {
|
lastUserPrompt: String(options?.userPrompt || prompt || "").trim() || prompt,
|
||||||
imageWorkflowMemory: buildImageWorkflowMemoryUpdate({
|
lastOptimizedPrompt:
|
||||||
existing: existingImageWorkflowMemory,
|
String(
|
||||||
lastIntent: options?.relationType === "edited_from" ? "image_edit" : "image_gen",
|
options?.optimizedPrompt ||
|
||||||
lastUserPrompt: prompt,
|
(options?.relationType !== "edited_from" ? prompt : "") ||
|
||||||
lastRequestedImageCount: savedOriginalUrls.length,
|
"",
|
||||||
lastSourceImageUrls: options?.sourceImageUrls || null,
|
).trim() || undefined,
|
||||||
lastResultImageUrls: savedOriginalUrls,
|
lastRequestedImageCount: options?.requestedImageCount || savedOriginalUrls.length,
|
||||||
}),
|
lastAspectRatio: options?.aspectRatio,
|
||||||
},
|
lastImageSize: options?.imageSize,
|
||||||
|
lastSourceImageUrls: options?.sourceImageUrls || null,
|
||||||
|
lastResultImageUrls: savedOriginalUrls,
|
||||||
});
|
});
|
||||||
} catch (memoryError) {
|
} catch (memoryError) {
|
||||||
console.warn("[Gallery] Failed to update working state with generated images:", memoryError);
|
console.warn("[Gallery] Failed to update working state with generated images:", memoryError);
|
||||||
@ -3163,6 +3168,8 @@ export async function POST(request: Request) {
|
|||||||
console.log(
|
console.log(
|
||||||
`[Smart Auto Mode] 🔐 Force agent conversation due to active storyboard task: ${activeStoryboardTaskForChat.task.id}`
|
`[Smart Auto Mode] 🔐 Force agent conversation due to active storyboard task: ${activeStoryboardTaskForChat.task.id}`
|
||||||
);
|
);
|
||||||
|
} else if (resolvedConversationAction.kind === "image_generation_followup") {
|
||||||
|
console.log("[Smart Auto Mode] 🪄 Force agent conversation due to image generation follow-up workflow");
|
||||||
} else if (resolvedConversationAction.kind === "image_edit_followup") {
|
} else if (resolvedConversationAction.kind === "image_edit_followup") {
|
||||||
console.log("[Smart Auto Mode] 🖼️ Force agent conversation due to image edit follow-up workflow");
|
console.log("[Smart Auto Mode] 🖼️ Force agent conversation due to image edit follow-up workflow");
|
||||||
}
|
}
|
||||||
@ -4662,51 +4669,35 @@ export async function POST(request: Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (shouldRunAgentConversation && smartAutoNeedsClarification) {
|
if (shouldRunAgentConversation && smartAutoNeedsClarification) {
|
||||||
const hasCurrentImageInput = message.parts.some((part: any) =>
|
const clarificationBypass = resolveClarificationBypass({
|
||||||
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
text: message.parts
|
||||||
);
|
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
||||||
const hasHistoryImageInput = smartAutoHistoryImages.length > 0;
|
.map((part) => part.text)
|
||||||
const latestUserText = message.parts
|
.join(" ")
|
||||||
.filter((part): part is { type: "text"; text: string } => part.type === "text")
|
.trim(),
|
||||||
.map((part) => part.text)
|
hasCurrentImageInput: message.parts.some((part: any) =>
|
||||||
.join(" ")
|
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
|
||||||
.trim();
|
),
|
||||||
|
hasHistoryImageInput: smartAutoHistoryImages.length > 0,
|
||||||
|
hasCanvasSelection: Boolean(contextPayload?.selectedNodeIds?.length),
|
||||||
|
targetResolutionMode: requestTargetResolution?.resolutionMode,
|
||||||
|
executionMode: smartAutoExecution?.mode,
|
||||||
|
friendlyMessage: smartAutoFriendlyMessage,
|
||||||
|
});
|
||||||
|
|
||||||
if (hasCurrentImageInput || hasHistoryImageInput || latestUserText.length > 0) {
|
if (clarificationBypass.shouldBypass) {
|
||||||
const canForceByIntent = /改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格/.test(
|
smartAutoNeedsClarification = false;
|
||||||
latestUserText
|
smartAutoClarificationQuestion = undefined;
|
||||||
);
|
smartAutoFriendlyMessage = clarificationBypass.nextFriendlyMessage;
|
||||||
const hasForceTargetHint =
|
console.log("[Smart Auto Mode] ✅ Force direct execution (skip clarification)");
|
||||||
Boolean(contextPayload?.selectedNodeIds?.length) ||
|
|
||||||
requestTargetResolution?.resolutionMode === "text_reference" ||
|
|
||||||
/第[一二三四五六七八九十\d]+张|左|右|上|下|中间|选中/.test(latestUserText);
|
|
||||||
const canForceExecution =
|
|
||||||
(smartAutoExecution?.mode === "edit_image" ||
|
|
||||||
smartAutoExecution?.mode === "generate_image" ||
|
|
||||||
smartAutoExecution?.mode === "generate_video") &&
|
|
||||||
canForceByIntent &&
|
|
||||||
hasForceTargetHint;
|
|
||||||
|
|
||||||
if (canForceExecution) {
|
|
||||||
smartAutoNeedsClarification = false;
|
|
||||||
smartAutoClarificationQuestion = undefined;
|
|
||||||
if (
|
|
||||||
!smartAutoFriendlyMessage ||
|
|
||||||
/[??]$/.test(smartAutoFriendlyMessage.trim()) ||
|
|
||||||
/哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage)
|
|
||||||
) {
|
|
||||||
smartAutoFriendlyMessage = "收到,我将按你的当前需求直接执行。";
|
|
||||||
}
|
|
||||||
console.log("[Smart Auto Mode] ✅ Force direct execution (skip clarification)");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (shouldRunAgentConversation && smartAutoNeedsClarification) {
|
if (shouldRunAgentConversation && smartAutoNeedsClarification) {
|
||||||
const clarificationMessage =
|
const clarificationMessage = resolveClarificationMessage({
|
||||||
smartAutoClarificationQuestion ||
|
clarificationQuestion: smartAutoClarificationQuestion,
|
||||||
smartAutoFriendlyMessage ||
|
friendlyMessage: smartAutoFriendlyMessage,
|
||||||
"我先确认一下,你希望我优先处理哪一部分?";
|
});
|
||||||
|
|
||||||
const clarificationStream = createUIMessageStream({
|
const clarificationStream = createUIMessageStream({
|
||||||
execute: async ({ writer }) => {
|
execute: async ({ writer }) => {
|
||||||
@ -4738,22 +4729,25 @@ export async function POST(request: Request) {
|
|||||||
],
|
],
|
||||||
});
|
});
|
||||||
|
|
||||||
if (shouldAskGridConfirmation && pendingGridConfirmation) {
|
try {
|
||||||
try {
|
await persistClarificationTurn({
|
||||||
await persistGridConfirmationTurn({
|
userId: session.user.id,
|
||||||
userId: session.user.id,
|
chatId: id,
|
||||||
chatId: id,
|
projectId: contextPayload?.projectId || null,
|
||||||
projectId: contextPayload?.projectId || null,
|
messageId,
|
||||||
messageId,
|
text: requestTextContent,
|
||||||
text: requestTextContent,
|
clarificationMessage,
|
||||||
pendingGridConfirmation,
|
currentIntent: smartAutoDetectedIntent || null,
|
||||||
});
|
pendingGridConfirmation:
|
||||||
} catch (gridConfirmationPersistError) {
|
shouldAskGridConfirmation && pendingGridConfirmation
|
||||||
console.warn(
|
? pendingGridConfirmation
|
||||||
"[Chat API] Failed to persist pending grid confirmation state:",
|
: null,
|
||||||
gridConfirmationPersistError
|
});
|
||||||
);
|
} catch (clarificationPersistError) {
|
||||||
}
|
console.warn(
|
||||||
|
"[Chat API] Failed to persist clarification state:",
|
||||||
|
clarificationPersistError
|
||||||
|
);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@ -4770,40 +4764,23 @@ export async function POST(request: Request) {
|
|||||||
const workflowReplyStream = createUIMessageStream({
|
const workflowReplyStream = createUIMessageStream({
|
||||||
execute: async ({ writer }) => {
|
execute: async ({ writer }) => {
|
||||||
const messageId = generateUUID();
|
const messageId = generateUUID();
|
||||||
let replyText =
|
const workflowFollowupResult = await handleStoryboardWorkflowFollowup({
|
||||||
smartAutoFriendlyMessage ||
|
action: smartAutoWorkflowDecision.action as "revision_requested" | "cancel_requested",
|
||||||
(smartAutoWorkflowDecision?.action === "cancel_requested"
|
workflowId: smartAutoWorkflowDecision.workflowId,
|
||||||
? "已收到取消指令,当前分镜流程不会继续自动执行。"
|
text: requestTextContent,
|
||||||
: "收到修改意见,我会先停在草案阶段,等待你继续调整。");
|
friendlyMessage: smartAutoFriendlyMessage,
|
||||||
|
onProgress: (event) => {
|
||||||
if (
|
safeWriteStreamChunk(
|
||||||
smartAutoWorkflowDecision?.action === "revision_requested" &&
|
writer,
|
||||||
smartAutoWorkflowDecision.workflowId
|
{
|
||||||
) {
|
type: "data-videoStoryboard",
|
||||||
const revisionTaskDetail = await getVideoStoryboardTaskDetail(
|
data: JSON.stringify(event),
|
||||||
smartAutoWorkflowDecision.workflowId
|
},
|
||||||
).catch(() => null);
|
`storyboard-revision:${event.stage}`,
|
||||||
const revisionResult = await reviseStoryboardTaskByCommand({
|
);
|
||||||
taskId: smartAutoWorkflowDecision.workflowId,
|
},
|
||||||
command: revisionTaskDetail
|
});
|
||||||
? await resolveStoryboardRevisionCommand({
|
const replyText = workflowFollowupResult.replyText;
|
||||||
text: requestTextContent,
|
|
||||||
detail: revisionTaskDetail,
|
|
||||||
})
|
|
||||||
: null,
|
|
||||||
onProgress: (event) => {
|
|
||||||
safeWriteStreamChunk(
|
|
||||||
writer,
|
|
||||||
{
|
|
||||||
type: "data-videoStoryboard",
|
|
||||||
data: JSON.stringify(event),
|
|
||||||
},
|
|
||||||
`storyboard-revision:${event.stage}`,
|
|
||||||
);
|
|
||||||
},
|
|
||||||
});
|
|
||||||
replyText = revisionResult.message || replyText;
|
|
||||||
}
|
|
||||||
|
|
||||||
writer.write({
|
writer.write({
|
||||||
type: "data-appendMessage",
|
type: "data-appendMessage",
|
||||||
@ -4828,20 +4805,43 @@ export async function POST(request: Request) {
|
|||||||
temporaryVideoUrl: null,
|
temporaryVideoUrl: null,
|
||||||
permanentVideoUrl: null,
|
permanentVideoUrl: null,
|
||||||
archivedAt: null,
|
archivedAt: null,
|
||||||
videoMetadata:
|
videoMetadata: workflowFollowupResult.videoMetadata,
|
||||||
smartAutoWorkflowDecision?.workflowId
|
|
||||||
? ({
|
|
||||||
storyboard: {
|
|
||||||
enabled: true,
|
|
||||||
taskId: smartAutoWorkflowDecision.workflowId,
|
|
||||||
action: smartAutoWorkflowDecision.action,
|
|
||||||
},
|
|
||||||
updatedAt: Date.now(),
|
|
||||||
} as any)
|
|
||||||
: null,
|
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (
|
||||||
|
smartAutoWorkflowDecision?.action === "revision_requested" &&
|
||||||
|
smartAutoWorkflowDecision.workflowId
|
||||||
|
) {
|
||||||
|
await persistStoryboardRevisionTurn({
|
||||||
|
userId: session.user.id,
|
||||||
|
chatId: id,
|
||||||
|
projectId: contextPayload?.projectId || null,
|
||||||
|
messageId,
|
||||||
|
text: requestTextContent,
|
||||||
|
taskId: smartAutoWorkflowDecision.workflowId,
|
||||||
|
status: workflowFollowupResult.nextWorkflowStatus,
|
||||||
|
});
|
||||||
|
} else if (smartAutoWorkflowDecision?.action === "cancel_requested") {
|
||||||
|
await persistWorkflowCancelledTurn({
|
||||||
|
userId: session.user.id,
|
||||||
|
chatId: id,
|
||||||
|
projectId: contextPayload?.projectId || null,
|
||||||
|
messageId,
|
||||||
|
text: requestTextContent,
|
||||||
|
workflowType: "storyboard_video",
|
||||||
|
workflowId: smartAutoWorkflowDecision?.workflowId,
|
||||||
|
status: workflowFollowupResult.nextWorkflowStatus,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch (workflowPersistError) {
|
||||||
|
console.warn(
|
||||||
|
"[Chat API] Failed to persist storyboard workflow interruption state:",
|
||||||
|
workflowPersistError
|
||||||
|
);
|
||||||
|
}
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -4858,7 +4858,8 @@ export async function POST(request: Request) {
|
|||||||
const confirmedStoryboardStream = createUIMessageStream({
|
const confirmedStoryboardStream = createUIMessageStream({
|
||||||
execute: async ({ writer }) => {
|
execute: async ({ writer }) => {
|
||||||
const acknowledgeMessageId = generateUUID();
|
const acknowledgeMessageId = generateUUID();
|
||||||
const acknowledgeMessage = "已收到确认,开始按分镜逐段生成视频,请稍候。";
|
const { acknowledgeMessage, acknowledgeMetadata } =
|
||||||
|
buildStoryboardConfirmedAck(confirmedTaskId);
|
||||||
|
|
||||||
writer.write({
|
writer.write({
|
||||||
type: "data-appendMessage",
|
type: "data-appendMessage",
|
||||||
@ -4883,14 +4884,7 @@ export async function POST(request: Request) {
|
|||||||
temporaryVideoUrl: null,
|
temporaryVideoUrl: null,
|
||||||
permanentVideoUrl: null,
|
permanentVideoUrl: null,
|
||||||
archivedAt: null,
|
archivedAt: null,
|
||||||
videoMetadata: {
|
videoMetadata: acknowledgeMetadata,
|
||||||
storyboard: {
|
|
||||||
enabled: true,
|
|
||||||
taskId: confirmedTaskId,
|
|
||||||
status: "confirmed",
|
|
||||||
},
|
|
||||||
updatedAt: Date.now(),
|
|
||||||
} as any,
|
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
@ -4902,6 +4896,8 @@ export async function POST(request: Request) {
|
|||||||
projectId: contextPayload?.projectId || null,
|
projectId: contextPayload?.projectId || null,
|
||||||
messageId: acknowledgeMessageId,
|
messageId: acknowledgeMessageId,
|
||||||
text: requestTextContent,
|
text: requestTextContent,
|
||||||
|
taskId: confirmedTaskId,
|
||||||
|
status: "confirmed",
|
||||||
});
|
});
|
||||||
} catch (storyboardConfirmMemoryError) {
|
} catch (storyboardConfirmMemoryError) {
|
||||||
console.warn(
|
console.warn(
|
||||||
@ -4910,19 +4906,37 @@ export async function POST(request: Request) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const executionResult = await executeConfirmedStoryboardTask({
|
try {
|
||||||
taskId: confirmedTaskId,
|
await persistStoryboardExecutionStartedTurn({
|
||||||
onProgress: (event) => {
|
userId: session.user.id,
|
||||||
safeWriteStreamChunk(
|
chatId: id,
|
||||||
writer,
|
projectId: contextPayload?.projectId || null,
|
||||||
{
|
messageId: acknowledgeMessageId,
|
||||||
type: "data-videoStoryboard",
|
text: requestTextContent,
|
||||||
data: JSON.stringify(event),
|
taskId: confirmedTaskId,
|
||||||
},
|
status: "running",
|
||||||
`storyboard-confirmed:${event.stage}`,
|
});
|
||||||
);
|
} catch (storyboardExecutionStartError) {
|
||||||
},
|
console.warn(
|
||||||
});
|
"[Chat API] Failed to persist storyboard execution start state:",
|
||||||
|
storyboardExecutionStartError
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const { executionResult, firstVideoUrl, finalVideoMetadata } =
|
||||||
|
await executeStoryboardConfirmedWorkflow({
|
||||||
|
taskId: confirmedTaskId,
|
||||||
|
onProgress: (event) => {
|
||||||
|
safeWriteStreamChunk(
|
||||||
|
writer,
|
||||||
|
{
|
||||||
|
type: "data-videoStoryboard",
|
||||||
|
data: JSON.stringify(event),
|
||||||
|
},
|
||||||
|
`storyboard-confirmed:${event.stage}`,
|
||||||
|
);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
const finalMessageId = generateUUID();
|
const finalMessageId = generateUUID();
|
||||||
writer.write({
|
writer.write({
|
||||||
@ -4935,7 +4949,6 @@ export async function POST(request: Request) {
|
|||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
const firstVideoUrl = executionResult.videoUrls[0] || "";
|
|
||||||
await saveMessages({
|
await saveMessages({
|
||||||
messages: [
|
messages: [
|
||||||
{
|
{
|
||||||
@ -4949,16 +4962,7 @@ export async function POST(request: Request) {
|
|||||||
temporaryVideoUrl: firstVideoUrl || null,
|
temporaryVideoUrl: firstVideoUrl || null,
|
||||||
permanentVideoUrl: null,
|
permanentVideoUrl: null,
|
||||||
archivedAt: null,
|
archivedAt: null,
|
||||||
videoMetadata: {
|
videoMetadata: finalVideoMetadata,
|
||||||
storyboard: {
|
|
||||||
enabled: true,
|
|
||||||
taskId: executionResult.taskId,
|
|
||||||
successCount: executionResult.successCount,
|
|
||||||
failedCount: executionResult.failedCount,
|
|
||||||
},
|
|
||||||
status: executionResult.status,
|
|
||||||
updatedAt: Date.now(),
|
|
||||||
} as any,
|
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
@ -4973,6 +4977,26 @@ export async function POST(request: Request) {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await persistStoryboardExecutionCompletedTurn({
|
||||||
|
userId: session.user.id,
|
||||||
|
chatId: id,
|
||||||
|
projectId: contextPayload?.projectId || null,
|
||||||
|
messageId: finalMessageId,
|
||||||
|
text: requestTextContent,
|
||||||
|
taskId: executionResult.taskId,
|
||||||
|
status: executionResult.status,
|
||||||
|
successCount: executionResult.successCount,
|
||||||
|
failedCount: executionResult.failedCount,
|
||||||
|
firstVideoUrl: firstVideoUrl || undefined,
|
||||||
|
});
|
||||||
|
} catch (storyboardExecutionCompletedError) {
|
||||||
|
console.warn(
|
||||||
|
"[Chat API] Failed to persist storyboard execution completion state:",
|
||||||
|
storyboardExecutionCompletedError
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
if (firstVideoUrl) {
|
if (firstVideoUrl) {
|
||||||
await consumeGeneratedVideoPoints({
|
await consumeGeneratedVideoPoints({
|
||||||
userId: session.user.id,
|
userId: session.user.id,
|
||||||
@ -5263,6 +5287,9 @@ export async function POST(request: Request) {
|
|||||||
sourceType:
|
sourceType:
|
||||||
sourceImageUrls.length > 0 ? "reference_generate" : "generated",
|
sourceImageUrls.length > 0 ? "reference_generate" : "generated",
|
||||||
projectId: contextPayload?.projectId || null,
|
projectId: contextPayload?.projectId || null,
|
||||||
|
userPrompt: requestTextContent,
|
||||||
|
optimizedPrompt: slotPrompt || undefined,
|
||||||
|
requestedImageCount: 1,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
@ -5372,13 +5399,18 @@ export async function POST(request: Request) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await persistVideoWorkflowTurn({
|
await persistVideoGenerationCompletedTurn({
|
||||||
userId: session.user.id,
|
userId: session.user.id,
|
||||||
chatId: id,
|
chatId: id,
|
||||||
projectId: contextPayload?.projectId || null,
|
projectId: contextPayload?.projectId || null,
|
||||||
lastMessageId: assistantMessageId,
|
messageId: assistantMessageId,
|
||||||
text: requestTextContent,
|
text: requestTextContent,
|
||||||
|
source: "agent_preset_direct_video",
|
||||||
|
providerType: "direct-video",
|
||||||
|
modelRef: result.modelRef || presetModelId,
|
||||||
|
mode,
|
||||||
status: "completed",
|
status: "completed",
|
||||||
|
temporaryVideoUrl: videoUrl,
|
||||||
});
|
});
|
||||||
} catch (memoryError) {
|
} catch (memoryError) {
|
||||||
console.warn(
|
console.warn(
|
||||||
@ -5458,6 +5490,27 @@ export async function POST(request: Request) {
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
|
if (selectedAgentPreset.type === "video") {
|
||||||
|
try {
|
||||||
|
await persistVideoGenerationFailedTurn({
|
||||||
|
userId: session.user.id,
|
||||||
|
chatId: id,
|
||||||
|
projectId: contextPayload?.projectId || null,
|
||||||
|
messageId: assistantMessageId,
|
||||||
|
text: requestTextContent,
|
||||||
|
source: "agent_preset_direct_video",
|
||||||
|
providerType: "direct-video",
|
||||||
|
modelRef: presetModelId,
|
||||||
|
status: "failed",
|
||||||
|
error: slotError instanceof Error ? slotError.message : String(slotError),
|
||||||
|
});
|
||||||
|
} catch (videoMemoryError) {
|
||||||
|
console.warn(
|
||||||
|
"[Chat API] Failed to persist agent preset video failure state:",
|
||||||
|
videoMemoryError,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
writeAssistantTextChunk(
|
writeAssistantTextChunk(
|
||||||
writer,
|
writer,
|
||||||
errorMessage,
|
errorMessage,
|
||||||
@ -5624,6 +5677,18 @@ export async function POST(request: Request) {
|
|||||||
? "reference_generate"
|
? "reference_generate"
|
||||||
: "generated",
|
: "generated",
|
||||||
projectId: contextPayload?.projectId || null,
|
projectId: contextPayload?.projectId || null,
|
||||||
|
userPrompt: requestTextContent,
|
||||||
|
optimizedPrompt: generationPrompt,
|
||||||
|
requestedImageCount:
|
||||||
|
effectiveImageParams?.imageCount ||
|
||||||
|
defaultImageQualityParams.imageCount ||
|
||||||
|
1,
|
||||||
|
aspectRatio:
|
||||||
|
effectiveImageParams?.aspectRatio ||
|
||||||
|
defaultImageQualityParams.aspectRatio,
|
||||||
|
imageSize:
|
||||||
|
effectiveImageParams?.imageSize ||
|
||||||
|
defaultImageQualityParams.imageSize,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
@ -5804,13 +5869,18 @@ export async function POST(request: Request) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await persistVideoWorkflowTurn({
|
await persistVideoGenerationCompletedTurn({
|
||||||
userId: session.user.id,
|
userId: session.user.id,
|
||||||
chatId: id,
|
chatId: id,
|
||||||
projectId: contextPayload?.projectId || null,
|
projectId: contextPayload?.projectId || null,
|
||||||
lastMessageId: assistantMessageId,
|
messageId: assistantMessageId,
|
||||||
text: requestTextContent,
|
text: requestTextContent,
|
||||||
|
source: "agent_direct_generate_video",
|
||||||
|
providerType: "direct-video",
|
||||||
|
modelRef: result.modelRef || finalModelId,
|
||||||
|
mode,
|
||||||
status: "completed",
|
status: "completed",
|
||||||
|
temporaryVideoUrl: videoUrl,
|
||||||
});
|
});
|
||||||
} catch (memoryError) {
|
} catch (memoryError) {
|
||||||
console.warn(
|
console.warn(
|
||||||
@ -5878,6 +5948,28 @@ export async function POST(request: Request) {
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
|
try {
|
||||||
|
await persistVideoGenerationFailedTurn({
|
||||||
|
userId: session.user.id,
|
||||||
|
chatId: id,
|
||||||
|
projectId: contextPayload?.projectId || null,
|
||||||
|
messageId: assistantMessageId,
|
||||||
|
text: requestTextContent,
|
||||||
|
source: "agent_direct_generate_video",
|
||||||
|
providerType: "direct-video",
|
||||||
|
modelRef: finalModelId,
|
||||||
|
status: "failed",
|
||||||
|
error:
|
||||||
|
directToolError instanceof Error
|
||||||
|
? directToolError.message
|
||||||
|
: String(directToolError),
|
||||||
|
});
|
||||||
|
} catch (videoMemoryError) {
|
||||||
|
console.warn(
|
||||||
|
"[Chat API] Failed to persist direct video failure state:",
|
||||||
|
videoMemoryError,
|
||||||
|
);
|
||||||
|
}
|
||||||
writeAssistantTextChunk(
|
writeAssistantTextChunk(
|
||||||
writer,
|
writer,
|
||||||
errorMessage,
|
errorMessage,
|
||||||
@ -6131,7 +6223,9 @@ export async function POST(request: Request) {
|
|||||||
lastResolvedTargetImageUrls: resultImageUrls,
|
lastResolvedTargetImageUrls: resultImageUrls,
|
||||||
lastMessageId: assistantMessageId,
|
lastMessageId: assistantMessageId,
|
||||||
lastUserPrompt: requestTextContent,
|
lastUserPrompt: requestTextContent,
|
||||||
|
lastOptimizedPrompt: smartAutoOptimizedPrompt || undefined,
|
||||||
lastRequestedImageCount: resultImageUrls?.length || 1,
|
lastRequestedImageCount: resultImageUrls?.length || 1,
|
||||||
|
lastImageSize: resultImageSize || undefined,
|
||||||
lastSourceImageUrls: directAgentToolTargetImageUrls,
|
lastSourceImageUrls: directAgentToolTargetImageUrls,
|
||||||
lastResultImageUrls: resultImageUrls,
|
lastResultImageUrls: resultImageUrls,
|
||||||
});
|
});
|
||||||
@ -6666,6 +6760,10 @@ export async function POST(request: Request) {
|
|||||||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||||||
sourceType: requestResolvedMediaInput?.hasImage ? "generated" : "generated",
|
sourceType: requestResolvedMediaInput?.hasImage ? "generated" : "generated",
|
||||||
projectId: contextPayload?.projectId || null,
|
projectId: contextPayload?.projectId || null,
|
||||||
|
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
|
||||||
|
requestedImageCount: effectiveImageParams?.imageCount,
|
||||||
|
aspectRatio: effectiveImageParams?.aspectRatio,
|
||||||
|
imageSize: effectiveImageParams?.imageSize,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
} catch (saveError) {
|
} catch (saveError) {
|
||||||
@ -6801,6 +6899,10 @@ export async function POST(request: Request) {
|
|||||||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||||||
sourceType: "generated",
|
sourceType: "generated",
|
||||||
projectId: contextPayload?.projectId || null,
|
projectId: contextPayload?.projectId || null,
|
||||||
|
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
|
||||||
|
requestedImageCount: effectiveImageParams?.imageCount,
|
||||||
|
aspectRatio: effectiveImageParams?.aspectRatio,
|
||||||
|
imageSize: effectiveImageParams?.imageSize,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
} catch (saveError) {
|
} catch (saveError) {
|
||||||
@ -7201,6 +7303,10 @@ export async function POST(request: Request) {
|
|||||||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||||||
sourceType: "generated",
|
sourceType: "generated",
|
||||||
projectId: contextPayload?.projectId || null,
|
projectId: contextPayload?.projectId || null,
|
||||||
|
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
|
||||||
|
requestedImageCount: effectiveImageParams?.imageCount,
|
||||||
|
aspectRatio: effectiveImageParams?.aspectRatio,
|
||||||
|
imageSize: effectiveImageParams?.imageSize,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
} catch (saveError) {
|
} catch (saveError) {
|
||||||
@ -7585,6 +7691,10 @@ export async function POST(request: Request) {
|
|||||||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||||||
sourceType: "generated",
|
sourceType: "generated",
|
||||||
projectId: contextPayload?.projectId || null,
|
projectId: contextPayload?.projectId || null,
|
||||||
|
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
|
||||||
|
requestedImageCount: effectiveImageParams?.imageCount,
|
||||||
|
aspectRatio: effectiveImageParams?.aspectRatio,
|
||||||
|
imageSize: effectiveImageParams?.imageSize,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
} catch (saveError) {
|
} catch (saveError) {
|
||||||
@ -7874,6 +7984,36 @@ export async function POST(request: Request) {
|
|||||||
storyboardMemoryPersistError
|
storyboardMemoryPersistError
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
await persistVideoGenerationCompletedTurn({
|
||||||
|
userId: session.user.id,
|
||||||
|
chatId: id,
|
||||||
|
projectId: contextPayload?.projectId || null,
|
||||||
|
messageId,
|
||||||
|
text: requestTextContent,
|
||||||
|
source: "native_video_provider",
|
||||||
|
providerType: currentVideoProvider,
|
||||||
|
modelRef: finalModelId,
|
||||||
|
mode:
|
||||||
|
typeof effectiveVideoParams?.mode === "string"
|
||||||
|
? effectiveVideoParams.mode
|
||||||
|
: undefined,
|
||||||
|
taskId: storyboardTaskId || undefined,
|
||||||
|
status:
|
||||||
|
typeof (videoMetadata as any)?.status === "string"
|
||||||
|
? (videoMetadata as any).status
|
||||||
|
: tempVideoUrl
|
||||||
|
? "completed"
|
||||||
|
: undefined,
|
||||||
|
temporaryVideoUrl: tempVideoUrl || undefined,
|
||||||
|
});
|
||||||
|
} catch (videoWorkflowPersistError) {
|
||||||
|
console.warn(
|
||||||
|
"[Chat API] Failed to persist native video workflow state:",
|
||||||
|
videoWorkflowPersistError,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
const shouldConsumeVideoPoints =
|
const shouldConsumeVideoPoints =
|
||||||
tempVideoUrl &&
|
tempVideoUrl &&
|
||||||
@ -7903,15 +8043,56 @@ export async function POST(request: Request) {
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`[Chat API] ${currentVideoProvider} video generation error:`, error);
|
console.error(`[Chat API] ${currentVideoProvider} video generation error:`, error);
|
||||||
const errorMessageId = generateUUID();
|
const errorMessageId = generateUUID();
|
||||||
|
const errorText = "视频生成失败,请稍后重试";
|
||||||
writer.write({
|
writer.write({
|
||||||
type: "data-appendMessage",
|
type: "data-appendMessage",
|
||||||
data: JSON.stringify({
|
data: JSON.stringify({
|
||||||
id: errorMessageId,
|
id: errorMessageId,
|
||||||
role: "assistant",
|
role: "assistant",
|
||||||
parts: [{ type: "text", text: "视频生成失败,请稍后重试" }],
|
parts: [{ type: "text", text: errorText }],
|
||||||
attachments: [],
|
attachments: [],
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
try {
|
||||||
|
await saveMessages({
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
id: errorMessageId,
|
||||||
|
chatId: id,
|
||||||
|
role: "assistant",
|
||||||
|
parts: [{ type: "text", text: errorText }],
|
||||||
|
attachments: [],
|
||||||
|
createdAt: new Date(),
|
||||||
|
videoStorageStatus: null,
|
||||||
|
temporaryVideoUrl: null,
|
||||||
|
permanentVideoUrl: null,
|
||||||
|
archivedAt: null,
|
||||||
|
videoMetadata: null,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
await persistVideoGenerationFailedTurn({
|
||||||
|
userId: session.user.id,
|
||||||
|
chatId: id,
|
||||||
|
projectId: contextPayload?.projectId || null,
|
||||||
|
messageId: errorMessageId,
|
||||||
|
text: requestTextContent,
|
||||||
|
source: "native_video_provider",
|
||||||
|
providerType: currentVideoProvider,
|
||||||
|
modelRef: finalModelId,
|
||||||
|
mode:
|
||||||
|
typeof effectiveVideoParams?.mode === "string"
|
||||||
|
? effectiveVideoParams.mode
|
||||||
|
: undefined,
|
||||||
|
status: "failed",
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
} catch (videoFailurePersistError) {
|
||||||
|
console.warn(
|
||||||
|
"[Chat API] Failed to persist native video failure state:",
|
||||||
|
videoFailurePersistError,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@ -8325,6 +8506,10 @@ export async function POST(request: Request) {
|
|||||||
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
|
||||||
sourceType: "generated",
|
sourceType: "generated",
|
||||||
projectId: contextPayload?.projectId || null,
|
projectId: contextPayload?.projectId || null,
|
||||||
|
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
|
||||||
|
requestedImageCount: effectiveImageParams?.imageCount,
|
||||||
|
aspectRatio: effectiveImageParams?.aspectRatio,
|
||||||
|
imageSize: effectiveImageParams?.imageSize,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
@ -8674,6 +8859,25 @@ export async function POST(request: Request) {
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
|
try {
|
||||||
|
await persistVideoGenerationStartedTurn({
|
||||||
|
userId: session.user.id,
|
||||||
|
chatId: id,
|
||||||
|
projectId: contextPayload?.projectId || null,
|
||||||
|
messageId: videoMessageId,
|
||||||
|
text: requestTextContent,
|
||||||
|
source: "stream_video_provider",
|
||||||
|
providerType: String(providerData.type || "stream"),
|
||||||
|
modelRef: finalVideoModelData.id || finalModelId,
|
||||||
|
mode: videoOptions.mode,
|
||||||
|
status: "running",
|
||||||
|
});
|
||||||
|
} catch (videoWorkflowStartError) {
|
||||||
|
console.warn(
|
||||||
|
"[Chat API] Failed to persist stream video start state:",
|
||||||
|
videoWorkflowStartError,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
dataStream.write({
|
dataStream.write({
|
||||||
type: "data-appendMessage",
|
type: "data-appendMessage",
|
||||||
@ -8749,6 +8953,30 @@ export async function POST(request: Request) {
|
|||||||
archivedAt: null,
|
archivedAt: null,
|
||||||
videoMetadata: successMetadata as any,
|
videoMetadata: successMetadata as any,
|
||||||
});
|
});
|
||||||
|
try {
|
||||||
|
await persistVideoGenerationCompletedTurn({
|
||||||
|
userId: session.user.id,
|
||||||
|
chatId: id,
|
||||||
|
projectId: contextPayload?.projectId || null,
|
||||||
|
messageId: videoMessageId,
|
||||||
|
text: requestTextContent,
|
||||||
|
source: "stream_video_provider",
|
||||||
|
providerType: String(providerData.type || "stream"),
|
||||||
|
modelRef: finalVideoModelData.id || finalModelId,
|
||||||
|
mode: videoOptions.mode,
|
||||||
|
taskId:
|
||||||
|
typeof successMetadata.taskId === "string" && successMetadata.taskId
|
||||||
|
? successMetadata.taskId
|
||||||
|
: undefined,
|
||||||
|
status: "completed",
|
||||||
|
temporaryVideoUrl: temporaryVideoUrl || undefined,
|
||||||
|
});
|
||||||
|
} catch (videoWorkflowCompletedError) {
|
||||||
|
console.warn(
|
||||||
|
"[Chat API] Failed to persist stream video completion state:",
|
||||||
|
videoWorkflowCompletedError,
|
||||||
|
);
|
||||||
|
}
|
||||||
await consumeGeneratedVideoPoints({
|
await consumeGeneratedVideoPoints({
|
||||||
userId: session.user.id,
|
userId: session.user.id,
|
||||||
chatId: id,
|
chatId: id,
|
||||||
@ -8807,6 +9035,29 @@ export async function POST(request: Request) {
|
|||||||
archivedAt: null,
|
archivedAt: null,
|
||||||
videoMetadata: failedMetadata as any,
|
videoMetadata: failedMetadata as any,
|
||||||
});
|
});
|
||||||
|
try {
|
||||||
|
await persistVideoGenerationFailedTurn({
|
||||||
|
userId: session.user.id,
|
||||||
|
chatId: id,
|
||||||
|
projectId: contextPayload?.projectId || null,
|
||||||
|
messageId: videoMessageId,
|
||||||
|
text: requestTextContent,
|
||||||
|
source: "stream_video_provider",
|
||||||
|
providerType: "stream",
|
||||||
|
modelRef: finalModelId,
|
||||||
|
mode:
|
||||||
|
typeof effectiveVideoParams?.mode === "string"
|
||||||
|
? effectiveVideoParams.mode
|
||||||
|
: undefined,
|
||||||
|
status: "failed",
|
||||||
|
error: error?.message || "未知错误",
|
||||||
|
});
|
||||||
|
} catch (videoWorkflowFailedError) {
|
||||||
|
console.warn(
|
||||||
|
"[Chat API] Failed to persist stream video failure state:",
|
||||||
|
videoWorkflowFailedError,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
dataStream.write({
|
dataStream.write({
|
||||||
type: "data-appendMessage",
|
type: "data-appendMessage",
|
||||||
@ -8846,6 +9097,29 @@ export async function POST(request: Request) {
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
|
try {
|
||||||
|
await persistVideoGenerationFailedTurn({
|
||||||
|
userId: session.user.id,
|
||||||
|
chatId: id,
|
||||||
|
projectId: contextPayload?.projectId || null,
|
||||||
|
messageId: fallbackMessageId,
|
||||||
|
text: requestTextContent,
|
||||||
|
source: "stream_video_provider",
|
||||||
|
providerType: "stream",
|
||||||
|
modelRef: finalModelId,
|
||||||
|
mode:
|
||||||
|
typeof effectiveVideoParams?.mode === "string"
|
||||||
|
? effectiveVideoParams.mode
|
||||||
|
: undefined,
|
||||||
|
status: "failed",
|
||||||
|
error: error?.message || "未知错误",
|
||||||
|
});
|
||||||
|
} catch (videoWorkflowFailedError) {
|
||||||
|
console.warn(
|
||||||
|
"[Chat API] Failed to persist fallback stream video failure state:",
|
||||||
|
videoWorkflowFailedError,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return;
|
return;
|
||||||
@ -8971,6 +9245,25 @@ export async function POST(request: Request) {
|
|||||||
archivedAt: null,
|
archivedAt: null,
|
||||||
videoMetadata: genericVideoMetadata as any,
|
videoMetadata: genericVideoMetadata as any,
|
||||||
});
|
});
|
||||||
|
try {
|
||||||
|
await persistVideoGenerationCompletedTurn({
|
||||||
|
userId: session.user.id,
|
||||||
|
chatId: id,
|
||||||
|
projectId: contextPayload?.projectId || null,
|
||||||
|
messageId: msg.id,
|
||||||
|
text: requestTextContent,
|
||||||
|
source: "chat_stream_output",
|
||||||
|
providerType: "stream",
|
||||||
|
modelRef: actualModelId,
|
||||||
|
status: "completed",
|
||||||
|
temporaryVideoUrl: tempVideoUrl,
|
||||||
|
});
|
||||||
|
} catch (videoWorkflowCompletedError) {
|
||||||
|
console.warn(
|
||||||
|
"[Chat API] Failed to persist streamed assistant video state:",
|
||||||
|
videoWorkflowCompletedError,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
scheduleVideoArchive({
|
scheduleVideoArchive({
|
||||||
chatId: id,
|
chatId: id,
|
||||||
|
|||||||
@ -325,6 +325,10 @@ export function Chat({
|
|||||||
Array<{ id: string; at: number; payload: Record<string, unknown> }>
|
Array<{ id: string; at: number; payload: Record<string, unknown> }>
|
||||||
>([]);
|
>([]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
setProtocolDebugEvents([]);
|
||||||
|
}, [id]);
|
||||||
|
|
||||||
const { visibilityType } = useChatVisibility({
|
const { visibilityType } = useChatVisibility({
|
||||||
chatId: id,
|
chatId: id,
|
||||||
initialVisibilityType,
|
initialVisibilityType,
|
||||||
@ -2094,7 +2098,11 @@ export function Chat({
|
|||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
<ProtocolDebugPanel enabled={protocolDebugEnabled} events={protocolDebugEvents} />
|
<ProtocolDebugPanel
|
||||||
|
enabled={protocolDebugEnabled}
|
||||||
|
events={protocolDebugEvents}
|
||||||
|
onClear={() => setProtocolDebugEvents([])}
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<Artifact
|
<Artifact
|
||||||
|
|||||||
@ -1,5 +1,7 @@
|
|||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
|
import { useMemo } from "react";
|
||||||
|
|
||||||
type ProtocolDebugEvent = {
|
type ProtocolDebugEvent = {
|
||||||
id: string;
|
id: string;
|
||||||
at: number;
|
at: number;
|
||||||
@ -9,12 +11,28 @@ type ProtocolDebugEvent = {
|
|||||||
export function ProtocolDebugPanel({
|
export function ProtocolDebugPanel({
|
||||||
enabled,
|
enabled,
|
||||||
events,
|
events,
|
||||||
|
onClear,
|
||||||
}: {
|
}: {
|
||||||
enabled: boolean;
|
enabled: boolean;
|
||||||
events: ProtocolDebugEvent[];
|
events: ProtocolDebugEvent[];
|
||||||
|
onClear?: () => void;
|
||||||
}) {
|
}) {
|
||||||
if (!enabled) return null;
|
if (!enabled) return null;
|
||||||
|
|
||||||
|
const groupedEvents = useMemo(() => {
|
||||||
|
const groups = new Map<string, ProtocolDebugEvent[]>();
|
||||||
|
for (const event of events) {
|
||||||
|
const key = String(event.payload.phase || event.payload.type || "unknown");
|
||||||
|
const list = groups.get(key) || [];
|
||||||
|
list.push(event);
|
||||||
|
groups.set(key, list);
|
||||||
|
}
|
||||||
|
return Array.from(groups.entries()).map(([phase, items]) => ({
|
||||||
|
phase,
|
||||||
|
items,
|
||||||
|
}));
|
||||||
|
}, [events]);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="mx-auto mt-2 w-full max-w-4xl px-2 md:px-4">
|
<div className="mx-auto mt-2 w-full max-w-4xl px-2 md:px-4">
|
||||||
<details className="rounded-xl border border-border/60 bg-muted/30 p-3 text-xs">
|
<details className="rounded-xl border border-border/60 bg-muted/30 p-3 text-xs">
|
||||||
@ -22,25 +40,58 @@ export function ProtocolDebugPanel({
|
|||||||
Protocol Debug
|
Protocol Debug
|
||||||
</summary>
|
</summary>
|
||||||
<div className="mt-3 space-y-2">
|
<div className="mt-3 space-y-2">
|
||||||
|
<div className="flex flex-wrap items-center justify-between gap-2 rounded-lg border border-border/60 bg-background px-3 py-2">
|
||||||
|
<div className="flex flex-wrap items-center gap-2">
|
||||||
|
<span className="rounded-md bg-muted px-2 py-1 text-[11px] text-muted-foreground">
|
||||||
|
总事件 {events.length}
|
||||||
|
</span>
|
||||||
|
<span className="rounded-md bg-muted px-2 py-1 text-[11px] text-muted-foreground">
|
||||||
|
阶段 {groupedEvents.length}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="rounded-md border border-border px-2 py-1 text-[11px] text-foreground hover:bg-muted"
|
||||||
|
onClick={(event) => {
|
||||||
|
event.preventDefault();
|
||||||
|
onClear?.();
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
清空
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
{events.length === 0 ? (
|
{events.length === 0 ? (
|
||||||
<div className="rounded-lg border border-dashed border-border/60 px-3 py-2 text-muted-foreground">
|
<div className="rounded-lg border border-dashed border-border/60 px-3 py-2 text-muted-foreground">
|
||||||
暂无协议调试事件
|
暂无协议调试事件
|
||||||
</div>
|
</div>
|
||||||
) : (
|
) : (
|
||||||
events.map((event) => (
|
groupedEvents.map((group) => (
|
||||||
<div
|
<details
|
||||||
key={event.id}
|
key={group.phase}
|
||||||
className="overflow-hidden rounded-lg border border-border/60 bg-background"
|
className="overflow-hidden rounded-lg border border-border/60 bg-background"
|
||||||
|
open
|
||||||
>
|
>
|
||||||
<div className="border-b border-border/60 px-3 py-2 font-medium">
|
<summary className="cursor-pointer border-b border-border/60 px-3 py-2 font-medium">
|
||||||
{new Date(event.at).toLocaleTimeString()}
|
{group.phase}
|
||||||
{" · "}
|
{" · "}
|
||||||
{String(event.payload.phase || event.payload.type || "unknown")}
|
{group.items.length} 条
|
||||||
|
</summary>
|
||||||
|
<div className="space-y-2 px-2 py-2">
|
||||||
|
{group.items.map((event) => (
|
||||||
|
<div
|
||||||
|
key={event.id}
|
||||||
|
className="overflow-hidden rounded-md border border-border/60"
|
||||||
|
>
|
||||||
|
<div className="border-b border-border/60 px-3 py-1.5 text-[11px] font-medium text-foreground">
|
||||||
|
{new Date(event.at).toLocaleTimeString()}
|
||||||
|
</div>
|
||||||
|
<pre className="overflow-x-auto px-3 py-2 whitespace-pre-wrap break-words text-[11px] leading-relaxed text-muted-foreground">
|
||||||
|
{JSON.stringify(event.payload, null, 2)}
|
||||||
|
</pre>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
</div>
|
</div>
|
||||||
<pre className="overflow-x-auto px-3 py-2 whitespace-pre-wrap break-words text-[11px] leading-relaxed text-muted-foreground">
|
</details>
|
||||||
{JSON.stringify(event.payload, null, 2)}
|
|
||||||
</pre>
|
|
||||||
</div>
|
|
||||||
))
|
))
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
@ -48,4 +99,3 @@ export function ProtocolDebugPanel({
|
|||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -2,6 +2,7 @@ import "server-only";
|
|||||||
|
|
||||||
import type { ImageWorkflowMemory } from "@/lib/ai/image-workflow-memory";
|
import type { ImageWorkflowMemory } from "@/lib/ai/image-workflow-memory";
|
||||||
import { mentionsImageEditFollowup } from "@/lib/ai/image-edit-subagent";
|
import { mentionsImageEditFollowup } from "@/lib/ai/image-edit-subagent";
|
||||||
|
import { mentionsImageGenerationFollowup } from "@/lib/ai/image-generation-subagent";
|
||||||
import { isPromptCarryForwardRequest } from "@/lib/ai/prompt-memory";
|
import { isPromptCarryForwardRequest } from "@/lib/ai/prompt-memory";
|
||||||
|
|
||||||
export interface PendingGridConfirmation {
|
export interface PendingGridConfirmation {
|
||||||
@ -16,6 +17,7 @@ export type ConversationActionKind =
|
|||||||
| "grid_confirmation_followup"
|
| "grid_confirmation_followup"
|
||||||
| "active_storyboard_followup"
|
| "active_storyboard_followup"
|
||||||
| "image_edit_followup"
|
| "image_edit_followup"
|
||||||
|
| "image_generation_followup"
|
||||||
| "prompt_carry_forward";
|
| "prompt_carry_forward";
|
||||||
|
|
||||||
export interface ConversationActionResolution {
|
export interface ConversationActionResolution {
|
||||||
@ -175,7 +177,7 @@ export function resolveConversationAction(params: {
|
|||||||
} satisfies ConversationActionResolution;
|
} satisfies ConversationActionResolution;
|
||||||
}
|
}
|
||||||
|
|
||||||
const usePromptCarryForward =
|
const promptCarryForwardRequested =
|
||||||
Boolean(params.confirmedPrompt) &&
|
Boolean(params.confirmedPrompt) &&
|
||||||
(isConfirmedPromptGenerationRequest(params.text) ||
|
(isConfirmedPromptGenerationRequest(params.text) ||
|
||||||
isPromptCarryForwardRequest({
|
isPromptCarryForwardRequest({
|
||||||
@ -183,6 +185,21 @@ export function resolveConversationAction(params: {
|
|||||||
hasImage: params.hasImageInputForPromptCarryForward,
|
hasImage: params.hasImageInputForPromptCarryForward,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
const hasImageGenerationFollowup =
|
||||||
|
params.imageWorkflowMemory?.lastIntent === "image_gen" &&
|
||||||
|
(mentionsImageGenerationFollowup(params.text) || promptCarryForwardRequested);
|
||||||
|
|
||||||
|
if (hasImageGenerationFollowup) {
|
||||||
|
return {
|
||||||
|
kind: "image_generation_followup",
|
||||||
|
forceAgentConversation: true,
|
||||||
|
shouldKeepSelectedModel: true,
|
||||||
|
usePromptCarryForward: promptCarryForwardRequested,
|
||||||
|
} satisfies ConversationActionResolution;
|
||||||
|
}
|
||||||
|
|
||||||
|
const usePromptCarryForward = promptCarryForwardRequested;
|
||||||
|
|
||||||
if (usePromptCarryForward) {
|
if (usePromptCarryForward) {
|
||||||
return {
|
return {
|
||||||
kind: "prompt_carry_forward",
|
kind: "prompt_carry_forward",
|
||||||
|
|||||||
@ -29,12 +29,32 @@ export const conversationStateEventSchemas = {
|
|||||||
originalPrompt: z.string().min(1),
|
originalPrompt: z.string().min(1),
|
||||||
}),
|
}),
|
||||||
}),
|
}),
|
||||||
|
clarification_requested: z.object({
|
||||||
|
userId: z.string().uuid(),
|
||||||
|
chatId: z.string().uuid(),
|
||||||
|
projectId: z.string().uuid().nullable().optional(),
|
||||||
|
messageId: z.string().uuid(),
|
||||||
|
text: z.string(),
|
||||||
|
clarificationMessage: z.string().min(1),
|
||||||
|
currentIntent: z.string().nullable().optional(),
|
||||||
|
}),
|
||||||
storyboard_confirmed: z.object({
|
storyboard_confirmed: z.object({
|
||||||
userId: z.string().uuid(),
|
userId: z.string().uuid(),
|
||||||
chatId: z.string().uuid(),
|
chatId: z.string().uuid(),
|
||||||
projectId: z.string().uuid().nullable().optional(),
|
projectId: z.string().uuid().nullable().optional(),
|
||||||
messageId: z.string().uuid(),
|
messageId: z.string().uuid(),
|
||||||
text: z.string(),
|
text: z.string(),
|
||||||
|
taskId: z.string().min(1).optional(),
|
||||||
|
status: z.string().optional(),
|
||||||
|
}),
|
||||||
|
storyboard_execution_started: z.object({
|
||||||
|
userId: z.string().uuid(),
|
||||||
|
chatId: z.string().uuid(),
|
||||||
|
projectId: z.string().uuid().nullable().optional(),
|
||||||
|
messageId: z.string().uuid(),
|
||||||
|
text: z.string(),
|
||||||
|
taskId: z.string().min(1),
|
||||||
|
status: z.string().optional(),
|
||||||
}),
|
}),
|
||||||
image_workflow_updated: z.object({
|
image_workflow_updated: z.object({
|
||||||
userId: z.string().uuid(),
|
userId: z.string().uuid(),
|
||||||
@ -50,10 +70,75 @@ export const conversationStateEventSchemas = {
|
|||||||
lastResolvedTargetImageUrls: nullableStringArraySchema,
|
lastResolvedTargetImageUrls: nullableStringArraySchema,
|
||||||
lastMessageId: z.string().uuid(),
|
lastMessageId: z.string().uuid(),
|
||||||
lastUserPrompt: z.string(),
|
lastUserPrompt: z.string(),
|
||||||
|
lastOptimizedPrompt: z.string().optional(),
|
||||||
lastRequestedImageCount: z.number().int().min(1).max(12).optional(),
|
lastRequestedImageCount: z.number().int().min(1).max(12).optional(),
|
||||||
|
lastAspectRatio: z.string().optional(),
|
||||||
|
lastImageSize: z.string().optional(),
|
||||||
lastSourceImageUrls: nullableStringArraySchema,
|
lastSourceImageUrls: nullableStringArraySchema,
|
||||||
lastResultImageUrls: nullableStringArraySchema,
|
lastResultImageUrls: nullableStringArraySchema,
|
||||||
}),
|
}),
|
||||||
|
storyboard_revision_requested: z.object({
|
||||||
|
userId: z.string().uuid(),
|
||||||
|
chatId: z.string().uuid(),
|
||||||
|
projectId: z.string().uuid().nullable().optional(),
|
||||||
|
messageId: z.string().uuid(),
|
||||||
|
text: z.string(),
|
||||||
|
taskId: z.string().min(1),
|
||||||
|
status: z.string().optional(),
|
||||||
|
}),
|
||||||
|
storyboard_execution_completed: z.object({
|
||||||
|
userId: z.string().uuid(),
|
||||||
|
chatId: z.string().uuid(),
|
||||||
|
projectId: z.string().uuid().nullable().optional(),
|
||||||
|
messageId: z.string().uuid(),
|
||||||
|
text: z.string(),
|
||||||
|
taskId: z.string().min(1),
|
||||||
|
status: z.string().optional(),
|
||||||
|
successCount: z.number().int().min(0).optional(),
|
||||||
|
failedCount: z.number().int().min(0).optional(),
|
||||||
|
firstVideoUrl: z.string().optional(),
|
||||||
|
}),
|
||||||
|
video_generation_started: z.object({
|
||||||
|
userId: z.string().uuid(),
|
||||||
|
chatId: z.string().uuid(),
|
||||||
|
projectId: z.string().uuid().nullable().optional(),
|
||||||
|
messageId: z.string().uuid(),
|
||||||
|
text: z.string(),
|
||||||
|
source: z.string().min(1),
|
||||||
|
providerType: z.string().optional(),
|
||||||
|
modelRef: z.string().optional(),
|
||||||
|
mode: z.string().optional(),
|
||||||
|
taskId: z.string().optional(),
|
||||||
|
status: z.string().optional(),
|
||||||
|
}),
|
||||||
|
video_generation_completed: z.object({
|
||||||
|
userId: z.string().uuid(),
|
||||||
|
chatId: z.string().uuid(),
|
||||||
|
projectId: z.string().uuid().nullable().optional(),
|
||||||
|
messageId: z.string().uuid(),
|
||||||
|
text: z.string(),
|
||||||
|
source: z.string().min(1),
|
||||||
|
providerType: z.string().optional(),
|
||||||
|
modelRef: z.string().optional(),
|
||||||
|
mode: z.string().optional(),
|
||||||
|
taskId: z.string().optional(),
|
||||||
|
status: z.string().optional(),
|
||||||
|
temporaryVideoUrl: z.string().optional(),
|
||||||
|
}),
|
||||||
|
video_generation_failed: z.object({
|
||||||
|
userId: z.string().uuid(),
|
||||||
|
chatId: z.string().uuid(),
|
||||||
|
projectId: z.string().uuid().nullable().optional(),
|
||||||
|
messageId: z.string().uuid(),
|
||||||
|
text: z.string(),
|
||||||
|
source: z.string().min(1),
|
||||||
|
providerType: z.string().optional(),
|
||||||
|
modelRef: z.string().optional(),
|
||||||
|
mode: z.string().optional(),
|
||||||
|
taskId: z.string().optional(),
|
||||||
|
status: z.string().optional(),
|
||||||
|
error: z.string().optional(),
|
||||||
|
}),
|
||||||
target_resolved: z.object({
|
target_resolved: z.object({
|
||||||
userId: z.string().uuid(),
|
userId: z.string().uuid(),
|
||||||
chatId: z.string().uuid(),
|
chatId: z.string().uuid(),
|
||||||
@ -84,6 +169,16 @@ export const conversationStateEventSchemas = {
|
|||||||
text: z.string().optional(),
|
text: z.string().optional(),
|
||||||
status: z.string().optional(),
|
status: z.string().optional(),
|
||||||
}),
|
}),
|
||||||
|
workflow_cancelled: z.object({
|
||||||
|
userId: z.string().uuid(),
|
||||||
|
chatId: z.string().uuid(),
|
||||||
|
projectId: z.string().uuid().nullable().optional(),
|
||||||
|
messageId: z.string().uuid(),
|
||||||
|
text: z.string(),
|
||||||
|
workflowType: z.string().min(1),
|
||||||
|
workflowId: z.string().min(1).optional(),
|
||||||
|
status: z.string().optional(),
|
||||||
|
}),
|
||||||
agent_preset_completed: z.object({
|
agent_preset_completed: z.object({
|
||||||
userId: z.string().uuid(),
|
userId: z.string().uuid(),
|
||||||
chatId: z.string().uuid(),
|
chatId: z.string().uuid(),
|
||||||
@ -120,4 +215,3 @@ export type ConversationStateEvent =
|
|||||||
payload: ConversationStateEventPayload<K>;
|
payload: ConversationStateEventPayload<K>;
|
||||||
};
|
};
|
||||||
}[ConversationStateEventType];
|
}[ConversationStateEventType];
|
||||||
|
|
||||||
|
|||||||
@ -393,6 +393,23 @@ export async function persistConversationStateEvent<T extends ConversationStateE
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case "clarification_requested": {
|
||||||
|
const typed = parsed as ConversationStateEventPayload<"clarification_requested">;
|
||||||
|
return persistConversationStateProtocol({
|
||||||
|
userId: typed.userId,
|
||||||
|
chatId: typed.chatId,
|
||||||
|
projectId: typed.projectId,
|
||||||
|
currentIntent: typed.currentIntent,
|
||||||
|
lastMessageId: typed.messageId,
|
||||||
|
lastTurnSummaryPatch: {
|
||||||
|
text: typed.text.slice(0, 500),
|
||||||
|
resolutionMode: "clarification",
|
||||||
|
confidence: 80,
|
||||||
|
clarificationMessage: typed.clarificationMessage.slice(0, 1000),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
case "storyboard_confirmed": {
|
case "storyboard_confirmed": {
|
||||||
const typed = parsed as ConversationStateEventPayload<"storyboard_confirmed">;
|
const typed = parsed as ConversationStateEventPayload<"storyboard_confirmed">;
|
||||||
const existing = await getConversationWorkingStateByChatId({
|
const existing = await getConversationWorkingStateByChatId({
|
||||||
@ -410,6 +427,9 @@ export async function persistConversationStateEvent<T extends ConversationStateE
|
|||||||
text: typed.text.slice(0, 500),
|
text: typed.text.slice(0, 500),
|
||||||
resolutionMode: "storyboard_confirmed",
|
resolutionMode: "storyboard_confirmed",
|
||||||
confidence: 95,
|
confidence: 95,
|
||||||
|
videoWorkflowStatus: typed.status || "confirmed",
|
||||||
|
storyboardTaskId: typed.taskId,
|
||||||
|
storyboardWorkflowAction: "confirmed",
|
||||||
storyboardMemory: buildStoryboardPreferenceMemoryUpdate({
|
storyboardMemory: buildStoryboardPreferenceMemoryUpdate({
|
||||||
existing: existingStoryboardMemory,
|
existing: existingStoryboardMemory,
|
||||||
prefersConfirmation: true,
|
prefersConfirmation: true,
|
||||||
@ -418,12 +438,32 @@ export async function persistConversationStateEvent<T extends ConversationStateE
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case "storyboard_execution_started": {
|
||||||
|
const typed = parsed as ConversationStateEventPayload<"storyboard_execution_started">;
|
||||||
|
return persistConversationStateProtocol({
|
||||||
|
userId: typed.userId,
|
||||||
|
chatId: typed.chatId,
|
||||||
|
projectId: typed.projectId,
|
||||||
|
currentIntent: "video_gen",
|
||||||
|
lastMessageId: typed.messageId,
|
||||||
|
lastTurnSummaryPatch: {
|
||||||
|
text: typed.text.slice(0, 500),
|
||||||
|
resolutionMode: "storyboard_execution_started",
|
||||||
|
confidence: 95,
|
||||||
|
videoWorkflowStatus: typed.status || "running",
|
||||||
|
storyboardTaskId: typed.taskId,
|
||||||
|
storyboardWorkflowAction: "execution_started",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
case "image_workflow_updated": {
|
case "image_workflow_updated": {
|
||||||
const typed = parsed as ConversationStateEventPayload<"image_workflow_updated">;
|
const typed = parsed as ConversationStateEventPayload<"image_workflow_updated">;
|
||||||
const existing = await getConversationWorkingStateByChatId({
|
const existing = await getConversationWorkingStateByChatId({
|
||||||
chatId: typed.chatId,
|
chatId: typed.chatId,
|
||||||
}).catch(() => null);
|
}).catch(() => null);
|
||||||
const existingImageWorkflowMemory = extractImageWorkflowMemory(existing);
|
const existingImageWorkflowMemory = extractImageWorkflowMemory(existing);
|
||||||
|
const nextOptimizedPrompt = typed.lastOptimizedPrompt?.trim();
|
||||||
|
|
||||||
return persistConversationStateProtocol({
|
return persistConversationStateProtocol({
|
||||||
userId: typed.userId,
|
userId: typed.userId,
|
||||||
@ -439,11 +479,22 @@ export async function persistConversationStateEvent<T extends ConversationStateE
|
|||||||
lastResolvedTargetImageUrls: typed.lastResolvedTargetImageUrls,
|
lastResolvedTargetImageUrls: typed.lastResolvedTargetImageUrls,
|
||||||
lastMessageId: typed.lastMessageId,
|
lastMessageId: typed.lastMessageId,
|
||||||
lastTurnSummaryPatch: {
|
lastTurnSummaryPatch: {
|
||||||
|
promptMemory: nextOptimizedPrompt
|
||||||
|
? buildPromptMemoryUpdate({
|
||||||
|
optimizedPrompt: nextOptimizedPrompt.slice(0, 1000),
|
||||||
|
sourceIntent: typed.currentIntent,
|
||||||
|
sourceText: typed.lastUserPrompt.slice(0, 500),
|
||||||
|
reusable: true,
|
||||||
|
})
|
||||||
|
: undefined,
|
||||||
imageWorkflowMemory: buildImageWorkflowMemoryUpdate({
|
imageWorkflowMemory: buildImageWorkflowMemoryUpdate({
|
||||||
existing: existingImageWorkflowMemory,
|
existing: existingImageWorkflowMemory,
|
||||||
lastIntent: typed.currentIntent,
|
lastIntent: typed.currentIntent,
|
||||||
|
lastOptimizedPrompt: nextOptimizedPrompt?.slice(0, 1000),
|
||||||
lastUserPrompt: typed.lastUserPrompt.slice(0, 500),
|
lastUserPrompt: typed.lastUserPrompt.slice(0, 500),
|
||||||
lastRequestedImageCount: typed.lastRequestedImageCount,
|
lastRequestedImageCount: typed.lastRequestedImageCount,
|
||||||
|
lastAspectRatio: typed.lastAspectRatio?.slice(0, 32),
|
||||||
|
lastImageSize: typed.lastImageSize?.slice(0, 32),
|
||||||
lastSourceImageUrls: typed.lastSourceImageUrls || null,
|
lastSourceImageUrls: typed.lastSourceImageUrls || null,
|
||||||
lastResultImageUrls: typed.lastResultImageUrls || null,
|
lastResultImageUrls: typed.lastResultImageUrls || null,
|
||||||
}),
|
}),
|
||||||
@ -451,6 +502,124 @@ export async function persistConversationStateEvent<T extends ConversationStateE
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case "storyboard_revision_requested": {
|
||||||
|
const typed = parsed as ConversationStateEventPayload<"storyboard_revision_requested">;
|
||||||
|
const existing = await getConversationWorkingStateByChatId({
|
||||||
|
chatId: typed.chatId,
|
||||||
|
}).catch(() => null);
|
||||||
|
const existingStoryboardMemory = extractStoryboardPreferenceMemory(existing);
|
||||||
|
|
||||||
|
return persistConversationStateProtocol({
|
||||||
|
userId: typed.userId,
|
||||||
|
chatId: typed.chatId,
|
||||||
|
projectId: typed.projectId,
|
||||||
|
currentIntent: "video_gen",
|
||||||
|
lastMessageId: typed.messageId,
|
||||||
|
lastTurnSummaryPatch: {
|
||||||
|
text: typed.text.slice(0, 500),
|
||||||
|
resolutionMode: "storyboard_revision",
|
||||||
|
confidence: 90,
|
||||||
|
videoWorkflowStatus: typed.status || "awaiting_user_confirmation",
|
||||||
|
storyboardTaskId: typed.taskId,
|
||||||
|
storyboardWorkflowAction: "revision_requested",
|
||||||
|
storyboardMemory: buildStoryboardPreferenceMemoryUpdate({
|
||||||
|
existing: existingStoryboardMemory,
|
||||||
|
prefersConfirmation: true,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
case "storyboard_execution_completed": {
|
||||||
|
const typed = parsed as ConversationStateEventPayload<"storyboard_execution_completed">;
|
||||||
|
return persistConversationStateProtocol({
|
||||||
|
userId: typed.userId,
|
||||||
|
chatId: typed.chatId,
|
||||||
|
projectId: typed.projectId,
|
||||||
|
currentIntent: "video_gen",
|
||||||
|
lastMessageId: typed.messageId,
|
||||||
|
lastTurnSummaryPatch: {
|
||||||
|
text: typed.text.slice(0, 500),
|
||||||
|
resolutionMode: "storyboard_execution_completed",
|
||||||
|
confidence: 95,
|
||||||
|
videoWorkflowStatus: typed.status || "completed",
|
||||||
|
storyboardTaskId: typed.taskId,
|
||||||
|
storyboardWorkflowAction: "execution_completed",
|
||||||
|
successCount: typed.successCount,
|
||||||
|
failedCount: typed.failedCount,
|
||||||
|
firstVideoUrl: typed.firstVideoUrl,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
case "video_generation_started": {
|
||||||
|
const typed = parsed as ConversationStateEventPayload<"video_generation_started">;
|
||||||
|
return persistConversationStateProtocol({
|
||||||
|
userId: typed.userId,
|
||||||
|
chatId: typed.chatId,
|
||||||
|
projectId: typed.projectId,
|
||||||
|
currentIntent: "video_gen",
|
||||||
|
lastMessageId: typed.messageId,
|
||||||
|
lastTurnSummaryPatch: {
|
||||||
|
text: typed.text.slice(0, 500),
|
||||||
|
resolutionMode: "video_generation_started",
|
||||||
|
confidence: 90,
|
||||||
|
videoWorkflowStatus: typed.status || "running",
|
||||||
|
videoWorkflowSource: typed.source,
|
||||||
|
videoProviderType: typed.providerType,
|
||||||
|
videoModelRef: typed.modelRef,
|
||||||
|
videoGenerationMode: typed.mode,
|
||||||
|
videoTaskId: typed.taskId,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
case "video_generation_completed": {
|
||||||
|
const typed = parsed as ConversationStateEventPayload<"video_generation_completed">;
|
||||||
|
return persistConversationStateProtocol({
|
||||||
|
userId: typed.userId,
|
||||||
|
chatId: typed.chatId,
|
||||||
|
projectId: typed.projectId,
|
||||||
|
currentIntent: "video_gen",
|
||||||
|
lastMessageId: typed.messageId,
|
||||||
|
lastTurnSummaryPatch: {
|
||||||
|
text: typed.text.slice(0, 500),
|
||||||
|
resolutionMode: "video_generation_completed",
|
||||||
|
confidence: 95,
|
||||||
|
videoWorkflowStatus: typed.status || "completed",
|
||||||
|
videoWorkflowSource: typed.source,
|
||||||
|
videoProviderType: typed.providerType,
|
||||||
|
videoModelRef: typed.modelRef,
|
||||||
|
videoGenerationMode: typed.mode,
|
||||||
|
videoTaskId: typed.taskId,
|
||||||
|
videoTemporaryUrl: typed.temporaryVideoUrl,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
case "video_generation_failed": {
|
||||||
|
const typed = parsed as ConversationStateEventPayload<"video_generation_failed">;
|
||||||
|
return persistConversationStateProtocol({
|
||||||
|
userId: typed.userId,
|
||||||
|
chatId: typed.chatId,
|
||||||
|
projectId: typed.projectId,
|
||||||
|
currentIntent: "video_gen",
|
||||||
|
lastMessageId: typed.messageId,
|
||||||
|
lastTurnSummaryPatch: {
|
||||||
|
text: typed.text.slice(0, 500),
|
||||||
|
resolutionMode: "video_generation_failed",
|
||||||
|
confidence: 85,
|
||||||
|
videoWorkflowStatus: typed.status || "failed",
|
||||||
|
videoWorkflowSource: typed.source,
|
||||||
|
videoProviderType: typed.providerType,
|
||||||
|
videoModelRef: typed.modelRef,
|
||||||
|
videoGenerationMode: typed.mode,
|
||||||
|
videoTaskId: typed.taskId,
|
||||||
|
videoWorkflowError: typed.error?.slice(0, 1000),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
case "target_resolved": {
|
case "target_resolved": {
|
||||||
const typed = parsed as ConversationStateEventPayload<"target_resolved">;
|
const typed = parsed as ConversationStateEventPayload<"target_resolved">;
|
||||||
return persistConversationStateProtocol({
|
return persistConversationStateProtocol({
|
||||||
@ -498,6 +667,26 @@ export async function persistConversationStateEvent<T extends ConversationStateE
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case "workflow_cancelled": {
|
||||||
|
const typed = parsed as ConversationStateEventPayload<"workflow_cancelled">;
|
||||||
|
return persistConversationStateProtocol({
|
||||||
|
userId: typed.userId,
|
||||||
|
chatId: typed.chatId,
|
||||||
|
projectId: typed.projectId,
|
||||||
|
currentIntent: typed.workflowType === "storyboard_video" ? "video_gen" : undefined,
|
||||||
|
lastMessageId: typed.messageId,
|
||||||
|
lastTurnSummaryPatch: {
|
||||||
|
text: typed.text.slice(0, 500),
|
||||||
|
resolutionMode: "workflow_cancelled",
|
||||||
|
confidence: 95,
|
||||||
|
videoWorkflowStatus: typed.status || "cancelled",
|
||||||
|
cancelledWorkflowType: typed.workflowType,
|
||||||
|
cancelledWorkflowId: typed.workflowId,
|
||||||
|
workflowAction: "cancel_requested",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
case "agent_preset_completed": {
|
case "agent_preset_completed": {
|
||||||
const typed = parsed as ConversationStateEventPayload<"agent_preset_completed">;
|
const typed = parsed as ConversationStateEventPayload<"agent_preset_completed">;
|
||||||
return persistConversationStateProtocol({
|
return persistConversationStateProtocol({
|
||||||
@ -594,12 +783,46 @@ export async function persistGridConfirmationTurn(params: {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function persistClarificationTurn(params: {
|
||||||
|
userId: string;
|
||||||
|
chatId: string;
|
||||||
|
projectId?: string | null;
|
||||||
|
messageId: string;
|
||||||
|
text: string;
|
||||||
|
clarificationMessage: string;
|
||||||
|
currentIntent?: string | null;
|
||||||
|
pendingGridConfirmation?: PendingGridConfirmation | null;
|
||||||
|
}) {
|
||||||
|
if (params.pendingGridConfirmation) {
|
||||||
|
return persistConversationStateEvent("grid_confirmation_requested", {
|
||||||
|
userId: params.userId,
|
||||||
|
chatId: params.chatId,
|
||||||
|
projectId: params.projectId,
|
||||||
|
messageId: params.messageId,
|
||||||
|
text: params.text,
|
||||||
|
pendingGridConfirmation: params.pendingGridConfirmation,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return persistConversationStateEvent("clarification_requested", {
|
||||||
|
userId: params.userId,
|
||||||
|
chatId: params.chatId,
|
||||||
|
projectId: params.projectId,
|
||||||
|
messageId: params.messageId,
|
||||||
|
text: params.text,
|
||||||
|
clarificationMessage: params.clarificationMessage,
|
||||||
|
currentIntent: params.currentIntent,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export async function persistStoryboardConfirmedTurn(params: {
|
export async function persistStoryboardConfirmedTurn(params: {
|
||||||
userId: string;
|
userId: string;
|
||||||
chatId: string;
|
chatId: string;
|
||||||
projectId?: string | null;
|
projectId?: string | null;
|
||||||
messageId: string;
|
messageId: string;
|
||||||
text: string;
|
text: string;
|
||||||
|
taskId?: string;
|
||||||
|
status?: string;
|
||||||
}) {
|
}) {
|
||||||
return persistConversationStateEvent("storyboard_confirmed", {
|
return persistConversationStateEvent("storyboard_confirmed", {
|
||||||
userId: params.userId,
|
userId: params.userId,
|
||||||
@ -607,6 +830,162 @@ export async function persistStoryboardConfirmedTurn(params: {
|
|||||||
projectId: params.projectId,
|
projectId: params.projectId,
|
||||||
messageId: params.messageId,
|
messageId: params.messageId,
|
||||||
text: params.text,
|
text: params.text,
|
||||||
|
taskId: params.taskId,
|
||||||
|
status: params.status,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function persistStoryboardExecutionStartedTurn(params: {
|
||||||
|
userId: string;
|
||||||
|
chatId: string;
|
||||||
|
projectId?: string | null;
|
||||||
|
messageId: string;
|
||||||
|
text: string;
|
||||||
|
taskId: string;
|
||||||
|
status?: string;
|
||||||
|
}) {
|
||||||
|
return persistConversationStateEvent("storyboard_execution_started", {
|
||||||
|
userId: params.userId,
|
||||||
|
chatId: params.chatId,
|
||||||
|
projectId: params.projectId,
|
||||||
|
messageId: params.messageId,
|
||||||
|
text: params.text,
|
||||||
|
taskId: params.taskId,
|
||||||
|
status: params.status,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function persistStoryboardRevisionTurn(params: {
|
||||||
|
userId: string;
|
||||||
|
chatId: string;
|
||||||
|
projectId?: string | null;
|
||||||
|
messageId: string;
|
||||||
|
text: string;
|
||||||
|
taskId: string;
|
||||||
|
status?: string;
|
||||||
|
}) {
|
||||||
|
return persistConversationStateEvent("storyboard_revision_requested", {
|
||||||
|
userId: params.userId,
|
||||||
|
chatId: params.chatId,
|
||||||
|
projectId: params.projectId,
|
||||||
|
messageId: params.messageId,
|
||||||
|
text: params.text,
|
||||||
|
taskId: params.taskId,
|
||||||
|
status: params.status,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function persistStoryboardExecutionCompletedTurn(params: {
|
||||||
|
userId: string;
|
||||||
|
chatId: string;
|
||||||
|
projectId?: string | null;
|
||||||
|
messageId: string;
|
||||||
|
text: string;
|
||||||
|
taskId: string;
|
||||||
|
status?: string;
|
||||||
|
successCount?: number;
|
||||||
|
failedCount?: number;
|
||||||
|
firstVideoUrl?: string;
|
||||||
|
}) {
|
||||||
|
return persistConversationStateEvent("storyboard_execution_completed", {
|
||||||
|
userId: params.userId,
|
||||||
|
chatId: params.chatId,
|
||||||
|
projectId: params.projectId,
|
||||||
|
messageId: params.messageId,
|
||||||
|
text: params.text,
|
||||||
|
taskId: params.taskId,
|
||||||
|
status: params.status,
|
||||||
|
successCount: params.successCount,
|
||||||
|
failedCount: params.failedCount,
|
||||||
|
firstVideoUrl: params.firstVideoUrl,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function persistVideoGenerationStartedTurn(params: {
|
||||||
|
userId: string;
|
||||||
|
chatId: string;
|
||||||
|
projectId?: string | null;
|
||||||
|
messageId: string;
|
||||||
|
text: string;
|
||||||
|
source: string;
|
||||||
|
providerType?: string;
|
||||||
|
modelRef?: string;
|
||||||
|
mode?: string;
|
||||||
|
taskId?: string;
|
||||||
|
status?: string;
|
||||||
|
}) {
|
||||||
|
return persistConversationStateEvent("video_generation_started", {
|
||||||
|
userId: params.userId,
|
||||||
|
chatId: params.chatId,
|
||||||
|
projectId: params.projectId,
|
||||||
|
messageId: params.messageId,
|
||||||
|
text: params.text,
|
||||||
|
source: params.source,
|
||||||
|
providerType: params.providerType,
|
||||||
|
modelRef: params.modelRef,
|
||||||
|
mode: params.mode,
|
||||||
|
taskId: params.taskId,
|
||||||
|
status: params.status,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function persistVideoGenerationCompletedTurn(params: {
|
||||||
|
userId: string;
|
||||||
|
chatId: string;
|
||||||
|
projectId?: string | null;
|
||||||
|
messageId: string;
|
||||||
|
text: string;
|
||||||
|
source: string;
|
||||||
|
providerType?: string;
|
||||||
|
modelRef?: string;
|
||||||
|
mode?: string;
|
||||||
|
taskId?: string;
|
||||||
|
status?: string;
|
||||||
|
temporaryVideoUrl?: string;
|
||||||
|
}) {
|
||||||
|
return persistConversationStateEvent("video_generation_completed", {
|
||||||
|
userId: params.userId,
|
||||||
|
chatId: params.chatId,
|
||||||
|
projectId: params.projectId,
|
||||||
|
messageId: params.messageId,
|
||||||
|
text: params.text,
|
||||||
|
source: params.source,
|
||||||
|
providerType: params.providerType,
|
||||||
|
modelRef: params.modelRef,
|
||||||
|
mode: params.mode,
|
||||||
|
taskId: params.taskId,
|
||||||
|
status: params.status,
|
||||||
|
temporaryVideoUrl: params.temporaryVideoUrl,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function persistVideoGenerationFailedTurn(params: {
|
||||||
|
userId: string;
|
||||||
|
chatId: string;
|
||||||
|
projectId?: string | null;
|
||||||
|
messageId: string;
|
||||||
|
text: string;
|
||||||
|
source: string;
|
||||||
|
providerType?: string;
|
||||||
|
modelRef?: string;
|
||||||
|
mode?: string;
|
||||||
|
taskId?: string;
|
||||||
|
status?: string;
|
||||||
|
error?: string;
|
||||||
|
}) {
|
||||||
|
return persistConversationStateEvent("video_generation_failed", {
|
||||||
|
userId: params.userId,
|
||||||
|
chatId: params.chatId,
|
||||||
|
projectId: params.projectId,
|
||||||
|
messageId: params.messageId,
|
||||||
|
text: params.text,
|
||||||
|
source: params.source,
|
||||||
|
providerType: params.providerType,
|
||||||
|
modelRef: params.modelRef,
|
||||||
|
mode: params.mode,
|
||||||
|
taskId: params.taskId,
|
||||||
|
status: params.status,
|
||||||
|
error: params.error,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -624,7 +1003,10 @@ export async function persistImageWorkflowTurn(params: {
|
|||||||
lastResolvedTargetImageUrls?: string[] | null;
|
lastResolvedTargetImageUrls?: string[] | null;
|
||||||
lastMessageId: string;
|
lastMessageId: string;
|
||||||
lastUserPrompt: string;
|
lastUserPrompt: string;
|
||||||
|
lastOptimizedPrompt?: string;
|
||||||
lastRequestedImageCount?: number;
|
lastRequestedImageCount?: number;
|
||||||
|
lastAspectRatio?: string;
|
||||||
|
lastImageSize?: string;
|
||||||
lastSourceImageUrls?: string[] | null;
|
lastSourceImageUrls?: string[] | null;
|
||||||
lastResultImageUrls?: string[] | null;
|
lastResultImageUrls?: string[] | null;
|
||||||
}) {
|
}) {
|
||||||
@ -642,7 +1024,10 @@ export async function persistImageWorkflowTurn(params: {
|
|||||||
lastResolvedTargetImageUrls: params.lastResolvedTargetImageUrls,
|
lastResolvedTargetImageUrls: params.lastResolvedTargetImageUrls,
|
||||||
lastMessageId: params.lastMessageId,
|
lastMessageId: params.lastMessageId,
|
||||||
lastUserPrompt: params.lastUserPrompt,
|
lastUserPrompt: params.lastUserPrompt,
|
||||||
|
lastOptimizedPrompt: params.lastOptimizedPrompt,
|
||||||
lastRequestedImageCount: params.lastRequestedImageCount,
|
lastRequestedImageCount: params.lastRequestedImageCount,
|
||||||
|
lastAspectRatio: params.lastAspectRatio,
|
||||||
|
lastImageSize: params.lastImageSize,
|
||||||
lastSourceImageUrls: params.lastSourceImageUrls || null,
|
lastSourceImageUrls: params.lastSourceImageUrls || null,
|
||||||
lastResultImageUrls: params.lastResultImageUrls || null,
|
lastResultImageUrls: params.lastResultImageUrls || null,
|
||||||
});
|
});
|
||||||
@ -684,6 +1069,28 @@ export async function persistVideoWorkflowTurn(params: {
|
|||||||
return persistConversationStateEvent("video_workflow_updated", params);
|
return persistConversationStateEvent("video_workflow_updated", params);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function persistWorkflowCancelledTurn(params: {
|
||||||
|
userId: string;
|
||||||
|
chatId: string;
|
||||||
|
projectId?: string | null;
|
||||||
|
messageId: string;
|
||||||
|
text: string;
|
||||||
|
workflowType: string;
|
||||||
|
workflowId?: string;
|
||||||
|
status?: string;
|
||||||
|
}) {
|
||||||
|
return persistConversationStateEvent("workflow_cancelled", {
|
||||||
|
userId: params.userId,
|
||||||
|
chatId: params.chatId,
|
||||||
|
projectId: params.projectId,
|
||||||
|
messageId: params.messageId,
|
||||||
|
text: params.text,
|
||||||
|
workflowType: params.workflowType,
|
||||||
|
workflowId: params.workflowId,
|
||||||
|
status: params.status,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export async function persistAgentPresetTurn(params: {
|
export async function persistAgentPresetTurn(params: {
|
||||||
userId: string;
|
userId: string;
|
||||||
chatId: string;
|
chatId: string;
|
||||||
|
|||||||
@ -16,6 +16,15 @@ function normalizeCompactText(text: string) {
|
|||||||
return String(text || "").toLowerCase().replace(/\s+/g, "").trim();
|
return String(text || "").toLowerCase().replace(/\s+/g, "").trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function mentionsImageGenerationFollowup(text: string): boolean {
|
||||||
|
const normalized = normalizeCompactText(text);
|
||||||
|
if (!normalized) return false;
|
||||||
|
|
||||||
|
return /继续生成|继续出图|继续做图|再来(一|两|三|四|五|六|七|八|九|\d+)?张|再出(一|两|三|四|五|六|七|八|九|\d+)?张|再做(一|两|三|四|五|六|七|八|九|\d+)?组|再给我(一|两|三|四|五|六|七|八|九|\d+)?张|补(一|两|三|四|五|六|七|八|九|\d+)?张|多来(一|两|三|四|五|六|七|八|九|\d+)?张|换一版|再换一版|同风格再来|同样风格再来|保持.*(不变|一致).*再来|同提示词再来|沿用上一个效果再来/.test(
|
||||||
|
normalized,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export function buildImageGenerationSpecialistRoute(params: {
|
export function buildImageGenerationSpecialistRoute(params: {
|
||||||
text: string;
|
text: string;
|
||||||
threadState?: AgentThreadState | null;
|
threadState?: AgentThreadState | null;
|
||||||
@ -30,11 +39,18 @@ export function buildImageGenerationSpecialistRoute(params: {
|
|||||||
text: params.text,
|
text: params.text,
|
||||||
hasImage: params.hasImage,
|
hasImage: params.hasImage,
|
||||||
});
|
});
|
||||||
|
const mentionsGenerationFollowup = mentionsImageGenerationFollowup(params.text);
|
||||||
const mentionsPromptExecution =
|
const mentionsPromptExecution =
|
||||||
/提示词|prompt|关键词|咒语/.test(normalized) &&
|
/提示词|prompt|关键词|咒语/.test(normalized) &&
|
||||||
/生成|生图|出图|继续做|继续生成|执行/.test(normalized);
|
/生成|生图|出图|继续做|继续生成|执行/.test(normalized);
|
||||||
|
|
||||||
if (!hasPromptMemory && !hasImageWorkflowMemory && !carryForward && !mentionsPromptExecution) {
|
if (
|
||||||
|
!hasPromptMemory &&
|
||||||
|
!hasImageWorkflowMemory &&
|
||||||
|
!carryForward &&
|
||||||
|
!mentionsPromptExecution &&
|
||||||
|
!mentionsGenerationFollowup
|
||||||
|
) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -42,6 +58,9 @@ export function buildImageGenerationSpecialistRoute(params: {
|
|||||||
carryForward
|
carryForward
|
||||||
? "用户当前要求沿用上一轮优化提示词继续执行,应优先理解为连续生图执行。"
|
? "用户当前要求沿用上一轮优化提示词继续执行,应优先理解为连续生图执行。"
|
||||||
: "",
|
: "",
|
||||||
|
mentionsGenerationFollowup
|
||||||
|
? "如果用户说“再来几张 / 换一版 / 同风格再来”,优先理解为沿用上一轮生图上下文继续执行,而不是开启全新任务。"
|
||||||
|
: "",
|
||||||
hasPromptMemory
|
hasPromptMemory
|
||||||
? "当前会话里存在上一轮可复用提示词,如果用户在说“上一个提示词/刚才那个 prompt”,优先复用这段提示词。"
|
? "当前会话里存在上一轮可复用提示词,如果用户在说“上一个提示词/刚才那个 prompt”,优先复用这段提示词。"
|
||||||
: "",
|
: "",
|
||||||
@ -58,7 +77,11 @@ export function buildImageGenerationSpecialistRoute(params: {
|
|||||||
return {
|
return {
|
||||||
domain: "image_generation_memory",
|
domain: "image_generation_memory",
|
||||||
specialistId: "image-generation-subagent",
|
specialistId: "image-generation-subagent",
|
||||||
reason: carryForward ? "当前回合命中提示词接力生图" : "当前回合命中提示词相关的连续生图任务",
|
reason: carryForward
|
||||||
|
? "当前回合命中提示词接力生图"
|
||||||
|
: mentionsGenerationFollowup
|
||||||
|
? "当前回合命中连续生图跟进语义"
|
||||||
|
: "当前回合命中提示词相关的连续生图任务",
|
||||||
planningHints,
|
planningHints,
|
||||||
toolAllowlist: [
|
toolAllowlist: [
|
||||||
"respondDirect",
|
"respondDirect",
|
||||||
@ -101,9 +124,15 @@ export function refinePlanWithImageGenerationSubagent(params: {
|
|||||||
text: params.text,
|
text: params.text,
|
||||||
hasImage: params.hasImage,
|
hasImage: params.hasImage,
|
||||||
});
|
});
|
||||||
|
const mentionsGenerationFollowup = mentionsImageGenerationFollowup(params.text);
|
||||||
const hasPromptMemory = getPromptMemoryPlanningHints(params.threadState).length > 0;
|
const hasPromptMemory = getPromptMemoryPlanningHints(params.threadState).length > 0;
|
||||||
const hasImageWorkflowMemory = getImageWorkflowPlanningHints(params.threadState).length > 0;
|
const hasImageWorkflowMemory = getImageWorkflowPlanningHints(params.threadState).length > 0;
|
||||||
if (!carryForward && !hasPromptMemory && !hasImageWorkflowMemory) {
|
if (
|
||||||
|
!carryForward &&
|
||||||
|
!mentionsGenerationFollowup &&
|
||||||
|
!hasPromptMemory &&
|
||||||
|
!hasImageWorkflowMemory
|
||||||
|
) {
|
||||||
return params.plan;
|
return params.plan;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
69
lib/ai/runtime/clarification-interruption.ts
Normal file
69
lib/ai/runtime/clarification-interruption.ts
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
import "server-only";
|
||||||
|
|
||||||
|
function shouldReplaceClarificationLikeMessage(message?: string) {
|
||||||
|
if (!message) return true;
|
||||||
|
const normalized = message.trim();
|
||||||
|
if (!normalized) return true;
|
||||||
|
return /[??]$/.test(normalized) || /哪一张|哪个|请先确认/.test(normalized);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveClarificationBypass(params: {
|
||||||
|
text: string;
|
||||||
|
hasCurrentImageInput: boolean;
|
||||||
|
hasHistoryImageInput: boolean;
|
||||||
|
hasCanvasSelection: boolean;
|
||||||
|
targetResolutionMode?: string | null;
|
||||||
|
executionMode?: string;
|
||||||
|
friendlyMessage?: string;
|
||||||
|
}) {
|
||||||
|
const latestUserText = String(params.text || "").trim();
|
||||||
|
if (
|
||||||
|
!params.hasCurrentImageInput &&
|
||||||
|
!params.hasHistoryImageInput &&
|
||||||
|
latestUserText.length <= 0
|
||||||
|
) {
|
||||||
|
return {
|
||||||
|
shouldBypass: false,
|
||||||
|
nextFriendlyMessage: params.friendlyMessage,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const canForceByIntent = /改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格/.test(
|
||||||
|
latestUserText,
|
||||||
|
);
|
||||||
|
const hasForceTargetHint =
|
||||||
|
params.hasCanvasSelection ||
|
||||||
|
params.targetResolutionMode === "text_reference" ||
|
||||||
|
/第[一二三四五六七八九十\d]+张|左|右|上|下|中间|选中/.test(latestUserText);
|
||||||
|
const canForceExecution =
|
||||||
|
(params.executionMode === "edit_image" ||
|
||||||
|
params.executionMode === "generate_image" ||
|
||||||
|
params.executionMode === "generate_video") &&
|
||||||
|
canForceByIntent &&
|
||||||
|
hasForceTargetHint;
|
||||||
|
|
||||||
|
if (!canForceExecution) {
|
||||||
|
return {
|
||||||
|
shouldBypass: false,
|
||||||
|
nextFriendlyMessage: params.friendlyMessage,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
shouldBypass: true,
|
||||||
|
nextFriendlyMessage: shouldReplaceClarificationLikeMessage(params.friendlyMessage)
|
||||||
|
? "收到,我将按你的当前需求直接执行。"
|
||||||
|
: params.friendlyMessage,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveClarificationMessage(params: {
|
||||||
|
clarificationQuestion?: string;
|
||||||
|
friendlyMessage?: string;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
params.clarificationQuestion ||
|
||||||
|
params.friendlyMessage ||
|
||||||
|
"我先确认一下,你希望我优先处理哪一部分?"
|
||||||
|
);
|
||||||
|
}
|
||||||
110
lib/video-agent/storyboard-chat-followup.ts
Normal file
110
lib/video-agent/storyboard-chat-followup.ts
Normal file
@ -0,0 +1,110 @@
|
|||||||
|
import "server-only";
|
||||||
|
|
||||||
|
import { executeConfirmedStoryboardTask } from "@/lib/video-agent/execute-storyboard-task";
|
||||||
|
import {
|
||||||
|
reviseStoryboardTaskByCommand,
|
||||||
|
resolveStoryboardRevisionCommand,
|
||||||
|
} from "@/lib/video-agent/revise-storyboard-task";
|
||||||
|
import { getVideoStoryboardTaskDetail } from "@/lib/video-agent/task-store";
|
||||||
|
import type { StoryboardProgressEvent } from "@/lib/video-agent/types";
|
||||||
|
|
||||||
|
export async function handleStoryboardWorkflowFollowup(params: {
|
||||||
|
action: "revision_requested" | "cancel_requested";
|
||||||
|
workflowId?: string;
|
||||||
|
text: string;
|
||||||
|
friendlyMessage?: string;
|
||||||
|
onProgress?: (event: StoryboardProgressEvent) => void | Promise<void>;
|
||||||
|
}) {
|
||||||
|
let replyText =
|
||||||
|
params.friendlyMessage ||
|
||||||
|
(params.action === "cancel_requested"
|
||||||
|
? "已收到取消指令,当前分镜流程不会继续自动执行。"
|
||||||
|
: "收到修改意见,我会先停在草案阶段,等待你继续调整。");
|
||||||
|
let nextWorkflowStatus = params.action === "cancel_requested" ? "cancelled" : "draft";
|
||||||
|
|
||||||
|
if (params.action === "revision_requested" && params.workflowId) {
|
||||||
|
const revisionTaskDetail = await getVideoStoryboardTaskDetail(
|
||||||
|
params.workflowId,
|
||||||
|
).catch(() => null);
|
||||||
|
const revisionResult = await reviseStoryboardTaskByCommand({
|
||||||
|
taskId: params.workflowId,
|
||||||
|
command: revisionTaskDetail
|
||||||
|
? await resolveStoryboardRevisionCommand({
|
||||||
|
text: params.text,
|
||||||
|
detail: revisionTaskDetail,
|
||||||
|
})
|
||||||
|
: null,
|
||||||
|
onProgress: params.onProgress,
|
||||||
|
});
|
||||||
|
replyText = revisionResult.message || replyText;
|
||||||
|
nextWorkflowStatus = revisionResult.ok ? "awaiting_user_confirmation" : "draft";
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
replyText,
|
||||||
|
nextWorkflowStatus,
|
||||||
|
videoMetadata: params.workflowId
|
||||||
|
? ({
|
||||||
|
storyboard: {
|
||||||
|
enabled: true,
|
||||||
|
taskId: params.workflowId,
|
||||||
|
action: params.action,
|
||||||
|
},
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
} as any)
|
||||||
|
: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildStoryboardConfirmedAck(taskId: string) {
|
||||||
|
return {
|
||||||
|
acknowledgeMessage: "已收到确认,开始按分镜逐段生成视频,请稍候。",
|
||||||
|
acknowledgeMetadata: {
|
||||||
|
storyboard: {
|
||||||
|
enabled: true,
|
||||||
|
taskId,
|
||||||
|
status: "confirmed",
|
||||||
|
},
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
} as any,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildStoryboardExecutionVideoMetadata(params: {
|
||||||
|
taskId: string;
|
||||||
|
status: string;
|
||||||
|
successCount: number;
|
||||||
|
failedCount: number;
|
||||||
|
}) {
|
||||||
|
return {
|
||||||
|
storyboard: {
|
||||||
|
enabled: true,
|
||||||
|
taskId: params.taskId,
|
||||||
|
successCount: params.successCount,
|
||||||
|
failedCount: params.failedCount,
|
||||||
|
},
|
||||||
|
status: params.status,
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
} as any;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function executeStoryboardConfirmedWorkflow(params: {
|
||||||
|
taskId: string;
|
||||||
|
onProgress?: (event: StoryboardProgressEvent) => void | Promise<void>;
|
||||||
|
}) {
|
||||||
|
const executionResult = await executeConfirmedStoryboardTask({
|
||||||
|
taskId: params.taskId,
|
||||||
|
onProgress: params.onProgress,
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
executionResult,
|
||||||
|
firstVideoUrl: executionResult.videoUrls[0] || "",
|
||||||
|
finalVideoMetadata: buildStoryboardExecutionVideoMetadata({
|
||||||
|
taskId: executionResult.taskId,
|
||||||
|
status: executionResult.status,
|
||||||
|
successCount: executionResult.successCount,
|
||||||
|
failedCount: executionResult.failedCount,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user