lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: unify agent workflow protocol
Some checks failed
Lint / build (20) (push) Has been cancelled
Details
Playwright Tests / test (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Your Name 2026-03-13 16:16:09 +08:00
parent 159137b3d7
commit 5a4e9fd582
9 changed files with 1267 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

639
app/(chat)/api/chat/route.ts
View File

@ -52,7 +52,6 @@ import {
executeDirectVideoGeneration,
generateVideo,
} from "@/lib/ai/tools/generate-video";
import { executeConfirmedStoryboardTask } from "@/lib/video-agent/execute-storyboard-task";
import { getEnabledTools } from "@/lib/ai/tools-registry";
import { isProductionEnvironment } from "@/lib/constants";
import {
@ -90,7 +89,6 @@ import {
} from "@/lib/db/schema";
import { generateCanvasImage } from "@/lib/canvas/generate-image";
import {
buildGridConfirmationMessage,
detectRequestedStoryboardGrid,
isGridConfirmationReply,
type PendingGridConfirmation,
@ -99,17 +97,22 @@ import {
bootstrapConversationStateProtocol,
composeConversationStateProtocol,
persistAgentPresetTurn,
persistConversationStateProtocol,
persistClarificationTurn,
setConversationStateProtocolEventObserver,
persistResolvedTargetTurn,
persistGridConfirmationTurn,
persistImageWorkflowTurn,
persistPromptOnlyTurn,
resolveGridConfirmationRequest,
resolveTargetExecutionOverride,
persistStoryboardConfirmedTurn,
persistStoryboardExecutionCompletedTurn,
persistStoryboardExecutionStartedTurn,
persistStoryboardRevisionTurn,
persistStoryboardAssistantTurn,
persistVideoWorkflowTurn,
persistVideoGenerationCompletedTurn,
persistVideoGenerationFailedTurn,
persistVideoGenerationStartedTurn,
persistWorkflowCancelledTurn,
} from "@/lib/ai/conversation-state-protocol";
import { resolveChatMediaInput } from "@/lib/ai/context-memory/media-input";
import { resolveTargetAssets } from "@/lib/ai/context-memory/target-resolution";
@ -120,26 +123,22 @@ import type {
import { ChatSDKError } from "@/lib/errors";
import { extractImageUrlsFromText } from "@/lib/image-url";
import {
buildImageWorkflowMemoryUpdate,
extractImageWorkflowMemory,
} from "@/lib/ai/image-workflow-memory";
import {
buildPromptMemoryUpdate,
} from "@/lib/ai/prompt-memory";
resolveClarificationBypass,
resolveClarificationMessage,
} from "@/lib/ai/runtime/clarification-interruption";
import { uploadFile } from "@/lib/storage/service";
import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline";
import {
reviseStoryboardTaskByCommand,
resolveStoryboardRevisionCommand,
} from "@/lib/video-agent/revise-storyboard-task";
buildStoryboardConfirmedAck,
executeStoryboardConfirmedWorkflow,
handleStoryboardWorkflowFollowup,
} from "@/lib/video-agent/storyboard-chat-followup";
import {
buildStoryboardPreferenceMemoryUpdate,
extractStoryboardPreferenceMemory,
} from "@/lib/video-agent/storyboard-preference-memory";
import {
attachVideoStoryboardTaskMessage,
getVideoStoryboardTaskDetail,
getLatestActiveVideoStoryboardTaskByChatId,
markVideoStoryboardAwaitingConfirmation,
markVideoStoryboardConfirmed,
@ -2459,6 +2458,11 @@ async function saveGeneratedImages(
relationType?: "generated_from" | "edited_from";
sourceType?: string;
projectId?: string | null;
userPrompt?: string;
optimizedPrompt?: string;
requestedImageCount?: number;
aspectRatio?: string;
imageSize?: string;
},
): Promise<void> {
// 匹配 Markdown 图片格式: ![alt](url)
@ -2559,30 +2563,31 @@ async function saveGeneratedImages(
if (savedOriginalUrls.length > 0) {
try {
const existingWorkingState = await getConversationWorkingStateByChatId({ chatId }).catch(
() => null
);
const existingImageWorkflowMemory = extractImageWorkflowMemory(existingWorkingState);
await persistConversationStateProtocol({
const resultAssetIds = savedAssetIds.length > 0 ? savedAssetIds : null;
await persistImageWorkflowTurn({
userId,
chatId,
currentFocusAssetIds: savedAssetIds,
projectId: options?.projectId || null,
currentIntent: options?.relationType === "edited_from" ? "image_edit" : "image_gen",
currentFocusAssetIds: resultAssetIds,
currentFocusImageUrls: savedOriginalUrls,
lastGeneratedAssetIds: savedAssetIds,
lastGeneratedAssetIds: resultAssetIds,
lastGeneratedImageUrls: savedOriginalUrls,
lastResolvedTargetAssetIds: savedAssetIds,
lastResolvedTargetAssetIds: resultAssetIds,
lastResolvedTargetImageUrls: savedOriginalUrls,
lastMessageId: messageId,
lastTurnSummaryPatch: {
imageWorkflowMemory: buildImageWorkflowMemoryUpdate({
existing: existingImageWorkflowMemory,
lastIntent: options?.relationType === "edited_from" ? "image_edit" : "image_gen",
lastUserPrompt: prompt,
lastRequestedImageCount: savedOriginalUrls.length,
lastSourceImageUrls: options?.sourceImageUrls || null,
lastResultImageUrls: savedOriginalUrls,
}),
},
lastUserPrompt: String(options?.userPrompt || prompt || "").trim() || prompt,
lastOptimizedPrompt:
String(
options?.optimizedPrompt ||
(options?.relationType !== "edited_from" ? prompt : "") ||
"",
).trim() || undefined,
lastRequestedImageCount: options?.requestedImageCount || savedOriginalUrls.length,
lastAspectRatio: options?.aspectRatio,
lastImageSize: options?.imageSize,
lastSourceImageUrls: options?.sourceImageUrls || null,
lastResultImageUrls: savedOriginalUrls,
});
} catch (memoryError) {
console.warn("[Gallery] Failed to update working state with generated images:", memoryError);
@ -3163,6 +3168,8 @@ export async function POST(request: Request) {
console.log(
`[Smart Auto Mode] 🔐 Force agent conversation due to active storyboard task: ${activeStoryboardTaskForChat.task.id}`
);
} else if (resolvedConversationAction.kind === "image_generation_followup") {
console.log("[Smart Auto Mode] 🪄 Force agent conversation due to image generation follow-up workflow");
} else if (resolvedConversationAction.kind === "image_edit_followup") {
console.log("[Smart Auto Mode] 🖼️ Force agent conversation due to image edit follow-up workflow");
}
@ -4662,51 +4669,35 @@ export async function POST(request: Request) {
}
if (shouldRunAgentConversation && smartAutoNeedsClarification) {
const hasCurrentImageInput = message.parts.some((part: any) =>
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
);
const hasHistoryImageInput = smartAutoHistoryImages.length > 0;
const latestUserText = message.parts
.filter((part): part is { type: "text"; text: string } => part.type === "text")
.map((part) => part.text)
.join(" ")
.trim();
const clarificationBypass = resolveClarificationBypass({
text: message.parts
.filter((part): part is { type: "text"; text: string } => part.type === "text")
.map((part) => part.text)
.join(" ")
.trim(),
hasCurrentImageInput: message.parts.some((part: any) =>
part?.type === "file" || part?.type === "image" || part?.type === "image_url"
),
hasHistoryImageInput: smartAutoHistoryImages.length > 0,
hasCanvasSelection: Boolean(contextPayload?.selectedNodeIds?.length),
targetResolutionMode: requestTargetResolution?.resolutionMode,
executionMode: smartAutoExecution?.mode,
friendlyMessage: smartAutoFriendlyMessage,
});
if (hasCurrentImageInput || hasHistoryImageInput || latestUserText.length > 0) {
const canForceByIntent = /改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格/.test(
latestUserText
);
const hasForceTargetHint =
Boolean(contextPayload?.selectedNodeIds?.length) ||
requestTargetResolution?.resolutionMode === "text_reference" ||
/第[一二三四五六七八九十\d]+张|左|右|上|下|中间|选中/.test(latestUserText);
const canForceExecution =
(smartAutoExecution?.mode === "edit_image" ||
smartAutoExecution?.mode === "generate_image" ||
smartAutoExecution?.mode === "generate_video") &&
canForceByIntent &&
hasForceTargetHint;
if (canForceExecution) {
smartAutoNeedsClarification = false;
smartAutoClarificationQuestion = undefined;
if (
!smartAutoFriendlyMessage ||
/[?？]$/.test(smartAutoFriendlyMessage.trim()) ||
/哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage)
) {
smartAutoFriendlyMessage = "收到，我将按你的当前需求直接执行。";
}
console.log("[Smart Auto Mode] ✅ Force direct execution (skip clarification)");
}
if (clarificationBypass.shouldBypass) {
smartAutoNeedsClarification = false;
smartAutoClarificationQuestion = undefined;
smartAutoFriendlyMessage = clarificationBypass.nextFriendlyMessage;
console.log("[Smart Auto Mode] ✅ Force direct execution (skip clarification)");
}
}
if (shouldRunAgentConversation && smartAutoNeedsClarification) {
const clarificationMessage =
smartAutoClarificationQuestion ||
smartAutoFriendlyMessage ||
"我先确认一下，你希望我优先处理哪一部分？";
const clarificationMessage = resolveClarificationMessage({
clarificationQuestion: smartAutoClarificationQuestion,
friendlyMessage: smartAutoFriendlyMessage,
});
const clarificationStream = createUIMessageStream({
execute: async ({ writer }) => {
@ -4738,22 +4729,25 @@ export async function POST(request: Request) {
],
});
if (shouldAskGridConfirmation && pendingGridConfirmation) {
try {
await persistGridConfirmationTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
messageId,
text: requestTextContent,
pendingGridConfirmation,
});
} catch (gridConfirmationPersistError) {
console.warn(
"[Chat API] Failed to persist pending grid confirmation state:",
gridConfirmationPersistError
);
}
try {
await persistClarificationTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
messageId,
text: requestTextContent,
clarificationMessage,
currentIntent: smartAutoDetectedIntent || null,
pendingGridConfirmation:
shouldAskGridConfirmation && pendingGridConfirmation
? pendingGridConfirmation
: null,
});
} catch (clarificationPersistError) {
console.warn(
"[Chat API] Failed to persist clarification state:",
clarificationPersistError
);
}
},
});
@ -4770,40 +4764,23 @@ export async function POST(request: Request) {
const workflowReplyStream = createUIMessageStream({
execute: async ({ writer }) => {
const messageId = generateUUID();
let replyText =
smartAutoFriendlyMessage ||
(smartAutoWorkflowDecision?.action === "cancel_requested"
? "已收到取消指令，当前分镜流程不会继续自动执行。"
: "收到修改意见，我会先停在草案阶段，等待你继续调整。");
if (
smartAutoWorkflowDecision?.action === "revision_requested" &&
smartAutoWorkflowDecision.workflowId
) {
const revisionTaskDetail = await getVideoStoryboardTaskDetail(
smartAutoWorkflowDecision.workflowId
).catch(() => null);
const revisionResult = await reviseStoryboardTaskByCommand({
taskId: smartAutoWorkflowDecision.workflowId,
command: revisionTaskDetail
? await resolveStoryboardRevisionCommand({
text: requestTextContent,
detail: revisionTaskDetail,
})
: null,
onProgress: (event) => {
safeWriteStreamChunk(
writer,
{
type: "data-videoStoryboard",
data: JSON.stringify(event),
},
`storyboard-revision:${event.stage}`,
);
},
});
replyText = revisionResult.message || replyText;
}
const workflowFollowupResult = await handleStoryboardWorkflowFollowup({
action: smartAutoWorkflowDecision.action as "revision_requested" | "cancel_requested",
workflowId: smartAutoWorkflowDecision.workflowId,
text: requestTextContent,
friendlyMessage: smartAutoFriendlyMessage,
onProgress: (event) => {
safeWriteStreamChunk(
writer,
{
type: "data-videoStoryboard",
data: JSON.stringify(event),
},
`storyboard-revision:${event.stage}`,
);
},
});
const replyText = workflowFollowupResult.replyText;
writer.write({
type: "data-appendMessage",
@ -4828,20 +4805,43 @@ export async function POST(request: Request) {
temporaryVideoUrl: null,
permanentVideoUrl: null,
archivedAt: null,
videoMetadata:
smartAutoWorkflowDecision?.workflowId
? ({
storyboard: {
enabled: true,
taskId: smartAutoWorkflowDecision.workflowId,
action: smartAutoWorkflowDecision.action,
},
updatedAt: Date.now(),
} as any)
: null,
videoMetadata: workflowFollowupResult.videoMetadata,
},
],
});
try {
if (
smartAutoWorkflowDecision?.action === "revision_requested" &&
smartAutoWorkflowDecision.workflowId
) {
await persistStoryboardRevisionTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
messageId,
text: requestTextContent,
taskId: smartAutoWorkflowDecision.workflowId,
status: workflowFollowupResult.nextWorkflowStatus,
});
} else if (smartAutoWorkflowDecision?.action === "cancel_requested") {
await persistWorkflowCancelledTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
messageId,
text: requestTextContent,
workflowType: "storyboard_video",
workflowId: smartAutoWorkflowDecision?.workflowId,
status: workflowFollowupResult.nextWorkflowStatus,
});
}
} catch (workflowPersistError) {
console.warn(
"[Chat API] Failed to persist storyboard workflow interruption state:",
workflowPersistError
);
}
},
});
@ -4858,7 +4858,8 @@ export async function POST(request: Request) {
const confirmedStoryboardStream = createUIMessageStream({
execute: async ({ writer }) => {
const acknowledgeMessageId = generateUUID();
const acknowledgeMessage = "已收到确认，开始按分镜逐段生成视频，请稍候。";
const { acknowledgeMessage, acknowledgeMetadata } =
buildStoryboardConfirmedAck(confirmedTaskId);
writer.write({
type: "data-appendMessage",
@ -4883,14 +4884,7 @@ export async function POST(request: Request) {
temporaryVideoUrl: null,
permanentVideoUrl: null,
archivedAt: null,
videoMetadata: {
storyboard: {
enabled: true,
taskId: confirmedTaskId,
status: "confirmed",
},
updatedAt: Date.now(),
} as any,
videoMetadata: acknowledgeMetadata,
},
],
});
@ -4902,6 +4896,8 @@ export async function POST(request: Request) {
projectId: contextPayload?.projectId || null,
messageId: acknowledgeMessageId,
text: requestTextContent,
taskId: confirmedTaskId,
status: "confirmed",
});
} catch (storyboardConfirmMemoryError) {
console.warn(
@ -4910,19 +4906,37 @@ export async function POST(request: Request) {
);
}
const executionResult = await executeConfirmedStoryboardTask({
taskId: confirmedTaskId,
onProgress: (event) => {
safeWriteStreamChunk(
writer,
{
type: "data-videoStoryboard",
data: JSON.stringify(event),
},
`storyboard-confirmed:${event.stage}`,
);
},
});
try {
await persistStoryboardExecutionStartedTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
messageId: acknowledgeMessageId,
text: requestTextContent,
taskId: confirmedTaskId,
status: "running",
});
} catch (storyboardExecutionStartError) {
console.warn(
"[Chat API] Failed to persist storyboard execution start state:",
storyboardExecutionStartError
);
}
const { executionResult, firstVideoUrl, finalVideoMetadata } =
await executeStoryboardConfirmedWorkflow({
taskId: confirmedTaskId,
onProgress: (event) => {
safeWriteStreamChunk(
writer,
{
type: "data-videoStoryboard",
data: JSON.stringify(event),
},
`storyboard-confirmed:${event.stage}`,
);
},
});
const finalMessageId = generateUUID();
writer.write({
@ -4935,7 +4949,6 @@ export async function POST(request: Request) {
}),
});
const firstVideoUrl = executionResult.videoUrls[0] || "";
await saveMessages({
messages: [
{
@ -4949,16 +4962,7 @@ export async function POST(request: Request) {
temporaryVideoUrl: firstVideoUrl || null,
permanentVideoUrl: null,
archivedAt: null,
videoMetadata: {
storyboard: {
enabled: true,
taskId: executionResult.taskId,
successCount: executionResult.successCount,
failedCount: executionResult.failedCount,
},
status: executionResult.status,
updatedAt: Date.now(),
} as any,
videoMetadata: finalVideoMetadata,
},
],
});
@ -4973,6 +4977,26 @@ export async function POST(request: Request) {
);
});
try {
await persistStoryboardExecutionCompletedTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
messageId: finalMessageId,
text: requestTextContent,
taskId: executionResult.taskId,
status: executionResult.status,
successCount: executionResult.successCount,
failedCount: executionResult.failedCount,
firstVideoUrl: firstVideoUrl || undefined,
});
} catch (storyboardExecutionCompletedError) {
console.warn(
"[Chat API] Failed to persist storyboard execution completion state:",
storyboardExecutionCompletedError
);
}
if (firstVideoUrl) {
await consumeGeneratedVideoPoints({
userId: session.user.id,
@ -5263,6 +5287,9 @@ export async function POST(request: Request) {
sourceType:
sourceImageUrls.length > 0 ? "reference_generate" : "generated",
projectId: contextPayload?.projectId || null,
userPrompt: requestTextContent,
optimizedPrompt: slotPrompt || undefined,
requestedImageCount: 1,
},
);
@ -5372,13 +5399,18 @@ export async function POST(request: Request) {
});
try {
await persistVideoWorkflowTurn({
await persistVideoGenerationCompletedTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
lastMessageId: assistantMessageId,
messageId: assistantMessageId,
text: requestTextContent,
source: "agent_preset_direct_video",
providerType: "direct-video",
modelRef: result.modelRef || presetModelId,
mode,
status: "completed",
temporaryVideoUrl: videoUrl,
});
} catch (memoryError) {
console.warn(
@ -5458,6 +5490,27 @@ export async function POST(request: Request) {
},
],
});
if (selectedAgentPreset.type === "video") {
try {
await persistVideoGenerationFailedTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
messageId: assistantMessageId,
text: requestTextContent,
source: "agent_preset_direct_video",
providerType: "direct-video",
modelRef: presetModelId,
status: "failed",
error: slotError instanceof Error ? slotError.message : String(slotError),
});
} catch (videoMemoryError) {
console.warn(
"[Chat API] Failed to persist agent preset video failure state:",
videoMemoryError,
);
}
}
writeAssistantTextChunk(
writer,
errorMessage,
@ -5624,6 +5677,18 @@ export async function POST(request: Request) {
? "reference_generate"
: "generated",
projectId: contextPayload?.projectId || null,
userPrompt: requestTextContent,
optimizedPrompt: generationPrompt,
requestedImageCount:
effectiveImageParams?.imageCount ||
defaultImageQualityParams.imageCount ||
1,
aspectRatio:
effectiveImageParams?.aspectRatio ||
defaultImageQualityParams.aspectRatio,
imageSize:
effectiveImageParams?.imageSize ||
defaultImageQualityParams.imageSize,
},
);
@ -5804,13 +5869,18 @@ export async function POST(request: Request) {
});
try {
await persistVideoWorkflowTurn({
await persistVideoGenerationCompletedTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
lastMessageId: assistantMessageId,
messageId: assistantMessageId,
text: requestTextContent,
source: "agent_direct_generate_video",
providerType: "direct-video",
modelRef: result.modelRef || finalModelId,
mode,
status: "completed",
temporaryVideoUrl: videoUrl,
});
} catch (memoryError) {
console.warn(
@ -5878,6 +5948,28 @@ export async function POST(request: Request) {
},
],
});
try {
await persistVideoGenerationFailedTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
messageId: assistantMessageId,
text: requestTextContent,
source: "agent_direct_generate_video",
providerType: "direct-video",
modelRef: finalModelId,
status: "failed",
error:
directToolError instanceof Error
? directToolError.message
: String(directToolError),
});
} catch (videoMemoryError) {
console.warn(
"[Chat API] Failed to persist direct video failure state:",
videoMemoryError,
);
}
writeAssistantTextChunk(
writer,
errorMessage,
@ -6131,7 +6223,9 @@ export async function POST(request: Request) {
lastResolvedTargetImageUrls: resultImageUrls,
lastMessageId: assistantMessageId,
lastUserPrompt: requestTextContent,
lastOptimizedPrompt: smartAutoOptimizedPrompt || undefined,
lastRequestedImageCount: resultImageUrls?.length || 1,
lastImageSize: resultImageSize || undefined,
lastSourceImageUrls: directAgentToolTargetImageUrls,
lastResultImageUrls: resultImageUrls,
});
@ -6666,6 +6760,10 @@ export async function POST(request: Request) {
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
sourceType: requestResolvedMediaInput?.hasImage ? "generated" : "generated",
projectId: contextPayload?.projectId || null,
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
requestedImageCount: effectiveImageParams?.imageCount,
aspectRatio: effectiveImageParams?.aspectRatio,
imageSize: effectiveImageParams?.imageSize,
},
);
} catch (saveError) {
@ -6801,6 +6899,10 @@ export async function POST(request: Request) {
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
sourceType: "generated",
projectId: contextPayload?.projectId || null,
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
requestedImageCount: effectiveImageParams?.imageCount,
aspectRatio: effectiveImageParams?.aspectRatio,
imageSize: effectiveImageParams?.imageSize,
},
);
} catch (saveError) {
@ -7201,6 +7303,10 @@ export async function POST(request: Request) {
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
sourceType: "generated",
projectId: contextPayload?.projectId || null,
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
requestedImageCount: effectiveImageParams?.imageCount,
aspectRatio: effectiveImageParams?.aspectRatio,
imageSize: effectiveImageParams?.imageSize,
},
);
} catch (saveError) {
@ -7585,6 +7691,10 @@ export async function POST(request: Request) {
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
sourceType: "generated",
projectId: contextPayload?.projectId || null,
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
requestedImageCount: effectiveImageParams?.imageCount,
aspectRatio: effectiveImageParams?.aspectRatio,
imageSize: effectiveImageParams?.imageSize,
},
);
} catch (saveError) {
@ -7874,6 +7984,36 @@ export async function POST(request: Request) {
storyboardMemoryPersistError
);
}
} else {
try {
await persistVideoGenerationCompletedTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
messageId,
text: requestTextContent,
source: "native_video_provider",
providerType: currentVideoProvider,
modelRef: finalModelId,
mode:
typeof effectiveVideoParams?.mode === "string"
? effectiveVideoParams.mode
: undefined,
taskId: storyboardTaskId || undefined,
status:
typeof (videoMetadata as any)?.status === "string"
? (videoMetadata as any).status
: tempVideoUrl
? "completed"
: undefined,
temporaryVideoUrl: tempVideoUrl || undefined,
});
} catch (videoWorkflowPersistError) {
console.warn(
"[Chat API] Failed to persist native video workflow state:",
videoWorkflowPersistError,
);
}
}
const shouldConsumeVideoPoints =
tempVideoUrl &&
@ -7903,15 +8043,56 @@ export async function POST(request: Request) {
} catch (error) {
console.error(`[Chat API] ${currentVideoProvider} video generation error:`, error);
const errorMessageId = generateUUID();
const errorText = "视频生成失败，请稍后重试";
writer.write({
type: "data-appendMessage",
data: JSON.stringify({
id: errorMessageId,
role: "assistant",
parts: [{ type: "text", text: "视频生成失败，请稍后重试" }],
parts: [{ type: "text", text: errorText }],
attachments: [],
}),
});
try {
await saveMessages({
messages: [
{
id: errorMessageId,
chatId: id,
role: "assistant",
parts: [{ type: "text", text: errorText }],
attachments: [],
createdAt: new Date(),
videoStorageStatus: null,
temporaryVideoUrl: null,
permanentVideoUrl: null,
archivedAt: null,
videoMetadata: null,
},
],
});
await persistVideoGenerationFailedTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
messageId: errorMessageId,
text: requestTextContent,
source: "native_video_provider",
providerType: currentVideoProvider,
modelRef: finalModelId,
mode:
typeof effectiveVideoParams?.mode === "string"
? effectiveVideoParams.mode
: undefined,
status: "failed",
error: error instanceof Error ? error.message : String(error),
});
} catch (videoFailurePersistError) {
console.warn(
"[Chat API] Failed to persist native video failure state:",
videoFailurePersistError,
);
}
}
},
});
@ -8325,6 +8506,10 @@ export async function POST(request: Request) {
relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from",
sourceType: "generated",
projectId: contextPayload?.projectId || null,
optimizedPrompt: smartAutoOptimizedPrompt || userPrompt,
requestedImageCount: effectiveImageParams?.imageCount,
aspectRatio: effectiveImageParams?.aspectRatio,
imageSize: effectiveImageParams?.imageSize,
},
);
return;
@ -8674,6 +8859,25 @@ export async function POST(request: Request) {
},
],
});
try {
await persistVideoGenerationStartedTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
messageId: videoMessageId,
text: requestTextContent,
source: "stream_video_provider",
providerType: String(providerData.type || "stream"),
modelRef: finalVideoModelData.id || finalModelId,
mode: videoOptions.mode,
status: "running",
});
} catch (videoWorkflowStartError) {
console.warn(
"[Chat API] Failed to persist stream video start state:",
videoWorkflowStartError,
);
}
dataStream.write({
type: "data-appendMessage",
@ -8749,6 +8953,30 @@ export async function POST(request: Request) {
archivedAt: null,
videoMetadata: successMetadata as any,
});
try {
await persistVideoGenerationCompletedTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
messageId: videoMessageId,
text: requestTextContent,
source: "stream_video_provider",
providerType: String(providerData.type || "stream"),
modelRef: finalVideoModelData.id || finalModelId,
mode: videoOptions.mode,
taskId:
typeof successMetadata.taskId === "string" && successMetadata.taskId
? successMetadata.taskId
: undefined,
status: "completed",
temporaryVideoUrl: temporaryVideoUrl || undefined,
});
} catch (videoWorkflowCompletedError) {
console.warn(
"[Chat API] Failed to persist stream video completion state:",
videoWorkflowCompletedError,
);
}
await consumeGeneratedVideoPoints({
userId: session.user.id,
chatId: id,
@ -8807,6 +9035,29 @@ export async function POST(request: Request) {
archivedAt: null,
videoMetadata: failedMetadata as any,
});
try {
await persistVideoGenerationFailedTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
messageId: videoMessageId,
text: requestTextContent,
source: "stream_video_provider",
providerType: "stream",
modelRef: finalModelId,
mode:
typeof effectiveVideoParams?.mode === "string"
? effectiveVideoParams.mode
: undefined,
status: "failed",
error: error?.message || "未知错误",
});
} catch (videoWorkflowFailedError) {
console.warn(
"[Chat API] Failed to persist stream video failure state:",
videoWorkflowFailedError,
);
}
dataStream.write({
type: "data-appendMessage",
@ -8846,6 +9097,29 @@ export async function POST(request: Request) {
},
],
});
try {
await persistVideoGenerationFailedTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
messageId: fallbackMessageId,
text: requestTextContent,
source: "stream_video_provider",
providerType: "stream",
modelRef: finalModelId,
mode:
typeof effectiveVideoParams?.mode === "string"
? effectiveVideoParams.mode
: undefined,
status: "failed",
error: error?.message || "未知错误",
});
} catch (videoWorkflowFailedError) {
console.warn(
"[Chat API] Failed to persist fallback stream video failure state:",
videoWorkflowFailedError,
);
}
}
return;
@ -8971,6 +9245,25 @@ export async function POST(request: Request) {
archivedAt: null,
videoMetadata: genericVideoMetadata as any,
});
try {
await persistVideoGenerationCompletedTurn({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
messageId: msg.id,
text: requestTextContent,
source: "chat_stream_output",
providerType: "stream",
modelRef: actualModelId,
status: "completed",
temporaryVideoUrl: tempVideoUrl,
});
} catch (videoWorkflowCompletedError) {
console.warn(
"[Chat API] Failed to persist streamed assistant video state:",
videoWorkflowCompletedError,
);
}
scheduleVideoArchive({
chatId: id,

10
components/chat.tsx
View File

@ -325,6 +325,10 @@ export function Chat({
Array<{ id: string; at: number; payload: Record<string, unknown> }>
>([]);
useEffect(() => {
setProtocolDebugEvents([]);
}, [id]);
const { visibilityType } = useChatVisibility({
chatId: id,
initialVisibilityType,
@ -2094,7 +2098,11 @@ export function Chat({
/>
)}
</div>
<ProtocolDebugPanel enabled={protocolDebugEnabled} events={protocolDebugEvents} />
<ProtocolDebugPanel
enabled={protocolDebugEnabled}
events={protocolDebugEvents}
onClear={() => setProtocolDebugEvents([])}
/>
</div>
<Artifact

72
components/protocol-debug-panel.tsx
View File

@ -1,5 +1,7 @@
"use client";
import { useMemo } from "react";
type ProtocolDebugEvent = {
id: string;
at: number;
@ -9,12 +11,28 @@ type ProtocolDebugEvent = {
export function ProtocolDebugPanel({
enabled,
events,
onClear,
}: {
enabled: boolean;
events: ProtocolDebugEvent[];
onClear?: () => void;
}) {
if (!enabled) return null;
const groupedEvents = useMemo(() => {
const groups = new Map<string, ProtocolDebugEvent[]>();
for (const event of events) {
const key = String(event.payload.phase || event.payload.type || "unknown");
const list = groups.get(key) || [];
list.push(event);
groups.set(key, list);
}
return Array.from(groups.entries()).map(([phase, items]) => ({
phase,
items,
}));
}, [events]);
return (
<div className="mx-auto mt-2 w-full max-w-4xl px-2 md:px-4">
<details className="rounded-xl border border-border/60 bg-muted/30 p-3 text-xs">
@ -22,25 +40,58 @@ export function ProtocolDebugPanel({
Protocol Debug
</summary>
<div className="mt-3 space-y-2">
<div className="flex flex-wrap items-center justify-between gap-2 rounded-lg border border-border/60 bg-background px-3 py-2">
<div className="flex flex-wrap items-center gap-2">
<span className="rounded-md bg-muted px-2 py-1 text-[11px] text-muted-foreground">
总事件 {events.length}
</span>
<span className="rounded-md bg-muted px-2 py-1 text-[11px] text-muted-foreground">
阶段 {groupedEvents.length}
</span>
</div>
<button
type="button"
className="rounded-md border border-border px-2 py-1 text-[11px] text-foreground hover:bg-muted"
onClick={(event) => {
event.preventDefault();
onClear?.();
}}
>
清空
</button>
</div>
{events.length === 0 ? (
<div className="rounded-lg border border-dashed border-border/60 px-3 py-2 text-muted-foreground">
暂无协议调试事件
</div>
) : (
events.map((event) => (
<div
key={event.id}
groupedEvents.map((group) => (
<details
key={group.phase}
className="overflow-hidden rounded-lg border border-border/60 bg-background"
open
>
<div className="border-b border-border/60 px-3 py-2 font-medium">
{new Date(event.at).toLocaleTimeString()}
<summary className="cursor-pointer border-b border-border/60 px-3 py-2 font-medium">
{group.phase}
{" · "}
{String(event.payload.phase || event.payload.type || "unknown")}
{group.items.length} 条
</summary>
<div className="space-y-2 px-2 py-2">
{group.items.map((event) => (
<div
key={event.id}
className="overflow-hidden rounded-md border border-border/60"
>
<div className="border-b border-border/60 px-3 py-1.5 text-[11px] font-medium text-foreground">
{new Date(event.at).toLocaleTimeString()}
</div>
<pre className="overflow-x-auto px-3 py-2 whitespace-pre-wrap break-words text-[11px] leading-relaxed text-muted-foreground">
{JSON.stringify(event.payload, null, 2)}
</pre>
</div>
))}
</div>
<pre className="overflow-x-auto px-3 py-2 whitespace-pre-wrap break-words text-[11px] leading-relaxed text-muted-foreground">
{JSON.stringify(event.payload, null, 2)}
</pre>
</div>
</details>
))
)}
</div>
@ -48,4 +99,3 @@ export function ProtocolDebugPanel({
</div>
);
}

19
lib/ai/conversation-action-resolver.ts
View File

@ -2,6 +2,7 @@ import "server-only";
import type { ImageWorkflowMemory } from "@/lib/ai/image-workflow-memory";
import { mentionsImageEditFollowup } from "@/lib/ai/image-edit-subagent";
import { mentionsImageGenerationFollowup } from "@/lib/ai/image-generation-subagent";
import { isPromptCarryForwardRequest } from "@/lib/ai/prompt-memory";
export interface PendingGridConfirmation {
@ -16,6 +17,7 @@ export type ConversationActionKind =
| "grid_confirmation_followup"
| "active_storyboard_followup"
| "image_edit_followup"
| "image_generation_followup"
| "prompt_carry_forward";
export interface ConversationActionResolution {
@ -175,7 +177,7 @@ export function resolveConversationAction(params: {
} satisfies ConversationActionResolution;
}
const usePromptCarryForward =
const promptCarryForwardRequested =
Boolean(params.confirmedPrompt) &&
(isConfirmedPromptGenerationRequest(params.text) ||
isPromptCarryForwardRequest({
@ -183,6 +185,21 @@ export function resolveConversationAction(params: {
hasImage: params.hasImageInputForPromptCarryForward,
}));
const hasImageGenerationFollowup =
params.imageWorkflowMemory?.lastIntent === "image_gen" &&
(mentionsImageGenerationFollowup(params.text) || promptCarryForwardRequested);
if (hasImageGenerationFollowup) {
return {
kind: "image_generation_followup",
forceAgentConversation: true,
shouldKeepSelectedModel: true,
usePromptCarryForward: promptCarryForwardRequested,
} satisfies ConversationActionResolution;
}
const usePromptCarryForward = promptCarryForwardRequested;
if (usePromptCarryForward) {
return {
kind: "prompt_carry_forward",

96
lib/ai/conversation-state-protocol-events.ts
View File

@ -29,12 +29,32 @@ export const conversationStateEventSchemas = {
originalPrompt: z.string().min(1),
}),
}),
clarification_requested: z.object({
userId: z.string().uuid(),
chatId: z.string().uuid(),
projectId: z.string().uuid().nullable().optional(),
messageId: z.string().uuid(),
text: z.string(),
clarificationMessage: z.string().min(1),
currentIntent: z.string().nullable().optional(),
}),
storyboard_confirmed: z.object({
userId: z.string().uuid(),
chatId: z.string().uuid(),
projectId: z.string().uuid().nullable().optional(),
messageId: z.string().uuid(),
text: z.string(),
taskId: z.string().min(1).optional(),
status: z.string().optional(),
}),
storyboard_execution_started: z.object({
userId: z.string().uuid(),
chatId: z.string().uuid(),
projectId: z.string().uuid().nullable().optional(),
messageId: z.string().uuid(),
text: z.string(),
taskId: z.string().min(1),
status: z.string().optional(),
}),
image_workflow_updated: z.object({
userId: z.string().uuid(),
@ -50,10 +70,75 @@ export const conversationStateEventSchemas = {
lastResolvedTargetImageUrls: nullableStringArraySchema,
lastMessageId: z.string().uuid(),
lastUserPrompt: z.string(),
lastOptimizedPrompt: z.string().optional(),
lastRequestedImageCount: z.number().int().min(1).max(12).optional(),
lastAspectRatio: z.string().optional(),
lastImageSize: z.string().optional(),
lastSourceImageUrls: nullableStringArraySchema,
lastResultImageUrls: nullableStringArraySchema,
}),
storyboard_revision_requested: z.object({
userId: z.string().uuid(),
chatId: z.string().uuid(),
projectId: z.string().uuid().nullable().optional(),
messageId: z.string().uuid(),
text: z.string(),
taskId: z.string().min(1),
status: z.string().optional(),
}),
storyboard_execution_completed: z.object({
userId: z.string().uuid(),
chatId: z.string().uuid(),
projectId: z.string().uuid().nullable().optional(),
messageId: z.string().uuid(),
text: z.string(),
taskId: z.string().min(1),
status: z.string().optional(),
successCount: z.number().int().min(0).optional(),
failedCount: z.number().int().min(0).optional(),
firstVideoUrl: z.string().optional(),
}),
video_generation_started: z.object({
userId: z.string().uuid(),
chatId: z.string().uuid(),
projectId: z.string().uuid().nullable().optional(),
messageId: z.string().uuid(),
text: z.string(),
source: z.string().min(1),
providerType: z.string().optional(),
modelRef: z.string().optional(),
mode: z.string().optional(),
taskId: z.string().optional(),
status: z.string().optional(),
}),
video_generation_completed: z.object({
userId: z.string().uuid(),
chatId: z.string().uuid(),
projectId: z.string().uuid().nullable().optional(),
messageId: z.string().uuid(),
text: z.string(),
source: z.string().min(1),
providerType: z.string().optional(),
modelRef: z.string().optional(),
mode: z.string().optional(),
taskId: z.string().optional(),
status: z.string().optional(),
temporaryVideoUrl: z.string().optional(),
}),
video_generation_failed: z.object({
userId: z.string().uuid(),
chatId: z.string().uuid(),
projectId: z.string().uuid().nullable().optional(),
messageId: z.string().uuid(),
text: z.string(),
source: z.string().min(1),
providerType: z.string().optional(),
modelRef: z.string().optional(),
mode: z.string().optional(),
taskId: z.string().optional(),
status: z.string().optional(),
error: z.string().optional(),
}),
target_resolved: z.object({
userId: z.string().uuid(),
chatId: z.string().uuid(),
@ -84,6 +169,16 @@ export const conversationStateEventSchemas = {
text: z.string().optional(),
status: z.string().optional(),
}),
workflow_cancelled: z.object({
userId: z.string().uuid(),
chatId: z.string().uuid(),
projectId: z.string().uuid().nullable().optional(),
messageId: z.string().uuid(),
text: z.string(),
workflowType: z.string().min(1),
workflowId: z.string().min(1).optional(),
status: z.string().optional(),
}),
agent_preset_completed: z.object({
userId: z.string().uuid(),
chatId: z.string().uuid(),
@ -120,4 +215,3 @@ export type ConversationStateEvent =
payload: ConversationStateEventPayload<K>;
};
}[ConversationStateEventType];

407
lib/ai/conversation-state-protocol.ts
View File

@ -393,6 +393,23 @@ export async function persistConversationStateEvent<T extends ConversationStateE
});
}
case "clarification_requested": {
const typed = parsed as ConversationStateEventPayload<"clarification_requested">;
return persistConversationStateProtocol({
userId: typed.userId,
chatId: typed.chatId,
projectId: typed.projectId,
currentIntent: typed.currentIntent,
lastMessageId: typed.messageId,
lastTurnSummaryPatch: {
text: typed.text.slice(0, 500),
resolutionMode: "clarification",
confidence: 80,
clarificationMessage: typed.clarificationMessage.slice(0, 1000),
},
});
}
case "storyboard_confirmed": {
const typed = parsed as ConversationStateEventPayload<"storyboard_confirmed">;
const existing = await getConversationWorkingStateByChatId({
@ -410,6 +427,9 @@ export async function persistConversationStateEvent<T extends ConversationStateE
text: typed.text.slice(0, 500),
resolutionMode: "storyboard_confirmed",
confidence: 95,
videoWorkflowStatus: typed.status || "confirmed",
storyboardTaskId: typed.taskId,
storyboardWorkflowAction: "confirmed",
storyboardMemory: buildStoryboardPreferenceMemoryUpdate({
existing: existingStoryboardMemory,
prefersConfirmation: true,
@ -418,12 +438,32 @@ export async function persistConversationStateEvent<T extends ConversationStateE
});
}
case "storyboard_execution_started": {
const typed = parsed as ConversationStateEventPayload<"storyboard_execution_started">;
return persistConversationStateProtocol({
userId: typed.userId,
chatId: typed.chatId,
projectId: typed.projectId,
currentIntent: "video_gen",
lastMessageId: typed.messageId,
lastTurnSummaryPatch: {
text: typed.text.slice(0, 500),
resolutionMode: "storyboard_execution_started",
confidence: 95,
videoWorkflowStatus: typed.status || "running",
storyboardTaskId: typed.taskId,
storyboardWorkflowAction: "execution_started",
},
});
}
case "image_workflow_updated": {
const typed = parsed as ConversationStateEventPayload<"image_workflow_updated">;
const existing = await getConversationWorkingStateByChatId({
chatId: typed.chatId,
}).catch(() => null);
const existingImageWorkflowMemory = extractImageWorkflowMemory(existing);
const nextOptimizedPrompt = typed.lastOptimizedPrompt?.trim();
return persistConversationStateProtocol({
userId: typed.userId,
@ -439,11 +479,22 @@ export async function persistConversationStateEvent<T extends ConversationStateE
lastResolvedTargetImageUrls: typed.lastResolvedTargetImageUrls,
lastMessageId: typed.lastMessageId,
lastTurnSummaryPatch: {
promptMemory: nextOptimizedPrompt
? buildPromptMemoryUpdate({
optimizedPrompt: nextOptimizedPrompt.slice(0, 1000),
sourceIntent: typed.currentIntent,
sourceText: typed.lastUserPrompt.slice(0, 500),
reusable: true,
})
: undefined,
imageWorkflowMemory: buildImageWorkflowMemoryUpdate({
existing: existingImageWorkflowMemory,
lastIntent: typed.currentIntent,
lastOptimizedPrompt: nextOptimizedPrompt?.slice(0, 1000),
lastUserPrompt: typed.lastUserPrompt.slice(0, 500),
lastRequestedImageCount: typed.lastRequestedImageCount,
lastAspectRatio: typed.lastAspectRatio?.slice(0, 32),
lastImageSize: typed.lastImageSize?.slice(0, 32),
lastSourceImageUrls: typed.lastSourceImageUrls || null,
lastResultImageUrls: typed.lastResultImageUrls || null,
}),
@ -451,6 +502,124 @@ export async function persistConversationStateEvent<T extends ConversationStateE
});
}
case "storyboard_revision_requested": {
const typed = parsed as ConversationStateEventPayload<"storyboard_revision_requested">;
const existing = await getConversationWorkingStateByChatId({
chatId: typed.chatId,
}).catch(() => null);
const existingStoryboardMemory = extractStoryboardPreferenceMemory(existing);
return persistConversationStateProtocol({
userId: typed.userId,
chatId: typed.chatId,
projectId: typed.projectId,
currentIntent: "video_gen",
lastMessageId: typed.messageId,
lastTurnSummaryPatch: {
text: typed.text.slice(0, 500),
resolutionMode: "storyboard_revision",
confidence: 90,
videoWorkflowStatus: typed.status || "awaiting_user_confirmation",
storyboardTaskId: typed.taskId,
storyboardWorkflowAction: "revision_requested",
storyboardMemory: buildStoryboardPreferenceMemoryUpdate({
existing: existingStoryboardMemory,
prefersConfirmation: true,
}),
},
});
}
case "storyboard_execution_completed": {
const typed = parsed as ConversationStateEventPayload<"storyboard_execution_completed">;
return persistConversationStateProtocol({
userId: typed.userId,
chatId: typed.chatId,
projectId: typed.projectId,
currentIntent: "video_gen",
lastMessageId: typed.messageId,
lastTurnSummaryPatch: {
text: typed.text.slice(0, 500),
resolutionMode: "storyboard_execution_completed",
confidence: 95,
videoWorkflowStatus: typed.status || "completed",
storyboardTaskId: typed.taskId,
storyboardWorkflowAction: "execution_completed",
successCount: typed.successCount,
failedCount: typed.failedCount,
firstVideoUrl: typed.firstVideoUrl,
},
});
}
case "video_generation_started": {
const typed = parsed as ConversationStateEventPayload<"video_generation_started">;
return persistConversationStateProtocol({
userId: typed.userId,
chatId: typed.chatId,
projectId: typed.projectId,
currentIntent: "video_gen",
lastMessageId: typed.messageId,
lastTurnSummaryPatch: {
text: typed.text.slice(0, 500),
resolutionMode: "video_generation_started",
confidence: 90,
videoWorkflowStatus: typed.status || "running",
videoWorkflowSource: typed.source,
videoProviderType: typed.providerType,
videoModelRef: typed.modelRef,
videoGenerationMode: typed.mode,
videoTaskId: typed.taskId,
},
});
}
case "video_generation_completed": {
const typed = parsed as ConversationStateEventPayload<"video_generation_completed">;
return persistConversationStateProtocol({
userId: typed.userId,
chatId: typed.chatId,
projectId: typed.projectId,
currentIntent: "video_gen",
lastMessageId: typed.messageId,
lastTurnSummaryPatch: {
text: typed.text.slice(0, 500),
resolutionMode: "video_generation_completed",
confidence: 95,
videoWorkflowStatus: typed.status || "completed",
videoWorkflowSource: typed.source,
videoProviderType: typed.providerType,
videoModelRef: typed.modelRef,
videoGenerationMode: typed.mode,
videoTaskId: typed.taskId,
videoTemporaryUrl: typed.temporaryVideoUrl,
},
});
}
case "video_generation_failed": {
const typed = parsed as ConversationStateEventPayload<"video_generation_failed">;
return persistConversationStateProtocol({
userId: typed.userId,
chatId: typed.chatId,
projectId: typed.projectId,
currentIntent: "video_gen",
lastMessageId: typed.messageId,
lastTurnSummaryPatch: {
text: typed.text.slice(0, 500),
resolutionMode: "video_generation_failed",
confidence: 85,
videoWorkflowStatus: typed.status || "failed",
videoWorkflowSource: typed.source,
videoProviderType: typed.providerType,
videoModelRef: typed.modelRef,
videoGenerationMode: typed.mode,
videoTaskId: typed.taskId,
videoWorkflowError: typed.error?.slice(0, 1000),
},
});
}
case "target_resolved": {
const typed = parsed as ConversationStateEventPayload<"target_resolved">;
return persistConversationStateProtocol({
@ -498,6 +667,26 @@ export async function persistConversationStateEvent<T extends ConversationStateE
});
}
case "workflow_cancelled": {
const typed = parsed as ConversationStateEventPayload<"workflow_cancelled">;
return persistConversationStateProtocol({
userId: typed.userId,
chatId: typed.chatId,
projectId: typed.projectId,
currentIntent: typed.workflowType === "storyboard_video" ? "video_gen" : undefined,
lastMessageId: typed.messageId,
lastTurnSummaryPatch: {
text: typed.text.slice(0, 500),
resolutionMode: "workflow_cancelled",
confidence: 95,
videoWorkflowStatus: typed.status || "cancelled",
cancelledWorkflowType: typed.workflowType,
cancelledWorkflowId: typed.workflowId,
workflowAction: "cancel_requested",
},
});
}
case "agent_preset_completed": {
const typed = parsed as ConversationStateEventPayload<"agent_preset_completed">;
return persistConversationStateProtocol({
@ -594,12 +783,46 @@ export async function persistGridConfirmationTurn(params: {
});
}
export async function persistClarificationTurn(params: {
userId: string;
chatId: string;
projectId?: string | null;
messageId: string;
text: string;
clarificationMessage: string;
currentIntent?: string | null;
pendingGridConfirmation?: PendingGridConfirmation | null;
}) {
if (params.pendingGridConfirmation) {
return persistConversationStateEvent("grid_confirmation_requested", {
userId: params.userId,
chatId: params.chatId,
projectId: params.projectId,
messageId: params.messageId,
text: params.text,
pendingGridConfirmation: params.pendingGridConfirmation,
});
}
return persistConversationStateEvent("clarification_requested", {
userId: params.userId,
chatId: params.chatId,
projectId: params.projectId,
messageId: params.messageId,
text: params.text,
clarificationMessage: params.clarificationMessage,
currentIntent: params.currentIntent,
});
}
export async function persistStoryboardConfirmedTurn(params: {
userId: string;
chatId: string;
projectId?: string | null;
messageId: string;
text: string;
taskId?: string;
status?: string;
}) {
return persistConversationStateEvent("storyboard_confirmed", {
userId: params.userId,
@ -607,6 +830,162 @@ export async function persistStoryboardConfirmedTurn(params: {
projectId: params.projectId,
messageId: params.messageId,
text: params.text,
taskId: params.taskId,
status: params.status,
});
}
export async function persistStoryboardExecutionStartedTurn(params: {
userId: string;
chatId: string;
projectId?: string | null;
messageId: string;
text: string;
taskId: string;
status?: string;
}) {
return persistConversationStateEvent("storyboard_execution_started", {
userId: params.userId,
chatId: params.chatId,
projectId: params.projectId,
messageId: params.messageId,
text: params.text,
taskId: params.taskId,
status: params.status,
});
}
export async function persistStoryboardRevisionTurn(params: {
userId: string;
chatId: string;
projectId?: string | null;
messageId: string;
text: string;
taskId: string;
status?: string;
}) {
return persistConversationStateEvent("storyboard_revision_requested", {
userId: params.userId,
chatId: params.chatId,
projectId: params.projectId,
messageId: params.messageId,
text: params.text,
taskId: params.taskId,
status: params.status,
});
}
export async function persistStoryboardExecutionCompletedTurn(params: {
userId: string;
chatId: string;
projectId?: string | null;
messageId: string;
text: string;
taskId: string;
status?: string;
successCount?: number;
failedCount?: number;
firstVideoUrl?: string;
}) {
return persistConversationStateEvent("storyboard_execution_completed", {
userId: params.userId,
chatId: params.chatId,
projectId: params.projectId,
messageId: params.messageId,
text: params.text,
taskId: params.taskId,
status: params.status,
successCount: params.successCount,
failedCount: params.failedCount,
firstVideoUrl: params.firstVideoUrl,
});
}
export async function persistVideoGenerationStartedTurn(params: {
userId: string;
chatId: string;
projectId?: string | null;
messageId: string;
text: string;
source: string;
providerType?: string;
modelRef?: string;
mode?: string;
taskId?: string;
status?: string;
}) {
return persistConversationStateEvent("video_generation_started", {
userId: params.userId,
chatId: params.chatId,
projectId: params.projectId,
messageId: params.messageId,
text: params.text,
source: params.source,
providerType: params.providerType,
modelRef: params.modelRef,
mode: params.mode,
taskId: params.taskId,
status: params.status,
});
}
export async function persistVideoGenerationCompletedTurn(params: {
userId: string;
chatId: string;
projectId?: string | null;
messageId: string;
text: string;
source: string;
providerType?: string;
modelRef?: string;
mode?: string;
taskId?: string;
status?: string;
temporaryVideoUrl?: string;
}) {
return persistConversationStateEvent("video_generation_completed", {
userId: params.userId,
chatId: params.chatId,
projectId: params.projectId,
messageId: params.messageId,
text: params.text,
source: params.source,
providerType: params.providerType,
modelRef: params.modelRef,
mode: params.mode,
taskId: params.taskId,
status: params.status,
temporaryVideoUrl: params.temporaryVideoUrl,
});
}
export async function persistVideoGenerationFailedTurn(params: {
userId: string;
chatId: string;
projectId?: string | null;
messageId: string;
text: string;
source: string;
providerType?: string;
modelRef?: string;
mode?: string;
taskId?: string;
status?: string;
error?: string;
}) {
return persistConversationStateEvent("video_generation_failed", {
userId: params.userId,
chatId: params.chatId,
projectId: params.projectId,
messageId: params.messageId,
text: params.text,
source: params.source,
providerType: params.providerType,
modelRef: params.modelRef,
mode: params.mode,
taskId: params.taskId,
status: params.status,
error: params.error,
});
}
@ -624,7 +1003,10 @@ export async function persistImageWorkflowTurn(params: {
lastResolvedTargetImageUrls?: string[] | null;
lastMessageId: string;
lastUserPrompt: string;
lastOptimizedPrompt?: string;
lastRequestedImageCount?: number;
lastAspectRatio?: string;
lastImageSize?: string;
lastSourceImageUrls?: string[] | null;
lastResultImageUrls?: string[] | null;
}) {
@ -642,7 +1024,10 @@ export async function persistImageWorkflowTurn(params: {
lastResolvedTargetImageUrls: params.lastResolvedTargetImageUrls,
lastMessageId: params.lastMessageId,
lastUserPrompt: params.lastUserPrompt,
lastOptimizedPrompt: params.lastOptimizedPrompt,
lastRequestedImageCount: params.lastRequestedImageCount,
lastAspectRatio: params.lastAspectRatio,
lastImageSize: params.lastImageSize,
lastSourceImageUrls: params.lastSourceImageUrls || null,
lastResultImageUrls: params.lastResultImageUrls || null,
});
@ -684,6 +1069,28 @@ export async function persistVideoWorkflowTurn(params: {
return persistConversationStateEvent("video_workflow_updated", params);
}
export async function persistWorkflowCancelledTurn(params: {
userId: string;
chatId: string;
projectId?: string | null;
messageId: string;
text: string;
workflowType: string;
workflowId?: string;
status?: string;
}) {
return persistConversationStateEvent("workflow_cancelled", {
userId: params.userId,
chatId: params.chatId,
projectId: params.projectId,
messageId: params.messageId,
text: params.text,
workflowType: params.workflowType,
workflowId: params.workflowId,
status: params.status,
});
}
export async function persistAgentPresetTurn(params: {
userId: string;
chatId: string;

35
lib/ai/image-generation-subagent.ts
View File

@ -16,6 +16,15 @@ function normalizeCompactText(text: string) {
return String(text || "").toLowerCase().replace(/\s+/g, "").trim();
}
export function mentionsImageGenerationFollowup(text: string): boolean {
const normalized = normalizeCompactText(text);
if (!normalized) return false;
return /继续生成|继续出图|继续做图|再来(一|两|三|四|五|六|七|八|九|\d+)?张|再出(一|两|三|四|五|六|七|八|九|\d+)?张|再做(一|两|三|四|五|六|七|八|九|\d+)?组|再给我(一|两|三|四|五|六|七|八|九|\d+)?张|补(一|两|三|四|五|六|七|八|九|\d+)?张|多来(一|两|三|四|五|六|七|八|九|\d+)?张|换一版|再换一版|同风格再来|同样风格再来|保持.*(不变|一致).*再来|同提示词再来|沿用上一个效果再来/.test(
normalized,
);
}
export function buildImageGenerationSpecialistRoute(params: {
text: string;
threadState?: AgentThreadState | null;
@ -30,11 +39,18 @@ export function buildImageGenerationSpecialistRoute(params: {
text: params.text,
hasImage: params.hasImage,
});
const mentionsGenerationFollowup = mentionsImageGenerationFollowup(params.text);
const mentionsPromptExecution =
/提示词|prompt|关键词|咒语/.test(normalized) &&
/生成|生图|出图|继续做|继续生成|执行/.test(normalized);
if (!hasPromptMemory && !hasImageWorkflowMemory && !carryForward && !mentionsPromptExecution) {
if (
!hasPromptMemory &&
!hasImageWorkflowMemory &&
!carryForward &&
!mentionsPromptExecution &&
!mentionsGenerationFollowup
) {
return null;
}
@ -42,6 +58,9 @@ export function buildImageGenerationSpecialistRoute(params: {
carryForward
? "用户当前要求沿用上一轮优化提示词继续执行，应优先理解为连续生图执行。"
: "",
mentionsGenerationFollowup
? "如果用户说“再来几张 / 换一版 / 同风格再来”，优先理解为沿用上一轮生图上下文继续执行，而不是开启全新任务。"
: "",
hasPromptMemory
? "当前会话里存在上一轮可复用提示词，如果用户在说“上一个提示词/刚才那个 prompt”，优先复用这段提示词。"
: "",
@ -58,7 +77,11 @@ export function buildImageGenerationSpecialistRoute(params: {
return {
domain: "image_generation_memory",
specialistId: "image-generation-subagent",
reason: carryForward ? "当前回合命中提示词接力生图" : "当前回合命中提示词相关的连续生图任务",
reason: carryForward
? "当前回合命中提示词接力生图"
: mentionsGenerationFollowup
? "当前回合命中连续生图跟进语义"
: "当前回合命中提示词相关的连续生图任务",
planningHints,
toolAllowlist: [
"respondDirect",
@ -101,9 +124,15 @@ export function refinePlanWithImageGenerationSubagent(params: {
text: params.text,
hasImage: params.hasImage,
});
const mentionsGenerationFollowup = mentionsImageGenerationFollowup(params.text);
const hasPromptMemory = getPromptMemoryPlanningHints(params.threadState).length > 0;
const hasImageWorkflowMemory = getImageWorkflowPlanningHints(params.threadState).length > 0;
if (!carryForward && !hasPromptMemory && !hasImageWorkflowMemory) {
if (
!carryForward &&
!mentionsGenerationFollowup &&
!hasPromptMemory &&
!hasImageWorkflowMemory
) {
return params.plan;
}

69
lib/ai/runtime/clarification-interruption.ts Normal file
View File

@ -0,0 +1,69 @@
import "server-only";
function shouldReplaceClarificationLikeMessage(message?: string) {
if (!message) return true;
const normalized = message.trim();
if (!normalized) return true;
return /[?？]$/.test(normalized) || /哪一张|哪个|请先确认/.test(normalized);
}
export function resolveClarificationBypass(params: {
text: string;
hasCurrentImageInput: boolean;
hasHistoryImageInput: boolean;
hasCanvasSelection: boolean;
targetResolutionMode?: string | null;
executionMode?: string;
friendlyMessage?: string;
}) {
const latestUserText = String(params.text || "").trim();
if (
!params.hasCurrentImageInput &&
!params.hasHistoryImageInput &&
latestUserText.length <= 0
) {
return {
shouldBypass: false,
nextFriendlyMessage: params.friendlyMessage,
};
}
const canForceByIntent = /改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格/.test(
latestUserText,
);
const hasForceTargetHint =
params.hasCanvasSelection ||
params.targetResolutionMode === "text_reference" ||
/第[一二三四五六七八九十\d]+张|左|右|上|下|中间|选中/.test(latestUserText);
const canForceExecution =
(params.executionMode === "edit_image" ||
params.executionMode === "generate_image" ||
params.executionMode === "generate_video") &&
canForceByIntent &&
hasForceTargetHint;
if (!canForceExecution) {
return {
shouldBypass: false,
nextFriendlyMessage: params.friendlyMessage,
};
}
return {
shouldBypass: true,
nextFriendlyMessage: shouldReplaceClarificationLikeMessage(params.friendlyMessage)
? "收到，我将按你的当前需求直接执行。"
: params.friendlyMessage,
};
}
export function resolveClarificationMessage(params: {
clarificationQuestion?: string;
friendlyMessage?: string;
}) {
return (
params.clarificationQuestion ||
params.friendlyMessage ||
"我先确认一下，你希望我优先处理哪一部分？"
);
}

110
lib/video-agent/storyboard-chat-followup.ts Normal file
View File

@ -0,0 +1,110 @@
import "server-only";
import { executeConfirmedStoryboardTask } from "@/lib/video-agent/execute-storyboard-task";
import {
reviseStoryboardTaskByCommand,
resolveStoryboardRevisionCommand,
} from "@/lib/video-agent/revise-storyboard-task";
import { getVideoStoryboardTaskDetail } from "@/lib/video-agent/task-store";
import type { StoryboardProgressEvent } from "@/lib/video-agent/types";
export async function handleStoryboardWorkflowFollowup(params: {
action: "revision_requested" | "cancel_requested";
workflowId?: string;
text: string;
friendlyMessage?: string;
onProgress?: (event: StoryboardProgressEvent) => void | Promise<void>;
}) {
let replyText =
params.friendlyMessage ||
(params.action === "cancel_requested"
? "已收到取消指令，当前分镜流程不会继续自动执行。"
: "收到修改意见，我会先停在草案阶段，等待你继续调整。");
let nextWorkflowStatus = params.action === "cancel_requested" ? "cancelled" : "draft";
if (params.action === "revision_requested" && params.workflowId) {
const revisionTaskDetail = await getVideoStoryboardTaskDetail(
params.workflowId,
).catch(() => null);
const revisionResult = await reviseStoryboardTaskByCommand({
taskId: params.workflowId,
command: revisionTaskDetail
? await resolveStoryboardRevisionCommand({
text: params.text,
detail: revisionTaskDetail,
})
: null,
onProgress: params.onProgress,
});
replyText = revisionResult.message || replyText;
nextWorkflowStatus = revisionResult.ok ? "awaiting_user_confirmation" : "draft";
}
return {
replyText,
nextWorkflowStatus,
videoMetadata: params.workflowId
? ({
storyboard: {
enabled: true,
taskId: params.workflowId,
action: params.action,
},
updatedAt: Date.now(),
} as any)
: null,
};
}
export function buildStoryboardConfirmedAck(taskId: string) {
return {
acknowledgeMessage: "已收到确认，开始按分镜逐段生成视频，请稍候。",
acknowledgeMetadata: {
storyboard: {
enabled: true,
taskId,
status: "confirmed",
},
updatedAt: Date.now(),
} as any,
};
}
export function buildStoryboardExecutionVideoMetadata(params: {
taskId: string;
status: string;
successCount: number;
failedCount: number;
}) {
return {
storyboard: {
enabled: true,
taskId: params.taskId,
successCount: params.successCount,
failedCount: params.failedCount,
},
status: params.status,
updatedAt: Date.now(),
} as any;
}
export async function executeStoryboardConfirmedWorkflow(params: {
taskId: string;
onProgress?: (event: StoryboardProgressEvent) => void | Promise<void>;
}) {
const executionResult = await executeConfirmedStoryboardTask({
taskId: params.taskId,
onProgress: params.onProgress,
});
return {
executionResult,
firstVideoUrl: executionResult.videoUrls[0] || "",
finalVideoMetadata: buildStoryboardExecutionVideoMetadata({
taskId: executionResult.taskId,
status: executionResult.status,
successCount: executionResult.successCount,
failedCount: executionResult.failedCount,
}),
};
}