diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 4cc080bb0..988b37803 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -52,7 +52,6 @@ import { executeDirectVideoGeneration, generateVideo, } from "@/lib/ai/tools/generate-video"; -import { executeConfirmedStoryboardTask } from "@/lib/video-agent/execute-storyboard-task"; import { getEnabledTools } from "@/lib/ai/tools-registry"; import { isProductionEnvironment } from "@/lib/constants"; import { @@ -90,7 +89,6 @@ import { } from "@/lib/db/schema"; import { generateCanvasImage } from "@/lib/canvas/generate-image"; import { - buildGridConfirmationMessage, detectRequestedStoryboardGrid, isGridConfirmationReply, type PendingGridConfirmation, @@ -99,17 +97,22 @@ import { bootstrapConversationStateProtocol, composeConversationStateProtocol, persistAgentPresetTurn, - persistConversationStateProtocol, + persistClarificationTurn, setConversationStateProtocolEventObserver, persistResolvedTargetTurn, - persistGridConfirmationTurn, persistImageWorkflowTurn, persistPromptOnlyTurn, resolveGridConfirmationRequest, resolveTargetExecutionOverride, persistStoryboardConfirmedTurn, + persistStoryboardExecutionCompletedTurn, + persistStoryboardExecutionStartedTurn, + persistStoryboardRevisionTurn, persistStoryboardAssistantTurn, - persistVideoWorkflowTurn, + persistVideoGenerationCompletedTurn, + persistVideoGenerationFailedTurn, + persistVideoGenerationStartedTurn, + persistWorkflowCancelledTurn, } from "@/lib/ai/conversation-state-protocol"; import { resolveChatMediaInput } from "@/lib/ai/context-memory/media-input"; import { resolveTargetAssets } from "@/lib/ai/context-memory/target-resolution"; @@ -120,26 +123,22 @@ import type { import { ChatSDKError } from "@/lib/errors"; import { extractImageUrlsFromText } from "@/lib/image-url"; import { - buildImageWorkflowMemoryUpdate, - extractImageWorkflowMemory, -} from "@/lib/ai/image-workflow-memory"; -import { - buildPromptMemoryUpdate, -} from "@/lib/ai/prompt-memory"; + resolveClarificationBypass, + resolveClarificationMessage, +} from "@/lib/ai/runtime/clarification-interruption"; import { uploadFile } from "@/lib/storage/service"; import { getVideoStoryboardConfig } from "@/lib/video-agent/config"; import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline"; import { - reviseStoryboardTaskByCommand, - resolveStoryboardRevisionCommand, -} from "@/lib/video-agent/revise-storyboard-task"; + buildStoryboardConfirmedAck, + executeStoryboardConfirmedWorkflow, + handleStoryboardWorkflowFollowup, +} from "@/lib/video-agent/storyboard-chat-followup"; import { buildStoryboardPreferenceMemoryUpdate, - extractStoryboardPreferenceMemory, } from "@/lib/video-agent/storyboard-preference-memory"; import { attachVideoStoryboardTaskMessage, - getVideoStoryboardTaskDetail, getLatestActiveVideoStoryboardTaskByChatId, markVideoStoryboardAwaitingConfirmation, markVideoStoryboardConfirmed, @@ -2459,6 +2458,11 @@ async function saveGeneratedImages( relationType?: "generated_from" | "edited_from"; sourceType?: string; projectId?: string | null; + userPrompt?: string; + optimizedPrompt?: string; + requestedImageCount?: number; + aspectRatio?: string; + imageSize?: string; }, ): Promise { // 匹配 Markdown 图片格式: ![alt](url) @@ -2559,30 +2563,31 @@ async function saveGeneratedImages( if (savedOriginalUrls.length > 0) { try { - const existingWorkingState = await getConversationWorkingStateByChatId({ chatId }).catch( - () => null - ); - const existingImageWorkflowMemory = extractImageWorkflowMemory(existingWorkingState); - await persistConversationStateProtocol({ + const resultAssetIds = savedAssetIds.length > 0 ? savedAssetIds : null; + await persistImageWorkflowTurn({ userId, chatId, - currentFocusAssetIds: savedAssetIds, + projectId: options?.projectId || null, + currentIntent: options?.relationType === "edited_from" ? "image_edit" : "image_gen", + currentFocusAssetIds: resultAssetIds, currentFocusImageUrls: savedOriginalUrls, - lastGeneratedAssetIds: savedAssetIds, + lastGeneratedAssetIds: resultAssetIds, lastGeneratedImageUrls: savedOriginalUrls, - lastResolvedTargetAssetIds: savedAssetIds, + lastResolvedTargetAssetIds: resultAssetIds, lastResolvedTargetImageUrls: savedOriginalUrls, lastMessageId: messageId, - lastTurnSummaryPatch: { - imageWorkflowMemory: buildImageWorkflowMemoryUpdate({ - existing: existingImageWorkflowMemory, - lastIntent: options?.relationType === "edited_from" ? "image_edit" : "image_gen", - lastUserPrompt: prompt, - lastRequestedImageCount: savedOriginalUrls.length, - lastSourceImageUrls: options?.sourceImageUrls || null, - lastResultImageUrls: savedOriginalUrls, - }), - }, + lastUserPrompt: String(options?.userPrompt || prompt || "").trim() || prompt, + lastOptimizedPrompt: + String( + options?.optimizedPrompt || + (options?.relationType !== "edited_from" ? prompt : "") || + "", + ).trim() || undefined, + lastRequestedImageCount: options?.requestedImageCount || savedOriginalUrls.length, + lastAspectRatio: options?.aspectRatio, + lastImageSize: options?.imageSize, + lastSourceImageUrls: options?.sourceImageUrls || null, + lastResultImageUrls: savedOriginalUrls, }); } catch (memoryError) { console.warn("[Gallery] Failed to update working state with generated images:", memoryError); @@ -3163,6 +3168,8 @@ export async function POST(request: Request) { console.log( `[Smart Auto Mode] 🔐 Force agent conversation due to active storyboard task: ${activeStoryboardTaskForChat.task.id}` ); + } else if (resolvedConversationAction.kind === "image_generation_followup") { + console.log("[Smart Auto Mode] 🪄 Force agent conversation due to image generation follow-up workflow"); } else if (resolvedConversationAction.kind === "image_edit_followup") { console.log("[Smart Auto Mode] 🖼️ Force agent conversation due to image edit follow-up workflow"); } @@ -4662,51 +4669,35 @@ export async function POST(request: Request) { } if (shouldRunAgentConversation && smartAutoNeedsClarification) { - const hasCurrentImageInput = message.parts.some((part: any) => - part?.type === "file" || part?.type === "image" || part?.type === "image_url" - ); - const hasHistoryImageInput = smartAutoHistoryImages.length > 0; - const latestUserText = message.parts - .filter((part): part is { type: "text"; text: string } => part.type === "text") - .map((part) => part.text) - .join(" ") - .trim(); + const clarificationBypass = resolveClarificationBypass({ + text: message.parts + .filter((part): part is { type: "text"; text: string } => part.type === "text") + .map((part) => part.text) + .join(" ") + .trim(), + hasCurrentImageInput: message.parts.some((part: any) => + part?.type === "file" || part?.type === "image" || part?.type === "image_url" + ), + hasHistoryImageInput: smartAutoHistoryImages.length > 0, + hasCanvasSelection: Boolean(contextPayload?.selectedNodeIds?.length), + targetResolutionMode: requestTargetResolution?.resolutionMode, + executionMode: smartAutoExecution?.mode, + friendlyMessage: smartAutoFriendlyMessage, + }); - if (hasCurrentImageInput || hasHistoryImageInput || latestUserText.length > 0) { - const canForceByIntent = /改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格/.test( - latestUserText - ); - const hasForceTargetHint = - Boolean(contextPayload?.selectedNodeIds?.length) || - requestTargetResolution?.resolutionMode === "text_reference" || - /第[一二三四五六七八九十\d]+张|左|右|上|下|中间|选中/.test(latestUserText); - const canForceExecution = - (smartAutoExecution?.mode === "edit_image" || - smartAutoExecution?.mode === "generate_image" || - smartAutoExecution?.mode === "generate_video") && - canForceByIntent && - hasForceTargetHint; - - if (canForceExecution) { - smartAutoNeedsClarification = false; - smartAutoClarificationQuestion = undefined; - if ( - !smartAutoFriendlyMessage || - /[?？]$/.test(smartAutoFriendlyMessage.trim()) || - /哪一张|哪个|请先确认/.test(smartAutoFriendlyMessage) - ) { - smartAutoFriendlyMessage = "收到，我将按你的当前需求直接执行。"; - } - console.log("[Smart Auto Mode] ✅ Force direct execution (skip clarification)"); - } + if (clarificationBypass.shouldBypass) { + smartAutoNeedsClarification = false; + smartAutoClarificationQuestion = undefined; + smartAutoFriendlyMessage = clarificationBypass.nextFriendlyMessage; + console.log("[Smart Auto Mode] ✅ Force direct execution (skip clarification)"); } } if (shouldRunAgentConversation && smartAutoNeedsClarification) { - const clarificationMessage = - smartAutoClarificationQuestion || - smartAutoFriendlyMessage || - "我先确认一下，你希望我优先处理哪一部分？"; + const clarificationMessage = resolveClarificationMessage({ + clarificationQuestion: smartAutoClarificationQuestion, + friendlyMessage: smartAutoFriendlyMessage, + }); const clarificationStream = createUIMessageStream({ execute: async ({ writer }) => { @@ -4738,22 +4729,25 @@ export async function POST(request: Request) { ], }); - if (shouldAskGridConfirmation && pendingGridConfirmation) { - try { - await persistGridConfirmationTurn({ - userId: session.user.id, - chatId: id, - projectId: contextPayload?.projectId || null, - messageId, - text: requestTextContent, - pendingGridConfirmation, - }); - } catch (gridConfirmationPersistError) { - console.warn( - "[Chat API] Failed to persist pending grid confirmation state:", - gridConfirmationPersistError - ); - } + try { + await persistClarificationTurn({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + messageId, + text: requestTextContent, + clarificationMessage, + currentIntent: smartAutoDetectedIntent || null, + pendingGridConfirmation: + shouldAskGridConfirmation && pendingGridConfirmation + ? pendingGridConfirmation + : null, + }); + } catch (clarificationPersistError) { + console.warn( + "[Chat API] Failed to persist clarification state:", + clarificationPersistError + ); } }, }); @@ -4770,40 +4764,23 @@ export async function POST(request: Request) { const workflowReplyStream = createUIMessageStream({ execute: async ({ writer }) => { const messageId = generateUUID(); - let replyText = - smartAutoFriendlyMessage || - (smartAutoWorkflowDecision?.action === "cancel_requested" - ? "已收到取消指令，当前分镜流程不会继续自动执行。" - : "收到修改意见，我会先停在草案阶段，等待你继续调整。"); - - if ( - smartAutoWorkflowDecision?.action === "revision_requested" && - smartAutoWorkflowDecision.workflowId - ) { - const revisionTaskDetail = await getVideoStoryboardTaskDetail( - smartAutoWorkflowDecision.workflowId - ).catch(() => null); - const revisionResult = await reviseStoryboardTaskByCommand({ - taskId: smartAutoWorkflowDecision.workflowId, - command: revisionTaskDetail - ? await resolveStoryboardRevisionCommand({ - text: requestTextContent, - detail: revisionTaskDetail, - }) - : null, - onProgress: (event) => { - safeWriteStreamChunk( - writer, - { - type: "data-videoStoryboard", - data: JSON.stringify(event), - }, - `storyboard-revision:${event.stage}`, - ); - }, - }); - replyText = revisionResult.message || replyText; - } + const workflowFollowupResult = await handleStoryboardWorkflowFollowup({ + action: smartAutoWorkflowDecision.action as "revision_requested" | "cancel_requested", + workflowId: smartAutoWorkflowDecision.workflowId, + text: requestTextContent, + friendlyMessage: smartAutoFriendlyMessage, + onProgress: (event) => { + safeWriteStreamChunk( + writer, + { + type: "data-videoStoryboard", + data: JSON.stringify(event), + }, + `storyboard-revision:${event.stage}`, + ); + }, + }); + const replyText = workflowFollowupResult.replyText; writer.write({ type: "data-appendMessage", @@ -4828,20 +4805,43 @@ export async function POST(request: Request) { temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, - videoMetadata: - smartAutoWorkflowDecision?.workflowId - ? ({ - storyboard: { - enabled: true, - taskId: smartAutoWorkflowDecision.workflowId, - action: smartAutoWorkflowDecision.action, - }, - updatedAt: Date.now(), - } as any) - : null, + videoMetadata: workflowFollowupResult.videoMetadata, }, ], }); + + try { + if ( + smartAutoWorkflowDecision?.action === "revision_requested" && + smartAutoWorkflowDecision.workflowId + ) { + await persistStoryboardRevisionTurn({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + messageId, + text: requestTextContent, + taskId: smartAutoWorkflowDecision.workflowId, + status: workflowFollowupResult.nextWorkflowStatus, + }); + } else if (smartAutoWorkflowDecision?.action === "cancel_requested") { + await persistWorkflowCancelledTurn({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + messageId, + text: requestTextContent, + workflowType: "storyboard_video", + workflowId: smartAutoWorkflowDecision?.workflowId, + status: workflowFollowupResult.nextWorkflowStatus, + }); + } + } catch (workflowPersistError) { + console.warn( + "[Chat API] Failed to persist storyboard workflow interruption state:", + workflowPersistError + ); + } }, }); @@ -4858,7 +4858,8 @@ export async function POST(request: Request) { const confirmedStoryboardStream = createUIMessageStream({ execute: async ({ writer }) => { const acknowledgeMessageId = generateUUID(); - const acknowledgeMessage = "已收到确认，开始按分镜逐段生成视频，请稍候。"; + const { acknowledgeMessage, acknowledgeMetadata } = + buildStoryboardConfirmedAck(confirmedTaskId); writer.write({ type: "data-appendMessage", @@ -4883,14 +4884,7 @@ export async function POST(request: Request) { temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, - videoMetadata: { - storyboard: { - enabled: true, - taskId: confirmedTaskId, - status: "confirmed", - }, - updatedAt: Date.now(), - } as any, + videoMetadata: acknowledgeMetadata, }, ], }); @@ -4902,6 +4896,8 @@ export async function POST(request: Request) { projectId: contextPayload?.projectId || null, messageId: acknowledgeMessageId, text: requestTextContent, + taskId: confirmedTaskId, + status: "confirmed", }); } catch (storyboardConfirmMemoryError) { console.warn( @@ -4910,19 +4906,37 @@ export async function POST(request: Request) { ); } - const executionResult = await executeConfirmedStoryboardTask({ - taskId: confirmedTaskId, - onProgress: (event) => { - safeWriteStreamChunk( - writer, - { - type: "data-videoStoryboard", - data: JSON.stringify(event), - }, - `storyboard-confirmed:${event.stage}`, - ); - }, - }); + try { + await persistStoryboardExecutionStartedTurn({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + messageId: acknowledgeMessageId, + text: requestTextContent, + taskId: confirmedTaskId, + status: "running", + }); + } catch (storyboardExecutionStartError) { + console.warn( + "[Chat API] Failed to persist storyboard execution start state:", + storyboardExecutionStartError + ); + } + + const { executionResult, firstVideoUrl, finalVideoMetadata } = + await executeStoryboardConfirmedWorkflow({ + taskId: confirmedTaskId, + onProgress: (event) => { + safeWriteStreamChunk( + writer, + { + type: "data-videoStoryboard", + data: JSON.stringify(event), + }, + `storyboard-confirmed:${event.stage}`, + ); + }, + }); const finalMessageId = generateUUID(); writer.write({ @@ -4935,7 +4949,6 @@ export async function POST(request: Request) { }), }); - const firstVideoUrl = executionResult.videoUrls[0] || ""; await saveMessages({ messages: [ { @@ -4949,16 +4962,7 @@ export async function POST(request: Request) { temporaryVideoUrl: firstVideoUrl || null, permanentVideoUrl: null, archivedAt: null, - videoMetadata: { - storyboard: { - enabled: true, - taskId: executionResult.taskId, - successCount: executionResult.successCount, - failedCount: executionResult.failedCount, - }, - status: executionResult.status, - updatedAt: Date.now(), - } as any, + videoMetadata: finalVideoMetadata, }, ], }); @@ -4973,6 +4977,26 @@ export async function POST(request: Request) { ); }); + try { + await persistStoryboardExecutionCompletedTurn({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + messageId: finalMessageId, + text: requestTextContent, + taskId: executionResult.taskId, + status: executionResult.status, + successCount: executionResult.successCount, + failedCount: executionResult.failedCount, + firstVideoUrl: firstVideoUrl || undefined, + }); + } catch (storyboardExecutionCompletedError) { + console.warn( + "[Chat API] Failed to persist storyboard execution completion state:", + storyboardExecutionCompletedError + ); + } + if (firstVideoUrl) { await consumeGeneratedVideoPoints({ userId: session.user.id, @@ -5263,6 +5287,9 @@ export async function POST(request: Request) { sourceType: sourceImageUrls.length > 0 ? "reference_generate" : "generated", projectId: contextPayload?.projectId || null, + userPrompt: requestTextContent, + optimizedPrompt: slotPrompt || undefined, + requestedImageCount: 1, }, ); @@ -5372,13 +5399,18 @@ export async function POST(request: Request) { }); try { - await persistVideoWorkflowTurn({ + await persistVideoGenerationCompletedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, - lastMessageId: assistantMessageId, + messageId: assistantMessageId, text: requestTextContent, + source: "agent_preset_direct_video", + providerType: "direct-video", + modelRef: result.modelRef || presetModelId, + mode, status: "completed", + temporaryVideoUrl: videoUrl, }); } catch (memoryError) { console.warn( @@ -5458,6 +5490,27 @@ export async function POST(request: Request) { }, ], }); + if (selectedAgentPreset.type === "video") { + try { + await persistVideoGenerationFailedTurn({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + messageId: assistantMessageId, + text: requestTextContent, + source: "agent_preset_direct_video", + providerType: "direct-video", + modelRef: presetModelId, + status: "failed", + error: slotError instanceof Error ? slotError.message : String(slotError), + }); + } catch (videoMemoryError) { + console.warn( + "[Chat API] Failed to persist agent preset video failure state:", + videoMemoryError, + ); + } + } writeAssistantTextChunk( writer, errorMessage, @@ -5624,6 +5677,18 @@ export async function POST(request: Request) { ? "reference_generate" : "generated", projectId: contextPayload?.projectId || null, + userPrompt: requestTextContent, + optimizedPrompt: generationPrompt, + requestedImageCount: + effectiveImageParams?.imageCount || + defaultImageQualityParams.imageCount || + 1, + aspectRatio: + effectiveImageParams?.aspectRatio || + defaultImageQualityParams.aspectRatio, + imageSize: + effectiveImageParams?.imageSize || + defaultImageQualityParams.imageSize, }, ); @@ -5804,13 +5869,18 @@ export async function POST(request: Request) { }); try { - await persistVideoWorkflowTurn({ + await persistVideoGenerationCompletedTurn({ userId: session.user.id, chatId: id, projectId: contextPayload?.projectId || null, - lastMessageId: assistantMessageId, + messageId: assistantMessageId, text: requestTextContent, + source: "agent_direct_generate_video", + providerType: "direct-video", + modelRef: result.modelRef || finalModelId, + mode, status: "completed", + temporaryVideoUrl: videoUrl, }); } catch (memoryError) { console.warn( @@ -5878,6 +5948,28 @@ export async function POST(request: Request) { }, ], }); + try { + await persistVideoGenerationFailedTurn({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + messageId: assistantMessageId, + text: requestTextContent, + source: "agent_direct_generate_video", + providerType: "direct-video", + modelRef: finalModelId, + status: "failed", + error: + directToolError instanceof Error + ? directToolError.message + : String(directToolError), + }); + } catch (videoMemoryError) { + console.warn( + "[Chat API] Failed to persist direct video failure state:", + videoMemoryError, + ); + } writeAssistantTextChunk( writer, errorMessage, @@ -6131,7 +6223,9 @@ export async function POST(request: Request) { lastResolvedTargetImageUrls: resultImageUrls, lastMessageId: assistantMessageId, lastUserPrompt: requestTextContent, + lastOptimizedPrompt: smartAutoOptimizedPrompt || undefined, lastRequestedImageCount: resultImageUrls?.length || 1, + lastImageSize: resultImageSize || undefined, lastSourceImageUrls: directAgentToolTargetImageUrls, lastResultImageUrls: resultImageUrls, }); @@ -6666,6 +6760,10 @@ export async function POST(request: Request) { relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from", sourceType: requestResolvedMediaInput?.hasImage ? "generated" : "generated", projectId: contextPayload?.projectId || null, + optimizedPrompt: smartAutoOptimizedPrompt || userPrompt, + requestedImageCount: effectiveImageParams?.imageCount, + aspectRatio: effectiveImageParams?.aspectRatio, + imageSize: effectiveImageParams?.imageSize, }, ); } catch (saveError) { @@ -6801,6 +6899,10 @@ export async function POST(request: Request) { relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from", sourceType: "generated", projectId: contextPayload?.projectId || null, + optimizedPrompt: smartAutoOptimizedPrompt || userPrompt, + requestedImageCount: effectiveImageParams?.imageCount, + aspectRatio: effectiveImageParams?.aspectRatio, + imageSize: effectiveImageParams?.imageSize, }, ); } catch (saveError) { @@ -7201,6 +7303,10 @@ export async function POST(request: Request) { relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from", sourceType: "generated", projectId: contextPayload?.projectId || null, + optimizedPrompt: smartAutoOptimizedPrompt || userPrompt, + requestedImageCount: effectiveImageParams?.imageCount, + aspectRatio: effectiveImageParams?.aspectRatio, + imageSize: effectiveImageParams?.imageSize, }, ); } catch (saveError) { @@ -7585,6 +7691,10 @@ export async function POST(request: Request) { relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from", sourceType: "generated", projectId: contextPayload?.projectId || null, + optimizedPrompt: smartAutoOptimizedPrompt || userPrompt, + requestedImageCount: effectiveImageParams?.imageCount, + aspectRatio: effectiveImageParams?.aspectRatio, + imageSize: effectiveImageParams?.imageSize, }, ); } catch (saveError) { @@ -7874,6 +7984,36 @@ export async function POST(request: Request) { storyboardMemoryPersistError ); } + } else { + try { + await persistVideoGenerationCompletedTurn({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + messageId, + text: requestTextContent, + source: "native_video_provider", + providerType: currentVideoProvider, + modelRef: finalModelId, + mode: + typeof effectiveVideoParams?.mode === "string" + ? effectiveVideoParams.mode + : undefined, + taskId: storyboardTaskId || undefined, + status: + typeof (videoMetadata as any)?.status === "string" + ? (videoMetadata as any).status + : tempVideoUrl + ? "completed" + : undefined, + temporaryVideoUrl: tempVideoUrl || undefined, + }); + } catch (videoWorkflowPersistError) { + console.warn( + "[Chat API] Failed to persist native video workflow state:", + videoWorkflowPersistError, + ); + } } const shouldConsumeVideoPoints = tempVideoUrl && @@ -7903,15 +8043,56 @@ export async function POST(request: Request) { } catch (error) { console.error(`[Chat API] ${currentVideoProvider} video generation error:`, error); const errorMessageId = generateUUID(); + const errorText = "视频生成失败，请稍后重试"; writer.write({ type: "data-appendMessage", data: JSON.stringify({ id: errorMessageId, role: "assistant", - parts: [{ type: "text", text: "视频生成失败，请稍后重试" }], + parts: [{ type: "text", text: errorText }], attachments: [], }), }); + try { + await saveMessages({ + messages: [ + { + id: errorMessageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: errorText }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + await persistVideoGenerationFailedTurn({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + messageId: errorMessageId, + text: requestTextContent, + source: "native_video_provider", + providerType: currentVideoProvider, + modelRef: finalModelId, + mode: + typeof effectiveVideoParams?.mode === "string" + ? effectiveVideoParams.mode + : undefined, + status: "failed", + error: error instanceof Error ? error.message : String(error), + }); + } catch (videoFailurePersistError) { + console.warn( + "[Chat API] Failed to persist native video failure state:", + videoFailurePersistError, + ); + } } }, }); @@ -8325,6 +8506,10 @@ export async function POST(request: Request) { relationType: requestResolvedMediaInput?.hasImage ? "edited_from" : "generated_from", sourceType: "generated", projectId: contextPayload?.projectId || null, + optimizedPrompt: smartAutoOptimizedPrompt || userPrompt, + requestedImageCount: effectiveImageParams?.imageCount, + aspectRatio: effectiveImageParams?.aspectRatio, + imageSize: effectiveImageParams?.imageSize, }, ); return; @@ -8674,6 +8859,25 @@ export async function POST(request: Request) { }, ], }); + try { + await persistVideoGenerationStartedTurn({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + messageId: videoMessageId, + text: requestTextContent, + source: "stream_video_provider", + providerType: String(providerData.type || "stream"), + modelRef: finalVideoModelData.id || finalModelId, + mode: videoOptions.mode, + status: "running", + }); + } catch (videoWorkflowStartError) { + console.warn( + "[Chat API] Failed to persist stream video start state:", + videoWorkflowStartError, + ); + } dataStream.write({ type: "data-appendMessage", @@ -8749,6 +8953,30 @@ export async function POST(request: Request) { archivedAt: null, videoMetadata: successMetadata as any, }); + try { + await persistVideoGenerationCompletedTurn({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + messageId: videoMessageId, + text: requestTextContent, + source: "stream_video_provider", + providerType: String(providerData.type || "stream"), + modelRef: finalVideoModelData.id || finalModelId, + mode: videoOptions.mode, + taskId: + typeof successMetadata.taskId === "string" && successMetadata.taskId + ? successMetadata.taskId + : undefined, + status: "completed", + temporaryVideoUrl: temporaryVideoUrl || undefined, + }); + } catch (videoWorkflowCompletedError) { + console.warn( + "[Chat API] Failed to persist stream video completion state:", + videoWorkflowCompletedError, + ); + } await consumeGeneratedVideoPoints({ userId: session.user.id, chatId: id, @@ -8807,6 +9035,29 @@ export async function POST(request: Request) { archivedAt: null, videoMetadata: failedMetadata as any, }); + try { + await persistVideoGenerationFailedTurn({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + messageId: videoMessageId, + text: requestTextContent, + source: "stream_video_provider", + providerType: "stream", + modelRef: finalModelId, + mode: + typeof effectiveVideoParams?.mode === "string" + ? effectiveVideoParams.mode + : undefined, + status: "failed", + error: error?.message || "未知错误", + }); + } catch (videoWorkflowFailedError) { + console.warn( + "[Chat API] Failed to persist stream video failure state:", + videoWorkflowFailedError, + ); + } dataStream.write({ type: "data-appendMessage", @@ -8846,6 +9097,29 @@ export async function POST(request: Request) { }, ], }); + try { + await persistVideoGenerationFailedTurn({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + messageId: fallbackMessageId, + text: requestTextContent, + source: "stream_video_provider", + providerType: "stream", + modelRef: finalModelId, + mode: + typeof effectiveVideoParams?.mode === "string" + ? effectiveVideoParams.mode + : undefined, + status: "failed", + error: error?.message || "未知错误", + }); + } catch (videoWorkflowFailedError) { + console.warn( + "[Chat API] Failed to persist fallback stream video failure state:", + videoWorkflowFailedError, + ); + } } return; @@ -8971,6 +9245,25 @@ export async function POST(request: Request) { archivedAt: null, videoMetadata: genericVideoMetadata as any, }); + try { + await persistVideoGenerationCompletedTurn({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + messageId: msg.id, + text: requestTextContent, + source: "chat_stream_output", + providerType: "stream", + modelRef: actualModelId, + status: "completed", + temporaryVideoUrl: tempVideoUrl, + }); + } catch (videoWorkflowCompletedError) { + console.warn( + "[Chat API] Failed to persist streamed assistant video state:", + videoWorkflowCompletedError, + ); + } scheduleVideoArchive({ chatId: id, diff --git a/components/chat.tsx b/components/chat.tsx index e6c19ff1a..8133fd6e9 100644 --- a/components/chat.tsx +++ b/components/chat.tsx @@ -325,6 +325,10 @@ export function Chat({ Array<{ id: string; at: number; payload: Record }> >([]); + useEffect(() => { + setProtocolDebugEvents([]); + }, [id]); + const { visibilityType } = useChatVisibility({ chatId: id, initialVisibilityType, @@ -2094,7 +2098,11 @@ export function Chat({ /> )} - + setProtocolDebugEvents([])} + /> void; }) { if (!enabled) return null; + const groupedEvents = useMemo(() => { + const groups = new Map(); + for (const event of events) { + const key = String(event.payload.phase || event.payload.type || "unknown"); + const list = groups.get(key) || []; + list.push(event); + groups.set(key, list); + } + return Array.from(groups.entries()).map(([phase, items]) => ({ + phase, + items, + })); + }, [events]); + return (
@@ -22,25 +40,58 @@ export function ProtocolDebugPanel({ Protocol Debug
+
+
+ + 总事件 {events.length} + + + 阶段 {groupedEvents.length} + +
+ +
{events.length === 0 ? (
暂无协议调试事件
) : ( - events.map((event) => ( -
( +
-
- {new Date(event.at).toLocaleTimeString()} + + {group.phase} {" · "} - {String(event.payload.phase || event.payload.type || "unknown")} + {group.items.length} 条 + +
+ {group.items.map((event) => ( +
+
+ {new Date(event.at).toLocaleTimeString()} +
+ 
+                        {JSON.stringify(event.payload, null, 2)}
+
+
+ ))}
- 
-                  {JSON.stringify(event.payload, null, 2)}
-
-
+
)) )}
@@ -48,4 +99,3 @@ export function ProtocolDebugPanel({
); } - diff --git a/lib/ai/conversation-action-resolver.ts b/lib/ai/conversation-action-resolver.ts index a5955fea0..8ef59e5bc 100644 --- a/lib/ai/conversation-action-resolver.ts +++ b/lib/ai/conversation-action-resolver.ts @@ -2,6 +2,7 @@ import "server-only"; import type { ImageWorkflowMemory } from "@/lib/ai/image-workflow-memory"; import { mentionsImageEditFollowup } from "@/lib/ai/image-edit-subagent"; +import { mentionsImageGenerationFollowup } from "@/lib/ai/image-generation-subagent"; import { isPromptCarryForwardRequest } from "@/lib/ai/prompt-memory"; export interface PendingGridConfirmation { @@ -16,6 +17,7 @@ export type ConversationActionKind = | "grid_confirmation_followup" | "active_storyboard_followup" | "image_edit_followup" + | "image_generation_followup" | "prompt_carry_forward"; export interface ConversationActionResolution { @@ -175,7 +177,7 @@ export function resolveConversationAction(params: { } satisfies ConversationActionResolution; } - const usePromptCarryForward = + const promptCarryForwardRequested = Boolean(params.confirmedPrompt) && (isConfirmedPromptGenerationRequest(params.text) || isPromptCarryForwardRequest({ @@ -183,6 +185,21 @@ export function resolveConversationAction(params: { hasImage: params.hasImageInputForPromptCarryForward, })); + const hasImageGenerationFollowup = + params.imageWorkflowMemory?.lastIntent === "image_gen" && + (mentionsImageGenerationFollowup(params.text) || promptCarryForwardRequested); + + if (hasImageGenerationFollowup) { + return { + kind: "image_generation_followup", + forceAgentConversation: true, + shouldKeepSelectedModel: true, + usePromptCarryForward: promptCarryForwardRequested, + } satisfies ConversationActionResolution; + } + + const usePromptCarryForward = promptCarryForwardRequested; + if (usePromptCarryForward) { return { kind: "prompt_carry_forward", diff --git a/lib/ai/conversation-state-protocol-events.ts b/lib/ai/conversation-state-protocol-events.ts index b46c10648..c55db1915 100644 --- a/lib/ai/conversation-state-protocol-events.ts +++ b/lib/ai/conversation-state-protocol-events.ts @@ -29,12 +29,32 @@ export const conversationStateEventSchemas = { originalPrompt: z.string().min(1), }), }), + clarification_requested: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + messageId: z.string().uuid(), + text: z.string(), + clarificationMessage: z.string().min(1), + currentIntent: z.string().nullable().optional(), + }), storyboard_confirmed: z.object({ userId: z.string().uuid(), chatId: z.string().uuid(), projectId: z.string().uuid().nullable().optional(), messageId: z.string().uuid(), text: z.string(), + taskId: z.string().min(1).optional(), + status: z.string().optional(), + }), + storyboard_execution_started: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + messageId: z.string().uuid(), + text: z.string(), + taskId: z.string().min(1), + status: z.string().optional(), }), image_workflow_updated: z.object({ userId: z.string().uuid(), @@ -50,10 +70,75 @@ export const conversationStateEventSchemas = { lastResolvedTargetImageUrls: nullableStringArraySchema, lastMessageId: z.string().uuid(), lastUserPrompt: z.string(), + lastOptimizedPrompt: z.string().optional(), lastRequestedImageCount: z.number().int().min(1).max(12).optional(), + lastAspectRatio: z.string().optional(), + lastImageSize: z.string().optional(), lastSourceImageUrls: nullableStringArraySchema, lastResultImageUrls: nullableStringArraySchema, }), + storyboard_revision_requested: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + messageId: z.string().uuid(), + text: z.string(), + taskId: z.string().min(1), + status: z.string().optional(), + }), + storyboard_execution_completed: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + messageId: z.string().uuid(), + text: z.string(), + taskId: z.string().min(1), + status: z.string().optional(), + successCount: z.number().int().min(0).optional(), + failedCount: z.number().int().min(0).optional(), + firstVideoUrl: z.string().optional(), + }), + video_generation_started: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + messageId: z.string().uuid(), + text: z.string(), + source: z.string().min(1), + providerType: z.string().optional(), + modelRef: z.string().optional(), + mode: z.string().optional(), + taskId: z.string().optional(), + status: z.string().optional(), + }), + video_generation_completed: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + messageId: z.string().uuid(), + text: z.string(), + source: z.string().min(1), + providerType: z.string().optional(), + modelRef: z.string().optional(), + mode: z.string().optional(), + taskId: z.string().optional(), + status: z.string().optional(), + temporaryVideoUrl: z.string().optional(), + }), + video_generation_failed: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + messageId: z.string().uuid(), + text: z.string(), + source: z.string().min(1), + providerType: z.string().optional(), + modelRef: z.string().optional(), + mode: z.string().optional(), + taskId: z.string().optional(), + status: z.string().optional(), + error: z.string().optional(), + }), target_resolved: z.object({ userId: z.string().uuid(), chatId: z.string().uuid(), @@ -84,6 +169,16 @@ export const conversationStateEventSchemas = { text: z.string().optional(), status: z.string().optional(), }), + workflow_cancelled: z.object({ + userId: z.string().uuid(), + chatId: z.string().uuid(), + projectId: z.string().uuid().nullable().optional(), + messageId: z.string().uuid(), + text: z.string(), + workflowType: z.string().min(1), + workflowId: z.string().min(1).optional(), + status: z.string().optional(), + }), agent_preset_completed: z.object({ userId: z.string().uuid(), chatId: z.string().uuid(), @@ -120,4 +215,3 @@ export type ConversationStateEvent = payload: ConversationStateEventPayload; }; }[ConversationStateEventType]; - diff --git a/lib/ai/conversation-state-protocol.ts b/lib/ai/conversation-state-protocol.ts index 1b1820bfd..80b06b2ac 100644 --- a/lib/ai/conversation-state-protocol.ts +++ b/lib/ai/conversation-state-protocol.ts @@ -393,6 +393,23 @@ export async function persistConversationStateEvent; + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: typed.currentIntent, + lastMessageId: typed.messageId, + lastTurnSummaryPatch: { + text: typed.text.slice(0, 500), + resolutionMode: "clarification", + confidence: 80, + clarificationMessage: typed.clarificationMessage.slice(0, 1000), + }, + }); + } + case "storyboard_confirmed": { const typed = parsed as ConversationStateEventPayload<"storyboard_confirmed">; const existing = await getConversationWorkingStateByChatId({ @@ -410,6 +427,9 @@ export async function persistConversationStateEvent; + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: "video_gen", + lastMessageId: typed.messageId, + lastTurnSummaryPatch: { + text: typed.text.slice(0, 500), + resolutionMode: "storyboard_execution_started", + confidence: 95, + videoWorkflowStatus: typed.status || "running", + storyboardTaskId: typed.taskId, + storyboardWorkflowAction: "execution_started", + }, + }); + } + case "image_workflow_updated": { const typed = parsed as ConversationStateEventPayload<"image_workflow_updated">; const existing = await getConversationWorkingStateByChatId({ chatId: typed.chatId, }).catch(() => null); const existingImageWorkflowMemory = extractImageWorkflowMemory(existing); + const nextOptimizedPrompt = typed.lastOptimizedPrompt?.trim(); return persistConversationStateProtocol({ userId: typed.userId, @@ -439,11 +479,22 @@ export async function persistConversationStateEvent; + const existing = await getConversationWorkingStateByChatId({ + chatId: typed.chatId, + }).catch(() => null); + const existingStoryboardMemory = extractStoryboardPreferenceMemory(existing); + + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: "video_gen", + lastMessageId: typed.messageId, + lastTurnSummaryPatch: { + text: typed.text.slice(0, 500), + resolutionMode: "storyboard_revision", + confidence: 90, + videoWorkflowStatus: typed.status || "awaiting_user_confirmation", + storyboardTaskId: typed.taskId, + storyboardWorkflowAction: "revision_requested", + storyboardMemory: buildStoryboardPreferenceMemoryUpdate({ + existing: existingStoryboardMemory, + prefersConfirmation: true, + }), + }, + }); + } + + case "storyboard_execution_completed": { + const typed = parsed as ConversationStateEventPayload<"storyboard_execution_completed">; + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: "video_gen", + lastMessageId: typed.messageId, + lastTurnSummaryPatch: { + text: typed.text.slice(0, 500), + resolutionMode: "storyboard_execution_completed", + confidence: 95, + videoWorkflowStatus: typed.status || "completed", + storyboardTaskId: typed.taskId, + storyboardWorkflowAction: "execution_completed", + successCount: typed.successCount, + failedCount: typed.failedCount, + firstVideoUrl: typed.firstVideoUrl, + }, + }); + } + + case "video_generation_started": { + const typed = parsed as ConversationStateEventPayload<"video_generation_started">; + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: "video_gen", + lastMessageId: typed.messageId, + lastTurnSummaryPatch: { + text: typed.text.slice(0, 500), + resolutionMode: "video_generation_started", + confidence: 90, + videoWorkflowStatus: typed.status || "running", + videoWorkflowSource: typed.source, + videoProviderType: typed.providerType, + videoModelRef: typed.modelRef, + videoGenerationMode: typed.mode, + videoTaskId: typed.taskId, + }, + }); + } + + case "video_generation_completed": { + const typed = parsed as ConversationStateEventPayload<"video_generation_completed">; + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: "video_gen", + lastMessageId: typed.messageId, + lastTurnSummaryPatch: { + text: typed.text.slice(0, 500), + resolutionMode: "video_generation_completed", + confidence: 95, + videoWorkflowStatus: typed.status || "completed", + videoWorkflowSource: typed.source, + videoProviderType: typed.providerType, + videoModelRef: typed.modelRef, + videoGenerationMode: typed.mode, + videoTaskId: typed.taskId, + videoTemporaryUrl: typed.temporaryVideoUrl, + }, + }); + } + + case "video_generation_failed": { + const typed = parsed as ConversationStateEventPayload<"video_generation_failed">; + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: "video_gen", + lastMessageId: typed.messageId, + lastTurnSummaryPatch: { + text: typed.text.slice(0, 500), + resolutionMode: "video_generation_failed", + confidence: 85, + videoWorkflowStatus: typed.status || "failed", + videoWorkflowSource: typed.source, + videoProviderType: typed.providerType, + videoModelRef: typed.modelRef, + videoGenerationMode: typed.mode, + videoTaskId: typed.taskId, + videoWorkflowError: typed.error?.slice(0, 1000), + }, + }); + } + case "target_resolved": { const typed = parsed as ConversationStateEventPayload<"target_resolved">; return persistConversationStateProtocol({ @@ -498,6 +667,26 @@ export async function persistConversationStateEvent; + return persistConversationStateProtocol({ + userId: typed.userId, + chatId: typed.chatId, + projectId: typed.projectId, + currentIntent: typed.workflowType === "storyboard_video" ? "video_gen" : undefined, + lastMessageId: typed.messageId, + lastTurnSummaryPatch: { + text: typed.text.slice(0, 500), + resolutionMode: "workflow_cancelled", + confidence: 95, + videoWorkflowStatus: typed.status || "cancelled", + cancelledWorkflowType: typed.workflowType, + cancelledWorkflowId: typed.workflowId, + workflowAction: "cancel_requested", + }, + }); + } + case "agent_preset_completed": { const typed = parsed as ConversationStateEventPayload<"agent_preset_completed">; return persistConversationStateProtocol({ @@ -594,12 +783,46 @@ export async function persistGridConfirmationTurn(params: { }); } +export async function persistClarificationTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + messageId: string; + text: string; + clarificationMessage: string; + currentIntent?: string | null; + pendingGridConfirmation?: PendingGridConfirmation | null; +}) { + if (params.pendingGridConfirmation) { + return persistConversationStateEvent("grid_confirmation_requested", { + userId: params.userId, + chatId: params.chatId, + projectId: params.projectId, + messageId: params.messageId, + text: params.text, + pendingGridConfirmation: params.pendingGridConfirmation, + }); + } + + return persistConversationStateEvent("clarification_requested", { + userId: params.userId, + chatId: params.chatId, + projectId: params.projectId, + messageId: params.messageId, + text: params.text, + clarificationMessage: params.clarificationMessage, + currentIntent: params.currentIntent, + }); +} + export async function persistStoryboardConfirmedTurn(params: { userId: string; chatId: string; projectId?: string | null; messageId: string; text: string; + taskId?: string; + status?: string; }) { return persistConversationStateEvent("storyboard_confirmed", { userId: params.userId, @@ -607,6 +830,162 @@ export async function persistStoryboardConfirmedTurn(params: { projectId: params.projectId, messageId: params.messageId, text: params.text, + taskId: params.taskId, + status: params.status, + }); +} + +export async function persistStoryboardExecutionStartedTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + messageId: string; + text: string; + taskId: string; + status?: string; +}) { + return persistConversationStateEvent("storyboard_execution_started", { + userId: params.userId, + chatId: params.chatId, + projectId: params.projectId, + messageId: params.messageId, + text: params.text, + taskId: params.taskId, + status: params.status, + }); +} + +export async function persistStoryboardRevisionTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + messageId: string; + text: string; + taskId: string; + status?: string; +}) { + return persistConversationStateEvent("storyboard_revision_requested", { + userId: params.userId, + chatId: params.chatId, + projectId: params.projectId, + messageId: params.messageId, + text: params.text, + taskId: params.taskId, + status: params.status, + }); +} + +export async function persistStoryboardExecutionCompletedTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + messageId: string; + text: string; + taskId: string; + status?: string; + successCount?: number; + failedCount?: number; + firstVideoUrl?: string; +}) { + return persistConversationStateEvent("storyboard_execution_completed", { + userId: params.userId, + chatId: params.chatId, + projectId: params.projectId, + messageId: params.messageId, + text: params.text, + taskId: params.taskId, + status: params.status, + successCount: params.successCount, + failedCount: params.failedCount, + firstVideoUrl: params.firstVideoUrl, + }); +} + +export async function persistVideoGenerationStartedTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + messageId: string; + text: string; + source: string; + providerType?: string; + modelRef?: string; + mode?: string; + taskId?: string; + status?: string; +}) { + return persistConversationStateEvent("video_generation_started", { + userId: params.userId, + chatId: params.chatId, + projectId: params.projectId, + messageId: params.messageId, + text: params.text, + source: params.source, + providerType: params.providerType, + modelRef: params.modelRef, + mode: params.mode, + taskId: params.taskId, + status: params.status, + }); +} + +export async function persistVideoGenerationCompletedTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + messageId: string; + text: string; + source: string; + providerType?: string; + modelRef?: string; + mode?: string; + taskId?: string; + status?: string; + temporaryVideoUrl?: string; +}) { + return persistConversationStateEvent("video_generation_completed", { + userId: params.userId, + chatId: params.chatId, + projectId: params.projectId, + messageId: params.messageId, + text: params.text, + source: params.source, + providerType: params.providerType, + modelRef: params.modelRef, + mode: params.mode, + taskId: params.taskId, + status: params.status, + temporaryVideoUrl: params.temporaryVideoUrl, + }); +} + +export async function persistVideoGenerationFailedTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + messageId: string; + text: string; + source: string; + providerType?: string; + modelRef?: string; + mode?: string; + taskId?: string; + status?: string; + error?: string; +}) { + return persistConversationStateEvent("video_generation_failed", { + userId: params.userId, + chatId: params.chatId, + projectId: params.projectId, + messageId: params.messageId, + text: params.text, + source: params.source, + providerType: params.providerType, + modelRef: params.modelRef, + mode: params.mode, + taskId: params.taskId, + status: params.status, + error: params.error, }); } @@ -624,7 +1003,10 @@ export async function persistImageWorkflowTurn(params: { lastResolvedTargetImageUrls?: string[] | null; lastMessageId: string; lastUserPrompt: string; + lastOptimizedPrompt?: string; lastRequestedImageCount?: number; + lastAspectRatio?: string; + lastImageSize?: string; lastSourceImageUrls?: string[] | null; lastResultImageUrls?: string[] | null; }) { @@ -642,7 +1024,10 @@ export async function persistImageWorkflowTurn(params: { lastResolvedTargetImageUrls: params.lastResolvedTargetImageUrls, lastMessageId: params.lastMessageId, lastUserPrompt: params.lastUserPrompt, + lastOptimizedPrompt: params.lastOptimizedPrompt, lastRequestedImageCount: params.lastRequestedImageCount, + lastAspectRatio: params.lastAspectRatio, + lastImageSize: params.lastImageSize, lastSourceImageUrls: params.lastSourceImageUrls || null, lastResultImageUrls: params.lastResultImageUrls || null, }); @@ -684,6 +1069,28 @@ export async function persistVideoWorkflowTurn(params: { return persistConversationStateEvent("video_workflow_updated", params); } +export async function persistWorkflowCancelledTurn(params: { + userId: string; + chatId: string; + projectId?: string | null; + messageId: string; + text: string; + workflowType: string; + workflowId?: string; + status?: string; +}) { + return persistConversationStateEvent("workflow_cancelled", { + userId: params.userId, + chatId: params.chatId, + projectId: params.projectId, + messageId: params.messageId, + text: params.text, + workflowType: params.workflowType, + workflowId: params.workflowId, + status: params.status, + }); +} + export async function persistAgentPresetTurn(params: { userId: string; chatId: string; diff --git a/lib/ai/image-generation-subagent.ts b/lib/ai/image-generation-subagent.ts index 6aa53f3b2..7f046af70 100644 --- a/lib/ai/image-generation-subagent.ts +++ b/lib/ai/image-generation-subagent.ts @@ -16,6 +16,15 @@ function normalizeCompactText(text: string) { return String(text || "").toLowerCase().replace(/\s+/g, "").trim(); } +export function mentionsImageGenerationFollowup(text: string): boolean { + const normalized = normalizeCompactText(text); + if (!normalized) return false; + + return /继续生成|继续出图|继续做图|再来(一|两|三|四|五|六|七|八|九|\d+)?张|再出(一|两|三|四|五|六|七|八|九|\d+)?张|再做(一|两|三|四|五|六|七|八|九|\d+)?组|再给我(一|两|三|四|五|六|七|八|九|\d+)?张|补(一|两|三|四|五|六|七|八|九|\d+)?张|多来(一|两|三|四|五|六|七|八|九|\d+)?张|换一版|再换一版|同风格再来|同样风格再来|保持.*(不变|一致).*再来|同提示词再来|沿用上一个效果再来/.test( + normalized, + ); +} + export function buildImageGenerationSpecialistRoute(params: { text: string; threadState?: AgentThreadState | null; @@ -30,11 +39,18 @@ export function buildImageGenerationSpecialistRoute(params: { text: params.text, hasImage: params.hasImage, }); + const mentionsGenerationFollowup = mentionsImageGenerationFollowup(params.text); const mentionsPromptExecution = /提示词|prompt|关键词|咒语/.test(normalized) && /生成|生图|出图|继续做|继续生成|执行/.test(normalized); - if (!hasPromptMemory && !hasImageWorkflowMemory && !carryForward && !mentionsPromptExecution) { + if ( + !hasPromptMemory && + !hasImageWorkflowMemory && + !carryForward && + !mentionsPromptExecution && + !mentionsGenerationFollowup + ) { return null; } @@ -42,6 +58,9 @@ export function buildImageGenerationSpecialistRoute(params: { carryForward ? "用户当前要求沿用上一轮优化提示词继续执行，应优先理解为连续生图执行。" : "", + mentionsGenerationFollowup + ? "如果用户说“再来几张 / 换一版 / 同风格再来”，优先理解为沿用上一轮生图上下文继续执行，而不是开启全新任务。" + : "", hasPromptMemory ? "当前会话里存在上一轮可复用提示词，如果用户在说“上一个提示词/刚才那个 prompt”，优先复用这段提示词。" : "", @@ -58,7 +77,11 @@ export function buildImageGenerationSpecialistRoute(params: { return { domain: "image_generation_memory", specialistId: "image-generation-subagent", - reason: carryForward ? "当前回合命中提示词接力生图" : "当前回合命中提示词相关的连续生图任务", + reason: carryForward + ? "当前回合命中提示词接力生图" + : mentionsGenerationFollowup + ? "当前回合命中连续生图跟进语义" + : "当前回合命中提示词相关的连续生图任务", planningHints, toolAllowlist: [ "respondDirect", @@ -101,9 +124,15 @@ export function refinePlanWithImageGenerationSubagent(params: { text: params.text, hasImage: params.hasImage, }); + const mentionsGenerationFollowup = mentionsImageGenerationFollowup(params.text); const hasPromptMemory = getPromptMemoryPlanningHints(params.threadState).length > 0; const hasImageWorkflowMemory = getImageWorkflowPlanningHints(params.threadState).length > 0; - if (!carryForward && !hasPromptMemory && !hasImageWorkflowMemory) { + if ( + !carryForward && + !mentionsGenerationFollowup && + !hasPromptMemory && + !hasImageWorkflowMemory + ) { return params.plan; } diff --git a/lib/ai/runtime/clarification-interruption.ts b/lib/ai/runtime/clarification-interruption.ts new file mode 100644 index 000000000..868183340 --- /dev/null +++ b/lib/ai/runtime/clarification-interruption.ts @@ -0,0 +1,69 @@ +import "server-only"; + +function shouldReplaceClarificationLikeMessage(message?: string) { + if (!message) return true; + const normalized = message.trim(); + if (!normalized) return true; + return /[?？]$/.test(normalized) || /哪一张|哪个|请先确认/.test(normalized); +} + +export function resolveClarificationBypass(params: { + text: string; + hasCurrentImageInput: boolean; + hasHistoryImageInput: boolean; + hasCanvasSelection: boolean; + targetResolutionMode?: string | null; + executionMode?: string; + friendlyMessage?: string; +}) { + const latestUserText = String(params.text || "").trim(); + if ( + !params.hasCurrentImageInput && + !params.hasHistoryImageInput && + latestUserText.length <= 0 + ) { + return { + shouldBypass: false, + nextFriendlyMessage: params.friendlyMessage, + }; + } + + const canForceByIntent = /改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格/.test( + latestUserText, + ); + const hasForceTargetHint = + params.hasCanvasSelection || + params.targetResolutionMode === "text_reference" || + /第[一二三四五六七八九十\d]+张|左|右|上|下|中间|选中/.test(latestUserText); + const canForceExecution = + (params.executionMode === "edit_image" || + params.executionMode === "generate_image" || + params.executionMode === "generate_video") && + canForceByIntent && + hasForceTargetHint; + + if (!canForceExecution) { + return { + shouldBypass: false, + nextFriendlyMessage: params.friendlyMessage, + }; + } + + return { + shouldBypass: true, + nextFriendlyMessage: shouldReplaceClarificationLikeMessage(params.friendlyMessage) + ? "收到，我将按你的当前需求直接执行。" + : params.friendlyMessage, + }; +} + +export function resolveClarificationMessage(params: { + clarificationQuestion?: string; + friendlyMessage?: string; +}) { + return ( + params.clarificationQuestion || + params.friendlyMessage || + "我先确认一下，你希望我优先处理哪一部分？" + ); +} diff --git a/lib/video-agent/storyboard-chat-followup.ts b/lib/video-agent/storyboard-chat-followup.ts new file mode 100644 index 000000000..c47ce9e71 --- /dev/null +++ b/lib/video-agent/storyboard-chat-followup.ts @@ -0,0 +1,110 @@ +import "server-only"; + +import { executeConfirmedStoryboardTask } from "@/lib/video-agent/execute-storyboard-task"; +import { + reviseStoryboardTaskByCommand, + resolveStoryboardRevisionCommand, +} from "@/lib/video-agent/revise-storyboard-task"; +import { getVideoStoryboardTaskDetail } from "@/lib/video-agent/task-store"; +import type { StoryboardProgressEvent } from "@/lib/video-agent/types"; + +export async function handleStoryboardWorkflowFollowup(params: { + action: "revision_requested" | "cancel_requested"; + workflowId?: string; + text: string; + friendlyMessage?: string; + onProgress?: (event: StoryboardProgressEvent) => void | Promise; +}) { + let replyText = + params.friendlyMessage || + (params.action === "cancel_requested" + ? "已收到取消指令，当前分镜流程不会继续自动执行。" + : "收到修改意见，我会先停在草案阶段，等待你继续调整。"); + let nextWorkflowStatus = params.action === "cancel_requested" ? "cancelled" : "draft"; + + if (params.action === "revision_requested" && params.workflowId) { + const revisionTaskDetail = await getVideoStoryboardTaskDetail( + params.workflowId, + ).catch(() => null); + const revisionResult = await reviseStoryboardTaskByCommand({ + taskId: params.workflowId, + command: revisionTaskDetail + ? await resolveStoryboardRevisionCommand({ + text: params.text, + detail: revisionTaskDetail, + }) + : null, + onProgress: params.onProgress, + }); + replyText = revisionResult.message || replyText; + nextWorkflowStatus = revisionResult.ok ? "awaiting_user_confirmation" : "draft"; + } + + return { + replyText, + nextWorkflowStatus, + videoMetadata: params.workflowId + ? ({ + storyboard: { + enabled: true, + taskId: params.workflowId, + action: params.action, + }, + updatedAt: Date.now(), + } as any) + : null, + }; +} + +export function buildStoryboardConfirmedAck(taskId: string) { + return { + acknowledgeMessage: "已收到确认，开始按分镜逐段生成视频，请稍候。", + acknowledgeMetadata: { + storyboard: { + enabled: true, + taskId, + status: "confirmed", + }, + updatedAt: Date.now(), + } as any, + }; +} + +export function buildStoryboardExecutionVideoMetadata(params: { + taskId: string; + status: string; + successCount: number; + failedCount: number; +}) { + return { + storyboard: { + enabled: true, + taskId: params.taskId, + successCount: params.successCount, + failedCount: params.failedCount, + }, + status: params.status, + updatedAt: Date.now(), + } as any; +} + +export async function executeStoryboardConfirmedWorkflow(params: { + taskId: string; + onProgress?: (event: StoryboardProgressEvent) => void | Promise; +}) { + const executionResult = await executeConfirmedStoryboardTask({ + taskId: params.taskId, + onProgress: params.onProgress, + }); + + return { + executionResult, + firstVideoUrl: executionResult.videoUrls[0] || "", + finalVideoMetadata: buildStoryboardExecutionVideoMetadata({ + taskId: executionResult.taskId, + status: executionResult.status, + successCount: executionResult.successCount, + failedCount: executionResult.failedCount, + }), + }; +}