diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx
index 36b4c5163..d8394a10c 100644
--- a/app/(admin)/settings/admin-settings-client.tsx
+++ b/app/(admin)/settings/admin-settings-client.tsx
@@ -236,6 +236,11 @@ type SiteSettings = {
usageScenarios: string[];
brands: string[];
};
+ agentSecondaryTypeConfig: {
+ image: string[];
+ video: string[];
+ text: string[];
+ };
realtimeCanvasEnabled: boolean;
aiDetailGenerationEnabled: boolean;
canvasDefaultModelId: string | null;
@@ -632,6 +637,11 @@ export function AdminSettingsClient() {
usageScenarios: [],
brands: [],
},
+ agentSecondaryTypeConfig: {
+ image: [],
+ video: [],
+ text: [],
+ },
realtimeCanvasEnabled: true,
aiDetailGenerationEnabled: false,
canvasDefaultModelId: null,
@@ -718,6 +728,17 @@ export function AdminSettingsClient() {
? (siteSettingsData as any).agentCategoryTreeConfig.brands
: [],
},
+ agentSecondaryTypeConfig: {
+ image: Array.isArray((siteSettingsData as any).agentSecondaryTypeConfig?.image)
+ ? (siteSettingsData as any).agentSecondaryTypeConfig.image
+ : [],
+ video: Array.isArray((siteSettingsData as any).agentSecondaryTypeConfig?.video)
+ ? (siteSettingsData as any).agentSecondaryTypeConfig.video
+ : [],
+ text: Array.isArray((siteSettingsData as any).agentSecondaryTypeConfig?.text)
+ ? (siteSettingsData as any).agentSecondaryTypeConfig.text
+ : [],
+ },
realtimeCanvasEnabled: (siteSettingsData as any).realtimeCanvasEnabled ?? true,
aiDetailGenerationEnabled: siteSettingsData.aiDetailGenerationEnabled ?? false,
canvasDefaultModelId: siteSettingsData.canvasDefaultModelId || null,
@@ -5431,6 +5452,82 @@ export function AdminSettingsClient() {
)}
+ {siteSettings.agentEnabled && (
+
+
+
智能体二级类型配置
+
+ 用户创建智能体时,“二级类型”下拉从这里读取。按主类型分别维护:生图 / 视频 / 生文。
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ )}
+
diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts
index 2e7ef9305..9cc24944f 100644
--- a/app/(chat)/api/chat/route.ts
+++ b/app/(chat)/api/chat/route.ts
@@ -52,6 +52,7 @@ import {
executeDirectVideoGeneration,
generateVideo,
} from "@/lib/ai/tools/generate-video";
+import { executeConfirmedStoryboardTask } from "@/lib/video-agent/execute-storyboard-task";
import { getEnabledTools } from "@/lib/ai/tools-registry";
import { isProductionEnvironment } from "@/lib/constants";
import {
@@ -100,7 +101,24 @@ import { extractImageUrlsFromText } from "@/lib/image-url";
import { uploadFile } from "@/lib/storage/service";
import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline";
-import { attachVideoStoryboardTaskMessage } from "@/lib/video-agent/task-store";
+import {
+ reviseStoryboardTaskByCommand,
+ resolveStoryboardRevisionCommand,
+} from "@/lib/video-agent/revise-storyboard-task";
+import {
+ buildStoryboardPreferenceMemoryUpdate,
+ extractStoryboardPreferenceMemory,
+} from "@/lib/video-agent/storyboard-preference-memory";
+import {
+ attachVideoStoryboardTaskMessage,
+ getVideoStoryboardTaskDetail,
+ getLatestActiveVideoStoryboardTaskByChatId,
+ markVideoStoryboardAwaitingConfirmation,
+ markVideoStoryboardConfirmed,
+ markVideoStoryboardDraftForRevision,
+ cancelVideoStoryboardTaskByUser,
+} from "@/lib/video-agent/task-store";
+import { buildAgentThreadStateFromStoryboardTask } from "@/lib/video-agent/agent-thread-state";
import type { StoryboardModelCandidate } from "@/lib/video-agent/types";
import type { ChatMessage } from "@/lib/types";
import type { AppUsage } from "@/lib/usage";
@@ -1753,21 +1771,118 @@ function isConfirmedPromptGenerationRequest(text: string): boolean {
function extractConfirmedPromptFromWorkingState(
workingState?: {
- lastTurnSummary?: Record
| null;
+ lastTurnSummary?: unknown;
} | null,
): string | undefined {
const summary = workingState?.lastTurnSummary;
if (!summary || typeof summary !== "object") return undefined;
+ const summaryRecord = summary as Record;
const resolutionMode =
- typeof summary.resolutionMode === "string" ? summary.resolutionMode.trim() : "";
+ typeof summaryRecord.resolutionMode === "string" ? summaryRecord.resolutionMode.trim() : "";
const optimizedPrompt =
- typeof summary.optimizedPrompt === "string" ? summary.optimizedPrompt.trim() : "";
+ typeof summaryRecord.optimizedPrompt === "string" ? summaryRecord.optimizedPrompt.trim() : "";
if (resolutionMode !== "prompt_only" || !optimizedPrompt) return undefined;
return optimizedPrompt;
}
+type PendingGridConfirmation = {
+ gridType: "grid_2x2" | "grid_3x2" | "grid_3x3";
+ shotCount: 4 | 6 | 9;
+ label: string;
+ originalPrompt: string;
+};
+
+function detectRequestedStoryboardGrid(text: string): PendingGridConfirmation | null {
+ const normalized = String(text || "").toLowerCase().replace(/\s+/g, "");
+ if (!normalized) return null;
+
+ if (/4宫格|四宫格|2x2|2×2/.test(normalized)) {
+ return {
+ gridType: "grid_2x2",
+ shotCount: 4,
+ label: "4宫格",
+ originalPrompt: String(text || "").trim(),
+ };
+ }
+
+ if (/6宫格|六宫格|3x2|3×2|2x3|2×3/.test(normalized)) {
+ return {
+ gridType: "grid_3x2",
+ shotCount: 6,
+ label: "6宫格",
+ originalPrompt: String(text || "").trim(),
+ };
+ }
+
+ if (/9宫格|九宫格|3x3|3×3/.test(normalized)) {
+ return {
+ gridType: "grid_3x3",
+ shotCount: 9,
+ label: "9宫格",
+ originalPrompt: String(text || "").trim(),
+ };
+ }
+
+ return null;
+}
+
+function isGridConfirmationReply(text: string): boolean {
+ const normalized = String(text || "").toLowerCase().replace(/\s+/g, "");
+ if (!normalized) return false;
+ return /确认|就按这个|开始生成|可以开始了|没问题|对的|就这样/.test(normalized);
+}
+
+function buildGridConfirmationMessage(grid: PendingGridConfirmation): string {
+ return [
+ `我看到你希望按 ${grid.label} 来生成分镜图。`,
+ "为了确保后续自动切片稳定,我会按标准网格、统一格线、固定顺序来出图。",
+ `请确认是否按 ${grid.label} 标准版式开始生成。`,
+ "",
+ "如果确认,请回复:",
+ `- 确认按${grid.label}生成`,
+ "",
+ "如果要修改,也可以直接回复:",
+ "- 改成 9 宫格",
+ "- 改成 6 宫格",
+ "- 改成 4 宫格",
+ ].join("\n");
+}
+
+function extractPendingGridConfirmationFromWorkingState(
+ workingState?: {
+ lastTurnSummary?: unknown;
+ } | null,
+): PendingGridConfirmation | null {
+ const summary = workingState?.lastTurnSummary;
+ if (!summary || typeof summary !== "object") return null;
+ const summaryRecord = summary as Record;
+ const pending = summaryRecord.pendingGridConfirmation;
+ if (!pending || typeof pending !== "object") return null;
+ const record = pending as Record;
+ const gridType = String(record.gridType || "").trim();
+ const shotCount = Number(record.shotCount || 0);
+ const label = String(record.label || "").trim();
+ const originalPrompt = String(record.originalPrompt || "").trim();
+
+ if (
+ !originalPrompt ||
+ !label ||
+ !["grid_2x2", "grid_3x2", "grid_3x3"].includes(gridType) ||
+ ![4, 6, 9].includes(shotCount)
+ ) {
+ return null;
+ }
+
+ return {
+ gridType: gridType as PendingGridConfirmation["gridType"],
+ shotCount: shotCount as PendingGridConfirmation["shotCount"],
+ label,
+ originalPrompt,
+ };
+}
+
function hasStructuredAgentSections(message?: string): boolean {
if (!message) return false;
return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test(
@@ -2943,8 +3058,13 @@ export async function POST(request: Request) {
let requestWorkingState:
| Awaited>
| null = null;
+ let storyboardPreferenceMemory = null;
let smartAutoHistoryMessagesFromDb: DBMessage[] = [];
let preferredAgentModelId: string | undefined;
+ let pendingGridConfirmation:
+ | PendingGridConfirmation
+ | null = null;
+ let shouldAskGridConfirmation = false;
const deepThinkingOn = deepThinkingEnabled !== false;
let shouldRunAgentConversation = finalModelId === "auto";
let shouldKeepSelectedModel = false;
@@ -3038,6 +3158,57 @@ export async function POST(request: Request) {
const allowAgentVideoGeneration =
Boolean(requestBody.videoParams) || isVideoProviderType(requestedModelProviderType);
+ let smartAutoWorkflowDecision:
+ | {
+ action?: string;
+ workflowId?: string;
+ workflowType?: string;
+ allowWorkflowExecution?: boolean;
+ }
+ | undefined;
+ const activeStoryboardTaskForChat = await getLatestActiveVideoStoryboardTaskByChatId({
+ chatId: id,
+ }).catch((storyboardLookupError) => {
+ console.warn(
+ "[Chat API] Failed to preload active storyboard task before smart auto mode:",
+ storyboardLookupError
+ );
+ return null;
+ });
+
+ requestWorkingState = await getConversationWorkingStateByChatId({
+ chatId: id,
+ }).catch((workingStatePreloadError) => {
+ console.warn(
+ "[Chat API] Failed to preload conversation working state before smart auto mode:",
+ workingStatePreloadError
+ );
+ return null;
+ });
+ storyboardPreferenceMemory = extractStoryboardPreferenceMemory(requestWorkingState);
+ pendingGridConfirmation =
+ extractPendingGridConfirmationFromWorkingState(requestWorkingState);
+
+ if (
+ pendingGridConfirmation &&
+ (isGridConfirmationReply(requestTextContent) ||
+ detectRequestedStoryboardGrid(requestTextContent) !== null) &&
+ !shouldRunAgentConversation
+ ) {
+ shouldRunAgentConversation = true;
+ shouldKeepSelectedModel = true;
+ console.log(
+ `[Smart Auto Mode] 🧩 Force agent conversation due to pending grid confirmation: ${pendingGridConfirmation.label}`
+ );
+ }
+
+ if (activeStoryboardTaskForChat?.task?.id && !shouldRunAgentConversation) {
+ shouldRunAgentConversation = true;
+ shouldKeepSelectedModel = true;
+ console.log(
+ `[Smart Auto Mode] 🔐 Force agent conversation due to active storyboard task: ${activeStoryboardTaskForChat.task.id}`
+ );
+ }
// ========== Smart Auto Mode Logic ==========
if (shouldRunAgentConversation) {
@@ -3047,13 +3218,17 @@ export async function POST(request: Request) {
let autoCurrentImageUrlsForFallback: string[] = [];
let confirmedPromptForGeneration: string | undefined;
let forcePromptDrivenImageGeneration = false;
+ const smartAutoInputText =
+ pendingGridConfirmation && isGridConfirmationReply(requestTextContent)
+ ? pendingGridConfirmation.originalPrompt
+ : requestTextContent;
try {
// 🆕 首先获取历史消息(在 Smart Auto Mode 内部单独获取,避免依赖后面的 messagesFromDb)
smartAutoHistoryMessagesFromDb = await getMessagesByChatId({ id });
console.log("[Smart Auto Mode] Loaded", smartAutoHistoryMessagesFromDb.length, "messages from DB");
- autoTextForFallback = requestTextContent;
+ autoTextForFallback = smartAutoInputText;
if (!requestWorkingState) {
requestWorkingState = await getConversationWorkingStateByChatId({
chatId: id,
@@ -3157,7 +3332,7 @@ export async function POST(request: Request) {
}
const autoGraphResult = await withOperationTimeout(
runAutoAgentGraph({
- text: requestTextContent,
+ text: smartAutoInputText,
hasImage,
imageUrls,
assetIds: Array.from(
@@ -3180,14 +3355,89 @@ export async function POST(request: Request) {
historyMessages,
context: autoContextText || undefined,
preferredModelId: preferredAgentModelId,
+ threadState: buildAgentThreadStateFromStoryboardTask(
+ activeStoryboardTaskForChat,
+ {
+ storyboardPreferenceMemory,
+ }
+ ),
}),
"runAutoAgentGraph",
autoGraphTimeoutMs,
);
+ const requestedStoryboardGrid = detectRequestedStoryboardGrid(
+ pendingGridConfirmation && isGridConfirmationReply(requestTextContent)
+ ? pendingGridConfirmation.originalPrompt
+ : requestTextContent
+ );
+ if (requestedStoryboardGrid) {
+ storyboardPreferenceMemory = buildStoryboardPreferenceMemoryUpdate({
+ existing: storyboardPreferenceMemory,
+ preferredGridType: requestedStoryboardGrid.gridType,
+ preferredShotCount: requestedStoryboardGrid.shotCount,
+ prefersConfirmation: true,
+ lastUsedGridType: requestedStoryboardGrid.gridType,
+ lastUsedShotCount: requestedStoryboardGrid.shotCount,
+ });
+ }
+
smartAutoPlan = autoGraphResult.plan;
smartAutoExecution = autoGraphResult.execution;
+ if (autoGraphResult.workflowDecision) {
+ smartAutoWorkflowDecision = autoGraphResult.workflowDecision;
+ console.log("[Smart Auto Mode] Workflow decision:", {
+ action: autoGraphResult.workflowDecision.action,
+ intent: autoGraphResult.workflowDecision.confirmationIntent,
+ workflowId: autoGraphResult.workflowDecision.workflowId,
+ workflowType: autoGraphResult.workflowDecision.workflowType,
+ nextWorkflowStatus: autoGraphResult.workflowDecision.nextWorkflowStatus,
+ allowWorkflowExecution: autoGraphResult.workflowDecision.allowWorkflowExecution,
+ });
+
+ if (
+ autoGraphResult.workflowDecision.workflowType === "storyboard_video" &&
+ autoGraphResult.workflowDecision.workflowId
+ ) {
+ const storyboardTaskId = autoGraphResult.workflowDecision.workflowId;
+ try {
+ switch (autoGraphResult.workflowDecision.action) {
+ case "awaiting_confirmation":
+ await markVideoStoryboardAwaitingConfirmation({
+ taskId: storyboardTaskId,
+ });
+ break;
+ case "confirmed":
+ await markVideoStoryboardConfirmed({
+ taskId: storyboardTaskId,
+ messageId: message.id,
+ });
+ break;
+ case "revision_requested":
+ await markVideoStoryboardDraftForRevision({
+ taskId: storyboardTaskId,
+ });
+ break;
+ case "cancel_requested":
+ await cancelVideoStoryboardTaskByUser({
+ taskId: storyboardTaskId,
+ error: "user_cancelled_via_chat",
+ });
+ break;
+ case "pass":
+ default:
+ break;
+ }
+ } catch (workflowPersistError) {
+ console.warn(
+ "[Smart Auto Mode] Failed to persist storyboard workflow decision:",
+ workflowPersistError
+ );
+ }
+ }
+ }
+
if (forcePromptDrivenImageGeneration && confirmedPromptForGeneration) {
smartAutoDetectedIntent = "image_gen";
smartAutoOptimizedPrompt = confirmedPromptForGeneration;
@@ -3263,11 +3513,44 @@ export async function POST(request: Request) {
}
}
+ const requestedGrid =
+ !pendingGridConfirmation || smartAutoInputText === requestTextContent
+ ? detectRequestedStoryboardGrid(requestTextContent)
+ : pendingGridConfirmation;
+ const isGridTaskCandidate =
+ smartAutoExecution?.mode === "generate_image" ||
+ smartAutoExecution?.mode === "generate_video" ||
+ autoGraphResult.analysis?.intent === "image_gen" ||
+ autoGraphResult.analysis?.intent === "video_gen";
+ if (
+ requestedGrid &&
+ !isGridConfirmationReply(requestTextContent) &&
+ !activeStoryboardTaskForChat?.task?.id &&
+ isGridTaskCandidate
+ ) {
+ shouldAskGridConfirmation = true;
+ smartAutoNeedsClarification = true;
+ smartAutoClarificationQuestion = buildGridConfirmationMessage(requestedGrid);
+ smartAutoFriendlyMessage = smartAutoClarificationQuestion;
+ pendingGridConfirmation = {
+ ...requestedGrid,
+ originalPrompt: smartAutoInputText,
+ };
+ console.log("[Smart Auto Mode] 🧱 Require grid confirmation before generation:", {
+ gridType: requestedGrid.gridType,
+ shotCount: requestedGrid.shotCount,
+ });
+ }
+
if (autoGraphResult.friendlyMessage) {
smartAutoFriendlyMessage = autoGraphResult.friendlyMessage;
console.log("[Smart Auto Mode] 💬 Conversational reply:", smartAutoFriendlyMessage);
}
+ if (shouldAskGridConfirmation && smartAutoClarificationQuestion) {
+ smartAutoFriendlyMessage = smartAutoClarificationQuestion;
+ }
+
if (
!smartAutoNeedsClarification &&
hasImage &&
@@ -4324,7 +4607,21 @@ export async function POST(request: Request) {
const optimizedResult = await optimizePromptWithLLM({
originalPrompt: promptSeed,
intent: "image_gen",
- historyMessages: smartAutoHistoryMessagesFromDb,
+ historyMessages: smartAutoHistoryMessagesFromDb
+ .map((msg) => {
+ const parts = Array.isArray(msg.parts) ? (msg.parts as any[]) : [];
+ const content = parts
+ .filter((part: any) => part?.type === "text")
+ .map((part: any) => String(part?.text || ""))
+ .join(" ")
+ .trim();
+ if (!content) return null;
+ return {
+ role: msg.role as "user" | "assistant",
+ content: content.slice(0, 300),
+ };
+ })
+ .filter(Boolean) as Array<{ role: "user" | "assistant"; content: string }>,
context: typeof autoContext === "string" ? autoContext : undefined,
}).catch(() =>
optimizePrompt({
@@ -4481,12 +4778,301 @@ export async function POST(request: Request) {
},
],
});
+
+ if (shouldAskGridConfirmation && pendingGridConfirmation) {
+ try {
+ await upsertConversationWorkingState({
+ userId: session.user.id,
+ chatId: id,
+ projectId: contextPayload?.projectId || null,
+ currentIntent: "image_gen",
+ lastMessageId: messageId,
+ lastTurnSummary: {
+ text: requestTextContent.slice(0, 500),
+ resolutionMode: "grid_confirmation",
+ confidence: 90,
+ storyboardMemory: buildStoryboardPreferenceMemoryUpdate({
+ existing: storyboardPreferenceMemory,
+ preferredGridType: pendingGridConfirmation.gridType,
+ preferredShotCount: pendingGridConfirmation.shotCount,
+ prefersConfirmation: true,
+ lastUsedGridType: pendingGridConfirmation.gridType,
+ lastUsedShotCount: pendingGridConfirmation.shotCount,
+ }),
+ pendingGridConfirmation: {
+ gridType: pendingGridConfirmation.gridType,
+ shotCount: pendingGridConfirmation.shotCount,
+ label: pendingGridConfirmation.label,
+ originalPrompt: pendingGridConfirmation.originalPrompt.slice(0, 1000),
+ },
+ },
+ });
+ } catch (gridConfirmationPersistError) {
+ console.warn(
+ "[Chat API] Failed to persist pending grid confirmation state:",
+ gridConfirmationPersistError
+ );
+ }
+ }
},
});
return createSseResponse(clarificationStream);
}
+ if (
+ shouldRunAgentConversation &&
+ smartAutoWorkflowDecision?.workflowType === "storyboard_video" &&
+ (smartAutoWorkflowDecision?.action === "revision_requested" ||
+ smartAutoWorkflowDecision?.action === "cancel_requested")
+ ) {
+ const workflowReplyStream = createUIMessageStream({
+ execute: async ({ writer }) => {
+ const messageId = generateUUID();
+ let replyText =
+ smartAutoFriendlyMessage ||
+ (smartAutoWorkflowDecision?.action === "cancel_requested"
+ ? "已收到取消指令,当前分镜流程不会继续自动执行。"
+ : "收到修改意见,我会先停在草案阶段,等待你继续调整。");
+
+ if (
+ smartAutoWorkflowDecision?.action === "revision_requested" &&
+ smartAutoWorkflowDecision.workflowId
+ ) {
+ const revisionTaskDetail = await getVideoStoryboardTaskDetail(
+ smartAutoWorkflowDecision.workflowId
+ ).catch(() => null);
+ const revisionResult = await reviseStoryboardTaskByCommand({
+ taskId: smartAutoWorkflowDecision.workflowId,
+ command: revisionTaskDetail
+ ? await resolveStoryboardRevisionCommand({
+ text: requestTextContent,
+ detail: revisionTaskDetail,
+ })
+ : null,
+ onProgress: (event) => {
+ safeWriteStreamChunk(
+ writer,
+ {
+ type: "data-videoStoryboard",
+ data: JSON.stringify(event),
+ },
+ `storyboard-revision:${event.stage}`,
+ );
+ },
+ });
+ replyText = revisionResult.message || replyText;
+ }
+
+ writer.write({
+ type: "data-appendMessage",
+ data: JSON.stringify({
+ id: messageId,
+ role: "assistant",
+ parts: [{ type: "text", text: replyText }],
+ attachments: [],
+ }),
+ });
+
+ await saveMessages({
+ messages: [
+ {
+ id: messageId,
+ chatId: id,
+ role: "assistant",
+ parts: [{ type: "text", text: replyText }],
+ attachments: [],
+ createdAt: new Date(),
+ videoStorageStatus: null,
+ temporaryVideoUrl: null,
+ permanentVideoUrl: null,
+ archivedAt: null,
+ videoMetadata:
+ smartAutoWorkflowDecision?.workflowId
+ ? ({
+ storyboard: {
+ enabled: true,
+ taskId: smartAutoWorkflowDecision.workflowId,
+ action: smartAutoWorkflowDecision.action,
+ },
+ updatedAt: Date.now(),
+ } as any)
+ : null,
+ },
+ ],
+ });
+ },
+ });
+
+ return createSseResponse(workflowReplyStream);
+ }
+
+ if (
+ shouldRunAgentConversation &&
+ smartAutoWorkflowDecision?.workflowType === "storyboard_video" &&
+ smartAutoWorkflowDecision?.action === "confirmed" &&
+ smartAutoWorkflowDecision.workflowId
+ ) {
+ const confirmedTaskId = smartAutoWorkflowDecision.workflowId;
+ const confirmedStoryboardStream = createUIMessageStream({
+ execute: async ({ writer }) => {
+ const acknowledgeMessageId = generateUUID();
+ const acknowledgeMessage = "已收到确认,开始按分镜逐段生成视频,请稍候。";
+
+ writer.write({
+ type: "data-appendMessage",
+ data: JSON.stringify({
+ id: acknowledgeMessageId,
+ role: "assistant",
+ parts: [{ type: "text", text: acknowledgeMessage }],
+ attachments: [],
+ }),
+ });
+
+ await saveMessages({
+ messages: [
+ {
+ id: acknowledgeMessageId,
+ chatId: id,
+ role: "assistant",
+ parts: [{ type: "text", text: acknowledgeMessage }],
+ attachments: [],
+ createdAt: new Date(),
+ videoStorageStatus: null,
+ temporaryVideoUrl: null,
+ permanentVideoUrl: null,
+ archivedAt: null,
+ videoMetadata: {
+ storyboard: {
+ enabled: true,
+ taskId: confirmedTaskId,
+ status: "confirmed",
+ },
+ updatedAt: Date.now(),
+ } as any,
+ },
+ ],
+ });
+
+ try {
+ await upsertConversationWorkingState({
+ userId: session.user.id,
+ chatId: id,
+ projectId: contextPayload?.projectId || null,
+ currentIntent: "video_gen",
+ lastMessageId: acknowledgeMessageId,
+ lastTurnSummary: {
+ text: requestTextContent.slice(0, 500),
+ resolutionMode: "storyboard_confirmed",
+ confidence: 95,
+ storyboardMemory: buildStoryboardPreferenceMemoryUpdate({
+ existing: storyboardPreferenceMemory,
+ prefersConfirmation: true,
+ }),
+ },
+ });
+ } catch (storyboardConfirmMemoryError) {
+ console.warn(
+ "[Chat API] Failed to persist storyboard confirmation memory:",
+ storyboardConfirmMemoryError
+ );
+ }
+
+ const executionResult = await executeConfirmedStoryboardTask({
+ taskId: confirmedTaskId,
+ onProgress: (event) => {
+ safeWriteStreamChunk(
+ writer,
+ {
+ type: "data-videoStoryboard",
+ data: JSON.stringify(event),
+ },
+ `storyboard-confirmed:${event.stage}`,
+ );
+ },
+ });
+
+ const finalMessageId = generateUUID();
+ writer.write({
+ type: "data-appendMessage",
+ data: JSON.stringify({
+ id: finalMessageId,
+ role: "assistant",
+ parts: [{ type: "text", text: executionResult.markdown }],
+ attachments: [],
+ }),
+ });
+
+ const firstVideoUrl = executionResult.videoUrls[0] || "";
+ await saveMessages({
+ messages: [
+ {
+ id: finalMessageId,
+ chatId: id,
+ role: "assistant",
+ parts: [{ type: "text", text: executionResult.markdown }],
+ attachments: [],
+ createdAt: new Date(),
+ videoStorageStatus: firstVideoUrl ? "temporary" : null,
+ temporaryVideoUrl: firstVideoUrl || null,
+ permanentVideoUrl: null,
+ archivedAt: null,
+ videoMetadata: {
+ storyboard: {
+ enabled: true,
+ taskId: executionResult.taskId,
+ successCount: executionResult.successCount,
+ failedCount: executionResult.failedCount,
+ },
+ status: executionResult.status,
+ updatedAt: Date.now(),
+ } as any,
+ },
+ ],
+ });
+
+ await attachVideoStoryboardTaskMessage({
+ taskId: executionResult.taskId,
+ messageId: finalMessageId,
+ }).catch((taskAttachError) => {
+ console.warn(
+ "[Chat API] Failed to attach confirmed storyboard task with message:",
+ taskAttachError
+ );
+ });
+
+ if (firstVideoUrl) {
+ await consumeGeneratedVideoPoints({
+ userId: session.user.id,
+ chatId: id,
+ messageId: finalMessageId,
+ modelRefs: executionResult.modelRefs,
+ });
+
+ scheduleVideoArchive({
+ chatId: id,
+ messageId: finalMessageId,
+ text: executionResult.markdown,
+ temporaryVideoUrl: firstVideoUrl,
+ taskId: finalMessageId,
+ metadata: {
+ storyboard: {
+ enabled: true,
+ taskId: executionResult.taskId,
+ successCount: executionResult.successCount,
+ failedCount: executionResult.failedCount,
+ },
+ status: executionResult.status,
+ updatedAt: Date.now(),
+ },
+ });
+ }
+ },
+ });
+
+ return createSseResponse(confirmedStoryboardStream);
+ }
+
if (agentPresetId) {
const selectedAgentPreset = await getAccessibleAgentPresetById({
id: agentPresetId,
@@ -4503,6 +5089,13 @@ export async function POST(request: Request) {
);
}
+ if (selectedAgentPreset.type === "text") {
+ // 生文智能体走常规多模态对话链路:
+ // - 模型已由前端切到智能体预设模型
+ // - 提示词通过 hiddenPrompt 注入
+ // - 图片输入沿用普通多模态附件
+ } else {
+
const orderedSlots = [...(selectedAgentPreset.slots || [])].sort(
(left, right) => left.slotIndex - right.slotIndex,
);
@@ -4984,6 +5577,7 @@ export async function POST(request: Request) {
});
return createSseResponse(agentSequenceStream);
+ }
}
const directAgentExecution = shouldRunAgentConversation
@@ -7224,6 +7818,8 @@ export async function POST(request: Request) {
usedStoryboardPipeline = true;
videoMarkdown = storyboardResult.markdown || "视频生成失败,请稍后重试";
tempVideoUrl = storyboardResult.videoUrls?.[0] || "";
+ const isAwaitingConfirmation =
+ storyboardResult.reason === "awaiting_user_confirmation";
billedModelRefs =
storyboardResult.modelRefs && storyboardResult.modelRefs.length > 0
? storyboardResult.modelRefs
@@ -7233,12 +7829,17 @@ export async function POST(request: Request) {
videoMetadata = {
...videoMetadata,
...(storyboardResult.metadata || {}),
- status: successCount > 0 ? "succeeded" : "failed",
+ status: isAwaitingConfirmation
+ ? "awaiting_user_confirmation"
+ : successCount > 0
+ ? "succeeded"
+ : "failed",
storyboard: {
enabled: true,
taskId: storyboardResult.taskId,
confidence: storyboardResult.confidence,
successCount,
+ awaitingConfirmation: isAwaitingConfirmation,
},
updatedAt: Date.now(),
};
@@ -7307,12 +7908,57 @@ export async function POST(request: Request) {
});
}
console.log(`[Chat API] ✅ ${currentVideoProvider} video message saved successfully!`);
- await consumeGeneratedVideoPoints({
- userId: session.user.id,
- chatId: id,
- messageId,
- modelRefs: billedModelRefs,
- });
+
+ if ((videoMetadata as any)?.storyboard?.enabled) {
+ try {
+ const storyboardGridType = String((videoMetadata as any)?.gridType || "").trim();
+ const storyboardShotCount = Number((videoMetadata as any)?.shotCount || 0);
+ await upsertConversationWorkingState({
+ userId: session.user.id,
+ chatId: id,
+ projectId: contextPayload?.projectId || null,
+ currentIntent: "video_gen",
+ lastMessageId: messageId,
+ lastTurnSummary: {
+ text: requestTextContent.slice(0, 500),
+ resolutionMode: "storyboard",
+ confidence: 95,
+ storyboardMemory: buildStoryboardPreferenceMemoryUpdate({
+ existing: storyboardPreferenceMemory,
+ prefersConfirmation: true,
+ lastUsedGridType:
+ storyboardGridType === "grid_2x2" ||
+ storyboardGridType === "grid_3x2" ||
+ storyboardGridType === "grid_3x3"
+ ? (storyboardGridType as "grid_2x2" | "grid_3x2" | "grid_3x3")
+ : undefined,
+ lastUsedShotCount:
+ storyboardShotCount === 4 ||
+ storyboardShotCount === 6 ||
+ storyboardShotCount === 9
+ ? storyboardShotCount
+ : undefined,
+ }),
+ },
+ });
+ } catch (storyboardMemoryPersistError) {
+ console.warn(
+ "[Chat API] Failed to persist storyboard memory after assistant response:",
+ storyboardMemoryPersistError
+ );
+ }
+ }
+ const shouldConsumeVideoPoints =
+ tempVideoUrl &&
+ (videoMetadata as any)?.status !== "awaiting_user_confirmation";
+ if (shouldConsumeVideoPoints) {
+ await consumeGeneratedVideoPoints({
+ userId: session.user.id,
+ chatId: id,
+ messageId,
+ modelRefs: billedModelRefs,
+ });
+ }
if (tempVideoUrl) {
scheduleVideoArchive({
diff --git a/app/(chat)/api/video/storyboard/tasks/[taskId]/cancel/route.ts b/app/(chat)/api/video/storyboard/tasks/[taskId]/cancel/route.ts
new file mode 100644
index 000000000..9a9fce8e7
--- /dev/null
+++ b/app/(chat)/api/video/storyboard/tasks/[taskId]/cancel/route.ts
@@ -0,0 +1,75 @@
+import { NextResponse } from "next/server";
+import { z } from "zod";
+import { auth } from "@/app/(auth)/auth";
+import {
+ cancelVideoStoryboardTaskByUser,
+ getVideoStoryboardTaskDetail,
+} from "@/lib/video-agent/task-store";
+import {
+ requireStoryboardTaskAccess,
+ StoryboardTaskAccessError,
+} from "@/lib/video-agent/user-task-access";
+
+const paramsSchema = z.object({
+ taskId: z.string().uuid(),
+});
+
+export async function POST(
+ _request: Request,
+ { params }: { params: Promise<{ taskId: string }> }
+) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+
+ const parsed = paramsSchema.parse(await params);
+ const detail = await requireStoryboardTaskAccess({
+ taskId: parsed.taskId,
+ userId: session.user.id,
+ });
+
+ const status = String((detail.task as any).status || "").trim();
+ if (status === "cancelled") {
+ return NextResponse.json({
+ success: true,
+ task: detail.task,
+ shots: detail.shots,
+ });
+ }
+
+ await cancelVideoStoryboardTaskByUser({
+ taskId: parsed.taskId,
+ error: "user_cancelled_via_api",
+ });
+
+ const nextDetail = await getVideoStoryboardTaskDetail(parsed.taskId);
+ return NextResponse.json({
+ success: true,
+ task: nextDetail?.task || detail.task,
+ shots: nextDetail?.shots || detail.shots,
+ });
+ } catch (error) {
+ if (error instanceof StoryboardTaskAccessError) {
+ return NextResponse.json(
+ { success: false, error: error.message },
+ { status: error.status }
+ );
+ }
+ if (error instanceof z.ZodError) {
+ return NextResponse.json(
+ { success: false, error: error.message },
+ { status: 400 }
+ );
+ }
+ return NextResponse.json(
+ {
+ success: false,
+ error: error instanceof Error ? error.message : "取消任务失败",
+ },
+ { status: 500 }
+ );
+ }
+}
+
diff --git a/app/(chat)/api/video/storyboard/tasks/[taskId]/confirm/route.ts b/app/(chat)/api/video/storyboard/tasks/[taskId]/confirm/route.ts
new file mode 100644
index 000000000..a83a9bfb3
--- /dev/null
+++ b/app/(chat)/api/video/storyboard/tasks/[taskId]/confirm/route.ts
@@ -0,0 +1,101 @@
+import { after, NextResponse } from "next/server";
+import { z } from "zod";
+import { auth } from "@/app/(auth)/auth";
+import { executeConfirmedStoryboardTask } from "@/lib/video-agent/execute-storyboard-task";
+import { getVideoStoryboardTaskDetail, markVideoStoryboardConfirmed } from "@/lib/video-agent/task-store";
+import {
+ requireStoryboardTaskAccess,
+ StoryboardTaskAccessError,
+} from "@/lib/video-agent/user-task-access";
+
+const paramsSchema = z.object({
+ taskId: z.string().uuid(),
+});
+
+export async function POST(
+ _request: Request,
+ { params }: { params: Promise<{ taskId: string }> }
+) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+
+ const parsed = paramsSchema.parse(await params);
+ const detail = await requireStoryboardTaskAccess({
+ taskId: parsed.taskId,
+ userId: session.user.id,
+ });
+
+ const task = detail.task as any;
+ const status = String(task.status || "").trim();
+ const confirmationStatus = String(task.confirmationStatus || "").trim();
+
+ if (status === "cancelled") {
+ return NextResponse.json(
+ { success: false, error: "任务已取消,无法确认执行" },
+ { status: 400 }
+ );
+ }
+ if (["succeeded", "partial_success", "failed"].includes(status)) {
+ return NextResponse.json(
+ { success: false, error: "任务已结束,无法再次确认执行" },
+ { status: 400 }
+ );
+ }
+ if (status === "running" && confirmationStatus === "confirmed") {
+ return NextResponse.json(
+ { success: true, queued: false, task: detail.task, shots: detail.shots },
+ { status: 200 }
+ );
+ }
+
+ await markVideoStoryboardConfirmed({
+ taskId: parsed.taskId,
+ });
+
+ after(async () => {
+ try {
+ await executeConfirmedStoryboardTask({
+ taskId: parsed.taskId,
+ });
+ } catch (error) {
+ console.error("[Storyboard Confirm API] Background execution failed:", error);
+ }
+ });
+
+ const nextDetail = await getVideoStoryboardTaskDetail(parsed.taskId);
+
+ return NextResponse.json(
+ {
+ success: true,
+ queued: true,
+ task: nextDetail?.task || detail.task,
+ shots: nextDetail?.shots || detail.shots,
+ },
+ { status: 202 }
+ );
+ } catch (error) {
+ if (error instanceof StoryboardTaskAccessError) {
+ return NextResponse.json(
+ { success: false, error: error.message },
+ { status: error.status }
+ );
+ }
+ if (error instanceof z.ZodError) {
+ return NextResponse.json(
+ { success: false, error: error.message },
+ { status: 400 }
+ );
+ }
+ return NextResponse.json(
+ {
+ success: false,
+ error: error instanceof Error ? error.message : "确认执行失败",
+ },
+ { status: 500 }
+ );
+ }
+}
+
diff --git a/app/(chat)/api/video/storyboard/tasks/[taskId]/route.ts b/app/(chat)/api/video/storyboard/tasks/[taskId]/route.ts
new file mode 100644
index 000000000..dc2601174
--- /dev/null
+++ b/app/(chat)/api/video/storyboard/tasks/[taskId]/route.ts
@@ -0,0 +1,56 @@
+import { NextResponse } from "next/server";
+import { z } from "zod";
+import { auth } from "@/app/(auth)/auth";
+import {
+ requireStoryboardTaskAccess,
+ StoryboardTaskAccessError,
+} from "@/lib/video-agent/user-task-access";
+
+const paramsSchema = z.object({
+ taskId: z.string().uuid(),
+});
+
+export async function GET(
+ _request: Request,
+ { params }: { params: Promise<{ taskId: string }> }
+) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+
+ const parsed = paramsSchema.parse(await params);
+ const detail = await requireStoryboardTaskAccess({
+ taskId: parsed.taskId,
+ userId: session.user.id,
+ });
+
+ return NextResponse.json({
+ success: true,
+ task: detail.task,
+ shots: detail.shots,
+ });
+ } catch (error) {
+ if (error instanceof StoryboardTaskAccessError) {
+ return NextResponse.json(
+ { success: false, error: error.message },
+ { status: error.status }
+ );
+ }
+ if (error instanceof z.ZodError) {
+ return NextResponse.json(
+ { success: false, error: error.message },
+ { status: 400 }
+ );
+ }
+ return NextResponse.json(
+ {
+ success: false,
+ error: error instanceof Error ? error.message : "查询任务失败",
+ },
+ { status: 500 }
+ );
+ }
+}
+
diff --git a/app/(chat)/api/video/storyboard/tasks/[taskId]/shots/[shotIndex]/regenerate-hd/route.ts b/app/(chat)/api/video/storyboard/tasks/[taskId]/shots/[shotIndex]/regenerate-hd/route.ts
new file mode 100644
index 000000000..8bc59caab
--- /dev/null
+++ b/app/(chat)/api/video/storyboard/tasks/[taskId]/shots/[shotIndex]/regenerate-hd/route.ts
@@ -0,0 +1,73 @@
+import { NextResponse } from "next/server";
+import { z } from "zod";
+import { auth } from "@/app/(auth)/auth";
+import { regenerateStoryboardShotHd } from "@/lib/video-agent/revise-storyboard-task";
+import {
+ requireStoryboardTaskAccess,
+ StoryboardTaskAccessError,
+} from "@/lib/video-agent/user-task-access";
+
+const paramsSchema = z.object({
+ taskId: z.string().uuid(),
+ shotIndex: z.coerce.number().int().min(1).max(99),
+});
+
+export async function POST(
+ _request: Request,
+ { params }: { params: Promise<{ taskId: string; shotIndex: string }> }
+) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+
+ const parsedParams = paramsSchema.parse(await params);
+ const detail = await requireStoryboardTaskAccess({
+ taskId: parsedParams.taskId,
+ userId: session.user.id,
+ });
+
+ const status = String((detail.task as any).status || "").trim();
+ const confirmationStatus = String((detail.task as any).confirmationStatus || "").trim();
+ if (status === "running" && confirmationStatus === "confirmed") {
+ return NextResponse.json(
+ { success: false, error: "任务正在执行视频,暂时不能重生高清预览" },
+ { status: 409 }
+ );
+ }
+
+ const result = await regenerateStoryboardShotHd({
+ taskId: parsedParams.taskId,
+ shotIndex: parsedParams.shotIndex,
+ });
+
+ return NextResponse.json({
+ success: result.ok,
+ message: result.message,
+ task: result.detail?.task || detail.task,
+ shots: result.detail?.shots || detail.shots,
+ });
+ } catch (error) {
+ if (error instanceof StoryboardTaskAccessError) {
+ return NextResponse.json(
+ { success: false, error: error.message },
+ { status: error.status }
+ );
+ }
+ if (error instanceof z.ZodError) {
+ return NextResponse.json(
+ { success: false, error: error.message },
+ { status: 400 }
+ );
+ }
+ return NextResponse.json(
+ {
+ success: false,
+ error: error instanceof Error ? error.message : "重生高清预览失败",
+ },
+ { status: 500 }
+ );
+ }
+}
+
diff --git a/app/(chat)/api/video/storyboard/tasks/[taskId]/shots/[shotIndex]/route.ts b/app/(chat)/api/video/storyboard/tasks/[taskId]/shots/[shotIndex]/route.ts
new file mode 100644
index 000000000..11d87eb52
--- /dev/null
+++ b/app/(chat)/api/video/storyboard/tasks/[taskId]/shots/[shotIndex]/route.ts
@@ -0,0 +1,103 @@
+import { NextResponse } from "next/server";
+import { z } from "zod";
+import { auth } from "@/app/(auth)/auth";
+import {
+ getVideoStoryboardTaskDetail,
+ markVideoStoryboardDraftForRevision,
+ updateVideoStoryboardShotPrompt,
+} from "@/lib/video-agent/task-store";
+import { updateStoryboardShotPromptAndRegenerateHd } from "@/lib/video-agent/revise-storyboard-task";
+import {
+ requireStoryboardTaskAccess,
+ StoryboardTaskAccessError,
+} from "@/lib/video-agent/user-task-access";
+
+const paramsSchema = z.object({
+ taskId: z.string().uuid(),
+ shotIndex: z.coerce.number().int().min(1).max(99),
+});
+
+const bodySchema = z.object({
+ prompt: z.string().trim().min(1).max(4000),
+ regenerateHd: z.boolean().optional().default(true),
+});
+
+export async function PATCH(
+ request: Request,
+ { params }: { params: Promise<{ taskId: string; shotIndex: string }> }
+) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+
+ const parsedParams = paramsSchema.parse(await params);
+ const parsedBody = bodySchema.parse(await request.json().catch(() => ({})));
+ const detail = await requireStoryboardTaskAccess({
+ taskId: parsedParams.taskId,
+ userId: session.user.id,
+ });
+
+ const status = String((detail.task as any).status || "").trim();
+ const confirmationStatus = String((detail.task as any).confirmationStatus || "").trim();
+ if (status === "running" && confirmationStatus === "confirmed") {
+ return NextResponse.json(
+ { success: false, error: "任务正在执行视频,暂时不能修改镜头" },
+ { status: 409 }
+ );
+ }
+
+ if (parsedBody.regenerateHd) {
+ const result = await updateStoryboardShotPromptAndRegenerateHd({
+ taskId: parsedParams.taskId,
+ shotIndex: parsedParams.shotIndex,
+ prompt: parsedBody.prompt,
+ });
+ return NextResponse.json({
+ success: result.ok,
+ message: result.message,
+ task: result.detail?.task || detail.task,
+ shots: result.detail?.shots || detail.shots,
+ });
+ }
+
+ await markVideoStoryboardDraftForRevision({
+ taskId: parsedParams.taskId,
+ });
+ await updateVideoStoryboardShotPrompt({
+ taskId: parsedParams.taskId,
+ shotIndex: parsedParams.shotIndex,
+ prompt: parsedBody.prompt,
+ });
+ const nextDetail = await getVideoStoryboardTaskDetail(parsedParams.taskId);
+
+ return NextResponse.json({
+ success: true,
+ message: `已更新第 ${parsedParams.shotIndex} 镜头文案。`,
+ task: nextDetail?.task || detail.task,
+ shots: nextDetail?.shots || detail.shots,
+ });
+ } catch (error) {
+ if (error instanceof StoryboardTaskAccessError) {
+ return NextResponse.json(
+ { success: false, error: error.message },
+ { status: error.status }
+ );
+ }
+ if (error instanceof z.ZodError) {
+ return NextResponse.json(
+ { success: false, error: error.message },
+ { status: 400 }
+ );
+ }
+ return NextResponse.json(
+ {
+ success: false,
+ error: error instanceof Error ? error.message : "更新镜头失败",
+ },
+ { status: 500 }
+ );
+ }
+}
+
diff --git a/app/api/agents/[id]/route.ts b/app/api/agents/[id]/route.ts
index 723be2ac9..a21cd2aec 100644
--- a/app/api/agents/[id]/route.ts
+++ b/app/api/agents/[id]/route.ts
@@ -5,8 +5,14 @@ import { ChatSDKError } from "@/lib/errors";
import {
deleteAgentPresetById,
getAccessibleAgentPresetById,
+ getSystemSetting,
updateAgentPreset,
} from "@/lib/db/queries";
+import {
+ AGENT_SECONDARY_TYPE_CONFIG_KEY,
+ isValidSecondaryTypeForPresetType,
+ normalizeAgentSecondaryTypeConfig,
+} from "@/lib/agents/secondary-type-config";
const slotSchema = z.object({
title: z.string().trim().max(128).optional(),
@@ -20,7 +26,8 @@ const slotSchema = z.object({
const updateAgentSchema = z
.object({
name: z.string().trim().min(1).max(128).optional(),
- type: z.enum(["image", "video"]).optional(),
+ type: z.enum(["image", "video", "text"]).optional(),
+ secondaryType: z.string().trim().max(128).nullable().optional(),
visibility: z.enum(["public", "private"]).optional(),
categoryPath: z
.object({
@@ -84,12 +91,43 @@ export async function PUT(
const { id } = await params;
const body = await request.json();
const data = updateAgentSchema.parse(body);
+ const existingPreset = await getAccessibleAgentPresetById({
+ id,
+ userId: session.user.id,
+ });
+
+ if (!existingPreset) {
+ return NextResponse.json({ success: false, error: "智能体不存在" }, { status: 404 });
+ }
+
+ const effectiveType = data.type || existingPreset.type;
+ const effectiveSecondaryType =
+ data.secondaryType === undefined ? existingPreset.secondaryType : data.secondaryType;
+ const secondaryTypeConfig = normalizeAgentSecondaryTypeConfig(
+ (
+ await getSystemSetting({ key: AGENT_SECONDARY_TYPE_CONFIG_KEY }).catch(() => undefined)
+ )?.value
+ );
+
+ if (
+ !isValidSecondaryTypeForPresetType({
+ presetType: effectiveType,
+ secondaryType: effectiveSecondaryType,
+ config: secondaryTypeConfig,
+ })
+ ) {
+ return NextResponse.json(
+ { success: false, error: "所选二级类型不在后台配置列表中" },
+ { status: 400 }
+ );
+ }
const preset = await updateAgentPreset({
id,
userId: session.user.id,
name: data.name,
type: data.type,
+ secondaryType: data.secondaryType ?? undefined,
visibility: data.visibility,
categoryPath: data.categoryPath ?? undefined,
description: data.description ?? undefined,
diff --git a/app/api/agents/route.ts b/app/api/agents/route.ts
index 21bf0b1bf..e37eae2b9 100644
--- a/app/api/agents/route.ts
+++ b/app/api/agents/route.ts
@@ -4,12 +4,18 @@ import { auth } from "@/app/(auth)/auth";
import { ChatSDKError } from "@/lib/errors";
import {
createAgentPreset,
+ getSystemSetting,
getVisibleAgentPresets,
} from "@/lib/db/queries";
+import {
+ AGENT_SECONDARY_TYPE_CONFIG_KEY,
+ isValidSecondaryTypeForPresetType,
+ normalizeAgentSecondaryTypeConfig,
+} from "@/lib/agents/secondary-type-config";
const listQuerySchema = z.object({
scope: z.enum(["all", "public", "private"]).default("all"),
- type: z.enum(["image", "video"]).optional(),
+ type: z.enum(["image", "video", "text"]).optional(),
productCategory: z.string().trim().max(64).optional(),
audienceTag: z.string().trim().max(64).optional(),
usageScenario: z.string().trim().max(64).optional(),
@@ -30,7 +36,8 @@ const slotSchema = z.object({
const createAgentSchema = z
.object({
name: z.string().trim().min(1).max(128),
- type: z.enum(["image", "video"]),
+ type: z.enum(["image", "video", "text"]),
+ secondaryType: z.string().trim().max(128).optional(),
visibility: z.enum(["public", "private"]).default("private"),
categoryPath: z
.object({
@@ -118,11 +125,30 @@ export async function POST(request: NextRequest) {
const body = await request.json();
const data = createAgentSchema.parse(body);
+ const secondaryTypeConfig = normalizeAgentSecondaryTypeConfig(
+ (
+ await getSystemSetting({ key: AGENT_SECONDARY_TYPE_CONFIG_KEY }).catch(() => undefined)
+ )?.value
+ );
+
+ if (
+ !isValidSecondaryTypeForPresetType({
+ presetType: data.type,
+ secondaryType: data.secondaryType,
+ config: secondaryTypeConfig,
+ })
+ ) {
+ return NextResponse.json(
+ { success: false, error: "所选二级类型不在后台配置列表中" },
+ { status: 400 }
+ );
+ }
const preset = await createAgentPreset({
userId: session.user.id,
name: data.name,
type: data.type,
+ secondaryType: data.secondaryType,
visibility: data.visibility,
categoryPath: data.categoryPath,
description: data.description,
diff --git a/app/api/site-settings/route.ts b/app/api/site-settings/route.ts
index b9cb520dd..247092dfc 100644
--- a/app/api/site-settings/route.ts
+++ b/app/api/site-settings/route.ts
@@ -24,6 +24,11 @@ import {
normalizeDulupayConfig,
normalizeDulupayForPublic,
} from "@/lib/payment/dulupay-config";
+import {
+ AGENT_SECONDARY_TYPE_CONFIG_KEY,
+ defaultAgentSecondaryTypeConfig,
+ normalizeAgentSecondaryTypeConfig,
+} from "@/lib/agents/secondary-type-config";
import {
defaultOssImageMigrationConfig,
normalizeOssImageMigrationConfig,
@@ -208,6 +213,7 @@ export async function GET() {
storageFailoverConfigSetting,
ossImageMigrationConfigSetting,
agentCategoryTreeConfigSetting,
+ agentSecondaryTypeConfigSetting,
] = await Promise.all([
db.select().from(siteSettings).limit(1).then(rows => rows[0]),
db
@@ -312,6 +318,12 @@ export async function GET() {
.where(eq(systemSettings.key, AGENT_CATEGORY_TREE_CONFIG_KEY))
.limit(1)
.then(rows => rows[0]),
+ db
+ .select()
+ .from(systemSettings)
+ .where(eq(systemSettings.key, AGENT_SECONDARY_TYPE_CONFIG_KEY))
+ .limit(1)
+ .then(rows => rows[0]),
]);
const notification = notificationSetting?.value || { enabled: false, title: "", content: "" };
@@ -350,6 +362,9 @@ export async function GET() {
const agentCategoryTreeConfig = normalizeAgentCategoryTreeConfig(
agentCategoryTreeConfigSetting?.value
);
+ const agentSecondaryTypeConfig = normalizeAgentSecondaryTypeConfig(
+ agentSecondaryTypeConfigSetting?.value
+ );
// 如果没有设置记录,返回默认值
const baseSettings = settings || {
@@ -376,6 +391,7 @@ export async function GET() {
canvasEnabled: true,
agentEnabled: true,
agentCategoryTreeConfig: normalizeAgentCategoryTreeConfig(undefined),
+ agentSecondaryTypeConfig: defaultAgentSecondaryTypeConfig,
realtimeCanvasEnabled: true,
realtimeCanvasServiceEnabled: true,
storageFailoverConfig,
@@ -418,6 +434,7 @@ export async function GET() {
lovartPromptTagVisibility,
contactConfig,
agentCategoryTreeConfig,
+ agentSecondaryTypeConfig,
membershipConfig,
membershipUserOverrides,
pointsSystemConfig,
@@ -474,6 +491,13 @@ const updateSchema = z.object({
brands: z.array(z.string().trim().min(1).max(64)).max(200).optional(),
})
.optional(),
+ agentSecondaryTypeConfig: z
+ .object({
+ image: z.array(z.string().trim().min(1).max(64)).max(200).optional(),
+ video: z.array(z.string().trim().min(1).max(64)).max(200).optional(),
+ text: z.array(z.string().trim().min(1).max(64)).max(200).optional(),
+ })
+ .optional(),
realtimeCanvasEnabled: z.boolean().optional(),
aiDetailGenerationEnabled: z.boolean().optional(),
canvasDefaultModelId: z.string().nullable().optional(),
@@ -550,6 +574,7 @@ export async function PUT(request: NextRequest) {
lovartPromptTagVisibility,
contactConfig,
agentCategoryTreeConfig,
+ agentSecondaryTypeConfig,
membershipConfig,
membershipUserOverrides,
pointsSystemConfig,
@@ -734,6 +759,26 @@ export async function PUT(request: NextRequest) {
});
}
+ if (agentSecondaryTypeConfig !== undefined) {
+ const normalized = normalizeAgentSecondaryTypeConfig(agentSecondaryTypeConfig);
+ await db
+ .insert(systemSettings)
+ .values({
+ key: AGENT_SECONDARY_TYPE_CONFIG_KEY,
+ value: normalized,
+ updatedAt: new Date(),
+ updatedBy: session.user.id,
+ })
+ .onConflictDoUpdate({
+ target: systemSettings.key,
+ set: {
+ value: normalized,
+ updatedAt: new Date(),
+ updatedBy: session.user.id,
+ },
+ });
+ }
+
if (membershipConfig !== undefined) {
const normalized = normalizeMembershipConfig(membershipConfig);
await db
@@ -902,6 +947,10 @@ export async function PUT(request: NextRequest) {
agentCategoryTreeConfig !== undefined
? normalizeAgentCategoryTreeConfig(agentCategoryTreeConfig)
: normalizeAgentCategoryTreeConfig(undefined),
+ agentSecondaryTypeConfig:
+ agentSecondaryTypeConfig !== undefined
+ ? normalizeAgentSecondaryTypeConfig(agentSecondaryTypeConfig)
+ : defaultAgentSecondaryTypeConfig,
membershipConfig:
membershipConfig !== undefined
? normalizeMembershipConfig(membershipConfig)
diff --git a/components/agents/AgentPresetCard.tsx b/components/agents/AgentPresetCard.tsx
index e34cfe53c..ecf288c69 100644
--- a/components/agents/AgentPresetCard.tsx
+++ b/components/agents/AgentPresetCard.tsx
@@ -77,8 +77,13 @@ export function AgentPresetCard({
)}
- {preset.type === "video" ? "视频" : "生图"}
+ {preset.type === "video" ? "视频" : preset.type === "text" ? "生文" : "生图"}
+ {preset.secondaryType ? (
+
+ {preset.secondaryType}
+
+ ) : null}
{categoryBadges.map((item) => (
{item}
diff --git a/components/agents/AgentPresetDialog.tsx b/components/agents/AgentPresetDialog.tsx
index 80dd6f6f3..e93e03f39 100644
--- a/components/agents/AgentPresetDialog.tsx
+++ b/components/agents/AgentPresetDialog.tsx
@@ -27,6 +27,7 @@ import { fetcher } from "@/lib/utils";
import { imageGenerationService } from "@/services/canvas";
import { agentService } from "@/services/agents";
import type {
+ AgentPresetType,
AgentCategoryTreeConfig,
AgentPreset,
AgentPresetSlot,
@@ -44,6 +45,11 @@ interface AgentPresetDialogProps {
open: boolean;
onOpenChange: (open: boolean) => void;
categoryTreeConfig: AgentCategoryTreeConfig;
+ secondaryTypeConfig: {
+ image: string[];
+ video: string[];
+ text: string[];
+ };
initialPreset?: AgentPreset | null;
onSaved?: (preset: AgentPreset) => void;
}
@@ -52,6 +58,7 @@ export function AgentPresetDialog({
open,
onOpenChange,
categoryTreeConfig,
+ secondaryTypeConfig,
initialPreset,
onSaved,
}: AgentPresetDialogProps) {
@@ -61,11 +68,13 @@ export function AgentPresetDialog({
displayName: string;
supportsImageGen?: boolean;
providerType?: string;
+ isMultimodal?: boolean;
}>;
}>("/api/models", fetcher);
const [name, setName] = useState("");
- const [type, setType] = useState<"image" | "video">("image");
+ const [type, setType] = useState<"image" | "video" | "text">("image");
+ const [secondaryType, setSecondaryType] = useState("");
const [visibility, setVisibility] = useState<"public" | "private">("private");
const [categoryPath, setCategoryPath] = useState<{
productCategory: string;
@@ -109,11 +118,22 @@ export function AgentPresetDialog({
if (type === "video") {
return models.filter((model) => videoProviderTypes.has(String(model.providerType || "")));
}
+ if (type === "text") {
+ return models.filter(
+ (model) =>
+ model.isMultimodal === true &&
+ !videoProviderTypes.has(String(model.providerType || ""))
+ );
+ }
return models.filter(
(model) =>
model.supportsImageGen && !videoProviderTypes.has(String(model.providerType || ""))
);
}, [modelsData?.models, type, videoProviderTypes]);
+ const secondaryTypeOptions = useMemo(
+ () => secondaryTypeConfig[type] || [],
+ [secondaryTypeConfig, type]
+ );
useEffect(() => {
if (!open) return;
@@ -141,6 +161,7 @@ export function AgentPresetDialog({
const resetState = () => {
setName("");
setType("image");
+ setSecondaryType("");
setVisibility("private");
setCategoryPath({
productCategory: "",
@@ -176,6 +197,7 @@ export function AgentPresetDialog({
setName(initialPreset.name || "");
setType(initialPreset.type);
+ setSecondaryType(String(initialPreset.secondaryType || ""));
setVisibility(initialPreset.visibility);
setCategoryPath({
productCategory: String(initialPreset.categoryPath?.productCategory || ""),
@@ -330,6 +352,7 @@ export function AgentPresetDialog({
const payload = {
name: name.trim(),
type,
+ secondaryType: secondaryType || undefined,
visibility,
categoryPath: {
productCategory: categoryPath.productCategory || undefined,
@@ -455,8 +478,9 @@ export function AgentPresetDialog({
类型
+
+
二级类型
+
+
+ 二级类型由管理员在系统设置中统一配置。
+
+
产品品类
- {hasAgentUploadSlots ? (
+ {selectedAgentPreset.type === "text" ? (
+
+ 可上传图片做分析,也可直接输入文案需求
+
+ ) : hasAgentUploadSlots ? (
请上传产品图1、产品图2、模特图
diff --git a/docs/DeerFlow调研与Agent升级建议-2026-03-12.md b/docs/DeerFlow调研与Agent升级建议-2026-03-12.md
new file mode 100644
index 000000000..0d55ac50f
--- /dev/null
+++ b/docs/DeerFlow调研与Agent升级建议-2026-03-12.md
@@ -0,0 +1,559 @@
+# DeerFlow 调研与 Agent 升级建议(2026-03-12)
+
+> 目标:在不推翻现有架构的前提下,吸收开源 `DeerFlow` 在对话理解、技能调用、上下文工程、子代理协作上的强项,提升本项目 Agent 能力。
+> 适用范围:`ai-chatbot-main` 当前聊天 Agent、视频分镜 Agent、无限画布 Agent。
+
+---
+
+## 1. 先说结论
+
+**建议学习 DeerFlow,但不建议直接整套迁移。**
+
+原因:
+
+1. DeerFlow 强项在于:
+ - 复杂任务拆解
+ - 子代理协作
+ - 工具/技能编排
+ - 人类确认与工作流状态管理
+ - 长期记忆与上下文工程
+2. 我们当前项目已经有:
+ - LangGraph
+ - 工具注册表
+ - specialist/skill route
+ - context memory
+ - 画布上下文
+ - 视频 storyboard pipeline 骨架
+3. 真正缺的不是“换框架”,而是把这些能力从“分散模块”升级成更稳定的 **统一 Agent Runtime**。
+
+所以最合理的路线是:
+
+- **不替换现有 Agent**
+- **吸收 DeerFlow 的 runtime 设计思想**
+- **补齐我们自己的 workflow state / middleware / subagent / human-in-the-loop**
+
+---
+
+## 2. DeerFlow 值得学习的核心能力
+
+根据 DeerFlow 官方仓库与文档,它不是一个简单的聊天机器人,而是一个偏“super agent runtime”的框架。对我们最有价值的点主要有 6 个。
+
+### 2.1 技能系统(Skills)
+
+DeerFlow 把技能视为可复用的高阶能力,不只是单个 tool call。
+
+对我们有启发的是:
+
+- 技能应该带有清晰的职责边界
+- 技能可以组合成工作流
+- 技能不等于单个模型调用
+
+这和我们当前“九宫格检测 / 分镜映射 / 视频调度”其实天然契合。
+
+### 2.2 子代理系统(Subagents)
+
+DeerFlow 强调把复杂任务拆给不同子代理,而不是一个主代理包办所有规划与执行。
+
+这对我们尤其重要,因为我们现在的任务已经明显分裂成多个 domain:
+
+- 普通对话
+- 看图分析
+- 修图
+- 生图
+- 生视频
+- 画布发布
+- 分镜任务
+
+再往后还有:
+
+- 多宫格分镜图生成
+- 高清镜头图重生
+- 用户确认管理
+- 视频镜头调度
+
+这些都非常适合子代理化。
+
+### 2.3 Middleware 机制
+
+DeerFlow 2.0 文档里一个很重要的方向,是把“状态处理、确认、内存更新、节点前后处理”抽成 middleware,而不是全写在单个 graph 节点里。
+
+这正是我们当前最缺的部分。
+
+我们现在虽然有:
+
+- `smart-analyzer`
+- `skill-router`
+- `specialists`
+- `auto-agent-graph`
+
+但缺少一个统一的 middleware 层来做:
+
+- 目标图绑定
+- 任务确认门
+- 安全/策略过滤
+- 记忆更新
+- 结果发布策略
+
+### 2.4 明确的 Workflow State
+
+DeerFlow 明显在往“显式工作流状态”方向演进,而不是只靠 prompt 和记忆里的隐式语义。
+
+这对我们的视频分镜任务尤其关键。
+
+比如我们现在正在设计的多宫格视频流程,天然就应该有:
+
+- `draft`
+- `slicing`
+- `hd_image_generating`
+- `awaiting_user_confirmation`
+- `confirmed`
+- `video_generating`
+- `completed`
+
+这类状态不应该只存在数据库里,也应该存在 Agent Runtime 的显式状态里。
+
+### 2.5 Human-in-the-loop
+
+DeerFlow 支持人在流程里介入确认,这一点和我们当前需求高度一致。
+
+我们后面做多宫格分镜视频时,最重要的一条就是:
+
+- 用户没确认,不允许开始生视频
+
+如果没有“人类确认是 runtime 的一等公民”这个理念,后续会很容易又退回“模型自己判断继续执行”的老路。
+
+### 2.6 Long-term Memory / Context Engineering
+
+DeerFlow 已经把长期记忆和上下文工程放进核心能力里。
+
+我们当前项目也已经开始做这件事,但还偏“局部补丁式”:
+
+- 当前图像目标定位
+- working memory
+- 画布选中上下文
+
+真正还没形成的是:
+
+- 用户偏好长期记忆
+- agent 可消费的结构化记忆摘要
+- 任务过程中的 thread state 与 memory state 分层
+
+---
+
+## 3. 我们当前项目已经有什么
+
+从当前代码来看,我们其实已经有了一个不错的基础,不需要从零重做。
+
+### 3.1 当前已有能力
+
+#### Agent Graph
+
+- `lib/ai/auto-agent-graph.ts`
+
+已经使用 `LangGraph`,而不是简单的 if/else 路由,这一点和 DeerFlow 的底层理念是一致的。
+
+#### Intent Analyzer
+
+- `lib/ai/smart-analyzer.ts`
+
+已经在做:
+
+- 意图识别
+- prompt 优化
+- 视频/图片/分析任务区分
+
+#### Skill Route / Specialists
+
+- `lib/ai/skill-router.ts`
+- `lib/ai/specialists/index.ts`
+
+已经有了“specialist + tool allowlist”的雏形,这可以视为 DeerFlow 风格 subagent 的前身。
+
+#### Tool Registry
+
+- `lib/ai/tools-registry.ts`
+
+已经有运行时工具注册和上下文过滤能力。
+
+#### Context Memory
+
+- `lib/ai/context-memory/types.ts`
+
+已经支持:
+
+- 当前上传图
+- 画布选中图
+- 历史结果图
+- working memory
+
+#### 视频 Storyboard Pipeline 骨架
+
+- `lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts`
+- `lib/video-agent/task-store.ts`
+
+已经具备:
+
+- 切片检测
+- 分镜映射
+- 调度
+- DB 任务状态
+
+这说明我们不是“要不要上 Agent”的阶段,而是“Agent Runtime 要不要体系化升级”的阶段。
+
+---
+
+## 4. 我们当前真正缺什么
+
+### 4.1 缺少统一的 Agent Runtime State
+
+当前很多信息分散在:
+
+- `analysis`
+- `plan`
+- `execution`
+- DB task
+- canvas state
+- working memory
+
+但缺少一层统一状态对象把这些东西串起来。
+
+后果是:
+
+- 复杂任务越多,graph 越难维护
+- 确认态、重试态、部分完成态难做
+- 很难把“Agent 理解能力”和“任务执行能力”解耦
+
+### 4.2 缺少 Middleware 层
+
+我们现在很多逻辑都直接塞在 graph planning 或 API route 里。
+
+这会导致:
+
+- 加一个新能力就要改很多地方
+- 很难复用“确认门”“上下文整理”“结果回写策略”
+- 视频、画布、详情生成无法共享行为层
+
+### 4.3 缺少真正的 Subagent Delegation
+
+现在我们虽然有 specialist,但更多像“专家路由 + 工具白名单”,还不是完整子代理。
+
+完整子代理至少应该具备:
+
+- 自己的职责边界
+- 自己的输入/输出结构
+- 可以被主代理调用
+- 可以独立测试
+
+### 4.4 缺少 Skill Package 化
+
+我们现在的“skill”更像是代码模块,不像 DeerFlow 那样是可以被稳定调度和演进的能力包。
+
+当前问题:
+
+- 技能定义分散
+- 调用方式不统一
+- 很难做版本化和可观测性
+
+### 4.5 缺少强约束的人类确认机制
+
+我们现在可以“提示用户确认”,但还没有把“确认门”上升为系统级策略。
+
+这会导致:
+
+- 一旦某个链路忘记拦截,就可能直接执行
+- 视频、多宫格、批量任务这些高成本流程风险很高
+
+---
+
+## 5. 建议吸收 DeerFlow 的方式
+
+不做框架替换,做 **能力吸收式升级**。
+
+### 5.1 第一层:统一 Agent Runtime
+
+建议新增:
+
+- `lib/ai/runtime/agent-thread-state.ts`
+- `lib/ai/runtime/agent-workflow-state.ts`
+- `lib/ai/runtime/agent-context-builder.ts`
+
+职责:
+
+- 管理单轮对话状态
+- 管理任务工作流状态
+- 统一拼装上下文、记忆、画布、任务状态
+
+### 5.2 第二层:Middleware 体系
+
+建议新增:
+
+- `lib/ai/runtime/middlewares/intent-guard.ts`
+- `lib/ai/runtime/middlewares/target-binding.ts`
+- `lib/ai/runtime/middlewares/human-confirmation-gate.ts`
+- `lib/ai/runtime/middlewares/memory-update.ts`
+- `lib/ai/runtime/middlewares/publish-policy.ts`
+
+这样后续像“多宫格确认门”就不是写死在某个 route 里,而是一个通用中间件。
+
+### 5.3 第三层:Subagent 体系
+
+建议把现有 specialist 升级成真正可调用的 subagent:
+
+- `conversation-subagent`
+- `vision-subagent`
+- `image-edit-subagent`
+- `image-generate-subagent`
+- `video-subagent`
+- `canvas-subagent`
+- `storyboard-subagent`
+
+新增最重要的两个:
+
+- `storyboard-subagent`
+- `confirmation-subagent`
+
+前者负责:
+
+- 宫格规划
+- 镜头归一化
+- 高清镜头图规划
+
+后者负责:
+
+- 输出确认文案
+- 识别用户确认/修改意图
+- 阻止未确认执行
+
+### 5.4 第四层:Skill Package 化
+
+建议把“技能”升级为可注册能力包,而不是只保留 TS 函数。
+
+可以采用轻量形式:
+
+- `lib/agent-skills//index.ts`
+- `lib/agent-skills//manifest.ts`
+
+`manifest.ts` 建议包含:
+
+- `id`
+- `name`
+- `version`
+- `domain`
+- `inputSchema`
+- `outputSchema`
+- `requiresHumanConfirmation`
+- `producesAssets`
+- `canPublishToCanvas`
+
+优先 package 化的技能:
+
+1. `storyboard-grid-detector`
+2. `storyboard-script-normalizer`
+3. `storyboard-hd-redraw`
+4. `storyboard-confirmation-gate`
+5. `storyboard-video-sequencer`
+
+### 5.5 第五层:长期记忆增强
+
+建议新增:
+
+- `lib/ai/memory/user-preference-memory.ts`
+- `lib/ai/memory/task-memory.ts`
+
+分开存两类内容:
+
+1. **用户长期偏好**
+ - 喜欢哪种风格
+ - 常用比例
+ - 常用模型
+ - 是否偏好先确认再执行
+2. **任务过程记忆**
+ - 当前宫格类型
+ - 当前镜头草案
+ - 已确认镜头
+ - 待重生镜头
+
+这样才能让 Agent 既懂“这个用户平时怎么用”,又懂“这次任务现在到了哪一步”。
+
+---
+
+## 6. 对多宫格视频需求的直接帮助
+
+DeerFlow 思路对我们当前多宫格视频需求最直接的帮助,不是“更会聊天”,而是让整个流程更稳定。
+
+### 6.1 生成多宫格图前
+
+由 `storyboard-subagent`:
+
+- 判断该用 4/6/9 宫格
+- 自动追加标准宫格 prompt
+- 输出结构化镜头规划
+
+### 6.2 切片与高清图阶段
+
+由 skill chain:
+
+- `grid-detector-skill`
+- `shot-mapper-skill`
+- `hd-redraw-skill`
+
+每个技能都有固定输入输出,不混在一个大函数里。
+
+### 6.3 用户确认阶段
+
+由 `human-confirmation-gate` middleware:
+
+- 拦住视频执行
+- 给用户输出可修改镜头文案
+- 监听“确认开始生成视频”之类确认语义
+
+### 6.4 视频执行阶段
+
+由 `video-subagent + sequencer-skill`:
+
+- 按确认后的镜头串行执行
+- 支持失败重试
+- 支持只重跑单镜头
+
+---
+
+## 7. 不建议直接照搬 DeerFlow 的地方
+
+### 7.1 不建议整套替换 UI / 交互模型
+
+DeerFlow 更偏通用 super agent / deep research 形态。
+
+我们当前项目是:
+
+- 多模态创作
+- 无限画布
+- 图片/视频生成
+- 强任务流
+
+所以 UI、消息结构、画布联动、媒体资产回流这些都不能直接照搬。
+
+### 7.2 不建议立刻重构成完全独立 repo 架构
+
+我们当前代码已经深度绑定:
+
+- `Next.js route handlers`
+- `Vercel AI SDK`
+- `LangGraph`
+- `canvas` 状态
+- `DB task store`
+
+直接大迁移只会让功能倒退。
+
+### 7.3 不建议先做“通用超级代理”,再回头补产品链路
+
+当前最正确顺序应该是:
+
+1. 先用 DeerFlow 思路补强我们的产品主链路
+2. 再抽象出通用 Agent Runtime
+
+而不是反过来。
+
+---
+
+## 8. 推荐升级路线
+
+### Phase 0:最小升级
+
+目标:不换架构,只补关键 runtime 能力
+
+- [ ] 增加 workflow state
+- [ ] 增加 confirmation middleware
+- [ ] 增加 storyboard-subagent
+- [ ] 增加 skill manifest 机制
+
+### Phase 1:多宫格视频先吃到红利
+
+- [ ] 用 subagent 重构多宫格分镜理解
+- [ ] 用 middleware 实现“未确认不得执行”
+- [ ] 用 skill package 管理分镜检测/切片/高清图/视频调度
+
+### Phase 2:扩到全局 Agent
+
+- [ ] 图片编辑
+- [ ] 生图
+- [ ] 视频
+- [ ] 画布
+- [ ] AI 详情
+
+统一切到同一个 Agent Runtime 层。
+
+### Phase 3:长期记忆与偏好学习
+
+- [ ] 用户偏好长期记忆
+- [ ] 任务级 thread state
+- [ ] 记忆摘要和召回策略
+
+---
+
+## 9. 我建议现在立刻做的事情
+
+如果目标是“先把多宫格视频方案做好”,我建议不要先做大重构,而是先做 4 个高价值改造:
+
+### 9.1 新增 `human-confirmation-gate`
+
+这是当前收益最大的一步。
+
+直接服务于:
+
+- 多宫格分镜视频
+- 高成本视频任务
+- 批量图片任务
+
+### 9.2 新增 `storyboard-subagent`
+
+把下面几件事从当前大 graph 里拆出去:
+
+- 宫格数量判断
+- 分镜归一
+- 标准宫格 prompt 注入
+- 高清镜头图阶段规划
+
+### 9.3 技能 manifest 化
+
+先从视频分镜链路做试点:
+
+- `grid-detector`
+- `script-normalizer`
+- `hd-redraw`
+- `video-sequencer`
+
+### 9.4 Thread State 显式化
+
+至少先给视频任务做 thread state:
+
+- 当前 gridType
+- 当前 shots
+- 当前 confirmationStatus
+- 当前 hd image progress
+- 当前 video progress
+
+这一步做完后,后续所有复杂 Agent 流程都会更容易。
+
+---
+
+## 10. 最终判断
+
+DeerFlow 值得认真研究,原因不是它“更强”,而是它已经把很多 super agent 必须补的工程层问题想清楚了:
+
+- workflow state
+- middleware
+- subagent
+- skills
+- long-term memory
+- human-in-the-loop
+
+这些正好都是我们现在开始碰到的瓶颈。
+
+所以建议是:
+
+- **理念全面学习**
+- **实现局部吸收**
+- **架构不整套迁移**
+- **先用在多宫格分镜视频链路上试点**
+
+如果这条链路跑通,再把同样的 runtime 能力推广到图片编辑、生图、画布和 AI 详情。
diff --git a/docs/多宫格分镜视频开发清单-2026-03-12.md b/docs/多宫格分镜视频开发清单-2026-03-12.md
new file mode 100644
index 000000000..7c3deb53f
--- /dev/null
+++ b/docs/多宫格分镜视频开发清单-2026-03-12.md
@@ -0,0 +1,436 @@
+# 多宫格分镜视频开发清单(2026-03-12)
+
+> 面向开发落地
+> 范围:`4宫格 / 6宫格 / 9宫格` 标准切片、高清镜头图阶段、用户确认后再生视频
+> 对应需求文档:`docs/多宫格分镜视频需求补充-2026-03-12.md`
+
+---
+
+## 0. 开发目标
+
+本轮不是“继续优化九宫格视频”,而是把整条链路正式升级为:
+
+`Agent 规划多宫格 -> 标准分镜图生成 -> 自动识别切片 -> 每片生成高清镜头图 -> 用户确认镜头文案 -> 串行生成视频 -> 回写数据库与画布`
+
+建议拆成四大模块并行推进:
+
+1. `Agent`
+2. `数据库`
+3. `API`
+4. `画布交互`
+
+建议开发顺序:
+
+1. 先完成 `数据库`
+2. 再完成 `Agent`
+3. 然后完成 `API`
+4. 最后接 `画布交互`
+
+原因:
+
+- 数据结构要先定,否则前后端字段会反复改
+- Agent 规则要先稳定,否则后端接口难以确定输入输出
+- API 要在状态机清晰后再落
+- 画布是最依赖整体状态的最后一层
+
+---
+
+## 1. Agent 开发清单
+
+目标:让 Agent 不只是“识别九宫格”,而是能稳定产出 **可切片的标准多宫格图**,并在视频前承担“分镜确认门”的职责。
+
+### 1.1 意图与宫格数判断
+
+- [ ] 在 `lib/ai/smart-analyzer.ts` 增加多宫格分镜识别词:
+ - `四宫格`
+ - `六宫格`
+ - `九宫格`
+ - `分镜拼图`
+ - `分镜图`
+ - `镜头图`
+- [ ] 在 `lib/ai/auto-agent-graph.ts` 增加 `storyboardGridType` 推断字段
+- [ ] 默认规则:
+ - 未指定镜头数但目标是视频分镜时,默认建议 `9宫格`
+ - 用户明确说 4 段时,使用 `grid_2x2`
+ - 用户明确说 6 段时,使用 `grid_3x2`
+ - 用户明确说 9 段时,使用 `grid_3x3`
+
+### 1.2 标准分割提示词注入
+
+- [ ] 在 Agent 提示词优化链路里增加“宫格结构化约束注入”
+- [ ] 为 `grid_2x2 / grid_3x2 / grid_3x3` 分别维护固定模板
+- [ ] 明确禁止跨格、透视倾斜、拼贴边框、编号文字、水印、字幕条
+- [ ] 对用户透明提示一句:
+ - “我会按标准宫格版式生成,方便系统自动切片”
+
+建议落点:
+
+- `lib/ai/auto-agent-graph.ts`
+- `lib/ai/smart-analyzer.ts`
+- 如需抽离模板:新增 `lib/video-agent/grid-prompt-templates.ts`
+
+### 1.3 分镜文案归一化
+
+- [ ] 现有 `script-normalizer-skill` 从固定 9 段改为“按 gridType 输出 N 段”
+- [ ] 规范输出字段:
+ - `gridType`
+ - `rows`
+ - `cols`
+ - `shots`
+ - `confirmationRequired`
+- [ ] 当镜头不足时,只生成草案,不自动补全并执行视频
+- [ ] 当用户后续改某一镜头时,仅更新对应镜头
+
+建议落点:
+
+- `lib/video-agent/skills/script-normalizer.ts`
+- `lib/video-agent/types.ts`
+
+### 1.4 用户确认门
+
+- [ ] 增加“确认态回复模板”
+- [ ] Agent 输出镜头预览时必须明确提示:
+ - 可修改任意镜头
+ - 回复“确认开始生成视频”才会真正开始视频
+- [ ] 在确认前,Agent 只允许执行:
+ - 修改镜头文案
+ - 重生单张高清镜头图
+ - 重新整理宫格布局
+- [ ] 在确认后,才允许发起视频任务调度
+
+建议落点:
+
+- `lib/ai/auto-agent-graph.ts`
+- `app/(chat)/api/chat/route.ts`
+
+### 1.5 Agent 模块验收
+
+- [ ] 能正确判断 4/6/9 宫格
+- [ ] 生图 prompt 中已自动附加标准结构约束
+- [ ] 分镜输出镜头数与宫格数一致
+- [ ] 未确认前不会触发视频生成
+
+---
+
+## 2. 数据库开发清单
+
+目标:让“切片图 / 高清镜头图 / 视频结果 / 用户确认状态”全部可持久化、可恢复、可追踪。
+
+### 2.1 主任务表 `VideoStoryboardTask`
+
+当前文件:
+
+- `lib/db/schema.ts`
+- `lib/db/migrations/0048_add_video_storyboard_tasks.sql`
+
+需要新增字段:
+
+- [ ] `gridType`:`grid_2x2 | grid_3x2 | grid_3x3`
+- [ ] `gridRows`
+- [ ] `gridCols`
+- [ ] `confirmationStatus`:`draft | awaiting_user_confirmation | confirmed | rejected`
+- [ ] `confirmedAt`
+- [ ] `confirmedByMessageId`
+- [ ] `hdImageStageStatus`:`pending | running | completed | failed`
+
+### 2.2 子任务表 `VideoStoryboardShotTask`
+
+需要新增字段:
+
+- [ ] `sourceTileAssetId`
+- [ ] `hdImageUrl`
+- [ ] `hdImageAssetId`
+- [ ] `hdImagePrompt`
+- [ ] `hdImageStatus`:`pending | running | completed | failed`
+- [ ] `videoStatus`:`pending | waiting_confirmation | running | completed | failed`
+- [ ] `userApproved`
+- [ ] `userApprovedAt`
+- [ ] `displayOrder`
+
+### 2.3 统一资产挂接
+
+如果要和当前资产图谱方案一致,需要补以下资产类型:
+
+- [ ] `storyboard_tile`
+- [ ] `storyboard_hd_image`
+- [ ] `storyboard_video`
+
+并保证都能反查:
+
+- `taskId`
+- `shotIndex`
+- `chatId`
+- `canvasNodeId`
+
+建议落点:
+
+- `lib/db/schema.ts`
+- `docs/上下文长期记忆-图片目标定位-资产图谱表结构设计.md`
+- 如果已有资产表实现,补充对应写入逻辑
+
+### 2.4 查询与状态同步
+
+- [ ] 更新 `lib/video-agent/task-store.ts`
+- [ ] 新增高清图阶段的状态更新方法:
+ - `markVideoStoryboardShotHdRunning`
+ - `markVideoStoryboardShotHdCompleted`
+ - `markVideoStoryboardShotHdFailed`
+- [ ] 新增确认状态方法:
+ - `markVideoStoryboardAwaitingConfirmation`
+ - `markVideoStoryboardConfirmed`
+ - `markVideoStoryboardRejected`
+- [ ] `syncVideoStoryboardTaskStatus` 要能区分:
+ - 高清图阶段
+ - 等待确认阶段
+ - 视频阶段
+
+### 2.5 数据库模块验收
+
+- [ ] 一个任务能同时追踪切片图、高清镜头图、视频结果
+- [ ] 系统重启后还能恢复等待确认中的任务
+- [ ] 单镜头重生不会破坏主任务顺序
+
+---
+
+## 3. API 开发清单
+
+目标:把“检测切片、高清镜头图生成、确认放行、视频调度”拆成清晰接口,而不是全部压在一个聊天接口里。
+
+### 3.1 配置结构扩展
+
+当前文件:
+
+- `lib/video-agent/config.ts`
+
+需要新增配置:
+
+- [ ] `supportedGridTypes`
+- [ ] `enableHdImageStage`
+- [ ] `hdImageModelWhitelist`
+- [ ] `hdImageConcurrency`
+- [ ] `confirmationTimeoutHours`
+- [ ] `requireUserConfirmationBeforeVideo`,默认 `true`
+
+### 3.2 新增/改造内部 pipeline
+
+当前骨架:
+
+- `lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts`
+
+需要改造为更通用的多宫格 pipeline:
+
+- [ ] 输入不再只接受 9 宫格
+- [ ] 输出包含 `gridType / totalShots / hdImages`
+- [ ] 先做切片,再做高清镜头图,不直接串行发视频
+- [ ] 任务结束点从“视频全部完成”改为“进入待确认态”
+
+建议:
+
+- 保留原文件,内部升级
+- 或新增 `lib/video-agent/pipeline/storyboard-grid-pipeline.ts`
+
+### 3.3 新增确认接口
+
+建议新增:
+
+- [ ] `POST /api/video/storyboard/tasks/:taskId/confirm`
+- [ ] `POST /api/video/storyboard/tasks/:taskId/reject`
+- [ ] `PATCH /api/video/storyboard/tasks/:taskId/shots/:shotIndex`
+- [ ] `POST /api/video/storyboard/tasks/:taskId/shots/:shotIndex/regenerate-hd`
+
+职责:
+
+- `confirm`:把任务状态切到 `confirmed`
+- `reject`:退回 `draft` 或 `rejected`
+- `PATCH shot`:更新单镜头文案
+- `regenerate-hd`:重生单张高清镜头图
+
+### 3.4 视频调度接口
+
+已有方向:
+
+- admin 查询:`app/api/admin/video-storyboard-tasks/...`
+- 重跑:`app/api/admin/video-storyboard-tasks/[taskId]/rerun/route.ts`
+
+需要补充正式业务接口:
+
+- [ ] `POST /api/jobs/video-storyboard-dispatch`
+ - 只处理 `confirmationStatus=confirmed` 的任务
+- [ ] 调度器读取待执行镜头时,跳过未确认任务
+- [ ] 增加“只重跑失败视频镜头”的业务接口
+
+### 3.5 聊天主入口联动
+
+当前聊天入口:
+
+- `app/(chat)/api/chat/route.ts`
+
+这里需要做的不是“直接生成视频”,而是分流:
+
+- [ ] 命中多宫格 storyboard 时,走多宫格 pipeline
+- [ ] pipeline 完成高清图阶段后,把确认消息回写到聊天流
+- [ ] 用户确认语义命中后,再调用 confirm 接口或内部 confirm 逻辑
+
+### 3.6 API 模块验收
+
+- [ ] 创建任务后能进入 `awaiting_user_confirmation`
+- [ ] 用户确认接口触发后才会进入视频调度
+- [ ] 单镜头修改和重生高清图都能单独调用
+
+---
+
+## 4. 画布交互开发清单
+
+目标:用户在画布里先看到“有顺序的高清镜头图”,确认后再看到视频结果,整个过程不能乱序。
+
+### 4.1 画布节点阶段化展示
+
+当前相关文件:
+
+- `components/canvas/CanvasChatPanel.tsx`
+- `store/canvas/index.ts`
+- `types/canvas/index.ts`
+
+当前已有九宫格视频的画布处理偏向直接视频节点,需要改成两阶段:
+
+- [ ] 阶段 A:高清镜头图阶段,创建 `image` 占位节点
+- [ ] 阶段 B:视频生成阶段,更新为 `video` 节点或追加视频节点
+
+### 4.2 新增流式事件类型
+
+建议扩展现有 `data-videoStoryboard` 事件:
+
+- [ ] `hd_start`
+- [ ] `hd_shot_completed`
+- [ ] `hd_shot_failed`
+- [ ] `awaiting_confirmation`
+- [ ] `confirmation_received`
+- [ ] `video_start`
+- [ ] `video_shot_completed`
+- [ ] `video_shot_failed`
+
+当前 `CanvasChatPanel` 里已有:
+
+- `start`
+- `shot_completed`
+- `shot_failed`
+- `completed`
+
+需要按“高清图阶段”和“视频阶段”拆开。
+
+### 4.3 画布布局规则
+
+- [ ] 4宫格时按 `2x2` 落位
+- [ ] 6宫格时按 `3x2` 落位
+- [ ] 9宫格时按 `3x3` 落位
+- [ ] 高清镜头图和视频节点都保持同一镜头顺序
+- [ ] 节点标题要显示:
+ - `镜头 1`
+ - `镜头 2`
+ - `镜头 3`
+
+### 4.4 用户确认交互
+
+建议增加一块轻量确认 UI:
+
+- [ ] 在聊天消息里展示镜头草案确认卡
+- [ ] 每个镜头支持:
+ - 编辑文案
+ - 重新生成高清图
+- [ ] 提供总按钮:
+ - `确认开始生成视频`
+ - `返回修改`
+
+如不做复杂卡片,至少要:
+
+- [ ] 在聊天流里清晰列出镜头 1~N 文案
+- [ ] 用户回复确认前,画布只显示高清图,不显示视频进度
+
+### 4.5 画布持久化
+
+- [ ] 高清镜头图节点写入当前 `canvasProject.canvasData`
+- [ ] 视频结果节点与镜头图保持可追溯关系
+- [ ] 刷新页面后仍能恢复:
+ - 哪些镜头已生成高清图
+ - 当前是否在等待确认
+ - 哪些镜头视频已开始/完成
+
+### 4.6 画布模块验收
+
+- [ ] 画布能先显示镜头静帧,再显示视频
+- [ ] 顺序始终稳定,不乱插
+- [ ] 用户修改单镜头后,只更新对应节点
+
+---
+
+## 5. 推荐实施阶段
+
+### Phase A:状态机和数据结构
+
+- [ ] 扩表
+- [ ] 更新 `task-store`
+- [ ] 扩 `types`
+
+### Phase B:多宫格检测与高清镜头图阶段
+
+- [ ] 扩 `nine-grid-detector` 为多宫格 detector
+- [ ] 增加 `tile-hd-redraw-skill`
+- [ ] pipeline 停在 `awaiting_user_confirmation`
+
+### Phase C:用户确认门
+
+- [ ] 增加 confirm/reject/update-shot API
+- [ ] 增加 Agent 确认模板
+- [ ] 增加聊天侧确认逻辑
+
+### Phase D:视频调度与画布联动
+
+- [ ] 仅对 confirmed 任务发视频
+- [ ] 画布上分阶段展示高清图和视频
+- [ ] 单镜头修订和重跑
+
+---
+
+## 6. 重点文件清单
+
+### Agent
+
+- `lib/ai/smart-analyzer.ts`
+- `lib/ai/auto-agent-graph.ts`
+- `app/(chat)/api/chat/route.ts`
+
+### 数据库
+
+- `lib/db/schema.ts`
+- `lib/db/migrations/0048_add_video_storyboard_tasks.sql`
+- `lib/video-agent/task-store.ts`
+
+### API / Pipeline
+
+- `lib/video-agent/config.ts`
+- `lib/video-agent/types.ts`
+- `lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts`
+- `lib/video-agent/skills/nine-grid-detector.ts`
+- `lib/video-agent/skills/script-normalizer.ts`
+- `lib/video-agent/skills/storyboard-mapper.ts`
+- `lib/video-agent/skills/video-sequencer.ts`
+
+### 画布交互
+
+- `components/canvas/CanvasChatPanel.tsx`
+- `store/canvas/index.ts`
+- `types/canvas/index.ts`
+
+---
+
+## 7. 本轮建议先做什么
+
+如果要最快推进,我建议先开这 5 个子任务:
+
+1. 扩 `schema + task-store`,把 `gridType / confirmationStatus / hdImage*` 字段补齐
+2. 扩 `types + detector`,把 9 宫格改成通用多宫格
+3. 新增 `tile-hd-redraw-skill`
+4. 让 pipeline 停在 `awaiting_user_confirmation`
+5. 给聊天流加 `confirm` 放行逻辑
+
+这样做完后,即使视频阶段还没完全改好,用户也已经能看到“多宫格切片 -> 高清镜头图 -> 确认”的完整闭环。
diff --git a/docs/多宫格分镜视频需求补充-2026-03-12.md b/docs/多宫格分镜视频需求补充-2026-03-12.md
new file mode 100644
index 000000000..e3f832bea
--- /dev/null
+++ b/docs/多宫格分镜视频需求补充-2026-03-12.md
@@ -0,0 +1,382 @@
+# 多宫格分镜视频需求补充(2026-03-12)
+
+> 适用范围:`RUXA 视频` / 画布分镜视频工作流 / Agent 自动规划
+> 本文是对现有“九宫格视频 Agent 技能链”方案的补充,重点新增:
+> 1. 支持 `4宫格 / 6宫格 / 9宫格` 标准化切片
+> 2. 切片后先自动生成高清大图,并按顺序入库、回显到画布
+> 3. 必须经过用户确认分镜文案后,才允许开始调用生视频模型
+>
+> 开发拆解清单见:`docs/多宫格分镜视频开发清单-2026-03-12.md`
+> Agent 升级调研见:`docs/DeerFlow调研与Agent升级建议-2026-03-12.md`
+
+---
+
+## 1. 需求目标
+
+我们不再只支持“识别九宫格再直接生视频”,而是升级为统一的 **多宫格分镜图 -> 标准切片 -> 高清镜头图 -> 用户确认 -> 分镜视频** 流程。
+
+核心目标:
+
+1. Agent 在前置生图阶段就强制加入“标准分割版式提示词”,提高后续切片识别成功率
+2. 支持标准 `2x2 / 3x2 / 3x3` 三种宫格版式
+3. 切片后每个分镜先生成一张大尺寸高清静帧图,作为“视频前置确认素材”
+4. 高清静帧图必须按顺序写入数据库、同步显示到画布
+5. Agent 必须和用户逐段确认分镜文案;未确认前,禁止调用视频模型
+
+---
+
+## 2. 标准宫格类型定义
+
+### 2.1 标准版式
+
+系统当前只认以下三种标准版式:
+
+| 业务名称 | 标准版式 | 行列 | 顺序 |
+| --- | --- | --- | --- |
+| 4宫格 | `grid_2x2` | 2 行 2 列 | 从左到右、从上到下 |
+| 6宫格 | `grid_3x2` | 2 行 3 列 | 从左到右、从上到下 |
+| 9宫格 | `grid_3x3` | 3 行 3 列 | 从左到右、从上到下 |
+
+约束:
+
+- `6宫格` 当前统一采用 `3列 x 2行`
+- 不支持自由拼贴、不规则分栏、跨格大图、瀑布流拼图
+- 如用户上传的是非标准拼图,Agent 需提示“当前只支持 2x2 / 3x2 / 3x3 标准版式”
+
+### 2.2 推荐输出尺寸
+
+| 版式 | 推荐画布比例 | 推荐最小边长 |
+| --- | --- | --- |
+| `grid_2x2` | 1:1 | 2048 |
+| `grid_3x2` | 3:2 | 2304 |
+| `grid_3x3` | 1:1 | 2304 |
+
+说明:
+
+- 目标不是让用户记参数,而是 Agent 自动把这些结构要求注入到底层生图提示词
+- 若底层模型不稳定,优先保证“分割清晰可识别”,其次再追求艺术表现
+
+---
+
+## 3. Agent 必须追加的结构化提示词
+
+## 3.1 总体规则
+
+当 Agent 需要让模型生成“可切片的多宫格分镜图”时,必须自动追加结构化约束,不允许只使用自然语言脚本直接出图。
+
+必须追加的约束包括:
+
+- 严格等分网格
+- 固定外边距
+- 固定分隔线宽度
+- 所有主体必须完整落在单一格内
+- 主体、光效、文字、特效都不得跨越分隔线
+- 禁止倾斜透视、胶片边框、拼贴阴影、圆角相框、手撕纸边、重叠排版
+- 禁止编号文字、水印、Logo、字幕条覆盖在格线附近
+
+## 3.2 标准版式提示词模板
+
+### `grid_2x2`
+
+Agent 需要在最终生图 prompt 末尾追加:
+
+`严格 2x2 分镜拼图,2行2列,所有格子等宽等高,纯净清晰的统一分隔线,固定外边距,画面按从左到右从上到下阅读。每个主体必须完整处于单独格内,不得跨格,不得遮挡边界,不得透视倾斜,不得拼贴重叠,不得出现文字编号、水印、胶片边框。`
+
+### `grid_3x2`
+
+Agent 需要在最终生图 prompt 末尾追加:
+
+`严格 3x2 分镜拼图,3列2行,所有格子等宽等高,纯净清晰的统一分隔线,固定外边距,画面按从左到右从上到下阅读。每个主体必须完整处于单独格内,不得跨格,不得遮挡边界,不得透视倾斜,不得拼贴重叠,不得出现文字编号、水印、胶片边框。`
+
+### `grid_3x3`
+
+Agent 需要在最终生图 prompt 末尾追加:
+
+`严格 3x3 分镜拼图,3行3列,所有格子等宽等高,纯净清晰的统一分隔线,固定外边距,画面按从左到右从上到下阅读。每个主体必须完整处于单独格内,不得跨格,不得遮挡边界,不得透视倾斜,不得拼贴重叠,不得出现文字编号、水印、胶片边框。`
+
+## 3.3 Agent 对用户的引导要求
+
+如果用户说“帮我做九宫格/六宫格/四宫格分镜图”,Agent 需要主动补一句简短说明:
+
+- 系统会按标准宫格版式生成,便于后续自动切片
+- 如需继续做分镜视频,建议先确认每格对应一个镜头
+
+如果用户只说“做一个分镜拼图”,Agent 默认策略:
+
+1. 优先判断是否要做视频分镜
+2. 若是视频分镜且镜头数未知,默认先建议 `9宫格`
+3. 若用户明确说“只要 4 段 / 6 段”,分别切到 `4宫格 / 6宫格`
+
+---
+
+## 4. 多宫格识别与切片规则
+
+## 4.1 检测输出结构
+
+检测技能输出统一扩展为:
+
+```json
+{
+ "isStoryboardGrid": true,
+ "gridType": "grid_3x3",
+ "rows": 3,
+ "cols": 3,
+ "confidence": 0.93,
+ "cropBoxes": [
+ { "index": 1, "row": 1, "col": 1, "x": 0, "y": 0, "w": 640, "h": 640 }
+ ],
+ "qualityScore": 0.9
+}
+```
+
+## 4.2 切片顺序
+
+统一采用“阅读顺序”:
+
+1. 先行后列
+2. 每行从左到右
+3. 再进入下一行
+
+即:
+
+- `grid_2x2`:1-2 / 3-4
+- `grid_3x2`:1-2-3 / 4-5-6
+- `grid_3x3`:1-2-3 / 4-5-6 / 7-8-9
+
+## 4.3 识别失败时的退路
+
+- 置信度不足:Agent 提示用户改用标准宫格版式重新生成
+- 分隔线不清晰:Agent 提示用户“重新按标准宫格输出”
+- 非标准布局:不进入后续高清图/视频链路
+
+---
+
+## 5. 切片后先生成高清镜头图
+
+## 5.1 新增强制步骤
+
+切片完成后,不能直接开始生视频,必须先进入 **高清镜头图生成阶段**:
+
+1. 每个切片图作为一张独立输入图
+2. 调用生图模型或图像增强模型,生成大尺寸高清镜头图
+3. 高清图按切片顺序写入数据库
+4. 高清图按切片顺序回显到画布
+5. Agent 用这些高清图和对应文案向用户做分镜确认
+
+## 5.2 目标能力
+
+高清镜头图阶段要解决两个问题:
+
+1. 原始切片图尺寸过小,不适合直接做视频首帧
+2. 用户需要在正式生视频前先看到“每镜头的大图预览”
+
+## 5.3 推荐技能命名
+
+建议新增:
+
+- `tile-hd-redraw-skill`
+
+职责:
+
+- 输入:`tileImage + globalStyle + shotPrompt`
+- 输出:`hdImageUrl + hdPrompt + seed/metadata`
+- 支持失败重试
+- 支持只重生单个镜头
+
+## 5.4 生成顺序
+
+高清镜头图也要按顺序执行:
+
+- 可串行,也可小并发(最多 2~3)
+- 画布上按照镜头顺序落位
+- 未完成的镜头显示占位卡
+
+---
+
+## 6. 数据库存储要求
+
+## 6.1 主任务表新增字段建议
+
+`VideoStoryboardTask` 建议增加:
+
+- `gridType`:`grid_2x2 | grid_3x2 | grid_3x3`
+- `gridRows`
+- `gridCols`
+- `confirmationStatus`:`draft | awaiting_user_confirmation | confirmed | rejected`
+- `confirmedAt`
+- `confirmedByMessageId`
+
+## 6.2 子任务表新增字段建议
+
+`VideoStoryboardShotTask` 建议增加:
+
+- `sourceTileAssetId`
+- `hdImageUrl`
+- `hdImageAssetId`
+- `hdImageStatus`:`pending | running | completed | failed`
+- `videoStatus`:`pending | waiting_confirmation | running | completed | failed`
+- `userApproved`:boolean
+- `userApprovedAt`
+
+## 6.3 资产类型建议
+
+新增或明确以下资产类型:
+
+- `storyboard_tile`:切片原图
+- `storyboard_hd_image`:高清镜头图
+- `storyboard_video`:最终视频结果
+
+要求:
+
+- 以上三类资产都必须可追溯到 `parentTaskId + shotIndex`
+- 都要能在画布节点和历史记录里反查
+
+---
+
+## 7. 画布展示规则
+
+## 7.1 画布先展示高清镜头图
+
+切片完成后,画布不应直接只显示视频占位卡,而应分两阶段:
+
+### 阶段 A:高清镜头图阶段
+
+- 每个镜头创建一个 `image` 节点占位
+- 高清图生成成功后替换占位图
+- 节点标题显示:`镜头 1 / 镜头 2 / 镜头 3 ...`
+
+### 阶段 B:视频阶段
+
+- 用户确认后,再在对应镜头位置创建或替换为 `video` 节点
+- 保留高清图与视频结果的关联关系
+
+## 7.2 顺序展示要求
+
+- 永远按切片顺序落位
+- 不允许哪个镜头先完成就乱序插入
+- 画布上应让用户一眼看出“镜头顺序”
+
+---
+
+## 8. 用户确认门机制(强制)
+
+## 8.1 硬性规则
+
+**没有用户确认,不允许开始生视频。**
+
+这条规则必须在 Agent、后端任务状态机、调度器三层同时生效。
+
+## 8.2 确认前流程
+
+1. Agent 识别宫格类型
+2. 切片
+3. 为每个切片生成高清镜头图
+4. 归一出镜头文案
+5. 将“镜头图 + 镜头文案”同步给用户确认
+6. 任务状态进入 `awaiting_user_confirmation`
+
+## 8.3 用户确认文案格式
+
+Agent 需要给用户清晰列出镜头草案,例如:
+
+```text
+我已经按 9 段分镜整理好了,并生成了对应高清镜头预览。
+
+镜头1:……
+镜头2:……
+镜头3:……
+
+如果需要修改,你可以直接回复:
+- 修改第3镜头:……
+- 修改第5镜头动作:……
+
+如果确认无误,请回复:
+“确认开始生成视频”
+```
+
+## 8.4 确认后的唯一放行口令
+
+系统只在识别到明确确认语义后放行,例如:
+
+- `确认开始生成视频`
+- `开始生成`
+- `按这个分镜生成`
+
+如果用户说的是修改意见,则只能更新分镜草案,不得偷偷开始视频任务。
+
+## 8.5 修改后的处理
+
+如果用户修改第 N 镜头:
+
+1. 只重算该镜头文案映射
+2. 必要时只重生该镜头高清图
+3. 更新画布对应静帧
+4. 重新回到 `awaiting_user_confirmation`
+
+---
+
+## 9. Agent 的新增职责
+
+Agent 在这条链路中的职责不再只是“识别意图 + 优化 prompt”,还必须承担:
+
+1. 判断应该使用 `4 / 6 / 9` 宫格中的哪一种
+2. 自动追加标准版式 prompt 约束
+3. 将用户自然语言脚本归一成与宫格数量一致的镜头数
+4. 在高清镜头图生成完成后,负责向用户解释和确认
+5. 在用户未确认前,阻止视频模型调用
+
+### 默认镜头数规则
+
+建议默认规则:
+
+- `4宫格 -> 4段分镜`
+- `6宫格 -> 6段分镜`
+- `9宫格 -> 9段分镜`
+
+如果业务强行要求“6宫格或4宫格也统一做成9段视频”,那不是当前默认流程,必须额外定义:
+
+- 缺失镜头如何补齐
+- 是否允许复用静帧
+- 是否允许 Agent 自动脑补新镜头
+
+当前阶段不建议自动补到 9 段后直接执行,必须先经用户明确确认。
+
+---
+
+## 10. 推荐状态机
+
+建议任务主状态改为:
+
+`draft -> slicing -> hd_image_generating -> awaiting_user_confirmation -> approved -> video_generating -> completed`
+
+失败态:
+
+- `slice_failed`
+- `hd_image_failed`
+- `confirmation_timeout`
+- `video_partial_failed`
+- `video_failed`
+
+---
+
+## 11. 验收标准(更新)
+
+新增验收标准:
+
+1. `4宫格 / 6宫格 / 9宫格` 都能稳定识别并按标准顺序切片
+2. 每个切片都能产出一张高清镜头图,并成功写库
+3. 高清镜头图会按顺序显示到画布
+4. 用户未确认前,系统不会触发任何视频模型调用
+5. 用户只修改局部镜头时,系统支持单镜头重算、单镜头重生
+
+---
+
+## 12. 建议开发顺序
+
+1. 先把 `gridType` 从“仅 9 宫格”扩成 `2x2 / 3x2 / 3x3`
+2. 再加 Agent 结构化版式 prompt 注入
+3. 再加 `tile-hd-redraw-skill`
+4. 打通“高清镜头图入库 + 画布展示”
+5. 最后再加“用户确认门 + 视频调度放行”
+
+这样能把风险最大的一步放在最后,同时让用户尽早看到“可确认的高清镜头预览”。
diff --git a/docs/系统全链路开发文档-含九宫格视频Agent技能化方案.md b/docs/系统全链路开发文档-含九宫格视频Agent技能化方案.md
index 8f6713333..293df5134 100644
--- a/docs/系统全链路开发文档-含九宫格视频Agent技能化方案.md
+++ b/docs/系统全链路开发文档-含九宫格视频Agent技能化方案.md
@@ -222,6 +222,10 @@ API:
## 6. 新需求:RUXA 视频九宫格 + Agent 技能化(待开发)
+> 2026-03-12 补充需求:多宫格(4/6/9)标准切片、切片后先生成高清镜头图、用户确认分镜后才允许开始生视频。
+> 需求详见:`docs/多宫格分镜视频需求补充-2026-03-12.md`
+> 开发清单详见:`docs/多宫格分镜视频开发清单-2026-03-12.md`
+
## 6.1 需求归纳
当用户在视频模式选中 `RUXA 视频` 时:
diff --git a/lib/agents/secondary-type-config.ts b/lib/agents/secondary-type-config.ts
new file mode 100644
index 000000000..17929990d
--- /dev/null
+++ b/lib/agents/secondary-type-config.ts
@@ -0,0 +1,47 @@
+import type { AgentPresetType } from "@/types/agent";
+
+export const AGENT_SECONDARY_TYPE_CONFIG_KEY = "agent_secondary_type_config";
+
+export type AgentSecondaryTypeConfig = Record;
+
+export const defaultAgentSecondaryTypeConfig: AgentSecondaryTypeConfig = {
+ image: [],
+ video: [],
+ text: [],
+};
+
+export function normalizeAgentSecondaryTypeConfig(value: unknown): AgentSecondaryTypeConfig {
+ const raw =
+ value && typeof value === "object" && !Array.isArray(value)
+ ? (value as Record)
+ : {};
+
+ const normalizeList = (input: unknown) => {
+ if (!Array.isArray(input)) return [] as string[];
+ return Array.from(
+ new Set(
+ input
+ .map((item) => String(item || "").trim())
+ .filter(Boolean)
+ .slice(0, 200)
+ )
+ );
+ };
+
+ return {
+ image: normalizeList(raw.image),
+ video: normalizeList(raw.video),
+ text: normalizeList(raw.text),
+ };
+}
+
+export function isValidSecondaryTypeForPresetType(params: {
+ presetType: AgentPresetType;
+ secondaryType?: string | null;
+ config: AgentSecondaryTypeConfig;
+}) {
+ const normalized = String(params.secondaryType || "").trim();
+ if (!normalized) return true;
+ const allowed = params.config[params.presetType] || [];
+ return allowed.includes(normalized);
+}
diff --git a/lib/ai/auto-agent-graph.ts b/lib/ai/auto-agent-graph.ts
index 4eb19bdfb..0ceddb024 100644
--- a/lib/ai/auto-agent-graph.ts
+++ b/lib/ai/auto-agent-graph.ts
@@ -1375,6 +1375,9 @@ async function buildExecutionPlanWithAgent(
skillRoute?.planningHints?.length
? `专家约束:\n${skillRoute.planningHints.map((item, index) => `${index + 1}. ${item}`).join("\n")}`
: "",
+ state.threadState?.workflowState
+ ? `当前工作流:type=${state.threadState.workflowState.type}; status=${state.threadState.workflowState.status}; summary=${compactText(state.threadState.workflowState.summary || "", 240)}`
+ : "",
state.context ? `画布上下文:${compactText(state.context, 600)}` : "",
historyContext ? `最近对话:\n${historyContext}` : "",
`可用工具:\n${toolCatalog}`,
@@ -1401,6 +1404,7 @@ async function buildExecutionPlanWithAgent(
targetResolutionMode: state.targetResolutionMode,
mediaSource: state.mediaSource,
selectedImageCount: state.selectedImageCount,
+ threadState: state.threadState,
});
}
}
@@ -1850,7 +1854,7 @@ const analyzeNode = async (state: AutoAgentStateType) => {
};
const confirmationGateNode = async (state: AutoAgentStateType) => {
- const workflowDecision = runHumanConfirmationGate({
+ const workflowDecision = await runHumanConfirmationGate({
text: state.text,
threadState: state.threadState,
});
@@ -1885,6 +1889,7 @@ const planNode = async (state: AutoAgentStateType) => {
targetResolutionMode: state.targetResolutionMode,
mediaSource: state.mediaSource,
selectedImageCount: state.selectedImageCount,
+ threadState: state.threadState,
});
const availableTools = filterToolsForSkillRoute(baseAvailableTools, skillRoute);
console.log("[Auto Agent] Skill route:", {
diff --git a/lib/ai/runtime/middlewares/human-confirmation-gate.ts b/lib/ai/runtime/middlewares/human-confirmation-gate.ts
index e42c77be1..7dac0c47f 100644
--- a/lib/ai/runtime/middlewares/human-confirmation-gate.ts
+++ b/lib/ai/runtime/middlewares/human-confirmation-gate.ts
@@ -5,6 +5,7 @@ import {
type AgentWorkflowType,
} from "@/lib/ai/runtime/agent-workflow-state";
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
+import { interpretStoryboardCommand } from "@/lib/video-agent/storyboard-command-interpreter";
export type HumanConfirmationIntent =
| "none"
@@ -31,63 +32,6 @@ export interface HumanConfirmationGateDecision {
friendlyMessage?: string;
}
-const CONFIRM_PATTERNS = [
- /确认开始生成视频/,
- /确认开始生成/,
- /按这个分镜生成/,
- /开始生成视频/,
- /开始生成$/,
- /可以开始了/,
- /可以执行了/,
- /就按这个/,
- /没问题开始/,
- /确认无误/,
-];
-
-const REVISE_PATTERNS = [
- /修改第\d+镜头/,
- /第\d+镜头/,
- /调整第\d+镜头/,
- /重生第\d+/,
- /重做第\d+/,
- /改一下/,
- /修改/,
- /调整/,
- /不要这样/,
- /不对/,
- /不行/,
- /换成/,
-];
-
-const CANCEL_PATTERNS = [
- /取消/,
- /先不要/,
- /暂停/,
- /停止/,
- /先别生成/,
- /不用生成了/,
-];
-
-function normalizeCompactText(text: string) {
- return String(text || "").toLowerCase().replace(/\s+/g, "").trim();
-}
-
-export function detectHumanConfirmationIntent(text: string): HumanConfirmationIntent {
- const normalized = normalizeCompactText(text);
- if (!normalized) return "unclear";
-
- if (CANCEL_PATTERNS.some((pattern) => pattern.test(normalized))) {
- return "cancel";
- }
- if (REVISE_PATTERNS.some((pattern) => pattern.test(normalized))) {
- return "revise";
- }
- if (CONFIRM_PATTERNS.some((pattern) => pattern.test(normalized))) {
- return "confirm";
- }
- return "unclear";
-}
-
function buildAwaitingConfirmationMessage(workflowState: AgentWorkflowState): string {
const title = workflowState.title || "当前工作流";
const summary = workflowState.summary
@@ -131,10 +75,10 @@ function buildCancelMessage(workflowState: AgentWorkflowState): string {
return `已收到取消指令。${title} 会继续停留在人工确认前状态,不会继续自动执行。`;
}
-export function runHumanConfirmationGate(params: {
+export async function runHumanConfirmationGate(params: {
text: string;
threadState?: AgentThreadState | null;
-}): HumanConfirmationGateDecision {
+}): Promise {
const workflowState = params.threadState?.workflowState;
if (!workflowState || !isAwaitingUserConfirmationWorkflow(workflowState)) {
return {
@@ -145,7 +89,16 @@ export function runHumanConfirmationGate(params: {
};
}
- const confirmationIntent = detectHumanConfirmationIntent(params.text);
+ const interpreted = await interpretStoryboardCommand({
+ text: params.text,
+ totalShots: workflowState.pendingItems.length || undefined,
+ items: workflowState.pendingItems.map((item) => ({
+ index: item.index || 0,
+ title: item.title,
+ summary: item.summary,
+ })),
+ });
+ const confirmationIntent = interpreted.intent;
const baseDecision = {
workflowId: workflowState.workflowId,
workflowType: workflowState.type,
@@ -181,7 +134,6 @@ export function runHumanConfirmationGate(params: {
friendlyMessage: buildCancelMessage(workflowState),
};
case "unclear":
- case "none":
default:
return {
...baseDecision,
diff --git a/lib/ai/skill-router.ts b/lib/ai/skill-router.ts
index d9f49e0ed..2bac2f326 100644
--- a/lib/ai/skill-router.ts
+++ b/lib/ai/skill-router.ts
@@ -6,6 +6,7 @@ import type {
AgentSpecialistContext,
AgentSpecialistRoute,
} from "@/lib/ai/specialists";
+import { buildStoryboardSpecialistRoute } from "@/lib/video-agent/storyboard-subagent";
export interface AgentSkillRouteResult {
primary: AgentSpecialistRoute;
@@ -210,23 +211,31 @@ export function routeAgentSkill(
? [canvasOverlay]
: [];
+ const storyboardOverlay = buildStoryboardSpecialistRoute({
+ text: context.text,
+ threadState: context.threadState,
+ });
+ const mergedOverlays = storyboardOverlay
+ ? [...overlays, storyboardOverlay]
+ : overlays;
+
const toolAllowlist = dedupeToolNames([
...(primary.toolAllowlist || []),
- ...overlays.flatMap((item) => item.toolAllowlist || []),
+ ...mergedOverlays.flatMap((item) => item.toolAllowlist || []),
]);
const planningHints = [
...primary.planningHints,
- ...overlays.flatMap((item) => item.planningHints),
+ ...mergedOverlays.flatMap((item) => item.planningHints),
];
- const reason = [primary.reason, ...overlays.map((item) => item.reason)]
+ const reason = [primary.reason, ...mergedOverlays.map((item) => item.reason)]
.filter(Boolean)
.join(";");
return {
primary,
- overlays,
+ overlays: mergedOverlays,
toolAllowlist,
planningHints,
reason,
diff --git a/lib/ai/specialists/index.ts b/lib/ai/specialists/index.ts
index 6979450c0..73c839074 100644
--- a/lib/ai/specialists/index.ts
+++ b/lib/ai/specialists/index.ts
@@ -1,4 +1,6 @@
import type { AnalysisResult } from "@/lib/ai/smart-analyzer";
+import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
+import { refinePlanWithStoryboardSubagent } from "@/lib/video-agent/storyboard-subagent";
import type {
AgentExecutionPlan,
AgentPlanNextAction,
@@ -18,6 +20,7 @@ export type AgentSpecialistDomain =
| "image_edit"
| "image_generate"
| "video_generate"
+ | "storyboard"
| "canvas";
export type AgentSpecialistId =
@@ -26,6 +29,7 @@ export type AgentSpecialistId =
| "image-edit-specialist"
| "image-generate-specialist"
| "video-specialist"
+ | "storyboard-subagent"
| "canvas-specialist";
export interface AgentSpecialistRoute {
@@ -44,6 +48,7 @@ export interface AgentSpecialistContext {
targetResolutionMode: AgentTargetResolutionMode;
mediaSource?: "current_message" | "canvas_selection" | "history" | "none";
selectedImageCount?: number;
+ threadState?: AgentThreadState | null;
}
export interface AgentPlanRefineInput extends AgentSpecialistContext {
@@ -260,6 +265,15 @@ export function refinePlanWithSpecialist(
});
}
+ if (specialist.domain === "storyboard") {
+ return refinePlanWithStoryboardSubagent({
+ text,
+ plan,
+ availableTools,
+ threadState: input.threadState,
+ });
+ }
+
if (specialist.domain === "canvas") {
const isExecutionAction =
plan.nextAction === "edit_image" ||
diff --git a/lib/db/migrations/0057_add_video_storyboard_confirmation_state.sql b/lib/db/migrations/0057_add_video_storyboard_confirmation_state.sql
new file mode 100644
index 000000000..856cbdf6a
--- /dev/null
+++ b/lib/db/migrations/0057_add_video_storyboard_confirmation_state.sql
@@ -0,0 +1,77 @@
+ALTER TABLE "VideoStoryboardTask"
+ ADD COLUMN IF NOT EXISTS "gridType" varchar(32);
+--> statement-breakpoint
+ALTER TABLE "VideoStoryboardTask"
+ ADD COLUMN IF NOT EXISTS "gridRows" integer;
+--> statement-breakpoint
+ALTER TABLE "VideoStoryboardTask"
+ ADD COLUMN IF NOT EXISTS "gridCols" integer;
+--> statement-breakpoint
+ALTER TABLE "VideoStoryboardTask"
+ ADD COLUMN IF NOT EXISTS "confirmationStatus" varchar(32) DEFAULT 'draft' NOT NULL;
+--> statement-breakpoint
+ALTER TABLE "VideoStoryboardTask"
+ ADD COLUMN IF NOT EXISTS "hdImageStageStatus" varchar(32) DEFAULT 'pending' NOT NULL;
+--> statement-breakpoint
+ALTER TABLE "VideoStoryboardTask"
+ ADD COLUMN IF NOT EXISTS "confirmedAt" timestamp;
+--> statement-breakpoint
+ALTER TABLE "VideoStoryboardTask"
+ ADD COLUMN IF NOT EXISTS "confirmedByMessageId" uuid;
+--> statement-breakpoint
+DO $$ BEGIN
+ ALTER TABLE "VideoStoryboardTask" ADD CONSTRAINT "VideoStoryboardTask_confirmedByMessageId_Message_v2_id_fk" FOREIGN KEY ("confirmedByMessageId") REFERENCES "public"."Message_v2"("id") ON DELETE set null ON UPDATE no action;
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+--> statement-breakpoint
+ALTER TABLE "VideoStoryboardShotTask"
+ ADD COLUMN IF NOT EXISTS "hdImageUrl" text;
+--> statement-breakpoint
+ALTER TABLE "VideoStoryboardShotTask"
+ ADD COLUMN IF NOT EXISTS "hdImagePrompt" text;
+--> statement-breakpoint
+ALTER TABLE "VideoStoryboardShotTask"
+ ADD COLUMN IF NOT EXISTS "hdImageStatus" varchar(32) DEFAULT 'pending' NOT NULL;
+--> statement-breakpoint
+ALTER TABLE "VideoStoryboardShotTask"
+ ADD COLUMN IF NOT EXISTS "videoStatus" varchar(32) DEFAULT 'pending' NOT NULL;
+--> statement-breakpoint
+ALTER TABLE "VideoStoryboardShotTask"
+ ADD COLUMN IF NOT EXISTS "userApproved" boolean DEFAULT false NOT NULL;
+--> statement-breakpoint
+ALTER TABLE "VideoStoryboardShotTask"
+ ADD COLUMN IF NOT EXISTS "userApprovedAt" timestamp;
+--> statement-breakpoint
+UPDATE "VideoStoryboardTask"
+SET "confirmationStatus" = CASE
+ WHEN "status" IN ('succeeded', 'partial_success', 'failed') THEN 'confirmed'
+ WHEN "status" = 'cancelled' THEN 'rejected'
+ ELSE "confirmationStatus"
+END
+WHERE "confirmationStatus" = 'draft';
+--> statement-breakpoint
+UPDATE "VideoStoryboardShotTask"
+SET
+ "hdImageStatus" = CASE
+ WHEN "status" = 'completed' THEN 'completed'
+ WHEN "status" = 'failed' THEN 'failed'
+ WHEN "status" = 'running' THEN 'running'
+ WHEN "status" = 'cancelled' THEN 'cancelled'
+ ELSE "hdImageStatus"
+ END,
+ "videoStatus" = CASE
+ WHEN "status" = 'completed' THEN 'completed'
+ WHEN "status" = 'failed' THEN 'failed'
+ WHEN "status" = 'running' THEN 'running'
+ WHEN "status" = 'cancelled' THEN 'cancelled'
+ ELSE "videoStatus"
+ END,
+ "userApproved" = CASE
+ WHEN "status" IN ('completed', 'failed', 'running') THEN true
+ ELSE "userApproved"
+ END,
+ "userApprovedAt" = CASE
+ WHEN "status" IN ('completed', 'failed', 'running') AND "userApprovedAt" IS NULL THEN NOW()
+ ELSE "userApprovedAt"
+ END;
diff --git a/lib/db/migrations/0058_add_agent_secondary_type.sql b/lib/db/migrations/0058_add_agent_secondary_type.sql
new file mode 100644
index 000000000..9cd2c439b
--- /dev/null
+++ b/lib/db/migrations/0058_add_agent_secondary_type.sql
@@ -0,0 +1,5 @@
+ALTER TABLE "AgentPreset"
+ ADD COLUMN IF NOT EXISTS "secondaryType" varchar(128);
+--> statement-breakpoint
+ALTER TABLE "AgentPresetExecutionSnapshot"
+ ADD COLUMN IF NOT EXISTS "secondaryType" varchar(128);
diff --git a/lib/db/migrations/meta/_journal.json b/lib/db/migrations/meta/_journal.json
index f2a86d6ad..52dbcd17e 100644
--- a/lib/db/migrations/meta/_journal.json
+++ b/lib/db/migrations/meta/_journal.json
@@ -302,6 +302,20 @@
"when": 1773151800000,
"tag": "0056_add_agent_category_path",
"breakpoints": true
+ },
+ {
+ "idx": 43,
+ "version": "7",
+ "when": 1773303600000,
+ "tag": "0057_add_video_storyboard_confirmation_state",
+ "breakpoints": true
+ },
+ {
+ "idx": 44,
+ "version": "7",
+ "when": 1773390000000,
+ "tag": "0058_add_agent_secondary_type",
+ "breakpoints": true
}
]
}
diff --git a/lib/db/queries.ts b/lib/db/queries.ts
index ce84933dd..4cbe2267a 100644
--- a/lib/db/queries.ts
+++ b/lib/db/queries.ts
@@ -79,6 +79,7 @@ import {
import { generateHashedPassword } from "./utils";
import {
isAiModelAgentAuto,
+ isAiModelMultimodal,
withAgentAutoModelFlag,
withAgentCodingModelId,
withMultimodalModelFlag,
@@ -1749,7 +1750,7 @@ export async function getPointsSystemConfig() {
}
type AgentPresetVisibility = "public" | "private";
-type AgentPresetKind = "image" | "video";
+type AgentPresetKind = "image" | "video" | "text";
type AgentPresetScope = "all" | AgentPresetVisibility;
type AgentCategoryScope = "all" | "public" | "private" | "global";
type AgentPresetSort = "latest" | "hottest" | "most_used";
@@ -1786,6 +1787,8 @@ async function validateAgentDefaultModel(params: {
modelId: aiModel.modelId,
displayName: aiModel.displayName,
supportsImageGen: aiModel.supportsImageGen,
+ capabilities: aiModel.capabilities,
+ defaultParams: aiModel.defaultParams,
isEnabled: aiModel.isEnabled,
providerType: aiProvider.type,
providerEnabled: aiProvider.isEnabled,
@@ -1812,6 +1815,20 @@ async function validateAgentDefaultModel(params: {
return model;
}
+ if (params.presetType === "text") {
+ const isMultimodal = isAiModelMultimodal({
+ capabilities: model.capabilities,
+ defaultParams: model.defaultParams,
+ });
+ if (isVideoModel || !isMultimodal) {
+ throw new ChatSDKError(
+ "bad_request:database",
+ "生文智能体只能选择带多模态标签的模型"
+ );
+ }
+ return model;
+ }
+
if (isVideoModel || !model.supportsImageGen) {
throw new ChatSDKError(
"bad_request:database",
@@ -2472,6 +2489,7 @@ export async function createAgentPreset(params: {
userId: string;
name: string;
type: AgentPresetKind;
+ secondaryType?: string | null;
visibility: AgentPresetVisibility;
categoryPath?: {
productCategory?: string | null;
@@ -2515,6 +2533,7 @@ export async function createAgentPreset(params: {
ownerUserId: params.userId,
name: params.name.trim(),
type: params.type,
+ secondaryType: params.secondaryType || null,
visibility: params.visibility,
categoryId: null,
categoryPath: params.categoryPath || null,
@@ -2583,6 +2602,7 @@ export async function updateAgentPreset(params: {
userId: string;
name?: string;
type?: AgentPresetKind;
+ secondaryType?: string | null;
visibility?: AgentPresetVisibility;
categoryPath?: {
productCategory?: string | null;
@@ -2704,6 +2724,9 @@ export async function updateAgentPreset(params: {
if (params.name !== undefined) updateData.name = params.name.trim();
if (params.type !== undefined) updateData.type = params.type;
+ if (params.secondaryType !== undefined) {
+ updateData.secondaryType = params.secondaryType || null;
+ }
if (params.visibility !== undefined) updateData.visibility = params.visibility;
if (params.categoryPath !== undefined) updateData.categoryPath = params.categoryPath || null;
if (params.description !== undefined) updateData.description = params.description || null;
@@ -2844,6 +2867,7 @@ export async function forkAgentPreset(params: {
ownerUserId: params.userId,
name: String(params.name || "").trim() || `${source.name}(副本)`,
type: source.type,
+ secondaryType: source.secondaryType,
visibility: "private",
categoryId: source.categoryId,
categoryPath: source.categoryPath,
@@ -2915,6 +2939,7 @@ export async function createAgentPresetExecutionSnapshot(params: {
messageId?: string | null;
presetName: string;
presetType: AgentPresetKind;
+ secondaryType?: string | null;
visibility: AgentPresetVisibility;
categoryId?: string | null;
categoryName?: string | null;
@@ -2950,6 +2975,7 @@ export async function createAgentPresetExecutionSnapshot(params: {
userId: params.userId,
presetName: params.presetName,
presetType: params.presetType,
+ secondaryType: params.secondaryType || null,
visibility: params.visibility,
categoryId: params.categoryId || null,
categoryName: params.categoryName || null,
diff --git a/lib/db/schema.ts b/lib/db/schema.ts
index 4162174e5..c4583eb85 100644
--- a/lib/db/schema.ts
+++ b/lib/db/schema.ts
@@ -521,9 +521,10 @@ export const agentPreset = pgTable("AgentPreset", {
.notNull()
.references(() => user.id, { onDelete: "cascade" }),
name: varchar("name", { length: 128 }).notNull(),
- type: varchar("type", { enum: ["image", "video"] })
+ type: varchar("type", { enum: ["image", "video", "text"] })
.notNull()
.default("image"),
+ secondaryType: varchar("secondaryType", { length: 128 }),
visibility: varchar("visibility", { enum: ["public", "private"] })
.notNull()
.default("private"),
@@ -708,17 +709,30 @@ export const videoStoryboardTask = pgTable("VideoStoryboardTask", {
.notNull()
.default("ruxa-video"),
modelRef: varchar("modelRef", { length: 128 }),
+ gridType: varchar("gridType", { length: 32 }),
+ gridRows: integer("gridRows"),
+ gridCols: integer("gridCols"),
sourceImageUrl: text("sourceImageUrl").notNull(),
isNineGrid: boolean("isNineGrid").notNull().default(false),
detectionConfidence: integer("detectionConfidence").notNull().default(0),
normalizedScript: jsonb("normalizedScript"),
status: varchar("status", { length: 32 }).notNull().default("pending"),
+ confirmationStatus: varchar("confirmationStatus", { length: 32 })
+ .notNull()
+ .default("draft"),
+ hdImageStageStatus: varchar("hdImageStageStatus", { length: 32 })
+ .notNull()
+ .default("pending"),
currentShot: integer("currentShot").notNull().default(0),
totalShots: integer("totalShots").notNull().default(9),
error: text("error"),
createdAt: timestamp("createdAt").notNull().defaultNow(),
updatedAt: timestamp("updatedAt").notNull().defaultNow(),
startedAt: timestamp("startedAt"),
+ confirmedAt: timestamp("confirmedAt"),
+ confirmedByMessageId: uuid("confirmedByMessageId").references(() => message.id, {
+ onDelete: "set null",
+ }),
completedAt: timestamp("completedAt"),
});
@@ -734,11 +748,17 @@ export const videoStoryboardShotTask = pgTable("VideoStoryboardShotTask", {
tileIndex: integer("tileIndex"),
tileImageUrl: text("tileImageUrl"),
upscaledImageUrl: text("upscaledImageUrl"),
+ hdImageUrl: text("hdImageUrl"),
+ hdImagePrompt: text("hdImagePrompt"),
prompt: text("prompt"),
modelId: varchar("modelId", { length: 128 }),
fallbackChain: jsonb("fallbackChain"),
attemptCount: integer("attemptCount").notNull().default(0),
status: varchar("status", { length: 32 }).notNull().default("pending"),
+ hdImageStatus: varchar("hdImageStatus", { length: 32 }).notNull().default("pending"),
+ videoStatus: varchar("videoStatus", { length: 32 }).notNull().default("pending"),
+ userApproved: boolean("userApproved").notNull().default(false),
+ userApprovedAt: timestamp("userApprovedAt"),
videoUrl: text("videoUrl"),
errorCode: varchar("errorCode", { length: 64 }),
errorMessage: text("errorMessage"),
@@ -960,7 +980,8 @@ export const agentPresetExecutionSnapshot = pgTable("AgentPresetExecutionSnapsho
.notNull()
.references(() => user.id, { onDelete: "cascade" }),
presetName: varchar("presetName", { length: 128 }).notNull(),
- presetType: varchar("presetType", { enum: ["image", "video"] }).notNull(),
+ presetType: varchar("presetType", { enum: ["image", "video", "text"] }).notNull(),
+ secondaryType: varchar("secondaryType", { length: 128 }),
visibility: varchar("visibility", { enum: ["public", "private"] }).notNull(),
categoryId: uuid("categoryId").references(() => agentPresetCategory.id, {
onDelete: "set null",
diff --git a/lib/video-agent/agent-thread-state.ts b/lib/video-agent/agent-thread-state.ts
new file mode 100644
index 000000000..e475bb6eb
--- /dev/null
+++ b/lib/video-agent/agent-thread-state.ts
@@ -0,0 +1,152 @@
+import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
+import type { AgentWorkflowStatus } from "@/lib/ai/runtime/agent-workflow-state";
+import type { StoryboardPreferenceMemory } from "@/lib/video-agent/storyboard-preference-memory";
+
+type StoryboardTaskDetail = {
+ task: {
+ id: string;
+ status: string | null;
+ confirmationStatus?: string | null;
+ hdImageStageStatus?: string | null;
+ gridType?: string | null;
+ gridRows?: number | null;
+ gridCols?: number | null;
+ currentShot: number | null;
+ totalShots: number | null;
+ sourceImageUrl?: string | null;
+ providerType?: string | null;
+ normalizedScript?: unknown;
+ };
+ shots: Array<{
+ shotIndex: number | null;
+ prompt?: string | null;
+ status?: string | null;
+ tileImageUrl?: string | null;
+ videoUrl?: string | null;
+ }>;
+};
+
+function normalizeText(value: unknown, max = 240): string | undefined {
+ const normalized = String(value || "").trim();
+ if (!normalized) return undefined;
+ return normalized.slice(0, max);
+}
+
+function mapStoryboardTaskStatusToWorkflowStatus(
+ task: StoryboardTaskDetail["task"],
+): AgentWorkflowStatus {
+ const rawConfirmationStatus = (task as Record).confirmationStatus;
+ const confirmationStatus = normalizeText(rawConfirmationStatus, 40);
+ if (confirmationStatus === "awaiting_user_confirmation") {
+ return "awaiting_user_confirmation";
+ }
+ if (confirmationStatus === "confirmed") {
+ return "confirmed";
+ }
+ if (confirmationStatus === "rejected") {
+ return "rejected";
+ }
+
+ const status = normalizeText(task.status, 40);
+ switch (status) {
+ case "pending":
+ return "draft";
+ case "running":
+ return "running";
+ case "succeeded":
+ case "partial_success":
+ return "completed";
+ case "failed":
+ return "failed";
+ case "cancelled":
+ return "cancelled";
+ default:
+ return "draft";
+ }
+}
+
+export function buildAgentThreadStateFromStoryboardTask(
+ detail: StoryboardTaskDetail | null | undefined,
+ options?: {
+ storyboardPreferenceMemory?: StoryboardPreferenceMemory | null;
+ },
+): AgentThreadState | null {
+ if (!detail?.task?.id) {
+ if (!options?.storyboardPreferenceMemory) return null;
+ return {
+ workflowState: null,
+ memoryHints: [
+ "当前会话里存在 storyboard 偏好记忆,可在分镜/宫格任务里优先参考用户常用版式和确认习惯。",
+ ],
+ metadata: {
+ workflowSource: "storyboard_preference_memory",
+ storyboardPreferenceMemory: options.storyboardPreferenceMemory,
+ },
+ };
+ }
+
+ const totalShots = Math.max(0, Number(detail.task.totalShots || detail.shots.length || 0));
+ const currentShot = Math.max(0, Number(detail.task.currentShot || 0));
+ const gridRows = Math.max(0, Number(detail.task.gridRows || 0));
+ const gridCols = Math.max(0, Number(detail.task.gridCols || 0));
+ const gridLabel =
+ gridRows > 0 && gridCols > 0 ? `${gridCols}x${gridRows}` : `${totalShots} 段`;
+
+ return {
+ workflowState: {
+ workflowId: detail.task.id,
+ type: "storyboard_video",
+ status: mapStoryboardTaskStatusToWorkflowStatus(detail.task),
+ title: `分镜视频任务(${gridLabel})`,
+ summary:
+ totalShots > 0
+ ? `当前分镜任务进度:第 ${Math.min(currentShot, totalShots)}/${totalShots} 段。`
+ : "当前存在一个分镜视频任务。",
+ confirmPhrases: ["确认开始生成视频", "开始生成", "按这个分镜生成"],
+ revisionHints: ["修改第3镜头", "调整第5镜头动作", "重生第2张高清图"],
+ pendingItems: detail.shots.slice(0, 9).map((shot) => ({
+ id:
+ Number.isFinite(Number(shot.shotIndex)) && Number(shot.shotIndex) > 0
+ ? `shot-${Math.round(Number(shot.shotIndex))}`
+ : undefined,
+ index:
+ Number.isFinite(Number(shot.shotIndex)) && Number(shot.shotIndex) > 0
+ ? Math.round(Number(shot.shotIndex))
+ : undefined,
+ title:
+ Number.isFinite(Number(shot.shotIndex)) && Number(shot.shotIndex) > 0
+ ? `镜头 ${Math.round(Number(shot.shotIndex))}`
+ : "镜头",
+ summary: normalizeText(shot.prompt, 120),
+ status: normalizeText(shot.status, 32),
+ })),
+ metadata: {
+ providerType: normalizeText(detail.task.providerType, 40),
+ gridType: normalizeText(detail.task.gridType, 32),
+ gridRows: gridRows || undefined,
+ gridCols: gridCols || undefined,
+ sourceImageUrl: normalizeText(detail.task.sourceImageUrl, 500),
+ normalizedScript:
+ detail.task.normalizedScript &&
+ typeof detail.task.normalizedScript === "object" &&
+ !Array.isArray(detail.task.normalizedScript)
+ ? detail.task.normalizedScript
+ : undefined,
+ },
+ },
+ memoryHints: [
+ "如果当前对话是在继续一个分镜视频任务,优先理解为对当前分镜流程的确认、修改或取消。",
+ "如果用户提到第N镜头、重生某张图、调整某个镜头,优先解释为对当前 storyboard workflow 的修订。",
+ ],
+ contextSummary:
+ totalShots > 0
+ ? `当前 chat 存在一个分镜视频任务,共 ${totalShots} 段,当前进度第 ${Math.min(currentShot, totalShots)}/${totalShots} 段。`
+ : "当前 chat 存在一个分镜视频任务。",
+ metadata: {
+ workflowSource: "video_storyboard_task",
+ taskId: detail.task.id,
+ shotCount: totalShots,
+ storyboardPreferenceMemory: options?.storyboardPreferenceMemory || undefined,
+ },
+ };
+}
diff --git a/lib/video-agent/execute-storyboard-task.ts b/lib/video-agent/execute-storyboard-task.ts
new file mode 100644
index 000000000..2c5f88251
--- /dev/null
+++ b/lib/video-agent/execute-storyboard-task.ts
@@ -0,0 +1,304 @@
+import "server-only";
+
+import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
+import { getAiModelById } from "@/lib/db/queries";
+import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
+import {
+ finalizeVideoStoryboardTask,
+ getVideoStoryboardTaskDetail,
+ markVideoStoryboardShotCompleted,
+ markVideoStoryboardShotFailed,
+ markVideoStoryboardShotRetry,
+ markVideoStoryboardShotRunning,
+ syncVideoStoryboardTaskStatus,
+} from "@/lib/video-agent/task-store";
+import type { StoryboardProgressEvent } from "@/lib/video-agent/types";
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+function normalizeText(value: unknown) {
+ return String(value || "").trim();
+}
+
+function extractVideoUrlFromText(text: string): string | null {
+ const normalizedText = String(text || "");
+ if (!normalizedText) return null;
+
+ const htmlMatch = normalizedText.match(/