lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update agent workflow and video task handling
Some checks failed
Lint / build (20) (push) Has been cancelled
Details
Playwright Tests / test (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Your Name 2026-03-13 11:21:42 +08:00
parent 37db7e2d3a
commit ec988be30a
44 changed files with 5260 additions and 966 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
app/(admin)/settings/admin-settings-client.tsx
View File

@ -236,6 +236,11 @@ type SiteSettings = {
usageScenarios: string[];
brands: string[];
};
agentSecondaryTypeConfig: {
image: string[];
video: string[];
text: string[];
};
realtimeCanvasEnabled: boolean;
aiDetailGenerationEnabled: boolean;
canvasDefaultModelId: string | null;
@ -632,6 +637,11 @@ export function AdminSettingsClient() {
usageScenarios: [],
brands: [],
},
agentSecondaryTypeConfig: {
image: [],
video: [],
text: [],
},
realtimeCanvasEnabled: true,
aiDetailGenerationEnabled: false,
canvasDefaultModelId: null,
@ -718,6 +728,17 @@ export function AdminSettingsClient() {
? (siteSettingsData as any).agentCategoryTreeConfig.brands
: [],
},
agentSecondaryTypeConfig: {
image: Array.isArray((siteSettingsData as any).agentSecondaryTypeConfig?.image)
? (siteSettingsData as any).agentSecondaryTypeConfig.image
: [],
video: Array.isArray((siteSettingsData as any).agentSecondaryTypeConfig?.video)
? (siteSettingsData as any).agentSecondaryTypeConfig.video
: [],
text: Array.isArray((siteSettingsData as any).agentSecondaryTypeConfig?.text)
? (siteSettingsData as any).agentSecondaryTypeConfig.text
: [],
},
realtimeCanvasEnabled: (siteSettingsData as any).realtimeCanvasEnabled ?? true,
aiDetailGenerationEnabled: siteSettingsData.aiDetailGenerationEnabled ?? false,
canvasDefaultModelId: siteSettingsData.canvasDefaultModelId || null,
@ -5431,6 +5452,82 @@ export function AdminSettingsClient() {
</div>
)}
{siteSettings.agentEnabled && (
<div className="rounded-lg border p-4 space-y-4">
<div>
<div className="font-medium">智能体二级类型配置</div>
<div className="text-xs text-muted-foreground mt-1">
用户创建智能体时，“二级类型”下拉从这里读取。按主类型分别维护：生图 / 视频 / 生文。
</div>
</div>
<div className="grid grid-cols-1 gap-4 md:grid-cols-3">
<div className="space-y-2">
<Label>生图二级类型</Label>
<Textarea
value={siteSettings.agentSecondaryTypeConfig.image.join("\n")}
onChange={(event) =>
setSiteSettings({
...siteSettings,
agentSecondaryTypeConfig: {
...siteSettings.agentSecondaryTypeConfig,
image: event.target.value
.split("\n")
.map((item) => item.trim())
.filter(Boolean),
},
})
}
rows={6}
placeholder={"例如：\n产品图优化\n场景图生成\n海报图生成"}
/>
</div>
<div className="space-y-2">
<Label>视频二级类型</Label>
<Textarea
value={siteSettings.agentSecondaryTypeConfig.video.join("\n")}
onChange={(event) =>
setSiteSettings({
...siteSettings,
agentSecondaryTypeConfig: {
...siteSettings.agentSecondaryTypeConfig,
video: event.target.value
.split("\n")
.map((item) => item.trim())
.filter(Boolean),
},
})
}
rows={6}
placeholder={"例如：\n口播视频\n商品演示\n分镜脚本"}
/>
</div>
<div className="space-y-2">
<Label>生文二级类型</Label>
<Textarea
value={siteSettings.agentSecondaryTypeConfig.text.join("\n")}
onChange={(event) =>
setSiteSettings({
...siteSettings,
agentSecondaryTypeConfig: {
...siteSettings.agentSecondaryTypeConfig,
text: event.target.value
.split("\n")
.map((item) => item.trim())
.filter(Boolean),
},
})
}
rows={6}
placeholder={"例如：\n文案分析\n提示词拆解\n图片分析报告"}
/>
</div>
</div>
</div>
)}
<div className="flex items-center justify-between rounded-lg border p-3">
<div className="space-y-0.5">
<Label htmlFor="realtimeCanvasEnabled">实时画布功能</Label>

674
app/(chat)/api/chat/route.ts
View File

@ -52,6 +52,7 @@ import {
executeDirectVideoGeneration,
generateVideo,
} from "@/lib/ai/tools/generate-video";
import { executeConfirmedStoryboardTask } from "@/lib/video-agent/execute-storyboard-task";
import { getEnabledTools } from "@/lib/ai/tools-registry";
import { isProductionEnvironment } from "@/lib/constants";
import {
@ -100,7 +101,24 @@ import { extractImageUrlsFromText } from "@/lib/image-url";
import { uploadFile } from "@/lib/storage/service";
import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline";
import { attachVideoStoryboardTaskMessage } from "@/lib/video-agent/task-store";
import {
reviseStoryboardTaskByCommand,
resolveStoryboardRevisionCommand,
} from "@/lib/video-agent/revise-storyboard-task";
import {
buildStoryboardPreferenceMemoryUpdate,
extractStoryboardPreferenceMemory,
} from "@/lib/video-agent/storyboard-preference-memory";
import {
attachVideoStoryboardTaskMessage,
getVideoStoryboardTaskDetail,
getLatestActiveVideoStoryboardTaskByChatId,
markVideoStoryboardAwaitingConfirmation,
markVideoStoryboardConfirmed,
markVideoStoryboardDraftForRevision,
cancelVideoStoryboardTaskByUser,
} from "@/lib/video-agent/task-store";
import { buildAgentThreadStateFromStoryboardTask } from "@/lib/video-agent/agent-thread-state";
import type { StoryboardModelCandidate } from "@/lib/video-agent/types";
import type { ChatMessage } from "@/lib/types";
import type { AppUsage } from "@/lib/usage";
@ -1753,21 +1771,118 @@ function isConfirmedPromptGenerationRequest(text: string): boolean {
function extractConfirmedPromptFromWorkingState(
workingState?: {
lastTurnSummary?: Record<string, unknown> | null;
lastTurnSummary?: unknown;
} | null,
): string | undefined {
const summary = workingState?.lastTurnSummary;
if (!summary || typeof summary !== "object") return undefined;
const summaryRecord = summary as Record<string, unknown>;
const resolutionMode =
typeof summary.resolutionMode === "string" ? summary.resolutionMode.trim() : "";
typeof summaryRecord.resolutionMode === "string" ? summaryRecord.resolutionMode.trim() : "";
const optimizedPrompt =
typeof summary.optimizedPrompt === "string" ? summary.optimizedPrompt.trim() : "";
typeof summaryRecord.optimizedPrompt === "string" ? summaryRecord.optimizedPrompt.trim() : "";
if (resolutionMode !== "prompt_only" || !optimizedPrompt) return undefined;
return optimizedPrompt;
}
type PendingGridConfirmation = {
gridType: "grid_2x2" | "grid_3x2" | "grid_3x3";
shotCount: 4 | 6 | 9;
label: string;
originalPrompt: string;
};
function detectRequestedStoryboardGrid(text: string): PendingGridConfirmation | null {
const normalized = String(text || "").toLowerCase().replace(/\s+/g, "");
if (!normalized) return null;
if (/4宫格|四宫格|2x2|2×2/.test(normalized)) {
return {
gridType: "grid_2x2",
shotCount: 4,
label: "4宫格",
originalPrompt: String(text || "").trim(),
};
}
if (/6宫格|六宫格|3x2|3×2|2x3|2×3/.test(normalized)) {
return {
gridType: "grid_3x2",
shotCount: 6,
label: "6宫格",
originalPrompt: String(text || "").trim(),
};
}
if (/9宫格|九宫格|3x3|3×3/.test(normalized)) {
return {
gridType: "grid_3x3",
shotCount: 9,
label: "9宫格",
originalPrompt: String(text || "").trim(),
};
}
return null;
}
function isGridConfirmationReply(text: string): boolean {
const normalized = String(text || "").toLowerCase().replace(/\s+/g, "");
if (!normalized) return false;
return /确认|就按这个|开始生成|可以开始了|没问题|对的|就这样/.test(normalized);
}
function buildGridConfirmationMessage(grid: PendingGridConfirmation): string {
return [
`我看到你希望按 ${grid.label} 来生成分镜图。`,
"为了确保后续自动切片稳定，我会按标准网格、统一格线、固定顺序来出图。",
`请确认是否按 ${grid.label} 标准版式开始生成。`,
"",
"如果确认，请回复：",
`- 确认按${grid.label}生成`,
"",
"如果要修改，也可以直接回复：",
"- 改成 9 宫格",
"- 改成 6 宫格",
"- 改成 4 宫格",
].join("\n");
}
function extractPendingGridConfirmationFromWorkingState(
workingState?: {
lastTurnSummary?: unknown;
} | null,
): PendingGridConfirmation | null {
const summary = workingState?.lastTurnSummary;
if (!summary || typeof summary !== "object") return null;
const summaryRecord = summary as Record<string, unknown>;
const pending = summaryRecord.pendingGridConfirmation;
if (!pending || typeof pending !== "object") return null;
const record = pending as Record<string, unknown>;
const gridType = String(record.gridType || "").trim();
const shotCount = Number(record.shotCount || 0);
const label = String(record.label || "").trim();
const originalPrompt = String(record.originalPrompt || "").trim();
if (
!originalPrompt ||
!label ||
!["grid_2x2", "grid_3x2", "grid_3x3"].includes(gridType) ||
![4, 6, 9].includes(shotCount)
) {
return null;
}
return {
gridType: gridType as PendingGridConfirmation["gridType"],
shotCount: shotCount as PendingGridConfirmation["shotCount"],
label,
originalPrompt,
};
}
function hasStructuredAgentSections(message?: string): boolean {
if (!message) return false;
return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test(
@ -2943,8 +3058,13 @@ export async function POST(request: Request) {
let requestWorkingState:
| Awaited<ReturnType<typeof getConversationWorkingStateByChatId>>
| null = null;
let storyboardPreferenceMemory = null;
let smartAutoHistoryMessagesFromDb: DBMessage[] = [];
let preferredAgentModelId: string | undefined;
let pendingGridConfirmation:
| PendingGridConfirmation
| null = null;
let shouldAskGridConfirmation = false;
const deepThinkingOn = deepThinkingEnabled !== false;
let shouldRunAgentConversation = finalModelId === "auto";
let shouldKeepSelectedModel = false;
@ -3038,6 +3158,57 @@ export async function POST(request: Request) {
const allowAgentVideoGeneration =
Boolean(requestBody.videoParams) || isVideoProviderType(requestedModelProviderType);
let smartAutoWorkflowDecision:
| {
action?: string;
workflowId?: string;
workflowType?: string;
allowWorkflowExecution?: boolean;
}
| undefined;
const activeStoryboardTaskForChat = await getLatestActiveVideoStoryboardTaskByChatId({
chatId: id,
}).catch((storyboardLookupError) => {
console.warn(
"[Chat API] Failed to preload active storyboard task before smart auto mode:",
storyboardLookupError
);
return null;
});
requestWorkingState = await getConversationWorkingStateByChatId({
chatId: id,
}).catch((workingStatePreloadError) => {
console.warn(
"[Chat API] Failed to preload conversation working state before smart auto mode:",
workingStatePreloadError
);
return null;
});
storyboardPreferenceMemory = extractStoryboardPreferenceMemory(requestWorkingState);
pendingGridConfirmation =
extractPendingGridConfirmationFromWorkingState(requestWorkingState);
if (
pendingGridConfirmation &&
(isGridConfirmationReply(requestTextContent) ||
detectRequestedStoryboardGrid(requestTextContent) !== null) &&
!shouldRunAgentConversation
) {
shouldRunAgentConversation = true;
shouldKeepSelectedModel = true;
console.log(
`[Smart Auto Mode] 🧩 Force agent conversation due to pending grid confirmation: ${pendingGridConfirmation.label}`
);
}
if (activeStoryboardTaskForChat?.task?.id && !shouldRunAgentConversation) {
shouldRunAgentConversation = true;
shouldKeepSelectedModel = true;
console.log(
`[Smart Auto Mode] 🔐 Force agent conversation due to active storyboard task: ${activeStoryboardTaskForChat.task.id}`
);
}
// ========== Smart Auto Mode Logic ==========
if (shouldRunAgentConversation) {
@ -3047,13 +3218,17 @@ export async function POST(request: Request) {
let autoCurrentImageUrlsForFallback: string[] = [];
let confirmedPromptForGeneration: string | undefined;
let forcePromptDrivenImageGeneration = false;
const smartAutoInputText =
pendingGridConfirmation && isGridConfirmationReply(requestTextContent)
? pendingGridConfirmation.originalPrompt
: requestTextContent;
try {
// 🆕 首先获取历史消息（在 Smart Auto Mode 内部单独获取，避免依赖后面的 messagesFromDb）
smartAutoHistoryMessagesFromDb = await getMessagesByChatId({ id });
console.log("[Smart Auto Mode] Loaded", smartAutoHistoryMessagesFromDb.length, "messages from DB");
autoTextForFallback = requestTextContent;
autoTextForFallback = smartAutoInputText;
if (!requestWorkingState) {
requestWorkingState = await getConversationWorkingStateByChatId({
chatId: id,
@ -3157,7 +3332,7 @@ export async function POST(request: Request) {
}
const autoGraphResult = await withOperationTimeout(
runAutoAgentGraph({
text: requestTextContent,
text: smartAutoInputText,
hasImage,
imageUrls,
assetIds: Array.from(
@ -3180,14 +3355,89 @@ export async function POST(request: Request) {
historyMessages,
context: autoContextText || undefined,
preferredModelId: preferredAgentModelId,
threadState: buildAgentThreadStateFromStoryboardTask(
activeStoryboardTaskForChat,
{
storyboardPreferenceMemory,
}
),
}),
"runAutoAgentGraph",
autoGraphTimeoutMs,
);
const requestedStoryboardGrid = detectRequestedStoryboardGrid(
pendingGridConfirmation && isGridConfirmationReply(requestTextContent)
? pendingGridConfirmation.originalPrompt
: requestTextContent
);
if (requestedStoryboardGrid) {
storyboardPreferenceMemory = buildStoryboardPreferenceMemoryUpdate({
existing: storyboardPreferenceMemory,
preferredGridType: requestedStoryboardGrid.gridType,
preferredShotCount: requestedStoryboardGrid.shotCount,
prefersConfirmation: true,
lastUsedGridType: requestedStoryboardGrid.gridType,
lastUsedShotCount: requestedStoryboardGrid.shotCount,
});
}
smartAutoPlan = autoGraphResult.plan;
smartAutoExecution = autoGraphResult.execution;
if (autoGraphResult.workflowDecision) {
smartAutoWorkflowDecision = autoGraphResult.workflowDecision;
console.log("[Smart Auto Mode] Workflow decision:", {
action: autoGraphResult.workflowDecision.action,
intent: autoGraphResult.workflowDecision.confirmationIntent,
workflowId: autoGraphResult.workflowDecision.workflowId,
workflowType: autoGraphResult.workflowDecision.workflowType,
nextWorkflowStatus: autoGraphResult.workflowDecision.nextWorkflowStatus,
allowWorkflowExecution: autoGraphResult.workflowDecision.allowWorkflowExecution,
});
if (
autoGraphResult.workflowDecision.workflowType === "storyboard_video" &&
autoGraphResult.workflowDecision.workflowId
) {
const storyboardTaskId = autoGraphResult.workflowDecision.workflowId;
try {
switch (autoGraphResult.workflowDecision.action) {
case "awaiting_confirmation":
await markVideoStoryboardAwaitingConfirmation({
taskId: storyboardTaskId,
});
break;
case "confirmed":
await markVideoStoryboardConfirmed({
taskId: storyboardTaskId,
messageId: message.id,
});
break;
case "revision_requested":
await markVideoStoryboardDraftForRevision({
taskId: storyboardTaskId,
});
break;
case "cancel_requested":
await cancelVideoStoryboardTaskByUser({
taskId: storyboardTaskId,
error: "user_cancelled_via_chat",
});
break;
case "pass":
default:
break;
}
} catch (workflowPersistError) {
console.warn(
"[Smart Auto Mode] Failed to persist storyboard workflow decision:",
workflowPersistError
);
}
}
}
if (forcePromptDrivenImageGeneration && confirmedPromptForGeneration) {
smartAutoDetectedIntent = "image_gen";
smartAutoOptimizedPrompt = confirmedPromptForGeneration;
@ -3263,11 +3513,44 @@ export async function POST(request: Request) {
}
}
const requestedGrid =
!pendingGridConfirmation || smartAutoInputText === requestTextContent
? detectRequestedStoryboardGrid(requestTextContent)
: pendingGridConfirmation;
const isGridTaskCandidate =
smartAutoExecution?.mode === "generate_image" ||
smartAutoExecution?.mode === "generate_video" ||
autoGraphResult.analysis?.intent === "image_gen" ||
autoGraphResult.analysis?.intent === "video_gen";
if (
requestedGrid &&
!isGridConfirmationReply(requestTextContent) &&
!activeStoryboardTaskForChat?.task?.id &&
isGridTaskCandidate
) {
shouldAskGridConfirmation = true;
smartAutoNeedsClarification = true;
smartAutoClarificationQuestion = buildGridConfirmationMessage(requestedGrid);
smartAutoFriendlyMessage = smartAutoClarificationQuestion;
pendingGridConfirmation = {
...requestedGrid,
originalPrompt: smartAutoInputText,
};
console.log("[Smart Auto Mode] 🧱 Require grid confirmation before generation:", {
gridType: requestedGrid.gridType,
shotCount: requestedGrid.shotCount,
});
}
if (autoGraphResult.friendlyMessage) {
smartAutoFriendlyMessage = autoGraphResult.friendlyMessage;
console.log("[Smart Auto Mode] 💬 Conversational reply:", smartAutoFriendlyMessage);
}
if (shouldAskGridConfirmation && smartAutoClarificationQuestion) {
smartAutoFriendlyMessage = smartAutoClarificationQuestion;
}
if (
!smartAutoNeedsClarification &&
hasImage &&
@ -4324,7 +4607,21 @@ export async function POST(request: Request) {
const optimizedResult = await optimizePromptWithLLM({
originalPrompt: promptSeed,
intent: "image_gen",
historyMessages: smartAutoHistoryMessagesFromDb,
historyMessages: smartAutoHistoryMessagesFromDb
.map((msg) => {
const parts = Array.isArray(msg.parts) ? (msg.parts as any[]) : [];
const content = parts
.filter((part: any) => part?.type === "text")
.map((part: any) => String(part?.text || ""))
.join(" ")
.trim();
if (!content) return null;
return {
role: msg.role as "user" | "assistant",
content: content.slice(0, 300),
};
})
.filter(Boolean) as Array<{ role: "user" | "assistant"; content: string }>,
context: typeof autoContext === "string" ? autoContext : undefined,
}).catch(() =>
optimizePrompt({
@ -4481,12 +4778,301 @@ export async function POST(request: Request) {
},
],
});
if (shouldAskGridConfirmation && pendingGridConfirmation) {
try {
await upsertConversationWorkingState({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
currentIntent: "image_gen",
lastMessageId: messageId,
lastTurnSummary: {
text: requestTextContent.slice(0, 500),
resolutionMode: "grid_confirmation",
confidence: 90,
storyboardMemory: buildStoryboardPreferenceMemoryUpdate({
existing: storyboardPreferenceMemory,
preferredGridType: pendingGridConfirmation.gridType,
preferredShotCount: pendingGridConfirmation.shotCount,
prefersConfirmation: true,
lastUsedGridType: pendingGridConfirmation.gridType,
lastUsedShotCount: pendingGridConfirmation.shotCount,
}),
pendingGridConfirmation: {
gridType: pendingGridConfirmation.gridType,
shotCount: pendingGridConfirmation.shotCount,
label: pendingGridConfirmation.label,
originalPrompt: pendingGridConfirmation.originalPrompt.slice(0, 1000),
},
},
});
} catch (gridConfirmationPersistError) {
console.warn(
"[Chat API] Failed to persist pending grid confirmation state:",
gridConfirmationPersistError
);
}
}
},
});
return createSseResponse(clarificationStream);
}
if (
shouldRunAgentConversation &&
smartAutoWorkflowDecision?.workflowType === "storyboard_video" &&
(smartAutoWorkflowDecision?.action === "revision_requested" ||
smartAutoWorkflowDecision?.action === "cancel_requested")
) {
const workflowReplyStream = createUIMessageStream({
execute: async ({ writer }) => {
const messageId = generateUUID();
let replyText =
smartAutoFriendlyMessage ||
(smartAutoWorkflowDecision?.action === "cancel_requested"
? "已收到取消指令，当前分镜流程不会继续自动执行。"
: "收到修改意见，我会先停在草案阶段，等待你继续调整。");
if (
smartAutoWorkflowDecision?.action === "revision_requested" &&
smartAutoWorkflowDecision.workflowId
) {
const revisionTaskDetail = await getVideoStoryboardTaskDetail(
smartAutoWorkflowDecision.workflowId
).catch(() => null);
const revisionResult = await reviseStoryboardTaskByCommand({
taskId: smartAutoWorkflowDecision.workflowId,
command: revisionTaskDetail
? await resolveStoryboardRevisionCommand({
text: requestTextContent,
detail: revisionTaskDetail,
})
: null,
onProgress: (event) => {
safeWriteStreamChunk(
writer,
{
type: "data-videoStoryboard",
data: JSON.stringify(event),
},
`storyboard-revision:${event.stage}`,
);
},
});
replyText = revisionResult.message || replyText;
}
writer.write({
type: "data-appendMessage",
data: JSON.stringify({
id: messageId,
role: "assistant",
parts: [{ type: "text", text: replyText }],
attachments: [],
}),
});
await saveMessages({
messages: [
{
id: messageId,
chatId: id,
role: "assistant",
parts: [{ type: "text", text: replyText }],
attachments: [],
createdAt: new Date(),
videoStorageStatus: null,
temporaryVideoUrl: null,
permanentVideoUrl: null,
archivedAt: null,
videoMetadata:
smartAutoWorkflowDecision?.workflowId
? ({
storyboard: {
enabled: true,
taskId: smartAutoWorkflowDecision.workflowId,
action: smartAutoWorkflowDecision.action,
},
updatedAt: Date.now(),
} as any)
: null,
},
],
});
},
});
return createSseResponse(workflowReplyStream);
}
if (
shouldRunAgentConversation &&
smartAutoWorkflowDecision?.workflowType === "storyboard_video" &&
smartAutoWorkflowDecision?.action === "confirmed" &&
smartAutoWorkflowDecision.workflowId
) {
const confirmedTaskId = smartAutoWorkflowDecision.workflowId;
const confirmedStoryboardStream = createUIMessageStream({
execute: async ({ writer }) => {
const acknowledgeMessageId = generateUUID();
const acknowledgeMessage = "已收到确认，开始按分镜逐段生成视频，请稍候。";
writer.write({
type: "data-appendMessage",
data: JSON.stringify({
id: acknowledgeMessageId,
role: "assistant",
parts: [{ type: "text", text: acknowledgeMessage }],
attachments: [],
}),
});
await saveMessages({
messages: [
{
id: acknowledgeMessageId,
chatId: id,
role: "assistant",
parts: [{ type: "text", text: acknowledgeMessage }],
attachments: [],
createdAt: new Date(),
videoStorageStatus: null,
temporaryVideoUrl: null,
permanentVideoUrl: null,
archivedAt: null,
videoMetadata: {
storyboard: {
enabled: true,
taskId: confirmedTaskId,
status: "confirmed",
},
updatedAt: Date.now(),
} as any,
},
],
});
try {
await upsertConversationWorkingState({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
currentIntent: "video_gen",
lastMessageId: acknowledgeMessageId,
lastTurnSummary: {
text: requestTextContent.slice(0, 500),
resolutionMode: "storyboard_confirmed",
confidence: 95,
storyboardMemory: buildStoryboardPreferenceMemoryUpdate({
existing: storyboardPreferenceMemory,
prefersConfirmation: true,
}),
},
});
} catch (storyboardConfirmMemoryError) {
console.warn(
"[Chat API] Failed to persist storyboard confirmation memory:",
storyboardConfirmMemoryError
);
}
const executionResult = await executeConfirmedStoryboardTask({
taskId: confirmedTaskId,
onProgress: (event) => {
safeWriteStreamChunk(
writer,
{
type: "data-videoStoryboard",
data: JSON.stringify(event),
},
`storyboard-confirmed:${event.stage}`,
);
},
});
const finalMessageId = generateUUID();
writer.write({
type: "data-appendMessage",
data: JSON.stringify({
id: finalMessageId,
role: "assistant",
parts: [{ type: "text", text: executionResult.markdown }],
attachments: [],
}),
});
const firstVideoUrl = executionResult.videoUrls[0] || "";
await saveMessages({
messages: [
{
id: finalMessageId,
chatId: id,
role: "assistant",
parts: [{ type: "text", text: executionResult.markdown }],
attachments: [],
createdAt: new Date(),
videoStorageStatus: firstVideoUrl ? "temporary" : null,
temporaryVideoUrl: firstVideoUrl || null,
permanentVideoUrl: null,
archivedAt: null,
videoMetadata: {
storyboard: {
enabled: true,
taskId: executionResult.taskId,
successCount: executionResult.successCount,
failedCount: executionResult.failedCount,
},
status: executionResult.status,
updatedAt: Date.now(),
} as any,
},
],
});
await attachVideoStoryboardTaskMessage({
taskId: executionResult.taskId,
messageId: finalMessageId,
}).catch((taskAttachError) => {
console.warn(
"[Chat API] Failed to attach confirmed storyboard task with message:",
taskAttachError
);
});
if (firstVideoUrl) {
await consumeGeneratedVideoPoints({
userId: session.user.id,
chatId: id,
messageId: finalMessageId,
modelRefs: executionResult.modelRefs,
});
scheduleVideoArchive({
chatId: id,
messageId: finalMessageId,
text: executionResult.markdown,
temporaryVideoUrl: firstVideoUrl,
taskId: finalMessageId,
metadata: {
storyboard: {
enabled: true,
taskId: executionResult.taskId,
successCount: executionResult.successCount,
failedCount: executionResult.failedCount,
},
status: executionResult.status,
updatedAt: Date.now(),
},
});
}
},
});
return createSseResponse(confirmedStoryboardStream);
}
if (agentPresetId) {
const selectedAgentPreset = await getAccessibleAgentPresetById({
id: agentPresetId,
@ -4503,6 +5089,13 @@ export async function POST(request: Request) {
);
}
if (selectedAgentPreset.type === "text") {
// 生文智能体走常规多模态对话链路：
// - 模型已由前端切到智能体预设模型
// - 提示词通过 hiddenPrompt 注入
// - 图片输入沿用普通多模态附件
} else {
const orderedSlots = [...(selectedAgentPreset.slots || [])].sort(
(left, right) => left.slotIndex - right.slotIndex,
);
@ -4984,6 +5577,7 @@ export async function POST(request: Request) {
});
return createSseResponse(agentSequenceStream);
}
}
const directAgentExecution = shouldRunAgentConversation
@ -7224,6 +7818,8 @@ export async function POST(request: Request) {
usedStoryboardPipeline = true;
videoMarkdown = storyboardResult.markdown || "视频生成失败，请稍后重试";
tempVideoUrl = storyboardResult.videoUrls?.[0] || "";
const isAwaitingConfirmation =
storyboardResult.reason === "awaiting_user_confirmation";
billedModelRefs =
storyboardResult.modelRefs && storyboardResult.modelRefs.length > 0
? storyboardResult.modelRefs
@ -7233,12 +7829,17 @@ export async function POST(request: Request) {
videoMetadata = {
...videoMetadata,
...(storyboardResult.metadata || {}),
status: successCount > 0 ? "succeeded" : "failed",
status: isAwaitingConfirmation
? "awaiting_user_confirmation"
: successCount > 0
? "succeeded"
: "failed",
storyboard: {
enabled: true,
taskId: storyboardResult.taskId,
confidence: storyboardResult.confidence,
successCount,
awaitingConfirmation: isAwaitingConfirmation,
},
updatedAt: Date.now(),
};
@ -7307,12 +7908,57 @@ export async function POST(request: Request) {
});
}
console.log(`[Chat API] ✅ ${currentVideoProvider} video message saved successfully!`);
await consumeGeneratedVideoPoints({
userId: session.user.id,
chatId: id,
messageId,
modelRefs: billedModelRefs,
});
if ((videoMetadata as any)?.storyboard?.enabled) {
try {
const storyboardGridType = String((videoMetadata as any)?.gridType || "").trim();
const storyboardShotCount = Number((videoMetadata as any)?.shotCount || 0);
await upsertConversationWorkingState({
userId: session.user.id,
chatId: id,
projectId: contextPayload?.projectId || null,
currentIntent: "video_gen",
lastMessageId: messageId,
lastTurnSummary: {
text: requestTextContent.slice(0, 500),
resolutionMode: "storyboard",
confidence: 95,
storyboardMemory: buildStoryboardPreferenceMemoryUpdate({
existing: storyboardPreferenceMemory,
prefersConfirmation: true,
lastUsedGridType:
storyboardGridType === "grid_2x2" ||
storyboardGridType === "grid_3x2" ||
storyboardGridType === "grid_3x3"
? (storyboardGridType as "grid_2x2" | "grid_3x2" | "grid_3x3")
: undefined,
lastUsedShotCount:
storyboardShotCount === 4 ||
storyboardShotCount === 6 ||
storyboardShotCount === 9
? storyboardShotCount
: undefined,
}),
},
});
} catch (storyboardMemoryPersistError) {
console.warn(
"[Chat API] Failed to persist storyboard memory after assistant response:",
storyboardMemoryPersistError
);
}
}
const shouldConsumeVideoPoints =
tempVideoUrl &&
(videoMetadata as any)?.status !== "awaiting_user_confirmation";
if (shouldConsumeVideoPoints) {
await consumeGeneratedVideoPoints({
userId: session.user.id,
chatId: id,
messageId,
modelRefs: billedModelRefs,
});
}
if (tempVideoUrl) {
scheduleVideoArchive({

75
app/(chat)/api/video/storyboard/tasks/[taskId]/cancel/route.ts Normal file
View File

@ -0,0 +1,75 @@
import { NextResponse } from "next/server";
import { z } from "zod";
import { auth } from "@/app/(auth)/auth";
import {
cancelVideoStoryboardTaskByUser,
getVideoStoryboardTaskDetail,
} from "@/lib/video-agent/task-store";
import {
requireStoryboardTaskAccess,
StoryboardTaskAccessError,
} from "@/lib/video-agent/user-task-access";
const paramsSchema = z.object({
taskId: z.string().uuid(),
});
export async function POST(
_request: Request,
{ params }: { params: Promise<{ taskId: string }> }
) {
try {
const session = await auth();
if (!session?.user?.id) {
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
}
const parsed = paramsSchema.parse(await params);
const detail = await requireStoryboardTaskAccess({
taskId: parsed.taskId,
userId: session.user.id,
});
const status = String((detail.task as any).status || "").trim();
if (status === "cancelled") {
return NextResponse.json({
success: true,
task: detail.task,
shots: detail.shots,
});
}
await cancelVideoStoryboardTaskByUser({
taskId: parsed.taskId,
error: "user_cancelled_via_api",
});
const nextDetail = await getVideoStoryboardTaskDetail(parsed.taskId);
return NextResponse.json({
success: true,
task: nextDetail?.task || detail.task,
shots: nextDetail?.shots || detail.shots,
});
} catch (error) {
if (error instanceof StoryboardTaskAccessError) {
return NextResponse.json(
{ success: false, error: error.message },
{ status: error.status }
);
}
if (error instanceof z.ZodError) {
return NextResponse.json(
{ success: false, error: error.message },
{ status: 400 }
);
}
return NextResponse.json(
{
success: false,
error: error instanceof Error ? error.message : "取消任务失败",
},
{ status: 500 }
);
}
}

101
app/(chat)/api/video/storyboard/tasks/[taskId]/confirm/route.ts Normal file
View File

@ -0,0 +1,101 @@
import { after, NextResponse } from "next/server";
import { z } from "zod";
import { auth } from "@/app/(auth)/auth";
import { executeConfirmedStoryboardTask } from "@/lib/video-agent/execute-storyboard-task";
import { getVideoStoryboardTaskDetail, markVideoStoryboardConfirmed } from "@/lib/video-agent/task-store";
import {
requireStoryboardTaskAccess,
StoryboardTaskAccessError,
} from "@/lib/video-agent/user-task-access";
const paramsSchema = z.object({
taskId: z.string().uuid(),
});
export async function POST(
_request: Request,
{ params }: { params: Promise<{ taskId: string }> }
) {
try {
const session = await auth();
if (!session?.user?.id) {
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
}
const parsed = paramsSchema.parse(await params);
const detail = await requireStoryboardTaskAccess({
taskId: parsed.taskId,
userId: session.user.id,
});
const task = detail.task as any;
const status = String(task.status || "").trim();
const confirmationStatus = String(task.confirmationStatus || "").trim();
if (status === "cancelled") {
return NextResponse.json(
{ success: false, error: "任务已取消，无法确认执行" },
{ status: 400 }
);
}
if (["succeeded", "partial_success", "failed"].includes(status)) {
return NextResponse.json(
{ success: false, error: "任务已结束，无法再次确认执行" },
{ status: 400 }
);
}
if (status === "running" && confirmationStatus === "confirmed") {
return NextResponse.json(
{ success: true, queued: false, task: detail.task, shots: detail.shots },
{ status: 200 }
);
}
await markVideoStoryboardConfirmed({
taskId: parsed.taskId,
});
after(async () => {
try {
await executeConfirmedStoryboardTask({
taskId: parsed.taskId,
});
} catch (error) {
console.error("[Storyboard Confirm API] Background execution failed:", error);
}
});
const nextDetail = await getVideoStoryboardTaskDetail(parsed.taskId);
return NextResponse.json(
{
success: true,
queued: true,
task: nextDetail?.task || detail.task,
shots: nextDetail?.shots || detail.shots,
},
{ status: 202 }
);
} catch (error) {
if (error instanceof StoryboardTaskAccessError) {
return NextResponse.json(
{ success: false, error: error.message },
{ status: error.status }
);
}
if (error instanceof z.ZodError) {
return NextResponse.json(
{ success: false, error: error.message },
{ status: 400 }
);
}
return NextResponse.json(
{
success: false,
error: error instanceof Error ? error.message : "确认执行失败",
},
{ status: 500 }
);
}
}

56
app/(chat)/api/video/storyboard/tasks/[taskId]/route.ts Normal file
View File

@ -0,0 +1,56 @@
import { NextResponse } from "next/server";
import { z } from "zod";
import { auth } from "@/app/(auth)/auth";
import {
requireStoryboardTaskAccess,
StoryboardTaskAccessError,
} from "@/lib/video-agent/user-task-access";
const paramsSchema = z.object({
taskId: z.string().uuid(),
});
export async function GET(
_request: Request,
{ params }: { params: Promise<{ taskId: string }> }
) {
try {
const session = await auth();
if (!session?.user?.id) {
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
}
const parsed = paramsSchema.parse(await params);
const detail = await requireStoryboardTaskAccess({
taskId: parsed.taskId,
userId: session.user.id,
});
return NextResponse.json({
success: true,
task: detail.task,
shots: detail.shots,
});
} catch (error) {
if (error instanceof StoryboardTaskAccessError) {
return NextResponse.json(
{ success: false, error: error.message },
{ status: error.status }
);
}
if (error instanceof z.ZodError) {
return NextResponse.json(
{ success: false, error: error.message },
{ status: 400 }
);
}
return NextResponse.json(
{
success: false,
error: error instanceof Error ? error.message : "查询任务失败",
},
{ status: 500 }
);
}
}

73
app/(chat)/api/video/storyboard/tasks/[taskId]/shots/[shotIndex]/regenerate-hd/route.ts Normal file
View File

@ -0,0 +1,73 @@
import { NextResponse } from "next/server";
import { z } from "zod";
import { auth } from "@/app/(auth)/auth";
import { regenerateStoryboardShotHd } from "@/lib/video-agent/revise-storyboard-task";
import {
requireStoryboardTaskAccess,
StoryboardTaskAccessError,
} from "@/lib/video-agent/user-task-access";
const paramsSchema = z.object({
taskId: z.string().uuid(),
shotIndex: z.coerce.number().int().min(1).max(99),
});
export async function POST(
_request: Request,
{ params }: { params: Promise<{ taskId: string; shotIndex: string }> }
) {
try {
const session = await auth();
if (!session?.user?.id) {
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
}
const parsedParams = paramsSchema.parse(await params);
const detail = await requireStoryboardTaskAccess({
taskId: parsedParams.taskId,
userId: session.user.id,
});
const status = String((detail.task as any).status || "").trim();
const confirmationStatus = String((detail.task as any).confirmationStatus || "").trim();
if (status === "running" && confirmationStatus === "confirmed") {
return NextResponse.json(
{ success: false, error: "任务正在执行视频，暂时不能重生高清预览" },
{ status: 409 }
);
}
const result = await regenerateStoryboardShotHd({
taskId: parsedParams.taskId,
shotIndex: parsedParams.shotIndex,
});
return NextResponse.json({
success: result.ok,
message: result.message,
task: result.detail?.task || detail.task,
shots: result.detail?.shots || detail.shots,
});
} catch (error) {
if (error instanceof StoryboardTaskAccessError) {
return NextResponse.json(
{ success: false, error: error.message },
{ status: error.status }
);
}
if (error instanceof z.ZodError) {
return NextResponse.json(
{ success: false, error: error.message },
{ status: 400 }
);
}
return NextResponse.json(
{
success: false,
error: error instanceof Error ? error.message : "重生高清预览失败",
},
{ status: 500 }
);
}
}

103
app/(chat)/api/video/storyboard/tasks/[taskId]/shots/[shotIndex]/route.ts Normal file
View File

@ -0,0 +1,103 @@
import { NextResponse } from "next/server";
import { z } from "zod";
import { auth } from "@/app/(auth)/auth";
import {
getVideoStoryboardTaskDetail,
markVideoStoryboardDraftForRevision,
updateVideoStoryboardShotPrompt,
} from "@/lib/video-agent/task-store";
import { updateStoryboardShotPromptAndRegenerateHd } from "@/lib/video-agent/revise-storyboard-task";
import {
requireStoryboardTaskAccess,
StoryboardTaskAccessError,
} from "@/lib/video-agent/user-task-access";
const paramsSchema = z.object({
taskId: z.string().uuid(),
shotIndex: z.coerce.number().int().min(1).max(99),
});
const bodySchema = z.object({
prompt: z.string().trim().min(1).max(4000),
regenerateHd: z.boolean().optional().default(true),
});
export async function PATCH(
request: Request,
{ params }: { params: Promise<{ taskId: string; shotIndex: string }> }
) {
try {
const session = await auth();
if (!session?.user?.id) {
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
}
const parsedParams = paramsSchema.parse(await params);
const parsedBody = bodySchema.parse(await request.json().catch(() => ({})));
const detail = await requireStoryboardTaskAccess({
taskId: parsedParams.taskId,
userId: session.user.id,
});
const status = String((detail.task as any).status || "").trim();
const confirmationStatus = String((detail.task as any).confirmationStatus || "").trim();
if (status === "running" && confirmationStatus === "confirmed") {
return NextResponse.json(
{ success: false, error: "任务正在执行视频，暂时不能修改镜头" },
{ status: 409 }
);
}
if (parsedBody.regenerateHd) {
const result = await updateStoryboardShotPromptAndRegenerateHd({
taskId: parsedParams.taskId,
shotIndex: parsedParams.shotIndex,
prompt: parsedBody.prompt,
});
return NextResponse.json({
success: result.ok,
message: result.message,
task: result.detail?.task || detail.task,
shots: result.detail?.shots || detail.shots,
});
}
await markVideoStoryboardDraftForRevision({
taskId: parsedParams.taskId,
});
await updateVideoStoryboardShotPrompt({
taskId: parsedParams.taskId,
shotIndex: parsedParams.shotIndex,
prompt: parsedBody.prompt,
});
const nextDetail = await getVideoStoryboardTaskDetail(parsedParams.taskId);
return NextResponse.json({
success: true,
message: `已更新第 ${parsedParams.shotIndex} 镜头文案。`,
task: nextDetail?.task || detail.task,
shots: nextDetail?.shots || detail.shots,
});
} catch (error) {
if (error instanceof StoryboardTaskAccessError) {
return NextResponse.json(
{ success: false, error: error.message },
{ status: error.status }
);
}
if (error instanceof z.ZodError) {
return NextResponse.json(
{ success: false, error: error.message },
{ status: 400 }
);
}
return NextResponse.json(
{
success: false,
error: error instanceof Error ? error.message : "更新镜头失败",
},
{ status: 500 }
);
}
}

40
app/api/agents/[id]/route.ts
View File

@ -5,8 +5,14 @@ import { ChatSDKError } from "@/lib/errors";
import {
deleteAgentPresetById,
getAccessibleAgentPresetById,
getSystemSetting,
updateAgentPreset,
} from "@/lib/db/queries";
import {
AGENT_SECONDARY_TYPE_CONFIG_KEY,
isValidSecondaryTypeForPresetType,
normalizeAgentSecondaryTypeConfig,
} from "@/lib/agents/secondary-type-config";
const slotSchema = z.object({
title: z.string().trim().max(128).optional(),
@ -20,7 +26,8 @@ const slotSchema = z.object({
const updateAgentSchema = z
.object({
name: z.string().trim().min(1).max(128).optional(),
type: z.enum(["image", "video"]).optional(),
type: z.enum(["image", "video", "text"]).optional(),
secondaryType: z.string().trim().max(128).nullable().optional(),
visibility: z.enum(["public", "private"]).optional(),
categoryPath: z
.object({
@ -84,12 +91,43 @@ export async function PUT(
const { id } = await params;
const body = await request.json();
const data = updateAgentSchema.parse(body);
const existingPreset = await getAccessibleAgentPresetById({
id,
userId: session.user.id,
});
if (!existingPreset) {
return NextResponse.json({ success: false, error: "智能体不存在" }, { status: 404 });
}
const effectiveType = data.type || existingPreset.type;
const effectiveSecondaryType =
data.secondaryType === undefined ? existingPreset.secondaryType : data.secondaryType;
const secondaryTypeConfig = normalizeAgentSecondaryTypeConfig(
(
await getSystemSetting({ key: AGENT_SECONDARY_TYPE_CONFIG_KEY }).catch(() => undefined)
)?.value
);
if (
!isValidSecondaryTypeForPresetType({
presetType: effectiveType,
secondaryType: effectiveSecondaryType,
config: secondaryTypeConfig,
})
) {
return NextResponse.json(
{ success: false, error: "所选二级类型不在后台配置列表中" },
{ status: 400 }
);
}
const preset = await updateAgentPreset({
id,
userId: session.user.id,
name: data.name,
type: data.type,
secondaryType: data.secondaryType ?? undefined,
visibility: data.visibility,
categoryPath: data.categoryPath ?? undefined,
description: data.description ?? undefined,

30
app/api/agents/route.ts
View File

@ -4,12 +4,18 @@ import { auth } from "@/app/(auth)/auth";
import { ChatSDKError } from "@/lib/errors";
import {
createAgentPreset,
getSystemSetting,
getVisibleAgentPresets,
} from "@/lib/db/queries";
import {
AGENT_SECONDARY_TYPE_CONFIG_KEY,
isValidSecondaryTypeForPresetType,
normalizeAgentSecondaryTypeConfig,
} from "@/lib/agents/secondary-type-config";
const listQuerySchema = z.object({
scope: z.enum(["all", "public", "private"]).default("all"),
type: z.enum(["image", "video"]).optional(),
type: z.enum(["image", "video", "text"]).optional(),
productCategory: z.string().trim().max(64).optional(),
audienceTag: z.string().trim().max(64).optional(),
usageScenario: z.string().trim().max(64).optional(),
@ -30,7 +36,8 @@ const slotSchema = z.object({
const createAgentSchema = z
.object({
name: z.string().trim().min(1).max(128),
type: z.enum(["image", "video"]),
type: z.enum(["image", "video", "text"]),
secondaryType: z.string().trim().max(128).optional(),
visibility: z.enum(["public", "private"]).default("private"),
categoryPath: z
.object({
@ -118,11 +125,30 @@ export async function POST(request: NextRequest) {
const body = await request.json();
const data = createAgentSchema.parse(body);
const secondaryTypeConfig = normalizeAgentSecondaryTypeConfig(
(
await getSystemSetting({ key: AGENT_SECONDARY_TYPE_CONFIG_KEY }).catch(() => undefined)
)?.value
);
if (
!isValidSecondaryTypeForPresetType({
presetType: data.type,
secondaryType: data.secondaryType,
config: secondaryTypeConfig,
})
) {
return NextResponse.json(
{ success: false, error: "所选二级类型不在后台配置列表中" },
{ status: 400 }
);
}
const preset = await createAgentPreset({
userId: session.user.id,
name: data.name,
type: data.type,
secondaryType: data.secondaryType,
visibility: data.visibility,
categoryPath: data.categoryPath,
description: data.description,

49
app/api/site-settings/route.ts
View File

@ -24,6 +24,11 @@ import {
normalizeDulupayConfig,
normalizeDulupayForPublic,
} from "@/lib/payment/dulupay-config";
import {
AGENT_SECONDARY_TYPE_CONFIG_KEY,
defaultAgentSecondaryTypeConfig,
normalizeAgentSecondaryTypeConfig,
} from "@/lib/agents/secondary-type-config";
import {
defaultOssImageMigrationConfig,
normalizeOssImageMigrationConfig,
@ -208,6 +213,7 @@ export async function GET() {
storageFailoverConfigSetting,
ossImageMigrationConfigSetting,
agentCategoryTreeConfigSetting,
agentSecondaryTypeConfigSetting,
] = await Promise.all([
db.select().from(siteSettings).limit(1).then(rows => rows[0]),
db
@ -312,6 +318,12 @@ export async function GET() {
.where(eq(systemSettings.key, AGENT_CATEGORY_TREE_CONFIG_KEY))
.limit(1)
.then(rows => rows[0]),
db
.select()
.from(systemSettings)
.where(eq(systemSettings.key, AGENT_SECONDARY_TYPE_CONFIG_KEY))
.limit(1)
.then(rows => rows[0]),
]);
const notification = notificationSetting?.value || { enabled: false, title: "", content: "" };
@ -350,6 +362,9 @@ export async function GET() {
const agentCategoryTreeConfig = normalizeAgentCategoryTreeConfig(
agentCategoryTreeConfigSetting?.value
);
const agentSecondaryTypeConfig = normalizeAgentSecondaryTypeConfig(
agentSecondaryTypeConfigSetting?.value
);
// 如果没有设置记录，返回默认值
const baseSettings = settings || {
@ -376,6 +391,7 @@ export async function GET() {
canvasEnabled: true,
agentEnabled: true,
agentCategoryTreeConfig: normalizeAgentCategoryTreeConfig(undefined),
agentSecondaryTypeConfig: defaultAgentSecondaryTypeConfig,
realtimeCanvasEnabled: true,
realtimeCanvasServiceEnabled: true,
storageFailoverConfig,
@ -418,6 +434,7 @@ export async function GET() {
lovartPromptTagVisibility,
contactConfig,
agentCategoryTreeConfig,
agentSecondaryTypeConfig,
membershipConfig,
membershipUserOverrides,
pointsSystemConfig,
@ -474,6 +491,13 @@ const updateSchema = z.object({
brands: z.array(z.string().trim().min(1).max(64)).max(200).optional(),
})
.optional(),
agentSecondaryTypeConfig: z
.object({
image: z.array(z.string().trim().min(1).max(64)).max(200).optional(),
video: z.array(z.string().trim().min(1).max(64)).max(200).optional(),
text: z.array(z.string().trim().min(1).max(64)).max(200).optional(),
})
.optional(),
realtimeCanvasEnabled: z.boolean().optional(),
aiDetailGenerationEnabled: z.boolean().optional(),
canvasDefaultModelId: z.string().nullable().optional(),
@ -550,6 +574,7 @@ export async function PUT(request: NextRequest) {
lovartPromptTagVisibility,
contactConfig,
agentCategoryTreeConfig,
agentSecondaryTypeConfig,
membershipConfig,
membershipUserOverrides,
pointsSystemConfig,
@ -734,6 +759,26 @@ export async function PUT(request: NextRequest) {
});
}
if (agentSecondaryTypeConfig !== undefined) {
const normalized = normalizeAgentSecondaryTypeConfig(agentSecondaryTypeConfig);
await db
.insert(systemSettings)
.values({
key: AGENT_SECONDARY_TYPE_CONFIG_KEY,
value: normalized,
updatedAt: new Date(),
updatedBy: session.user.id,
})
.onConflictDoUpdate({
target: systemSettings.key,
set: {
value: normalized,
updatedAt: new Date(),
updatedBy: session.user.id,
},
});
}
if (membershipConfig !== undefined) {
const normalized = normalizeMembershipConfig(membershipConfig);
await db
@ -902,6 +947,10 @@ export async function PUT(request: NextRequest) {
agentCategoryTreeConfig !== undefined
? normalizeAgentCategoryTreeConfig(agentCategoryTreeConfig)
: normalizeAgentCategoryTreeConfig(undefined),
agentSecondaryTypeConfig:
agentSecondaryTypeConfig !== undefined
? normalizeAgentSecondaryTypeConfig(agentSecondaryTypeConfig)
: defaultAgentSecondaryTypeConfig,
membershipConfig:
membershipConfig !== undefined
? normalizeMembershipConfig(membershipConfig)

7
components/agents/AgentPresetCard.tsx
View File

@ -77,8 +77,13 @@ export function AgentPresetCard({
)}
<div className="absolute left-2 top-2 flex flex-wrap gap-1.5">
<Badge variant="secondary" className="bg-white/90 text-black shadow-sm">
{preset.type === "video" ? "视频" : "生图"}
{preset.type === "video" ? "视频" : preset.type === "text" ? "生文" : "生图"}
</Badge>
{preset.secondaryType ? (
<Badge key={preset.secondaryType} variant="secondary" className="bg-white/90 text-black shadow-sm">
{preset.secondaryType}
</Badge>
) : null}
{categoryBadges.map((item) => (
<Badge key={item} variant="secondary" className="bg-white/90 text-black shadow-sm">
{item}

51
components/agents/AgentPresetDialog.tsx
View File

@ -27,6 +27,7 @@ import { fetcher } from "@/lib/utils";
import { imageGenerationService } from "@/services/canvas";
import { agentService } from "@/services/agents";
import type {
AgentPresetType,
AgentCategoryTreeConfig,
AgentPreset,
AgentPresetSlot,
@ -44,6 +45,11 @@ interface AgentPresetDialogProps {
open: boolean;
onOpenChange: (open: boolean) => void;
categoryTreeConfig: AgentCategoryTreeConfig;
secondaryTypeConfig: {
image: string[];
video: string[];
text: string[];
};
initialPreset?: AgentPreset | null;
onSaved?: (preset: AgentPreset) => void;
}
@ -52,6 +58,7 @@ export function AgentPresetDialog({
open,
onOpenChange,
categoryTreeConfig,
secondaryTypeConfig,
initialPreset,
onSaved,
}: AgentPresetDialogProps) {
@ -61,11 +68,13 @@ export function AgentPresetDialog({
displayName: string;
supportsImageGen?: boolean;
providerType?: string;
isMultimodal?: boolean;
}>;
}>("/api/models", fetcher);
const [name, setName] = useState("");
const [type, setType] = useState<"image" | "video">("image");
const [type, setType] = useState<"image" | "video" | "text">("image");
const [secondaryType, setSecondaryType] = useState("");
const [visibility, setVisibility] = useState<"public" | "private">("private");
const [categoryPath, setCategoryPath] = useState<{
productCategory: string;
@ -109,11 +118,22 @@ export function AgentPresetDialog({
if (type === "video") {
return models.filter((model) => videoProviderTypes.has(String(model.providerType || "")));
}
if (type === "text") {
return models.filter(
(model) =>
model.isMultimodal === true &&
!videoProviderTypes.has(String(model.providerType || ""))
);
}
return models.filter(
(model) =>
model.supportsImageGen && !videoProviderTypes.has(String(model.providerType || ""))
);
}, [modelsData?.models, type, videoProviderTypes]);
const secondaryTypeOptions = useMemo(
() => secondaryTypeConfig[type] || [],
[secondaryTypeConfig, type]
);
useEffect(() => {
if (!open) return;
@ -141,6 +161,7 @@ export function AgentPresetDialog({
const resetState = () => {
setName("");
setType("image");
setSecondaryType("");
setVisibility("private");
setCategoryPath({
productCategory: "",
@ -176,6 +197,7 @@ export function AgentPresetDialog({
setName(initialPreset.name || "");
setType(initialPreset.type);
setSecondaryType(String(initialPreset.secondaryType || ""));
setVisibility(initialPreset.visibility);
setCategoryPath({
productCategory: String(initialPreset.categoryPath?.productCategory || ""),
@ -330,6 +352,7 @@ export function AgentPresetDialog({
const payload = {
name: name.trim(),
type,
secondaryType: secondaryType || undefined,
visibility,
categoryPath: {
productCategory: categoryPath.productCategory || undefined,
@ -455,8 +478,9 @@ export function AgentPresetDialog({
<div className="text-sm font-medium">类型</div>
<Select
value={type}
onValueChange={(value: "image" | "video") => {
onValueChange={(value: AgentPresetType) => {
setType(value);
setSecondaryType("");
setDefaultModelId("");
}}
>
@ -466,9 +490,32 @@ export function AgentPresetDialog({
<SelectContent>
<SelectItem value="image">生图</SelectItem>
<SelectItem value="video">视频</SelectItem>
<SelectItem value="text">生文</SelectItem>
</SelectContent>
</Select>
</div>
<div className="space-y-2">
<div className="text-sm font-medium">二级类型</div>
<Select
value={secondaryType || "__none__"}
onValueChange={(value) => setSecondaryType(value === "__none__" ? "" : value)}
>
<SelectTrigger>
<SelectValue placeholder="选择二级类型" />
</SelectTrigger>
<SelectContent>
<SelectItem value="__none__">不选择</SelectItem>
{secondaryTypeOptions.map((item) => (
<SelectItem key={item} value={item}>
{item}
</SelectItem>
))}
</SelectContent>
</Select>
<div className="text-xs text-muted-foreground">
二级类型由管理员在系统设置中统一配置。
</div>
</div>
<div className="space-y-2">
<div className="text-sm font-medium">产品品类</div>
<Select

14
components/agents/AgentPresetPanel.tsx
View File

@ -45,6 +45,11 @@ export function AgentPresetPanel({
usageScenarios?: string[];
brands?: string[];
};
agentSecondaryTypeConfig?: {
image?: string[];
video?: string[];
text?: string[];
};
}>("/api/site-settings", fetcher);
const { data, isLoading, mutate } = useSWR(
["agent-presets"],
@ -69,6 +74,14 @@ export function AgentPresetPanel({
}),
[siteSettingsData?.agentCategoryTreeConfig]
);
const secondaryTypeConfig = useMemo(
() => ({
image: siteSettingsData?.agentSecondaryTypeConfig?.image || [],
video: siteSettingsData?.agentSecondaryTypeConfig?.video || [],
text: siteSettingsData?.agentSecondaryTypeConfig?.text || [],
}),
[siteSettingsData?.agentSecondaryTypeConfig]
);
const filteredPublicPresets = useMemo(() => {
return (data?.public || []).filter((preset) => {
const path = preset.categoryPath || {};
@ -313,6 +326,7 @@ export function AgentPresetPanel({
setShowCreateDialog(true);
}}
categoryTreeConfig={categoryTreeConfig}
secondaryTypeConfig={secondaryTypeConfig}
initialPreset={editingPreset}
onSaved={async (preset) => {
await mutate();

222
components/canvas/CanvasChatPanel.tsx
View File

@ -123,6 +123,9 @@ type VideoStoryboardStreamData =
provider?: string;
taskId: string;
totalShots: number;
gridType?: "grid_2x2" | "grid_3x2" | "grid_3x3";
rows?: number;
cols?: number;
confidence?: number;
sourceImageUrl?: string;
shots?: Array<{
@ -131,6 +134,37 @@ type VideoStoryboardStreamData =
prompt?: string;
}>;
}
| {
stage: "hd_shot_completed";
provider?: string;
taskId: string;
shotIndex: number;
imageUrl: string;
prompt?: string;
}
| {
stage: "hd_shot_failed";
provider?: string;
taskId: string;
shotIndex: number;
error?: string;
}
| {
stage: "awaiting_confirmation";
provider?: string;
taskId: string;
totalShots: number;
gridType?: "grid_2x2" | "grid_3x2" | "grid_3x3";
rows?: number;
cols?: number;
message?: string;
shots?: Array<{
shotIndex: number;
tileImageUrl?: string;
previewImageUrl?: string;
prompt?: string;
}>;
}
| {
stage: "shot_completed";
provider?: string;
@ -2889,7 +2923,11 @@ export function CanvasChatPanel({
return "image";
}, [isAgentModel, isDetailGenerationMode, isVideoModel]);
const agentUploadSlots = useMemo(() => {
if (activeComposerMode !== "agent" || !selectedAgentPreset) {
if (
activeComposerMode !== "agent" ||
!selectedAgentPreset ||
selectedAgentPreset.type === "text"
) {
return [] as Array<{
slotIndex: number;
label: string;
@ -3146,26 +3184,17 @@ export function CanvasChatPanel({
useEffect(() => {
if (!selectedAgentPresetId) return;
if (activeComposerMode === "agent") return;
if (!selectedAgentPreset?.defaultModelId) return;
if (currentModelIdRef.current === selectedAgentPreset.defaultModelId) return;
const preferredAgentModelId =
allModels.find(
(model) =>
model.id === lastConversationModelId &&
model.isMultimodal &&
!VIDEO_PROVIDER_TYPES.has(model.providerType)
)?.id || "auto";
currentModelIdRef.current = preferredAgentModelId;
setSelectedModelId(preferredAgentModelId);
currentModelIdRef.current = selectedAgentPreset.defaultModelId;
setSelectedModelId(selectedAgentPreset.defaultModelId);
if (isDetailGenerationMode) {
setIsDetailGenerationMode(false);
}
}, [
activeComposerMode,
allModels,
isDetailGenerationMode,
lastConversationModelId,
selectedAgentPreset,
selectedAgentPresetId,
]);
@ -3885,6 +3914,8 @@ export function CanvasChatPanel({
if (storyboardNodeMapRef.current[taskId]) return;
const totalShots = Math.max(1, Math.min(9, Number(payload.totalShots || 9)));
const layoutCols =
Math.max(1, Math.min(3, Number(payload.cols || (totalShots === 4 ? 2 : 3)))) || 3;
const shotNodeMap: Record<number, string> = {};
const currentNodes = useCanvasStore.getState().nodes;
const baseOffset = currentNodes.length;
@ -3897,20 +3928,26 @@ export function CanvasChatPanel({
for (let shotIndex = 1; shotIndex <= totalShots; shotIndex += 1) {
const shotData = payload.shots?.find((item) => Number(item.shotIndex) === shotIndex);
const col = (shotIndex - 1) % 3;
const row = Math.floor((shotIndex - 1) / 3);
const col = (shotIndex - 1) % layoutCols;
const row = Math.floor((shotIndex - 1) / layoutCols);
const nodeId = addNode({
type: "video",
type: "image",
position: {
x: startX + col * (cardWidth + gapX),
y: startY + row * (cardHeight + gapY),
},
size: { width: cardWidth, height: cardHeight },
data: {
src: "",
src: shotData?.tileImageUrl || "",
status: "generating",
title: `镜头 ${shotIndex}`,
prompt: shotData?.prompt || `分镜 ${shotIndex} 生成中`,
coverImage: shotData?.tileImageUrl || "",
originalSrc: shotData?.tileImageUrl || "",
metadata: {
storyboardTaskId: taskId,
shotIndex,
stage: "preview",
},
},
connections: [],
});
@ -3918,7 +3955,114 @@ export function CanvasChatPanel({
}
storyboardNodeMapRef.current[taskId] = shotNodeMap;
toast.message(`九宫格视频任务已开始（${totalShots} 段）`);
toast.message(`分镜预览任务已开始（${totalShots} 段）`);
return;
}
if (payload.stage === "hd_shot_completed") {
const taskId = String(payload.taskId || "").trim();
const shotIndex = Math.max(1, Number(payload.shotIndex || 0));
const imageUrl = String(payload.imageUrl || "").trim();
if (!taskId || !shotIndex || !imageUrl) return;
let nodeId = storyboardNodeMapRef.current[taskId]?.[shotIndex];
if (!nodeId) {
nodeId = addNode({
type: "image",
position: { x: 140 + ((shotIndex - 1) % 3) * 350, y: 150 + Math.floor((shotIndex - 1) / 3) * 230 },
size: { width: 320, height: 196 },
data: {
src: imageUrl,
status: "completed",
title: `镜头 ${shotIndex}`,
prompt: payload.prompt || `镜头 ${shotIndex}`,
metadata: {
storyboardTaskId: taskId,
shotIndex,
stage: "preview",
},
},
connections: [],
});
if (!storyboardNodeMapRef.current[taskId]) {
storyboardNodeMapRef.current[taskId] = {};
}
storyboardNodeMapRef.current[taskId]![shotIndex] = nodeId;
return;
}
const currentNode = useCanvasStore.getState().getNode(nodeId);
const currentData = ((currentNode?.data as any) || {}) as Record<string, any>;
updateNode(nodeId, {
data: {
...currentData,
src: imageUrl,
status: "completed",
title: currentData.title || `镜头 ${shotIndex}`,
prompt: payload.prompt || currentData.prompt || `镜头 ${shotIndex}`,
metadata: {
...(currentData.metadata || {}),
storyboardTaskId: taskId,
shotIndex,
stage: "preview",
},
},
} as any);
return;
}
if (payload.stage === "hd_shot_failed") {
const taskId = String(payload.taskId || "").trim();
const shotIndex = Math.max(1, Number(payload.shotIndex || 0));
if (!taskId || !shotIndex) return;
const nodeId = storyboardNodeMapRef.current[taskId]?.[shotIndex];
if (!nodeId) return;
const currentNode = useCanvasStore.getState().getNode(nodeId);
const currentData = ((currentNode?.data as any) || {}) as Record<string, any>;
updateNode(nodeId, {
data: {
...currentData,
status: "failed",
metadata: {
...(currentData.metadata || {}),
previewError: payload.error || "预览图生成失败",
},
},
} as any);
return;
}
if (payload.stage === "awaiting_confirmation") {
const taskId = String(payload.taskId || "").trim();
if (!taskId) return;
const previewShots = Array.isArray(payload.shots) ? payload.shots : [];
for (const shot of previewShots) {
const shotIndex = Math.max(1, Number(shot.shotIndex || 0));
const nodeId = storyboardNodeMapRef.current[taskId]?.[shotIndex];
if (!nodeId) continue;
const currentNode = useCanvasStore.getState().getNode(nodeId);
const currentData = ((currentNode?.data as any) || {}) as Record<string, any>;
updateNode(nodeId, {
data: {
...currentData,
src: String(shot.previewImageUrl || currentData.src || shot.tileImageUrl || ""),
originalSrc: String(shot.tileImageUrl || currentData.originalSrc || ""),
status: "completed",
title: currentData.title || `镜头 ${shotIndex}`,
prompt: shot.prompt || currentData.prompt || `镜头 ${shotIndex}`,
metadata: {
...(currentData.metadata || {}),
storyboardTaskId: taskId,
shotIndex,
stage: "awaiting_confirmation",
},
},
} as any);
}
toast.message(payload.message || "分镜预览已完成，请先确认镜头内容");
return;
}
@ -3951,10 +4095,18 @@ export function CanvasChatPanel({
const currentNode = useCanvasStore.getState().getNode(nodeId);
const currentData = ((currentNode?.data as any) || {}) as Record<string, any>;
updateNode(nodeId, {
type: "video",
data: {
...currentData,
src: videoUrl,
status: "completed",
prompt: currentData.prompt || `分镜 ${shotIndex}`,
coverImage: currentData.src || currentData.originalSrc || "",
metadata: {
...(currentData.metadata || {}),
storyboardTaskId: taskId,
shotIndex,
stage: "video",
},
},
} as any);
return;
@ -3980,12 +4132,12 @@ export function CanvasChatPanel({
if (payload.stage === "completed") {
const successCount = Math.max(0, Number(payload.successCount || 0));
const failedCount = Math.max(0, Number(payload.failedCount || 0));
toast.success(`九宫格视频完成：成功 ${successCount}，失败 ${failedCount}`);
toast.success(`分镜视频完成：成功 ${successCount}，失败 ${failedCount}`);
return;
}
if (payload.stage === "failed") {
toast.error(payload.error || "九宫格视频任务失败");
toast.error(payload.error || "分镜视频任务失败");
}
} catch (e) {
console.error("[CanvasChat] Failed to parse data-videoStoryboard:", e);
@ -6730,6 +6882,20 @@ export function CanvasChatPanel({
`【快捷标签：${selectedSkillTag.name}】\n${selectedSkillTag.prompt}`
);
}
if (selectedAgentPreset?.type === "text" && selectedAgentPreset.slots?.length) {
const presetPrompt = selectedAgentPreset.slots
.map((slot, index) => {
const title = String(slot.title || slot.uploadLabel || `模板${index + 1}`).trim();
const promptText = String(slot.promptTemplate || "").trim();
if (!promptText) return "";
return `【智能体：${selectedAgentPreset.name}｜${title}】\n${promptText}`;
})
.filter(Boolean)
.join("\n\n");
if (presetPrompt) {
hiddenSegments.push(presetPrompt);
}
}
if (pickedObjectInfos.length > 0) {
const objectsPrompt = pickedObjectInfos.join('\n');
hiddenSegments.push(
@ -7311,6 +7477,8 @@ export function CanvasChatPanel({
const hasEnoughAgentUploads = !hasAgentUploadSlots || attachments.length >= agentRequiredUploadCount;
const composerPlaceholder = isDetailGenerationMode
? "可选补充：风格、情绪氛围、禁用元素、期望生成张数（如：生成8张）"
: activeComposerMode === "agent" && selectedAgentPreset?.type === "text"
? "可上传图片做分析，也可直接输入文案需求"
: hasAgentUploadSlots
? "请先上传产品图1、产品图2、模特图，再补充说明"
: selectedSkillTag?.placeholder?.trim() ||
@ -9068,7 +9236,11 @@ export function CanvasChatPanel({
<Sparkles className="h-4 w-4" />
<span>{selectedAgentPreset.name}</span>
</div>
{hasAgentUploadSlots ? (
{selectedAgentPreset.type === "text" ? (
<div className="inline-flex items-center rounded-full bg-[#f4f5f7] px-3 py-1.5 text-xs text-[#5b6472]">
可上传图片做分析，也可直接输入文案需求
</div>
) : hasAgentUploadSlots ? (
<div className="inline-flex items-center rounded-full bg-[#f4f5f7] px-3 py-1.5 text-xs text-[#5b6472]">
请上传产品图1、产品图2、模特图
</div>

559
docs/DeerFlow调研与Agent升级建议-2026-03-12.md Normal file
View File

@ -0,0 +1,559 @@
# DeerFlow 调研与 Agent 升级建议（2026-03-12）
> 目标：在不推翻现有架构的前提下，吸收开源 `DeerFlow` 在对话理解、技能调用、上下文工程、子代理协作上的强项，提升本项目 Agent 能力。
> 适用范围：`ai-chatbot-main` 当前聊天 Agent、视频分镜 Agent、无限画布 Agent。
---
## 1. 先说结论
**建议学习 DeerFlow，但不建议直接整套迁移。**
原因：
1. DeerFlow 强项在于：
- 复杂任务拆解
- 子代理协作
- 工具/技能编排
- 人类确认与工作流状态管理
- 长期记忆与上下文工程
2. 我们当前项目已经有：
- LangGraph
- 工具注册表
- specialist/skill route
- context memory
- 画布上下文
- 视频 storyboard pipeline 骨架
3. 真正缺的不是“换框架”，而是把这些能力从“分散模块”升级成更稳定的 **统一 Agent Runtime**。
所以最合理的路线是：
- **不替换现有 Agent**
- **吸收 DeerFlow 的 runtime 设计思想**
- **补齐我们自己的 workflow state / middleware / subagent / human-in-the-loop**
---
## 2. DeerFlow 值得学习的核心能力
根据 DeerFlow 官方仓库与文档，它不是一个简单的聊天机器人，而是一个偏“super agent runtime”的框架。对我们最有价值的点主要有 6 个。
### 2.1 技能系统（Skills）
DeerFlow 把技能视为可复用的高阶能力，不只是单个 tool call。
对我们有启发的是：
- 技能应该带有清晰的职责边界
- 技能可以组合成工作流
- 技能不等于单个模型调用
这和我们当前“九宫格检测 / 分镜映射 / 视频调度”其实天然契合。
### 2.2 子代理系统（Subagents）
DeerFlow 强调把复杂任务拆给不同子代理，而不是一个主代理包办所有规划与执行。
这对我们尤其重要，因为我们现在的任务已经明显分裂成多个 domain：
- 普通对话
- 看图分析
- 修图
- 生图
- 生视频
- 画布发布
- 分镜任务
再往后还有：
- 多宫格分镜图生成
- 高清镜头图重生
- 用户确认管理
- 视频镜头调度
这些都非常适合子代理化。
### 2.3 Middleware 机制
DeerFlow 2.0 文档里一个很重要的方向，是把“状态处理、确认、内存更新、节点前后处理”抽成 middleware，而不是全写在单个 graph 节点里。
这正是我们当前最缺的部分。
我们现在虽然有：
- `smart-analyzer`
- `skill-router`
- `specialists`
- `auto-agent-graph`
但缺少一个统一的 middleware 层来做：
- 目标图绑定
- 任务确认门
- 安全/策略过滤
- 记忆更新
- 结果发布策略
### 2.4 明确的 Workflow State
DeerFlow 明显在往“显式工作流状态”方向演进，而不是只靠 prompt 和记忆里的隐式语义。
这对我们的视频分镜任务尤其关键。
比如我们现在正在设计的多宫格视频流程，天然就应该有：
- `draft`
- `slicing`
- `hd_image_generating`
- `awaiting_user_confirmation`
- `confirmed`
- `video_generating`
- `completed`
这类状态不应该只存在数据库里，也应该存在 Agent Runtime 的显式状态里。
### 2.5 Human-in-the-loop
DeerFlow 支持人在流程里介入确认，这一点和我们当前需求高度一致。
我们后面做多宫格分镜视频时，最重要的一条就是：
- 用户没确认，不允许开始生视频
如果没有“人类确认是 runtime 的一等公民”这个理念，后续会很容易又退回“模型自己判断继续执行”的老路。
### 2.6 Long-term Memory / Context Engineering
DeerFlow 已经把长期记忆和上下文工程放进核心能力里。
我们当前项目也已经开始做这件事，但还偏“局部补丁式”：
- 当前图像目标定位
- working memory
- 画布选中上下文
真正还没形成的是：
- 用户偏好长期记忆
- agent 可消费的结构化记忆摘要
- 任务过程中的 thread state 与 memory state 分层
---
## 3. 我们当前项目已经有什么
从当前代码来看，我们其实已经有了一个不错的基础，不需要从零重做。
### 3.1 当前已有能力
#### Agent Graph
- `lib/ai/auto-agent-graph.ts`
已经使用 `LangGraph`，而不是简单的 if/else 路由，这一点和 DeerFlow 的底层理念是一致的。
#### Intent Analyzer
- `lib/ai/smart-analyzer.ts`
已经在做：
- 意图识别
- prompt 优化
- 视频/图片/分析任务区分
#### Skill Route / Specialists
- `lib/ai/skill-router.ts`
- `lib/ai/specialists/index.ts`
已经有了“specialist + tool allowlist”的雏形，这可以视为 DeerFlow 风格 subagent 的前身。
#### Tool Registry
- `lib/ai/tools-registry.ts`
已经有运行时工具注册和上下文过滤能力。
#### Context Memory
- `lib/ai/context-memory/types.ts`
已经支持：
- 当前上传图
- 画布选中图
- 历史结果图
- working memory
#### 视频 Storyboard Pipeline 骨架
- `lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts`
- `lib/video-agent/task-store.ts`
已经具备：
- 切片检测
- 分镜映射
- 调度
- DB 任务状态
这说明我们不是“要不要上 Agent”的阶段，而是“Agent Runtime 要不要体系化升级”的阶段。
---
## 4. 我们当前真正缺什么
### 4.1 缺少统一的 Agent Runtime State
当前很多信息分散在：
- `analysis`
- `plan`
- `execution`
- DB task
- canvas state
- working memory
但缺少一层统一状态对象把这些东西串起来。
后果是：
- 复杂任务越多，graph 越难维护
- 确认态、重试态、部分完成态难做
- 很难把“Agent 理解能力”和“任务执行能力”解耦
### 4.2 缺少 Middleware 层
我们现在很多逻辑都直接塞在 graph planning 或 API route 里。
这会导致：
- 加一个新能力就要改很多地方
- 很难复用“确认门”“上下文整理”“结果回写策略”
- 视频、画布、详情生成无法共享行为层
### 4.3 缺少真正的 Subagent Delegation
现在我们虽然有 specialist，但更多像“专家路由 + 工具白名单”，还不是完整子代理。
完整子代理至少应该具备：
- 自己的职责边界
- 自己的输入/输出结构
- 可以被主代理调用
- 可以独立测试
### 4.4 缺少 Skill Package 化
我们现在的“skill”更像是代码模块，不像 DeerFlow 那样是可以被稳定调度和演进的能力包。
当前问题：
- 技能定义分散
- 调用方式不统一
- 很难做版本化和可观测性
### 4.5 缺少强约束的人类确认机制
我们现在可以“提示用户确认”，但还没有把“确认门”上升为系统级策略。
这会导致：
- 一旦某个链路忘记拦截，就可能直接执行
- 视频、多宫格、批量任务这些高成本流程风险很高
---
## 5. 建议吸收 DeerFlow 的方式
不做框架替换，做 **能力吸收式升级**。
### 5.1 第一层：统一 Agent Runtime
建议新增：
- `lib/ai/runtime/agent-thread-state.ts`
- `lib/ai/runtime/agent-workflow-state.ts`
- `lib/ai/runtime/agent-context-builder.ts`
职责：
- 管理单轮对话状态
- 管理任务工作流状态
- 统一拼装上下文、记忆、画布、任务状态
### 5.2 第二层：Middleware 体系
建议新增：
- `lib/ai/runtime/middlewares/intent-guard.ts`
- `lib/ai/runtime/middlewares/target-binding.ts`
- `lib/ai/runtime/middlewares/human-confirmation-gate.ts`
- `lib/ai/runtime/middlewares/memory-update.ts`
- `lib/ai/runtime/middlewares/publish-policy.ts`
这样后续像“多宫格确认门”就不是写死在某个 route 里，而是一个通用中间件。
### 5.3 第三层：Subagent 体系
建议把现有 specialist 升级成真正可调用的 subagent：
- `conversation-subagent`
- `vision-subagent`
- `image-edit-subagent`
- `image-generate-subagent`
- `video-subagent`
- `canvas-subagent`
- `storyboard-subagent`
新增最重要的两个：
- `storyboard-subagent`
- `confirmation-subagent`
前者负责：
- 宫格规划
- 镜头归一化
- 高清镜头图规划
后者负责：
- 输出确认文案
- 识别用户确认/修改意图
- 阻止未确认执行
### 5.4 第四层：Skill Package 化
建议把“技能”升级为可注册能力包，而不是只保留 TS 函数。
可以采用轻量形式：
- `lib/agent-skills/<skill-id>/index.ts`
- `lib/agent-skills/<skill-id>/manifest.ts`
`manifest.ts` 建议包含：
- `id`
- `name`
- `version`
- `domain`
- `inputSchema`
- `outputSchema`
- `requiresHumanConfirmation`
- `producesAssets`
- `canPublishToCanvas`
优先 package 化的技能：
1. `storyboard-grid-detector`
2. `storyboard-script-normalizer`
3. `storyboard-hd-redraw`
4. `storyboard-confirmation-gate`
5. `storyboard-video-sequencer`
### 5.5 第五层：长期记忆增强
建议新增：
- `lib/ai/memory/user-preference-memory.ts`
- `lib/ai/memory/task-memory.ts`
分开存两类内容：
1. **用户长期偏好**
- 喜欢哪种风格
- 常用比例
- 常用模型
- 是否偏好先确认再执行
2. **任务过程记忆**
- 当前宫格类型
- 当前镜头草案
- 已确认镜头
- 待重生镜头
这样才能让 Agent 既懂“这个用户平时怎么用”，又懂“这次任务现在到了哪一步”。
---
## 6. 对多宫格视频需求的直接帮助
DeerFlow 思路对我们当前多宫格视频需求最直接的帮助，不是“更会聊天”，而是让整个流程更稳定。
### 6.1 生成多宫格图前
由 `storyboard-subagent`：
- 判断该用 4/6/9 宫格
- 自动追加标准宫格 prompt
- 输出结构化镜头规划
### 6.2 切片与高清图阶段
由 skill chain：
- `grid-detector-skill`
- `shot-mapper-skill`
- `hd-redraw-skill`
每个技能都有固定输入输出，不混在一个大函数里。
### 6.3 用户确认阶段
由 `human-confirmation-gate` middleware：
- 拦住视频执行
- 给用户输出可修改镜头文案
- 监听“确认开始生成视频”之类确认语义
### 6.4 视频执行阶段
由 `video-subagent + sequencer-skill`：
- 按确认后的镜头串行执行
- 支持失败重试
- 支持只重跑单镜头
---
## 7. 不建议直接照搬 DeerFlow 的地方
### 7.1 不建议整套替换 UI / 交互模型
DeerFlow 更偏通用 super agent / deep research 形态。
我们当前项目是：
- 多模态创作
- 无限画布
- 图片/视频生成
- 强任务流
所以 UI、消息结构、画布联动、媒体资产回流这些都不能直接照搬。
### 7.2 不建议立刻重构成完全独立 repo 架构
我们当前代码已经深度绑定：
- `Next.js route handlers`
- `Vercel AI SDK`
- `LangGraph`
- `canvas` 状态
- `DB task store`
直接大迁移只会让功能倒退。
### 7.3 不建议先做“通用超级代理”，再回头补产品链路
当前最正确顺序应该是：
1. 先用 DeerFlow 思路补强我们的产品主链路
2. 再抽象出通用 Agent Runtime
而不是反过来。
---
## 8. 推荐升级路线
### Phase 0：最小升级
目标：不换架构，只补关键 runtime 能力
- [ ] 增加 workflow state
- [ ] 增加 confirmation middleware
- [ ] 增加 storyboard-subagent
- [ ] 增加 skill manifest 机制
### Phase 1：多宫格视频先吃到红利
- [ ] 用 subagent 重构多宫格分镜理解
- [ ] 用 middleware 实现“未确认不得执行”
- [ ] 用 skill package 管理分镜检测/切片/高清图/视频调度
### Phase 2：扩到全局 Agent
- [ ] 图片编辑
- [ ] 生图
- [ ] 视频
- [ ] 画布
- [ ] AI 详情
统一切到同一个 Agent Runtime 层。
### Phase 3：长期记忆与偏好学习
- [ ] 用户偏好长期记忆
- [ ] 任务级 thread state
- [ ] 记忆摘要和召回策略
---
## 9. 我建议现在立刻做的事情
如果目标是“先把多宫格视频方案做好”，我建议不要先做大重构，而是先做 4 个高价值改造：
### 9.1 新增 `human-confirmation-gate`
这是当前收益最大的一步。
直接服务于：
- 多宫格分镜视频
- 高成本视频任务
- 批量图片任务
### 9.2 新增 `storyboard-subagent`
把下面几件事从当前大 graph 里拆出去：
- 宫格数量判断
- 分镜归一
- 标准宫格 prompt 注入
- 高清镜头图阶段规划
### 9.3 技能 manifest 化
先从视频分镜链路做试点：
- `grid-detector`
- `script-normalizer`
- `hd-redraw`
- `video-sequencer`
### 9.4 Thread State 显式化
至少先给视频任务做 thread state：
- 当前 gridType
- 当前 shots
- 当前 confirmationStatus
- 当前 hd image progress
- 当前 video progress
这一步做完后，后续所有复杂 Agent 流程都会更容易。
---
## 10. 最终判断
DeerFlow 值得认真研究，原因不是它“更强”，而是它已经把很多 super agent 必须补的工程层问题想清楚了：
- workflow state
- middleware
- subagent
- skills
- long-term memory
- human-in-the-loop
这些正好都是我们现在开始碰到的瓶颈。
所以建议是：
- **理念全面学习**
- **实现局部吸收**
- **架构不整套迁移**
- **先用在多宫格分镜视频链路上试点**
如果这条链路跑通，再把同样的 runtime 能力推广到图片编辑、生图、画布和 AI 详情。

436
docs/多宫格分镜视频开发清单-2026-03-12.md Normal file
View File

@ -0,0 +1,436 @@
# 多宫格分镜视频开发清单（2026-03-12）
> 面向开发落地
> 范围：`4宫格 / 6宫格 / 9宫格` 标准切片、高清镜头图阶段、用户确认后再生视频
> 对应需求文档：`docs/多宫格分镜视频需求补充-2026-03-12.md`
---
## 0. 开发目标
本轮不是“继续优化九宫格视频”，而是把整条链路正式升级为：
`Agent 规划多宫格 -> 标准分镜图生成 -> 自动识别切片 -> 每片生成高清镜头图 -> 用户确认镜头文案 -> 串行生成视频 -> 回写数据库与画布`
建议拆成四大模块并行推进：
1. `Agent`
2. `数据库`
3. `API`
4. `画布交互`
建议开发顺序：
1. 先完成 `数据库`
2. 再完成 `Agent`
3. 然后完成 `API`
4. 最后接 `画布交互`
原因：
- 数据结构要先定，否则前后端字段会反复改
- Agent 规则要先稳定，否则后端接口难以确定输入输出
- API 要在状态机清晰后再落
- 画布是最依赖整体状态的最后一层
---
## 1. Agent 开发清单
目标：让 Agent 不只是“识别九宫格”，而是能稳定产出 **可切片的标准多宫格图**，并在视频前承担“分镜确认门”的职责。
### 1.1 意图与宫格数判断
- [ ] 在 `lib/ai/smart-analyzer.ts` 增加多宫格分镜识别词：
- `四宫格`
- `六宫格`
- `九宫格`
- `分镜拼图`
- `分镜图`
- `镜头图`
- [ ] 在 `lib/ai/auto-agent-graph.ts` 增加 `storyboardGridType` 推断字段
- [ ] 默认规则：
- 未指定镜头数但目标是视频分镜时，默认建议 `9宫格`
- 用户明确说 4 段时，使用 `grid_2x2`
- 用户明确说 6 段时，使用 `grid_3x2`
- 用户明确说 9 段时，使用 `grid_3x3`
### 1.2 标准分割提示词注入
- [ ] 在 Agent 提示词优化链路里增加“宫格结构化约束注入”
- [ ] 为 `grid_2x2 / grid_3x2 / grid_3x3` 分别维护固定模板
- [ ] 明确禁止跨格、透视倾斜、拼贴边框、编号文字、水印、字幕条
- [ ] 对用户透明提示一句：
- “我会按标准宫格版式生成，方便系统自动切片”
建议落点：
- `lib/ai/auto-agent-graph.ts`
- `lib/ai/smart-analyzer.ts`
- 如需抽离模板：新增 `lib/video-agent/grid-prompt-templates.ts`
### 1.3 分镜文案归一化
- [ ] 现有 `script-normalizer-skill` 从固定 9 段改为“按 gridType 输出 N 段”
- [ ] 规范输出字段：
- `gridType`
- `rows`
- `cols`
- `shots`
- `confirmationRequired`
- [ ] 当镜头不足时，只生成草案，不自动补全并执行视频
- [ ] 当用户后续改某一镜头时，仅更新对应镜头
建议落点：
- `lib/video-agent/skills/script-normalizer.ts`
- `lib/video-agent/types.ts`
### 1.4 用户确认门
- [ ] 增加“确认态回复模板”
- [ ] Agent 输出镜头预览时必须明确提示：
- 可修改任意镜头
- 回复“确认开始生成视频”才会真正开始视频
- [ ] 在确认前，Agent 只允许执行：
- 修改镜头文案
- 重生单张高清镜头图
- 重新整理宫格布局
- [ ] 在确认后，才允许发起视频任务调度
建议落点：
- `lib/ai/auto-agent-graph.ts`
- `app/(chat)/api/chat/route.ts`
### 1.5 Agent 模块验收
- [ ] 能正确判断 4/6/9 宫格
- [ ] 生图 prompt 中已自动附加标准结构约束
- [ ] 分镜输出镜头数与宫格数一致
- [ ] 未确认前不会触发视频生成
---
## 2. 数据库开发清单
目标：让“切片图 / 高清镜头图 / 视频结果 / 用户确认状态”全部可持久化、可恢复、可追踪。
### 2.1 主任务表 `VideoStoryboardTask`
当前文件：
- `lib/db/schema.ts`
- `lib/db/migrations/0048_add_video_storyboard_tasks.sql`
需要新增字段：
- [ ] `gridType`：`grid_2x2 | grid_3x2 | grid_3x3`
- [ ] `gridRows`
- [ ] `gridCols`
- [ ] `confirmationStatus`：`draft | awaiting_user_confirmation | confirmed | rejected`
- [ ] `confirmedAt`
- [ ] `confirmedByMessageId`
- [ ] `hdImageStageStatus`：`pending | running | completed | failed`
### 2.2 子任务表 `VideoStoryboardShotTask`
需要新增字段：
- [ ] `sourceTileAssetId`
- [ ] `hdImageUrl`
- [ ] `hdImageAssetId`
- [ ] `hdImagePrompt`
- [ ] `hdImageStatus`：`pending | running | completed | failed`
- [ ] `videoStatus`：`pending | waiting_confirmation | running | completed | failed`
- [ ] `userApproved`
- [ ] `userApprovedAt`
- [ ] `displayOrder`
### 2.3 统一资产挂接
如果要和当前资产图谱方案一致，需要补以下资产类型：
- [ ] `storyboard_tile`
- [ ] `storyboard_hd_image`
- [ ] `storyboard_video`
并保证都能反查：
- `taskId`
- `shotIndex`
- `chatId`
- `canvasNodeId`
建议落点：
- `lib/db/schema.ts`
- `docs/上下文长期记忆-图片目标定位-资产图谱表结构设计.md`
- 如果已有资产表实现，补充对应写入逻辑
### 2.4 查询与状态同步
- [ ] 更新 `lib/video-agent/task-store.ts`
- [ ] 新增高清图阶段的状态更新方法：
- `markVideoStoryboardShotHdRunning`
- `markVideoStoryboardShotHdCompleted`
- `markVideoStoryboardShotHdFailed`
- [ ] 新增确认状态方法：
- `markVideoStoryboardAwaitingConfirmation`
- `markVideoStoryboardConfirmed`
- `markVideoStoryboardRejected`
- [ ] `syncVideoStoryboardTaskStatus` 要能区分：
- 高清图阶段
- 等待确认阶段
- 视频阶段
### 2.5 数据库模块验收
- [ ] 一个任务能同时追踪切片图、高清镜头图、视频结果
- [ ] 系统重启后还能恢复等待确认中的任务
- [ ] 单镜头重生不会破坏主任务顺序
---
## 3. API 开发清单
目标：把“检测切片、高清镜头图生成、确认放行、视频调度”拆成清晰接口，而不是全部压在一个聊天接口里。
### 3.1 配置结构扩展
当前文件：
- `lib/video-agent/config.ts`
需要新增配置：
- [ ] `supportedGridTypes`
- [ ] `enableHdImageStage`
- [ ] `hdImageModelWhitelist`
- [ ] `hdImageConcurrency`
- [ ] `confirmationTimeoutHours`
- [ ] `requireUserConfirmationBeforeVideo`，默认 `true`
### 3.2 新增/改造内部 pipeline
当前骨架：
- `lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts`
需要改造为更通用的多宫格 pipeline：
- [ ] 输入不再只接受 9 宫格
- [ ] 输出包含 `gridType / totalShots / hdImages`
- [ ] 先做切片，再做高清镜头图，不直接串行发视频
- [ ] 任务结束点从“视频全部完成”改为“进入待确认态”
建议：
- 保留原文件，内部升级
- 或新增 `lib/video-agent/pipeline/storyboard-grid-pipeline.ts`
### 3.3 新增确认接口
建议新增：
- [ ] `POST /api/video/storyboard/tasks/:taskId/confirm`
- [ ] `POST /api/video/storyboard/tasks/:taskId/reject`
- [ ] `PATCH /api/video/storyboard/tasks/:taskId/shots/:shotIndex`
- [ ] `POST /api/video/storyboard/tasks/:taskId/shots/:shotIndex/regenerate-hd`
职责：
- `confirm`：把任务状态切到 `confirmed`
- `reject`：退回 `draft` 或 `rejected`
- `PATCH shot`：更新单镜头文案
- `regenerate-hd`：重生单张高清镜头图
### 3.4 视频调度接口
已有方向：
- admin 查询：`app/api/admin/video-storyboard-tasks/...`
- 重跑：`app/api/admin/video-storyboard-tasks/[taskId]/rerun/route.ts`
需要补充正式业务接口：
- [ ] `POST /api/jobs/video-storyboard-dispatch`
- 只处理 `confirmationStatus=confirmed` 的任务
- [ ] 调度器读取待执行镜头时，跳过未确认任务
- [ ] 增加“只重跑失败视频镜头”的业务接口
### 3.5 聊天主入口联动
当前聊天入口：
- `app/(chat)/api/chat/route.ts`
这里需要做的不是“直接生成视频”，而是分流：
- [ ] 命中多宫格 storyboard 时，走多宫格 pipeline
- [ ] pipeline 完成高清图阶段后，把确认消息回写到聊天流
- [ ] 用户确认语义命中后，再调用 confirm 接口或内部 confirm 逻辑
### 3.6 API 模块验收
- [ ] 创建任务后能进入 `awaiting_user_confirmation`
- [ ] 用户确认接口触发后才会进入视频调度
- [ ] 单镜头修改和重生高清图都能单独调用
---
## 4. 画布交互开发清单
目标：用户在画布里先看到“有顺序的高清镜头图”，确认后再看到视频结果，整个过程不能乱序。
### 4.1 画布节点阶段化展示
当前相关文件：
- `components/canvas/CanvasChatPanel.tsx`
- `store/canvas/index.ts`
- `types/canvas/index.ts`
当前已有九宫格视频的画布处理偏向直接视频节点，需要改成两阶段：
- [ ] 阶段 A：高清镜头图阶段，创建 `image` 占位节点
- [ ] 阶段 B：视频生成阶段，更新为 `video` 节点或追加视频节点
### 4.2 新增流式事件类型
建议扩展现有 `data-videoStoryboard` 事件：
- [ ] `hd_start`
- [ ] `hd_shot_completed`
- [ ] `hd_shot_failed`
- [ ] `awaiting_confirmation`
- [ ] `confirmation_received`
- [ ] `video_start`
- [ ] `video_shot_completed`
- [ ] `video_shot_failed`
当前 `CanvasChatPanel` 里已有：
- `start`
- `shot_completed`
- `shot_failed`
- `completed`
需要按“高清图阶段”和“视频阶段”拆开。
### 4.3 画布布局规则
- [ ] 4宫格时按 `2x2` 落位
- [ ] 6宫格时按 `3x2` 落位
- [ ] 9宫格时按 `3x3` 落位
- [ ] 高清镜头图和视频节点都保持同一镜头顺序
- [ ] 节点标题要显示：
- `镜头 1`
- `镜头 2`
- `镜头 3`
### 4.4 用户确认交互
建议增加一块轻量确认 UI：
- [ ] 在聊天消息里展示镜头草案确认卡
- [ ] 每个镜头支持：
- 编辑文案
- 重新生成高清图
- [ ] 提供总按钮：
- `确认开始生成视频`
- `返回修改`
如不做复杂卡片，至少要：
- [ ] 在聊天流里清晰列出镜头 1~N 文案
- [ ] 用户回复确认前，画布只显示高清图，不显示视频进度
### 4.5 画布持久化
- [ ] 高清镜头图节点写入当前 `canvasProject.canvasData`
- [ ] 视频结果节点与镜头图保持可追溯关系
- [ ] 刷新页面后仍能恢复：
- 哪些镜头已生成高清图
- 当前是否在等待确认
- 哪些镜头视频已开始/完成
### 4.6 画布模块验收
- [ ] 画布能先显示镜头静帧，再显示视频
- [ ] 顺序始终稳定，不乱插
- [ ] 用户修改单镜头后，只更新对应节点
---
## 5. 推荐实施阶段
### Phase A：状态机和数据结构
- [ ] 扩表
- [ ] 更新 `task-store`
- [ ] 扩 `types`
### Phase B：多宫格检测与高清镜头图阶段
- [ ] 扩 `nine-grid-detector` 为多宫格 detector
- [ ] 增加 `tile-hd-redraw-skill`
- [ ] pipeline 停在 `awaiting_user_confirmation`
### Phase C：用户确认门
- [ ] 增加 confirm/reject/update-shot API
- [ ] 增加 Agent 确认模板
- [ ] 增加聊天侧确认逻辑
### Phase D：视频调度与画布联动
- [ ] 仅对 confirmed 任务发视频
- [ ] 画布上分阶段展示高清图和视频
- [ ] 单镜头修订和重跑
---
## 6. 重点文件清单
### Agent
- `lib/ai/smart-analyzer.ts`
- `lib/ai/auto-agent-graph.ts`
- `app/(chat)/api/chat/route.ts`
### 数据库
- `lib/db/schema.ts`
- `lib/db/migrations/0048_add_video_storyboard_tasks.sql`
- `lib/video-agent/task-store.ts`
### API / Pipeline
- `lib/video-agent/config.ts`
- `lib/video-agent/types.ts`
- `lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts`
- `lib/video-agent/skills/nine-grid-detector.ts`
- `lib/video-agent/skills/script-normalizer.ts`
- `lib/video-agent/skills/storyboard-mapper.ts`
- `lib/video-agent/skills/video-sequencer.ts`
### 画布交互
- `components/canvas/CanvasChatPanel.tsx`
- `store/canvas/index.ts`
- `types/canvas/index.ts`
---
## 7. 本轮建议先做什么
如果要最快推进，我建议先开这 5 个子任务：
1. 扩 `schema + task-store`，把 `gridType / confirmationStatus / hdImage*` 字段补齐
2. 扩 `types + detector`，把 9 宫格改成通用多宫格
3. 新增 `tile-hd-redraw-skill`
4. 让 pipeline 停在 `awaiting_user_confirmation`
5. 给聊天流加 `confirm` 放行逻辑
这样做完后，即使视频阶段还没完全改好，用户也已经能看到“多宫格切片 -> 高清镜头图 -> 确认”的完整闭环。

382
docs/多宫格分镜视频需求补充-2026-03-12.md Normal file
View File

@ -0,0 +1,382 @@
# 多宫格分镜视频需求补充（2026-03-12）
> 适用范围：`RUXA 视频` / 画布分镜视频工作流 / Agent 自动规划
> 本文是对现有“九宫格视频 Agent 技能链”方案的补充，重点新增：
> 1. 支持 `4宫格 / 6宫格 / 9宫格` 标准化切片
> 2. 切片后先自动生成高清大图，并按顺序入库、回显到画布
> 3. 必须经过用户确认分镜文案后，才允许开始调用生视频模型
>
> 开发拆解清单见：`docs/多宫格分镜视频开发清单-2026-03-12.md`
> Agent 升级调研见：`docs/DeerFlow调研与Agent升级建议-2026-03-12.md`
---
## 1. 需求目标
我们不再只支持“识别九宫格再直接生视频”，而是升级为统一的 **多宫格分镜图 -> 标准切片 -> 高清镜头图 -> 用户确认 -> 分镜视频** 流程。
核心目标：
1. Agent 在前置生图阶段就强制加入“标准分割版式提示词”，提高后续切片识别成功率
2. 支持标准 `2x2 / 3x2 / 3x3` 三种宫格版式
3. 切片后每个分镜先生成一张大尺寸高清静帧图，作为“视频前置确认素材”
4. 高清静帧图必须按顺序写入数据库、同步显示到画布
5. Agent 必须和用户逐段确认分镜文案；未确认前，禁止调用视频模型
---
## 2. 标准宫格类型定义
### 2.1 标准版式
系统当前只认以下三种标准版式：
| 业务名称 | 标准版式 | 行列 | 顺序 |
| --- | --- | --- | --- |
| 4宫格 | `grid_2x2` | 2 行 2 列 | 从左到右、从上到下 |
| 6宫格 | `grid_3x2` | 2 行 3 列 | 从左到右、从上到下 |
| 9宫格 | `grid_3x3` | 3 行 3 列 | 从左到右、从上到下 |
约束：
- `6宫格` 当前统一采用 `3列 x 2行`
- 不支持自由拼贴、不规则分栏、跨格大图、瀑布流拼图
- 如用户上传的是非标准拼图，Agent 需提示“当前只支持 2x2 / 3x2 / 3x3 标准版式”
### 2.2 推荐输出尺寸
| 版式 | 推荐画布比例 | 推荐最小边长 |
| --- | --- | --- |
| `grid_2x2` | 1:1 | 2048 |
| `grid_3x2` | 3:2 | 2304 |
| `grid_3x3` | 1:1 | 2304 |
说明：
- 目标不是让用户记参数，而是 Agent 自动把这些结构要求注入到底层生图提示词
- 若底层模型不稳定，优先保证“分割清晰可识别”，其次再追求艺术表现
---
## 3. Agent 必须追加的结构化提示词
## 3.1 总体规则
当 Agent 需要让模型生成“可切片的多宫格分镜图”时，必须自动追加结构化约束，不允许只使用自然语言脚本直接出图。
必须追加的约束包括：
- 严格等分网格
- 固定外边距
- 固定分隔线宽度
- 所有主体必须完整落在单一格内
- 主体、光效、文字、特效都不得跨越分隔线
- 禁止倾斜透视、胶片边框、拼贴阴影、圆角相框、手撕纸边、重叠排版
- 禁止编号文字、水印、Logo、字幕条覆盖在格线附近
## 3.2 标准版式提示词模板
### `grid_2x2`
Agent 需要在最终生图 prompt 末尾追加：
`严格 2x2 分镜拼图，2行2列，所有格子等宽等高，纯净清晰的统一分隔线，固定外边距，画面按从左到右从上到下阅读。每个主体必须完整处于单独格内，不得跨格，不得遮挡边界，不得透视倾斜，不得拼贴重叠，不得出现文字编号、水印、胶片边框。`
### `grid_3x2`
Agent 需要在最终生图 prompt 末尾追加：
`严格 3x2 分镜拼图，3列2行，所有格子等宽等高，纯净清晰的统一分隔线，固定外边距，画面按从左到右从上到下阅读。每个主体必须完整处于单独格内，不得跨格，不得遮挡边界，不得透视倾斜，不得拼贴重叠，不得出现文字编号、水印、胶片边框。`
### `grid_3x3`
Agent 需要在最终生图 prompt 末尾追加：
`严格 3x3 分镜拼图，3行3列，所有格子等宽等高，纯净清晰的统一分隔线，固定外边距，画面按从左到右从上到下阅读。每个主体必须完整处于单独格内，不得跨格，不得遮挡边界，不得透视倾斜，不得拼贴重叠，不得出现文字编号、水印、胶片边框。`
## 3.3 Agent 对用户的引导要求
如果用户说“帮我做九宫格/六宫格/四宫格分镜图”，Agent 需要主动补一句简短说明：
- 系统会按标准宫格版式生成，便于后续自动切片
- 如需继续做分镜视频，建议先确认每格对应一个镜头
如果用户只说“做一个分镜拼图”，Agent 默认策略：
1. 优先判断是否要做视频分镜
2. 若是视频分镜且镜头数未知，默认先建议 `9宫格`
3. 若用户明确说“只要 4 段 / 6 段”，分别切到 `4宫格 / 6宫格`
---
## 4. 多宫格识别与切片规则
## 4.1 检测输出结构
检测技能输出统一扩展为：
```json
{
"isStoryboardGrid": true,
"gridType": "grid_3x3",
"rows": 3,
"cols": 3,
"confidence": 0.93,
"cropBoxes": [
{ "index": 1, "row": 1, "col": 1, "x": 0, "y": 0, "w": 640, "h": 640 }
],
"qualityScore": 0.9
}
```
## 4.2 切片顺序
统一采用“阅读顺序”：
1. 先行后列
2. 每行从左到右
3. 再进入下一行
即：
- `grid_2x2`：1-2 / 3-4
- `grid_3x2`：1-2-3 / 4-5-6
- `grid_3x3`：1-2-3 / 4-5-6 / 7-8-9
## 4.3 识别失败时的退路
- 置信度不足：Agent 提示用户改用标准宫格版式重新生成
- 分隔线不清晰：Agent 提示用户“重新按标准宫格输出”
- 非标准布局：不进入后续高清图/视频链路
---
## 5. 切片后先生成高清镜头图
## 5.1 新增强制步骤
切片完成后，不能直接开始生视频，必须先进入 **高清镜头图生成阶段**：
1. 每个切片图作为一张独立输入图
2. 调用生图模型或图像增强模型，生成大尺寸高清镜头图
3. 高清图按切片顺序写入数据库
4. 高清图按切片顺序回显到画布
5. Agent 用这些高清图和对应文案向用户做分镜确认
## 5.2 目标能力
高清镜头图阶段要解决两个问题：
1. 原始切片图尺寸过小，不适合直接做视频首帧
2. 用户需要在正式生视频前先看到“每镜头的大图预览”
## 5.3 推荐技能命名
建议新增：
- `tile-hd-redraw-skill`
职责：
- 输入：`tileImage + globalStyle + shotPrompt`
- 输出：`hdImageUrl + hdPrompt + seed/metadata`
- 支持失败重试
- 支持只重生单个镜头
## 5.4 生成顺序
高清镜头图也要按顺序执行：
- 可串行，也可小并发（最多 2~3）
- 画布上按照镜头顺序落位
- 未完成的镜头显示占位卡
---
## 6. 数据库存储要求
## 6.1 主任务表新增字段建议
`VideoStoryboardTask` 建议增加：
- `gridType`：`grid_2x2 | grid_3x2 | grid_3x3`
- `gridRows`
- `gridCols`
- `confirmationStatus`：`draft | awaiting_user_confirmation | confirmed | rejected`
- `confirmedAt`
- `confirmedByMessageId`
## 6.2 子任务表新增字段建议
`VideoStoryboardShotTask` 建议增加：
- `sourceTileAssetId`
- `hdImageUrl`
- `hdImageAssetId`
- `hdImageStatus`：`pending | running | completed | failed`
- `videoStatus`：`pending | waiting_confirmation | running | completed | failed`
- `userApproved`：boolean
- `userApprovedAt`
## 6.3 资产类型建议
新增或明确以下资产类型：
- `storyboard_tile`：切片原图
- `storyboard_hd_image`：高清镜头图
- `storyboard_video`：最终视频结果
要求：
- 以上三类资产都必须可追溯到 `parentTaskId + shotIndex`
- 都要能在画布节点和历史记录里反查
---
## 7. 画布展示规则
## 7.1 画布先展示高清镜头图
切片完成后，画布不应直接只显示视频占位卡，而应分两阶段：
### 阶段 A：高清镜头图阶段
- 每个镜头创建一个 `image` 节点占位
- 高清图生成成功后替换占位图
- 节点标题显示：`镜头 1 / 镜头 2 / 镜头 3 ...`
### 阶段 B：视频阶段
- 用户确认后，再在对应镜头位置创建或替换为 `video` 节点
- 保留高清图与视频结果的关联关系
## 7.2 顺序展示要求
- 永远按切片顺序落位
- 不允许哪个镜头先完成就乱序插入
- 画布上应让用户一眼看出“镜头顺序”
---
## 8. 用户确认门机制（强制）
## 8.1 硬性规则
**没有用户确认，不允许开始生视频。**
这条规则必须在 Agent、后端任务状态机、调度器三层同时生效。
## 8.2 确认前流程
1. Agent 识别宫格类型
2. 切片
3. 为每个切片生成高清镜头图
4. 归一出镜头文案
5. 将“镜头图 + 镜头文案”同步给用户确认
6. 任务状态进入 `awaiting_user_confirmation`
## 8.3 用户确认文案格式
Agent 需要给用户清晰列出镜头草案，例如：
```text
我已经按 9 段分镜整理好了，并生成了对应高清镜头预览。
镜头1：……
镜头2：……
镜头3：……
如果需要修改，你可以直接回复：
- 修改第3镜头：……
- 修改第5镜头动作：……
如果确认无误，请回复：
“确认开始生成视频”
```
## 8.4 确认后的唯一放行口令
系统只在识别到明确确认语义后放行，例如：
- `确认开始生成视频`
- `开始生成`
- `按这个分镜生成`
如果用户说的是修改意见，则只能更新分镜草案，不得偷偷开始视频任务。
## 8.5 修改后的处理
如果用户修改第 N 镜头：
1. 只重算该镜头文案映射
2. 必要时只重生该镜头高清图
3. 更新画布对应静帧
4. 重新回到 `awaiting_user_confirmation`
---
## 9. Agent 的新增职责
Agent 在这条链路中的职责不再只是“识别意图 + 优化 prompt”，还必须承担：
1. 判断应该使用 `4 / 6 / 9` 宫格中的哪一种
2. 自动追加标准版式 prompt 约束
3. 将用户自然语言脚本归一成与宫格数量一致的镜头数
4. 在高清镜头图生成完成后，负责向用户解释和确认
5. 在用户未确认前，阻止视频模型调用
### 默认镜头数规则
建议默认规则：
- `4宫格 -> 4段分镜`
- `6宫格 -> 6段分镜`
- `9宫格 -> 9段分镜`
如果业务强行要求“6宫格或4宫格也统一做成9段视频”，那不是当前默认流程，必须额外定义：
- 缺失镜头如何补齐
- 是否允许复用静帧
- 是否允许 Agent 自动脑补新镜头
当前阶段不建议自动补到 9 段后直接执行，必须先经用户明确确认。
---
## 10. 推荐状态机
建议任务主状态改为：
`draft -> slicing -> hd_image_generating -> awaiting_user_confirmation -> approved -> video_generating -> completed`
失败态：
- `slice_failed`
- `hd_image_failed`
- `confirmation_timeout`
- `video_partial_failed`
- `video_failed`
---
## 11. 验收标准（更新）
新增验收标准：
1. `4宫格 / 6宫格 / 9宫格` 都能稳定识别并按标准顺序切片
2. 每个切片都能产出一张高清镜头图，并成功写库
3. 高清镜头图会按顺序显示到画布
4. 用户未确认前，系统不会触发任何视频模型调用
5. 用户只修改局部镜头时，系统支持单镜头重算、单镜头重生
---
## 12. 建议开发顺序
1. 先把 `gridType` 从“仅 9 宫格”扩成 `2x2 / 3x2 / 3x3`
2. 再加 Agent 结构化版式 prompt 注入
3. 再加 `tile-hd-redraw-skill`
4. 打通“高清镜头图入库 + 画布展示”
5. 最后再加“用户确认门 + 视频调度放行”
这样能把风险最大的一步放在最后，同时让用户尽早看到“可确认的高清镜头预览”。

4
docs/系统全链路开发文档-含九宫格视频Agent技能化方案.md
View File

@ -222,6 +222,10 @@ API：
## 6. 新需求：RUXA 视频九宫格 + Agent 技能化（待开发）
> 2026-03-12 补充需求：多宫格（4/6/9）标准切片、切片后先生成高清镜头图、用户确认分镜后才允许开始生视频。
> 需求详见：`docs/多宫格分镜视频需求补充-2026-03-12.md`
> 开发清单详见：`docs/多宫格分镜视频开发清单-2026-03-12.md`
## 6.1 需求归纳
当用户在视频模式选中 `RUXA 视频` 时：

47
lib/agents/secondary-type-config.ts Normal file
View File

@ -0,0 +1,47 @@
import type { AgentPresetType } from "@/types/agent";
export const AGENT_SECONDARY_TYPE_CONFIG_KEY = "agent_secondary_type_config";
export type AgentSecondaryTypeConfig = Record<AgentPresetType, string[]>;
export const defaultAgentSecondaryTypeConfig: AgentSecondaryTypeConfig = {
image: [],
video: [],
text: [],
};
export function normalizeAgentSecondaryTypeConfig(value: unknown): AgentSecondaryTypeConfig {
const raw =
value && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
const normalizeList = (input: unknown) => {
if (!Array.isArray(input)) return [] as string[];
return Array.from(
new Set(
input
.map((item) => String(item || "").trim())
.filter(Boolean)
.slice(0, 200)
)
);
};
return {
image: normalizeList(raw.image),
video: normalizeList(raw.video),
text: normalizeList(raw.text),
};
}
export function isValidSecondaryTypeForPresetType(params: {
presetType: AgentPresetType;
secondaryType?: string | null;
config: AgentSecondaryTypeConfig;
}) {
const normalized = String(params.secondaryType || "").trim();
if (!normalized) return true;
const allowed = params.config[params.presetType] || [];
return allowed.includes(normalized);
}

7
lib/ai/auto-agent-graph.ts
View File

@ -1375,6 +1375,9 @@ async function buildExecutionPlanWithAgent(
skillRoute?.planningHints?.length
? `专家约束：\n${skillRoute.planningHints.map((item, index) => `${index + 1}. ${item}`).join("\n")}`
: "",
state.threadState?.workflowState
? `当前工作流：type=${state.threadState.workflowState.type}; status=${state.threadState.workflowState.status}; summary=${compactText(state.threadState.workflowState.summary || "", 240)}`
: "",
state.context ? `画布上下文：${compactText(state.context, 600)}` : "",
historyContext ? `最近对话：\n${historyContext}` : "",
`可用工具：\n${toolCatalog}`,
@ -1401,6 +1404,7 @@ async function buildExecutionPlanWithAgent(
targetResolutionMode: state.targetResolutionMode,
mediaSource: state.mediaSource,
selectedImageCount: state.selectedImageCount,
threadState: state.threadState,
});
}
}
@ -1850,7 +1854,7 @@ const analyzeNode = async (state: AutoAgentStateType) => {
};
const confirmationGateNode = async (state: AutoAgentStateType) => {
const workflowDecision = runHumanConfirmationGate({
const workflowDecision = await runHumanConfirmationGate({
text: state.text,
threadState: state.threadState,
});
@ -1885,6 +1889,7 @@ const planNode = async (state: AutoAgentStateType) => {
targetResolutionMode: state.targetResolutionMode,
mediaSource: state.mediaSource,
selectedImageCount: state.selectedImageCount,
threadState: state.threadState,
});
const availableTools = filterToolsForSkillRoute(baseAvailableTools, skillRoute);
console.log("[Auto Agent] Skill route:", {

74
lib/ai/runtime/middlewares/human-confirmation-gate.ts
View File

@ -5,6 +5,7 @@ import {
type AgentWorkflowType,
} from "@/lib/ai/runtime/agent-workflow-state";
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
import { interpretStoryboardCommand } from "@/lib/video-agent/storyboard-command-interpreter";
export type HumanConfirmationIntent =
| "none"
@ -31,63 +32,6 @@ export interface HumanConfirmationGateDecision {
friendlyMessage?: string;
}
const CONFIRM_PATTERNS = [
/确认开始生成视频/,
/确认开始生成/,
/按这个分镜生成/,
/开始生成视频/,
/开始生成$/,
/可以开始了/,
/可以执行了/,
/就按这个/,
/没问题开始/,
/确认无误/,
];
const REVISE_PATTERNS = [
/修改第\d+镜头/,
/第\d+镜头/,
/调整第\d+镜头/,
/重生第\d+/,
/重做第\d+/,
/改一下/,
/修改/,
/调整/,
/不要这样/,
/不对/,
/不行/,
/换成/,
];
const CANCEL_PATTERNS = [
/取消/,
/先不要/,
/暂停/,
/停止/,
/先别生成/,
/不用生成了/,
];
function normalizeCompactText(text: string) {
return String(text || "").toLowerCase().replace(/\s+/g, "").trim();
}
export function detectHumanConfirmationIntent(text: string): HumanConfirmationIntent {
const normalized = normalizeCompactText(text);
if (!normalized) return "unclear";
if (CANCEL_PATTERNS.some((pattern) => pattern.test(normalized))) {
return "cancel";
}
if (REVISE_PATTERNS.some((pattern) => pattern.test(normalized))) {
return "revise";
}
if (CONFIRM_PATTERNS.some((pattern) => pattern.test(normalized))) {
return "confirm";
}
return "unclear";
}
function buildAwaitingConfirmationMessage(workflowState: AgentWorkflowState): string {
const title = workflowState.title || "当前工作流";
const summary = workflowState.summary
@ -131,10 +75,10 @@ function buildCancelMessage(workflowState: AgentWorkflowState): string {
return `已收到取消指令。${title} 会继续停留在人工确认前状态，不会继续自动执行。`;
}
export function runHumanConfirmationGate(params: {
export async function runHumanConfirmationGate(params: {
text: string;
threadState?: AgentThreadState | null;
}): HumanConfirmationGateDecision {
}): Promise<HumanConfirmationGateDecision> {
const workflowState = params.threadState?.workflowState;
if (!workflowState || !isAwaitingUserConfirmationWorkflow(workflowState)) {
return {
@ -145,7 +89,16 @@ export function runHumanConfirmationGate(params: {
};
}
const confirmationIntent = detectHumanConfirmationIntent(params.text);
const interpreted = await interpretStoryboardCommand({
text: params.text,
totalShots: workflowState.pendingItems.length || undefined,
items: workflowState.pendingItems.map((item) => ({
index: item.index || 0,
title: item.title,
summary: item.summary,
})),
});
const confirmationIntent = interpreted.intent;
const baseDecision = {
workflowId: workflowState.workflowId,
workflowType: workflowState.type,
@ -181,7 +134,6 @@ export function runHumanConfirmationGate(params: {
friendlyMessage: buildCancelMessage(workflowState),
};
case "unclear":
case "none":
default:
return {
...baseDecision,

17
lib/ai/skill-router.ts
View File

@ -6,6 +6,7 @@ import type {
AgentSpecialistContext,
AgentSpecialistRoute,
} from "@/lib/ai/specialists";
import { buildStoryboardSpecialistRoute } from "@/lib/video-agent/storyboard-subagent";
export interface AgentSkillRouteResult {
primary: AgentSpecialistRoute;
@ -210,23 +211,31 @@ export function routeAgentSkill(
? [canvasOverlay]
: [];
const storyboardOverlay = buildStoryboardSpecialistRoute({
text: context.text,
threadState: context.threadState,
});
const mergedOverlays = storyboardOverlay
? [...overlays, storyboardOverlay]
: overlays;
const toolAllowlist = dedupeToolNames([
...(primary.toolAllowlist || []),
...overlays.flatMap((item) => item.toolAllowlist || []),
...mergedOverlays.flatMap((item) => item.toolAllowlist || []),
]);
const planningHints = [
...primary.planningHints,
...overlays.flatMap((item) => item.planningHints),
...mergedOverlays.flatMap((item) => item.planningHints),
];
const reason = [primary.reason, ...overlays.map((item) => item.reason)]
const reason = [primary.reason, ...mergedOverlays.map((item) => item.reason)]
.filter(Boolean)
.join("；");
return {
primary,
overlays,
overlays: mergedOverlays,
toolAllowlist,
planningHints,
reason,

14
lib/ai/specialists/index.ts
View File

@ -1,4 +1,6 @@
import type { AnalysisResult } from "@/lib/ai/smart-analyzer";
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
import { refinePlanWithStoryboardSubagent } from "@/lib/video-agent/storyboard-subagent";
import type {
AgentExecutionPlan,
AgentPlanNextAction,
@ -18,6 +20,7 @@ export type AgentSpecialistDomain =
| "image_edit"
| "image_generate"
| "video_generate"
| "storyboard"
| "canvas";
export type AgentSpecialistId =
@ -26,6 +29,7 @@ export type AgentSpecialistId =
| "image-edit-specialist"
| "image-generate-specialist"
| "video-specialist"
| "storyboard-subagent"
| "canvas-specialist";
export interface AgentSpecialistRoute {
@ -44,6 +48,7 @@ export interface AgentSpecialistContext {
targetResolutionMode: AgentTargetResolutionMode;
mediaSource?: "current_message" | "canvas_selection" | "history" | "none";
selectedImageCount?: number;
threadState?: AgentThreadState | null;
}
export interface AgentPlanRefineInput extends AgentSpecialistContext {
@ -260,6 +265,15 @@ export function refinePlanWithSpecialist(
});
}
if (specialist.domain === "storyboard") {
return refinePlanWithStoryboardSubagent({
text,
plan,
availableTools,
threadState: input.threadState,
});
}
if (specialist.domain === "canvas") {
const isExecutionAction =
plan.nextAction === "edit_image" ||

77
lib/db/migrations/0057_add_video_storyboard_confirmation_state.sql Normal file
View File

@ -0,0 +1,77 @@
ALTER TABLE "VideoStoryboardTask"
ADD COLUMN IF NOT EXISTS "gridType" varchar(32);
--> statement-breakpoint
ALTER TABLE "VideoStoryboardTask"
ADD COLUMN IF NOT EXISTS "gridRows" integer;
--> statement-breakpoint
ALTER TABLE "VideoStoryboardTask"
ADD COLUMN IF NOT EXISTS "gridCols" integer;
--> statement-breakpoint
ALTER TABLE "VideoStoryboardTask"
ADD COLUMN IF NOT EXISTS "confirmationStatus" varchar(32) DEFAULT 'draft' NOT NULL;
--> statement-breakpoint
ALTER TABLE "VideoStoryboardTask"
ADD COLUMN IF NOT EXISTS "hdImageStageStatus" varchar(32) DEFAULT 'pending' NOT NULL;
--> statement-breakpoint
ALTER TABLE "VideoStoryboardTask"
ADD COLUMN IF NOT EXISTS "confirmedAt" timestamp;
--> statement-breakpoint
ALTER TABLE "VideoStoryboardTask"
ADD COLUMN IF NOT EXISTS "confirmedByMessageId" uuid;
--> statement-breakpoint
DO $$ BEGIN
ALTER TABLE "VideoStoryboardTask" ADD CONSTRAINT "VideoStoryboardTask_confirmedByMessageId_Message_v2_id_fk" FOREIGN KEY ("confirmedByMessageId") REFERENCES "public"."Message_v2"("id") ON DELETE set null ON UPDATE no action;
EXCEPTION
WHEN duplicate_object THEN null;
END $$;
--> statement-breakpoint
ALTER TABLE "VideoStoryboardShotTask"
ADD COLUMN IF NOT EXISTS "hdImageUrl" text;
--> statement-breakpoint
ALTER TABLE "VideoStoryboardShotTask"
ADD COLUMN IF NOT EXISTS "hdImagePrompt" text;
--> statement-breakpoint
ALTER TABLE "VideoStoryboardShotTask"
ADD COLUMN IF NOT EXISTS "hdImageStatus" varchar(32) DEFAULT 'pending' NOT NULL;
--> statement-breakpoint
ALTER TABLE "VideoStoryboardShotTask"
ADD COLUMN IF NOT EXISTS "videoStatus" varchar(32) DEFAULT 'pending' NOT NULL;
--> statement-breakpoint
ALTER TABLE "VideoStoryboardShotTask"
ADD COLUMN IF NOT EXISTS "userApproved" boolean DEFAULT false NOT NULL;
--> statement-breakpoint
ALTER TABLE "VideoStoryboardShotTask"
ADD COLUMN IF NOT EXISTS "userApprovedAt" timestamp;
--> statement-breakpoint
UPDATE "VideoStoryboardTask"
SET "confirmationStatus" = CASE
WHEN "status" IN ('succeeded', 'partial_success', 'failed') THEN 'confirmed'
WHEN "status" = 'cancelled' THEN 'rejected'
ELSE "confirmationStatus"
END
WHERE "confirmationStatus" = 'draft';
--> statement-breakpoint
UPDATE "VideoStoryboardShotTask"
SET
"hdImageStatus" = CASE
WHEN "status" = 'completed' THEN 'completed'
WHEN "status" = 'failed' THEN 'failed'
WHEN "status" = 'running' THEN 'running'
WHEN "status" = 'cancelled' THEN 'cancelled'
ELSE "hdImageStatus"
END,
"videoStatus" = CASE
WHEN "status" = 'completed' THEN 'completed'
WHEN "status" = 'failed' THEN 'failed'
WHEN "status" = 'running' THEN 'running'
WHEN "status" = 'cancelled' THEN 'cancelled'
ELSE "videoStatus"
END,
"userApproved" = CASE
WHEN "status" IN ('completed', 'failed', 'running') THEN true
ELSE "userApproved"
END,
"userApprovedAt" = CASE
WHEN "status" IN ('completed', 'failed', 'running') AND "userApprovedAt" IS NULL THEN NOW()
ELSE "userApprovedAt"
END;

5
lib/db/migrations/0058_add_agent_secondary_type.sql Normal file
View File

@ -0,0 +1,5 @@
ALTER TABLE "AgentPreset"
ADD COLUMN IF NOT EXISTS "secondaryType" varchar(128);
--> statement-breakpoint
ALTER TABLE "AgentPresetExecutionSnapshot"
ADD COLUMN IF NOT EXISTS "secondaryType" varchar(128);

14
lib/db/migrations/meta/_journal.json
View File

@ -302,6 +302,20 @@
"when": 1773151800000,
"tag": "0056_add_agent_category_path",
"breakpoints": true
},
{
"idx": 43,
"version": "7",
"when": 1773303600000,
"tag": "0057_add_video_storyboard_confirmation_state",
"breakpoints": true
},
{
"idx": 44,
"version": "7",
"when": 1773390000000,
"tag": "0058_add_agent_secondary_type",
"breakpoints": true
}
]
}

28
lib/db/queries.ts
View File

@ -79,6 +79,7 @@ import {
import { generateHashedPassword } from "./utils";
import {
isAiModelAgentAuto,
isAiModelMultimodal,
withAgentAutoModelFlag,
withAgentCodingModelId,
withMultimodalModelFlag,
@ -1749,7 +1750,7 @@ export async function getPointsSystemConfig() {
}
type AgentPresetVisibility = "public" | "private";
type AgentPresetKind = "image" | "video";
type AgentPresetKind = "image" | "video" | "text";
type AgentPresetScope = "all" | AgentPresetVisibility;
type AgentCategoryScope = "all" | "public" | "private" | "global";
type AgentPresetSort = "latest" | "hottest" | "most_used";
@ -1786,6 +1787,8 @@ async function validateAgentDefaultModel(params: {
modelId: aiModel.modelId,
displayName: aiModel.displayName,
supportsImageGen: aiModel.supportsImageGen,
capabilities: aiModel.capabilities,
defaultParams: aiModel.defaultParams,
isEnabled: aiModel.isEnabled,
providerType: aiProvider.type,
providerEnabled: aiProvider.isEnabled,
@ -1812,6 +1815,20 @@ async function validateAgentDefaultModel(params: {
return model;
}
if (params.presetType === "text") {
const isMultimodal = isAiModelMultimodal({
capabilities: model.capabilities,
defaultParams: model.defaultParams,
});
if (isVideoModel || !isMultimodal) {
throw new ChatSDKError(
"bad_request:database",
"生文智能体只能选择带多模态标签的模型"
);
}
return model;
}
if (isVideoModel || !model.supportsImageGen) {
throw new ChatSDKError(
"bad_request:database",
@ -2472,6 +2489,7 @@ export async function createAgentPreset(params: {
userId: string;
name: string;
type: AgentPresetKind;
secondaryType?: string | null;
visibility: AgentPresetVisibility;
categoryPath?: {
productCategory?: string | null;
@ -2515,6 +2533,7 @@ export async function createAgentPreset(params: {
ownerUserId: params.userId,
name: params.name.trim(),
type: params.type,
secondaryType: params.secondaryType || null,
visibility: params.visibility,
categoryId: null,
categoryPath: params.categoryPath || null,
@ -2583,6 +2602,7 @@ export async function updateAgentPreset(params: {
userId: string;
name?: string;
type?: AgentPresetKind;
secondaryType?: string | null;
visibility?: AgentPresetVisibility;
categoryPath?: {
productCategory?: string | null;
@ -2704,6 +2724,9 @@ export async function updateAgentPreset(params: {
if (params.name !== undefined) updateData.name = params.name.trim();
if (params.type !== undefined) updateData.type = params.type;
if (params.secondaryType !== undefined) {
updateData.secondaryType = params.secondaryType || null;
}
if (params.visibility !== undefined) updateData.visibility = params.visibility;
if (params.categoryPath !== undefined) updateData.categoryPath = params.categoryPath || null;
if (params.description !== undefined) updateData.description = params.description || null;
@ -2844,6 +2867,7 @@ export async function forkAgentPreset(params: {
ownerUserId: params.userId,
name: String(params.name || "").trim() || `${source.name}（副本）`,
type: source.type,
secondaryType: source.secondaryType,
visibility: "private",
categoryId: source.categoryId,
categoryPath: source.categoryPath,
@ -2915,6 +2939,7 @@ export async function createAgentPresetExecutionSnapshot(params: {
messageId?: string | null;
presetName: string;
presetType: AgentPresetKind;
secondaryType?: string | null;
visibility: AgentPresetVisibility;
categoryId?: string | null;
categoryName?: string | null;
@ -2950,6 +2975,7 @@ export async function createAgentPresetExecutionSnapshot(params: {
userId: params.userId,
presetName: params.presetName,
presetType: params.presetType,
secondaryType: params.secondaryType || null,
visibility: params.visibility,
categoryId: params.categoryId || null,
categoryName: params.categoryName || null,

25
lib/db/schema.ts
View File

@ -521,9 +521,10 @@ export const agentPreset = pgTable("AgentPreset", {
.notNull()
.references(() => user.id, { onDelete: "cascade" }),
name: varchar("name", { length: 128 }).notNull(),
type: varchar("type", { enum: ["image", "video"] })
type: varchar("type", { enum: ["image", "video", "text"] })
.notNull()
.default("image"),
secondaryType: varchar("secondaryType", { length: 128 }),
visibility: varchar("visibility", { enum: ["public", "private"] })
.notNull()
.default("private"),
@ -708,17 +709,30 @@ export const videoStoryboardTask = pgTable("VideoStoryboardTask", {
.notNull()
.default("ruxa-video"),
modelRef: varchar("modelRef", { length: 128 }),
gridType: varchar("gridType", { length: 32 }),
gridRows: integer("gridRows"),
gridCols: integer("gridCols"),
sourceImageUrl: text("sourceImageUrl").notNull(),
isNineGrid: boolean("isNineGrid").notNull().default(false),
detectionConfidence: integer("detectionConfidence").notNull().default(0),
normalizedScript: jsonb("normalizedScript"),
status: varchar("status", { length: 32 }).notNull().default("pending"),
confirmationStatus: varchar("confirmationStatus", { length: 32 })
.notNull()
.default("draft"),
hdImageStageStatus: varchar("hdImageStageStatus", { length: 32 })
.notNull()
.default("pending"),
currentShot: integer("currentShot").notNull().default(0),
totalShots: integer("totalShots").notNull().default(9),
error: text("error"),
createdAt: timestamp("createdAt").notNull().defaultNow(),
updatedAt: timestamp("updatedAt").notNull().defaultNow(),
startedAt: timestamp("startedAt"),
confirmedAt: timestamp("confirmedAt"),
confirmedByMessageId: uuid("confirmedByMessageId").references(() => message.id, {
onDelete: "set null",
}),
completedAt: timestamp("completedAt"),
});
@ -734,11 +748,17 @@ export const videoStoryboardShotTask = pgTable("VideoStoryboardShotTask", {
tileIndex: integer("tileIndex"),
tileImageUrl: text("tileImageUrl"),
upscaledImageUrl: text("upscaledImageUrl"),
hdImageUrl: text("hdImageUrl"),
hdImagePrompt: text("hdImagePrompt"),
prompt: text("prompt"),
modelId: varchar("modelId", { length: 128 }),
fallbackChain: jsonb("fallbackChain"),
attemptCount: integer("attemptCount").notNull().default(0),
status: varchar("status", { length: 32 }).notNull().default("pending"),
hdImageStatus: varchar("hdImageStatus", { length: 32 }).notNull().default("pending"),
videoStatus: varchar("videoStatus", { length: 32 }).notNull().default("pending"),
userApproved: boolean("userApproved").notNull().default(false),
userApprovedAt: timestamp("userApprovedAt"),
videoUrl: text("videoUrl"),
errorCode: varchar("errorCode", { length: 64 }),
errorMessage: text("errorMessage"),
@ -960,7 +980,8 @@ export const agentPresetExecutionSnapshot = pgTable("AgentPresetExecutionSnapsho
.notNull()
.references(() => user.id, { onDelete: "cascade" }),
presetName: varchar("presetName", { length: 128 }).notNull(),
presetType: varchar("presetType", { enum: ["image", "video"] }).notNull(),
presetType: varchar("presetType", { enum: ["image", "video", "text"] }).notNull(),
secondaryType: varchar("secondaryType", { length: 128 }),
visibility: varchar("visibility", { enum: ["public", "private"] }).notNull(),
categoryId: uuid("categoryId").references(() => agentPresetCategory.id, {
onDelete: "set null",

152
lib/video-agent/agent-thread-state.ts Normal file
View File

@ -0,0 +1,152 @@
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
import type { AgentWorkflowStatus } from "@/lib/ai/runtime/agent-workflow-state";
import type { StoryboardPreferenceMemory } from "@/lib/video-agent/storyboard-preference-memory";
type StoryboardTaskDetail = {
task: {
id: string;
status: string | null;
confirmationStatus?: string | null;
hdImageStageStatus?: string | null;
gridType?: string | null;
gridRows?: number | null;
gridCols?: number | null;
currentShot: number | null;
totalShots: number | null;
sourceImageUrl?: string | null;
providerType?: string | null;
normalizedScript?: unknown;
};
shots: Array<{
shotIndex: number | null;
prompt?: string | null;
status?: string | null;
tileImageUrl?: string | null;
videoUrl?: string | null;
}>;
};
function normalizeText(value: unknown, max = 240): string | undefined {
const normalized = String(value || "").trim();
if (!normalized) return undefined;
return normalized.slice(0, max);
}
function mapStoryboardTaskStatusToWorkflowStatus(
task: StoryboardTaskDetail["task"],
): AgentWorkflowStatus {
const rawConfirmationStatus = (task as Record<string, unknown>).confirmationStatus;
const confirmationStatus = normalizeText(rawConfirmationStatus, 40);
if (confirmationStatus === "awaiting_user_confirmation") {
return "awaiting_user_confirmation";
}
if (confirmationStatus === "confirmed") {
return "confirmed";
}
if (confirmationStatus === "rejected") {
return "rejected";
}
const status = normalizeText(task.status, 40);
switch (status) {
case "pending":
return "draft";
case "running":
return "running";
case "succeeded":
case "partial_success":
return "completed";
case "failed":
return "failed";
case "cancelled":
return "cancelled";
default:
return "draft";
}
}
export function buildAgentThreadStateFromStoryboardTask(
detail: StoryboardTaskDetail | null | undefined,
options?: {
storyboardPreferenceMemory?: StoryboardPreferenceMemory | null;
},
): AgentThreadState | null {
if (!detail?.task?.id) {
if (!options?.storyboardPreferenceMemory) return null;
return {
workflowState: null,
memoryHints: [
"当前会话里存在 storyboard 偏好记忆，可在分镜/宫格任务里优先参考用户常用版式和确认习惯。",
],
metadata: {
workflowSource: "storyboard_preference_memory",
storyboardPreferenceMemory: options.storyboardPreferenceMemory,
},
};
}
const totalShots = Math.max(0, Number(detail.task.totalShots || detail.shots.length || 0));
const currentShot = Math.max(0, Number(detail.task.currentShot || 0));
const gridRows = Math.max(0, Number(detail.task.gridRows || 0));
const gridCols = Math.max(0, Number(detail.task.gridCols || 0));
const gridLabel =
gridRows > 0 && gridCols > 0 ? `${gridCols}x${gridRows}` : `${totalShots} 段`;
return {
workflowState: {
workflowId: detail.task.id,
type: "storyboard_video",
status: mapStoryboardTaskStatusToWorkflowStatus(detail.task),
title: `分镜视频任务（${gridLabel}）`,
summary:
totalShots > 0
? `当前分镜任务进度：第 ${Math.min(currentShot, totalShots)}/${totalShots} 段。`
: "当前存在一个分镜视频任务。",
confirmPhrases: ["确认开始生成视频", "开始生成", "按这个分镜生成"],
revisionHints: ["修改第3镜头", "调整第5镜头动作", "重生第2张高清图"],
pendingItems: detail.shots.slice(0, 9).map((shot) => ({
id:
Number.isFinite(Number(shot.shotIndex)) && Number(shot.shotIndex) > 0
? `shot-${Math.round(Number(shot.shotIndex))}`
: undefined,
index:
Number.isFinite(Number(shot.shotIndex)) && Number(shot.shotIndex) > 0
? Math.round(Number(shot.shotIndex))
: undefined,
title:
Number.isFinite(Number(shot.shotIndex)) && Number(shot.shotIndex) > 0
? `镜头 ${Math.round(Number(shot.shotIndex))}`
: "镜头",
summary: normalizeText(shot.prompt, 120),
status: normalizeText(shot.status, 32),
})),
metadata: {
providerType: normalizeText(detail.task.providerType, 40),
gridType: normalizeText(detail.task.gridType, 32),
gridRows: gridRows || undefined,
gridCols: gridCols || undefined,
sourceImageUrl: normalizeText(detail.task.sourceImageUrl, 500),
normalizedScript:
detail.task.normalizedScript &&
typeof detail.task.normalizedScript === "object" &&
!Array.isArray(detail.task.normalizedScript)
? detail.task.normalizedScript
: undefined,
},
},
memoryHints: [
"如果当前对话是在继续一个分镜视频任务，优先理解为对当前分镜流程的确认、修改或取消。",
"如果用户提到第N镜头、重生某张图、调整某个镜头，优先解释为对当前 storyboard workflow 的修订。",
],
contextSummary:
totalShots > 0
? `当前 chat 存在一个分镜视频任务，共 ${totalShots} 段，当前进度第 ${Math.min(currentShot, totalShots)}/${totalShots} 段。`
: "当前 chat 存在一个分镜视频任务。",
metadata: {
workflowSource: "video_storyboard_task",
taskId: detail.task.id,
shotCount: totalShots,
storyboardPreferenceMemory: options?.storyboardPreferenceMemory || undefined,
},
};
}

304
lib/video-agent/execute-storyboard-task.ts Normal file
View File

@ -0,0 +1,304 @@
import "server-only";
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
import { getAiModelById } from "@/lib/db/queries";
import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
import {
finalizeVideoStoryboardTask,
getVideoStoryboardTaskDetail,
markVideoStoryboardShotCompleted,
markVideoStoryboardShotFailed,
markVideoStoryboardShotRetry,
markVideoStoryboardShotRunning,
syncVideoStoryboardTaskStatus,
} from "@/lib/video-agent/task-store";
import type { StoryboardProgressEvent } from "@/lib/video-agent/types";
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
function normalizeText(value: unknown) {
return String(value || "").trim();
}
function extractVideoUrlFromText(text: string): string | null {
const normalizedText = String(text || "");
if (!normalizedText) return null;
const htmlMatch = normalizedText.match(/<video[^>]+src="([^"]+)"/i);
if (htmlMatch?.[1]) return htmlMatch[1];
const markdownMatch = normalizedText.match(
/\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i
);
if (markdownMatch?.[1]) return markdownMatch[1];
const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i);
return plainMatch?.[1] || null;
}
function normalizeFallbackChain(value: unknown): string[] {
if (!Array.isArray(value)) return [];
return Array.from(
new Set(
value
.map((item) => String(item || "").trim())
.filter(Boolean)
)
);
}
function buildCombinedMarkdown(params: {
success: Array<{ shotIndex: number; text: string }>;
failed: Array<{ shotIndex: number; error: string }>;
}) {
const sections: string[] = [];
for (const shot of params.success.sort((a, b) => a.shotIndex - b.shotIndex)) {
sections.push(`### 分镜 ${shot.shotIndex}\n${shot.text}`);
}
if (params.failed.length > 0) {
const failedLines = params.failed
.sort((a, b) => a.shotIndex - b.shotIndex)
.map((item) => `- 分镜 ${item.shotIndex}: ${item.error}`);
sections.push(["### 失败分镜", ...failedLines].join("\n"));
}
return sections.join("\n\n").trim();
}
export async function executeConfirmedStoryboardTask(params: {
taskId: string;
onProgress?: (event: StoryboardProgressEvent) => void | Promise<void>;
}) {
const detail = await getVideoStoryboardTaskDetail(params.taskId);
if (!detail) {
throw new Error("分镜任务不存在");
}
if (detail.task.providerType !== "ruxa-video") {
throw new Error("当前仅支持执行 ruxa-video 分镜任务");
}
const modelRef = normalizeText(detail.task.modelRef);
if (!modelRef) {
throw new Error("任务缺少 modelRef，无法执行");
}
const selectedModel = await getAiModelById({ id: modelRef }).catch(() => undefined);
const selectedModelId = normalizeText(selectedModel?.modelId);
const languageModel = await getLanguageModelById(modelRef);
if (!languageModel || typeof (languageModel as any).doGenerate !== "function") {
throw new Error("模型不可用或不支持视频执行");
}
const config = await getVideoStoryboardConfig().catch(() => null);
const dispatchIntervalSec = Math.max(1, Number(config?.dispatchIntervalSec || 10));
const maxRetry = Math.max(0, Number(config?.maxRetry || 3));
const enableFallback = config?.enableFallback !== false;
await finalizeVideoStoryboardTask({
taskId: detail.task.id,
status: "running",
});
const targetShots = detail.shots.filter((shot: any) => {
const currentStatus = normalizeText(shot.videoStatus || shot.status);
return currentStatus !== "completed" && currentStatus !== "cancelled";
});
const results: Array<{
shotIndex: number;
success: boolean;
videoUrl?: string;
modelId?: string;
text?: string;
error?: string;
}> = [];
for (let index = 0; index < targetShots.length; index += 1) {
const shot = targetShots[index]!;
const shotIndex = Math.max(1, Number(shot.shotIndex || 0));
const prompt = normalizeText(shot.prompt) || "让画面动起来";
const inputImageUrl =
normalizeText(shot.hdImageUrl) ||
normalizeText(shot.upscaledImageUrl) ||
normalizeText(shot.tileImageUrl);
if (!shotIndex || !inputImageUrl) {
const error = "缺少镜头输入图，无法生成视频";
await markVideoStoryboardShotFailed({
taskId: detail.task.id,
shotIndex,
errorMessage: error,
});
results.push({ shotIndex, success: false, error });
await params.onProgress?.({
stage: "shot_failed",
provider: "ruxa-video",
taskId: detail.task.id,
shotIndex,
error,
});
continue;
}
const fallbackChain = normalizeFallbackChain(shot.fallbackChain);
const candidateModelIds = Array.from(
new Set(
[normalizeText(shot.modelId), ...fallbackChain, selectedModelId].filter(Boolean)
)
);
const modelPool =
enableFallback && candidateModelIds.length > 0
? candidateModelIds
: candidateModelIds.slice(0, 1);
let succeeded = false;
let lastError = "";
let usedModelId = "";
let resultText = "";
let resultUrl = "";
for (const candidateModelId of modelPool) {
for (let attempt = 1; attempt <= maxRetry + 1; attempt += 1) {
try {
await markVideoStoryboardShotRunning({
taskId: detail.task.id,
shotIndex,
modelId: candidateModelId,
});
const result = await (languageModel as any).doGenerate({
messages: [
{
role: "user",
content: [
{ type: "text", text: prompt },
{ type: "image_url", image_url: { url: inputImageUrl } },
],
},
],
videoParams: { mode: "first-frame" },
overrideModelId: candidateModelId,
});
const text = normalizeText(result?.text);
const videoUrl = extractVideoUrlFromText(text);
if (!videoUrl) {
throw new Error(text || "未返回视频地址");
}
await markVideoStoryboardShotCompleted({
taskId: detail.task.id,
shotIndex,
modelId: candidateModelId,
videoUrl,
});
succeeded = true;
usedModelId = candidateModelId;
resultText = text;
resultUrl = videoUrl;
break;
} catch (error) {
lastError = error instanceof Error ? error.message : String(error);
await markVideoStoryboardShotRetry({
taskId: detail.task.id,
shotIndex,
errorMessage: lastError,
});
if (attempt <= maxRetry) {
await params.onProgress?.({
stage: "shot_retry",
provider: "ruxa-video",
taskId: detail.task.id,
shotIndex,
attempt,
modelId: candidateModelId,
error: lastError,
});
const backoffSec = Math.min(40, Math.pow(2, attempt - 1) * 10);
await sleep(backoffSec * 1000);
}
}
}
if (succeeded) break;
}
if (succeeded) {
results.push({
shotIndex,
success: true,
videoUrl: resultUrl,
modelId: usedModelId,
text: resultText,
});
await params.onProgress?.({
stage: "shot_completed",
provider: "ruxa-video",
taskId: detail.task.id,
shotIndex,
modelId: usedModelId,
videoUrl: resultUrl,
text: resultText,
});
} else {
await markVideoStoryboardShotFailed({
taskId: detail.task.id,
shotIndex,
errorMessage: lastError || "视频生成失败",
});
results.push({
shotIndex,
success: false,
error: lastError || "视频生成失败",
});
await params.onProgress?.({
stage: "shot_failed",
provider: "ruxa-video",
taskId: detail.task.id,
shotIndex,
error: lastError || "视频生成失败",
});
}
if (index < targetShots.length - 1 && dispatchIntervalSec > 0) {
await sleep(dispatchIntervalSec * 1000);
}
}
const summary = await syncVideoStoryboardTaskStatus(detail.task.id);
await params.onProgress?.({
stage: "completed",
provider: "ruxa-video",
taskId: detail.task.id,
totalShots: results.length,
successCount: summary.success,
failedCount: summary.failed,
});
return {
taskId: detail.task.id,
videoUrls: results.filter((item) => item.success && item.videoUrl).map((item) => item.videoUrl!),
modelRefs: results.filter((item) => item.success && item.modelId).map((item) => item.modelId!),
markdown: buildCombinedMarkdown({
success: results
.filter((item) => item.success)
.map((item) => ({
shotIndex: item.shotIndex,
text: normalizeText(item.text),
})),
failed: results
.filter((item) => !item.success)
.map((item) => ({
shotIndex: item.shotIndex,
error: normalizeText(item.error) || "video_generation_failed",
})),
}),
successCount: summary.success,
failedCount: summary.failed,
status: summary.status,
};
}

358
lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts
View File

@ -1,48 +1,31 @@
import "server-only";
import { generateCanvasImage } from "@/lib/canvas/generate-image";
import { generateUUID } from "@/lib/utils";
import {
createVideoStoryboardShotTasks,
createVideoStoryboardTask,
finalizeVideoStoryboardTask,
markVideoStoryboardShotCompleted,
markVideoStoryboardShotFailed,
markVideoStoryboardShotRetry,
markVideoStoryboardShotRunning,
markVideoStoryboardAwaitingConfirmation,
markVideoStoryboardHdCompleted,
markVideoStoryboardHdFailed,
markVideoStoryboardHdRunning,
syncVideoStoryboardHdStageStatus,
} from "@/lib/video-agent/task-store";
import { runNineGridDetectorSkill } from "@/lib/video-agent/skills/nine-grid-detector";
import { runScriptNormalizerSkill } from "@/lib/video-agent/skills/script-normalizer";
import { runStoryboardMapperSkill } from "@/lib/video-agent/skills/storyboard-mapper";
import { runTileUpscalerSkill } from "@/lib/video-agent/skills/tile-upscaler";
import { runVideoSequencerSkill } from "@/lib/video-agent/skills/video-sequencer";
import type {
RuxaNineGridPipelineParams,
RuxaNineGridPipelineResult,
StoryboardGridType,
StoryboardModelCandidate,
} from "@/lib/video-agent/types";
const NINE_GRID_SHOT_COUNT = 9;
function normalizeText(value: unknown) {
return String(value || "").trim();
}
function extractVideoUrlFromText(text: string): string | null {
const normalizedText = String(text || "");
if (!normalizedText) return null;
const htmlMatch = normalizedText.match(/<video[^>]+src="([^"]+)"/i);
if (htmlMatch?.[1]) return htmlMatch[1];
const markdownMatch = normalizedText.match(
/\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i
);
if (markdownMatch?.[1]) return markdownMatch[1];
const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i);
return plainMatch?.[1] || null;
}
function extractPromptAndImages(messages: Array<{ role: string; content: any }>) {
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
if (!lastUserMessage) {
@ -107,24 +90,51 @@ function normalizeModelCandidates(params: {
});
}
function buildCombinedMarkdown(params: {
success: Array<{ shotIndex: number; text: string }>;
failed: Array<{ shotIndex: number; error: string }>;
function buildAwaitingConfirmationMarkdown(params: {
shots: Array<{
shotIndex: number;
prompt: string;
}>;
}) {
const sections: string[] = [];
const lines = [
"### 分镜草案已准备完成",
"我已经根据当前宫格切片生成了每段镜头的预览图，请先确认镜头内容。",
"",
...params.shots.map(
(shot) => `- 镜头 ${shot.shotIndex}：${normalizeText(shot.prompt) || "待补充分镜文案"}`
),
"",
"如果需要修改，请直接回复：",
"- 修改第3镜头：……",
"- 调整第5镜头动作：……",
"- 重生第2张高清图",
"",
"如果确认无误，请回复：",
"- 确认开始生成视频",
];
return lines.join("\n").trim();
}
for (const shot of params.success.sort((a, b) => a.shotIndex - b.shotIndex)) {
sections.push(`### 分镜 ${shot.shotIndex}\n${shot.text}`);
function inferPreviewAspectRatio(width?: number, height?: number): string {
const safeWidth = Math.max(1, Number(width || 0));
const safeHeight = Math.max(1, Number(height || 0));
const ratio = safeWidth / safeHeight;
if (ratio > 1.35) return "16:9";
if (ratio < 0.8) return "9:16";
return "1:1";
}
function getGridLabel(gridType: StoryboardGridType | null, shotCount: number) {
switch (gridType) {
case "grid_2x2":
return "4宫格";
case "grid_3x2":
return "6宫格";
case "grid_3x3":
return "9宫格";
default:
return `${shotCount}宫格`;
}
if (params.failed.length > 0) {
const failedLines = params.failed
.sort((a, b) => a.shotIndex - b.shotIndex)
.map((item) => `- 分镜 ${item.shotIndex}: ${item.error}`);
sections.push(["### 失败分镜", ...failedLines].join("\n"));
}
return sections.join("\n\n").trim();
}
export async function runRuxaNineGridPipeline(
@ -140,7 +150,6 @@ export async function runRuxaNineGridPipeline(
videoParams,
config,
onProgress,
doGenerate,
} = params;
if (!config.enabled) {
@ -157,7 +166,7 @@ export async function runRuxaNineGridPipeline(
onProgress?.({
stage: "detecting",
provider: "ruxa-video",
message: "正在识别九宫格与解析分镜...",
message: "正在识别分镜拼图与解析分镜...",
});
const { prompt, imageUrls } = extractPromptAndImages(modelMessages);
@ -170,34 +179,47 @@ export async function runRuxaNineGridPipeline(
minConfidence: config.minGridConfidence,
});
if (!detection.isNineGrid || detection.tiles.length !== NINE_GRID_SHOT_COUNT) {
if (!detection.isStoryboardGrid || detection.tiles.length !== detection.shotCount) {
return {
triggered: false,
reason: detection.isNineGrid ? "slice_incomplete" : "not_nine_grid",
reason: detection.isStoryboardGrid ? "slice_incomplete" : "not_storyboard_grid",
confidence: detection.confidence,
};
}
const normalizedScript = runScriptNormalizerSkill({
scriptText: prompt,
maxShots: NINE_GRID_SHOT_COUNT,
maxShots: detection.shotCount,
totalDurationSec: Number(videoParams?.duration || 10),
});
const mappedShots = runStoryboardMapperSkill({
script: normalizedScript,
tiles: detection.tiles,
});
const preparedShots = await runTileUpscalerSkill({
shots: mappedShots,
enableTileUpscale: config.enableTileUpscale,
sliceUpscaleMinEdge: config.sliceUpscaleMinEdge,
});
const tileSummary = detection.tiles.map((tile) => ({
index: tile.index,
imageUrl: tile.imageUrl,
width: tile.width,
height: tile.height,
}));
const tileMetaByIndex = new Map(
detection.tiles.map((tile) => [
tile.index,
{
width: tile.width,
height: tile.height,
},
])
);
const shotPlanSummary = preparedShots.map((shot) => ({
shotIndex: shot.shotIndex,
tileIndex: shot.tileIndex,
@ -214,7 +236,6 @@ export async function runRuxaNineGridPipeline(
selectedModelRef,
modelCandidates,
});
const fallbackChain = resolvedModelCandidates
.map((item) => normalizeText(item.modelId))
.filter(Boolean);
@ -224,14 +245,19 @@ export async function runRuxaNineGridPipeline(
chatId,
providerType: "ruxa-video",
modelRef: selectedModelRef,
gridType: detection.gridType,
gridRows: detection.rows,
gridCols: detection.cols,
sourceImageUrl: imageUrls[0],
isNineGrid: true,
isNineGrid: detection.gridType === "grid_3x3",
detectionConfidence: detection.confidence,
normalizedScript: normalizedScript as any,
totalShots: preparedShots.length,
status: "running",
});
const taskId = createdTask.id || `ruxa-storyboard-${generateUUID()}`;
await createVideoStoryboardShotTasks({
parentTaskId: taskId,
shots: preparedShots,
@ -243,6 +269,9 @@ export async function runRuxaNineGridPipeline(
provider: "ruxa-video",
taskId,
totalShots: preparedShots.length,
gridType: detection.gridType || undefined,
rows: detection.rows,
cols: detection.cols,
confidence: detection.confidence,
sourceImageUrl: imageUrls[0],
shots: preparedShots.map((shot) => ({
@ -252,163 +281,140 @@ export async function runRuxaNineGridPipeline(
})),
});
const sequencer = await runVideoSequencerSkill({
provider: "ruxa-video",
taskId,
shots: preparedShots,
modelCandidates: resolvedModelCandidates,
maxRetry: config.maxRetry,
dispatchIntervalSec: config.dispatchIntervalSec,
enableFallback: config.enableFallback,
onProgress: async (event) => {
if (event.stage === "shot_retry") {
await markVideoStoryboardShotRetry({
taskId,
shotIndex: event.shotIndex,
errorMessage: event.error,
}).catch(() => undefined);
}
if (event.stage === "shot_completed") {
await markVideoStoryboardShotCompleted({
taskId,
shotIndex: event.shotIndex,
modelId: event.modelId,
videoUrl: event.videoUrl,
}).catch(() => undefined);
}
if (event.stage === "shot_failed") {
await markVideoStoryboardShotFailed({
taskId,
shotIndex: event.shotIndex,
errorMessage: event.error,
}).catch(() => undefined);
}
await onProgress?.(event);
},
generateOneShot: async ({ shot, model }) => {
await markVideoStoryboardShotRunning({
taskId,
shotIndex: shot.shotIndex,
modelId: model.modelId,
}).catch(() => undefined);
const previewShots: Array<{
shotIndex: number;
tileImageUrl: string;
previewImageUrl: string;
prompt: string;
fallbackUsed: boolean;
}> = [];
const shotMessages = [
{
role: "user",
content: [
{ type: "text", text: shot.prompt },
{ type: "image_url", image_url: { url: shot.inputImageUrl } },
],
},
];
const output = await doGenerate({
messages: shotMessages,
videoParams: {
...(videoParams || {}),
mode: "first-frame",
},
overrideModelId: model.modelId,
});
const text = normalizeText(output?.text);
const videoUrl = extractVideoUrlFromText(text);
if (!videoUrl) {
throw new Error(text || "未返回视频地址");
}
return { videoUrl, text };
},
});
const successShots = sequencer.results
.filter((item) => item.success && item.videoUrl)
.map((item) => ({
shotIndex: item.shotIndex,
text: normalizeText(item.text),
videoUrl: normalizeText(item.videoUrl),
modelId: normalizeText(item.modelId),
}));
const failedShots = sequencer.results
.filter((item) => !item.success)
.map((item) => ({
shotIndex: item.shotIndex,
error: normalizeText(item.error) || "video_generation_failed",
}));
if (successShots.length === 0) {
await finalizeVideoStoryboardTask({
for (const shot of preparedShots) {
await markVideoStoryboardHdRunning({
taskId,
status: "failed",
error: "all_shots_failed",
shotIndex: shot.shotIndex,
}).catch(() => undefined);
await onProgress?.({
stage: "failed",
provider: "ruxa-video",
taskId,
error: "all_shots_failed",
});
return {
triggered: true,
taskId,
confidence: detection.confidence,
reason: "all_shots_failed",
videoUrls: [],
modelRefs: [],
markdown: "视频生成失败，请稍后重试",
metadata: {
try {
const tileMeta = tileMetaByIndex.get(shot.tileIndex);
const hdResult = await generateCanvasImage({
prompt: shot.prompt,
referenceImage: shot.inputImageUrl,
imageCount: 1,
imageSize: "2K",
aspectRatio: inferPreviewAspectRatio(tileMeta?.width, tileMeta?.height),
});
const previewImageUrl =
hdResult.images[0]?.originalUrl || hdResult.images[0]?.thumbnailUrl || shot.inputImageUrl;
await markVideoStoryboardHdCompleted({
taskId,
successCount: 0,
failedCount: failedShots.length,
confidence: detection.confidence,
tiles: tileSummary,
shotPlan: shotPlanSummary,
normalizedScript,
failedShots,
},
};
shotIndex: shot.shotIndex,
hdImageUrl: previewImageUrl,
hdImagePrompt: shot.prompt,
}).catch(() => undefined);
previewShots.push({
shotIndex: shot.shotIndex,
tileImageUrl: shot.tileImageUrl,
previewImageUrl,
prompt: shot.prompt,
fallbackUsed: false,
});
await onProgress?.({
stage: "hd_shot_completed",
provider: "ruxa-video",
taskId,
shotIndex: shot.shotIndex,
imageUrl: previewImageUrl,
prompt: shot.prompt,
});
} catch (error) {
const errorMessage =
normalizeText(error instanceof Error ? error.message : String(error)) ||
"hd_image_generation_failed";
await markVideoStoryboardHdFailed({
taskId,
shotIndex: shot.shotIndex,
errorMessage,
}).catch(() => undefined);
previewShots.push({
shotIndex: shot.shotIndex,
tileImageUrl: shot.tileImageUrl,
previewImageUrl: shot.inputImageUrl,
prompt: shot.prompt,
fallbackUsed: true,
});
await onProgress?.({
stage: "hd_shot_failed",
provider: "ruxa-video",
taskId,
shotIndex: shot.shotIndex,
error: errorMessage,
});
}
}
const finalTaskStatus =
sequencer.failedCount > 0 ? "partial_success" : "succeeded";
await finalizeVideoStoryboardTask({
await syncVideoStoryboardHdStageStatus(taskId).catch(() => undefined);
await markVideoStoryboardAwaitingConfirmation({
taskId,
status: finalTaskStatus,
}).catch(() => undefined);
const awaitingConfirmationMessage = buildAwaitingConfirmationMarkdown({
shots: previewShots.map((shot) => ({
shotIndex: shot.shotIndex,
prompt: shot.prompt,
})),
}).replace(
"我已经根据当前宫格切片生成了每段镜头的预览图，请先确认镜头内容。",
`我已经根据当前${getGridLabel(detection.gridType, detection.shotCount)}切片生成了每段镜头的预览图，请先确认镜头内容。`
);
await onProgress?.({
stage: "completed",
stage: "awaiting_confirmation",
provider: "ruxa-video",
taskId,
totalShots: sequencer.results.length,
successCount: sequencer.successCount,
failedCount: sequencer.failedCount,
totalShots: previewShots.length,
gridType: detection.gridType || undefined,
rows: detection.rows,
cols: detection.cols,
message: awaitingConfirmationMessage,
shots: previewShots.map((shot) => ({
shotIndex: shot.shotIndex,
tileImageUrl: shot.tileImageUrl,
previewImageUrl: shot.previewImageUrl,
prompt: shot.prompt,
})),
});
return {
triggered: true,
taskId,
confidence: detection.confidence,
markdown: buildCombinedMarkdown({
success: successShots.map((item) => ({
shotIndex: item.shotIndex,
text: item.text,
})),
failed: failedShots,
}),
videoUrls: successShots.map((item) => item.videoUrl),
modelRefs: successShots.map((item) => item.modelId).filter(Boolean),
markdown: awaitingConfirmationMessage,
videoUrls: [],
previewImageUrls: previewShots.map((shot) => shot.previewImageUrl),
modelRefs: [],
metadata: {
taskId,
confidence: detection.confidence,
qualityScore: detection.qualityScore,
sourceImageUrl: detection.sourceImageUrl,
gridType: detection.gridType,
rows: detection.rows,
cols: detection.cols,
shotCount: detection.shotCount,
tiles: tileSummary,
shotPlan: shotPlanSummary,
normalizedScript,
successCount: sequencer.successCount,
failedCount: sequencer.failedCount,
failedShots,
awaitingConfirmation: true,
previewShots,
},
reason: "awaiting_user_confirmation",
};
}

288
lib/video-agent/revise-storyboard-task.ts Normal file
View File

@ -0,0 +1,288 @@
import "server-only";
import { generateCanvasImage } from "@/lib/canvas/generate-image";
import {
getVideoStoryboardTaskDetail,
markVideoStoryboardAwaitingConfirmation,
markVideoStoryboardDraftForRevision,
markVideoStoryboardHdCompleted,
markVideoStoryboardHdFailed,
markVideoStoryboardHdRunning,
syncVideoStoryboardHdStageStatus,
updateVideoStoryboardShotPrompt,
} from "@/lib/video-agent/task-store";
import {
interpretStoryboardCommand,
parseStoryboardRevisionCommand as parseStoryboardRevisionCommandWithContext,
type StoryboardRevisionCommand,
} from "@/lib/video-agent/storyboard-command-interpreter";
import type {
StoryboardGridType,
StoryboardProgressEvent,
} from "@/lib/video-agent/types";
function normalizeText(value: unknown) {
return String(value || "").trim();
}
function buildAwaitingConfirmationEvent(params: {
taskId: string;
detail: NonNullable<Awaited<ReturnType<typeof getVideoStoryboardTaskDetail>>>;
message: string;
}): StoryboardProgressEvent {
const task = params.detail.task as typeof params.detail.task & {
gridType?: StoryboardGridType | null;
gridRows?: number | null;
gridCols?: number | null;
};
return {
stage: "awaiting_confirmation",
provider: "ruxa-video",
taskId: params.taskId,
totalShots: Number(task.totalShots || params.detail.shots.length || 0),
gridType: task.gridType || undefined,
rows: Number(task.gridRows || 0) || undefined,
cols: Number(task.gridCols || 0) || undefined,
message: params.message,
shots: params.detail.shots.map((shot: any) => ({
shotIndex: Number(shot.shotIndex || 0),
tileImageUrl: normalizeText(shot.tileImageUrl),
previewImageUrl:
normalizeText(shot.hdImageUrl) ||
normalizeText(shot.upscaledImageUrl) ||
normalizeText(shot.tileImageUrl),
prompt: normalizeText(shot.prompt),
})),
};
}
function inferPreviewAspectRatio(gridType?: StoryboardGridType | null): string {
switch (gridType) {
case "grid_2x2":
case "grid_3x2":
case "grid_3x3":
default:
return "1:1";
}
}
export function parseStoryboardRevisionCommand(params: {
text: string;
totalShots?: number;
}): StoryboardRevisionCommand {
return parseStoryboardRevisionCommandWithContext(params);
}
export async function resolveStoryboardRevisionCommand(params: {
text: string;
detail: NonNullable<Awaited<ReturnType<typeof getVideoStoryboardTaskDetail>>>;
}) {
const interpreted = await interpretStoryboardCommand({
text: params.text,
totalShots: Number(params.detail.task.totalShots || params.detail.shots.length || 0),
items: params.detail.shots.map((shot: any) => ({
index: Number(shot.shotIndex || 0),
title: Number(shot.shotIndex || 0) > 0 ? `镜头 ${Number(shot.shotIndex || 0)}` : undefined,
summary: normalizeText(shot.prompt),
})),
});
return interpreted.revision;
}
export async function reviseStoryboardTaskByCommand(params: {
taskId: string;
command: StoryboardRevisionCommand;
onProgress?: (event: StoryboardProgressEvent) => void | Promise<void>;
}) {
if (!params.command) {
return {
ok: false,
message: "我收到了修改意图，但还没定位到具体镜头。请直接回复例如“修改第3镜头：……”或“重生第2张高清图”。",
detail: await getVideoStoryboardTaskDetail(params.taskId),
};
}
const detail = await getVideoStoryboardTaskDetail(params.taskId);
if (!detail) {
return {
ok: false,
message: "当前分镜任务不存在或已失效。",
detail: null,
};
}
const targetShot = detail.shots.find(
(shot: any) => Number(shot.shotIndex || 0) === params.command?.shotIndex
);
if (!targetShot) {
return {
ok: false,
message: `没有找到第 ${params.command.shotIndex} 镜头，请检查镜头编号后再试。`,
detail,
};
}
await markVideoStoryboardDraftForRevision({
taskId: params.taskId,
});
if (params.command.type === "update_prompt") {
await updateVideoStoryboardShotPrompt({
taskId: params.taskId,
shotIndex: params.command.shotIndex,
prompt: params.command.prompt,
});
}
const latestDetail = await getVideoStoryboardTaskDetail(params.taskId);
const latestShot = latestDetail?.shots.find(
(shot: any) => Number(shot.shotIndex || 0) === params.command?.shotIndex
);
const currentPrompt =
params.command.type === "update_prompt"
? params.command.prompt
: normalizeText(latestShot?.prompt) || normalizeText(targetShot.prompt);
const inputImageUrl =
normalizeText(latestShot?.upscaledImageUrl) ||
normalizeText(latestShot?.tileImageUrl) ||
normalizeText(targetShot.upscaledImageUrl) ||
normalizeText(targetShot.tileImageUrl);
if (!inputImageUrl) {
return {
ok: false,
message: `第 ${params.command.shotIndex} 镜头缺少输入图，无法重生高清预览。`,
detail: latestDetail || detail,
};
}
try {
await markVideoStoryboardHdRunning({
taskId: params.taskId,
shotIndex: params.command.shotIndex,
});
const hdResult = await generateCanvasImage({
prompt: currentPrompt,
referenceImage: inputImageUrl,
imageCount: 1,
imageSize: "2K",
aspectRatio: inferPreviewAspectRatio(
(latestDetail?.task as any)?.gridType || (detail.task as any)?.gridType
),
});
const previewImageUrl =
hdResult.images[0]?.originalUrl || hdResult.images[0]?.thumbnailUrl || inputImageUrl;
await markVideoStoryboardHdCompleted({
taskId: params.taskId,
shotIndex: params.command.shotIndex,
hdImageUrl: previewImageUrl,
hdImagePrompt: currentPrompt,
});
await syncVideoStoryboardHdStageStatus(params.taskId);
await markVideoStoryboardAwaitingConfirmation({
taskId: params.taskId,
});
await params.onProgress?.({
stage: "hd_shot_completed",
provider: "ruxa-video",
taskId: params.taskId,
shotIndex: params.command.shotIndex,
imageUrl: previewImageUrl,
prompt: currentPrompt,
});
const refreshedDetail = await getVideoStoryboardTaskDetail(params.taskId);
if (refreshedDetail) {
const message =
params.command.type === "update_prompt"
? `已更新第 ${params.command.shotIndex} 镜头，并重新生成了高清预览，请继续确认。`
: `已重生第 ${params.command.shotIndex} 张高清预览，请继续确认。`;
await params.onProgress?.(
buildAwaitingConfirmationEvent({
taskId: params.taskId,
detail: refreshedDetail,
message,
})
);
return {
ok: true,
message,
detail: refreshedDetail,
};
}
return {
ok: true,
message:
params.command.type === "update_prompt"
? `已更新第 ${params.command.shotIndex} 镜头，并重新生成高清预览。`
: `已重生第 ${params.command.shotIndex} 张高清预览。`,
detail: latestDetail || detail,
};
} catch (error) {
const errorMessage =
normalizeText(error instanceof Error ? error.message : String(error)) ||
"高清预览重生失败";
await markVideoStoryboardHdFailed({
taskId: params.taskId,
shotIndex: params.command.shotIndex,
errorMessage,
});
await params.onProgress?.({
stage: "hd_shot_failed",
provider: "ruxa-video",
taskId: params.taskId,
shotIndex: params.command.shotIndex,
error: errorMessage,
});
return {
ok: false,
message: `第 ${params.command.shotIndex} 镜头的高清预览重生失败：${errorMessage}`,
detail: await getVideoStoryboardTaskDetail(params.taskId),
};
}
}
export async function updateStoryboardShotPromptAndRegenerateHd(params: {
taskId: string;
shotIndex: number;
prompt: string;
onProgress?: (event: StoryboardProgressEvent) => void | Promise<void>;
}) {
return reviseStoryboardTaskByCommand({
taskId: params.taskId,
command: {
type: "update_prompt",
shotIndex: params.shotIndex,
prompt: params.prompt,
},
onProgress: params.onProgress,
});
}
export async function regenerateStoryboardShotHd(params: {
taskId: string;
shotIndex: number;
onProgress?: (event: StoryboardProgressEvent) => void | Promise<void>;
}) {
return reviseStoryboardTaskByCommand({
taskId: params.taskId,
command: {
type: "regenerate_hd",
shotIndex: params.shotIndex,
},
onProgress: params.onProgress,
});
}

150
lib/video-agent/skills/nine-grid-detector.ts
View File

@ -2,7 +2,12 @@ import "server-only";
import sharp from "sharp";
import { uploadFile } from "@/lib/storage/service";
import type { NineGridCropBox, NineGridDetectionResult, NineGridTile } from "@/lib/video-agent/types";
import type {
StoryboardCropBox,
StoryboardDetectionResult,
StoryboardGridType,
StoryboardTile,
} from "@/lib/video-agent/types";
type ProjectionPeak = {
index: number;
@ -81,41 +86,53 @@ function findPeakNear(
return { index: bestIndex, value: Number.isFinite(bestValue) ? bestValue : 0 };
}
function scoreGridAxis(scores: number[], firstTarget: number, secondTarget: number) {
function scoreGridAxis(scores: number[], targets: number[]) {
if (targets.length === 0) {
return {
score: 0,
peaks: [] as ProjectionPeak[],
};
}
const radius = Math.max(8, Math.round(scores.length * 0.12));
const firstPeak = findPeakNear(scores, firstTarget, radius);
const secondPeak = findPeakNear(scores, secondTarget, radius);
const peaks = targets.map((target) => findPeakNear(scores, target, radius));
const avg = mean(scores);
const sigma = stddev(scores);
const dynamicBase = avg + sigma;
const closenessA = clamp01(1 - Math.abs(firstPeak.index - firstTarget) / radius);
const closenessB = clamp01(1 - Math.abs(secondPeak.index - secondTarget) / radius);
const closeness = (closenessA + closenessB) / 2;
const denom = Math.max(dynamicBase, 1);
const strengthA = clamp01(firstPeak.value / (denom * 2.2));
const strengthB = clamp01(secondPeak.value / (denom * 2.2));
const strength = (strengthA + strengthB) / 2;
const closenessScores = peaks.map((peak, index) =>
clamp01(1 - Math.abs(peak.index - targets[index]!) / radius)
);
const strengthScores = peaks.map((peak) =>
clamp01(peak.value / (denom * 2.2))
);
const closeness = mean(closenessScores);
const strength = mean(strengthScores);
return {
score: clamp01(closeness * 0.55 + strength * 0.45),
peaks: [firstPeak, secondPeak],
peaks,
};
}
function buildNineGridCropBoxes(width: number, height: number): NineGridCropBox[] {
const baseW = Math.floor(width / 3);
const baseH = Math.floor(height / 3);
const boxes: NineGridCropBox[] = [];
function buildStoryboardCropBoxes(
width: number,
height: number,
rows: number,
cols: number
): StoryboardCropBox[] {
const baseW = Math.floor(width / cols);
const baseH = Math.floor(height / rows);
const boxes: StoryboardCropBox[] = [];
let index = 1;
for (let row = 0; row < 3; row += 1) {
for (let col = 0; col < 3; col += 1) {
for (let row = 0; row < rows; row += 1) {
for (let col = 0; col < cols; col += 1) {
const x = col * baseW;
const y = row * baseH;
const w = col === 2 ? width - x : baseW;
const h = row === 2 ? height - y : baseH;
boxes.push({ index, x, y, w, h });
const w = col === cols - 1 ? width - x : baseW;
const h = row === rows - 1 ? height - y : baseH;
boxes.push({ index, row: row + 1, col: col + 1, x, y, w, h });
index += 1;
}
}
@ -136,11 +153,11 @@ async function fetchImageBuffer(imageUrl: string) {
async function sliceAndUploadTiles(params: {
sourceBuffer: Buffer;
cropBoxes: NineGridCropBox[];
}): Promise<NineGridTile[]> {
cropBoxes: StoryboardCropBox[];
}): Promise<StoryboardTile[]> {
const { sourceBuffer, cropBoxes } = params;
const timestamp = Date.now();
const tiles: NineGridTile[] = [];
const tiles: StoryboardTile[] = [];
for (const cropBox of cropBoxes) {
const tileBuffer = await sharp(sourceBuffer)
.extract({
@ -156,6 +173,8 @@ async function sliceAndUploadTiles(params: {
const uploaded = await uploadFile(filename, toArrayBuffer(tileBuffer));
tiles.push({
index: cropBox.index,
row: cropBox.row,
col: cropBox.col,
imageUrl: uploaded.url,
width: cropBox.w,
height: cropBox.h,
@ -164,13 +183,39 @@ async function sliceAndUploadTiles(params: {
return tiles;
}
type GridCandidate = {
gridType: StoryboardGridType;
rows: number;
cols: number;
expectedRatio: number;
};
const GRID_CANDIDATES: GridCandidate[] = [
{ gridType: "grid_2x2", rows: 2, cols: 2, expectedRatio: 1 },
{ gridType: "grid_3x2", rows: 2, cols: 3, expectedRatio: 1.5 },
{ gridType: "grid_3x3", rows: 3, cols: 3, expectedRatio: 1 },
];
function buildAxisTargets(length: number, splitCount: number) {
const targets: number[] = [];
for (let index = 1; index < splitCount; index += 1) {
targets.push((length * index) / splitCount);
}
return targets;
}
function scoreAspectRatio(actualRatio: number, expectedRatio: number) {
const tolerance = expectedRatio === 1 ? 0.45 : 0.6;
return clamp01(1 - Math.abs(actualRatio - expectedRatio) / tolerance);
}
export async function runNineGridDetectorSkill(params: {
imageUrl: string;
minConfidence: number;
}): Promise<NineGridDetectionResult> {
}): Promise<StoryboardDetectionResult> {
const sourceImageUrl = String(params.imageUrl || "").trim();
if (!sourceImageUrl) {
throw new Error("缺少九宫格输入图片");
throw new Error("缺少分镜拼图输入图片");
}
const sourceBuffer = await fetchImageBuffer(sourceImageUrl);
@ -181,7 +226,12 @@ export async function runNineGridDetectorSkill(params: {
if (width < 240 || height < 240) {
return {
isStoryboardGrid: false,
isNineGrid: false,
gridType: null,
rows: 0,
cols: 0,
shotCount: 0,
confidence: 0,
qualityScore: 0,
cropBoxes: [],
@ -223,24 +273,52 @@ export async function runNineGridDetectorSkill(params: {
const smoothCols = movingAverage(colScores, 5);
const smoothRows = movingAverage(rowScores, 5);
const colAxis = scoreGridAxis(smoothCols, imgW / 3, (imgW * 2) / 3);
const rowAxis = scoreGridAxis(smoothRows, imgH / 3, (imgH * 2) / 3);
const ratio = width / Math.max(height, 1);
const aspectScore = clamp01(1 - Math.min(Math.abs(ratio - 1), 1));
const gridScore = clamp01((colAxis.score + rowAxis.score) / 2);
const confidence = clamp01(aspectScore * 0.3 + gridScore * 0.7);
const qualityScore = clamp01((sampleStddevFromUint8(raw, 20000) || 0) / 72);
const isNineGrid = confidence >= params.minConfidence && aspectScore >= 0.62;
const cropBoxes = isNineGrid ? buildNineGridCropBoxes(width, height) : [];
const tiles = isNineGrid
const scoredCandidates = GRID_CANDIDATES.map((candidate) => {
const colTargets = buildAxisTargets(imgW, candidate.cols);
const rowTargets = buildAxisTargets(imgH, candidate.rows);
const colAxis = scoreGridAxis(smoothCols, colTargets);
const rowAxis = scoreGridAxis(smoothRows, rowTargets);
const axisScores = [colAxis.score, rowAxis.score];
const gridScore = clamp01(mean(axisScores));
const aspectScore = scoreAspectRatio(ratio, candidate.expectedRatio);
const confidence = clamp01(aspectScore * 0.34 + gridScore * 0.66);
return {
...candidate,
confidence,
aspectScore,
gridScore,
};
}).sort((left, right) => right.confidence - left.confidence);
const bestCandidate = scoredCandidates[0];
const isStoryboardGrid =
Boolean(bestCandidate) &&
bestCandidate.confidence >= params.minConfidence &&
bestCandidate.aspectScore >= 0.55;
const rows = isStoryboardGrid ? bestCandidate.rows : 0;
const cols = isStoryboardGrid ? bestCandidate.cols : 0;
const shotCount = rows * cols;
const gridType = isStoryboardGrid ? bestCandidate.gridType : null;
const isNineGrid = gridType === "grid_3x3";
const cropBoxes = isStoryboardGrid
? buildStoryboardCropBoxes(width, height, rows, cols)
: [];
const tiles = isStoryboardGrid
? await sliceAndUploadTiles({ sourceBuffer: await baseImage.toBuffer(), cropBoxes })
: [];
return {
isStoryboardGrid,
isNineGrid,
confidence,
gridType,
rows,
cols,
shotCount,
confidence: isStoryboardGrid ? bestCandidate.confidence : 0,
qualityScore,
cropBoxes,
width,

15
lib/video-agent/skills/script-normalizer.ts
View File

@ -33,17 +33,17 @@ function splitScriptLines(script: string): string[] {
.filter(Boolean);
}
function parsePreferredTileIndex(text: string): number | undefined {
function parsePreferredTileIndex(text: string, maxShots: number): number | undefined {
const source = String(text || "");
const patterns = [
/第\s*([1-9])\s*(?:张|格|图|镜头)/i,
/(?:图|格|镜头|shot)\s*([1-9])/i,
/#\s*([1-9])/i,
/第\s*(\d{1,2})\s*(?:张|格|图|镜头)/i,
/(?:图|格|镜头|shot)\s*(\d{1,2})/i,
/#\s*(\d{1,2})/i,
];
for (const pattern of patterns) {
const matched = source.match(pattern);
const candidate = Number(matched?.[1] || 0);
if (candidate >= 1 && candidate <= 9) return candidate;
if (candidate >= 1 && candidate <= maxShots) return candidate;
}
return undefined;
}
@ -78,7 +78,7 @@ function buildDefaultAction(index: number) {
"主体完成动作收束",
"镜头定格形成结尾",
];
return defaults[index - 1] || "主体做自然动作";
return defaults[index - 1] || `主体完成第 ${index} 段自然动作`;
}
export function runScriptNormalizerSkill(params: {
@ -114,13 +114,14 @@ export function runScriptNormalizerSkill(params: {
language === "en"
? "jitter, blur, artifacts, low quality"
: "抖动, 畸变, 低清, 伪影",
preferredTileIndex: parsePreferredTileIndex(line),
preferredTileIndex: parsePreferredTileIndex(line, maxShots),
});
}
return {
language,
globalStyle: fallbackStyle,
shotCount: normalizedShots.length,
shots: normalizedShots,
raw,
};

4
lib/video-agent/skills/storyboard-mapper.ts
View File

@ -2,7 +2,7 @@ import "server-only";
import type {
MappedStoryboardShot,
NineGridTile,
StoryboardTile,
NormalizedStoryboardScript,
} from "@/lib/video-agent/types";
@ -43,7 +43,7 @@ function buildShotPrompt(params: {
export function runStoryboardMapperSkill(params: {
script: NormalizedStoryboardScript;
tiles: NineGridTile[];
tiles: StoryboardTile[];
}): MappedStoryboardShot[] {
const { script, tiles } = params;
if (!tiles.length) return [];

378
lib/video-agent/storyboard-command-interpreter.ts Normal file
View File

@ -0,0 +1,378 @@
import "server-only";
import { generateText } from "ai";
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
import { getAgentAiModelWithProvider, getEnabledAiModelsWithProvider } from "@/lib/db/queries";
export type StoryboardCommandIntent = "confirm" | "revise" | "cancel" | "unclear";
export type StoryboardRevisionCommand =
| {
type: "update_prompt";
shotIndex: number;
prompt: string;
}
| {
type: "regenerate_hd";
shotIndex: number;
}
| null;
export interface StoryboardCommandContextItem {
index: number;
title?: string;
summary?: string;
}
const CONFIRM_PATTERNS = [
/确认开始生成视频/,
/确认开始生成/,
/按这个分镜生成/,
/就按这个生成/,
/按这个来生成/,
/开始生成视频/,
/开始吧/,
/可以开始了/,
/可以执行了/,
/直接生成吧/,
/直接开始吧/,
/先这样/,
/没问题开始/,
/确认无误/,
/就这样吧/,
];
const CANCEL_PATTERNS = [
/取消/,
/先不要/,
/暂停/,
/停止/,
/先别生成/,
/先别做/,
/不用生成了/,
/不用做了/,
];
const REVISE_HINT_PATTERNS = [
/修改/,
/调整/,
/改成/,
/改为/,
/换成/,
/换成/,
/做成/,
/重生/,
/重做/,
/重新生成/,
/重新做/,
/不要/,
/太糊/,
/太平/,
/太死板/,
];
function normalizeCompactText(text: string) {
return String(text || "").toLowerCase().replace(/\s+/g, "").trim();
}
function chineseNumberToInt(source: string): number | null {
const normalized = String(source || "").trim();
if (!normalized) return null;
if (/^\d+$/.test(normalized)) {
const parsed = Number(normalized);
return Number.isFinite(parsed) ? parsed : null;
}
const map: Record<string, number> = {
一: 1,
二: 2,
两: 2,
三: 3,
四: 4,
五: 5,
六: 6,
七: 7,
八: 8,
九: 9,
十: 10,
};
if (normalized === "十") return 10;
if (normalized.length === 1) {
return map[normalized] ?? null;
}
if (/^十[一二三四五六七八九]$/.test(normalized)) {
return 10 + (map[normalized.slice(1)] || 0);
}
if (/^[一二三四五六七八九]十$/.test(normalized)) {
return (map[normalized[0]!] || 0) * 10;
}
if (/^[一二三四五六七八九]十[一二三四五六七八九]$/.test(normalized)) {
return (map[normalized[0]!] || 0) * 10 + (map[normalized[2]!] || 0);
}
return null;
}
function resolveShotIndexFromText(text: string, totalShots?: number): number | null {
const normalized = String(text || "").trim();
if (!normalized) return null;
const explicitPatterns = [
/第\s*([一二三四五六七八九十两\d]{1,3})\s*(?:张|格|图|镜头|段)/i,
/把\s*([一二三四五六七八九十两\d]{1,3})\s*(?:张|格|图|镜头|段)/i,
/([一二三四五六七八九十两\d]{1,3})\s*(?:号)?(?:张|格|图|镜头|段)/i,
];
for (const pattern of explicitPatterns) {
const match = normalized.match(pattern);
if (!match?.[1]) continue;
const resolved = chineseNumberToInt(match[1]);
if (resolved && resolved >= 1 && (!totalShots || resolved <= totalShots)) {
return resolved;
}
}
if (totalShots && /最后一(?:张|格|图|镜头|段)/.test(normalized)) {
return totalShots;
}
if (totalShots) {
const reverseMatch = normalized.match(/倒数第\s*([一二三四五六七八九十两\d]{1,3})\s*(?:张|格|图|镜头|段)/i);
if (reverseMatch?.[1]) {
const reversed = chineseNumberToInt(reverseMatch[1]);
if (reversed && reversed >= 1 && reversed <= totalShots) {
return totalShots - reversed + 1;
}
}
}
return null;
}
function extractRevisionPrompt(text: string): string {
return String(text || "")
.replace(/^(?:请)?(?:把)?\s*第?\s*[一二三四五六七八九十两\d]{1,3}\s*(?:张|格|图|镜头|段)(?:动作)?\s*(?:修改|调整|改成|改为|换成|做成)?\s*[:：]?\s*/i, "")
.replace(/^(?:请)?(?:把)?\s*最后一(?:张|格|图|镜头|段)\s*(?:修改|调整|改成|改为|换成|做成)?\s*[:：]?\s*/i, "")
.trim();
}
export function detectStoryboardCommandIntent(text: string): StoryboardCommandIntent {
const normalized = normalizeCompactText(text);
if (!normalized) return "unclear";
if (CANCEL_PATTERNS.some((pattern) => pattern.test(normalized))) return "cancel";
if (CONFIRM_PATTERNS.some((pattern) => pattern.test(normalized))) return "confirm";
if (REVISE_HINT_PATTERNS.some((pattern) => pattern.test(normalized))) return "revise";
if (resolveShotIndexFromText(normalized) !== null) return "revise";
return "unclear";
}
function parseJsonBlock<T = Record<string, unknown>>(text: string): T | null {
const match = String(text || "").match(/\{[\s\S]*\}/);
if (!match) return null;
try {
return JSON.parse(match[0]) as T;
} catch {
return null;
}
}
type EnabledModelWithProvider = Awaited<
ReturnType<typeof import("@/lib/db/queries")["getEnabledAiModelsWithProvider"]>
>[number];
function isAgentGenerateTextProviderType(providerType: string): boolean {
return ["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType);
}
function isAgentGenerateTextCompatibleModel(item: EnabledModelWithProvider) {
if (!item?.model?.isEnabled || item.model.isSubModel) return false;
return isAgentGenerateTextProviderType(String(item.provider?.type || ""));
}
function pickReasoningModel(
configuredAgentModel: EnabledModelWithProvider | undefined,
models: EnabledModelWithProvider[],
) {
if (
configuredAgentModel &&
isAgentGenerateTextCompatibleModel(configuredAgentModel)
) {
return configuredAgentModel;
}
const candidates = models.filter((item) => isAgentGenerateTextCompatibleModel(item));
const zhipuModel = candidates.find((item) => item.provider.type === "zhipu");
return zhipuModel || candidates[0];
}
async function interpretStoryboardCommandWithLLM(params: {
text: string;
totalShots?: number;
items?: StoryboardCommandContextItem[];
}): Promise<{ intent: StoryboardCommandIntent; revision: StoryboardRevisionCommand } | null> {
try {
const allModels = await getEnabledAiModelsWithProvider();
const configuredAgentModel = await getAgentAiModelWithProvider();
const reasoningModelConfig = pickReasoningModel(configuredAgentModel, allModels);
if (!reasoningModelConfig) return null;
const model = await getLanguageModelById(reasoningModelConfig.model.id);
if (!model) return null;
const contextLines = (params.items || [])
.slice(0, 12)
.map((item) => `- 镜头${item.index}：${item.summary || item.title || "无描述"}`)
.join("\n");
const { text } = await generateText({
model,
system: `你是一个分镜任务指令解释器。
你需要判断用户当前这句话，是在：
- confirm：确认开始执行
- revise：修改某个镜头或重生某张高清图
- cancel：取消当前流程
- unclear：无法判断
只输出 JSON，不要输出解释：
{
"intent": "confirm|revise|cancel|unclear",
"action": "update_prompt|regenerate_hd|null",
"shotIndex": 1,
"prompt": "修改后的镜头文案，没有就空字符串"
}
规则：
1. 用户表达“开始、确认、就这样、按这个生成”时，intent=confirm。
2. 用户表达“取消、暂停、先别生成”时，intent=cancel。
3. 用户表达“改一下、调整、换成、重生、重做、重新生成”且指向某镜头时，intent=revise。
4. 如果用户说“最后一张/最后一格/最后一个镜头”，shotIndex 取最后一个。
5. 如果用户说“倒数第二张/倒数第二个镜头”，按总镜头数推算。
6. 如果 intent=revise 且是文字修改，action=update_prompt。
7. 如果 intent=revise 且是重生高清预览，action=regenerate_hd。
8. 如果无法确定具体镜头，intent=unclear。`,
prompt: [
`总镜头数：${Math.max(0, Number(params.totalShots || 0))}`,
contextLines ? `当前镜头草案：\n${contextLines}` : "",
`用户输入：${params.text}`,
]
.filter(Boolean)
.join("\n\n"),
temperature: 0.1,
maxTokens: 200,
abortSignal: AbortSignal.timeout(12000),
} as any);
const parsed = parseJsonBlock<{
intent?: string;
action?: string | null;
shotIndex?: number;
prompt?: string;
}>(text);
if (!parsed) return null;
const intent =
parsed.intent === "confirm" ||
parsed.intent === "revise" ||
parsed.intent === "cancel" ||
parsed.intent === "unclear"
? parsed.intent
: "unclear";
if (intent !== "revise") {
return { intent, revision: null };
}
const shotIndex = Math.max(1, Number(parsed.shotIndex || 0));
if (!Number.isFinite(shotIndex) || shotIndex <= 0) {
return { intent: "unclear", revision: null };
}
if (parsed.action === "regenerate_hd") {
return {
intent,
revision: {
type: "regenerate_hd",
shotIndex,
},
};
}
const prompt = String(parsed.prompt || "").trim();
if (!prompt) {
return { intent: "unclear", revision: null };
}
return {
intent,
revision: {
type: "update_prompt",
shotIndex,
prompt,
},
};
} catch {
return null;
}
}
export function parseStoryboardRevisionCommand(params: {
text: string;
totalShots?: number;
}): StoryboardRevisionCommand {
const source = String(params.text || "").trim();
if (!source) return null;
const shotIndex = resolveShotIndexFromText(source, params.totalShots);
if (!shotIndex) return null;
if (
/重生|重做|重新生成|重新做|再生成一版|再来一版|重新出图|重出高清图|高清图重来/i.test(source)
) {
return {
type: "regenerate_hd",
shotIndex,
};
}
const prompt = extractRevisionPrompt(source);
if (!prompt) {
return null;
}
return {
type: "update_prompt",
shotIndex,
prompt,
};
}
export async function interpretStoryboardCommand(params: {
text: string;
totalShots?: number;
items?: StoryboardCommandContextItem[];
}): Promise<{ intent: StoryboardCommandIntent; revision: StoryboardRevisionCommand }> {
const heuristicIntent = detectStoryboardCommandIntent(params.text);
const heuristicRevision = parseStoryboardRevisionCommand({
text: params.text,
totalShots: params.totalShots,
});
if (heuristicIntent === "confirm" || heuristicIntent === "cancel") {
return { intent: heuristicIntent, revision: null };
}
if (heuristicIntent === "revise" && heuristicRevision) {
return { intent: heuristicIntent, revision: heuristicRevision };
}
const llmInterpretation = await interpretStoryboardCommandWithLLM(params);
if (llmInterpretation) {
return llmInterpretation;
}
return {
intent: heuristicIntent,
revision: heuristicRevision,
};
}

110
lib/video-agent/storyboard-preference-memory.ts Normal file
View File

@ -0,0 +1,110 @@
import "server-only";
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
export interface StoryboardPreferenceMemory {
preferredGridType?: "grid_2x2" | "grid_3x2" | "grid_3x3";
preferredShotCount?: number;
prefersConfirmation?: boolean;
lastUsedGridType?: "grid_2x2" | "grid_3x2" | "grid_3x3";
lastUsedShotCount?: number;
}
function normalizeGridType(value: unknown) {
const normalized = String(value || "").trim();
if (normalized === "grid_2x2" || normalized === "grid_3x2" || normalized === "grid_3x3") {
return normalized;
}
return undefined;
}
function normalizeShotCount(value: unknown) {
const parsed = Number(value);
if (!Number.isFinite(parsed)) return undefined;
const rounded = Math.round(parsed);
if (rounded === 4 || rounded === 6 || rounded === 9) return rounded;
return undefined;
}
export function extractStoryboardPreferenceMemory(
workingState?: {
lastTurnSummary?: unknown;
} | null,
): StoryboardPreferenceMemory | null {
const summary = workingState?.lastTurnSummary;
if (!summary || typeof summary !== "object") return null;
const summaryRecord = summary as Record<string, unknown>;
const memory = summaryRecord.storyboardMemory;
if (!memory || typeof memory !== "object") return null;
const memoryRecord = memory as Record<string, unknown>;
const normalized: StoryboardPreferenceMemory = {
preferredGridType: normalizeGridType(memoryRecord.preferredGridType),
preferredShotCount: normalizeShotCount(memoryRecord.preferredShotCount),
prefersConfirmation:
typeof memoryRecord.prefersConfirmation === "boolean"
? memoryRecord.prefersConfirmation
: undefined,
lastUsedGridType: normalizeGridType(memoryRecord.lastUsedGridType),
lastUsedShotCount: normalizeShotCount(memoryRecord.lastUsedShotCount),
};
if (
!normalized.preferredGridType &&
!normalized.preferredShotCount &&
normalized.prefersConfirmation === undefined &&
!normalized.lastUsedGridType &&
!normalized.lastUsedShotCount
) {
return null;
}
return normalized;
}
export function buildStoryboardPreferenceMemoryUpdate(params: {
existing?: StoryboardPreferenceMemory | null;
preferredGridType?: "grid_2x2" | "grid_3x2" | "grid_3x3";
preferredShotCount?: number;
prefersConfirmation?: boolean;
lastUsedGridType?: "grid_2x2" | "grid_3x2" | "grid_3x3";
lastUsedShotCount?: number;
}) {
const next: StoryboardPreferenceMemory = {
preferredGridType: params.preferredGridType ?? params.existing?.preferredGridType,
preferredShotCount: params.preferredShotCount ?? params.existing?.preferredShotCount,
prefersConfirmation: params.prefersConfirmation ?? params.existing?.prefersConfirmation,
lastUsedGridType: params.lastUsedGridType ?? params.existing?.lastUsedGridType,
lastUsedShotCount: params.lastUsedShotCount ?? params.existing?.lastUsedShotCount,
};
return next;
}
export function getStoryboardPreferencePlanningHints(
threadState?: AgentThreadState | null,
): string[] {
const metadata = threadState?.metadata;
const memory =
metadata && typeof metadata === "object" && !Array.isArray(metadata)
? ((metadata.storyboardPreferenceMemory as StoryboardPreferenceMemory | undefined) || undefined)
: undefined;
if (!memory) return [];
const hints: string[] = [];
if (memory.preferredGridType) {
hints.push(`用户最近偏好宫格版式：${memory.preferredGridType}。`);
}
if (memory.preferredShotCount) {
hints.push(`用户最近常用镜头数：${memory.preferredShotCount} 段。`);
}
if (memory.prefersConfirmation === true) {
hints.push("用户偏好先确认分镜草案，再开始执行。");
}
if (memory.lastUsedGridType && memory.lastUsedShotCount) {
hints.push(
`用户最近一次分镜任务使用的是 ${memory.lastUsedGridType}（${memory.lastUsedShotCount} 段）。`
);
}
return hints;
}

152
lib/video-agent/storyboard-subagent.ts Normal file
View File

@ -0,0 +1,152 @@
import "server-only";
import type { AgentThreadState } from "@/lib/ai/runtime/agent-thread-state";
import type { AgentExecutionPlan, AgentPlannedToolCall } from "@/lib/ai/auto-agent-graph";
import type { AgentSpecialistRoute } from "@/lib/ai/specialists";
import type { AgentProjectToolDefinition } from "@/lib/ai/tools-registry";
import { getStoryboardPreferencePlanningHints } from "@/lib/video-agent/storyboard-preference-memory";
function normalizeCompactText(text: string) {
return String(text || "").toLowerCase().replace(/\s+/g, "").trim();
}
export function mentionsStoryboardConcept(text: string): boolean {
const normalized = normalizeCompactText(text);
if (!normalized) return false;
return /分镜|镜头图|镜头草案|镜头预览|storyboard|宫格|四宫格|六宫格|九宫格|拼图视频|分镜视频|高清图/.test(
normalized
);
}
export function hasStoryboardWorkflow(threadState?: AgentThreadState | null): boolean {
return threadState?.workflowState?.type === "storyboard_video";
}
export function buildStoryboardSpecialistRoute(params: {
text: string;
threadState?: AgentThreadState | null;
}): AgentSpecialistRoute | null {
const hasWorkflow = hasStoryboardWorkflow(params.threadState);
const mentionsStoryboard = mentionsStoryboardConcept(params.text);
if (!hasWorkflow && !mentionsStoryboard) return null;
const workflowSummary = params.threadState?.workflowState?.summary;
const workflowStatus = params.threadState?.workflowState?.status;
const hasPendingConfirmation = workflowStatus === "awaiting_user_confirmation";
const preferenceHints = getStoryboardPreferencePlanningHints(params.threadState);
const planningHints = [
hasWorkflow
? "当前回合与一个 storyboard workflow 强相关，优先理解为分镜流程里的确认、修改、重生或继续执行。"
: "当前回合明显在处理分镜/宫格任务，不要按普通单图生图或普通视频问答理解。",
"如果用户提到第N镜头、最后一格、倒数第二张、高清图重生，优先解释为对 storyboard 的局部修改。",
hasPendingConfirmation
? "当前任务仍处于待确认阶段，除非用户明确确认，否则不要推进到视频执行。"
: "如果用户目标是先做分镜图或宫格图，优先保证输出适合后续切片与连续镜头生成。",
...preferenceHints,
workflowSummary ? `当前 workflow 摘要：${workflowSummary}` : "",
].filter(Boolean);
return {
domain: "storyboard",
specialistId: "storyboard-subagent",
reason: hasWorkflow ? "当前回合命中 storyboard 工作流" : "当前回合命中分镜/宫格任务",
planningHints,
toolAllowlist: [
"respondDirect",
"generateImage",
"generateVideo",
"useSelectedCanvasImage",
"useLatestResultImage",
"useOriginalUploadImage",
"publishToCanvas",
],
};
}
function hasTool(
availableTools: AgentProjectToolDefinition[],
name: AgentPlannedToolCall["name"],
) {
return availableTools.some((tool) => tool.name === name);
}
function upsertTool(
toolCalls: AgentPlannedToolCall[],
name: AgentPlannedToolCall["name"],
reason: string,
options?: { prepend?: boolean },
) {
const existing = toolCalls.find((tool) => tool.name === name);
if (existing) return toolCalls;
const nextTool = { name, reason } satisfies AgentPlannedToolCall;
return options?.prepend ? [nextTool, ...toolCalls] : [...toolCalls, nextTool];
}
export function refinePlanWithStoryboardSubagent(params: {
text: string;
plan: AgentExecutionPlan;
availableTools: AgentProjectToolDefinition[];
threadState?: AgentThreadState | null;
}): AgentExecutionPlan {
const { text, plan, availableTools, threadState } = params;
const normalized = normalizeCompactText(text);
const hasWorkflow = hasStoryboardWorkflow(threadState);
const status = threadState?.workflowState?.status;
let toolCalls = [...plan.toolCalls];
const wantsGridImage =
/四宫格|六宫格|九宫格|宫格图|分镜图|镜头图|storyboard/.test(normalized);
const wantsStoryboardVideo =
/分镜视频|按分镜生成视频|按这个分镜生成|图生视频|镜头视频/.test(normalized);
if (hasWorkflow && status === "awaiting_user_confirmation") {
if (hasTool(availableTools, "respondDirect")) {
toolCalls = [{ name: "respondDirect", reason: "storyboard 子代理要求先处理确认/修改，不直接切换普通工具链" }];
}
return {
...plan,
nextAction: "respond",
intent: "chat",
responseMode: "direct_answer",
preferredCapability: "respond",
shouldUseStructuredReply: false,
toolCalls,
};
}
if (wantsGridImage && plan.nextAction === "generate_image") {
if (hasTool(availableTools, "publishToCanvas")) {
toolCalls = upsertTool(
toolCalls,
"publishToCanvas",
"storyboard 子代理要求把分镜图结果同步到画布，便于后续切片和确认"
);
}
return {
...plan,
toolCalls,
shouldUseStructuredReply: true,
preferredCapability: "generate_image",
};
}
if ((wantsStoryboardVideo || hasWorkflow) && plan.nextAction === "generate_video") {
if (hasTool(availableTools, "useLatestResultImage")) {
toolCalls = upsertTool(
toolCalls,
"useLatestResultImage",
"storyboard 子代理优先绑定最近的分镜图/镜头预览作为视频输入",
{ prepend: true }
);
}
return {
...plan,
toolCalls,
shouldUseStructuredReply: true,
preferredCapability: "generate_video",
};
}
return plan;
}

314
lib/video-agent/task-store.ts
View File

@ -24,6 +24,12 @@ export type VideoStoryboardTaskStatus =
| "failed"
| "cancelled";
export type VideoStoryboardTaskConfirmationStatus =
| "draft"
| "awaiting_user_confirmation"
| "confirmed"
| "rejected";
export type VideoStoryboardShotStatus =
| "pending"
| "running"
@ -31,6 +37,21 @@ export type VideoStoryboardShotStatus =
| "failed"
| "cancelled";
export type VideoStoryboardHdImageStatus =
| "pending"
| "running"
| "completed"
| "failed"
| "cancelled";
export type VideoStoryboardVideoStatus =
| "pending"
| "waiting_confirmation"
| "running"
| "completed"
| "failed"
| "cancelled";
function toConfidenceInt(confidence: number) {
if (!Number.isFinite(confidence)) return 0;
return Math.max(0, Math.min(10000, Math.round(confidence * 10000)));
@ -41,12 +62,16 @@ export async function createVideoStoryboardTask(params: {
chatId?: string;
providerType: string;
modelRef?: string;
gridType?: string | null;
gridRows?: number | null;
gridCols?: number | null;
sourceImageUrl: string;
isNineGrid: boolean;
detectionConfidence: number;
normalizedScript?: Record<string, any> | null;
totalShots: number;
status?: VideoStoryboardTaskStatus;
confirmationStatus?: VideoStoryboardTaskConfirmationStatus;
}) {
const now = new Date();
const [task] = await db
@ -56,11 +81,22 @@ export async function createVideoStoryboardTask(params: {
chatId: params.chatId || null,
providerType: params.providerType,
modelRef: params.modelRef || null,
gridType: params.gridType || null,
gridRows:
typeof params.gridRows === "number" && Number.isFinite(params.gridRows)
? Math.max(1, Math.round(params.gridRows))
: null,
gridCols:
typeof params.gridCols === "number" && Number.isFinite(params.gridCols)
? Math.max(1, Math.round(params.gridCols))
: null,
sourceImageUrl: params.sourceImageUrl,
isNineGrid: params.isNineGrid,
detectionConfidence: toConfidenceInt(params.detectionConfidence),
normalizedScript: params.normalizedScript || null,
status: params.status || "running",
confirmationStatus: params.confirmationStatus || "draft",
hdImageStageStatus: "pending",
currentShot: 0,
totalShots: Math.max(0, Math.round(params.totalShots)),
createdAt: now,
@ -87,11 +123,17 @@ export async function createVideoStoryboardShotTasks(params: {
tileIndex: shot.tileIndex,
tileImageUrl: shot.tileImageUrl,
upscaledImageUrl: shot.upscaled ? shot.inputImageUrl : null,
hdImageUrl: null,
hdImagePrompt: shot.prompt,
prompt: shot.prompt,
modelId: null,
fallbackChain: params.fallbackChain,
attemptCount: 0,
status: "pending",
hdImageStatus: "pending",
videoStatus: "pending",
userApproved: false,
userApprovedAt: null,
createdAt: now,
updatedAt: now,
}))
@ -240,6 +282,251 @@ export async function finalizeVideoStoryboardTask(params: {
.where(eq(videoStoryboardTask.id, params.taskId));
}
export async function markVideoStoryboardAwaitingConfirmation(params: {
taskId: string;
}) {
const now = new Date();
await db
.update(videoStoryboardTask)
.set({
confirmationStatus: "awaiting_user_confirmation",
status: "pending",
updatedAt: now,
})
.where(eq(videoStoryboardTask.id, params.taskId));
await db
.update(videoStoryboardShotTask)
.set({
videoStatus: "waiting_confirmation",
userApproved: false,
userApprovedAt: null,
updatedAt: now,
})
.where(eq(videoStoryboardShotTask.parentTaskId, params.taskId));
}
export async function markVideoStoryboardHdRunning(params: {
taskId: string;
shotIndex: number;
}) {
const now = new Date();
await db
.update(videoStoryboardShotTask)
.set({
hdImageStatus: "running",
updatedAt: now,
})
.where(
and(
eq(videoStoryboardShotTask.parentTaskId, params.taskId),
eq(videoStoryboardShotTask.shotIndex, params.shotIndex)
)
);
await db
.update(videoStoryboardTask)
.set({
hdImageStageStatus: "running",
updatedAt: now,
})
.where(eq(videoStoryboardTask.id, params.taskId));
}
export async function markVideoStoryboardHdCompleted(params: {
taskId: string;
shotIndex: number;
hdImageUrl: string;
hdImagePrompt?: string;
}) {
const now = new Date();
await db
.update(videoStoryboardShotTask)
.set({
hdImageStatus: "completed",
hdImageUrl: params.hdImageUrl,
hdImagePrompt: params.hdImagePrompt || null,
updatedAt: now,
})
.where(
and(
eq(videoStoryboardShotTask.parentTaskId, params.taskId),
eq(videoStoryboardShotTask.shotIndex, params.shotIndex)
)
);
}
export async function markVideoStoryboardHdFailed(params: {
taskId: string;
shotIndex: number;
errorMessage: string;
}) {
const now = new Date();
await db
.update(videoStoryboardShotTask)
.set({
hdImageStatus: "failed",
errorMessage: params.errorMessage.slice(0, 4000),
updatedAt: now,
})
.where(
and(
eq(videoStoryboardShotTask.parentTaskId, params.taskId),
eq(videoStoryboardShotTask.shotIndex, params.shotIndex)
)
);
}
export async function syncVideoStoryboardHdStageStatus(taskId: string) {
const rows = await db
.select({
hdImageStatus: videoStoryboardShotTask.hdImageStatus,
})
.from(videoStoryboardShotTask)
.where(eq(videoStoryboardShotTask.parentTaskId, taskId));
const total = rows.length;
const completed = rows.filter((row) => row.hdImageStatus === "completed").length;
const failed = rows.filter((row) => row.hdImageStatus === "failed").length;
const running = rows.filter((row) => row.hdImageStatus === "running").length;
let hdImageStageStatus: "pending" | "running" | "completed" | "failed" = "pending";
if (running > 0) {
hdImageStageStatus = "running";
} else if (completed === total && total > 0) {
hdImageStageStatus = "completed";
} else if (failed === total && total > 0) {
hdImageStageStatus = "failed";
} else if (completed > 0 || failed > 0) {
hdImageStageStatus = "running";
}
await db
.update(videoStoryboardTask)
.set({
hdImageStageStatus,
updatedAt: new Date(),
})
.where(eq(videoStoryboardTask.id, taskId));
return { total, completed, failed, hdImageStageStatus };
}
export async function markVideoStoryboardConfirmed(params: {
taskId: string;
messageId?: string;
}) {
const now = new Date();
await db
.update(videoStoryboardTask)
.set({
confirmationStatus: "confirmed",
confirmedAt: now,
confirmedByMessageId: params.messageId || null,
updatedAt: now,
})
.where(eq(videoStoryboardTask.id, params.taskId));
await db
.update(videoStoryboardShotTask)
.set({
videoStatus: sql`CASE
WHEN ${videoStoryboardShotTask.status} = 'completed' THEN 'completed'
WHEN ${videoStoryboardShotTask.status} = 'failed' THEN 'failed'
WHEN ${videoStoryboardShotTask.status} = 'running' THEN 'running'
ELSE 'pending'
END`,
userApproved: true,
userApprovedAt: now,
updatedAt: now,
})
.where(eq(videoStoryboardShotTask.parentTaskId, params.taskId));
}
export async function markVideoStoryboardDraftForRevision(params: {
taskId: string;
}) {
const now = new Date();
await db
.update(videoStoryboardTask)
.set({
confirmationStatus: "draft",
confirmedAt: null,
confirmedByMessageId: null,
updatedAt: now,
})
.where(eq(videoStoryboardTask.id, params.taskId));
await db
.update(videoStoryboardShotTask)
.set({
videoStatus: "waiting_confirmation",
userApproved: false,
userApprovedAt: null,
updatedAt: now,
})
.where(eq(videoStoryboardShotTask.parentTaskId, params.taskId));
}
export async function updateVideoStoryboardShotPrompt(params: {
taskId: string;
shotIndex: number;
prompt: string;
}) {
const now = new Date();
await db
.update(videoStoryboardShotTask)
.set({
prompt: params.prompt.slice(0, 4000),
hdImagePrompt: params.prompt.slice(0, 4000),
updatedAt: now,
})
.where(
and(
eq(videoStoryboardShotTask.parentTaskId, params.taskId),
eq(videoStoryboardShotTask.shotIndex, params.shotIndex)
)
);
}
export async function cancelVideoStoryboardTaskByUser(params: {
taskId: string;
error?: string;
}) {
const now = new Date();
await db
.update(videoStoryboardTask)
.set({
status: "cancelled",
confirmationStatus: "rejected",
error: params.error ? params.error.slice(0, 4000) : "user_cancelled",
updatedAt: now,
completedAt: now,
})
.where(eq(videoStoryboardTask.id, params.taskId));
await db
.update(videoStoryboardShotTask)
.set({
status: sql`CASE
WHEN ${videoStoryboardShotTask.status} IN ('completed', 'failed') THEN ${videoStoryboardShotTask.status}
ELSE 'cancelled'
END`,
videoStatus: sql`CASE
WHEN ${videoStoryboardShotTask.videoStatus} IN ('completed', 'failed') THEN ${videoStoryboardShotTask.videoStatus}
ELSE 'cancelled'
END`,
userApproved: false,
userApprovedAt: null,
updatedAt: now,
completedAt: sql`CASE
WHEN ${videoStoryboardShotTask.completedAt} IS NULL THEN ${now}
ELSE ${videoStoryboardShotTask.completedAt}
END`,
})
.where(eq(videoStoryboardShotTask.parentTaskId, params.taskId));
}
export async function syncVideoStoryboardTaskStatus(taskId: string) {
const rows = await db
.select({
@ -315,6 +602,33 @@ export async function getVideoStoryboardTaskDetail(taskId: string) {
return { task, shots };
}
export async function getLatestActiveVideoStoryboardTaskByChatId(params: {
chatId: string;
}) {
const taskRows = await db
.select()
.from(videoStoryboardTask)
.where(
and(
eq(videoStoryboardTask.chatId, params.chatId),
inArray(videoStoryboardTask.status, ["pending", "running"])
)
)
.orderBy(desc(videoStoryboardTask.createdAt))
.limit(1);
const task = taskRows[0];
if (!task) return null;
const shots = await db
.select()
.from(videoStoryboardShotTask)
.where(eq(videoStoryboardShotTask.parentTaskId, task.id))
.orderBy(asc(videoStoryboardShotTask.shotIndex));
return { task, shots };
}
export async function listVideoStoryboardTasks(params: {
page: number;
pageSize: number;

61
lib/video-agent/types.ts
View File

@ -1,5 +1,7 @@
import type { VideoStoryboardConfig } from "@/lib/video-agent/config";
export type StoryboardGridType = "grid_2x2" | "grid_3x2" | "grid_3x3";
export type NormalizedStoryboardShot = {
index: number;
durationSec: number;
@ -13,36 +15,50 @@ export type NormalizedStoryboardShot = {
export type NormalizedStoryboardScript = {
language: "zh" | "en" | "mixed";
globalStyle: string;
shotCount: number;
shots: NormalizedStoryboardShot[];
raw: string;
};
export type NineGridCropBox = {
export type StoryboardCropBox = {
index: number;
row: number;
col: number;
x: number;
y: number;
w: number;
h: number;
};
export type NineGridTile = {
export type StoryboardTile = {
index: number;
row: number;
col: number;
imageUrl: string;
width: number;
height: number;
};
export type NineGridDetectionResult = {
export type StoryboardDetectionResult = {
isStoryboardGrid: boolean;
isNineGrid: boolean;
gridType: StoryboardGridType | null;
rows: number;
cols: number;
shotCount: number;
confidence: number;
qualityScore: number;
cropBoxes: NineGridCropBox[];
cropBoxes: StoryboardCropBox[];
width: number;
height: number;
sourceImageUrl: string;
tiles: NineGridTile[];
tiles: StoryboardTile[];
};
export type NineGridCropBox = StoryboardCropBox;
export type NineGridTile = StoryboardTile;
export type NineGridDetectionResult = StoryboardDetectionResult;
export type MappedStoryboardShot = {
shotIndex: number;
tileIndex: number;
@ -78,6 +94,9 @@ export type StoryboardProgressEvent =
provider: string;
taskId: string;
totalShots: number;
gridType?: StoryboardGridType;
rows?: number;
cols?: number;
confidence: number;
sourceImageUrl: string;
shots: Array<{
@ -86,6 +105,37 @@ export type StoryboardProgressEvent =
prompt: string;
}>;
}
| {
stage: "hd_shot_completed";
provider: string;
taskId: string;
shotIndex: number;
imageUrl: string;
prompt: string;
}
| {
stage: "hd_shot_failed";
provider: string;
taskId: string;
shotIndex: number;
error: string;
}
| {
stage: "awaiting_confirmation";
provider: string;
taskId: string;
totalShots: number;
gridType?: StoryboardGridType;
rows?: number;
cols?: number;
message: string;
shots: Array<{
shotIndex: number;
tileImageUrl: string;
previewImageUrl: string;
prompt: string;
}>;
}
| {
stage: "shot_retry";
provider: string;
@ -149,6 +199,7 @@ export type RuxaNineGridPipelineResult = {
confidence?: number;
markdown?: string;
videoUrls?: string[];
previewImageUrls?: string[];
modelRefs?: string[];
metadata?: Record<string, any>;
reason?: string;

29
lib/video-agent/user-task-access.ts Normal file
View File

@ -0,0 +1,29 @@
import "server-only";
import { getVideoStoryboardTaskDetail } from "@/lib/video-agent/task-store";
export class StoryboardTaskAccessError extends Error {
status: number;
constructor(message: string, status: number) {
super(message);
this.status = status;
}
}
export async function requireStoryboardTaskAccess(params: {
taskId: string;
userId: string;
}) {
const detail = await getVideoStoryboardTaskDetail(params.taskId);
if (!detail) {
throw new StoryboardTaskAccessError("任务不存在", 404);
}
if (String((detail.task as any).userId || "").trim() !== params.userId) {
throw new StoryboardTaskAccessError("无权访问该分镜任务", 403);
}
return detail;
}

5
types/agent.ts
View File

@ -1,4 +1,4 @@
export type AgentPresetType = "image" | "video";
export type AgentPresetType = "image" | "video" | "text";
export type AgentPresetVisibility = "public" | "private";
export type AgentCategoryScope = "public" | "private" | "global";
export type AgentPresetStatus = "active" | "disabled" | "hidden" | "blocked";
@ -70,6 +70,7 @@ export interface AgentPreset {
ownerUserId: string;
name: string;
type: AgentPresetType;
secondaryType?: string | null;
visibility: AgentPresetVisibility;
categoryId?: string | null;
categoryPath?: AgentCategoryPath | null;
@ -144,6 +145,7 @@ export interface AgentExecutionSnapshot {
export interface AgentCreatePayload {
name: string;
type: AgentPresetType;
secondaryType?: string | null;
visibility: AgentPresetVisibility;
categoryPath?: AgentCategoryPath | null;
description?: string;
@ -156,6 +158,7 @@ export interface AgentCreatePayload {
export interface AgentUpdatePayload {
name?: string;
type?: AgentPresetType;
secondaryType?: string | null;
visibility?: AgentPresetVisibility;
categoryPath?: AgentCategoryPath | null;
description?: string | null;

176
实时画布新功能文档/fal-realtime-image-api-contract.md
View File

@ -1,176 +0,0 @@
# fal 实时生图前后端接口契约（V1）
> 更新日期：2026-02-19
> 目的：统一前端、后端、测试对接口和状态码的理解
---
## 1. 约定
- Base URL：`/api`
- 所有时间戳：毫秒 UTC
- 所有 JSON：`application/json`
- 鉴权：Cookie Session 或 `Authorization: Bearer <token>`（按项目现状）
---
## 2. 配置下发
### `GET /api/realtime/config`
用途：前端启动时拉取实时参数、限制和开关。
响应 `200`：
```json
{
"session": {
"idleTimeoutMs": 120000
},
"transport": {
"throttleInterval": 100,
"maxBuffering": 2
},
"image": {
"maxInputResolution": 1024,
"jpegQuality": 0.55
},
"model": {
"id": "fal-ai/flux-2/klein/realtime",
"defaults": {
"steps": 3,
"scheduleMu": 2.3,
"feedback": 0.95,
"imageSize": "square"
},
"limits": {
"steps": { "min": 1, "max": 6 },
"feedback": { "min": 0.7, "max": 1.0 }
}
},
"features": {
"history": true,
"abCompare": true
}
}
```
错误：
- `401` 未登录
- `500` 服务异常
---
## 3. 指标上报
### `POST /api/realtime/metrics`
用途：收集端到端延迟、掉帧、重连、错误等指标。
请求：
```json
{
"sessionId": "rt_abc123",
"events": [
{
"name": "frame_result",
"ts": 1760000000000,
"seq": 102,
"latencyMs": 734,
"queueDepth": 1,
"networkType": "4g",
"deviceClass": "desktop"
}
]
}
```
响应：
```json
{
"ok": true
}
```
错误：
- `400` 参数不合法
- `413` 批量过大
- `429` 超频
---
## 4. fal 代理
### `GET/POST /api/fal/proxy/*`
用途：代理 fal API，隐藏 `FAL_KEY`。
说明：
- 前端 SDK 统一设置：`fal.config({ proxyUrl: "/api/fal/proxy" })`
- 该路由由 `@fal-ai/server-proxy` 托管，不建议自行改协议
后端必须加：
- 来源校验（Origin/Referer）
- 用户/IP 限流
- 请求日志（脱敏）
---
## 5. 前端状态码映射（用户提示）
- `401/403`：登录过期或权限不足 -> 提示重新登录
- `429`：操作过快 -> 自动降频并提示
- `5xx`：服务临时不可用 -> 自动重试 + 提示稍后再试
---
## 6. 帧序号策略（重要）
- FE 本地自增 `seq`
- 发送时记录：`lastSentSeq`
- 回包渲染时检查：`seq >= lastRenderedSeq` 才允许落屏
- 否则丢弃旧帧
---
## 7. 限流建议
- 用户维度：60 req/min
- IP 维度：120 req/min
- 指标上报接口：300 events/min
触发限流响应：
```json
{
"code": "RATE_LIMITED",
"message": "Too many requests",
"retryAfterMs": 1200
}
```
---
## 8. 错误对象约定
```json
{
"code": "INVALID_PARAM",
"message": "steps out of range",
"requestId": "req_xxx"
}
```
字段：
- `code`：机器可读错误码
- `message`：用户可读信息
- `requestId`：排障追踪 ID
---
## 9. 联调清单
- [ ] FE 能拿到 `/api/realtime/config`
- [ ] FE 通过 `/api/fal/proxy` 成功连接并回图
- [ ] 429 时 FE 自动调高节流
- [ ] 断线后自动重连并恢复
- [ ] 指标能在监控系统查询

449
实时画布新功能文档/fal-realtime-image-dev-guide.md
View File

@ -1,449 +0,0 @@
# fal.ai 实时生图开发文档（接近官方交互体验版）
> 版本：v2.0
> 更新日期：2026-02-19
> 范围：实时图片生成/编辑（不含视频）
> 目标模型：`fal-ai/flux-2/klein/realtime`
---
## 1. 文档目标
这份文档用于让前后端团队直接开工，重点不是“能跑起来”，而是“交互体验接近官方 Realtime”。
### 1.1 成功标准（必须量化）
- 首帧可见延迟（TFF）P50 <= 800ms，P95 <= 1500ms
- 连续交互刷新频率达到 6~12 FPS（取决于设备与网络）
- 拖拽/涂抹过程中 UI 不阻塞主线程（卡顿帧 < 5%）
- 断线重连成功率 > 99%
- 前端不泄露长期密钥 `FAL_KEY`
### 1.2 非目标
- 视频实时生成
- 复杂多模型路由
- 企业级权限系统（本期只做基础鉴权+限流）
---
## 2. 体验对标：你要做成什么感觉
接近官方实时感的核心不是“单次图好看”，而是“连续反馈稳定”。
### 2.1 用户可感知的关键点
1. 用户一动笔，画面很快有变化（低延迟）
2. 连续几帧风格连贯，不要每帧“跳风格”（时序一致性）
3. 结果总是追着“最新输入”走，不被旧请求拖住（最新优先）
4. 网络抖动时不会完全卡死（降级与重连）
### 2.2 交互策略总原则
- 本地先反馈，远端再纠正：先显示本地编辑痕迹，再被 AI 结果覆盖
- 只保留最新帧：旧帧到达就丢弃
- 高频输入要合并：不能每个鼠标事件都发请求
---
## 3. 系统架构（前后端职责）
```mermaid
flowchart LR
U[User] --> FE[Frontend Studio]
FE --> ENC[Encode Worker]
ENC --> RTC[Realtime Client]
RTC --> PX[/api/fal/proxy]
PX --> FAL[fal Realtime Model]
FAL --> RTC
RTC --> REN[Preview Renderer]
REN --> FE
FE --> OBS[Metrics]
PX --> OBS
```
### 3.1 前端职责
- 画布交互、参数面板、提示词输入
- 输入合并（节流、去抖、最新优先）
- 连接管理（连接、重连、失败提示）
- 预览渲染（只显示最新可用帧）
### 3.2 后端职责
- 代理 fal API（隐藏密钥）
- 对用户请求做速率限制、来源校验、审计日志
- 下发配置（参数上限、限流阈值、特性开关）
- 统一监控和告警
---
## 4. 前端功能清单与交互细节（重点）
## 4.1 页面模块
- `CanvasStage`：主画布，支持画笔、橡皮、遮罩、平移缩放
- `PromptBar`：Prompt 输入、负向词、随机种子开关
- `ControlPanel`：步数、强度、尺寸、风格预设
- `HistoryStrip`：结果快照列表（最多保留 N 张）
- `ConnectionBadge`：连接状态（连接中/实时中/重连中/失败）
## 4.2 用户交互流程（逐帧）
1. `pointerdown`：记录起点，进入本地绘制模式
2. `pointermove`：本地画布实时渲染笔触
3. 调度器每 80~120ms 取一次画布快照（不是每次 move 都取）
4. 快照进入编码 Worker 转 Data URI（JPEG）
5. 编码完成后推送到 `connection.send`
6. `onResult` 返回图像帧，渲染层替换预览
7. `pointerup` 立即触发一次 `flush`（强制送最终状态）
## 4.3 关键交互规则（必须实现）
- Prompt 输入时：
- 正在输入不触发（debounce 300~500ms）
- Enter 或输入停止后立即送一帧
- 滑杆参数变更时：
- 拖动过程中 120ms 节流
- 松手后强制再发一帧
- 画布连续操作时：
- 永远“latest wins”，旧序号响应不落屏
## 4.4 本地与远端双层渲染
- 层 1（本地编辑层）：即时反馈笔触，0 延迟
- 层 2（AI 结果层）：服务端回图，覆盖显示
- 层 3（UI Overlay）：网格、框选、蒙版高亮
这样用户不会感觉“操作没反应”。
---
## 5. 前端实现设计（工程级）
## 5.1 状态管理建议（Zustand/Redux 均可）
```ts
interface RealtimeState {
sessionId: string;
status: "idle" | "connecting" | "live" | "retrying" | "error";
prompt: string;
seed: number | null;
params: {
steps: number;
scheduleMu: number;
feedback: number;
imageSize: "square" | "square_hd";
};
seq: number;
lastSentSeq: number;
lastRenderedSeq: number;
queueDepth: number;
}
```
## 5.2 调度器（发送侧）
- 输入事件写入 `dirtyFlag`
- `requestAnimationFrame` 循环里检查：
- 距离上次发送 >= `throttleInterval`
- 当前无编码任务或队列未超阈值
- 满足条件才触发截图+编码+发送
## 5.3 编码 Worker（必须）
主线程不要做重编码，避免掉帧。
职责：
- 接收 `ImageBitmap` 或 `OffscreenCanvas`
- 压缩为 JPEG（质量 0.45~0.65）
- 返回 Data URI（或 bytes + MIME）
## 5.4 结果渲染策略
- 每条发送都附带 `seq`
- `onResult` 解析到响应时，如 `seq < lastRenderedSeq` 直接丢弃
- 只渲染最新帧
## 5.5 连接管理
- 使用 `connectionKey` 保证组件重渲染不重建连接
- 自动重连：500ms -> 1s -> 2s -> 4s -> 8s（上限 10s）
- 重连成功立即发送“最新完整画布+参数”
---
## 6. 前端代码骨架（可直接改造成项目代码）
```ts
import { fal } from "@fal-ai/client";
fal.config({ proxyUrl: "/api/fal/proxy" });
const realtime = fal.realtime.connect("fal-ai/flux-2/klein/realtime", {
connectionKey: "studio-main",
throttleInterval: 100,
maxBuffering: 2,
onResult: (result) => {
const image = result.images?.[result.images.length - 1];
if (!image?.content) return;
const seq = extractSeq(result) ?? 0;
if (seq < store.getState().lastRenderedSeq) return;
store.setState({ lastRenderedSeq: seq });
renderAIFrame(image.content, image.content_type || "image/jpeg");
},
onError: (err) => {
markConnectionError(err);
},
});
async function sendLatestFrame(payload: {
imageDataUri: string;
prompt: string;
seed?: number;
seq: number;
}) {
realtime.send({
image_url: payload.imageDataUri,
prompt: payload.prompt,
image_size: "square",
num_inference_steps: 3,
schedule_mu: 2.3,
seed: payload.seed,
output_feedback_strength: 0.95,
// 如果业务侧支持 metadata，可带 seq 用于对齐
// metadata: { seq: payload.seq }
});
}
```
---
## 7. 后端实现（详细）
## 7.1 最小后端模块
- `fal proxy route`：官方代理
- `auth middleware`：登录态检查（可选）
- `rate limiter`：按 userId/ip 双维限速
- `config endpoint`：给前端下发可调参数
- `metrics ingest`：收集前端埋点
## 7.2 目录建议
```txt
src/
app/
api/
fal/
proxy/
route.ts
realtime/
config/route.ts
metrics/route.ts
server/
rate-limit.ts
auth.ts
observability.ts
```
## 7.3 Proxy 路由（必须）
```ts
// src/app/api/fal/proxy/route.ts
import { route } from "@fal-ai/server-proxy/nextjs";
export const { GET, POST } = route;
```
```bash
# .env
FAL_KEY="key_id:key_secret"
```
## 7.4 参数下发接口（建议）
`GET /api/realtime/config`
返回示例：
```json
{
"throttleInterval": 100,
"maxBuffering": 2,
"jpegQuality": 0.55,
"maxInputResolution": 1024,
"defaults": {
"steps": 3,
"scheduleMu": 2.3,
"feedback": 0.95,
"imageSize": "square"
},
"limits": {
"steps": { "min": 1, "max": 6 },
"feedback": { "min": 0.7, "max": 1.0 }
}
}
```
## 7.5 埋点上报接口（建议）
`POST /api/realtime/metrics`
请求示例：
```json
{
"sessionId": "rt_abc123",
"event": "frame_result",
"ts": 1760000000000,
"latencyMs": 734,
"queueDepth": 1,
"network": "4g"
}
```
## 7.6 限流策略（建议起步）
- 每用户：60 请求/分钟（发送帧）
- 每 IP：120 请求/分钟
- 触发后返回 429 + `retryAfter`
- 前端收到 429：自动把节流增大到 150~200ms
---
## 8. 前后端协议与时序（必须统一）
## 8.1 时序图
```mermaid
sequenceDiagram
participant U as User
participant FE as Frontend
participant W as Encode Worker
participant PX as Backend Proxy
participant F as fal Realtime
U->>FE: 画布操作/改Prompt
FE->>W: snapshot + encode
W-->>FE: dataURI
FE->>PX: realtime send
PX->>F: forward
F-->>PX: result frame
PX-->>FE: result frame
FE-->>U: render latest frame
```
## 8.2 序号对齐（强烈建议）
- FE 发送每帧时维护本地 `seq`
- FE 端维护 `lastSentSeq` / `lastRenderedSeq`
- 到达乱序时只保留最新
---
## 9. 参数策略（默认值 + 调优步骤）
## 9.1 默认值
- `num_inference_steps=3`
- `schedule_mu=2.3`
- `output_feedback_strength=0.95`
- `image_size=square`
- `seed`：默认固定（用户可切换随机）
## 9.2 调优顺序
1. 先把延迟打下来：分辨率/JPEG 质量/节流
2. 再调连续性：固定 seed + feedback
3. 最后调画质：必要时增加 steps（慎重）
---
## 10. 性能预算（必须监控）
把每帧流程拆成预算：
- 画布快照：<= 8ms
- 编码：<= 20ms
- 网络+推理+回传：<= 900ms（P50）
- 解码+渲染：<= 16ms
超过预算时降级：
- 提高 `throttleInterval`
- 降输入分辨率
- 降 JPEG 质量
- 暂停插值/高级效果
---
## 11. 体验增强（接近官方手感的关键）
## 11.1 操作手感
- 光标附近显示笔刷圈和强度
- `space + drag` 平移，`wheel` 缩放
- 参数改动后有轻微过渡动画（100~180ms）
## 11.2 生成手感
- 显示“正在生成”但不阻塞输入
- 回图时使用淡入（80~120ms）避免闪屏
- 支持 A/B 快照对比（按住键切换前后）
## 11.3 可恢复性
- 自动保存最近 N 次快照到 IndexedDB
- 页面刷新后可恢复最近工作态
---
## 12. 错误与降级策略
## 12.1 用户可见文案规范
- 网络波动：`网络不稳定，正在自动重连...`
- 触发限流：`操作过于频繁，已自动降低刷新频率`
- 服务异常：`生成服务暂时不可用，请稍后重试`
## 12.2 技术处理
- WS 断开：自动重连 + 状态提示
- 429：提升节流、短暂停发
- 5xx：指数退避重试，最大 5 次
- 超时：超过阈值主动丢弃旧请求
---
## 13. 测试计划（按真实体验验收）
## 13.1 交互测试
- 连续涂抹 3 分钟，观察是否稳定刷新
- 快速修改 prompt，结果是否快速“跟手”
- 缩放+平移+绘制组合操作是否卡顿
## 13.2 弱网测试
- 模拟高延迟（200ms+）与抖动
- 验证重连和自动降频是否生效
## 13.3 安全测试
- 全站抓包确认无 `FAL_KEY`
- 代理接口防滥用（限流、来源）
---
## 14. 上线清单
- [ ] Proxy 可用且仅服务端持有长期密钥
- [ ] 前端已启用节流、丢旧帧、重连
- [ ] 监控看板有 TFF/P95/错误率/重连次数
- [ ] 429 与 5xx 有可验证的降级路径
- [ ] 移动端和桌面端都可用
---
## 15. 两周落地计划
- D1-D2：Proxy + Realtime 基础链路 + 画布最小版
- D3-D4：调度器、编码 Worker、序号对齐
- D5-D6：参数面板、Prompt 交互、快照历史
- D7-D8：错误与重连、429 降级、状态提示
- D9-D10：埋点接入、性能优化、弱网压测
- D11-D12：UI 手感优化（淡入、过渡、快捷键）
- D13-D14：灰度发布与回归
---
## 16. 官方参考链接
- [Real-time Quickstart](https://docs.fal.ai/model-apis/real-time/quickstart/)
- [Flux 2 klein realtime API](https://fal.ai/models/fal-ai/flux-2/klein/realtime/api)
- [Keeping fal API Secrets Safe](https://docs.fal.ai/model-apis/real-time/secrets/)
- [JavaScript Realtime Client](https://docs.fal.ai/reference/client-libraries/javascript/realtime)
- [Next.js Integration](https://docs.fal.ai/model-apis/integrations/nextjs)
- [Error Reference](https://docs.fal.ai/model-apis/errors/)
---
## 17. 给开发团队的实施备注
- 先“保证反馈连续”再“追求最高画质”
- 任何时候都优先响应最新输入
- 把输入合并、编码异步、旧帧丢弃做到位，体验会非常接近官方
- 如果后续要加视频，可复用当前状态机与连接管理，不要推倒重来