From 20898e1ac1bac2fbe755bd05e7e49f043f4043a8 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 5 Mar 2026 14:48:24 +0800 Subject: [PATCH] feat(video-agent): add ruxa nine-grid storyboard skill pipeline --- app/(chat)/api/chat/route.ts | 148 ++++- components/canvas/CanvasChatPanel.tsx | 182 ++++++ ...链路开发文档-含九宫格视频Agent技能化方案.md | 599 ++++++++++++++++++ lib/video-agent/config.ts | 81 +++ .../pipeline/ruxa-nine-grid-pipeline.ts | 316 +++++++++ lib/video-agent/skills/nine-grid-detector.ts | 251 ++++++++ lib/video-agent/skills/script-normalizer.ts | 109 ++++ lib/video-agent/skills/storyboard-mapper.ts | 73 +++ lib/video-agent/skills/tile-upscaler.ts | 18 + lib/video-agent/skills/video-sequencer.ts | 149 +++++ lib/video-agent/types.ts | 152 +++++ 11 files changed, 2061 insertions(+), 17 deletions(-) create mode 100644 docs/系统全链路开发文档-含九宫格视频Agent技能化方案.md create mode 100644 lib/video-agent/config.ts create mode 100644 lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts create mode 100644 lib/video-agent/skills/nine-grid-detector.ts create mode 100644 lib/video-agent/skills/script-normalizer.ts create mode 100644 lib/video-agent/skills/storyboard-mapper.ts create mode 100644 lib/video-agent/skills/tile-upscaler.ts create mode 100644 lib/video-agent/skills/video-sequencer.ts create mode 100644 lib/video-agent/types.ts diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 0d73d8b2b..12bca3478 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -57,6 +57,9 @@ import { ChatSDKError } from "@/lib/errors"; import { extractImageUrlsFromText } from "@/lib/image-url"; import { videoArchiver } from "@/lib/storage/video-archiver"; import { uploadFile } from "@/lib/storage/service"; +import { getVideoStoryboardConfig } from "@/lib/video-agent/config"; +import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline"; +import type { StoryboardModelCandidate } from "@/lib/video-agent/types"; import type { ChatMessage } from "@/lib/types"; import type { AppUsage } from "@/lib/usage"; import { convertToUIMessages, generateUUID } from "@/lib/utils"; @@ -359,6 +362,49 @@ function selectBestVideoSubModelForMode( })[0]; } +async function resolveStoryboardModelCandidates( + selectedModelRef: string +): Promise { + const selectedModel = await getAiModelById({ id: selectedModelRef }).catch(() => undefined); + if (!selectedModel) return []; + + const configuredFallbackIds = [ + selectedModel.fallbackModelId1, + selectedModel.fallbackModelId2, + ].filter((item): item is string => Boolean(String(item || "").trim())); + + const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []); + const sameProvider = allModels + .filter( + ({ model, provider }: any) => + provider?.id === selectedModel.providerId && + model?.id !== selectedModel.id && + model?.isEnabled + ) + .map(({ model }: any) => String(model?.id || "").trim()) + .filter(Boolean); + + const candidateIds = Array.from( + new Set([selectedModel.id, ...configuredFallbackIds, ...sameProvider]) + ); + const byId = new Map( + allModels + .map(({ model }: any) => model) + .filter(Boolean) + .map((model: any) => [String(model.id), model] as const) + ); + + return candidateIds + .map((id) => byId.get(id)) + .filter(Boolean) + .map((model: any) => ({ + id: String(model.id), + modelId: String(model.modelId || "").trim(), + displayName: String(model.displayName || model.modelId || "").trim(), + })) + .filter((item) => item.modelId); +} + function extractVideoUrlFromText(text: string): string | null { const normalizedText = String(text || ""); if (!normalizedText) return null; @@ -4123,23 +4169,11 @@ export async function POST(request: Request) { // 千问/FAL 视频直接使用原始 URL，不转换为 base64 const modelMessages = convertToModelMessages(messagesForVideo); console.log(`[Chat API] ${currentVideoProvider} messages prepared, count:`, modelMessages.length); - - // 调用 doGenerate 获取视频 - const result = await (languageModel as any).doGenerate({ - messages: modelMessages, - videoParams: effectiveVideoParams, - }); - - const videoMarkdown = result.text || "视频生成失败"; - console.log(`[Chat API] ${currentVideoProvider} video generated, length:`, videoMarkdown.length); - - // 提取视频 URL（后续会自动转存到永久存储） - const tempVideoUrl = extractVideoUrlFromText(videoMarkdown) || ""; - - // 生成一个统一的消息 ID - const messageId = generateUUID(); - const videoMetadata = { + let videoMarkdown = "视频生成失败"; + let tempVideoUrl = ""; + let billedModelRefs: string[] = [finalModelId]; + let videoMetadata: Record = { provider: currentVideoProvider, model: finalModelId, mode: effectiveVideoParams?.mode, @@ -4149,6 +4183,86 @@ export async function POST(request: Request) { status: "succeeded", updatedAt: Date.now(), }; + + let usedStoryboardPipeline = false; + if (currentVideoProvider === "ruxa-video") { + const storyboardConfig = await getVideoStoryboardConfig().catch(() => null); + if (storyboardConfig?.enabled) { + const storyboardModelCandidates = await resolveStoryboardModelCandidates(finalModelId); + const runtimeModelId = String((languageModel as any).modelId || "").trim(); + const selectedModelId = + runtimeModelId || + storyboardModelCandidates.find((item) => item.id === finalModelId)?.modelId || + ""; + + const storyboardResult = await runRuxaNineGridPipeline({ + modelMessages: modelMessages as any, + selectedModelId, + selectedModelRef: finalModelId, + modelCandidates: storyboardModelCandidates, + videoParams: effectiveVideoParams as any, + config: storyboardConfig, + doGenerate: (input) => + (languageModel as any).doGenerate({ + messages: input.messages, + videoParams: input.videoParams, + overrideModelId: input.overrideModelId, + }), + onProgress: (event) => { + safeWriteStreamChunk( + writer, + { + type: "data-videoStoryboard", + data: JSON.stringify(event), + }, + `ruxa-video:storyboard:${event.stage}`, + ); + }, + }); + + if (storyboardResult.triggered) { + usedStoryboardPipeline = true; + videoMarkdown = storyboardResult.markdown || "视频生成失败，请稍后重试"; + tempVideoUrl = storyboardResult.videoUrls?.[0] || ""; + billedModelRefs = + storyboardResult.modelRefs && storyboardResult.modelRefs.length > 0 + ? storyboardResult.modelRefs + : [finalModelId]; + + const successCount = storyboardResult.videoUrls?.length || 0; + videoMetadata = { + ...videoMetadata, + ...(storyboardResult.metadata || {}), + status: successCount > 0 ? "succeeded" : "failed", + storyboard: { + enabled: true, + taskId: storyboardResult.taskId, + confidence: storyboardResult.confidence, + successCount, + }, + updatedAt: Date.now(), + }; + } + } + } + + if (!usedStoryboardPipeline) { + const result = await (languageModel as any).doGenerate({ + messages: modelMessages, + videoParams: effectiveVideoParams, + }); + + videoMarkdown = result.text || "视频生成失败"; + tempVideoUrl = extractVideoUrlFromText(videoMarkdown) || ""; + } + + console.log( + `[Chat API] ${currentVideoProvider} video generated, length:`, + videoMarkdown.length + ); + + // 生成一个统一的消息 ID + const messageId = generateUUID(); // 写入消息到流 writer.write({ @@ -4185,7 +4299,7 @@ export async function POST(request: Request) { userId: session.user.id, chatId: id, messageId, - modelRefs: [finalModelId], + modelRefs: billedModelRefs, }); if (tempVideoUrl) { diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 21e621a07..222fd5b61 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -88,6 +88,65 @@ interface CanvasChatPanelProps { onOpenFileList?: () => void; } +type VideoStoryboardStreamData = + | { + stage: "detecting"; + provider?: string; + message?: string; + } + | { + stage: "start"; + provider?: string; + taskId: string; + totalShots: number; + confidence?: number; + sourceImageUrl?: string; + shots?: Array<{ + shotIndex: number; + tileImageUrl?: string; + prompt?: string; + }>; + } + | { + stage: "shot_completed"; + provider?: string; + taskId: string; + shotIndex: number; + modelId?: string; + videoUrl: string; + text?: string; + } + | { + stage: "shot_failed"; + provider?: string; + taskId: string; + shotIndex: number; + error?: string; + } + | { + stage: "completed"; + provider?: string; + taskId: string; + totalShots?: number; + successCount?: number; + failedCount?: number; + } + | { + stage: "failed"; + provider?: string; + taskId: string; + error?: string; + } + | { + stage: "shot_retry"; + provider?: string; + taskId: string; + shotIndex: number; + attempt?: number; + modelId?: string; + error?: string; + }; + // 建议卡片数据 - 已移除 const SUGGESTIONS: { title: string; desc: string; image: string }[] = []; const CANVAS_COMPOSER_PREFIX_DEFAULT = "让Buyi-ai"; @@ -1617,6 +1676,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF signature: "", timestamp: 0, }); + const storyboardNodeMapRef = useRef>>({}); const releaseSendGuard = useCallback(() => { sendGuardRef.current.inFlight = false; }, []); @@ -3190,6 +3250,128 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF } catch (e) { console.error("[CanvasChat] Failed to parse data-appendMessage:", e); } + return; + } + + if (dataPart.type === "data-videoStoryboard") { + try { + const payload: VideoStoryboardStreamData = + typeof dataPart.data === "string" + ? JSON.parse(dataPart.data) + : dataPart.data; + if (!payload || typeof payload !== "object") return; + + if (payload.stage === "start") { + const taskId = String(payload.taskId || "").trim(); + if (!taskId) return; + if (storyboardNodeMapRef.current[taskId]) return; + + const totalShots = Math.max(1, Math.min(9, Number(payload.totalShots || 9))); + const shotNodeMap: Record = {}; + const currentNodes = useCanvasStore.getState().nodes; + const baseOffset = currentNodes.length; + const cardWidth = 320; + const cardHeight = 196; + const gapX = 30; + const gapY = 24; + const startX = 120 + (baseOffset % 2) * 420; + const startY = 120 + Math.floor(baseOffset / 2) * 260; + + for (let shotIndex = 1; shotIndex <= totalShots; shotIndex += 1) { + const shotData = payload.shots?.find((item) => Number(item.shotIndex) === shotIndex); + const col = (shotIndex - 1) % 3; + const row = Math.floor((shotIndex - 1) / 3); + const nodeId = addNode({ + type: "video", + position: { + x: startX + col * (cardWidth + gapX), + y: startY + row * (cardHeight + gapY), + }, + size: { width: cardWidth, height: cardHeight }, + data: { + src: "", + status: "generating", + prompt: shotData?.prompt || `分镜 ${shotIndex} 生成中`, + coverImage: shotData?.tileImageUrl || "", + }, + connections: [], + }); + shotNodeMap[shotIndex] = nodeId; + } + + storyboardNodeMapRef.current[taskId] = shotNodeMap; + toast.message(`九宫格视频任务已开始（${totalShots} 段）`); + return; + } + + if (payload.stage === "shot_completed") { + const taskId = String(payload.taskId || "").trim(); + const shotIndex = Math.max(1, Number(payload.shotIndex || 0)); + const videoUrl = String(payload.videoUrl || "").trim(); + if (!taskId || !shotIndex || !videoUrl) return; + + let nodeId = storyboardNodeMapRef.current[taskId]?.[shotIndex]; + if (!nodeId) { + nodeId = addNode({ + type: "video", + position: { x: 140 + ((shotIndex - 1) % 3) * 350, y: 150 + Math.floor((shotIndex - 1) / 3) * 230 }, + size: { width: 320, height: 196 }, + data: { + src: videoUrl, + status: "completed", + prompt: `分镜 ${shotIndex}`, + }, + connections: [], + }); + if (!storyboardNodeMapRef.current[taskId]) { + storyboardNodeMapRef.current[taskId] = {}; + } + storyboardNodeMapRef.current[taskId]![shotIndex] = nodeId; + return; + } + + const currentNode = useCanvasStore.getState().getNode(nodeId); + const currentData = ((currentNode?.data as any) || {}) as Record; + updateNode(nodeId, { + data: { + ...currentData, + src: videoUrl, + status: "completed", + }, + } as any); + return; + } + + if (payload.stage === "shot_failed") { + const taskId = String(payload.taskId || "").trim(); + const shotIndex = Math.max(1, Number(payload.shotIndex || 0)); + if (!taskId || !shotIndex) return; + const nodeId = storyboardNodeMapRef.current[taskId]?.[shotIndex]; + if (!nodeId) return; + const currentNode = useCanvasStore.getState().getNode(nodeId); + const currentData = ((currentNode?.data as any) || {}) as Record; + updateNode(nodeId, { + data: { + ...currentData, + status: "failed", + }, + } as any); + return; + } + + if (payload.stage === "completed") { + const successCount = Math.max(0, Number(payload.successCount || 0)); + const failedCount = Math.max(0, Number(payload.failedCount || 0)); + toast.success(`九宫格视频完成：成功 ${successCount}，失败 ${failedCount}`); + return; + } + + if (payload.stage === "failed") { + toast.error(payload.error || "九宫格视频任务失败"); + } + } catch (e) { + console.error("[CanvasChat] Failed to parse data-videoStoryboard:", e); + } } }, onError: (error) => { diff --git a/docs/系统全链路开发文档-含九宫格视频Agent技能化方案.md b/docs/系统全链路开发文档-含九宫格视频Agent技能化方案.md new file mode 100644 index 000000000..8f6713333 --- /dev/null +++ b/docs/系统全链路开发文档-含九宫格视频Agent技能化方案.md @@ -0,0 +1,599 @@ +# 系统全链路开发文档（基于当前项目） + +> 项目：`ai-chatbot-main` +> 文档目标：沉淀当前系统已实现能力，并给出“RUXA 视频九宫格 + Agent 技能化”完整可开发方案 +> 适用范围：聊天页、无限画布、3D 生成、AI 详情、视频生成、存储迁移、后台模型配置 + +--- + +## 1. 项目定位与能力边界 + +本系统是一个多模态 AI 创作平台，核心能力覆盖： + +1. 文本/图片/视频对话式生成（统一在聊天链路） +2. 无限画布创作与多节点编排（图片、视频、文本、3D） +3. Agent 智能路由与智能参数推荐 +4. AI 详情生成（电商详情页自动分镜 + 质检回修） +5. 多供应商模型治理（后台可配置 provider/model） +6. 多存储后端（S3/MinIO/OSS）与 OSS 降本迁移任务 + +--- + +## 2. 技术架构总览 + +## 2.1 技术栈 + +- 前端：Next.js 16 + React 19 + Tailwind + shadcn/ui +- 后端：Next.js Route Handlers（Node Runtime） +- AI 编排：`ai` SDK + LangGraph（Agent 流程） +- 数据库：PostgreSQL + Drizzle ORM +- 存储：S3 兼容存储 / Aliyun OSS / Vercel Blob（兼容） +- 图像处理：`sharp` +- 状态管理：Zustand（画布） + +## 2.2 关键目录 + +- 聊天主链路：`/Users/lanmu/Desktop/ai-chatbot-main/app/(chat)/api/chat/route.ts` +- 模型动态加载：`/Users/lanmu/Desktop/ai-chatbot-main/lib/ai/dynamic-provider.ts` +- Agent 编排：`/Users/lanmu/Desktop/ai-chatbot-main/lib/ai/auto-agent-graph.ts` +- 智能分析：`/Users/lanmu/Desktop/ai-chatbot-main/lib/ai/smart-analyzer.ts` +- 无限画布页面：`/Users/lanmu/Desktop/ai-chatbot-main/app/canvas/[id]/page.tsx` +- 画布聊天面板：`/Users/lanmu/Desktop/ai-chatbot-main/components/canvas/CanvasChatPanel.tsx` +- 3D 页面：`/Users/lanmu/Desktop/ai-chatbot-main/app/canvas/[id]/model3d/page.tsx` +- 3D API：`/Users/lanmu/Desktop/ai-chatbot-main/app/api/canvas/3d/tasks/route.ts` +- 后台设置：`/Users/lanmu/Desktop/ai-chatbot-main/app/(admin)/settings/admin-settings-client.tsx` +- 数据模型：`/Users/lanmu/Desktop/ai-chatbot-main/lib/db/schema.ts` +- 存储迁移任务：`/Users/lanmu/Desktop/ai-chatbot-main/lib/jobs/oss-image-migration-job.ts` + +--- + +## 3. 当前模型与供应商体系（已实现） + +## 3.1 Provider 类型（后台可配置） + +已接入的 provider 类型（数据库 + 后台一致）： + +- `alibaba`、`openai`、`custom`、`local` +- `gemini`、`huanke-gemini` +- `alibaba-image`、`ruxa-image`、`fal-image`、`volcano-vision` +- `qianwen-video`、`doubao-video`、`fal-video`、`ruxa-video`、`volcano-video` +- `tripo-3d` +- 扩展兼容：`zhipu`、`volcano-multimodal`（由 `custom` 映射识别） + +参考实现： + +- `/Users/lanmu/Desktop/ai-chatbot-main/app/api/admin/ai-providers/route.ts` +- `/Users/lanmu/Desktop/ai-chatbot-main/lib/ai/provider-defaults.ts` + +## 3.2 视频模型与模式能力（代码内置） + +视频能力模型通过 `capabilities + videoToolCapability` 驱动前端 UI 与参数限制： + +- RUXA：仅 `first-frame`（图生视频），比例 `16:9/9:16/1:1`，时长 `5/10` +- Qianwen：`first-frame` +- FAL：`first-frame/first-last-frame` +- Volcano/Doubao：支持文生/首帧/首尾帧/多参考图（按模型能力动态裁剪） + +参考实现： + +- `/Users/lanmu/Desktop/ai-chatbot-main/lib/ai/video-tool-capability.ts` +- `/Users/lanmu/Desktop/ai-chatbot-main/components/video-params-bar.tsx` + +## 3.3 代码中已出现的具体模型名称（非运行态 DB 全量） + +- RUXA 视频：`veo3.1`（后台模型可配置） +- 实时画布默认模型：`fal-ai/flux-2/klein/realtime` +- 火山预设模型族： + - `doubao-seedance-1-0-pro-250528` + - `doubao-seedance-1-0-pro-fast-251015` + - `doubao-seedance-1-0-lite-t2v-250428` + - `doubao-seedance-1-0-lite-i2v-250428` +- Tripo3D 版本选项： + - `Turbo-v1.0-20250506` + - `v3.1-20260211` + - `v3.0-20250812` + - `v2.5-20250123` + - `v2.0-20240919` + - `v1.4-20240625` + +说明：系统最终“在用模型清单”来自数据库 `AiModel`，由后台“AI模型/本地AI模型”动态配置。 + +--- + +## 4. 核心链路说明（已实现） + +## 4.1 聊天与 Agent 模式 + +入口：`POST /api/chat` +实现：`/Users/lanmu/Desktop/ai-chatbot-main/app/(chat)/api/chat/route.ts` + +主流程： + +1. 鉴权 + 请求去重（同 `chatId:messageId` 防重入） +2. 自动模式判定（`selectedChatModel === "auto"` 或固定视觉模型 + deepThinking） +3. 历史图片补全（当前无图时回溯最近生成图/上传图） +4. 调用 LangGraph Agent： + - 意图识别（图编/图生/视频/问答） + - 友好回复 + - 提示词优化 + - 推荐图片/视频参数 + - 路由到最佳模型 +5. 按 provider 分支执行（文本、图片、视频） +6. 流式返回 + 消息持久化 + 积分/配额扣减 + +## 4.2 无限画布 + +页面：`/canvas/[id]` +实现： + +- `/Users/lanmu/Desktop/ai-chatbot-main/app/canvas/[id]/page.tsx` +- `/Users/lanmu/Desktop/ai-chatbot-main/components/canvas/CanvasWorkspace.tsx` +- `/Users/lanmu/Desktop/ai-chatbot-main/store/canvas/index.ts` + +能力： + +- 节点类型：`message/image/video/text/scene-fusion/model3d` +- 自动保存 + 节流重试（429 后重试） +- 节点操作：拖拽、缩放、撤销/重做、多选 +- 画布与聊天联动（图片可回注对话） + +## 4.3 AI 详情生成 + +核心接口： + +- `POST /api/canvas/detail-plan` +- `POST /api/canvas/detail-review` +- `GET|POST|PATCH|DELETE /api/canvas/[id]/detail-archives` + +核心能力： + +1. 电商详情 brief 结构化输入 +2. 分屏规划（传统生图 / `agent_layout` 可编辑文字层） +3. 逐屏生成 + 质检 + 二次修图 +4. 存档回放（`canvasDetailArchive`） + +主要实现： + +- `/Users/lanmu/Desktop/ai-chatbot-main/components/canvas/CanvasChatPanel.tsx` +- `/Users/lanmu/Desktop/ai-chatbot-main/app/api/canvas/detail-plan/route.ts` +- `/Users/lanmu/Desktop/ai-chatbot-main/app/api/canvas/detail-review/route.ts` + +## 4.4 3D 生成（Tripo3D） + +页面：`/canvas/[id]/model3d` +API： + +- `POST /api/canvas/3d/upload-image` +- `POST /api/canvas/3d/tasks` +- `GET /api/canvas/3d/tasks/[taskId]` +- `GET /api/canvas/3d/balance` + +后端关键点： + +- 从 `AiProvider(type="tripo-3d")` 读取 key/baseUrl +- 对 API key 做隐藏字符清洗（解决 ByteString 8232 问题） +- 统一 Tripo 错误结构返回（含 traceId） + +主要实现： + +- `/Users/lanmu/Desktop/ai-chatbot-main/lib/tripo/server.ts` +- `/Users/lanmu/Desktop/ai-chatbot-main/lib/tripo/client.ts` + +## 4.5 存储与 OSS 迁移降本 + +现状策略（已上线）： + +1. 上传优先走可用存储配置（支持 OSS 作为入口桶） +2. 记录迁移任务（`StorageMigrationTask`） +3. 定时任务将 OSS 图片转存到默认桶 +4. 回写数据库 URL（消息、画布、详情存档） +5. 到期清理 OSS 源对象 + +接口： + +- `POST /api/jobs/migrate-oss-images` +- `GET /api/jobs/migrate-oss-images` + +主要实现： + +- `/Users/lanmu/Desktop/ai-chatbot-main/lib/storage/service.ts` +- `/Users/lanmu/Desktop/ai-chatbot-main/lib/jobs/oss-image-migration-job.ts` +- `/Users/lanmu/Desktop/ai-chatbot-main/app/api/jobs/migrate-oss-images/route.ts` + +--- + +## 5. 数据库核心表（当前） + +主要业务表： + +- `AiProvider`：供应商配置 +- `AiModel`：模型配置、能力、故障迁移字段 +- `StorageConfig`：存储桶配置 +- `StorageMigrationTask`：OSS 迁移任务 +- `SiteSettings` + `SystemSettings`：站点与功能开关 +- `Message`：消息主表（含视频存储元数据） +- `GeneratedImage`：图片记录 +- `CanvasProject`、`CanvasNode`、`CanvasImageTask` +- `CanvasDetailArchive`：AI详情归档 + +定义文件：`/Users/lanmu/Desktop/ai-chatbot-main/lib/db/schema.ts` + +--- + +## 6. 新需求：RUXA 视频九宫格 + Agent 技能化（待开发） + +## 6.1 需求归纳 + +当用户在视频模式选中 `RUXA 视频` 时： + +1. Agent 判断用户上传图是否为九宫格 +2. 若是九宫格，切成 9 张图 +3. 用户脚本格式不固定，先由 Agent 归一为 9 镜头脚本 +4. 切图后尺寸偏小，后台预留“图片变清晰”调用链路（默认关闭，待测试后再开启） +5. 每张图按对应脚本生成视频，串行发送，间隔 10 秒 +6. 失败自动重试；超限/失败可降级到其它视频模型 +7. 全流程可追踪进度，可回放任务结果 + +## 6.2 合理性评估结论 + +该需求合理，且与现有架构兼容；但需工程化约束： + +- 超分能力先预留不启用，通过配置开关控制，默认 `false` +- 九宫格检测需置信度门槛，低置信度不自动切片 +- 串行队列必须持久化到 DB，避免重启丢任务 + +--- + +## 7. 九宫格技能链设计（建议实现） + +采用“技能链（Skill Chain）”封装，而不是把全部逻辑塞进 `chat/route.ts`。 + +> 实施约束：当前阶段仅输出方案文档，不改动业务代码；开发时必须按“技能封装”方式落地。 + +## 7.1 技能模块划分 + +1. `script-normalizer-skill` + 输入任意脚本文本，输出标准 JSON（9 段） + +2. `nine-grid-detector-skill` + 判断是否九宫格 + 透视校正 + 切片 + 质量评分 + +3. `tile-upscale-skill` + 对小图按阈值走增强（复用现有 enhance 能力，先预留默认关闭） + +4. `storyboard-mapper-skill` + 将 9 图与 9 段脚本一一映射，生成每段视频 prompt + +5. `video-sequencer-skill` + 串行调度（10s）+ 重试 + 降级模型 + 状态更新 + +## 7.2 技能触发规则（必须） + +仅在满足以下条件时触发九宫格技能链： + +1. 当前模式为视频生成，且选中模型 provider 为 `ruxa-video` +2. 用户消息包含至少 1 张图片 +3. `nine-grid-detector-skill` 识别结果 `isNineGrid=true` +4. `confidence >= minGridConfidence`（默认 0.85） + +触发后的流程： + +1. 进入 `script-normalizer-skill` +2. 再进入 `nine-grid-detector-skill` 切片 +3. 按开关决定是否进入 `tile-upscale-skill` +4. 进入 `storyboard-mapper-skill` +5. 最后进入 `video-sequencer-skill` + +未触发时（任一条件不满足）： + +- 直接走现有普通视频链路，不进入九宫格技能链 + +## 7.3 建议代码落点 + +- `lib/video-agent/skills/script-normalizer.ts` +- `lib/video-agent/skills/nine-grid-detector.ts` +- `lib/video-agent/skills/tile-upscaler.ts` +- `lib/video-agent/skills/storyboard-mapper.ts` +- `lib/video-agent/skills/video-sequencer.ts` +- `lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts` + +聊天主链路只做路由： + +- 修改 `/Users/lanmu/Desktop/ai-chatbot-main/app/(chat)/api/chat/route.ts` + - 当 provider=`ruxa-video` 且满足“技能触发规则”时调用 pipeline + +--- + +## 8. 防切错设计（重点） + +## 8.1 检测流程（四段式） + +1. **预检**：宽高比接近 1:1 或 3:3 拼图常见比例；分辨率 > 最低阈值 +2. **网格线检测**：行列投影 + 边缘峰值，识别 2 条纵线 + 2 条横线 +3. **几何校正**：若轻微透视，先矫正再切片 +4. **切后复检**：每格清晰度、边界截断、相邻差异、主体完整度评分 + +## 8.2 判定阈值建议 + +- `grid_confidence >= 0.85`：自动进入九宫格流程 +- `0.65 ~ 0.85`：进入“待确认”状态（Agent 提示用户确认） +- `< 0.65`：走普通单图视频流程 + +## 8.3 输出结构建议 + +```json +{ + "isNineGrid": true, + "confidence": 0.92, + "cropBoxes": [ + {"index": 1, "x": 0, "y": 0, "w": 342, "h": 342} + ], + "qualityScore": 0.88 +} +``` + +--- + +## 9. 脚本归一化与分镜映射 + +## 9.1 脚本标准 JSON + +```json +{ + "language": "zh", + "globalStyle": "写实电影感", + "shots": [ + { + "index": 1, + "durationSec": 3, + "action": "角色抬头看向镜头", + "camera": "轻微推进", + "mood": "期待", + "negativePrompt": "抖动,畸变,低清" + } + ] +} +``` + +## 9.2 映射策略 + +- 默认 `shot[i] -> tile[i]` +- 若脚本中有显式编号（如“第3镜头”）优先按编号映射 +- 脚本不足 9 段：补全占位镜头（继承全局风格） +- 脚本超过 9 段：截断并告知用户 + +--- + +## 10. 超分链路设计（复用现有能力） + +当前已有增强接口：`POST /api/enhance` + +建议： + +1. 抽出服务层（如 `lib/image/enhance-service.ts`），避免服务端内部再走 HTTP +2. 增加特性开关 `enableTileUpscale`，默认 `false`（预留不启用） +3. 仅当 `enableTileUpscale=true` 时，切片后按阈值触发超分（短边 < 768） +4. 超分失败降级：保留原切片继续后续流程 +5. 记录 `enhanceApplied/enhanceProvider/enhanceLatencyMs` + +--- + +## 11. 调度、重试、降级 + +## 11.1 串行调度规则 + +- 一次任务 9 段，严格串行 +- 相邻请求间隔 10 秒 +- 任一段失败：指数退避（10s/20s/40s），最多 3 次 + +## 11.2 降级模型规则 + +从后台 `AiModel` 读取，不硬编码： + +1. 先用当前选中模型 +2. 失败后按 `fallbackModelId1 -> fallbackModelId2` +3. 若仍失败，按同 provider 可用模型池挑选 + +--- + +## 12. 数据库扩展设计（建议新增） + +新增任务主表：`VideoStoryboardTask` + +- `id` +- `userId` +- `chatId` +- `providerType`（默认 `ruxa-video`） +- `sourceImageUrl` +- `isNineGrid` +- `detectionConfidence` +- `normalizedScript` (jsonb) +- `status`（pending/running/partial_success/succeeded/failed/cancelled） +- `currentShot` +- `totalShots`（固定 9） +- `createdAt/updatedAt/completedAt` + +新增子任务表：`VideoStoryboardShotTask` + +- `id` +- `parentTaskId` +- `shotIndex` +- `tileImageUrl` +- `upscaledImageUrl` +- `prompt` +- `modelId` +- `fallbackChain` (jsonb) +- `attemptCount` +- `status` +- `videoUrl` +- `errorCode/errorMessage` +- `startedAt/completedAt` + +--- + +## 13. API 设计（建议新增） + +1. `POST /api/video/storyboard/create` + 创建九宫格视频任务，返回 `taskId` + +2. `GET /api/video/storyboard/tasks/:taskId` + 获取任务与每段进度 + +3. `POST /api/video/storyboard/tasks/:taskId/cancel` + 取消任务 + +4. `POST /api/jobs/video-storyboard-dispatch` + 调度器轮询（可由 cron/pm2 定时触发） + +--- + +## 14. 与现有聊天链路集成方式 + +在 `/api/chat` 增加分支： + +1. 当前 provider 是否 `ruxa-video` +2. 用户是否上传图片 +3. 是否触发九宫格识别（可由配置开关控制） + +若命中： + +- 先回 SSE 占位消息（“正在识别九宫格与解析分镜...”） +- 创建 storyboard 任务 +- 前端按 taskId 轮询并展示 9 段进度 +- 最终输出 9 条视频或合并汇总消息 + +--- + +## 15. 前端改造点（建议） + +- `components/video-params-bar.tsx` + - 新增“九宫格自动分镜”开关（RUXA 模式显示） +- `components/messages.tsx` + - 支持展示分段进度条（1/9 ~ 9/9） +- `components/chat.tsx` / `components/canvas/CanvasChatPanel.tsx` + - 扩展任务状态卡片（处理中、重试中、降级中） + +--- + +## 16. 配置项设计（系统设置） + +在系统设置新增 `video_storyboard_config`： + +- `enabled`：是否启用九宫格流程 +- `providerWhitelist`：默认仅 `ruxa-video` +- `minGridConfidence`：默认 0.85 +- `enableTileUpscale`：是否启用切片超分（默认 false，预留） +- `sliceUpscaleMinEdge`：默认 768 +- `dispatchIntervalSec`：默认 10 +- `maxRetry`：默认 3 +- `enableFallback`：默认 true + +存储位置：`SystemSettings` + +--- + +## 17. 监控与告警 + +建议埋点： + +- `nine_grid_detect_success_rate` +- `nine_grid_misdetect_rate` +- `script_normalize_success_rate` +- `shot_retry_rate` +- `fallback_trigger_rate` +- `storyboard_task_success_rate` +- `avg_storyboard_total_latency` + +日志关键词建议统一： + +- `[VideoStoryboard]` +- `[NineGridDetector]` +- `[ScriptNormalizer]` +- `[TileUpscaler]` +- `[VideoSequencer]` + +--- + +## 18. 测试与验收 + +## 18.1 单元测试 + +- 九宫格检测：标准图、弱网格图、非九宫格图 +- 脚本归一：口语化、表格化、混合语言 +- 映射规则：缺段/超段/乱序编号 +- 重试降级：模拟 429、余额不足、超时 + +## 18.2 集成测试 + +1. 上传九宫格 + 9段脚本 -> 全部成功 +2. 上传九宫格 + 混乱脚本 -> 归一成功 +3. 部分失败 -> 自动重试 + 降级成功 +4. 非九宫格 -> 正常走单图视频 + +## 18.3 验收标准 + +- 切片正确率 ≥ 95% +- 9 段任务可追踪、可重试、可取消 +- 不出现并发超限（严格 10 秒间隔） +- 失败可降级且不丢任务状态 + +--- + +## 19. 发布计划（建议） + +1. Phase 1：先上“检测 + 归一 + 任务表 + 状态查询” +2. Phase 2：接入“超分能力预留 + 串行调度”（超分开关保持关闭） +3. Phase 3：接入降级链路与前端进度 UI +4. Phase 4：灰度开关 + 指标监控 + 回归优化 + +--- + +## 20. 风险与规避 + +1. **误判九宫格**：加入置信度阈值与人工确认分支 +2. **成本过高**：超分默认关闭，待效果验证后灰度开启 +3. **任务堆积**：DB 队列 + 限流 + 可取消机制 +4. **外部 API 不稳定**：重试 + fallback + 错误分级 +5. **多机部署一致性**：调度基于 DB 状态机，不依赖内存队列 + +--- + +## 21. 当前实施状态标记 + +- 聊天/画布/3D/AI详情/OSS迁移：**已实现并在代码中可见** +- RUXA 九宫格视频技能链：**本文档为待开发方案（尚未落地代码）** + +--- + +## 22. 运维与发布命令（项目现状） + +- 安装依赖：`pnpm install --no-frozen-lockfile` +- 数据库迁移：`pnpm db:migrate` +- 构建：`pnpm build` +- 启动：`pnpm start` +- PM2 重启：`pm2 restart ai-chatbot || pm2 restart all` + +--- + +## 23. 附：本方案主要影响文件清单（开发阶段） + +新增（建议）： + +- `lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts` +- `lib/video-agent/skills/script-normalizer.ts` +- `lib/video-agent/skills/nine-grid-detector.ts` +- `lib/video-agent/skills/tile-upscaler.ts` +- `lib/video-agent/skills/storyboard-mapper.ts` +- `lib/video-agent/skills/video-sequencer.ts` +- `app/api/video/storyboard/create/route.ts` +- `app/api/video/storyboard/tasks/[taskId]/route.ts` +- `app/api/jobs/video-storyboard-dispatch/route.ts` +- `lib/db/migrations/00xx_add_video_storyboard_tasks.sql` + +修改（建议）： + +- `app/(chat)/api/chat/route.ts` +- `lib/ai/providers/ruxa-video.ts` +- `app/(admin)/settings/admin-settings-client.tsx` +- `lib/db/schema.ts` +- `lib/db/queries.ts` diff --git a/lib/video-agent/config.ts b/lib/video-agent/config.ts new file mode 100644 index 000000000..0dd7c0057 --- /dev/null +++ b/lib/video-agent/config.ts @@ -0,0 +1,81 @@ +import "server-only"; + +import { getSystemSetting } from "@/lib/db/queries"; + +export const VIDEO_STORYBOARD_CONFIG_KEY = "video_storyboard_config"; + +export type VideoStoryboardConfig = { + enabled: boolean; + providerWhitelist: string[]; + minGridConfidence: number; + enableTileUpscale: boolean; + sliceUpscaleMinEdge: number; + dispatchIntervalSec: number; + maxRetry: number; + enableFallback: boolean; +}; + +export const defaultVideoStoryboardConfig: VideoStoryboardConfig = { + enabled: true, + providerWhitelist: ["ruxa-video"], + minGridConfidence: 0.85, + enableTileUpscale: false, + sliceUpscaleMinEdge: 768, + dispatchIntervalSec: 10, + maxRetry: 3, + enableFallback: true, +}; + +function toNumber(value: unknown, fallback: number) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : fallback; +} + +export function normalizeVideoStoryboardConfig( + value: unknown +): VideoStoryboardConfig { + const raw = + value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; + + const whitelist = Array.isArray(raw.providerWhitelist) + ? raw.providerWhitelist + .map((item) => String(item || "").trim()) + .filter(Boolean) + : defaultVideoStoryboardConfig.providerWhitelist; + + return { + enabled: raw.enabled !== false, + providerWhitelist: + whitelist.length > 0 + ? Array.from(new Set(whitelist)) + : defaultVideoStoryboardConfig.providerWhitelist, + minGridConfidence: Math.max( + 0.5, + Math.min(0.99, toNumber(raw.minGridConfidence, defaultVideoStoryboardConfig.minGridConfidence)) + ), + enableTileUpscale: raw.enableTileUpscale === true, + sliceUpscaleMinEdge: Math.max( + 256, + Math.min(4096, Math.round(toNumber(raw.sliceUpscaleMinEdge, defaultVideoStoryboardConfig.sliceUpscaleMinEdge))) + ), + dispatchIntervalSec: Math.max( + 1, + Math.min(120, Math.round(toNumber(raw.dispatchIntervalSec, defaultVideoStoryboardConfig.dispatchIntervalSec))) + ), + maxRetry: Math.max( + 0, + Math.min(5, Math.round(toNumber(raw.maxRetry, defaultVideoStoryboardConfig.maxRetry))) + ), + enableFallback: raw.enableFallback !== false, + }; +} + +export async function getVideoStoryboardConfig(): Promise { + const setting = await getSystemSetting({ key: VIDEO_STORYBOARD_CONFIG_KEY }).catch( + () => undefined + ); + return normalizeVideoStoryboardConfig(setting?.value); +} + diff --git a/lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts b/lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts new file mode 100644 index 000000000..bb17da795 --- /dev/null +++ b/lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts @@ -0,0 +1,316 @@ +import "server-only"; + +import { generateUUID } from "@/lib/utils"; +import { runNineGridDetectorSkill } from "@/lib/video-agent/skills/nine-grid-detector"; +import { runScriptNormalizerSkill } from "@/lib/video-agent/skills/script-normalizer"; +import { runStoryboardMapperSkill } from "@/lib/video-agent/skills/storyboard-mapper"; +import { runTileUpscalerSkill } from "@/lib/video-agent/skills/tile-upscaler"; +import { runVideoSequencerSkill } from "@/lib/video-agent/skills/video-sequencer"; +import type { + RuxaNineGridPipelineParams, + RuxaNineGridPipelineResult, + StoryboardModelCandidate, +} from "@/lib/video-agent/types"; + +const NINE_GRID_SHOT_COUNT = 9; + +function normalizeText(value: unknown) { + return String(value || "").trim(); +} + +function extractVideoUrlFromText(text: string): string | null { + const normalizedText = String(text || ""); + if (!normalizedText) return null; + + const htmlMatch = normalizedText.match(/]+src="([^"]+)"/i); + if (htmlMatch?.[1]) return htmlMatch[1]; + + const markdownMatch = normalizedText.match( + /\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i + ); + if (markdownMatch?.[1]) return markdownMatch[1]; + + const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i); + return plainMatch?.[1] || null; +} + +function extractPromptAndImages(messages: Array<{ role: string; content: any }>) { + const lastUserMessage = messages.filter((message) => message.role === "user").pop(); + if (!lastUserMessage) { + return { prompt: "", imageUrls: [] as string[] }; + } + + let prompt = ""; + const imageUrls: string[] = []; + + const pushImage = (raw?: string) => { + const url = normalizeText(raw); + if (!url || !/^https?:\/\//i.test(url)) return; + if (!imageUrls.includes(url)) imageUrls.push(url); + }; + + if (typeof lastUserMessage.content === "string") { + prompt = normalizeText(lastUserMessage.content); + } else if (Array.isArray(lastUserMessage.content)) { + for (const part of lastUserMessage.content) { + if (part?.type === "text" && typeof part.text === "string") { + prompt = `${prompt} ${part.text}`.trim(); + } else if (part?.type === "image_url" && part.image_url?.url) { + pushImage(part.image_url.url); + } else if (part?.type === "file") { + pushImage(part.url || part.data); + } else if (part?.type === "image" && typeof part.image === "string") { + pushImage(part.image); + } + } + } + + if (!prompt) { + prompt = "让画面动起来"; + } + + return { prompt, imageUrls }; +} + +function normalizeModelCandidates(params: { + selectedModelId: string; + selectedModelRef: string; + modelCandidates: StoryboardModelCandidate[]; +}) { + const selected: StoryboardModelCandidate = { + id: params.selectedModelRef || params.selectedModelId, + modelId: params.selectedModelId, + displayName: "", + }; + const candidates = [selected, ...params.modelCandidates] + .filter((item) => normalizeText(item.modelId)) + .map((item) => ({ + id: normalizeText(item.id) || normalizeText(item.modelId), + modelId: normalizeText(item.modelId), + displayName: normalizeText(item.displayName), + })); + + const seen = new Set(); + return candidates.filter((item) => { + if (seen.has(item.modelId)) return false; + seen.add(item.modelId); + return true; + }); +} + +function buildCombinedMarkdown(params: { + success: Array<{ shotIndex: number; text: string }>; + failed: Array<{ shotIndex: number; error: string }>; +}) { + const sections: string[] = []; + + for (const shot of params.success.sort((a, b) => a.shotIndex - b.shotIndex)) { + sections.push(`### 分镜 ${shot.shotIndex}\n${shot.text}`); + } + + if (params.failed.length > 0) { + const failedLines = params.failed + .sort((a, b) => a.shotIndex - b.shotIndex) + .map((item) => `- 分镜 ${item.shotIndex}: ${item.error}`); + sections.push(["### 失败分镜", ...failedLines].join("\n")); + } + + return sections.join("\n\n").trim(); +} + +export async function runRuxaNineGridPipeline( + params: RuxaNineGridPipelineParams +): Promise { + const { + modelMessages, + selectedModelId, + selectedModelRef, + modelCandidates, + videoParams, + config, + onProgress, + doGenerate, + } = params; + + if (!config.enabled) { + return { triggered: false, reason: "disabled" }; + } + + const providerAllowed = config.providerWhitelist.some( + (item) => normalizeText(item).toLowerCase() === "ruxa-video" + ); + if (!providerAllowed) { + return { triggered: false, reason: "provider_not_allowed" }; + } + + onProgress?.({ + stage: "detecting", + provider: "ruxa-video", + message: "正在识别九宫格与解析分镜...", + }); + + const { prompt, imageUrls } = extractPromptAndImages(modelMessages); + if (!imageUrls[0]) { + return { triggered: false, reason: "missing_image" }; + } + + const detection = await runNineGridDetectorSkill({ + imageUrl: imageUrls[0], + minConfidence: config.minGridConfidence, + }); + + if (!detection.isNineGrid || detection.tiles.length !== NINE_GRID_SHOT_COUNT) { + return { + triggered: false, + reason: detection.isNineGrid ? "slice_incomplete" : "not_nine_grid", + confidence: detection.confidence, + }; + } + + const normalizedScript = runScriptNormalizerSkill({ + scriptText: prompt, + totalShots: NINE_GRID_SHOT_COUNT, + totalDurationSec: Number(videoParams?.duration || 10), + }); + const mappedShots = runStoryboardMapperSkill({ + script: normalizedScript, + tiles: detection.tiles, + }); + const preparedShots = await runTileUpscalerSkill({ + shots: mappedShots, + enableTileUpscale: config.enableTileUpscale, + sliceUpscaleMinEdge: config.sliceUpscaleMinEdge, + }); + + const resolvedModelCandidates = normalizeModelCandidates({ + selectedModelId, + selectedModelRef, + modelCandidates, + }); + + const taskId = `ruxa-storyboard-${generateUUID()}`; + onProgress?.({ + stage: "start", + provider: "ruxa-video", + taskId, + totalShots: preparedShots.length, + confidence: detection.confidence, + sourceImageUrl: imageUrls[0], + shots: preparedShots.map((shot) => ({ + shotIndex: shot.shotIndex, + tileImageUrl: shot.inputImageUrl, + prompt: shot.prompt, + })), + }); + + const sequencer = await runVideoSequencerSkill({ + provider: "ruxa-video", + taskId, + shots: preparedShots, + modelCandidates: resolvedModelCandidates, + maxRetry: config.maxRetry, + dispatchIntervalSec: config.dispatchIntervalSec, + enableFallback: config.enableFallback, + onProgress, + generateOneShot: async ({ shot, model }) => { + const shotMessages = [ + { + role: "user", + content: [ + { type: "text", text: shot.prompt }, + { type: "image_url", image_url: { url: shot.inputImageUrl } }, + ], + }, + ]; + + const output = await doGenerate({ + messages: shotMessages, + videoParams: { + ...(videoParams || {}), + mode: "first-frame", + }, + overrideModelId: model.modelId, + }); + + const text = normalizeText(output?.text); + const videoUrl = extractVideoUrlFromText(text); + if (!videoUrl) { + throw new Error(text || "未返回视频地址"); + } + return { videoUrl, text }; + }, + }); + + const successShots = sequencer.results + .filter((item) => item.success && item.videoUrl) + .map((item) => ({ + shotIndex: item.shotIndex, + text: normalizeText(item.text), + videoUrl: normalizeText(item.videoUrl), + modelId: normalizeText(item.modelId), + })); + const failedShots = sequencer.results + .filter((item) => !item.success) + .map((item) => ({ + shotIndex: item.shotIndex, + error: normalizeText(item.error) || "video_generation_failed", + })); + + if (successShots.length === 0) { + onProgress?.({ + stage: "failed", + provider: "ruxa-video", + taskId, + error: "all_shots_failed", + }); + return { + triggered: true, + taskId, + confidence: detection.confidence, + reason: "all_shots_failed", + videoUrls: [], + modelRefs: [], + markdown: "视频生成失败，请稍后重试", + metadata: { + taskId, + successCount: 0, + failedCount: failedShots.length, + confidence: detection.confidence, + failedShots, + }, + }; + } + + onProgress?.({ + stage: "completed", + provider: "ruxa-video", + taskId, + totalShots: sequencer.results.length, + successCount: sequencer.successCount, + failedCount: sequencer.failedCount, + }); + + return { + triggered: true, + taskId, + confidence: detection.confidence, + markdown: buildCombinedMarkdown({ + success: successShots.map((item) => ({ + shotIndex: item.shotIndex, + text: item.text, + })), + failed: failedShots, + }), + videoUrls: successShots.map((item) => item.videoUrl), + modelRefs: successShots.map((item) => item.modelId).filter(Boolean), + metadata: { + taskId, + confidence: detection.confidence, + qualityScore: detection.qualityScore, + sourceImageUrl: detection.sourceImageUrl, + successCount: sequencer.successCount, + failedCount: sequencer.failedCount, + failedShots, + }, + }; +} diff --git a/lib/video-agent/skills/nine-grid-detector.ts b/lib/video-agent/skills/nine-grid-detector.ts new file mode 100644 index 000000000..571548b5a --- /dev/null +++ b/lib/video-agent/skills/nine-grid-detector.ts @@ -0,0 +1,251 @@ +import "server-only"; + +import sharp from "sharp"; +import { uploadFile } from "@/lib/storage/service"; +import type { NineGridCropBox, NineGridDetectionResult, NineGridTile } from "@/lib/video-agent/types"; + +type ProjectionPeak = { + index: number; + value: number; +}; + +function clamp01(value: number) { + if (!Number.isFinite(value)) return 0; + return Math.max(0, Math.min(1, value)); +} + +function sampleStddevFromUint8(data: Uint8Array, limit = 20000) { + const total = Math.min(data.length, Math.max(128, limit)); + if (total <= 0) return 0; + const stride = Math.max(1, Math.floor(data.length / total)); + const sampled: number[] = []; + for (let index = 0; index < data.length && sampled.length < total; index += stride) { + sampled.push(data[index] || 0); + } + return stddev(sampled); +} + +function toArrayBuffer(buffer: Buffer): ArrayBuffer { + return buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) as ArrayBuffer; +} + +function movingAverage(values: number[], radius = 3): number[] { + if (values.length === 0) return values; + const result = new Array(values.length).fill(0); + for (let index = 0; index < values.length; index += 1) { + let sum = 0; + let count = 0; + const start = Math.max(0, index - radius); + const end = Math.min(values.length - 1, index + radius); + for (let cursor = start; cursor <= end; cursor += 1) { + sum += values[cursor] || 0; + count += 1; + } + result[index] = count > 0 ? sum / count : 0; + } + return result; +} + +function mean(values: number[]) { + if (values.length === 0) return 0; + return values.reduce((total, item) => total + item, 0) / values.length; +} + +function stddev(values: number[]) { + if (values.length === 0) return 0; + const avg = mean(values); + const variance = + values.reduce((total, item) => total + Math.pow(item - avg, 2), 0) / values.length; + return Math.sqrt(Math.max(variance, 0)); +} + +function findPeakNear( + scores: number[], + targetIndex: number, + windowRadius: number +): ProjectionPeak { + const start = Math.max(0, Math.round(targetIndex - windowRadius)); + const end = Math.min(scores.length - 1, Math.round(targetIndex + windowRadius)); + let bestIndex = Math.round(targetIndex); + let bestValue = -Infinity; + for (let index = start; index <= end; index += 1) { + const value = scores[index] || 0; + if (value > bestValue) { + bestValue = value; + bestIndex = index; + } + } + return { index: bestIndex, value: Number.isFinite(bestValue) ? bestValue : 0 }; +} + +function scoreGridAxis(scores: number[], firstTarget: number, secondTarget: number) { + const radius = Math.max(8, Math.round(scores.length * 0.12)); + const firstPeak = findPeakNear(scores, firstTarget, radius); + const secondPeak = findPeakNear(scores, secondTarget, radius); + const avg = mean(scores); + const sigma = stddev(scores); + const dynamicBase = avg + sigma; + + const closenessA = clamp01(1 - Math.abs(firstPeak.index - firstTarget) / radius); + const closenessB = clamp01(1 - Math.abs(secondPeak.index - secondTarget) / radius); + const closeness = (closenessA + closenessB) / 2; + + const denom = Math.max(dynamicBase, 1); + const strengthA = clamp01(firstPeak.value / (denom * 2.2)); + const strengthB = clamp01(secondPeak.value / (denom * 2.2)); + const strength = (strengthA + strengthB) / 2; + + return { + score: clamp01(closeness * 0.55 + strength * 0.45), + peaks: [firstPeak, secondPeak], + }; +} + +function buildNineGridCropBoxes(width: number, height: number): NineGridCropBox[] { + const baseW = Math.floor(width / 3); + const baseH = Math.floor(height / 3); + const boxes: NineGridCropBox[] = []; + let index = 1; + for (let row = 0; row < 3; row += 1) { + for (let col = 0; col < 3; col += 1) { + const x = col * baseW; + const y = row * baseH; + const w = col === 2 ? width - x : baseW; + const h = row === 2 ? height - y : baseH; + boxes.push({ index, x, y, w, h }); + index += 1; + } + } + return boxes; +} + +async function fetchImageBuffer(imageUrl: string) { + const response = await fetch(imageUrl); + if (!response.ok) { + throw new Error(`图片下载失败 (HTTP ${response.status})`); + } + const contentType = response.headers.get("content-type") || ""; + if (!/^image\//i.test(contentType)) { + throw new Error(`输入不是图片类型: ${contentType || "unknown"}`); + } + return Buffer.from(await response.arrayBuffer()); +} + +async function sliceAndUploadTiles(params: { + sourceBuffer: Buffer; + cropBoxes: NineGridCropBox[]; +}): Promise { + const { sourceBuffer, cropBoxes } = params; + const timestamp = Date.now(); + const tiles: NineGridTile[] = []; + for (const cropBox of cropBoxes) { + const tileBuffer = await sharp(sourceBuffer) + .extract({ + left: cropBox.x, + top: cropBox.y, + width: cropBox.w, + height: cropBox.h, + }) + .jpeg({ quality: 92 }) + .toBuffer(); + + const filename = `video-storyboard-tile-${timestamp}-${cropBox.index}.jpg`; + const uploaded = await uploadFile(filename, toArrayBuffer(tileBuffer)); + tiles.push({ + index: cropBox.index, + imageUrl: uploaded.url, + width: cropBox.w, + height: cropBox.h, + }); + } + return tiles; +} + +export async function runNineGridDetectorSkill(params: { + imageUrl: string; + minConfidence: number; +}): Promise { + const sourceImageUrl = String(params.imageUrl || "").trim(); + if (!sourceImageUrl) { + throw new Error("缺少九宫格输入图片"); + } + + const sourceBuffer = await fetchImageBuffer(sourceImageUrl); + const baseImage = sharp(sourceBuffer, { failOn: "none" }).rotate(); + const metadata = await baseImage.metadata(); + const width = Number(metadata.width || 0); + const height = Number(metadata.height || 0); + + if (width < 240 || height < 240) { + return { + isNineGrid: false, + confidence: 0, + qualityScore: 0, + cropBoxes: [], + width, + height, + sourceImageUrl, + tiles: [], + }; + } + + const grayscale = await baseImage.greyscale().raw().toBuffer({ + resolveWithObject: true, + }); + const raw = grayscale.data; + const channels = grayscale.info.channels || 1; + const imgW = grayscale.info.width; + const imgH = grayscale.info.height; + const step = Math.max(1, Math.round(Math.min(imgW, imgH) / 900)); + + const colScores = new Array(imgW).fill(0); + const rowScores = new Array(imgH).fill(0); + + for (let y = 0; y < imgH; y += step) { + const rowOffset = y * imgW * channels; + for (let x = step; x < imgW; x += step) { + const current = raw[rowOffset + x * channels] || 0; + const prev = raw[rowOffset + (x - step) * channels] || 0; + colScores[x] += Math.abs(current - prev); + } + } + + for (let x = 0; x < imgW; x += step) { + for (let y = step; y < imgH; y += step) { + const current = raw[(y * imgW + x) * channels] || 0; + const prev = raw[((y - step) * imgW + x) * channels] || 0; + rowScores[y] += Math.abs(current - prev); + } + } + + const smoothCols = movingAverage(colScores, 5); + const smoothRows = movingAverage(rowScores, 5); + const colAxis = scoreGridAxis(smoothCols, imgW / 3, (imgW * 2) / 3); + const rowAxis = scoreGridAxis(smoothRows, imgH / 3, (imgH * 2) / 3); + + const ratio = width / Math.max(height, 1); + const aspectScore = clamp01(1 - Math.min(Math.abs(ratio - 1), 1)); + const gridScore = clamp01((colAxis.score + rowAxis.score) / 2); + const confidence = clamp01(aspectScore * 0.3 + gridScore * 0.7); + const qualityScore = clamp01((sampleStddevFromUint8(raw, 20000) || 0) / 72); + + const isNineGrid = confidence >= params.minConfidence && aspectScore >= 0.62; + const cropBoxes = isNineGrid ? buildNineGridCropBoxes(width, height) : []; + const tiles = isNineGrid + ? await sliceAndUploadTiles({ sourceBuffer: await baseImage.toBuffer(), cropBoxes }) + : []; + + return { + isNineGrid, + confidence, + qualityScore, + cropBoxes, + width, + height, + sourceImageUrl, + tiles, + }; +} diff --git a/lib/video-agent/skills/script-normalizer.ts b/lib/video-agent/skills/script-normalizer.ts new file mode 100644 index 000000000..ec6a56f8f --- /dev/null +++ b/lib/video-agent/skills/script-normalizer.ts @@ -0,0 +1,109 @@ +import "server-only"; + +import type { NormalizedStoryboardScript, NormalizedStoryboardShot } from "@/lib/video-agent/types"; + +function detectLanguage(text: string): "zh" | "en" | "mixed" { + const source = String(text || ""); + const zhCount = (source.match(/[\u4e00-\u9fff]/g) || []).length; + const enCount = (source.match(/[A-Za-z]/g) || []).length; + if (zhCount > 0 && enCount > 0) return "mixed"; + if (zhCount > 0) return "zh"; + return "en"; +} + +function cleanLine(line: string) { + return String(line || "") + .replace(/^\s*(?:第?\s*\d+\s*(?:镜头|段|幕)|\d+\s*[.)、：:\-]?|[一二三四五六七八九十]+\s*[、.：:\-]?)\s*/g, "") + .replace(/\s+/g, " ") + .trim(); +} + +function splitScriptLines(script: string): string[] { + const source = String(script || "").trim(); + if (!source) return []; + const byLine = source + .split(/\r?\n+/) + .map(cleanLine) + .filter(Boolean); + if (byLine.length >= 4) return byLine; + + return source + .split(/[。！？!?；;\n]+/g) + .map(cleanLine) + .filter(Boolean); +} + +function inferCamera(text: string) { + const source = String(text || ""); + if (/推进|推近|close[- ]?in|dolly in/i.test(source)) return "轻微推进"; + if (/拉远|远景|zoom out|wide/i.test(source)) return "缓慢拉远"; + if (/平移|横移|pan/i.test(source)) return "平稳平移"; + if (/旋转|环绕|orbit/i.test(source)) return "环绕镜头"; + return "稳定镜头"; +} + +function inferMood(text: string) { + const source = String(text || ""); + if (/紧张|压迫|惊悚|tense/i.test(source)) return "紧张"; + if (/温暖|治愈|暖/i.test(source)) return "温暖"; + if (/活力|动感|快节奏|energetic/i.test(source)) return "活力"; + if (/浪漫|梦幻|romantic/i.test(source)) return "浪漫"; + return "自然"; +} + +function buildDefaultAction(index: number) { + const defaults = [ + "主体从静止到轻微动作", + "主体与环境产生互动", + "主体视线发生转移", + "主体做细节动作展示", + "镜头展示关键部位", + "镜头拉近突出质感", + "镜头切换到侧面动态", + "主体完成动作收束", + "镜头定格形成结尾", + ]; + return defaults[index - 1] || "主体做自然动作"; +} + +export function runScriptNormalizerSkill(params: { + scriptText: string; + totalShots?: number; + totalDurationSec?: number; +}): NormalizedStoryboardScript { + const totalShots = Math.max(1, Math.min(20, Math.round(params.totalShots || 9))); + const raw = String(params.scriptText || "").trim(); + const lines = splitScriptLines(raw); + const durationPerShot = Math.max( + 2, + Math.round(Math.max(5, Number(params.totalDurationSec || 10)) / totalShots) + ); + + const normalizedShots: NormalizedStoryboardShot[] = []; + const language = detectLanguage(raw); + const fallbackStyle = + language === "en" ? "cinematic realistic style" : "写实电影感"; + + for (let index = 1; index <= totalShots; index += 1) { + const line = lines[index - 1] || lines[lines.length - 1] || ""; + const action = cleanLine(line) || buildDefaultAction(index); + normalizedShots.push({ + index, + durationSec: durationPerShot, + action, + camera: inferCamera(line), + mood: inferMood(line), + negativePrompt: + language === "en" + ? "jitter, blur, artifacts, low quality" + : "抖动, 畸变, 低清, 伪影", + }); + } + + return { + language, + globalStyle: fallbackStyle, + shots: normalizedShots, + raw, + }; +} diff --git a/lib/video-agent/skills/storyboard-mapper.ts b/lib/video-agent/skills/storyboard-mapper.ts new file mode 100644 index 000000000..ef8131e43 --- /dev/null +++ b/lib/video-agent/skills/storyboard-mapper.ts @@ -0,0 +1,73 @@ +import "server-only"; + +import type { + MappedStoryboardShot, + NineGridTile, + NormalizedStoryboardScript, +} from "@/lib/video-agent/types"; + +function clampIndex(value: number, length: number) { + if (length <= 0) return 0; + const rounded = Math.max(1, Math.round(value)); + return Math.min(length, rounded) - 1; +} + +function buildShotPrompt(params: { + style: string; + action: string; + camera: string; + mood: string; + language: "zh" | "en" | "mixed"; +}) { + const { style, action, camera, mood, language } = params; + if (language === "en") { + return [ + "Animate this still image into a coherent short scene.", + `Action: ${action}.`, + `Camera: ${camera}.`, + `Mood: ${mood}.`, + `Style: ${style}.`, + "Keep identity and key structure stable.", + ].join(" "); + } + + return [ + "请基于当前输入图片生成连贯的视频镜头。", + `动作：${action}。`, + `镜头：${camera}。`, + `氛围：${mood}。`, + `风格：${style}。`, + "保持主体身份、构图和关键细节稳定。", + ].join(""); +} + +export function runStoryboardMapperSkill(params: { + script: NormalizedStoryboardScript; + tiles: NineGridTile[]; +}): MappedStoryboardShot[] { + const { script, tiles } = params; + if (!tiles.length) return []; + + return script.shots.map((shot, index) => { + const tile = tiles[clampIndex(shot.index || index + 1, tiles.length)] || tiles[index] || tiles[0]; + const prompt = buildShotPrompt({ + style: script.globalStyle, + action: shot.action, + camera: shot.camera, + mood: shot.mood, + language: script.language, + }); + + return { + shotIndex: shot.index, + tileIndex: tile.index, + tileImageUrl: tile.imageUrl, + durationSec: shot.durationSec, + prompt, + negativePrompt: shot.negativePrompt, + action: shot.action, + camera: shot.camera, + mood: shot.mood, + }; + }); +} diff --git a/lib/video-agent/skills/tile-upscaler.ts b/lib/video-agent/skills/tile-upscaler.ts new file mode 100644 index 000000000..119e4bc6f --- /dev/null +++ b/lib/video-agent/skills/tile-upscaler.ts @@ -0,0 +1,18 @@ +import "server-only"; + +import type { MappedStoryboardShot, PreparedStoryboardShot } from "@/lib/video-agent/types"; + +export async function runTileUpscalerSkill(params: { + shots: MappedStoryboardShot[]; + enableTileUpscale: boolean; + sliceUpscaleMinEdge: number; +}): Promise { + const { shots, enableTileUpscale } = params; + + return shots.map((shot) => ({ + ...shot, + inputImageUrl: shot.tileImageUrl, + upscaled: false, + upscaleReason: enableTileUpscale ? "reserved_not_enabled" : "disabled", + })); +} diff --git a/lib/video-agent/skills/video-sequencer.ts b/lib/video-agent/skills/video-sequencer.ts new file mode 100644 index 000000000..f4440fb0b --- /dev/null +++ b/lib/video-agent/skills/video-sequencer.ts @@ -0,0 +1,149 @@ +import "server-only"; + +import type { + PreparedStoryboardShot, + StoryboardModelCandidate, + StoryboardProgressEvent, +} from "@/lib/video-agent/types"; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +function normalizeErrorMessage(error: unknown) { + if (!error) return "unknown_error"; + if (typeof error === "string") return error; + if (error instanceof Error) return error.message || String(error); + try { + return JSON.stringify(error); + } catch { + return String(error); + } +} + +export type SequencerShotResult = { + shotIndex: number; + success: boolean; + videoUrl?: string; + modelId?: string; + text?: string; + error?: string; +}; + +export async function runVideoSequencerSkill(params: { + provider: string; + taskId: string; + shots: PreparedStoryboardShot[]; + modelCandidates: StoryboardModelCandidate[]; + maxRetry: number; + dispatchIntervalSec: number; + enableFallback: boolean; + onProgress?: (event: StoryboardProgressEvent) => void; + generateOneShot: (input: { + shot: PreparedStoryboardShot; + model: StoryboardModelCandidate; + }) => Promise<{ videoUrl: string; text: string }>; +}): Promise<{ + results: SequencerShotResult[]; + successCount: number; + failedCount: number; +}> { + const { + provider, + taskId, + shots, + modelCandidates, + maxRetry, + dispatchIntervalSec, + enableFallback, + onProgress, + generateOneShot, + } = params; + + const candidateList = + enableFallback && modelCandidates.length > 0 + ? modelCandidates + : modelCandidates.slice(0, 1); + + const results: SequencerShotResult[] = []; + + for (let index = 0; index < shots.length; index += 1) { + const shot = shots[index]!; + let shotSuccess = false; + let lastError = ""; + let resultText = ""; + let resultUrl = ""; + let usedModelId = ""; + + for (const model of candidateList) { + const retries = Math.max(0, Math.round(maxRetry)); + for (let attempt = 1; attempt <= retries + 1; attempt += 1) { + try { + const output = await generateOneShot({ shot, model }); + resultText = output.text; + resultUrl = output.videoUrl; + usedModelId = model.modelId; + shotSuccess = true; + break; + } catch (error) { + lastError = normalizeErrorMessage(error); + if (attempt <= retries) { + onProgress?.({ + stage: "shot_retry", + provider, + taskId, + shotIndex: shot.shotIndex, + attempt, + modelId: model.modelId, + error: lastError, + }); + const backoffSec = Math.min(40, Math.pow(2, attempt - 1) * 10); + await sleep(backoffSec * 1000); + } + } + } + + if (shotSuccess) { + break; + } + } + + if (shotSuccess) { + results.push({ + shotIndex: shot.shotIndex, + success: true, + modelId: usedModelId, + videoUrl: resultUrl, + text: resultText, + }); + onProgress?.({ + stage: "shot_completed", + provider, + taskId, + shotIndex: shot.shotIndex, + modelId: usedModelId, + videoUrl: resultUrl, + text: resultText, + }); + } else { + results.push({ + shotIndex: shot.shotIndex, + success: false, + error: lastError || "video_generation_failed", + }); + onProgress?.({ + stage: "shot_failed", + provider, + taskId, + shotIndex: shot.shotIndex, + error: lastError || "video_generation_failed", + }); + } + + if (index < shots.length - 1 && dispatchIntervalSec > 0) { + await sleep(Math.round(dispatchIntervalSec * 1000)); + } + } + + const successCount = results.filter((item) => item.success).length; + const failedCount = results.length - successCount; + return { results, successCount, failedCount }; +} diff --git a/lib/video-agent/types.ts b/lib/video-agent/types.ts new file mode 100644 index 000000000..ab5606a2b --- /dev/null +++ b/lib/video-agent/types.ts @@ -0,0 +1,152 @@ +import type { VideoStoryboardConfig } from "@/lib/video-agent/config"; + +export type NormalizedStoryboardShot = { + index: number; + durationSec: number; + action: string; + camera: string; + mood: string; + negativePrompt: string; +}; + +export type NormalizedStoryboardScript = { + language: "zh" | "en" | "mixed"; + globalStyle: string; + shots: NormalizedStoryboardShot[]; + raw: string; +}; + +export type NineGridCropBox = { + index: number; + x: number; + y: number; + w: number; + h: number; +}; + +export type NineGridTile = { + index: number; + imageUrl: string; + width: number; + height: number; +}; + +export type NineGridDetectionResult = { + isNineGrid: boolean; + confidence: number; + qualityScore: number; + cropBoxes: NineGridCropBox[]; + width: number; + height: number; + sourceImageUrl: string; + tiles: NineGridTile[]; +}; + +export type MappedStoryboardShot = { + shotIndex: number; + tileIndex: number; + tileImageUrl: string; + durationSec: number; + prompt: string; + negativePrompt: string; + action: string; + camera: string; + mood: string; +}; + +export type PreparedStoryboardShot = MappedStoryboardShot & { + inputImageUrl: string; + upscaled: boolean; + upscaleReason?: string; +}; + +export type StoryboardModelCandidate = { + id: string; + modelId: string; + displayName?: string; +}; + +export type StoryboardProgressEvent = + | { + stage: "detecting"; + provider: string; + message?: string; + } + | { + stage: "start"; + provider: string; + taskId: string; + totalShots: number; + confidence: number; + sourceImageUrl: string; + shots: Array<{ + shotIndex: number; + tileImageUrl: string; + prompt: string; + }>; + } + | { + stage: "shot_retry"; + provider: string; + taskId: string; + shotIndex: number; + attempt: number; + modelId: string; + error: string; + } + | { + stage: "shot_completed"; + provider: string; + taskId: string; + shotIndex: number; + modelId: string; + videoUrl: string; + text: string; + } + | { + stage: "shot_failed"; + provider: string; + taskId: string; + shotIndex: number; + error: string; + } + | { + stage: "completed"; + provider: string; + taskId: string; + totalShots: number; + successCount: number; + failedCount: number; + } + | { + stage: "failed"; + provider: string; + taskId: string; + error: string; + }; + +export type RuxaNineGridPipelineParams = { + modelMessages: Array<{ role: string; content: any }>; + selectedModelId: string; + selectedModelRef: string; + modelCandidates: StoryboardModelCandidate[]; + videoParams?: Record; + config: VideoStoryboardConfig; + onProgress?: (event: StoryboardProgressEvent) => void; + doGenerate: (input: { + messages: Array<{ role: string; content: any }>; + videoParams?: Record; + overrideModelId?: string; + }) => Promise<{ text?: string }>; +}; + +export type RuxaNineGridPipelineResult = { + triggered: boolean; + taskId?: string; + confidence?: number; + markdown?: string; + videoUrls?: string[]; + modelRefs?: string[]; + metadata?: Record; + reason?: string; +};