feat(video-agent): add ruxa nine-grid storyboard skill pipeline

app/(chat)/api/chat/route.ts

@@ -57,6 +57,9 @@ import { ChatSDKError } from "@/lib/errors";
 import { extractImageUrlsFromText } from "@/lib/image-url";
 import { videoArchiver } from "@/lib/storage/video-archiver";
 import { uploadFile } from "@/lib/storage/service";
+import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
+import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline";
+import type { StoryboardModelCandidate } from "@/lib/video-agent/types";
 import type { ChatMessage } from "@/lib/types";
 import type { AppUsage } from "@/lib/usage";
 import { convertToUIMessages, generateUUID } from "@/lib/utils";
@@ -359,6 +362,49 @@ function selectBestVideoSubModelForMode(
   })[0];
 }
 
+async function resolveStoryboardModelCandidates(
+  selectedModelRef: string
+): Promise<StoryboardModelCandidate[]> {
+  const selectedModel = await getAiModelById({ id: selectedModelRef }).catch(() => undefined);
+  if (!selectedModel) return [];
+
+  const configuredFallbackIds = [
+    selectedModel.fallbackModelId1,
+    selectedModel.fallbackModelId2,
+  ].filter((item): item is string => Boolean(String(item || "").trim()));
+
+  const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []);
+  const sameProvider = allModels
+    .filter(
+      ({ model, provider }: any) =>
+        provider?.id === selectedModel.providerId &&
+        model?.id !== selectedModel.id &&
+        model?.isEnabled
+    )
+    .map(({ model }: any) => String(model?.id || "").trim())
+    .filter(Boolean);
+
+  const candidateIds = Array.from(
+    new Set([selectedModel.id, ...configuredFallbackIds, ...sameProvider])
+  );
+  const byId = new Map(
+    allModels
+      .map(({ model }: any) => model)
+      .filter(Boolean)
+      .map((model: any) => [String(model.id), model] as const)
+  );
+
+  return candidateIds
+    .map((id) => byId.get(id))
+    .filter(Boolean)
+    .map((model: any) => ({
+      id: String(model.id),
+      modelId: String(model.modelId || "").trim(),
+      displayName: String(model.displayName || model.modelId || "").trim(),
+    }))
+    .filter((item) => item.modelId);
+}
+
 function extractVideoUrlFromText(text: string): string | null {
   const normalizedText = String(text || "");
   if (!normalizedText) return null;
@@ -4123,23 +4169,11 @@ export async function POST(request: Request) {
             // 千问/FAL 视频直接使用原始 URL，不转换为 base64
             const modelMessages = convertToModelMessages(messagesForVideo);
             console.log(`[Chat API] ${currentVideoProvider} messages prepared, count:`, modelMessages.length);
-            
-            // 调用 doGenerate 获取视频
-            const result = await (languageModel as any).doGenerate({
-              messages: modelMessages,
-              videoParams: effectiveVideoParams,
-            });
-            
-            const videoMarkdown = result.text || "视频生成失败";
-            console.log(`[Chat API] ${currentVideoProvider} video generated, length:`, videoMarkdown.length);
-            
-            // 提取视频 URL（后续会自动转存到永久存储）
-            const tempVideoUrl = extractVideoUrlFromText(videoMarkdown) || "";
-            
-            // 生成一个统一的消息 ID
-            const messageId = generateUUID();
 
-            const videoMetadata = {
+            let videoMarkdown = "视频生成失败";
+            let tempVideoUrl = "";
+            let billedModelRefs: string[] = [finalModelId];
+            let videoMetadata: Record<string, any> = {
               provider: currentVideoProvider,
               model: finalModelId,
               mode: effectiveVideoParams?.mode,
@@ -4149,6 +4183,86 @@ export async function POST(request: Request) {
               status: "succeeded",
               updatedAt: Date.now(),
             };
+
+            let usedStoryboardPipeline = false;
+            if (currentVideoProvider === "ruxa-video") {
+              const storyboardConfig = await getVideoStoryboardConfig().catch(() => null);
+              if (storyboardConfig?.enabled) {
+                const storyboardModelCandidates = await resolveStoryboardModelCandidates(finalModelId);
+                const runtimeModelId = String((languageModel as any).modelId || "").trim();
+                const selectedModelId =
+                  runtimeModelId ||
+                  storyboardModelCandidates.find((item) => item.id === finalModelId)?.modelId ||
+                  "";
+
+                const storyboardResult = await runRuxaNineGridPipeline({
+                  modelMessages: modelMessages as any,
+                  selectedModelId,
+                  selectedModelRef: finalModelId,
+                  modelCandidates: storyboardModelCandidates,
+                  videoParams: effectiveVideoParams as any,
+                  config: storyboardConfig,
+                  doGenerate: (input) =>
+                    (languageModel as any).doGenerate({
+                      messages: input.messages,
+                      videoParams: input.videoParams,
+                      overrideModelId: input.overrideModelId,
+                    }),
+                  onProgress: (event) => {
+                    safeWriteStreamChunk(
+                      writer,
+                      {
+                        type: "data-videoStoryboard",
+                        data: JSON.stringify(event),
+                      },
+                      `ruxa-video:storyboard:${event.stage}`,
+                    );
+                  },
+                });
+
+                if (storyboardResult.triggered) {
+                  usedStoryboardPipeline = true;
+                  videoMarkdown = storyboardResult.markdown || "视频生成失败，请稍后重试";
+                  tempVideoUrl = storyboardResult.videoUrls?.[0] || "";
+                  billedModelRefs =
+                    storyboardResult.modelRefs && storyboardResult.modelRefs.length > 0
+                      ? storyboardResult.modelRefs
+                      : [finalModelId];
+
+                  const successCount = storyboardResult.videoUrls?.length || 0;
+                  videoMetadata = {
+                    ...videoMetadata,
+                    ...(storyboardResult.metadata || {}),
+                    status: successCount > 0 ? "succeeded" : "failed",
+                    storyboard: {
+                      enabled: true,
+                      taskId: storyboardResult.taskId,
+                      confidence: storyboardResult.confidence,
+                      successCount,
+                    },
+                    updatedAt: Date.now(),
+                  };
+                }
+              }
+            }
+
+            if (!usedStoryboardPipeline) {
+              const result = await (languageModel as any).doGenerate({
+                messages: modelMessages,
+                videoParams: effectiveVideoParams,
+              });
+
+              videoMarkdown = result.text || "视频生成失败";
+              tempVideoUrl = extractVideoUrlFromText(videoMarkdown) || "";
+            }
+
+            console.log(
+              `[Chat API] ${currentVideoProvider} video generated, length:`,
+              videoMarkdown.length
+            );
+            
+            // 生成一个统一的消息 ID
+            const messageId = generateUUID();
             
             // 写入消息到流
             writer.write({
@@ -4185,7 +4299,7 @@ export async function POST(request: Request) {
                 userId: session.user.id,
                 chatId: id,
                 messageId,
-                modelRefs: [finalModelId],
+                modelRefs: billedModelRefs,
               });
 
               if (tempVideoUrl) {

components/canvas/CanvasChatPanel.tsx

@@ -88,6 +88,65 @@ interface CanvasChatPanelProps {
   onOpenFileList?: () => void;
 }
 
+type VideoStoryboardStreamData =
+  | {
+      stage: "detecting";
+      provider?: string;
+      message?: string;
+    }
+  | {
+      stage: "start";
+      provider?: string;
+      taskId: string;
+      totalShots: number;
+      confidence?: number;
+      sourceImageUrl?: string;
+      shots?: Array<{
+        shotIndex: number;
+        tileImageUrl?: string;
+        prompt?: string;
+      }>;
+    }
+  | {
+      stage: "shot_completed";
+      provider?: string;
+      taskId: string;
+      shotIndex: number;
+      modelId?: string;
+      videoUrl: string;
+      text?: string;
+    }
+  | {
+      stage: "shot_failed";
+      provider?: string;
+      taskId: string;
+      shotIndex: number;
+      error?: string;
+    }
+  | {
+      stage: "completed";
+      provider?: string;
+      taskId: string;
+      totalShots?: number;
+      successCount?: number;
+      failedCount?: number;
+    }
+  | {
+      stage: "failed";
+      provider?: string;
+      taskId: string;
+      error?: string;
+    }
+  | {
+      stage: "shot_retry";
+      provider?: string;
+      taskId: string;
+      shotIndex: number;
+      attempt?: number;
+      modelId?: string;
+      error?: string;
+    };
+
 // 建议卡片数据 - 已移除
 const SUGGESTIONS: { title: string; desc: string; image: string }[] = [];
 const CANVAS_COMPOSER_PREFIX_DEFAULT = "让Buyi-ai";
@@ -1617,6 +1676,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
     signature: "",
     timestamp: 0,
   });
+  const storyboardNodeMapRef = useRef<Record<string, Record<number, string>>>({});
   const releaseSendGuard = useCallback(() => {
     sendGuardRef.current.inFlight = false;
   }, []);
@@ -3190,6 +3250,128 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
         } catch (e) {
           console.error("[CanvasChat] Failed to parse data-appendMessage:", e);
         }
+        return;
+      }
+
+      if (dataPart.type === "data-videoStoryboard") {
+        try {
+          const payload: VideoStoryboardStreamData =
+            typeof dataPart.data === "string"
+              ? JSON.parse(dataPart.data)
+              : dataPart.data;
+          if (!payload || typeof payload !== "object") return;
+
+          if (payload.stage === "start") {
+            const taskId = String(payload.taskId || "").trim();
+            if (!taskId) return;
+            if (storyboardNodeMapRef.current[taskId]) return;
+
+            const totalShots = Math.max(1, Math.min(9, Number(payload.totalShots || 9)));
+            const shotNodeMap: Record<number, string> = {};
+            const currentNodes = useCanvasStore.getState().nodes;
+            const baseOffset = currentNodes.length;
+            const cardWidth = 320;
+            const cardHeight = 196;
+            const gapX = 30;
+            const gapY = 24;
+            const startX = 120 + (baseOffset % 2) * 420;
+            const startY = 120 + Math.floor(baseOffset / 2) * 260;
+
+            for (let shotIndex = 1; shotIndex <= totalShots; shotIndex += 1) {
+              const shotData = payload.shots?.find((item) => Number(item.shotIndex) === shotIndex);
+              const col = (shotIndex - 1) % 3;
+              const row = Math.floor((shotIndex - 1) / 3);
+              const nodeId = addNode({
+                type: "video",
+                position: {
+                  x: startX + col * (cardWidth + gapX),
+                  y: startY + row * (cardHeight + gapY),
+                },
+                size: { width: cardWidth, height: cardHeight },
+                data: {
+                  src: "",
+                  status: "generating",
+                  prompt: shotData?.prompt || `分镜 ${shotIndex} 生成中`,
+                  coverImage: shotData?.tileImageUrl || "",
+                },
+                connections: [],
+              });
+              shotNodeMap[shotIndex] = nodeId;
+            }
+
+            storyboardNodeMapRef.current[taskId] = shotNodeMap;
+            toast.message(`九宫格视频任务已开始（${totalShots} 段）`);
+            return;
+          }
+
+          if (payload.stage === "shot_completed") {
+            const taskId = String(payload.taskId || "").trim();
+            const shotIndex = Math.max(1, Number(payload.shotIndex || 0));
+            const videoUrl = String(payload.videoUrl || "").trim();
+            if (!taskId || !shotIndex || !videoUrl) return;
+
+            let nodeId = storyboardNodeMapRef.current[taskId]?.[shotIndex];
+            if (!nodeId) {
+              nodeId = addNode({
+                type: "video",
+                position: { x: 140 + ((shotIndex - 1) % 3) * 350, y: 150 + Math.floor((shotIndex - 1) / 3) * 230 },
+                size: { width: 320, height: 196 },
+                data: {
+                  src: videoUrl,
+                  status: "completed",
+                  prompt: `分镜 ${shotIndex}`,
+                },
+                connections: [],
+              });
+              if (!storyboardNodeMapRef.current[taskId]) {
+                storyboardNodeMapRef.current[taskId] = {};
+              }
+              storyboardNodeMapRef.current[taskId]![shotIndex] = nodeId;
+              return;
+            }
+
+            const currentNode = useCanvasStore.getState().getNode(nodeId);
+            const currentData = ((currentNode?.data as any) || {}) as Record<string, any>;
+            updateNode(nodeId, {
+              data: {
+                ...currentData,
+                src: videoUrl,
+                status: "completed",
+              },
+            } as any);
+            return;
+          }
+
+          if (payload.stage === "shot_failed") {
+            const taskId = String(payload.taskId || "").trim();
+            const shotIndex = Math.max(1, Number(payload.shotIndex || 0));
+            if (!taskId || !shotIndex) return;
+            const nodeId = storyboardNodeMapRef.current[taskId]?.[shotIndex];
+            if (!nodeId) return;
+            const currentNode = useCanvasStore.getState().getNode(nodeId);
+            const currentData = ((currentNode?.data as any) || {}) as Record<string, any>;
+            updateNode(nodeId, {
+              data: {
+                ...currentData,
+                status: "failed",
+              },
+            } as any);
+            return;
+          }
+
+          if (payload.stage === "completed") {
+            const successCount = Math.max(0, Number(payload.successCount || 0));
+            const failedCount = Math.max(0, Number(payload.failedCount || 0));
+            toast.success(`九宫格视频完成：成功 ${successCount}，失败 ${failedCount}`);
+            return;
+          }
+
+          if (payload.stage === "failed") {
+            toast.error(payload.error || "九宫格视频任务失败");
+          }
+        } catch (e) {
+          console.error("[CanvasChat] Failed to parse data-videoStoryboard:", e);
+        }
       }
     },
     onError: (error) => {

docs/系统全链路开发文档-含九宫格视频Agent技能化方案.md

@@ -0,0 +1,599 @@
+# 系统全链路开发文档（基于当前项目）
+
+> 项目：`ai-chatbot-main`  
+> 文档目标：沉淀当前系统已实现能力，并给出“RUXA 视频九宫格 + Agent 技能化”完整可开发方案  
+> 适用范围：聊天页、无限画布、3D 生成、AI 详情、视频生成、存储迁移、后台模型配置
+
+---
+
+## 1. 项目定位与能力边界
+
+本系统是一个多模态 AI 创作平台，核心能力覆盖：
+
+1. 文本/图片/视频对话式生成（统一在聊天链路）
+2. 无限画布创作与多节点编排（图片、视频、文本、3D）
+3. Agent 智能路由与智能参数推荐
+4. AI 详情生成（电商详情页自动分镜 + 质检回修）
+5. 多供应商模型治理（后台可配置 provider/model）
+6. 多存储后端（S3/MinIO/OSS）与 OSS 降本迁移任务
+
+---
+
+## 2. 技术架构总览
+
+## 2.1 技术栈
+
+- 前端：Next.js 16 + React 19 + Tailwind + shadcn/ui
+- 后端：Next.js Route Handlers（Node Runtime）
+- AI 编排：`ai` SDK + LangGraph（Agent 流程）
+- 数据库：PostgreSQL + Drizzle ORM
+- 存储：S3 兼容存储 / Aliyun OSS / Vercel Blob（兼容）
+- 图像处理：`sharp`
+- 状态管理：Zustand（画布）
+
+## 2.2 关键目录
+
+- 聊天主链路：`/Users/lanmu/Desktop/ai-chatbot-main/app/(chat)/api/chat/route.ts`
+- 模型动态加载：`/Users/lanmu/Desktop/ai-chatbot-main/lib/ai/dynamic-provider.ts`
+- Agent 编排：`/Users/lanmu/Desktop/ai-chatbot-main/lib/ai/auto-agent-graph.ts`
+- 智能分析：`/Users/lanmu/Desktop/ai-chatbot-main/lib/ai/smart-analyzer.ts`
+- 无限画布页面：`/Users/lanmu/Desktop/ai-chatbot-main/app/canvas/[id]/page.tsx`
+- 画布聊天面板：`/Users/lanmu/Desktop/ai-chatbot-main/components/canvas/CanvasChatPanel.tsx`
+- 3D 页面：`/Users/lanmu/Desktop/ai-chatbot-main/app/canvas/[id]/model3d/page.tsx`
+- 3D API：`/Users/lanmu/Desktop/ai-chatbot-main/app/api/canvas/3d/tasks/route.ts`
+- 后台设置：`/Users/lanmu/Desktop/ai-chatbot-main/app/(admin)/settings/admin-settings-client.tsx`
+- 数据模型：`/Users/lanmu/Desktop/ai-chatbot-main/lib/db/schema.ts`
+- 存储迁移任务：`/Users/lanmu/Desktop/ai-chatbot-main/lib/jobs/oss-image-migration-job.ts`
+
+---
+
+## 3. 当前模型与供应商体系（已实现）
+
+## 3.1 Provider 类型（后台可配置）
+
+已接入的 provider 类型（数据库 + 后台一致）：
+
+- `alibaba`、`openai`、`custom`、`local`
+- `gemini`、`huanke-gemini`
+- `alibaba-image`、`ruxa-image`、`fal-image`、`volcano-vision`
+- `qianwen-video`、`doubao-video`、`fal-video`、`ruxa-video`、`volcano-video`
+- `tripo-3d`
+- 扩展兼容：`zhipu`、`volcano-multimodal`（由 `custom` 映射识别）
+
+参考实现：
+
+- `/Users/lanmu/Desktop/ai-chatbot-main/app/api/admin/ai-providers/route.ts`
+- `/Users/lanmu/Desktop/ai-chatbot-main/lib/ai/provider-defaults.ts`
+
+## 3.2 视频模型与模式能力（代码内置）
+
+视频能力模型通过 `capabilities + videoToolCapability` 驱动前端 UI 与参数限制：
+
+- RUXA：仅 `first-frame`（图生视频），比例 `16:9/9:16/1:1`，时长 `5/10`
+- Qianwen：`first-frame`
+- FAL：`first-frame/first-last-frame`
+- Volcano/Doubao：支持文生/首帧/首尾帧/多参考图（按模型能力动态裁剪）
+
+参考实现：
+
+- `/Users/lanmu/Desktop/ai-chatbot-main/lib/ai/video-tool-capability.ts`
+- `/Users/lanmu/Desktop/ai-chatbot-main/components/video-params-bar.tsx`
+
+## 3.3 代码中已出现的具体模型名称（非运行态 DB 全量）
+
+- RUXA 视频：`veo3.1`（后台模型可配置）
+- 实时画布默认模型：`fal-ai/flux-2/klein/realtime`
+- 火山预设模型族：
+  - `doubao-seedance-1-0-pro-250528`
+  - `doubao-seedance-1-0-pro-fast-251015`
+  - `doubao-seedance-1-0-lite-t2v-250428`
+  - `doubao-seedance-1-0-lite-i2v-250428`
+- Tripo3D 版本选项：
+  - `Turbo-v1.0-20250506`
+  - `v3.1-20260211`
+  - `v3.0-20250812`
+  - `v2.5-20250123`
+  - `v2.0-20240919`
+  - `v1.4-20240625`
+
+说明：系统最终“在用模型清单”来自数据库 `AiModel`，由后台“AI模型/本地AI模型”动态配置。
+
+---
+
+## 4. 核心链路说明（已实现）
+
+## 4.1 聊天与 Agent 模式
+
+入口：`POST /api/chat`  
+实现：`/Users/lanmu/Desktop/ai-chatbot-main/app/(chat)/api/chat/route.ts`
+
+主流程：
+
+1. 鉴权 + 请求去重（同 `chatId:messageId` 防重入）
+2. 自动模式判定（`selectedChatModel === "auto"` 或固定视觉模型 + deepThinking）
+3. 历史图片补全（当前无图时回溯最近生成图/上传图）
+4. 调用 LangGraph Agent：
+   - 意图识别（图编/图生/视频/问答）
+   - 友好回复
+   - 提示词优化
+   - 推荐图片/视频参数
+   - 路由到最佳模型
+5. 按 provider 分支执行（文本、图片、视频）
+6. 流式返回 + 消息持久化 + 积分/配额扣减
+
+## 4.2 无限画布
+
+页面：`/canvas/[id]`  
+实现：
+
+- `/Users/lanmu/Desktop/ai-chatbot-main/app/canvas/[id]/page.tsx`
+- `/Users/lanmu/Desktop/ai-chatbot-main/components/canvas/CanvasWorkspace.tsx`
+- `/Users/lanmu/Desktop/ai-chatbot-main/store/canvas/index.ts`
+
+能力：
+
+- 节点类型：`message/image/video/text/scene-fusion/model3d`
+- 自动保存 + 节流重试（429 后重试）
+- 节点操作：拖拽、缩放、撤销/重做、多选
+- 画布与聊天联动（图片可回注对话）
+
+## 4.3 AI 详情生成
+
+核心接口：
+
+- `POST /api/canvas/detail-plan`
+- `POST /api/canvas/detail-review`
+- `GET|POST|PATCH|DELETE /api/canvas/[id]/detail-archives`
+
+核心能力：
+
+1. 电商详情 brief 结构化输入
+2. 分屏规划（传统生图 / `agent_layout` 可编辑文字层）
+3. 逐屏生成 + 质检 + 二次修图
+4. 存档回放（`canvasDetailArchive`）
+
+主要实现：
+
+- `/Users/lanmu/Desktop/ai-chatbot-main/components/canvas/CanvasChatPanel.tsx`
+- `/Users/lanmu/Desktop/ai-chatbot-main/app/api/canvas/detail-plan/route.ts`
+- `/Users/lanmu/Desktop/ai-chatbot-main/app/api/canvas/detail-review/route.ts`
+
+## 4.4 3D 生成（Tripo3D）
+
+页面：`/canvas/[id]/model3d`  
+API：
+
+- `POST /api/canvas/3d/upload-image`
+- `POST /api/canvas/3d/tasks`
+- `GET /api/canvas/3d/tasks/[taskId]`
+- `GET /api/canvas/3d/balance`
+
+后端关键点：
+
+- 从 `AiProvider(type="tripo-3d")` 读取 key/baseUrl
+- 对 API key 做隐藏字符清洗（解决 ByteString 8232 问题）
+- 统一 Tripo 错误结构返回（含 traceId）
+
+主要实现：
+
+- `/Users/lanmu/Desktop/ai-chatbot-main/lib/tripo/server.ts`
+- `/Users/lanmu/Desktop/ai-chatbot-main/lib/tripo/client.ts`
+
+## 4.5 存储与 OSS 迁移降本
+
+现状策略（已上线）：
+
+1. 上传优先走可用存储配置（支持 OSS 作为入口桶）
+2. 记录迁移任务（`StorageMigrationTask`）
+3. 定时任务将 OSS 图片转存到默认桶
+4. 回写数据库 URL（消息、画布、详情存档）
+5. 到期清理 OSS 源对象
+
+接口：
+
+- `POST /api/jobs/migrate-oss-images`
+- `GET /api/jobs/migrate-oss-images`
+
+主要实现：
+
+- `/Users/lanmu/Desktop/ai-chatbot-main/lib/storage/service.ts`
+- `/Users/lanmu/Desktop/ai-chatbot-main/lib/jobs/oss-image-migration-job.ts`
+- `/Users/lanmu/Desktop/ai-chatbot-main/app/api/jobs/migrate-oss-images/route.ts`
+
+---
+
+## 5. 数据库核心表（当前）
+
+主要业务表：
+
+- `AiProvider`：供应商配置
+- `AiModel`：模型配置、能力、故障迁移字段
+- `StorageConfig`：存储桶配置
+- `StorageMigrationTask`：OSS 迁移任务
+- `SiteSettings` + `SystemSettings`：站点与功能开关
+- `Message`：消息主表（含视频存储元数据）
+- `GeneratedImage`：图片记录
+- `CanvasProject`、`CanvasNode`、`CanvasImageTask`
+- `CanvasDetailArchive`：AI详情归档
+
+定义文件：`/Users/lanmu/Desktop/ai-chatbot-main/lib/db/schema.ts`
+
+---
+
+## 6. 新需求：RUXA 视频九宫格 + Agent 技能化（待开发）
+
+## 6.1 需求归纳
+
+当用户在视频模式选中 `RUXA 视频` 时：
+
+1. Agent 判断用户上传图是否为九宫格
+2. 若是九宫格，切成 9 张图
+3. 用户脚本格式不固定，先由 Agent 归一为 9 镜头脚本
+4. 切图后尺寸偏小，后台预留“图片变清晰”调用链路（默认关闭，待测试后再开启）
+5. 每张图按对应脚本生成视频，串行发送，间隔 10 秒
+6. 失败自动重试；超限/失败可降级到其它视频模型
+7. 全流程可追踪进度，可回放任务结果
+
+## 6.2 合理性评估结论
+
+该需求合理，且与现有架构兼容；但需工程化约束：
+
+- 超分能力先预留不启用，通过配置开关控制，默认 `false`
+- 九宫格检测需置信度门槛，低置信度不自动切片
+- 串行队列必须持久化到 DB，避免重启丢任务
+
+---
+
+## 7. 九宫格技能链设计（建议实现）
+
+采用“技能链（Skill Chain）”封装，而不是把全部逻辑塞进 `chat/route.ts`。
+
+> 实施约束：当前阶段仅输出方案文档，不改动业务代码；开发时必须按“技能封装”方式落地。
+
+## 7.1 技能模块划分
+
+1. `script-normalizer-skill`  
+   输入任意脚本文本，输出标准 JSON（9 段）
+
+2. `nine-grid-detector-skill`  
+   判断是否九宫格 + 透视校正 + 切片 + 质量评分
+
+3. `tile-upscale-skill`  
+   对小图按阈值走增强（复用现有 enhance 能力，先预留默认关闭）
+
+4. `storyboard-mapper-skill`  
+   将 9 图与 9 段脚本一一映射，生成每段视频 prompt
+
+5. `video-sequencer-skill`  
+   串行调度（10s）+ 重试 + 降级模型 + 状态更新
+
+## 7.2 技能触发规则（必须）
+
+仅在满足以下条件时触发九宫格技能链：
+
+1. 当前模式为视频生成，且选中模型 provider 为 `ruxa-video`
+2. 用户消息包含至少 1 张图片
+3. `nine-grid-detector-skill` 识别结果 `isNineGrid=true`
+4. `confidence >= minGridConfidence`（默认 0.85）
+
+触发后的流程：
+
+1. 进入 `script-normalizer-skill`
+2. 再进入 `nine-grid-detector-skill` 切片
+3. 按开关决定是否进入 `tile-upscale-skill`
+4. 进入 `storyboard-mapper-skill`
+5. 最后进入 `video-sequencer-skill`
+
+未触发时（任一条件不满足）：
+
+- 直接走现有普通视频链路，不进入九宫格技能链
+
+## 7.3 建议代码落点
+
+- `lib/video-agent/skills/script-normalizer.ts`
+- `lib/video-agent/skills/nine-grid-detector.ts`
+- `lib/video-agent/skills/tile-upscaler.ts`
+- `lib/video-agent/skills/storyboard-mapper.ts`
+- `lib/video-agent/skills/video-sequencer.ts`
+- `lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts`
+
+聊天主链路只做路由：
+
+- 修改 `/Users/lanmu/Desktop/ai-chatbot-main/app/(chat)/api/chat/route.ts`
+  - 当 provider=`ruxa-video` 且满足“技能触发规则”时调用 pipeline
+
+---
+
+## 8. 防切错设计（重点）
+
+## 8.1 检测流程（四段式）
+
+1. **预检**：宽高比接近 1:1 或 3:3 拼图常见比例；分辨率 > 最低阈值  
+2. **网格线检测**：行列投影 + 边缘峰值，识别 2 条纵线 + 2 条横线  
+3. **几何校正**：若轻微透视，先矫正再切片  
+4. **切后复检**：每格清晰度、边界截断、相邻差异、主体完整度评分
+
+## 8.2 判定阈值建议
+
+- `grid_confidence >= 0.85`：自动进入九宫格流程
+- `0.65 ~ 0.85`：进入“待确认”状态（Agent 提示用户确认）
+- `< 0.65`：走普通单图视频流程
+
+## 8.3 输出结构建议
+
+```json
+{
+  "isNineGrid": true,
+  "confidence": 0.92,
+  "cropBoxes": [
+    {"index": 1, "x": 0, "y": 0, "w": 342, "h": 342}
+  ],
+  "qualityScore": 0.88
+}
+```
+
+---
+
+## 9. 脚本归一化与分镜映射
+
+## 9.1 脚本标准 JSON
+
+```json
+{
+  "language": "zh",
+  "globalStyle": "写实电影感",
+  "shots": [
+    {
+      "index": 1,
+      "durationSec": 3,
+      "action": "角色抬头看向镜头",
+      "camera": "轻微推进",
+      "mood": "期待",
+      "negativePrompt": "抖动,畸变,低清"
+    }
+  ]
+}
+```
+
+## 9.2 映射策略
+
+- 默认 `shot[i] -> tile[i]`
+- 若脚本中有显式编号（如“第3镜头”）优先按编号映射
+- 脚本不足 9 段：补全占位镜头（继承全局风格）
+- 脚本超过 9 段：截断并告知用户
+
+---
+
+## 10. 超分链路设计（复用现有能力）
+
+当前已有增强接口：`POST /api/enhance`
+
+建议：
+
+1. 抽出服务层（如 `lib/image/enhance-service.ts`），避免服务端内部再走 HTTP
+2. 增加特性开关 `enableTileUpscale`，默认 `false`（预留不启用）
+3. 仅当 `enableTileUpscale=true` 时，切片后按阈值触发超分（短边 < 768）
+4. 超分失败降级：保留原切片继续后续流程
+5. 记录 `enhanceApplied/enhanceProvider/enhanceLatencyMs`
+
+---
+
+## 11. 调度、重试、降级
+
+## 11.1 串行调度规则
+
+- 一次任务 9 段，严格串行
+- 相邻请求间隔 10 秒
+- 任一段失败：指数退避（10s/20s/40s），最多 3 次
+
+## 11.2 降级模型规则
+
+从后台 `AiModel` 读取，不硬编码：
+
+1. 先用当前选中模型
+2. 失败后按 `fallbackModelId1 -> fallbackModelId2`
+3. 若仍失败，按同 provider 可用模型池挑选
+
+---
+
+## 12. 数据库扩展设计（建议新增）
+
+新增任务主表：`VideoStoryboardTask`
+
+- `id`
+- `userId`
+- `chatId`
+- `providerType`（默认 `ruxa-video`）
+- `sourceImageUrl`
+- `isNineGrid`
+- `detectionConfidence`
+- `normalizedScript` (jsonb)
+- `status`（pending/running/partial_success/succeeded/failed/cancelled）
+- `currentShot`
+- `totalShots`（固定 9）
+- `createdAt/updatedAt/completedAt`
+
+新增子任务表：`VideoStoryboardShotTask`
+
+- `id`
+- `parentTaskId`
+- `shotIndex`
+- `tileImageUrl`
+- `upscaledImageUrl`
+- `prompt`
+- `modelId`
+- `fallbackChain` (jsonb)
+- `attemptCount`
+- `status`
+- `videoUrl`
+- `errorCode/errorMessage`
+- `startedAt/completedAt`
+
+---
+
+## 13. API 设计（建议新增）
+
+1. `POST /api/video/storyboard/create`  
+   创建九宫格视频任务，返回 `taskId`
+
+2. `GET /api/video/storyboard/tasks/:taskId`  
+   获取任务与每段进度
+
+3. `POST /api/video/storyboard/tasks/:taskId/cancel`  
+   取消任务
+
+4. `POST /api/jobs/video-storyboard-dispatch`  
+   调度器轮询（可由 cron/pm2 定时触发）
+
+---
+
+## 14. 与现有聊天链路集成方式
+
+在 `/api/chat` 增加分支：
+
+1. 当前 provider 是否 `ruxa-video`
+2. 用户是否上传图片
+3. 是否触发九宫格识别（可由配置开关控制）
+
+若命中：
+
+- 先回 SSE 占位消息（“正在识别九宫格与解析分镜...”）
+- 创建 storyboard 任务
+- 前端按 taskId 轮询并展示 9 段进度
+- 最终输出 9 条视频或合并汇总消息
+
+---
+
+## 15. 前端改造点（建议）
+
+- `components/video-params-bar.tsx`
+  - 新增“九宫格自动分镜”开关（RUXA 模式显示）
+- `components/messages.tsx`
+  - 支持展示分段进度条（1/9 ~ 9/9）
+- `components/chat.tsx` / `components/canvas/CanvasChatPanel.tsx`
+  - 扩展任务状态卡片（处理中、重试中、降级中）
+
+---
+
+## 16. 配置项设计（系统设置）
+
+在系统设置新增 `video_storyboard_config`：
+
+- `enabled`：是否启用九宫格流程
+- `providerWhitelist`：默认仅 `ruxa-video`
+- `minGridConfidence`：默认 0.85
+- `enableTileUpscale`：是否启用切片超分（默认 false，预留）
+- `sliceUpscaleMinEdge`：默认 768
+- `dispatchIntervalSec`：默认 10
+- `maxRetry`：默认 3
+- `enableFallback`：默认 true
+
+存储位置：`SystemSettings`
+
+---
+
+## 17. 监控与告警
+
+建议埋点：
+
+- `nine_grid_detect_success_rate`
+- `nine_grid_misdetect_rate`
+- `script_normalize_success_rate`
+- `shot_retry_rate`
+- `fallback_trigger_rate`
+- `storyboard_task_success_rate`
+- `avg_storyboard_total_latency`
+
+日志关键词建议统一：
+
+- `[VideoStoryboard]`
+- `[NineGridDetector]`
+- `[ScriptNormalizer]`
+- `[TileUpscaler]`
+- `[VideoSequencer]`
+
+---
+
+## 18. 测试与验收
+
+## 18.1 单元测试
+
+- 九宫格检测：标准图、弱网格图、非九宫格图
+- 脚本归一：口语化、表格化、混合语言
+- 映射规则：缺段/超段/乱序编号
+- 重试降级：模拟 429、余额不足、超时
+
+## 18.2 集成测试
+
+1. 上传九宫格 + 9段脚本 -> 全部成功
+2. 上传九宫格 + 混乱脚本 -> 归一成功
+3. 部分失败 -> 自动重试 + 降级成功
+4. 非九宫格 -> 正常走单图视频
+
+## 18.3 验收标准
+
+- 切片正确率 ≥ 95%
+- 9 段任务可追踪、可重试、可取消
+- 不出现并发超限（严格 10 秒间隔）
+- 失败可降级且不丢任务状态
+
+---
+
+## 19. 发布计划（建议）
+
+1. Phase 1：先上“检测 + 归一 + 任务表 + 状态查询”
+2. Phase 2：接入“超分能力预留 + 串行调度”（超分开关保持关闭）
+3. Phase 3：接入降级链路与前端进度 UI
+4. Phase 4：灰度开关 + 指标监控 + 回归优化
+
+---
+
+## 20. 风险与规避
+
+1. **误判九宫格**：加入置信度阈值与人工确认分支  
+2. **成本过高**：超分默认关闭，待效果验证后灰度开启  
+3. **任务堆积**：DB 队列 + 限流 + 可取消机制  
+4. **外部 API 不稳定**：重试 + fallback + 错误分级  
+5. **多机部署一致性**：调度基于 DB 状态机，不依赖内存队列
+
+---
+
+## 21. 当前实施状态标记
+
+- 聊天/画布/3D/AI详情/OSS迁移：**已实现并在代码中可见**
+- RUXA 九宫格视频技能链：**本文档为待开发方案（尚未落地代码）**
+
+---
+
+## 22. 运维与发布命令（项目现状）
+
+- 安装依赖：`pnpm install --no-frozen-lockfile`
+- 数据库迁移：`pnpm db:migrate`
+- 构建：`pnpm build`
+- 启动：`pnpm start`
+- PM2 重启：`pm2 restart ai-chatbot || pm2 restart all`
+
+---
+
+## 23. 附：本方案主要影响文件清单（开发阶段）
+
+新增（建议）：
+
+- `lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts`
+- `lib/video-agent/skills/script-normalizer.ts`
+- `lib/video-agent/skills/nine-grid-detector.ts`
+- `lib/video-agent/skills/tile-upscaler.ts`
+- `lib/video-agent/skills/storyboard-mapper.ts`
+- `lib/video-agent/skills/video-sequencer.ts`
+- `app/api/video/storyboard/create/route.ts`
+- `app/api/video/storyboard/tasks/[taskId]/route.ts`
+- `app/api/jobs/video-storyboard-dispatch/route.ts`
+- `lib/db/migrations/00xx_add_video_storyboard_tasks.sql`
+
+修改（建议）：
+
+- `app/(chat)/api/chat/route.ts`
+- `lib/ai/providers/ruxa-video.ts`
+- `app/(admin)/settings/admin-settings-client.tsx`
+- `lib/db/schema.ts`
+- `lib/db/queries.ts`

lib/video-agent/config.ts

@@ -0,0 +1,81 @@
+import "server-only";
+
+import { getSystemSetting } from "@/lib/db/queries";
+
+export const VIDEO_STORYBOARD_CONFIG_KEY = "video_storyboard_config";
+
+export type VideoStoryboardConfig = {
+  enabled: boolean;
+  providerWhitelist: string[];
+  minGridConfidence: number;
+  enableTileUpscale: boolean;
+  sliceUpscaleMinEdge: number;
+  dispatchIntervalSec: number;
+  maxRetry: number;
+  enableFallback: boolean;
+};
+
+export const defaultVideoStoryboardConfig: VideoStoryboardConfig = {
+  enabled: true,
+  providerWhitelist: ["ruxa-video"],
+  minGridConfidence: 0.85,
+  enableTileUpscale: false,
+  sliceUpscaleMinEdge: 768,
+  dispatchIntervalSec: 10,
+  maxRetry: 3,
+  enableFallback: true,
+};
+
+function toNumber(value: unknown, fallback: number) {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
+export function normalizeVideoStoryboardConfig(
+  value: unknown
+): VideoStoryboardConfig {
+  const raw =
+    value && typeof value === "object" && !Array.isArray(value)
+      ? (value as Record<string, unknown>)
+      : {};
+
+  const whitelist = Array.isArray(raw.providerWhitelist)
+    ? raw.providerWhitelist
+        .map((item) => String(item || "").trim())
+        .filter(Boolean)
+    : defaultVideoStoryboardConfig.providerWhitelist;
+
+  return {
+    enabled: raw.enabled !== false,
+    providerWhitelist:
+      whitelist.length > 0
+        ? Array.from(new Set(whitelist))
+        : defaultVideoStoryboardConfig.providerWhitelist,
+    minGridConfidence: Math.max(
+      0.5,
+      Math.min(0.99, toNumber(raw.minGridConfidence, defaultVideoStoryboardConfig.minGridConfidence))
+    ),
+    enableTileUpscale: raw.enableTileUpscale === true,
+    sliceUpscaleMinEdge: Math.max(
+      256,
+      Math.min(4096, Math.round(toNumber(raw.sliceUpscaleMinEdge, defaultVideoStoryboardConfig.sliceUpscaleMinEdge)))
+    ),
+    dispatchIntervalSec: Math.max(
+      1,
+      Math.min(120, Math.round(toNumber(raw.dispatchIntervalSec, defaultVideoStoryboardConfig.dispatchIntervalSec)))
+    ),
+    maxRetry: Math.max(
+      0,
+      Math.min(5, Math.round(toNumber(raw.maxRetry, defaultVideoStoryboardConfig.maxRetry)))
+    ),
+    enableFallback: raw.enableFallback !== false,
+  };
+}
+
+export async function getVideoStoryboardConfig(): Promise<VideoStoryboardConfig> {
+  const setting = await getSystemSetting({ key: VIDEO_STORYBOARD_CONFIG_KEY }).catch(
+    () => undefined
+  );
+  return normalizeVideoStoryboardConfig(setting?.value);
+}
+

lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts

@@ -0,0 +1,316 @@
+import "server-only";
+
+import { generateUUID } from "@/lib/utils";
+import { runNineGridDetectorSkill } from "@/lib/video-agent/skills/nine-grid-detector";
+import { runScriptNormalizerSkill } from "@/lib/video-agent/skills/script-normalizer";
+import { runStoryboardMapperSkill } from "@/lib/video-agent/skills/storyboard-mapper";
+import { runTileUpscalerSkill } from "@/lib/video-agent/skills/tile-upscaler";
+import { runVideoSequencerSkill } from "@/lib/video-agent/skills/video-sequencer";
+import type {
+  RuxaNineGridPipelineParams,
+  RuxaNineGridPipelineResult,
+  StoryboardModelCandidate,
+} from "@/lib/video-agent/types";
+
+const NINE_GRID_SHOT_COUNT = 9;
+
+function normalizeText(value: unknown) {
+  return String(value || "").trim();
+}
+
+function extractVideoUrlFromText(text: string): string | null {
+  const normalizedText = String(text || "");
+  if (!normalizedText) return null;
+
+  const htmlMatch = normalizedText.match(/<video[^>]+src="([^"]+)"/i);
+  if (htmlMatch?.[1]) return htmlMatch[1];
+
+  const markdownMatch = normalizedText.match(
+    /\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i
+  );
+  if (markdownMatch?.[1]) return markdownMatch[1];
+
+  const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i);
+  return plainMatch?.[1] || null;
+}
+
+function extractPromptAndImages(messages: Array<{ role: string; content: any }>) {
+  const lastUserMessage = messages.filter((message) => message.role === "user").pop();
+  if (!lastUserMessage) {
+    return { prompt: "", imageUrls: [] as string[] };
+  }
+
+  let prompt = "";
+  const imageUrls: string[] = [];
+
+  const pushImage = (raw?: string) => {
+    const url = normalizeText(raw);
+    if (!url || !/^https?:\/\//i.test(url)) return;
+    if (!imageUrls.includes(url)) imageUrls.push(url);
+  };
+
+  if (typeof lastUserMessage.content === "string") {
+    prompt = normalizeText(lastUserMessage.content);
+  } else if (Array.isArray(lastUserMessage.content)) {
+    for (const part of lastUserMessage.content) {
+      if (part?.type === "text" && typeof part.text === "string") {
+        prompt = `${prompt} ${part.text}`.trim();
+      } else if (part?.type === "image_url" && part.image_url?.url) {
+        pushImage(part.image_url.url);
+      } else if (part?.type === "file") {
+        pushImage(part.url || part.data);
+      } else if (part?.type === "image" && typeof part.image === "string") {
+        pushImage(part.image);
+      }
+    }
+  }
+
+  if (!prompt) {
+    prompt = "让画面动起来";
+  }
+
+  return { prompt, imageUrls };
+}
+
+function normalizeModelCandidates(params: {
+  selectedModelId: string;
+  selectedModelRef: string;
+  modelCandidates: StoryboardModelCandidate[];
+}) {
+  const selected: StoryboardModelCandidate = {
+    id: params.selectedModelRef || params.selectedModelId,
+    modelId: params.selectedModelId,
+    displayName: "",
+  };
+  const candidates = [selected, ...params.modelCandidates]
+    .filter((item) => normalizeText(item.modelId))
+    .map((item) => ({
+      id: normalizeText(item.id) || normalizeText(item.modelId),
+      modelId: normalizeText(item.modelId),
+      displayName: normalizeText(item.displayName),
+    }));
+
+  const seen = new Set<string>();
+  return candidates.filter((item) => {
+    if (seen.has(item.modelId)) return false;
+    seen.add(item.modelId);
+    return true;
+  });
+}
+
+function buildCombinedMarkdown(params: {
+  success: Array<{ shotIndex: number; text: string }>;
+  failed: Array<{ shotIndex: number; error: string }>;
+}) {
+  const sections: string[] = [];
+
+  for (const shot of params.success.sort((a, b) => a.shotIndex - b.shotIndex)) {
+    sections.push(`### 分镜 ${shot.shotIndex}\n${shot.text}`);
+  }
+
+  if (params.failed.length > 0) {
+    const failedLines = params.failed
+      .sort((a, b) => a.shotIndex - b.shotIndex)
+      .map((item) => `- 分镜 ${item.shotIndex}: ${item.error}`);
+    sections.push(["### 失败分镜", ...failedLines].join("\n"));
+  }
+
+  return sections.join("\n\n").trim();
+}
+
+export async function runRuxaNineGridPipeline(
+  params: RuxaNineGridPipelineParams
+): Promise<RuxaNineGridPipelineResult> {
+  const {
+    modelMessages,
+    selectedModelId,
+    selectedModelRef,
+    modelCandidates,
+    videoParams,
+    config,
+    onProgress,
+    doGenerate,
+  } = params;
+
+  if (!config.enabled) {
+    return { triggered: false, reason: "disabled" };
+  }
+
+  const providerAllowed = config.providerWhitelist.some(
+    (item) => normalizeText(item).toLowerCase() === "ruxa-video"
+  );
+  if (!providerAllowed) {
+    return { triggered: false, reason: "provider_not_allowed" };
+  }
+
+  onProgress?.({
+    stage: "detecting",
+    provider: "ruxa-video",
+    message: "正在识别九宫格与解析分镜...",
+  });
+
+  const { prompt, imageUrls } = extractPromptAndImages(modelMessages);
+  if (!imageUrls[0]) {
+    return { triggered: false, reason: "missing_image" };
+  }
+
+  const detection = await runNineGridDetectorSkill({
+    imageUrl: imageUrls[0],
+    minConfidence: config.minGridConfidence,
+  });
+
+  if (!detection.isNineGrid || detection.tiles.length !== NINE_GRID_SHOT_COUNT) {
+    return {
+      triggered: false,
+      reason: detection.isNineGrid ? "slice_incomplete" : "not_nine_grid",
+      confidence: detection.confidence,
+    };
+  }
+
+  const normalizedScript = runScriptNormalizerSkill({
+    scriptText: prompt,
+    totalShots: NINE_GRID_SHOT_COUNT,
+    totalDurationSec: Number(videoParams?.duration || 10),
+  });
+  const mappedShots = runStoryboardMapperSkill({
+    script: normalizedScript,
+    tiles: detection.tiles,
+  });
+  const preparedShots = await runTileUpscalerSkill({
+    shots: mappedShots,
+    enableTileUpscale: config.enableTileUpscale,
+    sliceUpscaleMinEdge: config.sliceUpscaleMinEdge,
+  });
+
+  const resolvedModelCandidates = normalizeModelCandidates({
+    selectedModelId,
+    selectedModelRef,
+    modelCandidates,
+  });
+
+  const taskId = `ruxa-storyboard-${generateUUID()}`;
+  onProgress?.({
+    stage: "start",
+    provider: "ruxa-video",
+    taskId,
+    totalShots: preparedShots.length,
+    confidence: detection.confidence,
+    sourceImageUrl: imageUrls[0],
+    shots: preparedShots.map((shot) => ({
+      shotIndex: shot.shotIndex,
+      tileImageUrl: shot.inputImageUrl,
+      prompt: shot.prompt,
+    })),
+  });
+
+  const sequencer = await runVideoSequencerSkill({
+    provider: "ruxa-video",
+    taskId,
+    shots: preparedShots,
+    modelCandidates: resolvedModelCandidates,
+    maxRetry: config.maxRetry,
+    dispatchIntervalSec: config.dispatchIntervalSec,
+    enableFallback: config.enableFallback,
+    onProgress,
+    generateOneShot: async ({ shot, model }) => {
+      const shotMessages = [
+        {
+          role: "user",
+          content: [
+            { type: "text", text: shot.prompt },
+            { type: "image_url", image_url: { url: shot.inputImageUrl } },
+          ],
+        },
+      ];
+
+      const output = await doGenerate({
+        messages: shotMessages,
+        videoParams: {
+          ...(videoParams || {}),
+          mode: "first-frame",
+        },
+        overrideModelId: model.modelId,
+      });
+
+      const text = normalizeText(output?.text);
+      const videoUrl = extractVideoUrlFromText(text);
+      if (!videoUrl) {
+        throw new Error(text || "未返回视频地址");
+      }
+      return { videoUrl, text };
+    },
+  });
+
+  const successShots = sequencer.results
+    .filter((item) => item.success && item.videoUrl)
+    .map((item) => ({
+      shotIndex: item.shotIndex,
+      text: normalizeText(item.text),
+      videoUrl: normalizeText(item.videoUrl),
+      modelId: normalizeText(item.modelId),
+    }));
+  const failedShots = sequencer.results
+    .filter((item) => !item.success)
+    .map((item) => ({
+      shotIndex: item.shotIndex,
+      error: normalizeText(item.error) || "video_generation_failed",
+    }));
+
+  if (successShots.length === 0) {
+    onProgress?.({
+      stage: "failed",
+      provider: "ruxa-video",
+      taskId,
+      error: "all_shots_failed",
+    });
+    return {
+      triggered: true,
+      taskId,
+      confidence: detection.confidence,
+      reason: "all_shots_failed",
+      videoUrls: [],
+      modelRefs: [],
+      markdown: "视频生成失败，请稍后重试",
+      metadata: {
+        taskId,
+        successCount: 0,
+        failedCount: failedShots.length,
+        confidence: detection.confidence,
+        failedShots,
+      },
+    };
+  }
+
+  onProgress?.({
+    stage: "completed",
+    provider: "ruxa-video",
+    taskId,
+    totalShots: sequencer.results.length,
+    successCount: sequencer.successCount,
+    failedCount: sequencer.failedCount,
+  });
+
+  return {
+    triggered: true,
+    taskId,
+    confidence: detection.confidence,
+    markdown: buildCombinedMarkdown({
+      success: successShots.map((item) => ({
+        shotIndex: item.shotIndex,
+        text: item.text,
+      })),
+      failed: failedShots,
+    }),
+    videoUrls: successShots.map((item) => item.videoUrl),
+    modelRefs: successShots.map((item) => item.modelId).filter(Boolean),
+    metadata: {
+      taskId,
+      confidence: detection.confidence,
+      qualityScore: detection.qualityScore,
+      sourceImageUrl: detection.sourceImageUrl,
+      successCount: sequencer.successCount,
+      failedCount: sequencer.failedCount,
+      failedShots,
+    },
+  };
+}

lib/video-agent/skills/nine-grid-detector.ts

@@ -0,0 +1,251 @@
+import "server-only";
+
+import sharp from "sharp";
+import { uploadFile } from "@/lib/storage/service";
+import type { NineGridCropBox, NineGridDetectionResult, NineGridTile } from "@/lib/video-agent/types";
+
+type ProjectionPeak = {
+  index: number;
+  value: number;
+};
+
+function clamp01(value: number) {
+  if (!Number.isFinite(value)) return 0;
+  return Math.max(0, Math.min(1, value));
+}
+
+function sampleStddevFromUint8(data: Uint8Array, limit = 20000) {
+  const total = Math.min(data.length, Math.max(128, limit));
+  if (total <= 0) return 0;
+  const stride = Math.max(1, Math.floor(data.length / total));
+  const sampled: number[] = [];
+  for (let index = 0; index < data.length && sampled.length < total; index += stride) {
+    sampled.push(data[index] || 0);
+  }
+  return stddev(sampled);
+}
+
+function toArrayBuffer(buffer: Buffer): ArrayBuffer {
+  return buffer.buffer.slice(
+    buffer.byteOffset,
+    buffer.byteOffset + buffer.byteLength
+  ) as ArrayBuffer;
+}
+
+function movingAverage(values: number[], radius = 3): number[] {
+  if (values.length === 0) return values;
+  const result = new Array(values.length).fill(0);
+  for (let index = 0; index < values.length; index += 1) {
+    let sum = 0;
+    let count = 0;
+    const start = Math.max(0, index - radius);
+    const end = Math.min(values.length - 1, index + radius);
+    for (let cursor = start; cursor <= end; cursor += 1) {
+      sum += values[cursor] || 0;
+      count += 1;
+    }
+    result[index] = count > 0 ? sum / count : 0;
+  }
+  return result;
+}
+
+function mean(values: number[]) {
+  if (values.length === 0) return 0;
+  return values.reduce((total, item) => total + item, 0) / values.length;
+}
+
+function stddev(values: number[]) {
+  if (values.length === 0) return 0;
+  const avg = mean(values);
+  const variance =
+    values.reduce((total, item) => total + Math.pow(item - avg, 2), 0) / values.length;
+  return Math.sqrt(Math.max(variance, 0));
+}
+
+function findPeakNear(
+  scores: number[],
+  targetIndex: number,
+  windowRadius: number
+): ProjectionPeak {
+  const start = Math.max(0, Math.round(targetIndex - windowRadius));
+  const end = Math.min(scores.length - 1, Math.round(targetIndex + windowRadius));
+  let bestIndex = Math.round(targetIndex);
+  let bestValue = -Infinity;
+  for (let index = start; index <= end; index += 1) {
+    const value = scores[index] || 0;
+    if (value > bestValue) {
+      bestValue = value;
+      bestIndex = index;
+    }
+  }
+  return { index: bestIndex, value: Number.isFinite(bestValue) ? bestValue : 0 };
+}
+
+function scoreGridAxis(scores: number[], firstTarget: number, secondTarget: number) {
+  const radius = Math.max(8, Math.round(scores.length * 0.12));
+  const firstPeak = findPeakNear(scores, firstTarget, radius);
+  const secondPeak = findPeakNear(scores, secondTarget, radius);
+  const avg = mean(scores);
+  const sigma = stddev(scores);
+  const dynamicBase = avg + sigma;
+
+  const closenessA = clamp01(1 - Math.abs(firstPeak.index - firstTarget) / radius);
+  const closenessB = clamp01(1 - Math.abs(secondPeak.index - secondTarget) / radius);
+  const closeness = (closenessA + closenessB) / 2;
+
+  const denom = Math.max(dynamicBase, 1);
+  const strengthA = clamp01(firstPeak.value / (denom * 2.2));
+  const strengthB = clamp01(secondPeak.value / (denom * 2.2));
+  const strength = (strengthA + strengthB) / 2;
+
+  return {
+    score: clamp01(closeness * 0.55 + strength * 0.45),
+    peaks: [firstPeak, secondPeak],
+  };
+}
+
+function buildNineGridCropBoxes(width: number, height: number): NineGridCropBox[] {
+  const baseW = Math.floor(width / 3);
+  const baseH = Math.floor(height / 3);
+  const boxes: NineGridCropBox[] = [];
+  let index = 1;
+  for (let row = 0; row < 3; row += 1) {
+    for (let col = 0; col < 3; col += 1) {
+      const x = col * baseW;
+      const y = row * baseH;
+      const w = col === 2 ? width - x : baseW;
+      const h = row === 2 ? height - y : baseH;
+      boxes.push({ index, x, y, w, h });
+      index += 1;
+    }
+  }
+  return boxes;
+}
+
+async function fetchImageBuffer(imageUrl: string) {
+  const response = await fetch(imageUrl);
+  if (!response.ok) {
+    throw new Error(`图片下载失败 (HTTP ${response.status})`);
+  }
+  const contentType = response.headers.get("content-type") || "";
+  if (!/^image\//i.test(contentType)) {
+    throw new Error(`输入不是图片类型: ${contentType || "unknown"}`);
+  }
+  return Buffer.from(await response.arrayBuffer());
+}
+
+async function sliceAndUploadTiles(params: {
+  sourceBuffer: Buffer;
+  cropBoxes: NineGridCropBox[];
+}): Promise<NineGridTile[]> {
+  const { sourceBuffer, cropBoxes } = params;
+  const timestamp = Date.now();
+  const tiles: NineGridTile[] = [];
+  for (const cropBox of cropBoxes) {
+    const tileBuffer = await sharp(sourceBuffer)
+      .extract({
+        left: cropBox.x,
+        top: cropBox.y,
+        width: cropBox.w,
+        height: cropBox.h,
+      })
+      .jpeg({ quality: 92 })
+      .toBuffer();
+
+    const filename = `video-storyboard-tile-${timestamp}-${cropBox.index}.jpg`;
+    const uploaded = await uploadFile(filename, toArrayBuffer(tileBuffer));
+    tiles.push({
+      index: cropBox.index,
+      imageUrl: uploaded.url,
+      width: cropBox.w,
+      height: cropBox.h,
+    });
+  }
+  return tiles;
+}
+
+export async function runNineGridDetectorSkill(params: {
+  imageUrl: string;
+  minConfidence: number;
+}): Promise<NineGridDetectionResult> {
+  const sourceImageUrl = String(params.imageUrl || "").trim();
+  if (!sourceImageUrl) {
+    throw new Error("缺少九宫格输入图片");
+  }
+
+  const sourceBuffer = await fetchImageBuffer(sourceImageUrl);
+  const baseImage = sharp(sourceBuffer, { failOn: "none" }).rotate();
+  const metadata = await baseImage.metadata();
+  const width = Number(metadata.width || 0);
+  const height = Number(metadata.height || 0);
+
+  if (width < 240 || height < 240) {
+    return {
+      isNineGrid: false,
+      confidence: 0,
+      qualityScore: 0,
+      cropBoxes: [],
+      width,
+      height,
+      sourceImageUrl,
+      tiles: [],
+    };
+  }
+
+  const grayscale = await baseImage.greyscale().raw().toBuffer({
+    resolveWithObject: true,
+  });
+  const raw = grayscale.data;
+  const channels = grayscale.info.channels || 1;
+  const imgW = grayscale.info.width;
+  const imgH = grayscale.info.height;
+  const step = Math.max(1, Math.round(Math.min(imgW, imgH) / 900));
+
+  const colScores = new Array(imgW).fill(0);
+  const rowScores = new Array(imgH).fill(0);
+
+  for (let y = 0; y < imgH; y += step) {
+    const rowOffset = y * imgW * channels;
+    for (let x = step; x < imgW; x += step) {
+      const current = raw[rowOffset + x * channels] || 0;
+      const prev = raw[rowOffset + (x - step) * channels] || 0;
+      colScores[x] += Math.abs(current - prev);
+    }
+  }
+
+  for (let x = 0; x < imgW; x += step) {
+    for (let y = step; y < imgH; y += step) {
+      const current = raw[(y * imgW + x) * channels] || 0;
+      const prev = raw[((y - step) * imgW + x) * channels] || 0;
+      rowScores[y] += Math.abs(current - prev);
+    }
+  }
+
+  const smoothCols = movingAverage(colScores, 5);
+  const smoothRows = movingAverage(rowScores, 5);
+  const colAxis = scoreGridAxis(smoothCols, imgW / 3, (imgW * 2) / 3);
+  const rowAxis = scoreGridAxis(smoothRows, imgH / 3, (imgH * 2) / 3);
+
+  const ratio = width / Math.max(height, 1);
+  const aspectScore = clamp01(1 - Math.min(Math.abs(ratio - 1), 1));
+  const gridScore = clamp01((colAxis.score + rowAxis.score) / 2);
+  const confidence = clamp01(aspectScore * 0.3 + gridScore * 0.7);
+  const qualityScore = clamp01((sampleStddevFromUint8(raw, 20000) || 0) / 72);
+
+  const isNineGrid = confidence >= params.minConfidence && aspectScore >= 0.62;
+  const cropBoxes = isNineGrid ? buildNineGridCropBoxes(width, height) : [];
+  const tiles = isNineGrid
+    ? await sliceAndUploadTiles({ sourceBuffer: await baseImage.toBuffer(), cropBoxes })
+    : [];
+
+  return {
+    isNineGrid,
+    confidence,
+    qualityScore,
+    cropBoxes,
+    width,
+    height,
+    sourceImageUrl,
+    tiles,
+  };
+}

lib/video-agent/skills/script-normalizer.ts

@@ -0,0 +1,109 @@
+import "server-only";
+
+import type { NormalizedStoryboardScript, NormalizedStoryboardShot } from "@/lib/video-agent/types";
+
+function detectLanguage(text: string): "zh" | "en" | "mixed" {
+  const source = String(text || "");
+  const zhCount = (source.match(/[\u4e00-\u9fff]/g) || []).length;
+  const enCount = (source.match(/[A-Za-z]/g) || []).length;
+  if (zhCount > 0 && enCount > 0) return "mixed";
+  if (zhCount > 0) return "zh";
+  return "en";
+}
+
+function cleanLine(line: string) {
+  return String(line || "")
+    .replace(/^\s*(?:第?\s*\d+\s*(?:镜头|段|幕)|\d+\s*[.)、：:\-]?|[一二三四五六七八九十]+\s*[、.：:\-]?)\s*/g, "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function splitScriptLines(script: string): string[] {
+  const source = String(script || "").trim();
+  if (!source) return [];
+  const byLine = source
+    .split(/\r?\n+/)
+    .map(cleanLine)
+    .filter(Boolean);
+  if (byLine.length >= 4) return byLine;
+
+  return source
+    .split(/[。！？!?；;\n]+/g)
+    .map(cleanLine)
+    .filter(Boolean);
+}
+
+function inferCamera(text: string) {
+  const source = String(text || "");
+  if (/推进|推近|close[- ]?in|dolly in/i.test(source)) return "轻微推进";
+  if (/拉远|远景|zoom out|wide/i.test(source)) return "缓慢拉远";
+  if (/平移|横移|pan/i.test(source)) return "平稳平移";
+  if (/旋转|环绕|orbit/i.test(source)) return "环绕镜头";
+  return "稳定镜头";
+}
+
+function inferMood(text: string) {
+  const source = String(text || "");
+  if (/紧张|压迫|惊悚|tense/i.test(source)) return "紧张";
+  if (/温暖|治愈|暖/i.test(source)) return "温暖";
+  if (/活力|动感|快节奏|energetic/i.test(source)) return "活力";
+  if (/浪漫|梦幻|romantic/i.test(source)) return "浪漫";
+  return "自然";
+}
+
+function buildDefaultAction(index: number) {
+  const defaults = [
+    "主体从静止到轻微动作",
+    "主体与环境产生互动",
+    "主体视线发生转移",
+    "主体做细节动作展示",
+    "镜头展示关键部位",
+    "镜头拉近突出质感",
+    "镜头切换到侧面动态",
+    "主体完成动作收束",
+    "镜头定格形成结尾",
+  ];
+  return defaults[index - 1] || "主体做自然动作";
+}
+
+export function runScriptNormalizerSkill(params: {
+  scriptText: string;
+  totalShots?: number;
+  totalDurationSec?: number;
+}): NormalizedStoryboardScript {
+  const totalShots = Math.max(1, Math.min(20, Math.round(params.totalShots || 9)));
+  const raw = String(params.scriptText || "").trim();
+  const lines = splitScriptLines(raw);
+  const durationPerShot = Math.max(
+    2,
+    Math.round(Math.max(5, Number(params.totalDurationSec || 10)) / totalShots)
+  );
+
+  const normalizedShots: NormalizedStoryboardShot[] = [];
+  const language = detectLanguage(raw);
+  const fallbackStyle =
+    language === "en" ? "cinematic realistic style" : "写实电影感";
+
+  for (let index = 1; index <= totalShots; index += 1) {
+    const line = lines[index - 1] || lines[lines.length - 1] || "";
+    const action = cleanLine(line) || buildDefaultAction(index);
+    normalizedShots.push({
+      index,
+      durationSec: durationPerShot,
+      action,
+      camera: inferCamera(line),
+      mood: inferMood(line),
+      negativePrompt:
+        language === "en"
+          ? "jitter, blur, artifacts, low quality"
+          : "抖动, 畸变, 低清, 伪影",
+    });
+  }
+
+  return {
+    language,
+    globalStyle: fallbackStyle,
+    shots: normalizedShots,
+    raw,
+  };
+}

lib/video-agent/skills/storyboard-mapper.ts

@@ -0,0 +1,73 @@
+import "server-only";
+
+import type {
+  MappedStoryboardShot,
+  NineGridTile,
+  NormalizedStoryboardScript,
+} from "@/lib/video-agent/types";
+
+function clampIndex(value: number, length: number) {
+  if (length <= 0) return 0;
+  const rounded = Math.max(1, Math.round(value));
+  return Math.min(length, rounded) - 1;
+}
+
+function buildShotPrompt(params: {
+  style: string;
+  action: string;
+  camera: string;
+  mood: string;
+  language: "zh" | "en" | "mixed";
+}) {
+  const { style, action, camera, mood, language } = params;
+  if (language === "en") {
+    return [
+      "Animate this still image into a coherent short scene.",
+      `Action: ${action}.`,
+      `Camera: ${camera}.`,
+      `Mood: ${mood}.`,
+      `Style: ${style}.`,
+      "Keep identity and key structure stable.",
+    ].join(" ");
+  }
+
+  return [
+    "请基于当前输入图片生成连贯的视频镜头。",
+    `动作：${action}。`,
+    `镜头：${camera}。`,
+    `氛围：${mood}。`,
+    `风格：${style}。`,
+    "保持主体身份、构图和关键细节稳定。",
+  ].join("");
+}
+
+export function runStoryboardMapperSkill(params: {
+  script: NormalizedStoryboardScript;
+  tiles: NineGridTile[];
+}): MappedStoryboardShot[] {
+  const { script, tiles } = params;
+  if (!tiles.length) return [];
+
+  return script.shots.map((shot, index) => {
+    const tile = tiles[clampIndex(shot.index || index + 1, tiles.length)] || tiles[index] || tiles[0];
+    const prompt = buildShotPrompt({
+      style: script.globalStyle,
+      action: shot.action,
+      camera: shot.camera,
+      mood: shot.mood,
+      language: script.language,
+    });
+
+    return {
+      shotIndex: shot.index,
+      tileIndex: tile.index,
+      tileImageUrl: tile.imageUrl,
+      durationSec: shot.durationSec,
+      prompt,
+      negativePrompt: shot.negativePrompt,
+      action: shot.action,
+      camera: shot.camera,
+      mood: shot.mood,
+    };
+  });
+}

lib/video-agent/skills/tile-upscaler.ts

@@ -0,0 +1,18 @@
+import "server-only";
+
+import type { MappedStoryboardShot, PreparedStoryboardShot } from "@/lib/video-agent/types";
+
+export async function runTileUpscalerSkill(params: {
+  shots: MappedStoryboardShot[];
+  enableTileUpscale: boolean;
+  sliceUpscaleMinEdge: number;
+}): Promise<PreparedStoryboardShot[]> {
+  const { shots, enableTileUpscale } = params;
+
+  return shots.map((shot) => ({
+    ...shot,
+    inputImageUrl: shot.tileImageUrl,
+    upscaled: false,
+    upscaleReason: enableTileUpscale ? "reserved_not_enabled" : "disabled",
+  }));
+}

lib/video-agent/skills/video-sequencer.ts

@@ -0,0 +1,149 @@
+import "server-only";
+
+import type {
+  PreparedStoryboardShot,
+  StoryboardModelCandidate,
+  StoryboardProgressEvent,
+} from "@/lib/video-agent/types";
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+function normalizeErrorMessage(error: unknown) {
+  if (!error) return "unknown_error";
+  if (typeof error === "string") return error;
+  if (error instanceof Error) return error.message || String(error);
+  try {
+    return JSON.stringify(error);
+  } catch {
+    return String(error);
+  }
+}
+
+export type SequencerShotResult = {
+  shotIndex: number;
+  success: boolean;
+  videoUrl?: string;
+  modelId?: string;
+  text?: string;
+  error?: string;
+};
+
+export async function runVideoSequencerSkill(params: {
+  provider: string;
+  taskId: string;
+  shots: PreparedStoryboardShot[];
+  modelCandidates: StoryboardModelCandidate[];
+  maxRetry: number;
+  dispatchIntervalSec: number;
+  enableFallback: boolean;
+  onProgress?: (event: StoryboardProgressEvent) => void;
+  generateOneShot: (input: {
+    shot: PreparedStoryboardShot;
+    model: StoryboardModelCandidate;
+  }) => Promise<{ videoUrl: string; text: string }>;
+}): Promise<{
+  results: SequencerShotResult[];
+  successCount: number;
+  failedCount: number;
+}> {
+  const {
+    provider,
+    taskId,
+    shots,
+    modelCandidates,
+    maxRetry,
+    dispatchIntervalSec,
+    enableFallback,
+    onProgress,
+    generateOneShot,
+  } = params;
+
+  const candidateList =
+    enableFallback && modelCandidates.length > 0
+      ? modelCandidates
+      : modelCandidates.slice(0, 1);
+
+  const results: SequencerShotResult[] = [];
+
+  for (let index = 0; index < shots.length; index += 1) {
+    const shot = shots[index]!;
+    let shotSuccess = false;
+    let lastError = "";
+    let resultText = "";
+    let resultUrl = "";
+    let usedModelId = "";
+
+    for (const model of candidateList) {
+      const retries = Math.max(0, Math.round(maxRetry));
+      for (let attempt = 1; attempt <= retries + 1; attempt += 1) {
+        try {
+          const output = await generateOneShot({ shot, model });
+          resultText = output.text;
+          resultUrl = output.videoUrl;
+          usedModelId = model.modelId;
+          shotSuccess = true;
+          break;
+        } catch (error) {
+          lastError = normalizeErrorMessage(error);
+          if (attempt <= retries) {
+            onProgress?.({
+              stage: "shot_retry",
+              provider,
+              taskId,
+              shotIndex: shot.shotIndex,
+              attempt,
+              modelId: model.modelId,
+              error: lastError,
+            });
+            const backoffSec = Math.min(40, Math.pow(2, attempt - 1) * 10);
+            await sleep(backoffSec * 1000);
+          }
+        }
+      }
+
+      if (shotSuccess) {
+        break;
+      }
+    }
+
+    if (shotSuccess) {
+      results.push({
+        shotIndex: shot.shotIndex,
+        success: true,
+        modelId: usedModelId,
+        videoUrl: resultUrl,
+        text: resultText,
+      });
+      onProgress?.({
+        stage: "shot_completed",
+        provider,
+        taskId,
+        shotIndex: shot.shotIndex,
+        modelId: usedModelId,
+        videoUrl: resultUrl,
+        text: resultText,
+      });
+    } else {
+      results.push({
+        shotIndex: shot.shotIndex,
+        success: false,
+        error: lastError || "video_generation_failed",
+      });
+      onProgress?.({
+        stage: "shot_failed",
+        provider,
+        taskId,
+        shotIndex: shot.shotIndex,
+        error: lastError || "video_generation_failed",
+      });
+    }
+
+    if (index < shots.length - 1 && dispatchIntervalSec > 0) {
+      await sleep(Math.round(dispatchIntervalSec * 1000));
+    }
+  }
+
+  const successCount = results.filter((item) => item.success).length;
+  const failedCount = results.length - successCount;
+  return { results, successCount, failedCount };
+}

lib/video-agent/types.ts

@@ -0,0 +1,152 @@
+import type { VideoStoryboardConfig } from "@/lib/video-agent/config";
+
+export type NormalizedStoryboardShot = {
+  index: number;
+  durationSec: number;
+  action: string;
+  camera: string;
+  mood: string;
+  negativePrompt: string;
+};
+
+export type NormalizedStoryboardScript = {
+  language: "zh" | "en" | "mixed";
+  globalStyle: string;
+  shots: NormalizedStoryboardShot[];
+  raw: string;
+};
+
+export type NineGridCropBox = {
+  index: number;
+  x: number;
+  y: number;
+  w: number;
+  h: number;
+};
+
+export type NineGridTile = {
+  index: number;
+  imageUrl: string;
+  width: number;
+  height: number;
+};
+
+export type NineGridDetectionResult = {
+  isNineGrid: boolean;
+  confidence: number;
+  qualityScore: number;
+  cropBoxes: NineGridCropBox[];
+  width: number;
+  height: number;
+  sourceImageUrl: string;
+  tiles: NineGridTile[];
+};
+
+export type MappedStoryboardShot = {
+  shotIndex: number;
+  tileIndex: number;
+  tileImageUrl: string;
+  durationSec: number;
+  prompt: string;
+  negativePrompt: string;
+  action: string;
+  camera: string;
+  mood: string;
+};
+
+export type PreparedStoryboardShot = MappedStoryboardShot & {
+  inputImageUrl: string;
+  upscaled: boolean;
+  upscaleReason?: string;
+};
+
+export type StoryboardModelCandidate = {
+  id: string;
+  modelId: string;
+  displayName?: string;
+};
+
+export type StoryboardProgressEvent =
+  | {
+      stage: "detecting";
+      provider: string;
+      message?: string;
+    }
+  | {
+      stage: "start";
+      provider: string;
+      taskId: string;
+      totalShots: number;
+      confidence: number;
+      sourceImageUrl: string;
+      shots: Array<{
+        shotIndex: number;
+        tileImageUrl: string;
+        prompt: string;
+      }>;
+    }
+  | {
+      stage: "shot_retry";
+      provider: string;
+      taskId: string;
+      shotIndex: number;
+      attempt: number;
+      modelId: string;
+      error: string;
+    }
+  | {
+      stage: "shot_completed";
+      provider: string;
+      taskId: string;
+      shotIndex: number;
+      modelId: string;
+      videoUrl: string;
+      text: string;
+    }
+  | {
+      stage: "shot_failed";
+      provider: string;
+      taskId: string;
+      shotIndex: number;
+      error: string;
+    }
+  | {
+      stage: "completed";
+      provider: string;
+      taskId: string;
+      totalShots: number;
+      successCount: number;
+      failedCount: number;
+    }
+  | {
+      stage: "failed";
+      provider: string;
+      taskId: string;
+      error: string;
+    };
+
+export type RuxaNineGridPipelineParams = {
+  modelMessages: Array<{ role: string; content: any }>;
+  selectedModelId: string;
+  selectedModelRef: string;
+  modelCandidates: StoryboardModelCandidate[];
+  videoParams?: Record<string, any>;
+  config: VideoStoryboardConfig;
+  onProgress?: (event: StoryboardProgressEvent) => void;
+  doGenerate: (input: {
+    messages: Array<{ role: string; content: any }>;
+    videoParams?: Record<string, any>;
+    overrideModelId?: string;
+  }) => Promise<{ text?: string }>;
+};
+
+export type RuxaNineGridPipelineResult = {
+  triggered: boolean;
+  taskId?: string;
+  confidence?: number;
+  markdown?: string;
+  videoUrls?: string[];
+  modelRefs?: string[];
+  metadata?: Record<string, any>;
+  reason?: string;
+};