feat(video-agent): add ruxa nine-grid storyboard skill pipeline
This commit is contained in:
parent
75f93976ef
commit
20898e1ac1
@ -57,6 +57,9 @@ import { ChatSDKError } from "@/lib/errors";
|
|||||||
import { extractImageUrlsFromText } from "@/lib/image-url";
|
import { extractImageUrlsFromText } from "@/lib/image-url";
|
||||||
import { videoArchiver } from "@/lib/storage/video-archiver";
|
import { videoArchiver } from "@/lib/storage/video-archiver";
|
||||||
import { uploadFile } from "@/lib/storage/service";
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
|
import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
|
||||||
|
import { runRuxaNineGridPipeline } from "@/lib/video-agent/pipeline/ruxa-nine-grid-pipeline";
|
||||||
|
import type { StoryboardModelCandidate } from "@/lib/video-agent/types";
|
||||||
import type { ChatMessage } from "@/lib/types";
|
import type { ChatMessage } from "@/lib/types";
|
||||||
import type { AppUsage } from "@/lib/usage";
|
import type { AppUsage } from "@/lib/usage";
|
||||||
import { convertToUIMessages, generateUUID } from "@/lib/utils";
|
import { convertToUIMessages, generateUUID } from "@/lib/utils";
|
||||||
@ -359,6 +362,49 @@ function selectBestVideoSubModelForMode(
|
|||||||
})[0];
|
})[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function resolveStoryboardModelCandidates(
|
||||||
|
selectedModelRef: string
|
||||||
|
): Promise<StoryboardModelCandidate[]> {
|
||||||
|
const selectedModel = await getAiModelById({ id: selectedModelRef }).catch(() => undefined);
|
||||||
|
if (!selectedModel) return [];
|
||||||
|
|
||||||
|
const configuredFallbackIds = [
|
||||||
|
selectedModel.fallbackModelId1,
|
||||||
|
selectedModel.fallbackModelId2,
|
||||||
|
].filter((item): item is string => Boolean(String(item || "").trim()));
|
||||||
|
|
||||||
|
const allModels = await getEnabledAiModelsWithProvider(true, true).catch(() => []);
|
||||||
|
const sameProvider = allModels
|
||||||
|
.filter(
|
||||||
|
({ model, provider }: any) =>
|
||||||
|
provider?.id === selectedModel.providerId &&
|
||||||
|
model?.id !== selectedModel.id &&
|
||||||
|
model?.isEnabled
|
||||||
|
)
|
||||||
|
.map(({ model }: any) => String(model?.id || "").trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
|
||||||
|
const candidateIds = Array.from(
|
||||||
|
new Set([selectedModel.id, ...configuredFallbackIds, ...sameProvider])
|
||||||
|
);
|
||||||
|
const byId = new Map(
|
||||||
|
allModels
|
||||||
|
.map(({ model }: any) => model)
|
||||||
|
.filter(Boolean)
|
||||||
|
.map((model: any) => [String(model.id), model] as const)
|
||||||
|
);
|
||||||
|
|
||||||
|
return candidateIds
|
||||||
|
.map((id) => byId.get(id))
|
||||||
|
.filter(Boolean)
|
||||||
|
.map((model: any) => ({
|
||||||
|
id: String(model.id),
|
||||||
|
modelId: String(model.modelId || "").trim(),
|
||||||
|
displayName: String(model.displayName || model.modelId || "").trim(),
|
||||||
|
}))
|
||||||
|
.filter((item) => item.modelId);
|
||||||
|
}
|
||||||
|
|
||||||
function extractVideoUrlFromText(text: string): string | null {
|
function extractVideoUrlFromText(text: string): string | null {
|
||||||
const normalizedText = String(text || "");
|
const normalizedText = String(text || "");
|
||||||
if (!normalizedText) return null;
|
if (!normalizedText) return null;
|
||||||
@ -4124,22 +4170,10 @@ export async function POST(request: Request) {
|
|||||||
const modelMessages = convertToModelMessages(messagesForVideo);
|
const modelMessages = convertToModelMessages(messagesForVideo);
|
||||||
console.log(`[Chat API] ${currentVideoProvider} messages prepared, count:`, modelMessages.length);
|
console.log(`[Chat API] ${currentVideoProvider} messages prepared, count:`, modelMessages.length);
|
||||||
|
|
||||||
// 调用 doGenerate 获取视频
|
let videoMarkdown = "视频生成失败";
|
||||||
const result = await (languageModel as any).doGenerate({
|
let tempVideoUrl = "";
|
||||||
messages: modelMessages,
|
let billedModelRefs: string[] = [finalModelId];
|
||||||
videoParams: effectiveVideoParams,
|
let videoMetadata: Record<string, any> = {
|
||||||
});
|
|
||||||
|
|
||||||
const videoMarkdown = result.text || "视频生成失败";
|
|
||||||
console.log(`[Chat API] ${currentVideoProvider} video generated, length:`, videoMarkdown.length);
|
|
||||||
|
|
||||||
// 提取视频 URL(后续会自动转存到永久存储)
|
|
||||||
const tempVideoUrl = extractVideoUrlFromText(videoMarkdown) || "";
|
|
||||||
|
|
||||||
// 生成一个统一的消息 ID
|
|
||||||
const messageId = generateUUID();
|
|
||||||
|
|
||||||
const videoMetadata = {
|
|
||||||
provider: currentVideoProvider,
|
provider: currentVideoProvider,
|
||||||
model: finalModelId,
|
model: finalModelId,
|
||||||
mode: effectiveVideoParams?.mode,
|
mode: effectiveVideoParams?.mode,
|
||||||
@ -4150,6 +4184,86 @@ export async function POST(request: Request) {
|
|||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let usedStoryboardPipeline = false;
|
||||||
|
if (currentVideoProvider === "ruxa-video") {
|
||||||
|
const storyboardConfig = await getVideoStoryboardConfig().catch(() => null);
|
||||||
|
if (storyboardConfig?.enabled) {
|
||||||
|
const storyboardModelCandidates = await resolveStoryboardModelCandidates(finalModelId);
|
||||||
|
const runtimeModelId = String((languageModel as any).modelId || "").trim();
|
||||||
|
const selectedModelId =
|
||||||
|
runtimeModelId ||
|
||||||
|
storyboardModelCandidates.find((item) => item.id === finalModelId)?.modelId ||
|
||||||
|
"";
|
||||||
|
|
||||||
|
const storyboardResult = await runRuxaNineGridPipeline({
|
||||||
|
modelMessages: modelMessages as any,
|
||||||
|
selectedModelId,
|
||||||
|
selectedModelRef: finalModelId,
|
||||||
|
modelCandidates: storyboardModelCandidates,
|
||||||
|
videoParams: effectiveVideoParams as any,
|
||||||
|
config: storyboardConfig,
|
||||||
|
doGenerate: (input) =>
|
||||||
|
(languageModel as any).doGenerate({
|
||||||
|
messages: input.messages,
|
||||||
|
videoParams: input.videoParams,
|
||||||
|
overrideModelId: input.overrideModelId,
|
||||||
|
}),
|
||||||
|
onProgress: (event) => {
|
||||||
|
safeWriteStreamChunk(
|
||||||
|
writer,
|
||||||
|
{
|
||||||
|
type: "data-videoStoryboard",
|
||||||
|
data: JSON.stringify(event),
|
||||||
|
},
|
||||||
|
`ruxa-video:storyboard:${event.stage}`,
|
||||||
|
);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (storyboardResult.triggered) {
|
||||||
|
usedStoryboardPipeline = true;
|
||||||
|
videoMarkdown = storyboardResult.markdown || "视频生成失败,请稍后重试";
|
||||||
|
tempVideoUrl = storyboardResult.videoUrls?.[0] || "";
|
||||||
|
billedModelRefs =
|
||||||
|
storyboardResult.modelRefs && storyboardResult.modelRefs.length > 0
|
||||||
|
? storyboardResult.modelRefs
|
||||||
|
: [finalModelId];
|
||||||
|
|
||||||
|
const successCount = storyboardResult.videoUrls?.length || 0;
|
||||||
|
videoMetadata = {
|
||||||
|
...videoMetadata,
|
||||||
|
...(storyboardResult.metadata || {}),
|
||||||
|
status: successCount > 0 ? "succeeded" : "failed",
|
||||||
|
storyboard: {
|
||||||
|
enabled: true,
|
||||||
|
taskId: storyboardResult.taskId,
|
||||||
|
confidence: storyboardResult.confidence,
|
||||||
|
successCount,
|
||||||
|
},
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!usedStoryboardPipeline) {
|
||||||
|
const result = await (languageModel as any).doGenerate({
|
||||||
|
messages: modelMessages,
|
||||||
|
videoParams: effectiveVideoParams,
|
||||||
|
});
|
||||||
|
|
||||||
|
videoMarkdown = result.text || "视频生成失败";
|
||||||
|
tempVideoUrl = extractVideoUrlFromText(videoMarkdown) || "";
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[Chat API] ${currentVideoProvider} video generated, length:`,
|
||||||
|
videoMarkdown.length
|
||||||
|
);
|
||||||
|
|
||||||
|
// 生成一个统一的消息 ID
|
||||||
|
const messageId = generateUUID();
|
||||||
|
|
||||||
// 写入消息到流
|
// 写入消息到流
|
||||||
writer.write({
|
writer.write({
|
||||||
type: "data-appendMessage",
|
type: "data-appendMessage",
|
||||||
@ -4185,7 +4299,7 @@ export async function POST(request: Request) {
|
|||||||
userId: session.user.id,
|
userId: session.user.id,
|
||||||
chatId: id,
|
chatId: id,
|
||||||
messageId,
|
messageId,
|
||||||
modelRefs: [finalModelId],
|
modelRefs: billedModelRefs,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (tempVideoUrl) {
|
if (tempVideoUrl) {
|
||||||
|
|||||||
@ -88,6 +88,65 @@ interface CanvasChatPanelProps {
|
|||||||
onOpenFileList?: () => void;
|
onOpenFileList?: () => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type VideoStoryboardStreamData =
|
||||||
|
| {
|
||||||
|
stage: "detecting";
|
||||||
|
provider?: string;
|
||||||
|
message?: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
stage: "start";
|
||||||
|
provider?: string;
|
||||||
|
taskId: string;
|
||||||
|
totalShots: number;
|
||||||
|
confidence?: number;
|
||||||
|
sourceImageUrl?: string;
|
||||||
|
shots?: Array<{
|
||||||
|
shotIndex: number;
|
||||||
|
tileImageUrl?: string;
|
||||||
|
prompt?: string;
|
||||||
|
}>;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
stage: "shot_completed";
|
||||||
|
provider?: string;
|
||||||
|
taskId: string;
|
||||||
|
shotIndex: number;
|
||||||
|
modelId?: string;
|
||||||
|
videoUrl: string;
|
||||||
|
text?: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
stage: "shot_failed";
|
||||||
|
provider?: string;
|
||||||
|
taskId: string;
|
||||||
|
shotIndex: number;
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
stage: "completed";
|
||||||
|
provider?: string;
|
||||||
|
taskId: string;
|
||||||
|
totalShots?: number;
|
||||||
|
successCount?: number;
|
||||||
|
failedCount?: number;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
stage: "failed";
|
||||||
|
provider?: string;
|
||||||
|
taskId: string;
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
stage: "shot_retry";
|
||||||
|
provider?: string;
|
||||||
|
taskId: string;
|
||||||
|
shotIndex: number;
|
||||||
|
attempt?: number;
|
||||||
|
modelId?: string;
|
||||||
|
error?: string;
|
||||||
|
};
|
||||||
|
|
||||||
// 建议卡片数据 - 已移除
|
// 建议卡片数据 - 已移除
|
||||||
const SUGGESTIONS: { title: string; desc: string; image: string }[] = [];
|
const SUGGESTIONS: { title: string; desc: string; image: string }[] = [];
|
||||||
const CANVAS_COMPOSER_PREFIX_DEFAULT = "让Buyi-ai";
|
const CANVAS_COMPOSER_PREFIX_DEFAULT = "让Buyi-ai";
|
||||||
@ -1617,6 +1676,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
|||||||
signature: "",
|
signature: "",
|
||||||
timestamp: 0,
|
timestamp: 0,
|
||||||
});
|
});
|
||||||
|
const storyboardNodeMapRef = useRef<Record<string, Record<number, string>>>({});
|
||||||
const releaseSendGuard = useCallback(() => {
|
const releaseSendGuard = useCallback(() => {
|
||||||
sendGuardRef.current.inFlight = false;
|
sendGuardRef.current.inFlight = false;
|
||||||
}, []);
|
}, []);
|
||||||
@ -3190,6 +3250,128 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
|||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error("[CanvasChat] Failed to parse data-appendMessage:", e);
|
console.error("[CanvasChat] Failed to parse data-appendMessage:", e);
|
||||||
}
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dataPart.type === "data-videoStoryboard") {
|
||||||
|
try {
|
||||||
|
const payload: VideoStoryboardStreamData =
|
||||||
|
typeof dataPart.data === "string"
|
||||||
|
? JSON.parse(dataPart.data)
|
||||||
|
: dataPart.data;
|
||||||
|
if (!payload || typeof payload !== "object") return;
|
||||||
|
|
||||||
|
if (payload.stage === "start") {
|
||||||
|
const taskId = String(payload.taskId || "").trim();
|
||||||
|
if (!taskId) return;
|
||||||
|
if (storyboardNodeMapRef.current[taskId]) return;
|
||||||
|
|
||||||
|
const totalShots = Math.max(1, Math.min(9, Number(payload.totalShots || 9)));
|
||||||
|
const shotNodeMap: Record<number, string> = {};
|
||||||
|
const currentNodes = useCanvasStore.getState().nodes;
|
||||||
|
const baseOffset = currentNodes.length;
|
||||||
|
const cardWidth = 320;
|
||||||
|
const cardHeight = 196;
|
||||||
|
const gapX = 30;
|
||||||
|
const gapY = 24;
|
||||||
|
const startX = 120 + (baseOffset % 2) * 420;
|
||||||
|
const startY = 120 + Math.floor(baseOffset / 2) * 260;
|
||||||
|
|
||||||
|
for (let shotIndex = 1; shotIndex <= totalShots; shotIndex += 1) {
|
||||||
|
const shotData = payload.shots?.find((item) => Number(item.shotIndex) === shotIndex);
|
||||||
|
const col = (shotIndex - 1) % 3;
|
||||||
|
const row = Math.floor((shotIndex - 1) / 3);
|
||||||
|
const nodeId = addNode({
|
||||||
|
type: "video",
|
||||||
|
position: {
|
||||||
|
x: startX + col * (cardWidth + gapX),
|
||||||
|
y: startY + row * (cardHeight + gapY),
|
||||||
|
},
|
||||||
|
size: { width: cardWidth, height: cardHeight },
|
||||||
|
data: {
|
||||||
|
src: "",
|
||||||
|
status: "generating",
|
||||||
|
prompt: shotData?.prompt || `分镜 ${shotIndex} 生成中`,
|
||||||
|
coverImage: shotData?.tileImageUrl || "",
|
||||||
|
},
|
||||||
|
connections: [],
|
||||||
|
});
|
||||||
|
shotNodeMap[shotIndex] = nodeId;
|
||||||
|
}
|
||||||
|
|
||||||
|
storyboardNodeMapRef.current[taskId] = shotNodeMap;
|
||||||
|
toast.message(`九宫格视频任务已开始(${totalShots} 段)`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (payload.stage === "shot_completed") {
|
||||||
|
const taskId = String(payload.taskId || "").trim();
|
||||||
|
const shotIndex = Math.max(1, Number(payload.shotIndex || 0));
|
||||||
|
const videoUrl = String(payload.videoUrl || "").trim();
|
||||||
|
if (!taskId || !shotIndex || !videoUrl) return;
|
||||||
|
|
||||||
|
let nodeId = storyboardNodeMapRef.current[taskId]?.[shotIndex];
|
||||||
|
if (!nodeId) {
|
||||||
|
nodeId = addNode({
|
||||||
|
type: "video",
|
||||||
|
position: { x: 140 + ((shotIndex - 1) % 3) * 350, y: 150 + Math.floor((shotIndex - 1) / 3) * 230 },
|
||||||
|
size: { width: 320, height: 196 },
|
||||||
|
data: {
|
||||||
|
src: videoUrl,
|
||||||
|
status: "completed",
|
||||||
|
prompt: `分镜 ${shotIndex}`,
|
||||||
|
},
|
||||||
|
connections: [],
|
||||||
|
});
|
||||||
|
if (!storyboardNodeMapRef.current[taskId]) {
|
||||||
|
storyboardNodeMapRef.current[taskId] = {};
|
||||||
|
}
|
||||||
|
storyboardNodeMapRef.current[taskId]![shotIndex] = nodeId;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const currentNode = useCanvasStore.getState().getNode(nodeId);
|
||||||
|
const currentData = ((currentNode?.data as any) || {}) as Record<string, any>;
|
||||||
|
updateNode(nodeId, {
|
||||||
|
data: {
|
||||||
|
...currentData,
|
||||||
|
src: videoUrl,
|
||||||
|
status: "completed",
|
||||||
|
},
|
||||||
|
} as any);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (payload.stage === "shot_failed") {
|
||||||
|
const taskId = String(payload.taskId || "").trim();
|
||||||
|
const shotIndex = Math.max(1, Number(payload.shotIndex || 0));
|
||||||
|
if (!taskId || !shotIndex) return;
|
||||||
|
const nodeId = storyboardNodeMapRef.current[taskId]?.[shotIndex];
|
||||||
|
if (!nodeId) return;
|
||||||
|
const currentNode = useCanvasStore.getState().getNode(nodeId);
|
||||||
|
const currentData = ((currentNode?.data as any) || {}) as Record<string, any>;
|
||||||
|
updateNode(nodeId, {
|
||||||
|
data: {
|
||||||
|
...currentData,
|
||||||
|
status: "failed",
|
||||||
|
},
|
||||||
|
} as any);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (payload.stage === "completed") {
|
||||||
|
const successCount = Math.max(0, Number(payload.successCount || 0));
|
||||||
|
const failedCount = Math.max(0, Number(payload.failedCount || 0));
|
||||||
|
toast.success(`九宫格视频完成:成功 ${successCount},失败 ${failedCount}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (payload.stage === "failed") {
|
||||||
|
toast.error(payload.error || "九宫格视频任务失败");
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.error("[CanvasChat] Failed to parse data-videoStoryboard:", e);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
onError: (error) => {
|
onError: (error) => {
|
||||||
|
|||||||
599
docs/系统全链路开发文档-含九宫格视频Agent技能化方案.md
Normal file
599
docs/系统全链路开发文档-含九宫格视频Agent技能化方案.md
Normal file
@ -0,0 +1,599 @@
|
|||||||
|
# 系统全链路开发文档(基于当前项目)
|
||||||
|
|
||||||
|
> 项目:`ai-chatbot-main`
|
||||||
|
> 文档目标:沉淀当前系统已实现能力,并给出“RUXA 视频九宫格 + Agent 技能化”完整可开发方案
|
||||||
|
> 适用范围:聊天页、无限画布、3D 生成、AI 详情、视频生成、存储迁移、后台模型配置
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. 项目定位与能力边界
|
||||||
|
|
||||||
|
本系统是一个多模态 AI 创作平台,核心能力覆盖:
|
||||||
|
|
||||||
|
1. 文本/图片/视频对话式生成(统一在聊天链路)
|
||||||
|
2. 无限画布创作与多节点编排(图片、视频、文本、3D)
|
||||||
|
3. Agent 智能路由与智能参数推荐
|
||||||
|
4. AI 详情生成(电商详情页自动分镜 + 质检回修)
|
||||||
|
5. 多供应商模型治理(后台可配置 provider/model)
|
||||||
|
6. 多存储后端(S3/MinIO/OSS)与 OSS 降本迁移任务
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. 技术架构总览
|
||||||
|
|
||||||
|
## 2.1 技术栈
|
||||||
|
|
||||||
|
- 前端:Next.js 16 + React 19 + Tailwind + shadcn/ui
|
||||||
|
- 后端:Next.js Route Handlers(Node Runtime)
|
||||||
|
- AI 编排:`ai` SDK + LangGraph(Agent 流程)
|
||||||
|
- 数据库:PostgreSQL + Drizzle ORM
|
||||||
|
- 存储:S3 兼容存储 / Aliyun OSS / Vercel Blob(兼容)
|
||||||
|
- 图像处理:`sharp`
|
||||||
|
- 状态管理:Zustand(画布)
|
||||||
|
|
||||||
|
## 2.2 关键目录
|
||||||
|
|
||||||
|
- 聊天主链路:`/Users/lanmu/Desktop/ai-chatbot-main/app/(chat)/api/chat/route.ts`
|
||||||
|
- 模型动态加载:`/Users/lanmu/Desktop/ai-chatbot-main/lib/ai/dynamic-provider.ts`
|
||||||
|
- Agent 编排:`/Users/lanmu/Desktop/ai-chatbot-main/lib/ai/auto-agent-graph.ts`
|
||||||
|
- 智能分析:`/Users/lanmu/Desktop/ai-chatbot-main/lib/ai/smart-analyzer.ts`
|
||||||
|
- 无限画布页面:`/Users/lanmu/Desktop/ai-chatbot-main/app/canvas/[id]/page.tsx`
|
||||||
|
- 画布聊天面板:`/Users/lanmu/Desktop/ai-chatbot-main/components/canvas/CanvasChatPanel.tsx`
|
||||||
|
- 3D 页面:`/Users/lanmu/Desktop/ai-chatbot-main/app/canvas/[id]/model3d/page.tsx`
|
||||||
|
- 3D API:`/Users/lanmu/Desktop/ai-chatbot-main/app/api/canvas/3d/tasks/route.ts`
|
||||||
|
- 后台设置:`/Users/lanmu/Desktop/ai-chatbot-main/app/(admin)/settings/admin-settings-client.tsx`
|
||||||
|
- 数据模型:`/Users/lanmu/Desktop/ai-chatbot-main/lib/db/schema.ts`
|
||||||
|
- 存储迁移任务:`/Users/lanmu/Desktop/ai-chatbot-main/lib/jobs/oss-image-migration-job.ts`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. 当前模型与供应商体系(已实现)
|
||||||
|
|
||||||
|
## 3.1 Provider 类型(后台可配置)
|
||||||
|
|
||||||
|
已接入的 provider 类型(数据库 + 后台一致):
|
||||||
|
|
||||||
|
- `alibaba`、`openai`、`custom`、`local`
|
||||||
|
- `gemini`、`huanke-gemini`
|
||||||
|
- `alibaba-image`、`ruxa-image`、`fal-image`、`volcano-vision`
|
||||||
|
- `qianwen-video`、`doubao-video`、`fal-video`、`ruxa-video`、`volcano-video`
|
||||||
|
- `tripo-3d`
|
||||||
|
- 扩展兼容:`zhipu`、`volcano-multimodal`(由 `custom` 映射识别)
|
||||||
|
|
||||||
|
参考实现:
|
||||||
|
|
||||||
|
- `/Users/lanmu/Desktop/ai-chatbot-main/app/api/admin/ai-providers/route.ts`
|
||||||
|
- `/Users/lanmu/Desktop/ai-chatbot-main/lib/ai/provider-defaults.ts`
|
||||||
|
|
||||||
|
## 3.2 视频模型与模式能力(代码内置)
|
||||||
|
|
||||||
|
视频能力模型通过 `capabilities + videoToolCapability` 驱动前端 UI 与参数限制:
|
||||||
|
|
||||||
|
- RUXA:仅 `first-frame`(图生视频),比例 `16:9/9:16/1:1`,时长 `5/10`
|
||||||
|
- Qianwen:`first-frame`
|
||||||
|
- FAL:`first-frame/first-last-frame`
|
||||||
|
- Volcano/Doubao:支持文生/首帧/首尾帧/多参考图(按模型能力动态裁剪)
|
||||||
|
|
||||||
|
参考实现:
|
||||||
|
|
||||||
|
- `/Users/lanmu/Desktop/ai-chatbot-main/lib/ai/video-tool-capability.ts`
|
||||||
|
- `/Users/lanmu/Desktop/ai-chatbot-main/components/video-params-bar.tsx`
|
||||||
|
|
||||||
|
## 3.3 代码中已出现的具体模型名称(非运行态 DB 全量)
|
||||||
|
|
||||||
|
- RUXA 视频:`veo3.1`(后台模型可配置)
|
||||||
|
- 实时画布默认模型:`fal-ai/flux-2/klein/realtime`
|
||||||
|
- 火山预设模型族:
|
||||||
|
- `doubao-seedance-1-0-pro-250528`
|
||||||
|
- `doubao-seedance-1-0-pro-fast-251015`
|
||||||
|
- `doubao-seedance-1-0-lite-t2v-250428`
|
||||||
|
- `doubao-seedance-1-0-lite-i2v-250428`
|
||||||
|
- Tripo3D 版本选项:
|
||||||
|
- `Turbo-v1.0-20250506`
|
||||||
|
- `v3.1-20260211`
|
||||||
|
- `v3.0-20250812`
|
||||||
|
- `v2.5-20250123`
|
||||||
|
- `v2.0-20240919`
|
||||||
|
- `v1.4-20240625`
|
||||||
|
|
||||||
|
说明:系统最终“在用模型清单”来自数据库 `AiModel`,由后台“AI模型/本地AI模型”动态配置。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. 核心链路说明(已实现)
|
||||||
|
|
||||||
|
## 4.1 聊天与 Agent 模式
|
||||||
|
|
||||||
|
入口:`POST /api/chat`
|
||||||
|
实现:`/Users/lanmu/Desktop/ai-chatbot-main/app/(chat)/api/chat/route.ts`
|
||||||
|
|
||||||
|
主流程:
|
||||||
|
|
||||||
|
1. 鉴权 + 请求去重(同 `chatId:messageId` 防重入)
|
||||||
|
2. 自动模式判定(`selectedChatModel === "auto"` 或固定视觉模型 + deepThinking)
|
||||||
|
3. 历史图片补全(当前无图时回溯最近生成图/上传图)
|
||||||
|
4. 调用 LangGraph Agent:
|
||||||
|
- 意图识别(图编/图生/视频/问答)
|
||||||
|
- 友好回复
|
||||||
|
- 提示词优化
|
||||||
|
- 推荐图片/视频参数
|
||||||
|
- 路由到最佳模型
|
||||||
|
5. 按 provider 分支执行(文本、图片、视频)
|
||||||
|
6. 流式返回 + 消息持久化 + 积分/配额扣减
|
||||||
|
|
||||||
|
## 4.2 无限画布
|
||||||
|
|
||||||
|
页面:`/canvas/[id]`
|
||||||
|
实现:
|
||||||
|
|
||||||
|
- `/Users/lanmu/Desktop/ai-chatbot-main/app/canvas/[id]/page.tsx`
|
||||||
|
- `/Users/lanmu/Desktop/ai-chatbot-main/components/canvas/CanvasWorkspace.tsx`
|
||||||
|
- `/Users/lanmu/Desktop/ai-chatbot-main/store/canvas/index.ts`
|
||||||
|
|
||||||
|
能力:
|
||||||
|
|
||||||
|
- 节点类型:`message/image/video/text/scene-fusion/model3d`
|
||||||
|
- 自动保存 + 节流重试(429 后重试)
|
||||||
|
- 节点操作:拖拽、缩放、撤销/重做、多选
|
||||||
|
- 画布与聊天联动(图片可回注对话)
|
||||||
|
|
||||||
|
## 4.3 AI 详情生成
|
||||||
|
|
||||||
|
核心接口:
|
||||||
|
|
||||||
|
- `POST /api/canvas/detail-plan`
|
||||||
|
- `POST /api/canvas/detail-review`
|
||||||
|
- `GET|POST|PATCH|DELETE /api/canvas/[id]/detail-archives`
|
||||||
|
|
||||||
|
核心能力:
|
||||||
|
|
||||||
|
1. 电商详情 brief 结构化输入
|
||||||
|
2. 分屏规划(传统生图 / `agent_layout` 可编辑文字层)
|
||||||
|
3. 逐屏生成 + 质检 + 二次修图
|
||||||
|
4. 存档回放(`canvasDetailArchive`)
|
||||||
|
|
||||||
|
主要实现:
|
||||||
|
|
||||||
|
- `/Users/lanmu/Desktop/ai-chatbot-main/components/canvas/CanvasChatPanel.tsx`
|
||||||
|
- `/Users/lanmu/Desktop/ai-chatbot-main/app/api/canvas/detail-plan/route.ts`
|
||||||
|
- `/Users/lanmu/Desktop/ai-chatbot-main/app/api/canvas/detail-review/route.ts`
|
||||||
|
|
||||||
|
## 4.4 3D 生成(Tripo3D)
|
||||||
|
|
||||||
|
页面:`/canvas/[id]/model3d`
|
||||||
|
API:
|
||||||
|
|
||||||
|
- `POST /api/canvas/3d/upload-image`
|
||||||
|
- `POST /api/canvas/3d/tasks`
|
||||||
|
- `GET /api/canvas/3d/tasks/[taskId]`
|
||||||
|
- `GET /api/canvas/3d/balance`
|
||||||
|
|
||||||
|
后端关键点:
|
||||||
|
|
||||||
|
- 从 `AiProvider(type="tripo-3d")` 读取 key/baseUrl
|
||||||
|
- 对 API key 做隐藏字符清洗(解决 ByteString 8232 问题)
|
||||||
|
- 统一 Tripo 错误结构返回(含 traceId)
|
||||||
|
|
||||||
|
主要实现:
|
||||||
|
|
||||||
|
- `/Users/lanmu/Desktop/ai-chatbot-main/lib/tripo/server.ts`
|
||||||
|
- `/Users/lanmu/Desktop/ai-chatbot-main/lib/tripo/client.ts`
|
||||||
|
|
||||||
|
## 4.5 存储与 OSS 迁移降本
|
||||||
|
|
||||||
|
现状策略(已上线):
|
||||||
|
|
||||||
|
1. 上传优先走可用存储配置(支持 OSS 作为入口桶)
|
||||||
|
2. 记录迁移任务(`StorageMigrationTask`)
|
||||||
|
3. 定时任务将 OSS 图片转存到默认桶
|
||||||
|
4. 回写数据库 URL(消息、画布、详情存档)
|
||||||
|
5. 到期清理 OSS 源对象
|
||||||
|
|
||||||
|
接口:
|
||||||
|
|
||||||
|
- `POST /api/jobs/migrate-oss-images`
|
||||||
|
- `GET /api/jobs/migrate-oss-images`
|
||||||
|
|
||||||
|
主要实现:
|
||||||
|
|
||||||
|
- `/Users/lanmu/Desktop/ai-chatbot-main/lib/storage/service.ts`
|
||||||
|
- `/Users/lanmu/Desktop/ai-chatbot-main/lib/jobs/oss-image-migration-job.ts`
|
||||||
|
- `/Users/lanmu/Desktop/ai-chatbot-main/app/api/jobs/migrate-oss-images/route.ts`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. 数据库核心表(当前)
|
||||||
|
|
||||||
|
主要业务表:
|
||||||
|
|
||||||
|
- `AiProvider`:供应商配置
|
||||||
|
- `AiModel`:模型配置、能力、故障迁移字段
|
||||||
|
- `StorageConfig`:存储桶配置
|
||||||
|
- `StorageMigrationTask`:OSS 迁移任务
|
||||||
|
- `SiteSettings` + `SystemSettings`:站点与功能开关
|
||||||
|
- `Message`:消息主表(含视频存储元数据)
|
||||||
|
- `GeneratedImage`:图片记录
|
||||||
|
- `CanvasProject`、`CanvasNode`、`CanvasImageTask`
|
||||||
|
- `CanvasDetailArchive`:AI详情归档
|
||||||
|
|
||||||
|
定义文件:`/Users/lanmu/Desktop/ai-chatbot-main/lib/db/schema.ts`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. 新需求:RUXA 视频九宫格 + Agent 技能化(待开发)
|
||||||
|
|
||||||
|
## 6.1 需求归纳
|
||||||
|
|
||||||
|
当用户在视频模式选中 `RUXA 视频` 时:
|
||||||
|
|
||||||
|
1. Agent 判断用户上传图是否为九宫格
|
||||||
|
2. 若是九宫格,切成 9 张图
|
||||||
|
3. 用户脚本格式不固定,先由 Agent 归一为 9 镜头脚本
|
||||||
|
4. 切图后尺寸偏小,后台预留“图片变清晰”调用链路(默认关闭,待测试后再开启)
|
||||||
|
5. 每张图按对应脚本生成视频,串行发送,间隔 10 秒
|
||||||
|
6. 失败自动重试;超限/失败可降级到其它视频模型
|
||||||
|
7. 全流程可追踪进度,可回放任务结果
|
||||||
|
|
||||||
|
## 6.2 合理性评估结论
|
||||||
|
|
||||||
|
该需求合理,且与现有架构兼容;但需工程化约束:
|
||||||
|
|
||||||
|
- 超分能力先预留不启用,通过配置开关控制,默认 `false`
|
||||||
|
- 九宫格检测需置信度门槛,低置信度不自动切片
|
||||||
|
- 串行队列必须持久化到 DB,避免重启丢任务
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. 九宫格技能链设计(建议实现)
|
||||||
|
|
||||||
|
采用“技能链(Skill Chain)”封装,而不是把全部逻辑塞进 `chat/route.ts`。
|
||||||
|
|
||||||
|
> 实施约束:当前阶段仅输出方案文档,不改动业务代码;开发时必须按“技能封装”方式落地。
|
||||||
|
|
||||||
|
## 7.1 技能模块划分
|
||||||
|
|
||||||
|
1. `script-normalizer-skill`
|
||||||
|
输入任意脚本文本,输出标准 JSON(9 段)
|
||||||
|
|
||||||
|
2. `nine-grid-detector-skill`
|
||||||
|
判断是否九宫格 + 透视校正 + 切片 + 质量评分
|
||||||
|
|
||||||
|
3. `tile-upscale-skill`
|
||||||
|
对小图按阈值走增强(复用现有 enhance 能力,先预留默认关闭)
|
||||||
|
|
||||||
|
4. `storyboard-mapper-skill`
|
||||||
|
将 9 图与 9 段脚本一一映射,生成每段视频 prompt
|
||||||
|
|
||||||
|
5. `video-sequencer-skill`
|
||||||
|
串行调度(10s)+ 重试 + 降级模型 + 状态更新
|
||||||
|
|
||||||
|
## 7.2 技能触发规则(必须)
|
||||||
|
|
||||||
|
仅在满足以下条件时触发九宫格技能链:
|
||||||
|
|
||||||
|
1. 当前模式为视频生成,且选中模型 provider 为 `ruxa-video`
|
||||||
|
2. 用户消息包含至少 1 张图片
|
||||||
|
3. `nine-grid-detector-skill` 识别结果 `isNineGrid=true`
|
||||||
|
4. `confidence >= minGridConfidence`(默认 0.85)
|
||||||
|
|
||||||
|
触发后的流程:
|
||||||
|
|
||||||
|
1. 进入 `script-normalizer-skill`
|
||||||
|
2. 再进入 `nine-grid-detector-skill` 切片
|
||||||
|
3. 按开关决定是否进入 `tile-upscale-skill`
|
||||||
|
4. 进入 `storyboard-mapper-skill`
|
||||||
|
5. 最后进入 `video-sequencer-skill`
|
||||||
|
|
||||||
|
未触发时(任一条件不满足):
|
||||||
|
|
||||||
|
- 直接走现有普通视频链路,不进入九宫格技能链
|
||||||
|
|
||||||
|
## 7.3 建议代码落点
|
||||||
|
|
||||||
|
- `lib/video-agent/skills/script-normalizer.ts`
|
||||||
|
- `lib/video-agent/skills/nine-grid-detector.ts`
|
||||||
|
- `lib/video-agent/skills/tile-upscaler.ts`
|
||||||
|
- `lib/video-agent/skills/storyboard-mapper.ts`
|
||||||
|
- `lib/video-agent/skills/video-sequencer.ts`
|
||||||
|
- `lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts`
|
||||||
|
|
||||||
|
聊天主链路只做路由:
|
||||||
|
|
||||||
|
- 修改 `/Users/lanmu/Desktop/ai-chatbot-main/app/(chat)/api/chat/route.ts`
|
||||||
|
- 当 provider=`ruxa-video` 且满足“技能触发规则”时调用 pipeline
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. 防切错设计(重点)
|
||||||
|
|
||||||
|
## 8.1 检测流程(四段式)
|
||||||
|
|
||||||
|
1. **预检**:宽高比接近 1:1 或 3:3 拼图常见比例;分辨率 > 最低阈值
|
||||||
|
2. **网格线检测**:行列投影 + 边缘峰值,识别 2 条纵线 + 2 条横线
|
||||||
|
3. **几何校正**:若轻微透视,先矫正再切片
|
||||||
|
4. **切后复检**:每格清晰度、边界截断、相邻差异、主体完整度评分
|
||||||
|
|
||||||
|
## 8.2 判定阈值建议
|
||||||
|
|
||||||
|
- `grid_confidence >= 0.85`:自动进入九宫格流程
|
||||||
|
- `0.65 ~ 0.85`:进入“待确认”状态(Agent 提示用户确认)
|
||||||
|
- `< 0.65`:走普通单图视频流程
|
||||||
|
|
||||||
|
## 8.3 输出结构建议
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"isNineGrid": true,
|
||||||
|
"confidence": 0.92,
|
||||||
|
"cropBoxes": [
|
||||||
|
{"index": 1, "x": 0, "y": 0, "w": 342, "h": 342}
|
||||||
|
],
|
||||||
|
"qualityScore": 0.88
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. 脚本归一化与分镜映射
|
||||||
|
|
||||||
|
## 9.1 脚本标准 JSON
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"language": "zh",
|
||||||
|
"globalStyle": "写实电影感",
|
||||||
|
"shots": [
|
||||||
|
{
|
||||||
|
"index": 1,
|
||||||
|
"durationSec": 3,
|
||||||
|
"action": "角色抬头看向镜头",
|
||||||
|
"camera": "轻微推进",
|
||||||
|
"mood": "期待",
|
||||||
|
"negativePrompt": "抖动,畸变,低清"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## 9.2 映射策略
|
||||||
|
|
||||||
|
- 默认 `shot[i] -> tile[i]`
|
||||||
|
- 若脚本中有显式编号(如“第3镜头”)优先按编号映射
|
||||||
|
- 脚本不足 9 段:补全占位镜头(继承全局风格)
|
||||||
|
- 脚本超过 9 段:截断并告知用户
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. 超分链路设计(复用现有能力)
|
||||||
|
|
||||||
|
当前已有增强接口:`POST /api/enhance`
|
||||||
|
|
||||||
|
建议:
|
||||||
|
|
||||||
|
1. 抽出服务层(如 `lib/image/enhance-service.ts`),避免服务端内部再走 HTTP
|
||||||
|
2. 增加特性开关 `enableTileUpscale`,默认 `false`(预留不启用)
|
||||||
|
3. 仅当 `enableTileUpscale=true` 时,切片后按阈值触发超分(短边 < 768)
|
||||||
|
4. 超分失败降级:保留原切片继续后续流程
|
||||||
|
5. 记录 `enhanceApplied/enhanceProvider/enhanceLatencyMs`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. 调度、重试、降级
|
||||||
|
|
||||||
|
## 11.1 串行调度规则
|
||||||
|
|
||||||
|
- 一次任务 9 段,严格串行
|
||||||
|
- 相邻请求间隔 10 秒
|
||||||
|
- 任一段失败:指数退避(10s/20s/40s),最多 3 次
|
||||||
|
|
||||||
|
## 11.2 降级模型规则
|
||||||
|
|
||||||
|
从后台 `AiModel` 读取,不硬编码:
|
||||||
|
|
||||||
|
1. 先用当前选中模型
|
||||||
|
2. 失败后按 `fallbackModelId1 -> fallbackModelId2`
|
||||||
|
3. 若仍失败,按同 provider 可用模型池挑选
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12. 数据库扩展设计(建议新增)
|
||||||
|
|
||||||
|
新增任务主表:`VideoStoryboardTask`
|
||||||
|
|
||||||
|
- `id`
|
||||||
|
- `userId`
|
||||||
|
- `chatId`
|
||||||
|
- `providerType`(默认 `ruxa-video`)
|
||||||
|
- `sourceImageUrl`
|
||||||
|
- `isNineGrid`
|
||||||
|
- `detectionConfidence`
|
||||||
|
- `normalizedScript` (jsonb)
|
||||||
|
- `status`(pending/running/partial_success/succeeded/failed/cancelled)
|
||||||
|
- `currentShot`
|
||||||
|
- `totalShots`(固定 9)
|
||||||
|
- `createdAt/updatedAt/completedAt`
|
||||||
|
|
||||||
|
新增子任务表:`VideoStoryboardShotTask`
|
||||||
|
|
||||||
|
- `id`
|
||||||
|
- `parentTaskId`
|
||||||
|
- `shotIndex`
|
||||||
|
- `tileImageUrl`
|
||||||
|
- `upscaledImageUrl`
|
||||||
|
- `prompt`
|
||||||
|
- `modelId`
|
||||||
|
- `fallbackChain` (jsonb)
|
||||||
|
- `attemptCount`
|
||||||
|
- `status`
|
||||||
|
- `videoUrl`
|
||||||
|
- `errorCode/errorMessage`
|
||||||
|
- `startedAt/completedAt`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 13. API 设计(建议新增)
|
||||||
|
|
||||||
|
1. `POST /api/video/storyboard/create`
|
||||||
|
创建九宫格视频任务,返回 `taskId`
|
||||||
|
|
||||||
|
2. `GET /api/video/storyboard/tasks/:taskId`
|
||||||
|
获取任务与每段进度
|
||||||
|
|
||||||
|
3. `POST /api/video/storyboard/tasks/:taskId/cancel`
|
||||||
|
取消任务
|
||||||
|
|
||||||
|
4. `POST /api/jobs/video-storyboard-dispatch`
|
||||||
|
调度器轮询(可由 cron/pm2 定时触发)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 14. 与现有聊天链路集成方式
|
||||||
|
|
||||||
|
在 `/api/chat` 增加分支:
|
||||||
|
|
||||||
|
1. 当前 provider 是否 `ruxa-video`
|
||||||
|
2. 用户是否上传图片
|
||||||
|
3. 是否触发九宫格识别(可由配置开关控制)
|
||||||
|
|
||||||
|
若命中:
|
||||||
|
|
||||||
|
- 先回 SSE 占位消息(“正在识别九宫格与解析分镜...”)
|
||||||
|
- 创建 storyboard 任务
|
||||||
|
- 前端按 taskId 轮询并展示 9 段进度
|
||||||
|
- 最终输出 9 条视频或合并汇总消息
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 15. 前端改造点(建议)
|
||||||
|
|
||||||
|
- `components/video-params-bar.tsx`
|
||||||
|
- 新增“九宫格自动分镜”开关(RUXA 模式显示)
|
||||||
|
- `components/messages.tsx`
|
||||||
|
- 支持展示分段进度条(1/9 ~ 9/9)
|
||||||
|
- `components/chat.tsx` / `components/canvas/CanvasChatPanel.tsx`
|
||||||
|
- 扩展任务状态卡片(处理中、重试中、降级中)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 16. 配置项设计(系统设置)
|
||||||
|
|
||||||
|
在系统设置新增 `video_storyboard_config`:
|
||||||
|
|
||||||
|
- `enabled`:是否启用九宫格流程
|
||||||
|
- `providerWhitelist`:默认仅 `ruxa-video`
|
||||||
|
- `minGridConfidence`:默认 0.85
|
||||||
|
- `enableTileUpscale`:是否启用切片超分(默认 false,预留)
|
||||||
|
- `sliceUpscaleMinEdge`:默认 768
|
||||||
|
- `dispatchIntervalSec`:默认 10
|
||||||
|
- `maxRetry`:默认 3
|
||||||
|
- `enableFallback`:默认 true
|
||||||
|
|
||||||
|
存储位置:`SystemSettings`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 17. 监控与告警
|
||||||
|
|
||||||
|
建议埋点:
|
||||||
|
|
||||||
|
- `nine_grid_detect_success_rate`
|
||||||
|
- `nine_grid_misdetect_rate`
|
||||||
|
- `script_normalize_success_rate`
|
||||||
|
- `shot_retry_rate`
|
||||||
|
- `fallback_trigger_rate`
|
||||||
|
- `storyboard_task_success_rate`
|
||||||
|
- `avg_storyboard_total_latency`
|
||||||
|
|
||||||
|
日志关键词建议统一:
|
||||||
|
|
||||||
|
- `[VideoStoryboard]`
|
||||||
|
- `[NineGridDetector]`
|
||||||
|
- `[ScriptNormalizer]`
|
||||||
|
- `[TileUpscaler]`
|
||||||
|
- `[VideoSequencer]`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 18. 测试与验收
|
||||||
|
|
||||||
|
## 18.1 单元测试
|
||||||
|
|
||||||
|
- 九宫格检测:标准图、弱网格图、非九宫格图
|
||||||
|
- 脚本归一:口语化、表格化、混合语言
|
||||||
|
- 映射规则:缺段/超段/乱序编号
|
||||||
|
- 重试降级:模拟 429、余额不足、超时
|
||||||
|
|
||||||
|
## 18.2 集成测试
|
||||||
|
|
||||||
|
1. 上传九宫格 + 9段脚本 -> 全部成功
|
||||||
|
2. 上传九宫格 + 混乱脚本 -> 归一成功
|
||||||
|
3. 部分失败 -> 自动重试 + 降级成功
|
||||||
|
4. 非九宫格 -> 正常走单图视频
|
||||||
|
|
||||||
|
## 18.3 验收标准
|
||||||
|
|
||||||
|
- 切片正确率 ≥ 95%
|
||||||
|
- 9 段任务可追踪、可重试、可取消
|
||||||
|
- 不出现并发超限(严格 10 秒间隔)
|
||||||
|
- 失败可降级且不丢任务状态
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 19. 发布计划(建议)
|
||||||
|
|
||||||
|
1. Phase 1:先上“检测 + 归一 + 任务表 + 状态查询”
|
||||||
|
2. Phase 2:接入“超分能力预留 + 串行调度”(超分开关保持关闭)
|
||||||
|
3. Phase 3:接入降级链路与前端进度 UI
|
||||||
|
4. Phase 4:灰度开关 + 指标监控 + 回归优化
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 20. 风险与规避
|
||||||
|
|
||||||
|
1. **误判九宫格**:加入置信度阈值与人工确认分支
|
||||||
|
2. **成本过高**:超分默认关闭,待效果验证后灰度开启
|
||||||
|
3. **任务堆积**:DB 队列 + 限流 + 可取消机制
|
||||||
|
4. **外部 API 不稳定**:重试 + fallback + 错误分级
|
||||||
|
5. **多机部署一致性**:调度基于 DB 状态机,不依赖内存队列
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 21. 当前实施状态标记
|
||||||
|
|
||||||
|
- 聊天/画布/3D/AI详情/OSS迁移:**已实现并在代码中可见**
|
||||||
|
- RUXA 九宫格视频技能链:**本文档为待开发方案(尚未落地代码)**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 22. 运维与发布命令(项目现状)
|
||||||
|
|
||||||
|
- 安装依赖:`pnpm install --no-frozen-lockfile`
|
||||||
|
- 数据库迁移:`pnpm db:migrate`
|
||||||
|
- 构建:`pnpm build`
|
||||||
|
- 启动:`pnpm start`
|
||||||
|
- PM2 重启:`pm2 restart ai-chatbot || pm2 restart all`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 23. 附:本方案主要影响文件清单(开发阶段)
|
||||||
|
|
||||||
|
新增(建议):
|
||||||
|
|
||||||
|
- `lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts`
|
||||||
|
- `lib/video-agent/skills/script-normalizer.ts`
|
||||||
|
- `lib/video-agent/skills/nine-grid-detector.ts`
|
||||||
|
- `lib/video-agent/skills/tile-upscaler.ts`
|
||||||
|
- `lib/video-agent/skills/storyboard-mapper.ts`
|
||||||
|
- `lib/video-agent/skills/video-sequencer.ts`
|
||||||
|
- `app/api/video/storyboard/create/route.ts`
|
||||||
|
- `app/api/video/storyboard/tasks/[taskId]/route.ts`
|
||||||
|
- `app/api/jobs/video-storyboard-dispatch/route.ts`
|
||||||
|
- `lib/db/migrations/00xx_add_video_storyboard_tasks.sql`
|
||||||
|
|
||||||
|
修改(建议):
|
||||||
|
|
||||||
|
- `app/(chat)/api/chat/route.ts`
|
||||||
|
- `lib/ai/providers/ruxa-video.ts`
|
||||||
|
- `app/(admin)/settings/admin-settings-client.tsx`
|
||||||
|
- `lib/db/schema.ts`
|
||||||
|
- `lib/db/queries.ts`
|
||||||
81
lib/video-agent/config.ts
Normal file
81
lib/video-agent/config.ts
Normal file
@ -0,0 +1,81 @@
|
|||||||
|
import "server-only";
|
||||||
|
|
||||||
|
import { getSystemSetting } from "@/lib/db/queries";
|
||||||
|
|
||||||
|
export const VIDEO_STORYBOARD_CONFIG_KEY = "video_storyboard_config";
|
||||||
|
|
||||||
|
export type VideoStoryboardConfig = {
|
||||||
|
enabled: boolean;
|
||||||
|
providerWhitelist: string[];
|
||||||
|
minGridConfidence: number;
|
||||||
|
enableTileUpscale: boolean;
|
||||||
|
sliceUpscaleMinEdge: number;
|
||||||
|
dispatchIntervalSec: number;
|
||||||
|
maxRetry: number;
|
||||||
|
enableFallback: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const defaultVideoStoryboardConfig: VideoStoryboardConfig = {
|
||||||
|
enabled: true,
|
||||||
|
providerWhitelist: ["ruxa-video"],
|
||||||
|
minGridConfidence: 0.85,
|
||||||
|
enableTileUpscale: false,
|
||||||
|
sliceUpscaleMinEdge: 768,
|
||||||
|
dispatchIntervalSec: 10,
|
||||||
|
maxRetry: 3,
|
||||||
|
enableFallback: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
function toNumber(value: unknown, fallback: number) {
|
||||||
|
const parsed = Number(value);
|
||||||
|
return Number.isFinite(parsed) ? parsed : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeVideoStoryboardConfig(
|
||||||
|
value: unknown
|
||||||
|
): VideoStoryboardConfig {
|
||||||
|
const raw =
|
||||||
|
value && typeof value === "object" && !Array.isArray(value)
|
||||||
|
? (value as Record<string, unknown>)
|
||||||
|
: {};
|
||||||
|
|
||||||
|
const whitelist = Array.isArray(raw.providerWhitelist)
|
||||||
|
? raw.providerWhitelist
|
||||||
|
.map((item) => String(item || "").trim())
|
||||||
|
.filter(Boolean)
|
||||||
|
: defaultVideoStoryboardConfig.providerWhitelist;
|
||||||
|
|
||||||
|
return {
|
||||||
|
enabled: raw.enabled !== false,
|
||||||
|
providerWhitelist:
|
||||||
|
whitelist.length > 0
|
||||||
|
? Array.from(new Set(whitelist))
|
||||||
|
: defaultVideoStoryboardConfig.providerWhitelist,
|
||||||
|
minGridConfidence: Math.max(
|
||||||
|
0.5,
|
||||||
|
Math.min(0.99, toNumber(raw.minGridConfidence, defaultVideoStoryboardConfig.minGridConfidence))
|
||||||
|
),
|
||||||
|
enableTileUpscale: raw.enableTileUpscale === true,
|
||||||
|
sliceUpscaleMinEdge: Math.max(
|
||||||
|
256,
|
||||||
|
Math.min(4096, Math.round(toNumber(raw.sliceUpscaleMinEdge, defaultVideoStoryboardConfig.sliceUpscaleMinEdge)))
|
||||||
|
),
|
||||||
|
dispatchIntervalSec: Math.max(
|
||||||
|
1,
|
||||||
|
Math.min(120, Math.round(toNumber(raw.dispatchIntervalSec, defaultVideoStoryboardConfig.dispatchIntervalSec)))
|
||||||
|
),
|
||||||
|
maxRetry: Math.max(
|
||||||
|
0,
|
||||||
|
Math.min(5, Math.round(toNumber(raw.maxRetry, defaultVideoStoryboardConfig.maxRetry)))
|
||||||
|
),
|
||||||
|
enableFallback: raw.enableFallback !== false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getVideoStoryboardConfig(): Promise<VideoStoryboardConfig> {
|
||||||
|
const setting = await getSystemSetting({ key: VIDEO_STORYBOARD_CONFIG_KEY }).catch(
|
||||||
|
() => undefined
|
||||||
|
);
|
||||||
|
return normalizeVideoStoryboardConfig(setting?.value);
|
||||||
|
}
|
||||||
|
|
||||||
316
lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts
Normal file
316
lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts
Normal file
@ -0,0 +1,316 @@
|
|||||||
|
import "server-only";
|
||||||
|
|
||||||
|
import { generateUUID } from "@/lib/utils";
|
||||||
|
import { runNineGridDetectorSkill } from "@/lib/video-agent/skills/nine-grid-detector";
|
||||||
|
import { runScriptNormalizerSkill } from "@/lib/video-agent/skills/script-normalizer";
|
||||||
|
import { runStoryboardMapperSkill } from "@/lib/video-agent/skills/storyboard-mapper";
|
||||||
|
import { runTileUpscalerSkill } from "@/lib/video-agent/skills/tile-upscaler";
|
||||||
|
import { runVideoSequencerSkill } from "@/lib/video-agent/skills/video-sequencer";
|
||||||
|
import type {
|
||||||
|
RuxaNineGridPipelineParams,
|
||||||
|
RuxaNineGridPipelineResult,
|
||||||
|
StoryboardModelCandidate,
|
||||||
|
} from "@/lib/video-agent/types";
|
||||||
|
|
||||||
|
const NINE_GRID_SHOT_COUNT = 9;
|
||||||
|
|
||||||
|
function normalizeText(value: unknown) {
|
||||||
|
return String(value || "").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractVideoUrlFromText(text: string): string | null {
|
||||||
|
const normalizedText = String(text || "");
|
||||||
|
if (!normalizedText) return null;
|
||||||
|
|
||||||
|
const htmlMatch = normalizedText.match(/<video[^>]+src="([^"]+)"/i);
|
||||||
|
if (htmlMatch?.[1]) return htmlMatch[1];
|
||||||
|
|
||||||
|
const markdownMatch = normalizedText.match(
|
||||||
|
/\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i
|
||||||
|
);
|
||||||
|
if (markdownMatch?.[1]) return markdownMatch[1];
|
||||||
|
|
||||||
|
const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i);
|
||||||
|
return plainMatch?.[1] || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractPromptAndImages(messages: Array<{ role: string; content: any }>) {
|
||||||
|
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
|
||||||
|
if (!lastUserMessage) {
|
||||||
|
return { prompt: "", imageUrls: [] as string[] };
|
||||||
|
}
|
||||||
|
|
||||||
|
let prompt = "";
|
||||||
|
const imageUrls: string[] = [];
|
||||||
|
|
||||||
|
const pushImage = (raw?: string) => {
|
||||||
|
const url = normalizeText(raw);
|
||||||
|
if (!url || !/^https?:\/\//i.test(url)) return;
|
||||||
|
if (!imageUrls.includes(url)) imageUrls.push(url);
|
||||||
|
};
|
||||||
|
|
||||||
|
if (typeof lastUserMessage.content === "string") {
|
||||||
|
prompt = normalizeText(lastUserMessage.content);
|
||||||
|
} else if (Array.isArray(lastUserMessage.content)) {
|
||||||
|
for (const part of lastUserMessage.content) {
|
||||||
|
if (part?.type === "text" && typeof part.text === "string") {
|
||||||
|
prompt = `${prompt} ${part.text}`.trim();
|
||||||
|
} else if (part?.type === "image_url" && part.image_url?.url) {
|
||||||
|
pushImage(part.image_url.url);
|
||||||
|
} else if (part?.type === "file") {
|
||||||
|
pushImage(part.url || part.data);
|
||||||
|
} else if (part?.type === "image" && typeof part.image === "string") {
|
||||||
|
pushImage(part.image);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!prompt) {
|
||||||
|
prompt = "让画面动起来";
|
||||||
|
}
|
||||||
|
|
||||||
|
return { prompt, imageUrls };
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeModelCandidates(params: {
|
||||||
|
selectedModelId: string;
|
||||||
|
selectedModelRef: string;
|
||||||
|
modelCandidates: StoryboardModelCandidate[];
|
||||||
|
}) {
|
||||||
|
const selected: StoryboardModelCandidate = {
|
||||||
|
id: params.selectedModelRef || params.selectedModelId,
|
||||||
|
modelId: params.selectedModelId,
|
||||||
|
displayName: "",
|
||||||
|
};
|
||||||
|
const candidates = [selected, ...params.modelCandidates]
|
||||||
|
.filter((item) => normalizeText(item.modelId))
|
||||||
|
.map((item) => ({
|
||||||
|
id: normalizeText(item.id) || normalizeText(item.modelId),
|
||||||
|
modelId: normalizeText(item.modelId),
|
||||||
|
displayName: normalizeText(item.displayName),
|
||||||
|
}));
|
||||||
|
|
||||||
|
const seen = new Set<string>();
|
||||||
|
return candidates.filter((item) => {
|
||||||
|
if (seen.has(item.modelId)) return false;
|
||||||
|
seen.add(item.modelId);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildCombinedMarkdown(params: {
|
||||||
|
success: Array<{ shotIndex: number; text: string }>;
|
||||||
|
failed: Array<{ shotIndex: number; error: string }>;
|
||||||
|
}) {
|
||||||
|
const sections: string[] = [];
|
||||||
|
|
||||||
|
for (const shot of params.success.sort((a, b) => a.shotIndex - b.shotIndex)) {
|
||||||
|
sections.push(`### 分镜 ${shot.shotIndex}\n${shot.text}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (params.failed.length > 0) {
|
||||||
|
const failedLines = params.failed
|
||||||
|
.sort((a, b) => a.shotIndex - b.shotIndex)
|
||||||
|
.map((item) => `- 分镜 ${item.shotIndex}: ${item.error}`);
|
||||||
|
sections.push(["### 失败分镜", ...failedLines].join("\n"));
|
||||||
|
}
|
||||||
|
|
||||||
|
return sections.join("\n\n").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runRuxaNineGridPipeline(
|
||||||
|
params: RuxaNineGridPipelineParams
|
||||||
|
): Promise<RuxaNineGridPipelineResult> {
|
||||||
|
const {
|
||||||
|
modelMessages,
|
||||||
|
selectedModelId,
|
||||||
|
selectedModelRef,
|
||||||
|
modelCandidates,
|
||||||
|
videoParams,
|
||||||
|
config,
|
||||||
|
onProgress,
|
||||||
|
doGenerate,
|
||||||
|
} = params;
|
||||||
|
|
||||||
|
if (!config.enabled) {
|
||||||
|
return { triggered: false, reason: "disabled" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const providerAllowed = config.providerWhitelist.some(
|
||||||
|
(item) => normalizeText(item).toLowerCase() === "ruxa-video"
|
||||||
|
);
|
||||||
|
if (!providerAllowed) {
|
||||||
|
return { triggered: false, reason: "provider_not_allowed" };
|
||||||
|
}
|
||||||
|
|
||||||
|
onProgress?.({
|
||||||
|
stage: "detecting",
|
||||||
|
provider: "ruxa-video",
|
||||||
|
message: "正在识别九宫格与解析分镜...",
|
||||||
|
});
|
||||||
|
|
||||||
|
const { prompt, imageUrls } = extractPromptAndImages(modelMessages);
|
||||||
|
if (!imageUrls[0]) {
|
||||||
|
return { triggered: false, reason: "missing_image" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const detection = await runNineGridDetectorSkill({
|
||||||
|
imageUrl: imageUrls[0],
|
||||||
|
minConfidence: config.minGridConfidence,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!detection.isNineGrid || detection.tiles.length !== NINE_GRID_SHOT_COUNT) {
|
||||||
|
return {
|
||||||
|
triggered: false,
|
||||||
|
reason: detection.isNineGrid ? "slice_incomplete" : "not_nine_grid",
|
||||||
|
confidence: detection.confidence,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const normalizedScript = runScriptNormalizerSkill({
|
||||||
|
scriptText: prompt,
|
||||||
|
totalShots: NINE_GRID_SHOT_COUNT,
|
||||||
|
totalDurationSec: Number(videoParams?.duration || 10),
|
||||||
|
});
|
||||||
|
const mappedShots = runStoryboardMapperSkill({
|
||||||
|
script: normalizedScript,
|
||||||
|
tiles: detection.tiles,
|
||||||
|
});
|
||||||
|
const preparedShots = await runTileUpscalerSkill({
|
||||||
|
shots: mappedShots,
|
||||||
|
enableTileUpscale: config.enableTileUpscale,
|
||||||
|
sliceUpscaleMinEdge: config.sliceUpscaleMinEdge,
|
||||||
|
});
|
||||||
|
|
||||||
|
const resolvedModelCandidates = normalizeModelCandidates({
|
||||||
|
selectedModelId,
|
||||||
|
selectedModelRef,
|
||||||
|
modelCandidates,
|
||||||
|
});
|
||||||
|
|
||||||
|
const taskId = `ruxa-storyboard-${generateUUID()}`;
|
||||||
|
onProgress?.({
|
||||||
|
stage: "start",
|
||||||
|
provider: "ruxa-video",
|
||||||
|
taskId,
|
||||||
|
totalShots: preparedShots.length,
|
||||||
|
confidence: detection.confidence,
|
||||||
|
sourceImageUrl: imageUrls[0],
|
||||||
|
shots: preparedShots.map((shot) => ({
|
||||||
|
shotIndex: shot.shotIndex,
|
||||||
|
tileImageUrl: shot.inputImageUrl,
|
||||||
|
prompt: shot.prompt,
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
|
||||||
|
const sequencer = await runVideoSequencerSkill({
|
||||||
|
provider: "ruxa-video",
|
||||||
|
taskId,
|
||||||
|
shots: preparedShots,
|
||||||
|
modelCandidates: resolvedModelCandidates,
|
||||||
|
maxRetry: config.maxRetry,
|
||||||
|
dispatchIntervalSec: config.dispatchIntervalSec,
|
||||||
|
enableFallback: config.enableFallback,
|
||||||
|
onProgress,
|
||||||
|
generateOneShot: async ({ shot, model }) => {
|
||||||
|
const shotMessages = [
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content: [
|
||||||
|
{ type: "text", text: shot.prompt },
|
||||||
|
{ type: "image_url", image_url: { url: shot.inputImageUrl } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const output = await doGenerate({
|
||||||
|
messages: shotMessages,
|
||||||
|
videoParams: {
|
||||||
|
...(videoParams || {}),
|
||||||
|
mode: "first-frame",
|
||||||
|
},
|
||||||
|
overrideModelId: model.modelId,
|
||||||
|
});
|
||||||
|
|
||||||
|
const text = normalizeText(output?.text);
|
||||||
|
const videoUrl = extractVideoUrlFromText(text);
|
||||||
|
if (!videoUrl) {
|
||||||
|
throw new Error(text || "未返回视频地址");
|
||||||
|
}
|
||||||
|
return { videoUrl, text };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const successShots = sequencer.results
|
||||||
|
.filter((item) => item.success && item.videoUrl)
|
||||||
|
.map((item) => ({
|
||||||
|
shotIndex: item.shotIndex,
|
||||||
|
text: normalizeText(item.text),
|
||||||
|
videoUrl: normalizeText(item.videoUrl),
|
||||||
|
modelId: normalizeText(item.modelId),
|
||||||
|
}));
|
||||||
|
const failedShots = sequencer.results
|
||||||
|
.filter((item) => !item.success)
|
||||||
|
.map((item) => ({
|
||||||
|
shotIndex: item.shotIndex,
|
||||||
|
error: normalizeText(item.error) || "video_generation_failed",
|
||||||
|
}));
|
||||||
|
|
||||||
|
if (successShots.length === 0) {
|
||||||
|
onProgress?.({
|
||||||
|
stage: "failed",
|
||||||
|
provider: "ruxa-video",
|
||||||
|
taskId,
|
||||||
|
error: "all_shots_failed",
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
triggered: true,
|
||||||
|
taskId,
|
||||||
|
confidence: detection.confidence,
|
||||||
|
reason: "all_shots_failed",
|
||||||
|
videoUrls: [],
|
||||||
|
modelRefs: [],
|
||||||
|
markdown: "视频生成失败,请稍后重试",
|
||||||
|
metadata: {
|
||||||
|
taskId,
|
||||||
|
successCount: 0,
|
||||||
|
failedCount: failedShots.length,
|
||||||
|
confidence: detection.confidence,
|
||||||
|
failedShots,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
onProgress?.({
|
||||||
|
stage: "completed",
|
||||||
|
provider: "ruxa-video",
|
||||||
|
taskId,
|
||||||
|
totalShots: sequencer.results.length,
|
||||||
|
successCount: sequencer.successCount,
|
||||||
|
failedCount: sequencer.failedCount,
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
triggered: true,
|
||||||
|
taskId,
|
||||||
|
confidence: detection.confidence,
|
||||||
|
markdown: buildCombinedMarkdown({
|
||||||
|
success: successShots.map((item) => ({
|
||||||
|
shotIndex: item.shotIndex,
|
||||||
|
text: item.text,
|
||||||
|
})),
|
||||||
|
failed: failedShots,
|
||||||
|
}),
|
||||||
|
videoUrls: successShots.map((item) => item.videoUrl),
|
||||||
|
modelRefs: successShots.map((item) => item.modelId).filter(Boolean),
|
||||||
|
metadata: {
|
||||||
|
taskId,
|
||||||
|
confidence: detection.confidence,
|
||||||
|
qualityScore: detection.qualityScore,
|
||||||
|
sourceImageUrl: detection.sourceImageUrl,
|
||||||
|
successCount: sequencer.successCount,
|
||||||
|
failedCount: sequencer.failedCount,
|
||||||
|
failedShots,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
251
lib/video-agent/skills/nine-grid-detector.ts
Normal file
251
lib/video-agent/skills/nine-grid-detector.ts
Normal file
@ -0,0 +1,251 @@
|
|||||||
|
import "server-only";
|
||||||
|
|
||||||
|
import sharp from "sharp";
|
||||||
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
|
import type { NineGridCropBox, NineGridDetectionResult, NineGridTile } from "@/lib/video-agent/types";
|
||||||
|
|
||||||
|
type ProjectionPeak = {
|
||||||
|
index: number;
|
||||||
|
value: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
function clamp01(value: number) {
|
||||||
|
if (!Number.isFinite(value)) return 0;
|
||||||
|
return Math.max(0, Math.min(1, value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function sampleStddevFromUint8(data: Uint8Array, limit = 20000) {
|
||||||
|
const total = Math.min(data.length, Math.max(128, limit));
|
||||||
|
if (total <= 0) return 0;
|
||||||
|
const stride = Math.max(1, Math.floor(data.length / total));
|
||||||
|
const sampled: number[] = [];
|
||||||
|
for (let index = 0; index < data.length && sampled.length < total; index += stride) {
|
||||||
|
sampled.push(data[index] || 0);
|
||||||
|
}
|
||||||
|
return stddev(sampled);
|
||||||
|
}
|
||||||
|
|
||||||
|
function toArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||||||
|
return buffer.buffer.slice(
|
||||||
|
buffer.byteOffset,
|
||||||
|
buffer.byteOffset + buffer.byteLength
|
||||||
|
) as ArrayBuffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
function movingAverage(values: number[], radius = 3): number[] {
|
||||||
|
if (values.length === 0) return values;
|
||||||
|
const result = new Array(values.length).fill(0);
|
||||||
|
for (let index = 0; index < values.length; index += 1) {
|
||||||
|
let sum = 0;
|
||||||
|
let count = 0;
|
||||||
|
const start = Math.max(0, index - radius);
|
||||||
|
const end = Math.min(values.length - 1, index + radius);
|
||||||
|
for (let cursor = start; cursor <= end; cursor += 1) {
|
||||||
|
sum += values[cursor] || 0;
|
||||||
|
count += 1;
|
||||||
|
}
|
||||||
|
result[index] = count > 0 ? sum / count : 0;
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
function mean(values: number[]) {
|
||||||
|
if (values.length === 0) return 0;
|
||||||
|
return values.reduce((total, item) => total + item, 0) / values.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
function stddev(values: number[]) {
|
||||||
|
if (values.length === 0) return 0;
|
||||||
|
const avg = mean(values);
|
||||||
|
const variance =
|
||||||
|
values.reduce((total, item) => total + Math.pow(item - avg, 2), 0) / values.length;
|
||||||
|
return Math.sqrt(Math.max(variance, 0));
|
||||||
|
}
|
||||||
|
|
||||||
|
function findPeakNear(
|
||||||
|
scores: number[],
|
||||||
|
targetIndex: number,
|
||||||
|
windowRadius: number
|
||||||
|
): ProjectionPeak {
|
||||||
|
const start = Math.max(0, Math.round(targetIndex - windowRadius));
|
||||||
|
const end = Math.min(scores.length - 1, Math.round(targetIndex + windowRadius));
|
||||||
|
let bestIndex = Math.round(targetIndex);
|
||||||
|
let bestValue = -Infinity;
|
||||||
|
for (let index = start; index <= end; index += 1) {
|
||||||
|
const value = scores[index] || 0;
|
||||||
|
if (value > bestValue) {
|
||||||
|
bestValue = value;
|
||||||
|
bestIndex = index;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { index: bestIndex, value: Number.isFinite(bestValue) ? bestValue : 0 };
|
||||||
|
}
|
||||||
|
|
||||||
|
function scoreGridAxis(scores: number[], firstTarget: number, secondTarget: number) {
|
||||||
|
const radius = Math.max(8, Math.round(scores.length * 0.12));
|
||||||
|
const firstPeak = findPeakNear(scores, firstTarget, radius);
|
||||||
|
const secondPeak = findPeakNear(scores, secondTarget, radius);
|
||||||
|
const avg = mean(scores);
|
||||||
|
const sigma = stddev(scores);
|
||||||
|
const dynamicBase = avg + sigma;
|
||||||
|
|
||||||
|
const closenessA = clamp01(1 - Math.abs(firstPeak.index - firstTarget) / radius);
|
||||||
|
const closenessB = clamp01(1 - Math.abs(secondPeak.index - secondTarget) / radius);
|
||||||
|
const closeness = (closenessA + closenessB) / 2;
|
||||||
|
|
||||||
|
const denom = Math.max(dynamicBase, 1);
|
||||||
|
const strengthA = clamp01(firstPeak.value / (denom * 2.2));
|
||||||
|
const strengthB = clamp01(secondPeak.value / (denom * 2.2));
|
||||||
|
const strength = (strengthA + strengthB) / 2;
|
||||||
|
|
||||||
|
return {
|
||||||
|
score: clamp01(closeness * 0.55 + strength * 0.45),
|
||||||
|
peaks: [firstPeak, secondPeak],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildNineGridCropBoxes(width: number, height: number): NineGridCropBox[] {
|
||||||
|
const baseW = Math.floor(width / 3);
|
||||||
|
const baseH = Math.floor(height / 3);
|
||||||
|
const boxes: NineGridCropBox[] = [];
|
||||||
|
let index = 1;
|
||||||
|
for (let row = 0; row < 3; row += 1) {
|
||||||
|
for (let col = 0; col < 3; col += 1) {
|
||||||
|
const x = col * baseW;
|
||||||
|
const y = row * baseH;
|
||||||
|
const w = col === 2 ? width - x : baseW;
|
||||||
|
const h = row === 2 ? height - y : baseH;
|
||||||
|
boxes.push({ index, x, y, w, h });
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return boxes;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchImageBuffer(imageUrl: string) {
|
||||||
|
const response = await fetch(imageUrl);
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`图片下载失败 (HTTP ${response.status})`);
|
||||||
|
}
|
||||||
|
const contentType = response.headers.get("content-type") || "";
|
||||||
|
if (!/^image\//i.test(contentType)) {
|
||||||
|
throw new Error(`输入不是图片类型: ${contentType || "unknown"}`);
|
||||||
|
}
|
||||||
|
return Buffer.from(await response.arrayBuffer());
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sliceAndUploadTiles(params: {
|
||||||
|
sourceBuffer: Buffer;
|
||||||
|
cropBoxes: NineGridCropBox[];
|
||||||
|
}): Promise<NineGridTile[]> {
|
||||||
|
const { sourceBuffer, cropBoxes } = params;
|
||||||
|
const timestamp = Date.now();
|
||||||
|
const tiles: NineGridTile[] = [];
|
||||||
|
for (const cropBox of cropBoxes) {
|
||||||
|
const tileBuffer = await sharp(sourceBuffer)
|
||||||
|
.extract({
|
||||||
|
left: cropBox.x,
|
||||||
|
top: cropBox.y,
|
||||||
|
width: cropBox.w,
|
||||||
|
height: cropBox.h,
|
||||||
|
})
|
||||||
|
.jpeg({ quality: 92 })
|
||||||
|
.toBuffer();
|
||||||
|
|
||||||
|
const filename = `video-storyboard-tile-${timestamp}-${cropBox.index}.jpg`;
|
||||||
|
const uploaded = await uploadFile(filename, toArrayBuffer(tileBuffer));
|
||||||
|
tiles.push({
|
||||||
|
index: cropBox.index,
|
||||||
|
imageUrl: uploaded.url,
|
||||||
|
width: cropBox.w,
|
||||||
|
height: cropBox.h,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return tiles;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runNineGridDetectorSkill(params: {
|
||||||
|
imageUrl: string;
|
||||||
|
minConfidence: number;
|
||||||
|
}): Promise<NineGridDetectionResult> {
|
||||||
|
const sourceImageUrl = String(params.imageUrl || "").trim();
|
||||||
|
if (!sourceImageUrl) {
|
||||||
|
throw new Error("缺少九宫格输入图片");
|
||||||
|
}
|
||||||
|
|
||||||
|
const sourceBuffer = await fetchImageBuffer(sourceImageUrl);
|
||||||
|
const baseImage = sharp(sourceBuffer, { failOn: "none" }).rotate();
|
||||||
|
const metadata = await baseImage.metadata();
|
||||||
|
const width = Number(metadata.width || 0);
|
||||||
|
const height = Number(metadata.height || 0);
|
||||||
|
|
||||||
|
if (width < 240 || height < 240) {
|
||||||
|
return {
|
||||||
|
isNineGrid: false,
|
||||||
|
confidence: 0,
|
||||||
|
qualityScore: 0,
|
||||||
|
cropBoxes: [],
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
sourceImageUrl,
|
||||||
|
tiles: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const grayscale = await baseImage.greyscale().raw().toBuffer({
|
||||||
|
resolveWithObject: true,
|
||||||
|
});
|
||||||
|
const raw = grayscale.data;
|
||||||
|
const channels = grayscale.info.channels || 1;
|
||||||
|
const imgW = grayscale.info.width;
|
||||||
|
const imgH = grayscale.info.height;
|
||||||
|
const step = Math.max(1, Math.round(Math.min(imgW, imgH) / 900));
|
||||||
|
|
||||||
|
const colScores = new Array(imgW).fill(0);
|
||||||
|
const rowScores = new Array(imgH).fill(0);
|
||||||
|
|
||||||
|
for (let y = 0; y < imgH; y += step) {
|
||||||
|
const rowOffset = y * imgW * channels;
|
||||||
|
for (let x = step; x < imgW; x += step) {
|
||||||
|
const current = raw[rowOffset + x * channels] || 0;
|
||||||
|
const prev = raw[rowOffset + (x - step) * channels] || 0;
|
||||||
|
colScores[x] += Math.abs(current - prev);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let x = 0; x < imgW; x += step) {
|
||||||
|
for (let y = step; y < imgH; y += step) {
|
||||||
|
const current = raw[(y * imgW + x) * channels] || 0;
|
||||||
|
const prev = raw[((y - step) * imgW + x) * channels] || 0;
|
||||||
|
rowScores[y] += Math.abs(current - prev);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const smoothCols = movingAverage(colScores, 5);
|
||||||
|
const smoothRows = movingAverage(rowScores, 5);
|
||||||
|
const colAxis = scoreGridAxis(smoothCols, imgW / 3, (imgW * 2) / 3);
|
||||||
|
const rowAxis = scoreGridAxis(smoothRows, imgH / 3, (imgH * 2) / 3);
|
||||||
|
|
||||||
|
const ratio = width / Math.max(height, 1);
|
||||||
|
const aspectScore = clamp01(1 - Math.min(Math.abs(ratio - 1), 1));
|
||||||
|
const gridScore = clamp01((colAxis.score + rowAxis.score) / 2);
|
||||||
|
const confidence = clamp01(aspectScore * 0.3 + gridScore * 0.7);
|
||||||
|
const qualityScore = clamp01((sampleStddevFromUint8(raw, 20000) || 0) / 72);
|
||||||
|
|
||||||
|
const isNineGrid = confidence >= params.minConfidence && aspectScore >= 0.62;
|
||||||
|
const cropBoxes = isNineGrid ? buildNineGridCropBoxes(width, height) : [];
|
||||||
|
const tiles = isNineGrid
|
||||||
|
? await sliceAndUploadTiles({ sourceBuffer: await baseImage.toBuffer(), cropBoxes })
|
||||||
|
: [];
|
||||||
|
|
||||||
|
return {
|
||||||
|
isNineGrid,
|
||||||
|
confidence,
|
||||||
|
qualityScore,
|
||||||
|
cropBoxes,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
sourceImageUrl,
|
||||||
|
tiles,
|
||||||
|
};
|
||||||
|
}
|
||||||
109
lib/video-agent/skills/script-normalizer.ts
Normal file
109
lib/video-agent/skills/script-normalizer.ts
Normal file
@ -0,0 +1,109 @@
|
|||||||
|
import "server-only";
|
||||||
|
|
||||||
|
import type { NormalizedStoryboardScript, NormalizedStoryboardShot } from "@/lib/video-agent/types";
|
||||||
|
|
||||||
|
function detectLanguage(text: string): "zh" | "en" | "mixed" {
|
||||||
|
const source = String(text || "");
|
||||||
|
const zhCount = (source.match(/[\u4e00-\u9fff]/g) || []).length;
|
||||||
|
const enCount = (source.match(/[A-Za-z]/g) || []).length;
|
||||||
|
if (zhCount > 0 && enCount > 0) return "mixed";
|
||||||
|
if (zhCount > 0) return "zh";
|
||||||
|
return "en";
|
||||||
|
}
|
||||||
|
|
||||||
|
function cleanLine(line: string) {
|
||||||
|
return String(line || "")
|
||||||
|
.replace(/^\s*(?:第?\s*\d+\s*(?:镜头|段|幕)|\d+\s*[.)、::\-]?|[一二三四五六七八九十]+\s*[、.::\-]?)\s*/g, "")
|
||||||
|
.replace(/\s+/g, " ")
|
||||||
|
.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function splitScriptLines(script: string): string[] {
|
||||||
|
const source = String(script || "").trim();
|
||||||
|
if (!source) return [];
|
||||||
|
const byLine = source
|
||||||
|
.split(/\r?\n+/)
|
||||||
|
.map(cleanLine)
|
||||||
|
.filter(Boolean);
|
||||||
|
if (byLine.length >= 4) return byLine;
|
||||||
|
|
||||||
|
return source
|
||||||
|
.split(/[。!?!?;;\n]+/g)
|
||||||
|
.map(cleanLine)
|
||||||
|
.filter(Boolean);
|
||||||
|
}
|
||||||
|
|
||||||
|
function inferCamera(text: string) {
|
||||||
|
const source = String(text || "");
|
||||||
|
if (/推进|推近|close[- ]?in|dolly in/i.test(source)) return "轻微推进";
|
||||||
|
if (/拉远|远景|zoom out|wide/i.test(source)) return "缓慢拉远";
|
||||||
|
if (/平移|横移|pan/i.test(source)) return "平稳平移";
|
||||||
|
if (/旋转|环绕|orbit/i.test(source)) return "环绕镜头";
|
||||||
|
return "稳定镜头";
|
||||||
|
}
|
||||||
|
|
||||||
|
function inferMood(text: string) {
|
||||||
|
const source = String(text || "");
|
||||||
|
if (/紧张|压迫|惊悚|tense/i.test(source)) return "紧张";
|
||||||
|
if (/温暖|治愈|暖/i.test(source)) return "温暖";
|
||||||
|
if (/活力|动感|快节奏|energetic/i.test(source)) return "活力";
|
||||||
|
if (/浪漫|梦幻|romantic/i.test(source)) return "浪漫";
|
||||||
|
return "自然";
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildDefaultAction(index: number) {
|
||||||
|
const defaults = [
|
||||||
|
"主体从静止到轻微动作",
|
||||||
|
"主体与环境产生互动",
|
||||||
|
"主体视线发生转移",
|
||||||
|
"主体做细节动作展示",
|
||||||
|
"镜头展示关键部位",
|
||||||
|
"镜头拉近突出质感",
|
||||||
|
"镜头切换到侧面动态",
|
||||||
|
"主体完成动作收束",
|
||||||
|
"镜头定格形成结尾",
|
||||||
|
];
|
||||||
|
return defaults[index - 1] || "主体做自然动作";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function runScriptNormalizerSkill(params: {
|
||||||
|
scriptText: string;
|
||||||
|
totalShots?: number;
|
||||||
|
totalDurationSec?: number;
|
||||||
|
}): NormalizedStoryboardScript {
|
||||||
|
const totalShots = Math.max(1, Math.min(20, Math.round(params.totalShots || 9)));
|
||||||
|
const raw = String(params.scriptText || "").trim();
|
||||||
|
const lines = splitScriptLines(raw);
|
||||||
|
const durationPerShot = Math.max(
|
||||||
|
2,
|
||||||
|
Math.round(Math.max(5, Number(params.totalDurationSec || 10)) / totalShots)
|
||||||
|
);
|
||||||
|
|
||||||
|
const normalizedShots: NormalizedStoryboardShot[] = [];
|
||||||
|
const language = detectLanguage(raw);
|
||||||
|
const fallbackStyle =
|
||||||
|
language === "en" ? "cinematic realistic style" : "写实电影感";
|
||||||
|
|
||||||
|
for (let index = 1; index <= totalShots; index += 1) {
|
||||||
|
const line = lines[index - 1] || lines[lines.length - 1] || "";
|
||||||
|
const action = cleanLine(line) || buildDefaultAction(index);
|
||||||
|
normalizedShots.push({
|
||||||
|
index,
|
||||||
|
durationSec: durationPerShot,
|
||||||
|
action,
|
||||||
|
camera: inferCamera(line),
|
||||||
|
mood: inferMood(line),
|
||||||
|
negativePrompt:
|
||||||
|
language === "en"
|
||||||
|
? "jitter, blur, artifacts, low quality"
|
||||||
|
: "抖动, 畸变, 低清, 伪影",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
language,
|
||||||
|
globalStyle: fallbackStyle,
|
||||||
|
shots: normalizedShots,
|
||||||
|
raw,
|
||||||
|
};
|
||||||
|
}
|
||||||
73
lib/video-agent/skills/storyboard-mapper.ts
Normal file
73
lib/video-agent/skills/storyboard-mapper.ts
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
import "server-only";
|
||||||
|
|
||||||
|
import type {
|
||||||
|
MappedStoryboardShot,
|
||||||
|
NineGridTile,
|
||||||
|
NormalizedStoryboardScript,
|
||||||
|
} from "@/lib/video-agent/types";
|
||||||
|
|
||||||
|
function clampIndex(value: number, length: number) {
|
||||||
|
if (length <= 0) return 0;
|
||||||
|
const rounded = Math.max(1, Math.round(value));
|
||||||
|
return Math.min(length, rounded) - 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildShotPrompt(params: {
|
||||||
|
style: string;
|
||||||
|
action: string;
|
||||||
|
camera: string;
|
||||||
|
mood: string;
|
||||||
|
language: "zh" | "en" | "mixed";
|
||||||
|
}) {
|
||||||
|
const { style, action, camera, mood, language } = params;
|
||||||
|
if (language === "en") {
|
||||||
|
return [
|
||||||
|
"Animate this still image into a coherent short scene.",
|
||||||
|
`Action: ${action}.`,
|
||||||
|
`Camera: ${camera}.`,
|
||||||
|
`Mood: ${mood}.`,
|
||||||
|
`Style: ${style}.`,
|
||||||
|
"Keep identity and key structure stable.",
|
||||||
|
].join(" ");
|
||||||
|
}
|
||||||
|
|
||||||
|
return [
|
||||||
|
"请基于当前输入图片生成连贯的视频镜头。",
|
||||||
|
`动作:${action}。`,
|
||||||
|
`镜头:${camera}。`,
|
||||||
|
`氛围:${mood}。`,
|
||||||
|
`风格:${style}。`,
|
||||||
|
"保持主体身份、构图和关键细节稳定。",
|
||||||
|
].join("");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function runStoryboardMapperSkill(params: {
|
||||||
|
script: NormalizedStoryboardScript;
|
||||||
|
tiles: NineGridTile[];
|
||||||
|
}): MappedStoryboardShot[] {
|
||||||
|
const { script, tiles } = params;
|
||||||
|
if (!tiles.length) return [];
|
||||||
|
|
||||||
|
return script.shots.map((shot, index) => {
|
||||||
|
const tile = tiles[clampIndex(shot.index || index + 1, tiles.length)] || tiles[index] || tiles[0];
|
||||||
|
const prompt = buildShotPrompt({
|
||||||
|
style: script.globalStyle,
|
||||||
|
action: shot.action,
|
||||||
|
camera: shot.camera,
|
||||||
|
mood: shot.mood,
|
||||||
|
language: script.language,
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
shotIndex: shot.index,
|
||||||
|
tileIndex: tile.index,
|
||||||
|
tileImageUrl: tile.imageUrl,
|
||||||
|
durationSec: shot.durationSec,
|
||||||
|
prompt,
|
||||||
|
negativePrompt: shot.negativePrompt,
|
||||||
|
action: shot.action,
|
||||||
|
camera: shot.camera,
|
||||||
|
mood: shot.mood,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
18
lib/video-agent/skills/tile-upscaler.ts
Normal file
18
lib/video-agent/skills/tile-upscaler.ts
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
import "server-only";
|
||||||
|
|
||||||
|
import type { MappedStoryboardShot, PreparedStoryboardShot } from "@/lib/video-agent/types";
|
||||||
|
|
||||||
|
export async function runTileUpscalerSkill(params: {
|
||||||
|
shots: MappedStoryboardShot[];
|
||||||
|
enableTileUpscale: boolean;
|
||||||
|
sliceUpscaleMinEdge: number;
|
||||||
|
}): Promise<PreparedStoryboardShot[]> {
|
||||||
|
const { shots, enableTileUpscale } = params;
|
||||||
|
|
||||||
|
return shots.map((shot) => ({
|
||||||
|
...shot,
|
||||||
|
inputImageUrl: shot.tileImageUrl,
|
||||||
|
upscaled: false,
|
||||||
|
upscaleReason: enableTileUpscale ? "reserved_not_enabled" : "disabled",
|
||||||
|
}));
|
||||||
|
}
|
||||||
149
lib/video-agent/skills/video-sequencer.ts
Normal file
149
lib/video-agent/skills/video-sequencer.ts
Normal file
@ -0,0 +1,149 @@
|
|||||||
|
import "server-only";
|
||||||
|
|
||||||
|
import type {
|
||||||
|
PreparedStoryboardShot,
|
||||||
|
StoryboardModelCandidate,
|
||||||
|
StoryboardProgressEvent,
|
||||||
|
} from "@/lib/video-agent/types";
|
||||||
|
|
||||||
|
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
|
||||||
|
function normalizeErrorMessage(error: unknown) {
|
||||||
|
if (!error) return "unknown_error";
|
||||||
|
if (typeof error === "string") return error;
|
||||||
|
if (error instanceof Error) return error.message || String(error);
|
||||||
|
try {
|
||||||
|
return JSON.stringify(error);
|
||||||
|
} catch {
|
||||||
|
return String(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export type SequencerShotResult = {
|
||||||
|
shotIndex: number;
|
||||||
|
success: boolean;
|
||||||
|
videoUrl?: string;
|
||||||
|
modelId?: string;
|
||||||
|
text?: string;
|
||||||
|
error?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function runVideoSequencerSkill(params: {
|
||||||
|
provider: string;
|
||||||
|
taskId: string;
|
||||||
|
shots: PreparedStoryboardShot[];
|
||||||
|
modelCandidates: StoryboardModelCandidate[];
|
||||||
|
maxRetry: number;
|
||||||
|
dispatchIntervalSec: number;
|
||||||
|
enableFallback: boolean;
|
||||||
|
onProgress?: (event: StoryboardProgressEvent) => void;
|
||||||
|
generateOneShot: (input: {
|
||||||
|
shot: PreparedStoryboardShot;
|
||||||
|
model: StoryboardModelCandidate;
|
||||||
|
}) => Promise<{ videoUrl: string; text: string }>;
|
||||||
|
}): Promise<{
|
||||||
|
results: SequencerShotResult[];
|
||||||
|
successCount: number;
|
||||||
|
failedCount: number;
|
||||||
|
}> {
|
||||||
|
const {
|
||||||
|
provider,
|
||||||
|
taskId,
|
||||||
|
shots,
|
||||||
|
modelCandidates,
|
||||||
|
maxRetry,
|
||||||
|
dispatchIntervalSec,
|
||||||
|
enableFallback,
|
||||||
|
onProgress,
|
||||||
|
generateOneShot,
|
||||||
|
} = params;
|
||||||
|
|
||||||
|
const candidateList =
|
||||||
|
enableFallback && modelCandidates.length > 0
|
||||||
|
? modelCandidates
|
||||||
|
: modelCandidates.slice(0, 1);
|
||||||
|
|
||||||
|
const results: SequencerShotResult[] = [];
|
||||||
|
|
||||||
|
for (let index = 0; index < shots.length; index += 1) {
|
||||||
|
const shot = shots[index]!;
|
||||||
|
let shotSuccess = false;
|
||||||
|
let lastError = "";
|
||||||
|
let resultText = "";
|
||||||
|
let resultUrl = "";
|
||||||
|
let usedModelId = "";
|
||||||
|
|
||||||
|
for (const model of candidateList) {
|
||||||
|
const retries = Math.max(0, Math.round(maxRetry));
|
||||||
|
for (let attempt = 1; attempt <= retries + 1; attempt += 1) {
|
||||||
|
try {
|
||||||
|
const output = await generateOneShot({ shot, model });
|
||||||
|
resultText = output.text;
|
||||||
|
resultUrl = output.videoUrl;
|
||||||
|
usedModelId = model.modelId;
|
||||||
|
shotSuccess = true;
|
||||||
|
break;
|
||||||
|
} catch (error) {
|
||||||
|
lastError = normalizeErrorMessage(error);
|
||||||
|
if (attempt <= retries) {
|
||||||
|
onProgress?.({
|
||||||
|
stage: "shot_retry",
|
||||||
|
provider,
|
||||||
|
taskId,
|
||||||
|
shotIndex: shot.shotIndex,
|
||||||
|
attempt,
|
||||||
|
modelId: model.modelId,
|
||||||
|
error: lastError,
|
||||||
|
});
|
||||||
|
const backoffSec = Math.min(40, Math.pow(2, attempt - 1) * 10);
|
||||||
|
await sleep(backoffSec * 1000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (shotSuccess) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (shotSuccess) {
|
||||||
|
results.push({
|
||||||
|
shotIndex: shot.shotIndex,
|
||||||
|
success: true,
|
||||||
|
modelId: usedModelId,
|
||||||
|
videoUrl: resultUrl,
|
||||||
|
text: resultText,
|
||||||
|
});
|
||||||
|
onProgress?.({
|
||||||
|
stage: "shot_completed",
|
||||||
|
provider,
|
||||||
|
taskId,
|
||||||
|
shotIndex: shot.shotIndex,
|
||||||
|
modelId: usedModelId,
|
||||||
|
videoUrl: resultUrl,
|
||||||
|
text: resultText,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
results.push({
|
||||||
|
shotIndex: shot.shotIndex,
|
||||||
|
success: false,
|
||||||
|
error: lastError || "video_generation_failed",
|
||||||
|
});
|
||||||
|
onProgress?.({
|
||||||
|
stage: "shot_failed",
|
||||||
|
provider,
|
||||||
|
taskId,
|
||||||
|
shotIndex: shot.shotIndex,
|
||||||
|
error: lastError || "video_generation_failed",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (index < shots.length - 1 && dispatchIntervalSec > 0) {
|
||||||
|
await sleep(Math.round(dispatchIntervalSec * 1000));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const successCount = results.filter((item) => item.success).length;
|
||||||
|
const failedCount = results.length - successCount;
|
||||||
|
return { results, successCount, failedCount };
|
||||||
|
}
|
||||||
152
lib/video-agent/types.ts
Normal file
152
lib/video-agent/types.ts
Normal file
@ -0,0 +1,152 @@
|
|||||||
|
import type { VideoStoryboardConfig } from "@/lib/video-agent/config";
|
||||||
|
|
||||||
|
export type NormalizedStoryboardShot = {
|
||||||
|
index: number;
|
||||||
|
durationSec: number;
|
||||||
|
action: string;
|
||||||
|
camera: string;
|
||||||
|
mood: string;
|
||||||
|
negativePrompt: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type NormalizedStoryboardScript = {
|
||||||
|
language: "zh" | "en" | "mixed";
|
||||||
|
globalStyle: string;
|
||||||
|
shots: NormalizedStoryboardShot[];
|
||||||
|
raw: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type NineGridCropBox = {
|
||||||
|
index: number;
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
w: number;
|
||||||
|
h: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type NineGridTile = {
|
||||||
|
index: number;
|
||||||
|
imageUrl: string;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type NineGridDetectionResult = {
|
||||||
|
isNineGrid: boolean;
|
||||||
|
confidence: number;
|
||||||
|
qualityScore: number;
|
||||||
|
cropBoxes: NineGridCropBox[];
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
sourceImageUrl: string;
|
||||||
|
tiles: NineGridTile[];
|
||||||
|
};
|
||||||
|
|
||||||
|
export type MappedStoryboardShot = {
|
||||||
|
shotIndex: number;
|
||||||
|
tileIndex: number;
|
||||||
|
tileImageUrl: string;
|
||||||
|
durationSec: number;
|
||||||
|
prompt: string;
|
||||||
|
negativePrompt: string;
|
||||||
|
action: string;
|
||||||
|
camera: string;
|
||||||
|
mood: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type PreparedStoryboardShot = MappedStoryboardShot & {
|
||||||
|
inputImageUrl: string;
|
||||||
|
upscaled: boolean;
|
||||||
|
upscaleReason?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type StoryboardModelCandidate = {
|
||||||
|
id: string;
|
||||||
|
modelId: string;
|
||||||
|
displayName?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type StoryboardProgressEvent =
|
||||||
|
| {
|
||||||
|
stage: "detecting";
|
||||||
|
provider: string;
|
||||||
|
message?: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
stage: "start";
|
||||||
|
provider: string;
|
||||||
|
taskId: string;
|
||||||
|
totalShots: number;
|
||||||
|
confidence: number;
|
||||||
|
sourceImageUrl: string;
|
||||||
|
shots: Array<{
|
||||||
|
shotIndex: number;
|
||||||
|
tileImageUrl: string;
|
||||||
|
prompt: string;
|
||||||
|
}>;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
stage: "shot_retry";
|
||||||
|
provider: string;
|
||||||
|
taskId: string;
|
||||||
|
shotIndex: number;
|
||||||
|
attempt: number;
|
||||||
|
modelId: string;
|
||||||
|
error: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
stage: "shot_completed";
|
||||||
|
provider: string;
|
||||||
|
taskId: string;
|
||||||
|
shotIndex: number;
|
||||||
|
modelId: string;
|
||||||
|
videoUrl: string;
|
||||||
|
text: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
stage: "shot_failed";
|
||||||
|
provider: string;
|
||||||
|
taskId: string;
|
||||||
|
shotIndex: number;
|
||||||
|
error: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
stage: "completed";
|
||||||
|
provider: string;
|
||||||
|
taskId: string;
|
||||||
|
totalShots: number;
|
||||||
|
successCount: number;
|
||||||
|
failedCount: number;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
stage: "failed";
|
||||||
|
provider: string;
|
||||||
|
taskId: string;
|
||||||
|
error: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type RuxaNineGridPipelineParams = {
|
||||||
|
modelMessages: Array<{ role: string; content: any }>;
|
||||||
|
selectedModelId: string;
|
||||||
|
selectedModelRef: string;
|
||||||
|
modelCandidates: StoryboardModelCandidate[];
|
||||||
|
videoParams?: Record<string, any>;
|
||||||
|
config: VideoStoryboardConfig;
|
||||||
|
onProgress?: (event: StoryboardProgressEvent) => void;
|
||||||
|
doGenerate: (input: {
|
||||||
|
messages: Array<{ role: string; content: any }>;
|
||||||
|
videoParams?: Record<string, any>;
|
||||||
|
overrideModelId?: string;
|
||||||
|
}) => Promise<{ text?: string }>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type RuxaNineGridPipelineResult = {
|
||||||
|
triggered: boolean;
|
||||||
|
taskId?: string;
|
||||||
|
confidence?: number;
|
||||||
|
markdown?: string;
|
||||||
|
videoUrls?: string[];
|
||||||
|
modelRefs?: string[];
|
||||||
|
metadata?: Record<string, any>;
|
||||||
|
reason?: string;
|
||||||
|
};
|
||||||
Loading…
x
Reference in New Issue
Block a user