ai-chatbot/lib/video-agent/pipeline/ruxa-nine-grid-pipeline.ts
Your Name 1967992de7
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled
feat(video): storyboard task tables and delayed archive flow
2026-03-05 15:58:43 +08:00

415 lines
12 KiB
TypeScript

import "server-only";
import { generateUUID } from "@/lib/utils";
import {
createVideoStoryboardShotTasks,
createVideoStoryboardTask,
finalizeVideoStoryboardTask,
markVideoStoryboardShotCompleted,
markVideoStoryboardShotFailed,
markVideoStoryboardShotRetry,
markVideoStoryboardShotRunning,
} from "@/lib/video-agent/task-store";
import { runNineGridDetectorSkill } from "@/lib/video-agent/skills/nine-grid-detector";
import { runScriptNormalizerSkill } from "@/lib/video-agent/skills/script-normalizer";
import { runStoryboardMapperSkill } from "@/lib/video-agent/skills/storyboard-mapper";
import { runTileUpscalerSkill } from "@/lib/video-agent/skills/tile-upscaler";
import { runVideoSequencerSkill } from "@/lib/video-agent/skills/video-sequencer";
import type {
RuxaNineGridPipelineParams,
RuxaNineGridPipelineResult,
StoryboardModelCandidate,
} from "@/lib/video-agent/types";
const NINE_GRID_SHOT_COUNT = 9;
function normalizeText(value: unknown) {
return String(value || "").trim();
}
function extractVideoUrlFromText(text: string): string | null {
const normalizedText = String(text || "");
if (!normalizedText) return null;
const htmlMatch = normalizedText.match(/<video[^>]+src="([^"]+)"/i);
if (htmlMatch?.[1]) return htmlMatch[1];
const markdownMatch = normalizedText.match(
/\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i
);
if (markdownMatch?.[1]) return markdownMatch[1];
const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i);
return plainMatch?.[1] || null;
}
function extractPromptAndImages(messages: Array<{ role: string; content: any }>) {
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
if (!lastUserMessage) {
return { prompt: "", imageUrls: [] as string[] };
}
let prompt = "";
const imageUrls: string[] = [];
const pushImage = (raw?: string) => {
const url = normalizeText(raw);
if (!url || !/^https?:\/\//i.test(url)) return;
if (!imageUrls.includes(url)) imageUrls.push(url);
};
if (typeof lastUserMessage.content === "string") {
prompt = normalizeText(lastUserMessage.content);
} else if (Array.isArray(lastUserMessage.content)) {
for (const part of lastUserMessage.content) {
if (part?.type === "text" && typeof part.text === "string") {
prompt = `${prompt} ${part.text}`.trim();
} else if (part?.type === "image_url" && part.image_url?.url) {
pushImage(part.image_url.url);
} else if (part?.type === "file") {
pushImage(part.url || part.data);
} else if (part?.type === "image" && typeof part.image === "string") {
pushImage(part.image);
}
}
}
if (!prompt) {
prompt = "让画面动起来";
}
return { prompt, imageUrls };
}
function normalizeModelCandidates(params: {
selectedModelId: string;
selectedModelRef: string;
modelCandidates: StoryboardModelCandidate[];
}) {
const selected: StoryboardModelCandidate = {
id: params.selectedModelRef || params.selectedModelId,
modelId: params.selectedModelId,
displayName: "",
};
const candidates = [selected, ...params.modelCandidates]
.filter((item) => normalizeText(item.modelId))
.map((item) => ({
id: normalizeText(item.id) || normalizeText(item.modelId),
modelId: normalizeText(item.modelId),
displayName: normalizeText(item.displayName),
}));
const seen = new Set<string>();
return candidates.filter((item) => {
if (seen.has(item.modelId)) return false;
seen.add(item.modelId);
return true;
});
}
function buildCombinedMarkdown(params: {
success: Array<{ shotIndex: number; text: string }>;
failed: Array<{ shotIndex: number; error: string }>;
}) {
const sections: string[] = [];
for (const shot of params.success.sort((a, b) => a.shotIndex - b.shotIndex)) {
sections.push(`### 分镜 ${shot.shotIndex}\n${shot.text}`);
}
if (params.failed.length > 0) {
const failedLines = params.failed
.sort((a, b) => a.shotIndex - b.shotIndex)
.map((item) => `- 分镜 ${item.shotIndex}: ${item.error}`);
sections.push(["### 失败分镜", ...failedLines].join("\n"));
}
return sections.join("\n\n").trim();
}
export async function runRuxaNineGridPipeline(
params: RuxaNineGridPipelineParams
): Promise<RuxaNineGridPipelineResult> {
const {
userId,
chatId,
modelMessages,
selectedModelId,
selectedModelRef,
modelCandidates,
videoParams,
config,
onProgress,
doGenerate,
} = params;
if (!config.enabled) {
return { triggered: false, reason: "disabled" };
}
const providerAllowed = config.providerWhitelist.some(
(item) => normalizeText(item).toLowerCase() === "ruxa-video"
);
if (!providerAllowed) {
return { triggered: false, reason: "provider_not_allowed" };
}
onProgress?.({
stage: "detecting",
provider: "ruxa-video",
message: "正在识别九宫格与解析分镜...",
});
const { prompt, imageUrls } = extractPromptAndImages(modelMessages);
if (!imageUrls[0]) {
return { triggered: false, reason: "missing_image" };
}
const detection = await runNineGridDetectorSkill({
imageUrl: imageUrls[0],
minConfidence: config.minGridConfidence,
});
if (!detection.isNineGrid || detection.tiles.length !== NINE_GRID_SHOT_COUNT) {
return {
triggered: false,
reason: detection.isNineGrid ? "slice_incomplete" : "not_nine_grid",
confidence: detection.confidence,
};
}
const normalizedScript = runScriptNormalizerSkill({
scriptText: prompt,
maxShots: NINE_GRID_SHOT_COUNT,
totalDurationSec: Number(videoParams?.duration || 10),
});
const mappedShots = runStoryboardMapperSkill({
script: normalizedScript,
tiles: detection.tiles,
});
const preparedShots = await runTileUpscalerSkill({
shots: mappedShots,
enableTileUpscale: config.enableTileUpscale,
sliceUpscaleMinEdge: config.sliceUpscaleMinEdge,
});
const tileSummary = detection.tiles.map((tile) => ({
index: tile.index,
imageUrl: tile.imageUrl,
width: tile.width,
height: tile.height,
}));
const shotPlanSummary = preparedShots.map((shot) => ({
shotIndex: shot.shotIndex,
tileIndex: shot.tileIndex,
tileImageUrl: shot.inputImageUrl,
prompt: shot.prompt,
action: shot.action,
camera: shot.camera,
mood: shot.mood,
durationSec: shot.durationSec,
}));
const resolvedModelCandidates = normalizeModelCandidates({
selectedModelId,
selectedModelRef,
modelCandidates,
});
const fallbackChain = resolvedModelCandidates
.map((item) => normalizeText(item.modelId))
.filter(Boolean);
const createdTask = await createVideoStoryboardTask({
userId,
chatId,
providerType: "ruxa-video",
modelRef: selectedModelRef,
sourceImageUrl: imageUrls[0],
isNineGrid: true,
detectionConfidence: detection.confidence,
normalizedScript: normalizedScript as any,
totalShots: preparedShots.length,
status: "running",
});
const taskId = createdTask.id || `ruxa-storyboard-${generateUUID()}`;
await createVideoStoryboardShotTasks({
parentTaskId: taskId,
shots: preparedShots,
fallbackChain,
});
await onProgress?.({
stage: "start",
provider: "ruxa-video",
taskId,
totalShots: preparedShots.length,
confidence: detection.confidence,
sourceImageUrl: imageUrls[0],
shots: preparedShots.map((shot) => ({
shotIndex: shot.shotIndex,
tileImageUrl: shot.inputImageUrl,
prompt: shot.prompt,
})),
});
const sequencer = await runVideoSequencerSkill({
provider: "ruxa-video",
taskId,
shots: preparedShots,
modelCandidates: resolvedModelCandidates,
maxRetry: config.maxRetry,
dispatchIntervalSec: config.dispatchIntervalSec,
enableFallback: config.enableFallback,
onProgress: async (event) => {
if (event.stage === "shot_retry") {
await markVideoStoryboardShotRetry({
taskId,
shotIndex: event.shotIndex,
errorMessage: event.error,
}).catch(() => undefined);
}
if (event.stage === "shot_completed") {
await markVideoStoryboardShotCompleted({
taskId,
shotIndex: event.shotIndex,
modelId: event.modelId,
videoUrl: event.videoUrl,
}).catch(() => undefined);
}
if (event.stage === "shot_failed") {
await markVideoStoryboardShotFailed({
taskId,
shotIndex: event.shotIndex,
errorMessage: event.error,
}).catch(() => undefined);
}
await onProgress?.(event);
},
generateOneShot: async ({ shot, model }) => {
await markVideoStoryboardShotRunning({
taskId,
shotIndex: shot.shotIndex,
modelId: model.modelId,
}).catch(() => undefined);
const shotMessages = [
{
role: "user",
content: [
{ type: "text", text: shot.prompt },
{ type: "image_url", image_url: { url: shot.inputImageUrl } },
],
},
];
const output = await doGenerate({
messages: shotMessages,
videoParams: {
...(videoParams || {}),
mode: "first-frame",
},
overrideModelId: model.modelId,
});
const text = normalizeText(output?.text);
const videoUrl = extractVideoUrlFromText(text);
if (!videoUrl) {
throw new Error(text || "未返回视频地址");
}
return { videoUrl, text };
},
});
const successShots = sequencer.results
.filter((item) => item.success && item.videoUrl)
.map((item) => ({
shotIndex: item.shotIndex,
text: normalizeText(item.text),
videoUrl: normalizeText(item.videoUrl),
modelId: normalizeText(item.modelId),
}));
const failedShots = sequencer.results
.filter((item) => !item.success)
.map((item) => ({
shotIndex: item.shotIndex,
error: normalizeText(item.error) || "video_generation_failed",
}));
if (successShots.length === 0) {
await finalizeVideoStoryboardTask({
taskId,
status: "failed",
error: "all_shots_failed",
}).catch(() => undefined);
await onProgress?.({
stage: "failed",
provider: "ruxa-video",
taskId,
error: "all_shots_failed",
});
return {
triggered: true,
taskId,
confidence: detection.confidence,
reason: "all_shots_failed",
videoUrls: [],
modelRefs: [],
markdown: "视频生成失败,请稍后重试",
metadata: {
taskId,
successCount: 0,
failedCount: failedShots.length,
confidence: detection.confidence,
tiles: tileSummary,
shotPlan: shotPlanSummary,
normalizedScript,
failedShots,
},
};
}
const finalTaskStatus =
sequencer.failedCount > 0 ? "partial_success" : "succeeded";
await finalizeVideoStoryboardTask({
taskId,
status: finalTaskStatus,
}).catch(() => undefined);
await onProgress?.({
stage: "completed",
provider: "ruxa-video",
taskId,
totalShots: sequencer.results.length,
successCount: sequencer.successCount,
failedCount: sequencer.failedCount,
});
return {
triggered: true,
taskId,
confidence: detection.confidence,
markdown: buildCombinedMarkdown({
success: successShots.map((item) => ({
shotIndex: item.shotIndex,
text: item.text,
})),
failed: failedShots,
}),
videoUrls: successShots.map((item) => item.videoUrl),
modelRefs: successShots.map((item) => item.modelId).filter(Boolean),
metadata: {
taskId,
confidence: detection.confidence,
qualityScore: detection.qualityScore,
sourceImageUrl: detection.sourceImageUrl,
tiles: tileSummary,
shotPlan: shotPlanSummary,
normalizedScript,
successCount: sequencer.successCount,
failedCount: sequencer.failedCount,
failedShots,
},
};
}