415 lines
12 KiB
TypeScript
415 lines
12 KiB
TypeScript
import "server-only";
|
|
|
|
import { generateUUID } from "@/lib/utils";
|
|
import {
|
|
createVideoStoryboardShotTasks,
|
|
createVideoStoryboardTask,
|
|
finalizeVideoStoryboardTask,
|
|
markVideoStoryboardShotCompleted,
|
|
markVideoStoryboardShotFailed,
|
|
markVideoStoryboardShotRetry,
|
|
markVideoStoryboardShotRunning,
|
|
} from "@/lib/video-agent/task-store";
|
|
import { runNineGridDetectorSkill } from "@/lib/video-agent/skills/nine-grid-detector";
|
|
import { runScriptNormalizerSkill } from "@/lib/video-agent/skills/script-normalizer";
|
|
import { runStoryboardMapperSkill } from "@/lib/video-agent/skills/storyboard-mapper";
|
|
import { runTileUpscalerSkill } from "@/lib/video-agent/skills/tile-upscaler";
|
|
import { runVideoSequencerSkill } from "@/lib/video-agent/skills/video-sequencer";
|
|
import type {
|
|
RuxaNineGridPipelineParams,
|
|
RuxaNineGridPipelineResult,
|
|
StoryboardModelCandidate,
|
|
} from "@/lib/video-agent/types";
|
|
|
|
const NINE_GRID_SHOT_COUNT = 9;
|
|
|
|
function normalizeText(value: unknown) {
|
|
return String(value || "").trim();
|
|
}
|
|
|
|
function extractVideoUrlFromText(text: string): string | null {
|
|
const normalizedText = String(text || "");
|
|
if (!normalizedText) return null;
|
|
|
|
const htmlMatch = normalizedText.match(/<video[^>]+src="([^"]+)"/i);
|
|
if (htmlMatch?.[1]) return htmlMatch[1];
|
|
|
|
const markdownMatch = normalizedText.match(
|
|
/\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i
|
|
);
|
|
if (markdownMatch?.[1]) return markdownMatch[1];
|
|
|
|
const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i);
|
|
return plainMatch?.[1] || null;
|
|
}
|
|
|
|
function extractPromptAndImages(messages: Array<{ role: string; content: any }>) {
|
|
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
|
|
if (!lastUserMessage) {
|
|
return { prompt: "", imageUrls: [] as string[] };
|
|
}
|
|
|
|
let prompt = "";
|
|
const imageUrls: string[] = [];
|
|
|
|
const pushImage = (raw?: string) => {
|
|
const url = normalizeText(raw);
|
|
if (!url || !/^https?:\/\//i.test(url)) return;
|
|
if (!imageUrls.includes(url)) imageUrls.push(url);
|
|
};
|
|
|
|
if (typeof lastUserMessage.content === "string") {
|
|
prompt = normalizeText(lastUserMessage.content);
|
|
} else if (Array.isArray(lastUserMessage.content)) {
|
|
for (const part of lastUserMessage.content) {
|
|
if (part?.type === "text" && typeof part.text === "string") {
|
|
prompt = `${prompt} ${part.text}`.trim();
|
|
} else if (part?.type === "image_url" && part.image_url?.url) {
|
|
pushImage(part.image_url.url);
|
|
} else if (part?.type === "file") {
|
|
pushImage(part.url || part.data);
|
|
} else if (part?.type === "image" && typeof part.image === "string") {
|
|
pushImage(part.image);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!prompt) {
|
|
prompt = "让画面动起来";
|
|
}
|
|
|
|
return { prompt, imageUrls };
|
|
}
|
|
|
|
function normalizeModelCandidates(params: {
|
|
selectedModelId: string;
|
|
selectedModelRef: string;
|
|
modelCandidates: StoryboardModelCandidate[];
|
|
}) {
|
|
const selected: StoryboardModelCandidate = {
|
|
id: params.selectedModelRef || params.selectedModelId,
|
|
modelId: params.selectedModelId,
|
|
displayName: "",
|
|
};
|
|
const candidates = [selected, ...params.modelCandidates]
|
|
.filter((item) => normalizeText(item.modelId))
|
|
.map((item) => ({
|
|
id: normalizeText(item.id) || normalizeText(item.modelId),
|
|
modelId: normalizeText(item.modelId),
|
|
displayName: normalizeText(item.displayName),
|
|
}));
|
|
|
|
const seen = new Set<string>();
|
|
return candidates.filter((item) => {
|
|
if (seen.has(item.modelId)) return false;
|
|
seen.add(item.modelId);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
function buildCombinedMarkdown(params: {
|
|
success: Array<{ shotIndex: number; text: string }>;
|
|
failed: Array<{ shotIndex: number; error: string }>;
|
|
}) {
|
|
const sections: string[] = [];
|
|
|
|
for (const shot of params.success.sort((a, b) => a.shotIndex - b.shotIndex)) {
|
|
sections.push(`### 分镜 ${shot.shotIndex}\n${shot.text}`);
|
|
}
|
|
|
|
if (params.failed.length > 0) {
|
|
const failedLines = params.failed
|
|
.sort((a, b) => a.shotIndex - b.shotIndex)
|
|
.map((item) => `- 分镜 ${item.shotIndex}: ${item.error}`);
|
|
sections.push(["### 失败分镜", ...failedLines].join("\n"));
|
|
}
|
|
|
|
return sections.join("\n\n").trim();
|
|
}
|
|
|
|
export async function runRuxaNineGridPipeline(
|
|
params: RuxaNineGridPipelineParams
|
|
): Promise<RuxaNineGridPipelineResult> {
|
|
const {
|
|
userId,
|
|
chatId,
|
|
modelMessages,
|
|
selectedModelId,
|
|
selectedModelRef,
|
|
modelCandidates,
|
|
videoParams,
|
|
config,
|
|
onProgress,
|
|
doGenerate,
|
|
} = params;
|
|
|
|
if (!config.enabled) {
|
|
return { triggered: false, reason: "disabled" };
|
|
}
|
|
|
|
const providerAllowed = config.providerWhitelist.some(
|
|
(item) => normalizeText(item).toLowerCase() === "ruxa-video"
|
|
);
|
|
if (!providerAllowed) {
|
|
return { triggered: false, reason: "provider_not_allowed" };
|
|
}
|
|
|
|
onProgress?.({
|
|
stage: "detecting",
|
|
provider: "ruxa-video",
|
|
message: "正在识别九宫格与解析分镜...",
|
|
});
|
|
|
|
const { prompt, imageUrls } = extractPromptAndImages(modelMessages);
|
|
if (!imageUrls[0]) {
|
|
return { triggered: false, reason: "missing_image" };
|
|
}
|
|
|
|
const detection = await runNineGridDetectorSkill({
|
|
imageUrl: imageUrls[0],
|
|
minConfidence: config.minGridConfidence,
|
|
});
|
|
|
|
if (!detection.isNineGrid || detection.tiles.length !== NINE_GRID_SHOT_COUNT) {
|
|
return {
|
|
triggered: false,
|
|
reason: detection.isNineGrid ? "slice_incomplete" : "not_nine_grid",
|
|
confidence: detection.confidence,
|
|
};
|
|
}
|
|
|
|
const normalizedScript = runScriptNormalizerSkill({
|
|
scriptText: prompt,
|
|
maxShots: NINE_GRID_SHOT_COUNT,
|
|
totalDurationSec: Number(videoParams?.duration || 10),
|
|
});
|
|
const mappedShots = runStoryboardMapperSkill({
|
|
script: normalizedScript,
|
|
tiles: detection.tiles,
|
|
});
|
|
const preparedShots = await runTileUpscalerSkill({
|
|
shots: mappedShots,
|
|
enableTileUpscale: config.enableTileUpscale,
|
|
sliceUpscaleMinEdge: config.sliceUpscaleMinEdge,
|
|
});
|
|
const tileSummary = detection.tiles.map((tile) => ({
|
|
index: tile.index,
|
|
imageUrl: tile.imageUrl,
|
|
width: tile.width,
|
|
height: tile.height,
|
|
}));
|
|
const shotPlanSummary = preparedShots.map((shot) => ({
|
|
shotIndex: shot.shotIndex,
|
|
tileIndex: shot.tileIndex,
|
|
tileImageUrl: shot.inputImageUrl,
|
|
prompt: shot.prompt,
|
|
action: shot.action,
|
|
camera: shot.camera,
|
|
mood: shot.mood,
|
|
durationSec: shot.durationSec,
|
|
}));
|
|
|
|
const resolvedModelCandidates = normalizeModelCandidates({
|
|
selectedModelId,
|
|
selectedModelRef,
|
|
modelCandidates,
|
|
});
|
|
|
|
const fallbackChain = resolvedModelCandidates
|
|
.map((item) => normalizeText(item.modelId))
|
|
.filter(Boolean);
|
|
|
|
const createdTask = await createVideoStoryboardTask({
|
|
userId,
|
|
chatId,
|
|
providerType: "ruxa-video",
|
|
modelRef: selectedModelRef,
|
|
sourceImageUrl: imageUrls[0],
|
|
isNineGrid: true,
|
|
detectionConfidence: detection.confidence,
|
|
normalizedScript: normalizedScript as any,
|
|
totalShots: preparedShots.length,
|
|
status: "running",
|
|
});
|
|
const taskId = createdTask.id || `ruxa-storyboard-${generateUUID()}`;
|
|
await createVideoStoryboardShotTasks({
|
|
parentTaskId: taskId,
|
|
shots: preparedShots,
|
|
fallbackChain,
|
|
});
|
|
|
|
await onProgress?.({
|
|
stage: "start",
|
|
provider: "ruxa-video",
|
|
taskId,
|
|
totalShots: preparedShots.length,
|
|
confidence: detection.confidence,
|
|
sourceImageUrl: imageUrls[0],
|
|
shots: preparedShots.map((shot) => ({
|
|
shotIndex: shot.shotIndex,
|
|
tileImageUrl: shot.inputImageUrl,
|
|
prompt: shot.prompt,
|
|
})),
|
|
});
|
|
|
|
const sequencer = await runVideoSequencerSkill({
|
|
provider: "ruxa-video",
|
|
taskId,
|
|
shots: preparedShots,
|
|
modelCandidates: resolvedModelCandidates,
|
|
maxRetry: config.maxRetry,
|
|
dispatchIntervalSec: config.dispatchIntervalSec,
|
|
enableFallback: config.enableFallback,
|
|
onProgress: async (event) => {
|
|
if (event.stage === "shot_retry") {
|
|
await markVideoStoryboardShotRetry({
|
|
taskId,
|
|
shotIndex: event.shotIndex,
|
|
errorMessage: event.error,
|
|
}).catch(() => undefined);
|
|
}
|
|
if (event.stage === "shot_completed") {
|
|
await markVideoStoryboardShotCompleted({
|
|
taskId,
|
|
shotIndex: event.shotIndex,
|
|
modelId: event.modelId,
|
|
videoUrl: event.videoUrl,
|
|
}).catch(() => undefined);
|
|
}
|
|
if (event.stage === "shot_failed") {
|
|
await markVideoStoryboardShotFailed({
|
|
taskId,
|
|
shotIndex: event.shotIndex,
|
|
errorMessage: event.error,
|
|
}).catch(() => undefined);
|
|
}
|
|
await onProgress?.(event);
|
|
},
|
|
generateOneShot: async ({ shot, model }) => {
|
|
await markVideoStoryboardShotRunning({
|
|
taskId,
|
|
shotIndex: shot.shotIndex,
|
|
modelId: model.modelId,
|
|
}).catch(() => undefined);
|
|
|
|
const shotMessages = [
|
|
{
|
|
role: "user",
|
|
content: [
|
|
{ type: "text", text: shot.prompt },
|
|
{ type: "image_url", image_url: { url: shot.inputImageUrl } },
|
|
],
|
|
},
|
|
];
|
|
|
|
const output = await doGenerate({
|
|
messages: shotMessages,
|
|
videoParams: {
|
|
...(videoParams || {}),
|
|
mode: "first-frame",
|
|
},
|
|
overrideModelId: model.modelId,
|
|
});
|
|
|
|
const text = normalizeText(output?.text);
|
|
const videoUrl = extractVideoUrlFromText(text);
|
|
if (!videoUrl) {
|
|
throw new Error(text || "未返回视频地址");
|
|
}
|
|
return { videoUrl, text };
|
|
},
|
|
});
|
|
|
|
const successShots = sequencer.results
|
|
.filter((item) => item.success && item.videoUrl)
|
|
.map((item) => ({
|
|
shotIndex: item.shotIndex,
|
|
text: normalizeText(item.text),
|
|
videoUrl: normalizeText(item.videoUrl),
|
|
modelId: normalizeText(item.modelId),
|
|
}));
|
|
const failedShots = sequencer.results
|
|
.filter((item) => !item.success)
|
|
.map((item) => ({
|
|
shotIndex: item.shotIndex,
|
|
error: normalizeText(item.error) || "video_generation_failed",
|
|
}));
|
|
|
|
if (successShots.length === 0) {
|
|
await finalizeVideoStoryboardTask({
|
|
taskId,
|
|
status: "failed",
|
|
error: "all_shots_failed",
|
|
}).catch(() => undefined);
|
|
|
|
await onProgress?.({
|
|
stage: "failed",
|
|
provider: "ruxa-video",
|
|
taskId,
|
|
error: "all_shots_failed",
|
|
});
|
|
return {
|
|
triggered: true,
|
|
taskId,
|
|
confidence: detection.confidence,
|
|
reason: "all_shots_failed",
|
|
videoUrls: [],
|
|
modelRefs: [],
|
|
markdown: "视频生成失败,请稍后重试",
|
|
metadata: {
|
|
taskId,
|
|
successCount: 0,
|
|
failedCount: failedShots.length,
|
|
confidence: detection.confidence,
|
|
tiles: tileSummary,
|
|
shotPlan: shotPlanSummary,
|
|
normalizedScript,
|
|
failedShots,
|
|
},
|
|
};
|
|
}
|
|
|
|
const finalTaskStatus =
|
|
sequencer.failedCount > 0 ? "partial_success" : "succeeded";
|
|
await finalizeVideoStoryboardTask({
|
|
taskId,
|
|
status: finalTaskStatus,
|
|
}).catch(() => undefined);
|
|
|
|
await onProgress?.({
|
|
stage: "completed",
|
|
provider: "ruxa-video",
|
|
taskId,
|
|
totalShots: sequencer.results.length,
|
|
successCount: sequencer.successCount,
|
|
failedCount: sequencer.failedCount,
|
|
});
|
|
|
|
return {
|
|
triggered: true,
|
|
taskId,
|
|
confidence: detection.confidence,
|
|
markdown: buildCombinedMarkdown({
|
|
success: successShots.map((item) => ({
|
|
shotIndex: item.shotIndex,
|
|
text: item.text,
|
|
})),
|
|
failed: failedShots,
|
|
}),
|
|
videoUrls: successShots.map((item) => item.videoUrl),
|
|
modelRefs: successShots.map((item) => item.modelId).filter(Boolean),
|
|
metadata: {
|
|
taskId,
|
|
confidence: detection.confidence,
|
|
qualityScore: detection.qualityScore,
|
|
sourceImageUrl: detection.sourceImageUrl,
|
|
tiles: tileSummary,
|
|
shotPlan: shotPlanSummary,
|
|
normalizedScript,
|
|
successCount: sequencer.successCount,
|
|
failedCount: sequencer.failedCount,
|
|
failedShots,
|
|
},
|
|
};
|
|
}
|