305 lines
9.2 KiB
TypeScript
305 lines
9.2 KiB
TypeScript
import "server-only";
|
||
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { getAiModelById } from "@/lib/db/queries";
|
||
import { getVideoStoryboardConfig } from "@/lib/video-agent/config";
|
||
import {
|
||
finalizeVideoStoryboardTask,
|
||
getVideoStoryboardTaskDetail,
|
||
markVideoStoryboardShotCompleted,
|
||
markVideoStoryboardShotFailed,
|
||
markVideoStoryboardShotRetry,
|
||
markVideoStoryboardShotRunning,
|
||
syncVideoStoryboardTaskStatus,
|
||
} from "@/lib/video-agent/task-store";
|
||
import type { StoryboardProgressEvent } from "@/lib/video-agent/types";
|
||
|
||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||
|
||
function normalizeText(value: unknown) {
|
||
return String(value || "").trim();
|
||
}
|
||
|
||
function extractVideoUrlFromText(text: string): string | null {
|
||
const normalizedText = String(text || "");
|
||
if (!normalizedText) return null;
|
||
|
||
const htmlMatch = normalizedText.match(/<video[^>]+src="([^"]+)"/i);
|
||
if (htmlMatch?.[1]) return htmlMatch[1];
|
||
|
||
const markdownMatch = normalizedText.match(
|
||
/\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i
|
||
);
|
||
if (markdownMatch?.[1]) return markdownMatch[1];
|
||
|
||
const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i);
|
||
return plainMatch?.[1] || null;
|
||
}
|
||
|
||
function normalizeFallbackChain(value: unknown): string[] {
|
||
if (!Array.isArray(value)) return [];
|
||
return Array.from(
|
||
new Set(
|
||
value
|
||
.map((item) => String(item || "").trim())
|
||
.filter(Boolean)
|
||
)
|
||
);
|
||
}
|
||
|
||
function buildCombinedMarkdown(params: {
|
||
success: Array<{ shotIndex: number; text: string }>;
|
||
failed: Array<{ shotIndex: number; error: string }>;
|
||
}) {
|
||
const sections: string[] = [];
|
||
|
||
for (const shot of params.success.sort((a, b) => a.shotIndex - b.shotIndex)) {
|
||
sections.push(`### 分镜 ${shot.shotIndex}\n${shot.text}`);
|
||
}
|
||
|
||
if (params.failed.length > 0) {
|
||
const failedLines = params.failed
|
||
.sort((a, b) => a.shotIndex - b.shotIndex)
|
||
.map((item) => `- 分镜 ${item.shotIndex}: ${item.error}`);
|
||
sections.push(["### 失败分镜", ...failedLines].join("\n"));
|
||
}
|
||
|
||
return sections.join("\n\n").trim();
|
||
}
|
||
|
||
export async function executeConfirmedStoryboardTask(params: {
|
||
taskId: string;
|
||
onProgress?: (event: StoryboardProgressEvent) => void | Promise<void>;
|
||
}) {
|
||
const detail = await getVideoStoryboardTaskDetail(params.taskId);
|
||
if (!detail) {
|
||
throw new Error("分镜任务不存在");
|
||
}
|
||
|
||
if (detail.task.providerType !== "ruxa-video") {
|
||
throw new Error("当前仅支持执行 ruxa-video 分镜任务");
|
||
}
|
||
|
||
const modelRef = normalizeText(detail.task.modelRef);
|
||
if (!modelRef) {
|
||
throw new Error("任务缺少 modelRef，无法执行");
|
||
}
|
||
|
||
const selectedModel = await getAiModelById({ id: modelRef }).catch(() => undefined);
|
||
const selectedModelId = normalizeText(selectedModel?.modelId);
|
||
const languageModel = await getLanguageModelById(modelRef);
|
||
if (!languageModel || typeof (languageModel as any).doGenerate !== "function") {
|
||
throw new Error("模型不可用或不支持视频执行");
|
||
}
|
||
|
||
const config = await getVideoStoryboardConfig().catch(() => null);
|
||
const dispatchIntervalSec = Math.max(1, Number(config?.dispatchIntervalSec || 10));
|
||
const maxRetry = Math.max(0, Number(config?.maxRetry || 3));
|
||
const enableFallback = config?.enableFallback !== false;
|
||
|
||
await finalizeVideoStoryboardTask({
|
||
taskId: detail.task.id,
|
||
status: "running",
|
||
});
|
||
|
||
const targetShots = detail.shots.filter((shot: any) => {
|
||
const currentStatus = normalizeText(shot.videoStatus || shot.status);
|
||
return currentStatus !== "completed" && currentStatus !== "cancelled";
|
||
});
|
||
|
||
const results: Array<{
|
||
shotIndex: number;
|
||
success: boolean;
|
||
videoUrl?: string;
|
||
modelId?: string;
|
||
text?: string;
|
||
error?: string;
|
||
}> = [];
|
||
|
||
for (let index = 0; index < targetShots.length; index += 1) {
|
||
const shot = targetShots[index]!;
|
||
const shotIndex = Math.max(1, Number(shot.shotIndex || 0));
|
||
const prompt = normalizeText(shot.prompt) || "让画面动起来";
|
||
const inputImageUrl =
|
||
normalizeText(shot.hdImageUrl) ||
|
||
normalizeText(shot.upscaledImageUrl) ||
|
||
normalizeText(shot.tileImageUrl);
|
||
|
||
if (!shotIndex || !inputImageUrl) {
|
||
const error = "缺少镜头输入图，无法生成视频";
|
||
await markVideoStoryboardShotFailed({
|
||
taskId: detail.task.id,
|
||
shotIndex,
|
||
errorMessage: error,
|
||
});
|
||
results.push({ shotIndex, success: false, error });
|
||
await params.onProgress?.({
|
||
stage: "shot_failed",
|
||
provider: "ruxa-video",
|
||
taskId: detail.task.id,
|
||
shotIndex,
|
||
error,
|
||
});
|
||
continue;
|
||
}
|
||
|
||
const fallbackChain = normalizeFallbackChain(shot.fallbackChain);
|
||
const candidateModelIds = Array.from(
|
||
new Set(
|
||
[normalizeText(shot.modelId), ...fallbackChain, selectedModelId].filter(Boolean)
|
||
)
|
||
);
|
||
const modelPool =
|
||
enableFallback && candidateModelIds.length > 0
|
||
? candidateModelIds
|
||
: candidateModelIds.slice(0, 1);
|
||
|
||
let succeeded = false;
|
||
let lastError = "";
|
||
let usedModelId = "";
|
||
let resultText = "";
|
||
let resultUrl = "";
|
||
|
||
for (const candidateModelId of modelPool) {
|
||
for (let attempt = 1; attempt <= maxRetry + 1; attempt += 1) {
|
||
try {
|
||
await markVideoStoryboardShotRunning({
|
||
taskId: detail.task.id,
|
||
shotIndex,
|
||
modelId: candidateModelId,
|
||
});
|
||
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{ type: "text", text: prompt },
|
||
{ type: "image_url", image_url: { url: inputImageUrl } },
|
||
],
|
||
},
|
||
],
|
||
videoParams: { mode: "first-frame" },
|
||
overrideModelId: candidateModelId,
|
||
});
|
||
|
||
const text = normalizeText(result?.text);
|
||
const videoUrl = extractVideoUrlFromText(text);
|
||
if (!videoUrl) {
|
||
throw new Error(text || "未返回视频地址");
|
||
}
|
||
|
||
await markVideoStoryboardShotCompleted({
|
||
taskId: detail.task.id,
|
||
shotIndex,
|
||
modelId: candidateModelId,
|
||
videoUrl,
|
||
});
|
||
succeeded = true;
|
||
usedModelId = candidateModelId;
|
||
resultText = text;
|
||
resultUrl = videoUrl;
|
||
break;
|
||
} catch (error) {
|
||
lastError = error instanceof Error ? error.message : String(error);
|
||
await markVideoStoryboardShotRetry({
|
||
taskId: detail.task.id,
|
||
shotIndex,
|
||
errorMessage: lastError,
|
||
});
|
||
if (attempt <= maxRetry) {
|
||
await params.onProgress?.({
|
||
stage: "shot_retry",
|
||
provider: "ruxa-video",
|
||
taskId: detail.task.id,
|
||
shotIndex,
|
||
attempt,
|
||
modelId: candidateModelId,
|
||
error: lastError,
|
||
});
|
||
const backoffSec = Math.min(40, Math.pow(2, attempt - 1) * 10);
|
||
await sleep(backoffSec * 1000);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (succeeded) break;
|
||
}
|
||
|
||
if (succeeded) {
|
||
results.push({
|
||
shotIndex,
|
||
success: true,
|
||
videoUrl: resultUrl,
|
||
modelId: usedModelId,
|
||
text: resultText,
|
||
});
|
||
await params.onProgress?.({
|
||
stage: "shot_completed",
|
||
provider: "ruxa-video",
|
||
taskId: detail.task.id,
|
||
shotIndex,
|
||
modelId: usedModelId,
|
||
videoUrl: resultUrl,
|
||
text: resultText,
|
||
});
|
||
} else {
|
||
await markVideoStoryboardShotFailed({
|
||
taskId: detail.task.id,
|
||
shotIndex,
|
||
errorMessage: lastError || "视频生成失败",
|
||
});
|
||
results.push({
|
||
shotIndex,
|
||
success: false,
|
||
error: lastError || "视频生成失败",
|
||
});
|
||
await params.onProgress?.({
|
||
stage: "shot_failed",
|
||
provider: "ruxa-video",
|
||
taskId: detail.task.id,
|
||
shotIndex,
|
||
error: lastError || "视频生成失败",
|
||
});
|
||
}
|
||
|
||
if (index < targetShots.length - 1 && dispatchIntervalSec > 0) {
|
||
await sleep(dispatchIntervalSec * 1000);
|
||
}
|
||
}
|
||
|
||
const summary = await syncVideoStoryboardTaskStatus(detail.task.id);
|
||
|
||
await params.onProgress?.({
|
||
stage: "completed",
|
||
provider: "ruxa-video",
|
||
taskId: detail.task.id,
|
||
totalShots: results.length,
|
||
successCount: summary.success,
|
||
failedCount: summary.failed,
|
||
});
|
||
|
||
return {
|
||
taskId: detail.task.id,
|
||
videoUrls: results.filter((item) => item.success && item.videoUrl).map((item) => item.videoUrl!),
|
||
modelRefs: results.filter((item) => item.success && item.modelId).map((item) => item.modelId!),
|
||
markdown: buildCombinedMarkdown({
|
||
success: results
|
||
.filter((item) => item.success)
|
||
.map((item) => ({
|
||
shotIndex: item.shotIndex,
|
||
text: normalizeText(item.text),
|
||
})),
|
||
failed: results
|
||
.filter((item) => !item.success)
|
||
.map((item) => ({
|
||
shotIndex: item.shotIndex,
|
||
error: normalizeText(item.error) || "video_generation_failed",
|
||
})),
|
||
}),
|
||
successCount: summary.success,
|
||
failedCount: summary.failed,
|
||
status: summary.status,
|
||
};
|
||
}
|