import "server-only"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { getAiModelById } from "@/lib/db/queries"; import { getVideoStoryboardConfig } from "@/lib/video-agent/config"; import { finalizeVideoStoryboardTask, getVideoStoryboardTaskDetail, markVideoStoryboardShotCompleted, markVideoStoryboardShotFailed, markVideoStoryboardShotRetry, markVideoStoryboardShotRunning, syncVideoStoryboardTaskStatus, } from "@/lib/video-agent/task-store"; import type { StoryboardProgressEvent } from "@/lib/video-agent/types"; const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); function normalizeText(value: unknown) { return String(value || "").trim(); } function extractVideoUrlFromText(text: string): string | null { const normalizedText = String(text || ""); if (!normalizedText) return null; const htmlMatch = normalizedText.match(/]+src="([^"]+)"/i); if (htmlMatch?.[1]) return htmlMatch[1]; const markdownMatch = normalizedText.match( /\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/i ); if (markdownMatch?.[1]) return markdownMatch[1]; const plainMatch = normalizedText.match(/(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/i); return plainMatch?.[1] || null; } function normalizeFallbackChain(value: unknown): string[] { if (!Array.isArray(value)) return []; return Array.from( new Set( value .map((item) => String(item || "").trim()) .filter(Boolean) ) ); } function buildCombinedMarkdown(params: { success: Array<{ shotIndex: number; text: string }>; failed: Array<{ shotIndex: number; error: string }>; }) { const sections: string[] = []; for (const shot of params.success.sort((a, b) => a.shotIndex - b.shotIndex)) { sections.push(`### 分镜 ${shot.shotIndex}\n${shot.text}`); } if (params.failed.length > 0) { const failedLines = params.failed .sort((a, b) => a.shotIndex - b.shotIndex) .map((item) => `- 分镜 ${item.shotIndex}: ${item.error}`); sections.push(["### 失败分镜", ...failedLines].join("\n")); } return sections.join("\n\n").trim(); } export async function executeConfirmedStoryboardTask(params: { taskId: string; onProgress?: (event: StoryboardProgressEvent) => void | Promise; }) { const detail = await getVideoStoryboardTaskDetail(params.taskId); if (!detail) { throw new Error("分镜任务不存在"); } if (detail.task.providerType !== "ruxa-video") { throw new Error("当前仅支持执行 ruxa-video 分镜任务"); } const modelRef = normalizeText(detail.task.modelRef); if (!modelRef) { throw new Error("任务缺少 modelRef,无法执行"); } const selectedModel = await getAiModelById({ id: modelRef }).catch(() => undefined); const selectedModelId = normalizeText(selectedModel?.modelId); const languageModel = await getLanguageModelById(modelRef); if (!languageModel || typeof (languageModel as any).doGenerate !== "function") { throw new Error("模型不可用或不支持视频执行"); } const config = await getVideoStoryboardConfig().catch(() => null); const dispatchIntervalSec = Math.max(1, Number(config?.dispatchIntervalSec || 10)); const maxRetry = Math.max(0, Number(config?.maxRetry || 3)); const enableFallback = config?.enableFallback !== false; await finalizeVideoStoryboardTask({ taskId: detail.task.id, status: "running", }); const targetShots = detail.shots.filter((shot: any) => { const currentStatus = normalizeText(shot.videoStatus || shot.status); return currentStatus !== "completed" && currentStatus !== "cancelled"; }); const results: Array<{ shotIndex: number; success: boolean; videoUrl?: string; modelId?: string; text?: string; error?: string; }> = []; for (let index = 0; index < targetShots.length; index += 1) { const shot = targetShots[index]!; const shotIndex = Math.max(1, Number(shot.shotIndex || 0)); const prompt = normalizeText(shot.prompt) || "让画面动起来"; const inputImageUrl = normalizeText(shot.hdImageUrl) || normalizeText(shot.upscaledImageUrl) || normalizeText(shot.tileImageUrl); if (!shotIndex || !inputImageUrl) { const error = "缺少镜头输入图,无法生成视频"; await markVideoStoryboardShotFailed({ taskId: detail.task.id, shotIndex, errorMessage: error, }); results.push({ shotIndex, success: false, error }); await params.onProgress?.({ stage: "shot_failed", provider: "ruxa-video", taskId: detail.task.id, shotIndex, error, }); continue; } const fallbackChain = normalizeFallbackChain(shot.fallbackChain); const candidateModelIds = Array.from( new Set( [normalizeText(shot.modelId), ...fallbackChain, selectedModelId].filter(Boolean) ) ); const modelPool = enableFallback && candidateModelIds.length > 0 ? candidateModelIds : candidateModelIds.slice(0, 1); let succeeded = false; let lastError = ""; let usedModelId = ""; let resultText = ""; let resultUrl = ""; for (const candidateModelId of modelPool) { for (let attempt = 1; attempt <= maxRetry + 1; attempt += 1) { try { await markVideoStoryboardShotRunning({ taskId: detail.task.id, shotIndex, modelId: candidateModelId, }); const result = await (languageModel as any).doGenerate({ messages: [ { role: "user", content: [ { type: "text", text: prompt }, { type: "image_url", image_url: { url: inputImageUrl } }, ], }, ], videoParams: { mode: "first-frame" }, overrideModelId: candidateModelId, }); const text = normalizeText(result?.text); const videoUrl = extractVideoUrlFromText(text); if (!videoUrl) { throw new Error(text || "未返回视频地址"); } await markVideoStoryboardShotCompleted({ taskId: detail.task.id, shotIndex, modelId: candidateModelId, videoUrl, }); succeeded = true; usedModelId = candidateModelId; resultText = text; resultUrl = videoUrl; break; } catch (error) { lastError = error instanceof Error ? error.message : String(error); await markVideoStoryboardShotRetry({ taskId: detail.task.id, shotIndex, errorMessage: lastError, }); if (attempt <= maxRetry) { await params.onProgress?.({ stage: "shot_retry", provider: "ruxa-video", taskId: detail.task.id, shotIndex, attempt, modelId: candidateModelId, error: lastError, }); const backoffSec = Math.min(40, Math.pow(2, attempt - 1) * 10); await sleep(backoffSec * 1000); } } } if (succeeded) break; } if (succeeded) { results.push({ shotIndex, success: true, videoUrl: resultUrl, modelId: usedModelId, text: resultText, }); await params.onProgress?.({ stage: "shot_completed", provider: "ruxa-video", taskId: detail.task.id, shotIndex, modelId: usedModelId, videoUrl: resultUrl, text: resultText, }); } else { await markVideoStoryboardShotFailed({ taskId: detail.task.id, shotIndex, errorMessage: lastError || "视频生成失败", }); results.push({ shotIndex, success: false, error: lastError || "视频生成失败", }); await params.onProgress?.({ stage: "shot_failed", provider: "ruxa-video", taskId: detail.task.id, shotIndex, error: lastError || "视频生成失败", }); } if (index < targetShots.length - 1 && dispatchIntervalSec > 0) { await sleep(dispatchIntervalSec * 1000); } } const summary = await syncVideoStoryboardTaskStatus(detail.task.id); await params.onProgress?.({ stage: "completed", provider: "ruxa-video", taskId: detail.task.id, totalShots: results.length, successCount: summary.success, failedCount: summary.failed, }); return { taskId: detail.task.id, videoUrls: results.filter((item) => item.success && item.videoUrl).map((item) => item.videoUrl!), modelRefs: results.filter((item) => item.success && item.modelId).map((item) => item.modelId!), markdown: buildCombinedMarkdown({ success: results .filter((item) => item.success) .map((item) => ({ shotIndex: item.shotIndex, text: normalizeText(item.text), })), failed: results .filter((item) => !item.success) .map((item) => ({ shotIndex: item.shotIndex, error: normalizeText(item.error) || "video_generation_failed", })), }), successCount: summary.success, failedCount: summary.failed, status: summary.status, }; }