import "server-only"; import { generateCanvasImage } from "@/lib/canvas/generate-image"; import { generateUUID } from "@/lib/utils"; import { createVideoStoryboardShotTasks, createVideoStoryboardTask, markVideoStoryboardAwaitingConfirmation, markVideoStoryboardHdCompleted, markVideoStoryboardHdFailed, markVideoStoryboardHdRunning, syncVideoStoryboardHdStageStatus, } from "@/lib/video-agent/task-store"; import { runNineGridDetectorSkill } from "@/lib/video-agent/skills/nine-grid-detector"; import { runScriptNormalizerSkill } from "@/lib/video-agent/skills/script-normalizer"; import { runStoryboardMapperSkill } from "@/lib/video-agent/skills/storyboard-mapper"; import { runTileUpscalerSkill } from "@/lib/video-agent/skills/tile-upscaler"; import type { RuxaNineGridPipelineParams, RuxaNineGridPipelineResult, StoryboardGridType, StoryboardModelCandidate, } from "@/lib/video-agent/types"; function normalizeText(value: unknown) { return String(value || "").trim(); } function extractPromptAndImages(messages: Array<{ role: string; content: any }>) { const lastUserMessage = messages.filter((message) => message.role === "user").pop(); if (!lastUserMessage) { return { prompt: "", imageUrls: [] as string[] }; } let prompt = ""; const imageUrls: string[] = []; const pushImage = (raw?: string) => { const url = normalizeText(raw); if (!url || !/^https?:\/\//i.test(url)) return; if (!imageUrls.includes(url)) imageUrls.push(url); }; if (typeof lastUserMessage.content === "string") { prompt = normalizeText(lastUserMessage.content); } else if (Array.isArray(lastUserMessage.content)) { for (const part of lastUserMessage.content) { if (part?.type === "text" && typeof part.text === "string") { prompt = `${prompt} ${part.text}`.trim(); } else if (part?.type === "image_url" && part.image_url?.url) { pushImage(part.image_url.url); } else if (part?.type === "file") { pushImage(part.url || part.data); } else if (part?.type === "image" && typeof part.image === "string") { pushImage(part.image); } } } if (!prompt) { prompt = "让画面动起来"; } return { prompt, imageUrls }; } function normalizeModelCandidates(params: { selectedModelId: string; selectedModelRef: string; modelCandidates: StoryboardModelCandidate[]; }) { const selected: StoryboardModelCandidate = { id: params.selectedModelRef || params.selectedModelId, modelId: params.selectedModelId, displayName: "", }; const candidates = [selected, ...params.modelCandidates] .filter((item) => normalizeText(item.modelId)) .map((item) => ({ id: normalizeText(item.id) || normalizeText(item.modelId), modelId: normalizeText(item.modelId), displayName: normalizeText(item.displayName), })); const seen = new Set(); return candidates.filter((item) => { if (seen.has(item.modelId)) return false; seen.add(item.modelId); return true; }); } function buildAwaitingConfirmationMarkdown(params: { shots: Array<{ shotIndex: number; prompt: string; }>; }) { const lines = [ "### 分镜草案已准备完成", "我已经根据当前宫格切片生成了每段镜头的预览图,请先确认镜头内容。", "", ...params.shots.map( (shot) => `- 镜头 ${shot.shotIndex}:${normalizeText(shot.prompt) || "待补充分镜文案"}` ), "", "如果需要修改,请直接回复:", "- 修改第3镜头:……", "- 调整第5镜头动作:……", "- 重生第2张高清图", "", "如果确认无误,请回复:", "- 确认开始生成视频", ]; return lines.join("\n").trim(); } function inferPreviewAspectRatio(width?: number, height?: number): string { const safeWidth = Math.max(1, Number(width || 0)); const safeHeight = Math.max(1, Number(height || 0)); const ratio = safeWidth / safeHeight; if (ratio > 1.35) return "16:9"; if (ratio < 0.8) return "9:16"; return "1:1"; } function getGridLabel(gridType: StoryboardGridType | null, shotCount: number) { switch (gridType) { case "grid_2x2": return "4宫格"; case "grid_3x2": return "6宫格"; case "grid_3x3": return "9宫格"; default: return `${shotCount}宫格`; } } export async function runRuxaNineGridPipeline( params: RuxaNineGridPipelineParams ): Promise { const { userId, chatId, modelMessages, selectedModelId, selectedModelRef, modelCandidates, videoParams, config, onProgress, } = params; if (!config.enabled) { return { triggered: false, reason: "disabled" }; } const providerAllowed = config.providerWhitelist.some( (item) => normalizeText(item).toLowerCase() === "ruxa-video" ); if (!providerAllowed) { return { triggered: false, reason: "provider_not_allowed" }; } onProgress?.({ stage: "detecting", provider: "ruxa-video", message: "正在识别分镜拼图与解析分镜...", }); const { prompt, imageUrls } = extractPromptAndImages(modelMessages); if (!imageUrls[0]) { return { triggered: false, reason: "missing_image" }; } const detection = await runNineGridDetectorSkill({ imageUrl: imageUrls[0], minConfidence: config.minGridConfidence, }); if (!detection.isStoryboardGrid || detection.tiles.length !== detection.shotCount) { return { triggered: false, reason: detection.isStoryboardGrid ? "slice_incomplete" : "not_storyboard_grid", confidence: detection.confidence, }; } const normalizedScript = runScriptNormalizerSkill({ scriptText: prompt, maxShots: detection.shotCount, totalDurationSec: Number(videoParams?.duration || 10), }); const mappedShots = runStoryboardMapperSkill({ script: normalizedScript, tiles: detection.tiles, }); const preparedShots = await runTileUpscalerSkill({ shots: mappedShots, enableTileUpscale: config.enableTileUpscale, sliceUpscaleMinEdge: config.sliceUpscaleMinEdge, }); const tileSummary = detection.tiles.map((tile) => ({ index: tile.index, imageUrl: tile.imageUrl, width: tile.width, height: tile.height, })); const tileMetaByIndex = new Map( detection.tiles.map((tile) => [ tile.index, { width: tile.width, height: tile.height, }, ]) ); const shotPlanSummary = preparedShots.map((shot) => ({ shotIndex: shot.shotIndex, tileIndex: shot.tileIndex, tileImageUrl: shot.inputImageUrl, prompt: shot.prompt, action: shot.action, camera: shot.camera, mood: shot.mood, durationSec: shot.durationSec, })); const resolvedModelCandidates = normalizeModelCandidates({ selectedModelId, selectedModelRef, modelCandidates, }); const fallbackChain = resolvedModelCandidates .map((item) => normalizeText(item.modelId)) .filter(Boolean); const createdTask = await createVideoStoryboardTask({ userId, chatId, providerType: "ruxa-video", modelRef: selectedModelRef, gridType: detection.gridType, gridRows: detection.rows, gridCols: detection.cols, sourceImageUrl: imageUrls[0], isNineGrid: detection.gridType === "grid_3x3", detectionConfidence: detection.confidence, normalizedScript: normalizedScript as any, totalShots: preparedShots.length, status: "running", }); const taskId = createdTask.id || `ruxa-storyboard-${generateUUID()}`; await createVideoStoryboardShotTasks({ parentTaskId: taskId, shots: preparedShots, fallbackChain, }); await onProgress?.({ stage: "start", provider: "ruxa-video", taskId, totalShots: preparedShots.length, gridType: detection.gridType || undefined, rows: detection.rows, cols: detection.cols, confidence: detection.confidence, sourceImageUrl: imageUrls[0], shots: preparedShots.map((shot) => ({ shotIndex: shot.shotIndex, tileImageUrl: shot.inputImageUrl, prompt: shot.prompt, })), }); const previewShots: Array<{ shotIndex: number; tileImageUrl: string; previewImageUrl: string; prompt: string; fallbackUsed: boolean; }> = []; for (const shot of preparedShots) { await markVideoStoryboardHdRunning({ taskId, shotIndex: shot.shotIndex, }).catch(() => undefined); try { const tileMeta = tileMetaByIndex.get(shot.tileIndex); const hdResult = await generateCanvasImage({ prompt: shot.prompt, referenceImage: shot.inputImageUrl, imageCount: 1, imageSize: "2K", aspectRatio: inferPreviewAspectRatio(tileMeta?.width, tileMeta?.height), }); const previewImageUrl = hdResult.images[0]?.originalUrl || hdResult.images[0]?.thumbnailUrl || shot.inputImageUrl; await markVideoStoryboardHdCompleted({ taskId, shotIndex: shot.shotIndex, hdImageUrl: previewImageUrl, hdImagePrompt: shot.prompt, }).catch(() => undefined); previewShots.push({ shotIndex: shot.shotIndex, tileImageUrl: shot.tileImageUrl, previewImageUrl, prompt: shot.prompt, fallbackUsed: false, }); await onProgress?.({ stage: "hd_shot_completed", provider: "ruxa-video", taskId, shotIndex: shot.shotIndex, imageUrl: previewImageUrl, prompt: shot.prompt, }); } catch (error) { const errorMessage = normalizeText(error instanceof Error ? error.message : String(error)) || "hd_image_generation_failed"; await markVideoStoryboardHdFailed({ taskId, shotIndex: shot.shotIndex, errorMessage, }).catch(() => undefined); previewShots.push({ shotIndex: shot.shotIndex, tileImageUrl: shot.tileImageUrl, previewImageUrl: shot.inputImageUrl, prompt: shot.prompt, fallbackUsed: true, }); await onProgress?.({ stage: "hd_shot_failed", provider: "ruxa-video", taskId, shotIndex: shot.shotIndex, error: errorMessage, }); } } await syncVideoStoryboardHdStageStatus(taskId).catch(() => undefined); await markVideoStoryboardAwaitingConfirmation({ taskId, }).catch(() => undefined); const awaitingConfirmationMessage = buildAwaitingConfirmationMarkdown({ shots: previewShots.map((shot) => ({ shotIndex: shot.shotIndex, prompt: shot.prompt, })), }).replace( "我已经根据当前宫格切片生成了每段镜头的预览图,请先确认镜头内容。", `我已经根据当前${getGridLabel(detection.gridType, detection.shotCount)}切片生成了每段镜头的预览图,请先确认镜头内容。` ); await onProgress?.({ stage: "awaiting_confirmation", provider: "ruxa-video", taskId, totalShots: previewShots.length, gridType: detection.gridType || undefined, rows: detection.rows, cols: detection.cols, message: awaitingConfirmationMessage, shots: previewShots.map((shot) => ({ shotIndex: shot.shotIndex, tileImageUrl: shot.tileImageUrl, previewImageUrl: shot.previewImageUrl, prompt: shot.prompt, })), }); return { triggered: true, taskId, confidence: detection.confidence, markdown: awaitingConfirmationMessage, videoUrls: [], previewImageUrls: previewShots.map((shot) => shot.previewImageUrl), modelRefs: [], metadata: { taskId, confidence: detection.confidence, qualityScore: detection.qualityScore, sourceImageUrl: detection.sourceImageUrl, gridType: detection.gridType, rows: detection.rows, cols: detection.cols, shotCount: detection.shotCount, tiles: tileSummary, shotPlan: shotPlanSummary, normalizedScript, awaitingConfirmation: true, previewShots, }, reason: "awaiting_user_confirmation", }; }