diff --git a/app/api/canvas/workflow/analyze/route.ts b/app/api/canvas/workflow/analyze/route.ts index 4730a0306..814195ab8 100644 --- a/app/api/canvas/workflow/analyze/route.ts +++ b/app/api/canvas/workflow/analyze/route.ts @@ -1,5 +1,11 @@ import { generateText } from "ai"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import { NextResponse } from "next/server"; +import sharp from "sharp"; import { z } from "zod"; import { auth } from "@/app/(auth)/auth"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; @@ -13,6 +19,11 @@ import { export const runtime = "nodejs"; export const maxDuration = 180; +const execFileAsync = promisify(execFile); +const VIDEO_ANALYSIS_FRAME_COUNT = 4; +const VIDEO_ANALYSIS_MAX_SIDE = 960; +const VIDEO_ANALYSIS_TIMEOUT_MS = 15000; + const requestSchema = z.object({ mode: z.enum(["video-analysis", "character-scene-extraction", "agent-plan"]), prompt: z.string().trim().optional(), @@ -93,6 +104,72 @@ function normalizeShotIdeas(value: unknown) { .slice(0, 6); } +function normalizeVideoAnalysisRows( + value: unknown, + framePreviews: string[], +) { + if (!Array.isArray(value)) return []; + return value + .map((item, index) => { + const raw = + item && typeof item === "object" && !Array.isArray(item) + ? (item as Record) + : {}; + const sceneDescription = normalizeText( + raw.sceneDescription || raw.scene || raw.title || raw.description, + ); + const backgroundMusic = normalizeText( + raw.backgroundMusic || raw.music || raw.bgm, + ); + const voiceAndSfx = normalizeText( + raw.voiceAndSfx || raw.sound || raw.audio || raw.sfx, + ); + const imagePrompt = normalizeText( + raw.imagePrompt || raw.image_generation_prompt || raw.image, + ); + const motionPrompt = normalizeText( + raw.motionPrompt || raw.motion || raw.videoPrompt || raw.video_motion_prompt, + ); + const keyframeIndex = Number( + raw.keyframeIndex || raw.frameIndex || raw.key_frame_index || index + 1, + ); + if ( + !sceneDescription && + !backgroundMusic && + !voiceAndSfx && + !imagePrompt && + !motionPrompt + ) { + return null; + } + const normalizedFrameIndex = + Number.isFinite(keyframeIndex) && keyframeIndex > 0 + ? Math.max(0, Math.min(framePreviews.length - 1, Math.round(keyframeIndex) - 1)) + : index % Math.max(framePreviews.length, 1); + return { + sceneDescription: sceneDescription || `镜头 ${index + 1}`, + backgroundMusic: backgroundMusic || "无", + voiceAndSfx: voiceAndSfx || "无", + imagePrompt: imagePrompt || sceneDescription || "延续当前视频气质生成关键帧", + motionPrompt: motionPrompt || "保持镜头运动自然、连贯、符合前后节奏。", + keyframeUrl: framePreviews[normalizedFrameIndex] || "", + }; + }) + .filter( + ( + item, + ): item is { + sceneDescription: string; + backgroundMusic: string; + voiceAndSfx: string; + imagePrompt: string; + motionPrompt: string; + keyframeUrl: string; + } => Boolean(item), + ) + .slice(0, 8); +} + function normalizeCharacters(value: unknown) { if (!Array.isArray(value)) return []; return value @@ -147,6 +224,7 @@ function normalizeScenes(value: unknown) { function buildVideoAnalysisPrompt(params: { nodeTitle?: string; prompt: string; + frameCount: number; imageCount: number; videoCount: number; }) { @@ -155,6 +233,7 @@ function buildVideoAnalysisPrompt(params: { params.nodeTitle ? `当前节点：${params.nodeTitle}` : "", `上游参考图数量：${params.imageCount}`, `上游视频数量：${params.videoCount}`, + `已提供按时间顺序抽取的视频关键帧数量：${params.frameCount}`, params.prompt ? `上游脚本/说明：\n${params.prompt}` : "上游没有额外脚本，请重点依据参考画面分析。", "", "请只输出 JSON，不要输出 markdown，不要输出解释。", @@ -162,18 +241,130 @@ function buildVideoAnalysisPrompt(params: { `{ "summary": "一句话总结当前视频/镜头风格，不超过120字", "highlights": ["3-5条镜头分析要点"], + "analysisRows": [ + { + "sceneDescription": "镜头内容与景别描述", + "backgroundMusic": "这一段适合的背景音乐氛围", + "voiceAndSfx": "人声/环境声/音效建议，没有就写无", + "imagePrompt": "后续生成关键帧时可直接使用的图像提示词", + "motionPrompt": "后续生成视频时可直接使用的运动提示词", + "keyframeIndex": 1 + } + ], "shotIdeas": [ { "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" } ], "visualMotifs": ["2-4个可复用的视觉母题"], "recommendedNextSteps": ["2-4条建议的下游动作"] }`, + "analysisRows 请尽量输出 2-6 行；keyframeIndex 使用 1 开始的序号，对应你看到的关键帧顺序。", "如果没有真实视频帧，只能依据参考图和脚本推断，也要正常输出，但措辞要稳健。", ] .filter(Boolean) .join("\n"); } +async function getVideoDurationSeconds(videoUrl: string) { + try { + const { stdout } = await execFileAsync( + "ffprobe", + [ + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + videoUrl, + ], + { + timeout: VIDEO_ANALYSIS_TIMEOUT_MS, + maxBuffer: 1024 * 1024, + }, + ); + const duration = Number.parseFloat(String(stdout || "").trim()); + return Number.isFinite(duration) && duration > 0 ? duration : null; + } catch { + return null; + } +} + +function buildVideoFrameTimestamps(durationSeconds: number | null, frameCount: number) { + const safeCount = Math.max(1, frameCount); + if (!durationSeconds || !Number.isFinite(durationSeconds) || durationSeconds <= 0.5) { + return Array.from({ length: safeCount }).map((_, index) => + Number((0.3 + index * 0.45).toFixed(2)), + ); + } + const start = Math.min(0.35, durationSeconds * 0.12); + const end = Math.max(start + 0.6, durationSeconds * 0.88); + if (safeCount === 1) return [Number(((start + end) / 2).toFixed(2))]; + const step = (end - start) / (safeCount - 1); + return Array.from({ length: safeCount }).map((_, index) => + Number((start + step * index).toFixed(2)), + ); +} + +async function downscaleVideoFrameForVision(inputBuffer: Buffer) { + return sharp(inputBuffer, { failOn: "none" }) + .rotate() + .resize({ + width: VIDEO_ANALYSIS_MAX_SIDE, + height: VIDEO_ANALYSIS_MAX_SIDE, + fit: "inside", + withoutEnlargement: true, + }) + .jpeg({ quality: 82, mozjpeg: true }) + .toBuffer(); +} + +async function extractVideoFrameDataUrls(videoUrl: string, frameCount = VIDEO_ANALYSIS_FRAME_COUNT) { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "canvas-video-analysis-")); + try { + const durationSeconds = await getVideoDurationSeconds(videoUrl); + const timestamps = buildVideoFrameTimestamps(durationSeconds, frameCount); + const frames = await Promise.all( + timestamps.map(async (timestamp, index) => { + const outputPath = path.join(tempDir, `frame-${index + 1}.jpg`); + await execFileAsync( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-ss", + String(timestamp), + "-i", + videoUrl, + "-frames:v", + "1", + "-q:v", + "4", + "-y", + outputPath, + ], + { + timeout: VIDEO_ANALYSIS_TIMEOUT_MS, + maxBuffer: 2 * 1024 * 1024, + }, + ); + const buffer = await readFile(outputPath); + const processed = await downscaleVideoFrameForVision(buffer); + return `data:image/jpeg;base64,${processed.toString("base64")}`; + }), + ); + return frames.filter(Boolean); + } catch (error) { + console.warn("[Canvas Workflow Analyze] extract video frames failed:", { + message: error instanceof Error ? error.message : String(error), + videoUrl, + }); + return []; + } finally { + await rm(tempDir, { force: true, recursive: true }).catch(() => undefined); + } +} + function buildCharacterScenePrompt(params: { nodeTitle?: string; prompt: string; @@ -265,6 +456,10 @@ export async function POST(request: Request) { const prompt = normalizeText(parsed.data.prompt); const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4); const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4); + const extractedVideoFrames = + parsed.data.mode === "video-analysis" && videoUrls.length > 0 + ? await extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT) + : []; const preferredModelRef = normalizeText(parsed.data.preferredModelRef || parsed.data.model); if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) { return NextResponse.json( @@ -290,14 +485,16 @@ export async function POST(request: Request) { ) : null; + const hasVisionInputs = imageUrls.length > 0 || extractedVideoFrames.length > 0; + let selectedModel = - preferredModel && (imageUrls.length === 0 || supportsVision(preferredModel.model)) + preferredModel && (!hasVisionInputs || supportsVision(preferredModel.model)) ? preferredModel - : imageUrls.length > 0 && + : hasVisionInputs && agentModel && supportsVision(agentModel.model) ? agentModel - : imageUrls.length > 0 + : hasVisionInputs ? compatibleModels.find((item) => supportsVision(item.model)) : agentModel || compatibleModels[0]; @@ -312,7 +509,7 @@ export async function POST(request: Request) { ); } - const useVision = imageUrls.length > 0 && supportsVision(selectedModel.model); + const useVision = hasVisionInputs && supportsVision(selectedModel.model); const codingModelId = getAiModelAgentCodingModelId(selectedModel.model); const model = await getLanguageModelById(selectedModel.model.id, codingModelId); if (!model) { @@ -325,6 +522,7 @@ export async function POST(request: Request) { const promptText = parsed.data.mode === "video-analysis" ? buildVideoAnalysisPrompt({ + frameCount: extractedVideoFrames.length, nodeTitle: normalizeText(parsed.data.nodeTitle), prompt, imageCount: imageUrls.length, @@ -343,7 +541,7 @@ export async function POST(request: Request) { videoCount: videoUrls.length, }); - const referenceImages = useVision + const inlinedImageReferences = useVision ? ( await inlineVisionImages(imageUrls, { maxImages: 4, @@ -354,6 +552,9 @@ export async function POST(request: Request) { }) ).inlineDataUrls : []; + const referenceImages = useVision + ? [...inlinedImageReferences, ...extractedVideoFrames].slice(0, 6) + : []; const { text } = await generateText({ model, @@ -376,12 +577,25 @@ export async function POST(request: Request) { normalizeText(text).slice(0, 160); if (parsed.data.mode === "video-analysis") { + const analysisRows = normalizeVideoAnalysisRows( + parsedJson.analysisRows, + extractedVideoFrames, + ); + const shotIdeas = + normalizeShotIdeas(parsedJson.shotIdeas).length > 0 + ? normalizeShotIdeas(parsedJson.shotIdeas) + : analysisRows.map((row, index) => ({ + title: `镜头 ${index + 1}`, + description: row.sceneDescription, + })); return NextResponse.json({ success: true, data: { summary, + analysisRows, + framePreviews: extractedVideoFrames, highlights: normalizeStringList(parsedJson.highlights, 6), - shotIdeas: normalizeShotIdeas(parsedJson.shotIdeas), + shotIdeas, visualMotifs: normalizeStringList(parsedJson.visualMotifs, 6), recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6), model: selectedModel.model.displayName, diff --git a/components/canvas/LiblibVideoCanvasReplica.tsx b/components/canvas/LiblibVideoCanvasReplica.tsx index 80340970d..c55e6351e 100644 --- a/components/canvas/LiblibVideoCanvasReplica.tsx +++ b/components/canvas/LiblibVideoCanvasReplica.tsx @@ -360,6 +360,23 @@ type VideoAnalysisRow = { keyframeUrl: string; }; +type VideoAnalysisResponsePayload = { + analysisRows?: Array<{ + backgroundMusic?: string; + imagePrompt?: string; + keyframeUrl?: string; + motionPrompt?: string; + sceneDescription?: string; + voiceAndSfx?: string; + }>; + framePreviews?: string[]; + highlights?: string[]; + raw?: string; + recommendedNextSteps?: string[]; + shotIdeas?: Array<{ title: string; description: string }>; + summary?: string; +}; + type VideoAnalysisState = { status: "idle" | "loading" | "ready" | "error"; summary: string; @@ -1324,14 +1341,26 @@ function normalizeVideoClipRange(value: unknown): VideoClipRange { }; } -function buildVideoAnalysisRows(node: CloneNode, responseData?: { - highlights?: string[]; - raw?: string; - recommendedNextSteps?: string[]; - shotIdeas?: Array<{ title: string; description: string }>; - summary?: string; -}): VideoAnalysisRow[] { +function buildVideoAnalysisRows(node: CloneNode, responseData?: VideoAnalysisResponsePayload): VideoAnalysisRow[] { const previewUrl = getVideoNodePreviewUrl(node) || getNodeMediaUrl(node) || ""; + const modelRows = Array.isArray(responseData?.analysisRows) ? responseData.analysisRows : []; + const framePreviews = Array.isArray(responseData?.framePreviews) + ? responseData.framePreviews.map((item) => String(item || "").trim()).filter(Boolean) + : []; + if (modelRows.length > 0) { + return modelRows.slice(0, 8).map((row, index) => ({ + id: `${node.id}-analysis-${index}`, + sceneDescription: String(row.sceneDescription || `镜头 ${index + 1}`).trim(), + backgroundMusic: String(row.backgroundMusic || "无").trim(), + voiceAndSfx: String(row.voiceAndSfx || "无").trim(), + imagePrompt: String(row.imagePrompt || row.sceneDescription || `${node.title} 关键帧`).trim(), + motionPrompt: String(row.motionPrompt || "保持镜头运动自然、连贯。").trim(), + keyframeUrl: + String(row.keyframeUrl || "").trim() || + framePreviews[index % Math.max(framePreviews.length, 1)] || + previewUrl, + })); + } const highlights = responseData?.highlights || []; const shotIdeas = responseData?.shotIdeas?.slice(0, 3) || []; const summary = responseData?.summary?.trim() || ""; @@ -1347,7 +1376,7 @@ function buildVideoAnalysisRows(node: CloneNode, responseData?: { motionPrompt: responseData?.recommendedNextSteps?.[index] || "保持镜头运动自然平稳，围绕主体动作推进。", - keyframeUrl: previewUrl, + keyframeUrl: framePreviews[index % Math.max(framePreviews.length, 1)] || previewUrl, })) : [ { @@ -1357,7 +1386,7 @@ function buildVideoAnalysisRows(node: CloneNode, responseData?: { voiceAndSfx: highlights[1] || "无", imagePrompt: fallbackPrompt, motionPrompt: "中景起势，镜头跟随主体动作缓慢推进。", - keyframeUrl: previewUrl, + keyframeUrl: framePreviews[0] || previewUrl, }, { id: `${node.id}-analysis-1`, @@ -1366,13 +1395,37 @@ function buildVideoAnalysisRows(node: CloneNode, responseData?: { voiceAndSfx: highlights[3] || "环境声增强", imagePrompt: fallbackPrompt, motionPrompt: "切换近景，强调表情和动作细节。", - keyframeUrl: previewUrl, + keyframeUrl: framePreviews[1] || framePreviews[0] || previewUrl, }, ]; return rows; } +function getPersistedVideoAnalysisState(node: CloneNode | null | undefined): VideoAnalysisState | null { + if (!node) return null; + const raw = + node?.draftValues?.videoAnalysis && + typeof node.draftValues.videoAnalysis === "object" && + !Array.isArray(node.draftValues.videoAnalysis) + ? (node.draftValues.videoAnalysis as Record) + : null; + if (!raw) return null; + const summary = String(raw.summary || "").trim() || "已保存视频解析结果"; + const responseData: VideoAnalysisResponsePayload = { + analysisRows: Array.isArray(raw.rows) ? (raw.rows as VideoAnalysisResponsePayload["analysisRows"]) : [], + framePreviews: Array.isArray(raw.framePreviews) ? raw.framePreviews.map((item) => String(item || "").trim()) : [], + summary, + }; + const rows = buildVideoAnalysisRows(node, responseData); + if (rows.length === 0) return null; + return { + rows, + status: "ready", + summary, + }; +} + function getGridSplitCellLabel(index: number, split: Pick) { const row = Math.floor(index / split.columns) + 1; const column = (index % split.columns) + 1; @@ -4502,6 +4555,7 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib const node = nodes.find((item) => item.id === nodeId); if (!node) return; const videoUrl = getNodeReferenceMediaUrl(node); + const previewImageUrl = getVideoNodePreviewUrl(node); if (!videoUrl) { toast.error("当前视频暂时没有可解析的素材地址"); return; @@ -4510,7 +4564,7 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib return; } - const definition = getConfiguredFunction(videoCanvasFunctionConfig, "story-script"); + const definition = getConfiguredFunction(videoCanvasFunctionConfig, "video-analysis"); const prompt = composeFunctionPrompt( definition, `请解析当前视频，输出结构化视频故事表。重点补充镜头描述、背景音乐、人声/音效、图像生成提示词、视频运动提示词与关键帧建议。视频标题：${node.title}` @@ -4528,6 +4582,8 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib try { const response = await workflowAnalysisService.analyze({ + imageUrls: + previewImageUrl && previewImageUrl !== videoUrl ? [previewImageUrl] : undefined, mode: "video-analysis", nodeTitle: node.title, preferredModelRef: getFunctionModel(videoCanvasFunctionConfig, definition), @@ -4542,6 +4598,23 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib response.data?.summary?.trim() || response.data?.highlights?.join(" · ") || "已整理出当前视频的镜头节奏与生成建议。"; + setNodes((current) => + current.map((item) => + item.id === nodeId + ? { + ...item, + draftValues: { + ...(item.draftValues || {}), + videoAnalysis: { + framePreviews: response.data?.framePreviews || [], + rows, + summary, + }, + }, + } + : item + ) + ); setVideoAnalysisStates((current) => ({ ...current, [nodeId]: { @@ -4551,11 +4624,28 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib }, })); } catch (error) { + const fallbackRows = buildVideoAnalysisRows(node); + setNodes((current) => + current.map((item) => + item.id === nodeId + ? { + ...item, + draftValues: { + ...(item.draftValues || {}), + videoAnalysis: { + rows: fallbackRows, + summary: "解析失败，先展示本地占位结构，方便继续编排。", + }, + }, + } + : item + ) + ); setVideoAnalysisStates((current) => ({ ...current, [nodeId]: { error: error instanceof Error ? error.message : "视频解析失败", - rows: buildVideoAnalysisRows(node), + rows: fallbackRows, status: "error", summary: "解析失败，先展示本地占位结构，方便继续编排。", }, @@ -5368,7 +5458,8 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib ; + framePreviews?: string[]; summary?: string; highlights?: string[]; shotIdeas?: Array<{ title: string; description: string }>;