lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve video analysis workflow
Some checks failed
Lint / build (20) (push) Has been cancelled
Details
Playwright Tests / test (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Your Name 2026-04-26 22:01:56 +08:00
parent c5c2d6eb93
commit 40af82a807
4 changed files with 344 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

226
app/api/canvas/workflow/analyze/route.ts
View File

@ -1,5 +1,11 @@
import { generateText } from "ai";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { NextResponse } from "next/server";
import sharp from "sharp";
import { z } from "zod";
import { auth } from "@/app/(auth)/auth";
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
@ -13,6 +19,11 @@ import {
export const runtime = "nodejs";
export const maxDuration = 180;
const execFileAsync = promisify(execFile);
const VIDEO_ANALYSIS_FRAME_COUNT = 4;
const VIDEO_ANALYSIS_MAX_SIDE = 960;
const VIDEO_ANALYSIS_TIMEOUT_MS = 15000;
const requestSchema = z.object({
mode: z.enum(["video-analysis", "character-scene-extraction", "agent-plan"]),
prompt: z.string().trim().optional(),
@ -93,6 +104,72 @@ function normalizeShotIdeas(value: unknown) {
.slice(0, 6);
}
function normalizeVideoAnalysisRows(
value: unknown,
framePreviews: string[],
) {
if (!Array.isArray(value)) return [];
return value
.map((item, index) => {
const raw =
item && typeof item === "object" && !Array.isArray(item)
? (item as Record<string, unknown>)
: {};
const sceneDescription = normalizeText(
raw.sceneDescription || raw.scene || raw.title || raw.description,
);
const backgroundMusic = normalizeText(
raw.backgroundMusic || raw.music || raw.bgm,
);
const voiceAndSfx = normalizeText(
raw.voiceAndSfx || raw.sound || raw.audio || raw.sfx,
);
const imagePrompt = normalizeText(
raw.imagePrompt || raw.image_generation_prompt || raw.image,
);
const motionPrompt = normalizeText(
raw.motionPrompt || raw.motion || raw.videoPrompt || raw.video_motion_prompt,
);
const keyframeIndex = Number(
raw.keyframeIndex || raw.frameIndex || raw.key_frame_index || index + 1,
);
if (
!sceneDescription &&
!backgroundMusic &&
!voiceAndSfx &&
!imagePrompt &&
!motionPrompt
) {
return null;
}
const normalizedFrameIndex =
Number.isFinite(keyframeIndex) && keyframeIndex > 0
? Math.max(0, Math.min(framePreviews.length - 1, Math.round(keyframeIndex) - 1))
: index % Math.max(framePreviews.length, 1);
return {
sceneDescription: sceneDescription || `镜头 ${index + 1}`,
backgroundMusic: backgroundMusic || "无",
voiceAndSfx: voiceAndSfx || "无",
imagePrompt: imagePrompt || sceneDescription || "延续当前视频气质生成关键帧",
motionPrompt: motionPrompt || "保持镜头运动自然、连贯、符合前后节奏。",
keyframeUrl: framePreviews[normalizedFrameIndex] || "",
};
})
.filter(
(
item,
): item is {
sceneDescription: string;
backgroundMusic: string;
voiceAndSfx: string;
imagePrompt: string;
motionPrompt: string;
keyframeUrl: string;
} => Boolean(item),
)
.slice(0, 8);
}
function normalizeCharacters(value: unknown) {
if (!Array.isArray(value)) return [];
return value
@ -147,6 +224,7 @@ function normalizeScenes(value: unknown) {
function buildVideoAnalysisPrompt(params: {
nodeTitle?: string;
prompt: string;
frameCount: number;
imageCount: number;
videoCount: number;
}) {
@ -155,6 +233,7 @@ function buildVideoAnalysisPrompt(params: {
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
`上游参考图数量：${params.imageCount}`,
`上游视频数量：${params.videoCount}`,
`已提供按时间顺序抽取的视频关键帧数量：${params.frameCount}`,
params.prompt ? `上游脚本/说明：\n${params.prompt}` : "上游没有额外脚本，请重点依据参考画面分析。",
"",
"请只输出 JSON，不要输出 markdown，不要输出解释。",
@ -162,18 +241,130 @@ function buildVideoAnalysisPrompt(params: {
`{
"summary": "一句话总结当前视频/镜头风格，不超过120字",
"highlights": ["3-5条镜头分析要点"],
"analysisRows": [
{
"sceneDescription": "镜头内容与景别描述",
"backgroundMusic": "这一段适合的背景音乐氛围",
"voiceAndSfx": "人声/环境声/音效建议，没有就写无",
"imagePrompt": "后续生成关键帧时可直接使用的图像提示词",
"motionPrompt": "后续生成视频时可直接使用的运动提示词",
"keyframeIndex": 1
}
],
"shotIdeas": [
{ "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" }
],
"visualMotifs": ["2-4个可复用的视觉母题"],
"recommendedNextSteps": ["2-4条建议的下游动作"]
}`,
"analysisRows 请尽量输出 2-6 行；keyframeIndex 使用 1 开始的序号，对应你看到的关键帧顺序。",
"如果没有真实视频帧，只能依据参考图和脚本推断，也要正常输出，但措辞要稳健。",
]
.filter(Boolean)
.join("\n");
}
async function getVideoDurationSeconds(videoUrl: string) {
try {
const { stdout } = await execFileAsync(
"ffprobe",
[
"-v",
"error",
"-show_entries",
"format=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
videoUrl,
],
{
timeout: VIDEO_ANALYSIS_TIMEOUT_MS,
maxBuffer: 1024 * 1024,
},
);
const duration = Number.parseFloat(String(stdout || "").trim());
return Number.isFinite(duration) && duration > 0 ? duration : null;
} catch {
return null;
}
}
function buildVideoFrameTimestamps(durationSeconds: number | null, frameCount: number) {
const safeCount = Math.max(1, frameCount);
if (!durationSeconds || !Number.isFinite(durationSeconds) || durationSeconds <= 0.5) {
return Array.from({ length: safeCount }).map((_, index) =>
Number((0.3 + index * 0.45).toFixed(2)),
);
}
const start = Math.min(0.35, durationSeconds * 0.12);
const end = Math.max(start + 0.6, durationSeconds * 0.88);
if (safeCount === 1) return [Number(((start + end) / 2).toFixed(2))];
const step = (end - start) / (safeCount - 1);
return Array.from({ length: safeCount }).map((_, index) =>
Number((start + step * index).toFixed(2)),
);
}
async function downscaleVideoFrameForVision(inputBuffer: Buffer) {
return sharp(inputBuffer, { failOn: "none" })
.rotate()
.resize({
width: VIDEO_ANALYSIS_MAX_SIDE,
height: VIDEO_ANALYSIS_MAX_SIDE,
fit: "inside",
withoutEnlargement: true,
})
.jpeg({ quality: 82, mozjpeg: true })
.toBuffer();
}
async function extractVideoFrameDataUrls(videoUrl: string, frameCount = VIDEO_ANALYSIS_FRAME_COUNT) {
const tempDir = await mkdtemp(path.join(os.tmpdir(), "canvas-video-analysis-"));
try {
const durationSeconds = await getVideoDurationSeconds(videoUrl);
const timestamps = buildVideoFrameTimestamps(durationSeconds, frameCount);
const frames = await Promise.all(
timestamps.map(async (timestamp, index) => {
const outputPath = path.join(tempDir, `frame-${index + 1}.jpg`);
await execFileAsync(
"ffmpeg",
[
"-hide_banner",
"-loglevel",
"error",
"-ss",
String(timestamp),
"-i",
videoUrl,
"-frames:v",
"1",
"-q:v",
"4",
"-y",
outputPath,
],
{
timeout: VIDEO_ANALYSIS_TIMEOUT_MS,
maxBuffer: 2 * 1024 * 1024,
},
);
const buffer = await readFile(outputPath);
const processed = await downscaleVideoFrameForVision(buffer);
return `data:image/jpeg;base64,${processed.toString("base64")}`;
}),
);
return frames.filter(Boolean);
} catch (error) {
console.warn("[Canvas Workflow Analyze] extract video frames failed:", {
message: error instanceof Error ? error.message : String(error),
videoUrl,
});
return [];
} finally {
await rm(tempDir, { force: true, recursive: true }).catch(() => undefined);
}
}
function buildCharacterScenePrompt(params: {
nodeTitle?: string;
prompt: string;
@ -265,6 +456,10 @@ export async function POST(request: Request) {
const prompt = normalizeText(parsed.data.prompt);
const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4);
const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4);
const extractedVideoFrames =
parsed.data.mode === "video-analysis" && videoUrls.length > 0
? await extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT)
: [];
const preferredModelRef = normalizeText(parsed.data.preferredModelRef || parsed.data.model);
if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) {
return NextResponse.json(
@ -290,14 +485,16 @@ export async function POST(request: Request) {
)
: null;
const hasVisionInputs = imageUrls.length > 0 || extractedVideoFrames.length > 0;
let selectedModel =
preferredModel && (imageUrls.length === 0 || supportsVision(preferredModel.model))
preferredModel && (!hasVisionInputs || supportsVision(preferredModel.model))
? preferredModel
: imageUrls.length > 0 &&
: hasVisionInputs &&
agentModel &&
supportsVision(agentModel.model)
? agentModel
: imageUrls.length > 0
: hasVisionInputs
? compatibleModels.find((item) => supportsVision(item.model))
: agentModel || compatibleModels[0];
@ -312,7 +509,7 @@ export async function POST(request: Request) {
);
}
const useVision = imageUrls.length > 0 && supportsVision(selectedModel.model);
const useVision = hasVisionInputs && supportsVision(selectedModel.model);
const codingModelId = getAiModelAgentCodingModelId(selectedModel.model);
const model = await getLanguageModelById(selectedModel.model.id, codingModelId);
if (!model) {
@ -325,6 +522,7 @@ export async function POST(request: Request) {
const promptText =
parsed.data.mode === "video-analysis"
? buildVideoAnalysisPrompt({
frameCount: extractedVideoFrames.length,
nodeTitle: normalizeText(parsed.data.nodeTitle),
prompt,
imageCount: imageUrls.length,
@ -343,7 +541,7 @@ export async function POST(request: Request) {
videoCount: videoUrls.length,
});
const referenceImages = useVision
const inlinedImageReferences = useVision
? (
await inlineVisionImages(imageUrls, {
maxImages: 4,
@ -354,6 +552,9 @@ export async function POST(request: Request) {
})
).inlineDataUrls
: [];
const referenceImages = useVision
? [...inlinedImageReferences, ...extractedVideoFrames].slice(0, 6)
: [];
const { text } = await generateText({
model,
@ -376,12 +577,25 @@ export async function POST(request: Request) {
normalizeText(text).slice(0, 160);
if (parsed.data.mode === "video-analysis") {
const analysisRows = normalizeVideoAnalysisRows(
parsedJson.analysisRows,
extractedVideoFrames,
);
const shotIdeas =
normalizeShotIdeas(parsedJson.shotIdeas).length > 0
? normalizeShotIdeas(parsedJson.shotIdeas)
: analysisRows.map((row, index) => ({
title: `镜头 ${index + 1}`,
description: row.sceneDescription,
}));
return NextResponse.json({
success: true,
data: {
summary,
analysisRows,
framePreviews: extractedVideoFrames,
highlights: normalizeStringList(parsedJson.highlights, 6),
shotIdeas: normalizeShotIdeas(parsedJson.shotIdeas),
shotIdeas,
visualMotifs: normalizeStringList(parsedJson.visualMotifs, 6),
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
model: selectedModel.model.displayName,

117
components/canvas/LiblibVideoCanvasReplica.tsx
View File

@ -360,6 +360,23 @@ type VideoAnalysisRow = {
keyframeUrl: string;
};
type VideoAnalysisResponsePayload = {
analysisRows?: Array<{
backgroundMusic?: string;
imagePrompt?: string;
keyframeUrl?: string;
motionPrompt?: string;
sceneDescription?: string;
voiceAndSfx?: string;
}>;
framePreviews?: string[];
highlights?: string[];
raw?: string;
recommendedNextSteps?: string[];
shotIdeas?: Array<{ title: string; description: string }>;
summary?: string;
};
type VideoAnalysisState = {
status: "idle" | "loading" | "ready" | "error";
summary: string;
@ -1324,14 +1341,26 @@ function normalizeVideoClipRange(value: unknown): VideoClipRange {
};
}
function buildVideoAnalysisRows(node: CloneNode, responseData?: {
highlights?: string[];
raw?: string;
recommendedNextSteps?: string[];
shotIdeas?: Array<{ title: string; description: string }>;
summary?: string;
}): VideoAnalysisRow[] {
function buildVideoAnalysisRows(node: CloneNode, responseData?: VideoAnalysisResponsePayload): VideoAnalysisRow[] {
const previewUrl = getVideoNodePreviewUrl(node) || getNodeMediaUrl(node) || "";
const modelRows = Array.isArray(responseData?.analysisRows) ? responseData.analysisRows : [];
const framePreviews = Array.isArray(responseData?.framePreviews)
? responseData.framePreviews.map((item) => String(item || "").trim()).filter(Boolean)
: [];
if (modelRows.length > 0) {
return modelRows.slice(0, 8).map((row, index) => ({
id: `${node.id}-analysis-${index}`,
sceneDescription: String(row.sceneDescription || `镜头 ${index + 1}`).trim(),
backgroundMusic: String(row.backgroundMusic || "无").trim(),
voiceAndSfx: String(row.voiceAndSfx || "无").trim(),
imagePrompt: String(row.imagePrompt || row.sceneDescription || `${node.title} 关键帧`).trim(),
motionPrompt: String(row.motionPrompt || "保持镜头运动自然、连贯。").trim(),
keyframeUrl:
String(row.keyframeUrl || "").trim() ||
framePreviews[index % Math.max(framePreviews.length, 1)] ||
previewUrl,
}));
}
const highlights = responseData?.highlights || [];
const shotIdeas = responseData?.shotIdeas?.slice(0, 3) || [];
const summary = responseData?.summary?.trim() || "";
@ -1347,7 +1376,7 @@ function buildVideoAnalysisRows(node: CloneNode, responseData?: {
motionPrompt:
responseData?.recommendedNextSteps?.[index] ||
"保持镜头运动自然平稳，围绕主体动作推进。",
keyframeUrl: previewUrl,
keyframeUrl: framePreviews[index % Math.max(framePreviews.length, 1)] || previewUrl,
}))
: [
{
@ -1357,7 +1386,7 @@ function buildVideoAnalysisRows(node: CloneNode, responseData?: {
voiceAndSfx: highlights[1] || "无",
imagePrompt: fallbackPrompt,
motionPrompt: "中景起势，镜头跟随主体动作缓慢推进。",
keyframeUrl: previewUrl,
keyframeUrl: framePreviews[0] || previewUrl,
},
{
id: `${node.id}-analysis-1`,
@ -1366,13 +1395,37 @@ function buildVideoAnalysisRows(node: CloneNode, responseData?: {
voiceAndSfx: highlights[3] || "环境声增强",
imagePrompt: fallbackPrompt,
motionPrompt: "切换近景，强调表情和动作细节。",
keyframeUrl: previewUrl,
keyframeUrl: framePreviews[1] || framePreviews[0] || previewUrl,
},
];
return rows;
}
function getPersistedVideoAnalysisState(node: CloneNode | null | undefined): VideoAnalysisState | null {
if (!node) return null;
const raw =
node?.draftValues?.videoAnalysis &&
typeof node.draftValues.videoAnalysis === "object" &&
!Array.isArray(node.draftValues.videoAnalysis)
? (node.draftValues.videoAnalysis as Record<string, unknown>)
: null;
if (!raw) return null;
const summary = String(raw.summary || "").trim() || "已保存视频解析结果";
const responseData: VideoAnalysisResponsePayload = {
analysisRows: Array.isArray(raw.rows) ? (raw.rows as VideoAnalysisResponsePayload["analysisRows"]) : [],
framePreviews: Array.isArray(raw.framePreviews) ? raw.framePreviews.map((item) => String(item || "").trim()) : [],
summary,
};
const rows = buildVideoAnalysisRows(node, responseData);
if (rows.length === 0) return null;
return {
rows,
status: "ready",
summary,
};
}
function getGridSplitCellLabel(index: number, split: Pick<GridSplitState, "columns">) {
const row = Math.floor(index / split.columns) + 1;
const column = (index % split.columns) + 1;
@ -4502,6 +4555,7 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib
const node = nodes.find((item) => item.id === nodeId);
if (!node) return;
const videoUrl = getNodeReferenceMediaUrl(node);
const previewImageUrl = getVideoNodePreviewUrl(node);
if (!videoUrl) {
toast.error("当前视频暂时没有可解析的素材地址");
return;
@ -4510,7 +4564,7 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib
return;
}
const definition = getConfiguredFunction(videoCanvasFunctionConfig, "story-script");
const definition = getConfiguredFunction(videoCanvasFunctionConfig, "video-analysis");
const prompt = composeFunctionPrompt(
definition,
`请解析当前视频，输出结构化视频故事表。重点补充镜头描述、背景音乐、人声/音效、图像生成提示词、视频运动提示词与关键帧建议。视频标题：${node.title}`
@ -4528,6 +4582,8 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib
try {
const response = await workflowAnalysisService.analyze({
imageUrls:
previewImageUrl && previewImageUrl !== videoUrl ? [previewImageUrl] : undefined,
mode: "video-analysis",
nodeTitle: node.title,
preferredModelRef: getFunctionModel(videoCanvasFunctionConfig, definition),
@ -4542,6 +4598,23 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib
response.data?.summary?.trim() ||
response.data?.highlights?.join(" · ") ||
"已整理出当前视频的镜头节奏与生成建议。";
setNodes((current) =>
current.map((item) =>
item.id === nodeId
? {
...item,
draftValues: {
...(item.draftValues || {}),
videoAnalysis: {
framePreviews: response.data?.framePreviews || [],
rows,
summary,
},
},
}
: item
)
);
setVideoAnalysisStates((current) => ({
...current,
[nodeId]: {
@ -4551,11 +4624,28 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib
},
}));
} catch (error) {
const fallbackRows = buildVideoAnalysisRows(node);
setNodes((current) =>
current.map((item) =>
item.id === nodeId
? {
...item,
draftValues: {
...(item.draftValues || {}),
videoAnalysis: {
rows: fallbackRows,
summary: "解析失败，先展示本地占位结构，方便继续编排。",
},
},
}
: item
)
);
setVideoAnalysisStates((current) => ({
...current,
[nodeId]: {
error: error instanceof Error ? error.message : "视频解析失败",
rows: buildVideoAnalysisRows(node),
rows: fallbackRows,
status: "error",
summary: "解析失败，先展示本地占位结构，方便继续编排。",
},
@ -5368,7 +5458,8 @@ export function LiblibVideoCanvasReplica({ canvasId, isLoading = false }: Liblib
<VideoAnalysisFloatingPanel
anchor={selectedVideoAnchor}
analysisState={
videoAnalysisStates[selectedVideoMediaNode.id] || {
videoAnalysisStates[selectedVideoMediaNode.id] ||
getPersistedVideoAnalysisState(selectedVideoMediaNode) || {
rows: [],
status: "idle",
summary: "准备解析当前视频故事...",

11
lib/video-canvas/config.ts
View File

@ -232,6 +232,17 @@ const defaultFunctions: VideoCanvasFunctionDefinition[] = [
prompt: "根据输入文本、图片或视频参考，生成结构化分镜脚本，包含镜号、场景、画面、声音和镜头节奏。",
description: "多模态大模型/文本模型生成脚本表格。",
},
{
key: "video-analysis",
label: "视频解析",
kind: "multimodal",
route: "canvas-workflow-analyze",
enabled: true,
creditCost: 6,
prompt:
"解析当前视频的镜头节奏、景别变化、主体动作、背景音乐、人声/音效、关键帧要点，并输出可直接衔接后续图像生成与视频生成节点的结构化故事表。",
description: "多模态视频解析：抽取关键帧并输出视频故事表。",
},
{
key: "image-to-prompt",
label: "图片反推提示词",

9
services/canvas/index.ts
View File

@ -743,6 +743,15 @@ export const workflowAnalysisService = {
preferredModelRef?: string;
}): Promise<
ApiResponse<{
analysisRows?: Array<{
sceneDescription?: string;
backgroundMusic?: string;
voiceAndSfx?: string;
imagePrompt?: string;
motionPrompt?: string;
keyframeUrl?: string;
}>;
framePreviews?: string[];
summary?: string;
highlights?: string[];
shotIdeas?: Array<{ title: string; description: string }>;