1003 lines
36 KiB
TypeScript
1003 lines
36 KiB
TypeScript
import { generateText } from "ai";
|
||
import { execFile } from "node:child_process";
|
||
import { promisify } from "node:util";
|
||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||
import os from "node:os";
|
||
import path from "node:path";
|
||
import { NextResponse } from "next/server";
|
||
import sharp from "sharp";
|
||
import { z } from "zod";
|
||
import { auth } from "@/app/(auth)/auth";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { inlineVisionImages } from "@/lib/ai/vision-inline";
|
||
import { getAiModelAgentCodingModelId, isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||
import {
|
||
getAgentAiModelWithProvider,
|
||
getEnabledAiModelsWithProvider,
|
||
} from "@/lib/db/queries";
|
||
|
||
export const runtime = "nodejs";
|
||
export const maxDuration = 180;
|
||
|
||
const execFileAsync = promisify(execFile);
|
||
const VIDEO_ANALYSIS_FRAME_COUNT = 4;
|
||
const VIDEO_ANALYSIS_MAX_SIDE = 960;
|
||
const VIDEO_ANALYSIS_TIMEOUT_MS = 15000;
|
||
|
||
const requestSchema = z.object({
|
||
mode: z.enum(["video-analysis", "character-scene-extraction", "story-script", "agent-plan"]),
|
||
prompt: z.string().trim().optional(),
|
||
imageUrls: z.array(z.string().trim()).optional(),
|
||
videoUrls: z.array(z.string().trim()).optional(),
|
||
nodeTitle: z.string().trim().optional(),
|
||
preferredModelRef: z.string().trim().optional(),
|
||
model: z.string().trim().optional(),
|
||
});
|
||
|
||
function sanitizeUrls(value: unknown, limit = 6) {
|
||
if (!Array.isArray(value)) return [];
|
||
return value
|
||
.map((item) => String(item || "").trim())
|
||
.filter((url) => /^https?:\/\//i.test(url))
|
||
.slice(0, limit);
|
||
}
|
||
|
||
function supportsVision(model: {
|
||
supportsVision?: boolean | null;
|
||
capabilities?: unknown;
|
||
defaultParams?: unknown;
|
||
}) {
|
||
return (
|
||
Boolean(model.supportsVision) ||
|
||
isAiModelMultimodal({
|
||
capabilities: model.capabilities,
|
||
defaultParams: model.defaultParams,
|
||
})
|
||
);
|
||
}
|
||
|
||
function extractJsonBlock<T = Record<string, unknown>>(text: string): T | null {
|
||
const matched = String(text || "").match(/\{[\s\S]*\}/);
|
||
if (!matched) return null;
|
||
try {
|
||
return JSON.parse(matched[0]) as T;
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function normalizeText(value: unknown) {
|
||
return String(value || "").trim();
|
||
}
|
||
|
||
function normalizeStringList(value: unknown, limit = 8) {
|
||
if (!Array.isArray(value)) return [];
|
||
return Array.from(
|
||
new Set(
|
||
value
|
||
.map((item) => String(item || "").trim())
|
||
.filter(Boolean)
|
||
.slice(0, limit)
|
||
)
|
||
);
|
||
}
|
||
|
||
function normalizeShotIdeas(value: unknown) {
|
||
if (!Array.isArray(value)) return [];
|
||
return value
|
||
.map((item, index) => {
|
||
const raw =
|
||
item && typeof item === "object" && !Array.isArray(item)
|
||
? (item as Record<string, unknown>)
|
||
: {};
|
||
const title = normalizeText(raw.title || `镜头 ${index + 1}`);
|
||
const description = normalizeText(raw.description || raw.prompt || raw.summary);
|
||
if (!title && !description) return null;
|
||
return {
|
||
title: title || `镜头 ${index + 1}`,
|
||
description,
|
||
};
|
||
})
|
||
.filter(
|
||
(item): item is { title: string; description: string } => Boolean(item)
|
||
)
|
||
.slice(0, 6);
|
||
}
|
||
|
||
function normalizeVideoAnalysisRows(
|
||
value: unknown,
|
||
framePreviews: string[],
|
||
) {
|
||
if (!Array.isArray(value)) return [];
|
||
return value
|
||
.map((item, index) => {
|
||
const raw =
|
||
item && typeof item === "object" && !Array.isArray(item)
|
||
? (item as Record<string, unknown>)
|
||
: {};
|
||
const sceneDescription = normalizeText(
|
||
raw.sceneDescription || raw.scene || raw.title || raw.description,
|
||
);
|
||
const narrative = normalizeText(
|
||
raw.narrative || raw.story || raw.storyContent || raw.narration || raw.summary,
|
||
);
|
||
const shot = normalizeText(raw.shot || raw.shotNumber || raw.index || index + 1);
|
||
const startTime = normalizeText(raw.startTime || raw.start || raw.start_time);
|
||
const endTime = normalizeText(raw.endTime || raw.end || raw.end_time);
|
||
const duration = normalizeText(raw.duration || raw.durationSeconds || raw.duration_seconds);
|
||
const shotSize = normalizeText(raw.shotSize || raw.frameSize || raw.shot_type || raw.sceneScale);
|
||
const cameraAngle = normalizeText(raw.cameraAngle || raw.angle || raw.camera_angle);
|
||
const cameraMovement = normalizeText(
|
||
raw.cameraMovement || raw.cameraMove || raw.camera_motion || raw.motionType,
|
||
);
|
||
const focusDepth = normalizeText(
|
||
raw.focusDepth || raw.depthOfField || raw.focalDepth || raw.focus_depth,
|
||
);
|
||
const lighting = normalizeText(raw.lighting || raw.light || raw.lightDesign);
|
||
const backgroundMusic = normalizeText(
|
||
raw.backgroundMusic || raw.music || raw.bgm,
|
||
);
|
||
const voiceAndSfx = normalizeText(
|
||
raw.voiceAndSfx || raw.sound || raw.audio || raw.sfx,
|
||
);
|
||
const imagePrompt = normalizeText(
|
||
raw.imagePrompt || raw.image_generation_prompt || raw.image,
|
||
);
|
||
const motionPrompt = normalizeText(
|
||
raw.motionPrompt || raw.motion || raw.videoPrompt || raw.video_motion_prompt,
|
||
);
|
||
const keyframeIndex = Number(
|
||
raw.keyframeIndex || raw.frameIndex || raw.key_frame_index || index + 1,
|
||
);
|
||
if (
|
||
!sceneDescription &&
|
||
!narrative &&
|
||
!backgroundMusic &&
|
||
!voiceAndSfx &&
|
||
!imagePrompt &&
|
||
!motionPrompt &&
|
||
!shotSize &&
|
||
!cameraAngle &&
|
||
!cameraMovement &&
|
||
!focusDepth &&
|
||
!lighting
|
||
) {
|
||
return null;
|
||
}
|
||
const normalizedFrameIndex =
|
||
Number.isFinite(keyframeIndex) && keyframeIndex > 0
|
||
? Math.max(0, Math.min(framePreviews.length - 1, Math.round(keyframeIndex) - 1))
|
||
: index % Math.max(framePreviews.length, 1);
|
||
return {
|
||
shot: shot || String(index + 1),
|
||
startTime: startTime || (index === 0 ? "0" : ""),
|
||
endTime,
|
||
duration,
|
||
sceneDescription: sceneDescription || `镜头 ${index + 1}`,
|
||
narrative: narrative || sceneDescription || `镜头 ${index + 1}`,
|
||
shotSize: shotSize || "中景至近景",
|
||
cameraAngle: cameraAngle || "平视",
|
||
cameraMovement: cameraMovement || "缓慢推镜头 (Slow Zoom In)",
|
||
focusDepth: focusDepth || "浅景深",
|
||
lighting: lighting || "冷调低沉照明，主体光线柔和。",
|
||
backgroundMusic: backgroundMusic || "无",
|
||
voiceAndSfx: voiceAndSfx || "无",
|
||
imagePrompt: imagePrompt || sceneDescription || "延续当前视频气质生成关键帧",
|
||
motionPrompt: motionPrompt || "保持镜头运动自然、连贯、符合前后节奏。",
|
||
keyframeUrl: framePreviews[normalizedFrameIndex] || "",
|
||
};
|
||
})
|
||
.filter(
|
||
(
|
||
item,
|
||
): item is {
|
||
shot: string;
|
||
startTime: string;
|
||
endTime: string;
|
||
duration: string;
|
||
sceneDescription: string;
|
||
narrative: string;
|
||
shotSize: string;
|
||
cameraAngle: string;
|
||
cameraMovement: string;
|
||
focusDepth: string;
|
||
lighting: string;
|
||
backgroundMusic: string;
|
||
voiceAndSfx: string;
|
||
imagePrompt: string;
|
||
motionPrompt: string;
|
||
keyframeUrl: string;
|
||
} => Boolean(item),
|
||
)
|
||
.slice(0, 8);
|
||
}
|
||
|
||
function normalizeStoryShot(value: unknown, index: number) {
|
||
const text = normalizeText(value);
|
||
if (/^\d+$/.test(text)) return text.padStart(2, "0");
|
||
return text || String(index + 1).padStart(2, "0");
|
||
}
|
||
|
||
type NormalizedStoryScriptRow = {
|
||
shot: string;
|
||
duration: string;
|
||
scene: string;
|
||
sceneDescription: string;
|
||
shotSize: string;
|
||
lighting: string;
|
||
dialogue: string;
|
||
backgroundMusic: string;
|
||
voiceAndSfx: string;
|
||
cameraMovement: string;
|
||
imagePrompt: string;
|
||
motionPrompt: string;
|
||
keyframeUrl: string;
|
||
};
|
||
|
||
function normalizeStoryScriptRows(value: unknown): NormalizedStoryScriptRow[] {
|
||
if (!Array.isArray(value)) return [];
|
||
return value
|
||
.map((item, index) => {
|
||
if (Array.isArray(item)) {
|
||
const cells = item.map((cell) => normalizeText(cell));
|
||
const shot = normalizeStoryShot(cells[0], index);
|
||
const secondCellLooksLikeDuration = /^\d+(\.\d+)?\s*(s|秒)?$/i.test(cells[1] || "");
|
||
const duration = secondCellLooksLikeDuration ? cells[1] : "5s";
|
||
const scene = secondCellLooksLikeDuration ? cells[2] || `镜头 ${shot}` : cells[1] || `镜头 ${shot}`;
|
||
const sceneDescription = secondCellLooksLikeDuration ? cells[2] || scene : cells[2] || cells[3] || scene;
|
||
return {
|
||
shot,
|
||
duration,
|
||
scene,
|
||
sceneDescription,
|
||
shotSize: (secondCellLooksLikeDuration ? cells[3] : "") || "中景",
|
||
lighting: (secondCellLooksLikeDuration ? cells[4] : cells[4]) || "电影感光影",
|
||
dialogue: (secondCellLooksLikeDuration ? cells[5] : "") || "+",
|
||
backgroundMusic: "无",
|
||
voiceAndSfx: (secondCellLooksLikeDuration ? cells[6] : cells[3]) || "无",
|
||
cameraMovement: (secondCellLooksLikeDuration ? cells[7] : "") || "缓慢推镜头",
|
||
imagePrompt: (secondCellLooksLikeDuration ? cells[8] : cells[5]) || sceneDescription,
|
||
motionPrompt:
|
||
(secondCellLooksLikeDuration ? cells[9] || cells[7] : cells[6]) ||
|
||
"保持镜头运动自然、连贯，围绕主体动作推进。",
|
||
keyframeUrl: "",
|
||
};
|
||
}
|
||
|
||
const raw =
|
||
item && typeof item === "object" && !Array.isArray(item)
|
||
? (item as Record<string, unknown>)
|
||
: {};
|
||
const shot = normalizeStoryShot(raw.shot || raw.shotNumber || raw.index || raw.id, index);
|
||
const duration = normalizeText(raw.duration || raw.length || raw.time || raw.seconds);
|
||
const scene = normalizeText(raw.scene || raw.location || raw.title || `镜头 ${shot}`);
|
||
const sceneDescription = normalizeText(
|
||
raw.sceneDescription ||
|
||
raw.description ||
|
||
raw.visual ||
|
||
raw.picture ||
|
||
raw.screen ||
|
||
raw.summary,
|
||
);
|
||
const shotSize = normalizeText(
|
||
raw.shotSize || raw.frameSize || raw.shot_type || raw.sceneScale || raw.cameraShot,
|
||
);
|
||
const lighting = normalizeText(
|
||
raw.lighting || raw.light || raw.atmosphere || raw.mood || raw.lightMood,
|
||
);
|
||
const dialogue = normalizeText(
|
||
raw.dialogue || raw.dialog || raw.narration || raw.voiceover || raw.monologue || raw.line,
|
||
);
|
||
const cameraMovement = normalizeText(
|
||
raw.cameraMovement || raw.cameraMove || raw.movement || raw.lensMovement || raw.motion,
|
||
);
|
||
const imagePrompt = normalizeText(
|
||
raw.imagePrompt ||
|
||
raw.image_generation_prompt ||
|
||
raw.image ||
|
||
raw.keyframePrompt ||
|
||
raw.keyFramePrompt,
|
||
);
|
||
const motionPrompt = normalizeText(
|
||
raw.motionPrompt ||
|
||
raw.videoPrompt ||
|
||
raw.video_motion_prompt ||
|
||
raw.motion ||
|
||
raw.cameraMovement,
|
||
);
|
||
const backgroundMusic = normalizeText(
|
||
raw.backgroundMusic || raw.music || raw.bgm || raw.soundtrack,
|
||
);
|
||
const voiceAndSfx = normalizeText(
|
||
raw.voiceAndSfx || raw.voice || raw.sound || raw.audio || raw.sfx,
|
||
);
|
||
|
||
if (
|
||
!scene &&
|
||
!sceneDescription &&
|
||
!imagePrompt &&
|
||
!motionPrompt &&
|
||
!backgroundMusic &&
|
||
!voiceAndSfx
|
||
) {
|
||
return null;
|
||
}
|
||
|
||
return {
|
||
shot,
|
||
duration: duration || "5s",
|
||
scene,
|
||
sceneDescription: sceneDescription || imagePrompt || scene || `镜头 ${shot}`,
|
||
shotSize: shotSize || "中景",
|
||
lighting: lighting || "电影感光影",
|
||
dialogue: dialogue || "+",
|
||
backgroundMusic: backgroundMusic || "无",
|
||
voiceAndSfx: voiceAndSfx || "无",
|
||
cameraMovement: cameraMovement || "缓慢推镜头",
|
||
imagePrompt: imagePrompt || sceneDescription || scene || `镜头 ${shot}关键帧`,
|
||
motionPrompt: motionPrompt || cameraMovement || "保持镜头运动自然、连贯，围绕主体动作推进。",
|
||
keyframeUrl: "",
|
||
};
|
||
})
|
||
.filter(
|
||
(
|
||
item,
|
||
): item is NormalizedStoryScriptRow => Boolean(item),
|
||
)
|
||
.slice(0, 18);
|
||
}
|
||
|
||
const storyExpansionPhases = [
|
||
{
|
||
description: "建立镜头，交代环境、人物位置和当前压力",
|
||
cameraMovement: "缓慢推镜头",
|
||
imagePrompt: "广角或中景建立画面，环境信息清晰，主体位置明确",
|
||
lighting: "环境主光明确，氛围基调清晰",
|
||
motionPrompt: "镜头缓慢推进或轻微横移，建立空间关系",
|
||
shotSize: "中景",
|
||
},
|
||
{
|
||
description: "动作推进，突出人物的关键行为或剧情转折",
|
||
cameraMovement: "跟随主体动作推进",
|
||
imagePrompt: "中景表现人物动作，服装、道具和光影保持连续",
|
||
lighting: "主体光影连续，动作区域清晰",
|
||
motionPrompt: "跟随主体动作推进，节奏自然连贯",
|
||
shotSize: "中景",
|
||
},
|
||
{
|
||
description: "情绪反应，用近景或特写强化人物心理和冲突",
|
||
cameraMovement: "轻微推近或稳定特写",
|
||
imagePrompt: "近景或特写，突出表情、眼神、手部或关键道具细节",
|
||
lighting: "面部光线突出，情绪反差增强",
|
||
motionPrompt: "轻微推近或稳定特写，强调情绪变化",
|
||
shotSize: "特写",
|
||
},
|
||
{
|
||
description: "关系与氛围，表现周围人物、环境反馈或声画张力",
|
||
cameraMovement: "横移扫过环境或切换视角",
|
||
imagePrompt: "加入环境反馈、群像反应或空间层次，电影感光影",
|
||
lighting: "场景氛围增强，空间层次明显",
|
||
motionPrompt: "镜头扫过环境或切换视角，保持节奏递进",
|
||
shotSize: "全景",
|
||
},
|
||
{
|
||
description: "转场收束，为下一镜头保留动作、视线或情绪钩子",
|
||
cameraMovement: "顺势后移、上摇或转场",
|
||
imagePrompt: "构图留出转场方向，动作或视线指向下一段剧情",
|
||
lighting: "光影引导视线并保留转场方向",
|
||
motionPrompt: "镜头后移、上摇或顺势转场，衔接下一镜头",
|
||
shotSize: "大远景",
|
||
},
|
||
];
|
||
|
||
function expandStoryScriptRowsToTarget(
|
||
rows: NormalizedStoryScriptRow[],
|
||
targetCount = 18,
|
||
): NormalizedStoryScriptRow[] {
|
||
if (rows.length === 0) return [];
|
||
if (rows.length >= targetCount) {
|
||
return rows.slice(0, targetCount).map((row, index) => ({
|
||
...row,
|
||
shot: String(index + 1).padStart(2, "0"),
|
||
}));
|
||
}
|
||
|
||
const baseCount = Math.floor(targetCount / rows.length);
|
||
const extraCount = targetCount % rows.length;
|
||
const expanded: NormalizedStoryScriptRow[] = [];
|
||
|
||
rows.forEach((row, rowIndex) => {
|
||
const segmentCount = baseCount + (rowIndex < extraCount ? 1 : 0);
|
||
for (let segmentIndex = 0; segmentIndex < segmentCount; segmentIndex += 1) {
|
||
const phase =
|
||
storyExpansionPhases[
|
||
Math.min(
|
||
storyExpansionPhases.length - 1,
|
||
Math.floor((segmentIndex / Math.max(segmentCount, 1)) * storyExpansionPhases.length),
|
||
)
|
||
];
|
||
expanded.push({
|
||
...row,
|
||
shot: String(expanded.length + 1).padStart(2, "0"),
|
||
sceneDescription: `${phase.description}：${row.sceneDescription}`,
|
||
shotSize: row.shotSize || phase.shotSize,
|
||
lighting: row.lighting || phase.lighting,
|
||
cameraMovement: row.cameraMovement || phase.cameraMovement,
|
||
imagePrompt: `${row.imagePrompt}；${phase.imagePrompt}`,
|
||
motionPrompt: `${phase.motionPrompt}；${row.motionPrompt}`,
|
||
});
|
||
}
|
||
});
|
||
|
||
return expanded.slice(0, targetCount);
|
||
}
|
||
|
||
function normalizeCharacters(value: unknown) {
|
||
if (!Array.isArray(value)) return [];
|
||
return value
|
||
.map((item) => {
|
||
const raw =
|
||
item && typeof item === "object" && !Array.isArray(item)
|
||
? (item as Record<string, unknown>)
|
||
: {};
|
||
const name = normalizeText(raw.name || raw.title);
|
||
const description = normalizeText(raw.description || raw.summary);
|
||
const prompt = normalizeText(raw.prompt);
|
||
if (!name && !description && !prompt) return null;
|
||
return {
|
||
name: name || "未命名角色",
|
||
description,
|
||
prompt,
|
||
look: normalizeText(raw.look) || undefined,
|
||
wardrobe: normalizeText(raw.wardrobe) || undefined,
|
||
signatureAction: normalizeText(raw.signatureAction) || undefined,
|
||
traits: normalizeStringList(raw.traits, 6),
|
||
};
|
||
})
|
||
.filter((item): item is NonNullable<typeof item> => Boolean(item))
|
||
.slice(0, 8);
|
||
}
|
||
|
||
function normalizeScenes(value: unknown) {
|
||
if (!Array.isArray(value)) return [];
|
||
return value
|
||
.map((item) => {
|
||
const raw =
|
||
item && typeof item === "object" && !Array.isArray(item)
|
||
? (item as Record<string, unknown>)
|
||
: {};
|
||
const name = normalizeText(raw.name || raw.title);
|
||
const description = normalizeText(raw.description || raw.summary);
|
||
const prompt = normalizeText(raw.prompt);
|
||
if (!name && !description && !prompt) return null;
|
||
return {
|
||
name: name || "未命名场景",
|
||
description,
|
||
prompt,
|
||
mood: normalizeText(raw.mood) || undefined,
|
||
lighting: normalizeText(raw.lighting) || undefined,
|
||
props: normalizeStringList(raw.props, 6),
|
||
};
|
||
})
|
||
.filter((item): item is NonNullable<typeof item> => Boolean(item))
|
||
.slice(0, 8);
|
||
}
|
||
|
||
function buildVideoAnalysisPrompt(params: {
|
||
nodeTitle?: string;
|
||
prompt: string;
|
||
frameCount: number;
|
||
imageCount: number;
|
||
videoCount: number;
|
||
}) {
|
||
return [
|
||
"你是视频画布工作流里的视频分析节点，需要为后续分镜和生成节点提供结构化输出。",
|
||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||
`上游参考图数量：${params.imageCount}`,
|
||
`上游视频数量：${params.videoCount}`,
|
||
`已提供按时间顺序抽取的视频关键帧数量：${params.frameCount}`,
|
||
params.prompt ? `上游脚本/说明：\n${params.prompt}` : "上游没有额外脚本，请重点依据参考画面分析。",
|
||
"",
|
||
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
||
"JSON 结构如下：",
|
||
`{
|
||
"summary": "一句话总结当前视频/镜头风格，不超过120字",
|
||
"highlights": ["3-5条镜头分析要点"],
|
||
"analysisRows": [
|
||
{
|
||
"shot": "1",
|
||
"startTime": "0",
|
||
"endTime": "5",
|
||
"duration": "5",
|
||
"sceneDescription": "画面描述，说明主体、环境、动作、构图和色调",
|
||
"narrative": "叙事内容，说明这一镜头承担的剧情/情绪/信息作用",
|
||
"shotSize": "景别，例如中景至近景",
|
||
"cameraAngle": "摄影机角度，例如平视",
|
||
"cameraMovement": "摄影机运动，例如缓慢推镜头 (Slow Zoom In)",
|
||
"focusDepth": "焦距与景深，例如浅景深",
|
||
"lighting": "光线，例如冷调低沉照明，屏幕光作为侧逆光",
|
||
"backgroundMusic": "这一段适合的背景音乐氛围",
|
||
"voiceAndSfx": "人声/环境声/音效建议，没有就写无",
|
||
"imagePrompt": "后续生成关键帧时可直接使用的英文图像提示词",
|
||
"motionPrompt": "后续生成视频时可直接使用的英文运动提示词",
|
||
"keyframeIndex": 1
|
||
}
|
||
],
|
||
"shotIdeas": [
|
||
{ "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" }
|
||
],
|
||
"visualMotifs": ["2-4个可复用的视觉母题"],
|
||
"recommendedNextSteps": ["2-4条建议的下游动作"]
|
||
}`,
|
||
"analysisRows 请按镜头时间顺序输出；短视频可以输出 1 行，复杂视频输出 2-6 行。",
|
||
"每行必须覆盖镜号、开始时间、结束时间、时长、画面描述、叙事内容、景别、摄影机角度、摄影机运动、焦距与景深、光线、背景音乐、人声/音效、图像生成提示词、视频运动提示词和关键帧。",
|
||
"imagePrompt 和 motionPrompt 优先使用英文；其他字段使用中文；keyframeIndex 使用 1 开始的序号，对应你看到的关键帧顺序。",
|
||
"如果没有真实视频帧，只能依据参考图和脚本推断，也要正常输出，但措辞要稳健。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
}
|
||
|
||
async function getVideoDurationSeconds(videoUrl: string) {
|
||
try {
|
||
const { stdout } = await execFileAsync(
|
||
"ffprobe",
|
||
[
|
||
"-v",
|
||
"error",
|
||
"-show_entries",
|
||
"format=duration",
|
||
"-of",
|
||
"default=noprint_wrappers=1:nokey=1",
|
||
videoUrl,
|
||
],
|
||
{
|
||
timeout: VIDEO_ANALYSIS_TIMEOUT_MS,
|
||
maxBuffer: 1024 * 1024,
|
||
},
|
||
);
|
||
const duration = Number.parseFloat(String(stdout || "").trim());
|
||
return Number.isFinite(duration) && duration > 0 ? duration : null;
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function buildVideoFrameTimestamps(durationSeconds: number | null, frameCount: number) {
|
||
const safeCount = Math.max(1, frameCount);
|
||
if (!durationSeconds || !Number.isFinite(durationSeconds) || durationSeconds <= 0.5) {
|
||
return Array.from({ length: safeCount }).map((_, index) =>
|
||
Number((0.3 + index * 0.45).toFixed(2)),
|
||
);
|
||
}
|
||
const start = Math.min(0.35, durationSeconds * 0.12);
|
||
const end = Math.max(start + 0.6, durationSeconds * 0.88);
|
||
if (safeCount === 1) return [Number(((start + end) / 2).toFixed(2))];
|
||
const step = (end - start) / (safeCount - 1);
|
||
return Array.from({ length: safeCount }).map((_, index) =>
|
||
Number((start + step * index).toFixed(2)),
|
||
);
|
||
}
|
||
|
||
async function downscaleVideoFrameForVision(inputBuffer: Buffer) {
|
||
return sharp(inputBuffer, { failOn: "none" })
|
||
.rotate()
|
||
.resize({
|
||
width: VIDEO_ANALYSIS_MAX_SIDE,
|
||
height: VIDEO_ANALYSIS_MAX_SIDE,
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
})
|
||
.jpeg({ quality: 82, mozjpeg: true })
|
||
.toBuffer();
|
||
}
|
||
|
||
async function extractVideoFrameDataUrls(videoUrl: string, frameCount = VIDEO_ANALYSIS_FRAME_COUNT) {
|
||
const tempDir = await mkdtemp(path.join(os.tmpdir(), "canvas-video-analysis-"));
|
||
try {
|
||
const durationSeconds = await getVideoDurationSeconds(videoUrl);
|
||
const timestamps = buildVideoFrameTimestamps(durationSeconds, frameCount);
|
||
const frames = await Promise.all(
|
||
timestamps.map(async (timestamp, index) => {
|
||
const outputPath = path.join(tempDir, `frame-${index + 1}.jpg`);
|
||
await execFileAsync(
|
||
"ffmpeg",
|
||
[
|
||
"-hide_banner",
|
||
"-loglevel",
|
||
"error",
|
||
"-ss",
|
||
String(timestamp),
|
||
"-i",
|
||
videoUrl,
|
||
"-frames:v",
|
||
"1",
|
||
"-q:v",
|
||
"4",
|
||
"-y",
|
||
outputPath,
|
||
],
|
||
{
|
||
timeout: VIDEO_ANALYSIS_TIMEOUT_MS,
|
||
maxBuffer: 2 * 1024 * 1024,
|
||
},
|
||
);
|
||
const buffer = await readFile(outputPath);
|
||
const processed = await downscaleVideoFrameForVision(buffer);
|
||
return `data:image/jpeg;base64,${processed.toString("base64")}`;
|
||
}),
|
||
);
|
||
return frames.filter(Boolean);
|
||
} catch (error) {
|
||
console.warn("[Canvas Workflow Analyze] extract video frames failed:", {
|
||
message: error instanceof Error ? error.message : String(error),
|
||
videoUrl,
|
||
});
|
||
return [];
|
||
} finally {
|
||
await rm(tempDir, { force: true, recursive: true }).catch(() => undefined);
|
||
}
|
||
}
|
||
|
||
function buildCharacterScenePrompt(params: {
|
||
nodeTitle?: string;
|
||
prompt: string;
|
||
imageCount: number;
|
||
}) {
|
||
return [
|
||
"你是视频画布工作流里的角色场景提取节点，需要从脚本、设定和参考图里提取角色与场景资产。",
|
||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||
`参考图数量：${params.imageCount}`,
|
||
params.prompt ? `输入内容：\n${params.prompt}` : "没有额外脚本，请根据参考图提取角色和场景线索。",
|
||
"",
|
||
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
||
"JSON 结构如下：",
|
||
`{
|
||
"summary": "一句话总结当前故事世界与资产线索",
|
||
"characters": [
|
||
{
|
||
"name": "角色名",
|
||
"description": "角色外观与身份简介",
|
||
"look": "外貌特征",
|
||
"wardrobe": "服装/道具",
|
||
"traits": ["性格关键词"],
|
||
"signatureAction": "代表动作",
|
||
"prompt": "可直接用于生成角色图/角色视频的提示词"
|
||
}
|
||
],
|
||
"scenes": [
|
||
{
|
||
"name": "场景名",
|
||
"description": "空间、布景和氛围简介",
|
||
"mood": "情绪与气氛",
|
||
"lighting": "布光特征",
|
||
"props": ["核心道具"],
|
||
"prompt": "可直接用于生成场景图/场景视频的提示词"
|
||
}
|
||
],
|
||
"relationships": ["角色与场景或角色之间的关系"],
|
||
"recommendedNextSteps": ["建议连接的后续节点动作"]
|
||
}`,
|
||
"角色和场景都要尽量具体，避免空泛形容词。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
}
|
||
|
||
function buildAgentPlanPrompt(params: {
|
||
nodeTitle?: string;
|
||
prompt: string;
|
||
imageCount: number;
|
||
videoCount: number;
|
||
}) {
|
||
return [
|
||
"你是视频画布里的自定义代理节点，需要把当前创作目标拆成可执行的工作流计划。",
|
||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||
`参考图数量：${params.imageCount}`,
|
||
`视频数量：${params.videoCount}`,
|
||
params.prompt ? `上游需求：\n${params.prompt}` : "没有额外脚本，请基于当前上下文给出稳妥计划。",
|
||
"",
|
||
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
||
"JSON 结构如下：",
|
||
`{
|
||
"summary": "一句话说明这个代理链的目标",
|
||
"agentPlan": ["3-6个执行步骤"],
|
||
"deliverables": ["2-4个预期交付物"],
|
||
"recommendedNodes": ["建议继续连接的节点类型或能力名"]
|
||
}`,
|
||
"步骤要偏工作流视角，而不是泛泛而谈。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
}
|
||
|
||
function buildStoryScriptPrompt(params: {
|
||
nodeTitle?: string;
|
||
prompt: string;
|
||
imageCount: number;
|
||
videoCount: number;
|
||
}) {
|
||
return [
|
||
"你是视频画布里的故事脚本生成节点，需要把用户输入、参考图和参考视频整理成可直接用于后续分镜/生图/生视频的结构化脚本表。",
|
||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||
`参考图数量：${params.imageCount}`,
|
||
`视频数量：${params.videoCount}`,
|
||
params.prompt ? `输入内容：\n${params.prompt}` : "没有额外脚本，请生成一个节奏清晰的短视频脚本。",
|
||
"",
|
||
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
||
"JSON 结构如下：",
|
||
`{
|
||
"summary": "一句话概括故事主题、情绪和视频方向，不超过120字",
|
||
"analysisRows": [
|
||
{
|
||
"shot": "01",
|
||
"duration": "5s",
|
||
"scene": "场景名或地点，如 盛唐·金銮殿·日",
|
||
"sceneDescription": "画面内容、人物动作和剧情推进，不要重复 scene 字段",
|
||
"shotSize": "景别，如中景、特写、全景、大远景",
|
||
"lighting": "光影氛围，如暖金主光带阴影，气氛紧张压抑",
|
||
"dialogue": "对白或旁白，没有就写 +",
|
||
"backgroundMusic": "背景音乐或情绪氛围，没有就写无",
|
||
"voiceAndSfx": "环境声、动作声或音效，没有就写无",
|
||
"cameraMovement": "运镜，如缓慢推镜头、固定、横移、上摇、环绕轨道跟拍",
|
||
"imagePrompt": "后续生成该镜头关键帧时可直接使用的中文图像提示词",
|
||
"motionPrompt": "后续生成视频时可直接使用的中文视频运动提示词"
|
||
}
|
||
],
|
||
"assetCandidates": {
|
||
"style": "全局视觉风格，如 盛唐·国风二次元",
|
||
"characters": ["需要生成的角色资产名"],
|
||
"scenes": ["需要生成的场景资产名"],
|
||
"props": ["需要生成的道具资产名"]
|
||
},
|
||
"shotIdeas": [
|
||
{ "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" }
|
||
],
|
||
"recommendedNextSteps": ["2-4条建议的下游动作"]
|
||
}`,
|
||
"analysisRows 必须直接输出 18 行，按镜头时间顺序编号 01-18；每行默认 5s 或 6s，剧情高潮可用 6s。",
|
||
"每行要覆盖竞品脚本表同等信息：镜号、时长、画面描述、景别、光影氛围、对白·旁白、音效、运镜、最终图像提示词和视频运动提示词。",
|
||
"画面描述中用 @角色名 或 @场景/道具名 标出可复用资产，例如 @沈昭昭、@金銮殿、@侍卫长刀。",
|
||
"imagePrompt 要像可直接生成分镜图的中文提示词，包含主体、环境、构图、光影、风格与一致性要求；motionPrompt 要按起始状态、动作过程、结束状态、音效组织。",
|
||
"assetCandidates 根据 18 镜头抽取 3-6 个角色、2-5 个场景和 1-4 个道具，方便后续准备资产。",
|
||
"每行要像专业视频画布脚本表：场景具体、画面可生成、声音可执行、镜头节奏明确，不要只写剧情摘要。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
}
|
||
|
||
export async function POST(request: Request) {
|
||
try {
|
||
const body = await request.json().catch(() => ({}));
|
||
const parsed = requestSchema.safeParse(body);
|
||
if (!parsed.success) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "请求参数不合法" },
|
||
{ status: 400 }
|
||
);
|
||
}
|
||
|
||
const prompt = normalizeText(parsed.data.prompt);
|
||
const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4);
|
||
const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4);
|
||
const extractedVideoFrames =
|
||
parsed.data.mode === "video-analysis" && videoUrls.length > 0
|
||
? await extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT)
|
||
: [];
|
||
const preferredModelRef = normalizeText(parsed.data.preferredModelRef || parsed.data.model);
|
||
if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "请至少提供脚本、图片或视频上下文" },
|
||
{ status: 400 }
|
||
);
|
||
}
|
||
|
||
const session = await auth();
|
||
if (!session?.user?.id) {
|
||
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
|
||
}
|
||
|
||
const [allModels, agentModel] = await Promise.all([
|
||
getEnabledAiModelsWithProvider(false),
|
||
getAgentAiModelWithProvider(),
|
||
]);
|
||
const compatibleModels = allModels.filter(
|
||
(item) => !item.model.isSubModel && item.model.isEnabled
|
||
);
|
||
|
||
const preferredModel =
|
||
preferredModelRef
|
||
? compatibleModels.find(
|
||
(item) =>
|
||
item.model.id === preferredModelRef ||
|
||
item.model.modelId === preferredModelRef,
|
||
)
|
||
: null;
|
||
|
||
const hasVisionInputs = imageUrls.length > 0 || extractedVideoFrames.length > 0;
|
||
|
||
let selectedModel =
|
||
preferredModel && (!hasVisionInputs || supportsVision(preferredModel.model))
|
||
? preferredModel
|
||
: hasVisionInputs &&
|
||
agentModel &&
|
||
supportsVision(agentModel.model)
|
||
? agentModel
|
||
: hasVisionInputs
|
||
? compatibleModels.find((item) => supportsVision(item.model))
|
||
: agentModel || compatibleModels[0];
|
||
|
||
if (!selectedModel) {
|
||
selectedModel = compatibleModels[0];
|
||
}
|
||
|
||
if (!selectedModel) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "当前站点没有可用的分析模型" },
|
||
{ status: 500 }
|
||
);
|
||
}
|
||
|
||
const useVision = hasVisionInputs && supportsVision(selectedModel.model);
|
||
const codingModelId = getAiModelAgentCodingModelId(selectedModel.model);
|
||
const model = await getLanguageModelById(selectedModel.model.id, codingModelId);
|
||
if (!model) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "分析模型初始化失败" },
|
||
{ status: 500 }
|
||
);
|
||
}
|
||
|
||
const promptText =
|
||
parsed.data.mode === "video-analysis"
|
||
? buildVideoAnalysisPrompt({
|
||
frameCount: extractedVideoFrames.length,
|
||
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
||
prompt,
|
||
imageCount: imageUrls.length,
|
||
videoCount: videoUrls.length,
|
||
})
|
||
: parsed.data.mode === "character-scene-extraction"
|
||
? buildCharacterScenePrompt({
|
||
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
||
prompt,
|
||
imageCount: imageUrls.length,
|
||
})
|
||
: parsed.data.mode === "story-script"
|
||
? buildStoryScriptPrompt({
|
||
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
||
prompt,
|
||
imageCount: imageUrls.length,
|
||
videoCount: videoUrls.length,
|
||
})
|
||
: buildAgentPlanPrompt({
|
||
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
||
prompt,
|
||
imageCount: imageUrls.length,
|
||
videoCount: videoUrls.length,
|
||
});
|
||
|
||
const inlinedImageReferences = useVision
|
||
? (
|
||
await inlineVisionImages(imageUrls, {
|
||
maxImages: 4,
|
||
maxSide: 960,
|
||
timeoutMs: 12000,
|
||
retries: 1,
|
||
loggerPrefix: "[Canvas Workflow Analyze]",
|
||
})
|
||
).inlineDataUrls
|
||
: [];
|
||
const referenceImages = useVision
|
||
? [...inlinedImageReferences, ...extractedVideoFrames].slice(0, 6)
|
||
: [];
|
||
|
||
const { text } = await generateText({
|
||
model,
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{ type: "text", text: promptText },
|
||
...referenceImages.map((image) => ({ type: "image" as const, image })),
|
||
],
|
||
},
|
||
],
|
||
temperature: 0.2,
|
||
maxTokens: parsed.data.mode === "story-script" ? 4200 : parsed.data.mode === "video-analysis" ? 2200 : 1600,
|
||
} as any);
|
||
|
||
const parsedJson = extractJsonBlock<Record<string, unknown>>(text) || {};
|
||
const summary =
|
||
normalizeText(parsedJson.summary) ||
|
||
normalizeText(text).slice(0, 160);
|
||
|
||
if (parsed.data.mode === "video-analysis") {
|
||
const analysisRows = normalizeVideoAnalysisRows(
|
||
parsedJson.analysisRows,
|
||
extractedVideoFrames,
|
||
);
|
||
const shotIdeas =
|
||
normalizeShotIdeas(parsedJson.shotIdeas).length > 0
|
||
? normalizeShotIdeas(parsedJson.shotIdeas)
|
||
: analysisRows.map((row, index) => ({
|
||
title: `镜头 ${index + 1}`,
|
||
description: row.sceneDescription,
|
||
}));
|
||
return NextResponse.json({
|
||
success: true,
|
||
data: {
|
||
summary,
|
||
analysisRows,
|
||
framePreviews: extractedVideoFrames,
|
||
highlights: normalizeStringList(parsedJson.highlights, 6),
|
||
shotIdeas,
|
||
visualMotifs: normalizeStringList(parsedJson.visualMotifs, 6),
|
||
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
|
||
model: selectedModel.model.displayName,
|
||
raw: text,
|
||
},
|
||
});
|
||
}
|
||
|
||
if (parsed.data.mode === "character-scene-extraction") {
|
||
return NextResponse.json({
|
||
success: true,
|
||
data: {
|
||
summary,
|
||
characters: normalizeCharacters(parsedJson.characters),
|
||
scenes: normalizeScenes(parsedJson.scenes),
|
||
relationships: normalizeStringList(parsedJson.relationships, 8),
|
||
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
|
||
model: selectedModel.model.displayName,
|
||
raw: text,
|
||
},
|
||
});
|
||
}
|
||
|
||
if (parsed.data.mode === "story-script") {
|
||
const analysisRows = expandStoryScriptRowsToTarget(normalizeStoryScriptRows(
|
||
parsedJson.analysisRows || parsedJson.storyRows || parsedJson.rows || parsedJson.shots,
|
||
));
|
||
return NextResponse.json({
|
||
success: true,
|
||
data: {
|
||
summary,
|
||
analysisRows,
|
||
shotIdeas:
|
||
normalizeShotIdeas(parsedJson.shotIdeas).length > 0
|
||
? normalizeShotIdeas(parsedJson.shotIdeas)
|
||
: analysisRows.map((row, index) => ({
|
||
title: `镜头 ${index + 1}`,
|
||
description: row.sceneDescription,
|
||
})),
|
||
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
|
||
model: selectedModel.model.displayName,
|
||
raw: text,
|
||
},
|
||
});
|
||
}
|
||
|
||
return NextResponse.json({
|
||
success: true,
|
||
data: {
|
||
summary,
|
||
agentPlan: normalizeStringList(parsedJson.agentPlan, 8),
|
||
deliverables: normalizeStringList(parsedJson.deliverables, 6),
|
||
recommendedNodes: normalizeStringList(parsedJson.recommendedNodes, 6),
|
||
model: selectedModel.model.displayName,
|
||
raw: text,
|
||
},
|
||
});
|
||
} catch (error) {
|
||
console.error("[Canvas Workflow Analyze] Error:", error);
|
||
return NextResponse.json(
|
||
{
|
||
success: false,
|
||
error: error instanceof Error ? error.message : "工作流分析失败",
|
||
},
|
||
{ status: 500 }
|
||
);
|
||
}
|
||
}
|