810 lines
31 KiB
TypeScript
810 lines
31 KiB
TypeScript
import { reconstructionPrompt, parseReconstructedScene } from "./director-reconstruction";
|
||
import { normalizeVideoAnalysisRows } from "./video-story";
|
||
import { analyzeAliyunVideo, isAliyunVideoModel, supportsVideoSound } from "./aliyun-video-analysis";
|
||
import "server-only";
|
||
import { normalizeStoryScriptRows } from "@/lib/video-canvas/story-script";
|
||
import { isCanvasLanguageModel, isCanvasVisionModel } from "@/lib/video-canvas/model-matching";
|
||
import { generateText } from "ai";
|
||
import { execFile } from "node:child_process";
|
||
import { promisify } from "node:util";
|
||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||
import os from "node:os";
|
||
import path from "node:path";
|
||
import { NextResponse } from "next/server";
|
||
import sharp from "sharp";
|
||
import { z } from "zod";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { inlineVisionImages } from "@/lib/ai/vision-inline";
|
||
import { getAiModelAgentCodingModelId, isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||
import { getAgentAiModelWithProvider, getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||
|
||
const execFileAsync = promisify(execFile);
|
||
const VIDEO_ANALYSIS_FRAME_COUNT = 4;
|
||
const VIDEO_ANALYSIS_MAX_SIDE = 960;
|
||
const VIDEO_ANALYSIS_TIMEOUT_MS = 30000;
|
||
|
||
const requestSchema = z.object({
|
||
mode: z.enum([
|
||
"video-analysis",
|
||
"character-scene-extraction",
|
||
"story-script",
|
||
"agent-plan",
|
||
"director-reconstruct",
|
||
]),
|
||
prompt: z.string().trim().optional(),
|
||
imageUrls: z.array(z.string().trim()).optional(),
|
||
videoUrls: z.array(z.string().trim()).optional(),
|
||
analysisDimensions: z.object({
|
||
shots: z.boolean().optional(),
|
||
motion: z.boolean().optional(),
|
||
music: z.boolean().optional(),
|
||
}).optional(),
|
||
nodeTitle: z.string().trim().optional(),
|
||
preferredModelRef: z.string().trim().optional(),
|
||
model: z.string().trim().optional(),
|
||
});
|
||
|
||
function sanitizeUrls(value: unknown, limit = 6) {
|
||
if (!Array.isArray(value)) return [];
|
||
return value
|
||
.map((item) => String(item || "").trim())
|
||
.filter((url) => /^https?:\/\//i.test(url))
|
||
.slice(0, limit);
|
||
}
|
||
|
||
function supportsVision(model: {
|
||
supportsVision?: boolean | null;
|
||
capabilities?: unknown;
|
||
defaultParams?: unknown;
|
||
}) {
|
||
return (
|
||
Boolean(model.supportsVision) ||
|
||
isAiModelMultimodal({
|
||
capabilities: model.capabilities,
|
||
defaultParams: model.defaultParams,
|
||
})
|
||
);
|
||
}
|
||
|
||
function extractJsonBlock<T = Record<string, unknown>>(text: string): T | null {
|
||
const matched = String(text || "").match(/\{[\s\S]*\}/);
|
||
if (!matched) return null;
|
||
try {
|
||
return JSON.parse(matched[0]) as T;
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function normalizeText(value: unknown) {
|
||
return String(value || "").trim();
|
||
}
|
||
|
||
function normalizeStringList(value: unknown, limit = 8) {
|
||
if (!Array.isArray(value)) return [];
|
||
return Array.from(
|
||
new Set(
|
||
value
|
||
.map((item) => String(item || "").trim())
|
||
.filter(Boolean)
|
||
.slice(0, limit),
|
||
),
|
||
);
|
||
}
|
||
|
||
function normalizeShotIdeas(value: unknown) {
|
||
if (!Array.isArray(value)) return [];
|
||
return value
|
||
.map((item, index) => {
|
||
const raw =
|
||
item && typeof item === "object" && !Array.isArray(item)
|
||
? (item as Record<string, unknown>)
|
||
: {};
|
||
const title = normalizeText(raw.title || `镜头 ${index + 1}`);
|
||
const description = normalizeText(raw.description || raw.prompt || raw.summary);
|
||
if (!title && !description) return null;
|
||
return {
|
||
title: title || `镜头 ${index + 1}`,
|
||
description,
|
||
};
|
||
})
|
||
.filter((item): item is { title: string; description: string } => Boolean(item))
|
||
.slice(0, 6);
|
||
}
|
||
|
||
function normalizeCharacters(value: unknown) {
|
||
if (!Array.isArray(value)) return [];
|
||
return value
|
||
.map((item) => {
|
||
const raw =
|
||
item && typeof item === "object" && !Array.isArray(item)
|
||
? (item as Record<string, unknown>)
|
||
: {};
|
||
const name = normalizeText(raw.name || raw.title);
|
||
const description = normalizeText(raw.description || raw.summary);
|
||
const prompt = normalizeText(raw.prompt);
|
||
if (!name && !description && !prompt) return null;
|
||
return {
|
||
name: name || "未命名角色",
|
||
description,
|
||
prompt,
|
||
look: normalizeText(raw.look) || undefined,
|
||
wardrobe: normalizeText(raw.wardrobe) || undefined,
|
||
signatureAction: normalizeText(raw.signatureAction) || undefined,
|
||
traits: normalizeStringList(raw.traits, 6),
|
||
};
|
||
})
|
||
.filter((item): item is NonNullable<typeof item> => Boolean(item))
|
||
.slice(0, 8);
|
||
}
|
||
|
||
function normalizeScenes(value: unknown) {
|
||
if (!Array.isArray(value)) return [];
|
||
return value
|
||
.map((item) => {
|
||
const raw =
|
||
item && typeof item === "object" && !Array.isArray(item)
|
||
? (item as Record<string, unknown>)
|
||
: {};
|
||
const name = normalizeText(raw.name || raw.title);
|
||
const description = normalizeText(raw.description || raw.summary);
|
||
const prompt = normalizeText(raw.prompt);
|
||
if (!name && !description && !prompt) return null;
|
||
return {
|
||
name: name || "未命名场景",
|
||
description,
|
||
prompt,
|
||
mood: normalizeText(raw.mood) || undefined,
|
||
lighting: normalizeText(raw.lighting) || undefined,
|
||
props: normalizeStringList(raw.props, 6),
|
||
};
|
||
})
|
||
.filter((item): item is NonNullable<typeof item> => Boolean(item))
|
||
.slice(0, 8);
|
||
}
|
||
|
||
function buildVideoAnalysisPrompt(params: {
|
||
nodeTitle?: string;
|
||
prompt: string;
|
||
frameCount: number;
|
||
imageCount: number;
|
||
videoCount: number;
|
||
directVideo: boolean;
|
||
soundAvailable: boolean;
|
||
}) {
|
||
return [
|
||
"你是视频画布工作流里的视频分析节点，需要为后续分镜和生成节点提供结构化输出。",
|
||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||
`上游参考图数量：${params.imageCount}`,
|
||
`上游视频数量：${params.videoCount}`,
|
||
params.directVideo
|
||
? "已直接提供原视频文件，请根据完整时间轴识别镜头，不要只依据少量预览关键帧。"
|
||
: `已提供按时间顺序抽取的视频关键帧数量：${params.frameCount}`,
|
||
params.soundAvailable
|
||
? "可分析原视频声音；背景音乐、人声和音效须以实际听到的内容为准。"
|
||
: "当前模型只能看画面，不能听声音；背景音乐和人声/音效字段必须留空，禁止推测。",
|
||
params.prompt
|
||
? `上游脚本/说明：\n${params.prompt}`
|
||
: "上游没有额外脚本，请重点依据参考画面分析。",
|
||
"",
|
||
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
||
"JSON 结构如下：",
|
||
`{
|
||
"summary": "一句话总结当前视频/镜头风格，不超过120字",
|
||
"highlights": ["3-5条镜头分析要点"],
|
||
"analysisRows": [
|
||
{
|
||
"shot": "1",
|
||
"startTime": "0",
|
||
"endTime": "5",
|
||
"duration": "5",
|
||
"sceneDescription": "画面描述，说明主体、环境、动作、构图和色调",
|
||
"narrative": "叙事内容，说明这一镜头承担的剧情/情绪/信息作用",
|
||
"shotSize": "景别，例如中景至近景",
|
||
"cameraAngle": "摄影机角度，例如平视",
|
||
"cameraMovement": "摄影机运动，例如缓慢推镜头 (Slow Zoom In)",
|
||
"focusDepth": "焦距与景深，例如浅景深",
|
||
"lighting": "光线，例如冷调低沉照明，屏幕光作为侧逆光",
|
||
"backgroundMusic": "这一段适合的背景音乐氛围",
|
||
"voiceAndSfx": "人声/环境声/音效建议，没有就写无",
|
||
"imagePrompt": "后续生成关键帧时可直接使用的英文图像提示词",
|
||
"motionPrompt": "后续生成视频时可直接使用的英文运动提示词",
|
||
"keyframeIndex": 1
|
||
}
|
||
],
|
||
"shotIdeas": [
|
||
{ "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" }
|
||
],
|
||
"visualMotifs": ["2-4个可复用的视觉母题"],
|
||
"recommendedNextSteps": ["2-4条建议的下游动作"]
|
||
}`,
|
||
"analysisRows 请按镜头时间顺序输出；短视频可以输出 1 行，复杂视频输出 2-6 行。",
|
||
"每行必须覆盖镜号、开始时间、结束时间、时长、画面描述、叙事内容、景别、摄影机角度、摄影机运动、焦距与景深、光线、背景音乐、人声/音效、图像生成提示词、视频运动提示词和关键帧。",
|
||
"imagePrompt 和 motionPrompt 优先使用英文；其他字段使用中文；keyframeIndex 使用 1 开始的序号，对应你看到的关键帧顺序。",
|
||
"没有实际看到或听到的内容必须留空，不可依据脚本虚构成视频事实。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
}
|
||
|
||
async function getVideoDurationSeconds(videoUrl: string) {
|
||
try {
|
||
const { stdout } = await execFileAsync(
|
||
"ffprobe",
|
||
[
|
||
"-v",
|
||
"error",
|
||
"-show_entries",
|
||
"format=duration",
|
||
"-of",
|
||
"default=noprint_wrappers=1:nokey=1",
|
||
videoUrl,
|
||
],
|
||
{
|
||
timeout: VIDEO_ANALYSIS_TIMEOUT_MS,
|
||
maxBuffer: 1024 * 1024,
|
||
},
|
||
);
|
||
const duration = Number.parseFloat(String(stdout || "").trim());
|
||
return Number.isFinite(duration) && duration > 0 ? duration : null;
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function buildVideoFrameTimestamps(durationSeconds: number | null, frameCount: number) {
|
||
const safeCount = Math.max(1, frameCount);
|
||
if (!durationSeconds || !Number.isFinite(durationSeconds) || durationSeconds <= 0.5) {
|
||
return Array.from({ length: safeCount }).map((_, index) =>
|
||
Number((0.3 + index * 0.45).toFixed(2)),
|
||
);
|
||
}
|
||
const start = Math.min(0.35, durationSeconds * 0.12);
|
||
const end = Math.max(start + 0.6, durationSeconds * 0.88);
|
||
if (safeCount === 1) return [Number(((start + end) / 2).toFixed(2))];
|
||
const step = (end - start) / (safeCount - 1);
|
||
return Array.from({ length: safeCount }).map((_, index) =>
|
||
Number((start + step * index).toFixed(2)),
|
||
);
|
||
}
|
||
|
||
async function downscaleVideoFrameForVision(inputBuffer: Buffer) {
|
||
return sharp(inputBuffer, { failOn: "none" })
|
||
.rotate()
|
||
.resize({
|
||
width: VIDEO_ANALYSIS_MAX_SIDE,
|
||
height: VIDEO_ANALYSIS_MAX_SIDE,
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
})
|
||
.jpeg({ quality: 82, mozjpeg: true })
|
||
.toBuffer();
|
||
}
|
||
|
||
async function extractVideoFrameDataUrls(
|
||
videoUrl: string,
|
||
frameCount = VIDEO_ANALYSIS_FRAME_COUNT,
|
||
) {
|
||
const tempDir = await mkdtemp(path.join(os.tmpdir(), "canvas-video-analysis-"));
|
||
try {
|
||
const durationSeconds = await getVideoDurationSeconds(videoUrl);
|
||
if (durationSeconds && durationSeconds <= 120) {
|
||
// Decode short clips in one pass. Four concurrent HTTP seeks are slow and
|
||
// some CDNs reject later range requests even when the video is valid.
|
||
await execFileAsync("ffmpeg", [
|
||
"-hide_banner", "-loglevel", "error", "-i", videoUrl,
|
||
"-vf", `fps=${(frameCount / durationSeconds).toFixed(6)}`,
|
||
"-frames:v", String(frameCount), "-q:v", "4", "-y",
|
||
path.join(tempDir, "frame-%d.jpg"),
|
||
], { timeout: VIDEO_ANALYSIS_TIMEOUT_MS, maxBuffer: 2 * 1024 * 1024 });
|
||
const frames = await Promise.allSettled(
|
||
Array.from({ length: frameCount }, async (_, index) => {
|
||
const bytes = await readFile(path.join(tempDir, `frame-${index + 1}.jpg`));
|
||
const processed = await downscaleVideoFrameForVision(bytes);
|
||
return `data:image/jpeg;base64,${processed.toString("base64")}`;
|
||
}),
|
||
);
|
||
return frames.flatMap((result) => result.status === "fulfilled" ? [result.value] : []);
|
||
}
|
||
const timestamps = buildVideoFrameTimestamps(durationSeconds, frameCount);
|
||
const frames = await Promise.allSettled(
|
||
timestamps.map(async (timestamp, index) => {
|
||
const outputPath = path.join(tempDir, `frame-${index + 1}.jpg`);
|
||
await execFileAsync(
|
||
"ffmpeg",
|
||
[
|
||
"-hide_banner",
|
||
"-loglevel",
|
||
"error",
|
||
"-ss",
|
||
String(timestamp),
|
||
"-i",
|
||
videoUrl,
|
||
"-frames:v",
|
||
"1",
|
||
"-q:v",
|
||
"4",
|
||
"-y",
|
||
outputPath,
|
||
],
|
||
{
|
||
timeout: VIDEO_ANALYSIS_TIMEOUT_MS,
|
||
maxBuffer: 2 * 1024 * 1024,
|
||
},
|
||
);
|
||
const buffer = await readFile(outputPath);
|
||
const processed = await downscaleVideoFrameForVision(buffer);
|
||
return `data:image/jpeg;base64,${processed.toString("base64")}`;
|
||
}),
|
||
);
|
||
const completed = frames.flatMap((result) => result.status === "fulfilled" ? [result.value] : []);
|
||
if (completed.length < frames.length) {
|
||
console.warn("[Canvas Workflow Analyze] some preview frames could not be extracted", {
|
||
completed: completed.length,
|
||
requested: frames.length,
|
||
});
|
||
}
|
||
return completed;
|
||
} catch (error) {
|
||
console.warn("[Canvas Workflow Analyze] extract video frames failed:", {
|
||
message: error instanceof Error ? error.message : String(error),
|
||
videoUrl,
|
||
});
|
||
return [];
|
||
} finally {
|
||
await rm(tempDir, { force: true, recursive: true }).catch(() => undefined);
|
||
}
|
||
}
|
||
|
||
function buildCharacterScenePrompt(params: {
|
||
nodeTitle?: string;
|
||
prompt: string;
|
||
imageCount: number;
|
||
}) {
|
||
return [
|
||
"你是视频画布工作流里的角色场景提取节点，需要从脚本、设定和参考图里提取角色与场景资产。",
|
||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||
`参考图数量：${params.imageCount}`,
|
||
params.prompt
|
||
? `输入内容：\n${params.prompt}`
|
||
: "没有额外脚本，请根据参考图提取角色和场景线索。",
|
||
"",
|
||
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
||
"JSON 结构如下：",
|
||
`{
|
||
"summary": "一句话总结当前故事世界与资产线索",
|
||
"characters": [
|
||
{
|
||
"name": "角色名",
|
||
"description": "角色外观与身份简介",
|
||
"look": "外貌特征",
|
||
"wardrobe": "服装/道具",
|
||
"traits": ["性格关键词"],
|
||
"signatureAction": "代表动作",
|
||
"prompt": "可直接用于生成角色图/角色视频的提示词"
|
||
}
|
||
],
|
||
"scenes": [
|
||
{
|
||
"name": "场景名",
|
||
"description": "空间、布景和氛围简介",
|
||
"mood": "情绪与气氛",
|
||
"lighting": "布光特征",
|
||
"props": ["核心道具"],
|
||
"prompt": "可直接用于生成场景图/场景视频的提示词"
|
||
}
|
||
],
|
||
"relationships": ["角色与场景或角色之间的关系"],
|
||
"recommendedNextSteps": ["建议连接的后续节点动作"]
|
||
}`,
|
||
"角色和场景都要尽量具体，避免空泛形容词。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
}
|
||
|
||
function buildAgentPlanPrompt(params: {
|
||
nodeTitle?: string;
|
||
prompt: string;
|
||
imageCount: number;
|
||
videoCount: number;
|
||
}) {
|
||
return [
|
||
"你是视频画布里的自定义代理节点，需要把当前创作目标拆成可执行的工作流计划。",
|
||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||
`参考图数量：${params.imageCount}`,
|
||
`视频数量：${params.videoCount}`,
|
||
params.prompt ? `上游需求：\n${params.prompt}` : "没有额外脚本，请基于当前上下文给出稳妥计划。",
|
||
"",
|
||
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
||
"JSON 结构如下：",
|
||
`{
|
||
"summary": "一句话说明这个代理链的目标",
|
||
"agentPlan": ["3-6个执行步骤"],
|
||
"deliverables": ["2-4个预期交付物"],
|
||
"recommendedNodes": ["建议继续连接的节点类型或能力名"]
|
||
}`,
|
||
"步骤要偏工作流视角，而不是泛泛而谈。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
}
|
||
|
||
function buildStoryScriptPrompt(params: {
|
||
nodeTitle?: string;
|
||
prompt: string;
|
||
imageCount: number;
|
||
videoCount: number;
|
||
}) {
|
||
return [
|
||
"你是视频画布里的故事脚本生成节点，需要把用户输入、参考图和参考视频整理成可直接用于后续分镜/生图/生视频的结构化脚本表。",
|
||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||
`参考图数量：${params.imageCount}`,
|
||
`视频数量：${params.videoCount}`,
|
||
params.prompt
|
||
? `输入内容：\n${params.prompt}`
|
||
: "没有额外脚本，请生成一个节奏清晰的短视频脚本。",
|
||
"",
|
||
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
||
"JSON 结构如下：",
|
||
`{
|
||
"summary": "一句话概括故事主题、情绪和视频方向，不超过120字",
|
||
"analysisRows": [
|
||
{
|
||
"shot": "01",
|
||
"duration": "5s",
|
||
"scene": "场景名或地点，如 盛唐·金銮殿·日",
|
||
"sceneDescription": "画面内容、人物动作和剧情推进，不要重复 scene 字段",
|
||
"shotSize": "景别，如中景、特写、全景、大远景",
|
||
"lighting": "光影氛围，如暖金主光带阴影，气氛紧张压抑",
|
||
"dialogue": "对白或旁白，没有就写 +",
|
||
"backgroundMusic": "背景音乐或情绪氛围，没有就写无",
|
||
"voiceAndSfx": "环境声、动作声或音效，没有就写无",
|
||
"cameraMovement": "运镜，如缓慢推镜头、固定、横移、上摇、环绕轨道跟拍",
|
||
"imagePrompt": "后续生成该镜头关键帧时可直接使用的中文图像提示词",
|
||
"motionPrompt": "后续生成视频时可直接使用的中文视频运动提示词"
|
||
}
|
||
],
|
||
"assetCandidates": {
|
||
"style": "全局视觉风格，如 盛唐·国风二次元",
|
||
"characters": ["需要生成的角色资产名"],
|
||
"scenes": ["需要生成的场景资产名"],
|
||
"props": ["需要生成的道具资产名"]
|
||
},
|
||
"shotIdeas": [
|
||
{ "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" }
|
||
],
|
||
"recommendedNextSteps": ["2-4条建议的下游动作"]
|
||
}`,
|
||
"优先严格遵守用户指定的镜头数量和总时长；未指定时生成 6 个镜头，最多 18 个。镜头时长之和必须符合用户要求，8 秒短片不得扩写成几十秒。只生成实际需要的镜头，不重复凑数。",
|
||
"每行要覆盖竞品脚本表同等信息：镜号、时长、画面描述、景别、光影氛围、对白·旁白、音效、运镜、最终图像提示词和视频运动提示词。",
|
||
"画面描述中用 @角色名 或 @场景/道具名 标出可复用资产，例如 @沈昭昭、@金銮殿、@侍卫长刀。",
|
||
"imagePrompt 要像可直接生成分镜图的中文提示词，包含主体、环境、构图、光影、风格与一致性要求；motionPrompt 要按起始状态、动作过程、结束状态、音效组织。",
|
||
"assetCandidates 只提取脚本中确实出现的角色、场景和道具；产品视频没有人物时 characters 必须为空，禁止为凑数虚构角色。",
|
||
"每行要像专业视频画布脚本表：场景具体、画面可生成、声音可执行、镜头节奏明确，不要只写剧情摘要。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
}
|
||
|
||
export async function executeWorkflowAnalysisResponse(body: unknown, signal?: AbortSignal) {
|
||
try {
|
||
const parsed = requestSchema.safeParse(body);
|
||
if (!parsed.success) {
|
||
return NextResponse.json({ success: false, error: "请求参数不合法" }, { status: 400 });
|
||
}
|
||
|
||
const prompt = normalizeText(parsed.data.prompt);
|
||
const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4);
|
||
const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4);
|
||
if (parsed.data.mode === "video-analysis" && videoUrls.length !== 1) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "视频拉片需要恰好一个可访问的视频地址" },
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
const preferredModelRef = normalizeText(parsed.data.preferredModelRef || parsed.data.model);
|
||
if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "请至少提供脚本、图片或视频上下文" },
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
|
||
const [allModels, agentModel] = await Promise.all([
|
||
getEnabledAiModelsWithProvider(false),
|
||
getAgentAiModelWithProvider(),
|
||
]);
|
||
const hasVisionInputs = imageUrls.length > 0 || videoUrls.length > 0;
|
||
const wantsSound = parsed.data.mode === "video-analysis" &&
|
||
parsed.data.analysisDimensions?.music === true;
|
||
const compatibleModels = allModels.filter((item) => {
|
||
const descriptor = { ...item.model, providerType: item.provider.type };
|
||
return (
|
||
!item.model.isSubModel &&
|
||
item.model.isEnabled &&
|
||
(hasVisionInputs ? isCanvasVisionModel(descriptor) : isCanvasLanguageModel(descriptor))
|
||
);
|
||
});
|
||
const requestedRef = preferredModelRef === "auto" ? "" : preferredModelRef;
|
||
const preferredModel = requestedRef
|
||
? compatibleModels.find(
|
||
(item) => item.model.id === requestedRef || item.model.modelId === requestedRef,
|
||
)
|
||
: undefined;
|
||
if (requestedRef && !preferredModel) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "所选模型不可用或不支持当前分析输入，请重新选择分析模型" },
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
const selectedModel =
|
||
(wantsSound
|
||
? compatibleModels.find((item) =>
|
||
item.provider.type === "alibaba" &&
|
||
supportsVideoSound(item.model.modelId),
|
||
)
|
||
: preferredModel) ||
|
||
(parsed.data.mode === "video-analysis"
|
||
? compatibleModels.find((item) =>
|
||
item.provider.type === "alibaba" && item.model.modelId === "qwen3.8-max",
|
||
)
|
||
: undefined) ||
|
||
compatibleModels.find((item) => item.model.id === agentModel?.model.id) ||
|
||
compatibleModels[0];
|
||
if (wantsSound && !selectedModel?.model.modelId?.includes("omni")) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "音乐/人声拉片需要启用阿里云 qwen3.8-omni-flash 视频音频理解模型" },
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
if (!selectedModel) {
|
||
return NextResponse.json(
|
||
{
|
||
success: false,
|
||
error: hasVisionInputs ? "当前没有可用的视觉理解模型" : "当前没有可用的文字分析模型",
|
||
},
|
||
{ status: 400 },
|
||
);
|
||
}
|
||
|
||
const directVideo = parsed.data.mode === "video-analysis" && videoUrls.length > 0 &&
|
||
isAliyunVideoModel(selectedModel.model.modelId, selectedModel.provider.type);
|
||
const soundAvailable = directVideo && supportsVideoSound(selectedModel.model.modelId);
|
||
const extractedVideoFrames = !directVideo && parsed.data.mode === "video-analysis"
|
||
? await extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT)
|
||
: [];
|
||
// The official model receives the full video. Extract UI previews alongside its request.
|
||
const framePreviewsPromise = directVideo
|
||
? extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT)
|
||
: Promise.resolve(extractedVideoFrames);
|
||
if (parsed.data.mode === "video-analysis" && videoUrls.length > 0 &&
|
||
!directVideo && extractedVideoFrames.length === 0) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "无法读取视频关键帧，请检查视频地址或选择阿里云视频理解模型" },
|
||
{ status: 422 },
|
||
);
|
||
}
|
||
const useVision = hasVisionInputs && supportsVision(selectedModel.model);
|
||
const codingModelId = getAiModelAgentCodingModelId(selectedModel.model);
|
||
const model = directVideo ? null : await getLanguageModelById(selectedModel.model.id, codingModelId);
|
||
if (!directVideo && !model) {
|
||
return NextResponse.json({ success: false, error: "分析模型初始化失败" }, { status: 500 });
|
||
}
|
||
|
||
const promptText =
|
||
parsed.data.mode === "director-reconstruct"
|
||
? reconstructionPrompt(prompt)
|
||
: parsed.data.mode === "video-analysis"
|
||
? buildVideoAnalysisPrompt({
|
||
frameCount: extractedVideoFrames.length,
|
||
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
||
prompt,
|
||
imageCount: imageUrls.length,
|
||
videoCount: videoUrls.length,
|
||
directVideo,
|
||
soundAvailable,
|
||
})
|
||
: parsed.data.mode === "character-scene-extraction"
|
||
? buildCharacterScenePrompt({
|
||
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
||
prompt,
|
||
imageCount: imageUrls.length,
|
||
})
|
||
: parsed.data.mode === "story-script"
|
||
? buildStoryScriptPrompt({
|
||
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
||
prompt,
|
||
imageCount: imageUrls.length,
|
||
videoCount: videoUrls.length,
|
||
})
|
||
: buildAgentPlanPrompt({
|
||
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
||
prompt,
|
||
imageCount: imageUrls.length,
|
||
videoCount: videoUrls.length,
|
||
});
|
||
|
||
const inlinedImageReferences = useVision && !directVideo
|
||
? (
|
||
await inlineVisionImages(imageUrls, {
|
||
maxImages: 4,
|
||
maxSide: 960,
|
||
timeoutMs: 12000,
|
||
retries: 1,
|
||
loggerPrefix: "[Canvas Workflow Analyze]",
|
||
})
|
||
).inlineDataUrls
|
||
: [];
|
||
const referenceImages = useVision
|
||
? [...inlinedImageReferences, ...extractedVideoFrames].slice(0, 6)
|
||
: [];
|
||
|
||
if (parsed.data.mode === "director-reconstruct" && !referenceImages.length)
|
||
throw new Error("识图重建需要一张可读取的图片");
|
||
const generated = directVideo
|
||
? { text: await analyzeAliyunVideo({
|
||
apiKey: selectedModel.provider.apiKey,
|
||
baseUrl: selectedModel.provider.baseUrl,
|
||
modelId: selectedModel.model.modelId as "qwen3.8-max" | "qwen3.8-flash" | "qwen3.8-omni-flash",
|
||
prompt: promptText,
|
||
videoUrl: videoUrls[0],
|
||
signal: signal
|
||
? AbortSignal.any([signal, AbortSignal.timeout(300_000)])
|
||
: AbortSignal.timeout(300_000),
|
||
}), finishReason: "stop" }
|
||
: await generateText({
|
||
model: model!,
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{ type: "text", text: promptText },
|
||
...referenceImages.map((image) => ({ type: "image" as const, image })),
|
||
],
|
||
},
|
||
],
|
||
temperature: 0.2,
|
||
maxOutputTokens: ["story-script", "director-reconstruct"].includes(parsed.data.mode)
|
||
? 12000
|
||
: parsed.data.mode === "video-analysis"
|
||
? 2200
|
||
: 1600,
|
||
maxRetries: 0,
|
||
abortSignal: signal
|
||
? AbortSignal.any([signal, AbortSignal.timeout(150_000)])
|
||
: AbortSignal.timeout(150_000),
|
||
});
|
||
const { text, finishReason } = generated;
|
||
|
||
if (finishReason === "length" || !text.trim()) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "分析结果不完整，请缩短输入或重新生成" },
|
||
{ status: 502 },
|
||
);
|
||
}
|
||
const parsedJson = extractJsonBlock<Record<string, unknown>>(text);
|
||
if (!parsedJson) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "分析模型未返回完整结构化结果，请重新生成" },
|
||
{ status: 502 },
|
||
);
|
||
}
|
||
const summary = normalizeText(parsedJson.summary) || normalizeText(text).slice(0, 160);
|
||
|
||
if (parsed.data.mode === "director-reconstruct")
|
||
return NextResponse.json({
|
||
success: true,
|
||
data: {
|
||
...parseReconstructedScene(parsedJson),
|
||
raw: text,
|
||
model: selectedModel.model.displayName,
|
||
},
|
||
});
|
||
if (parsed.data.mode === "video-analysis") {
|
||
const framePreviews = await framePreviewsPromise;
|
||
const analysisRows = normalizeVideoAnalysisRows(
|
||
parsedJson.analysisRows,
|
||
framePreviews,
|
||
).map((row) => soundAvailable ? row : { ...row, backgroundMusic: "", voiceAndSfx: "" });
|
||
if (!analysisRows.length)
|
||
return NextResponse.json(
|
||
{ success: false, error: "视频解析没有返回有效镜头，请重新解析" },
|
||
{ status: 502 },
|
||
);
|
||
const shotIdeas =
|
||
normalizeShotIdeas(parsedJson.shotIdeas).length > 0
|
||
? normalizeShotIdeas(parsedJson.shotIdeas)
|
||
: analysisRows.map((row, index) => ({
|
||
title: `镜头 ${index + 1}`,
|
||
description: row.sceneDescription,
|
||
}));
|
||
return NextResponse.json({
|
||
success: true,
|
||
data: {
|
||
summary,
|
||
analysisRows,
|
||
framePreviews,
|
||
highlights: normalizeStringList(parsedJson.highlights, 6),
|
||
shotIdeas,
|
||
visualMotifs: normalizeStringList(parsedJson.visualMotifs, 6),
|
||
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
|
||
model: selectedModel.model.displayName,
|
||
raw: text,
|
||
},
|
||
});
|
||
}
|
||
|
||
if (parsed.data.mode === "character-scene-extraction") {
|
||
return NextResponse.json({
|
||
success: true,
|
||
data: {
|
||
summary,
|
||
characters: normalizeCharacters(parsedJson.characters),
|
||
scenes: normalizeScenes(parsedJson.scenes),
|
||
relationships: normalizeStringList(parsedJson.relationships, 8),
|
||
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
|
||
model: selectedModel.model.displayName,
|
||
raw: text,
|
||
},
|
||
});
|
||
}
|
||
|
||
if (parsed.data.mode === "story-script") {
|
||
const analysisRows = normalizeStoryScriptRows(
|
||
parsedJson.analysisRows || parsedJson.storyRows || parsedJson.rows || parsedJson.shots,
|
||
);
|
||
if (analysisRows.length === 0) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "分析模型没有返回有效镜头，请重新生成" },
|
||
{ status: 502 },
|
||
);
|
||
}
|
||
return NextResponse.json({
|
||
success: true,
|
||
data: {
|
||
summary,
|
||
analysisRows,
|
||
shotIdeas:
|
||
normalizeShotIdeas(parsedJson.shotIdeas).length > 0
|
||
? normalizeShotIdeas(parsedJson.shotIdeas)
|
||
: analysisRows.map((row, index) => ({
|
||
title: `镜头 ${index + 1}`,
|
||
description: row.sceneDescription,
|
||
})),
|
||
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
|
||
model: selectedModel.model.displayName,
|
||
raw: text,
|
||
},
|
||
});
|
||
}
|
||
|
||
return NextResponse.json({
|
||
success: true,
|
||
data: {
|
||
summary,
|
||
agentPlan: normalizeStringList(parsedJson.agentPlan, 8),
|
||
deliverables: normalizeStringList(parsedJson.deliverables, 6),
|
||
recommendedNodes: normalizeStringList(parsedJson.recommendedNodes, 6),
|
||
model: selectedModel.model.displayName,
|
||
raw: text,
|
||
},
|
||
});
|
||
} catch (error) {
|
||
console.error("[Canvas Workflow Analyze] Error:", error);
|
||
const timedOut =
|
||
error instanceof Error && /timeout|timed out|abort/i.test(`${error.name} ${error.message}`);
|
||
return NextResponse.json(
|
||
{
|
||
success: false,
|
||
error: timedOut
|
||
? "分析超时，请稍后重试或选择其他分析模型"
|
||
: error instanceof Error
|
||
? error.message
|
||
: "工作流分析失败",
|
||
},
|
||
{ status: timedOut ? 504 : 500 },
|
||
);
|
||
}
|
||
}
|