import { generateText } from "ai"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { mkdtemp, readFile, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { NextResponse } from "next/server"; import sharp from "sharp"; import { z } from "zod"; import { auth } from "@/app/(auth)/auth"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { inlineVisionImages } from "@/lib/ai/vision-inline"; import { getAiModelAgentCodingModelId, isAiModelMultimodal } from "@/lib/db/ai-model-flags"; import { getAgentAiModelWithProvider, getEnabledAiModelsWithProvider, } from "@/lib/db/queries"; export const runtime = "nodejs"; export const maxDuration = 180; const execFileAsync = promisify(execFile); const VIDEO_ANALYSIS_FRAME_COUNT = 4; const VIDEO_ANALYSIS_MAX_SIDE = 960; const VIDEO_ANALYSIS_TIMEOUT_MS = 15000; const requestSchema = z.object({ mode: z.enum(["video-analysis", "character-scene-extraction", "agent-plan"]), prompt: z.string().trim().optional(), imageUrls: z.array(z.string().trim()).optional(), videoUrls: z.array(z.string().trim()).optional(), nodeTitle: z.string().trim().optional(), preferredModelRef: z.string().trim().optional(), model: z.string().trim().optional(), }); function sanitizeUrls(value: unknown, limit = 6) { if (!Array.isArray(value)) return []; return value .map((item) => String(item || "").trim()) .filter((url) => /^https?:\/\//i.test(url)) .slice(0, limit); } function supportsVision(model: { supportsVision?: boolean | null; capabilities?: unknown; defaultParams?: unknown; }) { return ( Boolean(model.supportsVision) || isAiModelMultimodal({ capabilities: model.capabilities, defaultParams: model.defaultParams, }) ); } function extractJsonBlock>(text: string): T | null { const matched = String(text || "").match(/\{[\s\S]*\}/); if (!matched) return null; try { return JSON.parse(matched[0]) as T; } catch { return null; } } function normalizeText(value: unknown) { return String(value || "").trim(); } function normalizeStringList(value: unknown, limit = 8) { if (!Array.isArray(value)) return []; return Array.from( new Set( value .map((item) => String(item || "").trim()) .filter(Boolean) .slice(0, limit) ) ); } function normalizeShotIdeas(value: unknown) { if (!Array.isArray(value)) return []; return value .map((item, index) => { const raw = item && typeof item === "object" && !Array.isArray(item) ? (item as Record) : {}; const title = normalizeText(raw.title || `镜头 ${index + 1}`); const description = normalizeText(raw.description || raw.prompt || raw.summary); if (!title && !description) return null; return { title: title || `镜头 ${index + 1}`, description, }; }) .filter( (item): item is { title: string; description: string } => Boolean(item) ) .slice(0, 6); } function normalizeVideoAnalysisRows( value: unknown, framePreviews: string[], ) { if (!Array.isArray(value)) return []; return value .map((item, index) => { const raw = item && typeof item === "object" && !Array.isArray(item) ? (item as Record) : {}; const sceneDescription = normalizeText( raw.sceneDescription || raw.scene || raw.title || raw.description, ); const backgroundMusic = normalizeText( raw.backgroundMusic || raw.music || raw.bgm, ); const voiceAndSfx = normalizeText( raw.voiceAndSfx || raw.sound || raw.audio || raw.sfx, ); const imagePrompt = normalizeText( raw.imagePrompt || raw.image_generation_prompt || raw.image, ); const motionPrompt = normalizeText( raw.motionPrompt || raw.motion || raw.videoPrompt || raw.video_motion_prompt, ); const keyframeIndex = Number( raw.keyframeIndex || raw.frameIndex || raw.key_frame_index || index + 1, ); if ( !sceneDescription && !backgroundMusic && !voiceAndSfx && !imagePrompt && !motionPrompt ) { return null; } const normalizedFrameIndex = Number.isFinite(keyframeIndex) && keyframeIndex > 0 ? Math.max(0, Math.min(framePreviews.length - 1, Math.round(keyframeIndex) - 1)) : index % Math.max(framePreviews.length, 1); return { sceneDescription: sceneDescription || `镜头 ${index + 1}`, backgroundMusic: backgroundMusic || "无", voiceAndSfx: voiceAndSfx || "无", imagePrompt: imagePrompt || sceneDescription || "延续当前视频气质生成关键帧", motionPrompt: motionPrompt || "保持镜头运动自然、连贯、符合前后节奏。", keyframeUrl: framePreviews[normalizedFrameIndex] || "", }; }) .filter( ( item, ): item is { sceneDescription: string; backgroundMusic: string; voiceAndSfx: string; imagePrompt: string; motionPrompt: string; keyframeUrl: string; } => Boolean(item), ) .slice(0, 8); } function normalizeCharacters(value: unknown) { if (!Array.isArray(value)) return []; return value .map((item) => { const raw = item && typeof item === "object" && !Array.isArray(item) ? (item as Record) : {}; const name = normalizeText(raw.name || raw.title); const description = normalizeText(raw.description || raw.summary); const prompt = normalizeText(raw.prompt); if (!name && !description && !prompt) return null; return { name: name || "未命名角色", description, prompt, look: normalizeText(raw.look) || undefined, wardrobe: normalizeText(raw.wardrobe) || undefined, signatureAction: normalizeText(raw.signatureAction) || undefined, traits: normalizeStringList(raw.traits, 6), }; }) .filter((item): item is NonNullable => Boolean(item)) .slice(0, 8); } function normalizeScenes(value: unknown) { if (!Array.isArray(value)) return []; return value .map((item) => { const raw = item && typeof item === "object" && !Array.isArray(item) ? (item as Record) : {}; const name = normalizeText(raw.name || raw.title); const description = normalizeText(raw.description || raw.summary); const prompt = normalizeText(raw.prompt); if (!name && !description && !prompt) return null; return { name: name || "未命名场景", description, prompt, mood: normalizeText(raw.mood) || undefined, lighting: normalizeText(raw.lighting) || undefined, props: normalizeStringList(raw.props, 6), }; }) .filter((item): item is NonNullable => Boolean(item)) .slice(0, 8); } function buildVideoAnalysisPrompt(params: { nodeTitle?: string; prompt: string; frameCount: number; imageCount: number; videoCount: number; }) { return [ "你是视频画布工作流里的视频分析节点,需要为后续分镜和生成节点提供结构化输出。", params.nodeTitle ? `当前节点:${params.nodeTitle}` : "", `上游参考图数量:${params.imageCount}`, `上游视频数量:${params.videoCount}`, `已提供按时间顺序抽取的视频关键帧数量:${params.frameCount}`, params.prompt ? `上游脚本/说明:\n${params.prompt}` : "上游没有额外脚本,请重点依据参考画面分析。", "", "请只输出 JSON,不要输出 markdown,不要输出解释。", "JSON 结构如下:", `{ "summary": "一句话总结当前视频/镜头风格,不超过120字", "highlights": ["3-5条镜头分析要点"], "analysisRows": [ { "sceneDescription": "镜头内容与景别描述", "backgroundMusic": "这一段适合的背景音乐氛围", "voiceAndSfx": "人声/环境声/音效建议,没有就写无", "imagePrompt": "后续生成关键帧时可直接使用的图像提示词", "motionPrompt": "后续生成视频时可直接使用的运动提示词", "keyframeIndex": 1 } ], "shotIdeas": [ { "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" } ], "visualMotifs": ["2-4个可复用的视觉母题"], "recommendedNextSteps": ["2-4条建议的下游动作"] }`, "analysisRows 请尽量输出 2-6 行;keyframeIndex 使用 1 开始的序号,对应你看到的关键帧顺序。", "如果没有真实视频帧,只能依据参考图和脚本推断,也要正常输出,但措辞要稳健。", ] .filter(Boolean) .join("\n"); } async function getVideoDurationSeconds(videoUrl: string) { try { const { stdout } = await execFileAsync( "ffprobe", [ "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", videoUrl, ], { timeout: VIDEO_ANALYSIS_TIMEOUT_MS, maxBuffer: 1024 * 1024, }, ); const duration = Number.parseFloat(String(stdout || "").trim()); return Number.isFinite(duration) && duration > 0 ? duration : null; } catch { return null; } } function buildVideoFrameTimestamps(durationSeconds: number | null, frameCount: number) { const safeCount = Math.max(1, frameCount); if (!durationSeconds || !Number.isFinite(durationSeconds) || durationSeconds <= 0.5) { return Array.from({ length: safeCount }).map((_, index) => Number((0.3 + index * 0.45).toFixed(2)), ); } const start = Math.min(0.35, durationSeconds * 0.12); const end = Math.max(start + 0.6, durationSeconds * 0.88); if (safeCount === 1) return [Number(((start + end) / 2).toFixed(2))]; const step = (end - start) / (safeCount - 1); return Array.from({ length: safeCount }).map((_, index) => Number((start + step * index).toFixed(2)), ); } async function downscaleVideoFrameForVision(inputBuffer: Buffer) { return sharp(inputBuffer, { failOn: "none" }) .rotate() .resize({ width: VIDEO_ANALYSIS_MAX_SIDE, height: VIDEO_ANALYSIS_MAX_SIDE, fit: "inside", withoutEnlargement: true, }) .jpeg({ quality: 82, mozjpeg: true }) .toBuffer(); } async function extractVideoFrameDataUrls(videoUrl: string, frameCount = VIDEO_ANALYSIS_FRAME_COUNT) { const tempDir = await mkdtemp(path.join(os.tmpdir(), "canvas-video-analysis-")); try { const durationSeconds = await getVideoDurationSeconds(videoUrl); const timestamps = buildVideoFrameTimestamps(durationSeconds, frameCount); const frames = await Promise.all( timestamps.map(async (timestamp, index) => { const outputPath = path.join(tempDir, `frame-${index + 1}.jpg`); await execFileAsync( "ffmpeg", [ "-hide_banner", "-loglevel", "error", "-ss", String(timestamp), "-i", videoUrl, "-frames:v", "1", "-q:v", "4", "-y", outputPath, ], { timeout: VIDEO_ANALYSIS_TIMEOUT_MS, maxBuffer: 2 * 1024 * 1024, }, ); const buffer = await readFile(outputPath); const processed = await downscaleVideoFrameForVision(buffer); return `data:image/jpeg;base64,${processed.toString("base64")}`; }), ); return frames.filter(Boolean); } catch (error) { console.warn("[Canvas Workflow Analyze] extract video frames failed:", { message: error instanceof Error ? error.message : String(error), videoUrl, }); return []; } finally { await rm(tempDir, { force: true, recursive: true }).catch(() => undefined); } } function buildCharacterScenePrompt(params: { nodeTitle?: string; prompt: string; imageCount: number; }) { return [ "你是视频画布工作流里的角色场景提取节点,需要从脚本、设定和参考图里提取角色与场景资产。", params.nodeTitle ? `当前节点:${params.nodeTitle}` : "", `参考图数量:${params.imageCount}`, params.prompt ? `输入内容:\n${params.prompt}` : "没有额外脚本,请根据参考图提取角色和场景线索。", "", "请只输出 JSON,不要输出 markdown,不要输出解释。", "JSON 结构如下:", `{ "summary": "一句话总结当前故事世界与资产线索", "characters": [ { "name": "角色名", "description": "角色外观与身份简介", "look": "外貌特征", "wardrobe": "服装/道具", "traits": ["性格关键词"], "signatureAction": "代表动作", "prompt": "可直接用于生成角色图/角色视频的提示词" } ], "scenes": [ { "name": "场景名", "description": "空间、布景和氛围简介", "mood": "情绪与气氛", "lighting": "布光特征", "props": ["核心道具"], "prompt": "可直接用于生成场景图/场景视频的提示词" } ], "relationships": ["角色与场景或角色之间的关系"], "recommendedNextSteps": ["建议连接的后续节点动作"] }`, "角色和场景都要尽量具体,避免空泛形容词。", ] .filter(Boolean) .join("\n"); } function buildAgentPlanPrompt(params: { nodeTitle?: string; prompt: string; imageCount: number; videoCount: number; }) { return [ "你是视频画布里的自定义代理节点,需要把当前创作目标拆成可执行的工作流计划。", params.nodeTitle ? `当前节点:${params.nodeTitle}` : "", `参考图数量:${params.imageCount}`, `视频数量:${params.videoCount}`, params.prompt ? `上游需求:\n${params.prompt}` : "没有额外脚本,请基于当前上下文给出稳妥计划。", "", "请只输出 JSON,不要输出 markdown,不要输出解释。", "JSON 结构如下:", `{ "summary": "一句话说明这个代理链的目标", "agentPlan": ["3-6个执行步骤"], "deliverables": ["2-4个预期交付物"], "recommendedNodes": ["建议继续连接的节点类型或能力名"] }`, "步骤要偏工作流视角,而不是泛泛而谈。", ] .filter(Boolean) .join("\n"); } export async function POST(request: Request) { try { const session = await auth(); if (!session?.user?.id) { return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); } const body = await request.json().catch(() => ({})); const parsed = requestSchema.safeParse(body); if (!parsed.success) { return NextResponse.json( { success: false, error: "请求参数不合法" }, { status: 400 } ); } const prompt = normalizeText(parsed.data.prompt); const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4); const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4); const extractedVideoFrames = parsed.data.mode === "video-analysis" && videoUrls.length > 0 ? await extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT) : []; const preferredModelRef = normalizeText(parsed.data.preferredModelRef || parsed.data.model); if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) { return NextResponse.json( { success: false, error: "请至少提供脚本、图片或视频上下文" }, { status: 400 } ); } const [allModels, agentModel] = await Promise.all([ getEnabledAiModelsWithProvider(false), getAgentAiModelWithProvider(), ]); const compatibleModels = allModels.filter( (item) => !item.model.isSubModel && item.model.isEnabled ); const preferredModel = preferredModelRef ? compatibleModels.find( (item) => item.model.id === preferredModelRef || item.model.modelId === preferredModelRef, ) : null; const hasVisionInputs = imageUrls.length > 0 || extractedVideoFrames.length > 0; let selectedModel = preferredModel && (!hasVisionInputs || supportsVision(preferredModel.model)) ? preferredModel : hasVisionInputs && agentModel && supportsVision(agentModel.model) ? agentModel : hasVisionInputs ? compatibleModels.find((item) => supportsVision(item.model)) : agentModel || compatibleModels[0]; if (!selectedModel) { selectedModel = compatibleModels[0]; } if (!selectedModel) { return NextResponse.json( { success: false, error: "当前站点没有可用的分析模型" }, { status: 500 } ); } const useVision = hasVisionInputs && supportsVision(selectedModel.model); const codingModelId = getAiModelAgentCodingModelId(selectedModel.model); const model = await getLanguageModelById(selectedModel.model.id, codingModelId); if (!model) { return NextResponse.json( { success: false, error: "分析模型初始化失败" }, { status: 500 } ); } const promptText = parsed.data.mode === "video-analysis" ? buildVideoAnalysisPrompt({ frameCount: extractedVideoFrames.length, nodeTitle: normalizeText(parsed.data.nodeTitle), prompt, imageCount: imageUrls.length, videoCount: videoUrls.length, }) : parsed.data.mode === "character-scene-extraction" ? buildCharacterScenePrompt({ nodeTitle: normalizeText(parsed.data.nodeTitle), prompt, imageCount: imageUrls.length, }) : buildAgentPlanPrompt({ nodeTitle: normalizeText(parsed.data.nodeTitle), prompt, imageCount: imageUrls.length, videoCount: videoUrls.length, }); const inlinedImageReferences = useVision ? ( await inlineVisionImages(imageUrls, { maxImages: 4, maxSide: 960, timeoutMs: 12000, retries: 1, loggerPrefix: "[Canvas Workflow Analyze]", }) ).inlineDataUrls : []; const referenceImages = useVision ? [...inlinedImageReferences, ...extractedVideoFrames].slice(0, 6) : []; const { text } = await generateText({ model, messages: [ { role: "user", content: [ { type: "text", text: promptText }, ...referenceImages.map((image) => ({ type: "image" as const, image })), ], }, ], temperature: 0.2, maxTokens: 1600, } as any); const parsedJson = extractJsonBlock>(text) || {}; const summary = normalizeText(parsedJson.summary) || normalizeText(text).slice(0, 160); if (parsed.data.mode === "video-analysis") { const analysisRows = normalizeVideoAnalysisRows( parsedJson.analysisRows, extractedVideoFrames, ); const shotIdeas = normalizeShotIdeas(parsedJson.shotIdeas).length > 0 ? normalizeShotIdeas(parsedJson.shotIdeas) : analysisRows.map((row, index) => ({ title: `镜头 ${index + 1}`, description: row.sceneDescription, })); return NextResponse.json({ success: true, data: { summary, analysisRows, framePreviews: extractedVideoFrames, highlights: normalizeStringList(parsedJson.highlights, 6), shotIdeas, visualMotifs: normalizeStringList(parsedJson.visualMotifs, 6), recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6), model: selectedModel.model.displayName, raw: text, }, }); } if (parsed.data.mode === "character-scene-extraction") { return NextResponse.json({ success: true, data: { summary, characters: normalizeCharacters(parsedJson.characters), scenes: normalizeScenes(parsedJson.scenes), relationships: normalizeStringList(parsedJson.relationships, 8), recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6), model: selectedModel.model.displayName, raw: text, }, }); } return NextResponse.json({ success: true, data: { summary, agentPlan: normalizeStringList(parsedJson.agentPlan, 8), deliverables: normalizeStringList(parsedJson.deliverables, 6), recommendedNodes: normalizeStringList(parsedJson.recommendedNodes, 6), model: selectedModel.model.displayName, raw: text, }, }); } catch (error) { console.error("[Canvas Workflow Analyze] Error:", error); return NextResponse.json( { success: false, error: error instanceof Error ? error.message : "工作流分析失败", }, { status: 500 } ); } }