import { generateText } from "ai"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { mkdtemp, readFile, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { NextResponse } from "next/server"; import sharp from "sharp"; import { z } from "zod"; import { auth } from "@/app/(auth)/auth"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { inlineVisionImages } from "@/lib/ai/vision-inline"; import { getAiModelAgentCodingModelId, isAiModelMultimodal } from "@/lib/db/ai-model-flags"; import { getAgentAiModelWithProvider, getEnabledAiModelsWithProvider, } from "@/lib/db/queries"; export const runtime = "nodejs"; export const maxDuration = 180; const execFileAsync = promisify(execFile); const VIDEO_ANALYSIS_FRAME_COUNT = 4; const VIDEO_ANALYSIS_MAX_SIDE = 960; const VIDEO_ANALYSIS_TIMEOUT_MS = 15000; const requestSchema = z.object({ mode: z.enum(["video-analysis", "character-scene-extraction", "story-script", "agent-plan"]), prompt: z.string().trim().optional(), imageUrls: z.array(z.string().trim()).optional(), videoUrls: z.array(z.string().trim()).optional(), nodeTitle: z.string().trim().optional(), preferredModelRef: z.string().trim().optional(), model: z.string().trim().optional(), }); function sanitizeUrls(value: unknown, limit = 6) { if (!Array.isArray(value)) return []; return value .map((item) => String(item || "").trim()) .filter((url) => /^https?:\/\//i.test(url)) .slice(0, limit); } function supportsVision(model: { supportsVision?: boolean | null; capabilities?: unknown; defaultParams?: unknown; }) { return ( Boolean(model.supportsVision) || isAiModelMultimodal({ capabilities: model.capabilities, defaultParams: model.defaultParams, }) ); } function extractJsonBlock>(text: string): T | null { const matched = String(text || "").match(/\{[\s\S]*\}/); if (!matched) return null; try { return JSON.parse(matched[0]) as T; } catch { return null; } } function normalizeText(value: unknown) { return String(value || "").trim(); } function shouldUseLocalWorkflowE2E(request: Request) { const hostname = new URL(request.url).hostname; return ( process.env.CANVAS_E2E_LOCAL === "true" && (hostname === "127.0.0.1" || hostname === "localhost" || hostname === "::1") ); } function normalizeStringList(value: unknown, limit = 8) { if (!Array.isArray(value)) return []; return Array.from( new Set( value .map((item) => String(item || "").trim()) .filter(Boolean) .slice(0, limit) ) ); } function normalizeShotIdeas(value: unknown) { if (!Array.isArray(value)) return []; return value .map((item, index) => { const raw = item && typeof item === "object" && !Array.isArray(item) ? (item as Record) : {}; const title = normalizeText(raw.title || `镜头 ${index + 1}`); const description = normalizeText(raw.description || raw.prompt || raw.summary); if (!title && !description) return null; return { title: title || `镜头 ${index + 1}`, description, }; }) .filter( (item): item is { title: string; description: string } => Boolean(item) ) .slice(0, 6); } function normalizeVideoAnalysisRows( value: unknown, framePreviews: string[], ) { if (!Array.isArray(value)) return []; return value .map((item, index) => { const raw = item && typeof item === "object" && !Array.isArray(item) ? (item as Record) : {}; const sceneDescription = normalizeText( raw.sceneDescription || raw.scene || raw.title || raw.description, ); const narrative = normalizeText( raw.narrative || raw.story || raw.storyContent || raw.narration || raw.summary, ); const shot = normalizeText(raw.shot || raw.shotNumber || raw.index || index + 1); const startTime = normalizeText(raw.startTime || raw.start || raw.start_time); const endTime = normalizeText(raw.endTime || raw.end || raw.end_time); const duration = normalizeText(raw.duration || raw.durationSeconds || raw.duration_seconds); const shotSize = normalizeText(raw.shotSize || raw.frameSize || raw.shot_type || raw.sceneScale); const cameraAngle = normalizeText(raw.cameraAngle || raw.angle || raw.camera_angle); const cameraMovement = normalizeText( raw.cameraMovement || raw.cameraMove || raw.camera_motion || raw.motionType, ); const focusDepth = normalizeText( raw.focusDepth || raw.depthOfField || raw.focalDepth || raw.focus_depth, ); const lighting = normalizeText(raw.lighting || raw.light || raw.lightDesign); const backgroundMusic = normalizeText( raw.backgroundMusic || raw.music || raw.bgm, ); const voiceAndSfx = normalizeText( raw.voiceAndSfx || raw.sound || raw.audio || raw.sfx, ); const imagePrompt = normalizeText( raw.imagePrompt || raw.image_generation_prompt || raw.image, ); const motionPrompt = normalizeText( raw.motionPrompt || raw.motion || raw.videoPrompt || raw.video_motion_prompt, ); const keyframeIndex = Number( raw.keyframeIndex || raw.frameIndex || raw.key_frame_index || index + 1, ); if ( !sceneDescription && !narrative && !backgroundMusic && !voiceAndSfx && !imagePrompt && !motionPrompt && !shotSize && !cameraAngle && !cameraMovement && !focusDepth && !lighting ) { return null; } const normalizedFrameIndex = Number.isFinite(keyframeIndex) && keyframeIndex > 0 ? Math.max(0, Math.min(framePreviews.length - 1, Math.round(keyframeIndex) - 1)) : index % Math.max(framePreviews.length, 1); return { shot: shot || String(index + 1), startTime: startTime || (index === 0 ? "0" : ""), endTime, duration, sceneDescription: sceneDescription || `镜头 ${index + 1}`, narrative: narrative || sceneDescription || `镜头 ${index + 1}`, shotSize: shotSize || "中景至近景", cameraAngle: cameraAngle || "平视", cameraMovement: cameraMovement || "缓慢推镜头 (Slow Zoom In)", focusDepth: focusDepth || "浅景深", lighting: lighting || "冷调低沉照明,主体光线柔和。", backgroundMusic: backgroundMusic || "无", voiceAndSfx: voiceAndSfx || "无", imagePrompt: imagePrompt || sceneDescription || "延续当前视频气质生成关键帧", motionPrompt: motionPrompt || "保持镜头运动自然、连贯、符合前后节奏。", keyframeUrl: framePreviews[normalizedFrameIndex] || "", }; }) .filter( ( item, ): item is { shot: string; startTime: string; endTime: string; duration: string; sceneDescription: string; narrative: string; shotSize: string; cameraAngle: string; cameraMovement: string; focusDepth: string; lighting: string; backgroundMusic: string; voiceAndSfx: string; imagePrompt: string; motionPrompt: string; keyframeUrl: string; } => Boolean(item), ) .slice(0, 8); } function normalizeStoryShot(value: unknown, index: number) { const text = normalizeText(value); if (/^\d+$/.test(text)) return text.padStart(2, "0"); return text || String(index + 1).padStart(2, "0"); } type NormalizedStoryScriptRow = { shot: string; duration: string; scene: string; sceneDescription: string; shotSize: string; lighting: string; dialogue: string; backgroundMusic: string; voiceAndSfx: string; cameraMovement: string; imagePrompt: string; motionPrompt: string; keyframeUrl: string; }; function normalizeStoryScriptRows(value: unknown): NormalizedStoryScriptRow[] { if (!Array.isArray(value)) return []; return value .map((item, index) => { if (Array.isArray(item)) { const cells = item.map((cell) => normalizeText(cell)); const shot = normalizeStoryShot(cells[0], index); const secondCellLooksLikeDuration = /^\d+(\.\d+)?\s*(s|秒)?$/i.test(cells[1] || ""); const duration = secondCellLooksLikeDuration ? cells[1] : "5s"; const scene = secondCellLooksLikeDuration ? cells[2] || `镜头 ${shot}` : cells[1] || `镜头 ${shot}`; const sceneDescription = secondCellLooksLikeDuration ? cells[2] || scene : cells[2] || cells[3] || scene; return { shot, duration, scene, sceneDescription, shotSize: (secondCellLooksLikeDuration ? cells[3] : "") || "中景", lighting: (secondCellLooksLikeDuration ? cells[4] : cells[4]) || "电影感光影", dialogue: (secondCellLooksLikeDuration ? cells[5] : "") || "+", backgroundMusic: "无", voiceAndSfx: (secondCellLooksLikeDuration ? cells[6] : cells[3]) || "无", cameraMovement: (secondCellLooksLikeDuration ? cells[7] : "") || "缓慢推镜头", imagePrompt: (secondCellLooksLikeDuration ? cells[8] : cells[5]) || sceneDescription, motionPrompt: (secondCellLooksLikeDuration ? cells[9] || cells[7] : cells[6]) || "保持镜头运动自然、连贯,围绕主体动作推进。", keyframeUrl: "", }; } const raw = item && typeof item === "object" && !Array.isArray(item) ? (item as Record) : {}; const shot = normalizeStoryShot(raw.shot || raw.shotNumber || raw.index || raw.id, index); const duration = normalizeText(raw.duration || raw.length || raw.time || raw.seconds); const scene = normalizeText(raw.scene || raw.location || raw.title || `镜头 ${shot}`); const sceneDescription = normalizeText( raw.sceneDescription || raw.description || raw.visual || raw.picture || raw.screen || raw.summary, ); const shotSize = normalizeText( raw.shotSize || raw.frameSize || raw.shot_type || raw.sceneScale || raw.cameraShot, ); const lighting = normalizeText( raw.lighting || raw.light || raw.atmosphere || raw.mood || raw.lightMood, ); const dialogue = normalizeText( raw.dialogue || raw.dialog || raw.narration || raw.voiceover || raw.monologue || raw.line, ); const cameraMovement = normalizeText( raw.cameraMovement || raw.cameraMove || raw.movement || raw.lensMovement || raw.motion, ); const imagePrompt = normalizeText( raw.imagePrompt || raw.image_generation_prompt || raw.image || raw.keyframePrompt || raw.keyFramePrompt, ); const motionPrompt = normalizeText( raw.motionPrompt || raw.videoPrompt || raw.video_motion_prompt || raw.motion || raw.cameraMovement, ); const backgroundMusic = normalizeText( raw.backgroundMusic || raw.music || raw.bgm || raw.soundtrack, ); const voiceAndSfx = normalizeText( raw.voiceAndSfx || raw.voice || raw.sound || raw.audio || raw.sfx, ); if ( !scene && !sceneDescription && !imagePrompt && !motionPrompt && !backgroundMusic && !voiceAndSfx ) { return null; } return { shot, duration: duration || "5s", scene, sceneDescription: sceneDescription || imagePrompt || scene || `镜头 ${shot}`, shotSize: shotSize || "中景", lighting: lighting || "电影感光影", dialogue: dialogue || "+", backgroundMusic: backgroundMusic || "无", voiceAndSfx: voiceAndSfx || "无", cameraMovement: cameraMovement || "缓慢推镜头", imagePrompt: imagePrompt || sceneDescription || scene || `镜头 ${shot}关键帧`, motionPrompt: motionPrompt || cameraMovement || "保持镜头运动自然、连贯,围绕主体动作推进。", keyframeUrl: "", }; }) .filter( ( item, ): item is NormalizedStoryScriptRow => Boolean(item), ) .slice(0, 18); } const storyExpansionPhases = [ { description: "建立镜头,交代环境、人物位置和当前压力", cameraMovement: "缓慢推镜头", imagePrompt: "广角或中景建立画面,环境信息清晰,主体位置明确", lighting: "环境主光明确,氛围基调清晰", motionPrompt: "镜头缓慢推进或轻微横移,建立空间关系", shotSize: "中景", }, { description: "动作推进,突出人物的关键行为或剧情转折", cameraMovement: "跟随主体动作推进", imagePrompt: "中景表现人物动作,服装、道具和光影保持连续", lighting: "主体光影连续,动作区域清晰", motionPrompt: "跟随主体动作推进,节奏自然连贯", shotSize: "中景", }, { description: "情绪反应,用近景或特写强化人物心理和冲突", cameraMovement: "轻微推近或稳定特写", imagePrompt: "近景或特写,突出表情、眼神、手部或关键道具细节", lighting: "面部光线突出,情绪反差增强", motionPrompt: "轻微推近或稳定特写,强调情绪变化", shotSize: "特写", }, { description: "关系与氛围,表现周围人物、环境反馈或声画张力", cameraMovement: "横移扫过环境或切换视角", imagePrompt: "加入环境反馈、群像反应或空间层次,电影感光影", lighting: "场景氛围增强,空间层次明显", motionPrompt: "镜头扫过环境或切换视角,保持节奏递进", shotSize: "全景", }, { description: "转场收束,为下一镜头保留动作、视线或情绪钩子", cameraMovement: "顺势后移、上摇或转场", imagePrompt: "构图留出转场方向,动作或视线指向下一段剧情", lighting: "光影引导视线并保留转场方向", motionPrompt: "镜头后移、上摇或顺势转场,衔接下一镜头", shotSize: "大远景", }, ]; function expandStoryScriptRowsToTarget( rows: NormalizedStoryScriptRow[], targetCount = 18, ): NormalizedStoryScriptRow[] { if (rows.length === 0) return []; if (rows.length >= targetCount) { return rows.slice(0, targetCount).map((row, index) => ({ ...row, shot: String(index + 1).padStart(2, "0"), })); } const baseCount = Math.floor(targetCount / rows.length); const extraCount = targetCount % rows.length; const expanded: NormalizedStoryScriptRow[] = []; rows.forEach((row, rowIndex) => { const segmentCount = baseCount + (rowIndex < extraCount ? 1 : 0); for (let segmentIndex = 0; segmentIndex < segmentCount; segmentIndex += 1) { const phase = storyExpansionPhases[ Math.min( storyExpansionPhases.length - 1, Math.floor((segmentIndex / Math.max(segmentCount, 1)) * storyExpansionPhases.length), ) ]; expanded.push({ ...row, shot: String(expanded.length + 1).padStart(2, "0"), sceneDescription: `${phase.description}:${row.sceneDescription}`, shotSize: row.shotSize || phase.shotSize, lighting: row.lighting || phase.lighting, cameraMovement: row.cameraMovement || phase.cameraMovement, imagePrompt: `${row.imagePrompt};${phase.imagePrompt}`, motionPrompt: `${phase.motionPrompt};${row.motionPrompt}`, }); } }); return expanded.slice(0, targetCount); } function normalizeCharacters(value: unknown) { if (!Array.isArray(value)) return []; return value .map((item) => { const raw = item && typeof item === "object" && !Array.isArray(item) ? (item as Record) : {}; const name = normalizeText(raw.name || raw.title); const description = normalizeText(raw.description || raw.summary); const prompt = normalizeText(raw.prompt); if (!name && !description && !prompt) return null; return { name: name || "未命名角色", description, prompt, look: normalizeText(raw.look) || undefined, wardrobe: normalizeText(raw.wardrobe) || undefined, signatureAction: normalizeText(raw.signatureAction) || undefined, traits: normalizeStringList(raw.traits, 6), }; }) .filter((item): item is NonNullable => Boolean(item)) .slice(0, 8); } function normalizeScenes(value: unknown) { if (!Array.isArray(value)) return []; return value .map((item) => { const raw = item && typeof item === "object" && !Array.isArray(item) ? (item as Record) : {}; const name = normalizeText(raw.name || raw.title); const description = normalizeText(raw.description || raw.summary); const prompt = normalizeText(raw.prompt); if (!name && !description && !prompt) return null; return { name: name || "未命名场景", description, prompt, mood: normalizeText(raw.mood) || undefined, lighting: normalizeText(raw.lighting) || undefined, props: normalizeStringList(raw.props, 6), }; }) .filter((item): item is NonNullable => Boolean(item)) .slice(0, 8); } function buildVideoAnalysisPrompt(params: { nodeTitle?: string; prompt: string; frameCount: number; imageCount: number; videoCount: number; }) { return [ "你是视频画布工作流里的视频分析节点,需要为后续分镜和生成节点提供结构化输出。", params.nodeTitle ? `当前节点:${params.nodeTitle}` : "", `上游参考图数量:${params.imageCount}`, `上游视频数量:${params.videoCount}`, `已提供按时间顺序抽取的视频关键帧数量:${params.frameCount}`, params.prompt ? `上游脚本/说明:\n${params.prompt}` : "上游没有额外脚本,请重点依据参考画面分析。", "", "请只输出 JSON,不要输出 markdown,不要输出解释。", "JSON 结构如下:", `{ "summary": "一句话总结当前视频/镜头风格,不超过120字", "highlights": ["3-5条镜头分析要点"], "analysisRows": [ { "shot": "1", "startTime": "0", "endTime": "5", "duration": "5", "sceneDescription": "画面描述,说明主体、环境、动作、构图和色调", "narrative": "叙事内容,说明这一镜头承担的剧情/情绪/信息作用", "shotSize": "景别,例如中景至近景", "cameraAngle": "摄影机角度,例如平视", "cameraMovement": "摄影机运动,例如缓慢推镜头 (Slow Zoom In)", "focusDepth": "焦距与景深,例如浅景深", "lighting": "光线,例如冷调低沉照明,屏幕光作为侧逆光", "backgroundMusic": "这一段适合的背景音乐氛围", "voiceAndSfx": "人声/环境声/音效建议,没有就写无", "imagePrompt": "后续生成关键帧时可直接使用的英文图像提示词", "motionPrompt": "后续生成视频时可直接使用的英文运动提示词", "keyframeIndex": 1 } ], "shotIdeas": [ { "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" } ], "visualMotifs": ["2-4个可复用的视觉母题"], "recommendedNextSteps": ["2-4条建议的下游动作"] }`, "analysisRows 请按镜头时间顺序输出;短视频可以输出 1 行,复杂视频输出 2-6 行。", "每行必须覆盖镜号、开始时间、结束时间、时长、画面描述、叙事内容、景别、摄影机角度、摄影机运动、焦距与景深、光线、背景音乐、人声/音效、图像生成提示词、视频运动提示词和关键帧。", "imagePrompt 和 motionPrompt 优先使用英文;其他字段使用中文;keyframeIndex 使用 1 开始的序号,对应你看到的关键帧顺序。", "如果没有真实视频帧,只能依据参考图和脚本推断,也要正常输出,但措辞要稳健。", ] .filter(Boolean) .join("\n"); } async function getVideoDurationSeconds(videoUrl: string) { try { const { stdout } = await execFileAsync( "ffprobe", [ "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", videoUrl, ], { timeout: VIDEO_ANALYSIS_TIMEOUT_MS, maxBuffer: 1024 * 1024, }, ); const duration = Number.parseFloat(String(stdout || "").trim()); return Number.isFinite(duration) && duration > 0 ? duration : null; } catch { return null; } } function buildVideoFrameTimestamps(durationSeconds: number | null, frameCount: number) { const safeCount = Math.max(1, frameCount); if (!durationSeconds || !Number.isFinite(durationSeconds) || durationSeconds <= 0.5) { return Array.from({ length: safeCount }).map((_, index) => Number((0.3 + index * 0.45).toFixed(2)), ); } const start = Math.min(0.35, durationSeconds * 0.12); const end = Math.max(start + 0.6, durationSeconds * 0.88); if (safeCount === 1) return [Number(((start + end) / 2).toFixed(2))]; const step = (end - start) / (safeCount - 1); return Array.from({ length: safeCount }).map((_, index) => Number((start + step * index).toFixed(2)), ); } async function downscaleVideoFrameForVision(inputBuffer: Buffer) { return sharp(inputBuffer, { failOn: "none" }) .rotate() .resize({ width: VIDEO_ANALYSIS_MAX_SIDE, height: VIDEO_ANALYSIS_MAX_SIDE, fit: "inside", withoutEnlargement: true, }) .jpeg({ quality: 82, mozjpeg: true }) .toBuffer(); } async function extractVideoFrameDataUrls(videoUrl: string, frameCount = VIDEO_ANALYSIS_FRAME_COUNT) { const tempDir = await mkdtemp(path.join(os.tmpdir(), "canvas-video-analysis-")); try { const durationSeconds = await getVideoDurationSeconds(videoUrl); const timestamps = buildVideoFrameTimestamps(durationSeconds, frameCount); const frames = await Promise.all( timestamps.map(async (timestamp, index) => { const outputPath = path.join(tempDir, `frame-${index + 1}.jpg`); await execFileAsync( "ffmpeg", [ "-hide_banner", "-loglevel", "error", "-ss", String(timestamp), "-i", videoUrl, "-frames:v", "1", "-q:v", "4", "-y", outputPath, ], { timeout: VIDEO_ANALYSIS_TIMEOUT_MS, maxBuffer: 2 * 1024 * 1024, }, ); const buffer = await readFile(outputPath); const processed = await downscaleVideoFrameForVision(buffer); return `data:image/jpeg;base64,${processed.toString("base64")}`; }), ); return frames.filter(Boolean); } catch (error) { console.warn("[Canvas Workflow Analyze] extract video frames failed:", { message: error instanceof Error ? error.message : String(error), videoUrl, }); return []; } finally { await rm(tempDir, { force: true, recursive: true }).catch(() => undefined); } } function buildCharacterScenePrompt(params: { nodeTitle?: string; prompt: string; imageCount: number; }) { return [ "你是视频画布工作流里的角色场景提取节点,需要从脚本、设定和参考图里提取角色与场景资产。", params.nodeTitle ? `当前节点:${params.nodeTitle}` : "", `参考图数量:${params.imageCount}`, params.prompt ? `输入内容:\n${params.prompt}` : "没有额外脚本,请根据参考图提取角色和场景线索。", "", "请只输出 JSON,不要输出 markdown,不要输出解释。", "JSON 结构如下:", `{ "summary": "一句话总结当前故事世界与资产线索", "characters": [ { "name": "角色名", "description": "角色外观与身份简介", "look": "外貌特征", "wardrobe": "服装/道具", "traits": ["性格关键词"], "signatureAction": "代表动作", "prompt": "可直接用于生成角色图/角色视频的提示词" } ], "scenes": [ { "name": "场景名", "description": "空间、布景和氛围简介", "mood": "情绪与气氛", "lighting": "布光特征", "props": ["核心道具"], "prompt": "可直接用于生成场景图/场景视频的提示词" } ], "relationships": ["角色与场景或角色之间的关系"], "recommendedNextSteps": ["建议连接的后续节点动作"] }`, "角色和场景都要尽量具体,避免空泛形容词。", ] .filter(Boolean) .join("\n"); } function buildAgentPlanPrompt(params: { nodeTitle?: string; prompt: string; imageCount: number; videoCount: number; }) { return [ "你是视频画布里的自定义代理节点,需要把当前创作目标拆成可执行的工作流计划。", params.nodeTitle ? `当前节点:${params.nodeTitle}` : "", `参考图数量:${params.imageCount}`, `视频数量:${params.videoCount}`, params.prompt ? `上游需求:\n${params.prompt}` : "没有额外脚本,请基于当前上下文给出稳妥计划。", "", "请只输出 JSON,不要输出 markdown,不要输出解释。", "JSON 结构如下:", `{ "summary": "一句话说明这个代理链的目标", "agentPlan": ["3-6个执行步骤"], "deliverables": ["2-4个预期交付物"], "recommendedNodes": ["建议继续连接的节点类型或能力名"] }`, "步骤要偏工作流视角,而不是泛泛而谈。", ] .filter(Boolean) .join("\n"); } function buildStoryScriptPrompt(params: { nodeTitle?: string; prompt: string; imageCount: number; videoCount: number; }) { return [ "你是视频画布里的故事脚本生成节点,需要把用户输入、参考图和参考视频整理成可直接用于后续分镜/生图/生视频的结构化脚本表。", params.nodeTitle ? `当前节点:${params.nodeTitle}` : "", `参考图数量:${params.imageCount}`, `视频数量:${params.videoCount}`, params.prompt ? `输入内容:\n${params.prompt}` : "没有额外脚本,请生成一个节奏清晰的短视频脚本。", "", "请只输出 JSON,不要输出 markdown,不要输出解释。", "JSON 结构如下:", `{ "summary": "一句话概括故事主题、情绪和视频方向,不超过120字", "analysisRows": [ { "shot": "01", "duration": "5s", "scene": "场景名或地点,如 盛唐·金銮殿·日", "sceneDescription": "画面内容、人物动作和剧情推进,不要重复 scene 字段", "shotSize": "景别,如中景、特写、全景、大远景", "lighting": "光影氛围,如暖金主光带阴影,气氛紧张压抑", "dialogue": "对白或旁白,没有就写 +", "backgroundMusic": "背景音乐或情绪氛围,没有就写无", "voiceAndSfx": "环境声、动作声或音效,没有就写无", "cameraMovement": "运镜,如缓慢推镜头、固定、横移、上摇、环绕轨道跟拍", "imagePrompt": "后续生成该镜头关键帧时可直接使用的中文图像提示词", "motionPrompt": "后续生成视频时可直接使用的中文视频运动提示词" } ], "assetCandidates": { "style": "全局视觉风格,如 盛唐·国风二次元", "characters": ["需要生成的角色资产名"], "scenes": ["需要生成的场景资产名"], "props": ["需要生成的道具资产名"] }, "shotIdeas": [ { "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" } ], "recommendedNextSteps": ["2-4条建议的下游动作"] }`, "analysisRows 必须直接输出 18 行,按镜头时间顺序编号 01-18;每行默认 5s 或 6s,剧情高潮可用 6s。", "每行要覆盖竞品脚本表同等信息:镜号、时长、画面描述、景别、光影氛围、对白·旁白、音效、运镜、最终图像提示词和视频运动提示词。", "画面描述中用 @角色名 或 @场景/道具名 标出可复用资产,例如 @沈昭昭、@金銮殿、@侍卫长刀。", "imagePrompt 要像可直接生成分镜图的中文提示词,包含主体、环境、构图、光影、风格与一致性要求;motionPrompt 要按起始状态、动作过程、结束状态、音效组织。", "assetCandidates 根据 18 镜头抽取 3-6 个角色、2-5 个场景和 1-4 个道具,方便后续准备资产。", "每行要像专业视频画布脚本表:场景具体、画面可生成、声音可执行、镜头节奏明确,不要只写剧情摘要。", ] .filter(Boolean) .join("\n"); } export async function POST(request: Request) { try { const body = await request.json().catch(() => ({})); const parsed = requestSchema.safeParse(body); if (!parsed.success) { return NextResponse.json( { success: false, error: "请求参数不合法" }, { status: 400 } ); } const prompt = normalizeText(parsed.data.prompt); const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4); const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4); const extractedVideoFrames = parsed.data.mode === "video-analysis" && videoUrls.length > 0 ? await extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT) : []; const preferredModelRef = normalizeText(parsed.data.preferredModelRef || parsed.data.model); if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) { return NextResponse.json( { success: false, error: "请至少提供脚本、图片或视频上下文" }, { status: 400 } ); } const session = await auth(); if (!session?.user?.id) { return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); } const [allModels, agentModel] = await Promise.all([ getEnabledAiModelsWithProvider(false), getAgentAiModelWithProvider(), ]); const compatibleModels = allModels.filter( (item) => !item.model.isSubModel && item.model.isEnabled ); const preferredModel = preferredModelRef ? compatibleModels.find( (item) => item.model.id === preferredModelRef || item.model.modelId === preferredModelRef, ) : null; const hasVisionInputs = imageUrls.length > 0 || extractedVideoFrames.length > 0; let selectedModel = preferredModel && (!hasVisionInputs || supportsVision(preferredModel.model)) ? preferredModel : hasVisionInputs && agentModel && supportsVision(agentModel.model) ? agentModel : hasVisionInputs ? compatibleModels.find((item) => supportsVision(item.model)) : agentModel || compatibleModels[0]; if (!selectedModel) { selectedModel = compatibleModels[0]; } if (!selectedModel) { return NextResponse.json( { success: false, error: "当前站点没有可用的分析模型" }, { status: 500 } ); } const useVision = hasVisionInputs && supportsVision(selectedModel.model); const codingModelId = getAiModelAgentCodingModelId(selectedModel.model); const model = await getLanguageModelById(selectedModel.model.id, codingModelId); if (!model) { return NextResponse.json( { success: false, error: "分析模型初始化失败" }, { status: 500 } ); } const promptText = parsed.data.mode === "video-analysis" ? buildVideoAnalysisPrompt({ frameCount: extractedVideoFrames.length, nodeTitle: normalizeText(parsed.data.nodeTitle), prompt, imageCount: imageUrls.length, videoCount: videoUrls.length, }) : parsed.data.mode === "character-scene-extraction" ? buildCharacterScenePrompt({ nodeTitle: normalizeText(parsed.data.nodeTitle), prompt, imageCount: imageUrls.length, }) : parsed.data.mode === "story-script" ? buildStoryScriptPrompt({ nodeTitle: normalizeText(parsed.data.nodeTitle), prompt, imageCount: imageUrls.length, videoCount: videoUrls.length, }) : buildAgentPlanPrompt({ nodeTitle: normalizeText(parsed.data.nodeTitle), prompt, imageCount: imageUrls.length, videoCount: videoUrls.length, }); const inlinedImageReferences = useVision ? ( await inlineVisionImages(imageUrls, { maxImages: 4, maxSide: 960, timeoutMs: 12000, retries: 1, loggerPrefix: "[Canvas Workflow Analyze]", }) ).inlineDataUrls : []; const referenceImages = useVision ? [...inlinedImageReferences, ...extractedVideoFrames].slice(0, 6) : []; const { text } = await generateText({ model, messages: [ { role: "user", content: [ { type: "text", text: promptText }, ...referenceImages.map((image) => ({ type: "image" as const, image })), ], }, ], temperature: 0.2, maxTokens: parsed.data.mode === "story-script" ? 4200 : parsed.data.mode === "video-analysis" ? 2200 : 1600, } as any); const parsedJson = extractJsonBlock>(text) || {}; const summary = normalizeText(parsedJson.summary) || normalizeText(text).slice(0, 160); if (parsed.data.mode === "video-analysis") { const analysisRows = normalizeVideoAnalysisRows( parsedJson.analysisRows, extractedVideoFrames, ); const shotIdeas = normalizeShotIdeas(parsedJson.shotIdeas).length > 0 ? normalizeShotIdeas(parsedJson.shotIdeas) : analysisRows.map((row, index) => ({ title: `镜头 ${index + 1}`, description: row.sceneDescription, })); return NextResponse.json({ success: true, data: { summary, analysisRows, framePreviews: extractedVideoFrames, highlights: normalizeStringList(parsedJson.highlights, 6), shotIdeas, visualMotifs: normalizeStringList(parsedJson.visualMotifs, 6), recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6), model: selectedModel.model.displayName, raw: text, }, }); } if (parsed.data.mode === "character-scene-extraction") { return NextResponse.json({ success: true, data: { summary, characters: normalizeCharacters(parsedJson.characters), scenes: normalizeScenes(parsedJson.scenes), relationships: normalizeStringList(parsedJson.relationships, 8), recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6), model: selectedModel.model.displayName, raw: text, }, }); } if (parsed.data.mode === "story-script") { const analysisRows = expandStoryScriptRowsToTarget(normalizeStoryScriptRows( parsedJson.analysisRows || parsedJson.storyRows || parsedJson.rows || parsedJson.shots, )); return NextResponse.json({ success: true, data: { summary, analysisRows, shotIdeas: normalizeShotIdeas(parsedJson.shotIdeas).length > 0 ? normalizeShotIdeas(parsedJson.shotIdeas) : analysisRows.map((row, index) => ({ title: `镜头 ${index + 1}`, description: row.sceneDescription, })), recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6), model: selectedModel.model.displayName, raw: text, }, }); } return NextResponse.json({ success: true, data: { summary, agentPlan: normalizeStringList(parsedJson.agentPlan, 8), deliverables: normalizeStringList(parsedJson.deliverables, 6), recommendedNodes: normalizeStringList(parsedJson.recommendedNodes, 6), model: selectedModel.model.displayName, raw: text, }, }); } catch (error) { console.error("[Canvas Workflow Analyze] Error:", error); return NextResponse.json( { success: false, error: error instanceof Error ? error.message : "工作流分析失败", }, { status: 500 } ); } }