import { generateText } from "ai"; import { NextResponse } from "next/server"; import { z } from "zod"; import { auth } from "@/app/(auth)/auth"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { inlineVisionImages } from "@/lib/ai/vision-inline"; import { getAiModelAgentCodingModelId, isAiModelMultimodal } from "@/lib/db/ai-model-flags"; import { getAgentAiModelWithProvider, getEnabledAiModelsWithProvider, } from "@/lib/db/queries"; export const runtime = "nodejs"; export const maxDuration = 180; const requestSchema = z.object({ mode: z.enum(["video-analysis", "character-scene-extraction", "agent-plan"]), prompt: z.string().trim().optional(), imageUrls: z.array(z.string().trim()).optional(), videoUrls: z.array(z.string().trim()).optional(), nodeTitle: z.string().trim().optional(), }); function sanitizeUrls(value: unknown, limit = 6) { if (!Array.isArray(value)) return []; return value .map((item) => String(item || "").trim()) .filter((url) => /^https?:\/\//i.test(url)) .slice(0, limit); } function supportsVision(model: { supportsVision?: boolean | null; capabilities?: unknown; defaultParams?: unknown; }) { return ( Boolean(model.supportsVision) || isAiModelMultimodal({ capabilities: model.capabilities, defaultParams: model.defaultParams, }) ); } function extractJsonBlock>(text: string): T | null { const matched = String(text || "").match(/\{[\s\S]*\}/); if (!matched) return null; try { return JSON.parse(matched[0]) as T; } catch { return null; } } function normalizeText(value: unknown) { return String(value || "").trim(); } function normalizeStringList(value: unknown, limit = 8) { if (!Array.isArray(value)) return []; return Array.from( new Set( value .map((item) => String(item || "").trim()) .filter(Boolean) .slice(0, limit) ) ); } function normalizeShotIdeas(value: unknown) { if (!Array.isArray(value)) return []; return value .map((item, index) => { const raw = item && typeof item === "object" && !Array.isArray(item) ? (item as Record) : {}; const title = normalizeText(raw.title || `镜头 ${index + 1}`); const description = normalizeText(raw.description || raw.prompt || raw.summary); if (!title && !description) return null; return { title: title || `镜头 ${index + 1}`, description, }; }) .filter( (item): item is { title: string; description: string } => Boolean(item) ) .slice(0, 6); } function normalizeCharacters(value: unknown) { if (!Array.isArray(value)) return []; return value .map((item) => { const raw = item && typeof item === "object" && !Array.isArray(item) ? (item as Record) : {}; const name = normalizeText(raw.name || raw.title); const description = normalizeText(raw.description || raw.summary); const prompt = normalizeText(raw.prompt); if (!name && !description && !prompt) return null; return { name: name || "未命名角色", description, prompt, look: normalizeText(raw.look) || undefined, wardrobe: normalizeText(raw.wardrobe) || undefined, signatureAction: normalizeText(raw.signatureAction) || undefined, traits: normalizeStringList(raw.traits, 6), }; }) .filter((item): item is NonNullable => Boolean(item)) .slice(0, 8); } function normalizeScenes(value: unknown) { if (!Array.isArray(value)) return []; return value .map((item) => { const raw = item && typeof item === "object" && !Array.isArray(item) ? (item as Record) : {}; const name = normalizeText(raw.name || raw.title); const description = normalizeText(raw.description || raw.summary); const prompt = normalizeText(raw.prompt); if (!name && !description && !prompt) return null; return { name: name || "未命名场景", description, prompt, mood: normalizeText(raw.mood) || undefined, lighting: normalizeText(raw.lighting) || undefined, props: normalizeStringList(raw.props, 6), }; }) .filter((item): item is NonNullable => Boolean(item)) .slice(0, 8); } function buildVideoAnalysisPrompt(params: { nodeTitle?: string; prompt: string; imageCount: number; videoCount: number; }) { return [ "你是视频画布工作流里的视频分析节点,需要为后续分镜和生成节点提供结构化输出。", params.nodeTitle ? `当前节点:${params.nodeTitle}` : "", `上游参考图数量:${params.imageCount}`, `上游视频数量:${params.videoCount}`, params.prompt ? `上游脚本/说明:\n${params.prompt}` : "上游没有额外脚本,请重点依据参考画面分析。", "", "请只输出 JSON,不要输出 markdown,不要输出解释。", "JSON 结构如下:", `{ "summary": "一句话总结当前视频/镜头风格,不超过120字", "highlights": ["3-5条镜头分析要点"], "shotIdeas": [ { "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" } ], "visualMotifs": ["2-4个可复用的视觉母题"], "recommendedNextSteps": ["2-4条建议的下游动作"] }`, "如果没有真实视频帧,只能依据参考图和脚本推断,也要正常输出,但措辞要稳健。", ] .filter(Boolean) .join("\n"); } function buildCharacterScenePrompt(params: { nodeTitle?: string; prompt: string; imageCount: number; }) { return [ "你是视频画布工作流里的角色场景提取节点,需要从脚本、设定和参考图里提取角色与场景资产。", params.nodeTitle ? `当前节点:${params.nodeTitle}` : "", `参考图数量:${params.imageCount}`, params.prompt ? `输入内容:\n${params.prompt}` : "没有额外脚本,请根据参考图提取角色和场景线索。", "", "请只输出 JSON,不要输出 markdown,不要输出解释。", "JSON 结构如下:", `{ "summary": "一句话总结当前故事世界与资产线索", "characters": [ { "name": "角色名", "description": "角色外观与身份简介", "look": "外貌特征", "wardrobe": "服装/道具", "traits": ["性格关键词"], "signatureAction": "代表动作", "prompt": "可直接用于生成角色图/角色视频的提示词" } ], "scenes": [ { "name": "场景名", "description": "空间、布景和氛围简介", "mood": "情绪与气氛", "lighting": "布光特征", "props": ["核心道具"], "prompt": "可直接用于生成场景图/场景视频的提示词" } ], "relationships": ["角色与场景或角色之间的关系"], "recommendedNextSteps": ["建议连接的后续节点动作"] }`, "角色和场景都要尽量具体,避免空泛形容词。", ] .filter(Boolean) .join("\n"); } function buildAgentPlanPrompt(params: { nodeTitle?: string; prompt: string; imageCount: number; videoCount: number; }) { return [ "你是视频画布里的自定义代理节点,需要把当前创作目标拆成可执行的工作流计划。", params.nodeTitle ? `当前节点:${params.nodeTitle}` : "", `参考图数量:${params.imageCount}`, `视频数量:${params.videoCount}`, params.prompt ? `上游需求:\n${params.prompt}` : "没有额外脚本,请基于当前上下文给出稳妥计划。", "", "请只输出 JSON,不要输出 markdown,不要输出解释。", "JSON 结构如下:", `{ "summary": "一句话说明这个代理链的目标", "agentPlan": ["3-6个执行步骤"], "deliverables": ["2-4个预期交付物"], "recommendedNodes": ["建议继续连接的节点类型或能力名"] }`, "步骤要偏工作流视角,而不是泛泛而谈。", ] .filter(Boolean) .join("\n"); } export async function POST(request: Request) { try { const session = await auth(); if (!session?.user?.id) { return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); } const body = await request.json().catch(() => ({})); const parsed = requestSchema.safeParse(body); if (!parsed.success) { return NextResponse.json( { success: false, error: "请求参数不合法" }, { status: 400 } ); } const prompt = normalizeText(parsed.data.prompt); const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4); const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4); if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) { return NextResponse.json( { success: false, error: "请至少提供脚本、图片或视频上下文" }, { status: 400 } ); } const [allModels, agentModel] = await Promise.all([ getEnabledAiModelsWithProvider(false), getAgentAiModelWithProvider(), ]); const compatibleModels = allModels.filter( (item) => !item.model.isSubModel && item.model.isEnabled ); let selectedModel = imageUrls.length > 0 && agentModel && supportsVision(agentModel.model) ? agentModel : imageUrls.length > 0 ? compatibleModels.find((item) => supportsVision(item.model)) : agentModel || compatibleModels[0]; if (!selectedModel) { selectedModel = compatibleModels[0]; } if (!selectedModel) { return NextResponse.json( { success: false, error: "当前站点没有可用的分析模型" }, { status: 500 } ); } const useVision = imageUrls.length > 0 && supportsVision(selectedModel.model); const codingModelId = getAiModelAgentCodingModelId(selectedModel.model); const model = await getLanguageModelById(selectedModel.model.id, codingModelId); if (!model) { return NextResponse.json( { success: false, error: "分析模型初始化失败" }, { status: 500 } ); } const promptText = parsed.data.mode === "video-analysis" ? buildVideoAnalysisPrompt({ nodeTitle: normalizeText(parsed.data.nodeTitle), prompt, imageCount: imageUrls.length, videoCount: videoUrls.length, }) : parsed.data.mode === "character-scene-extraction" ? buildCharacterScenePrompt({ nodeTitle: normalizeText(parsed.data.nodeTitle), prompt, imageCount: imageUrls.length, }) : buildAgentPlanPrompt({ nodeTitle: normalizeText(parsed.data.nodeTitle), prompt, imageCount: imageUrls.length, videoCount: videoUrls.length, }); const referenceImages = useVision ? ( await inlineVisionImages(imageUrls, { maxImages: 4, maxSide: 960, timeoutMs: 12000, retries: 1, loggerPrefix: "[Canvas Workflow Analyze]", }) ).inlineDataUrls : []; const { text } = await generateText({ model, messages: [ { role: "user", content: [ { type: "text", text: promptText }, ...referenceImages.map((image) => ({ type: "image" as const, image })), ], }, ], temperature: 0.2, maxTokens: 1600, } as any); const parsedJson = extractJsonBlock>(text) || {}; const summary = normalizeText(parsedJson.summary) || normalizeText(text).slice(0, 160); if (parsed.data.mode === "video-analysis") { return NextResponse.json({ success: true, data: { summary, highlights: normalizeStringList(parsedJson.highlights, 6), shotIdeas: normalizeShotIdeas(parsedJson.shotIdeas), visualMotifs: normalizeStringList(parsedJson.visualMotifs, 6), recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6), model: selectedModel.model.displayName, raw: text, }, }); } if (parsed.data.mode === "character-scene-extraction") { return NextResponse.json({ success: true, data: { summary, characters: normalizeCharacters(parsedJson.characters), scenes: normalizeScenes(parsedJson.scenes), relationships: normalizeStringList(parsedJson.relationships, 8), recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6), model: selectedModel.model.displayName, raw: text, }, }); } return NextResponse.json({ success: true, data: { summary, agentPlan: normalizeStringList(parsedJson.agentPlan, 8), deliverables: normalizeStringList(parsedJson.deliverables, 6), recommendedNodes: normalizeStringList(parsedJson.recommendedNodes, 6), model: selectedModel.model.displayName, raw: text, }, }); } catch (error) { console.error("[Canvas Workflow Analyze] Error:", error); return NextResponse.json( { success: false, error: error instanceof Error ? error.message : "工作流分析失败", }, { status: 500 } ); } }