import { defaultVideoCanvasFunctionConfig } from "@/lib/video-canvas/config"; import { createHash } from "node:crypto"; import { tool, type ToolSet } from "ai"; import type { DiscountableImageGenConfigLike } from "@/lib/ai/image-model-pricing"; import { z } from "zod"; import type { CanvasAgentTool } from "./loop"; import type { AgentRunSnapshot } from "./protocol"; import { readOwnedVideoCanvas, appendVideoAgentNode, replaceVideoAgentUrl } from "./video-store"; import { videoNodeType, videoNodeUrl, resolveVideoMode, type VideoRuntimeNode, } from "./video-protocol"; import { registerVideoCanvasAccessTools } from "./video-access-tools"; import type { VideoCanvasFunctionConfig, VideoCanvasFunctionDefinition, } from "@/lib/video-canvas/config"; import type { VideoCanvasMutation } from "./video-store"; export type VideoRuntimePorts = { featureConfig?: VideoCanvasFunctionConfig; processMedia?: ( input: import("./video-media-functions").VideoMediaInput, ) => Promise<{ url: string; type: "image" | "audio" | "video"; duration: number }>; emit?: (type: string, data: unknown) => void; mutate?: (id: string, input: VideoCanvasMutation) => Promise>; processImage?: (input: { definition: VideoCanvasFunctionDefinition; imageUrl: string; args: Record; prompt: string; callId: string; }) => Promise>; archiveResults?: () => Promise; read: () => Promise<{ nodes: VideoRuntimeNode[]; edges: Array<{ source: string; target: string }>; }>; append: (node: VideoRuntimeNode, sourceNodeIds: string[]) => Promise; analyze: (input: { mode: "video-analysis" | "agent-plan" | "story-script"; prompt: string; imageUrls: string[]; videoUrls: string[]; }) => Promise>; generateImage: (input: { prompt: string; references: string[]; callId: string; functionKey?: string; modelRef?: string; aspectRatio?: string; }) => Promise<{ url: string; model: string }>; generateVideo: (input: { prompt: string; references: string[]; callId: string; mode: ReturnType; }) => Promise<{ url: string; model: string; metadata?: unknown }>; }; export function buildVideoRuntimeTools(params: { runId: string; projectId?: string; selectedNodeIds: string[]; snapshot: AgentRunSnapshot; ports: VideoRuntimePorts; }) { const definitions: ToolSet = {}; const executable: Record = {}; function register( name: string, title: string, schema: T, execute: (input: z.infer, callId: string) => Promise>, readOnly = true, ) { definitions[name] = tool({ description: title, inputSchema: schema }); executable[name] = { title, readOnly, concurrentSafe: readOnly, execute: (input, callId) => execute(schema.parse(input), callId), }; } const ids = z.array(z.string().min(1).max(180)).max(4); const references = async (nodeIds: string[], type?: string) => { const current = await params.ports.read(); return nodeIds.map((id) => { const node = current.nodes.find((item) => item.id === id); if (!node || (type && videoNodeType(node) !== type) || !videoNodeUrl(node)) throw new Error(`素材节点不可用或尚未上传:${id}`); return node; }); }; const makeNode = ( callId: string, title: string, mediaType: "text" | "image" | "video" | "audio", draft: Record, ): VideoRuntimeNode => ({ id: `agent-${createHash("sha256").update(`${params.runId}:${callId}`).digest("hex").slice(0, 32)}`, kind: mediaType === "text" ? "script" : "media", title, subtitle: "Agent 生成结果", inputType: mediaType, x: 0, y: 0, width: mediaType === "text" ? 420 : 320, height: mediaType === "text" ? 300 : 220, accent: "#8790ff", draftValues: { ...(mediaType === "text" ? { action: "TEXT_RESOURCE", scriptRole: "resource" } : {}), mediaType, nodeType: mediaType, generated: true, generationStatus: "complete", generationProgress: 100, canvasAgentRunId: params.runId, ...draft, }, }); register( "planCanvasTask", "制定可核验的视频创作步骤,明确最终视频数量;不包含中间关键帧。", z.object({ steps: z.array(z.string().min(1).max(240)).min(1).max(20), expectedVideoCount: z.number().int().min(1).max(12).optional(), visualCriteria: z.array(z.string().max(240)).max(10).default([]), }), async (input) => ({ ...input }), ); register( "readCanvas", "读取真实视频画布节点、连线、文本和素材类型,不会读取其他项目。", z.object({ nodeIds: z.array(z.string().min(1).max(180)).max(60).optional() }), async (input) => { const current = await params.ports.read(); const candidates = input.nodeIds?.length ? current.nodes.filter((node) => input.nodeIds!.includes(node.id)) : [ ...current.nodes.filter((node) => params.selectedNodeIds.includes(node.id)), ...current.nodes.filter((node) => !params.selectedNodeIds.includes(node.id)).slice(-50), ]; return { selectedNodeIds: params.selectedNodeIds, totalNodes: current.nodes.length, nodes: candidates.slice(0, 60).map((node) => ({ id: node.id, title: node.title, type: videoNodeType(node), position: { x: node.x, y: node.y }, hasMedia: Boolean(videoNodeUrl(node)), content: JSON.stringify( node.draftValues?.rows || node.draftValues?.content || node.draftValues?.prompt || "", ).slice(0, 6000), })), edges: current.edges.slice(0, 120), }; }, ); register( "analyzeVideoMaterials", "分析真实图片或视频(视频抽取关键帧),返回事实与分镜建议。不能仅凭文件名臆测。", z.object({ nodeIds: ids.min(1), question: z.string().min(1).max(4000) }), async (input) => { const nodes = await references(input.nodeIds); if (nodes.some((node) => !["image", "video"].includes(videoNodeType(node)))) throw new Error("当前分析仅支持图片或视频素材"); return params.ports.analyze({ mode: nodes.some((node) => videoNodeType(node) === "video") ? "video-analysis" : "agent-plan", prompt: input.question, imageUrls: nodes.filter((node) => videoNodeType(node) === "image").map(videoNodeUrl), videoUrls: nodes.filter((node) => videoNodeType(node) === "video").map(videoNodeUrl), }); }, ); register( "writeVideoScript", "将真实分镜脚本写入新节点。只完成脚本,不会自动生成视频。", z.object({ title: z.string().min(1).max(100), sourceNodeIds: ids.default([]), rows: z .array( z.object({ sceneDescription: z.string().min(1).max(2000), imagePrompt: z.string().min(1).max(4000), motionPrompt: z.string().min(1).max(4000), duration: z.number().min(1).max(120), }), ) .min(1) .max(24), }), async (input, callId) => { const current = await params.ports.read(); if (input.sourceNodeIds.some((id) => !current.nodes.some((node) => node.id === id))) throw new Error("脚本参考节点不存在"); const rows = input.rows.map((row, index) => ({ ...row, shot: String(index + 1).padStart(2, "0"), scene: row.sceneDescription, })); const node = makeNode(callId, input.title, "text", { rows, content: [ rows .map((row) => `${row.shot}. ${row.sceneDescription}\n${row.imagePrompt}\n${row.motionPrompt}`) .join("\n\n"), ], }); return { createdNodeIds: [await params.ports.append(node, input.sourceNodeIds)], type: "script", shotCount: rows.length, }; }, false, ); register( "generateVideoKeyframe", "根据已写入的分镜或用户要求生成一张关键帧,保留所有原图。referenceNodeIds 只能是图片。", z.object({ prompt: z.string().min(1).max(8000), referenceNodeIds: ids.default([]), }), async (input, callId) => { const refs = await references(input.referenceNodeIds, "image"); const result = await params.ports.generateImage({ prompt: input.prompt, references: refs.map(videoNodeUrl), callId, }); if (!/^https?:\/\//i.test(result.url)) throw new Error("关键帧未返回有效图片地址"); const node = makeNode(callId, "分镜关键帧", "image", { mediaUrl: result.url, url: [result.url], prompt: input.prompt, model: result.model, }); return { createdNodeIds: [await params.ports.append(node, input.referenceNodeIds)], type: "image", }; }, false, ); register( "generateVideoShot", "生成一个视频镜头。首尾帧的 referenceNodeIds 必须按首帧、尾帧排序。沿用用户选择的模型和参数,不擅自重试付费请求。", z.object({ prompt: z.string().min(1).max(8000), referenceNodeIds: ids.default([]), mode: z .enum(["text-to-video", "first-frame", "first-last-frame", "reference-images"]) .optional(), }), async (input, callId) => { if (!params.snapshot.videoRequirements?.expectsVideo) throw new Error("用户尚未授权生成视频,请先明确是否需要生成成片。"); const started = params.snapshot.calls.filter( (call) => (call.name === "generateVideoShot" || (call.name === "runVideoCanvasFunction" && String(call.input.functionKey) !== "audio-to-video" && (params.ports.featureConfig || defaultVideoCanvasFunctionConfig).functions[ String(call.input.functionKey) ]?.route === "canvas-generate-video")) && (call.status === "succeeded" || call.status === "started"), ).length; const plannedCount = Number( params.snapshot.calls.find( (call) => call.name === "planCanvasTask" && call.status === "succeeded", )?.output?.expectedVideoCount || 1, ); const allowedCount = Math.min( 12, params.snapshot.videoRequirements?.expectedVideoCount ?? plannedCount, ); if (started > allowedCount) throw new Error(`本次已达到约定的 ${allowedCount} 条视频数量,未再发出付费请求。`); const refs = await references(input.referenceNodeIds, "image"); if ( input.mode && params.snapshot.videoSettings!.mode !== "auto" && input.mode !== params.snapshot.videoSettings!.mode ) throw new Error("功能模式与用户选择的模式不一致,请先调整模式"); const mode = resolveVideoMode(input.mode || params.snapshot.videoSettings!.mode, refs.length); const result = await params.ports.generateVideo({ prompt: input.prompt, references: refs.map(videoNodeUrl), callId, mode, }); if (!/^https?:\/\//i.test(result.url)) throw new Error("视频尚未返回有效结果,已停止后续付费请求,请核对任务状态。"); const node = makeNode(callId, "分镜视频", "video", { mediaUrl: result.url, url: [result.url], prompt: input.prompt, model: result.model, generatedByAgentVideo: true, duration: params.snapshot.videoSettings!.duration, ratio: params.snapshot.videoSettings!.ratio, resolution: params.snapshot.videoSettings!.resolution, referenceNodeIds: input.referenceNodeIds, mode, }); return { createdNodeIds: [await params.ports.append(node, input.referenceNodeIds)], type: "video", mode, }; }, false, ); registerVideoCanvasAccessTools({ ...params, definitions, executable, makeNode }); return { definitions, executable }; } /** Production ports delegate to existing provider, billing and storage implementations. */ export async function createVideoRuntimePorts(params: { userId: string; projectId: string; runId: string; chatId: string; assistantMessageId: string; request: Request; snapshot: AgentRunSnapshot; emit?: (type: string, data: unknown) => void; }): Promise { const { getEnabledAiModelsWithProvider, checkUserImageQuota } = await import("@/lib/db/queries"); const { isVideoModelDescriptor, isImageGenerationModelDescriptor } = await import( "@/lib/ai/model-classification" ); const { getModelGenerationCreditCost } = await import("@/lib/ai/image-model-pricing"); const { prepareFeaturePointsCharge } = await import("@/lib/points/feature-charge"); const models = await getEnabledAiModelsWithProvider(); const { getSystemSetting } = await import("@/lib/db/queries"); const { VIDEO_CANVAS_FUNCTION_CONFIG_KEY, normalizeVideoCanvasFunctionConfig } = await import( "@/lib/video-canvas/config" ); const featureConfig = normalizeVideoCanvasFunctionConfig( (await getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }))?.value, ); const pendingArchives: Array<{ nodeId: string; messageId: string; url: string }> = []; const choose = (kind: "video" | "image", ref?: string) => { const candidates = models.filter(({ model, provider }) => (kind === "video" ? isVideoModelDescriptor : isImageGenerationModelDescriptor)({ ...model, providerType: provider.type, }), ); const chosen = ref ? candidates.find(({ model }) => model.id === ref || model.modelId === ref) : candidates.find(({ model }) => model.isDefault) || candidates[0]; if (!chosen) throw new Error(`所选${kind === "video" ? "视频" : "图片"}模型不可用,请检查后台配置。`); return chosen; }; return { featureConfig, processMedia: async (input) => (await import("./video-media-functions")).processVideoMedia(input), emit: params.emit, mutate: async (id, input) => { const { mutateVideoAgentCanvas } = await import("./video-store"); const result = await mutateVideoAgentCanvas(params.userId, params.projectId, id, input); params.emit?.("data-videoCanvasMutation", { projectId: params.projectId, ...result, action: input.action, nodeIds: input.nodeIds, viewportChanged: input.action === "viewport", }); return result; }, processImage: async (input) => (await import("./video-image-functions")).processVideoCanvasImage({ ...input, userId: params.userId, request: params.request, }), read: async () => { const current = await readOwnedVideoCanvas(params.userId, params.projectId); return { nodes: current.nodes, edges: current.project.canvasData?.edges || [] }; }, append: async (node, sources) => { const id = await appendVideoAgentNode(params.userId, params.projectId, node, sources); if (videoNodeType(node) === "video") { const { saveMessages } = await import("@/lib/db/queries"); const messageId = crypto.randomUUID(); const url = videoNodeUrl(node); await saveMessages({ messages: [ { id: messageId, chatId: params.chatId, role: "assistant", parts: [{ type: "text", text: `[视频镜头](${url})` }], attachments: [], createdAt: new Date(), videoStorageStatus: "temporary", temporaryVideoUrl: url, permanentVideoUrl: null, archivedAt: null, videoMetadata: { taskId: node.id, duration: Number(node.draftValues?.duration), ratio: String(node.draftValues?.ratio), resolution: String(node.draftValues?.resolution), }, }, ], }); pendingArchives.push({ nodeId: id, messageId, url }); } return id; }, archiveResults: async () => { const { videoArchiver } = await import("@/lib/storage/video-archiver"); for (const result of pendingArchives) { try { const permanentUrl = await videoArchiver.archiveVideo({ messageId: result.messageId, taskId: result.nodeId, temporaryUrl: result.url, }); await replaceVideoAgentUrl( params.userId, params.projectId, result.nodeId, result.url, permanentUrl, ); } catch { // Keep the provider URL available and let the existing temporary-video job retry. const { db } = await import("@/lib/db"); const { message } = await import("@/lib/db/schema"); const { eq } = await import("drizzle-orm"); await db .update(message) .set({ videoStorageStatus: "temporary" }) .where(eq(message.id, result.messageId)); } } }, analyze: async (input) => { const { POST } = await import("@/app/api/canvas/workflow/analyze/route"); const headers = new Headers(params.request.headers); headers.set("Content-Type", "application/json"); const response = await POST( new Request(new URL("/api/canvas/workflow/analyze", params.request.url), { method: "POST", headers, body: JSON.stringify(input), }), ); const body = await response.json(); if (!response.ok || !body.success) throw new Error(body.error || "视频素材分析失败"); // Do not keep base64 previews in the long-lived model history. const { framePreviews: previews, ...data } = body.data; if (input.videoUrls.length && (!Array.isArray(previews) || !previews.length)) throw new Error("视频无法抽取真实画面,不能仅凭提示词完成分析"); if (Array.isArray(data.analysisRows)) data.analysisRows = data.analysisRows.map( ({ keyframeUrl: _url, ...row }: Record) => row, ); return { ...data, inspectedFrameCount: Array.isArray(previews) ? previews.length : 0 }; }, generateImage: async ({ prompt, references, functionKey, modelRef, aspectRatio }) => { const chosen = choose("image", params.snapshot.videoSettings?.imageModelRef || modelRef); const billingFeature = functionKey === "character-three-view" || functionKey === "multi-view-nine-grid" ? "multi-view" : functionKey === "image-repaint" ? "repaint" : null; const { getPointsSystemConfig } = await import("@/lib/db/queries"); const { getImageToolCreditCost } = await import("@/lib/points/config"); const pointsConfig = await getPointsSystemConfig(); const quota = await checkUserImageQuota(params.userId); if (!quota.allowed) throw new Error("本月图片生成配额不足"); const charge = await prepareFeaturePointsCharge({ userId: params.userId, unitCost: billingFeature ? getImageToolCreditCost(pointsConfig, billingFeature) : getModelGenerationCreditCost( { imageGenConfig: chosen.model .imageGenConfig as DiscountableImageGenConfigLike | null, }, "image", { imageSize: "2K" }, ), reason: "image_generation", modelId: chosen.model.id, messageId: params.assistantMessageId, note: "视频 Agent 关键帧生成", }); if (!charge.ok) throw new Error(charge.payload.message); const { generateCanvasImage } = await import("@/lib/canvas/generate-image"); const result = await generateCanvasImage({ prompt, model: chosen.model.id, imageCount: 1, imageSize: "2K", aspectRatio: aspectRatio || params.snapshot.videoSettings!.ratio, referenceImages: references, ownerKey: params.userId, }); const image = result.images[0]; if (!image) throw new Error("模型未返回关键帧"); // Existing persistence records generated assets and charges image generation once. const { saveGeneratedImages } = await import("@/lib/ai/runtime/generated-image-persistence"); await saveGeneratedImages( `![thumb|||${image.originalUrl}|||2K](${image.thumbnailUrl})`, params.userId, params.chatId, params.assistantMessageId, prompt, result.modelRef, { projectId: params.projectId, sourceImageUrls: references, relationType: references.length ? "edited_from" : "generated_from", skipBilling: Boolean(billingFeature), imageSize: "2K", aspectRatio: aspectRatio || params.snapshot.videoSettings!.ratio, }, ); if (billingFeature) await charge.consume(); return { url: image.originalUrl, model: result.modelRef }; }, generateVideo: async ({ prompt, references, mode }) => { const settings = params.snapshot.videoSettings!; const chosen = choose("video", settings.modelRef); const charge = await prepareFeaturePointsCharge({ userId: params.userId, unitCost: getModelGenerationCreditCost( { imageGenConfig: chosen.model.imageGenConfig as DiscountableImageGenConfigLike | null }, "video", { duration: settings.duration, resolution: settings.resolution, }, ), reason: "video_generation", modelId: chosen.model.id, messageId: params.assistantMessageId, note: "视频 Agent 镜头生成", }); if (!charge.ok) throw new Error(charge.payload.message); const { executeDirectVideoGeneration } = await import("@/lib/ai/tools/generate-video"); const result = await executeDirectVideoGeneration({ prompt, mode, images: references, duration: settings.duration, ratio: settings.ratio, resolution: settings.resolution, preferredModelRef: chosen.model.id, }); if (!result.success || !result.videoUrl) throw new Error("视频模型未返回有效成片,请核对服务商任务后继续。"); await charge.consume({ modelId: result.modelRef }); return { url: result.videoUrl, model: result.modelRef, metadata: result.metadata }; }, }; }