/** * 视频生成工具 * 支持火山引擎视频生成的多种模式 */ import { tool } from "ai"; import { z } from "zod"; import { doGenerateStream } from "@/lib/ai/providers/volcano-video"; import { getEnabledAiModelsWithProvider, getSubModelsByParentModelId } from "@/lib/db/queries"; import type { VideoGenerationMode, VideoModelCapabilities, VideoGenerationOptions, } from "@/types/ai"; /** * 🆕 智能选择最佳子模型 * 根据用户选择的模式和参数,从子模型列表中选择最合适的模型 */ async function selectBestSubModel( parentModelId: string, mode: VideoGenerationMode, params: Partial ): Promise<{ model: any; capabilities: VideoModelCapabilities }> { // 获取所有子模型 const subModels = await getSubModelsByParentModelId(parentModelId); if (subModels.length === 0) { throw new Error("没有可用的视频生成子模型"); } console.log(`[Smart Model Selection] 找到 ${subModels.length} 个子模型`); // 筛选支持指定模式的子模型 const compatibleModels = subModels.filter((m) => { const caps = m.model.capabilities as VideoModelCapabilities; const supportsMode = caps.supportedModes?.includes(mode); // 检查分辨率支持(如果用户指定了分辨率) const supportsResolution = !params.resolution || caps.supportedResolutions?.includes(params.resolution); return supportsMode && supportsResolution; }); if (compatibleModels.length === 0) { throw new Error(`没有模型支持 ${mode} 模式和指定的参数`); } console.log(`[Smart Model Selection] ${compatibleModels.length} 个模型支持 ${mode} 模式`); // 🎯 智能优先级排序 const priority: Record = { "doubao-seedance-1-0-pro-250528": 10, // 最高优先级(全功能) "doubao-seedance-1-0-pro-fast-251015": 8, // 快速模式 "doubao-seedance-1-0-lite-i2v-250428": 6, // 图生视频轻量版 "doubao-seedance-1-0-lite-t2v-250428": 5, // 文生视频轻量版 }; // 选择最佳模型 const bestModel = compatibleModels.sort((a, b) => { const priorityA = priority[a.model.modelId] || 0; const priorityB = priority[b.model.modelId] || 0; return priorityB - priorityA; })[0]; console.log(`[Smart Model Selection] ✅ 选择模型: ${bestModel.model.displayName} (${bestModel.model.modelId})`); return { model: bestModel, capabilities: bestModel.model.capabilities as VideoModelCapabilities, }; } /** * 获取支持指定模式的视频模型(兼容旧逻辑) */ async function getCompatibleModel( mode: VideoGenerationMode, parentModelId?: string, params?: Partial ): Promise<{ model: any; capabilities: VideoModelCapabilities }> { // 🆕 如果提供了parentModelId,使用智能子模型选择 if (parentModelId) { return selectBestSubModel(parentModelId, mode, params || {}); } // 兼容旧逻辑:直接从所有模型中选择(用于向后兼容) const models = await getEnabledAiModelsWithProvider(); const volcanoVideoModels = models.filter( (m) => m.provider.type === "volcano-video" && m.model.isEnabled && m.model.capabilities ); if (volcanoVideoModels.length === 0) { throw new Error("没有启用的火山引擎视频模型"); } const compatibleModels = volcanoVideoModels.filter((m) => { const caps = m.model.capabilities as VideoModelCapabilities; return caps.supportedModes?.includes(mode); }); if (compatibleModels.length === 0) { throw new Error(`没有模型支持 ${mode} 模式`); } const priority: Record = { "doubao-seedance-1-0-pro-250528": 10, "doubao-seedance-1-0-pro-fast-251015": 8, "doubao-seedance-1-0-lite-i2v-250428": 6, "doubao-seedance-1-0-lite-t2v-250428": 5, }; const bestModel = compatibleModels.sort((a, b) => { const priorityA = priority[a.model.modelId] || 0; const priorityB = priority[b.model.modelId] || 0; return priorityB - priorityA; })[0]; return { model: bestModel, capabilities: bestModel.model.capabilities as VideoModelCapabilities, }; } /** * 验证参数是否符合模型能力 */ function validateParamsAgainstCapabilities( params: Partial, capabilities: VideoModelCapabilities ): void { // 检查分辨率 if ( params.resolution && !capabilities.supportedResolutions.includes(params.resolution) ) { throw new Error( `模型不支持 ${params.resolution} 分辨率,支持的分辨率:${capabilities.supportedResolutions.join(", ")}` ); } // 检查宽高比 if (params.ratio && !capabilities.supportedRatios.includes(params.ratio)) { throw new Error( `模型不支持 ${params.ratio} 宽高比,支持的宽高比:${capabilities.supportedRatios.join(", ")}` ); } // 检查时长范围 if (params.duration) { const { min, max } = capabilities.durationRange; if (params.duration < min || params.duration > max) { throw new Error(`视频时长必须在 ${min}-${max} 秒之间`); } } // 检查参考图数量 if ( params.mode === "reference-images" && params.referenceImages && capabilities.maxReferenceImages ) { if (params.referenceImages.length > capabilities.maxReferenceImages) { throw new Error( `参考图数量不能超过 ${capabilities.maxReferenceImages} 张` ); } } } /** * 构建视频生成选项 */ function buildVideoOptions( mode: VideoGenerationMode, images: string[] | undefined, prompt: string, params: any, modelId: string ): VideoGenerationOptions { const options: VideoGenerationOptions = { prompt, model: modelId, mode, resolution: params.resolution || "720p", ratio: params.ratio || "16:9", duration: params.duration || 5, watermark: params.watermark ?? true, cameraFixed: params.cameraFixed ?? false, fps: 24, }; // 根据模式处理图片输入 if (mode === "first-frame" && images && images.length > 0) { options.firstFrameImage = images[0]; } else if (mode === "first-last-frame" && images && images.length >= 2) { options.firstFrameImage = images[0]; options.lastFrameImage = images[1]; } else if (mode === "reference-images" && images && images.length > 0) { options.referenceImages = images.slice(0, 4); } return options; } /** * 视频生成工具 */ export function generateVideo({ dataStream }: { dataStream: any }) { return tool({ description: "生成视频,支持文生视频、图生视频(首帧/首尾帧/参考图)等多种模式", inputSchema: z.object({ prompt: z.string().describe("视频生成提示词,描述视频内容、镜头运动等"), mode: z .enum(["text-to-video", "first-frame", "first-last-frame", "reference-images"]) .describe("生成模式:文生视频/图生视频-首帧/首尾帧/参考图"), images: z .array(z.string()) .optional() .describe("图片 URL 数组(根据模式不同,需要 0-4 张)"), resolution: z .enum(["480p", "720p", "1080p"]) .default("720p") .describe("视频分辨率"), ratio: z .enum(["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"]) .default("16:9") .describe("视频宽高比"), duration: z .number() .min(2) .max(12) .default(5) .describe("视频时长(2-12秒)"), watermark: z.boolean().default(true).describe("是否添加水印"), cameraFixed: z.boolean().default(false).describe("是否固定摄像头"), seed: z.number().optional().describe("随机种子(可选)"), returnLastFrame: z .boolean() .default(false) .describe("是否返回尾帧(用于连续生成)"), }), execute: async ({ prompt, mode, images, resolution, ratio, duration, watermark, cameraFixed, seed, returnLastFrame }) => { try { console.log("[Generate Video Tool] 🎬 开始生成视频"); console.log("[Generate Video Tool] 模式:", mode); console.log("[Generate Video Tool] 提示词:", prompt); // 1. 🆕 智能选择最佳子模型 // 先获取火山引擎视频主模型 const allModels = await getEnabledAiModelsWithProvider(); const volcanoMasterModel = allModels.find( m => m.provider.type === "volcano-video" && m.model.modelId === "volcano-video-master" ); if (!volcanoMasterModel) { throw new Error("未找到火山引擎视频主模型,请在系统设置中添加"); } // 准备参数用于智能选择 const params = { mode, resolution, ratio, duration, watermark, cameraFixed, seed, returnLastFrame, referenceImages: images, }; // 根据用户参数智能选择最佳子模型 const { model, capabilities } = await selectBestSubModel( volcanoMasterModel.model.id, mode, params ); console.log( "[Generate Video Tool] ✅ 智能选择模型:", model.model.displayName, `(${model.model.modelId})` ); // 2. 验证参数是否符合模型能力 validateParamsAgainstCapabilities(params, capabilities); // 3. 构建视频生成选项 const options = buildVideoOptions( mode, images, prompt, params, model.model.modelId ); // 4. 调用视频生成(流式输出) dataStream.writeData({ type: "video-generation-start", data: { mode, model: model.model.displayName, }, }); const apiKey = model.provider.apiKey; const baseURL = model.provider.baseUrl; let videoMetadata: any = null; // 使用异步生成器进行流式输出 for await (const chunk of doGenerateStream(apiKey, baseURL, options)) { dataStream.writeMessageAnnotation({ type: "video-progress", content: chunk, }); // 提取视频元数据(从注释中) const metadataMatch = chunk.match( // ); if (metadataMatch) { try { videoMetadata = JSON.parse(metadataMatch[1]); } catch (e) { console.error("[Generate Video Tool] 解析元数据失败:", e); } } } console.log("[Generate Video Tool] ✅ 视频生成完成"); // 5. 返回元数据(用于保存到数据库) return { success: true, videoUrl: videoMetadata?.temporaryVideoUrl, metadata: videoMetadata, }; } catch (error: any) { console.error("[Generate Video Tool] ❌ 生成失败:", error); dataStream.writeMessageAnnotation({ type: "video-error", content: `视频生成失败:${error.message}`, }); throw error; } }, }); }