/** * 视频生成工具 * 支持火山引擎视频生成的多种模式 */ import { tool } from "ai"; import { z } from "zod"; import { doGenerateStream } from "@/lib/ai/providers/volcano-video"; import { getEnabledAiModelsWithProvider, getSubModelsByParentModelId } from "@/lib/db/queries"; import { getConfiguredVideoDurationOptions } from "@/lib/ai/video-model-config"; import type { VideoGenerationMode, VideoModelCapabilities, VideoGenerationOptions, } from "@/types/ai"; export interface ExecuteDirectVideoGenerationParams { prompt: string; mode: VideoGenerationMode; images?: string[]; resolution?: "480p" | "720p" | "1080p"; ratio?: "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "21:9" | "adaptive"; duration?: number; watermark?: boolean; cameraFixed?: boolean; seed?: number; returnLastFrame?: boolean; preferredModelRef?: string; onProgress?: (chunk: string) => void | Promise; } export interface ExecuteDirectVideoGenerationResult { success: boolean; videoUrl?: string; metadata?: Record | null; modelDisplayName: string; modelId: string; modelRef: string; } function inferSupportedModesFromModelId(modelId: string): VideoGenerationMode[] { const normalized = modelId.toLowerCase(); if (normalized.includes("lite-t2v")) return ["text-to-video"]; if (normalized.includes("lite-i2v")) return ["first-frame", "first-last-frame", "reference-images"]; if (normalized.includes("pro-fast")) return ["text-to-video", "first-frame"]; if (normalized.includes("-pro-") || normalized.endsWith("-pro")) { return ["text-to-video", "first-frame", "first-last-frame"]; } return ["text-to-video", "first-frame", "first-last-frame", "reference-images"]; } function getSupportedModes(caps: Partial | undefined, modelId: string): VideoGenerationMode[] { const modes = (caps?.supportedModes || []) as VideoGenerationMode[]; if (Array.isArray(modes) && modes.length > 0) { return modes; } return inferSupportedModesFromModelId(modelId); } function withConfiguredVideoCapabilities( capabilities: Partial | undefined, imageGenConfig?: { videoDurationOptions?: number[] | null } | null ): VideoModelCapabilities { const baseCapabilities = { supportedModes: ["text-to-video", "first-frame", "first-last-frame", "reference-images"], supportedResolutions: ["480p", "720p", "1080p"], supportedRatios: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"], durationRange: { min: 2, max: 12 }, maxReferenceImages: 4, ...(capabilities || {}), } as VideoModelCapabilities; const durationOptions = getConfiguredVideoDurationOptions(imageGenConfig); if (durationOptions.length === 0) { return baseCapabilities; } return { ...baseCapabilities, durationOptions, durationRange: { min: durationOptions[0], max: durationOptions[durationOptions.length - 1], }, }; } /** * 🆕 智能选择最佳子模型 * 根据用户选择的模式和参数,从子模型列表中选择最合适的模型 */ async function selectBestSubModel( parentModelId: string, mode: VideoGenerationMode, params: Partial ): Promise<{ model: any; capabilities: VideoModelCapabilities }> { // 获取所有子模型 const subModels = await getSubModelsByParentModelId(parentModelId); if (subModels.length === 0) { const allModels = await getEnabledAiModelsWithProvider(); const directModel = allModels.find( (item) => item.model.id === parentModelId && (item.provider.type === "volcano-video" || item.provider.type === "doubao-video"), ); if (!directModel) { throw new Error("没有可用的视频生成子模型"); } const directCapabilities = withConfiguredVideoCapabilities( directModel.model.capabilities as VideoModelCapabilities | undefined, (directModel.model as any).imageGenConfig ); const supportedModes = getSupportedModes(directCapabilities, directModel.model.modelId); if (!supportedModes.includes(mode)) { throw new Error(`模型 ${directModel.model.displayName} 不支持 ${mode} 模式`); } return { model: directModel, capabilities: { ...directCapabilities, supportedModes, } as VideoModelCapabilities, }; } console.log(`[Smart Model Selection] 找到 ${subModels.length} 个子模型`); // 筛选支持指定模式的子模型 const compatibleModels = subModels.filter((m) => { const caps = withConfiguredVideoCapabilities( m.model.capabilities as VideoModelCapabilities, (m.model as any).imageGenConfig ); const supportsMode = getSupportedModes(caps, m.model.modelId).includes(mode); // 检查分辨率支持(如果用户指定了分辨率) const supportedResolutions = caps.supportedResolutions || ["480p", "720p", "1080p"]; const supportsResolution = !params.resolution || supportedResolutions.includes(params.resolution); return supportsMode && supportsResolution; }); if (compatibleModels.length === 0) { throw new Error(`没有模型支持 ${mode} 模式和指定的参数`); } console.log(`[Smart Model Selection] ${compatibleModels.length} 个模型支持 ${mode} 模式`); // 🎯 智能优先级排序 const priority: Record = { "doubao-seedance-1-0-pro-250528": 10, // 最高优先级(全功能) "doubao-seedance-1-0-pro-fast-251015": 8, // 快速模式 "doubao-seedance-1-0-lite-i2v-250428": 6, // 图生视频轻量版 "doubao-seedance-1-0-lite-t2v-250428": 5, // 文生视频轻量版 }; // 选择最佳模型 const bestModel = compatibleModels.sort((a, b) => { const priorityA = priority[a.model.modelId] || 0; const priorityB = priority[b.model.modelId] || 0; return priorityB - priorityA; })[0]; console.log(`[Smart Model Selection] ✅ 选择模型: ${bestModel.model.displayName} (${bestModel.model.modelId})`); return { model: bestModel, capabilities: withConfiguredVideoCapabilities( bestModel.model.capabilities as VideoModelCapabilities, (bestModel.model as any).imageGenConfig ), }; } /** * 获取支持指定模式的视频模型(兼容旧逻辑) */ async function getCompatibleModel( mode: VideoGenerationMode, parentModelId?: string, params?: Partial ): Promise<{ model: any; capabilities: VideoModelCapabilities }> { // 🆕 如果提供了parentModelId,使用智能子模型选择 if (parentModelId) { return selectBestSubModel(parentModelId, mode, params || {}); } // 兼容旧逻辑:直接从所有模型中选择(用于向后兼容) const models = await getEnabledAiModelsWithProvider(); const volcanoVideoModels = models.filter( (m) => (m.provider.type === "volcano-video" || m.provider.type === "doubao-video") && m.model.isEnabled && m.model.capabilities ); if (volcanoVideoModels.length === 0) { throw new Error("没有启用的火山引擎视频模型"); } const compatibleModels = volcanoVideoModels.filter((m) => { const caps = withConfiguredVideoCapabilities( m.model.capabilities as VideoModelCapabilities, (m.model as any).imageGenConfig ); return getSupportedModes(caps, m.model.modelId).includes(mode); }); if (compatibleModels.length === 0) { throw new Error(`没有模型支持 ${mode} 模式`); } const priority: Record = { "doubao-seedance-1-0-pro-250528": 10, "doubao-seedance-1-0-pro-fast-251015": 8, "doubao-seedance-1-0-lite-i2v-250428": 6, "doubao-seedance-1-0-lite-t2v-250428": 5, }; const bestModel = compatibleModels.sort((a, b) => { const priorityA = priority[a.model.modelId] || 0; const priorityB = priority[b.model.modelId] || 0; return priorityB - priorityA; })[0]; return { model: bestModel, capabilities: withConfiguredVideoCapabilities( bestModel.model.capabilities as VideoModelCapabilities, (bestModel.model as any).imageGenConfig ), }; } /** * 验证参数是否符合模型能力 */ function validateParamsAgainstCapabilities( params: Partial, capabilities: VideoModelCapabilities ): void { // 检查分辨率 if ( params.resolution && !capabilities.supportedResolutions.includes(params.resolution) ) { throw new Error( `模型不支持 ${params.resolution} 分辨率,支持的分辨率:${capabilities.supportedResolutions.join(", ")}` ); } // 检查宽高比 if (params.ratio && !capabilities.supportedRatios.includes(params.ratio)) { throw new Error( `模型不支持 ${params.ratio} 宽高比,支持的宽高比:${capabilities.supportedRatios.join(", ")}` ); } // 检查时长范围 if (params.duration) { const allowedDurations = Array.isArray(capabilities.durationOptions) ? capabilities.durationOptions : []; if (allowedDurations.length > 0) { if (!allowedDurations.includes(params.duration)) { throw new Error(`视频时长仅支持 ${allowedDurations.join(" / ")} 秒`); } } else { const { min, max } = capabilities.durationRange; if (params.duration < min || params.duration > max) { throw new Error(`视频时长必须在 ${min}-${max} 秒之间`); } } } // 检查参考图数量 if ( params.mode === "reference-images" && params.referenceImages && capabilities.maxReferenceImages ) { if (params.referenceImages.length > capabilities.maxReferenceImages) { throw new Error( `参考图数量不能超过 ${capabilities.maxReferenceImages} 张` ); } } } /** * 构建视频生成选项 */ function buildVideoOptions( mode: VideoGenerationMode, images: string[] | undefined, prompt: string, params: any, modelId: string ): VideoGenerationOptions { const options: VideoGenerationOptions = { prompt, model: modelId, mode, resolution: params.resolution || "720p", ratio: params.ratio || "16:9", duration: params.duration || 5, watermark: params.watermark ?? true, cameraFixed: params.cameraFixed ?? false, fps: 24, }; // 根据模式处理图片输入 if (mode === "first-frame" && images && images.length > 0) { options.firstFrameImage = images[0]; } else if (mode === "first-last-frame" && images && images.length >= 2) { options.firstFrameImage = images[0]; options.lastFrameImage = images[1]; } else if (mode === "reference-images" && images && images.length > 0) { options.referenceImages = images.slice(0, 4); } return options; } async function resolvePreferredVideoParentModelId(preferredModelRef?: string) { const normalizedRef = String(preferredModelRef || "").trim(); if (!normalizedRef) return undefined; const models = await getEnabledAiModelsWithProvider(); const matched = models.find( (item) => (item.provider.type === "volcano-video" || item.provider.type === "doubao-video") && (item.model.id === normalizedRef || item.model.modelId === normalizedRef), ); if (!matched) return undefined; if (matched.model.isSubModel && matched.model.parentModelId) { return matched.model.parentModelId; } return matched.model.id; } export async function executeDirectVideoGeneration( params: ExecuteDirectVideoGenerationParams, ): Promise { const prompt = String(params.prompt || "").trim(); if (!prompt) { throw new Error("缺少视频生成提示词"); } const preferredParentModelId = await resolvePreferredVideoParentModelId( params.preferredModelRef, ); const allModels = await getEnabledAiModelsWithProvider(); const fallbackParentModel = allModels.find( (item) => (item.provider.type === "volcano-video" || item.provider.type === "doubao-video") && !item.model.isSubModel && item.model.modelId === "volcano-video-master", ) || allModels.find( (item) => (item.provider.type === "volcano-video" || item.provider.type === "doubao-video") && !item.model.isSubModel, ); const parentModelId = preferredParentModelId || fallbackParentModel?.model.id || allModels.find( (item) => (item.provider.type === "volcano-video" || item.provider.type === "doubao-video") && item.model.isEnabled && item.model.isSubModel && item.model.parentModelId, )?.model.parentModelId || undefined; const generationParams = { mode: params.mode, resolution: params.resolution || "720p", ratio: params.ratio || "16:9", duration: params.duration || 5, watermark: params.watermark ?? true, cameraFixed: params.cameraFixed ?? false, seed: params.seed, returnLastFrame: params.returnLastFrame ?? false, referenceImages: params.images, } satisfies Partial; let selectedModelConfig: { model: any; capabilities: VideoModelCapabilities }; try { selectedModelConfig = await getCompatibleModel( params.mode, parentModelId, generationParams, ); } catch (preferredModelError) { if (parentModelId) { console.warn( "[Generate Video Tool] Preferred parent model selection failed, fallback to global compatible model:", preferredModelError, ); selectedModelConfig = await getCompatibleModel( params.mode, undefined, generationParams, ); } else { throw preferredModelError; } } const { model, capabilities } = selectedModelConfig; validateParamsAgainstCapabilities(generationParams, capabilities); const options = buildVideoOptions( params.mode, params.images, prompt, generationParams, model.model.modelId, ); let videoMetadata: Record | null = null; for await (const chunk of doGenerateStream(model.provider.apiKey, model.provider.baseUrl, options)) { await params.onProgress?.(chunk); const metadataMatch = chunk.match(//); if (metadataMatch) { try { videoMetadata = JSON.parse(metadataMatch[1]); } catch (error) { console.error("[Generate Video Tool] 解析元数据失败:", error); } } } return { success: true, videoUrl: videoMetadata?.temporaryVideoUrl || undefined, metadata: videoMetadata, modelDisplayName: String(model.model.displayName || model.model.modelId || "视频模型"), modelId: String(model.model.modelId || model.model.id), modelRef: String(model.model.id || model.model.modelId), }; } /** * 视频生成工具 */ export function generateVideo({ dataStream }: { dataStream: any }) { return tool({ description: "生成视频,支持文生视频、图生视频(首帧/首尾帧/参考图)等多种模式", inputSchema: z.object({ prompt: z.string().describe("视频生成提示词,描述视频内容、镜头运动等"), mode: z .enum(["text-to-video", "first-frame", "first-last-frame", "reference-images"]) .describe("生成模式:文生视频/图生视频-首帧/首尾帧/参考图"), images: z .array(z.string()) .optional() .describe("图片 URL 数组(根据模式不同,需要 0-4 张)"), resolution: z .enum(["480p", "720p", "1080p"]) .default("720p") .describe("视频分辨率"), ratio: z .enum(["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"]) .default("16:9") .describe("视频宽高比"), duration: z .number() .min(1) .max(600) .default(5) .describe("视频时长(秒)。优先使用当前视频模型配置的秒数档位,服务端会按模型能力动态校验。"), watermark: z.boolean().default(true).describe("是否添加水印"), cameraFixed: z.boolean().default(false).describe("是否固定摄像头"), seed: z.number().optional().describe("随机种子(可选)"), returnLastFrame: z .boolean() .default(false) .describe("是否返回尾帧(用于连续生成)"), }), execute: async ({ prompt, mode, images, resolution, ratio, duration, watermark, cameraFixed, seed, returnLastFrame }) => { try { console.log("[Generate Video Tool] 🎬 开始生成视频"); console.log("[Generate Video Tool] 模式:", mode); console.log("[Generate Video Tool] 提示词:", prompt); dataStream.writeData({ type: "video-generation-start", data: { mode, model: "智能选择中", }, }); const result = await executeDirectVideoGeneration({ prompt, mode, images, resolution, ratio, duration, watermark, cameraFixed, seed, returnLastFrame, onProgress: async (chunk) => { dataStream.writeMessageAnnotation({ type: "video-progress", content: chunk, }); }, }); console.log("[Generate Video Tool] ✅ 视频生成完成"); return { success: true, videoUrl: result.videoUrl, metadata: result.metadata, modelDisplayName: result.modelDisplayName, modelId: result.modelId, modelRef: result.modelRef, }; } catch (error: any) { console.error("[Generate Video Tool] ❌ 生成失败:", error); dataStream.writeMessageAnnotation({ type: "video-error", content: `视频生成失败:${error.message}`, }); throw error; } }, }); }