/** * 视频生成工具 * 支持火山引擎视频生成的多种模式 */ import { tool } from "ai"; import { z } from "zod"; import { doGenerateStream } from "@/lib/ai/providers/volcano-video"; import { getEnabledAiModelsWithProvider, getSubModelsByParentModelId } from "@/lib/db/queries"; import { getConfiguredVideoDurationOptions } from "@/lib/ai/video-model-config"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { getKreaModelPresetById } from "@/lib/ai/krea-models"; import { isVideoModelDescriptor } from "@/lib/ai/model-classification"; import type { VideoGenerationMode, VideoModelCapabilities, VideoGenerationOptions, } from "@/types/ai"; export interface ExecuteDirectVideoGenerationParams { prompt: string; mode: VideoGenerationMode; images?: string[]; audioUrls?: string[]; resolution?: "480p" | "720p" | "1080p" | "4k"; ratio?: "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "21:9" | "adaptive"; duration?: number; generateAudio?: boolean; watermark?: boolean; cameraFixed?: boolean; seed?: number; returnLastFrame?: boolean; preferredModelRef?: string; onProgress?: (chunk: string) => void | Promise; } export interface ExecuteDirectVideoGenerationResult { success: boolean; videoUrl?: string; metadata?: Record | null; modelDisplayName: string; modelId: string; modelRef: string; } function inferSupportedModesFromModelId(modelId: string): VideoGenerationMode[] { const normalized = modelId.toLowerCase(); if (normalized.includes("lite-t2v")) return ["text-to-video"]; if (normalized.includes("lite-i2v")) return ["first-frame", "first-last-frame", "reference-images"]; if (normalized.includes("pro-fast")) return ["text-to-video", "first-frame"]; if (normalized.includes("-pro-") || normalized.endsWith("-pro")) { return ["text-to-video", "first-frame", "first-last-frame"]; } return ["text-to-video", "first-frame", "first-last-frame", "reference-images"]; } function getModelSupportedModes(model: any): VideoGenerationMode[] { const configured = model.model.capabilities as Partial | undefined; const configuredModes = configured?.supportedModes; const modes = new Set( Array.isArray(configuredModes) ? (configuredModes as VideoGenerationMode[]) : [], ); if (model.provider.type === "krea") { const preset = getKreaModelPresetById(String(model.model.modelId || "")); const presetModes = preset?.capabilities?.supportedModes; if (Array.isArray(presetModes)) { for (const mode of presetModes as VideoGenerationMode[]) modes.add(mode); } } if (modes.size > 0) return Array.from(modes); return inferSupportedModesFromModelId(String(model.model.modelId || "")); } function extractVideoUrlFromGeneratedText(value: unknown) { const text = String(value || ""); const srcMatch = text.match(/]+src=["'](https?:\/\/[^"']+)["']/i); if (srcMatch?.[1]) return srcMatch[1]; const markdownMatch = text.match(/\[[^\]]*\]\((https?:\/\/[^)\s]+)\)/i); if (markdownMatch?.[1]) return markdownMatch[1]; return text.match(/https?:\/\/[^\s"'<>]+/i)?.[0]; } async function executeKreaAudioVideoGeneration(params: { prompt: string; audioUrls: string[]; resolution?: "480p" | "720p" | "1080p" | "4k"; ratio?: "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "21:9" | "adaptive"; duration?: number; generateAudio?: boolean; seed?: number; preferredModelRef?: string; }): Promise { if (params.audioUrls.length === 0) { throw new Error("音频生视频至少需要 1 个可访问的音频素材"); } if (params.resolution === "4k") { throw new Error("当前音频参考视频模型尚未验证 4K 输出,请选择 1080P 或更低清晰度"); } const allModels = await getEnabledAiModelsWithProvider(true); const candidates = allModels.filter( (item) => item.provider.type === "krea" && item.model.isEnabled && getModelSupportedModes(item).includes("audio-to-video"), ); if (candidates.length === 0) { throw new Error("没有启用支持音频参考的 Seedance 2.0 或 Wan 2.5 视频模型"); } const preferredRef = String(params.preferredModelRef || "").trim(); const preferred = preferredRef ? candidates.find( (item) => item.model.id === preferredRef || item.model.modelId === preferredRef || item.model.displayName === preferredRef, ) : undefined; if (preferredRef && !preferred) { throw new Error("所选模型不支持音频参考,请选择 Seedance 2.0 或 Wan 2.5"); } const selected = preferred || [...candidates].sort((left, right) => { const score = (modelId: string) => { if (modelId.includes("/seedance-2") && !modelId.includes("fast")) return 3; if (modelId.includes("/seedance-2-fast")) return 2; if (modelId.includes("/wan-2.5")) return 1; return 0; }; return score(right.model.modelId) - score(left.model.modelId); })[0]!; const languageModel = await getLanguageModelById(selected.model.id); if (!languageModel || typeof (languageModel as any).doGenerate !== "function") { throw new Error("音频参考视频模型初始化失败"); } const result = await (languageModel as any).doGenerate({ messages: [{ role: "user", content: [{ type: "text", text: params.prompt }] }], overrideModelId: selected.model.modelId, videoParams: { audioUrls: params.audioUrls, duration: params.duration || 5, generateAudio: params.generateAudio ?? true, mode: "audio-to-video", ratio: params.ratio || "16:9", resolution: params.resolution || "720p", seed: params.seed, }, }); if (result?.finishReason === "error") { throw new Error(String(result?.text || "音频生视频生成失败")); } const videoUrl = extractVideoUrlFromGeneratedText(result?.text); if (!videoUrl) { throw new Error("音频生视频模型未返回可用视频地址"); } return { success: true, videoUrl, metadata: { audioReferenceCount: params.audioUrls.length, provider: "krea", }, modelDisplayName: String(selected.model.displayName || selected.model.modelId), modelId: String(selected.model.modelId || selected.model.id), modelRef: String(selected.model.id || selected.model.modelId), }; } function getSupportedModes(caps: Partial | undefined, modelId: string): VideoGenerationMode[] { const modes = (caps?.supportedModes || []) as VideoGenerationMode[]; if (Array.isArray(modes) && modes.length > 0) { return modes; } return inferSupportedModesFromModelId(modelId); } function withConfiguredVideoCapabilities( capabilities: Partial | undefined, imageGenConfig?: { videoDurationOptions?: number[] | null } | null ): VideoModelCapabilities { const baseCapabilities = { supportedModes: ["text-to-video", "first-frame", "first-last-frame", "reference-images"], supportedResolutions: ["480p", "720p", "1080p"], supportedRatios: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"], durationRange: { min: 2, max: 12 }, maxReferenceImages: 4, ...(capabilities || {}), } as VideoModelCapabilities; const durationOptions = getConfiguredVideoDurationOptions(imageGenConfig); if (durationOptions.length === 0) { return baseCapabilities; } return { ...baseCapabilities, durationOptions, durationRange: { min: durationOptions[0], max: durationOptions[durationOptions.length - 1], }, }; } /** * 🆕 智能选择最佳子模型 * 根据用户选择的模式和参数,从子模型列表中选择最合适的模型 */ async function selectBestSubModel( parentModelId: string, mode: VideoGenerationMode, params: Partial ): Promise<{ model: any; capabilities: VideoModelCapabilities }> { // 获取所有子模型 const subModels = await getSubModelsByParentModelId(parentModelId); if (subModels.length === 0) { const allModels = await getEnabledAiModelsWithProvider(); const directModel = allModels.find( (item) => item.model.id === parentModelId && (item.provider.type === "volcano-video" || item.provider.type === "doubao-video"), ); if (!directModel) { throw new Error("没有可用的视频生成子模型"); } const directCapabilities = withConfiguredVideoCapabilities( directModel.model.capabilities as VideoModelCapabilities | undefined, (directModel.model as any).imageGenConfig ); const supportedModes = getSupportedModes(directCapabilities, directModel.model.modelId); if (!supportedModes.includes(mode)) { throw new Error(`模型 ${directModel.model.displayName} 不支持 ${mode} 模式`); } return { model: directModel, capabilities: { ...directCapabilities, supportedModes, } as VideoModelCapabilities, }; } console.log(`[Smart Model Selection] 找到 ${subModels.length} 个子模型`); // 筛选支持指定模式的子模型 const compatibleModels = subModels.filter((m) => { const caps = withConfiguredVideoCapabilities( m.model.capabilities as VideoModelCapabilities, (m.model as any).imageGenConfig ); const supportsMode = getSupportedModes(caps, m.model.modelId).includes(mode); // 检查分辨率支持(如果用户指定了分辨率) const supportedResolutions = caps.supportedResolutions || ["480p", "720p", "1080p"]; const supportsResolution = !params.resolution || supportedResolutions.includes(params.resolution); return supportsMode && supportsResolution; }); if (compatibleModels.length === 0) { throw new Error(`没有模型支持 ${mode} 模式和指定的参数`); } console.log(`[Smart Model Selection] ${compatibleModels.length} 个模型支持 ${mode} 模式`); // 🎯 智能优先级排序 const priority: Record = { "doubao-seedance-1-0-pro-250528": 10, // 最高优先级(全功能) "doubao-seedance-1-0-pro-fast-251015": 8, // 快速模式 "doubao-seedance-1-0-lite-i2v-250428": 6, // 图生视频轻量版 "doubao-seedance-1-0-lite-t2v-250428": 5, // 文生视频轻量版 }; // 选择最佳模型 const bestModel = compatibleModels.sort((a, b) => { const priorityA = priority[a.model.modelId] || 0; const priorityB = priority[b.model.modelId] || 0; return priorityB - priorityA; })[0]; console.log(`[Smart Model Selection] ✅ 选择模型: ${bestModel.model.displayName} (${bestModel.model.modelId})`); return { model: bestModel, capabilities: withConfiguredVideoCapabilities( bestModel.model.capabilities as VideoModelCapabilities, (bestModel.model as any).imageGenConfig ), }; } /** * 获取支持指定模式的视频模型(兼容旧逻辑) */ async function getCompatibleModel( mode: VideoGenerationMode, parentModelId?: string, params?: Partial ): Promise<{ model: any; capabilities: VideoModelCapabilities }> { // 🆕 如果提供了parentModelId,使用智能子模型选择 if (parentModelId) { return selectBestSubModel(parentModelId, mode, params || {}); } // 兼容旧逻辑:直接从所有模型中选择(用于向后兼容) const models = await getEnabledAiModelsWithProvider(); const volcanoVideoModels = models.filter( (m) => (m.provider.type === "volcano-video" || m.provider.type === "doubao-video") && m.model.isEnabled && m.model.capabilities ); if (volcanoVideoModels.length === 0) { throw new Error("没有启用的火山引擎视频模型"); } const compatibleModels = volcanoVideoModels.filter((m) => { const caps = withConfiguredVideoCapabilities( m.model.capabilities as VideoModelCapabilities, (m.model as any).imageGenConfig ); return getSupportedModes(caps, m.model.modelId).includes(mode); }); if (compatibleModels.length === 0) { throw new Error(`没有模型支持 ${mode} 模式`); } const priority: Record = { "doubao-seedance-1-0-pro-250528": 10, "doubao-seedance-1-0-pro-fast-251015": 8, "doubao-seedance-1-0-lite-i2v-250428": 6, "doubao-seedance-1-0-lite-t2v-250428": 5, }; const bestModel = compatibleModels.sort((a, b) => { const priorityA = priority[a.model.modelId] || 0; const priorityB = priority[b.model.modelId] || 0; return priorityB - priorityA; })[0]; return { model: bestModel, capabilities: withConfiguredVideoCapabilities( bestModel.model.capabilities as VideoModelCapabilities, (bestModel.model as any).imageGenConfig ), }; } /** * 验证参数是否符合模型能力 */ function validateParamsAgainstCapabilities( params: Partial, capabilities: VideoModelCapabilities ): void { // 检查分辨率 if ( params.resolution && !capabilities.supportedResolutions.includes(params.resolution) ) { throw new Error( `模型不支持 ${params.resolution} 分辨率,支持的分辨率:${capabilities.supportedResolutions.join(", ")}` ); } // 检查宽高比 if (params.ratio && !capabilities.supportedRatios.includes(params.ratio)) { throw new Error( `模型不支持 ${params.ratio} 宽高比,支持的宽高比:${capabilities.supportedRatios.join(", ")}` ); } // 检查时长范围 if (params.duration) { const allowedDurations = Array.isArray(capabilities.durationOptions) ? capabilities.durationOptions : []; if (allowedDurations.length > 0) { if (!allowedDurations.includes(params.duration)) { throw new Error(`视频时长仅支持 ${allowedDurations.join(" / ")} 秒`); } } else { const { min, max } = capabilities.durationRange; if (params.duration < min || params.duration > max) { throw new Error(`视频时长必须在 ${min}-${max} 秒之间`); } } } // 检查参考图数量 if ( params.mode === "reference-images" && params.referenceImages && capabilities.maxReferenceImages ) { if (params.referenceImages.length > capabilities.maxReferenceImages) { throw new Error( `参考图数量不能超过 ${capabilities.maxReferenceImages} 张` ); } } } /** * 构建视频生成选项 */ function buildVideoOptions( mode: VideoGenerationMode, images: string[] | undefined, prompt: string, params: any, modelId: string, maxReferenceImages = 4, ): VideoGenerationOptions { const options: VideoGenerationOptions = { prompt, model: modelId, mode, resolution: params.resolution || "720p", ratio: params.ratio || "16:9", duration: params.duration || 5, generateAudio: params.generateAudio ?? true, watermark: params.watermark ?? true, cameraFixed: params.cameraFixed ?? false, fps: 24, }; // 根据模式处理图片输入 if (mode === "first-frame" && images && images.length > 0) { options.firstFrameImage = images[0]; } else if (mode === "first-last-frame" && images && images.length >= 2) { options.firstFrameImage = images[0]; options.lastFrameImage = images[1]; } else if (mode === "reference-images" && images && images.length > 0) { options.referenceImages = images.slice( 0, Math.max(1, Math.round(maxReferenceImages)), ); } return options; } async function resolvePreferredVideoParentModelId(preferredModelRef?: string) { const normalizedRef = String(preferredModelRef || "").trim(); if (!normalizedRef) return undefined; const models = await getEnabledAiModelsWithProvider(); const matched = models.find( (item) => (item.provider.type === "volcano-video" || item.provider.type === "doubao-video") && (item.model.id === normalizedRef || item.model.modelId === normalizedRef), ); if (!matched) return undefined; if (matched.model.isSubModel && matched.model.parentModelId) { return matched.model.parentModelId; } return matched.model.id; } async function resolvePreferredGenericVideoModel( preferredModelRef: string | undefined, mode: VideoGenerationMode, params: Partial, ) { const normalizedRef = String(preferredModelRef || "").trim(); if (!normalizedRef) return null; const models = await getEnabledAiModelsWithProvider(); const matched = models.find( (item) => item.provider.type === "vip98k" && item.model.isEnabled && !item.model.isSubModel && (item.model.id === normalizedRef || item.model.modelId === normalizedRef || item.model.displayName === normalizedRef) && isVideoModelDescriptor({ providerType: item.provider.type, modelId: item.model.modelId, capabilities: item.model.capabilities, supportsImageGen: item.model.supportsImageGen, }), ); if (!matched) return null; const capabilities = withConfiguredVideoCapabilities( matched.model.capabilities as VideoModelCapabilities | undefined, (matched.model as any).imageGenConfig, ); const supportedModes = getSupportedModes(capabilities, matched.model.modelId); if (!supportedModes.includes(mode)) { throw new Error(`模型 ${matched.model.displayName} 不支持 ${mode} 模式`); } const resolvedCapabilities = { ...capabilities, supportedModes, } as VideoModelCapabilities; validateParamsAgainstCapabilities(params, resolvedCapabilities); return { model: matched, capabilities: resolvedCapabilities, }; } async function executeGenericProviderVideoGeneration(params: { request: ExecuteDirectVideoGenerationParams; prompt: string; selected: { model: any; capabilities: VideoModelCapabilities }; generationParams: Partial; }): Promise { const selected = params.selected.model; const languageModel = await getLanguageModelById(selected.model.id); if (!languageModel || typeof (languageModel as any).doGenerate !== "function") { throw new Error("所选视频模型初始化失败"); } await params.request.onProgress?.( `正在调用 ${selected.model.displayName || selected.model.modelId} 生成视频...`, ); const imageParts = (params.request.images || []).map((url) => ({ type: "image_url" as const, image_url: { url }, })); const result = await (languageModel as any).doGenerate({ messages: [ { role: "user", content: [{ type: "text", text: params.prompt }, ...imageParts], }, ], overrideModelId: selected.model.modelId, videoParams: { mode: params.request.mode, resolution: params.generationParams.resolution || "720p", ratio: params.generationParams.ratio || "16:9", duration: params.generationParams.duration || 8, generateAudio: params.generationParams.generateAudio ?? true, watermark: params.generationParams.watermark, cameraFixed: params.generationParams.cameraFixed, seed: params.generationParams.seed, returnLastFrame: params.generationParams.returnLastFrame, }, }); if (result?.finishReason === "error") { throw new Error(String(result?.text || "视频生成失败")); } const videoUrl = extractVideoUrlFromGeneratedText(result?.text); if (!videoUrl) { throw new Error("所选视频模型未返回可用视频地址"); } return { success: true, videoUrl, metadata: { provider: selected.provider.type, temporaryVideoUrl: videoUrl, mode: params.request.mode, resolution: params.generationParams.resolution, ratio: params.generationParams.ratio, duration: params.generationParams.duration, }, modelDisplayName: String(selected.model.displayName || selected.model.modelId), modelId: String(selected.model.modelId || selected.model.id), modelRef: String(selected.model.id || selected.model.modelId), }; } export async function executeDirectVideoGeneration( params: ExecuteDirectVideoGenerationParams, ): Promise { const prompt = String(params.prompt || "").trim(); if (!prompt) { throw new Error("缺少视频生成提示词"); } if (params.mode === "audio-to-video") { return executeKreaAudioVideoGeneration({ audioUrls: params.audioUrls || [], duration: params.duration, generateAudio: params.generateAudio, preferredModelRef: params.preferredModelRef, prompt, ratio: params.ratio, resolution: params.resolution, seed: params.seed, }); } const generationParams = { mode: params.mode, resolution: params.resolution || "720p", ratio: params.ratio || "16:9", duration: params.duration || 5, generateAudio: params.generateAudio ?? true, watermark: params.watermark ?? true, cameraFixed: params.cameraFixed ?? false, seed: params.seed, returnLastFrame: params.returnLastFrame ?? false, referenceImages: params.images, } satisfies Partial; const preferredGenericModel = await resolvePreferredGenericVideoModel( params.preferredModelRef, params.mode, generationParams, ); if (preferredGenericModel) { return executeGenericProviderVideoGeneration({ request: params, prompt, selected: preferredGenericModel, generationParams, }); } const preferredParentModelId = await resolvePreferredVideoParentModelId( params.preferredModelRef, ); const allModels = await getEnabledAiModelsWithProvider(); const fallbackParentModel = allModels.find( (item) => (item.provider.type === "volcano-video" || item.provider.type === "doubao-video") && !item.model.isSubModel && item.model.modelId === "volcano-video-master", ) || allModels.find( (item) => (item.provider.type === "volcano-video" || item.provider.type === "doubao-video") && !item.model.isSubModel, ); const parentModelId = preferredParentModelId || fallbackParentModel?.model.id || allModels.find( (item) => (item.provider.type === "volcano-video" || item.provider.type === "doubao-video") && item.model.isEnabled && item.model.isSubModel && item.model.parentModelId, )?.model.parentModelId || undefined; let selectedModelConfig: { model: any; capabilities: VideoModelCapabilities }; try { selectedModelConfig = await getCompatibleModel( params.mode, parentModelId, generationParams, ); } catch (preferredModelError) { if (parentModelId) { console.warn( "[Generate Video Tool] Preferred parent model selection failed, fallback to global compatible model:", preferredModelError, ); selectedModelConfig = await getCompatibleModel( params.mode, undefined, generationParams, ); } else { throw preferredModelError; } } const { model, capabilities } = selectedModelConfig; validateParamsAgainstCapabilities(generationParams, capabilities); const options = buildVideoOptions( params.mode, params.images, prompt, generationParams, model.model.modelId, capabilities.maxReferenceImages, ); let videoMetadata: Record | null = null; for await (const chunk of doGenerateStream(model.provider.apiKey, model.provider.baseUrl, options)) { await params.onProgress?.(chunk); const metadataMatch = chunk.match(//); if (metadataMatch) { try { videoMetadata = JSON.parse(metadataMatch[1]); } catch (error) { console.error("[Generate Video Tool] 解析元数据失败:", error); } } } return { success: true, videoUrl: videoMetadata?.temporaryVideoUrl || undefined, metadata: videoMetadata, modelDisplayName: String(model.model.displayName || model.model.modelId || "视频模型"), modelId: String(model.model.modelId || model.model.id), modelRef: String(model.model.id || model.model.modelId), }; } /** * 视频生成工具 */ export function generateVideo({ dataStream }: { dataStream: any }) { return tool({ description: "生成视频,支持文生视频、图生视频(首帧/首尾帧/参考图)等多种模式", inputSchema: z.object({ prompt: z.string().describe("视频生成提示词,描述视频内容、镜头运动等"), mode: z .enum(["text-to-video", "first-frame", "first-last-frame", "reference-images", "audio-to-video"]) .describe("生成模式:文生视频/图生视频-首帧/首尾帧/参考图/音频参考"), images: z .array(z.string()) .optional() .describe("图片 URL 数组(数量按当前视频模型能力动态校验)"), audioUrls: z .array(z.string()) .optional() .describe("音频参考 URL 数组(音频生视频模式至少需要 1 个)"), resolution: z .enum(["480p", "720p", "1080p"]) .default("720p") .describe("视频分辨率"), ratio: z .enum(["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"]) .default("16:9") .describe("视频宽高比"), duration: z .number() .min(1) .max(600) .default(5) .describe("视频时长(秒)。优先使用当前视频模型配置的秒数档位,服务端会按模型能力动态校验。"), watermark: z.boolean().default(true).describe("是否添加水印"), cameraFixed: z.boolean().default(false).describe("是否固定摄像头"), seed: z.number().optional().describe("随机种子(可选)"), returnLastFrame: z .boolean() .default(false) .describe("是否返回尾帧(用于连续生成)"), }), execute: async ({ prompt, mode, images, audioUrls, resolution, ratio, duration, watermark, cameraFixed, seed, returnLastFrame }) => { try { console.log("[Generate Video Tool] 🎬 开始生成视频"); console.log("[Generate Video Tool] 模式:", mode); console.log("[Generate Video Tool] 提示词:", prompt); dataStream.writeData({ type: "video-generation-start", data: { mode, model: "智能选择中", }, }); const result = await executeDirectVideoGeneration({ prompt, mode, images, audioUrls, resolution, ratio, duration, watermark, cameraFixed, seed, returnLastFrame, onProgress: async (chunk) => { dataStream.writeMessageAnnotation({ type: "video-progress", content: chunk, }); }, }); console.log("[Generate Video Tool] ✅ 视频生成完成"); return { success: true, videoUrl: result.videoUrl, metadata: result.metadata, modelDisplayName: result.modelDisplayName, modelId: result.modelId, modelRef: result.modelRef, }; } catch (error: any) { console.error("[Generate Video Tool] ❌ 生成失败:", error); dataStream.writeMessageAnnotation({ type: "video-error", content: `视频生成失败:${error.message}`, }); throw error; } }, }); }