381 lines
12 KiB
TypeScript
381 lines
12 KiB
TypeScript
/**
|
||
* 视频生成工具
|
||
* 支持火山引擎视频生成的多种模式
|
||
*/
|
||
|
||
import { tool } from "ai";
|
||
import { z } from "zod";
|
||
import { doGenerateStream } from "@/lib/ai/providers/volcano-video";
|
||
import {
|
||
getEnabledAiModelsWithProvider,
|
||
getSubModelsByParentModelId
|
||
} from "@/lib/db/queries";
|
||
import type {
|
||
VideoGenerationMode,
|
||
VideoModelCapabilities,
|
||
VideoGenerationOptions,
|
||
} from "@/types/ai";
|
||
|
||
function inferSupportedModesFromModelId(modelId: string): VideoGenerationMode[] {
|
||
const normalized = modelId.toLowerCase();
|
||
|
||
if (normalized.includes("lite-t2v")) return ["text-to-video"];
|
||
if (normalized.includes("lite-i2v")) return ["first-frame", "first-last-frame", "reference-images"];
|
||
if (normalized.includes("pro-fast")) return ["text-to-video", "first-frame"];
|
||
if (normalized.includes("-pro-") || normalized.endsWith("-pro")) {
|
||
return ["text-to-video", "first-frame", "first-last-frame"];
|
||
}
|
||
|
||
return ["text-to-video", "first-frame", "first-last-frame", "reference-images"];
|
||
}
|
||
|
||
function getSupportedModes(caps: Partial<VideoModelCapabilities> | undefined, modelId: string): VideoGenerationMode[] {
|
||
const modes = (caps?.supportedModes || []) as VideoGenerationMode[];
|
||
if (Array.isArray(modes) && modes.length > 0) {
|
||
return modes;
|
||
}
|
||
return inferSupportedModesFromModelId(modelId);
|
||
}
|
||
|
||
/**
|
||
* 🆕 智能选择最佳子模型
|
||
* 根据用户选择的模式和参数，从子模型列表中选择最合适的模型
|
||
*/
|
||
async function selectBestSubModel(
|
||
parentModelId: string,
|
||
mode: VideoGenerationMode,
|
||
params: Partial<VideoGenerationOptions>
|
||
): Promise<{ model: any; capabilities: VideoModelCapabilities }> {
|
||
// 获取所有子模型
|
||
const subModels = await getSubModelsByParentModelId(parentModelId);
|
||
|
||
if (subModels.length === 0) {
|
||
throw new Error("没有可用的视频生成子模型");
|
||
}
|
||
|
||
console.log(`[Smart Model Selection] 找到 ${subModels.length} 个子模型`);
|
||
|
||
// 筛选支持指定模式的子模型
|
||
const compatibleModels = subModels.filter((m) => {
|
||
const caps = m.model.capabilities as VideoModelCapabilities;
|
||
const supportsMode = getSupportedModes(caps, m.model.modelId).includes(mode);
|
||
|
||
// 检查分辨率支持（如果用户指定了分辨率）
|
||
const supportedResolutions = caps.supportedResolutions || ["480p", "720p", "1080p"];
|
||
const supportsResolution = !params.resolution || supportedResolutions.includes(params.resolution);
|
||
|
||
return supportsMode && supportsResolution;
|
||
});
|
||
|
||
if (compatibleModels.length === 0) {
|
||
throw new Error(`没有模型支持 ${mode} 模式和指定的参数`);
|
||
}
|
||
|
||
console.log(`[Smart Model Selection] ${compatibleModels.length} 个模型支持 ${mode} 模式`);
|
||
|
||
// 🎯 智能优先级排序
|
||
const priority: Record<string, number> = {
|
||
"doubao-seedance-1-0-pro-250528": 10, // 最高优先级（全功能）
|
||
"doubao-seedance-1-0-pro-fast-251015": 8, // 快速模式
|
||
"doubao-seedance-1-0-lite-i2v-250428": 6, // 图生视频轻量版
|
||
"doubao-seedance-1-0-lite-t2v-250428": 5, // 文生视频轻量版
|
||
};
|
||
|
||
// 选择最佳模型
|
||
const bestModel = compatibleModels.sort((a, b) => {
|
||
const priorityA = priority[a.model.modelId] || 0;
|
||
const priorityB = priority[b.model.modelId] || 0;
|
||
return priorityB - priorityA;
|
||
})[0];
|
||
|
||
console.log(`[Smart Model Selection] ✅ 选择模型: ${bestModel.model.displayName} (${bestModel.model.modelId})`);
|
||
|
||
return {
|
||
model: bestModel,
|
||
capabilities: bestModel.model.capabilities as VideoModelCapabilities,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 获取支持指定模式的视频模型（兼容旧逻辑）
|
||
*/
|
||
async function getCompatibleModel(
|
||
mode: VideoGenerationMode,
|
||
parentModelId?: string,
|
||
params?: Partial<VideoGenerationOptions>
|
||
): Promise<{ model: any; capabilities: VideoModelCapabilities }> {
|
||
// 🆕 如果提供了parentModelId，使用智能子模型选择
|
||
if (parentModelId) {
|
||
return selectBestSubModel(parentModelId, mode, params || {});
|
||
}
|
||
|
||
// 兼容旧逻辑：直接从所有模型中选择（用于向后兼容）
|
||
const models = await getEnabledAiModelsWithProvider();
|
||
|
||
const volcanoVideoModels = models.filter(
|
||
(m) =>
|
||
m.provider.type === "volcano-video" &&
|
||
m.model.isEnabled &&
|
||
m.model.capabilities
|
||
);
|
||
|
||
if (volcanoVideoModels.length === 0) {
|
||
throw new Error("没有启用的火山引擎视频模型");
|
||
}
|
||
|
||
const compatibleModels = volcanoVideoModels.filter((m) => {
|
||
const caps = m.model.capabilities as VideoModelCapabilities;
|
||
return getSupportedModes(caps, m.model.modelId).includes(mode);
|
||
});
|
||
|
||
if (compatibleModels.length === 0) {
|
||
throw new Error(`没有模型支持 ${mode} 模式`);
|
||
}
|
||
|
||
const priority: Record<string, number> = {
|
||
"doubao-seedance-1-0-pro-250528": 10,
|
||
"doubao-seedance-1-0-pro-fast-251015": 8,
|
||
"doubao-seedance-1-0-lite-i2v-250428": 6,
|
||
"doubao-seedance-1-0-lite-t2v-250428": 5,
|
||
};
|
||
|
||
const bestModel = compatibleModels.sort((a, b) => {
|
||
const priorityA = priority[a.model.modelId] || 0;
|
||
const priorityB = priority[b.model.modelId] || 0;
|
||
return priorityB - priorityA;
|
||
})[0];
|
||
|
||
return {
|
||
model: bestModel,
|
||
capabilities: bestModel.model.capabilities as VideoModelCapabilities,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 验证参数是否符合模型能力
|
||
*/
|
||
function validateParamsAgainstCapabilities(
|
||
params: Partial<VideoGenerationOptions>,
|
||
capabilities: VideoModelCapabilities
|
||
): void {
|
||
// 检查分辨率
|
||
if (
|
||
params.resolution &&
|
||
!capabilities.supportedResolutions.includes(params.resolution)
|
||
) {
|
||
throw new Error(
|
||
`模型不支持 ${params.resolution} 分辨率，支持的分辨率：${capabilities.supportedResolutions.join(", ")}`
|
||
);
|
||
}
|
||
|
||
// 检查宽高比
|
||
if (params.ratio && !capabilities.supportedRatios.includes(params.ratio)) {
|
||
throw new Error(
|
||
`模型不支持 ${params.ratio} 宽高比，支持的宽高比：${capabilities.supportedRatios.join(", ")}`
|
||
);
|
||
}
|
||
|
||
// 检查时长范围
|
||
if (params.duration) {
|
||
const { min, max } = capabilities.durationRange;
|
||
if (params.duration < min || params.duration > max) {
|
||
throw new Error(`视频时长必须在 ${min}-${max} 秒之间`);
|
||
}
|
||
}
|
||
|
||
// 检查参考图数量
|
||
if (
|
||
params.mode === "reference-images" &&
|
||
params.referenceImages &&
|
||
capabilities.maxReferenceImages
|
||
) {
|
||
if (params.referenceImages.length > capabilities.maxReferenceImages) {
|
||
throw new Error(
|
||
`参考图数量不能超过 ${capabilities.maxReferenceImages} 张`
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 构建视频生成选项
|
||
*/
|
||
function buildVideoOptions(
|
||
mode: VideoGenerationMode,
|
||
images: string[] | undefined,
|
||
prompt: string,
|
||
params: any,
|
||
modelId: string
|
||
): VideoGenerationOptions {
|
||
const options: VideoGenerationOptions = {
|
||
prompt,
|
||
model: modelId,
|
||
mode,
|
||
resolution: params.resolution || "720p",
|
||
ratio: params.ratio || "16:9",
|
||
duration: params.duration || 5,
|
||
watermark: params.watermark ?? true,
|
||
cameraFixed: params.cameraFixed ?? false,
|
||
fps: 24,
|
||
};
|
||
|
||
// 根据模式处理图片输入
|
||
if (mode === "first-frame" && images && images.length > 0) {
|
||
options.firstFrameImage = images[0];
|
||
} else if (mode === "first-last-frame" && images && images.length >= 2) {
|
||
options.firstFrameImage = images[0];
|
||
options.lastFrameImage = images[1];
|
||
} else if (mode === "reference-images" && images && images.length > 0) {
|
||
options.referenceImages = images.slice(0, 4);
|
||
}
|
||
|
||
return options;
|
||
}
|
||
|
||
/**
|
||
* 视频生成工具
|
||
*/
|
||
export function generateVideo({ dataStream }: { dataStream: any }) {
|
||
return tool({
|
||
description:
|
||
"生成视频，支持文生视频、图生视频（首帧/首尾帧/参考图）等多种模式",
|
||
inputSchema: z.object({
|
||
prompt: z.string().describe("视频生成提示词，描述视频内容、镜头运动等"),
|
||
mode: z
|
||
.enum(["text-to-video", "first-frame", "first-last-frame", "reference-images"])
|
||
.describe("生成模式：文生视频/图生视频-首帧/首尾帧/参考图"),
|
||
images: z
|
||
.array(z.string())
|
||
.optional()
|
||
.describe("图片 URL 数组（根据模式不同，需要 0-4 张）"),
|
||
resolution: z
|
||
.enum(["480p", "720p", "1080p"])
|
||
.default("720p")
|
||
.describe("视频分辨率"),
|
||
ratio: z
|
||
.enum(["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"])
|
||
.default("16:9")
|
||
.describe("视频宽高比"),
|
||
duration: z
|
||
.number()
|
||
.min(2)
|
||
.max(12)
|
||
.default(5)
|
||
.describe("视频时长（2-12秒）"),
|
||
watermark: z.boolean().default(true).describe("是否添加水印"),
|
||
cameraFixed: z.boolean().default(false).describe("是否固定摄像头"),
|
||
seed: z.number().optional().describe("随机种子（可选）"),
|
||
returnLastFrame: z
|
||
.boolean()
|
||
.default(false)
|
||
.describe("是否返回尾帧（用于连续生成）"),
|
||
}),
|
||
execute: async ({ prompt, mode, images, resolution, ratio, duration, watermark, cameraFixed, seed, returnLastFrame }) => {
|
||
try {
|
||
console.log("[Generate Video Tool] 🎬 开始生成视频");
|
||
console.log("[Generate Video Tool] 模式:", mode);
|
||
console.log("[Generate Video Tool] 提示词:", prompt);
|
||
|
||
// 1. 🆕 智能选择最佳子模型
|
||
// 先获取火山引擎视频主模型
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const volcanoMasterModel = allModels.find(
|
||
m => m.provider.type === "volcano-video" && m.model.modelId === "volcano-video-master"
|
||
);
|
||
|
||
if (!volcanoMasterModel) {
|
||
throw new Error("未找到火山引擎视频主模型，请在系统设置中添加");
|
||
}
|
||
|
||
// 准备参数用于智能选择
|
||
const params = {
|
||
mode,
|
||
resolution,
|
||
ratio,
|
||
duration,
|
||
watermark,
|
||
cameraFixed,
|
||
seed,
|
||
returnLastFrame,
|
||
referenceImages: images,
|
||
};
|
||
|
||
// 根据用户参数智能选择最佳子模型
|
||
const { model, capabilities } = await selectBestSubModel(
|
||
volcanoMasterModel.model.id,
|
||
mode,
|
||
params
|
||
);
|
||
|
||
console.log(
|
||
"[Generate Video Tool] ✅ 智能选择模型:",
|
||
model.model.displayName,
|
||
`(${model.model.modelId})`
|
||
);
|
||
|
||
// 2. 验证参数是否符合模型能力
|
||
validateParamsAgainstCapabilities(params, capabilities);
|
||
|
||
// 3. 构建视频生成选项
|
||
const options = buildVideoOptions(
|
||
mode,
|
||
images,
|
||
prompt,
|
||
params,
|
||
model.model.modelId
|
||
);
|
||
|
||
// 4. 调用视频生成（流式输出）
|
||
dataStream.writeData({
|
||
type: "video-generation-start",
|
||
data: {
|
||
mode,
|
||
model: model.model.displayName,
|
||
},
|
||
});
|
||
|
||
const apiKey = model.provider.apiKey;
|
||
const baseURL = model.provider.baseUrl;
|
||
|
||
let videoMetadata: any = null;
|
||
|
||
// 使用异步生成器进行流式输出
|
||
for await (const chunk of doGenerateStream(apiKey, baseURL, options)) {
|
||
dataStream.writeMessageAnnotation({
|
||
type: "video-progress",
|
||
content: chunk,
|
||
});
|
||
|
||
// 提取视频元数据（从注释中）
|
||
const metadataMatch = chunk.match(
|
||
/<!-- VIDEO_METADATA:(.*?) -->/
|
||
);
|
||
if (metadataMatch) {
|
||
try {
|
||
videoMetadata = JSON.parse(metadataMatch[1]);
|
||
} catch (e) {
|
||
console.error("[Generate Video Tool] 解析元数据失败:", e);
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log("[Generate Video Tool] ✅ 视频生成完成");
|
||
|
||
// 5. 返回元数据（用于保存到数据库）
|
||
return {
|
||
success: true,
|
||
videoUrl: videoMetadata?.temporaryVideoUrl,
|
||
metadata: videoMetadata,
|
||
};
|
||
} catch (error: any) {
|
||
console.error("[Generate Video Tool] ❌ 生成失败:", error);
|
||
dataStream.writeMessageAnnotation({
|
||
type: "video-error",
|
||
content: `视频生成失败：${error.message}`,
|
||
});
|
||
throw error;
|
||
}
|
||
},
|
||
});
|
||
}
|