552 lines
18 KiB
TypeScript
552 lines
18 KiB
TypeScript
/**
|
||
* 视频生成工具
|
||
* 支持火山引擎视频生成的多种模式
|
||
*/
|
||
|
||
import { tool } from "ai";
|
||
import { z } from "zod";
|
||
import { doGenerateStream } from "@/lib/ai/providers/volcano-video";
|
||
import {
|
||
getEnabledAiModelsWithProvider,
|
||
getSubModelsByParentModelId
|
||
} from "@/lib/db/queries";
|
||
import { getConfiguredVideoDurationOptions } from "@/lib/ai/video-model-config";
|
||
import type {
|
||
VideoGenerationMode,
|
||
VideoModelCapabilities,
|
||
VideoGenerationOptions,
|
||
} from "@/types/ai";
|
||
|
||
export interface ExecuteDirectVideoGenerationParams {
|
||
prompt: string;
|
||
mode: VideoGenerationMode;
|
||
images?: string[];
|
||
resolution?: "480p" | "720p" | "1080p";
|
||
ratio?: "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "21:9" | "adaptive";
|
||
duration?: number;
|
||
watermark?: boolean;
|
||
cameraFixed?: boolean;
|
||
seed?: number;
|
||
returnLastFrame?: boolean;
|
||
preferredModelRef?: string;
|
||
onProgress?: (chunk: string) => void | Promise<void>;
|
||
}
|
||
|
||
export interface ExecuteDirectVideoGenerationResult {
|
||
success: boolean;
|
||
videoUrl?: string;
|
||
metadata?: Record<string, any> | null;
|
||
modelDisplayName: string;
|
||
modelId: string;
|
||
modelRef: string;
|
||
}
|
||
|
||
function inferSupportedModesFromModelId(modelId: string): VideoGenerationMode[] {
|
||
const normalized = modelId.toLowerCase();
|
||
|
||
if (normalized.includes("lite-t2v")) return ["text-to-video"];
|
||
if (normalized.includes("lite-i2v")) return ["first-frame", "first-last-frame", "reference-images"];
|
||
if (normalized.includes("pro-fast")) return ["text-to-video", "first-frame"];
|
||
if (normalized.includes("-pro-") || normalized.endsWith("-pro")) {
|
||
return ["text-to-video", "first-frame", "first-last-frame"];
|
||
}
|
||
|
||
return ["text-to-video", "first-frame", "first-last-frame", "reference-images"];
|
||
}
|
||
|
||
function getSupportedModes(caps: Partial<VideoModelCapabilities> | undefined, modelId: string): VideoGenerationMode[] {
|
||
const modes = (caps?.supportedModes || []) as VideoGenerationMode[];
|
||
if (Array.isArray(modes) && modes.length > 0) {
|
||
return modes;
|
||
}
|
||
return inferSupportedModesFromModelId(modelId);
|
||
}
|
||
|
||
function withConfiguredVideoCapabilities(
|
||
capabilities: Partial<VideoModelCapabilities> | undefined,
|
||
imageGenConfig?: { videoDurationOptions?: number[] | null } | null
|
||
): VideoModelCapabilities {
|
||
const baseCapabilities = {
|
||
supportedModes: ["text-to-video", "first-frame", "first-last-frame", "reference-images"],
|
||
supportedResolutions: ["480p", "720p", "1080p"],
|
||
supportedRatios: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"],
|
||
durationRange: { min: 2, max: 12 },
|
||
maxReferenceImages: 4,
|
||
...(capabilities || {}),
|
||
} as VideoModelCapabilities;
|
||
|
||
const durationOptions = getConfiguredVideoDurationOptions(imageGenConfig);
|
||
if (durationOptions.length === 0) {
|
||
return baseCapabilities;
|
||
}
|
||
|
||
return {
|
||
...baseCapabilities,
|
||
durationOptions,
|
||
durationRange: {
|
||
min: durationOptions[0],
|
||
max: durationOptions[durationOptions.length - 1],
|
||
},
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 🆕 智能选择最佳子模型
|
||
* 根据用户选择的模式和参数，从子模型列表中选择最合适的模型
|
||
*/
|
||
async function selectBestSubModel(
|
||
parentModelId: string,
|
||
mode: VideoGenerationMode,
|
||
params: Partial<VideoGenerationOptions>
|
||
): Promise<{ model: any; capabilities: VideoModelCapabilities }> {
|
||
// 获取所有子模型
|
||
const subModels = await getSubModelsByParentModelId(parentModelId);
|
||
|
||
if (subModels.length === 0) {
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const directModel = allModels.find(
|
||
(item) =>
|
||
item.model.id === parentModelId &&
|
||
(item.provider.type === "volcano-video" || item.provider.type === "doubao-video"),
|
||
);
|
||
|
||
if (!directModel) {
|
||
throw new Error("没有可用的视频生成子模型");
|
||
}
|
||
|
||
const directCapabilities = withConfiguredVideoCapabilities(
|
||
directModel.model.capabilities as VideoModelCapabilities | undefined,
|
||
(directModel.model as any).imageGenConfig
|
||
);
|
||
const supportedModes = getSupportedModes(directCapabilities, directModel.model.modelId);
|
||
if (!supportedModes.includes(mode)) {
|
||
throw new Error(`模型 ${directModel.model.displayName} 不支持 ${mode} 模式`);
|
||
}
|
||
|
||
return {
|
||
model: directModel,
|
||
capabilities: {
|
||
...directCapabilities,
|
||
supportedModes,
|
||
} as VideoModelCapabilities,
|
||
};
|
||
}
|
||
|
||
console.log(`[Smart Model Selection] 找到 ${subModels.length} 个子模型`);
|
||
|
||
// 筛选支持指定模式的子模型
|
||
const compatibleModels = subModels.filter((m) => {
|
||
const caps = withConfiguredVideoCapabilities(
|
||
m.model.capabilities as VideoModelCapabilities,
|
||
(m.model as any).imageGenConfig
|
||
);
|
||
const supportsMode = getSupportedModes(caps, m.model.modelId).includes(mode);
|
||
|
||
// 检查分辨率支持（如果用户指定了分辨率）
|
||
const supportedResolutions = caps.supportedResolutions || ["480p", "720p", "1080p"];
|
||
const supportsResolution = !params.resolution || supportedResolutions.includes(params.resolution);
|
||
|
||
return supportsMode && supportsResolution;
|
||
});
|
||
|
||
if (compatibleModels.length === 0) {
|
||
throw new Error(`没有模型支持 ${mode} 模式和指定的参数`);
|
||
}
|
||
|
||
console.log(`[Smart Model Selection] ${compatibleModels.length} 个模型支持 ${mode} 模式`);
|
||
|
||
// 🎯 智能优先级排序
|
||
const priority: Record<string, number> = {
|
||
"doubao-seedance-1-0-pro-250528": 10, // 最高优先级（全功能）
|
||
"doubao-seedance-1-0-pro-fast-251015": 8, // 快速模式
|
||
"doubao-seedance-1-0-lite-i2v-250428": 6, // 图生视频轻量版
|
||
"doubao-seedance-1-0-lite-t2v-250428": 5, // 文生视频轻量版
|
||
};
|
||
|
||
// 选择最佳模型
|
||
const bestModel = compatibleModels.sort((a, b) => {
|
||
const priorityA = priority[a.model.modelId] || 0;
|
||
const priorityB = priority[b.model.modelId] || 0;
|
||
return priorityB - priorityA;
|
||
})[0];
|
||
|
||
console.log(`[Smart Model Selection] ✅ 选择模型: ${bestModel.model.displayName} (${bestModel.model.modelId})`);
|
||
|
||
return {
|
||
model: bestModel,
|
||
capabilities: withConfiguredVideoCapabilities(
|
||
bestModel.model.capabilities as VideoModelCapabilities,
|
||
(bestModel.model as any).imageGenConfig
|
||
),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 获取支持指定模式的视频模型（兼容旧逻辑）
|
||
*/
|
||
async function getCompatibleModel(
|
||
mode: VideoGenerationMode,
|
||
parentModelId?: string,
|
||
params?: Partial<VideoGenerationOptions>
|
||
): Promise<{ model: any; capabilities: VideoModelCapabilities }> {
|
||
// 🆕 如果提供了parentModelId，使用智能子模型选择
|
||
if (parentModelId) {
|
||
return selectBestSubModel(parentModelId, mode, params || {});
|
||
}
|
||
|
||
// 兼容旧逻辑：直接从所有模型中选择（用于向后兼容）
|
||
const models = await getEnabledAiModelsWithProvider();
|
||
|
||
const volcanoVideoModels = models.filter(
|
||
(m) =>
|
||
(m.provider.type === "volcano-video" || m.provider.type === "doubao-video") &&
|
||
m.model.isEnabled &&
|
||
m.model.capabilities
|
||
);
|
||
|
||
if (volcanoVideoModels.length === 0) {
|
||
throw new Error("没有启用的火山引擎视频模型");
|
||
}
|
||
|
||
const compatibleModels = volcanoVideoModels.filter((m) => {
|
||
const caps = withConfiguredVideoCapabilities(
|
||
m.model.capabilities as VideoModelCapabilities,
|
||
(m.model as any).imageGenConfig
|
||
);
|
||
return getSupportedModes(caps, m.model.modelId).includes(mode);
|
||
});
|
||
|
||
if (compatibleModels.length === 0) {
|
||
throw new Error(`没有模型支持 ${mode} 模式`);
|
||
}
|
||
|
||
const priority: Record<string, number> = {
|
||
"doubao-seedance-1-0-pro-250528": 10,
|
||
"doubao-seedance-1-0-pro-fast-251015": 8,
|
||
"doubao-seedance-1-0-lite-i2v-250428": 6,
|
||
"doubao-seedance-1-0-lite-t2v-250428": 5,
|
||
};
|
||
|
||
const bestModel = compatibleModels.sort((a, b) => {
|
||
const priorityA = priority[a.model.modelId] || 0;
|
||
const priorityB = priority[b.model.modelId] || 0;
|
||
return priorityB - priorityA;
|
||
})[0];
|
||
|
||
return {
|
||
model: bestModel,
|
||
capabilities: withConfiguredVideoCapabilities(
|
||
bestModel.model.capabilities as VideoModelCapabilities,
|
||
(bestModel.model as any).imageGenConfig
|
||
),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 验证参数是否符合模型能力
|
||
*/
|
||
function validateParamsAgainstCapabilities(
|
||
params: Partial<VideoGenerationOptions>,
|
||
capabilities: VideoModelCapabilities
|
||
): void {
|
||
// 检查分辨率
|
||
if (
|
||
params.resolution &&
|
||
!capabilities.supportedResolutions.includes(params.resolution)
|
||
) {
|
||
throw new Error(
|
||
`模型不支持 ${params.resolution} 分辨率，支持的分辨率：${capabilities.supportedResolutions.join(", ")}`
|
||
);
|
||
}
|
||
|
||
// 检查宽高比
|
||
if (params.ratio && !capabilities.supportedRatios.includes(params.ratio)) {
|
||
throw new Error(
|
||
`模型不支持 ${params.ratio} 宽高比，支持的宽高比：${capabilities.supportedRatios.join(", ")}`
|
||
);
|
||
}
|
||
|
||
// 检查时长范围
|
||
if (params.duration) {
|
||
const allowedDurations = Array.isArray(capabilities.durationOptions)
|
||
? capabilities.durationOptions
|
||
: [];
|
||
if (allowedDurations.length > 0) {
|
||
if (!allowedDurations.includes(params.duration)) {
|
||
throw new Error(`视频时长仅支持 ${allowedDurations.join(" / ")} 秒`);
|
||
}
|
||
} else {
|
||
const { min, max } = capabilities.durationRange;
|
||
if (params.duration < min || params.duration > max) {
|
||
throw new Error(`视频时长必须在 ${min}-${max} 秒之间`);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 检查参考图数量
|
||
if (
|
||
params.mode === "reference-images" &&
|
||
params.referenceImages &&
|
||
capabilities.maxReferenceImages
|
||
) {
|
||
if (params.referenceImages.length > capabilities.maxReferenceImages) {
|
||
throw new Error(
|
||
`参考图数量不能超过 ${capabilities.maxReferenceImages} 张`
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 构建视频生成选项
|
||
*/
|
||
function buildVideoOptions(
|
||
mode: VideoGenerationMode,
|
||
images: string[] | undefined,
|
||
prompt: string,
|
||
params: any,
|
||
modelId: string
|
||
): VideoGenerationOptions {
|
||
const options: VideoGenerationOptions = {
|
||
prompt,
|
||
model: modelId,
|
||
mode,
|
||
resolution: params.resolution || "720p",
|
||
ratio: params.ratio || "16:9",
|
||
duration: params.duration || 5,
|
||
watermark: params.watermark ?? true,
|
||
cameraFixed: params.cameraFixed ?? false,
|
||
fps: 24,
|
||
};
|
||
|
||
// 根据模式处理图片输入
|
||
if (mode === "first-frame" && images && images.length > 0) {
|
||
options.firstFrameImage = images[0];
|
||
} else if (mode === "first-last-frame" && images && images.length >= 2) {
|
||
options.firstFrameImage = images[0];
|
||
options.lastFrameImage = images[1];
|
||
} else if (mode === "reference-images" && images && images.length > 0) {
|
||
options.referenceImages = images.slice(0, 4);
|
||
}
|
||
|
||
return options;
|
||
}
|
||
|
||
async function resolvePreferredVideoParentModelId(preferredModelRef?: string) {
|
||
const normalizedRef = String(preferredModelRef || "").trim();
|
||
if (!normalizedRef) return undefined;
|
||
|
||
const models = await getEnabledAiModelsWithProvider();
|
||
const matched = models.find(
|
||
(item) =>
|
||
(item.provider.type === "volcano-video" || item.provider.type === "doubao-video") &&
|
||
(item.model.id === normalizedRef || item.model.modelId === normalizedRef),
|
||
);
|
||
|
||
if (!matched) return undefined;
|
||
if (matched.model.isSubModel && matched.model.parentModelId) {
|
||
return matched.model.parentModelId;
|
||
}
|
||
return matched.model.id;
|
||
}
|
||
|
||
export async function executeDirectVideoGeneration(
|
||
params: ExecuteDirectVideoGenerationParams,
|
||
): Promise<ExecuteDirectVideoGenerationResult> {
|
||
const prompt = String(params.prompt || "").trim();
|
||
if (!prompt) {
|
||
throw new Error("缺少视频生成提示词");
|
||
}
|
||
|
||
const preferredParentModelId = await resolvePreferredVideoParentModelId(
|
||
params.preferredModelRef,
|
||
);
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const fallbackParentModel =
|
||
allModels.find(
|
||
(item) =>
|
||
(item.provider.type === "volcano-video" || item.provider.type === "doubao-video") &&
|
||
!item.model.isSubModel &&
|
||
item.model.modelId === "volcano-video-master",
|
||
) ||
|
||
allModels.find(
|
||
(item) =>
|
||
(item.provider.type === "volcano-video" || item.provider.type === "doubao-video") &&
|
||
!item.model.isSubModel,
|
||
);
|
||
|
||
const parentModelId =
|
||
preferredParentModelId ||
|
||
fallbackParentModel?.model.id ||
|
||
allModels.find(
|
||
(item) =>
|
||
(item.provider.type === "volcano-video" || item.provider.type === "doubao-video") &&
|
||
item.model.isEnabled &&
|
||
item.model.isSubModel &&
|
||
item.model.parentModelId,
|
||
)?.model.parentModelId ||
|
||
undefined;
|
||
|
||
const generationParams = {
|
||
mode: params.mode,
|
||
resolution: params.resolution || "720p",
|
||
ratio: params.ratio || "16:9",
|
||
duration: params.duration || 5,
|
||
watermark: params.watermark ?? true,
|
||
cameraFixed: params.cameraFixed ?? false,
|
||
seed: params.seed,
|
||
returnLastFrame: params.returnLastFrame ?? false,
|
||
referenceImages: params.images,
|
||
} satisfies Partial<VideoGenerationOptions>;
|
||
|
||
let selectedModelConfig: { model: any; capabilities: VideoModelCapabilities };
|
||
try {
|
||
selectedModelConfig = await getCompatibleModel(
|
||
params.mode,
|
||
parentModelId,
|
||
generationParams,
|
||
);
|
||
} catch (preferredModelError) {
|
||
if (parentModelId) {
|
||
console.warn(
|
||
"[Generate Video Tool] Preferred parent model selection failed, fallback to global compatible model:",
|
||
preferredModelError,
|
||
);
|
||
selectedModelConfig = await getCompatibleModel(
|
||
params.mode,
|
||
undefined,
|
||
generationParams,
|
||
);
|
||
} else {
|
||
throw preferredModelError;
|
||
}
|
||
}
|
||
|
||
const { model, capabilities } = selectedModelConfig;
|
||
|
||
validateParamsAgainstCapabilities(generationParams, capabilities);
|
||
|
||
const options = buildVideoOptions(
|
||
params.mode,
|
||
params.images,
|
||
prompt,
|
||
generationParams,
|
||
model.model.modelId,
|
||
);
|
||
|
||
let videoMetadata: Record<string, any> | null = null;
|
||
for await (const chunk of doGenerateStream(model.provider.apiKey, model.provider.baseUrl, options)) {
|
||
await params.onProgress?.(chunk);
|
||
const metadataMatch = chunk.match(/<!-- VIDEO_METADATA:(.*?) -->/);
|
||
if (metadataMatch) {
|
||
try {
|
||
videoMetadata = JSON.parse(metadataMatch[1]);
|
||
} catch (error) {
|
||
console.error("[Generate Video Tool] 解析元数据失败:", error);
|
||
}
|
||
}
|
||
}
|
||
|
||
return {
|
||
success: true,
|
||
videoUrl: videoMetadata?.temporaryVideoUrl || undefined,
|
||
metadata: videoMetadata,
|
||
modelDisplayName: String(model.model.displayName || model.model.modelId || "视频模型"),
|
||
modelId: String(model.model.modelId || model.model.id),
|
||
modelRef: String(model.model.id || model.model.modelId),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 视频生成工具
|
||
*/
|
||
export function generateVideo({ dataStream }: { dataStream: any }) {
|
||
return tool({
|
||
description:
|
||
"生成视频，支持文生视频、图生视频（首帧/首尾帧/参考图）等多种模式",
|
||
inputSchema: z.object({
|
||
prompt: z.string().describe("视频生成提示词，描述视频内容、镜头运动等"),
|
||
mode: z
|
||
.enum(["text-to-video", "first-frame", "first-last-frame", "reference-images"])
|
||
.describe("生成模式：文生视频/图生视频-首帧/首尾帧/参考图"),
|
||
images: z
|
||
.array(z.string())
|
||
.optional()
|
||
.describe("图片 URL 数组（根据模式不同，需要 0-4 张）"),
|
||
resolution: z
|
||
.enum(["480p", "720p", "1080p"])
|
||
.default("720p")
|
||
.describe("视频分辨率"),
|
||
ratio: z
|
||
.enum(["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"])
|
||
.default("16:9")
|
||
.describe("视频宽高比"),
|
||
duration: z
|
||
.number()
|
||
.min(1)
|
||
.max(600)
|
||
.default(5)
|
||
.describe("视频时长（秒）。优先使用当前视频模型配置的秒数档位，服务端会按模型能力动态校验。"),
|
||
watermark: z.boolean().default(true).describe("是否添加水印"),
|
||
cameraFixed: z.boolean().default(false).describe("是否固定摄像头"),
|
||
seed: z.number().optional().describe("随机种子（可选）"),
|
||
returnLastFrame: z
|
||
.boolean()
|
||
.default(false)
|
||
.describe("是否返回尾帧（用于连续生成）"),
|
||
}),
|
||
execute: async ({ prompt, mode, images, resolution, ratio, duration, watermark, cameraFixed, seed, returnLastFrame }) => {
|
||
try {
|
||
console.log("[Generate Video Tool] 🎬 开始生成视频");
|
||
console.log("[Generate Video Tool] 模式:", mode);
|
||
console.log("[Generate Video Tool] 提示词:", prompt);
|
||
|
||
dataStream.writeData({
|
||
type: "video-generation-start",
|
||
data: {
|
||
mode,
|
||
model: "智能选择中",
|
||
},
|
||
});
|
||
|
||
const result = await executeDirectVideoGeneration({
|
||
prompt,
|
||
mode,
|
||
images,
|
||
resolution,
|
||
ratio,
|
||
duration,
|
||
watermark,
|
||
cameraFixed,
|
||
seed,
|
||
returnLastFrame,
|
||
onProgress: async (chunk) => {
|
||
dataStream.writeMessageAnnotation({
|
||
type: "video-progress",
|
||
content: chunk,
|
||
});
|
||
},
|
||
});
|
||
|
||
console.log("[Generate Video Tool] ✅ 视频生成完成");
|
||
|
||
return {
|
||
success: true,
|
||
videoUrl: result.videoUrl,
|
||
metadata: result.metadata,
|
||
modelDisplayName: result.modelDisplayName,
|
||
modelId: result.modelId,
|
||
modelRef: result.modelRef,
|
||
};
|
||
} catch (error: any) {
|
||
console.error("[Generate Video Tool] ❌ 生成失败:", error);
|
||
dataStream.writeMessageAnnotation({
|
||
type: "video-error",
|
||
content: `视频生成失败：${error.message}`,
|
||
});
|
||
throw error;
|
||
}
|
||
},
|
||
});
|
||
}
|