772 lines
24 KiB
TypeScript
772 lines
24 KiB
TypeScript
/**
|
||
* 视频生成工具
|
||
* 支持火山引擎视频生成的多种模式
|
||
*/
|
||
|
||
import { tool } from "ai";
|
||
import { z } from "zod";
|
||
import { doGenerateStream } from "@/lib/ai/providers/volcano-video";
|
||
import { executeAtlasCloudVideoGeneration } from "@/lib/ai/providers/atlascloud";
|
||
import {
|
||
getEnabledAiModelsWithProvider,
|
||
getSubModelsByParentModelId
|
||
} from "@/lib/db/queries";
|
||
import { getConfiguredVideoDurationOptions } from "@/lib/ai/video-model-config";
|
||
import type {
|
||
VideoGenerationMode,
|
||
VideoModelCapabilities,
|
||
VideoGenerationOptions,
|
||
} from "@/types/ai";
|
||
|
||
export interface ExecuteDirectVideoGenerationParams {
|
||
prompt: string;
|
||
mode: VideoGenerationMode;
|
||
images?: string[];
|
||
resolution?: "480p" | "720p" | "1080p";
|
||
ratio?: "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "21:9" | "adaptive";
|
||
duration?: number;
|
||
watermark?: boolean;
|
||
cameraFixed?: boolean;
|
||
seed?: number;
|
||
returnLastFrame?: boolean;
|
||
preferredModelRef?: string;
|
||
onProgress?: (chunk: string) => void | Promise<void>;
|
||
}
|
||
|
||
export interface ExecuteDirectVideoGenerationResult {
|
||
success: boolean;
|
||
videoUrl?: string;
|
||
metadata?: Record<string, any> | null;
|
||
modelDisplayName: string;
|
||
modelId: string;
|
||
modelRef: string;
|
||
}
|
||
|
||
function inferSupportedModesFromModelId(modelId: string): VideoGenerationMode[] {
|
||
const normalized = modelId.toLowerCase();
|
||
|
||
if (normalized.includes("lite-t2v")) return ["text-to-video"];
|
||
if (normalized.includes("lite-i2v")) return ["first-frame", "first-last-frame", "reference-images"];
|
||
if (normalized.includes("pro-fast")) return ["text-to-video", "first-frame"];
|
||
if (normalized.includes("-pro-") || normalized.endsWith("-pro")) {
|
||
return ["text-to-video", "first-frame", "first-last-frame"];
|
||
}
|
||
|
||
return ["text-to-video", "first-frame", "first-last-frame", "reference-images"];
|
||
}
|
||
|
||
function getSupportedModes(caps: Partial<VideoModelCapabilities> | undefined, modelId: string): VideoGenerationMode[] {
|
||
const modes = (caps?.supportedModes || []) as VideoGenerationMode[];
|
||
if (Array.isArray(modes) && modes.length > 0) {
|
||
return modes;
|
||
}
|
||
return inferSupportedModesFromModelId(modelId);
|
||
}
|
||
|
||
function withConfiguredVideoCapabilities(
|
||
capabilities: Partial<VideoModelCapabilities> | undefined,
|
||
imageGenConfig?: { videoDurationOptions?: number[] | null } | null
|
||
): VideoModelCapabilities {
|
||
const baseCapabilities = {
|
||
supportedModes: ["text-to-video", "first-frame", "first-last-frame", "reference-images"],
|
||
supportedResolutions: ["480p", "720p", "1080p"],
|
||
supportedRatios: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"],
|
||
durationRange: { min: 2, max: 12 },
|
||
maxReferenceImages: 4,
|
||
...(capabilities || {}),
|
||
} as VideoModelCapabilities;
|
||
|
||
const durationOptions = getConfiguredVideoDurationOptions(imageGenConfig);
|
||
if (durationOptions.length === 0) {
|
||
return baseCapabilities;
|
||
}
|
||
|
||
return {
|
||
...baseCapabilities,
|
||
durationOptions,
|
||
durationRange: {
|
||
min: durationOptions[0],
|
||
max: durationOptions[durationOptions.length - 1],
|
||
},
|
||
};
|
||
}
|
||
|
||
function isAgnesVideoModel(item: any) {
|
||
const capabilities = item?.model?.capabilities;
|
||
const provider = capabilities && typeof capabilities === "object"
|
||
? String((capabilities as Record<string, unknown>).provider || "")
|
||
: "";
|
||
const kind = capabilities && typeof capabilities === "object"
|
||
? String((capabilities as Record<string, unknown>).kind || "")
|
||
: "";
|
||
const modelId = String(item?.model?.modelId || "").toLowerCase();
|
||
return (
|
||
item?.provider?.type === "custom" &&
|
||
(provider === "agnes-ai" || (kind === "video" && modelId.startsWith("agnes-video")))
|
||
);
|
||
}
|
||
|
||
function isVolcanoVideoModel(item: any) {
|
||
return item?.provider?.type === "volcano-video" || item?.provider?.type === "doubao-video";
|
||
}
|
||
|
||
function isAtlasCloudVideoModel(item: any) {
|
||
const capabilities = item?.model?.capabilities;
|
||
return (
|
||
item?.provider?.type === "atlascloud" &&
|
||
capabilities &&
|
||
typeof capabilities === "object" &&
|
||
String((capabilities as Record<string, unknown>).mediaType || "") === "video"
|
||
);
|
||
}
|
||
|
||
function isDirectExecutableVideoModel(item: any) {
|
||
return (
|
||
isVolcanoVideoModel(item) ||
|
||
isAgnesVideoModel(item) ||
|
||
isAtlasCloudVideoModel(item)
|
||
);
|
||
}
|
||
|
||
/**
|
||
* 🆕 智能选择最佳子模型
|
||
* 根据用户选择的模式和参数，从子模型列表中选择最合适的模型
|
||
*/
|
||
async function selectBestSubModel(
|
||
parentModelId: string,
|
||
mode: VideoGenerationMode,
|
||
params: Partial<VideoGenerationOptions>
|
||
): Promise<{ model: any; capabilities: VideoModelCapabilities }> {
|
||
// 获取所有子模型
|
||
const subModels = await getSubModelsByParentModelId(parentModelId);
|
||
|
||
if (subModels.length === 0) {
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const directModel = allModels.find(
|
||
(item) =>
|
||
item.model.id === parentModelId &&
|
||
isDirectExecutableVideoModel(item),
|
||
);
|
||
|
||
if (!directModel) {
|
||
throw new Error("没有可用的视频生成子模型");
|
||
}
|
||
|
||
const directCapabilities = withConfiguredVideoCapabilities(
|
||
directModel.model.capabilities as VideoModelCapabilities | undefined,
|
||
(directModel.model as any).imageGenConfig
|
||
);
|
||
const supportedModes = getSupportedModes(directCapabilities, directModel.model.modelId);
|
||
if (!supportedModes.includes(mode)) {
|
||
throw new Error(`模型 ${directModel.model.displayName} 不支持 ${mode} 模式`);
|
||
}
|
||
|
||
return {
|
||
model: directModel,
|
||
capabilities: {
|
||
...directCapabilities,
|
||
supportedModes,
|
||
} as VideoModelCapabilities,
|
||
};
|
||
}
|
||
|
||
console.log(`[Smart Model Selection] 找到 ${subModels.length} 个子模型`);
|
||
|
||
// 筛选支持指定模式的子模型
|
||
const compatibleModels = subModels.filter((m) => {
|
||
const caps = withConfiguredVideoCapabilities(
|
||
m.model.capabilities as VideoModelCapabilities,
|
||
(m.model as any).imageGenConfig
|
||
);
|
||
const supportsMode = getSupportedModes(caps, m.model.modelId).includes(mode);
|
||
|
||
// 检查分辨率支持（如果用户指定了分辨率）
|
||
const supportedResolutions = caps.supportedResolutions || ["480p", "720p", "1080p"];
|
||
const supportsResolution = !params.resolution || supportedResolutions.includes(params.resolution);
|
||
|
||
return supportsMode && supportsResolution;
|
||
});
|
||
|
||
if (compatibleModels.length === 0) {
|
||
throw new Error(`没有模型支持 ${mode} 模式和指定的参数`);
|
||
}
|
||
|
||
console.log(`[Smart Model Selection] ${compatibleModels.length} 个模型支持 ${mode} 模式`);
|
||
|
||
// 🎯 智能优先级排序
|
||
const priority: Record<string, number> = {
|
||
"doubao-seedance-1-0-pro-250528": 10, // 最高优先级（全功能）
|
||
"doubao-seedance-1-0-pro-fast-251015": 8, // 快速模式
|
||
"doubao-seedance-1-0-lite-i2v-250428": 6, // 图生视频轻量版
|
||
"doubao-seedance-1-0-lite-t2v-250428": 5, // 文生视频轻量版
|
||
};
|
||
|
||
// 选择最佳模型
|
||
const bestModel = compatibleModels.sort((a, b) => {
|
||
const priorityA = priority[a.model.modelId] || 0;
|
||
const priorityB = priority[b.model.modelId] || 0;
|
||
return priorityB - priorityA;
|
||
})[0];
|
||
|
||
console.log(`[Smart Model Selection] ✅ 选择模型: ${bestModel.model.displayName} (${bestModel.model.modelId})`);
|
||
|
||
return {
|
||
model: bestModel,
|
||
capabilities: withConfiguredVideoCapabilities(
|
||
bestModel.model.capabilities as VideoModelCapabilities,
|
||
(bestModel.model as any).imageGenConfig
|
||
),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 获取支持指定模式的视频模型（兼容旧逻辑）
|
||
*/
|
||
async function getCompatibleModel(
|
||
mode: VideoGenerationMode,
|
||
parentModelId?: string,
|
||
params?: Partial<VideoGenerationOptions>
|
||
): Promise<{ model: any; capabilities: VideoModelCapabilities }> {
|
||
// 🆕 如果提供了parentModelId，使用智能子模型选择
|
||
if (parentModelId) {
|
||
return selectBestSubModel(parentModelId, mode, params || {});
|
||
}
|
||
|
||
// 兼容旧逻辑：直接从所有模型中选择（用于向后兼容）
|
||
const models = await getEnabledAiModelsWithProvider();
|
||
|
||
const volcanoVideoModels = models.filter(
|
||
(m) =>
|
||
isDirectExecutableVideoModel(m) &&
|
||
m.model.isEnabled &&
|
||
m.model.capabilities
|
||
);
|
||
|
||
if (volcanoVideoModels.length === 0) {
|
||
throw new Error("没有启用的可执行视频模型");
|
||
}
|
||
|
||
const compatibleModels = volcanoVideoModels.filter((m) => {
|
||
const caps = withConfiguredVideoCapabilities(
|
||
m.model.capabilities as VideoModelCapabilities,
|
||
(m.model as any).imageGenConfig
|
||
);
|
||
return getSupportedModes(caps, m.model.modelId).includes(mode);
|
||
});
|
||
|
||
if (compatibleModels.length === 0) {
|
||
throw new Error(`没有模型支持 ${mode} 模式`);
|
||
}
|
||
|
||
const priority: Record<string, number> = {
|
||
"doubao-seedance-1-0-pro-250528": 10,
|
||
"doubao-seedance-1-0-pro-fast-251015": 8,
|
||
"doubao-seedance-1-0-lite-i2v-250428": 6,
|
||
"doubao-seedance-1-0-lite-t2v-250428": 5,
|
||
};
|
||
|
||
const bestModel = compatibleModels.sort((a, b) => {
|
||
const priorityA = priority[a.model.modelId] || 0;
|
||
const priorityB = priority[b.model.modelId] || 0;
|
||
return priorityB - priorityA;
|
||
})[0];
|
||
|
||
return {
|
||
model: bestModel,
|
||
capabilities: withConfiguredVideoCapabilities(
|
||
bestModel.model.capabilities as VideoModelCapabilities,
|
||
(bestModel.model as any).imageGenConfig
|
||
),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 验证参数是否符合模型能力
|
||
*/
|
||
function validateParamsAgainstCapabilities(
|
||
params: Partial<VideoGenerationOptions>,
|
||
capabilities: VideoModelCapabilities
|
||
): void {
|
||
// 检查分辨率
|
||
if (
|
||
params.resolution &&
|
||
capabilities.supportedResolutions.length > 0 &&
|
||
!capabilities.supportedResolutions.includes(params.resolution)
|
||
) {
|
||
throw new Error(
|
||
`模型不支持 ${params.resolution} 分辨率，支持的分辨率：${capabilities.supportedResolutions.join(", ")}`
|
||
);
|
||
}
|
||
|
||
// 检查宽高比
|
||
if (
|
||
params.ratio &&
|
||
capabilities.supportedRatios.length > 0 &&
|
||
!capabilities.supportedRatios.includes(params.ratio)
|
||
) {
|
||
throw new Error(
|
||
`模型不支持 ${params.ratio} 宽高比，支持的宽高比：${capabilities.supportedRatios.join(", ")}`
|
||
);
|
||
}
|
||
|
||
// 检查时长范围
|
||
if (params.duration) {
|
||
const allowedDurations = Array.isArray(capabilities.durationOptions)
|
||
? capabilities.durationOptions
|
||
: [];
|
||
if (allowedDurations.length > 0) {
|
||
if (!allowedDurations.includes(params.duration)) {
|
||
throw new Error(`视频时长仅支持 ${allowedDurations.join(" / ")} 秒`);
|
||
}
|
||
} else {
|
||
const { min, max } = capabilities.durationRange;
|
||
if (params.duration < min || params.duration > max) {
|
||
throw new Error(`视频时长必须在 ${min}-${max} 秒之间`);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 检查参考图数量
|
||
if (
|
||
params.mode === "reference-images" &&
|
||
params.referenceImages &&
|
||
capabilities.maxReferenceImages
|
||
) {
|
||
if (params.referenceImages.length > capabilities.maxReferenceImages) {
|
||
throw new Error(
|
||
`参考图数量不能超过 ${capabilities.maxReferenceImages} 张`
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 构建视频生成选项
|
||
*/
|
||
function buildVideoOptions(
|
||
mode: VideoGenerationMode,
|
||
images: string[] | undefined,
|
||
prompt: string,
|
||
params: any,
|
||
modelId: string
|
||
): VideoGenerationOptions {
|
||
const options: VideoGenerationOptions = {
|
||
prompt,
|
||
model: modelId,
|
||
mode,
|
||
resolution: params.resolution || "720p",
|
||
ratio: params.ratio || "16:9",
|
||
duration: params.duration || 5,
|
||
watermark: params.watermark ?? true,
|
||
cameraFixed: params.cameraFixed ?? false,
|
||
fps: 24,
|
||
};
|
||
|
||
// 根据模式处理图片输入
|
||
if (mode === "first-frame" && images && images.length > 0) {
|
||
options.firstFrameImage = images[0];
|
||
} else if (mode === "first-last-frame" && images && images.length >= 2) {
|
||
options.firstFrameImage = images[0];
|
||
options.lastFrameImage = images[1];
|
||
} else if (mode === "reference-images" && images && images.length > 0) {
|
||
options.referenceImages = images.slice(0, 4);
|
||
}
|
||
|
||
return options;
|
||
}
|
||
|
||
async function resolvePreferredVideoParentModelId(preferredModelRef?: string) {
|
||
const normalizedRef = String(preferredModelRef || "").trim();
|
||
if (!normalizedRef) return undefined;
|
||
|
||
const models = await getEnabledAiModelsWithProvider();
|
||
const matched = models.find(
|
||
(item) =>
|
||
isDirectExecutableVideoModel(item) &&
|
||
(item.model.id === normalizedRef || item.model.modelId === normalizedRef),
|
||
);
|
||
|
||
if (!matched) return undefined;
|
||
if (matched.model.isSubModel && matched.model.parentModelId) {
|
||
return matched.model.parentModelId;
|
||
}
|
||
return matched.model.id;
|
||
}
|
||
|
||
function resolveAgnesVideoSize(
|
||
resolution: VideoGenerationOptions["resolution"],
|
||
ratio: VideoGenerationOptions["ratio"],
|
||
) {
|
||
const longSideByResolution: Record<string, number> = {
|
||
"480p": 854,
|
||
"720p": 1280,
|
||
"1080p": 1920,
|
||
};
|
||
const longSide = longSideByResolution[resolution || "720p"] || 1280;
|
||
const ratios: Record<string, [number, number]> = {
|
||
"16:9": [16, 9],
|
||
"4:3": [4, 3],
|
||
"1:1": [1, 1],
|
||
"3:4": [3, 4],
|
||
"9:16": [9, 16],
|
||
"21:9": [21, 9],
|
||
adaptive: [16, 9],
|
||
};
|
||
const [rw, rh] = ratios[ratio || "16:9"] || ratios["16:9"];
|
||
const landscape = rw >= rh;
|
||
const width = landscape ? longSide : Math.round((longSide * rw) / rh);
|
||
const height = landscape ? Math.round((longSide * rh) / rw) : longSide;
|
||
return {
|
||
width: Math.max(256, Math.round(width / 8) * 8),
|
||
height: Math.max(256, Math.round(height / 8) * 8),
|
||
};
|
||
}
|
||
|
||
function resolveAgnesFrameCount(duration?: number) {
|
||
const seconds = Math.max(1, Math.min(Number(duration) || 5, 18));
|
||
const rawFrames = Math.min(441, Math.max(9, Math.round(seconds * 24)));
|
||
return Math.min(441, Math.max(9, Math.floor((rawFrames - 1) / 8) * 8 + 1));
|
||
}
|
||
|
||
function buildAgnesResultUrl(baseUrl: string, videoId: string, modelId: string) {
|
||
const normalizedBase = String(baseUrl || "https://apihub.agnes-ai.com/v1").replace(/\/+$/, "");
|
||
const root = normalizedBase.replace(/\/v1$/i, "");
|
||
const url = new URL(`${root}/agnesapi`);
|
||
url.searchParams.set("video_id", videoId);
|
||
url.searchParams.set("model_name", modelId);
|
||
return url.toString();
|
||
}
|
||
|
||
async function executeAgnesVideoGeneration(params: {
|
||
apiKey: string;
|
||
baseUrl: string;
|
||
options: VideoGenerationOptions;
|
||
}) {
|
||
const baseUrl = String(params.baseUrl || "https://apihub.agnes-ai.com/v1").replace(/\/+$/, "");
|
||
const createUrl = `${baseUrl}/videos`;
|
||
const { width, height } = resolveAgnesVideoSize(params.options.resolution, params.options.ratio);
|
||
const body: Record<string, unknown> = {
|
||
model: params.options.model,
|
||
prompt: params.options.prompt,
|
||
width,
|
||
height,
|
||
num_frames: resolveAgnesFrameCount(params.options.duration),
|
||
frame_rate: 24,
|
||
};
|
||
if (params.options.seed !== undefined) body.seed = params.options.seed;
|
||
if (params.options.firstFrameImage && params.options.lastFrameImage) {
|
||
body.extra_body = {
|
||
image: [params.options.firstFrameImage, params.options.lastFrameImage],
|
||
mode: "keyframes",
|
||
};
|
||
} else if (params.options.referenceImages?.length) {
|
||
body.extra_body = {
|
||
image: params.options.referenceImages.slice(0, 4),
|
||
mode: "keyframes",
|
||
};
|
||
} else if (params.options.firstFrameImage) {
|
||
body.image = params.options.firstFrameImage;
|
||
body.mode = "ti2vid";
|
||
}
|
||
|
||
const createResponse = await fetch(createUrl, {
|
||
method: "POST",
|
||
headers: {
|
||
Authorization: `Bearer ${params.apiKey}`,
|
||
"Content-Type": "application/json",
|
||
},
|
||
body: JSON.stringify(body),
|
||
});
|
||
const createText = await createResponse.text();
|
||
let createData: Record<string, any> = {};
|
||
try {
|
||
createData = createText ? JSON.parse(createText) : {};
|
||
} catch {
|
||
createData = { raw: createText };
|
||
}
|
||
if (!createResponse.ok) {
|
||
throw new Error(
|
||
`Agnes 视频任务创建失败: ${createData?.error?.message || createData?.message || createResponse.statusText}`,
|
||
);
|
||
}
|
||
|
||
const videoId = String(createData.video_id || createData.videoId || createData.id || createData.task_id || "").trim();
|
||
const taskId = String(createData.task_id || createData.id || "").trim();
|
||
if (!videoId && !taskId) {
|
||
throw new Error("Agnes 视频任务创建成功但没有返回 video_id/task_id");
|
||
}
|
||
|
||
const startedAt = Date.now();
|
||
const timeoutMs = 285_000;
|
||
let lastData = createData;
|
||
while (Date.now() - startedAt < timeoutMs) {
|
||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||
const resultUrl = videoId
|
||
? buildAgnesResultUrl(baseUrl, videoId, params.options.model)
|
||
: `${baseUrl}/videos/${encodeURIComponent(taskId)}`;
|
||
const resultResponse = await fetch(resultUrl, {
|
||
headers: { Authorization: `Bearer ${params.apiKey}` },
|
||
});
|
||
const resultText = await resultResponse.text();
|
||
let resultData: Record<string, any> = {};
|
||
try {
|
||
resultData = resultText ? JSON.parse(resultText) : {};
|
||
} catch {
|
||
resultData = { raw: resultText };
|
||
}
|
||
if (!resultResponse.ok) {
|
||
throw new Error(
|
||
`Agnes 视频结果查询失败: ${resultData?.error?.message || resultData?.message || resultResponse.statusText}`,
|
||
);
|
||
}
|
||
lastData = resultData;
|
||
const status = String(resultData.status || "").toLowerCase();
|
||
if (status === "completed" && resultData.url) {
|
||
return {
|
||
url: String(resultData.url),
|
||
metadata: {
|
||
...resultData,
|
||
createTask: createData,
|
||
},
|
||
};
|
||
}
|
||
if (status === "failed") {
|
||
throw new Error(`Agnes 视频生成失败: ${resultData?.error?.message || resultData?.error || "任务失败"}`);
|
||
}
|
||
}
|
||
|
||
throw new Error(
|
||
`Agnes 视频生成超时: ${String(lastData.status || "unknown")}`,
|
||
);
|
||
}
|
||
|
||
export async function executeDirectVideoGeneration(
|
||
params: ExecuteDirectVideoGenerationParams,
|
||
): Promise<ExecuteDirectVideoGenerationResult> {
|
||
const prompt = String(params.prompt || "").trim();
|
||
if (!prompt) {
|
||
throw new Error("缺少视频生成提示词");
|
||
}
|
||
|
||
const preferredParentModelId = await resolvePreferredVideoParentModelId(
|
||
params.preferredModelRef,
|
||
);
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const fallbackParentModel =
|
||
allModels.find(
|
||
(item) =>
|
||
isVolcanoVideoModel(item) &&
|
||
!item.model.isSubModel &&
|
||
item.model.modelId === "volcano-video-master",
|
||
) ||
|
||
allModels.find(
|
||
(item) =>
|
||
isDirectExecutableVideoModel(item) &&
|
||
!item.model.isSubModel,
|
||
);
|
||
|
||
const parentModelId =
|
||
preferredParentModelId ||
|
||
fallbackParentModel?.model.id ||
|
||
allModels.find(
|
||
(item) =>
|
||
isDirectExecutableVideoModel(item) &&
|
||
item.model.isEnabled &&
|
||
item.model.isSubModel &&
|
||
item.model.parentModelId,
|
||
)?.model.parentModelId ||
|
||
undefined;
|
||
|
||
const generationParams = {
|
||
mode: params.mode,
|
||
resolution: params.resolution || "720p",
|
||
ratio: params.ratio || "16:9",
|
||
duration: params.duration || 5,
|
||
watermark: params.watermark ?? true,
|
||
cameraFixed: params.cameraFixed ?? false,
|
||
seed: params.seed,
|
||
returnLastFrame: params.returnLastFrame ?? false,
|
||
referenceImages: params.images,
|
||
} satisfies Partial<VideoGenerationOptions>;
|
||
|
||
let selectedModelConfig: { model: any; capabilities: VideoModelCapabilities };
|
||
try {
|
||
selectedModelConfig = await getCompatibleModel(
|
||
params.mode,
|
||
parentModelId,
|
||
generationParams,
|
||
);
|
||
} catch (preferredModelError) {
|
||
if (parentModelId) {
|
||
if (
|
||
preferredModelError instanceof Error &&
|
||
/不支持|没有模型支持/.test(preferredModelError.message)
|
||
) {
|
||
throw preferredModelError;
|
||
}
|
||
console.warn(
|
||
"[Generate Video Tool] Preferred parent model selection failed, fallback to global compatible model:",
|
||
preferredModelError,
|
||
);
|
||
selectedModelConfig = await getCompatibleModel(
|
||
params.mode,
|
||
undefined,
|
||
generationParams,
|
||
);
|
||
} else {
|
||
throw preferredModelError;
|
||
}
|
||
}
|
||
|
||
const { model, capabilities } = selectedModelConfig;
|
||
|
||
validateParamsAgainstCapabilities(generationParams, capabilities);
|
||
|
||
const options = buildVideoOptions(
|
||
params.mode,
|
||
params.images,
|
||
prompt,
|
||
generationParams,
|
||
model.model.modelId,
|
||
);
|
||
|
||
let videoMetadata: Record<string, any> | null = null;
|
||
if (isAgnesVideoModel(model)) {
|
||
const agnesResult = await executeAgnesVideoGeneration({
|
||
apiKey: model.provider.apiKey,
|
||
baseUrl: model.provider.baseUrl,
|
||
options,
|
||
});
|
||
videoMetadata = {
|
||
...(agnesResult.metadata || {}),
|
||
temporaryVideoUrl: agnesResult.url,
|
||
};
|
||
} else if (isAtlasCloudVideoModel(model)) {
|
||
const atlasResult = await executeAtlasCloudVideoGeneration({
|
||
apiKey: model.provider.apiKey,
|
||
baseURL: model.provider.baseUrl,
|
||
modelId: model.model.modelId,
|
||
modelConfig: model.model.imageGenConfig || {},
|
||
options,
|
||
});
|
||
videoMetadata = {
|
||
...(atlasResult.metadata || {}),
|
||
temporaryVideoUrl: atlasResult.url,
|
||
};
|
||
} else {
|
||
for await (const chunk of doGenerateStream(model.provider.apiKey, model.provider.baseUrl, options)) {
|
||
await params.onProgress?.(chunk);
|
||
const metadataMatch = chunk.match(/<!-- VIDEO_METADATA:(.*?) -->/);
|
||
if (metadataMatch) {
|
||
try {
|
||
videoMetadata = JSON.parse(metadataMatch[1]);
|
||
} catch (error) {
|
||
console.error("[Generate Video Tool] 解析元数据失败:", error);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
return {
|
||
success: true,
|
||
videoUrl: videoMetadata?.temporaryVideoUrl || undefined,
|
||
metadata: videoMetadata,
|
||
modelDisplayName: String(model.model.displayName || model.model.modelId || "视频模型"),
|
||
modelId: String(model.model.modelId || model.model.id),
|
||
modelRef: String(model.model.id || model.model.modelId),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 视频生成工具
|
||
*/
|
||
export function generateVideo({ dataStream }: { dataStream: any }) {
|
||
return tool({
|
||
description:
|
||
"生成视频，支持文生视频、图生视频（首帧/首尾帧/参考图）等多种模式",
|
||
inputSchema: z.object({
|
||
prompt: z.string().describe("视频生成提示词，描述视频内容、镜头运动等"),
|
||
mode: z
|
||
.enum(["text-to-video", "first-frame", "first-last-frame", "reference-images"])
|
||
.describe("生成模式：文生视频/图生视频-首帧/首尾帧/参考图"),
|
||
images: z
|
||
.array(z.string())
|
||
.optional()
|
||
.describe("图片 URL 数组（根据模式不同，需要 0-4 张）"),
|
||
resolution: z
|
||
.enum(["480p", "720p", "1080p"])
|
||
.default("720p")
|
||
.describe("视频分辨率"),
|
||
ratio: z
|
||
.enum(["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"])
|
||
.default("16:9")
|
||
.describe("视频宽高比"),
|
||
duration: z
|
||
.number()
|
||
.min(1)
|
||
.max(600)
|
||
.default(5)
|
||
.describe("视频时长（秒）。优先使用当前视频模型配置的秒数档位，服务端会按模型能力动态校验。"),
|
||
watermark: z.boolean().default(true).describe("是否添加水印"),
|
||
cameraFixed: z.boolean().default(false).describe("是否固定摄像头"),
|
||
seed: z.number().optional().describe("随机种子（可选）"),
|
||
returnLastFrame: z
|
||
.boolean()
|
||
.default(false)
|
||
.describe("是否返回尾帧（用于连续生成）"),
|
||
}),
|
||
execute: async ({ prompt, mode, images, resolution, ratio, duration, watermark, cameraFixed, seed, returnLastFrame }) => {
|
||
try {
|
||
console.log("[Generate Video Tool] 🎬 开始生成视频");
|
||
console.log("[Generate Video Tool] 模式:", mode);
|
||
console.log("[Generate Video Tool] 提示词:", prompt);
|
||
|
||
dataStream.writeData({
|
||
type: "video-generation-start",
|
||
data: {
|
||
mode,
|
||
model: "智能选择中",
|
||
},
|
||
});
|
||
|
||
const result = await executeDirectVideoGeneration({
|
||
prompt,
|
||
mode,
|
||
images,
|
||
resolution,
|
||
ratio,
|
||
duration,
|
||
watermark,
|
||
cameraFixed,
|
||
seed,
|
||
returnLastFrame,
|
||
onProgress: async (chunk) => {
|
||
dataStream.writeMessageAnnotation({
|
||
type: "video-progress",
|
||
content: chunk,
|
||
});
|
||
},
|
||
});
|
||
|
||
console.log("[Generate Video Tool] ✅ 视频生成完成");
|
||
|
||
return {
|
||
success: true,
|
||
videoUrl: result.videoUrl,
|
||
metadata: result.metadata,
|
||
modelDisplayName: result.modelDisplayName,
|
||
modelId: result.modelId,
|
||
modelRef: result.modelRef,
|
||
};
|
||
} catch (error: any) {
|
||
console.error("[Generate Video Tool] ❌ 生成失败:", error);
|
||
dataStream.writeMessageAnnotation({
|
||
type: "video-error",
|
||
content: `视频生成失败：${error.message}`,
|
||
});
|
||
throw error;
|
||
}
|
||
},
|
||
});
|
||
}
|