983 lines
34 KiB
TypeScript
983 lines
34 KiB
TypeScript
import {
|
||
bindVideoTaskLifecycle,
|
||
ProviderTaskFailedError,
|
||
ProviderTaskUncertainError,
|
||
type ProviderTaskLifecycle,
|
||
type VideoTaskLifecycle,
|
||
} from "@/lib/video-canvas/video-task-lifecycle";
|
||
import { executeVip98kExternalTask } from "@/lib/ai/providers/vip98k";
|
||
/**
|
||
* 视频生成工具
|
||
* 支持火山引擎视频生成的多种模式
|
||
*/
|
||
|
||
import { executeAliyunCanvasVideo } from "@/lib/ai/providers/aliyun-canvas-video";
|
||
import { getCanvasVideoModes, isExecutableCanvasVideoModel, supportsCanvasAudioVideo, validateCanvasVideoInputs, getCanvasVideoParameters } from "@/lib/video-canvas/model-matching";
|
||
import { tool } from "ai";
|
||
import { z } from "zod";
|
||
import { doGenerateStream } from "@/lib/ai/providers/volcano-video";
|
||
import { executeAtlasCloudVideoGeneration } from "@/lib/ai/providers/atlascloud";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { isVideoModelDescriptor } from "@/lib/ai/model-classification";
|
||
import {
|
||
getEnabledAiModelsWithProvider,
|
||
getSubModelsByParentModelId
|
||
} from "@/lib/db/queries";
|
||
import { getConfiguredVideoDurationOptions } from "@/lib/ai/video-model-config";
|
||
import type {
|
||
VideoGenerationMode,
|
||
VideoModelCapabilities,
|
||
VideoGenerationOptions,
|
||
} from "@/types/ai";
|
||
|
||
export interface ExecuteDirectVideoGenerationParams {
|
||
prompt: string;
|
||
mode: VideoGenerationMode;
|
||
images?: string[];
|
||
audioUrls?: string[];
|
||
generateAudio?: boolean;
|
||
resolution?: "480p" | "720p" | "1080p";
|
||
ratio?: "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "21:9" | "adaptive";
|
||
duration?: number;
|
||
watermark?: boolean;
|
||
cameraFixed?: boolean;
|
||
seed?: number;
|
||
returnLastFrame?: boolean;
|
||
preferredModelRef?: string;
|
||
onProgress?: (chunk: string) => void | Promise<void>;
|
||
taskLifecycle?: VideoTaskLifecycle;
|
||
}
|
||
|
||
export interface ExecuteDirectVideoGenerationResult {
|
||
success: boolean;
|
||
videoUrl?: string;
|
||
metadata?: Record<string, any> | null;
|
||
modelDisplayName: string;
|
||
modelId: string;
|
||
modelRef: string;
|
||
}
|
||
|
||
function inferSupportedModesFromModelId(modelId: string): VideoGenerationMode[] {
|
||
const normalized = modelId.toLowerCase();
|
||
|
||
if (normalized.includes("lite-t2v")) return ["text-to-video"];
|
||
if (normalized.includes("lite-i2v")) return ["first-frame", "first-last-frame", "reference-images"];
|
||
if (normalized.includes("pro-fast")) return ["text-to-video", "first-frame"];
|
||
if (normalized.includes("-pro-") || normalized.endsWith("-pro")) {
|
||
return ["text-to-video", "first-frame", "first-last-frame"];
|
||
}
|
||
|
||
return ["text-to-video", "first-frame", "first-last-frame", "reference-images"];
|
||
}
|
||
|
||
function extractVideoUrlFromGeneratedText(value: unknown) {
|
||
const text = String(value || "");
|
||
const srcMatch = text.match(/<video[^>]+src=["'](https?:\/\/[^"']+)["']/i);
|
||
if (srcMatch?.[1]) return srcMatch[1];
|
||
const markdownMatch = text.match(/\[[^\]]*\]\((https?:\/\/[^)\s]+)\)/i);
|
||
if (markdownMatch?.[1]) return markdownMatch[1];
|
||
const directMatch = text.match(/https?:\/\/[^\s<>"']+/i);
|
||
return directMatch?.[0]?.replace(/[),.;]+$/, "") || "";
|
||
}
|
||
|
||
function getSupportedModes(caps: Partial<VideoModelCapabilities> | undefined, modelId: string): VideoGenerationMode[] {
|
||
if (Array.isArray(caps?.supportedModes)) {
|
||
return getCanvasVideoModes({modelId, capabilities: {...caps, kind: "video"}});
|
||
}
|
||
return inferSupportedModesFromModelId(modelId);
|
||
}
|
||
|
||
function withConfiguredVideoCapabilities(
|
||
capabilities: Partial<VideoModelCapabilities> | undefined,
|
||
imageGenConfig?: { videoDurationOptions?: number[] | null } | null
|
||
): VideoModelCapabilities {
|
||
const baseCapabilities = {
|
||
supportedModes: [],
|
||
supportedResolutions: ["480p", "720p", "1080p"],
|
||
supportedRatios: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"],
|
||
durationRange: { min: 2, max: 12 },
|
||
maxReferenceImages: 4,
|
||
...(capabilities || {}),
|
||
} as VideoModelCapabilities;
|
||
|
||
const durationOptions = getConfiguredVideoDurationOptions(imageGenConfig);
|
||
if (durationOptions.length === 0) {
|
||
return baseCapabilities;
|
||
}
|
||
|
||
return {
|
||
...baseCapabilities,
|
||
durationOptions,
|
||
durationRange: {
|
||
min: durationOptions[0],
|
||
max: durationOptions[durationOptions.length - 1],
|
||
},
|
||
};
|
||
}
|
||
|
||
function isAgnesVideoModel(item: any) {
|
||
const capabilities = item?.model?.capabilities;
|
||
const provider = capabilities && typeof capabilities === "object"
|
||
? String((capabilities as Record<string, unknown>).provider || "")
|
||
: "";
|
||
const kind = capabilities && typeof capabilities === "object"
|
||
? String((capabilities as Record<string, unknown>).kind || "")
|
||
: "";
|
||
const modelId = String(item?.model?.modelId || "").toLowerCase();
|
||
return (
|
||
item?.provider?.type === "custom" &&
|
||
(kind === "video" && (provider === "agnes-ai" || modelId.startsWith("agnes-video")))
|
||
);
|
||
}
|
||
|
||
function isVolcanoVideoModel(item: any) {
|
||
return item?.provider?.type === "volcano-video" || item?.provider?.type === "doubao-video";
|
||
}
|
||
|
||
function isAtlasCloudVideoModel(item: any) {
|
||
const capabilities = item?.model?.capabilities;
|
||
return (
|
||
item?.provider?.type === "atlascloud" &&
|
||
capabilities &&
|
||
typeof capabilities === "object" &&
|
||
String((capabilities as Record<string, unknown>).mediaType || "") === "video"
|
||
);
|
||
}
|
||
|
||
function isVip98kVideoModel(item: any) {
|
||
return (
|
||
item?.provider?.type === "vip98k" &&
|
||
isVideoModelDescriptor({
|
||
providerType: item.provider.type,
|
||
modelId: item.model?.modelId,
|
||
capabilities: item.model?.capabilities,
|
||
supportsImageGen: item.model?.supportsImageGen,
|
||
})
|
||
);
|
||
}
|
||
|
||
function isDirectExecutableVideoModel(item: any) {
|
||
return isExecutableCanvasVideoModel({...item.model, providerType: item.provider?.type});
|
||
}
|
||
|
||
/**
|
||
* 🆕 智能选择最佳子模型
|
||
* 根据用户选择的模式和参数，从子模型列表中选择最合适的模型
|
||
*/
|
||
async function selectBestSubModel(
|
||
parentModelId: string,
|
||
mode: VideoGenerationMode,
|
||
params: Partial<VideoGenerationOptions>
|
||
): Promise<{ model: any; capabilities: VideoModelCapabilities }> {
|
||
// 获取所有子模型
|
||
const subModels = await getSubModelsByParentModelId(parentModelId);
|
||
|
||
if (subModels.length === 0) {
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const directModel = allModels.find(
|
||
(item) =>
|
||
item.model.id === parentModelId &&
|
||
isDirectExecutableVideoModel(item),
|
||
);
|
||
|
||
if (!directModel) {
|
||
throw new Error("没有可用的视频生成子模型");
|
||
}
|
||
|
||
const directCapabilities = withConfiguredVideoCapabilities(
|
||
directModel.model.capabilities as VideoModelCapabilities | undefined,
|
||
(directModel.model as any).imageGenConfig
|
||
);
|
||
const supportedModes = getSupportedModes(directCapabilities, directModel.model.modelId);
|
||
if (!supportedModes.includes(mode)) {
|
||
throw new Error(`模型 ${directModel.model.displayName} 不支持 ${mode} 模式`);
|
||
}
|
||
|
||
return {
|
||
model: directModel,
|
||
capabilities: {
|
||
...directCapabilities,
|
||
supportedModes,
|
||
} as VideoModelCapabilities,
|
||
};
|
||
}
|
||
|
||
console.log(`[Smart Model Selection] 找到 ${subModels.length} 个子模型`);
|
||
|
||
// 筛选支持指定模式的子模型
|
||
const compatibleModels = subModels.filter((m) => {
|
||
const caps = withConfiguredVideoCapabilities(
|
||
m.model.capabilities as VideoModelCapabilities,
|
||
(m.model as any).imageGenConfig
|
||
);
|
||
const supportsMode = getSupportedModes(caps, m.model.modelId).includes(mode);
|
||
|
||
// 检查分辨率支持（如果用户指定了分辨率）
|
||
const supportedResolutions = caps.supportedResolutions || ["480p", "720p", "1080p"];
|
||
const supportsResolution = !params.resolution || supportedResolutions.includes(params.resolution);
|
||
|
||
if (params.referenceAudios?.length && !supportsCanvasAudioVideo({...m.model,providerType:m.provider.type})) return false;
|
||
if (!isDirectExecutableVideoModel(m) || !supportsMode || !supportsResolution) return false;
|
||
try { validateParamsAgainstCapabilities({...params, mode}, caps); return true; } catch { return false; }
|
||
});
|
||
|
||
if (compatibleModels.length === 0) {
|
||
throw new Error(`没有模型支持 ${mode} 模式和指定的参数`);
|
||
}
|
||
|
||
console.log(`[Smart Model Selection] ${compatibleModels.length} 个模型支持 ${mode} 模式`);
|
||
|
||
// 🎯 智能优先级排序
|
||
const priority: Record<string, number> = {
|
||
"doubao-seedance-1-0-pro-250528": 10, // 最高优先级（全功能）
|
||
"doubao-seedance-1-0-pro-fast-251015": 8, // 快速模式
|
||
"doubao-seedance-1-0-lite-i2v-250428": 6, // 图生视频轻量版
|
||
"doubao-seedance-1-0-lite-t2v-250428": 5, // 文生视频轻量版
|
||
};
|
||
|
||
// 选择最佳模型
|
||
const bestModel = compatibleModels.sort((a, b) => {
|
||
const priorityA = priority[a.model.modelId] || 0;
|
||
const priorityB = priority[b.model.modelId] || 0;
|
||
return priorityB - priorityA;
|
||
})[0];
|
||
|
||
console.log(`[Smart Model Selection] ✅ 选择模型: ${bestModel.model.displayName} (${bestModel.model.modelId})`);
|
||
|
||
return {
|
||
model: bestModel,
|
||
capabilities: withConfiguredVideoCapabilities(
|
||
bestModel.model.capabilities as VideoModelCapabilities,
|
||
(bestModel.model as any).imageGenConfig
|
||
),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 获取支持指定模式的视频模型（兼容旧逻辑）
|
||
*/
|
||
async function getCompatibleModel(
|
||
mode: VideoGenerationMode,
|
||
parentModelId?: string,
|
||
params?: Partial<VideoGenerationOptions>
|
||
): Promise<{ model: any; capabilities: VideoModelCapabilities }> {
|
||
// 🆕 如果提供了parentModelId，使用智能子模型选择
|
||
if (parentModelId) {
|
||
return selectBestSubModel(parentModelId, mode, params || {});
|
||
}
|
||
|
||
// 兼容旧逻辑：直接从所有模型中选择（用于向后兼容）
|
||
const models = await getEnabledAiModelsWithProvider();
|
||
|
||
const volcanoVideoModels = models.filter(
|
||
(m) =>
|
||
isDirectExecutableVideoModel(m) &&
|
||
m.model.isEnabled &&
|
||
m.model.capabilities
|
||
);
|
||
|
||
if (volcanoVideoModels.length === 0) {
|
||
throw new Error("没有启用的可执行视频模型");
|
||
}
|
||
|
||
const compatibleModels = volcanoVideoModels.filter((m) => {
|
||
const caps = withConfiguredVideoCapabilities(
|
||
m.model.capabilities as VideoModelCapabilities,
|
||
(m.model as any).imageGenConfig
|
||
);
|
||
if (!getSupportedModes(caps, m.model.modelId).includes(mode)) return false;
|
||
if (params?.referenceAudios?.length && !supportsCanvasAudioVideo({...m.model, providerType:m.provider.type})) return false;
|
||
try { validateParamsAgainstCapabilities({...params, mode}, caps); return true; } catch { return false; }
|
||
});
|
||
|
||
if (compatibleModels.length === 0) {
|
||
throw new Error(`没有模型支持 ${mode} 模式`);
|
||
}
|
||
|
||
const priority: Record<string, number> = {
|
||
"doubao-seedance-1-0-pro-250528": 10,
|
||
"doubao-seedance-1-0-pro-fast-251015": 8,
|
||
"doubao-seedance-1-0-lite-i2v-250428": 6,
|
||
"doubao-seedance-1-0-lite-t2v-250428": 5,
|
||
};
|
||
|
||
const bestModel = compatibleModels.sort((a, b) => {
|
||
const priorityA = priority[a.model.modelId] || 0;
|
||
const priorityB = priority[b.model.modelId] || 0;
|
||
return priorityB - priorityA;
|
||
})[0];
|
||
|
||
return {
|
||
model: bestModel,
|
||
capabilities: withConfiguredVideoCapabilities(
|
||
bestModel.model.capabilities as VideoModelCapabilities,
|
||
(bestModel.model as any).imageGenConfig
|
||
),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 验证参数是否符合模型能力
|
||
*/
|
||
function validateParamsAgainstCapabilities(
|
||
params: Partial<VideoGenerationOptions>,
|
||
capabilities: VideoModelCapabilities
|
||
): void {
|
||
const scoped = getCanvasVideoParameters({ capabilities }, params.mode);
|
||
if (params.ratio && scoped.ratios.length && !scoped.ratios.includes(params.ratio)) throw new Error(`当前模式不支持 ${params.ratio} 比例`);
|
||
if (params.resolution && scoped.resolutions.length && !scoped.resolutions.includes(params.resolution)) throw new Error(`当前模式不支持 ${params.resolution} 分辨率`);
|
||
if (params.duration && scoped.durations.length && !scoped.durations.includes(params.duration)) throw new Error(`当前模式视频时长仅支持 ${scoped.durations.join(" / ")} 秒`);
|
||
// 检查分辨率
|
||
if (
|
||
params.resolution &&
|
||
capabilities.supportedResolutions.length > 0 &&
|
||
!capabilities.supportedResolutions.includes(params.resolution)
|
||
) {
|
||
throw new Error(
|
||
`模型不支持 ${params.resolution} 分辨率，支持的分辨率：${capabilities.supportedResolutions.join(", ")}`
|
||
);
|
||
}
|
||
|
||
// 检查宽高比
|
||
if (
|
||
params.ratio &&
|
||
capabilities.supportedRatios.length > 0 &&
|
||
!capabilities.supportedRatios.includes(params.ratio)
|
||
) {
|
||
throw new Error(
|
||
`模型不支持 ${params.ratio} 宽高比，支持的宽高比：${capabilities.supportedRatios.join(", ")}`
|
||
);
|
||
}
|
||
|
||
// 检查时长范围
|
||
if (params.duration) {
|
||
const allowedDurations = Array.isArray(capabilities.durationOptions)
|
||
? capabilities.durationOptions
|
||
: [];
|
||
if (allowedDurations.length > 0) {
|
||
if (!allowedDurations.includes(params.duration)) {
|
||
throw new Error(`视频时长仅支持 ${allowedDurations.join(" / ")} 秒`);
|
||
}
|
||
} else {
|
||
const { min, max } = capabilities.durationRange;
|
||
if (params.duration < min || params.duration > max) {
|
||
throw new Error(`视频时长必须在 ${min}-${max} 秒之间`);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 检查参考图数量
|
||
if (
|
||
params.mode === "reference-images" &&
|
||
params.referenceImages &&
|
||
capabilities.maxReferenceImages
|
||
) {
|
||
if (params.referenceImages.length > capabilities.maxReferenceImages) {
|
||
throw new Error(
|
||
`参考图数量不能超过 ${capabilities.maxReferenceImages} 张`
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 构建视频生成选项
|
||
*/
|
||
function buildVideoOptions(
|
||
mode: VideoGenerationMode,
|
||
images: string[] | undefined,
|
||
prompt: string,
|
||
params: any,
|
||
modelId: string
|
||
): VideoGenerationOptions {
|
||
const options: VideoGenerationOptions = {
|
||
prompt,
|
||
model: modelId,
|
||
mode,
|
||
resolution: params.resolution || "720p",
|
||
ratio: params.ratio || "16:9",
|
||
duration: params.duration || 5,
|
||
watermark: params.watermark ?? true,
|
||
cameraFixed: params.cameraFixed ?? false,
|
||
fps: 24,
|
||
seed: params.seed,
|
||
returnLastFrame: params.returnLastFrame,
|
||
referenceAudios: params.referenceAudios,
|
||
generateAudio: params.generateAudio,
|
||
};
|
||
|
||
// 根据模式处理图片输入
|
||
if (mode === "first-frame" && images && images.length > 0) {
|
||
options.firstFrameImage = images[0];
|
||
} else if (mode === "first-last-frame" && images && images.length >= 2) {
|
||
options.firstFrameImage = images[0];
|
||
options.lastFrameImage = images[1];
|
||
} else if (mode === "reference-images" && images && images.length > 0) {
|
||
options.referenceImages = images;
|
||
}
|
||
|
||
return options;
|
||
}
|
||
|
||
async function resolvePreferredVideoParentModelId(preferredModelRef?: string) {
|
||
const normalizedRef = String(preferredModelRef || "").trim();
|
||
if (!normalizedRef || normalizedRef === "auto") return undefined;
|
||
|
||
const models = await getEnabledAiModelsWithProvider();
|
||
const matched = models.find(
|
||
(item) =>
|
||
isDirectExecutableVideoModel(item) &&
|
||
(item.model.id === normalizedRef || item.model.modelId === normalizedRef),
|
||
);
|
||
|
||
if (!matched) throw new Error("所选视频模型不可用，请重新选择视频模型");
|
||
return matched.model.id;
|
||
}
|
||
|
||
function resolveAgnesVideoSize(
|
||
resolution: VideoGenerationOptions["resolution"],
|
||
ratio: VideoGenerationOptions["ratio"],
|
||
) {
|
||
const longSideByResolution: Record<string, number> = {
|
||
"480p": 854,
|
||
"720p": 1280,
|
||
"1080p": 1920,
|
||
};
|
||
const longSide = longSideByResolution[resolution || "720p"] || 1280;
|
||
const ratios: Record<string, [number, number]> = {
|
||
"16:9": [16, 9],
|
||
"4:3": [4, 3],
|
||
"1:1": [1, 1],
|
||
"3:4": [3, 4],
|
||
"9:16": [9, 16],
|
||
"21:9": [21, 9],
|
||
adaptive: [16, 9],
|
||
};
|
||
const [rw, rh] = ratios[ratio || "16:9"] || ratios["16:9"];
|
||
const landscape = rw >= rh;
|
||
const width = landscape ? longSide : Math.round((longSide * rw) / rh);
|
||
const height = landscape ? Math.round((longSide * rh) / rw) : longSide;
|
||
return {
|
||
width: Math.max(256, Math.round(width / 8) * 8),
|
||
height: Math.max(256, Math.round(height / 8) * 8),
|
||
};
|
||
}
|
||
|
||
function resolveAgnesFrameCount(duration?: number) {
|
||
const seconds = Math.max(1, Math.min(Number(duration) || 5, 18));
|
||
const rawFrames = Math.min(441, Math.max(9, Math.round(seconds * 24)));
|
||
return Math.min(441, Math.max(9, Math.floor((rawFrames - 1) / 8) * 8 + 1));
|
||
}
|
||
|
||
function buildAgnesResultUrl(baseUrl: string, videoId: string, modelId: string) {
|
||
const normalizedBase = String(baseUrl || "https://apihub.agnes-ai.com/v1").replace(/\/+$/, "");
|
||
const root = normalizedBase.replace(/\/v1$/i, "");
|
||
const url = new URL(`${root}/agnesapi`);
|
||
url.searchParams.set("video_id", videoId);
|
||
url.searchParams.set("model_name", modelId);
|
||
return url.toString();
|
||
}
|
||
|
||
async function executeAgnesVideoGeneration(params: {
|
||
apiKey: string;
|
||
baseUrl: string;
|
||
options: VideoGenerationOptions;
|
||
lifecycle?: ProviderTaskLifecycle;
|
||
}) {
|
||
const baseUrl = String(params.baseUrl || "https://apihub.agnes-ai.com/v1").replace(/\/+$/, "");
|
||
const createUrl = `${baseUrl}/videos`;
|
||
const { width, height } = resolveAgnesVideoSize(params.options.resolution, params.options.ratio);
|
||
const body: Record<string, unknown> = {
|
||
model: params.options.model,
|
||
prompt: params.options.prompt,
|
||
width,
|
||
height,
|
||
num_frames: resolveAgnesFrameCount(params.options.duration),
|
||
frame_rate: 24,
|
||
};
|
||
if (params.options.seed !== undefined) body.seed = params.options.seed;
|
||
if (params.options.firstFrameImage && params.options.lastFrameImage) {
|
||
body.extra_body = {
|
||
image: [params.options.firstFrameImage, params.options.lastFrameImage],
|
||
mode: "keyframes",
|
||
};
|
||
} else if (params.options.referenceImages?.length) {
|
||
body.extra_body = {
|
||
image: params.options.referenceImages.slice(0, 4),
|
||
mode: "keyframes",
|
||
};
|
||
} else if (params.options.firstFrameImage) {
|
||
body.image = params.options.firstFrameImage;
|
||
body.mode = "ti2vid";
|
||
}
|
||
|
||
await params.lifecycle?.assertActive();
|
||
const createResponse = params.lifecycle?.resume
|
||
? Response.json({
|
||
video_id: params.lifecycle.resume.videoId,
|
||
task_id: params.lifecycle.resume.taskId,
|
||
})
|
||
: await fetch(createUrl, {
|
||
method: "POST",
|
||
headers: {
|
||
Authorization: `Bearer ${params.apiKey}`,
|
||
"Content-Type": "application/json",
|
||
},
|
||
body: JSON.stringify(body),
|
||
});
|
||
const createText = await createResponse.text();
|
||
let createData: Record<string, any> = {};
|
||
try {
|
||
createData = createText ? JSON.parse(createText) : {};
|
||
} catch {
|
||
createData = { raw: createText };
|
||
}
|
||
if (!createResponse.ok) {
|
||
throw new Error(
|
||
`Agnes 视频任务创建失败: ${createData?.error?.message || createData?.message || createResponse.statusText}`,
|
||
);
|
||
}
|
||
|
||
const videoId = String(createData.video_id || createData.videoId || createData.id || createData.task_id || "").trim();
|
||
const taskId = String(createData.task_id || createData.id || "").trim();
|
||
if (!videoId && !taskId) {
|
||
throw new Error("Agnes 视频任务创建成功但没有返回 video_id/task_id");
|
||
}
|
||
|
||
if (!params.lifecycle?.resume)
|
||
await params.lifecycle?.accepted({ taskId: taskId || videoId, videoId });
|
||
const startedAt = Date.now();
|
||
const timeoutMs = 285_000;
|
||
let lastData = createData;
|
||
while (Date.now() - startedAt < timeoutMs) {
|
||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||
await params.lifecycle?.assertActive();
|
||
const resultUrl = videoId
|
||
? buildAgnesResultUrl(baseUrl, videoId, params.options.model)
|
||
: `${baseUrl}/videos/${encodeURIComponent(taskId)}`;
|
||
const resultResponse = await fetch(resultUrl, {
|
||
headers: { Authorization: `Bearer ${params.apiKey}` },
|
||
signal: AbortSignal.timeout(30000),
|
||
});
|
||
const resultText = await resultResponse.text();
|
||
let resultData: Record<string, any> = {};
|
||
try {
|
||
resultData = resultText ? JSON.parse(resultText) : {};
|
||
} catch {
|
||
resultData = { raw: resultText };
|
||
}
|
||
if (!resultResponse.ok) {
|
||
throw new Error(
|
||
`Agnes 视频结果查询失败: ${resultData?.error?.message || resultData?.message || resultResponse.statusText}`,
|
||
);
|
||
}
|
||
lastData = resultData;
|
||
const status = String(resultData.status || "").toLowerCase();
|
||
if (status === "completed" && resultData.url) {
|
||
return {
|
||
url: String(resultData.url),
|
||
metadata: {
|
||
...resultData,
|
||
createTask: createData,
|
||
},
|
||
};
|
||
}
|
||
if (status === "failed") {
|
||
throw new ProviderTaskFailedError(
|
||
`Agnes 视频生成失败: ${resultData?.error?.message || resultData?.error || "任务失败"}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
throw new Error(
|
||
`Agnes 视频生成超时: ${String(lastData.status || "unknown")}`,
|
||
);
|
||
}
|
||
|
||
async function resolvePreferredVip98kVideoModel(
|
||
preferredModelRef: string | undefined,
|
||
mode: VideoGenerationMode,
|
||
params: Partial<VideoGenerationOptions>,
|
||
) {
|
||
const normalizedRef = String(preferredModelRef || "").trim();
|
||
if (!normalizedRef) return null;
|
||
|
||
const models = await getEnabledAiModelsWithProvider();
|
||
const matched = models.find(
|
||
(item) =>
|
||
isVip98kVideoModel(item) &&
|
||
item.model.isEnabled &&
|
||
!item.model.isSubModel &&
|
||
(item.model.id === normalizedRef ||
|
||
item.model.modelId === normalizedRef ||
|
||
item.model.displayName === normalizedRef),
|
||
);
|
||
if (!matched) return null;
|
||
|
||
const capabilities = withConfiguredVideoCapabilities(
|
||
matched.model.capabilities as VideoModelCapabilities | undefined,
|
||
(matched.model as any).imageGenConfig,
|
||
);
|
||
const supportedModes = getSupportedModes(capabilities, matched.model.modelId);
|
||
if (!supportedModes.includes(mode)) {
|
||
throw new Error(`模型 ${matched.model.displayName} 不支持 ${mode} 模式`);
|
||
}
|
||
const resolvedCapabilities = {
|
||
...capabilities,
|
||
supportedModes,
|
||
} as VideoModelCapabilities;
|
||
validateParamsAgainstCapabilities(params, resolvedCapabilities);
|
||
return { model: matched, capabilities: resolvedCapabilities };
|
||
}
|
||
|
||
async function executeVip98kVideoGeneration(params: {
|
||
request: ExecuteDirectVideoGenerationParams;
|
||
prompt: string;
|
||
selected: { model: any; capabilities: VideoModelCapabilities };
|
||
generationParams: Partial<VideoGenerationOptions>;
|
||
}): Promise<ExecuteDirectVideoGenerationResult> {
|
||
const selected = params.selected.model;
|
||
params.generationParams.duration ??=
|
||
params.selected.capabilities.durationOptions?.[0] ||
|
||
params.selected.capabilities.durationRange?.min ||
|
||
5;
|
||
const lifecycle = bindVideoTaskLifecycle(params.request.taskLifecycle, selected, "vip98k");
|
||
if (lifecycle) {
|
||
const result = await executeVip98kExternalTask({
|
||
apiKey: selected.provider.apiKey,
|
||
baseURL: selected.provider.baseUrl,
|
||
modelId: selected.model.modelId,
|
||
kind: "video",
|
||
prompt: params.prompt,
|
||
imageUrls: params.request.images,
|
||
videoParams: params.generationParams as Record<string, unknown>,
|
||
modelConfig: {
|
||
imageGenConfig: selected.model.imageGenConfig || {},
|
||
capabilities: selected.model.capabilities,
|
||
},
|
||
lifecycle,
|
||
});
|
||
const videoUrl =
|
||
result.outputUrls.find((url) => /\.(mp4|webm|mov|m4v)(?:[?#].*)?$/i.test(url)) ||
|
||
result.outputUrls[0];
|
||
if (!videoUrl) throw new Error("VIP98K 视频任务完成但没有返回视频");
|
||
return {
|
||
success: true,
|
||
videoUrl,
|
||
metadata: { provider: "vip98k", taskId: result.taskId, temporaryVideoUrl: videoUrl },
|
||
modelId: String(selected.model.modelId),
|
||
modelRef: String(selected.model.id),
|
||
modelDisplayName: String(selected.model.displayName || selected.model.modelId),
|
||
};
|
||
}
|
||
const languageModel = await getLanguageModelById(selected.model.id);
|
||
if (!languageModel || typeof (languageModel as any).doGenerate !== "function") {
|
||
throw new Error("所选 VIP98K 视频模型初始化失败");
|
||
}
|
||
|
||
await params.request.onProgress?.(
|
||
`正在调用 ${selected.model.displayName || selected.model.modelId} 生成视频...`,
|
||
);
|
||
const imageParts = (params.request.images || []).map((url) => ({
|
||
type: "image_url" as const,
|
||
image_url: { url },
|
||
}));
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content: [{ type: "text", text: params.prompt }, ...imageParts],
|
||
},
|
||
],
|
||
overrideModelId: selected.model.modelId,
|
||
videoParams: {
|
||
mode: params.request.mode,
|
||
resolution: params.generationParams.resolution || "720p",
|
||
ratio: params.generationParams.ratio || "16:9",
|
||
duration: params.generationParams.duration || 8,
|
||
watermark: params.generationParams.watermark,
|
||
cameraFixed: params.generationParams.cameraFixed,
|
||
seed: params.generationParams.seed,
|
||
returnLastFrame: params.generationParams.returnLastFrame,
|
||
generateAudio: params.generationParams.generateAudio,
|
||
},
|
||
});
|
||
if (result?.finishReason === "error") {
|
||
throw new Error(String(result?.text || "VIP98K 视频生成失败"));
|
||
}
|
||
const videoUrl = extractVideoUrlFromGeneratedText(result?.text);
|
||
if (!videoUrl) throw new Error("VIP98K 视频模型未返回可用视频地址");
|
||
|
||
return {
|
||
success: true,
|
||
videoUrl,
|
||
metadata: {
|
||
provider: "vip98k",
|
||
temporaryVideoUrl: videoUrl,
|
||
mode: params.request.mode,
|
||
resolution: params.generationParams.resolution,
|
||
ratio: params.generationParams.ratio,
|
||
duration: params.generationParams.duration,
|
||
},
|
||
modelDisplayName: String(selected.model.displayName || selected.model.modelId),
|
||
modelId: String(selected.model.modelId || selected.model.id),
|
||
modelRef: String(selected.model.id || selected.model.modelId),
|
||
};
|
||
}
|
||
|
||
async function executeDirectVideoGenerationInternal(
|
||
params: ExecuteDirectVideoGenerationParams,
|
||
): Promise<ExecuteDirectVideoGenerationResult> {
|
||
const prompt = String(params.prompt || "").trim();
|
||
if (!prompt) {
|
||
throw new Error("缺少视频生成提示词");
|
||
}
|
||
|
||
if (params.taskLifecycle?.resume)
|
||
params = { ...params, preferredModelRef: params.taskLifecycle.resume.modelRef };
|
||
const preferredParentModelId = await resolvePreferredVideoParentModelId(params.preferredModelRef);
|
||
validateCanvasVideoInputs(params.mode, params.images);
|
||
const parentModelId = preferredParentModelId;
|
||
|
||
const generationParams = {
|
||
referenceAudios: params.audioUrls,
|
||
generateAudio: params.generateAudio,
|
||
mode: params.mode,
|
||
resolution: params.resolution || "720p",
|
||
ratio: params.ratio || "16:9",
|
||
duration: params.duration,
|
||
watermark: params.watermark ?? true,
|
||
cameraFixed: params.cameraFixed ?? false,
|
||
seed: params.seed,
|
||
returnLastFrame: params.returnLastFrame ?? false,
|
||
referenceImages: params.images,
|
||
} satisfies Partial<VideoGenerationOptions>;
|
||
|
||
const preferredVip98kModel = await resolvePreferredVip98kVideoModel(
|
||
params.preferredModelRef,
|
||
params.mode,
|
||
generationParams,
|
||
);
|
||
if (preferredVip98kModel) {
|
||
if (params.audioUrls?.length) throw new Error("所选模型不支持音频参考，请选择音频生视频模型");
|
||
return executeVip98kVideoGeneration({
|
||
request: params,
|
||
prompt,
|
||
selected: preferredVip98kModel,
|
||
generationParams,
|
||
});
|
||
}
|
||
|
||
const selectedModelConfig = await getCompatibleModel(params.mode, parentModelId, generationParams);
|
||
|
||
const { model, capabilities } = selectedModelConfig;
|
||
if (params.audioUrls?.length && !supportsCanvasAudioVideo({...model.model,providerType:model.provider.type})) throw new Error("当前没有可用的音频生视频模型");
|
||
generationParams.duration ??= capabilities.durationOptions?.[0] || capabilities.durationRange?.min || 5;
|
||
|
||
validateParamsAgainstCapabilities(generationParams, capabilities);
|
||
|
||
if (isVip98kVideoModel(model)) {
|
||
return executeVip98kVideoGeneration({request: params, prompt, selected: selectedModelConfig, generationParams});
|
||
}
|
||
|
||
const options = buildVideoOptions(
|
||
params.mode,
|
||
params.images,
|
||
prompt,
|
||
generationParams,
|
||
model.model.modelId,
|
||
);
|
||
|
||
let videoMetadata: Record<string, any> | null = null;
|
||
if (model.provider.type === "qianwen-video") {
|
||
const result = await executeAliyunCanvasVideo({
|
||
...params,
|
||
duration: generationParams.duration,
|
||
modelId: model.model.modelId,
|
||
apiKey: model.provider.apiKey,
|
||
baseURL: model.provider.baseUrl,
|
||
lifecycle: bindVideoTaskLifecycle(params.taskLifecycle, model, "aliyun"),
|
||
});
|
||
videoMetadata = { ...result.metadata, temporaryVideoUrl: result.url };
|
||
} else if (isAgnesVideoModel(model)) {
|
||
const agnesResult = await executeAgnesVideoGeneration({
|
||
apiKey: model.provider.apiKey,
|
||
baseUrl: model.provider.baseUrl,
|
||
options,
|
||
lifecycle: bindVideoTaskLifecycle(params.taskLifecycle, model, "agnes"),
|
||
});
|
||
videoMetadata = {
|
||
...(agnesResult.metadata || {}),
|
||
temporaryVideoUrl: agnesResult.url,
|
||
};
|
||
} else if (isAtlasCloudVideoModel(model)) {
|
||
const atlasResult = await executeAtlasCloudVideoGeneration({
|
||
apiKey: model.provider.apiKey,
|
||
baseURL: model.provider.baseUrl,
|
||
modelId: model.model.modelId,
|
||
modelConfig: model.model.imageGenConfig || {},
|
||
options,
|
||
lifecycle: bindVideoTaskLifecycle(params.taskLifecycle, model, "atlascloud"),
|
||
});
|
||
videoMetadata = {
|
||
...(atlasResult.metadata || {}),
|
||
temporaryVideoUrl: atlasResult.url,
|
||
};
|
||
} else {
|
||
for await (const chunk of doGenerateStream(
|
||
model.provider.apiKey,
|
||
model.provider.baseUrl,
|
||
options,
|
||
bindVideoTaskLifecycle(params.taskLifecycle, model, "volcano"),
|
||
)) {
|
||
await params.onProgress?.(chunk);
|
||
const metadataMatch = chunk.match(/<!-- VIDEO_METADATA:(.*?) -->/);
|
||
if (metadataMatch) {
|
||
try {
|
||
videoMetadata = JSON.parse(metadataMatch[1]);
|
||
} catch (error) {
|
||
console.error("[Generate Video Tool] 解析元数据失败:", error);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
return {
|
||
success: true,
|
||
videoUrl: videoMetadata?.temporaryVideoUrl || undefined,
|
||
metadata: videoMetadata,
|
||
modelDisplayName: String(model.model.displayName || model.model.modelId || "视频模型"),
|
||
modelId: String(model.model.modelId || model.model.id),
|
||
modelRef: String(model.model.id || model.model.modelId),
|
||
};
|
||
}
|
||
|
||
export async function executeDirectVideoGeneration(
|
||
params: ExecuteDirectVideoGenerationParams,
|
||
): Promise<ExecuteDirectVideoGenerationResult> {
|
||
let accepted = Boolean(params.taskLifecycle?.resume);
|
||
const lifecycle = params.taskLifecycle;
|
||
try {
|
||
return await executeDirectVideoGenerationInternal({
|
||
...params,
|
||
taskLifecycle: lifecycle
|
||
? {
|
||
...lifecycle,
|
||
save: async (checkpoint) => {
|
||
accepted = true;
|
||
await lifecycle.save(checkpoint);
|
||
},
|
||
}
|
||
: undefined,
|
||
});
|
||
} catch (error) {
|
||
const message = error instanceof Error ? error.message : String(error);
|
||
const unknownSubmission =
|
||
lifecycle &&
|
||
/fetch failed|network|socket|connection|timeout|timed out|abort|超时|未返回.*任务\s*ID|neither a task ID/i.test(
|
||
message,
|
||
);
|
||
if ((accepted || unknownSubmission) && !(error instanceof ProviderTaskFailedError)) {
|
||
throw new ProviderTaskUncertainError(
|
||
accepted
|
||
? message
|
||
: "视频提交连接中断，尚未取得任务编号，无法确认是否已受理；请先核对服务商记录，系统不会自动重新提交",
|
||
);
|
||
}
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 视频生成工具
|
||
*/
|
||
export function generateVideo({ dataStream }: { dataStream: any }) {
|
||
return tool({
|
||
description:
|
||
"生成视频，支持文生视频、图生视频（首帧/首尾帧/参考图）等多种模式",
|
||
inputSchema: z.object({
|
||
prompt: z.string().describe("视频生成提示词，描述视频内容、镜头运动等"),
|
||
mode: z
|
||
.enum(["text-to-video", "first-frame", "first-last-frame", "reference-images"])
|
||
.describe("生成模式：文生视频/图生视频-首帧/首尾帧/参考图"),
|
||
images: z
|
||
.array(z.string())
|
||
.optional()
|
||
.describe("图片 URL 数组（根据模式不同，需要 0-4 张）"),
|
||
resolution: z
|
||
.enum(["480p", "720p", "1080p"])
|
||
.default("720p")
|
||
.describe("视频分辨率"),
|
||
ratio: z
|
||
.enum(["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"])
|
||
.default("16:9")
|
||
.describe("视频宽高比"),
|
||
duration: z
|
||
.number()
|
||
.min(1)
|
||
.max(600)
|
||
.default(5)
|
||
.describe("视频时长（秒）。优先使用当前视频模型配置的秒数档位，服务端会按模型能力动态校验。"),
|
||
watermark: z.boolean().default(true).describe("是否添加水印"),
|
||
cameraFixed: z.boolean().default(false).describe("是否固定摄像头"),
|
||
seed: z.number().optional().describe("随机种子（可选）"),
|
||
returnLastFrame: z
|
||
.boolean()
|
||
.default(false)
|
||
.describe("是否返回尾帧（用于连续生成）"),
|
||
}),
|
||
execute: async ({ prompt, mode, images, resolution, ratio, duration, watermark, cameraFixed, seed, returnLastFrame }) => {
|
||
try {
|
||
console.log("[Generate Video Tool] 🎬 开始生成视频");
|
||
console.log("[Generate Video Tool] 模式:", mode);
|
||
console.log("[Generate Video Tool] 提示词:", prompt);
|
||
|
||
dataStream.writeData({
|
||
type: "video-generation-start",
|
||
data: {
|
||
mode,
|
||
model: "智能选择中",
|
||
},
|
||
});
|
||
|
||
const result = await executeDirectVideoGeneration({
|
||
prompt,
|
||
mode,
|
||
images,
|
||
resolution,
|
||
ratio,
|
||
duration,
|
||
watermark,
|
||
cameraFixed,
|
||
seed,
|
||
returnLastFrame,
|
||
onProgress: async (chunk) => {
|
||
dataStream.writeMessageAnnotation({
|
||
type: "video-progress",
|
||
content: chunk,
|
||
});
|
||
},
|
||
});
|
||
|
||
console.log("[Generate Video Tool] ✅ 视频生成完成");
|
||
|
||
return {
|
||
success: true,
|
||
videoUrl: result.videoUrl,
|
||
metadata: result.metadata,
|
||
modelDisplayName: result.modelDisplayName,
|
||
modelId: result.modelId,
|
||
modelRef: result.modelRef,
|
||
};
|
||
} catch (error: any) {
|
||
console.error("[Generate Video Tool] ❌ 生成失败:", error);
|
||
dataStream.writeMessageAnnotation({
|
||
type: "video-error",
|
||
content: `视频生成失败：${error.message}`,
|
||
});
|
||
throw error;
|
||
}
|
||
},
|
||
});
|
||
}
|