518 lines
16 KiB
TypeScript
518 lines
16 KiB
TypeScript
/**
|
||
* 火山引擎视频生成 Provider（豆包 Seedance 模型）
|
||
* 支持：文生视频、图生视频（首帧/首尾帧/参考图）
|
||
* API 文档：https://www.volcengine.com/docs/82379/1298442
|
||
*/
|
||
|
||
import type {
|
||
VideoGenerationOptions,
|
||
VideoGenerationResult,
|
||
VideoModelCapabilities,
|
||
} from "@/types/ai";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
|
||
/**
|
||
* 将图片 URL 转换为 Base64（火山引擎 API 要求）
|
||
*/
|
||
async function convertImageUrlToBase64(imageUrl: string): Promise<{
|
||
permanentUrl: string;
|
||
base64: string;
|
||
}> {
|
||
try {
|
||
// 如果已经是 Base64，直接返回
|
||
if (imageUrl.startsWith("data:image/")) {
|
||
const matches = imageUrl.match(/^data:image\/(\w+);base64,(.+)$/);
|
||
if (!matches) {
|
||
throw new Error("Invalid base64 image format");
|
||
}
|
||
|
||
// 上传到 S3 获取永久 URL
|
||
const buffer = Buffer.from(matches[2], "base64");
|
||
const filename = `volcano-video-input-${Date.now()}.${matches[1]}`;
|
||
const { url: permanentUrl } = await uploadFile(filename, buffer as any);
|
||
|
||
return { permanentUrl, base64: matches[2] };
|
||
}
|
||
|
||
// 下载图片
|
||
console.log(`[Volcano Video] Downloading image: ${imageUrl}`);
|
||
const response = await fetch(imageUrl);
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`Failed to download image: ${response.statusText}`);
|
||
}
|
||
|
||
const buffer = await response.arrayBuffer();
|
||
const bufferNode = Buffer.from(buffer);
|
||
|
||
// 上传到 S3 获取永久 URL
|
||
const filename = `volcano-video-input-${Date.now()}.jpg`;
|
||
const { url: permanentUrl } = await uploadFile(filename, bufferNode as any);
|
||
|
||
console.log(`[Volcano Video] Image uploaded to S3: ${permanentUrl}`);
|
||
|
||
// 转换为 Base64
|
||
const base64 = bufferNode.toString("base64");
|
||
|
||
return { permanentUrl, base64 };
|
||
} catch (error) {
|
||
console.error("[Volcano Video] Image processing failed:", error);
|
||
throw new Error("图片处理失败，请重试");
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 🎬 运镜模式映射表（转换为自然语言提示词）
|
||
* 根据火山引擎API文档，运镜效果通过文本描述实现
|
||
*/
|
||
// 运镜描述映射（与官方一致）
|
||
const CAMERA_MOVE_DESCRIPTIONS: Record<string, string> = {
|
||
"none": "", // 智能运镜，保持空字符串
|
||
"static": "固定镜头拍摄", // 固定镜头
|
||
"hitchcock-push": "希区柯克式镜头前推，背景同时拉远", // 希区柯克推进
|
||
"hitchcock-pull": "希区柯克式镜头后拉，背景同时推进", // 希区柯克拉远
|
||
"mechanical-arm": "机械臂运镜，多角度平滑运动", // 机械臂
|
||
"dynamic-orbit": "动感环绕运镜，快速360度旋转", // 动感环绕
|
||
"center-orbit": "以画面中心点环绕旋转", // 中心环绕
|
||
"crane": "起重机式垂直升降运镜", // 起重机
|
||
"super-pull": "镜头快速后退超级拉远", // 超级拉远
|
||
"ccw-rotate": "镜头逆时针旋转运动", // 逆时针回旋
|
||
"cw-rotate": "镜头顺时针旋转运动", // 顺时针回旋
|
||
"handheld": "手持摄像机拍摄效果", // 手持运镜
|
||
"fast-push-pull": "镜头快速推拉来回运动", // 快速推拉
|
||
};
|
||
|
||
/**
|
||
* 构建视频参数字符串
|
||
* 格式：--ratio 16:9 --dur 5 --rs 720p --cf false --wm true
|
||
*/
|
||
function buildParameterString(options: VideoGenerationOptions): string {
|
||
const params: string[] = [];
|
||
|
||
// 宽高比
|
||
if (options.ratio && options.ratio !== "adaptive") {
|
||
params.push(`--rt ${options.ratio}`);
|
||
}
|
||
|
||
// 时长
|
||
if (options.duration) {
|
||
params.push(`--dur ${options.duration}`);
|
||
}
|
||
|
||
// 分辨率
|
||
if (options.resolution) {
|
||
params.push(`--rs ${options.resolution}`);
|
||
}
|
||
|
||
// 🔧 修复：固定摄像头（使用正确的参数名）
|
||
if (options.cameraFixed !== undefined) {
|
||
params.push(`--cf ${options.cameraFixed}`);
|
||
}
|
||
|
||
// 水印
|
||
if (options.watermark !== undefined) {
|
||
params.push(`--wm ${options.watermark}`);
|
||
}
|
||
|
||
// 种子
|
||
if (options.seed) {
|
||
params.push(`--seed ${options.seed}`);
|
||
}
|
||
|
||
return params.join(" ");
|
||
}
|
||
|
||
/**
|
||
* 🆕 构建完整提示词（包含运镜描述）
|
||
* 将用户提示词 + 运镜描述 + API参数组合
|
||
*/
|
||
function buildFullPrompt(options: VideoGenerationOptions): string {
|
||
const parts: string[] = [options.prompt];
|
||
|
||
// 🎬 添加运镜描述（如果用户选择了运镜模式）
|
||
if (options.cameraMove && options.cameraMove !== "none") {
|
||
const cameraDescription = CAMERA_MOVE_DESCRIPTIONS[options.cameraMove];
|
||
if (cameraDescription) {
|
||
parts.push(cameraDescription);
|
||
}
|
||
}
|
||
|
||
// 添加API参数
|
||
const params = buildParameterString(options);
|
||
if (params) {
|
||
parts.push(params);
|
||
}
|
||
|
||
return parts.join(" ");
|
||
}
|
||
|
||
/**
|
||
* 创建视频生成任务
|
||
*/
|
||
async function createVideoTask(
|
||
baseURL: string,
|
||
apiKey: string,
|
||
options: VideoGenerationOptions
|
||
): Promise<{ taskId: string; inputImageUrls: any }> {
|
||
// 火山引擎官方文档：/contents/generations/tasks（复数形式）
|
||
// https://www.volcengine.com/docs/82379/1520758
|
||
const url = `${baseURL}/contents/generations/tasks`;
|
||
|
||
// 构建 content 数组
|
||
const content: any[] = [];
|
||
const inputImageUrls: any = {};
|
||
|
||
// 1. 🆕 添加文本提示词（包含运镜描述 + API参数）
|
||
const fullPrompt = buildFullPrompt(options);
|
||
|
||
console.log("[Volcano Video] Full Prompt:", fullPrompt);
|
||
|
||
content.push({
|
||
type: "text",
|
||
text: fullPrompt,
|
||
});
|
||
|
||
// 2. 处理图片输入（根据模式）
|
||
if (options.mode === "first-frame" && options.firstFrameImage) {
|
||
// 首帧模式
|
||
const { permanentUrl, base64 } = await convertImageUrlToBase64(
|
||
options.firstFrameImage
|
||
);
|
||
inputImageUrls.firstFrame = permanentUrl;
|
||
|
||
content.push({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: `data:image/jpeg;base64,${base64}`,
|
||
},
|
||
role: "first_frame",
|
||
});
|
||
} else if (
|
||
options.mode === "first-last-frame" &&
|
||
options.firstFrameImage &&
|
||
options.lastFrameImage
|
||
) {
|
||
// 首尾帧模式
|
||
const firstFrame = await convertImageUrlToBase64(options.firstFrameImage);
|
||
const lastFrame = await convertImageUrlToBase64(options.lastFrameImage);
|
||
|
||
inputImageUrls.firstFrame = firstFrame.permanentUrl;
|
||
inputImageUrls.lastFrame = lastFrame.permanentUrl;
|
||
|
||
content.push(
|
||
{
|
||
type: "image_url",
|
||
image_url: {
|
||
url: `data:image/jpeg;base64,${firstFrame.base64}`,
|
||
},
|
||
role: "first_frame",
|
||
},
|
||
{
|
||
type: "image_url",
|
||
image_url: {
|
||
url: `data:image/jpeg;base64,${lastFrame.base64}`,
|
||
},
|
||
role: "last_frame",
|
||
}
|
||
);
|
||
} else if (options.mode === "reference-images" && options.referenceImages) {
|
||
// 参考图模式
|
||
inputImageUrls.references = [];
|
||
|
||
for (const imageUrl of options.referenceImages) {
|
||
const { permanentUrl, base64 } = await convertImageUrlToBase64(imageUrl);
|
||
inputImageUrls.references.push(permanentUrl);
|
||
|
||
content.push({
|
||
type: "image_url",
|
||
image_url: {
|
||
url: `data:image/jpeg;base64,${base64}`,
|
||
},
|
||
role: "reference_image",
|
||
});
|
||
}
|
||
}
|
||
|
||
// 3. 🔧 构建请求体（火山引擎API格式）
|
||
const requestBody: any = {
|
||
model: options.model,
|
||
content: content, // 直接传content数组，不使用messages包装
|
||
};
|
||
|
||
// 可选参数：返回尾帧（用于连续生成视频）
|
||
if (options.returnLastFrame) {
|
||
requestBody.return_last_frame = true;
|
||
}
|
||
|
||
console.log("[Volcano Video] Creating task:", {
|
||
model: options.model,
|
||
mode: options.mode,
|
||
prompt: options.prompt,
|
||
fullPrompt,
|
||
cameraMove: options.cameraMove,
|
||
});
|
||
|
||
// 4. 发送请求
|
||
console.log("[Volcano Video] 🔍 Request URL:", url);
|
||
console.log("[Volcano Video] 🔍 Request method: POST");
|
||
console.log("[Volcano Video] 🔍 Request body:", JSON.stringify(requestBody, null, 2));
|
||
|
||
const response = await fetch(url, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${apiKey}`,
|
||
},
|
||
body: JSON.stringify(requestBody),
|
||
});
|
||
|
||
console.log("[Volcano Video] 🔍 Response status:", response.status);
|
||
console.log("[Volcano Video] 🔍 Response headers:", JSON.stringify(Object.fromEntries(response.headers.entries())));
|
||
|
||
if (!response.ok) {
|
||
const errorText = await response.text();
|
||
console.error("[Volcano Video] ❌ Task creation failed");
|
||
console.error("[Volcano Video] ❌ Status:", response.status);
|
||
console.error("[Volcano Video] ❌ Response:", errorText);
|
||
throw new Error(`视频任务创建失败: ${response.status} ${errorText}`);
|
||
}
|
||
|
||
const data = await response.json();
|
||
|
||
if (!data.id) {
|
||
console.error("[Volcano Video] Invalid response:", data);
|
||
throw new Error("任务创建失败：未返回任务ID");
|
||
}
|
||
|
||
console.log("[Volcano Video] Task created:", data.id);
|
||
|
||
return {
|
||
taskId: data.id,
|
||
inputImageUrls
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 查询任务状态
|
||
*/
|
||
async function fetchTaskStatus(
|
||
baseURL: string,
|
||
apiKey: string,
|
||
taskId: string
|
||
): Promise<any> {
|
||
const url = `${baseURL}/contents/generations/tasks/${taskId}`;
|
||
|
||
const response = await fetch(url, {
|
||
method: "GET",
|
||
headers: {
|
||
Authorization: `Bearer ${apiKey}`,
|
||
},
|
||
});
|
||
|
||
if (!response.ok) {
|
||
const errorText = await response.text();
|
||
throw new Error(`任务查询失败: ${response.status} ${errorText}`);
|
||
}
|
||
|
||
return await response.json();
|
||
}
|
||
|
||
/**
|
||
* 轮询任务状态直到完成
|
||
*/
|
||
async function pollTaskStatus(
|
||
baseURL: string,
|
||
apiKey: string,
|
||
taskId: string,
|
||
maxRetries = 60, // 最多轮询 60 次
|
||
pollInterval = 10000 // 每 10 秒轮询一次
|
||
): Promise<VideoGenerationResult> {
|
||
console.log("[Volcano Video] Polling task:", taskId);
|
||
|
||
for (let i = 0; i < maxRetries; i++) {
|
||
const status = await fetchTaskStatus(baseURL, apiKey, taskId);
|
||
|
||
console.log(
|
||
`[Volcano Video] Poll ${i + 1}/${maxRetries}, status: ${status.status}`
|
||
);
|
||
|
||
if (status.status === "succeeded") {
|
||
// 任务成功 - 火山引擎API返回格式：status.content.video_url
|
||
const videoUrl = status.content?.video_url || status.output?.video_url || status.video_url;
|
||
const lastFrameUrl = status.content?.last_frame_url || status.output?.last_frame_url || status.last_frame_url;
|
||
|
||
if (!videoUrl) {
|
||
throw new Error("视频生成成功但未返回视频URL");
|
||
}
|
||
|
||
console.log("[Volcano Video] ✅ Video generated:", videoUrl);
|
||
|
||
return {
|
||
taskId: taskId,
|
||
status: "succeeded",
|
||
videoUrl: videoUrl,
|
||
lastFrameUrl: lastFrameUrl,
|
||
usage: status.usage,
|
||
createdAt: Date.now(),
|
||
updatedAt: Date.now(),
|
||
};
|
||
}
|
||
|
||
if (status.status === "failed") {
|
||
// 任务失败
|
||
const errorMsg =
|
||
status.error?.message || status.message || "未知错误";
|
||
console.error("[Volcano Video] ❌ Task failed:", errorMsg);
|
||
throw new Error(`视频生成失败: ${errorMsg}`);
|
||
}
|
||
|
||
if (status.status === "cancelled") {
|
||
throw new Error("视频生成任务已取消");
|
||
}
|
||
|
||
// 继续等待
|
||
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
||
}
|
||
|
||
throw new Error("视频生成超时，请稍后重试");
|
||
}
|
||
|
||
/**
|
||
* 格式化视频响应为 Markdown
|
||
*/
|
||
function formatVideoResponse(result: VideoGenerationResult): string {
|
||
// 只显示视频和下载按钮，不显示额外文字
|
||
return `<video src="${result.videoUrl}" controls style="max-width: 100%; border-radius: 8px; margin-bottom: 12px;"></video>
|
||
|
||
<a href="${result.videoUrl}" download style="display: inline-block; padding: 8px 16px; background: #000000; color: white; text-decoration: none; border-radius: 6px; font-size: 14px; font-weight: 500; transition: all 0.2s; hover:background: #333333;">下载视频</a>`;
|
||
}
|
||
|
||
/**
|
||
* 主函数：生成视频
|
||
*/
|
||
export async function doGenerate(
|
||
apiKey: string,
|
||
baseURL: string,
|
||
options: VideoGenerationOptions
|
||
): Promise<{ text: string; videoMetadata: any }> {
|
||
try {
|
||
console.log("[Volcano Video] 🎬 Starting video generation...");
|
||
console.log("[Volcano Video] Options:", {
|
||
mode: options.mode,
|
||
model: options.model,
|
||
resolution: options.resolution,
|
||
ratio: options.ratio,
|
||
duration: options.duration,
|
||
});
|
||
|
||
// 1. 创建任务
|
||
const { taskId, inputImageUrls } = await createVideoTask(baseURL, apiKey, options);
|
||
|
||
// 2. 轮询任务状态
|
||
const result = await pollTaskStatus(baseURL, apiKey, taskId);
|
||
|
||
// 3. 格式化输出
|
||
const markdown = formatVideoResponse(result);
|
||
|
||
// 4. 返回视频元数据（用于保存到数据库）
|
||
const videoMetadata = {
|
||
taskId: result.taskId,
|
||
duration: options.duration,
|
||
resolution: options.resolution,
|
||
ratio: options.ratio,
|
||
inputImages: inputImageUrls,
|
||
};
|
||
|
||
return {
|
||
text: markdown,
|
||
videoMetadata: {
|
||
...videoMetadata,
|
||
temporaryVideoUrl: result.videoUrl,
|
||
lastFrameUrl: result.lastFrameUrl,
|
||
},
|
||
};
|
||
} catch (error: any) {
|
||
console.error("[Volcano Video] ❌ Generation failed:", error);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 流式生成（模拟进度推送）
|
||
*/
|
||
export async function* doGenerateStream(
|
||
apiKey: string,
|
||
baseURL: string,
|
||
options: VideoGenerationOptions
|
||
): AsyncGenerator<string, void, unknown> {
|
||
try {
|
||
// 1. 创建任务（不输出进度文字）
|
||
const { taskId, inputImageUrls } = await createVideoTask(baseURL, apiKey, options);
|
||
|
||
// 2. 轮询任务状态（静默等待）
|
||
const maxRetries = 60;
|
||
const pollInterval = 10000;
|
||
|
||
for (let i = 0; i < maxRetries; i++) {
|
||
const status = await fetchTaskStatus(baseURL, apiKey, taskId);
|
||
|
||
if (status.status === "succeeded") {
|
||
// 火山引擎API返回格式：status.content.video_url
|
||
const videoUrl = status.content?.video_url || status.output?.video_url || status.video_url;
|
||
const lastFrameUrl = status.content?.last_frame_url || status.output?.last_frame_url || status.last_frame_url;
|
||
|
||
console.log("[Volcano Video] ✅ Task succeeded!");
|
||
console.log("[Volcano Video] Video URL:", videoUrl);
|
||
console.log("[Volcano Video] Last frame URL:", lastFrameUrl);
|
||
console.log("[Volcano Video] Full status response:", JSON.stringify(status, null, 2));
|
||
|
||
const videoResponse = formatVideoResponse({
|
||
taskId: taskId,
|
||
status: "succeeded",
|
||
videoUrl: videoUrl,
|
||
lastFrameUrl: lastFrameUrl,
|
||
usage: status.usage,
|
||
createdAt: Date.now(),
|
||
updatedAt: Date.now(),
|
||
});
|
||
|
||
console.log("[Volcano Video] Formatted response:", videoResponse);
|
||
|
||
yield videoResponse;
|
||
|
||
// 输出视频元数据（JSON格式，前端可解析）
|
||
yield `\n\n<!-- VIDEO_METADATA:${JSON.stringify({
|
||
taskId: taskId,
|
||
temporaryVideoUrl: videoUrl,
|
||
lastFrameUrl: lastFrameUrl,
|
||
duration: options.duration,
|
||
resolution: options.resolution,
|
||
ratio: options.ratio,
|
||
inputImages: inputImageUrls,
|
||
})} -->`;
|
||
|
||
return;
|
||
}
|
||
|
||
if (status.status === "failed") {
|
||
const errorMsg = status.error?.message || status.message || "未知错误";
|
||
throw new Error(`视频生成失败: ${errorMsg}`);
|
||
}
|
||
|
||
if (status.status === "cancelled") {
|
||
throw new Error("视频生成任务已取消");
|
||
}
|
||
|
||
// 静默等待，不输出进度文字
|
||
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
||
}
|
||
|
||
throw new Error("视频生成超时，请稍后重试");
|
||
} catch (error: any) {
|
||
console.error("[Volcano Video] ❌ Stream generation failed:", error);
|
||
yield `\n\n❌ 错误：${error.message}\n`;
|
||
throw error;
|
||
}
|
||
}
|
||
|