/** * 火山引擎视频生成 Provider(豆包 Seedance 模型) * 支持:文生视频、图生视频(首帧/首尾帧/参考图) * API 文档:https://www.volcengine.com/docs/82379/1298442 */ import type { VideoGenerationOptions, VideoGenerationResult, VideoModelCapabilities, } from "@/types/ai"; import { uploadFile } from "@/lib/storage/service"; /** * 将图片 URL 转换为 Base64(火山引擎 API 要求) */ async function convertImageUrlToBase64(imageUrl: string): Promise<{ permanentUrl: string; base64: string; }> { try { // 如果已经是 Base64,直接返回 if (imageUrl.startsWith("data:image/")) { const matches = imageUrl.match(/^data:image\/(\w+);base64,(.+)$/); if (!matches) { throw new Error("Invalid base64 image format"); } // 上传到 S3 获取永久 URL const buffer = Buffer.from(matches[2], "base64"); const filename = `volcano-video-input-${Date.now()}.${matches[1]}`; const { url: permanentUrl } = await uploadFile(filename, buffer as any); return { permanentUrl, base64: matches[2] }; } // 下载图片 console.log(`[Volcano Video] Downloading image: ${imageUrl}`); const response = await fetch(imageUrl); if (!response.ok) { throw new Error(`Failed to download image: ${response.statusText}`); } const buffer = await response.arrayBuffer(); const bufferNode = Buffer.from(buffer); // 上传到 S3 获取永久 URL const filename = `volcano-video-input-${Date.now()}.jpg`; const { url: permanentUrl } = await uploadFile(filename, bufferNode as any); console.log(`[Volcano Video] Image uploaded to S3: ${permanentUrl}`); // 转换为 Base64 const base64 = bufferNode.toString("base64"); return { permanentUrl, base64 }; } catch (error) { console.error("[Volcano Video] Image processing failed:", error); throw new Error("图片处理失败,请重试"); } } /** * 🎬 运镜模式映射表(转换为自然语言提示词) * 根据火山引擎API文档,运镜效果通过文本描述实现 */ // 运镜描述映射(与官方一致) const CAMERA_MOVE_DESCRIPTIONS: Record = { "none": "", // 智能运镜,保持空字符串 "static": "固定镜头拍摄", // 固定镜头 "hitchcock-push": "希区柯克式镜头前推,背景同时拉远", // 希区柯克推进 "hitchcock-pull": "希区柯克式镜头后拉,背景同时推进", // 希区柯克拉远 "mechanical-arm": "机械臂运镜,多角度平滑运动", // 机械臂 "dynamic-orbit": "动感环绕运镜,快速360度旋转", // 动感环绕 "center-orbit": "以画面中心点环绕旋转", // 中心环绕 "crane": "起重机式垂直升降运镜", // 起重机 "super-pull": "镜头快速后退超级拉远", // 超级拉远 "ccw-rotate": "镜头逆时针旋转运动", // 逆时针回旋 "cw-rotate": "镜头顺时针旋转运动", // 顺时针回旋 "handheld": "手持摄像机拍摄效果", // 手持运镜 "fast-push-pull": "镜头快速推拉来回运动", // 快速推拉 }; /** * 构建视频参数字符串 * 格式:--ratio 16:9 --dur 5 --rs 720p --cf false --wm true */ function buildParameterString(options: VideoGenerationOptions): string { const params: string[] = []; // 宽高比 if (options.ratio && options.ratio !== "adaptive") { params.push(`--rt ${options.ratio}`); } // 时长 if (options.duration) { params.push(`--dur ${options.duration}`); } // 分辨率 if (options.resolution) { params.push(`--rs ${options.resolution}`); } // 🔧 修复:固定摄像头(使用正确的参数名) if (options.cameraFixed !== undefined) { params.push(`--cf ${options.cameraFixed}`); } // 水印 if (options.watermark !== undefined) { params.push(`--wm ${options.watermark}`); } // 种子 if (options.seed) { params.push(`--seed ${options.seed}`); } return params.join(" "); } /** * 🆕 构建完整提示词(包含运镜描述) * 将用户提示词 + 运镜描述 + API参数组合 */ function buildFullPrompt(options: VideoGenerationOptions): string { const parts: string[] = [options.prompt]; // 🎬 添加运镜描述(如果用户选择了运镜模式) if (options.cameraMove && options.cameraMove !== "none") { const cameraDescription = CAMERA_MOVE_DESCRIPTIONS[options.cameraMove]; if (cameraDescription) { parts.push(cameraDescription); } } // 添加API参数 const params = buildParameterString(options); if (params) { parts.push(params); } return parts.join(" "); } /** * 创建视频生成任务 */ async function createVideoTask( baseURL: string, apiKey: string, options: VideoGenerationOptions ): Promise<{ taskId: string; inputImageUrls: any }> { // 火山引擎官方文档:/contents/generations/tasks(复数形式) // https://www.volcengine.com/docs/82379/1520758 const url = `${baseURL}/contents/generations/tasks`; // 构建 content 数组 const content: any[] = []; const inputImageUrls: any = {}; // 1. 🆕 添加文本提示词(包含运镜描述 + API参数) const fullPrompt = buildFullPrompt(options); console.log("[Volcano Video] Full Prompt:", fullPrompt); content.push({ type: "text", text: fullPrompt, }); // 2. 处理图片输入(根据模式) if (options.mode === "first-frame" && options.firstFrameImage) { // 首帧模式 const { permanentUrl, base64 } = await convertImageUrlToBase64( options.firstFrameImage ); inputImageUrls.firstFrame = permanentUrl; content.push({ type: "image_url", image_url: { url: `data:image/jpeg;base64,${base64}`, }, role: "first_frame", }); } else if ( options.mode === "first-last-frame" && options.firstFrameImage && options.lastFrameImage ) { // 首尾帧模式 const firstFrame = await convertImageUrlToBase64(options.firstFrameImage); const lastFrame = await convertImageUrlToBase64(options.lastFrameImage); inputImageUrls.firstFrame = firstFrame.permanentUrl; inputImageUrls.lastFrame = lastFrame.permanentUrl; content.push( { type: "image_url", image_url: { url: `data:image/jpeg;base64,${firstFrame.base64}`, }, role: "first_frame", }, { type: "image_url", image_url: { url: `data:image/jpeg;base64,${lastFrame.base64}`, }, role: "last_frame", } ); } else if (options.mode === "reference-images" && options.referenceImages) { // 参考图模式 inputImageUrls.references = []; for (const imageUrl of options.referenceImages) { const { permanentUrl, base64 } = await convertImageUrlToBase64(imageUrl); inputImageUrls.references.push(permanentUrl); content.push({ type: "image_url", image_url: { url: `data:image/jpeg;base64,${base64}`, }, role: "reference_image", }); } } // 3. 🔧 构建请求体(火山引擎API格式) const requestBody: any = { model: options.model, content: content, // 直接传content数组,不使用messages包装 }; // 可选参数:返回尾帧(用于连续生成视频) if (options.returnLastFrame) { requestBody.return_last_frame = true; } console.log("[Volcano Video] Creating task:", { model: options.model, mode: options.mode, prompt: options.prompt, fullPrompt, cameraMove: options.cameraMove, }); // 4. 发送请求 console.log("[Volcano Video] 🔍 Request URL:", url); console.log("[Volcano Video] 🔍 Request method: POST"); console.log("[Volcano Video] 🔍 Request body:", JSON.stringify(requestBody, null, 2)); const response = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, }, body: JSON.stringify(requestBody), }); console.log("[Volcano Video] 🔍 Response status:", response.status); console.log("[Volcano Video] 🔍 Response headers:", JSON.stringify(Object.fromEntries(response.headers.entries()))); if (!response.ok) { const errorText = await response.text(); console.error("[Volcano Video] ❌ Task creation failed"); console.error("[Volcano Video] ❌ Status:", response.status); console.error("[Volcano Video] ❌ Response:", errorText); throw new Error(`视频任务创建失败: ${response.status} ${errorText}`); } const data = await response.json(); if (!data.id) { console.error("[Volcano Video] Invalid response:", data); throw new Error("任务创建失败:未返回任务ID"); } console.log("[Volcano Video] Task created:", data.id); return { taskId: data.id, inputImageUrls }; } /** * 查询任务状态 */ async function fetchTaskStatus( baseURL: string, apiKey: string, taskId: string ): Promise { const url = `${baseURL}/contents/generations/tasks/${taskId}`; const response = await fetch(url, { method: "GET", headers: { Authorization: `Bearer ${apiKey}`, }, }); if (!response.ok) { const errorText = await response.text(); throw new Error(`任务查询失败: ${response.status} ${errorText}`); } return await response.json(); } /** * 轮询任务状态直到完成 */ async function pollTaskStatus( baseURL: string, apiKey: string, taskId: string, maxRetries = 60, // 最多轮询 60 次 pollInterval = 10000 // 每 10 秒轮询一次 ): Promise { console.log("[Volcano Video] Polling task:", taskId); for (let i = 0; i < maxRetries; i++) { const status = await fetchTaskStatus(baseURL, apiKey, taskId); console.log( `[Volcano Video] Poll ${i + 1}/${maxRetries}, status: ${status.status}` ); if (status.status === "succeeded") { // 任务成功 - 火山引擎API返回格式:status.content.video_url const videoUrl = status.content?.video_url || status.output?.video_url || status.video_url; const lastFrameUrl = status.content?.last_frame_url || status.output?.last_frame_url || status.last_frame_url; if (!videoUrl) { throw new Error("视频生成成功但未返回视频URL"); } console.log("[Volcano Video] ✅ Video generated:", videoUrl); return { taskId: taskId, status: "succeeded", videoUrl: videoUrl, lastFrameUrl: lastFrameUrl, usage: status.usage, createdAt: Date.now(), updatedAt: Date.now(), }; } if (status.status === "failed") { // 任务失败 const errorMsg = status.error?.message || status.message || "未知错误"; console.error("[Volcano Video] ❌ Task failed:", errorMsg); throw new Error(`视频生成失败: ${errorMsg}`); } if (status.status === "cancelled") { throw new Error("视频生成任务已取消"); } // 继续等待 await new Promise((resolve) => setTimeout(resolve, pollInterval)); } throw new Error("视频生成超时,请稍后重试"); } /** * 格式化视频响应为 Markdown */ function formatVideoResponse(result: VideoGenerationResult): string { // 只显示视频和下载按钮,不显示额外文字 return ` 下载视频`; } /** * 主函数:生成视频 */ export async function doGenerate( apiKey: string, baseURL: string, options: VideoGenerationOptions ): Promise<{ text: string; videoMetadata: any }> { try { console.log("[Volcano Video] 🎬 Starting video generation..."); console.log("[Volcano Video] Options:", { mode: options.mode, model: options.model, resolution: options.resolution, ratio: options.ratio, duration: options.duration, }); // 1. 创建任务 const { taskId, inputImageUrls } = await createVideoTask(baseURL, apiKey, options); // 2. 轮询任务状态 const result = await pollTaskStatus(baseURL, apiKey, taskId); // 3. 格式化输出 const markdown = formatVideoResponse(result); // 4. 返回视频元数据(用于保存到数据库) const videoMetadata = { taskId: result.taskId, duration: options.duration, resolution: options.resolution, ratio: options.ratio, inputImages: inputImageUrls, }; return { text: markdown, videoMetadata: { ...videoMetadata, temporaryVideoUrl: result.videoUrl, lastFrameUrl: result.lastFrameUrl, }, }; } catch (error: any) { console.error("[Volcano Video] ❌ Generation failed:", error); throw error; } } /** * 流式生成(模拟进度推送) */ export async function* doGenerateStream( apiKey: string, baseURL: string, options: VideoGenerationOptions ): AsyncGenerator { try { // 1. 创建任务(不输出进度文字) const { taskId, inputImageUrls } = await createVideoTask(baseURL, apiKey, options); // 2. 轮询任务状态(静默等待) const maxRetries = 60; const pollInterval = 10000; for (let i = 0; i < maxRetries; i++) { const status = await fetchTaskStatus(baseURL, apiKey, taskId); if (status.status === "succeeded") { // 火山引擎API返回格式:status.content.video_url const videoUrl = status.content?.video_url || status.output?.video_url || status.video_url; const lastFrameUrl = status.content?.last_frame_url || status.output?.last_frame_url || status.last_frame_url; console.log("[Volcano Video] ✅ Task succeeded!"); console.log("[Volcano Video] Video URL:", videoUrl); console.log("[Volcano Video] Last frame URL:", lastFrameUrl); console.log("[Volcano Video] Full status response:", JSON.stringify(status, null, 2)); const videoResponse = formatVideoResponse({ taskId: taskId, status: "succeeded", videoUrl: videoUrl, lastFrameUrl: lastFrameUrl, usage: status.usage, createdAt: Date.now(), updatedAt: Date.now(), }); console.log("[Volcano Video] Formatted response:", videoResponse); yield videoResponse; // 输出视频元数据(JSON格式,前端可解析) yield `\n\n`; return; } if (status.status === "failed") { const errorMsg = status.error?.message || status.message || "未知错误"; throw new Error(`视频生成失败: ${errorMsg}`); } if (status.status === "cancelled") { throw new Error("视频生成任务已取消"); } // 静默等待,不输出进度文字 await new Promise((resolve) => setTimeout(resolve, pollInterval)); } throw new Error("视频生成超时,请稍后重试"); } catch (error: any) { console.error("[Volcano Video] ❌ Stream generation failed:", error); yield `\n\n❌ 错误:${error.message}\n`; throw error; } }