/** * 火山引擎视频生成 Provider(豆包 Seedance 模型) * 支持:文生视频、图生视频(首帧/首尾帧/参考图) * API 文档:https://www.volcengine.com/docs/82379/1298442 */ import type { VideoGenerationOptions, VideoGenerationResult, VideoModelCapabilities, } from "@/types/ai"; import { uploadFile } from "@/lib/storage/service"; const VOLCANO_MAX_INPUT_IMAGE_BYTES = 30 * 1024 * 1024; const VOLCANO_SAFE_INPUT_IMAGE_BYTES = VOLCANO_MAX_INPUT_IMAGE_BYTES - 2 * 1024 * 1024; const VOLCANO_OPTIMIZE_SIDE_STEPS = [3072, 2560, 2048, 1536, 1280, 1024]; const VOLCANO_OPTIMIZE_QUALITY_STEPS = [84, 76, 68, 60]; const RETRYABLE_HTTP_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]); const RETRYABLE_FETCH_CODES = new Set([ "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_BODY_TIMEOUT", "UND_ERR_SOCKET", "ETIMEDOUT", "ECONNRESET", "ECONNREFUSED", "EAI_AGAIN", "ENOTFOUND", ]); function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } function sanitizeUrlForLog(url: string): string { const trimmed = String(url || "").trim(); if (!trimmed) return ""; if (trimmed.startsWith("data:image/")) { return `data:image(base64,${trimmed.length} chars)`; } try { const parsed = new URL(trimmed); return `${parsed.origin}${parsed.pathname}`; } catch { return trimmed.slice(0, 120); } } function extractPublicHttpUrl(rawValue: string): string | null { const trimmed = rawValue.trim(); if (!trimmed) return null; if (/^https?:\/\//i.test(trimmed)) { return trimmed; } const markdownMatch = trimmed.match(/\((https?:\/\/[^)\s]+)\)/i); if (markdownMatch?.[1]) { return markdownMatch[1]; } const inlineMatch = trimmed.match(/https?:\/\/[^\s"'<>]+/i); if (inlineMatch?.[0]) { return inlineMatch[0]; } return null; } function mimeTypeToVolcanoImageExtension(mimeType?: string | null): string { const normalized = String(mimeType || "").toLowerCase(); if (normalized.includes("png")) return "png"; if (normalized.includes("webp")) return "webp"; if (normalized.includes("gif")) return "gif"; return "jpg"; } function isVolcanoOversizeImageError(errorText: string): boolean { const normalized = String(errorText || "").toLowerCase(); return ( normalized.includes("oversizeimage") || normalized.includes("exceeds the limit") || normalized.includes("image (") && normalized.includes("limit (") ); } async function optimizeImageBufferForVolcano( inputBuffer: Buffer, ): Promise<{ buffer: Buffer; extension: string; mimeType: string }> { const sharp = (await import("sharp")).default; const metadata = await sharp(inputBuffer, { failOn: "none" }).metadata(); const maxOriginalSide = Math.max(metadata.width || 0, metadata.height || 0); const sideCandidates = Array.from( new Set( [ maxOriginalSide > 0 ? Math.min(maxOriginalSide, VOLCANO_OPTIMIZE_SIDE_STEPS[0]!) : undefined, ...VOLCANO_OPTIMIZE_SIDE_STEPS, ].filter((value): value is number => Boolean(value && value > 0)) ) ); let bestBuffer = inputBuffer; for (const maxSide of sideCandidates) { for (const quality of VOLCANO_OPTIMIZE_QUALITY_STEPS) { const pipeline = sharp(inputBuffer, { failOn: "none" }) .rotate() .resize({ width: maxSide, height: maxSide, fit: "inside", withoutEnlargement: true, }) .flatten({ background: "#ffffff" }) .jpeg({ quality, mozjpeg: true, progressive: true, }); const optimizedBuffer = await pipeline.toBuffer(); if (optimizedBuffer.length < bestBuffer.length) { bestBuffer = optimizedBuffer; } if (optimizedBuffer.length <= VOLCANO_SAFE_INPUT_IMAGE_BYTES) { return { buffer: optimizedBuffer, extension: "jpg", mimeType: "image/jpeg", }; } } } return { buffer: bestBuffer, extension: "jpg", mimeType: "image/jpeg", }; } async function uploadVolcanoPreparedImage(params: { buffer: Buffer; extension?: string; }) { const filename = `volcano-video-input-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ params.extension || "jpg" }`; const { url } = await uploadFile(filename, params.buffer as any); return url; } function isRetryableFetchError(error: any): boolean { const code = error?.code || error?.cause?.code; if (typeof code === "string" && RETRYABLE_FETCH_CODES.has(code)) { return true; } if (error?.name === "AbortError") { return true; } const message = String(error?.message || "").toLowerCase(); return ( message.includes("timeout") || message.includes("timed out") || message.includes("fetch failed") ); } async function fetchWithRetry( url: string, init: RequestInit, options: { label: string; timeoutMs: number; retries: number; retryDelayMs?: number; } ): Promise { const { label, timeoutMs, retries, retryDelayMs = 1500 } = options; let attempt = 0; while (attempt <= retries) { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); try { const response = await fetch(url, { ...init, signal: controller.signal, }); clearTimeout(timer); if ( RETRYABLE_HTTP_STATUS.has(response.status) && attempt < retries ) { const bodyText = await response.text().catch(() => ""); console.warn( `[Volcano Video] ${label} transient status ${response.status}, retry ${attempt + 1}/${retries}`, bodyText?.slice(0, 300) ); await sleep(retryDelayMs * (attempt + 1)); attempt += 1; continue; } return response; } catch (error: any) { clearTimeout(timer); if (isRetryableFetchError(error) && attempt < retries) { console.warn( `[Volcano Video] ${label} request failed, retry ${attempt + 1}/${retries}`, error?.cause?.code || error?.code || error?.message ); await sleep(retryDelayMs * (attempt + 1)); attempt += 1; continue; } throw error; } } throw new Error(`[Volcano Video] ${label} request failed after retries`); } /** * 规范化图片地址为可公开访问 URL(优先直传 URL,避免大体积 base64) */ async function normalizeImageUrlForVolcano( imageUrl: string, options?: { forceOptimize?: boolean }, ): Promise { try { const rawInput = String(imageUrl || "").trim(); if (!rawInput) { throw new Error("Empty image url"); } // Base64 输入:上传到 S3,返回永久 URL if (rawInput.startsWith("data:image/")) { const matches = rawInput.match(/^data:(image\/[a-zA-Z0-9.+-]+);base64,(.+)$/); if (!matches) { throw new Error("Invalid base64 image format"); } const mimeType = matches[1]; const buffer = Buffer.from(matches[2], "base64"); const shouldOptimize = options?.forceOptimize === true || buffer.length > VOLCANO_SAFE_INPUT_IMAGE_BYTES; const prepared = shouldOptimize ? await optimizeImageBufferForVolcano(buffer) : { buffer, extension: mimeTypeToVolcanoImageExtension(mimeType), }; const permanentUrl = await uploadVolcanoPreparedImage(prepared); return permanentUrl; } // 已经是公网 URL,直接使用 const publicHttpUrl = extractPublicHttpUrl(rawInput); if (publicHttpUrl && options?.forceOptimize !== true) { return publicHttpUrl; } if (rawInput.startsWith("blob:") || rawInput.startsWith("file:")) { throw new Error("Unsupported local image URL. Please upload image first."); } // 其他格式:尝试下载后上传到 S3,返回永久 URL console.log(`[Volcano Video] Downloading image: ${sanitizeUrlForLog(rawInput)}`); const response = await fetchWithRetry( rawInput, { method: "GET" }, { label: "download-image", timeoutMs: 30000, retries: 2, } ); if (!response.ok) { throw new Error(`Failed to download image: ${response.statusText}`); } const buffer = await response.arrayBuffer(); const bufferNode = Buffer.from(buffer); const contentType = response.headers.get("content-type"); const shouldOptimize = options?.forceOptimize === true || bufferNode.length > VOLCANO_SAFE_INPUT_IMAGE_BYTES; const prepared = shouldOptimize ? await optimizeImageBufferForVolcano(bufferNode) : { buffer: bufferNode, extension: mimeTypeToVolcanoImageExtension(contentType), }; const permanentUrl = await uploadVolcanoPreparedImage(prepared); console.log(`[Volcano Video] Image uploaded to S3: ${sanitizeUrlForLog(permanentUrl)}`); return permanentUrl; } catch (error) { console.error("[Volcano Video] Image processing failed:", error); throw new Error("图片处理失败,请重试"); } } async function buildVolcanoTaskRequestPayload( options: VideoGenerationOptions, requestBody?: Record, requestMeta?: { forceOptimizeImages?: boolean }, ) { const content: any[] = []; const inputImageUrls: any = {}; const fullPrompt = buildFullPrompt(options); console.log("[Volcano Video] Full Prompt:", fullPrompt); content.push({ type: "text", text: fullPrompt, }); if (options.mode === "first-frame" && options.firstFrameImage) { const firstFrameUrl = await normalizeImageUrlForVolcano(options.firstFrameImage, { forceOptimize: requestMeta?.forceOptimizeImages, }); inputImageUrls.firstFrame = firstFrameUrl; content.push({ type: "image_url", image_url: { url: firstFrameUrl, }, role: "first_frame", }); } else if ( options.mode === "first-last-frame" && options.firstFrameImage && options.lastFrameImage ) { const firstFrameUrl = await normalizeImageUrlForVolcano(options.firstFrameImage, { forceOptimize: requestMeta?.forceOptimizeImages, }); const lastFrameUrl = await normalizeImageUrlForVolcano(options.lastFrameImage, { forceOptimize: requestMeta?.forceOptimizeImages, }); inputImageUrls.firstFrame = firstFrameUrl; inputImageUrls.lastFrame = lastFrameUrl; content.push( { type: "image_url", image_url: { url: firstFrameUrl, }, role: "first_frame", }, { type: "image_url", image_url: { url: lastFrameUrl, }, role: "last_frame", }, ); } else if (options.mode === "reference-images" && options.referenceImages) { inputImageUrls.references = []; for (const imageUrl of options.referenceImages) { const normalizedUrl = await normalizeImageUrlForVolcano(imageUrl, { forceOptimize: requestMeta?.forceOptimizeImages, }); inputImageUrls.references.push(normalizedUrl); content.push({ type: "image_url", image_url: { url: normalizedUrl, }, role: "reference_image", }); } } const nextRequestBody: any = requestBody || { model: options.model, }; nextRequestBody.content = content; const taskTypeMap: Record = { "text-to-video": "t2v", "first-frame": "i2v", "first-last-frame": "flf2v", "reference-images": "i2v", }; nextRequestBody.task_type = taskTypeMap[options.mode] || "t2v"; if (options.returnLastFrame) { nextRequestBody.return_last_frame = true; } return { requestBody: nextRequestBody, inputImageUrls, fullPrompt, contentCount: content.length, }; } async function sendCreateVideoTaskRequest(params: { url: string; apiKey: string; requestBody: Record; }) { return fetchWithRetry( params.url, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${params.apiKey}`, }, body: JSON.stringify(params.requestBody), }, { label: "create-task", timeoutMs: 90000, retries: 2, } ); } /** * 🎬 运镜模式映射表(转换为自然语言提示词) * 根据火山引擎API文档,运镜效果通过文本描述实现 */ // 运镜描述映射(与官方一致) const CAMERA_MOVE_DESCRIPTIONS: Record = { "none": "", // 智能运镜,保持空字符串 "static": "固定镜头拍摄", // 固定镜头 "hitchcock-push": "希区柯克式镜头前推,背景同时拉远", // 希区柯克推进 "hitchcock-pull": "希区柯克式镜头后拉,背景同时推进", // 希区柯克拉远 "mechanical-arm": "机械臂运镜,多角度平滑运动", // 机械臂 "dynamic-orbit": "动感环绕运镜,快速360度旋转", // 动感环绕 "center-orbit": "以画面中心点环绕旋转", // 中心环绕 "crane": "起重机式垂直升降运镜", // 起重机 "super-pull": "镜头快速后退超级拉远", // 超级拉远 "ccw-rotate": "镜头逆时针旋转运动", // 逆时针回旋 "cw-rotate": "镜头顺时针旋转运动", // 顺时针回旋 "handheld": "手持摄像机拍摄效果", // 手持运镜 "fast-push-pull": "镜头快速推拉来回运动", // 快速推拉 }; /** * 构建视频参数字符串 * 格式:--ratio 16:9 --dur 5 --rs 720p --cf false --wm true */ function buildParameterString(options: VideoGenerationOptions): string { const params: string[] = []; // 宽高比 if (options.ratio && options.ratio !== "adaptive") { params.push(`--rt ${options.ratio}`); } // 时长 if (options.duration) { params.push(`--dur ${options.duration}`); } // 分辨率 if (options.resolution) { params.push(`--rs ${options.resolution}`); } // 🔧 修复:固定摄像头(使用正确的参数名) if (options.cameraFixed !== undefined) { params.push(`--cf ${options.cameraFixed}`); } // 水印 if (options.watermark !== undefined) { params.push(`--wm ${options.watermark}`); } // 种子 if (options.seed) { params.push(`--seed ${options.seed}`); } return params.join(" "); } /** * 🆕 构建完整提示词(包含运镜描述) * 将用户提示词 + 运镜描述 + API参数组合 */ function buildFullPrompt(options: VideoGenerationOptions): string { const parts: string[] = [options.prompt]; // 🎬 添加运镜描述(如果用户选择了运镜模式) if (options.cameraMove && options.cameraMove !== "none") { const cameraDescription = CAMERA_MOVE_DESCRIPTIONS[options.cameraMove]; if (cameraDescription) { parts.push(cameraDescription); } } // 添加API参数 const params = buildParameterString(options); if (params) { parts.push(params); } return parts.join(" "); } /** * 创建视频生成任务 */ async function createVideoTask( baseURL: string, apiKey: string, options: VideoGenerationOptions ): Promise<{ taskId: string; inputImageUrls: any }> { // 火山引擎官方文档:/contents/generations/tasks(复数形式) // https://www.volcengine.com/docs/82379/1520758 const url = `${baseURL}/contents/generations/tasks`; // 3. 🔧 构建请求体(火山引擎API格式) const baseRequestBody: any = { model: options.model, }; let payload = await buildVolcanoTaskRequestPayload(options, baseRequestBody, { forceOptimizeImages: false, }); console.log("[Volcano Video] Creating task:", { model: options.model, mode: options.mode, promptLength: options.prompt?.length || 0, fullPromptLength: payload.fullPrompt.length, cameraMove: options.cameraMove, contentItems: payload.contentCount, imageRefs: Object.values(payload.inputImageUrls) .flat() .map((value: any) => sanitizeUrlForLog(String(value))) .slice(0, 6), }); // 4. 发送请求 console.log("[Volcano Video] Request URL:", url); let response = await sendCreateVideoTaskRequest({ url, apiKey, requestBody: payload.requestBody, }); console.log("[Volcano Video] Response status:", response.status); if (!response.ok) { let errorText = await response.text(); console.error("[Volcano Video] ❌ Task creation failed"); console.error("[Volcano Video] ❌ Status:", response.status); console.error("[Volcano Video] ❌ Response:", errorText); const hasImageInputs = Boolean(options.firstFrameImage) || Boolean(options.lastFrameImage) || Boolean(options.referenceImages?.length); if (hasImageInputs && isVolcanoOversizeImageError(errorText)) { console.warn( "[Volcano Video] Oversize image detected, retrying with optimized uploaded images", ); payload = await buildVolcanoTaskRequestPayload(options, baseRequestBody, { forceOptimizeImages: true, }); response = await sendCreateVideoTaskRequest({ url, apiKey, requestBody: payload.requestBody, }); console.log("[Volcano Video] Retry response status:", response.status); if (!response.ok) { errorText = await response.text(); console.error("[Volcano Video] ❌ Retry task creation failed"); console.error("[Volcano Video] ❌ Retry response:", errorText); throw new Error(`视频任务创建失败: ${response.status} ${errorText}`); } } else { throw new Error(`视频任务创建失败: ${response.status} ${errorText}`); } } const data = await response.json(); if (!data.id) { console.error("[Volcano Video] Invalid response:", data); throw new Error("任务创建失败:未返回任务ID"); } console.log("[Volcano Video] Task created:", data.id); return { taskId: data.id, inputImageUrls: payload.inputImageUrls }; } /** * 查询任务状态 */ async function fetchTaskStatus( baseURL: string, apiKey: string, taskId: string ): Promise { const url = `${baseURL}/contents/generations/tasks/${taskId}`; const response = await fetchWithRetry( url, { method: "GET", headers: { Authorization: `Bearer ${apiKey}`, }, }, { label: `task-status:${taskId}`, timeoutMs: 45000, retries: 2, } ); if (!response.ok) { const errorText = await response.text(); throw new Error(`任务查询失败: ${response.status} ${errorText}`); } return await response.json(); } /** * 轮询任务状态直到完成 */ async function pollTaskStatus( baseURL: string, apiKey: string, taskId: string, maxRetries = 60, // 最多轮询 60 次 pollInterval = 10000 // 每 10 秒轮询一次 ): Promise { console.log("[Volcano Video] Polling task:", taskId); for (let i = 0; i < maxRetries; i++) { let status: any; try { status = await fetchTaskStatus(baseURL, apiKey, taskId); } catch (error: any) { if (isRetryableFetchError(error)) { console.warn( `[Volcano Video] Poll ${i + 1}/${maxRetries} transient error, continue waiting:`, error?.cause?.code || error?.code || error?.message ); await sleep(pollInterval); continue; } throw error; } console.log( `[Volcano Video] Poll ${i + 1}/${maxRetries}, status: ${status.status}` ); if (status.status === "succeeded") { // 任务成功 - 火山引擎API返回格式:status.content.video_url const videoUrl = status.content?.video_url || status.output?.video_url || status.video_url; const lastFrameUrl = status.content?.last_frame_url || status.output?.last_frame_url || status.last_frame_url; if (!videoUrl) { throw new Error("视频生成成功但未返回视频URL"); } console.log("[Volcano Video] ✅ Video generated:", videoUrl); return { taskId: taskId, status: "succeeded", videoUrl: videoUrl, lastFrameUrl: lastFrameUrl, usage: status.usage, createdAt: Date.now(), updatedAt: Date.now(), }; } if (status.status === "failed") { // 任务失败 const errorMsg = status.error?.message || status.message || "未知错误"; console.error("[Volcano Video] ❌ Task failed:", errorMsg); throw new Error(`视频生成失败: ${errorMsg}`); } if (status.status === "cancelled") { throw new Error("视频生成任务已取消"); } // 继续等待 await sleep(pollInterval); } throw new Error("视频生成超时,请稍后重试"); } /** * 格式化视频响应为 Markdown */ function formatVideoResponse(result: VideoGenerationResult): string { // 只显示视频和下载按钮,不显示额外文字 return ` 下载视频`; } /** * 主函数:生成视频 */ export async function doGenerate( apiKey: string, baseURL: string, options: VideoGenerationOptions ): Promise<{ text: string; videoMetadata: any }> { try { console.log("[Volcano Video] 🎬 Starting video generation..."); console.log("[Volcano Video] Options:", { mode: options.mode, model: options.model, resolution: options.resolution, ratio: options.ratio, duration: options.duration, }); // 1. 创建任务 const { taskId, inputImageUrls } = await createVideoTask(baseURL, apiKey, options); // 2. 轮询任务状态 const result = await pollTaskStatus(baseURL, apiKey, taskId); // 3. 格式化输出 const markdown = formatVideoResponse(result); // 4. 返回视频元数据(用于保存到数据库) const videoMetadata = { taskId: result.taskId, duration: options.duration, resolution: options.resolution, ratio: options.ratio, inputImages: inputImageUrls, }; return { text: markdown, videoMetadata: { ...videoMetadata, temporaryVideoUrl: result.videoUrl, lastFrameUrl: result.lastFrameUrl, }, }; } catch (error: any) { console.error("[Volcano Video] ❌ Generation failed:", error); throw error; } } /** * 流式生成(模拟进度推送) */ export async function* doGenerateStream( apiKey: string, baseURL: string, options: VideoGenerationOptions ): AsyncGenerator { try { // 1. 创建任务(不输出进度文字) const { taskId, inputImageUrls } = await createVideoTask(baseURL, apiKey, options); // 2. 轮询任务状态(静默等待) const maxRetries = 60; const pollInterval = 10000; for (let i = 0; i < maxRetries; i++) { let status: any; try { status = await fetchTaskStatus(baseURL, apiKey, taskId); } catch (error: any) { if (isRetryableFetchError(error)) { console.warn( `[Volcano Video] Stream poll ${i + 1}/${maxRetries} transient error, continue waiting:`, error?.cause?.code || error?.code || error?.message ); await sleep(pollInterval); continue; } throw error; } if (status.status === "succeeded") { // 火山引擎API返回格式:status.content.video_url const videoUrl = status.content?.video_url || status.output?.video_url || status.video_url; const lastFrameUrl = status.content?.last_frame_url || status.output?.last_frame_url || status.last_frame_url; console.log("[Volcano Video] ✅ Task succeeded!"); console.log("[Volcano Video] Video URL:", sanitizeUrlForLog(videoUrl || "")); console.log("[Volcano Video] Last frame URL:", sanitizeUrlForLog(lastFrameUrl || "")); const videoResponse = formatVideoResponse({ taskId: taskId, status: "succeeded", videoUrl: videoUrl, lastFrameUrl: lastFrameUrl, usage: status.usage, createdAt: Date.now(), updatedAt: Date.now(), }); yield videoResponse; // 输出视频元数据(JSON格式,前端可解析) yield `\n\n`; return; } if (status.status === "failed") { const errorMsg = status.error?.message || status.message || "未知错误"; throw new Error(`视频生成失败: ${errorMsg}`); } if (status.status === "cancelled") { throw new Error("视频生成任务已取消"); } // 静默等待,不输出进度文字 await sleep(pollInterval); } throw new Error("视频生成超时,请稍后重试"); } catch (error: any) { console.error("[Volcano Video] ❌ Stream generation failed:", error); yield `\n\n❌ 错误:${error.message}\n`; throw error; } }