From 810b530eb020799a120745aec7d2c8f4b148efbd Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 13 Mar 2026 16:35:55 +0800 Subject: [PATCH] fix: harden video generation fallback --- lib/ai/providers/volcano-video.ts | 398 +++++++++++++++++++++--------- lib/ai/tools/generate-video.ts | 46 +++- 2 files changed, 316 insertions(+), 128 deletions(-) diff --git a/lib/ai/providers/volcano-video.ts b/lib/ai/providers/volcano-video.ts index 889c2705d..7d2971e6a 100644 --- a/lib/ai/providers/volcano-video.ts +++ b/lib/ai/providers/volcano-video.ts @@ -11,6 +11,11 @@ import type { } from "@/types/ai"; import { uploadFile } from "@/lib/storage/service"; +const VOLCANO_MAX_INPUT_IMAGE_BYTES = 30 * 1024 * 1024; +const VOLCANO_SAFE_INPUT_IMAGE_BYTES = VOLCANO_MAX_INPUT_IMAGE_BYTES - 2 * 1024 * 1024; +const VOLCANO_OPTIMIZE_SIDE_STEPS = [3072, 2560, 2048, 1536, 1280, 1024]; +const VOLCANO_OPTIMIZE_QUALITY_STEPS = [84, 76, 68, 60]; + const RETRYABLE_HTTP_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]); const RETRYABLE_FETCH_CODES = new Set([ "UND_ERR_HEADERS_TIMEOUT", @@ -62,6 +67,90 @@ function extractPublicHttpUrl(rawValue: string): string | null { return null; } +function mimeTypeToVolcanoImageExtension(mimeType?: string | null): string { + const normalized = String(mimeType || "").toLowerCase(); + if (normalized.includes("png")) return "png"; + if (normalized.includes("webp")) return "webp"; + if (normalized.includes("gif")) return "gif"; + return "jpg"; +} + +function isVolcanoOversizeImageError(errorText: string): boolean { + const normalized = String(errorText || "").toLowerCase(); + return ( + normalized.includes("oversizeimage") || + normalized.includes("exceeds the limit") || + normalized.includes("image (") && normalized.includes("limit (") + ); +} + +async function optimizeImageBufferForVolcano( + inputBuffer: Buffer, +): Promise<{ buffer: Buffer; extension: string; mimeType: string }> { + const sharp = (await import("sharp")).default; + const metadata = await sharp(inputBuffer, { failOn: "none" }).metadata(); + const maxOriginalSide = Math.max(metadata.width || 0, metadata.height || 0); + const sideCandidates = Array.from( + new Set( + [ + maxOriginalSide > 0 ? Math.min(maxOriginalSide, VOLCANO_OPTIMIZE_SIDE_STEPS[0]!) : undefined, + ...VOLCANO_OPTIMIZE_SIDE_STEPS, + ].filter((value): value is number => Boolean(value && value > 0)) + ) + ); + + let bestBuffer = inputBuffer; + + for (const maxSide of sideCandidates) { + for (const quality of VOLCANO_OPTIMIZE_QUALITY_STEPS) { + const pipeline = sharp(inputBuffer, { failOn: "none" }) + .rotate() + .resize({ + width: maxSide, + height: maxSide, + fit: "inside", + withoutEnlargement: true, + }) + .flatten({ background: "#ffffff" }) + .jpeg({ + quality, + mozjpeg: true, + progressive: true, + }); + const optimizedBuffer = await pipeline.toBuffer(); + + if (optimizedBuffer.length < bestBuffer.length) { + bestBuffer = optimizedBuffer; + } + + if (optimizedBuffer.length <= VOLCANO_SAFE_INPUT_IMAGE_BYTES) { + return { + buffer: optimizedBuffer, + extension: "jpg", + mimeType: "image/jpeg", + }; + } + } + } + + return { + buffer: bestBuffer, + extension: "jpg", + mimeType: "image/jpeg", + }; +} + +async function uploadVolcanoPreparedImage(params: { + buffer: Buffer; + extension?: string; +}) { + const filename = `volcano-video-input-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ + params.extension || "jpg" + }`; + const { url } = await uploadFile(filename, params.buffer as any); + return url; +} + function isRetryableFetchError(error: any): boolean { const code = error?.code || error?.cause?.code; if (typeof code === "string" && RETRYABLE_FETCH_CODES.has(code)) { @@ -142,7 +231,10 @@ async function fetchWithRetry( /** * 规范化图片地址为可公开访问 URL（优先直传 URL，避免大体积 base64） */ -async function normalizeImageUrlForVolcano(imageUrl: string): Promise { +async function normalizeImageUrlForVolcano( + imageUrl: string, + options?: { forceOptimize?: boolean }, +): Promise { try { const rawInput = String(imageUrl || "").trim(); if (!rawInput) { @@ -157,17 +249,23 @@ async function normalizeImageUrlForVolcano(imageUrl: string): Promise { } const mimeType = matches[1]; - const ext = mimeType.split("/")[1]?.toLowerCase() || "jpg"; const buffer = Buffer.from(matches[2], "base64"); - const filename = `volcano-video-input-${Date.now()}.${ext}`; - const { url: permanentUrl } = await uploadFile(filename, buffer as any); + const shouldOptimize = + options?.forceOptimize === true || buffer.length > VOLCANO_SAFE_INPUT_IMAGE_BYTES; + const prepared = shouldOptimize + ? await optimizeImageBufferForVolcano(buffer) + : { + buffer, + extension: mimeTypeToVolcanoImageExtension(mimeType), + }; + const permanentUrl = await uploadVolcanoPreparedImage(prepared); return permanentUrl; } // 已经是公网 URL，直接使用 const publicHttpUrl = extractPublicHttpUrl(rawInput); - if (publicHttpUrl) { + if (publicHttpUrl && options?.forceOptimize !== true) { return publicHttpUrl; } @@ -194,9 +292,16 @@ async function normalizeImageUrlForVolcano(imageUrl: string): Promise { const buffer = await response.arrayBuffer(); const bufferNode = Buffer.from(buffer); - // 上传到 S3 获取永久 URL - const filename = `volcano-video-input-${Date.now()}.jpg`; - const { url: permanentUrl } = await uploadFile(filename, bufferNode as any); + const contentType = response.headers.get("content-type"); + const shouldOptimize = + options?.forceOptimize === true || bufferNode.length > VOLCANO_SAFE_INPUT_IMAGE_BYTES; + const prepared = shouldOptimize + ? await optimizeImageBufferForVolcano(bufferNode) + : { + buffer: bufferNode, + extension: mimeTypeToVolcanoImageExtension(contentType), + }; + const permanentUrl = await uploadVolcanoPreparedImage(prepared); console.log(`[Volcano Video] Image uploaded to S3: ${sanitizeUrlForLog(permanentUrl)}`); @@ -207,6 +312,133 @@ async function normalizeImageUrlForVolcano(imageUrl: string): Promise { } } +async function buildVolcanoTaskRequestPayload( + options: VideoGenerationOptions, + requestBody?: Record, + requestMeta?: { forceOptimizeImages?: boolean }, +) { + const content: any[] = []; + const inputImageUrls: any = {}; + const fullPrompt = buildFullPrompt(options); + + console.log("[Volcano Video] Full Prompt:", fullPrompt); + + content.push({ + type: "text", + text: fullPrompt, + }); + + if (options.mode === "first-frame" && options.firstFrameImage) { + const firstFrameUrl = await normalizeImageUrlForVolcano(options.firstFrameImage, { + forceOptimize: requestMeta?.forceOptimizeImages, + }); + inputImageUrls.firstFrame = firstFrameUrl; + + content.push({ + type: "image_url", + image_url: { + url: firstFrameUrl, + }, + role: "first_frame", + }); + } else if ( + options.mode === "first-last-frame" && + options.firstFrameImage && + options.lastFrameImage + ) { + const firstFrameUrl = await normalizeImageUrlForVolcano(options.firstFrameImage, { + forceOptimize: requestMeta?.forceOptimizeImages, + }); + const lastFrameUrl = await normalizeImageUrlForVolcano(options.lastFrameImage, { + forceOptimize: requestMeta?.forceOptimizeImages, + }); + + inputImageUrls.firstFrame = firstFrameUrl; + inputImageUrls.lastFrame = lastFrameUrl; + + content.push( + { + type: "image_url", + image_url: { + url: firstFrameUrl, + }, + role: "first_frame", + }, + { + type: "image_url", + image_url: { + url: lastFrameUrl, + }, + role: "last_frame", + }, + ); + } else if (options.mode === "reference-images" && options.referenceImages) { + inputImageUrls.references = []; + + for (const imageUrl of options.referenceImages) { + const normalizedUrl = await normalizeImageUrlForVolcano(imageUrl, { + forceOptimize: requestMeta?.forceOptimizeImages, + }); + inputImageUrls.references.push(normalizedUrl); + + content.push({ + type: "image_url", + image_url: { + url: normalizedUrl, + }, + role: "reference_image", + }); + } + } + + const nextRequestBody: any = requestBody || { + model: options.model, + }; + nextRequestBody.content = content; + + const taskTypeMap: Record = { + "text-to-video": "t2v", + "first-frame": "i2v", + "first-last-frame": "flf2v", + "reference-images": "i2v", + }; + nextRequestBody.task_type = taskTypeMap[options.mode] || "t2v"; + + if (options.returnLastFrame) { + nextRequestBody.return_last_frame = true; + } + + return { + requestBody: nextRequestBody, + inputImageUrls, + fullPrompt, + contentCount: content.length, + }; +} + +async function sendCreateVideoTaskRequest(params: { + url: string; + apiKey: string; + requestBody: Record; +}) { + return fetchWithRetry( + params.url, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${params.apiKey}`, + }, + body: JSON.stringify(params.requestBody), + }, + { + label: "create-task", + timeoutMs: 90000, + retries: 2, + } + ); +} + /** * 🎬 运镜模式映射表（转换为自然语言提示词） * 根据火山引擎API文档，运镜效果通过文本描述实现 @@ -304,107 +536,22 @@ async function createVideoTask( // https://www.volcengine.com/docs/82379/1520758 const url = `${baseURL}/contents/generations/tasks`; - // 构建 content 数组 - const content: any[] = []; - const inputImageUrls: any = {}; - - // 1. 🆕 添加文本提示词（包含运镜描述 + API参数） - const fullPrompt = buildFullPrompt(options); - - console.log("[Volcano Video] Full Prompt:", fullPrompt); - - content.push({ - type: "text", - text: fullPrompt, - }); - - // 2. 处理图片输入（根据模式） - if (options.mode === "first-frame" && options.firstFrameImage) { - // 首帧模式 - const firstFrameUrl = await normalizeImageUrlForVolcano(options.firstFrameImage); - inputImageUrls.firstFrame = firstFrameUrl; - - content.push({ - type: "image_url", - image_url: { - url: firstFrameUrl, - }, - role: "first_frame", - }); - } else if ( - options.mode === "first-last-frame" && - options.firstFrameImage && - options.lastFrameImage - ) { - // 首尾帧模式 - const firstFrameUrl = await normalizeImageUrlForVolcano(options.firstFrameImage); - const lastFrameUrl = await normalizeImageUrlForVolcano(options.lastFrameImage); - - inputImageUrls.firstFrame = firstFrameUrl; - inputImageUrls.lastFrame = lastFrameUrl; - - content.push( - { - type: "image_url", - image_url: { - url: firstFrameUrl, - }, - role: "first_frame", - }, - { - type: "image_url", - image_url: { - url: lastFrameUrl, - }, - role: "last_frame", - } - ); - } else if (options.mode === "reference-images" && options.referenceImages) { - // 参考图模式 - inputImageUrls.references = []; - - for (const imageUrl of options.referenceImages) { - const normalizedUrl = await normalizeImageUrlForVolcano(imageUrl); - inputImageUrls.references.push(normalizedUrl); - - content.push({ - type: "image_url", - image_url: { - url: normalizedUrl, - }, - role: "reference_image", - }); - } - } - // 3. 🔧 构建请求体（火山引擎API格式） - const requestBody: any = { + const baseRequestBody: any = { model: options.model, - content: content, // 直接传content数组，不使用messages包装 }; - - // 显式指定 task_type，避免服务端推断错误 - const taskTypeMap: Record = { - "text-to-video": "t2v", - "first-frame": "i2v", - "first-last-frame": "flf2v", - "reference-images": "i2v", - }; - requestBody.task_type = taskTypeMap[options.mode] || "t2v"; - - // 可选参数：返回尾帧（用于连续生成视频） - if (options.returnLastFrame) { - requestBody.return_last_frame = true; - } + let payload = await buildVolcanoTaskRequestPayload(options, baseRequestBody, { + forceOptimizeImages: false, + }); console.log("[Volcano Video] Creating task:", { model: options.model, mode: options.mode, promptLength: options.prompt?.length || 0, - fullPromptLength: fullPrompt.length, + fullPromptLength: payload.fullPrompt.length, cameraMove: options.cameraMove, - contentItems: content.length, - imageRefs: Object.values(inputImageUrls) + contentItems: payload.contentCount, + imageRefs: Object.values(payload.inputImageUrls) .flat() .map((value: any) => sanitizeUrlForLog(String(value))) .slice(0, 6), @@ -413,31 +560,46 @@ async function createVideoTask( // 4. 发送请求 console.log("[Volcano Video] Request URL:", url); - const response = await fetchWithRetry( + let response = await sendCreateVideoTaskRequest({ url, - { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify(requestBody), - }, - { - label: "create-task", - timeoutMs: 90000, - retries: 2, - } - ); + apiKey, + requestBody: payload.requestBody, + }); console.log("[Volcano Video] Response status:", response.status); if (!response.ok) { - const errorText = await response.text(); + let errorText = await response.text(); console.error("[Volcano Video] ❌ Task creation failed"); console.error("[Volcano Video] ❌ Status:", response.status); console.error("[Volcano Video] ❌ Response:", errorText); - throw new Error(`视频任务创建失败: ${response.status} ${errorText}`); + + const hasImageInputs = + Boolean(options.firstFrameImage) || + Boolean(options.lastFrameImage) || + Boolean(options.referenceImages?.length); + if (hasImageInputs && isVolcanoOversizeImageError(errorText)) { + console.warn( + "[Volcano Video] Oversize image detected, retrying with optimized uploaded images", + ); + payload = await buildVolcanoTaskRequestPayload(options, baseRequestBody, { + forceOptimizeImages: true, + }); + response = await sendCreateVideoTaskRequest({ + url, + apiKey, + requestBody: payload.requestBody, + }); + console.log("[Volcano Video] Retry response status:", response.status); + if (!response.ok) { + errorText = await response.text(); + console.error("[Volcano Video] ❌ Retry task creation failed"); + console.error("[Volcano Video] ❌ Retry response:", errorText); + throw new Error(`视频任务创建失败: ${response.status} ${errorText}`); + } + } else { + throw new Error(`视频任务创建失败: ${response.status} ${errorText}`); + } } const data = await response.json(); @@ -451,7 +613,7 @@ async function createVideoTask( return { taskId: data.id, - inputImageUrls + inputImageUrls: payload.inputImageUrls }; } diff --git a/lib/ai/tools/generate-video.ts b/lib/ai/tools/generate-video.ts index f6954786e..4d5446a81 100644 --- a/lib/ai/tools/generate-video.ts +++ b/lib/ai/tools/generate-video.ts @@ -166,7 +166,7 @@ async function getCompatibleModel( const volcanoVideoModels = models.filter( (m) => - m.provider.type === "volcano-video" && + (m.provider.type === "volcano-video" || m.provider.type === "doubao-video") && m.model.isEnabled && m.model.capabilities ); @@ -327,10 +327,17 @@ export async function executeDirectVideoGeneration( !item.model.isSubModel, ); - const parentModelId = preferredParentModelId || fallbackParentModel?.model.id; - if (!parentModelId) { - throw new Error("未找到可用的视频生成主模型"); - } + const parentModelId = + preferredParentModelId || + fallbackParentModel?.model.id || + allModels.find( + (item) => + (item.provider.type === "volcano-video" || item.provider.type === "doubao-video") && + item.model.isEnabled && + item.model.isSubModel && + item.model.parentModelId, + )?.model.parentModelId || + undefined; const generationParams = { mode: params.mode, @@ -344,11 +351,30 @@ export async function executeDirectVideoGeneration( referenceImages: params.images, } satisfies Partial; - const { model, capabilities } = await getCompatibleModel( - params.mode, - parentModelId, - generationParams, - ); + let selectedModelConfig: { model: any; capabilities: VideoModelCapabilities }; + try { + selectedModelConfig = await getCompatibleModel( + params.mode, + parentModelId, + generationParams, + ); + } catch (preferredModelError) { + if (parentModelId) { + console.warn( + "[Generate Video Tool] Preferred parent model selection failed, fallback to global compatible model:", + preferredModelError, + ); + selectedModelConfig = await getCompatibleModel( + params.mode, + undefined, + generationParams, + ); + } else { + throw preferredModelError; + } + } + + const { model, capabilities } = selectedModelConfig; validateParamsAgainstCapabilities(generationParams, capabilities);