fix: harden video generation fallback
This commit is contained in:
Your Name
parent
5a4e9fd582
commit
810b530eb0
@ -11,6 +11,11 @@ import type {
|
||||
} from "@/types/ai";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
|
||||
const VOLCANO_MAX_INPUT_IMAGE_BYTES = 30 * 1024 * 1024;
|
||||
const VOLCANO_SAFE_INPUT_IMAGE_BYTES = VOLCANO_MAX_INPUT_IMAGE_BYTES - 2 * 1024 * 1024;
|
||||
const VOLCANO_OPTIMIZE_SIDE_STEPS = [3072, 2560, 2048, 1536, 1280, 1024];
|
||||
const VOLCANO_OPTIMIZE_QUALITY_STEPS = [84, 76, 68, 60];
|
||||
|
||||
const RETRYABLE_HTTP_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]);
|
||||
const RETRYABLE_FETCH_CODES = new Set([
|
||||
"UND_ERR_HEADERS_TIMEOUT",
|
||||
@ -62,6 +67,90 @@ function extractPublicHttpUrl(rawValue: string): string | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
function mimeTypeToVolcanoImageExtension(mimeType?: string | null): string {
|
||||
const normalized = String(mimeType || "").toLowerCase();
|
||||
if (normalized.includes("png")) return "png";
|
||||
if (normalized.includes("webp")) return "webp";
|
||||
if (normalized.includes("gif")) return "gif";
|
||||
return "jpg";
|
||||
}
|
||||
|
||||
function isVolcanoOversizeImageError(errorText: string): boolean {
|
||||
const normalized = String(errorText || "").toLowerCase();
|
||||
return (
|
||||
normalized.includes("oversizeimage") ||
|
||||
normalized.includes("exceeds the limit") ||
|
||||
normalized.includes("image (") && normalized.includes("limit (")
|
||||
);
|
||||
}
|
||||
|
||||
async function optimizeImageBufferForVolcano(
|
||||
inputBuffer: Buffer,
|
||||
): Promise<{ buffer: Buffer; extension: string; mimeType: string }> {
|
||||
const sharp = (await import("sharp")).default;
|
||||
const metadata = await sharp(inputBuffer, { failOn: "none" }).metadata();
|
||||
const maxOriginalSide = Math.max(metadata.width || 0, metadata.height || 0);
|
||||
const sideCandidates = Array.from(
|
||||
new Set(
|
||||
[
|
||||
maxOriginalSide > 0 ? Math.min(maxOriginalSide, VOLCANO_OPTIMIZE_SIDE_STEPS[0]!) : undefined,
|
||||
...VOLCANO_OPTIMIZE_SIDE_STEPS,
|
||||
].filter((value): value is number => Boolean(value && value > 0))
|
||||
)
|
||||
);
|
||||
|
||||
let bestBuffer = inputBuffer;
|
||||
|
||||
for (const maxSide of sideCandidates) {
|
||||
for (const quality of VOLCANO_OPTIMIZE_QUALITY_STEPS) {
|
||||
const pipeline = sharp(inputBuffer, { failOn: "none" })
|
||||
.rotate()
|
||||
.resize({
|
||||
width: maxSide,
|
||||
height: maxSide,
|
||||
fit: "inside",
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.flatten({ background: "#ffffff" })
|
||||
.jpeg({
|
||||
quality,
|
||||
mozjpeg: true,
|
||||
progressive: true,
|
||||
});
|
||||
const optimizedBuffer = await pipeline.toBuffer();
|
||||
|
||||
if (optimizedBuffer.length < bestBuffer.length) {
|
||||
bestBuffer = optimizedBuffer;
|
||||
}
|
||||
|
||||
if (optimizedBuffer.length <= VOLCANO_SAFE_INPUT_IMAGE_BYTES) {
|
||||
return {
|
||||
buffer: optimizedBuffer,
|
||||
extension: "jpg",
|
||||
mimeType: "image/jpeg",
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
buffer: bestBuffer,
|
||||
extension: "jpg",
|
||||
mimeType: "image/jpeg",
|
||||
};
|
||||
}
|
||||
|
||||
async function uploadVolcanoPreparedImage(params: {
|
||||
buffer: Buffer;
|
||||
extension?: string;
|
||||
}) {
|
||||
const filename = `volcano-video-input-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${
|
||||
params.extension || "jpg"
|
||||
}`;
|
||||
const { url } = await uploadFile(filename, params.buffer as any);
|
||||
return url;
|
||||
}
|
||||
|
||||
function isRetryableFetchError(error: any): boolean {
|
||||
const code = error?.code || error?.cause?.code;
|
||||
if (typeof code === "string" && RETRYABLE_FETCH_CODES.has(code)) {
|
||||
@ -142,7 +231,10 @@ async function fetchWithRetry(
|
||||
/**
|
||||
* 规范化图片地址为可公开访问 URL（优先直传 URL，避免大体积 base64）
|
||||
*/
|
||||
async function normalizeImageUrlForVolcano(imageUrl: string): Promise<string> {
|
||||
async function normalizeImageUrlForVolcano(
|
||||
imageUrl: string,
|
||||
options?: { forceOptimize?: boolean },
|
||||
): Promise<string> {
|
||||
try {
|
||||
const rawInput = String(imageUrl || "").trim();
|
||||
if (!rawInput) {
|
||||
@ -157,17 +249,23 @@ async function normalizeImageUrlForVolcano(imageUrl: string): Promise<string> {
|
||||
}
|
||||
|
||||
const mimeType = matches[1];
|
||||
const ext = mimeType.split("/")[1]?.toLowerCase() || "jpg";
|
||||
const buffer = Buffer.from(matches[2], "base64");
|
||||
const filename = `volcano-video-input-${Date.now()}.${ext}`;
|
||||
const { url: permanentUrl } = await uploadFile(filename, buffer as any);
|
||||
const shouldOptimize =
|
||||
options?.forceOptimize === true || buffer.length > VOLCANO_SAFE_INPUT_IMAGE_BYTES;
|
||||
const prepared = shouldOptimize
|
||||
? await optimizeImageBufferForVolcano(buffer)
|
||||
: {
|
||||
buffer,
|
||||
extension: mimeTypeToVolcanoImageExtension(mimeType),
|
||||
};
|
||||
const permanentUrl = await uploadVolcanoPreparedImage(prepared);
|
||||
|
||||
return permanentUrl;
|
||||
}
|
||||
|
||||
// 已经是公网 URL，直接使用
|
||||
const publicHttpUrl = extractPublicHttpUrl(rawInput);
|
||||
if (publicHttpUrl) {
|
||||
if (publicHttpUrl && options?.forceOptimize !== true) {
|
||||
return publicHttpUrl;
|
||||
}
|
||||
|
||||
@ -194,9 +292,16 @@ async function normalizeImageUrlForVolcano(imageUrl: string): Promise<string> {
|
||||
const buffer = await response.arrayBuffer();
|
||||
const bufferNode = Buffer.from(buffer);
|
||||
|
||||
// 上传到 S3 获取永久 URL
|
||||
const filename = `volcano-video-input-${Date.now()}.jpg`;
|
||||
const { url: permanentUrl } = await uploadFile(filename, bufferNode as any);
|
||||
const contentType = response.headers.get("content-type");
|
||||
const shouldOptimize =
|
||||
options?.forceOptimize === true || bufferNode.length > VOLCANO_SAFE_INPUT_IMAGE_BYTES;
|
||||
const prepared = shouldOptimize
|
||||
? await optimizeImageBufferForVolcano(bufferNode)
|
||||
: {
|
||||
buffer: bufferNode,
|
||||
extension: mimeTypeToVolcanoImageExtension(contentType),
|
||||
};
|
||||
const permanentUrl = await uploadVolcanoPreparedImage(prepared);
|
||||
|
||||
console.log(`[Volcano Video] Image uploaded to S3: ${sanitizeUrlForLog(permanentUrl)}`);
|
||||
|
||||
@ -207,6 +312,133 @@ async function normalizeImageUrlForVolcano(imageUrl: string): Promise<string> {
|
||||
}
|
||||
}
|
||||
|
||||
async function buildVolcanoTaskRequestPayload(
|
||||
options: VideoGenerationOptions,
|
||||
requestBody?: Record<string, any>,
|
||||
requestMeta?: { forceOptimizeImages?: boolean },
|
||||
) {
|
||||
const content: any[] = [];
|
||||
const inputImageUrls: any = {};
|
||||
const fullPrompt = buildFullPrompt(options);
|
||||
|
||||
console.log("[Volcano Video] Full Prompt:", fullPrompt);
|
||||
|
||||
content.push({
|
||||
type: "text",
|
||||
text: fullPrompt,
|
||||
});
|
||||
|
||||
if (options.mode === "first-frame" && options.firstFrameImage) {
|
||||
const firstFrameUrl = await normalizeImageUrlForVolcano(options.firstFrameImage, {
|
||||
forceOptimize: requestMeta?.forceOptimizeImages,
|
||||
});
|
||||
inputImageUrls.firstFrame = firstFrameUrl;
|
||||
|
||||
content.push({
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: firstFrameUrl,
|
||||
},
|
||||
role: "first_frame",
|
||||
});
|
||||
} else if (
|
||||
options.mode === "first-last-frame" &&
|
||||
options.firstFrameImage &&
|
||||
options.lastFrameImage
|
||||
) {
|
||||
const firstFrameUrl = await normalizeImageUrlForVolcano(options.firstFrameImage, {
|
||||
forceOptimize: requestMeta?.forceOptimizeImages,
|
||||
});
|
||||
const lastFrameUrl = await normalizeImageUrlForVolcano(options.lastFrameImage, {
|
||||
forceOptimize: requestMeta?.forceOptimizeImages,
|
||||
});
|
||||
|
||||
inputImageUrls.firstFrame = firstFrameUrl;
|
||||
inputImageUrls.lastFrame = lastFrameUrl;
|
||||
|
||||
content.push(
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: firstFrameUrl,
|
||||
},
|
||||
role: "first_frame",
|
||||
},
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: lastFrameUrl,
|
||||
},
|
||||
role: "last_frame",
|
||||
},
|
||||
);
|
||||
} else if (options.mode === "reference-images" && options.referenceImages) {
|
||||
inputImageUrls.references = [];
|
||||
|
||||
for (const imageUrl of options.referenceImages) {
|
||||
const normalizedUrl = await normalizeImageUrlForVolcano(imageUrl, {
|
||||
forceOptimize: requestMeta?.forceOptimizeImages,
|
||||
});
|
||||
inputImageUrls.references.push(normalizedUrl);
|
||||
|
||||
content.push({
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: normalizedUrl,
|
||||
},
|
||||
role: "reference_image",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const nextRequestBody: any = requestBody || {
|
||||
model: options.model,
|
||||
};
|
||||
nextRequestBody.content = content;
|
||||
|
||||
const taskTypeMap: Record<string, string> = {
|
||||
"text-to-video": "t2v",
|
||||
"first-frame": "i2v",
|
||||
"first-last-frame": "flf2v",
|
||||
"reference-images": "i2v",
|
||||
};
|
||||
nextRequestBody.task_type = taskTypeMap[options.mode] || "t2v";
|
||||
|
||||
if (options.returnLastFrame) {
|
||||
nextRequestBody.return_last_frame = true;
|
||||
}
|
||||
|
||||
return {
|
||||
requestBody: nextRequestBody,
|
||||
inputImageUrls,
|
||||
fullPrompt,
|
||||
contentCount: content.length,
|
||||
};
|
||||
}
|
||||
|
||||
async function sendCreateVideoTaskRequest(params: {
|
||||
url: string;
|
||||
apiKey: string;
|
||||
requestBody: Record<string, any>;
|
||||
}) {
|
||||
return fetchWithRetry(
|
||||
params.url,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${params.apiKey}`,
|
||||
},
|
||||
body: JSON.stringify(params.requestBody),
|
||||
},
|
||||
{
|
||||
label: "create-task",
|
||||
timeoutMs: 90000,
|
||||
retries: 2,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 🎬 运镜模式映射表（转换为自然语言提示词）
|
||||
* 根据火山引擎API文档，运镜效果通过文本描述实现
|
||||
@ -304,107 +536,22 @@ async function createVideoTask(
|
||||
// https://www.volcengine.com/docs/82379/1520758
|
||||
const url = `${baseURL}/contents/generations/tasks`;
|
||||
|
||||
// 构建 content 数组
|
||||
const content: any[] = [];
|
||||
const inputImageUrls: any = {};
|
||||
|
||||
// 1. 🆕 添加文本提示词（包含运镜描述 + API参数）
|
||||
const fullPrompt = buildFullPrompt(options);
|
||||
|
||||
console.log("[Volcano Video] Full Prompt:", fullPrompt);
|
||||
|
||||
content.push({
|
||||
type: "text",
|
||||
text: fullPrompt,
|
||||
});
|
||||
|
||||
// 2. 处理图片输入（根据模式）
|
||||
if (options.mode === "first-frame" && options.firstFrameImage) {
|
||||
// 首帧模式
|
||||
const firstFrameUrl = await normalizeImageUrlForVolcano(options.firstFrameImage);
|
||||
inputImageUrls.firstFrame = firstFrameUrl;
|
||||
|
||||
content.push({
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: firstFrameUrl,
|
||||
},
|
||||
role: "first_frame",
|
||||
});
|
||||
} else if (
|
||||
options.mode === "first-last-frame" &&
|
||||
options.firstFrameImage &&
|
||||
options.lastFrameImage
|
||||
) {
|
||||
// 首尾帧模式
|
||||
const firstFrameUrl = await normalizeImageUrlForVolcano(options.firstFrameImage);
|
||||
const lastFrameUrl = await normalizeImageUrlForVolcano(options.lastFrameImage);
|
||||
|
||||
inputImageUrls.firstFrame = firstFrameUrl;
|
||||
inputImageUrls.lastFrame = lastFrameUrl;
|
||||
|
||||
content.push(
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: firstFrameUrl,
|
||||
},
|
||||
role: "first_frame",
|
||||
},
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: lastFrameUrl,
|
||||
},
|
||||
role: "last_frame",
|
||||
}
|
||||
);
|
||||
} else if (options.mode === "reference-images" && options.referenceImages) {
|
||||
// 参考图模式
|
||||
inputImageUrls.references = [];
|
||||
|
||||
for (const imageUrl of options.referenceImages) {
|
||||
const normalizedUrl = await normalizeImageUrlForVolcano(imageUrl);
|
||||
inputImageUrls.references.push(normalizedUrl);
|
||||
|
||||
content.push({
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: normalizedUrl,
|
||||
},
|
||||
role: "reference_image",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 🔧 构建请求体（火山引擎API格式）
|
||||
const requestBody: any = {
|
||||
const baseRequestBody: any = {
|
||||
model: options.model,
|
||||
content: content, // 直接传content数组，不使用messages包装
|
||||
};
|
||||
|
||||
// 显式指定 task_type，避免服务端推断错误
|
||||
const taskTypeMap: Record<string, string> = {
|
||||
"text-to-video": "t2v",
|
||||
"first-frame": "i2v",
|
||||
"first-last-frame": "flf2v",
|
||||
"reference-images": "i2v",
|
||||
};
|
||||
requestBody.task_type = taskTypeMap[options.mode] || "t2v";
|
||||
|
||||
// 可选参数：返回尾帧（用于连续生成视频）
|
||||
if (options.returnLastFrame) {
|
||||
requestBody.return_last_frame = true;
|
||||
}
|
||||
let payload = await buildVolcanoTaskRequestPayload(options, baseRequestBody, {
|
||||
forceOptimizeImages: false,
|
||||
});
|
||||
|
||||
console.log("[Volcano Video] Creating task:", {
|
||||
model: options.model,
|
||||
mode: options.mode,
|
||||
promptLength: options.prompt?.length || 0,
|
||||
fullPromptLength: fullPrompt.length,
|
||||
fullPromptLength: payload.fullPrompt.length,
|
||||
cameraMove: options.cameraMove,
|
||||
contentItems: content.length,
|
||||
imageRefs: Object.values(inputImageUrls)
|
||||
contentItems: payload.contentCount,
|
||||
imageRefs: Object.values(payload.inputImageUrls)
|
||||
.flat()
|
||||
.map((value: any) => sanitizeUrlForLog(String(value)))
|
||||
.slice(0, 6),
|
||||
@ -413,31 +560,46 @@ async function createVideoTask(
|
||||
// 4. 发送请求
|
||||
console.log("[Volcano Video] Request URL:", url);
|
||||
|
||||
const response = await fetchWithRetry(
|
||||
let response = await sendCreateVideoTaskRequest({
|
||||
url,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify(requestBody),
|
||||
},
|
||||
{
|
||||
label: "create-task",
|
||||
timeoutMs: 90000,
|
||||
retries: 2,
|
||||
}
|
||||
);
|
||||
apiKey,
|
||||
requestBody: payload.requestBody,
|
||||
});
|
||||
|
||||
console.log("[Volcano Video] Response status:", response.status);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
let errorText = await response.text();
|
||||
console.error("[Volcano Video] ❌ Task creation failed");
|
||||
console.error("[Volcano Video] ❌ Status:", response.status);
|
||||
console.error("[Volcano Video] ❌ Response:", errorText);
|
||||
throw new Error(`视频任务创建失败: ${response.status} ${errorText}`);
|
||||
|
||||
const hasImageInputs =
|
||||
Boolean(options.firstFrameImage) ||
|
||||
Boolean(options.lastFrameImage) ||
|
||||
Boolean(options.referenceImages?.length);
|
||||
if (hasImageInputs && isVolcanoOversizeImageError(errorText)) {
|
||||
console.warn(
|
||||
"[Volcano Video] Oversize image detected, retrying with optimized uploaded images",
|
||||
);
|
||||
payload = await buildVolcanoTaskRequestPayload(options, baseRequestBody, {
|
||||
forceOptimizeImages: true,
|
||||
});
|
||||
response = await sendCreateVideoTaskRequest({
|
||||
url,
|
||||
apiKey,
|
||||
requestBody: payload.requestBody,
|
||||
});
|
||||
console.log("[Volcano Video] Retry response status:", response.status);
|
||||
if (!response.ok) {
|
||||
errorText = await response.text();
|
||||
console.error("[Volcano Video] ❌ Retry task creation failed");
|
||||
console.error("[Volcano Video] ❌ Retry response:", errorText);
|
||||
throw new Error(`视频任务创建失败: ${response.status} ${errorText}`);
|
||||
}
|
||||
} else {
|
||||
throw new Error(`视频任务创建失败: ${response.status} ${errorText}`);
|
||||
}
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
@ -451,7 +613,7 @@ async function createVideoTask(
|
||||
|
||||
return {
|
||||
taskId: data.id,
|
||||
inputImageUrls
|
||||
inputImageUrls: payload.inputImageUrls
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -166,7 +166,7 @@ async function getCompatibleModel(
|
||||
|
||||
const volcanoVideoModels = models.filter(
|
||||
(m) =>
|
||||
m.provider.type === "volcano-video" &&
|
||||
(m.provider.type === "volcano-video" || m.provider.type === "doubao-video") &&
|
||||
m.model.isEnabled &&
|
||||
m.model.capabilities
|
||||
);
|
||||
@ -327,10 +327,17 @@ export async function executeDirectVideoGeneration(
|
||||
!item.model.isSubModel,
|
||||
);
|
||||
|
||||
const parentModelId = preferredParentModelId || fallbackParentModel?.model.id;
|
||||
if (!parentModelId) {
|
||||
throw new Error("未找到可用的视频生成主模型");
|
||||
}
|
||||
const parentModelId =
|
||||
preferredParentModelId ||
|
||||
fallbackParentModel?.model.id ||
|
||||
allModels.find(
|
||||
(item) =>
|
||||
(item.provider.type === "volcano-video" || item.provider.type === "doubao-video") &&
|
||||
item.model.isEnabled &&
|
||||
item.model.isSubModel &&
|
||||
item.model.parentModelId,
|
||||
)?.model.parentModelId ||
|
||||
undefined;
|
||||
|
||||
const generationParams = {
|
||||
mode: params.mode,
|
||||
@ -344,11 +351,30 @@ export async function executeDirectVideoGeneration(
|
||||
referenceImages: params.images,
|
||||
} satisfies Partial<VideoGenerationOptions>;
|
||||
|
||||
const { model, capabilities } = await getCompatibleModel(
|
||||
params.mode,
|
||||
parentModelId,
|
||||
generationParams,
|
||||
);
|
||||
let selectedModelConfig: { model: any; capabilities: VideoModelCapabilities };
|
||||
try {
|
||||
selectedModelConfig = await getCompatibleModel(
|
||||
params.mode,
|
||||
parentModelId,
|
||||
generationParams,
|
||||
);
|
||||
} catch (preferredModelError) {
|
||||
if (parentModelId) {
|
||||
console.warn(
|
||||
"[Generate Video Tool] Preferred parent model selection failed, fallback to global compatible model:",
|
||||
preferredModelError,
|
||||
);
|
||||
selectedModelConfig = await getCompatibleModel(
|
||||
params.mode,
|
||||
undefined,
|
||||
generationParams,
|
||||
);
|
||||
} else {
|
||||
throw preferredModelError;
|
||||
}
|
||||
}
|
||||
|
||||
const { model, capabilities } = selectedModelConfig;
|
||||
|
||||
validateParamsAgainstCapabilities(generationParams, capabilities);
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user