diff --git a/lib/ai/providers/volcano-vision.ts b/lib/ai/providers/volcano-vision.ts index 4ec5d2cba..0e4447594 100644 --- a/lib/ai/providers/volcano-vision.ts +++ b/lib/ai/providers/volcano-vision.ts @@ -8,6 +8,8 @@ interface VolcanoVisionConfig { baseURL: string; } +type VolcanoImageSizeBucket = "1K" | "2K" | "4K"; + type VolcanoUrlRewriteConfig = { publicOrigin: string; endpointBase: string; @@ -20,7 +22,43 @@ const VOLCANO_INLINE_MAX_SIDE = 1024; const DEFAULT_VOLCANO_IMAGE_SIZE = "2K"; const SEEDREAM_MODEL_ID_PATTERN = /doubao-seedream-4-5/i; -function normalizeVolcanoImageSize(imageSize?: string): "1K" | "2K" | "4K" { +const VOLCANO_ASPECT_SIZE_OPTIONS: Record< + VolcanoImageSizeBucket, + Array<{ width: number; height: number }> +> = { + "1K": [ + { width: 1024, height: 1024 }, + { width: 1152, height: 864 }, + { width: 864, height: 1152 }, + { width: 1280, height: 720 }, + { width: 720, height: 1280 }, + { width: 1248, height: 832 }, + { width: 832, height: 1248 }, + { width: 1512, height: 648 }, + ], + "2K": [ + { width: 2048, height: 2048 }, + { width: 2304, height: 1728 }, + { width: 1728, height: 2304 }, + { width: 2560, height: 1440 }, + { width: 1440, height: 2560 }, + { width: 2496, height: 1664 }, + { width: 1664, height: 2496 }, + { width: 3024, height: 1296 }, + ], + "4K": [ + { width: 4096, height: 4096 }, + { width: 4704, height: 3520 }, + { width: 3520, height: 4704 }, + { width: 5504, height: 3040 }, + { width: 3040, height: 5504 }, + { width: 4992, height: 3328 }, + { width: 3328, height: 4992 }, + { width: 6240, height: 2656 }, + ], +}; + +function normalizeVolcanoImageSize(imageSize?: string): VolcanoImageSizeBucket { const normalized = String(imageSize || "").trim().toUpperCase(); if (normalized === "1K" || normalized === "2K" || normalized === "4K") { return normalized; @@ -31,8 +69,11 @@ function normalizeVolcanoImageSize(imageSize?: string): "1K" | "2K" | "4K" { function resolveVolcanoImageSizeForModel( modelId: string, imageSize?: string -): "1K" | "2K" | "4K" { - const normalized = normalizeVolcanoImageSize(imageSize); +): VolcanoImageSizeBucket { + const exactSize = normalizeVolcanoExactSize(imageSize); + const normalized = exactSize + ? inferVolcanoImageSizeBucketFromExactSize(exactSize) + : normalizeVolcanoImageSize(imageSize); if (SEEDREAM_MODEL_ID_PATTERN.test(modelId)) { // doubao-seedream-4-5 在当前链路下 1K 会返回 InvalidParameter,强制回落到 2K if (normalized === "1K") { @@ -45,6 +86,165 @@ function resolveVolcanoImageSizeForModel( return normalized; } +function inferVolcanoImageSizeBucketFromExactSize(exactSize: string): VolcanoImageSizeBucket { + const [widthRaw, heightRaw] = exactSize.split("x"); + const width = Number(widthRaw); + const height = Number(heightRaw); + const area = width * height; + if (area >= 7_000_000) return "4K"; + if (area >= 2_000_000) return "2K"; + return "1K"; +} + +function normalizeVolcanoExactSize(imageSize?: string): string | null { + const normalized = String(imageSize || "") + .trim() + .toLowerCase() + .replace(/[×*]/g, "x"); + const match = normalized.match(/^(\d{2,5})x(\d{2,5})$/); + if (!match) return null; + + const width = Number(match[1]); + const height = Number(match[2]); + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + return null; + } + + return `${width}x${height}`; +} + +function parseVolcanoAspectRatio(aspectRatio?: string): number | null { + const normalized = String(aspectRatio || "").trim().toLowerCase(); + if (!normalized || normalized === "auto") return null; + + const match = normalized.match(/^(\d+(?:\.\d+)?)\s*:\s*(\d+(?:\.\d+)?)$/); + if (!match) return null; + + const width = Number(match[1]); + const height = Number(match[2]); + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + return null; + } + + return width / height; +} + +function resolveVolcanoAspectAwareSize( + aspectRatio: string | undefined, + imageSize: VolcanoImageSizeBucket +): string | null { + const targetRatio = parseVolcanoAspectRatio(aspectRatio); + if (!targetRatio) return null; + + const candidates = VOLCANO_ASPECT_SIZE_OPTIONS[imageSize]; + const best = candidates.reduce((currentBest, option) => { + const currentDelta = Math.abs(currentBest.width / currentBest.height - targetRatio); + const optionDelta = Math.abs(option.width / option.height - targetRatio); + if (optionDelta < currentDelta) { + return option; + } + if (optionDelta === currentDelta) { + return option.width * option.height > currentBest.width * currentBest.height + ? option + : currentBest; + } + return currentBest; + }, candidates[0]); + + return `${best.width}x${best.height}`; +} + +function getFirstImageSource(image: string | string[] | undefined): string | null { + if (!image) return null; + if (Array.isArray(image)) { + return image.find((item) => typeof item === "string" && item.length > 0) || null; + } + return typeof image === "string" && image.length > 0 ? image : null; +} + +async function inferImageAspectRatioFromSource( + image: string | string[] | undefined +): Promise { + const source = getFirstImageSource(image); + if (!source) return null; + + try { + let buffer: Buffer | null = null; + + if (source.startsWith("data:image")) { + const match = source.match(/^data:[^;]+;base64,(.+)$/); + if (match?.[1]) { + buffer = Buffer.from(match[1], "base64"); + } + } else if (/^https?:\/\//i.test(source)) { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), VOLCANO_IMAGE_FETCH_TIMEOUT_MS); + try { + const response = await fetch(source, { signal: controller.signal }); + if (!response.ok) { + console.warn("[Volcano Vision] Failed to fetch reference image for aspect inference:", response.status); + return null; + } + buffer = Buffer.from(await response.arrayBuffer()); + } finally { + clearTimeout(timeoutId); + } + } + + if (!buffer || buffer.length === 0) return null; + + const sharp = await import("sharp"); + const metadata = await sharp.default(buffer).metadata(); + if (!metadata.width || !metadata.height) return null; + + return `${metadata.width}:${metadata.height}`; + } catch (error) { + console.warn("[Volcano Vision] Failed to infer reference aspect ratio:", error); + return null; + } +} + +async function resolveVolcanoRequestSize(params: { + modelId: string; + imageSize?: string; + aspectRatio?: string; + image?: string | string[]; +}): Promise<{ + size: string; + fallbackSize: VolcanoImageSizeBucket; + inferredAspectRatio: string | null; +}> { + const fallbackSize = resolveVolcanoImageSizeForModel(params.modelId, params.imageSize); + const explicitSize = normalizeVolcanoExactSize(params.imageSize); + if (explicitSize) { + return { size: explicitSize, fallbackSize, inferredAspectRatio: null }; + } + + const requestedAspectRatio = String(params.aspectRatio || "").trim(); + const inferredAspectRatio = + requestedAspectRatio === "auto" && params.image + ? await inferImageAspectRatioFromSource(params.image) + : null; + const resolvedAspectRatio = inferredAspectRatio || requestedAspectRatio; + const aspectAwareSize = resolveVolcanoAspectAwareSize(resolvedAspectRatio, fallbackSize); + + if (inferredAspectRatio) { + console.log(`[Volcano Vision] Inferred reference aspect ratio: ${inferredAspectRatio}`); + } + + if (aspectAwareSize) { + console.log( + `[Volcano Vision] Resolved size: aspectRatio=${resolvedAspectRatio}, imageSize=${fallbackSize} => ${aspectAwareSize}` + ); + return { size: aspectAwareSize, fallbackSize, inferredAspectRatio }; + } + + console.log( + `[Volcano Vision] Resolved size: aspectRatio=${resolvedAspectRatio || "default"}, imageSize=${fallbackSize} => ${fallbackSize}` + ); + return { size: fallbackSize, fallbackSize, inferredAspectRatio }; +} + function guessImageMimeTypeFromUrl(url: string): string { const cleanUrl = url.split("?")[0]?.toLowerCase() || ""; if (cleanUrl.endsWith(".png")) return "image/png"; @@ -61,6 +261,11 @@ function shouldRetryVolcanoWithInlineImage(errorText: string): boolean { ); } +function shouldRetryVolcanoWithBucketSize(errorText: string): boolean { + if (!errorText) return false; + return /invalidparameter|invalid parameter|size|resolution|dimension/i.test(errorText); +} + async function fetchImageAsDataUri(url: string): Promise { if (!/^https?:\/\//i.test(url)) return null; @@ -358,7 +563,7 @@ function formatImageResponse( // 🆕 统一使用 ||| 分隔符格式(与 Gemini 保持一致) // 格式: ![thumb|||原图URL|||尺寸](缩略图URL) - const normalizedSize = normalizeVolcanoImageSize(imageSize); + const normalizedSize = normalizeVolcanoExactSize(imageSize) || normalizeVolcanoImageSize(imageSize); return images .map((img) => `![thumb|||${img.original}|||${normalizedSize}](${img.thumbnail})`) .join("\n\n"); @@ -460,7 +665,8 @@ async function sendVolcanoRequest( messages: any[], modelId: string, maxImages: number = 3, // 默认 3 张,故障转移时传 1 - imageSize: string = DEFAULT_VOLCANO_IMAGE_SIZE + imageSize: string = DEFAULT_VOLCANO_IMAGE_SIZE, + aspectRatio?: string ): Promise { const url = `${baseURL}/api/v3/images/generations`; @@ -496,11 +702,16 @@ async function sendVolcanoRequest( console.log("[Volcano Vision] Image count:", image ? (Array.isArray(image) ? image.length : 1) : 0); // 构建请求体 - const normalizedSize = resolveVolcanoImageSizeForModel(modelId, imageSize); + const resolvedSize = await resolveVolcanoRequestSize({ + modelId, + imageSize, + aspectRatio, + image, + }); const requestBody: any = { model: modelId, prompt, - size: normalizedSize, + size: resolvedSize.size, watermark: false, response_format: "url", stream: false, @@ -591,16 +802,37 @@ async function sendVolcanoRequest( } const errorBody = (error as any)?.body || String(error || ""); - if (requestBody.image && shouldRetryVolcanoWithInlineImage(errorBody)) { + let inlineRetryBody = requestBody; + let latestError: unknown = error; + let latestErrorBody = errorBody; + + if ( + requestBody.size !== resolvedSize.fallbackSize && + shouldRetryVolcanoWithBucketSize(errorBody) + ) { + console.warn( + `[Volcano Vision] Retrying with bucket size fallback: ${requestBody.size} -> ${resolvedSize.fallbackSize}` + ); + const sizeFallbackBody = { ...requestBody, size: resolvedSize.fallbackSize }; + try { + return await executeRequest(sizeFallbackBody, "size-fallback"); + } catch (sizeError) { + latestError = sizeError; + latestErrorBody = (sizeError as any)?.body || String(sizeError || ""); + inlineRetryBody = sizeFallbackBody; + } + } + + if (inlineRetryBody.image && shouldRetryVolcanoWithInlineImage(latestErrorBody)) { console.warn("[Volcano Vision] Retrying with inline base64 image payload..."); - const inlineImage = await convertImageFieldToInlineDataUri(requestBody.image); + const inlineImage = await convertImageFieldToInlineDataUri(inlineRetryBody.image); if (inlineImage) { - const retryBody = { ...requestBody, image: inlineImage }; + const retryBody = { ...inlineRetryBody, image: inlineImage }; return await executeRequest(retryBody, "inline-retry"); } console.warn("[Volcano Vision] Inline retry skipped: failed to convert source image(s)."); } - throw error; + throw latestError; } } @@ -612,7 +844,8 @@ async function* sendVolcanoStreamRequest( apiKey: string, messages: any[], modelId: string, - imageSize: string = DEFAULT_VOLCANO_IMAGE_SIZE + imageSize: string = DEFAULT_VOLCANO_IMAGE_SIZE, + aspectRatio?: string ): AsyncGenerator { const url = `${baseURL}/api/v3/images/generations`; @@ -626,11 +859,16 @@ async function* sendVolcanoStreamRequest( console.log("[Volcano Vision] Image count:", image ? (Array.isArray(image) ? image.length : 1) : 0); // 构建请求体 - const normalizedSize = normalizeVolcanoImageSize(imageSize); + const resolvedSize = await resolveVolcanoRequestSize({ + modelId, + imageSize, + aspectRatio, + image, + }); const requestBody: any = { model: modelId, prompt, - size: normalizedSize, + size: resolvedSize.size, watermark: false, response_format: "url", stream: true, @@ -747,12 +985,14 @@ export function createVolcanoVision({ apiKey, baseURL }: VolcanoVisionConfig) { options.messages, activeModelId, maxImages, - imageSize + imageSize, + options.imageParams?.aspectRatio ); // 提取图片并上传到 S3 const imageUrls = await extractAndUploadImages(response); - const markdownText = formatImageResponse(imageUrls, imageSize); + const responseImageSize = response.data?.find((item) => item.size)?.size || imageSize; + const markdownText = formatImageResponse(imageUrls, responseImageSize); console.log( `[Volcano Vision] Generated ${imageUrls.length} images (uploaded to S3)` @@ -797,7 +1037,14 @@ export function createVolcanoVision({ apiKey, baseURL }: VolcanoVisionConfig) { modelId, options.imageParams?.imageSize ); - for await (const event of sendVolcanoStreamRequest(baseURL, apiKey, options.messages, modelId, imageSize)) { + for await (const event of sendVolcanoStreamRequest( + baseURL, + apiKey, + options.messages, + modelId, + imageSize, + options.imageParams?.aspectRatio + )) { if (event.type === "image_generation.partial_succeeded" && event.url) { // 每张图片生成完成后立即上传并发送 try {