import "server-only"; import { uploadFile } from "@/lib/storage/service"; import { getDefaultStorageConfig } from "@/lib/db/queries"; import { createLogger } from "@/lib/observability/logger"; const logger = createLogger("volcano-vision"); interface VolcanoVisionConfig { apiKey: string; baseURL: string; } type VolcanoImageSizeBucket = "1K" | "2K" | "4K"; type VolcanoUrlRewriteConfig = { publicOrigin: string; endpointBase: string; bucket: string; }; let cachedRewriteConfig: VolcanoUrlRewriteConfig | null | undefined; const VOLCANO_IMAGE_FETCH_TIMEOUT_MS = 20_000; const VOLCANO_INLINE_MAX_SIDE = 1024; const DEFAULT_VOLCANO_IMAGE_SIZE = "2K"; const SEEDREAM_MODEL_ID_PATTERN = /doubao-seedream-4-5/i; const VOLCANO_ASPECT_SIZE_OPTIONS: Record< VolcanoImageSizeBucket, Array<{ width: number; height: number }> > = { "1K": [ { width: 1024, height: 1024 }, { width: 1152, height: 864 }, { width: 864, height: 1152 }, { width: 1280, height: 720 }, { width: 720, height: 1280 }, { width: 1248, height: 832 }, { width: 832, height: 1248 }, { width: 1512, height: 648 }, ], "2K": [ { width: 2048, height: 2048 }, { width: 2304, height: 1728 }, { width: 1728, height: 2304 }, { width: 2560, height: 1440 }, { width: 1440, height: 2560 }, { width: 2496, height: 1664 }, { width: 1664, height: 2496 }, { width: 3024, height: 1296 }, ], "4K": [ { width: 4096, height: 4096 }, { width: 4704, height: 3520 }, { width: 3520, height: 4704 }, { width: 5504, height: 3040 }, { width: 3040, height: 5504 }, { width: 4992, height: 3328 }, { width: 3328, height: 4992 }, { width: 6240, height: 2656 }, ], }; function normalizeVolcanoImageSize(imageSize?: string): VolcanoImageSizeBucket { const normalized = String(imageSize || "").trim().toUpperCase(); if (normalized === "1K" || normalized === "2K" || normalized === "4K") { return normalized; } return DEFAULT_VOLCANO_IMAGE_SIZE; } function resolveVolcanoImageSizeForModel( modelId: string, imageSize?: string ): VolcanoImageSizeBucket { const exactSize = normalizeVolcanoExactSize(imageSize); const normalized = exactSize ? inferVolcanoImageSizeBucketFromExactSize(exactSize) : normalizeVolcanoImageSize(imageSize); if (SEEDREAM_MODEL_ID_PATTERN.test(modelId)) { // doubao-seedream-4-5 在当前链路下 1K 会返回 InvalidParameter,强制回落到 2K if (normalized === "1K") { logger.debug( `[Volcano Vision] Size normalized for ${modelId}: 1K -> 2K (model constraint)` ); return "2K"; } } return normalized; } function inferVolcanoImageSizeBucketFromExactSize(exactSize: string): VolcanoImageSizeBucket { const [widthRaw, heightRaw] = exactSize.split("x"); const width = Number(widthRaw); const height = Number(heightRaw); const area = width * height; if (area >= 7_000_000) return "4K"; if (area >= 2_000_000) return "2K"; return "1K"; } function normalizeVolcanoExactSize(imageSize?: string): string | null { const normalized = String(imageSize || "") .trim() .toLowerCase() .replace(/[×*]/g, "x"); const match = normalized.match(/^(\d{2,5})x(\d{2,5})$/); if (!match) return null; const width = Number(match[1]); const height = Number(match[2]); if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { return null; } return `${width}x${height}`; } function parseVolcanoAspectRatio(aspectRatio?: string): number | null { const normalized = String(aspectRatio || "").trim().toLowerCase(); if (!normalized || normalized === "auto") return null; const match = normalized.match(/^(\d+(?:\.\d+)?)\s*:\s*(\d+(?:\.\d+)?)$/); if (!match) return null; const width = Number(match[1]); const height = Number(match[2]); if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { return null; } return width / height; } function resolveVolcanoAspectAwareSize( aspectRatio: string | undefined, imageSize: VolcanoImageSizeBucket ): string | null { const targetRatio = parseVolcanoAspectRatio(aspectRatio); if (!targetRatio) return null; const candidates = VOLCANO_ASPECT_SIZE_OPTIONS[imageSize]; const best = candidates.reduce((currentBest, option) => { const currentDelta = Math.abs(currentBest.width / currentBest.height - targetRatio); const optionDelta = Math.abs(option.width / option.height - targetRatio); if (optionDelta < currentDelta) { return option; } if (optionDelta === currentDelta) { return option.width * option.height > currentBest.width * currentBest.height ? option : currentBest; } return currentBest; }, candidates[0]); return `${best.width}x${best.height}`; } function getFirstImageSource(image: string | string[] | undefined): string | null { if (!image) return null; if (Array.isArray(image)) { return image.find((item) => typeof item === "string" && item.length > 0) || null; } return typeof image === "string" && image.length > 0 ? image : null; } async function inferImageAspectRatioFromSource( image: string | string[] | undefined ): Promise { const source = getFirstImageSource(image); if (!source) return null; try { let buffer: Buffer | null = null; if (source.startsWith("data:image")) { const match = source.match(/^data:[^;]+;base64,(.+)$/); if (match?.[1]) { buffer = Buffer.from(match[1], "base64"); } } else if (/^https?:\/\//i.test(source)) { const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), VOLCANO_IMAGE_FETCH_TIMEOUT_MS); try { const response = await fetch(source, { signal: controller.signal }); if (!response.ok) { logger.warn("[Volcano Vision] Failed to fetch reference image for aspect inference:", response.status); return null; } buffer = Buffer.from(await response.arrayBuffer()); } finally { clearTimeout(timeoutId); } } if (!buffer || buffer.length === 0) return null; const sharp = await import("sharp"); const metadata = await sharp.default(buffer).metadata(); if (!metadata.width || !metadata.height) return null; return `${metadata.width}:${metadata.height}`; } catch (error) { logger.warn("[Volcano Vision] Failed to infer reference aspect ratio:", error); return null; } } async function resolveVolcanoRequestSize(params: { modelId: string; imageSize?: string; aspectRatio?: string; image?: string | string[]; }): Promise<{ size: string; fallbackSize: VolcanoImageSizeBucket; inferredAspectRatio: string | null; }> { const fallbackSize = resolveVolcanoImageSizeForModel(params.modelId, params.imageSize); const explicitSize = normalizeVolcanoExactSize(params.imageSize); if (explicitSize) { return { size: explicitSize, fallbackSize, inferredAspectRatio: null }; } const requestedAspectRatio = String(params.aspectRatio || "").trim(); const inferredAspectRatio = requestedAspectRatio === "auto" && params.image ? await inferImageAspectRatioFromSource(params.image) : null; const resolvedAspectRatio = inferredAspectRatio || requestedAspectRatio; const aspectAwareSize = resolveVolcanoAspectAwareSize(resolvedAspectRatio, fallbackSize); if (inferredAspectRatio) { logger.debug(`[Volcano Vision] Inferred reference aspect ratio: ${inferredAspectRatio}`); } if (aspectAwareSize) { logger.debug( `[Volcano Vision] Resolved size: aspectRatio=${resolvedAspectRatio}, imageSize=${fallbackSize} => ${aspectAwareSize}` ); return { size: aspectAwareSize, fallbackSize, inferredAspectRatio }; } logger.debug( `[Volcano Vision] Resolved size: aspectRatio=${resolvedAspectRatio || "default"}, imageSize=${fallbackSize} => ${fallbackSize}` ); return { size: fallbackSize, fallbackSize, inferredAspectRatio }; } function guessImageMimeTypeFromUrl(url: string): string { const cleanUrl = url.split("?")[0]?.toLowerCase() || ""; if (cleanUrl.endsWith(".png")) return "image/png"; if (cleanUrl.endsWith(".webp")) return "image/webp"; if (cleanUrl.endsWith(".gif")) return "image/gif"; if (cleanUrl.endsWith(".bmp")) return "image/bmp"; return "image/jpeg"; } function shouldRetryVolcanoWithInlineImage(errorText: string): boolean { if (!errorText) return false; return /timeout while downloading url|download multimodal file timed out|invalidparameter/i.test( errorText ); } function shouldRetryVolcanoWithBucketSize(errorText: string): boolean { if (!errorText) return false; return /invalidparameter|invalid parameter|size|resolution|dimension/i.test(errorText); } async function fetchImageAsDataUri(url: string): Promise { if (!/^https?:\/\//i.test(url)) return null; const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), VOLCANO_IMAGE_FETCH_TIMEOUT_MS); try { const response = await fetch(url, { signal: controller.signal }); if (!response.ok) { logger.warn("[Volcano Vision] Failed to fetch source image for inline retry:", response.status, url); return null; } const arrayBuffer = await response.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); if (buffer.length === 0) return null; const inputMime = response.headers.get("content-type") || guessImageMimeTypeFromUrl(url); // 优先保留原图,避免 base64 回退路径带来明显画质损失 if (buffer.length <= 6 * 1024 * 1024) { return `data:${inputMime};base64,${buffer.toString("base64")}`; } try { const sharp = await import("sharp"); const optimized = await sharp.default(buffer) .rotate() .resize(VOLCANO_INLINE_MAX_SIDE, VOLCANO_INLINE_MAX_SIDE, { fit: "inside", withoutEnlargement: true, }) .jpeg({ quality: 90, mozjpeg: true }) .toBuffer(); return `data:image/jpeg;base64,${optimized.toString("base64")}`; } catch { return `data:${inputMime};base64,${buffer.toString("base64")}`; } } catch (error) { logger.warn("[Volcano Vision] Inline image fetch failed:", error); return null; } finally { clearTimeout(timeoutId); } } async function convertImageFieldToInlineDataUri( image: string | string[] | undefined ): Promise { if (!image) return image; const list = Array.isArray(image) ? image : [image]; const converted = await Promise.all( list.map(async (item) => { if (typeof item !== "string") return null; if (item.startsWith("data:image")) return item; return fetchImageAsDataUri(item); }) ); const ready = converted.filter((item): item is string => typeof item === "string" && item.length > 0); if (ready.length === 0) return undefined; return Array.isArray(image) ? ready : ready[0]; } async function getVolcanoUrlRewriteConfig(): Promise { if (cachedRewriteConfig !== undefined) { return cachedRewriteConfig; } try { const storageConfig = await getDefaultStorageConfig(); if ( !storageConfig || storageConfig.type !== "s3" || !storageConfig.endpoint || !storageConfig.bucket || !storageConfig.publicUrl ) { cachedRewriteConfig = null; return null; } const publicOrigin = new URL(storageConfig.publicUrl).origin; const endpointBase = storageConfig.endpoint.replace(/\/+$/, ""); cachedRewriteConfig = { publicOrigin, endpointBase, bucket: storageConfig.bucket, }; return cachedRewriteConfig; } catch { cachedRewriteConfig = null; return null; } } async function rewriteUploadPublicUrlForVolcano(url: string): Promise { if (!/^https?:\/\//i.test(url)) return url; const config = await getVolcanoUrlRewriteConfig(); if (!config) return url; try { const parsed = new URL(url); if (parsed.origin !== config.publicOrigin) return url; if (!parsed.pathname.startsWith("/uploads/")) return url; const rewritten = `${config.endpointBase}/${config.bucket}${parsed.pathname}${parsed.search}`; logger.debug("[Volcano Vision] Rewrote upload URL for provider fetch:", rewritten); return rewritten; } catch { return url; } } /** * 压缩图片生成缩略图(使用 sharp) */ async function createThumbnail(buffer: Buffer, mimeType: string): Promise { try { const sharp = await import("sharp"); // 创建缩略图(宽度 500px,高度按比例,质量 80%) const thumbnailBuffer = await sharp.default(buffer) .resize(500, null, { fit: "inside", withoutEnlargement: true, }) .jpeg({ quality: 80 }) .toBuffer(); logger.debug(`[Volcano Vision] Thumbnail created, size: ${thumbnailBuffer.length} bytes (original: ${buffer.length} bytes)`); return thumbnailBuffer; } catch (error) { logger.error("[Volcano Vision] Failed to create thumbnail:", error); // 如果失败,返回原图 return buffer; } } /** * 将图片 URL 或 base64 上传到 S3,返回缩略图和原图的 URL */ async function uploadImageToS3(imageData: string): Promise<{ thumbnail: string; original: string }> { try { let buffer: Buffer; let mimeType = "image/jpeg"; // 检查是否是 base64 if (imageData.startsWith("data:")) { const matches = imageData.match(/^data:([^;]+);base64,(.+)$/); if (matches) { mimeType = matches[1]; buffer = Buffer.from(matches[2], "base64"); } else { throw new Error("Invalid base64 format"); } } else { // 是 URL,下载图片 logger.debug(`[Volcano Vision] Downloading image from URL...`); const response = await fetch(imageData); if (!response.ok) { throw new Error(`Failed to download image: ${response.statusText}`); } const arrayBuffer = await response.arrayBuffer(); buffer = Buffer.from(arrayBuffer); mimeType = response.headers.get("content-type") || "image/jpeg"; } // 生成文件名 const ext = mimeType.split("/")[1] || "jpg"; const timestamp = Date.now(); const randomStr = Math.random().toString(36).slice(2, 8); const thumbnailFilename = `volcano-thumb-${timestamp}-${randomStr}.${ext}`; const originalFilename = `volcano-orig-${timestamp}-${randomStr}.${ext}`; logger.debug(`[Volcano Vision] Original size: ${buffer.length} bytes`); // 生成缩略图 const thumbnailBuffer = await createThumbnail(buffer, mimeType); // 转换为 ArrayBuffer const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice( thumbnailBuffer.byteOffset, thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength ) as ArrayBuffer; const originalArrayBuffer = buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; // 并行上传缩略图和原图到 S3 logger.debug(`[Volcano Vision] Uploading thumbnail and original to S3...`); const [thumbnailResult, originalResult] = await Promise.all([ uploadFile(thumbnailFilename, thumbnailArrayBuffer), uploadFile(originalFilename, originalArrayBuffer), ]); logger.debug(`[Volcano Vision] Uploaded to S3:`); logger.debug(` - Thumbnail: ${thumbnailResult.url}`); logger.debug(` - Original: ${originalResult.url}`); return { thumbnail: thumbnailResult.url, original: originalResult.url, }; } catch (error) { logger.error("[Volcano Vision] Failed to upload to S3:", error); throw new Error(`图片上传到存储桶失败: ${error instanceof Error ? error.message : "未知错误"}。请检查系统设置中的存储配置。`); } } /** * 火山引擎 API 响应格式 */ interface VolcanoVisionResponse { model: string; created: number; data: Array<{ url?: string; b64_json?: string; size: string; }>; usage: { generated_images: number; output_tokens: number; total_tokens: number; }; error?: { code: string; message: string; }; } /** * 火山引擎流式响应事件 */ interface VolcanoVisionStreamEvent { type: "image_generation.partial_succeeded" | "image_generation.completed"; model: string; created: number; image_index?: number; url?: string; size?: string; usage?: { generated_images: number; output_tokens: number; total_tokens: number; }; } /** * 从 API 响应中提取图片并上传到 S3 */ async function extractAndUploadImages(response: VolcanoVisionResponse): Promise> { const images: Array<{ thumbnail: string; original: string }> = []; const uploadPromises: Promise[] = []; if (response.data && response.data.length > 0) { for (const item of response.data) { if (item.url) { // URL 格式 const uploadPromise = uploadImageToS3(item.url) .then(urls => { images.push(urls); }); uploadPromises.push(uploadPromise); } else if (item.b64_json) { // Base64 格式 const base64Data = `data:image/jpeg;base64,${item.b64_json}`; const uploadPromise = uploadImageToS3(base64Data) .then(urls => { images.push(urls); }); uploadPromises.push(uploadPromise); } } } // 等待所有上传完成 await Promise.all(uploadPromises); logger.debug("[Volcano Vision] Extracted and uploaded", images.length, "images (thumbnail + original)"); return images; } /** * 将图片数据转换为 Markdown 格式 * 使用特殊格式传递缩略图和原图 URL:![thumb|原图URL](缩略图URL) */ function formatImageResponse( images: Array<{ thumbnail: string; original: string }>, imageSize: string = DEFAULT_VOLCANO_IMAGE_SIZE ): string { if (images.length === 0) { return "生成图片失败,未找到图片数据"; } // 🆕 统一使用 ||| 分隔符格式(与 Gemini 保持一致) // 格式: ![thumb|||原图URL|||尺寸](缩略图URL) const normalizedSize = normalizeVolcanoExactSize(imageSize) || normalizeVolcanoImageSize(imageSize); return images .map((img) => `![thumb|||${img.original}|||${normalizedSize}](${img.thumbnail})`) .join("\n\n"); } /** * 将消息转换为火山引擎 API 格式 */ async function convertMessagesToVolcanoFormat(messages: any[]): Promise<{ prompt: string; image?: string | string[] }> { const lastMessage = messages[messages.length - 1]; let prompt = ""; const images: string[] = []; logger.debug("[Volcano Vision] Converting messages, last message:", JSON.stringify(lastMessage).substring(0, 500)); // 提取文本和图片 if (lastMessage.content) { if (typeof lastMessage.content === "string") { prompt = lastMessage.content; } else if (Array.isArray(lastMessage.content)) { for (const part of lastMessage.content) { logger.debug("[Volcano Vision] Processing part:", JSON.stringify(part).substring(0, 150)); if (part.type === "text") { prompt += part.text; } else if (part.type === "image_url") { // OpenAI 格式:{ type: "image_url", image_url: { url: "..." } } if (part.image_url && part.image_url.url) { images.push(part.image_url.url); logger.debug("[Volcano Vision] ✅ Found image_url:", part.image_url.url.substring(0, 100) + "..."); } } else if (part.type === "file") { // 文件格式:{ type: "file", mediaType: "image/...", data: "url" } logger.debug("[Volcano Vision] File part - mediaType:", part.mediaType, "data exists:", !!part.data); if (part.data && part.mediaType && part.mediaType.startsWith("image/")) { images.push(part.data); logger.debug("[Volcano Vision] ✅ Found file (image):", part.data.substring(0, 100) + "..."); } else { logger.debug("[Volcano Vision] ❌ File part skipped - mediaType:", part.mediaType, "data:", !!part.data); } } else if (part.type === "image") { // 其他格式:{ type: "image", image: "..." } 或 { type: "image", url: "..." } if (part.image) { images.push(part.image); logger.debug("[Volcano Vision] ✅ Found image:", part.image.substring(0, 100) + "..."); } else if (part.url) { images.push(part.url); logger.debug("[Volcano Vision] ✅ Found image url:", part.url.substring(0, 100) + "..."); } } } } } logger.debug("[Volcano Vision] Extracted prompt:", prompt.substring(0, 100)); logger.debug("[Volcano Vision] Extracted images count:", images.length); // 🆕 直接使用图片URL,不再转换Base64(既梦API支持URL格式) // 只有当图片是Base64格式时才保持原样 const processedImages: string[] = []; for (const imageUrl of images) { if (imageUrl.startsWith('data:image')) { // Base64格式:需要先上传到S3获取URL logger.debug("[Volcano Vision] Found Base64 image, uploading to S3..."); const uploaded = await uploadImageToS3(imageUrl); const rewritten = await rewriteUploadPublicUrlForVolcano(uploaded.original); processedImages.push(rewritten); } else { // URL格式:直接使用 const rewritten = await rewriteUploadPublicUrlForVolcano(imageUrl); processedImages.push(rewritten); logger.debug("[Volcano Vision] Using image URL directly:", rewritten.substring(0, 80) + "..."); } } // 不在这里添加"生成多个变体"的提示,由调用方根据maxImages决定 // 这样可以避免在场景裂变等只需要1张图片的场景中生成多张图片 const enhancedPrompt = prompt; logger.debug("[Volcano Vision] Prompt:", enhancedPrompt.substring(0, 150)); // 构建请求体 const result: { prompt: string; image?: string | string[] } = { prompt: enhancedPrompt }; if (processedImages.length === 1) { result.image = processedImages[0]; } else if (processedImages.length > 1) { result.image = processedImages; // 多图直接传数组 } return result; } /** * 发送火山引擎图像生成请求(非流式) */ async function sendVolcanoRequest( baseURL: string, apiKey: string, messages: any[], modelId: string, maxImages: number = 3, // 默认 3 张,故障转移时传 1 imageSize: string = DEFAULT_VOLCANO_IMAGE_SIZE, aspectRatio?: string ): Promise { const url = `${baseURL}/api/v3/images/generations`; logger.debug("[Volcano Vision] Sending non-stream request to:", url); logger.debug("[Volcano Vision] Using model:", modelId); // 转换消息格式(直接使用URL) let { prompt, image } = await convertMessagesToVolcanoFormat(messages); // 根据 maxImages 参数优化提示词 if (maxImages === 1) { // 只生成1张图片时,明确指示只生成1张,避免生成多张 if (!prompt.includes("只生成") && !prompt.includes("生成1张") && !prompt.includes("生成一张")) { prompt = prompt.trim(); // 移除末尾的句号(如果有),然后添加明确指令 if (prompt.endsWith("。") || prompt.endsWith(".")) { prompt = prompt.slice(0, -1); } prompt += "。只生成1张图片。"; } } else if (maxImages > 1) { // 生成多张图片时,明确要求生成多个变体 if (!prompt.includes("生成") || (!prompt.includes("变体") && !prompt.includes("不同"))) { prompt = prompt.trim(); if (prompt.endsWith("。") || prompt.endsWith(".")) { prompt = prompt.slice(0, -1); } prompt += `。请生成 ${maxImages} 个不同的变体。`; } } logger.debug("[Volcano Vision] Prompt:", prompt.substring(0, 100) + "..."); logger.debug("[Volcano Vision] Image count:", image ? (Array.isArray(image) ? image.length : 1) : 0); // 构建请求体 const resolvedSize = await resolveVolcanoRequestSize({ modelId, imageSize, aspectRatio, image, }); const requestBody: any = { model: modelId, prompt, size: resolvedSize.size, watermark: false, response_format: "url", stream: false, }; // 添加图片(如果有) if (image) { requestBody.image = image; } // 根据 maxImages 参数配置组图功能 if (maxImages > 1) { requestBody.sequential_image_generation = "auto"; requestBody.sequential_image_generation_options = { max_images: maxImages, }; logger.debug(`[Volcano Vision] Generating up to ${maxImages} images (mode: ${image ? "image-to-image" : "text-to-image"})`); } else { requestBody.sequential_image_generation = "disabled"; logger.debug(`[Volcano Vision] Generating 1 image (mode: ${image ? "image-to-image" : "text-to-image"})`); } logger.debug("[Volcano Vision] Request body:", JSON.stringify(requestBody).substring(0, 500) + "..."); logger.debug("[Volcano Vision] Full request body (prompt + image info):"); logger.debug(" - Prompt:", requestBody.prompt); logger.debug(" - Image field exists:", !!requestBody.image); logger.debug(" - Image type:", typeof requestBody.image); logger.debug(" - Image is array:", Array.isArray(requestBody.image)); logger.debug(" - Image count:", Array.isArray(requestBody.image) ? requestBody.image.length : (requestBody.image ? 1 : 0)); if (requestBody.image) { if (Array.isArray(requestBody.image)) { requestBody.image.forEach((img: string, idx: number) => { logger.debug(` - Image ${idx}: ${img.substring(0, 100)}...`); }); } else { logger.debug(` - Image: ${requestBody.image.substring(0, 100)}...`); } } const executeRequest = async (payload: any, tag: string): Promise => { const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 600000); try { const response = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, }, body: JSON.stringify(payload), signal: controller.signal, }); if (!response.ok) { const errorText = await response.text(); logger.error(`[Volcano Vision] Request failed (${tag}):`, response.status, errorText); const err = new Error( `Volcano Vision API error: ${response.status} ${response.statusText} - ${errorText}` ) as Error & { status?: number; body?: string }; err.status = response.status; err.body = errorText; throw err; } const data = await response.json(); logger.debug( `[Volcano Vision] Response received (${tag}):`, JSON.stringify(data).substring(0, 200) + "..." ); if (data.error) { throw new Error(`API Error: ${data.error.message || data.error.code}`); } return data as VolcanoVisionResponse; } finally { clearTimeout(timeoutId); } }; try { return await executeRequest(requestBody, "primary"); } catch (error) { if (error instanceof Error && error.name === "AbortError") { logger.error("[Volcano Vision] Request timeout after 10 minutes"); throw new Error("请求超时,图片处理时间过长,请稍后重试"); } const errorBody = (error as any)?.body || String(error || ""); let inlineRetryBody = requestBody; let latestError: unknown = error; let latestErrorBody = errorBody; if ( requestBody.size !== resolvedSize.fallbackSize && shouldRetryVolcanoWithBucketSize(errorBody) ) { logger.warn( `[Volcano Vision] Retrying with bucket size fallback: ${requestBody.size} -> ${resolvedSize.fallbackSize}` ); const sizeFallbackBody = { ...requestBody, size: resolvedSize.fallbackSize }; try { return await executeRequest(sizeFallbackBody, "size-fallback"); } catch (sizeError) { latestError = sizeError; latestErrorBody = (sizeError as any)?.body || String(sizeError || ""); inlineRetryBody = sizeFallbackBody; } } if (inlineRetryBody.image && shouldRetryVolcanoWithInlineImage(latestErrorBody)) { logger.warn("[Volcano Vision] Retrying with inline base64 image payload..."); const inlineImage = await convertImageFieldToInlineDataUri(inlineRetryBody.image); if (inlineImage) { const retryBody = { ...inlineRetryBody, image: inlineImage }; return await executeRequest(retryBody, "inline-retry"); } logger.warn("[Volcano Vision] Inline retry skipped: failed to convert source image(s)."); } throw latestError; } } /** * 发送火山引擎图像生成请求(流式) */ async function* sendVolcanoStreamRequest( baseURL: string, apiKey: string, messages: any[], modelId: string, imageSize: string = DEFAULT_VOLCANO_IMAGE_SIZE, aspectRatio?: string ): AsyncGenerator { const url = `${baseURL}/api/v3/images/generations`; logger.debug("[Volcano Vision] Sending stream request to:", url); logger.debug("[Volcano Vision] Using model:", modelId); // 转换消息格式(直接使用URL) const { prompt, image } = await convertMessagesToVolcanoFormat(messages); logger.debug("[Volcano Vision] Prompt:", prompt.substring(0, 100) + "..."); logger.debug("[Volcano Vision] Image count:", image ? (Array.isArray(image) ? image.length : 1) : 0); // 构建请求体 const resolvedSize = await resolveVolcanoRequestSize({ modelId, imageSize, aspectRatio, image, }); const requestBody: any = { model: modelId, prompt, size: resolvedSize.size, watermark: false, response_format: "url", stream: true, }; // 添加图片(如果有) if (image) { requestBody.image = image; } logger.debug("[Volcano Vision] Stream request body:", JSON.stringify(requestBody).substring(0, 500) + "..."); logger.debug("[Volcano Vision] Stream - Full request info:"); logger.debug(" - Prompt:", requestBody.prompt); logger.debug(" - Image field exists:", !!requestBody.image); logger.debug(" - Image type:", typeof requestBody.image); logger.debug(" - Image is array:", Array.isArray(requestBody.image)); logger.debug(" - Image count:", Array.isArray(requestBody.image) ? requestBody.image.length : (requestBody.image ? 1 : 0)); // 设置 10 分钟超时 const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 600000); try { const response = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, }, body: JSON.stringify(requestBody), signal: controller.signal, }); clearTimeout(timeoutId); if (!response.ok) { const errorText = await response.text(); logger.error("[Volcano Vision] Stream request failed:", response.status, errorText); throw new Error( `Volcano Vision API error: ${response.status} ${response.statusText}` ); } // 解析 SSE 流 const reader = response.body?.getReader(); if (!reader) { throw new Error("Failed to get response reader"); } const decoder = new TextDecoder(); let buffer = ""; while (true) { const { done, value } = await reader.read(); if (done) break; buffer += decoder.decode(value, { stream: true }); const lines = buffer.split("\n"); buffer = lines.pop() || ""; for (const line of lines) { const trimmedLine = line.trim(); if (!trimmedLine || trimmedLine === "data: [DONE]") continue; if (trimmedLine.startsWith("data: ")) { const jsonStr = trimmedLine.slice(6); try { const event = JSON.parse(jsonStr) as VolcanoVisionStreamEvent; logger.debug("[Volcano Vision] Stream event:", event.type, event.image_index); yield event; } catch (error) { logger.error("[Volcano Vision] Failed to parse stream event:", jsonStr); } } } } } catch (error) { clearTimeout(timeoutId); if (error instanceof Error && error.name === "AbortError") { logger.error("[Volcano Vision] Stream request timeout after 10 minutes"); throw new Error("请求超时,图片处理时间过长,请稍后重试"); } throw error; } } /** * 创建火山引擎视觉 Provider */ export function createVolcanoVision({ apiKey, baseURL }: VolcanoVisionConfig) { return { chat: (modelId: string) => ({ doGenerate: async (options: any) => { logger.debug("[Volcano Vision] doGenerate called!"); logger.debug("[Volcano Vision] Messages received:", JSON.stringify(options.messages).substring(0, 500) + "..."); const activeModelId = options.overrideModelId || modelId; logger.debug("[Volcano Vision] Using modelId:", activeModelId); logger.debug("[Volcano Vision] Image params from options:", options.imageParams); const imageSize = resolveVolcanoImageSizeForModel( activeModelId, options.imageParams?.imageSize ); // 🆕 优先使用用户选择的图片数量,其次是 maxImages 参数,默认 1 张 const imageCount = options.imageParams?.imageCount || options.maxImages || 1; const maxImages = Math.min(Math.max(imageCount, 1), 3); // 限制 1-3 张 logger.debug("[Volcano Vision] Max images:", maxImages); try { // 发送请求 const response = await sendVolcanoRequest( baseURL, apiKey, options.messages, activeModelId, maxImages, imageSize, options.imageParams?.aspectRatio ); // 提取图片并上传到 S3 const imageUrls = await extractAndUploadImages(response); const responseImageSize = response.data?.find((item) => item.size)?.size || imageSize; const markdownText = formatImageResponse(imageUrls, responseImageSize); logger.debug( `[Volcano Vision] Generated ${imageUrls.length} images (uploaded to S3)` ); // 返回 AI SDK v1 格式 return { text: markdownText, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: markdownText.length, }, }; } catch (error) { logger.error("[Volcano Vision] Generation failed:", error); // 返回错误信息 const errorMessage = error instanceof Error ? `图片生成失败: ${error.message}` : "图片生成失败,请稍后重试"; return { text: errorMessage, finishReason: "error" as const, usage: { promptTokens: 0, completionTokens: 0 }, }; } }, doStream: async function* (options: any) { logger.debug("[Volcano Vision] ===== doStream ENTERED! ====="); logger.debug("[Volcano Vision] Messages received:", JSON.stringify(options.messages).substring(0, 500) + "..."); logger.debug("[Volcano Vision] Using modelId:", modelId); try { const allImages: Array<{ thumbnail: string; original: string }> = []; // 使用流式请求 const imageSize = resolveVolcanoImageSizeForModel( modelId, options.imageParams?.imageSize ); for await (const event of sendVolcanoStreamRequest( baseURL, apiKey, options.messages, modelId, imageSize, options.imageParams?.aspectRatio )) { if (event.type === "image_generation.partial_succeeded" && event.url) { // 每张图片生成完成后立即上传并发送 try { const imageUrls = await uploadImageToS3(event.url); allImages.push(imageUrls); const markdown = `![thumb|||${imageUrls.original}|||${imageSize}](${imageUrls.thumbnail})`; logger.debug(`[Volcano Vision Stream] Image ${event.image_index} uploaded, sending...`); yield { type: "text-delta" as const, textDelta: markdown + "\n\n" }; } catch (error) { logger.error(`[Volcano Vision Stream] Failed to upload image ${event.image_index}:`, error); } } else if (event.type === "image_generation.completed") { logger.debug(`[Volcano Vision Stream] Generation completed, total images: ${allImages.length}`); // 发送完成信号 yield { type: "finish" as const, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: allImages.length * 50, // 估算 }, }; } } } catch (error) { logger.error("[Volcano Vision Stream] Generation failed:", error); const errorMessage = error instanceof Error ? `图片生成失败: ${error.message}` : "图片生成失败,请稍后重试"; // 发送错误信息 yield { type: "text-delta" as const, textDelta: errorMessage }; yield { type: "finish" as const, finishReason: "error" as const, usage: { promptTokens: 0, completionTokens: 0 }, }; } }, modelId, provider: "volcano-vision", specificationVersion: "v1" as const, defaultObjectGenerationMode: "json" as const, }), }; }