import "server-only"; import { uploadFile } from "@/lib/storage/service"; interface VolcanoVisionConfig { apiKey: string; baseURL: string; } /** * 压缩图片生成缩略图(使用 sharp) */ async function createThumbnail(buffer: Buffer, mimeType: string): Promise { try { const sharp = await import("sharp"); // 创建缩略图(宽度 500px,高度按比例,质量 80%) const thumbnailBuffer = await sharp.default(buffer) .resize(500, null, { fit: "inside", withoutEnlargement: true, }) .jpeg({ quality: 80 }) .toBuffer(); console.log(`[Volcano Vision] Thumbnail created, size: ${thumbnailBuffer.length} bytes (original: ${buffer.length} bytes)`); return thumbnailBuffer; } catch (error) { console.error("[Volcano Vision] Failed to create thumbnail:", error); // 如果失败,返回原图 return buffer; } } /** * 将图片 URL 或 base64 上传到 S3,返回缩略图和原图的 URL */ async function uploadImageToS3(imageData: string): Promise<{ thumbnail: string; original: string }> { try { let buffer: Buffer; let mimeType = "image/jpeg"; // 检查是否是 base64 if (imageData.startsWith("data:")) { const matches = imageData.match(/^data:([^;]+);base64,(.+)$/); if (matches) { mimeType = matches[1]; buffer = Buffer.from(matches[2], "base64"); } else { throw new Error("Invalid base64 format"); } } else { // 是 URL,下载图片 console.log(`[Volcano Vision] Downloading image from URL...`); const response = await fetch(imageData); if (!response.ok) { throw new Error(`Failed to download image: ${response.statusText}`); } const arrayBuffer = await response.arrayBuffer(); buffer = Buffer.from(arrayBuffer); mimeType = response.headers.get("content-type") || "image/jpeg"; } // 生成文件名 const ext = mimeType.split("/")[1] || "jpg"; const timestamp = Date.now(); const randomStr = Math.random().toString(36).slice(2, 8); const thumbnailFilename = `volcano-thumb-${timestamp}-${randomStr}.${ext}`; const originalFilename = `volcano-orig-${timestamp}-${randomStr}.${ext}`; console.log(`[Volcano Vision] Original size: ${buffer.length} bytes`); // 生成缩略图 const thumbnailBuffer = await createThumbnail(buffer, mimeType); // 转换为 ArrayBuffer const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice( thumbnailBuffer.byteOffset, thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength ) as ArrayBuffer; const originalArrayBuffer = buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; // 并行上传缩略图和原图到 S3 console.log(`[Volcano Vision] Uploading thumbnail and original to S3...`); const [thumbnailResult, originalResult] = await Promise.all([ uploadFile(thumbnailFilename, thumbnailArrayBuffer), uploadFile(originalFilename, originalArrayBuffer), ]); console.log(`[Volcano Vision] Uploaded to S3:`); console.log(` - Thumbnail: ${thumbnailResult.url}`); console.log(` - Original: ${originalResult.url}`); return { thumbnail: thumbnailResult.url, original: originalResult.url, }; } catch (error) { console.error("[Volcano Vision] Failed to upload to S3:", error); throw new Error(`图片上传到存储桶失败: ${error instanceof Error ? error.message : "未知错误"}。请检查系统设置中的存储配置。`); } } /** * 火山引擎 API 响应格式 */ interface VolcanoVisionResponse { model: string; created: number; data: Array<{ url?: string; b64_json?: string; size: string; }>; usage: { generated_images: number; output_tokens: number; total_tokens: number; }; error?: { code: string; message: string; }; } /** * 火山引擎流式响应事件 */ interface VolcanoVisionStreamEvent { type: "image_generation.partial_succeeded" | "image_generation.completed"; model: string; created: number; image_index?: number; url?: string; size?: string; usage?: { generated_images: number; output_tokens: number; total_tokens: number; }; } /** * 从 API 响应中提取图片并上传到 S3 */ async function extractAndUploadImages(response: VolcanoVisionResponse): Promise> { const images: Array<{ thumbnail: string; original: string }> = []; const uploadPromises: Promise[] = []; if (response.data && response.data.length > 0) { for (const item of response.data) { if (item.url) { // URL 格式 const uploadPromise = uploadImageToS3(item.url) .then(urls => { images.push(urls); }); uploadPromises.push(uploadPromise); } else if (item.b64_json) { // Base64 格式 const base64Data = `data:image/jpeg;base64,${item.b64_json}`; const uploadPromise = uploadImageToS3(base64Data) .then(urls => { images.push(urls); }); uploadPromises.push(uploadPromise); } } } // 等待所有上传完成 await Promise.all(uploadPromises); console.log("[Volcano Vision] Extracted and uploaded", images.length, "images (thumbnail + original)"); return images; } /** * 将图片数据转换为 Markdown 格式 * 使用特殊格式传递缩略图和原图 URL:![thumb|原图URL](缩略图URL) */ function formatImageResponse(images: Array<{ thumbnail: string; original: string }>): string { if (images.length === 0) { return "生成图片失败,未找到图片数据"; } // 🆕 统一使用 ||| 分隔符格式(与 Gemini 保持一致) // 格式: ![thumb|||原图URL|||尺寸](缩略图URL) // 豆包固定生成 2K 图片 return images .map((img) => `![thumb|||${img.original}|||2K](${img.thumbnail})`) .join("\n\n"); } /** * 下载图片并转换为 Base64 */ async function urlToBase64(url: string): Promise { try { console.log("[Volcano Vision] Converting image to Base64:", url.substring(0, 100) + "..."); const response = await fetch(url); if (!response.ok) { throw new Error(`Failed to fetch image: ${response.status} ${response.statusText}`); } const arrayBuffer = await response.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); const base64 = buffer.toString('base64'); // 从 URL 推断图片格式 let format = 'jpeg'; // 默认 if (url.includes('.png')) format = 'png'; else if (url.includes('.webp')) format = 'webp'; else if (url.includes('.gif')) format = 'gif'; else if (url.includes('.jpg') || url.includes('.jpeg')) format = 'jpeg'; const result = `data:image/${format};base64,${base64}`; console.log("[Volcano Vision] ✅ Converted to Base64 (format:", format, ", size:", (base64.length / 1024).toFixed(2), "KB)"); return result; } catch (error) { console.error("[Volcano Vision] ❌ Failed to convert image to Base64:", error); throw error; } } /** * 将消息转换为火山引擎 API 格式 */ async function convertMessagesToVolcanoFormat(messages: any[]): Promise<{ prompt: string; image?: string | string[] }> { const lastMessage = messages[messages.length - 1]; let prompt = ""; const images: string[] = []; console.log("[Volcano Vision] Converting messages, last message:", JSON.stringify(lastMessage).substring(0, 500)); // 提取文本和图片 if (lastMessage.content) { if (typeof lastMessage.content === "string") { prompt = lastMessage.content; } else if (Array.isArray(lastMessage.content)) { for (const part of lastMessage.content) { console.log("[Volcano Vision] Processing part:", JSON.stringify(part).substring(0, 150)); if (part.type === "text") { prompt += part.text; } else if (part.type === "image_url") { // OpenAI 格式:{ type: "image_url", image_url: { url: "..." } } if (part.image_url && part.image_url.url) { images.push(part.image_url.url); console.log("[Volcano Vision] ✅ Found image_url:", part.image_url.url.substring(0, 100) + "..."); } } else if (part.type === "file") { // 文件格式:{ type: "file", mediaType: "image/...", data: "url" } console.log("[Volcano Vision] File part - mediaType:", part.mediaType, "data exists:", !!part.data); if (part.data && part.mediaType && part.mediaType.startsWith("image/")) { images.push(part.data); console.log("[Volcano Vision] ✅ Found file (image):", part.data.substring(0, 100) + "..."); } else { console.log("[Volcano Vision] ❌ File part skipped - mediaType:", part.mediaType, "data:", !!part.data); } } else if (part.type === "image") { // 其他格式:{ type: "image", image: "..." } 或 { type: "image", url: "..." } if (part.image) { images.push(part.image); console.log("[Volcano Vision] ✅ Found image:", part.image.substring(0, 100) + "..."); } else if (part.url) { images.push(part.url); console.log("[Volcano Vision] ✅ Found image url:", part.url.substring(0, 100) + "..."); } } } } } console.log("[Volcano Vision] Extracted prompt:", prompt.substring(0, 100)); console.log("[Volcano Vision] Extracted images count:", images.length); // 将所有图片 URL 转换为 Base64 const base64Images: string[] = []; for (const imageUrl of images) { // 如果已经是 Base64 格式,直接使用 if (imageUrl.startsWith('data:image')) { base64Images.push(imageUrl); } else { // 否则下载并转换 const base64Image = await urlToBase64(imageUrl); base64Images.push(base64Image); } } // 强制要求生成 3 张图片:在提示词后添加明确指令 let enhancedPrompt = prompt; if (base64Images.length > 0) { // 图生图:明确要求生成 3 个变体 enhancedPrompt += "。请生成 3 个不同的变体。"; } else { // 文生图:明确要求生成 3 张不同图片 enhancedPrompt += "。请生成 3 张不同风格的图片。"; } console.log("[Volcano Vision] Enhanced prompt:", enhancedPrompt.substring(0, 150)); // 构建请求体 const result: { prompt: string; image?: string | string[] } = { prompt: enhancedPrompt }; if (base64Images.length === 1) { result.image = base64Images[0]; } else if (base64Images.length > 1) { result.image = base64Images; // 多图直接传数组 } return result; } /** * 发送火山引擎图像生成请求(非流式) */ async function sendVolcanoRequest( baseURL: string, apiKey: string, messages: any[], modelId: string, maxImages: number = 3 // 默认 3 张,故障转移时传 1 ): Promise { const url = `${baseURL}/api/v3/images/generations`; console.log("[Volcano Vision] Sending non-stream request to:", url); console.log("[Volcano Vision] Using model:", modelId); // 转换消息格式(含 Base64 转换) const { prompt, image } = await convertMessagesToVolcanoFormat(messages); console.log("[Volcano Vision] Prompt:", prompt.substring(0, 100) + "..."); console.log("[Volcano Vision] Image count:", image ? (Array.isArray(image) ? image.length : 1) : 0); // 构建请求体 const requestBody: any = { model: modelId, prompt, size: "2K", watermark: false, response_format: "url", stream: false, }; // 添加图片(如果有) if (image) { requestBody.image = image; } // 根据 maxImages 参数配置组图功能 if (maxImages > 1) { requestBody.sequential_image_generation = "auto"; requestBody.sequential_image_generation_options = { max_images: maxImages, }; console.log(`[Volcano Vision] Generating up to ${maxImages} images (mode: ${image ? "image-to-image" : "text-to-image"})`); } else { requestBody.sequential_image_generation = "disabled"; console.log(`[Volcano Vision] Generating 1 image (mode: ${image ? "image-to-image" : "text-to-image"})`); } console.log("[Volcano Vision] Request body:", JSON.stringify(requestBody).substring(0, 500) + "..."); console.log("[Volcano Vision] Full request body (prompt + image info):"); console.log(" - Prompt:", requestBody.prompt); console.log(" - Image field exists:", !!requestBody.image); console.log(" - Image type:", typeof requestBody.image); console.log(" - Image is array:", Array.isArray(requestBody.image)); console.log(" - Image count:", Array.isArray(requestBody.image) ? requestBody.image.length : (requestBody.image ? 1 : 0)); if (requestBody.image) { if (Array.isArray(requestBody.image)) { requestBody.image.forEach((img: string, idx: number) => { console.log(` - Image ${idx}: ${img.substring(0, 100)}...`); }); } else { console.log(` - Image: ${requestBody.image.substring(0, 100)}...`); } } // 设置 10 分钟超时 const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 600000); try { const response = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, }, body: JSON.stringify(requestBody), signal: controller.signal, }); clearTimeout(timeoutId); if (!response.ok) { const errorText = await response.text(); console.error("[Volcano Vision] Request failed:", response.status, errorText); throw new Error( `Volcano Vision API error: ${response.status} ${response.statusText}` ); } const data = await response.json(); console.log( "[Volcano Vision] Response received:", JSON.stringify(data).substring(0, 200) + "..." ); // 检查是否有错误 if (data.error) { throw new Error(`API Error: ${data.error.message || data.error.code}`); } return data as VolcanoVisionResponse; } catch (error) { clearTimeout(timeoutId); if (error instanceof Error && error.name === "AbortError") { console.error("[Volcano Vision] Request timeout after 10 minutes"); throw new Error("请求超时,图片处理时间过长,请稍后重试"); } throw error; } } /** * 发送火山引擎图像生成请求(流式) */ async function* sendVolcanoStreamRequest( baseURL: string, apiKey: string, messages: any[], modelId: string ): AsyncGenerator { const url = `${baseURL}/api/v3/images/generations`; console.log("[Volcano Vision] Sending stream request to:", url); console.log("[Volcano Vision] Using model:", modelId); // 转换消息格式(含 Base64 转换) const { prompt, image } = await convertMessagesToVolcanoFormat(messages); console.log("[Volcano Vision] Prompt:", prompt.substring(0, 100) + "..."); console.log("[Volcano Vision] Image count:", image ? (Array.isArray(image) ? image.length : 1) : 0); // 构建请求体 const requestBody: any = { model: modelId, prompt, size: "2K", watermark: false, response_format: "url", stream: true, }; // 添加图片(如果有) if (image) { requestBody.image = image; } console.log("[Volcano Vision] Stream request body:", JSON.stringify(requestBody).substring(0, 500) + "..."); console.log("[Volcano Vision] Stream - Full request info:"); console.log(" - Prompt:", requestBody.prompt); console.log(" - Image field exists:", !!requestBody.image); console.log(" - Image type:", typeof requestBody.image); console.log(" - Image is array:", Array.isArray(requestBody.image)); console.log(" - Image count:", Array.isArray(requestBody.image) ? requestBody.image.length : (requestBody.image ? 1 : 0)); // 设置 10 分钟超时 const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 600000); try { const response = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, }, body: JSON.stringify(requestBody), signal: controller.signal, }); clearTimeout(timeoutId); if (!response.ok) { const errorText = await response.text(); console.error("[Volcano Vision] Stream request failed:", response.status, errorText); throw new Error( `Volcano Vision API error: ${response.status} ${response.statusText}` ); } // 解析 SSE 流 const reader = response.body?.getReader(); if (!reader) { throw new Error("Failed to get response reader"); } const decoder = new TextDecoder(); let buffer = ""; while (true) { const { done, value } = await reader.read(); if (done) break; buffer += decoder.decode(value, { stream: true }); const lines = buffer.split("\n"); buffer = lines.pop() || ""; for (const line of lines) { const trimmedLine = line.trim(); if (!trimmedLine || trimmedLine === "data: [DONE]") continue; if (trimmedLine.startsWith("data: ")) { const jsonStr = trimmedLine.slice(6); try { const event = JSON.parse(jsonStr) as VolcanoVisionStreamEvent; console.log("[Volcano Vision] Stream event:", event.type, event.image_index); yield event; } catch (error) { console.error("[Volcano Vision] Failed to parse stream event:", jsonStr); } } } } } catch (error) { clearTimeout(timeoutId); if (error instanceof Error && error.name === "AbortError") { console.error("[Volcano Vision] Stream request timeout after 10 minutes"); throw new Error("请求超时,图片处理时间过长,请稍后重试"); } throw error; } } /** * 创建火山引擎视觉 Provider */ export function createVolcanoVision({ apiKey, baseURL }: VolcanoVisionConfig) { return { chat: (modelId: string) => ({ doGenerate: async (options: any) => { console.log("[Volcano Vision] doGenerate called!"); console.log("[Volcano Vision] Messages received:", JSON.stringify(options.messages).substring(0, 500) + "..."); console.log("[Volcano Vision] Using modelId:", modelId); console.log("[Volcano Vision] Image params from options:", options.imageParams); // 🆕 优先使用用户选择的图片数量,其次是 maxImages 参数,默认 1 张 const imageCount = options.imageParams?.imageCount || options.maxImages || 1; const maxImages = Math.min(Math.max(imageCount, 1), 3); // 限制 1-3 张 console.log("[Volcano Vision] Max images:", maxImages); try { // 发送请求 const response = await sendVolcanoRequest(baseURL, apiKey, options.messages, modelId, maxImages); // 提取图片并上传到 S3 const imageUrls = await extractAndUploadImages(response); const markdownText = formatImageResponse(imageUrls); console.log( `[Volcano Vision] Generated ${imageUrls.length} images (uploaded to S3)` ); // 返回 AI SDK v1 格式 return { text: markdownText, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: markdownText.length, }, }; } catch (error) { console.error("[Volcano Vision] Generation failed:", error); // 返回错误信息 const errorMessage = error instanceof Error ? `图片生成失败: ${error.message}` : "图片生成失败,请稍后重试"; return { text: errorMessage, finishReason: "error" as const, usage: { promptTokens: 0, completionTokens: 0 }, }; } }, doStream: async function* (options: any) { console.log("[Volcano Vision] ===== doStream ENTERED! ====="); console.log("[Volcano Vision] Messages received:", JSON.stringify(options.messages).substring(0, 500) + "..."); console.log("[Volcano Vision] Using modelId:", modelId); try { const allImages: Array<{ thumbnail: string; original: string }> = []; // 使用流式请求 for await (const event of sendVolcanoStreamRequest(baseURL, apiKey, options.messages, modelId)) { if (event.type === "image_generation.partial_succeeded" && event.url) { // 每张图片生成完成后立即上传并发送 try { const imageUrls = await uploadImageToS3(event.url); allImages.push(imageUrls); const markdown = `![thumb|${imageUrls.original}](${imageUrls.thumbnail})`; console.log(`[Volcano Vision Stream] Image ${event.image_index} uploaded, sending...`); yield { type: "text-delta" as const, textDelta: markdown + "\n\n" }; } catch (error) { console.error(`[Volcano Vision Stream] Failed to upload image ${event.image_index}:`, error); } } else if (event.type === "image_generation.completed") { console.log(`[Volcano Vision Stream] Generation completed, total images: ${allImages.length}`); // 发送完成信号 yield { type: "finish" as const, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: allImages.length * 50, // 估算 }, }; } } } catch (error) { console.error("[Volcano Vision Stream] Generation failed:", error); const errorMessage = error instanceof Error ? `图片生成失败: ${error.message}` : "图片生成失败,请稍后重试"; // 发送错误信息 yield { type: "text-delta" as const, textDelta: errorMessage }; yield { type: "finish" as const, finishReason: "error" as const, usage: { promptTokens: 0, completionTokens: 0 }, }; } }, modelId, provider: "volcano-vision", specificationVersion: "v1" as const, defaultObjectGenerationMode: "json" as const, }), }; }