import "server-only"; import { uploadFile } from "@/lib/storage/service"; interface HuankeGeminiConfig { apiKey: string; baseURL: string; } function normalizeHuankeBaseURL(baseURL: string): string { let normalized = (baseURL || "").trim().replace(/\/+$/, ""); normalized = normalized.replace(/\/v1beta\/models\/.*$/i, ""); normalized = normalized.replace(/\/v1beta$/i, ""); normalized = normalized.replace(/\/v1$/i, ""); if (!normalized) { return "https://api.huandutech.com"; } return normalized; } function redactSecrets(text: string, secrets: string[] = []) { let output = text; for (const secret of secrets) { if (!secret) continue; output = output.split(secret).join("***"); } output = output.replace(/([?&]key=)[^&\s"']+/gi, "$1***"); output = output.replace(/("apiKey"\s*:\s*")[^"]+(")/gi, "$1***$2"); return output; } /** * 压缩图片生成缩略图(使用 sharp) */ async function createThumbnail(buffer: Buffer, mimeType: string): Promise { try { const sharp = await import("sharp"); // 创建缩略图(宽度 500px,高度按比例,质量 80%) const thumbnailBuffer = await sharp.default(buffer) .resize(500, null, { fit: "inside", withoutEnlargement: true, }) .jpeg({ quality: 80 }) .toBuffer(); console.log(`[Huanke Gemini] Thumbnail created, size: ${thumbnailBuffer.length} bytes (original: ${buffer.length} bytes)`); return thumbnailBuffer; } catch (error) { console.error("[Huanke Gemini] Failed to create thumbnail:", error); // 如果失败,返回原图 return buffer; } } /** * 将 base64 图片上传到 S3,返回缩略图和原图的 URL */ async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/jpeg"): Promise<{ thumbnail: string; original: string }> { try { // 解析 base64 数据 const buffer = Buffer.from(base64Data, "base64"); // 生成文件名 const ext = mimeType.split("/")[1] || "jpg"; const timestamp = Date.now(); const randomStr = Math.random().toString(36).slice(2, 8); const thumbnailFilename = `huanke-thumb-${timestamp}-${randomStr}.${ext}`; const originalFilename = `huanke-orig-${timestamp}-${randomStr}.${ext}`; console.log(`[Huanke Gemini] Original size: ${buffer.length} bytes`); // 生成缩略图 const thumbnailBuffer = await createThumbnail(buffer, mimeType); // 转换为 ArrayBuffer const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice( thumbnailBuffer.byteOffset, thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength ) as ArrayBuffer; const originalArrayBuffer = buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; // 并行上传缩略图和原图到 S3 console.log(`[Huanke Gemini] Uploading thumbnail and original to S3...`); const [thumbnailResult, originalResult] = await Promise.all([ uploadFile(thumbnailFilename, thumbnailArrayBuffer), uploadFile(originalFilename, originalArrayBuffer), ]); console.log(`[Huanke Gemini] Uploaded to S3:`); console.log(` - Thumbnail: ${thumbnailResult.url}`); console.log(` - Original: ${originalResult.url}`); return { thumbnail: thumbnailResult.url, original: originalResult.url, }; } catch (error) { console.error("[Huanke Gemini] Failed to upload to S3:", error); throw new Error(`图片上传到存储桶失败: ${error instanceof Error ? error.message : "未知错误"}。请检查系统设置中的存储配置。`); } } /** * 下载远程URL图片并上传到 S3 */ async function downloadAndUploadToS3(imageUrl: string, retries = 3): Promise<{ thumbnail: string; original: string }> { for (let attempt = 0; attempt < retries; attempt++) { try { console.log(`[Huanke Gemini] Downloading image from URL (attempt ${attempt + 1}/${retries}): ${imageUrl.substring(0, 100)}...`); const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 30000); const response = await fetch(imageUrl, { signal: controller.signal, headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", }, }); clearTimeout(timeoutId); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); } const arrayBuffer = await response.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); if (buffer.length === 0) { throw new Error("下载的图片数据为空"); } const contentType = response.headers.get("content-type") || "image/jpeg"; const ext = contentType.split("/")[1] || "jpg"; const timestamp = Date.now(); const randomStr = Math.random().toString(36).slice(2, 8); const thumbnailFilename = `huanke-thumb-${timestamp}-${randomStr}.${ext}`; const originalFilename = `huanke-orig-${timestamp}-${randomStr}.${ext}`; console.log(`[Huanke Gemini] Downloaded size: ${buffer.length} bytes`); const thumbnailBuffer = await createThumbnail(buffer, contentType); const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice( thumbnailBuffer.byteOffset, thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength ) as ArrayBuffer; const originalArrayBuffer = buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; console.log(`[Huanke Gemini] Uploading downloaded image to S3...`); const [thumbnailResult, originalResult] = await Promise.all([ uploadFile(thumbnailFilename, thumbnailArrayBuffer), uploadFile(originalFilename, originalArrayBuffer), ]); console.log(`[Huanke Gemini] Uploaded to S3:`); console.log(` - Thumbnail: ${thumbnailResult.url}`); console.log(` - Original: ${originalResult.url}`); return { thumbnail: thumbnailResult.url, original: originalResult.url, }; } catch (error) { const isLastAttempt = attempt === retries - 1; console.error( `[Huanke Gemini] Download attempt ${attempt + 1}/${retries} failed:`, error instanceof Error ? error.message : String(error) ); if (isLastAttempt) { throw new Error( `图片下载并上传失败(已重试 ${retries} 次): ${error instanceof Error ? error.message : "未知错误"}` ); } const delay = Math.pow(2, attempt) * 1000; console.log(`[Huanke Gemini] Retrying in ${delay}ms...`); await new Promise((resolve) => setTimeout(resolve, delay)); } } throw new Error("下载失败:未知错误"); } /** * 确保图片是公网可访问的 URL(如果是 Base64,上传到 S3) */ async function ensureImageUrl(url: string): Promise { if (url.startsWith("http")) return url; if (url.startsWith("data:")) { try { console.log("[Huanke Gemini] Converting Base64 reference image to S3 URL..."); const matches = url.match(/^data:([A-Za-z-+\/]+);base64,(.+)$/); if (!matches) { console.warn("[Huanke Gemini] Invalid Base64 format, skipping upload"); return url; } const mimeType = matches[1]; const base64Data = matches[2]; const buffer = Buffer.from(base64Data, "base64"); const ext = mimeType.split("/")[1] || "png"; const filename = `ref-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`; const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength) as ArrayBuffer; const result = await uploadFile(filename, arrayBuffer); console.log(`[Huanke Gemini] Uploaded reference image to: ${result.url}`); return result.url; } catch (error) { console.error("[Huanke Gemini] Failed to upload reference image to S3:", error); return url; } } return url; } // 环科 Gemini API 响应格式 interface HuankeGeminiResponse { candidates?: Array<{ content: { parts: Array<{ text?: string; inlineData?: { mimeType: string; data: string; }; inline_data?: { mime_type: string; data: string; }; }>; }; finishReason?: string; }>; // OpenAI 兼容格式(可能的返回格式) choices?: Array<{ message: { content: string; }; }>; error?: { message: string; code?: string; }; } /** * 从 API 响应中提取图片并上传到 S3 */ async function extractAndUploadImages(response: HuankeGeminiResponse): Promise> { console.log("[Huanke Gemini] Extracting images from response:", JSON.stringify(response).substring(0, 500)); const images: Array<{ thumbnail: string; original: string }> = []; const uploadPromises: Promise[] = []; // 1. 尝试 Gemini 原生格式 (candidates[0].content.parts) if (response.candidates && response.candidates.length > 0) { const candidate = response.candidates[0]; if (candidate.content && candidate.content.parts) { for (const part of candidate.content.parts) { // inline_data 格式(小写下划线) if (part.inline_data && part.inline_data.data) { const mimeType = part.inline_data.mime_type || "image/png"; console.log(`[Huanke Gemini] Found inline_data image, mimeType: ${mimeType}`); const uploadPromise = uploadBase64ToS3(part.inline_data.data, mimeType) .then(urls => { images.push(urls); }); uploadPromises.push(uploadPromise); } // inlineData 格式(驼峰命名) else if (part.inlineData && part.inlineData.data) { const mimeType = part.inlineData.mimeType || "image/png"; console.log(`[Huanke Gemini] Found inlineData image, mimeType: ${mimeType}`); const uploadPromise = uploadBase64ToS3(part.inlineData.data, mimeType) .then(urls => { images.push(urls); }); uploadPromises.push(uploadPromise); } // 文本中的 Markdown 图片 else if (part.text) { const allImageMatches = part.text.matchAll(/!\[[^\]]*\]\(([^)]+)\)/g); for (const match of allImageMatches) { const url = match[1]; const base64Match = url.match(/^data:image\/([^;]+);base64,(.+)$/); if (base64Match) { const uploadPromise = uploadBase64ToS3(base64Match[2], `image/${base64Match[1]}`) .then(urls => { images.push(urls); }); uploadPromises.push(uploadPromise); } else if (url.startsWith("http")) { const uploadPromise = downloadAndUploadToS3(url) .then(urls => { images.push(urls); }) .catch(error => { console.error("[Huanke Gemini] Failed to download image:", error); throw error; }); uploadPromises.push(uploadPromise); } } } } } } // 2. 尝试 OpenAI 兼容格式 (choices[0].message.content) if (response.choices && response.choices.length > 0) { const content = response.choices[0]?.message?.content; if (content && typeof content === "string") { console.log("[Huanke Gemini] Parsing OpenAI format, content:", content.substring(0, 100) + "..."); const allImageMatches = content.matchAll(/!\[[^\]]*\]\(([^)]+)\)/g); for (const match of allImageMatches) { const url = match[1]; const base64Match = url.match(/^data:image\/([^;]+);base64,(.+)$/); if (base64Match) { const uploadPromise = uploadBase64ToS3(base64Match[2], `image/${base64Match[1]}`) .then(urls => { images.push(urls); }); uploadPromises.push(uploadPromise); } else if (url.startsWith("http")) { const uploadPromise = downloadAndUploadToS3(url) .then(urls => { images.push(urls); }) .catch(error => { console.error("[Huanke Gemini] Failed to download image:", error); throw error; }); uploadPromises.push(uploadPromise); } } } } // 等待所有上传完成 await Promise.all(uploadPromises); console.log("[Huanke Gemini] Extracted and uploaded", images.length, "images"); return images; } /** * 将图片数据转换为 Markdown 格式 */ function formatImageResponse( images: Array<{ thumbnail: string; original: string }>, imageSize: string = "2K" ): string { if (images.length === 0) { return "生成图片失败,未找到图片数据"; } return images.map((img) => `![thumb|||${img.original}|||${imageSize}](${img.thumbnail})`).join("\n\n"); } /** * 将 Base64 图片上传到 S3 并返回 URL * 环科 API 禁止使用 Base64 格式,必须使用 URL */ async function convertBase64ToUrl(base64Url: string): Promise { const matches = base64Url.match(/^data:([A-Za-z-+\/]+);base64,(.+)$/); if (!matches) { console.warn("[Huanke Gemini] Invalid Base64 format"); return base64Url; } const mimeType = matches[1]; const base64Data = matches[2]; const buffer = Buffer.from(base64Data, "base64"); const ext = mimeType.split("/")[1] || "png"; const filename = `huanke-ref-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`; const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength) as ArrayBuffer; const result = await uploadFile(filename, arrayBuffer); console.log(`[Huanke Gemini] Converted Base64 to URL: ${result.url}`); return result.url; } /** * 发送环科 Gemini 原生格式请求 * 端点格式: /v1beta/models/{model}:generateContent * * 重要:环科 API 禁止使用 Base64 格式,所有图片必须使用 URL 格式传输 */ async function sendHuankeGeminiRequest( baseURL: string, apiKey: string, messages: any[], modelId: string, imageParams?: { aspectRatio?: string; imageSize?: string; } ): Promise { const normalizedBaseURL = normalizeHuankeBaseURL(baseURL); // 构建 Gemini 原生格式的端点 // 例如: https://api.example.com/v1beta/models/gemini-3-pro-image-preview-4k:generateContent const url = `${normalizedBaseURL}/v1beta/models/${modelId}:generateContent?key=${apiKey}`; console.log("[Huanke Gemini] Sending request to:", redactSecrets(url, [apiKey])); console.log("[Huanke Gemini] Using model:", modelId); console.log("[Huanke Gemini] Image params:", imageParams); // 构建 Gemini 原生格式的请求体 const contents: any[] = []; for (const msg of messages) { const parts: any[] = []; if (typeof msg.content === "string") { parts.push({ text: msg.content }); } else if (Array.isArray(msg.content)) { for (const contentPart of msg.content) { if (contentPart.type === "text") { parts.push({ text: contentPart.text }); } else if (contentPart.type === "image_url" && contentPart.image_url?.url) { // 处理图片 URL - 禁止使用 Base64,必须转换为 URL let imageUrl = contentPart.image_url.url; if (imageUrl.startsWith("data:")) { // Base64 图片 -> 上传到 S3 获取 URL console.log("[Huanke Gemini] Converting Base64 image to URL (Base64 forbidden)..."); try { imageUrl = await convertBase64ToUrl(imageUrl); } catch (error) { console.error("[Huanke Gemini] Failed to convert Base64 to URL:", error); continue; // 跳过这张图片 } } // 使用 fileData 格式传递图片 URL(不使用 inlineData/Base64) parts.push({ fileData: { fileUri: imageUrl, mimeType: "image/jpeg", // 默认 MIME 类型 } }); console.log("[Huanke Gemini] Added image URL:", imageUrl.substring(0, 100)); } else if (contentPart.type === "image" && contentPart.image) { // CoreMessage 的 image 类型 let imageUrl = typeof contentPart.image === "string" ? contentPart.image : contentPart.image.toString(); if (imageUrl.startsWith("data:")) { // Base64 图片 -> 上传到 S3 获取 URL console.log("[Huanke Gemini] Converting Base64 image to URL (Base64 forbidden)..."); try { imageUrl = await convertBase64ToUrl(imageUrl); } catch (error) { console.error("[Huanke Gemini] Failed to convert Base64 to URL:", error); continue; } } // 使用 fileData 格式传递图片 URL parts.push({ fileData: { fileUri: imageUrl, mimeType: "image/jpeg", } }); console.log("[Huanke Gemini] Added image URL:", imageUrl.substring(0, 100)); } else if (contentPart.type === "file" && (contentPart.data || contentPart.url)) { // 🆕 处理 file 类型(从 Chat API 传过来的格式) // 格式: { type: "file", mediaType: "image/jpeg", data: "https://..." } let imageUrl = contentPart.data || contentPart.url; const mimeType = contentPart.mediaType || contentPart.mimeType || "image/jpeg"; console.log("[Huanke Gemini] Processing file type:", { url: imageUrl?.substring(0, 80), mimeType }); if (imageUrl && imageUrl.startsWith("data:")) { // Base64 图片 -> 上传到 S3 获取 URL console.log("[Huanke Gemini] Converting Base64 file to URL (Base64 forbidden)..."); try { imageUrl = await convertBase64ToUrl(imageUrl); } catch (error) { console.error("[Huanke Gemini] Failed to convert Base64 to URL:", error); continue; } } if (imageUrl && imageUrl.startsWith("http")) { // 使用 fileData 格式传递图片 URL parts.push({ fileData: { fileUri: imageUrl, mimeType: mimeType.startsWith("image/") ? mimeType : "image/jpeg", } }); console.log("[Huanke Gemini] Added file image URL:", imageUrl.substring(0, 100)); } } } } if (parts.length > 0) { contents.push({ role: msg.role === "assistant" ? "model" : "user", parts, }); } } // 构建请求体(Gemini 原生格式) const requestBody: any = { contents, generationConfig: { responseModalities: ["TEXT", "IMAGE"], }, }; // 添加图片配置 if (imageParams) { requestBody.generationConfig.imageConfig = {}; if (imageParams.aspectRatio && imageParams.aspectRatio !== "auto") { requestBody.generationConfig.imageConfig.aspectRatio = imageParams.aspectRatio; } if (imageParams.imageSize) { requestBody.generationConfig.imageConfig.imageSize = imageParams.imageSize; } } console.log("[Huanke Gemini] Request body (truncated):", JSON.stringify(requestBody).substring(0, 500) + "..."); // 根据分辨率动态设置超时时间 const timeout = imageParams?.imageSize === "4K" ? 300000 : 120000; console.log(`[Huanke Gemini] Timeout set to ${timeout/1000} seconds for ${imageParams?.imageSize || "default"}`); const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), timeout); try { const response = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify(requestBody), signal: controller.signal, }); clearTimeout(timeoutId); if (!response.ok) { const errorText = redactSecrets(await response.text(), [apiKey]); console.error("[Huanke Gemini] Request failed:", response.status, errorText); throw new Error( `Huanke Gemini API error: ${response.status} ${response.statusText} - ${errorText}` ); } const data = await response.json(); console.log("[Huanke Gemini] Response received:", JSON.stringify(data).substring(0, 500)); // 检查是否有错误 if (data.error) { throw new Error(`API Error: ${data.error.message || JSON.stringify(data.error)}`); } return data as HuankeGeminiResponse; } catch (error) { clearTimeout(timeoutId); if (error instanceof Error && error.name === "AbortError") { console.error("[Huanke Gemini] Request timeout"); throw new Error("环科 Gemini 请求超时,请稍后重试"); } throw error; } } /** * 创建环科 Gemini 图像生成 Provider * 使用 Gemini 原生 API 格式 */ export function createHuankeGemini({ apiKey, baseURL }: HuankeGeminiConfig) { const getMessagesFromOptions = (options: any): any[] => { if (Array.isArray(options?.messages)) { return options.messages; } if (!Array.isArray(options?.prompt)) { return []; } return options.prompt .map((message: any) => { if (!message || typeof message !== "object") return null; if (typeof message.content === "string") { return { role: message.role, content: message.content, }; } if (!Array.isArray(message.content)) { return null; } const content = message.content .map((part: any) => { if (!part || typeof part !== "object") return null; if (part.type === "text") { return { type: "text", text: part.text || "" }; } if (part.type === "file") { const data = typeof part.data === "string" ? part.data : part.data instanceof URL ? part.data.toString() : null; if (!data) return null; return { type: "file", data, mediaType: part.mediaType, filename: part.filename, }; } return null; }) .filter(Boolean); return { role: message.role, content, }; }) .filter(Boolean); }; const getImageParamsFromOptions = (options: any) => { const providerOptions = options?.providerOptions?.["huanke-gemini"] || options?.providerOptions?.huankeGemini || {}; const imageParams = options?.imageParams || {}; return { aspectRatio: imageParams.aspectRatio ?? providerOptions.aspectRatio ?? "auto", imageSize: imageParams.imageSize ?? providerOptions.imageSize ?? "2K", imageCount: imageParams.imageCount ?? providerOptions.imageCount ?? 1, }; }; const toV2Usage = (text: string) => { const outputTokens = text.length; return { inputTokens: 0, outputTokens, totalTokens: outputTokens, }; }; const createV2TextStream = (run: Promise<{ text: string; finishReason: "stop" | "error" }>) => new ReadableStream({ start(controller) { controller.enqueue({ type: "stream-start", warnings: [] }); void run .then((result) => { const usage = toV2Usage(result.text); if (result.text) { const textId = `txt-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`; controller.enqueue({ type: "text-start", id: textId }); controller.enqueue({ type: "text-delta", id: textId, delta: result.text, }); controller.enqueue({ type: "text-end", id: textId }); } controller.enqueue({ type: "finish", finishReason: result.finishReason, usage, }); }) .catch((error) => { controller.enqueue({ type: "error", error }); controller.enqueue({ type: "finish", finishReason: "error", usage: { inputTokens: 0, outputTokens: 0, totalTokens: 0, }, }); }) .finally(() => { controller.close(); }); }, }); return { chat: (modelId: string) => ({ doGenerate: async (options: any) => { console.log("[Huanke Gemini] doGenerate called!"); const messages = getMessagesFromOptions(options); console.log( "[Huanke Gemini] Messages received:", JSON.stringify(messages).substring(0, 500) + "..." ); const effectiveModelId = options?.overrideModelId || modelId; const imageParams = getImageParamsFromOptions(options); const imageCount = Math.min(Math.max(Number(imageParams.imageCount || 1), 1), 2); console.log("[Huanke Gemini] Using modelId:", effectiveModelId); console.log("[Huanke Gemini] Final imageParams:", imageParams); console.log(`[Huanke Gemini] Image count: ${imageCount}`); console.log(`[Huanke Gemini] Starting ${imageCount} concurrent request(s)...`); const startTime = Date.now(); const requestPromises = Array.from({ length: imageCount }, () => sendHuankeGeminiRequest(baseURL, apiKey, messages, effectiveModelId, imageParams) ); const responses = await Promise.all(requestPromises); console.log(`[Huanke Gemini] All ${imageCount} request(s) completed in ${Date.now() - startTime}ms`); const extractPromises = responses.map((response) => extractAndUploadImages(response)); const imageUrlsArrays = await Promise.all(extractPromises); const allImageUrls = imageUrlsArrays.flat(); const markdownText = formatImageResponse(allImageUrls, imageParams.imageSize); console.log( `[Huanke Gemini] Generated ${allImageUrls.length} images from ${imageCount} concurrent request(s)` ); const isV2Call = Array.isArray(options?.prompt); if (isV2Call) { return { content: [{ type: "text", text: markdownText }], finishReason: "stop" as const, usage: toV2Usage(markdownText), warnings: [], }; } return { text: markdownText, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: markdownText.length, }, }; }, doStream: function (this: any, options: any) { const isV2Call = Array.isArray(options?.prompt); if (!isV2Call) { const legacyStream = async function* (this: any) { try { const result = await (this as any).doGenerate(options); yield { type: "text-delta" as const, textDelta: result.text }; yield { type: "finish" as const, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: result.text?.length || 0, }, }; } catch (error) { const errorMessage = error instanceof Error ? `图片生成失败: ${error.message}` : "图片生成失败,请稍后重试"; yield { type: "text-delta" as const, textDelta: errorMessage }; yield { type: "finish" as const, finishReason: "error" as const, usage: { promptTokens: 0, completionTokens: 0 }, }; } }; return legacyStream.call(this); } const run = (async () => { try { const result = await this.doGenerate(options); const content = Array.isArray(result?.content) ? result.content : []; const text = content .filter((part: any) => part?.type === "text" && typeof part.text === "string") .map((part: any) => part.text) .join(""); return { text, finishReason: "stop" as const }; } catch (error) { console.error("[Huanke Gemini Stream] Generation failed:", error); const text = error instanceof Error ? `图片生成失败: ${error.message}` : "图片生成失败,请稍后重试"; return { text, finishReason: "error" as const }; } })(); return Promise.resolve({ stream: createV2TextStream(run), }); }, supportedUrls: {}, modelId, provider: "huanke-gemini", specificationVersion: "v2" as const, defaultObjectGenerationMode: "json" as const, }), }; }