1564 lines
54 KiB
TypeScript
1564 lines
54 KiB
TypeScript
import "server-only";
|
||
|
||
import { resolveGrsaiGptImageSize } from "@/lib/ai/grsai-gpt-image-sizes";
|
||
import { buildReferenceImageOrderPrompt } from "@/lib/ai/providers/reference-image-order";
|
||
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
import { createLogger } from "@/lib/observability/logger";
|
||
|
||
const logger = createLogger("gemini-image");
|
||
interface GeminiImageConfig {
|
||
apiKey: string;
|
||
baseURL: string;
|
||
}
|
||
|
||
/**
|
||
* 压缩图片生成缩略图（使用 sharp）
|
||
*/
|
||
async function createThumbnail(buffer: Buffer, mimeType: string): Promise<Buffer> {
|
||
try {
|
||
const sharp = await import("sharp");
|
||
|
||
// 创建缩略图（宽度 500px，高度按比例，质量 80%）
|
||
const thumbnailBuffer = await sharp.default(buffer)
|
||
.resize(500, null, {
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
})
|
||
.jpeg({ quality: 80 })
|
||
.toBuffer();
|
||
|
||
logger.debug(`[Gemini Image] Thumbnail created, size: ${thumbnailBuffer.length} bytes (original: ${buffer.length} bytes)`);
|
||
return thumbnailBuffer;
|
||
} catch (error) {
|
||
logger.error("[Gemini Image] Failed to create thumbnail:", error);
|
||
// 如果失败，返回原图
|
||
return buffer;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 确保图片是公网可访问的 URL（如果是 Base64，上传到 S3）
|
||
*/
|
||
async function ensureImageUrl(url: string): Promise<string> {
|
||
if (url.startsWith("http")) return url;
|
||
|
||
if (url.startsWith("data:")) {
|
||
try {
|
||
logger.debug("[Gemini Image] Converting Base64 reference image to S3 URL...");
|
||
// 解析 Base64
|
||
const matches = url.match(/^data:([A-Za-z-+\/]+);base64,(.+)$/);
|
||
if (!matches) {
|
||
logger.warn("[Gemini Image] Invalid Base64 format, skipping upload");
|
||
return url;
|
||
}
|
||
|
||
const mimeType = matches[1];
|
||
const base64Data = matches[2];
|
||
const buffer = Buffer.from(base64Data, "base64");
|
||
const ext = mimeType.split("/")[1] || "png";
|
||
const filename = `ref-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.${ext}`;
|
||
|
||
// 上传到 S3（需要转换为 ArrayBuffer）
|
||
const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength) as ArrayBuffer;
|
||
const result = await uploadFile(filename, arrayBuffer);
|
||
logger.debug(`[Gemini Image] Uploaded reference image to: ${result.url}`);
|
||
return result.url;
|
||
} catch (error) {
|
||
logger.error("[Gemini Image] Failed to upload reference image to S3:", error);
|
||
// 如果上传失败，返回原 Base64（虽然可能也会失败，但没别的办法）
|
||
return url;
|
||
}
|
||
}
|
||
|
||
return url;
|
||
}
|
||
|
||
async function inferImageAspectRatioFromUrl(url: string): Promise<string | null> {
|
||
const normalized = String(url || "").trim();
|
||
if (!normalized) return null;
|
||
|
||
try {
|
||
let buffer: Buffer | null = null;
|
||
if (normalized.startsWith("data:")) {
|
||
const match = normalized.match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/);
|
||
if (!match) return null;
|
||
buffer = Buffer.from(match[2], "base64");
|
||
} else if (/^https?:\/\//i.test(normalized)) {
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), 20000);
|
||
try {
|
||
const response = await fetch(normalized, {
|
||
signal: controller.signal,
|
||
headers: {
|
||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||
},
|
||
});
|
||
if (!response.ok) return null;
|
||
buffer = Buffer.from(await response.arrayBuffer());
|
||
} finally {
|
||
clearTimeout(timeoutId);
|
||
}
|
||
}
|
||
|
||
if (!buffer?.length) return null;
|
||
const sharp = await import("sharp");
|
||
const metadata = await sharp.default(buffer).metadata();
|
||
if (!metadata.width || !metadata.height || metadata.height <= 0) {
|
||
return null;
|
||
}
|
||
|
||
return `${metadata.width}:${metadata.height}`;
|
||
} catch (error) {
|
||
logger.warn("[Gemini Image] Failed to infer reference aspect ratio:", error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 将 base64 图片上传到 S3，返回缩略图和原图的 URL
|
||
*/
|
||
async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/jpeg"): Promise<{ thumbnail: string; original: string }> {
|
||
const deferred = await createDeferredUploadedImage(`data:${mimeType || "image/png"};base64,${base64Data}`, { providerType: "gemini-image" });
|
||
if (deferred) return deferred;
|
||
|
||
try {
|
||
// 解析 base64 数据
|
||
const buffer = Buffer.from(base64Data, "base64");
|
||
|
||
// 生成文件名
|
||
const ext = mimeType.split("/")[1] || "jpg";
|
||
const timestamp = Date.now();
|
||
const randomStr = Math.random().toString(36).slice(2, 8);
|
||
const thumbnailFilename = `gemini-thumb-${timestamp}-${randomStr}.${ext}`;
|
||
const originalFilename = `gemini-orig-${timestamp}-${randomStr}.${ext}`;
|
||
|
||
logger.debug(`[Gemini Image] Original size: ${buffer.length} bytes`);
|
||
|
||
// 生成缩略图
|
||
const thumbnailBuffer = await createThumbnail(buffer, mimeType);
|
||
|
||
// 转换为 ArrayBuffer
|
||
const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
|
||
thumbnailBuffer.byteOffset,
|
||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
|
||
) as ArrayBuffer;
|
||
const originalArrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
|
||
// 并行上传缩略图和原图到 S3
|
||
logger.debug(`[Gemini Image] Uploading thumbnail and original to S3...`);
|
||
const [thumbnailResult, originalResult] = await Promise.all([
|
||
uploadFile(thumbnailFilename, thumbnailArrayBuffer),
|
||
uploadFile(originalFilename, originalArrayBuffer),
|
||
]);
|
||
|
||
logger.debug(`[Gemini Image] Uploaded to S3:`);
|
||
logger.debug(` - Thumbnail: ${thumbnailResult.url}`);
|
||
logger.debug(` - Original: ${originalResult.url}`);
|
||
|
||
return {
|
||
thumbnail: thumbnailResult.url,
|
||
original: originalResult.url,
|
||
};
|
||
} catch (error) {
|
||
logger.error("[Gemini Image] Failed to upload to S3:", error);
|
||
// 如果上传失败，抛出错误，不要返回 Base64，否则会导致前端崩溃和数据库保存失败
|
||
throw new Error(`图片上传到存储桶失败: ${error instanceof Error ? error.message : "未知错误"}。请检查系统设置中的存储配置。`);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 下载远程URL图片并上传到 S3，返回缩略图和原图的 URL
|
||
* 带重试机制，确保成功下载并上传到国内中转
|
||
*/
|
||
async function downloadAndUploadToS3(imageUrl: string, retries = 3): Promise<{ thumbnail: string; original: string }> {
|
||
const deferred = await createDeferredUploadedImage(imageUrl, {
|
||
providerType: "gemini-image",
|
||
});
|
||
if (deferred) return deferred;
|
||
|
||
for (let attempt = 0; attempt < retries; attempt++) {
|
||
try {
|
||
logger.debug(`[Gemini Image] Downloading image from URL (attempt ${attempt + 1}/${retries}): ${imageUrl.substring(0, 100)}...`);
|
||
|
||
// 下载图片（增加超时时间到 30 秒）
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), 30000);
|
||
|
||
const response = await fetch(imageUrl, {
|
||
signal: controller.signal,
|
||
headers: {
|
||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||
},
|
||
});
|
||
|
||
clearTimeout(timeoutId);
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||
}
|
||
|
||
// 获取图片数据
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
const buffer = Buffer.from(arrayBuffer);
|
||
|
||
if (buffer.length === 0) {
|
||
throw new Error("下载的图片数据为空");
|
||
}
|
||
|
||
// 从响应头或URL获取MIME类型
|
||
const contentType = response.headers.get("content-type") || "image/jpeg";
|
||
const ext = contentType.split("/")[1] || "jpg";
|
||
|
||
// 生成文件名
|
||
const timestamp = Date.now();
|
||
const randomStr = Math.random().toString(36).slice(2, 8);
|
||
const thumbnailFilename = `gemini-thumb-${timestamp}-${randomStr}.${ext}`;
|
||
const originalFilename = `gemini-orig-${timestamp}-${randomStr}.${ext}`;
|
||
|
||
logger.debug(`[Gemini Image] Downloaded size: ${buffer.length} bytes`);
|
||
|
||
// 生成缩略图
|
||
const thumbnailBuffer = await createThumbnail(buffer, contentType);
|
||
|
||
// 转换为 ArrayBuffer
|
||
const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
|
||
thumbnailBuffer.byteOffset,
|
||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
|
||
) as ArrayBuffer;
|
||
const originalArrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
|
||
// 并行上传缩略图和原图到 S3
|
||
logger.debug(`[Gemini Image] Uploading downloaded image to S3...`);
|
||
const [thumbnailResult, originalResult] = await Promise.all([
|
||
uploadFile(thumbnailFilename, thumbnailArrayBuffer),
|
||
uploadFile(originalFilename, originalArrayBuffer),
|
||
]);
|
||
|
||
logger.debug(`[Gemini Image] Uploaded to S3:`);
|
||
logger.debug(` - Thumbnail: ${thumbnailResult.url}`);
|
||
logger.debug(` - Original: ${originalResult.url}`);
|
||
|
||
return {
|
||
thumbnail: thumbnailResult.url,
|
||
original: originalResult.url,
|
||
};
|
||
} catch (error) {
|
||
const isLastAttempt = attempt === retries - 1;
|
||
logger.error(
|
||
`[Gemini Image] Download attempt ${attempt + 1}/${retries} failed:`,
|
||
error instanceof Error ? error.message : String(error)
|
||
);
|
||
|
||
if (isLastAttempt) {
|
||
// 最后一次尝试失败，抛出错误（不要 fallback 到原始 URL）
|
||
throw new Error(
|
||
`图片下载并上传失败（已重试 ${retries} 次）: ${error instanceof Error ? error.message : "未知错误"}`
|
||
);
|
||
}
|
||
|
||
// 等待后重试（指数退避）
|
||
const delay = Math.pow(2, attempt) * 1000; // 1s, 2s, 4s...
|
||
logger.debug(`[Gemini Image] Retrying in ${delay}ms...`);
|
||
await new Promise((resolve) => setTimeout(resolve, delay));
|
||
}
|
||
}
|
||
|
||
// 理论上不会到达这里，但 TypeScript 需要
|
||
throw new Error("下载失败：未知错误");
|
||
}
|
||
|
||
// Gemini API 响应格式（参考灵魂助手3.0的格式）
|
||
interface GeminiResponse {
|
||
candidates: Array<{
|
||
content: {
|
||
parts: Array<{
|
||
text?: string;
|
||
inline_data?: {
|
||
mime_type: string;
|
||
data: string;
|
||
};
|
||
inlineData?: {
|
||
mimeType: string;
|
||
data: string;
|
||
};
|
||
}>;
|
||
};
|
||
}>;
|
||
}
|
||
|
||
function joinOpenAICompatibleBaseUrl(baseURL: string, path: string) {
|
||
const normalized = String(baseURL || "").trim().replace(/\/+$/, "");
|
||
const withoutVersion = normalized.replace(/\/v1$/i, "");
|
||
return `${withoutVersion}${path}`;
|
||
}
|
||
|
||
async function loadImageAsBlob(imageUrl: string): Promise<{ blob: Blob; filename: string }> {
|
||
const normalized = String(imageUrl || "").trim();
|
||
if (!normalized) {
|
||
throw new Error("参考图地址为空");
|
||
}
|
||
|
||
if (normalized.startsWith("data:")) {
|
||
const match = normalized.match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/);
|
||
if (!match) {
|
||
throw new Error("参考图 Base64 格式无效");
|
||
}
|
||
const mimeType = match[1] || "image/png";
|
||
const ext = mimeType.split("/")[1] || "png";
|
||
return {
|
||
blob: new Blob([Buffer.from(match[2], "base64")], { type: mimeType }),
|
||
filename: `reference.${ext}`,
|
||
};
|
||
}
|
||
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), 30000);
|
||
try {
|
||
const response = await fetch(normalized, {
|
||
signal: controller.signal,
|
||
headers: {
|
||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||
},
|
||
});
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||
}
|
||
|
||
const contentType = response.headers.get("content-type") || "image/png";
|
||
const extFromMime = contentType.split("/")[1]?.split(";")[0] || "png";
|
||
const extFromUrl = normalized
|
||
.split("?")[0]
|
||
.match(/\.([A-Za-z0-9]+)$/)?.[1];
|
||
const ext = extFromUrl || extFromMime;
|
||
|
||
return {
|
||
blob: new Blob([await response.arrayBuffer()], { type: contentType }),
|
||
filename: `reference.${ext}`,
|
||
};
|
||
} finally {
|
||
clearTimeout(timeoutId);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 从文本中提取图片数据
|
||
* 返回 { url, isBase64, mimeType, base64Data } 数组
|
||
*/
|
||
function extractImageDataFromText(text: string): Array<{ url: string; isBase64: boolean; mimeType?: string; base64Data?: string }> {
|
||
const images: Array<{ url: string; isBase64: boolean; mimeType?: string; base64Data?: string }> = [];
|
||
|
||
// 匹配所有 Markdown 图片格式: 
|
||
const allImageMatches = text.matchAll(/!\[[^\]]*\]\(([^)]+)\)/g);
|
||
for (const match of allImageMatches) {
|
||
const url = match[1];
|
||
// 检查是否是 base64 格式
|
||
const base64Match = url.match(/^data:image\/([^;]+);base64,(.+)$/);
|
||
if (base64Match) {
|
||
images.push({
|
||
url,
|
||
isBase64: true,
|
||
mimeType: `image/${base64Match[1]}`,
|
||
base64Data: base64Match[2],
|
||
});
|
||
} else {
|
||
images.push({ url, isBase64: false });
|
||
}
|
||
}
|
||
return images;
|
||
}
|
||
|
||
function parseDataImageUrl(value: string): { mimeType: string; base64Data: string } | null {
|
||
const match = String(value || "").trim().match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/);
|
||
if (!match) return null;
|
||
return {
|
||
mimeType: match[1] || "image/png",
|
||
base64Data: match[2],
|
||
};
|
||
}
|
||
|
||
/**
|
||
* 从 API 响应中提取图片并上传 base64 到 S3
|
||
* 返回图片数据数组（包含缩略图和原图 URL）
|
||
*/
|
||
async function extractAndUploadImages(response: any): Promise<Array<{ thumbnail: string; original: string }>> {
|
||
logger.debug("[Gemini Image] Extracting images from response:", JSON.stringify(response).substring(0, 500)); // 🔍 调试日志
|
||
const images: Array<{ thumbnail: string; original: string }> = [];
|
||
const uploadPromises: Promise<void>[] = [];
|
||
const queuedUrls = new Set<string>();
|
||
|
||
const queueRemoteImageUrl = (imageUrl: unknown, source: string) => {
|
||
const normalized = String(imageUrl || "").trim();
|
||
if (!normalized || queuedUrls.has(normalized)) return;
|
||
queuedUrls.add(normalized);
|
||
|
||
if (normalized.startsWith("data:")) {
|
||
const imageData = parseDataImageUrl(normalized);
|
||
if (imageData?.base64Data) {
|
||
const uploadPromise = uploadBase64ToS3(imageData.base64Data, imageData.mimeType)
|
||
.then((urls) => {
|
||
images.push(urls);
|
||
});
|
||
uploadPromises.push(uploadPromise);
|
||
}
|
||
return;
|
||
}
|
||
|
||
if (!/^https?:\/\//i.test(normalized)) {
|
||
for (const imageData of extractImageDataFromText(normalized)) {
|
||
if (imageData.isBase64 && imageData.base64Data) {
|
||
const uploadPromise = uploadBase64ToS3(imageData.base64Data, imageData.mimeType)
|
||
.then((urls) => {
|
||
images.push(urls);
|
||
});
|
||
uploadPromises.push(uploadPromise);
|
||
} else {
|
||
queueRemoteImageUrl(imageData.url, source);
|
||
}
|
||
}
|
||
return;
|
||
}
|
||
|
||
logger.debug(
|
||
`[Gemini Image] Found ${source} image URL: ${normalized.substring(0, 100)}...`
|
||
);
|
||
const uploadPromise = downloadAndUploadToS3(normalized)
|
||
.then((urls) => {
|
||
images.push(urls);
|
||
})
|
||
.catch((error) => {
|
||
logger.error(`[Gemini Image] Failed to download/upload ${source} URL after retries:`, error);
|
||
throw new Error(
|
||
`无法下载并上传图片到国内中转: ${error instanceof Error ? error.message : "未知错误"}`
|
||
);
|
||
});
|
||
uploadPromises.push(uploadPromise);
|
||
};
|
||
|
||
const queueImageValue = (value: unknown, source: string) => {
|
||
if (!value) return;
|
||
if (Array.isArray(value)) {
|
||
value.forEach((entry) => queueImageValue(entry, source));
|
||
return;
|
||
}
|
||
if (typeof value === "string") {
|
||
queueRemoteImageUrl(value, source);
|
||
return;
|
||
}
|
||
if (typeof value === "object") {
|
||
const item = value as Record<string, any>;
|
||
[
|
||
item.url,
|
||
item.imageUrl,
|
||
item.image_url,
|
||
item.image,
|
||
item.original,
|
||
item.originalUrl,
|
||
item.original_url,
|
||
item.thumbnail,
|
||
item.outputUrl,
|
||
item.output_url,
|
||
item.resultUrl,
|
||
item.result_url,
|
||
item.fileUrl,
|
||
item.file_url,
|
||
].forEach((candidate) => queueRemoteImageUrl(candidate, source));
|
||
|
||
const base64Data =
|
||
item.b64_json ||
|
||
item.base64 ||
|
||
item.base64Data ||
|
||
item.image_base64 ||
|
||
item.imageBase64;
|
||
if (typeof base64Data === "string" && base64Data.trim()) {
|
||
const mimeType = item.mime_type || item.mimeType || "image/png";
|
||
queueRemoteImageUrl(
|
||
base64Data.startsWith("data:image/")
|
||
? base64Data
|
||
: `data:${mimeType};base64,${base64Data.trim()}`,
|
||
source
|
||
);
|
||
}
|
||
}
|
||
};
|
||
|
||
queueImageValue(response?.url, "direct");
|
||
queueImageValue(response?.imageUrl, "direct");
|
||
queueImageValue(response?.image_url, "direct");
|
||
queueImageValue(response?.data?.url, "data");
|
||
queueImageValue(response?.data?.imageUrl, "data");
|
||
queueImageValue(response?.data?.image_url, "data");
|
||
|
||
const directArrays = [
|
||
{ value: response?.images, source: "images" },
|
||
{ value: response?.data?.images, source: "data.images" },
|
||
{ value: response?.data?.results, source: "data.results" },
|
||
{ value: response?.imageUrls, source: "imageUrls" },
|
||
{ value: response?.image_urls, source: "image_urls" },
|
||
{ value: response?.data?.imageUrls, source: "data.imageUrls" },
|
||
{ value: response?.data?.image_urls, source: "data.image_urls" },
|
||
{ value: response?.output, source: "output" },
|
||
{ value: response?.outputs, source: "outputs" },
|
||
{ value: response?.data?.output, source: "data.output" },
|
||
{ value: response?.data?.outputs, source: "data.outputs" },
|
||
];
|
||
for (const item of directArrays) {
|
||
queueImageValue(item.value, item.source);
|
||
}
|
||
|
||
// 1. 尝试 OpenAI Images API 格式 (data[].b64_json / data[].url)
|
||
if (Array.isArray(response.data)) {
|
||
logger.debug("[Gemini Image] Parsing OpenAI Images format, data count:", response.data.length);
|
||
for (const item of response.data) {
|
||
if (item?.b64_json && typeof item.b64_json === "string") {
|
||
const base64Data = item.b64_json.startsWith("data:")
|
||
? item.b64_json.split(",", 2)[1] || item.b64_json
|
||
: item.b64_json;
|
||
const uploadPromise = uploadBase64ToS3(base64Data, "image/png")
|
||
.then((urls) => {
|
||
images.push(urls);
|
||
});
|
||
uploadPromises.push(uploadPromise);
|
||
} else if (item?.url && typeof item.url === "string") {
|
||
queueRemoteImageUrl(item.url, "OpenAI Images");
|
||
}
|
||
}
|
||
}
|
||
|
||
// 2. 尝试 OpenAI Chat 格式 (choices[0].message.content)
|
||
if (response.choices && response.choices.length > 0) {
|
||
const content = response.choices[0]?.message?.content;
|
||
if (content && typeof content === "string") {
|
||
logger.debug("[Gemini Image] Parsing OpenAI format, content:", content.substring(0, 100) + "...");
|
||
const imageData = extractImageDataFromText(content);
|
||
|
||
for (const img of imageData) {
|
||
if (img.isBase64 && img.base64Data) {
|
||
// 异步上传 base64 到 S3（生成缩略图和原图）
|
||
const uploadPromise = uploadBase64ToS3(img.base64Data, img.mimeType)
|
||
.then(urls => { images.push(urls); });
|
||
uploadPromises.push(uploadPromise);
|
||
} else {
|
||
// 非 base64 图片（OpenAI URL 等），必须下载并上传到 S3（国内中转）
|
||
logger.debug("[Gemini Image] Detected external URL, will download and upload to S3:", img.url.substring(0, 100) + "...");
|
||
const uploadPromise = downloadAndUploadToS3(img.url)
|
||
.then(urls => {
|
||
logger.debug("[Gemini Image] Successfully downloaded and uploaded external image to S3");
|
||
images.push(urls);
|
||
})
|
||
.catch(error => {
|
||
logger.error("[Gemini Image] Failed to download/upload URL after retries:", error);
|
||
// 不再 fallback 到原始 URL，而是抛出错误
|
||
throw new Error(`无法下载并上传图片到国内中转: ${error instanceof Error ? error.message : "未知错误"}`);
|
||
});
|
||
uploadPromises.push(uploadPromise);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 3. 尝试 Nano Banana API 格式 (results[].url)
|
||
if (response.results && Array.isArray(response.results)) {
|
||
logger.debug("[Gemini Image] Parsing Nano Banana format, results count:", response.results.length);
|
||
for (const result of response.results) {
|
||
if (result.url && typeof result.url === "string") {
|
||
queueRemoteImageUrl(result.url, "Nano Banana");
|
||
}
|
||
}
|
||
}
|
||
|
||
// 4. 尝试 Gemini 格式 (candidates[0].content.parts)
|
||
if (response.candidates && response.candidates.length > 0) {
|
||
const candidate = response.candidates[0];
|
||
if (candidate.content && candidate.content.parts) {
|
||
for (const part of candidate.content.parts) {
|
||
if (part.text) {
|
||
const imageData = extractImageDataFromText(part.text);
|
||
for (const img of imageData) {
|
||
if (img.isBase64 && img.base64Data) {
|
||
const uploadPromise = uploadBase64ToS3(img.base64Data, img.mimeType)
|
||
.then(urls => { images.push(urls); });
|
||
uploadPromises.push(uploadPromise);
|
||
} else {
|
||
// 非 base64 图片（OpenAI URL 等），必须下载并上传到 S3（国内中转）
|
||
logger.debug("[Gemini Image] Detected external URL in Gemini response, will download and upload to S3:", img.url.substring(0, 100) + "...");
|
||
const uploadPromise = downloadAndUploadToS3(img.url)
|
||
.then(urls => {
|
||
logger.debug("[Gemini Image] Successfully downloaded and uploaded Gemini external image to S3");
|
||
images.push(urls);
|
||
})
|
||
.catch(error => {
|
||
logger.error("[Gemini Image] Failed to download/upload URL in Gemini response after retries:", error);
|
||
// 不再 fallback 到原始 URL，而是抛出错误
|
||
throw new Error(`无法下载并上传图片到国内中转: ${error instanceof Error ? error.message : "未知错误"}`);
|
||
});
|
||
uploadPromises.push(uploadPromise);
|
||
}
|
||
}
|
||
}
|
||
// inline_data 格式
|
||
if (part.inline_data && part.inline_data.data) {
|
||
const mimeType = part.inline_data.mime_type || "image/png";
|
||
const uploadPromise = uploadBase64ToS3(part.inline_data.data, mimeType)
|
||
.then(urls => { images.push(urls); });
|
||
uploadPromises.push(uploadPromise);
|
||
} else if (part.inlineData && part.inlineData.data) {
|
||
const mimeType = part.inlineData.mimeType || "image/png";
|
||
const uploadPromise = uploadBase64ToS3(part.inlineData.data, mimeType)
|
||
.then(urls => { images.push(urls); });
|
||
uploadPromises.push(uploadPromise);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 等待所有上传完成
|
||
await Promise.all(uploadPromises);
|
||
|
||
logger.debug("[Gemini Image] Extracted and uploaded", images.length, "images (thumbnail + original)");
|
||
return images;
|
||
}
|
||
|
||
/**
|
||
* 将图片数据转换为 Markdown 格式
|
||
* 使用特殊格式传递缩略图和原图 URL：
|
||
* 前端的 ImagePreview 组件会解析这个格式
|
||
*
|
||
* 注意：不能使用 $$ 作为分隔符，因为会被 Markdown 解析器当作 LaTeX 数学公式！
|
||
* 使用 ||| (三个竖线) 作为分隔符，不会与任何 Markdown 语法冲突
|
||
*/
|
||
function formatImageResponse(
|
||
images: Array<{ thumbnail: string; original: string }>,
|
||
imageSize: string = "2K"
|
||
): string {
|
||
if (images.length === 0) {
|
||
throw new Error("生成图片失败，未找到图片数据");
|
||
}
|
||
|
||
// 返回 Markdown 格式的图片，在 alt 文本中编码原图 URL 和 imageSize
|
||
// 使用 ||| 作为分隔符，避免与 Markdown 表格语法和 LaTeX 数学公式冲突
|
||
return images.map((img) => ``).join("\n\n");
|
||
}
|
||
|
||
const GRSAI_NANO_BANANA_SUBMIT_TIMEOUT_MS = 30000;
|
||
const GRSAI_NANO_BANANA_POLL_TIMEOUT_MS = 30000;
|
||
const GRSAI_NANO_BANANA_POLL_INTERVAL_MS = 5000;
|
||
const GRSAI_GPT_IMAGE_TIMEOUT_MS = 10 * 60 * 1000;
|
||
const GRSAI_GPT_IMAGE_4K_TIMEOUT_MS = 10 * 60 * 1000;
|
||
|
||
function isRecord(value: unknown): value is Record<string, any> {
|
||
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
||
}
|
||
|
||
function normalizeGrsaiPayload(payload: any): any {
|
||
if (isRecord(payload?.data)) {
|
||
return payload.data;
|
||
}
|
||
return payload;
|
||
}
|
||
|
||
async function sendGrsaiGptImageRequest(params: {
|
||
baseURL: string;
|
||
apiKey: string;
|
||
modelId: string;
|
||
prompt: string;
|
||
size: string;
|
||
imageUrls: string[];
|
||
timeout: number;
|
||
apiLogPrefix: string;
|
||
}): Promise<any> {
|
||
const { baseURL, apiKey, modelId, prompt, size, imageUrls, timeout, apiLogPrefix } = params;
|
||
const endpoint = `${baseURL.replace(/\/+$/, "")}/v1/draw/completions`;
|
||
const requestBody: Record<string, any> = {
|
||
model: modelId,
|
||
prompt,
|
||
size,
|
||
shutProgress: false,
|
||
};
|
||
|
||
if (imageUrls.length > 0) {
|
||
requestBody.urls = imageUrls;
|
||
}
|
||
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), timeout);
|
||
|
||
try {
|
||
logger.info(`${apiLogPrefix} Native submit`, {
|
||
endpoint,
|
||
modelId,
|
||
size,
|
||
referenceImageCount: imageUrls.length,
|
||
});
|
||
|
||
const response = await fetch(endpoint, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${apiKey}`,
|
||
},
|
||
body: JSON.stringify(requestBody),
|
||
signal: controller.signal,
|
||
});
|
||
|
||
clearTimeout(timeoutId);
|
||
|
||
const responseText = await response.text();
|
||
const data = parseGrsaiJsonOrStreamResponse(responseText);
|
||
|
||
if (!response.ok) {
|
||
logger.error(
|
||
`${apiLogPrefix} Native request failed:`,
|
||
response.status,
|
||
responseText.substring(0, 1000)
|
||
);
|
||
const providerError = getGrsaiErrorMessage(data);
|
||
const errorMessage =
|
||
providerError !== "未知错误"
|
||
? providerError
|
||
: responseText || `${response.status} ${response.statusText}`;
|
||
throw new Error(`图片生成失败: ${errorMessage}`);
|
||
}
|
||
|
||
logger.info(`${apiLogPrefix} Native response received`, {
|
||
status: response.status,
|
||
hasImages: hasGrsaiImages(data),
|
||
progress: getGrsaiProgress(data),
|
||
responseKeys: isRecord(data) ? Object.keys(data).slice(0, 12) : [],
|
||
});
|
||
|
||
if (!hasGrsaiImages(data)) {
|
||
const status = getGrsaiStatus(data);
|
||
if (status === "failed" || status === "error") {
|
||
throw new Error(`图片生成失败: ${getGrsaiErrorMessage(data)}`);
|
||
}
|
||
|
||
const taskId = getGrsaiTaskId(data);
|
||
if (taskId) {
|
||
logger.info(`${apiLogPrefix} Native response has no images, polling task result`, {
|
||
taskId,
|
||
progress: getGrsaiProgress(data),
|
||
status: status || null,
|
||
});
|
||
return pollGrsaiDrawResult({
|
||
baseURL,
|
||
apiKey,
|
||
taskId,
|
||
timeoutMs: timeout,
|
||
apiLogPrefix,
|
||
timeoutLabel: "Grsai GPT Image",
|
||
});
|
||
}
|
||
|
||
const providerError = getGrsaiErrorMessage(data);
|
||
throw new Error(
|
||
`图片生成失败: 未返回任务 ID 或可用图片${
|
||
providerError !== "未知错误" ? `: ${providerError}` : ""
|
||
}`
|
||
);
|
||
}
|
||
|
||
return data;
|
||
} catch (error) {
|
||
clearTimeout(timeoutId);
|
||
if (error instanceof Error && error.name === "AbortError") {
|
||
const timeoutSeconds = Math.round(timeout / 1000);
|
||
logger.error(`${apiLogPrefix} Request timeout after ${timeoutSeconds} seconds`);
|
||
throw new Error(`Grsai GPT Image 请求超时（${timeoutSeconds}秒），将自动切换备用模型`);
|
||
}
|
||
logger.error(`${apiLogPrefix} Native request error:`, {
|
||
name: error instanceof Error ? error.name : "UnknownError",
|
||
message: error instanceof Error ? error.message : String(error),
|
||
cause:
|
||
error instanceof Error && "cause" in error
|
||
? error.cause instanceof Error
|
||
? error.cause.message
|
||
: String(error.cause)
|
||
: undefined,
|
||
endpoint,
|
||
modelId,
|
||
size,
|
||
referenceImageCount: imageUrls.length,
|
||
});
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
function parseGrsaiJsonOrStreamResponse(responseText: string): any {
|
||
if (!responseText.trim()) return {};
|
||
|
||
try {
|
||
return JSON.parse(responseText);
|
||
} catch {
|
||
// Grsai draw/completions may stream progress as SSE-like "data: {...}" lines.
|
||
const payloads = responseText
|
||
.split(/\r?\n/)
|
||
.map((line) => line.trim())
|
||
.filter((line) => line.startsWith("data:"))
|
||
.map((line) => line.replace(/^data:\s*/, "").trim())
|
||
.filter((line) => line && line !== "[DONE]")
|
||
.map((line) => {
|
||
try {
|
||
return JSON.parse(line);
|
||
} catch {
|
||
return null;
|
||
}
|
||
})
|
||
.filter(Boolean);
|
||
|
||
if (payloads.length > 0) {
|
||
return payloads[payloads.length - 1];
|
||
}
|
||
}
|
||
|
||
return { raw: responseText };
|
||
}
|
||
|
||
function getGrsaiTaskId(payload: any): string | null {
|
||
const normalized = normalizeGrsaiPayload(payload);
|
||
const candidates = [
|
||
normalized?.id,
|
||
normalized?.taskId,
|
||
normalized?.task_id,
|
||
normalized?.task?.id,
|
||
normalized?.data?.id,
|
||
payload?.id,
|
||
payload?.taskId,
|
||
payload?.task_id,
|
||
];
|
||
|
||
for (const candidate of candidates) {
|
||
const value = String(candidate || "").trim();
|
||
if (value) return value;
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
function getGrsaiStatus(payload: any): string {
|
||
const normalized = normalizeGrsaiPayload(payload);
|
||
return String(
|
||
normalized?.status ||
|
||
normalized?.state ||
|
||
normalized?.taskStatus ||
|
||
payload?.status ||
|
||
""
|
||
).toLowerCase();
|
||
}
|
||
|
||
function getGrsaiProgress(payload: any): number | null {
|
||
const normalized = normalizeGrsaiPayload(payload);
|
||
const value =
|
||
normalized?.progress ??
|
||
normalized?.percentage ??
|
||
normalized?.percent ??
|
||
payload?.progress;
|
||
const numeric = Number(value);
|
||
return Number.isFinite(numeric) ? numeric : null;
|
||
}
|
||
|
||
function getGrsaiErrorMessage(payload: any): string {
|
||
const normalized = normalizeGrsaiPayload(payload);
|
||
const normalizePart = (part: unknown) => {
|
||
if (typeof part === "string" || typeof part === "number") {
|
||
return String(part).trim();
|
||
}
|
||
if (!isRecord(part)) return "";
|
||
return String(
|
||
part.message ||
|
||
part.msg ||
|
||
part.detail ||
|
||
part.reason ||
|
||
part.code ||
|
||
part.type ||
|
||
""
|
||
).trim();
|
||
};
|
||
const messages = [
|
||
normalized?.failure_reason,
|
||
normalized?.error?.message,
|
||
normalized?.error?.msg,
|
||
normalized?.error?.code,
|
||
normalized?.error,
|
||
normalized?.message,
|
||
normalized?.msg,
|
||
normalized?.detail,
|
||
normalized?.reason,
|
||
normalized?.code,
|
||
payload?.error?.message,
|
||
payload?.error?.msg,
|
||
payload?.error?.code,
|
||
payload?.msg,
|
||
payload?.message,
|
||
payload?.error,
|
||
payload?.detail,
|
||
payload?.reason,
|
||
payload?.code,
|
||
]
|
||
.map(normalizePart)
|
||
.filter(Boolean);
|
||
|
||
return Array.from(new Set(messages)).join(" / ") || "未知错误";
|
||
}
|
||
|
||
function hasGrsaiImages(payload: any): boolean {
|
||
const normalized = normalizeGrsaiPayload(payload);
|
||
return (
|
||
Array.isArray(normalized?.results) ||
|
||
Array.isArray(normalized?.images) ||
|
||
Boolean(normalized?.url || normalized?.imageUrl || normalized?.image_url)
|
||
);
|
||
}
|
||
|
||
async function fetchWithTimeout(
|
||
url: string,
|
||
init: RequestInit,
|
||
timeoutMs: number
|
||
): Promise<Response> {
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||
try {
|
||
return await fetch(url, {
|
||
...init,
|
||
signal: controller.signal,
|
||
});
|
||
} finally {
|
||
clearTimeout(timeoutId);
|
||
}
|
||
}
|
||
|
||
async function readJsonResponse(response: Response, label: string): Promise<any> {
|
||
const text = await response.text();
|
||
const payload = parseGrsaiJsonOrStreamResponse(text);
|
||
|
||
if (!response.ok) {
|
||
logger.error(`${label} failed:`, response.status, text.substring(0, 500));
|
||
const providerError = getGrsaiErrorMessage(payload);
|
||
const detail =
|
||
providerError !== "未知错误"
|
||
? providerError
|
||
: String(payload?.raw || text || "").trim().slice(0, 300);
|
||
throw new Error(
|
||
`${label} 接口错误：${response.status} ${response.statusText}${
|
||
detail ? `: ${detail}` : ""
|
||
}`
|
||
);
|
||
}
|
||
|
||
if (isRecord(payload) && !("raw" in payload)) {
|
||
return payload;
|
||
}
|
||
|
||
if (!isRecord(payload)) {
|
||
return payload;
|
||
}
|
||
|
||
if ("raw" in payload) {
|
||
logger.error(`${label} returned non-JSON response:`, text.substring(0, 500));
|
||
throw new Error(`${label} 返回格式异常，未能解析任务结果`);
|
||
}
|
||
|
||
return payload;
|
||
}
|
||
|
||
async function pollGrsaiNanoBananaResult(params: {
|
||
baseURL: string;
|
||
apiKey: string;
|
||
taskId: string;
|
||
timeoutMs: number;
|
||
apiLogPrefix: string;
|
||
}): Promise<any> {
|
||
return pollGrsaiDrawResult({
|
||
...params,
|
||
timeoutLabel: "GRSAI Nano Banana",
|
||
});
|
||
}
|
||
|
||
async function pollGrsaiDrawResult(params: {
|
||
baseURL: string;
|
||
apiKey: string;
|
||
taskId: string;
|
||
timeoutMs: number;
|
||
apiLogPrefix: string;
|
||
timeoutLabel: string;
|
||
}): Promise<any> {
|
||
const resultUrl = `${params.baseURL.replace(/\/+$/, "")}/v1/draw/result`;
|
||
const deadline = Date.now() + params.timeoutMs;
|
||
let lastPayload: any = null;
|
||
let lastProgress: number | null = null;
|
||
|
||
while (Date.now() < deadline) {
|
||
await new Promise((resolve) => setTimeout(resolve, GRSAI_NANO_BANANA_POLL_INTERVAL_MS));
|
||
|
||
const response = await fetchWithTimeout(
|
||
resultUrl,
|
||
{
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${params.apiKey}`,
|
||
},
|
||
body: JSON.stringify({ id: params.taskId }),
|
||
},
|
||
GRSAI_NANO_BANANA_POLL_TIMEOUT_MS
|
||
);
|
||
const payload = await readJsonResponse(response, `${params.apiLogPrefix} result`);
|
||
lastPayload = payload;
|
||
|
||
const normalized = normalizeGrsaiPayload(payload);
|
||
const status = getGrsaiStatus(payload);
|
||
const progress = getGrsaiProgress(payload);
|
||
if (progress !== null && progress !== lastProgress) {
|
||
lastProgress = progress;
|
||
logger.debug(`${params.apiLogPrefix} Async task progress: ${progress}%`);
|
||
}
|
||
|
||
if (
|
||
status === "succeeded" ||
|
||
status === "success" ||
|
||
status === "completed" ||
|
||
(progress !== null && progress >= 100 && hasGrsaiImages(payload))
|
||
) {
|
||
logger.debug(`${params.apiLogPrefix} Async task succeeded:`, {
|
||
taskId: params.taskId,
|
||
progress,
|
||
});
|
||
return normalized;
|
||
}
|
||
|
||
if (status === "failed" || status === "error") {
|
||
throw new Error(`图片生成失败: ${getGrsaiErrorMessage(payload)}`);
|
||
}
|
||
}
|
||
|
||
logger.error(`${params.apiLogPrefix} Async task timeout:`, {
|
||
taskId: params.taskId,
|
||
lastPayload: JSON.stringify(lastPayload || {}).substring(0, 500),
|
||
});
|
||
throw new Error(
|
||
`${params.timeoutLabel} 异步任务超时（${Math.round(params.timeoutMs / 1000)}秒），请稍后重试或切换备用模型`
|
||
);
|
||
}
|
||
|
||
async function submitAndPollGrsaiNanoBanana(params: {
|
||
baseURL: string;
|
||
apiKey: string;
|
||
requestBody: Record<string, any>;
|
||
imageParams?: {
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
};
|
||
apiLogPrefix: string;
|
||
}): Promise<any> {
|
||
const submitUrl = `${params.baseURL.replace(/\/+$/, "")}/v1/draw/nano-banana`;
|
||
const body: Record<string, any> = {
|
||
...params.requestBody,
|
||
webHook: "-1",
|
||
};
|
||
const imageSize = String(params.imageParams?.imageSize || "").toUpperCase();
|
||
const pollTimeoutMs = 600000;
|
||
|
||
logger.debug(`${params.apiLogPrefix} Submitting async Nano Banana task:`, {
|
||
model: body.model,
|
||
imageSize: params.imageParams?.imageSize,
|
||
aspectRatio: params.imageParams?.aspectRatio,
|
||
});
|
||
|
||
const response = await fetchWithTimeout(
|
||
submitUrl,
|
||
{
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${params.apiKey}`,
|
||
},
|
||
body: JSON.stringify(body),
|
||
},
|
||
GRSAI_NANO_BANANA_SUBMIT_TIMEOUT_MS
|
||
);
|
||
const submitPayload = await readJsonResponse(response, `${params.apiLogPrefix} submit`);
|
||
const submitStatus = getGrsaiStatus(submitPayload);
|
||
const normalizedSubmitPayload = normalizeGrsaiPayload(submitPayload);
|
||
|
||
if (
|
||
submitStatus === "succeeded" ||
|
||
submitStatus === "success" ||
|
||
submitStatus === "completed" ||
|
||
hasGrsaiImages(submitPayload)
|
||
) {
|
||
return normalizedSubmitPayload;
|
||
}
|
||
|
||
if (submitStatus === "failed" || submitStatus === "error") {
|
||
throw new Error(`图片生成失败: ${getGrsaiErrorMessage(submitPayload)}`);
|
||
}
|
||
|
||
const taskId = getGrsaiTaskId(submitPayload);
|
||
if (!taskId) {
|
||
const providerError = getGrsaiErrorMessage(submitPayload);
|
||
logger.error(
|
||
`${params.apiLogPrefix} Async submit response has no task id:`,
|
||
JSON.stringify(submitPayload).substring(0, 500)
|
||
);
|
||
throw new Error(
|
||
`GRSAI Nano Banana 异步任务提交失败：未返回任务 ID${
|
||
providerError !== "未知错误" ? `: ${providerError}` : ""
|
||
}`
|
||
);
|
||
}
|
||
|
||
logger.debug(`${params.apiLogPrefix} Async task submitted:`, { taskId });
|
||
return pollGrsaiNanoBananaResult({
|
||
baseURL: params.baseURL,
|
||
apiKey: params.apiKey,
|
||
taskId,
|
||
timeoutMs: pollTimeoutMs,
|
||
apiLogPrefix: params.apiLogPrefix,
|
||
});
|
||
}
|
||
|
||
/**
|
||
* 发送 Gemini 图像生成/编辑请求
|
||
* 使用独立的请求格式，不依赖 OpenAI SDK
|
||
* 支持发送完整的消息数组（包括文本和图片）
|
||
*/
|
||
async function sendGeminiRequest(
|
||
baseURL: string,
|
||
apiKey: string,
|
||
messages: any[],
|
||
modelId: string,
|
||
imageParams?: {
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
}
|
||
): Promise<GeminiResponse> {
|
||
const normalizeAspectRatio = (aspectRatio: string) => {
|
||
const normalized = String(aspectRatio || "").trim();
|
||
if (!normalized || normalized.toLowerCase() === "auto") {
|
||
return undefined;
|
||
}
|
||
if (!/^\d+\s*:\s*\d+$/i.test(normalized)) {
|
||
return undefined;
|
||
}
|
||
const [widthRaw, heightRaw] = normalized.split(":").map((item) => item.trim());
|
||
return `${Number(widthRaw)}:${Number(heightRaw)}`;
|
||
};
|
||
|
||
const isGptImageModel = /^gpt-image/i.test(modelId);
|
||
const shouldUseGrsaiAsyncPolling =
|
||
!isGptImageModel &&
|
||
/nano-banana/i.test(modelId) &&
|
||
/(?:grsai|dakka)/i.test(baseURL);
|
||
const apiLogPrefix = isGptImageModel ? "[Grsai GPT Image]" : "[Gemini Image]";
|
||
// GPT Image 和 Nano Banana 在 Grsai 上是两套接口，不能混用同一个 endpoint。
|
||
const url = isGptImageModel
|
||
? `${baseURL}/v1/draw/completions`
|
||
: `${baseURL}/v1/draw/nano-banana`;
|
||
|
||
// 提取最后一条消息的文本作为prompt
|
||
const lastMessage = messages[messages.length - 1];
|
||
const prompt = typeof lastMessage.content === 'string'
|
||
? lastMessage.content
|
||
: lastMessage.content?.find((c: any) => c.type === 'text')?.text || '生成图片';
|
||
|
||
// 提取所有参考图 (urls)
|
||
const urls: string[] = [];
|
||
|
||
// 遍历所有消息提取图片（通常用户会发送多张参考图）
|
||
messages.forEach((msg) => {
|
||
if (msg.role === "user" && Array.isArray(msg.content)) {
|
||
msg.content.forEach((contentPart: any) => {
|
||
const candidateImageUrl =
|
||
contentPart.type === "image_url"
|
||
? contentPart.image_url?.url
|
||
: contentPart.type === "file"
|
||
? contentPart.url || contentPart.data
|
||
: contentPart.type === "image"
|
||
? contentPart.image || contentPart.url
|
||
: undefined;
|
||
|
||
if (!candidateImageUrl || typeof candidateImageUrl !== "string") {
|
||
return;
|
||
}
|
||
|
||
urls.push(candidateImageUrl);
|
||
});
|
||
}
|
||
});
|
||
|
||
// 🆕 处理参考图：上传 Base64 到 S3
|
||
const processedUrls: string[] = [];
|
||
if (urls.length > 0) {
|
||
logger.debug(`${apiLogPrefix} Processing ${urls.length} reference images...`);
|
||
for (const url of urls) {
|
||
const s3Url = await ensureImageUrl(url);
|
||
processedUrls.push(s3Url);
|
||
}
|
||
}
|
||
const promptForRequest = buildReferenceImageOrderPrompt(prompt, processedUrls);
|
||
|
||
logger.debug(`${apiLogPrefix} Sending request to:`, url);
|
||
logger.debug(`${apiLogPrefix} Using model:`, modelId);
|
||
logger.debug(`${apiLogPrefix} Prompt:`, promptForRequest.substring(0, 100));
|
||
logger.debug(`${apiLogPrefix} Reference image count:`, urls.length);
|
||
logger.debug(`${apiLogPrefix} Image params:`, imageParams);
|
||
|
||
// 使用对应模型的原生格式
|
||
const requestBody: any = {
|
||
model: modelId,
|
||
prompt: promptForRequest,
|
||
};
|
||
|
||
if (isGptImageModel) {
|
||
const requestedAspectRatio = imageParams?.aspectRatio || "auto";
|
||
const inferredAspectRatio =
|
||
requestedAspectRatio === "auto" && processedUrls.length > 0
|
||
? await inferImageAspectRatioFromUrl(processedUrls[0])
|
||
: null;
|
||
const resolvedAspectRatio = inferredAspectRatio || requestedAspectRatio;
|
||
const resolvedSize = resolveGrsaiGptImageSize(resolvedAspectRatio, imageParams?.imageSize);
|
||
|
||
// Grsai GPT-image-2-vip 只接受固定像素 size；继续传比例或档位字段会触发
|
||
// “does not support 1:1/3:4 parameter, please use pixel value input”。
|
||
requestBody.size = resolvedSize;
|
||
requestBody.shutProgress = false;
|
||
|
||
if (processedUrls.length > 0) {
|
||
requestBody.image = processedUrls[0];
|
||
requestBody.imageUrls = processedUrls;
|
||
}
|
||
logger.debug(
|
||
`${apiLogPrefix} GPT image payload mapping: requestedAspectRatio=${requestedAspectRatio}, resolvedAspectRatio=${resolvedAspectRatio}, imageSize=${imageParams?.imageSize || "2K"}, resolvedSize=${resolvedSize}`
|
||
);
|
||
if (inferredAspectRatio) {
|
||
logger.debug(`${apiLogPrefix} Inferred reference aspect ratio: ${inferredAspectRatio}`);
|
||
}
|
||
logger.debug(
|
||
`${apiLogPrefix} Resolved size: aspectRatio=${resolvedAspectRatio}, imageSize=${imageParams?.imageSize || "2K"} => ${resolvedSize}`
|
||
);
|
||
|
||
const timeout =
|
||
imageParams?.imageSize === "4K"
|
||
? GRSAI_GPT_IMAGE_4K_TIMEOUT_MS
|
||
: GRSAI_GPT_IMAGE_TIMEOUT_MS;
|
||
logger.debug(`${apiLogPrefix} Timeout set to ${timeout / 1000} seconds for ${imageParams?.imageSize || "default"}`);
|
||
return sendGrsaiGptImageRequest({
|
||
baseURL,
|
||
apiKey,
|
||
modelId,
|
||
prompt: promptForRequest,
|
||
size: resolvedSize,
|
||
imageUrls: processedUrls,
|
||
timeout,
|
||
apiLogPrefix,
|
||
});
|
||
} else {
|
||
// 添加 Nano Banana 特定参数
|
||
if (processedUrls.length > 0) {
|
||
requestBody.urls = processedUrls;
|
||
}
|
||
|
||
if (imageParams?.aspectRatio) {
|
||
requestBody.aspectRatio = imageParams.aspectRatio;
|
||
}
|
||
if (imageParams?.imageSize) {
|
||
requestBody.imageSize = imageParams.imageSize;
|
||
}
|
||
}
|
||
|
||
logger.debug(`${apiLogPrefix} Request body (truncated):`, JSON.stringify(requestBody).substring(0, 500) + "...");
|
||
|
||
if (shouldUseGrsaiAsyncPolling) {
|
||
return submitAndPollGrsaiNanoBanana({
|
||
baseURL,
|
||
apiKey,
|
||
requestBody,
|
||
imageParams,
|
||
apiLogPrefix,
|
||
});
|
||
}
|
||
|
||
// 根据分辨率动态设置超时时间
|
||
const timeout = 600000; // 所有生图模型最长等待 10 分钟
|
||
logger.debug(`${apiLogPrefix} Timeout set to ${timeout/1000} seconds for ${imageParams?.imageSize || "default"}`);
|
||
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), timeout);
|
||
|
||
try {
|
||
const response = await fetch(url, {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
Authorization: `Bearer ${apiKey}`,
|
||
},
|
||
body: JSON.stringify(requestBody),
|
||
signal: controller.signal,
|
||
});
|
||
|
||
clearTimeout(timeoutId);
|
||
|
||
if (!response.ok) {
|
||
const errorText = await response.text();
|
||
logger.error(`${apiLogPrefix} Request failed:`, response.status, errorText);
|
||
throw new Error(
|
||
`${isGptImageModel ? "Grsai GPT Image" : "Gemini"} API error: ${response.status} ${response.statusText}`
|
||
);
|
||
}
|
||
|
||
// 处理流式响应（SSE格式）
|
||
logger.debug(`${apiLogPrefix} Reading stream response...`);
|
||
const reader = response.body?.getReader();
|
||
const decoder = new TextDecoder();
|
||
let buffer = "";
|
||
let lastValidData: any = null;
|
||
|
||
if (!reader) {
|
||
throw new Error("Response body is null");
|
||
}
|
||
|
||
while (true) {
|
||
const { done, value } = await reader.read();
|
||
|
||
if (done) {
|
||
logger.debug("[Gemini Image] Stream completed");
|
||
break;
|
||
}
|
||
|
||
// 将新数据添加到缓冲区
|
||
buffer += decoder.decode(value, { stream: true });
|
||
|
||
// 按行分割
|
||
const lines = buffer.split("\n");
|
||
buffer = lines.pop() || ""; // 保留最后一个不完整的行
|
||
|
||
for (const line of lines) {
|
||
const trimmedLine = line.trim();
|
||
|
||
// 解析 SSE 格式: "data: {...}"
|
||
if (trimmedLine.startsWith("data: ")) {
|
||
try {
|
||
const jsonStr = trimmedLine.substring(6); // 移除 "data: " 前缀
|
||
const data = JSON.parse(jsonStr);
|
||
|
||
// 保存最新的有效数据
|
||
lastValidData = data;
|
||
|
||
// 记录进度
|
||
if (data.progress !== undefined) {
|
||
logger.debug(`[Gemini Image] Progress: ${data.progress}%`);
|
||
}
|
||
|
||
// 如果成功完成，立即返回
|
||
if (data.status === "succeeded") {
|
||
logger.debug("[Gemini Image] Generation succeeded!");
|
||
logger.debug("[Gemini Image] Full success data:", JSON.stringify(data).substring(0, 1000)); // 🔍 打印完整成功数据
|
||
return data as GeminiResponse;
|
||
}
|
||
|
||
// 如果失败，抛出错误（不被catch捕获）
|
||
if (data.status === "failed") {
|
||
const failureReason = String(data.failure_reason || "").trim();
|
||
const providerError = String(data.error || "").trim();
|
||
const detailParts = [failureReason, providerError].filter(
|
||
(part, index, arr) =>
|
||
part && arr.findIndex((candidate) => candidate === part) === index
|
||
);
|
||
const errorMsg = `图片生成失败: ${detailParts.join(" / ") || "未知错误"}`;
|
||
logger.error("[Gemini Image] Generation failed:", data.failure_reason, data.error);
|
||
clearTimeout(timeoutId);
|
||
throw new Error(errorMsg);
|
||
}
|
||
} catch (e) {
|
||
// 只捕获JSON解析错误，重新抛出其他错误
|
||
if (e instanceof Error && e.message.includes("图片生成失败")) {
|
||
throw e; // 重新抛出失败错误
|
||
}
|
||
logger.warn("[Gemini Image] Failed to parse SSE line:", trimmedLine);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 流结束后，返回最后一个有效数据
|
||
if (lastValidData) {
|
||
logger.debug("[Gemini Image] Returning last valid data");
|
||
return lastValidData as GeminiResponse;
|
||
}
|
||
|
||
throw new Error("No valid data received from stream");
|
||
} catch (error) {
|
||
clearTimeout(timeoutId);
|
||
if (error instanceof Error && error.name === "AbortError") {
|
||
const timeoutSeconds = Math.round(timeout / 1000);
|
||
logger.error(`${apiLogPrefix} Request timeout after ${timeoutSeconds} seconds`);
|
||
throw new Error(
|
||
`${isGptImageModel ? "Grsai GPT Image" : "Gemini"} 请求超时（${timeoutSeconds}秒），将自动切换备用模型`
|
||
);
|
||
}
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 创建 Gemini 图像生成 Provider
|
||
* 使用与 local-qwen.ts 相同的格式
|
||
*/
|
||
export function createGeminiImage({ apiKey, baseURL }: GeminiImageConfig) {
|
||
return {
|
||
chat: (modelId: string) => ({
|
||
doGenerate: async (options: any) => {
|
||
logger.debug("[Gemini Image] doGenerate called!"); // 调试日志
|
||
logger.debug("[Gemini Image] Messages received:", JSON.stringify(options.messages).substring(0, 500) + "...");
|
||
// 🆕 支持覆盖 modelId（用于备用模型切换）
|
||
const effectiveModelId = options.overrideModelId || modelId;
|
||
logger.debug("[Gemini Image] Using modelId:", effectiveModelId, options.overrideModelId ? "(overridden)" : "(default)");
|
||
logger.debug("[Gemini Image] Image params from options:", options.imageParams);
|
||
|
||
try {
|
||
// 直接传递完整的消息数组（包括文本和图片）
|
||
// 确保imageSize有值，优先级：用户选择 > 默认2K
|
||
const imageParams = {
|
||
aspectRatio: options.imageParams?.aspectRatio || "auto",
|
||
imageSize: options.imageParams?.imageSize || "2K"
|
||
};
|
||
// 🆕 获取用户选择的图片数量，默认 1 张
|
||
const imageCount = Math.min(Math.max(options.imageParams?.imageCount || 1, 1), 2); // Gemini 限制 1-2 张
|
||
logger.debug("[Gemini Image] Final imageParams after defaults:", imageParams);
|
||
logger.debug(`[Gemini Image] Image count: ${imageCount}`);
|
||
|
||
// 🆕 根据用户选择的数量发起请求
|
||
logger.debug(`[Gemini Image] Starting ${imageCount} concurrent request(s)...`);
|
||
const startTime = Date.now();
|
||
|
||
// 创建请求数组（使用 effectiveModelId）
|
||
const requestPromises = Array.from({ length: imageCount }, () =>
|
||
sendGeminiRequest(baseURL, apiKey, options.messages, effectiveModelId, imageParams)
|
||
);
|
||
|
||
const responses = await Promise.all(requestPromises);
|
||
|
||
logger.debug(`[Gemini Image] All ${imageCount} request(s) completed in ${Date.now() - startTime}ms`);
|
||
|
||
// 提取图片并上传 base64 到 S3（并行处理）
|
||
const extractPromises = responses.map(response => extractAndUploadImages(response));
|
||
const imageUrlsArrays = await Promise.all(extractPromises);
|
||
|
||
// 合并所有请求的图片结果
|
||
const allImageUrls = imageUrlsArrays.flat();
|
||
const markdownText = formatImageResponse(allImageUrls, imageParams.imageSize);
|
||
|
||
logger.debug(
|
||
`[Gemini Image] Generated ${allImageUrls.length} images from ${imageCount} concurrent request(s) (uploaded to S3)`
|
||
);
|
||
|
||
// 返回 AI SDK v1 格式
|
||
return {
|
||
text: markdownText,
|
||
finishReason: "stop" as const,
|
||
usage: {
|
||
promptTokens: 0,
|
||
completionTokens: markdownText.length,
|
||
},
|
||
};
|
||
} catch (error) {
|
||
logger.error("[Gemini Image] Generation failed:", error);
|
||
|
||
// ❌ 抛出错误以触发故障转移（不要吞掉错误）
|
||
throw error;
|
||
}
|
||
},
|
||
|
||
doStream: async function* (options: any) {
|
||
logger.debug("[Gemini Image] ===== doStream ENTERED! ====="); // 调试日志
|
||
logger.debug("[Gemini Image] Messages received:", JSON.stringify(options.messages).substring(0, 500) + "...");
|
||
// 🆕 支持覆盖 modelId（用于备用模型切换）
|
||
const effectiveModelId = options.overrideModelId || modelId;
|
||
logger.debug("[Gemini Image] Using modelId:", effectiveModelId, options.overrideModelId ? "(overridden)" : "(default)");
|
||
logger.debug("[Gemini Image] Image params from options:", options.imageParams);
|
||
|
||
try {
|
||
// 直接传递完整的消息数组（包括文本和图片）
|
||
// 确保imageSize有值，优先级：用户选择 > 默认2K
|
||
const imageParams = {
|
||
aspectRatio: options.imageParams?.aspectRatio || "auto",
|
||
imageSize: options.imageParams?.imageSize || "2K"
|
||
};
|
||
// 🆕 获取用户选择的图片数量，默认 1 张
|
||
const imageCount = Math.min(Math.max(options.imageParams?.imageCount || 1, 1), 2); // Gemini 限制 1-2 张
|
||
logger.debug("[Gemini Image] Final imageParams after defaults:", imageParams);
|
||
logger.debug(`[Gemini Image Stream] Image count: ${imageCount}`);
|
||
|
||
// 🆕 根据用户选择的数量发起请求
|
||
logger.debug(`[Gemini Image Stream] Starting ${imageCount} concurrent request(s)...`);
|
||
const startTime = Date.now();
|
||
|
||
// 创建请求数组（使用 effectiveModelId）
|
||
const requestPromises = Array.from({ length: imageCount }, () =>
|
||
sendGeminiRequest(baseURL, apiKey, options.messages, effectiveModelId, imageParams)
|
||
);
|
||
|
||
const responses = await Promise.all(requestPromises);
|
||
|
||
logger.debug(`[Gemini Image Stream] All ${imageCount} request(s) completed in ${Date.now() - startTime}ms`);
|
||
|
||
// 提取图片并上传 base64 到 S3（并行处理）
|
||
const extractPromises = responses.map(response => extractAndUploadImages(response));
|
||
const imageUrlsArrays = await Promise.all(extractPromises);
|
||
|
||
// 合并所有请求的图片结果
|
||
const allImageUrls = imageUrlsArrays.flat();
|
||
const markdownText = formatImageResponse(allImageUrls, imageParams.imageSize);
|
||
|
||
logger.debug(
|
||
`[Gemini Image Stream] Generated ${allImageUrls.length} images from ${imageCount} concurrent request(s) (uploaded to S3)`
|
||
);
|
||
|
||
// 一次性发送整个图片 markdown（不再分块，因为 base64 图片很大）
|
||
yield { type: "text-delta" as const, textDelta: markdownText };
|
||
|
||
logger.debug("[Gemini Image Stream] Sent complete markdown");
|
||
|
||
// 发送完成信号
|
||
yield {
|
||
type: "finish" as const,
|
||
finishReason: "stop" as const,
|
||
usage: {
|
||
promptTokens: 0,
|
||
completionTokens: markdownText.length,
|
||
},
|
||
};
|
||
} catch (error) {
|
||
logger.error("[Gemini Image Stream] Generation failed:", error);
|
||
|
||
const errorMessage =
|
||
error instanceof Error
|
||
? `图片生成失败: ${error.message}`
|
||
: "图片生成失败，请稍后重试";
|
||
|
||
// 发送错误信息
|
||
yield { type: "text-delta" as const, textDelta: errorMessage };
|
||
|
||
yield {
|
||
type: "finish" as const,
|
||
finishReason: "error" as const,
|
||
usage: { promptTokens: 0, completionTokens: 0 },
|
||
};
|
||
}
|
||
},
|
||
|
||
modelId,
|
||
provider: "gemini-image",
|
||
specificationVersion: "v1" as const,
|
||
defaultObjectGenerationMode: "json" as const,
|
||
}),
|
||
};
|
||
}
|