962 lines
30 KiB
TypeScript
962 lines
30 KiB
TypeScript
import "server-only";
|
||
|
||
import sharp from "sharp";
|
||
import {
|
||
resolveLongestEdgeDimensions,
|
||
} from "@/lib/ai/image-size-constraints";
|
||
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
|
||
interface OpenAIImageConfig {
|
||
apiKey: string;
|
||
baseURL: string;
|
||
}
|
||
|
||
export interface OpenAIImageParams {
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
imageCount?: number;
|
||
size?: string;
|
||
quality?: string;
|
||
background?: string;
|
||
outputFormat?: string;
|
||
maskImageUrl?: string;
|
||
}
|
||
|
||
interface UploadedImage {
|
||
thumbnail: string;
|
||
original: string;
|
||
}
|
||
|
||
type InputImageReference = {
|
||
url: string;
|
||
width?: number;
|
||
height?: number;
|
||
};
|
||
|
||
const DEFAULT_IMAGE_SIZE = "2K";
|
||
const DEFAULT_TIMEOUT_MS = 600000;
|
||
const LONG_RUNNING_COMFLY_TIMEOUT_MS = 600000;
|
||
const OPENAI_IMAGE_MIN_TOTAL_PIXELS = 3_686_400;
|
||
const OPENAI_IMAGE_DIMENSION_MULTIPLE = 16;
|
||
const COMFLY_GPT_IMAGE_STABLE_EDIT_EDGE = 1536;
|
||
const COMFLY_GPT_IMAGE_MAX_EDGE = 3840;
|
||
const COMFLY_GPT_IMAGE_MAX_TOTAL_PIXELS = 8_294_400;
|
||
const AGNES_IMAGE_LONG_EDGE_BY_SIZE: Record<"1K" | "2K" | "4K", number> = {
|
||
"1K": 1024,
|
||
"2K": 1536,
|
||
"4K": 2048,
|
||
};
|
||
const AGNES_IMAGE_MAX_EDGE = 2048;
|
||
const AGNES_IMAGE_MAX_TOTAL_PIXELS = 4_194_304;
|
||
const BLTCY_HOST_REGEX = /(^|\.)bltcy\.ai$/i;
|
||
const COMFLY_HOST_REGEX = /(^|\.)ai\.comfly\.org$/i;
|
||
const AGNES_HOST_REGEX = /(^|\.)agnes-ai\.com$/i;
|
||
const SUPPORTED_ASPECT_RATIOS = [
|
||
"1:1",
|
||
"1:4",
|
||
"1:8",
|
||
"2:3",
|
||
"3:2",
|
||
"3:4",
|
||
"4:1",
|
||
"4:3",
|
||
"4:5",
|
||
"5:4",
|
||
"8:1",
|
||
"9:16",
|
||
"16:9",
|
||
"21:9",
|
||
] as const;
|
||
|
||
function normalizeBaseURL(baseURL: string) {
|
||
const normalized = String(baseURL || "").trim().replace(/\/+$/, "");
|
||
if (!normalized) return "https://api.openai.com/v1";
|
||
if (/\/v1$/i.test(normalized)) return normalized;
|
||
return `${normalized}/v1`;
|
||
}
|
||
|
||
function joinEndpoint(baseURL: string, path: string) {
|
||
const normalizedPath = path.startsWith("/") ? path : `/${path}`;
|
||
return `${normalizeBaseURL(baseURL)}${normalizedPath}`;
|
||
}
|
||
|
||
function isBltcyBaseURL(baseURL: string) {
|
||
try {
|
||
const parsed = new URL(normalizeBaseURL(baseURL));
|
||
return BLTCY_HOST_REGEX.test(parsed.hostname);
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
function shouldUseBltcyImageCompatParams(baseURL: string) {
|
||
return isBltcyBaseURL(baseURL);
|
||
}
|
||
|
||
function isComflyBaseURL(baseURL: string) {
|
||
try {
|
||
const parsed = new URL(normalizeBaseURL(baseURL));
|
||
return COMFLY_HOST_REGEX.test(parsed.hostname);
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
function isAgnesBaseURL(baseURL: string) {
|
||
try {
|
||
const parsed = new URL(normalizeBaseURL(baseURL));
|
||
return AGNES_HOST_REGEX.test(parsed.hostname);
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
function isAgnesImageModel(baseURL: string, modelId?: string) {
|
||
const normalizedModelId = String(modelId || "").trim().toLowerCase();
|
||
return isAgnesBaseURL(baseURL) && normalizedModelId.startsWith("agnes-image-");
|
||
}
|
||
|
||
function normalizeImageSizeLabel(imageSize?: string): "1K" | "2K" | "4K" {
|
||
const normalized = String(imageSize || "").trim().toUpperCase();
|
||
if (normalized === "1K" || normalized === "2K" || normalized === "4K") return normalized;
|
||
return DEFAULT_IMAGE_SIZE;
|
||
}
|
||
|
||
function inferImageSizeFromModelId(modelId?: string): "1K" | "2K" | "4K" | undefined {
|
||
const normalized = String(modelId || "").trim().toLowerCase();
|
||
const explicitBucket = normalized.match(/(?:^|[-_\s])([124])k(?:$|[-_\s])/);
|
||
if (explicitBucket?.[1] === "1") return "1K";
|
||
if (explicitBucket?.[1] === "2") return "2K";
|
||
if (explicitBucket?.[1] === "4") return "4K";
|
||
return undefined;
|
||
}
|
||
|
||
function normalizeAspectRatio(value?: string) {
|
||
const normalized = String(value || "").trim();
|
||
if (!normalized || normalized.toLowerCase() === "auto") return "auto";
|
||
if ((SUPPORTED_ASPECT_RATIOS as readonly string[]).includes(normalized)) return normalized;
|
||
return undefined;
|
||
}
|
||
|
||
function parseAspectRatioPair(ratio: string) {
|
||
const [width, height] = ratio.split(":").map((item) => Number(item));
|
||
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
|
||
return null;
|
||
}
|
||
return { width, height };
|
||
}
|
||
|
||
function parseAspectRatioValue(ratio: string) {
|
||
const pair = parseAspectRatioPair(ratio);
|
||
if (!pair) return 1;
|
||
const { width, height } = pair;
|
||
return width / height;
|
||
}
|
||
|
||
function ceilToMultiple(value: number, multiple: number) {
|
||
return Math.max(multiple, Math.ceil(value / multiple) * multiple);
|
||
}
|
||
|
||
function inferAspectRatioFromPrompt(prompt: string) {
|
||
const normalized = String(prompt || "").replace(/：/g, ":");
|
||
const match = normalized.match(/(\d{1,2})\s*(?::|比)\s*(\d{1,2})/);
|
||
if (!match) return undefined;
|
||
|
||
const ratio = `${Number(match[1])}:${Number(match[2])}`;
|
||
return (SUPPORTED_ASPECT_RATIOS as readonly string[]).includes(ratio) ? ratio : undefined;
|
||
}
|
||
|
||
function inferClosestAspectRatio(aspectValue: number) {
|
||
if (!Number.isFinite(aspectValue) || aspectValue <= 0) return "1:1";
|
||
let closest = "1:1";
|
||
let minDiff = Number.POSITIVE_INFINITY;
|
||
for (const candidate of SUPPORTED_ASPECT_RATIOS) {
|
||
const diff = Math.abs(parseAspectRatioValue(candidate) - aspectValue);
|
||
if (diff < minDiff) {
|
||
minDiff = diff;
|
||
closest = candidate;
|
||
}
|
||
}
|
||
return closest;
|
||
}
|
||
|
||
function inferAspectRatioFromDimensions(width?: number, height?: number) {
|
||
const safeWidth = Number(width);
|
||
const safeHeight = Number(height);
|
||
if (
|
||
!Number.isFinite(safeWidth) ||
|
||
!Number.isFinite(safeHeight) ||
|
||
safeWidth <= 0 ||
|
||
safeHeight <= 0
|
||
) {
|
||
return null;
|
||
}
|
||
return inferClosestAspectRatio(safeWidth / safeHeight);
|
||
}
|
||
|
||
async function inferAspectRatioFromImageUrl(imageUrl: string): Promise<string | null> {
|
||
const normalized = String(imageUrl || "").trim();
|
||
if (!normalized) return null;
|
||
|
||
try {
|
||
let buffer: Buffer | null = null;
|
||
if (normalized.startsWith("data:")) {
|
||
const match = normalized.match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/);
|
||
if (!match) return null;
|
||
buffer = Buffer.from(match[2], "base64");
|
||
} else if (/^https?:\/\//i.test(normalized)) {
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), 20000);
|
||
try {
|
||
const response = await fetch(normalized, {
|
||
signal: controller.signal,
|
||
cache: "no-store",
|
||
});
|
||
if (!response.ok) return null;
|
||
buffer = Buffer.from(await response.arrayBuffer());
|
||
} finally {
|
||
clearTimeout(timeoutId);
|
||
}
|
||
}
|
||
|
||
if (!buffer?.length) return null;
|
||
const metadata = await sharp(buffer).metadata();
|
||
if (!metadata.width || !metadata.height || metadata.height <= 0) return null;
|
||
return inferClosestAspectRatio(metadata.width / metadata.height);
|
||
} catch (error) {
|
||
console.warn("[OpenAI Image] Failed to infer aspect ratio:", error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function resolveOpenAISize(aspectRatio: string, imageSize: "1K" | "2K" | "4K") {
|
||
const baseMap: Record<typeof imageSize, number> = {
|
||
"1K": 1024,
|
||
"2K": 1536,
|
||
"4K": 2048,
|
||
};
|
||
|
||
const ratioPair = parseAspectRatioPair(aspectRatio) || { width: 1, height: 1 };
|
||
const base = baseMap[imageSize] || 1536;
|
||
const targetPixels = Math.max(base * base, OPENAI_IMAGE_MIN_TOTAL_PIXELS);
|
||
const scale = ceilToMultiple(
|
||
Math.sqrt(targetPixels / (ratioPair.width * ratioPair.height)),
|
||
OPENAI_IMAGE_DIMENSION_MULTIPLE,
|
||
);
|
||
const width = ratioPair.width * scale;
|
||
const height = ratioPair.height * scale;
|
||
|
||
return `${width}x${height}`;
|
||
}
|
||
|
||
function resolveComflyGptImageSize(aspectRatio: string, imageSize: "1K" | "2K" | "4K") {
|
||
const edgeMap: Record<typeof imageSize, number> = {
|
||
"1K": COMFLY_GPT_IMAGE_STABLE_EDIT_EDGE,
|
||
"2K": COMFLY_GPT_IMAGE_STABLE_EDIT_EDGE,
|
||
"4K": COMFLY_GPT_IMAGE_STABLE_EDIT_EDGE,
|
||
};
|
||
const { width, height } = resolveLongestEdgeDimensions({
|
||
aspectRatio,
|
||
longestEdge: edgeMap[imageSize] || edgeMap["2K"],
|
||
constraints: {
|
||
maxEdge: COMFLY_GPT_IMAGE_MAX_EDGE,
|
||
maxPixels: COMFLY_GPT_IMAGE_MAX_TOTAL_PIXELS,
|
||
multiple: OPENAI_IMAGE_DIMENSION_MULTIPLE,
|
||
},
|
||
});
|
||
|
||
return `${width}x${height}`;
|
||
}
|
||
|
||
function resolveAgnesImageSize(aspectRatio: string, imageSize: "1K" | "2K" | "4K") {
|
||
const { width, height } = resolveLongestEdgeDimensions({
|
||
aspectRatio,
|
||
longestEdge: AGNES_IMAGE_LONG_EDGE_BY_SIZE[imageSize] || AGNES_IMAGE_LONG_EDGE_BY_SIZE["2K"],
|
||
constraints: {
|
||
maxEdge: AGNES_IMAGE_MAX_EDGE,
|
||
maxPixels: AGNES_IMAGE_MAX_TOTAL_PIXELS,
|
||
multiple: 8,
|
||
},
|
||
});
|
||
|
||
return `${width}x${height}`;
|
||
}
|
||
|
||
function resolveProviderImageSize(
|
||
baseURL: string,
|
||
aspectRatio: string,
|
||
imageSize: "1K" | "2K" | "4K"
|
||
) {
|
||
if (isComflyBaseURL(baseURL)) {
|
||
return resolveComflyGptImageSize(aspectRatio, imageSize);
|
||
}
|
||
if (isAgnesBaseURL(baseURL)) {
|
||
return resolveAgnesImageSize(aspectRatio, imageSize);
|
||
}
|
||
return resolveOpenAISize(aspectRatio, imageSize);
|
||
}
|
||
|
||
function isLongRunningComflyModel(modelId?: string) {
|
||
const normalized = String(modelId || "").trim().toLowerCase();
|
||
return normalized.includes("gpt-image-2") && normalized.includes("4k");
|
||
}
|
||
|
||
function isComflyGptImage2Model(baseURL: string, modelId?: string) {
|
||
const normalizedModelId = String(modelId || "").trim().toLowerCase();
|
||
return isComflyBaseURL(baseURL) && normalizedModelId.includes("gpt-image-2");
|
||
}
|
||
|
||
function resolveProviderApiModelId(baseURL: string, modelId: string) {
|
||
if (isComflyGptImage2Model(baseURL, modelId)) {
|
||
return "gpt-image-2";
|
||
}
|
||
return modelId;
|
||
}
|
||
|
||
function resolveRequestTimeoutMs(
|
||
baseURL: string,
|
||
imageSize?: "1K" | "2K" | "4K",
|
||
modelId?: string
|
||
) {
|
||
if (isComflyBaseURL(baseURL) && (imageSize === "4K" || isLongRunningComflyModel(modelId))) {
|
||
return LONG_RUNNING_COMFLY_TIMEOUT_MS;
|
||
}
|
||
return DEFAULT_TIMEOUT_MS;
|
||
}
|
||
|
||
function normalizeQuality(value?: string) {
|
||
const normalized = String(value || "").trim().toLowerCase();
|
||
if (normalized === "low" || normalized === "medium" || normalized === "high") {
|
||
return normalized;
|
||
}
|
||
return undefined;
|
||
}
|
||
|
||
function normalizeBackground(value?: string) {
|
||
const normalized = String(value || "").trim().toLowerCase();
|
||
if (normalized === "transparent" || normalized === "opaque") return normalized;
|
||
return undefined;
|
||
}
|
||
|
||
function normalizeOutputFormat(value?: string) {
|
||
const normalized = String(value || "").trim().toLowerCase();
|
||
if (normalized === "jpeg" || normalized === "jpg") return "jpeg";
|
||
if (normalized === "webp") return "webp";
|
||
if (normalized === "png") return "png";
|
||
return undefined;
|
||
}
|
||
|
||
function detectImageExtension(imageUrl: string, contentType?: string | null) {
|
||
const type = String(contentType || "").toLowerCase();
|
||
if (type.includes("jpeg") || type.includes("jpg")) return "jpg";
|
||
if (type.includes("webp")) return "webp";
|
||
if (type.includes("png")) return "png";
|
||
if (type.includes("gif")) return "gif";
|
||
|
||
const cleanUrl = imageUrl.split("?")[0]?.toLowerCase() || "";
|
||
if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "jpg";
|
||
if (cleanUrl.endsWith(".webp")) return "webp";
|
||
if (cleanUrl.endsWith(".gif")) return "gif";
|
||
if (cleanUrl.endsWith(".png")) return "png";
|
||
return "png";
|
||
}
|
||
|
||
async function createThumbnail(buffer: Buffer) {
|
||
try {
|
||
return await sharp(buffer)
|
||
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
|
||
.jpeg({ quality: 80 })
|
||
.toBuffer();
|
||
} catch (error) {
|
||
console.error("[OpenAI Image] Failed to create thumbnail:", error);
|
||
return buffer;
|
||
}
|
||
}
|
||
|
||
function generateFilename(prefix: string, extension: string) {
|
||
const timestamp = Date.now();
|
||
const random = Math.random().toString(36).slice(2, 8);
|
||
return `${prefix}-${timestamp}-${random}.${extension}`;
|
||
}
|
||
|
||
async function uploadGeneratedBuffer(
|
||
buffer: Buffer,
|
||
extension: string,
|
||
mimeType: string
|
||
): Promise<UploadedImage> {
|
||
const thumbnailBuffer = await createThumbnail(buffer);
|
||
|
||
const originalFilename = generateFilename("openai-orig", extension);
|
||
const originalArrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer);
|
||
|
||
const thumbFilename = generateFilename("openai-thumb", "jpg");
|
||
const thumbArrayBuffer = thumbnailBuffer.buffer.slice(
|
||
thumbnailBuffer.byteOffset,
|
||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
|
||
) as ArrayBuffer;
|
||
const thumbUploaded = await uploadFile(thumbFilename, thumbArrayBuffer);
|
||
|
||
console.log("[OpenAI Image] Uploaded generated image:", {
|
||
original: originalUploaded.url,
|
||
thumbnail: thumbUploaded.url,
|
||
mimeType,
|
||
});
|
||
|
||
return {
|
||
thumbnail: thumbUploaded.url,
|
||
original: originalUploaded.url,
|
||
};
|
||
}
|
||
|
||
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
||
const deferred = await createDeferredUploadedImage(imageUrl, {
|
||
providerType: "openai-image",
|
||
});
|
||
if (deferred) return deferred;
|
||
|
||
const response = await fetch(imageUrl, { cache: "no-store" });
|
||
if (!response.ok) {
|
||
throw new Error(`Failed to download generated image: ${response.status}`);
|
||
}
|
||
const buffer = Buffer.from(await response.arrayBuffer());
|
||
const extension = detectImageExtension(imageUrl, response.headers.get("content-type"));
|
||
const mimeType = response.headers.get("content-type") || `image/${extension === "jpg" ? "jpeg" : extension}`;
|
||
return uploadGeneratedBuffer(buffer, extension, mimeType);
|
||
}
|
||
|
||
async function uploadBase64ToS3(base64Data: string, mimeType: string) {
|
||
const buffer = Buffer.from(base64Data, "base64");
|
||
const extension = detectImageExtension("", mimeType);
|
||
return uploadGeneratedBuffer(
|
||
buffer,
|
||
extension,
|
||
mimeType || `image/${extension === "jpg" ? "jpeg" : extension}`
|
||
);
|
||
}
|
||
|
||
function extractPromptAndImages(messages: Array<{ role: string; content: any }>) {
|
||
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
|
||
if (!lastUserMessage) {
|
||
throw new Error("No user message found");
|
||
}
|
||
|
||
let prompt = "";
|
||
const imageRefs: InputImageReference[] = [];
|
||
|
||
const pushImageUrl = (value?: string, geometry?: { width?: unknown; height?: unknown }) => {
|
||
if (!value) return;
|
||
const normalized = String(value).trim();
|
||
if (!normalized) return;
|
||
if (!imageRefs.some((item) => item.url === normalized)) {
|
||
const width = Number(geometry?.width);
|
||
const height = Number(geometry?.height);
|
||
imageRefs.push({
|
||
url: normalized,
|
||
width: Number.isFinite(width) && width > 0 ? Math.round(width) : undefined,
|
||
height: Number.isFinite(height) && height > 0 ? Math.round(height) : undefined,
|
||
});
|
||
}
|
||
};
|
||
|
||
if (typeof lastUserMessage.content === "string") {
|
||
prompt = lastUserMessage.content.trim();
|
||
} else if (Array.isArray(lastUserMessage.content)) {
|
||
for (const part of lastUserMessage.content) {
|
||
if (part?.type === "text" && typeof part.text === "string") {
|
||
prompt = `${prompt} ${part.text}`.trim();
|
||
} else if (part?.type === "image_url" && part.image_url?.url) {
|
||
pushImageUrl(part.image_url.url, part);
|
||
} else if (part?.type === "file") {
|
||
pushImageUrl(part.url || part.data, part);
|
||
} else if (part?.type === "image") {
|
||
pushImageUrl(part.image || part.url, part);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!prompt) {
|
||
throw new Error("No prompt found in user message");
|
||
}
|
||
|
||
return { prompt, imageUrls: imageRefs.map((item) => item.url), imageRefs };
|
||
}
|
||
|
||
async function loadInputImage(source: string) {
|
||
const normalized = String(source || "").trim();
|
||
if (!normalized) {
|
||
throw new Error("输入图片地址为空");
|
||
}
|
||
|
||
if (normalized.startsWith("data:")) {
|
||
const match = normalized.match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/);
|
||
if (!match) {
|
||
throw new Error("不支持的 Base64 图片格式");
|
||
}
|
||
const mimeType = match[1] || "image/png";
|
||
const buffer = Buffer.from(match[2], "base64");
|
||
const extension = detectImageExtension("", mimeType);
|
||
return {
|
||
buffer,
|
||
mimeType,
|
||
filename: generateFilename("openai-edit-input", extension),
|
||
};
|
||
}
|
||
|
||
const response = await fetch(normalized, { cache: "no-store" });
|
||
if (!response.ok) {
|
||
throw new Error(`Failed to fetch input image: ${response.status}`);
|
||
}
|
||
const buffer = Buffer.from(await response.arrayBuffer());
|
||
const mimeType = response.headers.get("content-type") || "image/png";
|
||
const extension = detectImageExtension(normalized, mimeType);
|
||
return {
|
||
buffer,
|
||
mimeType,
|
||
filename: generateFilename("openai-edit-input", extension),
|
||
};
|
||
}
|
||
|
||
function extractErrorMessage(payload: any) {
|
||
if (!payload) return "OpenAI Images 请求失败";
|
||
if (typeof payload === "string") return payload;
|
||
if (payload.error?.message) return String(payload.error.message);
|
||
if (payload.message) return String(payload.message);
|
||
try {
|
||
return JSON.stringify(payload);
|
||
} catch {
|
||
return "OpenAI Images 请求失败";
|
||
}
|
||
}
|
||
|
||
async function parseJsonResponse(response: Response) {
|
||
const rawText = await response.text();
|
||
try {
|
||
return JSON.parse(rawText);
|
||
} catch {
|
||
return rawText;
|
||
}
|
||
}
|
||
|
||
function extractImageUrlsFromText(text: string) {
|
||
const normalized = String(text || "");
|
||
if (!normalized) return [];
|
||
|
||
const urls = new Set<string>();
|
||
const markdownImageRegex = /!\[[^\]]*]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
|
||
for (const match of normalized.matchAll(markdownImageRegex)) {
|
||
if (match[1]) urls.add(match[1].trim());
|
||
}
|
||
|
||
const htmlImageRegex = /<img\b[^>]*\bsrc=["']([^"']+)["'][^>]*>/gi;
|
||
for (const match of normalized.matchAll(htmlImageRegex)) {
|
||
if (match[1]) urls.add(match[1].trim());
|
||
}
|
||
|
||
const dataImageRegex = /data:image\/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=]+/g;
|
||
for (const match of normalized.matchAll(dataImageRegex)) {
|
||
if (match[0]) urls.add(match[0].trim());
|
||
}
|
||
|
||
const rawUrlRegex = /https?:\/\/[^\s"'<>()[\]{}]+/g;
|
||
for (const match of normalized.matchAll(rawUrlRegex)) {
|
||
if (match[0]) urls.add(match[0].replace(/[),.;]+$/g, "").trim());
|
||
}
|
||
|
||
return Array.from(urls).filter(Boolean);
|
||
}
|
||
|
||
function collectImageCandidates(value: any, candidates = new Set<string>()) {
|
||
if (!value) return candidates;
|
||
|
||
if (typeof value === "string") {
|
||
for (const url of extractImageUrlsFromText(value)) {
|
||
candidates.add(url);
|
||
}
|
||
return candidates;
|
||
}
|
||
|
||
if (Array.isArray(value)) {
|
||
for (const item of value) collectImageCandidates(item, candidates);
|
||
return candidates;
|
||
}
|
||
|
||
if (typeof value !== "object") return candidates;
|
||
|
||
const directUrl =
|
||
value.url ||
|
||
value.image_url?.url ||
|
||
value.imageUrl ||
|
||
value.imageURL ||
|
||
value.output_url ||
|
||
value.outputUrl ||
|
||
value.file_url ||
|
||
value.fileUrl;
|
||
if (typeof directUrl === "string") {
|
||
candidates.add(directUrl.trim());
|
||
}
|
||
|
||
const base64Data =
|
||
value.b64_json ||
|
||
value.base64 ||
|
||
value.base64Data ||
|
||
value.image_base64 ||
|
||
value.imageBase64 ||
|
||
value.inline_data?.data ||
|
||
value.inlineData?.data;
|
||
if (typeof base64Data === "string" && base64Data.trim()) {
|
||
const mimeType =
|
||
value.mime_type ||
|
||
value.mimeType ||
|
||
value.inline_data?.mime_type ||
|
||
value.inlineData?.mimeType ||
|
||
"image/png";
|
||
if (base64Data.startsWith("data:image/")) {
|
||
candidates.add(base64Data.trim());
|
||
} else {
|
||
candidates.add(`data:${mimeType};base64,${base64Data.trim()}`);
|
||
}
|
||
}
|
||
|
||
for (const key of [
|
||
"content",
|
||
"message",
|
||
"choices",
|
||
"data",
|
||
"images",
|
||
"results",
|
||
"result",
|
||
"output",
|
||
"outputs",
|
||
"image",
|
||
"image_url",
|
||
]) {
|
||
if (value[key]) collectImageCandidates(value[key], candidates);
|
||
}
|
||
|
||
return candidates;
|
||
}
|
||
|
||
async function requestOpenAIImageGeneration(params: {
|
||
baseURL: string;
|
||
apiKey: string;
|
||
model: string;
|
||
prompt: string;
|
||
size?: string;
|
||
aspectRatio?: string;
|
||
imageSizeLabel?: "1K" | "2K" | "4K";
|
||
imageUrls: string[];
|
||
maskImageUrl?: string;
|
||
quality?: string;
|
||
background?: string;
|
||
outputFormat?: string;
|
||
}) {
|
||
const {
|
||
baseURL,
|
||
apiKey,
|
||
model,
|
||
prompt,
|
||
size,
|
||
aspectRatio,
|
||
imageSizeLabel,
|
||
imageUrls,
|
||
maskImageUrl,
|
||
quality,
|
||
background,
|
||
outputFormat,
|
||
} = params;
|
||
const useBltcyImageCompatParams = shouldUseBltcyImageCompatParams(baseURL);
|
||
const apiModel = resolveProviderApiModelId(baseURL, model);
|
||
const explicitAspectRatio =
|
||
typeof aspectRatio === "string" && aspectRatio.trim() && aspectRatio !== "auto"
|
||
? aspectRatio.trim()
|
||
: undefined;
|
||
const timeoutMs = resolveRequestTimeoutMs(baseURL, imageSizeLabel, model);
|
||
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||
|
||
try {
|
||
if (isAgnesImageModel(baseURL, model)) {
|
||
const endpoint = joinEndpoint(baseURL, "/images/generations");
|
||
const response = await fetch(endpoint, {
|
||
method: "POST",
|
||
headers: {
|
||
Authorization: `Bearer ${apiKey}`,
|
||
"Content-Type": "application/json",
|
||
},
|
||
body: JSON.stringify({
|
||
model: apiModel,
|
||
prompt,
|
||
...(size ? { size } : {}),
|
||
extra_body: {
|
||
...(imageUrls.length > 0 ? { image: imageUrls } : {}),
|
||
response_format: "url",
|
||
},
|
||
}),
|
||
signal: controller.signal,
|
||
});
|
||
const payload = await parseJsonResponse(response);
|
||
if (!response.ok) {
|
||
throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`);
|
||
}
|
||
return payload;
|
||
}
|
||
|
||
if (imageUrls.length > 0) {
|
||
const endpoint = joinEndpoint(baseURL, "/images/edits");
|
||
const formData = new FormData();
|
||
formData.append("model", apiModel);
|
||
formData.append("prompt", prompt);
|
||
if (size) formData.append("size", size);
|
||
if (useBltcyImageCompatParams) {
|
||
if (explicitAspectRatio) formData.append("aspect_ratio", explicitAspectRatio);
|
||
if (imageSizeLabel) formData.append("image_size", imageSizeLabel);
|
||
}
|
||
formData.append("response_format", "url");
|
||
|
||
if (quality) formData.append("quality", quality);
|
||
if (background) formData.append("background", background);
|
||
if (outputFormat) formData.append("output_format", outputFormat);
|
||
|
||
if (imageUrls.length === 1) {
|
||
const image = await loadInputImage(imageUrls[0]);
|
||
formData.append("image", new Blob([image.buffer], { type: image.mimeType }), image.filename);
|
||
} else {
|
||
for (const source of imageUrls) {
|
||
const image = await loadInputImage(source);
|
||
formData.append("image[]", new Blob([image.buffer], { type: image.mimeType }), image.filename);
|
||
}
|
||
}
|
||
if (maskImageUrl) {
|
||
const mask = await loadInputImage(maskImageUrl);
|
||
formData.append("mask", new Blob([mask.buffer], { type: mask.mimeType }), mask.filename);
|
||
}
|
||
|
||
const response = await fetch(endpoint, {
|
||
method: "POST",
|
||
headers: {
|
||
Authorization: `Bearer ${apiKey}`,
|
||
},
|
||
body: formData,
|
||
signal: controller.signal,
|
||
});
|
||
const payload = await parseJsonResponse(response);
|
||
if (!response.ok) {
|
||
throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`);
|
||
}
|
||
return payload;
|
||
}
|
||
|
||
const endpoint = joinEndpoint(baseURL, "/images/generations");
|
||
const response = await fetch(endpoint, {
|
||
method: "POST",
|
||
headers: {
|
||
Authorization: `Bearer ${apiKey}`,
|
||
"Content-Type": "application/json",
|
||
},
|
||
body: JSON.stringify({
|
||
model: apiModel,
|
||
prompt,
|
||
...(size ? { size } : {}),
|
||
...(useBltcyImageCompatParams && explicitAspectRatio
|
||
? { aspect_ratio: explicitAspectRatio }
|
||
: {}),
|
||
...(useBltcyImageCompatParams && imageSizeLabel ? { image_size: imageSizeLabel } : {}),
|
||
response_format: "url",
|
||
...(quality ? { quality } : {}),
|
||
...(background ? { background } : {}),
|
||
...(outputFormat ? { output_format: outputFormat } : {}),
|
||
}),
|
||
signal: controller.signal,
|
||
});
|
||
const payload = await parseJsonResponse(response);
|
||
if (!response.ok) {
|
||
throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`);
|
||
}
|
||
return payload;
|
||
} finally {
|
||
clearTimeout(timeoutId);
|
||
}
|
||
}
|
||
|
||
async function extractUploadedImages(payload: any): Promise<UploadedImage[]> {
|
||
const data = Array.isArray(payload?.data) ? payload.data : [];
|
||
const uploadedImages: UploadedImage[] = [];
|
||
const handledSources = new Set<string>();
|
||
|
||
for (const item of data) {
|
||
if (item?.url && typeof item.url === "string") {
|
||
handledSources.add(item.url);
|
||
uploadedImages.push(await downloadAndUploadImage(item.url));
|
||
continue;
|
||
}
|
||
|
||
if (item?.b64_json && typeof item.b64_json === "string") {
|
||
handledSources.add(item.b64_json);
|
||
uploadedImages.push(await uploadBase64ToS3(item.b64_json, "image/png"));
|
||
}
|
||
}
|
||
|
||
const candidates = Array.from(collectImageCandidates(payload));
|
||
for (const candidate of candidates) {
|
||
if (handledSources.has(candidate)) continue;
|
||
|
||
try {
|
||
if (candidate.startsWith("data:image/")) {
|
||
const match = candidate.match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/);
|
||
if (match) {
|
||
handledSources.add(candidate);
|
||
uploadedImages.push(await uploadBase64ToS3(match[2], match[1] || "image/png"));
|
||
}
|
||
} else if (/^https?:\/\//i.test(candidate)) {
|
||
handledSources.add(candidate);
|
||
uploadedImages.push(await downloadAndUploadImage(candidate));
|
||
}
|
||
} catch (error) {
|
||
console.warn("[OpenAI Image] Failed to ingest image candidate:", {
|
||
candidate,
|
||
error: error instanceof Error ? error.message : String(error),
|
||
});
|
||
}
|
||
}
|
||
|
||
return uploadedImages;
|
||
}
|
||
|
||
function formatMarkdown(images: UploadedImage[], imageSize: string) {
|
||
return images
|
||
.map((image) => ``)
|
||
.join("\n\n");
|
||
}
|
||
|
||
export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) {
|
||
return {
|
||
provider: "openai-image",
|
||
chat: (modelId: string) => ({
|
||
provider: "openai-image",
|
||
doGenerate: async (options: {
|
||
messages: Array<{ role: string; content: any }>;
|
||
imageParams?: OpenAIImageParams;
|
||
overrideModelId?: string;
|
||
}) => {
|
||
const { prompt, imageUrls, imageRefs } = extractPromptAndImages(options.messages);
|
||
const activeModelId = options.overrideModelId || modelId;
|
||
const normalizedAspectRatio = normalizeAspectRatio(options.imageParams?.aspectRatio);
|
||
const promptAspectRatio = inferAspectRatioFromPrompt(prompt);
|
||
const referenceGeometryAspectRatio =
|
||
imageRefs.length > 0
|
||
? inferAspectRatioFromDimensions(imageRefs[0]?.width, imageRefs[0]?.height)
|
||
: null;
|
||
const referenceAspectRatio =
|
||
referenceGeometryAspectRatio ||
|
||
(imageUrls.length > 0
|
||
? await inferAspectRatioFromImageUrl(imageUrls[0] || "")
|
||
: null);
|
||
const shouldPreferReferenceAspectRatio =
|
||
imageUrls.length > 0 && isAgnesImageModel(baseURL, activeModelId);
|
||
const aspectRatio =
|
||
shouldPreferReferenceAspectRatio
|
||
? referenceAspectRatio || promptAspectRatio || normalizedAspectRatio
|
||
: normalizedAspectRatio === "auto"
|
||
? promptAspectRatio || referenceAspectRatio
|
||
: normalizedAspectRatio;
|
||
const requestedImageSize = normalizeImageSizeLabel(options.imageParams?.imageSize);
|
||
const modelImageSize = isComflyBaseURL(baseURL)
|
||
? inferImageSizeFromModelId(activeModelId)
|
||
: undefined;
|
||
const imageSize = modelImageSize || requestedImageSize;
|
||
const shouldOmitAgnesAutoSize =
|
||
isAgnesImageModel(baseURL, activeModelId) &&
|
||
imageUrls.length > 0 &&
|
||
normalizedAspectRatio === "auto" &&
|
||
(!aspectRatio || aspectRatio === "auto") &&
|
||
!options.imageParams?.size;
|
||
let size: string | undefined;
|
||
if (shouldOmitAgnesAutoSize) {
|
||
size = undefined;
|
||
} else if (modelImageSize && aspectRatio) {
|
||
size = resolveProviderImageSize(baseURL, aspectRatio, imageSize);
|
||
} else {
|
||
size =
|
||
options.imageParams?.size ||
|
||
(aspectRatio ? resolveProviderImageSize(baseURL, aspectRatio, imageSize) : undefined);
|
||
}
|
||
const quality = normalizeQuality(options.imageParams?.quality);
|
||
const background = normalizeBackground(options.imageParams?.background);
|
||
const outputFormat = normalizeOutputFormat(options.imageParams?.outputFormat);
|
||
|
||
console.log("[OpenAI Image] Submitting request", {
|
||
model: activeModelId,
|
||
apiModel: resolveProviderApiModelId(baseURL, activeModelId),
|
||
size,
|
||
aspectRatio: aspectRatio || "auto",
|
||
referenceAspectRatio: referenceAspectRatio || undefined,
|
||
referenceGeometryAspectRatio: referenceGeometryAspectRatio || undefined,
|
||
preferReferenceAspectRatio: shouldPreferReferenceAspectRatio,
|
||
promptAspectRatio: promptAspectRatio || undefined,
|
||
omittedSizeForAgnesAuto: shouldOmitAgnesAutoSize || undefined,
|
||
imageSize,
|
||
requestedImageSize,
|
||
modelImageSize,
|
||
resolutionMode: isComflyBaseURL(baseURL)
|
||
? "comfly-gpt-image-constraints"
|
||
: isAgnesBaseURL(baseURL)
|
||
? "agnes-long-edge-constraints"
|
||
: "pixel-area",
|
||
timeoutMs: resolveRequestTimeoutMs(baseURL, imageSize, activeModelId),
|
||
useBltcyImageCompatParams: shouldUseBltcyImageCompatParams(baseURL),
|
||
imageCount: imageUrls.length,
|
||
hasMask: Boolean(options.imageParams?.maskImageUrl),
|
||
baseURL: normalizeBaseURL(baseURL),
|
||
});
|
||
|
||
const payload = await requestOpenAIImageGeneration({
|
||
baseURL,
|
||
apiKey,
|
||
model: activeModelId,
|
||
prompt,
|
||
size,
|
||
aspectRatio: aspectRatio || undefined,
|
||
imageSizeLabel: imageSize,
|
||
imageUrls,
|
||
maskImageUrl: options.imageParams?.maskImageUrl,
|
||
quality,
|
||
background,
|
||
outputFormat,
|
||
});
|
||
|
||
const uploadedImages = await extractUploadedImages(payload);
|
||
if (uploadedImages.length === 0) {
|
||
throw new Error("OpenAI Images 未返回可用图片");
|
||
}
|
||
|
||
const text = formatMarkdown(uploadedImages, imageSize);
|
||
return {
|
||
text,
|
||
finishReason: "stop" as const,
|
||
usage: {
|
||
promptTokens: 0,
|
||
completionTokens: text.length,
|
||
},
|
||
};
|
||
},
|
||
doStream: async function* (options: {
|
||
messages: Array<{ role: string; content: any }>;
|
||
imageParams?: OpenAIImageParams;
|
||
overrideModelId?: string;
|
||
}) {
|
||
const result = await this.doGenerate(options);
|
||
yield {
|
||
type: "text-delta",
|
||
textDelta: result.text,
|
||
};
|
||
},
|
||
}),
|
||
};
|
||
}
|