968 lines
28 KiB
TypeScript
968 lines
28 KiB
TypeScript
import "server-only";
|
||
|
||
import { createHash } from "node:crypto";
|
||
import sharp from "sharp";
|
||
import {
|
||
constrainImageDimensions,
|
||
resolveLongestEdgeDimensions,
|
||
} from "@/lib/ai/image-size-constraints";
|
||
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
|
||
interface FalImageConfig {
|
||
apiKey: string;
|
||
baseURL: string;
|
||
}
|
||
|
||
interface FalImageParams {
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
imageCount?: number;
|
||
outputFormat?: "png" | "jpg" | "jpeg" | "webp";
|
||
safetyTolerance?: string | number;
|
||
seed?: string | number;
|
||
enableWebSearch?: boolean;
|
||
quality?: string;
|
||
maskImageUrl?: string;
|
||
openaiApiKey?: string;
|
||
systemPrompt?: string;
|
||
thinkingLevel?: string;
|
||
syncMode?: boolean;
|
||
}
|
||
|
||
interface UploadedImage {
|
||
thumbnail: string;
|
||
original: string;
|
||
}
|
||
|
||
export interface FalSubmittedImageTask {
|
||
requestId: string;
|
||
endpointId: string;
|
||
modelId: string;
|
||
imageCount: number;
|
||
outputSizeLabel: string;
|
||
prompt: string;
|
||
sourceImageUrls: string[];
|
||
}
|
||
|
||
const DEFAULT_ASPECT_RATIO = "auto";
|
||
const DEFAULT_RESOLUTION = "1K";
|
||
const DEFAULT_OUTPUT_FORMAT = "png";
|
||
const DEFAULT_SAFETY_TOLERANCE = "4";
|
||
const DEFAULT_TIMEOUT_MS = 600000;
|
||
const FAL_GENERIC_MAX_EDGE = 4096;
|
||
const FAL_GENERIC_MAX_PIXELS = 4096 * 4096;
|
||
const GPT_IMAGE_2_MAX_EDGE = 3840;
|
||
const GPT_IMAGE_2_MAX_PIXELS = 8_294_400;
|
||
const FAL_NANO_BANANA_MODEL_IDS = new Set([
|
||
"fal-ai/nano-banana",
|
||
"fal-ai/nano-banana-2",
|
||
"fal-ai/nano-banana-pro",
|
||
]);
|
||
const FAL_GPT_IMAGE_2_EDIT_MODEL_ID = "openai/gpt-image-2/edit";
|
||
const GPT_IMAGE_2_EDIT_IMAGE_SIZE_PRESETS = new Set([
|
||
"auto",
|
||
"square_hd",
|
||
"square",
|
||
"portrait_4_3",
|
||
"portrait_16_9",
|
||
"landscape_4_3",
|
||
"landscape_16_9",
|
||
]);
|
||
const GPT_IMAGE_2_EDIT_QUALITIES = new Set(["low", "medium", "high"]);
|
||
const FAL_ASPECT_RATIO_VALUES = new Set([
|
||
"21:9",
|
||
"16:9",
|
||
"3:2",
|
||
"4:3",
|
||
"5:4",
|
||
"1:1",
|
||
"4:5",
|
||
"3:4",
|
||
"2:3",
|
||
"9:16",
|
||
"4:1",
|
||
"1:4",
|
||
"8:1",
|
||
"1:8",
|
||
]);
|
||
const FAL_IMAGE_SIZE_PRESETS = new Set([
|
||
"square_hd",
|
||
"square",
|
||
"portrait_4_3",
|
||
"portrait_16_9",
|
||
"landscape_4_3",
|
||
"landscape_16_9",
|
||
]);
|
||
const FAL_IMAGE_SIZE_BY_ASPECT_RATIO: Record<string, string> = {
|
||
"1:1": "square_hd",
|
||
"4:3": "landscape_4_3",
|
||
"3:4": "portrait_4_3",
|
||
"16:9": "landscape_16_9",
|
||
"9:16": "portrait_16_9",
|
||
};
|
||
|
||
type FalImageSize = string | { width: number; height: number };
|
||
|
||
const GPT_IMAGE_2_EDIT_SIZE_BY_BUCKET: Record<string, Record<string, FalImageSize>> = {
|
||
"1K": {
|
||
"1:1": "square_hd",
|
||
"4:3": "landscape_4_3",
|
||
"3:4": "portrait_4_3",
|
||
"16:9": "landscape_16_9",
|
||
"9:16": "portrait_16_9",
|
||
},
|
||
"2K": {
|
||
"1:1": { width: 2048, height: 2048 },
|
||
"3:2": { width: 2048, height: 1360 },
|
||
"2:3": { width: 1360, height: 2048 },
|
||
"4:3": { width: 2048, height: 1536 },
|
||
"3:4": { width: 1536, height: 2048 },
|
||
"16:9": { width: 1920, height: 1088 },
|
||
"9:16": { width: 1088, height: 1920 },
|
||
},
|
||
"4K": {
|
||
"1:1": { width: 3072, height: 3072 },
|
||
"3:2": { width: 3072, height: 2048 },
|
||
"2:3": { width: 2048, height: 3072 },
|
||
"4:3": { width: 3072, height: 2304 },
|
||
"3:4": { width: 2304, height: 3072 },
|
||
"16:9": { width: 3840, height: 2160 },
|
||
"9:16": { width: 2160, height: 3840 },
|
||
},
|
||
};
|
||
|
||
function normalizeFalCredentials(apiKey: string) {
|
||
const normalized = apiKey
|
||
.replace(/\u00a0/g, " ")
|
||
.replace(/\u3000/g, " ")
|
||
.replace(/：/g, ":")
|
||
.trim();
|
||
const stripped = normalized.replace(/^bearer\s+/i, "").replace(/^key\s+/i, "");
|
||
const kvMatch = stripped.match(/^(?:fal[_-]?key|api[_-]?key|token)\s*[:：]\s*(.+)$/i);
|
||
if (kvMatch?.[1]) {
|
||
return kvMatch[1].trim();
|
||
}
|
||
return stripped;
|
||
}
|
||
|
||
function normalizeUrlForCompare(url: string) {
|
||
try {
|
||
const parsed = new URL(url);
|
||
parsed.search = "";
|
||
parsed.hash = "";
|
||
return parsed.toString();
|
||
} catch {
|
||
return url.split("?")[0]?.split("#")[0] || url;
|
||
}
|
||
}
|
||
|
||
function filterSourceImages(urls: string[], sourceUrls: string[]) {
|
||
if (sourceUrls.length === 0) return urls;
|
||
const sourceSet = new Set(sourceUrls.map((url) => normalizeUrlForCompare(url)));
|
||
return urls.filter((url) => !sourceSet.has(normalizeUrlForCompare(url)));
|
||
}
|
||
|
||
async function getImageContentHash(url: string): Promise<string | null> {
|
||
try {
|
||
const response = await fetch(url, { signal: AbortSignal.timeout(15_000) });
|
||
if (!response.ok) return null;
|
||
const buffer = Buffer.from(await response.arrayBuffer());
|
||
return createHash("sha1").update(buffer).digest("hex");
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
async function filterSourceImagesByContent(urls: string[], sourceUrls: string[]) {
|
||
if (sourceUrls.length === 0 || urls.length === 0) return urls;
|
||
|
||
const sourceHashes = new Set<string>();
|
||
for (const sourceUrl of sourceUrls) {
|
||
const hash = await getImageContentHash(sourceUrl);
|
||
if (hash) sourceHashes.add(hash);
|
||
}
|
||
if (sourceHashes.size === 0) return urls;
|
||
|
||
const filtered: string[] = [];
|
||
for (const url of urls) {
|
||
const hash = await getImageContentHash(url);
|
||
if (hash && sourceHashes.has(hash)) {
|
||
continue;
|
||
}
|
||
filtered.push(url);
|
||
}
|
||
|
||
return filtered.length > 0 ? filtered : urls;
|
||
}
|
||
|
||
function normalizeAspectRatio(aspectRatio?: string) {
|
||
if (!aspectRatio) return DEFAULT_ASPECT_RATIO;
|
||
const ratio = aspectRatio.trim();
|
||
const normalized = ratio.toLowerCase();
|
||
|
||
if (
|
||
[
|
||
"auto",
|
||
"21:9",
|
||
"16:9",
|
||
"3:2",
|
||
"4:3",
|
||
"5:4",
|
||
"1:1",
|
||
"4:5",
|
||
"3:4",
|
||
"2:3",
|
||
"9:16",
|
||
"9:21",
|
||
"4:1",
|
||
"1:4",
|
||
"8:1",
|
||
"1:8",
|
||
].includes(normalized)
|
||
) {
|
||
return normalized === "auto" ? "auto" : ratio;
|
||
}
|
||
|
||
return DEFAULT_ASPECT_RATIO;
|
||
}
|
||
|
||
function normalizeResolution(imageSize?: string) {
|
||
if (!imageSize) return DEFAULT_RESOLUTION;
|
||
const normalized = imageSize.trim().toUpperCase();
|
||
if (["0.5K", "1K", "2K", "4K"].includes(normalized)) return normalized;
|
||
return DEFAULT_RESOLUTION;
|
||
}
|
||
|
||
function normalizeExplicitAspectRatio(aspectRatio?: string) {
|
||
const normalized = normalizeAspectRatio(aspectRatio);
|
||
if (normalized === "auto") return undefined;
|
||
return FAL_ASPECT_RATIO_VALUES.has(normalized) ? normalized : undefined;
|
||
}
|
||
|
||
function normalizeAnyExplicitAspectRatio(aspectRatio?: string) {
|
||
const normalized = normalizeAspectRatio(aspectRatio);
|
||
return normalized === "auto" ? undefined : normalized;
|
||
}
|
||
|
||
function resolveFalCustomImageSize(aspectRatio?: string, resolution?: string): FalImageSize {
|
||
const bucket = normalizeResolution(resolution);
|
||
const longestEdgeByBucket: Record<string, number> = {
|
||
"0.5K": 768,
|
||
"1K": 1024,
|
||
"2K": 1536,
|
||
"4K": 2048,
|
||
};
|
||
const { width, height } = resolveLongestEdgeDimensions({
|
||
aspectRatio,
|
||
longestEdge: longestEdgeByBucket[bucket] || longestEdgeByBucket["1K"],
|
||
constraints: {
|
||
maxEdge: FAL_GENERIC_MAX_EDGE,
|
||
maxPixels: FAL_GENERIC_MAX_PIXELS,
|
||
minEdge: 256,
|
||
multiple: 16,
|
||
},
|
||
});
|
||
return { width, height };
|
||
}
|
||
|
||
function constrainFalImageSize(size: FalImageSize, constraints = {
|
||
maxEdge: FAL_GENERIC_MAX_EDGE,
|
||
maxPixels: FAL_GENERIC_MAX_PIXELS,
|
||
}) {
|
||
if (typeof size === "string") return size;
|
||
return constrainImageDimensions(size, {
|
||
...constraints,
|
||
minEdge: 256,
|
||
multiple: 16,
|
||
});
|
||
}
|
||
|
||
function normalizeFalImageSize(imageSize?: string, aspectRatio?: string): FalImageSize {
|
||
const rawSize = String(imageSize || "").trim();
|
||
const normalizedSize = rawSize.toLowerCase().replace(/\s+/g, "");
|
||
if (FAL_IMAGE_SIZE_PRESETS.has(normalizedSize)) {
|
||
return normalizedSize;
|
||
}
|
||
|
||
const customSizeMatch = normalizedSize.match(/^(\d{2,5})[x×](\d{2,5})$/i);
|
||
if (customSizeMatch) {
|
||
const width = Number(customSizeMatch[1]);
|
||
const height = Number(customSizeMatch[2]);
|
||
if (
|
||
Number.isFinite(width) &&
|
||
Number.isFinite(height) &&
|
||
width >= 256 &&
|
||
height >= 256 &&
|
||
width <= 4096 &&
|
||
height <= 4096
|
||
) {
|
||
return constrainFalImageSize({ width, height });
|
||
}
|
||
}
|
||
|
||
const normalizedAspectRatio = normalizeAnyExplicitAspectRatio(aspectRatio);
|
||
if (normalizedAspectRatio && FAL_IMAGE_SIZE_BY_ASPECT_RATIO[normalizedAspectRatio]) {
|
||
return FAL_IMAGE_SIZE_BY_ASPECT_RATIO[normalizedAspectRatio];
|
||
}
|
||
|
||
return normalizedAspectRatio
|
||
? resolveFalCustomImageSize(normalizedAspectRatio, rawSize)
|
||
: "square_hd";
|
||
}
|
||
|
||
function normalizeOutputFormat(outputFormat?: string) {
|
||
if (!outputFormat) return DEFAULT_OUTPUT_FORMAT;
|
||
const normalized = outputFormat.trim().toLowerCase();
|
||
if (normalized === "jpg" || normalized === "jpeg") return "jpeg";
|
||
if (normalized === "webp") return "webp";
|
||
return "png";
|
||
}
|
||
|
||
function normalizeSafetyTolerance(value?: string | number) {
|
||
if (value === undefined || value === null) return DEFAULT_SAFETY_TOLERANCE;
|
||
const normalized = String(value).trim();
|
||
if (["1", "2", "3", "4", "5", "6"].includes(normalized)) return normalized;
|
||
return DEFAULT_SAFETY_TOLERANCE;
|
||
}
|
||
|
||
function normalizeSeed(value?: string | number) {
|
||
if (value === undefined || value === null || value === "") return undefined;
|
||
const seed = Number(value);
|
||
if (!Number.isInteger(seed) || seed < 0) return undefined;
|
||
return seed;
|
||
}
|
||
|
||
export function isFalGptImage2EditModelId(modelId: string) {
|
||
return modelId.trim().toLowerCase() === FAL_GPT_IMAGE_2_EDIT_MODEL_ID;
|
||
}
|
||
|
||
function normalizeFalEndpointId(modelId: string, hasInputImages: boolean) {
|
||
const endpoint = modelId.trim();
|
||
const normalized = endpoint.toLowerCase();
|
||
if (!hasInputImages) return endpoint;
|
||
if (normalized.endsWith("/edit")) return endpoint;
|
||
if (FAL_NANO_BANANA_MODEL_IDS.has(normalized)) {
|
||
return `${endpoint}/edit`;
|
||
}
|
||
return endpoint;
|
||
}
|
||
|
||
function isFalNanoBananaModelId(modelId: string) {
|
||
const normalized = modelId.trim().toLowerCase().replace(/\/edit$/, "");
|
||
return FAL_NANO_BANANA_MODEL_IDS.has(normalized);
|
||
}
|
||
|
||
function isFalFluxKontextDevModelId(modelId: string) {
|
||
return modelId.trim().toLowerCase().replace(/\/$/, "") === "fal-ai/flux-kontext/dev";
|
||
}
|
||
|
||
function isFalKontextModelId(modelId: string) {
|
||
return modelId.trim().toLowerCase().includes("kontext");
|
||
}
|
||
|
||
function shouldUseFalImageSize(modelId: string, hasInputImages: boolean) {
|
||
if (hasInputImages) return false;
|
||
const normalized = modelId.trim().toLowerCase();
|
||
if (isFalNanoBananaModelId(normalized) || isFalKontextModelId(normalized)) return false;
|
||
return (
|
||
normalized.startsWith("fal-ai/flux/") ||
|
||
normalized.startsWith("fal-ai/flux-lora") ||
|
||
normalized.includes("/flux/dev") ||
|
||
normalized.includes("/flux/schnell")
|
||
);
|
||
}
|
||
|
||
function normalizeGptImage2EditQuality(value?: string) {
|
||
if (!value) return "high";
|
||
const normalized = value.trim().toLowerCase();
|
||
return GPT_IMAGE_2_EDIT_QUALITIES.has(normalized) ? normalized : "high";
|
||
}
|
||
|
||
function normalizeGptImage2EditImageSize(
|
||
imageSize?: string,
|
||
aspectRatio?: string
|
||
): FalImageSize {
|
||
if (!imageSize) return "auto";
|
||
|
||
const rawSize = imageSize.trim();
|
||
const normalizedSize = rawSize.toLowerCase().replace(/\s+/g, "");
|
||
if (GPT_IMAGE_2_EDIT_IMAGE_SIZE_PRESETS.has(normalizedSize)) {
|
||
return normalizedSize;
|
||
}
|
||
|
||
const customSizeMatch = normalizedSize.match(/^(\d{2,5})[x×](\d{2,5})$/i);
|
||
if (customSizeMatch) {
|
||
const width = Number(customSizeMatch[1]);
|
||
const height = Number(customSizeMatch[2]);
|
||
if (
|
||
Number.isFinite(width) &&
|
||
Number.isFinite(height) &&
|
||
width >= 256 &&
|
||
height >= 256 &&
|
||
width <= 4096 &&
|
||
height <= 4096 &&
|
||
width % 16 === 0 &&
|
||
height % 16 === 0
|
||
) {
|
||
return constrainFalImageSize(
|
||
{ width, height },
|
||
{ maxEdge: GPT_IMAGE_2_MAX_EDGE, maxPixels: GPT_IMAGE_2_MAX_PIXELS }
|
||
);
|
||
}
|
||
}
|
||
|
||
const sizeBucket = rawSize.toUpperCase();
|
||
const ratio = normalizeAspectRatio(aspectRatio).toLowerCase();
|
||
const resolvedSize =
|
||
GPT_IMAGE_2_EDIT_SIZE_BY_BUCKET[sizeBucket]?.[ratio] ||
|
||
(normalizeAnyExplicitAspectRatio(aspectRatio)
|
||
? resolveFalCustomImageSize(aspectRatio, sizeBucket)
|
||
: "auto");
|
||
return constrainFalImageSize(resolvedSize, {
|
||
maxEdge: GPT_IMAGE_2_MAX_EDGE,
|
||
maxPixels: GPT_IMAGE_2_MAX_PIXELS,
|
||
});
|
||
}
|
||
|
||
function getFalImageSizeLabel(size: FalImageSize) {
|
||
return typeof size === "string" ? size : `${size.width}x${size.height}`;
|
||
}
|
||
|
||
function normalizeOptionalHttpUrl(value?: string) {
|
||
const normalized = String(value || "").trim();
|
||
return /^https?:\/\//i.test(normalized) ? normalized : "";
|
||
}
|
||
|
||
function detectImageExtension(imageUrl: string, contentType?: string | null) {
|
||
const type = (contentType || "").toLowerCase();
|
||
if (type.includes("jpeg") || type.includes("jpg")) return "jpg";
|
||
if (type.includes("webp")) return "webp";
|
||
if (type.includes("png")) return "png";
|
||
|
||
const cleanUrl = imageUrl.split("?")[0]?.toLowerCase() || "";
|
||
if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "jpg";
|
||
if (cleanUrl.endsWith(".webp")) return "webp";
|
||
if (cleanUrl.endsWith(".png")) return "png";
|
||
|
||
return "png";
|
||
}
|
||
|
||
async function createThumbnail(buffer: Buffer): Promise<Buffer> {
|
||
try {
|
||
return await sharp(buffer)
|
||
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
|
||
.jpeg({ quality: 80 })
|
||
.toBuffer();
|
||
} catch (error) {
|
||
console.error("[Fal Image] Failed to create thumbnail:", error);
|
||
return buffer;
|
||
}
|
||
}
|
||
|
||
function generateFilename(prefix: string, extension: string) {
|
||
const ts = Date.now();
|
||
const rand = Math.random().toString(36).slice(2, 8);
|
||
return `${prefix}-${ts}-${rand}.${extension}`;
|
||
}
|
||
|
||
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
||
const deferred = await createDeferredUploadedImage(imageUrl, {
|
||
providerType: "fal-image",
|
||
});
|
||
if (deferred) return deferred;
|
||
|
||
const response = await fetch(imageUrl);
|
||
if (!response.ok) {
|
||
throw new Error(`Failed to download image: ${response.status}`);
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
const buffer = Buffer.from(arrayBuffer);
|
||
const thumbnailBuffer = await createThumbnail(buffer);
|
||
|
||
const originalExt = detectImageExtension(imageUrl, response.headers.get("content-type"));
|
||
const originalFilename = generateFilename("fal-image-orig", originalExt);
|
||
const originalArrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer);
|
||
|
||
const thumbFilename = generateFilename("fal-image-thumb", "jpg");
|
||
const thumbArrayBuffer = thumbnailBuffer.buffer.slice(
|
||
thumbnailBuffer.byteOffset,
|
||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
|
||
) as ArrayBuffer;
|
||
const thumbUploaded = await uploadFile(thumbFilename, thumbArrayBuffer);
|
||
|
||
return {
|
||
thumbnail: thumbUploaded.url,
|
||
original: originalUploaded.url,
|
||
};
|
||
}
|
||
|
||
function extractPromptAndImages(messages: Array<{ role: string; content: any }>) {
|
||
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
|
||
if (!lastUserMessage) {
|
||
throw new Error("No user message found");
|
||
}
|
||
|
||
let prompt = "";
|
||
const imageUrls: string[] = [];
|
||
|
||
const pushImageUrl = (url?: string) => {
|
||
if (!url) return;
|
||
const normalized = String(url).trim();
|
||
|
||
if (normalized.startsWith("data:")) {
|
||
throw new Error("FAL 不支持 Base64 图片传输，请先上传图片后再编辑");
|
||
}
|
||
|
||
if (!normalized.startsWith("http")) return;
|
||
if (!imageUrls.includes(normalized)) {
|
||
imageUrls.push(normalized);
|
||
}
|
||
};
|
||
|
||
if (typeof lastUserMessage.content === "string") {
|
||
prompt = lastUserMessage.content.trim();
|
||
} else if (Array.isArray(lastUserMessage.content)) {
|
||
for (const part of lastUserMessage.content) {
|
||
if (part?.type === "text" && typeof part.text === "string") {
|
||
prompt = `${prompt} ${part.text}`.trim();
|
||
} else if (part?.type === "image_url" && part.image_url?.url) {
|
||
pushImageUrl(part.image_url.url);
|
||
} else if (part?.type === "file") {
|
||
pushImageUrl(part.url || part.data);
|
||
} else if (part?.type === "image") {
|
||
pushImageUrl(part.image || part.url);
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!prompt) {
|
||
throw new Error("No prompt found in user message");
|
||
}
|
||
|
||
return { prompt, imageUrls };
|
||
}
|
||
|
||
function collectImageUrls(payload: any): string[] {
|
||
const urls = new Set<string>();
|
||
|
||
const isLikelyImageUrl = (value: string) =>
|
||
/^https?:\/\//i.test(value) &&
|
||
(/\.(png|jpg|jpeg|webp|gif)(\?|$)/i.test(value) ||
|
||
value.includes("image") ||
|
||
value.includes("fal.media"));
|
||
|
||
const walk = (value: any, parentKey = "") => {
|
||
if (typeof value === "string") {
|
||
if (isLikelyImageUrl(value)) {
|
||
urls.add(value);
|
||
}
|
||
return;
|
||
}
|
||
|
||
if (Array.isArray(value)) {
|
||
for (const item of value) {
|
||
walk(item, parentKey);
|
||
}
|
||
return;
|
||
}
|
||
|
||
if (value && typeof value === "object") {
|
||
for (const [key, nestedValue] of Object.entries(value)) {
|
||
if (key === "url" && typeof nestedValue === "string" && /^https?:\/\//i.test(nestedValue)) {
|
||
if (/image|img|file|output|result|artifact/i.test(parentKey) || isLikelyImageUrl(nestedValue)) {
|
||
urls.add(nestedValue);
|
||
}
|
||
}
|
||
walk(nestedValue, key);
|
||
}
|
||
}
|
||
};
|
||
|
||
walk(payload);
|
||
return Array.from(urls);
|
||
}
|
||
|
||
function formatMarkdown(images: UploadedImage[], resolution: string) {
|
||
return images
|
||
.map((image) => ``)
|
||
.join("\n\n");
|
||
}
|
||
|
||
function summarizeFalError(error: any) {
|
||
const status = error?.status || error?.statusCode || error?.response?.status;
|
||
const body =
|
||
error?.body ||
|
||
error?.responseBody ||
|
||
error?.data ||
|
||
error?.response?.data ||
|
||
error?.cause?.data;
|
||
const message =
|
||
error?.message ||
|
||
(typeof body === "string" ? body : "") ||
|
||
"FAL request failed";
|
||
return {
|
||
name: error?.name,
|
||
status,
|
||
message,
|
||
body,
|
||
};
|
||
}
|
||
|
||
function stringifyFalErrorDetail(value: unknown) {
|
||
if (!value) return "";
|
||
if (typeof value === "string") return value;
|
||
try {
|
||
return JSON.stringify(value);
|
||
} catch {
|
||
return String(value);
|
||
}
|
||
}
|
||
|
||
function createDetailedFalError(error: unknown) {
|
||
const summary = summarizeFalError(error);
|
||
const detail = stringifyFalErrorDetail(summary.body);
|
||
const message = [
|
||
"FAL request failed",
|
||
summary.status ? `status=${summary.status}` : "",
|
||
summary.message ? `message=${summary.message}` : "",
|
||
detail ? `body=${detail}` : "",
|
||
]
|
||
.filter(Boolean)
|
||
.join("; ")
|
||
.slice(0, 1800);
|
||
const detailedError = new Error(message);
|
||
detailedError.name = "FalImageProviderError";
|
||
(detailedError as Error & { cause?: unknown }).cause = error;
|
||
return detailedError;
|
||
}
|
||
|
||
function createFalClientFromApiKey(apiKey: string) {
|
||
return import("@fal-ai/client").then(({ createFalClient }) =>
|
||
createFalClient({
|
||
credentials: normalizeFalCredentials(apiKey),
|
||
}),
|
||
);
|
||
}
|
||
|
||
function buildFalRequestPayload(params: {
|
||
modelId: string;
|
||
overrideModelId?: string;
|
||
messages: Array<{ role: string; content: any }>;
|
||
imageParams?: FalImageParams;
|
||
}) {
|
||
const { prompt, imageUrls } = extractPromptAndImages(params.messages);
|
||
const aspectRatio = normalizeAspectRatio(params.imageParams?.aspectRatio);
|
||
const resolution = normalizeResolution(params.imageParams?.imageSize);
|
||
const outputFormat = normalizeOutputFormat(params.imageParams?.outputFormat);
|
||
const safetyTolerance = normalizeSafetyTolerance(params.imageParams?.safetyTolerance);
|
||
const seed = normalizeSeed(params.imageParams?.seed);
|
||
const imageCount = Math.max(1, Math.min(6, Number(params.imageParams?.imageCount || 1)));
|
||
const systemPrompt = String(params.imageParams?.systemPrompt || "").trim();
|
||
const thinkingLevel = String(params.imageParams?.thinkingLevel || "").trim().toLowerCase();
|
||
const activeModelId = params.overrideModelId || params.modelId;
|
||
const isGptImage2Edit = isFalGptImage2EditModelId(activeModelId);
|
||
const falEndpointId = isGptImage2Edit
|
||
? activeModelId
|
||
: normalizeFalEndpointId(activeModelId, imageUrls.length > 0);
|
||
|
||
if (!isGptImage2Edit && falEndpointId.toLowerCase().endsWith("/edit") && imageUrls.length === 0) {
|
||
throw new Error("FAL 图片编辑模型需要至少一张可访问的参考图片 URL");
|
||
}
|
||
|
||
const gptImage2EditSize = normalizeGptImage2EditImageSize(
|
||
params.imageParams?.imageSize,
|
||
params.imageParams?.aspectRatio,
|
||
);
|
||
const outputSizeLabel = isGptImage2Edit ? getFalImageSizeLabel(gptImage2EditSize) : resolution;
|
||
|
||
let input: Record<string, any>;
|
||
if (isGptImage2Edit) {
|
||
if (imageUrls.length === 0) {
|
||
throw new Error("FAL GPT Image 2 Edit 需要至少一张可访问的参考图片 URL");
|
||
}
|
||
|
||
input = {
|
||
prompt,
|
||
image_urls: imageUrls,
|
||
image_size: gptImage2EditSize,
|
||
quality: normalizeGptImage2EditQuality(params.imageParams?.quality),
|
||
num_images: imageCount,
|
||
output_format: outputFormat,
|
||
};
|
||
|
||
const maskImageUrl = normalizeOptionalHttpUrl(params.imageParams?.maskImageUrl);
|
||
if (maskImageUrl) {
|
||
input.mask_url = maskImageUrl;
|
||
}
|
||
|
||
if (seed !== undefined) {
|
||
input.seed = seed;
|
||
}
|
||
|
||
const openaiApiKey = String(params.imageParams?.openaiApiKey || "").trim();
|
||
if (openaiApiKey) {
|
||
input.openai_api_key = openaiApiKey;
|
||
}
|
||
} else {
|
||
const explicitAspectRatio = normalizeExplicitAspectRatio(aspectRatio);
|
||
input = {
|
||
prompt,
|
||
num_images: imageCount,
|
||
output_format: outputFormat,
|
||
safety_tolerance: safetyTolerance,
|
||
};
|
||
|
||
if (isFalNanoBananaModelId(falEndpointId)) {
|
||
input.aspect_ratio = FAL_ASPECT_RATIO_VALUES.has(aspectRatio)
|
||
? aspectRatio
|
||
: DEFAULT_ASPECT_RATIO;
|
||
input.resolution = resolution;
|
||
input.limit_generations = true;
|
||
} else if (isFalFluxKontextDevModelId(falEndpointId) && imageUrls.length > 0) {
|
||
input.image_url = imageUrls[0];
|
||
input.enable_safety_checker = true;
|
||
input.resolution_mode = explicitAspectRatio || "match_input";
|
||
} else if (isFalKontextModelId(falEndpointId)) {
|
||
if (explicitAspectRatio) {
|
||
input.aspect_ratio = explicitAspectRatio;
|
||
}
|
||
if (imageUrls.length > 0) {
|
||
input.image_url = imageUrls[0];
|
||
if (imageUrls.length > 1 || falEndpointId.toLowerCase().includes("/multi")) {
|
||
input.image_urls = imageUrls;
|
||
}
|
||
}
|
||
} else if (shouldUseFalImageSize(falEndpointId, imageUrls.length > 0)) {
|
||
input.image_size = normalizeFalImageSize(params.imageParams?.imageSize, aspectRatio);
|
||
input.enable_safety_checker = true;
|
||
} else if (explicitAspectRatio) {
|
||
input.aspect_ratio = explicitAspectRatio;
|
||
}
|
||
|
||
if (seed !== undefined) {
|
||
input.seed = seed;
|
||
}
|
||
|
||
if (imageUrls.length > 0 && !input.image_url && !input.image_urls) {
|
||
input.image_urls = imageUrls;
|
||
}
|
||
|
||
if (typeof params.imageParams?.enableWebSearch === "boolean") {
|
||
input.enable_web_search = params.imageParams.enableWebSearch;
|
||
}
|
||
|
||
if (typeof params.imageParams?.syncMode === "boolean") {
|
||
input.sync_mode = params.imageParams.syncMode;
|
||
}
|
||
}
|
||
|
||
if (isFalNanoBananaModelId(falEndpointId)) {
|
||
if (systemPrompt) {
|
||
input.system_prompt = systemPrompt;
|
||
}
|
||
if (thinkingLevel === "minimal" || thinkingLevel === "high") {
|
||
input.thinking_level = thinkingLevel;
|
||
}
|
||
}
|
||
|
||
return {
|
||
prompt,
|
||
imageUrls,
|
||
aspectRatio,
|
||
resolution,
|
||
outputFormat,
|
||
safetyTolerance,
|
||
seed,
|
||
imageCount,
|
||
activeModelId,
|
||
isGptImage2Edit,
|
||
falEndpointId,
|
||
gptImage2EditSize,
|
||
outputSizeLabel,
|
||
input,
|
||
};
|
||
}
|
||
|
||
async function normalizeFalResultToMarkdown(params: {
|
||
payload: any;
|
||
sourceImageUrls: string[];
|
||
imageCount: number;
|
||
outputSizeLabel: string;
|
||
}) {
|
||
let outputUrls = filterSourceImages(collectImageUrls(params.payload), params.sourceImageUrls);
|
||
outputUrls = await filterSourceImagesByContent(outputUrls, params.sourceImageUrls);
|
||
|
||
if (outputUrls.length === 0 && params.payload?.data) {
|
||
outputUrls = filterSourceImages(collectImageUrls(params.payload.data), params.sourceImageUrls);
|
||
outputUrls = await filterSourceImagesByContent(outputUrls, params.sourceImageUrls);
|
||
}
|
||
|
||
if (outputUrls.length === 0) {
|
||
throw new Error("FAL 未返回可用图片");
|
||
}
|
||
|
||
const uploadedImages: UploadedImage[] = [];
|
||
for (const outputUrl of outputUrls.slice(0, params.imageCount)) {
|
||
uploadedImages.push(await downloadAndUploadImage(outputUrl));
|
||
}
|
||
|
||
if (uploadedImages.length === 0) {
|
||
throw new Error("FAL 未返回可用图片");
|
||
}
|
||
|
||
return formatMarkdown(uploadedImages, params.outputSizeLabel);
|
||
}
|
||
|
||
export async function submitFalImageTask(params: {
|
||
apiKey: string;
|
||
modelId: string;
|
||
messages: Array<{ role: string; content: any }>;
|
||
imageParams?: FalImageParams;
|
||
overrideModelId?: string;
|
||
}): Promise<FalSubmittedImageTask> {
|
||
const request = buildFalRequestPayload(params);
|
||
const fal = await createFalClientFromApiKey(params.apiKey);
|
||
|
||
console.log("[Fal Image] Queue submit", {
|
||
model: request.activeModelId,
|
||
endpoint: request.falEndpointId,
|
||
imageCount: request.imageCount,
|
||
hasInputImages: request.imageUrls.length > 0,
|
||
resolution: request.outputSizeLabel,
|
||
inputKeys: Object.keys(request.input),
|
||
});
|
||
|
||
const enqueueResult = await fal.queue.submit(request.falEndpointId, {
|
||
input: request.input,
|
||
});
|
||
|
||
return {
|
||
requestId: String(enqueueResult.request_id),
|
||
endpointId: request.falEndpointId,
|
||
modelId: request.activeModelId,
|
||
imageCount: request.imageCount,
|
||
outputSizeLabel: request.outputSizeLabel,
|
||
prompt: request.prompt,
|
||
sourceImageUrls: request.imageUrls,
|
||
};
|
||
}
|
||
|
||
export async function getFalImageTaskStatus(params: {
|
||
apiKey: string;
|
||
endpointId: string;
|
||
requestId: string;
|
||
}) {
|
||
const fal = await createFalClientFromApiKey(params.apiKey);
|
||
return fal.queue.status(params.endpointId, {
|
||
requestId: params.requestId,
|
||
logs: true,
|
||
});
|
||
}
|
||
|
||
export async function resolveFalImageTaskToMarkdown(params: {
|
||
apiKey: string;
|
||
endpointId: string;
|
||
requestId: string;
|
||
imageCount: number;
|
||
outputSizeLabel: string;
|
||
sourceImageUrls: string[];
|
||
}) {
|
||
const fal = await createFalClientFromApiKey(params.apiKey);
|
||
const result = await fal.queue.result<any>(params.endpointId, {
|
||
requestId: params.requestId,
|
||
});
|
||
return normalizeFalResultToMarkdown({
|
||
payload: result,
|
||
sourceImageUrls: params.sourceImageUrls,
|
||
imageCount: params.imageCount,
|
||
outputSizeLabel: params.outputSizeLabel,
|
||
});
|
||
}
|
||
|
||
export function createFalImage({ apiKey, baseURL }: FalImageConfig) {
|
||
return {
|
||
provider: "fal-image",
|
||
chat: (modelId: string) => ({
|
||
provider: "fal-image",
|
||
doGenerate: async (options: {
|
||
messages: Array<{ role: string; content: any }>;
|
||
imageParams?: FalImageParams;
|
||
overrideModelId?: string;
|
||
}) => {
|
||
const request = buildFalRequestPayload({
|
||
modelId,
|
||
overrideModelId: options.overrideModelId,
|
||
messages: options.messages,
|
||
imageParams: options.imageParams,
|
||
});
|
||
|
||
console.log("[Fal Image] Submitting request", {
|
||
model: request.activeModelId,
|
||
endpoint: request.falEndpointId,
|
||
baseURL,
|
||
imageCount: request.imageCount,
|
||
aspectRatio: request.aspectRatio,
|
||
resolution: request.outputSizeLabel,
|
||
hasInputImages: request.imageUrls.length > 0,
|
||
inputKeys: Object.keys(request.input),
|
||
});
|
||
|
||
const fal = await createFalClientFromApiKey(apiKey);
|
||
|
||
let result: any;
|
||
try {
|
||
result = await fal.subscribe(request.falEndpointId, {
|
||
input: request.input,
|
||
logs: true,
|
||
timeout: DEFAULT_TIMEOUT_MS,
|
||
onQueueUpdate: (update: any) => {
|
||
if (update?.status === "IN_PROGRESS" && Array.isArray(update.logs)) {
|
||
for (const log of update.logs) {
|
||
if (log?.message) {
|
||
console.log(`[Fal Image][${request.falEndpointId}]`, log.message);
|
||
}
|
||
}
|
||
}
|
||
},
|
||
});
|
||
} catch (error) {
|
||
console.error("[Fal Image] Request failed:", summarizeFalError(error));
|
||
throw createDetailedFalError(error);
|
||
}
|
||
|
||
const text = await normalizeFalResultToMarkdown({
|
||
payload: result,
|
||
sourceImageUrls: request.imageUrls,
|
||
imageCount: request.imageCount,
|
||
outputSizeLabel: request.outputSizeLabel,
|
||
});
|
||
return {
|
||
text,
|
||
finishReason: "stop" as const,
|
||
usage: {
|
||
promptTokens: 0,
|
||
completionTokens: text.length,
|
||
},
|
||
};
|
||
},
|
||
doStream: async function* (options: {
|
||
messages: Array<{ role: string; content: any }>;
|
||
imageParams?: FalImageParams;
|
||
overrideModelId?: string;
|
||
}) {
|
||
const result = await this.doGenerate(options);
|
||
yield {
|
||
type: "text-delta",
|
||
textDelta: result.text,
|
||
};
|
||
},
|
||
}),
|
||
};
|
||
}
|