diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx index 828d0194b..3da45bb46 100644 --- a/app/(admin)/settings/admin-settings-client.tsx +++ b/app/(admin)/settings/admin-settings-client.tsx @@ -121,7 +121,7 @@ type ToolConfig = { type AiProvider = { id: string; name: string; - type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "zhipu"; + type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "zhipu"; baseUrl: string; apiKey: string; isEnabled: boolean; @@ -319,6 +319,7 @@ const PROVIDER_TYPES = [ { value: "alibaba", label: "阿里云通义千问" }, { value: "alibaba-image", label: "阿里生图模型" }, // 🆕 千问图片生成/编辑 { value: "ruxa-image", label: "RUXA 生图模型" }, + { value: "fal-image", label: "FAL 生图模型" }, { value: "zhipu", label: "智普官方 4.7" }, { value: "openai", label: "OpenAI 兼容" }, { value: "custom", label: "自定义 OpenAI 兼容" }, @@ -865,7 +866,7 @@ export function AdminSettingsClient() { const [providerForm, setProviderForm] = useState({ id: "", name: "", - type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "zhipu", + type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "zhipu", baseUrl: "", apiKey: "", isEnabled: true, @@ -2756,6 +2757,7 @@ export function AdminSettingsClient() { "alibaba": "bg-blue-500", "alibaba-image": "bg-amber-500", "ruxa-image": "bg-emerald-500", + "fal-image": "bg-sky-500", "openai": "bg-green-500", "zhipu": "bg-lime-500", "qianwen-video": "bg-cyan-500", @@ -5087,6 +5089,8 @@ export function AdminSettingsClient() { baseUrl: !prev.baseUrl && value === "ruxa-image" ? "https://api.ruxa.ai" + : !prev.baseUrl && value === "fal-image" + ? "https://fal.run" : !prev.baseUrl && value === "zhipu" ? "https://open.bigmodel.cn/api/paas/v4" : prev.baseUrl, @@ -5109,6 +5113,8 @@ export function AdminSettingsClient() { placeholder={ providerForm.type === "ruxa-image" ? "https://api.ruxa.ai" + : providerForm.type === "fal-image" + ? "https://fal.run" : providerForm.type === "zhipu" ? "https://open.bigmodel.cn/api/paas/v4" : "https://api.openai.com/v1" @@ -6407,7 +6413,7 @@ export function AdminSettingsClient() { const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId); if (!provider) return false; // 支持阿里生图、Gemini、既梦等图片模型 - const imageProviderTypes = ["alibaba-image", "ruxa-image", "gemini-image", "gemini", "volcano-vision"]; + const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "gemini-image", "gemini", "volcano-vision"]; return imageProviderTypes.includes(provider.type); }) .map((m: AiModel) => { @@ -6422,7 +6428,7 @@ export function AdminSettingsClient() { {/* 如果没有找到生图模型,显示提示 */} {(modelsData?.models || []).filter((m: AiModel) => { const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId); - const imageProviderTypes = ["alibaba-image", "ruxa-image", "gemini-image", "gemini", "volcano-vision"]; + const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "gemini-image", "gemini", "volcano-vision"]; return provider && imageProviderTypes.includes(provider.type); }).length === 0 && ( diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 19bcd1d49..7c5fc0afb 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -380,7 +380,7 @@ function buildAutoModeFallbackMessage(params: { * 因为这些 provider 内部会自行处理图片上传到 S3 */ async function convertMessagesWithImages(messages: ChatMessage[], providerType: string) { - if (providerType === "ruxa-image") { + if (providerType === "ruxa-image" || providerType === "fal-image") { const convertedMessages = []; for (const message of messages) { @@ -408,7 +408,7 @@ async function convertMessagesWithImages(messages: ChatMessage[], providerType: if (!candidateUrl || typeof candidateUrl !== "string") continue; if (candidateUrl.startsWith("data:")) { - throw new Error("RUXA 不支持 Base64 图片传输,请先上传图片后再编辑"); + throw new Error("当前模型不支持 Base64 图片传输,请先上传图片后再编辑"); } if (!candidateUrl.startsWith("http")) continue; @@ -541,6 +541,8 @@ function parseImageAltMetadata(altText: string, thumbnailUrl: string): { originalUrl = thumbnailUrl.replace("alibaba-image-thumb-", "alibaba-image-orig-"); } else if (thumbnailUrl.includes("ruxa-image-thumb-")) { originalUrl = thumbnailUrl.replace("ruxa-image-thumb-", "ruxa-image-orig-"); + } else if (thumbnailUrl.includes("fal-image-thumb-")) { + originalUrl = thumbnailUrl.replace("fal-image-thumb-", "fal-image-orig-"); } } @@ -980,6 +982,7 @@ export async function POST(request: Request) { "huanke-gemini", "alibaba-image", "ruxa-image", + "fal-image", "volcano-vision", ]); const isFixedImageModel = @@ -1646,6 +1649,7 @@ export async function POST(request: Request) { (languageModel as any).provider === "volcano-vision" || (languageModel as any).provider === "alibaba-image" || (languageModel as any).provider === "ruxa-image" || + (languageModel as any).provider === "fal-image" || (languageModel as any).provider === "huanke-gemini"; const isUuidModelId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( @@ -2077,7 +2081,7 @@ export async function POST(request: Request) { // ========== Gemini 处理结束 ========== // ========== 阿里云千问图片生成独立处理 ========== - const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image"].includes((languageModel as any).provider); + const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image", "fal-image"].includes((languageModel as any).provider); if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using image generation provider:", (languageModel as any).provider); console.log("[Chat API] Image params:", effectiveImageParams); @@ -2162,9 +2166,13 @@ export async function POST(request: Request) { messagesForAlibaba = injectHistoryImagesToLastUserMessage(messagesForAlibaba, smartAutoHistoryImages); const tryGenerateImage = async (modelIdToUse: string, isRetry: boolean) => { - // 转换消息(阿里生图支持 URL,会自动上传 base64) + // 转换消息(图片编辑模型统一使用 URL 输入) const providerForConvert = - (languageModel as any).provider === "ruxa-image" ? "ruxa-image" : "alibaba-image"; + (languageModel as any).provider === "ruxa-image" + ? "ruxa-image" + : (languageModel as any).provider === "fal-image" + ? "fal-image" + : "alibaba-image"; const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert); // 调用 doGenerate 获取图片,传递模型显示名称 @@ -2226,6 +2234,7 @@ export async function POST(request: Request) { "huanke-gemini": 2, "alibaba-image": 3, "ruxa-image": 4, + "fal-image": 5, }; const crossProviderCandidates = allModels .filter((m) => { @@ -2258,6 +2267,8 @@ export async function POST(request: Request) { const convertProvider = candidate.provider.type === "ruxa-image" ? "ruxa-image" + : candidate.provider.type === "fal-image" + ? "fal-image" : candidate.provider.type === "gemini" ? "gemini" : candidate.provider.type === "huanke-gemini" @@ -2573,7 +2584,8 @@ export async function POST(request: Request) { "huanke-gemini": 1, "alibaba-image": 2, "ruxa-image": 3, - "volcano-vision": 4, + "fal-image": 4, + "volcano-vision": 5, }; const crossProviderCandidates = allModels .filter((m) => { @@ -2606,6 +2618,8 @@ export async function POST(request: Request) { const convertProvider = candidate.provider.type === "ruxa-image" ? "ruxa-image" + : candidate.provider.type === "fal-image" + ? "fal-image" : candidate.provider.type === "gemini" ? "gemini" : candidate.provider.type === "huanke-gemini" diff --git a/app/api/admin/ai-providers/route.ts b/app/api/admin/ai-providers/route.ts index 929077590..fa49fa337 100644 --- a/app/api/admin/ai-providers/route.ts +++ b/app/api/admin/ai-providers/route.ts @@ -21,6 +21,7 @@ const providerTypes = [ "volcano-video", "alibaba-image", "ruxa-image", + "fal-image", "zhipu", ] as const; type ProviderType = (typeof providerTypes)[number]; diff --git a/app/api/canvas/generate/image/route.ts b/app/api/canvas/generate/image/route.ts index a4582752e..ca8e3cc4e 100644 --- a/app/api/canvas/generate/image/route.ts +++ b/app/api/canvas/generate/image/route.ts @@ -4,6 +4,7 @@ import { getEnabledAiModelsWithProvider, getAiProviderById } from "@/lib/db/quer import { createGeminiImage } from "@/lib/ai/providers/gemini-image"; import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image"; import { createRuxaImage } from "@/lib/ai/providers/ruxa-image"; +import { createFalImage } from "@/lib/ai/providers/fal-image"; export const maxDuration = 120; @@ -157,6 +158,18 @@ export async function POST(request: Request) { imageParams, }); imageMarkdown = result.text || ""; + } else if (provider.type === "fal-image") { + const falProvider = createFalImage({ + apiKey: provider.apiKey, + baseURL: provider.baseUrl, + }); + + const model = falProvider.chat(selectedModel.model.modelId); + const result = await model.doGenerate({ + messages, + imageParams, + }); + imageMarkdown = result.text || ""; } else { return NextResponse.json( { success: false, error: "不支持的图片生成模型类型" }, diff --git a/app/api/scene-fusion/route.ts b/app/api/scene-fusion/route.ts index ed9a8bea2..a08074e85 100644 --- a/app/api/scene-fusion/route.ts +++ b/app/api/scene-fusion/route.ts @@ -17,6 +17,7 @@ import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image"; import { createGeminiImage } from "@/lib/ai/providers/gemini-image"; import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision"; import { createRuxaImage } from "@/lib/ai/providers/ruxa-image"; +import { createFalImage } from "@/lib/ai/providers/fal-image"; // 请求体 Schema const sceneFusionRequestSchema = z.object({ @@ -368,6 +369,9 @@ async function processSceneWithProvider( case "ruxa-image": return processWithRuxaImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize); + case "fal-image": + return processWithFalImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize); + case "gemini": return processWithGemini(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize); @@ -502,6 +506,46 @@ async function processWithRuxaImage( return { thumbnail, original, prompt }; } +async function processWithFalImage( + productImageUrl: string, + sceneImageUrl: string, + prompt: string, + provider: { baseUrl: string; apiKey: string }, + modelId: string, + aspectRatio?: string, + imageSize?: string +): Promise<{ thumbnail: string; original: string; prompt: string }> { + console.log(`[SceneFusion][FAL] Processing with model: ${modelId}`); + + const falProvider = createFalImage({ + baseURL: provider.baseUrl, + apiKey: provider.apiKey, + }); + + const messages = [ + { + role: "user", + content: [ + { type: "image_url", image_url: { url: productImageUrl } }, + { type: "image_url", image_url: { url: sceneImageUrl } }, + { type: "text", text: prompt }, + ], + }, + ]; + + const result = await falProvider.chat(modelId).doGenerate({ + messages, + imageParams: { + aspectRatio: aspectRatio || "auto", + imageSize: imageSize || "2K", + imageCount: 1, + }, + }); + + const { thumbnail, original } = parseImageFromMarkdown(result.text); + return { thumbnail, original, prompt }; +} + /** * 使用既梦/火山模型处理 */ @@ -683,8 +727,8 @@ export async function POST(request: Request) { // 如果没有配置或配置的模型不可用,使用默认的阿里生图模型 if (!selectedModel) { - // 按优先级查找:alibaba-image > ruxa-image > gemini > volcano-vision - const imageProviderTypes = ["alibaba-image", "ruxa-image", "gemini", "volcano-vision"]; + // 按优先级查找:alibaba-image > ruxa-image > fal-image > gemini > volcano-vision + const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "gemini", "volcano-vision"]; for (const providerType of imageProviderTypes) { selectedModel = allModels.find( (m) => m.provider.type === providerType && m.model.isEnabled diff --git a/components/chat.tsx b/components/chat.tsx index 998b1b9fd..7a676848a 100644 --- a/components/chat.tsx +++ b/components/chat.tsx @@ -654,7 +654,11 @@ export function Chat({ else if (model?.providerType === "volcano-vision") modelType = "volcano-vision"; else if (model?.providerType === "volcano-video") modelType = "volcano-video"; else if (model?.providerType === "qianwen-video") modelType = "qianwen-video"; - else if (model?.providerType === "alibaba-image" || model?.providerType === "ruxa-image") modelType = "alibaba-image"; + else if ( + model?.providerType === "alibaba-image" || + model?.providerType === "ruxa-image" || + model?.providerType === "fal-image" + ) modelType = "alibaba-image"; console.log(`[Chat] 🚀 记录生成任务: modelType=${modelType}, modelId=${currentModelId}`); startGeneration(currentModelId, modelType); diff --git a/components/image-generation-loading.tsx b/components/image-generation-loading.tsx index 4c68896b9..67c959f61 100644 --- a/components/image-generation-loading.tsx +++ b/components/image-generation-loading.tsx @@ -112,6 +112,8 @@ export const ImageGenerationLoading = memo(({ return "orange"; case "ruxa-image": return "green"; + case "fal-image": + return "blue"; case "volcano-vision": return "blue"; default: @@ -127,6 +129,7 @@ export const ImageGenerationLoading = memo(({ // 获取显示名称 const displayName = modelName || (providerType === "alibaba-image" ? "千问图片编辑" : providerType === "ruxa-image" ? "RUXA 图片编辑" : + providerType === "fal-image" ? "FAL 图片编辑" : providerType === "volcano-vision" ? "即梦4.5" : "AI"); const phaseTitle = diff --git a/components/image-preview.tsx b/components/image-preview.tsx index 577b53f64..a851edd80 100644 --- a/components/image-preview.tsx +++ b/components/image-preview.tsx @@ -141,6 +141,9 @@ export function ImagePreview({ } else if (src.includes("openai-thumb-")) { originalUrl = src.replace("openai-thumb-", "openai-orig-"); console.log("[ImagePreview] Inferred original URL from openai-thumb filename"); + } else if (src.includes("fal-image-thumb-")) { + originalUrl = src.replace("fal-image-thumb-", "fal-image-orig-"); + console.log("[ImagePreview] Inferred original URL from fal-image-thumb filename"); } else if (src.includes("/thumb/") || src.includes("-thumb")) { // 尝试从S3 URL中推断原图URL(常见格式:xxx-thumb.jpg -> xxx.jpg 或 xxx/thumb/xxx -> xxx/xxx) originalUrl = src.replace(/-thumb\./g, ".").replace(/\/thumb\//g, "/"); diff --git a/components/messages.tsx b/components/messages.tsx index 5d3589455..5b1375973 100644 --- a/components/messages.tsx +++ b/components/messages.tsx @@ -117,7 +117,11 @@ function PureMessages({ // 🆕 判断是否是阿里生图模型(或者智能自动模式默认使用阿里生图占位图) const isAlibabaImageModel = modelsData?.models?.some( - (m: DynamicModel) => m.id === selectedModelId && (m.providerType === "alibaba-image" || m.providerType === "ruxa-image") + (m: DynamicModel) => + m.id === selectedModelId && + (m.providerType === "alibaba-image" || + m.providerType === "ruxa-image" || + m.providerType === "fal-image") ) ?? false; // 🆕 获取当前模型的显示名称 @@ -639,6 +643,8 @@ function PureMessages({ providerType={ currentModelProviderType === "ruxa-image" ? "ruxa-image" + : currentModelProviderType === "fal-image" + ? "fal-image" : "alibaba-image" } modelName={currentModelDisplayName} diff --git a/lib/ai/dynamic-provider.ts b/lib/ai/dynamic-provider.ts index ff5faa554..e7ef5a5ef 100644 --- a/lib/ai/dynamic-provider.ts +++ b/lib/ai/dynamic-provider.ts @@ -19,6 +19,7 @@ import { createQianwenVideo } from "./providers/qianwen-video"; import { createVolcanoVision } from "./providers/volcano-vision"; import { createAlibabaImage } from "./providers/alibaba-image"; import { createRuxaImage } from "./providers/ruxa-image"; +import { createFalImage } from "./providers/fal-image"; // 缓存已创建的 provider 实例 const providerCache = new Map>(); @@ -154,6 +155,16 @@ function getOrCreateProvider(provider: Pick = { // 图片编辑:优先 alibaba-image(灵魂4.5,支持提示词扩展) image_edit: { - providerTypes: ["alibaba-image", "ruxa-image", "gemini", "huanke-gemini"], + providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "gemini", "huanke-gemini"], requireVision: true, requireImageGen: true, description: "图片编辑", @@ -45,7 +45,7 @@ const INTENT_MODEL_MAP: Record< // 图片生成:优先灵魂4.5 image_gen: { - providerTypes: ["alibaba-image", "ruxa-image", "gemini", "huanke-gemini", "volcano-vision"], + providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "gemini", "huanke-gemini", "volcano-vision"], requireImageGen: true, description: "图片生成", }, @@ -73,25 +73,25 @@ const SUB_INTENT_OVERRIDES: Record< > = { // 抠图:alibaba-image 效果最好 remove_background: { - providerTypes: ["alibaba-image", "ruxa-image", "gemini"], + providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "gemini"], reason: "抠图首选灵魂4.5", }, // 换背景:alibaba-image 支持场景生成 replace_background: { - providerTypes: ["alibaba-image", "ruxa-image", "gemini"], + providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "gemini"], reason: "换背景首选灵魂4.5", }, // 图片增强 enhance: { - providerTypes: ["alibaba-image", "ruxa-image", "gemini", "volcano-vision"], + providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "gemini", "volcano-vision"], reason: "增强首选灵魂4.5", }, // 风格迁移:gemini 可能更有创意 style_transfer: { - providerTypes: ["gemini", "alibaba-image", "ruxa-image"], + providerTypes: ["gemini", "alibaba-image", "ruxa-image", "fal-image"], reason: "风格迁移优选Gemini", }, }; diff --git a/lib/ai/providers/fal-image.ts b/lib/ai/providers/fal-image.ts new file mode 100644 index 000000000..9a113bd36 --- /dev/null +++ b/lib/ai/providers/fal-image.ts @@ -0,0 +1,360 @@ +import "server-only"; + +import sharp from "sharp"; +import { uploadFile } from "@/lib/storage/service"; + +interface FalImageConfig { + apiKey: string; + baseURL: string; +} + +interface FalImageParams { + aspectRatio?: string; + imageSize?: string; + imageCount?: number; + outputFormat?: "png" | "jpg" | "jpeg" | "webp"; + safetyTolerance?: string | number; + enableWebSearch?: boolean; +} + +interface UploadedImage { + thumbnail: string; + original: string; +} + +const DEFAULT_ASPECT_RATIO = "auto"; +const DEFAULT_RESOLUTION = "1K"; +const DEFAULT_OUTPUT_FORMAT = "png"; +const DEFAULT_SAFETY_TOLERANCE = "4"; +const DEFAULT_TIMEOUT_MS = 180000; + +function normalizeFalCredentials(apiKey: string) { + return apiKey.trim().replace(/^bearer\s+/i, "").replace(/^key\s+/i, ""); +} + +function normalizeUrlForCompare(url: string) { + try { + const parsed = new URL(url); + parsed.search = ""; + parsed.hash = ""; + return parsed.toString(); + } catch { + return url.split("?")[0]?.split("#")[0] || url; + } +} + +function filterSourceImages(urls: string[], sourceUrls: string[]) { + if (sourceUrls.length === 0) return urls; + const sourceSet = new Set(sourceUrls.map((url) => normalizeUrlForCompare(url))); + return urls.filter((url) => !sourceSet.has(normalizeUrlForCompare(url))); +} + +function normalizeAspectRatio(aspectRatio?: string) { + if (!aspectRatio) return DEFAULT_ASPECT_RATIO; + const ratio = aspectRatio.trim(); + const normalized = ratio.toLowerCase(); + + if ( + [ + "auto", + "21:9", + "16:9", + "3:2", + "4:3", + "5:4", + "1:1", + "4:5", + "3:4", + "2:3", + "9:16", + ].includes(normalized) + ) { + return normalized === "auto" ? "auto" : ratio; + } + + return DEFAULT_ASPECT_RATIO; +} + +function normalizeResolution(imageSize?: string) { + if (!imageSize) return DEFAULT_RESOLUTION; + const normalized = imageSize.trim().toUpperCase(); + if (["1K", "2K", "4K"].includes(normalized)) return normalized; + return DEFAULT_RESOLUTION; +} + +function normalizeOutputFormat(outputFormat?: string) { + if (!outputFormat) return DEFAULT_OUTPUT_FORMAT; + const normalized = outputFormat.trim().toLowerCase(); + if (normalized === "jpg" || normalized === "jpeg") return "jpeg"; + if (normalized === "webp") return "webp"; + return "png"; +} + +function normalizeSafetyTolerance(value?: string | number) { + if (value === undefined || value === null) return DEFAULT_SAFETY_TOLERANCE; + const normalized = String(value).trim(); + if (["1", "2", "3", "4", "5", "6"].includes(normalized)) return normalized; + return DEFAULT_SAFETY_TOLERANCE; +} + +function detectImageExtension(imageUrl: string, contentType?: string | null) { + const type = (contentType || "").toLowerCase(); + if (type.includes("jpeg") || type.includes("jpg")) return "jpg"; + if (type.includes("webp")) return "webp"; + if (type.includes("png")) return "png"; + + const cleanUrl = imageUrl.split("?")[0]?.toLowerCase() || ""; + if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "jpg"; + if (cleanUrl.endsWith(".webp")) return "webp"; + if (cleanUrl.endsWith(".png")) return "png"; + + return "png"; +} + +async function createThumbnail(buffer: Buffer): Promise { + try { + return await sharp(buffer) + .resize(400, 400, { fit: "inside", withoutEnlargement: true }) + .jpeg({ quality: 80 }) + .toBuffer(); + } catch (error) { + console.error("[Fal Image] Failed to create thumbnail:", error); + return buffer; + } +} + +function generateFilename(prefix: string, extension: string) { + const ts = Date.now(); + const rand = Math.random().toString(36).slice(2, 8); + return `${prefix}-${ts}-${rand}.${extension}`; +} + +async function downloadAndUploadImage(imageUrl: string): Promise { + const response = await fetch(imageUrl); + if (!response.ok) { + throw new Error(`Failed to download image: ${response.status}`); + } + + const arrayBuffer = await response.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + const thumbnailBuffer = await createThumbnail(buffer); + + const originalExt = detectImageExtension(imageUrl, response.headers.get("content-type")); + const originalFilename = generateFilename("fal-image-orig", originalExt); + const originalArrayBuffer = buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) as ArrayBuffer; + const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer); + + const thumbFilename = generateFilename("fal-image-thumb", "jpg"); + const thumbArrayBuffer = thumbnailBuffer.buffer.slice( + thumbnailBuffer.byteOffset, + thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength + ) as ArrayBuffer; + const thumbUploaded = await uploadFile(thumbFilename, thumbArrayBuffer); + + return { + thumbnail: thumbUploaded.url, + original: originalUploaded.url, + }; +} + +function extractPromptAndImages(messages: Array<{ role: string; content: any }>) { + const lastUserMessage = messages.filter((message) => message.role === "user").pop(); + if (!lastUserMessage) { + throw new Error("No user message found"); + } + + let prompt = ""; + const imageUrls: string[] = []; + + const pushImageUrl = (url?: string) => { + if (!url) return; + const normalized = String(url).trim(); + + if (normalized.startsWith("data:")) { + throw new Error("FAL 不支持 Base64 图片传输,请先上传图片后再编辑"); + } + + if (!normalized.startsWith("http")) return; + if (!imageUrls.includes(normalized)) { + imageUrls.push(normalized); + } + }; + + if (typeof lastUserMessage.content === "string") { + prompt = lastUserMessage.content.trim(); + } else if (Array.isArray(lastUserMessage.content)) { + for (const part of lastUserMessage.content) { + if (part?.type === "text" && typeof part.text === "string") { + prompt = `${prompt} ${part.text}`.trim(); + } else if (part?.type === "image_url" && part.image_url?.url) { + pushImageUrl(part.image_url.url); + } else if (part?.type === "file") { + pushImageUrl(part.url || part.data); + } else if (part?.type === "image") { + pushImageUrl(part.image || part.url); + } + } + } + + if (!prompt) { + throw new Error("No prompt found in user message"); + } + + return { prompt, imageUrls }; +} + +function collectImageUrls(payload: any): string[] { + const urls = new Set(); + + const isLikelyImageUrl = (value: string) => + /^https?:\/\//i.test(value) && + (/\.(png|jpg|jpeg|webp|gif)(\?|$)/i.test(value) || + value.includes("image") || + value.includes("fal.media")); + + const walk = (value: any, parentKey = "") => { + if (typeof value === "string") { + if (isLikelyImageUrl(value)) { + urls.add(value); + } + return; + } + + if (Array.isArray(value)) { + for (const item of value) { + walk(item, parentKey); + } + return; + } + + if (value && typeof value === "object") { + for (const [key, nestedValue] of Object.entries(value)) { + if (key === "url" && typeof nestedValue === "string" && /^https?:\/\//i.test(nestedValue)) { + if (/image|img|file|output|result|artifact/i.test(parentKey) || isLikelyImageUrl(nestedValue)) { + urls.add(nestedValue); + } + } + walk(nestedValue, key); + } + } + }; + + walk(payload); + return Array.from(urls); +} + +function formatMarkdown(images: UploadedImage[], resolution: string) { + return images + .map((image) => `![thumb|||${image.original}|||${resolution}](${image.thumbnail})`) + .join("\n\n"); +} + +export function createFalImage({ apiKey, baseURL }: FalImageConfig) { + return { + provider: "fal-image", + chat: (modelId: string) => ({ + provider: "fal-image", + doGenerate: async (options: { + messages: Array<{ role: string; content: any }>; + imageParams?: FalImageParams; + overrideModelId?: string; + }) => { + const { prompt, imageUrls } = extractPromptAndImages(options.messages); + const aspectRatio = normalizeAspectRatio(options.imageParams?.aspectRatio); + const resolution = normalizeResolution(options.imageParams?.imageSize); + const outputFormat = normalizeOutputFormat(options.imageParams?.outputFormat); + const safetyTolerance = normalizeSafetyTolerance(options.imageParams?.safetyTolerance); + const imageCount = Math.max(1, Math.min(6, Number(options.imageParams?.imageCount || 1))); + const activeModelId = options.overrideModelId || modelId; + + console.log("[Fal Image] Submitting request", { + model: activeModelId, + baseURL, + imageCount, + aspectRatio, + resolution, + }); + + const fal = await import("@fal-ai/serverless-client"); + fal.config({ + credentials: normalizeFalCredentials(apiKey), + }); + + const input: Record = { + prompt, + num_images: imageCount, + aspect_ratio: aspectRatio, + resolution, + output_format: outputFormat, + safety_tolerance: safetyTolerance, + }; + + if (imageUrls.length > 0) { + input.image_urls = imageUrls; + } + + if (typeof options.imageParams?.enableWebSearch === "boolean") { + input.enable_web_search = options.imageParams.enableWebSearch; + } + + const result: any = await fal.subscribe(activeModelId, { + input, + logs: true, + timeout: DEFAULT_TIMEOUT_MS, + onQueueUpdate: (update: any) => { + if (update?.status === "IN_PROGRESS" && Array.isArray(update.logs)) { + for (const log of update.logs) { + if (log?.message) { + console.log(`[Fal Image][${activeModelId}]`, log.message); + } + } + } + }, + }); + + let outputUrls = filterSourceImages(collectImageUrls(result), imageUrls); + + if (outputUrls.length === 0 && result?.data) { + outputUrls = filterSourceImages(collectImageUrls(result.data), imageUrls); + } + + if (outputUrls.length === 0) { + throw new Error("FAL 未返回可用图片"); + } + + const uploadedImages: UploadedImage[] = []; + for (const outputUrl of outputUrls.slice(0, imageCount)) { + uploadedImages.push(await downloadAndUploadImage(outputUrl)); + } + + if (uploadedImages.length === 0) { + throw new Error("FAL 未返回可用图片"); + } + + const text = formatMarkdown(uploadedImages, resolution); + return { + text, + finishReason: "stop" as const, + usage: { + promptTokens: 0, + completionTokens: text.length, + }, + }; + }, + doStream: async function* (options: { + messages: Array<{ role: string; content: any }>; + imageParams?: FalImageParams; + overrideModelId?: string; + }) { + const result = await this.doGenerate(options); + yield { + type: "text-delta", + textDelta: result.text, + }; + }, + }), + }; +} diff --git a/lib/db/migrate.ts b/lib/db/migrate.ts index 5144e0092..76f0282a5 100644 --- a/lib/db/migrate.ts +++ b/lib/db/migrate.ts @@ -67,7 +67,7 @@ const runMigrate = async () => { ADD CONSTRAINT "AiProvider_type_check" CHECK (type IN ( 'alibaba', 'openai', 'custom', 'local', 'gemini', - 'huanke-gemini', 'qianwen-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'ruxa-image', 'zhipu' + 'huanke-gemini', 'qianwen-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'ruxa-image', 'fal-image', 'zhipu' )); `); const end = Date.now(); diff --git a/lib/db/queries.ts b/lib/db/queries.ts index 03f802714..75bbe9da7 100644 --- a/lib/db/queries.ts +++ b/lib/db/queries.ts @@ -1088,7 +1088,7 @@ export async function createAiProvider({ updatedBy, }: { name: string; - type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "zhipu"; + type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "zhipu"; baseUrl: string; apiKey: string; isEnabled?: boolean; @@ -1126,7 +1126,7 @@ export async function updateAiProvider({ }: { id: string; name?: string; - type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "zhipu"; + type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "zhipu"; baseUrl?: string; apiKey?: string; isEnabled?: boolean; diff --git a/lib/db/schema.ts b/lib/db/schema.ts index 5d7389988..c72687b7d 100644 --- a/lib/db/schema.ts +++ b/lib/db/schema.ts @@ -258,7 +258,7 @@ export type ToolConfig = InferSelectModel; export const aiProvider = pgTable("AiProvider", { id: uuid("id").primaryKey().notNull().defaultRandom(), name: varchar("name", { length: 128 }).notNull(), // 提供商名称,如 "阿里云通义千问" - type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "zhipu"] }) + type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "zhipu"] }) .notNull() .default("custom"), // 提供商类型 baseUrl: text("baseUrl").notNull(), // API 基础 URL diff --git a/lib/image-url.ts b/lib/image-url.ts index 9f7f4a8cb..c18e208a9 100644 --- a/lib/image-url.ts +++ b/lib/image-url.ts @@ -15,7 +15,8 @@ export function isLikelyThumbnailUrl(url: string): boolean { lowerUrl.includes("openai-thumb-") || lowerUrl.includes("volcano-thumb-") || lowerUrl.includes("alibaba-image-thumb-") || - lowerUrl.includes("ruxa-image-thumb-") + lowerUrl.includes("ruxa-image-thumb-") || + lowerUrl.includes("fal-image-thumb-") ) { return true; } @@ -41,6 +42,9 @@ export function inferOriginalImageUrl(url: string): string { if (url.includes("ruxa-image-thumb-")) { return url.replace("ruxa-image-thumb-", "ruxa-image-orig-"); } + if (url.includes("fal-image-thumb-")) { + return url.replace("fal-image-thumb-", "fal-image-orig-"); + } if (url.includes("/thumb/")) { return url.replace("/thumb/", "/");