From d104efd7e9b15e8ea008882f0f5be3262e96187f Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 7 Mar 2026 10:51:56 +0800 Subject: [PATCH] feat: add openai compatible image provider --- .../settings/admin-settings-client.tsx | 11 +- app/(chat)/api/chat/route.ts | 74 ++- app/api/admin/ai-providers/route.ts | 4 + app/api/scene-fusion/route.ts | 45 ++ components/chat.tsx | 5 +- components/image-generation-loading.tsx | 1 + components/image-params-bar.tsx | 2 +- components/messages.tsx | 7 +- components/multimodal-input.tsx | 3 +- lib/ai/dynamic-provider.ts | 17 +- lib/ai/model-router.ts | 12 +- lib/ai/provider-defaults.ts | 1 + lib/ai/providers/openai-image.ts | 538 ++++++++++++++++++ lib/canvas/generate-image.ts | 14 +- lib/db/migrate.ts | 3 +- lib/db/queries.ts | 4 +- lib/db/schema.ts | 2 +- 17 files changed, 690 insertions(+), 53 deletions(-) create mode 100644 lib/ai/providers/openai-image.ts diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx index dde871a47..dde186d26 100644 --- a/app/(admin)/settings/admin-settings-client.tsx +++ b/app/(admin)/settings/admin-settings-client.tsx @@ -147,7 +147,7 @@ type ToolConfig = { type AiProvider = { id: string; name: string; - type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal"; + type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal"; baseUrl: string; apiKey: string; isEnabled: boolean; @@ -437,6 +437,7 @@ const PROVIDER_TYPES = [ { value: "alibaba-image", label: "阿里生图模型" }, // 🆕 千问图片生成/编辑 { value: "ruxa-image", label: "RUXA生图" }, { value: "fal-image", label: "FAL 生图模型" }, + { value: "openai-image", label: "OpenAI 图片兼容" }, { value: "tripo-3d", label: "Tripo3D 模型生成" }, { value: "zhipu", label: "智普官方 4.7" }, { value: "volcano-multimodal", label: "火山引擎多模态" }, @@ -459,6 +460,7 @@ const PROVIDER_DEFAULT_BASE_URL: Record = { "https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation", "ruxa-image": "https://api.ruxa.ai", "fal-image": "https://fal.run", + "openai-image": "https://api.openai.com/v1", "fal-video": "https://fal.run", "tripo-3d": "https://api.tripo3d.ai/v2/openapi", zhipu: "https://open.bigmodel.cn/api/paas/v4", @@ -1325,7 +1327,7 @@ export function AdminSettingsClient() { const [providerForm, setProviderForm] = useState({ id: "", name: "", - type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal", + type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal", baseUrl: "", apiKey: "", isEnabled: true, @@ -3694,6 +3696,7 @@ export function AdminSettingsClient() { "alibaba-image": "bg-amber-500", "ruxa-image": "bg-emerald-500", "fal-image": "bg-sky-500", + "openai-image": "bg-green-600", "tripo-3d": "bg-teal-500", "openai": "bg-green-500", "zhipu": "bg-lime-500", @@ -8725,7 +8728,7 @@ export function AdminSettingsClient() { const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId); if (!provider) return false; // 支持阿里生图、Gemini、既梦等图片模型 - const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "gemini-image", "gemini", "volcano-vision"]; + const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini-image", "gemini", "volcano-vision"]; return imageProviderTypes.includes(provider.type); }) .map((m: AiModel) => { @@ -8740,7 +8743,7 @@ export function AdminSettingsClient() { {/* 如果没有找到生图模型，显示提示 */} {(modelsData?.models || []).filter((m: AiModel) => { const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId); - const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "gemini-image", "gemini", "volcano-vision"]; + const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini-image", "gemini", "volcano-vision"]; return provider && imageProviderTypes.includes(provider.type); }).length === 0 && ( diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index bbc299f89..6b9c673e8 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -597,6 +597,7 @@ const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([ "alibaba-image", "ruxa-image", "fal-image", + "openai-image", "volcano-vision", "gemini", "huanke-gemini", @@ -1112,7 +1113,7 @@ function ensureImageAnalysisSummarySection(message: string, summary: string): st * 因为这些 provider 内部会自行处理图片上传到 S3 */ async function convertMessagesWithImages(messages: ChatMessage[], providerType: string) { - if (providerType === "ruxa-image" || providerType === "fal-image") { + if (providerType === "ruxa-image" || providerType === "fal-image" || providerType === "openai-image") { const convertedMessages = []; for (const message of messages) { @@ -2115,6 +2116,7 @@ export async function POST(request: Request) { "alibaba-image", "ruxa-image", "fal-image", + "openai-image", "volcano-vision", ]); const isFixedVisualModel = @@ -3231,6 +3233,7 @@ export async function POST(request: Request) { (languageModel as any).provider === "alibaba-image" || (languageModel as any).provider === "ruxa-image" || (languageModel as any).provider === "fal-image" || + (languageModel as any).provider === "openai-image" || (languageModel as any).provider === "huanke-gemini"; const isUuidModelId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( @@ -3718,7 +3721,7 @@ export async function POST(request: Request) { // ========== Gemini 处理结束 ========== // ========== 阿里云千问图片生成独立处理 ========== - const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image", "fal-image"].includes((languageModel as any).provider); + const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image", "fal-image", "openai-image"].includes((languageModel as any).provider); if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using image generation provider:", (languageModel as any).provider); console.log("[Chat API] Image params:", effectiveImageParams); @@ -3808,6 +3811,8 @@ export async function POST(request: Request) { ? "ruxa-image" : (languageModel as any).provider === "fal-image" ? "fal-image" + : (languageModel as any).provider === "openai-image" + ? "openai-image" : "alibaba-image"; const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert); @@ -3867,8 +3872,9 @@ export async function POST(request: Request) { "alibaba-image": 1, "ruxa-image": 2, "fal-image": 3, - gemini: 4, - "huanke-gemini": 5, + "openai-image": 4, + gemini: 5, + "huanke-gemini": 6, }; const crossProviderCandidates = getImageCrossProviderCandidates(allModels, { currentModelId: finalModelId, @@ -3893,13 +3899,15 @@ export async function POST(request: Request) { ? "ruxa-image" : candidate.provider.type === "fal-image" ? "fal-image" - : candidate.provider.type === "gemini" - ? "gemini" - : candidate.provider.type === "huanke-gemini" - ? "huanke-gemini" - : candidate.provider.type === "volcano-vision" - ? "volcano" - : "alibaba-image"; + : candidate.provider.type === "openai-image" + ? "openai-image" + : candidate.provider.type === "gemini" + ? "gemini" + : candidate.provider.type === "huanke-gemini" + ? "huanke-gemini" + : candidate.provider.type === "volcano-vision" + ? "volcano" + : "alibaba-image"; const candidateMessages = await convertMessagesWithImages( messagesForAlibaba, @@ -4216,9 +4224,10 @@ export async function POST(request: Request) { "alibaba-image": 0, "ruxa-image": 1, "fal-image": 2, - gemini: 3, - "huanke-gemini": 4, - "volcano-vision": 5, + "openai-image": 3, + gemini: 4, + "huanke-gemini": 5, + "volcano-vision": 6, }; const crossProviderCandidates = getImageCrossProviderCandidates(allModels, { currentModelId: finalModelId, @@ -4243,13 +4252,15 @@ export async function POST(request: Request) { ? "ruxa-image" : candidate.provider.type === "fal-image" ? "fal-image" - : candidate.provider.type === "gemini" - ? "gemini" - : candidate.provider.type === "huanke-gemini" - ? "huanke-gemini" - : candidate.provider.type === "volcano-vision" - ? "volcano" - : "alibaba-image"; + : candidate.provider.type === "openai-image" + ? "openai-image" + : candidate.provider.type === "gemini" + ? "gemini" + : candidate.provider.type === "huanke-gemini" + ? "huanke-gemini" + : candidate.provider.type === "volcano-vision" + ? "volcano" + : "alibaba-image"; const candidateMessages = await convertMessagesWithImages( messagesForVolcano, @@ -4844,8 +4855,9 @@ export async function POST(request: Request) { "alibaba-image": 1, "ruxa-image": 2, "fal-image": 3, - gemini: 4, - "huanke-gemini": 5, + "openai-image": 4, + gemini: 5, + "huanke-gemini": 6, }; const crossProviderCandidates = getImageCrossProviderCandidates(allModels, { currentModelId: finalModelId, @@ -4875,13 +4887,15 @@ export async function POST(request: Request) { ? "ruxa-image" : candidate.provider.type === "fal-image" ? "fal-image" - : candidate.provider.type === "gemini" - ? "gemini" - : candidate.provider.type === "huanke-gemini" - ? "huanke-gemini" - : candidate.provider.type === "volcano-vision" - ? "volcano" - : "alibaba-image"; + : candidate.provider.type === "openai-image" + ? "openai-image" + : candidate.provider.type === "gemini" + ? "gemini" + : candidate.provider.type === "huanke-gemini" + ? "huanke-gemini" + : candidate.provider.type === "volcano-vision" + ? "volcano" + : "alibaba-image"; const candidateMessages = await convertMessagesWithImages( messagesForCrossProvider, diff --git a/app/api/admin/ai-providers/route.ts b/app/api/admin/ai-providers/route.ts index ac9ddca71..eb578e475 100644 --- a/app/api/admin/ai-providers/route.ts +++ b/app/api/admin/ai-providers/route.ts @@ -24,6 +24,7 @@ const providerTypes = [ "alibaba-image", "ruxa-image", "fal-image", + "openai-image", "fal-video", "zhipu", "tripo-3d", @@ -54,6 +55,7 @@ const VOLCANO_MULTIMODAL_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3"; const DOUBAO_VIDEO_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3"; const RUXA_VIDEO_BASE_URL = "https://api.ruxa.ai"; const RUXA_IMAGE_BASE_URL = "https://api.ruxa.ai"; +const OPENAI_IMAGE_BASE_URL = "https://api.openai.com/v1"; const TRIPO_3D_BASE_URL = "https://api.tripo3d.ai/v2/openapi"; function normalizeProviderTypeForStorage( @@ -165,6 +167,8 @@ export async function POST(request: Request) { ? data.baseUrl || RUXA_VIDEO_BASE_URL : data.type === "ruxa-image" ? data.baseUrl || RUXA_IMAGE_BASE_URL + : data.type === "openai-image" + ? data.baseUrl || OPENAI_IMAGE_BASE_URL : data.type === "tripo-3d" ? data.baseUrl || TRIPO_3D_BASE_URL : data.baseUrl, diff --git a/app/api/scene-fusion/route.ts b/app/api/scene-fusion/route.ts index 01e0c1d08..0c6d4f232 100644 --- a/app/api/scene-fusion/route.ts +++ b/app/api/scene-fusion/route.ts @@ -23,6 +23,7 @@ import { createGeminiImage } from "@/lib/ai/providers/gemini-image"; import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision"; import { createRuxaImage } from "@/lib/ai/providers/ruxa-image"; import { createFalImage } from "@/lib/ai/providers/fal-image"; +import { createOpenAIImage } from "@/lib/ai/providers/openai-image"; import { callQwen3Vl } from "@/lib/ai/qwen3-vl"; // 请求体 Schema @@ -98,6 +99,7 @@ function canProcessSceneFusionProviderType(providerType: string): boolean { case "alibaba-image": case "ruxa-image": case "fal-image": + case "openai-image": case "gemini": case "volcano-vision": return true; @@ -1236,6 +1238,9 @@ async function processSceneWithProvider( case "fal-image": return processWithFalImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize); + case "openai-image": + return processWithOpenAIImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize); + case "gemini": case "gemini-image": return processWithGemini(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize); @@ -1411,6 +1416,46 @@ async function processWithFalImage( return { thumbnail, original, prompt }; } +async function processWithOpenAIImage( + productImageUrl: string, + sceneImageUrl: string, + prompt: string, + provider: { baseUrl: string; apiKey: string }, + modelId: string, + aspectRatio?: string, + imageSize?: string +): Promise<{ thumbnail: string; original: string; prompt: string }> { + console.log(`[SceneFusion][OpenAI Image] Processing with model: ${modelId}`); + + const openAIImageProvider = createOpenAIImage({ + baseURL: provider.baseUrl, + apiKey: provider.apiKey, + }); + + const messages = [ + { + role: "user", + content: [ + { type: "image_url", image_url: { url: productImageUrl } }, + { type: "image_url", image_url: { url: sceneImageUrl } }, + { type: "text", text: prompt }, + ], + }, + ]; + + const result = await openAIImageProvider.chat(modelId).doGenerate({ + messages, + imageParams: { + aspectRatio: aspectRatio || "auto", + imageSize: imageSize || "2K", + imageCount: 1, + }, + }); + + const { thumbnail, original } = parseImageFromMarkdown(result.text); + return { thumbnail, original, prompt }; +} + /** * 使用既梦/火山模型处理 */ diff --git a/components/chat.tsx b/components/chat.tsx index ece54612d..e4b4084d5 100644 --- a/components/chat.tsx +++ b/components/chat.tsx @@ -49,7 +49,7 @@ function normalizeImageParamsForProvider( const normalized = { ...params }; const currentCount = normalized.imageCount || 1; - if (providerType === "ruxa-image") { + if (providerType === "ruxa-image" || providerType === "openai-image") { normalized.imageCount = 1; return normalized; } @@ -1052,7 +1052,8 @@ export function Chat({ else if ( model?.providerType === "alibaba-image" || model?.providerType === "ruxa-image" || - model?.providerType === "fal-image" + model?.providerType === "fal-image" || + model?.providerType === "openai-image" ) modelType = "alibaba-image"; // 智能自动模式下，如果当前轮输入包含图片，默认按图片生成链路跟踪 diff --git a/components/image-generation-loading.tsx b/components/image-generation-loading.tsx index f96d1ec59..7de89e293 100644 --- a/components/image-generation-loading.tsx +++ b/components/image-generation-loading.tsx @@ -20,6 +20,7 @@ const PROVIDER_MODEL_LABEL: Record = { "alibaba-image": "千问图片", "ruxa-image": "RUXA 图片", "fal-image": "FAL 图片", + "openai-image": "OpenAI 图片", "volcano-vision": "即梦 4.5", gemini: "Gemini 图片", }; diff --git a/components/image-params-bar.tsx b/components/image-params-bar.tsx index edee16c6f..e5851c2e4 100644 --- a/components/image-params-bar.tsx +++ b/components/image-params-bar.tsx @@ -87,7 +87,7 @@ export function ImageParamsBar({ }: ImageParamsBarProps) { const getImageCountOptions = () => { // RUXA 当前按单图任务处理（优先级最高） - if (providerType === "ruxa-image") { + if (providerType === "ruxa-image" || providerType === "openai-image") { return [1]; } // 优先使用模型配置中的 maxImages diff --git a/components/messages.tsx b/components/messages.tsx index 53c43f945..29d9cdb8b 100644 --- a/components/messages.tsx +++ b/components/messages.tsx @@ -123,7 +123,8 @@ function PureMessages({ m.id === selectedModelId && (m.providerType === "alibaba-image" || m.providerType === "ruxa-image" || - m.providerType === "fal-image") + m.providerType === "fal-image" || + m.providerType === "openai-image") ) ?? false; // 🆕 获取当前模型的显示名称 @@ -682,12 +683,14 @@ function PureMessages({ return ( ; } diff --git a/lib/ai/dynamic-provider.ts b/lib/ai/dynamic-provider.ts index 9af4d2101..9db853cbf 100644 --- a/lib/ai/dynamic-provider.ts +++ b/lib/ai/dynamic-provider.ts @@ -20,13 +20,14 @@ import { createVolcanoVision } from "./providers/volcano-vision"; import { createAlibabaImage } from "./providers/alibaba-image"; import { createRuxaImage } from "./providers/ruxa-image"; import { createFalImage } from "./providers/fal-image"; +import { createOpenAIImage } from "./providers/openai-image"; import { createFalVideo } from "./providers/fal-video"; import { createRuxaVideo } from "./providers/ruxa-video"; import { createVolcanoMultimodal } from "./providers/volcano-multimodal"; import { getProviderDefaultBaseUrl } from "./provider-defaults"; // 缓存已创建的 provider 实例 -const providerCache = new Map>(); +const providerCache = new Map(); function isVolcanoMultimodalBaseUrl(baseUrl?: string | null) { if (!baseUrl) return false; @@ -207,6 +208,15 @@ function getOrCreateProvider(provider: Pick = { // 图片编辑：优先 alibaba-image（灵魂4.5，支持提示词扩展） image_edit: { - providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "gemini", "huanke-gemini"], + providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini", "huanke-gemini"], requireVision: true, requireImageGen: true, description: "图片编辑", @@ -45,7 +45,7 @@ const INTENT_MODEL_MAP: Record< // 图片生成：优先灵魂4.5 image_gen: { - providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "gemini", "huanke-gemini", "volcano-vision"], + providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini", "huanke-gemini", "volcano-vision"], requireImageGen: true, description: "图片生成", }, @@ -73,25 +73,25 @@ const SUB_INTENT_OVERRIDES: Record< > = { // 抠图：alibaba-image 效果最好 remove_background: { - providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "gemini"], + providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini"], reason: "抠图首选灵魂4.5", }, // 换背景：alibaba-image 支持场景生成 replace_background: { - providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "gemini"], + providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini"], reason: "换背景首选灵魂4.5", }, // 图片增强 enhance: { - providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "gemini", "volcano-vision"], + providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini", "volcano-vision"], reason: "增强首选灵魂4.5", }, // 风格迁移：gemini 可能更有创意 style_transfer: { - providerTypes: ["gemini", "alibaba-image", "ruxa-image", "fal-image"], + providerTypes: ["gemini", "alibaba-image", "ruxa-image", "fal-image", "openai-image"], reason: "风格迁移优选Gemini", }, }; diff --git a/lib/ai/provider-defaults.ts b/lib/ai/provider-defaults.ts index 6bd56af1a..323f7fd42 100644 --- a/lib/ai/provider-defaults.ts +++ b/lib/ai/provider-defaults.ts @@ -13,6 +13,7 @@ const PROVIDER_DEFAULT_BASE_URL: Partial> = { "https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation", "ruxa-image": "https://api.ruxa.ai", "fal-image": "https://fal.run", + "openai-image": "https://api.openai.com/v1", "fal-video": "https://fal.run", "tripo-3d": "https://api.tripo3d.ai/v2/openapi", }; diff --git a/lib/ai/providers/openai-image.ts b/lib/ai/providers/openai-image.ts new file mode 100644 index 000000000..4ebbbadd1 --- /dev/null +++ b/lib/ai/providers/openai-image.ts @@ -0,0 +1,538 @@ +import "server-only"; + +import sharp from "sharp"; +import { uploadFile } from "@/lib/storage/service"; + +interface OpenAIImageConfig { + apiKey: string; + baseURL: string; +} + +export interface OpenAIImageParams { + aspectRatio?: string; + imageSize?: string; + imageCount?: number; + size?: string; + quality?: string; + background?: string; + outputFormat?: string; +} + +interface UploadedImage { + thumbnail: string; + original: string; +} + +const DEFAULT_ASPECT_RATIO = "1:1"; +const DEFAULT_IMAGE_SIZE = "2K"; +const DEFAULT_TIMEOUT_MS = 180000; +const SUPPORTED_ASPECT_RATIOS = [ + "1:1", + "1:4", + "1:8", + "2:3", + "3:2", + "3:4", + "4:1", + "4:3", + "4:5", + "5:4", + "8:1", + "9:16", + "16:9", + "21:9", +] as const; + +function normalizeBaseURL(baseURL: string) { + const normalized = String(baseURL || "").trim().replace(/\/+$/, ""); + if (!normalized) return "https://api.openai.com/v1"; + if (/\/v1$/i.test(normalized)) return normalized; + return `${normalized}/v1`; +} + +function joinEndpoint(baseURL: string, path: string) { + const normalizedPath = path.startsWith("/") ? path : `/${path}`; + return `${normalizeBaseURL(baseURL)}${normalizedPath}`; +} + +function normalizeImageSizeLabel(imageSize?: string): "1K" | "2K" | "4K" { + const normalized = String(imageSize || "").trim().toUpperCase(); + if (normalized === "1K" || normalized === "2K" || normalized === "4K") return normalized; + return DEFAULT_IMAGE_SIZE; +} + +function normalizeAspectRatio(value?: string) { + const normalized = String(value || "").trim(); + if (!normalized || normalized.toLowerCase() === "auto") return "auto"; + if ((SUPPORTED_ASPECT_RATIOS as readonly string[]).includes(normalized)) return normalized; + return DEFAULT_ASPECT_RATIO; +} + +function parseAspectRatioValue(ratio: string) { + const [width, height] = ratio.split(":").map((item) => Number(item)); + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + return 1; + } + return width / height; +} + +function roundEven(value: number) { + return Math.max(2, Math.round(value / 2) * 2); +} + +function inferClosestAspectRatio(aspectValue: number) { + if (!Number.isFinite(aspectValue) || aspectValue <= 0) return DEFAULT_ASPECT_RATIO; + let closest = DEFAULT_ASPECT_RATIO; + let minDiff = Number.POSITIVE_INFINITY; + for (const candidate of SUPPORTED_ASPECT_RATIOS) { + const diff = Math.abs(parseAspectRatioValue(candidate) - aspectValue); + if (diff < minDiff) { + minDiff = diff; + closest = candidate; + } + } + return closest; +} + +async function inferAspectRatioFromImageUrl(imageUrl: string): Promise { + const normalized = String(imageUrl || "").trim(); + if (!normalized) return null; + + try { + let buffer: Buffer | null = null; + if (normalized.startsWith("data:")) { + const match = normalized.match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/); + if (!match) return null; + buffer = Buffer.from(match[2], "base64"); + } else if (/^https?:\/\//i.test(normalized)) { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 20000); + try { + const response = await fetch(normalized, { + signal: controller.signal, + cache: "no-store", + }); + if (!response.ok) return null; + buffer = Buffer.from(await response.arrayBuffer()); + } finally { + clearTimeout(timeoutId); + } + } + + if (!buffer?.length) return null; + const metadata = await sharp(buffer).metadata(); + if (!metadata.width || !metadata.height || metadata.height <= 0) return null; + return inferClosestAspectRatio(metadata.width / metadata.height); + } catch (error) { + console.warn("[OpenAI Image] Failed to infer aspect ratio:", error); + return null; + } +} + +function resolveOpenAISize(aspectRatio: string, imageSize: "1K" | "2K" | "4K") { + const baseMap: Record = { + "1K": 1024, + "2K": 1536, + "4K": 2048, + }; + + const safeRatio = parseAspectRatioValue(aspectRatio); + const base = baseMap[imageSize] || 1536; + + if (safeRatio >= 1) { + const width = base; + const height = roundEven(base / safeRatio); + return `${width}x${height}`; + } + + const height = base; + const width = roundEven(base * safeRatio); + return `${width}x${height}`; +} + +function normalizeQuality(value?: string) { + const normalized = String(value || "").trim().toLowerCase(); + if (normalized === "low" || normalized === "medium" || normalized === "high") { + return normalized; + } + return undefined; +} + +function normalizeBackground(value?: string) { + const normalized = String(value || "").trim().toLowerCase(); + if (normalized === "transparent" || normalized === "opaque") return normalized; + return undefined; +} + +function normalizeOutputFormat(value?: string) { + const normalized = String(value || "").trim().toLowerCase(); + if (normalized === "jpeg" || normalized === "jpg") return "jpeg"; + if (normalized === "webp") return "webp"; + if (normalized === "png") return "png"; + return undefined; +} + +function detectImageExtension(imageUrl: string, contentType?: string | null) { + const type = String(contentType || "").toLowerCase(); + if (type.includes("jpeg") || type.includes("jpg")) return "jpg"; + if (type.includes("webp")) return "webp"; + if (type.includes("png")) return "png"; + if (type.includes("gif")) return "gif"; + + const cleanUrl = imageUrl.split("?")[0]?.toLowerCase() || ""; + if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "jpg"; + if (cleanUrl.endsWith(".webp")) return "webp"; + if (cleanUrl.endsWith(".gif")) return "gif"; + if (cleanUrl.endsWith(".png")) return "png"; + return "png"; +} + +async function createThumbnail(buffer: Buffer) { + try { + return await sharp(buffer) + .resize(400, 400, { fit: "inside", withoutEnlargement: true }) + .jpeg({ quality: 80 }) + .toBuffer(); + } catch (error) { + console.error("[OpenAI Image] Failed to create thumbnail:", error); + return buffer; + } +} + +function generateFilename(prefix: string, extension: string) { + const timestamp = Date.now(); + const random = Math.random().toString(36).slice(2, 8); + return `${prefix}-${timestamp}-${random}.${extension}`; +} + +async function uploadGeneratedBuffer( + buffer: Buffer, + extension: string, + mimeType: string +): Promise { + const thumbnailBuffer = await createThumbnail(buffer); + + const originalFilename = generateFilename("openai-orig", extension); + const originalArrayBuffer = buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) as ArrayBuffer; + const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer); + + const thumbFilename = generateFilename("openai-thumb", "jpg"); + const thumbArrayBuffer = thumbnailBuffer.buffer.slice( + thumbnailBuffer.byteOffset, + thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength + ) as ArrayBuffer; + const thumbUploaded = await uploadFile(thumbFilename, thumbArrayBuffer); + + console.log("[OpenAI Image] Uploaded generated image:", { + original: originalUploaded.url, + thumbnail: thumbUploaded.url, + mimeType, + }); + + return { + thumbnail: thumbUploaded.url, + original: originalUploaded.url, + }; +} + +async function downloadAndUploadImage(imageUrl: string): Promise { + const response = await fetch(imageUrl, { cache: "no-store" }); + if (!response.ok) { + throw new Error(`Failed to download generated image: ${response.status}`); + } + const buffer = Buffer.from(await response.arrayBuffer()); + const extension = detectImageExtension(imageUrl, response.headers.get("content-type")); + const mimeType = response.headers.get("content-type") || `image/${extension === "jpg" ? "jpeg" : extension}`; + return uploadGeneratedBuffer(buffer, extension, mimeType); +} + +async function uploadBase64ToS3(base64Data: string, mimeType: string) { + const buffer = Buffer.from(base64Data, "base64"); + const extension = detectImageExtension("", mimeType); + return uploadGeneratedBuffer( + buffer, + extension, + mimeType || `image/${extension === "jpg" ? "jpeg" : extension}` + ); +} + +function extractPromptAndImages(messages: Array<{ role: string; content: any }>) { + const lastUserMessage = messages.filter((message) => message.role === "user").pop(); + if (!lastUserMessage) { + throw new Error("No user message found"); + } + + let prompt = ""; + const imageUrls: string[] = []; + + const pushImageUrl = (value?: string) => { + if (!value) return; + const normalized = String(value).trim(); + if (!normalized) return; + if (!imageUrls.includes(normalized)) { + imageUrls.push(normalized); + } + }; + + if (typeof lastUserMessage.content === "string") { + prompt = lastUserMessage.content.trim(); + } else if (Array.isArray(lastUserMessage.content)) { + for (const part of lastUserMessage.content) { + if (part?.type === "text" && typeof part.text === "string") { + prompt = `${prompt} ${part.text}`.trim(); + } else if (part?.type === "image_url" && part.image_url?.url) { + pushImageUrl(part.image_url.url); + } else if (part?.type === "file") { + pushImageUrl(part.url || part.data); + } else if (part?.type === "image") { + pushImageUrl(part.image || part.url); + } + } + } + + if (!prompt) { + throw new Error("No prompt found in user message"); + } + + return { prompt, imageUrls }; +} + +async function loadInputImage(source: string) { + const normalized = String(source || "").trim(); + if (!normalized) { + throw new Error("输入图片地址为空"); + } + + if (normalized.startsWith("data:")) { + const match = normalized.match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/); + if (!match) { + throw new Error("不支持的 Base64 图片格式"); + } + const mimeType = match[1] || "image/png"; + const buffer = Buffer.from(match[2], "base64"); + const extension = detectImageExtension("", mimeType); + return { + buffer, + mimeType, + filename: generateFilename("openai-edit-input", extension), + }; + } + + const response = await fetch(normalized, { cache: "no-store" }); + if (!response.ok) { + throw new Error(`Failed to fetch input image: ${response.status}`); + } + const buffer = Buffer.from(await response.arrayBuffer()); + const mimeType = response.headers.get("content-type") || "image/png"; + const extension = detectImageExtension(normalized, mimeType); + return { + buffer, + mimeType, + filename: generateFilename("openai-edit-input", extension), + }; +} + +function extractErrorMessage(payload: any) { + if (!payload) return "OpenAI Images 请求失败"; + if (typeof payload === "string") return payload; + if (payload.error?.message) return String(payload.error.message); + if (payload.message) return String(payload.message); + try { + return JSON.stringify(payload); + } catch { + return "OpenAI Images 请求失败"; + } +} + +async function parseJsonResponse(response: Response) { + const rawText = await response.text(); + try { + return JSON.parse(rawText); + } catch { + return rawText; + } +} + +async function requestOpenAIImageGeneration(params: { + baseURL: string; + apiKey: string; + model: string; + prompt: string; + size: string; + imageUrls: string[]; + quality?: string; + background?: string; + outputFormat?: string; +}) { + const { baseURL, apiKey, model, prompt, size, imageUrls, quality, background, outputFormat } = params; + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), DEFAULT_TIMEOUT_MS); + + try { + if (imageUrls.length > 0) { + const endpoint = joinEndpoint(baseURL, "/images/edits"); + const formData = new FormData(); + formData.append("model", model); + formData.append("prompt", prompt); + formData.append("size", size); + formData.append("response_format", "url"); + + if (quality) formData.append("quality", quality); + if (background) formData.append("background", background); + if (outputFormat) formData.append("output_format", outputFormat); + + if (imageUrls.length === 1) { + const image = await loadInputImage(imageUrls[0]); + formData.append("image", new Blob([image.buffer], { type: image.mimeType }), image.filename); + } else { + for (const source of imageUrls) { + const image = await loadInputImage(source); + formData.append("image[]", new Blob([image.buffer], { type: image.mimeType }), image.filename); + } + } + + const response = await fetch(endpoint, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + }, + body: formData, + signal: controller.signal, + }); + const payload = await parseJsonResponse(response); + if (!response.ok) { + throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`); + } + return payload; + } + + const endpoint = joinEndpoint(baseURL, "/images/generations"); + const response = await fetch(endpoint, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model, + prompt, + size, + response_format: "url", + ...(quality ? { quality } : {}), + ...(background ? { background } : {}), + ...(outputFormat ? { output_format: outputFormat } : {}), + }), + signal: controller.signal, + }); + const payload = await parseJsonResponse(response); + if (!response.ok) { + throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`); + } + return payload; + } finally { + clearTimeout(timeoutId); + } +} + +async function extractUploadedImages(payload: any): Promise { + const data = Array.isArray(payload?.data) ? payload.data : []; + const uploadedImages: UploadedImage[] = []; + + for (const item of data) { + if (item?.url && typeof item.url === "string") { + uploadedImages.push(await downloadAndUploadImage(item.url)); + continue; + } + + if (item?.b64_json && typeof item.b64_json === "string") { + uploadedImages.push(await uploadBase64ToS3(item.b64_json, "image/png")); + } + } + + return uploadedImages; +} + +function formatMarkdown(images: UploadedImage[], imageSize: string) { + return images + .map((image) => `![thumb|||${image.original}|||${imageSize}](${image.thumbnail})`) + .join("\n\n"); +} + +export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) { + return { + provider: "openai-image", + chat: (modelId: string) => ({ + provider: "openai-image", + doGenerate: async (options: { + messages: Array<{ role: string; content: any }>; + imageParams?: OpenAIImageParams; + overrideModelId?: string; + }) => { + const { prompt, imageUrls } = extractPromptAndImages(options.messages); + const activeModelId = options.overrideModelId || modelId; + const aspectRatio = + normalizeAspectRatio(options.imageParams?.aspectRatio) === "auto" + ? await inferAspectRatioFromImageUrl(imageUrls[0] || "") + : normalizeAspectRatio(options.imageParams?.aspectRatio); + const imageSize = normalizeImageSizeLabel(options.imageParams?.imageSize); + const effectiveAspectRatio = aspectRatio || DEFAULT_ASPECT_RATIO; + const size = + options.imageParams?.size || + resolveOpenAISize(effectiveAspectRatio, imageSize); + const quality = normalizeQuality(options.imageParams?.quality); + const background = normalizeBackground(options.imageParams?.background); + const outputFormat = normalizeOutputFormat(options.imageParams?.outputFormat); + + console.log("[OpenAI Image] Submitting request", { + model: activeModelId, + size, + aspectRatio: effectiveAspectRatio, + imageCount: imageUrls.length, + baseURL: normalizeBaseURL(baseURL), + }); + + const payload = await requestOpenAIImageGeneration({ + baseURL, + apiKey, + model: activeModelId, + prompt, + size, + imageUrls, + quality, + background, + outputFormat, + }); + + const uploadedImages = await extractUploadedImages(payload); + if (uploadedImages.length === 0) { + throw new Error("OpenAI Images 未返回可用图片"); + } + + const text = formatMarkdown(uploadedImages, imageSize); + return { + text, + finishReason: "stop" as const, + usage: { + promptTokens: 0, + completionTokens: text.length, + }, + }; + }, + doStream: async function* (options: { + messages: Array<{ role: string; content: any }>; + imageParams?: OpenAIImageParams; + overrideModelId?: string; + }) { + const result = await this.doGenerate(options); + yield { + type: "text-delta", + textDelta: result.text, + }; + }, + }), + }; +} diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index c2ede07fa..4b740f1d8 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -5,6 +5,7 @@ import { createGeminiImage } from "@/lib/ai/providers/gemini-image"; import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image"; import { createRuxaImage } from "@/lib/ai/providers/ruxa-image"; import { createFalImage } from "@/lib/ai/providers/fal-image"; +import { createOpenAIImage } from "@/lib/ai/providers/openai-image"; import { filterReachableRemoteUrls } from "@/lib/net/url-access"; export interface CanvasGenerateImageRequest { @@ -34,6 +35,7 @@ const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([ "alibaba-image", "ruxa-image", "fal-image", + "openai-image", ]); function parseImagesFromMarkdown(markdown: string): ParsedImage[] { @@ -130,7 +132,7 @@ export async function generateCanvasImage( } if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(fallbackProvider.type)) { throw new Error( - "当前图片模型类型暂不支持画布生成，请在后台选择 Gemini/阿里/RUXA/FAL 图片模型。" + "当前图片模型类型暂不支持画布生成，请在后台选择 Gemini/阿里/RUXA/FAL/OpenAI 图片模型。" ); } if (fallbackModel.model.id !== selectedModel.model.id) { @@ -224,9 +226,17 @@ export async function generateCanvasImage( const modelInstance = falProvider.chat(selectedModel.model.modelId); const result = await modelInstance.doGenerate({ messages, imageParams }); imageMarkdown = result.text || ""; + } else if (provider.type === "openai-image") { + const openAIImageProvider = createOpenAIImage({ + apiKey: provider.apiKey, + baseURL: provider.baseUrl, + }); + const modelInstance = openAIImageProvider.chat(selectedModel.model.modelId); + const result = await modelInstance.doGenerate({ messages, imageParams }); + imageMarkdown = result.text || ""; } else { throw new Error( - `不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/阿里/RUXA/FAL 图片模型` + `不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/阿里/RUXA/FAL/OpenAI 图片模型` ); } diff --git a/lib/db/migrate.ts b/lib/db/migrate.ts index 3f52061f5..dfbac3aa2 100644 --- a/lib/db/migrate.ts +++ b/lib/db/migrate.ts @@ -28,6 +28,7 @@ const providerBaseUrlDefaults = [ }, { type: "ruxa-image", baseUrl: "https://api.ruxa.ai" }, { type: "fal-image", baseUrl: "https://fal.run" }, + { type: "openai-image", baseUrl: "https://api.openai.com/v1" }, { type: "fal-video", baseUrl: "https://fal.run" }, { type: "tripo-3d", baseUrl: "https://api.tripo3d.ai/v2/openapi" }, ] as const; @@ -89,7 +90,7 @@ const runMigrate = async () => { ADD CONSTRAINT "AiProvider_type_check" CHECK (type IN ( 'alibaba', 'openai', 'custom', 'local', 'gemini', - 'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'ruxa-image', 'fal-image', 'fal-video', 'zhipu', 'tripo-3d' + 'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'ruxa-image', 'fal-image', 'openai-image', 'fal-video', 'zhipu', 'tripo-3d' )); `); for (const item of providerBaseUrlDefaults) { diff --git a/lib/db/queries.ts b/lib/db/queries.ts index 0d4034efa..7d8c4ef49 100644 --- a/lib/db/queries.ts +++ b/lib/db/queries.ts @@ -2005,7 +2005,7 @@ export async function createAiProvider({ updatedBy, }: { name: string; - type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d"; + type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d"; baseUrl: string; apiKey: string; isEnabled?: boolean; @@ -2043,7 +2043,7 @@ export async function updateAiProvider({ }: { id: string; name?: string; - type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d"; + type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d"; baseUrl?: string; apiKey?: string; isEnabled?: boolean; diff --git a/lib/db/schema.ts b/lib/db/schema.ts index dd2d5f8ec..dbcdcf37e 100644 --- a/lib/db/schema.ts +++ b/lib/db/schema.ts @@ -259,7 +259,7 @@ export type ToolConfig = InferSelectModel; export const aiProvider = pgTable("AiProvider", { id: uuid("id").primaryKey().notNull().defaultRandom(), name: varchar("name", { length: 128 }).notNull(), // 提供商名称，如 "阿里云通义千问" - type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "fal-video", "zhipu", "tripo-3d"] }) + type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "openai-image", "fal-video", "zhipu", "tripo-3d"] }) .notNull() .default("custom"), // 提供商类型 baseUrl: text("baseUrl").notNull(), // API 基础 URL