feat: add openai compatible image provider
This commit is contained in:
Your Name
parent
e8b71ddbe7
commit
d104efd7e9
@ -147,7 +147,7 @@ type ToolConfig = {
|
||||
type AiProvider = {
|
||||
id: string;
|
||||
name: string;
|
||||
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal";
|
||||
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal";
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
isEnabled: boolean;
|
||||
@ -437,6 +437,7 @@ const PROVIDER_TYPES = [
|
||||
{ value: "alibaba-image", label: "阿里生图模型" }, // 🆕 千问图片生成/编辑
|
||||
{ value: "ruxa-image", label: "RUXA生图" },
|
||||
{ value: "fal-image", label: "FAL 生图模型" },
|
||||
{ value: "openai-image", label: "OpenAI 图片兼容" },
|
||||
{ value: "tripo-3d", label: "Tripo3D 模型生成" },
|
||||
{ value: "zhipu", label: "智普官方 4.7" },
|
||||
{ value: "volcano-multimodal", label: "火山引擎多模态" },
|
||||
@ -459,6 +460,7 @@ const PROVIDER_DEFAULT_BASE_URL: Record<string, string> = {
|
||||
"https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation",
|
||||
"ruxa-image": "https://api.ruxa.ai",
|
||||
"fal-image": "https://fal.run",
|
||||
"openai-image": "https://api.openai.com/v1",
|
||||
"fal-video": "https://fal.run",
|
||||
"tripo-3d": "https://api.tripo3d.ai/v2/openapi",
|
||||
zhipu: "https://open.bigmodel.cn/api/paas/v4",
|
||||
@ -1325,7 +1327,7 @@ export function AdminSettingsClient() {
|
||||
const [providerForm, setProviderForm] = useState({
|
||||
id: "",
|
||||
name: "",
|
||||
type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal",
|
||||
type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal",
|
||||
baseUrl: "",
|
||||
apiKey: "",
|
||||
isEnabled: true,
|
||||
@ -3694,6 +3696,7 @@ export function AdminSettingsClient() {
|
||||
"alibaba-image": "bg-amber-500",
|
||||
"ruxa-image": "bg-emerald-500",
|
||||
"fal-image": "bg-sky-500",
|
||||
"openai-image": "bg-green-600",
|
||||
"tripo-3d": "bg-teal-500",
|
||||
"openai": "bg-green-500",
|
||||
"zhipu": "bg-lime-500",
|
||||
@ -8725,7 +8728,7 @@ export function AdminSettingsClient() {
|
||||
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
|
||||
if (!provider) return false;
|
||||
// 支持阿里生图、Gemini、既梦等图片模型
|
||||
const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "gemini-image", "gemini", "volcano-vision"];
|
||||
const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini-image", "gemini", "volcano-vision"];
|
||||
return imageProviderTypes.includes(provider.type);
|
||||
})
|
||||
.map((m: AiModel) => {
|
||||
@ -8740,7 +8743,7 @@ export function AdminSettingsClient() {
|
||||
{/* 如果没有找到生图模型，显示提示 */}
|
||||
{(modelsData?.models || []).filter((m: AiModel) => {
|
||||
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
|
||||
const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "gemini-image", "gemini", "volcano-vision"];
|
||||
const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini-image", "gemini", "volcano-vision"];
|
||||
return provider && imageProviderTypes.includes(provider.type);
|
||||
}).length === 0 && (
|
||||
<SelectItem value="no-model" disabled>
|
||||
|
||||
@ -597,6 +597,7 @@ const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([
|
||||
"alibaba-image",
|
||||
"ruxa-image",
|
||||
"fal-image",
|
||||
"openai-image",
|
||||
"volcano-vision",
|
||||
"gemini",
|
||||
"huanke-gemini",
|
||||
@ -1112,7 +1113,7 @@ function ensureImageAnalysisSummarySection(message: string, summary: string): st
|
||||
* 因为这些 provider 内部会自行处理图片上传到 S3
|
||||
*/
|
||||
async function convertMessagesWithImages(messages: ChatMessage[], providerType: string) {
|
||||
if (providerType === "ruxa-image" || providerType === "fal-image") {
|
||||
if (providerType === "ruxa-image" || providerType === "fal-image" || providerType === "openai-image") {
|
||||
const convertedMessages = [];
|
||||
|
||||
for (const message of messages) {
|
||||
@ -2115,6 +2116,7 @@ export async function POST(request: Request) {
|
||||
"alibaba-image",
|
||||
"ruxa-image",
|
||||
"fal-image",
|
||||
"openai-image",
|
||||
"volcano-vision",
|
||||
]);
|
||||
const isFixedVisualModel =
|
||||
@ -3231,6 +3233,7 @@ export async function POST(request: Request) {
|
||||
(languageModel as any).provider === "alibaba-image" ||
|
||||
(languageModel as any).provider === "ruxa-image" ||
|
||||
(languageModel as any).provider === "fal-image" ||
|
||||
(languageModel as any).provider === "openai-image" ||
|
||||
(languageModel as any).provider === "huanke-gemini";
|
||||
|
||||
const isUuidModelId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||||
@ -3718,7 +3721,7 @@ export async function POST(request: Request) {
|
||||
// ========== Gemini 处理结束 ==========
|
||||
|
||||
// ========== 阿里云千问图片生成独立处理 ==========
|
||||
const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image", "fal-image"].includes((languageModel as any).provider);
|
||||
const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image", "fal-image", "openai-image"].includes((languageModel as any).provider);
|
||||
if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||||
console.log("[Chat API] Using image generation provider:", (languageModel as any).provider);
|
||||
console.log("[Chat API] Image params:", effectiveImageParams);
|
||||
@ -3808,6 +3811,8 @@ export async function POST(request: Request) {
|
||||
? "ruxa-image"
|
||||
: (languageModel as any).provider === "fal-image"
|
||||
? "fal-image"
|
||||
: (languageModel as any).provider === "openai-image"
|
||||
? "openai-image"
|
||||
: "alibaba-image";
|
||||
const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert);
|
||||
|
||||
@ -3867,8 +3872,9 @@ export async function POST(request: Request) {
|
||||
"alibaba-image": 1,
|
||||
"ruxa-image": 2,
|
||||
"fal-image": 3,
|
||||
gemini: 4,
|
||||
"huanke-gemini": 5,
|
||||
"openai-image": 4,
|
||||
gemini: 5,
|
||||
"huanke-gemini": 6,
|
||||
};
|
||||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||||
currentModelId: finalModelId,
|
||||
@ -3893,13 +3899,15 @@ export async function POST(request: Request) {
|
||||
? "ruxa-image"
|
||||
: candidate.provider.type === "fal-image"
|
||||
? "fal-image"
|
||||
: candidate.provider.type === "gemini"
|
||||
? "gemini"
|
||||
: candidate.provider.type === "huanke-gemini"
|
||||
? "huanke-gemini"
|
||||
: candidate.provider.type === "volcano-vision"
|
||||
? "volcano"
|
||||
: "alibaba-image";
|
||||
: candidate.provider.type === "openai-image"
|
||||
? "openai-image"
|
||||
: candidate.provider.type === "gemini"
|
||||
? "gemini"
|
||||
: candidate.provider.type === "huanke-gemini"
|
||||
? "huanke-gemini"
|
||||
: candidate.provider.type === "volcano-vision"
|
||||
? "volcano"
|
||||
: "alibaba-image";
|
||||
|
||||
const candidateMessages = await convertMessagesWithImages(
|
||||
messagesForAlibaba,
|
||||
@ -4216,9 +4224,10 @@ export async function POST(request: Request) {
|
||||
"alibaba-image": 0,
|
||||
"ruxa-image": 1,
|
||||
"fal-image": 2,
|
||||
gemini: 3,
|
||||
"huanke-gemini": 4,
|
||||
"volcano-vision": 5,
|
||||
"openai-image": 3,
|
||||
gemini: 4,
|
||||
"huanke-gemini": 5,
|
||||
"volcano-vision": 6,
|
||||
};
|
||||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||||
currentModelId: finalModelId,
|
||||
@ -4243,13 +4252,15 @@ export async function POST(request: Request) {
|
||||
? "ruxa-image"
|
||||
: candidate.provider.type === "fal-image"
|
||||
? "fal-image"
|
||||
: candidate.provider.type === "gemini"
|
||||
? "gemini"
|
||||
: candidate.provider.type === "huanke-gemini"
|
||||
? "huanke-gemini"
|
||||
: candidate.provider.type === "volcano-vision"
|
||||
? "volcano"
|
||||
: "alibaba-image";
|
||||
: candidate.provider.type === "openai-image"
|
||||
? "openai-image"
|
||||
: candidate.provider.type === "gemini"
|
||||
? "gemini"
|
||||
: candidate.provider.type === "huanke-gemini"
|
||||
? "huanke-gemini"
|
||||
: candidate.provider.type === "volcano-vision"
|
||||
? "volcano"
|
||||
: "alibaba-image";
|
||||
|
||||
const candidateMessages = await convertMessagesWithImages(
|
||||
messagesForVolcano,
|
||||
@ -4844,8 +4855,9 @@ export async function POST(request: Request) {
|
||||
"alibaba-image": 1,
|
||||
"ruxa-image": 2,
|
||||
"fal-image": 3,
|
||||
gemini: 4,
|
||||
"huanke-gemini": 5,
|
||||
"openai-image": 4,
|
||||
gemini: 5,
|
||||
"huanke-gemini": 6,
|
||||
};
|
||||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||||
currentModelId: finalModelId,
|
||||
@ -4875,13 +4887,15 @@ export async function POST(request: Request) {
|
||||
? "ruxa-image"
|
||||
: candidate.provider.type === "fal-image"
|
||||
? "fal-image"
|
||||
: candidate.provider.type === "gemini"
|
||||
? "gemini"
|
||||
: candidate.provider.type === "huanke-gemini"
|
||||
? "huanke-gemini"
|
||||
: candidate.provider.type === "volcano-vision"
|
||||
? "volcano"
|
||||
: "alibaba-image";
|
||||
: candidate.provider.type === "openai-image"
|
||||
? "openai-image"
|
||||
: candidate.provider.type === "gemini"
|
||||
? "gemini"
|
||||
: candidate.provider.type === "huanke-gemini"
|
||||
? "huanke-gemini"
|
||||
: candidate.provider.type === "volcano-vision"
|
||||
? "volcano"
|
||||
: "alibaba-image";
|
||||
|
||||
const candidateMessages = await convertMessagesWithImages(
|
||||
messagesForCrossProvider,
|
||||
|
||||
@ -24,6 +24,7 @@ const providerTypes = [
|
||||
"alibaba-image",
|
||||
"ruxa-image",
|
||||
"fal-image",
|
||||
"openai-image",
|
||||
"fal-video",
|
||||
"zhipu",
|
||||
"tripo-3d",
|
||||
@ -54,6 +55,7 @@ const VOLCANO_MULTIMODAL_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3";
|
||||
const DOUBAO_VIDEO_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3";
|
||||
const RUXA_VIDEO_BASE_URL = "https://api.ruxa.ai";
|
||||
const RUXA_IMAGE_BASE_URL = "https://api.ruxa.ai";
|
||||
const OPENAI_IMAGE_BASE_URL = "https://api.openai.com/v1";
|
||||
const TRIPO_3D_BASE_URL = "https://api.tripo3d.ai/v2/openapi";
|
||||
|
||||
function normalizeProviderTypeForStorage(
|
||||
@ -165,6 +167,8 @@ export async function POST(request: Request) {
|
||||
? data.baseUrl || RUXA_VIDEO_BASE_URL
|
||||
: data.type === "ruxa-image"
|
||||
? data.baseUrl || RUXA_IMAGE_BASE_URL
|
||||
: data.type === "openai-image"
|
||||
? data.baseUrl || OPENAI_IMAGE_BASE_URL
|
||||
: data.type === "tripo-3d"
|
||||
? data.baseUrl || TRIPO_3D_BASE_URL
|
||||
: data.baseUrl,
|
||||
|
||||
@ -23,6 +23,7 @@ import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
|
||||
import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision";
|
||||
import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
|
||||
import { createFalImage } from "@/lib/ai/providers/fal-image";
|
||||
import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
|
||||
import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
|
||||
|
||||
// 请求体 Schema
|
||||
@ -98,6 +99,7 @@ function canProcessSceneFusionProviderType(providerType: string): boolean {
|
||||
case "alibaba-image":
|
||||
case "ruxa-image":
|
||||
case "fal-image":
|
||||
case "openai-image":
|
||||
case "gemini":
|
||||
case "volcano-vision":
|
||||
return true;
|
||||
@ -1236,6 +1238,9 @@ async function processSceneWithProvider(
|
||||
case "fal-image":
|
||||
return processWithFalImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
|
||||
|
||||
case "openai-image":
|
||||
return processWithOpenAIImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
|
||||
|
||||
case "gemini":
|
||||
case "gemini-image":
|
||||
return processWithGemini(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
|
||||
@ -1411,6 +1416,46 @@ async function processWithFalImage(
|
||||
return { thumbnail, original, prompt };
|
||||
}
|
||||
|
||||
async function processWithOpenAIImage(
|
||||
productImageUrl: string,
|
||||
sceneImageUrl: string,
|
||||
prompt: string,
|
||||
provider: { baseUrl: string; apiKey: string },
|
||||
modelId: string,
|
||||
aspectRatio?: string,
|
||||
imageSize?: string
|
||||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||||
console.log(`[SceneFusion][OpenAI Image] Processing with model: ${modelId}`);
|
||||
|
||||
const openAIImageProvider = createOpenAIImage({
|
||||
baseURL: provider.baseUrl,
|
||||
apiKey: provider.apiKey,
|
||||
});
|
||||
|
||||
const messages = [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "image_url", image_url: { url: productImageUrl } },
|
||||
{ type: "image_url", image_url: { url: sceneImageUrl } },
|
||||
{ type: "text", text: prompt },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const result = await openAIImageProvider.chat(modelId).doGenerate({
|
||||
messages,
|
||||
imageParams: {
|
||||
aspectRatio: aspectRatio || "auto",
|
||||
imageSize: imageSize || "2K",
|
||||
imageCount: 1,
|
||||
},
|
||||
});
|
||||
|
||||
const { thumbnail, original } = parseImageFromMarkdown(result.text);
|
||||
return { thumbnail, original, prompt };
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用既梦/火山模型处理
|
||||
*/
|
||||
|
||||
@ -49,7 +49,7 @@ function normalizeImageParamsForProvider(
|
||||
const normalized = { ...params };
|
||||
const currentCount = normalized.imageCount || 1;
|
||||
|
||||
if (providerType === "ruxa-image") {
|
||||
if (providerType === "ruxa-image" || providerType === "openai-image") {
|
||||
normalized.imageCount = 1;
|
||||
return normalized;
|
||||
}
|
||||
@ -1052,7 +1052,8 @@ export function Chat({
|
||||
else if (
|
||||
model?.providerType === "alibaba-image" ||
|
||||
model?.providerType === "ruxa-image" ||
|
||||
model?.providerType === "fal-image"
|
||||
model?.providerType === "fal-image" ||
|
||||
model?.providerType === "openai-image"
|
||||
) modelType = "alibaba-image";
|
||||
|
||||
// 智能自动模式下，如果当前轮输入包含图片，默认按图片生成链路跟踪
|
||||
|
||||
@ -20,6 +20,7 @@ const PROVIDER_MODEL_LABEL: Record<string, string> = {
|
||||
"alibaba-image": "千问图片",
|
||||
"ruxa-image": "RUXA 图片",
|
||||
"fal-image": "FAL 图片",
|
||||
"openai-image": "OpenAI 图片",
|
||||
"volcano-vision": "即梦 4.5",
|
||||
gemini: "Gemini 图片",
|
||||
};
|
||||
|
||||
@ -87,7 +87,7 @@ export function ImageParamsBar({
|
||||
}: ImageParamsBarProps) {
|
||||
const getImageCountOptions = () => {
|
||||
// RUXA 当前按单图任务处理（优先级最高）
|
||||
if (providerType === "ruxa-image") {
|
||||
if (providerType === "ruxa-image" || providerType === "openai-image") {
|
||||
return [1];
|
||||
}
|
||||
// 优先使用模型配置中的 maxImages
|
||||
|
||||
@ -123,7 +123,8 @@ function PureMessages({
|
||||
m.id === selectedModelId &&
|
||||
(m.providerType === "alibaba-image" ||
|
||||
m.providerType === "ruxa-image" ||
|
||||
m.providerType === "fal-image")
|
||||
m.providerType === "fal-image" ||
|
||||
m.providerType === "openai-image")
|
||||
) ?? false;
|
||||
|
||||
// 🆕 获取当前模型的显示名称
|
||||
@ -682,12 +683,14 @@ function PureMessages({
|
||||
return (
|
||||
<ImageGenerationLoading
|
||||
imageCount={imageCount}
|
||||
maxImages={6}
|
||||
maxImages={currentModelProviderType === "openai-image" ? 1 : 6}
|
||||
providerType={
|
||||
currentModelProviderType === "ruxa-image"
|
||||
? "ruxa-image"
|
||||
: currentModelProviderType === "fal-image"
|
||||
? "fal-image"
|
||||
: currentModelProviderType === "openai-image"
|
||||
? "openai-image"
|
||||
: "alibaba-image"
|
||||
}
|
||||
modelName={currentModelDisplayName}
|
||||
|
||||
@ -1790,7 +1790,8 @@ function PureModelSelectorCompact({
|
||||
providerType === "volcano-vision" ||
|
||||
providerType === "alibaba-image" ||
|
||||
providerType === "ruxa-image" ||
|
||||
providerType === "fal-image"
|
||||
providerType === "fal-image" ||
|
||||
providerType === "openai-image"
|
||||
) {
|
||||
return <ImageIcon className="h-4 w-4" strokeWidth={2.2} />;
|
||||
}
|
||||
|
||||
@ -20,13 +20,14 @@ import { createVolcanoVision } from "./providers/volcano-vision";
|
||||
import { createAlibabaImage } from "./providers/alibaba-image";
|
||||
import { createRuxaImage } from "./providers/ruxa-image";
|
||||
import { createFalImage } from "./providers/fal-image";
|
||||
import { createOpenAIImage } from "./providers/openai-image";
|
||||
import { createFalVideo } from "./providers/fal-video";
|
||||
import { createRuxaVideo } from "./providers/ruxa-video";
|
||||
import { createVolcanoMultimodal } from "./providers/volcano-multimodal";
|
||||
import { getProviderDefaultBaseUrl } from "./provider-defaults";
|
||||
|
||||
// 缓存已创建的 provider 实例
|
||||
const providerCache = new Map<string, ReturnType<typeof createOpenAI>>();
|
||||
const providerCache = new Map<string, any>();
|
||||
|
||||
function isVolcanoMultimodalBaseUrl(baseUrl?: string | null) {
|
||||
if (!baseUrl) return false;
|
||||
@ -207,6 +208,15 @@ function getOrCreateProvider(provider: Pick<AiProvider, "id" | "type" | "baseUrl
|
||||
return falImageProvider as any;
|
||||
}
|
||||
|
||||
case "openai-image": {
|
||||
const openAIImageProvider = createOpenAIImage({
|
||||
baseURL: resolvedBaseUrl,
|
||||
apiKey: provider.apiKey,
|
||||
});
|
||||
console.log(`[Provider] Created OpenAI Image generation provider: ${resolvedBaseUrl}`);
|
||||
return openAIImageProvider as any;
|
||||
}
|
||||
|
||||
case "custom":
|
||||
default: {
|
||||
if (isVolcanoMultimodalBaseUrl(resolvedBaseUrl)) {
|
||||
@ -324,6 +334,11 @@ export async function getDynamicModel(
|
||||
console.log("[getDynamicModel] Getting Fal Image chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] Fal Image model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "openai-image") {
|
||||
// OpenAI Images 兼容图片生成/编辑使用 chat() 方法
|
||||
console.log("[getDynamicModel] Getting OpenAI Image chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] OpenAI Image model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "huanke-gemini") {
|
||||
// 环科 Gemini 原生格式使用 chat() 方法
|
||||
console.log("[getDynamicModel] Getting Huanke Gemini chat model..."); // 调试日志
|
||||
|
||||
@ -30,7 +30,7 @@ const INTENT_MODEL_MAP: Record<
|
||||
> = {
|
||||
// 图片编辑：优先 alibaba-image（灵魂4.5，支持提示词扩展）
|
||||
image_edit: {
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "gemini", "huanke-gemini"],
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini", "huanke-gemini"],
|
||||
requireVision: true,
|
||||
requireImageGen: true,
|
||||
description: "图片编辑",
|
||||
@ -45,7 +45,7 @@ const INTENT_MODEL_MAP: Record<
|
||||
|
||||
// 图片生成：优先灵魂4.5
|
||||
image_gen: {
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "gemini", "huanke-gemini", "volcano-vision"],
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini", "huanke-gemini", "volcano-vision"],
|
||||
requireImageGen: true,
|
||||
description: "图片生成",
|
||||
},
|
||||
@ -73,25 +73,25 @@ const SUB_INTENT_OVERRIDES: Record<
|
||||
> = {
|
||||
// 抠图：alibaba-image 效果最好
|
||||
remove_background: {
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "gemini"],
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini"],
|
||||
reason: "抠图首选灵魂4.5",
|
||||
},
|
||||
|
||||
// 换背景：alibaba-image 支持场景生成
|
||||
replace_background: {
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "gemini"],
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini"],
|
||||
reason: "换背景首选灵魂4.5",
|
||||
},
|
||||
|
||||
// 图片增强
|
||||
enhance: {
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "gemini", "volcano-vision"],
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini", "volcano-vision"],
|
||||
reason: "增强首选灵魂4.5",
|
||||
},
|
||||
|
||||
// 风格迁移：gemini 可能更有创意
|
||||
style_transfer: {
|
||||
providerTypes: ["gemini", "alibaba-image", "ruxa-image", "fal-image"],
|
||||
providerTypes: ["gemini", "alibaba-image", "ruxa-image", "fal-image", "openai-image"],
|
||||
reason: "风格迁移优选Gemini",
|
||||
},
|
||||
};
|
||||
|
||||
@ -13,6 +13,7 @@ const PROVIDER_DEFAULT_BASE_URL: Partial<Record<AiProvider["type"], string>> = {
|
||||
"https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation",
|
||||
"ruxa-image": "https://api.ruxa.ai",
|
||||
"fal-image": "https://fal.run",
|
||||
"openai-image": "https://api.openai.com/v1",
|
||||
"fal-video": "https://fal.run",
|
||||
"tripo-3d": "https://api.tripo3d.ai/v2/openapi",
|
||||
};
|
||||
|
||||
538
lib/ai/providers/openai-image.ts
Normal file
538
lib/ai/providers/openai-image.ts
Normal file
@ -0,0 +1,538 @@
|
||||
import "server-only";
|
||||
|
||||
import sharp from "sharp";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
|
||||
interface OpenAIImageConfig {
|
||||
apiKey: string;
|
||||
baseURL: string;
|
||||
}
|
||||
|
||||
export interface OpenAIImageParams {
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
imageCount?: number;
|
||||
size?: string;
|
||||
quality?: string;
|
||||
background?: string;
|
||||
outputFormat?: string;
|
||||
}
|
||||
|
||||
interface UploadedImage {
|
||||
thumbnail: string;
|
||||
original: string;
|
||||
}
|
||||
|
||||
const DEFAULT_ASPECT_RATIO = "1:1";
|
||||
const DEFAULT_IMAGE_SIZE = "2K";
|
||||
const DEFAULT_TIMEOUT_MS = 180000;
|
||||
const SUPPORTED_ASPECT_RATIOS = [
|
||||
"1:1",
|
||||
"1:4",
|
||||
"1:8",
|
||||
"2:3",
|
||||
"3:2",
|
||||
"3:4",
|
||||
"4:1",
|
||||
"4:3",
|
||||
"4:5",
|
||||
"5:4",
|
||||
"8:1",
|
||||
"9:16",
|
||||
"16:9",
|
||||
"21:9",
|
||||
] as const;
|
||||
|
||||
function normalizeBaseURL(baseURL: string) {
|
||||
const normalized = String(baseURL || "").trim().replace(/\/+$/, "");
|
||||
if (!normalized) return "https://api.openai.com/v1";
|
||||
if (/\/v1$/i.test(normalized)) return normalized;
|
||||
return `${normalized}/v1`;
|
||||
}
|
||||
|
||||
function joinEndpoint(baseURL: string, path: string) {
|
||||
const normalizedPath = path.startsWith("/") ? path : `/${path}`;
|
||||
return `${normalizeBaseURL(baseURL)}${normalizedPath}`;
|
||||
}
|
||||
|
||||
function normalizeImageSizeLabel(imageSize?: string): "1K" | "2K" | "4K" {
|
||||
const normalized = String(imageSize || "").trim().toUpperCase();
|
||||
if (normalized === "1K" || normalized === "2K" || normalized === "4K") return normalized;
|
||||
return DEFAULT_IMAGE_SIZE;
|
||||
}
|
||||
|
||||
function normalizeAspectRatio(value?: string) {
|
||||
const normalized = String(value || "").trim();
|
||||
if (!normalized || normalized.toLowerCase() === "auto") return "auto";
|
||||
if ((SUPPORTED_ASPECT_RATIOS as readonly string[]).includes(normalized)) return normalized;
|
||||
return DEFAULT_ASPECT_RATIO;
|
||||
}
|
||||
|
||||
function parseAspectRatioValue(ratio: string) {
|
||||
const [width, height] = ratio.split(":").map((item) => Number(item));
|
||||
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
|
||||
return 1;
|
||||
}
|
||||
return width / height;
|
||||
}
|
||||
|
||||
function roundEven(value: number) {
|
||||
return Math.max(2, Math.round(value / 2) * 2);
|
||||
}
|
||||
|
||||
function inferClosestAspectRatio(aspectValue: number) {
|
||||
if (!Number.isFinite(aspectValue) || aspectValue <= 0) return DEFAULT_ASPECT_RATIO;
|
||||
let closest = DEFAULT_ASPECT_RATIO;
|
||||
let minDiff = Number.POSITIVE_INFINITY;
|
||||
for (const candidate of SUPPORTED_ASPECT_RATIOS) {
|
||||
const diff = Math.abs(parseAspectRatioValue(candidate) - aspectValue);
|
||||
if (diff < minDiff) {
|
||||
minDiff = diff;
|
||||
closest = candidate;
|
||||
}
|
||||
}
|
||||
return closest;
|
||||
}
|
||||
|
||||
async function inferAspectRatioFromImageUrl(imageUrl: string): Promise<string | null> {
|
||||
const normalized = String(imageUrl || "").trim();
|
||||
if (!normalized) return null;
|
||||
|
||||
try {
|
||||
let buffer: Buffer | null = null;
|
||||
if (normalized.startsWith("data:")) {
|
||||
const match = normalized.match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/);
|
||||
if (!match) return null;
|
||||
buffer = Buffer.from(match[2], "base64");
|
||||
} else if (/^https?:\/\//i.test(normalized)) {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), 20000);
|
||||
try {
|
||||
const response = await fetch(normalized, {
|
||||
signal: controller.signal,
|
||||
cache: "no-store",
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
buffer = Buffer.from(await response.arrayBuffer());
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
if (!buffer?.length) return null;
|
||||
const metadata = await sharp(buffer).metadata();
|
||||
if (!metadata.width || !metadata.height || metadata.height <= 0) return null;
|
||||
return inferClosestAspectRatio(metadata.width / metadata.height);
|
||||
} catch (error) {
|
||||
console.warn("[OpenAI Image] Failed to infer aspect ratio:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveOpenAISize(aspectRatio: string, imageSize: "1K" | "2K" | "4K") {
|
||||
const baseMap: Record<typeof imageSize, number> = {
|
||||
"1K": 1024,
|
||||
"2K": 1536,
|
||||
"4K": 2048,
|
||||
};
|
||||
|
||||
const safeRatio = parseAspectRatioValue(aspectRatio);
|
||||
const base = baseMap[imageSize] || 1536;
|
||||
|
||||
if (safeRatio >= 1) {
|
||||
const width = base;
|
||||
const height = roundEven(base / safeRatio);
|
||||
return `${width}x${height}`;
|
||||
}
|
||||
|
||||
const height = base;
|
||||
const width = roundEven(base * safeRatio);
|
||||
return `${width}x${height}`;
|
||||
}
|
||||
|
||||
function normalizeQuality(value?: string) {
|
||||
const normalized = String(value || "").trim().toLowerCase();
|
||||
if (normalized === "low" || normalized === "medium" || normalized === "high") {
|
||||
return normalized;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeBackground(value?: string) {
|
||||
const normalized = String(value || "").trim().toLowerCase();
|
||||
if (normalized === "transparent" || normalized === "opaque") return normalized;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeOutputFormat(value?: string) {
|
||||
const normalized = String(value || "").trim().toLowerCase();
|
||||
if (normalized === "jpeg" || normalized === "jpg") return "jpeg";
|
||||
if (normalized === "webp") return "webp";
|
||||
if (normalized === "png") return "png";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function detectImageExtension(imageUrl: string, contentType?: string | null) {
|
||||
const type = String(contentType || "").toLowerCase();
|
||||
if (type.includes("jpeg") || type.includes("jpg")) return "jpg";
|
||||
if (type.includes("webp")) return "webp";
|
||||
if (type.includes("png")) return "png";
|
||||
if (type.includes("gif")) return "gif";
|
||||
|
||||
const cleanUrl = imageUrl.split("?")[0]?.toLowerCase() || "";
|
||||
if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "jpg";
|
||||
if (cleanUrl.endsWith(".webp")) return "webp";
|
||||
if (cleanUrl.endsWith(".gif")) return "gif";
|
||||
if (cleanUrl.endsWith(".png")) return "png";
|
||||
return "png";
|
||||
}
|
||||
|
||||
async function createThumbnail(buffer: Buffer) {
|
||||
try {
|
||||
return await sharp(buffer)
|
||||
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
|
||||
.jpeg({ quality: 80 })
|
||||
.toBuffer();
|
||||
} catch (error) {
|
||||
console.error("[OpenAI Image] Failed to create thumbnail:", error);
|
||||
return buffer;
|
||||
}
|
||||
}
|
||||
|
||||
function generateFilename(prefix: string, extension: string) {
|
||||
const timestamp = Date.now();
|
||||
const random = Math.random().toString(36).slice(2, 8);
|
||||
return `${prefix}-${timestamp}-${random}.${extension}`;
|
||||
}
|
||||
|
||||
async function uploadGeneratedBuffer(
|
||||
buffer: Buffer,
|
||||
extension: string,
|
||||
mimeType: string
|
||||
): Promise<UploadedImage> {
|
||||
const thumbnailBuffer = await createThumbnail(buffer);
|
||||
|
||||
const originalFilename = generateFilename("openai-orig", extension);
|
||||
const originalArrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer);
|
||||
|
||||
const thumbFilename = generateFilename("openai-thumb", "jpg");
|
||||
const thumbArrayBuffer = thumbnailBuffer.buffer.slice(
|
||||
thumbnailBuffer.byteOffset,
|
||||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const thumbUploaded = await uploadFile(thumbFilename, thumbArrayBuffer);
|
||||
|
||||
console.log("[OpenAI Image] Uploaded generated image:", {
|
||||
original: originalUploaded.url,
|
||||
thumbnail: thumbUploaded.url,
|
||||
mimeType,
|
||||
});
|
||||
|
||||
return {
|
||||
thumbnail: thumbUploaded.url,
|
||||
original: originalUploaded.url,
|
||||
};
|
||||
}
|
||||
|
||||
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
||||
const response = await fetch(imageUrl, { cache: "no-store" });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download generated image: ${response.status}`);
|
||||
}
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
const extension = detectImageExtension(imageUrl, response.headers.get("content-type"));
|
||||
const mimeType = response.headers.get("content-type") || `image/${extension === "jpg" ? "jpeg" : extension}`;
|
||||
return uploadGeneratedBuffer(buffer, extension, mimeType);
|
||||
}
|
||||
|
||||
async function uploadBase64ToS3(base64Data: string, mimeType: string) {
|
||||
const buffer = Buffer.from(base64Data, "base64");
|
||||
const extension = detectImageExtension("", mimeType);
|
||||
return uploadGeneratedBuffer(
|
||||
buffer,
|
||||
extension,
|
||||
mimeType || `image/${extension === "jpg" ? "jpeg" : extension}`
|
||||
);
|
||||
}
|
||||
|
||||
function extractPromptAndImages(messages: Array<{ role: string; content: any }>) {
|
||||
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
|
||||
if (!lastUserMessage) {
|
||||
throw new Error("No user message found");
|
||||
}
|
||||
|
||||
let prompt = "";
|
||||
const imageUrls: string[] = [];
|
||||
|
||||
const pushImageUrl = (value?: string) => {
|
||||
if (!value) return;
|
||||
const normalized = String(value).trim();
|
||||
if (!normalized) return;
|
||||
if (!imageUrls.includes(normalized)) {
|
||||
imageUrls.push(normalized);
|
||||
}
|
||||
};
|
||||
|
||||
if (typeof lastUserMessage.content === "string") {
|
||||
prompt = lastUserMessage.content.trim();
|
||||
} else if (Array.isArray(lastUserMessage.content)) {
|
||||
for (const part of lastUserMessage.content) {
|
||||
if (part?.type === "text" && typeof part.text === "string") {
|
||||
prompt = `${prompt} ${part.text}`.trim();
|
||||
} else if (part?.type === "image_url" && part.image_url?.url) {
|
||||
pushImageUrl(part.image_url.url);
|
||||
} else if (part?.type === "file") {
|
||||
pushImageUrl(part.url || part.data);
|
||||
} else if (part?.type === "image") {
|
||||
pushImageUrl(part.image || part.url);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!prompt) {
|
||||
throw new Error("No prompt found in user message");
|
||||
}
|
||||
|
||||
return { prompt, imageUrls };
|
||||
}
|
||||
|
||||
async function loadInputImage(source: string) {
|
||||
const normalized = String(source || "").trim();
|
||||
if (!normalized) {
|
||||
throw new Error("输入图片地址为空");
|
||||
}
|
||||
|
||||
if (normalized.startsWith("data:")) {
|
||||
const match = normalized.match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/);
|
||||
if (!match) {
|
||||
throw new Error("不支持的 Base64 图片格式");
|
||||
}
|
||||
const mimeType = match[1] || "image/png";
|
||||
const buffer = Buffer.from(match[2], "base64");
|
||||
const extension = detectImageExtension("", mimeType);
|
||||
return {
|
||||
buffer,
|
||||
mimeType,
|
||||
filename: generateFilename("openai-edit-input", extension),
|
||||
};
|
||||
}
|
||||
|
||||
const response = await fetch(normalized, { cache: "no-store" });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch input image: ${response.status}`);
|
||||
}
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
const mimeType = response.headers.get("content-type") || "image/png";
|
||||
const extension = detectImageExtension(normalized, mimeType);
|
||||
return {
|
||||
buffer,
|
||||
mimeType,
|
||||
filename: generateFilename("openai-edit-input", extension),
|
||||
};
|
||||
}
|
||||
|
||||
function extractErrorMessage(payload: any) {
|
||||
if (!payload) return "OpenAI Images 请求失败";
|
||||
if (typeof payload === "string") return payload;
|
||||
if (payload.error?.message) return String(payload.error.message);
|
||||
if (payload.message) return String(payload.message);
|
||||
try {
|
||||
return JSON.stringify(payload);
|
||||
} catch {
|
||||
return "OpenAI Images 请求失败";
|
||||
}
|
||||
}
|
||||
|
||||
async function parseJsonResponse(response: Response) {
|
||||
const rawText = await response.text();
|
||||
try {
|
||||
return JSON.parse(rawText);
|
||||
} catch {
|
||||
return rawText;
|
||||
}
|
||||
}
|
||||
|
||||
async function requestOpenAIImageGeneration(params: {
|
||||
baseURL: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
prompt: string;
|
||||
size: string;
|
||||
imageUrls: string[];
|
||||
quality?: string;
|
||||
background?: string;
|
||||
outputFormat?: string;
|
||||
}) {
|
||||
const { baseURL, apiKey, model, prompt, size, imageUrls, quality, background, outputFormat } = params;
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), DEFAULT_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
if (imageUrls.length > 0) {
|
||||
const endpoint = joinEndpoint(baseURL, "/images/edits");
|
||||
const formData = new FormData();
|
||||
formData.append("model", model);
|
||||
formData.append("prompt", prompt);
|
||||
formData.append("size", size);
|
||||
formData.append("response_format", "url");
|
||||
|
||||
if (quality) formData.append("quality", quality);
|
||||
if (background) formData.append("background", background);
|
||||
if (outputFormat) formData.append("output_format", outputFormat);
|
||||
|
||||
if (imageUrls.length === 1) {
|
||||
const image = await loadInputImage(imageUrls[0]);
|
||||
formData.append("image", new Blob([image.buffer], { type: image.mimeType }), image.filename);
|
||||
} else {
|
||||
for (const source of imageUrls) {
|
||||
const image = await loadInputImage(source);
|
||||
formData.append("image[]", new Blob([image.buffer], { type: image.mimeType }), image.filename);
|
||||
}
|
||||
}
|
||||
|
||||
const response = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: formData,
|
||||
signal: controller.signal,
|
||||
});
|
||||
const payload = await parseJsonResponse(response);
|
||||
if (!response.ok) {
|
||||
throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`);
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
const endpoint = joinEndpoint(baseURL, "/images/generations");
|
||||
const response = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
prompt,
|
||||
size,
|
||||
response_format: "url",
|
||||
...(quality ? { quality } : {}),
|
||||
...(background ? { background } : {}),
|
||||
...(outputFormat ? { output_format: outputFormat } : {}),
|
||||
}),
|
||||
signal: controller.signal,
|
||||
});
|
||||
const payload = await parseJsonResponse(response);
|
||||
if (!response.ok) {
|
||||
throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`);
|
||||
}
|
||||
return payload;
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
async function extractUploadedImages(payload: any): Promise<UploadedImage[]> {
|
||||
const data = Array.isArray(payload?.data) ? payload.data : [];
|
||||
const uploadedImages: UploadedImage[] = [];
|
||||
|
||||
for (const item of data) {
|
||||
if (item?.url && typeof item.url === "string") {
|
||||
uploadedImages.push(await downloadAndUploadImage(item.url));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (item?.b64_json && typeof item.b64_json === "string") {
|
||||
uploadedImages.push(await uploadBase64ToS3(item.b64_json, "image/png"));
|
||||
}
|
||||
}
|
||||
|
||||
return uploadedImages;
|
||||
}
|
||||
|
||||
function formatMarkdown(images: UploadedImage[], imageSize: string) {
|
||||
return images
|
||||
.map((image) => ``)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
export function createOpenAIImage({ apiKey, baseURL }: OpenAIImageConfig) {
|
||||
return {
|
||||
provider: "openai-image",
|
||||
chat: (modelId: string) => ({
|
||||
provider: "openai-image",
|
||||
doGenerate: async (options: {
|
||||
messages: Array<{ role: string; content: any }>;
|
||||
imageParams?: OpenAIImageParams;
|
||||
overrideModelId?: string;
|
||||
}) => {
|
||||
const { prompt, imageUrls } = extractPromptAndImages(options.messages);
|
||||
const activeModelId = options.overrideModelId || modelId;
|
||||
const aspectRatio =
|
||||
normalizeAspectRatio(options.imageParams?.aspectRatio) === "auto"
|
||||
? await inferAspectRatioFromImageUrl(imageUrls[0] || "")
|
||||
: normalizeAspectRatio(options.imageParams?.aspectRatio);
|
||||
const imageSize = normalizeImageSizeLabel(options.imageParams?.imageSize);
|
||||
const effectiveAspectRatio = aspectRatio || DEFAULT_ASPECT_RATIO;
|
||||
const size =
|
||||
options.imageParams?.size ||
|
||||
resolveOpenAISize(effectiveAspectRatio, imageSize);
|
||||
const quality = normalizeQuality(options.imageParams?.quality);
|
||||
const background = normalizeBackground(options.imageParams?.background);
|
||||
const outputFormat = normalizeOutputFormat(options.imageParams?.outputFormat);
|
||||
|
||||
console.log("[OpenAI Image] Submitting request", {
|
||||
model: activeModelId,
|
||||
size,
|
||||
aspectRatio: effectiveAspectRatio,
|
||||
imageCount: imageUrls.length,
|
||||
baseURL: normalizeBaseURL(baseURL),
|
||||
});
|
||||
|
||||
const payload = await requestOpenAIImageGeneration({
|
||||
baseURL,
|
||||
apiKey,
|
||||
model: activeModelId,
|
||||
prompt,
|
||||
size,
|
||||
imageUrls,
|
||||
quality,
|
||||
background,
|
||||
outputFormat,
|
||||
});
|
||||
|
||||
const uploadedImages = await extractUploadedImages(payload);
|
||||
if (uploadedImages.length === 0) {
|
||||
throw new Error("OpenAI Images 未返回可用图片");
|
||||
}
|
||||
|
||||
const text = formatMarkdown(uploadedImages, imageSize);
|
||||
return {
|
||||
text,
|
||||
finishReason: "stop" as const,
|
||||
usage: {
|
||||
promptTokens: 0,
|
||||
completionTokens: text.length,
|
||||
},
|
||||
};
|
||||
},
|
||||
doStream: async function* (options: {
|
||||
messages: Array<{ role: string; content: any }>;
|
||||
imageParams?: OpenAIImageParams;
|
||||
overrideModelId?: string;
|
||||
}) {
|
||||
const result = await this.doGenerate(options);
|
||||
yield {
|
||||
type: "text-delta",
|
||||
textDelta: result.text,
|
||||
};
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
@ -5,6 +5,7 @@ import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
|
||||
import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image";
|
||||
import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
|
||||
import { createFalImage } from "@/lib/ai/providers/fal-image";
|
||||
import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
|
||||
import { filterReachableRemoteUrls } from "@/lib/net/url-access";
|
||||
|
||||
export interface CanvasGenerateImageRequest {
|
||||
@ -34,6 +35,7 @@ const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([
|
||||
"alibaba-image",
|
||||
"ruxa-image",
|
||||
"fal-image",
|
||||
"openai-image",
|
||||
]);
|
||||
|
||||
function parseImagesFromMarkdown(markdown: string): ParsedImage[] {
|
||||
@ -130,7 +132,7 @@ export async function generateCanvasImage(
|
||||
}
|
||||
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(fallbackProvider.type)) {
|
||||
throw new Error(
|
||||
"当前图片模型类型暂不支持画布生成，请在后台选择 Gemini/阿里/RUXA/FAL 图片模型。"
|
||||
"当前图片模型类型暂不支持画布生成，请在后台选择 Gemini/阿里/RUXA/FAL/OpenAI 图片模型。"
|
||||
);
|
||||
}
|
||||
if (fallbackModel.model.id !== selectedModel.model.id) {
|
||||
@ -224,9 +226,17 @@ export async function generateCanvasImage(
|
||||
const modelInstance = falProvider.chat(selectedModel.model.modelId);
|
||||
const result = await modelInstance.doGenerate({ messages, imageParams });
|
||||
imageMarkdown = result.text || "";
|
||||
} else if (provider.type === "openai-image") {
|
||||
const openAIImageProvider = createOpenAIImage({
|
||||
apiKey: provider.apiKey,
|
||||
baseURL: provider.baseUrl,
|
||||
});
|
||||
const modelInstance = openAIImageProvider.chat(selectedModel.model.modelId);
|
||||
const result = await modelInstance.doGenerate({ messages, imageParams });
|
||||
imageMarkdown = result.text || "";
|
||||
} else {
|
||||
throw new Error(
|
||||
`不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/阿里/RUXA/FAL 图片模型`
|
||||
`不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/阿里/RUXA/FAL/OpenAI 图片模型`
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@ -28,6 +28,7 @@ const providerBaseUrlDefaults = [
|
||||
},
|
||||
{ type: "ruxa-image", baseUrl: "https://api.ruxa.ai" },
|
||||
{ type: "fal-image", baseUrl: "https://fal.run" },
|
||||
{ type: "openai-image", baseUrl: "https://api.openai.com/v1" },
|
||||
{ type: "fal-video", baseUrl: "https://fal.run" },
|
||||
{ type: "tripo-3d", baseUrl: "https://api.tripo3d.ai/v2/openapi" },
|
||||
] as const;
|
||||
@ -89,7 +90,7 @@ const runMigrate = async () => {
|
||||
ADD CONSTRAINT "AiProvider_type_check"
|
||||
CHECK (type IN (
|
||||
'alibaba', 'openai', 'custom', 'local', 'gemini',
|
||||
'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'ruxa-image', 'fal-image', 'fal-video', 'zhipu', 'tripo-3d'
|
||||
'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'ruxa-image', 'fal-image', 'openai-image', 'fal-video', 'zhipu', 'tripo-3d'
|
||||
));
|
||||
`);
|
||||
for (const item of providerBaseUrlDefaults) {
|
||||
|
||||
@ -2005,7 +2005,7 @@ export async function createAiProvider({
|
||||
updatedBy,
|
||||
}: {
|
||||
name: string;
|
||||
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d";
|
||||
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d";
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
isEnabled?: boolean;
|
||||
@ -2043,7 +2043,7 @@ export async function updateAiProvider({
|
||||
}: {
|
||||
id: string;
|
||||
name?: string;
|
||||
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d";
|
||||
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d";
|
||||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
isEnabled?: boolean;
|
||||
|
||||
@ -259,7 +259,7 @@ export type ToolConfig = InferSelectModel<typeof toolConfig>;
|
||||
export const aiProvider = pgTable("AiProvider", {
|
||||
id: uuid("id").primaryKey().notNull().defaultRandom(),
|
||||
name: varchar("name", { length: 128 }).notNull(), // 提供商名称，如 "阿里云通义千问"
|
||||
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "fal-video", "zhipu", "tripo-3d"] })
|
||||
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "openai-image", "fal-video", "zhipu", "tripo-3d"] })
|
||||
.notNull()
|
||||
.default("custom"), // 提供商类型
|
||||
baseUrl: text("baseUrl").notNull(), // API 基础 URL
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user