新增RUXA生图服务商并打通聊天/画布链路
This commit is contained in:
parent
84e22bcccb
commit
3a20c5a36d
@ -98,7 +98,7 @@ type ToolConfig = {
|
||||
type AiProvider = {
|
||||
id: string;
|
||||
name: string;
|
||||
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image";
|
||||
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image";
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
isEnabled: boolean;
|
||||
@ -275,6 +275,7 @@ const AVAILABLE_TOOLS = [
|
||||
const PROVIDER_TYPES = [
|
||||
{ value: "alibaba", label: "阿里云通义千问" },
|
||||
{ value: "alibaba-image", label: "阿里生图模型" }, // 🆕 千问图片生成/编辑
|
||||
{ value: "ruxa-image", label: "RUXA 生图模型" },
|
||||
{ value: "openai", label: "OpenAI 兼容" },
|
||||
{ value: "custom", label: "自定义 OpenAI 兼容" },
|
||||
{ value: "local", label: "本地模型" },
|
||||
@ -739,7 +740,7 @@ export function AdminSettingsClient() {
|
||||
const [providerForm, setProviderForm] = useState({
|
||||
id: "",
|
||||
name: "",
|
||||
type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image",
|
||||
type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image",
|
||||
baseUrl: "",
|
||||
apiKey: "",
|
||||
isEnabled: true,
|
||||
@ -2303,6 +2304,8 @@ export function AdminSettingsClient() {
|
||||
"volcano-vision": "bg-orange-500",
|
||||
"volcano-video": "bg-red-500",
|
||||
"alibaba": "bg-blue-500",
|
||||
"alibaba-image": "bg-amber-500",
|
||||
"ruxa-image": "bg-emerald-500",
|
||||
"openai": "bg-green-500",
|
||||
"qianwen-video": "bg-cyan-500",
|
||||
"custom": "bg-gray-500",
|
||||
@ -3607,7 +3610,16 @@ export function AdminSettingsClient() {
|
||||
<Label>类型</Label>
|
||||
<Select
|
||||
value={providerForm.type}
|
||||
onValueChange={(value) => setProviderForm({ ...providerForm, type: value as any })}
|
||||
onValueChange={(value) =>
|
||||
setProviderForm((prev) => ({
|
||||
...prev,
|
||||
type: value as any,
|
||||
baseUrl:
|
||||
!prev.baseUrl && value === "ruxa-image"
|
||||
? "https://api.ruxa.ai"
|
||||
: prev.baseUrl,
|
||||
}))
|
||||
}
|
||||
>
|
||||
<SelectTrigger><SelectValue /></SelectTrigger>
|
||||
<SelectContent>
|
||||
@ -3622,6 +3634,7 @@ export function AdminSettingsClient() {
|
||||
<Input
|
||||
value={providerForm.baseUrl}
|
||||
onChange={(e) => setProviderForm({ ...providerForm, baseUrl: e.target.value })}
|
||||
placeholder={providerForm.type === "ruxa-image" ? "https://api.ruxa.ai" : "https://api.openai.com/v1"}
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
@ -4883,7 +4896,7 @@ export function AdminSettingsClient() {
|
||||
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
|
||||
if (!provider) return false;
|
||||
// 支持阿里生图、Gemini、既梦等图片模型
|
||||
const imageProviderTypes = ["alibaba-image", "gemini-image", "gemini", "volcano-vision"];
|
||||
const imageProviderTypes = ["alibaba-image", "ruxa-image", "gemini-image", "gemini", "volcano-vision"];
|
||||
return imageProviderTypes.includes(provider.type);
|
||||
})
|
||||
.map((m: AiModel) => {
|
||||
@ -4898,7 +4911,7 @@ export function AdminSettingsClient() {
|
||||
{/* 如果没有找到生图模型,显示提示 */}
|
||||
{(modelsData?.models || []).filter((m: AiModel) => {
|
||||
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
|
||||
const imageProviderTypes = ["alibaba-image", "gemini-image", "gemini", "volcano-vision"];
|
||||
const imageProviderTypes = ["alibaba-image", "ruxa-image", "gemini-image", "gemini", "volcano-vision"];
|
||||
return provider && imageProviderTypes.includes(provider.type);
|
||||
}).length === 0 && (
|
||||
<SelectItem value="no-model" disabled>
|
||||
|
||||
@ -305,6 +305,9 @@ async function saveGeneratedImages(
|
||||
} else if (thumbnailUrl.includes("alibaba-image-thumb-")) {
|
||||
// 🆕 阿里生图模型
|
||||
originalUrl = thumbnailUrl.replace("alibaba-image-thumb-", "alibaba-image-orig-");
|
||||
} else if (thumbnailUrl.includes("ruxa-image-thumb-")) {
|
||||
// 🆕 RUXA 生图模型
|
||||
originalUrl = thumbnailUrl.replace("ruxa-image-thumb-", "ruxa-image-orig-");
|
||||
}
|
||||
}
|
||||
|
||||
@ -776,7 +779,10 @@ export async function POST(request: Request) {
|
||||
// ========== 图片生成配额检查 ==========
|
||||
const isImageGenerationModel =
|
||||
(languageModel as any).provider === "gemini-image" ||
|
||||
(languageModel as any).provider === "volcano-vision";
|
||||
(languageModel as any).provider === "volcano-vision" ||
|
||||
(languageModel as any).provider === "alibaba-image" ||
|
||||
(languageModel as any).provider === "ruxa-image" ||
|
||||
(languageModel as any).provider === "huanke-gemini";
|
||||
|
||||
if (isImageGenerationModel) {
|
||||
const quota = await checkUserImageQuota(session.user.id);
|
||||
@ -1156,8 +1162,9 @@ export async function POST(request: Request) {
|
||||
// ========== Gemini 处理结束 ==========
|
||||
|
||||
// ========== 阿里云千问图片生成独立处理 ==========
|
||||
if ((languageModel as any).provider === "alibaba-image" && typeof (languageModel as any).doGenerate === "function") {
|
||||
console.log("[Chat API] Using Alibaba Image generation");
|
||||
const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image"].includes((languageModel as any).provider);
|
||||
if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||||
console.log("[Chat API] Using image generation provider:", (languageModel as any).provider);
|
||||
console.log("[Chat API] Image params:", requestBody.imageParams);
|
||||
|
||||
// 🆕 获取模型显示名称,用于错误提示
|
||||
|
||||
@ -11,7 +11,7 @@ import { ChatSDKError } from "@/lib/errors";
|
||||
|
||||
const createProviderSchema = z.object({
|
||||
name: z.string().min(1).max(128),
|
||||
type: z.enum(["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "volcano-vision", "volcano-video", "alibaba-image"]),
|
||||
type: z.enum(["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image"]),
|
||||
baseUrl: z.string().min(1), // 改为简单的字符串验证,支持 localhost
|
||||
apiKey: z.string().min(1),
|
||||
isEnabled: z.boolean().optional(),
|
||||
@ -20,7 +20,7 @@ const createProviderSchema = z.object({
|
||||
const updateProviderSchema = z.object({
|
||||
id: z.string().uuid(),
|
||||
name: z.string().min(1).max(128).optional(),
|
||||
type: z.enum(["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "volcano-vision", "volcano-video", "alibaba-image"]).optional(),
|
||||
type: z.enum(["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image"]).optional(),
|
||||
baseUrl: z.string().min(1).optional(), // 改为简单的字符串验证,支持 localhost
|
||||
apiKey: z.string().min(1).optional(),
|
||||
isEnabled: z.boolean().optional(),
|
||||
@ -142,4 +142,3 @@ export async function DELETE(request: Request) {
|
||||
return new ChatSDKError("offline:admin").toResponse();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@ import { auth } from "@/app/(auth)/auth";
|
||||
import { getEnabledAiModelsWithProvider, getAiProviderById } from "@/lib/db/queries";
|
||||
import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
|
||||
import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image";
|
||||
import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
|
||||
|
||||
export const maxDuration = 120;
|
||||
|
||||
@ -15,6 +16,41 @@ interface GenerateImageRequest {
|
||||
referenceImage?: string; // 参考图片(用于图生图)
|
||||
}
|
||||
|
||||
interface ParsedImage {
|
||||
thumbnailUrl: string;
|
||||
originalUrl: string;
|
||||
}
|
||||
|
||||
function parseImagesFromMarkdown(markdown: string): ParsedImage[] {
|
||||
const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
|
||||
const matches = [...markdown.matchAll(imageRegex)];
|
||||
const parsed: ParsedImage[] = [];
|
||||
|
||||
for (const match of matches) {
|
||||
const altText = match[1] || "";
|
||||
const thumbnailUrl = match[2];
|
||||
let originalUrl = thumbnailUrl;
|
||||
|
||||
if (altText.includes("thumb|||")) {
|
||||
const content = altText.substring(altText.indexOf("thumb|||") + 8);
|
||||
const parts = content.split("|||");
|
||||
originalUrl = parts[0] || thumbnailUrl;
|
||||
} else if (altText.includes("thumb$$")) {
|
||||
const content = altText.substring(altText.indexOf("thumb$$") + 7);
|
||||
const parts = content.split("$$");
|
||||
originalUrl = parts[0] || thumbnailUrl;
|
||||
} else if (altText.includes("thumb|")) {
|
||||
const content = altText.substring(altText.indexOf("thumb|") + 6);
|
||||
const parts = content.split("|");
|
||||
originalUrl = parts[0] || thumbnailUrl;
|
||||
}
|
||||
|
||||
parsed.push({ thumbnailUrl, originalUrl });
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
const session = await auth();
|
||||
@ -63,35 +99,26 @@ export async function POST(request: Request) {
|
||||
);
|
||||
}
|
||||
|
||||
// 计算尺寸
|
||||
const sizeMap: Record<string, { width: number; height: number }> = {
|
||||
"1:1": { width: 1024, height: 1024 },
|
||||
"3:2": { width: 1536, height: 1024 },
|
||||
"2:3": { width: 1024, height: 1536 },
|
||||
"4:3": { width: 1365, height: 1024 },
|
||||
"3:4": { width: 1024, height: 1365 },
|
||||
"16:9": { width: 1820, height: 1024 },
|
||||
"9:16": { width: 1024, height: 1820 },
|
||||
};
|
||||
const size = sizeMap[aspectRatio] || sizeMap["1:1"];
|
||||
|
||||
// 构建消息
|
||||
const messages = [
|
||||
{
|
||||
role: "user" as const,
|
||||
content: prompt,
|
||||
}
|
||||
];
|
||||
const messages = [{
|
||||
role: "user" as const,
|
||||
content: referenceImage
|
||||
? [
|
||||
{ type: "text" as const, text: prompt },
|
||||
{ type: "image_url" as const, image_url: { url: referenceImage } },
|
||||
]
|
||||
: prompt,
|
||||
}];
|
||||
|
||||
// 图片参数
|
||||
const imageParams = {
|
||||
aspectRatio,
|
||||
imageSize: `${size.width}x${size.height}`,
|
||||
imageSize: "2K",
|
||||
imageCount,
|
||||
negativePrompt,
|
||||
};
|
||||
|
||||
let result;
|
||||
let imageMarkdown = "";
|
||||
|
||||
// 根据提供商类型调用不同的生成器
|
||||
if (provider.type === "gemini") {
|
||||
@ -100,23 +127,36 @@ export async function POST(request: Request) {
|
||||
baseURL: provider.baseUrl,
|
||||
});
|
||||
|
||||
result = await geminiProvider.doGenerate({
|
||||
const model = geminiProvider.chat(selectedModel.model.modelId);
|
||||
const result = await model.doGenerate({
|
||||
messages,
|
||||
modelId: selectedModel.model.modelId,
|
||||
imageParams,
|
||||
});
|
||||
imageMarkdown = result.text || "";
|
||||
} else if (provider.type === "alibaba-image") {
|
||||
const alibabaProvider = createAlibabaImage({
|
||||
apiKey: provider.apiKey,
|
||||
baseURL: provider.baseUrl,
|
||||
});
|
||||
|
||||
result = await alibabaProvider.doGenerate({
|
||||
const model = alibabaProvider.chat(selectedModel.model.modelId);
|
||||
const result = await model.doGenerate({
|
||||
messages,
|
||||
modelId: selectedModel.model.modelId,
|
||||
imageParams,
|
||||
referenceImage,
|
||||
});
|
||||
imageMarkdown = result.text || "";
|
||||
} else if (provider.type === "ruxa-image") {
|
||||
const ruxaProvider = createRuxaImage({
|
||||
apiKey: provider.apiKey,
|
||||
baseURL: provider.baseUrl,
|
||||
});
|
||||
|
||||
const model = ruxaProvider.chat(selectedModel.model.modelId);
|
||||
const result = await model.doGenerate({
|
||||
messages,
|
||||
imageParams,
|
||||
});
|
||||
imageMarkdown = result.text || "";
|
||||
} else {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "不支持的图片生成模型类型" },
|
||||
@ -124,11 +164,13 @@ export async function POST(request: Request) {
|
||||
);
|
||||
}
|
||||
|
||||
if (result.images && result.images.length > 0) {
|
||||
const parsedImages = parseImagesFromMarkdown(imageMarkdown);
|
||||
|
||||
if (parsedImages.length > 0) {
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
images: result.images,
|
||||
images: parsedImages,
|
||||
model: selectedModel.model.displayName,
|
||||
prompt,
|
||||
}
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
* 场景裂变 API
|
||||
*
|
||||
* 将一张产品图批量融合到多个场景图中
|
||||
* 支持多种生图模型:阿里生图、Gemini、既梦
|
||||
* 支持多种生图模型:阿里生图、RUXA、Gemini、既梦
|
||||
*/
|
||||
|
||||
import { NextResponse } from "next/server";
|
||||
@ -16,6 +16,7 @@ import sharp from "sharp";
|
||||
import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image";
|
||||
import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
|
||||
import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision";
|
||||
import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
|
||||
|
||||
// 请求体 Schema
|
||||
const sceneFusionRequestSchema = z.object({
|
||||
@ -364,6 +365,9 @@ async function processSceneWithProvider(
|
||||
case "alibaba-image":
|
||||
return processWithAlibabaImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
|
||||
|
||||
case "ruxa-image":
|
||||
return processWithRuxaImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
|
||||
|
||||
case "gemini":
|
||||
return processWithGemini(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
|
||||
|
||||
@ -455,6 +459,49 @@ async function processWithGemini(
|
||||
return { thumbnail, original, prompt };
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用 RUXA 模型处理
|
||||
*/
|
||||
async function processWithRuxaImage(
|
||||
productImageUrl: string,
|
||||
sceneImageUrl: string,
|
||||
prompt: string,
|
||||
provider: { baseUrl: string; apiKey: string },
|
||||
modelId: string,
|
||||
aspectRatio?: string,
|
||||
imageSize?: string
|
||||
): Promise<{ thumbnail: string; original: string; prompt: string }> {
|
||||
console.log(`[SceneFusion][RUXA] Processing with model: ${modelId}`);
|
||||
|
||||
const ruxaProvider = createRuxaImage({
|
||||
baseURL: provider.baseUrl,
|
||||
apiKey: provider.apiKey,
|
||||
});
|
||||
|
||||
const messages = [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "image_url", image_url: { url: productImageUrl } },
|
||||
{ type: "image_url", image_url: { url: sceneImageUrl } },
|
||||
{ type: "text", text: prompt },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const result = await ruxaProvider.chat(modelId).doGenerate({
|
||||
messages,
|
||||
imageParams: {
|
||||
aspectRatio: aspectRatio || "auto",
|
||||
imageSize: imageSize || "2K",
|
||||
imageCount: 1,
|
||||
},
|
||||
});
|
||||
|
||||
const { thumbnail, original } = parseImageFromMarkdown(result.text);
|
||||
return { thumbnail, original, prompt };
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用既梦/火山模型处理
|
||||
*/
|
||||
@ -636,8 +683,8 @@ export async function POST(request: Request) {
|
||||
|
||||
// 如果没有配置或配置的模型不可用,使用默认的阿里生图模型
|
||||
if (!selectedModel) {
|
||||
// 按优先级查找:alibaba-image > gemini > volcano-vision
|
||||
const imageProviderTypes = ["alibaba-image", "gemini", "volcano-vision"];
|
||||
// 按优先级查找:alibaba-image > ruxa-image > gemini > volcano-vision
|
||||
const imageProviderTypes = ["alibaba-image", "ruxa-image", "gemini", "volcano-vision"];
|
||||
for (const providerType of imageProviderTypes) {
|
||||
selectedModel = allModels.find(
|
||||
(m) => m.provider.type === providerType && m.model.isEnabled
|
||||
|
||||
@ -493,7 +493,7 @@ export function Chat({
|
||||
else if (model?.providerType === "volcano-vision") modelType = "volcano-vision";
|
||||
else if (model?.providerType === "volcano-video") modelType = "volcano-video";
|
||||
else if (model?.providerType === "qianwen-video") modelType = "qianwen-video";
|
||||
else if (model?.providerType === "alibaba-image") modelType = "alibaba-image";
|
||||
else if (model?.providerType === "alibaba-image" || model?.providerType === "ruxa-image") modelType = "alibaba-image";
|
||||
|
||||
console.log(`[Chat] 🚀 记录生成任务: modelType=${modelType}, modelId=${currentModelId}`);
|
||||
startGeneration(currentModelId, modelType);
|
||||
|
||||
@ -106,6 +106,8 @@ export const ImageGenerationLoading = memo(({
|
||||
return "purple";
|
||||
case "alibaba-image":
|
||||
return "orange";
|
||||
case "ruxa-image":
|
||||
return "green";
|
||||
case "volcano-vision":
|
||||
return "blue";
|
||||
default:
|
||||
@ -119,7 +121,8 @@ export const ImageGenerationLoading = memo(({
|
||||
const count = Math.min(Math.max(imageCount, 1), maxImages);
|
||||
|
||||
// 获取显示名称
|
||||
const displayName = modelName || (providerType === "alibaba-image" ? "千问图片编辑" :
|
||||
const displayName = modelName || (providerType === "alibaba-image" ? "千问图片编辑" :
|
||||
providerType === "ruxa-image" ? "RUXA 图片编辑" :
|
||||
providerType === "volcano-vision" ? "即梦4.5" : "AI");
|
||||
|
||||
return (
|
||||
@ -256,4 +259,3 @@ export const ImageGenerationLoading = memo(({
|
||||
ImageGenerationLoading.displayName = "ImageGenerationLoading";
|
||||
|
||||
export default ImageGenerationLoading;
|
||||
|
||||
|
||||
@ -37,7 +37,7 @@ interface ImageParamsBarProps {
|
||||
initialParams?: Partial<ImageGenerationParams>;
|
||||
onParamsChange?: (params: ImageGenerationParams) => void;
|
||||
inline?: boolean; // 内联模式:只显示参数按钮,不显示外层容器
|
||||
providerType?: string; // 模型类型:gemini, volcano-vision, alibaba-image 等
|
||||
providerType?: string; // 模型类型:gemini, volcano-vision, alibaba-image, ruxa-image 等
|
||||
// 🆕 图片生成模型特性配置(从模型配置中读取)
|
||||
imageGenConfig?: {
|
||||
maxImages?: number;
|
||||
@ -99,6 +99,10 @@ export function ImageParamsBar({
|
||||
// 阿里生图:1~6 张(qwen-image-edit-plus)
|
||||
return [1, 2, 3, 4, 5, 6];
|
||||
}
|
||||
if (providerType === "ruxa-image") {
|
||||
// RUXA 当前按单图任务处理
|
||||
return [1];
|
||||
}
|
||||
if (providerType === "volcano-vision") {
|
||||
// 即梦4.5(豆包):1~3 张
|
||||
return [1, 2, 3];
|
||||
@ -271,4 +275,3 @@ export function ImageParamsBar({
|
||||
// 非内联模式(暂时不实现,因为只需要内联模式)
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@ -113,7 +113,7 @@ function PureMessages({
|
||||
|
||||
// 🆕 判断是否是阿里生图模型(或者智能自动模式默认使用阿里生图占位图)
|
||||
const isAlibabaImageModel = modelsData?.models?.some(
|
||||
(m: DynamicModel) => m.id === selectedModelId && m.providerType === "alibaba-image"
|
||||
(m: DynamicModel) => m.id === selectedModelId && (m.providerType === "alibaba-image" || m.providerType === "ruxa-image")
|
||||
) ?? false;
|
||||
|
||||
// 🆕 获取当前模型的显示名称
|
||||
|
||||
@ -18,6 +18,7 @@ import { createHuankeGemini } from "./providers/huanke-gemini";
|
||||
import { createQianwenVideo } from "./providers/qianwen-video";
|
||||
import { createVolcanoVision } from "./providers/volcano-vision";
|
||||
import { createAlibabaImage } from "./providers/alibaba-image";
|
||||
import { createRuxaImage } from "./providers/ruxa-image";
|
||||
|
||||
// 缓存已创建的 provider 实例
|
||||
const providerCache = new Map<string, ReturnType<typeof createOpenAI>>();
|
||||
@ -132,6 +133,16 @@ function getOrCreateProvider(provider: Pick<AiProvider, "id" | "type" | "baseUrl
|
||||
return alibabaImageProvider as any;
|
||||
}
|
||||
|
||||
case "ruxa-image": {
|
||||
// RUXA 图片编辑服务
|
||||
const ruxaImageProvider = createRuxaImage({
|
||||
baseURL: provider.baseUrl,
|
||||
apiKey: provider.apiKey,
|
||||
});
|
||||
console.log(`[Provider] Created Ruxa Image generation provider: ${provider.baseUrl}`);
|
||||
return ruxaImageProvider as any;
|
||||
}
|
||||
|
||||
case "custom":
|
||||
default: {
|
||||
// OpenAI 兼容的第三方 API(如 GPT-5 Nano、Claude via OpenAI proxy 等)
|
||||
@ -214,6 +225,11 @@ export async function getDynamicModel(
|
||||
console.log("[getDynamicModel] Getting Alibaba Image chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(model.modelId);
|
||||
console.log("[getDynamicModel] Alibaba Image model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "ruxa-image") {
|
||||
// RUXA 图片编辑生成使用 chat() 方法
|
||||
console.log("[getDynamicModel] Getting Ruxa Image chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(model.modelId);
|
||||
console.log("[getDynamicModel] Ruxa Image model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "huanke-gemini") {
|
||||
// 环科 Gemini 原生格式使用 chat() 方法
|
||||
console.log("[getDynamicModel] Getting Huanke Gemini chat model..."); // 调试日志
|
||||
@ -322,4 +338,3 @@ export async function getSystemModel(
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -30,7 +30,7 @@ const INTENT_MODEL_MAP: Record<
|
||||
> = {
|
||||
// 图片编辑:优先 alibaba-image(灵魂4.5,支持提示词扩展)
|
||||
image_edit: {
|
||||
providerTypes: ["alibaba-image", "gemini", "huanke-gemini"],
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "gemini", "huanke-gemini"],
|
||||
requireVision: true,
|
||||
requireImageGen: true,
|
||||
description: "图片编辑",
|
||||
@ -45,7 +45,7 @@ const INTENT_MODEL_MAP: Record<
|
||||
|
||||
// 图片生成:优先灵魂4.5
|
||||
image_gen: {
|
||||
providerTypes: ["alibaba-image", "gemini", "huanke-gemini", "volcano-vision"],
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "gemini", "huanke-gemini", "volcano-vision"],
|
||||
requireImageGen: true,
|
||||
description: "图片生成",
|
||||
},
|
||||
@ -73,25 +73,25 @@ const SUB_INTENT_OVERRIDES: Record<
|
||||
> = {
|
||||
// 抠图:alibaba-image 效果最好
|
||||
remove_background: {
|
||||
providerTypes: ["alibaba-image", "gemini"],
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "gemini"],
|
||||
reason: "抠图首选灵魂4.5",
|
||||
},
|
||||
|
||||
// 换背景:alibaba-image 支持场景生成
|
||||
replace_background: {
|
||||
providerTypes: ["alibaba-image", "gemini"],
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "gemini"],
|
||||
reason: "换背景首选灵魂4.5",
|
||||
},
|
||||
|
||||
// 图片增强
|
||||
enhance: {
|
||||
providerTypes: ["alibaba-image", "gemini", "volcano-vision"],
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "gemini", "volcano-vision"],
|
||||
reason: "增强首选灵魂4.5",
|
||||
},
|
||||
|
||||
// 风格迁移:gemini 可能更有创意
|
||||
style_transfer: {
|
||||
providerTypes: ["gemini", "alibaba-image"],
|
||||
providerTypes: ["gemini", "alibaba-image", "ruxa-image"],
|
||||
reason: "风格迁移优选Gemini",
|
||||
},
|
||||
};
|
||||
|
||||
378
lib/ai/providers/ruxa-image.ts
Normal file
378
lib/ai/providers/ruxa-image.ts
Normal file
@ -0,0 +1,378 @@
|
||||
import "server-only";
|
||||
|
||||
import sharp from "sharp";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
|
||||
interface RuxaImageConfig {
|
||||
apiKey: string;
|
||||
baseURL: string;
|
||||
}
|
||||
|
||||
interface RuxaImageParams {
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
imageCount?: number;
|
||||
outputFormat?: "png" | "jpg" | "jpeg" | "webp";
|
||||
}
|
||||
|
||||
interface UploadedImage {
|
||||
thumbnail: string;
|
||||
original: string;
|
||||
}
|
||||
|
||||
const DEFAULT_ASPECT_RATIO = "1:1";
|
||||
const DEFAULT_RESOLUTION = "1K";
|
||||
const DEFAULT_OUTPUT_FORMAT = "png";
|
||||
const POLL_INTERVAL_MS = 2000;
|
||||
const POLL_MAX_ATTEMPTS = 60;
|
||||
|
||||
function normalizeUrlForCompare(url: string) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
parsed.search = "";
|
||||
parsed.hash = "";
|
||||
return parsed.toString();
|
||||
} catch {
|
||||
return url.split("?")[0]?.split("#")[0] || url;
|
||||
}
|
||||
}
|
||||
|
||||
function filterSourceImages(urls: string[], sourceUrls: string[]) {
|
||||
if (sourceUrls.length === 0) return urls;
|
||||
const sourceSet = new Set(sourceUrls.map((url) => normalizeUrlForCompare(url)));
|
||||
return urls.filter((url) => !sourceSet.has(normalizeUrlForCompare(url)));
|
||||
}
|
||||
|
||||
function joinApiPath(baseURL: string, path: string) {
|
||||
const cleanBase = baseURL.replace(/\/+$/, "");
|
||||
const cleanPath = path.startsWith("/") ? path : `/${path}`;
|
||||
return `${cleanBase}${cleanPath}`;
|
||||
}
|
||||
|
||||
function normalizeAspectRatio(aspectRatio?: string) {
|
||||
if (!aspectRatio || aspectRatio === "auto") return DEFAULT_ASPECT_RATIO;
|
||||
return aspectRatio;
|
||||
}
|
||||
|
||||
function normalizeResolution(imageSize?: string) {
|
||||
if (!imageSize) return DEFAULT_RESOLUTION;
|
||||
const normalized = imageSize.toUpperCase();
|
||||
if (["1K", "2K", "4K"].includes(normalized)) return normalized;
|
||||
return DEFAULT_RESOLUTION;
|
||||
}
|
||||
|
||||
function extractTaskId(payload: any): string | undefined {
|
||||
return (
|
||||
payload?.taskId ||
|
||||
payload?.task_id ||
|
||||
payload?.id ||
|
||||
payload?.data?.taskId ||
|
||||
payload?.data?.task_id ||
|
||||
payload?.data?.id
|
||||
);
|
||||
}
|
||||
|
||||
function extractStatus(payload: any): string | undefined {
|
||||
const raw =
|
||||
payload?.status ||
|
||||
payload?.taskStatus ||
|
||||
payload?.task_status ||
|
||||
payload?.state ||
|
||||
payload?.data?.status ||
|
||||
payload?.data?.taskStatus ||
|
||||
payload?.data?.task_status ||
|
||||
payload?.data?.state;
|
||||
if (typeof raw !== "string") return undefined;
|
||||
return raw.toLowerCase();
|
||||
}
|
||||
|
||||
function extractErrorMessage(payload: any): string {
|
||||
return (
|
||||
payload?.message ||
|
||||
payload?.error ||
|
||||
payload?.errorMessage ||
|
||||
payload?.data?.message ||
|
||||
payload?.data?.error ||
|
||||
"RUXA 任务执行失败"
|
||||
);
|
||||
}
|
||||
|
||||
function extractPromptAndImages(messages: Array<{ role: string; content: any }>) {
|
||||
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
|
||||
if (!lastUserMessage) {
|
||||
throw new Error("No user message found");
|
||||
}
|
||||
|
||||
let prompt = "";
|
||||
const imageUrls: string[] = [];
|
||||
|
||||
const pushImageUrl = (url?: string) => {
|
||||
if (!url || !url.startsWith("http")) return;
|
||||
if (!imageUrls.includes(url)) {
|
||||
imageUrls.push(url);
|
||||
}
|
||||
};
|
||||
|
||||
if (typeof lastUserMessage.content === "string") {
|
||||
prompt = lastUserMessage.content.trim();
|
||||
} else if (Array.isArray(lastUserMessage.content)) {
|
||||
for (const part of lastUserMessage.content) {
|
||||
if (part?.type === "text" && typeof part.text === "string") {
|
||||
prompt = `${prompt} ${part.text}`.trim();
|
||||
} else if (part?.type === "image_url" && part.image_url?.url) {
|
||||
pushImageUrl(part.image_url.url);
|
||||
} else if (part?.type === "file") {
|
||||
pushImageUrl(part.url || part.data);
|
||||
} else if (part?.type === "image") {
|
||||
pushImageUrl(part.image || part.url);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!prompt) {
|
||||
throw new Error("No prompt found in user message");
|
||||
}
|
||||
|
||||
if (imageUrls.length === 0) {
|
||||
throw new Error("RUXA 图片编辑模型需要至少 1 张输入图片");
|
||||
}
|
||||
|
||||
return { prompt, imageUrls };
|
||||
}
|
||||
|
||||
function collectImageUrls(payload: any): string[] {
|
||||
const urls = new Set<string>();
|
||||
|
||||
const isImageLikeKey = (key: string) =>
|
||||
/image|img|url|output|result|artifact|file/i.test(key);
|
||||
|
||||
const isLikelyImageUrl = (value: string) =>
|
||||
/^https?:\/\//i.test(value) &&
|
||||
(/\.(png|jpg|jpeg|webp|gif)(\?|$)/i.test(value) || value.includes("image"));
|
||||
|
||||
const walk = (value: any, parentKey = "") => {
|
||||
if (typeof value === "string") {
|
||||
if (isLikelyImageUrl(value) || (isImageLikeKey(parentKey) && /^https?:\/\//i.test(value))) {
|
||||
urls.add(value);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) {
|
||||
walk(item, parentKey);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (value && typeof value === "object") {
|
||||
for (const [key, nestedValue] of Object.entries(value)) {
|
||||
walk(nestedValue, key);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
walk(payload);
|
||||
return Array.from(urls);
|
||||
}
|
||||
|
||||
function detectImageExtension(imageUrl: string, contentType?: string | null) {
|
||||
const type = (contentType || "").toLowerCase();
|
||||
if (type.includes("jpeg") || type.includes("jpg")) return "jpg";
|
||||
if (type.includes("webp")) return "webp";
|
||||
if (type.includes("png")) return "png";
|
||||
|
||||
const cleanUrl = imageUrl.split("?")[0]?.toLowerCase() || "";
|
||||
if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "jpg";
|
||||
if (cleanUrl.endsWith(".webp")) return "webp";
|
||||
if (cleanUrl.endsWith(".png")) return "png";
|
||||
return "png";
|
||||
}
|
||||
|
||||
async function createThumbnail(buffer: Buffer): Promise<Buffer> {
|
||||
try {
|
||||
return await sharp(buffer)
|
||||
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
|
||||
.jpeg({ quality: 80 })
|
||||
.toBuffer();
|
||||
} catch (error) {
|
||||
console.error("[Ruxa Image] Failed to create thumbnail:", error);
|
||||
return buffer;
|
||||
}
|
||||
}
|
||||
|
||||
function generateFilename(prefix: string, extension: string) {
|
||||
const ts = Date.now();
|
||||
const rand = Math.random().toString(36).slice(2, 8);
|
||||
return `${prefix}-${ts}-${rand}.${extension}`;
|
||||
}
|
||||
|
||||
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
||||
const response = await fetch(imageUrl);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download image: ${response.status}`);
|
||||
}
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
const buffer = Buffer.from(arrayBuffer);
|
||||
const thumbnailBuffer = await createThumbnail(buffer);
|
||||
|
||||
const originalExt = detectImageExtension(imageUrl, response.headers.get("content-type"));
|
||||
const originalFilename = generateFilename("ruxa-image-orig", originalExt);
|
||||
const originalArrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer);
|
||||
|
||||
const thumbFilename = generateFilename("ruxa-image-thumb", "jpg");
|
||||
const thumbArrayBuffer = thumbnailBuffer.buffer.slice(
|
||||
thumbnailBuffer.byteOffset,
|
||||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const thumbUploaded = await uploadFile(thumbFilename, thumbArrayBuffer);
|
||||
|
||||
return {
|
||||
thumbnail: thumbUploaded.url,
|
||||
original: originalUploaded.url,
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForTaskResult(baseURL: string, apiKey: string, taskId: string, sourceUrls: string[]) {
|
||||
const endpointCandidates = [
|
||||
(id: string) => joinApiPath(baseURL, `/api/v1/tasks/query/${id}`),
|
||||
(id: string) => joinApiPath(baseURL, `/api/v1/tasks/${id}`),
|
||||
(id: string) => `${joinApiPath(baseURL, "/api/v1/tasks/query")}?taskId=${encodeURIComponent(id)}`,
|
||||
];
|
||||
|
||||
for (let attempt = 1; attempt <= POLL_MAX_ATTEMPTS; attempt++) {
|
||||
for (const endpointBuilder of endpointCandidates) {
|
||||
try {
|
||||
const endpoint = endpointBuilder(taskId);
|
||||
const response = await fetch(endpoint, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const payload = await response.json();
|
||||
const imageUrls = filterSourceImages(collectImageUrls(payload), sourceUrls);
|
||||
const status = extractStatus(payload);
|
||||
|
||||
if (imageUrls.length > 0) {
|
||||
return imageUrls;
|
||||
}
|
||||
|
||||
if (status && ["failed", "error", "cancelled", "canceled"].includes(status)) {
|
||||
throw new Error(extractErrorMessage(payload));
|
||||
}
|
||||
|
||||
if (status && ["success", "succeeded", "completed", "done", "finished"].includes(status)) {
|
||||
throw new Error("任务已完成但未返回图片地址");
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message !== "任务已完成但未返回图片地址") {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
|
||||
}
|
||||
|
||||
throw new Error("RUXA 任务超时,请稍后重试");
|
||||
}
|
||||
|
||||
function formatMarkdown(images: UploadedImage[], resolution: string) {
|
||||
return images
|
||||
.map((image) => ``)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
export function createRuxaImage({ apiKey, baseURL }: RuxaImageConfig) {
|
||||
return {
|
||||
provider: "ruxa-image",
|
||||
chat: (modelId: string) => ({
|
||||
provider: "ruxa-image",
|
||||
doGenerate: async (options: {
|
||||
messages: Array<{ role: string; content: any }>;
|
||||
imageParams?: RuxaImageParams;
|
||||
overrideModelId?: string;
|
||||
}) => {
|
||||
const { prompt, imageUrls } = extractPromptAndImages(options.messages);
|
||||
const aspectRatio = normalizeAspectRatio(options.imageParams?.aspectRatio);
|
||||
const resolution = normalizeResolution(options.imageParams?.imageSize);
|
||||
const outputFormat = options.imageParams?.outputFormat || DEFAULT_OUTPUT_FORMAT;
|
||||
const activeModelId = options.overrideModelId || modelId;
|
||||
|
||||
const createEndpoint = joinApiPath(baseURL, "/api/v1/tasks/create");
|
||||
const createResponse = await fetch(createEndpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
input: {
|
||||
prompt,
|
||||
image_input: imageUrls,
|
||||
aspect_ratio: aspectRatio,
|
||||
resolution,
|
||||
output_format: outputFormat,
|
||||
},
|
||||
model: activeModelId,
|
||||
}),
|
||||
});
|
||||
|
||||
const createPayload = await createResponse.json();
|
||||
if (!createResponse.ok) {
|
||||
throw new Error(extractErrorMessage(createPayload));
|
||||
}
|
||||
|
||||
let outputUrls = filterSourceImages(collectImageUrls(createPayload), imageUrls);
|
||||
if (outputUrls.length === 0) {
|
||||
const taskId = extractTaskId(createPayload);
|
||||
if (!taskId) {
|
||||
throw new Error("RUXA 创建任务成功,但未返回 taskId");
|
||||
}
|
||||
outputUrls = await waitForTaskResult(baseURL, apiKey, taskId, imageUrls);
|
||||
}
|
||||
|
||||
const uploadedImages: UploadedImage[] = [];
|
||||
for (const outputUrl of outputUrls.slice(0, options.imageParams?.imageCount || 1)) {
|
||||
uploadedImages.push(await downloadAndUploadImage(outputUrl));
|
||||
}
|
||||
|
||||
if (uploadedImages.length === 0) {
|
||||
throw new Error("RUXA 未返回可用图片");
|
||||
}
|
||||
|
||||
const text = formatMarkdown(uploadedImages, resolution);
|
||||
return {
|
||||
text,
|
||||
finishReason: "stop" as const,
|
||||
usage: {
|
||||
promptTokens: 0,
|
||||
completionTokens: text.length,
|
||||
},
|
||||
};
|
||||
},
|
||||
doStream: async function* (options: {
|
||||
messages: Array<{ role: string; content: any }>;
|
||||
imageParams?: RuxaImageParams;
|
||||
overrideModelId?: string;
|
||||
}) {
|
||||
const result = await this.doGenerate(options);
|
||||
yield {
|
||||
type: "text-delta",
|
||||
textDelta: result.text,
|
||||
};
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
@ -1,4 +1,5 @@
|
||||
import { config } from "dotenv";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import { migrate } from "drizzle-orm/postgres-js/migrator";
|
||||
import postgres from "postgres";
|
||||
@ -19,6 +20,17 @@ const runMigrate = async () => {
|
||||
|
||||
const start = Date.now();
|
||||
await migrate(db, { migrationsFolder: "./lib/db/migrations" });
|
||||
await db.execute(sql`
|
||||
ALTER TABLE "AiProvider" DROP CONSTRAINT IF EXISTS "AiProvider_type_check";
|
||||
`);
|
||||
await db.execute(sql`
|
||||
ALTER TABLE "AiProvider"
|
||||
ADD CONSTRAINT "AiProvider_type_check"
|
||||
CHECK (type IN (
|
||||
'alibaba', 'openai', 'custom', 'local', 'gemini',
|
||||
'huanke-gemini', 'qianwen-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'ruxa-image'
|
||||
));
|
||||
`);
|
||||
const end = Date.now();
|
||||
|
||||
console.log("✅ Migrations completed in", end - start, "ms");
|
||||
|
||||
@ -941,7 +941,7 @@ export async function createAiProvider({
|
||||
updatedBy,
|
||||
}: {
|
||||
name: string;
|
||||
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image";
|
||||
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image";
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
isEnabled?: boolean;
|
||||
@ -979,7 +979,7 @@ export async function updateAiProvider({
|
||||
}: {
|
||||
id: string;
|
||||
name?: string;
|
||||
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image";
|
||||
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image";
|
||||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
isEnabled?: boolean;
|
||||
|
||||
@ -215,7 +215,7 @@ export type ToolConfig = InferSelectModel<typeof toolConfig>;
|
||||
export const aiProvider = pgTable("AiProvider", {
|
||||
id: uuid("id").primaryKey().notNull().defaultRandom(),
|
||||
name: varchar("name", { length: 128 }).notNull(), // 提供商名称,如 "阿里云通义千问"
|
||||
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "volcano-vision", "volcano-video", "alibaba-image"] })
|
||||
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image"] })
|
||||
.notNull()
|
||||
.default("custom"), // 提供商类型
|
||||
baseUrl: text("baseUrl").notNull(), // API 基础 URL
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user