新增RUXA生图服务商并打通聊天/画布链路
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled

This commit is contained in:
Your Name 2026-02-07 20:09:31 +08:00
parent 84e22bcccb
commit 3a20c5a36d
15 changed files with 575 additions and 57 deletions

View File

@ -98,7 +98,7 @@ type ToolConfig = {
type AiProvider = {
id: string;
name: string;
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image";
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image";
baseUrl: string;
apiKey: string;
isEnabled: boolean;
@ -275,6 +275,7 @@ const AVAILABLE_TOOLS = [
const PROVIDER_TYPES = [
{ value: "alibaba", label: "阿里云通义千问" },
{ value: "alibaba-image", label: "阿里生图模型" }, // 🆕 千问图片生成/编辑
{ value: "ruxa-image", label: "RUXA 生图模型" },
{ value: "openai", label: "OpenAI 兼容" },
{ value: "custom", label: "自定义 OpenAI 兼容" },
{ value: "local", label: "本地模型" },
@ -739,7 +740,7 @@ export function AdminSettingsClient() {
const [providerForm, setProviderForm] = useState({
id: "",
name: "",
type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image",
type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image",
baseUrl: "",
apiKey: "",
isEnabled: true,
@ -2303,6 +2304,8 @@ export function AdminSettingsClient() {
"volcano-vision": "bg-orange-500",
"volcano-video": "bg-red-500",
"alibaba": "bg-blue-500",
"alibaba-image": "bg-amber-500",
"ruxa-image": "bg-emerald-500",
"openai": "bg-green-500",
"qianwen-video": "bg-cyan-500",
"custom": "bg-gray-500",
@ -3607,7 +3610,16 @@ export function AdminSettingsClient() {
<Label>类型</Label>
<Select
value={providerForm.type}
onValueChange={(value) => setProviderForm({ ...providerForm, type: value as any })}
onValueChange={(value) =>
setProviderForm((prev) => ({
...prev,
type: value as any,
baseUrl:
!prev.baseUrl && value === "ruxa-image"
? "https://api.ruxa.ai"
: prev.baseUrl,
}))
}
>
<SelectTrigger><SelectValue /></SelectTrigger>
<SelectContent>
@ -3622,6 +3634,7 @@ export function AdminSettingsClient() {
<Input
value={providerForm.baseUrl}
onChange={(e) => setProviderForm({ ...providerForm, baseUrl: e.target.value })}
placeholder={providerForm.type === "ruxa-image" ? "https://api.ruxa.ai" : "https://api.openai.com/v1"}
/>
</div>
<div className="space-y-2">
@ -4883,7 +4896,7 @@ export function AdminSettingsClient() {
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
if (!provider) return false;
// 支持阿里生图、Gemini、既梦等图片模型
const imageProviderTypes = ["alibaba-image", "gemini-image", "gemini", "volcano-vision"];
const imageProviderTypes = ["alibaba-image", "ruxa-image", "gemini-image", "gemini", "volcano-vision"];
return imageProviderTypes.includes(provider.type);
})
.map((m: AiModel) => {
@ -4898,7 +4911,7 @@ export function AdminSettingsClient() {
{/* 如果没有找到生图模型,显示提示 */}
{(modelsData?.models || []).filter((m: AiModel) => {
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
const imageProviderTypes = ["alibaba-image", "gemini-image", "gemini", "volcano-vision"];
const imageProviderTypes = ["alibaba-image", "ruxa-image", "gemini-image", "gemini", "volcano-vision"];
return provider && imageProviderTypes.includes(provider.type);
}).length === 0 && (
<SelectItem value="no-model" disabled>

View File

@ -305,6 +305,9 @@ async function saveGeneratedImages(
} else if (thumbnailUrl.includes("alibaba-image-thumb-")) {
// 🆕 阿里生图模型
originalUrl = thumbnailUrl.replace("alibaba-image-thumb-", "alibaba-image-orig-");
} else if (thumbnailUrl.includes("ruxa-image-thumb-")) {
// 🆕 RUXA 生图模型
originalUrl = thumbnailUrl.replace("ruxa-image-thumb-", "ruxa-image-orig-");
}
}
@ -776,7 +779,10 @@ export async function POST(request: Request) {
// ========== 图片生成配额检查 ==========
const isImageGenerationModel =
(languageModel as any).provider === "gemini-image" ||
(languageModel as any).provider === "volcano-vision";
(languageModel as any).provider === "volcano-vision" ||
(languageModel as any).provider === "alibaba-image" ||
(languageModel as any).provider === "ruxa-image" ||
(languageModel as any).provider === "huanke-gemini";
if (isImageGenerationModel) {
const quota = await checkUserImageQuota(session.user.id);
@ -1156,8 +1162,9 @@ export async function POST(request: Request) {
// ========== Gemini 处理结束 ==========
// ========== 阿里云千问图片生成独立处理 ==========
if ((languageModel as any).provider === "alibaba-image" && typeof (languageModel as any).doGenerate === "function") {
console.log("[Chat API] Using Alibaba Image generation");
const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image"].includes((languageModel as any).provider);
if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") {
console.log("[Chat API] Using image generation provider:", (languageModel as any).provider);
console.log("[Chat API] Image params:", requestBody.imageParams);
// 🆕 获取模型显示名称,用于错误提示

View File

@ -11,7 +11,7 @@ import { ChatSDKError } from "@/lib/errors";
const createProviderSchema = z.object({
name: z.string().min(1).max(128),
type: z.enum(["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "volcano-vision", "volcano-video", "alibaba-image"]),
type: z.enum(["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image"]),
baseUrl: z.string().min(1), // 改为简单的字符串验证,支持 localhost
apiKey: z.string().min(1),
isEnabled: z.boolean().optional(),
@ -20,7 +20,7 @@ const createProviderSchema = z.object({
const updateProviderSchema = z.object({
id: z.string().uuid(),
name: z.string().min(1).max(128).optional(),
type: z.enum(["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "volcano-vision", "volcano-video", "alibaba-image"]).optional(),
type: z.enum(["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image"]).optional(),
baseUrl: z.string().min(1).optional(), // 改为简单的字符串验证,支持 localhost
apiKey: z.string().min(1).optional(),
isEnabled: z.boolean().optional(),
@ -142,4 +142,3 @@ export async function DELETE(request: Request) {
return new ChatSDKError("offline:admin").toResponse();
}
}

View File

@ -3,6 +3,7 @@ import { auth } from "@/app/(auth)/auth";
import { getEnabledAiModelsWithProvider, getAiProviderById } from "@/lib/db/queries";
import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image";
import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
export const maxDuration = 120;
@ -15,6 +16,41 @@ interface GenerateImageRequest {
referenceImage?: string; // 参考图片(用于图生图)
}
interface ParsedImage {
thumbnailUrl: string;
originalUrl: string;
}
function parseImagesFromMarkdown(markdown: string): ParsedImage[] {
const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
const matches = [...markdown.matchAll(imageRegex)];
const parsed: ParsedImage[] = [];
for (const match of matches) {
const altText = match[1] || "";
const thumbnailUrl = match[2];
let originalUrl = thumbnailUrl;
if (altText.includes("thumb|||")) {
const content = altText.substring(altText.indexOf("thumb|||") + 8);
const parts = content.split("|||");
originalUrl = parts[0] || thumbnailUrl;
} else if (altText.includes("thumb$$")) {
const content = altText.substring(altText.indexOf("thumb$$") + 7);
const parts = content.split("$$");
originalUrl = parts[0] || thumbnailUrl;
} else if (altText.includes("thumb|")) {
const content = altText.substring(altText.indexOf("thumb|") + 6);
const parts = content.split("|");
originalUrl = parts[0] || thumbnailUrl;
}
parsed.push({ thumbnailUrl, originalUrl });
}
return parsed;
}
export async function POST(request: Request) {
try {
const session = await auth();
@ -63,35 +99,26 @@ export async function POST(request: Request) {
);
}
// 计算尺寸
const sizeMap: Record<string, { width: number; height: number }> = {
"1:1": { width: 1024, height: 1024 },
"3:2": { width: 1536, height: 1024 },
"2:3": { width: 1024, height: 1536 },
"4:3": { width: 1365, height: 1024 },
"3:4": { width: 1024, height: 1365 },
"16:9": { width: 1820, height: 1024 },
"9:16": { width: 1024, height: 1820 },
};
const size = sizeMap[aspectRatio] || sizeMap["1:1"];
// 构建消息
const messages = [
{
role: "user" as const,
content: prompt,
}
];
const messages = [{
role: "user" as const,
content: referenceImage
? [
{ type: "text" as const, text: prompt },
{ type: "image_url" as const, image_url: { url: referenceImage } },
]
: prompt,
}];
// 图片参数
const imageParams = {
aspectRatio,
imageSize: `${size.width}x${size.height}`,
imageSize: "2K",
imageCount,
negativePrompt,
};
let result;
let imageMarkdown = "";
// 根据提供商类型调用不同的生成器
if (provider.type === "gemini") {
@ -100,23 +127,36 @@ export async function POST(request: Request) {
baseURL: provider.baseUrl,
});
result = await geminiProvider.doGenerate({
const model = geminiProvider.chat(selectedModel.model.modelId);
const result = await model.doGenerate({
messages,
modelId: selectedModel.model.modelId,
imageParams,
});
imageMarkdown = result.text || "";
} else if (provider.type === "alibaba-image") {
const alibabaProvider = createAlibabaImage({
apiKey: provider.apiKey,
baseURL: provider.baseUrl,
});
result = await alibabaProvider.doGenerate({
const model = alibabaProvider.chat(selectedModel.model.modelId);
const result = await model.doGenerate({
messages,
modelId: selectedModel.model.modelId,
imageParams,
referenceImage,
});
imageMarkdown = result.text || "";
} else if (provider.type === "ruxa-image") {
const ruxaProvider = createRuxaImage({
apiKey: provider.apiKey,
baseURL: provider.baseUrl,
});
const model = ruxaProvider.chat(selectedModel.model.modelId);
const result = await model.doGenerate({
messages,
imageParams,
});
imageMarkdown = result.text || "";
} else {
return NextResponse.json(
{ success: false, error: "不支持的图片生成模型类型" },
@ -124,11 +164,13 @@ export async function POST(request: Request) {
);
}
if (result.images && result.images.length > 0) {
const parsedImages = parseImagesFromMarkdown(imageMarkdown);
if (parsedImages.length > 0) {
return NextResponse.json({
success: true,
data: {
images: result.images,
images: parsedImages,
model: selectedModel.model.displayName,
prompt,
}

View File

@ -2,7 +2,7 @@
* 场景裂变 API
*
* 将一张产品图批量融合到多个场景图中
* 支持多种生图模型:阿里生图、Gemini、既梦
* 支持多种生图模型:阿里生图、RUXA、Gemini、既梦
*/
import { NextResponse } from "next/server";
@ -16,6 +16,7 @@ import sharp from "sharp";
import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image";
import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision";
import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
// 请求体 Schema
const sceneFusionRequestSchema = z.object({
@ -364,6 +365,9 @@ async function processSceneWithProvider(
case "alibaba-image":
return processWithAlibabaImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
case "ruxa-image":
return processWithRuxaImage(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
case "gemini":
return processWithGemini(productImageUrl, sceneImageUrl, prompt, provider, modelId, aspectRatio, imageSize);
@ -455,6 +459,49 @@ async function processWithGemini(
return { thumbnail, original, prompt };
}
/**
* 使用 RUXA 模型处理
*/
async function processWithRuxaImage(
productImageUrl: string,
sceneImageUrl: string,
prompt: string,
provider: { baseUrl: string; apiKey: string },
modelId: string,
aspectRatio?: string,
imageSize?: string
): Promise<{ thumbnail: string; original: string; prompt: string }> {
console.log(`[SceneFusion][RUXA] Processing with model: ${modelId}`);
const ruxaProvider = createRuxaImage({
baseURL: provider.baseUrl,
apiKey: provider.apiKey,
});
const messages = [
{
role: "user",
content: [
{ type: "image_url", image_url: { url: productImageUrl } },
{ type: "image_url", image_url: { url: sceneImageUrl } },
{ type: "text", text: prompt },
],
},
];
const result = await ruxaProvider.chat(modelId).doGenerate({
messages,
imageParams: {
aspectRatio: aspectRatio || "auto",
imageSize: imageSize || "2K",
imageCount: 1,
},
});
const { thumbnail, original } = parseImageFromMarkdown(result.text);
return { thumbnail, original, prompt };
}
/**
* 使用既梦/火山模型处理
*/
@ -636,8 +683,8 @@ export async function POST(request: Request) {
// 如果没有配置或配置的模型不可用,使用默认的阿里生图模型
if (!selectedModel) {
// 按优先级查找:alibaba-image > gemini > volcano-vision
const imageProviderTypes = ["alibaba-image", "gemini", "volcano-vision"];
// 按优先级查找:alibaba-image > ruxa-image > gemini > volcano-vision
const imageProviderTypes = ["alibaba-image", "ruxa-image", "gemini", "volcano-vision"];
for (const providerType of imageProviderTypes) {
selectedModel = allModels.find(
(m) => m.provider.type === providerType && m.model.isEnabled

View File

@ -493,7 +493,7 @@ export function Chat({
else if (model?.providerType === "volcano-vision") modelType = "volcano-vision";
else if (model?.providerType === "volcano-video") modelType = "volcano-video";
else if (model?.providerType === "qianwen-video") modelType = "qianwen-video";
else if (model?.providerType === "alibaba-image") modelType = "alibaba-image";
else if (model?.providerType === "alibaba-image" || model?.providerType === "ruxa-image") modelType = "alibaba-image";
console.log(`[Chat] 🚀 记录生成任务: modelType=${modelType}, modelId=${currentModelId}`);
startGeneration(currentModelId, modelType);

View File

@ -106,6 +106,8 @@ export const ImageGenerationLoading = memo(({
return "purple";
case "alibaba-image":
return "orange";
case "ruxa-image":
return "green";
case "volcano-vision":
return "blue";
default:
@ -119,7 +121,8 @@ export const ImageGenerationLoading = memo(({
const count = Math.min(Math.max(imageCount, 1), maxImages);
// 获取显示名称
const displayName = modelName || (providerType === "alibaba-image" ? "千问图片编辑" :
const displayName = modelName || (providerType === "alibaba-image" ? "千问图片编辑" :
providerType === "ruxa-image" ? "RUXA 图片编辑" :
providerType === "volcano-vision" ? "即梦4.5" : "AI");
return (
@ -256,4 +259,3 @@ export const ImageGenerationLoading = memo(({
ImageGenerationLoading.displayName = "ImageGenerationLoading";
export default ImageGenerationLoading;

View File

@ -37,7 +37,7 @@ interface ImageParamsBarProps {
initialParams?: Partial<ImageGenerationParams>;
onParamsChange?: (params: ImageGenerationParams) => void;
inline?: boolean; // 内联模式:只显示参数按钮,不显示外层容器
providerType?: string; // 模型类型:gemini, volcano-vision, alibaba-image 等
providerType?: string; // 模型类型:gemini, volcano-vision, alibaba-image, ruxa-image 等
// 🆕 图片生成模型特性配置(从模型配置中读取)
imageGenConfig?: {
maxImages?: number;
@ -99,6 +99,10 @@ export function ImageParamsBar({
// 阿里生图:1~6 张(qwen-image-edit-plus)
return [1, 2, 3, 4, 5, 6];
}
if (providerType === "ruxa-image") {
// RUXA 当前按单图任务处理
return [1];
}
if (providerType === "volcano-vision") {
// 即梦4.5(豆包):1~3 张
return [1, 2, 3];
@ -271,4 +275,3 @@ export function ImageParamsBar({
// 非内联模式(暂时不实现,因为只需要内联模式)
return null;
}

View File

@ -113,7 +113,7 @@ function PureMessages({
// 🆕 判断是否是阿里生图模型(或者智能自动模式默认使用阿里生图占位图)
const isAlibabaImageModel = modelsData?.models?.some(
(m: DynamicModel) => m.id === selectedModelId && m.providerType === "alibaba-image"
(m: DynamicModel) => m.id === selectedModelId && (m.providerType === "alibaba-image" || m.providerType === "ruxa-image")
) ?? false;
// 🆕 获取当前模型的显示名称

View File

@ -18,6 +18,7 @@ import { createHuankeGemini } from "./providers/huanke-gemini";
import { createQianwenVideo } from "./providers/qianwen-video";
import { createVolcanoVision } from "./providers/volcano-vision";
import { createAlibabaImage } from "./providers/alibaba-image";
import { createRuxaImage } from "./providers/ruxa-image";
// 缓存已创建的 provider 实例
const providerCache = new Map<string, ReturnType<typeof createOpenAI>>();
@ -132,6 +133,16 @@ function getOrCreateProvider(provider: Pick<AiProvider, "id" | "type" | "baseUrl
return alibabaImageProvider as any;
}
case "ruxa-image": {
// RUXA 图片编辑服务
const ruxaImageProvider = createRuxaImage({
baseURL: provider.baseUrl,
apiKey: provider.apiKey,
});
console.log(`[Provider] Created Ruxa Image generation provider: ${provider.baseUrl}`);
return ruxaImageProvider as any;
}
case "custom":
default: {
// OpenAI 兼容的第三方 API(如 GPT-5 Nano、Claude via OpenAI proxy 等)
@ -214,6 +225,11 @@ export async function getDynamicModel(
console.log("[getDynamicModel] Getting Alibaba Image chat model..."); // 调试日志
baseModel = providerInstance.chat(model.modelId);
console.log("[getDynamicModel] Alibaba Image model retrieved:", baseModel ? "success" : "failed"); // 调试日志
} else if (provider.type === "ruxa-image") {
// RUXA 图片编辑生成使用 chat() 方法
console.log("[getDynamicModel] Getting Ruxa Image chat model..."); // 调试日志
baseModel = providerInstance.chat(model.modelId);
console.log("[getDynamicModel] Ruxa Image model retrieved:", baseModel ? "success" : "failed"); // 调试日志
} else if (provider.type === "huanke-gemini") {
// 环科 Gemini 原生格式使用 chat() 方法
console.log("[getDynamicModel] Getting Huanke Gemini chat model..."); // 调试日志
@ -322,4 +338,3 @@ export async function getSystemModel(
return null;
}
}

View File

@ -30,7 +30,7 @@ const INTENT_MODEL_MAP: Record<
> = {
// 图片编辑:优先 alibaba-image(灵魂4.5,支持提示词扩展)
image_edit: {
providerTypes: ["alibaba-image", "gemini", "huanke-gemini"],
providerTypes: ["alibaba-image", "ruxa-image", "gemini", "huanke-gemini"],
requireVision: true,
requireImageGen: true,
description: "图片编辑",
@ -45,7 +45,7 @@ const INTENT_MODEL_MAP: Record<
// 图片生成:优先灵魂4.5
image_gen: {
providerTypes: ["alibaba-image", "gemini", "huanke-gemini", "volcano-vision"],
providerTypes: ["alibaba-image", "ruxa-image", "gemini", "huanke-gemini", "volcano-vision"],
requireImageGen: true,
description: "图片生成",
},
@ -73,25 +73,25 @@ const SUB_INTENT_OVERRIDES: Record<
> = {
// 抠图:alibaba-image 效果最好
remove_background: {
providerTypes: ["alibaba-image", "gemini"],
providerTypes: ["alibaba-image", "ruxa-image", "gemini"],
reason: "抠图首选灵魂4.5",
},
// 换背景:alibaba-image 支持场景生成
replace_background: {
providerTypes: ["alibaba-image", "gemini"],
providerTypes: ["alibaba-image", "ruxa-image", "gemini"],
reason: "换背景首选灵魂4.5",
},
// 图片增强
enhance: {
providerTypes: ["alibaba-image", "gemini", "volcano-vision"],
providerTypes: ["alibaba-image", "ruxa-image", "gemini", "volcano-vision"],
reason: "增强首选灵魂4.5",
},
// 风格迁移:gemini 可能更有创意
style_transfer: {
providerTypes: ["gemini", "alibaba-image"],
providerTypes: ["gemini", "alibaba-image", "ruxa-image"],
reason: "风格迁移优选Gemini",
},
};

View File

@ -0,0 +1,378 @@
import "server-only";
import sharp from "sharp";
import { uploadFile } from "@/lib/storage/service";
interface RuxaImageConfig {
apiKey: string;
baseURL: string;
}
interface RuxaImageParams {
aspectRatio?: string;
imageSize?: string;
imageCount?: number;
outputFormat?: "png" | "jpg" | "jpeg" | "webp";
}
interface UploadedImage {
thumbnail: string;
original: string;
}
const DEFAULT_ASPECT_RATIO = "1:1";
const DEFAULT_RESOLUTION = "1K";
const DEFAULT_OUTPUT_FORMAT = "png";
const POLL_INTERVAL_MS = 2000;
const POLL_MAX_ATTEMPTS = 60;
function normalizeUrlForCompare(url: string) {
try {
const parsed = new URL(url);
parsed.search = "";
parsed.hash = "";
return parsed.toString();
} catch {
return url.split("?")[0]?.split("#")[0] || url;
}
}
function filterSourceImages(urls: string[], sourceUrls: string[]) {
if (sourceUrls.length === 0) return urls;
const sourceSet = new Set(sourceUrls.map((url) => normalizeUrlForCompare(url)));
return urls.filter((url) => !sourceSet.has(normalizeUrlForCompare(url)));
}
function joinApiPath(baseURL: string, path: string) {
const cleanBase = baseURL.replace(/\/+$/, "");
const cleanPath = path.startsWith("/") ? path : `/${path}`;
return `${cleanBase}${cleanPath}`;
}
function normalizeAspectRatio(aspectRatio?: string) {
if (!aspectRatio || aspectRatio === "auto") return DEFAULT_ASPECT_RATIO;
return aspectRatio;
}
function normalizeResolution(imageSize?: string) {
if (!imageSize) return DEFAULT_RESOLUTION;
const normalized = imageSize.toUpperCase();
if (["1K", "2K", "4K"].includes(normalized)) return normalized;
return DEFAULT_RESOLUTION;
}
function extractTaskId(payload: any): string | undefined {
return (
payload?.taskId ||
payload?.task_id ||
payload?.id ||
payload?.data?.taskId ||
payload?.data?.task_id ||
payload?.data?.id
);
}
function extractStatus(payload: any): string | undefined {
const raw =
payload?.status ||
payload?.taskStatus ||
payload?.task_status ||
payload?.state ||
payload?.data?.status ||
payload?.data?.taskStatus ||
payload?.data?.task_status ||
payload?.data?.state;
if (typeof raw !== "string") return undefined;
return raw.toLowerCase();
}
function extractErrorMessage(payload: any): string {
return (
payload?.message ||
payload?.error ||
payload?.errorMessage ||
payload?.data?.message ||
payload?.data?.error ||
"RUXA 任务执行失败"
);
}
function extractPromptAndImages(messages: Array<{ role: string; content: any }>) {
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
if (!lastUserMessage) {
throw new Error("No user message found");
}
let prompt = "";
const imageUrls: string[] = [];
const pushImageUrl = (url?: string) => {
if (!url || !url.startsWith("http")) return;
if (!imageUrls.includes(url)) {
imageUrls.push(url);
}
};
if (typeof lastUserMessage.content === "string") {
prompt = lastUserMessage.content.trim();
} else if (Array.isArray(lastUserMessage.content)) {
for (const part of lastUserMessage.content) {
if (part?.type === "text" && typeof part.text === "string") {
prompt = `${prompt} ${part.text}`.trim();
} else if (part?.type === "image_url" && part.image_url?.url) {
pushImageUrl(part.image_url.url);
} else if (part?.type === "file") {
pushImageUrl(part.url || part.data);
} else if (part?.type === "image") {
pushImageUrl(part.image || part.url);
}
}
}
if (!prompt) {
throw new Error("No prompt found in user message");
}
if (imageUrls.length === 0) {
throw new Error("RUXA 图片编辑模型需要至少 1 张输入图片");
}
return { prompt, imageUrls };
}
function collectImageUrls(payload: any): string[] {
const urls = new Set<string>();
const isImageLikeKey = (key: string) =>
/image|img|url|output|result|artifact|file/i.test(key);
const isLikelyImageUrl = (value: string) =>
/^https?:\/\//i.test(value) &&
(/\.(png|jpg|jpeg|webp|gif)(\?|$)/i.test(value) || value.includes("image"));
const walk = (value: any, parentKey = "") => {
if (typeof value === "string") {
if (isLikelyImageUrl(value) || (isImageLikeKey(parentKey) && /^https?:\/\//i.test(value))) {
urls.add(value);
}
return;
}
if (Array.isArray(value)) {
for (const item of value) {
walk(item, parentKey);
}
return;
}
if (value && typeof value === "object") {
for (const [key, nestedValue] of Object.entries(value)) {
walk(nestedValue, key);
}
}
};
walk(payload);
return Array.from(urls);
}
function detectImageExtension(imageUrl: string, contentType?: string | null) {
const type = (contentType || "").toLowerCase();
if (type.includes("jpeg") || type.includes("jpg")) return "jpg";
if (type.includes("webp")) return "webp";
if (type.includes("png")) return "png";
const cleanUrl = imageUrl.split("?")[0]?.toLowerCase() || "";
if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "jpg";
if (cleanUrl.endsWith(".webp")) return "webp";
if (cleanUrl.endsWith(".png")) return "png";
return "png";
}
async function createThumbnail(buffer: Buffer): Promise<Buffer> {
try {
return await sharp(buffer)
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
.jpeg({ quality: 80 })
.toBuffer();
} catch (error) {
console.error("[Ruxa Image] Failed to create thumbnail:", error);
return buffer;
}
}
function generateFilename(prefix: string, extension: string) {
const ts = Date.now();
const rand = Math.random().toString(36).slice(2, 8);
return `${prefix}-${ts}-${rand}.${extension}`;
}
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
const response = await fetch(imageUrl);
if (!response.ok) {
throw new Error(`Failed to download image: ${response.status}`);
}
const arrayBuffer = await response.arrayBuffer();
const buffer = Buffer.from(arrayBuffer);
const thumbnailBuffer = await createThumbnail(buffer);
const originalExt = detectImageExtension(imageUrl, response.headers.get("content-type"));
const originalFilename = generateFilename("ruxa-image-orig", originalExt);
const originalArrayBuffer = buffer.buffer.slice(
buffer.byteOffset,
buffer.byteOffset + buffer.byteLength
) as ArrayBuffer;
const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer);
const thumbFilename = generateFilename("ruxa-image-thumb", "jpg");
const thumbArrayBuffer = thumbnailBuffer.buffer.slice(
thumbnailBuffer.byteOffset,
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
) as ArrayBuffer;
const thumbUploaded = await uploadFile(thumbFilename, thumbArrayBuffer);
return {
thumbnail: thumbUploaded.url,
original: originalUploaded.url,
};
}
async function waitForTaskResult(baseURL: string, apiKey: string, taskId: string, sourceUrls: string[]) {
const endpointCandidates = [
(id: string) => joinApiPath(baseURL, `/api/v1/tasks/query/${id}`),
(id: string) => joinApiPath(baseURL, `/api/v1/tasks/${id}`),
(id: string) => `${joinApiPath(baseURL, "/api/v1/tasks/query")}?taskId=${encodeURIComponent(id)}`,
];
for (let attempt = 1; attempt <= POLL_MAX_ATTEMPTS; attempt++) {
for (const endpointBuilder of endpointCandidates) {
try {
const endpoint = endpointBuilder(taskId);
const response = await fetch(endpoint, {
headers: {
Authorization: `Bearer ${apiKey}`,
"Content-Type": "application/json",
},
});
if (!response.ok) {
continue;
}
const payload = await response.json();
const imageUrls = filterSourceImages(collectImageUrls(payload), sourceUrls);
const status = extractStatus(payload);
if (imageUrls.length > 0) {
return imageUrls;
}
if (status && ["failed", "error", "cancelled", "canceled"].includes(status)) {
throw new Error(extractErrorMessage(payload));
}
if (status && ["success", "succeeded", "completed", "done", "finished"].includes(status)) {
throw new Error("任务已完成但未返回图片地址");
}
} catch (error) {
if (error instanceof Error && error.message !== "任务已完成但未返回图片地址") {
throw error;
}
}
}
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
}
throw new Error("RUXA 任务超时,请稍后重试");
}
function formatMarkdown(images: UploadedImage[], resolution: string) {
return images
.map((image) => `![thumb|||${image.original}|||${resolution}](${image.thumbnail})`)
.join("\n\n");
}
export function createRuxaImage({ apiKey, baseURL }: RuxaImageConfig) {
return {
provider: "ruxa-image",
chat: (modelId: string) => ({
provider: "ruxa-image",
doGenerate: async (options: {
messages: Array<{ role: string; content: any }>;
imageParams?: RuxaImageParams;
overrideModelId?: string;
}) => {
const { prompt, imageUrls } = extractPromptAndImages(options.messages);
const aspectRatio = normalizeAspectRatio(options.imageParams?.aspectRatio);
const resolution = normalizeResolution(options.imageParams?.imageSize);
const outputFormat = options.imageParams?.outputFormat || DEFAULT_OUTPUT_FORMAT;
const activeModelId = options.overrideModelId || modelId;
const createEndpoint = joinApiPath(baseURL, "/api/v1/tasks/create");
const createResponse = await fetch(createEndpoint, {
method: "POST",
headers: {
Authorization: `Bearer ${apiKey}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
input: {
prompt,
image_input: imageUrls,
aspect_ratio: aspectRatio,
resolution,
output_format: outputFormat,
},
model: activeModelId,
}),
});
const createPayload = await createResponse.json();
if (!createResponse.ok) {
throw new Error(extractErrorMessage(createPayload));
}
let outputUrls = filterSourceImages(collectImageUrls(createPayload), imageUrls);
if (outputUrls.length === 0) {
const taskId = extractTaskId(createPayload);
if (!taskId) {
throw new Error("RUXA 创建任务成功,但未返回 taskId");
}
outputUrls = await waitForTaskResult(baseURL, apiKey, taskId, imageUrls);
}
const uploadedImages: UploadedImage[] = [];
for (const outputUrl of outputUrls.slice(0, options.imageParams?.imageCount || 1)) {
uploadedImages.push(await downloadAndUploadImage(outputUrl));
}
if (uploadedImages.length === 0) {
throw new Error("RUXA 未返回可用图片");
}
const text = formatMarkdown(uploadedImages, resolution);
return {
text,
finishReason: "stop" as const,
usage: {
promptTokens: 0,
completionTokens: text.length,
},
};
},
doStream: async function* (options: {
messages: Array<{ role: string; content: any }>;
imageParams?: RuxaImageParams;
overrideModelId?: string;
}) {
const result = await this.doGenerate(options);
yield {
type: "text-delta",
textDelta: result.text,
};
},
}),
};
}

View File

@ -1,4 +1,5 @@
import { config } from "dotenv";
import { sql } from "drizzle-orm";
import { drizzle } from "drizzle-orm/postgres-js";
import { migrate } from "drizzle-orm/postgres-js/migrator";
import postgres from "postgres";
@ -19,6 +20,17 @@ const runMigrate = async () => {
const start = Date.now();
await migrate(db, { migrationsFolder: "./lib/db/migrations" });
await db.execute(sql`
ALTER TABLE "AiProvider" DROP CONSTRAINT IF EXISTS "AiProvider_type_check";
`);
await db.execute(sql`
ALTER TABLE "AiProvider"
ADD CONSTRAINT "AiProvider_type_check"
CHECK (type IN (
'alibaba', 'openai', 'custom', 'local', 'gemini',
'huanke-gemini', 'qianwen-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'ruxa-image'
));
`);
const end = Date.now();
console.log("✅ Migrations completed in", end - start, "ms");

View File

@ -941,7 +941,7 @@ export async function createAiProvider({
updatedBy,
}: {
name: string;
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image";
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image";
baseUrl: string;
apiKey: string;
isEnabled?: boolean;
@ -979,7 +979,7 @@ export async function updateAiProvider({
}: {
id: string;
name?: string;
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image";
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image";
baseUrl?: string;
apiKey?: string;
isEnabled?: boolean;

View File

@ -215,7 +215,7 @@ export type ToolConfig = InferSelectModel<typeof toolConfig>;
export const aiProvider = pgTable("AiProvider", {
id: uuid("id").primaryKey().notNull().defaultRandom(),
name: varchar("name", { length: 128 }).notNull(), // 提供商名称,如 "阿里云通义千问"
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "volcano-vision", "volcano-video", "alibaba-image"] })
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image"] })
.notNull()
.default("custom"), // 提供商类型
baseUrl: text("baseUrl").notNull(), // API 基础 URL