lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(video): add ruxa-video provider end-to-end

Browse Source
This commit is contained in:
Your Name 2026-03-05 11:16:24 +08:00
parent 8aadc721a5
commit 25f88e8d3d
20 changed files with 567 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/(admin)/settings/admin-settings-client.tsx
View File

@ -146,7 +146,7 @@ type ToolConfig = {
type AiProvider = {
id: string;
name: string;
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal";
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal";
baseUrl: string;
apiKey: string;
isEnabled: boolean;
@ -427,6 +427,7 @@ const PROVIDER_TYPES = [
{ value: "huanke-gemini", label: "环科Gemini生图" }, // 🆕 环科 Gemini 原生格式 API
{ value: "qianwen-video", label: "千问视频生成" },
{ value: "doubao-video", label: "豆包AI视频" },
{ value: "ruxa-video", label: "RUXA 视频生成" },
{ value: "volcano-vision", label: "火山引擎视觉" },
{ value: "volcano-video", label: "火山引擎视频" },
{ value: "fal-video", label: "FAL 视频生成" },
@ -446,6 +447,7 @@ const PROVIDER_DEFAULT_BASE_URL: Record<string, string> = {
"huanke-gemini": "https://api.huandutech.com/v1",
"qianwen-video": "https://dashscope.aliyuncs.com/api/v1/services/aigc/video-generation",
"doubao-video": "https://ark.cn-beijing.volces.com/api/v3",
"ruxa-video": "https://api.ruxa.ai",
"volcano-vision": "https://ark.cn-beijing.volces.com",
"volcano-video": "https://ark.cn-beijing.volces.com/api/v3",
};
@ -1277,7 +1279,7 @@ export function AdminSettingsClient() {
const [providerForm, setProviderForm] = useState({
id: "",
name: "",
type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal",
type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal",
baseUrl: "",
apiKey: "",
isEnabled: true,
@ -1366,6 +1368,7 @@ export function AdminSettingsClient() {
"doubao-video",
"qianwen-video",
"fal-video",
"ruxa-video",
].includes(selectedModelProviderType);
// 存储配置状态
@ -3620,6 +3623,7 @@ export function AdminSettingsClient() {
"volcano-video": "bg-red-500",
"doubao-video": "bg-red-400",
"fal-video": "bg-indigo-500",
"ruxa-video": "bg-violet-500",
"alibaba": "bg-blue-500",
"alibaba-image": "bg-amber-500",
"ruxa-image": "bg-emerald-500",

17
app/(chat)/api/chat/route.ts
View File

@ -257,6 +257,7 @@ const VIDEO_PROVIDER_TYPES = new Set([
"doubao-video",
"qianwen-video",
"fal-video",
"ruxa-video",
]);
function isVideoProviderType(providerType?: string | null): boolean {
@ -2972,6 +2973,7 @@ export async function POST(request: Request) {
const isVideoGenerationModel = [
"qianwen-video",
"fal-video",
"ruxa-video",
"volcano-video",
"doubao-video",
].includes(String(resolvedProviderType || (languageModel as any).provider || ""));
@ -4064,15 +4066,18 @@ export async function POST(request: Request) {
}
// ========== 火山引擎视觉处理结束 ==========
// ========== 千问/FAL 视频生成独立处理 ==========
// ========== 千问/FAL/RUXA 视频生成独立处理 ==========
if (
["qianwen-video", "fal-video"].includes((languageModel as any).provider) &&
["qianwen-video", "fal-video", "ruxa-video"].includes((languageModel as any).provider) &&
typeof (languageModel as any).doGenerate === "function"
) {
const currentVideoProvider = (languageModel as any).provider as "qianwen-video" | "fal-video";
const currentVideoProvider = (languageModel as any).provider as
| "qianwen-video"
| "fal-video"
| "ruxa-video";
console.log(`[Chat API] Using ${currentVideoProvider} video generation`);
const qianwenVideoStream = createUIMessageStream({
const videoGenerationStream = createUIMessageStream({
execute: async ({ writer }) => {
try {
const conversationMessage = smartAutoFriendlyMessage || null;
@ -4212,9 +4217,9 @@ export async function POST(request: Request) {
},
});
return createSseResponse(qianwenVideoStream);
return createSseResponse(videoGenerationStream);
}
// ========== 千问/FAL 视频处理结束 ==========
// ========== 千问/FAL/RUXA 视频处理结束 ==========
let stopStreamKeepalive: (() => void) | null = null;
const stream = createUIMessageStream({

1
app/(chat)/api/models/route.ts
View File

@ -13,6 +13,7 @@ const VIDEO_PROVIDER_TYPES = new Set([
"doubao-video",
"qianwen-video",
"fal-video",
"ruxa-video",
]);
function isNanoBananaProModel(model: { displayName?: string | null; modelId?: string | null }) {

1
app/(chat)/page.tsx
View File

@ -20,6 +20,7 @@ const VIDEO_PROVIDER_TYPES = new Set([
"doubao-video",
"qianwen-video",
"fal-video",
"ruxa-video",
]);
function isNanoBananaProModel(model: { displayName?: string | null; modelId?: string | null }) {

4
app/api/admin/ai-providers/route.ts
View File

@ -18,6 +18,7 @@ const providerTypes = [
"huanke-gemini",
"qianwen-video",
"doubao-video",
"ruxa-video",
"volcano-vision",
"volcano-video",
"alibaba-image",
@ -51,6 +52,7 @@ const updateProviderSchema = z.object({
const ZHIPU_BASE_URL = "https://open.bigmodel.cn/api/paas/v4";
const VOLCANO_MULTIMODAL_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3";
const DOUBAO_VIDEO_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3";
const RUXA_VIDEO_BASE_URL = "https://api.ruxa.ai";
const TRIPO_3D_BASE_URL = "https://api.tripo3d.ai/v2/openapi";
function normalizeProviderTypeForStorage(
@ -158,6 +160,8 @@ export async function POST(request: Request) {
? data.baseUrl || VOLCANO_MULTIMODAL_BASE_URL
: data.type === "doubao-video"
? data.baseUrl || DOUBAO_VIDEO_BASE_URL
: data.type === "ruxa-video"
? data.baseUrl || RUXA_VIDEO_BASE_URL
: data.type === "tripo-3d"
? data.baseUrl || TRIPO_3D_BASE_URL
: data.baseUrl,

22
components/canvas/CanvasChatPanel.tsx
View File

@ -763,8 +763,9 @@ function normalizeVideoParamsForProvider(
providerType?: string
): VideoGenerationParams {
const isFalVideo = providerType === "fal-video";
const isRuxaVideo = providerType === "ruxa-video";
const base: VideoGenerationParams = {
mode: isFalVideo ? "first-frame" : "text-to-video",
mode: isFalVideo || isRuxaVideo ? "first-frame" : "text-to-video",
resolution: "720p",
ratio: "16:9",
duration: 5,
@ -788,6 +789,19 @@ function normalizeVideoParamsForProvider(
return merged;
}
if (isRuxaVideo) {
merged.mode = "first-frame";
if (!["16:9", "9:16", "1:1"].includes(merged.ratio)) {
merged.ratio = "16:9";
}
merged.duration = merged.duration >= 8 ? 10 : 5;
merged.generateAudio = false;
merged.cameraMove = "none";
merged.cameraFixed = false;
merged.fps = 24;
return merged;
}
if (!["text-to-video", "first-frame", "first-last-frame", "reference-images"].includes(merged.mode)) {
merged.mode = "text-to-video";
}
@ -806,6 +820,10 @@ function getVideoImageLimit(
return { min: 0, max: 0 };
}
const isFalVideo = providerType === "fal-video";
const isRuxaVideo = providerType === "ruxa-video";
if (isRuxaVideo) {
return { min: 1, max: 1 };
}
if (isFalVideo) {
if (params.mode === "first-last-frame") {
return { min: 2, max: 2 };
@ -2457,7 +2475,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
}
}, [canvasSkillTags, selectedSkillTagId]);
const VIDEO_PROVIDER_TYPES = new Set(["volcano-video", "doubao-video", "qianwen-video", "fal-video"]);
const VIDEO_PROVIDER_TYPES = new Set(["volcano-video", "doubao-video", "qianwen-video", "fal-video", "ruxa-video"]);
const visualModels = useMemo(() => {
const filtered =
modelsData?.models?.filter(

21
components/chat.tsx
View File

@ -71,8 +71,9 @@ function normalizeVideoParamsForProvider(
providerType?: string
): VideoGenerationParams {
const isFalVideo = providerType === "fal-video";
const isRuxaVideo = providerType === "ruxa-video";
const base: VideoGenerationParams = {
mode: isFalVideo ? "first-frame" : "text-to-video",
mode: isFalVideo || isRuxaVideo ? "first-frame" : "text-to-video",
resolution: "720p",
ratio: "16:9",
duration: 5,
@ -99,6 +100,19 @@ function normalizeVideoParamsForProvider(
return merged;
}
if (isRuxaVideo) {
merged.mode = "first-frame";
if (!["16:9", "9:16", "1:1"].includes(merged.ratio)) {
merged.ratio = "16:9";
}
merged.duration = merged.duration >= 8 ? 10 : 5;
merged.generateAudio = false;
merged.cameraMove = "none";
merged.cameraFixed = false;
merged.fps = 24;
return merged;
}
if (!["text-to-video", "first-frame", "first-last-frame", "reference-images"].includes(merged.mode)) {
merged.mode = "text-to-video";
}
@ -416,7 +430,8 @@ export function Chat({
currentModel?.providerType === "volcano-video" ||
currentModel?.providerType === "doubao-video" ||
currentModel?.providerType === "qianwen-video" ||
currentModel?.providerType === "fal-video";
currentModel?.providerType === "fal-video" ||
currentModel?.providerType === "ruxa-video";
const isImageModel = currentModel?.supportsImageGen === true;
// 🔍 调试日志
@ -962,6 +977,7 @@ export function Chat({
else if (model?.providerType === "volcano-video" || model?.providerType === "doubao-video") modelType = "volcano-video";
else if (model?.providerType === "qianwen-video") modelType = "qianwen-video";
else if (model?.providerType === "fal-video") modelType = "fal-video";
else if (model?.providerType === "ruxa-video") modelType = "ruxa-video";
else if (
model?.providerType === "alibaba-image" ||
model?.providerType === "ruxa-image" ||
@ -1793,6 +1809,7 @@ export function Chat({
): PendingGeneration["modelType"] => {
if (provider === "qianwen-video") return "qianwen-video";
if (provider === "fal-video") return "fal-video";
if (provider === "ruxa-video") return "ruxa-video";
if (provider === "volcano-video" || provider === "doubao-video") return "volcano-video";
return "other";
};

6
components/messages.tsx
View File

@ -110,7 +110,7 @@ function PureMessages({
const isQianwenVideoModel = modelsData?.models?.some(
(m: DynamicModel) =>
m.id === selectedModelId &&
(m.providerType === "qianwen-video" || m.providerType === "fal-video")
(m.providerType === "qianwen-video" || m.providerType === "fal-video" || m.providerType === "ruxa-video")
) ?? false;
const isVolcanoVisionModel = modelsData?.models?.some(
@ -139,6 +139,9 @@ function PureMessages({
(m: DynamicModel) =>
m.id === selectedModelId &&
(m.providerType === "volcano-video" ||
m.providerType === "qianwen-video" ||
m.providerType === "fal-video" ||
m.providerType === "ruxa-video" ||
m.providerType === "doubao-video" ||
m.modelId?.toLowerCase().includes("video") ||
m.modelId?.toLowerCase().includes("sora") ||
@ -608,6 +611,7 @@ function PureMessages({
modelType === "volcano-video" ||
modelType === "qianwen-video" ||
modelType === "fal-video" ||
modelType === "ruxa-video" ||
(!modelType && (isQianwenVideoModel || isVideoModel));
const isAutoMode = selectedModelId === "auto";

16
components/multimodal-input.tsx
View File

@ -50,6 +50,7 @@ const VIDEO_PROVIDER_TYPES = new Set([
"doubao-video",
"qianwen-video",
"fal-video",
"ruxa-video",
]);
function isVideoProviderType(providerType?: string | null) {
@ -690,13 +691,14 @@ function PureMultimodalInput({
if (!isVideoModel || !videoParams) {
return { max: Number.POSITIVE_INFINITY, label: "" as const };
}
if (videoProviderType === "fal-video") {
if (videoProviderType === "fal-video" || videoProviderType === "ruxa-video") {
const providerLabel = videoProviderType === "ruxa-video" ? "RUXA" : "FAL";
switch (videoParams.mode) {
case "first-last-frame":
return { max: 2, label: "FAL 首尾帧模式需要 2 张图（首帧 + 尾帧）" as const };
return { max: 2, label: `${providerLabel} 首尾帧模式需要 2 张图（首帧 + 尾帧）` as const };
case "first-frame":
default:
return { max: 1, label: "FAL 图生视频模式需要 1 张起始图" as const };
return { max: 1, label: `${providerLabel} 图生视频模式需要 1 张起始图` as const };
}
}
switch (videoParams.mode) {
@ -1741,7 +1743,13 @@ function PureModelSelectorCompact({
<span className="flex h-5 w-5 items-center justify-center">
{(() => {
const providerType = model.providerType || "";
if (providerType === "volcano-video" || providerType === "doubao-video" || providerType === "qianwen-video" || providerType === "fal-video") {
if (
providerType === "volcano-video" ||
providerType === "doubao-video" ||
providerType === "qianwen-video" ||
providerType === "fal-video" ||
providerType === "ruxa-video"
) {
return <Video className="h-4 w-4" strokeWidth={2.2} />;
}
if (

113
components/video-params-bar.tsx
View File

@ -145,24 +145,32 @@ const FAL_MODE_OPTIONS: Array<{ value: VideoGenerationMode; label: string }> = [
{ value: "first-frame", label: "图生视频" },
{ value: "first-last-frame", label: "首尾帧" },
];
const RUXA_MODE_OPTIONS: Array<{ value: VideoGenerationMode; label: string }> = [
{ value: "first-frame", label: "图生视频" },
];
const FAL_RATIO_OPTIONS = [
{ value: "16:9", label: "16:9" },
{ value: "9:16", label: "9:16" },
{ value: "1:1", label: "1:1" },
] as const;
const RUXA_RATIO_OPTIONS = FAL_RATIO_OPTIONS;
const STANDARD_DURATION_OPTIONS = [5, 10];
const FAL_DURATION_OPTIONS = [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];
const RUXA_DURATION_OPTIONS = [5, 10];
const STANDARD_RATIO_SET = new Set(["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"]);
const FAL_RATIO_SET = new Set(["16:9", "9:16", "1:1"]);
const RUXA_RATIO_SET = new Set(["16:9", "9:16", "1:1"]);
function normalizeVideoParamsByProvider(
source: Partial<VideoGenerationParams> | undefined,
isFalVideo: boolean
providerType?: string
): VideoGenerationParams {
const isFalVideo = providerType === "fal-video";
const isRuxaVideo = providerType === "ruxa-video";
const base: VideoGenerationParams = {
mode: isFalVideo ? "first-frame" : "text-to-video",
mode: isFalVideo || isRuxaVideo ? "first-frame" : "text-to-video",
resolution: "720p",
ratio: "16:9",
duration: 5,
@ -184,6 +192,17 @@ function normalizeVideoParamsByProvider(
return merged;
}
if (isRuxaVideo) {
merged.mode = "first-frame";
merged.ratio = RUXA_RATIO_SET.has(merged.ratio) ? merged.ratio : "16:9";
merged.duration = merged.duration >= 8 ? 10 : 5;
merged.generateAudio = false;
merged.cameraMove = "none";
merged.cameraFixed = false;
merged.fps = 24;
return merged;
}
if (
merged.mode !== "text-to-video" &&
merged.mode !== "first-last-frame" &&
@ -206,25 +225,38 @@ export function VideoParamsBar({
providerType = "volcano-video",
}: VideoParamsBarProps) {
const isFalVideo = providerType === "fal-video";
const modeOptions = isFalVideo ? FAL_MODE_OPTIONS : STANDARD_MODE_OPTIONS;
const ratioOptions = isFalVideo ? FAL_RATIO_OPTIONS : RATIO_OPTIONS;
const durationOptions = isFalVideo ? FAL_DURATION_OPTIONS : STANDARD_DURATION_OPTIONS;
const isRuxaVideo = providerType === "ruxa-video";
const modeOptions = isFalVideo
? FAL_MODE_OPTIONS
: isRuxaVideo
? RUXA_MODE_OPTIONS
: STANDARD_MODE_OPTIONS;
const ratioOptions = isFalVideo
? FAL_RATIO_OPTIONS
: isRuxaVideo
? RUXA_RATIO_OPTIONS
: RATIO_OPTIONS;
const durationOptions = isFalVideo
? FAL_DURATION_OPTIONS
: isRuxaVideo
? RUXA_DURATION_OPTIONS
: STANDARD_DURATION_OPTIONS;
const [params, setParams] = useState<VideoGenerationParams>(() =>
normalizeVideoParamsByProvider(initialParams, isFalVideo)
normalizeVideoParamsByProvider(initialParams, providerType)
);
const [collapsed, setCollapsed] = useState(false);
useEffect(() => {
const normalized = normalizeVideoParamsByProvider(initialParams, isFalVideo);
const normalized = normalizeVideoParamsByProvider(initialParams, providerType);
setParams((prev) => {
if (JSON.stringify(prev) === JSON.stringify(normalized)) {
return prev;
}
return normalized;
});
}, [initialParams, isFalVideo]);
}, [initialParams, providerType]);
// 获取当前运镜模式的标签
const getCurrentCameraLabel = () => {
@ -236,30 +268,33 @@ export function VideoParamsBar({
};
// 🆕 根据模式获取图片上传配置（根据火山引擎官方文档）
const getImageUploadConfig = () => {
if (isFalVideo) {
switch (params.mode) {
case "first-last-frame":
return { min: 2, max: 2, label: "首尾帧（2 张）" };
case "first-frame":
default:
return { min: 1, max: 1, label: "图生视频（1 张起始图）" };
const getImageUploadConfig = () => {
if (isRuxaVideo) {
return { min: 1, max: 1, label: "图生视频（1 张起始图）" };
}
if (isFalVideo) {
switch (params.mode) {
case "first-last-frame":
return { min: 2, max: 2, label: "首尾帧（2 张）" };
case "first-frame":
default:
return { min: 1, max: 1, label: "图生视频（1 张起始图）" };
}
}
}
switch (params.mode) {
case "text-to-video":
return { min: 0, max: 0, label: "文生视频（无需图片）" };
case "first-last-frame":
return { min: 2, max: 2, label: "首尾帧（各 1 张）" };
case "first-frame":
return { min: 1, max: 1, label: "首帧模式（1 张）" };
case "reference-images":
return { min: 1, max: 4, label: "参考图（1-4 张）" };
default:
return { min: 0, max: 0, label: "" };
}
};
switch (params.mode) {
case "text-to-video":
return { min: 0, max: 0, label: "文生视频（无需图片）" };
case "first-last-frame":
return { min: 2, max: 2, label: "首尾帧（各 1 张）" };
case "first-frame":
return { min: 1, max: 1, label: "首帧模式（1 张）" };
case "reference-images":
return { min: 1, max: 4, label: "参考图（1-4 张）" };
default:
return { min: 0, max: 0, label: "" };
}
};
const imageConfig = getImageUploadConfig();
@ -347,7 +382,7 @@ const getImageUploadConfig = () => {
</div>
{/* 分辨率（非 fal） */}
{!isFalVideo && (
{!isFalVideo && !isRuxaVideo && (
<div>
<p className="text-[11px] font-medium text-[#8a94a3]">选择分辨率</p>
<div className="mt-1.5 grid grid-cols-3 gap-1.5 rounded-[12px] bg-[#eceff2]/90 p-1">
@ -449,6 +484,10 @@ const getImageUploadConfig = () => {
</button>
</div>
</div>
) : isRuxaVideo ? (
<div className="rounded-[12px] bg-[#eceff2]/90 p-2 text-[11px] text-[#7b8796]">
当前模型支持图生视频（1 张起始图），已自动按官方参数约束处理。
</div>
) : (
<>
{/* 帧率 */}
@ -659,7 +698,7 @@ const getImageUploadConfig = () => {
</DropdownMenuContent>
</DropdownMenu>
{!isFalVideo && (
{!isFalVideo && !isRuxaVideo && (
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button variant="ghost" size="sm" className="gap-1 px-2 h-8 focus-visible:ring-0 focus-visible:ring-offset-0">
@ -705,6 +744,16 @@ const getImageUploadConfig = () => {
<Sparkles size={14} />
{params.generateAudio === false ? "静音" : "带音频"}
</Button>
) : isRuxaVideo ? (
<Button
variant="ghost"
size="sm"
className="gap-1 px-2 h-8 focus-visible:ring-0 focus-visible:ring-offset-0"
disabled
>
<Sparkles size={14} />
图生视频
</Button>
) : (
<>
<DropdownMenu>

2
hooks/use-pending-generation.ts
View File

@ -8,7 +8,7 @@ import { useEffect, useState, useCallback } from "react";
export interface PendingGeneration {
chatId: string;
modelId: string;
modelType: "gemini" | "volcano-vision" | "volcano-video" | "qianwen-video" | "fal-video" | "alibaba-image" | "other";
modelType: "gemini" | "volcano-vision" | "volcano-video" | "qianwen-video" | "fal-video" | "ruxa-video" | "alibaba-image" | "other";
timestamp: number;
userMessageId?: string;
}

7
lib/ai/auto-agent-graph.ts
View File

@ -582,7 +582,12 @@ function shouldPreferRemoteVisionUrls(item?: EnabledModelWithProvider) {
const baseUrl = String((item.provider as { baseUrl?: string | null })?.baseUrl || "").toLowerCase();
const modelText = `${item.model.displayName || ""} ${item.model.modelId || ""}`.toLowerCase();
if (providerType === "volcano-multimodal" || providerType === "doubao-video" || providerType === "volcano-video") {
if (
providerType === "volcano-multimodal" ||
providerType === "doubao-video" ||
providerType === "volcano-video" ||
providerType === "ruxa-video"
) {
return true;
}
if (baseUrl.includes("ark.cn-beijing.volces.com")) {

16
lib/ai/dynamic-provider.ts
View File

@ -21,6 +21,7 @@ import { createAlibabaImage } from "./providers/alibaba-image";
import { createRuxaImage } from "./providers/ruxa-image";
import { createFalImage } from "./providers/fal-image";
import { createFalVideo } from "./providers/fal-video";
import { createRuxaVideo } from "./providers/ruxa-video";
import { createVolcanoMultimodal } from "./providers/volcano-multimodal";
import { getProviderDefaultBaseUrl } from "./provider-defaults";
@ -154,6 +155,16 @@ function getOrCreateProvider(provider: Pick<AiProvider, "id" | "type" | "baseUrl
return falVideoProvider as any;
}
case "ruxa-video": {
// RUXA 视频生成服务（Veo 3.1）
const ruxaVideoProvider = createRuxaVideo({
baseURL: resolvedBaseUrl,
apiKey: provider.apiKey,
});
console.log(`[Provider] Created Ruxa Video generation provider: ${resolvedBaseUrl}`);
return ruxaVideoProvider as any;
}
case "volcano-vision": {
// 火山引擎视觉图片生成服务（Seedream）
// 支持文生图、图生图、多图融合等功能
@ -288,6 +299,11 @@ export async function getDynamicModel(
console.log("[getDynamicModel] Getting Fal Video chat model..."); // 调试日志
baseModel = providerInstance.chat(resolvedModelId);
console.log("[getDynamicModel] Fal Video model retrieved:", baseModel ? "success" : "failed"); // 调试日志
} else if (provider.type === "ruxa-video") {
// RUXA 视频生成使用 chat() 方法
console.log("[getDynamicModel] Getting Ruxa Video chat model..."); // 调试日志
baseModel = providerInstance.chat(resolvedModelId);
console.log("[getDynamicModel] Ruxa Video model retrieved:", baseModel ? "success" : "failed"); // 调试日志
} else if (provider.type === "volcano-vision") {
// 火山引擎视觉生成使用 chat() 方法
console.log("[getDynamicModel] Getting Volcano Vision chat model..."); // 调试日志

2
lib/ai/model-router.ts
View File

@ -52,7 +52,7 @@ const INTENT_MODEL_MAP: Record<
// 视频生成
video_gen: {
providerTypes: ["doubao-video", "volcano-video", "qianwen-video", "fal-video"],
providerTypes: ["doubao-video", "volcano-video", "qianwen-video", "fal-video", "ruxa-video"],
description: "视频生成",
},

1
lib/ai/provider-defaults.ts
View File

@ -6,6 +6,7 @@ const PROVIDER_DEFAULT_BASE_URL: Partial<Record<AiProvider["type"], string>> = {
"huanke-gemini": "https://api.huandutech.com/v1",
"qianwen-video": "https://dashscope.aliyuncs.com/api/v1/services/aigc/video-generation",
"doubao-video": "https://ark.cn-beijing.volces.com/api/v3",
"ruxa-video": "https://api.ruxa.ai",
"volcano-vision": "https://ark.cn-beijing.volces.com",
"volcano-video": "https://ark.cn-beijing.volces.com/api/v3",
"alibaba-image":

374
lib/ai/providers/ruxa-video.ts Normal file
View File

@ -0,0 +1,374 @@
import "server-only";
import { uploadFile } from "@/lib/storage/service";
interface RuxaVideoConfig {
apiKey: string;
baseURL: string;
}
interface RuxaVideoParams {
mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
ratio?: string;
duration?: number;
enhancePrompt?: boolean;
enableUpsample?: boolean;
}
const POLL_INTERVAL_MS = 2500;
const POLL_MAX_ATTEMPTS = 90;
function joinApiPath(baseURL: string, path: string) {
const cleanBase = String(baseURL || "").replace(/\/+$/, "");
const cleanPath = path.startsWith("/") ? path : `/${path}`;
return `${cleanBase}${cleanPath}`;
}
function normalizeRuxaTextInput(value: string) {
return String(value || "")
.replace(/\u00a0/g, " ")
.trim()
.normalize("NFKC");
}
function normalizeRuxaCredentials(apiKey: string) {
const normalized = normalizeRuxaTextInput(apiKey);
const stripped = normalized
.replace(/^bearer\s+/i, "")
.replace(/^key\s+/i, "");
const kvMatch = stripped.match(/^(?:api[_-]?key|token)\s*[:：]\s*(.+)$/i);
const candidate = kvMatch?.[1] ? kvMatch[1] : stripped;
return candidate.trim().replace(/[^\x20-\x7E]/g, "");
}
function normalizeRuxaRatio(ratio?: string) {
const normalized = String(ratio || "").trim();
if (normalized === "16:9" || normalized === "9:16" || normalized === "1:1") {
return normalized;
}
return "16:9";
}
function normalizeRuxaDuration(duration?: number) {
if (!Number.isFinite(duration)) return 10;
return Math.max(5, Math.min(10, Math.round(Number(duration))));
}
function normalizeRuxaModelId(modelId: string) {
return normalizeRuxaTextInput(modelId);
}
function extractTaskId(payload: any): string | undefined {
if (typeof payload === "string" && payload.trim()) return payload.trim();
if (typeof payload?.data === "string" && payload.data.trim()) return payload.data.trim();
return (
payload?.taskId ||
payload?.task_id ||
payload?.id ||
payload?.data?.taskId ||
payload?.data?.task_id ||
payload?.data?.id
);
}
function extractStatus(payload: any): string | undefined {
const raw =
payload?.status ||
payload?.taskStatus ||
payload?.task_status ||
payload?.state ||
payload?.data?.status ||
payload?.data?.taskStatus ||
payload?.data?.task_status ||
payload?.data?.state;
if (typeof raw !== "string") return undefined;
return raw.toLowerCase();
}
function extractErrorMessage(payload: any): string {
return (
payload?.message ||
payload?.error ||
payload?.errorMessage ||
payload?.data?.message ||
payload?.data?.error ||
"RUXA 视频任务执行失败"
);
}
function collectVideoUrls(payload: any): string[] {
const urls = new Set<string>();
const isVideoLikeKey = (key: string) => /video|url|output|result|file/i.test(key);
const isLikelyVideoUrl = (value: string) =>
/^https?:\/\//i.test(value) &&
(/\.(mp4|webm|mov|mkv)(\?|$)/i.test(value) ||
value.includes("/video") ||
value.includes("video"));
const walk = (value: any, parentKey = "") => {
if (typeof value === "string") {
if (isLikelyVideoUrl(value) || (isVideoLikeKey(parentKey) && /^https?:\/\//i.test(value))) {
urls.add(value);
}
return;
}
if (Array.isArray(value)) {
value.forEach((item) => walk(item, parentKey));
return;
}
if (value && typeof value === "object") {
Object.entries(value).forEach(([key, nested]) => walk(nested, key));
}
};
walk(payload);
return Array.from(urls);
}
function extractPromptAndImages(messages: Array<{ role: string; content: any }>) {
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
if (!lastUserMessage) {
throw new Error("No user message found");
}
let prompt = "";
const imageUrls: string[] = [];
const pushImageUrl = (url?: string) => {
if (!url) return;
const normalized = normalizeRuxaTextInput(String(url));
if (normalized.startsWith("data:")) {
throw new Error("RUXA 视频模型不支持 Base64 图片，请先上传图片");
}
if (!normalized.startsWith("http")) return;
if (!imageUrls.includes(normalized)) imageUrls.push(normalized);
};
if (typeof lastUserMessage.content === "string") {
prompt = lastUserMessage.content.trim();
} else if (Array.isArray(lastUserMessage.content)) {
for (const part of lastUserMessage.content) {
if (part?.type === "text" && typeof part.text === "string") {
prompt = `${prompt} ${part.text}`.trim();
} else if (part?.type === "image_url" && part.image_url?.url) {
pushImageUrl(part.image_url.url);
} else if (part?.type === "file") {
pushImageUrl(part.url || part.data);
} else if (part?.type === "image") {
pushImageUrl(part.image || part.url);
}
}
}
if (!prompt) {
prompt = "让画面动起来";
}
return { prompt, imageUrls };
}
async function waitForVideoUrl(baseURL: string, apiKey: string, taskId: string) {
const endpointCandidates = [
(id: string) => joinApiPath(baseURL, `/api/v1/tasks/query/${id}`),
(id: string) => joinApiPath(baseURL, `/api/v1/tasks/${id}`),
(id: string) => `${joinApiPath(baseURL, "/api/v1/tasks/query")}?taskId=${encodeURIComponent(id)}`,
];
let endpointIndex = 0;
for (let attempt = 1; attempt <= POLL_MAX_ATTEMPTS; attempt++) {
const endpoint = endpointCandidates[endpointIndex]!(taskId);
const response = await fetch(endpoint, {
headers: {
Authorization: `Bearer ${apiKey}`,
"Content-Type": "application/json",
},
});
if (!response.ok) {
if (response.status === 404 && endpointIndex < endpointCandidates.length - 1) {
endpointIndex += 1;
}
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
continue;
}
const payload = await response.json();
const videoUrls = collectVideoUrls(payload);
if (videoUrls.length > 0) {
return videoUrls[0]!;
}
const status = extractStatus(payload);
if (status && ["failed", "error", "cancelled", "canceled"].includes(status)) {
throw new Error(extractErrorMessage(payload));
}
if (status && ["success", "succeeded", "completed", "done", "finished"].includes(status)) {
throw new Error("任务已完成但未返回视频地址");
}
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
}
throw new Error("RUXA 视频任务超时，请稍后重试");
}
async function downloadAndUploadVideo(videoUrl: string) {
try {
const response = await fetch(videoUrl);
if (!response.ok) {
throw new Error(`Failed to download video: ${response.status}`);
}
const buffer = Buffer.from(await response.arrayBuffer());
const filename = `ruxa-video-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.mp4`;
const uploaded = await uploadFile(filename, buffer as any);
return uploaded.url;
} catch (error) {
console.error("[Ruxa Video] Upload to storage failed, fallback remote URL:", error);
return videoUrl;
}
}
function formatVideoMarkdown(videoUrl: string) {
return `<video src="${videoUrl}" controls style="max-width: 100%; border-radius: 8px; margin-bottom: 12px;"></video>\n\n[下载视频](${videoUrl})`;
}
function buildVideoInput(params: {
imageUrls: string[];
prompt: string;
videoParams?: RuxaVideoParams;
}) {
const { imageUrls, prompt, videoParams } = params;
if (!imageUrls[0]) {
throw new Error("RUXA Veo 3.1 需要至少 1 张起始图");
}
const duration = normalizeRuxaDuration(videoParams?.duration);
const ratio = normalizeRuxaRatio(videoParams?.ratio);
const enhancePrompt =
typeof videoParams?.enhancePrompt === "boolean" ? videoParams.enhancePrompt : true;
const enableUpsample =
typeof videoParams?.enableUpsample === "boolean"
? videoParams.enableUpsample
: false;
return {
prompt: normalizeRuxaTextInput(prompt),
image_url: imageUrls[0],
seconds: String(duration),
aspect_ratio: ratio,
enhance_prompt: enhancePrompt,
enable_upsample: enableUpsample,
};
}
export function createRuxaVideo({ apiKey, baseURL }: RuxaVideoConfig) {
return {
provider: "ruxa-video",
chat: (modelId: string) => ({
provider: "ruxa-video",
doGenerate: async (options: {
messages: Array<{ role: string; content: any }>;
videoParams?: RuxaVideoParams;
overrideModelId?: string;
}) => {
try {
const credentials = normalizeRuxaCredentials(apiKey);
if (!credentials) {
throw new Error("RUXA API Key 未配置或格式无效");
}
const activeModelId = normalizeRuxaModelId(options.overrideModelId || modelId);
const { prompt, imageUrls } = extractPromptAndImages(options.messages);
const input = buildVideoInput({
imageUrls,
prompt,
videoParams: options.videoParams,
});
const callbackUrl =
process.env.RUXA_VIDEO_CALLBACK_URL?.trim() || process.env.RUXA_CALLBACK_URL?.trim();
const requestBody = {
...(callbackUrl ? { callBackUrl: callbackUrl } : {}),
input,
model: activeModelId,
};
const createEndpoint = joinApiPath(baseURL, "/api/v1/tasks/create");
const createResponse = await fetch(createEndpoint, {
method: "POST",
headers: {
Authorization: `Bearer ${credentials}`,
"Content-Type": "application/json",
},
body: JSON.stringify(requestBody),
});
const createRawText = await createResponse.text();
let createPayload: any = null;
try {
createPayload = JSON.parse(createRawText);
} catch {
createPayload = createRawText;
}
if (!createResponse.ok) {
throw new Error(`${extractErrorMessage(createPayload)} (HTTP ${createResponse.status})`);
}
let remoteVideoUrl = collectVideoUrls(createPayload)[0];
if (!remoteVideoUrl) {
const taskId = extractTaskId(createPayload);
if (!taskId) {
throw new Error("RUXA 创建任务成功，但未返回任务ID");
}
remoteVideoUrl = await waitForVideoUrl(baseURL, credentials, taskId);
}
if (!remoteVideoUrl) {
throw new Error("RUXA 未返回可用视频地址");
}
const hostedVideoUrl = await downloadAndUploadVideo(remoteVideoUrl);
const text = formatVideoMarkdown(hostedVideoUrl);
return {
text,
finishReason: "stop" as const,
usage: {
promptTokens: 0,
completionTokens: text.length,
},
};
} catch (error) {
console.error("[Ruxa Video] Generation failed:", error);
const errorText =
error instanceof Error
? `视频生成失败: ${error.message}`
: "视频生成失败，请稍后重试";
return {
text: errorText,
finishReason: "error" as const,
usage: {
promptTokens: 0,
completionTokens: 0,
},
};
}
},
doStream: async function* (options: {
messages: Array<{ role: string; content: any }>;
videoParams?: RuxaVideoParams;
overrideModelId?: string;
}) {
const result = await this.doGenerate(options);
yield {
type: "text-delta",
textDelta: result.text,
};
},
modelId,
specificationVersion: "v1" as const,
defaultObjectGenerationMode: "json" as const,
}),
};
}

5
lib/ai/smart-analyzer.ts
View File

@ -368,7 +368,8 @@ function shouldPreferRemoteVisionUrls(modelWithProvider?: {
providerType === "volcano-vision" ||
providerType === "volcano-multimodal" ||
providerType === "doubao-video" ||
providerType === "volcano-video"
providerType === "volcano-video" ||
providerType === "ruxa-video"
) {
return true;
}
@ -1074,7 +1075,7 @@ type VisionSummaryModel = Awaited<
function isVisionSummaryCandidateModel(modelWithProvider: VisionSummaryModel): boolean {
const providerType = String(modelWithProvider.provider?.type || "");
if (["fal-video", "volcano-video", "qianwen-video", "doubao-video"].includes(providerType)) {
if (["fal-video", "volcano-video", "qianwen-video", "doubao-video", "ruxa-video"].includes(providerType)) {
return false;
}
return isLikelyVisionCapableModel(modelWithProvider);

3
lib/db/migrate.ts
View File

@ -19,6 +19,7 @@ const providerBaseUrlDefaults = [
baseUrl: "https://dashscope.aliyuncs.com/api/v1/services/aigc/video-generation",
},
{ type: "doubao-video", baseUrl: "https://ark.cn-beijing.volces.com/api/v3" },
{ type: "ruxa-video", baseUrl: "https://api.ruxa.ai" },
{ type: "volcano-vision", baseUrl: "https://ark.cn-beijing.volces.com" },
{ type: "volcano-video", baseUrl: "https://ark.cn-beijing.volces.com/api/v3" },
{
@ -88,7 +89,7 @@ const runMigrate = async () => {
ADD CONSTRAINT "AiProvider_type_check"
CHECK (type IN (
'alibaba', 'openai', 'custom', 'local', 'gemini',
'huanke-gemini', 'qianwen-video', 'doubao-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'ruxa-image', 'fal-image', 'fal-video', 'zhipu', 'tripo-3d'
'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'ruxa-image', 'fal-image', 'fal-video', 'zhipu', 'tripo-3d'
));
`);
for (const item of providerBaseUrlDefaults) {

4
lib/db/queries.ts
View File

@ -1250,7 +1250,7 @@ export async function createAiProvider({
updatedBy,
}: {
name: string;
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d";
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d";
baseUrl: string;
apiKey: string;
isEnabled?: boolean;
@ -1288,7 +1288,7 @@ export async function updateAiProvider({
}: {
id: string;
name?: string;
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d";
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "fal-video" | "zhipu" | "tripo-3d";
baseUrl?: string;
apiKey?: string;
isEnabled?: boolean;

2
lib/db/schema.ts
View File

@ -259,7 +259,7 @@ export type ToolConfig = InferSelectModel<typeof toolConfig>;
export const aiProvider = pgTable("AiProvider", {
id: uuid("id").primaryKey().notNull().defaultRandom(),
name: varchar("name", { length: 128 }).notNull(), // 提供商名称，如 "阿里云通义千问"
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "doubao-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "fal-video", "zhipu", "tripo-3d"] })
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "fal-video", "zhipu", "tripo-3d"] })
.notNull()
.default("custom"), // 提供商类型
baseUrl: text("baseUrl").notNull(), // API 基础 URL