feat: add midjourney provider and failover exclusion
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled

This commit is contained in:
Your Name 2026-03-07 16:25:48 +08:00
parent 864b6d41e3
commit bbdda81686
16 changed files with 659 additions and 71 deletions

View File

@ -147,7 +147,7 @@ type ToolConfig = {
type AiProvider = {
id: string;
name: string;
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal";
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal";
baseUrl: string;
apiKey: string;
isEnabled: boolean;
@ -184,6 +184,9 @@ type AiModel = {
isHidden: boolean;
isDefault: boolean;
sortOrder: number;
fallbackModelId1?: string;
fallbackModelId2?: string;
excludeFromFailover?: boolean;
isAgentModel?: boolean;
agentCodingModelId?: string;
isMultimodal?: boolean;
@ -438,6 +441,7 @@ const PROVIDER_TYPES = [
{ value: "ruxa-image", label: "RUXA生图" },
{ value: "fal-image", label: "FAL 生图模型" },
{ value: "openai-image", label: "OpenAI 图片兼容" },
{ value: "midjourney", label: "Midjourney" },
{ value: "tripo-3d", label: "Tripo3D 模型生成" },
{ value: "zhipu", label: "智普官方 4.7" },
{ value: "volcano-multimodal", label: "火山引擎多模态" },
@ -461,6 +465,7 @@ const PROVIDER_DEFAULT_BASE_URL: Record<string, string> = {
"ruxa-image": "https://api.ruxa.ai",
"fal-image": "https://fal.run",
"openai-image": "https://api.openai.com/v1",
midjourney: "https://your-midjourney-api.example.com",
"fal-video": "https://fal.run",
"tripo-3d": "https://api.tripo3d.ai/v2/openapi",
zhipu: "https://open.bigmodel.cn/api/paas/v4",
@ -1327,7 +1332,7 @@ export function AdminSettingsClient() {
const [providerForm, setProviderForm] = useState({
id: "",
name: "",
type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal",
type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal",
baseUrl: "",
apiKey: "",
isEnabled: true,
@ -1354,6 +1359,7 @@ export function AdminSettingsClient() {
sortOrder: 0,
fallbackModelId1: "", // 备用模型 ID 1
fallbackModelId2: "", // 备用模型 ID 2
excludeFromFailover: false,
// 🆕 图片生成模型特性配置
imageGenConfig: {
maxImages: 2,
@ -2426,6 +2432,7 @@ export function AdminSettingsClient() {
sortOrder: 0,
fallbackModelId1: "",
fallbackModelId2: "",
excludeFromFailover: false,
imageGenConfig: {
maxImages: 2,
supportsSize: false,
@ -3697,6 +3704,7 @@ export function AdminSettingsClient() {
"ruxa-image": "bg-emerald-500",
"fal-image": "bg-sky-500",
"openai-image": "bg-green-600",
"midjourney": "bg-fuchsia-600",
"tripo-3d": "bg-teal-500",
"openai": "bg-green-500",
"zhipu": "bg-lime-500",
@ -6860,6 +6868,15 @@ export function AdminSettingsClient() {
/>
<Label>多模态模型(Agent 识图与分析使用同一模型)</Label>
</div>
<div className="flex items-center gap-2">
<Switch
checked={modelForm.excludeFromFailover || false}
onCheckedChange={(checked) =>
setModelForm({ ...modelForm, excludeFromFailover: checked })
}
/>
<Label>禁止参与故障迁移(不作为其他模型的自动备用模型)</Label>
</div>
{/* 🆕 图片生成模型特性配置 */}
{(modelForm.supportsImageGen || isVideoProviderForModelConfig) && (
@ -7096,6 +7113,7 @@ export function AdminSettingsClient() {
{model.isAgentModel && <span className="text-xs bg-violet-100 text-violet-800 px-2 py-0.5 rounded">Agent</span>}
{model.isMultimodal && <span className="text-xs bg-emerald-100 text-emerald-800 px-2 py-0.5 rounded">多模态</span>}
{model.isHidden && <span className="text-xs bg-amber-100 text-amber-800 px-2 py-0.5 rounded">前台隐藏</span>}
{(model as any).excludeFromFailover && <span className="text-xs bg-rose-100 text-rose-700 px-2 py-0.5 rounded">不参与故障迁移</span>}
</div>
<div className="text-sm text-muted-foreground truncate">
ID: {model.modelId} | 提供商: {provider?.name ?? "未知"}
@ -7122,6 +7140,7 @@ export function AdminSettingsClient() {
sortOrder: model.sortOrder || 0,
fallbackModelId1: (model as any).fallbackModelId1 || "",
fallbackModelId2: (model as any).fallbackModelId2 || "",
excludeFromFailover: (model as any).excludeFromFailover || false,
imageGenConfig: (model as any).imageGenConfig || {
maxImages: 2,
supportsSize: false,
@ -8728,7 +8747,7 @@ export function AdminSettingsClient() {
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
if (!provider) return false;
// 支持阿里生图、Gemini、既梦等图片模型
const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini-image", "gemini", "volcano-vision"];
const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "gemini-image", "gemini", "volcano-vision"];
return imageProviderTypes.includes(provider.type);
})
.map((m: AiModel) => {
@ -8743,7 +8762,7 @@ export function AdminSettingsClient() {
{/* 如果没有找到生图模型,显示提示 */}
{(modelsData?.models || []).filter((m: AiModel) => {
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini-image", "gemini", "volcano-vision"];
const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "gemini-image", "gemini", "volcano-vision"];
return provider && imageProviderTypes.includes(provider.type);
}).length === 0 && (
<SelectItem value="no-model" disabled>

View File

@ -280,6 +280,7 @@ function normalizeImageLoadingProviderType(providerType?: string | null): string
normalized === "ruxa-image" ||
normalized === "fal-image" ||
normalized === "openai-image" ||
normalized === "midjourney" ||
normalized === "volcano-vision" ||
normalized === "gemini"
) {
@ -974,6 +975,56 @@ function setImageFallbackModelCooldown(modelId: string, reason: string): void {
});
}
function isImageFailoverExcludedModel(model: { excludeFromFailover?: boolean } | null | undefined) {
return Boolean(model?.excludeFromFailover);
}
function buildImageFallbackModelIds(
allModels: Array<{ model: any; provider: any }>,
options: {
currentProviderId?: string;
currentModelId?: string;
currentModelKey?: string;
configuredFallbackIds?: string[];
}
) {
const { currentProviderId, currentModelId, currentModelKey, configuredFallbackIds = [] } = options;
const eligibleFallbackModelIds = new Set(
allModels
.filter(
(item) =>
item.model.supportsImageGen &&
item.model.isEnabled &&
!item.model.isSubModel &&
!isImageFailoverExcludedModel(item.model) &&
item.model.modelId !== currentModelKey &&
item.model.id !== currentModelId
)
.map((item) => String(item.model.modelId || ""))
.filter(Boolean)
);
const sameProviderFallbacks = currentProviderId
? allModels
.filter(
(item) =>
item.provider.id === currentProviderId &&
item.model.supportsImageGen &&
item.model.isEnabled &&
!item.model.isSubModel &&
!isImageFailoverExcludedModel(item.model) &&
item.model.modelId !== currentModelKey &&
item.model.id !== currentModelId
)
.map((item) => item.model.modelId)
: [];
return Array.from(new Set([...configuredFallbackIds, ...sameProviderFallbacks])).filter(
(modelId) => Boolean(modelId) && eligibleFallbackModelIds.has(String(modelId))
);
}
function getImageCrossProviderCandidates(
allModels: any[],
options: {
@ -984,7 +1035,12 @@ function getImageCrossProviderCandidates(
) {
return allModels
.filter((m) => {
if (!m?.model?.supportsImageGen || !m?.model?.isEnabled || m?.model?.isSubModel) {
if (
!m?.model?.supportsImageGen ||
!m?.model?.isEnabled ||
m?.model?.isSubModel ||
isImageFailoverExcludedModel(m?.model)
) {
return false;
}
if (!IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(String(m?.provider?.type || ""))) {
@ -1438,7 +1494,12 @@ function ensureImageAnalysisSummarySection(message: string, summary: string): st
* 因为这些 provider 内部会自行处理图片上传到 S3
*/
async function convertMessagesWithImages(messages: ChatMessage[], providerType: string) {
if (providerType === "ruxa-image" || providerType === "fal-image" || providerType === "openai-image") {
if (
providerType === "ruxa-image" ||
providerType === "fal-image" ||
providerType === "openai-image" ||
providerType === "midjourney"
) {
const convertedMessages = [];
for (const message of messages) {
@ -2407,6 +2468,8 @@ export async function POST(request: Request) {
let shouldKeepSelectedModel = false;
const requestedModelId = selectedChatModel;
let requestedModelProviderType: string | undefined;
let requestedModelIsFixedVisual = false;
let requestedModelIsConversationMultimodal = false;
if (!shouldRunAgentConversation && requestedModelId !== "auto") {
const selectedModelIsUUID =
@ -2444,6 +2507,7 @@ export async function POST(request: Request) {
"ruxa-image",
"fal-image",
"openai-image",
"midjourney",
"volcano-vision",
]);
const isFixedVisualModel =
@ -2460,6 +2524,8 @@ export async function POST(request: Request) {
selectedModelConfig.supportsVision &&
!selectedModelConfig.supportsImageGen &&
!isVideoProviderType(selectedModelProviderType);
requestedModelIsFixedVisual = isFixedVisualModel;
requestedModelIsConversationMultimodal = isConversationMultimodalModel;
if (deepThinkingOn) {
if (isAiModelAgentAuto(selectedModelConfig)) {
@ -2755,7 +2821,13 @@ export async function POST(request: Request) {
console.log("[Smart Auto Mode] 🎛️ Agent recommended image params:", smartAutoRecommendedImageParams);
}
if (autoGraphResult.route && !shouldKeepSelectedModel) {
const shouldForceConversationalRouteForVisualModel =
shouldKeepSelectedModel &&
requestedModelIsFixedVisual &&
!requestedModelIsConversationMultimodal &&
Boolean(smartAutoExecution?.shouldStayConversational);
if (autoGraphResult.route && (!shouldKeepSelectedModel || shouldForceConversationalRouteForVisualModel)) {
const routeIsVideo = isVideoProviderType(autoGraphResult.route.providerType);
if (routeIsVideo && !allowAgentVideoGeneration) {
const fallbackIntent: "image_edit" | "image_gen" = hasImage ? "image_edit" : "image_gen";
@ -2808,7 +2880,9 @@ export async function POST(request: Request) {
} else {
finalModelId = autoGraphResult.route.id;
console.log(
`[Smart Auto Mode] ✅ Routed to: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`,
shouldForceConversationalRouteForVisualModel
? `[Smart Auto Mode] ✅ Switched visual model to conversational route: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`
: `[Smart Auto Mode] ✅ Routed to: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`,
);
}
} else if (autoGraphResult.route && shouldKeepSelectedModel) {
@ -4102,6 +4176,7 @@ export async function POST(request: Request) {
(languageModel as any).provider === "ruxa-image" ||
(languageModel as any).provider === "fal-image" ||
(languageModel as any).provider === "openai-image" ||
(languageModel as any).provider === "midjourney" ||
(languageModel as any).provider === "huanke-gemini";
const isUuidModelId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
@ -4218,18 +4293,12 @@ export async function POST(request: Request) {
currentModelConfig?.fallbackModelId2,
].filter(Boolean) as string[];
const allModels = await getEnabledAiModelsWithProvider();
const sameProviderFallbacks = currentModelConfig
? allModels
.filter(m =>
m.provider.id === currentModelConfig.providerId &&
m.model.supportsImageGen &&
m.model.isEnabled &&
!m.model.isSubModel &&
m.model.modelId !== actualModelId
)
.map(m => m.model.modelId)
: [];
const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks]));
const fallbackModelIds = buildImageFallbackModelIds(allModels, {
currentProviderId: currentModelConfig?.providerId,
currentModelId: currentModelConfig?.id,
currentModelKey: actualModelId,
configuredFallbackIds: configuredFallbacks,
});
console.log("[Chat API] Fallback model IDs:", fallbackModelIds);
const geminiStream = createUIMessageStream({
@ -4649,7 +4718,7 @@ export async function POST(request: Request) {
// ========== Gemini 处理结束 ==========
// ========== 阿里云千问图片生成独立处理 ==========
const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image", "fal-image", "openai-image"].includes((languageModel as any).provider);
const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney"].includes((languageModel as any).provider);
if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") {
console.log("[Chat API] Using image generation provider:", (languageModel as any).provider);
console.log("[Chat API] Image params:", effectiveImageParams);
@ -4662,18 +4731,12 @@ export async function POST(request: Request) {
alibabaModelConfig?.fallbackModelId2,
].filter(Boolean) as string[];
const allModels = await getEnabledAiModelsWithProvider();
const sameProviderFallbacks = alibabaModelConfig
? allModels
.filter(m =>
m.provider.id === alibabaModelConfig.providerId &&
m.model.supportsImageGen &&
m.model.isEnabled &&
!m.model.isSubModel &&
m.model.modelId !== actualModelId
)
.map(m => m.model.modelId)
: [];
const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks]));
const fallbackModelIds = buildImageFallbackModelIds(allModels, {
currentProviderId: alibabaModelConfig?.providerId,
currentModelId: alibabaModelConfig?.id,
currentModelKey: actualModelId,
configuredFallbackIds: configuredFallbacks,
});
console.log("[Chat API] Alibaba fallback model IDs:", fallbackModelIds);
const alibabaStream = createUIMessageStream({
@ -4741,6 +4804,8 @@ export async function POST(request: Request) {
? "fal-image"
: (languageModel as any).provider === "openai-image"
? "openai-image"
: (languageModel as any).provider === "midjourney"
? "midjourney"
: "alibaba-image";
const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert);
@ -4841,6 +4906,8 @@ export async function POST(request: Request) {
? "fal-image"
: candidate.provider.type === "openai-image"
? "openai-image"
: candidate.provider.type === "midjourney"
? "midjourney"
: candidate.provider.type === "gemini"
? "gemini"
: candidate.provider.type === "huanke-gemini"
@ -5060,18 +5127,12 @@ export async function POST(request: Request) {
volcanoModelConfig?.fallbackModelId2,
].filter(Boolean) as string[];
const allModels = await getEnabledAiModelsWithProvider();
const sameProviderFallbacks = volcanoModelConfig
? allModels
.filter(m =>
m.provider.id === volcanoModelConfig.providerId &&
m.model.supportsImageGen &&
m.model.isEnabled &&
!m.model.isSubModel &&
m.model.modelId !== actualModelId
)
.map(m => m.model.modelId)
: [];
const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks]));
const fallbackModelIds = buildImageFallbackModelIds(allModels, {
currentProviderId: volcanoModelConfig?.providerId,
currentModelId: volcanoModelConfig?.id,
currentModelKey: actualModelId,
configuredFallbackIds: configuredFallbacks,
});
console.log("[Chat API] Volcano fallback model IDs:", fallbackModelIds);
if (hasAnnotation) {
@ -5230,6 +5291,8 @@ export async function POST(request: Request) {
? "fal-image"
: candidate.provider.type === "openai-image"
? "openai-image"
: candidate.provider.type === "midjourney"
? "midjourney"
: candidate.provider.type === "gemini"
? "gemini"
: candidate.provider.type === "huanke-gemini"
@ -5754,21 +5817,12 @@ export async function POST(request: Request) {
currentModelConfig?.fallbackModelId2,
].filter(Boolean) as string[];
const allModels = await getEnabledAiModelsWithProvider();
const sameProviderFallbacks = currentModelConfig
? allModels
.filter(
(m) =>
m.provider.id === currentModelConfig.providerId &&
m.model.supportsImageGen &&
m.model.isEnabled &&
!m.model.isSubModel &&
m.model.modelId !== actualModelId
)
.map((m) => m.model.modelId)
: [];
const fallbackModelIds = Array.from(
new Set([...configuredFallbacks, ...sameProviderFallbacks])
).filter((modelId) => modelId !== actualModelId);
const fallbackModelIds = buildImageFallbackModelIds(allModels, {
currentProviderId: currentModelConfig?.providerId,
currentModelId: currentModelConfig?.id,
currentModelKey: actualModelId,
configuredFallbackIds: configuredFallbacks,
});
console.log("[Chat API] OpenAI-compatible fallback model IDs:", fallbackModelIds);
const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop();
@ -5901,6 +5955,8 @@ export async function POST(request: Request) {
? "fal-image"
: candidate.provider.type === "openai-image"
? "openai-image"
: candidate.provider.type === "midjourney"
? "midjourney"
: candidate.provider.type === "gemini"
? "gemini"
: candidate.provider.type === "huanke-gemini"

View File

@ -47,6 +47,7 @@ const createModelSchema = z.object({
sortOrder: z.number().optional(),
fallbackModelId1: z.string().max(128).optional(),
fallbackModelId2: z.string().max(128).optional(),
excludeFromFailover: z.boolean().optional(),
isAgentModel: z.boolean().optional(),
agentCodingModelId: z.string().max(128).optional(),
isMultimodal: z.boolean().optional(),
@ -68,6 +69,7 @@ const updateModelSchema = z.object({
sortOrder: z.number().optional(),
fallbackModelId1: z.string().max(128).optional(),
fallbackModelId2: z.string().max(128).optional(),
excludeFromFailover: z.boolean().optional(),
isAgentModel: z.boolean().optional(),
agentCodingModelId: z.string().max(128).optional(),
isMultimodal: z.boolean().optional(),

View File

@ -25,6 +25,7 @@ const providerTypes = [
"ruxa-image",
"fal-image",
"openai-image",
"midjourney",
"fal-video",
"zhipu",
"tripo-3d",
@ -56,6 +57,7 @@ const DOUBAO_VIDEO_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3";
const RUXA_VIDEO_BASE_URL = "https://api.ruxa.ai";
const RUXA_IMAGE_BASE_URL = "https://api.ruxa.ai";
const OPENAI_IMAGE_BASE_URL = "https://api.openai.com/v1";
const MIDJOURNEY_BASE_URL = "https://your-midjourney-api.example.com";
const TRIPO_3D_BASE_URL = "https://api.tripo3d.ai/v2/openapi";
function normalizeProviderTypeForStorage(
@ -169,6 +171,8 @@ export async function POST(request: Request) {
? data.baseUrl || RUXA_IMAGE_BASE_URL
: data.type === "openai-image"
? data.baseUrl || OPENAI_IMAGE_BASE_URL
: data.type === "midjourney"
? data.baseUrl || MIDJOURNEY_BASE_URL
: data.type === "tripo-3d"
? data.baseUrl || TRIPO_3D_BASE_URL
: data.baseUrl,

View File

@ -49,7 +49,7 @@ function normalizeImageParamsForProvider(
const normalized = { ...params };
const currentCount = normalized.imageCount || 1;
if (providerType === "ruxa-image" || providerType === "openai-image") {
if (providerType === "ruxa-image" || providerType === "openai-image" || providerType === "midjourney") {
normalized.imageCount = 1;
return normalized;
}
@ -1053,7 +1053,8 @@ export function Chat({
model?.providerType === "alibaba-image" ||
model?.providerType === "ruxa-image" ||
model?.providerType === "fal-image" ||
model?.providerType === "openai-image"
model?.providerType === "openai-image" ||
model?.providerType === "midjourney"
) modelType = "alibaba-image";
// 智能自动模式下,如果当前轮输入包含图片,默认按图片生成链路跟踪

View File

@ -21,6 +21,7 @@ const PROVIDER_MODEL_LABEL: Record<string, string> = {
"ruxa-image": "RUXA 图片",
"fal-image": "FAL 图片",
"openai-image": "OpenAI 图片",
midjourney: "Midjourney",
"volcano-vision": "即梦 4.5",
gemini: "Gemini 图片",
};

View File

@ -87,7 +87,7 @@ export function ImageParamsBar({
}: ImageParamsBarProps) {
const getImageCountOptions = () => {
// RUXA 当前按单图任务处理(优先级最高)
if (providerType === "ruxa-image" || providerType === "openai-image") {
if (providerType === "ruxa-image" || providerType === "openai-image" || providerType === "midjourney") {
return [1];
}
// 优先使用模型配置中的 maxImages

View File

@ -124,7 +124,8 @@ function PureMessages({
(m.providerType === "alibaba-image" ||
m.providerType === "ruxa-image" ||
m.providerType === "fal-image" ||
m.providerType === "openai-image")
m.providerType === "openai-image" ||
m.providerType === "midjourney")
) ?? false;
// 🆕 获取当前模型的显示名称
@ -683,7 +684,12 @@ function PureMessages({
return (
<ImageGenerationLoading
imageCount={imageCount}
maxImages={currentModelProviderType === "openai-image" ? 1 : 6}
maxImages={
currentModelProviderType === "openai-image" ||
currentModelProviderType === "midjourney"
? 1
: 6
}
providerType={
currentModelProviderType === "ruxa-image"
? "ruxa-image"
@ -691,6 +697,8 @@ function PureMessages({
? "fal-image"
: currentModelProviderType === "openai-image"
? "openai-image"
: currentModelProviderType === "midjourney"
? "midjourney"
: "alibaba-image"
}
modelName={currentModelDisplayName}

View File

@ -1792,7 +1792,8 @@ function PureModelSelectorCompact({
providerType === "alibaba-image" ||
providerType === "ruxa-image" ||
providerType === "fal-image" ||
providerType === "openai-image"
providerType === "openai-image" ||
providerType === "midjourney"
) {
return <ImageIcon className="h-4 w-4" strokeWidth={2.2} />;
}

View File

@ -21,6 +21,7 @@ import { createAlibabaImage } from "./providers/alibaba-image";
import { createRuxaImage } from "./providers/ruxa-image";
import { createFalImage } from "./providers/fal-image";
import { createOpenAIImage } from "./providers/openai-image";
import { createMidjourney } from "./providers/midjourney";
import { createFalVideo } from "./providers/fal-video";
import { createRuxaVideo } from "./providers/ruxa-video";
import { createVolcanoMultimodal } from "./providers/volcano-multimodal";
@ -217,6 +218,15 @@ function getOrCreateProvider(provider: Pick<AiProvider, "id" | "type" | "baseUrl
return openAIImageProvider as any;
}
case "midjourney": {
const midjourneyProvider = createMidjourney({
baseURL: resolvedBaseUrl,
apiKey: provider.apiKey,
});
console.log(`[Provider] Created Midjourney generation provider: ${resolvedBaseUrl}`);
return midjourneyProvider as any;
}
case "custom":
default: {
if (isVolcanoMultimodalBaseUrl(resolvedBaseUrl)) {
@ -339,6 +349,10 @@ export async function getDynamicModel(
console.log("[getDynamicModel] Getting OpenAI Image chat model..."); // 调试日志
baseModel = providerInstance.chat(resolvedModelId);
console.log("[getDynamicModel] OpenAI Image model retrieved:", baseModel ? "success" : "failed"); // 调试日志
} else if (provider.type === "midjourney") {
console.log("[getDynamicModel] Getting Midjourney chat model..."); // 调试日志
baseModel = providerInstance.chat(resolvedModelId);
console.log("[getDynamicModel] Midjourney model retrieved:", baseModel ? "success" : "failed"); // 调试日志
} else if (provider.type === "huanke-gemini") {
// 环科 Gemini 原生格式使用 chat() 方法
console.log("[getDynamicModel] Getting Huanke Gemini chat model..."); // 调试日志

View File

@ -14,6 +14,7 @@ const PROVIDER_DEFAULT_BASE_URL: Partial<Record<AiProvider["type"], string>> = {
"ruxa-image": "https://api.ruxa.ai",
"fal-image": "https://fal.run",
"openai-image": "https://api.openai.com/v1",
midjourney: "https://your-midjourney-api.example.com",
"fal-video": "https://fal.run",
"tripo-3d": "https://api.tripo3d.ai/v2/openapi",
};

View File

@ -0,0 +1,459 @@
import "server-only";
import sharp from "sharp";
import { uploadFile } from "@/lib/storage/service";
interface MidjourneyConfig {
apiKey: string;
baseURL: string;
}
export interface MidjourneyImageParams {
aspectRatio?: string;
imageSize?: string;
imageCount?: number;
}
interface UploadedImage {
thumbnail: string;
original: string;
}
interface MidjourneyTaskButton {
customId?: string;
label?: string;
emoji?: string;
style?: number;
type?: number;
}
interface MidjourneyTaskResult {
id?: string;
imageUrl?: string;
status?: string;
progress?: string;
failReason?: string;
description?: string;
prompt?: string;
buttons?: MidjourneyTaskButton[];
}
const DEFAULT_TIMEOUT_MS = 300000;
const POLL_INTERVAL_MS = 3500;
const DEFAULT_IMAGE_SIZE = "2K";
function normalizeBaseURL(baseURL: string) {
return String(baseURL || "").trim().replace(/\/+$/, "");
}
function resolveMode(baseURL: string, modelId: string) {
const normalizedBaseURL = normalizeBaseURL(baseURL).toLowerCase();
const normalizedModelId = String(modelId || "").trim().toLowerCase();
if (
normalizedBaseURL.endsWith("/relax") ||
normalizedBaseURL.endsWith("/mj-relax") ||
normalizedModelId.includes("relax")
) {
return "relax" as const;
}
return "fast" as const;
}
function buildEndpointCandidates(baseURL: string, mode: "fast" | "relax", suffix: string) {
const normalizedBaseURL = normalizeBaseURL(baseURL);
const normalizedSuffix = suffix.startsWith("/") ? suffix : `/${suffix}`;
const directPattern = /\/(fast|relax|mj-fast|mj-relax)$/i;
if (directPattern.test(normalizedBaseURL)) {
return [`${normalizedBaseURL}${normalizedSuffix}`];
}
const modePrefix = mode === "relax" ? "relax" : "fast";
const altPrefix = mode === "relax" ? "mj-relax" : "mj-fast";
return [
`${normalizedBaseURL}/${modePrefix}${normalizedSuffix}`,
`${normalizedBaseURL}/${altPrefix}${normalizedSuffix}`,
];
}
function normalizeImageSizeLabel(imageSize?: string): "1K" | "2K" | "4K" {
const normalized = String(imageSize || "").trim().toUpperCase();
if (normalized === "1K" || normalized === "2K" || normalized === "4K") return normalized;
return DEFAULT_IMAGE_SIZE;
}
function normalizeAspectRatio(value?: string) {
const normalized = String(value || "").trim();
if (!normalized || normalized.toLowerCase() === "auto") return undefined;
if (/^\d+:\d+$/.test(normalized)) return normalized;
return undefined;
}
function appendAspectRatio(prompt: string, aspectRatio?: string) {
const normalizedPrompt = String(prompt || "").trim();
const normalizedAspectRatio = normalizeAspectRatio(aspectRatio);
if (!normalizedAspectRatio) return normalizedPrompt;
if (/--ar\s+\d+:\d+/i.test(normalizedPrompt)) return normalizedPrompt;
return `${normalizedPrompt} --ar ${normalizedAspectRatio}`.trim();
}
function detectImageExtension(imageUrl: string, contentType?: string | null) {
const type = String(contentType || "").toLowerCase();
if (type.includes("jpeg") || type.includes("jpg")) return "jpg";
if (type.includes("webp")) return "webp";
if (type.includes("png")) return "png";
if (type.includes("gif")) return "gif";
const cleanUrl = imageUrl.split("?")[0]?.toLowerCase() || "";
if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "jpg";
if (cleanUrl.endsWith(".webp")) return "webp";
if (cleanUrl.endsWith(".gif")) return "gif";
if (cleanUrl.endsWith(".png")) return "png";
return "png";
}
async function createThumbnail(buffer: Buffer) {
try {
return await sharp(buffer)
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
.jpeg({ quality: 80 })
.toBuffer();
} catch (error) {
console.error("[Midjourney] Failed to create thumbnail:", error);
return buffer;
}
}
function generateFilename(prefix: string, extension: string) {
const timestamp = Date.now();
const random = Math.random().toString(36).slice(2, 8);
return `${prefix}-${timestamp}-${random}.${extension}`;
}
async function uploadGeneratedBuffer(
buffer: Buffer,
extension: string,
mimeType: string
): Promise<UploadedImage> {
const thumbnailBuffer = await createThumbnail(buffer);
const originalFilename = generateFilename("midjourney-orig", extension);
const originalArrayBuffer = buffer.buffer.slice(
buffer.byteOffset,
buffer.byteOffset + buffer.byteLength
) as ArrayBuffer;
const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer);
const thumbFilename = generateFilename("midjourney-thumb", "jpg");
const thumbArrayBuffer = thumbnailBuffer.buffer.slice(
thumbnailBuffer.byteOffset,
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
) as ArrayBuffer;
const thumbUploaded = await uploadFile(thumbFilename, thumbArrayBuffer);
return {
thumbnail: thumbUploaded.url,
original: originalUploaded.url,
};
}
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
const response = await fetch(imageUrl, { cache: "no-store" });
if (!response.ok) {
throw new Error(`Failed to download Midjourney image: ${response.status}`);
}
const buffer = Buffer.from(await response.arrayBuffer());
const extension = detectImageExtension(imageUrl, response.headers.get("content-type"));
const mimeType =
response.headers.get("content-type") ||
`image/${extension === "jpg" ? "jpeg" : extension}`;
return uploadGeneratedBuffer(buffer, extension, mimeType);
}
function extractPromptAndImages(messages: Array<{ role: string; content: any }>) {
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
if (!lastUserMessage) {
throw new Error("No user message found");
}
let prompt = "";
const imageUrls: string[] = [];
const pushImageUrl = (value?: string) => {
if (!value) return;
const normalized = String(value).trim();
if (!normalized) return;
if (!imageUrls.includes(normalized)) {
imageUrls.push(normalized);
}
};
if (typeof lastUserMessage.content === "string") {
prompt = lastUserMessage.content.trim();
} else if (Array.isArray(lastUserMessage.content)) {
for (const part of lastUserMessage.content) {
if (part?.type === "text" && typeof part.text === "string") {
prompt = `${prompt} ${part.text}`.trim();
} else if (part?.type === "image_url" && part.image_url?.url) {
pushImageUrl(part.image_url.url);
} else if (part?.type === "file") {
pushImageUrl(part.url || part.data);
} else if (part?.type === "image") {
pushImageUrl(part.image || part.url);
}
}
}
if (!prompt) {
throw new Error("No prompt found in user message");
}
return { prompt, imageUrls };
}
async function loadInputImageAsDataUrl(source: string) {
const normalized = String(source || "").trim();
if (!normalized) {
throw new Error("Midjourney 输入图片地址为空");
}
if (normalized.startsWith("data:")) {
return normalized;
}
const response = await fetch(normalized, { cache: "no-store" });
if (!response.ok) {
throw new Error(`Failed to fetch Midjourney input image: ${response.status}`);
}
const mimeType = response.headers.get("content-type") || "image/png";
const buffer = Buffer.from(await response.arrayBuffer());
return `data:${mimeType};base64,${buffer.toString("base64")}`;
}
async function parseJsonResponse(response: Response) {
const rawText = await response.text();
try {
return JSON.parse(rawText);
} catch {
return rawText;
}
}
function extractErrorMessage(payload: any) {
if (!payload) return "Midjourney 请求失败";
if (typeof payload === "string") return payload;
if (payload.failReason) return String(payload.failReason);
if (payload.description) return String(payload.description);
if (payload.message) return String(payload.message);
if (payload.error?.message) return String(payload.error.message);
try {
return JSON.stringify(payload);
} catch {
return "Midjourney 请求失败";
}
}
async function requestWithEndpointFallback(params: {
baseURL: string;
mode: "fast" | "relax";
suffix: string;
method: "GET" | "POST";
apiKey: string;
body?: string;
contentType?: string;
}) {
const { baseURL, mode, suffix, method, apiKey, body, contentType } = params;
const endpoints = buildEndpointCandidates(baseURL, mode, suffix);
let lastPayload: any = null;
let lastStatus: number | null = null;
let lastEndpoint = endpoints[0];
for (const endpoint of endpoints) {
lastEndpoint = endpoint;
const response = await fetch(endpoint, {
method,
headers: {
Authorization: `Bearer ${apiKey}`,
...(contentType ? { "Content-Type": contentType } : {}),
},
...(body ? { body } : {}),
signal: AbortSignal.timeout(45000),
});
const payload = await parseJsonResponse(response);
lastPayload = payload;
lastStatus = response.status;
if (response.ok || response.status !== 404) {
return { endpoint, response, payload };
}
}
throw new Error(
`Midjourney request failed: ${extractErrorMessage(lastPayload)} (${lastStatus || "no_status"}) @ ${lastEndpoint}`
);
}
async function submitImagineTask(params: {
baseURL: string;
apiKey: string;
mode: "fast" | "relax";
prompt: string;
imageUrls: string[];
}) {
const { baseURL, apiKey, mode, prompt, imageUrls } = params;
const base64Array = await Promise.all(imageUrls.map((imageUrl) => loadInputImageAsDataUrl(imageUrl)));
const { payload } = await requestWithEndpointFallback({
baseURL,
mode,
suffix: "/mj/submit/imagine",
method: "POST",
apiKey,
body: JSON.stringify({
base64Array,
prompt,
}),
contentType: "application/json",
});
const code = Number((payload as any)?.code ?? 0);
const taskId = String((payload as any)?.result || "").trim();
if (![1, 22].includes(code) || !taskId) {
throw new Error(extractErrorMessage(payload));
}
return taskId;
}
async function fetchMidjourneyTask(params: {
baseURL: string;
apiKey: string;
mode: "fast" | "relax";
taskId: string;
}) {
const { payload } = await requestWithEndpointFallback({
baseURL: params.baseURL,
mode: params.mode,
suffix: `/mj/task/${params.taskId}/fetch`,
method: "GET",
apiKey: params.apiKey,
});
return (payload || {}) as MidjourneyTaskResult;
}
async function waitForMidjourneyTask(params: {
baseURL: string;
apiKey: string;
mode: "fast" | "relax";
taskId: string;
}) {
const deadlineAt = Date.now() + DEFAULT_TIMEOUT_MS;
let lastTask: MidjourneyTaskResult | null = null;
while (Date.now() < deadlineAt) {
const task = await fetchMidjourneyTask(params);
lastTask = task;
const status = String(task.status || "").toUpperCase();
if (status === "SUCCESS" && task.imageUrl) {
return task;
}
if (status === "FAILURE") {
throw new Error(task.failReason || task.description || "Midjourney 任务失败");
}
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
}
throw new Error(
`Midjourney task timeout: ${params.taskId} (${lastTask?.status || "unknown"})`
);
}
function formatMarkdown(image: UploadedImage, imageSize: string) {
return `![thumb|||${image.original}|||${imageSize}](${image.thumbnail})`;
}
export function createMidjourney({ apiKey, baseURL }: MidjourneyConfig) {
return {
provider: "midjourney",
chat: (modelId: string) => ({
provider: "midjourney",
doGenerate: async (options: {
messages: Array<{ role: string; content: any }>;
imageParams?: MidjourneyImageParams;
overrideModelId?: string;
}) => {
const { prompt, imageUrls } = extractPromptAndImages(options.messages);
const activeModelId = options.overrideModelId || modelId;
const mode = resolveMode(baseURL, activeModelId);
const imageSize = normalizeImageSizeLabel(options.imageParams?.imageSize);
const finalPrompt = appendAspectRatio(prompt, options.imageParams?.aspectRatio);
console.log("[Midjourney] Submitting request", {
model: activeModelId,
mode,
imageCount: imageUrls.length,
promptPreview: finalPrompt.slice(0, 120),
});
const taskId = await submitImagineTask({
baseURL,
apiKey,
mode,
prompt: finalPrompt,
imageUrls,
});
console.log("[Midjourney] Task submitted", { taskId, mode });
const taskResult = await waitForMidjourneyTask({
baseURL,
apiKey,
mode,
taskId,
});
if (!taskResult.imageUrl) {
throw new Error(taskResult.failReason || taskResult.description || "Midjourney 未返回图片地址");
}
const uploadedImage = await downloadAndUploadImage(taskResult.imageUrl);
const text = formatMarkdown(uploadedImage, imageSize);
return {
text,
finishReason: "stop" as const,
usage: {
promptTokens: 0,
completionTokens: text.length,
},
providerMetadata: {
midjourney: {
taskId,
mode,
progress: taskResult.progress || "",
buttons: taskResult.buttons || [],
imageUrl: taskResult.imageUrl,
},
},
};
},
doStream: async function* (options: {
messages: Array<{ role: string; content: any }>;
imageParams?: MidjourneyImageParams;
overrideModelId?: string;
}) {
const result = await this.doGenerate(options);
yield {
type: "text-delta",
textDelta: result.text,
};
},
}),
};
}

View File

@ -6,6 +6,7 @@ import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image";
import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
import { createFalImage } from "@/lib/ai/providers/fal-image";
import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
import { createMidjourney } from "@/lib/ai/providers/midjourney";
import { filterReachableRemoteUrls } from "@/lib/net/url-access";
export interface CanvasGenerateImageRequest {
@ -36,6 +37,7 @@ const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([
"ruxa-image",
"fal-image",
"openai-image",
"midjourney",
]);
function parseImagesFromMarkdown(markdown: string): ParsedImage[] {
@ -234,9 +236,17 @@ export async function generateCanvasImage(
const modelInstance = openAIImageProvider.chat(selectedModel.model.modelId);
const result = await modelInstance.doGenerate({ messages, imageParams });
imageMarkdown = result.text || "";
} else if (provider.type === "midjourney") {
const midjourneyProvider = createMidjourney({
apiKey: provider.apiKey,
baseURL: provider.baseUrl,
});
const modelInstance = midjourneyProvider.chat(selectedModel.model.modelId);
const result = await modelInstance.doGenerate({ messages, imageParams });
imageMarkdown = result.text || "";
} else {
throw new Error(
`不支持的图片生成模型类型: ${provider.type || "unknown"},请改用 Gemini/阿里/RUXA/FAL/OpenAI 图片模型`
`不支持的图片生成模型类型: ${provider.type || "unknown"},请改用 Gemini/阿里/RUXA/FAL/OpenAI/Midjourney 图片模型`
);
}

View File

@ -29,6 +29,7 @@ const providerBaseUrlDefaults = [
{ type: "ruxa-image", baseUrl: "https://api.ruxa.ai" },
{ type: "fal-image", baseUrl: "https://fal.run" },
{ type: "openai-image", baseUrl: "https://api.openai.com/v1" },
{ type: "midjourney", baseUrl: "https://your-midjourney-api.example.com" },
{ type: "fal-video", baseUrl: "https://fal.run" },
{ type: "tripo-3d", baseUrl: "https://api.tripo3d.ai/v2/openapi" },
] as const;
@ -90,9 +91,13 @@ const runMigrate = async () => {
ADD CONSTRAINT "AiProvider_type_check"
CHECK (type IN (
'alibaba', 'openai', 'custom', 'local', 'gemini',
'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'ruxa-image', 'fal-image', 'openai-image', 'fal-video', 'zhipu', 'tripo-3d'
'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'ruxa-image', 'fal-image', 'openai-image', 'midjourney', 'fal-video', 'zhipu', 'tripo-3d'
));
`);
await db.execute(sql`
ALTER TABLE "AiModel"
ADD COLUMN IF NOT EXISTS "excludeFromFailover" boolean NOT NULL DEFAULT false;
`);
for (const item of providerBaseUrlDefaults) {
await db.execute(sql`
UPDATE "AiProvider"

View File

@ -2005,7 +2005,7 @@ export async function createAiProvider({
updatedBy,
}: {
name: string;
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d";
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d";
baseUrl: string;
apiKey: string;
isEnabled?: boolean;
@ -2043,7 +2043,7 @@ export async function updateAiProvider({
}: {
id: string;
name?: string;
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d";
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d";
baseUrl?: string;
apiKey?: string;
isEnabled?: boolean;
@ -2196,6 +2196,7 @@ export async function createAiModel({
sortOrder,
fallbackModelId1,
fallbackModelId2,
excludeFromFailover,
imageGenConfig,
isAgentModel,
isMultimodal,
@ -2214,6 +2215,7 @@ export async function createAiModel({
sortOrder?: number;
fallbackModelId1?: string;
fallbackModelId2?: string;
excludeFromFailover?: boolean;
imageGenConfig?: ImageGenConfig;
isAgentModel?: boolean;
isMultimodal?: boolean;
@ -2254,6 +2256,7 @@ export async function createAiModel({
sortOrder: sortOrder ?? 0,
fallbackModelId1: fallbackModelId1 || null,
fallbackModelId2: fallbackModelId2 || null,
excludeFromFailover: excludeFromFailover ?? false,
imageGenConfig: imageGenConfig || null,
defaultParams,
capabilities:
@ -2285,6 +2288,7 @@ export async function updateAiModel({
sortOrder,
fallbackModelId1,
fallbackModelId2,
excludeFromFailover,
imageGenConfig,
isAgentModel,
isMultimodal,
@ -2304,6 +2308,7 @@ export async function updateAiModel({
sortOrder?: number;
fallbackModelId1?: string;
fallbackModelId2?: string;
excludeFromFailover?: boolean;
imageGenConfig?: ImageGenConfig;
isAgentModel?: boolean;
isMultimodal?: boolean;
@ -2334,6 +2339,7 @@ export async function updateAiModel({
// 备用模型 ID(空字符串转为 null)
if (fallbackModelId1 !== undefined) updateData.fallbackModelId1 = fallbackModelId1 || null;
if (fallbackModelId2 !== undefined) updateData.fallbackModelId2 = fallbackModelId2 || null;
if (excludeFromFailover !== undefined) updateData.excludeFromFailover = excludeFromFailover;
// 🆕 图片生成配置
if (imageGenConfig !== undefined) updateData.imageGenConfig = imageGenConfig || null;
if (

View File

@ -259,7 +259,7 @@ export type ToolConfig = InferSelectModel<typeof toolConfig>;
export const aiProvider = pgTable("AiProvider", {
id: uuid("id").primaryKey().notNull().defaultRandom(),
name: varchar("name", { length: 128 }).notNull(), // 提供商名称,如 "阿里云通义千问"
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "openai-image", "fal-video", "zhipu", "tripo-3d"] })
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "fal-video", "zhipu", "tripo-3d"] })
.notNull()
.default("custom"), // 提供商类型
baseUrl: text("baseUrl").notNull(), // API 基础 URL
@ -296,6 +296,7 @@ export const aiModel = pgTable("AiModel", {
// 🆕 备用模型 ID(当主模型请求失败时自动切换)
fallbackModelId1: varchar("fallbackModelId1", { length: 128 }), // 备用模型 ID 1
fallbackModelId2: varchar("fallbackModelId2", { length: 128 }), // 备用模型 ID 2
excludeFromFailover: boolean("excludeFromFailover").notNull().default(false), // 不参与故障迁移
// 🆕 图片生成模型特性配置(用于阿里生图、Gemini等)
imageGenConfig: jsonb("imageGenConfig"), // { maxImages: 6, supportsSize: true, supportsPromptExtend: true, supportsNegativePrompt: true, enableLoadingPlaceholder: true }
createdAt: timestamp("createdAt").notNull().defaultNow(),