feat: add midjourney provider and failover exclusion
This commit is contained in:
parent
864b6d41e3
commit
bbdda81686
@ -147,7 +147,7 @@ type ToolConfig = {
|
||||
type AiProvider = {
|
||||
id: string;
|
||||
name: string;
|
||||
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal";
|
||||
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal";
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
isEnabled: boolean;
|
||||
@ -184,6 +184,9 @@ type AiModel = {
|
||||
isHidden: boolean;
|
||||
isDefault: boolean;
|
||||
sortOrder: number;
|
||||
fallbackModelId1?: string;
|
||||
fallbackModelId2?: string;
|
||||
excludeFromFailover?: boolean;
|
||||
isAgentModel?: boolean;
|
||||
agentCodingModelId?: string;
|
||||
isMultimodal?: boolean;
|
||||
@ -438,6 +441,7 @@ const PROVIDER_TYPES = [
|
||||
{ value: "ruxa-image", label: "RUXA生图" },
|
||||
{ value: "fal-image", label: "FAL 生图模型" },
|
||||
{ value: "openai-image", label: "OpenAI 图片兼容" },
|
||||
{ value: "midjourney", label: "Midjourney" },
|
||||
{ value: "tripo-3d", label: "Tripo3D 模型生成" },
|
||||
{ value: "zhipu", label: "智普官方 4.7" },
|
||||
{ value: "volcano-multimodal", label: "火山引擎多模态" },
|
||||
@ -461,6 +465,7 @@ const PROVIDER_DEFAULT_BASE_URL: Record<string, string> = {
|
||||
"ruxa-image": "https://api.ruxa.ai",
|
||||
"fal-image": "https://fal.run",
|
||||
"openai-image": "https://api.openai.com/v1",
|
||||
midjourney: "https://your-midjourney-api.example.com",
|
||||
"fal-video": "https://fal.run",
|
||||
"tripo-3d": "https://api.tripo3d.ai/v2/openapi",
|
||||
zhipu: "https://open.bigmodel.cn/api/paas/v4",
|
||||
@ -1327,7 +1332,7 @@ export function AdminSettingsClient() {
|
||||
const [providerForm, setProviderForm] = useState({
|
||||
id: "",
|
||||
name: "",
|
||||
type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal",
|
||||
type: "openai" as "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "volcano-multimodal",
|
||||
baseUrl: "",
|
||||
apiKey: "",
|
||||
isEnabled: true,
|
||||
@ -1354,6 +1359,7 @@ export function AdminSettingsClient() {
|
||||
sortOrder: 0,
|
||||
fallbackModelId1: "", // 备用模型 ID 1
|
||||
fallbackModelId2: "", // 备用模型 ID 2
|
||||
excludeFromFailover: false,
|
||||
// 🆕 图片生成模型特性配置
|
||||
imageGenConfig: {
|
||||
maxImages: 2,
|
||||
@ -2426,6 +2432,7 @@ export function AdminSettingsClient() {
|
||||
sortOrder: 0,
|
||||
fallbackModelId1: "",
|
||||
fallbackModelId2: "",
|
||||
excludeFromFailover: false,
|
||||
imageGenConfig: {
|
||||
maxImages: 2,
|
||||
supportsSize: false,
|
||||
@ -3697,6 +3704,7 @@ export function AdminSettingsClient() {
|
||||
"ruxa-image": "bg-emerald-500",
|
||||
"fal-image": "bg-sky-500",
|
||||
"openai-image": "bg-green-600",
|
||||
"midjourney": "bg-fuchsia-600",
|
||||
"tripo-3d": "bg-teal-500",
|
||||
"openai": "bg-green-500",
|
||||
"zhipu": "bg-lime-500",
|
||||
@ -6860,6 +6868,15 @@ export function AdminSettingsClient() {
|
||||
/>
|
||||
<Label>多模态模型(Agent 识图与分析使用同一模型)</Label>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Switch
|
||||
checked={modelForm.excludeFromFailover || false}
|
||||
onCheckedChange={(checked) =>
|
||||
setModelForm({ ...modelForm, excludeFromFailover: checked })
|
||||
}
|
||||
/>
|
||||
<Label>禁止参与故障迁移(不作为其他模型的自动备用模型)</Label>
|
||||
</div>
|
||||
|
||||
{/* 🆕 图片生成模型特性配置 */}
|
||||
{(modelForm.supportsImageGen || isVideoProviderForModelConfig) && (
|
||||
@ -7096,6 +7113,7 @@ export function AdminSettingsClient() {
|
||||
{model.isAgentModel && <span className="text-xs bg-violet-100 text-violet-800 px-2 py-0.5 rounded">Agent</span>}
|
||||
{model.isMultimodal && <span className="text-xs bg-emerald-100 text-emerald-800 px-2 py-0.5 rounded">多模态</span>}
|
||||
{model.isHidden && <span className="text-xs bg-amber-100 text-amber-800 px-2 py-0.5 rounded">前台隐藏</span>}
|
||||
{(model as any).excludeFromFailover && <span className="text-xs bg-rose-100 text-rose-700 px-2 py-0.5 rounded">不参与故障迁移</span>}
|
||||
</div>
|
||||
<div className="text-sm text-muted-foreground truncate">
|
||||
ID: {model.modelId} | 提供商: {provider?.name ?? "未知"}
|
||||
@ -7122,6 +7140,7 @@ export function AdminSettingsClient() {
|
||||
sortOrder: model.sortOrder || 0,
|
||||
fallbackModelId1: (model as any).fallbackModelId1 || "",
|
||||
fallbackModelId2: (model as any).fallbackModelId2 || "",
|
||||
excludeFromFailover: (model as any).excludeFromFailover || false,
|
||||
imageGenConfig: (model as any).imageGenConfig || {
|
||||
maxImages: 2,
|
||||
supportsSize: false,
|
||||
@ -8728,7 +8747,7 @@ export function AdminSettingsClient() {
|
||||
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
|
||||
if (!provider) return false;
|
||||
// 支持阿里生图、Gemini、既梦等图片模型
|
||||
const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini-image", "gemini", "volcano-vision"];
|
||||
const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "gemini-image", "gemini", "volcano-vision"];
|
||||
return imageProviderTypes.includes(provider.type);
|
||||
})
|
||||
.map((m: AiModel) => {
|
||||
@ -8743,7 +8762,7 @@ export function AdminSettingsClient() {
|
||||
{/* 如果没有找到生图模型,显示提示 */}
|
||||
{(modelsData?.models || []).filter((m: AiModel) => {
|
||||
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
|
||||
const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini-image", "gemini", "volcano-vision"];
|
||||
const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "gemini-image", "gemini", "volcano-vision"];
|
||||
return provider && imageProviderTypes.includes(provider.type);
|
||||
}).length === 0 && (
|
||||
<SelectItem value="no-model" disabled>
|
||||
|
||||
@ -280,6 +280,7 @@ function normalizeImageLoadingProviderType(providerType?: string | null): string
|
||||
normalized === "ruxa-image" ||
|
||||
normalized === "fal-image" ||
|
||||
normalized === "openai-image" ||
|
||||
normalized === "midjourney" ||
|
||||
normalized === "volcano-vision" ||
|
||||
normalized === "gemini"
|
||||
) {
|
||||
@ -974,6 +975,56 @@ function setImageFallbackModelCooldown(modelId: string, reason: string): void {
|
||||
});
|
||||
}
|
||||
|
||||
function isImageFailoverExcludedModel(model: { excludeFromFailover?: boolean } | null | undefined) {
|
||||
return Boolean(model?.excludeFromFailover);
|
||||
}
|
||||
|
||||
function buildImageFallbackModelIds(
|
||||
allModels: Array<{ model: any; provider: any }>,
|
||||
options: {
|
||||
currentProviderId?: string;
|
||||
currentModelId?: string;
|
||||
currentModelKey?: string;
|
||||
configuredFallbackIds?: string[];
|
||||
}
|
||||
) {
|
||||
const { currentProviderId, currentModelId, currentModelKey, configuredFallbackIds = [] } = options;
|
||||
|
||||
const eligibleFallbackModelIds = new Set(
|
||||
allModels
|
||||
.filter(
|
||||
(item) =>
|
||||
item.model.supportsImageGen &&
|
||||
item.model.isEnabled &&
|
||||
!item.model.isSubModel &&
|
||||
!isImageFailoverExcludedModel(item.model) &&
|
||||
item.model.modelId !== currentModelKey &&
|
||||
item.model.id !== currentModelId
|
||||
)
|
||||
.map((item) => String(item.model.modelId || ""))
|
||||
.filter(Boolean)
|
||||
);
|
||||
|
||||
const sameProviderFallbacks = currentProviderId
|
||||
? allModels
|
||||
.filter(
|
||||
(item) =>
|
||||
item.provider.id === currentProviderId &&
|
||||
item.model.supportsImageGen &&
|
||||
item.model.isEnabled &&
|
||||
!item.model.isSubModel &&
|
||||
!isImageFailoverExcludedModel(item.model) &&
|
||||
item.model.modelId !== currentModelKey &&
|
||||
item.model.id !== currentModelId
|
||||
)
|
||||
.map((item) => item.model.modelId)
|
||||
: [];
|
||||
|
||||
return Array.from(new Set([...configuredFallbackIds, ...sameProviderFallbacks])).filter(
|
||||
(modelId) => Boolean(modelId) && eligibleFallbackModelIds.has(String(modelId))
|
||||
);
|
||||
}
|
||||
|
||||
function getImageCrossProviderCandidates(
|
||||
allModels: any[],
|
||||
options: {
|
||||
@ -984,7 +1035,12 @@ function getImageCrossProviderCandidates(
|
||||
) {
|
||||
return allModels
|
||||
.filter((m) => {
|
||||
if (!m?.model?.supportsImageGen || !m?.model?.isEnabled || m?.model?.isSubModel) {
|
||||
if (
|
||||
!m?.model?.supportsImageGen ||
|
||||
!m?.model?.isEnabled ||
|
||||
m?.model?.isSubModel ||
|
||||
isImageFailoverExcludedModel(m?.model)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (!IMAGE_FAILOVER_PROVIDER_ALLOWLIST.has(String(m?.provider?.type || ""))) {
|
||||
@ -1438,7 +1494,12 @@ function ensureImageAnalysisSummarySection(message: string, summary: string): st
|
||||
* 因为这些 provider 内部会自行处理图片上传到 S3
|
||||
*/
|
||||
async function convertMessagesWithImages(messages: ChatMessage[], providerType: string) {
|
||||
if (providerType === "ruxa-image" || providerType === "fal-image" || providerType === "openai-image") {
|
||||
if (
|
||||
providerType === "ruxa-image" ||
|
||||
providerType === "fal-image" ||
|
||||
providerType === "openai-image" ||
|
||||
providerType === "midjourney"
|
||||
) {
|
||||
const convertedMessages = [];
|
||||
|
||||
for (const message of messages) {
|
||||
@ -2407,6 +2468,8 @@ export async function POST(request: Request) {
|
||||
let shouldKeepSelectedModel = false;
|
||||
const requestedModelId = selectedChatModel;
|
||||
let requestedModelProviderType: string | undefined;
|
||||
let requestedModelIsFixedVisual = false;
|
||||
let requestedModelIsConversationMultimodal = false;
|
||||
|
||||
if (!shouldRunAgentConversation && requestedModelId !== "auto") {
|
||||
const selectedModelIsUUID =
|
||||
@ -2444,6 +2507,7 @@ export async function POST(request: Request) {
|
||||
"ruxa-image",
|
||||
"fal-image",
|
||||
"openai-image",
|
||||
"midjourney",
|
||||
"volcano-vision",
|
||||
]);
|
||||
const isFixedVisualModel =
|
||||
@ -2460,6 +2524,8 @@ export async function POST(request: Request) {
|
||||
selectedModelConfig.supportsVision &&
|
||||
!selectedModelConfig.supportsImageGen &&
|
||||
!isVideoProviderType(selectedModelProviderType);
|
||||
requestedModelIsFixedVisual = isFixedVisualModel;
|
||||
requestedModelIsConversationMultimodal = isConversationMultimodalModel;
|
||||
|
||||
if (deepThinkingOn) {
|
||||
if (isAiModelAgentAuto(selectedModelConfig)) {
|
||||
@ -2755,7 +2821,13 @@ export async function POST(request: Request) {
|
||||
console.log("[Smart Auto Mode] 🎛️ Agent recommended image params:", smartAutoRecommendedImageParams);
|
||||
}
|
||||
|
||||
if (autoGraphResult.route && !shouldKeepSelectedModel) {
|
||||
const shouldForceConversationalRouteForVisualModel =
|
||||
shouldKeepSelectedModel &&
|
||||
requestedModelIsFixedVisual &&
|
||||
!requestedModelIsConversationMultimodal &&
|
||||
Boolean(smartAutoExecution?.shouldStayConversational);
|
||||
|
||||
if (autoGraphResult.route && (!shouldKeepSelectedModel || shouldForceConversationalRouteForVisualModel)) {
|
||||
const routeIsVideo = isVideoProviderType(autoGraphResult.route.providerType);
|
||||
if (routeIsVideo && !allowAgentVideoGeneration) {
|
||||
const fallbackIntent: "image_edit" | "image_gen" = hasImage ? "image_edit" : "image_gen";
|
||||
@ -2808,7 +2880,9 @@ export async function POST(request: Request) {
|
||||
} else {
|
||||
finalModelId = autoGraphResult.route.id;
|
||||
console.log(
|
||||
`[Smart Auto Mode] ✅ Routed to: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`,
|
||||
shouldForceConversationalRouteForVisualModel
|
||||
? `[Smart Auto Mode] ✅ Switched visual model to conversational route: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`
|
||||
: `[Smart Auto Mode] ✅ Routed to: ${autoGraphResult.route.displayName} (${autoGraphResult.route.reason})`,
|
||||
);
|
||||
}
|
||||
} else if (autoGraphResult.route && shouldKeepSelectedModel) {
|
||||
@ -4102,6 +4176,7 @@ export async function POST(request: Request) {
|
||||
(languageModel as any).provider === "ruxa-image" ||
|
||||
(languageModel as any).provider === "fal-image" ||
|
||||
(languageModel as any).provider === "openai-image" ||
|
||||
(languageModel as any).provider === "midjourney" ||
|
||||
(languageModel as any).provider === "huanke-gemini";
|
||||
|
||||
const isUuidModelId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||||
@ -4218,18 +4293,12 @@ export async function POST(request: Request) {
|
||||
currentModelConfig?.fallbackModelId2,
|
||||
].filter(Boolean) as string[];
|
||||
const allModels = await getEnabledAiModelsWithProvider();
|
||||
const sameProviderFallbacks = currentModelConfig
|
||||
? allModels
|
||||
.filter(m =>
|
||||
m.provider.id === currentModelConfig.providerId &&
|
||||
m.model.supportsImageGen &&
|
||||
m.model.isEnabled &&
|
||||
!m.model.isSubModel &&
|
||||
m.model.modelId !== actualModelId
|
||||
)
|
||||
.map(m => m.model.modelId)
|
||||
: [];
|
||||
const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks]));
|
||||
const fallbackModelIds = buildImageFallbackModelIds(allModels, {
|
||||
currentProviderId: currentModelConfig?.providerId,
|
||||
currentModelId: currentModelConfig?.id,
|
||||
currentModelKey: actualModelId,
|
||||
configuredFallbackIds: configuredFallbacks,
|
||||
});
|
||||
console.log("[Chat API] Fallback model IDs:", fallbackModelIds);
|
||||
|
||||
const geminiStream = createUIMessageStream({
|
||||
@ -4649,7 +4718,7 @@ export async function POST(request: Request) {
|
||||
// ========== Gemini 处理结束 ==========
|
||||
|
||||
// ========== 阿里云千问图片生成独立处理 ==========
|
||||
const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image", "fal-image", "openai-image"].includes((languageModel as any).provider);
|
||||
const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney"].includes((languageModel as any).provider);
|
||||
if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||||
console.log("[Chat API] Using image generation provider:", (languageModel as any).provider);
|
||||
console.log("[Chat API] Image params:", effectiveImageParams);
|
||||
@ -4662,18 +4731,12 @@ export async function POST(request: Request) {
|
||||
alibabaModelConfig?.fallbackModelId2,
|
||||
].filter(Boolean) as string[];
|
||||
const allModels = await getEnabledAiModelsWithProvider();
|
||||
const sameProviderFallbacks = alibabaModelConfig
|
||||
? allModels
|
||||
.filter(m =>
|
||||
m.provider.id === alibabaModelConfig.providerId &&
|
||||
m.model.supportsImageGen &&
|
||||
m.model.isEnabled &&
|
||||
!m.model.isSubModel &&
|
||||
m.model.modelId !== actualModelId
|
||||
)
|
||||
.map(m => m.model.modelId)
|
||||
: [];
|
||||
const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks]));
|
||||
const fallbackModelIds = buildImageFallbackModelIds(allModels, {
|
||||
currentProviderId: alibabaModelConfig?.providerId,
|
||||
currentModelId: alibabaModelConfig?.id,
|
||||
currentModelKey: actualModelId,
|
||||
configuredFallbackIds: configuredFallbacks,
|
||||
});
|
||||
console.log("[Chat API] Alibaba fallback model IDs:", fallbackModelIds);
|
||||
|
||||
const alibabaStream = createUIMessageStream({
|
||||
@ -4741,6 +4804,8 @@ export async function POST(request: Request) {
|
||||
? "fal-image"
|
||||
: (languageModel as any).provider === "openai-image"
|
||||
? "openai-image"
|
||||
: (languageModel as any).provider === "midjourney"
|
||||
? "midjourney"
|
||||
: "alibaba-image";
|
||||
const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert);
|
||||
|
||||
@ -4841,6 +4906,8 @@ export async function POST(request: Request) {
|
||||
? "fal-image"
|
||||
: candidate.provider.type === "openai-image"
|
||||
? "openai-image"
|
||||
: candidate.provider.type === "midjourney"
|
||||
? "midjourney"
|
||||
: candidate.provider.type === "gemini"
|
||||
? "gemini"
|
||||
: candidate.provider.type === "huanke-gemini"
|
||||
@ -5060,18 +5127,12 @@ export async function POST(request: Request) {
|
||||
volcanoModelConfig?.fallbackModelId2,
|
||||
].filter(Boolean) as string[];
|
||||
const allModels = await getEnabledAiModelsWithProvider();
|
||||
const sameProviderFallbacks = volcanoModelConfig
|
||||
? allModels
|
||||
.filter(m =>
|
||||
m.provider.id === volcanoModelConfig.providerId &&
|
||||
m.model.supportsImageGen &&
|
||||
m.model.isEnabled &&
|
||||
!m.model.isSubModel &&
|
||||
m.model.modelId !== actualModelId
|
||||
)
|
||||
.map(m => m.model.modelId)
|
||||
: [];
|
||||
const fallbackModelIds = Array.from(new Set([...configuredFallbacks, ...sameProviderFallbacks]));
|
||||
const fallbackModelIds = buildImageFallbackModelIds(allModels, {
|
||||
currentProviderId: volcanoModelConfig?.providerId,
|
||||
currentModelId: volcanoModelConfig?.id,
|
||||
currentModelKey: actualModelId,
|
||||
configuredFallbackIds: configuredFallbacks,
|
||||
});
|
||||
console.log("[Chat API] Volcano fallback model IDs:", fallbackModelIds);
|
||||
|
||||
if (hasAnnotation) {
|
||||
@ -5230,6 +5291,8 @@ export async function POST(request: Request) {
|
||||
? "fal-image"
|
||||
: candidate.provider.type === "openai-image"
|
||||
? "openai-image"
|
||||
: candidate.provider.type === "midjourney"
|
||||
? "midjourney"
|
||||
: candidate.provider.type === "gemini"
|
||||
? "gemini"
|
||||
: candidate.provider.type === "huanke-gemini"
|
||||
@ -5754,21 +5817,12 @@ export async function POST(request: Request) {
|
||||
currentModelConfig?.fallbackModelId2,
|
||||
].filter(Boolean) as string[];
|
||||
const allModels = await getEnabledAiModelsWithProvider();
|
||||
const sameProviderFallbacks = currentModelConfig
|
||||
? allModels
|
||||
.filter(
|
||||
(m) =>
|
||||
m.provider.id === currentModelConfig.providerId &&
|
||||
m.model.supportsImageGen &&
|
||||
m.model.isEnabled &&
|
||||
!m.model.isSubModel &&
|
||||
m.model.modelId !== actualModelId
|
||||
)
|
||||
.map((m) => m.model.modelId)
|
||||
: [];
|
||||
const fallbackModelIds = Array.from(
|
||||
new Set([...configuredFallbacks, ...sameProviderFallbacks])
|
||||
).filter((modelId) => modelId !== actualModelId);
|
||||
const fallbackModelIds = buildImageFallbackModelIds(allModels, {
|
||||
currentProviderId: currentModelConfig?.providerId,
|
||||
currentModelId: currentModelConfig?.id,
|
||||
currentModelKey: actualModelId,
|
||||
configuredFallbackIds: configuredFallbacks,
|
||||
});
|
||||
console.log("[Chat API] OpenAI-compatible fallback model IDs:", fallbackModelIds);
|
||||
|
||||
const lastUserMessage = uiMessagesForModel.filter((m) => m.role === "user").pop();
|
||||
@ -5901,6 +5955,8 @@ export async function POST(request: Request) {
|
||||
? "fal-image"
|
||||
: candidate.provider.type === "openai-image"
|
||||
? "openai-image"
|
||||
: candidate.provider.type === "midjourney"
|
||||
? "midjourney"
|
||||
: candidate.provider.type === "gemini"
|
||||
? "gemini"
|
||||
: candidate.provider.type === "huanke-gemini"
|
||||
|
||||
@ -47,6 +47,7 @@ const createModelSchema = z.object({
|
||||
sortOrder: z.number().optional(),
|
||||
fallbackModelId1: z.string().max(128).optional(),
|
||||
fallbackModelId2: z.string().max(128).optional(),
|
||||
excludeFromFailover: z.boolean().optional(),
|
||||
isAgentModel: z.boolean().optional(),
|
||||
agentCodingModelId: z.string().max(128).optional(),
|
||||
isMultimodal: z.boolean().optional(),
|
||||
@ -68,6 +69,7 @@ const updateModelSchema = z.object({
|
||||
sortOrder: z.number().optional(),
|
||||
fallbackModelId1: z.string().max(128).optional(),
|
||||
fallbackModelId2: z.string().max(128).optional(),
|
||||
excludeFromFailover: z.boolean().optional(),
|
||||
isAgentModel: z.boolean().optional(),
|
||||
agentCodingModelId: z.string().max(128).optional(),
|
||||
isMultimodal: z.boolean().optional(),
|
||||
|
||||
@ -25,6 +25,7 @@ const providerTypes = [
|
||||
"ruxa-image",
|
||||
"fal-image",
|
||||
"openai-image",
|
||||
"midjourney",
|
||||
"fal-video",
|
||||
"zhipu",
|
||||
"tripo-3d",
|
||||
@ -56,6 +57,7 @@ const DOUBAO_VIDEO_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3";
|
||||
const RUXA_VIDEO_BASE_URL = "https://api.ruxa.ai";
|
||||
const RUXA_IMAGE_BASE_URL = "https://api.ruxa.ai";
|
||||
const OPENAI_IMAGE_BASE_URL = "https://api.openai.com/v1";
|
||||
const MIDJOURNEY_BASE_URL = "https://your-midjourney-api.example.com";
|
||||
const TRIPO_3D_BASE_URL = "https://api.tripo3d.ai/v2/openapi";
|
||||
|
||||
function normalizeProviderTypeForStorage(
|
||||
@ -169,6 +171,8 @@ export async function POST(request: Request) {
|
||||
? data.baseUrl || RUXA_IMAGE_BASE_URL
|
||||
: data.type === "openai-image"
|
||||
? data.baseUrl || OPENAI_IMAGE_BASE_URL
|
||||
: data.type === "midjourney"
|
||||
? data.baseUrl || MIDJOURNEY_BASE_URL
|
||||
: data.type === "tripo-3d"
|
||||
? data.baseUrl || TRIPO_3D_BASE_URL
|
||||
: data.baseUrl,
|
||||
|
||||
@ -49,7 +49,7 @@ function normalizeImageParamsForProvider(
|
||||
const normalized = { ...params };
|
||||
const currentCount = normalized.imageCount || 1;
|
||||
|
||||
if (providerType === "ruxa-image" || providerType === "openai-image") {
|
||||
if (providerType === "ruxa-image" || providerType === "openai-image" || providerType === "midjourney") {
|
||||
normalized.imageCount = 1;
|
||||
return normalized;
|
||||
}
|
||||
@ -1053,7 +1053,8 @@ export function Chat({
|
||||
model?.providerType === "alibaba-image" ||
|
||||
model?.providerType === "ruxa-image" ||
|
||||
model?.providerType === "fal-image" ||
|
||||
model?.providerType === "openai-image"
|
||||
model?.providerType === "openai-image" ||
|
||||
model?.providerType === "midjourney"
|
||||
) modelType = "alibaba-image";
|
||||
|
||||
// 智能自动模式下,如果当前轮输入包含图片,默认按图片生成链路跟踪
|
||||
|
||||
@ -21,6 +21,7 @@ const PROVIDER_MODEL_LABEL: Record<string, string> = {
|
||||
"ruxa-image": "RUXA 图片",
|
||||
"fal-image": "FAL 图片",
|
||||
"openai-image": "OpenAI 图片",
|
||||
midjourney: "Midjourney",
|
||||
"volcano-vision": "即梦 4.5",
|
||||
gemini: "Gemini 图片",
|
||||
};
|
||||
|
||||
@ -87,7 +87,7 @@ export function ImageParamsBar({
|
||||
}: ImageParamsBarProps) {
|
||||
const getImageCountOptions = () => {
|
||||
// RUXA 当前按单图任务处理(优先级最高)
|
||||
if (providerType === "ruxa-image" || providerType === "openai-image") {
|
||||
if (providerType === "ruxa-image" || providerType === "openai-image" || providerType === "midjourney") {
|
||||
return [1];
|
||||
}
|
||||
// 优先使用模型配置中的 maxImages
|
||||
|
||||
@ -124,7 +124,8 @@ function PureMessages({
|
||||
(m.providerType === "alibaba-image" ||
|
||||
m.providerType === "ruxa-image" ||
|
||||
m.providerType === "fal-image" ||
|
||||
m.providerType === "openai-image")
|
||||
m.providerType === "openai-image" ||
|
||||
m.providerType === "midjourney")
|
||||
) ?? false;
|
||||
|
||||
// 🆕 获取当前模型的显示名称
|
||||
@ -683,7 +684,12 @@ function PureMessages({
|
||||
return (
|
||||
<ImageGenerationLoading
|
||||
imageCount={imageCount}
|
||||
maxImages={currentModelProviderType === "openai-image" ? 1 : 6}
|
||||
maxImages={
|
||||
currentModelProviderType === "openai-image" ||
|
||||
currentModelProviderType === "midjourney"
|
||||
? 1
|
||||
: 6
|
||||
}
|
||||
providerType={
|
||||
currentModelProviderType === "ruxa-image"
|
||||
? "ruxa-image"
|
||||
@ -691,6 +697,8 @@ function PureMessages({
|
||||
? "fal-image"
|
||||
: currentModelProviderType === "openai-image"
|
||||
? "openai-image"
|
||||
: currentModelProviderType === "midjourney"
|
||||
? "midjourney"
|
||||
: "alibaba-image"
|
||||
}
|
||||
modelName={currentModelDisplayName}
|
||||
|
||||
@ -1792,7 +1792,8 @@ function PureModelSelectorCompact({
|
||||
providerType === "alibaba-image" ||
|
||||
providerType === "ruxa-image" ||
|
||||
providerType === "fal-image" ||
|
||||
providerType === "openai-image"
|
||||
providerType === "openai-image" ||
|
||||
providerType === "midjourney"
|
||||
) {
|
||||
return <ImageIcon className="h-4 w-4" strokeWidth={2.2} />;
|
||||
}
|
||||
|
||||
@ -21,6 +21,7 @@ import { createAlibabaImage } from "./providers/alibaba-image";
|
||||
import { createRuxaImage } from "./providers/ruxa-image";
|
||||
import { createFalImage } from "./providers/fal-image";
|
||||
import { createOpenAIImage } from "./providers/openai-image";
|
||||
import { createMidjourney } from "./providers/midjourney";
|
||||
import { createFalVideo } from "./providers/fal-video";
|
||||
import { createRuxaVideo } from "./providers/ruxa-video";
|
||||
import { createVolcanoMultimodal } from "./providers/volcano-multimodal";
|
||||
@ -217,6 +218,15 @@ function getOrCreateProvider(provider: Pick<AiProvider, "id" | "type" | "baseUrl
|
||||
return openAIImageProvider as any;
|
||||
}
|
||||
|
||||
case "midjourney": {
|
||||
const midjourneyProvider = createMidjourney({
|
||||
baseURL: resolvedBaseUrl,
|
||||
apiKey: provider.apiKey,
|
||||
});
|
||||
console.log(`[Provider] Created Midjourney generation provider: ${resolvedBaseUrl}`);
|
||||
return midjourneyProvider as any;
|
||||
}
|
||||
|
||||
case "custom":
|
||||
default: {
|
||||
if (isVolcanoMultimodalBaseUrl(resolvedBaseUrl)) {
|
||||
@ -339,6 +349,10 @@ export async function getDynamicModel(
|
||||
console.log("[getDynamicModel] Getting OpenAI Image chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] OpenAI Image model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "midjourney") {
|
||||
console.log("[getDynamicModel] Getting Midjourney chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] Midjourney model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "huanke-gemini") {
|
||||
// 环科 Gemini 原生格式使用 chat() 方法
|
||||
console.log("[getDynamicModel] Getting Huanke Gemini chat model..."); // 调试日志
|
||||
|
||||
@ -14,6 +14,7 @@ const PROVIDER_DEFAULT_BASE_URL: Partial<Record<AiProvider["type"], string>> = {
|
||||
"ruxa-image": "https://api.ruxa.ai",
|
||||
"fal-image": "https://fal.run",
|
||||
"openai-image": "https://api.openai.com/v1",
|
||||
midjourney: "https://your-midjourney-api.example.com",
|
||||
"fal-video": "https://fal.run",
|
||||
"tripo-3d": "https://api.tripo3d.ai/v2/openapi",
|
||||
};
|
||||
|
||||
459
lib/ai/providers/midjourney.ts
Normal file
459
lib/ai/providers/midjourney.ts
Normal file
@ -0,0 +1,459 @@
|
||||
import "server-only";
|
||||
|
||||
import sharp from "sharp";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
|
||||
interface MidjourneyConfig {
|
||||
apiKey: string;
|
||||
baseURL: string;
|
||||
}
|
||||
|
||||
export interface MidjourneyImageParams {
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
imageCount?: number;
|
||||
}
|
||||
|
||||
interface UploadedImage {
|
||||
thumbnail: string;
|
||||
original: string;
|
||||
}
|
||||
|
||||
interface MidjourneyTaskButton {
|
||||
customId?: string;
|
||||
label?: string;
|
||||
emoji?: string;
|
||||
style?: number;
|
||||
type?: number;
|
||||
}
|
||||
|
||||
interface MidjourneyTaskResult {
|
||||
id?: string;
|
||||
imageUrl?: string;
|
||||
status?: string;
|
||||
progress?: string;
|
||||
failReason?: string;
|
||||
description?: string;
|
||||
prompt?: string;
|
||||
buttons?: MidjourneyTaskButton[];
|
||||
}
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 300000;
|
||||
const POLL_INTERVAL_MS = 3500;
|
||||
const DEFAULT_IMAGE_SIZE = "2K";
|
||||
|
||||
function normalizeBaseURL(baseURL: string) {
|
||||
return String(baseURL || "").trim().replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
function resolveMode(baseURL: string, modelId: string) {
|
||||
const normalizedBaseURL = normalizeBaseURL(baseURL).toLowerCase();
|
||||
const normalizedModelId = String(modelId || "").trim().toLowerCase();
|
||||
|
||||
if (
|
||||
normalizedBaseURL.endsWith("/relax") ||
|
||||
normalizedBaseURL.endsWith("/mj-relax") ||
|
||||
normalizedModelId.includes("relax")
|
||||
) {
|
||||
return "relax" as const;
|
||||
}
|
||||
|
||||
return "fast" as const;
|
||||
}
|
||||
|
||||
function buildEndpointCandidates(baseURL: string, mode: "fast" | "relax", suffix: string) {
|
||||
const normalizedBaseURL = normalizeBaseURL(baseURL);
|
||||
const normalizedSuffix = suffix.startsWith("/") ? suffix : `/${suffix}`;
|
||||
const directPattern = /\/(fast|relax|mj-fast|mj-relax)$/i;
|
||||
|
||||
if (directPattern.test(normalizedBaseURL)) {
|
||||
return [`${normalizedBaseURL}${normalizedSuffix}`];
|
||||
}
|
||||
|
||||
const modePrefix = mode === "relax" ? "relax" : "fast";
|
||||
const altPrefix = mode === "relax" ? "mj-relax" : "mj-fast";
|
||||
|
||||
return [
|
||||
`${normalizedBaseURL}/${modePrefix}${normalizedSuffix}`,
|
||||
`${normalizedBaseURL}/${altPrefix}${normalizedSuffix}`,
|
||||
];
|
||||
}
|
||||
|
||||
function normalizeImageSizeLabel(imageSize?: string): "1K" | "2K" | "4K" {
|
||||
const normalized = String(imageSize || "").trim().toUpperCase();
|
||||
if (normalized === "1K" || normalized === "2K" || normalized === "4K") return normalized;
|
||||
return DEFAULT_IMAGE_SIZE;
|
||||
}
|
||||
|
||||
function normalizeAspectRatio(value?: string) {
|
||||
const normalized = String(value || "").trim();
|
||||
if (!normalized || normalized.toLowerCase() === "auto") return undefined;
|
||||
if (/^\d+:\d+$/.test(normalized)) return normalized;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function appendAspectRatio(prompt: string, aspectRatio?: string) {
|
||||
const normalizedPrompt = String(prompt || "").trim();
|
||||
const normalizedAspectRatio = normalizeAspectRatio(aspectRatio);
|
||||
if (!normalizedAspectRatio) return normalizedPrompt;
|
||||
if (/--ar\s+\d+:\d+/i.test(normalizedPrompt)) return normalizedPrompt;
|
||||
return `${normalizedPrompt} --ar ${normalizedAspectRatio}`.trim();
|
||||
}
|
||||
|
||||
function detectImageExtension(imageUrl: string, contentType?: string | null) {
|
||||
const type = String(contentType || "").toLowerCase();
|
||||
if (type.includes("jpeg") || type.includes("jpg")) return "jpg";
|
||||
if (type.includes("webp")) return "webp";
|
||||
if (type.includes("png")) return "png";
|
||||
if (type.includes("gif")) return "gif";
|
||||
|
||||
const cleanUrl = imageUrl.split("?")[0]?.toLowerCase() || "";
|
||||
if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "jpg";
|
||||
if (cleanUrl.endsWith(".webp")) return "webp";
|
||||
if (cleanUrl.endsWith(".gif")) return "gif";
|
||||
if (cleanUrl.endsWith(".png")) return "png";
|
||||
return "png";
|
||||
}
|
||||
|
||||
async function createThumbnail(buffer: Buffer) {
|
||||
try {
|
||||
return await sharp(buffer)
|
||||
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
|
||||
.jpeg({ quality: 80 })
|
||||
.toBuffer();
|
||||
} catch (error) {
|
||||
console.error("[Midjourney] Failed to create thumbnail:", error);
|
||||
return buffer;
|
||||
}
|
||||
}
|
||||
|
||||
function generateFilename(prefix: string, extension: string) {
|
||||
const timestamp = Date.now();
|
||||
const random = Math.random().toString(36).slice(2, 8);
|
||||
return `${prefix}-${timestamp}-${random}.${extension}`;
|
||||
}
|
||||
|
||||
async function uploadGeneratedBuffer(
|
||||
buffer: Buffer,
|
||||
extension: string,
|
||||
mimeType: string
|
||||
): Promise<UploadedImage> {
|
||||
const thumbnailBuffer = await createThumbnail(buffer);
|
||||
|
||||
const originalFilename = generateFilename("midjourney-orig", extension);
|
||||
const originalArrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer);
|
||||
|
||||
const thumbFilename = generateFilename("midjourney-thumb", "jpg");
|
||||
const thumbArrayBuffer = thumbnailBuffer.buffer.slice(
|
||||
thumbnailBuffer.byteOffset,
|
||||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const thumbUploaded = await uploadFile(thumbFilename, thumbArrayBuffer);
|
||||
|
||||
return {
|
||||
thumbnail: thumbUploaded.url,
|
||||
original: originalUploaded.url,
|
||||
};
|
||||
}
|
||||
|
||||
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
||||
const response = await fetch(imageUrl, { cache: "no-store" });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download Midjourney image: ${response.status}`);
|
||||
}
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
const extension = detectImageExtension(imageUrl, response.headers.get("content-type"));
|
||||
const mimeType =
|
||||
response.headers.get("content-type") ||
|
||||
`image/${extension === "jpg" ? "jpeg" : extension}`;
|
||||
return uploadGeneratedBuffer(buffer, extension, mimeType);
|
||||
}
|
||||
|
||||
function extractPromptAndImages(messages: Array<{ role: string; content: any }>) {
|
||||
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
|
||||
if (!lastUserMessage) {
|
||||
throw new Error("No user message found");
|
||||
}
|
||||
|
||||
let prompt = "";
|
||||
const imageUrls: string[] = [];
|
||||
|
||||
const pushImageUrl = (value?: string) => {
|
||||
if (!value) return;
|
||||
const normalized = String(value).trim();
|
||||
if (!normalized) return;
|
||||
if (!imageUrls.includes(normalized)) {
|
||||
imageUrls.push(normalized);
|
||||
}
|
||||
};
|
||||
|
||||
if (typeof lastUserMessage.content === "string") {
|
||||
prompt = lastUserMessage.content.trim();
|
||||
} else if (Array.isArray(lastUserMessage.content)) {
|
||||
for (const part of lastUserMessage.content) {
|
||||
if (part?.type === "text" && typeof part.text === "string") {
|
||||
prompt = `${prompt} ${part.text}`.trim();
|
||||
} else if (part?.type === "image_url" && part.image_url?.url) {
|
||||
pushImageUrl(part.image_url.url);
|
||||
} else if (part?.type === "file") {
|
||||
pushImageUrl(part.url || part.data);
|
||||
} else if (part?.type === "image") {
|
||||
pushImageUrl(part.image || part.url);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!prompt) {
|
||||
throw new Error("No prompt found in user message");
|
||||
}
|
||||
|
||||
return { prompt, imageUrls };
|
||||
}
|
||||
|
||||
async function loadInputImageAsDataUrl(source: string) {
|
||||
const normalized = String(source || "").trim();
|
||||
if (!normalized) {
|
||||
throw new Error("Midjourney 输入图片地址为空");
|
||||
}
|
||||
|
||||
if (normalized.startsWith("data:")) {
|
||||
return normalized;
|
||||
}
|
||||
|
||||
const response = await fetch(normalized, { cache: "no-store" });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch Midjourney input image: ${response.status}`);
|
||||
}
|
||||
|
||||
const mimeType = response.headers.get("content-type") || "image/png";
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
return `data:${mimeType};base64,${buffer.toString("base64")}`;
|
||||
}
|
||||
|
||||
async function parseJsonResponse(response: Response) {
|
||||
const rawText = await response.text();
|
||||
try {
|
||||
return JSON.parse(rawText);
|
||||
} catch {
|
||||
return rawText;
|
||||
}
|
||||
}
|
||||
|
||||
function extractErrorMessage(payload: any) {
|
||||
if (!payload) return "Midjourney 请求失败";
|
||||
if (typeof payload === "string") return payload;
|
||||
if (payload.failReason) return String(payload.failReason);
|
||||
if (payload.description) return String(payload.description);
|
||||
if (payload.message) return String(payload.message);
|
||||
if (payload.error?.message) return String(payload.error.message);
|
||||
try {
|
||||
return JSON.stringify(payload);
|
||||
} catch {
|
||||
return "Midjourney 请求失败";
|
||||
}
|
||||
}
|
||||
|
||||
async function requestWithEndpointFallback(params: {
|
||||
baseURL: string;
|
||||
mode: "fast" | "relax";
|
||||
suffix: string;
|
||||
method: "GET" | "POST";
|
||||
apiKey: string;
|
||||
body?: string;
|
||||
contentType?: string;
|
||||
}) {
|
||||
const { baseURL, mode, suffix, method, apiKey, body, contentType } = params;
|
||||
const endpoints = buildEndpointCandidates(baseURL, mode, suffix);
|
||||
let lastPayload: any = null;
|
||||
let lastStatus: number | null = null;
|
||||
let lastEndpoint = endpoints[0];
|
||||
|
||||
for (const endpoint of endpoints) {
|
||||
lastEndpoint = endpoint;
|
||||
const response = await fetch(endpoint, {
|
||||
method,
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
...(contentType ? { "Content-Type": contentType } : {}),
|
||||
},
|
||||
...(body ? { body } : {}),
|
||||
signal: AbortSignal.timeout(45000),
|
||||
});
|
||||
const payload = await parseJsonResponse(response);
|
||||
lastPayload = payload;
|
||||
lastStatus = response.status;
|
||||
|
||||
if (response.ok || response.status !== 404) {
|
||||
return { endpoint, response, payload };
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Midjourney request failed: ${extractErrorMessage(lastPayload)} (${lastStatus || "no_status"}) @ ${lastEndpoint}`
|
||||
);
|
||||
}
|
||||
|
||||
async function submitImagineTask(params: {
|
||||
baseURL: string;
|
||||
apiKey: string;
|
||||
mode: "fast" | "relax";
|
||||
prompt: string;
|
||||
imageUrls: string[];
|
||||
}) {
|
||||
const { baseURL, apiKey, mode, prompt, imageUrls } = params;
|
||||
const base64Array = await Promise.all(imageUrls.map((imageUrl) => loadInputImageAsDataUrl(imageUrl)));
|
||||
|
||||
const { payload } = await requestWithEndpointFallback({
|
||||
baseURL,
|
||||
mode,
|
||||
suffix: "/mj/submit/imagine",
|
||||
method: "POST",
|
||||
apiKey,
|
||||
body: JSON.stringify({
|
||||
base64Array,
|
||||
prompt,
|
||||
}),
|
||||
contentType: "application/json",
|
||||
});
|
||||
|
||||
const code = Number((payload as any)?.code ?? 0);
|
||||
const taskId = String((payload as any)?.result || "").trim();
|
||||
if (![1, 22].includes(code) || !taskId) {
|
||||
throw new Error(extractErrorMessage(payload));
|
||||
}
|
||||
|
||||
return taskId;
|
||||
}
|
||||
|
||||
async function fetchMidjourneyTask(params: {
|
||||
baseURL: string;
|
||||
apiKey: string;
|
||||
mode: "fast" | "relax";
|
||||
taskId: string;
|
||||
}) {
|
||||
const { payload } = await requestWithEndpointFallback({
|
||||
baseURL: params.baseURL,
|
||||
mode: params.mode,
|
||||
suffix: `/mj/task/${params.taskId}/fetch`,
|
||||
method: "GET",
|
||||
apiKey: params.apiKey,
|
||||
});
|
||||
|
||||
return (payload || {}) as MidjourneyTaskResult;
|
||||
}
|
||||
|
||||
async function waitForMidjourneyTask(params: {
|
||||
baseURL: string;
|
||||
apiKey: string;
|
||||
mode: "fast" | "relax";
|
||||
taskId: string;
|
||||
}) {
|
||||
const deadlineAt = Date.now() + DEFAULT_TIMEOUT_MS;
|
||||
let lastTask: MidjourneyTaskResult | null = null;
|
||||
|
||||
while (Date.now() < deadlineAt) {
|
||||
const task = await fetchMidjourneyTask(params);
|
||||
lastTask = task;
|
||||
|
||||
const status = String(task.status || "").toUpperCase();
|
||||
if (status === "SUCCESS" && task.imageUrl) {
|
||||
return task;
|
||||
}
|
||||
if (status === "FAILURE") {
|
||||
throw new Error(task.failReason || task.description || "Midjourney 任务失败");
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Midjourney task timeout: ${params.taskId} (${lastTask?.status || "unknown"})`
|
||||
);
|
||||
}
|
||||
|
||||
function formatMarkdown(image: UploadedImage, imageSize: string) {
|
||||
return ``;
|
||||
}
|
||||
|
||||
export function createMidjourney({ apiKey, baseURL }: MidjourneyConfig) {
|
||||
return {
|
||||
provider: "midjourney",
|
||||
chat: (modelId: string) => ({
|
||||
provider: "midjourney",
|
||||
doGenerate: async (options: {
|
||||
messages: Array<{ role: string; content: any }>;
|
||||
imageParams?: MidjourneyImageParams;
|
||||
overrideModelId?: string;
|
||||
}) => {
|
||||
const { prompt, imageUrls } = extractPromptAndImages(options.messages);
|
||||
const activeModelId = options.overrideModelId || modelId;
|
||||
const mode = resolveMode(baseURL, activeModelId);
|
||||
const imageSize = normalizeImageSizeLabel(options.imageParams?.imageSize);
|
||||
const finalPrompt = appendAspectRatio(prompt, options.imageParams?.aspectRatio);
|
||||
|
||||
console.log("[Midjourney] Submitting request", {
|
||||
model: activeModelId,
|
||||
mode,
|
||||
imageCount: imageUrls.length,
|
||||
promptPreview: finalPrompt.slice(0, 120),
|
||||
});
|
||||
|
||||
const taskId = await submitImagineTask({
|
||||
baseURL,
|
||||
apiKey,
|
||||
mode,
|
||||
prompt: finalPrompt,
|
||||
imageUrls,
|
||||
});
|
||||
|
||||
console.log("[Midjourney] Task submitted", { taskId, mode });
|
||||
const taskResult = await waitForMidjourneyTask({
|
||||
baseURL,
|
||||
apiKey,
|
||||
mode,
|
||||
taskId,
|
||||
});
|
||||
|
||||
if (!taskResult.imageUrl) {
|
||||
throw new Error(taskResult.failReason || taskResult.description || "Midjourney 未返回图片地址");
|
||||
}
|
||||
|
||||
const uploadedImage = await downloadAndUploadImage(taskResult.imageUrl);
|
||||
const text = formatMarkdown(uploadedImage, imageSize);
|
||||
|
||||
return {
|
||||
text,
|
||||
finishReason: "stop" as const,
|
||||
usage: {
|
||||
promptTokens: 0,
|
||||
completionTokens: text.length,
|
||||
},
|
||||
providerMetadata: {
|
||||
midjourney: {
|
||||
taskId,
|
||||
mode,
|
||||
progress: taskResult.progress || "",
|
||||
buttons: taskResult.buttons || [],
|
||||
imageUrl: taskResult.imageUrl,
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
doStream: async function* (options: {
|
||||
messages: Array<{ role: string; content: any }>;
|
||||
imageParams?: MidjourneyImageParams;
|
||||
overrideModelId?: string;
|
||||
}) {
|
||||
const result = await this.doGenerate(options);
|
||||
yield {
|
||||
type: "text-delta",
|
||||
textDelta: result.text,
|
||||
};
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
@ -6,6 +6,7 @@ import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image";
|
||||
import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
|
||||
import { createFalImage } from "@/lib/ai/providers/fal-image";
|
||||
import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
|
||||
import { createMidjourney } from "@/lib/ai/providers/midjourney";
|
||||
import { filterReachableRemoteUrls } from "@/lib/net/url-access";
|
||||
|
||||
export interface CanvasGenerateImageRequest {
|
||||
@ -36,6 +37,7 @@ const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([
|
||||
"ruxa-image",
|
||||
"fal-image",
|
||||
"openai-image",
|
||||
"midjourney",
|
||||
]);
|
||||
|
||||
function parseImagesFromMarkdown(markdown: string): ParsedImage[] {
|
||||
@ -234,9 +236,17 @@ export async function generateCanvasImage(
|
||||
const modelInstance = openAIImageProvider.chat(selectedModel.model.modelId);
|
||||
const result = await modelInstance.doGenerate({ messages, imageParams });
|
||||
imageMarkdown = result.text || "";
|
||||
} else if (provider.type === "midjourney") {
|
||||
const midjourneyProvider = createMidjourney({
|
||||
apiKey: provider.apiKey,
|
||||
baseURL: provider.baseUrl,
|
||||
});
|
||||
const modelInstance = midjourneyProvider.chat(selectedModel.model.modelId);
|
||||
const result = await modelInstance.doGenerate({ messages, imageParams });
|
||||
imageMarkdown = result.text || "";
|
||||
} else {
|
||||
throw new Error(
|
||||
`不支持的图片生成模型类型: ${provider.type || "unknown"},请改用 Gemini/阿里/RUXA/FAL/OpenAI 图片模型`
|
||||
`不支持的图片生成模型类型: ${provider.type || "unknown"},请改用 Gemini/阿里/RUXA/FAL/OpenAI/Midjourney 图片模型`
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@ -29,6 +29,7 @@ const providerBaseUrlDefaults = [
|
||||
{ type: "ruxa-image", baseUrl: "https://api.ruxa.ai" },
|
||||
{ type: "fal-image", baseUrl: "https://fal.run" },
|
||||
{ type: "openai-image", baseUrl: "https://api.openai.com/v1" },
|
||||
{ type: "midjourney", baseUrl: "https://your-midjourney-api.example.com" },
|
||||
{ type: "fal-video", baseUrl: "https://fal.run" },
|
||||
{ type: "tripo-3d", baseUrl: "https://api.tripo3d.ai/v2/openapi" },
|
||||
] as const;
|
||||
@ -90,9 +91,13 @@ const runMigrate = async () => {
|
||||
ADD CONSTRAINT "AiProvider_type_check"
|
||||
CHECK (type IN (
|
||||
'alibaba', 'openai', 'custom', 'local', 'gemini',
|
||||
'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'ruxa-image', 'fal-image', 'openai-image', 'fal-video', 'zhipu', 'tripo-3d'
|
||||
'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'ruxa-image', 'fal-image', 'openai-image', 'midjourney', 'fal-video', 'zhipu', 'tripo-3d'
|
||||
));
|
||||
`);
|
||||
await db.execute(sql`
|
||||
ALTER TABLE "AiModel"
|
||||
ADD COLUMN IF NOT EXISTS "excludeFromFailover" boolean NOT NULL DEFAULT false;
|
||||
`);
|
||||
for (const item of providerBaseUrlDefaults) {
|
||||
await db.execute(sql`
|
||||
UPDATE "AiProvider"
|
||||
|
||||
@ -2005,7 +2005,7 @@ export async function createAiProvider({
|
||||
updatedBy,
|
||||
}: {
|
||||
name: string;
|
||||
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d";
|
||||
type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d";
|
||||
baseUrl: string;
|
||||
apiKey: string;
|
||||
isEnabled?: boolean;
|
||||
@ -2043,7 +2043,7 @@ export async function updateAiProvider({
|
||||
}: {
|
||||
id: string;
|
||||
name?: string;
|
||||
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "fal-video" | "zhipu" | "tripo-3d";
|
||||
type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d";
|
||||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
isEnabled?: boolean;
|
||||
@ -2196,6 +2196,7 @@ export async function createAiModel({
|
||||
sortOrder,
|
||||
fallbackModelId1,
|
||||
fallbackModelId2,
|
||||
excludeFromFailover,
|
||||
imageGenConfig,
|
||||
isAgentModel,
|
||||
isMultimodal,
|
||||
@ -2214,6 +2215,7 @@ export async function createAiModel({
|
||||
sortOrder?: number;
|
||||
fallbackModelId1?: string;
|
||||
fallbackModelId2?: string;
|
||||
excludeFromFailover?: boolean;
|
||||
imageGenConfig?: ImageGenConfig;
|
||||
isAgentModel?: boolean;
|
||||
isMultimodal?: boolean;
|
||||
@ -2254,6 +2256,7 @@ export async function createAiModel({
|
||||
sortOrder: sortOrder ?? 0,
|
||||
fallbackModelId1: fallbackModelId1 || null,
|
||||
fallbackModelId2: fallbackModelId2 || null,
|
||||
excludeFromFailover: excludeFromFailover ?? false,
|
||||
imageGenConfig: imageGenConfig || null,
|
||||
defaultParams,
|
||||
capabilities:
|
||||
@ -2285,6 +2288,7 @@ export async function updateAiModel({
|
||||
sortOrder,
|
||||
fallbackModelId1,
|
||||
fallbackModelId2,
|
||||
excludeFromFailover,
|
||||
imageGenConfig,
|
||||
isAgentModel,
|
||||
isMultimodal,
|
||||
@ -2304,6 +2308,7 @@ export async function updateAiModel({
|
||||
sortOrder?: number;
|
||||
fallbackModelId1?: string;
|
||||
fallbackModelId2?: string;
|
||||
excludeFromFailover?: boolean;
|
||||
imageGenConfig?: ImageGenConfig;
|
||||
isAgentModel?: boolean;
|
||||
isMultimodal?: boolean;
|
||||
@ -2334,6 +2339,7 @@ export async function updateAiModel({
|
||||
// 备用模型 ID(空字符串转为 null)
|
||||
if (fallbackModelId1 !== undefined) updateData.fallbackModelId1 = fallbackModelId1 || null;
|
||||
if (fallbackModelId2 !== undefined) updateData.fallbackModelId2 = fallbackModelId2 || null;
|
||||
if (excludeFromFailover !== undefined) updateData.excludeFromFailover = excludeFromFailover;
|
||||
// 🆕 图片生成配置
|
||||
if (imageGenConfig !== undefined) updateData.imageGenConfig = imageGenConfig || null;
|
||||
if (
|
||||
|
||||
@ -259,7 +259,7 @@ export type ToolConfig = InferSelectModel<typeof toolConfig>;
|
||||
export const aiProvider = pgTable("AiProvider", {
|
||||
id: uuid("id").primaryKey().notNull().defaultRandom(),
|
||||
name: varchar("name", { length: 128 }).notNull(), // 提供商名称,如 "阿里云通义千问"
|
||||
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "openai-image", "fal-video", "zhipu", "tripo-3d"] })
|
||||
type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "fal-video", "zhipu", "tripo-3d"] })
|
||||
.notNull()
|
||||
.default("custom"), // 提供商类型
|
||||
baseUrl: text("baseUrl").notNull(), // API 基础 URL
|
||||
@ -296,6 +296,7 @@ export const aiModel = pgTable("AiModel", {
|
||||
// 🆕 备用模型 ID(当主模型请求失败时自动切换)
|
||||
fallbackModelId1: varchar("fallbackModelId1", { length: 128 }), // 备用模型 ID 1
|
||||
fallbackModelId2: varchar("fallbackModelId2", { length: 128 }), // 备用模型 ID 2
|
||||
excludeFromFailover: boolean("excludeFromFailover").notNull().default(false), // 不参与故障迁移
|
||||
// 🆕 图片生成模型特性配置(用于阿里生图、Gemini等)
|
||||
imageGenConfig: jsonb("imageGenConfig"), // { maxImages: 6, supportsSize: true, supportsPromptExtend: true, supportsNegativePrompt: true, enableLoadingPlaceholder: true }
|
||||
createdAt: timestamp("createdAt").notNull().defaultNow(),
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user