From 947af6d26fbc02628266184d70a4fc2ffe7fa8ff Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 20 Mar 2026 22:33:09 +0800 Subject: [PATCH] feat: add runway image and video runtime support --- .../settings/admin-settings-client.tsx | 10 +- app/(chat)/api/chat/route.ts | 198 ++++-- app/(chat)/page.tsx | 16 +- lib/ai/dynamic-provider.ts | 14 + lib/ai/model-classification.ts | 114 ++++ lib/ai/model-router.ts | 66 +- lib/ai/providers/runway.ts | 581 ++++++++++++++++++ lib/ai/video-tool-capability.ts | 27 + lib/canvas/generate-image.ts | 20 +- lib/db/queries.ts | 16 +- 10 files changed, 947 insertions(+), 115 deletions(-) create mode 100644 lib/ai/model-classification.ts create mode 100644 lib/ai/providers/runway.ts diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx index ca122b776..b4a292286 100644 --- a/app/(admin)/settings/admin-settings-client.tsx +++ b/app/(admin)/settings/admin-settings-client.tsx @@ -1562,6 +1562,7 @@ export function AdminSettingsClient() { "qianwen-video", "fal-video", "ruxa-video", + "runway", ].includes(selectedModelProviderType); // 存储配置状态 @@ -9363,12 +9364,8 @@ export function AdminSettingsClient() { 不指定首选（完全随机轮换） {(modelsData?.models || []) .filter((m: AiModel) => { - // 过滤出支持图片生成/编辑的模型 const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId); - if (!provider) return false; - // 支持阿里生图、Gemini、既梦等图片模型 - const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "gemini-image", "gemini", "volcano-vision"]; - return imageProviderTypes.includes(provider.type); + return Boolean(provider && m.supportsImageGen); }) .map((m: AiModel) => { const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId); @@ -9382,8 +9379,7 @@ export function AdminSettingsClient() { {/* 如果没有找到生图模型，显示提示 */} {(modelsData?.models || []).filter((m: AiModel) => { const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId); - const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "gemini-image", "gemini", "volcano-vision"]; - return provider && imageProviderTypes.includes(provider.type); + return Boolean(provider && m.supportsImageGen); }).length === 0 && ( 暂无可用的生图模型，请先配置生图供应商 diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index da9137f25..677611f59 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -98,6 +98,10 @@ import { isGridConfirmationReply, type PendingGridConfirmation, } from "@/lib/ai/conversation-action-resolver"; +import { + isImageGenerationModelDescriptor, + isVideoModelDescriptor, +} from "@/lib/ai/model-classification"; import { bootstrapConversationStateProtocol, composeConversationStateProtocol, @@ -457,6 +461,7 @@ function normalizeImageLoadingProviderType(providerType?: string | null): string normalized === "fal-image" || normalized === "openai-image" || normalized === "midjourney" || + normalized === "runway" || normalized === "volcano-vision" || normalized === "gemini" ) { @@ -583,21 +588,53 @@ function startStreamKeepalive(writer: StreamWriter, label: string, intervalMs = type VideoModeType = "text-to-video" | "first-frame" | "first-last-frame" | "reference-images"; -const VIDEO_PROVIDER_TYPES = new Set([ - "volcano-video", - "doubao-video", - "qianwen-video", - "fal-video", - "ruxa-video", -]); +function isVideoProviderType( + providerType?: string | null, + model?: { + modelId?: string | null; + capabilities?: unknown; + supportsImageGen?: boolean | null; + } +): boolean { + return isVideoModelDescriptor({ + providerType, + modelId: model?.modelId, + capabilities: model?.capabilities, + supportsImageGen: model?.supportsImageGen, + }); +} -function isVideoProviderType(providerType?: string | null): boolean { - return Boolean(providerType && VIDEO_PROVIDER_TYPES.has(providerType)); +function isImageGenerationModelType( + providerType?: string | null, + model?: { + modelId?: string | null; + capabilities?: unknown; + supportsImageGen?: boolean | null; + } +): boolean { + return isImageGenerationModelDescriptor({ + providerType, + modelId: model?.modelId, + capabilities: model?.capabilities, + supportsImageGen: model?.supportsImageGen, + }); } function inferSupportedVideoModesByModelId(modelId: string): VideoModeType[] { const normalized = modelId.toLowerCase(); + if ( + normalized === "gen4.5" || + normalized === "gen4_turbo" || + normalized === "gen4_aleph" || + normalized === "gen3a_turbo" || + normalized === "veo3" || + normalized === "veo3.1" || + normalized === "veo3.1_fast" + ) { + return ["text-to-video", "first-frame"]; + } + if (normalized.includes("lite-t2v")) { return ["text-to-video"]; } @@ -1341,6 +1378,7 @@ const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([ "ruxa-image", "fal-image", "openai-image", + "runway", "volcano-vision", "gemini", "huanke-gemini", @@ -1980,7 +2018,8 @@ async function convertMessagesWithImages(messages: ChatMessage[], providerType: providerType === "ruxa-image" || providerType === "fal-image" || providerType === "openai-image" || - providerType === "midjourney" + providerType === "midjourney" || + providerType === "runway" ) { const convertedMessages = []; @@ -3156,6 +3195,7 @@ export async function POST(request: Request) { let requestedModelProviderType: string | undefined; let requestedModelIsFixedVisual = false; let requestedModelIsConversationMultimodal = false; + let requestedModelConfig: Awaited> | null = null; if (!shouldRunAgentConversation && requestedModelId !== "auto") { const selectedModelIsUUID = @@ -3183,33 +3223,21 @@ export async function POST(request: Request) { selectedModelProviderType = selectedProvider?.type as string | undefined; } + requestedModelConfig = selectedModelConfig; requestedModelProviderType = selectedModelProviderType; if (selectedModelConfig) { - const fixedImageProviderTypes = new Set([ - "gemini", - "huanke-gemini", - "alibaba-image", - "ruxa-image", - "fal-image", - "openai-image", - "midjourney", - "volcano-vision", - ]); const isFixedVisualModel = !selectedModelConfig.isSubModel && - (selectedModelConfig.supportsImageGen || - (selectedModelProviderType - ? isVideoProviderType(selectedModelProviderType) - : false) || - (selectedModelProviderType - ? fixedImageProviderTypes.has(selectedModelProviderType) - : false)); + ( + isImageGenerationModelType(selectedModelProviderType, selectedModelConfig) || + isVideoProviderType(selectedModelProviderType, selectedModelConfig) + ); const isConversationMultimodalModel = !selectedModelConfig.isSubModel && isAiModelMultimodal(selectedModelConfig) && - !selectedModelConfig.supportsImageGen && - !isVideoProviderType(selectedModelProviderType); + !isImageGenerationModelType(selectedModelProviderType, selectedModelConfig) && + !isVideoProviderType(selectedModelProviderType, selectedModelConfig); requestedModelIsFixedVisual = isFixedVisualModel; requestedModelIsConversationMultimodal = isConversationMultimodalModel; @@ -3331,7 +3359,8 @@ export async function POST(request: Request) { } const allowAgentVideoGeneration = - Boolean(requestBody.videoParams) || isVideoProviderType(requestedModelProviderType); + Boolean(requestBody.videoParams) || + isVideoProviderType(requestedModelProviderType, requestedModelConfig || undefined); let smartAutoWorkflowDecision: | { action?: string; @@ -3985,7 +4014,9 @@ export async function POST(request: Request) { Boolean(smartAutoExecution?.shouldStayConversational); if (autoGraphResult.route && (!shouldKeepSelectedModel || shouldForceConversationalRouteForVisualModel)) { - const routeIsVideo = isVideoProviderType(autoGraphResult.route.providerType); + const routeIsVideo = isVideoProviderType(autoGraphResult.route.providerType, { + modelId: autoGraphResult.route.modelId, + }); if (routeIsVideo && !allowAgentVideoGeneration) { const fallbackIntent: "image_edit" | "image_gen" = hasImage ? "image_edit" : "image_gen"; smartAutoDetectedIntent = fallbackIntent; @@ -4021,7 +4052,7 @@ export async function POST(request: Request) { subIntent: autoGraphResult.analysis?.subIntent, hasImage, }); - if (!isVideoProviderType(fallbackRoute.providerType)) { + if (!isVideoProviderType(fallbackRoute.providerType, { modelId: fallbackRoute.modelId })) { finalModelId = fallbackRoute.id; console.log( `[Smart Auto Mode] ✅ Fallback routed to: ${fallbackRoute.displayName} (${fallbackRoute.reason})`, @@ -4281,7 +4312,7 @@ export async function POST(request: Request) { (m) => m.model.supportsImageGen && !m.model.isSubModel && - !isVideoProviderType(m.provider?.type), + !isVideoProviderType(m.provider?.type, m.model), ); if (isAutoTimeout) { smartAutoFriendlyMessage = buildAutoModeFallbackMessage({ @@ -4309,12 +4340,15 @@ export async function POST(request: Request) { } if (!bestModel) { bestModel = allModels.find( - (m) => m.model.isDefault && !m.model.isSubModel && !isVideoProviderType(m.provider?.type), + (m) => + m.model.isDefault && + !m.model.isSubModel && + !isVideoProviderType(m.provider?.type, m.model), ); } if (!bestModel && allModels.length > 0) { bestModel = allModels.find( - (m) => !m.model.isSubModel && !isVideoProviderType(m.provider?.type), + (m) => !m.model.isSubModel && !isVideoProviderType(m.provider?.type, m.model), ) || allModels.find((m) => !m.model.isSubModel) || allModels[0]; } @@ -4335,9 +4369,10 @@ export async function POST(request: Request) { const finalModelIsUuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(finalModelId); let finalProviderType: string | undefined; + let finalModelConfig: Awaited> | null = null; if (finalModelIsUuid) { - const finalModelConfig = await getAiModelById({ id: finalModelId }).catch(() => null); + finalModelConfig = await getAiModelById({ id: finalModelId }).catch(() => null); if (finalModelConfig) { finalProviderType = ( await getAiProviderById({ id: finalModelConfig.providerId }).catch(() => null) @@ -4345,19 +4380,19 @@ export async function POST(request: Request) { } } - if (isVideoProviderType(finalProviderType)) { + if (isVideoProviderType(finalProviderType, finalModelConfig || undefined)) { const allModels = await getEnabledAiModelsWithProvider(false); const fallbackModel = allModels.find( (m) => !m.model.isSubModel && m.model.supportsImageGen && - !isVideoProviderType(m.provider?.type), + !isVideoProviderType(m.provider?.type, m.model), ) || allModels.find( (m) => !m.model.isSubModel && - !isVideoProviderType(m.provider?.type), + !isVideoProviderType(m.provider?.type, m.model), ); if (fallbackModel) { @@ -7364,22 +7399,31 @@ export async function POST(request: Request) { } // ========== 图片生成配额检查 ========== - const isImageGenerationModel = - (languageModel as any).provider === "gemini-image" || - (languageModel as any).provider === "volcano-vision" || - (languageModel as any).provider === "alibaba-image" || - (languageModel as any).provider === "ruxa-image" || - (languageModel as any).provider === "fal-image" || - (languageModel as any).provider === "openai-image" || - (languageModel as any).provider === "midjourney" || - (languageModel as any).provider === "huanke-gemini"; - const isUuidModelId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( finalModelId ); const currentModelConfig = isUuidModelId ? await getAiModelById({ id: finalModelId }).catch(() => undefined) : undefined; + const runtimeProviderType = String( + resolvedProviderType || (languageModel as any).provider || "" + ); + const normalizedRuntimeProviderType = + runtimeProviderType === "gemini-image" + ? "gemini" + : runtimeProviderType === "huanke-gemini" + ? "huanke-gemini" + : runtimeProviderType; + const isImageGenerationModel = isImageGenerationModelType( + normalizedRuntimeProviderType, + currentModelConfig + ? { + modelId: currentModelConfig.modelId, + capabilities: currentModelConfig.capabilities, + supportsImageGen: currentModelConfig.supportsImageGen, + } + : undefined + ); const modelCreditCostPerImage = Number( (currentModelConfig as any)?.imageGenConfig?.creditCostPerImage ?? 0 ); @@ -7438,13 +7482,16 @@ export async function POST(request: Request) { } } - const isVideoGenerationModel = [ - "qianwen-video", - "fal-video", - "ruxa-video", - "volcano-video", - "doubao-video", - ].includes(String(resolvedProviderType || (languageModel as any).provider || "")); + const isVideoGenerationModel = isVideoProviderType( + normalizedRuntimeProviderType, + currentModelConfig + ? { + modelId: currentModelConfig.modelId, + capabilities: currentModelConfig.capabilities, + supportsImageGen: currentModelConfig.supportsImageGen, + } + : undefined + ); if (isVideoGenerationModel) { const pointsConfig = await getPointsSystemConfig(); @@ -7925,7 +7972,14 @@ export async function POST(request: Request) { // ========== Gemini 处理结束 ========== // ========== 阿里云千问图片生成独立处理 ========== - const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney"].includes((languageModel as any).provider); + const isAlibabaLikeImageProvider = [ + "alibaba-image", + "ruxa-image", + "fal-image", + "openai-image", + "midjourney", + "runway", + ].includes((languageModel as any).provider); if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using image generation provider:", (languageModel as any).provider); console.log("[Chat API] Image params:", effectiveImageParams); @@ -8013,6 +8067,8 @@ export async function POST(request: Request) { ? "openai-image" : (languageModel as any).provider === "midjourney" ? "midjourney" + : (languageModel as any).provider === "runway" + ? "runway" : "alibaba-image"; const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert); @@ -8085,8 +8141,9 @@ export async function POST(request: Request) { "ruxa-image": 2, "fal-image": 3, "openai-image": 4, - gemini: 5, - "huanke-gemini": 6, + runway: 5, + gemini: 6, + "huanke-gemini": 7, }; const crossProviderCandidates = getImageCrossProviderCandidates(allModels, { currentModelId: finalModelId, @@ -8115,6 +8172,8 @@ export async function POST(request: Request) { ? "openai-image" : candidate.provider.type === "midjourney" ? "midjourney" + : candidate.provider.type === "runway" + ? "runway" : candidate.provider.type === "gemini" ? "gemini" : candidate.provider.type === "huanke-gemini" @@ -8473,9 +8532,10 @@ export async function POST(request: Request) { "ruxa-image": 1, "fal-image": 2, "openai-image": 3, - gemini: 4, - "huanke-gemini": 5, - "volcano-vision": 6, + runway: 4, + gemini: 5, + "huanke-gemini": 6, + "volcano-vision": 7, }; const crossProviderCandidates = getImageCrossProviderCandidates(allModels, { currentModelId: finalModelId, @@ -8504,6 +8564,8 @@ export async function POST(request: Request) { ? "openai-image" : candidate.provider.type === "midjourney" ? "midjourney" + : candidate.provider.type === "runway" + ? "runway" : candidate.provider.type === "gemini" ? "gemini" : candidate.provider.type === "huanke-gemini" @@ -8710,13 +8772,14 @@ export async function POST(request: Request) { // ========== 千问/FAL/RUXA 视频生成独立处理 ========== if ( - ["qianwen-video", "fal-video", "ruxa-video"].includes((languageModel as any).provider) && + ["qianwen-video", "fal-video", "ruxa-video", "runway"].includes((languageModel as any).provider) && typeof (languageModel as any).doGenerate === "function" ) { const currentVideoProvider = (languageModel as any).provider as | "qianwen-video" | "fal-video" - | "ruxa-video"; + | "ruxa-video" + | "runway"; console.log(`[Chat API] Using ${currentVideoProvider} video generation`); const videoGenerationStream = createUIMessageStream({ @@ -9286,8 +9349,9 @@ export async function POST(request: Request) { "ruxa-image": 2, "fal-image": 3, "openai-image": 4, - gemini: 5, - "huanke-gemini": 6, + runway: 5, + gemini: 6, + "huanke-gemini": 7, }; const crossProviderCandidates = getImageCrossProviderCandidates(allModels, { currentModelId: finalModelId, @@ -9321,6 +9385,8 @@ export async function POST(request: Request) { ? "openai-image" : candidate.provider.type === "midjourney" ? "midjourney" + : candidate.provider.type === "runway" + ? "runway" : candidate.provider.type === "gemini" ? "gemini" : candidate.provider.type === "huanke-gemini" diff --git a/app/(chat)/page.tsx b/app/(chat)/page.tsx index dc324994c..c8df373d6 100644 --- a/app/(chat)/page.tsx +++ b/app/(chat)/page.tsx @@ -14,14 +14,7 @@ import { getSystemSetting, } from "@/lib/db/queries"; import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags"; - -const VIDEO_PROVIDER_TYPES = new Set([ - "volcano-video", - "doubao-video", - "qianwen-video", - "fal-video", - "ruxa-video", -]); +import { isVideoModelDescriptor } from "@/lib/ai/model-classification"; function isNanoBananaProModel(model: { displayName?: string | null; modelId?: string | null }) { const displayName = String(model.displayName || "").toLowerCase(); @@ -109,7 +102,12 @@ async function NewChatPage() { const preferredImageModels = visibleModels.filter( ({ model, provider }) => model.supportsImageGen && - !VIDEO_PROVIDER_TYPES.has(provider.type) + !isVideoModelDescriptor({ + providerType: provider.type, + modelId: model.modelId, + capabilities: model.capabilities, + supportsImageGen: model.supportsImageGen, + }) ); const nanoBananaModel = preferredImageModels.find(({ model }) => isNanoBananaProModel(model) diff --git a/lib/ai/dynamic-provider.ts b/lib/ai/dynamic-provider.ts index 947687631..016cdda72 100644 --- a/lib/ai/dynamic-provider.ts +++ b/lib/ai/dynamic-provider.ts @@ -24,6 +24,7 @@ import { createOpenAIImage } from "./providers/openai-image"; import { createMidjourney } from "./providers/midjourney"; import { createFalVideo } from "./providers/fal-video"; import { createRuxaVideo } from "./providers/ruxa-video"; +import { createRunway } from "./providers/runway"; import { createVolcanoMultimodal } from "./providers/volcano-multimodal"; import { getProviderDefaultBaseUrl } from "./provider-defaults"; @@ -227,6 +228,15 @@ function getOrCreateProvider(provider: Pick VIDEO_MODE_KEYS.has(String(mode || "").trim())); +} + +export function inferRunwayModelKind(modelId?: string | null): "image" | "video" | null { + const normalized = normalizeText(modelId); + if (!normalized) return null; + if (RUNWAY_IMAGE_MODEL_IDS.has(normalized)) return "image"; + if (RUNWAY_VIDEO_MODEL_PREFIXES.some((prefix) => normalized === prefix || normalized.startsWith(`${prefix}_`))) { + return "video"; + } + return null; +} + +export function isStaticVideoProviderType(providerType?: string | null): boolean { + return STATIC_VIDEO_PROVIDER_TYPES.has(normalizeText(providerType)); +} + +export function isVideoModelDescriptor(params: { + providerType?: string | null; + modelId?: string | null; + capabilities?: unknown; + supportsImageGen?: boolean | null; +}): boolean { + const providerType = normalizeText(params.providerType); + + if (isStaticVideoProviderType(providerType)) { + return true; + } + + if (hasVideoModes(params.capabilities)) { + return true; + } + + if (providerType === "runway") { + return inferRunwayModelKind(params.modelId) === "video"; + } + + return false; +} + +export function isImageGenerationModelDescriptor(params: { + providerType?: string | null; + modelId?: string | null; + capabilities?: unknown; + supportsImageGen?: boolean | null; +}): boolean { + const providerType = normalizeText(params.providerType); + + if (params.supportsImageGen === true) { + return true; + } + + if (providerType === "runway") { + return inferRunwayModelKind(params.modelId) === "image"; + } + + if (isVideoModelDescriptor(params)) { + return false; + } + + return STATIC_IMAGE_PROVIDER_TYPES.has(providerType); +} diff --git a/lib/ai/model-router.ts b/lib/ai/model-router.ts index 09e0896e5..f8eaed78e 100644 --- a/lib/ai/model-router.ts +++ b/lib/ai/model-router.ts @@ -6,6 +6,10 @@ */ import { isAiModelMultimodal } from "@/lib/db/ai-model-flags"; +import { + isImageGenerationModelDescriptor, + isVideoModelDescriptor, +} from "@/lib/ai/model-classification"; type EnabledModelWithProvider = Awaited< ReturnType @@ -42,14 +46,6 @@ const CHAT_PROVIDER_TYPES = [ "volcano-multimodal", ] as const; -const VIDEO_PROVIDER_TYPES = new Set([ - "doubao-video", - "volcano-video", - "qianwen-video", - "fal-video", - "ruxa-video", -]); - const IMAGE_ONLY_PROVIDER_TYPES = new Set([ "alibaba-image", "ruxa-image", @@ -59,8 +55,20 @@ const IMAGE_ONLY_PROVIDER_TYPES = new Set([ "gemini-image", ]); -function isVideoProviderType(providerType: string) { - return VIDEO_PROVIDER_TYPES.has(providerType); +function isVideoProviderType( + providerType: string, + model?: { + modelId?: string | null; + capabilities?: unknown; + supportsImageGen?: boolean | null; + } +) { + return isVideoModelDescriptor({ + providerType, + modelId: model?.modelId, + capabilities: model?.capabilities, + supportsImageGen: model?.supportsImageGen, + }); } function isConversationalCandidate( @@ -69,7 +77,7 @@ function isConversationalCandidate( ) { const providerType = String(item.provider.type || ""); if (!item.model.isEnabled || item.model.isSubModel) return false; - if (isVideoProviderType(providerType)) return false; + if (isVideoProviderType(providerType, item.model)) return false; const requireVision = Boolean(options?.requireVision); const matchesPreferredProvider = CHAT_PROVIDER_TYPES.includes( @@ -108,13 +116,27 @@ function matchesIntentRequirements( if (params.intent === "vision") { return ( - !isVideoProviderType(String(item.provider.type || "")) && + !isVideoProviderType(String(item.provider.type || ""), item.model) && hasVisionCapability(item.model) ); } + if (params.intent === "video_gen") { + return isVideoProviderType(String(item.provider.type || ""), item.model); + } + if (params.requireVision && !hasVisionCapability(item.model)) return false; - if (params.requireImageGen && !item.model.supportsImageGen) return false; + if ( + params.requireImageGen && + !isImageGenerationModelDescriptor({ + providerType: item.provider.type, + modelId: item.model.modelId, + capabilities: item.model.capabilities, + supportsImageGen: item.model.supportsImageGen, + }) + ) { + return false; + } return true; } @@ -132,7 +154,7 @@ const INTENT_MODEL_MAP: Record< > = { // 图片编辑：优先 alibaba-image（灵魂4.5，支持提示词扩展） image_edit: { - providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini", "huanke-gemini"], + providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini", "huanke-gemini"], requireVision: true, requireImageGen: true, description: "图片编辑", @@ -147,14 +169,14 @@ const INTENT_MODEL_MAP: Record< // 图片生成：优先灵魂4.5 image_gen: { - providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini", "huanke-gemini", "volcano-vision"], + providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini", "huanke-gemini", "volcano-vision"], requireImageGen: true, description: "图片生成", }, // 视频生成 video_gen: { - providerTypes: ["doubao-video", "volcano-video", "qianwen-video", "fal-video", "ruxa-video"], + providerTypes: ["doubao-video", "volcano-video", "qianwen-video", "fal-video", "ruxa-video", "runway"], description: "视频生成", }, @@ -175,25 +197,25 @@ const SUB_INTENT_OVERRIDES: Record< > = { // 抠图：alibaba-image 效果最好 remove_background: { - providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini"], + providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini"], reason: "抠图首选灵魂4.5", }, // 换背景：alibaba-image 支持场景生成 replace_background: { - providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini"], + providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini"], reason: "换背景首选灵魂4.5", }, // 图片增强 enhance: { - providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini", "volcano-vision"], + providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini", "volcano-vision"], reason: "增强首选灵魂4.5", }, // 风格迁移：gemini 可能更有创意 style_transfer: { - providerTypes: ["gemini", "alibaba-image", "ruxa-image", "fal-image", "openai-image"], + providerTypes: ["gemini", "alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway"], reason: "风格迁移优选Gemini", }, }; @@ -321,7 +343,7 @@ export async function routeToModel(params: { return ( !item.model.isSubModel && item.model.isEnabled && - !isVideoProviderType(String(item.provider.type || "")) && + !isVideoProviderType(String(item.provider.type || ""), item.model) && (item.model.supportsImageGen || hasVisionCapability(item.model)) ); }); @@ -345,7 +367,7 @@ export async function routeToModel(params: { (m) => !m.model.isSubModel && m.model.isEnabled && - !isVideoProviderType(String(m.provider.type || "")) && + !isVideoProviderType(String(m.provider.type || ""), m.model) && !m.model.supportsImageGen, ) : undefined) || diff --git a/lib/ai/providers/runway.ts b/lib/ai/providers/runway.ts new file mode 100644 index 000000000..96fd84e8c --- /dev/null +++ b/lib/ai/providers/runway.ts @@ -0,0 +1,581 @@ +import "server-only"; + +import sharp from "sharp"; +import { uploadFile } from "@/lib/storage/service"; +import { inferRunwayModelKind } from "@/lib/ai/model-classification"; + +interface RunwayConfig { + apiKey: string; + baseURL: string; +} + +interface RunwayImageParams { + aspectRatio?: string; + imageSize?: string; + imageCount?: number; + seed?: number; +} + +interface RunwayVideoParams { + mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images"; + ratio?: string; + duration?: number; + seed?: number; +} + +interface UploadedImage { + thumbnail: string; + original: string; +} + +const RUNWAY_VERSION = "2024-11-06"; +const DEFAULT_BASE_URL = "https://api.dev.runwayml.com"; +const TASK_POLL_INTERVAL_MS = 5000; +const TASK_TIMEOUT_MS = 15 * 60 * 1000; + +const IMAGE_RESOLUTION_MAP: Record> = { + "1K": { + "21:9": "1584:672", + "16:9": "1360:768", + "9:16": "768:1360", + "4:3": "1104:832", + "3:4": "832:1104", + "1:1": "960:960", + }, + "2K": { + "21:9": "1584:672", + "16:9": "1360:768", + "9:16": "768:1360", + "4:3": "1104:832", + "3:4": "832:1104", + "1:1": "960:960", + }, + "4K": { + "21:9": "1584:672", + "16:9": "1360:768", + "9:16": "768:1360", + "4:3": "1104:832", + "3:4": "832:1104", + "1:1": "960:960", + }, +}; + +function normalizeBaseURL(baseURL: string) { + const normalized = String(baseURL || "").trim().replace(/\/+$/, ""); + return normalized || DEFAULT_BASE_URL; +} + +function joinEndpoint(baseURL: string, path: string) { + const cleanPath = path.startsWith("/") ? path : `/${path}`; + return `${normalizeBaseURL(baseURL)}${cleanPath}`; +} + +function buildHeaders(apiKey: string) { + return { + Authorization: `Bearer ${String(apiKey || "").trim()}`, + "Content-Type": "application/json", + "X-Runway-Version": RUNWAY_VERSION, + }; +} + +function normalizeText(value: string) { + return String(value || "").replace(/\u00a0/g, " ").trim().normalize("NFKC"); +} + +function normalizeImageSizeLabel(imageSize?: string): "1K" | "2K" | "4K" { + const normalized = String(imageSize || "").trim().toUpperCase(); + if (normalized === "1K" || normalized === "2K" || normalized === "4K") { + return normalized; + } + return "2K"; +} + +function normalizeImageResolution(aspectRatio?: string, imageSize?: string) { + const normalizedRatio = String(aspectRatio || "").trim() || "16:9"; + const normalizedSize = normalizeImageSizeLabel(imageSize); + return ( + IMAGE_RESOLUTION_MAP[normalizedSize]?.[normalizedRatio] || + IMAGE_RESOLUTION_MAP[normalizedSize]?.["16:9"] || + "1360:768" + ); +} + +function normalizeVideoRatio(ratio?: string) { + const normalized = String(ratio || "").trim(); + if (["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"].includes(normalized)) { + return normalized; + } + return "16:9"; +} + +function normalizeVideoDuration(duration?: number) { + if (!Number.isFinite(duration)) return 5; + const normalized = Math.round(Number(duration)); + return normalized >= 8 ? 10 : 5; +} + +function normalizeSeed(seed?: number) { + if (!Number.isFinite(seed)) return undefined; + return Math.max(0, Math.min(4294967295, Math.round(Number(seed)))); +} + +function detectExtensionFromContentType(contentType?: string | null) { + const normalized = String(contentType || "").toLowerCase(); + if (normalized.includes("jpeg") || normalized.includes("jpg")) return "jpg"; + if (normalized.includes("webp")) return "webp"; + if (normalized.includes("gif")) return "gif"; + if (normalized.includes("png")) return "png"; + if (normalized.includes("mp4")) return "mp4"; + if (normalized.includes("quicktime")) return "mov"; + if (normalized.includes("webm")) return "webm"; + return "bin"; +} + +function extractDataUriParts(dataUri: string) { + const match = dataUri.match(/^data:([^;]+);base64,(.+)$/); + if (!match) { + throw new Error("不支持的 Data URI 格式"); + } + return { + mimeType: match[1], + buffer: Buffer.from(match[2], "base64"), + }; +} + +function generateFilename(prefix: string, extension: string) { + const timestamp = Date.now(); + const random = Math.random().toString(36).slice(2, 8); + return `${prefix}-${timestamp}-${random}.${extension}`; +} + +async function uploadGeneratedBuffer( + buffer: Buffer, + extension: string, + mimeType: string +): Promise { + const thumbnailBuffer = await sharp(buffer) + .resize(400, 400, { fit: "inside", withoutEnlargement: true }) + .jpeg({ quality: 80 }) + .toBuffer(); + + const originalFilename = generateFilename("runway-image", extension); + const originalArrayBuffer = buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) as ArrayBuffer; + const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer); + + const thumbnailFilename = generateFilename("runway-thumb", "jpg"); + const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice( + thumbnailBuffer.byteOffset, + thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength + ) as ArrayBuffer; + const thumbnailUploaded = await uploadFile(thumbnailFilename, thumbnailArrayBuffer); + + console.log("[Runway] Uploaded image to storage:", { + original: originalUploaded.url, + thumbnail: thumbnailUploaded.url, + mimeType, + }); + + return { + thumbnail: thumbnailUploaded.url, + original: originalUploaded.url, + }; +} + +async function uploadDataUriAsset(dataUri: string, prefix: "runway-input" | "runway-video") { + const { mimeType, buffer } = extractDataUriParts(dataUri); + const extension = detectExtensionFromContentType(mimeType); + const filename = generateFilename(prefix, extension); + const arrayBuffer = buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) as ArrayBuffer; + const uploaded = await uploadFile(filename, arrayBuffer); + return uploaded.url; +} + +async function normalizeAssetUrl(url?: string | null) { + const normalized = String(url || "").trim(); + if (!normalized) return ""; + if (/^https:\/\//i.test(normalized) || /^runway:\/\//i.test(normalized)) { + return normalized; + } + if (/^data:(image|video)\//i.test(normalized)) { + return uploadDataUriAsset(normalized, /^data:video\//i.test(normalized) ? "runway-video" : "runway-input"); + } + return normalized; +} + +function collectHttpUrls(payload: unknown): string[] { + const urls = new Set(); + + const visit = (value: unknown) => { + if (typeof value === "string") { + const normalized = value.trim(); + if (/^https?:\/\//i.test(normalized)) { + urls.add(normalized); + } + return; + } + + if (Array.isArray(value)) { + value.forEach(visit); + return; + } + + if (value && typeof value === "object") { + Object.values(value as Record).forEach(visit); + } + }; + + visit(payload); + return Array.from(urls); +} + +async function downloadRemoteAsset(url: string) { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 120000); + + try { + const response = await fetch(url, { + cache: "no-store", + signal: controller.signal, + }); + if (!response.ok) { + throw new Error(`下载 Runway 输出失败: ${response.status}`); + } + const buffer = Buffer.from(await response.arrayBuffer()); + return { + buffer, + contentType: response.headers.get("content-type"), + }; + } finally { + clearTimeout(timeoutId); + } +} + +async function uploadVideoToStorage(remoteVideoUrl: string) { + try { + const { buffer, contentType } = await downloadRemoteAsset(remoteVideoUrl); + const extension = detectExtensionFromContentType(contentType || "video/mp4"); + const filename = generateFilename("runway-video", extension === "bin" ? "mp4" : extension); + const arrayBuffer = buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) as ArrayBuffer; + const uploaded = await uploadFile(filename, arrayBuffer); + return uploaded.url; + } catch (error) { + console.warn("[Runway] Failed to archive video output, using temporary URL:", error); + return remoteVideoUrl; + } +} + +function formatImageMarkdown(images: UploadedImage[], imageSize: string) { + return images + .map((image) => `![thumb|||${image.original}|||${imageSize}](${image.thumbnail})`) + .join("\n\n"); +} + +function formatVideoMarkdown(videoUrl: string) { + return `\n\n[下载视频](${videoUrl})`; +} + +function extractPromptAndAssets(messages: Array<{ role: string; content: any }>) { + const lastUserMessage = messages.filter((message) => message.role === "user").pop(); + if (!lastUserMessage) { + throw new Error("未找到用户输入内容"); + } + + let prompt = ""; + const imageUrls: string[] = []; + + const pushImageUrl = (value?: string) => { + if (!value) return; + const normalized = String(value || "").trim(); + if (!normalized) return; + if (!imageUrls.includes(normalized)) { + imageUrls.push(normalized); + } + }; + + if (typeof lastUserMessage.content === "string") { + prompt = normalizeText(lastUserMessage.content); + } else if (Array.isArray(lastUserMessage.content)) { + for (const part of lastUserMessage.content) { + if (part?.type === "text" && typeof part.text === "string") { + prompt = `${prompt} ${part.text}`.trim(); + } else if (part?.type === "image_url" && part.image_url?.url) { + pushImageUrl(part.image_url.url); + } else if (part?.type === "file") { + pushImageUrl(part.url || part.data); + } else if (part?.type === "image") { + pushImageUrl(part.image || part.url); + } + } + } + + return { + prompt: normalizeText(prompt) || "生成内容", + imageUrls, + }; +} + +async function createTask(baseURL: string, apiKey: string, path: string, body: Record) { + const response = await fetch(joinEndpoint(baseURL, path), { + method: "POST", + headers: buildHeaders(apiKey), + body: JSON.stringify(body), + }); + + const rawText = await response.text(); + let payload: any = null; + try { + payload = rawText ? JSON.parse(rawText) : null; + } catch { + payload = rawText; + } + + if (!response.ok) { + const message = + typeof payload?.error === "string" + ? payload.error + : typeof payload?.message === "string" + ? payload.message + : rawText || "Runway 请求失败"; + throw new Error(`${message} (HTTP ${response.status})`); + } + + const taskId = String(payload?.id || "").trim(); + if (!taskId) { + throw new Error("Runway 未返回任务 ID"); + } + + return taskId; +} + +async function waitForTaskOutput(baseURL: string, apiKey: string, taskId: string) { + const deadline = Date.now() + TASK_TIMEOUT_MS; + + while (Date.now() < deadline) { + const response = await fetch(joinEndpoint(baseURL, `/v1/tasks/${taskId}`), { + method: "GET", + headers: buildHeaders(apiKey), + cache: "no-store", + }); + + const rawText = await response.text(); + let payload: any = null; + try { + payload = rawText ? JSON.parse(rawText) : null; + } catch { + payload = rawText; + } + + if (!response.ok) { + const message = + typeof payload?.error === "string" + ? payload.error + : typeof payload?.message === "string" + ? payload.message + : rawText || "Runway 轮询失败"; + throw new Error(`${message} (HTTP ${response.status})`); + } + + const status = String(payload?.status || "").trim().toUpperCase(); + if (status === "SUCCEEDED") { + return payload; + } + if (status === "FAILED" || status === "CANCELLED") { + throw new Error( + String(payload?.failure || payload?.error || payload?.message || "Runway 任务失败") + ); + } + + await new Promise((resolve) => setTimeout(resolve, TASK_POLL_INTERVAL_MS)); + } + + throw new Error("Runway 任务超时，请稍后重试"); +} + +async function uploadRunwayImages(taskOutput: unknown) { + const outputUrls = collectHttpUrls(taskOutput); + const uploadedImages: UploadedImage[] = []; + + for (const url of outputUrls) { + try { + const { buffer, contentType } = await downloadRemoteAsset(url); + if (!String(contentType || "").startsWith("image/")) { + continue; + } + const extension = detectExtensionFromContentType(contentType); + uploadedImages.push( + await uploadGeneratedBuffer( + buffer, + extension === "bin" ? "png" : extension, + contentType || "image/png" + ) + ); + } catch (error) { + console.warn("[Runway] Failed to archive image output:", error); + } + } + + if (uploadedImages.length === 0) { + throw new Error("Runway 未返回可用图片"); + } + + return uploadedImages; +} + +async function resolveRunwayVideoUrl(taskOutput: unknown) { + const outputUrls = collectHttpUrls(taskOutput); + if (outputUrls.length === 0) { + throw new Error("Runway 未返回可用视频地址"); + } + + for (const url of outputUrls) { + if (/(\.mp4|\.mov|\.webm)(\?|$)/i.test(url) || /video/i.test(url)) { + return uploadVideoToStorage(url); + } + } + + return uploadVideoToStorage(outputUrls[0]!); +} + +function getRunwayVideoEndpoint(modelId: string, hasPromptImage: boolean) { + const normalized = normalizeText(modelId); + if (hasPromptImage) return "/v1/image_to_video"; + if (normalized === "gen4.5") return "/v1/image_to_video"; + return "/v1/text_to_video"; +} + +function buildReferenceImages(imageUrls: string[]) { + return imageUrls.slice(0, 3).map((uri, index) => ({ + uri, + tag: `ref_${index + 1}`.replace(/[^a-z0-9_]/g, "_"), + })); +} + +export function createRunway({ apiKey, baseURL }: RunwayConfig) { + return { + provider: "runway", + chat: (modelId: string) => ({ + provider: "runway", + doGenerate: async (options: { + messages: Array<{ role: string; content: any }>; + imageParams?: RunwayImageParams; + videoParams?: RunwayVideoParams; + overrideModelId?: string; + }) => { + try { + const activeModelId = normalizeText(options.overrideModelId || modelId); + const { prompt, imageUrls } = extractPromptAndAssets(options.messages); + const normalizedImageUrls = ( + await Promise.all(imageUrls.map((url) => normalizeAssetUrl(url))) + ).filter(Boolean); + const modelKind = + inferRunwayModelKind(activeModelId) || + (options.videoParams ? "video" : options.imageParams ? "image" : null) || + "video"; + + if (modelKind === "image") { + const imageSize = normalizeImageSizeLabel(options.imageParams?.imageSize); + const taskId = await createTask(baseURL, apiKey, "/v1/text_to_image", { + model: activeModelId, + promptText: prompt, + ratio: normalizeImageResolution(options.imageParams?.aspectRatio, imageSize), + ...(normalizedImageUrls.length > 0 + ? { referenceImages: buildReferenceImages(normalizedImageUrls) } + : {}), + ...(normalizeSeed(options.imageParams?.seed) !== undefined + ? { seed: normalizeSeed(options.imageParams?.seed) } + : {}), + }); + + const task = await waitForTaskOutput(baseURL, apiKey, taskId); + const uploadedImages = await uploadRunwayImages(task?.output ?? task); + const text = formatImageMarkdown(uploadedImages, imageSize); + + return { + text, + finishReason: "stop" as const, + usage: { + promptTokens: 0, + completionTokens: text.length, + }, + }; + } + + const videoMode = options.videoParams?.mode || (normalizedImageUrls[0] ? "first-frame" : "text-to-video"); + if ( + (videoMode === "first-frame" || videoMode === "first-last-frame" || videoMode === "reference-images") && + !normalizedImageUrls[0] + ) { + throw new Error("Runway 图生视频至少需要 1 张参考图"); + } + + const endpoint = getRunwayVideoEndpoint(activeModelId, Boolean(normalizedImageUrls[0])); + const taskId = await createTask(baseURL, apiKey, endpoint, { + model: activeModelId, + promptText: prompt, + ratio: normalizeVideoRatio(options.videoParams?.ratio), + duration: normalizeVideoDuration(options.videoParams?.duration), + ...(normalizedImageUrls[0] ? { promptImage: normalizedImageUrls[0] } : {}), + ...(normalizeSeed(options.videoParams?.seed) !== undefined + ? { seed: normalizeSeed(options.videoParams?.seed) } + : {}), + }); + + const task = await waitForTaskOutput(baseURL, apiKey, taskId); + const videoUrl = await resolveRunwayVideoUrl(task?.output ?? task); + const text = formatVideoMarkdown(videoUrl); + + return { + text, + finishReason: "stop" as const, + usage: { + promptTokens: 0, + completionTokens: text.length, + }, + }; + } catch (error) { + console.error("[Runway] Generation failed:", error); + return { + text: + error instanceof Error + ? `${ + (inferRunwayModelKind(options.overrideModelId || modelId) || + (options.videoParams ? "video" : options.imageParams ? "image" : "video")) === "image" + ? "图片" + : "视频" + }生成失败: ${error.message}` + : "Runway 生成失败，请稍后重试", + finishReason: "error" as const, + usage: { + promptTokens: 0, + completionTokens: 0, + }, + }; + } + }, + doStream: async function* (options: { + messages: Array<{ role: string; content: any }>; + imageParams?: RunwayImageParams; + videoParams?: RunwayVideoParams; + overrideModelId?: string; + }) { + const result = await this.doGenerate(options); + yield { + type: "text-delta", + textDelta: result.text, + }; + }, + modelId, + specificationVersion: "v1" as const, + defaultObjectGenerationMode: "json" as const, + }), + }; +} diff --git a/lib/ai/video-tool-capability.ts b/lib/ai/video-tool-capability.ts index a6686893a..e101aaf59 100644 --- a/lib/ai/video-tool-capability.ts +++ b/lib/ai/video-tool-capability.ts @@ -193,6 +193,33 @@ function createProviderDefaultCapability(providerType: string): VideoToolCapabil }; } + if (providerType === "runway") { + return { + schemaVersion: "v1", + source: "provider-default", + providerType, + modeOptions: normalizeModeOptions(["text-to-video", "first-frame"]), + ratioOptions: [...STANDARD_RATIOS], + durationOptions: [5, 10], + resolutionOptions: [], + ui: { + showResolution: false, + showFps: false, + showCameraMove: false, + showGenerateAudio: false, + showReturnLastFrame: false, + showSeed: true, + }, + defaults: { + mode: "text-to-video", + ratio: "16:9", + duration: 5, + generateAudio: false, + fps: 24, + }, + }; + } + return { schemaVersion: "v1", source: "provider-default", diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 871d37124..87cfddd22 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -7,6 +7,7 @@ import { createRuxaImage } from "@/lib/ai/providers/ruxa-image"; import { createFalImage } from "@/lib/ai/providers/fal-image"; import { createOpenAIImage } from "@/lib/ai/providers/openai-image"; import { createMidjourney } from "@/lib/ai/providers/midjourney"; +import { createRunway } from "@/lib/ai/providers/runway"; import { filterReachableRemoteUrls } from "@/lib/net/url-access"; export interface CanvasGenerateImageRequest { @@ -39,6 +40,7 @@ const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([ "fal-image", "openai-image", "midjourney", + "runway", ]); function parseImagesFromMarkdown(markdown: string): ParsedImage[] { @@ -76,6 +78,7 @@ function getProviderBatchLimit(providerType: string) { case "ruxa-image": case "openai-image": case "midjourney": + case "runway": return 1; case "gemini": return 2; @@ -151,7 +154,7 @@ export async function generateCanvasImage( } if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(fallbackProvider.type)) { throw new Error( - "当前图片模型类型暂不支持画布生成，请在后台选择 Gemini/阿里/RUXA/FAL/OpenAI 图片模型。" + "当前图片模型类型暂不支持画布生成，请在后台选择 Gemini/阿里/RUXA/FAL/OpenAI/Runway 图片模型。" ); } if (fallbackModel.model.id !== selectedModel.model.id) { @@ -311,9 +314,22 @@ export async function generateCanvasImage( }); return result.text || ""; }; + } else if (provider.type === "runway") { + const runwayProvider = createRunway({ + apiKey: provider.apiKey, + baseURL: provider.baseUrl, + }); + const modelInstance = runwayProvider.chat(selectedModel.model.modelId); + executeGenerateBatch = async (_batchImageCount) => { + const result = await modelInstance.doGenerate({ + messages, + imageParams, + }); + return result.text || ""; + }; } else { throw new Error( - `不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/阿里/RUXA/FAL/OpenAI/Midjourney 图片模型` + `不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/阿里/RUXA/FAL/OpenAI/Midjourney/Runway 图片模型` ); } diff --git a/lib/db/queries.ts b/lib/db/queries.ts index d0f1620af..59c47b553 100644 --- a/lib/db/queries.ts +++ b/lib/db/queries.ts @@ -91,6 +91,7 @@ import { } from "../points/config"; import { inferOriginalImageUrl } from "../image-url"; import { getAgentRequiredUploadCount } from "../agents/preset-mode-config"; +import { isVideoModelDescriptor } from "../ai/model-classification"; // Optionally, if not using email/pass login, you can // use the Drizzle adapter for Auth.js / NextAuth @@ -1755,14 +1756,6 @@ type AgentPresetKind = "image" | "video" | "text"; type AgentPresetScope = "all" | AgentPresetVisibility; type AgentCategoryScope = "all" | "public" | "private" | "global"; type AgentPresetSort = "latest" | "hottest" | "most_used"; -const AGENT_VIDEO_PROVIDER_TYPES = new Set([ - "volcano-video", - "doubao-video", - "qianwen-video", - "fal-video", - "ruxa-video", -]); - function normalizeAgentCategoryName(value: string) { return String(value || "").trim().replace(/\s+/g, " "); } @@ -1804,7 +1797,12 @@ async function validateAgentDefaultModel(params: { throw new ChatSDKError("bad_request:database", "Invalid default model"); } - const isVideoModel = AGENT_VIDEO_PROVIDER_TYPES.has(String(model.providerType || "")); + const isVideoModel = isVideoModelDescriptor({ + providerType: model.providerType, + modelId: model.modelId, + capabilities: model.capabilities, + supportsImageGen: model.supportsImageGen, + }); if (params.presetType === "video") { if (!isVideoModel) {