From dbfc8e97d29d2433eee4ea9779039033c654ee82 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 21 Mar 2026 01:15:24 +0800 Subject: [PATCH] fix: align runway video params with model capabilities --- app/(chat)/api/chat/route.ts | 4 + components/canvas/CanvasChatPanel.tsx | 44 ++- components/chat.tsx | 46 ++- components/video-params-bar.tsx | 95 +++-- lib/ai/providers/runway.ts | 130 +++++-- lib/ai/runway-video-models.ts | 482 ++++++++++++++++++++++++++ lib/ai/video-tool-capability.ts | 310 +++++++++++++++-- types/video-tool-capability.ts | 8 + 8 files changed, 994 insertions(+), 125 deletions(-) create mode 100644 lib/ai/runway-video-models.ts diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 4ac130ddd..7294bf264 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -8922,6 +8922,10 @@ export async function POST(request: Request) { videoParams: effectiveVideoParams, }); + if (result?.finishReason === "error") { + throw new Error(result.text || "视频生成失败"); + } + videoMarkdown = result.text || "视频生成失败"; tempVideoUrl = extractVideoUrlFromText(videoMarkdown) || ""; } diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 373371b70..596eeca65 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -51,6 +51,7 @@ import { isImageGenerationModelDescriptor, isVideoModelDescriptor, } from "@/lib/ai/model-classification"; +import { getVideoToolOptionsForMode } from "@/lib/ai/video-tool-capability"; import type { Attachment, ChatMessage } from "@/lib/types"; import type { AgentPreset, @@ -893,24 +894,14 @@ function normalizeVideoParamsForProvider( ): VideoGenerationParams { if (capability) { const modeOptions = capability.modeOptions || []; - const ratioOptions = capability.ratioOptions || []; - const durationOptions = capability.durationOptions || []; - const resolutionOptions = capability.resolutionOptions || []; const modeSet = new Set(modeOptions.map((item) => item.mode)); - const ratioSet = new Set(ratioOptions); - const durationSet = new Set(durationOptions); - const resolutionSet = new Set(resolutionOptions); - const fallbackMode = capability.defaults?.mode || modeOptions[0]?.mode || "text-to-video"; - const fallbackRatio = capability.defaults?.ratio || ratioOptions[0] || "16:9"; - const fallbackDuration = capability.defaults?.duration || durationOptions[0] || 5; - const fallbackResolution = capability.defaults?.resolution || resolutionOptions[0] || "720p"; const merged = { mode: fallbackMode, - resolution: fallbackResolution as VideoGenerationParams["resolution"], - ratio: fallbackRatio as VideoGenerationParams["ratio"], - duration: Number(fallbackDuration) || 5, + resolution: (capability.defaults?.resolution || "720p") as VideoGenerationParams["resolution"], + ratio: (capability.defaults?.ratio || "16:9") as VideoGenerationParams["ratio"], + duration: Number(capability.defaults?.duration) || 5, generateAudio: capability.defaults?.generateAudio ?? false, watermark: false, cameraFixed: false, @@ -922,16 +913,39 @@ function normalizeVideoParamsForProvider( if (!modeSet.has(merged.mode)) { merged.mode = fallbackMode as VideoGenerationParams["mode"]; } + + const scopedOptions = getVideoToolOptionsForMode(capability, merged.mode); + const ratioOptions = scopedOptions.ratioOptions || []; + const durationOptions = scopedOptions.durationOptions || []; + const resolutionOptions = scopedOptions.resolutionOptions || []; + const ratioSet = new Set(ratioOptions); + const durationSet = new Set(durationOptions); + const resolutionSet = new Set(resolutionOptions); + const fallbackRatio = + scopedOptions.defaults?.ratio || ratioOptions[0] || capability.defaults?.ratio || "16:9"; + const fallbackDuration = + Number(scopedOptions.defaults?.duration) || + durationOptions[0] || + Number(capability.defaults?.duration) || + 5; + const fallbackResolution = + scopedOptions.defaults?.resolution || + resolutionOptions[0] || + capability.defaults?.resolution || + "720p"; + if (!ratioSet.has(merged.ratio)) { merged.ratio = fallbackRatio as VideoGenerationParams["ratio"]; } if (!durationSet.has(Number(merged.duration))) { - merged.duration = Number(fallbackDuration) || 5; + merged.duration = fallbackDuration; } else { - merged.duration = Number(merged.duration) || Number(fallbackDuration) || 5; + merged.duration = Number(merged.duration) || fallbackDuration; } if (resolutionOptions.length > 0 && !resolutionSet.has(merged.resolution)) { merged.resolution = fallbackResolution as VideoGenerationParams["resolution"]; + } else if (resolutionOptions.length === 0) { + merged.resolution = fallbackResolution as VideoGenerationParams["resolution"]; } if (!capability.ui.showGenerateAudio) { merged.generateAudio = false; diff --git a/components/chat.tsx b/components/chat.tsx index 965a05510..83c5895e6 100644 --- a/components/chat.tsx +++ b/components/chat.tsx @@ -32,6 +32,7 @@ import { isImageGenerationModelDescriptor, isVideoModelDescriptor, } from "@/lib/ai/model-classification"; +import { getVideoToolOptionsForMode } from "@/lib/ai/video-tool-capability"; import { fetcher, fetchWithErrorHandlers, generateUUID } from "@/lib/utils"; import { Artifact } from "@/components/artifact"; import { ProtocolDebugPanel } from "@/components/protocol-debug-panel"; @@ -85,24 +86,14 @@ function normalizeVideoParamsForProvider( ): VideoGenerationParams { if (capability) { const modeOptions = capability.modeOptions || []; - const ratioOptions = capability.ratioOptions || []; - const durationOptions = capability.durationOptions || []; - const resolutionOptions = capability.resolutionOptions || []; const modeSet = new Set(modeOptions.map((item) => item.mode)); - const ratioSet = new Set(ratioOptions); - const durationSet = new Set(durationOptions); - const resolutionSet = new Set(resolutionOptions); - const fallbackMode = capability.defaults?.mode || modeOptions[0]?.mode || "text-to-video"; - const fallbackRatio = capability.defaults?.ratio || ratioOptions[0] || "16:9"; - const fallbackDuration = capability.defaults?.duration || durationOptions[0] || 5; - const fallbackResolution = capability.defaults?.resolution || resolutionOptions[0] || "720p"; const merged = { mode: fallbackMode, - resolution: fallbackResolution as VideoGenerationParams["resolution"], - ratio: fallbackRatio as VideoGenerationParams["ratio"], - duration: Number(fallbackDuration) || 5, + resolution: (capability.defaults?.resolution || "720p") as VideoGenerationParams["resolution"], + ratio: (capability.defaults?.ratio || "16:9") as VideoGenerationParams["ratio"], + duration: Number(capability.defaults?.duration) || 5, generateAudio: capability.defaults?.generateAudio ?? false, watermark: false, cameraFixed: false, @@ -114,15 +105,38 @@ function normalizeVideoParamsForProvider( if (!modeSet.has(merged.mode)) { merged.mode = fallbackMode as VideoGenerationParams["mode"]; } + + const scopedOptions = getVideoToolOptionsForMode(capability, merged.mode); + const ratioOptions = scopedOptions.ratioOptions || []; + const durationOptions = scopedOptions.durationOptions || []; + const resolutionOptions = scopedOptions.resolutionOptions || []; + const ratioSet = new Set(ratioOptions); + const durationSet = new Set(durationOptions); + const resolutionSet = new Set(resolutionOptions); + const fallbackRatio = + scopedOptions.defaults?.ratio || ratioOptions[0] || capability.defaults?.ratio || "16:9"; + const fallbackDuration = + Number(scopedOptions.defaults?.duration) || + durationOptions[0] || + Number(capability.defaults?.duration) || + 5; + const fallbackResolution = + scopedOptions.defaults?.resolution || + resolutionOptions[0] || + capability.defaults?.resolution || + "720p"; + if (!ratioSet.has(merged.ratio)) { merged.ratio = fallbackRatio as VideoGenerationParams["ratio"]; } if (!durationSet.has(Number(merged.duration))) { - merged.duration = Number(fallbackDuration) || 5; + merged.duration = fallbackDuration; } else { - merged.duration = Number(merged.duration) || Number(fallbackDuration) || 5; + merged.duration = Number(merged.duration) || fallbackDuration; } - if (!resolutionSet.has(merged.resolution)) { + if (resolutionOptions.length > 0 && !resolutionSet.has(merged.resolution)) { + merged.resolution = fallbackResolution as VideoGenerationParams["resolution"]; + } else if (resolutionOptions.length === 0) { merged.resolution = fallbackResolution as VideoGenerationParams["resolution"]; } diff --git a/components/video-params-bar.tsx b/components/video-params-bar.tsx index f8ba2c6e0..e774910ec 100644 --- a/components/video-params-bar.tsx +++ b/components/video-params-bar.tsx @@ -40,6 +40,7 @@ import { Volume2, VolumeX, } from "lucide-react"; +import { getVideoToolOptionsForMode } from "@/lib/ai/video-tool-capability"; import { cn } from "@/lib/utils"; import type { VideoToolCapabilityContract } from "@/types/video-tool-capability"; @@ -172,21 +173,13 @@ function normalizeVideoParamsByProvider( ): VideoGenerationParams { if (capability) { const modes = capability.modeOptions.map((item) => item.mode); - const ratios = capability.ratioOptions; - const durations = capability.durationOptions; - const resolutions = capability.resolutionOptions; - const fallbackMode = capability.defaults.mode || modes[0] || "text-to-video"; - const fallbackRatio = capability.defaults.ratio || ratios[0] || "16:9"; - const fallbackDuration = capability.defaults.duration || durations[0] || 5; - const fallbackResolution = - capability.defaults.resolution || resolutions[0] || "720p"; const merged = { mode: fallbackMode, - resolution: fallbackResolution as VideoGenerationParams["resolution"], - ratio: fallbackRatio as VideoGenerationParams["ratio"], - duration: Number(fallbackDuration) || 5, + resolution: (capability.defaults.resolution || "720p") as VideoGenerationParams["resolution"], + ratio: (capability.defaults.ratio || "16:9") as VideoGenerationParams["ratio"], + duration: Number(capability.defaults.duration) || 5, generateAudio: capability.defaults.generateAudio ?? false, watermark: false, cameraFixed: false, @@ -198,20 +191,39 @@ function normalizeVideoParamsByProvider( if (!modes.includes(merged.mode)) { merged.mode = fallbackMode as VideoGenerationParams["mode"]; } + + const scopedOptions = getVideoToolOptionsForMode(capability, merged.mode); + const ratios = scopedOptions.ratioOptions; + const durations = scopedOptions.durationOptions; + const resolutions = scopedOptions.resolutionOptions; + const fallbackRatio = + scopedOptions.defaults?.ratio || ratios[0] || capability.defaults.ratio || "16:9"; + const fallbackDuration = + Number(scopedOptions.defaults?.duration) || durations[0] || Number(capability.defaults.duration) || 5; + const fallbackResolution = + scopedOptions.defaults?.resolution || + resolutions[0] || + capability.defaults.resolution || + "720p"; + if (!ratios.includes(merged.ratio)) { merged.ratio = fallbackRatio as VideoGenerationParams["ratio"]; } if (!durations.includes(Number(merged.duration))) { - merged.duration = Number(fallbackDuration) || 5; + merged.duration = fallbackDuration; } else { - merged.duration = Number(merged.duration) || Number(fallbackDuration) || 5; + merged.duration = Number(merged.duration) || fallbackDuration; } if (resolutions.length > 0 && !resolutions.includes(merged.resolution)) { merged.resolution = fallbackResolution as VideoGenerationParams["resolution"]; + } else if (resolutions.length === 0) { + merged.resolution = fallbackResolution as VideoGenerationParams["resolution"]; } if (!capability.ui.showGenerateAudio) { merged.generateAudio = false; + } else if (typeof merged.generateAudio !== "boolean") { + merged.generateAudio = scopedOptions.defaults?.generateAudio ?? capability.defaults.generateAudio ?? false; } if (!capability.ui.showFps) { merged.fps = capability.defaults.fps || 24; @@ -290,6 +302,7 @@ export function VideoParamsBar({ }: VideoParamsBarProps) { const isFalVideo = providerType === "fal-video"; const isRuxaVideo = providerType === "ruxa-video"; + const capabilityModeOptions = capabilities?.modeOptions?.length ? capabilities.modeOptions.map((item) => ({ value: item.mode, @@ -301,30 +314,6 @@ export function VideoParamsBar({ : isRuxaVideo ? RUXA_MODE_OPTIONS : STANDARD_MODE_OPTIONS); - const capabilityRatioOptions = capabilities?.ratioOptions?.length - ? capabilities.ratioOptions.map((value) => ({ value, label: value })) - : null; - const ratioOptions = capabilityRatioOptions || (isFalVideo - ? FAL_RATIO_OPTIONS - : isRuxaVideo - ? RUXA_RATIO_OPTIONS - : RATIO_OPTIONS); - const durationOptions = capabilities?.durationOptions?.length - ? capabilities.durationOptions - : (isFalVideo - ? FAL_DURATION_OPTIONS - : isRuxaVideo - ? RUXA_DURATION_OPTIONS - : STANDARD_DURATION_OPTIONS); - const resolutionOptions = capabilities?.resolutionOptions?.length - ? capabilities.resolutionOptions.map((value) => ({ - value, - label: value.toUpperCase(), - })) - : RESOLUTION_OPTIONS; - const showResolutionControl = capabilities - ? capabilities.ui.showResolution && resolutionOptions.length > 0 - : !isFalVideo && !isRuxaVideo; const showGenerateAudioControl = capabilities ? capabilities.ui.showGenerateAudio : isFalVideo; @@ -339,6 +328,32 @@ export function VideoParamsBar({ normalizeVideoParamsByProvider(initialParams, providerType, capabilities) ); + const scopedCapabilityOptions = getVideoToolOptionsForMode(capabilities, params.mode); + const capabilityRatioOptions = scopedCapabilityOptions.ratioOptions.length + ? scopedCapabilityOptions.ratioOptions.map((value) => ({ value, label: value })) + : null; + const ratioOptions = capabilityRatioOptions || (isFalVideo + ? FAL_RATIO_OPTIONS + : isRuxaVideo + ? RUXA_RATIO_OPTIONS + : RATIO_OPTIONS); + const durationOptions = scopedCapabilityOptions.durationOptions.length + ? scopedCapabilityOptions.durationOptions + : (isFalVideo + ? FAL_DURATION_OPTIONS + : isRuxaVideo + ? RUXA_DURATION_OPTIONS + : STANDARD_DURATION_OPTIONS); + const resolutionOptions = scopedCapabilityOptions.resolutionOptions.length + ? scopedCapabilityOptions.resolutionOptions.map((value) => ({ + value, + label: value.toUpperCase(), + })) + : RESOLUTION_OPTIONS; + const showResolutionControl = capabilities + ? capabilities.ui.showResolution && scopedCapabilityOptions.resolutionOptions.length > 0 + : !isFalVideo && !isRuxaVideo; + const [collapsed, setCollapsed] = useState(false); useEffect(() => { @@ -412,7 +427,11 @@ export function VideoParamsBar({ key: K, value: VideoGenerationParams[K] ) => { - const newParams = { ...params, [key]: value }; + const newParams = normalizeVideoParamsByProvider( + { ...params, [key]: value }, + providerType, + capabilities + ); setParams(newParams); onParamsChange?.(newParams); }; diff --git a/lib/ai/providers/runway.ts b/lib/ai/providers/runway.ts index 9c17e2f37..c64567e79 100644 --- a/lib/ai/providers/runway.ts +++ b/lib/ai/providers/runway.ts @@ -1,6 +1,13 @@ import "server-only"; import sharp from "sharp"; +import { + getRunwayVideoModeDefinition, + isRunwayAudioSupported, + isRunwaySeedSupported, + normalizeRunwayDuration, + resolveRunwayApiRatio, +} from "@/lib/ai/runway-video-models"; import { uploadFile } from "@/lib/storage/service"; import { inferRunwayModelKind } from "@/lib/ai/model-classification"; @@ -21,6 +28,7 @@ interface RunwayVideoParams { ratio?: string; duration?: number; seed?: number; + generateAudio?: boolean; } interface UploadedImage { @@ -109,20 +117,6 @@ function normalizeImageResolution(aspectRatio?: string, imageSize?: string) { ); } -function normalizeVideoRatio(ratio?: string) { - const normalized = String(ratio || "").trim(); - if (["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"].includes(normalized)) { - return normalized; - } - return "16:9"; -} - -function normalizeVideoDuration(duration?: number) { - if (!Number.isFinite(duration)) return 5; - const normalized = Math.round(Number(duration)); - return normalized >= 8 ? 10 : 5; -} - function normalizeSeed(seed?: number) { if (!Number.isFinite(seed)) return undefined; return Math.max(0, Math.min(4294967295, Math.round(Number(seed)))); @@ -348,6 +342,23 @@ async function createTask(baseURL: string, apiKey: string, path: string, body: R } if (!response.ok) { + console.error("[Runway] Task request failed:", { + path, + status: response.status, + bodySummary: { + model: body.model, + ratio: body.ratio, + duration: body.duration, + hasPromptImage: Boolean(body.promptImage), + promptImageCount: Array.isArray(body.promptImage) ? body.promptImage.length : body.promptImage ? 1 : 0, + hasReferenceImages: Array.isArray(body.referenceImages) ? body.referenceImages.length : 0, + hasPromptText: typeof body.promptText === "string" && body.promptText.length > 0, + promptTextLength: typeof body.promptText === "string" ? body.promptText.length : 0, + audio: body.audio, + seed: typeof body.seed === "number" ? body.seed : undefined, + }, + response: payload, + }); const message = typeof payload?.error === "string" ? payload.error @@ -461,6 +472,43 @@ function getRunwayVideoEndpoint(modelId: string, hasPromptImage: boolean) { return "/v1/text_to_video"; } +function normalizeVideoMode( + mode: RunwayVideoParams["mode"] | undefined, + hasPromptImage: boolean +): NonNullable { + if ( + mode === "text-to-video" || + mode === "first-frame" || + mode === "first-last-frame" || + mode === "reference-images" + ) { + return mode; + } + + return hasPromptImage ? "first-frame" : "text-to-video"; +} + +function buildRunwayPromptImagePayload(params: { + modelId: string; + mode: NonNullable; + imageUrls: string[]; +}) { + const resolved = getRunwayVideoModeDefinition(params.modelId, params.mode); + if (!resolved || params.imageUrls.length === 0) { + return undefined; + } + + const limitedImageUrls = params.imageUrls.slice(0, Math.max(1, resolved.definition.imageMax)); + if (params.mode === "first-last-frame" && limitedImageUrls.length >= 2) { + return [ + { uri: limitedImageUrls[0], position: "first" as const }, + { uri: limitedImageUrls[1], position: "last" as const }, + ]; + } + + return limitedImageUrls[0]; +} + function buildReferenceImages(imageUrls: string[]) { return imageUrls.slice(0, 3).map((uri, index) => ({ uri, @@ -518,7 +566,10 @@ export function createRunway({ apiKey, baseURL }: RunwayConfig) { }; } - const videoMode = options.videoParams?.mode || (normalizedImageUrls[0] ? "first-frame" : "text-to-video"); + const videoMode = normalizeVideoMode( + options.videoParams?.mode, + Boolean(normalizedImageUrls[0]) + ); if ( (videoMode === "first-frame" || videoMode === "first-last-frame" || videoMode === "reference-images") && !normalizedImageUrls[0] @@ -526,16 +577,51 @@ export function createRunway({ apiKey, baseURL }: RunwayConfig) { throw new Error("Runway 图生视频至少需要 1 张参考图"); } - const endpoint = getRunwayVideoEndpoint(activeModelId, Boolean(normalizedImageUrls[0])); + const promptImage = buildRunwayPromptImagePayload({ + modelId: activeModelId, + mode: videoMode, + imageUrls: normalizedImageUrls, + }); + const endpoint = getRunwayVideoEndpoint(activeModelId, Boolean(promptImage)); + const ratio = resolveRunwayApiRatio({ + modelId: activeModelId, + mode: videoMode, + ratio: options.videoParams?.ratio, + }); + const duration = normalizeRunwayDuration({ + modelId: activeModelId, + mode: videoMode, + duration: options.videoParams?.duration, + }); + const seed = isRunwaySeedSupported(activeModelId, videoMode) + ? normalizeSeed(options.videoParams?.seed) + : undefined; + const audio = + isRunwayAudioSupported(activeModelId, videoMode) && + typeof options.videoParams?.generateAudio === "boolean" + ? options.videoParams.generateAudio + : undefined; + + console.log("[Runway] Video request config:", { + model: activeModelId, + mode: videoMode, + endpoint, + ratio, + duration, + hasPromptImage: Boolean(promptImage), + promptImageCount: Array.isArray(promptImage) ? promptImage.length : promptImage ? 1 : 0, + audio, + seedEnabled: seed !== undefined, + }); + const taskId = await createTask(baseURL, apiKey, endpoint, { model: activeModelId, promptText: prompt, - ratio: normalizeVideoRatio(options.videoParams?.ratio), - duration: normalizeVideoDuration(options.videoParams?.duration), - ...(normalizedImageUrls[0] ? { promptImage: normalizedImageUrls[0] } : {}), - ...(normalizeSeed(options.videoParams?.seed) !== undefined - ? { seed: normalizeSeed(options.videoParams?.seed) } - : {}), + ratio, + duration, + ...(promptImage ? { promptImage } : {}), + ...(seed !== undefined ? { seed } : {}), + ...(typeof audio === "boolean" ? { audio } : {}), }); const task = await waitForTaskOutput(baseURL, apiKey, taskId); diff --git a/lib/ai/runway-video-models.ts b/lib/ai/runway-video-models.ts new file mode 100644 index 000000000..b6fc453a9 --- /dev/null +++ b/lib/ai/runway-video-models.ts @@ -0,0 +1,482 @@ +import type { + VideoToolDefaults, + VideoToolMode, + VideoToolModeConstraint, + VideoToolUiToggles, +} from "@/types/video-tool-capability"; + +type RunwayModeDefinition = VideoToolModeConstraint & { + imageMin: number; + imageMax: number; + apiRatios: Record; + supportsAudio?: boolean; + supportsSeed?: boolean; +}; + +type RunwayVideoModelDefinition = { + defaults: VideoToolDefaults; + ui: VideoToolUiToggles; + modes: Partial>; +}; + +const COMMON_IMAGE_TO_VIDEO_RATIOS = { + "21:9": "1584:672", + "16:9": "1280:720", + "4:3": "1104:832", + "1:1": "960:960", + "3:4": "832:1104", + "9:16": "720:1280", +} as const; + +const PORTRAIT_LANDSCAPE_RATIOS = { + "16:9": "1280:720", + "9:16": "720:1280", +} as const; + +const GEN3A_RATIOS = { + "16:9": "1280:768", + "9:16": "768:1280", +} as const; + +const VEO_RATIOS = { + "16:9": "1280:720", + "9:16": "720:1280", +} as const; + +const DURATIONS_2_TO_10 = Array.from({ length: 9 }, (_, index) => index + 2); +const DURATIONS_5_OR_10 = [5, 10]; +const DURATIONS_4_6_8 = [4, 6, 8]; + +function buildModeDefinition(params: { + imageMin: number; + imageMax: number; + ratioMap: Record; + durationOptions: number[]; + defaults?: Partial; + supportsAudio?: boolean; + supportsSeed?: boolean; +}): RunwayModeDefinition { + return { + imageMin: params.imageMin, + imageMax: params.imageMax, + ratioOptions: Object.keys(params.ratioMap), + durationOptions: [...params.durationOptions], + resolutionOptions: [], + defaults: params.defaults, + apiRatios: { ...params.ratioMap }, + supportsAudio: params.supportsAudio, + supportsSeed: params.supportsSeed, + }; +} + +const RUNWAY_VIDEO_MODEL_DEFINITIONS: Array<[string, RunwayVideoModelDefinition]> = [ + [ + "gen4.5", + { + defaults: { + mode: "text-to-video", + ratio: "16:9", + duration: 5, + generateAudio: false, + fps: 24, + }, + ui: { + showResolution: false, + showFps: false, + showCameraMove: false, + showGenerateAudio: false, + showReturnLastFrame: false, + showSeed: true, + }, + modes: { + "text-to-video": buildModeDefinition({ + imageMin: 0, + imageMax: 0, + ratioMap: PORTRAIT_LANDSCAPE_RATIOS, + durationOptions: DURATIONS_2_TO_10, + defaults: { + ratio: "16:9", + duration: 5, + }, + supportsSeed: true, + }), + "first-frame": buildModeDefinition({ + imageMin: 1, + imageMax: 1, + ratioMap: COMMON_IMAGE_TO_VIDEO_RATIOS, + durationOptions: DURATIONS_2_TO_10, + defaults: { + ratio: "16:9", + duration: 5, + }, + supportsSeed: true, + }), + }, + }, + ], + [ + "gen4_turbo", + { + defaults: { + mode: "first-frame", + ratio: "16:9", + duration: 5, + generateAudio: false, + fps: 24, + }, + ui: { + showResolution: false, + showFps: false, + showCameraMove: false, + showGenerateAudio: false, + showReturnLastFrame: false, + showSeed: true, + }, + modes: { + "first-frame": buildModeDefinition({ + imageMin: 1, + imageMax: 1, + ratioMap: COMMON_IMAGE_TO_VIDEO_RATIOS, + durationOptions: DURATIONS_5_OR_10, + defaults: { + ratio: "16:9", + duration: 5, + }, + supportsSeed: true, + }), + }, + }, + ], + [ + "gen3a_turbo", + { + defaults: { + mode: "first-frame", + ratio: "16:9", + duration: 5, + generateAudio: false, + fps: 24, + }, + ui: { + showResolution: false, + showFps: false, + showCameraMove: false, + showGenerateAudio: false, + showReturnLastFrame: false, + showSeed: true, + }, + modes: { + "first-frame": buildModeDefinition({ + imageMin: 1, + imageMax: 1, + ratioMap: GEN3A_RATIOS, + durationOptions: DURATIONS_5_OR_10, + defaults: { + ratio: "16:9", + duration: 5, + }, + supportsSeed: true, + }), + "first-last-frame": buildModeDefinition({ + imageMin: 2, + imageMax: 2, + ratioMap: GEN3A_RATIOS, + durationOptions: DURATIONS_5_OR_10, + defaults: { + ratio: "16:9", + duration: 5, + }, + supportsSeed: true, + }), + }, + }, + ], + [ + "veo3.1", + { + defaults: { + mode: "text-to-video", + ratio: "16:9", + duration: 4, + generateAudio: false, + fps: 24, + }, + ui: { + showResolution: false, + showFps: false, + showCameraMove: false, + showGenerateAudio: true, + showReturnLastFrame: false, + showSeed: false, + }, + modes: { + "text-to-video": buildModeDefinition({ + imageMin: 0, + imageMax: 0, + ratioMap: VEO_RATIOS, + durationOptions: DURATIONS_4_6_8, + defaults: { + ratio: "16:9", + duration: 4, + generateAudio: false, + }, + supportsAudio: true, + supportsSeed: false, + }), + "first-frame": buildModeDefinition({ + imageMin: 1, + imageMax: 1, + ratioMap: VEO_RATIOS, + durationOptions: DURATIONS_4_6_8, + defaults: { + ratio: "16:9", + duration: 4, + generateAudio: false, + }, + supportsAudio: true, + supportsSeed: false, + }), + "first-last-frame": buildModeDefinition({ + imageMin: 2, + imageMax: 2, + ratioMap: VEO_RATIOS, + durationOptions: DURATIONS_4_6_8, + defaults: { + ratio: "16:9", + duration: 4, + generateAudio: false, + }, + supportsAudio: true, + supportsSeed: false, + }), + }, + }, + ], + [ + "veo3.1_fast", + { + defaults: { + mode: "text-to-video", + ratio: "16:9", + duration: 4, + generateAudio: false, + fps: 24, + }, + ui: { + showResolution: false, + showFps: false, + showCameraMove: false, + showGenerateAudio: true, + showReturnLastFrame: false, + showSeed: false, + }, + modes: { + "text-to-video": buildModeDefinition({ + imageMin: 0, + imageMax: 0, + ratioMap: VEO_RATIOS, + durationOptions: DURATIONS_4_6_8, + defaults: { + ratio: "16:9", + duration: 4, + generateAudio: false, + }, + supportsAudio: true, + supportsSeed: false, + }), + "first-frame": buildModeDefinition({ + imageMin: 1, + imageMax: 1, + ratioMap: VEO_RATIOS, + durationOptions: DURATIONS_4_6_8, + defaults: { + ratio: "16:9", + duration: 4, + generateAudio: false, + }, + supportsAudio: true, + supportsSeed: false, + }), + "first-last-frame": buildModeDefinition({ + imageMin: 2, + imageMax: 2, + ratioMap: VEO_RATIOS, + durationOptions: DURATIONS_4_6_8, + defaults: { + ratio: "16:9", + duration: 4, + generateAudio: false, + }, + supportsAudio: true, + supportsSeed: false, + }), + }, + }, + ], + [ + "veo3", + { + defaults: { + mode: "text-to-video", + ratio: "16:9", + duration: 8, + generateAudio: false, + fps: 24, + }, + ui: { + showResolution: false, + showFps: false, + showCameraMove: false, + showGenerateAudio: false, + showReturnLastFrame: false, + showSeed: false, + }, + modes: { + "text-to-video": buildModeDefinition({ + imageMin: 0, + imageMax: 0, + ratioMap: VEO_RATIOS, + durationOptions: [8], + defaults: { + ratio: "16:9", + duration: 8, + }, + supportsSeed: false, + }), + "first-frame": buildModeDefinition({ + imageMin: 1, + imageMax: 1, + ratioMap: VEO_RATIOS, + durationOptions: [8], + defaults: { + ratio: "16:9", + duration: 8, + }, + supportsSeed: false, + }), + }, + }, + ], +]; + +function normalizeModelId(modelId?: string | null) { + return String(modelId || "").trim().toLowerCase(); +} + +export function getRunwayVideoModelDefinition(modelId?: string | null) { + const normalizedModelId = normalizeModelId(modelId); + if (!normalizedModelId) return null; + + for (const [key, definition] of RUNWAY_VIDEO_MODEL_DEFINITIONS) { + if ( + normalizedModelId === key || + normalizedModelId.startsWith(`${key}_`) || + normalizedModelId.startsWith(`${key}:`) || + normalizedModelId.startsWith(`${key}-`) + ) { + return definition; + } + } + + return null; +} + +export function getRunwayVideoModeDefinition( + modelId?: string | null, + mode?: VideoToolMode | null +) { + const definition = getRunwayVideoModelDefinition(modelId); + if (!definition) return null; + + const availableModes = Object.entries(definition.modes).filter( + (entry): entry is [VideoToolMode, RunwayModeDefinition] => Boolean(entry[1]) + ); + if (availableModes.length === 0) return null; + + if (mode && definition.modes[mode]) { + return { + mode, + definition: definition.modes[mode]!, + model: definition, + }; + } + + const fallbackMode = definition.defaults.mode; + if (definition.modes[fallbackMode]) { + return { + mode: fallbackMode, + definition: definition.modes[fallbackMode]!, + model: definition, + }; + } + + const [firstMode, firstDefinition] = availableModes[0]; + return { + mode: firstMode, + definition: firstDefinition, + model: definition, + }; +} + +export function resolveRunwayApiRatio(params: { + modelId?: string | null; + mode?: VideoToolMode | null; + ratio?: string | null; +}) { + const resolved = getRunwayVideoModeDefinition(params.modelId, params.mode); + if (!resolved) { + return params.ratio ? String(params.ratio).trim() : "1280:720"; + } + + const normalizedRatio = String(params.ratio || "").trim(); + if (normalizedRatio && resolved.definition.apiRatios[normalizedRatio]) { + return resolved.definition.apiRatios[normalizedRatio]; + } + + const fallbackRatio = + resolved.definition.defaults?.ratio || + resolved.model.defaults.ratio || + Object.keys(resolved.definition.apiRatios)[0] || + "16:9"; + + return resolved.definition.apiRatios[fallbackRatio] || "1280:720"; +} + +export function normalizeRunwayDuration(params: { + modelId?: string | null; + mode?: VideoToolMode | null; + duration?: number | null; +}) { + const resolved = getRunwayVideoModeDefinition(params.modelId, params.mode); + const allowedDurations = resolved?.definition.durationOptions || [5]; + const fallbackDuration = + Number(resolved?.definition.defaults?.duration) || + Number(resolved?.model.defaults.duration) || + allowedDurations[0] || + 5; + + if (!Number.isFinite(params.duration)) { + return fallbackDuration; + } + + const normalizedDuration = Math.round(Number(params.duration)); + if (allowedDurations.includes(normalizedDuration)) { + return normalizedDuration; + } + + return allowedDurations.reduce((closest, current) => { + return Math.abs(current - normalizedDuration) < Math.abs(closest - normalizedDuration) + ? current + : closest; + }, fallbackDuration); +} + +export function isRunwayAudioSupported(modelId?: string | null, mode?: VideoToolMode | null) { + const resolved = getRunwayVideoModeDefinition(modelId, mode); + return Boolean(resolved?.definition.supportsAudio); +} + +export function isRunwaySeedSupported(modelId?: string | null, mode?: VideoToolMode | null) { + const resolved = getRunwayVideoModeDefinition(modelId, mode); + return Boolean(resolved?.definition.supportsSeed); +} diff --git a/lib/ai/video-tool-capability.ts b/lib/ai/video-tool-capability.ts index e101aaf59..7e2c3e4f8 100644 --- a/lib/ai/video-tool-capability.ts +++ b/lib/ai/video-tool-capability.ts @@ -1,7 +1,10 @@ import type { VideoModelCapabilities } from "@/types/ai"; +import { getRunwayVideoModelDefinition } from "@/lib/ai/runway-video-models"; import type { VideoToolCapabilityContract, + VideoToolDefaults, VideoToolMode, + VideoToolModeConstraint, VideoToolModeOption, VideoToolResolution, } from "@/types/video-tool-capability"; @@ -25,6 +28,10 @@ const PORTRAIT_RATIOS = ["16:9", "9:16", "1:1"]; const STANDARD_RESOLUTIONS: VideoToolResolution[] = ["480p", "720p", "1080p"]; const STANDARD_DURATIONS = [5, 10]; +type ParsedVideoCapabilities = Partial & { + constraints?: Partial>; +}; + function asObject(value: unknown) { return value && typeof value === "object" ? (value as Record) : null; } @@ -65,7 +72,64 @@ function normalizeModeOptions( }); } -function normalizeVideoCapabilities(rawCapabilities: unknown): Partial { +function normalizeModeConstraints( + rawConstraints: unknown +): Partial> | undefined { + const raw = asObject(rawConstraints); + if (!raw) return undefined; + + const modeConstraints: Partial> = {}; + + for (const mode of MODE_ORDER) { + const entry = asObject(raw[mode]); + if (!entry) continue; + + const ratioOptions = Array.isArray(entry.ratioOptions) + ? entry.ratioOptions.map((item) => String(item || "").trim()).filter(Boolean) + : undefined; + const durationOptions = Array.isArray(entry.durationOptions) + ? entry.durationOptions + .map((item) => Number(item)) + .filter((item) => Number.isFinite(item)) + .map((item) => Math.round(item)) + : undefined; + const resolutionOptions = Array.isArray(entry.resolutionOptions) + ? entry.resolutionOptions.filter( + (item): item is VideoToolResolution => + item === "480p" || item === "720p" || item === "1080p" + ) + : undefined; + const defaults = asObject(entry.defaults); + + modeConstraints[mode] = { + ratioOptions: ratioOptions?.length ? Array.from(new Set(ratioOptions)) : undefined, + durationOptions: durationOptions?.length ? Array.from(new Set(durationOptions)) : undefined, + resolutionOptions: resolutionOptions?.length ? Array.from(new Set(resolutionOptions)) : undefined, + defaults: defaults + ? { + mode, + ratio: typeof defaults.ratio === "string" ? defaults.ratio : undefined, + duration: Number.isFinite(Number(defaults.duration)) + ? Math.round(Number(defaults.duration)) + : undefined, + resolution: + defaults.resolution === "480p" || + defaults.resolution === "720p" || + defaults.resolution === "1080p" + ? defaults.resolution + : undefined, + generateAudio: + typeof defaults.generateAudio === "boolean" ? defaults.generateAudio : undefined, + fps: Number.isFinite(Number(defaults.fps)) ? Math.round(Number(defaults.fps)) : undefined, + } + : undefined, + }; + } + + return Object.keys(modeConstraints).length > 0 ? modeConstraints : undefined; +} + +function normalizeVideoCapabilities(rawCapabilities: unknown): ParsedVideoCapabilities { const raw = asObject(rawCapabilities); if (!raw) return {}; @@ -108,10 +172,101 @@ function normalizeVideoCapabilities(rawCapabilities: unknown): Partial { + const modeDefinition = definition.modes[mode]; + if (!modeDefinition) return null; + return { + mode, + label: MODE_LABELS[mode], + imageMin: modeDefinition.imageMin, + imageMax: modeDefinition.imageMax, + } satisfies VideoToolModeOption; + }).filter((item): item is VideoToolModeOption => Boolean(item)); + + const modeConstraints = MODE_ORDER.reduce>>( + (acc, mode) => { + const modeDefinition = definition.modes[mode]; + if (!modeDefinition) return acc; + acc[mode] = { + ratioOptions: modeDefinition.ratioOptions, + durationOptions: modeDefinition.durationOptions, + resolutionOptions: modeDefinition.resolutionOptions || [], + defaults: modeDefinition.defaults, + }; + return acc; + }, + {} + ); + + const ratioOptions = Array.from( + new Set( + Object.values(definition.modes) + .flatMap((modeDefinition) => modeDefinition?.ratioOptions || []) + .filter(Boolean) + ) + ); + const durationOptions = Array.from( + new Set( + Object.values(definition.modes) + .flatMap((modeDefinition) => modeDefinition?.durationOptions || []) + .filter((item): item is number => Number.isFinite(item)) + ) + ).sort((a, b) => a - b); + + return { + schemaVersion: "v1", + source: "provider-default", + providerType: "runway", + modelId: modelId || undefined, + modeOptions: modeEntries, + ratioOptions, + durationOptions, + resolutionOptions: [], + modeConstraints, + ui: { ...definition.ui }, + defaults: { ...definition.defaults }, + }; +} + +function createProviderDefaultCapability( + providerType: string, + modelId?: string | null +): VideoToolCapabilityContract { if (providerType === "fal-video") { return { schemaVersion: "v1", @@ -194,30 +349,7 @@ function createProviderDefaultCapability(providerType: string): VideoToolCapabil } if (providerType === "runway") { - return { - schemaVersion: "v1", - source: "provider-default", - providerType, - modeOptions: normalizeModeOptions(["text-to-video", "first-frame"]), - ratioOptions: [...STANDARD_RATIOS], - durationOptions: [5, 10], - resolutionOptions: [], - ui: { - showResolution: false, - showFps: false, - showCameraMove: false, - showGenerateAudio: false, - showReturnLastFrame: false, - showSeed: true, - }, - defaults: { - mode: "text-to-video", - ratio: "16:9", - duration: 5, - generateAudio: false, - fps: 24, - }, - }; + return createRunwayDefaultCapability(modelId); } return { @@ -271,6 +403,65 @@ function pickPreferredMode(modeOptions: VideoToolModeOption[], fallback: VideoTo return modeOptions[0]?.mode || fallback; } +function pickPreferredResolution(resolutionOptions: VideoToolResolution[]) { + return resolutionOptions.includes("720p") ? "720p" : resolutionOptions[0] || undefined; +} + +function getModeConstraintOptions( + modeConstraints: Partial> | undefined, + mode: VideoToolMode | undefined, + defaults: VideoToolDefaults +) { + const constraint = mode ? modeConstraints?.[mode] : undefined; + const ratioOptions = constraint?.ratioOptions?.length ? constraint.ratioOptions : []; + const durationOptions = constraint?.durationOptions?.length ? constraint.durationOptions : []; + const resolutionOptions = constraint?.resolutionOptions?.length + ? constraint.resolutionOptions + : []; + + return { + ratioOptions, + durationOptions, + resolutionOptions, + defaults: { + ...defaults, + ...(constraint?.defaults || {}), + } as VideoToolDefaults, + }; +} + +export function getVideoToolOptionsForMode( + capability: VideoToolCapabilityContract | null | undefined, + mode?: VideoToolMode | null +) { + if (!capability) { + return { + ratioOptions: [] as string[], + durationOptions: [] as number[], + resolutionOptions: [] as VideoToolResolution[], + defaults: null as VideoToolDefaults | null, + }; + } + + const fallbackMode = + (mode && capability.modeOptions.some((item) => item.mode === mode) ? mode : undefined) || + capability.defaults.mode || + capability.modeOptions[0]?.mode; + + const scoped = getModeConstraintOptions(capability.modeConstraints, fallbackMode, capability.defaults); + + return { + ratioOptions: scoped.ratioOptions.length ? scoped.ratioOptions : capability.ratioOptions, + durationOptions: scoped.durationOptions.length + ? scoped.durationOptions + : capability.durationOptions, + resolutionOptions: scoped.resolutionOptions.length + ? scoped.resolutionOptions + : capability.resolutionOptions, + defaults: scoped.defaults, + }; +} + export function buildVideoToolCapability(params: { providerType?: string | null; modelId?: string | null; @@ -278,7 +469,7 @@ export function buildVideoToolCapability(params: { }): VideoToolCapabilityContract | null { const providerType = String(params.providerType || "").trim(); if (!providerType) return null; - const base = createProviderDefaultCapability(providerType); + const base = createProviderDefaultCapability(providerType, params.modelId); if (providerType === "fal-video" || providerType === "qianwen-video" || providerType === "ruxa-video") { return { @@ -292,7 +483,11 @@ export function buildVideoToolCapability(params: { Boolean(parsed.supportedModes?.length) || Boolean(parsed.supportedRatios?.length) || Boolean(parsed.supportedResolutions?.length) || - Boolean(parsed.durationRange); + Boolean(parsed.durationRange) || + Boolean( + parsed.constraints && + Object.values(parsed.constraints).some((constraint) => Boolean(constraint)) + ); if (!hasModelCapability) { return { @@ -304,6 +499,8 @@ export function buildVideoToolCapability(params: { const supportedModes = parsed.supportedModes && parsed.supportedModes.length > 0 ? parsed.supportedModes + : parsed.constraints && Object.keys(parsed.constraints).length > 0 + ? MODE_ORDER.filter((mode) => Boolean(parsed.constraints?.[mode])) : base.modeOptions.map((item) => item.mode); const modeOptions = normalizeModeOptions(supportedModes, parsed.maxReferenceImages || 4); @@ -318,6 +515,51 @@ export function buildVideoToolCapability(params: { const durationOptions = parsed.durationRange ? normalizeDurations(parsed.durationRange.min, parsed.durationRange.max) : base.durationOptions; + const baseModeConstraints = base.modeConstraints || {}; + const nextModeConstraints = + parsed.constraints && Object.keys(parsed.constraints).length > 0 + ? MODE_ORDER.reduce>>((acc, mode) => { + const baseConstraint = baseModeConstraints[mode]; + const parsedConstraint = parsed.constraints?.[mode]; + if (!baseConstraint && !parsedConstraint) return acc; + acc[mode] = { + ratioOptions: + parsedConstraint?.ratioOptions?.length + ? parsedConstraint.ratioOptions + : baseConstraint?.ratioOptions, + durationOptions: + parsedConstraint?.durationOptions?.length + ? parsedConstraint.durationOptions + : parsedConstraint + ? durationOptions + : baseConstraint?.durationOptions, + resolutionOptions: + parsedConstraint?.resolutionOptions?.length + ? parsedConstraint.resolutionOptions + : baseConstraint?.resolutionOptions, + defaults: { + ...(baseConstraint?.defaults || {}), + ...(parsedConstraint?.defaults || {}), + }, + }; + return acc; + }, {}) + : base.modeConstraints; + const defaultMode = pickPreferredMode(modeOptions, base.defaults.mode); + const modeScopedDefaults = getModeConstraintOptions( + nextModeConstraints, + defaultMode, + base.defaults + ); + const effectiveRatioOptions = modeScopedDefaults.ratioOptions.length + ? modeScopedDefaults.ratioOptions + : ratioOptions; + const effectiveDurationOptions = modeScopedDefaults.durationOptions.length + ? modeScopedDefaults.durationOptions + : durationOptions; + const effectiveResolutionOptions = modeScopedDefaults.resolutionOptions.length + ? modeScopedDefaults.resolutionOptions + : resolutionOptions; return { ...base, @@ -327,6 +569,7 @@ export function buildVideoToolCapability(params: { ratioOptions, resolutionOptions, durationOptions, + modeConstraints: nextModeConstraints, ui: { ...base.ui, showReturnLastFrame: @@ -338,12 +581,11 @@ export function buildVideoToolCapability(params: { }, defaults: { ...base.defaults, - mode: pickPreferredMode(modeOptions, base.defaults.mode), - ratio: pickPreferredRatio(ratioOptions), - duration: pickPreferredDuration(durationOptions), - resolution: resolutionOptions.includes("720p") - ? "720p" - : resolutionOptions[0] || undefined, + ...modeScopedDefaults.defaults, + mode: defaultMode, + ratio: pickPreferredRatio(effectiveRatioOptions), + duration: pickPreferredDuration(effectiveDurationOptions), + resolution: pickPreferredResolution(effectiveResolutionOptions), }, }; } diff --git a/types/video-tool-capability.ts b/types/video-tool-capability.ts index 61e33bc5b..56ad77253 100644 --- a/types/video-tool-capability.ts +++ b/types/video-tool-capability.ts @@ -31,6 +31,13 @@ export interface VideoToolDefaults { fps?: number; } +export interface VideoToolModeConstraint { + ratioOptions?: string[]; + durationOptions?: number[]; + resolutionOptions?: VideoToolResolution[]; + defaults?: Partial; +} + export interface VideoToolCapabilityContract { schemaVersion: "v1"; source: "provider-default" | "model-capabilities"; @@ -40,6 +47,7 @@ export interface VideoToolCapabilityContract { ratioOptions: string[]; durationOptions: number[]; resolutionOptions: VideoToolResolution[]; + modeConstraints?: Partial>; ui: VideoToolUiToggles; defaults: VideoToolDefaults; }