diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx
index ca122b776..b4a292286 100644
--- a/app/(admin)/settings/admin-settings-client.tsx
+++ b/app/(admin)/settings/admin-settings-client.tsx
@@ -1562,6 +1562,7 @@ export function AdminSettingsClient() {
"qianwen-video",
"fal-video",
"ruxa-video",
+ "runway",
].includes(selectedModelProviderType);
// 存储配置状态
@@ -9363,12 +9364,8 @@ export function AdminSettingsClient() {
不指定首选(完全随机轮换)
{(modelsData?.models || [])
.filter((m: AiModel) => {
- // 过滤出支持图片生成/编辑的模型
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
- if (!provider) return false;
- // 支持阿里生图、Gemini、既梦等图片模型
- const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "gemini-image", "gemini", "volcano-vision"];
- return imageProviderTypes.includes(provider.type);
+ return Boolean(provider && m.supportsImageGen);
})
.map((m: AiModel) => {
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
@@ -9382,8 +9379,7 @@ export function AdminSettingsClient() {
{/* 如果没有找到生图模型,显示提示 */}
{(modelsData?.models || []).filter((m: AiModel) => {
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
- const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "gemini-image", "gemini", "volcano-vision"];
- return provider && imageProviderTypes.includes(provider.type);
+ return Boolean(provider && m.supportsImageGen);
}).length === 0 && (
暂无可用的生图模型,请先配置生图供应商
diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts
index da9137f25..677611f59 100644
--- a/app/(chat)/api/chat/route.ts
+++ b/app/(chat)/api/chat/route.ts
@@ -98,6 +98,10 @@ import {
isGridConfirmationReply,
type PendingGridConfirmation,
} from "@/lib/ai/conversation-action-resolver";
+import {
+ isImageGenerationModelDescriptor,
+ isVideoModelDescriptor,
+} from "@/lib/ai/model-classification";
import {
bootstrapConversationStateProtocol,
composeConversationStateProtocol,
@@ -457,6 +461,7 @@ function normalizeImageLoadingProviderType(providerType?: string | null): string
normalized === "fal-image" ||
normalized === "openai-image" ||
normalized === "midjourney" ||
+ normalized === "runway" ||
normalized === "volcano-vision" ||
normalized === "gemini"
) {
@@ -583,21 +588,53 @@ function startStreamKeepalive(writer: StreamWriter, label: string, intervalMs =
type VideoModeType = "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
-const VIDEO_PROVIDER_TYPES = new Set([
- "volcano-video",
- "doubao-video",
- "qianwen-video",
- "fal-video",
- "ruxa-video",
-]);
+function isVideoProviderType(
+ providerType?: string | null,
+ model?: {
+ modelId?: string | null;
+ capabilities?: unknown;
+ supportsImageGen?: boolean | null;
+ }
+): boolean {
+ return isVideoModelDescriptor({
+ providerType,
+ modelId: model?.modelId,
+ capabilities: model?.capabilities,
+ supportsImageGen: model?.supportsImageGen,
+ });
+}
-function isVideoProviderType(providerType?: string | null): boolean {
- return Boolean(providerType && VIDEO_PROVIDER_TYPES.has(providerType));
+function isImageGenerationModelType(
+ providerType?: string | null,
+ model?: {
+ modelId?: string | null;
+ capabilities?: unknown;
+ supportsImageGen?: boolean | null;
+ }
+): boolean {
+ return isImageGenerationModelDescriptor({
+ providerType,
+ modelId: model?.modelId,
+ capabilities: model?.capabilities,
+ supportsImageGen: model?.supportsImageGen,
+ });
}
function inferSupportedVideoModesByModelId(modelId: string): VideoModeType[] {
const normalized = modelId.toLowerCase();
+ if (
+ normalized === "gen4.5" ||
+ normalized === "gen4_turbo" ||
+ normalized === "gen4_aleph" ||
+ normalized === "gen3a_turbo" ||
+ normalized === "veo3" ||
+ normalized === "veo3.1" ||
+ normalized === "veo3.1_fast"
+ ) {
+ return ["text-to-video", "first-frame"];
+ }
+
if (normalized.includes("lite-t2v")) {
return ["text-to-video"];
}
@@ -1341,6 +1378,7 @@ const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([
"ruxa-image",
"fal-image",
"openai-image",
+ "runway",
"volcano-vision",
"gemini",
"huanke-gemini",
@@ -1980,7 +2018,8 @@ async function convertMessagesWithImages(messages: ChatMessage[], providerType:
providerType === "ruxa-image" ||
providerType === "fal-image" ||
providerType === "openai-image" ||
- providerType === "midjourney"
+ providerType === "midjourney" ||
+ providerType === "runway"
) {
const convertedMessages = [];
@@ -3156,6 +3195,7 @@ export async function POST(request: Request) {
let requestedModelProviderType: string | undefined;
let requestedModelIsFixedVisual = false;
let requestedModelIsConversationMultimodal = false;
+ let requestedModelConfig: Awaited> | null = null;
if (!shouldRunAgentConversation && requestedModelId !== "auto") {
const selectedModelIsUUID =
@@ -3183,33 +3223,21 @@ export async function POST(request: Request) {
selectedModelProviderType = selectedProvider?.type as string | undefined;
}
+ requestedModelConfig = selectedModelConfig;
requestedModelProviderType = selectedModelProviderType;
if (selectedModelConfig) {
- const fixedImageProviderTypes = new Set([
- "gemini",
- "huanke-gemini",
- "alibaba-image",
- "ruxa-image",
- "fal-image",
- "openai-image",
- "midjourney",
- "volcano-vision",
- ]);
const isFixedVisualModel =
!selectedModelConfig.isSubModel &&
- (selectedModelConfig.supportsImageGen ||
- (selectedModelProviderType
- ? isVideoProviderType(selectedModelProviderType)
- : false) ||
- (selectedModelProviderType
- ? fixedImageProviderTypes.has(selectedModelProviderType)
- : false));
+ (
+ isImageGenerationModelType(selectedModelProviderType, selectedModelConfig) ||
+ isVideoProviderType(selectedModelProviderType, selectedModelConfig)
+ );
const isConversationMultimodalModel =
!selectedModelConfig.isSubModel &&
isAiModelMultimodal(selectedModelConfig) &&
- !selectedModelConfig.supportsImageGen &&
- !isVideoProviderType(selectedModelProviderType);
+ !isImageGenerationModelType(selectedModelProviderType, selectedModelConfig) &&
+ !isVideoProviderType(selectedModelProviderType, selectedModelConfig);
requestedModelIsFixedVisual = isFixedVisualModel;
requestedModelIsConversationMultimodal = isConversationMultimodalModel;
@@ -3331,7 +3359,8 @@ export async function POST(request: Request) {
}
const allowAgentVideoGeneration =
- Boolean(requestBody.videoParams) || isVideoProviderType(requestedModelProviderType);
+ Boolean(requestBody.videoParams) ||
+ isVideoProviderType(requestedModelProviderType, requestedModelConfig || undefined);
let smartAutoWorkflowDecision:
| {
action?: string;
@@ -3985,7 +4014,9 @@ export async function POST(request: Request) {
Boolean(smartAutoExecution?.shouldStayConversational);
if (autoGraphResult.route && (!shouldKeepSelectedModel || shouldForceConversationalRouteForVisualModel)) {
- const routeIsVideo = isVideoProviderType(autoGraphResult.route.providerType);
+ const routeIsVideo = isVideoProviderType(autoGraphResult.route.providerType, {
+ modelId: autoGraphResult.route.modelId,
+ });
if (routeIsVideo && !allowAgentVideoGeneration) {
const fallbackIntent: "image_edit" | "image_gen" = hasImage ? "image_edit" : "image_gen";
smartAutoDetectedIntent = fallbackIntent;
@@ -4021,7 +4052,7 @@ export async function POST(request: Request) {
subIntent: autoGraphResult.analysis?.subIntent,
hasImage,
});
- if (!isVideoProviderType(fallbackRoute.providerType)) {
+ if (!isVideoProviderType(fallbackRoute.providerType, { modelId: fallbackRoute.modelId })) {
finalModelId = fallbackRoute.id;
console.log(
`[Smart Auto Mode] ✅ Fallback routed to: ${fallbackRoute.displayName} (${fallbackRoute.reason})`,
@@ -4281,7 +4312,7 @@ export async function POST(request: Request) {
(m) =>
m.model.supportsImageGen &&
!m.model.isSubModel &&
- !isVideoProviderType(m.provider?.type),
+ !isVideoProviderType(m.provider?.type, m.model),
);
if (isAutoTimeout) {
smartAutoFriendlyMessage = buildAutoModeFallbackMessage({
@@ -4309,12 +4340,15 @@ export async function POST(request: Request) {
}
if (!bestModel) {
bestModel = allModels.find(
- (m) => m.model.isDefault && !m.model.isSubModel && !isVideoProviderType(m.provider?.type),
+ (m) =>
+ m.model.isDefault &&
+ !m.model.isSubModel &&
+ !isVideoProviderType(m.provider?.type, m.model),
);
}
if (!bestModel && allModels.length > 0) {
bestModel = allModels.find(
- (m) => !m.model.isSubModel && !isVideoProviderType(m.provider?.type),
+ (m) => !m.model.isSubModel && !isVideoProviderType(m.provider?.type, m.model),
) || allModels.find((m) => !m.model.isSubModel) || allModels[0];
}
@@ -4335,9 +4369,10 @@ export async function POST(request: Request) {
const finalModelIsUuid =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(finalModelId);
let finalProviderType: string | undefined;
+ let finalModelConfig: Awaited> | null = null;
if (finalModelIsUuid) {
- const finalModelConfig = await getAiModelById({ id: finalModelId }).catch(() => null);
+ finalModelConfig = await getAiModelById({ id: finalModelId }).catch(() => null);
if (finalModelConfig) {
finalProviderType = (
await getAiProviderById({ id: finalModelConfig.providerId }).catch(() => null)
@@ -4345,19 +4380,19 @@ export async function POST(request: Request) {
}
}
- if (isVideoProviderType(finalProviderType)) {
+ if (isVideoProviderType(finalProviderType, finalModelConfig || undefined)) {
const allModels = await getEnabledAiModelsWithProvider(false);
const fallbackModel =
allModels.find(
(m) =>
!m.model.isSubModel &&
m.model.supportsImageGen &&
- !isVideoProviderType(m.provider?.type),
+ !isVideoProviderType(m.provider?.type, m.model),
) ||
allModels.find(
(m) =>
!m.model.isSubModel &&
- !isVideoProviderType(m.provider?.type),
+ !isVideoProviderType(m.provider?.type, m.model),
);
if (fallbackModel) {
@@ -7364,22 +7399,31 @@ export async function POST(request: Request) {
}
// ========== 图片生成配额检查 ==========
- const isImageGenerationModel =
- (languageModel as any).provider === "gemini-image" ||
- (languageModel as any).provider === "volcano-vision" ||
- (languageModel as any).provider === "alibaba-image" ||
- (languageModel as any).provider === "ruxa-image" ||
- (languageModel as any).provider === "fal-image" ||
- (languageModel as any).provider === "openai-image" ||
- (languageModel as any).provider === "midjourney" ||
- (languageModel as any).provider === "huanke-gemini";
-
const isUuidModelId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
finalModelId
);
const currentModelConfig = isUuidModelId
? await getAiModelById({ id: finalModelId }).catch(() => undefined)
: undefined;
+ const runtimeProviderType = String(
+ resolvedProviderType || (languageModel as any).provider || ""
+ );
+ const normalizedRuntimeProviderType =
+ runtimeProviderType === "gemini-image"
+ ? "gemini"
+ : runtimeProviderType === "huanke-gemini"
+ ? "huanke-gemini"
+ : runtimeProviderType;
+ const isImageGenerationModel = isImageGenerationModelType(
+ normalizedRuntimeProviderType,
+ currentModelConfig
+ ? {
+ modelId: currentModelConfig.modelId,
+ capabilities: currentModelConfig.capabilities,
+ supportsImageGen: currentModelConfig.supportsImageGen,
+ }
+ : undefined
+ );
const modelCreditCostPerImage = Number(
(currentModelConfig as any)?.imageGenConfig?.creditCostPerImage ?? 0
);
@@ -7438,13 +7482,16 @@ export async function POST(request: Request) {
}
}
- const isVideoGenerationModel = [
- "qianwen-video",
- "fal-video",
- "ruxa-video",
- "volcano-video",
- "doubao-video",
- ].includes(String(resolvedProviderType || (languageModel as any).provider || ""));
+ const isVideoGenerationModel = isVideoProviderType(
+ normalizedRuntimeProviderType,
+ currentModelConfig
+ ? {
+ modelId: currentModelConfig.modelId,
+ capabilities: currentModelConfig.capabilities,
+ supportsImageGen: currentModelConfig.supportsImageGen,
+ }
+ : undefined
+ );
if (isVideoGenerationModel) {
const pointsConfig = await getPointsSystemConfig();
@@ -7925,7 +7972,14 @@ export async function POST(request: Request) {
// ========== Gemini 处理结束 ==========
// ========== 阿里云千问图片生成独立处理 ==========
- const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney"].includes((languageModel as any).provider);
+ const isAlibabaLikeImageProvider = [
+ "alibaba-image",
+ "ruxa-image",
+ "fal-image",
+ "openai-image",
+ "midjourney",
+ "runway",
+ ].includes((languageModel as any).provider);
if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") {
console.log("[Chat API] Using image generation provider:", (languageModel as any).provider);
console.log("[Chat API] Image params:", effectiveImageParams);
@@ -8013,6 +8067,8 @@ export async function POST(request: Request) {
? "openai-image"
: (languageModel as any).provider === "midjourney"
? "midjourney"
+ : (languageModel as any).provider === "runway"
+ ? "runway"
: "alibaba-image";
const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert);
@@ -8085,8 +8141,9 @@ export async function POST(request: Request) {
"ruxa-image": 2,
"fal-image": 3,
"openai-image": 4,
- gemini: 5,
- "huanke-gemini": 6,
+ runway: 5,
+ gemini: 6,
+ "huanke-gemini": 7,
};
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
currentModelId: finalModelId,
@@ -8115,6 +8172,8 @@ export async function POST(request: Request) {
? "openai-image"
: candidate.provider.type === "midjourney"
? "midjourney"
+ : candidate.provider.type === "runway"
+ ? "runway"
: candidate.provider.type === "gemini"
? "gemini"
: candidate.provider.type === "huanke-gemini"
@@ -8473,9 +8532,10 @@ export async function POST(request: Request) {
"ruxa-image": 1,
"fal-image": 2,
"openai-image": 3,
- gemini: 4,
- "huanke-gemini": 5,
- "volcano-vision": 6,
+ runway: 4,
+ gemini: 5,
+ "huanke-gemini": 6,
+ "volcano-vision": 7,
};
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
currentModelId: finalModelId,
@@ -8504,6 +8564,8 @@ export async function POST(request: Request) {
? "openai-image"
: candidate.provider.type === "midjourney"
? "midjourney"
+ : candidate.provider.type === "runway"
+ ? "runway"
: candidate.provider.type === "gemini"
? "gemini"
: candidate.provider.type === "huanke-gemini"
@@ -8710,13 +8772,14 @@ export async function POST(request: Request) {
// ========== 千问/FAL/RUXA 视频生成独立处理 ==========
if (
- ["qianwen-video", "fal-video", "ruxa-video"].includes((languageModel as any).provider) &&
+ ["qianwen-video", "fal-video", "ruxa-video", "runway"].includes((languageModel as any).provider) &&
typeof (languageModel as any).doGenerate === "function"
) {
const currentVideoProvider = (languageModel as any).provider as
| "qianwen-video"
| "fal-video"
- | "ruxa-video";
+ | "ruxa-video"
+ | "runway";
console.log(`[Chat API] Using ${currentVideoProvider} video generation`);
const videoGenerationStream = createUIMessageStream({
@@ -9286,8 +9349,9 @@ export async function POST(request: Request) {
"ruxa-image": 2,
"fal-image": 3,
"openai-image": 4,
- gemini: 5,
- "huanke-gemini": 6,
+ runway: 5,
+ gemini: 6,
+ "huanke-gemini": 7,
};
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
currentModelId: finalModelId,
@@ -9321,6 +9385,8 @@ export async function POST(request: Request) {
? "openai-image"
: candidate.provider.type === "midjourney"
? "midjourney"
+ : candidate.provider.type === "runway"
+ ? "runway"
: candidate.provider.type === "gemini"
? "gemini"
: candidate.provider.type === "huanke-gemini"
diff --git a/app/(chat)/page.tsx b/app/(chat)/page.tsx
index dc324994c..c8df373d6 100644
--- a/app/(chat)/page.tsx
+++ b/app/(chat)/page.tsx
@@ -14,14 +14,7 @@ import {
getSystemSetting,
} from "@/lib/db/queries";
import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags";
-
-const VIDEO_PROVIDER_TYPES = new Set([
- "volcano-video",
- "doubao-video",
- "qianwen-video",
- "fal-video",
- "ruxa-video",
-]);
+import { isVideoModelDescriptor } from "@/lib/ai/model-classification";
function isNanoBananaProModel(model: { displayName?: string | null; modelId?: string | null }) {
const displayName = String(model.displayName || "").toLowerCase();
@@ -109,7 +102,12 @@ async function NewChatPage() {
const preferredImageModels = visibleModels.filter(
({ model, provider }) =>
model.supportsImageGen &&
- !VIDEO_PROVIDER_TYPES.has(provider.type)
+ !isVideoModelDescriptor({
+ providerType: provider.type,
+ modelId: model.modelId,
+ capabilities: model.capabilities,
+ supportsImageGen: model.supportsImageGen,
+ })
);
const nanoBananaModel = preferredImageModels.find(({ model }) =>
isNanoBananaProModel(model)
diff --git a/lib/ai/dynamic-provider.ts b/lib/ai/dynamic-provider.ts
index 947687631..016cdda72 100644
--- a/lib/ai/dynamic-provider.ts
+++ b/lib/ai/dynamic-provider.ts
@@ -24,6 +24,7 @@ import { createOpenAIImage } from "./providers/openai-image";
import { createMidjourney } from "./providers/midjourney";
import { createFalVideo } from "./providers/fal-video";
import { createRuxaVideo } from "./providers/ruxa-video";
+import { createRunway } from "./providers/runway";
import { createVolcanoMultimodal } from "./providers/volcano-multimodal";
import { getProviderDefaultBaseUrl } from "./provider-defaults";
@@ -227,6 +228,15 @@ function getOrCreateProvider(provider: Pick VIDEO_MODE_KEYS.has(String(mode || "").trim()));
+}
+
+export function inferRunwayModelKind(modelId?: string | null): "image" | "video" | null {
+ const normalized = normalizeText(modelId);
+ if (!normalized) return null;
+ if (RUNWAY_IMAGE_MODEL_IDS.has(normalized)) return "image";
+ if (RUNWAY_VIDEO_MODEL_PREFIXES.some((prefix) => normalized === prefix || normalized.startsWith(`${prefix}_`))) {
+ return "video";
+ }
+ return null;
+}
+
+export function isStaticVideoProviderType(providerType?: string | null): boolean {
+ return STATIC_VIDEO_PROVIDER_TYPES.has(normalizeText(providerType));
+}
+
+export function isVideoModelDescriptor(params: {
+ providerType?: string | null;
+ modelId?: string | null;
+ capabilities?: unknown;
+ supportsImageGen?: boolean | null;
+}): boolean {
+ const providerType = normalizeText(params.providerType);
+
+ if (isStaticVideoProviderType(providerType)) {
+ return true;
+ }
+
+ if (hasVideoModes(params.capabilities)) {
+ return true;
+ }
+
+ if (providerType === "runway") {
+ return inferRunwayModelKind(params.modelId) === "video";
+ }
+
+ return false;
+}
+
+export function isImageGenerationModelDescriptor(params: {
+ providerType?: string | null;
+ modelId?: string | null;
+ capabilities?: unknown;
+ supportsImageGen?: boolean | null;
+}): boolean {
+ const providerType = normalizeText(params.providerType);
+
+ if (params.supportsImageGen === true) {
+ return true;
+ }
+
+ if (providerType === "runway") {
+ return inferRunwayModelKind(params.modelId) === "image";
+ }
+
+ if (isVideoModelDescriptor(params)) {
+ return false;
+ }
+
+ return STATIC_IMAGE_PROVIDER_TYPES.has(providerType);
+}
diff --git a/lib/ai/model-router.ts b/lib/ai/model-router.ts
index 09e0896e5..f8eaed78e 100644
--- a/lib/ai/model-router.ts
+++ b/lib/ai/model-router.ts
@@ -6,6 +6,10 @@
*/
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
+import {
+ isImageGenerationModelDescriptor,
+ isVideoModelDescriptor,
+} from "@/lib/ai/model-classification";
type EnabledModelWithProvider = Awaited<
ReturnType
@@ -42,14 +46,6 @@ const CHAT_PROVIDER_TYPES = [
"volcano-multimodal",
] as const;
-const VIDEO_PROVIDER_TYPES = new Set([
- "doubao-video",
- "volcano-video",
- "qianwen-video",
- "fal-video",
- "ruxa-video",
-]);
-
const IMAGE_ONLY_PROVIDER_TYPES = new Set([
"alibaba-image",
"ruxa-image",
@@ -59,8 +55,20 @@ const IMAGE_ONLY_PROVIDER_TYPES = new Set([
"gemini-image",
]);
-function isVideoProviderType(providerType: string) {
- return VIDEO_PROVIDER_TYPES.has(providerType);
+function isVideoProviderType(
+ providerType: string,
+ model?: {
+ modelId?: string | null;
+ capabilities?: unknown;
+ supportsImageGen?: boolean | null;
+ }
+) {
+ return isVideoModelDescriptor({
+ providerType,
+ modelId: model?.modelId,
+ capabilities: model?.capabilities,
+ supportsImageGen: model?.supportsImageGen,
+ });
}
function isConversationalCandidate(
@@ -69,7 +77,7 @@ function isConversationalCandidate(
) {
const providerType = String(item.provider.type || "");
if (!item.model.isEnabled || item.model.isSubModel) return false;
- if (isVideoProviderType(providerType)) return false;
+ if (isVideoProviderType(providerType, item.model)) return false;
const requireVision = Boolean(options?.requireVision);
const matchesPreferredProvider = CHAT_PROVIDER_TYPES.includes(
@@ -108,13 +116,27 @@ function matchesIntentRequirements(
if (params.intent === "vision") {
return (
- !isVideoProviderType(String(item.provider.type || "")) &&
+ !isVideoProviderType(String(item.provider.type || ""), item.model) &&
hasVisionCapability(item.model)
);
}
+ if (params.intent === "video_gen") {
+ return isVideoProviderType(String(item.provider.type || ""), item.model);
+ }
+
if (params.requireVision && !hasVisionCapability(item.model)) return false;
- if (params.requireImageGen && !item.model.supportsImageGen) return false;
+ if (
+ params.requireImageGen &&
+ !isImageGenerationModelDescriptor({
+ providerType: item.provider.type,
+ modelId: item.model.modelId,
+ capabilities: item.model.capabilities,
+ supportsImageGen: item.model.supportsImageGen,
+ })
+ ) {
+ return false;
+ }
return true;
}
@@ -132,7 +154,7 @@ const INTENT_MODEL_MAP: Record<
> = {
// 图片编辑:优先 alibaba-image(灵魂4.5,支持提示词扩展)
image_edit: {
- providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini", "huanke-gemini"],
+ providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini", "huanke-gemini"],
requireVision: true,
requireImageGen: true,
description: "图片编辑",
@@ -147,14 +169,14 @@ const INTENT_MODEL_MAP: Record<
// 图片生成:优先灵魂4.5
image_gen: {
- providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini", "huanke-gemini", "volcano-vision"],
+ providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini", "huanke-gemini", "volcano-vision"],
requireImageGen: true,
description: "图片生成",
},
// 视频生成
video_gen: {
- providerTypes: ["doubao-video", "volcano-video", "qianwen-video", "fal-video", "ruxa-video"],
+ providerTypes: ["doubao-video", "volcano-video", "qianwen-video", "fal-video", "ruxa-video", "runway"],
description: "视频生成",
},
@@ -175,25 +197,25 @@ const SUB_INTENT_OVERRIDES: Record<
> = {
// 抠图:alibaba-image 效果最好
remove_background: {
- providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini"],
+ providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini"],
reason: "抠图首选灵魂4.5",
},
// 换背景:alibaba-image 支持场景生成
replace_background: {
- providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini"],
+ providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini"],
reason: "换背景首选灵魂4.5",
},
// 图片增强
enhance: {
- providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini", "volcano-vision"],
+ providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini", "volcano-vision"],
reason: "增强首选灵魂4.5",
},
// 风格迁移:gemini 可能更有创意
style_transfer: {
- providerTypes: ["gemini", "alibaba-image", "ruxa-image", "fal-image", "openai-image"],
+ providerTypes: ["gemini", "alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway"],
reason: "风格迁移优选Gemini",
},
};
@@ -321,7 +343,7 @@ export async function routeToModel(params: {
return (
!item.model.isSubModel &&
item.model.isEnabled &&
- !isVideoProviderType(String(item.provider.type || "")) &&
+ !isVideoProviderType(String(item.provider.type || ""), item.model) &&
(item.model.supportsImageGen || hasVisionCapability(item.model))
);
});
@@ -345,7 +367,7 @@ export async function routeToModel(params: {
(m) =>
!m.model.isSubModel &&
m.model.isEnabled &&
- !isVideoProviderType(String(m.provider.type || "")) &&
+ !isVideoProviderType(String(m.provider.type || ""), m.model) &&
!m.model.supportsImageGen,
)
: undefined) ||
diff --git a/lib/ai/providers/runway.ts b/lib/ai/providers/runway.ts
new file mode 100644
index 000000000..96fd84e8c
--- /dev/null
+++ b/lib/ai/providers/runway.ts
@@ -0,0 +1,581 @@
+import "server-only";
+
+import sharp from "sharp";
+import { uploadFile } from "@/lib/storage/service";
+import { inferRunwayModelKind } from "@/lib/ai/model-classification";
+
+interface RunwayConfig {
+ apiKey: string;
+ baseURL: string;
+}
+
+interface RunwayImageParams {
+ aspectRatio?: string;
+ imageSize?: string;
+ imageCount?: number;
+ seed?: number;
+}
+
+interface RunwayVideoParams {
+ mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
+ ratio?: string;
+ duration?: number;
+ seed?: number;
+}
+
+interface UploadedImage {
+ thumbnail: string;
+ original: string;
+}
+
+const RUNWAY_VERSION = "2024-11-06";
+const DEFAULT_BASE_URL = "https://api.dev.runwayml.com";
+const TASK_POLL_INTERVAL_MS = 5000;
+const TASK_TIMEOUT_MS = 15 * 60 * 1000;
+
+const IMAGE_RESOLUTION_MAP: Record> = {
+ "1K": {
+ "21:9": "1584:672",
+ "16:9": "1360:768",
+ "9:16": "768:1360",
+ "4:3": "1104:832",
+ "3:4": "832:1104",
+ "1:1": "960:960",
+ },
+ "2K": {
+ "21:9": "1584:672",
+ "16:9": "1360:768",
+ "9:16": "768:1360",
+ "4:3": "1104:832",
+ "3:4": "832:1104",
+ "1:1": "960:960",
+ },
+ "4K": {
+ "21:9": "1584:672",
+ "16:9": "1360:768",
+ "9:16": "768:1360",
+ "4:3": "1104:832",
+ "3:4": "832:1104",
+ "1:1": "960:960",
+ },
+};
+
+function normalizeBaseURL(baseURL: string) {
+ const normalized = String(baseURL || "").trim().replace(/\/+$/, "");
+ return normalized || DEFAULT_BASE_URL;
+}
+
+function joinEndpoint(baseURL: string, path: string) {
+ const cleanPath = path.startsWith("/") ? path : `/${path}`;
+ return `${normalizeBaseURL(baseURL)}${cleanPath}`;
+}
+
+function buildHeaders(apiKey: string) {
+ return {
+ Authorization: `Bearer ${String(apiKey || "").trim()}`,
+ "Content-Type": "application/json",
+ "X-Runway-Version": RUNWAY_VERSION,
+ };
+}
+
+function normalizeText(value: string) {
+ return String(value || "").replace(/\u00a0/g, " ").trim().normalize("NFKC");
+}
+
+function normalizeImageSizeLabel(imageSize?: string): "1K" | "2K" | "4K" {
+ const normalized = String(imageSize || "").trim().toUpperCase();
+ if (normalized === "1K" || normalized === "2K" || normalized === "4K") {
+ return normalized;
+ }
+ return "2K";
+}
+
+function normalizeImageResolution(aspectRatio?: string, imageSize?: string) {
+ const normalizedRatio = String(aspectRatio || "").trim() || "16:9";
+ const normalizedSize = normalizeImageSizeLabel(imageSize);
+ return (
+ IMAGE_RESOLUTION_MAP[normalizedSize]?.[normalizedRatio] ||
+ IMAGE_RESOLUTION_MAP[normalizedSize]?.["16:9"] ||
+ "1360:768"
+ );
+}
+
+function normalizeVideoRatio(ratio?: string) {
+ const normalized = String(ratio || "").trim();
+ if (["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"].includes(normalized)) {
+ return normalized;
+ }
+ return "16:9";
+}
+
+function normalizeVideoDuration(duration?: number) {
+ if (!Number.isFinite(duration)) return 5;
+ const normalized = Math.round(Number(duration));
+ return normalized >= 8 ? 10 : 5;
+}
+
+function normalizeSeed(seed?: number) {
+ if (!Number.isFinite(seed)) return undefined;
+ return Math.max(0, Math.min(4294967295, Math.round(Number(seed))));
+}
+
+function detectExtensionFromContentType(contentType?: string | null) {
+ const normalized = String(contentType || "").toLowerCase();
+ if (normalized.includes("jpeg") || normalized.includes("jpg")) return "jpg";
+ if (normalized.includes("webp")) return "webp";
+ if (normalized.includes("gif")) return "gif";
+ if (normalized.includes("png")) return "png";
+ if (normalized.includes("mp4")) return "mp4";
+ if (normalized.includes("quicktime")) return "mov";
+ if (normalized.includes("webm")) return "webm";
+ return "bin";
+}
+
+function extractDataUriParts(dataUri: string) {
+ const match = dataUri.match(/^data:([^;]+);base64,(.+)$/);
+ if (!match) {
+ throw new Error("不支持的 Data URI 格式");
+ }
+ return {
+ mimeType: match[1],
+ buffer: Buffer.from(match[2], "base64"),
+ };
+}
+
+function generateFilename(prefix: string, extension: string) {
+ const timestamp = Date.now();
+ const random = Math.random().toString(36).slice(2, 8);
+ return `${prefix}-${timestamp}-${random}.${extension}`;
+}
+
+async function uploadGeneratedBuffer(
+ buffer: Buffer,
+ extension: string,
+ mimeType: string
+): Promise {
+ const thumbnailBuffer = await sharp(buffer)
+ .resize(400, 400, { fit: "inside", withoutEnlargement: true })
+ .jpeg({ quality: 80 })
+ .toBuffer();
+
+ const originalFilename = generateFilename("runway-image", extension);
+ const originalArrayBuffer = buffer.buffer.slice(
+ buffer.byteOffset,
+ buffer.byteOffset + buffer.byteLength
+ ) as ArrayBuffer;
+ const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer);
+
+ const thumbnailFilename = generateFilename("runway-thumb", "jpg");
+ const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
+ thumbnailBuffer.byteOffset,
+ thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
+ ) as ArrayBuffer;
+ const thumbnailUploaded = await uploadFile(thumbnailFilename, thumbnailArrayBuffer);
+
+ console.log("[Runway] Uploaded image to storage:", {
+ original: originalUploaded.url,
+ thumbnail: thumbnailUploaded.url,
+ mimeType,
+ });
+
+ return {
+ thumbnail: thumbnailUploaded.url,
+ original: originalUploaded.url,
+ };
+}
+
+async function uploadDataUriAsset(dataUri: string, prefix: "runway-input" | "runway-video") {
+ const { mimeType, buffer } = extractDataUriParts(dataUri);
+ const extension = detectExtensionFromContentType(mimeType);
+ const filename = generateFilename(prefix, extension);
+ const arrayBuffer = buffer.buffer.slice(
+ buffer.byteOffset,
+ buffer.byteOffset + buffer.byteLength
+ ) as ArrayBuffer;
+ const uploaded = await uploadFile(filename, arrayBuffer);
+ return uploaded.url;
+}
+
+async function normalizeAssetUrl(url?: string | null) {
+ const normalized = String(url || "").trim();
+ if (!normalized) return "";
+ if (/^https:\/\//i.test(normalized) || /^runway:\/\//i.test(normalized)) {
+ return normalized;
+ }
+ if (/^data:(image|video)\//i.test(normalized)) {
+ return uploadDataUriAsset(normalized, /^data:video\//i.test(normalized) ? "runway-video" : "runway-input");
+ }
+ return normalized;
+}
+
+function collectHttpUrls(payload: unknown): string[] {
+ const urls = new Set();
+
+ const visit = (value: unknown) => {
+ if (typeof value === "string") {
+ const normalized = value.trim();
+ if (/^https?:\/\//i.test(normalized)) {
+ urls.add(normalized);
+ }
+ return;
+ }
+
+ if (Array.isArray(value)) {
+ value.forEach(visit);
+ return;
+ }
+
+ if (value && typeof value === "object") {
+ Object.values(value as Record).forEach(visit);
+ }
+ };
+
+ visit(payload);
+ return Array.from(urls);
+}
+
+async function downloadRemoteAsset(url: string) {
+ const controller = new AbortController();
+ const timeoutId = setTimeout(() => controller.abort(), 120000);
+
+ try {
+ const response = await fetch(url, {
+ cache: "no-store",
+ signal: controller.signal,
+ });
+ if (!response.ok) {
+ throw new Error(`下载 Runway 输出失败: ${response.status}`);
+ }
+ const buffer = Buffer.from(await response.arrayBuffer());
+ return {
+ buffer,
+ contentType: response.headers.get("content-type"),
+ };
+ } finally {
+ clearTimeout(timeoutId);
+ }
+}
+
+async function uploadVideoToStorage(remoteVideoUrl: string) {
+ try {
+ const { buffer, contentType } = await downloadRemoteAsset(remoteVideoUrl);
+ const extension = detectExtensionFromContentType(contentType || "video/mp4");
+ const filename = generateFilename("runway-video", extension === "bin" ? "mp4" : extension);
+ const arrayBuffer = buffer.buffer.slice(
+ buffer.byteOffset,
+ buffer.byteOffset + buffer.byteLength
+ ) as ArrayBuffer;
+ const uploaded = await uploadFile(filename, arrayBuffer);
+ return uploaded.url;
+ } catch (error) {
+ console.warn("[Runway] Failed to archive video output, using temporary URL:", error);
+ return remoteVideoUrl;
+ }
+}
+
+function formatImageMarkdown(images: UploadedImage[], imageSize: string) {
+ return images
+ .map((image) => ``)
+ .join("\n\n");
+}
+
+function formatVideoMarkdown(videoUrl: string) {
+ return `\n\n[下载视频](${videoUrl})`;
+}
+
+function extractPromptAndAssets(messages: Array<{ role: string; content: any }>) {
+ const lastUserMessage = messages.filter((message) => message.role === "user").pop();
+ if (!lastUserMessage) {
+ throw new Error("未找到用户输入内容");
+ }
+
+ let prompt = "";
+ const imageUrls: string[] = [];
+
+ const pushImageUrl = (value?: string) => {
+ if (!value) return;
+ const normalized = String(value || "").trim();
+ if (!normalized) return;
+ if (!imageUrls.includes(normalized)) {
+ imageUrls.push(normalized);
+ }
+ };
+
+ if (typeof lastUserMessage.content === "string") {
+ prompt = normalizeText(lastUserMessage.content);
+ } else if (Array.isArray(lastUserMessage.content)) {
+ for (const part of lastUserMessage.content) {
+ if (part?.type === "text" && typeof part.text === "string") {
+ prompt = `${prompt} ${part.text}`.trim();
+ } else if (part?.type === "image_url" && part.image_url?.url) {
+ pushImageUrl(part.image_url.url);
+ } else if (part?.type === "file") {
+ pushImageUrl(part.url || part.data);
+ } else if (part?.type === "image") {
+ pushImageUrl(part.image || part.url);
+ }
+ }
+ }
+
+ return {
+ prompt: normalizeText(prompt) || "生成内容",
+ imageUrls,
+ };
+}
+
+async function createTask(baseURL: string, apiKey: string, path: string, body: Record) {
+ const response = await fetch(joinEndpoint(baseURL, path), {
+ method: "POST",
+ headers: buildHeaders(apiKey),
+ body: JSON.stringify(body),
+ });
+
+ const rawText = await response.text();
+ let payload: any = null;
+ try {
+ payload = rawText ? JSON.parse(rawText) : null;
+ } catch {
+ payload = rawText;
+ }
+
+ if (!response.ok) {
+ const message =
+ typeof payload?.error === "string"
+ ? payload.error
+ : typeof payload?.message === "string"
+ ? payload.message
+ : rawText || "Runway 请求失败";
+ throw new Error(`${message} (HTTP ${response.status})`);
+ }
+
+ const taskId = String(payload?.id || "").trim();
+ if (!taskId) {
+ throw new Error("Runway 未返回任务 ID");
+ }
+
+ return taskId;
+}
+
+async function waitForTaskOutput(baseURL: string, apiKey: string, taskId: string) {
+ const deadline = Date.now() + TASK_TIMEOUT_MS;
+
+ while (Date.now() < deadline) {
+ const response = await fetch(joinEndpoint(baseURL, `/v1/tasks/${taskId}`), {
+ method: "GET",
+ headers: buildHeaders(apiKey),
+ cache: "no-store",
+ });
+
+ const rawText = await response.text();
+ let payload: any = null;
+ try {
+ payload = rawText ? JSON.parse(rawText) : null;
+ } catch {
+ payload = rawText;
+ }
+
+ if (!response.ok) {
+ const message =
+ typeof payload?.error === "string"
+ ? payload.error
+ : typeof payload?.message === "string"
+ ? payload.message
+ : rawText || "Runway 轮询失败";
+ throw new Error(`${message} (HTTP ${response.status})`);
+ }
+
+ const status = String(payload?.status || "").trim().toUpperCase();
+ if (status === "SUCCEEDED") {
+ return payload;
+ }
+ if (status === "FAILED" || status === "CANCELLED") {
+ throw new Error(
+ String(payload?.failure || payload?.error || payload?.message || "Runway 任务失败")
+ );
+ }
+
+ await new Promise((resolve) => setTimeout(resolve, TASK_POLL_INTERVAL_MS));
+ }
+
+ throw new Error("Runway 任务超时,请稍后重试");
+}
+
+async function uploadRunwayImages(taskOutput: unknown) {
+ const outputUrls = collectHttpUrls(taskOutput);
+ const uploadedImages: UploadedImage[] = [];
+
+ for (const url of outputUrls) {
+ try {
+ const { buffer, contentType } = await downloadRemoteAsset(url);
+ if (!String(contentType || "").startsWith("image/")) {
+ continue;
+ }
+ const extension = detectExtensionFromContentType(contentType);
+ uploadedImages.push(
+ await uploadGeneratedBuffer(
+ buffer,
+ extension === "bin" ? "png" : extension,
+ contentType || "image/png"
+ )
+ );
+ } catch (error) {
+ console.warn("[Runway] Failed to archive image output:", error);
+ }
+ }
+
+ if (uploadedImages.length === 0) {
+ throw new Error("Runway 未返回可用图片");
+ }
+
+ return uploadedImages;
+}
+
+async function resolveRunwayVideoUrl(taskOutput: unknown) {
+ const outputUrls = collectHttpUrls(taskOutput);
+ if (outputUrls.length === 0) {
+ throw new Error("Runway 未返回可用视频地址");
+ }
+
+ for (const url of outputUrls) {
+ if (/(\.mp4|\.mov|\.webm)(\?|$)/i.test(url) || /video/i.test(url)) {
+ return uploadVideoToStorage(url);
+ }
+ }
+
+ return uploadVideoToStorage(outputUrls[0]!);
+}
+
+function getRunwayVideoEndpoint(modelId: string, hasPromptImage: boolean) {
+ const normalized = normalizeText(modelId);
+ if (hasPromptImage) return "/v1/image_to_video";
+ if (normalized === "gen4.5") return "/v1/image_to_video";
+ return "/v1/text_to_video";
+}
+
+function buildReferenceImages(imageUrls: string[]) {
+ return imageUrls.slice(0, 3).map((uri, index) => ({
+ uri,
+ tag: `ref_${index + 1}`.replace(/[^a-z0-9_]/g, "_"),
+ }));
+}
+
+export function createRunway({ apiKey, baseURL }: RunwayConfig) {
+ return {
+ provider: "runway",
+ chat: (modelId: string) => ({
+ provider: "runway",
+ doGenerate: async (options: {
+ messages: Array<{ role: string; content: any }>;
+ imageParams?: RunwayImageParams;
+ videoParams?: RunwayVideoParams;
+ overrideModelId?: string;
+ }) => {
+ try {
+ const activeModelId = normalizeText(options.overrideModelId || modelId);
+ const { prompt, imageUrls } = extractPromptAndAssets(options.messages);
+ const normalizedImageUrls = (
+ await Promise.all(imageUrls.map((url) => normalizeAssetUrl(url)))
+ ).filter(Boolean);
+ const modelKind =
+ inferRunwayModelKind(activeModelId) ||
+ (options.videoParams ? "video" : options.imageParams ? "image" : null) ||
+ "video";
+
+ if (modelKind === "image") {
+ const imageSize = normalizeImageSizeLabel(options.imageParams?.imageSize);
+ const taskId = await createTask(baseURL, apiKey, "/v1/text_to_image", {
+ model: activeModelId,
+ promptText: prompt,
+ ratio: normalizeImageResolution(options.imageParams?.aspectRatio, imageSize),
+ ...(normalizedImageUrls.length > 0
+ ? { referenceImages: buildReferenceImages(normalizedImageUrls) }
+ : {}),
+ ...(normalizeSeed(options.imageParams?.seed) !== undefined
+ ? { seed: normalizeSeed(options.imageParams?.seed) }
+ : {}),
+ });
+
+ const task = await waitForTaskOutput(baseURL, apiKey, taskId);
+ const uploadedImages = await uploadRunwayImages(task?.output ?? task);
+ const text = formatImageMarkdown(uploadedImages, imageSize);
+
+ return {
+ text,
+ finishReason: "stop" as const,
+ usage: {
+ promptTokens: 0,
+ completionTokens: text.length,
+ },
+ };
+ }
+
+ const videoMode = options.videoParams?.mode || (normalizedImageUrls[0] ? "first-frame" : "text-to-video");
+ if (
+ (videoMode === "first-frame" || videoMode === "first-last-frame" || videoMode === "reference-images") &&
+ !normalizedImageUrls[0]
+ ) {
+ throw new Error("Runway 图生视频至少需要 1 张参考图");
+ }
+
+ const endpoint = getRunwayVideoEndpoint(activeModelId, Boolean(normalizedImageUrls[0]));
+ const taskId = await createTask(baseURL, apiKey, endpoint, {
+ model: activeModelId,
+ promptText: prompt,
+ ratio: normalizeVideoRatio(options.videoParams?.ratio),
+ duration: normalizeVideoDuration(options.videoParams?.duration),
+ ...(normalizedImageUrls[0] ? { promptImage: normalizedImageUrls[0] } : {}),
+ ...(normalizeSeed(options.videoParams?.seed) !== undefined
+ ? { seed: normalizeSeed(options.videoParams?.seed) }
+ : {}),
+ });
+
+ const task = await waitForTaskOutput(baseURL, apiKey, taskId);
+ const videoUrl = await resolveRunwayVideoUrl(task?.output ?? task);
+ const text = formatVideoMarkdown(videoUrl);
+
+ return {
+ text,
+ finishReason: "stop" as const,
+ usage: {
+ promptTokens: 0,
+ completionTokens: text.length,
+ },
+ };
+ } catch (error) {
+ console.error("[Runway] Generation failed:", error);
+ return {
+ text:
+ error instanceof Error
+ ? `${
+ (inferRunwayModelKind(options.overrideModelId || modelId) ||
+ (options.videoParams ? "video" : options.imageParams ? "image" : "video")) === "image"
+ ? "图片"
+ : "视频"
+ }生成失败: ${error.message}`
+ : "Runway 生成失败,请稍后重试",
+ finishReason: "error" as const,
+ usage: {
+ promptTokens: 0,
+ completionTokens: 0,
+ },
+ };
+ }
+ },
+ doStream: async function* (options: {
+ messages: Array<{ role: string; content: any }>;
+ imageParams?: RunwayImageParams;
+ videoParams?: RunwayVideoParams;
+ overrideModelId?: string;
+ }) {
+ const result = await this.doGenerate(options);
+ yield {
+ type: "text-delta",
+ textDelta: result.text,
+ };
+ },
+ modelId,
+ specificationVersion: "v1" as const,
+ defaultObjectGenerationMode: "json" as const,
+ }),
+ };
+}
diff --git a/lib/ai/video-tool-capability.ts b/lib/ai/video-tool-capability.ts
index a6686893a..e101aaf59 100644
--- a/lib/ai/video-tool-capability.ts
+++ b/lib/ai/video-tool-capability.ts
@@ -193,6 +193,33 @@ function createProviderDefaultCapability(providerType: string): VideoToolCapabil
};
}
+ if (providerType === "runway") {
+ return {
+ schemaVersion: "v1",
+ source: "provider-default",
+ providerType,
+ modeOptions: normalizeModeOptions(["text-to-video", "first-frame"]),
+ ratioOptions: [...STANDARD_RATIOS],
+ durationOptions: [5, 10],
+ resolutionOptions: [],
+ ui: {
+ showResolution: false,
+ showFps: false,
+ showCameraMove: false,
+ showGenerateAudio: false,
+ showReturnLastFrame: false,
+ showSeed: true,
+ },
+ defaults: {
+ mode: "text-to-video",
+ ratio: "16:9",
+ duration: 5,
+ generateAudio: false,
+ fps: 24,
+ },
+ };
+ }
+
return {
schemaVersion: "v1",
source: "provider-default",
diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts
index 871d37124..87cfddd22 100644
--- a/lib/canvas/generate-image.ts
+++ b/lib/canvas/generate-image.ts
@@ -7,6 +7,7 @@ import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
import { createFalImage } from "@/lib/ai/providers/fal-image";
import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
import { createMidjourney } from "@/lib/ai/providers/midjourney";
+import { createRunway } from "@/lib/ai/providers/runway";
import { filterReachableRemoteUrls } from "@/lib/net/url-access";
export interface CanvasGenerateImageRequest {
@@ -39,6 +40,7 @@ const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([
"fal-image",
"openai-image",
"midjourney",
+ "runway",
]);
function parseImagesFromMarkdown(markdown: string): ParsedImage[] {
@@ -76,6 +78,7 @@ function getProviderBatchLimit(providerType: string) {
case "ruxa-image":
case "openai-image":
case "midjourney":
+ case "runway":
return 1;
case "gemini":
return 2;
@@ -151,7 +154,7 @@ export async function generateCanvasImage(
}
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(fallbackProvider.type)) {
throw new Error(
- "当前图片模型类型暂不支持画布生成,请在后台选择 Gemini/阿里/RUXA/FAL/OpenAI 图片模型。"
+ "当前图片模型类型暂不支持画布生成,请在后台选择 Gemini/阿里/RUXA/FAL/OpenAI/Runway 图片模型。"
);
}
if (fallbackModel.model.id !== selectedModel.model.id) {
@@ -311,9 +314,22 @@ export async function generateCanvasImage(
});
return result.text || "";
};
+ } else if (provider.type === "runway") {
+ const runwayProvider = createRunway({
+ apiKey: provider.apiKey,
+ baseURL: provider.baseUrl,
+ });
+ const modelInstance = runwayProvider.chat(selectedModel.model.modelId);
+ executeGenerateBatch = async (_batchImageCount) => {
+ const result = await modelInstance.doGenerate({
+ messages,
+ imageParams,
+ });
+ return result.text || "";
+ };
} else {
throw new Error(
- `不支持的图片生成模型类型: ${provider.type || "unknown"},请改用 Gemini/阿里/RUXA/FAL/OpenAI/Midjourney 图片模型`
+ `不支持的图片生成模型类型: ${provider.type || "unknown"},请改用 Gemini/阿里/RUXA/FAL/OpenAI/Midjourney/Runway 图片模型`
);
}
diff --git a/lib/db/queries.ts b/lib/db/queries.ts
index d0f1620af..59c47b553 100644
--- a/lib/db/queries.ts
+++ b/lib/db/queries.ts
@@ -91,6 +91,7 @@ import {
} from "../points/config";
import { inferOriginalImageUrl } from "../image-url";
import { getAgentRequiredUploadCount } from "../agents/preset-mode-config";
+import { isVideoModelDescriptor } from "../ai/model-classification";
// Optionally, if not using email/pass login, you can
// use the Drizzle adapter for Auth.js / NextAuth
@@ -1755,14 +1756,6 @@ type AgentPresetKind = "image" | "video" | "text";
type AgentPresetScope = "all" | AgentPresetVisibility;
type AgentCategoryScope = "all" | "public" | "private" | "global";
type AgentPresetSort = "latest" | "hottest" | "most_used";
-const AGENT_VIDEO_PROVIDER_TYPES = new Set([
- "volcano-video",
- "doubao-video",
- "qianwen-video",
- "fal-video",
- "ruxa-video",
-]);
-
function normalizeAgentCategoryName(value: string) {
return String(value || "").trim().replace(/\s+/g, " ");
}
@@ -1804,7 +1797,12 @@ async function validateAgentDefaultModel(params: {
throw new ChatSDKError("bad_request:database", "Invalid default model");
}
- const isVideoModel = AGENT_VIDEO_PROVIDER_TYPES.has(String(model.providerType || ""));
+ const isVideoModel = isVideoModelDescriptor({
+ providerType: model.providerType,
+ modelId: model.modelId,
+ capabilities: model.capabilities,
+ supportsImageGen: model.supportsImageGen,
+ });
if (params.presetType === "video") {
if (!isVideoModel) {