feat: add runway image and video runtime support
This commit is contained in:
Your Name
parent
f09d8f1d25
commit
947af6d26f
@ -1562,6 +1562,7 @@ export function AdminSettingsClient() {
|
||||
"qianwen-video",
|
||||
"fal-video",
|
||||
"ruxa-video",
|
||||
"runway",
|
||||
].includes(selectedModelProviderType);
|
||||
|
||||
// 存储配置状态
|
||||
@ -9363,12 +9364,8 @@ export function AdminSettingsClient() {
|
||||
<SelectItem value="auto">不指定首选（完全随机轮换）</SelectItem>
|
||||
{(modelsData?.models || [])
|
||||
.filter((m: AiModel) => {
|
||||
// 过滤出支持图片生成/编辑的模型
|
||||
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
|
||||
if (!provider) return false;
|
||||
// 支持阿里生图、Gemini、既梦等图片模型
|
||||
const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "gemini-image", "gemini", "volcano-vision"];
|
||||
return imageProviderTypes.includes(provider.type);
|
||||
return Boolean(provider && m.supportsImageGen);
|
||||
})
|
||||
.map((m: AiModel) => {
|
||||
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
|
||||
@ -9382,8 +9379,7 @@ export function AdminSettingsClient() {
|
||||
{/* 如果没有找到生图模型，显示提示 */}
|
||||
{(modelsData?.models || []).filter((m: AiModel) => {
|
||||
const provider = (providersData?.providers || []).find((p: AiProvider) => p.id === m.providerId);
|
||||
const imageProviderTypes = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "gemini-image", "gemini", "volcano-vision"];
|
||||
return provider && imageProviderTypes.includes(provider.type);
|
||||
return Boolean(provider && m.supportsImageGen);
|
||||
}).length === 0 && (
|
||||
<SelectItem value="no-model" disabled>
|
||||
暂无可用的生图模型，请先配置生图供应商
|
||||
|
||||
@ -98,6 +98,10 @@ import {
|
||||
isGridConfirmationReply,
|
||||
type PendingGridConfirmation,
|
||||
} from "@/lib/ai/conversation-action-resolver";
|
||||
import {
|
||||
isImageGenerationModelDescriptor,
|
||||
isVideoModelDescriptor,
|
||||
} from "@/lib/ai/model-classification";
|
||||
import {
|
||||
bootstrapConversationStateProtocol,
|
||||
composeConversationStateProtocol,
|
||||
@ -457,6 +461,7 @@ function normalizeImageLoadingProviderType(providerType?: string | null): string
|
||||
normalized === "fal-image" ||
|
||||
normalized === "openai-image" ||
|
||||
normalized === "midjourney" ||
|
||||
normalized === "runway" ||
|
||||
normalized === "volcano-vision" ||
|
||||
normalized === "gemini"
|
||||
) {
|
||||
@ -583,21 +588,53 @@ function startStreamKeepalive(writer: StreamWriter, label: string, intervalMs =
|
||||
|
||||
type VideoModeType = "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
|
||||
|
||||
const VIDEO_PROVIDER_TYPES = new Set([
|
||||
"volcano-video",
|
||||
"doubao-video",
|
||||
"qianwen-video",
|
||||
"fal-video",
|
||||
"ruxa-video",
|
||||
]);
|
||||
function isVideoProviderType(
|
||||
providerType?: string | null,
|
||||
model?: {
|
||||
modelId?: string | null;
|
||||
capabilities?: unknown;
|
||||
supportsImageGen?: boolean | null;
|
||||
}
|
||||
): boolean {
|
||||
return isVideoModelDescriptor({
|
||||
providerType,
|
||||
modelId: model?.modelId,
|
||||
capabilities: model?.capabilities,
|
||||
supportsImageGen: model?.supportsImageGen,
|
||||
});
|
||||
}
|
||||
|
||||
function isVideoProviderType(providerType?: string | null): boolean {
|
||||
return Boolean(providerType && VIDEO_PROVIDER_TYPES.has(providerType));
|
||||
function isImageGenerationModelType(
|
||||
providerType?: string | null,
|
||||
model?: {
|
||||
modelId?: string | null;
|
||||
capabilities?: unknown;
|
||||
supportsImageGen?: boolean | null;
|
||||
}
|
||||
): boolean {
|
||||
return isImageGenerationModelDescriptor({
|
||||
providerType,
|
||||
modelId: model?.modelId,
|
||||
capabilities: model?.capabilities,
|
||||
supportsImageGen: model?.supportsImageGen,
|
||||
});
|
||||
}
|
||||
|
||||
function inferSupportedVideoModesByModelId(modelId: string): VideoModeType[] {
|
||||
const normalized = modelId.toLowerCase();
|
||||
|
||||
if (
|
||||
normalized === "gen4.5" ||
|
||||
normalized === "gen4_turbo" ||
|
||||
normalized === "gen4_aleph" ||
|
||||
normalized === "gen3a_turbo" ||
|
||||
normalized === "veo3" ||
|
||||
normalized === "veo3.1" ||
|
||||
normalized === "veo3.1_fast"
|
||||
) {
|
||||
return ["text-to-video", "first-frame"];
|
||||
}
|
||||
|
||||
if (normalized.includes("lite-t2v")) {
|
||||
return ["text-to-video"];
|
||||
}
|
||||
@ -1341,6 +1378,7 @@ const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([
|
||||
"ruxa-image",
|
||||
"fal-image",
|
||||
"openai-image",
|
||||
"runway",
|
||||
"volcano-vision",
|
||||
"gemini",
|
||||
"huanke-gemini",
|
||||
@ -1980,7 +2018,8 @@ async function convertMessagesWithImages(messages: ChatMessage[], providerType:
|
||||
providerType === "ruxa-image" ||
|
||||
providerType === "fal-image" ||
|
||||
providerType === "openai-image" ||
|
||||
providerType === "midjourney"
|
||||
providerType === "midjourney" ||
|
||||
providerType === "runway"
|
||||
) {
|
||||
const convertedMessages = [];
|
||||
|
||||
@ -3156,6 +3195,7 @@ export async function POST(request: Request) {
|
||||
let requestedModelProviderType: string | undefined;
|
||||
let requestedModelIsFixedVisual = false;
|
||||
let requestedModelIsConversationMultimodal = false;
|
||||
let requestedModelConfig: Awaited<ReturnType<typeof getAiModelById>> | null = null;
|
||||
|
||||
if (!shouldRunAgentConversation && requestedModelId !== "auto") {
|
||||
const selectedModelIsUUID =
|
||||
@ -3183,33 +3223,21 @@ export async function POST(request: Request) {
|
||||
selectedModelProviderType = selectedProvider?.type as string | undefined;
|
||||
}
|
||||
|
||||
requestedModelConfig = selectedModelConfig;
|
||||
requestedModelProviderType = selectedModelProviderType;
|
||||
|
||||
if (selectedModelConfig) {
|
||||
const fixedImageProviderTypes = new Set([
|
||||
"gemini",
|
||||
"huanke-gemini",
|
||||
"alibaba-image",
|
||||
"ruxa-image",
|
||||
"fal-image",
|
||||
"openai-image",
|
||||
"midjourney",
|
||||
"volcano-vision",
|
||||
]);
|
||||
const isFixedVisualModel =
|
||||
!selectedModelConfig.isSubModel &&
|
||||
(selectedModelConfig.supportsImageGen ||
|
||||
(selectedModelProviderType
|
||||
? isVideoProviderType(selectedModelProviderType)
|
||||
: false) ||
|
||||
(selectedModelProviderType
|
||||
? fixedImageProviderTypes.has(selectedModelProviderType)
|
||||
: false));
|
||||
(
|
||||
isImageGenerationModelType(selectedModelProviderType, selectedModelConfig) ||
|
||||
isVideoProviderType(selectedModelProviderType, selectedModelConfig)
|
||||
);
|
||||
const isConversationMultimodalModel =
|
||||
!selectedModelConfig.isSubModel &&
|
||||
isAiModelMultimodal(selectedModelConfig) &&
|
||||
!selectedModelConfig.supportsImageGen &&
|
||||
!isVideoProviderType(selectedModelProviderType);
|
||||
!isImageGenerationModelType(selectedModelProviderType, selectedModelConfig) &&
|
||||
!isVideoProviderType(selectedModelProviderType, selectedModelConfig);
|
||||
requestedModelIsFixedVisual = isFixedVisualModel;
|
||||
requestedModelIsConversationMultimodal = isConversationMultimodalModel;
|
||||
|
||||
@ -3331,7 +3359,8 @@ export async function POST(request: Request) {
|
||||
}
|
||||
|
||||
const allowAgentVideoGeneration =
|
||||
Boolean(requestBody.videoParams) || isVideoProviderType(requestedModelProviderType);
|
||||
Boolean(requestBody.videoParams) ||
|
||||
isVideoProviderType(requestedModelProviderType, requestedModelConfig || undefined);
|
||||
let smartAutoWorkflowDecision:
|
||||
| {
|
||||
action?: string;
|
||||
@ -3985,7 +4014,9 @@ export async function POST(request: Request) {
|
||||
Boolean(smartAutoExecution?.shouldStayConversational);
|
||||
|
||||
if (autoGraphResult.route && (!shouldKeepSelectedModel || shouldForceConversationalRouteForVisualModel)) {
|
||||
const routeIsVideo = isVideoProviderType(autoGraphResult.route.providerType);
|
||||
const routeIsVideo = isVideoProviderType(autoGraphResult.route.providerType, {
|
||||
modelId: autoGraphResult.route.modelId,
|
||||
});
|
||||
if (routeIsVideo && !allowAgentVideoGeneration) {
|
||||
const fallbackIntent: "image_edit" | "image_gen" = hasImage ? "image_edit" : "image_gen";
|
||||
smartAutoDetectedIntent = fallbackIntent;
|
||||
@ -4021,7 +4052,7 @@ export async function POST(request: Request) {
|
||||
subIntent: autoGraphResult.analysis?.subIntent,
|
||||
hasImage,
|
||||
});
|
||||
if (!isVideoProviderType(fallbackRoute.providerType)) {
|
||||
if (!isVideoProviderType(fallbackRoute.providerType, { modelId: fallbackRoute.modelId })) {
|
||||
finalModelId = fallbackRoute.id;
|
||||
console.log(
|
||||
`[Smart Auto Mode] ✅ Fallback routed to: ${fallbackRoute.displayName} (${fallbackRoute.reason})`,
|
||||
@ -4281,7 +4312,7 @@ export async function POST(request: Request) {
|
||||
(m) =>
|
||||
m.model.supportsImageGen &&
|
||||
!m.model.isSubModel &&
|
||||
!isVideoProviderType(m.provider?.type),
|
||||
!isVideoProviderType(m.provider?.type, m.model),
|
||||
);
|
||||
if (isAutoTimeout) {
|
||||
smartAutoFriendlyMessage = buildAutoModeFallbackMessage({
|
||||
@ -4309,12 +4340,15 @@ export async function POST(request: Request) {
|
||||
}
|
||||
if (!bestModel) {
|
||||
bestModel = allModels.find(
|
||||
(m) => m.model.isDefault && !m.model.isSubModel && !isVideoProviderType(m.provider?.type),
|
||||
(m) =>
|
||||
m.model.isDefault &&
|
||||
!m.model.isSubModel &&
|
||||
!isVideoProviderType(m.provider?.type, m.model),
|
||||
);
|
||||
}
|
||||
if (!bestModel && allModels.length > 0) {
|
||||
bestModel = allModels.find(
|
||||
(m) => !m.model.isSubModel && !isVideoProviderType(m.provider?.type),
|
||||
(m) => !m.model.isSubModel && !isVideoProviderType(m.provider?.type, m.model),
|
||||
) || allModels.find((m) => !m.model.isSubModel) || allModels[0];
|
||||
}
|
||||
|
||||
@ -4335,9 +4369,10 @@ export async function POST(request: Request) {
|
||||
const finalModelIsUuid =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(finalModelId);
|
||||
let finalProviderType: string | undefined;
|
||||
let finalModelConfig: Awaited<ReturnType<typeof getAiModelById>> | null = null;
|
||||
|
||||
if (finalModelIsUuid) {
|
||||
const finalModelConfig = await getAiModelById({ id: finalModelId }).catch(() => null);
|
||||
finalModelConfig = await getAiModelById({ id: finalModelId }).catch(() => null);
|
||||
if (finalModelConfig) {
|
||||
finalProviderType = (
|
||||
await getAiProviderById({ id: finalModelConfig.providerId }).catch(() => null)
|
||||
@ -4345,19 +4380,19 @@ export async function POST(request: Request) {
|
||||
}
|
||||
}
|
||||
|
||||
if (isVideoProviderType(finalProviderType)) {
|
||||
if (isVideoProviderType(finalProviderType, finalModelConfig || undefined)) {
|
||||
const allModels = await getEnabledAiModelsWithProvider(false);
|
||||
const fallbackModel =
|
||||
allModels.find(
|
||||
(m) =>
|
||||
!m.model.isSubModel &&
|
||||
m.model.supportsImageGen &&
|
||||
!isVideoProviderType(m.provider?.type),
|
||||
!isVideoProviderType(m.provider?.type, m.model),
|
||||
) ||
|
||||
allModels.find(
|
||||
(m) =>
|
||||
!m.model.isSubModel &&
|
||||
!isVideoProviderType(m.provider?.type),
|
||||
!isVideoProviderType(m.provider?.type, m.model),
|
||||
);
|
||||
|
||||
if (fallbackModel) {
|
||||
@ -7364,22 +7399,31 @@ export async function POST(request: Request) {
|
||||
}
|
||||
|
||||
// ========== 图片生成配额检查 ==========
|
||||
const isImageGenerationModel =
|
||||
(languageModel as any).provider === "gemini-image" ||
|
||||
(languageModel as any).provider === "volcano-vision" ||
|
||||
(languageModel as any).provider === "alibaba-image" ||
|
||||
(languageModel as any).provider === "ruxa-image" ||
|
||||
(languageModel as any).provider === "fal-image" ||
|
||||
(languageModel as any).provider === "openai-image" ||
|
||||
(languageModel as any).provider === "midjourney" ||
|
||||
(languageModel as any).provider === "huanke-gemini";
|
||||
|
||||
const isUuidModelId = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
|
||||
finalModelId
|
||||
);
|
||||
const currentModelConfig = isUuidModelId
|
||||
? await getAiModelById({ id: finalModelId }).catch(() => undefined)
|
||||
: undefined;
|
||||
const runtimeProviderType = String(
|
||||
resolvedProviderType || (languageModel as any).provider || ""
|
||||
);
|
||||
const normalizedRuntimeProviderType =
|
||||
runtimeProviderType === "gemini-image"
|
||||
? "gemini"
|
||||
: runtimeProviderType === "huanke-gemini"
|
||||
? "huanke-gemini"
|
||||
: runtimeProviderType;
|
||||
const isImageGenerationModel = isImageGenerationModelType(
|
||||
normalizedRuntimeProviderType,
|
||||
currentModelConfig
|
||||
? {
|
||||
modelId: currentModelConfig.modelId,
|
||||
capabilities: currentModelConfig.capabilities,
|
||||
supportsImageGen: currentModelConfig.supportsImageGen,
|
||||
}
|
||||
: undefined
|
||||
);
|
||||
const modelCreditCostPerImage = Number(
|
||||
(currentModelConfig as any)?.imageGenConfig?.creditCostPerImage ?? 0
|
||||
);
|
||||
@ -7438,13 +7482,16 @@ export async function POST(request: Request) {
|
||||
}
|
||||
}
|
||||
|
||||
const isVideoGenerationModel = [
|
||||
"qianwen-video",
|
||||
"fal-video",
|
||||
"ruxa-video",
|
||||
"volcano-video",
|
||||
"doubao-video",
|
||||
].includes(String(resolvedProviderType || (languageModel as any).provider || ""));
|
||||
const isVideoGenerationModel = isVideoProviderType(
|
||||
normalizedRuntimeProviderType,
|
||||
currentModelConfig
|
||||
? {
|
||||
modelId: currentModelConfig.modelId,
|
||||
capabilities: currentModelConfig.capabilities,
|
||||
supportsImageGen: currentModelConfig.supportsImageGen,
|
||||
}
|
||||
: undefined
|
||||
);
|
||||
|
||||
if (isVideoGenerationModel) {
|
||||
const pointsConfig = await getPointsSystemConfig();
|
||||
@ -7925,7 +7972,14 @@ export async function POST(request: Request) {
|
||||
// ========== Gemini 处理结束 ==========
|
||||
|
||||
// ========== 阿里云千问图片生成独立处理 ==========
|
||||
const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney"].includes((languageModel as any).provider);
|
||||
const isAlibabaLikeImageProvider = [
|
||||
"alibaba-image",
|
||||
"ruxa-image",
|
||||
"fal-image",
|
||||
"openai-image",
|
||||
"midjourney",
|
||||
"runway",
|
||||
].includes((languageModel as any).provider);
|
||||
if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") {
|
||||
console.log("[Chat API] Using image generation provider:", (languageModel as any).provider);
|
||||
console.log("[Chat API] Image params:", effectiveImageParams);
|
||||
@ -8013,6 +8067,8 @@ export async function POST(request: Request) {
|
||||
? "openai-image"
|
||||
: (languageModel as any).provider === "midjourney"
|
||||
? "midjourney"
|
||||
: (languageModel as any).provider === "runway"
|
||||
? "runway"
|
||||
: "alibaba-image";
|
||||
const modelMessages = await convertMessagesWithImages(messagesForAlibaba, providerForConvert);
|
||||
|
||||
@ -8085,8 +8141,9 @@ export async function POST(request: Request) {
|
||||
"ruxa-image": 2,
|
||||
"fal-image": 3,
|
||||
"openai-image": 4,
|
||||
gemini: 5,
|
||||
"huanke-gemini": 6,
|
||||
runway: 5,
|
||||
gemini: 6,
|
||||
"huanke-gemini": 7,
|
||||
};
|
||||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||||
currentModelId: finalModelId,
|
||||
@ -8115,6 +8172,8 @@ export async function POST(request: Request) {
|
||||
? "openai-image"
|
||||
: candidate.provider.type === "midjourney"
|
||||
? "midjourney"
|
||||
: candidate.provider.type === "runway"
|
||||
? "runway"
|
||||
: candidate.provider.type === "gemini"
|
||||
? "gemini"
|
||||
: candidate.provider.type === "huanke-gemini"
|
||||
@ -8473,9 +8532,10 @@ export async function POST(request: Request) {
|
||||
"ruxa-image": 1,
|
||||
"fal-image": 2,
|
||||
"openai-image": 3,
|
||||
gemini: 4,
|
||||
"huanke-gemini": 5,
|
||||
"volcano-vision": 6,
|
||||
runway: 4,
|
||||
gemini: 5,
|
||||
"huanke-gemini": 6,
|
||||
"volcano-vision": 7,
|
||||
};
|
||||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||||
currentModelId: finalModelId,
|
||||
@ -8504,6 +8564,8 @@ export async function POST(request: Request) {
|
||||
? "openai-image"
|
||||
: candidate.provider.type === "midjourney"
|
||||
? "midjourney"
|
||||
: candidate.provider.type === "runway"
|
||||
? "runway"
|
||||
: candidate.provider.type === "gemini"
|
||||
? "gemini"
|
||||
: candidate.provider.type === "huanke-gemini"
|
||||
@ -8710,13 +8772,14 @@ export async function POST(request: Request) {
|
||||
|
||||
// ========== 千问/FAL/RUXA 视频生成独立处理 ==========
|
||||
if (
|
||||
["qianwen-video", "fal-video", "ruxa-video"].includes((languageModel as any).provider) &&
|
||||
["qianwen-video", "fal-video", "ruxa-video", "runway"].includes((languageModel as any).provider) &&
|
||||
typeof (languageModel as any).doGenerate === "function"
|
||||
) {
|
||||
const currentVideoProvider = (languageModel as any).provider as
|
||||
| "qianwen-video"
|
||||
| "fal-video"
|
||||
| "ruxa-video";
|
||||
| "ruxa-video"
|
||||
| "runway";
|
||||
console.log(`[Chat API] Using ${currentVideoProvider} video generation`);
|
||||
|
||||
const videoGenerationStream = createUIMessageStream({
|
||||
@ -9286,8 +9349,9 @@ export async function POST(request: Request) {
|
||||
"ruxa-image": 2,
|
||||
"fal-image": 3,
|
||||
"openai-image": 4,
|
||||
gemini: 5,
|
||||
"huanke-gemini": 6,
|
||||
runway: 5,
|
||||
gemini: 6,
|
||||
"huanke-gemini": 7,
|
||||
};
|
||||
const crossProviderCandidates = getImageCrossProviderCandidates(allModels, {
|
||||
currentModelId: finalModelId,
|
||||
@ -9321,6 +9385,8 @@ export async function POST(request: Request) {
|
||||
? "openai-image"
|
||||
: candidate.provider.type === "midjourney"
|
||||
? "midjourney"
|
||||
: candidate.provider.type === "runway"
|
||||
? "runway"
|
||||
: candidate.provider.type === "gemini"
|
||||
? "gemini"
|
||||
: candidate.provider.type === "huanke-gemini"
|
||||
|
||||
@ -14,14 +14,7 @@ import {
|
||||
getSystemSetting,
|
||||
} from "@/lib/db/queries";
|
||||
import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags";
|
||||
|
||||
const VIDEO_PROVIDER_TYPES = new Set([
|
||||
"volcano-video",
|
||||
"doubao-video",
|
||||
"qianwen-video",
|
||||
"fal-video",
|
||||
"ruxa-video",
|
||||
]);
|
||||
import { isVideoModelDescriptor } from "@/lib/ai/model-classification";
|
||||
|
||||
function isNanoBananaProModel(model: { displayName?: string | null; modelId?: string | null }) {
|
||||
const displayName = String(model.displayName || "").toLowerCase();
|
||||
@ -109,7 +102,12 @@ async function NewChatPage() {
|
||||
const preferredImageModels = visibleModels.filter(
|
||||
({ model, provider }) =>
|
||||
model.supportsImageGen &&
|
||||
!VIDEO_PROVIDER_TYPES.has(provider.type)
|
||||
!isVideoModelDescriptor({
|
||||
providerType: provider.type,
|
||||
modelId: model.modelId,
|
||||
capabilities: model.capabilities,
|
||||
supportsImageGen: model.supportsImageGen,
|
||||
})
|
||||
);
|
||||
const nanoBananaModel = preferredImageModels.find(({ model }) =>
|
||||
isNanoBananaProModel(model)
|
||||
|
||||
@ -24,6 +24,7 @@ import { createOpenAIImage } from "./providers/openai-image";
|
||||
import { createMidjourney } from "./providers/midjourney";
|
||||
import { createFalVideo } from "./providers/fal-video";
|
||||
import { createRuxaVideo } from "./providers/ruxa-video";
|
||||
import { createRunway } from "./providers/runway";
|
||||
import { createVolcanoMultimodal } from "./providers/volcano-multimodal";
|
||||
import { getProviderDefaultBaseUrl } from "./provider-defaults";
|
||||
|
||||
@ -227,6 +228,15 @@ function getOrCreateProvider(provider: Pick<AiProvider, "id" | "type" | "baseUrl
|
||||
return midjourneyProvider as any;
|
||||
}
|
||||
|
||||
case "runway": {
|
||||
const runwayProvider = createRunway({
|
||||
baseURL: resolvedBaseUrl,
|
||||
apiKey: provider.apiKey,
|
||||
});
|
||||
console.log(`[Provider] Created Runway provider: ${resolvedBaseUrl}`);
|
||||
return runwayProvider as any;
|
||||
}
|
||||
|
||||
case "custom":
|
||||
default: {
|
||||
if (isVolcanoMultimodalBaseUrl(resolvedBaseUrl)) {
|
||||
@ -353,6 +363,10 @@ export async function getDynamicModel(
|
||||
console.log("[getDynamicModel] Getting Midjourney chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] Midjourney model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "runway") {
|
||||
console.log("[getDynamicModel] Getting Runway chat model..."); // 调试日志
|
||||
baseModel = providerInstance.chat(resolvedModelId);
|
||||
console.log("[getDynamicModel] Runway model retrieved:", baseModel ? "success" : "failed"); // 调试日志
|
||||
} else if (provider.type === "huanke-gemini") {
|
||||
// 环科 Gemini 原生格式使用 chat() 方法
|
||||
console.log("[getDynamicModel] Getting Huanke Gemini chat model..."); // 调试日志
|
||||
|
||||
114
lib/ai/model-classification.ts
Normal file
114
lib/ai/model-classification.ts
Normal file
@ -0,0 +1,114 @@
|
||||
const STATIC_VIDEO_PROVIDER_TYPES = new Set([
|
||||
"doubao-video",
|
||||
"volcano-video",
|
||||
"qianwen-video",
|
||||
"fal-video",
|
||||
"ruxa-video",
|
||||
]);
|
||||
|
||||
const STATIC_IMAGE_PROVIDER_TYPES = new Set([
|
||||
"gemini",
|
||||
"huanke-gemini",
|
||||
"alibaba-image",
|
||||
"ruxa-image",
|
||||
"fal-image",
|
||||
"openai-image",
|
||||
"midjourney",
|
||||
"volcano-vision",
|
||||
]);
|
||||
|
||||
const RUNWAY_IMAGE_MODEL_IDS = new Set([
|
||||
"gen4_image",
|
||||
"gen4_image_turbo",
|
||||
"gemini_2.5_flash",
|
||||
]);
|
||||
|
||||
const RUNWAY_VIDEO_MODEL_PREFIXES = [
|
||||
"gen4.5",
|
||||
"gen4_turbo",
|
||||
"gen4_aleph",
|
||||
"gen3a_turbo",
|
||||
"veo3.1_fast",
|
||||
"veo3.1",
|
||||
"veo3",
|
||||
"act_two",
|
||||
"upscale_v1",
|
||||
];
|
||||
|
||||
const VIDEO_MODE_KEYS = new Set([
|
||||
"text-to-video",
|
||||
"first-frame",
|
||||
"first-last-frame",
|
||||
"reference-images",
|
||||
]);
|
||||
|
||||
function normalizeText(value?: string | null) {
|
||||
return String(value || "").trim().toLowerCase();
|
||||
}
|
||||
|
||||
function hasVideoModes(capabilities: unknown) {
|
||||
if (!capabilities || typeof capabilities !== "object") return false;
|
||||
const supportedModes = (capabilities as { supportedModes?: unknown }).supportedModes;
|
||||
if (!Array.isArray(supportedModes)) return false;
|
||||
return supportedModes.some((mode) => VIDEO_MODE_KEYS.has(String(mode || "").trim()));
|
||||
}
|
||||
|
||||
export function inferRunwayModelKind(modelId?: string | null): "image" | "video" | null {
|
||||
const normalized = normalizeText(modelId);
|
||||
if (!normalized) return null;
|
||||
if (RUNWAY_IMAGE_MODEL_IDS.has(normalized)) return "image";
|
||||
if (RUNWAY_VIDEO_MODEL_PREFIXES.some((prefix) => normalized === prefix || normalized.startsWith(`${prefix}_`))) {
|
||||
return "video";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function isStaticVideoProviderType(providerType?: string | null): boolean {
|
||||
return STATIC_VIDEO_PROVIDER_TYPES.has(normalizeText(providerType));
|
||||
}
|
||||
|
||||
export function isVideoModelDescriptor(params: {
|
||||
providerType?: string | null;
|
||||
modelId?: string | null;
|
||||
capabilities?: unknown;
|
||||
supportsImageGen?: boolean | null;
|
||||
}): boolean {
|
||||
const providerType = normalizeText(params.providerType);
|
||||
|
||||
if (isStaticVideoProviderType(providerType)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (hasVideoModes(params.capabilities)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (providerType === "runway") {
|
||||
return inferRunwayModelKind(params.modelId) === "video";
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export function isImageGenerationModelDescriptor(params: {
|
||||
providerType?: string | null;
|
||||
modelId?: string | null;
|
||||
capabilities?: unknown;
|
||||
supportsImageGen?: boolean | null;
|
||||
}): boolean {
|
||||
const providerType = normalizeText(params.providerType);
|
||||
|
||||
if (params.supportsImageGen === true) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (providerType === "runway") {
|
||||
return inferRunwayModelKind(params.modelId) === "image";
|
||||
}
|
||||
|
||||
if (isVideoModelDescriptor(params)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return STATIC_IMAGE_PROVIDER_TYPES.has(providerType);
|
||||
}
|
||||
@ -6,6 +6,10 @@
|
||||
*/
|
||||
|
||||
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||||
import {
|
||||
isImageGenerationModelDescriptor,
|
||||
isVideoModelDescriptor,
|
||||
} from "@/lib/ai/model-classification";
|
||||
|
||||
type EnabledModelWithProvider = Awaited<
|
||||
ReturnType<typeof import("@/lib/db/queries")["getEnabledAiModelsWithProvider"]>
|
||||
@ -42,14 +46,6 @@ const CHAT_PROVIDER_TYPES = [
|
||||
"volcano-multimodal",
|
||||
] as const;
|
||||
|
||||
const VIDEO_PROVIDER_TYPES = new Set([
|
||||
"doubao-video",
|
||||
"volcano-video",
|
||||
"qianwen-video",
|
||||
"fal-video",
|
||||
"ruxa-video",
|
||||
]);
|
||||
|
||||
const IMAGE_ONLY_PROVIDER_TYPES = new Set([
|
||||
"alibaba-image",
|
||||
"ruxa-image",
|
||||
@ -59,8 +55,20 @@ const IMAGE_ONLY_PROVIDER_TYPES = new Set([
|
||||
"gemini-image",
|
||||
]);
|
||||
|
||||
function isVideoProviderType(providerType: string) {
|
||||
return VIDEO_PROVIDER_TYPES.has(providerType);
|
||||
function isVideoProviderType(
|
||||
providerType: string,
|
||||
model?: {
|
||||
modelId?: string | null;
|
||||
capabilities?: unknown;
|
||||
supportsImageGen?: boolean | null;
|
||||
}
|
||||
) {
|
||||
return isVideoModelDescriptor({
|
||||
providerType,
|
||||
modelId: model?.modelId,
|
||||
capabilities: model?.capabilities,
|
||||
supportsImageGen: model?.supportsImageGen,
|
||||
});
|
||||
}
|
||||
|
||||
function isConversationalCandidate(
|
||||
@ -69,7 +77,7 @@ function isConversationalCandidate(
|
||||
) {
|
||||
const providerType = String(item.provider.type || "");
|
||||
if (!item.model.isEnabled || item.model.isSubModel) return false;
|
||||
if (isVideoProviderType(providerType)) return false;
|
||||
if (isVideoProviderType(providerType, item.model)) return false;
|
||||
|
||||
const requireVision = Boolean(options?.requireVision);
|
||||
const matchesPreferredProvider = CHAT_PROVIDER_TYPES.includes(
|
||||
@ -108,13 +116,27 @@ function matchesIntentRequirements(
|
||||
|
||||
if (params.intent === "vision") {
|
||||
return (
|
||||
!isVideoProviderType(String(item.provider.type || "")) &&
|
||||
!isVideoProviderType(String(item.provider.type || ""), item.model) &&
|
||||
hasVisionCapability(item.model)
|
||||
);
|
||||
}
|
||||
|
||||
if (params.intent === "video_gen") {
|
||||
return isVideoProviderType(String(item.provider.type || ""), item.model);
|
||||
}
|
||||
|
||||
if (params.requireVision && !hasVisionCapability(item.model)) return false;
|
||||
if (params.requireImageGen && !item.model.supportsImageGen) return false;
|
||||
if (
|
||||
params.requireImageGen &&
|
||||
!isImageGenerationModelDescriptor({
|
||||
providerType: item.provider.type,
|
||||
modelId: item.model.modelId,
|
||||
capabilities: item.model.capabilities,
|
||||
supportsImageGen: item.model.supportsImageGen,
|
||||
})
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
@ -132,7 +154,7 @@ const INTENT_MODEL_MAP: Record<
|
||||
> = {
|
||||
// 图片编辑：优先 alibaba-image（灵魂4.5，支持提示词扩展）
|
||||
image_edit: {
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini", "huanke-gemini"],
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini", "huanke-gemini"],
|
||||
requireVision: true,
|
||||
requireImageGen: true,
|
||||
description: "图片编辑",
|
||||
@ -147,14 +169,14 @@ const INTENT_MODEL_MAP: Record<
|
||||
|
||||
// 图片生成：优先灵魂4.5
|
||||
image_gen: {
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini", "huanke-gemini", "volcano-vision"],
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini", "huanke-gemini", "volcano-vision"],
|
||||
requireImageGen: true,
|
||||
description: "图片生成",
|
||||
},
|
||||
|
||||
// 视频生成
|
||||
video_gen: {
|
||||
providerTypes: ["doubao-video", "volcano-video", "qianwen-video", "fal-video", "ruxa-video"],
|
||||
providerTypes: ["doubao-video", "volcano-video", "qianwen-video", "fal-video", "ruxa-video", "runway"],
|
||||
description: "视频生成",
|
||||
},
|
||||
|
||||
@ -175,25 +197,25 @@ const SUB_INTENT_OVERRIDES: Record<
|
||||
> = {
|
||||
// 抠图：alibaba-image 效果最好
|
||||
remove_background: {
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini"],
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini"],
|
||||
reason: "抠图首选灵魂4.5",
|
||||
},
|
||||
|
||||
// 换背景：alibaba-image 支持场景生成
|
||||
replace_background: {
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini"],
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini"],
|
||||
reason: "换背景首选灵魂4.5",
|
||||
},
|
||||
|
||||
// 图片增强
|
||||
enhance: {
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "gemini", "volcano-vision"],
|
||||
providerTypes: ["alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway", "gemini", "volcano-vision"],
|
||||
reason: "增强首选灵魂4.5",
|
||||
},
|
||||
|
||||
// 风格迁移：gemini 可能更有创意
|
||||
style_transfer: {
|
||||
providerTypes: ["gemini", "alibaba-image", "ruxa-image", "fal-image", "openai-image"],
|
||||
providerTypes: ["gemini", "alibaba-image", "ruxa-image", "fal-image", "openai-image", "runway"],
|
||||
reason: "风格迁移优选Gemini",
|
||||
},
|
||||
};
|
||||
@ -321,7 +343,7 @@ export async function routeToModel(params: {
|
||||
return (
|
||||
!item.model.isSubModel &&
|
||||
item.model.isEnabled &&
|
||||
!isVideoProviderType(String(item.provider.type || "")) &&
|
||||
!isVideoProviderType(String(item.provider.type || ""), item.model) &&
|
||||
(item.model.supportsImageGen || hasVisionCapability(item.model))
|
||||
);
|
||||
});
|
||||
@ -345,7 +367,7 @@ export async function routeToModel(params: {
|
||||
(m) =>
|
||||
!m.model.isSubModel &&
|
||||
m.model.isEnabled &&
|
||||
!isVideoProviderType(String(m.provider.type || "")) &&
|
||||
!isVideoProviderType(String(m.provider.type || ""), m.model) &&
|
||||
!m.model.supportsImageGen,
|
||||
)
|
||||
: undefined) ||
|
||||
|
||||
581
lib/ai/providers/runway.ts
Normal file
581
lib/ai/providers/runway.ts
Normal file
@ -0,0 +1,581 @@
|
||||
import "server-only";
|
||||
|
||||
import sharp from "sharp";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
import { inferRunwayModelKind } from "@/lib/ai/model-classification";
|
||||
|
||||
interface RunwayConfig {
|
||||
apiKey: string;
|
||||
baseURL: string;
|
||||
}
|
||||
|
||||
interface RunwayImageParams {
|
||||
aspectRatio?: string;
|
||||
imageSize?: string;
|
||||
imageCount?: number;
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
interface RunwayVideoParams {
|
||||
mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
|
||||
ratio?: string;
|
||||
duration?: number;
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
interface UploadedImage {
|
||||
thumbnail: string;
|
||||
original: string;
|
||||
}
|
||||
|
||||
const RUNWAY_VERSION = "2024-11-06";
|
||||
const DEFAULT_BASE_URL = "https://api.dev.runwayml.com";
|
||||
const TASK_POLL_INTERVAL_MS = 5000;
|
||||
const TASK_TIMEOUT_MS = 15 * 60 * 1000;
|
||||
|
||||
const IMAGE_RESOLUTION_MAP: Record<string, Record<string, string>> = {
|
||||
"1K": {
|
||||
"21:9": "1584:672",
|
||||
"16:9": "1360:768",
|
||||
"9:16": "768:1360",
|
||||
"4:3": "1104:832",
|
||||
"3:4": "832:1104",
|
||||
"1:1": "960:960",
|
||||
},
|
||||
"2K": {
|
||||
"21:9": "1584:672",
|
||||
"16:9": "1360:768",
|
||||
"9:16": "768:1360",
|
||||
"4:3": "1104:832",
|
||||
"3:4": "832:1104",
|
||||
"1:1": "960:960",
|
||||
},
|
||||
"4K": {
|
||||
"21:9": "1584:672",
|
||||
"16:9": "1360:768",
|
||||
"9:16": "768:1360",
|
||||
"4:3": "1104:832",
|
||||
"3:4": "832:1104",
|
||||
"1:1": "960:960",
|
||||
},
|
||||
};
|
||||
|
||||
function normalizeBaseURL(baseURL: string) {
|
||||
const normalized = String(baseURL || "").trim().replace(/\/+$/, "");
|
||||
return normalized || DEFAULT_BASE_URL;
|
||||
}
|
||||
|
||||
function joinEndpoint(baseURL: string, path: string) {
|
||||
const cleanPath = path.startsWith("/") ? path : `/${path}`;
|
||||
return `${normalizeBaseURL(baseURL)}${cleanPath}`;
|
||||
}
|
||||
|
||||
function buildHeaders(apiKey: string) {
|
||||
return {
|
||||
Authorization: `Bearer ${String(apiKey || "").trim()}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-Runway-Version": RUNWAY_VERSION,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeText(value: string) {
|
||||
return String(value || "").replace(/\u00a0/g, " ").trim().normalize("NFKC");
|
||||
}
|
||||
|
||||
function normalizeImageSizeLabel(imageSize?: string): "1K" | "2K" | "4K" {
|
||||
const normalized = String(imageSize || "").trim().toUpperCase();
|
||||
if (normalized === "1K" || normalized === "2K" || normalized === "4K") {
|
||||
return normalized;
|
||||
}
|
||||
return "2K";
|
||||
}
|
||||
|
||||
function normalizeImageResolution(aspectRatio?: string, imageSize?: string) {
|
||||
const normalizedRatio = String(aspectRatio || "").trim() || "16:9";
|
||||
const normalizedSize = normalizeImageSizeLabel(imageSize);
|
||||
return (
|
||||
IMAGE_RESOLUTION_MAP[normalizedSize]?.[normalizedRatio] ||
|
||||
IMAGE_RESOLUTION_MAP[normalizedSize]?.["16:9"] ||
|
||||
"1360:768"
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeVideoRatio(ratio?: string) {
|
||||
const normalized = String(ratio || "").trim();
|
||||
if (["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"].includes(normalized)) {
|
||||
return normalized;
|
||||
}
|
||||
return "16:9";
|
||||
}
|
||||
|
||||
function normalizeVideoDuration(duration?: number) {
|
||||
if (!Number.isFinite(duration)) return 5;
|
||||
const normalized = Math.round(Number(duration));
|
||||
return normalized >= 8 ? 10 : 5;
|
||||
}
|
||||
|
||||
function normalizeSeed(seed?: number) {
|
||||
if (!Number.isFinite(seed)) return undefined;
|
||||
return Math.max(0, Math.min(4294967295, Math.round(Number(seed))));
|
||||
}
|
||||
|
||||
function detectExtensionFromContentType(contentType?: string | null) {
|
||||
const normalized = String(contentType || "").toLowerCase();
|
||||
if (normalized.includes("jpeg") || normalized.includes("jpg")) return "jpg";
|
||||
if (normalized.includes("webp")) return "webp";
|
||||
if (normalized.includes("gif")) return "gif";
|
||||
if (normalized.includes("png")) return "png";
|
||||
if (normalized.includes("mp4")) return "mp4";
|
||||
if (normalized.includes("quicktime")) return "mov";
|
||||
if (normalized.includes("webm")) return "webm";
|
||||
return "bin";
|
||||
}
|
||||
|
||||
function extractDataUriParts(dataUri: string) {
|
||||
const match = dataUri.match(/^data:([^;]+);base64,(.+)$/);
|
||||
if (!match) {
|
||||
throw new Error("不支持的 Data URI 格式");
|
||||
}
|
||||
return {
|
||||
mimeType: match[1],
|
||||
buffer: Buffer.from(match[2], "base64"),
|
||||
};
|
||||
}
|
||||
|
||||
function generateFilename(prefix: string, extension: string) {
|
||||
const timestamp = Date.now();
|
||||
const random = Math.random().toString(36).slice(2, 8);
|
||||
return `${prefix}-${timestamp}-${random}.${extension}`;
|
||||
}
|
||||
|
||||
async function uploadGeneratedBuffer(
|
||||
buffer: Buffer,
|
||||
extension: string,
|
||||
mimeType: string
|
||||
): Promise<UploadedImage> {
|
||||
const thumbnailBuffer = await sharp(buffer)
|
||||
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
|
||||
.jpeg({ quality: 80 })
|
||||
.toBuffer();
|
||||
|
||||
const originalFilename = generateFilename("runway-image", extension);
|
||||
const originalArrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer);
|
||||
|
||||
const thumbnailFilename = generateFilename("runway-thumb", "jpg");
|
||||
const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
|
||||
thumbnailBuffer.byteOffset,
|
||||
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const thumbnailUploaded = await uploadFile(thumbnailFilename, thumbnailArrayBuffer);
|
||||
|
||||
console.log("[Runway] Uploaded image to storage:", {
|
||||
original: originalUploaded.url,
|
||||
thumbnail: thumbnailUploaded.url,
|
||||
mimeType,
|
||||
});
|
||||
|
||||
return {
|
||||
thumbnail: thumbnailUploaded.url,
|
||||
original: originalUploaded.url,
|
||||
};
|
||||
}
|
||||
|
||||
async function uploadDataUriAsset(dataUri: string, prefix: "runway-input" | "runway-video") {
|
||||
const { mimeType, buffer } = extractDataUriParts(dataUri);
|
||||
const extension = detectExtensionFromContentType(mimeType);
|
||||
const filename = generateFilename(prefix, extension);
|
||||
const arrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const uploaded = await uploadFile(filename, arrayBuffer);
|
||||
return uploaded.url;
|
||||
}
|
||||
|
||||
async function normalizeAssetUrl(url?: string | null) {
|
||||
const normalized = String(url || "").trim();
|
||||
if (!normalized) return "";
|
||||
if (/^https:\/\//i.test(normalized) || /^runway:\/\//i.test(normalized)) {
|
||||
return normalized;
|
||||
}
|
||||
if (/^data:(image|video)\//i.test(normalized)) {
|
||||
return uploadDataUriAsset(normalized, /^data:video\//i.test(normalized) ? "runway-video" : "runway-input");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function collectHttpUrls(payload: unknown): string[] {
|
||||
const urls = new Set<string>();
|
||||
|
||||
const visit = (value: unknown) => {
|
||||
if (typeof value === "string") {
|
||||
const normalized = value.trim();
|
||||
if (/^https?:\/\//i.test(normalized)) {
|
||||
urls.add(normalized);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach(visit);
|
||||
return;
|
||||
}
|
||||
|
||||
if (value && typeof value === "object") {
|
||||
Object.values(value as Record<string, unknown>).forEach(visit);
|
||||
}
|
||||
};
|
||||
|
||||
visit(payload);
|
||||
return Array.from(urls);
|
||||
}
|
||||
|
||||
async function downloadRemoteAsset(url: string) {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), 120000);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
cache: "no-store",
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`下载 Runway 输出失败: ${response.status}`);
|
||||
}
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
return {
|
||||
buffer,
|
||||
contentType: response.headers.get("content-type"),
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadVideoToStorage(remoteVideoUrl: string) {
|
||||
try {
|
||||
const { buffer, contentType } = await downloadRemoteAsset(remoteVideoUrl);
|
||||
const extension = detectExtensionFromContentType(contentType || "video/mp4");
|
||||
const filename = generateFilename("runway-video", extension === "bin" ? "mp4" : extension);
|
||||
const arrayBuffer = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength
|
||||
) as ArrayBuffer;
|
||||
const uploaded = await uploadFile(filename, arrayBuffer);
|
||||
return uploaded.url;
|
||||
} catch (error) {
|
||||
console.warn("[Runway] Failed to archive video output, using temporary URL:", error);
|
||||
return remoteVideoUrl;
|
||||
}
|
||||
}
|
||||
|
||||
function formatImageMarkdown(images: UploadedImage[], imageSize: string) {
|
||||
return images
|
||||
.map((image) => ``)
|
||||
.join("\n\n");
|
||||
}
|
||||
|
||||
function formatVideoMarkdown(videoUrl: string) {
|
||||
return `<video src="${videoUrl}" controls style="max-width: 100%; border-radius: 8px; margin-bottom: 12px;"></video>\n\n[下载视频](${videoUrl})`;
|
||||
}
|
||||
|
||||
function extractPromptAndAssets(messages: Array<{ role: string; content: any }>) {
|
||||
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
|
||||
if (!lastUserMessage) {
|
||||
throw new Error("未找到用户输入内容");
|
||||
}
|
||||
|
||||
let prompt = "";
|
||||
const imageUrls: string[] = [];
|
||||
|
||||
const pushImageUrl = (value?: string) => {
|
||||
if (!value) return;
|
||||
const normalized = String(value || "").trim();
|
||||
if (!normalized) return;
|
||||
if (!imageUrls.includes(normalized)) {
|
||||
imageUrls.push(normalized);
|
||||
}
|
||||
};
|
||||
|
||||
if (typeof lastUserMessage.content === "string") {
|
||||
prompt = normalizeText(lastUserMessage.content);
|
||||
} else if (Array.isArray(lastUserMessage.content)) {
|
||||
for (const part of lastUserMessage.content) {
|
||||
if (part?.type === "text" && typeof part.text === "string") {
|
||||
prompt = `${prompt} ${part.text}`.trim();
|
||||
} else if (part?.type === "image_url" && part.image_url?.url) {
|
||||
pushImageUrl(part.image_url.url);
|
||||
} else if (part?.type === "file") {
|
||||
pushImageUrl(part.url || part.data);
|
||||
} else if (part?.type === "image") {
|
||||
pushImageUrl(part.image || part.url);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
prompt: normalizeText(prompt) || "生成内容",
|
||||
imageUrls,
|
||||
};
|
||||
}
|
||||
|
||||
async function createTask(baseURL: string, apiKey: string, path: string, body: Record<string, any>) {
|
||||
const response = await fetch(joinEndpoint(baseURL, path), {
|
||||
method: "POST",
|
||||
headers: buildHeaders(apiKey),
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
const rawText = await response.text();
|
||||
let payload: any = null;
|
||||
try {
|
||||
payload = rawText ? JSON.parse(rawText) : null;
|
||||
} catch {
|
||||
payload = rawText;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const message =
|
||||
typeof payload?.error === "string"
|
||||
? payload.error
|
||||
: typeof payload?.message === "string"
|
||||
? payload.message
|
||||
: rawText || "Runway 请求失败";
|
||||
throw new Error(`${message} (HTTP ${response.status})`);
|
||||
}
|
||||
|
||||
const taskId = String(payload?.id || "").trim();
|
||||
if (!taskId) {
|
||||
throw new Error("Runway 未返回任务 ID");
|
||||
}
|
||||
|
||||
return taskId;
|
||||
}
|
||||
|
||||
async function waitForTaskOutput(baseURL: string, apiKey: string, taskId: string) {
|
||||
const deadline = Date.now() + TASK_TIMEOUT_MS;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
const response = await fetch(joinEndpoint(baseURL, `/v1/tasks/${taskId}`), {
|
||||
method: "GET",
|
||||
headers: buildHeaders(apiKey),
|
||||
cache: "no-store",
|
||||
});
|
||||
|
||||
const rawText = await response.text();
|
||||
let payload: any = null;
|
||||
try {
|
||||
payload = rawText ? JSON.parse(rawText) : null;
|
||||
} catch {
|
||||
payload = rawText;
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const message =
|
||||
typeof payload?.error === "string"
|
||||
? payload.error
|
||||
: typeof payload?.message === "string"
|
||||
? payload.message
|
||||
: rawText || "Runway 轮询失败";
|
||||
throw new Error(`${message} (HTTP ${response.status})`);
|
||||
}
|
||||
|
||||
const status = String(payload?.status || "").trim().toUpperCase();
|
||||
if (status === "SUCCEEDED") {
|
||||
return payload;
|
||||
}
|
||||
if (status === "FAILED" || status === "CANCELLED") {
|
||||
throw new Error(
|
||||
String(payload?.failure || payload?.error || payload?.message || "Runway 任务失败")
|
||||
);
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, TASK_POLL_INTERVAL_MS));
|
||||
}
|
||||
|
||||
throw new Error("Runway 任务超时，请稍后重试");
|
||||
}
|
||||
|
||||
async function uploadRunwayImages(taskOutput: unknown) {
|
||||
const outputUrls = collectHttpUrls(taskOutput);
|
||||
const uploadedImages: UploadedImage[] = [];
|
||||
|
||||
for (const url of outputUrls) {
|
||||
try {
|
||||
const { buffer, contentType } = await downloadRemoteAsset(url);
|
||||
if (!String(contentType || "").startsWith("image/")) {
|
||||
continue;
|
||||
}
|
||||
const extension = detectExtensionFromContentType(contentType);
|
||||
uploadedImages.push(
|
||||
await uploadGeneratedBuffer(
|
||||
buffer,
|
||||
extension === "bin" ? "png" : extension,
|
||||
contentType || "image/png"
|
||||
)
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn("[Runway] Failed to archive image output:", error);
|
||||
}
|
||||
}
|
||||
|
||||
if (uploadedImages.length === 0) {
|
||||
throw new Error("Runway 未返回可用图片");
|
||||
}
|
||||
|
||||
return uploadedImages;
|
||||
}
|
||||
|
||||
async function resolveRunwayVideoUrl(taskOutput: unknown) {
|
||||
const outputUrls = collectHttpUrls(taskOutput);
|
||||
if (outputUrls.length === 0) {
|
||||
throw new Error("Runway 未返回可用视频地址");
|
||||
}
|
||||
|
||||
for (const url of outputUrls) {
|
||||
if (/(\.mp4|\.mov|\.webm)(\?|$)/i.test(url) || /video/i.test(url)) {
|
||||
return uploadVideoToStorage(url);
|
||||
}
|
||||
}
|
||||
|
||||
return uploadVideoToStorage(outputUrls[0]!);
|
||||
}
|
||||
|
||||
function getRunwayVideoEndpoint(modelId: string, hasPromptImage: boolean) {
|
||||
const normalized = normalizeText(modelId);
|
||||
if (hasPromptImage) return "/v1/image_to_video";
|
||||
if (normalized === "gen4.5") return "/v1/image_to_video";
|
||||
return "/v1/text_to_video";
|
||||
}
|
||||
|
||||
function buildReferenceImages(imageUrls: string[]) {
|
||||
return imageUrls.slice(0, 3).map((uri, index) => ({
|
||||
uri,
|
||||
tag: `ref_${index + 1}`.replace(/[^a-z0-9_]/g, "_"),
|
||||
}));
|
||||
}
|
||||
|
||||
export function createRunway({ apiKey, baseURL }: RunwayConfig) {
|
||||
return {
|
||||
provider: "runway",
|
||||
chat: (modelId: string) => ({
|
||||
provider: "runway",
|
||||
doGenerate: async (options: {
|
||||
messages: Array<{ role: string; content: any }>;
|
||||
imageParams?: RunwayImageParams;
|
||||
videoParams?: RunwayVideoParams;
|
||||
overrideModelId?: string;
|
||||
}) => {
|
||||
try {
|
||||
const activeModelId = normalizeText(options.overrideModelId || modelId);
|
||||
const { prompt, imageUrls } = extractPromptAndAssets(options.messages);
|
||||
const normalizedImageUrls = (
|
||||
await Promise.all(imageUrls.map((url) => normalizeAssetUrl(url)))
|
||||
).filter(Boolean);
|
||||
const modelKind =
|
||||
inferRunwayModelKind(activeModelId) ||
|
||||
(options.videoParams ? "video" : options.imageParams ? "image" : null) ||
|
||||
"video";
|
||||
|
||||
if (modelKind === "image") {
|
||||
const imageSize = normalizeImageSizeLabel(options.imageParams?.imageSize);
|
||||
const taskId = await createTask(baseURL, apiKey, "/v1/text_to_image", {
|
||||
model: activeModelId,
|
||||
promptText: prompt,
|
||||
ratio: normalizeImageResolution(options.imageParams?.aspectRatio, imageSize),
|
||||
...(normalizedImageUrls.length > 0
|
||||
? { referenceImages: buildReferenceImages(normalizedImageUrls) }
|
||||
: {}),
|
||||
...(normalizeSeed(options.imageParams?.seed) !== undefined
|
||||
? { seed: normalizeSeed(options.imageParams?.seed) }
|
||||
: {}),
|
||||
});
|
||||
|
||||
const task = await waitForTaskOutput(baseURL, apiKey, taskId);
|
||||
const uploadedImages = await uploadRunwayImages(task?.output ?? task);
|
||||
const text = formatImageMarkdown(uploadedImages, imageSize);
|
||||
|
||||
return {
|
||||
text,
|
||||
finishReason: "stop" as const,
|
||||
usage: {
|
||||
promptTokens: 0,
|
||||
completionTokens: text.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const videoMode = options.videoParams?.mode || (normalizedImageUrls[0] ? "first-frame" : "text-to-video");
|
||||
if (
|
||||
(videoMode === "first-frame" || videoMode === "first-last-frame" || videoMode === "reference-images") &&
|
||||
!normalizedImageUrls[0]
|
||||
) {
|
||||
throw new Error("Runway 图生视频至少需要 1 张参考图");
|
||||
}
|
||||
|
||||
const endpoint = getRunwayVideoEndpoint(activeModelId, Boolean(normalizedImageUrls[0]));
|
||||
const taskId = await createTask(baseURL, apiKey, endpoint, {
|
||||
model: activeModelId,
|
||||
promptText: prompt,
|
||||
ratio: normalizeVideoRatio(options.videoParams?.ratio),
|
||||
duration: normalizeVideoDuration(options.videoParams?.duration),
|
||||
...(normalizedImageUrls[0] ? { promptImage: normalizedImageUrls[0] } : {}),
|
||||
...(normalizeSeed(options.videoParams?.seed) !== undefined
|
||||
? { seed: normalizeSeed(options.videoParams?.seed) }
|
||||
: {}),
|
||||
});
|
||||
|
||||
const task = await waitForTaskOutput(baseURL, apiKey, taskId);
|
||||
const videoUrl = await resolveRunwayVideoUrl(task?.output ?? task);
|
||||
const text = formatVideoMarkdown(videoUrl);
|
||||
|
||||
return {
|
||||
text,
|
||||
finishReason: "stop" as const,
|
||||
usage: {
|
||||
promptTokens: 0,
|
||||
completionTokens: text.length,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("[Runway] Generation failed:", error);
|
||||
return {
|
||||
text:
|
||||
error instanceof Error
|
||||
? `${
|
||||
(inferRunwayModelKind(options.overrideModelId || modelId) ||
|
||||
(options.videoParams ? "video" : options.imageParams ? "image" : "video")) === "image"
|
||||
? "图片"
|
||||
: "视频"
|
||||
}生成失败: ${error.message}`
|
||||
: "Runway 生成失败，请稍后重试",
|
||||
finishReason: "error" as const,
|
||||
usage: {
|
||||
promptTokens: 0,
|
||||
completionTokens: 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
},
|
||||
doStream: async function* (options: {
|
||||
messages: Array<{ role: string; content: any }>;
|
||||
imageParams?: RunwayImageParams;
|
||||
videoParams?: RunwayVideoParams;
|
||||
overrideModelId?: string;
|
||||
}) {
|
||||
const result = await this.doGenerate(options);
|
||||
yield {
|
||||
type: "text-delta",
|
||||
textDelta: result.text,
|
||||
};
|
||||
},
|
||||
modelId,
|
||||
specificationVersion: "v1" as const,
|
||||
defaultObjectGenerationMode: "json" as const,
|
||||
}),
|
||||
};
|
||||
}
|
||||
@ -193,6 +193,33 @@ function createProviderDefaultCapability(providerType: string): VideoToolCapabil
|
||||
};
|
||||
}
|
||||
|
||||
if (providerType === "runway") {
|
||||
return {
|
||||
schemaVersion: "v1",
|
||||
source: "provider-default",
|
||||
providerType,
|
||||
modeOptions: normalizeModeOptions(["text-to-video", "first-frame"]),
|
||||
ratioOptions: [...STANDARD_RATIOS],
|
||||
durationOptions: [5, 10],
|
||||
resolutionOptions: [],
|
||||
ui: {
|
||||
showResolution: false,
|
||||
showFps: false,
|
||||
showCameraMove: false,
|
||||
showGenerateAudio: false,
|
||||
showReturnLastFrame: false,
|
||||
showSeed: true,
|
||||
},
|
||||
defaults: {
|
||||
mode: "text-to-video",
|
||||
ratio: "16:9",
|
||||
duration: 5,
|
||||
generateAudio: false,
|
||||
fps: 24,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
schemaVersion: "v1",
|
||||
source: "provider-default",
|
||||
|
||||
@ -7,6 +7,7 @@ import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
|
||||
import { createFalImage } from "@/lib/ai/providers/fal-image";
|
||||
import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
|
||||
import { createMidjourney } from "@/lib/ai/providers/midjourney";
|
||||
import { createRunway } from "@/lib/ai/providers/runway";
|
||||
import { filterReachableRemoteUrls } from "@/lib/net/url-access";
|
||||
|
||||
export interface CanvasGenerateImageRequest {
|
||||
@ -39,6 +40,7 @@ const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([
|
||||
"fal-image",
|
||||
"openai-image",
|
||||
"midjourney",
|
||||
"runway",
|
||||
]);
|
||||
|
||||
function parseImagesFromMarkdown(markdown: string): ParsedImage[] {
|
||||
@ -76,6 +78,7 @@ function getProviderBatchLimit(providerType: string) {
|
||||
case "ruxa-image":
|
||||
case "openai-image":
|
||||
case "midjourney":
|
||||
case "runway":
|
||||
return 1;
|
||||
case "gemini":
|
||||
return 2;
|
||||
@ -151,7 +154,7 @@ export async function generateCanvasImage(
|
||||
}
|
||||
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(fallbackProvider.type)) {
|
||||
throw new Error(
|
||||
"当前图片模型类型暂不支持画布生成，请在后台选择 Gemini/阿里/RUXA/FAL/OpenAI 图片模型。"
|
||||
"当前图片模型类型暂不支持画布生成，请在后台选择 Gemini/阿里/RUXA/FAL/OpenAI/Runway 图片模型。"
|
||||
);
|
||||
}
|
||||
if (fallbackModel.model.id !== selectedModel.model.id) {
|
||||
@ -311,9 +314,22 @@ export async function generateCanvasImage(
|
||||
});
|
||||
return result.text || "";
|
||||
};
|
||||
} else if (provider.type === "runway") {
|
||||
const runwayProvider = createRunway({
|
||||
apiKey: provider.apiKey,
|
||||
baseURL: provider.baseUrl,
|
||||
});
|
||||
const modelInstance = runwayProvider.chat(selectedModel.model.modelId);
|
||||
executeGenerateBatch = async (_batchImageCount) => {
|
||||
const result = await modelInstance.doGenerate({
|
||||
messages,
|
||||
imageParams,
|
||||
});
|
||||
return result.text || "";
|
||||
};
|
||||
} else {
|
||||
throw new Error(
|
||||
`不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/阿里/RUXA/FAL/OpenAI/Midjourney 图片模型`
|
||||
`不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/阿里/RUXA/FAL/OpenAI/Midjourney/Runway 图片模型`
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@ -91,6 +91,7 @@ import {
|
||||
} from "../points/config";
|
||||
import { inferOriginalImageUrl } from "../image-url";
|
||||
import { getAgentRequiredUploadCount } from "../agents/preset-mode-config";
|
||||
import { isVideoModelDescriptor } from "../ai/model-classification";
|
||||
|
||||
// Optionally, if not using email/pass login, you can
|
||||
// use the Drizzle adapter for Auth.js / NextAuth
|
||||
@ -1755,14 +1756,6 @@ type AgentPresetKind = "image" | "video" | "text";
|
||||
type AgentPresetScope = "all" | AgentPresetVisibility;
|
||||
type AgentCategoryScope = "all" | "public" | "private" | "global";
|
||||
type AgentPresetSort = "latest" | "hottest" | "most_used";
|
||||
const AGENT_VIDEO_PROVIDER_TYPES = new Set([
|
||||
"volcano-video",
|
||||
"doubao-video",
|
||||
"qianwen-video",
|
||||
"fal-video",
|
||||
"ruxa-video",
|
||||
]);
|
||||
|
||||
function normalizeAgentCategoryName(value: string) {
|
||||
return String(value || "").trim().replace(/\s+/g, " ");
|
||||
}
|
||||
@ -1804,7 +1797,12 @@ async function validateAgentDefaultModel(params: {
|
||||
throw new ChatSDKError("bad_request:database", "Invalid default model");
|
||||
}
|
||||
|
||||
const isVideoModel = AGENT_VIDEO_PROVIDER_TYPES.has(String(model.providerType || ""));
|
||||
const isVideoModel = isVideoModelDescriptor({
|
||||
providerType: model.providerType,
|
||||
modelId: model.modelId,
|
||||
capabilities: model.capabilities,
|
||||
supportsImageGen: model.supportsImageGen,
|
||||
});
|
||||
|
||||
if (params.presetType === "video") {
|
||||
if (!isVideoModel) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user