292 lines
8.5 KiB
TypeScript
292 lines
8.5 KiB
TypeScript
import "server-only";
|
|
|
|
import { uploadFile } from "@/lib/storage/service";
|
|
|
|
interface FalVideoConfig {
|
|
apiKey: string;
|
|
baseURL: string;
|
|
}
|
|
|
|
interface FalVideoParams {
|
|
mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
|
|
ratio?: string;
|
|
duration?: number;
|
|
generateAudio?: boolean;
|
|
negativePrompt?: string;
|
|
cfgScale?: number;
|
|
shotType?: "customize" | "intelligent";
|
|
}
|
|
|
|
const DEFAULT_DURATION = 5;
|
|
const DEFAULT_RATIO = "16:9";
|
|
const DEFAULT_SHOT_TYPE = "customize";
|
|
const DEFAULT_TIMEOUT_MS = 10 * 60 * 1000;
|
|
|
|
function normalizeFalCredentials(apiKey: string) {
|
|
return apiKey.trim().replace(/^bearer\s+/i, "").replace(/^key\s+/i, "");
|
|
}
|
|
|
|
function normalizeDuration(value?: number) {
|
|
if (!Number.isFinite(value)) return DEFAULT_DURATION;
|
|
return Math.max(3, Math.min(15, Math.round(value as number)));
|
|
}
|
|
|
|
function normalizeRatio(value?: string) {
|
|
if (!value) return DEFAULT_RATIO;
|
|
const ratio = value.trim();
|
|
if (["16:9", "9:16", "1:1"].includes(ratio)) return ratio;
|
|
return DEFAULT_RATIO;
|
|
}
|
|
|
|
function normalizeShotType(value?: string) {
|
|
if (value === "intelligent") return "intelligent";
|
|
return DEFAULT_SHOT_TYPE;
|
|
}
|
|
|
|
function extractPromptAndImages(messages: Array<{ role: string; content: any }>) {
|
|
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
|
|
if (!lastUserMessage) {
|
|
throw new Error("No user message found");
|
|
}
|
|
|
|
let prompt = "";
|
|
const imageUrls: string[] = [];
|
|
|
|
const pushImageUrl = (url?: string) => {
|
|
if (!url) return;
|
|
const normalized = String(url).trim();
|
|
|
|
if (normalized.startsWith("data:")) {
|
|
throw new Error("FAL 视频模型不支持 Base64 图片，请先上传图片后再生成视频");
|
|
}
|
|
|
|
if (!normalized.startsWith("http")) return;
|
|
if (!imageUrls.includes(normalized)) {
|
|
imageUrls.push(normalized);
|
|
}
|
|
};
|
|
|
|
if (typeof lastUserMessage.content === "string") {
|
|
prompt = lastUserMessage.content.trim();
|
|
} else if (Array.isArray(lastUserMessage.content)) {
|
|
for (const part of lastUserMessage.content) {
|
|
if (part?.type === "text" && typeof part.text === "string") {
|
|
prompt = `${prompt} ${part.text}`.trim();
|
|
} else if (part?.type === "image_url" && part.image_url?.url) {
|
|
pushImageUrl(part.image_url.url);
|
|
} else if (part?.type === "file") {
|
|
pushImageUrl(part.url || part.data);
|
|
} else if (part?.type === "image") {
|
|
pushImageUrl(part.image || part.url);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!prompt) {
|
|
prompt = "生成视频";
|
|
}
|
|
|
|
return { prompt, imageUrls };
|
|
}
|
|
|
|
function extractVideoUrl(payload: any): string | undefined {
|
|
if (!payload) return undefined;
|
|
|
|
if (typeof payload === "string" && payload.startsWith("http")) {
|
|
return payload;
|
|
}
|
|
|
|
if (typeof payload !== "object") return undefined;
|
|
|
|
const direct =
|
|
payload?.video?.url ||
|
|
payload?.output?.video?.url ||
|
|
payload?.data?.video?.url ||
|
|
payload?.video_url ||
|
|
payload?.output?.video_url ||
|
|
payload?.data?.video_url;
|
|
|
|
if (typeof direct === "string" && direct.startsWith("http")) {
|
|
return direct;
|
|
}
|
|
|
|
if (Array.isArray(payload)) {
|
|
for (const item of payload) {
|
|
const nested = extractVideoUrl(item);
|
|
if (nested) return nested;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
for (const value of Object.values(payload)) {
|
|
const nested = extractVideoUrl(value);
|
|
if (nested) return nested;
|
|
}
|
|
|
|
return undefined;
|
|
}
|
|
|
|
async function downloadAndUploadVideo(videoUrl: string): Promise<string> {
|
|
try {
|
|
const response = await fetch(videoUrl);
|
|
if (!response.ok) {
|
|
throw new Error(`Failed to download video: ${response.status}`);
|
|
}
|
|
|
|
const buffer = Buffer.from(await response.arrayBuffer());
|
|
const filename = `fal-video-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.mp4`;
|
|
const uploadResult = await uploadFile(filename, buffer as any);
|
|
return uploadResult.url;
|
|
} catch (error) {
|
|
console.error("[Fal Video] Failed to upload video to storage, fallback original URL", error);
|
|
return videoUrl;
|
|
}
|
|
}
|
|
|
|
function formatVideoResponse(videoUrl: string): string {
|
|
return `<video src="${videoUrl}" controls style="max-width: 100%; border-radius: 8px; margin-bottom: 12px;"></video>\n\n[下载视频](${videoUrl})`;
|
|
}
|
|
|
|
function buildFalInput(params: {
|
|
prompt: string;
|
|
imageUrls: string[];
|
|
videoParams?: FalVideoParams;
|
|
}) {
|
|
const { prompt, imageUrls, videoParams } = params;
|
|
const mode = videoParams?.mode || "first-frame";
|
|
|
|
const input: Record<string, any> = {
|
|
prompt,
|
|
duration: String(normalizeDuration(videoParams?.duration)),
|
|
aspect_ratio: normalizeRatio(videoParams?.ratio),
|
|
shot_type: normalizeShotType(videoParams?.shotType),
|
|
};
|
|
|
|
if (typeof videoParams?.generateAudio === "boolean") {
|
|
input.generate_audio = videoParams.generateAudio;
|
|
}
|
|
|
|
if (videoParams?.negativePrompt?.trim()) {
|
|
input.negative_prompt = videoParams.negativePrompt.trim();
|
|
}
|
|
|
|
if (typeof videoParams?.cfgScale === "number" && Number.isFinite(videoParams.cfgScale)) {
|
|
input.cfg_scale = videoParams.cfgScale;
|
|
}
|
|
|
|
if (mode === "first-last-frame") {
|
|
if (!imageUrls[0] || !imageUrls[1]) {
|
|
throw new Error("请至少上传 2 张图片（首帧 + 尾帧）");
|
|
}
|
|
input.image_url = imageUrls[0];
|
|
input.end_image_url = imageUrls[1];
|
|
return input;
|
|
}
|
|
|
|
if (!imageUrls[0]) {
|
|
throw new Error("FAL O3 图生视频需要至少 1 张起始图片");
|
|
}
|
|
|
|
input.image_url = imageUrls[0];
|
|
if (imageUrls[1]) {
|
|
input.end_image_url = imageUrls[1];
|
|
}
|
|
|
|
return input;
|
|
}
|
|
|
|
export function createFalVideo({ apiKey, baseURL }: FalVideoConfig) {
|
|
return {
|
|
provider: "fal-video",
|
|
chat: (modelId: string) => ({
|
|
provider: "fal-video",
|
|
doGenerate: async (options: {
|
|
messages: Array<{ role: string; content: any }>;
|
|
videoParams?: FalVideoParams;
|
|
overrideModelId?: string;
|
|
}) => {
|
|
try {
|
|
const activeModelId = options.overrideModelId || modelId;
|
|
const { prompt, imageUrls } = extractPromptAndImages(options.messages);
|
|
const input = buildFalInput({
|
|
prompt,
|
|
imageUrls,
|
|
videoParams: options.videoParams,
|
|
});
|
|
|
|
console.log("[Fal Video] Submitting task", {
|
|
model: activeModelId,
|
|
baseURL,
|
|
duration: input.duration,
|
|
aspect_ratio: input.aspect_ratio,
|
|
hasEndFrame: Boolean(input.end_image_url),
|
|
});
|
|
|
|
const fal = await import("@fal-ai/serverless-client");
|
|
fal.config({
|
|
credentials: normalizeFalCredentials(apiKey),
|
|
});
|
|
|
|
const result: any = await fal.subscribe(activeModelId, {
|
|
input,
|
|
logs: true,
|
|
timeout: DEFAULT_TIMEOUT_MS,
|
|
onQueueUpdate: (update: any) => {
|
|
if (update?.status === "IN_PROGRESS" && Array.isArray(update.logs)) {
|
|
for (const log of update.logs) {
|
|
if (log?.message) {
|
|
console.log(`[Fal Video][${activeModelId}]`, log.message);
|
|
}
|
|
}
|
|
}
|
|
},
|
|
});
|
|
|
|
const remoteVideoUrl = extractVideoUrl(result) || extractVideoUrl(result?.data);
|
|
if (!remoteVideoUrl) {
|
|
throw new Error("FAL 未返回视频地址");
|
|
}
|
|
|
|
const hostedVideoUrl = await downloadAndUploadVideo(remoteVideoUrl);
|
|
const text = formatVideoResponse(hostedVideoUrl);
|
|
|
|
return {
|
|
text,
|
|
finishReason: "stop" as const,
|
|
usage: {
|
|
promptTokens: 0,
|
|
completionTokens: text.length,
|
|
},
|
|
};
|
|
} catch (error) {
|
|
console.error("[Fal Video] Generation failed", error);
|
|
const errorText =
|
|
error instanceof Error ? `视频生成失败: ${error.message}` : "视频生成失败，请稍后重试";
|
|
return {
|
|
text: errorText,
|
|
finishReason: "error" as const,
|
|
usage: {
|
|
promptTokens: 0,
|
|
completionTokens: 0,
|
|
},
|
|
};
|
|
}
|
|
},
|
|
doStream: async function* (options: {
|
|
messages: Array<{ role: string; content: any }>;
|
|
videoParams?: FalVideoParams;
|
|
overrideModelId?: string;
|
|
}) {
|
|
const result = await this.doGenerate(options);
|
|
yield {
|
|
type: "text-delta",
|
|
textDelta: result.text,
|
|
};
|
|
},
|
|
modelId,
|
|
specificationVersion: "v1" as const,
|
|
defaultObjectGenerationMode: "json" as const,
|
|
}),
|
|
};
|
|
}
|