582 lines
18 KiB
TypeScript
582 lines
18 KiB
TypeScript
import "server-only";
|
|
|
|
import sharp from "sharp";
|
|
import { uploadFile } from "@/lib/storage/service";
|
|
import { inferRunwayModelKind } from "@/lib/ai/model-classification";
|
|
|
|
interface RunwayConfig {
|
|
apiKey: string;
|
|
baseURL: string;
|
|
}
|
|
|
|
interface RunwayImageParams {
|
|
aspectRatio?: string;
|
|
imageSize?: string;
|
|
imageCount?: number;
|
|
seed?: number;
|
|
}
|
|
|
|
interface RunwayVideoParams {
|
|
mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
|
|
ratio?: string;
|
|
duration?: number;
|
|
seed?: number;
|
|
}
|
|
|
|
interface UploadedImage {
|
|
thumbnail: string;
|
|
original: string;
|
|
}
|
|
|
|
const RUNWAY_VERSION = "2024-11-06";
|
|
const DEFAULT_BASE_URL = "https://api.dev.runwayml.com";
|
|
const TASK_POLL_INTERVAL_MS = 5000;
|
|
const TASK_TIMEOUT_MS = 15 * 60 * 1000;
|
|
|
|
const IMAGE_RESOLUTION_MAP: Record<string, Record<string, string>> = {
|
|
"1K": {
|
|
"21:9": "1584:672",
|
|
"16:9": "1360:768",
|
|
"9:16": "768:1360",
|
|
"4:3": "1104:832",
|
|
"3:4": "832:1104",
|
|
"1:1": "960:960",
|
|
},
|
|
"2K": {
|
|
"21:9": "1584:672",
|
|
"16:9": "1360:768",
|
|
"9:16": "768:1360",
|
|
"4:3": "1104:832",
|
|
"3:4": "832:1104",
|
|
"1:1": "960:960",
|
|
},
|
|
"4K": {
|
|
"21:9": "1584:672",
|
|
"16:9": "1360:768",
|
|
"9:16": "768:1360",
|
|
"4:3": "1104:832",
|
|
"3:4": "832:1104",
|
|
"1:1": "960:960",
|
|
},
|
|
};
|
|
|
|
function normalizeBaseURL(baseURL: string) {
|
|
const normalized = String(baseURL || "").trim().replace(/\/+$/, "");
|
|
return normalized || DEFAULT_BASE_URL;
|
|
}
|
|
|
|
function joinEndpoint(baseURL: string, path: string) {
|
|
const cleanPath = path.startsWith("/") ? path : `/${path}`;
|
|
return `${normalizeBaseURL(baseURL)}${cleanPath}`;
|
|
}
|
|
|
|
function buildHeaders(apiKey: string) {
|
|
return {
|
|
Authorization: `Bearer ${String(apiKey || "").trim()}`,
|
|
"Content-Type": "application/json",
|
|
"X-Runway-Version": RUNWAY_VERSION,
|
|
};
|
|
}
|
|
|
|
function normalizeText(value: string) {
|
|
return String(value || "").replace(/\u00a0/g, " ").trim().normalize("NFKC");
|
|
}
|
|
|
|
function normalizeImageSizeLabel(imageSize?: string): "1K" | "2K" | "4K" {
|
|
const normalized = String(imageSize || "").trim().toUpperCase();
|
|
if (normalized === "1K" || normalized === "2K" || normalized === "4K") {
|
|
return normalized;
|
|
}
|
|
return "2K";
|
|
}
|
|
|
|
function normalizeImageResolution(aspectRatio?: string, imageSize?: string) {
|
|
const normalizedRatio = String(aspectRatio || "").trim() || "16:9";
|
|
const normalizedSize = normalizeImageSizeLabel(imageSize);
|
|
return (
|
|
IMAGE_RESOLUTION_MAP[normalizedSize]?.[normalizedRatio] ||
|
|
IMAGE_RESOLUTION_MAP[normalizedSize]?.["16:9"] ||
|
|
"1360:768"
|
|
);
|
|
}
|
|
|
|
function normalizeVideoRatio(ratio?: string) {
|
|
const normalized = String(ratio || "").trim();
|
|
if (["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"].includes(normalized)) {
|
|
return normalized;
|
|
}
|
|
return "16:9";
|
|
}
|
|
|
|
function normalizeVideoDuration(duration?: number) {
|
|
if (!Number.isFinite(duration)) return 5;
|
|
const normalized = Math.round(Number(duration));
|
|
return normalized >= 8 ? 10 : 5;
|
|
}
|
|
|
|
function normalizeSeed(seed?: number) {
|
|
if (!Number.isFinite(seed)) return undefined;
|
|
return Math.max(0, Math.min(4294967295, Math.round(Number(seed))));
|
|
}
|
|
|
|
function detectExtensionFromContentType(contentType?: string | null) {
|
|
const normalized = String(contentType || "").toLowerCase();
|
|
if (normalized.includes("jpeg") || normalized.includes("jpg")) return "jpg";
|
|
if (normalized.includes("webp")) return "webp";
|
|
if (normalized.includes("gif")) return "gif";
|
|
if (normalized.includes("png")) return "png";
|
|
if (normalized.includes("mp4")) return "mp4";
|
|
if (normalized.includes("quicktime")) return "mov";
|
|
if (normalized.includes("webm")) return "webm";
|
|
return "bin";
|
|
}
|
|
|
|
function extractDataUriParts(dataUri: string) {
|
|
const match = dataUri.match(/^data:([^;]+);base64,(.+)$/);
|
|
if (!match) {
|
|
throw new Error("不支持的 Data URI 格式");
|
|
}
|
|
return {
|
|
mimeType: match[1],
|
|
buffer: Buffer.from(match[2], "base64"),
|
|
};
|
|
}
|
|
|
|
function generateFilename(prefix: string, extension: string) {
|
|
const timestamp = Date.now();
|
|
const random = Math.random().toString(36).slice(2, 8);
|
|
return `${prefix}-${timestamp}-${random}.${extension}`;
|
|
}
|
|
|
|
async function uploadGeneratedBuffer(
|
|
buffer: Buffer,
|
|
extension: string,
|
|
mimeType: string
|
|
): Promise<UploadedImage> {
|
|
const thumbnailBuffer = await sharp(buffer)
|
|
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
|
|
.jpeg({ quality: 80 })
|
|
.toBuffer();
|
|
|
|
const originalFilename = generateFilename("runway-image", extension);
|
|
const originalArrayBuffer = buffer.buffer.slice(
|
|
buffer.byteOffset,
|
|
buffer.byteOffset + buffer.byteLength
|
|
) as ArrayBuffer;
|
|
const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer);
|
|
|
|
const thumbnailFilename = generateFilename("runway-thumb", "jpg");
|
|
const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
|
|
thumbnailBuffer.byteOffset,
|
|
thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength
|
|
) as ArrayBuffer;
|
|
const thumbnailUploaded = await uploadFile(thumbnailFilename, thumbnailArrayBuffer);
|
|
|
|
console.log("[Runway] Uploaded image to storage:", {
|
|
original: originalUploaded.url,
|
|
thumbnail: thumbnailUploaded.url,
|
|
mimeType,
|
|
});
|
|
|
|
return {
|
|
thumbnail: thumbnailUploaded.url,
|
|
original: originalUploaded.url,
|
|
};
|
|
}
|
|
|
|
async function uploadDataUriAsset(dataUri: string, prefix: "runway-input" | "runway-video") {
|
|
const { mimeType, buffer } = extractDataUriParts(dataUri);
|
|
const extension = detectExtensionFromContentType(mimeType);
|
|
const filename = generateFilename(prefix, extension);
|
|
const arrayBuffer = buffer.buffer.slice(
|
|
buffer.byteOffset,
|
|
buffer.byteOffset + buffer.byteLength
|
|
) as ArrayBuffer;
|
|
const uploaded = await uploadFile(filename, arrayBuffer);
|
|
return uploaded.url;
|
|
}
|
|
|
|
async function normalizeAssetUrl(url?: string | null) {
|
|
const normalized = String(url || "").trim();
|
|
if (!normalized) return "";
|
|
if (/^https:\/\//i.test(normalized) || /^runway:\/\//i.test(normalized)) {
|
|
return normalized;
|
|
}
|
|
if (/^data:(image|video)\//i.test(normalized)) {
|
|
return uploadDataUriAsset(normalized, /^data:video\//i.test(normalized) ? "runway-video" : "runway-input");
|
|
}
|
|
return normalized;
|
|
}
|
|
|
|
function collectHttpUrls(payload: unknown): string[] {
|
|
const urls = new Set<string>();
|
|
|
|
const visit = (value: unknown) => {
|
|
if (typeof value === "string") {
|
|
const normalized = value.trim();
|
|
if (/^https?:\/\//i.test(normalized)) {
|
|
urls.add(normalized);
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (Array.isArray(value)) {
|
|
value.forEach(visit);
|
|
return;
|
|
}
|
|
|
|
if (value && typeof value === "object") {
|
|
Object.values(value as Record<string, unknown>).forEach(visit);
|
|
}
|
|
};
|
|
|
|
visit(payload);
|
|
return Array.from(urls);
|
|
}
|
|
|
|
async function downloadRemoteAsset(url: string) {
|
|
const controller = new AbortController();
|
|
const timeoutId = setTimeout(() => controller.abort(), 120000);
|
|
|
|
try {
|
|
const response = await fetch(url, {
|
|
cache: "no-store",
|
|
signal: controller.signal,
|
|
});
|
|
if (!response.ok) {
|
|
throw new Error(`下载 Runway 输出失败: ${response.status}`);
|
|
}
|
|
const buffer = Buffer.from(await response.arrayBuffer());
|
|
return {
|
|
buffer,
|
|
contentType: response.headers.get("content-type"),
|
|
};
|
|
} finally {
|
|
clearTimeout(timeoutId);
|
|
}
|
|
}
|
|
|
|
async function uploadVideoToStorage(remoteVideoUrl: string) {
|
|
try {
|
|
const { buffer, contentType } = await downloadRemoteAsset(remoteVideoUrl);
|
|
const extension = detectExtensionFromContentType(contentType || "video/mp4");
|
|
const filename = generateFilename("runway-video", extension === "bin" ? "mp4" : extension);
|
|
const arrayBuffer = buffer.buffer.slice(
|
|
buffer.byteOffset,
|
|
buffer.byteOffset + buffer.byteLength
|
|
) as ArrayBuffer;
|
|
const uploaded = await uploadFile(filename, arrayBuffer);
|
|
return uploaded.url;
|
|
} catch (error) {
|
|
console.warn("[Runway] Failed to archive video output, using temporary URL:", error);
|
|
return remoteVideoUrl;
|
|
}
|
|
}
|
|
|
|
function formatImageMarkdown(images: UploadedImage[], imageSize: string) {
|
|
return images
|
|
.map((image) => ``)
|
|
.join("\n\n");
|
|
}
|
|
|
|
function formatVideoMarkdown(videoUrl: string) {
|
|
return `<video src="${videoUrl}" controls style="max-width: 100%; border-radius: 8px; margin-bottom: 12px;"></video>\n\n[下载视频](${videoUrl})`;
|
|
}
|
|
|
|
function extractPromptAndAssets(messages: Array<{ role: string; content: any }>) {
|
|
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
|
|
if (!lastUserMessage) {
|
|
throw new Error("未找到用户输入内容");
|
|
}
|
|
|
|
let prompt = "";
|
|
const imageUrls: string[] = [];
|
|
|
|
const pushImageUrl = (value?: string) => {
|
|
if (!value) return;
|
|
const normalized = String(value || "").trim();
|
|
if (!normalized) return;
|
|
if (!imageUrls.includes(normalized)) {
|
|
imageUrls.push(normalized);
|
|
}
|
|
};
|
|
|
|
if (typeof lastUserMessage.content === "string") {
|
|
prompt = normalizeText(lastUserMessage.content);
|
|
} else if (Array.isArray(lastUserMessage.content)) {
|
|
for (const part of lastUserMessage.content) {
|
|
if (part?.type === "text" && typeof part.text === "string") {
|
|
prompt = `${prompt} ${part.text}`.trim();
|
|
} else if (part?.type === "image_url" && part.image_url?.url) {
|
|
pushImageUrl(part.image_url.url);
|
|
} else if (part?.type === "file") {
|
|
pushImageUrl(part.url || part.data);
|
|
} else if (part?.type === "image") {
|
|
pushImageUrl(part.image || part.url);
|
|
}
|
|
}
|
|
}
|
|
|
|
return {
|
|
prompt: normalizeText(prompt) || "生成内容",
|
|
imageUrls,
|
|
};
|
|
}
|
|
|
|
async function createTask(baseURL: string, apiKey: string, path: string, body: Record<string, any>) {
|
|
const response = await fetch(joinEndpoint(baseURL, path), {
|
|
method: "POST",
|
|
headers: buildHeaders(apiKey),
|
|
body: JSON.stringify(body),
|
|
});
|
|
|
|
const rawText = await response.text();
|
|
let payload: any = null;
|
|
try {
|
|
payload = rawText ? JSON.parse(rawText) : null;
|
|
} catch {
|
|
payload = rawText;
|
|
}
|
|
|
|
if (!response.ok) {
|
|
const message =
|
|
typeof payload?.error === "string"
|
|
? payload.error
|
|
: typeof payload?.message === "string"
|
|
? payload.message
|
|
: rawText || "Runway 请求失败";
|
|
throw new Error(`${message} (HTTP ${response.status})`);
|
|
}
|
|
|
|
const taskId = String(payload?.id || "").trim();
|
|
if (!taskId) {
|
|
throw new Error("Runway 未返回任务 ID");
|
|
}
|
|
|
|
return taskId;
|
|
}
|
|
|
|
async function waitForTaskOutput(baseURL: string, apiKey: string, taskId: string) {
|
|
const deadline = Date.now() + TASK_TIMEOUT_MS;
|
|
|
|
while (Date.now() < deadline) {
|
|
const response = await fetch(joinEndpoint(baseURL, `/v1/tasks/${taskId}`), {
|
|
method: "GET",
|
|
headers: buildHeaders(apiKey),
|
|
cache: "no-store",
|
|
});
|
|
|
|
const rawText = await response.text();
|
|
let payload: any = null;
|
|
try {
|
|
payload = rawText ? JSON.parse(rawText) : null;
|
|
} catch {
|
|
payload = rawText;
|
|
}
|
|
|
|
if (!response.ok) {
|
|
const message =
|
|
typeof payload?.error === "string"
|
|
? payload.error
|
|
: typeof payload?.message === "string"
|
|
? payload.message
|
|
: rawText || "Runway 轮询失败";
|
|
throw new Error(`${message} (HTTP ${response.status})`);
|
|
}
|
|
|
|
const status = String(payload?.status || "").trim().toUpperCase();
|
|
if (status === "SUCCEEDED") {
|
|
return payload;
|
|
}
|
|
if (status === "FAILED" || status === "CANCELLED") {
|
|
throw new Error(
|
|
String(payload?.failure || payload?.error || payload?.message || "Runway 任务失败")
|
|
);
|
|
}
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, TASK_POLL_INTERVAL_MS));
|
|
}
|
|
|
|
throw new Error("Runway 任务超时，请稍后重试");
|
|
}
|
|
|
|
async function uploadRunwayImages(taskOutput: unknown) {
|
|
const outputUrls = collectHttpUrls(taskOutput);
|
|
const uploadedImages: UploadedImage[] = [];
|
|
|
|
for (const url of outputUrls) {
|
|
try {
|
|
const { buffer, contentType } = await downloadRemoteAsset(url);
|
|
if (!String(contentType || "").startsWith("image/")) {
|
|
continue;
|
|
}
|
|
const extension = detectExtensionFromContentType(contentType);
|
|
uploadedImages.push(
|
|
await uploadGeneratedBuffer(
|
|
buffer,
|
|
extension === "bin" ? "png" : extension,
|
|
contentType || "image/png"
|
|
)
|
|
);
|
|
} catch (error) {
|
|
console.warn("[Runway] Failed to archive image output:", error);
|
|
}
|
|
}
|
|
|
|
if (uploadedImages.length === 0) {
|
|
throw new Error("Runway 未返回可用图片");
|
|
}
|
|
|
|
return uploadedImages;
|
|
}
|
|
|
|
async function resolveRunwayVideoUrl(taskOutput: unknown) {
|
|
const outputUrls = collectHttpUrls(taskOutput);
|
|
if (outputUrls.length === 0) {
|
|
throw new Error("Runway 未返回可用视频地址");
|
|
}
|
|
|
|
for (const url of outputUrls) {
|
|
if (/(\.mp4|\.mov|\.webm)(\?|$)/i.test(url) || /video/i.test(url)) {
|
|
return uploadVideoToStorage(url);
|
|
}
|
|
}
|
|
|
|
return uploadVideoToStorage(outputUrls[0]!);
|
|
}
|
|
|
|
function getRunwayVideoEndpoint(modelId: string, hasPromptImage: boolean) {
|
|
const normalized = normalizeText(modelId);
|
|
if (hasPromptImage) return "/v1/image_to_video";
|
|
if (normalized === "gen4.5") return "/v1/image_to_video";
|
|
return "/v1/text_to_video";
|
|
}
|
|
|
|
function buildReferenceImages(imageUrls: string[]) {
|
|
return imageUrls.slice(0, 3).map((uri, index) => ({
|
|
uri,
|
|
tag: `ref_${index + 1}`.replace(/[^a-z0-9_]/g, "_"),
|
|
}));
|
|
}
|
|
|
|
export function createRunway({ apiKey, baseURL }: RunwayConfig) {
|
|
return {
|
|
provider: "runway",
|
|
chat: (modelId: string) => ({
|
|
provider: "runway",
|
|
doGenerate: async (options: {
|
|
messages: Array<{ role: string; content: any }>;
|
|
imageParams?: RunwayImageParams;
|
|
videoParams?: RunwayVideoParams;
|
|
overrideModelId?: string;
|
|
}) => {
|
|
try {
|
|
const activeModelId = normalizeText(options.overrideModelId || modelId);
|
|
const { prompt, imageUrls } = extractPromptAndAssets(options.messages);
|
|
const normalizedImageUrls = (
|
|
await Promise.all(imageUrls.map((url) => normalizeAssetUrl(url)))
|
|
).filter(Boolean);
|
|
const modelKind =
|
|
inferRunwayModelKind(activeModelId) ||
|
|
(options.videoParams ? "video" : options.imageParams ? "image" : null) ||
|
|
"video";
|
|
|
|
if (modelKind === "image") {
|
|
const imageSize = normalizeImageSizeLabel(options.imageParams?.imageSize);
|
|
const taskId = await createTask(baseURL, apiKey, "/v1/text_to_image", {
|
|
model: activeModelId,
|
|
promptText: prompt,
|
|
ratio: normalizeImageResolution(options.imageParams?.aspectRatio, imageSize),
|
|
...(normalizedImageUrls.length > 0
|
|
? { referenceImages: buildReferenceImages(normalizedImageUrls) }
|
|
: {}),
|
|
...(normalizeSeed(options.imageParams?.seed) !== undefined
|
|
? { seed: normalizeSeed(options.imageParams?.seed) }
|
|
: {}),
|
|
});
|
|
|
|
const task = await waitForTaskOutput(baseURL, apiKey, taskId);
|
|
const uploadedImages = await uploadRunwayImages(task?.output ?? task);
|
|
const text = formatImageMarkdown(uploadedImages, imageSize);
|
|
|
|
return {
|
|
text,
|
|
finishReason: "stop" as const,
|
|
usage: {
|
|
promptTokens: 0,
|
|
completionTokens: text.length,
|
|
},
|
|
};
|
|
}
|
|
|
|
const videoMode = options.videoParams?.mode || (normalizedImageUrls[0] ? "first-frame" : "text-to-video");
|
|
if (
|
|
(videoMode === "first-frame" || videoMode === "first-last-frame" || videoMode === "reference-images") &&
|
|
!normalizedImageUrls[0]
|
|
) {
|
|
throw new Error("Runway 图生视频至少需要 1 张参考图");
|
|
}
|
|
|
|
const endpoint = getRunwayVideoEndpoint(activeModelId, Boolean(normalizedImageUrls[0]));
|
|
const taskId = await createTask(baseURL, apiKey, endpoint, {
|
|
model: activeModelId,
|
|
promptText: prompt,
|
|
ratio: normalizeVideoRatio(options.videoParams?.ratio),
|
|
duration: normalizeVideoDuration(options.videoParams?.duration),
|
|
...(normalizedImageUrls[0] ? { promptImage: normalizedImageUrls[0] } : {}),
|
|
...(normalizeSeed(options.videoParams?.seed) !== undefined
|
|
? { seed: normalizeSeed(options.videoParams?.seed) }
|
|
: {}),
|
|
});
|
|
|
|
const task = await waitForTaskOutput(baseURL, apiKey, taskId);
|
|
const videoUrl = await resolveRunwayVideoUrl(task?.output ?? task);
|
|
const text = formatVideoMarkdown(videoUrl);
|
|
|
|
return {
|
|
text,
|
|
finishReason: "stop" as const,
|
|
usage: {
|
|
promptTokens: 0,
|
|
completionTokens: text.length,
|
|
},
|
|
};
|
|
} catch (error) {
|
|
console.error("[Runway] Generation failed:", error);
|
|
return {
|
|
text:
|
|
error instanceof Error
|
|
? `${
|
|
(inferRunwayModelKind(options.overrideModelId || modelId) ||
|
|
(options.videoParams ? "video" : options.imageParams ? "image" : "video")) === "image"
|
|
? "图片"
|
|
: "视频"
|
|
}生成失败: ${error.message}`
|
|
: "Runway 生成失败，请稍后重试",
|
|
finishReason: "error" as const,
|
|
usage: {
|
|
promptTokens: 0,
|
|
completionTokens: 0,
|
|
},
|
|
};
|
|
}
|
|
},
|
|
doStream: async function* (options: {
|
|
messages: Array<{ role: string; content: any }>;
|
|
imageParams?: RunwayImageParams;
|
|
videoParams?: RunwayVideoParams;
|
|
overrideModelId?: string;
|
|
}) {
|
|
const result = await this.doGenerate(options);
|
|
yield {
|
|
type: "text-delta",
|
|
textDelta: result.text,
|
|
};
|
|
},
|
|
modelId,
|
|
specificationVersion: "v1" as const,
|
|
defaultObjectGenerationMode: "json" as const,
|
|
}),
|
|
};
|
|
}
|