import "server-only"; import sharp from "sharp"; import { getRunwayVideoModeDefinition, isRunwayAudioSupported, isRunwaySeedSupported, normalizeRunwayDuration, resolveRunwayApiRatio, } from "@/lib/ai/runway-video-models"; import { uploadFile } from "@/lib/storage/service"; import { inferRunwayModelKind } from "@/lib/ai/model-classification"; interface RunwayConfig { apiKey: string; baseURL: string; } interface RunwayImageParams { aspectRatio?: string; imageSize?: string; imageCount?: number; seed?: number; } interface RunwayVideoParams { mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images"; ratio?: string; duration?: number; seed?: number; generateAudio?: boolean; } interface UploadedImage { thumbnail: string; original: string; } const RUNWAY_VERSION = "2024-11-06"; const DEFAULT_BASE_URL = "https://api.dev.runwayml.com"; const TASK_POLL_INTERVAL_MS = 5000; const TASK_TIMEOUT_MS = 15 * 60 * 1000; const IMAGE_RESOLUTION_MAP: Record> = { "1K": { "21:9": "1584:672", "16:9": "1360:768", "9:16": "768:1360", "4:3": "1104:832", "3:4": "832:1104", "1:1": "960:960", }, "2K": { "21:9": "1584:672", "16:9": "1360:768", "9:16": "768:1360", "4:3": "1104:832", "3:4": "832:1104", "1:1": "960:960", }, "4K": { "21:9": "1584:672", "16:9": "1360:768", "9:16": "768:1360", "4:3": "1104:832", "3:4": "832:1104", "1:1": "960:960", }, }; function normalizeBaseURL(baseURL: string) { const normalized = String(baseURL || "").trim().replace(/\/+$/, ""); return normalized || DEFAULT_BASE_URL; } function joinEndpoint(baseURL: string, path: string) { const cleanPath = path.startsWith("/") ? path : `/${path}`; return `${normalizeBaseURL(baseURL)}${cleanPath}`; } function buildHeaders(apiKey: string) { const normalizedApiKey = normalizeRunwayCredentials(apiKey); return { Authorization: `Bearer ${normalizedApiKey}`, "Content-Type": "application/json", "X-Runway-Version": RUNWAY_VERSION, }; } function normalizeText(value: string) { return String(value || "").replace(/\u00a0/g, " ").trim().normalize("NFKC"); } function normalizeRunwayCredentials(apiKey: string) { const normalized = normalizeText(apiKey); const stripped = normalized.replace(/^bearer\s+/i, "").replace(/^key\s+/i, ""); const kvMatch = stripped.match(/^(?:runway[_-]?api[_-]?key|api[_-]?key|token)\s*[::]\s*(.+)$/i); const candidate = kvMatch?.[1] ? kvMatch[1] : stripped; return candidate.trim().replace(/[^\x20-\x7E]/g, ""); } function normalizeImageSizeLabel(imageSize?: string): "1K" | "2K" | "4K" { const normalized = String(imageSize || "").trim().toUpperCase(); if (normalized === "1K" || normalized === "2K" || normalized === "4K") { return normalized; } return "2K"; } function normalizeImageResolution(aspectRatio?: string, imageSize?: string) { const normalizedRatio = String(aspectRatio || "").trim() || "16:9"; const normalizedSize = normalizeImageSizeLabel(imageSize); return ( IMAGE_RESOLUTION_MAP[normalizedSize]?.[normalizedRatio] || IMAGE_RESOLUTION_MAP[normalizedSize]?.["16:9"] || "1360:768" ); } function normalizeSeed(seed?: number) { if (!Number.isFinite(seed)) return undefined; return Math.max(0, Math.min(4294967295, Math.round(Number(seed)))); } function detectExtensionFromContentType(contentType?: string | null) { const normalized = String(contentType || "").toLowerCase(); if (normalized.includes("jpeg") || normalized.includes("jpg")) return "jpg"; if (normalized.includes("webp")) return "webp"; if (normalized.includes("gif")) return "gif"; if (normalized.includes("png")) return "png"; if (normalized.includes("mp4")) return "mp4"; if (normalized.includes("quicktime")) return "mov"; if (normalized.includes("webm")) return "webm"; return "bin"; } function extractDataUriParts(dataUri: string) { const match = dataUri.match(/^data:([^;]+);base64,(.+)$/); if (!match) { throw new Error("不支持的 Data URI 格式"); } return { mimeType: match[1], buffer: Buffer.from(match[2], "base64"), }; } function generateFilename(prefix: string, extension: string) { const timestamp = Date.now(); const random = Math.random().toString(36).slice(2, 8); return `${prefix}-${timestamp}-${random}.${extension}`; } async function uploadGeneratedBuffer( buffer: Buffer, extension: string, mimeType: string ): Promise { const thumbnailBuffer = await sharp(buffer) .resize(400, 400, { fit: "inside", withoutEnlargement: true }) .jpeg({ quality: 80 }) .toBuffer(); const originalFilename = generateFilename("runway-image", extension); const originalArrayBuffer = buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer); const thumbnailFilename = generateFilename("runway-thumb", "jpg"); const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice( thumbnailBuffer.byteOffset, thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength ) as ArrayBuffer; const thumbnailUploaded = await uploadFile(thumbnailFilename, thumbnailArrayBuffer); console.log("[Runway] Uploaded image to storage:", { original: originalUploaded.url, thumbnail: thumbnailUploaded.url, mimeType, }); return { thumbnail: thumbnailUploaded.url, original: originalUploaded.url, }; } async function uploadDataUriAsset(dataUri: string, prefix: "runway-input" | "runway-video") { const { mimeType, buffer } = extractDataUriParts(dataUri); const extension = detectExtensionFromContentType(mimeType); const filename = generateFilename(prefix, extension); const arrayBuffer = buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; const uploaded = await uploadFile(filename, arrayBuffer); return uploaded.url; } async function normalizeAssetUrl(url?: string | null) { const normalized = String(url || "").trim(); if (!normalized) return ""; if (/^https:\/\//i.test(normalized) || /^runway:\/\//i.test(normalized)) { return normalized; } if (/^data:(image|video)\//i.test(normalized)) { return uploadDataUriAsset(normalized, /^data:video\//i.test(normalized) ? "runway-video" : "runway-input"); } return normalized; } function collectHttpUrls(payload: unknown): string[] { const urls = new Set(); const visit = (value: unknown) => { if (typeof value === "string") { const normalized = value.trim(); if (/^https?:\/\//i.test(normalized)) { urls.add(normalized); } return; } if (Array.isArray(value)) { value.forEach(visit); return; } if (value && typeof value === "object") { Object.values(value as Record).forEach(visit); } }; visit(payload); return Array.from(urls); } async function downloadRemoteAsset(url: string) { const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 120000); try { const response = await fetch(url, { cache: "no-store", signal: controller.signal, }); if (!response.ok) { throw new Error(`下载 Runway 输出失败: ${response.status}`); } const buffer = Buffer.from(await response.arrayBuffer()); return { buffer, contentType: response.headers.get("content-type"), }; } finally { clearTimeout(timeoutId); } } async function uploadVideoToStorage(remoteVideoUrl: string) { try { const { buffer, contentType } = await downloadRemoteAsset(remoteVideoUrl); const extension = detectExtensionFromContentType(contentType || "video/mp4"); const filename = generateFilename("runway-video", extension === "bin" ? "mp4" : extension); const arrayBuffer = buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; const uploaded = await uploadFile(filename, arrayBuffer); return uploaded.url; } catch (error) { console.warn("[Runway] Failed to archive video output, using temporary URL:", error); return remoteVideoUrl; } } function formatImageMarkdown(images: UploadedImage[], imageSize: string) { return images .map((image) => `![thumb|||${image.original}|||${imageSize}](${image.thumbnail})`) .join("\n\n"); } function formatVideoMarkdown(videoUrl: string) { return `\n\n[下载视频](${videoUrl})`; } function extractPromptAndAssets(messages: Array<{ role: string; content: any }>) { const lastUserMessage = messages.filter((message) => message.role === "user").pop(); if (!lastUserMessage) { throw new Error("未找到用户输入内容"); } let prompt = ""; const imageUrls: string[] = []; const pushImageUrl = (value?: string) => { if (!value) return; const normalized = String(value || "").trim(); if (!normalized) return; if (!imageUrls.includes(normalized)) { imageUrls.push(normalized); } }; if (typeof lastUserMessage.content === "string") { prompt = normalizeText(lastUserMessage.content); } else if (Array.isArray(lastUserMessage.content)) { for (const part of lastUserMessage.content) { if (part?.type === "text" && typeof part.text === "string") { prompt = `${prompt} ${part.text}`.trim(); } else if (part?.type === "image_url" && part.image_url?.url) { pushImageUrl(part.image_url.url); } else if (part?.type === "file") { pushImageUrl(part.url || part.data); } else if (part?.type === "image") { pushImageUrl(part.image || part.url); } } } return { prompt: normalizeText(prompt) || "生成内容", imageUrls, }; } async function createTask(baseURL: string, apiKey: string, path: string, body: Record) { const response = await fetch(joinEndpoint(baseURL, path), { method: "POST", headers: buildHeaders(apiKey), body: JSON.stringify(body), }); const rawText = await response.text(); let payload: any = null; try { payload = rawText ? JSON.parse(rawText) : null; } catch { payload = rawText; } if (!response.ok) { console.error("[Runway] Task request failed:", { path, status: response.status, bodySummary: { model: body.model, ratio: body.ratio, duration: body.duration, hasPromptImage: Boolean(body.promptImage), promptImageCount: Array.isArray(body.promptImage) ? body.promptImage.length : body.promptImage ? 1 : 0, hasReferenceImages: Array.isArray(body.referenceImages) ? body.referenceImages.length : 0, hasPromptText: typeof body.promptText === "string" && body.promptText.length > 0, promptTextLength: typeof body.promptText === "string" ? body.promptText.length : 0, audio: body.audio, seed: typeof body.seed === "number" ? body.seed : undefined, }, response: payload, }); const message = typeof payload?.error === "string" ? payload.error : typeof payload?.message === "string" ? payload.message : rawText || "Runway 请求失败"; throw new Error(`${message} (HTTP ${response.status})`); } const taskId = String(payload?.id || "").trim(); if (!taskId) { throw new Error("Runway 未返回任务 ID"); } return taskId; } async function waitForTaskOutput(baseURL: string, apiKey: string, taskId: string) { const deadline = Date.now() + TASK_TIMEOUT_MS; while (Date.now() < deadline) { const response = await fetch(joinEndpoint(baseURL, `/v1/tasks/${taskId}`), { method: "GET", headers: buildHeaders(apiKey), cache: "no-store", }); const rawText = await response.text(); let payload: any = null; try { payload = rawText ? JSON.parse(rawText) : null; } catch { payload = rawText; } if (!response.ok) { const message = typeof payload?.error === "string" ? payload.error : typeof payload?.message === "string" ? payload.message : rawText || "Runway 轮询失败"; throw new Error(`${message} (HTTP ${response.status})`); } const status = String(payload?.status || "").trim().toUpperCase(); if (status === "SUCCEEDED") { return payload; } if (status === "FAILED" || status === "CANCELLED") { throw new Error( String(payload?.failure || payload?.error || payload?.message || "Runway 任务失败") ); } await new Promise((resolve) => setTimeout(resolve, TASK_POLL_INTERVAL_MS)); } throw new Error("Runway 任务超时,请稍后重试"); } async function uploadRunwayImages(taskOutput: unknown) { const outputUrls = collectHttpUrls(taskOutput); const uploadedImages: UploadedImage[] = []; for (const url of outputUrls) { try { const { buffer, contentType } = await downloadRemoteAsset(url); if (!String(contentType || "").startsWith("image/")) { continue; } const extension = detectExtensionFromContentType(contentType); uploadedImages.push( await uploadGeneratedBuffer( buffer, extension === "bin" ? "png" : extension, contentType || "image/png" ) ); } catch (error) { console.warn("[Runway] Failed to archive image output:", error); } } if (uploadedImages.length === 0) { throw new Error("Runway 未返回可用图片"); } return uploadedImages; } async function resolveRunwayVideoUrl(taskOutput: unknown) { const outputUrls = collectHttpUrls(taskOutput); if (outputUrls.length === 0) { throw new Error("Runway 未返回可用视频地址"); } for (const url of outputUrls) { if (/(\.mp4|\.mov|\.webm)(\?|$)/i.test(url) || /video/i.test(url)) { return uploadVideoToStorage(url); } } return uploadVideoToStorage(outputUrls[0]!); } function getRunwayVideoEndpoint(modelId: string, hasPromptImage: boolean) { const normalized = normalizeText(modelId); if (hasPromptImage) return "/v1/image_to_video"; if (normalized === "gen4.5") return "/v1/image_to_video"; return "/v1/text_to_video"; } function normalizeVideoMode( mode: RunwayVideoParams["mode"] | undefined, hasPromptImage: boolean ): NonNullable { if ( mode === "text-to-video" || mode === "first-frame" || mode === "first-last-frame" || mode === "reference-images" ) { return mode; } return hasPromptImage ? "first-frame" : "text-to-video"; } function buildRunwayPromptImagePayload(params: { modelId: string; mode: NonNullable; imageUrls: string[]; }) { const resolved = getRunwayVideoModeDefinition(params.modelId, params.mode); if (!resolved || params.imageUrls.length === 0) { return undefined; } const limitedImageUrls = params.imageUrls.slice(0, Math.max(1, resolved.definition.imageMax)); if (params.mode === "first-last-frame" && limitedImageUrls.length >= 2) { return [ { uri: limitedImageUrls[0], position: "first" as const }, { uri: limitedImageUrls[1], position: "last" as const }, ]; } return limitedImageUrls[0]; } function buildReferenceImages(imageUrls: string[]) { return imageUrls.slice(0, 3).map((uri, index) => ({ uri, tag: `ref_${index + 1}`.replace(/[^a-z0-9_]/g, "_"), })); } export function createRunway({ apiKey, baseURL }: RunwayConfig) { return { provider: "runway", chat: (modelId: string) => ({ provider: "runway", doGenerate: async (options: { messages: Array<{ role: string; content: any }>; imageParams?: RunwayImageParams; videoParams?: RunwayVideoParams; overrideModelId?: string; }) => { try { const activeModelId = normalizeText(options.overrideModelId || modelId); const { prompt, imageUrls } = extractPromptAndAssets(options.messages); const normalizedImageUrls = ( await Promise.all(imageUrls.map((url) => normalizeAssetUrl(url))) ).filter(Boolean); const modelKind = inferRunwayModelKind(activeModelId) || (options.videoParams ? "video" : options.imageParams ? "image" : null) || "video"; if (modelKind === "image") { const imageSize = normalizeImageSizeLabel(options.imageParams?.imageSize); const taskId = await createTask(baseURL, apiKey, "/v1/text_to_image", { model: activeModelId, promptText: prompt, ratio: normalizeImageResolution(options.imageParams?.aspectRatio, imageSize), ...(normalizedImageUrls.length > 0 ? { referenceImages: buildReferenceImages(normalizedImageUrls) } : {}), ...(normalizeSeed(options.imageParams?.seed) !== undefined ? { seed: normalizeSeed(options.imageParams?.seed) } : {}), }); const task = await waitForTaskOutput(baseURL, apiKey, taskId); const uploadedImages = await uploadRunwayImages(task?.output ?? task); const text = formatImageMarkdown(uploadedImages, imageSize); return { text, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: text.length, }, }; } const videoMode = normalizeVideoMode( options.videoParams?.mode, Boolean(normalizedImageUrls[0]) ); if ( (videoMode === "first-frame" || videoMode === "first-last-frame" || videoMode === "reference-images") && !normalizedImageUrls[0] ) { throw new Error("Runway 图生视频至少需要 1 张参考图"); } const promptImage = buildRunwayPromptImagePayload({ modelId: activeModelId, mode: videoMode, imageUrls: normalizedImageUrls, }); const endpoint = getRunwayVideoEndpoint(activeModelId, Boolean(promptImage)); const ratio = resolveRunwayApiRatio({ modelId: activeModelId, mode: videoMode, ratio: options.videoParams?.ratio, }); const duration = normalizeRunwayDuration({ modelId: activeModelId, mode: videoMode, duration: options.videoParams?.duration, }); const seed = isRunwaySeedSupported(activeModelId, videoMode) ? normalizeSeed(options.videoParams?.seed) : undefined; const audio = isRunwayAudioSupported(activeModelId, videoMode) && typeof options.videoParams?.generateAudio === "boolean" ? options.videoParams.generateAudio : undefined; console.log("[Runway] Video request config:", { model: activeModelId, mode: videoMode, endpoint, ratio, duration, hasPromptImage: Boolean(promptImage), promptImageCount: Array.isArray(promptImage) ? promptImage.length : promptImage ? 1 : 0, audio, seedEnabled: seed !== undefined, }); const taskId = await createTask(baseURL, apiKey, endpoint, { model: activeModelId, promptText: prompt, ratio, duration, ...(promptImage ? { promptImage } : {}), ...(seed !== undefined ? { seed } : {}), ...(typeof audio === "boolean" ? { audio } : {}), }); const task = await waitForTaskOutput(baseURL, apiKey, taskId); const videoUrl = await resolveRunwayVideoUrl(task?.output ?? task); const text = formatVideoMarkdown(videoUrl); return { text, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: text.length, }, }; } catch (error) { console.error("[Runway] Generation failed:", error); return { text: error instanceof Error ? `${ (inferRunwayModelKind(options.overrideModelId || modelId) || (options.videoParams ? "video" : options.imageParams ? "image" : "video")) === "image" ? "图片" : "视频" }生成失败: ${error.message}` : "Runway 生成失败,请稍后重试", finishReason: "error" as const, usage: { promptTokens: 0, completionTokens: 0, }, }; } }, doStream: async function* (options: { messages: Array<{ role: string; content: any }>; imageParams?: RunwayImageParams; videoParams?: RunwayVideoParams; overrideModelId?: string; }) { const result = await this.doGenerate(options); yield { type: "text-delta", textDelta: result.text, }; }, modelId, specificationVersion: "v1" as const, defaultObjectGenerationMode: "json" as const, }), }; }