531 lines
16 KiB
TypeScript
531 lines
16 KiB
TypeScript
import "server-only";
|
||
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
import { normalizeBltcyVideoBaseUrl } from "@/lib/ai/bltcy-video-models";
|
||
|
||
interface BltcyVideoConfig {
|
||
apiKey: string;
|
||
baseURL: string;
|
||
}
|
||
|
||
interface BltcyVideoParams {
|
||
mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
|
||
ratio?: string;
|
||
duration?: number;
|
||
resolution?: string;
|
||
}
|
||
|
||
const POLL_INTERVAL_MS = 3000;
|
||
const POLL_MAX_ATTEMPTS = 240;
|
||
const DEFAULT_RATIO = "16:9";
|
||
const DEFAULT_DURATION = 5;
|
||
|
||
function normalizeText(value?: string | null) {
|
||
return String(value || "")
|
||
.replace(/\u00a0/g, " ")
|
||
.trim()
|
||
.normalize("NFKC");
|
||
}
|
||
|
||
function normalizeApiKey(value: string) {
|
||
const normalized = normalizeText(value)
|
||
.replace(/^bearer\s+/i, "")
|
||
.replace(/^key\s+/i, "")
|
||
.replace(/^(?:bltcy[_-]?api[_-]?key|api[_-]?key|token)\s*[:：]\s*/i, "");
|
||
return normalized.replace(/[^\x20-\x7E]/g, "");
|
||
}
|
||
|
||
function joinApiPath(baseURL: string, path: string) {
|
||
const cleanBase = normalizeBltcyVideoBaseUrl(baseURL).replace(/\/+$/, "");
|
||
const cleanPath = path.startsWith("/") ? path : `/${path}`;
|
||
return `${cleanBase}${cleanPath}`;
|
||
}
|
||
|
||
function normalizeRatio(value?: string) {
|
||
const normalized = normalizeText(value);
|
||
if (
|
||
normalized === "21:9" ||
|
||
normalized === "16:9" ||
|
||
normalized === "4:3" ||
|
||
normalized === "1:1" ||
|
||
normalized === "3:4" ||
|
||
normalized === "9:16"
|
||
) {
|
||
return normalized;
|
||
}
|
||
return DEFAULT_RATIO;
|
||
}
|
||
|
||
function normalizeDuration(value?: number) {
|
||
if (!Number.isFinite(value)) return DEFAULT_DURATION;
|
||
return Math.max(3, Math.min(20, Math.round(Number(value))));
|
||
}
|
||
|
||
function normalizeResolution(value?: string) {
|
||
const normalized = normalizeText(value).toLowerCase();
|
||
if (normalized === "480p") return "480P";
|
||
if (normalized === "720p") return "720P";
|
||
if (normalized === "1080p") return "1080P";
|
||
return undefined;
|
||
}
|
||
|
||
function detectExtensionFromContentType(contentType?: string | null) {
|
||
const normalized = String(contentType || "").toLowerCase();
|
||
if (normalized.includes("jpeg") || normalized.includes("jpg")) return "jpg";
|
||
if (normalized.includes("webp")) return "webp";
|
||
if (normalized.includes("gif")) return "gif";
|
||
if (normalized.includes("png")) return "png";
|
||
if (normalized.includes("mp4")) return "mp4";
|
||
if (normalized.includes("quicktime")) return "mov";
|
||
if (normalized.includes("webm")) return "webm";
|
||
return "bin";
|
||
}
|
||
|
||
function generateFilename(prefix: string, extension: string) {
|
||
const timestamp = Date.now();
|
||
const random = Math.random().toString(36).slice(2, 8);
|
||
return `${prefix}-${timestamp}-${random}.${extension}`;
|
||
}
|
||
|
||
function extractDataUriParts(dataUri: string) {
|
||
const match = dataUri.match(/^data:([^;]+);base64,(.+)$/);
|
||
if (!match) {
|
||
throw new Error("不支持的 Data URI 格式");
|
||
}
|
||
return {
|
||
mimeType: match[1],
|
||
buffer: Buffer.from(match[2], "base64"),
|
||
};
|
||
}
|
||
|
||
async function uploadDataUriAsset(dataUri: string, prefix: "bltcy-input" | "bltcy-video") {
|
||
const { mimeType, buffer } = extractDataUriParts(dataUri);
|
||
const extension = detectExtensionFromContentType(mimeType);
|
||
const arrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
const uploaded = await uploadFile(
|
||
generateFilename(prefix, extension === "bin" ? (prefix === "bltcy-video" ? "mp4" : "png") : extension),
|
||
arrayBuffer
|
||
);
|
||
return uploaded.url;
|
||
}
|
||
|
||
function isLikelyVideoUrl(url: string, mediaType?: string | null) {
|
||
const normalizedMediaType = String(mediaType || "").toLowerCase();
|
||
if (normalizedMediaType.startsWith("video/")) return true;
|
||
return /\.(mp4|mov|webm|m4v)(\?|#|$)/i.test(url) || /^data:video\//i.test(url);
|
||
}
|
||
|
||
async function normalizeAssetUrl(url?: string | null, kind?: "image" | "video") {
|
||
const normalized = normalizeText(url);
|
||
if (!normalized) return "";
|
||
if (/^https?:\/\//i.test(normalized)) return normalized;
|
||
if (/^data:image\//i.test(normalized)) {
|
||
return uploadDataUriAsset(normalized, "bltcy-input");
|
||
}
|
||
if (/^data:video\//i.test(normalized)) {
|
||
return uploadDataUriAsset(normalized, "bltcy-video");
|
||
}
|
||
if (kind === "video") {
|
||
throw new Error("柏拉图统一视频的参考视频需要可访问的 URL");
|
||
}
|
||
return normalized;
|
||
}
|
||
|
||
function extractPromptAndAssets(messages: Array<{ role: string; content: any }>) {
|
||
const lastUserMessage = messages.filter((message) => message.role === "user").pop();
|
||
if (!lastUserMessage) {
|
||
throw new Error("未找到用户输入内容");
|
||
}
|
||
|
||
let prompt = "";
|
||
const imageUrls: string[] = [];
|
||
const videoUrls: string[] = [];
|
||
|
||
const pushAsset = (value?: string, mediaType?: string | null) => {
|
||
if (!value) return;
|
||
const normalized = String(value || "").trim();
|
||
if (!normalized) return;
|
||
if (isLikelyVideoUrl(normalized, mediaType)) {
|
||
if (!videoUrls.includes(normalized)) {
|
||
videoUrls.push(normalized);
|
||
}
|
||
return;
|
||
}
|
||
if (!imageUrls.includes(normalized)) {
|
||
imageUrls.push(normalized);
|
||
}
|
||
};
|
||
|
||
const readPart = (part: any) => {
|
||
if (!part) return;
|
||
if (part.type === "text" && typeof part.text === "string") {
|
||
prompt = `${prompt} ${part.text}`.trim();
|
||
return;
|
||
}
|
||
if (part.type === "image_url" && part.image_url?.url) {
|
||
pushAsset(part.image_url.url, "image/jpeg");
|
||
return;
|
||
}
|
||
if (part.type === "file") {
|
||
pushAsset(part.url || part.data, part.mediaType || part.mimeType);
|
||
return;
|
||
}
|
||
if (part.type === "image") {
|
||
pushAsset(part.image || part.url || part.data, part.mediaType || "image/jpeg");
|
||
return;
|
||
}
|
||
if (part.type === "video") {
|
||
pushAsset(part.video || part.url || part.data, part.mediaType || "video/mp4");
|
||
}
|
||
};
|
||
|
||
if (typeof lastUserMessage.content === "string") {
|
||
prompt = normalizeText(lastUserMessage.content);
|
||
} else if (Array.isArray(lastUserMessage.content)) {
|
||
for (const part of lastUserMessage.content) {
|
||
readPart(part);
|
||
}
|
||
}
|
||
|
||
return {
|
||
prompt: normalizeText(prompt) || "生成视频",
|
||
imageUrls,
|
||
videoUrls,
|
||
};
|
||
}
|
||
|
||
function extractTaskId(payload: any): string | undefined {
|
||
const candidates = [
|
||
payload?.task_id,
|
||
payload?.taskId,
|
||
payload?.id,
|
||
payload?.data?.task_id,
|
||
payload?.data?.taskId,
|
||
payload?.data?.id,
|
||
];
|
||
return candidates.find((value) => typeof value === "string" && value.trim())?.trim();
|
||
}
|
||
|
||
function extractStatus(payload: any): string | undefined {
|
||
const raw =
|
||
payload?.status ||
|
||
payload?.task_status ||
|
||
payload?.taskStatus ||
|
||
payload?.state ||
|
||
payload?.data?.status ||
|
||
payload?.data?.task_status ||
|
||
payload?.data?.taskStatus ||
|
||
payload?.data?.state;
|
||
if (typeof raw !== "string") return undefined;
|
||
return raw.trim().toLowerCase();
|
||
}
|
||
|
||
function extractErrorMessage(payload: any): string {
|
||
return (
|
||
payload?.fail_reason ||
|
||
payload?.message ||
|
||
payload?.error ||
|
||
payload?.data?.fail_reason ||
|
||
payload?.data?.message ||
|
||
payload?.data?.error ||
|
||
"柏拉图视频任务执行失败"
|
||
);
|
||
}
|
||
|
||
function collectHttpUrls(payload: unknown): string[] {
|
||
const urls = new Set<string>();
|
||
const visit = (value: unknown) => {
|
||
if (typeof value === "string") {
|
||
const normalized = value.trim();
|
||
if (/^https?:\/\//i.test(normalized)) {
|
||
urls.add(normalized);
|
||
}
|
||
return;
|
||
}
|
||
if (Array.isArray(value)) {
|
||
value.forEach(visit);
|
||
return;
|
||
}
|
||
if (value && typeof value === "object") {
|
||
Object.values(value as Record<string, unknown>).forEach(visit);
|
||
}
|
||
};
|
||
visit(payload);
|
||
return Array.from(urls);
|
||
}
|
||
|
||
async function readJsonOrText(response: Response) {
|
||
const rawText = await response.text();
|
||
if (!rawText) return null;
|
||
try {
|
||
return JSON.parse(rawText);
|
||
} catch {
|
||
return rawText;
|
||
}
|
||
}
|
||
|
||
async function downloadRemoteAsset(url: string) {
|
||
const controller = new AbortController();
|
||
const timeoutId = setTimeout(() => controller.abort(), 120000);
|
||
|
||
try {
|
||
const response = await fetch(url, {
|
||
cache: "no-store",
|
||
signal: controller.signal,
|
||
});
|
||
if (!response.ok) {
|
||
throw new Error(`下载柏拉图视频输出失败: ${response.status}`);
|
||
}
|
||
const buffer = Buffer.from(await response.arrayBuffer());
|
||
return {
|
||
buffer,
|
||
contentType: response.headers.get("content-type"),
|
||
};
|
||
} finally {
|
||
clearTimeout(timeoutId);
|
||
}
|
||
}
|
||
|
||
async function uploadVideoToStorage(remoteVideoUrl: string) {
|
||
try {
|
||
const { buffer, contentType } = await downloadRemoteAsset(remoteVideoUrl);
|
||
const extension = detectExtensionFromContentType(contentType || "video/mp4");
|
||
const arrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
const uploaded = await uploadFile(
|
||
generateFilename("bltcy-video", extension === "bin" ? "mp4" : extension),
|
||
arrayBuffer
|
||
);
|
||
return uploaded.url;
|
||
} catch (error) {
|
||
console.warn("[BLTCY Video] Failed to archive video output, using temporary URL:", error);
|
||
return remoteVideoUrl;
|
||
}
|
||
}
|
||
|
||
function formatVideoMarkdown(videoUrl: string) {
|
||
return `<video src="${videoUrl}" controls style="max-width: 100%; border-radius: 8px; margin-bottom: 12px;"></video>\n\n[下载视频](${videoUrl})`;
|
||
}
|
||
|
||
async function waitForVideoUrl(baseURL: string, apiKey: string, taskId: string) {
|
||
const endpoint = joinApiPath(baseURL, `/v2/videos/generations/${encodeURIComponent(taskId)}`);
|
||
|
||
for (let attempt = 1; attempt <= POLL_MAX_ATTEMPTS; attempt++) {
|
||
const response = await fetch(endpoint, {
|
||
headers: {
|
||
Authorization: `Bearer ${apiKey}`,
|
||
Accept: "application/json",
|
||
},
|
||
cache: "no-store",
|
||
});
|
||
|
||
const payload = await readJsonOrText(response);
|
||
|
||
if (!response.ok) {
|
||
throw new Error(
|
||
`${extractErrorMessage(payload)} (HTTP ${response.status})`
|
||
);
|
||
}
|
||
|
||
const status = extractStatus(payload);
|
||
const urls = collectHttpUrls(payload);
|
||
const remoteVideoUrl =
|
||
urls.find((url) => /\.(mp4|mov|webm|m4v)(\?|#|$)/i.test(url) || /video/i.test(url)) ||
|
||
urls[0];
|
||
|
||
if (remoteVideoUrl && (!status || ["success", "succeeded", "completed", "done", "finished"].includes(status))) {
|
||
return uploadVideoToStorage(remoteVideoUrl);
|
||
}
|
||
|
||
if (status && ["failed", "error", "cancelled", "canceled"].includes(status)) {
|
||
throw new Error(extractErrorMessage(payload));
|
||
}
|
||
|
||
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
|
||
}
|
||
|
||
throw new Error("柏拉图视频任务超时，请稍后重试");
|
||
}
|
||
|
||
function buildSubmitBody(params: {
|
||
modelId: string;
|
||
prompt: string;
|
||
imageUrls: string[];
|
||
videoUrls: string[];
|
||
videoParams?: BltcyVideoParams;
|
||
}) {
|
||
const { modelId, prompt, imageUrls, videoUrls, videoParams } = params;
|
||
const mode = videoParams?.mode || (imageUrls[0] || videoUrls[0] ? "first-frame" : "text-to-video");
|
||
const images =
|
||
mode === "first-frame"
|
||
? imageUrls.slice(0, 1)
|
||
: mode === "first-last-frame"
|
||
? imageUrls.slice(0, 2)
|
||
: mode === "reference-images"
|
||
? imageUrls.slice(0, 8)
|
||
: [];
|
||
const videos = videoUrls.slice(0, 2);
|
||
|
||
if (mode === "first-frame" && images.length < 1 && videos.length < 1) {
|
||
throw new Error("柏拉图图生视频至少需要 1 张参考图或 1 段参考视频");
|
||
}
|
||
if (mode === "first-last-frame" && images.length < 2) {
|
||
throw new Error("柏拉图首尾帧视频至少需要 2 张图片");
|
||
}
|
||
if (mode === "reference-images" && images.length < 1 && videos.length < 1) {
|
||
throw new Error("柏拉图多参考视频至少需要 1 张图片或 1 段视频");
|
||
}
|
||
|
||
const body: Record<string, any> = {
|
||
model: normalizeText(modelId),
|
||
prompt: normalizeText(prompt) || "生成视频",
|
||
duration: normalizeDuration(videoParams?.duration),
|
||
aspect_ratio: normalizeRatio(videoParams?.ratio),
|
||
watermark: false,
|
||
};
|
||
|
||
const resolution = normalizeResolution(videoParams?.resolution);
|
||
if (resolution) {
|
||
body.resolution = resolution;
|
||
}
|
||
|
||
if (images.length > 0) {
|
||
body.images = images;
|
||
}
|
||
|
||
if (videos.length > 0) {
|
||
body.videos = videos;
|
||
}
|
||
|
||
return body;
|
||
}
|
||
|
||
export function createBltcyVideo({ apiKey, baseURL }: BltcyVideoConfig) {
|
||
return {
|
||
provider: "bltcy-video",
|
||
chat: (modelId: string) => ({
|
||
provider: "bltcy-video",
|
||
doGenerate: async (options: {
|
||
messages: Array<{ role: string; content: any }>;
|
||
videoParams?: BltcyVideoParams;
|
||
overrideModelId?: string;
|
||
}) => {
|
||
try {
|
||
const credentials = normalizeApiKey(apiKey);
|
||
if (!credentials) {
|
||
throw new Error("柏拉图 API Key 未配置或格式无效");
|
||
}
|
||
|
||
const activeModelId = normalizeText(options.overrideModelId || modelId);
|
||
const { prompt, imageUrls, videoUrls } = extractPromptAndAssets(options.messages);
|
||
const normalizedImageUrls = (
|
||
await Promise.all(imageUrls.map((url) => normalizeAssetUrl(url, "image")))
|
||
).filter(Boolean);
|
||
const normalizedVideoUrls = (
|
||
await Promise.all(videoUrls.map((url) => normalizeAssetUrl(url, "video")))
|
||
).filter(Boolean);
|
||
const requestBody = buildSubmitBody({
|
||
modelId: activeModelId,
|
||
prompt,
|
||
imageUrls: normalizedImageUrls,
|
||
videoUrls: normalizedVideoUrls,
|
||
videoParams: options.videoParams,
|
||
});
|
||
|
||
console.log("[BLTCY Video] Submitting task:", {
|
||
model: activeModelId,
|
||
baseURL: normalizeBltcyVideoBaseUrl(baseURL),
|
||
hasImages: Array.isArray(requestBody.images) ? requestBody.images.length : 0,
|
||
hasVideos: Array.isArray(requestBody.videos) ? requestBody.videos.length : 0,
|
||
duration: requestBody.duration,
|
||
aspectRatio: requestBody.aspect_ratio,
|
||
resolution: requestBody.resolution || null,
|
||
});
|
||
|
||
const submitResponse = await fetch(joinApiPath(baseURL, "/v2/videos/generations"), {
|
||
method: "POST",
|
||
headers: {
|
||
Authorization: `Bearer ${credentials}`,
|
||
"Content-Type": "application/json",
|
||
Accept: "application/json",
|
||
},
|
||
body: JSON.stringify(requestBody),
|
||
});
|
||
|
||
const submitPayload = await readJsonOrText(submitResponse);
|
||
|
||
if (!submitResponse.ok) {
|
||
throw new Error(
|
||
`${extractErrorMessage(submitPayload)} (HTTP ${submitResponse.status})`
|
||
);
|
||
}
|
||
|
||
const taskId = extractTaskId(submitPayload);
|
||
if (!taskId) {
|
||
const directVideoUrl = collectHttpUrls(submitPayload).find((url) =>
|
||
/\.(mp4|mov|webm|m4v)(\?|#|$)/i.test(url)
|
||
);
|
||
if (directVideoUrl) {
|
||
const archivedUrl = await uploadVideoToStorage(directVideoUrl);
|
||
const text = formatVideoMarkdown(archivedUrl);
|
||
return {
|
||
text,
|
||
finishReason: "stop" as const,
|
||
usage: {
|
||
promptTokens: 0,
|
||
completionTokens: text.length,
|
||
},
|
||
};
|
||
}
|
||
throw new Error("柏拉图创建任务成功，但未返回任务 ID");
|
||
}
|
||
|
||
const archivedVideoUrl = await waitForVideoUrl(baseURL, credentials, taskId);
|
||
const text = formatVideoMarkdown(archivedVideoUrl);
|
||
|
||
return {
|
||
text,
|
||
finishReason: "stop" as const,
|
||
usage: {
|
||
promptTokens: 0,
|
||
completionTokens: text.length,
|
||
},
|
||
};
|
||
} catch (error) {
|
||
console.error("[BLTCY Video] Generation failed:", error);
|
||
return {
|
||
text:
|
||
error instanceof Error
|
||
? `视频生成失败: ${error.message}`
|
||
: "柏拉图视频生成失败，请稍后重试",
|
||
finishReason: "error" as const,
|
||
usage: {
|
||
promptTokens: 0,
|
||
completionTokens: 0,
|
||
},
|
||
};
|
||
}
|
||
},
|
||
doStream: async function* (options: {
|
||
messages: Array<{ role: string; content: any }>;
|
||
videoParams?: BltcyVideoParams;
|
||
overrideModelId?: string;
|
||
}) {
|
||
const result = await this.doGenerate(options);
|
||
yield {
|
||
type: "text-delta",
|
||
textDelta: result.text,
|
||
};
|
||
},
|
||
modelId,
|
||
specificationVersion: "v1" as const,
|
||
defaultObjectGenerationMode: "json" as const,
|
||
}),
|
||
};
|
||
}
|