ai-chatbot/lib/video-canvas/vip98k-audio-generation.ts

134 lines
8.0 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import "server-only";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { randomUUID } from "node:crypto";
import { executeVip98kExternalTask } from "@/lib/ai/providers/vip98k";
import { getVip98kInvocationHeaders } from "@/lib/ai/vip98k-task-invocation";
import { uploadFile } from "@/lib/storage/service";
import { archiveMediaThroughQiniu } from "@/lib/storage/qiniu-image-archive";
import { downloadCanvasMedia } from "./download-media";
import { ProviderTaskUncertainError } from "./video-task-lifecycle";
import { buildVip98kAudioBody, vip98kAudioCheckpointSchema } from "./vip98k-audio-contract";
import { readVip98kAudioFile } from "./vip98k-audio-file";
import type { AudioRequest } from "./audio-contract";
import type { AiModel, AiProvider } from "@/lib/db/schema";
export async function executeVip98kCanvasAudio(input: AudioRequest, item: { model: AiModel; provider: AiProvider }, lifecycle: {
resume?: unknown; assertActive: () => Promise<void>; checkpoint: (value: Record<string, unknown>) => Promise<void>;
}) {
const { model, provider } = item;
const metadata = (model.capabilities as any)?.vip98k;
const invocation = metadata?.invocation;
if (!invocation?.path) throw new Error("音频模型缺少调用配置,请同步模型目录");
const identity = { operation: "audio" as const, provider: "vip98k" as const,
modelRef: model.id, modelId: model.modelId, providerId: provider.id,
baseUrl: provider.baseUrl.replace(/\/+$/, "").replace(/\/v1$/i, ""), format: input.format };
let saved = lifecycle.resume ? vip98kAudioCheckpointSchema.parse(lifecycle.resume) : undefined;
if (saved && Object.entries(identity).some(([key, value]) => (saved as any)[key] !== value))
throw new ProviderTaskUncertainError("原音频任务的服务线路或模型已变更,请核对原任务");
const save = async (value: typeof saved) => {
await lifecycle.assertActive();
await lifecycle.checkpoint({ upstreamAudio: value });
saved = value;
};
const dir = await mkdtemp(join(tmpdir(), "canvas-98k-audio-"));
try {
if (!saved?.url && !saved?.data && !saved?.text) {
let audioData: string | undefined;
if (!saved && input.referenceAudioUrl) {
const reference = join(dir, "reference");
await downloadCanvasMedia(input.referenceAudioUrl, reference);
const bytes = await readFile(reference);
if (bytes.length > 20 * 1024 * 1024) throw new Error("参考音频需小于 20 MiB");
const referenceAudio = readVip98kAudioFile(bytes);
audioData = `data:audio/${referenceAudio.format === "mp3" ? "mpeg" : "wav"};base64,${referenceAudio.bytes.toString("base64")}`;
}
const body = saved ? undefined : buildVip98kAudioBody(input, model.modelId, invocation.requestExample || {}, audioData);
if (invocation.path === "/v1/chat/completions") {
await lifecycle.assertActive();
let response: Response;
let payload: any;
try {
response = await fetch(identity.baseUrl + invocation.path, {
method: "POST", redirect: "error", signal: AbortSignal.timeout(180000),
headers: { Authorization: `Bearer ${provider.apiKey}`, "Content-Type": "application/json",
...getVip98kInvocationHeaders(invocation.headers), "Idempotency-Key": randomUUID() },
body: JSON.stringify(body),
});
payload = await response.json();
} catch {
throw new ProviderTaskUncertainError("98K 音频请求结果未确认,请核对原请求,系统不会自动重发");
}
if (!response.ok) {
if (response.status >= 500) throw new ProviderTaskUncertainError(`98K 音频结果未确认(HTTP ${response.status}),未重复提交`);
throw new Error(`98K 音频请求失败(HTTP ${response.status}),请核对模型和输入`);
}
const message = payload.choices?.[0]?.message;
const text = typeof message?.content === "string" ? message.content.trim() : "";
const audio = message?.audio;
const next = input.mode === "asr" ? { text } : { data: audio?.data, url: audio?.url };
if (!next.text && !next.data && !next.url) throw new ProviderTaskUncertainError("98K 未返回音频结果,请核对原请求,勿重复生成");
try { await save({ ...identity, ...next, requestId: payload.id }); }
catch { throw new ProviderTaskUncertainError("音频已生成,但结果保存失败,请核对原请求"); }
} else {
const result = await executeVip98kExternalTask({
apiKey: provider.apiKey, baseURL: identity.baseUrl, modelId: model.modelId, kind: "audio", prompt: String(body?.prompt || input.text),
modelConfig: { capabilities: { ...model.capabilities as any, vip98k: { ...metadata,
invocation: { ...invocation, requestExample: body || invocation.requestExample },
invocations: [],
} } },
lifecycle: { assertActive: lifecycle.assertActive,
resume: saved?.taskId ? { taskId: saved.taskId, resultPath: saved.resultPath } : undefined,
accepted: reference => save({ ...identity, ...reference }),
},
});
const payload = result.payload as any;
const outputs = payload?.result?.outputs || payload?.outputs;
const audioUrls = Array.isArray(outputs) ? outputs.filter(output => output.type === "audio" && typeof output.url === "string").map(output => output.url as string) : [];
const urls: string[] = audioUrls.length ? audioUrls : result.outputUrls.slice(0, 1);
const url = urls[0];
if (!url) throw new ProviderTaskUncertainError("音频任务已完成但未返回文件,请核对原任务");
await save({ ...identity, taskId: result.taskId, url, urls });
}
}
if (saved?.text) return { text: saved.text, modelRef: model.id, modelId: model.modelId, metadata: { outputType: "text", requestId: saved.requestId } };
const sources = saved?.urls?.length ? saved.urls : [saved?.url];
const archivedUrls = [...(saved?.archivedUrls || [])];
let format = saved!.format;
for (let index = archivedUrls.length; index < sources.length; index++) {
if (sources[index] && !model.modelId.startsWith("speech-")) {
const archived = await archiveMediaThroughQiniu(sources[index]!, `${saved!.taskId || saved!.requestId}-${index}`, "audio");
if (archived) {
const response = await fetch(archived.url, { headers: { Range: "bytes=0-1023" }, signal: AbortSignal.timeout(20000) });
if (!response.ok) throw new Error("音频文件尚不可下载");
const reader = response.body!.getReader();
const first = await reader.read(); await reader.cancel();
format = readVip98kAudioFile(Buffer.from(first.value || [])).format;
archivedUrls.push(archived.url);
await save({ ...saved!, archivedUrls });
continue;
}
}
let bytes: Buffer;
if (saved?.data) bytes = Buffer.from(saved.data.replace(/^data:[^,]+,/, ""), "base64");
else {
const output = join(dir, `result-${index}`);
await downloadCanvasMedia(sources[index]!, output);
bytes = await readFile(output);
}
const audio = readVip98kAudioFile(bytes);
format = audio.format;
await lifecycle.assertActive();
const result = await uploadFile(`canvas-audio-${randomUUID()}.${format}`, Uint8Array.from(audio.bytes).buffer);
archivedUrls.push(result.url);
await save({ ...saved!, archivedUrls });
}
return { url: archivedUrls[0], urls: archivedUrls, modelRef: model.id, modelId: model.modelId,
metadata: { storageStatus: "archived", format, requestId: saved?.requestId, taskId: saved?.taskId } };
} catch (error) {
if (saved?.url || saved?.data) throw new ProviderTaskUncertainError("音频已生成,保存文件尚未完成,将继续保存原文件");
throw error;
} finally { await rm(dir, { recursive: true, force: true }); }
}