134 lines
8.0 KiB
TypeScript
134 lines
8.0 KiB
TypeScript
import "server-only";
|
||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||
import { tmpdir } from "node:os";
|
||
import { join } from "node:path";
|
||
import { randomUUID } from "node:crypto";
|
||
import { executeVip98kExternalTask } from "@/lib/ai/providers/vip98k";
|
||
import { getVip98kInvocationHeaders } from "@/lib/ai/vip98k-task-invocation";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
import { archiveMediaThroughQiniu } from "@/lib/storage/qiniu-image-archive";
|
||
import { downloadCanvasMedia } from "./download-media";
|
||
import { ProviderTaskUncertainError } from "./video-task-lifecycle";
|
||
import { buildVip98kAudioBody, vip98kAudioCheckpointSchema } from "./vip98k-audio-contract";
|
||
import { readVip98kAudioFile } from "./vip98k-audio-file";
|
||
import type { AudioRequest } from "./audio-contract";
|
||
import type { AiModel, AiProvider } from "@/lib/db/schema";
|
||
|
||
export async function executeVip98kCanvasAudio(input: AudioRequest, item: { model: AiModel; provider: AiProvider }, lifecycle: {
|
||
resume?: unknown; assertActive: () => Promise<void>; checkpoint: (value: Record<string, unknown>) => Promise<void>;
|
||
}) {
|
||
const { model, provider } = item;
|
||
const metadata = (model.capabilities as any)?.vip98k;
|
||
const invocation = metadata?.invocation;
|
||
if (!invocation?.path) throw new Error("音频模型缺少调用配置,请同步模型目录");
|
||
const identity = { operation: "audio" as const, provider: "vip98k" as const,
|
||
modelRef: model.id, modelId: model.modelId, providerId: provider.id,
|
||
baseUrl: provider.baseUrl.replace(/\/+$/, "").replace(/\/v1$/i, ""), format: input.format };
|
||
let saved = lifecycle.resume ? vip98kAudioCheckpointSchema.parse(lifecycle.resume) : undefined;
|
||
if (saved && Object.entries(identity).some(([key, value]) => (saved as any)[key] !== value))
|
||
throw new ProviderTaskUncertainError("原音频任务的服务线路或模型已变更,请核对原任务");
|
||
const save = async (value: typeof saved) => {
|
||
await lifecycle.assertActive();
|
||
await lifecycle.checkpoint({ upstreamAudio: value });
|
||
saved = value;
|
||
};
|
||
const dir = await mkdtemp(join(tmpdir(), "canvas-98k-audio-"));
|
||
try {
|
||
if (!saved?.url && !saved?.data && !saved?.text) {
|
||
let audioData: string | undefined;
|
||
if (!saved && input.referenceAudioUrl) {
|
||
const reference = join(dir, "reference");
|
||
await downloadCanvasMedia(input.referenceAudioUrl, reference);
|
||
const bytes = await readFile(reference);
|
||
if (bytes.length > 20 * 1024 * 1024) throw new Error("参考音频需小于 20 MiB");
|
||
const referenceAudio = readVip98kAudioFile(bytes);
|
||
audioData = `data:audio/${referenceAudio.format === "mp3" ? "mpeg" : "wav"};base64,${referenceAudio.bytes.toString("base64")}`;
|
||
}
|
||
const body = saved ? undefined : buildVip98kAudioBody(input, model.modelId, invocation.requestExample || {}, audioData);
|
||
if (invocation.path === "/v1/chat/completions") {
|
||
await lifecycle.assertActive();
|
||
let response: Response;
|
||
let payload: any;
|
||
try {
|
||
response = await fetch(identity.baseUrl + invocation.path, {
|
||
method: "POST", redirect: "error", signal: AbortSignal.timeout(180000),
|
||
headers: { Authorization: `Bearer ${provider.apiKey}`, "Content-Type": "application/json",
|
||
...getVip98kInvocationHeaders(invocation.headers), "Idempotency-Key": randomUUID() },
|
||
body: JSON.stringify(body),
|
||
});
|
||
payload = await response.json();
|
||
} catch {
|
||
throw new ProviderTaskUncertainError("98K 音频请求结果未确认,请核对原请求,系统不会自动重发");
|
||
}
|
||
if (!response.ok) {
|
||
if (response.status >= 500) throw new ProviderTaskUncertainError(`98K 音频结果未确认(HTTP ${response.status}),未重复提交`);
|
||
throw new Error(`98K 音频请求失败(HTTP ${response.status}),请核对模型和输入`);
|
||
}
|
||
const message = payload.choices?.[0]?.message;
|
||
const text = typeof message?.content === "string" ? message.content.trim() : "";
|
||
const audio = message?.audio;
|
||
const next = input.mode === "asr" ? { text } : { data: audio?.data, url: audio?.url };
|
||
if (!next.text && !next.data && !next.url) throw new ProviderTaskUncertainError("98K 未返回音频结果,请核对原请求,勿重复生成");
|
||
try { await save({ ...identity, ...next, requestId: payload.id }); }
|
||
catch { throw new ProviderTaskUncertainError("音频已生成,但结果保存失败,请核对原请求"); }
|
||
} else {
|
||
const result = await executeVip98kExternalTask({
|
||
apiKey: provider.apiKey, baseURL: identity.baseUrl, modelId: model.modelId, kind: "audio", prompt: String(body?.prompt || input.text),
|
||
modelConfig: { capabilities: { ...model.capabilities as any, vip98k: { ...metadata,
|
||
invocation: { ...invocation, requestExample: body || invocation.requestExample },
|
||
invocations: [],
|
||
} } },
|
||
lifecycle: { assertActive: lifecycle.assertActive,
|
||
resume: saved?.taskId ? { taskId: saved.taskId, resultPath: saved.resultPath } : undefined,
|
||
accepted: reference => save({ ...identity, ...reference }),
|
||
},
|
||
});
|
||
const payload = result.payload as any;
|
||
const outputs = payload?.result?.outputs || payload?.outputs;
|
||
const audioUrls = Array.isArray(outputs) ? outputs.filter(output => output.type === "audio" && typeof output.url === "string").map(output => output.url as string) : [];
|
||
const urls: string[] = audioUrls.length ? audioUrls : result.outputUrls.slice(0, 1);
|
||
const url = urls[0];
|
||
if (!url) throw new ProviderTaskUncertainError("音频任务已完成但未返回文件,请核对原任务");
|
||
await save({ ...identity, taskId: result.taskId, url, urls });
|
||
}
|
||
}
|
||
if (saved?.text) return { text: saved.text, modelRef: model.id, modelId: model.modelId, metadata: { outputType: "text", requestId: saved.requestId } };
|
||
const sources = saved?.urls?.length ? saved.urls : [saved?.url];
|
||
const archivedUrls = [...(saved?.archivedUrls || [])];
|
||
let format = saved!.format;
|
||
for (let index = archivedUrls.length; index < sources.length; index++) {
|
||
if (sources[index] && !model.modelId.startsWith("speech-")) {
|
||
const archived = await archiveMediaThroughQiniu(sources[index]!, `${saved!.taskId || saved!.requestId}-${index}`, "audio");
|
||
if (archived) {
|
||
const response = await fetch(archived.url, { headers: { Range: "bytes=0-1023" }, signal: AbortSignal.timeout(20000) });
|
||
if (!response.ok) throw new Error("音频文件尚不可下载");
|
||
const reader = response.body!.getReader();
|
||
const first = await reader.read(); await reader.cancel();
|
||
format = readVip98kAudioFile(Buffer.from(first.value || [])).format;
|
||
archivedUrls.push(archived.url);
|
||
await save({ ...saved!, archivedUrls });
|
||
continue;
|
||
}
|
||
}
|
||
let bytes: Buffer;
|
||
if (saved?.data) bytes = Buffer.from(saved.data.replace(/^data:[^,]+,/, ""), "base64");
|
||
else {
|
||
const output = join(dir, `result-${index}`);
|
||
await downloadCanvasMedia(sources[index]!, output);
|
||
bytes = await readFile(output);
|
||
}
|
||
const audio = readVip98kAudioFile(bytes);
|
||
format = audio.format;
|
||
await lifecycle.assertActive();
|
||
const result = await uploadFile(`canvas-audio-${randomUUID()}.${format}`, Uint8Array.from(audio.bytes).buffer);
|
||
archivedUrls.push(result.url);
|
||
await save({ ...saved!, archivedUrls });
|
||
}
|
||
return { url: archivedUrls[0], urls: archivedUrls, modelRef: model.id, modelId: model.modelId,
|
||
metadata: { storageStatus: "archived", format, requestId: saved?.requestId, taskId: saved?.taskId } };
|
||
} catch (error) {
|
||
if (saved?.url || saved?.data) throw new ProviderTaskUncertainError("音频已生成,保存文件尚未完成,将继续保存原文件");
|
||
throw error;
|
||
} finally { await rm(dir, { recursive: true, force: true }); }
|
||
}
|