import "server-only"; import { mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { randomUUID } from "node:crypto"; import { executeVip98kExternalTask } from "@/lib/ai/providers/vip98k"; import { getVip98kInvocationHeaders } from "@/lib/ai/vip98k-task-invocation"; import { uploadFile } from "@/lib/storage/service"; import { archiveMediaThroughQiniu } from "@/lib/storage/qiniu-image-archive"; import { downloadCanvasMedia } from "./download-media"; import { ProviderTaskUncertainError } from "./video-task-lifecycle"; import { buildVip98kAudioBody, vip98kAudioCheckpointSchema } from "./vip98k-audio-contract"; import { readVip98kAudioFile } from "./vip98k-audio-file"; import type { AudioRequest } from "./audio-contract"; import type { AiModel, AiProvider } from "@/lib/db/schema"; export async function executeVip98kCanvasAudio(input: AudioRequest, item: { model: AiModel; provider: AiProvider }, lifecycle: { resume?: unknown; assertActive: () => Promise; checkpoint: (value: Record) => Promise; }) { const { model, provider } = item; const metadata = (model.capabilities as any)?.vip98k; const invocation = metadata?.invocation; if (!invocation?.path) throw new Error("音频模型缺少调用配置,请同步模型目录"); const identity = { operation: "audio" as const, provider: "vip98k" as const, modelRef: model.id, modelId: model.modelId, providerId: provider.id, baseUrl: provider.baseUrl.replace(/\/+$/, "").replace(/\/v1$/i, ""), format: input.format }; let saved = lifecycle.resume ? vip98kAudioCheckpointSchema.parse(lifecycle.resume) : undefined; if (saved && Object.entries(identity).some(([key, value]) => (saved as any)[key] !== value)) throw new ProviderTaskUncertainError("原音频任务的服务线路或模型已变更,请核对原任务"); const save = async (value: typeof saved) => { await lifecycle.assertActive(); await lifecycle.checkpoint({ upstreamAudio: value }); saved = value; }; const dir = await mkdtemp(join(tmpdir(), "canvas-98k-audio-")); try { if (!saved?.url && !saved?.data && !saved?.text) { let audioData: string | undefined; if (!saved && input.referenceAudioUrl) { const reference = join(dir, "reference"); await downloadCanvasMedia(input.referenceAudioUrl, reference); const bytes = await readFile(reference); if (bytes.length > 20 * 1024 * 1024) throw new Error("参考音频需小于 20 MiB"); const referenceAudio = readVip98kAudioFile(bytes); audioData = `data:audio/${referenceAudio.format === "mp3" ? "mpeg" : "wav"};base64,${referenceAudio.bytes.toString("base64")}`; } const body = saved ? undefined : buildVip98kAudioBody(input, model.modelId, invocation.requestExample || {}, audioData); if (invocation.path === "/v1/chat/completions") { await lifecycle.assertActive(); let response: Response; let payload: any; try { response = await fetch(identity.baseUrl + invocation.path, { method: "POST", redirect: "error", signal: AbortSignal.timeout(180000), headers: { Authorization: `Bearer ${provider.apiKey}`, "Content-Type": "application/json", ...getVip98kInvocationHeaders(invocation.headers), "Idempotency-Key": randomUUID() }, body: JSON.stringify(body), }); payload = await response.json(); } catch { throw new ProviderTaskUncertainError("98K 音频请求结果未确认,请核对原请求,系统不会自动重发"); } if (!response.ok) { if (response.status >= 500) throw new ProviderTaskUncertainError(`98K 音频结果未确认(HTTP ${response.status}),未重复提交`); throw new Error(`98K 音频请求失败(HTTP ${response.status}),请核对模型和输入`); } const message = payload.choices?.[0]?.message; const text = typeof message?.content === "string" ? message.content.trim() : ""; const audio = message?.audio; const next = input.mode === "asr" ? { text } : { data: audio?.data, url: audio?.url }; if (!next.text && !next.data && !next.url) throw new ProviderTaskUncertainError("98K 未返回音频结果,请核对原请求,勿重复生成"); try { await save({ ...identity, ...next, requestId: payload.id }); } catch { throw new ProviderTaskUncertainError("音频已生成,但结果保存失败,请核对原请求"); } } else { const result = await executeVip98kExternalTask({ apiKey: provider.apiKey, baseURL: identity.baseUrl, modelId: model.modelId, kind: "audio", prompt: String(body?.prompt || input.text), modelConfig: { capabilities: { ...model.capabilities as any, vip98k: { ...metadata, invocation: { ...invocation, requestExample: body || invocation.requestExample }, invocations: [], } } }, lifecycle: { assertActive: lifecycle.assertActive, resume: saved?.taskId ? { taskId: saved.taskId, resultPath: saved.resultPath } : undefined, accepted: reference => save({ ...identity, ...reference }), }, }); const payload = result.payload as any; const outputs = payload?.result?.outputs || payload?.outputs; const audioUrls = Array.isArray(outputs) ? outputs.filter(output => output.type === "audio" && typeof output.url === "string").map(output => output.url as string) : []; const urls: string[] = audioUrls.length ? audioUrls : result.outputUrls.slice(0, 1); const url = urls[0]; if (!url) throw new ProviderTaskUncertainError("音频任务已完成但未返回文件,请核对原任务"); await save({ ...identity, taskId: result.taskId, url, urls }); } } if (saved?.text) return { text: saved.text, modelRef: model.id, modelId: model.modelId, metadata: { outputType: "text", requestId: saved.requestId } }; const sources = saved?.urls?.length ? saved.urls : [saved?.url]; const archivedUrls = [...(saved?.archivedUrls || [])]; let format = saved!.format; for (let index = archivedUrls.length; index < sources.length; index++) { if (sources[index] && !model.modelId.startsWith("speech-")) { const archived = await archiveMediaThroughQiniu(sources[index]!, `${saved!.taskId || saved!.requestId}-${index}`, "audio"); if (archived) { const response = await fetch(archived.url, { headers: { Range: "bytes=0-1023" }, signal: AbortSignal.timeout(20000) }); if (!response.ok) throw new Error("音频文件尚不可下载"); const reader = response.body!.getReader(); const first = await reader.read(); await reader.cancel(); format = readVip98kAudioFile(Buffer.from(first.value || [])).format; archivedUrls.push(archived.url); await save({ ...saved!, archivedUrls }); continue; } } let bytes: Buffer; if (saved?.data) bytes = Buffer.from(saved.data.replace(/^data:[^,]+,/, ""), "base64"); else { const output = join(dir, `result-${index}`); await downloadCanvasMedia(sources[index]!, output); bytes = await readFile(output); } const audio = readVip98kAudioFile(bytes); format = audio.format; await lifecycle.assertActive(); const result = await uploadFile(`canvas-audio-${randomUUID()}.${format}`, Uint8Array.from(audio.bytes).buffer); archivedUrls.push(result.url); await save({ ...saved!, archivedUrls }); } return { url: archivedUrls[0], urls: archivedUrls, modelRef: model.id, modelId: model.modelId, metadata: { storageStatus: "archived", format, requestId: saved?.requestId, taskId: saved?.taskId } }; } catch (error) { if (saved?.url || saved?.data) throw new ProviderTaskUncertainError("音频已生成,保存文件尚未完成,将继续保存原文件"); throw error; } finally { await rm(dir, { recursive: true, force: true }); } }