107 lines
6.3 KiB
TypeScript
107 lines
6.3 KiB
TypeScript
import { z } from "zod";
|
|
|
|
export type AudioModelContract = {
|
|
kind: string; label: string; voices: { id: string; name: string }[];
|
|
instructions: boolean; controls: boolean; formats: string[]; supportsLanguageControl?: boolean;
|
|
supportsGender?: boolean; genderRequiresLyrics?: boolean; descriptionLimit?: number;
|
|
};
|
|
const voice = (id: string, name = id) => ({ id, name });
|
|
const tts = (label: string, voices: { id: string; name: string }[] = [], instructions = false): AudioModelContract =>
|
|
({ kind: "tts", label, voices, instructions, controls: false, formats: ["wav"], supportsLanguageControl: false });
|
|
const music = (label: string): AudioModelContract =>
|
|
({ kind: "music", label, voices: [], instructions: false, controls: false, formats: ["mp3"], supportsGender: false });
|
|
|
|
export const vip98kAudioContracts: Record<string, AudioModelContract> = {
|
|
"mimo-v2.5-asr": { ...tts("MiMo 语音识别"), kind: "asr", formats: [] },
|
|
"mimo-v2.5-tts": tts("MiMo 文字配音", [], true),
|
|
"mimo-v2.5-tts-voiceclone": { ...tts("MiMo 声音克隆", [], true), kind: "voiceclone" },
|
|
"mimo-v2.5-tts-voicedesign": { ...tts("MiMo 声音设计", [], true), kind: "voicedesign" },
|
|
"gem-2.5-tts": tts("Gemini 2.5 配音", [voice("Zephyr"), voice("Puck"), voice("Kore"), voice("Charon")]),
|
|
"gem-3.1-tts": tts("Gemini 3.1 配音", [voice("Zephyr"), voice("Puck"), voice("Kore"), voice("Charon")]),
|
|
"speech-2.8-hd": { ...tts("Speech 2.8 高清", [voice("male-qn-qingse", "青涩男声")]), formats: ["mp3", "wav"], controls: true },
|
|
"speech-2.8-turbo": { ...tts("Speech 2.8 快速", [voice("male-qn-qingse", "青涩男声")]), formats: ["mp3", "wav"], controls: true },
|
|
"music-2.5": music("Music 2.5"),
|
|
"music-2.5+": music("Music 2.5+"),
|
|
"suno-v4.5": { ...music("Suno 4.5"), supportsGender: true, descriptionLimit: 1000 },
|
|
"suno-v5": { ...music("Suno 5"), supportsGender: true, genderRequiresLyrics: true, descriptionLimit: 500 },
|
|
"suno-v5-5": { ...music("Suno 5.5"), supportsGender: true, genderRequiresLyrics: true, descriptionLimit: 500 },
|
|
"suno-v6": { ...music("Suno 6"), supportsGender: true, descriptionLimit: 1000 },
|
|
};
|
|
export function vip98kAudioContract(id: string) {
|
|
return Object.hasOwn(vip98kAudioContracts, id) ? vip98kAudioContracts[id] : undefined;
|
|
}
|
|
|
|
export type Vip98kAudioInput = {
|
|
mode: string; text: string; lyrics: string; instructions: string; voice?: string;
|
|
format: string; instrumental: boolean; gender: string; rate: number; pitch: number;
|
|
volume: number; referenceAudioUrl?: string;
|
|
};
|
|
export function validateVip98kAudioInput(input: Vip98kAudioInput, modelId: string) {
|
|
const contract = vip98kAudioContract(modelId);
|
|
if (!contract || contract.kind !== input.mode) throw new Error("音频模式与模型不匹配");
|
|
if (["asr", "voiceclone"].includes(input.mode) && !input.referenceAudioUrl) throw new Error("请先上传参考音频");
|
|
if (input.mode !== "asr" && !input.text.trim() && !input.lyrics.trim()) throw new Error("请输入配音文本或音乐描述");
|
|
if (input.mode === "voicedesign" && !input.instructions.trim()) throw new Error("请填写声音风格指令");
|
|
if (contract.descriptionLimit && input.text.length > contract.descriptionLimit) throw new Error(`这个模型的音乐描述最多 ${contract.descriptionLimit} 字`);
|
|
if (input.mode !== "asr" && !contract.formats.includes(input.format)) throw new Error("所选模型不支持这个音频格式");
|
|
}
|
|
|
|
export function buildVip98kAudioBody(input: Vip98kAudioInput, modelId: string, example: Record<string, any>, audioData?: string) {
|
|
validateVip98kAudioInput(input, modelId);
|
|
const body = structuredClone(example);
|
|
body.model = modelId;
|
|
if (modelId.startsWith("mimo-")) {
|
|
body.stream = false;
|
|
if (input.mode === "asr") {
|
|
if (!audioData) throw new Error("参考音频尚未读取");
|
|
body.messages = [{ role: "user", content: [{ type: "input_audio", input_audio: { data: audioData, format: audioData.includes("audio/wav") ? "wav" : "mp3" } }] }];
|
|
} else {
|
|
body.messages = [
|
|
{ role: "user", content: input.instructions || "请用自然的中文朗读。" },
|
|
{ role: "assistant", content: input.text },
|
|
];
|
|
if (input.mode === "voiceclone") {
|
|
if (!audioData) throw new Error("参考音频尚未读取");
|
|
body.audio = { ...body.audio, voice: audioData };
|
|
}
|
|
}
|
|
return body;
|
|
}
|
|
body.prompt = input.text;
|
|
const set = (key: string, value: unknown) => {
|
|
if (body.metadata && key in body.metadata) body.metadata[key] = value;
|
|
else body[key] = value;
|
|
};
|
|
if (modelId.startsWith("gem-")) set("voice_id", input.voice || "Zephyr");
|
|
if (modelId.startsWith("speech-")) {
|
|
set("voice_id", input.voice || "male-qn-qingse");
|
|
set("speed", input.rate);
|
|
set("pitch", Math.round((input.pitch - 1) * 12));
|
|
set("vol", Math.max(0.01, input.volume / 50));
|
|
set("format", input.format);
|
|
}
|
|
if (input.mode === "music") {
|
|
if (modelId === "suno-v5" || modelId === "suno-v5-5") {
|
|
body.instrumental = input.instrumental;
|
|
body.custom_mode = Boolean(input.lyrics && !input.instrumental);
|
|
if (body.custom_mode) { body.prompt = input.lyrics; body.style = input.text; body.title = input.text.slice(0, 60) || "歌曲"; }
|
|
if (body.custom_mode) set("vocal_gender", input.gender === "male" ? "m" : "f");
|
|
else { delete body.vocal_gender; if (body.metadata) delete body.metadata.vocal_gender; }
|
|
} else {
|
|
set(modelId.startsWith("music-") ? "is_instrumental" : "make_instrumental", input.instrumental ? "instrumental" : "song");
|
|
if (!input.instrumental && input.lyrics) set("lyrics", input.lyrics);
|
|
if (modelId.startsWith("suno-")) set("vocal_gender", input.gender === "male" ? "m" : "f");
|
|
}
|
|
}
|
|
return body;
|
|
}
|
|
|
|
export const vip98kAudioCheckpointSchema = z.object({
|
|
operation: z.literal("audio"), provider: z.literal("vip98k"),
|
|
modelRef: z.string(), modelId: z.string(), providerId: z.string(), baseUrl: z.string(),
|
|
taskId: z.string().optional(), resultPath: z.string().optional(), requestId: z.string().optional(),
|
|
url: z.string().url().optional(), urls: z.array(z.string().url()).optional(), archivedUrls: z.array(z.string().url()).optional(),
|
|
data: z.string().optional(), text: z.string().optional(),
|
|
format: z.enum(["mp3", "wav"]),
|
|
}).refine(value => Boolean(value.taskId || value.url || value.data || value.text));
|