ai-chatbot/components/canvas/AudioGenerationComposer.tsx

335 lines
13 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"use client";
import { useEffect, useState } from "react";
import {
audioContract,
audioRequestSchema,
buildAudioRequest,
type AudioRequest,
} from "@/lib/video-canvas/audio-contract";
type Model = { id: string; modelId: string; name: string; kind: string };
export function AudioGenerationComposer({
initial,
connectedText = "",
busy,
onSubmit,
onUpload,
onDraft,
}: {
initial?: Record<string, unknown>;
connectedText?: string;
busy: boolean;
onSubmit: (params: AudioRequest, draft: AudioRequest) => Promise<unknown>;
onUpload: (file: File) => Promise<void>;
onDraft: (params: Record<string, unknown>) => void;
}) {
const [models, setModels] = useState<Model[]>([]),
[error, setError] = useState(""),
[loading, setLoading] = useState(true),
[sending, setSending] = useState(false);
const [draft, setDraft] = useState<Record<string, any>>(() => ({
mode: "tts",
text: "",
voice: "Cherry",
format: "wav",
language: "Chinese",
instrumental: true,
gender: "female",
rate: 1,
pitch: 1,
volume: 50,
...initial,
}));
const patch = (value: Record<string, unknown>) => {
const next = { ...draft, ...value };
setDraft(next);
onDraft(next);
};
useEffect(() => {
const controller = new AbortController();
fetch("/api/canvas/audio/models", { signal: controller.signal })
.then(async (r) => {
const data = await r.json();
if (!r.ok) throw Error(data.error || "音频模型读取失败");
setModels(data.models || []);
})
.catch((e) => {
if (!controller.signal.aborted) setError(e.message);
})
.finally(() => {
if (!controller.signal.aborted) setLoading(false);
});
return () => controller.abort();
}, []);
const options = models.filter((m) => m.kind === draft.mode),
model = options.find((m) => m.id === draft.modelRef) || options[0],
contract = model ? audioContract(model.modelId) : undefined;
const voice = contract?.voices.some((v) => v.id === draft.voice)
? draft.voice
: contract?.voices[0]?.id;
const format = contract?.formats.includes(draft.format) ? draft.format : contract?.formats[0];
const input = "mt-1 w-full rounded-lg border border-white/15 bg-[#303030] p-2 text-sm";
const run = async () => {
setError("");
setSending(true);
try {
if (!model) throw Error("当前模式没有已启用的模型");
const params = audioRequestSchema.parse({
...draft,
text: [draft.text, connectedText].filter(Boolean).join("\n\n"),
modelRef: model.id,
voice,
format: format || "wav",
...(!contract?.controls ? { rate: 1, pitch: 1, volume: 50 } : {}),
...(!contract?.instructions ? { instructions: "" } : {}),
});
buildAudioRequest(params, model.modelId);
const savedDraft = { ...params, text: String(draft.text || "") };
patch(savedDraft);
await onSubmit(params, savedDraft);
} catch (e) {
setError(e instanceof Error ? e.message : "提交失败");
} finally {
setSending(false);
}
};
return (
<section
className="fixed bottom-8 left-1/2 z-[100] max-h-[75vh] w-[min(620px,94vw)] -translate-x-1/2 overflow-y-auto rounded-2xl border border-white/10 bg-[#222] p-5 text-white shadow-xl"
data-canvas-overlay="true"
aria-label="语音与音乐"
>
<h3 className="font-semibold">语音与音乐</h3>
<fieldset disabled={busy || sending} className="mt-3 space-y-3 disabled:opacity-60">
{connectedText && <p className="mb-3 whitespace-pre-wrap rounded bg-white/5 p-2 text-xs" aria-label="已连接文本">已连接文本:{connectedText}</p>}
<div className="grid grid-cols-2 gap-3">
<label>
生成类型
<select
aria-label="音频生成类型"
className={input}
value={draft.mode}
onChange={(e) => patch({ mode: e.target.value, modelRef: "" })}
>
<option value="tts">文字配音</option>
<option value="music">音乐 / 歌曲</option>
<option value="asr">语音转文字</option>
<option value="voiceclone">声音克隆配音</option>
<option value="voicedesign">声音设计配音</option>
</select>
</label>
<label>
模型
<select
aria-label="音频模型"
className={input}
value={model?.id || ""}
onChange={(e) => patch({ modelRef: e.target.value })}
>
{!options.length ? (
<option value="">{loading ? "读取模型中…" : "暂无已开通模型"}</option>
) : (
options.map((m) => (
<option key={m.id} value={m.id}>
{m.name}
</option>
))
)}
</select>
</label>
</div>
{["asr", "voiceclone"].includes(draft.mode) && (
<label className="block">
{draft.mode === "asr" ? "待识别音频" : "参考声音"}(MP3 / WAV,最多 20 MiB)
<input type="file" accept="audio/mpeg,audio/wav,.mp3,.wav" aria-label="上传参考音频" className={input}
onChange={async event => {
const file = event.target.files?.[0]; event.target.value = ""; if (!file) return;
if (file.size > 20 * 1024 * 1024) { setError("参考音频需小于 20 MiB"); return; }
setSending(true); setError("");
try {
const form = new FormData(); form.append("file", file);
const response = await fetch("/api/canvas/media/upload", { method: "POST", body: form, signal: AbortSignal.timeout(120000) });
const result = await response.json(); const url = result.url || result.data?.url;
if (!response.ok || !url) throw Error(result.error || "参考音频上传失败");
patch({ referenceAudioUrl: String(url), referenceAudioName: file.name });
} catch (error) { setError(error instanceof Error ? error.message : "上传失败"); }
finally { setSending(false); }
}} />
{draft.referenceAudioUrl && <span className="mt-1 block text-sm text-white/80">已上传:{draft.referenceAudioName || "参考音频"}</span>}
</label>
)}
{draft.mode !== "asr" && <label className="block">
{draft.mode !== "music" ? "配音文本(最多 600 字)" : `音乐描述(最多 ${contract?.descriptionLimit || 2000} 字)`}
<textarea
aria-label="音频内容"
className={input}
rows={4}
maxLength={draft.mode !== "music" ? 600 : contract?.descriptionLimit || 2000}
value={draft.text}
onChange={(e) => patch({ text: e.target.value })}
/>
</label>}
{!["music", "asr"].includes(draft.mode) ? (
<>
{!!contract?.voices.length && <div className="grid grid-cols-2 gap-3">
<label>
音色
<select
aria-label="配音音色"
className={input}
value={voice || ""}
onChange={(e) => patch({ voice: e.target.value })}
>
{contract?.voices.map((v) => (
<option key={v.id} value={v.id}>
{v.name}
</option>
))}
</select>
</label>
{contract?.supportsLanguageControl !== false && <label>
语言
<select
aria-label="配音语言"
className={input}
value={draft.language}
onChange={(e) => patch({ language: e.target.value })}
>
<option value="Chinese">中文</option>
<option value="English">英文</option>
<option value="Auto">自动 / 中英混合</option>
</select>
</label>}
</div>}
{contract?.instructions ? (
<label className="block">
声音风格指令
<input
className={input}
aria-label="声音风格指令"
placeholder="例如:温暖自然,语速略慢"
maxLength={1200}
value={draft.instructions || ""}
onChange={(e) => patch({ instructions: e.target.value })}
/>
</label>
) : null}
{contract?.controls ? (
<div className="grid grid-cols-3 gap-2">
{(["rate", "pitch", "volume"] as const).map((key, i) => (
<label key={key}>
{["语速", "音调", "音量"][i]}
<input
aria-label={["语速", "音调", "音量"][i]}
className={input}
type="number"
min={key === "volume" ? 0 : 0.5}
max={key === "volume" ? 100 : 2}
step={key === "volume" ? 1 : 0.1}
value={draft[key]}
onChange={(e) => patch({ [key]: Number(e.target.value) })}
/>
</label>
))}
</div>
) : null}
</>
) : draft.mode === "music" ? (
<>
<label className="flex gap-2">
<input
type="checkbox"
checked={draft.instrumental}
onChange={(e) => patch({ instrumental: e.target.checked })}
/>
纯音乐(无人声)
</label>
{!draft.instrumental ? (
<>
<label className="block">
自定义歌词(选填,填写后优先于描述)
<textarea
aria-label="音乐歌词"
className={input}
rows={3}
maxLength={2000}
value={draft.lyrics || ""}
onChange={(e) => patch({ lyrics: e.target.value })}
/>
</label>
{contract?.supportsGender !== false && (!contract?.genderRequiresLyrics || draft.lyrics?.trim()) && <label>
演唱声音
<select
className={input}
aria-label="演唱声音"
value={draft.gender}
onChange={(e) => patch({ gender: e.target.value })}
>
<option value="female">女声</option>
<option value="male">男声</option>
</select>
</label>}
</>
) : null}
{!options.length ? (
<p className="text-sm text-amber-200">
当前没有可用的音乐模型,请在模型设置中启用。
</p>
) : null}
</>
) : null}
<div className="flex items-end gap-3">
{draft.mode !== "asr" && <label>
格式
<select
aria-label="音频格式"
className={input}
value={format || "wav"}
onChange={(e) => patch({ format: e.target.value })}
>
{(contract?.formats || ["wav"]).map((f) => (
<option key={f} value={f}>
{f.toUpperCase()}
</option>
))}
</select>
</label>}
<button
type="button"
disabled={!model || loading}
className="rounded-lg bg-white px-4 py-2 text-black disabled:opacity-40"
onClick={() => void run()}
>
{busy || sending ? "处理中…" : draft.mode === "asr" ? "识别文字" : "生成音频"}
</button>
<label className="cursor-pointer rounded-lg border border-white/20 px-3 py-2">
上传音频
<input
type="file"
accept="audio/*"
className="hidden"
onChange={async (e) => {
const file = e.target.files?.[0];
e.target.value = "";
if (!file) return;
setSending(true);
try {
await onUpload(file);
} catch (e) {
setError(e instanceof Error ? e.message : "上传失败");
} finally {
setSending(false);
}
}}
/>
</label>
</div>
</fieldset>
{error ? (
<p role="alert" className="mt-3 text-sm text-red-300">
{error}
</p>
) : null}
</section>
);
}