feat: support 98K image, video, chat and audio model contracts
This commit is contained in:
parent
1a682343c7
commit
ba24efc8e7
@ -9,10 +9,10 @@ export async function GET() {
|
|||||||
const models = await listCanvasAudioModels();
|
const models = await listCanvasAudioModels();
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{
|
{
|
||||||
models: models.map(({ model }) => ({
|
models: models.map(({ model, provider }) => ({
|
||||||
id: model.id,
|
id: model.id,
|
||||||
modelId: model.modelId,
|
modelId: model.modelId,
|
||||||
name: model.displayName,
|
name: `${model.displayName} · ${provider.type === "vip98k" ? "98K" : "阿里云"}`,
|
||||||
...audioContract(model.modelId),
|
...audioContract(model.modelId),
|
||||||
})),
|
})),
|
||||||
},
|
},
|
||||||
|
|||||||
@ -72,13 +72,13 @@ export function AudioGenerationComposer({
|
|||||||
setError("");
|
setError("");
|
||||||
setSending(true);
|
setSending(true);
|
||||||
try {
|
try {
|
||||||
if (!model) throw Error("当前模式没有已启用的官方模型");
|
if (!model) throw Error("当前模式没有已启用的模型");
|
||||||
const params = audioRequestSchema.parse({
|
const params = audioRequestSchema.parse({
|
||||||
...draft,
|
...draft,
|
||||||
text: [draft.text, connectedText].filter(Boolean).join("\n\n"),
|
text: [draft.text, connectedText].filter(Boolean).join("\n\n"),
|
||||||
modelRef: model.id,
|
modelRef: model.id,
|
||||||
voice,
|
voice,
|
||||||
format,
|
format: format || "wav",
|
||||||
...(!contract?.controls ? { rate: 1, pitch: 1, volume: 50 } : {}),
|
...(!contract?.controls ? { rate: 1, pitch: 1, volume: 50 } : {}),
|
||||||
...(!contract?.instructions ? { instructions: "" } : {}),
|
...(!contract?.instructions ? { instructions: "" } : {}),
|
||||||
});
|
});
|
||||||
@ -96,9 +96,9 @@ export function AudioGenerationComposer({
|
|||||||
<section
|
<section
|
||||||
className="fixed bottom-8 left-1/2 z-[100] max-h-[75vh] w-[min(620px,94vw)] -translate-x-1/2 overflow-y-auto rounded-2xl border border-white/10 bg-[#222] p-5 text-white shadow-xl"
|
className="fixed bottom-8 left-1/2 z-[100] max-h-[75vh] w-[min(620px,94vw)] -translate-x-1/2 overflow-y-auto rounded-2xl border border-white/10 bg-[#222] p-5 text-white shadow-xl"
|
||||||
data-canvas-overlay="true"
|
data-canvas-overlay="true"
|
||||||
aria-label="阿里云音频生成"
|
aria-label="语音与音乐"
|
||||||
>
|
>
|
||||||
<h3 className="font-semibold">语音与音乐 · 阿里云官方</h3>
|
<h3 className="font-semibold">语音与音乐</h3>
|
||||||
<fieldset disabled={busy || sending} className="mt-3 space-y-3 disabled:opacity-60">
|
<fieldset disabled={busy || sending} className="mt-3 space-y-3 disabled:opacity-60">
|
||||||
{connectedText && <p className="mb-3 whitespace-pre-wrap rounded bg-white/5 p-2 text-xs" aria-label="已连接文本">已连接文本:{connectedText}</p>}
|
{connectedText && <p className="mb-3 whitespace-pre-wrap rounded bg-white/5 p-2 text-xs" aria-label="已连接文本">已连接文本:{connectedText}</p>}
|
||||||
<div className="grid grid-cols-2 gap-3">
|
<div className="grid grid-cols-2 gap-3">
|
||||||
@ -112,6 +112,9 @@ export function AudioGenerationComposer({
|
|||||||
>
|
>
|
||||||
<option value="tts">文字配音</option>
|
<option value="tts">文字配音</option>
|
||||||
<option value="music">音乐 / 歌曲</option>
|
<option value="music">音乐 / 歌曲</option>
|
||||||
|
<option value="asr">语音转文字</option>
|
||||||
|
<option value="voiceclone">声音克隆配音</option>
|
||||||
|
<option value="voicedesign">声音设计配音</option>
|
||||||
</select>
|
</select>
|
||||||
</label>
|
</label>
|
||||||
<label>
|
<label>
|
||||||
@ -134,20 +137,40 @@ export function AudioGenerationComposer({
|
|||||||
</select>
|
</select>
|
||||||
</label>
|
</label>
|
||||||
</div>
|
</div>
|
||||||
<label className="block">
|
{["asr", "voiceclone"].includes(draft.mode) && (
|
||||||
{draft.mode === "tts" ? "配音文本(最多 600 字)" : "音乐描述"}
|
<label className="block">
|
||||||
|
{draft.mode === "asr" ? "待识别音频" : "参考声音"}(MP3 / WAV,最多 20 MiB)
|
||||||
|
<input type="file" accept="audio/mpeg,audio/wav,.mp3,.wav" aria-label="上传参考音频" className={input}
|
||||||
|
onChange={async event => {
|
||||||
|
const file = event.target.files?.[0]; event.target.value = ""; if (!file) return;
|
||||||
|
if (file.size > 20 * 1024 * 1024) { setError("参考音频需小于 20 MiB"); return; }
|
||||||
|
setSending(true); setError("");
|
||||||
|
try {
|
||||||
|
const form = new FormData(); form.append("file", file);
|
||||||
|
const response = await fetch("/api/canvas/media/upload", { method: "POST", body: form, signal: AbortSignal.timeout(120000) });
|
||||||
|
const result = await response.json(); const url = result.url || result.data?.url;
|
||||||
|
if (!response.ok || !url) throw Error(result.error || "参考音频上传失败");
|
||||||
|
patch({ referenceAudioUrl: String(url), referenceAudioName: file.name });
|
||||||
|
} catch (error) { setError(error instanceof Error ? error.message : "上传失败"); }
|
||||||
|
finally { setSending(false); }
|
||||||
|
}} />
|
||||||
|
{draft.referenceAudioUrl && <span className="mt-1 block text-sm text-white/80">已上传:{draft.referenceAudioName || "参考音频"}</span>}
|
||||||
|
</label>
|
||||||
|
)}
|
||||||
|
{draft.mode !== "asr" && <label className="block">
|
||||||
|
{draft.mode !== "music" ? "配音文本(最多 600 字)" : `音乐描述(最多 ${contract?.descriptionLimit || 2000} 字)`}
|
||||||
<textarea
|
<textarea
|
||||||
aria-label="音频内容"
|
aria-label="音频内容"
|
||||||
className={input}
|
className={input}
|
||||||
rows={4}
|
rows={4}
|
||||||
maxLength={draft.mode === "tts" ? 600 : 2000}
|
maxLength={draft.mode !== "music" ? 600 : contract?.descriptionLimit || 2000}
|
||||||
value={draft.text}
|
value={draft.text}
|
||||||
onChange={(e) => patch({ text: e.target.value })}
|
onChange={(e) => patch({ text: e.target.value })}
|
||||||
/>
|
/>
|
||||||
</label>
|
</label>}
|
||||||
{draft.mode === "tts" ? (
|
{!["music", "asr"].includes(draft.mode) ? (
|
||||||
<>
|
<>
|
||||||
<div className="grid grid-cols-2 gap-3">
|
{!!contract?.voices.length && <div className="grid grid-cols-2 gap-3">
|
||||||
<label>
|
<label>
|
||||||
音色
|
音色
|
||||||
<select
|
<select
|
||||||
@ -163,7 +186,7 @@ export function AudioGenerationComposer({
|
|||||||
))}
|
))}
|
||||||
</select>
|
</select>
|
||||||
</label>
|
</label>
|
||||||
<label>
|
{contract?.supportsLanguageControl !== false && <label>
|
||||||
语言
|
语言
|
||||||
<select
|
<select
|
||||||
aria-label="配音语言"
|
aria-label="配音语言"
|
||||||
@ -175,8 +198,8 @@ export function AudioGenerationComposer({
|
|||||||
<option value="English">英文</option>
|
<option value="English">英文</option>
|
||||||
<option value="Auto">自动 / 中英混合</option>
|
<option value="Auto">自动 / 中英混合</option>
|
||||||
</select>
|
</select>
|
||||||
</label>
|
</label>}
|
||||||
</div>
|
</div>}
|
||||||
{contract?.instructions ? (
|
{contract?.instructions ? (
|
||||||
<label className="block">
|
<label className="block">
|
||||||
声音风格指令
|
声音风格指令
|
||||||
@ -210,7 +233,7 @@ export function AudioGenerationComposer({
|
|||||||
</div>
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
</>
|
</>
|
||||||
) : (
|
) : draft.mode === "music" ? (
|
||||||
<>
|
<>
|
||||||
<label className="flex gap-2">
|
<label className="flex gap-2">
|
||||||
<input
|
<input
|
||||||
@ -233,7 +256,7 @@ export function AudioGenerationComposer({
|
|||||||
onChange={(e) => patch({ lyrics: e.target.value })}
|
onChange={(e) => patch({ lyrics: e.target.value })}
|
||||||
/>
|
/>
|
||||||
</label>
|
</label>
|
||||||
<label>
|
{contract?.supportsGender !== false && (!contract?.genderRequiresLyrics || draft.lyrics?.trim()) && <label>
|
||||||
演唱声音
|
演唱声音
|
||||||
<select
|
<select
|
||||||
className={input}
|
className={input}
|
||||||
@ -244,18 +267,18 @@ export function AudioGenerationComposer({
|
|||||||
<option value="female">女声</option>
|
<option value="female">女声</option>
|
||||||
<option value="male">男声</option>
|
<option value="male">男声</option>
|
||||||
</select>
|
</select>
|
||||||
</label>
|
</label>}
|
||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
{!options.length ? (
|
{!options.length ? (
|
||||||
<p className="text-sm text-amber-200">
|
<p className="text-sm text-amber-200">
|
||||||
Fun-Music 需要阿里云邀测权限,开通并启用模型后即可生成。
|
当前没有可用的音乐模型,请在模型设置中启用。
|
||||||
</p>
|
</p>
|
||||||
) : null}
|
) : null}
|
||||||
</>
|
</>
|
||||||
)}
|
) : null}
|
||||||
<div className="flex items-end gap-3">
|
<div className="flex items-end gap-3">
|
||||||
<label>
|
{draft.mode !== "asr" && <label>
|
||||||
格式
|
格式
|
||||||
<select
|
<select
|
||||||
aria-label="音频格式"
|
aria-label="音频格式"
|
||||||
@ -269,14 +292,14 @@ export function AudioGenerationComposer({
|
|||||||
</option>
|
</option>
|
||||||
))}
|
))}
|
||||||
</select>
|
</select>
|
||||||
</label>
|
</label>}
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
disabled={!model || loading}
|
disabled={!model || loading}
|
||||||
className="rounded-lg bg-white px-4 py-2 text-black disabled:opacity-40"
|
className="rounded-lg bg-white px-4 py-2 text-black disabled:opacity-40"
|
||||||
onClick={() => void run()}
|
onClick={() => void run()}
|
||||||
>
|
>
|
||||||
{busy || sending ? "处理中…" : "生成音频"}
|
{busy || sending ? "处理中…" : draft.mode === "asr" ? "识别文字" : "生成音频"}
|
||||||
</button>
|
</button>
|
||||||
<label className="cursor-pointer rounded-lg border border-white/20 px-3 py-2">
|
<label className="cursor-pointer rounded-lg border border-white/20 px-3 py-2">
|
||||||
上传音频
|
上传音频
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
import { createOpenAI } from "@ai-sdk/openai";
|
import { createOpenAI } from "@ai-sdk/openai";
|
||||||
|
import { createAnthropic } from "@ai-sdk/anthropic";
|
||||||
import { APICallError, type LanguageModelV2, type LanguageModelV2CallOptions } from "@ai-sdk/provider";
|
import { APICallError, type LanguageModelV2, type LanguageModelV2CallOptions } from "@ai-sdk/provider";
|
||||||
import { selectVip98kChatInvocation } from "../vip98k-chat-contract";
|
import { selectVip98kChatInvocation } from "../vip98k-chat-contract";
|
||||||
|
|
||||||
@ -128,6 +129,14 @@ export function createVip98kChatModel(params: {
|
|||||||
const root = params.baseURL.replace(/\/+$/, "").replace(/\/v1$/i, "");
|
const root = params.baseURL.replace(/\/+$/, "").replace(/\/v1$/i, "");
|
||||||
const baseFetch = params.fetch || fetch;
|
const baseFetch = params.fetch || fetch;
|
||||||
const createModel = (contract: typeof invocation) => {
|
const createModel = (contract: typeof invocation) => {
|
||||||
|
if (contract?.protocol === "anthropic-messages") {
|
||||||
|
return createAnthropic({
|
||||||
|
apiKey: params.apiKey,
|
||||||
|
baseURL: `${root}/v1`,
|
||||||
|
headers: { ...contract.headers, Authorization: `Bearer ${params.apiKey}` },
|
||||||
|
fetch: createChatFetch(root, baseFetch),
|
||||||
|
})(params.modelId);
|
||||||
|
}
|
||||||
const provider = createOpenAI({
|
const provider = createOpenAI({
|
||||||
apiKey: params.apiKey,
|
apiKey: params.apiKey,
|
||||||
baseURL: `${root}/v1`,
|
baseURL: `${root}/v1`,
|
||||||
@ -140,7 +149,8 @@ export function createVip98kChatModel(params: {
|
|||||||
};
|
};
|
||||||
const model = createModel(invocation);
|
const model = createModel(invocation);
|
||||||
const needsDiscovery = params.discoverLegacyContract &&
|
const needsDiscovery = params.discoverLegacyContract &&
|
||||||
!invocation?.headers["X-Relay-Route-Option"];
|
(!invocation?.headers["X-Relay-Route-Option"] ||
|
||||||
|
(!metadata.chatInvocation && invocation.protocol === "anthropic-messages"));
|
||||||
const resolveModel = async (signal?: AbortSignal) => {
|
const resolveModel = async (signal?: AbortSignal) => {
|
||||||
signal?.throwIfAborted();
|
signal?.throwIfAborted();
|
||||||
if (!needsDiscovery) return model;
|
if (!needsDiscovery) return model;
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
import { mapVip98kVideoParameters } from "../vip98k-video-parameters";
|
import { mapVip98kVideoParameters, bindVip98kVideoContent } from "../vip98k-video-parameters";
|
||||||
import {
|
import {
|
||||||
ProviderTaskFailedError,
|
ProviderTaskFailedError,
|
||||||
ProviderTaskUncertainError,
|
ProviderTaskUncertainError,
|
||||||
@ -205,8 +205,9 @@ function invocationSupportsBodyField(
|
|||||||
key: string,
|
key: string,
|
||||||
) {
|
) {
|
||||||
if (!invocation) return true;
|
if (!invocation) return true;
|
||||||
if (key in invocation.requestExample) return true;
|
|
||||||
const properties = toRecord(toRecord(invocation.inputSchema)?.properties);
|
const properties = toRecord(toRecord(invocation.inputSchema)?.properties);
|
||||||
|
if (properties?.[key] === false) return false;
|
||||||
|
if (key in invocation.requestExample) return true;
|
||||||
return Boolean(properties && key in properties);
|
return Boolean(properties && key in properties);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1574,7 +1575,7 @@ export function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvoca
|
|||||||
if ("displayName" in body || !invocation) {
|
if ("displayName" in body || !invocation) {
|
||||||
body.displayName = promptTitle || params.modelId;
|
body.displayName = promptTitle || params.modelId;
|
||||||
}
|
}
|
||||||
if ("prompt" in body || !invocation) body.prompt = params.prompt;
|
if (invocationSupportsBodyField(invocation, "prompt")) body.prompt = params.prompt;
|
||||||
if (params.kind === "image" && invocationSupportsBodyField(invocation, "n")) {
|
if (params.kind === "image" && invocationSupportsBodyField(invocation, "n")) {
|
||||||
body.n = imageParams.imageCount ?? imageParams.n ?? 1;
|
body.n = imageParams.imageCount ?? imageParams.n ?? 1;
|
||||||
}
|
}
|
||||||
@ -1627,14 +1628,14 @@ export function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvoca
|
|||||||
!invocationSupportsBodyField(invocation, "resolution")
|
!invocationSupportsBodyField(invocation, "resolution")
|
||||||
) {
|
) {
|
||||||
const preserveResolutionBucket =
|
const preserveResolutionBucket =
|
||||||
invocation?.path === "/v1/images/generations";
|
invocation?.path === "/v1/images/generations" &&
|
||||||
|
(!invocation.inputSchema || (toRecord(toRecord(invocation.inputSchema.properties)?.size)?.enum as unknown[] | undefined)?.includes(resolution));
|
||||||
const resolvedTaskSize = resolveVip98kImageTaskSize({
|
const resolvedTaskSize = resolveVip98kImageTaskSize({
|
||||||
imageSize: resolution,
|
imageSize: resolution,
|
||||||
aspectRatio,
|
aspectRatio,
|
||||||
fallbackSize: body.size,
|
fallbackSize: body.size,
|
||||||
preserveResolutionBucket,
|
preserveResolutionBucket,
|
||||||
useGptImage25AspectSizing:
|
useGptImage25AspectSizing:
|
||||||
preserveResolutionBucket &&
|
|
||||||
hasVerifiedVip98kOpenAiImageArrayContract(params.modelId),
|
hasVerifiedVip98kOpenAiImageArrayContract(params.modelId),
|
||||||
});
|
});
|
||||||
if (resolvedTaskSize) {
|
if (resolvedTaskSize) {
|
||||||
@ -1664,8 +1665,8 @@ export function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvoca
|
|||||||
if (value !== undefined && targetKey in body) body[targetKey] = value;
|
if (value !== undefined && targetKey in body) body[targetKey] = value;
|
||||||
}
|
}
|
||||||
if (typeof videoParams.generateAudio === "boolean") {
|
if (typeof videoParams.generateAudio === "boolean") {
|
||||||
if ("generateAudio" in body || !invocation) body.generateAudio = videoParams.generateAudio;
|
if (invocationSupportsBodyField(invocation, "generateAudio")) body.generateAudio = videoParams.generateAudio;
|
||||||
if ("generate_audio" in body || !invocation) body.generate_audio = videoParams.generateAudio;
|
if (invocationSupportsBodyField(invocation, "generate_audio")) body.generate_audio = videoParams.generateAudio;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reference fields can be optional in the schema and absent from its text-only example.
|
// Reference fields can be optional in the schema and absent from its text-only example.
|
||||||
@ -1694,6 +1695,11 @@ export function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvoca
|
|||||||
delete body.size;
|
delete body.size;
|
||||||
}
|
}
|
||||||
const imageBindingKeys = bindVip98kImages(body, params.imageUrls);
|
const imageBindingKeys = bindVip98kImages(body, params.imageUrls);
|
||||||
|
const imageProperties = toRecord(invocation?.inputSchema?.properties) || {};
|
||||||
|
for (const key of imageBindingKeys) {
|
||||||
|
const limit = toRecord(imageProperties[key])?.maxItems;
|
||||||
|
if (typeof limit === "number" && params.imageUrls.length > limit) throw Error(`当前模型最多支持 ${limit} 张参考图`);
|
||||||
|
}
|
||||||
if (params.imageUrls.length > 0 && !invocation && imageBindingKeys.length === 0) {
|
if (params.imageUrls.length > 0 && !invocation && imageBindingKeys.length === 0) {
|
||||||
body.images = params.imageUrls;
|
body.images = params.imageUrls;
|
||||||
imageBindingKeys.push("images");
|
imageBindingKeys.push("images");
|
||||||
@ -1728,7 +1734,46 @@ export function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvoca
|
|||||||
body.mode = params.imageUrls.length > 0 ? "image-to-video" : "text-to-video";
|
body.mode = params.imageUrls.length > 0 ? "image-to-video" : "text-to-video";
|
||||||
}
|
}
|
||||||
|
|
||||||
if (params.kind === "video") mapVip98kVideoParameters(body, invocation?.inputSchema);
|
if (params.kind === "video") {
|
||||||
|
const properties = toRecord(invocation?.inputSchema?.properties) || {};
|
||||||
|
if (params.imageUrls.length < Number(toRecord(properties.file_urls)?.minItems || 0)) throw Error("请先添加视频参考图");
|
||||||
|
if (bindVip98kVideoContent(body, invocation?.inputSchema, {
|
||||||
|
prompt: params.prompt, images: params.imageUrls, videos: params.videoUrls,
|
||||||
|
audios: Array.isArray(videoParams.referenceAudios) ? videoParams.referenceAudios as string[] : [],
|
||||||
|
mode: videoParams.mode,
|
||||||
|
}) && params.imageUrls.length) imageBindingKeys.push("content");
|
||||||
|
mapVip98kVideoParameters(body, invocation?.inputSchema);
|
||||||
|
}
|
||||||
|
if (params.kind === "image") {
|
||||||
|
const props = toRecord(invocation?.inputSchema?.properties) || {};
|
||||||
|
const sizeSchema = toRecord(props.size);
|
||||||
|
const pattern = typeof sizeSchema?.pattern === "string" ? new RegExp(sizeSchema.pattern) : null;
|
||||||
|
const sizes = Array.isArray(sizeSchema?.enum) ? sizeSchema.enum.filter(value => !pattern || pattern.test(String(value))) : undefined;
|
||||||
|
const ratios = Array.isArray(sizes) ? sizes.filter(value => typeof value === "string" && /^\d+:\d+$/.test(value)) : [];
|
||||||
|
const pixels = Array.isArray(sizes) ? sizes.flatMap(value => {
|
||||||
|
const match = String(value).match(/^(\d+)x(\d+)$/);
|
||||||
|
return match ? [{ size: value, width: Number(match[1]), height: Number(match[2]) }] : [];
|
||||||
|
}) : [];
|
||||||
|
const selectedRatio = aspectRatio || "1:1";
|
||||||
|
if (ratios.includes(selectedRatio)) body.size = selectedRatio;
|
||||||
|
else if (pixels.length && (resolution || aspectRatio)) {
|
||||||
|
const [rw, rh] = selectedRatio.split(":").map(Number);
|
||||||
|
const matching = pixels.filter(size => Math.abs(size.width / size.height - rw / rh) < 0.025);
|
||||||
|
if (!matching.length) throw Error("当前模型不支持所选图片比例");
|
||||||
|
const area = resolution === "4K" ? 2880 ** 2 : resolution === "2K" ? 2048 ** 2 : 1024 ** 2;
|
||||||
|
matching.sort((a, b) => Math.abs(a.width * a.height - area) - Math.abs(b.width * b.height - area));
|
||||||
|
body.size = matching[0].size;
|
||||||
|
}
|
||||||
|
if (ratios.length || pixels.length) {
|
||||||
|
const suffix = "\n\n[Output constraints]";
|
||||||
|
if (typeof body.prompt === "string" && body.prompt.includes(suffix)) body.prompt = body.prompt.split(suffix)[0];
|
||||||
|
}
|
||||||
|
for (const key of Object.keys(body)) if (props[key] === false) delete body[key];
|
||||||
|
if (body.orientation && aspectRatio) {
|
||||||
|
const [width, height] = aspectRatio.split(":").map(Number);
|
||||||
|
body.orientation = width > height ? "landscape" : "portrait";
|
||||||
|
}
|
||||||
|
}
|
||||||
return { body, imageBindingKeys, outputConstraintFallbacks };
|
return { body, imageBindingKeys, outputConstraintFallbacks };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -2,8 +2,8 @@ import { getVip98kInvocationHeaders } from "./vip98k-task-invocation";
|
|||||||
|
|
||||||
type RecordValue = Record<string, unknown>;
|
type RecordValue = Record<string, unknown>;
|
||||||
export type Vip98kChatInvocation = {
|
export type Vip98kChatInvocation = {
|
||||||
protocol: "openai-chat" | "openai-responses";
|
protocol: "openai-chat" | "openai-responses" | "anthropic-messages";
|
||||||
path: "/v1/chat/completions" | "/v1/responses";
|
path: "/v1/chat/completions" | "/v1/responses" | "/v1/messages";
|
||||||
headers: Record<string, string>;
|
headers: Record<string, string>;
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -43,21 +43,21 @@ export function selectVip98kChatInvocation(value: unknown): Vip98kChatInvocation
|
|||||||
return (!item.method || String(item.method).toUpperCase() === "POST") &&
|
return (!item.method || String(item.method).toUpperCase() === "POST") &&
|
||||||
(!item.kind || item.kind === "chat") &&
|
(!item.kind || item.kind === "chat") &&
|
||||||
(!modelId || !example?.model || example.model === modelId) &&
|
(!modelId || !example?.model || example.model === modelId) &&
|
||||||
(item.path === "/v1/chat/completions" || item.path === "/v1/responses");
|
["/v1/chat/completions", "/v1/responses", "/v1/messages"].includes(String(item.path));
|
||||||
});
|
});
|
||||||
// Prefer the published default route when it supports this protocol. An
|
// Prefer the published default route when it supports this protocol. An
|
||||||
// unscoped protocol example must not discard a route's execution contract.
|
// unscoped protocol example must not discard a route's execution contract.
|
||||||
candidates.sort((a, b) => {
|
candidates.sort((a, b) => {
|
||||||
const score = (item: RecordValue) => {
|
const score = (item: RecordValue) => {
|
||||||
const route = getVip98kInvocationHeaders(item.headers)["X-Relay-Route-Option"];
|
const route = getVip98kInvocationHeaders(item.headers)["X-Relay-Route-Option"];
|
||||||
return route && route === defaultRoute ? 2 : route ? 1 : 0;
|
return (route && route === defaultRoute ? 4 : route ? 2 : 0) + (item.path === "/v1/messages" ? 0 : 1);
|
||||||
};
|
};
|
||||||
return score(b) - score(a);
|
return score(b) - score(a);
|
||||||
});
|
});
|
||||||
const selected = candidates[0];
|
const selected = candidates[0];
|
||||||
if (!selected) return null;
|
if (!selected) return null;
|
||||||
return {
|
return {
|
||||||
protocol: selected.path === "/v1/responses" ? "openai-responses" : "openai-chat",
|
protocol: selected.path === "/v1/messages" ? "anthropic-messages" : selected.path === "/v1/responses" ? "openai-responses" : "openai-chat",
|
||||||
path: selected.path as Vip98kChatInvocation["path"],
|
path: selected.path as Vip98kChatInvocation["path"],
|
||||||
headers: getVip98kInvocationHeaders(selected.headers),
|
headers: getVip98kInvocationHeaders(selected.headers),
|
||||||
};
|
};
|
||||||
|
|||||||
@ -1,9 +1,10 @@
|
|||||||
import { canonicalVip98kVideoRatio, vip98kVideoDurationBounds } from "./vip98k-video-parameters";
|
import { canonicalVip98kVideoRatio, vip98kVideoDurationBounds, vip98kVideoContentTypes, vip98kVideoImageRoles } from "./vip98k-video-parameters";
|
||||||
import { VIP98K_TASK_POLL_SECONDS } from "./vip98k-runtime-config";
|
import { VIP98K_TASK_POLL_SECONDS } from "./vip98k-runtime-config";
|
||||||
import {
|
import {
|
||||||
hasVerifiedVip98kGptImage25ResolutionTiers,
|
hasVerifiedVip98kGptImage25ResolutionTiers,
|
||||||
hasVerifiedVip98kOpenAiImageArrayContract,
|
hasVerifiedVip98kOpenAiImageArrayContract,
|
||||||
normalizeVip98kInvocationHeaders,
|
normalizeVip98kInvocationHeaders,
|
||||||
|
getVip98kImageBindingKeys,
|
||||||
} from "@/lib/ai/vip98k-task-invocation";
|
} from "@/lib/ai/vip98k-task-invocation";
|
||||||
import { selectVip98kChatInvocation, vip98kChatSupportsVision } from "./vip98k-chat-contract";
|
import { selectVip98kChatInvocation, vip98kChatSupportsVision } from "./vip98k-chat-contract";
|
||||||
|
|
||||||
@ -349,7 +350,8 @@ function getInvocationRequestKeys(model: Vip98kCatalogModel) {
|
|||||||
return [
|
return [
|
||||||
...Object.keys(toRecord(invocation.request_example) || {}),
|
...Object.keys(toRecord(invocation.request_example) || {}),
|
||||||
...Object.keys(schemaProperties || {}),
|
...Object.keys(schemaProperties || {}),
|
||||||
].map(normalizeInvocationRequestKey);
|
].filter(key => schemaProperties?.[key] !== false && toRecord(schemaProperties?.[key])?.maxItems !== 0)
|
||||||
|
.map(normalizeInvocationRequestKey);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -427,7 +429,10 @@ function hasMediaReferenceInput(
|
|||||||
// published request body only accepts a text prompt.
|
// published request body only accepts a text prompt.
|
||||||
if (type === "image" && getInvocationCandidates(model).length > 0) {
|
if (type === "image" && getInvocationCandidates(model).length > 0) {
|
||||||
return (
|
return (
|
||||||
hasMediaRequestField ||
|
getInvocationCandidates(model).some(invocation => getVip98kImageBindingKeys({
|
||||||
|
requestExample: invocation.request_example || {},
|
||||||
|
inputSchema: invocation.input_schema,
|
||||||
|
} as any).length > 0) ||
|
||||||
hasVerifiedVip98kOpenAiImageArrayContract(modelId)
|
hasVerifiedVip98kOpenAiImageArrayContract(modelId)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -468,6 +473,7 @@ function hasMultiImageReferenceInput(model: Vip98kCatalogModel) {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
const schema = toRecord(schemaProperties[key]);
|
const schema = toRecord(schemaProperties[key]);
|
||||||
|
if (schemaProperties[key] === false || schema?.maxItems === 0) return false;
|
||||||
return (
|
return (
|
||||||
Array.isArray(requestExample[key]) ||
|
Array.isArray(requestExample[key]) ||
|
||||||
String(schema?.type || "").trim().toLowerCase() === "array"
|
String(schema?.type || "").trim().toLowerCase() === "array"
|
||||||
@ -507,7 +513,10 @@ export function isVip98kCatalogModelEligible(model: Vip98kCatalogModel) {
|
|||||||
if (!getModelId(model) || !isVip98kCatalogModelAvailable(model)) return false;
|
if (!getModelId(model) || !isVip98kCatalogModelAvailable(model)) return false;
|
||||||
const type = resolvePresetType(model);
|
const type = resolvePresetType(model);
|
||||||
if (type === "image") {
|
if (type === "image") {
|
||||||
return isVip98kMultiImageEditCatalogModel(model);
|
if (VIP98K_MULTI_IMAGE_MODEL_EXCLUSIONS.has(getModelId(model).toLowerCase())) return false;
|
||||||
|
return isVip98kMultiImageEditCatalogModel(model) ||
|
||||||
|
(isVip98kTextToImageCatalogModel(model) && getInvocationCandidates(model).some(invocation =>
|
||||||
|
invocation.path === "/v1/images/generations" || invocation.path === "/extensions/v1/tasks"));
|
||||||
}
|
}
|
||||||
if (type === "video") return hasMediaReferenceInput(model, type);
|
if (type === "video") return hasMediaReferenceInput(model, type);
|
||||||
return true;
|
return true;
|
||||||
@ -557,6 +566,9 @@ function getSupportedImageSizes(model: Vip98kCatalogModel) {
|
|||||||
if (hasVerifiedVip98kGptImage25ResolutionTiers(getModelId(model))) {
|
if (hasVerifiedVip98kGptImage25ResolutionTiers(getModelId(model))) {
|
||||||
return ["1K", "2K", "4K"];
|
return ["1K", "2K", "4K"];
|
||||||
}
|
}
|
||||||
|
const primary = toRecord(toRecord(model.invocation?.input_schema)?.properties) || {};
|
||||||
|
const resolutionEnum = parseStringArray(toRecord(primary.resolution)?.enum);
|
||||||
|
if (resolutionEnum.length) return resolutionEnum.map(normalizeImageSize).filter((size): size is string => Boolean(size));
|
||||||
const schemaSizes = getInvocationCandidates(model).flatMap((invocation) => {
|
const schemaSizes = getInvocationCandidates(model).flatMap((invocation) => {
|
||||||
const properties =
|
const properties =
|
||||||
toRecord(toRecord(invocation.input_schema)?.properties) || {};
|
toRecord(toRecord(invocation.input_schema)?.properties) || {};
|
||||||
@ -742,7 +754,11 @@ function createImagePreset(
|
|||||||
].some((key) => invocationKeys.has(key)) || hasVerifiedVip98kOpenAiImageArrayContract(modelId);
|
].some((key) => invocationKeys.has(key)) || hasVerifiedVip98kOpenAiImageArrayContract(modelId);
|
||||||
const supportsQuality = invocationKeys.has("quality");
|
const supportsQuality = invocationKeys.has("quality");
|
||||||
// Flare repeatedly returns 3:4/4:3 for 4:5/5:4, even with aspect_ratio supplied.
|
// Flare repeatedly returns 3:4/4:3 for 4:5/5:4, even with aspect_ratio supplied.
|
||||||
const supportedAspectRatios = supportsNativeAspectRatio
|
const declaredRatios = getInvocationCandidates(model).flatMap(invocation => {
|
||||||
|
const props = toRecord(toRecord(invocation.input_schema)?.properties) || {};
|
||||||
|
return ["aspect_ratio", "ratio", "size"].flatMap(key => parseStringArray(toRecord(props[key])?.enum).filter(value => /^\d+:\d+$/.test(value)));
|
||||||
|
});
|
||||||
|
const supportedAspectRatios = declaredRatios.length ? ["auto", ...new Set(declaredRatios)] : supportsNativeAspectRatio
|
||||||
? DEFAULT_ASPECT_RATIOS.filter((ratio) => modelId !== "gpt-image-2.5-flare" || !["4:5", "5:4"].includes(ratio))
|
? DEFAULT_ASPECT_RATIOS.filter((ratio) => modelId !== "gpt-image-2.5-flare" || !["4:5", "5:4"].includes(ratio))
|
||||||
: ["auto"];
|
: ["auto"];
|
||||||
const supportedModes = [
|
const supportedModes = [
|
||||||
@ -751,7 +767,14 @@ function createImagePreset(
|
|||||||
: []),
|
: []),
|
||||||
...(supportsReferenceImages ? ["image-to-image", "reference-images"] : []),
|
...(supportsReferenceImages ? ["image-to-image", "reference-images"] : []),
|
||||||
];
|
];
|
||||||
const maxReferenceImages = supportsMultiImageReference
|
const declaredReferenceLimits = getInvocationCandidates(model).flatMap(invocation => {
|
||||||
|
const properties = toRecord(toRecord(invocation.input_schema)?.properties) || {};
|
||||||
|
return ["file_urls", "image", "images", "reference_images"].flatMap(key => {
|
||||||
|
const limit = toRecord(properties[key])?.maxItems;
|
||||||
|
return typeof limit === "number" && limit > 0 ? [limit] : [];
|
||||||
|
});
|
||||||
|
});
|
||||||
|
const maxReferenceImages = declaredReferenceLimits.length ? Math.max(...declaredReferenceLimits) : supportsMultiImageReference
|
||||||
? 8
|
? 8
|
||||||
: supportsReferenceImages
|
: supportsReferenceImages
|
||||||
? 1
|
? 1
|
||||||
@ -763,7 +786,7 @@ function createImagePreset(
|
|||||||
|
|
||||||
return {
|
return {
|
||||||
modelId,
|
modelId,
|
||||||
displayName,
|
displayName: supportsReferenceImages ? displayName : `${displayName}(仅文生图)`,
|
||||||
description: model.description || `VIP98K image model (${modelId})`,
|
description: model.description || `VIP98K image model (${modelId})`,
|
||||||
type: "image",
|
type: "image",
|
||||||
supportsVision: supportsReferenceImages,
|
supportsVision: supportsReferenceImages,
|
||||||
@ -843,21 +866,27 @@ function createVideoPreset(
|
|||||||
: getDurations(model));
|
: getDurations(model));
|
||||||
const defaultDuration =
|
const defaultDuration =
|
||||||
parseNumber(model.defaultDuration || model.default_duration) ||
|
parseNumber(model.defaultDuration || model.default_duration) ||
|
||||||
durations[durations.length - 1] ||
|
parseNumber(invocation?.request_example?.duration_seconds) ||
|
||||||
|
(Number(invocation?.request_example?.duration) > 0 ? Number(invocation?.request_example?.duration) : 0) ||
|
||||||
|
durations[0] ||
|
||||||
8;
|
8;
|
||||||
const keys = new Set(getInvocationRequestKeys(model));
|
const keys = new Set(getInvocationRequestKeys(model));
|
||||||
const imageModes = parseStringArray(toRecord(schema.mode_image)?.enum);
|
const imageModes = parseStringArray(toRecord(schema.mode_image)?.enum);
|
||||||
const supportsReferenceImages = capabilities.includes("image-to-video") || keys.has("file-urls") || keys.has("image") || keys.has("image-url");
|
const contentTypes = vip98kVideoContentTypes(invocation?.input_schema);
|
||||||
const supportsFramePair = imageModes.includes("frame");
|
const contentRoles = vip98kVideoImageRoles(invocation?.input_schema);
|
||||||
const supportsIngredients = imageModes.includes("ingredient");
|
const supportsReferenceImages = contentTypes.length ? contentTypes.includes("image_url") : capabilities.includes("image-to-video") || keys.has("file-urls") || keys.has("image") || keys.has("image-url");
|
||||||
|
const supportsFirstFrame = supportsReferenceImages && (!contentRoles || contentRoles.includes("first_frame"));
|
||||||
|
const supportsFramePair = imageModes.includes("frame") || (contentTypes.includes("image_url") && (!contentRoles || contentRoles.includes("last_frame")));
|
||||||
|
const supportsIngredients = imageModes.includes("ingredient") || (contentTypes.includes("image_url") && (!contentRoles || contentRoles.includes("reference_image")));
|
||||||
|
const requiresReference = Number(toRecord(schema.file_urls)?.minItems) > 0;
|
||||||
const supportedModes = [
|
const supportedModes = [
|
||||||
...(capabilities.includes("text-to-video") || keys.has("prompt") ? ["text-to-video"] : []),
|
...(!requiresReference && (capabilities.includes("text-to-video") || keys.has("prompt") || (contentTypes.length === 1 && contentTypes[0] === "text")) ? ["text-to-video"] : []),
|
||||||
...(supportsReferenceImages ? ["first-frame"] : []),
|
...(supportsFirstFrame ? ["first-frame"] : []),
|
||||||
...(supportsFramePair ? ["first-last-frame"] : []),
|
...(supportsFramePair ? ["first-last-frame"] : []),
|
||||||
...(supportsIngredients ? ["reference-images"] : []),
|
...(supportsIngredients ? ["reference-images"] : []),
|
||||||
];
|
];
|
||||||
const ratios = parseStringArray(toRecord(schema.aspect_ratio)?.enum).map(canonicalVip98kVideoRatio);
|
const ratios = parseStringArray(toRecord(schema.aspect_ratio || schema.ratio)?.enum).map(canonicalVip98kVideoRatio).filter(value => value !== "adaptive");
|
||||||
const maxReferenceImages = Number(toRecord(schema.file_urls)?.maxItems) || (supportsReferenceImages ? 1 : 0);
|
const maxReferenceImages = Number(toRecord(schema.file_urls)?.maxItems) || (contentTypes.includes("image_url") ? 30 : supportsReferenceImages ? 1 : 0);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
modelId,
|
modelId,
|
||||||
|
|||||||
@ -220,9 +220,11 @@ export function getVip98kImageBindingKeys(invocation: Vip98kTaskInvocation | nul
|
|||||||
if (!invocation) return [];
|
if (!invocation) return [];
|
||||||
const body = invocation.requestExample || {};
|
const body = invocation.requestExample || {};
|
||||||
const properties = (invocation.inputSchema?.properties || {}) as Record<string, unknown>;
|
const properties = (invocation.inputSchema?.properties || {}) as Record<string, unknown>;
|
||||||
|
const accepts = (key: string) => properties[key] !== false &&
|
||||||
|
(properties[key] as { maxItems?: number } | undefined)?.maxItems !== 0;
|
||||||
return [
|
return [
|
||||||
...IMAGE_ARRAY_INPUT_KEYS.filter((key) => key in body || key in properties),
|
...IMAGE_ARRAY_INPUT_KEYS.filter((key) => accepts(key) && (key in body || key in properties)),
|
||||||
...IMAGE_SINGLE_INPUT_KEYS.filter((key) => key in body || key in properties),
|
...IMAGE_SINGLE_INPUT_KEYS.filter((key) => accepts(key) && (key in body || key in properties)),
|
||||||
...(Array.isArray(body.video_clips) ? ["video_clips" as const] : []),
|
...(Array.isArray(body.video_clips) ? ["video_clips" as const] : []),
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|||||||
@ -12,6 +12,11 @@ export function vip98kVideoDurationBounds(schema: unknown, resolution?: string)
|
|||||||
const visit = (value: unknown) => {
|
const visit = (value: unknown) => {
|
||||||
const current = object(value), properties = object(current.properties);
|
const current = object(value), properties = object(current.properties);
|
||||||
const field = object(properties.duration_seconds || properties.duration);
|
const field = object(properties.duration_seconds || properties.duration);
|
||||||
|
const positiveRange = (Array.isArray(field.anyOf) ? field.anyOf : []).find((x: any) => x.minimum > 0);
|
||||||
|
if (positiveRange) {
|
||||||
|
min = Math.max(min, positiveRange.minimum);
|
||||||
|
max = Math.min(max, positiveRange.maximum ?? Infinity);
|
||||||
|
}
|
||||||
if (Number.isFinite(field.minimum)) min = Math.max(min, field.minimum);
|
if (Number.isFinite(field.minimum)) min = Math.max(min, field.minimum);
|
||||||
if (Number.isFinite(field.maximum)) max = Math.min(max, field.maximum);
|
if (Number.isFinite(field.maximum)) max = Math.min(max, field.maximum);
|
||||||
const allowed = Array.isArray(field.enum) ? field.enum : Number.isFinite(field.const) ? [field.const] : undefined;
|
const allowed = Array.isArray(field.enum) ? field.enum : Number.isFinite(field.const) ? [field.const] : undefined;
|
||||||
@ -36,8 +41,11 @@ export function mapVip98kVideoParameters(body: Record<string, unknown>, schema:
|
|||||||
body.aspect_ratio = alias;
|
body.aspect_ratio = alias;
|
||||||
}
|
}
|
||||||
const resolutions = object(properties.resolution).enum;
|
const resolutions = object(properties.resolution).enum;
|
||||||
if (body.resolution && Array.isArray(resolutions) && !resolutions.includes(body.resolution))
|
if (body.resolution && Array.isArray(resolutions) && !resolutions.includes(body.resolution)) {
|
||||||
throw Error("供应商当前线路不支持所选视频清晰度,请同步模型规格后重试");
|
const matched = resolutions.find(value => String(value).toLowerCase() === String(body.resolution).toLowerCase());
|
||||||
|
if (!matched) throw Error("供应商当前线路不支持所选视频清晰度,请同步模型规格后重试");
|
||||||
|
body.resolution = matched;
|
||||||
|
}
|
||||||
const seconds = Number(body.duration_seconds ?? body.duration);
|
const seconds = Number(body.duration_seconds ?? body.duration);
|
||||||
if (seconds) {
|
if (seconds) {
|
||||||
const bounds = vip98kVideoDurationBounds(schema, String(body.resolution || ""));
|
const bounds = vip98kVideoDurationBounds(schema, String(body.resolution || ""));
|
||||||
@ -45,3 +53,45 @@ export function mapVip98kVideoParameters(body: Record<string, unknown>, schema:
|
|||||||
throw Error("供应商当前清晰度不支持所选视频时长,请调整镜头时长");
|
throw Error("供应商当前清晰度不支持所选视频时长,请调整镜头时长");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function vip98kVideoContentTypes(schema: unknown): string[] {
|
||||||
|
const content = object(object(object(schema).properties).content);
|
||||||
|
const items = object(content.items);
|
||||||
|
return (Array.isArray(items.oneOf) ? items.oneOf : []).flatMap((variant: unknown) => {
|
||||||
|
const type = object(object(object(variant).properties).type).const;
|
||||||
|
return typeof type === "string" ? [type] : [];
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function vip98kVideoImageRoles(schema: unknown): string[] | undefined {
|
||||||
|
const variants = object(object(object(object(schema).properties).content).items).oneOf;
|
||||||
|
const image = (Array.isArray(variants) ? variants : []).find(value => object(object(object(value).properties).type).const === "image_url");
|
||||||
|
const roles = object(object(object(image).properties).role).enum;
|
||||||
|
return Array.isArray(roles) ? roles.map(String) : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function bindVip98kVideoContent(body: Record<string, unknown>, schema: unknown, input: {
|
||||||
|
prompt: string; images: string[]; videos: string[]; audios: string[]; mode?: unknown;
|
||||||
|
}) {
|
||||||
|
const types = vip98kVideoContentTypes(schema);
|
||||||
|
if (!types.length) return false;
|
||||||
|
if (input.images.length && !types.includes("image_url")) throw Error("所选视频模型只支持文字生成,不能添加参考图片");
|
||||||
|
if (input.videos.length && !types.includes("video_url")) throw Error("所选模型不支持参考视频");
|
||||||
|
if (input.audios.length && !types.includes("audio_url")) throw Error("所选模型不支持参考音频");
|
||||||
|
const reference = input.mode === "reference-images";
|
||||||
|
const roles = vip98kVideoImageRoles(schema);
|
||||||
|
const role = reference ? "reference_image" : "first_frame";
|
||||||
|
if (input.images.length && roles && !roles.includes(role)) throw Error("所选视频模型不支持这个参考图模式");
|
||||||
|
if (!reference && input.images.length > 2) throw Error("首尾帧模式最多支持两张图片");
|
||||||
|
if (!reference && input.images.length === 2 && roles && !roles.includes("last_frame")) throw Error("所选视频模型不支持尾帧");
|
||||||
|
if (["first-frame", "first-last-frame", "reference-images"].includes(String(input.mode)) && !input.images.length) throw Error("请先添加视频参考图");
|
||||||
|
body.content = [
|
||||||
|
...(input.prompt ? [{ type: "text", text: input.prompt }] : []),
|
||||||
|
...input.images.map((url, index) => ({ type: "image_url", image_url: { url }, role: reference ? "reference_image" : index ? "last_frame" : "first_frame" })),
|
||||||
|
...input.videos.map(url => ({ type: "video_url", video_url: { url }, role: "reference_video" })),
|
||||||
|
...input.audios.map(url => ({ type: "audio_url", audio_url: { url }, role: "reference_audio" })),
|
||||||
|
];
|
||||||
|
delete body.prompt;
|
||||||
|
delete body.file_urls;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|||||||
@ -74,6 +74,14 @@ export async function getReadyQiniuImageDownload(url: string, target?: Awaited<R
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function archiveImageThroughQiniu(sourceUrl: string, archiveId: string) {
|
export async function archiveImageThroughQiniu(sourceUrl: string, archiveId: string) {
|
||||||
|
return archiveThroughQiniu(sourceUrl, archiveId, "image");
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function archiveMediaThroughQiniu(sourceUrl: string, archiveId: string, kind: "audio" | "video") {
|
||||||
|
return archiveThroughQiniu(sourceUrl, archiveId, kind);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function archiveThroughQiniu(sourceUrl: string, archiveId: string, kind: "image" | "audio" | "video") {
|
||||||
let source: URL;
|
let source: URL;
|
||||||
try { source = new URL(sourceUrl); } catch { return null; }
|
try { source = new URL(sourceUrl); } catch { return null; }
|
||||||
if (!/^https?:$/.test(source.protocol) || source.username || source.password) return null;
|
if (!/^https?:$/.test(source.protocol) || source.username || source.password) return null;
|
||||||
@ -90,8 +98,8 @@ export async function archiveImageThroughQiniu(sourceUrl: string, archiveId: str
|
|||||||
|| /^(?:127\.|10\.|192\.168\.|169\.254\.|172\.(?:1[6-9]|2\d|3[01])\.)/.test(source.hostname)) return null;
|
|| /^(?:127\.|10\.|192\.168\.|169\.254\.|172\.(?:1[6-9]|2\d|3[01])\.)/.test(source.hostname)) return null;
|
||||||
|
|
||||||
// The archive task owns this object key, including retries after a timeout.
|
// The archive task owns this object key, including retries after a timeout.
|
||||||
const pathname = `uploads/ai-generated-archive-${archiveId}`;
|
const pathname = kind === "image" ? `uploads/ai-generated-archive-${archiveId}` : `uploads/canvas-${kind}-${archiveId}`;
|
||||||
return fetchImageIntoQiniu(sourceUrl, pathname, target!);
|
return fetchImageIntoQiniu(sourceUrl, pathname, target!, kind);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Prepare uploaded WebP images inside Kodo, retaining their embedded colour profile. */
|
/** Prepare uploaded WebP images inside Kodo, retaining their embedded colour profile. */
|
||||||
@ -111,6 +119,7 @@ async function fetchImageIntoQiniu(
|
|||||||
sourceUrl: string,
|
sourceUrl: string,
|
||||||
pathname: string,
|
pathname: string,
|
||||||
target: NonNullable<Awaited<ReturnType<typeof resolveS3UploadTarget>>>,
|
target: NonNullable<Awaited<ReturnType<typeof resolveS3UploadTarget>>>,
|
||||||
|
kind: "image" | "audio" | "video" = "image",
|
||||||
) {
|
) {
|
||||||
const config = target.config;
|
const config = target.config;
|
||||||
const path = `/fetch/${encodeQiniu(sourceUrl)}/to/${encodeQiniu(`${config.bucket}:${pathname}`)}`;
|
const path = `/fetch/${encodeQiniu(sourceUrl)}/to/${encodeQiniu(`${config.bucket}:${pathname}`)}`;
|
||||||
@ -132,8 +141,8 @@ async function fetchImageIntoQiniu(
|
|||||||
throw new Error(`Qiniu archive fetch returned HTTP ${response.status}`);
|
throw new Error(`Qiniu archive fetch returned HTTP ${response.status}`);
|
||||||
}
|
}
|
||||||
const result = await response.json() as { key?: string; fsize?: number; mimeType?: string };
|
const result = await response.json() as { key?: string; fsize?: number; mimeType?: string };
|
||||||
if (result.key !== pathname || !result.fsize || !result.mimeType?.startsWith("image/")) {
|
if (result.key !== pathname || !result.fsize || !result.mimeType?.startsWith(`${kind}/`)) {
|
||||||
throw new Error("Qiniu archive fetch did not return an image");
|
throw new Error(`Qiniu archive fetch did not return ${kind}`);
|
||||||
}
|
}
|
||||||
const url = buildS3PublicUrl(config, pathname);
|
const url = buildS3PublicUrl(config, pathname);
|
||||||
return {
|
return {
|
||||||
|
|||||||
@ -156,6 +156,7 @@ export function isStyleProImageModel(
|
|||||||
entry: StyleProModelEntry | StyleProDisplayModelEntry
|
entry: StyleProModelEntry | StyleProDisplayModelEntry
|
||||||
) {
|
) {
|
||||||
if (!entry.model.supportsImageGen) return false;
|
if (!entry.model.supportsImageGen) return false;
|
||||||
|
if ((entry.model.imageGenConfig as Record<string, unknown> | null)?.maxReferenceImages === 0) return false;
|
||||||
if (isExplicitImageGenerationModel(entry)) return true;
|
if (isExplicitImageGenerationModel(entry)) return true;
|
||||||
return !isVideoLikeModel(entry);
|
return !isVideoLikeModel(entry);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
|
import { vip98kAudioContract, validateVip98kAudioInput, type AudioModelContract } from "./vip98k-audio-contract";
|
||||||
const qwenVoices = [
|
const qwenVoices = [
|
||||||
{ id: "Cherry", name: "芊悦" },
|
{ id: "Cherry", name: "芊悦" },
|
||||||
{ id: "Serena", name: "苏瑶" },
|
{ id: "Serena", name: "苏瑶" },
|
||||||
@ -65,7 +66,8 @@ export const audioModelContracts = {
|
|||||||
>;
|
>;
|
||||||
export type AudioModelId = keyof typeof audioModelContracts;
|
export type AudioModelId = keyof typeof audioModelContracts;
|
||||||
export const audioRequestSchema = z.object({
|
export const audioRequestSchema = z.object({
|
||||||
mode: z.enum(["tts", "music"]),
|
mode: z.enum(["tts", "music", "asr", "voiceclone", "voicedesign"]),
|
||||||
|
referenceAudioUrl: z.string().url().refine(value => /^https?:\/\//i.test(value)).optional(),
|
||||||
modelRef: z.string().min(1),
|
modelRef: z.string().min(1),
|
||||||
text: z.string().trim().max(2000).default(""),
|
text: z.string().trim().max(2000).default(""),
|
||||||
lyrics: z.string().trim().max(2000).default(""),
|
lyrics: z.string().trim().max(2000).default(""),
|
||||||
@ -94,10 +96,10 @@ export const audioRequestSchema = z.object({
|
|||||||
volume: z.number().int().min(0).max(100).default(50),
|
volume: z.number().int().min(0).max(100).default(50),
|
||||||
});
|
});
|
||||||
export type AudioRequest = z.infer<typeof audioRequestSchema>;
|
export type AudioRequest = z.infer<typeof audioRequestSchema>;
|
||||||
export function audioContract(id: string) {
|
export function audioContract(id: string): AudioModelContract | undefined {
|
||||||
return Object.hasOwn(audioModelContracts, id)
|
return Object.hasOwn(audioModelContracts, id)
|
||||||
? audioModelContracts[id as AudioModelId]
|
? audioModelContracts[id as AudioModelId]
|
||||||
: undefined;
|
: vip98kAudioContract(id);
|
||||||
}
|
}
|
||||||
export function officialAudioBase(base: string, modelId?: string) {
|
export function officialAudioBase(base: string, modelId?: string) {
|
||||||
const url = new URL(base);
|
const url = new URL(base);
|
||||||
@ -121,6 +123,10 @@ export function officialAudioBase(base: string, modelId?: string) {
|
|||||||
export function buildAudioRequest(value: unknown, modelId: string) {
|
export function buildAudioRequest(value: unknown, modelId: string) {
|
||||||
const p = audioRequestSchema.parse(value),
|
const p = audioRequestSchema.parse(value),
|
||||||
contract = audioContract(modelId);
|
contract = audioContract(modelId);
|
||||||
|
if (vip98kAudioContract(modelId)) {
|
||||||
|
validateVip98kAudioInput(p, modelId);
|
||||||
|
return { path: "", body: {} };
|
||||||
|
}
|
||||||
if (!contract || contract.kind !== p.mode) throw Error("音频模式与模型不匹配");
|
if (!contract || contract.kind !== p.mode) throw Error("音频模式与模型不匹配");
|
||||||
if (!contract.formats.includes(p.format)) throw Error("所选模型不支持这个音频格式");
|
if (!contract.formats.includes(p.format)) throw Error("所选模型不支持这个音频格式");
|
||||||
if (p.mode === "music") {
|
if (p.mode === "music") {
|
||||||
|
|||||||
@ -13,9 +13,12 @@ import {
|
|||||||
officialAudioBase,
|
officialAudioBase,
|
||||||
} from "./audio-contract";
|
} from "./audio-contract";
|
||||||
import { ProviderTaskUncertainError } from "./video-task-lifecycle";
|
import { ProviderTaskUncertainError } from "./video-task-lifecycle";
|
||||||
|
import { vip98kAudioContract } from "./vip98k-audio-contract";
|
||||||
|
import { executeVip98kCanvasAudio } from "./vip98k-audio-generation";
|
||||||
export async function listCanvasAudioModels() {
|
export async function listCanvasAudioModels() {
|
||||||
return (await getEnabledAiModelsWithProvider()).filter(({ model, provider }) => {
|
return (await getEnabledAiModelsWithProvider()).filter(({ model, provider }) => {
|
||||||
try {
|
try {
|
||||||
|
if (provider.type === "vip98k") return Boolean(vip98kAudioContract(model.modelId));
|
||||||
return (
|
return (
|
||||||
provider.type === "alibaba" &&
|
provider.type === "alibaba" &&
|
||||||
!!audioContract(model.modelId) &&
|
!!audioContract(model.modelId) &&
|
||||||
@ -35,6 +38,8 @@ export async function executeCanvasAudio(
|
|||||||
},
|
},
|
||||||
) {
|
) {
|
||||||
const input = audioRequestSchema.parse(params);
|
const input = audioRequestSchema.parse(params);
|
||||||
|
const selected = (await listCanvasAudioModels()).find(item => item.model.id === input.modelRef);
|
||||||
|
if (selected?.provider.type === "vip98k") return executeVip98kCanvasAudio(input, selected, lifecycle);
|
||||||
let checkpoint = lifecycle.resume ? audioCheckpointSchema.parse(lifecycle.resume) : undefined;
|
let checkpoint = lifecycle.resume ? audioCheckpointSchema.parse(lifecycle.resume) : undefined;
|
||||||
if (checkpoint && checkpoint.modelRef !== input.modelRef) throw Error("音频记录与原模型不匹配");
|
if (checkpoint && checkpoint.modelRef !== input.modelRef) throw Error("音频记录与原模型不匹配");
|
||||||
if (!checkpoint) {
|
if (!checkpoint) {
|
||||||
|
|||||||
@ -227,10 +227,9 @@ export function reconcileCanvasTasks<
|
|||||||
}
|
}
|
||||||
if (task.request.kind === "audio")
|
if (task.request.kind === "audio")
|
||||||
Object.assign(nextDraft, {
|
Object.assign(nextDraft, {
|
||||||
mediaUrl: data.url,
|
...(data.text ? { text: data.text, content: data.text, prompt: data.text, action: "TEXT_RESOURCE", mediaType: "text" } : {
|
||||||
url: [data.url],
|
mediaUrl: data.url, url: data.urls || [data.url], action: "AUDIO_RESOURCE", mediaType: "audio",
|
||||||
action: "AUDIO_RESOURCE",
|
}),
|
||||||
mediaType: "audio",
|
|
||||||
audioMeta: data.metadata,
|
audioMeta: data.metadata,
|
||||||
audioParams:
|
audioParams:
|
||||||
task.request.node.draftValues?.audioParams || task.request.params,
|
task.request.node.draftValues?.audioParams || task.request.params,
|
||||||
@ -243,16 +242,16 @@ export function reconcileCanvasTasks<
|
|||||||
});
|
});
|
||||||
if (
|
if (
|
||||||
task.request.kind === "image" ||
|
task.request.kind === "image" ||
|
||||||
task.request.kind === "image-operation"
|
task.request.kind === "image-operation" || (task.request.kind === "audio" && !data.text)
|
||||||
) {
|
) {
|
||||||
const urls = (data.images || [])
|
const urls = task.request.kind === "audio" ? data.urls || [data.url] : (data.images || [])
|
||||||
.map((image: any) => image.originalUrl || image.thumbnailUrl)
|
.map((image: any) => image.originalUrl || image.thumbnailUrl)
|
||||||
.filter(Boolean);
|
.filter(Boolean);
|
||||||
Object.assign(nextDraft, {
|
Object.assign(nextDraft, {
|
||||||
mediaUrl: urls[0],
|
mediaUrl: urls[0],
|
||||||
url: urls,
|
url: urls,
|
||||||
action:
|
action:
|
||||||
draft.action === "IMAGE_GENERATE_NINE_GRID"
|
task.request.kind === "audio" ? "AUDIO_RESOURCE" : draft.action === "IMAGE_GENERATE_NINE_GRID"
|
||||||
? draft.action
|
? draft.action
|
||||||
: "IMAGE_RESOURCE",
|
: "IMAGE_RESOURCE",
|
||||||
});
|
});
|
||||||
@ -263,6 +262,7 @@ export function reconcileCanvasTasks<
|
|||||||
...result,
|
...result,
|
||||||
{
|
{
|
||||||
...node,
|
...node,
|
||||||
|
...(task.request.kind === "audio" ? { kind: "media", inputType: "audio" } : {}),
|
||||||
id,
|
id,
|
||||||
title: `${node.title} ${i + 1}`,
|
title: `${node.title} ${i + 1}`,
|
||||||
y: node.y + i * (node.height + 40),
|
y: node.y + i * (node.height + 40),
|
||||||
@ -357,7 +357,7 @@ export function reconcileCanvasTasks<
|
|||||||
task.request.kind === "smart-edit"
|
task.request.kind === "smart-edit"
|
||||||
? "video"
|
? "video"
|
||||||
: task.request.kind === "audio"
|
: task.request.kind === "audio"
|
||||||
? "audio"
|
? task.result?.text ? "text" : "audio"
|
||||||
: "image",
|
: "image",
|
||||||
}
|
}
|
||||||
: {}),
|
: {}),
|
||||||
|
|||||||
@ -1,5 +1,6 @@
|
|||||||
import type { SmartEditCheckpoint } from "./smart-edit/execute";
|
import type { SmartEditCheckpoint } from "./smart-edit/execute";
|
||||||
import { audioCheckpointSchema } from "./audio-contract";
|
import { audioCheckpointSchema } from "./audio-contract";
|
||||||
|
import { vip98kAudioCheckpointSchema } from "./vip98k-audio-contract";
|
||||||
import { videoCheckpointSchema } from "./video-task-lifecycle";
|
import { videoCheckpointSchema } from "./video-task-lifecycle";
|
||||||
// Resume only server-recorded provider jobs. A lost request without an upstream ID is never resubmitted.
|
// Resume only server-recorded provider jobs. A lost request without an upstream ID is never resubmitted.
|
||||||
export type RecoveryTask = {
|
export type RecoveryTask = {
|
||||||
@ -49,7 +50,7 @@ export function getAudioCheckpoint(
|
|||||||
task: Pick<RecoveryTask, "request" | "result">
|
task: Pick<RecoveryTask, "request" | "result">
|
||||||
) {
|
) {
|
||||||
if (task.request.kind !== "audio") return null;
|
if (task.request.kind !== "audio") return null;
|
||||||
const parsed = audioCheckpointSchema.safeParse(task.result?.upstreamAudio);
|
const parsed = vip98kAudioCheckpointSchema.or(audioCheckpointSchema).safeParse(task.result?.upstreamAudio);
|
||||||
return parsed.success && parsed.data.modelRef === task.request.params.modelRef
|
return parsed.success && parsed.data.modelRef === task.request.params.modelRef
|
||||||
? parsed.data
|
? parsed.data
|
||||||
: null;
|
: null;
|
||||||
|
|||||||
106
lib/video-canvas/vip98k-audio-contract.ts
Normal file
106
lib/video-canvas/vip98k-audio-contract.ts
Normal file
@ -0,0 +1,106 @@
|
|||||||
|
import { z } from "zod";
|
||||||
|
|
||||||
|
export type AudioModelContract = {
|
||||||
|
kind: string; label: string; voices: { id: string; name: string }[];
|
||||||
|
instructions: boolean; controls: boolean; formats: string[]; supportsLanguageControl?: boolean;
|
||||||
|
supportsGender?: boolean; genderRequiresLyrics?: boolean; descriptionLimit?: number;
|
||||||
|
};
|
||||||
|
const voice = (id: string, name = id) => ({ id, name });
|
||||||
|
const tts = (label: string, voices: { id: string; name: string }[] = [], instructions = false): AudioModelContract =>
|
||||||
|
({ kind: "tts", label, voices, instructions, controls: false, formats: ["wav"], supportsLanguageControl: false });
|
||||||
|
const music = (label: string): AudioModelContract =>
|
||||||
|
({ kind: "music", label, voices: [], instructions: false, controls: false, formats: ["mp3"], supportsGender: false });
|
||||||
|
|
||||||
|
export const vip98kAudioContracts: Record<string, AudioModelContract> = {
|
||||||
|
"mimo-v2.5-asr": { ...tts("MiMo 语音识别"), kind: "asr", formats: [] },
|
||||||
|
"mimo-v2.5-tts": tts("MiMo 文字配音", [], true),
|
||||||
|
"mimo-v2.5-tts-voiceclone": { ...tts("MiMo 声音克隆", [], true), kind: "voiceclone" },
|
||||||
|
"mimo-v2.5-tts-voicedesign": { ...tts("MiMo 声音设计", [], true), kind: "voicedesign" },
|
||||||
|
"gem-2.5-tts": tts("Gemini 2.5 配音", [voice("Zephyr"), voice("Puck"), voice("Kore"), voice("Charon")]),
|
||||||
|
"gem-3.1-tts": tts("Gemini 3.1 配音", [voice("Zephyr"), voice("Puck"), voice("Kore"), voice("Charon")]),
|
||||||
|
"speech-2.8-hd": { ...tts("Speech 2.8 高清", [voice("male-qn-qingse", "青涩男声")]), formats: ["mp3", "wav"], controls: true },
|
||||||
|
"speech-2.8-turbo": { ...tts("Speech 2.8 快速", [voice("male-qn-qingse", "青涩男声")]), formats: ["mp3", "wav"], controls: true },
|
||||||
|
"music-2.5": music("Music 2.5"),
|
||||||
|
"music-2.5+": music("Music 2.5+"),
|
||||||
|
"suno-v4.5": { ...music("Suno 4.5"), supportsGender: true, descriptionLimit: 1000 },
|
||||||
|
"suno-v5": { ...music("Suno 5"), supportsGender: true, genderRequiresLyrics: true, descriptionLimit: 500 },
|
||||||
|
"suno-v5-5": { ...music("Suno 5.5"), supportsGender: true, genderRequiresLyrics: true, descriptionLimit: 500 },
|
||||||
|
"suno-v6": { ...music("Suno 6"), supportsGender: true, descriptionLimit: 1000 },
|
||||||
|
};
|
||||||
|
export function vip98kAudioContract(id: string) {
|
||||||
|
return Object.hasOwn(vip98kAudioContracts, id) ? vip98kAudioContracts[id] : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type Vip98kAudioInput = {
|
||||||
|
mode: string; text: string; lyrics: string; instructions: string; voice?: string;
|
||||||
|
format: string; instrumental: boolean; gender: string; rate: number; pitch: number;
|
||||||
|
volume: number; referenceAudioUrl?: string;
|
||||||
|
};
|
||||||
|
export function validateVip98kAudioInput(input: Vip98kAudioInput, modelId: string) {
|
||||||
|
const contract = vip98kAudioContract(modelId);
|
||||||
|
if (!contract || contract.kind !== input.mode) throw new Error("音频模式与模型不匹配");
|
||||||
|
if (["asr", "voiceclone"].includes(input.mode) && !input.referenceAudioUrl) throw new Error("请先上传参考音频");
|
||||||
|
if (input.mode !== "asr" && !input.text.trim() && !input.lyrics.trim()) throw new Error("请输入配音文本或音乐描述");
|
||||||
|
if (input.mode === "voicedesign" && !input.instructions.trim()) throw new Error("请填写声音风格指令");
|
||||||
|
if (contract.descriptionLimit && input.text.length > contract.descriptionLimit) throw new Error(`这个模型的音乐描述最多 ${contract.descriptionLimit} 字`);
|
||||||
|
if (input.mode !== "asr" && !contract.formats.includes(input.format)) throw new Error("所选模型不支持这个音频格式");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildVip98kAudioBody(input: Vip98kAudioInput, modelId: string, example: Record<string, any>, audioData?: string) {
|
||||||
|
validateVip98kAudioInput(input, modelId);
|
||||||
|
const body = structuredClone(example);
|
||||||
|
body.model = modelId;
|
||||||
|
if (modelId.startsWith("mimo-")) {
|
||||||
|
body.stream = false;
|
||||||
|
if (input.mode === "asr") {
|
||||||
|
if (!audioData) throw new Error("参考音频尚未读取");
|
||||||
|
body.messages = [{ role: "user", content: [{ type: "input_audio", input_audio: { data: audioData, format: audioData.includes("audio/wav") ? "wav" : "mp3" } }] }];
|
||||||
|
} else {
|
||||||
|
body.messages = [
|
||||||
|
{ role: "user", content: input.instructions || "请用自然的中文朗读。" },
|
||||||
|
{ role: "assistant", content: input.text },
|
||||||
|
];
|
||||||
|
if (input.mode === "voiceclone") {
|
||||||
|
if (!audioData) throw new Error("参考音频尚未读取");
|
||||||
|
body.audio = { ...body.audio, voice: audioData };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return body;
|
||||||
|
}
|
||||||
|
body.prompt = input.text;
|
||||||
|
const set = (key: string, value: unknown) => {
|
||||||
|
if (body.metadata && key in body.metadata) body.metadata[key] = value;
|
||||||
|
else body[key] = value;
|
||||||
|
};
|
||||||
|
if (modelId.startsWith("gem-")) set("voice_id", input.voice || "Zephyr");
|
||||||
|
if (modelId.startsWith("speech-")) {
|
||||||
|
set("voice_id", input.voice || "male-qn-qingse");
|
||||||
|
set("speed", input.rate);
|
||||||
|
set("pitch", Math.round((input.pitch - 1) * 12));
|
||||||
|
set("vol", Math.max(0.01, input.volume / 50));
|
||||||
|
set("format", input.format);
|
||||||
|
}
|
||||||
|
if (input.mode === "music") {
|
||||||
|
if (modelId === "suno-v5" || modelId === "suno-v5-5") {
|
||||||
|
body.instrumental = input.instrumental;
|
||||||
|
body.custom_mode = Boolean(input.lyrics && !input.instrumental);
|
||||||
|
if (body.custom_mode) { body.prompt = input.lyrics; body.style = input.text; body.title = input.text.slice(0, 60) || "歌曲"; }
|
||||||
|
if (body.custom_mode) set("vocal_gender", input.gender === "male" ? "m" : "f");
|
||||||
|
else { delete body.vocal_gender; if (body.metadata) delete body.metadata.vocal_gender; }
|
||||||
|
} else {
|
||||||
|
set(modelId.startsWith("music-") ? "is_instrumental" : "make_instrumental", input.instrumental ? "instrumental" : "song");
|
||||||
|
if (!input.instrumental && input.lyrics) set("lyrics", input.lyrics);
|
||||||
|
if (modelId.startsWith("suno-")) set("vocal_gender", input.gender === "male" ? "m" : "f");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return body;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const vip98kAudioCheckpointSchema = z.object({
|
||||||
|
operation: z.literal("audio"), provider: z.literal("vip98k"),
|
||||||
|
modelRef: z.string(), modelId: z.string(), providerId: z.string(), baseUrl: z.string(),
|
||||||
|
taskId: z.string().optional(), resultPath: z.string().optional(), requestId: z.string().optional(),
|
||||||
|
url: z.string().url().optional(), urls: z.array(z.string().url()).optional(), archivedUrls: z.array(z.string().url()).optional(),
|
||||||
|
data: z.string().optional(), text: z.string().optional(),
|
||||||
|
format: z.enum(["mp3", "wav"]),
|
||||||
|
}).refine(value => Boolean(value.taskId || value.url || value.data || value.text));
|
||||||
24
lib/video-canvas/vip98k-audio-file.ts
Normal file
24
lib/video-canvas/vip98k-audio-file.ts
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
function format(bytes: Buffer): "mp3" | "wav" | undefined {
|
||||||
|
if (bytes.length < 12) return;
|
||||||
|
if (bytes.toString("ascii", 0, 4) === "RIFF" && bytes.toString("ascii", 8, 12) === "WAVE") return "wav";
|
||||||
|
if (bytes.toString("ascii", 0, 3) === "ID3" || (bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0)) return "mp3";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function readVip98kAudioFile(bytes: Buffer) {
|
||||||
|
const direct = format(bytes);
|
||||||
|
if (direct) return { bytes, format: direct };
|
||||||
|
// Speech 2.8 returns a TAR containing audio and captions. Read the audio
|
||||||
|
// directly from memory; archive member paths are never written to disk.
|
||||||
|
if (bytes.toString("ascii", 257, 262) === "ustar") {
|
||||||
|
for (let offset = 0; offset + 512 <= bytes.length;) {
|
||||||
|
const header = bytes.subarray(offset, offset + 512);
|
||||||
|
const size = Number.parseInt(header.toString("ascii", 124, 136).replace(/\0.*$/, "").trim(), 8);
|
||||||
|
if (!Number.isFinite(size) || size < 0 || offset + 512 + size > bytes.length) break;
|
||||||
|
const data = bytes.subarray(offset + 512, offset + 512 + size);
|
||||||
|
const memberFormat = format(data);
|
||||||
|
if ((header[156] === 0 || header[156] === 48) && memberFormat) return { bytes: data, format: memberFormat };
|
||||||
|
offset += 512 + Math.ceil(size / 512) * 512;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new Error("返回结果不是有效的 MP3 或 WAV 音频文件");
|
||||||
|
}
|
||||||
133
lib/video-canvas/vip98k-audio-generation.ts
Normal file
133
lib/video-canvas/vip98k-audio-generation.ts
Normal file
@ -0,0 +1,133 @@
|
|||||||
|
import "server-only";
|
||||||
|
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
import { executeVip98kExternalTask } from "@/lib/ai/providers/vip98k";
|
||||||
|
import { getVip98kInvocationHeaders } from "@/lib/ai/vip98k-task-invocation";
|
||||||
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
|
import { archiveMediaThroughQiniu } from "@/lib/storage/qiniu-image-archive";
|
||||||
|
import { downloadCanvasMedia } from "./download-media";
|
||||||
|
import { ProviderTaskUncertainError } from "./video-task-lifecycle";
|
||||||
|
import { buildVip98kAudioBody, vip98kAudioCheckpointSchema } from "./vip98k-audio-contract";
|
||||||
|
import { readVip98kAudioFile } from "./vip98k-audio-file";
|
||||||
|
import type { AudioRequest } from "./audio-contract";
|
||||||
|
import type { AiModel, AiProvider } from "@/lib/db/schema";
|
||||||
|
|
||||||
|
export async function executeVip98kCanvasAudio(input: AudioRequest, item: { model: AiModel; provider: AiProvider }, lifecycle: {
|
||||||
|
resume?: unknown; assertActive: () => Promise<void>; checkpoint: (value: Record<string, unknown>) => Promise<void>;
|
||||||
|
}) {
|
||||||
|
const { model, provider } = item;
|
||||||
|
const metadata = (model.capabilities as any)?.vip98k;
|
||||||
|
const invocation = metadata?.invocation;
|
||||||
|
if (!invocation?.path) throw new Error("音频模型缺少调用配置,请同步模型目录");
|
||||||
|
const identity = { operation: "audio" as const, provider: "vip98k" as const,
|
||||||
|
modelRef: model.id, modelId: model.modelId, providerId: provider.id,
|
||||||
|
baseUrl: provider.baseUrl.replace(/\/+$/, "").replace(/\/v1$/i, ""), format: input.format };
|
||||||
|
let saved = lifecycle.resume ? vip98kAudioCheckpointSchema.parse(lifecycle.resume) : undefined;
|
||||||
|
if (saved && Object.entries(identity).some(([key, value]) => (saved as any)[key] !== value))
|
||||||
|
throw new ProviderTaskUncertainError("原音频任务的服务线路或模型已变更,请核对原任务");
|
||||||
|
const save = async (value: typeof saved) => {
|
||||||
|
await lifecycle.assertActive();
|
||||||
|
await lifecycle.checkpoint({ upstreamAudio: value });
|
||||||
|
saved = value;
|
||||||
|
};
|
||||||
|
const dir = await mkdtemp(join(tmpdir(), "canvas-98k-audio-"));
|
||||||
|
try {
|
||||||
|
if (!saved?.url && !saved?.data && !saved?.text) {
|
||||||
|
let audioData: string | undefined;
|
||||||
|
if (!saved && input.referenceAudioUrl) {
|
||||||
|
const reference = join(dir, "reference");
|
||||||
|
await downloadCanvasMedia(input.referenceAudioUrl, reference);
|
||||||
|
const bytes = await readFile(reference);
|
||||||
|
if (bytes.length > 20 * 1024 * 1024) throw new Error("参考音频需小于 20 MiB");
|
||||||
|
const referenceAudio = readVip98kAudioFile(bytes);
|
||||||
|
audioData = `data:audio/${referenceAudio.format === "mp3" ? "mpeg" : "wav"};base64,${referenceAudio.bytes.toString("base64")}`;
|
||||||
|
}
|
||||||
|
const body = saved ? undefined : buildVip98kAudioBody(input, model.modelId, invocation.requestExample || {}, audioData);
|
||||||
|
if (invocation.path === "/v1/chat/completions") {
|
||||||
|
await lifecycle.assertActive();
|
||||||
|
let response: Response;
|
||||||
|
let payload: any;
|
||||||
|
try {
|
||||||
|
response = await fetch(identity.baseUrl + invocation.path, {
|
||||||
|
method: "POST", redirect: "error", signal: AbortSignal.timeout(180000),
|
||||||
|
headers: { Authorization: `Bearer ${provider.apiKey}`, "Content-Type": "application/json",
|
||||||
|
...getVip98kInvocationHeaders(invocation.headers), "Idempotency-Key": randomUUID() },
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
});
|
||||||
|
payload = await response.json();
|
||||||
|
} catch {
|
||||||
|
throw new ProviderTaskUncertainError("98K 音频请求结果未确认,请核对原请求,系统不会自动重发");
|
||||||
|
}
|
||||||
|
if (!response.ok) {
|
||||||
|
if (response.status >= 500) throw new ProviderTaskUncertainError(`98K 音频结果未确认(HTTP ${response.status}),未重复提交`);
|
||||||
|
throw new Error(`98K 音频请求失败(HTTP ${response.status}),请核对模型和输入`);
|
||||||
|
}
|
||||||
|
const message = payload.choices?.[0]?.message;
|
||||||
|
const text = typeof message?.content === "string" ? message.content.trim() : "";
|
||||||
|
const audio = message?.audio;
|
||||||
|
const next = input.mode === "asr" ? { text } : { data: audio?.data, url: audio?.url };
|
||||||
|
if (!next.text && !next.data && !next.url) throw new ProviderTaskUncertainError("98K 未返回音频结果,请核对原请求,勿重复生成");
|
||||||
|
try { await save({ ...identity, ...next, requestId: payload.id }); }
|
||||||
|
catch { throw new ProviderTaskUncertainError("音频已生成,但结果保存失败,请核对原请求"); }
|
||||||
|
} else {
|
||||||
|
const result = await executeVip98kExternalTask({
|
||||||
|
apiKey: provider.apiKey, baseURL: identity.baseUrl, modelId: model.modelId, kind: "audio", prompt: String(body?.prompt || input.text),
|
||||||
|
modelConfig: { capabilities: { ...model.capabilities as any, vip98k: { ...metadata,
|
||||||
|
invocation: { ...invocation, requestExample: body || invocation.requestExample },
|
||||||
|
invocations: [],
|
||||||
|
} } },
|
||||||
|
lifecycle: { assertActive: lifecycle.assertActive,
|
||||||
|
resume: saved?.taskId ? { taskId: saved.taskId, resultPath: saved.resultPath } : undefined,
|
||||||
|
accepted: reference => save({ ...identity, ...reference }),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const payload = result.payload as any;
|
||||||
|
const outputs = payload?.result?.outputs || payload?.outputs;
|
||||||
|
const audioUrls = Array.isArray(outputs) ? outputs.filter(output => output.type === "audio" && typeof output.url === "string").map(output => output.url as string) : [];
|
||||||
|
const urls: string[] = audioUrls.length ? audioUrls : result.outputUrls.slice(0, 1);
|
||||||
|
const url = urls[0];
|
||||||
|
if (!url) throw new ProviderTaskUncertainError("音频任务已完成但未返回文件,请核对原任务");
|
||||||
|
await save({ ...identity, taskId: result.taskId, url, urls });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (saved?.text) return { text: saved.text, modelRef: model.id, modelId: model.modelId, metadata: { outputType: "text", requestId: saved.requestId } };
|
||||||
|
const sources = saved?.urls?.length ? saved.urls : [saved?.url];
|
||||||
|
const archivedUrls = [...(saved?.archivedUrls || [])];
|
||||||
|
let format = saved!.format;
|
||||||
|
for (let index = archivedUrls.length; index < sources.length; index++) {
|
||||||
|
if (sources[index] && !model.modelId.startsWith("speech-")) {
|
||||||
|
const archived = await archiveMediaThroughQiniu(sources[index]!, `${saved!.taskId || saved!.requestId}-${index}`, "audio");
|
||||||
|
if (archived) {
|
||||||
|
const response = await fetch(archived.url, { headers: { Range: "bytes=0-1023" }, signal: AbortSignal.timeout(20000) });
|
||||||
|
if (!response.ok) throw new Error("音频文件尚不可下载");
|
||||||
|
const reader = response.body!.getReader();
|
||||||
|
const first = await reader.read(); await reader.cancel();
|
||||||
|
format = readVip98kAudioFile(Buffer.from(first.value || [])).format;
|
||||||
|
archivedUrls.push(archived.url);
|
||||||
|
await save({ ...saved!, archivedUrls });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let bytes: Buffer;
|
||||||
|
if (saved?.data) bytes = Buffer.from(saved.data.replace(/^data:[^,]+,/, ""), "base64");
|
||||||
|
else {
|
||||||
|
const output = join(dir, `result-${index}`);
|
||||||
|
await downloadCanvasMedia(sources[index]!, output);
|
||||||
|
bytes = await readFile(output);
|
||||||
|
}
|
||||||
|
const audio = readVip98kAudioFile(bytes);
|
||||||
|
format = audio.format;
|
||||||
|
await lifecycle.assertActive();
|
||||||
|
const result = await uploadFile(`canvas-audio-${randomUUID()}.${format}`, Uint8Array.from(audio.bytes).buffer);
|
||||||
|
archivedUrls.push(result.url);
|
||||||
|
await save({ ...saved!, archivedUrls });
|
||||||
|
}
|
||||||
|
return { url: archivedUrls[0], urls: archivedUrls, modelRef: model.id, modelId: model.modelId,
|
||||||
|
metadata: { storageStatus: "archived", format, requestId: saved?.requestId, taskId: saved?.taskId } };
|
||||||
|
} catch (error) {
|
||||||
|
if (saved?.url || saved?.data) throw new ProviderTaskUncertainError("音频已生成,保存文件尚未完成,将继续保存原文件");
|
||||||
|
throw error;
|
||||||
|
} finally { await rm(dir, { recursive: true, force: true }); }
|
||||||
|
}
|
||||||
@ -65,6 +65,7 @@
|
|||||||
"providers:test-vip98k-images": "pnpm exec tsx scripts/test-vip98k-image-models.ts"
|
"providers:test-vip98k-images": "pnpm exec tsx scripts/test-vip98k-image-models.ts"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@ai-sdk/anthropic": "2.0.13",
|
||||||
"@ai-sdk/gateway": "^1.0.15",
|
"@ai-sdk/gateway": "^1.0.15",
|
||||||
"@ai-sdk/openai": "2.0.13",
|
"@ai-sdk/openai": "2.0.13",
|
||||||
"@ai-sdk/provider": "2.0.0",
|
"@ai-sdk/provider": "2.0.0",
|
||||||
|
|||||||
@ -72,6 +72,7 @@ export default defineConfig({
|
|||||||
/routes\/hicad-manufacturing-extension\.test\.ts/,
|
/routes\/hicad-manufacturing-extension\.test\.ts/,
|
||||||
/routes\/openapi-access-key\.test\.ts/,
|
/routes\/openapi-access-key\.test\.ts/,
|
||||||
/routes\/vip98k-reference-routing\.test\.ts/,
|
/routes\/vip98k-reference-routing\.test\.ts/,
|
||||||
|
/routes\/vip98k-model-expansion\.test\.ts/,
|
||||||
/routes\/ordinal-image-references\.test\.ts/,
|
/routes\/ordinal-image-references\.test\.ts/,
|
||||||
/routes\/extend-provider-config\.test\.ts/,
|
/routes\/extend-provider-config\.test\.ts/,
|
||||||
],
|
],
|
||||||
|
|||||||
32
pnpm-lock.yaml
generated
32
pnpm-lock.yaml
generated
@ -33,6 +33,9 @@ importers:
|
|||||||
|
|
||||||
.:
|
.:
|
||||||
dependencies:
|
dependencies:
|
||||||
|
'@ai-sdk/anthropic':
|
||||||
|
specifier: 2.0.13
|
||||||
|
version: 2.0.13(zod@3.25.76)
|
||||||
'@ai-sdk/gateway':
|
'@ai-sdk/gateway':
|
||||||
specifier: ^1.0.15
|
specifier: ^1.0.15
|
||||||
version: 1.0.41(zod@3.25.76)
|
version: 1.0.41(zod@3.25.76)
|
||||||
@ -463,6 +466,20 @@ importers:
|
|||||||
|
|
||||||
packages:
|
packages:
|
||||||
|
|
||||||
|
'@ai-sdk/anthropic@2.0.13':
|
||||||
|
resolution: {integrity: sha512-5Kt1sNy80pKJsThe3snL9OajgNfdjH7GiWj/CyDoMNY82+Pb5mHimCwZzMUUoAM77GjfSd3Ico8sE7pPdJOuYQ==}
|
||||||
|
engines: {node: '>=18'}
|
||||||
|
peerDependencies:
|
||||||
|
zod: ^3.25.76 || ^4
|
||||||
|
|
||||||
|
|
||||||
|
'@ai-sdk/provider-utils@3.0.8':
|
||||||
|
resolution: {integrity: sha512-cDj1iigu7MW2tgAQeBzOiLhjHOUM9vENsgh4oAVitek0d//WdgfPCsKO3euP7m7LyO/j9a1vr/So+BGNdpFXYw==}
|
||||||
|
engines: {node: '>=18'}
|
||||||
|
peerDependencies:
|
||||||
|
zod: ^3.25.76 || ^4
|
||||||
|
|
||||||
|
|
||||||
gsap@3.14.2:
|
gsap@3.14.2:
|
||||||
resolution: {integrity: sha512-P8/mMxVLU7o4+55+1TCnQrPmgjPKnwkzkXOK1asnR9Jg2lna4tEY5qBJjMmAaOBDDZWtlRjBXjLa0w53G/uBLA==}
|
resolution: {integrity: sha512-P8/mMxVLU7o4+55+1TCnQrPmgjPKnwkzkXOK1asnR9Jg2lna4tEY5qBJjMmAaOBDDZWtlRjBXjLa0w53G/uBLA==}
|
||||||
|
|
||||||
@ -6973,6 +6990,21 @@ packages:
|
|||||||
|
|
||||||
snapshots:
|
snapshots:
|
||||||
|
|
||||||
|
'@ai-sdk/anthropic@2.0.13(zod@3.25.76)':
|
||||||
|
dependencies:
|
||||||
|
'@ai-sdk/provider': 2.0.0
|
||||||
|
'@ai-sdk/provider-utils': 3.0.8(zod@3.25.76)
|
||||||
|
zod: 3.25.76
|
||||||
|
|
||||||
|
|
||||||
|
'@ai-sdk/provider-utils@3.0.8(zod@3.25.76)':
|
||||||
|
dependencies:
|
||||||
|
'@ai-sdk/provider': 2.0.0
|
||||||
|
'@standard-schema/spec': 1.1.0
|
||||||
|
eventsource-parser: 3.0.6
|
||||||
|
zod: 3.25.76
|
||||||
|
|
||||||
|
|
||||||
gsap@3.14.2: {}
|
gsap@3.14.2: {}
|
||||||
|
|
||||||
'@ai-sdk/gateway@1.0.15(zod@3.25.76)':
|
'@ai-sdk/gateway@1.0.15(zod@3.25.76)':
|
||||||
|
|||||||
@ -55,6 +55,13 @@ test("restores accepted tasks missing from a pre-submit canvas snapshot, includi
|
|||||||
expect(result[0].draftValues?.url).toHaveLength(2);
|
expect(result[0].draftValues?.url).toHaveLength(2);
|
||||||
expect(reconcileCanvasTasks<TaskNode>(result, [task], "2026-09-17T01:00:00Z")).toBe(result);
|
expect(reconcileCanvasTasks<TaskNode>(result, [task], "2026-09-17T01:00:00Z")).toBe(result);
|
||||||
});
|
});
|
||||||
|
test("music variations become separate playable resources without duplicate cards on polling", () => {
|
||||||
|
const music:CanvasGenerationTask={...task,request:{...task.request,kind:"audio"},result:{url:"https://example.com/one.mp3",urls:["https://example.com/one.mp3","https://example.com/two.mp3"]}};
|
||||||
|
const result=reconcileCanvasTasks<TaskNode>([],[music],"2026-09-17T01:00:00Z");
|
||||||
|
expect(result).toHaveLength(2);
|
||||||
|
expect(result[1]).toMatchObject({inputType:"audio",draftValues:{action:"AUDIO_RESOURCE",mediaUrl:"https://example.com/two.mp3"}});
|
||||||
|
expect(reconcileCanvasTasks(result,[music],"2026-09-17T01:00:00Z")).toBe(result);
|
||||||
|
});
|
||||||
test("does not resurrect nodes removed after task creation or hidden on the server", () => {
|
test("does not resurrect nodes removed after task creation or hidden on the server", () => {
|
||||||
expect(reconcileCanvasTasks<TaskNode>([], [task], "2026-09-17T01:02:00Z")).toEqual([]);
|
expect(reconcileCanvasTasks<TaskNode>([], [task], "2026-09-17T01:02:00Z")).toEqual([]);
|
||||||
expect(
|
expect(
|
||||||
|
|||||||
106
tests/routes/vip98k-model-expansion.test.ts
Normal file
106
tests/routes/vip98k-model-expansion.test.ts
Normal file
@ -0,0 +1,106 @@
|
|||||||
|
import { test, expect } from "@playwright/test";
|
||||||
|
import { createVip98kChatModel } from "@/lib/ai/providers/vip98k-chat";
|
||||||
|
import { buildVip98kModelPresets } from "@/lib/ai/vip98k-models";
|
||||||
|
import { buildTaskBody } from "@/lib/ai/providers/vip98k";
|
||||||
|
import { getVip98kImageBindingKeys } from "@/lib/ai/vip98k-task-invocation";
|
||||||
|
import { getAudioCheckpoint } from "@/lib/video-canvas/task-recovery";
|
||||||
|
import { audioRequestSchema } from "@/lib/video-canvas/audio-contract";
|
||||||
|
import { buildVip98kAudioBody, vip98kAudioContracts } from "@/lib/video-canvas/vip98k-audio-contract";
|
||||||
|
import { vip98kVideoDurationBounds } from "@/lib/ai/vip98k-video-parameters";
|
||||||
|
import { readVip98kAudioFile } from "@/lib/video-canvas/vip98k-audio-file";
|
||||||
|
|
||||||
|
test("Anthropic-only routes preserve tool calls and relay routing", async () => {
|
||||||
|
let sent: any;
|
||||||
|
const model = createVip98kChatModel({ apiKey:"fixture", baseURL:"https://relay.example", modelId:"claude-haiku-5-5",
|
||||||
|
metadata:{invocation:{kind:"chat",path:"/v1/messages",headers:{"X-Relay-Route-Option":"native_route"}}},
|
||||||
|
fetch:async (url, init) => {
|
||||||
|
expect(String(url)).toBe("https://relay.example/v1/messages");
|
||||||
|
const headers = new Headers(init?.headers);
|
||||||
|
expect(headers.get("x-relay-route-option")).toBe("native_route");
|
||||||
|
expect(headers.get("authorization")).toBe("Bearer fixture");
|
||||||
|
sent=JSON.parse(String(init?.body));
|
||||||
|
return Response.json({id:"message_test",type:"message",role:"assistant",model:"claude-haiku-5-5",
|
||||||
|
content:[{type:"tool_use",id:"call_test",name:"record_result",input:{message:"ok"}}],
|
||||||
|
stop_reason:"tool_use",stop_sequence:null,usage:{input_tokens:10,output_tokens:8}});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const result = await model.doGenerate({ prompt:[{role:"user",content:[{type:"text",text:"Run the test tool"}]}],
|
||||||
|
tools:[{type:"function",name:"record_result",inputSchema:{type:"object",properties:{message:{type:"string"}},required:["message"]}}],toolChoice:{type:"auto"},maxOutputTokens:128 });
|
||||||
|
expect(sent.messages[0].content[0].text).toBe("Run the test tool");
|
||||||
|
expect(result.content).toContainEqual(expect.objectContaining({type:"tool-call",toolName:"record_result"}));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("zero image slots and forbidden fields do not become reference support", () => {
|
||||||
|
const invocation = {kind:"tool",path:"/v1/images/generations",request_example:{model:"text_image",prompt:"a cup"},
|
||||||
|
input_schema:{properties:{file_urls:{type:"array",maxItems:0},image:false,partial_images:{type:"integer"}}}};
|
||||||
|
const [model] = buildVip98kModelPresets([{id:"text_image",type:"image",capabilities:["text_to_image"],invocation}]);
|
||||||
|
expect(model.capabilities).toMatchObject({maxReferenceImages:0,supportedModes:["text-to-image"]});
|
||||||
|
expect(model.displayName).toContain("仅文生图");
|
||||||
|
expect(getVip98kImageBindingKeys({requestExample:invocation.request_example,inputSchema:invocation.input_schema} as any)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
const params = {apiKey:"fixture",modelId:"image",kind:"image",prompt:"a blue cup",imageUrls:[],videoUrls:[],modelConfig:{},imageParams:{aspectRatio:"16:9",imageSize:"2K"}};
|
||||||
|
const invocation = (properties: Record<string, unknown>, requestExample: Record<string, unknown> = {}) => ({kind:"tool",method:"POST",path:"/v1/images/generations",contentType:"application/json",headers:{},requestExample:{model:"image",prompt:"example",...requestExample},inputSchema:{properties},resultMethod:"GET",resultPath:""});
|
||||||
|
|
||||||
|
test("ratio-sized image contracts bind size and resolution without stale square examples", () => {
|
||||||
|
const result = buildTaskBody(params,invocation({size:{enum:["1:1","16:9"]},resolution:{enum:["1K","2K"]}},{size:"1:1",resolution:"1K"}));
|
||||||
|
expect(result.body).toMatchObject({size:"16:9",resolution:"2K",prompt:"a blue cup"});
|
||||||
|
});
|
||||||
|
test("pixel-only image contracts use a supported size in the requested tier", () => {
|
||||||
|
const result = buildTaskBody(params,invocation({size:{enum:["1024x1024","1280x720","2048x1152","3840x2160"]}},{size:"1024x1024"}));
|
||||||
|
expect(result.body.size).toBe("2048x1152");
|
||||||
|
});
|
||||||
|
test("OpenAI image size fields receive pixel dimensions instead of a resolution label", () => {
|
||||||
|
const result = buildTaskBody({...params,modelId:"gpt-image-2"},invocation({size:{type:["string","null"]}}));
|
||||||
|
expect(result.body.size).toMatch(/^\d+x\d+$/);
|
||||||
|
});
|
||||||
|
test("task size patterns take precedence over ratio aliases in a catalog enum", () => {
|
||||||
|
const result = buildTaskBody({...params,imageParams:{imageSize:"1K",aspectRatio:"1:1"}},invocation({size:{enum:["1:1","1024x1024"],pattern:"^(auto|[1-9][0-9]*x[1-9][0-9]*)$"},resolution:{enum:["1K","2K"]}}));
|
||||||
|
expect(result.body.size).toBe("1024x1024");
|
||||||
|
});
|
||||||
|
test("image reference limits reject excess inputs before submission", () => {
|
||||||
|
expect(() => buildTaskBody({...params,imageUrls:["https://example.com/1.png","https://example.com/2.png"]},invocation({image:{type:"array",maxItems:1}},{image:[]}))).toThrow("最多支持 1 张");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Seedance content binds prompts and reference roles", () => {
|
||||||
|
const i=invocation({content:{items:{oneOf:[{properties:{type:{const:"text"}}},{properties:{type:{const:"image_url"}}}]}},duration:{anyOf:[{const:-1},{minimum:4,maximum:30}]},resolution:{enum:["480P","720P"]},ratio:{enum:["16:9","9:16"]}});
|
||||||
|
const result=buildTaskBody({...params,kind:"video",imageUrls:["https://example.com/one.png","https://example.com/two.png"],videoParams:{mode:"first-last-frame",resolution:"480p",duration:4,ratio:"16:9"}},i);
|
||||||
|
expect(result.body).toMatchObject({resolution:"480P",duration:4,content:[{type:"text",text:"a blue cup"},{type:"image_url",role:"first_frame"},{type:"image_url",role:"last_frame"}]});
|
||||||
|
expect(result.imageBindingKeys).toContain("content");
|
||||||
|
expect(vip98kVideoDurationBounds(i.inputSchema)).toMatchObject({min:4,max:30});
|
||||||
|
});
|
||||||
|
|
||||||
|
test("all fourteen audio entries expose distinct supported modes", () => {
|
||||||
|
expect(Object.keys(vip98kAudioContracts)).toHaveLength(14);
|
||||||
|
expect(vip98kAudioContracts["mimo-v2.5-asr"].kind).toBe("asr");
|
||||||
|
expect(vip98kAudioContracts["mimo-v2.5-tts-voiceclone"].kind).toBe("voiceclone");
|
||||||
|
expect(vip98kAudioContracts["suno-v6"].kind).toBe("music");
|
||||||
|
});
|
||||||
|
test("reference-only video routes reject an empty image list before submission", () => {
|
||||||
|
expect(()=>buildTaskBody({...params,kind:"video"},invocation({file_urls:{type:"array",minItems:1,maxItems:1}}))).toThrow("参考图");
|
||||||
|
});
|
||||||
|
test("Speech archives return the audio member instead of the TAR bytes", () => {
|
||||||
|
const audio=Buffer.from("RIFF1234WAVEfmt fixture audio");
|
||||||
|
const header=Buffer.alloc(512);header.write("output.wav");header.write(audio.length.toString(8).padStart(11,"0"),124);header[156]=48;header.write("ustar",257);
|
||||||
|
const file=readVip98kAudioFile(Buffer.concat([header,audio,Buffer.alloc(512-audio.length),Buffer.alloc(1024)]));
|
||||||
|
expect(file.format).toBe("wav");expect(file.bytes).toEqual(audio);
|
||||||
|
expect(()=>readVip98kAudioFile(Buffer.from('{"error":"unavailable"}'))).toThrow("音频文件");
|
||||||
|
});
|
||||||
|
test("audio input requires a reference for cloning and maps the user's audio", () => {
|
||||||
|
const input=audioRequestSchema.parse({modelRef:"test",mode:"voiceclone",text:"你好"});
|
||||||
|
expect(()=>buildVip98kAudioBody(input,"mimo-v2.5-tts-voiceclone",{})).toThrow("参考音频");
|
||||||
|
const body=buildVip98kAudioBody({...input,referenceAudioUrl:"https://example.com/audio.wav"},"mimo-v2.5-tts-voiceclone",{},"data:audio/wav;base64,fixture");
|
||||||
|
expect(body.audio.voice).toBe("data:audio/wav;base64,fixture");
|
||||||
|
expect(body.messages[1]).toEqual({role:"assistant",content:"你好"});
|
||||||
|
});
|
||||||
|
test("Suno automatic music omits the gender parameter that requires custom lyrics mode", () => {
|
||||||
|
const input=audioRequestSchema.parse({modelRef:"test",mode:"music",text:"gentle piano",format:"mp3",instrumental:true});
|
||||||
|
const body=buildVip98kAudioBody(input,"suno-v5",{custom_mode:false,metadata:{vocal_gender:"m"}});
|
||||||
|
expect(body.custom_mode).toBe(false);expect(body.metadata.vocal_gender).toBeUndefined();
|
||||||
|
const song=buildVip98kAudioBody({...input,instrumental:false,lyrics:"阳光照进窗台"},"suno-v5",{custom_mode:false,metadata:{vocal_gender:"m"}});
|
||||||
|
expect(song).toMatchObject({custom_mode:true,prompt:"阳光照进窗台",style:"gentle piano",metadata:{vocal_gender:"f"}});
|
||||||
|
});
|
||||||
|
test("completed audio checkpoints retain the original provider and resumable task", () => {
|
||||||
|
const checkpoint={operation:"audio",provider:"vip98k",modelRef:"test",modelId:"gem-2.5-tts",providerId:"source",baseUrl:"https://relay.example",taskId:"original",url:"https://example.com/audio.wav",format:"wav"};
|
||||||
|
expect(getAudioCheckpoint({request:{kind:"audio",params:{modelRef:"test"}},result:{upstreamAudio:checkpoint}})).toEqual(checkpoint);
|
||||||
|
});
|
||||||
@ -755,7 +755,7 @@ test.describe("VIP98K reference-image routing", () => {
|
|||||||
).toBe(false);
|
).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("excludes unverified text-only VIP98K models from the image picker", () => {
|
test("imports published text-only models without advertising image editing", () => {
|
||||||
const presets = buildVip98kModelPresets([
|
const presets = buildVip98kModelPresets([
|
||||||
{
|
{
|
||||||
id: "unverified-image-model",
|
id: "unverified-image-model",
|
||||||
@ -771,7 +771,9 @@ test.describe("VIP98K reference-image routing", () => {
|
|||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
|
|
||||||
expect(presets).toEqual([]);
|
expect(presets).toHaveLength(1);
|
||||||
|
expect(presets[0]).toMatchObject({ supportsVision: false,
|
||||||
|
capabilities: { supportedModes: ["text-to-image"], maxReferenceImages: 0 } });
|
||||||
});
|
});
|
||||||
|
|
||||||
test("keeps gpt-image-2-vip when its published image field is an array", () => {
|
test("keeps gpt-image-2-vip when its published image field is an array", () => {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user