feat: support 98K image, video, chat and audio model contracts

This commit is contained in:
WuYongJie 2026-11-16 10:57:53 +08:00
parent 1a682343c7
commit ba24efc8e7
23 changed files with 667 additions and 74 deletions

View File

@ -9,10 +9,10 @@ export async function GET() {
const models = await listCanvasAudioModels(); const models = await listCanvasAudioModels();
return NextResponse.json( return NextResponse.json(
{ {
models: models.map(({ model }) => ({ models: models.map(({ model, provider }) => ({
id: model.id, id: model.id,
modelId: model.modelId, modelId: model.modelId,
name: model.displayName, name: `${model.displayName} · ${provider.type === "vip98k" ? "98K" : "阿里云"}`,
...audioContract(model.modelId), ...audioContract(model.modelId),
})), })),
}, },

View File

@ -72,13 +72,13 @@ export function AudioGenerationComposer({
setError(""); setError("");
setSending(true); setSending(true);
try { try {
if (!model) throw Error("当前模式没有已启用的官方模型"); if (!model) throw Error("当前模式没有已启用的模型");
const params = audioRequestSchema.parse({ const params = audioRequestSchema.parse({
...draft, ...draft,
text: [draft.text, connectedText].filter(Boolean).join("\n\n"), text: [draft.text, connectedText].filter(Boolean).join("\n\n"),
modelRef: model.id, modelRef: model.id,
voice, voice,
format, format: format || "wav",
...(!contract?.controls ? { rate: 1, pitch: 1, volume: 50 } : {}), ...(!contract?.controls ? { rate: 1, pitch: 1, volume: 50 } : {}),
...(!contract?.instructions ? { instructions: "" } : {}), ...(!contract?.instructions ? { instructions: "" } : {}),
}); });
@ -96,9 +96,9 @@ export function AudioGenerationComposer({
<section <section
className="fixed bottom-8 left-1/2 z-[100] max-h-[75vh] w-[min(620px,94vw)] -translate-x-1/2 overflow-y-auto rounded-2xl border border-white/10 bg-[#222] p-5 text-white shadow-xl" className="fixed bottom-8 left-1/2 z-[100] max-h-[75vh] w-[min(620px,94vw)] -translate-x-1/2 overflow-y-auto rounded-2xl border border-white/10 bg-[#222] p-5 text-white shadow-xl"
data-canvas-overlay="true" data-canvas-overlay="true"
aria-label="阿里云音频生成" aria-label="语音与音乐"
> >
<h3 className="font-semibold">语音与音乐 · 阿里云官方</h3> <h3 className="font-semibold">语音与音乐</h3>
<fieldset disabled={busy || sending} className="mt-3 space-y-3 disabled:opacity-60"> <fieldset disabled={busy || sending} className="mt-3 space-y-3 disabled:opacity-60">
{connectedText && <p className="mb-3 whitespace-pre-wrap rounded bg-white/5 p-2 text-xs" aria-label="已连接文本">已连接文本:{connectedText}</p>} {connectedText && <p className="mb-3 whitespace-pre-wrap rounded bg-white/5 p-2 text-xs" aria-label="已连接文本">已连接文本:{connectedText}</p>}
<div className="grid grid-cols-2 gap-3"> <div className="grid grid-cols-2 gap-3">
@ -112,6 +112,9 @@ export function AudioGenerationComposer({
> >
<option value="tts">文字配音</option> <option value="tts">文字配音</option>
<option value="music">音乐 / 歌曲</option> <option value="music">音乐 / 歌曲</option>
<option value="asr">语音转文字</option>
<option value="voiceclone">声音克隆配音</option>
<option value="voicedesign">声音设计配音</option>
</select> </select>
</label> </label>
<label> <label>
@ -134,20 +137,40 @@ export function AudioGenerationComposer({
</select> </select>
</label> </label>
</div> </div>
{["asr", "voiceclone"].includes(draft.mode) && (
<label className="block"> <label className="block">
{draft.mode === "tts" ? "配音文本(最多 600 字)" : "音乐描述"} {draft.mode === "asr" ? "待识别音频" : "参考声音"}(MP3 / WAV,最多 20 MiB)
<input type="file" accept="audio/mpeg,audio/wav,.mp3,.wav" aria-label="上传参考音频" className={input}
onChange={async event => {
const file = event.target.files?.[0]; event.target.value = ""; if (!file) return;
if (file.size > 20 * 1024 * 1024) { setError("参考音频需小于 20 MiB"); return; }
setSending(true); setError("");
try {
const form = new FormData(); form.append("file", file);
const response = await fetch("/api/canvas/media/upload", { method: "POST", body: form, signal: AbortSignal.timeout(120000) });
const result = await response.json(); const url = result.url || result.data?.url;
if (!response.ok || !url) throw Error(result.error || "参考音频上传失败");
patch({ referenceAudioUrl: String(url), referenceAudioName: file.name });
} catch (error) { setError(error instanceof Error ? error.message : "上传失败"); }
finally { setSending(false); }
}} />
{draft.referenceAudioUrl && <span className="mt-1 block text-sm text-white/80">已上传:{draft.referenceAudioName || "参考音频"}</span>}
</label>
)}
{draft.mode !== "asr" && <label className="block">
{draft.mode !== "music" ? "配音文本(最多 600 字)" : `音乐描述(最多 ${contract?.descriptionLimit || 2000} 字)`}
<textarea <textarea
aria-label="音频内容" aria-label="音频内容"
className={input} className={input}
rows={4} rows={4}
maxLength={draft.mode === "tts" ? 600 : 2000} maxLength={draft.mode !== "music" ? 600 : contract?.descriptionLimit || 2000}
value={draft.text} value={draft.text}
onChange={(e) => patch({ text: e.target.value })} onChange={(e) => patch({ text: e.target.value })}
/> />
</label> </label>}
{draft.mode === "tts" ? ( {!["music", "asr"].includes(draft.mode) ? (
<> <>
<div className="grid grid-cols-2 gap-3"> {!!contract?.voices.length && <div className="grid grid-cols-2 gap-3">
<label> <label>
音色 音色
<select <select
@ -163,7 +186,7 @@ export function AudioGenerationComposer({
))} ))}
</select> </select>
</label> </label>
<label> {contract?.supportsLanguageControl !== false && <label>
语言 语言
<select <select
aria-label="配音语言" aria-label="配音语言"
@ -175,8 +198,8 @@ export function AudioGenerationComposer({
<option value="English">英文</option> <option value="English">英文</option>
<option value="Auto">自动 / 中英混合</option> <option value="Auto">自动 / 中英混合</option>
</select> </select>
</label> </label>}
</div> </div>}
{contract?.instructions ? ( {contract?.instructions ? (
<label className="block"> <label className="block">
声音风格指令 声音风格指令
@ -210,7 +233,7 @@ export function AudioGenerationComposer({
</div> </div>
) : null} ) : null}
</> </>
) : ( ) : draft.mode === "music" ? (
<> <>
<label className="flex gap-2"> <label className="flex gap-2">
<input <input
@ -233,7 +256,7 @@ export function AudioGenerationComposer({
onChange={(e) => patch({ lyrics: e.target.value })} onChange={(e) => patch({ lyrics: e.target.value })}
/> />
</label> </label>
<label> {contract?.supportsGender !== false && (!contract?.genderRequiresLyrics || draft.lyrics?.trim()) && <label>
演唱声音 演唱声音
<select <select
className={input} className={input}
@ -244,18 +267,18 @@ export function AudioGenerationComposer({
<option value="female">女声</option> <option value="female">女声</option>
<option value="male">男声</option> <option value="male">男声</option>
</select> </select>
</label> </label>}
</> </>
) : null} ) : null}
{!options.length ? ( {!options.length ? (
<p className="text-sm text-amber-200"> <p className="text-sm text-amber-200">
Fun-Music 需要阿里云邀测权限,开通并启用模型后即可生成。 当前没有可用的音乐模型,请在模型设置中启用。
</p> </p>
) : null} ) : null}
</> </>
)} ) : null}
<div className="flex items-end gap-3"> <div className="flex items-end gap-3">
<label> {draft.mode !== "asr" && <label>
格式 格式
<select <select
aria-label="音频格式" aria-label="音频格式"
@ -269,14 +292,14 @@ export function AudioGenerationComposer({
</option> </option>
))} ))}
</select> </select>
</label> </label>}
<button <button
type="button" type="button"
disabled={!model || loading} disabled={!model || loading}
className="rounded-lg bg-white px-4 py-2 text-black disabled:opacity-40" className="rounded-lg bg-white px-4 py-2 text-black disabled:opacity-40"
onClick={() => void run()} onClick={() => void run()}
> >
{busy || sending ? "处理中…" : "生成音频"} {busy || sending ? "处理中…" : draft.mode === "asr" ? "识别文字" : "生成音频"}
</button> </button>
<label className="cursor-pointer rounded-lg border border-white/20 px-3 py-2"> <label className="cursor-pointer rounded-lg border border-white/20 px-3 py-2">
上传音频 上传音频

View File

@ -1,4 +1,5 @@
import { createOpenAI } from "@ai-sdk/openai"; import { createOpenAI } from "@ai-sdk/openai";
import { createAnthropic } from "@ai-sdk/anthropic";
import { APICallError, type LanguageModelV2, type LanguageModelV2CallOptions } from "@ai-sdk/provider"; import { APICallError, type LanguageModelV2, type LanguageModelV2CallOptions } from "@ai-sdk/provider";
import { selectVip98kChatInvocation } from "../vip98k-chat-contract"; import { selectVip98kChatInvocation } from "../vip98k-chat-contract";
@ -128,6 +129,14 @@ export function createVip98kChatModel(params: {
const root = params.baseURL.replace(/\/+$/, "").replace(/\/v1$/i, ""); const root = params.baseURL.replace(/\/+$/, "").replace(/\/v1$/i, "");
const baseFetch = params.fetch || fetch; const baseFetch = params.fetch || fetch;
const createModel = (contract: typeof invocation) => { const createModel = (contract: typeof invocation) => {
if (contract?.protocol === "anthropic-messages") {
return createAnthropic({
apiKey: params.apiKey,
baseURL: `${root}/v1`,
headers: { ...contract.headers, Authorization: `Bearer ${params.apiKey}` },
fetch: createChatFetch(root, baseFetch),
})(params.modelId);
}
const provider = createOpenAI({ const provider = createOpenAI({
apiKey: params.apiKey, apiKey: params.apiKey,
baseURL: `${root}/v1`, baseURL: `${root}/v1`,
@ -140,7 +149,8 @@ export function createVip98kChatModel(params: {
}; };
const model = createModel(invocation); const model = createModel(invocation);
const needsDiscovery = params.discoverLegacyContract && const needsDiscovery = params.discoverLegacyContract &&
!invocation?.headers["X-Relay-Route-Option"]; (!invocation?.headers["X-Relay-Route-Option"] ||
(!metadata.chatInvocation && invocation.protocol === "anthropic-messages"));
const resolveModel = async (signal?: AbortSignal) => { const resolveModel = async (signal?: AbortSignal) => {
signal?.throwIfAborted(); signal?.throwIfAborted();
if (!needsDiscovery) return model; if (!needsDiscovery) return model;

View File

@ -1,4 +1,4 @@
import { mapVip98kVideoParameters } from "../vip98k-video-parameters"; import { mapVip98kVideoParameters, bindVip98kVideoContent } from "../vip98k-video-parameters";
import { import {
ProviderTaskFailedError, ProviderTaskFailedError,
ProviderTaskUncertainError, ProviderTaskUncertainError,
@ -205,8 +205,9 @@ function invocationSupportsBodyField(
key: string, key: string,
) { ) {
if (!invocation) return true; if (!invocation) return true;
if (key in invocation.requestExample) return true;
const properties = toRecord(toRecord(invocation.inputSchema)?.properties); const properties = toRecord(toRecord(invocation.inputSchema)?.properties);
if (properties?.[key] === false) return false;
if (key in invocation.requestExample) return true;
return Boolean(properties && key in properties); return Boolean(properties && key in properties);
} }
@ -1574,7 +1575,7 @@ export function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvoca
if ("displayName" in body || !invocation) { if ("displayName" in body || !invocation) {
body.displayName = promptTitle || params.modelId; body.displayName = promptTitle || params.modelId;
} }
if ("prompt" in body || !invocation) body.prompt = params.prompt; if (invocationSupportsBodyField(invocation, "prompt")) body.prompt = params.prompt;
if (params.kind === "image" && invocationSupportsBodyField(invocation, "n")) { if (params.kind === "image" && invocationSupportsBodyField(invocation, "n")) {
body.n = imageParams.imageCount ?? imageParams.n ?? 1; body.n = imageParams.imageCount ?? imageParams.n ?? 1;
} }
@ -1627,14 +1628,14 @@ export function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvoca
!invocationSupportsBodyField(invocation, "resolution") !invocationSupportsBodyField(invocation, "resolution")
) { ) {
const preserveResolutionBucket = const preserveResolutionBucket =
invocation?.path === "/v1/images/generations"; invocation?.path === "/v1/images/generations" &&
(!invocation.inputSchema || (toRecord(toRecord(invocation.inputSchema.properties)?.size)?.enum as unknown[] | undefined)?.includes(resolution));
const resolvedTaskSize = resolveVip98kImageTaskSize({ const resolvedTaskSize = resolveVip98kImageTaskSize({
imageSize: resolution, imageSize: resolution,
aspectRatio, aspectRatio,
fallbackSize: body.size, fallbackSize: body.size,
preserveResolutionBucket, preserveResolutionBucket,
useGptImage25AspectSizing: useGptImage25AspectSizing:
preserveResolutionBucket &&
hasVerifiedVip98kOpenAiImageArrayContract(params.modelId), hasVerifiedVip98kOpenAiImageArrayContract(params.modelId),
}); });
if (resolvedTaskSize) { if (resolvedTaskSize) {
@ -1664,8 +1665,8 @@ export function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvoca
if (value !== undefined && targetKey in body) body[targetKey] = value; if (value !== undefined && targetKey in body) body[targetKey] = value;
} }
if (typeof videoParams.generateAudio === "boolean") { if (typeof videoParams.generateAudio === "boolean") {
if ("generateAudio" in body || !invocation) body.generateAudio = videoParams.generateAudio; if (invocationSupportsBodyField(invocation, "generateAudio")) body.generateAudio = videoParams.generateAudio;
if ("generate_audio" in body || !invocation) body.generate_audio = videoParams.generateAudio; if (invocationSupportsBodyField(invocation, "generate_audio")) body.generate_audio = videoParams.generateAudio;
} }
// Reference fields can be optional in the schema and absent from its text-only example. // Reference fields can be optional in the schema and absent from its text-only example.
@ -1694,6 +1695,11 @@ export function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvoca
delete body.size; delete body.size;
} }
const imageBindingKeys = bindVip98kImages(body, params.imageUrls); const imageBindingKeys = bindVip98kImages(body, params.imageUrls);
const imageProperties = toRecord(invocation?.inputSchema?.properties) || {};
for (const key of imageBindingKeys) {
const limit = toRecord(imageProperties[key])?.maxItems;
if (typeof limit === "number" && params.imageUrls.length > limit) throw Error(`当前模型最多支持 ${limit} 张参考图`);
}
if (params.imageUrls.length > 0 && !invocation && imageBindingKeys.length === 0) { if (params.imageUrls.length > 0 && !invocation && imageBindingKeys.length === 0) {
body.images = params.imageUrls; body.images = params.imageUrls;
imageBindingKeys.push("images"); imageBindingKeys.push("images");
@ -1728,7 +1734,46 @@ export function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvoca
body.mode = params.imageUrls.length > 0 ? "image-to-video" : "text-to-video"; body.mode = params.imageUrls.length > 0 ? "image-to-video" : "text-to-video";
} }
if (params.kind === "video") mapVip98kVideoParameters(body, invocation?.inputSchema); if (params.kind === "video") {
const properties = toRecord(invocation?.inputSchema?.properties) || {};
if (params.imageUrls.length < Number(toRecord(properties.file_urls)?.minItems || 0)) throw Error("请先添加视频参考图");
if (bindVip98kVideoContent(body, invocation?.inputSchema, {
prompt: params.prompt, images: params.imageUrls, videos: params.videoUrls,
audios: Array.isArray(videoParams.referenceAudios) ? videoParams.referenceAudios as string[] : [],
mode: videoParams.mode,
}) && params.imageUrls.length) imageBindingKeys.push("content");
mapVip98kVideoParameters(body, invocation?.inputSchema);
}
if (params.kind === "image") {
const props = toRecord(invocation?.inputSchema?.properties) || {};
const sizeSchema = toRecord(props.size);
const pattern = typeof sizeSchema?.pattern === "string" ? new RegExp(sizeSchema.pattern) : null;
const sizes = Array.isArray(sizeSchema?.enum) ? sizeSchema.enum.filter(value => !pattern || pattern.test(String(value))) : undefined;
const ratios = Array.isArray(sizes) ? sizes.filter(value => typeof value === "string" && /^\d+:\d+$/.test(value)) : [];
const pixels = Array.isArray(sizes) ? sizes.flatMap(value => {
const match = String(value).match(/^(\d+)x(\d+)$/);
return match ? [{ size: value, width: Number(match[1]), height: Number(match[2]) }] : [];
}) : [];
const selectedRatio = aspectRatio || "1:1";
if (ratios.includes(selectedRatio)) body.size = selectedRatio;
else if (pixels.length && (resolution || aspectRatio)) {
const [rw, rh] = selectedRatio.split(":").map(Number);
const matching = pixels.filter(size => Math.abs(size.width / size.height - rw / rh) < 0.025);
if (!matching.length) throw Error("当前模型不支持所选图片比例");
const area = resolution === "4K" ? 2880 ** 2 : resolution === "2K" ? 2048 ** 2 : 1024 ** 2;
matching.sort((a, b) => Math.abs(a.width * a.height - area) - Math.abs(b.width * b.height - area));
body.size = matching[0].size;
}
if (ratios.length || pixels.length) {
const suffix = "\n\n[Output constraints]";
if (typeof body.prompt === "string" && body.prompt.includes(suffix)) body.prompt = body.prompt.split(suffix)[0];
}
for (const key of Object.keys(body)) if (props[key] === false) delete body[key];
if (body.orientation && aspectRatio) {
const [width, height] = aspectRatio.split(":").map(Number);
body.orientation = width > height ? "landscape" : "portrait";
}
}
return { body, imageBindingKeys, outputConstraintFallbacks }; return { body, imageBindingKeys, outputConstraintFallbacks };
} }

View File

@ -2,8 +2,8 @@ import { getVip98kInvocationHeaders } from "./vip98k-task-invocation";
type RecordValue = Record<string, unknown>; type RecordValue = Record<string, unknown>;
export type Vip98kChatInvocation = { export type Vip98kChatInvocation = {
protocol: "openai-chat" | "openai-responses"; protocol: "openai-chat" | "openai-responses" | "anthropic-messages";
path: "/v1/chat/completions" | "/v1/responses"; path: "/v1/chat/completions" | "/v1/responses" | "/v1/messages";
headers: Record<string, string>; headers: Record<string, string>;
}; };
@ -43,21 +43,21 @@ export function selectVip98kChatInvocation(value: unknown): Vip98kChatInvocation
return (!item.method || String(item.method).toUpperCase() === "POST") && return (!item.method || String(item.method).toUpperCase() === "POST") &&
(!item.kind || item.kind === "chat") && (!item.kind || item.kind === "chat") &&
(!modelId || !example?.model || example.model === modelId) && (!modelId || !example?.model || example.model === modelId) &&
(item.path === "/v1/chat/completions" || item.path === "/v1/responses"); ["/v1/chat/completions", "/v1/responses", "/v1/messages"].includes(String(item.path));
}); });
// Prefer the published default route when it supports this protocol. An // Prefer the published default route when it supports this protocol. An
// unscoped protocol example must not discard a route's execution contract. // unscoped protocol example must not discard a route's execution contract.
candidates.sort((a, b) => { candidates.sort((a, b) => {
const score = (item: RecordValue) => { const score = (item: RecordValue) => {
const route = getVip98kInvocationHeaders(item.headers)["X-Relay-Route-Option"]; const route = getVip98kInvocationHeaders(item.headers)["X-Relay-Route-Option"];
return route && route === defaultRoute ? 2 : route ? 1 : 0; return (route && route === defaultRoute ? 4 : route ? 2 : 0) + (item.path === "/v1/messages" ? 0 : 1);
}; };
return score(b) - score(a); return score(b) - score(a);
}); });
const selected = candidates[0]; const selected = candidates[0];
if (!selected) return null; if (!selected) return null;
return { return {
protocol: selected.path === "/v1/responses" ? "openai-responses" : "openai-chat", protocol: selected.path === "/v1/messages" ? "anthropic-messages" : selected.path === "/v1/responses" ? "openai-responses" : "openai-chat",
path: selected.path as Vip98kChatInvocation["path"], path: selected.path as Vip98kChatInvocation["path"],
headers: getVip98kInvocationHeaders(selected.headers), headers: getVip98kInvocationHeaders(selected.headers),
}; };

View File

@ -1,9 +1,10 @@
import { canonicalVip98kVideoRatio, vip98kVideoDurationBounds } from "./vip98k-video-parameters"; import { canonicalVip98kVideoRatio, vip98kVideoDurationBounds, vip98kVideoContentTypes, vip98kVideoImageRoles } from "./vip98k-video-parameters";
import { VIP98K_TASK_POLL_SECONDS } from "./vip98k-runtime-config"; import { VIP98K_TASK_POLL_SECONDS } from "./vip98k-runtime-config";
import { import {
hasVerifiedVip98kGptImage25ResolutionTiers, hasVerifiedVip98kGptImage25ResolutionTiers,
hasVerifiedVip98kOpenAiImageArrayContract, hasVerifiedVip98kOpenAiImageArrayContract,
normalizeVip98kInvocationHeaders, normalizeVip98kInvocationHeaders,
getVip98kImageBindingKeys,
} from "@/lib/ai/vip98k-task-invocation"; } from "@/lib/ai/vip98k-task-invocation";
import { selectVip98kChatInvocation, vip98kChatSupportsVision } from "./vip98k-chat-contract"; import { selectVip98kChatInvocation, vip98kChatSupportsVision } from "./vip98k-chat-contract";
@ -349,7 +350,8 @@ function getInvocationRequestKeys(model: Vip98kCatalogModel) {
return [ return [
...Object.keys(toRecord(invocation.request_example) || {}), ...Object.keys(toRecord(invocation.request_example) || {}),
...Object.keys(schemaProperties || {}), ...Object.keys(schemaProperties || {}),
].map(normalizeInvocationRequestKey); ].filter(key => schemaProperties?.[key] !== false && toRecord(schemaProperties?.[key])?.maxItems !== 0)
.map(normalizeInvocationRequestKey);
}); });
} }
@ -427,7 +429,10 @@ function hasMediaReferenceInput(
// published request body only accepts a text prompt. // published request body only accepts a text prompt.
if (type === "image" && getInvocationCandidates(model).length > 0) { if (type === "image" && getInvocationCandidates(model).length > 0) {
return ( return (
hasMediaRequestField || getInvocationCandidates(model).some(invocation => getVip98kImageBindingKeys({
requestExample: invocation.request_example || {},
inputSchema: invocation.input_schema,
} as any).length > 0) ||
hasVerifiedVip98kOpenAiImageArrayContract(modelId) hasVerifiedVip98kOpenAiImageArrayContract(modelId)
); );
} }
@ -468,6 +473,7 @@ function hasMultiImageReferenceInput(model: Vip98kCatalogModel) {
return false; return false;
} }
const schema = toRecord(schemaProperties[key]); const schema = toRecord(schemaProperties[key]);
if (schemaProperties[key] === false || schema?.maxItems === 0) return false;
return ( return (
Array.isArray(requestExample[key]) || Array.isArray(requestExample[key]) ||
String(schema?.type || "").trim().toLowerCase() === "array" String(schema?.type || "").trim().toLowerCase() === "array"
@ -507,7 +513,10 @@ export function isVip98kCatalogModelEligible(model: Vip98kCatalogModel) {
if (!getModelId(model) || !isVip98kCatalogModelAvailable(model)) return false; if (!getModelId(model) || !isVip98kCatalogModelAvailable(model)) return false;
const type = resolvePresetType(model); const type = resolvePresetType(model);
if (type === "image") { if (type === "image") {
return isVip98kMultiImageEditCatalogModel(model); if (VIP98K_MULTI_IMAGE_MODEL_EXCLUSIONS.has(getModelId(model).toLowerCase())) return false;
return isVip98kMultiImageEditCatalogModel(model) ||
(isVip98kTextToImageCatalogModel(model) && getInvocationCandidates(model).some(invocation =>
invocation.path === "/v1/images/generations" || invocation.path === "/extensions/v1/tasks"));
} }
if (type === "video") return hasMediaReferenceInput(model, type); if (type === "video") return hasMediaReferenceInput(model, type);
return true; return true;
@ -557,6 +566,9 @@ function getSupportedImageSizes(model: Vip98kCatalogModel) {
if (hasVerifiedVip98kGptImage25ResolutionTiers(getModelId(model))) { if (hasVerifiedVip98kGptImage25ResolutionTiers(getModelId(model))) {
return ["1K", "2K", "4K"]; return ["1K", "2K", "4K"];
} }
const primary = toRecord(toRecord(model.invocation?.input_schema)?.properties) || {};
const resolutionEnum = parseStringArray(toRecord(primary.resolution)?.enum);
if (resolutionEnum.length) return resolutionEnum.map(normalizeImageSize).filter((size): size is string => Boolean(size));
const schemaSizes = getInvocationCandidates(model).flatMap((invocation) => { const schemaSizes = getInvocationCandidates(model).flatMap((invocation) => {
const properties = const properties =
toRecord(toRecord(invocation.input_schema)?.properties) || {}; toRecord(toRecord(invocation.input_schema)?.properties) || {};
@ -742,7 +754,11 @@ function createImagePreset(
].some((key) => invocationKeys.has(key)) || hasVerifiedVip98kOpenAiImageArrayContract(modelId); ].some((key) => invocationKeys.has(key)) || hasVerifiedVip98kOpenAiImageArrayContract(modelId);
const supportsQuality = invocationKeys.has("quality"); const supportsQuality = invocationKeys.has("quality");
// Flare repeatedly returns 3:4/4:3 for 4:5/5:4, even with aspect_ratio supplied. // Flare repeatedly returns 3:4/4:3 for 4:5/5:4, even with aspect_ratio supplied.
const supportedAspectRatios = supportsNativeAspectRatio const declaredRatios = getInvocationCandidates(model).flatMap(invocation => {
const props = toRecord(toRecord(invocation.input_schema)?.properties) || {};
return ["aspect_ratio", "ratio", "size"].flatMap(key => parseStringArray(toRecord(props[key])?.enum).filter(value => /^\d+:\d+$/.test(value)));
});
const supportedAspectRatios = declaredRatios.length ? ["auto", ...new Set(declaredRatios)] : supportsNativeAspectRatio
? DEFAULT_ASPECT_RATIOS.filter((ratio) => modelId !== "gpt-image-2.5-flare" || !["4:5", "5:4"].includes(ratio)) ? DEFAULT_ASPECT_RATIOS.filter((ratio) => modelId !== "gpt-image-2.5-flare" || !["4:5", "5:4"].includes(ratio))
: ["auto"]; : ["auto"];
const supportedModes = [ const supportedModes = [
@ -751,7 +767,14 @@ function createImagePreset(
: []), : []),
...(supportsReferenceImages ? ["image-to-image", "reference-images"] : []), ...(supportsReferenceImages ? ["image-to-image", "reference-images"] : []),
]; ];
const maxReferenceImages = supportsMultiImageReference const declaredReferenceLimits = getInvocationCandidates(model).flatMap(invocation => {
const properties = toRecord(toRecord(invocation.input_schema)?.properties) || {};
return ["file_urls", "image", "images", "reference_images"].flatMap(key => {
const limit = toRecord(properties[key])?.maxItems;
return typeof limit === "number" && limit > 0 ? [limit] : [];
});
});
const maxReferenceImages = declaredReferenceLimits.length ? Math.max(...declaredReferenceLimits) : supportsMultiImageReference
? 8 ? 8
: supportsReferenceImages : supportsReferenceImages
? 1 ? 1
@ -763,7 +786,7 @@ function createImagePreset(
return { return {
modelId, modelId,
displayName, displayName: supportsReferenceImages ? displayName : `${displayName}(仅文生图)`,
description: model.description || `VIP98K image model (${modelId})`, description: model.description || `VIP98K image model (${modelId})`,
type: "image", type: "image",
supportsVision: supportsReferenceImages, supportsVision: supportsReferenceImages,
@ -843,21 +866,27 @@ function createVideoPreset(
: getDurations(model)); : getDurations(model));
const defaultDuration = const defaultDuration =
parseNumber(model.defaultDuration || model.default_duration) || parseNumber(model.defaultDuration || model.default_duration) ||
durations[durations.length - 1] || parseNumber(invocation?.request_example?.duration_seconds) ||
(Number(invocation?.request_example?.duration) > 0 ? Number(invocation?.request_example?.duration) : 0) ||
durations[0] ||
8; 8;
const keys = new Set(getInvocationRequestKeys(model)); const keys = new Set(getInvocationRequestKeys(model));
const imageModes = parseStringArray(toRecord(schema.mode_image)?.enum); const imageModes = parseStringArray(toRecord(schema.mode_image)?.enum);
const supportsReferenceImages = capabilities.includes("image-to-video") || keys.has("file-urls") || keys.has("image") || keys.has("image-url"); const contentTypes = vip98kVideoContentTypes(invocation?.input_schema);
const supportsFramePair = imageModes.includes("frame"); const contentRoles = vip98kVideoImageRoles(invocation?.input_schema);
const supportsIngredients = imageModes.includes("ingredient"); const supportsReferenceImages = contentTypes.length ? contentTypes.includes("image_url") : capabilities.includes("image-to-video") || keys.has("file-urls") || keys.has("image") || keys.has("image-url");
const supportsFirstFrame = supportsReferenceImages && (!contentRoles || contentRoles.includes("first_frame"));
const supportsFramePair = imageModes.includes("frame") || (contentTypes.includes("image_url") && (!contentRoles || contentRoles.includes("last_frame")));
const supportsIngredients = imageModes.includes("ingredient") || (contentTypes.includes("image_url") && (!contentRoles || contentRoles.includes("reference_image")));
const requiresReference = Number(toRecord(schema.file_urls)?.minItems) > 0;
const supportedModes = [ const supportedModes = [
...(capabilities.includes("text-to-video") || keys.has("prompt") ? ["text-to-video"] : []), ...(!requiresReference && (capabilities.includes("text-to-video") || keys.has("prompt") || (contentTypes.length === 1 && contentTypes[0] === "text")) ? ["text-to-video"] : []),
...(supportsReferenceImages ? ["first-frame"] : []), ...(supportsFirstFrame ? ["first-frame"] : []),
...(supportsFramePair ? ["first-last-frame"] : []), ...(supportsFramePair ? ["first-last-frame"] : []),
...(supportsIngredients ? ["reference-images"] : []), ...(supportsIngredients ? ["reference-images"] : []),
]; ];
const ratios = parseStringArray(toRecord(schema.aspect_ratio)?.enum).map(canonicalVip98kVideoRatio); const ratios = parseStringArray(toRecord(schema.aspect_ratio || schema.ratio)?.enum).map(canonicalVip98kVideoRatio).filter(value => value !== "adaptive");
const maxReferenceImages = Number(toRecord(schema.file_urls)?.maxItems) || (supportsReferenceImages ? 1 : 0); const maxReferenceImages = Number(toRecord(schema.file_urls)?.maxItems) || (contentTypes.includes("image_url") ? 30 : supportsReferenceImages ? 1 : 0);
return { return {
modelId, modelId,

View File

@ -220,9 +220,11 @@ export function getVip98kImageBindingKeys(invocation: Vip98kTaskInvocation | nul
if (!invocation) return []; if (!invocation) return [];
const body = invocation.requestExample || {}; const body = invocation.requestExample || {};
const properties = (invocation.inputSchema?.properties || {}) as Record<string, unknown>; const properties = (invocation.inputSchema?.properties || {}) as Record<string, unknown>;
const accepts = (key: string) => properties[key] !== false &&
(properties[key] as { maxItems?: number } | undefined)?.maxItems !== 0;
return [ return [
...IMAGE_ARRAY_INPUT_KEYS.filter((key) => key in body || key in properties), ...IMAGE_ARRAY_INPUT_KEYS.filter((key) => accepts(key) && (key in body || key in properties)),
...IMAGE_SINGLE_INPUT_KEYS.filter((key) => key in body || key in properties), ...IMAGE_SINGLE_INPUT_KEYS.filter((key) => accepts(key) && (key in body || key in properties)),
...(Array.isArray(body.video_clips) ? ["video_clips" as const] : []), ...(Array.isArray(body.video_clips) ? ["video_clips" as const] : []),
]; ];
} }

View File

@ -12,6 +12,11 @@ export function vip98kVideoDurationBounds(schema: unknown, resolution?: string)
const visit = (value: unknown) => { const visit = (value: unknown) => {
const current = object(value), properties = object(current.properties); const current = object(value), properties = object(current.properties);
const field = object(properties.duration_seconds || properties.duration); const field = object(properties.duration_seconds || properties.duration);
const positiveRange = (Array.isArray(field.anyOf) ? field.anyOf : []).find((x: any) => x.minimum > 0);
if (positiveRange) {
min = Math.max(min, positiveRange.minimum);
max = Math.min(max, positiveRange.maximum ?? Infinity);
}
if (Number.isFinite(field.minimum)) min = Math.max(min, field.minimum); if (Number.isFinite(field.minimum)) min = Math.max(min, field.minimum);
if (Number.isFinite(field.maximum)) max = Math.min(max, field.maximum); if (Number.isFinite(field.maximum)) max = Math.min(max, field.maximum);
const allowed = Array.isArray(field.enum) ? field.enum : Number.isFinite(field.const) ? [field.const] : undefined; const allowed = Array.isArray(field.enum) ? field.enum : Number.isFinite(field.const) ? [field.const] : undefined;
@ -36,8 +41,11 @@ export function mapVip98kVideoParameters(body: Record<string, unknown>, schema:
body.aspect_ratio = alias; body.aspect_ratio = alias;
} }
const resolutions = object(properties.resolution).enum; const resolutions = object(properties.resolution).enum;
if (body.resolution && Array.isArray(resolutions) && !resolutions.includes(body.resolution)) if (body.resolution && Array.isArray(resolutions) && !resolutions.includes(body.resolution)) {
throw Error("供应商当前线路不支持所选视频清晰度,请同步模型规格后重试"); const matched = resolutions.find(value => String(value).toLowerCase() === String(body.resolution).toLowerCase());
if (!matched) throw Error("供应商当前线路不支持所选视频清晰度,请同步模型规格后重试");
body.resolution = matched;
}
const seconds = Number(body.duration_seconds ?? body.duration); const seconds = Number(body.duration_seconds ?? body.duration);
if (seconds) { if (seconds) {
const bounds = vip98kVideoDurationBounds(schema, String(body.resolution || "")); const bounds = vip98kVideoDurationBounds(schema, String(body.resolution || ""));
@ -45,3 +53,45 @@ export function mapVip98kVideoParameters(body: Record<string, unknown>, schema:
throw Error("供应商当前清晰度不支持所选视频时长,请调整镜头时长"); throw Error("供应商当前清晰度不支持所选视频时长,请调整镜头时长");
} }
} }
export function vip98kVideoContentTypes(schema: unknown): string[] {
const content = object(object(object(schema).properties).content);
const items = object(content.items);
return (Array.isArray(items.oneOf) ? items.oneOf : []).flatMap((variant: unknown) => {
const type = object(object(object(variant).properties).type).const;
return typeof type === "string" ? [type] : [];
});
}
export function vip98kVideoImageRoles(schema: unknown): string[] | undefined {
const variants = object(object(object(object(schema).properties).content).items).oneOf;
const image = (Array.isArray(variants) ? variants : []).find(value => object(object(object(value).properties).type).const === "image_url");
const roles = object(object(object(image).properties).role).enum;
return Array.isArray(roles) ? roles.map(String) : undefined;
}
export function bindVip98kVideoContent(body: Record<string, unknown>, schema: unknown, input: {
prompt: string; images: string[]; videos: string[]; audios: string[]; mode?: unknown;
}) {
const types = vip98kVideoContentTypes(schema);
if (!types.length) return false;
if (input.images.length && !types.includes("image_url")) throw Error("所选视频模型只支持文字生成,不能添加参考图片");
if (input.videos.length && !types.includes("video_url")) throw Error("所选模型不支持参考视频");
if (input.audios.length && !types.includes("audio_url")) throw Error("所选模型不支持参考音频");
const reference = input.mode === "reference-images";
const roles = vip98kVideoImageRoles(schema);
const role = reference ? "reference_image" : "first_frame";
if (input.images.length && roles && !roles.includes(role)) throw Error("所选视频模型不支持这个参考图模式");
if (!reference && input.images.length > 2) throw Error("首尾帧模式最多支持两张图片");
if (!reference && input.images.length === 2 && roles && !roles.includes("last_frame")) throw Error("所选视频模型不支持尾帧");
if (["first-frame", "first-last-frame", "reference-images"].includes(String(input.mode)) && !input.images.length) throw Error("请先添加视频参考图");
body.content = [
...(input.prompt ? [{ type: "text", text: input.prompt }] : []),
...input.images.map((url, index) => ({ type: "image_url", image_url: { url }, role: reference ? "reference_image" : index ? "last_frame" : "first_frame" })),
...input.videos.map(url => ({ type: "video_url", video_url: { url }, role: "reference_video" })),
...input.audios.map(url => ({ type: "audio_url", audio_url: { url }, role: "reference_audio" })),
];
delete body.prompt;
delete body.file_urls;
return true;
}

View File

@ -74,6 +74,14 @@ export async function getReadyQiniuImageDownload(url: string, target?: Awaited<R
} }
export async function archiveImageThroughQiniu(sourceUrl: string, archiveId: string) { export async function archiveImageThroughQiniu(sourceUrl: string, archiveId: string) {
return archiveThroughQiniu(sourceUrl, archiveId, "image");
}
export async function archiveMediaThroughQiniu(sourceUrl: string, archiveId: string, kind: "audio" | "video") {
return archiveThroughQiniu(sourceUrl, archiveId, kind);
}
async function archiveThroughQiniu(sourceUrl: string, archiveId: string, kind: "image" | "audio" | "video") {
let source: URL; let source: URL;
try { source = new URL(sourceUrl); } catch { return null; } try { source = new URL(sourceUrl); } catch { return null; }
if (!/^https?:$/.test(source.protocol) || source.username || source.password) return null; if (!/^https?:$/.test(source.protocol) || source.username || source.password) return null;
@ -90,8 +98,8 @@ export async function archiveImageThroughQiniu(sourceUrl: string, archiveId: str
|| /^(?:127\.|10\.|192\.168\.|169\.254\.|172\.(?:1[6-9]|2\d|3[01])\.)/.test(source.hostname)) return null; || /^(?:127\.|10\.|192\.168\.|169\.254\.|172\.(?:1[6-9]|2\d|3[01])\.)/.test(source.hostname)) return null;
// The archive task owns this object key, including retries after a timeout. // The archive task owns this object key, including retries after a timeout.
const pathname = `uploads/ai-generated-archive-${archiveId}`; const pathname = kind === "image" ? `uploads/ai-generated-archive-${archiveId}` : `uploads/canvas-${kind}-${archiveId}`;
return fetchImageIntoQiniu(sourceUrl, pathname, target!); return fetchImageIntoQiniu(sourceUrl, pathname, target!, kind);
} }
/** Prepare uploaded WebP images inside Kodo, retaining their embedded colour profile. */ /** Prepare uploaded WebP images inside Kodo, retaining their embedded colour profile. */
@ -111,6 +119,7 @@ async function fetchImageIntoQiniu(
sourceUrl: string, sourceUrl: string,
pathname: string, pathname: string,
target: NonNullable<Awaited<ReturnType<typeof resolveS3UploadTarget>>>, target: NonNullable<Awaited<ReturnType<typeof resolveS3UploadTarget>>>,
kind: "image" | "audio" | "video" = "image",
) { ) {
const config = target.config; const config = target.config;
const path = `/fetch/${encodeQiniu(sourceUrl)}/to/${encodeQiniu(`${config.bucket}:${pathname}`)}`; const path = `/fetch/${encodeQiniu(sourceUrl)}/to/${encodeQiniu(`${config.bucket}:${pathname}`)}`;
@ -132,8 +141,8 @@ async function fetchImageIntoQiniu(
throw new Error(`Qiniu archive fetch returned HTTP ${response.status}`); throw new Error(`Qiniu archive fetch returned HTTP ${response.status}`);
} }
const result = await response.json() as { key?: string; fsize?: number; mimeType?: string }; const result = await response.json() as { key?: string; fsize?: number; mimeType?: string };
if (result.key !== pathname || !result.fsize || !result.mimeType?.startsWith("image/")) { if (result.key !== pathname || !result.fsize || !result.mimeType?.startsWith(`${kind}/`)) {
throw new Error("Qiniu archive fetch did not return an image"); throw new Error(`Qiniu archive fetch did not return ${kind}`);
} }
const url = buildS3PublicUrl(config, pathname); const url = buildS3PublicUrl(config, pathname);
return { return {

View File

@ -156,6 +156,7 @@ export function isStyleProImageModel(
entry: StyleProModelEntry | StyleProDisplayModelEntry entry: StyleProModelEntry | StyleProDisplayModelEntry
) { ) {
if (!entry.model.supportsImageGen) return false; if (!entry.model.supportsImageGen) return false;
if ((entry.model.imageGenConfig as Record<string, unknown> | null)?.maxReferenceImages === 0) return false;
if (isExplicitImageGenerationModel(entry)) return true; if (isExplicitImageGenerationModel(entry)) return true;
return !isVideoLikeModel(entry); return !isVideoLikeModel(entry);
} }

View File

@ -1,4 +1,5 @@
import { z } from "zod"; import { z } from "zod";
import { vip98kAudioContract, validateVip98kAudioInput, type AudioModelContract } from "./vip98k-audio-contract";
const qwenVoices = [ const qwenVoices = [
{ id: "Cherry", name: "芊悦" }, { id: "Cherry", name: "芊悦" },
{ id: "Serena", name: "苏瑶" }, { id: "Serena", name: "苏瑶" },
@ -65,7 +66,8 @@ export const audioModelContracts = {
>; >;
export type AudioModelId = keyof typeof audioModelContracts; export type AudioModelId = keyof typeof audioModelContracts;
export const audioRequestSchema = z.object({ export const audioRequestSchema = z.object({
mode: z.enum(["tts", "music"]), mode: z.enum(["tts", "music", "asr", "voiceclone", "voicedesign"]),
referenceAudioUrl: z.string().url().refine(value => /^https?:\/\//i.test(value)).optional(),
modelRef: z.string().min(1), modelRef: z.string().min(1),
text: z.string().trim().max(2000).default(""), text: z.string().trim().max(2000).default(""),
lyrics: z.string().trim().max(2000).default(""), lyrics: z.string().trim().max(2000).default(""),
@ -94,10 +96,10 @@ export const audioRequestSchema = z.object({
volume: z.number().int().min(0).max(100).default(50), volume: z.number().int().min(0).max(100).default(50),
}); });
export type AudioRequest = z.infer<typeof audioRequestSchema>; export type AudioRequest = z.infer<typeof audioRequestSchema>;
export function audioContract(id: string) { export function audioContract(id: string): AudioModelContract | undefined {
return Object.hasOwn(audioModelContracts, id) return Object.hasOwn(audioModelContracts, id)
? audioModelContracts[id as AudioModelId] ? audioModelContracts[id as AudioModelId]
: undefined; : vip98kAudioContract(id);
} }
export function officialAudioBase(base: string, modelId?: string) { export function officialAudioBase(base: string, modelId?: string) {
const url = new URL(base); const url = new URL(base);
@ -121,6 +123,10 @@ export function officialAudioBase(base: string, modelId?: string) {
export function buildAudioRequest(value: unknown, modelId: string) { export function buildAudioRequest(value: unknown, modelId: string) {
const p = audioRequestSchema.parse(value), const p = audioRequestSchema.parse(value),
contract = audioContract(modelId); contract = audioContract(modelId);
if (vip98kAudioContract(modelId)) {
validateVip98kAudioInput(p, modelId);
return { path: "", body: {} };
}
if (!contract || contract.kind !== p.mode) throw Error("音频模式与模型不匹配"); if (!contract || contract.kind !== p.mode) throw Error("音频模式与模型不匹配");
if (!contract.formats.includes(p.format)) throw Error("所选模型不支持这个音频格式"); if (!contract.formats.includes(p.format)) throw Error("所选模型不支持这个音频格式");
if (p.mode === "music") { if (p.mode === "music") {

View File

@ -13,9 +13,12 @@ import {
officialAudioBase, officialAudioBase,
} from "./audio-contract"; } from "./audio-contract";
import { ProviderTaskUncertainError } from "./video-task-lifecycle"; import { ProviderTaskUncertainError } from "./video-task-lifecycle";
import { vip98kAudioContract } from "./vip98k-audio-contract";
import { executeVip98kCanvasAudio } from "./vip98k-audio-generation";
export async function listCanvasAudioModels() { export async function listCanvasAudioModels() {
return (await getEnabledAiModelsWithProvider()).filter(({ model, provider }) => { return (await getEnabledAiModelsWithProvider()).filter(({ model, provider }) => {
try { try {
if (provider.type === "vip98k") return Boolean(vip98kAudioContract(model.modelId));
return ( return (
provider.type === "alibaba" && provider.type === "alibaba" &&
!!audioContract(model.modelId) && !!audioContract(model.modelId) &&
@ -35,6 +38,8 @@ export async function executeCanvasAudio(
}, },
) { ) {
const input = audioRequestSchema.parse(params); const input = audioRequestSchema.parse(params);
const selected = (await listCanvasAudioModels()).find(item => item.model.id === input.modelRef);
if (selected?.provider.type === "vip98k") return executeVip98kCanvasAudio(input, selected, lifecycle);
let checkpoint = lifecycle.resume ? audioCheckpointSchema.parse(lifecycle.resume) : undefined; let checkpoint = lifecycle.resume ? audioCheckpointSchema.parse(lifecycle.resume) : undefined;
if (checkpoint && checkpoint.modelRef !== input.modelRef) throw Error("音频记录与原模型不匹配"); if (checkpoint && checkpoint.modelRef !== input.modelRef) throw Error("音频记录与原模型不匹配");
if (!checkpoint) { if (!checkpoint) {

View File

@ -227,10 +227,9 @@ export function reconcileCanvasTasks<
} }
if (task.request.kind === "audio") if (task.request.kind === "audio")
Object.assign(nextDraft, { Object.assign(nextDraft, {
mediaUrl: data.url, ...(data.text ? { text: data.text, content: data.text, prompt: data.text, action: "TEXT_RESOURCE", mediaType: "text" } : {
url: [data.url], mediaUrl: data.url, url: data.urls || [data.url], action: "AUDIO_RESOURCE", mediaType: "audio",
action: "AUDIO_RESOURCE", }),
mediaType: "audio",
audioMeta: data.metadata, audioMeta: data.metadata,
audioParams: audioParams:
task.request.node.draftValues?.audioParams || task.request.params, task.request.node.draftValues?.audioParams || task.request.params,
@ -243,16 +242,16 @@ export function reconcileCanvasTasks<
}); });
if ( if (
task.request.kind === "image" || task.request.kind === "image" ||
task.request.kind === "image-operation" task.request.kind === "image-operation" || (task.request.kind === "audio" && !data.text)
) { ) {
const urls = (data.images || []) const urls = task.request.kind === "audio" ? data.urls || [data.url] : (data.images || [])
.map((image: any) => image.originalUrl || image.thumbnailUrl) .map((image: any) => image.originalUrl || image.thumbnailUrl)
.filter(Boolean); .filter(Boolean);
Object.assign(nextDraft, { Object.assign(nextDraft, {
mediaUrl: urls[0], mediaUrl: urls[0],
url: urls, url: urls,
action: action:
draft.action === "IMAGE_GENERATE_NINE_GRID" task.request.kind === "audio" ? "AUDIO_RESOURCE" : draft.action === "IMAGE_GENERATE_NINE_GRID"
? draft.action ? draft.action
: "IMAGE_RESOURCE", : "IMAGE_RESOURCE",
}); });
@ -263,6 +262,7 @@ export function reconcileCanvasTasks<
...result, ...result,
{ {
...node, ...node,
...(task.request.kind === "audio" ? { kind: "media", inputType: "audio" } : {}),
id, id,
title: `${node.title} ${i + 1}`, title: `${node.title} ${i + 1}`,
y: node.y + i * (node.height + 40), y: node.y + i * (node.height + 40),
@ -357,7 +357,7 @@ export function reconcileCanvasTasks<
task.request.kind === "smart-edit" task.request.kind === "smart-edit"
? "video" ? "video"
: task.request.kind === "audio" : task.request.kind === "audio"
? "audio" ? task.result?.text ? "text" : "audio"
: "image", : "image",
} }
: {}), : {}),

View File

@ -1,5 +1,6 @@
import type { SmartEditCheckpoint } from "./smart-edit/execute"; import type { SmartEditCheckpoint } from "./smart-edit/execute";
import { audioCheckpointSchema } from "./audio-contract"; import { audioCheckpointSchema } from "./audio-contract";
import { vip98kAudioCheckpointSchema } from "./vip98k-audio-contract";
import { videoCheckpointSchema } from "./video-task-lifecycle"; import { videoCheckpointSchema } from "./video-task-lifecycle";
// Resume only server-recorded provider jobs. A lost request without an upstream ID is never resubmitted. // Resume only server-recorded provider jobs. A lost request without an upstream ID is never resubmitted.
export type RecoveryTask = { export type RecoveryTask = {
@ -49,7 +50,7 @@ export function getAudioCheckpoint(
task: Pick<RecoveryTask, "request" | "result"> task: Pick<RecoveryTask, "request" | "result">
) { ) {
if (task.request.kind !== "audio") return null; if (task.request.kind !== "audio") return null;
const parsed = audioCheckpointSchema.safeParse(task.result?.upstreamAudio); const parsed = vip98kAudioCheckpointSchema.or(audioCheckpointSchema).safeParse(task.result?.upstreamAudio);
return parsed.success && parsed.data.modelRef === task.request.params.modelRef return parsed.success && parsed.data.modelRef === task.request.params.modelRef
? parsed.data ? parsed.data
: null; : null;

View File

@ -0,0 +1,106 @@
import { z } from "zod";
export type AudioModelContract = {
kind: string; label: string; voices: { id: string; name: string }[];
instructions: boolean; controls: boolean; formats: string[]; supportsLanguageControl?: boolean;
supportsGender?: boolean; genderRequiresLyrics?: boolean; descriptionLimit?: number;
};
const voice = (id: string, name = id) => ({ id, name });
const tts = (label: string, voices: { id: string; name: string }[] = [], instructions = false): AudioModelContract =>
({ kind: "tts", label, voices, instructions, controls: false, formats: ["wav"], supportsLanguageControl: false });
const music = (label: string): AudioModelContract =>
({ kind: "music", label, voices: [], instructions: false, controls: false, formats: ["mp3"], supportsGender: false });
export const vip98kAudioContracts: Record<string, AudioModelContract> = {
"mimo-v2.5-asr": { ...tts("MiMo 语音识别"), kind: "asr", formats: [] },
"mimo-v2.5-tts": tts("MiMo 文字配音", [], true),
"mimo-v2.5-tts-voiceclone": { ...tts("MiMo 声音克隆", [], true), kind: "voiceclone" },
"mimo-v2.5-tts-voicedesign": { ...tts("MiMo 声音设计", [], true), kind: "voicedesign" },
"gem-2.5-tts": tts("Gemini 2.5 配音", [voice("Zephyr"), voice("Puck"), voice("Kore"), voice("Charon")]),
"gem-3.1-tts": tts("Gemini 3.1 配音", [voice("Zephyr"), voice("Puck"), voice("Kore"), voice("Charon")]),
"speech-2.8-hd": { ...tts("Speech 2.8 高清", [voice("male-qn-qingse", "青涩男声")]), formats: ["mp3", "wav"], controls: true },
"speech-2.8-turbo": { ...tts("Speech 2.8 快速", [voice("male-qn-qingse", "青涩男声")]), formats: ["mp3", "wav"], controls: true },
"music-2.5": music("Music 2.5"),
"music-2.5+": music("Music 2.5+"),
"suno-v4.5": { ...music("Suno 4.5"), supportsGender: true, descriptionLimit: 1000 },
"suno-v5": { ...music("Suno 5"), supportsGender: true, genderRequiresLyrics: true, descriptionLimit: 500 },
"suno-v5-5": { ...music("Suno 5.5"), supportsGender: true, genderRequiresLyrics: true, descriptionLimit: 500 },
"suno-v6": { ...music("Suno 6"), supportsGender: true, descriptionLimit: 1000 },
};
export function vip98kAudioContract(id: string) {
return Object.hasOwn(vip98kAudioContracts, id) ? vip98kAudioContracts[id] : undefined;
}
export type Vip98kAudioInput = {
mode: string; text: string; lyrics: string; instructions: string; voice?: string;
format: string; instrumental: boolean; gender: string; rate: number; pitch: number;
volume: number; referenceAudioUrl?: string;
};
export function validateVip98kAudioInput(input: Vip98kAudioInput, modelId: string) {
const contract = vip98kAudioContract(modelId);
if (!contract || contract.kind !== input.mode) throw new Error("音频模式与模型不匹配");
if (["asr", "voiceclone"].includes(input.mode) && !input.referenceAudioUrl) throw new Error("请先上传参考音频");
if (input.mode !== "asr" && !input.text.trim() && !input.lyrics.trim()) throw new Error("请输入配音文本或音乐描述");
if (input.mode === "voicedesign" && !input.instructions.trim()) throw new Error("请填写声音风格指令");
if (contract.descriptionLimit && input.text.length > contract.descriptionLimit) throw new Error(`这个模型的音乐描述最多 ${contract.descriptionLimit} 字`);
if (input.mode !== "asr" && !contract.formats.includes(input.format)) throw new Error("所选模型不支持这个音频格式");
}
export function buildVip98kAudioBody(input: Vip98kAudioInput, modelId: string, example: Record<string, any>, audioData?: string) {
validateVip98kAudioInput(input, modelId);
const body = structuredClone(example);
body.model = modelId;
if (modelId.startsWith("mimo-")) {
body.stream = false;
if (input.mode === "asr") {
if (!audioData) throw new Error("参考音频尚未读取");
body.messages = [{ role: "user", content: [{ type: "input_audio", input_audio: { data: audioData, format: audioData.includes("audio/wav") ? "wav" : "mp3" } }] }];
} else {
body.messages = [
{ role: "user", content: input.instructions || "请用自然的中文朗读。" },
{ role: "assistant", content: input.text },
];
if (input.mode === "voiceclone") {
if (!audioData) throw new Error("参考音频尚未读取");
body.audio = { ...body.audio, voice: audioData };
}
}
return body;
}
body.prompt = input.text;
const set = (key: string, value: unknown) => {
if (body.metadata && key in body.metadata) body.metadata[key] = value;
else body[key] = value;
};
if (modelId.startsWith("gem-")) set("voice_id", input.voice || "Zephyr");
if (modelId.startsWith("speech-")) {
set("voice_id", input.voice || "male-qn-qingse");
set("speed", input.rate);
set("pitch", Math.round((input.pitch - 1) * 12));
set("vol", Math.max(0.01, input.volume / 50));
set("format", input.format);
}
if (input.mode === "music") {
if (modelId === "suno-v5" || modelId === "suno-v5-5") {
body.instrumental = input.instrumental;
body.custom_mode = Boolean(input.lyrics && !input.instrumental);
if (body.custom_mode) { body.prompt = input.lyrics; body.style = input.text; body.title = input.text.slice(0, 60) || "歌曲"; }
if (body.custom_mode) set("vocal_gender", input.gender === "male" ? "m" : "f");
else { delete body.vocal_gender; if (body.metadata) delete body.metadata.vocal_gender; }
} else {
set(modelId.startsWith("music-") ? "is_instrumental" : "make_instrumental", input.instrumental ? "instrumental" : "song");
if (!input.instrumental && input.lyrics) set("lyrics", input.lyrics);
if (modelId.startsWith("suno-")) set("vocal_gender", input.gender === "male" ? "m" : "f");
}
}
return body;
}
export const vip98kAudioCheckpointSchema = z.object({
operation: z.literal("audio"), provider: z.literal("vip98k"),
modelRef: z.string(), modelId: z.string(), providerId: z.string(), baseUrl: z.string(),
taskId: z.string().optional(), resultPath: z.string().optional(), requestId: z.string().optional(),
url: z.string().url().optional(), urls: z.array(z.string().url()).optional(), archivedUrls: z.array(z.string().url()).optional(),
data: z.string().optional(), text: z.string().optional(),
format: z.enum(["mp3", "wav"]),
}).refine(value => Boolean(value.taskId || value.url || value.data || value.text));

View File

@ -0,0 +1,24 @@
function format(bytes: Buffer): "mp3" | "wav" | undefined {
if (bytes.length < 12) return;
if (bytes.toString("ascii", 0, 4) === "RIFF" && bytes.toString("ascii", 8, 12) === "WAVE") return "wav";
if (bytes.toString("ascii", 0, 3) === "ID3" || (bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0)) return "mp3";
}
export function readVip98kAudioFile(bytes: Buffer) {
const direct = format(bytes);
if (direct) return { bytes, format: direct };
// Speech 2.8 returns a TAR containing audio and captions. Read the audio
// directly from memory; archive member paths are never written to disk.
if (bytes.toString("ascii", 257, 262) === "ustar") {
for (let offset = 0; offset + 512 <= bytes.length;) {
const header = bytes.subarray(offset, offset + 512);
const size = Number.parseInt(header.toString("ascii", 124, 136).replace(/\0.*$/, "").trim(), 8);
if (!Number.isFinite(size) || size < 0 || offset + 512 + size > bytes.length) break;
const data = bytes.subarray(offset + 512, offset + 512 + size);
const memberFormat = format(data);
if ((header[156] === 0 || header[156] === 48) && memberFormat) return { bytes: data, format: memberFormat };
offset += 512 + Math.ceil(size / 512) * 512;
}
}
throw new Error("返回结果不是有效的 MP3 或 WAV 音频文件");
}

View File

@ -0,0 +1,133 @@
import "server-only";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { randomUUID } from "node:crypto";
import { executeVip98kExternalTask } from "@/lib/ai/providers/vip98k";
import { getVip98kInvocationHeaders } from "@/lib/ai/vip98k-task-invocation";
import { uploadFile } from "@/lib/storage/service";
import { archiveMediaThroughQiniu } from "@/lib/storage/qiniu-image-archive";
import { downloadCanvasMedia } from "./download-media";
import { ProviderTaskUncertainError } from "./video-task-lifecycle";
import { buildVip98kAudioBody, vip98kAudioCheckpointSchema } from "./vip98k-audio-contract";
import { readVip98kAudioFile } from "./vip98k-audio-file";
import type { AudioRequest } from "./audio-contract";
import type { AiModel, AiProvider } from "@/lib/db/schema";
export async function executeVip98kCanvasAudio(input: AudioRequest, item: { model: AiModel; provider: AiProvider }, lifecycle: {
resume?: unknown; assertActive: () => Promise<void>; checkpoint: (value: Record<string, unknown>) => Promise<void>;
}) {
const { model, provider } = item;
const metadata = (model.capabilities as any)?.vip98k;
const invocation = metadata?.invocation;
if (!invocation?.path) throw new Error("音频模型缺少调用配置,请同步模型目录");
const identity = { operation: "audio" as const, provider: "vip98k" as const,
modelRef: model.id, modelId: model.modelId, providerId: provider.id,
baseUrl: provider.baseUrl.replace(/\/+$/, "").replace(/\/v1$/i, ""), format: input.format };
let saved = lifecycle.resume ? vip98kAudioCheckpointSchema.parse(lifecycle.resume) : undefined;
if (saved && Object.entries(identity).some(([key, value]) => (saved as any)[key] !== value))
throw new ProviderTaskUncertainError("原音频任务的服务线路或模型已变更,请核对原任务");
const save = async (value: typeof saved) => {
await lifecycle.assertActive();
await lifecycle.checkpoint({ upstreamAudio: value });
saved = value;
};
const dir = await mkdtemp(join(tmpdir(), "canvas-98k-audio-"));
try {
if (!saved?.url && !saved?.data && !saved?.text) {
let audioData: string | undefined;
if (!saved && input.referenceAudioUrl) {
const reference = join(dir, "reference");
await downloadCanvasMedia(input.referenceAudioUrl, reference);
const bytes = await readFile(reference);
if (bytes.length > 20 * 1024 * 1024) throw new Error("参考音频需小于 20 MiB");
const referenceAudio = readVip98kAudioFile(bytes);
audioData = `data:audio/${referenceAudio.format === "mp3" ? "mpeg" : "wav"};base64,${referenceAudio.bytes.toString("base64")}`;
}
const body = saved ? undefined : buildVip98kAudioBody(input, model.modelId, invocation.requestExample || {}, audioData);
if (invocation.path === "/v1/chat/completions") {
await lifecycle.assertActive();
let response: Response;
let payload: any;
try {
response = await fetch(identity.baseUrl + invocation.path, {
method: "POST", redirect: "error", signal: AbortSignal.timeout(180000),
headers: { Authorization: `Bearer ${provider.apiKey}`, "Content-Type": "application/json",
...getVip98kInvocationHeaders(invocation.headers), "Idempotency-Key": randomUUID() },
body: JSON.stringify(body),
});
payload = await response.json();
} catch {
throw new ProviderTaskUncertainError("98K 音频请求结果未确认,请核对原请求,系统不会自动重发");
}
if (!response.ok) {
if (response.status >= 500) throw new ProviderTaskUncertainError(`98K 音频结果未确认(HTTP ${response.status}),未重复提交`);
throw new Error(`98K 音频请求失败(HTTP ${response.status}),请核对模型和输入`);
}
const message = payload.choices?.[0]?.message;
const text = typeof message?.content === "string" ? message.content.trim() : "";
const audio = message?.audio;
const next = input.mode === "asr" ? { text } : { data: audio?.data, url: audio?.url };
if (!next.text && !next.data && !next.url) throw new ProviderTaskUncertainError("98K 未返回音频结果,请核对原请求,勿重复生成");
try { await save({ ...identity, ...next, requestId: payload.id }); }
catch { throw new ProviderTaskUncertainError("音频已生成,但结果保存失败,请核对原请求"); }
} else {
const result = await executeVip98kExternalTask({
apiKey: provider.apiKey, baseURL: identity.baseUrl, modelId: model.modelId, kind: "audio", prompt: String(body?.prompt || input.text),
modelConfig: { capabilities: { ...model.capabilities as any, vip98k: { ...metadata,
invocation: { ...invocation, requestExample: body || invocation.requestExample },
invocations: [],
} } },
lifecycle: { assertActive: lifecycle.assertActive,
resume: saved?.taskId ? { taskId: saved.taskId, resultPath: saved.resultPath } : undefined,
accepted: reference => save({ ...identity, ...reference }),
},
});
const payload = result.payload as any;
const outputs = payload?.result?.outputs || payload?.outputs;
const audioUrls = Array.isArray(outputs) ? outputs.filter(output => output.type === "audio" && typeof output.url === "string").map(output => output.url as string) : [];
const urls: string[] = audioUrls.length ? audioUrls : result.outputUrls.slice(0, 1);
const url = urls[0];
if (!url) throw new ProviderTaskUncertainError("音频任务已完成但未返回文件,请核对原任务");
await save({ ...identity, taskId: result.taskId, url, urls });
}
}
if (saved?.text) return { text: saved.text, modelRef: model.id, modelId: model.modelId, metadata: { outputType: "text", requestId: saved.requestId } };
const sources = saved?.urls?.length ? saved.urls : [saved?.url];
const archivedUrls = [...(saved?.archivedUrls || [])];
let format = saved!.format;
for (let index = archivedUrls.length; index < sources.length; index++) {
if (sources[index] && !model.modelId.startsWith("speech-")) {
const archived = await archiveMediaThroughQiniu(sources[index]!, `${saved!.taskId || saved!.requestId}-${index}`, "audio");
if (archived) {
const response = await fetch(archived.url, { headers: { Range: "bytes=0-1023" }, signal: AbortSignal.timeout(20000) });
if (!response.ok) throw new Error("音频文件尚不可下载");
const reader = response.body!.getReader();
const first = await reader.read(); await reader.cancel();
format = readVip98kAudioFile(Buffer.from(first.value || [])).format;
archivedUrls.push(archived.url);
await save({ ...saved!, archivedUrls });
continue;
}
}
let bytes: Buffer;
if (saved?.data) bytes = Buffer.from(saved.data.replace(/^data:[^,]+,/, ""), "base64");
else {
const output = join(dir, `result-${index}`);
await downloadCanvasMedia(sources[index]!, output);
bytes = await readFile(output);
}
const audio = readVip98kAudioFile(bytes);
format = audio.format;
await lifecycle.assertActive();
const result = await uploadFile(`canvas-audio-${randomUUID()}.${format}`, Uint8Array.from(audio.bytes).buffer);
archivedUrls.push(result.url);
await save({ ...saved!, archivedUrls });
}
return { url: archivedUrls[0], urls: archivedUrls, modelRef: model.id, modelId: model.modelId,
metadata: { storageStatus: "archived", format, requestId: saved?.requestId, taskId: saved?.taskId } };
} catch (error) {
if (saved?.url || saved?.data) throw new ProviderTaskUncertainError("音频已生成,保存文件尚未完成,将继续保存原文件");
throw error;
} finally { await rm(dir, { recursive: true, force: true }); }
}

View File

@ -65,6 +65,7 @@
"providers:test-vip98k-images": "pnpm exec tsx scripts/test-vip98k-image-models.ts" "providers:test-vip98k-images": "pnpm exec tsx scripts/test-vip98k-image-models.ts"
}, },
"dependencies": { "dependencies": {
"@ai-sdk/anthropic": "2.0.13",
"@ai-sdk/gateway": "^1.0.15", "@ai-sdk/gateway": "^1.0.15",
"@ai-sdk/openai": "2.0.13", "@ai-sdk/openai": "2.0.13",
"@ai-sdk/provider": "2.0.0", "@ai-sdk/provider": "2.0.0",

View File

@ -72,6 +72,7 @@ export default defineConfig({
/routes\/hicad-manufacturing-extension\.test\.ts/, /routes\/hicad-manufacturing-extension\.test\.ts/,
/routes\/openapi-access-key\.test\.ts/, /routes\/openapi-access-key\.test\.ts/,
/routes\/vip98k-reference-routing\.test\.ts/, /routes\/vip98k-reference-routing\.test\.ts/,
/routes\/vip98k-model-expansion\.test\.ts/,
/routes\/ordinal-image-references\.test\.ts/, /routes\/ordinal-image-references\.test\.ts/,
/routes\/extend-provider-config\.test\.ts/, /routes\/extend-provider-config\.test\.ts/,
], ],

32
pnpm-lock.yaml generated
View File

@ -33,6 +33,9 @@ importers:
.: .:
dependencies: dependencies:
'@ai-sdk/anthropic':
specifier: 2.0.13
version: 2.0.13(zod@3.25.76)
'@ai-sdk/gateway': '@ai-sdk/gateway':
specifier: ^1.0.15 specifier: ^1.0.15
version: 1.0.41(zod@3.25.76) version: 1.0.41(zod@3.25.76)
@ -463,6 +466,20 @@ importers:
packages: packages:
'@ai-sdk/anthropic@2.0.13':
resolution: {integrity: sha512-5Kt1sNy80pKJsThe3snL9OajgNfdjH7GiWj/CyDoMNY82+Pb5mHimCwZzMUUoAM77GjfSd3Ico8sE7pPdJOuYQ==}
engines: {node: '>=18'}
peerDependencies:
zod: ^3.25.76 || ^4
'@ai-sdk/provider-utils@3.0.8':
resolution: {integrity: sha512-cDj1iigu7MW2tgAQeBzOiLhjHOUM9vENsgh4oAVitek0d//WdgfPCsKO3euP7m7LyO/j9a1vr/So+BGNdpFXYw==}
engines: {node: '>=18'}
peerDependencies:
zod: ^3.25.76 || ^4
gsap@3.14.2: gsap@3.14.2:
resolution: {integrity: sha512-P8/mMxVLU7o4+55+1TCnQrPmgjPKnwkzkXOK1asnR9Jg2lna4tEY5qBJjMmAaOBDDZWtlRjBXjLa0w53G/uBLA==} resolution: {integrity: sha512-P8/mMxVLU7o4+55+1TCnQrPmgjPKnwkzkXOK1asnR9Jg2lna4tEY5qBJjMmAaOBDDZWtlRjBXjLa0w53G/uBLA==}
@ -6973,6 +6990,21 @@ packages:
snapshots: snapshots:
'@ai-sdk/anthropic@2.0.13(zod@3.25.76)':
dependencies:
'@ai-sdk/provider': 2.0.0
'@ai-sdk/provider-utils': 3.0.8(zod@3.25.76)
zod: 3.25.76
'@ai-sdk/provider-utils@3.0.8(zod@3.25.76)':
dependencies:
'@ai-sdk/provider': 2.0.0
'@standard-schema/spec': 1.1.0
eventsource-parser: 3.0.6
zod: 3.25.76
gsap@3.14.2: {} gsap@3.14.2: {}
'@ai-sdk/gateway@1.0.15(zod@3.25.76)': '@ai-sdk/gateway@1.0.15(zod@3.25.76)':

View File

@ -55,6 +55,13 @@ test("restores accepted tasks missing from a pre-submit canvas snapshot, includi
expect(result[0].draftValues?.url).toHaveLength(2); expect(result[0].draftValues?.url).toHaveLength(2);
expect(reconcileCanvasTasks<TaskNode>(result, [task], "2026-09-17T01:00:00Z")).toBe(result); expect(reconcileCanvasTasks<TaskNode>(result, [task], "2026-09-17T01:00:00Z")).toBe(result);
}); });
test("music variations become separate playable resources without duplicate cards on polling", () => {
const music:CanvasGenerationTask={...task,request:{...task.request,kind:"audio"},result:{url:"https://example.com/one.mp3",urls:["https://example.com/one.mp3","https://example.com/two.mp3"]}};
const result=reconcileCanvasTasks<TaskNode>([],[music],"2026-09-17T01:00:00Z");
expect(result).toHaveLength(2);
expect(result[1]).toMatchObject({inputType:"audio",draftValues:{action:"AUDIO_RESOURCE",mediaUrl:"https://example.com/two.mp3"}});
expect(reconcileCanvasTasks(result,[music],"2026-09-17T01:00:00Z")).toBe(result);
});
test("does not resurrect nodes removed after task creation or hidden on the server", () => { test("does not resurrect nodes removed after task creation or hidden on the server", () => {
expect(reconcileCanvasTasks<TaskNode>([], [task], "2026-09-17T01:02:00Z")).toEqual([]); expect(reconcileCanvasTasks<TaskNode>([], [task], "2026-09-17T01:02:00Z")).toEqual([]);
expect( expect(

View File

@ -0,0 +1,106 @@
import { test, expect } from "@playwright/test";
import { createVip98kChatModel } from "@/lib/ai/providers/vip98k-chat";
import { buildVip98kModelPresets } from "@/lib/ai/vip98k-models";
import { buildTaskBody } from "@/lib/ai/providers/vip98k";
import { getVip98kImageBindingKeys } from "@/lib/ai/vip98k-task-invocation";
import { getAudioCheckpoint } from "@/lib/video-canvas/task-recovery";
import { audioRequestSchema } from "@/lib/video-canvas/audio-contract";
import { buildVip98kAudioBody, vip98kAudioContracts } from "@/lib/video-canvas/vip98k-audio-contract";
import { vip98kVideoDurationBounds } from "@/lib/ai/vip98k-video-parameters";
import { readVip98kAudioFile } from "@/lib/video-canvas/vip98k-audio-file";
test("Anthropic-only routes preserve tool calls and relay routing", async () => {
let sent: any;
const model = createVip98kChatModel({ apiKey:"fixture", baseURL:"https://relay.example", modelId:"claude-haiku-5-5",
metadata:{invocation:{kind:"chat",path:"/v1/messages",headers:{"X-Relay-Route-Option":"native_route"}}},
fetch:async (url, init) => {
expect(String(url)).toBe("https://relay.example/v1/messages");
const headers = new Headers(init?.headers);
expect(headers.get("x-relay-route-option")).toBe("native_route");
expect(headers.get("authorization")).toBe("Bearer fixture");
sent=JSON.parse(String(init?.body));
return Response.json({id:"message_test",type:"message",role:"assistant",model:"claude-haiku-5-5",
content:[{type:"tool_use",id:"call_test",name:"record_result",input:{message:"ok"}}],
stop_reason:"tool_use",stop_sequence:null,usage:{input_tokens:10,output_tokens:8}});
},
});
const result = await model.doGenerate({ prompt:[{role:"user",content:[{type:"text",text:"Run the test tool"}]}],
tools:[{type:"function",name:"record_result",inputSchema:{type:"object",properties:{message:{type:"string"}},required:["message"]}}],toolChoice:{type:"auto"},maxOutputTokens:128 });
expect(sent.messages[0].content[0].text).toBe("Run the test tool");
expect(result.content).toContainEqual(expect.objectContaining({type:"tool-call",toolName:"record_result"}));
});
test("zero image slots and forbidden fields do not become reference support", () => {
const invocation = {kind:"tool",path:"/v1/images/generations",request_example:{model:"text_image",prompt:"a cup"},
input_schema:{properties:{file_urls:{type:"array",maxItems:0},image:false,partial_images:{type:"integer"}}}};
const [model] = buildVip98kModelPresets([{id:"text_image",type:"image",capabilities:["text_to_image"],invocation}]);
expect(model.capabilities).toMatchObject({maxReferenceImages:0,supportedModes:["text-to-image"]});
expect(model.displayName).toContain("仅文生图");
expect(getVip98kImageBindingKeys({requestExample:invocation.request_example,inputSchema:invocation.input_schema} as any)).toEqual([]);
});
const params = {apiKey:"fixture",modelId:"image",kind:"image",prompt:"a blue cup",imageUrls:[],videoUrls:[],modelConfig:{},imageParams:{aspectRatio:"16:9",imageSize:"2K"}};
const invocation = (properties: Record<string, unknown>, requestExample: Record<string, unknown> = {}) => ({kind:"tool",method:"POST",path:"/v1/images/generations",contentType:"application/json",headers:{},requestExample:{model:"image",prompt:"example",...requestExample},inputSchema:{properties},resultMethod:"GET",resultPath:""});
test("ratio-sized image contracts bind size and resolution without stale square examples", () => {
const result = buildTaskBody(params,invocation({size:{enum:["1:1","16:9"]},resolution:{enum:["1K","2K"]}},{size:"1:1",resolution:"1K"}));
expect(result.body).toMatchObject({size:"16:9",resolution:"2K",prompt:"a blue cup"});
});
test("pixel-only image contracts use a supported size in the requested tier", () => {
const result = buildTaskBody(params,invocation({size:{enum:["1024x1024","1280x720","2048x1152","3840x2160"]}},{size:"1024x1024"}));
expect(result.body.size).toBe("2048x1152");
});
test("OpenAI image size fields receive pixel dimensions instead of a resolution label", () => {
const result = buildTaskBody({...params,modelId:"gpt-image-2"},invocation({size:{type:["string","null"]}}));
expect(result.body.size).toMatch(/^\d+x\d+$/);
});
test("task size patterns take precedence over ratio aliases in a catalog enum", () => {
const result = buildTaskBody({...params,imageParams:{imageSize:"1K",aspectRatio:"1:1"}},invocation({size:{enum:["1:1","1024x1024"],pattern:"^(auto|[1-9][0-9]*x[1-9][0-9]*)$"},resolution:{enum:["1K","2K"]}}));
expect(result.body.size).toBe("1024x1024");
});
test("image reference limits reject excess inputs before submission", () => {
expect(() => buildTaskBody({...params,imageUrls:["https://example.com/1.png","https://example.com/2.png"]},invocation({image:{type:"array",maxItems:1}},{image:[]}))).toThrow("最多支持 1 张");
});
test("Seedance content binds prompts and reference roles", () => {
const i=invocation({content:{items:{oneOf:[{properties:{type:{const:"text"}}},{properties:{type:{const:"image_url"}}}]}},duration:{anyOf:[{const:-1},{minimum:4,maximum:30}]},resolution:{enum:["480P","720P"]},ratio:{enum:["16:9","9:16"]}});
const result=buildTaskBody({...params,kind:"video",imageUrls:["https://example.com/one.png","https://example.com/two.png"],videoParams:{mode:"first-last-frame",resolution:"480p",duration:4,ratio:"16:9"}},i);
expect(result.body).toMatchObject({resolution:"480P",duration:4,content:[{type:"text",text:"a blue cup"},{type:"image_url",role:"first_frame"},{type:"image_url",role:"last_frame"}]});
expect(result.imageBindingKeys).toContain("content");
expect(vip98kVideoDurationBounds(i.inputSchema)).toMatchObject({min:4,max:30});
});
test("all fourteen audio entries expose distinct supported modes", () => {
expect(Object.keys(vip98kAudioContracts)).toHaveLength(14);
expect(vip98kAudioContracts["mimo-v2.5-asr"].kind).toBe("asr");
expect(vip98kAudioContracts["mimo-v2.5-tts-voiceclone"].kind).toBe("voiceclone");
expect(vip98kAudioContracts["suno-v6"].kind).toBe("music");
});
test("reference-only video routes reject an empty image list before submission", () => {
expect(()=>buildTaskBody({...params,kind:"video"},invocation({file_urls:{type:"array",minItems:1,maxItems:1}}))).toThrow("参考图");
});
test("Speech archives return the audio member instead of the TAR bytes", () => {
const audio=Buffer.from("RIFF1234WAVEfmt fixture audio");
const header=Buffer.alloc(512);header.write("output.wav");header.write(audio.length.toString(8).padStart(11,"0"),124);header[156]=48;header.write("ustar",257);
const file=readVip98kAudioFile(Buffer.concat([header,audio,Buffer.alloc(512-audio.length),Buffer.alloc(1024)]));
expect(file.format).toBe("wav");expect(file.bytes).toEqual(audio);
expect(()=>readVip98kAudioFile(Buffer.from('{"error":"unavailable"}'))).toThrow("音频文件");
});
test("audio input requires a reference for cloning and maps the user's audio", () => {
const input=audioRequestSchema.parse({modelRef:"test",mode:"voiceclone",text:"你好"});
expect(()=>buildVip98kAudioBody(input,"mimo-v2.5-tts-voiceclone",{})).toThrow("参考音频");
const body=buildVip98kAudioBody({...input,referenceAudioUrl:"https://example.com/audio.wav"},"mimo-v2.5-tts-voiceclone",{},"data:audio/wav;base64,fixture");
expect(body.audio.voice).toBe("data:audio/wav;base64,fixture");
expect(body.messages[1]).toEqual({role:"assistant",content:"你好"});
});
test("Suno automatic music omits the gender parameter that requires custom lyrics mode", () => {
const input=audioRequestSchema.parse({modelRef:"test",mode:"music",text:"gentle piano",format:"mp3",instrumental:true});
const body=buildVip98kAudioBody(input,"suno-v5",{custom_mode:false,metadata:{vocal_gender:"m"}});
expect(body.custom_mode).toBe(false);expect(body.metadata.vocal_gender).toBeUndefined();
const song=buildVip98kAudioBody({...input,instrumental:false,lyrics:"阳光照进窗台"},"suno-v5",{custom_mode:false,metadata:{vocal_gender:"m"}});
expect(song).toMatchObject({custom_mode:true,prompt:"阳光照进窗台",style:"gentle piano",metadata:{vocal_gender:"f"}});
});
test("completed audio checkpoints retain the original provider and resumable task", () => {
const checkpoint={operation:"audio",provider:"vip98k",modelRef:"test",modelId:"gem-2.5-tts",providerId:"source",baseUrl:"https://relay.example",taskId:"original",url:"https://example.com/audio.wav",format:"wav"};
expect(getAudioCheckpoint({request:{kind:"audio",params:{modelRef:"test"}},result:{upstreamAudio:checkpoint}})).toEqual(checkpoint);
});

View File

@ -755,7 +755,7 @@ test.describe("VIP98K reference-image routing", () => {
).toBe(false); ).toBe(false);
}); });
test("excludes unverified text-only VIP98K models from the image picker", () => { test("imports published text-only models without advertising image editing", () => {
const presets = buildVip98kModelPresets([ const presets = buildVip98kModelPresets([
{ {
id: "unverified-image-model", id: "unverified-image-model",
@ -771,7 +771,9 @@ test.describe("VIP98K reference-image routing", () => {
}, },
]); ]);
expect(presets).toEqual([]); expect(presets).toHaveLength(1);
expect(presets[0]).toMatchObject({ supportsVision: false,
capabilities: { supportedModes: ["text-to-image"], maxReferenceImages: 0 } });
}); });
test("keeps gpt-image-2-vip when its published image field is an array", () => { test("keeps gpt-image-2-vip when its published image field is an array", () => {