diff --git a/lib/ai/providers/vip98k.ts b/lib/ai/providers/vip98k.ts index 424bef452..4d7833db5 100644 --- a/lib/ai/providers/vip98k.ts +++ b/lib/ai/providers/vip98k.ts @@ -30,6 +30,16 @@ type Vip98kModelRuntimeConfig = { imageGenConfig?: Record | null; }; +type Vip98kInvocation = { + kind: string; + method: string; + path: string; + contentType: string; + requestExample: Record; + resultMethod: string; + resultPath: string; +}; + type UploadedImage = { thumbnail: string; original: string; @@ -75,6 +85,23 @@ function getVip98kMetadata(config: Vip98kModelRuntimeConfig) { return capabilitiesVip98k || imageConfigVip98k || {}; } +function getVip98kInvocation(config: Vip98kModelRuntimeConfig): Vip98kInvocation | null { + const metadata = getVip98kMetadata(config); + const invocation = toRecord(metadata.invocation); + if (!invocation) return null; + const path = String(invocation.path || "").trim(); + if (!path.startsWith("/")) return null; + return { + kind: String(invocation.kind || "task").trim().toLowerCase(), + method: String(invocation.method || "POST").trim().toUpperCase(), + path, + contentType: String(invocation.contentType || "application/json").trim(), + requestExample: { ...(toRecord(invocation.requestExample) || {}) }, + resultMethod: String(invocation.resultMethod || "GET").trim().toUpperCase(), + resultPath: String(invocation.resultPath || "").trim(), + }; +} + function inferKind(modelId: string, config: Vip98kModelRuntimeConfig) { const capabilities = config.capabilities; const metadata = getVip98kMetadata(config); @@ -391,14 +418,19 @@ function getConfiguredPollSeconds(config: Vip98kModelRuntimeConfig, key: string, async function submitTask(params: { apiKey: string; baseURL?: string | null; + path?: string | null; + method?: string | null; + contentType?: string | null; body: Record; }) { const baseURL = normalizeVip98kBaseUrl(params.baseURL); - const response = await fetch(`${baseURL}/api/external/task/generate`, { - method: "POST", + const path = String(params.path || "/api/external/task/generate").trim(); + if (!path.startsWith("/")) throw new Error("VIP98K invocation path is invalid"); + const response = await fetch(`${baseURL}${path}`, { + method: String(params.method || "POST").toUpperCase(), headers: { Authorization: `Bearer ${params.apiKey}`, - "Content-Type": "application/json", + "Content-Type": params.contentType || "application/json", }, body: JSON.stringify(params.body), signal: AbortSignal.timeout(120_000), @@ -459,14 +491,21 @@ export async function submitVip98kExternalTask(params: { imageParams: params.imageParams, modelConfig: params.modelConfig || {}, }; + const invocation = getVip98kInvocation(taskParams.modelConfig); const submitted = await submitTask({ apiKey: params.apiKey, baseURL: params.baseURL, - body: buildTaskBody(taskParams), + path: invocation?.path, + method: invocation?.method, + contentType: invocation?.contentType, + body: buildTaskBody(taskParams, invocation), }); + const resultPath = submitted.resultPath || invocation?.resultPath || ""; return { taskId: submitted.taskId, - resultPath: submitted.resultPath, + resultPath: submitted.taskId + ? resultPath.replaceAll("{task_id}", encodeURIComponent(submitted.taskId)) + : resultPath, outputUrls: submitted.outputs, payload: submitted.payload, }; @@ -573,15 +612,16 @@ function normalizeDuration(value: unknown) { return Math.round(numeric); } -function buildTaskBody(params: Vip98kTaskParams) { +function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvocation | null) { const videoParams = params.videoParams || {}; const imageParams = params.imageParams || {}; const promptTitle = params.prompt.split(/\s+/).slice(0, 8).join(" ").slice(0, 80); - const body: Record = { - model: params.modelId, - displayName: promptTitle || params.modelId, - prompt: params.prompt, - }; + const body: Record = { ...(invocation?.requestExample || {}) }; + if (invocation?.kind !== "tool" || "model" in body) body.model = params.modelId; + if ("displayName" in body || invocation?.kind !== "tool") { + body.displayName = promptTitle || params.modelId; + } + if ("prompt" in body || !invocation) body.prompt = params.prompt; const aspectRatio = normalizeRatio(videoParams.ratio) || @@ -592,54 +632,64 @@ function buildTaskBody(params: Vip98kTaskParams) { const negativePrompt = String(videoParams.negativePrompt || imageParams.negativePrompt || "").trim(); if (aspectRatio) { - body.ratio = aspectRatio; - body.aspectRatio = aspectRatio; - body.aspect_ratio = aspectRatio; + if ("ratio" in body || !invocation) body.ratio = aspectRatio; + if ("aspectRatio" in body || !invocation) body.aspectRatio = aspectRatio; + if ("aspect_ratio" in body || !invocation) body.aspect_ratio = aspectRatio; } - if (duration) body.duration = duration; + if (duration && ("duration" in body || !invocation)) body.duration = duration; if (resolution) { - body.resolution = resolution; - body.imageSize = resolution; - body.image_size = resolution; + if ("resolution" in body || !invocation) body.resolution = resolution; + if ("imageSize" in body || !invocation) body.imageSize = resolution; + if ("image_size" in body || !invocation) body.image_size = resolution; + if ("size" in body && typeof body.size === "string") body.size = resolution; } if (negativePrompt) { - body.negativePrompt = negativePrompt; - body.negative_prompt = negativePrompt; + if ("negativePrompt" in body || !invocation) body.negativePrompt = negativePrompt; + if ("negative_prompt" in body || !invocation) body.negative_prompt = negativePrompt; } if (typeof videoParams.generateAudio === "boolean") { - body.generateAudio = videoParams.generateAudio; - body.generate_audio = videoParams.generateAudio; + if ("generateAudio" in body || !invocation) body.generateAudio = videoParams.generateAudio; + if ("generate_audio" in body || !invocation) body.generate_audio = videoParams.generateAudio; } if (params.imageUrls.length > 0) { - body.images = params.imageUrls; - body.imageUrls = params.imageUrls; - body.referenceImages = params.imageUrls; - body.reference_images = params.imageUrls; - body.image = params.imageUrls[0]; - body.imageUrl = params.imageUrls[0]; - body.image_url = params.imageUrls[0]; - body.firstFrameImage = params.imageUrls[0]; - body.first_frame_image = params.imageUrls[0]; - body.firstFrameImageUrl = params.imageUrls[0]; - body.first_frame_image_url = params.imageUrls[0]; + const setImageField = (key: string, value: unknown) => { + if (key in body || !invocation) body[key] = value; + }; + setImageField("images", params.imageUrls); + setImageField("imageUrls", params.imageUrls); + setImageField("referenceImages", params.imageUrls); + setImageField("reference_images", params.imageUrls); + setImageField("image", params.imageUrls[0]); + setImageField("imageUrl", params.imageUrls[0]); + setImageField("image_url", params.imageUrls[0]); + setImageField("firstFrameImage", params.imageUrls[0]); + setImageField("first_frame_image", params.imageUrls[0]); + setImageField("firstFrameImageUrl", params.imageUrls[0]); + setImageField("first_frame_image_url", params.imageUrls[0]); + if (Array.isArray(body.video_clips)) { + const template = toRecord(body.video_clips[0]) || {}; + body.video_clips = params.imageUrls.map((url) => ({ ...template, url })); + } if (params.imageUrls[1]) { - body.lastFrameImage = params.imageUrls[1]; - body.last_frame_image = params.imageUrls[1]; - body.endImageUrl = params.imageUrls[1]; - body.end_image_url = params.imageUrls[1]; + setImageField("lastFrameImage", params.imageUrls[1]); + setImageField("last_frame_image", params.imageUrls[1]); + setImageField("endImageUrl", params.imageUrls[1]); + setImageField("end_image_url", params.imageUrls[1]); } } if (params.videoUrls.length > 0) { - body.videos = params.videoUrls; - body.videoUrls = params.videoUrls; - body.referenceVideos = params.videoUrls; + if ("videos" in body || !invocation) body.videos = params.videoUrls; + if ("videoUrls" in body || !invocation) body.videoUrls = params.videoUrls; + if ("referenceVideos" in body || !invocation) body.referenceVideos = params.videoUrls; + if ("video" in body) body.video = params.videoUrls[0]; + if ("video_url" in body) body.video_url = params.videoUrls[0]; } - if (params.kind === "world") { + if (params.kind === "world" && ("mode" in body || !invocation)) { body.mode = "world-generation"; - } else if (params.kind === "video") { + } else if (params.kind === "video" && ("mode" in body || !invocation)) { body.mode = params.imageUrls.length > 0 ? "image-to-video" : "text-to-video"; } diff --git a/lib/ai/vip98k-models.ts b/lib/ai/vip98k-models.ts index a62a5d0fa..a2f7637d5 100644 --- a/lib/ai/vip98k-models.ts +++ b/lib/ai/vip98k-models.ts @@ -10,6 +10,16 @@ export type Vip98kCatalogModelType = | "world" | string; +export interface Vip98kInvocationContract { + kind?: "chat" | "task" | "tool" | string; + method?: string; + path?: string; + content_type?: string; + request_example?: Record; + result_method?: string; + result_path?: string; +} + export interface Vip98kCatalogModel { modelId?: string; id?: string; @@ -47,6 +57,9 @@ export interface Vip98kCatalogModel { pricing?: unknown; price?: unknown; status?: string; + invocation?: Vip98kInvocationContract; + taskTypes?: unknown; + task_types?: unknown; [key: string]: unknown; } @@ -248,9 +261,24 @@ function getBillingMode(model: Vip98kCatalogModel) { } function getVip98kMetadata(model: Vip98kCatalogModel, type: string) { + const invocation = toRecord(model.invocation); return { modelType: type, - taskApi: "external-task", + taskApi: + String(invocation?.kind || "").toLowerCase() === "tool" + ? "external-tool" + : "external-task", + invocation: invocation + ? { + kind: String(invocation.kind || ""), + method: String(invocation.method || "POST"), + path: String(invocation.path || ""), + contentType: String(invocation.content_type || "application/json"), + requestExample: toRecord(invocation.request_example) || {}, + resultMethod: String(invocation.result_method || "GET"), + resultPath: String(invocation.result_path || ""), + } + : null, supportsAsync: Boolean(model.supportsAsync ?? model.supports_async ?? true), pollIntervalSeconds: parseNumber( model.pollIntervalSeconds || model.poll_interval_seconds, @@ -646,6 +674,17 @@ function extractCatalogModels(payload: unknown): Vip98kCatalogModel[] { return []; } +function extractPublicCatalogModels(payload: unknown): Vip98kCatalogModel[] { + const record = toRecord(payload); + const data = toRecord(record?.data); + const sections = Array.isArray(data?.sections) ? data.sections : []; + return sections.flatMap((section) => { + const sectionRecord = toRecord(section); + const models = Array.isArray(sectionRecord?.models) ? sectionRecord.models : []; + return models.filter((model): model is Vip98kCatalogModel => Boolean(toRecord(model))); + }); +} + async function parseJsonResponse(response: Response) { const text = await response.text(); if (!text.trim()) return null; @@ -676,23 +715,58 @@ export async function fetchVip98kCatalog(params: { apiKey: string; }): Promise { const baseURL = normalizeVip98kBaseUrl(params.baseURL); - const response = await fetch(`${baseURL}/api/external/task/models`, { - cache: "no-store", - headers: { - Authorization: `Bearer ${params.apiKey}`, - Accept: "application/json", - }, - signal: AbortSignal.timeout(60_000), - }); - const payload = await parseJsonResponse(response); - if (!response.ok) { - throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`); + const [authorizedResponse, publicResponse] = await Promise.all([ + fetch(`${baseURL}/api/external/task/models`, { + cache: "no-store", + headers: { + Authorization: `Bearer ${params.apiKey}`, + Accept: "application/json", + }, + signal: AbortSignal.timeout(60_000), + }), + fetch(`${baseURL}/api/v1/public/models`, { + cache: "no-store", + headers: { Accept: "application/json" }, + signal: AbortSignal.timeout(60_000), + }), + ]); + const [authorizedPayload, publicPayload] = await Promise.all([ + parseJsonResponse(authorizedResponse), + parseJsonResponse(publicResponse), + ]); + if (!authorizedResponse.ok) { + throw new Error( + `${extractErrorMessage(authorizedPayload)} (HTTP ${authorizedResponse.status})`, + ); } - const models = extractCatalogModels(payload); - if (models.length === 0) { + if (!publicResponse.ok) { + throw new Error( + `VIP98K public invocation catalog failed (HTTP ${publicResponse.status})`, + ); + } + const authorizedModels = extractCatalogModels(authorizedPayload); + const publicModels = extractPublicCatalogModels(publicPayload); + if (authorizedModels.length === 0) { throw new Error("VIP98K model catalog is empty or has an unsupported response shape"); } - return models; + if (publicModels.length === 0) { + throw new Error("VIP98K public invocation catalog is empty"); + } + + const publicById = new Map( + publicModels.map((model) => [getModelId(model).toLowerCase(), model]), + ); + return authorizedModels.map((model) => { + const publicModel = publicById.get(getModelId(model).toLowerCase()); + if (!publicModel) return model; + return { + ...publicModel, + ...model, + invocation: publicModel.invocation, + taskTypes: model.taskTypes || model.task_types || publicModel.taskTypes || publicModel.task_types, + capabilities: model.capabilities || publicModel.capabilities, + }; + }); } export async function fetchVip98kModelPresets(params: { diff --git a/scripts/test-vip98k-capability.ts b/scripts/test-vip98k-capability.ts index b553b309d..91eb30871 100644 --- a/scripts/test-vip98k-capability.ts +++ b/scripts/test-vip98k-capability.ts @@ -15,6 +15,19 @@ type Vip98kCatalogModel = { model_id?: string; slug?: string; name?: string; + displayName?: string; + display_name?: string; + invocation?: Vip98kInvocation; +}; + +type Vip98kInvocation = { + kind?: string; + method?: string; + path?: string; + content_type?: string; + request_example?: Record; + result_method?: string; + result_path?: string; }; const CAPABILITIES = [ @@ -50,8 +63,8 @@ const DEFAULT_MODEL_IDS: Record = { multimodal: ["gpt-5.4-mini", "gpt-5.4", "claude-sonnet-4-6"], image: ["nano-banana-2", "nano-banana-pro", "gpt-image-2-vip"], "multi-image-edit": [ - "google/nano-banana-2/reference-to-image", "google/nano-banana-pro/edit", + "google/nano-banana-2/edit", "openai/gpt-image-2/edit", ], rmbg: ["remove-background"], @@ -93,25 +106,53 @@ function normalizeVip98kBaseUrl(baseUrl?: string | null) { } async function fetchVip98kCatalog(params: { baseURL: string; apiKey: string }) { - const payload = await parseResponse( - await fetch(`${normalizeVip98kBaseUrl(params.baseURL)}/api/external/task/models`, { - headers: { - Authorization: `Bearer ${params.apiKey}`, - Accept: "application/json", - }, - signal: AbortSignal.timeout(60_000), - }), - ); - const record = payload && typeof payload === "object" ? (payload as Record) : {}; + const baseUrl = normalizeVip98kBaseUrl(params.baseURL); + const [authorizedPayload, publicPayload] = await Promise.all([ + parseResponse( + await fetch(`${baseUrl}/api/external/task/models`, { + headers: { + Authorization: `Bearer ${params.apiKey}`, + Accept: "application/json", + }, + signal: AbortSignal.timeout(60_000), + }), + ), + parseResponse( + await fetch(`${baseUrl}/api/v1/public/models`, { + headers: { Accept: "application/json" }, + signal: AbortSignal.timeout(60_000), + }), + ), + ]); + const record = + authorizedPayload && typeof authorizedPayload === "object" + ? (authorizedPayload as Record) + : {}; const data = record.data; - const models = + const authorized = (Array.isArray(data?.models) && data.models) || (Array.isArray(data?.items) && data.items) || (Array.isArray(data) && data) || (Array.isArray(record.models) && record.models) || []; - if (models.length === 0) throw new Error("VIP98K model catalog is empty"); - return models as Vip98kCatalogModel[]; + const publicRecord = + publicPayload && typeof publicPayload === "object" + ? (publicPayload as Record) + : {}; + const publicModels = (Array.isArray(publicRecord.data?.sections) + ? publicRecord.data.sections + : [] + ).flatMap((section: any) => (Array.isArray(section?.models) ? section.models : [])); + if (authorized.length === 0) throw new Error("VIP98K authorized model catalog is empty"); + if (publicModels.length === 0) throw new Error("VIP98K public invocation catalog is empty"); + const publicById = new Map( + publicModels.map((model: Vip98kCatalogModel) => [getModelId(model).toLowerCase(), model]), + ); + return authorized.map((model: Vip98kCatalogModel) => ({ + ...(publicById.get(getModelId(model).toLowerCase()) || {}), + ...model, + invocation: publicById.get(getModelId(model).toLowerCase())?.invocation, + })) as Vip98kCatalogModel[]; } function sanitizeError(error: unknown) { @@ -304,14 +345,16 @@ async function runExternalTask(params: { apiKey: string; modelId: string; capability: Capability; + invocation: Vip98kInvocation; }) { const imageUrl = "https://dummyimage.com/512x512/2563eb/ffffff.png&text=VIP98K"; const secondImageUrl = "https://dummyimage.com/512x512/dc2626/ffffff.png&text=REFERENCE"; - const body: Record = { - model: params.modelId, - displayName: `provider-smoke-${params.capability}`, - prompt: "Create a minimal test result with one simple object", - }; + const body: Record = JSON.parse( + JSON.stringify(params.invocation.request_example || {}), + ); + if ("model" in body) body.model = params.modelId; + if ("displayName" in body) body.displayName = `provider-smoke-${params.capability}`; + if ("prompt" in body) body.prompt = "Create a minimal test result with one simple object"; if ( ["multi-image-edit", "rmbg", "enhance", "extend", "image-to-3d"].includes( params.capability, @@ -319,40 +362,62 @@ async function runExternalTask(params: { ) { const images = params.capability === "multi-image-edit" ? [imageUrl, secondImageUrl] : [imageUrl]; - Object.assign(body, { - image: imageUrl, - imageUrl, - image_url: imageUrl, - images, - imageUrls: images, - referenceImages: images, - reference_images: images, - }); + if ("image" in body) body.image = imageUrl; + if ("imageUrl" in body) body.imageUrl = imageUrl; + if ("image_url" in body) body.image_url = imageUrl; + if ("images" in body) body.images = images; + if ("imageUrls" in body) body.imageUrls = images; + if ("referenceImages" in body) body.referenceImages = images; + if ("reference_images" in body) body.reference_images = images; + if (Array.isArray(body.video_clips)) { + const template = + body.video_clips[0] && typeof body.video_clips[0] === "object" + ? (body.video_clips[0] as Record) + : {}; + body.video_clips = images.map((url) => ({ ...template, url })); + } + if ( + params.capability === "multi-image-edit" && + !Array.isArray(body.images) && + !Array.isArray(body.referenceImages) && + !Array.isArray(body.reference_images) && + !Array.isArray(body.video_clips) + ) { + throw new Error(`${params.modelId} official invocation does not expose multi-image input`); + } } if (params.capability === "extend") { - Object.assign(body, { aspectRatio: "16:9", aspect_ratio: "16:9", ratio: "16:9" }); + if ("aspectRatio" in body) body.aspectRatio = "16:9"; + if ("aspect_ratio" in body) body.aspect_ratio = "16:9"; + if ("ratio" in body) body.ratio = "16:9"; } - if (params.capability === "video") { - Object.assign(body, { mode: "text-to-video", duration: 4, resolution: "480p", ratio: "16:9" }); - } - if (params.capability === "world") body.mode = "world-generation"; + if (params.capability === "world" && "mode" in body) body.mode = "world-generation"; + + const path = String(params.invocation.path || "").trim(); + if (!path.startsWith("/")) throw new Error(`${params.modelId} has no valid invocation path`); const submitted = await parseResponse( - await fetch(`${params.baseUrl}/api/external/task/generate`, { - method: "POST", + await fetch(`${params.baseUrl}${path}`, { + method: String(params.invocation.method || "POST").toUpperCase(), headers: { Authorization: `Bearer ${params.apiKey}`, - "Content-Type": "application/json", + "Content-Type": params.invocation.content_type || "application/json", }, body: JSON.stringify(body), signal: AbortSignal.timeout(180_000), }), ); - const immediate = Array.from(collectUrls(submitted)); + const inputUrls = new Set([imageUrl, secondImageUrl]); + const immediate = Array.from(collectUrls(submitted)).filter((url) => !inputUrls.has(url)); if (immediate.length > 0) return immediate; const taskId = extractTaskId(submitted); if (!taskId) throw new Error("External task returned neither output nor task ID"); - const resultPath = extractResultPath(submitted); + const resultPath = + extractResultPath(submitted) || + String(params.invocation.result_path || "").replaceAll( + "{task_id}", + encodeURIComponent(taskId), + ); const startedAt = Date.now(); while (Date.now() - startedAt < 1_200_000) { await new Promise((resolvePromise) => setTimeout(resolvePromise, 5000)); @@ -368,7 +433,7 @@ async function runExternalTask(params: { }), ); const status = extractStatus(payload); - const outputs = Array.from(collectUrls(payload)); + const outputs = Array.from(collectUrls(payload)).filter((url) => !inputUrls.has(url)); if (["failed", "error", "timeout", "canceled", "cancelled"].includes(status)) { throw new Error(`External task failed with status ${status}`); } @@ -400,6 +465,13 @@ async function main() { if (!modelId || !availableModelIds.has(modelId)) { throw new Error(`No catalog model is available for ${capability}; pass --model=`); } + const selectedModel = catalog.find((model) => getModelId(model) === modelId); + if ( + !["chat", "multimodal", "image"].includes(capability) && + !selectedModel?.invocation?.path + ) { + throw new Error(`${modelId} is missing its official invocation contract`); + } const ledgerPath = resolve( getArgValue("--ledger") || @@ -445,7 +517,13 @@ async function main() { multiImage: false, }); } else { - outputs = await runExternalTask({ baseUrl, apiKey, modelId, capability }); + outputs = await runExternalTask({ + baseUrl, + apiKey, + modelId, + capability, + invocation: selectedModel!.invocation!, + }); } const finishedAt = new Date(); const entry: LedgerEntry = {