import { createHash } from "node:crypto"; import { appendFile, mkdir, readFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import dotenv from "dotenv"; import sharp from "sharp"; dotenv.config({ path: ".env.local" }); dotenv.config(); const VIP98K_BASE_URL = "https://api.vip98k.com"; type Vip98kCatalogModel = { id?: string; modelId?: string; model_id?: string; slug?: string; name?: string; displayName?: string; display_name?: string; invocation?: Vip98kInvocation; }; type Vip98kInvocation = { kind?: string; method?: string; path?: string; content_type?: string; request_example?: Record; result_method?: string; result_path?: string; }; const CAPABILITIES = [ "chat", "multimodal", "image", "multi-image-edit", "rmbg", "enhance", "extend", "video", "world", "text-to-3d", "image-to-3d", ] as const; type Capability = (typeof CAPABILITIES)[number]; type LedgerEntry = { runId: string; capability: Capability; modelId: string; status: "started" | "success" | "failure" | "skipped"; startedAt: string; finishedAt?: string; durationMs?: number; outputCount?: number; outputDigest?: string; error?: string; }; const DEFAULT_MODEL_IDS: Record = { chat: ["gpt-5.4-mini", "gpt-5.4", "claude-sonnet-4-6"], multimodal: ["gpt-5.4-mini", "gpt-5.4", "claude-sonnet-4-6"], image: ["nano-banana-2", "nano-banana-pro", "gpt-image-2-vip"], "multi-image-edit": [ "google/nano-banana-pro/edit", "google/nano-banana-2/edit", "openai/gpt-image-2/edit", ], rmbg: ["remove-background"], enhance: ["enhance", "image-upscaler"], extend: ["image-expansion"], video: [ "bytedance/seedance-v1-pro-fast/text-to-video", "veo-3.1-fast", "google/veo3.1-fast/text-to-video", ], world: ["world-generation"], "text-to-3d": [ "tencent/hunyuan3d-pro/text-to-3d", "tencent/hunyuan3d-rapid/text-to-3d", ], "image-to-3d": [ "tencent/hunyuan3d-pro/image-to-3d", "tencent/hunyuan3d-rapid/image-to-3d", ], }; function getArgValue(name: string) { const prefix = `${name}=`; return process.argv.find((item) => item.startsWith(prefix))?.slice(prefix.length) || ""; } function hasFlag(name: string) { return process.argv.includes(name); } function getModelId(model: Vip98kCatalogModel) { return String(model.id || model.modelId || model.model_id || model.slug || model.name || "").trim(); } function normalizeVip98kBaseUrl(baseUrl?: string | null) { const normalized = String(baseUrl || VIP98K_BASE_URL).trim().replace(/\/+$/, ""); if (!normalized) return VIP98K_BASE_URL; return normalized.replace(/\/v1$/i, "").replace(/\/api\/external\/task$/i, ""); } async function fetchVip98kCatalog(params: { baseURL: string; apiKey: string }) { const baseUrl = normalizeVip98kBaseUrl(params.baseURL); const [authorizedPayload, publicPayload] = await Promise.all([ parseResponse( await fetch(`${baseUrl}/api/external/task/models`, { headers: { Authorization: `Bearer ${params.apiKey}`, Accept: "application/json", }, signal: AbortSignal.timeout(60_000), }), ), parseResponse( await fetch(`${baseUrl}/api/v1/public/models`, { headers: { Accept: "application/json" }, signal: AbortSignal.timeout(60_000), }), ), ]); const record = authorizedPayload && typeof authorizedPayload === "object" ? (authorizedPayload as Record) : {}; const data = record.data; const authorized = (Array.isArray(data?.models) && data.models) || (Array.isArray(data?.items) && data.items) || (Array.isArray(data) && data) || (Array.isArray(record.models) && record.models) || []; const publicRecord = publicPayload && typeof publicPayload === "object" ? (publicPayload as Record) : {}; const publicModels = (Array.isArray(publicRecord.data?.sections) ? publicRecord.data.sections : [] ).flatMap((section: any) => (Array.isArray(section?.models) ? section.models : [])); if (authorized.length === 0) throw new Error("VIP98K authorized model catalog is empty"); if (publicModels.length === 0) throw new Error("VIP98K public invocation catalog is empty"); const publicById = new Map( publicModels.map((model: Vip98kCatalogModel) => [ getModelId(model).toLowerCase(), model, ]), ); return authorized.map((model: Vip98kCatalogModel) => ({ ...(publicById.get(getModelId(model).toLowerCase()) || {}), ...model, invocation: publicById.get(getModelId(model).toLowerCase())?.invocation, })) as Vip98kCatalogModel[]; } function sanitizeError(error: unknown) { const message = error instanceof Error ? error.message : String(error); return message.replace(/Bearer\s+\S+/gi, "Bearer [REDACTED]").slice(0, 2000); } function digestOutputs(outputs: string[]) { return createHash("sha256").update(outputs.join("\n")).digest("hex"); } async function readLedger(path: string): Promise { const content = await readFile(path, "utf8").catch(() => ""); return content .split(/\r?\n/) .filter(Boolean) .flatMap((line) => { try { return [JSON.parse(line) as LedgerEntry]; } catch { return []; } }); } async function writeLedger(path: string, entry: LedgerEntry) { await mkdir(dirname(path), { recursive: true }); await appendFile(path, `${JSON.stringify(entry)}\n`, "utf8"); } async function parseResponse(response: Response) { const text = await response.text(); let payload: unknown = text; try { payload = text ? JSON.parse(text) : null; } catch { // Keep the response body for a useful provider error. } if (!response.ok) { const record = payload && typeof payload === "object" ? (payload as Record) : {}; const message = record.error?.message || record.error || record.message || text || `HTTP ${response.status}`; throw new Error(`${String(message)} (HTTP ${response.status})`); } return payload; } function collectUrls(value: unknown, urls = new Set()) { if (!value) return urls; if (typeof value === "string") { for (const match of value.match(/https?:\/\/[^\s"'<>()[\]{}]+/g) || []) { urls.add(match.replace(/[),.;]+$/g, "")); } if (/^data:[^;]+;base64,/i.test(value)) urls.add(value); return urls; } if (Array.isArray(value)) { value.forEach((item) => collectUrls(item, urls)); return urls; } if (typeof value === "object") { Object.values(value as Record).forEach((item) => collectUrls(item, urls)); } return urls; } function extractTaskId(payload: unknown) { const root = payload && typeof payload === "object" ? (payload as Record) : {}; const data = root.data && typeof root.data === "object" ? root.data : root; return String(data.taskId || data.task_id || data.id || data.requestId || "").trim(); } function extractResultPath(payload: unknown) { const root = payload && typeof payload === "object" ? (payload as Record) : {}; const data = root.data && typeof root.data === "object" ? root.data : root; return String(data.resultPath || data.result_path || "").trim(); } function extractStatus(payload: unknown) { const root = payload && typeof payload === "object" ? (payload as Record) : {}; const data = root.data && typeof root.data === "object" ? root.data : root; return String(data.status || data.state || data.taskStatus || "").trim().toLowerCase(); } async function makeReferencePng(color: string, label: string) { return sharp({ create: { width: 96, height: 96, channels: 4, background: color }, }) .composite([ { input: Buffer.from( `${label}`, ), }, ]) .png() .toBuffer(); } async function runOpenAIImage(params: { baseUrl: string; apiKey: string; modelId: string; multiImage: boolean; }) { const endpoint = `${params.baseUrl}/v1/images/${params.multiImage ? "edits" : "generations"}`; if (!params.multiImage) { const payload = await parseResponse( await fetch(endpoint, { method: "POST", headers: { Authorization: `Bearer ${params.apiKey}`, "Content-Type": "application/json", }, body: JSON.stringify({ model: params.modelId, prompt: "A small blue ceramic cup on a white studio background", size: "1024x1024", response_format: "url", }), signal: AbortSignal.timeout(900_000), }), ); const outputs = Array.from(collectUrls(payload)); if (outputs.length === 0) throw new Error("Image generation returned no image output"); return outputs; } const [left, right] = await Promise.all([ makeReferencePng("#2563eb", "A"), makeReferencePng("#dc2626", "B"), ]); const form = new FormData(); form.append("model", params.modelId); form.append("prompt", "Combine both references into one clean side-by-side product comparison image"); form.append("size", "1024x1024"); form.append("response_format", "url"); form.append("image[]", new Blob([left], { type: "image/png" }), "reference-a.png"); form.append("image[]", new Blob([right], { type: "image/png" }), "reference-b.png"); const payload = await parseResponse( await fetch(endpoint, { method: "POST", headers: { Authorization: `Bearer ${params.apiKey}` }, body: form, signal: AbortSignal.timeout(900_000), }), ); const outputs = Array.from(collectUrls(payload)); if (outputs.length === 0) throw new Error("Multi-image edit returned no image output"); return outputs; } async function runChat(params: { baseUrl: string; apiKey: string; modelId: string; multimodal: boolean; }) { const content = params.multimodal ? [ { type: "text", text: "Reply with the dominant color in one English word." }, { type: "image_url", image_url: { url: "https://dummyimage.com/96x96/2563eb/ffffff.png&text=A" }, }, ] : "Reply with exactly: vip98k-ok"; const payload = await parseResponse( await fetch(`${params.baseUrl}/v1/chat/completions`, { method: "POST", headers: { Authorization: `Bearer ${params.apiKey}`, "Content-Type": "application/json", }, body: JSON.stringify({ model: params.modelId, messages: [{ role: "user", content }], max_tokens: 64, }), signal: AbortSignal.timeout(180_000), }), ); const text = JSON.stringify(payload); if (!text || text === "{}") throw new Error("Chat request returned no content"); return [text.slice(0, 500)]; } async function runExternalTask(params: { baseUrl: string; apiKey: string; modelId: string; capability: Capability; invocation: Vip98kInvocation; }) { const imageUrl = "https://dummyimage.com/512x512/2563eb/ffffff.png&text=VIP98K"; const secondImageUrl = "https://dummyimage.com/512x512/dc2626/ffffff.png&text=REFERENCE"; const body: Record = JSON.parse( JSON.stringify(params.invocation.request_example || {}), ); if ("model" in body) body.model = params.modelId; if ("displayName" in body) body.displayName = `provider-smoke-${params.capability}`; if ("prompt" in body) body.prompt = "Create a minimal test result with one simple object"; if ( ["multi-image-edit", "rmbg", "enhance", "extend", "image-to-3d"].includes( params.capability, ) ) { const images = params.capability === "multi-image-edit" ? [imageUrl, secondImageUrl] : [imageUrl]; if ("image" in body) body.image = imageUrl; if ("imageUrl" in body) body.imageUrl = imageUrl; if ("image_url" in body) body.image_url = imageUrl; if ("images" in body) body.images = images; if ("imageUrls" in body) body.imageUrls = images; if ("referenceImages" in body) body.referenceImages = images; if ("reference_images" in body) body.reference_images = images; if (Array.isArray(body.video_clips)) { const template = body.video_clips[0] && typeof body.video_clips[0] === "object" ? (body.video_clips[0] as Record) : {}; body.video_clips = images.map((url) => ({ ...template, url })); } if ( params.capability === "multi-image-edit" && !Array.isArray(body.images) && !Array.isArray(body.referenceImages) && !Array.isArray(body.reference_images) && !Array.isArray(body.video_clips) ) { throw new Error(`${params.modelId} official invocation does not expose multi-image input`); } } if (params.capability === "extend") { if ("aspectRatio" in body) body.aspectRatio = "16:9"; if ("aspect_ratio" in body) body.aspect_ratio = "16:9"; if ("ratio" in body) body.ratio = "16:9"; } if (params.capability === "world" && "mode" in body) body.mode = "world-generation"; const path = String(params.invocation.path || "").trim(); if (!path.startsWith("/")) throw new Error(`${params.modelId} has no valid invocation path`); const submitted = await parseResponse( await fetch(`${params.baseUrl}${path}`, { method: String(params.invocation.method || "POST").toUpperCase(), headers: { Authorization: `Bearer ${params.apiKey}`, "Content-Type": params.invocation.content_type || "application/json", }, body: JSON.stringify(body), signal: AbortSignal.timeout(180_000), }), ); const inputUrls = new Set([imageUrl, secondImageUrl]); const immediate = Array.from(collectUrls(submitted)).filter((url) => !inputUrls.has(url)); if (immediate.length > 0) return immediate; const taskId = extractTaskId(submitted); if (!taskId) throw new Error("External task returned neither output nor task ID"); const resultPath = extractResultPath(submitted) || String(params.invocation.result_path || "").replaceAll( "{task_id}", encodeURIComponent(taskId), ); const startedAt = Date.now(); while (Date.now() - startedAt < 1_200_000) { await new Promise((resolvePromise) => setTimeout(resolvePromise, 5000)); const resultUrl = resultPath ? resultPath.startsWith("http") ? resultPath : `${params.baseUrl}${resultPath.startsWith("/") ? "" : "/"}${resultPath}` : `${params.baseUrl}/api/external/task/result?taskId=${encodeURIComponent(taskId)}`; const payload = await parseResponse( await fetch(resultUrl, { headers: { Authorization: `Bearer ${params.apiKey}` }, signal: AbortSignal.timeout(60_000), }), ); const status = extractStatus(payload); const outputs = Array.from(collectUrls(payload)).filter((url) => !inputUrls.has(url)); if (["failed", "error", "timeout", "canceled", "cancelled"].includes(status)) { throw new Error(`External task failed with status ${status}`); } if ( ["succeeded", "success", "completed", "done", "finished"].includes(status) || outputs.length > 0 ) { if (outputs.length === 0) throw new Error("External task completed without output URL"); return outputs; } } throw new Error("External task did not complete within 20 minutes"); } async function main() { const capabilityArg = getArgValue("--capability"); if (!CAPABILITIES.includes(capabilityArg as Capability)) { throw new Error(`Pass exactly one --capability=${CAPABILITIES.join("|")}`); } const capability = capabilityArg as Capability; const apiKey = String(process.env.VIP98K_API_KEY || "").trim(); if (!apiKey) throw new Error("VIP98K_API_KEY is required and must remain server-side"); const baseUrl = normalizeVip98kBaseUrl(process.env.VIP98K_BASE_URL || VIP98K_BASE_URL); const catalog = await fetchVip98kCatalog({ baseURL: baseUrl, apiKey }); const availableModelIds = new Set(catalog.map(getModelId)); const explicitModelId = getArgValue("--model"); const modelId = explicitModelId || DEFAULT_MODEL_IDS[capability].find((candidate) => availableModelIds.has(candidate)); if (!modelId || !availableModelIds.has(modelId)) { throw new Error(`No catalog model is available for ${capability}; pass --model=`); } const selectedModel = catalog.find((model) => getModelId(model) === modelId); if ( !["chat", "multimodal", "image"].includes(capability) && !selectedModel?.invocation?.path ) { throw new Error(`${modelId} is missing its official invocation contract`); } const ledgerPath = resolve( getArgValue("--ledger") || process.env.VIP98K_TEST_LEDGER || ".local/vip98k-capability-tests.jsonl", ); const ledger = await readLedger(ledgerPath); if ( !hasFlag("--force") && ledger.some( (entry) => entry.capability === capability && entry.modelId === modelId && entry.status === "success", ) ) { console.log(JSON.stringify({ skipped: true, capability, modelId, ledgerPath }, null, 2)); return; } const runId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; const startedAt = new Date(); await writeLedger(ledgerPath, { runId, capability, modelId, status: "started", startedAt: startedAt.toISOString(), }); try { let outputs: string[]; if (capability === "chat" || capability === "multimodal") { outputs = await runChat({ baseUrl, apiKey, modelId, multimodal: capability === "multimodal", }); } else if (capability === "image") { outputs = await runOpenAIImage({ baseUrl, apiKey, modelId, multiImage: false, }); } else { outputs = await runExternalTask({ baseUrl, apiKey, modelId, capability, invocation: selectedModel!.invocation!, }); } const finishedAt = new Date(); const entry: LedgerEntry = { runId, capability, modelId, status: "success", startedAt: startedAt.toISOString(), finishedAt: finishedAt.toISOString(), durationMs: finishedAt.getTime() - startedAt.getTime(), outputCount: outputs.length, outputDigest: digestOutputs(outputs), }; await writeLedger(ledgerPath, entry); console.log(JSON.stringify({ ...entry, ledgerPath }, null, 2)); } catch (error) { const finishedAt = new Date(); const entry: LedgerEntry = { runId, capability, modelId, status: "failure", startedAt: startedAt.toISOString(), finishedAt: finishedAt.toISOString(), durationMs: finishedAt.getTime() - startedAt.getTime(), error: sanitizeError(error), }; await writeLedger(ledgerPath, entry); console.error(JSON.stringify({ ...entry, ledgerPath }, null, 2)); process.exitCode = 1; } } main().catch((error) => { console.error(sanitizeError(error)); process.exit(1); });