import { createHash } from "node:crypto"; import { appendFile, mkdir, readFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; import dotenv from "dotenv"; import sharp from "sharp"; dotenv.config({ path: ".env.local" }); dotenv.config(); const VIP98K_BASE_URL = "https://api.vip98k.com"; type Vip98kCatalogModel = { id?: string; modelId?: string; model_id?: string; slug?: string; name?: string; }; const CAPABILITIES = [ "chat", "multimodal", "image", "multi-image-edit", "rmbg", "enhance", "extend", "video", "world", "text-to-3d", "image-to-3d", ] as const; type Capability = (typeof CAPABILITIES)[number]; type LedgerEntry = { runId: string; capability: Capability; modelId: string; status: "started" | "success" | "failure" | "skipped"; startedAt: string; finishedAt?: string; durationMs?: number; outputCount?: number; outputDigest?: string; error?: string; }; const DEFAULT_MODEL_IDS: Record = { chat: ["gpt-5.4-mini", "gpt-5.4", "claude-sonnet-4-6"], multimodal: ["gpt-5.4-mini", "gpt-5.4", "claude-sonnet-4-6"], image: ["nano-banana-2", "nano-banana-pro", "gpt-image-2-vip"], "multi-image-edit": ["nano-banana-2", "nano-banana-pro", "gpt-image-2-vip"], rmbg: ["remove-background"], enhance: ["enhance", "image-upscaler"], extend: ["image-expansion"], video: [ "bytedance/seedance-v1-pro-fast/text-to-video", "veo-3.1-fast", "google/veo3.1-fast/text-to-video", ], world: ["world-generation"], "text-to-3d": [ "tencent/hunyuan3d-pro/text-to-3d", "tencent/hunyuan3d-rapid/text-to-3d", ], "image-to-3d": [ "tencent/hunyuan3d-pro/image-to-3d", "tencent/hunyuan3d-rapid/image-to-3d", ], }; function getArgValue(name: string) { const prefix = `${name}=`; return process.argv.find((item) => item.startsWith(prefix))?.slice(prefix.length) || ""; } function hasFlag(name: string) { return process.argv.includes(name); } function getModelId(model: Vip98kCatalogModel) { return String(model.id || model.modelId || model.model_id || model.slug || model.name || "").trim(); } function normalizeVip98kBaseUrl(baseUrl?: string | null) { const normalized = String(baseUrl || VIP98K_BASE_URL).trim().replace(/\/+$/, ""); if (!normalized) return VIP98K_BASE_URL; return normalized.replace(/\/v1$/i, "").replace(/\/api\/external\/task$/i, ""); } async function fetchVip98kCatalog(params: { baseURL: string; apiKey: string }) { const payload = await parseResponse( await fetch(`${normalizeVip98kBaseUrl(params.baseURL)}/api/external/task/models`, { headers: { Authorization: `Bearer ${params.apiKey}`, Accept: "application/json", }, signal: AbortSignal.timeout(60_000), }), ); const record = payload && typeof payload === "object" ? (payload as Record) : {}; const data = record.data; const models = (Array.isArray(data?.models) && data.models) || (Array.isArray(data?.items) && data.items) || (Array.isArray(data) && data) || (Array.isArray(record.models) && record.models) || []; if (models.length === 0) throw new Error("VIP98K model catalog is empty"); return models as Vip98kCatalogModel[]; } function sanitizeError(error: unknown) { const message = error instanceof Error ? error.message : String(error); return message.replace(/Bearer\s+\S+/gi, "Bearer [REDACTED]").slice(0, 2000); } function digestOutputs(outputs: string[]) { return createHash("sha256").update(outputs.join("\n")).digest("hex"); } async function readLedger(path: string): Promise { const content = await readFile(path, "utf8").catch(() => ""); return content .split(/\r?\n/) .filter(Boolean) .flatMap((line) => { try { return [JSON.parse(line) as LedgerEntry]; } catch { return []; } }); } async function writeLedger(path: string, entry: LedgerEntry) { await mkdir(dirname(path), { recursive: true }); await appendFile(path, `${JSON.stringify(entry)}\n`, "utf8"); } async function parseResponse(response: Response) { const text = await response.text(); let payload: unknown = text; try { payload = text ? JSON.parse(text) : null; } catch { // Keep the response body for a useful provider error. } if (!response.ok) { const record = payload && typeof payload === "object" ? (payload as Record) : {}; const message = record.error?.message || record.error || record.message || text || `HTTP ${response.status}`; throw new Error(`${String(message)} (HTTP ${response.status})`); } return payload; } function collectUrls(value: unknown, urls = new Set()) { if (!value) return urls; if (typeof value === "string") { for (const match of value.match(/https?:\/\/[^\s"'<>()[\]{}]+/g) || []) { urls.add(match.replace(/[),.;]+$/g, "")); } if (/^data:[^;]+;base64,/i.test(value)) urls.add(value); return urls; } if (Array.isArray(value)) { value.forEach((item) => collectUrls(item, urls)); return urls; } if (typeof value === "object") { Object.values(value as Record).forEach((item) => collectUrls(item, urls)); } return urls; } function extractTaskId(payload: unknown) { const root = payload && typeof payload === "object" ? (payload as Record) : {}; const data = root.data && typeof root.data === "object" ? root.data : root; return String(data.taskId || data.task_id || data.id || data.requestId || "").trim(); } function extractResultPath(payload: unknown) { const root = payload && typeof payload === "object" ? (payload as Record) : {}; const data = root.data && typeof root.data === "object" ? root.data : root; return String(data.resultPath || data.result_path || "").trim(); } function extractStatus(payload: unknown) { const root = payload && typeof payload === "object" ? (payload as Record) : {}; const data = root.data && typeof root.data === "object" ? root.data : root; return String(data.status || data.state || data.taskStatus || "").trim().toLowerCase(); } async function makeReferencePng(color: string, label: string) { return sharp({ create: { width: 96, height: 96, channels: 4, background: color }, }) .composite([ { input: Buffer.from( `${label}`, ), }, ]) .png() .toBuffer(); } async function runOpenAIImage(params: { baseUrl: string; apiKey: string; modelId: string; multiImage: boolean; }) { const endpoint = `${params.baseUrl}/v1/images/${params.multiImage ? "edits" : "generations"}`; if (!params.multiImage) { const payload = await parseResponse( await fetch(endpoint, { method: "POST", headers: { Authorization: `Bearer ${params.apiKey}`, "Content-Type": "application/json", }, body: JSON.stringify({ model: params.modelId, prompt: "A small blue ceramic cup on a white studio background", size: "1024x1024", response_format: "url", }), signal: AbortSignal.timeout(900_000), }), ); const outputs = Array.from(collectUrls(payload)); if (outputs.length === 0) throw new Error("Image generation returned no image output"); return outputs; } const [left, right] = await Promise.all([ makeReferencePng("#2563eb", "A"), makeReferencePng("#dc2626", "B"), ]); const form = new FormData(); form.append("model", params.modelId); form.append("prompt", "Combine both references into one clean side-by-side product comparison image"); form.append("size", "1024x1024"); form.append("response_format", "url"); form.append("image[]", new Blob([left], { type: "image/png" }), "reference-a.png"); form.append("image[]", new Blob([right], { type: "image/png" }), "reference-b.png"); const payload = await parseResponse( await fetch(endpoint, { method: "POST", headers: { Authorization: `Bearer ${params.apiKey}` }, body: form, signal: AbortSignal.timeout(900_000), }), ); const outputs = Array.from(collectUrls(payload)); if (outputs.length === 0) throw new Error("Multi-image edit returned no image output"); return outputs; } async function runChat(params: { baseUrl: string; apiKey: string; modelId: string; multimodal: boolean; }) { const content = params.multimodal ? [ { type: "text", text: "Reply with the dominant color in one English word." }, { type: "image_url", image_url: { url: "https://dummyimage.com/96x96/2563eb/ffffff.png&text=A" }, }, ] : "Reply with exactly: vip98k-ok"; const payload = await parseResponse( await fetch(`${params.baseUrl}/v1/chat/completions`, { method: "POST", headers: { Authorization: `Bearer ${params.apiKey}`, "Content-Type": "application/json", }, body: JSON.stringify({ model: params.modelId, messages: [{ role: "user", content }], max_tokens: 64, }), signal: AbortSignal.timeout(180_000), }), ); const text = JSON.stringify(payload); if (!text || text === "{}") throw new Error("Chat request returned no content"); return [text.slice(0, 500)]; } async function runExternalTask(params: { baseUrl: string; apiKey: string; modelId: string; capability: Capability; }) { const imageUrl = "https://dummyimage.com/512x512/2563eb/ffffff.png&text=VIP98K"; const body: Record = { model: params.modelId, displayName: `provider-smoke-${params.capability}`, prompt: "Create a minimal test result with one simple object", }; if (["rmbg", "enhance", "extend", "image-to-3d"].includes(params.capability)) { Object.assign(body, { image: imageUrl, imageUrl, image_url: imageUrl, images: [imageUrl], imageUrls: [imageUrl], }); } if (params.capability === "extend") { Object.assign(body, { aspectRatio: "16:9", aspect_ratio: "16:9", ratio: "16:9" }); } if (params.capability === "video") { Object.assign(body, { mode: "text-to-video", duration: 4, resolution: "480p", ratio: "16:9" }); } if (params.capability === "world") body.mode = "world-generation"; const submitted = await parseResponse( await fetch(`${params.baseUrl}/api/external/task/generate`, { method: "POST", headers: { Authorization: `Bearer ${params.apiKey}`, "Content-Type": "application/json", }, body: JSON.stringify(body), signal: AbortSignal.timeout(180_000), }), ); const immediate = Array.from(collectUrls(submitted)); if (immediate.length > 0) return immediate; const taskId = extractTaskId(submitted); if (!taskId) throw new Error("External task returned neither output nor task ID"); const resultPath = extractResultPath(submitted); const startedAt = Date.now(); while (Date.now() - startedAt < 1_200_000) { await new Promise((resolvePromise) => setTimeout(resolvePromise, 5000)); const resultUrl = resultPath ? resultPath.startsWith("http") ? resultPath : `${params.baseUrl}${resultPath.startsWith("/") ? "" : "/"}${resultPath}` : `${params.baseUrl}/api/external/task/result?taskId=${encodeURIComponent(taskId)}`; const payload = await parseResponse( await fetch(resultUrl, { headers: { Authorization: `Bearer ${params.apiKey}` }, signal: AbortSignal.timeout(60_000), }), ); const status = extractStatus(payload); const outputs = Array.from(collectUrls(payload)); if (["failed", "error", "timeout", "canceled", "cancelled"].includes(status)) { throw new Error(`External task failed with status ${status}`); } if ( ["succeeded", "success", "completed", "done", "finished"].includes(status) || outputs.length > 0 ) { if (outputs.length === 0) throw new Error("External task completed without output URL"); return outputs; } } throw new Error("External task did not complete within 20 minutes"); } async function main() { const capabilityArg = getArgValue("--capability"); if (!CAPABILITIES.includes(capabilityArg as Capability)) { throw new Error(`Pass exactly one --capability=${CAPABILITIES.join("|")}`); } const capability = capabilityArg as Capability; const apiKey = String(process.env.VIP98K_API_KEY || "").trim(); if (!apiKey) throw new Error("VIP98K_API_KEY is required and must remain server-side"); const baseUrl = normalizeVip98kBaseUrl(process.env.VIP98K_BASE_URL || VIP98K_BASE_URL); const catalog = await fetchVip98kCatalog({ baseURL: baseUrl, apiKey }); const availableModelIds = new Set(catalog.map(getModelId)); const explicitModelId = getArgValue("--model"); const modelId = explicitModelId || DEFAULT_MODEL_IDS[capability].find((candidate) => availableModelIds.has(candidate)); if (!modelId || !availableModelIds.has(modelId)) { throw new Error(`No catalog model is available for ${capability}; pass --model=`); } const ledgerPath = resolve( getArgValue("--ledger") || process.env.VIP98K_TEST_LEDGER || ".local/vip98k-capability-tests.jsonl", ); const ledger = await readLedger(ledgerPath); if ( !hasFlag("--force") && ledger.some( (entry) => entry.capability === capability && entry.modelId === modelId && entry.status === "success", ) ) { console.log(JSON.stringify({ skipped: true, capability, modelId, ledgerPath }, null, 2)); return; } const runId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; const startedAt = new Date(); await writeLedger(ledgerPath, { runId, capability, modelId, status: "started", startedAt: startedAt.toISOString(), }); try { let outputs: string[]; if (capability === "chat" || capability === "multimodal") { outputs = await runChat({ baseUrl, apiKey, modelId, multimodal: capability === "multimodal", }); } else if (capability === "image" || capability === "multi-image-edit") { outputs = await runOpenAIImage({ baseUrl, apiKey, modelId, multiImage: capability === "multi-image-edit", }); } else { outputs = await runExternalTask({ baseUrl, apiKey, modelId, capability }); } const finishedAt = new Date(); const entry: LedgerEntry = { runId, capability, modelId, status: "success", startedAt: startedAt.toISOString(), finishedAt: finishedAt.toISOString(), durationMs: finishedAt.getTime() - startedAt.getTime(), outputCount: outputs.length, outputDigest: digestOutputs(outputs), }; await writeLedger(ledgerPath, entry); console.log(JSON.stringify({ ...entry, ledgerPath }, null, 2)); } catch (error) { const finishedAt = new Date(); const entry: LedgerEntry = { runId, capability, modelId, status: "failure", startedAt: startedAt.toISOString(), finishedAt: finishedAt.toISOString(), durationMs: finishedAt.getTime() - startedAt.getTime(), error: sanitizeError(error), }; await writeLedger(ledgerPath, entry); console.error(JSON.stringify({ ...entry, ledgerPath }, null, 2)); process.exitCode = 1; } } main().catch((error) => { console.error(sanitizeError(error)); process.exit(1); });