568 lines
19 KiB
TypeScript
568 lines
19 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { appendFile, mkdir, readFile } from "node:fs/promises";
|
|
import { dirname, resolve } from "node:path";
|
|
import dotenv from "dotenv";
|
|
import sharp from "sharp";
|
|
|
|
dotenv.config({ path: ".env.local" });
|
|
dotenv.config();
|
|
|
|
const VIP98K_BASE_URL = "https://api.vip98k.com";
|
|
|
|
type Vip98kCatalogModel = {
|
|
id?: string;
|
|
modelId?: string;
|
|
model_id?: string;
|
|
slug?: string;
|
|
name?: string;
|
|
displayName?: string;
|
|
display_name?: string;
|
|
invocation?: Vip98kInvocation;
|
|
};
|
|
|
|
type Vip98kInvocation = {
|
|
kind?: string;
|
|
method?: string;
|
|
path?: string;
|
|
content_type?: string;
|
|
request_example?: Record<string, unknown>;
|
|
result_method?: string;
|
|
result_path?: string;
|
|
};
|
|
|
|
const CAPABILITIES = [
|
|
"chat",
|
|
"multimodal",
|
|
"image",
|
|
"multi-image-edit",
|
|
"rmbg",
|
|
"enhance",
|
|
"extend",
|
|
"video",
|
|
"world",
|
|
"text-to-3d",
|
|
"image-to-3d",
|
|
] as const;
|
|
|
|
type Capability = (typeof CAPABILITIES)[number];
|
|
type LedgerEntry = {
|
|
runId: string;
|
|
capability: Capability;
|
|
modelId: string;
|
|
status: "started" | "success" | "failure" | "skipped";
|
|
startedAt: string;
|
|
finishedAt?: string;
|
|
durationMs?: number;
|
|
outputCount?: number;
|
|
outputDigest?: string;
|
|
error?: string;
|
|
};
|
|
|
|
const DEFAULT_MODEL_IDS: Record<Capability, string[]> = {
|
|
chat: ["gpt-5.4-mini", "gpt-5.4", "claude-sonnet-4-6"],
|
|
multimodal: ["gpt-5.4-mini", "gpt-5.4", "claude-sonnet-4-6"],
|
|
image: ["nano-banana-2", "nano-banana-pro", "gpt-image-2-vip"],
|
|
"multi-image-edit": [
|
|
"google/nano-banana-pro/edit",
|
|
"google/nano-banana-2/edit",
|
|
"openai/gpt-image-2/edit",
|
|
],
|
|
rmbg: ["remove-background"],
|
|
enhance: ["enhance", "image-upscaler"],
|
|
extend: ["image-expansion"],
|
|
video: [
|
|
"bytedance/seedance-v1-pro-fast/text-to-video",
|
|
"veo-3.1-fast",
|
|
"google/veo3.1-fast/text-to-video",
|
|
],
|
|
world: ["world-generation"],
|
|
"text-to-3d": [
|
|
"tencent/hunyuan3d-pro/text-to-3d",
|
|
"tencent/hunyuan3d-rapid/text-to-3d",
|
|
],
|
|
"image-to-3d": [
|
|
"tencent/hunyuan3d-pro/image-to-3d",
|
|
"tencent/hunyuan3d-rapid/image-to-3d",
|
|
],
|
|
};
|
|
|
|
function getArgValue(name: string) {
|
|
const prefix = `${name}=`;
|
|
return process.argv.find((item) => item.startsWith(prefix))?.slice(prefix.length) || "";
|
|
}
|
|
|
|
function hasFlag(name: string) {
|
|
return process.argv.includes(name);
|
|
}
|
|
|
|
function getModelId(model: Vip98kCatalogModel) {
|
|
return String(model.id || model.modelId || model.model_id || model.slug || model.name || "").trim();
|
|
}
|
|
|
|
function normalizeVip98kBaseUrl(baseUrl?: string | null) {
|
|
const normalized = String(baseUrl || VIP98K_BASE_URL).trim().replace(/\/+$/, "");
|
|
if (!normalized) return VIP98K_BASE_URL;
|
|
return normalized.replace(/\/v1$/i, "").replace(/\/api\/external\/task$/i, "");
|
|
}
|
|
|
|
async function fetchVip98kCatalog(params: { baseURL: string; apiKey: string }) {
|
|
const baseUrl = normalizeVip98kBaseUrl(params.baseURL);
|
|
const [authorizedPayload, publicPayload] = await Promise.all([
|
|
parseResponse(
|
|
await fetch(`${baseUrl}/v1/models`, {
|
|
headers: {
|
|
Authorization: `Bearer ${params.apiKey}`,
|
|
Accept: "application/json",
|
|
},
|
|
signal: AbortSignal.timeout(60_000),
|
|
}),
|
|
),
|
|
parseResponse(
|
|
await fetch(`${baseUrl}/api/v2/catalog/models`, {
|
|
headers: { Accept: "application/json" },
|
|
signal: AbortSignal.timeout(60_000),
|
|
}),
|
|
),
|
|
]);
|
|
const record =
|
|
authorizedPayload && typeof authorizedPayload === "object"
|
|
? (authorizedPayload as Record<string, any>)
|
|
: {};
|
|
const data = record.data;
|
|
const authorized =
|
|
(Array.isArray(record.data) && record.data) ||
|
|
(Array.isArray(data?.models) && data.models) ||
|
|
(Array.isArray(data?.items) && data.items) ||
|
|
(Array.isArray(data) && data) ||
|
|
(Array.isArray(record.models) && record.models) ||
|
|
[];
|
|
const publicRecord =
|
|
publicPayload && typeof publicPayload === "object"
|
|
? (publicPayload as Record<string, any>)
|
|
: {};
|
|
const publicModels = (Array.isArray(publicRecord.data?.sections)
|
|
? publicRecord.data.sections
|
|
: []
|
|
).flatMap((section: any) => (Array.isArray(section?.models) ? section.models : []));
|
|
if (authorized.length === 0) throw new Error("VIP98K authorized model catalog is empty");
|
|
if (publicModels.length === 0) throw new Error("VIP98K public invocation catalog is empty");
|
|
const publicById = new Map<string, Vip98kCatalogModel>(
|
|
publicModels.map((model: Vip98kCatalogModel) => [
|
|
getModelId(model).toLowerCase(),
|
|
model,
|
|
]),
|
|
);
|
|
return authorized.map((model: Vip98kCatalogModel) => ({
|
|
...(publicById.get(getModelId(model).toLowerCase()) || {}),
|
|
...model,
|
|
invocation: publicById.get(getModelId(model).toLowerCase())?.invocation,
|
|
})) as Vip98kCatalogModel[];
|
|
}
|
|
|
|
function sanitizeError(error: unknown) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
return message.replace(/Bearer\s+\S+/gi, "Bearer [REDACTED]").slice(0, 2000);
|
|
}
|
|
|
|
function digestOutputs(outputs: string[]) {
|
|
return createHash("sha256").update(outputs.join("\n")).digest("hex");
|
|
}
|
|
|
|
async function readLedger(path: string): Promise<LedgerEntry[]> {
|
|
const content = await readFile(path, "utf8").catch(() => "");
|
|
return content
|
|
.split(/\r?\n/)
|
|
.filter(Boolean)
|
|
.flatMap((line) => {
|
|
try {
|
|
return [JSON.parse(line) as LedgerEntry];
|
|
} catch {
|
|
return [];
|
|
}
|
|
});
|
|
}
|
|
|
|
async function writeLedger(path: string, entry: LedgerEntry) {
|
|
await mkdir(dirname(path), { recursive: true });
|
|
await appendFile(path, `${JSON.stringify(entry)}\n`, "utf8");
|
|
}
|
|
|
|
async function parseResponse(response: Response) {
|
|
const text = await response.text();
|
|
let payload: unknown = text;
|
|
try {
|
|
payload = text ? JSON.parse(text) : null;
|
|
} catch {
|
|
// Keep the response body for a useful provider error.
|
|
}
|
|
if (!response.ok) {
|
|
const record = payload && typeof payload === "object" ? (payload as Record<string, any>) : {};
|
|
const message =
|
|
record.error?.message || record.error || record.message || text || `HTTP ${response.status}`;
|
|
throw new Error(`${String(message)} (HTTP ${response.status})`);
|
|
}
|
|
return payload;
|
|
}
|
|
|
|
function collectUrls(value: unknown, urls = new Set<string>()) {
|
|
if (!value) return urls;
|
|
if (typeof value === "string") {
|
|
for (const match of value.match(/https?:\/\/[^\s"'<>()[\]{}]+/g) || []) {
|
|
urls.add(match.replace(/[),.;]+$/g, ""));
|
|
}
|
|
if (/^data:[^;]+;base64,/i.test(value)) urls.add(value);
|
|
return urls;
|
|
}
|
|
if (Array.isArray(value)) {
|
|
value.forEach((item) => collectUrls(item, urls));
|
|
return urls;
|
|
}
|
|
if (typeof value === "object") {
|
|
Object.values(value as Record<string, unknown>).forEach((item) => collectUrls(item, urls));
|
|
}
|
|
return urls;
|
|
}
|
|
|
|
function extractTaskId(payload: unknown) {
|
|
const root = payload && typeof payload === "object" ? (payload as Record<string, any>) : {};
|
|
const data = root.data && typeof root.data === "object" ? root.data : root;
|
|
return String(data.taskId || data.task_id || data.id || data.requestId || "").trim();
|
|
}
|
|
|
|
function extractResultPath(payload: unknown) {
|
|
const root = payload && typeof payload === "object" ? (payload as Record<string, any>) : {};
|
|
const data = root.data && typeof root.data === "object" ? root.data : root;
|
|
return String(data.resultPath || data.result_path || "").trim();
|
|
}
|
|
|
|
function extractStatus(payload: unknown) {
|
|
const root = payload && typeof payload === "object" ? (payload as Record<string, any>) : {};
|
|
const data = root.data && typeof root.data === "object" ? root.data : root;
|
|
return String(data.status || data.state || data.taskStatus || "").trim().toLowerCase();
|
|
}
|
|
|
|
async function makeReferencePng(color: string, label: string) {
|
|
return sharp({
|
|
create: { width: 96, height: 96, channels: 4, background: color },
|
|
})
|
|
.composite([
|
|
{
|
|
input: Buffer.from(
|
|
`<svg width="96" height="96"><text x="48" y="58" text-anchor="middle" font-size="30" fill="white">${label}</text></svg>`,
|
|
),
|
|
},
|
|
])
|
|
.png()
|
|
.toBuffer();
|
|
}
|
|
|
|
async function runOpenAIImage(params: {
|
|
baseUrl: string;
|
|
apiKey: string;
|
|
modelId: string;
|
|
multiImage: boolean;
|
|
}) {
|
|
const endpoint = `${params.baseUrl}/v1/images/${params.multiImage ? "edits" : "generations"}`;
|
|
if (!params.multiImage) {
|
|
const payload = await parseResponse(
|
|
await fetch(endpoint, {
|
|
method: "POST",
|
|
headers: {
|
|
Authorization: `Bearer ${params.apiKey}`,
|
|
"Content-Type": "application/json",
|
|
},
|
|
body: JSON.stringify({
|
|
model: params.modelId,
|
|
prompt: "A small blue ceramic cup on a white studio background",
|
|
size: "1024x1024",
|
|
response_format: "url",
|
|
}),
|
|
signal: AbortSignal.timeout(900_000),
|
|
}),
|
|
);
|
|
const outputs = Array.from(collectUrls(payload));
|
|
if (outputs.length === 0) throw new Error("Image generation returned no image output");
|
|
return outputs;
|
|
}
|
|
|
|
const [left, right] = await Promise.all([
|
|
makeReferencePng("#2563eb", "A"),
|
|
makeReferencePng("#dc2626", "B"),
|
|
]);
|
|
const form = new FormData();
|
|
form.append("model", params.modelId);
|
|
form.append("prompt", "Combine both references into one clean side-by-side product comparison image");
|
|
form.append("size", "1024x1024");
|
|
form.append("response_format", "url");
|
|
form.append("image[]", new Blob([left], { type: "image/png" }), "reference-a.png");
|
|
form.append("image[]", new Blob([right], { type: "image/png" }), "reference-b.png");
|
|
const payload = await parseResponse(
|
|
await fetch(endpoint, {
|
|
method: "POST",
|
|
headers: { Authorization: `Bearer ${params.apiKey}` },
|
|
body: form,
|
|
signal: AbortSignal.timeout(900_000),
|
|
}),
|
|
);
|
|
const outputs = Array.from(collectUrls(payload));
|
|
if (outputs.length === 0) throw new Error("Multi-image edit returned no image output");
|
|
return outputs;
|
|
}
|
|
|
|
async function runChat(params: {
|
|
baseUrl: string;
|
|
apiKey: string;
|
|
modelId: string;
|
|
multimodal: boolean;
|
|
}) {
|
|
const content = params.multimodal
|
|
? [
|
|
{ type: "text", text: "Reply with the dominant color in one English word." },
|
|
{
|
|
type: "image_url",
|
|
image_url: { url: "https://dummyimage.com/96x96/2563eb/ffffff.png&text=A" },
|
|
},
|
|
]
|
|
: "Reply with exactly: vip98k-ok";
|
|
const payload = await parseResponse(
|
|
await fetch(`${params.baseUrl}/v1/chat/completions`, {
|
|
method: "POST",
|
|
headers: {
|
|
Authorization: `Bearer ${params.apiKey}`,
|
|
"Content-Type": "application/json",
|
|
},
|
|
body: JSON.stringify({
|
|
model: params.modelId,
|
|
messages: [{ role: "user", content }],
|
|
max_tokens: 64,
|
|
}),
|
|
signal: AbortSignal.timeout(180_000),
|
|
}),
|
|
);
|
|
const text = JSON.stringify(payload);
|
|
if (!text || text === "{}") throw new Error("Chat request returned no content");
|
|
return [text.slice(0, 500)];
|
|
}
|
|
|
|
async function runExternalTask(params: {
|
|
baseUrl: string;
|
|
apiKey: string;
|
|
modelId: string;
|
|
capability: Capability;
|
|
invocation: Vip98kInvocation;
|
|
}) {
|
|
const imageUrl = "https://dummyimage.com/512x512/2563eb/ffffff.png&text=VIP98K";
|
|
const secondImageUrl = "https://dummyimage.com/512x512/dc2626/ffffff.png&text=REFERENCE";
|
|
const body: Record<string, unknown> = JSON.parse(
|
|
JSON.stringify(params.invocation.request_example || {}),
|
|
);
|
|
if ("model" in body) body.model = params.modelId;
|
|
if ("displayName" in body) body.displayName = `provider-smoke-${params.capability}`;
|
|
if ("prompt" in body) body.prompt = "Create a minimal test result with one simple object";
|
|
if (
|
|
["multi-image-edit", "rmbg", "enhance", "extend", "image-to-3d"].includes(
|
|
params.capability,
|
|
)
|
|
) {
|
|
const images =
|
|
params.capability === "multi-image-edit" ? [imageUrl, secondImageUrl] : [imageUrl];
|
|
if ("image" in body) body.image = imageUrl;
|
|
if ("imageUrl" in body) body.imageUrl = imageUrl;
|
|
if ("image_url" in body) body.image_url = imageUrl;
|
|
if ("images" in body) body.images = images;
|
|
if ("imageUrls" in body) body.imageUrls = images;
|
|
if ("referenceImages" in body) body.referenceImages = images;
|
|
if ("reference_images" in body) body.reference_images = images;
|
|
if (Array.isArray(body.video_clips)) {
|
|
const template =
|
|
body.video_clips[0] && typeof body.video_clips[0] === "object"
|
|
? (body.video_clips[0] as Record<string, unknown>)
|
|
: {};
|
|
body.video_clips = images.map((url) => ({ ...template, url }));
|
|
}
|
|
if (
|
|
params.capability === "multi-image-edit" &&
|
|
!Array.isArray(body.images) &&
|
|
!Array.isArray(body.referenceImages) &&
|
|
!Array.isArray(body.reference_images) &&
|
|
!Array.isArray(body.video_clips)
|
|
) {
|
|
throw new Error(`${params.modelId} official invocation does not expose multi-image input`);
|
|
}
|
|
}
|
|
if (params.capability === "extend") {
|
|
if ("aspectRatio" in body) body.aspectRatio = "16:9";
|
|
if ("aspect_ratio" in body) body.aspect_ratio = "16:9";
|
|
if ("ratio" in body) body.ratio = "16:9";
|
|
}
|
|
if (params.capability === "world" && "mode" in body) body.mode = "world-generation";
|
|
|
|
const path = String(params.invocation.path || "").trim();
|
|
if (!path.startsWith("/")) throw new Error(`${params.modelId} has no valid invocation path`);
|
|
|
|
const submitted = await parseResponse(
|
|
await fetch(`${params.baseUrl}${path}`, {
|
|
method: String(params.invocation.method || "POST").toUpperCase(),
|
|
headers: {
|
|
Authorization: `Bearer ${params.apiKey}`,
|
|
"Content-Type": params.invocation.content_type || "application/json",
|
|
},
|
|
body: JSON.stringify(body),
|
|
signal: AbortSignal.timeout(180_000),
|
|
}),
|
|
);
|
|
const inputUrls = new Set([imageUrl, secondImageUrl]);
|
|
const immediate = Array.from(collectUrls(submitted)).filter((url) => !inputUrls.has(url));
|
|
if (immediate.length > 0) return immediate;
|
|
const taskId = extractTaskId(submitted);
|
|
if (!taskId) throw new Error("External task returned neither output nor task ID");
|
|
const resultPath =
|
|
extractResultPath(submitted) ||
|
|
String(params.invocation.result_path || "").replaceAll(
|
|
"{task_id}",
|
|
encodeURIComponent(taskId),
|
|
);
|
|
const startedAt = Date.now();
|
|
while (Date.now() - startedAt < 1_200_000) {
|
|
await new Promise((resolvePromise) => setTimeout(resolvePromise, 5000));
|
|
const resultUrl = resultPath
|
|
? resultPath.startsWith("http")
|
|
? resultPath
|
|
: `${params.baseUrl}${resultPath.startsWith("/") ? "" : "/"}${resultPath}`
|
|
: `${params.baseUrl}/api/external/task/result?taskId=${encodeURIComponent(taskId)}`;
|
|
const payload = await parseResponse(
|
|
await fetch(resultUrl, {
|
|
headers: { Authorization: `Bearer ${params.apiKey}` },
|
|
signal: AbortSignal.timeout(60_000),
|
|
}),
|
|
);
|
|
const status = extractStatus(payload);
|
|
const outputs = Array.from(collectUrls(payload)).filter((url) => !inputUrls.has(url));
|
|
if (["failed", "error", "timeout", "canceled", "cancelled"].includes(status)) {
|
|
throw new Error(`External task failed with status ${status}`);
|
|
}
|
|
if (
|
|
["succeeded", "success", "completed", "done", "finished"].includes(status) ||
|
|
outputs.length > 0
|
|
) {
|
|
if (outputs.length === 0) throw new Error("External task completed without output URL");
|
|
return outputs;
|
|
}
|
|
}
|
|
throw new Error("External task did not complete within 20 minutes");
|
|
}
|
|
|
|
async function main() {
|
|
const capabilityArg = getArgValue("--capability");
|
|
if (!CAPABILITIES.includes(capabilityArg as Capability)) {
|
|
throw new Error(`Pass exactly one --capability=${CAPABILITIES.join("|")}`);
|
|
}
|
|
const capability = capabilityArg as Capability;
|
|
const apiKey = String(process.env.VIP98K_API_KEY || "").trim();
|
|
if (!apiKey) throw new Error("VIP98K_API_KEY is required and must remain server-side");
|
|
const baseUrl = normalizeVip98kBaseUrl(process.env.VIP98K_BASE_URL || VIP98K_BASE_URL);
|
|
const catalog = await fetchVip98kCatalog({ baseURL: baseUrl, apiKey });
|
|
const availableModelIds = new Set(catalog.map(getModelId));
|
|
const explicitModelId = getArgValue("--model");
|
|
const modelId =
|
|
explicitModelId || DEFAULT_MODEL_IDS[capability].find((candidate) => availableModelIds.has(candidate));
|
|
if (!modelId || !availableModelIds.has(modelId)) {
|
|
throw new Error(`No catalog model is available for ${capability}; pass --model=<catalog-id>`);
|
|
}
|
|
const selectedModel = catalog.find((model) => getModelId(model) === modelId);
|
|
if (
|
|
!["chat", "multimodal", "image"].includes(capability) &&
|
|
!selectedModel?.invocation?.path
|
|
) {
|
|
throw new Error(`${modelId} is missing its official invocation contract`);
|
|
}
|
|
|
|
const ledgerPath = resolve(
|
|
getArgValue("--ledger") ||
|
|
process.env.VIP98K_TEST_LEDGER ||
|
|
".local/vip98k-capability-tests.jsonl",
|
|
);
|
|
const ledger = await readLedger(ledgerPath);
|
|
if (
|
|
!hasFlag("--force") &&
|
|
ledger.some(
|
|
(entry) =>
|
|
entry.capability === capability && entry.modelId === modelId && entry.status === "success",
|
|
)
|
|
) {
|
|
console.log(JSON.stringify({ skipped: true, capability, modelId, ledgerPath }, null, 2));
|
|
return;
|
|
}
|
|
|
|
const runId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
const startedAt = new Date();
|
|
await writeLedger(ledgerPath, {
|
|
runId,
|
|
capability,
|
|
modelId,
|
|
status: "started",
|
|
startedAt: startedAt.toISOString(),
|
|
});
|
|
|
|
try {
|
|
let outputs: string[];
|
|
if (capability === "chat" || capability === "multimodal") {
|
|
outputs = await runChat({
|
|
baseUrl,
|
|
apiKey,
|
|
modelId,
|
|
multimodal: capability === "multimodal",
|
|
});
|
|
} else if (capability === "image") {
|
|
outputs = await runOpenAIImage({
|
|
baseUrl,
|
|
apiKey,
|
|
modelId,
|
|
multiImage: false,
|
|
});
|
|
} else {
|
|
outputs = await runExternalTask({
|
|
baseUrl,
|
|
apiKey,
|
|
modelId,
|
|
capability,
|
|
invocation: selectedModel!.invocation!,
|
|
});
|
|
}
|
|
const finishedAt = new Date();
|
|
const entry: LedgerEntry = {
|
|
runId,
|
|
capability,
|
|
modelId,
|
|
status: "success",
|
|
startedAt: startedAt.toISOString(),
|
|
finishedAt: finishedAt.toISOString(),
|
|
durationMs: finishedAt.getTime() - startedAt.getTime(),
|
|
outputCount: outputs.length,
|
|
outputDigest: digestOutputs(outputs),
|
|
};
|
|
await writeLedger(ledgerPath, entry);
|
|
console.log(JSON.stringify({ ...entry, ledgerPath }, null, 2));
|
|
} catch (error) {
|
|
const finishedAt = new Date();
|
|
const entry: LedgerEntry = {
|
|
runId,
|
|
capability,
|
|
modelId,
|
|
status: "failure",
|
|
startedAt: startedAt.toISOString(),
|
|
finishedAt: finishedAt.toISOString(),
|
|
durationMs: finishedAt.getTime() - startedAt.getTime(),
|
|
error: sanitizeError(error),
|
|
};
|
|
await writeLedger(ledgerPath, entry);
|
|
console.error(JSON.stringify({ ...entry, ledgerPath }, null, 2));
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(sanitizeError(error));
|
|
process.exit(1);
|
|
});
|