1385 lines
45 KiB
TypeScript
1385 lines
45 KiB
TypeScript
import "server-only";
|
|
|
|
import { GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
|
|
import { createOpenAI } from "@ai-sdk/openai";
|
|
import sharp from "sharp";
|
|
import { getDefaultStorageConfig } from "@/lib/db/queries";
|
|
import {
|
|
bindVip98kImages,
|
|
selectVip98kTaskInvocation,
|
|
type Vip98kTaskInvocation,
|
|
} from "@/lib/ai/vip98k-task-invocation";
|
|
import { VIP98K_BASE_URL, normalizeVip98kBaseUrl } from "@/lib/ai/vip98k-models";
|
|
import { shouldResolveVip98kResultDelivery } from "@/lib/ai/vip98k-result-delivery";
|
|
import {
|
|
addVip98kReferenceRetryToken,
|
|
getStorageObjectKeyFromPublicUrl,
|
|
getVip98kStagedMediaRetryDelay,
|
|
getVip98kSubmissionRetryDelay,
|
|
shouldStageVip98kReferenceUrl,
|
|
} from "@/lib/ai/vip98k-reference-staging";
|
|
import { createLogger } from "@/lib/observability/logger";
|
|
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
|
import {
|
|
buildS3PublicUrl,
|
|
createS3Client,
|
|
type S3UploadConfig,
|
|
toDbS3UploadConfig,
|
|
uploadFile,
|
|
} from "@/lib/storage/service";
|
|
|
|
const logger = createLogger("vip98k-provider");
|
|
|
|
interface Vip98kConfig {
|
|
apiKey: string;
|
|
baseURL?: string | null;
|
|
}
|
|
|
|
interface Vip98kTaskParams {
|
|
apiKey: string;
|
|
baseURL?: string | null;
|
|
modelId: string;
|
|
kind: string;
|
|
prompt: string;
|
|
imageUrls: string[];
|
|
videoUrls: string[];
|
|
videoParams?: Record<string, unknown>;
|
|
imageParams?: Record<string, unknown>;
|
|
modelConfig: Vip98kModelRuntimeConfig;
|
|
}
|
|
|
|
type Vip98kModelRuntimeConfig = {
|
|
capabilities?: Record<string, unknown> | null;
|
|
imageGenConfig?: Record<string, unknown> | null;
|
|
};
|
|
|
|
type Vip98kInvocation = Vip98kTaskInvocation;
|
|
|
|
type UploadedImage = {
|
|
thumbnail: string;
|
|
original: string;
|
|
};
|
|
|
|
const DOWNLOAD_TIMEOUT_MS = 120_000;
|
|
const DEFAULT_POLL_INTERVAL_SECONDS = 8;
|
|
const DEFAULT_MAX_POLL_SECONDS = 600;
|
|
const REFERENCE_DOWNLOAD_ATTEMPTS = 3;
|
|
const REFERENCE_DOWNLOAD_TIMEOUT_MS = 120_000;
|
|
const MAX_REFERENCE_DOWNLOAD_BYTES = 64 * 1024 * 1024;
|
|
const REFERENCE_STAGING_CACHE_MS = 60_000;
|
|
const REFERENCE_STAGING_KEY_PREFIX = "uploads/vip98k-";
|
|
const MAX_SUBMISSION_ATTEMPTS = 3;
|
|
|
|
type ReferenceStagingConfig = {
|
|
sourcePublicUrl: string;
|
|
storageConfig: S3UploadConfig;
|
|
};
|
|
|
|
let referenceStagingConfigCache:
|
|
| { expiresAt: number; value: ReferenceStagingConfig | null }
|
|
| undefined;
|
|
const referenceStagingTasks = new Map<string, Promise<string>>();
|
|
|
|
function toRecord(value: unknown): Record<string, unknown> | null {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
return value as Record<string, unknown>;
|
|
}
|
|
|
|
function normalizeOpenAIBaseUrl(baseURL?: string | null) {
|
|
const normalized = normalizeVip98kBaseUrl(baseURL || VIP98K_BASE_URL);
|
|
return /\/v1$/i.test(normalized) ? normalized : `${normalized}/v1`;
|
|
}
|
|
|
|
function parseRuntimeConfig(value?: unknown): Vip98kModelRuntimeConfig {
|
|
const record = toRecord(value);
|
|
if (!record) return {};
|
|
if ("capabilities" in record || "imageGenConfig" in record) {
|
|
return {
|
|
capabilities: toRecord(record.capabilities),
|
|
imageGenConfig: toRecord(record.imageGenConfig),
|
|
};
|
|
}
|
|
return {
|
|
capabilities: null,
|
|
imageGenConfig: record,
|
|
};
|
|
}
|
|
|
|
function getStringField(source: unknown, key: string) {
|
|
const record = toRecord(source);
|
|
return String(record?.[key] || "").trim().toLowerCase();
|
|
}
|
|
|
|
function getVip98kMetadata(config: Vip98kModelRuntimeConfig) {
|
|
const capabilitiesVip98k = toRecord(toRecord(config.capabilities)?.vip98k);
|
|
const imageConfigVip98k = toRecord(toRecord(config.imageGenConfig)?.vip98k);
|
|
return capabilitiesVip98k || imageConfigVip98k || {};
|
|
}
|
|
|
|
function parseVip98kInvocation(value: unknown): Vip98kInvocation | null {
|
|
const invocation = toRecord(value);
|
|
if (!invocation) return null;
|
|
const path = String(invocation.path || "").trim();
|
|
if (!path.startsWith("/")) return null;
|
|
return {
|
|
kind: String(invocation.kind || "task").trim().toLowerCase(),
|
|
method: String(invocation.method || "POST").trim().toUpperCase(),
|
|
path,
|
|
contentType: String(invocation.contentType || "application/json").trim(),
|
|
requestExample: { ...(toRecord(invocation.requestExample) || {}) },
|
|
resultMethod: String(invocation.resultMethod || "GET").trim().toUpperCase(),
|
|
resultPath: String(invocation.resultPath || "").trim(),
|
|
};
|
|
}
|
|
|
|
function getVip98kInvocations(config: Vip98kModelRuntimeConfig) {
|
|
const metadata = getVip98kMetadata(config);
|
|
const configured = Array.isArray(metadata.invocations) ? metadata.invocations : [];
|
|
const candidates = [metadata.invocation, ...configured]
|
|
.map(parseVip98kInvocation)
|
|
.filter((item): item is Vip98kInvocation => Boolean(item));
|
|
return candidates.filter(
|
|
(candidate, index, items) =>
|
|
items.findIndex(
|
|
(item) =>
|
|
item.method === candidate.method &&
|
|
item.path === candidate.path &&
|
|
JSON.stringify(item.requestExample) === JSON.stringify(candidate.requestExample),
|
|
) === index,
|
|
);
|
|
}
|
|
|
|
function inferKind(modelId: string, config: Vip98kModelRuntimeConfig) {
|
|
const capabilities = config.capabilities;
|
|
const metadata = getVip98kMetadata(config);
|
|
const configuredKind =
|
|
getStringField(capabilities, "kind") ||
|
|
getStringField(capabilities, "modelType") ||
|
|
getStringField(capabilities, "mediaType") ||
|
|
getStringField(metadata, "modelType");
|
|
if (configuredKind.includes("world")) return "world";
|
|
if (configuredKind.includes("video")) return "video";
|
|
if (configuredKind.includes("model-3d") || configuredKind === "3d") {
|
|
return "model-3d";
|
|
}
|
|
if (configuredKind.includes("image-tool")) return "image-tool";
|
|
if (configuredKind.includes("image")) return "image";
|
|
if (configuredKind.includes("audio")) return "audio";
|
|
if (configuredKind.includes("document")) return "document-tool";
|
|
if (configuredKind.includes("chat") || configuredKind.includes("model")) return "chat";
|
|
|
|
const normalizedModelId = modelId.trim().toLowerCase();
|
|
if (normalizedModelId === "world-generation") return "world";
|
|
if (/^veo[-_.]/.test(normalizedModelId) || normalizedModelId.startsWith("veo-")) {
|
|
return "video";
|
|
}
|
|
if (
|
|
normalizedModelId.startsWith("gpt-image") ||
|
|
normalizedModelId.startsWith("nano-banana")
|
|
) {
|
|
return "image";
|
|
}
|
|
return "chat";
|
|
}
|
|
|
|
function delay(ms: number) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
async function parseJsonResponse(response: Response) {
|
|
const text = await response.text();
|
|
if (!text.trim()) return null;
|
|
try {
|
|
return JSON.parse(text);
|
|
} catch {
|
|
return text;
|
|
}
|
|
}
|
|
|
|
const ERROR_MESSAGE_FIELDS = [
|
|
"failureReason",
|
|
"failure_reason",
|
|
"errorMessage",
|
|
"error_message",
|
|
"errorDescription",
|
|
"error_description",
|
|
"reason",
|
|
"detail",
|
|
"description",
|
|
"message",
|
|
"error",
|
|
] as const;
|
|
|
|
const ERROR_CONTAINER_FIELDS = [
|
|
"data",
|
|
"result",
|
|
"task",
|
|
"output",
|
|
"response",
|
|
] as const;
|
|
|
|
function findNestedErrorMessage(value: unknown, depth = 0): string {
|
|
if (depth > 4 || value == null) return "";
|
|
if (typeof value === "string") return value.trim();
|
|
if (Array.isArray(value)) {
|
|
for (const item of value) {
|
|
const message = findNestedErrorMessage(item, depth + 1);
|
|
if (message) return message;
|
|
}
|
|
return "";
|
|
}
|
|
|
|
const record = toRecord(value);
|
|
if (!record) return "";
|
|
|
|
for (const key of ERROR_MESSAGE_FIELDS) {
|
|
if (!(key in record)) continue;
|
|
const message = findNestedErrorMessage(record[key], depth + 1);
|
|
if (message) return message;
|
|
}
|
|
for (const key of ERROR_CONTAINER_FIELDS) {
|
|
if (!(key in record)) continue;
|
|
const message = findNestedErrorMessage(record[key], depth + 1);
|
|
if (message) return message;
|
|
}
|
|
return "";
|
|
}
|
|
|
|
function extractErrorMessage(payload: unknown, fallback = "VIP98K request failed") {
|
|
return findNestedErrorMessage(payload) || fallback;
|
|
}
|
|
|
|
function extractPayloadShape(payload: unknown) {
|
|
const record = toRecord(payload);
|
|
const data = toRecord(record?.data);
|
|
const task = toRecord(data?.task) || toRecord(record?.task);
|
|
return {
|
|
payloadKeys: record ? Object.keys(record).sort() : [],
|
|
dataKeys: data ? Object.keys(data).sort() : [],
|
|
taskKeys: task ? Object.keys(task).sort() : [],
|
|
};
|
|
}
|
|
|
|
function unwrapData(payload: unknown) {
|
|
const record = toRecord(payload);
|
|
return record?.data && typeof record.data === "object" ? record.data : payload;
|
|
}
|
|
|
|
function extractTaskId(payload: unknown) {
|
|
const data = unwrapData(payload);
|
|
const record = toRecord(data);
|
|
return String(
|
|
record?.taskId ||
|
|
record?.task_id ||
|
|
record?.id ||
|
|
record?.requestId ||
|
|
record?.request_id ||
|
|
"",
|
|
).trim();
|
|
}
|
|
|
|
function extractResultPath(payload: unknown) {
|
|
const data = unwrapData(payload);
|
|
const record = toRecord(data);
|
|
return String(record?.resultPath || record?.result_path || "").trim();
|
|
}
|
|
|
|
function extractStatus(payload: unknown) {
|
|
const data = unwrapData(payload);
|
|
const record = toRecord(data);
|
|
return String(record?.status || record?.state || record?.taskStatus || "").trim().toLowerCase();
|
|
}
|
|
|
|
function extractDone(payload: unknown) {
|
|
const data = unwrapData(payload);
|
|
const record = toRecord(data);
|
|
if (typeof record?.done === "boolean") return record.done;
|
|
if (typeof record?.isDone === "boolean") return record.isDone;
|
|
if (typeof record?.finished === "boolean") return record.finished;
|
|
if (typeof record?.completed === "boolean") return record.completed;
|
|
if (typeof record?.ready === "boolean") return record.ready;
|
|
return false;
|
|
}
|
|
|
|
function collectHttpUrls(value: unknown, urls = new Set<string>()) {
|
|
if (!value) return urls;
|
|
if (typeof value === "string") {
|
|
const rawUrlMatches = value.match(/https?:\/\/[^\s"'<>()[\]{}]+/g) || [];
|
|
for (const match of rawUrlMatches) {
|
|
urls.add(match.replace(/[),.;]+$/g, ""));
|
|
}
|
|
if (/^data:image\/[a-zA-Z0-9.+-]+;base64,/i.test(value)) urls.add(value);
|
|
return urls;
|
|
}
|
|
if (Array.isArray(value)) {
|
|
value.forEach((item) => collectHttpUrls(item, urls));
|
|
return urls;
|
|
}
|
|
if (typeof value !== "object") return urls;
|
|
Object.values(value as Record<string, unknown>).forEach((nested) =>
|
|
collectHttpUrls(nested, urls),
|
|
);
|
|
return urls;
|
|
}
|
|
|
|
function collectOutputUrls(payload: unknown) {
|
|
const data = unwrapData(payload);
|
|
const candidateRecord = toRecord(data);
|
|
const preferred = [
|
|
candidateRecord?.result,
|
|
candidateRecord?.results,
|
|
candidateRecord?.output,
|
|
candidateRecord?.outputs,
|
|
candidateRecord?.url,
|
|
candidateRecord?.urls,
|
|
candidateRecord?.imageUrl,
|
|
candidateRecord?.image_url,
|
|
candidateRecord?.videoUrl,
|
|
candidateRecord?.video_url,
|
|
candidateRecord?.assetUrl,
|
|
candidateRecord?.asset_url,
|
|
candidateRecord?.files,
|
|
candidateRecord?.assets,
|
|
data,
|
|
];
|
|
const urls = new Set<string>();
|
|
preferred.forEach((candidate) => collectHttpUrls(candidate, urls));
|
|
return Array.from(urls);
|
|
}
|
|
|
|
async function fetchTaskResult(params: {
|
|
apiKey: string;
|
|
baseURL: string;
|
|
taskId: string;
|
|
resultUrl: string;
|
|
}) {
|
|
const response = await fetch(params.resultUrl, {
|
|
cache: "no-store",
|
|
headers: { Authorization: `Bearer ${params.apiKey}` },
|
|
signal: AbortSignal.timeout(60_000),
|
|
});
|
|
const payload = await parseJsonResponse(response);
|
|
if (!response.ok) {
|
|
throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`);
|
|
}
|
|
|
|
const status = extractStatus(payload);
|
|
let outputUrls = collectOutputUrls(payload);
|
|
let deliveryPayload: unknown = null;
|
|
// V2 media tasks return opaque asset_id values from their published result path.
|
|
// The compatibility result route resolves those IDs into expiring download URLs.
|
|
if (
|
|
shouldResolveVip98kResultDelivery({
|
|
payload,
|
|
status,
|
|
outputUrlCount: outputUrls.length,
|
|
})
|
|
) {
|
|
const deliveryUrl = `${params.baseURL}/api/external/task/result?taskId=${encodeURIComponent(
|
|
params.taskId,
|
|
)}`;
|
|
const deliveryResponse = await fetch(deliveryUrl, {
|
|
cache: "no-store",
|
|
headers: { Authorization: `Bearer ${params.apiKey}` },
|
|
signal: AbortSignal.timeout(60_000),
|
|
});
|
|
deliveryPayload = await parseJsonResponse(deliveryResponse);
|
|
if (!deliveryResponse.ok) {
|
|
throw new Error(
|
|
`VIP98K result delivery lookup failed: ${extractErrorMessage(
|
|
deliveryPayload,
|
|
)} (HTTP ${deliveryResponse.status})`,
|
|
);
|
|
}
|
|
outputUrls = collectOutputUrls(deliveryPayload);
|
|
}
|
|
|
|
return {
|
|
payload: deliveryPayload ? { result: payload, delivery: deliveryPayload } : payload,
|
|
status: extractStatus(deliveryPayload) || status,
|
|
outputUrls,
|
|
};
|
|
}
|
|
|
|
function parseDataUrl(value: string) {
|
|
const match = value.match(/^data:([^;,]+);base64,(.+)$/s);
|
|
if (!match) throw new Error("VIP98K returned invalid Base64 media");
|
|
return {
|
|
mimeType: match[1] || "image/png",
|
|
buffer: Buffer.from(match[2] || "", "base64"),
|
|
};
|
|
}
|
|
|
|
function detectExtension(mimeType: string) {
|
|
if (/jpe?g/i.test(mimeType)) return "jpg";
|
|
if (/webp/i.test(mimeType)) return "webp";
|
|
if (/gif/i.test(mimeType)) return "gif";
|
|
if (/mp4/i.test(mimeType)) return "mp4";
|
|
if (/webm/i.test(mimeType)) return "webm";
|
|
if (/quicktime/i.test(mimeType)) return "mov";
|
|
return "png";
|
|
}
|
|
|
|
function toArrayBuffer(buffer: Buffer) {
|
|
return buffer.buffer.slice(
|
|
buffer.byteOffset,
|
|
buffer.byteOffset + buffer.byteLength,
|
|
) as ArrayBuffer;
|
|
}
|
|
|
|
async function uploadBuffer(buffer: Buffer, mimeType: string) {
|
|
const extension = detectExtension(mimeType);
|
|
const thumbnailBuffer = await sharp(buffer, { failOn: "none" })
|
|
.resize(400, 400, { fit: "inside", withoutEnlargement: true })
|
|
.jpeg({ quality: 82 })
|
|
.toBuffer()
|
|
.catch(() => buffer);
|
|
const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
const [original, thumbnail] = await Promise.all([
|
|
uploadFile(`vip98k-orig-${suffix}.${extension}`, toArrayBuffer(buffer)),
|
|
uploadFile(`vip98k-thumb-${suffix}.jpg`, toArrayBuffer(thumbnailBuffer)),
|
|
]);
|
|
return { original: original.url, thumbnail: thumbnail.url };
|
|
}
|
|
|
|
async function archiveOutputImage(imageUrl: string): Promise<UploadedImage> {
|
|
if (imageUrl.startsWith("data:")) {
|
|
const parsed = parseDataUrl(imageUrl);
|
|
return uploadBuffer(parsed.buffer, parsed.mimeType);
|
|
}
|
|
const deferred = await createDeferredUploadedImage(imageUrl, {
|
|
providerType: "vip98k",
|
|
});
|
|
if (deferred) return deferred;
|
|
const response = await fetch(imageUrl, {
|
|
cache: "no-store",
|
|
redirect: "follow",
|
|
headers: { "User-Agent": "yedir-ai-vip98k/1.0" },
|
|
signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS),
|
|
});
|
|
if (!response.ok) {
|
|
throw new Error(`VIP98K result image download failed (HTTP ${response.status})`);
|
|
}
|
|
return uploadBuffer(
|
|
Buffer.from(await response.arrayBuffer()),
|
|
response.headers.get("content-type") || "image/png",
|
|
);
|
|
}
|
|
|
|
async function resolveReferenceStagingConfig() {
|
|
const now = Date.now();
|
|
if (referenceStagingConfigCache?.expiresAt && referenceStagingConfigCache.expiresAt > now) {
|
|
return referenceStagingConfigCache.value;
|
|
}
|
|
|
|
let value: ReferenceStagingConfig | null = null;
|
|
try {
|
|
const sourceStorage = await getDefaultStorageConfig();
|
|
const storageConfig = toDbS3UploadConfig(sourceStorage);
|
|
if (sourceStorage?.publicUrl && storageConfig) {
|
|
value = {
|
|
sourcePublicUrl: sourceStorage.publicUrl,
|
|
storageConfig,
|
|
};
|
|
}
|
|
} catch (error) {
|
|
logger.warn("reference staging config lookup failed", {
|
|
message: error instanceof Error ? error.message : String(error),
|
|
});
|
|
}
|
|
|
|
referenceStagingConfigCache = {
|
|
expiresAt: now + REFERENCE_STAGING_CACHE_MS,
|
|
value,
|
|
};
|
|
return value;
|
|
}
|
|
|
|
function getUrlHost(value: string) {
|
|
try {
|
|
return new URL(value).host;
|
|
} catch {
|
|
return "invalid";
|
|
}
|
|
}
|
|
|
|
function isStagedReferenceUrl(value: string) {
|
|
try {
|
|
return new URL(value).pathname.startsWith(`/${REFERENCE_STAGING_KEY_PREFIX}`);
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async function downloadReferenceImage(mediaUrl: string) {
|
|
let lastError: unknown;
|
|
for (let attempt = 1; attempt <= REFERENCE_DOWNLOAD_ATTEMPTS; attempt += 1) {
|
|
try {
|
|
const response = await fetch(mediaUrl, {
|
|
cache: "no-store",
|
|
redirect: "follow",
|
|
headers: { "User-Agent": "yedir-ai-vip98k-staging/1.0" },
|
|
signal: AbortSignal.timeout(REFERENCE_DOWNLOAD_TIMEOUT_MS),
|
|
});
|
|
if (!response.ok) {
|
|
throw new Error(`source returned HTTP ${response.status}`);
|
|
}
|
|
|
|
const declaredLength = Number(response.headers.get("content-length"));
|
|
if (
|
|
Number.isFinite(declaredLength) &&
|
|
declaredLength > MAX_REFERENCE_DOWNLOAD_BYTES
|
|
) {
|
|
throw new Error("source image exceeds the staging size limit");
|
|
}
|
|
|
|
const buffer = Buffer.from(await response.arrayBuffer());
|
|
if (buffer.byteLength > MAX_REFERENCE_DOWNLOAD_BYTES) {
|
|
throw new Error("source image exceeds the staging size limit");
|
|
}
|
|
if (
|
|
Number.isFinite(declaredLength) &&
|
|
declaredLength > 0 &&
|
|
buffer.byteLength !== declaredLength
|
|
) {
|
|
throw new Error("source image download was incomplete");
|
|
}
|
|
return {
|
|
buffer,
|
|
mimeType: response.headers.get("content-type") || "image/png",
|
|
};
|
|
} catch (error) {
|
|
lastError = error;
|
|
logger.warn("reference staging download attempt failed", {
|
|
attempt,
|
|
sourceHost: getUrlHost(mediaUrl),
|
|
message: error instanceof Error ? error.message : String(error),
|
|
});
|
|
if (attempt < REFERENCE_DOWNLOAD_ATTEMPTS) await delay(attempt * 1000);
|
|
}
|
|
}
|
|
|
|
throw new Error(
|
|
`VIP98K reference staging failed: ${
|
|
lastError instanceof Error ? lastError.message : "source download failed"
|
|
}`,
|
|
);
|
|
}
|
|
|
|
async function downloadReferenceImageFromStorage(
|
|
mediaUrl: string,
|
|
stagingConfig: ReferenceStagingConfig,
|
|
) {
|
|
const objectKey = getStorageObjectKeyFromPublicUrl(
|
|
mediaUrl,
|
|
stagingConfig.sourcePublicUrl,
|
|
);
|
|
if (!objectKey) return null;
|
|
|
|
try {
|
|
const response = await createS3Client(stagingConfig.storageConfig).send(
|
|
new GetObjectCommand({
|
|
Bucket: stagingConfig.storageConfig.bucket,
|
|
Key: objectKey,
|
|
}),
|
|
{ abortSignal: AbortSignal.timeout(REFERENCE_DOWNLOAD_TIMEOUT_MS) },
|
|
);
|
|
if (!response.Body) throw new Error("source object body is empty");
|
|
if (
|
|
typeof response.ContentLength === "number" &&
|
|
response.ContentLength > MAX_REFERENCE_DOWNLOAD_BYTES
|
|
) {
|
|
throw new Error("source image exceeds the staging size limit");
|
|
}
|
|
const buffer = Buffer.from(await response.Body.transformToByteArray());
|
|
if (buffer.byteLength > MAX_REFERENCE_DOWNLOAD_BYTES) {
|
|
throw new Error("source image exceeds the staging size limit");
|
|
}
|
|
if (
|
|
typeof response.ContentLength === "number" &&
|
|
response.ContentLength > 0 &&
|
|
buffer.byteLength !== response.ContentLength
|
|
) {
|
|
throw new Error("source object download was incomplete");
|
|
}
|
|
return {
|
|
buffer,
|
|
mimeType: response.ContentType || "image/png",
|
|
};
|
|
} catch (error) {
|
|
logger.warn("reference source-storage download failed; retrying public URL", {
|
|
sourceHost: getUrlHost(mediaUrl),
|
|
message: error instanceof Error ? error.message : String(error),
|
|
});
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function uploadReferenceImage(
|
|
buffer: Buffer,
|
|
stagingConfig: ReferenceStagingConfig,
|
|
) {
|
|
const key = `${REFERENCE_STAGING_KEY_PREFIX}${Date.now()}-${Math.random()
|
|
.toString(36)
|
|
.slice(2, 8)}.jpg`;
|
|
await createS3Client(stagingConfig.storageConfig).send(
|
|
new PutObjectCommand({
|
|
Bucket: stagingConfig.storageConfig.bucket,
|
|
Key: key,
|
|
Body: buffer,
|
|
ContentType: "image/jpeg",
|
|
CacheControl: "public, max-age=31536000, immutable",
|
|
...(stagingConfig.storageConfig.provider === "qiniu-kodo"
|
|
? {}
|
|
: { ACL: "public-read" as const }),
|
|
}),
|
|
{ abortSignal: AbortSignal.timeout(REFERENCE_DOWNLOAD_TIMEOUT_MS) },
|
|
);
|
|
return buildS3PublicUrl(stagingConfig.storageConfig, key);
|
|
}
|
|
|
|
async function normalizeReferenceImage(buffer: Buffer, mimeType: string) {
|
|
try {
|
|
const normalized = await sharp(buffer, { failOn: "none" })
|
|
.rotate()
|
|
.resize(1920, 1920, { fit: "inside", withoutEnlargement: true })
|
|
.flatten({ background: "#ffffff" })
|
|
.jpeg({ quality: 92, chromaSubsampling: "4:4:4", mozjpeg: true })
|
|
.toBuffer();
|
|
return {
|
|
buffer: normalized,
|
|
extension: "jpg",
|
|
};
|
|
} catch (error) {
|
|
logger.warn("reference image normalization failed; preserving original bytes", {
|
|
message: error instanceof Error ? error.message : String(error),
|
|
});
|
|
return { buffer, extension: detectExtension(mimeType) };
|
|
}
|
|
}
|
|
|
|
async function stageReferenceImage(
|
|
mediaUrl: string,
|
|
stagingConfig: ReferenceStagingConfig,
|
|
) {
|
|
const downloaded =
|
|
(await downloadReferenceImageFromStorage(mediaUrl, stagingConfig)) ||
|
|
(await downloadReferenceImage(mediaUrl));
|
|
const normalized = await normalizeReferenceImage(
|
|
downloaded.buffer,
|
|
downloaded.mimeType,
|
|
);
|
|
const uploadedUrl = await uploadReferenceImage(normalized.buffer, stagingConfig);
|
|
logger.info("reference image staged", {
|
|
sourceHost: getUrlHost(mediaUrl),
|
|
targetHost: getUrlHost(uploadedUrl),
|
|
sourceBytes: downloaded.buffer.byteLength,
|
|
stagedBytes: normalized.buffer.byteLength,
|
|
});
|
|
return uploadedUrl;
|
|
}
|
|
|
|
async function ensurePublicMediaUrl(mediaUrl: string, mediaKind: "image" | "video") {
|
|
const normalized = String(mediaUrl || "").trim();
|
|
if (/^https?:\/\//i.test(normalized)) {
|
|
if (mediaKind !== "image") return normalized;
|
|
const stagingConfig = await resolveReferenceStagingConfig();
|
|
const sourceObjectKey = stagingConfig
|
|
? getStorageObjectKeyFromPublicUrl(normalized, stagingConfig.sourcePublicUrl)
|
|
: null;
|
|
if (
|
|
!stagingConfig ||
|
|
sourceObjectKey?.startsWith(REFERENCE_STAGING_KEY_PREFIX) ||
|
|
!shouldStageVip98kReferenceUrl({
|
|
mediaUrl: normalized,
|
|
sourcePublicUrl: stagingConfig.sourcePublicUrl,
|
|
})
|
|
) {
|
|
return normalized;
|
|
}
|
|
|
|
const existingTask = referenceStagingTasks.get(normalized);
|
|
if (existingTask) return existingTask;
|
|
const task = stageReferenceImage(normalized, stagingConfig).finally(() => {
|
|
referenceStagingTasks.delete(normalized);
|
|
});
|
|
referenceStagingTasks.set(normalized, task);
|
|
return task;
|
|
}
|
|
if (!normalized.startsWith("data:")) return normalized;
|
|
const parsed = parseDataUrl(normalized);
|
|
const extension = detectExtension(parsed.mimeType);
|
|
const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
const stagingConfig = await resolveReferenceStagingConfig();
|
|
if (stagingConfig && mediaKind === "image") {
|
|
const normalizedImage = await normalizeReferenceImage(
|
|
parsed.buffer,
|
|
parsed.mimeType,
|
|
);
|
|
return uploadReferenceImage(normalizedImage.buffer, stagingConfig);
|
|
}
|
|
const uploaded = await uploadFile(
|
|
`vip98k-ref-${suffix}.${extension}`,
|
|
toArrayBuffer(parsed.buffer),
|
|
);
|
|
return uploaded.url;
|
|
}
|
|
|
|
function extractPromptAndMedia(messages: Array<{ role: string; content?: any; parts?: any[] }>) {
|
|
const message = messages.filter((item) => item.role === "user").pop();
|
|
if (!message) throw new Error("No user message found");
|
|
|
|
const texts: string[] = [];
|
|
const imageUrls: string[] = [];
|
|
const videoUrls: string[] = [];
|
|
|
|
const pushText = (value: unknown) => {
|
|
if (typeof value === "string" && value.trim()) texts.push(value.trim());
|
|
};
|
|
|
|
const pushMedia = (value: unknown, mediaType?: unknown) => {
|
|
const url = String(
|
|
typeof value === "string"
|
|
? value
|
|
: (value as any)?.url ||
|
|
(value as any)?.imageUrl ||
|
|
(value as any)?.fileUrl ||
|
|
(value as any)?.src ||
|
|
"",
|
|
).trim();
|
|
if (!url || !/^(?:https?:\/\/|data:)/i.test(url)) return;
|
|
const normalizedMediaType = String(mediaType || "").toLowerCase();
|
|
const isVideo =
|
|
normalizedMediaType.startsWith("video/") ||
|
|
url.startsWith("data:video/") ||
|
|
/\.(?:mp4|mov|webm|m4v)(?:[?#].*)?$/i.test(url);
|
|
const target = isVideo ? videoUrls : imageUrls;
|
|
if (!target.includes(url)) target.push(url);
|
|
};
|
|
|
|
const visitPart = (part: any) => {
|
|
if (!part) return;
|
|
if (typeof part === "string") return pushText(part);
|
|
if (part.type === "text" || typeof part.text === "string") pushText(part.text);
|
|
if (part.type === "image_url") pushMedia(part.image_url, "image/jpeg");
|
|
if (part.type === "image") pushMedia(part.image || part.url, "image/jpeg");
|
|
if (part.type === "file") {
|
|
pushMedia(
|
|
part.url || part.file || part.data,
|
|
part.mediaType || part.contentType || part.mimeType,
|
|
);
|
|
}
|
|
if (part.inlineData?.data) {
|
|
pushMedia(
|
|
`data:${part.inlineData.mimeType || "image/png"};base64,${part.inlineData.data}`,
|
|
part.inlineData.mimeType,
|
|
);
|
|
}
|
|
};
|
|
|
|
if (typeof message.content === "string") pushText(message.content);
|
|
else if (Array.isArray(message.content)) message.content.forEach(visitPart);
|
|
if (Array.isArray(message.parts)) message.parts.forEach(visitPart);
|
|
|
|
const prompt = texts.join("\n").trim() || "Generate media";
|
|
return { prompt, imageUrls, videoUrls };
|
|
}
|
|
|
|
function getConfiguredPollSeconds(config: Vip98kModelRuntimeConfig, key: string, fallback: number) {
|
|
const metadata = getVip98kMetadata(config);
|
|
const value = Number(metadata[key]);
|
|
if (!Number.isFinite(value) || value <= 0) return fallback;
|
|
return Math.min(Math.max(value, 1), 3600);
|
|
}
|
|
|
|
async function submitTask(params: {
|
|
apiKey: string;
|
|
baseURL?: string | null;
|
|
path?: string | null;
|
|
method?: string | null;
|
|
contentType?: string | null;
|
|
body: Record<string, unknown>;
|
|
}) {
|
|
const baseURL = normalizeVip98kBaseUrl(params.baseURL);
|
|
const path = String(params.path || "/api/external/task/generate").trim();
|
|
if (!path.startsWith("/")) throw new Error("VIP98K invocation path is invalid");
|
|
const stagedReferences = Array.isArray(params.body.file_urls)
|
|
? params.body.file_urls.filter(
|
|
(value): value is string =>
|
|
typeof value === "string" && isStagedReferenceUrl(value),
|
|
)
|
|
: [];
|
|
const maxAttempts = MAX_SUBMISSION_ATTEMPTS;
|
|
let refreshStagedReferences = false;
|
|
|
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
const requestBody =
|
|
!refreshStagedReferences || !Array.isArray(params.body.file_urls)
|
|
? params.body
|
|
: {
|
|
...params.body,
|
|
file_urls: params.body.file_urls.map((value) =>
|
|
typeof value === "string" && isStagedReferenceUrl(value)
|
|
? addVip98kReferenceRetryToken(
|
|
value,
|
|
`${Date.now()}-${attempt}`,
|
|
)
|
|
: value,
|
|
),
|
|
};
|
|
const response = await fetch(`${baseURL}${path}`, {
|
|
method: String(params.method || "POST").toUpperCase(),
|
|
headers: {
|
|
Authorization: `Bearer ${params.apiKey}`,
|
|
"Content-Type": params.contentType || "application/json",
|
|
},
|
|
body: JSON.stringify(requestBody),
|
|
signal: AbortSignal.timeout(120_000),
|
|
});
|
|
const payload = await parseJsonResponse(response);
|
|
if (response.ok) {
|
|
return {
|
|
payload,
|
|
taskId: extractTaskId(payload),
|
|
resultPath: extractResultPath(payload),
|
|
outputs: collectOutputUrls(payload),
|
|
};
|
|
}
|
|
|
|
const errorMessage = extractErrorMessage(payload);
|
|
const stagedMediaRetryDelayMs = getVip98kStagedMediaRetryDelay({
|
|
status: response.status,
|
|
message: errorMessage,
|
|
attempt,
|
|
maxAttempts,
|
|
stagedReferenceCount: stagedReferences.length,
|
|
});
|
|
if (stagedMediaRetryDelayMs !== null) {
|
|
logger.warn("staged reference not ready for provider; retrying submission", {
|
|
attempt,
|
|
retryDelayMs: stagedMediaRetryDelayMs,
|
|
stagedReferenceCount: stagedReferences.length,
|
|
});
|
|
refreshStagedReferences = true;
|
|
await delay(stagedMediaRetryDelayMs);
|
|
continue;
|
|
}
|
|
|
|
const retryAfterSeconds = Number(response.headers.get("retry-after"));
|
|
const reconciliationRetryDelayMs = getVip98kSubmissionRetryDelay({
|
|
status: response.status,
|
|
message: errorMessage,
|
|
attempt,
|
|
maxAttempts,
|
|
retryAfterMs: Number.isFinite(retryAfterSeconds)
|
|
? retryAfterSeconds * 1000
|
|
: null,
|
|
});
|
|
if (reconciliationRetryDelayMs !== null) {
|
|
logger.warn("provider is reconciling task submission; retrying", {
|
|
attempt,
|
|
retryDelayMs: reconciliationRetryDelayMs,
|
|
});
|
|
refreshStagedReferences = false;
|
|
await delay(reconciliationRetryDelayMs);
|
|
continue;
|
|
}
|
|
throw new Error(`${errorMessage} (HTTP ${response.status})`);
|
|
}
|
|
|
|
throw new Error("VIP98K request failed after media readiness retries");
|
|
}
|
|
|
|
export type Vip98kExternalTaskSubmission = {
|
|
taskId: string;
|
|
resultPath: string;
|
|
outputUrls: string[];
|
|
payload: unknown;
|
|
};
|
|
|
|
export type Vip98kExternalTaskStatus = {
|
|
taskId: string;
|
|
status: string;
|
|
done: boolean;
|
|
failed: boolean;
|
|
outputUrls: string[];
|
|
payload: unknown;
|
|
};
|
|
|
|
export async function submitVip98kExternalTask(params: {
|
|
apiKey: string;
|
|
baseURL?: string | null;
|
|
modelId: string;
|
|
kind: string;
|
|
prompt?: string;
|
|
imageUrls?: string[];
|
|
videoUrls?: string[];
|
|
videoParams?: Record<string, unknown>;
|
|
imageParams?: Record<string, unknown>;
|
|
modelConfig?: Vip98kModelRuntimeConfig;
|
|
}): Promise<Vip98kExternalTaskSubmission> {
|
|
const [imageUrls, videoUrls] = await Promise.all([
|
|
Promise.all(
|
|
(params.imageUrls || []).map((url) => ensurePublicMediaUrl(url, "image")),
|
|
),
|
|
Promise.all(
|
|
(params.videoUrls || []).map((url) => ensurePublicMediaUrl(url, "video")),
|
|
),
|
|
]);
|
|
const taskParams: Vip98kTaskParams = {
|
|
apiKey: params.apiKey,
|
|
baseURL: params.baseURL,
|
|
modelId: params.modelId,
|
|
kind: params.kind,
|
|
prompt: String(params.prompt || "Process media").trim() || "Process media",
|
|
imageUrls,
|
|
videoUrls,
|
|
videoParams: params.videoParams,
|
|
imageParams: params.imageParams,
|
|
modelConfig: params.modelConfig || {},
|
|
};
|
|
const configuredInvocations = getVip98kInvocations(taskParams.modelConfig);
|
|
const invocation = selectVip98kTaskInvocation({
|
|
modelId: taskParams.modelId,
|
|
configuredInvocation: configuredInvocations[0] || null,
|
|
configuredInvocations,
|
|
imageCount: imageUrls.length,
|
|
});
|
|
const { body, imageBindingKeys, outputConstraintFallbacks } = buildTaskBody(
|
|
taskParams,
|
|
invocation,
|
|
);
|
|
if (imageUrls.length > 0 && imageBindingKeys.length === 0) {
|
|
throw new Error(
|
|
`VIP98K model does not publish a reference-image request contract: ${taskParams.modelId}`,
|
|
);
|
|
}
|
|
logger.info("task submit audit", {
|
|
modelId: taskParams.modelId,
|
|
kind: taskParams.kind,
|
|
invocationKind: invocation?.kind || "task",
|
|
invocationPath: invocation?.path || "/api/external/task",
|
|
imageCount: imageUrls.length,
|
|
videoCount: videoUrls.length,
|
|
imageBindingKeys,
|
|
outputConstraintFallbacks,
|
|
bodyKeys: Object.keys(body).sort(),
|
|
});
|
|
const submitted = await submitTask({
|
|
apiKey: params.apiKey,
|
|
baseURL: params.baseURL,
|
|
path: invocation?.path,
|
|
method: invocation?.method,
|
|
contentType: invocation?.contentType,
|
|
body,
|
|
});
|
|
const resultPath = submitted.resultPath || invocation?.resultPath || "";
|
|
return {
|
|
taskId: submitted.taskId,
|
|
resultPath: submitted.taskId
|
|
? resultPath.replaceAll("{task_id}", encodeURIComponent(submitted.taskId))
|
|
: resultPath,
|
|
outputUrls: submitted.outputs,
|
|
payload: submitted.payload,
|
|
};
|
|
}
|
|
|
|
export async function getVip98kExternalTaskStatus(params: {
|
|
apiKey: string;
|
|
baseURL?: string | null;
|
|
taskId: string;
|
|
resultPath?: string | null;
|
|
}): Promise<Vip98kExternalTaskStatus> {
|
|
const baseURL = normalizeVip98kBaseUrl(params.baseURL);
|
|
const resultUrl = params.resultPath
|
|
? params.resultPath.startsWith("http")
|
|
? params.resultPath
|
|
: `${baseURL}${params.resultPath.startsWith("/") ? "" : "/"}${params.resultPath}`
|
|
: `${baseURL}/api/external/task/result?taskId=${encodeURIComponent(params.taskId)}`;
|
|
const { payload, status, outputUrls } = await fetchTaskResult({
|
|
apiKey: params.apiKey,
|
|
baseURL,
|
|
taskId: params.taskId,
|
|
resultUrl,
|
|
});
|
|
const failed = ["failed", "error", "timeout", "canceled", "cancelled"].includes(status);
|
|
const done =
|
|
["succeeded", "success", "completed", "done", "finished"].includes(status) ||
|
|
(extractDone(payload) && outputUrls.length > 0);
|
|
return {
|
|
taskId: params.taskId,
|
|
status,
|
|
done,
|
|
failed,
|
|
outputUrls,
|
|
payload,
|
|
};
|
|
}
|
|
|
|
async function waitForTask(params: {
|
|
apiKey: string;
|
|
baseURL?: string | null;
|
|
taskId: string;
|
|
modelId: string;
|
|
resultPath?: string;
|
|
modelConfig: Vip98kModelRuntimeConfig;
|
|
}) {
|
|
const baseURL = normalizeVip98kBaseUrl(params.baseURL);
|
|
const intervalMs =
|
|
getConfiguredPollSeconds(
|
|
params.modelConfig,
|
|
"pollIntervalSeconds",
|
|
DEFAULT_POLL_INTERVAL_SECONDS,
|
|
) * 1000;
|
|
const timeoutMs =
|
|
getConfiguredPollSeconds(params.modelConfig, "maxPollSeconds", DEFAULT_MAX_POLL_SECONDS) *
|
|
1000;
|
|
const startedAt = Date.now();
|
|
|
|
while (Date.now() - startedAt < timeoutMs) {
|
|
await delay(intervalMs);
|
|
const resultUrl = params.resultPath
|
|
? params.resultPath.startsWith("http")
|
|
? params.resultPath
|
|
: `${baseURL}${params.resultPath.startsWith("/") ? "" : "/"}${params.resultPath}`
|
|
: `${baseURL}/api/external/task/result?taskId=${encodeURIComponent(params.taskId)}`;
|
|
const { payload, status, outputUrls: outputs } = await fetchTaskResult({
|
|
apiKey: params.apiKey,
|
|
baseURL,
|
|
taskId: params.taskId,
|
|
resultUrl,
|
|
});
|
|
const done = extractDone(payload);
|
|
if (
|
|
["succeeded", "success", "completed", "done", "finished"].includes(status) ||
|
|
(done && outputs.length > 0)
|
|
) {
|
|
return { payload, outputs };
|
|
}
|
|
if (["failed", "error", "timeout", "canceled", "cancelled"].includes(status)) {
|
|
const providerMessage = extractErrorMessage(payload, "");
|
|
logger.error("task failed", {
|
|
taskId: params.taskId,
|
|
modelId: params.modelId,
|
|
status,
|
|
providerMessage: providerMessage || null,
|
|
...extractPayloadShape(payload),
|
|
});
|
|
const detail = providerMessage ? `: ${providerMessage}` : "";
|
|
throw new Error(
|
|
`VIP98K task failed${detail} (task_id: ${params.taskId}, model: ${params.modelId})`,
|
|
);
|
|
}
|
|
}
|
|
|
|
throw new Error("VIP98K task timed out; it may still be running");
|
|
}
|
|
|
|
function normalizeRatio(value: unknown) {
|
|
const normalized = String(value || "").trim();
|
|
return normalized && normalized !== "auto" ? normalized : undefined;
|
|
}
|
|
|
|
function normalizeDuration(value: unknown) {
|
|
const numeric = Number(value);
|
|
if (!Number.isFinite(numeric) || numeric <= 0) return undefined;
|
|
return Math.round(numeric);
|
|
}
|
|
|
|
function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvocation | null) {
|
|
const videoParams = params.videoParams || {};
|
|
const imageParams = params.imageParams || {};
|
|
const promptTitle = params.prompt.split(/\s+/).slice(0, 8).join(" ").slice(0, 80);
|
|
const body: Record<string, unknown> = { ...(invocation?.requestExample || {}) };
|
|
if ("model" in body || !invocation) body.model = params.modelId;
|
|
if ("displayName" in body || !invocation) {
|
|
body.displayName = promptTitle || params.modelId;
|
|
}
|
|
if ("prompt" in body || !invocation) body.prompt = params.prompt;
|
|
|
|
const aspectRatio =
|
|
normalizeRatio(videoParams.ratio) ||
|
|
normalizeRatio(videoParams.aspectRatio) ||
|
|
normalizeRatio(imageParams.aspectRatio);
|
|
const duration = normalizeDuration(videoParams.duration);
|
|
const resolution = String(videoParams.resolution || imageParams.imageSize || "").trim();
|
|
const negativePrompt = String(videoParams.negativePrompt || imageParams.negativePrompt || "").trim();
|
|
const outputConstraintFallbacks: string[] = [];
|
|
let hasNativeAspectRatio = false;
|
|
let hasNativeResolution = false;
|
|
|
|
if (aspectRatio) {
|
|
if ("ratio" in body || !invocation) {
|
|
body.ratio = aspectRatio;
|
|
hasNativeAspectRatio = true;
|
|
}
|
|
if ("aspectRatio" in body || !invocation) {
|
|
body.aspectRatio = aspectRatio;
|
|
hasNativeAspectRatio = true;
|
|
}
|
|
if ("aspect_ratio" in body || !invocation) {
|
|
body.aspect_ratio = aspectRatio;
|
|
hasNativeAspectRatio = true;
|
|
}
|
|
}
|
|
if (duration && ("duration" in body || !invocation)) body.duration = duration;
|
|
if (duration && "duration_seconds" in body) body.duration_seconds = duration;
|
|
if (resolution) {
|
|
if ("resolution" in body || !invocation) {
|
|
body.resolution = resolution;
|
|
hasNativeResolution = true;
|
|
}
|
|
if ("imageSize" in body || !invocation) {
|
|
body.imageSize = resolution;
|
|
hasNativeResolution = true;
|
|
}
|
|
if ("image_size" in body || !invocation) {
|
|
body.image_size = resolution;
|
|
hasNativeResolution = true;
|
|
}
|
|
if ("size" in body && typeof body.size === "string") {
|
|
body.size = resolution;
|
|
hasNativeResolution = true;
|
|
}
|
|
}
|
|
if (negativePrompt) {
|
|
if ("negativePrompt" in body || !invocation) body.negativePrompt = negativePrompt;
|
|
if ("negative_prompt" in body || !invocation) body.negative_prompt = negativePrompt;
|
|
}
|
|
const imageToolParamAliases = [
|
|
["scaleFactor", "scale_factor"],
|
|
["xScale", "x_scale"],
|
|
["yScale", "y_scale"],
|
|
] as const;
|
|
for (const [sourceKey, targetKey] of imageToolParamAliases) {
|
|
const value = imageParams[sourceKey];
|
|
if (value !== undefined && targetKey in body) body[targetKey] = value;
|
|
}
|
|
if (typeof videoParams.generateAudio === "boolean") {
|
|
if ("generateAudio" in body || !invocation) body.generateAudio = videoParams.generateAudio;
|
|
if ("generate_audio" in body || !invocation) body.generate_audio = videoParams.generateAudio;
|
|
}
|
|
|
|
const imageBindingKeys = bindVip98kImages(body, params.imageUrls);
|
|
if (params.imageUrls.length > 0 && !invocation && imageBindingKeys.length === 0) {
|
|
body.images = params.imageUrls;
|
|
imageBindingKeys.push("images");
|
|
}
|
|
|
|
const promptConstraints: string[] = [];
|
|
if (aspectRatio && !hasNativeAspectRatio) {
|
|
promptConstraints.push(`aspect ratio ${aspectRatio}`);
|
|
outputConstraintFallbacks.push("aspectRatio:prompt");
|
|
}
|
|
if (resolution && !hasNativeResolution) {
|
|
promptConstraints.push(`resolution ${resolution}`);
|
|
outputConstraintFallbacks.push("imageSize:prompt");
|
|
}
|
|
if (promptConstraints.length > 0 && typeof body.prompt === "string") {
|
|
body.prompt = `${body.prompt}\n\n[Output constraints] ${promptConstraints.join(
|
|
"; ",
|
|
)}.`;
|
|
}
|
|
|
|
if (params.videoUrls.length > 0) {
|
|
if ("videos" in body || !invocation) body.videos = params.videoUrls;
|
|
if ("videoUrls" in body || !invocation) body.videoUrls = params.videoUrls;
|
|
if ("referenceVideos" in body || !invocation) body.referenceVideos = params.videoUrls;
|
|
if ("video" in body) body.video = params.videoUrls[0];
|
|
if ("video_url" in body) body.video_url = params.videoUrls[0];
|
|
}
|
|
|
|
if (params.kind === "world" && ("mode" in body || !invocation)) {
|
|
body.mode = "world-generation";
|
|
} else if (params.kind === "video" && ("mode" in body || !invocation)) {
|
|
body.mode = params.imageUrls.length > 0 ? "image-to-video" : "text-to-video";
|
|
}
|
|
|
|
return { body, imageBindingKeys, outputConstraintFallbacks };
|
|
}
|
|
|
|
function formatImageMarkdown(images: UploadedImage[], imageSize: string) {
|
|
return images
|
|
.map((image) => ``)
|
|
.join("\n\n");
|
|
}
|
|
|
|
function formatVideoMarkdown(videoUrl: string) {
|
|
return `<video src="${videoUrl}" controls style="max-width: 100%; border-radius: 8px; margin-bottom: 12px;"></video>\n\n[下载视频](${videoUrl})`;
|
|
}
|
|
|
|
function formatWorldMarkdown(urls: string[]) {
|
|
if (urls.length === 0) return "3D 世界生成成功，但未返回可访问的结果地址";
|
|
return urls.map((url, index) => `[查看 3D 世界 ${index + 1}](${url})`).join("\n\n");
|
|
}
|
|
|
|
export type Vip98kExternalTaskResult = {
|
|
taskId: string;
|
|
outputUrls: string[];
|
|
payload: unknown;
|
|
};
|
|
|
|
export async function executeVip98kExternalTask(params: {
|
|
apiKey: string;
|
|
baseURL?: string | null;
|
|
modelId: string;
|
|
kind: string;
|
|
prompt?: string;
|
|
imageUrls?: string[];
|
|
videoUrls?: string[];
|
|
videoParams?: Record<string, unknown>;
|
|
imageParams?: Record<string, unknown>;
|
|
modelConfig?: Vip98kModelRuntimeConfig;
|
|
}): Promise<Vip98kExternalTaskResult> {
|
|
const submitted = await submitVip98kExternalTask({
|
|
...params,
|
|
});
|
|
|
|
if (submitted.outputUrls.length > 0) {
|
|
return {
|
|
taskId: submitted.taskId,
|
|
outputUrls: submitted.outputUrls,
|
|
payload: submitted.payload,
|
|
};
|
|
}
|
|
if (!submitted.taskId) {
|
|
throw new Error("VIP98K task submission returned neither a task ID nor output URLs");
|
|
}
|
|
|
|
const completed = await waitForTask({
|
|
apiKey: params.apiKey,
|
|
baseURL: params.baseURL,
|
|
taskId: submitted.taskId,
|
|
modelId: params.modelId,
|
|
resultPath: submitted.resultPath,
|
|
modelConfig: params.modelConfig || {},
|
|
});
|
|
return {
|
|
taskId: submitted.taskId,
|
|
outputUrls: completed.outputs,
|
|
payload: completed.payload,
|
|
};
|
|
}
|
|
|
|
async function runTask(params: Vip98kTaskParams) {
|
|
const completed = await executeVip98kExternalTask({
|
|
apiKey: params.apiKey,
|
|
baseURL: params.baseURL,
|
|
modelId: params.modelId,
|
|
kind: params.kind,
|
|
prompt: params.prompt,
|
|
imageUrls: params.imageUrls,
|
|
videoUrls: params.videoUrls,
|
|
videoParams: params.videoParams,
|
|
imageParams: params.imageParams,
|
|
modelConfig: params.modelConfig,
|
|
});
|
|
const outputUrls = completed.outputUrls;
|
|
|
|
if (params.kind === "video") {
|
|
const videoUrl =
|
|
outputUrls.find((url) => /\.(mp4|webm|mov|m4v)(?:[?#].*)?$/i.test(url)) ||
|
|
outputUrls[0];
|
|
if (!videoUrl) throw new Error("VIP98K task completed but returned no video URL");
|
|
return formatVideoMarkdown(videoUrl);
|
|
}
|
|
|
|
if (params.kind === "image" || params.kind === "image-tool") {
|
|
if (outputUrls.length === 0) {
|
|
throw new Error("VIP98K task completed but returned no image URL");
|
|
}
|
|
const imageSize = String(params.imageParams?.imageSize || "2K").trim() || "2K";
|
|
const archived = await Promise.all(outputUrls.slice(0, 1).map(archiveOutputImage));
|
|
return formatImageMarkdown(archived, imageSize);
|
|
}
|
|
|
|
if (params.kind === "world" || params.kind === "model-3d") {
|
|
return formatWorldMarkdown(outputUrls);
|
|
}
|
|
|
|
return outputUrls.length ? outputUrls.join("\n") : "VIP98K task completed";
|
|
}
|
|
|
|
export function createVip98k({ apiKey, baseURL }: Vip98kConfig) {
|
|
const rootBaseURL = normalizeVip98kBaseUrl(baseURL);
|
|
const openAIBaseURL = normalizeOpenAIBaseUrl(rootBaseURL);
|
|
const openAITextProvider = createOpenAI({
|
|
apiKey,
|
|
baseURL: openAIBaseURL,
|
|
});
|
|
return {
|
|
provider: "vip98k",
|
|
chat: (modelId: string, defaultModelConfig?: unknown): any => {
|
|
const modelConfig = parseRuntimeConfig(defaultModelConfig);
|
|
const kind = inferKind(modelId, modelConfig);
|
|
|
|
if (kind === "chat") {
|
|
return openAITextProvider.chat(modelId) as any;
|
|
}
|
|
|
|
return {
|
|
provider: "vip98k",
|
|
doGenerate: async (options: {
|
|
messages: Array<{ role: string; content?: any; parts?: any[] }>;
|
|
imageParams?: Record<string, unknown>;
|
|
videoParams?: Record<string, unknown>;
|
|
imageGenConfig?: unknown;
|
|
overrideModelId?: string;
|
|
}) => {
|
|
const runtimeConfig =
|
|
options.imageGenConfig !== undefined
|
|
? parseRuntimeConfig({
|
|
capabilities: modelConfig.capabilities,
|
|
imageGenConfig: options.imageGenConfig,
|
|
})
|
|
: modelConfig;
|
|
const activeModelId = String(options.overrideModelId || modelId || "").trim();
|
|
if (!activeModelId) throw new Error("VIP98K model ID is not configured");
|
|
const activeKind = inferKind(activeModelId, runtimeConfig);
|
|
const { prompt, imageUrls, videoUrls } = extractPromptAndMedia(
|
|
options.messages,
|
|
);
|
|
const [publicImageUrls, publicVideoUrls] = await Promise.all([
|
|
Promise.all(
|
|
imageUrls.map((url) => ensurePublicMediaUrl(url, "image")),
|
|
),
|
|
Promise.all(
|
|
videoUrls.map((url) => ensurePublicMediaUrl(url, "video")),
|
|
),
|
|
]);
|
|
const text = await runTask({
|
|
apiKey,
|
|
baseURL: rootBaseURL,
|
|
modelId: activeModelId,
|
|
kind: activeKind,
|
|
prompt,
|
|
imageUrls: publicImageUrls,
|
|
videoUrls: publicVideoUrls,
|
|
videoParams: options.videoParams,
|
|
imageParams: options.imageParams,
|
|
modelConfig: runtimeConfig,
|
|
});
|
|
return {
|
|
text,
|
|
finishReason: "stop" as const,
|
|
usage: {
|
|
promptTokens: 0,
|
|
completionTokens: text.length,
|
|
},
|
|
};
|
|
},
|
|
doStream: async function* (options: {
|
|
messages: Array<{ role: string; content?: any; parts?: any[] }>;
|
|
imageParams?: Record<string, unknown>;
|
|
videoParams?: Record<string, unknown>;
|
|
imageGenConfig?: unknown;
|
|
overrideModelId?: string;
|
|
}) {
|
|
const result = await this.doGenerate(options);
|
|
yield { type: "text-delta", textDelta: result.text };
|
|
},
|
|
};
|
|
},
|
|
};
|
|
}
|