import "server-only"; import { GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"; import { createOpenAI } from "@ai-sdk/openai"; import sharp from "sharp"; import { getDefaultStorageConfig } from "@/lib/db/queries"; import { bindVip98kImages, selectVip98kTaskInvocation, type Vip98kTaskInvocation, } from "@/lib/ai/vip98k-task-invocation"; import { VIP98K_BASE_URL, normalizeVip98kBaseUrl } from "@/lib/ai/vip98k-models"; import { shouldResolveVip98kResultDelivery } from "@/lib/ai/vip98k-result-delivery"; import { addVip98kReferenceRetryToken, getStorageObjectKeyFromPublicUrl, getVip98kStagedMediaRetryDelay, getVip98kSubmissionRetryDelay, shouldStageVip98kReferenceUrl, } from "@/lib/ai/vip98k-reference-staging"; import { createLogger } from "@/lib/observability/logger"; import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { buildS3PublicUrl, createS3Client, type S3UploadConfig, toDbS3UploadConfig, uploadFile, } from "@/lib/storage/service"; const logger = createLogger("vip98k-provider"); interface Vip98kConfig { apiKey: string; baseURL?: string | null; } interface Vip98kTaskParams { apiKey: string; baseURL?: string | null; modelId: string; kind: string; prompt: string; imageUrls: string[]; videoUrls: string[]; videoParams?: Record; imageParams?: Record; modelConfig: Vip98kModelRuntimeConfig; } type Vip98kModelRuntimeConfig = { capabilities?: Record | null; imageGenConfig?: Record | null; }; type Vip98kInvocation = Vip98kTaskInvocation; type UploadedImage = { thumbnail: string; original: string; }; const DOWNLOAD_TIMEOUT_MS = 120_000; const DEFAULT_POLL_INTERVAL_SECONDS = 8; const DEFAULT_MAX_POLL_SECONDS = 600; const REFERENCE_DOWNLOAD_ATTEMPTS = 3; const REFERENCE_DOWNLOAD_TIMEOUT_MS = 120_000; const MAX_REFERENCE_DOWNLOAD_BYTES = 40 * 1024 * 1024; const REFERENCE_STAGING_CACHE_MS = 60_000; const REFERENCE_STAGING_KEY_PREFIX = "uploads/vip98k-"; const MAX_SUBMISSION_ATTEMPTS = 3; type ReferenceStagingConfig = { sourcePublicUrl: string; storageConfig: S3UploadConfig; }; let referenceStagingConfigCache: | { expiresAt: number; value: ReferenceStagingConfig | null } | undefined; const referenceStagingTasks = new Map>(); function toRecord(value: unknown): Record | null { if (!value || typeof value !== "object" || Array.isArray(value)) return null; return value as Record; } function normalizeOpenAIBaseUrl(baseURL?: string | null) { const normalized = normalizeVip98kBaseUrl(baseURL || VIP98K_BASE_URL); return /\/v1$/i.test(normalized) ? normalized : `${normalized}/v1`; } function parseRuntimeConfig(value?: unknown): Vip98kModelRuntimeConfig { const record = toRecord(value); if (!record) return {}; if ("capabilities" in record || "imageGenConfig" in record) { return { capabilities: toRecord(record.capabilities), imageGenConfig: toRecord(record.imageGenConfig), }; } return { capabilities: null, imageGenConfig: record, }; } function getStringField(source: unknown, key: string) { const record = toRecord(source); return String(record?.[key] || "").trim().toLowerCase(); } function getVip98kMetadata(config: Vip98kModelRuntimeConfig) { const capabilitiesVip98k = toRecord(toRecord(config.capabilities)?.vip98k); const imageConfigVip98k = toRecord(toRecord(config.imageGenConfig)?.vip98k); return capabilitiesVip98k || imageConfigVip98k || {}; } function parseVip98kInvocation(value: unknown): Vip98kInvocation | null { const invocation = toRecord(value); if (!invocation) return null; const path = String(invocation.path || "").trim(); if (!path.startsWith("/")) return null; return { kind: String(invocation.kind || "task").trim().toLowerCase(), method: String(invocation.method || "POST").trim().toUpperCase(), path, contentType: String(invocation.contentType || "application/json").trim(), requestExample: { ...(toRecord(invocation.requestExample) || {}) }, resultMethod: String(invocation.resultMethod || "GET").trim().toUpperCase(), resultPath: String(invocation.resultPath || "").trim(), }; } function getVip98kInvocations(config: Vip98kModelRuntimeConfig) { const metadata = getVip98kMetadata(config); const configured = Array.isArray(metadata.invocations) ? metadata.invocations : []; const candidates = [metadata.invocation, ...configured] .map(parseVip98kInvocation) .filter((item): item is Vip98kInvocation => Boolean(item)); return candidates.filter( (candidate, index, items) => items.findIndex( (item) => item.method === candidate.method && item.path === candidate.path && JSON.stringify(item.requestExample) === JSON.stringify(candidate.requestExample), ) === index, ); } function inferKind(modelId: string, config: Vip98kModelRuntimeConfig) { const capabilities = config.capabilities; const metadata = getVip98kMetadata(config); const configuredKind = getStringField(capabilities, "kind") || getStringField(capabilities, "modelType") || getStringField(capabilities, "mediaType") || getStringField(metadata, "modelType"); if (configuredKind.includes("world")) return "world"; if (configuredKind.includes("video")) return "video"; if (configuredKind.includes("model-3d") || configuredKind === "3d") { return "model-3d"; } if (configuredKind.includes("image-tool")) return "image-tool"; if (configuredKind.includes("image")) return "image"; if (configuredKind.includes("audio")) return "audio"; if (configuredKind.includes("document")) return "document-tool"; if (configuredKind.includes("chat") || configuredKind.includes("model")) return "chat"; const normalizedModelId = modelId.trim().toLowerCase(); if (normalizedModelId === "world-generation") return "world"; if (/^veo[-_.]/.test(normalizedModelId) || normalizedModelId.startsWith("veo-")) { return "video"; } if ( normalizedModelId.startsWith("gpt-image") || normalizedModelId.startsWith("nano-banana") ) { return "image"; } return "chat"; } function delay(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } async function parseJsonResponse(response: Response) { const text = await response.text(); if (!text.trim()) return null; try { return JSON.parse(text); } catch { return text; } } const ERROR_MESSAGE_FIELDS = [ "failureReason", "failure_reason", "errorMessage", "error_message", "errorDescription", "error_description", "reason", "detail", "description", "message", "error", ] as const; const ERROR_CONTAINER_FIELDS = [ "data", "result", "task", "output", "response", ] as const; function findNestedErrorMessage(value: unknown, depth = 0): string { if (depth > 4 || value == null) return ""; if (typeof value === "string") return value.trim(); if (Array.isArray(value)) { for (const item of value) { const message = findNestedErrorMessage(item, depth + 1); if (message) return message; } return ""; } const record = toRecord(value); if (!record) return ""; for (const key of ERROR_MESSAGE_FIELDS) { if (!(key in record)) continue; const message = findNestedErrorMessage(record[key], depth + 1); if (message) return message; } for (const key of ERROR_CONTAINER_FIELDS) { if (!(key in record)) continue; const message = findNestedErrorMessage(record[key], depth + 1); if (message) return message; } return ""; } function extractErrorMessage(payload: unknown, fallback = "VIP98K request failed") { return findNestedErrorMessage(payload) || fallback; } function extractPayloadShape(payload: unknown) { const record = toRecord(payload); const data = toRecord(record?.data); const task = toRecord(data?.task) || toRecord(record?.task); return { payloadKeys: record ? Object.keys(record).sort() : [], dataKeys: data ? Object.keys(data).sort() : [], taskKeys: task ? Object.keys(task).sort() : [], }; } function unwrapData(payload: unknown) { const record = toRecord(payload); return record?.data && typeof record.data === "object" ? record.data : payload; } function extractTaskId(payload: unknown) { const data = unwrapData(payload); const record = toRecord(data); return String( record?.taskId || record?.task_id || record?.id || record?.requestId || record?.request_id || "", ).trim(); } function extractResultPath(payload: unknown) { const data = unwrapData(payload); const record = toRecord(data); return String(record?.resultPath || record?.result_path || "").trim(); } function extractStatus(payload: unknown) { const data = unwrapData(payload); const record = toRecord(data); return String(record?.status || record?.state || record?.taskStatus || "").trim().toLowerCase(); } function extractDone(payload: unknown) { const data = unwrapData(payload); const record = toRecord(data); if (typeof record?.done === "boolean") return record.done; if (typeof record?.isDone === "boolean") return record.isDone; if (typeof record?.finished === "boolean") return record.finished; if (typeof record?.completed === "boolean") return record.completed; if (typeof record?.ready === "boolean") return record.ready; return false; } function collectHttpUrls(value: unknown, urls = new Set()) { if (!value) return urls; if (typeof value === "string") { const rawUrlMatches = value.match(/https?:\/\/[^\s"'<>()[\]{}]+/g) || []; for (const match of rawUrlMatches) { urls.add(match.replace(/[),.;]+$/g, "")); } if (/^data:image\/[a-zA-Z0-9.+-]+;base64,/i.test(value)) urls.add(value); return urls; } if (Array.isArray(value)) { value.forEach((item) => collectHttpUrls(item, urls)); return urls; } if (typeof value !== "object") return urls; Object.values(value as Record).forEach((nested) => collectHttpUrls(nested, urls), ); return urls; } function collectOutputUrls(payload: unknown) { const data = unwrapData(payload); const candidateRecord = toRecord(data); const preferred = [ candidateRecord?.result, candidateRecord?.results, candidateRecord?.output, candidateRecord?.outputs, candidateRecord?.url, candidateRecord?.urls, candidateRecord?.imageUrl, candidateRecord?.image_url, candidateRecord?.videoUrl, candidateRecord?.video_url, candidateRecord?.assetUrl, candidateRecord?.asset_url, candidateRecord?.files, candidateRecord?.assets, data, ]; const urls = new Set(); preferred.forEach((candidate) => collectHttpUrls(candidate, urls)); return Array.from(urls); } async function fetchTaskResult(params: { apiKey: string; baseURL: string; taskId: string; resultUrl: string; }) { const response = await fetch(params.resultUrl, { cache: "no-store", headers: { Authorization: `Bearer ${params.apiKey}` }, signal: AbortSignal.timeout(60_000), }); const payload = await parseJsonResponse(response); if (!response.ok) { throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`); } const status = extractStatus(payload); let outputUrls = collectOutputUrls(payload); let deliveryPayload: unknown = null; // V2 media tasks return opaque asset_id values from their published result path. // The compatibility result route resolves those IDs into expiring download URLs. if ( shouldResolveVip98kResultDelivery({ payload, status, outputUrlCount: outputUrls.length, }) ) { const deliveryUrl = `${params.baseURL}/api/external/task/result?taskId=${encodeURIComponent( params.taskId, )}`; const deliveryResponse = await fetch(deliveryUrl, { cache: "no-store", headers: { Authorization: `Bearer ${params.apiKey}` }, signal: AbortSignal.timeout(60_000), }); deliveryPayload = await parseJsonResponse(deliveryResponse); if (!deliveryResponse.ok) { throw new Error( `VIP98K result delivery lookup failed: ${extractErrorMessage( deliveryPayload, )} (HTTP ${deliveryResponse.status})`, ); } outputUrls = collectOutputUrls(deliveryPayload); } return { payload: deliveryPayload ? { result: payload, delivery: deliveryPayload } : payload, status: extractStatus(deliveryPayload) || status, outputUrls, }; } function parseDataUrl(value: string) { const match = value.match(/^data:([^;,]+);base64,(.+)$/s); if (!match) throw new Error("VIP98K returned invalid Base64 media"); return { mimeType: match[1] || "image/png", buffer: Buffer.from(match[2] || "", "base64"), }; } function detectExtension(mimeType: string) { if (/jpe?g/i.test(mimeType)) return "jpg"; if (/webp/i.test(mimeType)) return "webp"; if (/gif/i.test(mimeType)) return "gif"; if (/mp4/i.test(mimeType)) return "mp4"; if (/webm/i.test(mimeType)) return "webm"; if (/quicktime/i.test(mimeType)) return "mov"; return "png"; } function toArrayBuffer(buffer: Buffer) { return buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength, ) as ArrayBuffer; } async function uploadBuffer(buffer: Buffer, mimeType: string) { const extension = detectExtension(mimeType); const thumbnailBuffer = await sharp(buffer, { failOn: "none" }) .resize(400, 400, { fit: "inside", withoutEnlargement: true }) .jpeg({ quality: 82 }) .toBuffer() .catch(() => buffer); const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; const [original, thumbnail] = await Promise.all([ uploadFile(`vip98k-orig-${suffix}.${extension}`, toArrayBuffer(buffer)), uploadFile(`vip98k-thumb-${suffix}.jpg`, toArrayBuffer(thumbnailBuffer)), ]); return { original: original.url, thumbnail: thumbnail.url }; } async function archiveOutputImage(imageUrl: string): Promise { if (imageUrl.startsWith("data:")) { const parsed = parseDataUrl(imageUrl); return uploadBuffer(parsed.buffer, parsed.mimeType); } const deferred = await createDeferredUploadedImage(imageUrl, { providerType: "vip98k", }); if (deferred) return deferred; const response = await fetch(imageUrl, { cache: "no-store", redirect: "follow", headers: { "User-Agent": "yedir-ai-vip98k/1.0" }, signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS), }); if (!response.ok) { throw new Error(`VIP98K result image download failed (HTTP ${response.status})`); } return uploadBuffer( Buffer.from(await response.arrayBuffer()), response.headers.get("content-type") || "image/png", ); } async function resolveReferenceStagingConfig() { const now = Date.now(); if (referenceStagingConfigCache?.expiresAt && referenceStagingConfigCache.expiresAt > now) { return referenceStagingConfigCache.value; } let value: ReferenceStagingConfig | null = null; try { const sourceStorage = await getDefaultStorageConfig(); const storageConfig = toDbS3UploadConfig(sourceStorage); if (sourceStorage?.publicUrl && storageConfig) { value = { sourcePublicUrl: sourceStorage.publicUrl, storageConfig, }; } } catch (error) { logger.warn("reference staging config lookup failed", { message: error instanceof Error ? error.message : String(error), }); } referenceStagingConfigCache = { expiresAt: now + REFERENCE_STAGING_CACHE_MS, value, }; return value; } function getUrlHost(value: string) { try { return new URL(value).host; } catch { return "invalid"; } } function isStagedReferenceUrl(value: string) { try { return new URL(value).pathname.startsWith(`/${REFERENCE_STAGING_KEY_PREFIX}`); } catch { return false; } } async function downloadReferenceImage(mediaUrl: string) { let lastError: unknown; for (let attempt = 1; attempt <= REFERENCE_DOWNLOAD_ATTEMPTS; attempt += 1) { try { const response = await fetch(mediaUrl, { cache: "no-store", redirect: "follow", headers: { "User-Agent": "yedir-ai-vip98k-staging/1.0" }, signal: AbortSignal.timeout(REFERENCE_DOWNLOAD_TIMEOUT_MS), }); if (!response.ok) { throw new Error(`source returned HTTP ${response.status}`); } const declaredLength = Number(response.headers.get("content-length")); if ( Number.isFinite(declaredLength) && declaredLength > MAX_REFERENCE_DOWNLOAD_BYTES ) { throw new Error("source image exceeds the staging size limit"); } const buffer = Buffer.from(await response.arrayBuffer()); if (buffer.byteLength > MAX_REFERENCE_DOWNLOAD_BYTES) { throw new Error("source image exceeds the staging size limit"); } if ( Number.isFinite(declaredLength) && declaredLength > 0 && buffer.byteLength !== declaredLength ) { throw new Error("source image download was incomplete"); } return { buffer, mimeType: response.headers.get("content-type") || "image/png", }; } catch (error) { lastError = error; logger.warn("reference staging download attempt failed", { attempt, sourceHost: getUrlHost(mediaUrl), message: error instanceof Error ? error.message : String(error), }); if (attempt < REFERENCE_DOWNLOAD_ATTEMPTS) await delay(attempt * 1000); } } throw new Error( `VIP98K reference staging failed: ${ lastError instanceof Error ? lastError.message : "source download failed" }`, ); } async function downloadReferenceImageFromStorage( mediaUrl: string, stagingConfig: ReferenceStagingConfig, ) { const objectKey = getStorageObjectKeyFromPublicUrl( mediaUrl, stagingConfig.sourcePublicUrl, ); if (!objectKey) return null; try { const response = await createS3Client(stagingConfig.storageConfig).send( new GetObjectCommand({ Bucket: stagingConfig.storageConfig.bucket, Key: objectKey, }), { abortSignal: AbortSignal.timeout(REFERENCE_DOWNLOAD_TIMEOUT_MS) }, ); if (!response.Body) throw new Error("source object body is empty"); if ( typeof response.ContentLength === "number" && response.ContentLength > MAX_REFERENCE_DOWNLOAD_BYTES ) { throw new Error("source image exceeds the staging size limit"); } const buffer = Buffer.from(await response.Body.transformToByteArray()); if (buffer.byteLength > MAX_REFERENCE_DOWNLOAD_BYTES) { throw new Error("source image exceeds the staging size limit"); } if ( typeof response.ContentLength === "number" && response.ContentLength > 0 && buffer.byteLength !== response.ContentLength ) { throw new Error("source object download was incomplete"); } return { buffer, mimeType: response.ContentType || "image/png", }; } catch (error) { logger.warn("reference source-storage download failed; retrying public URL", { sourceHost: getUrlHost(mediaUrl), message: error instanceof Error ? error.message : String(error), }); return null; } } async function uploadReferenceImage( buffer: Buffer, stagingConfig: ReferenceStagingConfig, ) { const key = `${REFERENCE_STAGING_KEY_PREFIX}${Date.now()}-${Math.random() .toString(36) .slice(2, 8)}.jpg`; await createS3Client(stagingConfig.storageConfig).send( new PutObjectCommand({ Bucket: stagingConfig.storageConfig.bucket, Key: key, Body: buffer, ContentType: "image/jpeg", CacheControl: "public, max-age=31536000, immutable", ...(stagingConfig.storageConfig.provider === "qiniu-kodo" ? {} : { ACL: "public-read" as const }), }), { abortSignal: AbortSignal.timeout(REFERENCE_DOWNLOAD_TIMEOUT_MS) }, ); return buildS3PublicUrl(stagingConfig.storageConfig, key); } async function normalizeReferenceImage(buffer: Buffer, mimeType: string) { try { const normalized = await sharp(buffer, { failOn: "none" }) .rotate() .resize(1920, 1920, { fit: "inside", withoutEnlargement: true }) .flatten({ background: "#ffffff" }) .jpeg({ quality: 92, chromaSubsampling: "4:4:4", mozjpeg: true }) .toBuffer(); return { buffer: normalized, extension: "jpg", }; } catch (error) { logger.warn("reference image normalization failed; preserving original bytes", { message: error instanceof Error ? error.message : String(error), }); return { buffer, extension: detectExtension(mimeType) }; } } async function stageReferenceImage( mediaUrl: string, stagingConfig: ReferenceStagingConfig, ) { const downloaded = (await downloadReferenceImageFromStorage(mediaUrl, stagingConfig)) || (await downloadReferenceImage(mediaUrl)); const normalized = await normalizeReferenceImage( downloaded.buffer, downloaded.mimeType, ); const uploadedUrl = await uploadReferenceImage(normalized.buffer, stagingConfig); logger.info("reference image staged", { sourceHost: getUrlHost(mediaUrl), targetHost: getUrlHost(uploadedUrl), sourceBytes: downloaded.buffer.byteLength, stagedBytes: normalized.buffer.byteLength, }); return uploadedUrl; } async function ensurePublicMediaUrl(mediaUrl: string, mediaKind: "image" | "video") { const normalized = String(mediaUrl || "").trim(); if (/^https?:\/\//i.test(normalized)) { if (mediaKind !== "image") return normalized; const stagingConfig = await resolveReferenceStagingConfig(); const sourceObjectKey = stagingConfig ? getStorageObjectKeyFromPublicUrl(normalized, stagingConfig.sourcePublicUrl) : null; if ( !stagingConfig || sourceObjectKey?.startsWith(REFERENCE_STAGING_KEY_PREFIX) || !shouldStageVip98kReferenceUrl({ mediaUrl: normalized, sourcePublicUrl: stagingConfig.sourcePublicUrl, }) ) { return normalized; } const existingTask = referenceStagingTasks.get(normalized); if (existingTask) return existingTask; const task = stageReferenceImage(normalized, stagingConfig).finally(() => { referenceStagingTasks.delete(normalized); }); referenceStagingTasks.set(normalized, task); return task; } if (!normalized.startsWith("data:")) return normalized; const parsed = parseDataUrl(normalized); const extension = detectExtension(parsed.mimeType); const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; const stagingConfig = await resolveReferenceStagingConfig(); if (stagingConfig && mediaKind === "image") { const normalizedImage = await normalizeReferenceImage( parsed.buffer, parsed.mimeType, ); return uploadReferenceImage(normalizedImage.buffer, stagingConfig); } const uploaded = await uploadFile( `vip98k-ref-${suffix}.${extension}`, toArrayBuffer(parsed.buffer), ); return uploaded.url; } function extractPromptAndMedia(messages: Array<{ role: string; content?: any; parts?: any[] }>) { const message = messages.filter((item) => item.role === "user").pop(); if (!message) throw new Error("No user message found"); const texts: string[] = []; const imageUrls: string[] = []; const videoUrls: string[] = []; const pushText = (value: unknown) => { if (typeof value === "string" && value.trim()) texts.push(value.trim()); }; const pushMedia = (value: unknown, mediaType?: unknown) => { const url = String( typeof value === "string" ? value : (value as any)?.url || (value as any)?.imageUrl || (value as any)?.fileUrl || (value as any)?.src || "", ).trim(); if (!url || !/^(?:https?:\/\/|data:)/i.test(url)) return; const normalizedMediaType = String(mediaType || "").toLowerCase(); const isVideo = normalizedMediaType.startsWith("video/") || url.startsWith("data:video/") || /\.(?:mp4|mov|webm|m4v)(?:[?#].*)?$/i.test(url); const target = isVideo ? videoUrls : imageUrls; if (!target.includes(url)) target.push(url); }; const visitPart = (part: any) => { if (!part) return; if (typeof part === "string") return pushText(part); if (part.type === "text" || typeof part.text === "string") pushText(part.text); if (part.type === "image_url") pushMedia(part.image_url, "image/jpeg"); if (part.type === "image") pushMedia(part.image || part.url, "image/jpeg"); if (part.type === "file") { pushMedia( part.url || part.file || part.data, part.mediaType || part.contentType || part.mimeType, ); } if (part.inlineData?.data) { pushMedia( `data:${part.inlineData.mimeType || "image/png"};base64,${part.inlineData.data}`, part.inlineData.mimeType, ); } }; if (typeof message.content === "string") pushText(message.content); else if (Array.isArray(message.content)) message.content.forEach(visitPart); if (Array.isArray(message.parts)) message.parts.forEach(visitPart); const prompt = texts.join("\n").trim() || "Generate media"; return { prompt, imageUrls, videoUrls }; } function getConfiguredPollSeconds(config: Vip98kModelRuntimeConfig, key: string, fallback: number) { const metadata = getVip98kMetadata(config); const value = Number(metadata[key]); if (!Number.isFinite(value) || value <= 0) return fallback; return Math.min(Math.max(value, 1), 3600); } async function submitTask(params: { apiKey: string; baseURL?: string | null; path?: string | null; method?: string | null; contentType?: string | null; body: Record; }) { const baseURL = normalizeVip98kBaseUrl(params.baseURL); const path = String(params.path || "/api/external/task/generate").trim(); if (!path.startsWith("/")) throw new Error("VIP98K invocation path is invalid"); const stagedReferences = Array.isArray(params.body.file_urls) ? params.body.file_urls.filter( (value): value is string => typeof value === "string" && isStagedReferenceUrl(value), ) : []; const maxAttempts = MAX_SUBMISSION_ATTEMPTS; let refreshStagedReferences = false; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const requestBody = !refreshStagedReferences || !Array.isArray(params.body.file_urls) ? params.body : { ...params.body, file_urls: params.body.file_urls.map((value) => typeof value === "string" && isStagedReferenceUrl(value) ? addVip98kReferenceRetryToken( value, `${Date.now()}-${attempt}`, ) : value, ), }; const response = await fetch(`${baseURL}${path}`, { method: String(params.method || "POST").toUpperCase(), headers: { Authorization: `Bearer ${params.apiKey}`, "Content-Type": params.contentType || "application/json", }, body: JSON.stringify(requestBody), signal: AbortSignal.timeout(120_000), }); const payload = await parseJsonResponse(response); if (response.ok) { return { payload, taskId: extractTaskId(payload), resultPath: extractResultPath(payload), outputs: collectOutputUrls(payload), }; } const errorMessage = extractErrorMessage(payload); const stagedMediaRetryDelayMs = getVip98kStagedMediaRetryDelay({ status: response.status, message: errorMessage, attempt, maxAttempts, stagedReferenceCount: stagedReferences.length, }); if (stagedMediaRetryDelayMs !== null) { logger.warn("staged reference not ready for provider; retrying submission", { attempt, retryDelayMs: stagedMediaRetryDelayMs, stagedReferenceCount: stagedReferences.length, }); refreshStagedReferences = true; await delay(stagedMediaRetryDelayMs); continue; } const retryAfterSeconds = Number(response.headers.get("retry-after")); const reconciliationRetryDelayMs = getVip98kSubmissionRetryDelay({ status: response.status, message: errorMessage, attempt, maxAttempts, retryAfterMs: Number.isFinite(retryAfterSeconds) ? retryAfterSeconds * 1000 : null, }); if (reconciliationRetryDelayMs !== null) { logger.warn("provider is reconciling task submission; retrying", { attempt, retryDelayMs: reconciliationRetryDelayMs, }); refreshStagedReferences = false; await delay(reconciliationRetryDelayMs); continue; } throw new Error(`${errorMessage} (HTTP ${response.status})`); } throw new Error("VIP98K request failed after media readiness retries"); } export type Vip98kExternalTaskSubmission = { taskId: string; resultPath: string; outputUrls: string[]; payload: unknown; }; export type Vip98kExternalTaskStatus = { taskId: string; status: string; done: boolean; failed: boolean; outputUrls: string[]; payload: unknown; }; export async function submitVip98kExternalTask(params: { apiKey: string; baseURL?: string | null; modelId: string; kind: string; prompt?: string; imageUrls?: string[]; videoUrls?: string[]; videoParams?: Record; imageParams?: Record; modelConfig?: Vip98kModelRuntimeConfig; }): Promise { const [imageUrls, videoUrls] = await Promise.all([ Promise.all( (params.imageUrls || []).map((url) => ensurePublicMediaUrl(url, "image")), ), Promise.all( (params.videoUrls || []).map((url) => ensurePublicMediaUrl(url, "video")), ), ]); const taskParams: Vip98kTaskParams = { apiKey: params.apiKey, baseURL: params.baseURL, modelId: params.modelId, kind: params.kind, prompt: String(params.prompt || "Process media").trim() || "Process media", imageUrls, videoUrls, videoParams: params.videoParams, imageParams: params.imageParams, modelConfig: params.modelConfig || {}, }; const configuredInvocations = getVip98kInvocations(taskParams.modelConfig); const invocation = selectVip98kTaskInvocation({ modelId: taskParams.modelId, configuredInvocation: configuredInvocations[0] || null, configuredInvocations, imageCount: imageUrls.length, }); const { body, imageBindingKeys, outputConstraintFallbacks } = buildTaskBody( taskParams, invocation, ); if (imageUrls.length > 0 && imageBindingKeys.length === 0) { throw new Error( `VIP98K model does not publish a reference-image request contract: ${taskParams.modelId}`, ); } logger.info("task submit audit", { modelId: taskParams.modelId, kind: taskParams.kind, invocationKind: invocation?.kind || "task", invocationPath: invocation?.path || "/api/external/task", imageCount: imageUrls.length, videoCount: videoUrls.length, imageBindingKeys, outputConstraintFallbacks, bodyKeys: Object.keys(body).sort(), }); const submitted = await submitTask({ apiKey: params.apiKey, baseURL: params.baseURL, path: invocation?.path, method: invocation?.method, contentType: invocation?.contentType, body, }); const resultPath = submitted.resultPath || invocation?.resultPath || ""; return { taskId: submitted.taskId, resultPath: submitted.taskId ? resultPath.replaceAll("{task_id}", encodeURIComponent(submitted.taskId)) : resultPath, outputUrls: submitted.outputs, payload: submitted.payload, }; } export async function getVip98kExternalTaskStatus(params: { apiKey: string; baseURL?: string | null; taskId: string; resultPath?: string | null; }): Promise { const baseURL = normalizeVip98kBaseUrl(params.baseURL); const resultUrl = params.resultPath ? params.resultPath.startsWith("http") ? params.resultPath : `${baseURL}${params.resultPath.startsWith("/") ? "" : "/"}${params.resultPath}` : `${baseURL}/api/external/task/result?taskId=${encodeURIComponent(params.taskId)}`; const { payload, status, outputUrls } = await fetchTaskResult({ apiKey: params.apiKey, baseURL, taskId: params.taskId, resultUrl, }); const failed = ["failed", "error", "timeout", "canceled", "cancelled"].includes(status); const done = ["succeeded", "success", "completed", "done", "finished"].includes(status) || (extractDone(payload) && outputUrls.length > 0); return { taskId: params.taskId, status, done, failed, outputUrls, payload, }; } async function waitForTask(params: { apiKey: string; baseURL?: string | null; taskId: string; modelId: string; resultPath?: string; modelConfig: Vip98kModelRuntimeConfig; }) { const baseURL = normalizeVip98kBaseUrl(params.baseURL); const intervalMs = getConfiguredPollSeconds( params.modelConfig, "pollIntervalSeconds", DEFAULT_POLL_INTERVAL_SECONDS, ) * 1000; const timeoutMs = getConfiguredPollSeconds(params.modelConfig, "maxPollSeconds", DEFAULT_MAX_POLL_SECONDS) * 1000; const startedAt = Date.now(); while (Date.now() - startedAt < timeoutMs) { await delay(intervalMs); const resultUrl = params.resultPath ? params.resultPath.startsWith("http") ? params.resultPath : `${baseURL}${params.resultPath.startsWith("/") ? "" : "/"}${params.resultPath}` : `${baseURL}/api/external/task/result?taskId=${encodeURIComponent(params.taskId)}`; const { payload, status, outputUrls: outputs } = await fetchTaskResult({ apiKey: params.apiKey, baseURL, taskId: params.taskId, resultUrl, }); const done = extractDone(payload); if ( ["succeeded", "success", "completed", "done", "finished"].includes(status) || (done && outputs.length > 0) ) { return { payload, outputs }; } if (["failed", "error", "timeout", "canceled", "cancelled"].includes(status)) { const providerMessage = extractErrorMessage(payload, ""); logger.error("task failed", { taskId: params.taskId, modelId: params.modelId, status, providerMessage: providerMessage || null, ...extractPayloadShape(payload), }); const detail = providerMessage ? `: ${providerMessage}` : ""; throw new Error( `VIP98K task failed${detail} (task_id: ${params.taskId}, model: ${params.modelId})`, ); } } throw new Error("VIP98K task timed out; it may still be running"); } function normalizeRatio(value: unknown) { const normalized = String(value || "").trim(); return normalized && normalized !== "auto" ? normalized : undefined; } function normalizeDuration(value: unknown) { const numeric = Number(value); if (!Number.isFinite(numeric) || numeric <= 0) return undefined; return Math.round(numeric); } function buildTaskBody(params: Vip98kTaskParams, invocation: Vip98kInvocation | null) { const videoParams = params.videoParams || {}; const imageParams = params.imageParams || {}; const promptTitle = params.prompt.split(/\s+/).slice(0, 8).join(" ").slice(0, 80); const body: Record = { ...(invocation?.requestExample || {}) }; if ("model" in body || !invocation) body.model = params.modelId; if ("displayName" in body || !invocation) { body.displayName = promptTitle || params.modelId; } if ("prompt" in body || !invocation) body.prompt = params.prompt; const aspectRatio = normalizeRatio(videoParams.ratio) || normalizeRatio(videoParams.aspectRatio) || normalizeRatio(imageParams.aspectRatio); const duration = normalizeDuration(videoParams.duration); const resolution = String(videoParams.resolution || imageParams.imageSize || "").trim(); const negativePrompt = String(videoParams.negativePrompt || imageParams.negativePrompt || "").trim(); const outputConstraintFallbacks: string[] = []; let hasNativeAspectRatio = false; let hasNativeResolution = false; if (aspectRatio) { if ("ratio" in body || !invocation) { body.ratio = aspectRatio; hasNativeAspectRatio = true; } if ("aspectRatio" in body || !invocation) { body.aspectRatio = aspectRatio; hasNativeAspectRatio = true; } if ("aspect_ratio" in body || !invocation) { body.aspect_ratio = aspectRatio; hasNativeAspectRatio = true; } } if (duration && ("duration" in body || !invocation)) body.duration = duration; if (duration && "duration_seconds" in body) body.duration_seconds = duration; if (resolution) { if ("resolution" in body || !invocation) { body.resolution = resolution; hasNativeResolution = true; } if ("imageSize" in body || !invocation) { body.imageSize = resolution; hasNativeResolution = true; } if ("image_size" in body || !invocation) { body.image_size = resolution; hasNativeResolution = true; } if ("size" in body && typeof body.size === "string") { body.size = resolution; hasNativeResolution = true; } } if (negativePrompt) { if ("negativePrompt" in body || !invocation) body.negativePrompt = negativePrompt; if ("negative_prompt" in body || !invocation) body.negative_prompt = negativePrompt; } const imageToolParamAliases = [ ["scaleFactor", "scale_factor"], ["xScale", "x_scale"], ["yScale", "y_scale"], ] as const; for (const [sourceKey, targetKey] of imageToolParamAliases) { const value = imageParams[sourceKey]; if (value !== undefined && targetKey in body) body[targetKey] = value; } if (typeof videoParams.generateAudio === "boolean") { if ("generateAudio" in body || !invocation) body.generateAudio = videoParams.generateAudio; if ("generate_audio" in body || !invocation) body.generate_audio = videoParams.generateAudio; } const imageBindingKeys = bindVip98kImages(body, params.imageUrls); if (params.imageUrls.length > 0 && !invocation && imageBindingKeys.length === 0) { body.images = params.imageUrls; imageBindingKeys.push("images"); } const promptConstraints: string[] = []; if (aspectRatio && !hasNativeAspectRatio) { promptConstraints.push(`aspect ratio ${aspectRatio}`); outputConstraintFallbacks.push("aspectRatio:prompt"); } if (resolution && !hasNativeResolution) { promptConstraints.push(`resolution ${resolution}`); outputConstraintFallbacks.push("imageSize:prompt"); } if (promptConstraints.length > 0 && typeof body.prompt === "string") { body.prompt = `${body.prompt}\n\n[Output constraints] ${promptConstraints.join( "; ", )}.`; } if (params.videoUrls.length > 0) { if ("videos" in body || !invocation) body.videos = params.videoUrls; if ("videoUrls" in body || !invocation) body.videoUrls = params.videoUrls; if ("referenceVideos" in body || !invocation) body.referenceVideos = params.videoUrls; if ("video" in body) body.video = params.videoUrls[0]; if ("video_url" in body) body.video_url = params.videoUrls[0]; } if (params.kind === "world" && ("mode" in body || !invocation)) { body.mode = "world-generation"; } else if (params.kind === "video" && ("mode" in body || !invocation)) { body.mode = params.imageUrls.length > 0 ? "image-to-video" : "text-to-video"; } return { body, imageBindingKeys, outputConstraintFallbacks }; } function formatImageMarkdown(images: UploadedImage[], imageSize: string) { return images .map((image) => `![thumb|||${image.original}|||${imageSize}](${image.thumbnail})`) .join("\n\n"); } function formatVideoMarkdown(videoUrl: string) { return `\n\n[下载视频](${videoUrl})`; } function formatWorldMarkdown(urls: string[]) { if (urls.length === 0) return "3D 世界生成成功,但未返回可访问的结果地址"; return urls.map((url, index) => `[查看 3D 世界 ${index + 1}](${url})`).join("\n\n"); } export type Vip98kExternalTaskResult = { taskId: string; outputUrls: string[]; payload: unknown; }; export async function executeVip98kExternalTask(params: { apiKey: string; baseURL?: string | null; modelId: string; kind: string; prompt?: string; imageUrls?: string[]; videoUrls?: string[]; videoParams?: Record; imageParams?: Record; modelConfig?: Vip98kModelRuntimeConfig; }): Promise { const submitted = await submitVip98kExternalTask({ ...params, }); if (submitted.outputUrls.length > 0) { return { taskId: submitted.taskId, outputUrls: submitted.outputUrls, payload: submitted.payload, }; } if (!submitted.taskId) { throw new Error("VIP98K task submission returned neither a task ID nor output URLs"); } const completed = await waitForTask({ apiKey: params.apiKey, baseURL: params.baseURL, taskId: submitted.taskId, modelId: params.modelId, resultPath: submitted.resultPath, modelConfig: params.modelConfig || {}, }); return { taskId: submitted.taskId, outputUrls: completed.outputs, payload: completed.payload, }; } async function runTask(params: Vip98kTaskParams) { const completed = await executeVip98kExternalTask({ apiKey: params.apiKey, baseURL: params.baseURL, modelId: params.modelId, kind: params.kind, prompt: params.prompt, imageUrls: params.imageUrls, videoUrls: params.videoUrls, videoParams: params.videoParams, imageParams: params.imageParams, modelConfig: params.modelConfig, }); const outputUrls = completed.outputUrls; if (params.kind === "video") { const videoUrl = outputUrls.find((url) => /\.(mp4|webm|mov|m4v)(?:[?#].*)?$/i.test(url)) || outputUrls[0]; if (!videoUrl) throw new Error("VIP98K task completed but returned no video URL"); return formatVideoMarkdown(videoUrl); } if (params.kind === "image" || params.kind === "image-tool") { if (outputUrls.length === 0) { throw new Error("VIP98K task completed but returned no image URL"); } const imageSize = String(params.imageParams?.imageSize || "2K").trim() || "2K"; const archived = await Promise.all(outputUrls.slice(0, 1).map(archiveOutputImage)); return formatImageMarkdown(archived, imageSize); } if (params.kind === "world" || params.kind === "model-3d") { return formatWorldMarkdown(outputUrls); } return outputUrls.length ? outputUrls.join("\n") : "VIP98K task completed"; } export function createVip98k({ apiKey, baseURL }: Vip98kConfig) { const rootBaseURL = normalizeVip98kBaseUrl(baseURL); const openAIBaseURL = normalizeOpenAIBaseUrl(rootBaseURL); const openAITextProvider = createOpenAI({ apiKey, baseURL: openAIBaseURL, }); return { provider: "vip98k", chat: (modelId: string, defaultModelConfig?: unknown): any => { const modelConfig = parseRuntimeConfig(defaultModelConfig); const kind = inferKind(modelId, modelConfig); if (kind === "chat") { return openAITextProvider.chat(modelId) as any; } return { provider: "vip98k", doGenerate: async (options: { messages: Array<{ role: string; content?: any; parts?: any[] }>; imageParams?: Record; videoParams?: Record; imageGenConfig?: unknown; overrideModelId?: string; }) => { const runtimeConfig = options.imageGenConfig !== undefined ? parseRuntimeConfig({ capabilities: modelConfig.capabilities, imageGenConfig: options.imageGenConfig, }) : modelConfig; const activeModelId = String(options.overrideModelId || modelId || "").trim(); if (!activeModelId) throw new Error("VIP98K model ID is not configured"); const activeKind = inferKind(activeModelId, runtimeConfig); const { prompt, imageUrls, videoUrls } = extractPromptAndMedia( options.messages, ); const [publicImageUrls, publicVideoUrls] = await Promise.all([ Promise.all( imageUrls.map((url) => ensurePublicMediaUrl(url, "image")), ), Promise.all( videoUrls.map((url) => ensurePublicMediaUrl(url, "video")), ), ]); const text = await runTask({ apiKey, baseURL: rootBaseURL, modelId: activeModelId, kind: activeKind, prompt, imageUrls: publicImageUrls, videoUrls: publicVideoUrls, videoParams: options.videoParams, imageParams: options.imageParams, modelConfig: runtimeConfig, }); return { text, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: text.length, }, }; }, doStream: async function* (options: { messages: Array<{ role: string; content?: any; parts?: any[] }>; imageParams?: Record; videoParams?: Record; imageGenConfig?: unknown; overrideModelId?: string; }) { const result = await this.doGenerate(options); yield { type: "text-delta", textDelta: result.text }; }, }; }, }; }