diff --git a/lib/ai/providers/vip98k.ts b/lib/ai/providers/vip98k.ts index ba6b5c430..6ac9059ff 100644 --- a/lib/ai/providers/vip98k.ts +++ b/lib/ai/providers/vip98k.ts @@ -1,7 +1,13 @@ import "server-only"; +import { GetObjectCommand } from "@aws-sdk/client-s3"; import { createOpenAI } from "@ai-sdk/openai"; import sharp from "sharp"; +import { + getDefaultStorageConfig, + getStorageConfigById, + getSystemSetting, +} from "@/lib/db/queries"; import { bindVip98kImages, selectVip98kTaskInvocation, @@ -9,9 +15,19 @@ import { } from "@/lib/ai/vip98k-task-invocation"; import { VIP98K_BASE_URL, normalizeVip98kBaseUrl } from "@/lib/ai/vip98k-models"; import { shouldResolveVip98kResultDelivery } from "@/lib/ai/vip98k-result-delivery"; +import { + getStorageObjectKeyFromPublicUrl, + shouldStageVip98kReferenceUrl, +} from "@/lib/ai/vip98k-reference-staging"; import { createLogger } from "@/lib/observability/logger"; import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; -import { uploadFile } from "@/lib/storage/service"; +import { + createS3Client, + type S3UploadConfig, + toDbS3UploadConfig, + uploadFile, + uploadFileWithS3Config, +} from "@/lib/storage/service"; const logger = createLogger("vip98k-provider"); @@ -48,6 +64,23 @@ type UploadedImage = { const DOWNLOAD_TIMEOUT_MS = 120_000; const DEFAULT_POLL_INTERVAL_SECONDS = 8; const DEFAULT_MAX_POLL_SECONDS = 600; +const REFERENCE_DOWNLOAD_ATTEMPTS = 3; +const REFERENCE_DOWNLOAD_TIMEOUT_MS = 120_000; +const MAX_REFERENCE_DOWNLOAD_BYTES = 40 * 1024 * 1024; +const REFERENCE_STAGING_CACHE_MS = 60_000; +const STORAGE_FAILOVER_CONFIG_KEY = "storage_failover_config"; + +type ReferenceStagingConfig = { + sourcePublicUrl: string; + sourceDownloadConfig?: S3UploadConfig; + targetPublicUrl?: string; + uploadConfig: S3UploadConfig; +}; + +let referenceStagingConfigCache: + | { expiresAt: number; value: ReferenceStagingConfig | null } + | undefined; +const referenceStagingTasks = new Map>(); function toRecord(value: unknown): Record | null { if (!value || typeof value !== "object" || Array.isArray(value)) return null; @@ -435,13 +468,269 @@ async function archiveOutputImage(imageUrl: string): Promise { ); } -async function ensurePublicMediaUrl(mediaUrl: string) { +function readStorageFailoverConfig(value: unknown) { + let parsed = value; + if (typeof parsed === "string") { + try { + parsed = JSON.parse(parsed); + } catch { + parsed = null; + } + } + const record = toRecord(parsed); + return { + enabled: record?.enabled === true, + backupStorageConfigId: String(record?.backupStorageConfigId || "").trim(), + }; +} + +async function resolveReferenceStagingConfig() { + const now = Date.now(); + if (referenceStagingConfigCache?.expiresAt && referenceStagingConfigCache.expiresAt > now) { + return referenceStagingConfigCache.value; + } + + let value: ReferenceStagingConfig | null = null; + try { + const [sourceStorage, failoverSetting] = await Promise.all([ + getDefaultStorageConfig(), + getSystemSetting({ key: STORAGE_FAILOVER_CONFIG_KEY }), + ]); + const failover = readStorageFailoverConfig(failoverSetting?.value); + if ( + sourceStorage?.publicUrl && + failover.enabled && + failover.backupStorageConfigId && + failover.backupStorageConfigId !== sourceStorage.id + ) { + const targetStorage = await getStorageConfigById({ + id: failover.backupStorageConfigId, + }); + const uploadConfig = toDbS3UploadConfig(targetStorage); + if (uploadConfig) { + value = { + sourcePublicUrl: sourceStorage.publicUrl, + sourceDownloadConfig: toDbS3UploadConfig(sourceStorage) || undefined, + targetPublicUrl: targetStorage?.publicUrl || uploadConfig.publicUrl, + uploadConfig, + }; + } + } + } catch (error) { + logger.warn("reference staging config lookup failed", { + message: error instanceof Error ? error.message : String(error), + }); + } + + referenceStagingConfigCache = { + expiresAt: now + REFERENCE_STAGING_CACHE_MS, + value, + }; + return value; +} + +function getUrlHost(value: string) { + try { + return new URL(value).host; + } catch { + return "invalid"; + } +} + +async function downloadReferenceImage(mediaUrl: string) { + let lastError: unknown; + for (let attempt = 1; attempt <= REFERENCE_DOWNLOAD_ATTEMPTS; attempt += 1) { + try { + const response = await fetch(mediaUrl, { + cache: "no-store", + redirect: "follow", + headers: { "User-Agent": "yedir-ai-vip98k-staging/1.0" }, + signal: AbortSignal.timeout(REFERENCE_DOWNLOAD_TIMEOUT_MS), + }); + if (!response.ok) { + throw new Error(`source returned HTTP ${response.status}`); + } + + const declaredLength = Number(response.headers.get("content-length")); + if ( + Number.isFinite(declaredLength) && + declaredLength > MAX_REFERENCE_DOWNLOAD_BYTES + ) { + throw new Error("source image exceeds the staging size limit"); + } + + const buffer = Buffer.from(await response.arrayBuffer()); + if (buffer.byteLength > MAX_REFERENCE_DOWNLOAD_BYTES) { + throw new Error("source image exceeds the staging size limit"); + } + if ( + Number.isFinite(declaredLength) && + declaredLength > 0 && + buffer.byteLength !== declaredLength + ) { + throw new Error("source image download was incomplete"); + } + return { + buffer, + mimeType: response.headers.get("content-type") || "image/png", + }; + } catch (error) { + lastError = error; + logger.warn("reference staging download attempt failed", { + attempt, + sourceHost: getUrlHost(mediaUrl), + message: error instanceof Error ? error.message : String(error), + }); + if (attempt < REFERENCE_DOWNLOAD_ATTEMPTS) await delay(attempt * 1000); + } + } + + throw new Error( + `VIP98K reference staging failed: ${ + lastError instanceof Error ? lastError.message : "source download failed" + }`, + ); +} + +async function downloadReferenceImageFromStorage( + mediaUrl: string, + stagingConfig: ReferenceStagingConfig, +) { + if (!stagingConfig.sourceDownloadConfig) return null; + const objectKey = getStorageObjectKeyFromPublicUrl( + mediaUrl, + stagingConfig.sourcePublicUrl, + ); + if (!objectKey) return null; + + try { + const response = await createS3Client(stagingConfig.sourceDownloadConfig).send( + new GetObjectCommand({ + Bucket: stagingConfig.sourceDownloadConfig.bucket, + Key: objectKey, + }), + { abortSignal: AbortSignal.timeout(REFERENCE_DOWNLOAD_TIMEOUT_MS) }, + ); + if (!response.Body) throw new Error("source object body is empty"); + if ( + typeof response.ContentLength === "number" && + response.ContentLength > MAX_REFERENCE_DOWNLOAD_BYTES + ) { + throw new Error("source image exceeds the staging size limit"); + } + const buffer = Buffer.from(await response.Body.transformToByteArray()); + if (buffer.byteLength > MAX_REFERENCE_DOWNLOAD_BYTES) { + throw new Error("source image exceeds the staging size limit"); + } + if ( + typeof response.ContentLength === "number" && + response.ContentLength > 0 && + buffer.byteLength !== response.ContentLength + ) { + throw new Error("source object download was incomplete"); + } + return { + buffer, + mimeType: response.ContentType || "image/png", + }; + } catch (error) { + logger.warn("reference source-storage download failed; retrying public URL", { + sourceHost: getUrlHost(mediaUrl), + message: error instanceof Error ? error.message : String(error), + }); + return null; + } +} + +async function normalizeReferenceImage(buffer: Buffer, mimeType: string) { + try { + const metadata = await sharp(buffer, { failOn: "none" }).metadata(); + const pipeline = sharp(buffer, { failOn: "none" }) + .rotate() + .resize(2048, 2048, { fit: "inside", withoutEnlargement: true }); + if (metadata.hasAlpha || /png/i.test(mimeType)) { + return { + buffer: await pipeline.png({ compressionLevel: 6 }).toBuffer(), + extension: "png", + }; + } + return { + buffer: await pipeline + .jpeg({ quality: 95, chromaSubsampling: "4:4:4", mozjpeg: true }) + .toBuffer(), + extension: "jpg", + }; + } catch (error) { + logger.warn("reference image normalization failed; preserving original bytes", { + message: error instanceof Error ? error.message : String(error), + }); + return { buffer, extension: detectExtension(mimeType) }; + } +} + +async function stageReferenceImage( + mediaUrl: string, + stagingConfig: ReferenceStagingConfig, +) { + const downloaded = + (await downloadReferenceImageFromStorage(mediaUrl, stagingConfig)) || + (await downloadReferenceImage(mediaUrl)); + const normalized = await normalizeReferenceImage( + downloaded.buffer, + downloaded.mimeType, + ); + const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + const uploaded = await uploadFileWithS3Config( + `vip98k-reference-${suffix}.${normalized.extension}`, + toArrayBuffer(normalized.buffer), + stagingConfig.uploadConfig, + ); + logger.info("reference image staged", { + sourceHost: getUrlHost(mediaUrl), + targetHost: getUrlHost(uploaded.url), + sourceBytes: downloaded.buffer.byteLength, + stagedBytes: normalized.buffer.byteLength, + }); + return uploaded.url; +} + +async function ensurePublicMediaUrl(mediaUrl: string, mediaKind: "image" | "video") { const normalized = String(mediaUrl || "").trim(); - if (/^https?:\/\//i.test(normalized)) return normalized; + if (/^https?:\/\//i.test(normalized)) { + if (mediaKind !== "image") return normalized; + const stagingConfig = await resolveReferenceStagingConfig(); + if ( + !stagingConfig || + !shouldStageVip98kReferenceUrl({ + mediaUrl: normalized, + sourcePublicUrl: stagingConfig.sourcePublicUrl, + targetPublicUrl: stagingConfig.targetPublicUrl, + }) + ) { + return normalized; + } + + const existingTask = referenceStagingTasks.get(normalized); + if (existingTask) return existingTask; + const task = stageReferenceImage(normalized, stagingConfig).finally(() => { + referenceStagingTasks.delete(normalized); + }); + referenceStagingTasks.set(normalized, task); + return task; + } if (!normalized.startsWith("data:")) return normalized; const parsed = parseDataUrl(normalized); const extension = detectExtension(parsed.mimeType); const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + const stagingConfig = await resolveReferenceStagingConfig(); + if (stagingConfig) { + const uploaded = await uploadFileWithS3Config( + `vip98k-ref-${suffix}.${extension}`, + toArrayBuffer(parsed.buffer), + stagingConfig.uploadConfig, + ); + return uploaded.url; + } const uploaded = await uploadFile( `vip98k-ref-${suffix}.${extension}`, toArrayBuffer(parsed.buffer), @@ -577,8 +866,12 @@ export async function submitVip98kExternalTask(params: { modelConfig?: Vip98kModelRuntimeConfig; }): Promise { const [imageUrls, videoUrls] = await Promise.all([ - Promise.all((params.imageUrls || []).map(ensurePublicMediaUrl)), - Promise.all((params.videoUrls || []).map(ensurePublicMediaUrl)), + Promise.all( + (params.imageUrls || []).map((url) => ensurePublicMediaUrl(url, "image")), + ), + Promise.all( + (params.videoUrls || []).map((url) => ensurePublicMediaUrl(url, "video")), + ), ]); const taskParams: Vip98kTaskParams = { apiKey: params.apiKey, @@ -992,8 +1285,12 @@ export function createVip98k({ apiKey, baseURL }: Vip98kConfig) { const activeKind = inferKind(activeModelId, runtimeConfig); const { prompt, imageUrls, videoUrls } = extractPromptAndMedia(options.messages); const [publicImageUrls, publicVideoUrls] = await Promise.all([ - Promise.all(imageUrls.map(ensurePublicMediaUrl)), - Promise.all(videoUrls.map(ensurePublicMediaUrl)), + Promise.all( + imageUrls.map((url) => ensurePublicMediaUrl(url, "image")), + ), + Promise.all( + videoUrls.map((url) => ensurePublicMediaUrl(url, "video")), + ), ]); const text = await runTask({ apiKey, diff --git a/lib/ai/runtime/model-health.ts b/lib/ai/runtime/model-health.ts index 6205c6a35..0eb4a02c1 100644 --- a/lib/ai/runtime/model-health.ts +++ b/lib/ai/runtime/model-health.ts @@ -7,6 +7,10 @@ import { aiModelHealthEvent, type AiModel, } from "@/lib/db/schema"; +import { + isMediaInputFailure, + isNonProviderRequestStatus, +} from "@/lib/ai/runtime/request-failure-classification"; export type AiModelHealthEventType = | "success" @@ -173,6 +177,15 @@ export function classifyAiModelHealthFailure(error: unknown): { }; } + if (isMediaInputFailure(normalized)) { + return { + eventType: "error", + severity: "warning", + counted: false, + reason: "request_media_rejected", + }; + } + if ( normalized.includes("fetch failed") || normalized.includes("econnreset") || @@ -226,6 +239,15 @@ export function classifyAiModelHealthFailure(error: unknown): { }; } + if (isNonProviderRequestStatus(httpStatus, normalized)) { + return { + eventType: "error", + severity: "warning", + counted: false, + reason: "request_input_rejected", + }; + } + if ( isOperationalAbort(error, normalized) || normalized.includes("timeout") || diff --git a/lib/ai/runtime/request-failure-classification.ts b/lib/ai/runtime/request-failure-classification.ts new file mode 100644 index 000000000..efabf9747 --- /dev/null +++ b/lib/ai/runtime/request-failure-classification.ts @@ -0,0 +1,24 @@ +const NON_PROVIDER_REQUEST_STATUSES = new Set([400, 413, 415, 422]); + +export function isMediaInputFailure(message: string) { + const normalized = String(message || "").trim().toLowerCase(); + return ( + normalized.includes("media request is invalid") || + normalized.includes("invalid media request") || + normalized.includes("invalid input media") || + normalized.includes("unsupported media type") || + normalized.includes("vip98k reference staging failed") + ); +} + +export function isNonProviderRequestStatus( + httpStatus: number | null, + message: string, +) { + if (httpStatus !== null && NON_PROVIDER_REQUEST_STATUSES.has(httpStatus)) { + return true; + } + + const normalized = String(message || "").trim(); + return /(?:^|\D)(?:400|413|415|422)(?:\D|$)/.test(normalized); +} diff --git a/lib/ai/vip98k-reference-staging.ts b/lib/ai/vip98k-reference-staging.ts new file mode 100644 index 000000000..ee7d5e5a5 --- /dev/null +++ b/lib/ai/vip98k-reference-staging.ts @@ -0,0 +1,51 @@ +function parseUrl(value: string) { + try { + return new URL(String(value || "").trim()); + } catch { + return null; + } +} + +export function isUrlWithinPublicBase(mediaUrl: string, publicBaseUrl: string) { + const media = parseUrl(mediaUrl); + const base = parseUrl(publicBaseUrl); + if (!media || !base || media.host.toLowerCase() !== base.host.toLowerCase()) { + return false; + } + + const basePath = base.pathname.replace(/\/+$/, ""); + if (!basePath) return true; + return media.pathname === basePath || media.pathname.startsWith(`${basePath}/`); +} + +export function shouldStageVip98kReferenceUrl(params: { + mediaUrl: string; + sourcePublicUrl: string; + targetPublicUrl?: string; +}) { + if ( + params.targetPublicUrl && + isUrlWithinPublicBase(params.mediaUrl, params.targetPublicUrl) + ) { + return false; + } + return isUrlWithinPublicBase(params.mediaUrl, params.sourcePublicUrl); +} + +export function getStorageObjectKeyFromPublicUrl( + mediaUrl: string, + publicBaseUrl: string, +) { + if (!isUrlWithinPublicBase(mediaUrl, publicBaseUrl)) return null; + const media = parseUrl(mediaUrl); + const base = parseUrl(publicBaseUrl); + if (!media || !base) return null; + const basePath = base.pathname.replace(/\/+$/, ""); + const encodedKey = media.pathname.slice(basePath.length).replace(/^\/+/, ""); + if (!encodedKey) return null; + try { + return decodeURIComponent(encodedKey); + } catch { + return encodedKey; + } +} diff --git a/tests/routes/model-health-classification.test.ts b/tests/routes/model-health-classification.test.ts new file mode 100644 index 000000000..6435a7073 --- /dev/null +++ b/tests/routes/model-health-classification.test.ts @@ -0,0 +1,24 @@ +import { expect, test } from "@playwright/test"; +import { + isMediaInputFailure, + isNonProviderRequestStatus, +} from "@/lib/ai/runtime/request-failure-classification"; + +test.describe("AI model health request classification", () => { + test("does not treat rejected media as a provider outage", () => { + expect( + isMediaInputFailure("The media request is invalid. (HTTP 400)"), + ).toBe(true); + expect( + isMediaInputFailure("VIP98K reference staging failed: fetch failed"), + ).toBe(true); + }); + + test("recognizes request-specific HTTP failures", () => { + expect(isNonProviderRequestStatus(413, "payload too large")).toBe(true); + expect(isNonProviderRequestStatus(null, "request failed (HTTP 422)")).toBe( + true, + ); + expect(isNonProviderRequestStatus(503, "service unavailable")).toBe(false); + }); +}); diff --git a/tests/routes/vip98k-reference-routing.test.ts b/tests/routes/vip98k-reference-routing.test.ts index 274f76c8a..289cc289b 100644 --- a/tests/routes/vip98k-reference-routing.test.ts +++ b/tests/routes/vip98k-reference-routing.test.ts @@ -9,6 +9,8 @@ import { buildVip98kModelPresets, isVip98kImageModelVisible, } from "@/lib/ai/vip98k-models"; +import { shouldStageVip98kReferenceUrl } from "@/lib/ai/vip98k-reference-staging"; +import { getStorageObjectKeyFromPublicUrl } from "@/lib/ai/vip98k-reference-staging"; import { shouldResolveVip98kResultDelivery } from "@/lib/ai/vip98k-result-delivery"; const textToImageInvocation: Vip98kTaskInvocation = { @@ -23,6 +25,36 @@ const textToImageInvocation: Vip98kTaskInvocation = { }; test.describe("VIP98K reference-image routing", () => { + test("stages references from the configured default storage into backup storage", () => { + expect( + shouldStageVip98kReferenceUrl({ + mediaUrl: "https://default.example/uploads/reference.jpg", + sourcePublicUrl: "https://default.example", + targetPublicUrl: "https://backup.example/project", + }), + ).toBe(true); + expect( + shouldStageVip98kReferenceUrl({ + mediaUrl: "https://backup.example/project/uploads/reference.jpg", + sourcePublicUrl: "https://default.example", + targetPublicUrl: "https://backup.example/project", + }), + ).toBe(false); + expect( + shouldStageVip98kReferenceUrl({ + mediaUrl: "https://external.example/reference.jpg", + sourcePublicUrl: "https://default.example", + targetPublicUrl: "https://backup.example/project", + }), + ).toBe(false); + expect( + getStorageObjectKeyFromPublicUrl( + "https://default.example/assets/folder%20name/reference.jpg?token=secret", + "https://default.example/assets", + ), + ).toBe("folder name/reference.jpg"); + }); + test("shows the newly published Nano Banana Pro image editor", () => { expect( isVip98kImageModelVisible({