diff --git a/lib/storage/image-archive.ts b/lib/storage/image-archive.ts index 26b31e9f0..a3235cb90 100644 --- a/lib/storage/image-archive.ts +++ b/lib/storage/image-archive.ts @@ -12,7 +12,7 @@ import { imageArchiveSourceKey, } from "@/lib/storage/image-archive-db"; import { uploadFile } from "@/lib/storage/service"; -import { archiveImageThroughQiniu } from "./qiniu-image-archive"; +import { archiveImageThroughQiniu, QiniuArchiveSourceMissingError } from "./qiniu-image-archive"; const logger = createLogger("image-archive"); @@ -195,6 +195,7 @@ export async function downloadTemporaryImage(params: { sourceUrl: string; retrie export async function archiveTemporaryImage(sourceUrl: string, archiveId?: string) { if (archiveId) { const direct = await archiveImageThroughQiniu(sourceUrl, archiveId).catch(error => { + if (error instanceof QiniuArchiveSourceMissingError) throw error; logger.warn("cloud_archive_fallback", error, { archiveId }); return null; }); diff --git a/lib/storage/qiniu-image-archive.ts b/lib/storage/qiniu-image-archive.ts index c229dda66..74ac747cf 100644 --- a/lib/storage/qiniu-image-archive.ts +++ b/lib/storage/qiniu-image-archive.ts @@ -1,7 +1,8 @@ import "server-only"; -import { GetObjectCommand } from "@aws-sdk/client-s3"; +import { GetObjectCommand, HeadObjectCommand } from "@aws-sdk/client-s3"; import { createHmac } from "node:crypto"; +import { createLogger } from "@/lib/observability/logger"; import { buildS3PublicUrl, createS3Client, @@ -11,6 +12,23 @@ import { const QINIU_EAST_CHINA_IO = "iovip-z0.qiniuio.com"; const PNG_HEADER_LIMIT = 64 * 1024; const PNG_SIGNATURE = Buffer.from("89504e470d0a1a0a", "hex"); +const QINIU_FETCH_TIMEOUT_MS = 40_000; +const RELAY_RETRY_TIMEOUT_MS = 25_000; +const logger = createLogger("qiniu-image-archive"); + +type QiniuUploadTarget = NonNullable>>; + +class QiniuFetchError extends Error { + constructor(readonly status: number, readonly requestId: string | null) { + super(`Qiniu archive fetch returned HTTP ${status}`); + } +} + +export class QiniuArchiveSourceMissingError extends Error { + constructor(status: number) { + super(`98K original image is no longer available (HTTP ${status})`); + } +} export type ReadyImageDownload = { status: "ready"; @@ -99,9 +117,84 @@ async function archiveThroughQiniu(sourceUrl: string, archiveId: string, kind: " // The archive task owns this object key, including retries after a timeout. const pathname = kind === "image" ? `uploads/ai-generated-archive-${archiveId}` : `uploads/canvas-${kind}-${archiveId}`; + if (kind === "image" && source.origin === "https://api.vip98k.com" && + source.pathname.startsWith("/api/external/assets/")) { + return fetchRelayImageIntoQiniu(sourceUrl, pathname, target!, archiveId); + } return fetchImageIntoQiniu(sourceUrl, pathname, target!, kind); } +function isRetryableQiniuFetch(error: unknown) { + if (error instanceof QiniuFetchError) { + return [408, 478, 502, 503, 504, 599].includes(error.status); + } + if (!(error instanceof Error)) return false; + const cause = error.cause as { code?: string } | undefined; + return error.name === "TimeoutError" || error.name === "AbortError" || + ["ECONNRESET", "ETIMEDOUT", "UND_ERR_SOCKET"].includes(cause?.code ?? ""); +} + +function relaySourceStatus(error: unknown) { + if (error instanceof QiniuFetchError && [404, 410].includes(error.status)) return error.status; +} + +async function readStoredRelayImage(pathname: string, target: QiniuUploadTarget) { + const client = createS3Client(target.config); + try { + const response = await client.send(new HeadObjectCommand({ + Bucket: target.config.bucket, Key: pathname, + }), { abortSignal: AbortSignal.timeout(4_000) }); + if (response.ContentType?.startsWith("image/") && response.ContentLength && response.ContentLength > 0) { + return describeArchivedObject(pathname, target, response.ContentType, response.ContentLength); + } + } catch { + return; + } finally { + client.destroy(); + } +} + +async function fetchRelayImageIntoQiniu( + sourceUrl: string, + pathname: string, + target: QiniuUploadTarget, + archiveId: string, +) { + const startedAt = Date.now(); + try { + return await fetchImageIntoQiniu(sourceUrl, pathname, target); + } catch (error) { + const missing = relaySourceStatus(error); + const retryable = isRetryableQiniuFetch(error); + const alreadyExists = error instanceof QiniuFetchError && error.status === 614; + if (!missing && !retryable && !alreadyExists) throw error; + + // Losing the fetch response does not establish whether Kodo stored the file. + const stored = await readStoredRelayImage(pathname, target); + if (stored) return stored; + if (missing) throw new QiniuArchiveSourceMissingError(missing); + if (!retryable) throw error; + + logger.warn("relay_fetch_retry", { + archiveId, + elapsedMs: Date.now() - startedAt, + ...(error instanceof QiniuFetchError + ? { status: error.status, requestId: error.requestId } + : { errorType: error instanceof Error ? error.name : "unknown" }), + }); + } + + try { + return await fetchImageIntoQiniu(sourceUrl, pathname, target, "image", RELAY_RETRY_TIMEOUT_MS); + } catch (error) { + const stored = await readStoredRelayImage(pathname, target); + if (stored) return stored; + const missing = relaySourceStatus(error); + if (missing) throw new QiniuArchiveSourceMissingError(missing); + throw error; + } +} + /** Prepare uploaded WebP images inside Kodo, retaining their embedded colour profile. */ export async function prepareQiniuWebpDownload(url: string, taskId: string) { const target = await resolveS3UploadTarget(); @@ -118,8 +211,9 @@ export async function prepareQiniuWebpDownload(url: string, taskId: string) { async function fetchImageIntoQiniu( sourceUrl: string, pathname: string, - target: NonNullable>>, + target: QiniuUploadTarget, kind: "image" | "audio" | "video" = "image", + timeoutMs = QINIU_FETCH_TIMEOUT_MS, ) { const config = target.config; const path = `/fetch/${encodeQiniu(sourceUrl)}/to/${encodeQiniu(`${config.bucket}:${pathname}`)}`; @@ -134,24 +228,28 @@ async function fetchImageIntoQiniu( }, body: "", redirect: "error", - signal: AbortSignal.timeout(40_000), + signal: AbortSignal.timeout(timeoutMs), }); if (!response.ok) { await response.body?.cancel(); - throw new Error(`Qiniu archive fetch returned HTTP ${response.status}`); + throw new QiniuFetchError(response.status, response.headers.get("x-reqid")); } const result = await response.json() as { key?: string; fsize?: number; mimeType?: string }; if (result.key !== pathname || !result.fsize || !result.mimeType?.startsWith(`${kind}/`)) { throw new Error(`Qiniu archive fetch did not return ${kind}`); } - const url = buildS3PublicUrl(config, pathname); + return describeArchivedObject(pathname, target, result.mimeType, result.fsize); +} + +async function describeArchivedObject(pathname: string, target: QiniuUploadTarget, contentType: string, bytes: number) { + const url = buildS3PublicUrl(target.config, pathname); return { url, thumbnailUrl: url, pathname, - contentType: result.mimeType, - bytes: result.fsize, - downloadAsset: result.mimeType === "image/png" + contentType, + bytes, + downloadAsset: contentType === "image/png" ? await getReadyQiniuImageDownload(url, target).catch(() => undefined) : undefined, }; diff --git a/tests/routes/image-archive-flow.test.ts b/tests/routes/image-archive-flow.test.ts index b543146ae..92fa65869 100644 --- a/tests/routes/image-archive-flow.test.ts +++ b/tests/routes/image-archive-flow.test.ts @@ -132,6 +132,73 @@ test("Qiniu archives 98K originals without relaying full image bytes through the expect(reads).toHaveLength(3); }); +test("98K fetch checks the original archive key and retries a temporary failure once", async () => { + const source = "https://api.vip98k.com/api/external/assets/test_asset"; + const id = "archive-1"; + const key = `uploads/ai-generated-archive-${id}`; + const publicUrl = "https://bucket.example"; + const config = { provider: "qiniu-kodo", region: "cn-east-1", bucket: "archive", + accessKeyId: "test-key", secretAccessKey: "test-secret", publicUrl }; + const calls: string[] = []; + let existing = false; + let firstStatus = 478; + const load = serverLoader({ + "@/lib/observability/logger": loggerMock, + "@/lib/storage/service": { + resolveS3UploadTarget: async () => ({ config }), + buildS3PublicUrl: (_: unknown, pathname: string) => `${publicUrl}/${pathname}`, + createS3Client: () => ({ + async send(cmd: { input: { Bucket: string; Key: string } }) { + expect(cmd.input).toEqual({ Bucket: "archive", Key: key }); + calls.push("head"); + if (!existing) throw new Error("Object does not exist"); + return { ContentType: "image/jpeg", ContentLength: 20 }; + }, + destroy() {}, + }), + }, + fetch: async (url: string) => { + const to = new URL(url).pathname.split("/to/")[1]; + expect(Buffer.from(to, "base64url").toString()).toBe(`archive:${key}`); + calls.push("fetch"); + return calls.length === 1 + ? { ok: false, status: firstStatus, headers: new Headers() } + : Response.json({ key, fsize: 20, mimeType: "image/jpeg" }); + }, + })("lib/storage/qiniu-image-archive.ts"); + + expect(await load.archiveImageThroughQiniu(source, id)).toMatchObject({ pathname: key, bytes: 20 }); + expect(calls.splice(0)).toEqual(["fetch", "head", "fetch"]); + + existing = true; + firstStatus = 614; + expect(await load.archiveImageThroughQiniu(source, id)).toMatchObject({ pathname: key, bytes: 20 }); + expect(calls.splice(0)).toEqual(["fetch", "head"]); + + existing = false; + firstStatus = 404; + await expect(load.archiveImageThroughQiniu(source, id)).rejects.toBeInstanceOf(load.QiniuArchiveSourceMissingError); + expect(calls).toEqual(["fetch", "head"]); +}); + +test("a confirmed missing 98K image does not start a production transfer", async () => { + class QiniuArchiveSourceMissingError extends Error {} + const sourceError = new QiniuArchiveSourceMissingError("98K original image is no longer available (HTTP 404)"); + const load = serverLoader({ + "@/lib/storage/image-byte-cache": { fetchImageBytes() { throw new Error("Unexpected production download"); } }, + "@/lib/observability/logger": loggerMock, + "@/lib/storage/image-archive-db": {}, + "@/lib/storage/service": {}, + "./image-archive-worker": {}, + "./qiniu-image-archive": { + QiniuArchiveSourceMissingError, + async archiveImageThroughQiniu() { throw sourceError; }, + }, + })("lib/storage/image-archive.ts"); + await expect(load.archiveTemporaryImage("https://api.vip98k.com/api/external/assets/test_asset", "task")) + .rejects.toBe(sourceError); +}); + test("160 requests from the same user and model start without admission waiting", async () => { const module = serverLoader({})("lib/canvas/image-concurrency.ts"); let queued = 0; diff --git a/tests/routes/qiniu-direct-paths.test.ts b/tests/routes/qiniu-direct-paths.test.ts index 740b4b17c..df71ac74c 100644 --- a/tests/routes/qiniu-direct-paths.test.ts +++ b/tests/routes/qiniu-direct-paths.test.ts @@ -62,7 +62,9 @@ test("download preparation only registers unsigned files in the configured bucke const module = serverLoader({ "@/lib/db/queries": { getDefaultStorageConfig: async () => config }, "./image-archive-db": { findImageArchiveDownloadTask: async () => null, + enqueueImageArchiveTask: async () => ({ id: "waiting", status: "pending", targetUrl: null }), ensureStoredImageDownloadTask: async (url: string) => { stored.push(url); return { id: "1", targetUrl: url }; } }, + "./image-archive-worker": { scheduleImageArchiveWorker() {} }, }, { CANVAS_QINIU_ORIGIN_URL: "https://origin.example" })("lib/storage/canvas-cdn-download.ts"); for (const url of ["https://other.example/uploads/a.png", "https://cdn.example/uploads/a.png?token=signed", "https://cdn.example/uploads/a.png?imageView2/2/w/160", "https://cdn.example/private/a.png"]) {