From a6f79998bde05be926489f70fea5746bb961c610 Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Thu, 12 Nov 2026 16:43:52 +0800 Subject: [PATCH] Prepare uploaded WebP downloads in Qiniu with embedded colour profiles --- lib/storage/image-download-preparation.ts | 7 ++- lib/storage/qiniu-image-archive.ts | 22 ++++++++ tests/qiniu-delivery.config.ts | 2 +- tests/routes/canvas-download-recovery.test.ts | 2 +- tests/routes/canvas-download-speed.test.ts | 2 +- tests/routes/qiniu-direct-paths.test.ts | 51 +++++++++++++++++++ 6 files changed, 82 insertions(+), 4 deletions(-) diff --git a/lib/storage/image-download-preparation.ts b/lib/storage/image-download-preparation.ts index 215c3a387..716d29b45 100644 --- a/lib/storage/image-download-preparation.ts +++ b/lib/storage/image-download-preparation.ts @@ -4,7 +4,7 @@ import { fetchImageBytes, cacheImageBytes } from "./image-byte-cache"; import { normalizeCanvasDownloadAsset } from "@/lib/canvas/photoshop-safe-download"; import { claimImageDownloadTask, saveImageDownloadAsset, getImageDownloadTasks, hasPendingImageDownloads } from "./image-archive-db"; import { uploadFile } from "./service"; -import { getReadyQiniuImageDownload } from "./qiniu-image-archive"; +import { getReadyQiniuImageDownload, prepareQiniuWebpDownload } from "./qiniu-image-archive"; import { getDownloadFailure } from "@/lib/canvas/download-error"; import { createLogger } from "@/lib/observability/logger"; import { trackRuntimeWork } from "@/lib/observability/active-work"; @@ -21,6 +21,11 @@ export async function prepareImageArchiveDownload(id: string) { await saveImageDownloadAsset(id, ready); return; } + const converted = await prepareQiniuWebpDownload(task.targetUrl, task.id).catch(() => undefined); + if (converted) { + await saveImageDownloadAsset(id, converted); + return; + } const source = await fetchImageBytes(task.targetUrl); phase = "convert"; const asset = await normalizeCanvasDownloadAsset({ diff --git a/lib/storage/qiniu-image-archive.ts b/lib/storage/qiniu-image-archive.ts index 431ef7683..f02f54681 100644 --- a/lib/storage/qiniu-image-archive.ts +++ b/lib/storage/qiniu-image-archive.ts @@ -91,6 +91,28 @@ export async function archiveImageThroughQiniu(sourceUrl: string, archiveId: str // The archive task owns this object key, including retries after a timeout. const pathname = `uploads/ai-generated-archive-${archiveId}`; + return fetchImageIntoQiniu(sourceUrl, pathname, target!); +} + +/** Prepare uploaded WebP images inside Kodo, retaining their embedded colour profile. */ +export async function prepareQiniuWebpDownload(url: string, taskId: string) { + const target = await resolveS3UploadTarget(); + const config = target?.config; + if (config?.provider !== "qiniu-kodo" || config.region !== "cn-east-1" || !config.publicUrl) return; + const base = buildS3PublicUrl(config, ""); + if (!url.startsWith(base) || !/^uploads\/(?:[\w.-]+\/)*[\w.-]+\.webp$/i.test(url.slice(base.length))) return; + // No resizing or lossy re-encoding; apply orientation before removing EXIF. + const source = `${url}?imageMogr2/auto-orient/format/png/strip/meta-keep-list/${encodeQiniu("icc")}`; + const result = await fetchImageIntoQiniu(source, `uploads/canvas-download-${taskId}.png`, target!); + return result.downloadAsset; +} + +async function fetchImageIntoQiniu( + sourceUrl: string, + pathname: string, + target: NonNullable>>, +) { + const config = target.config; const path = `/fetch/${encodeQiniu(sourceUrl)}/to/${encodeQiniu(`${config.bucket}:${pathname}`)}`; const contentType = "application/x-www-form-urlencoded"; const signingData = `POST ${path}\nHost: ${QINIU_EAST_CHINA_IO}\nContent-Type: ${contentType}\n\n`; diff --git a/tests/qiniu-delivery.config.ts b/tests/qiniu-delivery.config.ts index e54d25b67..1d4c6f4fc 100644 --- a/tests/qiniu-delivery.config.ts +++ b/tests/qiniu-delivery.config.ts @@ -2,7 +2,7 @@ import { defineConfig } from "@playwright/test"; export default defineConfig({ testDir: "./routes", - testMatch: /(?:qiniu-direct-paths|image-delivery-qiniu|image-archive-flow|original-image-download|canvas-download-speed)\.test\.ts/, + testMatch: /(?:qiniu-direct-paths|image-delivery-qiniu|image-archive-flow|original-image-download|canvas-download-speed|canvas-download-recovery)\.test\.ts/, workers: 1, reporter: "list", timeout: 30_000, diff --git a/tests/routes/canvas-download-recovery.test.ts b/tests/routes/canvas-download-recovery.test.ts index 831471ae1..6d489e9c7 100644 --- a/tests/routes/canvas-download-recovery.test.ts +++ b/tests/routes/canvas-download-recovery.test.ts @@ -124,7 +124,7 @@ test("requested downloads are prioritized with two workers before scanning backg }, "./image-byte-cache": { fetchImageBytes: async () => { active++; maximum = Math.max(maximum, active); await new Promise(resolve => releases.push(resolve)); active--; return { buffer: bytes, contentType: "image/png" }; }, cacheImageBytes: async () => {} }, "@/lib/canvas/photoshop-safe-download": { normalizeCanvasDownloadAsset: async () => ({ ok: true, bytes, contentType: "image/png" }) }, - "./qiniu-image-archive": { getReadyQiniuImageDownload: async () => undefined }, + "./qiniu-image-archive": { getReadyQiniuImageDownload: async () => undefined, prepareQiniuWebpDownload: async () => undefined }, "./service": { uploadFile() { throw new Error("Compatible PNG must retain its object"); } }, })("lib/storage/image-download-preparation.ts"); worker.scheduleImageDownloadPreparation("one"); worker.scheduleImageDownloadPreparation("two"); worker.scheduleImageDownloadPreparation("three"); diff --git a/tests/routes/canvas-download-speed.test.ts b/tests/routes/canvas-download-speed.test.ts index 51b4bca07..f8530a82c 100644 --- a/tests/routes/canvas-download-speed.test.ts +++ b/tests/routes/canvas-download-speed.test.ts @@ -137,7 +137,7 @@ test("PNG preparation failure leaves the original archived and schedules a retry const module = serverLoader({ "./image-archive-db": { claimImageDownloadTask: async () => ({ id: "task", targetUrl: "https://cdn.example/original.png", metadata: { downloadAsset: { attempts: 2 } } }), saveImageDownloadAsset: async (_id: string, asset: any) => { saved = asset; } }, "./image-byte-cache": { fetchImageBytes: async () => { throw new Error("storage offline"); } }, - "./qiniu-image-archive": { getReadyQiniuImageDownload: async () => undefined }, + "./qiniu-image-archive": { getReadyQiniuImageDownload: async () => undefined, prepareQiniuWebpDownload: async () => undefined }, "./service": {}, })("lib/storage/image-download-preparation.ts"); await module.prepareImageArchiveDownload("task"); diff --git a/tests/routes/qiniu-direct-paths.test.ts b/tests/routes/qiniu-direct-paths.test.ts index ae1b9bd37..740b4b17c 100644 --- a/tests/routes/qiniu-direct-paths.test.ts +++ b/tests/routes/qiniu-direct-paths.test.ts @@ -113,3 +113,54 @@ test("saved preview URLs redirect to an archived CDN thumbnail without reading t expect(response.status).toBe(307); expect(response.headers.get("location")).toBe("https://cdn.example/uploads/a.png?imageView2/2/w/360/format/webp/q/88"); }); + +test("uploaded WebP is converted inside Qiniu with orientation and ICC retention", async () => { + const png = await sharp({ create: { width: 20, height: 12, channels: 4, background: "#22887780" } }).withIccProfile("p3").png().toBuffer(); + let calls = 0; + const module = serverLoader({ + "@/lib/storage/service": { resolveS3UploadTarget: async () => ({ config }), + buildS3PublicUrl: (_: unknown, key: string) => `${config.publicUrl}/${key}`, + createS3Client: () => ({ destroy() {}, send: async (cmd: any) => { + expect(cmd.input.Key).toBe("uploads/canvas-download-task-1.png"); + expect(cmd.input.Range).toBe("bytes=0-65535"); + return { $metadata: { httpStatusCode: 206 }, ContentType: "image/png", ContentLength: png.length, + ContentRange: `bytes 0-${png.length - 1}/${png.length}`, Body: { transformToByteArray: async () => png } }; + } }) }, + fetch: async (url: string) => { + calls++; + const encoded = new URL(url).pathname.split("/fetch/")[1].split("/to/")[0]; + const source = Buffer.from(encoded, "base64url").toString(); + expect(source).toBe("https://cdn.example/uploads/direct/user/source.webp?imageMogr2/auto-orient/format/png/strip/meta-keep-list/aWNj"); + return Response.json({ key: "uploads/canvas-download-task-1.png", fsize: png.length, mimeType: "image/png" }); + }, + })("lib/storage/qiniu-image-archive.ts"); + const source = "https://cdn.example/uploads/direct/user/source.webp"; + const ready = await module.prepareQiniuWebpDownload(source, "task-1"); + expect(ready).toMatchObject({ status: "ready", url: "https://cdn.example/uploads/canvas-download-task-1.png", bytes: png.length }); + for (const excluded of [source + "?token=signed", source.replace("cdn.example", "other.example"), source.replace(".webp", ".png")]) { + expect(await module.prepareQiniuWebpDownload(excluded, "task-1")).toBeUndefined(); + } + expect(calls).toBe(1); +}); + +test("download worker uses a cloud-prepared PNG and retains local conversion on cloud failure", async () => { + const bytes = await sharp({ create: { width: 4, height: 4, channels: 4, background: "#bb442280" } }).png().toBuffer(); + let cloudFails = false, reads = 0; + const saved: any[] = []; + const task = { id: "webp", targetUrl: "https://cdn.example/uploads/a.webp", metadata: {} }; + const module = serverLoader({ + "./image-archive-db": { claimImageDownloadTask: async () => task, saveImageDownloadAsset: async (_: string, asset: any) => saved.push(asset) }, + "./qiniu-image-archive": { getReadyQiniuImageDownload: async () => undefined, + prepareQiniuWebpDownload: async () => { if (cloudFails) throw Error("cloud temporarily unavailable"); return { status: "ready", url: "https://cdn.example/uploads/converted.png", bytes: bytes.length, contentType: "image/png" }; } }, + "./image-byte-cache": { fetchImageBytes: async () => { reads++; return { buffer: bytes, contentType: "image/png" }; }, cacheImageBytes: async () => undefined }, + "@/lib/storage/image-byte-cache": { onceImageBytes: async (_: string, fn: any) => fn() }, + "./service": { uploadFile: async () => ({ url: "https://cdn.example/uploads/fallback.png" }) }, + })("lib/storage/image-download-preparation.ts"); + await module.prepareImageArchiveDownload("webp"); + expect(reads).toBe(0); + expect(saved[0]).toMatchObject({ status: "ready", url: "https://cdn.example/uploads/converted.png" }); + cloudFails = true; + await module.prepareImageArchiveDownload("webp"); + expect(reads).toBe(1); + expect(saved[1].status).toBe("ready"); +});