From b3549f25d7f4e2c76453da10ee8e1c447a42b098 Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Tue, 10 Nov 2026 11:42:30 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E9=80=9F=2098K=20=E5=8E=9F=E5=9B=BE?= =?UTF-8?q?=E5=BD=92=E6=A1=A3=E5=92=8C=E7=94=BB=E5=B8=83=E4=B8=8B=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/jobs/image-archive-job.ts | 5 +- lib/storage/image-archive-db.ts | 3 +- lib/storage/image-archive.ts | 13 ++- lib/storage/image-download-preparation.ts | 6 + lib/storage/qiniu-image-archive.ts | 121 +++++++++++++++++++++ tests/routes/canvas-download-speed.test.ts | 1 + tests/routes/image-archive-flow.test.ts | 47 ++++++++ 7 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 lib/storage/qiniu-image-archive.ts diff --git a/lib/jobs/image-archive-job.ts b/lib/jobs/image-archive-job.ts index c8fe5779a..2d3329202 100644 --- a/lib/jobs/image-archive-job.ts +++ b/lib/jobs/image-archive-job.ts @@ -41,12 +41,13 @@ export async function runImageArchiveJob() { try { if (!(await markImageArchiveTaskArchiving(task.id))) return; const metadata = task.metadata as { inlineSource?: string } | null; - const archived = await archiveTemporaryImage(metadata?.inlineSource || task.sourceUrl); + const archived = await archiveTemporaryImage(metadata?.inlineSource || task.sourceUrl, task.id); await markImageArchiveTaskStored({ id: task.id, targetUrl: archived.url, + downloadAsset: archived.downloadAsset, }); - scheduleImageDownloadPreparation(task.id); + if (!archived.downloadAsset) scheduleImageDownloadPreparation(task.id); storeSucceeded += 1; console.log("[ImageArchiveJob] stored", { taskId: task.id, diff --git a/lib/storage/image-archive-db.ts b/lib/storage/image-archive-db.ts index bb28f4ff0..f0517ba90 100644 --- a/lib/storage/image-archive-db.ts +++ b/lib/storage/image-archive-db.ts @@ -138,6 +138,7 @@ export async function markImageArchiveTaskStored(params: { id: string; targetUrl: string; targetStorageConfigId?: string | null; + downloadAsset?: import("./qiniu-image-archive").ReadyImageDownload; }) { await db .update(imageArchiveTask) @@ -145,7 +146,7 @@ export async function markImageArchiveTaskStored(params: { status: "stored", targetUrl: params.targetUrl, targetStorageConfigId: params.targetStorageConfigId || null, - metadata: sql`coalesce(${imageArchiveTask.metadata}, '{}'::jsonb) || jsonb_build_object('downloadAsset', jsonb_build_object('status', 'pending'))`, + metadata: sql`coalesce(${imageArchiveTask.metadata}, '{}'::jsonb) || jsonb_build_object('downloadAsset', ${JSON.stringify(params.downloadAsset ?? { status: "pending" })}::jsonb)`, archivedAt: new Date(), replaceAfterAt: new Date(), lastError: null, diff --git a/lib/storage/image-archive.ts b/lib/storage/image-archive.ts index 4ac8023c5..a634e6e1f 100644 --- a/lib/storage/image-archive.ts +++ b/lib/storage/image-archive.ts @@ -12,6 +12,7 @@ import { imageArchiveSourceKey, } from "@/lib/storage/image-archive-db"; import { uploadFile } from "@/lib/storage/service"; +import { archiveImageThroughQiniu } from "./qiniu-image-archive"; const logger = createLogger("image-archive"); @@ -182,7 +183,7 @@ export async function downloadTemporaryImage(params: { sourceUrl: string; retrie }); return { ...data, extension: getImageExtension(data.contentType, params.sourceUrl) }; } catch (error) { - if (attempt >= retries) throw error; + if (attempt >= retries || /\bHTTP (?:404|410)\b/.test(String(error))) throw error; await new Promise((resolve) => setTimeout(resolve, 1000 * (attempt + 1))); } } @@ -190,7 +191,14 @@ export async function downloadTemporaryImage(params: { sourceUrl: string; retrie throw new Error("Image download failed"); } -export async function archiveTemporaryImage(sourceUrl: string) { +export async function archiveTemporaryImage(sourceUrl: string, archiveId?: string) { + if (archiveId) { + const direct = await archiveImageThroughQiniu(sourceUrl, archiveId).catch(error => { + logger.warn("cloud_archive_fallback", error, { archiveId }); + return null; + }); + if (direct) return direct; + } const downloaded = await downloadTemporaryImage({ sourceUrl }); const objectSuffix = `${Date.now()}-${randomUUID()}`; const filename = `ai-generated-archive-${objectSuffix}.${downloaded.extension}`; @@ -202,5 +210,6 @@ export async function archiveTemporaryImage(sourceUrl: string) { pathname: uploaded.pathname, contentType: uploaded.contentType, bytes: downloaded.buffer.length, + downloadAsset: undefined, }; } diff --git a/lib/storage/image-download-preparation.ts b/lib/storage/image-download-preparation.ts index bb179fc9d..215c3a387 100644 --- a/lib/storage/image-download-preparation.ts +++ b/lib/storage/image-download-preparation.ts @@ -4,6 +4,7 @@ import { fetchImageBytes, cacheImageBytes } from "./image-byte-cache"; import { normalizeCanvasDownloadAsset } from "@/lib/canvas/photoshop-safe-download"; import { claimImageDownloadTask, saveImageDownloadAsset, getImageDownloadTasks, hasPendingImageDownloads } from "./image-archive-db"; import { uploadFile } from "./service"; +import { getReadyQiniuImageDownload } from "./qiniu-image-archive"; import { getDownloadFailure } from "@/lib/canvas/download-error"; import { createLogger } from "@/lib/observability/logger"; import { trackRuntimeWork } from "@/lib/observability/active-work"; @@ -15,6 +16,11 @@ export async function prepareImageArchiveDownload(id: string) { if (!task?.targetUrl) return; let phase: "read" | "convert" | "store" = "read"; try { + const ready = await getReadyQiniuImageDownload(task.targetUrl).catch(() => undefined); + if (ready) { + await saveImageDownloadAsset(id, ready); + return; + } const source = await fetchImageBytes(task.targetUrl); phase = "convert"; const asset = await normalizeCanvasDownloadAsset({ diff --git a/lib/storage/qiniu-image-archive.ts b/lib/storage/qiniu-image-archive.ts new file mode 100644 index 000000000..f9d17707a --- /dev/null +++ b/lib/storage/qiniu-image-archive.ts @@ -0,0 +1,121 @@ +import "server-only"; + +import { GetObjectCommand } from "@aws-sdk/client-s3"; +import { createHmac } from "node:crypto"; +import { + buildS3PublicUrl, + createS3Client, + resolveS3UploadTarget, +} from "@/lib/storage/service"; + +const QINIU_EAST_CHINA_IO = "iovip-z0.qiniuio.com"; +const PNG_HEADER_LIMIT = 64 * 1024; +const PNG_SIGNATURE = Buffer.from("89504e470d0a1a0a", "hex"); + +export type ReadyImageDownload = { + status: "ready"; + url: string; + contentType: "image/png"; + bytes: number; + preparedAt: string; +}; + +function encodeQiniu(value: string | Buffer) { + return (typeof value === "string" ? Buffer.from(value) : value) + .toString("base64").replace(/\+/g, "-").replace(/\//g, "_"); +} + +function isDownloadReadyPng(prefix: Buffer) { + if (prefix.length < 41 || !prefix.subarray(0, 8).equals(PNG_SIGNATURE) + || prefix.readUInt32BE(8) !== 13 || prefix.toString("ascii", 12, 16) !== "IHDR" + || !prefix.readUInt32BE(16) || !prefix.readUInt32BE(20) + || prefix[24] !== 8 || (prefix[25] !== 2 && prefix[25] !== 6)) return false; + + // Animation and EXIF orientation need the existing full-image preparation. + // An embedded color profile is fine: the browser gets the exact stored bytes. + for (let offset = 33; offset + 8 <= prefix.length;) { + const type = prefix.toString("ascii", offset + 4, offset + 8); + if (type === "IDAT") return true; + if (type === "acTL" || type === "eXIf" || type === "IEND") return false; + offset += prefix.readUInt32BE(offset) + 12; + } + return false; +} + +export async function getReadyQiniuImageDownload(url: string, target?: Awaited>) { + const config = (target ?? await resolveS3UploadTarget())?.config; + if (config?.provider !== "qiniu-kodo" || !config.publicUrl) return; + const base = buildS3PublicUrl(config, ""); + if (!url.startsWith(base)) return; + const key = url.slice(base.length); + if (!/^uploads\/[\w.-]+$/.test(key)) return; + + const client = createS3Client(config); + try { + const response = await client.send(new GetObjectCommand({ + Bucket: config.bucket, Key: key, Range: `bytes=0-${PNG_HEADER_LIMIT - 1}`, + }), { abortSignal: AbortSignal.timeout(10_000) }); + const range = response.ContentRange?.match(/^bytes 0-(\d+)\/(\d+)$/); + const length = response.ContentLength ?? 0; + const bytes = Number(range?.[2]); + if (response.$metadata.httpStatusCode !== 206 || response.ContentType !== "image/png" + || !range || !response.Body || length <= 0 || length > PNG_HEADER_LIMIT + || Number(range[1]) + 1 !== length || bytes < length) { + (response.Body as NodeJS.ReadableStream & { destroy?: () => void } | undefined)?.destroy?.(); + return; + } + const prefix = Buffer.from(await response.Body.transformToByteArray()); + if (prefix.length === length && isDownloadReadyPng(prefix)) { + return { status: "ready", url, contentType: "image/png", bytes, preparedAt: new Date().toISOString() } satisfies ReadyImageDownload; + } + } finally { + client.destroy(); + } +} + +export async function archiveImageThroughQiniu(sourceUrl: string, archiveId: string) { + if (!/^https:\/\/api\.vip98k\.com\/api\/external\/assets\/[A-Za-z0-9_-]+$/.test(sourceUrl)) { + return null; + } + const target = await resolveS3UploadTarget(); + const config = target?.config; + if (config?.provider !== "qiniu-kodo" || config.region !== "cn-east-1" || !config.publicUrl) { + return null; + } + + // The archive task owns this object key, including retries after a timeout. + const pathname = `uploads/ai-generated-archive-${archiveId}`; + const path = `/fetch/${encodeQiniu(sourceUrl)}/to/${encodeQiniu(`${config.bucket}:${pathname}`)}`; + const contentType = "application/x-www-form-urlencoded"; + const signingData = `POST ${path}\nHost: ${QINIU_EAST_CHINA_IO}\nContent-Type: ${contentType}\n\n`; + const signature = encodeQiniu(createHmac("sha1", config.secretAccessKey).update(signingData).digest()); + const response = await fetch(`https://${QINIU_EAST_CHINA_IO}${path}`, { + method: "POST", + headers: { + Authorization: `Qiniu ${config.accessKeyId}:${signature}`, + "Content-Type": contentType, + }, + body: "", + redirect: "error", + signal: AbortSignal.timeout(40_000), + }); + if (!response.ok) { + await response.body?.cancel(); + throw new Error(`Qiniu archive fetch returned HTTP ${response.status}`); + } + const result = await response.json() as { key?: string; fsize?: number; mimeType?: string }; + if (result.key !== pathname || !result.fsize || !result.mimeType?.startsWith("image/")) { + throw new Error("Qiniu archive fetch did not return an image"); + } + const url = buildS3PublicUrl(config, pathname); + return { + url, + thumbnailUrl: url, + pathname, + contentType: result.mimeType, + bytes: result.fsize, + downloadAsset: result.mimeType === "image/png" + ? await getReadyQiniuImageDownload(url, target).catch(() => undefined) + : undefined, + }; +} diff --git a/tests/routes/canvas-download-speed.test.ts b/tests/routes/canvas-download-speed.test.ts index 1e25c5aad..a86023176 100644 --- a/tests/routes/canvas-download-speed.test.ts +++ b/tests/routes/canvas-download-speed.test.ts @@ -131,6 +131,7 @@ test("PNG preparation failure leaves the original archived and schedules a retry const module = serverLoader({ "./image-archive-db": { claimImageDownloadTask: async () => ({ id: "task", targetUrl: "https://cdn.example/original.png", metadata: { downloadAsset: { attempts: 2 } } }), saveImageDownloadAsset: async (_id: string, asset: any) => { saved = asset; } }, "./image-byte-cache": { fetchImageBytes: async () => { throw new Error("storage offline"); } }, + "./qiniu-image-archive": { getReadyQiniuImageDownload: async () => undefined }, "./service": {}, })("lib/storage/image-download-preparation.ts"); await module.prepareImageArchiveDownload("task"); diff --git a/tests/routes/image-archive-flow.test.ts b/tests/routes/image-archive-flow.test.ts index db9881fb0..fce0db247 100644 --- a/tests/routes/image-archive-flow.test.ts +++ b/tests/routes/image-archive-flow.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import vm from "node:vm"; import * as crypto from "node:crypto"; import ts from "typescript"; +import sharp from "sharp"; import { collectImageArchiveSources, replaceArchivedImageUrls, @@ -82,6 +83,52 @@ function serverLoader(mocks: Record, env: Record ({ info() {}, warn() {}, error() {} }) }; +test("Qiniu archives 98K originals without relaying full image bytes through the app", async () => { + const image = sharp({ create: { width: 4, height: 4, channels: 3, background: "blue" } }); + const rgb = await image.clone().withIccProfile("srgb").png().toBuffer(); + const indexed = await image.clone().png({ palette: true }).toBuffer(); + const config = { provider: "qiniu-kodo", region: "cn-east-1", bucket: "archive", + accessKeyId: "test-key", secretAccessKey: "test-secret", publicUrl: "https://bucket.example" }; + const id = "test-task"; + const key = `uploads/ai-generated-archive-${id}`; + const source = "https://api.vip98k.com/api/external/assets/test_asset"; + const archiveUrl = `${config.publicUrl}/${key}`; + let stored = rgb; + const cloudCalls: string[] = []; + const reads: string[] = []; + const load = serverLoader({ + "@/lib/storage/service": { + resolveS3UploadTarget: async () => ({ config }), + buildS3PublicUrl: (_: unknown, path: string) => `${config.publicUrl}/${path}`, + createS3Client: () => ({ + send: async (cmd: { input: { Bucket: string; Key: string; Range: string } }) => { + expect(cmd.input).toEqual({ Bucket: config.bucket, Key: key, Range: "bytes=0-65535" }); + reads.push(cmd.input.Range); + return { $metadata: { httpStatusCode: 206 }, ContentType: "image/png", + ContentLength: stored.length, ContentRange: `bytes 0-${stored.length - 1}/${stored.length}`, + Body: { transformToByteArray: async () => stored } }; + }, + destroy() {}, + }), + }, + fetch: async (url: string, options: RequestInit) => { + cloudCalls.push(url); + expect(options.method).toBe("POST"); + const [, from, to] = new URL(url).pathname.split("/fetch/")[1].match(/^(.+)\/to\/(.+)$/)!; + expect(Buffer.from(from, "base64url").toString()).toBe(source); + expect(Buffer.from(to, "base64url").toString()).toBe(`${config.bucket}:${key}`); + return Response.json({ key, fsize: stored.length, mimeType: "image/png" }); + }, + })("lib/storage/qiniu-image-archive.ts"); + const archived = await load.archiveImageThroughQiniu(source, id); + expect(archived.downloadAsset).toMatchObject({ status: "ready", url: archiveUrl, bytes: rgb.length }); + stored = indexed; + expect((await load.archiveImageThroughQiniu(source, id)).downloadAsset).toBeUndefined(); + expect(await load.archiveImageThroughQiniu("https://another-provider.example/result.png", id)).toBeNull(); + expect(cloudCalls).toHaveLength(2); + expect(reads).toHaveLength(2); +}); + test("160 requests from the same user and model start without admission waiting", async () => { const module = serverLoader({})("lib/canvas/image-concurrency.ts"); let queued = 0; -- 2.47.2