加速 98K 原图归档和画布下载

lib/jobs/image-archive-job.ts

@@ -41,12 +41,13 @@ export async function runImageArchiveJob() {
     try {
       if (!(await markImageArchiveTaskArchiving(task.id))) return;
       const metadata = task.metadata as { inlineSource?: string } | null;
-      const archived = await archiveTemporaryImage(metadata?.inlineSource || task.sourceUrl);
+      const archived = await archiveTemporaryImage(metadata?.inlineSource || task.sourceUrl, task.id);
       await markImageArchiveTaskStored({
         id: task.id,
         targetUrl: archived.url,
+        downloadAsset: archived.downloadAsset,
       });
-      scheduleImageDownloadPreparation(task.id);
+      if (!archived.downloadAsset) scheduleImageDownloadPreparation(task.id);
       storeSucceeded += 1;
       console.log("[ImageArchiveJob] stored", {
         taskId: task.id,

lib/storage/image-archive-db.ts

@@ -138,6 +138,7 @@ export async function markImageArchiveTaskStored(params: {
   id: string;
   targetUrl: string;
   targetStorageConfigId?: string | null;
+  downloadAsset?: import("./qiniu-image-archive").ReadyImageDownload;
 }) {
   await db
     .update(imageArchiveTask)
@@ -145,7 +146,7 @@ export async function markImageArchiveTaskStored(params: {
       status: "stored",
       targetUrl: params.targetUrl,
       targetStorageConfigId: params.targetStorageConfigId || null,
-      metadata: sql`coalesce(${imageArchiveTask.metadata}, '{}'::jsonb) || jsonb_build_object('downloadAsset', jsonb_build_object('status', 'pending'))`,
+      metadata: sql`coalesce(${imageArchiveTask.metadata}, '{}'::jsonb) || jsonb_build_object('downloadAsset', ${JSON.stringify(params.downloadAsset ?? { status: "pending" })}::jsonb)`,
       archivedAt: new Date(),
       replaceAfterAt: new Date(),
       lastError: null,

lib/storage/image-archive.ts

@@ -12,6 +12,7 @@ import {
   imageArchiveSourceKey,
 } from "@/lib/storage/image-archive-db";
 import { uploadFile } from "@/lib/storage/service";
+import { archiveImageThroughQiniu } from "./qiniu-image-archive";
 
 const logger = createLogger("image-archive");
 
@@ -182,7 +183,7 @@ export async function downloadTemporaryImage(params: { sourceUrl: string; retrie
       });
       return { ...data, extension: getImageExtension(data.contentType, params.sourceUrl) };
     } catch (error) {
-      if (attempt >= retries) throw error;
+      if (attempt >= retries || /\bHTTP (?:404|410)\b/.test(String(error))) throw error;
       await new Promise((resolve) => setTimeout(resolve, 1000 * (attempt + 1)));
     }
   }
@@ -190,7 +191,14 @@ export async function downloadTemporaryImage(params: { sourceUrl: string; retrie
   throw new Error("Image download failed");
 }
 
-export async function archiveTemporaryImage(sourceUrl: string) {
+export async function archiveTemporaryImage(sourceUrl: string, archiveId?: string) {
+  if (archiveId) {
+    const direct = await archiveImageThroughQiniu(sourceUrl, archiveId).catch(error => {
+      logger.warn("cloud_archive_fallback", error, { archiveId });
+      return null;
+    });
+    if (direct) return direct;
+  }
   const downloaded = await downloadTemporaryImage({ sourceUrl });
   const objectSuffix = `${Date.now()}-${randomUUID()}`;
   const filename = `ai-generated-archive-${objectSuffix}.${downloaded.extension}`;
@@ -202,5 +210,6 @@ export async function archiveTemporaryImage(sourceUrl: string) {
     pathname: uploaded.pathname,
     contentType: uploaded.contentType,
     bytes: downloaded.buffer.length,
+    downloadAsset: undefined,
   };
 }

lib/storage/image-download-preparation.ts

@@ -4,6 +4,7 @@ import { fetchImageBytes, cacheImageBytes } from "./image-byte-cache";
 import { normalizeCanvasDownloadAsset } from "@/lib/canvas/photoshop-safe-download";
 import { claimImageDownloadTask, saveImageDownloadAsset, getImageDownloadTasks, hasPendingImageDownloads } from "./image-archive-db";
 import { uploadFile } from "./service";
+import { getReadyQiniuImageDownload } from "./qiniu-image-archive";
 import { getDownloadFailure } from "@/lib/canvas/download-error";
 import { createLogger } from "@/lib/observability/logger";
 import { trackRuntimeWork } from "@/lib/observability/active-work";
@@ -15,6 +16,11 @@ export async function prepareImageArchiveDownload(id: string) {
   if (!task?.targetUrl) return;
   let phase: "read" | "convert" | "store" = "read";
   try {
+    const ready = await getReadyQiniuImageDownload(task.targetUrl).catch(() => undefined);
+    if (ready) {
+      await saveImageDownloadAsset(id, ready);
+      return;
+    }
     const source = await fetchImageBytes(task.targetUrl);
     phase = "convert";
     const asset = await normalizeCanvasDownloadAsset({

lib/storage/qiniu-image-archive.ts

@@ -0,0 +1,121 @@
+import "server-only";
+
+import { GetObjectCommand } from "@aws-sdk/client-s3";
+import { createHmac } from "node:crypto";
+import {
+  buildS3PublicUrl,
+  createS3Client,
+  resolveS3UploadTarget,
+} from "@/lib/storage/service";
+
+const QINIU_EAST_CHINA_IO = "iovip-z0.qiniuio.com";
+const PNG_HEADER_LIMIT = 64 * 1024;
+const PNG_SIGNATURE = Buffer.from("89504e470d0a1a0a", "hex");
+
+export type ReadyImageDownload = {
+  status: "ready";
+  url: string;
+  contentType: "image/png";
+  bytes: number;
+  preparedAt: string;
+};
+
+function encodeQiniu(value: string | Buffer) {
+  return (typeof value === "string" ? Buffer.from(value) : value)
+    .toString("base64").replace(/\+/g, "-").replace(/\//g, "_");
+}
+
+function isDownloadReadyPng(prefix: Buffer) {
+  if (prefix.length < 41 || !prefix.subarray(0, 8).equals(PNG_SIGNATURE)
+    || prefix.readUInt32BE(8) !== 13 || prefix.toString("ascii", 12, 16) !== "IHDR"
+    || !prefix.readUInt32BE(16) || !prefix.readUInt32BE(20)
+    || prefix[24] !== 8 || (prefix[25] !== 2 && prefix[25] !== 6)) return false;
+
+  // Animation and EXIF orientation need the existing full-image preparation.
+  // An embedded color profile is fine: the browser gets the exact stored bytes.
+  for (let offset = 33; offset + 8 <= prefix.length;) {
+    const type = prefix.toString("ascii", offset + 4, offset + 8);
+    if (type === "IDAT") return true;
+    if (type === "acTL" || type === "eXIf" || type === "IEND") return false;
+    offset += prefix.readUInt32BE(offset) + 12;
+  }
+  return false;
+}
+
+export async function getReadyQiniuImageDownload(url: string, target?: Awaited<ReturnType<typeof resolveS3UploadTarget>>) {
+  const config = (target ?? await resolveS3UploadTarget())?.config;
+  if (config?.provider !== "qiniu-kodo" || !config.publicUrl) return;
+  const base = buildS3PublicUrl(config, "");
+  if (!url.startsWith(base)) return;
+  const key = url.slice(base.length);
+  if (!/^uploads\/[\w.-]+$/.test(key)) return;
+
+  const client = createS3Client(config);
+  try {
+    const response = await client.send(new GetObjectCommand({
+      Bucket: config.bucket, Key: key, Range: `bytes=0-${PNG_HEADER_LIMIT - 1}`,
+    }), { abortSignal: AbortSignal.timeout(10_000) });
+    const range = response.ContentRange?.match(/^bytes 0-(\d+)\/(\d+)$/);
+    const length = response.ContentLength ?? 0;
+    const bytes = Number(range?.[2]);
+    if (response.$metadata.httpStatusCode !== 206 || response.ContentType !== "image/png"
+      || !range || !response.Body || length <= 0 || length > PNG_HEADER_LIMIT
+      || Number(range[1]) + 1 !== length || bytes < length) {
+      (response.Body as NodeJS.ReadableStream & { destroy?: () => void } | undefined)?.destroy?.();
+      return;
+    }
+    const prefix = Buffer.from(await response.Body.transformToByteArray());
+    if (prefix.length === length && isDownloadReadyPng(prefix)) {
+      return { status: "ready", url, contentType: "image/png", bytes, preparedAt: new Date().toISOString() } satisfies ReadyImageDownload;
+    }
+  } finally {
+    client.destroy();
+  }
+}
+
+export async function archiveImageThroughQiniu(sourceUrl: string, archiveId: string) {
+  if (!/^https:\/\/api\.vip98k\.com\/api\/external\/assets\/[A-Za-z0-9_-]+$/.test(sourceUrl)) {
+    return null;
+  }
+  const target = await resolveS3UploadTarget();
+  const config = target?.config;
+  if (config?.provider !== "qiniu-kodo" || config.region !== "cn-east-1" || !config.publicUrl) {
+    return null;
+  }
+
+  // The archive task owns this object key, including retries after a timeout.
+  const pathname = `uploads/ai-generated-archive-${archiveId}`;
+  const path = `/fetch/${encodeQiniu(sourceUrl)}/to/${encodeQiniu(`${config.bucket}:${pathname}`)}`;
+  const contentType = "application/x-www-form-urlencoded";
+  const signingData = `POST ${path}\nHost: ${QINIU_EAST_CHINA_IO}\nContent-Type: ${contentType}\n\n`;
+  const signature = encodeQiniu(createHmac("sha1", config.secretAccessKey).update(signingData).digest());
+  const response = await fetch(`https://${QINIU_EAST_CHINA_IO}${path}`, {
+    method: "POST",
+    headers: {
+      Authorization: `Qiniu ${config.accessKeyId}:${signature}`,
+      "Content-Type": contentType,
+    },
+    body: "",
+    redirect: "error",
+    signal: AbortSignal.timeout(40_000),
+  });
+  if (!response.ok) {
+    await response.body?.cancel();
+    throw new Error(`Qiniu archive fetch returned HTTP ${response.status}`);
+  }
+  const result = await response.json() as { key?: string; fsize?: number; mimeType?: string };
+  if (result.key !== pathname || !result.fsize || !result.mimeType?.startsWith("image/")) {
+    throw new Error("Qiniu archive fetch did not return an image");
+  }
+  const url = buildS3PublicUrl(config, pathname);
+  return {
+    url,
+    thumbnailUrl: url,
+    pathname,
+    contentType: result.mimeType,
+    bytes: result.fsize,
+    downloadAsset: result.mimeType === "image/png"
+      ? await getReadyQiniuImageDownload(url, target).catch(() => undefined)
+      : undefined,
+  };
+}

tests/routes/canvas-download-speed.test.ts

@@ -131,6 +131,7 @@ test("PNG preparation failure leaves the original archived and schedules a retry
   const module = serverLoader({
     "./image-archive-db": { claimImageDownloadTask: async () => ({ id: "task", targetUrl: "https://cdn.example/original.png", metadata: { downloadAsset: { attempts: 2 } } }), saveImageDownloadAsset: async (_id: string, asset: any) => { saved = asset; } },
     "./image-byte-cache": { fetchImageBytes: async () => { throw new Error("storage offline"); } },
+    "./qiniu-image-archive": { getReadyQiniuImageDownload: async () => undefined },
     "./service": {},
   })("lib/storage/image-download-preparation.ts");
   await module.prepareImageArchiveDownload("task");

tests/routes/image-archive-flow.test.ts

@@ -5,6 +5,7 @@ import path from "node:path";
 import vm from "node:vm";
 import * as crypto from "node:crypto";
 import ts from "typescript";
+import sharp from "sharp";
 import {
   collectImageArchiveSources,
   replaceArchivedImageUrls,
@@ -82,6 +83,52 @@ function serverLoader(mocks: Record<string, unknown>, env: Record<string, string
 }
 const loggerMock = { createLogger: () => ({ info() {}, warn() {}, error() {} }) };
 
+test("Qiniu archives 98K originals without relaying full image bytes through the app", async () => {
+  const image = sharp({ create: { width: 4, height: 4, channels: 3, background: "blue" } });
+  const rgb = await image.clone().withIccProfile("srgb").png().toBuffer();
+  const indexed = await image.clone().png({ palette: true }).toBuffer();
+  const config = { provider: "qiniu-kodo", region: "cn-east-1", bucket: "archive",
+    accessKeyId: "test-key", secretAccessKey: "test-secret", publicUrl: "https://bucket.example" };
+  const id = "test-task";
+  const key = `uploads/ai-generated-archive-${id}`;
+  const source = "https://api.vip98k.com/api/external/assets/test_asset";
+  const archiveUrl = `${config.publicUrl}/${key}`;
+  let stored = rgb;
+  const cloudCalls: string[] = [];
+  const reads: string[] = [];
+  const load = serverLoader({
+    "@/lib/storage/service": {
+      resolveS3UploadTarget: async () => ({ config }),
+      buildS3PublicUrl: (_: unknown, path: string) => `${config.publicUrl}/${path}`,
+      createS3Client: () => ({
+        send: async (cmd: { input: { Bucket: string; Key: string; Range: string } }) => {
+          expect(cmd.input).toEqual({ Bucket: config.bucket, Key: key, Range: "bytes=0-65535" });
+          reads.push(cmd.input.Range);
+          return { $metadata: { httpStatusCode: 206 }, ContentType: "image/png",
+            ContentLength: stored.length, ContentRange: `bytes 0-${stored.length - 1}/${stored.length}`,
+            Body: { transformToByteArray: async () => stored } };
+        },
+        destroy() {},
+      }),
+    },
+    fetch: async (url: string, options: RequestInit) => {
+      cloudCalls.push(url);
+      expect(options.method).toBe("POST");
+      const [, from, to] = new URL(url).pathname.split("/fetch/")[1].match(/^(.+)\/to\/(.+)$/)!;
+      expect(Buffer.from(from, "base64url").toString()).toBe(source);
+      expect(Buffer.from(to, "base64url").toString()).toBe(`${config.bucket}:${key}`);
+      return Response.json({ key, fsize: stored.length, mimeType: "image/png" });
+    },
+  })("lib/storage/qiniu-image-archive.ts");
+  const archived = await load.archiveImageThroughQiniu(source, id);
+  expect(archived.downloadAsset).toMatchObject({ status: "ready", url: archiveUrl, bytes: rgb.length });
+  stored = indexed;
+  expect((await load.archiveImageThroughQiniu(source, id)).downloadAsset).toBeUndefined();
+  expect(await load.archiveImageThroughQiniu("https://another-provider.example/result.png", id)).toBeNull();
+  expect(cloudCalls).toHaveLength(2);
+  expect(reads).toHaveLength(2);
+});
+
 test("160 requests from the same user and model start without admission waiting", async () => {
   const module = serverLoader({})("lib/canvas/image-concurrency.ts");
   let queued = 0;