perf(canvas): share image bytes and deliver prepared downloads through Qiniu CDN
This commit is contained in:
134
parent
47e1009ee0
commit
1a806b4275
@ -1,4 +1,6 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { resolveCanvasCdnDownload } from "@/lib/storage/canvas-cdn-download";
|
||||
import { prepareImageArchiveDownload } from "@/lib/storage/image-download-preparation";
|
||||
import { after, NextRequest, NextResponse } from "next/server";
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import {
|
||||
buildAttachmentContentDisposition,
|
||||
@ -60,6 +62,14 @@ export async function POST(request: NextRequest) {
|
||||
}
|
||||
|
||||
const name = sanitizeCanvasDownloadFileName(String(body.name || ""), "canvas-image");
|
||||
if (body.preferDirect === true) {
|
||||
const { task, direct } = await resolveCanvasCdnDownload(urls, `${name}.png`).catch(() => ({ task: null, direct: null }));
|
||||
if (direct) return NextResponse.json(direct, { headers: { "Cache-Control": "no-store" } });
|
||||
if (task?.targetUrl) {
|
||||
if (!urls.includes(task.targetUrl)) urls.unshift(task.targetUrl);
|
||||
after(() => prepareImageArchiveDownload(task.id));
|
||||
}
|
||||
}
|
||||
const cookie = request.headers.get("cookie") || "";
|
||||
let result: Awaited<ReturnType<typeof fetchCanvasDownloadAsset>> | null = null;
|
||||
const failures: string[] = [];
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import { writeDownloadZip, readDownloadZip, removeDownloadZip } from "@/lib/canvas/download-zip-stream";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import JSZip from "jszip";
|
||||
@ -27,7 +28,7 @@ type ZipTask = {
|
||||
status: ZipTaskStatus;
|
||||
stage: string;
|
||||
error?: string;
|
||||
zipBuffer?: Buffer;
|
||||
zipFile?: { path: string; bytes: number };
|
||||
contentDisposition?: string;
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
@ -116,6 +117,7 @@ function cleanupExpiredTasks() {
|
||||
for (const [taskId, task] of store.entries()) {
|
||||
if (task.expiresAt <= now) {
|
||||
store.delete(taskId);
|
||||
if (task.zipFile) void removeDownloadZip(task.zipFile.path);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -242,17 +244,10 @@ async function processZipTask(params: {
|
||||
|
||||
task.stage = "正在生成 ZIP 文件";
|
||||
task.updatedAt = Date.now();
|
||||
task.zipBuffer = await zip.generateAsync(
|
||||
{
|
||||
type: "nodebuffer",
|
||||
compression: "DEFLATE",
|
||||
compressionOptions: { level: 6 },
|
||||
},
|
||||
(metadata) => {
|
||||
task.compressPercent = Math.max(0, Math.min(100, metadata.percent || 0));
|
||||
task.updatedAt = Date.now();
|
||||
}
|
||||
);
|
||||
task.zipFile = await writeDownloadZip(zip, task.id, percent => {
|
||||
task.compressPercent = Math.max(0, Math.min(100, percent || 0));
|
||||
task.updatedAt = Date.now();
|
||||
});
|
||||
task.contentDisposition = buildContentDisposition(task.folderName);
|
||||
task.status = "completed";
|
||||
task.stage = "打包完成";
|
||||
@ -340,15 +335,15 @@ export async function GET(request: NextRequest) {
|
||||
}
|
||||
|
||||
if (request.nextUrl.searchParams.get("download") === "1") {
|
||||
if (task.status !== "completed" || !task.zipBuffer) {
|
||||
if (task.status !== "completed" || !task.zipFile) {
|
||||
return NextResponse.json({ success: false, error: "ZIP 文件尚未准备好" }, { status: 409 });
|
||||
}
|
||||
return new NextResponse(task.zipBuffer, {
|
||||
return new NextResponse(readDownloadZip(task.zipFile.path), {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": "application/zip",
|
||||
"Content-Disposition": task.contentDisposition || buildContentDisposition(task.folderName),
|
||||
"Content-Length": String(task.zipBuffer.length),
|
||||
"Content-Length": String(task.zipFile.bytes),
|
||||
"Cache-Control": "no-store",
|
||||
"X-Download-Success-Count": String(task.successCount),
|
||||
"X-Download-Failed-Count": String(task.failedCount),
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import { streamDownloadZip } from "@/lib/canvas/download-zip-stream";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import JSZip from "jszip";
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
@ -159,13 +160,7 @@ export async function POST(request: NextRequest) {
|
||||
console.warn("[CanvasDownloadZip] Partial download failures:", failures);
|
||||
}
|
||||
|
||||
const zipBuffer = await zip.generateAsync({
|
||||
type: "nodebuffer",
|
||||
compression: "DEFLATE",
|
||||
compressionOptions: { level: 6 },
|
||||
});
|
||||
|
||||
return new NextResponse(zipBuffer, {
|
||||
return new NextResponse(streamDownloadZip(zip), {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": "application/zip",
|
||||
|
||||
@ -4463,7 +4463,7 @@ export function CanvasChatPanel({
|
||||
signal: controller.signal,
|
||||
onProgress: setDownloadProgress,
|
||||
});
|
||||
toast.success("原图下载成功");
|
||||
toast.success("原图已开始下载");
|
||||
} catch (error) {
|
||||
if (controller.signal.aborted) {
|
||||
toast.message("已取消下载");
|
||||
|
||||
@ -8335,7 +8335,7 @@ export function ImageNode({
|
||||
signal: controller.signal,
|
||||
onProgress: setDownloadProgress,
|
||||
});
|
||||
toast.success("原图下载成功");
|
||||
toast.success("原图已开始下载");
|
||||
} catch (error) {
|
||||
if (controller.signal.aborted) {
|
||||
toast.message("已取消下载");
|
||||
|
||||
19
docs/canvas-download-cdn.md
Normal file
19
docs/canvas-download-cdn.md
Normal file
@ -0,0 +1,19 @@
|
||||
# Canvas image downloads through Qiniu CDN
|
||||
|
||||
Generation still returns the provider preview first and archives the original asynchronously, without model admission limits. Archiving and downloads share a one-hour disk cache and a single in-flight source request. The cache hashes identifiers, isolates authenticated same-origin responses by cookie digest, uses private files, and is expendable.
|
||||
|
||||
After storing the original, ImageArchiveTask.metadata.downloadAsset queues PNG preparation independently from URL replacement. Compatible 8-bit RGB/RGBA PNG files are reused byte-for-byte; other images are normalized once without resizing. Failures retry independently with leases and do not revert the original. Older archived images are prepared on demand. The download endpoint remains authenticated and supports its original binary contract. Clients opt into a small JSON response for ready PNG objects from the configured Qiniu origin; the browser then downloads directly using Kodo's attname filename parameter. Unready files use the shared cache and binary fallback.
|
||||
|
||||
Production settings (no credentials):
|
||||
|
||||
- Qiniu bucket: gongxiangpan-sc
|
||||
- StorageConfig.publicUrl: https://scqnycdn.jiansun.vip
|
||||
- CANVAS_DOWNLOAD_CDN_ORIGIN=https://scqnycdn.jiansun.vip (download iframe CSP)
|
||||
- NEXT_PUBLIC_CANVAS_CDN_ORIGIN=https://scqnycdn.jiansun.vip (canvas display delivery; requires build)
|
||||
- CANVAS_QINIU_ORIGIN_URL=https://scqny.jiansun.vip (map older ready originals to the same bucket's CDN)
|
||||
|
||||
The CDN keeps query parameters in its cache key so image processing and attname filenames cannot collide. HTTPS, HTTP/2, byte ranges and CORS response headers are enabled. Existing source URLs are retained. Credentials stay in the configured storage/management credentials and must never be committed.
|
||||
|
||||
Batch ZIPs reuse prepared bytes, use STORE for compressed image/video files, and stream the archive. Task ZIP files expire and reserve at least 1 GiB of server disk space. Files remain available for authenticated task owners only.
|
||||
|
||||
Validation: pnpm exec playwright test -c tests/canvas-download-speed.config.ts and pnpm exec tsc --noEmit. Build off-server with webpack; deploy only against the current production source to retain flow-board updates. Rollback needs the previous build, environment file and StorageConfig.publicUrl.
|
||||
37
lib/canvas/download-zip-stream.ts
Normal file
37
lib/canvas/download-zip-stream.ts
Normal file
@ -0,0 +1,37 @@
|
||||
import "server-only";
|
||||
import { createReadStream, createWriteStream } from "node:fs";
|
||||
import { mkdir, stat, statfs, unlink, readdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { Readable, Transform } from "node:stream";
|
||||
import JSZip from "jszip";
|
||||
const directory = join(tmpdir(), "canvas-download-zips-v1");
|
||||
export async function writeDownloadZip(zip: JSZip, id: string, progress: (percent: number) => void) {
|
||||
await mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
// Remove abandoned files after restart as well as expired in-process tasks.
|
||||
for (const name of await readdir(directory)) {
|
||||
const path = join(directory, name);
|
||||
if ((await stat(path)).mtimeMs < Date.now() - 60 * 60_000) await unlink(path).catch(() => undefined);
|
||||
}
|
||||
const path = join(directory, `${id}.zip`);
|
||||
try {
|
||||
const disk = await statfs(directory);
|
||||
const budget = disk.bavail * disk.bsize - 1024 * 1024 * 1024;
|
||||
let written = 0;
|
||||
const guard = new Transform({ transform(chunk, _encoding, callback) {
|
||||
written += chunk.length;
|
||||
callback(written > budget ? new Error("服务器下载空间不足，请减少本次图片数量") : null, chunk);
|
||||
} });
|
||||
await pipeline(zip.generateNodeStream({ type: "nodebuffer", streamFiles: true, compression: "STORE" },
|
||||
metadata => progress(metadata.percent)), guard, createWriteStream(path, { mode: 0o600 }));
|
||||
return { path, bytes: (await stat(path)).size };
|
||||
} catch (error) { await unlink(path).catch(() => undefined); throw error; }
|
||||
}
|
||||
export function readDownloadZip(path: string) {
|
||||
return Readable.toWeb(createReadStream(path)) as ReadableStream<Uint8Array>;
|
||||
}
|
||||
export function streamDownloadZip(zip: JSZip) {
|
||||
return Readable.toWeb(zip.generateNodeStream({ type: "nodebuffer", streamFiles: true, compression: "STORE" }) as Readable) as ReadableStream<Uint8Array>;
|
||||
}
|
||||
export async function removeDownloadZip(path: string) { await unlink(path).catch(() => undefined); }
|
||||
@ -1,3 +1,5 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { fetchImageBytes, onceImageBytes } from "@/lib/storage/image-byte-cache";
|
||||
import { extractImageProxySourceUrl, inferOriginalImageUrl } from "@/lib/image-url";
|
||||
|
||||
type DownloadFileItem = {
|
||||
@ -21,25 +23,11 @@ export type CanvasDownloadAssetResult =
|
||||
|
||||
const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_FETCH_RETRY_COUNT = 3;
|
||||
const RETRYABLE_STATUS_CODES = new Set([408, 409, 425, 429, 500, 502, 503, 504]);
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function fetchWithTimeout(url: string, init: RequestInit, timeoutMs: number): Promise<Response> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
return await fetch(url, {
|
||||
...init,
|
||||
signal: controller.signal,
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeDownloadError(error: unknown): string {
|
||||
if (error instanceof Error) return error.message || "download failed";
|
||||
if (typeof error === "string") return error;
|
||||
@ -214,7 +202,7 @@ function detectAsset(bytes: Buffer, contentType: string): { kind: "image" | "vid
|
||||
return null;
|
||||
}
|
||||
|
||||
async function convertImageToPhotoshopPng(bytes: Buffer): Promise<Buffer> {
|
||||
export async function convertImageToPhotoshopPng(bytes: Buffer): Promise<Buffer> {
|
||||
const sharp = await import("sharp");
|
||||
const image = sharp.default(bytes, {
|
||||
animated: false,
|
||||
@ -225,7 +213,10 @@ async function convertImageToPhotoshopPng(bytes: Buffer): Promise<Buffer> {
|
||||
if (!metadata.width || !metadata.height) {
|
||||
throw new Error("无法解析图片尺寸");
|
||||
}
|
||||
return image.rotate().toColorspace("srgb").png({ compressionLevel: 6, adaptiveFiltering: true }).toBuffer();
|
||||
// RGB/RGBA, 8-bit, single-frame PNG is already compatible; preserve its exact bytes.
|
||||
if (metadata.format === "png" && metadata.space === "srgb" && metadata.depth === "uchar"
|
||||
&& !metadata.isPalette && (metadata.pages || 1) === 1 && (!metadata.orientation || metadata.orientation === 1)) return bytes;
|
||||
return image.rotate().toColorspace("srgb").png({ compressionLevel: 3, adaptiveFiltering: true }).toBuffer();
|
||||
}
|
||||
|
||||
function buildCanvasAssetFilename(params: {
|
||||
@ -245,7 +236,7 @@ function buildCanvasAssetFilename(params: {
|
||||
return `${sequencePrefix}-${baseName}.${params.ext}`;
|
||||
}
|
||||
|
||||
async function normalizeCanvasDownloadAsset(params: {
|
||||
export async function normalizeCanvasDownloadAsset(params: {
|
||||
item: DownloadFileItem;
|
||||
index: number;
|
||||
bytes: Buffer;
|
||||
@ -278,7 +269,10 @@ async function normalizeCanvasDownloadAsset(params: {
|
||||
};
|
||||
}
|
||||
|
||||
const pngBytes = await convertImageToPhotoshopPng(params.bytes);
|
||||
const digest = createHash("sha256").update(params.bytes).digest("hex");
|
||||
const { buffer: pngBytes } = await onceImageBytes(`photoshop:${digest}`, async () => ({
|
||||
buffer: await convertImageToPhotoshopPng(params.bytes), contentType: "image/png",
|
||||
}));
|
||||
return {
|
||||
ok: true,
|
||||
filename: buildCanvasAssetFilename({
|
||||
@ -312,32 +306,18 @@ export async function fetchCanvasDownloadAsset(params: {
|
||||
for (const candidateUrl of candidateUrls) {
|
||||
for (let attempt = 1; attempt <= retryCount; attempt += 1) {
|
||||
try {
|
||||
const response = await fetchWithTimeout(
|
||||
candidateUrl,
|
||||
{
|
||||
method: "GET",
|
||||
cache: "no-store",
|
||||
redirect: "follow",
|
||||
headers: buildRemoteFetchHeaders(candidateUrl, params.requestOrigin, params.cookie),
|
||||
},
|
||||
timeoutMs
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
lastError = `HTTP ${response.status}`;
|
||||
if (attempt < retryCount && RETRYABLE_STATUS_CODES.has(response.status)) {
|
||||
await sleep(350 * attempt);
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
const bytes = Buffer.from(await response.arrayBuffer());
|
||||
const data = await fetchImageBytes(candidateUrl, {
|
||||
timeoutMs,
|
||||
headers: buildRemoteFetchHeaders(candidateUrl, params.requestOrigin, params.cookie),
|
||||
// Authenticated local assets must never share a cache across users.
|
||||
scope: new URL(candidateUrl).origin === params.requestOrigin && params.cookie
|
||||
? createHash("sha256").update(params.cookie).digest("hex") : "",
|
||||
});
|
||||
return await normalizeCanvasDownloadAsset({
|
||||
item: params.file,
|
||||
index: params.index,
|
||||
bytes,
|
||||
contentType: response.headers.get("content-type") || "",
|
||||
bytes: data.buffer,
|
||||
contentType: data.contentType,
|
||||
includeSequencePrefix: params.includeSequencePrefix !== false,
|
||||
});
|
||||
} catch (error) {
|
||||
|
||||
@ -256,6 +256,7 @@ export async function downloadCanvasImageAsPng({
|
||||
url: urls[0],
|
||||
urls,
|
||||
name: filenameBase,
|
||||
preferDirect: true,
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
@ -300,6 +301,20 @@ export async function downloadCanvasImageAsPng({
|
||||
);
|
||||
}
|
||||
|
||||
if (response.headers.get("content-type")?.includes("application/json")) {
|
||||
const result = await response.json();
|
||||
if (result.delivery !== "cdn" || typeof result.downloadUrl !== "string"
|
||||
|| new URL(result.downloadUrl).protocol !== "https:") throw new Error("下载链接无效");
|
||||
if (signal?.aborted) throw signal.reason || new DOMException("已取消下载", "AbortError");
|
||||
const frame = document.createElement("iframe");
|
||||
frame.hidden = true;
|
||||
frame.src = result.downloadUrl;
|
||||
document.body.appendChild(frame);
|
||||
window.setTimeout(() => frame.remove(), 10 * 60_000);
|
||||
onProgress?.({ phase: "downloading", message: "已开始下载，可在浏览器下载列表查看进度" });
|
||||
return { filename: result.filename, bytes: result.bytes || 0 };
|
||||
}
|
||||
|
||||
const filename =
|
||||
parseAttachmentFilename(response.headers.get("content-disposition")) ||
|
||||
`${filenameBase}.png`;
|
||||
|
||||
@ -44,7 +44,10 @@ function isAliyunOssImageHost(hostname: string): boolean {
|
||||
|
||||
function isTrustedDirectStorageImageHost(hostname: string): boolean {
|
||||
const normalized = hostname.trim().toLowerCase();
|
||||
let configuredCdnHost = "";
|
||||
try { configuredCdnHost = new URL(process.env.NEXT_PUBLIC_CANVAS_CDN_ORIGIN || "").hostname; } catch { /* optional */ }
|
||||
return (
|
||||
(configuredCdnHost !== "" && normalized === configuredCdnHost) ||
|
||||
normalized === "yybb.top" ||
|
||||
normalized.endsWith(".qiniucs.com") ||
|
||||
normalized.endsWith(".qiniudn.com") ||
|
||||
|
||||
@ -1,8 +1,10 @@
|
||||
import { preparePendingImageDownloads } from "@/lib/storage/image-download-preparation";
|
||||
import "server-only";
|
||||
|
||||
import { archiveTemporaryImage } from "@/lib/storage/image-archive";
|
||||
import {
|
||||
getImageArchiveTaskStats,
|
||||
hasPendingImageDownloads,
|
||||
recoverStaleImageArchiveTasks,
|
||||
getImageArchiveTasksToReplace,
|
||||
getImageArchiveTasksToStore,
|
||||
@ -60,9 +62,10 @@ export async function runImageArchiveJob() {
|
||||
};
|
||||
|
||||
// Bucket work has a small worker pool independent of unrestricted model generation.
|
||||
for (let index = 0; index < storeTasks.length; index += 4) {
|
||||
await Promise.all(storeTasks.slice(index, index + 4).map(storeOne));
|
||||
}
|
||||
let storeCursor = 0;
|
||||
await Promise.all(Array.from({ length: Math.min(4, storeTasks.length) }, async () => {
|
||||
while (storeCursor < storeTasks.length) await storeOne(storeTasks[storeCursor++]);
|
||||
}));
|
||||
const replaceTasks = await getImageArchiveTasksToReplace({ limit: 50 });
|
||||
let replaceSucceeded = 0;
|
||||
let replaceFailed = 0;
|
||||
@ -103,6 +106,7 @@ export async function runImageArchiveJob() {
|
||||
}
|
||||
}
|
||||
|
||||
const downloadTasks = await preparePendingImageDownloads();
|
||||
const stats = await getImageArchiveTaskStats();
|
||||
return {
|
||||
success: true,
|
||||
@ -118,5 +122,7 @@ export async function runImageArchiveJob() {
|
||||
failed: replaceFailed,
|
||||
},
|
||||
stats,
|
||||
downloadTasks,
|
||||
pendingDownloads: await hasPendingImageDownloads(),
|
||||
};
|
||||
}
|
||||
|
||||
26
lib/storage/canvas-cdn-download.ts
Normal file
26
lib/storage/canvas-cdn-download.ts
Normal file
@ -0,0 +1,26 @@
|
||||
import "server-only";
|
||||
import { getDefaultStorageConfig } from "@/lib/db/queries";
|
||||
import { findImageArchiveDownloadTask } from "./image-archive-db";
|
||||
import { extractImageProxySourceUrl, inferOriginalImageUrl } from "@/lib/image-url";
|
||||
|
||||
export async function resolveCanvasCdnDownload(urls: string[], filename: string) {
|
||||
const sources = Array.from(new Set(urls.flatMap(url => {
|
||||
const source = extractImageProxySourceUrl(url) || url;
|
||||
return [source, inferOriginalImageUrl(source) || source];
|
||||
})));
|
||||
const task = await findImageArchiveDownloadTask(sources);
|
||||
const asset = (task?.metadata as { downloadAsset?: { status?: string; url?: string; contentType?: string; bytes?: number } } | null)?.downloadAsset;
|
||||
if (asset?.status !== "ready" || !asset.url || asset.contentType !== "image/png") return { task, direct: null };
|
||||
const config = await getDefaultStorageConfig();
|
||||
if (config?.type !== "qiniu-kodo" || !config.publicUrl || !config.isEnabled) return { task, direct: null };
|
||||
const url = new URL(asset.url);
|
||||
const base = new URL(config.publicUrl);
|
||||
const origin = process.env.CANVAS_QINIU_ORIGIN_URL;
|
||||
if (origin && url.origin === new URL(origin).origin) url.host = base.host;
|
||||
if (base.protocol !== "https:" || url.origin !== base.origin
|
||||
|| !(url.pathname.startsWith(base.pathname.replace(/\/$/, "") + "/"))) return { task, direct: null };
|
||||
// attname is interpreted by Kodo/CDN as Content-Disposition, including Chinese
|
||||
// filenames. Keep all processing/auth parameters intact for correct cache keys.
|
||||
url.searchParams.set("attname", filename);
|
||||
return { task, direct: { delivery: "cdn", downloadUrl: url.toString(), filename, bytes: asset.bytes || 0 } };
|
||||
}
|
||||
@ -142,6 +142,7 @@ export async function markImageArchiveTaskStored(params: {
|
||||
status: "stored",
|
||||
targetUrl: params.targetUrl,
|
||||
targetStorageConfigId: params.targetStorageConfigId || null,
|
||||
metadata: sql`coalesce(${imageArchiveTask.metadata}, '{}'::jsonb) || jsonb_build_object('downloadAsset', jsonb_build_object('status', 'pending'))`,
|
||||
archivedAt: new Date(),
|
||||
replaceAfterAt: new Date(),
|
||||
lastError: null,
|
||||
@ -313,3 +314,47 @@ export async function resolveArchivedImageUrls<T>(value: T): Promise<T> {
|
||||
if (!sources.length) return value;
|
||||
return replaceArchivedImageUrls(value, await getArchivedImageUrlMappings(sources));
|
||||
}
|
||||
|
||||
// Download assets use an independent JSONB lease; original archiving/replacement
|
||||
// can finish even if PNG preparation or thumbnail storage is temporarily unavailable.
|
||||
export async function findImageArchiveDownloadTask(urls: string[]) {
|
||||
const [task] = await db.select().from(imageArchiveTask).where(or(
|
||||
inArray(imageArchiveTask.sourceUrl, urls), inArray(imageArchiveTask.targetUrl, urls),
|
||||
)).limit(1);
|
||||
if (task?.targetUrl && !(task.metadata as { downloadAsset?: unknown } | null)?.downloadAsset) {
|
||||
await db.update(imageArchiveTask).set({
|
||||
metadata: sql`coalesce(${imageArchiveTask.metadata}, '{}'::jsonb) || jsonb_build_object('downloadAsset', jsonb_build_object('status', 'pending'))`,
|
||||
}).where(and(eq(imageArchiveTask.id, task.id), sql`${imageArchiveTask.metadata}->'downloadAsset' is null`));
|
||||
}
|
||||
return task || null;
|
||||
}
|
||||
export async function hasPendingImageDownloads() {
|
||||
const [task] = await db.select({ id: imageArchiveTask.id }).from(imageArchiveTask).where(
|
||||
sql`${imageArchiveTask.metadata}->'downloadAsset'->>'status' in ('pending', 'preparing', 'failed')`,
|
||||
).limit(1);
|
||||
return Boolean(task);
|
||||
}
|
||||
function downloadPreparationDue() {
|
||||
return and(
|
||||
sql`${imageArchiveTask.targetUrl} is not null`,
|
||||
sql`(${imageArchiveTask.metadata}->'downloadAsset'->>'status' = 'pending'
|
||||
or (${imageArchiveTask.metadata}->'downloadAsset'->>'status' in ('preparing', 'failed')
|
||||
and (${imageArchiveTask.metadata}->'downloadAsset'->>'retryAt')::timestamptz <= now()))`,
|
||||
);
|
||||
}
|
||||
export async function getImageDownloadTasks(limit = 4) {
|
||||
return db.select().from(imageArchiveTask).where(downloadPreparationDue())
|
||||
.orderBy(asc(imageArchiveTask.createdAt)).limit(limit);
|
||||
}
|
||||
export async function claimImageDownloadTask(id: string) {
|
||||
const [task] = await db.update(imageArchiveTask).set({
|
||||
metadata: sql`coalesce(${imageArchiveTask.metadata}, '{}'::jsonb) || jsonb_build_object('downloadAsset',
|
||||
jsonb_build_object('status', 'preparing', 'attempts', coalesce((${imageArchiveTask.metadata}->'downloadAsset'->>'attempts')::int, 0) + 1, 'retryAt', now() + interval '10 minutes'))`,
|
||||
}).where(and(eq(imageArchiveTask.id, id), downloadPreparationDue())).returning();
|
||||
return task || null;
|
||||
}
|
||||
export async function saveImageDownloadAsset(id: string, asset: Record<string, unknown>) {
|
||||
await db.update(imageArchiveTask).set({
|
||||
metadata: sql`coalesce(${imageArchiveTask.metadata}, '{}'::jsonb) || jsonb_build_object('downloadAsset', ${JSON.stringify(asset)}::jsonb)`,
|
||||
}).where(eq(imageArchiveTask.id, id));
|
||||
}
|
||||
|
||||
@ -32,9 +32,9 @@ export function scheduleImageArchiveWorker(deliveryConfirmed = false) {
|
||||
do {
|
||||
worker.requested = false;
|
||||
const result = await runImageArchiveJob();
|
||||
retryNeeded = result.stats.pendingStore > 0 || result.stats.pendingReplace > 0;
|
||||
retryNeeded = result.stats.pendingStore > 0 || result.stats.pendingReplace > 0 || result.pendingDownloads === true;
|
||||
// Stop when only backoff tasks remain; do not spin while storage is down.
|
||||
if (!worker.requested && result.store.total === 0 && result.replace.total === 0) break;
|
||||
if (!worker.requested && result.store.total === 0 && result.replace.total === 0 && result.downloadTasks === 0) break;
|
||||
} while (true);
|
||||
} catch (error) {
|
||||
retryNeeded = true;
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import { fetchImageBytes, cacheImageBytes } from "@/lib/storage/image-byte-cache";
|
||||
import "server-only";
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import sharp from "sharp";
|
||||
import { isGeneratedImageSource } from "./image-archive-urls";
|
||||
import { scheduleImageArchiveWorker } from "./image-archive-worker";
|
||||
import { createLogger } from "@/lib/observability/logger";
|
||||
@ -170,42 +170,15 @@ function toArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||||
}
|
||||
|
||||
export async function downloadTemporaryImage(params: { sourceUrl: string; retries?: number }) {
|
||||
if (params.sourceUrl.startsWith("data:")) {
|
||||
const match = params.sourceUrl.match(/^data:(image\/[a-z0-9.+-]+);base64,([a-z0-9+/=]+)$/i);
|
||||
if (!match) throw new Error("Invalid inline generated image");
|
||||
return {
|
||||
buffer: Buffer.from(match[2], "base64"),
|
||||
contentType: match[1],
|
||||
extension: getImageExtension(match[1], ""),
|
||||
};
|
||||
}
|
||||
const retries = Math.max(0, Math.min(5, params.retries ?? 2));
|
||||
|
||||
for (let attempt = 0; attempt <= retries; attempt += 1) {
|
||||
try {
|
||||
const response = await fetch(params.sourceUrl, {
|
||||
signal: AbortSignal.timeout(getDownloadTimeoutMs()),
|
||||
headers: {
|
||||
"User-Agent": "AI-Chatbot-Image-Archiver/1.0",
|
||||
Accept: "image/avif,image/webp,image/png,image/jpeg,image/*,*/*;q=0.8",
|
||||
},
|
||||
const data = await fetchImageBytes(params.sourceUrl, {
|
||||
timeoutMs: getDownloadTimeoutMs(),
|
||||
headers: { Accept: "image/*,*/*;q=0.8" },
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const contentType = response.headers.get("content-type");
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
if (!arrayBuffer || arrayBuffer.byteLength === 0) {
|
||||
throw new Error("Downloaded image is empty");
|
||||
}
|
||||
|
||||
return {
|
||||
buffer: Buffer.from(arrayBuffer),
|
||||
contentType,
|
||||
extension: getImageExtension(contentType, params.sourceUrl),
|
||||
};
|
||||
return { ...data, extension: getImageExtension(data.contentType, params.sourceUrl) };
|
||||
} catch (error) {
|
||||
if (attempt >= retries) throw error;
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000 * (attempt + 1)));
|
||||
@ -220,25 +193,10 @@ export async function archiveTemporaryImage(sourceUrl: string) {
|
||||
const objectSuffix = `${Date.now()}-${randomUUID()}`;
|
||||
const filename = `ai-generated-archive-${objectSuffix}.${downloaded.extension}`;
|
||||
const uploaded = await uploadFile(filename, toArrayBuffer(downloaded.buffer));
|
||||
const thumbnailBuffer = await sharp(downloaded.buffer, { failOn: "none" })
|
||||
.resize(640, 640, { fit: "inside", withoutEnlargement: true })
|
||||
.webp({ quality: 82 })
|
||||
.toBuffer()
|
||||
.catch(() => null);
|
||||
const thumbnailUploaded = thumbnailBuffer
|
||||
? await uploadFile(
|
||||
`ai-generated-archive-thumb-${objectSuffix}.webp`,
|
||||
toArrayBuffer(thumbnailBuffer),
|
||||
).catch((error) => {
|
||||
logger.warn("generated image thumbnail upload failed", {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
});
|
||||
return null;
|
||||
})
|
||||
: null;
|
||||
await cacheImageBytes(`source::${uploaded.url}`, downloaded);
|
||||
return {
|
||||
url: uploaded.url,
|
||||
thumbnailUrl: thumbnailUploaded?.url || uploaded.url,
|
||||
thumbnailUrl: uploaded.url,
|
||||
pathname: uploaded.pathname,
|
||||
contentType: uploaded.contentType,
|
||||
bytes: downloaded.buffer.length,
|
||||
|
||||
90
lib/storage/image-byte-cache.ts
Normal file
90
lib/storage/image-byte-cache.ts
Normal file
@ -0,0 +1,90 @@
|
||||
import "server-only";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { mkdir, readFile, writeFile, rename, readdir, stat, statfs, unlink } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
export type ImageBytes = { buffer: Buffer; contentType: string };
|
||||
const directory = process.env.CANVAS_IMAGE_CACHE_DIR || join(tmpdir(), "canvas-image-bytes-v1");
|
||||
const ttl = 60 * 60 * 1000;
|
||||
const maxBytes = 512 * 1024 * 1024;
|
||||
const state = globalThis as typeof globalThis & {
|
||||
__imageByteFlights?: Map<string, Promise<ImageBytes>>;
|
||||
__imageCacheCleanupAt?: number;
|
||||
};
|
||||
const flights = (state.__imageByteFlights ??= new Map());
|
||||
function key(value: string) { return createHash("sha256").update(value).digest("hex"); }
|
||||
|
||||
async function cleanup() {
|
||||
if ((state.__imageCacheCleanupAt || 0) > Date.now() - 60_000) return;
|
||||
state.__imageCacheCleanupAt = Date.now();
|
||||
const files = await Promise.all((await readdir(directory)).map(async (name) => {
|
||||
const path = join(directory, name);
|
||||
return { path, ...(await stat(path)) };
|
||||
}));
|
||||
let total = files.reduce((sum, file) => sum + file.size, 0);
|
||||
for (const file of files.sort((a, b) => a.mtimeMs - b.mtimeMs)) {
|
||||
if (file.mtimeMs < Date.now() - ttl || total > maxBytes) {
|
||||
await unlink(file.path).catch(() => undefined);
|
||||
total -= file.size;
|
||||
}
|
||||
}
|
||||
}
|
||||
export async function cacheImageBytes(identity: string, data: ImageBytes) {
|
||||
if (data.buffer.length > maxBytes / 4) return;
|
||||
try {
|
||||
await mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
const disk = await statfs(directory);
|
||||
if (disk.bavail * disk.bsize < data.buffer.length + 1024 * 1024 * 1024) return;
|
||||
// One atomic file keeps metadata and bytes together, even across process restarts.
|
||||
const header = Buffer.from(JSON.stringify({ type: data.contentType, expires: Date.now() + ttl }) + "\n");
|
||||
const path = join(directory, key(identity));
|
||||
const temp = `${path}.${randomUUID()}.tmp`;
|
||||
try {
|
||||
await writeFile(temp, Buffer.concat([header, data.buffer]), { mode: 0o600 });
|
||||
await rename(temp, path);
|
||||
} finally { await unlink(temp).catch(() => undefined); }
|
||||
await cleanup();
|
||||
} catch { /* A full/unavailable cache must never fail generation or downloads. */ }
|
||||
}
|
||||
export async function readImageBytes(identity: string): Promise<ImageBytes | null> {
|
||||
try {
|
||||
const bytes = await readFile(join(directory, key(identity)));
|
||||
const end = bytes.indexOf(10);
|
||||
if (end < 0 || end > 1024) return null;
|
||||
const header = JSON.parse(bytes.subarray(0, end).toString());
|
||||
if (header.expires <= Date.now()) return null;
|
||||
return { buffer: bytes.subarray(end + 1), contentType: header.type };
|
||||
} catch { return null; }
|
||||
}
|
||||
export async function onceImageBytes(identity: string, load: () => Promise<ImageBytes>): Promise<ImageBytes> {
|
||||
const existing = flights.get(identity);
|
||||
if (existing) return existing;
|
||||
const promise = (async () => {
|
||||
const cached = await readImageBytes(identity);
|
||||
if (cached) return cached;
|
||||
const result = await load();
|
||||
await cacheImageBytes(identity, result);
|
||||
return result;
|
||||
})();
|
||||
flights.set(identity, promise);
|
||||
try { return await promise; } finally { if (flights.get(identity) === promise) flights.delete(identity); }
|
||||
}
|
||||
export async function fetchImageBytes(url: string, options: { headers?: HeadersInit; timeoutMs?: number; scope?: string } = {}) {
|
||||
return onceImageBytes(`source:${options.scope || ""}:${url}`, async () => {
|
||||
if (url.startsWith("data:")) {
|
||||
const match = url.match(/^data:(image\/[a-z0-9.+-]+);base64,([a-z0-9+/=]+)$/i);
|
||||
if (!match) throw new Error("Invalid inline generated image");
|
||||
return { buffer: Buffer.from(match[2], "base64"), contentType: match[1] };
|
||||
}
|
||||
const response = await fetch(url, {
|
||||
headers: options.headers, cache: "no-store", redirect: "follow",
|
||||
// The deadline covers the body as well as the response headers.
|
||||
signal: AbortSignal.timeout(options.timeoutMs || 60_000),
|
||||
});
|
||||
if (!response.ok) { await response.body?.cancel(); throw new Error(`HTTP ${response.status}`); }
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
if (!buffer.length) throw new Error("Downloaded image is empty");
|
||||
return { buffer, contentType: response.headers.get("content-type") || "" };
|
||||
});
|
||||
}
|
||||
38
lib/storage/image-download-preparation.ts
Normal file
38
lib/storage/image-download-preparation.ts
Normal file
@ -0,0 +1,38 @@
|
||||
import "server-only";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { fetchImageBytes, cacheImageBytes } from "./image-byte-cache";
|
||||
import { normalizeCanvasDownloadAsset } from "@/lib/canvas/photoshop-safe-download";
|
||||
import { claimImageDownloadTask, saveImageDownloadAsset, getImageDownloadTasks } from "./image-archive-db";
|
||||
import { uploadFile } from "./service";
|
||||
|
||||
export async function prepareImageArchiveDownload(id: string) {
|
||||
const task = await claimImageDownloadTask(id);
|
||||
if (!task?.targetUrl) return;
|
||||
try {
|
||||
const source = await fetchImageBytes(task.targetUrl);
|
||||
const asset = await normalizeCanvasDownloadAsset({
|
||||
item: { url: task.targetUrl }, index: 0, bytes: source.buffer,
|
||||
contentType: source.contentType, includeSequencePrefix: false,
|
||||
});
|
||||
if (!asset.ok) throw new Error(asset.error);
|
||||
// An already compatible PNG shares the original object; other formats get
|
||||
// one prepared PNG, without changing the original dimensions or transparency.
|
||||
const url = source.buffer.equals(asset.bytes) && asset.contentType === "image/png"
|
||||
? task.targetUrl
|
||||
: (await uploadFile(`canvas-download-${randomUUID()}.${asset.filename.split(".").pop() || "png"}`, Uint8Array.from(asset.bytes).buffer)).url;
|
||||
await saveImageDownloadAsset(id, { status: "ready", url, bytes: asset.bytes.length,
|
||||
contentType: asset.contentType, preparedAt: new Date().toISOString() });
|
||||
await cacheImageBytes(`source::${url}`, { buffer: asset.bytes, contentType: asset.contentType });
|
||||
} catch {
|
||||
const attempts = Number((task.metadata as { downloadAsset?: { attempts?: number } } | null)?.downloadAsset?.attempts || 1);
|
||||
await saveImageDownloadAsset(id, { status: attempts >= 12 ? "abandoned" : "failed", attempts, retryAt: new Date(Date.now() + 10 * 60_000).toISOString() });
|
||||
}
|
||||
}
|
||||
export async function preparePendingImageDownloads() {
|
||||
const tasks = await getImageDownloadTasks();
|
||||
let cursor = 0;
|
||||
await Promise.all(Array.from({ length: Math.min(2, tasks.length) }, async () => {
|
||||
while (cursor < tasks.length) await prepareImageArchiveDownload(tasks[cursor++].id);
|
||||
}));
|
||||
return tasks.length;
|
||||
}
|
||||
@ -92,7 +92,9 @@ const nextConfig: NextConfig = {
|
||||
// HTTP 安全头
|
||||
async headers() {
|
||||
const isDev = process.env.NODE_ENV === "development";
|
||||
const frameSources = resolvePascalEditorFrameSources(isDev);
|
||||
const downloadCdn = process.env.CANVAS_DOWNLOAD_CDN_ORIGIN || "";
|
||||
const downloadFrameSource = /^https:\/\/[^/\s]+\/?$/.test(downloadCdn) ? new URL(downloadCdn).origin : "";
|
||||
const frameSources = [resolvePascalEditorFrameSources(isDev), downloadFrameSource].filter(Boolean).join(" ");
|
||||
const devMediaOrigins = resolveDevMediaOrigins(isDev);
|
||||
const csp = [
|
||||
"default-src 'self'",
|
||||
|
||||
2
tests/canvas-download-speed.config.ts
Normal file
2
tests/canvas-download-speed.config.ts
Normal file
@ -0,0 +1,2 @@
|
||||
import { defineConfig } from "@playwright/test";
|
||||
export default defineConfig({ testDir: "./routes", testMatch: /(?:canvas-download-speed|canvas-download-rate-limit|image-archive-flow)\.test\.ts/, workers: 1, reporter: "list", timeout: 30_000 });
|
||||
162
tests/routes/canvas-download-speed.test.ts
Normal file
162
tests/routes/canvas-download-speed.test.ts
Normal file
@ -0,0 +1,162 @@
|
||||
import { test, expect } from "@playwright/test";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import path from "node:path";
|
||||
import vm from "node:vm";
|
||||
import * as crypto from "node:crypto";
|
||||
import ts from "typescript";
|
||||
import sharp from "sharp";
|
||||
import JSZip from "jszip";
|
||||
const root = path.resolve(__dirname, "../..");
|
||||
function serverLoader(mocks: Record<string, unknown>, env: Record<string, string> = {}) {
|
||||
const modules = new Map<string, { exports: any }>();
|
||||
const runtimeGlobal = {};
|
||||
const load = (relative: string): any => {
|
||||
const filename = path.resolve(root, relative);
|
||||
if (modules.has(filename)) return modules.get(filename)!.exports;
|
||||
const module = { exports: {} };
|
||||
modules.set(filename, module);
|
||||
const source = ts.transpileModule(readFileSync(filename, "utf8"), {
|
||||
compilerOptions: {
|
||||
module: ts.ModuleKind.CommonJS,
|
||||
target: ts.ScriptTarget.ES2022,
|
||||
esModuleInterop: true,
|
||||
},
|
||||
}).outputText;
|
||||
const nativeRequire = createRequire(filename);
|
||||
const requireMock = (name: string) => {
|
||||
if (name === "server-only") return {};
|
||||
if (Object.prototype.hasOwnProperty.call(mocks, name)) return mocks[name];
|
||||
if (name.startsWith("@/")) return load(`${name.slice(2)}.ts`);
|
||||
if (name.startsWith("."))
|
||||
return load(path.relative(root, path.resolve(path.dirname(filename), `${name}.ts`)));
|
||||
return nativeRequire(name);
|
||||
};
|
||||
const context = vm.createContext({
|
||||
module,
|
||||
exports: module.exports,
|
||||
require: requireMock,
|
||||
globalThis: runtimeGlobal,
|
||||
process: { env },
|
||||
Buffer,
|
||||
URL,
|
||||
Response,
|
||||
crypto,
|
||||
setTimeout,
|
||||
clearTimeout,
|
||||
setImmediate,
|
||||
AbortSignal,
|
||||
AbortController,
|
||||
ReadableStream,
|
||||
console: { log() {}, warn() {}, error() {}, info() {} },
|
||||
fetch:
|
||||
mocks.fetch ||
|
||||
(() => {
|
||||
throw new Error("Unexpected network call");
|
||||
}),
|
||||
});
|
||||
new vm.Script(source, { filename }).runInContext(context);
|
||||
return module.exports;
|
||||
};
|
||||
return load;
|
||||
}
|
||||
|
||||
test("simultaneous archive and download fetch the source once; stored aliases survive module restart", async () => {
|
||||
let requests = 0;
|
||||
const source = `https://provider.example/${crypto.randomUUID()}.png`;
|
||||
const bytes = await sharp({ create: { width: 16, height: 24, channels: 4, background: "#12345680" } }).png().toBuffer();
|
||||
const loader = serverLoader({ fetch: async () => { requests++; await new Promise(r => setTimeout(r, 40)); return new Response(bytes, { headers: { "content-type": "image/png" } }); } });
|
||||
const cache = loader("lib/storage/image-byte-cache.ts");
|
||||
const results = await Promise.all(Array.from({ length: 10 }, () => cache.fetchImageBytes(source)));
|
||||
expect(requests).toBe(1);
|
||||
expect(results.every((r: any) => r.buffer.equals(bytes))).toBe(true);
|
||||
const alias = `https://cdn.example/${crypto.randomUUID()}.png`;
|
||||
await cache.cacheImageBytes(`source::${alias}`, results[0]);
|
||||
const restarted = serverLoader({})("lib/storage/image-byte-cache.ts");
|
||||
expect((await restarted.fetchImageBytes(alias)).buffer.equals(bytes)).toBe(true);
|
||||
});
|
||||
test("source body timeout aborts a slow body even after successful headers", async () => {
|
||||
const cache = serverLoader({ fetch: async (_url: string, options: any) => ({
|
||||
ok: true, headers: new Headers({ "content-type": "image/png" }),
|
||||
arrayBuffer: () => new Promise((_resolve, reject) => options.signal.addEventListener("abort", () => reject(new Error("body timeout")), { once: true })),
|
||||
}) })("lib/storage/image-byte-cache.ts");
|
||||
await expect(cache.fetchImageBytes(`https://provider.example/${crypto.randomUUID()}`, { timeoutMs: 40 })).rejects.toThrow("body timeout");
|
||||
});
|
||||
test("compatible PNG remains byte-identical; JPEG becomes RGB PNG without resizing", async () => {
|
||||
const module = serverLoader({})("lib/canvas/photoshop-safe-download.ts");
|
||||
const bytes = await sharp({ create: { width: 32, height: 48, channels: 4, background: "#12345680" } }).png().toBuffer();
|
||||
expect((await module.convertImageToPhotoshopPng(bytes)).equals(bytes)).toBe(true);
|
||||
const jpeg = await sharp(bytes).jpeg().toBuffer();
|
||||
const converted = await module.convertImageToPhotoshopPng(jpeg);
|
||||
const metadata = await sharp(converted).metadata();
|
||||
expect(metadata).toMatchObject({ format: "png", space: "srgb", width: 32, height: 48, depth: "uchar" });
|
||||
});
|
||||
test("CDN delivery requires a ready compatible asset and a configured HTTPS storage origin", async () => {
|
||||
let url = "https://cdn.example/uploads/test.png";
|
||||
let status = "ready";
|
||||
const module = serverLoader({
|
||||
"@/lib/db/queries": { getDefaultStorageConfig: async () => ({ type: "qiniu-kodo", publicUrl: "https://cdn.example", isEnabled: true }) },
|
||||
"./image-archive-db": { findImageArchiveDownloadTask: async () => ({ metadata: { downloadAsset: { status, url, contentType: "image/png", bytes: 321 } } }) },
|
||||
})("lib/storage/canvas-cdn-download.ts");
|
||||
const ready = await module.resolveCanvasCdnDownload(["https://provider.example/out.jpg"], "花瓶.png");
|
||||
expect(ready.direct.bytes).toBe(321);
|
||||
expect(new URL(ready.direct.downloadUrl).searchParams.get("attname")).toBe("花瓶.png");
|
||||
status = "pending";
|
||||
expect((await module.resolveCanvasCdnDownload([url], "a.png")).direct).toBeNull();
|
||||
status = "ready"; url = "https://evil.example/uploads/test.png";
|
||||
expect((await module.resolveCanvasCdnDownload([url], "a.png")).direct).toBeNull();
|
||||
});
|
||||
test("PNG preparation failure leaves the original archived and schedules a retry", async () => {
|
||||
let saved: any;
|
||||
const module = serverLoader({
|
||||
"./image-archive-db": { claimImageDownloadTask: async () => ({ id: "task", targetUrl: "https://cdn.example/original.png", metadata: { downloadAsset: { attempts: 2 } } }), saveImageDownloadAsset: async (_id: string, asset: any) => { saved = asset; } },
|
||||
"./image-byte-cache": { fetchImageBytes: async () => { throw new Error("storage offline"); } },
|
||||
"./service": {},
|
||||
})("lib/storage/image-download-preparation.ts");
|
||||
await module.prepareImageArchiveDownload("task");
|
||||
expect(saved.status).toBe("failed");
|
||||
expect(saved.attempts).toBe(2);
|
||||
expect(Date.parse(saved.retryAt)).toBeGreaterThan(Date.now());
|
||||
});
|
||||
test("ZIP streams retain original bytes and Unicode names without recompression", async () => {
|
||||
const module = serverLoader({})("lib/canvas/download-zip-stream.ts");
|
||||
const bytes = await sharp({ create: { width: 8, height: 8, channels: 3, background: "red" } }).png().toBuffer();
|
||||
const zip = new JSZip(); zip.file("作品/花瓶.png", bytes);
|
||||
const data = Buffer.from(await new Response(module.streamDownloadZip(zip)).arrayBuffer());
|
||||
const unpacked = await JSZip.loadAsync(data);
|
||||
expect((await unpacked.file("作品/花瓶.png")!.async("nodebuffer")).equals(bytes)).toBe(true);
|
||||
// Local-file compression method 0 means STORE.
|
||||
expect(data.readUInt16LE(8)).toBe(0);
|
||||
});
|
||||
|
||||
test("ready downloads hand the CDN URL to the browser without buffering the file", async () => {
|
||||
let frame: any; let payload: any; let appended = false;
|
||||
const module = serverLoader({
|
||||
"@/lib/image-delivery": { buildImageProxyUrl: (url: string) => url },
|
||||
fetch: async (_url: string, init: any) => {
|
||||
payload = JSON.parse(init.body);
|
||||
return new Response(JSON.stringify({ delivery: "cdn", downloadUrl: "https://cdn.example/a.png?attname=a.png", filename: "a.png", bytes: 123 }), { headers: { "Content-Type": "application/json" } });
|
||||
},
|
||||
document: { createElement: (tag: string) => { expect(tag).toBe("iframe"); return frame = { remove() {} }; }, body: { appendChild: () => { appended = true; } } },
|
||||
window: { setTimeout() {} },
|
||||
})("lib/client/canvas-image-download.ts");
|
||||
const result = await module.downloadCanvasImageAsPng({ url: "https://provider.example/a.png" });
|
||||
expect(payload.preferDirect).toBe(true);
|
||||
expect(appended).toBe(true); expect(frame.hidden).toBe(true);
|
||||
expect(frame.src).toContain("https://cdn.example/"); expect(result.bytes).toBe(123);
|
||||
});
|
||||
test("single-image route preserves authentication and returns a small CDN response only when requested", async () => {
|
||||
let signedIn = false; let lookups = 0;
|
||||
const module = serverLoader({
|
||||
"@/app/(auth)/auth": { auth: async () => signedIn ? { user: { id: "owner" } } : null },
|
||||
"@/lib/storage/canvas-cdn-download": { resolveCanvasCdnDownload: async () => { lookups++; return { direct: { delivery: "cdn", downloadUrl: "https://cdn.example/a.png?attname=a.png", filename: "a.png", bytes: 100 }, task: null }; } },
|
||||
"@/lib/storage/image-download-preparation": {},
|
||||
"next/server": { after() {}, NextResponse: class extends Response { static json(body: unknown, init?: ResponseInit) { return new Response(JSON.stringify(body), { ...init, headers: { "Content-Type": "application/json", ...(init?.headers || {}) } }); } } },
|
||||
})("app/api/canvas/download-image/route.ts");
|
||||
const request = { url: "https://app.example/api/canvas/download-image", headers: new Headers(), json: async () => ({ url: "https://provider.example/a.png", preferDirect: true }) };
|
||||
expect((await module.POST(request)).status).toBe(401); expect(lookups).toBe(0);
|
||||
signedIn = true;
|
||||
const response = await module.POST(request);
|
||||
expect(response.status).toBe(200); expect(response.headers.get("Cache-Control")).toBe("no-store");
|
||||
expect((await response.json()).delivery).toBe("cdn"); expect(lookups).toBe(1);
|
||||
});
|
||||
@ -18,7 +18,7 @@ const png = Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=",
|
||||
"base64",
|
||||
);
|
||||
const raw = "https://provider.example/result.png?signature=keep-me";
|
||||
const raw = `https://provider.example/result.png?signature=test-${crypto.randomUUID()}`;
|
||||
const permanent = "https://bucket.example/result.png";
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void;
|
||||
@ -206,6 +206,7 @@ test("background storage starts after delivery and coalesces repeated wakeups",
|
||||
store: { total: runs === 1 ? 1 : 0 },
|
||||
replace: { total: 0 },
|
||||
stats: { pendingStore: 0, pendingReplace: 0 },
|
||||
downloadTasks: 0,
|
||||
};
|
||||
},
|
||||
},
|
||||
@ -223,6 +224,7 @@ test("bucket failure retains the provider URL and retry task", async () => {
|
||||
let stored = 0;
|
||||
let replaced = 0;
|
||||
const module = serverLoader({
|
||||
"@/lib/storage/image-download-preparation": { preparePendingImageDownloads: async () => 0 },
|
||||
"@/lib/storage/image-archive": {
|
||||
archiveTemporaryImage: async () => {
|
||||
throw new Error("Bucket unavailable");
|
||||
@ -239,6 +241,7 @@ test("bucket failure retains the provider URL and retry task", async () => {
|
||||
failed += 1;
|
||||
},
|
||||
getImageArchiveTasksToReplace: async () => [],
|
||||
hasPendingImageDownloads: async () => false,
|
||||
getImageArchiveTaskStats: async () => ({ pendingStore: 1, pendingReplace: 0 }),
|
||||
},
|
||||
"@/lib/storage/image-url-replacement": {
|
||||
@ -411,6 +414,7 @@ test("successful background archive replaces saved references immediately", asyn
|
||||
let completed = false;
|
||||
const calls: string[][] = [];
|
||||
const module = serverLoader({
|
||||
"@/lib/storage/image-download-preparation": { preparePendingImageDownloads: async () => 0 },
|
||||
"@/lib/storage/image-archive": {
|
||||
archiveTemporaryImage: async () => ({ url: permanent, bytes: png.length }),
|
||||
},
|
||||
@ -434,6 +438,7 @@ test("successful background archive replaces saved references immediately", asyn
|
||||
markImageArchiveTaskCompleted: async () => {
|
||||
completed = true;
|
||||
},
|
||||
hasPendingImageDownloads: async () => false,
|
||||
getImageArchiveTaskStats: async () => ({ pendingStore: 0, pendingReplace: 0 }),
|
||||
},
|
||||
"@/lib/storage/image-url-replacement": {
|
||||
@ -616,6 +621,7 @@ test("detached generation still archives when its original request has already c
|
||||
store: { total: 0 },
|
||||
replace: { total: 0 },
|
||||
stats: { pendingStore: 0, pendingReplace: 0 },
|
||||
downloadTasks: 0,
|
||||
};
|
||||
},
|
||||
},
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user