fix(canvas): prepare original downloads independently of archiving
This commit is contained in:
134
parent
45c554fc27
commit
aec7134ad3
@ -2,6 +2,8 @@ import { resolveCanvasCdnDownload } from "@/lib/storage/canvas-cdn-download";
|
||||
import { prepareImageArchiveDownload } from "@/lib/storage/image-download-preparation";
|
||||
import { after, NextRequest, NextResponse } from "next/server";
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { getDownloadPreparationQueue, type DownloadPreparationInput } from "@/lib/canvas/download-preparation";
|
||||
import { extractImageProxySourceUrl, inferOriginalImageUrl } from "@/lib/image-url";
|
||||
import {
|
||||
buildAttachmentContentDisposition,
|
||||
fetchCanvasDownloadAsset,
|
||||
@ -11,6 +13,31 @@ import {
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 120;
|
||||
|
||||
async function prepareDownload(input: DownloadPreparationInput, signal: AbortSignal) {
|
||||
// A proxy candidate for the same original must not repeat the same slow
|
||||
// upstream request through our own server (and create another HTTP timeout).
|
||||
const sources = [...new Set(input.urls.map(url => {
|
||||
const source = extractImageProxySourceUrl(url) || url;
|
||||
return inferOriginalImageUrl(source) || source;
|
||||
}))];
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
signal.throwIfAborted();
|
||||
for (const url of sources) {
|
||||
signal.throwIfAborted();
|
||||
const result = await fetchCanvasDownloadAsset({
|
||||
file: { url, name: "original" }, index: 0,
|
||||
requestOrigin: input.requestOrigin, cookie: input.cookie,
|
||||
includeSequencePrefix: false, timeoutMs: 12_000, retryCount: 1, signal,
|
||||
});
|
||||
if (result.ok) return { buffer: result.bytes, contentType: result.contentType };
|
||||
}
|
||||
// Each pass resolves the latest archive URL inside fetchCanvasDownloadAsset.
|
||||
// No AI/model request is involved and a completed source fetch is shared.
|
||||
if (attempt < 2) await new Promise(resolve => setTimeout(resolve, 1_000));
|
||||
}
|
||||
throw new Error("Original image sources unavailable");
|
||||
}
|
||||
|
||||
function getRequestOrigin(request: NextRequest): string {
|
||||
const forwardedHost = (request.headers.get("x-forwarded-host") || "").split(",")[0]?.trim();
|
||||
const forwardedProto = (request.headers.get("x-forwarded-proto") || "").split(",")[0]?.trim();
|
||||
@ -57,6 +84,7 @@ export async function POST(request: NextRequest) {
|
||||
const body = (await request.json().catch(() => ({}))) as Record<string, unknown>;
|
||||
const requestOrigin = getRequestOrigin(request);
|
||||
const urls = resolveDownloadUrls(body, requestOrigin);
|
||||
const requestedUrls = [...urls];
|
||||
if (urls.length === 0) {
|
||||
return NextResponse.json({ success: false, error: "图片地址无效" }, { status: 400 });
|
||||
}
|
||||
@ -71,6 +99,26 @@ export async function POST(request: NextRequest) {
|
||||
}
|
||||
}
|
||||
const cookie = request.headers.get("cookie") || "";
|
||||
if (body.preparation === "async") {
|
||||
const result = await getDownloadPreparationQueue(prepareDownload, after).request({
|
||||
userId: session.user.id, urls: requestedUrls, requestOrigin, cookie,
|
||||
});
|
||||
if (result.status === "preparing") {
|
||||
return NextResponse.json({ delivery: "preparing", retryAfter: 3,
|
||||
message: "正在准备原图，完成后将自动下载" }, {
|
||||
status: 202, headers: { "Retry-After": "3", "Cache-Control": "no-store" },
|
||||
});
|
||||
}
|
||||
if (result.status === "failed") {
|
||||
return NextResponse.json({ success: false, error: result.error }, { status: 502 });
|
||||
}
|
||||
return new NextResponse(new Uint8Array(result.asset.buffer), { headers: {
|
||||
"Content-Type": result.asset.contentType,
|
||||
"Content-Length": String(result.asset.buffer.length),
|
||||
"Content-Disposition": buildAttachmentContentDisposition(`${name}.png`),
|
||||
"Cache-Control": "no-store",
|
||||
} });
|
||||
}
|
||||
let result: Awaited<ReturnType<typeof fetchCanvasDownloadAsset>> | null = null;
|
||||
const failures: string[] = [];
|
||||
|
||||
|
||||
146
lib/canvas/download-preparation.ts
Normal file
146
lib/canvas/download-preparation.ts
Normal file
@ -0,0 +1,146 @@
|
||||
import "server-only";
|
||||
import { createHash } from "node:crypto";
|
||||
import { cacheImageBytes, readImageBytes, type ImageBytes } from "@/lib/storage/image-byte-cache";
|
||||
|
||||
export type DownloadPreparationInput = {
|
||||
userId: string;
|
||||
urls: string[];
|
||||
requestOrigin: string;
|
||||
cookie: string;
|
||||
};
|
||||
type Result =
|
||||
| { status: "preparing" }
|
||||
| { status: "ready"; asset: ImageBytes }
|
||||
| { status: "failed"; error: string };
|
||||
type Job = {
|
||||
userId: string;
|
||||
input: DownloadPreparationInput;
|
||||
status: "queued" | "running" | "ready" | "failed";
|
||||
updatedAt: number;
|
||||
asset?: ImageBytes;
|
||||
};
|
||||
const FAILURE_MESSAGE = "原图暂时无法读取，图片仍保留在画布中，请稍后重试";
|
||||
const MAX_MEMORY_BYTES = 128 * 1024 * 1024;
|
||||
|
||||
// A download is independent of node autosave and of the archive worker's batch.
|
||||
// Polls share one preparation; only bytes are cached, never cookies or signed URLs.
|
||||
export function createDownloadPreparationQueue(deps: {
|
||||
load: (input: DownloadPreparationInput, signal: AbortSignal) => Promise<ImageBytes>;
|
||||
read: (identity: string) => Promise<ImageBytes | null>;
|
||||
write: (identity: string, asset: ImageBytes) => Promise<unknown>;
|
||||
schedule: (run: () => Promise<void>) => void;
|
||||
now?: () => number;
|
||||
timeoutMs?: number;
|
||||
concurrency?: number;
|
||||
}) {
|
||||
const jobs = new Map<string, Job>();
|
||||
const now = deps.now || Date.now;
|
||||
const concurrency = deps.concurrency || 4;
|
||||
let active = 0;
|
||||
|
||||
const prune = () => {
|
||||
let bytes = 0;
|
||||
for (const [key, job] of jobs) {
|
||||
if ((job.status === "ready" && now() - job.updatedAt > 10 * 60_000) ||
|
||||
(job.status === "failed" && now() - job.updatedAt > 15_000)) jobs.delete(key);
|
||||
else bytes += job.asset?.buffer.length || 0;
|
||||
}
|
||||
// Completed jobs are in insertion order; keep the newest fallback bytes if
|
||||
// the optional disk cache is full. Never evict a running preparation.
|
||||
for (const [key, job] of jobs) {
|
||||
if (bytes <= MAX_MEMORY_BYTES) break;
|
||||
if (job.status === "ready") { bytes -= job.asset?.buffer.length || 0; jobs.delete(key); }
|
||||
}
|
||||
};
|
||||
const drain = async (): Promise<void> => {
|
||||
const running: Promise<void>[] = [];
|
||||
for (const [key, job] of jobs) {
|
||||
if (active >= concurrency) break;
|
||||
if (job.status !== "queued") continue;
|
||||
active++;
|
||||
job.status = "running";
|
||||
running.push((async () => {
|
||||
const controller = new AbortController();
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
const asset = await Promise.race([
|
||||
deps.load(job.input, controller.signal),
|
||||
new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => {
|
||||
controller.abort();
|
||||
reject(new Error("Download preparation deadline"));
|
||||
}, deps.timeoutMs || 90_000);
|
||||
}),
|
||||
]);
|
||||
if (asset.contentType !== "image/png" || !asset.buffer.length || asset.buffer.length > MAX_MEMORY_BYTES) {
|
||||
throw new Error("Invalid prepared original");
|
||||
}
|
||||
// Keep a bounded in-memory fallback until persistence is confirmed.
|
||||
job.asset = asset;
|
||||
await deps.write(key, asset).catch(() => undefined);
|
||||
const stored = await deps.read(key).catch(() => null);
|
||||
if (stored?.contentType === "image/png" && stored.buffer.length) job.asset = undefined;
|
||||
job.status = "ready";
|
||||
} catch {
|
||||
job.status = "failed";
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
// Do not retain authentication data after a preparation finishes.
|
||||
job.input = { ...job.input, cookie: "", urls: [] };
|
||||
job.updatedAt = now();
|
||||
active--;
|
||||
prune();
|
||||
}
|
||||
})().then(() => drain()));
|
||||
}
|
||||
await Promise.all(running);
|
||||
};
|
||||
|
||||
return {
|
||||
async request(input: DownloadPreparationInput): Promise<Result> {
|
||||
const key = "canvas-download:v2:" + createHash("sha256")
|
||||
.update(JSON.stringify([input.userId, input.requestOrigin, input.cookie, input.urls])).digest("hex");
|
||||
const cached = await deps.read(key).catch(() => null);
|
||||
if (cached?.contentType === "image/png") return { status: "ready", asset: cached };
|
||||
prune();
|
||||
const current = jobs.get(key);
|
||||
if (current?.status === "ready" && current.asset) return { status: "ready", asset: current.asset };
|
||||
if (current?.status === "failed") return { status: "failed", error: FAILURE_MESSAGE };
|
||||
if (current && current.status !== "ready") return { status: "preparing" };
|
||||
if (jobs.size >= 64) {
|
||||
for (const [oldKey, job] of jobs) {
|
||||
if (job.status === "ready" || job.status === "failed") jobs.delete(oldKey);
|
||||
if (jobs.size < 64) break;
|
||||
}
|
||||
}
|
||||
if (jobs.size >= 64 || [...jobs.values()].filter(job => job.userId === input.userId &&
|
||||
(job.status === "queued" || job.status === "running")).length >= 4) {
|
||||
return { status: "failed", error: "正在准备的原图较多，请等待当前下载完成后再试" };
|
||||
}
|
||||
jobs.set(key, { userId: input.userId, input, status: "queued", updatedAt: now() });
|
||||
deps.schedule(drain);
|
||||
return { status: "preparing" };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const state = globalThis as typeof globalThis & {
|
||||
__canvasDownloadPreparationV2?: ReturnType<typeof createDownloadPreparationQueue>;
|
||||
};
|
||||
|
||||
export function getDownloadPreparationQueue(load: Parameters<typeof createDownloadPreparationQueue>[0]["load"],
|
||||
after: (run: () => Promise<void>) => void) {
|
||||
return (state.__canvasDownloadPreparationV2 ??= createDownloadPreparationQueue({
|
||||
load,
|
||||
read: readImageBytes,
|
||||
write: cacheImageBytes,
|
||||
schedule(run) {
|
||||
// Also run when invoked outside a live Next request. The queue itself
|
||||
// deduplicates both callbacks and limits concurrent work.
|
||||
const timer = setTimeout(() => { void run(); }, 1_000);
|
||||
timer.unref?.();
|
||||
try { after(async () => { clearTimeout(timer); await run(); }); }
|
||||
catch { clearTimeout(timer); setImmediate(() => { void run(); }); }
|
||||
},
|
||||
}));
|
||||
}
|
||||
@ -288,31 +288,35 @@ export async function fetchCanvasDownloadAsset(params: {
|
||||
includeSequencePrefix?: boolean;
|
||||
timeoutMs?: number;
|
||||
retryCount?: number;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<CanvasDownloadAssetResult> {
|
||||
let lastError = "";
|
||||
const timeoutMs = params.timeoutMs || DEFAULT_FETCH_TIMEOUT_MS;
|
||||
const retryCount = params.retryCount || DEFAULT_FETCH_RETRY_COUNT;
|
||||
const candidateUrls = buildCanvasDownloadSourceCandidates(
|
||||
const originalCandidates = buildCanvasDownloadSourceCandidates(
|
||||
params.file.url,
|
||||
params.requestOrigin
|
||||
);
|
||||
|
||||
if (process.env.POSTGRES_URL) {
|
||||
// ZIPs and binary fallback also reuse the durable prepared object after the
|
||||
// local byte cache expires, instead of converting the original JPEG again.
|
||||
const { findImageArchiveDownloadTask } = await import("@/lib/storage/image-archive-db");
|
||||
const task = await findImageArchiveDownloadTask(candidateUrls).catch(() => null);
|
||||
const asset = (task?.metadata as { downloadAsset?: { status?: string; url?: string; contentType?: string } } | null)?.downloadAsset;
|
||||
if (asset?.status === "ready" && asset.url && asset.contentType === "image/png") {
|
||||
const readyUrl = resolveHttpUrl(asset.url, params.requestOrigin);
|
||||
if (readyUrl && !candidateUrls.includes(readyUrl)) candidateUrls.unshift(readyUrl);
|
||||
} else if (task?.targetUrl && !candidateUrls.includes(task.targetUrl)) {
|
||||
candidateUrls.unshift(task.targetUrl);
|
||||
for (let attempt = 1; attempt <= retryCount; attempt += 1) {
|
||||
params.signal?.throwIfAborted();
|
||||
const candidateUrls = [...originalCandidates];
|
||||
if (process.env.POSTGRES_URL) {
|
||||
// ZIPs and binary fallback also reuse the durable prepared object after the
|
||||
// local byte cache expires, instead of converting the original JPEG again.
|
||||
const { findImageArchiveDownloadTask } = await import("@/lib/storage/image-archive-db");
|
||||
const task = await findImageArchiveDownloadTask(candidateUrls).catch(() => null);
|
||||
const asset = (task?.metadata as { downloadAsset?: { status?: string; url?: string; contentType?: string } } | null)?.downloadAsset;
|
||||
if (asset?.status === "ready" && asset.url && asset.contentType === "image/png") {
|
||||
const readyUrl = resolveHttpUrl(asset.url, params.requestOrigin);
|
||||
if (readyUrl && !candidateUrls.includes(readyUrl)) candidateUrls.unshift(readyUrl);
|
||||
} else if (task?.targetUrl && !candidateUrls.includes(task.targetUrl)) {
|
||||
candidateUrls.unshift(task.targetUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const candidateUrl of candidateUrls) {
|
||||
for (let attempt = 1; attempt <= retryCount; attempt += 1) {
|
||||
for (const candidateUrl of candidateUrls) {
|
||||
params.signal?.throwIfAborted();
|
||||
try {
|
||||
const data = await fetchImageBytes(candidateUrl, {
|
||||
timeoutMs,
|
||||
@ -329,12 +333,13 @@ export async function fetchCanvasDownloadAsset(params: {
|
||||
includeSequencePrefix: params.includeSequencePrefix !== false,
|
||||
});
|
||||
} catch (error) {
|
||||
params.signal?.throwIfAborted();
|
||||
lastError = normalizeDownloadError(error);
|
||||
if (attempt < retryCount) {
|
||||
await sleep(350 * attempt);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Archiving may have completed during the failed fetch. Resolve aliases
|
||||
// again before retrying, rather than spending every attempt on the old URL.
|
||||
if (attempt < retryCount) await sleep(350 * attempt);
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@ -19,6 +19,7 @@ export type CanvasImageDownloadResult = {
|
||||
type CanvasImageDownloadErrorCode =
|
||||
| "rate_limited"
|
||||
| "source_unavailable"
|
||||
| "preparation_timeout"
|
||||
| "request_failed";
|
||||
|
||||
export class CanvasImageDownloadError extends Error {
|
||||
@ -50,7 +51,10 @@ export function getCanvasImageDownloadErrorMessage(error: unknown) {
|
||||
: "图片下载请求过于频繁，请稍后重试";
|
||||
}
|
||||
if (error.code === "source_unavailable") {
|
||||
return `${error.message}，源图可能已失效，请稍后重试或重新生成图片`;
|
||||
return "原图暂时无法读取，图片仍保留在画布中，请稍后重试";
|
||||
}
|
||||
if (error.code === "preparation_timeout") {
|
||||
return "原图仍在后台准备中，请稍后再次点击下载";
|
||||
}
|
||||
}
|
||||
return error instanceof Error ? error.message : "图片下载失败，请稍后重试";
|
||||
@ -248,7 +252,11 @@ export async function downloadCanvasImageAsPng({
|
||||
let response: Response | null = null;
|
||||
let errorPayload: DownloadErrorPayload | null = null;
|
||||
|
||||
for (let attempt = 0; attempt <= MAX_RATE_LIMIT_RETRIES; attempt++) {
|
||||
const preparationDeadline = Date.now() + 180_000;
|
||||
let rateLimitRetries = 0;
|
||||
let preparationPolls = 0;
|
||||
for (;;) {
|
||||
signal?.throwIfAborted();
|
||||
response = await fetch("/api/canvas/download-image", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
@ -257,10 +265,26 @@ export async function downloadCanvasImageAsPng({
|
||||
urls,
|
||||
name: filenameBase,
|
||||
preferDirect: true,
|
||||
preparation: "async",
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
|
||||
if (response.status === 202) {
|
||||
rateLimitRetries = 0;
|
||||
preparationPolls++;
|
||||
const preparing = await response.json().catch(() => null);
|
||||
if (preparing?.delivery !== "preparing") {
|
||||
throw new CanvasImageDownloadError("下载准备状态无效", { code: "request_failed", status: 202 });
|
||||
}
|
||||
if (Date.now() >= preparationDeadline || preparationPolls >= 60) {
|
||||
throw new CanvasImageDownloadError("原图仍在准备中", { code: "preparation_timeout", status: 202 });
|
||||
}
|
||||
onProgress?.({ phase: "preparing", message: preparing.message || "正在准备原图，完成后将自动下载" });
|
||||
const delaySeconds = Math.max(preparationPolls > 5 ? 5 : 1, Math.min(5, Number(preparing.retryAfter) || 3));
|
||||
await waitForRetry(delaySeconds * 1000, signal);
|
||||
continue;
|
||||
}
|
||||
if (response.ok) break;
|
||||
errorPayload = (await response.json().catch(() => null)) as
|
||||
| DownloadErrorPayload
|
||||
@ -268,7 +292,7 @@ export async function downloadCanvasImageAsPng({
|
||||
if (response.status !== 429) break;
|
||||
|
||||
const retryAfterSeconds = parseRetryAfterSeconds(response, errorPayload);
|
||||
if (attempt >= MAX_RATE_LIMIT_RETRIES) {
|
||||
if (rateLimitRetries++ >= MAX_RATE_LIMIT_RETRIES) {
|
||||
throw new CanvasImageDownloadError(
|
||||
errorPayload?.error || "图片下载请求过于频繁",
|
||||
{
|
||||
|
||||
@ -57,19 +57,29 @@ export async function readImageBytes(identity: string): Promise<ImageBytes | nul
|
||||
return { buffer: bytes.subarray(end + 1), contentType: header.type };
|
||||
} catch { return null; }
|
||||
}
|
||||
export async function onceImageBytes(identity: string, load: () => Promise<ImageBytes>): Promise<ImageBytes> {
|
||||
export async function onceImageBytes(identity: string, load: () => Promise<ImageBytes>, validate?: (data: ImageBytes) => boolean): Promise<ImageBytes> {
|
||||
const existing = flights.get(identity);
|
||||
if (existing) return existing;
|
||||
const promise = (async () => {
|
||||
const cached = await readImageBytes(identity);
|
||||
if (cached) return cached;
|
||||
if (cached && (!validate || validate(cached))) return cached;
|
||||
const result = await load();
|
||||
if (validate && !validate(result)) throw new Error("Image source returned an error document instead of an image");
|
||||
await cacheImageBytes(identity, result);
|
||||
return result;
|
||||
})();
|
||||
flights.set(identity, promise);
|
||||
try { return await promise; } finally { if (flights.get(identity) === promise) flights.delete(identity); }
|
||||
}
|
||||
|
||||
function isMediaResponse(data: ImageBytes) {
|
||||
const type = data.contentType.split(";")[0].trim().toLowerCase();
|
||||
const prefix = data.buffer.subarray(0, 1024).toString("utf8").trimStart();
|
||||
if (type === "text/html" || type === "application/json" || type === "application/xhtml+xml") return false;
|
||||
if (/^(?:<!doctype\s+html|<html[\s>]|<error[\s>]|[{[])/i.test(prefix)) return false;
|
||||
if (/^<\?xml/i.test(prefix) && !/<svg[\s>]/i.test(prefix)) return false;
|
||||
return data.buffer.length > 0;
|
||||
}
|
||||
export async function fetchImageBytes(url: string, options: { headers?: HeadersInit; timeoutMs?: number; scope?: string } = {}) {
|
||||
return onceImageBytes(`source:${options.scope || ""}:${url}`, async () => {
|
||||
if (url.startsWith("data:")) {
|
||||
@ -86,5 +96,5 @@ export async function fetchImageBytes(url: string, options: { headers?: HeadersI
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
if (!buffer.length) throw new Error("Downloaded image is empty");
|
||||
return { buffer, contentType: response.headers.get("content-type") || "" };
|
||||
});
|
||||
}, isMediaResponse);
|
||||
}
|
||||
|
||||
@ -85,7 +85,8 @@ test.describe("canvas image download rate limiting", () => {
|
||||
|
||||
expect(rateLimitMessage).toBe("图片下载请求过于频繁，请稍后重试");
|
||||
expect(rateLimitMessage).not.toContain("源图");
|
||||
expect(sourceMessage).toContain("源图可能已失效");
|
||||
expect(sourceMessage).toContain("图片仍保留在画布中");
|
||||
expect(sourceMessage).not.toContain("重新生成");
|
||||
});
|
||||
|
||||
test("uses the shared error formatter in every canvas image download entry", () => {
|
||||
|
||||
@ -40,6 +40,8 @@ function serverLoader(mocks: Record<string, unknown>, env: Record<string, string
|
||||
process: { env },
|
||||
Buffer,
|
||||
URL,
|
||||
URLSearchParams,
|
||||
DOMException,
|
||||
Response,
|
||||
crypto,
|
||||
setTimeout,
|
||||
@ -50,6 +52,7 @@ function serverLoader(mocks: Record<string, unknown>, env: Record<string, string
|
||||
ReadableStream,
|
||||
Headers,
|
||||
Blob,
|
||||
Date: mocks.Date || Date,
|
||||
document: mocks.document,
|
||||
window: mocks.window,
|
||||
console: { log() {}, warn() {}, error() {}, info() {} },
|
||||
@ -86,6 +89,19 @@ test("source body timeout aborts a slow body even after successful headers", asy
|
||||
}) })("lib/storage/image-byte-cache.ts");
|
||||
await expect(cache.fetchImageBytes(`https://provider.example/${crypto.randomUUID()}`, { timeoutMs: 40 })).rejects.toThrow("body timeout");
|
||||
});
|
||||
test("a provider error page with HTTP 200 never poisons the image cache", async () => {
|
||||
let requests = 0;
|
||||
const bytes = await sharp({ create: { width: 25, height: 39, channels: 3, background: "red" } }).png().toBuffer();
|
||||
const module = serverLoader({ fetch: async () => {
|
||||
requests++;
|
||||
return requests === 1 ? new Response("<!DOCTYPE html><html>upstream timeout</html>", { headers: { "content-type": "image/png" } }) :
|
||||
new Response(bytes, { headers: { "content-type": "image/png" } });
|
||||
} })("lib/storage/image-byte-cache.ts");
|
||||
const url = `https://provider.example/${crypto.randomUUID()}.png`;
|
||||
await expect(module.fetchImageBytes(url)).rejects.toThrow("error document");
|
||||
expect((await module.fetchImageBytes(url)).buffer.equals(bytes)).toBe(true);
|
||||
expect(requests).toBe(2);
|
||||
});
|
||||
test("compatible PNG remains byte-identical; JPEG becomes RGB PNG without resizing", async () => {
|
||||
const module = serverLoader({})("lib/canvas/photoshop-safe-download.ts");
|
||||
const bytes = await sharp({ create: { width: 32, height: 48, channels: 4, background: "#12345680" } }).png().toBuffer();
|
||||
@ -139,7 +155,7 @@ test("ready downloads hand the CDN URL to the browser without buffering the file
|
||||
"@/lib/image-delivery": { buildImageProxyUrl: (url: string) => url },
|
||||
fetch: async (_url: string, init: any) => {
|
||||
payload = JSON.parse(init.body);
|
||||
return new Response(JSON.stringify({ delivery: "cdn", downloadUrl: "https://cdn.example/a.png?attname=a.png", filename: "a.png", bytes: 123 }), { headers: { "Content-Type": "application/json" } });
|
||||
return Response.json({ delivery: "cdn", downloadUrl: "https://cdn.example/a.png?attname=a.png", filename: "a.png", bytes: 123 });
|
||||
},
|
||||
document: { createElement: (tag: string) => { expect(tag).toBe("iframe"); return frame = { remove() {} }; }, body: { appendChild: () => { appended = true; } } },
|
||||
window: { setTimeout() {} },
|
||||
@ -177,3 +193,156 @@ test("batch downloads use the durable prepared PNG even when the source cache is
|
||||
expect(result.ok).toBe(true); expect(result.bytes.equals(bytes)).toBe(true);
|
||||
expect(result.filename).toBe("001-作品.png"); expect(fetched).toEqual([prepared]);
|
||||
});
|
||||
|
||||
test("an archive completed during a source failure is resolved before the next attempt", async () => {
|
||||
const source = `https://provider.example/${crypto.randomUUID()}.png`;
|
||||
const archived = `https://cdn.example/${crypto.randomUUID()}.png`;
|
||||
const bytes = await sharp({ create: { width: 75, height: 113, channels: 4, background: "#abcdef80" } }).png().toBuffer();
|
||||
let targetUrl: string | undefined;
|
||||
const fetched: string[] = [];
|
||||
const module = serverLoader({
|
||||
"@/lib/storage/image-archive-db": { findImageArchiveDownloadTask: async () => ({ targetUrl }) },
|
||||
fetch: async (url: string) => {
|
||||
fetched.push(url);
|
||||
if (url === source) { targetUrl = archived; throw new Error("Old provider URL timed out"); }
|
||||
return new Response(bytes, { headers: { "content-type": "image/png" } });
|
||||
},
|
||||
}, { POSTGRES_URL: "mock-only" })("lib/canvas/photoshop-safe-download.ts");
|
||||
const result = await module.fetchCanvasDownloadAsset({ file: { url: source }, index: 0,
|
||||
requestOrigin: "https://app.example", cookie: "", retryCount: 2 });
|
||||
expect(result.ok).toBe(true);
|
||||
expect(result.bytes.equals(bytes)).toBe(true);
|
||||
expect(fetched).toEqual([source, archived]);
|
||||
});
|
||||
|
||||
test("parallel preparation polls share one task, retaining bytes if the disk cache is unavailable", async () => {
|
||||
const { createDownloadPreparationQueue } = serverLoader({})("lib/canvas/download-preparation.ts");
|
||||
const runs: Array<() => Promise<void>> = [];
|
||||
const asset = { buffer: Buffer.from("test-normalized-png"), contentType: "image/png" };
|
||||
let release!: () => void;
|
||||
let loads = 0;
|
||||
const gate = new Promise<void>(resolve => { release = resolve; });
|
||||
const queue = createDownloadPreparationQueue({
|
||||
load: async () => { loads++; await gate; return asset; },
|
||||
read: async () => null, write: async () => { throw new Error("Disk full"); },
|
||||
schedule: (run: () => Promise<void>) => runs.push(run),
|
||||
});
|
||||
const input = { userId: "owner", urls: ["https://provider.example/a.png"], requestOrigin: "https://app.example", cookie: "session=private" };
|
||||
const initial = await Promise.all(Array.from({ length: 10 }, () => queue.request(input)));
|
||||
expect(initial.every(item => item.status === "preparing")).toBe(true);
|
||||
expect(runs.length).toBe(1);
|
||||
const running = runs[0]();
|
||||
expect((await queue.request(input)).status).toBe("preparing");
|
||||
expect(loads).toBe(1);
|
||||
release(); await running;
|
||||
const result = await queue.request(input);
|
||||
expect(result.status).toBe("ready");
|
||||
expect(result.asset.buffer.equals(asset.buffer)).toBe(true);
|
||||
});
|
||||
|
||||
test("preparation cache survives a process restart and isolates authenticated users", async () => {
|
||||
const { createDownloadPreparationQueue } = serverLoader({})("lib/canvas/download-preparation.ts");
|
||||
const runs: Array<() => Promise<void>> = [];
|
||||
const cache = new Map<string, any>();
|
||||
let loads = 0;
|
||||
const deps = { load: async () => { loads++; return { buffer: Buffer.from("png-original"), contentType: "image/png" }; },
|
||||
read: async (key: string) => cache.get(key) || null,
|
||||
write: async (key: string, value: any) => { cache.set(key, value); },
|
||||
schedule: (run: () => Promise<void>) => runs.push(run) };
|
||||
const input = { userId: "owner-a", urls: ["https://app.example/private.png"], requestOrigin: "https://app.example", cookie: "session=a" };
|
||||
const queue = createDownloadPreparationQueue(deps);
|
||||
expect((await queue.request(input)).status).toBe("preparing");
|
||||
await runs.shift()!();
|
||||
const restarted = createDownloadPreparationQueue(deps);
|
||||
expect((await restarted.request(input)).status).toBe("ready");
|
||||
expect(loads).toBe(1);
|
||||
expect((await restarted.request({ ...input, userId: "owner-b", cookie: "session=b" })).status).toBe("preparing");
|
||||
await runs.shift()!();
|
||||
expect(loads).toBe(2);
|
||||
expect([...cache.keys()].every(key => !key.includes("private.png") && !key.includes("session="))).toBe(true);
|
||||
});
|
||||
|
||||
test("preparations have bounded concurrency and a deadline for unreachable originals", async () => {
|
||||
const { createDownloadPreparationQueue } = serverLoader({})("lib/canvas/download-preparation.ts");
|
||||
const runs: Array<() => Promise<void>> = [];
|
||||
let active = 0; let maximum = 0;
|
||||
const releases: Array<() => void> = [];
|
||||
const queue = createDownloadPreparationQueue({ concurrency: 2,
|
||||
load: async () => { active++; maximum = Math.max(maximum, active);
|
||||
await new Promise<void>(resolve => releases.push(resolve)); active--;
|
||||
return { buffer: Buffer.from("png"), contentType: "image/png" }; },
|
||||
read: async () => null, write: async () => {}, schedule: (run: () => Promise<void>) => runs.push(run) });
|
||||
for (let i = 0; i < 3; i++) await queue.request({ userId: `owner-${i}`, urls: [`https://provider.example/${i}.png`], requestOrigin: "https://app.example", cookie: "" });
|
||||
const running = Promise.all(runs.map(run => run()));
|
||||
expect(active).toBe(2);
|
||||
releases.shift()!(); releases.shift()!();
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
releases.shift()!(); await running;
|
||||
expect(maximum).toBe(2);
|
||||
const failedRuns: Array<() => Promise<void>> = [];
|
||||
const deadlineQueue = createDownloadPreparationQueue({ timeoutMs: 25,
|
||||
load: () => new Promise(() => {}), read: async () => null, write: async () => {},
|
||||
schedule: (run: () => Promise<void>) => failedRuns.push(run) });
|
||||
const input = { userId: "owner", urls: ["https://provider.example/unreachable.png"], requestOrigin: "https://app.example", cookie: "" };
|
||||
await deadlineQueue.request(input); await failedRuns[0]();
|
||||
expect((await deadlineQueue.request(input)).status).toBe("failed");
|
||||
});
|
||||
|
||||
test("the download route returns promptly then serves a full-size PNG while archive upload is unfinished", async () => {
|
||||
const bytes = await sharp({ create: { width: 122, height: 186, channels: 4, background: "#12345680" } }).png().toBuffer();
|
||||
const runs: Array<() => Promise<void>> = [];
|
||||
let calls = 0;
|
||||
let targetUrl: string | undefined;
|
||||
const module = serverLoader({
|
||||
"@/app/(auth)/auth": { auth: async () => ({ user: { id: "owner" } }) },
|
||||
"@/lib/storage/canvas-cdn-download": { resolveCanvasCdnDownload: async () => ({ task: targetUrl ? { id: "archived", targetUrl } : null, direct: null }) },
|
||||
"@/lib/storage/image-download-preparation": { prepareImageArchiveDownload: async () => {} },
|
||||
"next/server": { NextResponse: Response, after: (run: () => Promise<void>) => runs.push(run) },
|
||||
fetch: async () => { calls++; return new Response(bytes, { headers: { "content-type": "image/png" } }); },
|
||||
})("app/api/canvas/download-image/route.ts");
|
||||
const request = { url: "https://app.example/api/canvas/download-image", headers: new Headers(),
|
||||
json: async () => ({ url: `https://provider.example/${crypto.randomUUID()}.png`, preparation: "async", name: "原图" }) };
|
||||
const body = await request.json(); request.json = async () => body;
|
||||
const first = await module.POST(request);
|
||||
expect(first.status).toBe(202); expect(calls).toBe(0);
|
||||
expect((await first.json()).delivery).toBe("preparing");
|
||||
await runs[0]();
|
||||
targetUrl = "https://cdn.example/archive-completed.png";
|
||||
const second = await module.POST(request);
|
||||
expect(second.status).toBe(200);
|
||||
const downloaded = Buffer.from(await second.arrayBuffer());
|
||||
expect(downloaded.equals(bytes)).toBe(true);
|
||||
expect(await sharp(downloaded).metadata()).toMatchObject({ width: 122, height: 186, hasAlpha: true });
|
||||
expect(calls).toBe(1);
|
||||
});
|
||||
|
||||
test("the browser waits for preparation and cancellation never starts a second request", async () => {
|
||||
let calls = 0; const messages: string[] = []; let appended = false;
|
||||
const module = serverLoader({
|
||||
fetch: async (_url: string, options: any) => {
|
||||
expect(JSON.parse(options.body).preparation).toBe("async"); calls++;
|
||||
return calls === 1 ? Response.json({ delivery: "preparing", retryAfter: 1 }, { status: 202 }) :
|
||||
Response.json({ delivery: "cdn", downloadUrl: "https://cdn.example/ready.png", filename: "ready.png" });
|
||||
},
|
||||
document: { createElement: () => ({ remove() {} }), body: { appendChild: () => { appended = true; } } },
|
||||
window: { setTimeout() {} },
|
||||
})("lib/client/canvas-image-download.ts");
|
||||
const running = module.downloadCanvasImageAsPng({ url: "https://provider.example/a.png", onProgress: (p: any) => messages.push(p.message) });
|
||||
await new Promise(resolve => setTimeout(resolve, 20)); expect(appended).toBe(false);
|
||||
await running; expect(calls).toBe(2); expect(appended).toBe(true);
|
||||
expect(messages.some(message => message.includes("准备"))).toBe(true);
|
||||
const abort = new AbortController(); calls = 0;
|
||||
const cancelled = module.downloadCanvasImageAsPng({ url: "https://provider.example/a.png", signal: abort.signal });
|
||||
setTimeout(() => abort.abort(), 10);
|
||||
await expect(cancelled).rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(calls).toBe(1);
|
||||
});
|
||||
|
||||
test("preparation polling has a finite deadline instead of an endless spinner", async () => {
|
||||
let time = 0;
|
||||
const module = serverLoader({ Date: { now: () => time },
|
||||
fetch: async () => { time = 181_000; return Response.json({ delivery: "preparing" }, { status: 202 }); },
|
||||
})("lib/client/canvas-image-download.ts");
|
||||
await expect(module.downloadCanvasImageAsPng({ url: "https://provider.example/a.png" }))
|
||||
.rejects.toMatchObject({ code: "preparation_timeout" });
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user