修复大 ZIP 七牛交付与断点重试
This commit is contained in:
parent
445d756a19
commit
e221027b74
@ -10,11 +10,13 @@ import JSZip from "jszip";
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { fetchCanvasDownloadAsset } from "@/lib/canvas/photoshop-safe-download";
|
||||
import { publishCanvasZip } from "@/lib/storage/canvas-zip-delivery";
|
||||
import { trackRuntimeWork } from "@/lib/observability/active-work";
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 300;
|
||||
type DownloadFileItem = { url: string; name?: string };
|
||||
const MAX_DOWNLOAD_FILES = 200;
|
||||
const TASK_TTL_MS = 60 * 60_000;
|
||||
const TASK_TTL_MS = 24 * 60 * 60_000;
|
||||
const PREPARATION_TTL_MS = 6 * 60 * 60_000;
|
||||
const running = (globalThis as typeof globalThis & { __canvasZipRunning?: Set<string> });
|
||||
const active = running.__canvasZipRunning ??= new Set<string>();
|
||||
function sanitizeFolderName(value: string): string {
|
||||
@ -98,63 +100,71 @@ async function processTask(id: string, userId: string, requestOrigin: string, co
|
||||
task = await loadZipTask(id, userId);
|
||||
if (!task || task.status === "completed" || task.status === "failed") return;
|
||||
await assertZipDiskSpace();
|
||||
task.status = "running"; task.stage = "正在准备原图"; task.updatedAt = Date.now();
|
||||
const existingZip = task.zipFile && task.contentDisposition && await stat(task.zipFile.path).catch(() => null);
|
||||
const reuseZip = existingZip && existingZip.size === task.zipFile!.bytes;
|
||||
task.status = "running"; task.stage = reuseZip ? "正在准备加速下载" : "正在准备原图"; task.updatedAt = Date.now();
|
||||
await saveZipTask(task);
|
||||
// Stage each image on disk, not all PNG buffers in the web process heap.
|
||||
// A restart resumes from these files using the current owner's session.
|
||||
let cursor = 0;
|
||||
let writes = Promise.resolve();
|
||||
const persist = () => { const current = task!; writes = writes.then(() => saveZipTask(current)); return writes; };
|
||||
const results = await Promise.allSettled(Array.from({ length: Math.min(2, task.files.length) }, async () => {
|
||||
while (cursor < task!.files.length) {
|
||||
const index = cursor++; const file = task!.files[index];
|
||||
const path = join(zipTaskPath(id), `${index}.image`);
|
||||
if (file.prepared && file.filename && (await stat(path).catch(() => null))?.size) continue;
|
||||
delete file.prepared; delete file.filename;
|
||||
const result = await fetchCanvasDownloadAsset({ file, index, requestOrigin, cookie,
|
||||
signal: AbortSignal.timeout(120_000) }).catch(() => ({ ok: false as const, error: "原图读取超时或源地址暂时不可用" }));
|
||||
if (result.ok) {
|
||||
await assertZipDiskSpace(result.bytes.length);
|
||||
await writeFile(path, result.bytes, { mode: 0o600 });
|
||||
file.prepared = `${index}.image`; file.filename = result.filename; delete file.error;
|
||||
} else file.error = result.error;
|
||||
task!.processedCount = task!.files.filter(f => f.prepared || f.error).length;
|
||||
task!.successCount = task!.files.filter(f => f.prepared).length;
|
||||
task!.failedCount = task!.files.filter(f => f.error).length;
|
||||
task!.stage = `正在准备原图 ${task!.processedCount}/${task!.totalCount}`;
|
||||
task!.updatedAt = Date.now(); await persist();
|
||||
}
|
||||
}));
|
||||
const rejected = results.find(result => result.status === "rejected");
|
||||
if (rejected?.status === "rejected") throw rejected.reason;
|
||||
if (!task.successCount) throw new Error("所选原图暂时都无法读取，请稍后重试；画布图片不会被删除");
|
||||
const zip = new JSZip(); const folder = zip.folder(task.folderName) || zip;
|
||||
for (const file of task.files) if (file.prepared && file.filename)
|
||||
folder.file(file.filename, createReadStream(join(zipTaskPath(id), file.prepared))
|
||||
.once("close", () => { void rm(join(zipTaskPath(id), file.prepared!), { force: true }).catch(() => undefined); }));
|
||||
if (task.failedCount) folder.file("_download-failures.txt", task.files.filter(f => f.error)
|
||||
.map(f => `${f.name || "图片"}：${f.error}`).join("\n"));
|
||||
task.stage = "正在生成完整 ZIP 文件"; await saveZipTask(task);
|
||||
task.zipFile = await writeDownloadZip(zip, task.id, percent => { task!.compressPercent = percent; });
|
||||
task.contentDisposition = buildContentDisposition(task.folderName);
|
||||
if (!reuseZip) {
|
||||
delete task.zipFile; delete task.contentDisposition; delete task.cdnUrl;
|
||||
// Stage each image on disk, not all PNG buffers in the web process heap.
|
||||
// A restart resumes from these files using the current owner's session.
|
||||
let cursor = 0;
|
||||
let writes = Promise.resolve();
|
||||
const persist = () => { const current = task!; writes = writes.then(() => saveZipTask(current)); return writes; };
|
||||
const results = await Promise.allSettled(Array.from({ length: Math.min(2, task.files.length) }, async () => {
|
||||
while (cursor < task!.files.length) {
|
||||
const index = cursor++; const file = task!.files[index];
|
||||
const path = join(zipTaskPath(id), `${index}.image`);
|
||||
if (file.prepared && file.filename && (await stat(path).catch(() => null))?.size) continue;
|
||||
delete file.prepared; delete file.filename;
|
||||
const result = await fetchCanvasDownloadAsset({ file, index, requestOrigin, cookie,
|
||||
signal: AbortSignal.timeout(120_000) }).catch(() => ({ ok: false as const, error: "原图读取超时或源地址暂时不可用" }));
|
||||
if (result.ok) {
|
||||
await assertZipDiskSpace(result.bytes.length);
|
||||
await writeFile(path, result.bytes, { mode: 0o600 });
|
||||
file.prepared = `${index}.image`; file.filename = result.filename; delete file.error;
|
||||
} else file.error = result.error;
|
||||
task!.processedCount = task!.files.filter(f => f.prepared || f.error).length;
|
||||
task!.successCount = task!.files.filter(f => f.prepared).length;
|
||||
task!.failedCount = task!.files.filter(f => f.error).length;
|
||||
task!.stage = `正在准备原图 ${task!.processedCount}/${task!.totalCount}`;
|
||||
task!.updatedAt = Date.now(); await persist();
|
||||
}
|
||||
}));
|
||||
const rejected = results.find(result => result.status === "rejected");
|
||||
if (rejected?.status === "rejected") throw rejected.reason;
|
||||
if (!task.successCount) throw new Error("所选原图暂时都无法读取，请稍后重试；画布图片不会被删除");
|
||||
const zip = new JSZip(); const folder = zip.folder(task.folderName) || zip;
|
||||
for (const file of task.files) if (file.prepared && file.filename)
|
||||
folder.file(file.filename, createReadStream(join(zipTaskPath(id), file.prepared))
|
||||
.once("close", () => { void rm(join(zipTaskPath(id), file.prepared!), { force: true }).catch(() => undefined); }));
|
||||
if (task.failedCount) folder.file("_download-failures.txt", task.files.filter(f => f.error)
|
||||
.map(f => `${f.name || "图片"}：${f.error}`).join("\n"));
|
||||
task.stage = "正在生成完整 ZIP 文件"; await saveZipTask(task);
|
||||
task.zipFile = await writeDownloadZip(zip, task.id, percent => { task!.compressPercent = percent; });
|
||||
task.contentDisposition = buildContentDisposition(task.folderName);
|
||||
}
|
||||
task.stage = "正在准备下载链接"; await saveZipTask(task);
|
||||
try {
|
||||
const cdn = await publishCanvasZip({ id: task.id, ...task.zipFile, disposition: task.contentDisposition });
|
||||
if (cdn) {
|
||||
task.cdnObject = { storageConfigId: cdn.storageConfigId, key: cdn.key };
|
||||
if (cdn.url) task.cdnUrl = cdn.url;
|
||||
}
|
||||
} catch { console.warn("[CanvasDownloadZipTask] CDN delivery unavailable; complete local ZIP retained", { taskId: id }); }
|
||||
const cdn = await publishCanvasZip({ id: task.id, ...task.zipFile!, disposition: task.contentDisposition!,
|
||||
onProgress: async bytes => {
|
||||
task!.stage = `正在传送至七牛 ${Math.round(bytes / task!.zipFile!.bytes * 100)}%`;
|
||||
task!.updatedAt = Date.now(); await saveZipTask(task!);
|
||||
} });
|
||||
if (cdn) {
|
||||
task.cdnObject = { storageConfigId: cdn.storageConfigId, key: cdn.key };
|
||||
if (!cdn.url) throw new Error(cdn.error || `ZIP 已打包，但加速下载未就绪，请再次点击下载继续传送原 ZIP。下载编号：${task.id}`);
|
||||
task.cdnUrl = cdn.url;
|
||||
}
|
||||
task.status = "completed"; task.stage = task.failedCount ? `已打包 ${task.successCount} 张，${task.failedCount} 张未能读取` : "打包完成";
|
||||
task.compressPercent = 100; task.updatedAt = Date.now(); task.expiresAt = Date.now() + TASK_TTL_MS;
|
||||
await saveZipTask(task);
|
||||
// The complete ZIP is enough for retries; release staged duplicate bytes.
|
||||
await Promise.all(task.files.filter(f => f.prepared).map(f => rm(join(zipTaskPath(id), f.prepared!), { force: true }).catch(() => undefined)));
|
||||
console.info("[CanvasDownloadZipTask] Complete", { taskId: id, count: task.successCount, failed: task.failedCount,
|
||||
bytes: task.zipFile.bytes, delivery: task.cdnUrl ? "cdn" : "local" });
|
||||
bytes: task.zipFile!.bytes, delivery: task.cdnUrl ? "cdn" : "local" });
|
||||
} catch (error) {
|
||||
if (task) {
|
||||
task.status = "failed"; task.stage = "打包未完成";
|
||||
task.status = "failed"; task.stage = task.zipFile ? "ZIP 已打包，加速下载未就绪" : "打包未完成";
|
||||
task.error = error instanceof Error ? error.message : "打包失败，请稍后重试";
|
||||
task.updatedAt = Date.now(); task.expiresAt = Date.now() + TASK_TTL_MS; await saveZipTask(task).catch(() => undefined);
|
||||
}
|
||||
@ -162,7 +172,7 @@ async function processTask(id: string, userId: string, requestOrigin: string, co
|
||||
} finally { await release?.().catch(() => undefined); active.delete(id); }
|
||||
}
|
||||
function schedule(task: ZipTask, request: NextRequest) {
|
||||
const run = () => processTask(task.id, task.userId, getRequestOrigin(request), request.headers.get("cookie") || "");
|
||||
const run = () => trackRuntimeWork(() => processTask(task.id, task.userId, getRequestOrigin(request), request.headers.get("cookie") || ""));
|
||||
after(run);
|
||||
}
|
||||
export async function POST(request: NextRequest) {
|
||||
@ -187,14 +197,17 @@ export async function POST(request: NextRequest) {
|
||||
const now = Date.now();
|
||||
task = { id, userId: session.user.id, folderName: sanitizeFolderName(String(body.folderName || "")), files,
|
||||
totalCount: files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0,
|
||||
status: "pending", stage: "等待后台准备", createdAt: now, updatedAt: now, expiresAt: now + 6 * TASK_TTL_MS };
|
||||
status: "pending", stage: "等待后台准备", createdAt: now, updatedAt: now, expiresAt: now + PREPARATION_TTL_MS };
|
||||
await saveZipTask(task);
|
||||
} else if (task.status === "failed" && body.retry === true) {
|
||||
} else if ((task.status === "failed" || task.status === "completed" && task.cdnObject && !task.cdnUrl) && body.retry === true) {
|
||||
task.status = "pending"; task.error = undefined;
|
||||
await assertZipTaskCapacity(session.user.id);
|
||||
for (const file of task.files) delete file.error;
|
||||
task.processedCount = task.successCount = task.files.filter(file => file.prepared).length;
|
||||
task.failedCount = 0; task.updatedAt = Date.now(); task.expiresAt = Date.now() + 6 * TASK_TTL_MS;
|
||||
if (!task.zipFile) {
|
||||
for (const file of task.files) delete file.error;
|
||||
task.processedCount = task.successCount = task.files.filter(file => file.prepared).length;
|
||||
task.failedCount = 0;
|
||||
}
|
||||
task.updatedAt = Date.now(); task.expiresAt = Date.now() + PREPARATION_TTL_MS;
|
||||
await saveZipTask(task);
|
||||
}
|
||||
if (task.status === "pending" || task.status === "running") schedule(task, request);
|
||||
|
||||
@ -98,3 +98,10 @@ public 先从当前运行版本创建写入时复制副本，保留未纳入 Git
|
||||
|
||||
仓库历史全量 lint 仍有已记录的问题，不能声称全部质量门禁通过。
|
||||
每次 PR 须记录本次改动的验证、历史失败和未覆盖项目；当前流程不会自动触发生产发布。
|
||||
|
||||
## ZIP 下载交付
|
||||
|
||||
默认存储为七牛时，完整 ZIP 通过分片上传至七牛，确认下载地址可用后才提供下载。
|
||||
大文件使用三个并发分片，上传进度显示在原打包任务中。传送或链接检查失败会返回具体阶段、
|
||||
下载编号及重试建议，保留已生成的 ZIP；再次点击下载复用原文件，已上传的完整对象不重复上传。
|
||||
后台 ZIP 任务参与发布期间的工作等待。完成或失败的任务及文件保留 24 小时，待处理任务保留 6 小时。
|
||||
|
||||
@ -1,11 +1,72 @@
|
||||
import "server-only";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { PutObjectCommand } from "@aws-sdk/client-s3";
|
||||
import { open } from "node:fs/promises";
|
||||
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, HeadObjectCommand, PutObjectCommand, UploadPartCommand, type S3Client } from "@aws-sdk/client-s3";
|
||||
import { getDefaultStorageConfig, getStorageConfigById } from "@/lib/db/queries";
|
||||
import { createS3Client, toDbS3UploadConfig, buildS3PublicUrl } from "./service";
|
||||
|
||||
/** Only a completely written archive is uploaded. Local GET remains a fallback. */
|
||||
export async function publishCanvasZip(params: { id: string; path: string; bytes: number; disposition: string }) {
|
||||
type ZipPublication = { id: string; path: string; bytes: number; disposition: string; onProgress?: (bytes: number) => Promise<void> };
|
||||
const PART_BYTES = 8 * 1024 ** 2;
|
||||
|
||||
async function uploadZip(client: S3Client, bucket: string, key: string, params: ZipPublication) {
|
||||
const metadata = { Bucket: bucket, Key: key, ContentType: "application/zip", ContentDisposition: params.disposition, CacheControl: "public, max-age=1800" };
|
||||
if (params.bytes <= 2 * PART_BYTES) {
|
||||
const body = createReadStream(params.path);
|
||||
try {
|
||||
await client.send(new PutObjectCommand({ ...metadata, Body: body, ContentLength: params.bytes }), { abortSignal: AbortSignal.timeout(90_000) });
|
||||
await params.onProgress?.(params.bytes);
|
||||
} finally { body.destroy(); }
|
||||
return;
|
||||
}
|
||||
const initiated = await client.send(new CreateMultipartUploadCommand(metadata), { abortSignal: AbortSignal.timeout(15_000) });
|
||||
if (!initiated.UploadId) throw new Error("七牛未返回分片上传编号");
|
||||
const upload = { Bucket: bucket, Key: key, UploadId: initiated.UploadId };
|
||||
const handle = await open(params.path, "r").catch(async error => {
|
||||
await client.send(new AbortMultipartUploadCommand(upload), { abortSignal: AbortSignal.timeout(10_000) }).catch(() => undefined);
|
||||
throw error;
|
||||
});
|
||||
const parts: Array<{ PartNumber: number; ETag: string }> = [];
|
||||
let cursor = 0;
|
||||
let uploaded = 0;
|
||||
let progress = Promise.resolve();
|
||||
const controller = new AbortController();
|
||||
try {
|
||||
const workers = await Promise.allSettled(Array.from({ length: 3 }, async () => {
|
||||
try {
|
||||
while (!controller.signal.aborted) {
|
||||
const index = cursor++;
|
||||
const start = index * PART_BYTES;
|
||||
if (start >= params.bytes) return;
|
||||
const body = Buffer.allocUnsafe(Math.min(PART_BYTES, params.bytes - start));
|
||||
let read = 0;
|
||||
while (read < body.length) {
|
||||
const result = await handle.read(body, read, body.length - read, start + read);
|
||||
if (!result.bytesRead) throw new Error("完整 ZIP 文件读取中断，请重试下载");
|
||||
read += result.bytesRead;
|
||||
}
|
||||
const part = await client.send(new UploadPartCommand({ ...upload, PartNumber: index + 1, Body: body, ContentLength: body.length }), {
|
||||
abortSignal: AbortSignal.any([controller.signal, AbortSignal.timeout(180_000)]),
|
||||
});
|
||||
if (!part.ETag) throw new Error("七牛未确认上传分片");
|
||||
parts.push({ PartNumber: index + 1, ETag: part.ETag });
|
||||
uploaded += body.length;
|
||||
const bytes = uploaded;
|
||||
progress = progress.then(() => params.onProgress?.(bytes));
|
||||
await progress;
|
||||
}
|
||||
} catch (error) { controller.abort(); throw error; }
|
||||
}));
|
||||
const failure = workers.find(worker => worker.status === "rejected");
|
||||
if (failure?.status === "rejected") throw failure.reason;
|
||||
await client.send(new CompleteMultipartUploadCommand({ ...upload, MultipartUpload: { Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber) } }), { abortSignal: AbortSignal.timeout(30_000) });
|
||||
} catch (error) {
|
||||
await client.send(new AbortMultipartUploadCommand(upload), { abortSignal: AbortSignal.timeout(10_000) }).catch(() => undefined);
|
||||
throw error;
|
||||
} finally { await handle.close(); }
|
||||
}
|
||||
|
||||
/** Publish the complete ZIP before offering the accelerated download. */
|
||||
export async function publishCanvasZip(params: ZipPublication) {
|
||||
const raw = await getDefaultStorageConfig();
|
||||
const config = toDbS3UploadConfig(raw);
|
||||
if (config?.provider !== "qiniu-kodo" || !config.publicUrl?.startsWith("https://")) return null;
|
||||
@ -13,20 +74,37 @@ export async function publishCanvasZip(params: { id: string; path: string; bytes
|
||||
const client = createS3Client(config);
|
||||
// Qiniu's S3 endpoint expects an ordinary known-length body, without AWS trailers.
|
||||
client.config.requestChecksumCalculation = async () => "WHEN_REQUIRED";
|
||||
const body = createReadStream(params.path);
|
||||
let phase: "check" | "upload" | "verify" = "check";
|
||||
const startedAt = Date.now();
|
||||
let uploadedBytes = 0;
|
||||
try {
|
||||
await client.send(new PutObjectCommand({ Bucket: config.bucket, Key: key, Body: body,
|
||||
ContentLength: params.bytes, ContentType: "application/zip", ContentDisposition: params.disposition,
|
||||
CacheControl: "public, max-age=1800" }), { abortSignal: AbortSignal.timeout(90_000) });
|
||||
const url = buildS3PublicUrl(config, key);
|
||||
const existing = await client.send(new HeadObjectCommand({ Bucket: config.bucket, Key: key }), { abortSignal: AbortSignal.timeout(15_000) }).catch(error => {
|
||||
if (error.$metadata?.httpStatusCode === 404) return null;
|
||||
throw error;
|
||||
});
|
||||
if (existing?.ContentLength !== params.bytes) {
|
||||
phase = "upload";
|
||||
await uploadZip(client, config.bucket, key, { ...params, onProgress: async bytes => {
|
||||
uploadedBytes = bytes;
|
||||
await params.onProgress?.(bytes);
|
||||
} });
|
||||
}
|
||||
phase = "verify";
|
||||
const target = new URL(buildS3PublicUrl(config, key));
|
||||
target.searchParams.set("v", String(params.bytes));
|
||||
const url = target.toString();
|
||||
const response = await fetch(url, { method: "HEAD", redirect: "error", signal: AbortSignal.timeout(10_000) });
|
||||
const length = Number(response.headers.get("content-length"));
|
||||
if (!response.ok || length !== params.bytes) throw new Error("CDN object is not ready");
|
||||
return { url, storageConfigId: raw!.id, key };
|
||||
} catch (error) {
|
||||
console.warn("[CanvasDownloadZipTask] CDN upload/check failed; using complete local file", { taskId: params.id, kind: (error as Error).name });
|
||||
return { url: null, storageConfigId: raw!.id, key };
|
||||
} finally { body.destroy(); client.destroy(); }
|
||||
const kind = (error as Error).name;
|
||||
const status = (error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode;
|
||||
console.warn("[CanvasDownloadZipTask] CDN publication failed", { taskId: params.id, phase, kind, status, bytes: params.bytes, uploadedBytes, elapsedMs: Date.now() - startedAt });
|
||||
const reason = status === 401 || status === 403 ? `七牛拒绝访问（HTTP ${status}），请管理员检查存储权限` : phase === "check" ? "暂时无法确认七牛文件状态" : phase === "verify" ? "七牛下载链接尚未可用" : /AbortError|TimeoutError/.test(kind) ? "传送至七牛时超时" : "传送至七牛失败";
|
||||
const advice = status === 401 || status === 403 ? "存储权限恢复后再次点击下载即可继续传送原 ZIP" : "请再次点击下载以继续传送原 ZIP";
|
||||
return { url: null, storageConfigId: raw!.id, key, error: `ZIP 已打包完成，但${reason}。${advice}；不会重新生成图片。若持续失败，请联系管理员并提供下载编号：${params.id}` };
|
||||
} finally { client.destroy(); }
|
||||
}
|
||||
|
||||
export async function removeCanvasZipObject(object: { storageConfigId: string; key: string }) {
|
||||
|
||||
@ -7,6 +7,9 @@ import { randomUUID } from "node:crypto";
|
||||
import JSZip from "jszip";
|
||||
import { serverLoader } from "../helpers/canvas-download-server-loader";
|
||||
import { parseDownloadByteRange } from "@/lib/canvas/download-byte-range";
|
||||
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, HeadObjectCommand, PutObjectCommand, UploadPartCommand } from "@aws-sdk/client-s3";
|
||||
|
||||
const runtimeWork = { trackRuntimeWork: async (run: () => Promise<void>) => run() };
|
||||
|
||||
test("ranges reject invalid offsets and preserve normal, open and suffix byte ranges", () => {
|
||||
expect(parseDownloadByteRange("bytes=0-3", 20)).toEqual({start:0,end:3});
|
||||
@ -20,7 +23,7 @@ test("multi-select ZIP admission is immediate; complete archives survive restart
|
||||
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-test-')); let owner:string|null='owner';let fetches=0;
|
||||
const callbacks:Array<()=>Promise<void>>=[];
|
||||
const mocks={
|
||||
'@/app/(auth)/auth': {auth:async()=>owner?{user:{id:owner}}:null},
|
||||
'@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth': {auth:async()=>owner?{user:{id:owner}}:null},
|
||||
'next/server': {NextResponse,after:(run:()=>Promise<void>)=>callbacks.push(run)},
|
||||
'@/lib/canvas/photoshop-safe-download':{fetchCanvasDownloadAsset:async({index}:any)=>{
|
||||
fetches++;return {ok:true,filename:`${index}-花瓶.png`,bytes:Buffer.from(`image-${index}`),contentType:'image/png'};
|
||||
@ -56,7 +59,7 @@ test("multi-select ZIP admission is immediate; complete archives survive restart
|
||||
test("all-source failures never produce an empty successful ZIP; retries reuse the task and resolve sources again", async()=>{
|
||||
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-retry-'));const callbacks:Array<()=>Promise<void>>=[];let available=false;
|
||||
const route=serverLoader({
|
||||
'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})},
|
||||
'@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})},
|
||||
'next/server':{NextResponse,after:(run:()=>Promise<void>)=>callbacks.push(run)},
|
||||
'@/lib/canvas/photoshop-safe-download':{fetchCanvasDownloadAsset:async()=>available?{ok:true,filename:'one.png',bytes:Buffer.from('one'),contentType:'image/png'}:{ok:false,error:'expired provider URL'}},
|
||||
'@/lib/storage/canvas-zip-delivery':{publishCanvasZip:async()=>null},
|
||||
@ -108,7 +111,7 @@ test("the browser polls a complete ZIP, preserves an idempotent task after a los
|
||||
test("partial source failures are explicit; CDN redirects retain an authenticated local range fallback",async()=>{
|
||||
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-partial-'));const callbacks:Array<()=>Promise<void>>=[];let owner='owner';
|
||||
const route=serverLoader({
|
||||
'@/app/(auth)/auth':{auth:async()=>({user:{id:owner}})},
|
||||
'@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth':{auth:async()=>({user:{id:owner}})},
|
||||
'next/server':{NextResponse,after:(run:()=>Promise<void>)=>callbacks.push(run)},
|
||||
'@/lib/canvas/photoshop-safe-download':{fetchCanvasDownloadAsset:async({index}:any)=>index?{ok:false,error:'source expired'}:{ok:true,filename:'original.png',bytes:Buffer.from('original'),contentType:'image/png'}},
|
||||
'@/lib/storage/canvas-zip-delivery':{publishCanvasZip:async()=>({url:'https://cdn.example/complete.zip',key:'canvas-downloads/x.zip',storageConfigId:'config'})},
|
||||
@ -128,7 +131,7 @@ test("partial source failures are explicit; CDN redirects retain an authenticate
|
||||
test("recovery validates staged files instead of treating missing bytes as successful images",async()=>{
|
||||
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-missing-'));const callbacks:Array<()=>Promise<void>>=[];const id=randomUUID();
|
||||
const load=serverLoader({
|
||||
'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})},
|
||||
'@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})},
|
||||
'next/server':{NextResponse,after:(run:()=>Promise<void>)=>callbacks.push(run)},
|
||||
'@/lib/canvas/photoshop-safe-download':{fetchCanvasDownloadAsset:async()=>({ok:false,error:'source expired'})},
|
||||
'@/lib/storage/canvas-zip-delivery':{publishCanvasZip:async()=>null},
|
||||
@ -157,7 +160,7 @@ test("large ZIP writing checks current free space as staged input bytes are rele
|
||||
test("slow expired-object cleanup cannot keep the ZIP admission request open",async()=>{
|
||||
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-cleanup-'));const callbacks:Array<()=>Promise<void>>=[];
|
||||
const mocks:Record<string,unknown>={
|
||||
'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})},
|
||||
'@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})},
|
||||
'next/server':{NextResponse,after:(run:()=>Promise<void>)=>callbacks.push(run)},
|
||||
'@/lib/canvas/photoshop-safe-download':{},'@/lib/storage/canvas-zip-delivery':{},
|
||||
};
|
||||
@ -169,3 +172,127 @@ test("slow expired-object cleanup cannot keep the ZIP admission request open",as
|
||||
expect(result).not.toBe('blocked');expect((result as Response).status).toBe(202);
|
||||
}finally{await rm(dir,{recursive:true,force:true});}
|
||||
});
|
||||
|
||||
test("large ZIPs use bounded parallel parts and publish only the complete ordered archive", async () => {
|
||||
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-parts-'));
|
||||
const bytes=Buffer.alloc(25*1024**2+19,0x5a);bytes[16*1024**2]=0xff;
|
||||
const file=join(dir,'large.zip');await writeFile(file,bytes);
|
||||
const parts=new Map<number,Buffer>();let inFlight=0;let peak=0;let finished=false;let destroyed=false;let trailers='';
|
||||
const progress:number[]=[];
|
||||
const client={config:{requestChecksumCalculation:async()=>''},destroy(){destroyed=true;},async send(command:any){
|
||||
if(command instanceof HeadObjectCommand)throw Object.assign(new Error('missing'),{$metadata:{httpStatusCode:404}});
|
||||
if(command instanceof CreateMultipartUploadCommand){expect(command.input.ContentDisposition).toContain('attachment');return {UploadId:'upload'};}
|
||||
if(command instanceof UploadPartCommand){
|
||||
const input=command.input;inFlight++;peak=Math.max(peak,inFlight);expect(input.ContentLength).toBeLessThanOrEqual(8*1024**2);
|
||||
await new Promise(resolve=>setTimeout(resolve,input.PartNumber===1?30:20));
|
||||
expect(Buffer.isBuffer(input.Body)).toBe(true);
|
||||
parts.set(input.PartNumber!,Buffer.from(input.Body as Buffer));inFlight--;return {ETag:`part-${input.PartNumber}`};
|
||||
}
|
||||
expect(command).toBeInstanceOf(CompleteMultipartUploadCommand);
|
||||
expect(command.input.MultipartUpload.Parts.map((p:any)=>p.PartNumber)).toEqual([1,2,3,4]);
|
||||
expect(Buffer.concat(command.input.MultipartUpload.Parts.map((p:any)=>parts.get(p.PartNumber)!)).equals(bytes)).toBe(true);
|
||||
finished=true;trailers=await client.config.requestChecksumCalculation();return {};
|
||||
}};
|
||||
const publish=serverLoader({
|
||||
'@/lib/db/queries':{getDefaultStorageConfig:async()=>({id:'storage'})},
|
||||
'./service':{toDbS3UploadConfig:()=>({provider:'qiniu-kodo',bucket:'bucket',publicUrl:'https://cdn.example'}),createS3Client:()=>client,buildS3PublicUrl:(_:any,key:string)=>'https://cdn.example/'+key},
|
||||
fetch:async(url:string,init:any)=>{expect(finished).toBe(true);expect(init.method).toBe('HEAD');expect(new URL(url).searchParams.get('v')).toBe(String(bytes.length));return new Response(null,{headers:{'content-length':String(bytes.length)}});},
|
||||
})('lib/storage/canvas-zip-delivery.ts').publishCanvasZip;
|
||||
try{
|
||||
const result=await publish({id:randomUUID(),path:file,bytes:bytes.length,disposition:'attachment; filename="large.zip"',onProgress:async(n:number)=>{progress.push(n);}});
|
||||
expect(result.url).toContain('https://cdn.example/canvas-downloads/');expect(peak).toBeGreaterThan(1);expect(peak).toBeLessThanOrEqual(3);expect(progress.at(-1)).toBe(bytes.length);
|
||||
expect(progress).toEqual([...progress].sort((a,b)=>a-b));expect(trailers).toBe('WHEN_REQUIRED');expect(destroyed).toBe(true);
|
||||
}finally{await rm(dir,{recursive:true,force:true});}
|
||||
});
|
||||
|
||||
test("failed multipart uploads abort their parts and retain the original ZIP with actionable advice",async()=>{
|
||||
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-abort-'));const file=join(dir,'large.zip');const bytes=17*1024**2;
|
||||
const handle=await (await import('node:fs/promises')).open(file,'w');await handle.truncate(bytes);await handle.close();
|
||||
let aborted=false;let completed=false;
|
||||
const client={config:{},destroy(){},async send(command:any){
|
||||
if(command instanceof HeadObjectCommand)throw Object.assign(new Error('missing'),{$metadata:{httpStatusCode:404}});
|
||||
if(command instanceof CreateMultipartUploadCommand)return {UploadId:'upload'};
|
||||
if(command instanceof UploadPartCommand){if(command.input.PartNumber===2)throw new DOMException('part timeout','TimeoutError');return {ETag:'part'};}
|
||||
if(command instanceof AbortMultipartUploadCommand){aborted=true;return {};}
|
||||
completed=true;return {};
|
||||
}};
|
||||
const publish=serverLoader({
|
||||
'@/lib/db/queries':{getDefaultStorageConfig:async()=>({id:'storage'})},
|
||||
'./service':{toDbS3UploadConfig:()=>({provider:'qiniu-kodo',bucket:'bucket',publicUrl:'https://cdn.example'}),createS3Client:()=>client},
|
||||
})('lib/storage/canvas-zip-delivery.ts').publishCanvasZip;
|
||||
try{
|
||||
const result=await publish({id:randomUUID(),path:file,bytes,disposition:'attachment'});
|
||||
expect(result.url).toBeNull();expect(result.error).toContain('传送至七牛时超时');expect(result.error).toContain('继续传送原 ZIP');
|
||||
expect(aborted).toBe(true);expect(completed).toBe(false);expect((await (await import('node:fs/promises')).stat(file)).size).toBe(bytes);
|
||||
}finally{await rm(dir,{recursive:true,force:true});}
|
||||
});
|
||||
|
||||
test("retrying CDN verification reuses an already uploaded ZIP without uploading it again",async()=>{
|
||||
let uploads=0;let available=false;
|
||||
const client={config:{},destroy(){},async send(command:any){
|
||||
if(command instanceof HeadObjectCommand)return {ContentLength:100};
|
||||
if(command instanceof PutObjectCommand||command instanceof CreateMultipartUploadCommand)uploads++;
|
||||
throw new Error('Unexpected upload');
|
||||
}};
|
||||
const publish=serverLoader({
|
||||
'@/lib/db/queries':{getDefaultStorageConfig:async()=>({id:'storage'})},
|
||||
'./service':{toDbS3UploadConfig:()=>({provider:'qiniu-kodo',bucket:'bucket',publicUrl:'https://cdn.example'}),createS3Client:()=>client,buildS3PublicUrl:(_:any,key:string)=>'https://cdn.example/'+key},
|
||||
fetch:async()=>new Response(null,{status:available?200:404,headers:{'content-length':available?'100':'0'}}),
|
||||
})('lib/storage/canvas-zip-delivery.ts').publishCanvasZip;
|
||||
const params={id:randomUUID(),path:'unused-already-published.zip',bytes:100,disposition:'attachment'};
|
||||
const first=await publish(params);expect(first.url).toBeNull();expect(first.error).toContain('七牛下载链接尚未可用');
|
||||
available=true;expect((await publish(params)).url).toContain('https://cdn.example/');expect(uploads).toBe(0);
|
||||
});
|
||||
|
||||
test("CDN failures never silently start local downloads; retries reuse the ZIP and keep it resumable beyond one hour",async()=>{
|
||||
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-cdn-retry-'));const callbacks:Array<()=>Promise<void>>=[];
|
||||
let fetches=0;let publications=0;let available=false;let now=Date.now();
|
||||
let activeWork=0;
|
||||
const runtimeWork={trackRuntimeWork:async(run:()=>Promise<void>)=>{activeWork++;try{return await run();}finally{activeWork--;}}};
|
||||
class Clock extends Date{static now(){return now;}}
|
||||
const load=serverLoader({
|
||||
Date:Clock,'@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})},
|
||||
'next/server':{NextResponse,after:(run:()=>Promise<void>)=>callbacks.push(run)},
|
||||
'@/lib/canvas/photoshop-safe-download':{fetchCanvasDownloadAsset:async()=>{fetches++;return {ok:true,filename:'original.png',bytes:Buffer.from('original'),contentType:'image/png'};}},
|
||||
'@/lib/storage/canvas-zip-delivery':{publishCanvasZip:async()=>{expect(activeWork).toBe(1);publications++;return {url:available?'https://cdn.example/complete.zip':null,key:'canvas-downloads/key.zip',storageConfigId:'storage',error:'ZIP 已打包，传送至七牛时超时；请再次点击下载继续原 ZIP，不会重新生成图片。'};},removeCanvasZipObject:async()=>{}},
|
||||
},{CANVAS_DOWNLOAD_ZIP_DIR:dir});
|
||||
const route=load('app/api/canvas/download-zip-task/route.ts');const id=randomUUID();
|
||||
const post=()=>new NextRequest('http://localhost/api/canvas/download-zip-task',{method:'POST',body:JSON.stringify({taskId:id,retry:true,files:[{url:'https://cdn.example/original.png'}]})});
|
||||
const request=(suffix='')=>new NextRequest(`http://localhost/api/canvas/download-zip-task?taskId=${id}${suffix}`,{headers:{range:'bytes=0-3'}});
|
||||
try{
|
||||
await route.POST(post());while(callbacks.length)await callbacks.shift()!();
|
||||
const failed=(await(await route.GET(request())).json()).task;
|
||||
expect(failed).toMatchObject({id,status:'failed',successCount:1,downloadUrl:''});expect(failed.error).toContain('不会重新生成图片');
|
||||
expect((await route.GET(request('&download=1'))).status).toBe(409);
|
||||
const before=await readFile(join(dir,id,'task.json'),'utf8');const original=Buffer.from(await readFile(JSON.parse(before).zipFile.path));
|
||||
available=true;await route.POST(post());while(callbacks.length)await callbacks.shift()!();
|
||||
expect(fetches).toBe(1);expect(publications).toBe(2);
|
||||
const manifest=JSON.parse(await readFile(join(dir,id,'task.json'),'utf8'));expect(await readFile(manifest.zipFile.path)).toEqual(original);
|
||||
delete manifest.cdnUrl;
|
||||
await load('lib/canvas/download-zip-task-store.ts').saveZipTask(manifest);
|
||||
await route.POST(post());while(callbacks.length)await callbacks.shift()!();
|
||||
expect(fetches).toBe(1);expect(publications).toBe(3);
|
||||
expect(activeWork).toBe(0);
|
||||
now+=2*60*60_000;
|
||||
expect((await route.GET(request('&download=1'))).status).toBe(307);
|
||||
const local=await route.GET(request('&download=1&local=1'));expect(local.status).toBe(206);expect(Buffer.from(await local.arrayBuffer())).toEqual(original.subarray(0,4));
|
||||
}finally{await rm(dir,{recursive:true,force:true});}
|
||||
});
|
||||
|
||||
test("small ZIP delivery preserves the complete file and download filename",async()=>{
|
||||
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-small-'));const file=join(dir,'small.zip');const bytes=Buffer.from('PK small complete archive');await writeFile(file,bytes);
|
||||
let written=false;
|
||||
const client={config:{},destroy(){},async send(command:any){
|
||||
if(command instanceof HeadObjectCommand)throw Object.assign(new Error('missing'),{$metadata:{httpStatusCode:404}});
|
||||
expect(command).toBeInstanceOf(PutObjectCommand);expect(command.input.ContentLength).toBe(bytes.length);expect(command.input.ContentDisposition).toContain('small.zip');
|
||||
const chunks:Buffer[]=[];for await(const chunk of command.input.Body)chunks.push(Buffer.from(chunk));
|
||||
expect(Buffer.concat(chunks).equals(bytes)).toBe(true);written=true;return {};
|
||||
}};
|
||||
const publish=serverLoader({
|
||||
'@/lib/db/queries':{getDefaultStorageConfig:async()=>({id:'storage'})},
|
||||
'./service':{toDbS3UploadConfig:()=>({provider:'qiniu-kodo',bucket:'bucket',publicUrl:'https://cdn.example'}),createS3Client:()=>client,buildS3PublicUrl:(_:any,key:string)=>'https://cdn.example/'+key},
|
||||
fetch:async()=>{expect(written).toBe(true);return new Response(null,{headers:{'content-length':String(bytes.length)}});},
|
||||
})('lib/storage/canvas-zip-delivery.ts').publishCanvasZip;
|
||||
try{expect((await publish({id:randomUUID(),path:file,bytes:bytes.length,disposition:'attachment; filename="small.zip"'})).url).toContain('https://cdn.example/');}
|
||||
finally{await rm(dir,{recursive:true,force:true});}
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user