lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复大 ZIP 上传超时及慢速回退下载 #53

Merged
WuYongJie merged 1 commits from codex/zip-cdn-delivery-20261106 into main 2026-11-06 12:10:51 +08:00
Conversation 0 Commits 1 Files Changed 4 +295 -70
4 changed files with 295 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
app/api/canvas/download-zip-task/route.ts
View File

@ -10,11 +10,13 @@ import JSZip from "jszip";
import { auth } from "@/app/(auth)/auth";
import { fetchCanvasDownloadAsset } from "@/lib/canvas/photoshop-safe-download";
import { publishCanvasZip } from "@/lib/storage/canvas-zip-delivery";
import { trackRuntimeWork } from "@/lib/observability/active-work";
export const runtime = "nodejs";
export const maxDuration = 300;
type DownloadFileItem = { url: string; name?: string };
const MAX_DOWNLOAD_FILES = 200;
const TASK_TTL_MS = 60 * 60_000;
const TASK_TTL_MS = 24 * 60 * 60_000;
const PREPARATION_TTL_MS = 6 * 60 * 60_000;
const running = (globalThis as typeof globalThis & { __canvasZipRunning?: Set<string> });
const active = running.__canvasZipRunning ??= new Set<string>();
function sanitizeFolderName(value: string): string {
@ -98,63 +100,71 @@ async function processTask(id: string, userId: string, requestOrigin: string, co
task = await loadZipTask(id, userId);
if (!task || task.status === "completed" || task.status === "failed") return;
await assertZipDiskSpace();
task.status = "running"; task.stage = "正在准备原图"; task.updatedAt = Date.now();
const existingZip = task.zipFile && task.contentDisposition && await stat(task.zipFile.path).catch(() => null);
const reuseZip = existingZip && existingZip.size === task.zipFile!.bytes;
task.status = "running"; task.stage = reuseZip ? "正在准备加速下载" : "正在准备原图"; task.updatedAt = Date.now();
await saveZipTask(task);
// Stage each image on disk, not all PNG buffers in the web process heap.
// A restart resumes from these files using the current owner's session.
let cursor = 0;
let writes = Promise.resolve();
const persist = () => { const current = task!; writes = writes.then(() => saveZipTask(current)); return writes; };
const results = await Promise.allSettled(Array.from({ length: Math.min(2, task.files.length) }, async () => {
while (cursor < task!.files.length) {
const index = cursor++; const file = task!.files[index];
const path = join(zipTaskPath(id), `${index}.image`);
if (file.prepared && file.filename && (await stat(path).catch(() => null))?.size) continue;
delete file.prepared; delete file.filename;
const result = await fetchCanvasDownloadAsset({ file, index, requestOrigin, cookie,
signal: AbortSignal.timeout(120_000) }).catch(() => ({ ok: false as const, error: "原图读取超时或源地址暂时不可用" }));
if (result.ok) {
await assertZipDiskSpace(result.bytes.length);
await writeFile(path, result.bytes, { mode: 0o600 });
file.prepared = `${index}.image`; file.filename = result.filename; delete file.error;
} else file.error = result.error;
task!.processedCount = task!.files.filter(f => f.prepared || f.error).length;
task!.successCount = task!.files.filter(f => f.prepared).length;
task!.failedCount = task!.files.filter(f => f.error).length;
task!.stage = `正在准备原图 ${task!.processedCount}/${task!.totalCount}`;
task!.updatedAt = Date.now(); await persist();
}
}));
const rejected = results.find(result => result.status === "rejected");
if (rejected?.status === "rejected") throw rejected.reason;
if (!task.successCount) throw new Error("所选原图暂时都无法读取，请稍后重试；画布图片不会被删除");
const zip = new JSZip(); const folder = zip.folder(task.folderName) || zip;
for (const file of task.files) if (file.prepared && file.filename)
folder.file(file.filename, createReadStream(join(zipTaskPath(id), file.prepared))
.once("close", () => { void rm(join(zipTaskPath(id), file.prepared!), { force: true }).catch(() => undefined); }));
if (task.failedCount) folder.file("_download-failures.txt", task.files.filter(f => f.error)
.map(f => `${f.name || "图片"}：${f.error}`).join("\n"));
task.stage = "正在生成完整 ZIP 文件"; await saveZipTask(task);
task.zipFile = await writeDownloadZip(zip, task.id, percent => { task!.compressPercent = percent; });
task.contentDisposition = buildContentDisposition(task.folderName);
if (!reuseZip) {
delete task.zipFile; delete task.contentDisposition; delete task.cdnUrl;
// Stage each image on disk, not all PNG buffers in the web process heap.
// A restart resumes from these files using the current owner's session.
let cursor = 0;
let writes = Promise.resolve();
const persist = () => { const current = task!; writes = writes.then(() => saveZipTask(current)); return writes; };
const results = await Promise.allSettled(Array.from({ length: Math.min(2, task.files.length) }, async () => {
while (cursor < task!.files.length) {
const index = cursor++; const file = task!.files[index];
const path = join(zipTaskPath(id), `${index}.image`);
if (file.prepared && file.filename && (await stat(path).catch(() => null))?.size) continue;
delete file.prepared; delete file.filename;
const result = await fetchCanvasDownloadAsset({ file, index, requestOrigin, cookie,
signal: AbortSignal.timeout(120_000) }).catch(() => ({ ok: false as const, error: "原图读取超时或源地址暂时不可用" }));
if (result.ok) {
await assertZipDiskSpace(result.bytes.length);
await writeFile(path, result.bytes, { mode: 0o600 });
file.prepared = `${index}.image`; file.filename = result.filename; delete file.error;
} else file.error = result.error;
task!.processedCount = task!.files.filter(f => f.prepared || f.error).length;
task!.successCount = task!.files.filter(f => f.prepared).length;
task!.failedCount = task!.files.filter(f => f.error).length;
task!.stage = `正在准备原图 ${task!.processedCount}/${task!.totalCount}`;
task!.updatedAt = Date.now(); await persist();
}
}));
const rejected = results.find(result => result.status === "rejected");
if (rejected?.status === "rejected") throw rejected.reason;
if (!task.successCount) throw new Error("所选原图暂时都无法读取，请稍后重试；画布图片不会被删除");
const zip = new JSZip(); const folder = zip.folder(task.folderName) || zip;
for (const file of task.files) if (file.prepared && file.filename)
folder.file(file.filename, createReadStream(join(zipTaskPath(id), file.prepared))
.once("close", () => { void rm(join(zipTaskPath(id), file.prepared!), { force: true }).catch(() => undefined); }));
if (task.failedCount) folder.file("_download-failures.txt", task.files.filter(f => f.error)
.map(f => `${f.name || "图片"}：${f.error}`).join("\n"));
task.stage = "正在生成完整 ZIP 文件"; await saveZipTask(task);
task.zipFile = await writeDownloadZip(zip, task.id, percent => { task!.compressPercent = percent; });
task.contentDisposition = buildContentDisposition(task.folderName);
}
task.stage = "正在准备下载链接"; await saveZipTask(task);
try {
const cdn = await publishCanvasZip({ id: task.id, ...task.zipFile, disposition: task.contentDisposition });
if (cdn) {
task.cdnObject = { storageConfigId: cdn.storageConfigId, key: cdn.key };
if (cdn.url) task.cdnUrl = cdn.url;
}
} catch { console.warn("[CanvasDownloadZipTask] CDN delivery unavailable; complete local ZIP retained", { taskId: id }); }
const cdn = await publishCanvasZip({ id: task.id, ...task.zipFile!, disposition: task.contentDisposition!,
onProgress: async bytes => {
task!.stage = `正在传送至七牛 ${Math.round(bytes / task!.zipFile!.bytes * 100)}%`;
task!.updatedAt = Date.now(); await saveZipTask(task!);
} });
if (cdn) {
task.cdnObject = { storageConfigId: cdn.storageConfigId, key: cdn.key };
if (!cdn.url) throw new Error(cdn.error || `ZIP 已打包，但加速下载未就绪，请再次点击下载继续传送原 ZIP。下载编号：${task.id}`);
task.cdnUrl = cdn.url;
}
task.status = "completed"; task.stage = task.failedCount ? `已打包 ${task.successCount} 张，${task.failedCount} 张未能读取` : "打包完成";
task.compressPercent = 100; task.updatedAt = Date.now(); task.expiresAt = Date.now() + TASK_TTL_MS;
await saveZipTask(task);
// The complete ZIP is enough for retries; release staged duplicate bytes.
await Promise.all(task.files.filter(f => f.prepared).map(f => rm(join(zipTaskPath(id), f.prepared!), { force: true }).catch(() => undefined)));
console.info("[CanvasDownloadZipTask] Complete", { taskId: id, count: task.successCount, failed: task.failedCount,
bytes: task.zipFile.bytes, delivery: task.cdnUrl ? "cdn" : "local" });
bytes: task.zipFile!.bytes, delivery: task.cdnUrl ? "cdn" : "local" });
} catch (error) {
if (task) {
task.status = "failed"; task.stage = "打包未完成";
task.status = "failed"; task.stage = task.zipFile ? "ZIP 已打包，加速下载未就绪" : "打包未完成";
task.error = error instanceof Error ? error.message : "打包失败，请稍后重试";
task.updatedAt = Date.now(); task.expiresAt = Date.now() + TASK_TTL_MS; await saveZipTask(task).catch(() => undefined);
}
@ -162,7 +172,7 @@ async function processTask(id: string, userId: string, requestOrigin: string, co
} finally { await release?.().catch(() => undefined); active.delete(id); }
}
function schedule(task: ZipTask, request: NextRequest) {
const run = () => processTask(task.id, task.userId, getRequestOrigin(request), request.headers.get("cookie") || "");
const run = () => trackRuntimeWork(() => processTask(task.id, task.userId, getRequestOrigin(request), request.headers.get("cookie") || ""));
after(run);
}
export async function POST(request: NextRequest) {
@ -187,14 +197,17 @@ export async function POST(request: NextRequest) {
const now = Date.now();
task = { id, userId: session.user.id, folderName: sanitizeFolderName(String(body.folderName || "")), files,
totalCount: files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0,
status: "pending", stage: "等待后台准备", createdAt: now, updatedAt: now, expiresAt: now + 6 * TASK_TTL_MS };
status: "pending", stage: "等待后台准备", createdAt: now, updatedAt: now, expiresAt: now + PREPARATION_TTL_MS };
await saveZipTask(task);
} else if (task.status === "failed" && body.retry === true) {
} else if ((task.status === "failed" || task.status === "completed" && task.cdnObject && !task.cdnUrl) && body.retry === true) {
task.status = "pending"; task.error = undefined;
await assertZipTaskCapacity(session.user.id);
for (const file of task.files) delete file.error;
task.processedCount = task.successCount = task.files.filter(file => file.prepared).length;
task.failedCount = 0; task.updatedAt = Date.now(); task.expiresAt = Date.now() + 6 * TASK_TTL_MS;
if (!task.zipFile) {
for (const file of task.files) delete file.error;
task.processedCount = task.successCount = task.files.filter(file => file.prepared).length;
task.failedCount = 0;
}
task.updatedAt = Date.now(); task.expiresAt = Date.now() + PREPARATION_TTL_MS;
await saveZipTask(task);
}
if (task.status === "pending" || task.status === "running") schedule(task, request);

7
docs/production-48.md
View File

@ -98,3 +98,10 @@ public 先从当前运行版本创建写入时复制副本，保留未纳入 Git
仓库历史全量 lint 仍有已记录的问题，不能声称全部质量门禁通过。
每次 PR 须记录本次改动的验证、历史失败和未覆盖项目；当前流程不会自动触发生产发布。
## ZIP 下载交付
默认存储为七牛时，完整 ZIP 通过分片上传至七牛，确认下载地址可用后才提供下载。
大文件使用三个并发分片，上传进度显示在原打包任务中。传送或链接检查失败会返回具体阶段、
下载编号及重试建议，保留已生成的 ZIP；再次点击下载复用原文件，已上传的完整对象不重复上传。
后台 ZIP 任务参与发布期间的工作等待。完成或失败的任务及文件保留 24 小时，待处理任务保留 6 小时。

100
lib/storage/canvas-zip-delivery.ts
View File

@ -1,11 +1,72 @@
import "server-only";
import { createReadStream } from "node:fs";
import { PutObjectCommand } from "@aws-sdk/client-s3";
import { open } from "node:fs/promises";
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, HeadObjectCommand, PutObjectCommand, UploadPartCommand, type S3Client } from "@aws-sdk/client-s3";
import { getDefaultStorageConfig, getStorageConfigById } from "@/lib/db/queries";
import { createS3Client, toDbS3UploadConfig, buildS3PublicUrl } from "./service";
/** Only a completely written archive is uploaded. Local GET remains a fallback. */
export async function publishCanvasZip(params: { id: string; path: string; bytes: number; disposition: string }) {
type ZipPublication = { id: string; path: string; bytes: number; disposition: string; onProgress?: (bytes: number) => Promise<void> };
const PART_BYTES = 8 * 1024 ** 2;
async function uploadZip(client: S3Client, bucket: string, key: string, params: ZipPublication) {
const metadata = { Bucket: bucket, Key: key, ContentType: "application/zip", ContentDisposition: params.disposition, CacheControl: "public, max-age=1800" };
if (params.bytes <= 2 * PART_BYTES) {
const body = createReadStream(params.path);
try {
await client.send(new PutObjectCommand({ ...metadata, Body: body, ContentLength: params.bytes }), { abortSignal: AbortSignal.timeout(90_000) });
await params.onProgress?.(params.bytes);
} finally { body.destroy(); }
return;
}
const initiated = await client.send(new CreateMultipartUploadCommand(metadata), { abortSignal: AbortSignal.timeout(15_000) });
if (!initiated.UploadId) throw new Error("七牛未返回分片上传编号");
const upload = { Bucket: bucket, Key: key, UploadId: initiated.UploadId };
const handle = await open(params.path, "r").catch(async error => {
await client.send(new AbortMultipartUploadCommand(upload), { abortSignal: AbortSignal.timeout(10_000) }).catch(() => undefined);
throw error;
});
const parts: Array<{ PartNumber: number; ETag: string }> = [];
let cursor = 0;
let uploaded = 0;
let progress = Promise.resolve();
const controller = new AbortController();
try {
const workers = await Promise.allSettled(Array.from({ length: 3 }, async () => {
try {
while (!controller.signal.aborted) {
const index = cursor++;
const start = index * PART_BYTES;
if (start >= params.bytes) return;
const body = Buffer.allocUnsafe(Math.min(PART_BYTES, params.bytes - start));
let read = 0;
while (read < body.length) {
const result = await handle.read(body, read, body.length - read, start + read);
if (!result.bytesRead) throw new Error("完整 ZIP 文件读取中断，请重试下载");
read += result.bytesRead;
}
const part = await client.send(new UploadPartCommand({ ...upload, PartNumber: index + 1, Body: body, ContentLength: body.length }), {
abortSignal: AbortSignal.any([controller.signal, AbortSignal.timeout(180_000)]),
});
if (!part.ETag) throw new Error("七牛未确认上传分片");
parts.push({ PartNumber: index + 1, ETag: part.ETag });
uploaded += body.length;
const bytes = uploaded;
progress = progress.then(() => params.onProgress?.(bytes));
await progress;
}
} catch (error) { controller.abort(); throw error; }
}));
const failure = workers.find(worker => worker.status === "rejected");
if (failure?.status === "rejected") throw failure.reason;
await client.send(new CompleteMultipartUploadCommand({ ...upload, MultipartUpload: { Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber) } }), { abortSignal: AbortSignal.timeout(30_000) });
} catch (error) {
await client.send(new AbortMultipartUploadCommand(upload), { abortSignal: AbortSignal.timeout(10_000) }).catch(() => undefined);
throw error;
} finally { await handle.close(); }
}
/** Publish the complete ZIP before offering the accelerated download. */
export async function publishCanvasZip(params: ZipPublication) {
const raw = await getDefaultStorageConfig();
const config = toDbS3UploadConfig(raw);
if (config?.provider !== "qiniu-kodo" || !config.publicUrl?.startsWith("https://")) return null;
@ -13,20 +74,37 @@ export async function publishCanvasZip(params: { id: string; path: string; bytes
const client = createS3Client(config);
// Qiniu's S3 endpoint expects an ordinary known-length body, without AWS trailers.
client.config.requestChecksumCalculation = async () => "WHEN_REQUIRED";
const body = createReadStream(params.path);
let phase: "check" | "upload" | "verify" = "check";
const startedAt = Date.now();
let uploadedBytes = 0;
try {
await client.send(new PutObjectCommand({ Bucket: config.bucket, Key: key, Body: body,
ContentLength: params.bytes, ContentType: "application/zip", ContentDisposition: params.disposition,
CacheControl: "public, max-age=1800" }), { abortSignal: AbortSignal.timeout(90_000) });
const url = buildS3PublicUrl(config, key);
const existing = await client.send(new HeadObjectCommand({ Bucket: config.bucket, Key: key }), { abortSignal: AbortSignal.timeout(15_000) }).catch(error => {
if (error.$metadata?.httpStatusCode === 404) return null;
throw error;
});
if (existing?.ContentLength !== params.bytes) {
phase = "upload";
await uploadZip(client, config.bucket, key, { ...params, onProgress: async bytes => {
uploadedBytes = bytes;
await params.onProgress?.(bytes);
} });
}
phase = "verify";
const target = new URL(buildS3PublicUrl(config, key));
target.searchParams.set("v", String(params.bytes));
const url = target.toString();
const response = await fetch(url, { method: "HEAD", redirect: "error", signal: AbortSignal.timeout(10_000) });
const length = Number(response.headers.get("content-length"));
if (!response.ok || length !== params.bytes) throw new Error("CDN object is not ready");
return { url, storageConfigId: raw!.id, key };
} catch (error) {
console.warn("[CanvasDownloadZipTask] CDN upload/check failed; using complete local file", { taskId: params.id, kind: (error as Error).name });
return { url: null, storageConfigId: raw!.id, key };
} finally { body.destroy(); client.destroy(); }
const kind = (error as Error).name;
const status = (error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode;
console.warn("[CanvasDownloadZipTask] CDN publication failed", { taskId: params.id, phase, kind, status, bytes: params.bytes, uploadedBytes, elapsedMs: Date.now() - startedAt });
const reason = status === 401 || status === 403 ? `七牛拒绝访问（HTTP ${status}），请管理员检查存储权限` : phase === "check" ? "暂时无法确认七牛文件状态" : phase === "verify" ? "七牛下载链接尚未可用" : /AbortError|TimeoutError/.test(kind) ? "传送至七牛时超时" : "传送至七牛失败";
const advice = status === 401 || status === 403 ? "存储权限恢复后再次点击下载即可继续传送原 ZIP" : "请再次点击下载以继续传送原 ZIP";
return { url: null, storageConfigId: raw!.id, key, error: `ZIP 已打包完成，但${reason}。${advice}；不会重新生成图片。若持续失败，请联系管理员并提供下载编号：${params.id}` };
} finally { client.destroy(); }
}
export async function removeCanvasZipObject(object: { storageConfigId: string; key: string }) {

137
tests/routes/canvas-zip-download.test.ts
View File

@ -7,6 +7,9 @@ import { randomUUID } from "node:crypto";
import JSZip from "jszip";
import { serverLoader } from "../helpers/canvas-download-server-loader";
import { parseDownloadByteRange } from "@/lib/canvas/download-byte-range";
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, HeadObjectCommand, PutObjectCommand, UploadPartCommand } from "@aws-sdk/client-s3";
const runtimeWork = { trackRuntimeWork: async (run: () => Promise<void>) => run() };
test("ranges reject invalid offsets and preserve normal, open and suffix byte ranges", () => {
expect(parseDownloadByteRange("bytes=0-3", 20)).toEqual({start:0,end:3});
@ -20,7 +23,7 @@ test("multi-select ZIP admission is immediate; complete archives survive restart
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-test-')); let owner:string|null='owner';let fetches=0;
const callbacks:Array<()=>Promise<void>>=[];
const mocks={
'@/app/(auth)/auth': {auth:async()=>owner?{user:{id:owner}}:null},
'@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth': {auth:async()=>owner?{user:{id:owner}}:null},
'next/server': {NextResponse,after:(run:()=>Promise<void>)=>callbacks.push(run)},
'@/lib/canvas/photoshop-safe-download':{fetchCanvasDownloadAsset:async({index}:any)=>{
fetches++;return {ok:true,filename:`${index}-花瓶.png`,bytes:Buffer.from(`image-${index}`),contentType:'image/png'};
@ -56,7 +59,7 @@ test("multi-select ZIP admission is immediate; complete archives survive restart
test("all-source failures never produce an empty successful ZIP; retries reuse the task and resolve sources again", async()=>{
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-retry-'));const callbacks:Array<()=>Promise<void>>=[];let available=false;
const route=serverLoader({
'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})},
'@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})},
'next/server':{NextResponse,after:(run:()=>Promise<void>)=>callbacks.push(run)},
'@/lib/canvas/photoshop-safe-download':{fetchCanvasDownloadAsset:async()=>available?{ok:true,filename:'one.png',bytes:Buffer.from('one'),contentType:'image/png'}:{ok:false,error:'expired provider URL'}},
'@/lib/storage/canvas-zip-delivery':{publishCanvasZip:async()=>null},
@ -108,7 +111,7 @@ test("the browser polls a complete ZIP, preserves an idempotent task after a los
test("partial source failures are explicit; CDN redirects retain an authenticated local range fallback",async()=>{
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-partial-'));const callbacks:Array<()=>Promise<void>>=[];let owner='owner';
const route=serverLoader({
'@/app/(auth)/auth':{auth:async()=>({user:{id:owner}})},
'@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth':{auth:async()=>({user:{id:owner}})},
'next/server':{NextResponse,after:(run:()=>Promise<void>)=>callbacks.push(run)},
'@/lib/canvas/photoshop-safe-download':{fetchCanvasDownloadAsset:async({index}:any)=>index?{ok:false,error:'source expired'}:{ok:true,filename:'original.png',bytes:Buffer.from('original'),contentType:'image/png'}},
'@/lib/storage/canvas-zip-delivery':{publishCanvasZip:async()=>({url:'https://cdn.example/complete.zip',key:'canvas-downloads/x.zip',storageConfigId:'config'})},
@ -128,7 +131,7 @@ test("partial source failures are explicit; CDN redirects retain an authenticate
test("recovery validates staged files instead of treating missing bytes as successful images",async()=>{
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-missing-'));const callbacks:Array<()=>Promise<void>>=[];const id=randomUUID();
const load=serverLoader({
'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})},
'@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})},
'next/server':{NextResponse,after:(run:()=>Promise<void>)=>callbacks.push(run)},
'@/lib/canvas/photoshop-safe-download':{fetchCanvasDownloadAsset:async()=>({ok:false,error:'source expired'})},
'@/lib/storage/canvas-zip-delivery':{publishCanvasZip:async()=>null},
@ -157,7 +160,7 @@ test("large ZIP writing checks current free space as staged input bytes are rele
test("slow expired-object cleanup cannot keep the ZIP admission request open",async()=>{
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-cleanup-'));const callbacks:Array<()=>Promise<void>>=[];
const mocks:Record<string,unknown>={
'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})},
'@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})},
'next/server':{NextResponse,after:(run:()=>Promise<void>)=>callbacks.push(run)},
'@/lib/canvas/photoshop-safe-download':{},'@/lib/storage/canvas-zip-delivery':{},
};
@ -169,3 +172,127 @@ test("slow expired-object cleanup cannot keep the ZIP admission request open",as
expect(result).not.toBe('blocked');expect((result as Response).status).toBe(202);
}finally{await rm(dir,{recursive:true,force:true});}
});
test("large ZIPs use bounded parallel parts and publish only the complete ordered archive", async () => {
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-parts-'));
const bytes=Buffer.alloc(25*1024**2+19,0x5a);bytes[16*1024**2]=0xff;
const file=join(dir,'large.zip');await writeFile(file,bytes);
const parts=new Map<number,Buffer>();let inFlight=0;let peak=0;let finished=false;let destroyed=false;let trailers='';
const progress:number[]=[];
const client={config:{requestChecksumCalculation:async()=>''},destroy(){destroyed=true;},async send(command:any){
if(command instanceof HeadObjectCommand)throw Object.assign(new Error('missing'),{$metadata:{httpStatusCode:404}});
if(command instanceof CreateMultipartUploadCommand){expect(command.input.ContentDisposition).toContain('attachment');return {UploadId:'upload'};}
if(command instanceof UploadPartCommand){
const input=command.input;inFlight++;peak=Math.max(peak,inFlight);expect(input.ContentLength).toBeLessThanOrEqual(8*1024**2);
await new Promise(resolve=>setTimeout(resolve,input.PartNumber===1?30:20));
expect(Buffer.isBuffer(input.Body)).toBe(true);
parts.set(input.PartNumber!,Buffer.from(input.Body as Buffer));inFlight--;return {ETag:`part-${input.PartNumber}`};
}
expect(command).toBeInstanceOf(CompleteMultipartUploadCommand);
expect(command.input.MultipartUpload.Parts.map((p:any)=>p.PartNumber)).toEqual([1,2,3,4]);
expect(Buffer.concat(command.input.MultipartUpload.Parts.map((p:any)=>parts.get(p.PartNumber)!)).equals(bytes)).toBe(true);
finished=true;trailers=await client.config.requestChecksumCalculation();return {};
}};
const publish=serverLoader({
'@/lib/db/queries':{getDefaultStorageConfig:async()=>({id:'storage'})},
'./service':{toDbS3UploadConfig:()=>({provider:'qiniu-kodo',bucket:'bucket',publicUrl:'https://cdn.example'}),createS3Client:()=>client,buildS3PublicUrl:(_:any,key:string)=>'https://cdn.example/'+key},
fetch:async(url:string,init:any)=>{expect(finished).toBe(true);expect(init.method).toBe('HEAD');expect(new URL(url).searchParams.get('v')).toBe(String(bytes.length));return new Response(null,{headers:{'content-length':String(bytes.length)}});},
})('lib/storage/canvas-zip-delivery.ts').publishCanvasZip;
try{
const result=await publish({id:randomUUID(),path:file,bytes:bytes.length,disposition:'attachment; filename="large.zip"',onProgress:async(n:number)=>{progress.push(n);}});
expect(result.url).toContain('https://cdn.example/canvas-downloads/');expect(peak).toBeGreaterThan(1);expect(peak).toBeLessThanOrEqual(3);expect(progress.at(-1)).toBe(bytes.length);
expect(progress).toEqual([...progress].sort((a,b)=>a-b));expect(trailers).toBe('WHEN_REQUIRED');expect(destroyed).toBe(true);
}finally{await rm(dir,{recursive:true,force:true});}
});
test("failed multipart uploads abort their parts and retain the original ZIP with actionable advice",async()=>{
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-abort-'));const file=join(dir,'large.zip');const bytes=17*1024**2;
const handle=await (await import('node:fs/promises')).open(file,'w');await handle.truncate(bytes);await handle.close();
let aborted=false;let completed=false;
const client={config:{},destroy(){},async send(command:any){
if(command instanceof HeadObjectCommand)throw Object.assign(new Error('missing'),{$metadata:{httpStatusCode:404}});
if(command instanceof CreateMultipartUploadCommand)return {UploadId:'upload'};
if(command instanceof UploadPartCommand){if(command.input.PartNumber===2)throw new DOMException('part timeout','TimeoutError');return {ETag:'part'};}
if(command instanceof AbortMultipartUploadCommand){aborted=true;return {};}
completed=true;return {};
}};
const publish=serverLoader({
'@/lib/db/queries':{getDefaultStorageConfig:async()=>({id:'storage'})},
'./service':{toDbS3UploadConfig:()=>({provider:'qiniu-kodo',bucket:'bucket',publicUrl:'https://cdn.example'}),createS3Client:()=>client},
})('lib/storage/canvas-zip-delivery.ts').publishCanvasZip;
try{
const result=await publish({id:randomUUID(),path:file,bytes,disposition:'attachment'});
expect(result.url).toBeNull();expect(result.error).toContain('传送至七牛时超时');expect(result.error).toContain('继续传送原 ZIP');
expect(aborted).toBe(true);expect(completed).toBe(false);expect((await (await import('node:fs/promises')).stat(file)).size).toBe(bytes);
}finally{await rm(dir,{recursive:true,force:true});}
});
test("retrying CDN verification reuses an already uploaded ZIP without uploading it again",async()=>{
let uploads=0;let available=false;
const client={config:{},destroy(){},async send(command:any){
if(command instanceof HeadObjectCommand)return {ContentLength:100};
if(command instanceof PutObjectCommand||command instanceof CreateMultipartUploadCommand)uploads++;
throw new Error('Unexpected upload');
}};
const publish=serverLoader({
'@/lib/db/queries':{getDefaultStorageConfig:async()=>({id:'storage'})},
'./service':{toDbS3UploadConfig:()=>({provider:'qiniu-kodo',bucket:'bucket',publicUrl:'https://cdn.example'}),createS3Client:()=>client,buildS3PublicUrl:(_:any,key:string)=>'https://cdn.example/'+key},
fetch:async()=>new Response(null,{status:available?200:404,headers:{'content-length':available?'100':'0'}}),
})('lib/storage/canvas-zip-delivery.ts').publishCanvasZip;
const params={id:randomUUID(),path:'unused-already-published.zip',bytes:100,disposition:'attachment'};
const first=await publish(params);expect(first.url).toBeNull();expect(first.error).toContain('七牛下载链接尚未可用');
available=true;expect((await publish(params)).url).toContain('https://cdn.example/');expect(uploads).toBe(0);
});
test("CDN failures never silently start local downloads; retries reuse the ZIP and keep it resumable beyond one hour",async()=>{
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-cdn-retry-'));const callbacks:Array<()=>Promise<void>>=[];
let fetches=0;let publications=0;let available=false;let now=Date.now();
let activeWork=0;
const runtimeWork={trackRuntimeWork:async(run:()=>Promise<void>)=>{activeWork++;try{return await run();}finally{activeWork--;}}};
class Clock extends Date{static now(){return now;}}
const load=serverLoader({
Date:Clock,'@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})},
'next/server':{NextResponse,after:(run:()=>Promise<void>)=>callbacks.push(run)},
'@/lib/canvas/photoshop-safe-download':{fetchCanvasDownloadAsset:async()=>{fetches++;return {ok:true,filename:'original.png',bytes:Buffer.from('original'),contentType:'image/png'};}},
'@/lib/storage/canvas-zip-delivery':{publishCanvasZip:async()=>{expect(activeWork).toBe(1);publications++;return {url:available?'https://cdn.example/complete.zip':null,key:'canvas-downloads/key.zip',storageConfigId:'storage',error:'ZIP 已打包，传送至七牛时超时；请再次点击下载继续原 ZIP，不会重新生成图片。'};},removeCanvasZipObject:async()=>{}},
},{CANVAS_DOWNLOAD_ZIP_DIR:dir});
const route=load('app/api/canvas/download-zip-task/route.ts');const id=randomUUID();
const post=()=>new NextRequest('http://localhost/api/canvas/download-zip-task',{method:'POST',body:JSON.stringify({taskId:id,retry:true,files:[{url:'https://cdn.example/original.png'}]})});
const request=(suffix='')=>new NextRequest(`http://localhost/api/canvas/download-zip-task?taskId=${id}${suffix}`,{headers:{range:'bytes=0-3'}});
try{
await route.POST(post());while(callbacks.length)await callbacks.shift()!();
const failed=(await(await route.GET(request())).json()).task;
expect(failed).toMatchObject({id,status:'failed',successCount:1,downloadUrl:''});expect(failed.error).toContain('不会重新生成图片');
expect((await route.GET(request('&download=1'))).status).toBe(409);
const before=await readFile(join(dir,id,'task.json'),'utf8');const original=Buffer.from(await readFile(JSON.parse(before).zipFile.path));
available=true;await route.POST(post());while(callbacks.length)await callbacks.shift()!();
expect(fetches).toBe(1);expect(publications).toBe(2);
const manifest=JSON.parse(await readFile(join(dir,id,'task.json'),'utf8'));expect(await readFile(manifest.zipFile.path)).toEqual(original);
delete manifest.cdnUrl;
await load('lib/canvas/download-zip-task-store.ts').saveZipTask(manifest);
await route.POST(post());while(callbacks.length)await callbacks.shift()!();
expect(fetches).toBe(1);expect(publications).toBe(3);
expect(activeWork).toBe(0);
now+=2*60*60_000;
expect((await route.GET(request('&download=1'))).status).toBe(307);
const local=await route.GET(request('&download=1&local=1'));expect(local.status).toBe(206);expect(Buffer.from(await local.arrayBuffer())).toEqual(original.subarray(0,4));
}finally{await rm(dir,{recursive:true,force:true});}
});
test("small ZIP delivery preserves the complete file and download filename",async()=>{
const dir=await mkdtemp(join(tmpdir(),'canvas-zip-small-'));const file=join(dir,'small.zip');const bytes=Buffer.from('PK small complete archive');await writeFile(file,bytes);
let written=false;
const client={config:{},destroy(){},async send(command:any){
if(command instanceof HeadObjectCommand)throw Object.assign(new Error('missing'),{$metadata:{httpStatusCode:404}});
expect(command).toBeInstanceOf(PutObjectCommand);expect(command.input.ContentLength).toBe(bytes.length);expect(command.input.ContentDisposition).toContain('small.zip');
const chunks:Buffer[]=[];for await(const chunk of command.input.Body)chunks.push(Buffer.from(chunk));
expect(Buffer.concat(chunks).equals(bytes)).toBe(true);written=true;return {};
}};
const publish=serverLoader({
'@/lib/db/queries':{getDefaultStorageConfig:async()=>({id:'storage'})},
'./service':{toDbS3UploadConfig:()=>({provider:'qiniu-kodo',bucket:'bucket',publicUrl:'https://cdn.example'}),createS3Client:()=>client,buildS3PublicUrl:(_:any,key:string)=>'https://cdn.example/'+key},
fetch:async()=>{expect(written).toBe(true);return new Response(null,{headers:{'content-length':String(bytes.length)}});},
})('lib/storage/canvas-zip-delivery.ts').publishCanvasZip;
try{expect((await publish({id:randomUUID(),path:file,bytes:bytes.length,disposition:'attachment; filename="small.zip"'})).url).toContain('https://cdn.example/');}
finally{await rm(dir,{recursive:true,force:true});}
});