diff --git a/app/api/canvas/download-zip-task/route.ts b/app/api/canvas/download-zip-task/route.ts index 2f5de4a34..33f281d1c 100644 --- a/app/api/canvas/download-zip-task/route.ts +++ b/app/api/canvas/download-zip-task/route.ts @@ -10,11 +10,13 @@ import JSZip from "jszip"; import { auth } from "@/app/(auth)/auth"; import { fetchCanvasDownloadAsset } from "@/lib/canvas/photoshop-safe-download"; import { publishCanvasZip } from "@/lib/storage/canvas-zip-delivery"; +import { trackRuntimeWork } from "@/lib/observability/active-work"; export const runtime = "nodejs"; export const maxDuration = 300; type DownloadFileItem = { url: string; name?: string }; const MAX_DOWNLOAD_FILES = 200; -const TASK_TTL_MS = 60 * 60_000; +const TASK_TTL_MS = 24 * 60 * 60_000; +const PREPARATION_TTL_MS = 6 * 60 * 60_000; const running = (globalThis as typeof globalThis & { __canvasZipRunning?: Set }); const active = running.__canvasZipRunning ??= new Set(); function sanitizeFolderName(value: string): string { @@ -98,63 +100,71 @@ async function processTask(id: string, userId: string, requestOrigin: string, co task = await loadZipTask(id, userId); if (!task || task.status === "completed" || task.status === "failed") return; await assertZipDiskSpace(); - task.status = "running"; task.stage = "正在准备原图"; task.updatedAt = Date.now(); + const existingZip = task.zipFile && task.contentDisposition && await stat(task.zipFile.path).catch(() => null); + const reuseZip = existingZip && existingZip.size === task.zipFile!.bytes; + task.status = "running"; task.stage = reuseZip ? "正在准备加速下载" : "正在准备原图"; task.updatedAt = Date.now(); await saveZipTask(task); - // Stage each image on disk, not all PNG buffers in the web process heap. - // A restart resumes from these files using the current owner's session. - let cursor = 0; - let writes = Promise.resolve(); - const persist = () => { const current = task!; writes = writes.then(() => saveZipTask(current)); return writes; }; - const results = await Promise.allSettled(Array.from({ length: Math.min(2, task.files.length) }, async () => { - while (cursor < task!.files.length) { - const index = cursor++; const file = task!.files[index]; - const path = join(zipTaskPath(id), `${index}.image`); - if (file.prepared && file.filename && (await stat(path).catch(() => null))?.size) continue; - delete file.prepared; delete file.filename; - const result = await fetchCanvasDownloadAsset({ file, index, requestOrigin, cookie, - signal: AbortSignal.timeout(120_000) }).catch(() => ({ ok: false as const, error: "原图读取超时或源地址暂时不可用" })); - if (result.ok) { - await assertZipDiskSpace(result.bytes.length); - await writeFile(path, result.bytes, { mode: 0o600 }); - file.prepared = `${index}.image`; file.filename = result.filename; delete file.error; - } else file.error = result.error; - task!.processedCount = task!.files.filter(f => f.prepared || f.error).length; - task!.successCount = task!.files.filter(f => f.prepared).length; - task!.failedCount = task!.files.filter(f => f.error).length; - task!.stage = `正在准备原图 ${task!.processedCount}/${task!.totalCount}`; - task!.updatedAt = Date.now(); await persist(); - } - })); - const rejected = results.find(result => result.status === "rejected"); - if (rejected?.status === "rejected") throw rejected.reason; - if (!task.successCount) throw new Error("所选原图暂时都无法读取,请稍后重试;画布图片不会被删除"); - const zip = new JSZip(); const folder = zip.folder(task.folderName) || zip; - for (const file of task.files) if (file.prepared && file.filename) - folder.file(file.filename, createReadStream(join(zipTaskPath(id), file.prepared)) - .once("close", () => { void rm(join(zipTaskPath(id), file.prepared!), { force: true }).catch(() => undefined); })); - if (task.failedCount) folder.file("_download-failures.txt", task.files.filter(f => f.error) - .map(f => `${f.name || "图片"}:${f.error}`).join("\n")); - task.stage = "正在生成完整 ZIP 文件"; await saveZipTask(task); - task.zipFile = await writeDownloadZip(zip, task.id, percent => { task!.compressPercent = percent; }); - task.contentDisposition = buildContentDisposition(task.folderName); + if (!reuseZip) { + delete task.zipFile; delete task.contentDisposition; delete task.cdnUrl; + // Stage each image on disk, not all PNG buffers in the web process heap. + // A restart resumes from these files using the current owner's session. + let cursor = 0; + let writes = Promise.resolve(); + const persist = () => { const current = task!; writes = writes.then(() => saveZipTask(current)); return writes; }; + const results = await Promise.allSettled(Array.from({ length: Math.min(2, task.files.length) }, async () => { + while (cursor < task!.files.length) { + const index = cursor++; const file = task!.files[index]; + const path = join(zipTaskPath(id), `${index}.image`); + if (file.prepared && file.filename && (await stat(path).catch(() => null))?.size) continue; + delete file.prepared; delete file.filename; + const result = await fetchCanvasDownloadAsset({ file, index, requestOrigin, cookie, + signal: AbortSignal.timeout(120_000) }).catch(() => ({ ok: false as const, error: "原图读取超时或源地址暂时不可用" })); + if (result.ok) { + await assertZipDiskSpace(result.bytes.length); + await writeFile(path, result.bytes, { mode: 0o600 }); + file.prepared = `${index}.image`; file.filename = result.filename; delete file.error; + } else file.error = result.error; + task!.processedCount = task!.files.filter(f => f.prepared || f.error).length; + task!.successCount = task!.files.filter(f => f.prepared).length; + task!.failedCount = task!.files.filter(f => f.error).length; + task!.stage = `正在准备原图 ${task!.processedCount}/${task!.totalCount}`; + task!.updatedAt = Date.now(); await persist(); + } + })); + const rejected = results.find(result => result.status === "rejected"); + if (rejected?.status === "rejected") throw rejected.reason; + if (!task.successCount) throw new Error("所选原图暂时都无法读取,请稍后重试;画布图片不会被删除"); + const zip = new JSZip(); const folder = zip.folder(task.folderName) || zip; + for (const file of task.files) if (file.prepared && file.filename) + folder.file(file.filename, createReadStream(join(zipTaskPath(id), file.prepared)) + .once("close", () => { void rm(join(zipTaskPath(id), file.prepared!), { force: true }).catch(() => undefined); })); + if (task.failedCount) folder.file("_download-failures.txt", task.files.filter(f => f.error) + .map(f => `${f.name || "图片"}:${f.error}`).join("\n")); + task.stage = "正在生成完整 ZIP 文件"; await saveZipTask(task); + task.zipFile = await writeDownloadZip(zip, task.id, percent => { task!.compressPercent = percent; }); + task.contentDisposition = buildContentDisposition(task.folderName); + } task.stage = "正在准备下载链接"; await saveZipTask(task); - try { - const cdn = await publishCanvasZip({ id: task.id, ...task.zipFile, disposition: task.contentDisposition }); - if (cdn) { - task.cdnObject = { storageConfigId: cdn.storageConfigId, key: cdn.key }; - if (cdn.url) task.cdnUrl = cdn.url; - } - } catch { console.warn("[CanvasDownloadZipTask] CDN delivery unavailable; complete local ZIP retained", { taskId: id }); } + const cdn = await publishCanvasZip({ id: task.id, ...task.zipFile!, disposition: task.contentDisposition!, + onProgress: async bytes => { + task!.stage = `正在传送至七牛 ${Math.round(bytes / task!.zipFile!.bytes * 100)}%`; + task!.updatedAt = Date.now(); await saveZipTask(task!); + } }); + if (cdn) { + task.cdnObject = { storageConfigId: cdn.storageConfigId, key: cdn.key }; + if (!cdn.url) throw new Error(cdn.error || `ZIP 已打包,但加速下载未就绪,请再次点击下载继续传送原 ZIP。下载编号:${task.id}`); + task.cdnUrl = cdn.url; + } task.status = "completed"; task.stage = task.failedCount ? `已打包 ${task.successCount} 张,${task.failedCount} 张未能读取` : "打包完成"; task.compressPercent = 100; task.updatedAt = Date.now(); task.expiresAt = Date.now() + TASK_TTL_MS; await saveZipTask(task); // The complete ZIP is enough for retries; release staged duplicate bytes. await Promise.all(task.files.filter(f => f.prepared).map(f => rm(join(zipTaskPath(id), f.prepared!), { force: true }).catch(() => undefined))); console.info("[CanvasDownloadZipTask] Complete", { taskId: id, count: task.successCount, failed: task.failedCount, - bytes: task.zipFile.bytes, delivery: task.cdnUrl ? "cdn" : "local" }); + bytes: task.zipFile!.bytes, delivery: task.cdnUrl ? "cdn" : "local" }); } catch (error) { if (task) { - task.status = "failed"; task.stage = "打包未完成"; + task.status = "failed"; task.stage = task.zipFile ? "ZIP 已打包,加速下载未就绪" : "打包未完成"; task.error = error instanceof Error ? error.message : "打包失败,请稍后重试"; task.updatedAt = Date.now(); task.expiresAt = Date.now() + TASK_TTL_MS; await saveZipTask(task).catch(() => undefined); } @@ -162,7 +172,7 @@ async function processTask(id: string, userId: string, requestOrigin: string, co } finally { await release?.().catch(() => undefined); active.delete(id); } } function schedule(task: ZipTask, request: NextRequest) { - const run = () => processTask(task.id, task.userId, getRequestOrigin(request), request.headers.get("cookie") || ""); + const run = () => trackRuntimeWork(() => processTask(task.id, task.userId, getRequestOrigin(request), request.headers.get("cookie") || "")); after(run); } export async function POST(request: NextRequest) { @@ -187,14 +197,17 @@ export async function POST(request: NextRequest) { const now = Date.now(); task = { id, userId: session.user.id, folderName: sanitizeFolderName(String(body.folderName || "")), files, totalCount: files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0, - status: "pending", stage: "等待后台准备", createdAt: now, updatedAt: now, expiresAt: now + 6 * TASK_TTL_MS }; + status: "pending", stage: "等待后台准备", createdAt: now, updatedAt: now, expiresAt: now + PREPARATION_TTL_MS }; await saveZipTask(task); - } else if (task.status === "failed" && body.retry === true) { + } else if ((task.status === "failed" || task.status === "completed" && task.cdnObject && !task.cdnUrl) && body.retry === true) { task.status = "pending"; task.error = undefined; await assertZipTaskCapacity(session.user.id); - for (const file of task.files) delete file.error; - task.processedCount = task.successCount = task.files.filter(file => file.prepared).length; - task.failedCount = 0; task.updatedAt = Date.now(); task.expiresAt = Date.now() + 6 * TASK_TTL_MS; + if (!task.zipFile) { + for (const file of task.files) delete file.error; + task.processedCount = task.successCount = task.files.filter(file => file.prepared).length; + task.failedCount = 0; + } + task.updatedAt = Date.now(); task.expiresAt = Date.now() + PREPARATION_TTL_MS; await saveZipTask(task); } if (task.status === "pending" || task.status === "running") schedule(task, request); diff --git a/docs/production-48.md b/docs/production-48.md index 4d830cfad..5ef5adf13 100644 --- a/docs/production-48.md +++ b/docs/production-48.md @@ -98,3 +98,10 @@ public 先从当前运行版本创建写入时复制副本,保留未纳入 Git 仓库历史全量 lint 仍有已记录的问题,不能声称全部质量门禁通过。 每次 PR 须记录本次改动的验证、历史失败和未覆盖项目;当前流程不会自动触发生产发布。 + +## ZIP 下载交付 + +默认存储为七牛时,完整 ZIP 通过分片上传至七牛,确认下载地址可用后才提供下载。 +大文件使用三个并发分片,上传进度显示在原打包任务中。传送或链接检查失败会返回具体阶段、 +下载编号及重试建议,保留已生成的 ZIP;再次点击下载复用原文件,已上传的完整对象不重复上传。 +后台 ZIP 任务参与发布期间的工作等待。完成或失败的任务及文件保留 24 小时,待处理任务保留 6 小时。 diff --git a/lib/storage/canvas-zip-delivery.ts b/lib/storage/canvas-zip-delivery.ts index 9049f1bb5..a8b0a18a0 100644 --- a/lib/storage/canvas-zip-delivery.ts +++ b/lib/storage/canvas-zip-delivery.ts @@ -1,11 +1,72 @@ import "server-only"; import { createReadStream } from "node:fs"; -import { PutObjectCommand } from "@aws-sdk/client-s3"; +import { open } from "node:fs/promises"; +import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, HeadObjectCommand, PutObjectCommand, UploadPartCommand, type S3Client } from "@aws-sdk/client-s3"; import { getDefaultStorageConfig, getStorageConfigById } from "@/lib/db/queries"; import { createS3Client, toDbS3UploadConfig, buildS3PublicUrl } from "./service"; -/** Only a completely written archive is uploaded. Local GET remains a fallback. */ -export async function publishCanvasZip(params: { id: string; path: string; bytes: number; disposition: string }) { +type ZipPublication = { id: string; path: string; bytes: number; disposition: string; onProgress?: (bytes: number) => Promise }; +const PART_BYTES = 8 * 1024 ** 2; + +async function uploadZip(client: S3Client, bucket: string, key: string, params: ZipPublication) { + const metadata = { Bucket: bucket, Key: key, ContentType: "application/zip", ContentDisposition: params.disposition, CacheControl: "public, max-age=1800" }; + if (params.bytes <= 2 * PART_BYTES) { + const body = createReadStream(params.path); + try { + await client.send(new PutObjectCommand({ ...metadata, Body: body, ContentLength: params.bytes }), { abortSignal: AbortSignal.timeout(90_000) }); + await params.onProgress?.(params.bytes); + } finally { body.destroy(); } + return; + } + const initiated = await client.send(new CreateMultipartUploadCommand(metadata), { abortSignal: AbortSignal.timeout(15_000) }); + if (!initiated.UploadId) throw new Error("七牛未返回分片上传编号"); + const upload = { Bucket: bucket, Key: key, UploadId: initiated.UploadId }; + const handle = await open(params.path, "r").catch(async error => { + await client.send(new AbortMultipartUploadCommand(upload), { abortSignal: AbortSignal.timeout(10_000) }).catch(() => undefined); + throw error; + }); + const parts: Array<{ PartNumber: number; ETag: string }> = []; + let cursor = 0; + let uploaded = 0; + let progress = Promise.resolve(); + const controller = new AbortController(); + try { + const workers = await Promise.allSettled(Array.from({ length: 3 }, async () => { + try { + while (!controller.signal.aborted) { + const index = cursor++; + const start = index * PART_BYTES; + if (start >= params.bytes) return; + const body = Buffer.allocUnsafe(Math.min(PART_BYTES, params.bytes - start)); + let read = 0; + while (read < body.length) { + const result = await handle.read(body, read, body.length - read, start + read); + if (!result.bytesRead) throw new Error("完整 ZIP 文件读取中断,请重试下载"); + read += result.bytesRead; + } + const part = await client.send(new UploadPartCommand({ ...upload, PartNumber: index + 1, Body: body, ContentLength: body.length }), { + abortSignal: AbortSignal.any([controller.signal, AbortSignal.timeout(180_000)]), + }); + if (!part.ETag) throw new Error("七牛未确认上传分片"); + parts.push({ PartNumber: index + 1, ETag: part.ETag }); + uploaded += body.length; + const bytes = uploaded; + progress = progress.then(() => params.onProgress?.(bytes)); + await progress; + } + } catch (error) { controller.abort(); throw error; } + })); + const failure = workers.find(worker => worker.status === "rejected"); + if (failure?.status === "rejected") throw failure.reason; + await client.send(new CompleteMultipartUploadCommand({ ...upload, MultipartUpload: { Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber) } }), { abortSignal: AbortSignal.timeout(30_000) }); + } catch (error) { + await client.send(new AbortMultipartUploadCommand(upload), { abortSignal: AbortSignal.timeout(10_000) }).catch(() => undefined); + throw error; + } finally { await handle.close(); } +} + +/** Publish the complete ZIP before offering the accelerated download. */ +export async function publishCanvasZip(params: ZipPublication) { const raw = await getDefaultStorageConfig(); const config = toDbS3UploadConfig(raw); if (config?.provider !== "qiniu-kodo" || !config.publicUrl?.startsWith("https://")) return null; @@ -13,20 +74,37 @@ export async function publishCanvasZip(params: { id: string; path: string; bytes const client = createS3Client(config); // Qiniu's S3 endpoint expects an ordinary known-length body, without AWS trailers. client.config.requestChecksumCalculation = async () => "WHEN_REQUIRED"; - const body = createReadStream(params.path); + let phase: "check" | "upload" | "verify" = "check"; + const startedAt = Date.now(); + let uploadedBytes = 0; try { - await client.send(new PutObjectCommand({ Bucket: config.bucket, Key: key, Body: body, - ContentLength: params.bytes, ContentType: "application/zip", ContentDisposition: params.disposition, - CacheControl: "public, max-age=1800" }), { abortSignal: AbortSignal.timeout(90_000) }); - const url = buildS3PublicUrl(config, key); + const existing = await client.send(new HeadObjectCommand({ Bucket: config.bucket, Key: key }), { abortSignal: AbortSignal.timeout(15_000) }).catch(error => { + if (error.$metadata?.httpStatusCode === 404) return null; + throw error; + }); + if (existing?.ContentLength !== params.bytes) { + phase = "upload"; + await uploadZip(client, config.bucket, key, { ...params, onProgress: async bytes => { + uploadedBytes = bytes; + await params.onProgress?.(bytes); + } }); + } + phase = "verify"; + const target = new URL(buildS3PublicUrl(config, key)); + target.searchParams.set("v", String(params.bytes)); + const url = target.toString(); const response = await fetch(url, { method: "HEAD", redirect: "error", signal: AbortSignal.timeout(10_000) }); const length = Number(response.headers.get("content-length")); if (!response.ok || length !== params.bytes) throw new Error("CDN object is not ready"); return { url, storageConfigId: raw!.id, key }; } catch (error) { - console.warn("[CanvasDownloadZipTask] CDN upload/check failed; using complete local file", { taskId: params.id, kind: (error as Error).name }); - return { url: null, storageConfigId: raw!.id, key }; - } finally { body.destroy(); client.destroy(); } + const kind = (error as Error).name; + const status = (error as { $metadata?: { httpStatusCode?: number } }).$metadata?.httpStatusCode; + console.warn("[CanvasDownloadZipTask] CDN publication failed", { taskId: params.id, phase, kind, status, bytes: params.bytes, uploadedBytes, elapsedMs: Date.now() - startedAt }); + const reason = status === 401 || status === 403 ? `七牛拒绝访问(HTTP ${status}),请管理员检查存储权限` : phase === "check" ? "暂时无法确认七牛文件状态" : phase === "verify" ? "七牛下载链接尚未可用" : /AbortError|TimeoutError/.test(kind) ? "传送至七牛时超时" : "传送至七牛失败"; + const advice = status === 401 || status === 403 ? "存储权限恢复后再次点击下载即可继续传送原 ZIP" : "请再次点击下载以继续传送原 ZIP"; + return { url: null, storageConfigId: raw!.id, key, error: `ZIP 已打包完成,但${reason}。${advice};不会重新生成图片。若持续失败,请联系管理员并提供下载编号:${params.id}` }; + } finally { client.destroy(); } } export async function removeCanvasZipObject(object: { storageConfigId: string; key: string }) { diff --git a/tests/routes/canvas-zip-download.test.ts b/tests/routes/canvas-zip-download.test.ts index 157c1eee7..3d450b704 100644 --- a/tests/routes/canvas-zip-download.test.ts +++ b/tests/routes/canvas-zip-download.test.ts @@ -7,6 +7,9 @@ import { randomUUID } from "node:crypto"; import JSZip from "jszip"; import { serverLoader } from "../helpers/canvas-download-server-loader"; import { parseDownloadByteRange } from "@/lib/canvas/download-byte-range"; +import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, HeadObjectCommand, PutObjectCommand, UploadPartCommand } from "@aws-sdk/client-s3"; + +const runtimeWork = { trackRuntimeWork: async (run: () => Promise) => run() }; test("ranges reject invalid offsets and preserve normal, open and suffix byte ranges", () => { expect(parseDownloadByteRange("bytes=0-3", 20)).toEqual({start:0,end:3}); @@ -20,7 +23,7 @@ test("multi-select ZIP admission is immediate; complete archives survive restart const dir=await mkdtemp(join(tmpdir(),'canvas-zip-test-')); let owner:string|null='owner';let fetches=0; const callbacks:Array<()=>Promise>=[]; const mocks={ - '@/app/(auth)/auth': {auth:async()=>owner?{user:{id:owner}}:null}, + '@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth': {auth:async()=>owner?{user:{id:owner}}:null}, 'next/server': {NextResponse,after:(run:()=>Promise)=>callbacks.push(run)}, '@/lib/canvas/photoshop-safe-download':{fetchCanvasDownloadAsset:async({index}:any)=>{ fetches++;return {ok:true,filename:`${index}-花瓶.png`,bytes:Buffer.from(`image-${index}`),contentType:'image/png'}; @@ -56,7 +59,7 @@ test("multi-select ZIP admission is immediate; complete archives survive restart test("all-source failures never produce an empty successful ZIP; retries reuse the task and resolve sources again", async()=>{ const dir=await mkdtemp(join(tmpdir(),'canvas-zip-retry-'));const callbacks:Array<()=>Promise>=[];let available=false; const route=serverLoader({ - '@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})}, + '@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})}, 'next/server':{NextResponse,after:(run:()=>Promise)=>callbacks.push(run)}, '@/lib/canvas/photoshop-safe-download':{fetchCanvasDownloadAsset:async()=>available?{ok:true,filename:'one.png',bytes:Buffer.from('one'),contentType:'image/png'}:{ok:false,error:'expired provider URL'}}, '@/lib/storage/canvas-zip-delivery':{publishCanvasZip:async()=>null}, @@ -108,7 +111,7 @@ test("the browser polls a complete ZIP, preserves an idempotent task after a los test("partial source failures are explicit; CDN redirects retain an authenticated local range fallback",async()=>{ const dir=await mkdtemp(join(tmpdir(),'canvas-zip-partial-'));const callbacks:Array<()=>Promise>=[];let owner='owner'; const route=serverLoader({ - '@/app/(auth)/auth':{auth:async()=>({user:{id:owner}})}, + '@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth':{auth:async()=>({user:{id:owner}})}, 'next/server':{NextResponse,after:(run:()=>Promise)=>callbacks.push(run)}, '@/lib/canvas/photoshop-safe-download':{fetchCanvasDownloadAsset:async({index}:any)=>index?{ok:false,error:'source expired'}:{ok:true,filename:'original.png',bytes:Buffer.from('original'),contentType:'image/png'}}, '@/lib/storage/canvas-zip-delivery':{publishCanvasZip:async()=>({url:'https://cdn.example/complete.zip',key:'canvas-downloads/x.zip',storageConfigId:'config'})}, @@ -128,7 +131,7 @@ test("partial source failures are explicit; CDN redirects retain an authenticate test("recovery validates staged files instead of treating missing bytes as successful images",async()=>{ const dir=await mkdtemp(join(tmpdir(),'canvas-zip-missing-'));const callbacks:Array<()=>Promise>=[];const id=randomUUID(); const load=serverLoader({ - '@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})}, + '@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})}, 'next/server':{NextResponse,after:(run:()=>Promise)=>callbacks.push(run)}, '@/lib/canvas/photoshop-safe-download':{fetchCanvasDownloadAsset:async()=>({ok:false,error:'source expired'})}, '@/lib/storage/canvas-zip-delivery':{publishCanvasZip:async()=>null}, @@ -157,7 +160,7 @@ test("large ZIP writing checks current free space as staged input bytes are rele test("slow expired-object cleanup cannot keep the ZIP admission request open",async()=>{ const dir=await mkdtemp(join(tmpdir(),'canvas-zip-cleanup-'));const callbacks:Array<()=>Promise>=[]; const mocks:Record={ - '@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})}, + '@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})}, 'next/server':{NextResponse,after:(run:()=>Promise)=>callbacks.push(run)}, '@/lib/canvas/photoshop-safe-download':{},'@/lib/storage/canvas-zip-delivery':{}, }; @@ -169,3 +172,127 @@ test("slow expired-object cleanup cannot keep the ZIP admission request open",as expect(result).not.toBe('blocked');expect((result as Response).status).toBe(202); }finally{await rm(dir,{recursive:true,force:true});} }); + +test("large ZIPs use bounded parallel parts and publish only the complete ordered archive", async () => { + const dir=await mkdtemp(join(tmpdir(),'canvas-zip-parts-')); + const bytes=Buffer.alloc(25*1024**2+19,0x5a);bytes[16*1024**2]=0xff; + const file=join(dir,'large.zip');await writeFile(file,bytes); + const parts=new Map();let inFlight=0;let peak=0;let finished=false;let destroyed=false;let trailers=''; + const progress:number[]=[]; + const client={config:{requestChecksumCalculation:async()=>''},destroy(){destroyed=true;},async send(command:any){ + if(command instanceof HeadObjectCommand)throw Object.assign(new Error('missing'),{$metadata:{httpStatusCode:404}}); + if(command instanceof CreateMultipartUploadCommand){expect(command.input.ContentDisposition).toContain('attachment');return {UploadId:'upload'};} + if(command instanceof UploadPartCommand){ + const input=command.input;inFlight++;peak=Math.max(peak,inFlight);expect(input.ContentLength).toBeLessThanOrEqual(8*1024**2); + await new Promise(resolve=>setTimeout(resolve,input.PartNumber===1?30:20)); + expect(Buffer.isBuffer(input.Body)).toBe(true); + parts.set(input.PartNumber!,Buffer.from(input.Body as Buffer));inFlight--;return {ETag:`part-${input.PartNumber}`}; + } + expect(command).toBeInstanceOf(CompleteMultipartUploadCommand); + expect(command.input.MultipartUpload.Parts.map((p:any)=>p.PartNumber)).toEqual([1,2,3,4]); + expect(Buffer.concat(command.input.MultipartUpload.Parts.map((p:any)=>parts.get(p.PartNumber)!)).equals(bytes)).toBe(true); + finished=true;trailers=await client.config.requestChecksumCalculation();return {}; + }}; + const publish=serverLoader({ + '@/lib/db/queries':{getDefaultStorageConfig:async()=>({id:'storage'})}, + './service':{toDbS3UploadConfig:()=>({provider:'qiniu-kodo',bucket:'bucket',publicUrl:'https://cdn.example'}),createS3Client:()=>client,buildS3PublicUrl:(_:any,key:string)=>'https://cdn.example/'+key}, + fetch:async(url:string,init:any)=>{expect(finished).toBe(true);expect(init.method).toBe('HEAD');expect(new URL(url).searchParams.get('v')).toBe(String(bytes.length));return new Response(null,{headers:{'content-length':String(bytes.length)}});}, + })('lib/storage/canvas-zip-delivery.ts').publishCanvasZip; + try{ + const result=await publish({id:randomUUID(),path:file,bytes:bytes.length,disposition:'attachment; filename="large.zip"',onProgress:async(n:number)=>{progress.push(n);}}); + expect(result.url).toContain('https://cdn.example/canvas-downloads/');expect(peak).toBeGreaterThan(1);expect(peak).toBeLessThanOrEqual(3);expect(progress.at(-1)).toBe(bytes.length); + expect(progress).toEqual([...progress].sort((a,b)=>a-b));expect(trailers).toBe('WHEN_REQUIRED');expect(destroyed).toBe(true); + }finally{await rm(dir,{recursive:true,force:true});} +}); + +test("failed multipart uploads abort their parts and retain the original ZIP with actionable advice",async()=>{ + const dir=await mkdtemp(join(tmpdir(),'canvas-zip-abort-'));const file=join(dir,'large.zip');const bytes=17*1024**2; + const handle=await (await import('node:fs/promises')).open(file,'w');await handle.truncate(bytes);await handle.close(); + let aborted=false;let completed=false; + const client={config:{},destroy(){},async send(command:any){ + if(command instanceof HeadObjectCommand)throw Object.assign(new Error('missing'),{$metadata:{httpStatusCode:404}}); + if(command instanceof CreateMultipartUploadCommand)return {UploadId:'upload'}; + if(command instanceof UploadPartCommand){if(command.input.PartNumber===2)throw new DOMException('part timeout','TimeoutError');return {ETag:'part'};} + if(command instanceof AbortMultipartUploadCommand){aborted=true;return {};} + completed=true;return {}; + }}; + const publish=serverLoader({ + '@/lib/db/queries':{getDefaultStorageConfig:async()=>({id:'storage'})}, + './service':{toDbS3UploadConfig:()=>({provider:'qiniu-kodo',bucket:'bucket',publicUrl:'https://cdn.example'}),createS3Client:()=>client}, + })('lib/storage/canvas-zip-delivery.ts').publishCanvasZip; + try{ + const result=await publish({id:randomUUID(),path:file,bytes,disposition:'attachment'}); + expect(result.url).toBeNull();expect(result.error).toContain('传送至七牛时超时');expect(result.error).toContain('继续传送原 ZIP'); + expect(aborted).toBe(true);expect(completed).toBe(false);expect((await (await import('node:fs/promises')).stat(file)).size).toBe(bytes); + }finally{await rm(dir,{recursive:true,force:true});} +}); + +test("retrying CDN verification reuses an already uploaded ZIP without uploading it again",async()=>{ + let uploads=0;let available=false; + const client={config:{},destroy(){},async send(command:any){ + if(command instanceof HeadObjectCommand)return {ContentLength:100}; + if(command instanceof PutObjectCommand||command instanceof CreateMultipartUploadCommand)uploads++; + throw new Error('Unexpected upload'); + }}; + const publish=serverLoader({ + '@/lib/db/queries':{getDefaultStorageConfig:async()=>({id:'storage'})}, + './service':{toDbS3UploadConfig:()=>({provider:'qiniu-kodo',bucket:'bucket',publicUrl:'https://cdn.example'}),createS3Client:()=>client,buildS3PublicUrl:(_:any,key:string)=>'https://cdn.example/'+key}, + fetch:async()=>new Response(null,{status:available?200:404,headers:{'content-length':available?'100':'0'}}), + })('lib/storage/canvas-zip-delivery.ts').publishCanvasZip; + const params={id:randomUUID(),path:'unused-already-published.zip',bytes:100,disposition:'attachment'}; + const first=await publish(params);expect(first.url).toBeNull();expect(first.error).toContain('七牛下载链接尚未可用'); + available=true;expect((await publish(params)).url).toContain('https://cdn.example/');expect(uploads).toBe(0); +}); + +test("CDN failures never silently start local downloads; retries reuse the ZIP and keep it resumable beyond one hour",async()=>{ + const dir=await mkdtemp(join(tmpdir(),'canvas-zip-cdn-retry-'));const callbacks:Array<()=>Promise>=[]; + let fetches=0;let publications=0;let available=false;let now=Date.now(); + let activeWork=0; + const runtimeWork={trackRuntimeWork:async(run:()=>Promise)=>{activeWork++;try{return await run();}finally{activeWork--;}}}; + class Clock extends Date{static now(){return now;}} + const load=serverLoader({ + Date:Clock,'@/lib/observability/active-work':runtimeWork,'@/app/(auth)/auth':{auth:async()=>({user:{id:'owner'}})}, + 'next/server':{NextResponse,after:(run:()=>Promise)=>callbacks.push(run)}, + '@/lib/canvas/photoshop-safe-download':{fetchCanvasDownloadAsset:async()=>{fetches++;return {ok:true,filename:'original.png',bytes:Buffer.from('original'),contentType:'image/png'};}}, + '@/lib/storage/canvas-zip-delivery':{publishCanvasZip:async()=>{expect(activeWork).toBe(1);publications++;return {url:available?'https://cdn.example/complete.zip':null,key:'canvas-downloads/key.zip',storageConfigId:'storage',error:'ZIP 已打包,传送至七牛时超时;请再次点击下载继续原 ZIP,不会重新生成图片。'};},removeCanvasZipObject:async()=>{}}, + },{CANVAS_DOWNLOAD_ZIP_DIR:dir}); + const route=load('app/api/canvas/download-zip-task/route.ts');const id=randomUUID(); + const post=()=>new NextRequest('http://localhost/api/canvas/download-zip-task',{method:'POST',body:JSON.stringify({taskId:id,retry:true,files:[{url:'https://cdn.example/original.png'}]})}); + const request=(suffix='')=>new NextRequest(`http://localhost/api/canvas/download-zip-task?taskId=${id}${suffix}`,{headers:{range:'bytes=0-3'}}); + try{ + await route.POST(post());while(callbacks.length)await callbacks.shift()!(); + const failed=(await(await route.GET(request())).json()).task; + expect(failed).toMatchObject({id,status:'failed',successCount:1,downloadUrl:''});expect(failed.error).toContain('不会重新生成图片'); + expect((await route.GET(request('&download=1'))).status).toBe(409); + const before=await readFile(join(dir,id,'task.json'),'utf8');const original=Buffer.from(await readFile(JSON.parse(before).zipFile.path)); + available=true;await route.POST(post());while(callbacks.length)await callbacks.shift()!(); + expect(fetches).toBe(1);expect(publications).toBe(2); + const manifest=JSON.parse(await readFile(join(dir,id,'task.json'),'utf8'));expect(await readFile(manifest.zipFile.path)).toEqual(original); + delete manifest.cdnUrl; + await load('lib/canvas/download-zip-task-store.ts').saveZipTask(manifest); + await route.POST(post());while(callbacks.length)await callbacks.shift()!(); + expect(fetches).toBe(1);expect(publications).toBe(3); + expect(activeWork).toBe(0); + now+=2*60*60_000; + expect((await route.GET(request('&download=1'))).status).toBe(307); + const local=await route.GET(request('&download=1&local=1'));expect(local.status).toBe(206);expect(Buffer.from(await local.arrayBuffer())).toEqual(original.subarray(0,4)); + }finally{await rm(dir,{recursive:true,force:true});} +}); + +test("small ZIP delivery preserves the complete file and download filename",async()=>{ + const dir=await mkdtemp(join(tmpdir(),'canvas-zip-small-'));const file=join(dir,'small.zip');const bytes=Buffer.from('PK small complete archive');await writeFile(file,bytes); + let written=false; + const client={config:{},destroy(){},async send(command:any){ + if(command instanceof HeadObjectCommand)throw Object.assign(new Error('missing'),{$metadata:{httpStatusCode:404}}); + expect(command).toBeInstanceOf(PutObjectCommand);expect(command.input.ContentLength).toBe(bytes.length);expect(command.input.ContentDisposition).toContain('small.zip'); + const chunks:Buffer[]=[];for await(const chunk of command.input.Body)chunks.push(Buffer.from(chunk)); + expect(Buffer.concat(chunks).equals(bytes)).toBe(true);written=true;return {}; + }}; + const publish=serverLoader({ + '@/lib/db/queries':{getDefaultStorageConfig:async()=>({id:'storage'})}, + './service':{toDbS3UploadConfig:()=>({provider:'qiniu-kodo',bucket:'bucket',publicUrl:'https://cdn.example'}),createS3Client:()=>client,buildS3PublicUrl:(_:any,key:string)=>'https://cdn.example/'+key}, + fetch:async()=>{expect(written).toBe(true);return new Response(null,{headers:{'content-length':String(bytes.length)}});}, + })('lib/storage/canvas-zip-delivery.ts').publishCanvasZip; + try{expect((await publish({id:randomUUID(),path:file,bytes:bytes.length,disposition:'attachment; filename="small.zip"'})).url).toContain('https://cdn.example/');} + finally{await rm(dir,{recursive:true,force:true});} +});