/** @jest-environment node */ import { mkdtemp, mkdir, readdir, readFile, rm, stat, symlink, writeFile, } from 'fs/promises' import { appendFileSync } from 'fs' import AdmZip from 'adm-zip' import os from 'os' import path from 'path' import { createZipFromFiles, createZipFromZips, } from '@/lib/task-material-processor' import { deleteStoredUploadFileFromRemote, syncStoredUploadFileToRemote, } from '@/lib/remote-upload-sync' jest.mock('@/lib/prisma', () => ({ prisma: {}, })) jest.mock('@/lib/s3', () => ({ deleteFromS3: jest.fn(), })) jest.mock('@/lib/public-display', () => ({ buildMaterialPublicDisplayPath: jest.fn(), uploadPublicDisplayToS3: jest.fn(), })) jest.mock('@/lib/image-processor', () => ({ compressImage: jest.fn(), })) jest.mock('@/lib/remote-upload-sync', () => ({ deleteStoredUploadFileFromRemote: jest.fn().mockResolvedValue(undefined), syncStoredUploadFileToRemote: jest.fn(), })) // Keep the ZIP test focused on the real filesystem/archive path. Disk reservation // has its own unit tests and would otherwise make this test depend on free space. jest.mock('@/lib/material-upload-merge', () => ({ acquireUploadMergeDiskReservation: jest.fn().mockResolvedValue(() => undefined), })) const syncRemoteMock = syncStoredUploadFileToRemote as jest.Mock const deleteRemoteMock = deleteStoredUploadFileFromRemote as jest.Mock type SyncObservation = { localFilePath: string storedPath: string fileSize: number expectedSize?: number partialNamesAtSync: string[] } describe('task material ZIP atomic filesystem behavior', () => { let uploadRoot: string let previousUploadDir: string | undefined let syncObservations: SyncObservation[] const materialsDir = () => path.join(uploadRoot, 'materials') beforeEach(async () => { previousUploadDir = process.env.UPLOAD_DIR uploadRoot = await mkdtemp(path.join(os.tmpdir(), 'task-material-zip-')) process.env.UPLOAD_DIR = uploadRoot syncObservations = [] jest.clearAllMocks() syncRemoteMock.mockImplementation( async (localFilePath: string, storedPath: string, expectedSize?: number) => { // This callback is intentionally a real stat/readdir: the production // function must rename the final ZIP before remote sync is invoked. const fileStats = await stat(localFilePath) const names = await readdir(path.dirname(localFilePath)) syncObservations.push({ localFilePath, storedPath, fileSize: fileStats.size, expectedSize, partialNamesAtSync: names.filter((name) => name.includes('.partial-')), }) return storedPath }, ) }) afterEach(async () => { syncRemoteMock.mockReset() deleteRemoteMock.mockReset() if (previousUploadDir === undefined) { delete process.env.UPLOAD_DIR } else { process.env.UPLOAD_DIR = previousUploadDir } await rm(uploadRoot, { recursive: true, force: true }) }) async function assertFinalZip(result: { filePath: string; fileName: string; fileSize: number }) { const finalPath = path.join(materialsDir(), result.fileName) const finalStats = await stat(finalPath) const finalBytes = await readFile(finalPath) const names = await readdir(materialsDir()) expect(result.filePath).toBe(`uploads/materials/${result.fileName}`) expect(finalStats.isFile()).toBe(true) expect(finalStats.size).toBeGreaterThan(0) expect(result.fileSize).toBe(finalStats.size) expect(finalBytes.subarray(0, 2).toString()).toBe('PK') expect(names.filter((name) => name.includes('.partial-'))).toEqual([]) return finalPath } async function assertNoZipResidue() { const names = await readdir(materialsDir()).catch(() => []) expect(names.filter((name) => name.endsWith('.zip') || name.includes('.partial-'))).toEqual([]) } test('createZipFromFiles 写出非空最终 ZIP,并在同步时已完成原子改名', async () => { const sourcePath = path.join(uploadRoot, 'temp', 'source.txt') const sourceBytes = Buffer.from('task-material-source') await mkdir(path.dirname(sourcePath), { recursive: true }) await writeFile(sourcePath, sourceBytes) const result = await createZipFromFiles( [{ path: sourcePath, name: 'source.txt', size: sourceBytes.length }], 'material-files-success', 'MODEL-FILES', '任务-详情', ) const finalPath = await assertFinalZip(result) const archive = new AdmZip(finalPath) expect(archive.getEntry('source.txt')?.getData()).toEqual(sourceBytes) expect(syncRemoteMock).toHaveBeenCalledTimes(1) expect(syncObservations).toEqual([ { localFilePath: finalPath, storedPath: result.filePath, fileSize: result.fileSize, expectedSize: result.fileSize, partialNamesAtSync: [], }, ]) }) test('createZipFromZips 写出非空最终 ZIP,并在同步时已完成原子改名', async () => { const sourceZipPath = path.join(uploadRoot, 'materials', 'source.zip') await mkdir(path.dirname(sourceZipPath), { recursive: true }) // The function validates the source as a regular file; the nested archive // contents are intentionally outside this test's scope. await writeFile(sourceZipPath, Buffer.from('PK\x03\x04nested-zip-source')) const result = await createZipFromZips( [{ path: sourceZipPath, name: 'source.zip' }], 'material-zips-success', 'MODEL-ZIPS', '任务-详情', ) const finalPath = await assertFinalZip(result) const archive = new AdmZip(finalPath) expect(archive.getEntry('source.zip')?.getData()).toEqual( Buffer.from('PK\x03\x04nested-zip-source'), ) expect(syncRemoteMock).toHaveBeenCalledTimes(1) expect(syncObservations).toEqual([ { localFilePath: finalPath, storedPath: result.filePath, fileSize: result.fileSize, expectedSize: result.fileSize, partialNamesAtSync: [], }, ]) }) test('createZipFromFiles 对目录、符号链接和缺失输入失败且不留下 ZIP 或 partial', async () => { const directoryPath = path.join(uploadRoot, 'invalid-directory') await mkdir(directoryPath, { recursive: true }) const symlinkTarget = path.join(uploadRoot, 'symlink-target') const symlinkPath = path.join(uploadRoot, 'symlink-input') await mkdir(symlinkTarget, { recursive: true }) await symlink( symlinkTarget, symlinkPath, process.platform === 'win32' ? 'junction' : 'dir', ) const cases = [ { label: '目录', inputPath: directoryPath }, { label: '符号链接', inputPath: symlinkPath }, { label: '缺失文件', inputPath: path.join(uploadRoot, 'missing-input') }, ] for (let index = 0; index < cases.length; index += 1) { const item = cases[index] await expect( createZipFromFiles( [{ path: item.inputPath, name: `${item.label}.bin`, size: 1 }], `material-files-failure-${index}`, 'MODEL-INVALID', '任务-详情', ), ).rejects.toThrow() await assertNoZipResidue() } expect(syncRemoteMock).not.toHaveBeenCalled() }) test('createZipFromZips 对符号链接和缺失输入失败且不留下 ZIP 或 partial', async () => { const symlinkTarget = path.join(uploadRoot, 'zip-symlink-target') const symlinkPath = path.join(uploadRoot, 'zip-symlink-input') await mkdir(symlinkTarget, { recursive: true }) await symlink( symlinkTarget, symlinkPath, process.platform === 'win32' ? 'junction' : 'dir', ) const cases = [ { label: '符号链接', inputPath: symlinkPath }, { label: '缺失文件', inputPath: path.join(uploadRoot, 'missing-source.zip') }, ] for (let index = 0; index < cases.length; index += 1) { const item = cases[index] await expect( createZipFromZips( [{ path: item.inputPath, name: `${item.label}.zip` }], `material-zips-failure-${index}`, 'MODEL-INVALID', '任务-详情', ), ).rejects.toThrow() await assertNoZipResidue() } expect(syncRemoteMock).not.toHaveBeenCalled() }) test('输入在 archiver 取样后消失时失败,且不留下正式 ZIP 或 partial', async () => { const sourcePath = path.join(uploadRoot, 'temp', 'disappearing-source.bin') await mkdir(path.dirname(sourcePath), { recursive: true }) await writeFile(sourcePath, Buffer.from('will-disappear')) // archiver uses callback-style fs.lstat after the production preflight // (which uses fs/promises.lstat). Delete the source between those two // observations to deterministically exercise a stream/input disappearance. const nodeFs = require('fs') as typeof import('fs') const originalLstat = nodeFs.lstat.bind(nodeFs) const lstatMock = jest.spyOn(nodeFs, 'lstat') as jest.Mock let removed = false lstatMock.mockImplementation((filePath: string, ...args: unknown[]) => { const callback = args[args.length - 1] as (...callbackArgs: unknown[]) => void originalLstat(filePath, (...callbackArgs: unknown[]) => { if (!removed && path.resolve(filePath) === path.resolve(sourcePath)) { removed = true void (async () => { await rm(sourcePath, { force: true }) callback(...callbackArgs) })() return } callback(...callbackArgs) }) }) try { await expect( createZipFromFiles( [{ path: sourcePath, name: 'disappearing-source.bin', size: 14 }], 'material-disappearing-input', 'MODEL-DISAPPEARING', '任务-详情', ), ).rejects.toThrow() } finally { lstatMock.mockRestore() } expect(removed).toBe(true) await assertNoZipResidue() expect(syncRemoteMock).not.toHaveBeenCalled() }) test('ZIP 输出超过预留上限时失败并清理 partial', async () => { const sourcePath = path.join(uploadRoot, 'temp', 'growing-source.bin') await mkdir(path.dirname(sourcePath), { recursive: true }) await writeFile(sourcePath, Buffer.from('x')) // The preflight sees one byte, reserving 16MiB of overhead. Grow the file // immediately after archiver's later callback-style lstat so the real // archive stream exceeds that bound. Compression level 0 makes the size // relationship deterministic. const oversizedPayload = Buffer.alloc(16 * 1024 * 1024 + 1024, 0x61) const nodeFs = require('fs') as typeof import('fs') const originalLstat = nodeFs.lstat.bind(nodeFs) const lstatMock = jest.spyOn(nodeFs, 'lstat') as jest.Mock let expanded = false lstatMock.mockImplementation((filePath: string, ...args: unknown[]) => { const callback = args[args.length - 1] as (...callbackArgs: unknown[]) => void originalLstat(filePath, (...callbackArgs: unknown[]) => { if (!expanded && path.resolve(filePath) === path.resolve(sourcePath)) { expanded = true appendFileSync(sourcePath, oversizedPayload) } callback(...callbackArgs) }) }) try { await expect( createZipFromFiles( [{ path: sourcePath, name: 'growing-source.bin', size: 1 }], 'material-output-over-limit', 'MODEL-OVER-LIMIT', '任务-详情', 'all', undefined, undefined, 30000, 0, ), ).rejects.toThrow() } finally { lstatMock.mockRestore() } expect(expanded).toBe(true) await assertNoZipResidue() expect(syncRemoteMock).not.toHaveBeenCalled() }) test('同名 ZIP 并发创建时只保留一个成功结果,失败方不删除成功方', async () => { const sourceA = path.join(uploadRoot, 'temp', 'same-name-a.txt') const sourceB = path.join(uploadRoot, 'temp', 'same-name-b.txt') await mkdir(path.dirname(sourceA), { recursive: true }) await writeFile(sourceA, Buffer.from('first concurrent source')) await writeFile(sourceB, Buffer.from('second concurrent source')) const dateNowSpy = jest.spyOn(Date, 'now').mockReturnValue(1711111111111) let settled: PromiseSettledResult<{ filePath: string fileName: string fileSize: number }>[] try { settled = await Promise.allSettled([ createZipFromFiles( [{ path: sourceA, name: 'same-name-a.txt', size: 23 }], 'material-same-name', 'MODEL-SAME-NAME', '任务-详情', ), createZipFromFiles( [{ path: sourceB, name: 'same-name-b.txt', size: 24 }], 'material-same-name', 'MODEL-SAME-NAME', '任务-详情', ), ]) } finally { dateNowSpy.mockRestore() } const fulfilled = settled.filter( (item): item is PromiseFulfilledResult<{ filePath: string fileName: string fileSize: number }> => item.status === 'fulfilled', ) const rejected = settled.filter((item) => item.status === 'rejected') expect(fulfilled).toHaveLength(1) expect(rejected).toHaveLength(1) const result = fulfilled[0].value const finalPath = await assertFinalZip(result) expect(await stat(finalPath)).toMatchObject({ isFile: expect.any(Function) }) expect((await readdir(materialsDir())).filter((name) => name.endsWith('.zip'))).toEqual([ result.fileName, ]) expect(syncRemoteMock).toHaveBeenCalledTimes(1) expect(syncObservations[0]?.localFilePath).toBe(finalPath) }) })