7326e273e1
Protect upload cleanup and atomic completion paths, add durable media worker leases and brand asset validation, and introduce verified migration/bootstrap/deployment gates.
401 lines
13 KiB
TypeScript
401 lines
13 KiB
TypeScript
/** @jest-environment node */
|
||
|
||
import {
|
||
mkdtemp,
|
||
mkdir,
|
||
readdir,
|
||
readFile,
|
||
rm,
|
||
stat,
|
||
symlink,
|
||
writeFile,
|
||
} from 'fs/promises'
|
||
import { appendFileSync } from 'fs'
|
||
import AdmZip from 'adm-zip'
|
||
import os from 'os'
|
||
import path from 'path'
|
||
|
||
import {
|
||
createZipFromFiles,
|
||
createZipFromZips,
|
||
} from '@/lib/task-material-processor'
|
||
import {
|
||
deleteStoredUploadFileFromRemote,
|
||
syncStoredUploadFileToRemote,
|
||
} from '@/lib/remote-upload-sync'
|
||
|
||
jest.mock('@/lib/prisma', () => ({
|
||
prisma: {},
|
||
}))
|
||
|
||
jest.mock('@/lib/s3', () => ({
|
||
deleteFromS3: jest.fn(),
|
||
}))
|
||
|
||
jest.mock('@/lib/public-display', () => ({
|
||
buildMaterialPublicDisplayPath: jest.fn(),
|
||
uploadPublicDisplayToS3: jest.fn(),
|
||
}))
|
||
|
||
jest.mock('@/lib/image-processor', () => ({
|
||
compressImage: jest.fn(),
|
||
}))
|
||
|
||
jest.mock('@/lib/remote-upload-sync', () => ({
|
||
deleteStoredUploadFileFromRemote: jest.fn().mockResolvedValue(undefined),
|
||
syncStoredUploadFileToRemote: jest.fn(),
|
||
}))
|
||
|
||
// Keep the ZIP test focused on the real filesystem/archive path. Disk reservation
|
||
// has its own unit tests and would otherwise make this test depend on free space.
|
||
jest.mock('@/lib/material-upload-merge', () => ({
|
||
acquireUploadMergeDiskReservation: jest.fn().mockResolvedValue(() => undefined),
|
||
}))
|
||
|
||
const syncRemoteMock = syncStoredUploadFileToRemote as jest.Mock
|
||
const deleteRemoteMock = deleteStoredUploadFileFromRemote as jest.Mock
|
||
|
||
type SyncObservation = {
|
||
localFilePath: string
|
||
storedPath: string
|
||
fileSize: number
|
||
expectedSize?: number
|
||
partialNamesAtSync: string[]
|
||
}
|
||
|
||
describe('task material ZIP atomic filesystem behavior', () => {
|
||
let uploadRoot: string
|
||
let previousUploadDir: string | undefined
|
||
let syncObservations: SyncObservation[]
|
||
|
||
const materialsDir = () => path.join(uploadRoot, 'materials')
|
||
|
||
beforeEach(async () => {
|
||
previousUploadDir = process.env.UPLOAD_DIR
|
||
uploadRoot = await mkdtemp(path.join(os.tmpdir(), 'task-material-zip-'))
|
||
process.env.UPLOAD_DIR = uploadRoot
|
||
syncObservations = []
|
||
jest.clearAllMocks()
|
||
|
||
syncRemoteMock.mockImplementation(
|
||
async (localFilePath: string, storedPath: string, expectedSize?: number) => {
|
||
// This callback is intentionally a real stat/readdir: the production
|
||
// function must rename the final ZIP before remote sync is invoked.
|
||
const fileStats = await stat(localFilePath)
|
||
const names = await readdir(path.dirname(localFilePath))
|
||
syncObservations.push({
|
||
localFilePath,
|
||
storedPath,
|
||
fileSize: fileStats.size,
|
||
expectedSize,
|
||
partialNamesAtSync: names.filter((name) => name.includes('.partial-')),
|
||
})
|
||
return storedPath
|
||
},
|
||
)
|
||
})
|
||
|
||
afterEach(async () => {
|
||
syncRemoteMock.mockReset()
|
||
deleteRemoteMock.mockReset()
|
||
if (previousUploadDir === undefined) {
|
||
delete process.env.UPLOAD_DIR
|
||
} else {
|
||
process.env.UPLOAD_DIR = previousUploadDir
|
||
}
|
||
await rm(uploadRoot, { recursive: true, force: true })
|
||
})
|
||
|
||
async function assertFinalZip(result: { filePath: string; fileName: string; fileSize: number }) {
|
||
const finalPath = path.join(materialsDir(), result.fileName)
|
||
const finalStats = await stat(finalPath)
|
||
const finalBytes = await readFile(finalPath)
|
||
const names = await readdir(materialsDir())
|
||
|
||
expect(result.filePath).toBe(`uploads/materials/${result.fileName}`)
|
||
expect(finalStats.isFile()).toBe(true)
|
||
expect(finalStats.size).toBeGreaterThan(0)
|
||
expect(result.fileSize).toBe(finalStats.size)
|
||
expect(finalBytes.subarray(0, 2).toString()).toBe('PK')
|
||
expect(names.filter((name) => name.includes('.partial-'))).toEqual([])
|
||
|
||
return finalPath
|
||
}
|
||
|
||
async function assertNoZipResidue() {
|
||
const names = await readdir(materialsDir()).catch(() => [])
|
||
expect(names.filter((name) => name.endsWith('.zip') || name.includes('.partial-'))).toEqual([])
|
||
}
|
||
|
||
test('createZipFromFiles 写出非空最终 ZIP，并在同步时已完成原子改名', async () => {
|
||
const sourcePath = path.join(uploadRoot, 'temp', 'source.txt')
|
||
const sourceBytes = Buffer.from('task-material-source')
|
||
await mkdir(path.dirname(sourcePath), { recursive: true })
|
||
await writeFile(sourcePath, sourceBytes)
|
||
|
||
const result = await createZipFromFiles(
|
||
[{ path: sourcePath, name: 'source.txt', size: sourceBytes.length }],
|
||
'material-files-success',
|
||
'MODEL-FILES',
|
||
'任务-详情',
|
||
)
|
||
|
||
const finalPath = await assertFinalZip(result)
|
||
const archive = new AdmZip(finalPath)
|
||
expect(archive.getEntry('source.txt')?.getData()).toEqual(sourceBytes)
|
||
expect(syncRemoteMock).toHaveBeenCalledTimes(1)
|
||
expect(syncObservations).toEqual([
|
||
{
|
||
localFilePath: finalPath,
|
||
storedPath: result.filePath,
|
||
fileSize: result.fileSize,
|
||
expectedSize: result.fileSize,
|
||
partialNamesAtSync: [],
|
||
},
|
||
])
|
||
})
|
||
|
||
test('createZipFromZips 写出非空最终 ZIP，并在同步时已完成原子改名', async () => {
|
||
const sourceZipPath = path.join(uploadRoot, 'materials', 'source.zip')
|
||
await mkdir(path.dirname(sourceZipPath), { recursive: true })
|
||
// The function validates the source as a regular file; the nested archive
|
||
// contents are intentionally outside this test's scope.
|
||
await writeFile(sourceZipPath, Buffer.from('PK\x03\x04nested-zip-source'))
|
||
|
||
const result = await createZipFromZips(
|
||
[{ path: sourceZipPath, name: 'source.zip' }],
|
||
'material-zips-success',
|
||
'MODEL-ZIPS',
|
||
'任务-详情',
|
||
)
|
||
|
||
const finalPath = await assertFinalZip(result)
|
||
const archive = new AdmZip(finalPath)
|
||
expect(archive.getEntry('source.zip')?.getData()).toEqual(
|
||
Buffer.from('PK\x03\x04nested-zip-source'),
|
||
)
|
||
expect(syncRemoteMock).toHaveBeenCalledTimes(1)
|
||
expect(syncObservations).toEqual([
|
||
{
|
||
localFilePath: finalPath,
|
||
storedPath: result.filePath,
|
||
fileSize: result.fileSize,
|
||
expectedSize: result.fileSize,
|
||
partialNamesAtSync: [],
|
||
},
|
||
])
|
||
})
|
||
|
||
test('createZipFromFiles 对目录、符号链接和缺失输入失败且不留下 ZIP 或 partial', async () => {
|
||
const directoryPath = path.join(uploadRoot, 'invalid-directory')
|
||
await mkdir(directoryPath, { recursive: true })
|
||
|
||
const symlinkTarget = path.join(uploadRoot, 'symlink-target')
|
||
const symlinkPath = path.join(uploadRoot, 'symlink-input')
|
||
await mkdir(symlinkTarget, { recursive: true })
|
||
await symlink(
|
||
symlinkTarget,
|
||
symlinkPath,
|
||
process.platform === 'win32' ? 'junction' : 'dir',
|
||
)
|
||
|
||
const cases = [
|
||
{ label: '目录', inputPath: directoryPath },
|
||
{ label: '符号链接', inputPath: symlinkPath },
|
||
{ label: '缺失文件', inputPath: path.join(uploadRoot, 'missing-input') },
|
||
]
|
||
|
||
for (let index = 0; index < cases.length; index += 1) {
|
||
const item = cases[index]
|
||
await expect(
|
||
createZipFromFiles(
|
||
[{ path: item.inputPath, name: `${item.label}.bin`, size: 1 }],
|
||
`material-files-failure-${index}`,
|
||
'MODEL-INVALID',
|
||
'任务-详情',
|
||
),
|
||
).rejects.toThrow()
|
||
await assertNoZipResidue()
|
||
}
|
||
|
||
expect(syncRemoteMock).not.toHaveBeenCalled()
|
||
})
|
||
|
||
test('createZipFromZips 对符号链接和缺失输入失败且不留下 ZIP 或 partial', async () => {
|
||
const symlinkTarget = path.join(uploadRoot, 'zip-symlink-target')
|
||
const symlinkPath = path.join(uploadRoot, 'zip-symlink-input')
|
||
await mkdir(symlinkTarget, { recursive: true })
|
||
await symlink(
|
||
symlinkTarget,
|
||
symlinkPath,
|
||
process.platform === 'win32' ? 'junction' : 'dir',
|
||
)
|
||
|
||
const cases = [
|
||
{ label: '符号链接', inputPath: symlinkPath },
|
||
{ label: '缺失文件', inputPath: path.join(uploadRoot, 'missing-source.zip') },
|
||
]
|
||
|
||
for (let index = 0; index < cases.length; index += 1) {
|
||
const item = cases[index]
|
||
await expect(
|
||
createZipFromZips(
|
||
[{ path: item.inputPath, name: `${item.label}.zip` }],
|
||
`material-zips-failure-${index}`,
|
||
'MODEL-INVALID',
|
||
'任务-详情',
|
||
),
|
||
).rejects.toThrow()
|
||
await assertNoZipResidue()
|
||
}
|
||
|
||
expect(syncRemoteMock).not.toHaveBeenCalled()
|
||
})
|
||
|
||
test('输入在 archiver 取样后消失时失败，且不留下正式 ZIP 或 partial', async () => {
|
||
const sourcePath = path.join(uploadRoot, 'temp', 'disappearing-source.bin')
|
||
await mkdir(path.dirname(sourcePath), { recursive: true })
|
||
await writeFile(sourcePath, Buffer.from('will-disappear'))
|
||
|
||
// archiver uses callback-style fs.lstat after the production preflight
|
||
// (which uses fs/promises.lstat). Delete the source between those two
|
||
// observations to deterministically exercise a stream/input disappearance.
|
||
const nodeFs = require('fs') as typeof import('fs')
|
||
const originalLstat = nodeFs.lstat.bind(nodeFs)
|
||
const lstatMock = jest.spyOn(nodeFs, 'lstat') as jest.Mock
|
||
let removed = false
|
||
lstatMock.mockImplementation((filePath: string, ...args: unknown[]) => {
|
||
const callback = args[args.length - 1] as (...callbackArgs: unknown[]) => void
|
||
originalLstat(filePath, (...callbackArgs: unknown[]) => {
|
||
if (!removed && path.resolve(filePath) === path.resolve(sourcePath)) {
|
||
removed = true
|
||
void (async () => {
|
||
await rm(sourcePath, { force: true })
|
||
callback(...callbackArgs)
|
||
})()
|
||
return
|
||
}
|
||
callback(...callbackArgs)
|
||
})
|
||
})
|
||
|
||
try {
|
||
await expect(
|
||
createZipFromFiles(
|
||
[{ path: sourcePath, name: 'disappearing-source.bin', size: 14 }],
|
||
'material-disappearing-input',
|
||
'MODEL-DISAPPEARING',
|
||
'任务-详情',
|
||
),
|
||
).rejects.toThrow()
|
||
} finally {
|
||
lstatMock.mockRestore()
|
||
}
|
||
|
||
expect(removed).toBe(true)
|
||
await assertNoZipResidue()
|
||
expect(syncRemoteMock).not.toHaveBeenCalled()
|
||
})
|
||
|
||
test('ZIP 输出超过预留上限时失败并清理 partial', async () => {
|
||
const sourcePath = path.join(uploadRoot, 'temp', 'growing-source.bin')
|
||
await mkdir(path.dirname(sourcePath), { recursive: true })
|
||
await writeFile(sourcePath, Buffer.from('x'))
|
||
|
||
// The preflight sees one byte, reserving 16MiB of overhead. Grow the file
|
||
// immediately after archiver's later callback-style lstat so the real
|
||
// archive stream exceeds that bound. Compression level 0 makes the size
|
||
// relationship deterministic.
|
||
const oversizedPayload = Buffer.alloc(16 * 1024 * 1024 + 1024, 0x61)
|
||
const nodeFs = require('fs') as typeof import('fs')
|
||
const originalLstat = nodeFs.lstat.bind(nodeFs)
|
||
const lstatMock = jest.spyOn(nodeFs, 'lstat') as jest.Mock
|
||
let expanded = false
|
||
lstatMock.mockImplementation((filePath: string, ...args: unknown[]) => {
|
||
const callback = args[args.length - 1] as (...callbackArgs: unknown[]) => void
|
||
originalLstat(filePath, (...callbackArgs: unknown[]) => {
|
||
if (!expanded && path.resolve(filePath) === path.resolve(sourcePath)) {
|
||
expanded = true
|
||
appendFileSync(sourcePath, oversizedPayload)
|
||
}
|
||
callback(...callbackArgs)
|
||
})
|
||
})
|
||
|
||
try {
|
||
await expect(
|
||
createZipFromFiles(
|
||
[{ path: sourcePath, name: 'growing-source.bin', size: 1 }],
|
||
'material-output-over-limit',
|
||
'MODEL-OVER-LIMIT',
|
||
'任务-详情',
|
||
'all',
|
||
undefined,
|
||
undefined,
|
||
30000,
|
||
0,
|
||
),
|
||
).rejects.toThrow()
|
||
} finally {
|
||
lstatMock.mockRestore()
|
||
}
|
||
|
||
expect(expanded).toBe(true)
|
||
await assertNoZipResidue()
|
||
expect(syncRemoteMock).not.toHaveBeenCalled()
|
||
})
|
||
|
||
test('同名 ZIP 并发创建时只保留一个成功结果，失败方不删除成功方', async () => {
|
||
const sourceA = path.join(uploadRoot, 'temp', 'same-name-a.txt')
|
||
const sourceB = path.join(uploadRoot, 'temp', 'same-name-b.txt')
|
||
await mkdir(path.dirname(sourceA), { recursive: true })
|
||
await writeFile(sourceA, Buffer.from('first concurrent source'))
|
||
await writeFile(sourceB, Buffer.from('second concurrent source'))
|
||
|
||
const dateNowSpy = jest.spyOn(Date, 'now').mockReturnValue(1711111111111)
|
||
let settled: PromiseSettledResult<{
|
||
filePath: string
|
||
fileName: string
|
||
fileSize: number
|
||
}>[]
|
||
try {
|
||
settled = await Promise.allSettled([
|
||
createZipFromFiles(
|
||
[{ path: sourceA, name: 'same-name-a.txt', size: 23 }],
|
||
'material-same-name',
|
||
'MODEL-SAME-NAME',
|
||
'任务-详情',
|
||
),
|
||
createZipFromFiles(
|
||
[{ path: sourceB, name: 'same-name-b.txt', size: 24 }],
|
||
'material-same-name',
|
||
'MODEL-SAME-NAME',
|
||
'任务-详情',
|
||
),
|
||
])
|
||
} finally {
|
||
dateNowSpy.mockRestore()
|
||
}
|
||
|
||
const fulfilled = settled.filter(
|
||
(item): item is PromiseFulfilledResult<{
|
||
filePath: string
|
||
fileName: string
|
||
fileSize: number
|
||
}> => item.status === 'fulfilled',
|
||
)
|
||
const rejected = settled.filter((item) => item.status === 'rejected')
|
||
|
||
expect(fulfilled).toHaveLength(1)
|
||
expect(rejected).toHaveLength(1)
|
||
const result = fulfilled[0].value
|
||
const finalPath = await assertFinalZip(result)
|
||
expect(await stat(finalPath)).toMatchObject({ isFile: expect.any(Function) })
|
||
expect((await readdir(materialsDir())).filter((name) => name.endsWith('.zip'))).toEqual([
|
||
result.fileName,
|
||
])
|
||
expect(syncRemoteMock).toHaveBeenCalledTimes(1)
|
||
expect(syncObservations[0]?.localFilePath).toBe(finalPath)
|
||
})
|
||
})
|