7326e273e1
Protect upload cleanup and atomic completion paths, add durable media worker leases and brand asset validation, and introduce verified migration/bootstrap/deployment gates.
319 lines
9.4 KiB
TypeScript
319 lines
9.4 KiB
TypeScript
import { createHash, randomBytes } from 'crypto'
|
||
import { lstat, open, readFile, rename, unlink } from 'fs/promises'
|
||
import path from 'path'
|
||
|
||
import {
|
||
ensureSafeDirectoryWithinRoot,
|
||
isPathFreeOfSymbolicLinks,
|
||
normalizeSafeFileName,
|
||
resolveStrictChildPath,
|
||
} from './path-security'
|
||
import { getUploadRootDir } from './upload-paths'
|
||
|
||
const JOURNAL_DIRECTORY_NAME = '.material-upload-consumption'
|
||
const JOURNAL_VERSION = 1
|
||
const MAX_JOURNAL_BYTES = 16 * 1024
|
||
const SAFE_UPLOAD_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/
|
||
|
||
export interface MaterialUploadConsumptionRecord {
|
||
version: 1
|
||
uploadId: string
|
||
ownerId: string
|
||
mergedPath: string
|
||
finalPath: string
|
||
storedFinalPath: string
|
||
fileName: string
|
||
size: number
|
||
createdAt: number
|
||
}
|
||
|
||
export interface RecoveredMaterialUploadConsumption {
|
||
record: MaterialUploadConsumptionRecord
|
||
sourcePath: string
|
||
sourceLocation: 'merged' | 'final'
|
||
}
|
||
|
||
export class MaterialUploadConsumptionJournalError extends Error {
|
||
constructor(message = '上传源文件恢复记录异常，请稍后重试') {
|
||
super(message)
|
||
this.name = 'MaterialUploadConsumptionJournalError'
|
||
}
|
||
}
|
||
|
||
function isSafeOwnerId(ownerId: string): boolean {
|
||
return (
|
||
ownerId.length > 0
|
||
&& Buffer.byteLength(ownerId, 'utf8') <= 512
|
||
&& !/[\x00-\x1f\x7f]/.test(ownerId)
|
||
)
|
||
}
|
||
|
||
function isPathInside(basePath: string, targetPath: string): boolean {
|
||
const relativePath = path.relative(path.resolve(basePath), path.resolve(targetPath))
|
||
return Boolean(
|
||
relativePath
|
||
&& relativePath !== '..'
|
||
&& !relativePath.startsWith(`..${path.sep}`)
|
||
&& !path.isAbsolute(relativePath),
|
||
)
|
||
}
|
||
|
||
function resolveJournalPath(ownerId: string, uploadId: string): string {
|
||
const uploadRoot = getUploadRootDir()
|
||
const digest = createHash('sha256')
|
||
.update(ownerId)
|
||
.update('\0')
|
||
.update(uploadId)
|
||
.digest('hex')
|
||
const journalPath = resolveStrictChildPath(
|
||
path.join(uploadRoot, JOURNAL_DIRECTORY_NAME),
|
||
`${digest}.json`,
|
||
)
|
||
if (!journalPath) {
|
||
throw new MaterialUploadConsumptionJournalError()
|
||
}
|
||
return journalPath
|
||
}
|
||
|
||
function normalizeRecord(input: unknown): MaterialUploadConsumptionRecord {
|
||
const value = input as Partial<MaterialUploadConsumptionRecord> | null
|
||
const uploadId = String(value?.uploadId || '').trim()
|
||
const ownerId = String(value?.ownerId || '').trim()
|
||
const mergedPath = path.resolve(String(value?.mergedPath || ''))
|
||
const finalPath = path.resolve(String(value?.finalPath || ''))
|
||
const storedFinalPath = String(value?.storedFinalPath || '').trim()
|
||
const fileName = normalizeSafeFileName(String(value?.fileName || ''))
|
||
const size = Number(value?.size)
|
||
const createdAt = Number(value?.createdAt)
|
||
const uploadRoot = path.resolve(getUploadRootDir())
|
||
const tempRoot = path.join(uploadRoot, 'temp')
|
||
const expectedSessionDir = path.join(tempRoot, uploadId)
|
||
|
||
if (
|
||
value?.version !== JOURNAL_VERSION
|
||
|| !SAFE_UPLOAD_ID_PATTERN.test(uploadId)
|
||
|| !isSafeOwnerId(ownerId)
|
||
|| !fileName
|
||
|| !Number.isSafeInteger(size)
|
||
|| size <= 0
|
||
|| !Number.isFinite(createdAt)
|
||
|| createdAt <= 0
|
||
|| path.dirname(mergedPath) !== expectedSessionDir
|
||
|| !path.basename(mergedPath).startsWith('merged-')
|
||
|| !isPathInside(tempRoot, mergedPath)
|
||
|| path.dirname(finalPath) !== uploadRoot
|
||
|| !normalizeSafeFileName(path.basename(finalPath))
|
||
|| path.basename(finalPath) !== fileName
|
||
|| path.resolve(storedFinalPath) !== finalPath
|
||
) {
|
||
throw new MaterialUploadConsumptionJournalError()
|
||
}
|
||
|
||
return {
|
||
version: JOURNAL_VERSION,
|
||
uploadId,
|
||
ownerId,
|
||
mergedPath,
|
||
finalPath,
|
||
storedFinalPath,
|
||
fileName,
|
||
size,
|
||
createdAt,
|
||
}
|
||
}
|
||
|
||
async function readJournalFile(
|
||
ownerId: string,
|
||
uploadId: string,
|
||
): Promise<MaterialUploadConsumptionRecord | null> {
|
||
if (!SAFE_UPLOAD_ID_PATTERN.test(uploadId) || !isSafeOwnerId(ownerId)) {
|
||
throw new MaterialUploadConsumptionJournalError()
|
||
}
|
||
|
||
const journalPath = resolveJournalPath(ownerId, uploadId)
|
||
let stats
|
||
try {
|
||
stats = await lstat(journalPath)
|
||
} catch (error: any) {
|
||
if (error?.code === 'ENOENT') {
|
||
return null
|
||
}
|
||
throw new MaterialUploadConsumptionJournalError()
|
||
}
|
||
if (!stats.isFile() || stats.isSymbolicLink() || stats.size <= 0 || stats.size > MAX_JOURNAL_BYTES) {
|
||
throw new MaterialUploadConsumptionJournalError()
|
||
}
|
||
|
||
let record: MaterialUploadConsumptionRecord
|
||
try {
|
||
record = normalizeRecord(JSON.parse(await readFile(journalPath, 'utf8')))
|
||
} catch (error) {
|
||
if (error instanceof MaterialUploadConsumptionJournalError) {
|
||
throw error
|
||
}
|
||
throw new MaterialUploadConsumptionJournalError()
|
||
}
|
||
if (record.ownerId !== ownerId || record.uploadId !== uploadId) {
|
||
throw new MaterialUploadConsumptionJournalError()
|
||
}
|
||
return record
|
||
}
|
||
|
||
async function inspectRegularFile(
|
||
uploadRoot: string,
|
||
filePath: string,
|
||
expectedSize: number,
|
||
): Promise<'missing' | 'valid' | 'invalid'> {
|
||
let stats
|
||
try {
|
||
stats = await lstat(filePath)
|
||
} catch (error: any) {
|
||
if (error?.code === 'ENOENT') {
|
||
return 'missing'
|
||
}
|
||
throw new MaterialUploadConsumptionJournalError()
|
||
}
|
||
|
||
if (
|
||
stats.isSymbolicLink()
|
||
|| !stats.isFile()
|
||
|| stats.size !== expectedSize
|
||
|| !(await isPathFreeOfSymbolicLinks(uploadRoot, filePath))
|
||
) {
|
||
return 'invalid'
|
||
}
|
||
return 'valid'
|
||
}
|
||
|
||
async function syncDirectory(directoryPath: string): Promise<void> {
|
||
const directory = await open(directoryPath, 'r')
|
||
try {
|
||
await directory.sync()
|
||
} catch (error: any) {
|
||
// Windows 不支持目录 fsync；Linux 生产环境的其他错误必须上抛。
|
||
if (process.platform !== 'win32' || !['EINVAL', 'EPERM', 'ENOTSUP'].includes(error?.code)) {
|
||
throw error
|
||
}
|
||
} finally {
|
||
await directory.close()
|
||
}
|
||
}
|
||
|
||
export async function writeMaterialUploadConsumptionJournal(input: {
|
||
uploadId: string
|
||
ownerId: string
|
||
mergedPath: string
|
||
finalPath: string
|
||
storedFinalPath: string
|
||
fileName: string
|
||
size: number
|
||
}): Promise<MaterialUploadConsumptionRecord> {
|
||
const record = normalizeRecord({
|
||
version: JOURNAL_VERSION,
|
||
...input,
|
||
createdAt: Date.now(),
|
||
})
|
||
const uploadRoot = path.resolve(getUploadRootDir())
|
||
if (await inspectRegularFile(uploadRoot, record.mergedPath, record.size) !== 'valid') {
|
||
throw new MaterialUploadConsumptionJournalError('上传源文件状态异常，未执行移动')
|
||
}
|
||
if (await inspectRegularFile(uploadRoot, record.finalPath, record.size) !== 'missing') {
|
||
throw new MaterialUploadConsumptionJournalError('上传目标文件状态异常，未执行移动')
|
||
}
|
||
|
||
const existing = await readJournalFile(record.ownerId, record.uploadId)
|
||
if (existing) {
|
||
const comparableExisting = { ...existing, createdAt: 0 }
|
||
const comparableRecord = { ...record, createdAt: 0 }
|
||
if (JSON.stringify(comparableExisting) !== JSON.stringify(comparableRecord)) {
|
||
throw new MaterialUploadConsumptionJournalError()
|
||
}
|
||
return existing
|
||
}
|
||
|
||
const journalDirectory = await ensureSafeDirectoryWithinRoot(
|
||
uploadRoot,
|
||
JOURNAL_DIRECTORY_NAME,
|
||
)
|
||
if (!journalDirectory) {
|
||
throw new MaterialUploadConsumptionJournalError()
|
||
}
|
||
const journalPath = resolveJournalPath(record.ownerId, record.uploadId)
|
||
const temporaryPath = resolveStrictChildPath(
|
||
journalDirectory,
|
||
`${path.basename(journalPath)}.${randomBytes(8).toString('hex')}.tmp`,
|
||
)
|
||
if (!temporaryPath) {
|
||
throw new MaterialUploadConsumptionJournalError()
|
||
}
|
||
|
||
const file = await open(temporaryPath, 'wx', 0o600)
|
||
try {
|
||
await file.writeFile(`${JSON.stringify(record)}\n`, 'utf8')
|
||
await file.sync()
|
||
} catch (error) {
|
||
await file.close().catch(() => undefined)
|
||
await unlink(temporaryPath).catch(() => undefined)
|
||
throw error
|
||
}
|
||
await file.close()
|
||
|
||
try {
|
||
await rename(temporaryPath, journalPath)
|
||
await syncDirectory(journalDirectory)
|
||
} catch (error) {
|
||
await unlink(temporaryPath).catch(() => undefined)
|
||
throw error
|
||
}
|
||
return record
|
||
}
|
||
|
||
export async function recoverMaterialUploadConsumptionJournal(input: {
|
||
ownerId: string
|
||
uploadId: string
|
||
}): Promise<RecoveredMaterialUploadConsumption | null> {
|
||
const record = await readJournalFile(input.ownerId, input.uploadId)
|
||
if (!record) {
|
||
return null
|
||
}
|
||
|
||
const uploadRoot = path.resolve(getUploadRootDir())
|
||
const [mergedState, finalState] = await Promise.all([
|
||
inspectRegularFile(uploadRoot, record.mergedPath, record.size),
|
||
inspectRegularFile(uploadRoot, record.finalPath, record.size),
|
||
])
|
||
if (mergedState === 'valid' && finalState === 'missing') {
|
||
return { record, sourcePath: record.mergedPath, sourceLocation: 'merged' }
|
||
}
|
||
if (mergedState === 'missing' && finalState === 'valid') {
|
||
return { record, sourcePath: record.finalPath, sourceLocation: 'final' }
|
||
}
|
||
throw new MaterialUploadConsumptionJournalError()
|
||
}
|
||
|
||
export async function syncMaterialUploadRootDirectory(): Promise<void> {
|
||
await syncDirectory(getUploadRootDir())
|
||
}
|
||
|
||
export async function removeMaterialUploadConsumptionJournal(input: {
|
||
ownerId: string
|
||
uploadId: string
|
||
expectedFinalPath?: string
|
||
}): Promise<boolean> {
|
||
const record = await readJournalFile(input.ownerId, input.uploadId)
|
||
if (!record) {
|
||
return false
|
||
}
|
||
if (
|
||
input.expectedFinalPath
|
||
&& path.resolve(input.expectedFinalPath) !== record.finalPath
|
||
) {
|
||
throw new MaterialUploadConsumptionJournalError()
|
||
}
|
||
|
||
const journalPath = resolveJournalPath(record.ownerId, record.uploadId)
|
||
await unlink(journalPath)
|
||
await syncDirectory(path.dirname(journalPath))
|
||
return true
|
||
}
|