perf: speed up durable material file cleanup
This commit is contained in:
parent
df0e765f3e
commit
de2d7595ce
11
docs/回收站清理性能修复-2026-10-09.md
Normal file
11
docs/回收站清理性能修复-2026-10-09.md
Normal file
@ -0,0 +1,11 @@
|
||||
# 回收站清理性能修复
|
||||
|
||||
用户提交的 18 条素材包含 1276 个文件，原版本从提交到全部完成用了约 6 分 33 秒。任务无错误，但每条素材都读取全库引用：399050 条资产与其他引用形成约 49.9 万个文件身份，单次检查约 19 秒；只有 4 个文件的任务也耗时 18.6 秒。
|
||||
|
||||
此次修改只涉及后台清理：按当前待处理文件的完整地址筛选资产引用，常规地址在数据库内做集合匹配，再使用原有路径、桶与存储边界规则核对。编码、控制字符、点路径等特殊格式，以及白底图本地派生引用仍纳入检查；每个任务重新查询，保护其他素材和回收站中的共用文件。
|
||||
|
||||
物理清理每批最多并行 8 个文件，逐文件保留结果并按批持久化。批内部分失败时先保存已成功的结果，再进入原有自动重试；租约和心跳规则保留。进程中断时最多重做一个批次，缺失文件删除仍幂等。页面、删除权限和存储写入范围没有调整，无新增数据库迁移。
|
||||
|
||||
验证：20 项相关 Jest 测试、类型检查、11 组真实 PostgreSQL/MinIO/文件系统集成检查通过，覆盖特殊文件名、编码历史域名、同名不同路径、白底派生共用文件、2000 个文件候选上限、部分失败续跑、过期租约接管及审计保留。跨三个批次处理 19 个真实本地文件，结果全部持久化，隔离环境耗时 91ms。
|
||||
|
||||
线上数据仅执行只读检查：同一 4 文件任务的引用检查由约 19 秒降至 4.9 秒，原 18 条任务均已由用户提交并完成；没有为验收新增生产删除。此数字是引用检查耗时，不能等同于整批物理删除耗时。精确发布提交、镜像、健康及只读对照证据保存在 `tmp/material-purge-slow-qa/`。
|
||||
@ -2,7 +2,7 @@
|
||||
import { promises as fs } from 'fs'
|
||||
import os from 'os'
|
||||
import path from 'path'
|
||||
import { collectPurgeFiles, identifyPurgeFile, removePurgeFile } from '@/lib/material-purge-files'
|
||||
import { collectPurgeFiles, identifyPurgeFile, removePurgeFile, findPurgeFileReferences } from '@/lib/material-purge-files'
|
||||
|
||||
jest.mock('@/lib/prisma', () => ({ prisma: {} }))
|
||||
jest.mock('@/lib/s3', () => ({
|
||||
@ -16,11 +16,13 @@ jest.mock('@/lib/s3', () => ({
|
||||
|
||||
let root: string
|
||||
let oldUpload: string | undefined
|
||||
let oldBucket: string | undefined
|
||||
const material = (source: string) => ({ id: '11111111-1111-4111-8111-111111111111', previewUrl: source, zipFilePath: source,
|
||||
displayImages: [], videoSourceUrl: null, videoPreview540Url: null, videoPreviewClipUrl: null, videoCoverUrl: null, assets: [], skus: [] })
|
||||
|
||||
beforeEach(async () => {
|
||||
oldUpload = process.env.UPLOAD_DIR
|
||||
oldBucket = process.env.S3_BUCKET
|
||||
root = await fs.mkdtemp(path.join(os.tmpdir(), 'gongxiangp2-purge-files-'))
|
||||
process.env.UPLOAD_DIR = root
|
||||
})
|
||||
@ -28,6 +30,8 @@ afterEach(async () => {
|
||||
await fs.rm(root, { recursive: true, force: true })
|
||||
if (oldUpload === undefined) delete process.env.UPLOAD_DIR
|
||||
else process.env.UPLOAD_DIR = oldUpload
|
||||
if (oldBucket === undefined) delete process.env.S3_BUCKET
|
||||
else process.env.S3_BUCKET = oldBucket
|
||||
})
|
||||
|
||||
test('规范化等价的 uploads URL 并去重，重复删除缺失文件可续跑', async () => {
|
||||
@ -64,3 +68,39 @@ test('任务入队后同路径文件被替换时拒绝删除', async () => {
|
||||
test('没有物理文件的 data 预览不产生删除目标', async () => {
|
||||
expect(await collectPurgeFiles(material('data:image/svg+xml;base64,Zm9v'))).toEqual([])
|
||||
})
|
||||
|
||||
test('候选筛选保留特殊文件名、编码 URL 和控制字符 URL 的等价引用', async () => {
|
||||
process.env.S3_BUCKET = 'test'
|
||||
const targets = [
|
||||
{ source: 'uploads/shared[1]+.bin', storage: 'local' as const, key: 'shared[1]+.bin', root, outcome: 'pending' as const },
|
||||
{ source: 'http://storage.test/project/shared.bin', storage: 's3' as const, key: 'project/shared.bin', bucket: 'test', outcome: 'pending' as const },
|
||||
]
|
||||
const sources = [
|
||||
'/uploads/shared[1]+.bin',
|
||||
'/uploads/other/shared[1]+.bin',
|
||||
'http://legacy.test/project/%73hared.bin?cache=1',
|
||||
'http://legacy.test/project/sha\tred.bin',
|
||||
'http://legacy.test/project/unrelated.bin',
|
||||
]
|
||||
const rows = sources.map(fileUrl => ({ fileUrl }))
|
||||
const empty = { findMany: jest.fn().mockResolvedValue([]) }
|
||||
const db: any = { material: empty, materialAsset: { findMany: jest.fn().mockResolvedValue(rows) },
|
||||
materialSku: empty, brandKnowledge: empty, systemSettings: empty, performanceSurveyWorkItem: empty,
|
||||
workReviewVersion: empty, productPackageUploadJob: empty, taskDetailChangeRequest: empty,
|
||||
$queryRaw: jest.fn().mockResolvedValue(rows) }
|
||||
const full = await findPurgeFileReferences(db)
|
||||
const targeted = await findPurgeFileReferences(db, targets)
|
||||
expect(targeted.has('local::shared[1]+.bin')).toBe(true)
|
||||
expect(targeted.has('s3:test:project/shared.bin')).toBe(true)
|
||||
expect(targeted.has('s3:test:project/unrelated.bin')).toBe(false)
|
||||
for (const identity of ['local::shared[1]+.bin', 's3:test:project/shared.bin']) {
|
||||
expect(targeted.has(identity)).toBe(full.has(identity))
|
||||
}
|
||||
})
|
||||
|
||||
test('没有待处理文件时不访问数据库', async () => {
|
||||
const db = { material: { findMany: jest.fn() }, $queryRaw: jest.fn() }
|
||||
expect(await findPurgeFileReferences(db as any, [])).toEqual(new Set())
|
||||
expect(db.material.findMany).not.toHaveBeenCalled()
|
||||
expect(db.$queryRaw).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
@ -133,12 +133,62 @@ function stringsIn(value: unknown): string[] {
|
||||
return []
|
||||
}
|
||||
|
||||
/** 包含回收站与 SKU 软删除记录，保留其他素材恢复时仍需要的共用文件。 */
|
||||
export async function findPurgeFileReferences(db: Prisma.TransactionClient): Promise<Set<string>> {
|
||||
/**
|
||||
* 常规路径按本次文件的完整地址匹配；编码、控制字符和 URL 点路径
|
||||
* 不能按字面排除，全部纳入候选。最终仍由原生路径校验核对完整身份。
|
||||
* 白底派生图的本地引用也单独纳入，不依赖源文件与公网文件的路径相同。
|
||||
* 包含回收站与 SKU 软删除记录，保留其他素材恢复时仍需要的共用文件。
|
||||
*/
|
||||
export async function findPurgeFileReferences(db: Prisma.TransactionClient, files?: readonly PurgeFile[]): Promise<Set<string>> {
|
||||
if (files && !files.length) return new Set()
|
||||
const caseFold = process.platform === 'win32'
|
||||
const sources = files ? new Set<string>() : null
|
||||
const root = path.resolve(getUploadRootDir())
|
||||
for (const file of files || []) {
|
||||
const values = file.storage === 's3'
|
||||
? [`/${file.key}`, `/${file.bucket}/${file.key}`]
|
||||
: [file.key, `uploads/${file.key}`, `/uploads/${file.key}`, `./uploads/${file.key}`,
|
||||
path.join(root, file.key), path.relative(process.cwd(), path.join(root, file.key)),
|
||||
`./${path.relative(process.cwd(), path.join(root, file.key))}`]
|
||||
for (const value of values) {
|
||||
const normalized = value.replace(/\\/g, '/')
|
||||
sources!.add(caseFold ? normalized.toLowerCase() : normalized)
|
||||
}
|
||||
}
|
||||
const candidate = (source: string) => {
|
||||
if (!sources || /[%\x00-\x1f\x7f]/.test(source)) return true
|
||||
const value = source.trim().replace(/\\/g, '/')
|
||||
const pathname = /^https?:\/\//i.test(value) ? value.split(/[?#]/, 1)[0] : value
|
||||
const relative = /^https?:\/\//i.test(pathname) ? pathname.slice(pathname.indexOf('/', pathname.indexOf('://') + 3)) : pathname
|
||||
if (relative.includes('/.') || relative.includes('//')) return true
|
||||
return sources.has(caseFold ? relative.toLowerCase() : relative)
|
||||
}
|
||||
// 格式特殊的地址先全取，再用原生规则核对。Windows 非 ASCII 地址也全取，
|
||||
// 避免数据库 locale 与 JavaScript 文件系统大小写规则不同。
|
||||
const edgeWhitespace = '[[:space:]\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]'
|
||||
const [materials, assets, skus, knowledge, settings, workItems, versions, uploads, changes] = await Promise.all([
|
||||
db.material.findMany({ select: { previewUrl: true, zipFilePath: true, displayImages: true,
|
||||
videoSourceUrl: true, videoPreview540Url: true, videoPreviewClipUrl: true, videoCoverUrl: true } }),
|
||||
db.materialAsset.findMany({ select: { fileUrl: true } }),
|
||||
sources
|
||||
? db.$queryRaw<{ fileUrl: string }[]>`
|
||||
WITH paths AS (
|
||||
SELECT file_url, CASE
|
||||
WHEN lower(left(file_url, 7)) = 'http://'
|
||||
THEN substring(file_url FROM 7 + strpos(substring(file_url FROM 8), '/'))
|
||||
WHEN lower(left(file_url, 8)) = 'https://'
|
||||
THEN substring(file_url FROM 8 + strpos(substring(file_url FROM 9), '/'))
|
||||
ELSE file_url END AS reference FROM material_assets
|
||||
)
|
||||
SELECT asset.file_url AS "fileUrl" FROM paths AS asset
|
||||
WHERE (CASE WHEN ${caseFold} THEN lower(asset.reference) ELSE asset.reference END)
|
||||
IN (SELECT unnest(${Array.from(sources)}::text[]))
|
||||
OR asset.file_url ~ '[%[:cntrl:]]'
|
||||
OR strpos(asset.file_url, ${'\\'}) > 0 OR strpos(asset.file_url, '/.') > 0
|
||||
OR strpos(asset.file_url, '?') > 0 OR strpos(asset.file_url, '#') > 0
|
||||
OR asset.file_url ~ '(^|[^:])//' OR asset.file_url ~ ${`^${edgeWhitespace}|${edgeWhitespace}$`}
|
||||
OR strpos(asset.file_url, '/white-background/') > 0
|
||||
OR (${caseFold} AND asset.file_url ~ ${'[^\u0001-\u007f]'})`
|
||||
: db.materialAsset.findMany({ select: { fileUrl: true } }),
|
||||
db.materialSku.findMany({ select: { filePath: true, previewUrl: true } }),
|
||||
db.brandKnowledge.findMany({ select: { filePath: true, previewImagePath: true, previewImageSlices: true } }),
|
||||
db.systemSettings.findMany({ select: { headerLogoUrl: true, loginLogoUrl: true } }),
|
||||
@ -149,6 +199,7 @@ export async function findPurgeFileReferences(db: Prisma.TransactionClient): Pro
|
||||
])
|
||||
const identities = new Set<string>()
|
||||
for (const source of stringsIn([materials, assets, skus, knowledge, settings, workItems, versions, uploads, changes])) {
|
||||
if (!candidate(source)) continue
|
||||
try {
|
||||
const file = identifyPurgeFile(source)
|
||||
if (file) identities.add(purgeFileIdentity(file))
|
||||
@ -168,7 +219,7 @@ export async function findPurgeFileReferences(db: Prisma.TransactionClient): Pro
|
||||
for (const asset of assets) {
|
||||
const derived = buildWhiteBackgroundPublicDisplayPathFromAssetPath(asset.fileUrl)
|
||||
const source = derived && buildPublicDisplayUrl(derived)
|
||||
if (source) {
|
||||
if (source && candidate(source)) {
|
||||
const file = identifyPurgeFile(source)
|
||||
if (file) identities.add(purgeFileIdentity(file))
|
||||
}
|
||||
|
||||
@ -10,6 +10,7 @@ export class MaterialPurgeError extends Error {
|
||||
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
|
||||
const LEASE_MS = 10 * 60_000
|
||||
const MAX_ATTEMPTS = 3
|
||||
const FILE_BATCH_SIZE = 8
|
||||
|
||||
export function validatePurgeIds(input: unknown): string[] {
|
||||
if (!Array.isArray(input) || input.length === 0 || input.length > 100 || input.some(id => typeof id !== 'string' || !UUID.test(id))) {
|
||||
@ -120,7 +121,7 @@ async function updateOwnedJob(job: MaterialPurgeJob, data: Prisma.MaterialPurgeJ
|
||||
})
|
||||
}
|
||||
|
||||
/** 每个文件的结果单独落库，重复删除缺失文件视为成功，可从进程中断处续跑。 */
|
||||
/** 每批最多 8 个文件，逐文件记录结果；缺失文件视为成功，可从进程中断处续跑。 */
|
||||
export async function processNextMaterialPurge(): Promise<boolean> {
|
||||
const job = await claimJob()
|
||||
if (!job) return false
|
||||
@ -131,14 +132,24 @@ export async function processNextMaterialPurge(): Promise<boolean> {
|
||||
}, 30_000)
|
||||
heartbeat.unref?.()
|
||||
try {
|
||||
const references = await findPurgeFileReferences(prisma)
|
||||
for (const file of files) {
|
||||
if (file.outcome !== 'pending') continue
|
||||
const references = await findPurgeFileReferences(prisma, files.filter(file => file.outcome === 'pending'))
|
||||
const pending = files.filter(file => file.outcome === 'pending')
|
||||
for (let offset = 0; offset < pending.length; offset += FILE_BATCH_SIZE) {
|
||||
if (leaseLost) throw new Error('文件清理任务已移交，等待新的处理结果')
|
||||
await updateOwnedJob(job, {})
|
||||
if (references.has(purgeFileIdentity(file))) file.outcome = 'shared'
|
||||
else { await removePurgeFile(file); file.outcome = 'deleted' }
|
||||
const batch = pending.slice(offset, offset + FILE_BATCH_SIZE)
|
||||
const results = await Promise.allSettled(batch.map(async file => {
|
||||
if (references.has(purgeFileIdentity(file))) return 'shared' as const
|
||||
await removePurgeFile(file)
|
||||
return 'deleted' as const
|
||||
}))
|
||||
results.forEach((result, index) => {
|
||||
if (result.status === 'fulfilled') batch[index].outcome = result.value
|
||||
})
|
||||
// 即使批内部分失败，也先保存成功文件的结果，再进入原有重试流程。
|
||||
await updateOwnedJob(job, { files: files as unknown as Prisma.InputJsonValue })
|
||||
const failure = results.find((result): result is PromiseRejectedResult => result.status === 'rejected')
|
||||
if (failure) throw failure.reason
|
||||
}
|
||||
await updateOwnedJob(job, { status: 'completed', completedAt: new Date(), leaseToken: null, leaseExpiresAt: null })
|
||||
} catch (error: any) {
|
||||
|
||||
@ -40,6 +40,7 @@ async function main() {
|
||||
assert.equal(migrated.status, 0, migrated.stderr)
|
||||
const { prisma } = await import('../lib/prisma')
|
||||
const { enqueueMaterialPurge, processNextMaterialPurge, retryMaterialPurge } = await import('../lib/material-purge')
|
||||
const { collectPurgeFiles, findPurgeFileReferences, purgeFileIdentity } = await import('../lib/material-purge-files')
|
||||
const run = randomUUID()
|
||||
const passed: string[] = []
|
||||
passed.push('真实升级 SQL 创建清理表、重试索引与审计表，并解除历史日志的素材级联删除')
|
||||
@ -100,6 +101,27 @@ async function main() {
|
||||
assert(await prisma.material.findUnique({ where: { id: keeper.id } }))
|
||||
passed.push('URL 等价引用与回收站引用保护共用文件')
|
||||
|
||||
const specialSource = await file('shared[1]+.bin')
|
||||
const specialKeeper = await make('special-keeper')
|
||||
await prisma.materialAsset.create({ data: { materialId: specialKeeper.id, assetType: 'test', fileUrl: '/uploads/shared[1]+.bin' } })
|
||||
const special = await make('special-delete', { zipFilePath: specialSource })
|
||||
const specialFiles = await collectPurgeFiles({ ...special, assets: [], skus: [] })
|
||||
const fullReferences = await findPurgeFileReferences(prisma)
|
||||
const targetedReferences = await findPurgeFileReferences(prisma, specialFiles)
|
||||
for (const target of specialFiles) assert.equal(targetedReferences.has(purgeFileIdentity(target)), fullReferences.has(purgeFileIdentity(target)))
|
||||
await enqueueMaterialPurge([special.id], admin.id)
|
||||
await processNextMaterialPurge()
|
||||
assert(await exists('shared[1]+.bin'))
|
||||
const otherDirectory = await make('same-filename', { zipFilePath: await file('same-filename.bin') })
|
||||
await prisma.materialAsset.create({ data: { materialId: specialKeeper.id, assetType: 'test', fileUrl: 'uploads/other/same-filename.bin' } })
|
||||
await enqueueMaterialPurge([otherDirectory.id], admin.id)
|
||||
await processNextMaterialPurge()
|
||||
assert(!await exists('same-filename.bin'))
|
||||
await findPurgeFileReferences(prisma, Array.from({ length: 2000 }, () => ({
|
||||
source: '', storage: 'local' as const, key: `${randomUUID()}.bin`, outcome: 'pending' as const,
|
||||
})))
|
||||
passed.push('真实 PostgreSQL 候选筛选与全量引用结果一致：特殊文件名共用资产保留，同名不同路径文件正常删除')
|
||||
|
||||
const batch = await Promise.all([make('batch-a', { zipFilePath: await file('batch-a.bin') }), make('batch-b', { zipFilePath: await file('batch-b.bin') })])
|
||||
const batchJobs = await enqueueMaterialPurge(batch.map(material => material.id), admin.id)
|
||||
assert.equal(batchJobs.length, 2)
|
||||
@ -110,6 +132,18 @@ async function main() {
|
||||
assert.equal(await prisma.materialPurgeJob.count({ where: { id: { in: batchJobs.map(job => job.id) }, status: 'completed' } }), 2)
|
||||
passed.push('批量作业并发领取不重复，重启后过期租约可接管并完成')
|
||||
|
||||
const wide = await make('wide-batch')
|
||||
const wideFiles = await Promise.all(Array.from({ length: 19 }, (_, index) => file(`wide-${index}.bin`)))
|
||||
await prisma.materialAsset.createMany({ data: wideFiles.map(fileUrl => ({ materialId: wide.id, assetType: 'test', fileUrl })) })
|
||||
const [wideJob] = await enqueueMaterialPurge([wide.id], admin.id)
|
||||
const wideStarted = Date.now()
|
||||
await processNextMaterialPurge()
|
||||
const wideResult = await prisma.materialPurgeJob.findUniqueOrThrow({ where: { id: wideJob.id } })
|
||||
assert.equal(wideResult.status, 'completed')
|
||||
assert.equal((wideResult.files as any[]).filter(f => f.outcome === 'deleted').length, 19)
|
||||
assert((await Promise.all(wideFiles.map((_, index) => exists(`wide-${index}.bin`)))).every(value => !value))
|
||||
passed.push(`真实文件系统跨三个清理批次完成 19 个文件，结果全部持久化，耗时 ${Date.now() - wideStarted}ms`)
|
||||
|
||||
const broken = await make('retry', { zipFilePath: await file('retry-first.bin'), displayImages: [await file('retry-second.bin')] })
|
||||
const [retryJob] = await enqueueMaterialPurge([broken.id], admin.id)
|
||||
await fs.rename(path.join(root, 'retry-second.bin'), path.join(root, 'saved-second.bin'))
|
||||
@ -155,6 +189,25 @@ async function main() {
|
||||
await processNextMaterialPurge()
|
||||
await client.send(new HeadObjectCommand({ Bucket, Key: sharedKey }))
|
||||
assert(await prisma.material.findUnique({ where: { id: objectKeeper.id } }))
|
||||
const encodedKey = `project-purge-qa/${run}/encoded[1]+.bin`
|
||||
await client.send(new PutObjectCommand({ Bucket, Key: encodedKey, Body: 'isolated-object' }))
|
||||
await prisma.materialAsset.create({ data: { materialId: objectKeeper.id, assetType: 'test',
|
||||
fileUrl: `https://legacy-storage.test/${Bucket}/${encodedKey.replace('encoded', '%65ncoded')}?cache=1` } })
|
||||
const encoded = await make('encoded-shared', { zipFilePath: source(encodedKey) })
|
||||
await enqueueMaterialPurge([encoded.id], admin.id)
|
||||
await processNextMaterialPurge()
|
||||
await client.send(new HeadObjectCommand({ Bucket, Key: encodedKey }))
|
||||
const localWhiteBackground = `uploads/skus/${objectKeeper.id}/white-background/${run}/preview/shared-display.png`
|
||||
await prisma.materialAsset.create({ data: { materialId: objectKeeper.id, assetType: 'test', fileUrl: localWhiteBackground } })
|
||||
const { buildPublicDisplayUrl, buildWhiteBackgroundPublicDisplayPathFromAssetPath } = await import('../lib/public-display')
|
||||
const displaySource = buildPublicDisplayUrl(buildWhiteBackgroundPublicDisplayPathFromAssetPath(localWhiteBackground)!)!
|
||||
const { getTrustedS3MutationKey } = await import('../lib/s3')
|
||||
const displayKey = getTrustedS3MutationKey(displaySource)
|
||||
await client.send(new PutObjectCommand({ Bucket, Key: displayKey, Body: 'isolated-display' }))
|
||||
const display = await make('derived-shared', { zipFilePath: displaySource })
|
||||
await enqueueMaterialPurge([display.id], admin.id)
|
||||
await processNextMaterialPurge()
|
||||
await client.send(new HeadObjectCommand({ Bucket, Key: displayKey }))
|
||||
const objectForeign = await make('object-foreign', { zipFilePath: source(foreignKey) })
|
||||
await assert.rejects(enqueueMaterialPurge([objectForeign.id], admin.id), /业务前缀之外/)
|
||||
await client.send(new HeadObjectCommand({ Bucket, Key: foreignKey }))
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user