diff --git a/docs/回收站清理性能修复-2026-10-09.md b/docs/回收站清理性能修复-2026-10-09.md new file mode 100644 index 0000000..25a904d --- /dev/null +++ b/docs/回收站清理性能修复-2026-10-09.md @@ -0,0 +1,11 @@ +# 回收站清理性能修复 + +用户提交的 18 条素材包含 1276 个文件,原版本从提交到全部完成用了约 6 分 33 秒。任务无错误,但每条素材都读取全库引用:399050 条资产与其他引用形成约 49.9 万个文件身份,单次检查约 19 秒;只有 4 个文件的任务也耗时 18.6 秒。 + +此次修改只涉及后台清理:按当前待处理文件的完整地址筛选资产引用,常规地址在数据库内做集合匹配,再使用原有路径、桶与存储边界规则核对。编码、控制字符、点路径等特殊格式,以及白底图本地派生引用仍纳入检查;每个任务重新查询,保护其他素材和回收站中的共用文件。 + +物理清理每批最多并行 8 个文件,逐文件保留结果并按批持久化。批内部分失败时先保存已成功的结果,再进入原有自动重试;租约和心跳规则保留。进程中断时最多重做一个批次,缺失文件删除仍幂等。页面、删除权限和存储写入范围没有调整,无新增数据库迁移。 + +验证:20 项相关 Jest 测试、类型检查、11 组真实 PostgreSQL/MinIO/文件系统集成检查通过,覆盖特殊文件名、编码历史域名、同名不同路径、白底派生共用文件、2000 个文件候选上限、部分失败续跑、过期租约接管及审计保留。跨三个批次处理 19 个真实本地文件,结果全部持久化,隔离环境耗时 91ms。 + +线上数据仅执行只读检查:同一 4 文件任务的引用检查由约 19 秒降至 4.9 秒,原 18 条任务均已由用户提交并完成;没有为验收新增生产删除。此数字是引用检查耗时,不能等同于整批物理删除耗时。精确发布提交、镜像、健康及只读对照证据保存在 `tmp/material-purge-slow-qa/`。 diff --git a/lib/__tests__/material-purge-files.test.ts b/lib/__tests__/material-purge-files.test.ts index 23cbeaa..c817a6b 100644 --- a/lib/__tests__/material-purge-files.test.ts +++ b/lib/__tests__/material-purge-files.test.ts @@ -2,7 +2,7 @@ import { promises as fs } from 'fs' import os from 'os' import path from 'path' -import { collectPurgeFiles, identifyPurgeFile, removePurgeFile } from '@/lib/material-purge-files' +import { collectPurgeFiles, identifyPurgeFile, removePurgeFile, findPurgeFileReferences } from '@/lib/material-purge-files' jest.mock('@/lib/prisma', () => ({ prisma: {} })) jest.mock('@/lib/s3', () => ({ @@ -16,11 +16,13 @@ jest.mock('@/lib/s3', () => ({ let root: string let oldUpload: string | undefined +let oldBucket: string | undefined const material = (source: string) => ({ id: '11111111-1111-4111-8111-111111111111', previewUrl: source, zipFilePath: source, displayImages: [], videoSourceUrl: null, videoPreview540Url: null, videoPreviewClipUrl: null, videoCoverUrl: null, assets: [], skus: [] }) beforeEach(async () => { oldUpload = process.env.UPLOAD_DIR + oldBucket = process.env.S3_BUCKET root = await fs.mkdtemp(path.join(os.tmpdir(), 'gongxiangp2-purge-files-')) process.env.UPLOAD_DIR = root }) @@ -28,6 +30,8 @@ afterEach(async () => { await fs.rm(root, { recursive: true, force: true }) if (oldUpload === undefined) delete process.env.UPLOAD_DIR else process.env.UPLOAD_DIR = oldUpload + if (oldBucket === undefined) delete process.env.S3_BUCKET + else process.env.S3_BUCKET = oldBucket }) test('规范化等价的 uploads URL 并去重,重复删除缺失文件可续跑', async () => { @@ -64,3 +68,39 @@ test('任务入队后同路径文件被替换时拒绝删除', async () => { test('没有物理文件的 data 预览不产生删除目标', async () => { expect(await collectPurgeFiles(material('data:image/svg+xml;base64,Zm9v'))).toEqual([]) }) + +test('候选筛选保留特殊文件名、编码 URL 和控制字符 URL 的等价引用', async () => { + process.env.S3_BUCKET = 'test' + const targets = [ + { source: 'uploads/shared[1]+.bin', storage: 'local' as const, key: 'shared[1]+.bin', root, outcome: 'pending' as const }, + { source: 'http://storage.test/project/shared.bin', storage: 's3' as const, key: 'project/shared.bin', bucket: 'test', outcome: 'pending' as const }, + ] + const sources = [ + '/uploads/shared[1]+.bin', + '/uploads/other/shared[1]+.bin', + 'http://legacy.test/project/%73hared.bin?cache=1', + 'http://legacy.test/project/sha\tred.bin', + 'http://legacy.test/project/unrelated.bin', + ] + const rows = sources.map(fileUrl => ({ fileUrl })) + const empty = { findMany: jest.fn().mockResolvedValue([]) } + const db: any = { material: empty, materialAsset: { findMany: jest.fn().mockResolvedValue(rows) }, + materialSku: empty, brandKnowledge: empty, systemSettings: empty, performanceSurveyWorkItem: empty, + workReviewVersion: empty, productPackageUploadJob: empty, taskDetailChangeRequest: empty, + $queryRaw: jest.fn().mockResolvedValue(rows) } + const full = await findPurgeFileReferences(db) + const targeted = await findPurgeFileReferences(db, targets) + expect(targeted.has('local::shared[1]+.bin')).toBe(true) + expect(targeted.has('s3:test:project/shared.bin')).toBe(true) + expect(targeted.has('s3:test:project/unrelated.bin')).toBe(false) + for (const identity of ['local::shared[1]+.bin', 's3:test:project/shared.bin']) { + expect(targeted.has(identity)).toBe(full.has(identity)) + } +}) + +test('没有待处理文件时不访问数据库', async () => { + const db = { material: { findMany: jest.fn() }, $queryRaw: jest.fn() } + expect(await findPurgeFileReferences(db as any, [])).toEqual(new Set()) + expect(db.material.findMany).not.toHaveBeenCalled() + expect(db.$queryRaw).not.toHaveBeenCalled() +}) diff --git a/lib/material-purge-files.ts b/lib/material-purge-files.ts index 32e95b9..424628d 100644 --- a/lib/material-purge-files.ts +++ b/lib/material-purge-files.ts @@ -133,12 +133,62 @@ function stringsIn(value: unknown): string[] { return [] } -/** 包含回收站与 SKU 软删除记录,保留其他素材恢复时仍需要的共用文件。 */ -export async function findPurgeFileReferences(db: Prisma.TransactionClient): Promise> { +/** + * 常规路径按本次文件的完整地址匹配;编码、控制字符和 URL 点路径 + * 不能按字面排除,全部纳入候选。最终仍由原生路径校验核对完整身份。 + * 白底派生图的本地引用也单独纳入,不依赖源文件与公网文件的路径相同。 + * 包含回收站与 SKU 软删除记录,保留其他素材恢复时仍需要的共用文件。 + */ +export async function findPurgeFileReferences(db: Prisma.TransactionClient, files?: readonly PurgeFile[]): Promise> { + if (files && !files.length) return new Set() + const caseFold = process.platform === 'win32' + const sources = files ? new Set() : null + const root = path.resolve(getUploadRootDir()) + for (const file of files || []) { + const values = file.storage === 's3' + ? [`/${file.key}`, `/${file.bucket}/${file.key}`] + : [file.key, `uploads/${file.key}`, `/uploads/${file.key}`, `./uploads/${file.key}`, + path.join(root, file.key), path.relative(process.cwd(), path.join(root, file.key)), + `./${path.relative(process.cwd(), path.join(root, file.key))}`] + for (const value of values) { + const normalized = value.replace(/\\/g, '/') + sources!.add(caseFold ? normalized.toLowerCase() : normalized) + } + } + const candidate = (source: string) => { + if (!sources || /[%\x00-\x1f\x7f]/.test(source)) return true + const value = source.trim().replace(/\\/g, '/') + const pathname = /^https?:\/\//i.test(value) ? value.split(/[?#]/, 1)[0] : value + const relative = /^https?:\/\//i.test(pathname) ? pathname.slice(pathname.indexOf('/', pathname.indexOf('://') + 3)) : pathname + if (relative.includes('/.') || relative.includes('//')) return true + return sources.has(caseFold ? relative.toLowerCase() : relative) + } + // 格式特殊的地址先全取,再用原生规则核对。Windows 非 ASCII 地址也全取, + // 避免数据库 locale 与 JavaScript 文件系统大小写规则不同。 + const edgeWhitespace = '[[:space:]\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]' const [materials, assets, skus, knowledge, settings, workItems, versions, uploads, changes] = await Promise.all([ db.material.findMany({ select: { previewUrl: true, zipFilePath: true, displayImages: true, videoSourceUrl: true, videoPreview540Url: true, videoPreviewClipUrl: true, videoCoverUrl: true } }), - db.materialAsset.findMany({ select: { fileUrl: true } }), + sources + ? db.$queryRaw<{ fileUrl: string }[]>` + WITH paths AS ( + SELECT file_url, CASE + WHEN lower(left(file_url, 7)) = 'http://' + THEN substring(file_url FROM 7 + strpos(substring(file_url FROM 8), '/')) + WHEN lower(left(file_url, 8)) = 'https://' + THEN substring(file_url FROM 8 + strpos(substring(file_url FROM 9), '/')) + ELSE file_url END AS reference FROM material_assets + ) + SELECT asset.file_url AS "fileUrl" FROM paths AS asset + WHERE (CASE WHEN ${caseFold} THEN lower(asset.reference) ELSE asset.reference END) + IN (SELECT unnest(${Array.from(sources)}::text[])) + OR asset.file_url ~ '[%[:cntrl:]]' + OR strpos(asset.file_url, ${'\\'}) > 0 OR strpos(asset.file_url, '/.') > 0 + OR strpos(asset.file_url, '?') > 0 OR strpos(asset.file_url, '#') > 0 + OR asset.file_url ~ '(^|[^:])//' OR asset.file_url ~ ${`^${edgeWhitespace}|${edgeWhitespace}$`} + OR strpos(asset.file_url, '/white-background/') > 0 + OR (${caseFold} AND asset.file_url ~ ${'[^\u0001-\u007f]'})` + : db.materialAsset.findMany({ select: { fileUrl: true } }), db.materialSku.findMany({ select: { filePath: true, previewUrl: true } }), db.brandKnowledge.findMany({ select: { filePath: true, previewImagePath: true, previewImageSlices: true } }), db.systemSettings.findMany({ select: { headerLogoUrl: true, loginLogoUrl: true } }), @@ -149,6 +199,7 @@ export async function findPurgeFileReferences(db: Prisma.TransactionClient): Pro ]) const identities = new Set() for (const source of stringsIn([materials, assets, skus, knowledge, settings, workItems, versions, uploads, changes])) { + if (!candidate(source)) continue try { const file = identifyPurgeFile(source) if (file) identities.add(purgeFileIdentity(file)) @@ -168,7 +219,7 @@ export async function findPurgeFileReferences(db: Prisma.TransactionClient): Pro for (const asset of assets) { const derived = buildWhiteBackgroundPublicDisplayPathFromAssetPath(asset.fileUrl) const source = derived && buildPublicDisplayUrl(derived) - if (source) { + if (source && candidate(source)) { const file = identifyPurgeFile(source) if (file) identities.add(purgeFileIdentity(file)) } diff --git a/lib/material-purge.ts b/lib/material-purge.ts index 57daae2..b8ddbda 100644 --- a/lib/material-purge.ts +++ b/lib/material-purge.ts @@ -10,6 +10,7 @@ export class MaterialPurgeError extends Error { const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i const LEASE_MS = 10 * 60_000 const MAX_ATTEMPTS = 3 +const FILE_BATCH_SIZE = 8 export function validatePurgeIds(input: unknown): string[] { if (!Array.isArray(input) || input.length === 0 || input.length > 100 || input.some(id => typeof id !== 'string' || !UUID.test(id))) { @@ -120,7 +121,7 @@ async function updateOwnedJob(job: MaterialPurgeJob, data: Prisma.MaterialPurgeJ }) } -/** 每个文件的结果单独落库,重复删除缺失文件视为成功,可从进程中断处续跑。 */ +/** 每批最多 8 个文件,逐文件记录结果;缺失文件视为成功,可从进程中断处续跑。 */ export async function processNextMaterialPurge(): Promise { const job = await claimJob() if (!job) return false @@ -131,14 +132,24 @@ export async function processNextMaterialPurge(): Promise { }, 30_000) heartbeat.unref?.() try { - const references = await findPurgeFileReferences(prisma) - for (const file of files) { - if (file.outcome !== 'pending') continue + const references = await findPurgeFileReferences(prisma, files.filter(file => file.outcome === 'pending')) + const pending = files.filter(file => file.outcome === 'pending') + for (let offset = 0; offset < pending.length; offset += FILE_BATCH_SIZE) { if (leaseLost) throw new Error('文件清理任务已移交,等待新的处理结果') await updateOwnedJob(job, {}) - if (references.has(purgeFileIdentity(file))) file.outcome = 'shared' - else { await removePurgeFile(file); file.outcome = 'deleted' } + const batch = pending.slice(offset, offset + FILE_BATCH_SIZE) + const results = await Promise.allSettled(batch.map(async file => { + if (references.has(purgeFileIdentity(file))) return 'shared' as const + await removePurgeFile(file) + return 'deleted' as const + })) + results.forEach((result, index) => { + if (result.status === 'fulfilled') batch[index].outcome = result.value + }) + // 即使批内部分失败,也先保存成功文件的结果,再进入原有重试流程。 await updateOwnedJob(job, { files: files as unknown as Prisma.InputJsonValue }) + const failure = results.find((result): result is PromiseRejectedResult => result.status === 'rejected') + if (failure) throw failure.reason } await updateOwnedJob(job, { status: 'completed', completedAt: new Date(), leaseToken: null, leaseExpiresAt: null }) } catch (error: any) { diff --git a/scripts/test-material-purge-integration.ts b/scripts/test-material-purge-integration.ts index 79868da..c8fef94 100644 --- a/scripts/test-material-purge-integration.ts +++ b/scripts/test-material-purge-integration.ts @@ -40,6 +40,7 @@ async function main() { assert.equal(migrated.status, 0, migrated.stderr) const { prisma } = await import('../lib/prisma') const { enqueueMaterialPurge, processNextMaterialPurge, retryMaterialPurge } = await import('../lib/material-purge') + const { collectPurgeFiles, findPurgeFileReferences, purgeFileIdentity } = await import('../lib/material-purge-files') const run = randomUUID() const passed: string[] = [] passed.push('真实升级 SQL 创建清理表、重试索引与审计表,并解除历史日志的素材级联删除') @@ -100,6 +101,27 @@ async function main() { assert(await prisma.material.findUnique({ where: { id: keeper.id } })) passed.push('URL 等价引用与回收站引用保护共用文件') + const specialSource = await file('shared[1]+.bin') + const specialKeeper = await make('special-keeper') + await prisma.materialAsset.create({ data: { materialId: specialKeeper.id, assetType: 'test', fileUrl: '/uploads/shared[1]+.bin' } }) + const special = await make('special-delete', { zipFilePath: specialSource }) + const specialFiles = await collectPurgeFiles({ ...special, assets: [], skus: [] }) + const fullReferences = await findPurgeFileReferences(prisma) + const targetedReferences = await findPurgeFileReferences(prisma, specialFiles) + for (const target of specialFiles) assert.equal(targetedReferences.has(purgeFileIdentity(target)), fullReferences.has(purgeFileIdentity(target))) + await enqueueMaterialPurge([special.id], admin.id) + await processNextMaterialPurge() + assert(await exists('shared[1]+.bin')) + const otherDirectory = await make('same-filename', { zipFilePath: await file('same-filename.bin') }) + await prisma.materialAsset.create({ data: { materialId: specialKeeper.id, assetType: 'test', fileUrl: 'uploads/other/same-filename.bin' } }) + await enqueueMaterialPurge([otherDirectory.id], admin.id) + await processNextMaterialPurge() + assert(!await exists('same-filename.bin')) + await findPurgeFileReferences(prisma, Array.from({ length: 2000 }, () => ({ + source: '', storage: 'local' as const, key: `${randomUUID()}.bin`, outcome: 'pending' as const, + }))) + passed.push('真实 PostgreSQL 候选筛选与全量引用结果一致:特殊文件名共用资产保留,同名不同路径文件正常删除') + const batch = await Promise.all([make('batch-a', { zipFilePath: await file('batch-a.bin') }), make('batch-b', { zipFilePath: await file('batch-b.bin') })]) const batchJobs = await enqueueMaterialPurge(batch.map(material => material.id), admin.id) assert.equal(batchJobs.length, 2) @@ -110,6 +132,18 @@ async function main() { assert.equal(await prisma.materialPurgeJob.count({ where: { id: { in: batchJobs.map(job => job.id) }, status: 'completed' } }), 2) passed.push('批量作业并发领取不重复,重启后过期租约可接管并完成') + const wide = await make('wide-batch') + const wideFiles = await Promise.all(Array.from({ length: 19 }, (_, index) => file(`wide-${index}.bin`))) + await prisma.materialAsset.createMany({ data: wideFiles.map(fileUrl => ({ materialId: wide.id, assetType: 'test', fileUrl })) }) + const [wideJob] = await enqueueMaterialPurge([wide.id], admin.id) + const wideStarted = Date.now() + await processNextMaterialPurge() + const wideResult = await prisma.materialPurgeJob.findUniqueOrThrow({ where: { id: wideJob.id } }) + assert.equal(wideResult.status, 'completed') + assert.equal((wideResult.files as any[]).filter(f => f.outcome === 'deleted').length, 19) + assert((await Promise.all(wideFiles.map((_, index) => exists(`wide-${index}.bin`)))).every(value => !value)) + passed.push(`真实文件系统跨三个清理批次完成 19 个文件,结果全部持久化,耗时 ${Date.now() - wideStarted}ms`) + const broken = await make('retry', { zipFilePath: await file('retry-first.bin'), displayImages: [await file('retry-second.bin')] }) const [retryJob] = await enqueueMaterialPurge([broken.id], admin.id) await fs.rename(path.join(root, 'retry-second.bin'), path.join(root, 'saved-second.bin')) @@ -155,6 +189,25 @@ async function main() { await processNextMaterialPurge() await client.send(new HeadObjectCommand({ Bucket, Key: sharedKey })) assert(await prisma.material.findUnique({ where: { id: objectKeeper.id } })) + const encodedKey = `project-purge-qa/${run}/encoded[1]+.bin` + await client.send(new PutObjectCommand({ Bucket, Key: encodedKey, Body: 'isolated-object' })) + await prisma.materialAsset.create({ data: { materialId: objectKeeper.id, assetType: 'test', + fileUrl: `https://legacy-storage.test/${Bucket}/${encodedKey.replace('encoded', '%65ncoded')}?cache=1` } }) + const encoded = await make('encoded-shared', { zipFilePath: source(encodedKey) }) + await enqueueMaterialPurge([encoded.id], admin.id) + await processNextMaterialPurge() + await client.send(new HeadObjectCommand({ Bucket, Key: encodedKey })) + const localWhiteBackground = `uploads/skus/${objectKeeper.id}/white-background/${run}/preview/shared-display.png` + await prisma.materialAsset.create({ data: { materialId: objectKeeper.id, assetType: 'test', fileUrl: localWhiteBackground } }) + const { buildPublicDisplayUrl, buildWhiteBackgroundPublicDisplayPathFromAssetPath } = await import('../lib/public-display') + const displaySource = buildPublicDisplayUrl(buildWhiteBackgroundPublicDisplayPathFromAssetPath(localWhiteBackground)!)! + const { getTrustedS3MutationKey } = await import('../lib/s3') + const displayKey = getTrustedS3MutationKey(displaySource) + await client.send(new PutObjectCommand({ Bucket, Key: displayKey, Body: 'isolated-display' })) + const display = await make('derived-shared', { zipFilePath: displaySource }) + await enqueueMaterialPurge([display.id], admin.id) + await processNextMaterialPurge() + await client.send(new HeadObjectCommand({ Bucket, Key: displayKey })) const objectForeign = await make('object-foreign', { zipFilePath: source(foreignKey) }) await assert.rejects(enqueueMaterialPurge([objectForeign.id], admin.id), /业务前缀之外/) await client.send(new HeadObjectCommand({ Bucket, Key: foreignKey }))