diff --git a/docs/生产数据库安全增量迁移SOP-2026-08-12.md b/docs/生产数据库安全增量迁移SOP-2026-08-12.md index 014fd7c3..31ad3f6f 100644 --- a/docs/生产数据库安全增量迁移SOP-2026-08-12.md +++ b/docs/生产数据库安全增量迁移SOP-2026-08-12.md @@ -2,7 +2,7 @@ ## 结论 -当前应用发布工具验证四十三份目标 migration。普通部署脚本**只读验证，不会自动修改数据库**。只有独立 migrator 在同时拿到精确 Git 提交、数据库身份和备份凭证后，才允许执行 `prisma migrate deploy`。目标名称以该精确提交的 `TARGET_MIGRATIONS` 为准，不能沿用历史批次的数量或凭证。 +当前应用发布工具验证四十四份目标 migration。普通部署脚本**只读验证，不会自动修改数据库**。只有独立 migrator 在同时拿到精确 Git 提交、数据库身份和备份凭证后，才允许执行 `prisma migrate deploy`。目标名称以该精确提交的 `TARGET_MIGRATIONS` 为准，不能沿用历史批次的数量或凭证。 `20261106110000_material_access_scope` 为所有素材增加默认 `general` 的使用范围，新增索引，并限制专属素材为运营中心 G 内共享。迁移不改变现有素材的可见范围；审计同时验证范围枚举和禁止叠加个人私密的数据库约束。 @@ -49,6 +49,7 @@ - `20261020150000_desktop_project_deletion` - `20261023130000_personnel_salary_base` - `20261106110000_material_access_scope` +- `20261112154000_task_detail_zip_cleanup` `20261012_detail_generator` 仅新增个人详情页项目、私有文件、持久任务和品牌风格草案四张表及其约束／索引，不回填素材库或修改用户权限。正式验收前，启用配置仍只开放给当前有效超级管理员；私有文件目录独立持久挂载。 @@ -93,7 +94,7 @@ 1. 构建并固定同一 exact SHA/digest 的 runner 与 migrator； 2. legacy bootstrap `Audit`； 3. legacy bootstrap `Apply`，外部验收 503，确认旧容器停止且保留； -4. 在同一维护窗内运行本文的真实备份、隔离恢复、四十三条目标 migration 和迁移后审计； +4. 在同一维护窗内运行本文的真实备份、隔离恢复、四十四条目标 migration 和迁移后审计； 5. 仅在所有门禁通过后，运行普通同 SHA 应用部署。 顺序不得改为“先 migration，后 bootstrap”。本文备份工具的生产前后摘要若有变化，说明维护窗内仍可能存在写入，会直接失败且不生成 receipt。 @@ -419,7 +420,7 @@ docker run --rm \ "$MIGRATOR_IMAGE" --apply ``` -工具会在修改数据库前先验证审计目录可执行“新建→原子改名→删除”，再依次完成：核对镜像 SHA、配置端口与数据库实际端口、核对机器生成的 receipt 和完整备份产物、拒绝未知待执行/失败 migration、执行 `prisma migrate deploy`，最后核对四十三条目标 migration 的数据库 checksum、完成状态、关键人员组织表、素材空壳标记、钉钉流程快照表及需求类型和表单字段、员工花名册历史快照表、业绩转化经营数据快照及同步状态表、字段类型与可空性、检测报告原五个可空字段和独立检测日期字段、`api_keys` 三个可空加密列、`api_key_reveal_audits` 关键列、租约/需求类型/API Key reveal 审计索引的精确列顺序、`materials_uploadId_key` 的唯一性/btree/单列/有效就绪状态/active-only 谓词、Logo 默认值，以及 `materials.task_detail_demand_type` 无默认值、`materials.task_detail_private` / `material_permission_pack_items.can_access_private` 默认值为 `false`，并用稳定语义核对 `materials_private_optimization_check`，确认私密任务详情仍被精确限制在“任务-详情 + 优化”条件下，同时确认品牌知识预览切片列为非空文本数组且默认空数组；AI 模型五表、能力证据及用途验证约束／索引也须通过前述结构检查，最后原子写审计 JSON。 +工具会在修改数据库前先验证审计目录可执行“新建→原子改名→删除”，再依次完成：核对镜像 SHA、配置端口与数据库实际端口、核对机器生成的 receipt 和完整备份产物、拒绝未知待执行/失败 migration、执行 `prisma migrate deploy`，最后核对四十四条目标 migration 的数据库 checksum、完成状态、关键人员组织表、素材空壳标记、钉钉流程快照表及需求类型和表单字段、员工花名册历史快照表、业绩转化经营数据快照及同步状态表、字段类型与可空性、检测报告原五个可空字段和独立检测日期字段、`api_keys` 三个可空加密列、`api_key_reveal_audits` 关键列、租约/需求类型/API Key reveal 审计索引的精确列顺序、`materials_uploadId_key` 的唯一性/btree/单列/有效就绪状态/active-only 谓词、Logo 默认值，以及 `materials.task_detail_demand_type` 无默认值、`materials.task_detail_private` / `material_permission_pack_items.can_access_private` 默认值为 `false`，并用稳定语义核对 `materials_private_optimization_check`，确认私密任务详情仍被精确限制在“任务-详情 + 优化”条件下，同时确认品牌知识预览切片列为非空文本数组且默认空数组；AI 模型五表、能力证据及用途验证约束／索引也须通过前述结构检查，最后原子写审计 JSON。 apply 成功后立即固定审计文件本身；后续“迁移批次部署”读取这一个文件，不能手工重写： diff --git a/lib/__tests__/material-purge-worker.test.ts b/lib/__tests__/material-purge-worker.test.ts index 29d54324..f35065e0 100644 --- a/lib/__tests__/material-purge-worker.test.ts +++ b/lib/__tests__/material-purge-worker.test.ts @@ -4,6 +4,7 @@ import { prisma } from '@/lib/prisma' jest.mock('@/lib/prisma', () => ({ prisma: { $transaction: jest.fn() } })) jest.mock('@/lib/material-purge-files', () => ({})) +jest.mock('@/lib/task-detail-zip-cleanup', () => ({ processNextTaskDetailZipCleanup: jest.fn().mockResolvedValue(false) })) test('空队列检查期间收到入队信号，立即再检查，不丢失唤醒并等待轮询', async () => { jest.useFakeTimers() diff --git a/lib/__tests__/task-detail-asset-edit-batch.test.ts b/lib/__tests__/task-detail-asset-edit-batch.test.ts index 0dd065cc..7e2cf529 100644 --- a/lib/__tests__/task-detail-asset-edit-batch.test.ts +++ b/lib/__tests__/task-detail-asset-edit-batch.test.ts @@ -72,6 +72,7 @@ jest.mock('@/lib/prisma', () => ({ taskDetailChangeRequest: { updateMany: jest.fn(), }, + taskDetailZipCleanup: { create: jest.fn(), updateMany: jest.fn(), deleteMany: jest.fn() }, $queryRaw: jest.fn(), $transaction: jest.fn(), }, @@ -144,13 +145,25 @@ const material = { describe('task-detail asset edit batch data-plane', () => { let tempRoot: string let previousUploadRoot: string | undefined + let archiveReferences: { sources: string[]; derived: boolean }[] + let zipCleanupFiles: unknown[] beforeEach(async () => { jest.clearAllMocks() tempRoot = await mkdtemp(path.join(os.tmpdir(), 'task-detail-batch-')) previousUploadRoot = process.env.UPLOAD_DIR process.env.UPLOAD_DIR = tempRoot - mockReferenceQuery.mockReset().mockResolvedValue([]) + archiveReferences = [] + zipCleanupFiles = [] + mockReferenceQuery.mockReset().mockImplementation(async (sql: TemplateStringsArray) => ( + sql.join('').includes('UPDATE task_detail_zip_cleanups') + ? [{ id: 'zip-cleanup-1', files: zipCleanupFiles, attempts: 1 }] + : archiveReferences + )) + ;(prisma.taskDetailZipCleanup.create as jest.Mock).mockImplementation(async ({ data }) => { + zipCleanupFiles = data.files + return { id: 'zip-cleanup-1' } + }) mockCanManageMaterial.mockResolvedValue(true) mockCanAccessPrivateTaskDetail.mockResolvedValue(false) @@ -174,6 +187,7 @@ describe('task-detail asset edit batch data-plane', () => { materialAsset: prisma.materialAsset, materialAssetBarcode: prisma.materialAssetBarcode, taskDetailChangeRequest: prisma.taskDetailChangeRequest, + taskDetailZipCleanup: prisma.taskDetailZipCleanup, })) mockUploadPreview.mockImplementation(async (_buffer: Buffer, key: string) => `https://preview/${key}`) mockCreateZipFromFiles.mockResolvedValue({ @@ -465,7 +479,7 @@ describe('task-detail asset edit batch data-plane', () => { await writeFile(archivePath(name), name) } if (sharedAll) { - mockReferenceQuery.mockResolvedValue([{ sources: [archivePath('all-old.zip')], derived: false }]) + archiveReferences = [{ sources: [archivePath('all-old.zip')], derived: false }] } if (commitFails) { ;(prisma.$transaction as jest.Mock).mockImplementationOnce(async (callback) => { @@ -523,6 +537,10 @@ describe('task-detail asset edit batch data-plane', () => { return } await expect(editing).resolves.toMatchObject({ success: true, appliedCount: 3, entryCount: 3 }) + expect(zipCleanupFiles).toEqual([ + expect.objectContaining({ key: 'materials/sku-old.zip', fingerprint: expect.objectContaining({ size: 11 }) }), + expect.objectContaining({ key: 'materials/all-old.zip', fingerprint: expect.objectContaining({ size: 11 }) }), + ]) await expect(access(archivePath('sku-old.zip'))).rejects.toMatchObject({ code: 'ENOENT' }) if (sharedAll) await expect(readFile(archivePath('all-old.zip'), 'utf8')).resolves.toBe('all-old.zip') else await expect(access(archivePath('all-old.zip'))).rejects.toMatchObject({ code: 'ENOENT' }) diff --git a/lib/__tests__/task-detail-zip-cleanup.test.ts b/lib/__tests__/task-detail-zip-cleanup.test.ts new file mode 100644 index 00000000..4acccf6f --- /dev/null +++ b/lib/__tests__/task-detail-zip-cleanup.test.ts @@ -0,0 +1,81 @@ +/** @jest-environment node */ +import { findPurgeFileReferences, purgeFileIdentity, removePurgeFile, type PurgeFile } from '@/lib/material-purge-files' +import { prisma } from '@/lib/prisma' +import { deleteStoredUploadFileFromRemote } from '@/lib/remote-upload-sync' +import { processNextTaskDetailZipCleanup } from '@/lib/task-detail-zip-cleanup' + +jest.mock('@/lib/prisma', () => ({ + prisma: { + $queryRaw: jest.fn(), + taskDetailZipCleanup: { updateMany: jest.fn(), deleteMany: jest.fn() }, + }, +})) +jest.mock('@/lib/material-purge-files', () => ({ + ...jest.requireActual('@/lib/material-purge-files'), + findPurgeFileReferences: jest.fn(), + removePurgeFile: jest.fn(), +})) +jest.mock('@/lib/remote-upload-sync', () => ({ deleteStoredUploadFileFromRemote: jest.fn() })) + +function archive(name: string): PurgeFile { + return { storage: 'local', key: 'materials/' + name, source: 'uploads/materials/' + name, outcome: 'pending' } +} + +const claim = prisma.$queryRaw as jest.Mock +const references = findPurgeFileReferences as jest.Mock +const remove = removePurgeFile as jest.Mock +const deleteRemote = deleteStoredUploadFileFromRemote as jest.Mock + +beforeEach(() => { + jest.resetAllMocks() + references.mockResolvedValue(new Set()) + remove.mockResolvedValue(undefined) + deleteRemote.mockResolvedValue(undefined) +}) + +test('保留仍被引用的旧包，并重试已经删掉本地副本的远程清理', async () => { + const shared = archive('shared.zip') + const unused = archive('unused.zip') + const remoteRetry = archive('remote-retry.zip') + claim.mockResolvedValueOnce([{ id: 'cleanup-id', attempts: 1, files: [shared, unused, remoteRetry] }]) + references.mockResolvedValueOnce(new Set([purgeFileIdentity(shared)])) + deleteRemote.mockImplementationOnce(async () => undefined).mockRejectedValueOnce(new Error('remote temporarily unavailable')) + const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined) + try { + await expect(processNextTaskDetailZipCleanup('cleanup-id')).resolves.toBe(true) + expect(remove).not.toHaveBeenCalledWith(shared) + expect(remove).toHaveBeenCalledWith(unused) + expect(remove).toHaveBeenCalledWith(remoteRetry) + expect(prisma.taskDetailZipCleanup.updateMany).toHaveBeenCalledWith({ + where: { id: 'cleanup-id', attempts: 1 }, + data: { files: [shared, remoteRetry], nextAttemptAt: expect.any(Date) }, + }) + + claim.mockResolvedValueOnce([{ id: 'cleanup-id', attempts: 2, files: [shared, remoteRetry] }]) + await expect(processNextTaskDetailZipCleanup()).resolves.toBe(true) + expect(references).toHaveBeenLastCalledWith(prisma, [shared, remoteRetry]) + expect(remove).toHaveBeenCalledWith(shared) + expect(deleteRemote).toHaveBeenLastCalledWith(remoteRetry.source) + expect(prisma.taskDetailZipCleanup.deleteMany).toHaveBeenCalledWith({ where: { id: 'cleanup-id', attempts: 2 } }) + } finally { + warn.mockRestore() + } +}) + +test('引用查询失败时保留任务，跳过所有物理删除', async () => { + const file = archive('keep-until-verified.zip') + claim.mockResolvedValue([{ id: 'cleanup-id', attempts: 2, files: [file] }]) + references.mockRejectedValue(new Error('database unavailable')) + const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined) + try { + await expect(processNextTaskDetailZipCleanup()).resolves.toBe(true) + expect(remove).not.toHaveBeenCalled() + expect(deleteRemote).not.toHaveBeenCalled() + expect(prisma.taskDetailZipCleanup.updateMany).toHaveBeenCalledWith({ + where: { id: 'cleanup-id', attempts: 2 }, + data: { files: [file], nextAttemptAt: expect.any(Date) }, + }) + } finally { + warn.mockRestore() + } +}) diff --git a/lib/material-purge-files.ts b/lib/material-purge-files.ts index a78ae748..f6e61f9f 100644 --- a/lib/material-purge-files.ts +++ b/lib/material-purge-files.ts @@ -73,6 +73,14 @@ async function inspectLocalFile(key: string, directory = false) { return null } +export async function snapshotPurgeFile(file: Omit): Promise { + return { + ...file, + outcome: 'pending', + ...(file.storage === 'local' ? { fingerprint: await inspectLocalFile(file.key) } : {}), + } +} + export async function collectPurgeFiles(material: { id: string; previewUrl: string; zipFilePath: string; displayImages: string[] videoSourceUrl: string | null; videoPreview540Url: string | null @@ -118,10 +126,7 @@ export async function collectPurgeFiles(material: { const file = identifyPurgeFile(source) if (!file) continue const identity = purgeFileIdentity(file) - if (!files.has(identity)) files.set(identity, { - ...file, outcome: 'pending', - ...(file.storage === 'local' ? { fingerprint: await inspectLocalFile(file.key) } : {}), - }) + if (!files.has(identity)) files.set(identity, await snapshotPurgeFile(file)) if (files.size > MAX_FILES) throw new Error('素材文件过多，请联系管理员分批清理') } return Array.from(files.values()) diff --git a/lib/material-purge.ts b/lib/material-purge.ts index 79932353..761a8f75 100644 --- a/lib/material-purge.ts +++ b/lib/material-purge.ts @@ -2,6 +2,7 @@ import { invalidateMaterialFacetsCache } from '@/lib/material-facets-cache' import { randomUUID } from 'crypto' import { Prisma, type MaterialPurgeJob } from '@prisma/client' import { prisma } from '@/lib/prisma' +import { processNextTaskDetailZipCleanup } from '@/lib/task-detail-zip-cleanup' import { isRestrictedMaterial } from '@/lib/material-access-scope' import { buildMaterialPrivacyWhere, canAccessMaterialPrivacy, type TaskDetailPrivacyMaterial } from '@/lib/task-detail-privacy' import { collectPurgeFiles, findPurgeFileReferences, purgeFileIdentity, removePurgeFile, type PurgeFile } from '@/lib/material-purge-files' @@ -219,7 +220,14 @@ export function ensureMaterialPurgeWorkerStarted() { wakeRequested = false let more = false try { - for (let count = 0; count < 20 && await processNextMaterialPurge(); count += 1) more = count === 19 + for (let count = 0; count < 20; count += 1) { + const [purged, cleaned] = await Promise.all([ + processNextMaterialPurge(), + processNextTaskDetailZipCleanup(), + ]) + if (!purged && !cleaned) break + more = count === 19 + } } catch (error) { console.error('[MaterialPurge] 清理队列暂不可用，将在下一轮重试:', error) } finally { diff --git a/lib/task-detail-asset-edit.ts b/lib/task-detail-asset-edit.ts index d89c31d5..5ef02237 100644 --- a/lib/task-detail-asset-edit.ts +++ b/lib/task-detail-asset-edit.ts @@ -5,8 +5,9 @@ import { canManageMaterial, } from '@/lib/material-authorization' import { createMaterialLog } from '@/lib/material-log' -import { deleteFileIfUnreferenced, deleteLocalFile } from '@/lib/file-cleanup' -import { findPurgeFileReferences, identifyPurgeFile, purgeFileIdentity, type PurgeFile } from '@/lib/material-purge-files' +import { deleteFileIfUnreferenced } from '@/lib/file-cleanup' +import { identifyPurgeFile, purgeFileIdentity, snapshotPurgeFile, type PurgeFile } from '@/lib/material-purge-files' +import { processNextTaskDetailZipCleanup } from '@/lib/task-detail-zip-cleanup' import { acquireTaskMaterialCompletionLock, } from '@/lib/material-upload-cross-process-lock' @@ -2643,6 +2644,7 @@ async function commitRebuild( rebuilds: GroupRebuild[], newAllZip: RebuiltZip | null, operationLabel: EditLogLabel, + retiredZips: PurgeFile[], ) { const first = rebuilds[0] const { context } = first @@ -2657,7 +2659,7 @@ async function commitRebuild( }).find(Boolean) || '' } - await prisma.$transaction(async (tx) => { + const zipCleanupId = await prisma.$transaction(async (tx) => { for (const rebuild of rebuilds) { const { group } = rebuild.context await updateArchiveAsset(tx, context.materialId, group, context.groupZipAssets[group], rebuild.newGroupZip) @@ -2684,6 +2686,13 @@ async function commitRebuild( throw new TaskDetailAssetEditError('审批单状态已变化，正式素材未应用，请刷新后重试', 409) } } + + if (!retiredZips.length) return null + const job = await tx.taskDetailZipCleanup.create({ + data: { files: retiredZips as unknown as Prisma.InputJsonValue }, + select: { id: true }, + }) + return job.id }) await createMaterialLog({ @@ -2706,50 +2715,39 @@ async function commitRebuild( console.error('[task-detail-asset-edit] 素材日志写入失败:', error) }) - return { nextIsFullSet } + return { nextIsFullSet, zipCleanupId } } -async function cleanupReplacedZipFiles(storedPaths: string[]) { - const files = new Map() - for (const storedPath of storedPaths) { - try { - const file = identifyPurgeFile(storedPath) - if (file?.storage !== 'local' || !file.key.toLowerCase().endsWith('.zip')) continue - files.set(purgeFileIdentity(file), { ...file, outcome: 'pending' }) - } catch (error) { - console.warn('[task-detail-asset-edit] 无法识别旧压缩包路径:', storedPath, error) - } - } - - // 回收站中的素材仍可恢复，也会保留对 ZIP 的引用。 - const references = await findPurgeFileReferences(prisma, Array.from(files.values())) - await Promise.all(Array.from(files, async ([identity, file]) => { - if (references.has(identity)) return - const storedPath = buildStoredUploadPath(file.key) - try { - const removed = await deleteLocalFile(storedPath) - if (!removed.success) throw new Error(removed.error || '本地旧压缩包清理失败') - await deleteStoredUploadFileFromRemote(storedPath) - } catch (error) { - console.warn('[task-detail-asset-edit] 旧压缩包清理失败:', storedPath, error) - } - })) -} - -async function cleanupAfterCommit(rebuilds: GroupRebuild[], newAllZip: RebuiltZip | null) { +async function collectRetiredZips(rebuilds: GroupRebuild[], newAllZip: RebuiltZip | null): Promise { const { context } = rebuilds[0] - const staleStoredPaths = rebuilds.flatMap((item) => { + const previousPaths = rebuilds.flatMap((item) => { const zip = context.groupZipAssets[item.context.group] return zip && zip.fileUrl !== item.newGroupZip?.filePath ? [zip.fileUrl] : [] }) for (const previousPath of [context.allZipAsset?.fileUrl, context.currentZipFilePath]) { - if (previousPath && previousPath !== newAllZip?.filePath) staleStoredPaths.push(previousPath) + if (previousPath && previousPath !== newAllZip?.filePath) previousPaths.push(previousPath) } + const files = new Map() + for (const source of previousPaths) { + try { + const file = identifyPurgeFile(source) + if (file?.storage !== 'local' || !file.key.toLowerCase().endsWith('.zip')) continue + const identity = purgeFileIdentity(file) + if (!files.has(identity)) files.set(identity, await snapshotPurgeFile(file)) + } catch (error) { + console.warn('[task-detail-asset-edit] 无法核实旧压缩包:', source, error) + } + } + return Array.from(files.values()) +} + +async function cleanupAfterCommit(rebuilds: GroupRebuild[], zipCleanupId: string | null) { + const { context } = rebuilds[0] await Promise.allSettled([ - cleanupReplacedZipFiles(staleStoredPaths).catch((error) => { - console.warn('[task-detail-asset-edit] 旧压缩包引用检查失败:', error) - }), + zipCleanupId ? processNextTaskDetailZipCleanup(zipCleanupId).catch((error) => { + console.warn('[task-detail-asset-edit] 旧压缩包将由后台重试:', zipCleanupId, error) + }) : undefined, ...rebuilds.flatMap((item) => item.db.removedPreviewUrls.map(({ url }) => deleteFileIfUnreferenced(url, { excludeMaterialIds: [context.materialId] }), )), @@ -2780,7 +2778,7 @@ async function commitRebuildWithCleanup( const { context } = rebuilds[0] const byGroup = new Map(rebuilds.map((item) => [item.context.group, item])) let newAllZip: RebuiltZip | null = null - let result: { nextIsFullSet: boolean } + let result: Awaited> try { if (options.enforceLimits) assertFinalBatchLimits(context, rebuilds) const members = GROUPS.flatMap((group) => { @@ -2796,14 +2794,15 @@ async function commitRebuildWithCleanup( const zipOptions = options.fastZip ? [undefined, 1] as const : [] as const newAllZip = await createZipFromZips(members, context.materialId, context.productModel || 'product', '任务-详情', ...zipOptions) } + const retiredZips = await collectRetiredZips(rebuilds, newAllZip) options.assertHeld?.() - result = await commitRebuild(rebuilds, newAllZip, operationLabel) + result = await commitRebuild(rebuilds, newAllZip, operationLabel, retiredZips) } catch (error) { await cleanupUnappliedRebuilds(rebuilds, newAllZip) throw error } - await cleanupAfterCommit(rebuilds, newAllZip) - return result + await cleanupAfterCommit(rebuilds, result.zipCleanupId) + return { nextIsFullSet: result.nextIsFullSet } } async function runEditOperation(materialId: string, operation: (assertHeld: () => void) => Promise): Promise { diff --git a/lib/task-detail-zip-cleanup.ts b/lib/task-detail-zip-cleanup.ts new file mode 100644 index 00000000..e9645239 --- /dev/null +++ b/lib/task-detail-zip-cleanup.ts @@ -0,0 +1,63 @@ +import { Prisma } from '@prisma/client' +import { findPurgeFileReferences, purgeFileIdentity, removePurgeFile, type PurgeFile } from '@/lib/material-purge-files' +import { prisma } from '@/lib/prisma' +import { deleteStoredUploadFileFromRemote } from '@/lib/remote-upload-sync' +import { buildStoredUploadPath } from '@/lib/upload-paths' + +type ClaimedCleanup = { id: string; files: Prisma.JsonValue; attempts: number } + +const SHARED_FILE_RECHECK_MS = 60 * 60_000 +const MAX_FAILURE_RETRY_MS = 10 * 60_000 + +export async function processNextTaskDetailZipCleanup(id?: string): Promise { + // 领取和锁一起提交，崩溃后等待租期届满即可重试。 + const [job] = await prisma.$queryRaw` + WITH due AS ( + SELECT id FROM task_detail_zip_cleanups + WHERE next_attempt_at <= CURRENT_TIMESTAMP + ${id ? Prisma.sql`AND id = ${id}` : Prisma.empty} + ORDER BY next_attempt_at, id LIMIT 1 FOR UPDATE SKIP LOCKED + ) + UPDATE task_detail_zip_cleanups AS cleanup + SET attempts = cleanup.attempts + 1, next_attempt_at = CURRENT_TIMESTAMP + interval '10 minutes' + FROM due WHERE cleanup.id = due.id + RETURNING cleanup.id, cleanup.files, cleanup.attempts + ` + if (!job) return false + + const files = job.files as unknown as PurgeFile[] + let pending = files + let retryMs = SHARED_FILE_RECHECK_MS + const retryOnFailure = (error: unknown) => { + retryMs = Math.min(job.attempts * 30_000, MAX_FAILURE_RETRY_MS) + console.warn('[TaskDetailZipCleanup] 旧包清理将重试:', job.id, error) + } + try { + const references = await findPurgeFileReferences(prisma, files) + const results = await Promise.allSettled(files.map(async (file) => { + if (references.has(purgeFileIdentity(file))) return false + await removePurgeFile(file) + await deleteStoredUploadFileFromRemote(buildStoredUploadPath(file.key)) + return true + })) + pending = files.filter((_, index) => { + const result = results[index] + if (result.status === 'rejected') retryOnFailure(result.reason) + return result.status === 'rejected' || !result.value + }) + } catch (error) { + retryOnFailure(error) + } + + // 已被其他进程重新领取的任务由新的领取者写回。 + const where = { id: job.id, attempts: job.attempts } + if (pending.length) { + await prisma.taskDetailZipCleanup.updateMany({ + where, + data: { files: pending as unknown as Prisma.InputJsonValue, nextAttemptAt: new Date(Date.now() + retryMs) }, + }) + } else { + await prisma.taskDetailZipCleanup.deleteMany({ where }) + } + return true +} diff --git a/prisma/migrations/20261112154000_task_detail_zip_cleanup/migration.sql b/prisma/migrations/20261112154000_task_detail_zip_cleanup/migration.sql new file mode 100644 index 00000000..262098ca --- /dev/null +++ b/prisma/migrations/20261112154000_task_detail_zip_cleanup/migration.sql @@ -0,0 +1,15 @@ +BEGIN; + +CREATE TABLE "task_detail_zip_cleanups" ( + "id" TEXT NOT NULL, + "files" JSONB NOT NULL, + "attempts" INTEGER NOT NULL DEFAULT 0, + "next_attempt_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "task_detail_zip_cleanups_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX "task_detail_zip_cleanups_next_attempt_at_idx" + ON "task_detail_zip_cleanups"("next_attempt_at"); + +COMMIT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 77d3aefd..917cf368 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -569,6 +569,17 @@ model MaterialPurgeJob { @@map("material_purge_jobs") } +// 图片编辑事务提交后待清理的旧压缩包。 +model TaskDetailZipCleanup { + id String @id @default(uuid()) + files Json + attempts Int @default(0) + nextAttemptAt DateTime @default(now()) @map("next_attempt_at") + + @@index([nextAttemptAt]) + @@map("task_detail_zip_cleanups") +} + model MaterialPurgeAudit { id String @id @default(uuid()) jobId String @map("job_id") diff --git a/scripts/safe-production-migrate.mjs b/scripts/safe-production-migrate.mjs index 937686ed..9b804151 100644 --- a/scripts/safe-production-migrate.mjs +++ b/scripts/safe-production-migrate.mjs @@ -52,6 +52,7 @@ const TARGET_MIGRATIONS = Object.freeze([ '20261028090000_salary_position_configuration', '20261106160000_salary_personnel_grade', '20261106110000_material_access_scope', + '20261112154000_task_detail_zip_cleanup', ]) const VERIFIED_BACKUP_GENERATOR = 'scripts/create-verified-production-backup.mjs' const VERIFIED_BACKUP_MAINTENANCE_CONFIRMATION = 'NO_WRITES_DURING_SNAPSHOT' diff --git a/scripts/safe-production-migrate.test.mjs b/scripts/safe-production-migrate.test.mjs index d550e96f..9c34b518 100644 --- a/scripts/safe-production-migrate.test.mjs +++ b/scripts/safe-production-migrate.test.mjs @@ -756,6 +756,8 @@ test('target evidence rejects nullable logo columns and structurally wrong index for (const key of ['columns', 'indexes', 'constraints']) evidence[key].push(...organization[key], ...purge[key]) migrations.set('20261106110000_material_access_scope', { checksum: 'material-scope' }) evidence.migrationRows.push({ migration_name: '20261106110000_material_access_scope', checksum: 'material-scope', finished_at: 'now', rolled_back_at: null }) + migrations.set('20261112154000_task_detail_zip_cleanup', { checksum: 'task-detail-zip-cleanup' }) + evidence.migrationRows.push({ migration_name: '20261112154000_task_detail_zip_cleanup', checksum: 'task-detail-zip-cleanup', finished_at: 'now', rolled_back_at: null }) const scopeEvidence = materialScopeEvidence() evidence.columns.push(...scopeEvidence.columns) evidence.indexes.push(...scopeEvidence.indexes) @@ -1121,12 +1123,12 @@ test('container supply chain and migrator image are immutable while SOP keeps co assert.match(sop, /-v "\$HOST_RESULT_DIR:\/audit"/) assert.match(sop, /test -f "\$PREFLIGHT_AUDIT_FILE" && test ! -L "\$PREFLIGHT_AUDIT_FILE"/) assert.doesNotMatch(sop, /-v \/root\/gongxiangp2-migration-audit:\/audit/) - assert.match(sop, /四十三份目标 migration/) - assert.match(sop, /四十三条目标 migration/) + assert.match(sop, /四十四份目标 migration/) + assert.match(sop, /四十四条目标 migration/) const targetBlock = migratorSource.match(/const TARGET_MIGRATIONS = Object\.freeze\(\[([\s\S]*?)\]\)/)?.[1] assert.ok(targetBlock, 'Migrator target list must be inspectable') const targets = [...targetBlock.matchAll(/'([^']+)'/g)].map(match => match[1]) - assert.equal(targets.length, 43) + assert.equal(targets.length, 44) for (const migration of targets) assert.ok(sop.includes(`\`${migration}\``), `SOP must name ${migration}`) const applyMigration = migratorSource.indexOf("['migrate', 'deploy'")