fix(materials): retry cleanup of replaced task detail ZIPs
This commit is contained in:
parent
29980e697a
commit
be0d4fe7be
@ -2,7 +2,7 @@
|
||||
|
||||
## 结论
|
||||
|
||||
当前应用发布工具验证四十三份目标 migration。普通部署脚本**只读验证，不会自动修改数据库**。只有独立 migrator 在同时拿到精确 Git 提交、数据库身份和备份凭证后，才允许执行 `prisma migrate deploy`。目标名称以该精确提交的 `TARGET_MIGRATIONS` 为准，不能沿用历史批次的数量或凭证。
|
||||
当前应用发布工具验证四十四份目标 migration。普通部署脚本**只读验证，不会自动修改数据库**。只有独立 migrator 在同时拿到精确 Git 提交、数据库身份和备份凭证后，才允许执行 `prisma migrate deploy`。目标名称以该精确提交的 `TARGET_MIGRATIONS` 为准，不能沿用历史批次的数量或凭证。
|
||||
|
||||
`20261106110000_material_access_scope` 为所有素材增加默认 `general` 的使用范围，新增索引，并限制专属素材为运营中心 G 内共享。迁移不改变现有素材的可见范围；审计同时验证范围枚举和禁止叠加个人私密的数据库约束。
|
||||
|
||||
@ -49,6 +49,7 @@
|
||||
- `20261020150000_desktop_project_deletion`
|
||||
- `20261023130000_personnel_salary_base`
|
||||
- `20261106110000_material_access_scope`
|
||||
- `20261112154000_task_detail_zip_cleanup`
|
||||
|
||||
`20261012_detail_generator` 仅新增个人详情页项目、私有文件、持久任务和品牌风格草案四张表及其约束／索引，不回填素材库或修改用户权限。正式验收前，启用配置仍只开放给当前有效超级管理员；私有文件目录独立持久挂载。
|
||||
|
||||
@ -93,7 +94,7 @@
|
||||
1. 构建并固定同一 exact SHA/digest 的 runner 与 migrator；
|
||||
2. legacy bootstrap `Audit`；
|
||||
3. legacy bootstrap `Apply`，外部验收 503，确认旧容器停止且保留；
|
||||
4. 在同一维护窗内运行本文的真实备份、隔离恢复、四十三条目标 migration 和迁移后审计；
|
||||
4. 在同一维护窗内运行本文的真实备份、隔离恢复、四十四条目标 migration 和迁移后审计；
|
||||
5. 仅在所有门禁通过后，运行普通同 SHA 应用部署。
|
||||
|
||||
顺序不得改为“先 migration，后 bootstrap”。本文备份工具的生产前后摘要若有变化，说明维护窗内仍可能存在写入，会直接失败且不生成 receipt。
|
||||
@ -419,7 +420,7 @@ docker run --rm \
|
||||
"$MIGRATOR_IMAGE" --apply
|
||||
```
|
||||
|
||||
工具会在修改数据库前先验证审计目录可执行“新建→原子改名→删除”，再依次完成：核对镜像 SHA、配置端口与数据库实际端口、核对机器生成的 receipt 和完整备份产物、拒绝未知待执行/失败 migration、执行 `prisma migrate deploy`，最后核对四十三条目标 migration 的数据库 checksum、完成状态、关键人员组织表、素材空壳标记、钉钉流程快照表及需求类型和表单字段、员工花名册历史快照表、业绩转化经营数据快照及同步状态表、字段类型与可空性、检测报告原五个可空字段和独立检测日期字段、`api_keys` 三个可空加密列、`api_key_reveal_audits` 关键列、租约/需求类型/API Key reveal 审计索引的精确列顺序、`materials_uploadId_key` 的唯一性/btree/单列/有效就绪状态/active-only 谓词、Logo 默认值，以及 `materials.task_detail_demand_type` 无默认值、`materials.task_detail_private` / `material_permission_pack_items.can_access_private` 默认值为 `false`，并用稳定语义核对 `materials_private_optimization_check`，确认私密任务详情仍被精确限制在“任务-详情 + 优化”条件下，同时确认品牌知识预览切片列为非空文本数组且默认空数组；AI 模型五表、能力证据及用途验证约束／索引也须通过前述结构检查，最后原子写审计 JSON。
|
||||
工具会在修改数据库前先验证审计目录可执行“新建→原子改名→删除”，再依次完成：核对镜像 SHA、配置端口与数据库实际端口、核对机器生成的 receipt 和完整备份产物、拒绝未知待执行/失败 migration、执行 `prisma migrate deploy`，最后核对四十四条目标 migration 的数据库 checksum、完成状态、关键人员组织表、素材空壳标记、钉钉流程快照表及需求类型和表单字段、员工花名册历史快照表、业绩转化经营数据快照及同步状态表、字段类型与可空性、检测报告原五个可空字段和独立检测日期字段、`api_keys` 三个可空加密列、`api_key_reveal_audits` 关键列、租约/需求类型/API Key reveal 审计索引的精确列顺序、`materials_uploadId_key` 的唯一性/btree/单列/有效就绪状态/active-only 谓词、Logo 默认值，以及 `materials.task_detail_demand_type` 无默认值、`materials.task_detail_private` / `material_permission_pack_items.can_access_private` 默认值为 `false`，并用稳定语义核对 `materials_private_optimization_check`，确认私密任务详情仍被精确限制在“任务-详情 + 优化”条件下，同时确认品牌知识预览切片列为非空文本数组且默认空数组；AI 模型五表、能力证据及用途验证约束／索引也须通过前述结构检查，最后原子写审计 JSON。
|
||||
|
||||
apply 成功后立即固定审计文件本身；后续“迁移批次部署”读取这一个文件，不能手工重写：
|
||||
|
||||
|
||||
@ -4,6 +4,7 @@ import { prisma } from '@/lib/prisma'
|
||||
|
||||
jest.mock('@/lib/prisma', () => ({ prisma: { $transaction: jest.fn() } }))
|
||||
jest.mock('@/lib/material-purge-files', () => ({}))
|
||||
jest.mock('@/lib/task-detail-zip-cleanup', () => ({ processNextTaskDetailZipCleanup: jest.fn().mockResolvedValue(false) }))
|
||||
|
||||
test('空队列检查期间收到入队信号，立即再检查，不丢失唤醒并等待轮询', async () => {
|
||||
jest.useFakeTimers()
|
||||
|
||||
@ -72,6 +72,7 @@ jest.mock('@/lib/prisma', () => ({
|
||||
taskDetailChangeRequest: {
|
||||
updateMany: jest.fn(),
|
||||
},
|
||||
taskDetailZipCleanup: { create: jest.fn(), updateMany: jest.fn(), deleteMany: jest.fn() },
|
||||
$queryRaw: jest.fn(),
|
||||
$transaction: jest.fn(),
|
||||
},
|
||||
@ -144,13 +145,25 @@ const material = {
|
||||
describe('task-detail asset edit batch data-plane', () => {
|
||||
let tempRoot: string
|
||||
let previousUploadRoot: string | undefined
|
||||
let archiveReferences: { sources: string[]; derived: boolean }[]
|
||||
let zipCleanupFiles: unknown[]
|
||||
|
||||
beforeEach(async () => {
|
||||
jest.clearAllMocks()
|
||||
tempRoot = await mkdtemp(path.join(os.tmpdir(), 'task-detail-batch-'))
|
||||
previousUploadRoot = process.env.UPLOAD_DIR
|
||||
process.env.UPLOAD_DIR = tempRoot
|
||||
mockReferenceQuery.mockReset().mockResolvedValue([])
|
||||
archiveReferences = []
|
||||
zipCleanupFiles = []
|
||||
mockReferenceQuery.mockReset().mockImplementation(async (sql: TemplateStringsArray) => (
|
||||
sql.join('').includes('UPDATE task_detail_zip_cleanups')
|
||||
? [{ id: 'zip-cleanup-1', files: zipCleanupFiles, attempts: 1 }]
|
||||
: archiveReferences
|
||||
))
|
||||
;(prisma.taskDetailZipCleanup.create as jest.Mock).mockImplementation(async ({ data }) => {
|
||||
zipCleanupFiles = data.files
|
||||
return { id: 'zip-cleanup-1' }
|
||||
})
|
||||
|
||||
mockCanManageMaterial.mockResolvedValue(true)
|
||||
mockCanAccessPrivateTaskDetail.mockResolvedValue(false)
|
||||
@ -174,6 +187,7 @@ describe('task-detail asset edit batch data-plane', () => {
|
||||
materialAsset: prisma.materialAsset,
|
||||
materialAssetBarcode: prisma.materialAssetBarcode,
|
||||
taskDetailChangeRequest: prisma.taskDetailChangeRequest,
|
||||
taskDetailZipCleanup: prisma.taskDetailZipCleanup,
|
||||
}))
|
||||
mockUploadPreview.mockImplementation(async (_buffer: Buffer, key: string) => `https://preview/${key}`)
|
||||
mockCreateZipFromFiles.mockResolvedValue({
|
||||
@ -465,7 +479,7 @@ describe('task-detail asset edit batch data-plane', () => {
|
||||
await writeFile(archivePath(name), name)
|
||||
}
|
||||
if (sharedAll) {
|
||||
mockReferenceQuery.mockResolvedValue([{ sources: [archivePath('all-old.zip')], derived: false }])
|
||||
archiveReferences = [{ sources: [archivePath('all-old.zip')], derived: false }]
|
||||
}
|
||||
if (commitFails) {
|
||||
;(prisma.$transaction as jest.Mock).mockImplementationOnce(async (callback) => {
|
||||
@ -523,6 +537,10 @@ describe('task-detail asset edit batch data-plane', () => {
|
||||
return
|
||||
}
|
||||
await expect(editing).resolves.toMatchObject({ success: true, appliedCount: 3, entryCount: 3 })
|
||||
expect(zipCleanupFiles).toEqual([
|
||||
expect.objectContaining({ key: 'materials/sku-old.zip', fingerprint: expect.objectContaining({ size: 11 }) }),
|
||||
expect.objectContaining({ key: 'materials/all-old.zip', fingerprint: expect.objectContaining({ size: 11 }) }),
|
||||
])
|
||||
await expect(access(archivePath('sku-old.zip'))).rejects.toMatchObject({ code: 'ENOENT' })
|
||||
if (sharedAll) await expect(readFile(archivePath('all-old.zip'), 'utf8')).resolves.toBe('all-old.zip')
|
||||
else await expect(access(archivePath('all-old.zip'))).rejects.toMatchObject({ code: 'ENOENT' })
|
||||
|
||||
81
lib/__tests__/task-detail-zip-cleanup.test.ts
Normal file
81
lib/__tests__/task-detail-zip-cleanup.test.ts
Normal file
@ -0,0 +1,81 @@
|
||||
/** @jest-environment node */
|
||||
import { findPurgeFileReferences, purgeFileIdentity, removePurgeFile, type PurgeFile } from '@/lib/material-purge-files'
|
||||
import { prisma } from '@/lib/prisma'
|
||||
import { deleteStoredUploadFileFromRemote } from '@/lib/remote-upload-sync'
|
||||
import { processNextTaskDetailZipCleanup } from '@/lib/task-detail-zip-cleanup'
|
||||
|
||||
jest.mock('@/lib/prisma', () => ({
|
||||
prisma: {
|
||||
$queryRaw: jest.fn(),
|
||||
taskDetailZipCleanup: { updateMany: jest.fn(), deleteMany: jest.fn() },
|
||||
},
|
||||
}))
|
||||
jest.mock('@/lib/material-purge-files', () => ({
|
||||
...jest.requireActual('@/lib/material-purge-files'),
|
||||
findPurgeFileReferences: jest.fn(),
|
||||
removePurgeFile: jest.fn(),
|
||||
}))
|
||||
jest.mock('@/lib/remote-upload-sync', () => ({ deleteStoredUploadFileFromRemote: jest.fn() }))
|
||||
|
||||
function archive(name: string): PurgeFile {
|
||||
return { storage: 'local', key: 'materials/' + name, source: 'uploads/materials/' + name, outcome: 'pending' }
|
||||
}
|
||||
|
||||
const claim = prisma.$queryRaw as jest.Mock
|
||||
const references = findPurgeFileReferences as jest.Mock
|
||||
const remove = removePurgeFile as jest.Mock
|
||||
const deleteRemote = deleteStoredUploadFileFromRemote as jest.Mock
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks()
|
||||
references.mockResolvedValue(new Set())
|
||||
remove.mockResolvedValue(undefined)
|
||||
deleteRemote.mockResolvedValue(undefined)
|
||||
})
|
||||
|
||||
test('保留仍被引用的旧包，并重试已经删掉本地副本的远程清理', async () => {
|
||||
const shared = archive('shared.zip')
|
||||
const unused = archive('unused.zip')
|
||||
const remoteRetry = archive('remote-retry.zip')
|
||||
claim.mockResolvedValueOnce([{ id: 'cleanup-id', attempts: 1, files: [shared, unused, remoteRetry] }])
|
||||
references.mockResolvedValueOnce(new Set([purgeFileIdentity(shared)]))
|
||||
deleteRemote.mockImplementationOnce(async () => undefined).mockRejectedValueOnce(new Error('remote temporarily unavailable'))
|
||||
const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
try {
|
||||
await expect(processNextTaskDetailZipCleanup('cleanup-id')).resolves.toBe(true)
|
||||
expect(remove).not.toHaveBeenCalledWith(shared)
|
||||
expect(remove).toHaveBeenCalledWith(unused)
|
||||
expect(remove).toHaveBeenCalledWith(remoteRetry)
|
||||
expect(prisma.taskDetailZipCleanup.updateMany).toHaveBeenCalledWith({
|
||||
where: { id: 'cleanup-id', attempts: 1 },
|
||||
data: { files: [shared, remoteRetry], nextAttemptAt: expect.any(Date) },
|
||||
})
|
||||
|
||||
claim.mockResolvedValueOnce([{ id: 'cleanup-id', attempts: 2, files: [shared, remoteRetry] }])
|
||||
await expect(processNextTaskDetailZipCleanup()).resolves.toBe(true)
|
||||
expect(references).toHaveBeenLastCalledWith(prisma, [shared, remoteRetry])
|
||||
expect(remove).toHaveBeenCalledWith(shared)
|
||||
expect(deleteRemote).toHaveBeenLastCalledWith(remoteRetry.source)
|
||||
expect(prisma.taskDetailZipCleanup.deleteMany).toHaveBeenCalledWith({ where: { id: 'cleanup-id', attempts: 2 } })
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
test('引用查询失败时保留任务，跳过所有物理删除', async () => {
|
||||
const file = archive('keep-until-verified.zip')
|
||||
claim.mockResolvedValue([{ id: 'cleanup-id', attempts: 2, files: [file] }])
|
||||
references.mockRejectedValue(new Error('database unavailable'))
|
||||
const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
try {
|
||||
await expect(processNextTaskDetailZipCleanup()).resolves.toBe(true)
|
||||
expect(remove).not.toHaveBeenCalled()
|
||||
expect(deleteRemote).not.toHaveBeenCalled()
|
||||
expect(prisma.taskDetailZipCleanup.updateMany).toHaveBeenCalledWith({
|
||||
where: { id: 'cleanup-id', attempts: 2 },
|
||||
data: { files: [file], nextAttemptAt: expect.any(Date) },
|
||||
})
|
||||
} finally {
|
||||
warn.mockRestore()
|
||||
}
|
||||
})
|
||||
@ -73,6 +73,14 @@ async function inspectLocalFile(key: string, directory = false) {
|
||||
return null
|
||||
}
|
||||
|
||||
export async function snapshotPurgeFile(file: Omit<PurgeFile, 'outcome'>): Promise<PurgeFile> {
|
||||
return {
|
||||
...file,
|
||||
outcome: 'pending',
|
||||
...(file.storage === 'local' ? { fingerprint: await inspectLocalFile(file.key) } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
export async function collectPurgeFiles(material: {
|
||||
id: string; previewUrl: string; zipFilePath: string; displayImages: string[]
|
||||
videoSourceUrl: string | null; videoPreview540Url: string | null
|
||||
@ -118,10 +126,7 @@ export async function collectPurgeFiles(material: {
|
||||
const file = identifyPurgeFile(source)
|
||||
if (!file) continue
|
||||
const identity = purgeFileIdentity(file)
|
||||
if (!files.has(identity)) files.set(identity, {
|
||||
...file, outcome: 'pending',
|
||||
...(file.storage === 'local' ? { fingerprint: await inspectLocalFile(file.key) } : {}),
|
||||
})
|
||||
if (!files.has(identity)) files.set(identity, await snapshotPurgeFile(file))
|
||||
if (files.size > MAX_FILES) throw new Error('素材文件过多，请联系管理员分批清理')
|
||||
}
|
||||
return Array.from(files.values())
|
||||
|
||||
@ -2,6 +2,7 @@ import { invalidateMaterialFacetsCache } from '@/lib/material-facets-cache'
|
||||
import { randomUUID } from 'crypto'
|
||||
import { Prisma, type MaterialPurgeJob } from '@prisma/client'
|
||||
import { prisma } from '@/lib/prisma'
|
||||
import { processNextTaskDetailZipCleanup } from '@/lib/task-detail-zip-cleanup'
|
||||
import { isRestrictedMaterial } from '@/lib/material-access-scope'
|
||||
import { buildMaterialPrivacyWhere, canAccessMaterialPrivacy, type TaskDetailPrivacyMaterial } from '@/lib/task-detail-privacy'
|
||||
import { collectPurgeFiles, findPurgeFileReferences, purgeFileIdentity, removePurgeFile, type PurgeFile } from '@/lib/material-purge-files'
|
||||
@ -219,7 +220,14 @@ export function ensureMaterialPurgeWorkerStarted() {
|
||||
wakeRequested = false
|
||||
let more = false
|
||||
try {
|
||||
for (let count = 0; count < 20 && await processNextMaterialPurge(); count += 1) more = count === 19
|
||||
for (let count = 0; count < 20; count += 1) {
|
||||
const [purged, cleaned] = await Promise.all([
|
||||
processNextMaterialPurge(),
|
||||
processNextTaskDetailZipCleanup(),
|
||||
])
|
||||
if (!purged && !cleaned) break
|
||||
more = count === 19
|
||||
}
|
||||
}
|
||||
catch (error) { console.error('[MaterialPurge] 清理队列暂不可用，将在下一轮重试:', error) }
|
||||
finally {
|
||||
|
||||
@ -5,8 +5,9 @@ import {
|
||||
canManageMaterial,
|
||||
} from '@/lib/material-authorization'
|
||||
import { createMaterialLog } from '@/lib/material-log'
|
||||
import { deleteFileIfUnreferenced, deleteLocalFile } from '@/lib/file-cleanup'
|
||||
import { findPurgeFileReferences, identifyPurgeFile, purgeFileIdentity, type PurgeFile } from '@/lib/material-purge-files'
|
||||
import { deleteFileIfUnreferenced } from '@/lib/file-cleanup'
|
||||
import { identifyPurgeFile, purgeFileIdentity, snapshotPurgeFile, type PurgeFile } from '@/lib/material-purge-files'
|
||||
import { processNextTaskDetailZipCleanup } from '@/lib/task-detail-zip-cleanup'
|
||||
import {
|
||||
acquireTaskMaterialCompletionLock,
|
||||
} from '@/lib/material-upload-cross-process-lock'
|
||||
@ -2643,6 +2644,7 @@ async function commitRebuild(
|
||||
rebuilds: GroupRebuild[],
|
||||
newAllZip: RebuiltZip | null,
|
||||
operationLabel: EditLogLabel,
|
||||
retiredZips: PurgeFile[],
|
||||
) {
|
||||
const first = rebuilds[0]
|
||||
const { context } = first
|
||||
@ -2657,7 +2659,7 @@ async function commitRebuild(
|
||||
}).find(Boolean) || ''
|
||||
}
|
||||
|
||||
await prisma.$transaction(async (tx) => {
|
||||
const zipCleanupId = await prisma.$transaction(async (tx) => {
|
||||
for (const rebuild of rebuilds) {
|
||||
const { group } = rebuild.context
|
||||
await updateArchiveAsset(tx, context.materialId, group, context.groupZipAssets[group], rebuild.newGroupZip)
|
||||
@ -2684,6 +2686,13 @@ async function commitRebuild(
|
||||
throw new TaskDetailAssetEditError('审批单状态已变化，正式素材未应用，请刷新后重试', 409)
|
||||
}
|
||||
}
|
||||
|
||||
if (!retiredZips.length) return null
|
||||
const job = await tx.taskDetailZipCleanup.create({
|
||||
data: { files: retiredZips as unknown as Prisma.InputJsonValue },
|
||||
select: { id: true },
|
||||
})
|
||||
return job.id
|
||||
})
|
||||
|
||||
await createMaterialLog({
|
||||
@ -2706,50 +2715,39 @@ async function commitRebuild(
|
||||
console.error('[task-detail-asset-edit] 素材日志写入失败:', error)
|
||||
})
|
||||
|
||||
return { nextIsFullSet }
|
||||
return { nextIsFullSet, zipCleanupId }
|
||||
}
|
||||
|
||||
async function cleanupReplacedZipFiles(storedPaths: string[]) {
|
||||
const files = new Map<string, PurgeFile>()
|
||||
for (const storedPath of storedPaths) {
|
||||
try {
|
||||
const file = identifyPurgeFile(storedPath)
|
||||
if (file?.storage !== 'local' || !file.key.toLowerCase().endsWith('.zip')) continue
|
||||
files.set(purgeFileIdentity(file), { ...file, outcome: 'pending' })
|
||||
} catch (error) {
|
||||
console.warn('[task-detail-asset-edit] 无法识别旧压缩包路径:', storedPath, error)
|
||||
}
|
||||
}
|
||||
|
||||
// 回收站中的素材仍可恢复，也会保留对 ZIP 的引用。
|
||||
const references = await findPurgeFileReferences(prisma, Array.from(files.values()))
|
||||
await Promise.all(Array.from(files, async ([identity, file]) => {
|
||||
if (references.has(identity)) return
|
||||
const storedPath = buildStoredUploadPath(file.key)
|
||||
try {
|
||||
const removed = await deleteLocalFile(storedPath)
|
||||
if (!removed.success) throw new Error(removed.error || '本地旧压缩包清理失败')
|
||||
await deleteStoredUploadFileFromRemote(storedPath)
|
||||
} catch (error) {
|
||||
console.warn('[task-detail-asset-edit] 旧压缩包清理失败:', storedPath, error)
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
async function cleanupAfterCommit(rebuilds: GroupRebuild[], newAllZip: RebuiltZip | null) {
|
||||
async function collectRetiredZips(rebuilds: GroupRebuild[], newAllZip: RebuiltZip | null): Promise<PurgeFile[]> {
|
||||
const { context } = rebuilds[0]
|
||||
const staleStoredPaths = rebuilds.flatMap((item) => {
|
||||
const previousPaths = rebuilds.flatMap((item) => {
|
||||
const zip = context.groupZipAssets[item.context.group]
|
||||
return zip && zip.fileUrl !== item.newGroupZip?.filePath ? [zip.fileUrl] : []
|
||||
})
|
||||
for (const previousPath of [context.allZipAsset?.fileUrl, context.currentZipFilePath]) {
|
||||
if (previousPath && previousPath !== newAllZip?.filePath) staleStoredPaths.push(previousPath)
|
||||
if (previousPath && previousPath !== newAllZip?.filePath) previousPaths.push(previousPath)
|
||||
}
|
||||
|
||||
const files = new Map<string, PurgeFile>()
|
||||
for (const source of previousPaths) {
|
||||
try {
|
||||
const file = identifyPurgeFile(source)
|
||||
if (file?.storage !== 'local' || !file.key.toLowerCase().endsWith('.zip')) continue
|
||||
const identity = purgeFileIdentity(file)
|
||||
if (!files.has(identity)) files.set(identity, await snapshotPurgeFile(file))
|
||||
} catch (error) {
|
||||
console.warn('[task-detail-asset-edit] 无法核实旧压缩包:', source, error)
|
||||
}
|
||||
}
|
||||
return Array.from(files.values())
|
||||
}
|
||||
|
||||
async function cleanupAfterCommit(rebuilds: GroupRebuild[], zipCleanupId: string | null) {
|
||||
const { context } = rebuilds[0]
|
||||
await Promise.allSettled([
|
||||
cleanupReplacedZipFiles(staleStoredPaths).catch((error) => {
|
||||
console.warn('[task-detail-asset-edit] 旧压缩包引用检查失败:', error)
|
||||
}),
|
||||
zipCleanupId ? processNextTaskDetailZipCleanup(zipCleanupId).catch((error) => {
|
||||
console.warn('[task-detail-asset-edit] 旧压缩包将由后台重试:', zipCleanupId, error)
|
||||
}) : undefined,
|
||||
...rebuilds.flatMap((item) => item.db.removedPreviewUrls.map(({ url }) =>
|
||||
deleteFileIfUnreferenced(url, { excludeMaterialIds: [context.materialId] }),
|
||||
)),
|
||||
@ -2780,7 +2778,7 @@ async function commitRebuildWithCleanup(
|
||||
const { context } = rebuilds[0]
|
||||
const byGroup = new Map(rebuilds.map((item) => [item.context.group, item]))
|
||||
let newAllZip: RebuiltZip | null = null
|
||||
let result: { nextIsFullSet: boolean }
|
||||
let result: Awaited<ReturnType<typeof commitRebuild>>
|
||||
try {
|
||||
if (options.enforceLimits) assertFinalBatchLimits(context, rebuilds)
|
||||
const members = GROUPS.flatMap((group) => {
|
||||
@ -2796,14 +2794,15 @@ async function commitRebuildWithCleanup(
|
||||
const zipOptions = options.fastZip ? [undefined, 1] as const : [] as const
|
||||
newAllZip = await createZipFromZips(members, context.materialId, context.productModel || 'product', '任务-详情', ...zipOptions)
|
||||
}
|
||||
const retiredZips = await collectRetiredZips(rebuilds, newAllZip)
|
||||
options.assertHeld?.()
|
||||
result = await commitRebuild(rebuilds, newAllZip, operationLabel)
|
||||
result = await commitRebuild(rebuilds, newAllZip, operationLabel, retiredZips)
|
||||
} catch (error) {
|
||||
await cleanupUnappliedRebuilds(rebuilds, newAllZip)
|
||||
throw error
|
||||
}
|
||||
await cleanupAfterCommit(rebuilds, newAllZip)
|
||||
return result
|
||||
await cleanupAfterCommit(rebuilds, result.zipCleanupId)
|
||||
return { nextIsFullSet: result.nextIsFullSet }
|
||||
}
|
||||
|
||||
async function runEditOperation<T>(materialId: string, operation: (assertHeld: () => void) => Promise<T>): Promise<T> {
|
||||
|
||||
63
lib/task-detail-zip-cleanup.ts
Normal file
63
lib/task-detail-zip-cleanup.ts
Normal file
@ -0,0 +1,63 @@
|
||||
import { Prisma } from '@prisma/client'
|
||||
import { findPurgeFileReferences, purgeFileIdentity, removePurgeFile, type PurgeFile } from '@/lib/material-purge-files'
|
||||
import { prisma } from '@/lib/prisma'
|
||||
import { deleteStoredUploadFileFromRemote } from '@/lib/remote-upload-sync'
|
||||
import { buildStoredUploadPath } from '@/lib/upload-paths'
|
||||
|
||||
type ClaimedCleanup = { id: string; files: Prisma.JsonValue; attempts: number }
|
||||
|
||||
const SHARED_FILE_RECHECK_MS = 60 * 60_000
|
||||
const MAX_FAILURE_RETRY_MS = 10 * 60_000
|
||||
|
||||
export async function processNextTaskDetailZipCleanup(id?: string): Promise<boolean> {
|
||||
// 领取和锁一起提交，崩溃后等待租期届满即可重试。
|
||||
const [job] = await prisma.$queryRaw<ClaimedCleanup[]>`
|
||||
WITH due AS (
|
||||
SELECT id FROM task_detail_zip_cleanups
|
||||
WHERE next_attempt_at <= CURRENT_TIMESTAMP
|
||||
${id ? Prisma.sql`AND id = ${id}` : Prisma.empty}
|
||||
ORDER BY next_attempt_at, id LIMIT 1 FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE task_detail_zip_cleanups AS cleanup
|
||||
SET attempts = cleanup.attempts + 1, next_attempt_at = CURRENT_TIMESTAMP + interval '10 minutes'
|
||||
FROM due WHERE cleanup.id = due.id
|
||||
RETURNING cleanup.id, cleanup.files, cleanup.attempts
|
||||
`
|
||||
if (!job) return false
|
||||
|
||||
const files = job.files as unknown as PurgeFile[]
|
||||
let pending = files
|
||||
let retryMs = SHARED_FILE_RECHECK_MS
|
||||
const retryOnFailure = (error: unknown) => {
|
||||
retryMs = Math.min(job.attempts * 30_000, MAX_FAILURE_RETRY_MS)
|
||||
console.warn('[TaskDetailZipCleanup] 旧包清理将重试:', job.id, error)
|
||||
}
|
||||
try {
|
||||
const references = await findPurgeFileReferences(prisma, files)
|
||||
const results = await Promise.allSettled(files.map(async (file) => {
|
||||
if (references.has(purgeFileIdentity(file))) return false
|
||||
await removePurgeFile(file)
|
||||
await deleteStoredUploadFileFromRemote(buildStoredUploadPath(file.key))
|
||||
return true
|
||||
}))
|
||||
pending = files.filter((_, index) => {
|
||||
const result = results[index]
|
||||
if (result.status === 'rejected') retryOnFailure(result.reason)
|
||||
return result.status === 'rejected' || !result.value
|
||||
})
|
||||
} catch (error) {
|
||||
retryOnFailure(error)
|
||||
}
|
||||
|
||||
// 已被其他进程重新领取的任务由新的领取者写回。
|
||||
const where = { id: job.id, attempts: job.attempts }
|
||||
if (pending.length) {
|
||||
await prisma.taskDetailZipCleanup.updateMany({
|
||||
where,
|
||||
data: { files: pending as unknown as Prisma.InputJsonValue, nextAttemptAt: new Date(Date.now() + retryMs) },
|
||||
})
|
||||
} else {
|
||||
await prisma.taskDetailZipCleanup.deleteMany({ where })
|
||||
}
|
||||
return true
|
||||
}
|
||||
@ -0,0 +1,15 @@
|
||||
BEGIN;
|
||||
|
||||
CREATE TABLE "task_detail_zip_cleanups" (
|
||||
"id" TEXT NOT NULL,
|
||||
"files" JSONB NOT NULL,
|
||||
"attempts" INTEGER NOT NULL DEFAULT 0,
|
||||
"next_attempt_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "task_detail_zip_cleanups_pkey" PRIMARY KEY ("id")
|
||||
);
|
||||
|
||||
CREATE INDEX "task_detail_zip_cleanups_next_attempt_at_idx"
|
||||
ON "task_detail_zip_cleanups"("next_attempt_at");
|
||||
|
||||
COMMIT;
|
||||
@ -569,6 +569,17 @@ model MaterialPurgeJob {
|
||||
@@map("material_purge_jobs")
|
||||
}
|
||||
|
||||
// 图片编辑事务提交后待清理的旧压缩包。
|
||||
model TaskDetailZipCleanup {
|
||||
id String @id @default(uuid())
|
||||
files Json
|
||||
attempts Int @default(0)
|
||||
nextAttemptAt DateTime @default(now()) @map("next_attempt_at")
|
||||
|
||||
@@index([nextAttemptAt])
|
||||
@@map("task_detail_zip_cleanups")
|
||||
}
|
||||
|
||||
model MaterialPurgeAudit {
|
||||
id String @id @default(uuid())
|
||||
jobId String @map("job_id")
|
||||
|
||||
@ -52,6 +52,7 @@ const TARGET_MIGRATIONS = Object.freeze([
|
||||
'20261028090000_salary_position_configuration',
|
||||
'20261106160000_salary_personnel_grade',
|
||||
'20261106110000_material_access_scope',
|
||||
'20261112154000_task_detail_zip_cleanup',
|
||||
])
|
||||
const VERIFIED_BACKUP_GENERATOR = 'scripts/create-verified-production-backup.mjs'
|
||||
const VERIFIED_BACKUP_MAINTENANCE_CONFIRMATION = 'NO_WRITES_DURING_SNAPSHOT'
|
||||
|
||||
@ -756,6 +756,8 @@ test('target evidence rejects nullable logo columns and structurally wrong index
|
||||
for (const key of ['columns', 'indexes', 'constraints']) evidence[key].push(...organization[key], ...purge[key])
|
||||
migrations.set('20261106110000_material_access_scope', { checksum: 'material-scope' })
|
||||
evidence.migrationRows.push({ migration_name: '20261106110000_material_access_scope', checksum: 'material-scope', finished_at: 'now', rolled_back_at: null })
|
||||
migrations.set('20261112154000_task_detail_zip_cleanup', { checksum: 'task-detail-zip-cleanup' })
|
||||
evidence.migrationRows.push({ migration_name: '20261112154000_task_detail_zip_cleanup', checksum: 'task-detail-zip-cleanup', finished_at: 'now', rolled_back_at: null })
|
||||
const scopeEvidence = materialScopeEvidence()
|
||||
evidence.columns.push(...scopeEvidence.columns)
|
||||
evidence.indexes.push(...scopeEvidence.indexes)
|
||||
@ -1121,12 +1123,12 @@ test('container supply chain and migrator image are immutable while SOP keeps co
|
||||
assert.match(sop, /-v "\$HOST_RESULT_DIR:\/audit"/)
|
||||
assert.match(sop, /test -f "\$PREFLIGHT_AUDIT_FILE" && test ! -L "\$PREFLIGHT_AUDIT_FILE"/)
|
||||
assert.doesNotMatch(sop, /-v \/root\/gongxiangp2-migration-audit:\/audit/)
|
||||
assert.match(sop, /四十三份目标 migration/)
|
||||
assert.match(sop, /四十三条目标 migration/)
|
||||
assert.match(sop, /四十四份目标 migration/)
|
||||
assert.match(sop, /四十四条目标 migration/)
|
||||
const targetBlock = migratorSource.match(/const TARGET_MIGRATIONS = Object\.freeze\(\[([\s\S]*?)\]\)/)?.[1]
|
||||
assert.ok(targetBlock, 'Migrator target list must be inspectable')
|
||||
const targets = [...targetBlock.matchAll(/'([^']+)'/g)].map(match => match[1])
|
||||
assert.equal(targets.length, 43)
|
||||
assert.equal(targets.length, 44)
|
||||
for (const migration of targets) assert.ok(sop.includes(`\`${migration}\``), `SOP must name ${migration}`)
|
||||
|
||||
const applyMigration = migratorSource.indexOf("['migrate', 'deploy'")
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user