diff --git a/Dockerfile b/Dockerfile index 92501d4..5332549 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,7 +103,8 @@ RUN test -s "/app/public/brand/header-logo.png" && \ test -f "/app/prisma/migrations/20260923_add_certificate_report_metadata/migration.sql" && \ test -f "/app/prisma/migrations/20260924_add_certificate_tested_on/migration.sql" RUN test -f "/app/prisma/migrations/20261008_permission_group_department_identity/migration.sql" && \ - test -f "/app/prisma/migrations/20261008_team_function_configs/migration.sql" + test -f "/app/prisma/migrations/20261008_team_function_configs/migration.sql" && \ + test -f "/app/prisma/migrations/20261009_material_purge_outbox/migration.sql" # 这两个动态路由负责兜底访问本地上传文件；如果它们在构建上下文里丢失， # 线上音频/视频就会直接 404。这里提前失败，避免再把缺路由的镜像发出去。 diff --git a/app/admin/page.tsx b/app/admin/page.tsx index 9f2705c..accbec1 100644 --- a/app/admin/page.tsx +++ b/app/admin/page.tsx @@ -10,6 +10,7 @@ import MaterialManagement from '@/components/admin/MaterialManagement' import CategoryManagement from '@/components/admin/CategoryManagement' import DataStatisticsTab from '@/components/admin/DataStatisticsTab' import PublicTrafficTab from '@/components/admin/PublicTrafficTab' +import ToastBubble from '@/components/ToastBubble' // 类型定义 interface Material { @@ -266,7 +267,7 @@ export default function AdminPage() { }, [isLoadingMoreMaterials, hasMoreMaterials, currentMaterialsPage, showRecycleBin, materialSearchTerm]) // 加载状态 - if (status === 'loading' || loading) { + if (status === 'loading' || (loading && activeTab !== 'materials')) { return (
加载中...
@@ -284,13 +285,7 @@ export default function AdminPage() { {/* Toast通知 */} {toast && ( -
- {toast.message} -
+ )}
@@ -410,6 +405,7 @@ export default function AdminPage() { {/* 素材管理 Tab */} {canManageSystemTabs && activeTab === 'materials' && ( 100) { return NextResponse.json({ error: '请提供要删除的素材ID列表' }, { status: 400 }) } @@ -46,11 +50,10 @@ export async function POST(request: NextRequest) { }, { status: 400 }) } - if (permanent) { - return NextResponse.json({ - error: '批量永久删除已暂停：请使用可恢复删除；待持久化清理任务上线后再开放物理删除', - code: 'PERMANENT_DELETE_DISABLED', - }, { status: 409 }) + if (permanent === true) { + const jobs = await enqueueMaterialPurge(ids, operatorId) + ensureMaterialPurgeWorkerStarted() + return NextResponse.json({ success: true, jobs }, { status: 202 }) } else { // ========== 软删除：仅标记 deletedAt ========== @@ -124,6 +127,7 @@ export async function POST(request: NextRequest) { }) } } catch (error) { + if (error instanceof MaterialPurgeError) return NextResponse.json({ error: error.message }, { status: error.status }) console.error('[BatchDelete] Error:', error) return NextResponse.json({ error: '批量删除失败' }, { status: 500 }) } diff --git a/app/api/admin/materials/purge-jobs/route.ts b/app/api/admin/materials/purge-jobs/route.ts new file mode 100644 index 0000000..077d8cb --- /dev/null +++ b/app/api/admin/materials/purge-jobs/route.ts @@ -0,0 +1,46 @@ +import { NextRequest, NextResponse } from 'next/server' +import { getServerSession } from 'next-auth' +import { authOptions } from '@/lib/auth' +import { isSystemAdmin } from '@/lib/department-permission' +import { prisma } from '@/lib/prisma' +import { ensureMaterialPurgeWorkerStarted, MaterialPurgeError, retryMaterialPurge, toPurgeJobView } from '@/lib/material-purge' + +export const dynamic = 'force-dynamic' + +async function operator() { + const session = await getServerSession(authOptions) + if (!session?.user || !(session.user as any).id) return null + return isSystemAdmin((session.user as any).role) ? (session.user as any).id as string : null +} + +export async function GET(request: NextRequest) { + if (!await operator()) return NextResponse.json({ error: '无权限' }, { status: 403 }) + const ids = request.nextUrl.searchParams.getAll('id') + if (ids.length > 100 || ids.some(id => !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id))) { + return NextResponse.json({ error: '清理任务无效' }, { status: 400 }) + } + try { + const [pending, tracked] = await Promise.all([ + prisma.materialPurgeJob.findMany({ where: { status: { not: 'completed' } }, orderBy: { createdAt: 'asc' }, take: 100 }), + ids.length ? prisma.materialPurgeJob.findMany({ where: { id: { in: ids } } }) : Promise.resolve([]), + ]) + const jobs = [...tracked, ...pending.filter(job => !tracked.some(item => item.id === job.id))] + return NextResponse.json({ jobs: jobs.map(toPurgeJobView) }, { headers: { 'Cache-Control': 'no-store' } }) + } catch { + return NextResponse.json({ error: '暂时无法读取清理进度，请重试' }, { status: 503 }) + } +} + +export async function POST(request: NextRequest) { + const operatorId = await operator() + if (!operatorId) return NextResponse.json({ error: '无权限' }, { status: 403 }) + try { + const { jobId } = await request.json() + const job = await retryMaterialPurge(jobId, operatorId) + ensureMaterialPurgeWorkerStarted() + return NextResponse.json({ job }, { status: 202 }) + } catch (error) { + if (error instanceof MaterialPurgeError) return NextResponse.json({ error: error.message }, { status: error.status }) + return NextResponse.json({ error: '暂时无法重试，请稍后再试' }, { status: 503 }) + } +} diff --git a/app/api/materials/route.ts b/app/api/materials/route.ts index 00152b2..0e4b2a6 100644 --- a/app/api/materials/route.ts +++ b/app/api/materials/route.ts @@ -5,6 +5,7 @@ import { authOptions } from '@/lib/auth'; import { prisma } from '@/lib/prisma'; import { isSystemAdmin } from '@/lib/department-permission'; import { createMaterialLog } from '@/lib/material-log'; +import { enqueueMaterialPurge, ensureMaterialPurgeWorkerStarted, MaterialPurgeError } from '@/lib/material-purge'; import { normalizeUrl, normalizeMaterial } from '@/lib/utils'; import { rewriteMediaUrlForClient } from '@/lib/url-rewrite'; import { detectIsIntranet } from '@/lib/network-detection'; @@ -926,6 +927,12 @@ export async function DELETE(request: NextRequest) { ); } + if (permanent) { + const jobs = await enqueueMaterialPurge([id], (session.user as any).id); + ensureMaterialPurgeWorkerStarted(); + return NextResponse.json({ success: true, jobs }, { status: 202 }); + } + // 查询素材 const material = await prisma.material.findUnique({ where: { id }, @@ -988,14 +995,8 @@ export async function DELETE(request: NextRequest) { return NextResponse.json({ message: '删除成功（可恢复），文件已保留' }); } - // 永久删除 - { - return NextResponse.json({ - error: '永久删除已暂停：请先使用可恢复删除；待持久化清理任务上线后再开放物理删除', - code: 'PERMANENT_DELETE_DISABLED', - }, { status: 409 }); - } } catch (error: any) { + if (error instanceof MaterialPurgeError) return NextResponse.json({ error: error.message }, { status: error.status }); // 缺少软删除列时必须失败关闭；绝不能悄悄退化成不可恢复的物理删除。 if (error?.meta?.column?.includes('deletedAt')) { console.error('[Materials API] deletedAt field is unavailable; refusing destructive fallback'); diff --git a/components/ToastBubble.tsx b/components/ToastBubble.tsx index 63568cc..2e626e3 100644 --- a/components/ToastBubble.tsx +++ b/components/ToastBubble.tsx @@ -16,24 +16,25 @@ export default function ToastBubble({ zIndexClass = 'z-[200]', }: ToastBubbleProps) { return ( -
+
{variant === 'success' ? ( - + ) : ( - + )} - {message} + {message}
) diff --git a/components/admin/MaterialManagement.tsx b/components/admin/MaterialManagement.tsx index e718c51..b8275bc 100644 --- a/components/admin/MaterialManagement.tsx +++ b/components/admin/MaterialManagement.tsx @@ -10,6 +10,8 @@ import { useRef, useEffect, useCallback, useDeferredValue, useState } from 'react' import { useRouter } from 'next/navigation' +import { useConfirmActionDialog, type ConfirmActionOptions } from '@/components/ConfirmActionDialog' +import { useMaterialPurgeJobs } from '@/components/admin/MaterialPurgeStatus' import { format } from 'date-fns' import type { Material } from '@/hooks/useAdminState' import { parseFetchError, handleError } from '@/lib/error-handler' @@ -20,6 +22,7 @@ import { isVideoMaterialType } from '@/lib/video-material-types' const SEARCH_PAGE_SIZE = 50 interface MaterialManagementProps { + isLoadingMaterials?: boolean materials: Material[] setMaterials: React.Dispatch> selectedMaterials: Set @@ -41,6 +44,7 @@ interface MaterialManagementProps { } export default function MaterialManagement({ + isLoadingMaterials = false, materials, setMaterials, selectedMaterials, @@ -70,7 +74,35 @@ export default function MaterialManagement({ const [totalSearchCount, setTotalSearchCount] = useState(0) // 批量操作加载状态 - const [isBatchProcessing, setIsBatchProcessing] = useState(false) + const [actionProcessing, setIsBatchProcessing] = useState(false) + const isBatchProcessing = actionProcessing || isLoadingMaterials + const { confirmAction, confirmActionDialog } = useConfirmActionDialog() + const { track: trackPurgeJobs, status: purgeStatus } = useMaterialPurgeJobs() + const actionOpen = useRef(false) + + const confirmMaterialAction = async (options: ConfirmActionOptions) => { + if (actionOpen.current) return + actionOpen.current = true + try { + await confirmAction({ ...options, action: async reason => { + setIsBatchProcessing(true) + try { await options.action?.(reason) } + catch (error) { throw new Error(handleError(error).message) } + finally { setIsBatchProcessing(false) } + } }) + } finally { actionOpen.current = false } + } + + const removeRows = (ids: string[]) => { + setMaterials(previous => previous.filter(material => !ids.includes(material.id))) + setSelectedMaterials(previous => new Set(Array.from(previous).filter(id => !ids.includes(id)))) + void fetchData() + } + + const requireSuccess = async (response: Response) => { + if (!response.ok) throw new Error((await parseFetchError(response)).message) + return response.json() + } // 使用 useDeferredValue 替代手动防抖，与前台保持一致 const deferredSearchKeyword = useDeferredValue(materialSearchTerm) @@ -189,75 +221,33 @@ export default function MaterialManagement({ } }, [isLoadingMoreSearch, hasMoreSearchResults, isInSearchMode, searchPage, showRecycleBin, deferredSearchKeyword, setMaterials, setDisplayedMaterialsCount]) - // 删除单个素材 + // 删除操作在共享确认框内执行，失败保留上下文和选择。 const handleDeleteMaterial = async (id: string) => { - if (!confirm('确定要删除这个素材吗？')) return - - try { - const response = await fetch(`/api/materials?id=${id}`, { - method: 'DELETE', - }) - - if (response.ok) { - setMaterials(prev => prev.filter(m => m.id !== id)) - showToast('删除成功') - } else { - const error = await parseFetchError(response) - showToast(error.message, 'error') - } - } catch (error) { - console.error('Error deleting material:', error) - const { message } = handleError(error) - showToast(message, 'error') - } + await confirmMaterialAction({ title: '移入回收站？', description: '素材之后可以恢复。', confirmLabel: '移入回收站', tone: 'danger', action: async () => { + await requireSuccess(await fetch(`/api/materials?id=${id}`, { method: 'DELETE' })) + removeRows([id]) + showToast('已移入回收站') + } }) } - // 批量删除素材 const handleBatchDeleteMaterials = async () => { const selectedIds = Array.from(selectedMaterials) - if (selectedIds.length === 0) { - showToast('请先选择要删除的素材', 'error') - return - } - - const confirmMessage = showRecycleBin - ? `确定要永久删除选中的 ${selectedIds.length} 条素材吗？此操作不可恢复！` - : `确定要删除选中的 ${selectedIds.length} 条素材吗？素材将移至回收站。` - - if (!confirm(confirmMessage)) return - - setIsBatchProcessing(true) - try { - const response = await fetch('/api/admin/materials/batch-delete', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - ids: selectedIds, - permanent: showRecycleBin, - }), - }) - - if (response.ok) { - const result = await response.json() - setMaterials(prev => prev.filter(m => !selectedIds.includes(m.id))) - setSelectedMaterials(new Set()) - // 显示详细信息（如果是永久删除，显示文件清理信息） - if (showRecycleBin && result.details) { - showToast(`批量永久删除完成：删除 ${result.details.deletedRecords} 条记录，清理 ${result.details.filesDeleted} 个文件`) - } else { - showToast(showRecycleBin ? '批量永久删除完成' : '批量删除完成') - } - } else { - const error = await parseFetchError(response) - showToast(error.message, 'error') - } - } catch (error) { - console.error('Batch delete materials error:', error) - const { message } = handleError(error) - showToast(message, 'error') - } finally { - setIsBatchProcessing(false) - } + if (!selectedIds.length) return + const permanent = showRecycleBin + await confirmMaterialAction({ + title: permanent ? `彻底删除 ${selectedIds.length} 条素材？` : `将 ${selectedIds.length} 条素材移入回收站？`, + description: permanent ? '删除后无法恢复，关联文件将一并清理。' : '素材之后可以恢复。', + confirmLabel: permanent ? '彻底删除' : '移入回收站', loadingLabel: '正在提交…', tone: 'danger', + action: async () => { + const result = await requireSuccess(await fetch('/api/admin/materials/batch-delete', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ids: selectedIds, permanent }), + })) + if (permanent) trackPurgeJobs(result.jobs) + removeRows(selectedIds) + if (!permanent) showToast(`已将 ${selectedIds.length} 条素材移入回收站`) + }, + }) } // 切换素材选择状态 @@ -292,54 +282,45 @@ export default function MaterialManagement({ } } - // 恢复素材 const handleRestoreMaterial = async (id: string) => { - if (!confirm('确定要恢复该素材吗？')) return - - try { - const response = await fetch('/api/materials/restore', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ id }) - }) - - if (response.ok) { - setMaterials(prev => prev.filter(m => m.id !== id)) - showToast('恢复成功') - fetchData() - } else { - const error = await parseFetchError(response) - showToast(error.message, 'error') - } - } catch (error) { - console.error('Restore material error:', error) - const { message } = handleError(error) - showToast(message, 'error') - } + await confirmMaterialAction({ title: '恢复素材？', confirmLabel: '恢复', action: async () => { + await requireSuccess(await fetch('/api/materials/restore', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ id }), + })) + removeRows([id]) + showToast('已恢复素材') + } }) } - // 永久删除素材 const handlePermanentDelete = async (id: string) => { - if (!confirm('确定要永久删除该素材吗？此操作不可恢复！')) return + await confirmMaterialAction({ title: '彻底删除这条素材？', description: '删除后无法恢复，关联文件将一并清理。', + confirmLabel: '彻底删除', loadingLabel: '正在提交…', tone: 'danger', action: async () => { + const result = await requireSuccess(await fetch(`/api/materials?id=${id}&permanent=true`, { method: 'DELETE' })) + trackPurgeJobs(result.jobs) + removeRows([id]) + }, + }) + } - try { - const response = await fetch(`/api/materials?id=${id}&permanent=true`, { - method: 'DELETE' - }) - - if (response.ok) { - setMaterials(prev => prev.filter(m => m.id !== id)) - showToast('永久删除成功') - fetchData() - } else { - const error = await parseFetchError(response) - showToast(error.message, 'error') + const handleBatchRestore = async () => { + let remaining = Array.from(selectedMaterials) + const originalCount = remaining.length + await confirmMaterialAction({ title: `恢复 ${originalCount} 条素材？`, confirmLabel: '恢复', action: async () => { + const restored: string[] = [] + let message = '请稍后重试' + for (const id of remaining) { + try { + await requireSuccess(await fetch('/api/materials/restore', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ id }), + })) + restored.push(id) + } catch (error) { message = handleError(error).message } } - } catch (error) { - console.error('Permanent delete error:', error) - const { message } = handleError(error) - showToast(message, 'error') - } + remaining = remaining.filter(id => !restored.includes(id)) + removeRows(restored) + if (remaining.length) throw new Error(`已恢复 ${originalCount - remaining.length} 条，${remaining.length} 条未恢复：${message}`) + showToast(`已恢复 ${originalCount} 条素材`) + } }) } // 获取素材类型角标颜色 @@ -399,7 +380,7 @@ export default function MaterialManagement({ }, [isInSearchMode, hasMoreSearchResults, isLoadingMoreSearch, loadMoreSearchResults, hasMoreMaterials, isLoadingMoreMaterials, loadMoreMaterials]) return ( -
+
{/* 顶部工具栏：状态切换 + 操作 + 搜索框 */}
@@ -409,7 +390,8 @@ export default function MaterialManagement({
- {/* 回收站模式提示 */} - {showRecycleBin && ( -
- - - - 回收站模式 - - 显示已删除素材，删除操作为永久删除 - {totalMaterialsCount !== undefined && ( - （共 {totalMaterialsCount} 条） - )} -
- )} -
{/* 批量操作按钮组 */} @@ -458,7 +427,7 @@ export default function MaterialManagement({ @@ -545,6 +487,9 @@ export default function MaterialManagement({
+ {purgeStatus} + {confirmActionDialog} + {/* 卡片网格布局 */}
{materials.slice(0, displayedMaterialsCount).map((material) => { @@ -568,7 +513,9 @@ export default function MaterialManagement({ }`} > {/* 选择指示器 (更像原生选择) */} -
{ event.stopPropagation(); toggleMaterialSelection(material.id) }} + className={`absolute z-20 top-1 right-1 min-w-11 min-h-11 rounded-full flex items-center justify-center transition-colors focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-blue-600 ${ isSelected ? 'bg-blue-500 text-white shadow-md scale-100' : 'bg-black/20 hover:bg-black/40 border border-white/30 backdrop-blur-sm' @@ -578,7 +525,7 @@ export default function MaterialManagement({ )} -
+ {/* 预览图/视频 */}
@@ -627,45 +574,49 @@ export default function MaterialManagement({
{/* 操作按钮浮层 */} -
+
{showRecycleBin ? ( <> ) : ( <> @@ -697,7 +648,8 @@ export default function MaterialManagement({
)} - {materials.length === 0 && ( + {isLoadingMaterials &&
加载中…
} + {!isLoadingMaterials && materials.length === 0 && (
暂无素材
diff --git a/components/admin/MaterialPurgeStatus.test.tsx b/components/admin/MaterialPurgeStatus.test.tsx new file mode 100644 index 0000000..9463499 --- /dev/null +++ b/components/admin/MaterialPurgeStatus.test.tsx @@ -0,0 +1,33 @@ +import React from 'react' +import { act, fireEvent, render, screen } from '@testing-library/react' +import { useMaterialPurgeJobs } from './MaterialPurgeStatus' + +const id = '11111111-1111-4111-8111-111111111111' +const job = { id, materialId: id, name: '测试素材', status: 'queued', totalFiles: 2, processedFiles: 0, sharedFiles: 0, error: null } +const response = (body: unknown, ok = true) => Promise.resolve({ ok, json: async () => body }) +function Fixture() { const { status } = useMaterialPurgeJobs(); return <>{status} } + +beforeEach(() => { jest.useFakeTimers(); global.fetch = jest.fn() }) +afterEach(() => { jest.useRealTimers(); jest.restoreAllMocks() }) + +test('刷新后重新接管的清理任务也显示完成结果', async () => { + ;(fetch as jest.Mock).mockImplementationOnce(() => response({ jobs: [job] })) + .mockImplementation(() => response({ jobs: [{ ...job, status: 'completed', processedFiles: 2 }] })) + await act(async () => { render() }) + expect(screen.getByText('正在清理 1 条素材')).toBeInTheDocument() + await act(async () => { jest.advanceTimersByTime(3000) }) + expect((fetch as jest.Mock).mock.calls[1][0]).toContain(`id=${id}`) + expect(screen.getByText('已彻底删除 1 条')).toBeInTheDocument() +}) + +test('后台进度轮询成功不能抹掉人工重试失败的提示', async () => { + const failed = { ...job, status: 'failed', error: '文件清理未完成' } + ;(fetch as jest.Mock).mockImplementation((_url, options) => options?.method === 'POST' + ? response({ error: '存储暂不可用，请稍后重试' }, false) : response({ jobs: [failed] })) + await act(async () => { render() }) + await act(async () => { fireEvent.click(screen.getByRole('button', { name: '重试清理' })) }) + expect(screen.getByRole('alert')).toHaveTextContent('存储暂不可用，请稍后重试') + await act(async () => { jest.advanceTimersByTime(3000) }) + expect(screen.getByRole('alert')).toHaveTextContent('存储暂不可用，请稍后重试') + expect(screen.getByRole('button', { name: '重试清理' })).toBeEnabled() +}) diff --git a/components/admin/MaterialPurgeStatus.tsx b/components/admin/MaterialPurgeStatus.tsx new file mode 100644 index 0000000..183d61b --- /dev/null +++ b/components/admin/MaterialPurgeStatus.tsx @@ -0,0 +1,116 @@ +'use client' + +import { useCallback, useEffect, useRef, useState } from 'react' +import { AlertCircle, Check, Loader2, X } from 'lucide-react' + +export type PurgeJobView = { + id: string; materialId: string; name: string; status: string + totalFiles: number; processedFiles: number; sharedFiles: number; error: string | null +} + +export function useMaterialPurgeJobs() { + const [jobs, setJobs] = useState([]) + const [error, setError] = useState(null) + const [retryError, setRetryError] = useState(null) + const [retrying, setRetrying] = useState(null) + const tracked = useRef([]) + const mounted = useRef(false) + const dismissed = useRef(new Set()) + const loading = useRef(false) + + const track = useCallback((next: PurgeJobView[]) => { + tracked.current = Array.from(new Set([...next.map(job => job.id), ...tracked.current])).slice(0, 100) + setJobs(previous => [...next, ...previous.filter(job => !next.some(item => item.id === job.id))]) + }, []) + + const refresh = useCallback(async () => { + if (loading.current) return + loading.current = true + try { + const params = new URLSearchParams() + tracked.current.forEach(id => params.append('id', id)) + const response = await fetch(`/api/admin/materials/purge-jobs?${params}`, { cache: 'no-store' }) + if (!response.ok) throw new Error('清理进度暂不可用') + const result = await response.json() + if (!mounted.current) return + // 页面刷新后接管正在处理的任务，也要跟踪它们的最终完成状态。 + tracked.current = Array.from(new Set([ + ...result.jobs.filter((job: PurgeJobView) => job.status !== 'completed').map((job: PurgeJobView) => job.id), + ...tracked.current, + ])).slice(0, 100) + setJobs(previous => { + const next: PurgeJobView[] = result.jobs.filter((job: PurgeJobView) => !dismissed.current.has(job.id)) + // 请求开始之后新提交的任务，不能被旧的列表响应抹掉。 + return [...next, ...previous.filter(job => tracked.current.includes(job.id) && !next.some(item => item.id === job.id))] + }) + setError(null) + } catch { + if (mounted.current) setError('清理进度暂不可用') + } finally { loading.current = false } + }, []) + + useEffect(() => { + mounted.current = true + void refresh() + const timer = setInterval(() => { if (!document.hidden) void refresh() }, 3000) + return () => { mounted.current = false; clearInterval(timer) } + }, [refresh]) + + const retry = async (id: string) => { + setRetrying(id) + setRetryError(null) + try { + const response = await fetch('/api/admin/materials/purge-jobs', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ jobId: id }), + }) + const result = await response.json() + if (!response.ok) throw new Error(result.error || '重试失败，请稍后再试') + track([result.job]) + setError(null) + } catch (failure: any) { setRetryError(failure.message || '重试失败，请稍后再试') } + finally { setRetrying(null) } + } + + const failed = jobs.filter(job => job.status === 'failed') + const pending = jobs.filter(job => job.status === 'queued' || job.status === 'processing') + const completed = jobs.filter(job => job.status === 'completed') + const shared = completed.reduce((sum, job) => sum + job.sharedFiles, 0) + + const status = jobs.length || error || retryError ? ( +
+
+ {pending.length > 0 && + } + {completed.length > 0 && + } + {error && + {error} + } +
+ {retryError &&

{retryError}

} + {failed.length > 0 &&
    + {failed.map(job =>
  • +
    +
    + +
    • )} +
} +
+ ) : null + return { track, status } +} diff --git a/docs/回收站永久删除修复与验收-2026-10-09.md b/docs/回收站永久删除修复与验收-2026-10-09.md new file mode 100644 index 0000000..0087c2c --- /dev/null +++ b/docs/回收站永久删除修复与验收-2026-10-09.md @@ -0,0 +1,62 @@ +# 回收站永久删除修复与验收 + +日期：2026-10-09。本文记录发布前的本地代码与隔离验收结果；验收阶段未删除生产回收站的真实素材。用户已授权提交与部署，实际发布状态以精确 Git 提交、生产镜像身份和迁移审计回执为准。原有任务详情相关未提交工作保持原状。 + +## 行为 + +- 素材管理中的单条/批量删除、恢复使用现有 `ConfirmActionDialog`。危险操作默认聚焦取消；支持键盘焦点循环、Escape 取消和焦点恢复。 +- 提交失败留在确认框内，并保留素材选择。成功反馈使用共享居中气泡，限制宽度并提供 `status/alert` 语义。 +- 彻底删除返回 202 和持久化清理任务，列表显示正在清理、完成或需重试。文件尚未清理完时不宣称操作完成；失败记录刷新后仍可读取。 +- 批量恢复只移除成功项，失败项保留并可以续试。 + +## 删除一致性与存储边界 + +`enqueueMaterialPurge` 同一个数据库事务内检查回收站状态与在途处理，保存文件清单、素材快照、操作日志和清理审计，再删除素材记录及关联资产/SKU。任何失败整批回滚，请求路径不执行文件删除。最多提交 100 条素材，每条清单最多 2000 个文件，重复和并发提交按素材 ID 幂等。 + +上传作业创建/领取、视频转码领取与删除使用一致的数据库锁顺序；仍有上传作业、应用中的变更或转码租约的素材拒绝清理。恢复与删除由素材行锁协调。素材操作与下载日志不再随素材级联删除，历史导入查重改为显式关联仍在使用的素材。 + +旧历史素材备份脚本改为按素材 ID 显式查询两类日志，保持备份与证据预检的兼容；仅进行定向类型检查，没有执行该脚本或扩大历史删除范围。 + +后台从 PostgreSQL 领取任务，使用数据库时钟和租约令牌。租约 10 分钟、每 30 秒续期，每个文件的处理结果单独落库；进程中断后可接管过期任务。失败自动重试至三次，之后显示人工重试入口，保留失败、重试操作人和完成审计。应用启动与请求提交均可启动调度器，构建阶段不启动删除工作器。 + +只接受当前上传根目录或当前配置的 S3 桶、端点和项目前缀；清单绑定这些存储身份。根目录或端点变化后停止清理。拒绝越界、符号链接和非普通文件，不递归删除目录。属于素材 UUID 的目录仅枚举普通文件加入清单。本地文件大小与纳秒修改时间作为替换保护，时间戳以字符串保存，避免数据库 JSON 浮点往返误差。 + +删除前检查其他素材、资产、SKU、品牌知识、站点标识、作品图片与在途作业/变更的文件引用，包括回收站和软删除 SKU。保留共用文件。历史 HTTP 地址只能提供保守的引用保护，不增加物理删除权限；未受控地址的素材不会直接永久删除。 + +## 验证证据 + +| 验证 | 结果 | +| --- | --- | +| 相关 Jest 测试 | 38 个通过，覆盖权限、输入上限、请求不直接删除文件、路径/链接/替换保护、原素材/SKU 授权、刷新后完成状态和重试错误保留 | +| 安全迁移验证 | 13 个通过，新增清理表结构、唯一性、索引和日志不级联删除检查 | +| 实际 PostgreSQL 与 MinIO | 9 组场景通过；执行真实升级 SQL、整批回滚、审计失败回滚、在途处理拒绝、并发幂等、共用文件保护、过期租约恢复、部分失败续跑和对象前缀隔离 | +| 类型与客户端 | `npm run typecheck` 和 Prisma Client 69 项声明核对通过 | +| 构建 | `npm run build` 通过；存储历史只读边界检查通过。保留项目原有 lint/Browserslist 提示，未扩展修改范围 | +| 浏览器 | 本地真实删除/恢复 API；1280px 桌面与 390×844 手机视口。检查确认、409 失败上下文、居中气泡、单条/批量 202 清理中、刷新后失败记录、人工重试完成、取消焦点恢复；无页面控制台错误或横向溢出 | + +隔离数据库使用独立 Docker 容器 `gongxiangp2-purge-qa-20261009`，仅绑定本机 55439。真实对象测试使用独立本机 MinIO 59000，未连接生产对象存储。验收脚本 `scripts/test-material-purge-integration.ts` 拒绝非指定本机验收数据库，每次建立新的独立数据库和临时文件目录。 + +本地 UI 预览为 `http://localhost:3114/admin`，仅身份和无关统计页使用隔离替身；删除、恢复、任务查询与重试调用本项目真实路由。页脚标注示例数据并提供复原入口。构建日志、请求记录、测试结果和截图保存在忽略目录 `tmp/material-purge-qa/`。 + +## Impeccable harden / polish / audit + +本次页面按 Operate 模式处理，保留素材管理现有视觉语言，复用共享对话框和通知；不改造后台整体结构或添加装饰动画。 + +| 维度 | 范围评分 | 验证 | +| --- | --- | --- | +| 可访问性 | 3/4 | 共享确认框焦点管理、危险操作默认取消、错误/进度语义、相关操作 44px 触达；未宣称整站无障碍认证 | +| 性能 | 3/4 | 原分页与懒加载保持；状态查询限量、隐藏页面暂停轮询、请求不重叠；未进行大数据量负载测试 | +| 响应式 | 4/4 | 本次对话框、气泡和进度区域在两个视口中无溢出；手机动作可见并可操作 | +| 主题与一致性 | 3/4 | 复用现有产品组件，提高气泡前景对比；存量样式未全面重写 | +| 实现完整性 | 4/4 | 持久化链路与 UI 承诺一致，提交/完成区分，失败可见、可续试 | +| 合计 | 17/20 | 本次操作范围良好；主要原有噪音与无关样式不在修复范围 | + +一次源码 detector 检查没有阻断问题，五个提示均为存量 10/11/13px 字号与主要描述部门壳的 DESIGN.md 字号阶梯不同。没有扩大到后台整体字体/主题重构。原始 JSON 为 `tmp/material-purge-qa/impeccable-audit.json`。 + +截图包括 `desktop-final-confirm.png`、`desktop-error.png`、`mobile-error.png`、`mobile-toast.png`、`desktop-cleanup-failed.png`、`mobile-cleanup-failed.png`、`desktop-retry-complete.png` 和 `mobile-retry-complete.png`。 + +## 后续发布 + +新增迁移 `20261009_material_purge_outbox` 已加入安全迁移与发布审计列表、构建文件存在性检查；完整迁移目录随 migrator 镜像提供。发布必须先通过现有备份/恢复验证和安全迁移流程，再启动新应用；不能只删除旧禁用判断或跳过迁移。 + +本地验收阶段尚未进行 Git 提交；提交后按项目规则同步真实 Git 元数据。线上验收应先使用专用可清理测试素材，真实回收站内容的永久删除仍由管理员明确确认。 diff --git a/instrumentation.ts b/instrumentation.ts index fe8c376..bae21b3 100644 --- a/instrumentation.ts +++ b/instrumentation.ts @@ -10,6 +10,9 @@ export async function register() { // 只在 Node.js 服务端运行（不在 Edge 运行时） if (process.env.NEXT_RUNTIME === 'nodejs') { console.log('[Instrumentation] Next.js 服务端启动...') + + const { ensureMaterialPurgeWorkerStarted } = await import('./lib/material-purge') + ensureMaterialPurgeWorkerStarted() // 启动临时文件清理任务 try { diff --git a/lib/__tests__/material-permanent-delete-disabled.test.ts b/lib/__tests__/material-permanent-delete-disabled.test.ts deleted file mode 100644 index aa4f1ab..0000000 --- a/lib/__tests__/material-permanent-delete-disabled.test.ts +++ /dev/null @@ -1,18 +0,0 @@ -/** @jest-environment node */ - -import { readFile } from 'fs/promises' -import path from 'path' - -describe('material permanent delete safety gate', () => { - test.each([ - 'app/api/materials/route.ts', - 'app/api/admin/materials/batch-delete/route.ts', - ])('%s 在持久化 purge/outbox 上线前禁止物理删除', async (relativePath) => { - const source = await readFile(path.join(process.cwd(), relativePath), 'utf8') - - expect(source).toContain("code: 'PERMANENT_DELETE_DISABLED'") - expect(source).not.toContain('deleteFiles(') - expect(source).not.toContain('deleteMaterialRelatedRecords(') - expect(source).not.toContain('prisma.material.delete(') - }) -}) diff --git a/lib/__tests__/material-permanent-delete.test.ts b/lib/__tests__/material-permanent-delete.test.ts new file mode 100644 index 0000000..af1f2d6 --- /dev/null +++ b/lib/__tests__/material-permanent-delete.test.ts @@ -0,0 +1,60 @@ +/** @jest-environment node */ +import { NextRequest } from 'next/server' +import { getServerSession } from 'next-auth' +import { POST } from '@/app/api/admin/materials/batch-delete/route' +import { enqueueMaterialPurge, MaterialPurgeError } from '@/lib/material-purge' +import { readFile } from 'fs/promises' +import path from 'path' + +jest.mock('next-auth', () => ({ getServerSession: jest.fn() })) +jest.mock('@/lib/auth', () => ({ authOptions: {} })) +jest.mock('@/lib/prisma', () => ({ prisma: {} })) +jest.mock('@/lib/material-log', () => ({ createMaterialLog: jest.fn() })) +jest.mock('@/lib/department-permission', () => ({ isSystemAdmin: (role: string) => ['ADMIN', 'SYSTEM_ADMIN'].includes(role) })) +jest.mock('@/lib/material-purge', () => ({ + enqueueMaterialPurge: jest.fn(), ensureMaterialPurgeWorkerStarted: jest.fn(), + MaterialPurgeError: class extends Error { constructor(message: string, readonly status = 409) { super(message) } }, +})) + +const id = '11111111-1111-4111-8111-111111111111' +const request = (body: unknown) => new NextRequest('http://localhost/api/admin/materials/batch-delete', { + method: 'POST', body: JSON.stringify(body), headers: { 'Content-Type': 'application/json' }, +}) + +beforeEach(() => { + jest.clearAllMocks() + ;(getServerSession as jest.Mock).mockResolvedValue({ user: { id: 'admin', role: 'ADMIN' } }) +}) + +test.each([null, { user: { id: 'user', role: 'USER' } }, { user: { id: 'department', role: 'DEPARTMENT_ADMIN' } }])('永久删除拒绝非系统管理员', async session => { + ;(getServerSession as jest.Mock).mockResolvedValue(session) + expect((await POST(request({ ids: [id], permanent: true }))).status).toBe(403) + expect(enqueueMaterialPurge).not.toHaveBeenCalled() +}) + +test('只在任务可靠提交后返回 202 和真实任务，不宣称文件已经清理', async () => { + const jobs = [{ id: 'job', status: 'queued' }] + ;(enqueueMaterialPurge as jest.Mock).mockResolvedValue(jobs) + const result = await POST(request({ ids: [id], permanent: true })) + expect(result.status).toBe(202) + expect(await result.json()).toEqual({ success: true, jobs }) + expect(enqueueMaterialPurge).toHaveBeenCalledWith([id], 'admin') +}) + +test.each([{ ids: [id], permanent: 'true' }, { ids: ['invalid'], permanent: true }, { ids: Array(101).fill(id), permanent: true }])('拒绝错误删除方式和超限批量输入', async body => { + expect((await POST(request(body))).status).toBe(400) + expect(enqueueMaterialPurge).not.toHaveBeenCalled() +}) + +test('批量条件冲突返回可恢复的上下文错误', async () => { + ;(enqueueMaterialPurge as jest.Mock).mockRejectedValue(new MaterialPurgeError('只能彻底删除回收站中的素材')) + const result = await POST(request({ ids: [id], permanent: true })) + expect(result.status).toBe(409) + expect(await result.json()).toEqual({ error: '只能彻底删除回收站中的素材' }) +}) + +test.each(['app/api/materials/route.ts', 'app/api/admin/materials/batch-delete/route.ts'])('%s 的请求路径不直接做物理删除', async file => { + const source = await readFile(path.join(process.cwd(), file), 'utf8') + expect(source).toContain('enqueueMaterialPurge(') + expect(source).not.toMatch(/deleteFiles\(|deleteMaterialRelatedRecords\(|prisma\.material\.delete\(/) +}) diff --git a/lib/__tests__/material-purge-files.test.ts b/lib/__tests__/material-purge-files.test.ts new file mode 100644 index 0000000..23cbeaa --- /dev/null +++ b/lib/__tests__/material-purge-files.test.ts @@ -0,0 +1,66 @@ +/** @jest-environment node */ +import { promises as fs } from 'fs' +import os from 'os' +import path from 'path' +import { collectPurgeFiles, identifyPurgeFile, removePurgeFile } from '@/lib/material-purge-files' + +jest.mock('@/lib/prisma', () => ({ prisma: {} })) +jest.mock('@/lib/s3', () => ({ + getTrustedS3MutationKey: (url: string) => { + const parsed = new URL(url) + if (parsed.origin !== 'http://storage.test' || !parsed.pathname.startsWith('/project/')) throw new Error('拒绝外部对象') + return parsed.pathname.slice(1) + }, + deleteFromS3: jest.fn(), +})) + +let root: string +let oldUpload: string | undefined +const material = (source: string) => ({ id: '11111111-1111-4111-8111-111111111111', previewUrl: source, zipFilePath: source, + displayImages: [], videoSourceUrl: null, videoPreview540Url: null, videoPreviewClipUrl: null, videoCoverUrl: null, assets: [], skus: [] }) + +beforeEach(async () => { + oldUpload = process.env.UPLOAD_DIR + root = await fs.mkdtemp(path.join(os.tmpdir(), 'gongxiangp2-purge-files-')) + process.env.UPLOAD_DIR = root +}) +afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }) + if (oldUpload === undefined) delete process.env.UPLOAD_DIR + else process.env.UPLOAD_DIR = oldUpload +}) + +test('规范化等价的 uploads URL 并去重，重复删除缺失文件可续跑', async () => { + await fs.writeFile(path.join(root, 'asset.bin'), 'owned') + const input = material('uploads/asset.bin') + input.displayImages = ['/uploads/asset.bin'] as never[] + const files = await collectPurgeFiles(input) + expect(files).toHaveLength(1) + await removePurgeFile(files[0]) + await expect(removePurgeFile(files[0])).resolves.toBeUndefined() +}) + +test.each(['uploads/../outside', '/etc/passwd', 'https://foreign.test/project/file', 'uploads/'])('拒绝不受控删除地址 %s', source => { + expect(() => identifyPurgeFile(source)).toThrow() +}) + +test('拒绝符号链接，不跟随链接删除文件', async () => { + const file = path.join(root, 'owned.bin') + await fs.writeFile(file, 'keep') + await fs.symlink(file, path.join(root, 'link.bin')) + await expect(collectPurgeFiles(material('uploads/link.bin'))).rejects.toThrow('符号链接') + expect(await fs.readFile(file, 'utf8')).toBe('keep') +}) + +test('任务入队后同路径文件被替换时拒绝删除', async () => { + const file = path.join(root, 'owned.bin') + await fs.writeFile(file, 'first') + const files = await collectPurgeFiles(material('uploads/owned.bin')) + await fs.writeFile(file, 'replacement content') + await expect(removePurgeFile(files[0])).rejects.toThrow('文件已发生变化') + expect(await fs.readFile(file, 'utf8')).toBe('replacement content') +}) + +test('没有物理文件的 data 预览不产生删除目标', async () => { + expect(await collectPurgeFiles(material('data:image/svg+xml;base64,Zm9v'))).toEqual([]) +}) diff --git a/lib/history-material-import.ts b/lib/history-material-import.ts index ce0d1c2..07fee20 100644 --- a/lib/history-material-import.ts +++ b/lib/history-material-import.ts @@ -759,20 +759,15 @@ async function ensureCategoryAndSceneNames(categoryName: string, sceneName: stri } async function hasImportedHistoryRecord(record: HistoryImportRecord) { - const existingByFingerprint = await prisma.materialLog.findFirst({ - where: { - action: 'HISTORY_IMPORT', - material: { - deletedAt: null, - }, - changes: { - contains: record.fingerprint, - } as any, - }, - select: { id: true }, - }) + const existingByFingerprint = await prisma.$queryRaw>` + SELECT logs.id FROM "material_logs" AS logs + INNER JOIN "materials" AS materials ON materials.id = logs."materialId" + WHERE logs.action = 'HISTORY_IMPORT' AND materials."deletedAt" IS NULL + AND position(${record.fingerprint} in logs.changes::text) > 0 + LIMIT 1 + ` - if (existingByFingerprint) { + if (existingByFingerprint.length) { return true } diff --git a/lib/material-purge-files.ts b/lib/material-purge-files.ts new file mode 100644 index 0000000..32e95b9 --- /dev/null +++ b/lib/material-purge-files.ts @@ -0,0 +1,195 @@ +import { promises as fs } from 'fs' +import path from 'path' +import type { Prisma } from '@prisma/client' +import { deleteFromS3, getTrustedS3MutationKey } from '@/lib/s3' +import { getUploadRootDir } from '@/lib/upload-paths' +import { buildPublicDisplayUrl, buildWhiteBackgroundPublicDisplayPathFromAssetPath } from '@/lib/public-display' + +export type PurgeFile = { + source: string + storage: 'local' | 's3' + key: string + bucket?: string + endpoint?: string + root?: string + fingerprint?: { size: number; mtimeNs: string } | null + outcome: 'pending' | 'deleted' | 'shared' +} + +const MAX_FILES = 2000 + +/** 地址只决定当前项目的写入范围，历史域名不扩展删除权限。 */ +export function identifyPurgeFile(source: string): Omit | null { + const value = String(source || '').trim() + if (!value || value.startsWith('data:image/') || /^\/(?:brand|images|placeholder)[/.]/.test(value)) return null + if (/^https?:\/\//i.test(value)) { + const key = getTrustedS3MutationKey(value) + if (!process.env.S3_BUCKET) throw new Error('对象存储未配置，暂不能清理文件') + return { source: value, storage: 's3', key, bucket: process.env.S3_BUCKET, + endpoint: String(process.env.S3_ENDPOINT || '').split('#')[0].trim() } + } + const normalized = value.replace(/\\/g, '/') + if (/[\x00-\x1f\x7f]/.test(normalized) || normalized.split('/').includes('..')) { + throw new Error('文件地址不安全，已拒绝删除') + } + const root = path.resolve(getUploadRootDir()) + const relative = normalized.replace(/^\.?\/?uploads\//, '') + const absolute = normalized !== relative + ? path.resolve(root, relative) + : path.isAbsolute(normalized) ? path.resolve(normalized) : path.resolve(process.cwd(), normalized) + const key = path.relative(root, absolute).split(path.sep).join('/') + if (!key || key === '..' || key.startsWith('../') || path.isAbsolute(key)) { + throw new Error('文件不在本项目上传目录，已拒绝删除') + } + return { source: value, storage: 'local', key, root } +} + +export function purgeFileIdentity(file: Pick) { + const key = file.storage === 'local' && process.platform === 'win32' ? file.key.toLowerCase() : file.key + return `${file.storage}:${file.bucket || ''}:${key}` +} + +async function inspectLocalFile(key: string, directory = false) { + const root = path.resolve(getUploadRootDir()) + let current = root + // 每一层都拒绝符号链接；不递归删除目录，也不沿链接枚举文件。 + const parts = ['', ...key.split('/')] + for (let index = 0; index < parts.length; index += 1) { + if (parts[index]) current = path.join(current, parts[index]) + let stat + try { stat = await fs.lstat(current, { bigint: true }) } catch (error: any) { + if (error.code === 'ENOENT') return null + throw error + } + if (stat.isSymbolicLink()) throw new Error('文件路径包含符号链接，已拒绝删除') + if (index === parts.length - 1) { + if (directory ? !stat.isDirectory() : !stat.isFile()) throw new Error('文件清理目标类型不符，已拒绝删除') + // 纳秒时间戳用字符串保存，避免 JSON/数据库浮点往返导致误判。 + return { size: Number(stat.size), mtimeNs: stat.mtimeNs.toString() } + } + if (!stat.isDirectory()) throw new Error('文件目录无效，已拒绝删除') + } + return null +} + +export async function collectPurgeFiles(material: { + id: string; previewUrl: string; zipFilePath: string; displayImages: string[] + videoSourceUrl: string | null; videoPreview540Url: string | null + videoPreviewClipUrl: string | null; videoCoverUrl: string | null + assets: { fileUrl: string }[]; skus: { filePath: string; previewUrl: string | null }[] +}): Promise { + const sources = [material.previewUrl, material.zipFilePath, ...material.displayImages, + material.videoSourceUrl, material.videoPreview540Url, material.videoPreviewClipUrl, material.videoCoverUrl, + ...material.assets.map(asset => asset.fileUrl), + ...material.skus.flatMap(sku => [sku.filePath, sku.previewUrl])] + for (const asset of material.assets) { + const derived = buildWhiteBackgroundPublicDisplayPathFromAssetPath(asset.fileUrl) + if (derived) sources.push(buildPublicDisplayUrl(derived)) + } + // 这些目录由素材 UUID 独占；列出文件写入清单，不做目录递归删除。 + const root = path.resolve(getUploadRootDir()) + async function walk(relative: string) { + const absolute = path.join(root, relative) + let entries + try { + if (!await inspectLocalFile(relative, true)) return + entries = await fs.readdir(absolute, { withFileTypes: true }) + } catch (error: any) { if (error.code === 'ENOENT') return; throw error } + for (const entry of entries) { + if (entry.isSymbolicLink()) throw new Error('素材目录包含符号链接，已拒绝删除') + const child = `${relative}/${entry.name}` + if (entry.isDirectory()) await walk(child) + else if (entry.isFile()) sources.push(`uploads/${child}`) + else throw new Error('素材目录包含特殊文件，已拒绝删除') + if (sources.length > MAX_FILES) throw new Error('素材文件过多，请联系管理员分批清理') + } + } + // 先校验上传根目录本身，避免只检查最末级目录时沿根目录链接走出边界。 + try { + const stat = await fs.lstat(root) + if (stat.isSymbolicLink() || !stat.isDirectory()) throw new Error('上传目录无效，已拒绝删除') + } catch (error: any) { if (error.code !== 'ENOENT') throw error } + await walk(`product-package/${material.id}`) + await walk(`skus/${material.id}`) + const files = new Map() + for (const source of sources) { + if (!source) continue + const file = identifyPurgeFile(source) + if (!file) continue + const identity = purgeFileIdentity(file) + if (!files.has(identity)) files.set(identity, { + ...file, outcome: 'pending', + ...(file.storage === 'local' ? { fingerprint: await inspectLocalFile(file.key) } : {}), + }) + if (files.size > MAX_FILES) throw new Error('素材文件过多，请联系管理员分批清理') + } + return Array.from(files.values()) +} + +function stringsIn(value: unknown): string[] { + if (typeof value === 'string') return [value] + if (Array.isArray(value)) return value.flatMap(stringsIn) + if (value && typeof value === 'object') return Object.values(value).flatMap(stringsIn) + return [] +} + +/** 包含回收站与 SKU 软删除记录，保留其他素材恢复时仍需要的共用文件。 */ +export async function findPurgeFileReferences(db: Prisma.TransactionClient): Promise> { + const [materials, assets, skus, knowledge, settings, workItems, versions, uploads, changes] = await Promise.all([ + db.material.findMany({ select: { previewUrl: true, zipFilePath: true, displayImages: true, + videoSourceUrl: true, videoPreview540Url: true, videoPreviewClipUrl: true, videoCoverUrl: true } }), + db.materialAsset.findMany({ select: { fileUrl: true } }), + db.materialSku.findMany({ select: { filePath: true, previewUrl: true } }), + db.brandKnowledge.findMany({ select: { filePath: true, previewImagePath: true, previewImageSlices: true } }), + db.systemSettings.findMany({ select: { headerLogoUrl: true, loginLogoUrl: true } }), + db.performanceSurveyWorkItem.findMany({ select: { imageUrls: true } }), + db.workReviewVersion.findMany({ select: { imageUrls: true } }), + db.productPackageUploadJob.findMany({ where: { status: { in: ['queued', 'processing'] } }, select: { assets: true } }), + db.taskDetailChangeRequest.findMany({ where: { status: { notIn: ['applied', 'rejected', 'expired', 'cancelled'] } }, select: { operations: true } }), + ]) + const identities = new Set() + for (const source of stringsIn([materials, assets, skus, knowledge, settings, workItems, versions, uploads, changes])) { + try { + const file = identifyPurgeFile(source) + if (file) identities.add(purgeFileIdentity(file)) + } catch { + // 外部只读引用不扩大写权限。路径相同的历史 HTTP 引用仍保护当前桶中的对象。 + if (/^https?:\/\//i.test(source)) { + try { + let key = decodeURIComponent(new URL(source).pathname.slice(1)) + const bucket = process.env.S3_BUCKET || '' + if (bucket && key.startsWith(`${bucket}/`)) key = key.slice(bucket.length + 1) + identities.add(purgeFileIdentity({ storage: 's3', bucket, key })) + } catch { /* 非文件字符串不构成引用 */ } + } + } + } + // 白底图的公网派生文件也可能被其他 SKU 的本地源文件引用。 + for (const asset of assets) { + const derived = buildWhiteBackgroundPublicDisplayPathFromAssetPath(asset.fileUrl) + const source = derived && buildPublicDisplayUrl(derived) + if (source) { + const file = identifyPurgeFile(source) + if (file) identities.add(purgeFileIdentity(file)) + } + } + return identities +} + +export async function removePurgeFile(file: PurgeFile): Promise { + const current = identifyPurgeFile(file.source) + if (!current || purgeFileIdentity(current) !== purgeFileIdentity(file) || current.root !== file.root || current.endpoint !== file.endpoint) { + throw new Error('存储配置已变化，已停止文件清理，请联系管理员') + } + if (file.storage === 's3') { + try { await deleteFromS3(file.source) } + catch { throw new Error('存储暂时无法访问，文件清理未完成，请重试') } + return + } + const stat = await inspectLocalFile(file.key) + if (!stat) return // 删除成功但写回之前进程中断时，重试仍可完成。 + if (!file.fingerprint || stat.size !== file.fingerprint.size || stat.mtimeNs !== file.fingerprint.mtimeNs) { + throw new Error('文件已发生变化，已停止清理，请联系管理员') + } + await fs.unlink(path.join(getUploadRootDir(), file.key)) +} diff --git a/lib/material-purge.ts b/lib/material-purge.ts new file mode 100644 index 0000000..57daae2 --- /dev/null +++ b/lib/material-purge.ts @@ -0,0 +1,189 @@ +import { randomUUID } from 'crypto' +import { Prisma, type MaterialPurgeJob } from '@prisma/client' +import { prisma } from '@/lib/prisma' +import { collectPurgeFiles, findPurgeFileReferences, purgeFileIdentity, removePurgeFile, type PurgeFile } from '@/lib/material-purge-files' + +export class MaterialPurgeError extends Error { + constructor(message: string, readonly status = 409) { super(message); this.name = 'MaterialPurgeError' } +} + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i +const LEASE_MS = 10 * 60_000 +const MAX_ATTEMPTS = 3 + +export function validatePurgeIds(input: unknown): string[] { + if (!Array.isArray(input) || input.length === 0 || input.length > 100 || input.some(id => typeof id !== 'string' || !UUID.test(id))) { + throw new MaterialPurgeError('请选择 1–100 条有效素材', 400) + } + return Array.from(new Set(input.map(id => id.toLowerCase()))).sort() +} + +export function toPurgeJobView(job: MaterialPurgeJob) { + const files = job.files as unknown as PurgeFile[] + const snapshot = job.materialSnapshot as { productModel?: string } + return { + id: job.id, materialId: job.materialId, name: snapshot.productModel || '素材', + status: job.status, totalFiles: files.length, + processedFiles: files.filter(file => file.outcome !== 'pending').length, + sharedFiles: files.filter(file => file.outcome === 'shared').length, + error: job.errorMessage, createdAt: job.createdAt.toISOString(), + } +} + +async function lock(tx: Prisma.TransactionClient, name: string) { + await tx.$queryRaw`SELECT 1 AS locked FROM (SELECT pg_advisory_xact_lock(hashtext(${name}))) AS locked` +} + +/** 清单、审计和素材记录删除必须一起提交；请求路径不做物理删除。 */ +export async function enqueueMaterialPurge(input: unknown, operatorUserId: string) { + const ids = validatePurgeIds(input) + return prisma.$transaction(async tx => { + await lock(tx, 'material-purge-enqueue') + await lock(tx, 'product-package-upload-create') + await lock(tx, 'product-package-upload-worker') + await tx.$queryRaw`SELECT 1 AS locked FROM pg_advisory_xact_lock(193582731)` + const jobs = await tx.materialPurgeJob.findMany({ where: { materialId: { in: ids } } }) + const newIds = ids.filter(id => !jobs.some(job => job.materialId === id)) + if (!newIds.length) return jobs.map(toPurgeJobView) + await tx.$queryRaw`SELECT id FROM materials WHERE id IN (${Prisma.join(newIds)}) ORDER BY id FOR UPDATE` + const materials = await tx.material.findMany({ + where: { id: { in: newIds } }, include: { assets: true, skus: true }, + }) + if (materials.length !== newIds.length) throw new MaterialPurgeError('部分素材已不存在，请刷新后重试', 404) + if (materials.some(material => !material.deletedAt)) throw new MaterialPurgeError('只能彻底删除回收站中的素材') + const uploading = await tx.productPackageUploadJob.count({ where: { materialId: { in: newIds }, status: { in: ['queued', 'processing'] } } }) + const applying = await tx.taskDetailChangeRequest.count({ where: { materialId: { in: newIds }, status: 'applying' } }) + if (uploading || applying || materials.some(material => material.transcodeLeaseToken)) { + throw new MaterialPurgeError('部分素材仍在处理中，请处理完成后重试') + } + for (const material of materials) { + let files: PurgeFile[] + try { files = await collectPurgeFiles(material) } catch (error: any) { + throw new MaterialPurgeError(`${material.productModel || '素材'}：${error.message}`) + } + const snapshot = { + id: material.id, productModel: material.productModel, materialType: material.materialType, + taskNo: material.taskNo, ownerUserId: material.userId, deletedAt: material.deletedAt!.toISOString(), + } + const job = await tx.materialPurgeJob.create({ data: { + materialId: material.id, operatorUserId, materialSnapshot: snapshot, + files: files as unknown as Prisma.InputJsonValue, + } }) + await tx.materialPurgeAudit.create({ data: { jobId: job.id, event: 'queued', operatorUserId } }) + await tx.materialLog.create({ data: { + materialId: material.id, userId: operatorUserId, action: 'DELETE', + changes: JSON.stringify({ + materialSnapshot: { oldValue: null, newValue: snapshot }, + deleteMode: { oldValue: 'soft', newValue: 'permanent' }, + purgeJobId: { oldValue: null, newValue: job.id }, + }), + } }) + // FK 清理关联资产、收藏等记录；操作/下载日志已由迁移解除素材级联关系。 + await tx.material.delete({ where: { id: material.id } }) + jobs.push(job) + } + return jobs.map(toPurgeJobView) + }, { timeout: 30_000 }) +} + +async function claimJob() { + return prisma.$transaction(async tx => { + const rows = await tx.$queryRaw<{ id: string }[]>` + SELECT id FROM material_purge_jobs + WHERE (status = 'queued' AND next_attempt_at <= clock_timestamp()) + OR (status = 'processing' AND lease_expires_at <= clock_timestamp()) + ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED` + if (!rows.length) return null + const leaseToken = randomUUID() + const [clock] = await tx.$queryRaw<{ now: Date }[]>`SELECT clock_timestamp() AS now` + const job = await tx.materialPurgeJob.update({ where: { id: rows[0].id }, data: { + status: 'processing', leaseToken, leaseExpiresAt: new Date(clock.now.getTime() + LEASE_MS), + attempts: { increment: 1 }, errorMessage: null, + } }) + await tx.materialPurgeAudit.create({ data: { jobId: job.id, event: 'processing' } }) + return job + }) +} + +async function updateOwnedJob(job: MaterialPurgeJob, data: Prisma.MaterialPurgeJobUpdateManyMutationInput) { + const updated = await prisma.$executeRaw` + UPDATE material_purge_jobs SET lease_expires_at = clock_timestamp() + interval '10 minutes' + WHERE id = ${job.id} AND status = 'processing' AND lease_token = ${job.leaseToken} + AND lease_expires_at > clock_timestamp()` + if (!updated) throw new Error('文件清理任务已移交，等待新的处理结果') + await prisma.$transaction(async tx => { + const result = await tx.materialPurgeJob.updateMany({ + where: { id: job.id, status: 'processing', leaseToken: job.leaseToken }, data: { ...data, updatedAt: new Date() }, + }) + if (!result.count) throw new Error('文件清理任务已移交，等待新的处理结果') + if (data.status === 'completed') await tx.materialPurgeAudit.create({ data: { jobId: job.id, event: 'completed' } }) + }) +} + +/** 每个文件的结果单独落库，重复删除缺失文件视为成功，可从进程中断处续跑。 */ +export async function processNextMaterialPurge(): Promise { + const job = await claimJob() + if (!job) return false + const files = job.files as unknown as PurgeFile[] + let leaseLost = false + const heartbeat = setInterval(() => { + void updateOwnedJob(job, {}).catch(() => { leaseLost = true }) + }, 30_000) + heartbeat.unref?.() + try { + const references = await findPurgeFileReferences(prisma) + for (const file of files) { + if (file.outcome !== 'pending') continue + if (leaseLost) throw new Error('文件清理任务已移交，等待新的处理结果') + await updateOwnedJob(job, {}) + if (references.has(purgeFileIdentity(file))) file.outcome = 'shared' + else { await removePurgeFile(file); file.outcome = 'deleted' } + await updateOwnedJob(job, { files: files as unknown as Prisma.InputJsonValue }) + } + await updateOwnedJob(job, { status: 'completed', completedAt: new Date(), leaseToken: null, leaseExpiresAt: null }) + } catch (error: any) { + const retry = job.attempts < MAX_ATTEMPTS + const message = error instanceof Error && /^(文件|存储|对象存储)/.test(error.message) + ? error.message : '暂时无法清理文件，请稍后重试' + await prisma.$transaction(async tx => { + const result = await tx.materialPurgeJob.updateMany({ + where: { id: job.id, status: 'processing', leaseToken: job.leaseToken }, + data: { + status: retry ? 'queued' : 'failed', errorMessage: message, + nextAttemptAt: new Date(Date.now() + job.attempts * 30_000), leaseToken: null, leaseExpiresAt: null, + }, + }) + if (result.count) await tx.materialPurgeAudit.create({ data: { jobId: job.id, event: retry ? 'retry_scheduled' : 'failed', message } }) + }) + } finally { clearInterval(heartbeat) } + return true +} + +export async function retryMaterialPurge(jobId: string, operatorUserId: string) { + if (typeof jobId !== 'string' || !UUID.test(jobId)) throw new MaterialPurgeError('清理任务无效', 400) + return prisma.$transaction(async tx => { + const result = await tx.materialPurgeJob.updateMany({ where: { id: jobId, status: 'failed' }, data: { + status: 'queued', attempts: 0, errorMessage: null, nextAttemptAt: new Date(), + } }) + if (!result.count) throw new MaterialPurgeError('该任务已在处理或已经完成，请刷新状态') + await tx.materialPurgeAudit.create({ data: { jobId, event: 'manual_retry', operatorUserId } }) + return toPurgeJobView(await tx.materialPurgeJob.findUniqueOrThrow({ where: { id: jobId } })) + }) +} + +export function ensureMaterialPurgeWorkerStarted() { + if (process.env.NEXT_BUILD === '1' || process.env.NEXT_PHASE === 'phase-production-build') return + const global = globalThis as typeof globalThis & { __materialPurgeWorkerStarted?: boolean } + if (global.__materialPurgeWorkerStarted) return + global.__materialPurgeWorkerStarted = true + let running = false + const pump = async () => { + if (running) return + running = true + try { for (let count = 0; count < 20 && await processNextMaterialPurge(); count += 1) { /* 限定每轮批量 */ } } + catch (error) { console.error('[MaterialPurge] 清理队列暂不可用，将在下一轮重试:', error) } + finally { running = false } + } + setInterval(() => { void pump() }, 5_000).unref?.() + void pump() +} diff --git a/prisma/migrations/20261009_material_purge_outbox/migration.sql b/prisma/migrations/20261009_material_purge_outbox/migration.sql new file mode 100644 index 0000000..01d9b1e --- /dev/null +++ b/prisma/migrations/20261009_material_purge_outbox/migration.sql @@ -0,0 +1,37 @@ +-- 日志的 materialId 保持历史标识，不随素材级联删除。 +ALTER TABLE "material_logs" DROP CONSTRAINT IF EXISTS "material_logs_materialId_fkey"; +ALTER TABLE "material_logs" DROP CONSTRAINT IF EXISTS "MaterialLog_materialId_fkey"; +ALTER TABLE "material_download_logs" DROP CONSTRAINT IF EXISTS "material_download_logs_materialId_fkey"; +ALTER TABLE "material_download_logs" DROP CONSTRAINT IF EXISTS "MaterialDownloadLog_materialId_fkey"; + +CREATE TABLE "material_purge_jobs" ( + "id" TEXT NOT NULL PRIMARY KEY, + "material_id" TEXT NOT NULL, + "operator_user_id" TEXT NOT NULL, + "material_snapshot" JSONB NOT NULL, + "files" JSONB NOT NULL, + "status" TEXT NOT NULL DEFAULT 'queued', + "attempts" INTEGER NOT NULL DEFAULT 0, + "error_message" TEXT, + "lease_token" TEXT, + "lease_expires_at" TIMESTAMP(3), + "next_attempt_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + "completed_at" TIMESTAMP(3), + CONSTRAINT "material_purge_jobs_status_check" CHECK ("status" IN ('queued', 'processing', 'failed', 'completed')), + CONSTRAINT "material_purge_jobs_attempts_check" CHECK ("attempts" >= 0) +); +CREATE UNIQUE INDEX "material_purge_jobs_material_id_key" ON "material_purge_jobs"("material_id"); +CREATE INDEX "material_purge_jobs_status_next_attempt_at_idx" ON "material_purge_jobs"("status", "next_attempt_at"); + +CREATE TABLE "material_purge_audits" ( + "id" TEXT NOT NULL PRIMARY KEY, + "job_id" TEXT NOT NULL, + "event" TEXT NOT NULL, + "operator_user_id" TEXT, + "message" TEXT, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "material_purge_audits_job_id_fkey" FOREIGN KEY ("job_id") REFERENCES "material_purge_jobs"("id") ON DELETE RESTRICT ON UPDATE CASCADE +); +CREATE INDEX "material_purge_audits_job_id_created_at_idx" ON "material_purge_audits"("job_id", "created_at"); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 4292e61..60bdaba 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -138,10 +138,8 @@ model Material { /// 不再每次去资产表现场判断是不是上传空壳。 isIncompleteShell Boolean @default(false) @map("is_incomplete_shell") assets MaterialAsset[] - downloadLogs MaterialDownloadLog[] favorites MaterialFavorite[] likes MaterialLike[] - logs MaterialLog[] skus MaterialSku[] transfers MaterialTransfer[] transcodeProgresses VideoTranscodeProgress[] @relation("MaterialTranscodeProgress") @@ -302,7 +300,6 @@ model MaterialDownloadLog { materialId String userId String downloadedAt DateTime @default(now()) - material Material @relation(fields: [materialId], references: [id], onDelete: Cascade) user User @relation(fields: [userId], references: [id], onDelete: Cascade) @@index([materialId]) @@ -319,7 +316,6 @@ model MaterialLog { action String changes String? timestamp DateTime @default(now()) - material Material @relation(fields: [materialId], references: [id], onDelete: Cascade) user User @relation(fields: [userId], references: [id], onDelete: Cascade) @@index([materialId]) @@ -328,6 +324,41 @@ model MaterialLog { @@map("material_logs") } +// 永久删除的持久化凭证；不关联 Material，素材删除后仍保留清理清单与操作人。 +model MaterialPurgeJob { + id String @id @default(uuid()) + materialId String @unique @map("material_id") + operatorUserId String @map("operator_user_id") + materialSnapshot Json @map("material_snapshot") + files Json + status String @default("queued") + attempts Int @default(0) + errorMessage String? @map("error_message") + leaseToken String? @map("lease_token") + leaseExpiresAt DateTime? @map("lease_expires_at") + nextAttemptAt DateTime @default(now()) @map("next_attempt_at") + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + completedAt DateTime? @map("completed_at") + audits MaterialPurgeAudit[] + + @@index([status, nextAttemptAt]) + @@map("material_purge_jobs") +} + +model MaterialPurgeAudit { + id String @id @default(uuid()) + jobId String @map("job_id") + event String + operatorUserId String? @map("operator_user_id") + message String? + createdAt DateTime @default(now()) @map("created_at") + job MaterialPurgeJob @relation(fields: [jobId], references: [id], onDelete: Restrict) + + @@index([jobId, createdAt]) + @@map("material_purge_audits") +} + model MaterialSku { id String @id @default(uuid()) materialId String @map("material_id") diff --git a/scripts/git-incremental-deploy.ps1 b/scripts/git-incremental-deploy.ps1 index 3be5b01..29d59d4 100644 --- a/scripts/git-incremental-deploy.ps1 +++ b/scripts/git-incremental-deploy.ps1 @@ -1523,6 +1523,7 @@ const expectedMigrationNames = new Set([ '20260924_add_certificate_tested_on', '20261008_permission_group_department_identity', '20261008_team_function_configs', + '20261009_material_purge_outbox', ]); if (migrations.length !== expectedMigrationNames.size || migrations.some((item) => !expectedMigrationNames.has(item && item.name) diff --git a/scripts/hard-delete-history-materials.ts b/scripts/hard-delete-history-materials.ts index 0925fc9..cab028f 100644 --- a/scripts/hard-delete-history-materials.ts +++ b/scripts/hard-delete-history-materials.ts @@ -75,7 +75,7 @@ async function loadCandidates(candidatePath: string) { } async function loadMaterialSnapshot(materialId: string) { - return prisma.material.findUnique({ + const material = await prisma.material.findUnique({ where: { id: materialId }, include: { assets: { @@ -84,13 +84,21 @@ async function loadMaterialSnapshot(materialId: string) { }, }, skus: true, - logs: true, likes: true, favorites: true, - downloadLogs: true, transfers: true, }, }) + + if (!material) return null + + // Deletion history is retained independently from the material record. + const [logs, downloadLogs] = await Promise.all([ + prisma.materialLog.findMany({ where: { materialId } }), + prisma.materialDownloadLog.findMany({ where: { materialId } }), + ]) + + return { ...material, logs, downloadLogs } } function hasHistoryEvidence(material: Awaited>, candidate: DeleteCandidate) { diff --git a/scripts/safe-production-migrate.mjs b/scripts/safe-production-migrate.mjs index 8de1185..170b1c3 100644 --- a/scripts/safe-production-migrate.mjs +++ b/scripts/safe-production-migrate.mjs @@ -39,6 +39,7 @@ const TARGET_MIGRATIONS = Object.freeze([ '20260924_add_certificate_tested_on', '20261008_permission_group_department_identity', '20261008_team_function_configs', + '20261009_material_purge_outbox', ]) const VERIFIED_BACKUP_GENERATOR = 'scripts/create-verified-production-backup.mjs' const VERIFIED_BACKUP_MAINTENANCE_CONFIRMATION = 'NO_WRITES_DURING_SNAPSHOT' @@ -351,7 +352,7 @@ async function queryEvidence(prisma, localMigrations) { OR (table_name = 'materials' AND column_name IN ('custody_department_id', 'custody_at', 'custody_reason', 'is_incomplete_shell', 'task_detail_demand_type', 'task_detail_private', 'report_kind', 'certificate_expires_on', 'certificate_tested_on', 'certificate_number', 'certificate_usage_instructions', 'report_usage_rules')) OR (table_name = 'product_package_upload_jobs' AND column_name IN ('worker_id', 'lease_token', 'lease_expires_at')) OR (table_name = 'material_permission_pack_items' AND column_name IN ('can_access_private')) - OR table_name IN ('material_permission_pack_departments', 'team_function_configs', 'team_function_config_audits') + OR table_name IN ('material_permission_pack_departments', 'team_function_configs', 'team_function_config_audits', 'material_purge_jobs', 'material_purge_audits') OR (table_name = 'api_keys' AND column_name IN ('secret_ciphertext', 'secret_nonce', 'secret_auth_tag')) OR (table_name = 'api_key_reveal_audits' AND column_name IN ('id', 'api_key_id', 'viewer_user_id', 'created_at')) OR (table_name = 'system_settings' AND column_name IN ('headerLogoUrl', 'loginLogoUrl', 'logoUrl')) @@ -418,7 +419,8 @@ async function queryEvidence(prisma, localMigrations) { 'performance_conversion_metric_date_idx', 'performance_conversion_sync_states_lease_idx' ,'material_permission_pack_dept_unit_key', 'material_permission_pack_departments_pack_id_idx', - 'team_function_configs_organization_unit_id_key', 'team_function_config_audits_config_id_created_at_idx' + 'team_function_configs_organization_unit_id_key', 'team_function_config_audits_config_id_created_at_idx', + 'material_purge_jobs_material_id_key', 'material_purge_jobs_status_next_attempt_at_idx', 'material_purge_audits_job_id_created_at_idx' ) ORDER BY idx.relname`, ) @@ -450,7 +452,7 @@ async function queryEvidence(prisma, localMigrations) { LEFT JOIN pg_class ref ON ref.oid = con.confrelid JOIN pg_namespace ns ON ns.oid = cls.relnamespace WHERE ns.nspname = current_schema() - AND con.conname IN ( + AND (con.conname IN ( 'materials_private_optimization_check', 'brand_knowledge_likes_knowledge_id_fkey', 'brand_knowledge_likes_user_id_fkey', @@ -458,8 +460,9 @@ async function queryEvidence(prisma, localMigrations) { 'brand_knowledge_favorites_user_id_fkey' ,'material_permission_pack_departments_organization_unit_id_fkey', 'team_function_configs_organization_unit_id_fkey', 'team_function_configs_lead_employee_profile_id_fkey', - 'team_function_config_audits_config_id_fkey', 'team_function_configs_revision_check' - ) + 'team_function_config_audits_config_id_fkey', 'team_function_configs_revision_check', + 'material_purge_audits_job_id_fkey' + ) OR (cls.relname IN ('material_logs', 'material_download_logs') AND ref.relname = 'materials')) ORDER BY con.conname`, ) const brandRows = await prisma.$queryRawUnsafe( @@ -854,6 +857,38 @@ export function assertTargetMigrationEvidence(localMigrations, evidence) { } } assertOrganizationConfigurationEvidence(evidence) + assertMaterialPurgeEvidence(evidence) +} + +export function assertMaterialPurgeEvidence(evidence) { + const columns = new Map(evidence.columns.map(row => [`${row.table_name}.${row.column_name}`, row])) + const job = 'material_purge_jobs' + const audit = 'material_purge_audits' + for (const [table, column, type, nullable] of [ + ...['id', 'material_id', 'operator_user_id', 'status'].map(column => [job, column, 'text', 'NO']), + ...['material_snapshot', 'files'].map(column => [job, column, 'jsonb', 'NO']), + [job, 'attempts', 'integer', 'NO'], + ...['error_message', 'lease_token'].map(column => [job, column, 'text', 'YES']), + ...['lease_expires_at', 'completed_at'].map(column => [job, column, 'timestamp without time zone', 'YES']), + ...['next_attempt_at', 'created_at', 'updated_at'].map(column => [job, column, 'timestamp without time zone', 'NO']), + ...['id', 'job_id', 'event'].map(column => [audit, column, 'text', 'NO']), + ...['operator_user_id', 'message'].map(column => [audit, column, 'text', 'YES']), + [audit, 'created_at', 'timestamp without time zone', 'NO'], + ]) { + const row = columns.get(`${table}.${column}`) + if (!row || row.data_type !== type || row.is_nullable !== nullable) fail(`Database column mismatch: ${table}.${column}`) + } + const uniqueName = 'material_purge_jobs_material_id_key' + assertExactUniqueBtreeIndex(evidence.indexes.find(row => row.indexname === uniqueName), job, uniqueName, ['material_id']) + for (const [table, name, fields] of [ + [job, 'material_purge_jobs_status_next_attempt_at_idx', ['status', 'next_attempt_at']], + [audit, 'material_purge_audits_job_id_created_at_idx', ['job_id', 'created_at']], + ]) assertExactBtreeIndex(evidence.indexes.find(row => row.indexname === name)?.indexdef, table, fields) + const fk = 'material_purge_audits_job_id_fkey' + assertExactForeignKeyConstraint(evidence.constraints.find(row => row.constraint_name === fk), audit, fk, 'job_id', job, 'id') + if (evidence.constraints.some(row => ['material_logs', 'material_download_logs'].includes(row.tablename) && row.referenced_table === 'materials')) { + fail('Material audit logs must survive permanent deletion; material foreign key is still present') + } } export function assertOrganizationConfigurationEvidence(evidence) { diff --git a/scripts/safe-production-migrate.test.mjs b/scripts/safe-production-migrate.test.mjs index 0d25af2..8d762a8 100644 --- a/scripts/safe-production-migrate.test.mjs +++ b/scripts/safe-production-migrate.test.mjs @@ -14,6 +14,7 @@ import { assertExpectedDatabaseIdentity, assertTargetMigrationEvidence, assertOrganizationConfigurationEvidence, + assertMaterialPurgeEvidence, databaseFingerprintHash, normalizeColumnDefault, requireExactCommit, @@ -43,6 +44,38 @@ function organizationEvidence() { } } +function purgeEvidence() { + const job = 'material_purge_jobs' + const audit = 'material_purge_audits' + const column = (table_name, column_name, data_type, is_nullable) => ({ table_name, column_name, data_type, is_nullable }) + return { + columns: [ + ...['id', 'material_id', 'operator_user_id', 'status'].map(name => column(job, name, 'text', 'NO')), + ...['material_snapshot', 'files'].map(name => column(job, name, 'jsonb', 'NO')), + column(job, 'attempts', 'integer', 'NO'), + ...['error_message', 'lease_token'].map(name => column(job, name, 'text', 'YES')), + ...['lease_expires_at', 'completed_at'].map(name => column(job, name, 'timestamp without time zone', 'YES')), + ...['next_attempt_at', 'created_at', 'updated_at'].map(name => column(job, name, 'timestamp without time zone', 'NO')), + ...['id', 'job_id', 'event'].map(name => column(audit, name, 'text', 'NO')), + ...['operator_user_id', 'message'].map(name => column(audit, name, 'text', 'YES')), + column(audit, 'created_at', 'timestamp without time zone', 'NO'), + ], + indexes: [ + { tablename: job, indexname: 'material_purge_jobs_material_id_key', indisunique: true, indisprimary: false, indisvalid: true, indisready: true, access_method: 'btree', predicate: '', columns: ['material_id'] }, + { indexname: 'material_purge_jobs_status_next_attempt_at_idx', indexdef: 'CREATE INDEX material_purge_jobs_status_next_attempt_at_idx ON material_purge_jobs USING btree (status, next_attempt_at)' }, + { indexname: 'material_purge_audits_job_id_created_at_idx', indexdef: 'CREATE INDEX material_purge_audits_job_id_created_at_idx ON material_purge_audits USING btree (job_id, created_at)' }, + ], + constraints: [{ tablename: audit, constraint_name: 'material_purge_audits_job_id_fkey', contype: 'f', convalidated: true, referenced_table: job, columns: ['job_id'], referenced_columns: ['id'] }], + } +} + +test('purge verification requires durable manifests, uniqueness, retry index and preserved material logs', () => { + const evidence = purgeEvidence() + assert.doesNotThrow(() => assertMaterialPurgeEvidence(evidence)) + assert.throws(() => assertMaterialPurgeEvidence({ ...evidence, columns: evidence.columns.filter(row => row.column_name !== 'files') }), /column mismatch/) + assert.throws(() => assertMaterialPurgeEvidence({ ...evidence, constraints: [...evidence.constraints, { tablename: 'material_logs', referenced_table: 'materials' }] }), /logs must survive/) +}) + test('organization configuration verification rejects missing identity uniqueness, invalid audit foreign key and nonpositive revision', () => { const evidence = organizationEvidence() assert.doesNotThrow(() => assertOrganizationConfigurationEvidence(evidence)) @@ -386,6 +419,7 @@ test('target evidence rejects nullable logo columns and structurally wrong index ['20260924_add_certificate_tested_on', { checksum: 'ab' }], ['20261008_permission_group_department_identity', { checksum: 'ac' }], ['20261008_team_function_configs', { checksum: 'ad' }], + ['20261009_material_purge_outbox', { checksum: 'ae' }], ]) const evidence = { migrationRows: [ @@ -656,8 +690,10 @@ test('target evidence rejects nullable logo columns and structurally wrong index { id: 'custom', headerLogoUrl: 'https://s3.jiansun.vip/gongxiangpan/header.png', loginLogoUrl: 'https://s3.jiansun.vip/gongxiangpan/login.png' }, ], } + evidence.migrationRows.push({ migration_name: '20261009_material_purge_outbox', checksum: 'ae', finished_at: 'now', rolled_back_at: null }) const organization = organizationEvidence() - for (const key of ['columns', 'indexes', 'constraints']) evidence[key].push(...organization[key]) + const purge = purgeEvidence() + for (const key of ['columns', 'indexes', 'constraints']) evidence[key].push(...organization[key], ...purge[key]) assert.doesNotThrow(() => assertTargetMigrationEvidence(migrations, evidence)) const nullableLogo = { ...evidence, diff --git a/scripts/test-material-purge-integration.ts b/scripts/test-material-purge-integration.ts new file mode 100644 index 0000000..79868da --- /dev/null +++ b/scripts/test-material-purge-integration.ts @@ -0,0 +1,178 @@ +/** 仅使用独立的本机 PostgreSQL 和临时上传目录，不读取生产配置。 */ +import assert from 'node:assert/strict' +import { randomUUID } from 'node:crypto' +import { promises as fs } from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +async function main() { + const url = new URL(process.env.DATABASE_URL || '') + assert(['127.0.0.1', 'localhost'].includes(url.hostname) && url.port === '55439' && /^\/material_purge(?:_migration)?_qa$/.test(url.pathname), '拒绝连接非独立验收数据库') + process.env.NEXT_BUILD = '1' + process.env.NODE_ENV = 'production' + const s3Enabled = process.argv.includes('--s3') + if (s3Enabled) { + Object.assign(process.env, { S3_ENDPOINT: 'http://127.0.0.1:59000', S3_PUBLIC_DOMAIN: 'http://127.0.0.1:59000/purge-qa', + S3_BUCKET: 'purge-qa', S3_FILE_PATH: 'project-purge-qa', S3_ACCESS_KEY_ID: 'purgeqa', S3_SECRET_ACCESS_KEY: 'purge_local_storage_qa' }) + } + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'gongxiangp2-purge-integration-')) + process.env.UPLOAD_DIR = root + // 每次使用独立数据库，原生 SQL 与 Prisma 使用同一个 public schema。 + const { PrismaClient } = await import('@prisma/client') + const database = `material_purge_qa_${randomUUID().replace(/-/g, '')}` + const setup = new PrismaClient({ datasourceUrl: url.toString() }) + await setup.$executeRawUnsafe(`CREATE DATABASE "${database}"`) + await setup.$disconnect() + url.pathname = `/${database}` + url.searchParams.set('schema', 'public') + process.env.DATABASE_URL = url.toString() + const { spawnSync } = await import('node:child_process') + const pushed = spawnSync(process.execPath, ['node_modules/prisma/build/index.js', 'db', 'push', '--skip-generate'], { encoding: 'utf8', env: process.env }) + assert.equal(pushed.status, 0, pushed.stderr) + // 重建迁移前的日志级联关系，在空的专用数据库中执行真实 reviewed SQL。 + const migrationSql = await fs.readFile('prisma/migrations/20261009_material_purge_outbox/migration.sql', 'utf8') + const baselineSql = `DROP TABLE material_purge_audits; DROP TABLE material_purge_jobs; + ALTER TABLE material_logs ADD CONSTRAINT "material_logs_materialId_fkey" FOREIGN KEY ("materialId") REFERENCES materials(id) ON DELETE CASCADE; + ALTER TABLE material_download_logs ADD CONSTRAINT "material_download_logs_materialId_fkey" FOREIGN KEY ("materialId") REFERENCES materials(id) ON DELETE CASCADE;` + const migrated = spawnSync('docker', ['exec', '-i', 'gongxiangp2-purge-qa-20261009', 'psql', '-U', 'postgres', '-d', database, '-v', 'ON_ERROR_STOP=1'], { + encoding: 'utf8', input: `${baselineSql}\n${migrationSql}`, + }) + assert.equal(migrated.status, 0, migrated.stderr) + const { prisma } = await import('../lib/prisma') + const { enqueueMaterialPurge, processNextMaterialPurge, retryMaterialPurge } = await import('../lib/material-purge') + const run = randomUUID() + const passed: string[] = [] + passed.push('真实升级 SQL 创建清理表、重试索引与审计表，并解除历史日志的素材级联删除') + const admin = await prisma.user.create({ data: { email: `${run}@purge.test`, username: run, password: 'local-qa-only', role: 'ADMIN' } }) + const make = async (name: string, extra: any = {}) => prisma.material.create({ data: { + productModel: `${run}-${name}`, previewUrl: 'data:image/svg+xml;base64,Zm9v', categoryName: '验收', + sceneName: '验收', materialType: '标准模型', authorName: '隔离验收', userId: admin.id, deletedAt: new Date(), ...extra, + } }) + const file = async (name: string) => { await fs.writeFile(path.join(root, name), 'isolated content'); return `uploads/${name}` } + const exists = async (name: string) => fs.stat(path.join(root, name)).then(() => true, () => false) + try { + const active = await make('active', { deletedAt: null }) + const bin = await make('atomic', { zipFilePath: await file('atomic.bin') }) + await assert.rejects(enqueueMaterialPurge([bin.id, active.id], admin.id), /只能彻底删除/) + assert(await prisma.material.findUnique({ where: { id: bin.id } })) + assert.equal(await prisma.materialPurgeJob.count({ where: { materialId: bin.id } }), 0) + assert(await exists('atomic.bin')) + passed.push('混合批量整批拒绝，素材/文件/任务均无部分提交') + + await assert.rejects(enqueueMaterialPurge([bin.id], 'missing-operator'), /Foreign key/) + assert(await prisma.material.findUnique({ where: { id: bin.id } })) + assert.equal(await prisma.materialPurgeJob.count({ where: { materialId: bin.id } }), 0) + passed.push('审计写入失败时整个删除事务回滚') + + const uploading = await make('uploading') + await prisma.productPackageUploadJob.create({ data: { materialId: uploading.id, operatorUserId: admin.id, assets: [], status: 'queued', stage: 'queued' } }) + await assert.rejects(enqueueMaterialPurge([uploading.id], admin.id), /仍在处理中/) + const transcoding = await make('transcoding', { transcodeLeaseToken: 'active-lease' }) + await assert.rejects(enqueueMaterialPurge([transcoding.id], admin.id), /仍在处理中/) + passed.push('上传/转码未结束的素材拒绝进入清理队列') + + const own = await make('single', { zipFilePath: await file('single.bin') }) + await prisma.materialAsset.create({ data: { materialId: own.id, assetType: 'arbitrary-type', fileUrl: await file('asset.bin') } }) + await prisma.materialSku.create({ data: { materialId: own.id, skuName: 'test', filePath: await file('sku.bin') } }) + await prisma.materialLog.create({ data: { materialId: own.id, userId: admin.id, action: 'UPLOAD' } }) + await prisma.materialDownloadLog.create({ data: { materialId: own.id, userId: admin.id } }) + const requests = await Promise.all([enqueueMaterialPurge([own.id, own.id], admin.id), enqueueMaterialPurge([own.id], admin.id)]) + assert.equal(requests[0][0].id, requests[1][0].id) + assert.equal(await prisma.materialPurgeJob.count({ where: { materialId: own.id } }), 1) + assert.equal(await prisma.materialLog.count({ where: { materialId: own.id } }), 2) + assert.equal(await prisma.materialDownloadLog.count({ where: { materialId: own.id } }), 1) + assert.equal(await prisma.materialAsset.count({ where: { materialId: own.id } }), 0) + assert.equal(await prisma.materialSku.count({ where: { materialId: own.id } }), 0) + assert(await exists('single.bin')) + await processNextMaterialPurge() + assert(!await exists('single.bin') && !await exists('asset.bin') && !await exists('sku.bin')) + assert.equal((await prisma.materialPurgeJob.findUniqueOrThrow({ where: { materialId: own.id } })).status, 'completed') + passed.push('并发/重复提交幂等；关联记录清理，旧日志和下载日志保留；所有类型资产纳入清单') + + const sharedSource = await file('shared.bin') + const keeper = await make('shared-keeper', { previewUrl: '/uploads/shared.bin' }) + const sharer = await make('shared-delete', { zipFilePath: sharedSource }) + await enqueueMaterialPurge([sharer.id], admin.id) + await processNextMaterialPurge() + assert(await exists('shared.bin')) + const sharedJob = await prisma.materialPurgeJob.findUniqueOrThrow({ where: { materialId: sharer.id } }) + assert.equal((sharedJob.files as any)[0].outcome, 'shared') + assert(await prisma.material.findUnique({ where: { id: keeper.id } })) + passed.push('URL 等价引用与回收站引用保护共用文件') + + const batch = await Promise.all([make('batch-a', { zipFilePath: await file('batch-a.bin') }), make('batch-b', { zipFilePath: await file('batch-b.bin') })]) + const batchJobs = await enqueueMaterialPurge(batch.map(material => material.id), admin.id) + assert.equal(batchJobs.length, 2) + // 模拟作业进程中断：持久化处理中状态和已过期租约，下一轮自动接管。 + await prisma.materialPurgeJob.update({ where: { id: batchJobs[0].id }, data: { status: 'processing', leaseToken: 'interrupted-process', leaseExpiresAt: new Date(0) } }) + await Promise.all([processNextMaterialPurge(), processNextMaterialPurge()]) + assert(!await exists('batch-a.bin') && !await exists('batch-b.bin')) + assert.equal(await prisma.materialPurgeJob.count({ where: { id: { in: batchJobs.map(job => job.id) }, status: 'completed' } }), 2) + passed.push('批量作业并发领取不重复，重启后过期租约可接管并完成') + + const broken = await make('retry', { zipFilePath: await file('retry-first.bin'), displayImages: [await file('retry-second.bin')] }) + const [retryJob] = await enqueueMaterialPurge([broken.id], admin.id) + await fs.rename(path.join(root, 'retry-second.bin'), path.join(root, 'saved-second.bin')) + await fs.mkdir(path.join(root, 'retry-second.bin')) + for (let attempt = 0; attempt < 3; attempt += 1) { + await prisma.materialPurgeJob.update({ where: { id: retryJob.id }, data: { nextAttemptAt: new Date(0) } }) + await processNextMaterialPurge() + } + let failed = await prisma.materialPurgeJob.findUniqueOrThrow({ where: { id: retryJob.id } }) + assert.equal(failed.status, 'failed') + assert(!await exists('retry-first.bin')) + assert.equal((failed.files as any)[0].outcome, 'deleted') + await fs.rmdir(path.join(root, 'retry-second.bin')) + await fs.rename(path.join(root, 'saved-second.bin'), path.join(root, 'retry-second.bin')) + await retryMaterialPurge(retryJob.id, admin.id) + await processNextMaterialPurge() + failed = await prisma.materialPurgeJob.findUniqueOrThrow({ where: { id: retryJob.id } }) + assert.equal(failed.status, 'completed') + assert(!await exists('retry-second.bin')) + const audits = await prisma.materialPurgeAudit.findMany({ where: { jobId: retryJob.id }, orderBy: { createdAt: 'asc' } }) + assert(audits.some(audit => audit.event === 'failed')) + assert(audits.some(audit => audit.event === 'manual_retry' && audit.operatorUserId === admin.id)) + passed.push('部分文件失败保存进度，自动重试耗尽后人工续跑；失败/重试操作人审计完整') + + if (s3Enabled) { + const { S3Client, CreateBucketCommand, PutObjectCommand, HeadObjectCommand } = await import('@aws-sdk/client-s3') + const client = new S3Client({ endpoint: process.env.S3_ENDPOINT, region: 'us-east-1', forcePathStyle: true, + credentials: { accessKeyId: process.env.S3_ACCESS_KEY_ID!, secretAccessKey: process.env.S3_SECRET_ACCESS_KEY! }, maxAttempts: 1 }) + const Bucket = process.env.S3_BUCKET! + try { await client.send(new CreateBucketCommand({ Bucket })) } catch (error: any) { if (error.name !== 'BucketAlreadyOwnedByYou') throw error } + const ownKey = `project-purge-qa/${run}/owned.bin` + const sharedKey = `project-purge-qa/${run}/shared.bin` + const foreignKey = `other-project/${run}/sentinel.bin` + for (const Key of [ownKey, sharedKey, foreignKey]) await client.send(new PutObjectCommand({ Bucket, Key, Body: 'isolated-object' })) + const source = (key: string) => `http://127.0.0.1:59000/${Bucket}/${key}` + const objectMaterial = await make('object-own', { zipFilePath: source(ownKey) }) + await enqueueMaterialPurge([objectMaterial.id], admin.id) + await processNextMaterialPurge() + await assert.rejects(client.send(new HeadObjectCommand({ Bucket, Key: ownKey })), error => (error as any).$metadata.httpStatusCode === 404) + const objectKeeper = await make('object-keeper', { previewUrl: `${source(sharedKey)}?cache=keeper` }) + const objectShared = await make('object-shared', { zipFilePath: source(sharedKey) }) + await enqueueMaterialPurge([objectShared.id], admin.id) + await processNextMaterialPurge() + await client.send(new HeadObjectCommand({ Bucket, Key: sharedKey })) + assert(await prisma.material.findUnique({ where: { id: objectKeeper.id } })) + const objectForeign = await make('object-foreign', { zipFilePath: source(foreignKey) }) + await assert.rejects(enqueueMaterialPurge([objectForeign.id], admin.id), /业务前缀之外/) + await client.send(new HeadObjectCommand({ Bucket, Key: foreignKey })) + assert(await prisma.material.findUnique({ where: { id: objectForeign.id } })) + passed.push('真实 MinIO 验证：本项目对象删除成功，共用对象保留，其他项目前缀拒绝删除') + client.destroy() + } + + const output = { database: url.pathname.slice(1), passed, checks: passed.length, at: new Date().toISOString() } + await fs.mkdir('tmp/material-purge-qa', { recursive: true }) + await fs.writeFile('tmp/material-purge-qa/integration.json', JSON.stringify(output, null, 2)) + console.log(JSON.stringify(output, null, 2)) + } finally { + await prisma.$disconnect() + // root 是本次 mkdtemp 创建的目录，仅移除这次隔离文件。 + assert(path.dirname(root) === os.tmpdir() && path.basename(root).startsWith('gongxiangp2-purge-integration-')) + await fs.rm(root, { recursive: true, force: true }) + } +} + +main().catch(error => { console.error(error); process.exitCode = 1 })