465 lines
18 KiB
TypeScript
465 lines
18 KiB
TypeScript
/** @jest-environment node */
|
||
|
||
import type { Prisma } from '@prisma/client'
|
||
import { prisma } from '@/lib/prisma'
|
||
import {
|
||
__taskDetailUploadJobTesting,
|
||
acquireTaskDetailUploadLock,
|
||
createTaskDetailUploadJob,
|
||
recoverTaskDetailUploadJobs,
|
||
TaskDetailUploadQueueCapacityError,
|
||
triggerTaskDetailUploadJob,
|
||
} from '@/lib/task-detail-upload-jobs'
|
||
import { completeProductPackageUpload } from '@/lib/product-package-completion'
|
||
import {
|
||
acquireTaskDetailUploadExecutionLock,
|
||
ensureTaskDetailUploadDrainCapability,
|
||
isTaskDetailUploadDraining,
|
||
removeTaskDetailUploadActiveMarker,
|
||
writeTaskDetailUploadActiveMarker,
|
||
} from '@/lib/task-detail-upload-runtime'
|
||
|
||
jest.mock('@/lib/prisma', () => ({
|
||
prisma: {
|
||
$transaction: jest.fn(),
|
||
$queryRawUnsafe: jest.fn(),
|
||
productPackageUploadJob: {
|
||
findFirst: jest.fn(),
|
||
findUnique: jest.fn(),
|
||
findMany: jest.fn(),
|
||
count: jest.fn(),
|
||
create: jest.fn(),
|
||
updateMany: jest.fn(),
|
||
},
|
||
materialAsset: {
|
||
count: jest.fn(),
|
||
},
|
||
material: {
|
||
update: jest.fn(),
|
||
updateMany: jest.fn(),
|
||
},
|
||
},
|
||
}))
|
||
|
||
jest.mock('@/lib/task-detail-upload-runtime', () => ({
|
||
acquireTaskDetailUploadExecutionLock: jest.fn(),
|
||
ensureTaskDetailUploadDrainCapability: jest.fn().mockResolvedValue(undefined),
|
||
getTaskDetailUploadWorkerIdentity: jest.fn(() => ({ workerId: 'worker-test', instanceId: 'instance-test' })),
|
||
isTaskDetailUploadDraining: jest.fn().mockResolvedValue(false),
|
||
removeTaskDetailUploadActiveMarker: jest.fn().mockResolvedValue(undefined),
|
||
writeTaskDetailUploadActiveMarker: jest.fn().mockResolvedValue(undefined),
|
||
}))
|
||
|
||
jest.mock('@/lib/product-package-completion', () => ({
|
||
assertTaskDetailUploadAssetLimits: jest.fn(),
|
||
assertOwnedProductPackageUploadSessions: jest.fn(),
|
||
completeProductPackageUpload: jest.fn(),
|
||
}))
|
||
|
||
jest.mock('@/lib/upload-sessions', () => ({
|
||
refreshOwnedUploadSessionTtls: jest.fn().mockResolvedValue(undefined),
|
||
}))
|
||
|
||
const now = new Date('2026-08-01T00:00:00.000Z')
|
||
const releaseExecutionLock = Object.assign(jest.fn(async () => undefined), {
|
||
assertHeld: jest.fn(),
|
||
})
|
||
|
||
function job(overrides: Record<string, unknown> = {}) {
|
||
return {
|
||
id: 'job-1',
|
||
materialId: 'material-1',
|
||
operatorUserId: 'user-1',
|
||
taskNo: null,
|
||
dingtalkApprovalNo: null,
|
||
assets: { main: [] },
|
||
options: null,
|
||
status: 'queued',
|
||
stage: 'queued',
|
||
progress: 0,
|
||
errorMessage: null,
|
||
createdAt: now,
|
||
startedAt: null,
|
||
completedAt: null,
|
||
updatedAt: now,
|
||
...overrides,
|
||
}
|
||
}
|
||
|
||
function transactionClient(options: { activeCount?: number; claimedCount?: number } = {}) {
|
||
return {
|
||
$queryRaw: jest.fn().mockResolvedValue([{ locked: 1 }]),
|
||
$queryRawUnsafe: jest.fn().mockResolvedValue([{ now }]),
|
||
productPackageUploadJob: {
|
||
count: jest.fn()
|
||
.mockResolvedValueOnce(options.activeCount ?? 0)
|
||
.mockResolvedValue(1),
|
||
updateMany: jest.fn().mockResolvedValue({ count: options.claimedCount ?? 1 }),
|
||
findFirst: jest.fn(),
|
||
create: jest.fn(),
|
||
},
|
||
material: {
|
||
update: jest.fn(),
|
||
updateMany: jest.fn().mockResolvedValue({ count: 1 }),
|
||
},
|
||
} as any
|
||
}
|
||
|
||
async function flushMicrotasks(rounds = 8) {
|
||
for (let index = 0; index < rounds; index += 1) await Promise.resolve()
|
||
}
|
||
|
||
describe('task-detail upload queue', () => {
|
||
beforeEach(() => {
|
||
jest.clearAllMocks()
|
||
releaseExecutionLock.mockReset().mockResolvedValue(undefined)
|
||
releaseExecutionLock.assertHeld.mockReset()
|
||
__taskDetailUploadJobTesting.resetRuntimeState()
|
||
;(prisma.productPackageUploadJob.findMany as jest.Mock).mockResolvedValue([])
|
||
;(prisma.productPackageUploadJob.updateMany as jest.Mock).mockResolvedValue({ count: 1 })
|
||
;(prisma.material.updateMany as jest.Mock).mockResolvedValue({ count: 1 })
|
||
;(prisma.materialAsset.count as jest.Mock).mockResolvedValue(0)
|
||
;(completeProductPackageUpload as jest.Mock).mockResolvedValue(undefined)
|
||
;(prisma.$queryRawUnsafe as jest.Mock).mockResolvedValue([{ now }])
|
||
;(ensureTaskDetailUploadDrainCapability as jest.Mock).mockResolvedValue(undefined)
|
||
;(acquireTaskDetailUploadExecutionLock as jest.Mock).mockResolvedValue(releaseExecutionLock)
|
||
;(isTaskDetailUploadDraining as jest.Mock).mockResolvedValue(false)
|
||
;(writeTaskDetailUploadActiveMarker as jest.Mock).mockResolvedValue(undefined)
|
||
;(removeTaskDetailUploadActiveMarker as jest.Mock).mockResolvedValue(undefined)
|
||
})
|
||
|
||
afterEach(() => {
|
||
__taskDetailUploadJobTesting.resetRuntimeState()
|
||
jest.useRealTimers()
|
||
jest.restoreAllMocks()
|
||
})
|
||
|
||
test('advisory lock 只返回 Prisma 可反序列化的整数列', async () => {
|
||
const tx = {
|
||
$queryRaw: jest.fn().mockResolvedValue([{ locked: 1 }]),
|
||
} as unknown as Prisma.TransactionClient
|
||
|
||
await acquireTaskDetailUploadLock(tx, 'product-package-upload-create')
|
||
|
||
const [template, lockName] = (tx.$queryRaw as jest.Mock).mock.calls[0]
|
||
const sql = Array.from(template as TemplateStringsArray).join('?')
|
||
expect(sql).toContain('SELECT 1 AS locked')
|
||
expect(sql).toContain('pg_advisory_xact_lock(hashtext(?)')
|
||
expect(sql.trim()).not.toMatch(/^SELECT\s+pg_advisory_xact_lock/i)
|
||
expect(lockName).toBe('product-package-upload-create')
|
||
})
|
||
|
||
test('重复 create 复用同一任务且不新增队列记录', async () => {
|
||
const existing = job({ status: 'completed', stage: 'completed', progress: 100, completedAt: now })
|
||
const tx = transactionClient()
|
||
tx.productPackageUploadJob.findFirst.mockResolvedValue(existing)
|
||
;(prisma.$transaction as jest.Mock).mockImplementation(async (callback) => callback(tx))
|
||
;(prisma.productPackageUploadJob.findUnique as jest.Mock).mockResolvedValue(existing)
|
||
|
||
const input = {
|
||
materialId: 'material-1',
|
||
operatorUserId: 'user-1',
|
||
assets: { main: [] },
|
||
}
|
||
const first = await createTaskDetailUploadJob(input)
|
||
const second = await createTaskDetailUploadJob(input)
|
||
await flushMicrotasks()
|
||
|
||
expect(first.jobId).toBe('job-1')
|
||
expect(second.jobId).toBe('job-1')
|
||
expect(tx.productPackageUploadJob.create).not.toHaveBeenCalled()
|
||
expect(tx.productPackageUploadJob.count).not.toHaveBeenCalled()
|
||
})
|
||
|
||
test.each([
|
||
['user', [3], TaskDetailUploadQueueCapacityError, 429],
|
||
['global', [2, 20], TaskDetailUploadQueueCapacityError, 503],
|
||
])('等待队列 %s 容量满时拒绝创建', async (scope, counts, errorType, statusCode) => {
|
||
const tx = transactionClient()
|
||
tx.productPackageUploadJob.findFirst.mockResolvedValue(null)
|
||
tx.productPackageUploadJob.count.mockReset().mockImplementation(async () => counts.shift() ?? 0)
|
||
;(prisma.$transaction as jest.Mock).mockImplementation(async (callback) => callback(tx))
|
||
|
||
await expect(createTaskDetailUploadJob({
|
||
materialId: `material-${scope}`,
|
||
operatorUserId: 'user-1',
|
||
assets: { main: [] },
|
||
})).rejects.toMatchObject({
|
||
constructor: errorType,
|
||
scope,
|
||
statusCode,
|
||
})
|
||
expect(tx.productPackageUploadJob.create).not.toHaveBeenCalled()
|
||
})
|
||
|
||
test('跨容器领取在 advisory lock 内检查全局 processing 并使用 queued 条件更新', async () => {
|
||
const firstTx = transactionClient({ activeCount: 0, claimedCount: 1 })
|
||
const secondTx = transactionClient({ activeCount: 1, claimedCount: 1 })
|
||
;(prisma.$transaction as jest.Mock)
|
||
.mockImplementationOnce(async (callback) => callback(firstTx))
|
||
.mockImplementationOnce(async (callback) => callback(secondTx))
|
||
|
||
await expect(__taskDetailUploadJobTesting.claimTaskDetailUploadJob('job-1')).resolves.toEqual(
|
||
expect.objectContaining({ status: 'acquired', token: expect.any(String) }),
|
||
)
|
||
await expect(__taskDetailUploadJobTesting.claimTaskDetailUploadJob('job-2')).resolves.toEqual({ status: 'busy' })
|
||
|
||
expect(firstTx.productPackageUploadJob.updateMany).toHaveBeenCalledWith(expect.objectContaining({
|
||
where: { id: 'job-1', status: 'queued' },
|
||
data: expect.objectContaining({ status: 'processing' }),
|
||
}))
|
||
expect(secondTx.productPackageUploadJob.updateMany).not.toHaveBeenCalled()
|
||
})
|
||
|
||
test('已提交素材会直接补记 completed，不会重跑已清理的上传会话', async () => {
|
||
;(prisma.productPackageUploadJob.findUnique as jest.Mock).mockResolvedValue({
|
||
id: 'job-1',
|
||
status: 'processing',
|
||
materialId: 'material-1',
|
||
material: { taskStatus: 'completed', zipFilePath: 'all.zip' },
|
||
})
|
||
;(prisma.materialAsset.count as jest.Mock).mockResolvedValue(3)
|
||
|
||
await expect(
|
||
__taskDetailUploadJobTesting.reconcileCommittedTaskDetailUploadJob('job-1'),
|
||
).resolves.toBe(true)
|
||
|
||
expect(prisma.productPackageUploadJob.updateMany).toHaveBeenCalledWith(expect.objectContaining({
|
||
data: expect.objectContaining({ status: 'completed', progress: 100 }),
|
||
}))
|
||
expect(completeProductPackageUpload).not.toHaveBeenCalled()
|
||
})
|
||
|
||
test('worker 处理期间每 30 秒写一次 heartbeat，完成后清除并写 completed', async () => {
|
||
jest.useFakeTimers()
|
||
const tx = transactionClient({ activeCount: 0, claimedCount: 1 })
|
||
;(prisma.$transaction as jest.Mock).mockImplementation(async (callback) => callback(tx))
|
||
;(prisma.productPackageUploadJob.findUnique as jest.Mock)
|
||
.mockResolvedValueOnce({
|
||
id: 'job-1',
|
||
status: 'queued',
|
||
materialId: 'material-1',
|
||
material: { taskStatus: 'uploading', zipFilePath: '' },
|
||
})
|
||
.mockResolvedValueOnce(job())
|
||
let finish!: () => void
|
||
;(completeProductPackageUpload as jest.Mock).mockImplementation(() => new Promise<void>((resolve) => {
|
||
finish = resolve
|
||
}))
|
||
|
||
const processing = __taskDetailUploadJobTesting.processOneTaskDetailUploadJob('job-1')
|
||
await flushMicrotasks()
|
||
await jest.advanceTimersByTimeAsync(30_000)
|
||
|
||
expect(tx.productPackageUploadJob.updateMany).toHaveBeenCalledWith(expect.objectContaining({
|
||
where: expect.objectContaining({
|
||
id: 'job-1',
|
||
status: 'processing',
|
||
leaseToken: expect.any(String),
|
||
}),
|
||
data: expect.objectContaining({ leaseExpiresAt: expect.any(Date) }),
|
||
}))
|
||
|
||
finish()
|
||
await expect(processing).resolves.toBe('processed')
|
||
expect(tx.productPackageUploadJob.updateMany).toHaveBeenCalledWith(expect.objectContaining({
|
||
data: expect.objectContaining({ status: 'completed', progress: 100 }),
|
||
}))
|
||
expect(acquireTaskDetailUploadExecutionLock).toHaveBeenCalledTimes(1)
|
||
expect(releaseExecutionLock.assertHeld).toHaveBeenCalled()
|
||
expect(releaseExecutionLock).toHaveBeenCalledTimes(1)
|
||
})
|
||
|
||
test('active marker 落盘后才出现 drain 时释放租约并保持 queued，不开始重处理', async () => {
|
||
const tx = transactionClient({ activeCount: 0, claimedCount: 1 })
|
||
;(prisma.$transaction as jest.Mock).mockImplementation(async (callback) => callback(tx))
|
||
;(prisma.productPackageUploadJob.findUnique as jest.Mock)
|
||
.mockResolvedValueOnce({
|
||
id: 'job-1',
|
||
status: 'queued',
|
||
materialId: 'material-1',
|
||
material: { taskStatus: 'uploading', zipFilePath: '' },
|
||
})
|
||
.mockResolvedValueOnce(job())
|
||
;(isTaskDetailUploadDraining as jest.Mock)
|
||
.mockResolvedValueOnce(false)
|
||
.mockResolvedValueOnce(true)
|
||
|
||
await expect(
|
||
__taskDetailUploadJobTesting.processOneTaskDetailUploadJob('job-1'),
|
||
).resolves.toBe('busy')
|
||
|
||
expect(writeTaskDetailUploadActiveMarker).toHaveBeenCalledTimes(1)
|
||
expect(completeProductPackageUpload).not.toHaveBeenCalled()
|
||
expect(tx.productPackageUploadJob.updateMany).toHaveBeenCalledWith(expect.objectContaining({
|
||
where: expect.objectContaining({ id: 'job-1', leaseToken: expect.any(String) }),
|
||
data: expect.objectContaining({
|
||
status: 'queued',
|
||
stage: 'queued',
|
||
leaseToken: null,
|
||
leaseExpiresAt: null,
|
||
}),
|
||
}))
|
||
expect(removeTaskDetailUploadActiveMarker).toHaveBeenCalledTimes(1)
|
||
expect(releaseExecutionLock).toHaveBeenCalledTimes(1)
|
||
})
|
||
|
||
test('heartbeat 续租失败后中止作业，不写 completed/failed 也不改素材失败状态', async () => {
|
||
jest.useFakeTimers()
|
||
const tx = transactionClient({ activeCount: 0, claimedCount: 1 })
|
||
tx.productPackageUploadJob.updateMany
|
||
.mockResolvedValueOnce({ count: 1 }) // claim
|
||
.mockResolvedValueOnce({ count: 0 }) // heartbeat lease renew lost
|
||
.mockResolvedValue({ count: 0 })
|
||
jest.spyOn(console, 'error').mockImplementation(() => undefined)
|
||
jest.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||
;(prisma.$transaction as jest.Mock).mockImplementation(async (callback) => callback(tx))
|
||
;(prisma.productPackageUploadJob.findUnique as jest.Mock)
|
||
.mockResolvedValueOnce({
|
||
id: 'job-1',
|
||
status: 'queued',
|
||
materialId: 'material-1',
|
||
material: { taskStatus: 'uploading', zipFilePath: '' },
|
||
})
|
||
.mockResolvedValueOnce(job())
|
||
let finish!: () => void
|
||
;(completeProductPackageUpload as jest.Mock).mockImplementation(() => new Promise<void>((resolve) => {
|
||
finish = resolve
|
||
}))
|
||
|
||
const processing = __taskDetailUploadJobTesting.processOneTaskDetailUploadJob('job-1')
|
||
await flushMicrotasks()
|
||
expect(releaseExecutionLock).not.toHaveBeenCalled()
|
||
await jest.advanceTimersByTimeAsync(30_000)
|
||
finish()
|
||
await expect(processing).resolves.toBe('processed')
|
||
|
||
const statusWrites = tx.productPackageUploadJob.updateMany.mock.calls
|
||
.map(([input]: [any]) => input?.data?.status)
|
||
.filter((status: unknown) => status === 'completed' || status === 'failed')
|
||
expect(statusWrites).toEqual([])
|
||
expect(prisma.material.updateMany).not.toHaveBeenCalled()
|
||
expect(tx.material.updateMany).not.toHaveBeenCalled()
|
||
const processorOptions = (completeProductPackageUpload as jest.Mock).mock.calls[0][0].options
|
||
expect(processorOptions.signal.aborted).toBe(true)
|
||
expect(releaseExecutionLock).toHaveBeenCalledTimes(1)
|
||
})
|
||
|
||
test('共享 uploads 执行锁不可用时只背压重试，不领租约也不运行处理器', async () => {
|
||
jest.spyOn(console, 'error').mockImplementation(() => undefined)
|
||
;(prisma.productPackageUploadJob.findUnique as jest.Mock).mockResolvedValueOnce({
|
||
id: 'job-1',
|
||
status: 'queued',
|
||
materialId: 'material-1',
|
||
material: { taskStatus: 'uploading', zipFilePath: '' },
|
||
})
|
||
;(acquireTaskDetailUploadExecutionLock as jest.Mock).mockRejectedValueOnce(
|
||
new Error('flock unavailable'),
|
||
)
|
||
|
||
await expect(
|
||
__taskDetailUploadJobTesting.processOneTaskDetailUploadJob('job-1'),
|
||
).resolves.toBe('busy')
|
||
|
||
expect(prisma.$transaction).not.toHaveBeenCalled()
|
||
expect(completeProductPackageUpload).not.toHaveBeenCalled()
|
||
expect(writeTaskDetailUploadActiveMarker).not.toHaveBeenCalled()
|
||
})
|
||
|
||
test('处理失败会写 failed 并同步素材状态', async () => {
|
||
jest.spyOn(console, 'error').mockImplementation(() => undefined)
|
||
const tx = transactionClient({ activeCount: 0, claimedCount: 1 })
|
||
;(prisma.$transaction as jest.Mock).mockImplementation(async (callback) => callback(tx))
|
||
;(prisma.productPackageUploadJob.findUnique as jest.Mock)
|
||
.mockResolvedValueOnce({
|
||
id: 'job-1',
|
||
status: 'queued',
|
||
materialId: 'material-1',
|
||
material: { taskStatus: 'uploading', zipFilePath: '' },
|
||
})
|
||
.mockResolvedValueOnce(job())
|
||
;(completeProductPackageUpload as jest.Mock).mockRejectedValue(new Error('恶意预览被拒绝'))
|
||
|
||
await expect(
|
||
__taskDetailUploadJobTesting.processOneTaskDetailUploadJob('job-1'),
|
||
).resolves.toBe('processed')
|
||
|
||
expect(tx.productPackageUploadJob.updateMany).toHaveBeenCalledWith(expect.objectContaining({
|
||
data: expect.objectContaining({
|
||
status: 'failed',
|
||
stage: 'failed',
|
||
errorMessage: '恶意预览被拒绝',
|
||
}),
|
||
}))
|
||
expect(tx.material.updateMany).toHaveBeenCalledWith({
|
||
where: { id: 'material-1', taskStatus: { not: 'completed' } },
|
||
data: { taskStatus: 'failed' },
|
||
})
|
||
})
|
||
|
||
test('leaseToken 有值但 leaseExpiresAt 为 null 时在 DB clock + advisory lock 内 CAS 回收', async () => {
|
||
;(prisma.productPackageUploadJob.findMany as jest.Mock)
|
||
.mockResolvedValueOnce([{ id: 'stale-1', materialId: 'material-stale', leaseToken: 'expired-token' }])
|
||
.mockResolvedValueOnce([])
|
||
;(prisma.productPackageUploadJob.findUnique as jest.Mock).mockResolvedValue({
|
||
id: 'stale-1',
|
||
status: 'processing',
|
||
materialId: 'material-stale',
|
||
material: { taskStatus: 'uploading', zipFilePath: '' },
|
||
})
|
||
|
||
const recoveryTx = transactionClient()
|
||
;(prisma.$transaction as jest.Mock).mockImplementation(async (callback) => callback(recoveryTx))
|
||
|
||
await expect(recoverTaskDetailUploadJobs()).resolves.toBe(0)
|
||
|
||
expect(recoveryTx.productPackageUploadJob.updateMany).toHaveBeenCalledWith(expect.objectContaining({
|
||
where: expect.objectContaining({
|
||
id: 'stale-1',
|
||
status: 'processing',
|
||
leaseToken: 'expired-token',
|
||
OR: [
|
||
{ leaseExpiresAt: { lte: now } },
|
||
{ leaseExpiresAt: null },
|
||
],
|
||
}),
|
||
data: expect.objectContaining({ status: 'queued', stage: 'queued', leaseToken: null }),
|
||
}))
|
||
expect(recoveryTx.$queryRaw).toHaveBeenCalled()
|
||
expect(recoveryTx.$queryRawUnsafe).toHaveBeenCalledWith('SELECT clock_timestamp() AS "now"')
|
||
expect(recoveryTx.material.updateMany).toHaveBeenCalledWith(expect.objectContaining({
|
||
where: { id: 'material-stale', taskStatus: { not: 'completed' } },
|
||
data: { taskStatus: 'uploading' },
|
||
}))
|
||
})
|
||
|
||
test('全局已有 processing 时 20 个 queued 只保留一个 retry timer，不形成 per-job 自旋', async () => {
|
||
jest.useFakeTimers()
|
||
const tx = transactionClient({ activeCount: 1, claimedCount: 0 })
|
||
;(prisma.$transaction as jest.Mock).mockImplementation(async (callback) => callback(tx))
|
||
;(prisma.productPackageUploadJob.findUnique as jest.Mock)
|
||
.mockResolvedValueOnce({
|
||
id: 'job-0',
|
||
status: 'queued',
|
||
materialId: 'material-0',
|
||
material: { taskStatus: 'uploading', zipFilePath: '' },
|
||
})
|
||
.mockResolvedValueOnce({ status: 'queued' })
|
||
|
||
for (let index = 0; index < 20; index += 1) triggerTaskDetailUploadJob(`job-${index}`)
|
||
await flushMicrotasks(20)
|
||
|
||
const state = __taskDetailUploadJobTesting.getRuntimeState()
|
||
expect(state.pendingJobIds).toHaveLength(20)
|
||
expect(state.retryScheduled).toBe(true)
|
||
expect(state.globallyBackpressured).toBe(true)
|
||
expect(jest.getTimerCount()).toBe(1)
|
||
|
||
// recovery/新请求在背压期间只能补 pending，不能再创建第二条重试链。
|
||
triggerTaskDetailUploadJob('job-extra')
|
||
await flushMicrotasks()
|
||
expect(__taskDetailUploadJobTesting.getRuntimeState().pendingJobIds).toHaveLength(21)
|
||
expect(jest.getTimerCount()).toBe(1)
|
||
})
|
||
})
|