928a7b37a9
Persist image submissions with fixed idempotency keys, leased execution, provider identity, reference snapshots and staged results. Integrate DIY, assisted assembly and non-integrated white-background sessions without holding generation in a browser request. Recover ambiguous submissions through owner-scoped reads and stop a new eight-view batch when restoring an earlier request. Add the NativeImageGenerationJob migration and a dedicated worker to both release entry points. Protect job inputs/results from storage GC. Existing production configuration and business data are not changed by this commit. Validation: 256 focused Jest tests, isolated PostgreSQL migration/lease/transaction checks (8), deployment guard tests (9), type-check, production build and desktop/ mobile browser component scenarios. Live native 2K reference-image request worked; reusing its idempotency key returned the same task. Build uses local placeholders for otherwise absent Drawing System configuration. Three unrelated material tests also fail with the original mainline test file; they remain outside this change. Architecture/visual/root-cause review: passed with warnings. Independent review completed for the integration and shared input/storage paths; worker additionally reviewed by the primary agent. Production merge/deployment requires authorization.
239 lines
17 KiB
TypeScript
239 lines
17 KiB
TypeScript
/** @jest-environment node */
|
|
import { nativeImageGenerationWorker } from '../native-image-generation.worker'
|
|
import { nativeImageGenerationRepository as repo } from '@/server/repositories/native-image-generation.repository'
|
|
import { providerConfigService } from '../provider-config.service'
|
|
import { nativeImageProviderFingerprint } from '../native-image-generation.service'
|
|
import { createRelayImageTask, getRelayImageTaskResult, RelayImageTaskError } from '../relay-image-task.client'
|
|
import { completeNativeRenderGeneration, failNativeRenderGeneration, markNativeRenderBlockedGeneration } from '../native-render-completion.service'
|
|
import { persistNativeImageOutputs } from '../native-image-files'
|
|
import { AppError, ErrorCode } from '@/server/core/errors'
|
|
|
|
jest.mock('@/server/repositories/native-image-generation.repository', () => ({ nativeImageGenerationRepository: { claimNext: jest.fn(), saveClaim: jest.fn() } }))
|
|
jest.mock('../provider-config.service', () => ({ providerConfigService: { getById: jest.fn() } }))
|
|
jest.mock('../native-image-generation.service', () => ({ nativeImageProviderFingerprint: jest.fn(() => 'fingerprint') }))
|
|
jest.mock('../relay-image-task.client', () => {
|
|
const actual = jest.requireActual('../relay-image-task.client')
|
|
return { ...actual, createRelayImageTask: jest.fn(), getRelayImageTaskResult: jest.fn() }
|
|
})
|
|
jest.mock('../native-render-completion.service', () => ({ completeNativeRenderGeneration: jest.fn(), failNativeRenderGeneration: jest.fn(), markNativeRenderBlockedGeneration: jest.fn() }))
|
|
jest.mock('../native-image-files', () => ({ persistNativeImageOutputs: jest.fn() }))
|
|
|
|
const makeJob = (overrides = {}) => ({
|
|
id: 'job', createdBy: 'owner', status: 'QUEUED', retryCount: 0, createdAt: new Date(), providerConfigId: 'provider', providerFingerprint: 'fingerprint',
|
|
request: { model: 'image-model', imageCount: 1, prompt: 'cabinet', aspectRatio: '16:9', imageSize: '2K', referenceImageUrls: ['https://images.example/ref.png'] },
|
|
progress: [{ idempotencyKey: 'fixed-key' }], completion: { kind: 'render', renderId: 'render', source: 'diy' }, ...overrides,
|
|
})
|
|
beforeEach(() => {
|
|
jest.clearAllMocks()
|
|
jest.mocked(repo.saveClaim).mockResolvedValue(undefined)
|
|
jest.mocked(providerConfigService.getById).mockResolvedValue({ baseUrl: 'https://relay.example', apiKey: 'test-key' } as never)
|
|
jest.mocked(nativeImageProviderFingerprint).mockReturnValue('fingerprint')
|
|
jest.mocked(createRelayImageTask).mockResolvedValue({ taskId: 'remote-1' })
|
|
jest.mocked(getRelayImageTaskResult).mockResolvedValue({ status: 'pending' })
|
|
jest.mocked(completeNativeRenderGeneration).mockResolvedValue(undefined)
|
|
jest.mocked(failNativeRenderGeneration).mockResolvedValue(undefined)
|
|
jest.mocked(markNativeRenderBlockedGeneration).mockResolvedValue(undefined)
|
|
jest.mocked(persistNativeImageOutputs).mockImplementation(async (_job, _task, images) => images)
|
|
})
|
|
test('stores remote ID before finalization, then a restarted worker polls without another create', async () => {
|
|
jest.mocked(repo.claimNext).mockResolvedValue(makeJob() as never)
|
|
await nativeImageGenerationWorker.runNext()
|
|
const saved = jest.mocked(repo.saveClaim).mock.calls.at(-1)![2]
|
|
expect(saved.progress).toEqual([{ idempotencyKey: 'fixed-key', taskId: 'remote-1', lastAttemptedAt: expect.any(String) }])
|
|
expect(completeNativeRenderGeneration).not.toHaveBeenCalled()
|
|
jest.mocked(repo.claimNext).mockResolvedValue(makeJob({ status: 'RUNNING', progress: saved.progress }) as never)
|
|
await nativeImageGenerationWorker.runNext()
|
|
expect(createRelayImageTask).toHaveBeenCalledTimes(1)
|
|
expect(getRelayImageTaskResult).toHaveBeenCalledWith(expect.anything(), 'remote-1')
|
|
})
|
|
test('unknown POST outcome retries with exactly the original key and payload', async () => {
|
|
jest.mocked(repo.claimNext).mockImplementation(async () => makeJob() as never)
|
|
jest.mocked(createRelayImageTask).mockRejectedValueOnce(new RelayImageTaskError('network', 'unknown', { retryable: true, outcomeUnknown: true }))
|
|
await nativeImageGenerationWorker.runNext()
|
|
await nativeImageGenerationWorker.runNext()
|
|
expect(jest.mocked(createRelayImageTask).mock.calls.map(call => call[2])).toEqual(['fixed-key', 'fixed-key'])
|
|
expect(jest.mocked(createRelayImageTask).mock.calls[0]![1]).toEqual(jest.mocked(createRelayImageTask).mock.calls[1]![1])
|
|
})
|
|
test('completed results survive finalizer failure without regenerating', async () => {
|
|
const progress = [{ idempotencyKey: 'fixed-key', taskId: 'remote-1', images: [{ originalUrl: 'https://images.example/result.png' }] }]
|
|
jest.mocked(repo.claimNext).mockResolvedValue(makeJob({ status: 'FINALIZING', progress }) as never)
|
|
jest.mocked(completeNativeRenderGeneration).mockRejectedValueOnce(new Error('storage unavailable'))
|
|
await nativeImageGenerationWorker.runNext()
|
|
await nativeImageGenerationWorker.runNext()
|
|
expect(completeNativeRenderGeneration).toHaveBeenCalledTimes(2)
|
|
expect(createRelayImageTask).not.toHaveBeenCalled()
|
|
expect(getRelayImageTaskResult).not.toHaveBeenCalled()
|
|
expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ status: 'SUCCEEDED' }))
|
|
})
|
|
test('credential changes block a job instead of submitting to a different account', async () => {
|
|
jest.mocked(repo.claimNext).mockResolvedValue(makeJob() as never)
|
|
jest.mocked(nativeImageProviderFingerprint).mockReturnValue('changed')
|
|
await nativeImageGenerationWorker.runNext()
|
|
expect(createRelayImageTask).not.toHaveBeenCalled()
|
|
expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ status: 'BLOCKED' }))
|
|
})
|
|
test('mixed outcomes retain successful images and expose partial failure', async () => {
|
|
const job = makeJob({ status: 'FINALIZING', progress: [
|
|
{ idempotencyKey: 'one', taskId: 'remote-1', images: [{ originalUrl: 'https://images.example/result.png' }] },
|
|
{ idempotencyKey: 'two', taskId: 'remote-2', errorMessage: 'generation failed' },
|
|
] })
|
|
job.request.imageCount = 2
|
|
jest.mocked(repo.claimNext).mockResolvedValue(job as never)
|
|
await nativeImageGenerationWorker.runNext()
|
|
expect(completeNativeRenderGeneration).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ images: [{ originalUrl: 'https://images.example/result.png' }], attempts: expect.arrayContaining([expect.objectContaining({ status: 'failed' })]) }))
|
|
expect(failNativeRenderGeneration).not.toHaveBeenCalled()
|
|
})
|
|
|
|
test('persists BLOCKED and releases its lease even when the business notification throws', async () => {
|
|
jest.mocked(repo.claimNext).mockResolvedValue(makeJob() as never)
|
|
jest.mocked(nativeImageProviderFingerprint).mockReturnValue('changed')
|
|
jest.mocked(markNativeRenderBlockedGeneration).mockImplementation(async () => {
|
|
expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({
|
|
status: 'BLOCKED', leaseOwner: null, leaseUntil: null,
|
|
}))
|
|
throw new Error('product was deleted')
|
|
})
|
|
const warning = jest.spyOn(console, 'warn').mockImplementation(() => {})
|
|
try {
|
|
await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'blocked' })
|
|
expect(createRelayImageTask).not.toHaveBeenCalled()
|
|
expect(warning).toHaveBeenCalledWith('[native-ai.business-status-pending]', { jobId: 'job' })
|
|
} finally { warning.mockRestore() }
|
|
})
|
|
|
|
test('pending review retains the remote ID and blocks instead of resubmitting', async () => {
|
|
jest.mocked(repo.claimNext).mockResolvedValue(makeJob({
|
|
status: 'RUNNING', progress: [{ idempotencyKey: 'fixed-key', taskId: 'remote-1' }],
|
|
}) as never)
|
|
jest.mocked(getRelayImageTaskResult).mockResolvedValue({ status: 'pending', needsReview: true })
|
|
await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'blocked' })
|
|
expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({
|
|
status: 'BLOCKED', progress: [expect.objectContaining({ taskId: 'remote-1', idempotencyKey: 'fixed-key', needsReview: true })],
|
|
}))
|
|
expect(createRelayImageTask).not.toHaveBeenCalled()
|
|
expect(completeNativeRenderGeneration).not.toHaveBeenCalled()
|
|
})
|
|
|
|
test('retrieves and archives other successful images before blocking a mixed review job', async () => {
|
|
const job = makeJob({ status: 'RUNNING', progress: [
|
|
{ idempotencyKey: 'one', taskId: 'remote-review' },
|
|
{ idempotencyKey: 'two', taskId: 'remote-success' },
|
|
] })
|
|
job.request.imageCount = 2
|
|
jest.mocked(repo.claimNext).mockResolvedValue(job as never)
|
|
jest.mocked(getRelayImageTaskResult).mockResolvedValueOnce({ status: 'pending', needsReview: true })
|
|
.mockResolvedValueOnce({ status: 'succeeded', images: [{ originalUrl: 'https://upstream.example/result.png' }] })
|
|
jest.mocked(persistNativeImageOutputs).mockResolvedValue([{ originalUrl: 'https://project.example/native-ai-results/result.png' }])
|
|
await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'processed' })
|
|
const savedFirst = jest.mocked(repo.saveClaim).mock.calls.at(-1)![2]
|
|
expect(savedFirst.status).toBe('RUNNING')
|
|
jest.mocked(repo.claimNext).mockResolvedValue({ ...job, progress: savedFirst.progress } as never)
|
|
await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'blocked' })
|
|
expect(jest.mocked(getRelayImageTaskResult).mock.calls.map(call => call[1])).toEqual(['remote-review', 'remote-success'])
|
|
expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({
|
|
status: 'BLOCKED', progress: [
|
|
expect.objectContaining({ taskId: 'remote-review', needsReview: true }),
|
|
expect.objectContaining({ taskId: 'remote-success', images: [{ originalUrl: 'https://project.example/native-ai-results/result.png' }] }),
|
|
],
|
|
}))
|
|
expect(createRelayImageTask).not.toHaveBeenCalled()
|
|
expect(completeNativeRenderGeneration).not.toHaveBeenCalled()
|
|
})
|
|
|
|
test('rotates pending polls so the first image cannot starve later images', async () => {
|
|
const job = makeJob({ status: 'RUNNING', progress: [
|
|
{ idempotencyKey: 'one', taskId: 'remote-1' },
|
|
{ idempotencyKey: 'two', taskId: 'remote-2' },
|
|
] })
|
|
job.request.imageCount = 2
|
|
jest.mocked(repo.claimNext).mockResolvedValue(job as never)
|
|
await nativeImageGenerationWorker.runNext()
|
|
const saved = jest.mocked(repo.saveClaim).mock.calls.at(-1)![2]
|
|
jest.mocked(repo.claimNext).mockResolvedValue({ ...job, progress: saved.progress } as never)
|
|
await nativeImageGenerationWorker.runNext()
|
|
expect(jest.mocked(getRelayImageTaskResult).mock.calls.map(call => call[1])).toEqual(['remote-1', 'remote-2'])
|
|
})
|
|
|
|
test('an uncertain first submission does not prevent attempting the second image with its own fixed key', async () => {
|
|
const job = makeJob({ progress: [{ idempotencyKey: 'one' }, { idempotencyKey: 'two' }] })
|
|
job.request.imageCount = 2
|
|
jest.mocked(repo.claimNext).mockResolvedValue(job as never)
|
|
jest.mocked(createRelayImageTask).mockRejectedValueOnce(new RelayImageTaskError('network', 'unknown', { retryable: true, outcomeUnknown: true }))
|
|
await nativeImageGenerationWorker.runNext()
|
|
const saved = jest.mocked(repo.saveClaim).mock.calls.at(-1)![2]
|
|
jest.mocked(repo.claimNext).mockResolvedValue({ ...job, progress: saved.progress } as never)
|
|
await nativeImageGenerationWorker.runNext()
|
|
expect(jest.mocked(createRelayImageTask).mock.calls.map(call => call[2])).toEqual(['one', 'two'])
|
|
})
|
|
|
|
test('output archive failure retries GET for the existing task and never creates another generation', async () => {
|
|
const job = makeJob({ status: 'RUNNING', progress: [{ idempotencyKey: 'fixed-key', taskId: 'remote-1' }] })
|
|
jest.mocked(repo.claimNext).mockResolvedValue(job as never)
|
|
jest.mocked(getRelayImageTaskResult).mockResolvedValue({ status: 'succeeded', images: [{ originalUrl: 'https://upstream.example/result.png' }] })
|
|
jest.mocked(persistNativeImageOutputs).mockRejectedValueOnce(new Error('storage offline'))
|
|
.mockResolvedValueOnce([{ originalUrl: 'https://project.example/native-ai-results/result.png' }])
|
|
await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'retry' })
|
|
const failed = jest.mocked(repo.saveClaim).mock.calls.at(-1)![2]
|
|
expect(failed.progress).toEqual([expect.objectContaining({ taskId: 'remote-1' })])
|
|
expect(failed.progress).not.toEqual([expect.objectContaining({ images: expect.anything() })])
|
|
jest.mocked(repo.claimNext).mockResolvedValue({ ...job, progress: failed.progress } as never)
|
|
await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'processed' })
|
|
expect(createRelayImageTask).not.toHaveBeenCalled()
|
|
expect(jest.mocked(getRelayImageTaskResult).mock.calls.map(call => call[1])).toEqual(['remote-1', 'remote-1'])
|
|
expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ status: 'FINALIZING' }))
|
|
})
|
|
|
|
test('pending work older than 24 hours blocks without another remote operation', async () => {
|
|
jest.mocked(repo.claimNext).mockResolvedValue(makeJob({ createdAt: new Date(Date.now() - 24 * 60 * 60 * 1000) }) as never)
|
|
await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'blocked' })
|
|
expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ status: 'BLOCKED', errorMessage: expect.stringContaining('24 小时') }))
|
|
expect(createRelayImageTask).not.toHaveBeenCalled()
|
|
expect(getRelayImageTaskResult).not.toHaveBeenCalled()
|
|
})
|
|
|
|
test('finished work still archives beyond the 24-hour generation limit', async () => {
|
|
jest.mocked(repo.claimNext).mockResolvedValue(makeJob({
|
|
status: 'FINALIZING', createdAt: new Date(Date.now() - 48 * 60 * 60 * 1000),
|
|
progress: [{ idempotencyKey: 'fixed-key', taskId: 'remote-1', images: [{ originalUrl: 'https://project.example/result.png' }] }],
|
|
}) as never)
|
|
await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'finalized' })
|
|
expect(completeNativeRenderGeneration).toHaveBeenCalledTimes(1)
|
|
expect(markNativeRenderBlockedGeneration).not.toHaveBeenCalled()
|
|
expect(createRelayImageTask).not.toHaveBeenCalled()
|
|
})
|
|
|
|
test('known terminal failure retries its business projection before marking the durable job FAILED', async () => {
|
|
jest.mocked(repo.claimNext).mockResolvedValue(makeJob({ status: 'FINALIZING', progress: [{ idempotencyKey: 'fixed-key', errorMessage: 'generation rejected' }] }) as never)
|
|
jest.mocked(failNativeRenderGeneration).mockRejectedValueOnce(new Error('database unavailable'))
|
|
await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'retry' })
|
|
expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ retryCount: 1, leaseOwner: null, leaseUntil: null }))
|
|
expect(jest.mocked(repo.saveClaim).mock.calls.at(-1)![2]).not.toHaveProperty('status', 'FAILED')
|
|
await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'finalized' })
|
|
expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ status: 'FAILED', leaseOwner: null, leaseUntil: null }))
|
|
})
|
|
|
|
test('context-validation failure keeps retrying until the business failed state is saved', async () => {
|
|
jest.mocked(repo.claimNext).mockResolvedValue(makeJob({ status: 'FINALIZING', progress: [{ idempotencyKey: 'fixed-key', images: [{ originalUrl: 'https://project.example/result.png' }] }] }) as never)
|
|
jest.mocked(completeNativeRenderGeneration).mockRejectedValue(new AppError(ErrorCode.VALIDATION_FAILED, 'source changed'))
|
|
jest.mocked(failNativeRenderGeneration).mockRejectedValueOnce(new Error('database unavailable'))
|
|
await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'retry' })
|
|
expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ retryCount: 1, leaseOwner: null, leaseUntil: null }))
|
|
await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'failed' })
|
|
expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ status: 'FAILED', leaseOwner: null, leaseUntil: null }))
|
|
})
|
|
|
|
test('a lost lease during provider lookup prevents a subsequent generation request', async () => {
|
|
jest.useFakeTimers()
|
|
let releaseLookup!: (value: never) => void
|
|
jest.mocked(repo.claimNext).mockResolvedValue(makeJob() as never)
|
|
jest.mocked(providerConfigService.getById).mockImplementation(() => new Promise(resolve => { releaseLookup = resolve }))
|
|
jest.mocked(repo.saveClaim).mockRejectedValue(new Error('lease transferred'))
|
|
const work = nativeImageGenerationWorker.runNext()
|
|
try {
|
|
await jest.advanceTimersByTimeAsync(30_000)
|
|
releaseLookup({ baseUrl: 'https://relay.example', apiKey: 'test-key' } as never)
|
|
await expect(work).resolves.toMatchObject({ status: 'lease-lost' })
|
|
expect(createRelayImageTask).not.toHaveBeenCalled()
|
|
} finally { jest.useRealTimers() }
|
|
})
|