/** @jest-environment node */ import { nativeImageGenerationWorker } from '../native-image-generation.worker' import { nativeImageGenerationRepository as repo } from '@/server/repositories/native-image-generation.repository' import { providerConfigService } from '../provider-config.service' import { nativeImageProviderFingerprint } from '../native-image-generation.service' import { createRelayImageTask, getRelayImageTaskResult, RelayImageTaskError } from '../relay-image-task.client' import { completeNativeRenderGeneration, failNativeRenderGeneration, markNativeRenderBlockedGeneration } from '../native-render-completion.service' import { persistNativeImageOutputs } from '../native-image-files' import { AppError, ErrorCode } from '@/server/core/errors' jest.mock('@/server/repositories/native-image-generation.repository', () => ({ nativeImageGenerationRepository: { claimNext: jest.fn(), saveClaim: jest.fn() } })) jest.mock('../provider-config.service', () => ({ providerConfigService: { getById: jest.fn() } })) jest.mock('../native-image-generation.service', () => ({ nativeImageProviderFingerprint: jest.fn(() => 'fingerprint') })) jest.mock('../relay-image-task.client', () => { const actual = jest.requireActual('../relay-image-task.client') return { ...actual, createRelayImageTask: jest.fn(), getRelayImageTaskResult: jest.fn() } }) jest.mock('../native-render-completion.service', () => ({ completeNativeRenderGeneration: jest.fn(), failNativeRenderGeneration: jest.fn(), markNativeRenderBlockedGeneration: jest.fn() })) jest.mock('../native-image-files', () => ({ persistNativeImageOutputs: jest.fn() })) const makeJob = (overrides = {}) => ({ id: 'job', createdBy: 'owner', status: 'QUEUED', retryCount: 0, createdAt: new Date(), providerConfigId: 'provider', providerFingerprint: 'fingerprint', request: { model: 'image-model', imageCount: 1, prompt: 'cabinet', aspectRatio: '16:9', imageSize: '2K', referenceImageUrls: ['https://images.example/ref.png'] }, progress: [{ idempotencyKey: 'fixed-key' }], completion: { kind: 'render', renderId: 'render', source: 'diy' }, ...overrides, }) beforeEach(() => { jest.clearAllMocks() jest.mocked(repo.saveClaim).mockResolvedValue(undefined) jest.mocked(providerConfigService.getById).mockResolvedValue({ baseUrl: 'https://relay.example', apiKey: 'test-key' } as never) jest.mocked(nativeImageProviderFingerprint).mockReturnValue('fingerprint') jest.mocked(createRelayImageTask).mockResolvedValue({ taskId: 'remote-1' }) jest.mocked(getRelayImageTaskResult).mockResolvedValue({ status: 'pending' }) jest.mocked(completeNativeRenderGeneration).mockResolvedValue(undefined) jest.mocked(failNativeRenderGeneration).mockResolvedValue(undefined) jest.mocked(markNativeRenderBlockedGeneration).mockResolvedValue(undefined) jest.mocked(persistNativeImageOutputs).mockImplementation(async (_job, _task, images) => images) }) test('stores remote ID before finalization, then a restarted worker polls without another create', async () => { jest.mocked(repo.claimNext).mockResolvedValue(makeJob() as never) await nativeImageGenerationWorker.runNext() const saved = jest.mocked(repo.saveClaim).mock.calls.at(-1)![2] expect(saved.progress).toEqual([{ idempotencyKey: 'fixed-key', taskId: 'remote-1', lastAttemptedAt: expect.any(String) }]) expect(completeNativeRenderGeneration).not.toHaveBeenCalled() jest.mocked(repo.claimNext).mockResolvedValue(makeJob({ status: 'RUNNING', progress: saved.progress }) as never) await nativeImageGenerationWorker.runNext() expect(createRelayImageTask).toHaveBeenCalledTimes(1) expect(getRelayImageTaskResult).toHaveBeenCalledWith(expect.anything(), 'remote-1') }) test('unknown POST outcome retries with exactly the original key and payload', async () => { jest.mocked(repo.claimNext).mockImplementation(async () => makeJob() as never) jest.mocked(createRelayImageTask).mockRejectedValueOnce(new RelayImageTaskError('network', 'unknown', { retryable: true, outcomeUnknown: true })) await nativeImageGenerationWorker.runNext() await nativeImageGenerationWorker.runNext() expect(jest.mocked(createRelayImageTask).mock.calls.map(call => call[2])).toEqual(['fixed-key', 'fixed-key']) expect(jest.mocked(createRelayImageTask).mock.calls[0]![1]).toEqual(jest.mocked(createRelayImageTask).mock.calls[1]![1]) }) test('completed results survive finalizer failure without regenerating', async () => { const progress = [{ idempotencyKey: 'fixed-key', taskId: 'remote-1', images: [{ originalUrl: 'https://images.example/result.png' }] }] jest.mocked(repo.claimNext).mockResolvedValue(makeJob({ status: 'FINALIZING', progress }) as never) jest.mocked(completeNativeRenderGeneration).mockRejectedValueOnce(new Error('storage unavailable')) await nativeImageGenerationWorker.runNext() await nativeImageGenerationWorker.runNext() expect(completeNativeRenderGeneration).toHaveBeenCalledTimes(2) expect(createRelayImageTask).not.toHaveBeenCalled() expect(getRelayImageTaskResult).not.toHaveBeenCalled() expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ status: 'SUCCEEDED' })) }) test('credential changes block a job instead of submitting to a different account', async () => { jest.mocked(repo.claimNext).mockResolvedValue(makeJob() as never) jest.mocked(nativeImageProviderFingerprint).mockReturnValue('changed') await nativeImageGenerationWorker.runNext() expect(createRelayImageTask).not.toHaveBeenCalled() expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ status: 'BLOCKED' })) }) test('mixed outcomes retain successful images and expose partial failure', async () => { const job = makeJob({ status: 'FINALIZING', progress: [ { idempotencyKey: 'one', taskId: 'remote-1', images: [{ originalUrl: 'https://images.example/result.png' }] }, { idempotencyKey: 'two', taskId: 'remote-2', errorMessage: 'generation failed' }, ] }) job.request.imageCount = 2 jest.mocked(repo.claimNext).mockResolvedValue(job as never) await nativeImageGenerationWorker.runNext() expect(completeNativeRenderGeneration).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ images: [{ originalUrl: 'https://images.example/result.png' }], attempts: expect.arrayContaining([expect.objectContaining({ status: 'failed' })]) })) expect(failNativeRenderGeneration).not.toHaveBeenCalled() }) test('persists BLOCKED and releases its lease even when the business notification throws', async () => { jest.mocked(repo.claimNext).mockResolvedValue(makeJob() as never) jest.mocked(nativeImageProviderFingerprint).mockReturnValue('changed') jest.mocked(markNativeRenderBlockedGeneration).mockImplementation(async () => { expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ status: 'BLOCKED', leaseOwner: null, leaseUntil: null, })) throw new Error('product was deleted') }) const warning = jest.spyOn(console, 'warn').mockImplementation(() => {}) try { await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'blocked' }) expect(createRelayImageTask).not.toHaveBeenCalled() expect(warning).toHaveBeenCalledWith('[native-ai.business-status-pending]', { jobId: 'job' }) } finally { warning.mockRestore() } }) test('pending review retains the remote ID and blocks instead of resubmitting', async () => { jest.mocked(repo.claimNext).mockResolvedValue(makeJob({ status: 'RUNNING', progress: [{ idempotencyKey: 'fixed-key', taskId: 'remote-1' }], }) as never) jest.mocked(getRelayImageTaskResult).mockResolvedValue({ status: 'pending', needsReview: true }) await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'blocked' }) expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ status: 'BLOCKED', progress: [expect.objectContaining({ taskId: 'remote-1', idempotencyKey: 'fixed-key', needsReview: true })], })) expect(createRelayImageTask).not.toHaveBeenCalled() expect(completeNativeRenderGeneration).not.toHaveBeenCalled() }) test('retrieves and archives other successful images before blocking a mixed review job', async () => { const job = makeJob({ status: 'RUNNING', progress: [ { idempotencyKey: 'one', taskId: 'remote-review' }, { idempotencyKey: 'two', taskId: 'remote-success' }, ] }) job.request.imageCount = 2 jest.mocked(repo.claimNext).mockResolvedValue(job as never) jest.mocked(getRelayImageTaskResult).mockResolvedValueOnce({ status: 'pending', needsReview: true }) .mockResolvedValueOnce({ status: 'succeeded', images: [{ originalUrl: 'https://upstream.example/result.png' }] }) jest.mocked(persistNativeImageOutputs).mockResolvedValue([{ originalUrl: 'https://project.example/native-ai-results/result.png' }]) await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'processed' }) const savedFirst = jest.mocked(repo.saveClaim).mock.calls.at(-1)![2] expect(savedFirst.status).toBe('RUNNING') jest.mocked(repo.claimNext).mockResolvedValue({ ...job, progress: savedFirst.progress } as never) await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'blocked' }) expect(jest.mocked(getRelayImageTaskResult).mock.calls.map(call => call[1])).toEqual(['remote-review', 'remote-success']) expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ status: 'BLOCKED', progress: [ expect.objectContaining({ taskId: 'remote-review', needsReview: true }), expect.objectContaining({ taskId: 'remote-success', images: [{ originalUrl: 'https://project.example/native-ai-results/result.png' }] }), ], })) expect(createRelayImageTask).not.toHaveBeenCalled() expect(completeNativeRenderGeneration).not.toHaveBeenCalled() }) test('rotates pending polls so the first image cannot starve later images', async () => { const job = makeJob({ status: 'RUNNING', progress: [ { idempotencyKey: 'one', taskId: 'remote-1' }, { idempotencyKey: 'two', taskId: 'remote-2' }, ] }) job.request.imageCount = 2 jest.mocked(repo.claimNext).mockResolvedValue(job as never) await nativeImageGenerationWorker.runNext() const saved = jest.mocked(repo.saveClaim).mock.calls.at(-1)![2] jest.mocked(repo.claimNext).mockResolvedValue({ ...job, progress: saved.progress } as never) await nativeImageGenerationWorker.runNext() expect(jest.mocked(getRelayImageTaskResult).mock.calls.map(call => call[1])).toEqual(['remote-1', 'remote-2']) }) test('an uncertain first submission does not prevent attempting the second image with its own fixed key', async () => { const job = makeJob({ progress: [{ idempotencyKey: 'one' }, { idempotencyKey: 'two' }] }) job.request.imageCount = 2 jest.mocked(repo.claimNext).mockResolvedValue(job as never) jest.mocked(createRelayImageTask).mockRejectedValueOnce(new RelayImageTaskError('network', 'unknown', { retryable: true, outcomeUnknown: true })) await nativeImageGenerationWorker.runNext() const saved = jest.mocked(repo.saveClaim).mock.calls.at(-1)![2] jest.mocked(repo.claimNext).mockResolvedValue({ ...job, progress: saved.progress } as never) await nativeImageGenerationWorker.runNext() expect(jest.mocked(createRelayImageTask).mock.calls.map(call => call[2])).toEqual(['one', 'two']) }) test('output archive failure retries GET for the existing task and never creates another generation', async () => { const job = makeJob({ status: 'RUNNING', progress: [{ idempotencyKey: 'fixed-key', taskId: 'remote-1' }] }) jest.mocked(repo.claimNext).mockResolvedValue(job as never) jest.mocked(getRelayImageTaskResult).mockResolvedValue({ status: 'succeeded', images: [{ originalUrl: 'https://upstream.example/result.png' }] }) jest.mocked(persistNativeImageOutputs).mockRejectedValueOnce(new Error('storage offline')) .mockResolvedValueOnce([{ originalUrl: 'https://project.example/native-ai-results/result.png' }]) await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'retry' }) const failed = jest.mocked(repo.saveClaim).mock.calls.at(-1)![2] expect(failed.progress).toEqual([expect.objectContaining({ taskId: 'remote-1' })]) expect(failed.progress).not.toEqual([expect.objectContaining({ images: expect.anything() })]) jest.mocked(repo.claimNext).mockResolvedValue({ ...job, progress: failed.progress } as never) await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'processed' }) expect(createRelayImageTask).not.toHaveBeenCalled() expect(jest.mocked(getRelayImageTaskResult).mock.calls.map(call => call[1])).toEqual(['remote-1', 'remote-1']) expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ status: 'FINALIZING' })) }) test('pending work older than 24 hours blocks without another remote operation', async () => { jest.mocked(repo.claimNext).mockResolvedValue(makeJob({ createdAt: new Date(Date.now() - 24 * 60 * 60 * 1000) }) as never) await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'blocked' }) expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ status: 'BLOCKED', errorMessage: expect.stringContaining('24 小时') })) expect(createRelayImageTask).not.toHaveBeenCalled() expect(getRelayImageTaskResult).not.toHaveBeenCalled() }) test('finished work still archives beyond the 24-hour generation limit', async () => { jest.mocked(repo.claimNext).mockResolvedValue(makeJob({ status: 'FINALIZING', createdAt: new Date(Date.now() - 48 * 60 * 60 * 1000), progress: [{ idempotencyKey: 'fixed-key', taskId: 'remote-1', images: [{ originalUrl: 'https://project.example/result.png' }] }], }) as never) await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'finalized' }) expect(completeNativeRenderGeneration).toHaveBeenCalledTimes(1) expect(markNativeRenderBlockedGeneration).not.toHaveBeenCalled() expect(createRelayImageTask).not.toHaveBeenCalled() }) test('known terminal failure retries its business projection before marking the durable job FAILED', async () => { jest.mocked(repo.claimNext).mockResolvedValue(makeJob({ status: 'FINALIZING', progress: [{ idempotencyKey: 'fixed-key', errorMessage: 'generation rejected' }] }) as never) jest.mocked(failNativeRenderGeneration).mockRejectedValueOnce(new Error('database unavailable')) await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'retry' }) expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ retryCount: 1, leaseOwner: null, leaseUntil: null })) expect(jest.mocked(repo.saveClaim).mock.calls.at(-1)![2]).not.toHaveProperty('status', 'FAILED') await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'finalized' }) expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ status: 'FAILED', leaseOwner: null, leaseUntil: null })) }) test('context-validation failure keeps retrying until the business failed state is saved', async () => { jest.mocked(repo.claimNext).mockResolvedValue(makeJob({ status: 'FINALIZING', progress: [{ idempotencyKey: 'fixed-key', images: [{ originalUrl: 'https://project.example/result.png' }] }] }) as never) jest.mocked(completeNativeRenderGeneration).mockRejectedValue(new AppError(ErrorCode.VALIDATION_FAILED, 'source changed')) jest.mocked(failNativeRenderGeneration).mockRejectedValueOnce(new Error('database unavailable')) await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'retry' }) expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ retryCount: 1, leaseOwner: null, leaseUntil: null })) await expect(nativeImageGenerationWorker.runNext()).resolves.toMatchObject({ status: 'failed' }) expect(repo.saveClaim).toHaveBeenLastCalledWith('job', expect.any(String), expect.objectContaining({ status: 'FAILED', leaseOwner: null, leaseUntil: null })) }) test('a lost lease during provider lookup prevents a subsequent generation request', async () => { jest.useFakeTimers() let releaseLookup!: (value: never) => void jest.mocked(repo.claimNext).mockResolvedValue(makeJob() as never) jest.mocked(providerConfigService.getById).mockImplementation(() => new Promise(resolve => { releaseLookup = resolve })) jest.mocked(repo.saveClaim).mockRejectedValue(new Error('lease transferred')) const work = nativeImageGenerationWorker.runNext() try { await jest.advanceTimersByTimeAsync(30_000) releaseLookup({ baseUrl: 'https://relay.example', apiKey: 'test-key' } as never) await expect(work).resolves.toMatchObject({ status: 'lease-lost' }) expect(createRelayImageTask).not.toHaveBeenCalled() } finally { jest.useRealTimers() } })