import { preparePendingImageDownloads } from "@/lib/storage/image-download-preparation"; import "server-only"; import { archiveTemporaryImage } from "@/lib/storage/image-archive"; import { getImageArchiveTaskStats, hasPendingImageDownloads, recoverStaleImageArchiveTasks, getImageArchiveTasksToReplace, getImageArchiveTasksToStore, markImageArchiveTaskArchiving, markImageArchiveTaskCompleted, markImageArchiveTaskFailed, markImageArchiveTaskReplaceFailed, markImageArchiveTaskReplacing, markImageArchiveTaskStored, } from "@/lib/storage/image-archive-db"; import { replaceImageUrlInDatabase } from "@/lib/storage/image-url-replacement"; const DEFAULT_BATCH_SIZE = 10; function getBatchSize() { const parsed = Number(process.env.IMAGE_ARCHIVE_BATCH_SIZE || DEFAULT_BATCH_SIZE); if (!Number.isFinite(parsed)) return DEFAULT_BATCH_SIZE; return Math.max(1, Math.min(50, Math.round(parsed))); } export async function runImageArchiveJob() { const batchSize = getBatchSize(); console.log(`[ImageArchiveJob] Start image archive job (batchSize=${batchSize})`); await recoverStaleImageArchiveTasks(); const storeTasks = await getImageArchiveTasksToStore({ limit: batchSize }); let storeSucceeded = 0; let storeFailed = 0; const storeOne = async (task: (typeof storeTasks)[number]) => { try { if (!(await markImageArchiveTaskArchiving(task.id))) return; const metadata = task.metadata as { inlineSource?: string } | null; const archived = await archiveTemporaryImage(metadata?.inlineSource || task.sourceUrl); await markImageArchiveTaskStored({ id: task.id, targetUrl: archived.url, }); storeSucceeded += 1; console.log("[ImageArchiveJob] stored", { taskId: task.id, bytes: archived.bytes, sourceUrl: task.sourceUrl, targetUrl: archived.url, }); } catch (error) { storeFailed += 1; await markImageArchiveTaskFailed({ id: task.id, error }); console.error("[ImageArchiveJob] store failed", { taskId: task.id, sourceUrl: task.sourceUrl, error: error instanceof Error ? error.message : String(error), }); } }; // Bucket work has a small worker pool independent of unrestricted model generation. let storeCursor = 0; await Promise.all(Array.from({ length: Math.min(4, storeTasks.length) }, async () => { while (storeCursor < storeTasks.length) await storeOne(storeTasks[storeCursor++]); })); const replaceTasks = await getImageArchiveTasksToReplace({ limit: 50 }); let replaceSucceeded = 0; let replaceFailed = 0; for (const task of replaceTasks) { if (!task.targetUrl) { await markImageArchiveTaskReplaceFailed({ id: task.id, error: new Error("Missing targetUrl for stored image archive task"), }); replaceFailed += 1; continue; } try { if (!(await markImageArchiveTaskReplacing(task.id))) continue; const result = await replaceImageUrlInDatabase(task.sourceUrl, task.targetUrl); await markImageArchiveTaskCompleted({ id: task.id, replacedCount: result.changed, }); replaceSucceeded += 1; console.log("[ImageArchiveJob] replaced", { taskId: task.id, sourceUrl: task.sourceUrl, targetUrl: task.targetUrl, replacedCount: result.changed, }); } catch (error) { replaceFailed += 1; await markImageArchiveTaskReplaceFailed({ id: task.id, error }); console.error("[ImageArchiveJob] replace failed", { taskId: task.id, sourceUrl: task.sourceUrl, targetUrl: task.targetUrl, error: error instanceof Error ? error.message : String(error), }); } } const downloadTasks = await preparePendingImageDownloads(); const stats = await getImageArchiveTaskStats(); return { success: true, message: "Image archive job completed", store: { total: storeTasks.length, succeeded: storeSucceeded, failed: storeFailed, }, replace: { total: replaceTasks.length, succeeded: replaceSucceeded, failed: replaceFailed, }, stats, downloadTasks, pendingDownloads: await hasPendingImageDownloads(), }; }