import { scheduleImageDownloadPreparation } from "@/lib/storage/image-download-preparation"; import "server-only"; import { trackRuntimeWork } from "@/lib/observability/active-work"; import { archiveTemporaryImage } from "@/lib/storage/image-archive"; import { getImageArchiveTaskStats, hasPendingImageDownloads, recoverStaleImageArchiveTasks, getImageArchiveTasksToReplace, getImageArchiveTasksToStore, markImageArchiveTaskArchiving, markImageArchiveTaskCompleted, markImageArchiveTaskFailed, markImageArchiveTaskReplaceFailed, markImageArchiveTaskReplacing, markImageArchiveTaskStored, claimNextImageArchiveTask, type ImageArchiveTaskRecord, } from "@/lib/storage/image-archive-db"; import { replaceImageUrlInDatabase } from "@/lib/storage/image-url-replacement"; const DEFAULT_BATCH_SIZE = 10; async function storeClaimedImage(task: ImageArchiveTaskRecord) { try { const metadata = task.metadata as { inlineSource?: string } | null; const archived = await archiveTemporaryImage(metadata?.inlineSource || task.sourceUrl, task.id); await markImageArchiveTaskStored({ id: task.id, targetUrl: archived.url, downloadAsset: archived.downloadAsset }); if (!archived.downloadAsset) scheduleImageDownloadPreparation(task.id); console.log("[ImageArchiveJob] stored", { taskId: task.id, bytes: archived.bytes, targetUrl: archived.url }); return true; } catch (error) { await markImageArchiveTaskFailed({ id: task.id, error }); console.error("[ImageArchiveJob] store failed", { taskId: task.id, error: String(error) }); return false; } } export async function storeNextImageArchive(preferredTaskId?: string) { return trackRuntimeWork(async () => { const task = await claimNextImageArchiveTask(preferredTaskId); if (!task) return false; await storeClaimedImage(task); return true; }, { durable: true }); } export async function replacePendingImageArchives() { return trackRuntimeWork(async () => { const tasks = await getImageArchiveTasksToReplace({ limit: 10 }); let claimed = 0; for (const task of tasks) { if (!(await markImageArchiveTaskReplacing(task.id))) continue; claimed++; try { if (!task.targetUrl) throw new Error("Missing targetUrl"); const result = await replaceImageUrlInDatabase(task.sourceUrl, task.targetUrl); await markImageArchiveTaskCompleted({ id: task.id, replacedCount: result.changed }); } catch (error) { await markImageArchiveTaskReplaceFailed({ id: task.id, error }); } } return claimed; }, { durable: true }); } function getBatchSize() { const parsed = Number(process.env.IMAGE_ARCHIVE_BATCH_SIZE || DEFAULT_BATCH_SIZE); if (!Number.isFinite(parsed)) return DEFAULT_BATCH_SIZE; return Math.max(1, Math.min(50, Math.round(parsed))); } export async function runImageArchiveJob({ preferredTaskId }: { preferredTaskId?: string } = {}) { return trackRuntimeWork(async () => { const batchSize = getBatchSize(); console.log(`[ImageArchiveJob] Start image archive job (batchSize=${batchSize})`); await recoverStaleImageArchiveTasks(); scheduleImageDownloadPreparation(); const storeTasks = await getImageArchiveTasksToStore({ limit: batchSize, preferredTaskId }); let storeSucceeded = 0; let storeFailed = 0; const storeOne = async (task: (typeof storeTasks)[number]) => { if (!(await markImageArchiveTaskArchiving(task.id))) return; if (await storeClaimedImage(task)) storeSucceeded++; else storeFailed++; }; // Bucket work has a small worker pool independent of unrestricted model generation. let storeCursor = 0; await Promise.all(Array.from({ length: Math.min(4, storeTasks.length) }, async () => { while (storeCursor < storeTasks.length) await storeOne(storeTasks[storeCursor++]); })); const replaceTasks = await getImageArchiveTasksToReplace({ limit: 50 }); let replaceSucceeded = 0; let replaceFailed = 0; for (const task of replaceTasks) { if (!task.targetUrl) { await markImageArchiveTaskReplaceFailed({ id: task.id, error: new Error("Missing targetUrl for stored image archive task"), }); replaceFailed += 1; continue; } try { if (!(await markImageArchiveTaskReplacing(task.id))) continue; const result = await replaceImageUrlInDatabase(task.sourceUrl, task.targetUrl); await markImageArchiveTaskCompleted({ id: task.id, replacedCount: result.changed, }); replaceSucceeded += 1; console.log("[ImageArchiveJob] replaced", { taskId: task.id, sourceUrl: task.sourceUrl, targetUrl: task.targetUrl, replacedCount: result.changed, }); } catch (error) { replaceFailed += 1; await markImageArchiveTaskReplaceFailed({ id: task.id, error }); console.error("[ImageArchiveJob] replace failed", { taskId: task.id, sourceUrl: task.sourceUrl, targetUrl: task.targetUrl, error: error instanceof Error ? error.message : String(error), }); } } const stats = await getImageArchiveTaskStats(); return { success: true, message: "Image archive job completed", store: { total: storeTasks.length, succeeded: storeSucceeded, failed: storeFailed, }, replace: { total: replaceTasks.length, succeeded: replaceSucceeded, failed: replaceFailed, }, stats, downloadTasks: 0, pendingDownloads: await hasPendingImageDownloads(), }; }, { durable: true }); }