ai-chatbot/lib/jobs/image-archive-job.ts

156 lines
5.4 KiB
TypeScript

import { scheduleImageDownloadPreparation } from "@/lib/storage/image-download-preparation";
import "server-only";
import { trackRuntimeWork } from "@/lib/observability/active-work";
import { archiveTemporaryImage } from "@/lib/storage/image-archive";
import {
getImageArchiveTaskStats,
hasPendingImageDownloads,
recoverStaleImageArchiveTasks,
getImageArchiveTasksToReplace,
getImageArchiveTasksToStore,
markImageArchiveTaskArchiving,
markImageArchiveTaskCompleted,
markImageArchiveTaskFailed,
markImageArchiveTaskReplaceFailed,
markImageArchiveTaskReplacing,
markImageArchiveTaskStored,
claimNextImageArchiveTask,
type ImageArchiveTaskRecord,
} from "@/lib/storage/image-archive-db";
import { replaceImageUrlInDatabase } from "@/lib/storage/image-url-replacement";
const DEFAULT_BATCH_SIZE = 10;
async function storeClaimedImage(task: ImageArchiveTaskRecord) {
try {
const metadata = task.metadata as { inlineSource?: string } | null;
const archived = await archiveTemporaryImage(metadata?.inlineSource || task.sourceUrl, task.id);
await markImageArchiveTaskStored({ id: task.id, targetUrl: archived.url, downloadAsset: archived.downloadAsset });
if (!archived.downloadAsset) scheduleImageDownloadPreparation(task.id);
console.log("[ImageArchiveJob] stored", { taskId: task.id, bytes: archived.bytes, targetUrl: archived.url });
return true;
} catch (error) {
await markImageArchiveTaskFailed({ id: task.id, error });
console.error("[ImageArchiveJob] store failed", { taskId: task.id, error: String(error) });
return false;
}
}
export async function storeNextImageArchive(preferredTaskId?: string) {
return trackRuntimeWork(async () => {
const task = await claimNextImageArchiveTask(preferredTaskId);
if (!task) return false;
await storeClaimedImage(task);
return true;
}, { durable: true });
}
export async function replacePendingImageArchives() {
return trackRuntimeWork(async () => {
const tasks = await getImageArchiveTasksToReplace({ limit: 10 });
let claimed = 0;
for (const task of tasks) {
if (!(await markImageArchiveTaskReplacing(task.id))) continue;
claimed++;
try {
if (!task.targetUrl) throw new Error("Missing targetUrl");
const result = await replaceImageUrlInDatabase(task.sourceUrl, task.targetUrl);
await markImageArchiveTaskCompleted({ id: task.id, replacedCount: result.changed });
} catch (error) {
await markImageArchiveTaskReplaceFailed({ id: task.id, error });
}
}
return claimed;
}, { durable: true });
}
function getBatchSize() {
const parsed = Number(process.env.IMAGE_ARCHIVE_BATCH_SIZE || DEFAULT_BATCH_SIZE);
if (!Number.isFinite(parsed)) return DEFAULT_BATCH_SIZE;
return Math.max(1, Math.min(50, Math.round(parsed)));
}
export async function runImageArchiveJob({ preferredTaskId }: { preferredTaskId?: string } = {}) {
return trackRuntimeWork(async () => {
const batchSize = getBatchSize();
console.log(`[ImageArchiveJob] Start image archive job (batchSize=${batchSize})`);
await recoverStaleImageArchiveTasks();
scheduleImageDownloadPreparation();
const storeTasks = await getImageArchiveTasksToStore({ limit: batchSize, preferredTaskId });
let storeSucceeded = 0;
let storeFailed = 0;
const storeOne = async (task: (typeof storeTasks)[number]) => {
if (!(await markImageArchiveTaskArchiving(task.id))) return;
if (await storeClaimedImage(task)) storeSucceeded++;
else storeFailed++;
};
// Bucket work has a small worker pool independent of unrestricted model generation.
let storeCursor = 0;
await Promise.all(Array.from({ length: Math.min(4, storeTasks.length) }, async () => {
while (storeCursor < storeTasks.length) await storeOne(storeTasks[storeCursor++]);
}));
const replaceTasks = await getImageArchiveTasksToReplace({ limit: 50 });
let replaceSucceeded = 0;
let replaceFailed = 0;
for (const task of replaceTasks) {
if (!task.targetUrl) {
await markImageArchiveTaskReplaceFailed({
id: task.id,
error: new Error("Missing targetUrl for stored image archive task"),
});
replaceFailed += 1;
continue;
}
try {
if (!(await markImageArchiveTaskReplacing(task.id))) continue;
const result = await replaceImageUrlInDatabase(task.sourceUrl, task.targetUrl);
await markImageArchiveTaskCompleted({
id: task.id,
replacedCount: result.changed,
});
replaceSucceeded += 1;
console.log("[ImageArchiveJob] replaced", {
taskId: task.id,
sourceUrl: task.sourceUrl,
targetUrl: task.targetUrl,
replacedCount: result.changed,
});
} catch (error) {
replaceFailed += 1;
await markImageArchiveTaskReplaceFailed({ id: task.id, error });
console.error("[ImageArchiveJob] replace failed", {
taskId: task.id,
sourceUrl: task.sourceUrl,
targetUrl: task.targetUrl,
error: error instanceof Error ? error.message : String(error),
});
}
}
const stats = await getImageArchiveTaskStats();
return {
success: true,
message: "Image archive job completed",
store: {
total: storeTasks.length,
succeeded: storeSucceeded,
failed: storeFailed,
},
replace: {
total: replaceTasks.length,
succeeded: replaceSucceeded,
failed: replaceFailed,
},
stats,
downloadTasks: 0,
pendingDownloads: await hasPendingImageDownloads(),
};
}, { durable: true });
}