feat: defer generated image archiving

This commit is contained in:
134 2026-07-10 20:27:09 +08:00
parent 6c83826e1d
commit 748273d917
23 changed files with 1241 additions and 4 deletions

View File

@ -0,0 +1,65 @@
import { NextResponse } from "next/server";
import { runImageArchiveJob } from "@/lib/jobs/image-archive-job";
import { getImageArchiveTaskStats } from "@/lib/storage/image-archive-db";
function verifyAuthToken(request: Request): boolean {
const authHeader = request.headers.get("authorization");
const token = authHeader?.replace("Bearer ", "");
const cronSecret = process.env.CRON_SECRET;
if (!cronSecret) {
console.error("[ImageArchiveAPI] CRON_SECRET is not configured");
return false;
}
return token === cronSecret;
}
export async function POST(request: Request) {
try {
if (!verifyAuthToken(request)) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
const result = await runImageArchiveJob();
return NextResponse.json({
...result,
timestamp: new Date().toISOString(),
});
} catch (error) {
console.error("[ImageArchiveAPI] Job execution failed:", error);
return NextResponse.json(
{
success: false,
error: error instanceof Error ? error.message : String(error),
timestamp: new Date().toISOString(),
},
{ status: 500 }
);
}
}
export async function GET(request: Request) {
try {
if (!verifyAuthToken(request)) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
const stats = await getImageArchiveTaskStats();
return NextResponse.json({
success: true,
stats,
timestamp: new Date().toISOString(),
});
} catch (error) {
console.error("[ImageArchiveAPI] Stats query failed:", error);
return NextResponse.json(
{
success: false,
error: error instanceof Error ? error.message : String(error),
timestamp: new Date().toISOString(),
},
{ status: 500 }
);
}
}

View File

@ -10,6 +10,7 @@
import sharp from "sharp"; import sharp from "sharp";
import { constrainImageDimensions } from "@/lib/ai/image-size-constraints"; import { constrainImageDimensions } from "@/lib/ai/image-size-constraints";
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
import { uploadFile } from "@/lib/storage/service"; import { uploadFile } from "@/lib/storage/service";
interface AlibabaImageConfig { interface AlibabaImageConfig {
@ -358,6 +359,11 @@ async function downloadAndUploadToS3(
prefix: string = "alibaba-image" prefix: string = "alibaba-image"
): Promise<{ thumbnail: string; original: string }> { ): Promise<{ thumbnail: string; original: string }> {
try { try {
const deferred = await createDeferredUploadedImage(imageUrl, {
providerType: "alibaba-image",
});
if (deferred) return deferred;
console.log("[Alibaba Image] Downloading image from:", imageUrl.substring(0, 100) + "..."); console.log("[Alibaba Image] Downloading image from:", imageUrl.substring(0, 100) + "...");
const { buffer } = await fetchImageBufferWithRetry(imageUrl, 30000, 3); const { buffer } = await fetchImageBufferWithRetry(imageUrl, 30000, 3);

View File

@ -2,6 +2,7 @@ import "server-only";
import sharp from "sharp"; import sharp from "sharp";
import { resolveLongestEdgeDimensions } from "@/lib/ai/image-size-constraints"; import { resolveLongestEdgeDimensions } from "@/lib/ai/image-size-constraints";
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
import { uploadFile } from "@/lib/storage/service"; import { uploadFile } from "@/lib/storage/service";
type BflImageConfig = { type BflImageConfig = {
@ -273,6 +274,11 @@ async function uploadGeneratedBuffer(
} }
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> { async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
const deferred = await createDeferredUploadedImage(imageUrl, {
providerType: "bfl-image",
});
if (deferred) return deferred;
const response = await fetch(imageUrl, { cache: "no-store" }); const response = await fetch(imageUrl, { cache: "no-store" });
if (!response.ok) { if (!response.ok) {
throw new Error(`Failed to download generated image: ${response.status}`); throw new Error(`Failed to download generated image: ${response.status}`);

View File

@ -6,6 +6,7 @@ import {
constrainImageDimensions, constrainImageDimensions,
resolveLongestEdgeDimensions, resolveLongestEdgeDimensions,
} from "@/lib/ai/image-size-constraints"; } from "@/lib/ai/image-size-constraints";
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
import { uploadFile } from "@/lib/storage/service"; import { uploadFile } from "@/lib/storage/service";
interface FalImageConfig { interface FalImageConfig {
@ -466,6 +467,11 @@ function generateFilename(prefix: string, extension: string) {
} }
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> { async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
const deferred = await createDeferredUploadedImage(imageUrl, {
providerType: "fal-image",
});
if (deferred) return deferred;
const response = await fetch(imageUrl); const response = await fetch(imageUrl);
if (!response.ok) { if (!response.ok) {
throw new Error(`Failed to download image: ${response.status}`); throw new Error(`Failed to download image: ${response.status}`);

View File

@ -1,6 +1,7 @@
import "server-only"; import "server-only";
import { resolveGrsaiGptImageSize } from "@/lib/ai/grsai-gpt-image-sizes"; import { resolveGrsaiGptImageSize } from "@/lib/ai/grsai-gpt-image-sizes";
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
import { uploadFile } from "@/lib/storage/service"; import { uploadFile } from "@/lib/storage/service";
import { createLogger } from "@/lib/observability/logger"; import { createLogger } from "@/lib/observability/logger";
@ -170,6 +171,11 @@ async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/jp
* 带重试机制,确保成功下载并上传到国内中转 * 带重试机制,确保成功下载并上传到国内中转
*/ */
async function downloadAndUploadToS3(imageUrl: string, retries = 3): Promise<{ thumbnail: string; original: string }> { async function downloadAndUploadToS3(imageUrl: string, retries = 3): Promise<{ thumbnail: string; original: string }> {
const deferred = await createDeferredUploadedImage(imageUrl, {
providerType: "gemini-image",
});
if (deferred) return deferred;
for (let attempt = 0; attempt < retries; attempt++) { for (let attempt = 0; attempt < retries; attempt++) {
try { try {
logger.debug(`[Gemini Image] Downloading image from URL (attempt ${attempt + 1}/${retries}): ${imageUrl.substring(0, 100)}...`); logger.debug(`[Gemini Image] Downloading image from URL (attempt ${attempt + 1}/${retries}): ${imageUrl.substring(0, 100)}...`);

View File

@ -1,5 +1,6 @@
import "server-only"; import "server-only";
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
import { uploadFile } from "@/lib/storage/service"; import { uploadFile } from "@/lib/storage/service";
interface HuankeGeminiConfig { interface HuankeGeminiConfig {
@ -112,6 +113,11 @@ async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/jp
* 下载远程URL图片并上传到 S3 * 下载远程URL图片并上传到 S3
*/ */
async function downloadAndUploadToS3(imageUrl: string, retries = 3): Promise<{ thumbnail: string; original: string }> { async function downloadAndUploadToS3(imageUrl: string, retries = 3): Promise<{ thumbnail: string; original: string }> {
const deferred = await createDeferredUploadedImage(imageUrl, {
providerType: "huanke-gemini",
});
if (deferred) return deferred;
for (let attempt = 0; attempt < retries; attempt++) { for (let attempt = 0; attempt < retries; attempt++) {
try { try {
console.log(`[Huanke Gemini] Downloading image from URL (attempt ${attempt + 1}/${retries}): ${imageUrl.substring(0, 100)}...`); console.log(`[Huanke Gemini] Downloading image from URL (attempt ${attempt + 1}/${retries}): ${imageUrl.substring(0, 100)}...`);

View File

@ -7,6 +7,7 @@ import {
inferKreaModelKind, inferKreaModelKind,
type KreaModelKind, type KreaModelKind,
} from "@/lib/ai/krea-models"; } from "@/lib/ai/krea-models";
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
import { uploadFile } from "@/lib/storage/service"; import { uploadFile } from "@/lib/storage/service";
interface KreaConfig { interface KreaConfig {
@ -651,6 +652,14 @@ async function uploadKreaImages(payload: unknown) {
const uploadedImages: UploadedImage[] = []; const uploadedImages: UploadedImage[] = [];
for (const url of collectHttpUrls(payload)) { for (const url of collectHttpUrls(payload)) {
try { try {
const deferred = await createDeferredUploadedImage(url, {
providerType: "krea",
});
if (deferred) {
uploadedImages.push(deferred);
continue;
}
const { buffer, contentType } = await downloadRemoteAsset(url); const { buffer, contentType } = await downloadRemoteAsset(url);
if (!String(contentType || "").startsWith("image/")) continue; if (!String(contentType || "").startsWith("image/")) continue;
uploadedImages.push(await uploadGeneratedBuffer(buffer, detectExtensionFromContentType(contentType))); uploadedImages.push(await uploadGeneratedBuffer(buffer, detectExtensionFromContentType(contentType)));

View File

@ -1,6 +1,7 @@
import "server-only"; import "server-only";
import sharp from "sharp"; import sharp from "sharp";
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
import { uploadFile } from "@/lib/storage/service"; import { uploadFile } from "@/lib/storage/service";
interface MidjourneyConfig { interface MidjourneyConfig {
@ -161,6 +162,11 @@ async function uploadGeneratedBuffer(
} }
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> { async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
const deferred = await createDeferredUploadedImage(imageUrl, {
providerType: "midjourney",
});
if (deferred) return deferred;
const response = await fetch(imageUrl, { cache: "no-store" }); const response = await fetch(imageUrl, { cache: "no-store" });
if (!response.ok) { if (!response.ok) {
throw new Error(`Failed to download Midjourney image: ${response.status}`); throw new Error(`Failed to download Midjourney image: ${response.status}`);

View File

@ -4,6 +4,7 @@ import sharp from "sharp";
import { import {
resolveLongestEdgeDimensions, resolveLongestEdgeDimensions,
} from "@/lib/ai/image-size-constraints"; } from "@/lib/ai/image-size-constraints";
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
import { uploadFile } from "@/lib/storage/service"; import { uploadFile } from "@/lib/storage/service";
interface OpenAIImageConfig { interface OpenAIImageConfig {
@ -413,6 +414,11 @@ async function uploadGeneratedBuffer(
} }
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> { async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
const deferred = await createDeferredUploadedImage(imageUrl, {
providerType: "openai-image",
});
if (deferred) return deferred;
const response = await fetch(imageUrl, { cache: "no-store" }); const response = await fetch(imageUrl, { cache: "no-store" });
if (!response.ok) { if (!response.ok) {
throw new Error(`Failed to download generated image: ${response.status}`); throw new Error(`Failed to download generated image: ${response.status}`);

View File

@ -8,6 +8,7 @@ import {
normalizeRunwayDuration, normalizeRunwayDuration,
resolveRunwayApiRatio, resolveRunwayApiRatio,
} from "@/lib/ai/runway-video-models"; } from "@/lib/ai/runway-video-models";
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
import { uploadFile } from "@/lib/storage/service"; import { uploadFile } from "@/lib/storage/service";
import { inferRunwayModelKind } from "@/lib/ai/model-classification"; import { inferRunwayModelKind } from "@/lib/ai/model-classification";
@ -432,6 +433,14 @@ async function uploadRunwayImages(taskOutput: unknown) {
for (const url of outputUrls) { for (const url of outputUrls) {
try { try {
const deferred = await createDeferredUploadedImage(url, {
providerType: "runway",
});
if (deferred) {
uploadedImages.push(deferred);
continue;
}
const { buffer, contentType } = await downloadRemoteAsset(url); const { buffer, contentType } = await downloadRemoteAsset(url);
if (!String(contentType || "").startsWith("image/")) { if (!String(contentType || "").startsWith("image/")) {
continue; continue;

View File

@ -1,6 +1,7 @@
import "server-only"; import "server-only";
import sharp from "sharp"; import sharp from "sharp";
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
import { uploadFile } from "@/lib/storage/service"; import { uploadFile } from "@/lib/storage/service";
import { import {
resolveRuxaImageRuntimeModelId, resolveRuxaImageRuntimeModelId,
@ -369,6 +370,11 @@ function generateFilename(prefix: string, extension: string) {
} }
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> { async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
const deferred = await createDeferredUploadedImage(imageUrl, {
providerType: "ruxa-image",
});
if (deferred) return deferred;
const response = await fetch(imageUrl); const response = await fetch(imageUrl);
if (!response.ok) { if (!response.ok) {
throw new Error(`Failed to download image: ${response.status}`); throw new Error(`Failed to download image: ${response.status}`);

View File

@ -1,6 +1,7 @@
import "server-only"; import "server-only";
import sharp from "sharp"; import sharp from "sharp";
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
import { uploadFile } from "@/lib/storage/service"; import { uploadFile } from "@/lib/storage/service";
import { buildSnapgenReferenceOrderPrompt } from "@/lib/ai/providers/snapgen-reference-order"; import { buildSnapgenReferenceOrderPrompt } from "@/lib/ai/providers/snapgen-reference-order";
import { import {
@ -690,6 +691,11 @@ async function downloadAndUploadImage(
imageUrl: string, imageUrl: string,
timeoutMs = IMAGE_DOWNLOAD_TIMEOUT_MS timeoutMs = IMAGE_DOWNLOAD_TIMEOUT_MS
): Promise<UploadedImage> { ): Promise<UploadedImage> {
const deferred = await createDeferredUploadedImage(imageUrl, {
providerType: "snapgen-image",
});
if (deferred) return deferred;
const response = await fetchWithTimeout(imageUrl, {}, timeoutMs); const response = await fetchWithTimeout(imageUrl, {}, timeoutMs);
if (!response.ok) { if (!response.ok) {
throw new Error(`Failed to download SnapGen image: ${response.status}`); throw new Error(`Failed to download SnapGen image: ${response.status}`);

View File

@ -2,6 +2,7 @@ import "server-only";
import sharp from "sharp"; import sharp from "sharp";
import { constrainImageDimensions } from "@/lib/ai/image-size-constraints"; import { constrainImageDimensions } from "@/lib/ai/image-size-constraints";
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
import { uploadFile } from "@/lib/storage/service"; import { uploadFile } from "@/lib/storage/service";
interface StepfunImageConfig { interface StepfunImageConfig {
@ -160,6 +161,11 @@ async function uploadGeneratedBuffer(
} }
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> { async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
const deferred = await createDeferredUploadedImage(imageUrl, {
providerType: "stepfun-image",
});
if (deferred) return deferred;
const response = await fetch(imageUrl, { cache: "no-store" }); const response = await fetch(imageUrl, { cache: "no-store" });
if (!response.ok) { if (!response.ok) {
throw new Error(`Failed to download StepFun generated image: ${response.status}`); throw new Error(`Failed to download StepFun generated image: ${response.status}`);

View File

@ -1,6 +1,7 @@
import "server-only"; import "server-only";
import { constrainImageDimensions } from "@/lib/ai/image-size-constraints"; import { constrainImageDimensions } from "@/lib/ai/image-size-constraints";
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
import { uploadFile } from "@/lib/storage/service"; import { uploadFile } from "@/lib/storage/service";
import { getDefaultStorageConfig } from "@/lib/db/queries"; import { getDefaultStorageConfig } from "@/lib/db/queries";
import { createLogger } from "@/lib/observability/logger"; import { createLogger } from "@/lib/observability/logger";
@ -429,6 +430,13 @@ async function createThumbnail(buffer: Buffer, mimeType: string): Promise<Buffer
*/ */
async function uploadImageToS3(imageData: string): Promise<{ thumbnail: string; original: string }> { async function uploadImageToS3(imageData: string): Promise<{ thumbnail: string; original: string }> {
try { try {
if (/^https?:\/\//i.test(imageData)) {
const deferred = await createDeferredUploadedImage(imageData, {
providerType: "volcano-vision",
});
if (deferred) return deferred;
}
let buffer: Buffer; let buffer: Buffer;
let mimeType = "image/jpeg"; let mimeType = "image/jpeg";

View File

@ -0,0 +1,22 @@
CREATE TABLE IF NOT EXISTS "ImageArchiveTask" (
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
"sourceUrl" text NOT NULL UNIQUE,
"targetUrl" text,
"targetStorageConfigId" uuid REFERENCES "StorageConfig"("id"),
"status" varchar(32) NOT NULL DEFAULT 'pending',
"attemptCount" integer NOT NULL DEFAULT 0,
"lastError" text,
"archiveAfterAt" timestamp NOT NULL,
"replaceAfterAt" timestamp NOT NULL,
"archivedAt" timestamp,
"replacedAt" timestamp,
"metadata" jsonb,
"createdAt" timestamp NOT NULL DEFAULT now(),
"updatedAt" timestamp NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS "ImageArchiveTask_status_archive_idx"
ON "ImageArchiveTask"("status", "archiveAfterAt", "createdAt");
CREATE INDEX IF NOT EXISTS "ImageArchiveTask_status_replace_idx"
ON "ImageArchiveTask"("status", "replaceAfterAt", "createdAt");

View File

@ -512,6 +512,13 @@
"when": 1781174400000, "when": 1781174400000,
"tag": "0086_external_image_log_model_delete_set_null", "tag": "0086_external_image_log_model_delete_set_null",
"breakpoints": true "breakpoints": true
},
{
"idx": 73,
"version": "7",
"when": 1783680000000,
"tag": "0087_add_image_archive_tasks",
"breakpoints": true
} }
] ]
} }

View File

@ -441,6 +441,26 @@ export const storageMigrationTask = pgTable("StorageMigrationTask", {
export type StorageMigrationTask = InferSelectModel<typeof storageMigrationTask>; export type StorageMigrationTask = InferSelectModel<typeof storageMigrationTask>;
// AI 生成图片临时地址归档任务表
export const imageArchiveTask = pgTable("ImageArchiveTask", {
id: uuid("id").primaryKey().notNull().defaultRandom(),
sourceUrl: text("sourceUrl").notNull().unique(),
targetUrl: text("targetUrl"),
targetStorageConfigId: uuid("targetStorageConfigId").references(() => storageConfig.id),
status: varchar("status", { length: 32 }).notNull().default("pending"),
attemptCount: integer("attemptCount").notNull().default(0),
lastError: text("lastError"),
archiveAfterAt: timestamp("archiveAfterAt").notNull(),
replaceAfterAt: timestamp("replaceAfterAt").notNull(),
archivedAt: timestamp("archivedAt"),
replacedAt: timestamp("replacedAt"),
metadata: jsonb("metadata"),
createdAt: timestamp("createdAt").notNull().defaultNow(),
updatedAt: timestamp("updatedAt").notNull().defaultNow(),
});
export type ImageArchiveTask = InferSelectModel<typeof imageArchiveTask>;
// 网站设置表 // 网站设置表
export const siteSettings = pgTable("SiteSettings", { export const siteSettings = pgTable("SiteSettings", {
id: uuid("id").primaryKey().notNull().defaultRandom(), id: uuid("id").primaryKey().notNull().defaultRandom(),

View File

@ -0,0 +1,118 @@
import "server-only";
import { archiveTemporaryImage } from "@/lib/storage/image-archive";
import {
getImageArchiveTaskStats,
getImageArchiveTasksToReplace,
getImageArchiveTasksToStore,
markImageArchiveTaskArchiving,
markImageArchiveTaskCompleted,
markImageArchiveTaskFailed,
markImageArchiveTaskReplaceFailed,
markImageArchiveTaskReplacing,
markImageArchiveTaskStored,
} from "@/lib/storage/image-archive-db";
import { replaceImageUrlInDatabase } from "@/lib/storage/image-url-replacement";
const DEFAULT_BATCH_SIZE = 10;
function getBatchSize() {
const parsed = Number(process.env.IMAGE_ARCHIVE_BATCH_SIZE || DEFAULT_BATCH_SIZE);
if (!Number.isFinite(parsed)) return DEFAULT_BATCH_SIZE;
return Math.max(1, Math.min(50, Math.round(parsed)));
}
export async function runImageArchiveJob() {
const batchSize = getBatchSize();
console.log(`[ImageArchiveJob] Start image archive job (batchSize=${batchSize})`);
const storeTasks = await getImageArchiveTasksToStore({ limit: batchSize });
let storeSucceeded = 0;
let storeFailed = 0;
for (const task of storeTasks) {
try {
await markImageArchiveTaskArchiving(task.id);
const archived = await archiveTemporaryImage(task.sourceUrl);
await markImageArchiveTaskStored({
id: task.id,
targetUrl: archived.url,
});
storeSucceeded += 1;
console.log("[ImageArchiveJob] stored", {
taskId: task.id,
bytes: archived.bytes,
sourceUrl: task.sourceUrl,
targetUrl: archived.url,
});
} catch (error) {
storeFailed += 1;
await markImageArchiveTaskFailed({ id: task.id, error });
console.error("[ImageArchiveJob] store failed", {
taskId: task.id,
sourceUrl: task.sourceUrl,
error: error instanceof Error ? error.message : String(error),
});
}
}
const remainingSlots = Math.max(1, batchSize - storeTasks.length);
const replaceTasks = await getImageArchiveTasksToReplace({
limit: remainingSlots,
});
let replaceSucceeded = 0;
let replaceFailed = 0;
for (const task of replaceTasks) {
if (!task.targetUrl) {
await markImageArchiveTaskReplaceFailed({
id: task.id,
error: new Error("Missing targetUrl for stored image archive task"),
});
replaceFailed += 1;
continue;
}
try {
await markImageArchiveTaskReplacing(task.id);
const result = await replaceImageUrlInDatabase(task.sourceUrl, task.targetUrl);
await markImageArchiveTaskCompleted({
id: task.id,
replacedCount: result.changed,
});
replaceSucceeded += 1;
console.log("[ImageArchiveJob] replaced", {
taskId: task.id,
sourceUrl: task.sourceUrl,
targetUrl: task.targetUrl,
replacedCount: result.changed,
});
} catch (error) {
replaceFailed += 1;
await markImageArchiveTaskReplaceFailed({ id: task.id, error });
console.error("[ImageArchiveJob] replace failed", {
taskId: task.id,
sourceUrl: task.sourceUrl,
targetUrl: task.targetUrl,
error: error instanceof Error ? error.message : String(error),
});
}
}
const stats = await getImageArchiveTaskStats();
return {
success: true,
message: "Image archive job completed",
store: {
total: storeTasks.length,
succeeded: storeSucceeded,
failed: storeFailed,
},
replace: {
total: replaceTasks.length,
succeeded: replaceSucceeded,
failed: replaceFailed,
},
stats,
};
}

View File

@ -0,0 +1,240 @@
import "server-only";
import { and, asc, eq, inArray, lte, sql } from "drizzle-orm";
import { db } from "@/lib/db";
import { imageArchiveTask, type ImageArchiveTask } from "@/lib/db/schema";
export type ImageArchiveTaskRecord = ImageArchiveTask;
const DEFAULT_DISPLAY_HOLD_HOURS = 1;
function getDisplayHoldHours() {
const raw = Number(
process.env.IMAGE_ARCHIVE_DISPLAY_HOLD_HOURS || DEFAULT_DISPLAY_HOLD_HOURS
);
if (!Number.isFinite(raw)) return DEFAULT_DISPLAY_HOLD_HOURS;
return Math.max(0, Math.min(24 * 30, raw));
}
export function getImageArchiveDisplayHoldMs() {
return getDisplayHoldHours() * 60 * 60 * 1000;
}
function isArchivableHttpUrl(value: string) {
if (!/^https?:\/\//i.test(value)) return false;
if (/^(data|blob):/i.test(value)) return false;
return true;
}
export async function enqueueImageArchiveTask(params: {
sourceUrl: string;
archiveAfterAt?: Date;
replaceAfterAt?: Date;
metadata?: Record<string, unknown> | null;
}) {
const sourceUrl = String(params.sourceUrl || "").trim();
if (!isArchivableHttpUrl(sourceUrl)) return null;
const now = new Date();
const archiveAfterAt = params.archiveAfterAt || now;
const replaceAfterAt =
params.replaceAfterAt ||
new Date(now.getTime() + getImageArchiveDisplayHoldMs());
const [task] = await db
.insert(imageArchiveTask)
.values({
sourceUrl,
status: "pending",
archiveAfterAt,
replaceAfterAt,
metadata: params.metadata || null,
updatedAt: now,
})
.onConflictDoUpdate({
target: imageArchiveTask.sourceUrl,
set: {
replaceAfterAt,
metadata: params.metadata || undefined,
updatedAt: now,
},
})
.returning();
return task || null;
}
export async function enqueueImageArchiveTasks(params: {
urls: string[];
metadata?: Record<string, unknown> | null;
}) {
const urls = Array.from(
new Set(
(params.urls || [])
.map((url) => String(url || "").trim())
.filter(isArchivableHttpUrl)
)
);
const tasks: ImageArchiveTaskRecord[] = [];
for (const sourceUrl of urls) {
const task = await enqueueImageArchiveTask({
sourceUrl,
metadata: params.metadata || null,
});
if (task) tasks.push(task);
}
return tasks;
}
export async function getImageArchiveTasksToStore(params: { limit: number }) {
const now = new Date();
return await db
.select()
.from(imageArchiveTask)
.where(
and(
inArray(imageArchiveTask.status, ["pending", "failed"]),
lte(imageArchiveTask.archiveAfterAt, now)
)
)
.orderBy(asc(imageArchiveTask.createdAt), asc(imageArchiveTask.id))
.limit(Math.max(1, Math.min(50, params.limit)));
}
export async function markImageArchiveTaskArchiving(id: string) {
await db
.update(imageArchiveTask)
.set({
status: "archiving",
attemptCount: sql`${imageArchiveTask.attemptCount} + 1`,
lastError: null,
updatedAt: new Date(),
})
.where(eq(imageArchiveTask.id, id));
}
export async function markImageArchiveTaskStored(params: {
id: string;
targetUrl: string;
targetStorageConfigId?: string | null;
}) {
await db
.update(imageArchiveTask)
.set({
status: "stored",
targetUrl: params.targetUrl,
targetStorageConfigId: params.targetStorageConfigId || null,
archivedAt: new Date(),
lastError: null,
updatedAt: new Date(),
})
.where(eq(imageArchiveTask.id, params.id));
}
export async function markImageArchiveTaskFailed(params: {
id: string;
error: unknown;
}) {
const message = String(
params.error instanceof Error ? params.error.message : params.error || "Unknown error"
).slice(0, 2000);
await db
.update(imageArchiveTask)
.set({
status: "failed",
lastError: message,
archiveAfterAt: new Date(Date.now() + 10 * 60 * 1000),
updatedAt: new Date(),
})
.where(eq(imageArchiveTask.id, params.id));
}
export async function getImageArchiveTasksToReplace(params: { limit: number }) {
const now = new Date();
return await db
.select()
.from(imageArchiveTask)
.where(
and(
eq(imageArchiveTask.status, "stored"),
lte(imageArchiveTask.replaceAfterAt, now)
)
)
.orderBy(asc(imageArchiveTask.createdAt), asc(imageArchiveTask.id))
.limit(Math.max(1, Math.min(50, params.limit)));
}
export async function markImageArchiveTaskReplacing(id: string) {
await db
.update(imageArchiveTask)
.set({
status: "replacing",
updatedAt: new Date(),
})
.where(eq(imageArchiveTask.id, id));
}
export async function markImageArchiveTaskCompleted(params: {
id: string;
replacedCount?: number;
}) {
await db
.update(imageArchiveTask)
.set({
status: "completed",
replacedAt: new Date(),
lastError: null,
metadata:
params.replacedCount === undefined
? undefined
: sql`coalesce(${imageArchiveTask.metadata}, '{}'::jsonb) || ${JSON.stringify({
replacedCount: params.replacedCount,
})}::jsonb`,
updatedAt: new Date(),
})
.where(eq(imageArchiveTask.id, params.id));
}
export async function markImageArchiveTaskReplaceFailed(params: {
id: string;
error: unknown;
}) {
const message = String(
params.error instanceof Error ? params.error.message : params.error || "Unknown error"
).slice(0, 2000);
await db
.update(imageArchiveTask)
.set({
status: "stored",
lastError: message,
replaceAfterAt: new Date(Date.now() + 10 * 60 * 1000),
updatedAt: new Date(),
})
.where(eq(imageArchiveTask.id, params.id));
}
export async function getImageArchiveTaskStats() {
const grouped = await db
.select({
status: imageArchiveTask.status,
count: sql<number>`count(*)::int`,
})
.from(imageArchiveTask)
.groupBy(imageArchiveTask.status);
const pendingStore = grouped
.filter((item) => item.status === "pending" || item.status === "failed")
.reduce((sum, item) => sum + (item.count || 0), 0);
const pendingReplace = grouped
.filter((item) => item.status === "stored")
.reduce((sum, item) => sum + (item.count || 0), 0);
return {
grouped,
pendingStore,
pendingReplace,
};
}

View File

@ -0,0 +1,198 @@
import "server-only";
import { createLogger } from "@/lib/observability/logger";
import {
enqueueImageArchiveTasks,
getImageArchiveDisplayHoldMs,
} from "@/lib/storage/image-archive-db";
import { uploadFile } from "@/lib/storage/service";
const logger = createLogger("image-archive");
const DEFAULT_DOWNLOAD_TIMEOUT_MS = 60_000;
type DeferredImageArchiveMetadata = Record<string, unknown> & {
providerType?: string;
modelId?: string;
imageSize?: string;
};
function getDownloadTimeoutMs() {
const parsed = Number(process.env.IMAGE_ARCHIVE_DOWNLOAD_TIMEOUT_MS);
if (!Number.isFinite(parsed)) return DEFAULT_DOWNLOAD_TIMEOUT_MS;
return Math.max(5_000, Math.min(5 * 60_000, Math.round(parsed)));
}
function isHttpUrl(value: string) {
return /^https?:\/\//i.test(String(value || "").trim());
}
export function isDeferredImageArchiveEnabled() {
return process.env.AI_IMAGE_DEFERRED_ARCHIVE !== "false";
}
export async function registerGeneratedImageTemporaryUrls(params: {
urls: string[];
metadata?: DeferredImageArchiveMetadata | null;
}) {
if (!isDeferredImageArchiveEnabled()) return [];
const urls = Array.from(
new Set(
(params.urls || [])
.map((url) => String(url || "").trim())
.filter(isHttpUrl)
)
);
if (urls.length === 0) return [];
try {
return await enqueueImageArchiveTasks({
urls,
metadata: {
...(params.metadata || {}),
displayHoldMs: getImageArchiveDisplayHoldMs(),
},
});
} catch (error) {
logger.warn("enqueue temporary image archive failed", {
urls,
error: error instanceof Error ? error.message : String(error),
});
return [];
}
}
export async function formatDeferredGeneratedImageMarkdown(params: {
urls: string[];
imageSize: string;
providerType?: string;
modelId?: string;
metadata?: DeferredImageArchiveMetadata | null;
}) {
const urls = Array.from(
new Set(
(params.urls || [])
.map((url) => String(url || "").trim())
.filter(isHttpUrl)
)
);
if (urls.length === 0) {
throw new Error("AI 未返回可用图片地址");
}
await registerGeneratedImageTemporaryUrls({
urls,
metadata: {
providerType: params.providerType,
modelId: params.modelId,
imageSize: params.imageSize,
...(params.metadata || {}),
},
});
return urls
.map((url) => `![thumb|||${url}|||${params.imageSize}](${url})`)
.join("\n\n");
}
export async function createDeferredUploadedImage(
sourceUrl: string,
metadata?: DeferredImageArchiveMetadata | null
) {
const url = String(sourceUrl || "").trim();
if (!isDeferredImageArchiveEnabled() || !isHttpUrl(url)) return null;
await registerGeneratedImageTemporaryUrls({
urls: [url],
metadata: metadata || null,
});
return {
thumbnail: url,
original: url,
};
}
function getImageExtension(contentType: string | null, sourceUrl: string) {
const normalized = String(contentType || "").split(";")[0].trim().toLowerCase();
const byType: Record<string, string> = {
"image/jpeg": "jpg",
"image/jpg": "jpg",
"image/png": "png",
"image/webp": "webp",
"image/gif": "gif",
"image/svg+xml": "svg",
};
if (byType[normalized]) return byType[normalized];
try {
const pathname = new URL(sourceUrl).pathname.toLowerCase();
const ext = pathname.split(".").pop()?.replace(/[^a-z0-9]+/g, "") || "";
if (["jpg", "jpeg", "png", "webp", "gif", "svg"].includes(ext)) {
return ext === "jpeg" ? "jpg" : ext;
}
} catch {
// keep fallback
}
return "png";
}
function toArrayBuffer(buffer: Buffer): ArrayBuffer {
return buffer.buffer.slice(
buffer.byteOffset,
buffer.byteOffset + buffer.byteLength
) as ArrayBuffer;
}
export async function downloadTemporaryImage(params: {
sourceUrl: string;
retries?: number;
}) {
const retries = Math.max(0, Math.min(5, params.retries ?? 2));
for (let attempt = 0; attempt <= retries; attempt += 1) {
try {
const response = await fetch(params.sourceUrl, {
signal: AbortSignal.timeout(getDownloadTimeoutMs()),
headers: {
"User-Agent": "AI-Chatbot-Image-Archiver/1.0",
Accept: "image/avif,image/webp,image/png,image/jpeg,image/*,*/*;q=0.8",
},
});
if (!response.ok) {
throw new Error(`HTTP ${response.status} ${response.statusText}`);
}
const contentType = response.headers.get("content-type");
const arrayBuffer = await response.arrayBuffer();
if (!arrayBuffer || arrayBuffer.byteLength === 0) {
throw new Error("Downloaded image is empty");
}
return {
buffer: Buffer.from(arrayBuffer),
contentType,
extension: getImageExtension(contentType, params.sourceUrl),
};
} catch (error) {
if (attempt >= retries) throw error;
await new Promise((resolve) => setTimeout(resolve, 1000 * (attempt + 1)));
}
}
throw new Error("Image download failed");
}
export async function archiveTemporaryImage(sourceUrl: string) {
const downloaded = await downloadTemporaryImage({ sourceUrl });
const filename = `ai-generated-archive-${Date.now()}.${downloaded.extension}`;
const uploaded = await uploadFile(filename, toArrayBuffer(downloaded.buffer));
return {
url: uploaded.url,
pathname: uploaded.pathname,
contentType: uploaded.contentType,
bytes: downloaded.buffer.length,
};
}

View File

@ -0,0 +1,476 @@
import "server-only";
import { eq, like, or, sql } from "drizzle-orm";
import { db } from "@/lib/db";
import {
assetMemorySnapshot,
canvasDetailArchive,
canvasNode,
canvasProject,
conversationWorkingState,
externalImageApiRequestLog,
generatedImage,
mediaAsset,
message,
messageDeprecated,
model3dAsset,
} from "@/lib/db/schema";
import { inferOriginalImageUrl } from "@/lib/image-url";
function normalizeMediaUrl(value: string): string {
const raw = String(value || "").trim();
if (!raw) return "";
const canonical = inferOriginalImageUrl(raw);
try {
const parsed = new URL(canonical);
return `${parsed.origin}${parsed.pathname}`.replace(/\/+$/, "").toLowerCase();
} catch {
return canonical
.split("#")[0]
.split("?")[0]
.replace(/\/+$/, "")
.toLowerCase();
}
}
export function replaceDeepUrls(
value: unknown,
sourceUrl: string,
targetUrl: string
): { value: unknown; changed: boolean } {
if (typeof value === "string") {
if (!value.includes(sourceUrl)) {
return { value, changed: false };
}
return { value: value.split(sourceUrl).join(targetUrl), changed: true };
}
if (Array.isArray(value)) {
let changed = false;
const next = value.map((item) => {
const result = replaceDeepUrls(item, sourceUrl, targetUrl);
if (result.changed) changed = true;
return result.value;
});
return { value: changed ? next : value, changed };
}
if (value && typeof value === "object") {
let changed = false;
const next: Record<string, unknown> = {};
for (const [key, item] of Object.entries(value as Record<string, unknown>)) {
const result = replaceDeepUrls(item, sourceUrl, targetUrl);
next[key] = result.value;
if (result.changed) changed = true;
}
return { value: changed ? next : value, changed };
}
return { value, changed: false };
}
export async function replaceImageUrlInDatabase(
sourceUrl: string,
targetUrl: string
) {
if (!sourceUrl || !targetUrl || sourceUrl === targetUrl) {
return { changed: 0 };
}
let changed = 0;
const sourceNormalizedUrl = normalizeMediaUrl(sourceUrl);
const targetNormalizedUrl = normalizeMediaUrl(targetUrl);
const generatedThumb = await db
.update(generatedImage)
.set({ thumbnailUrl: targetUrl })
.where(eq(generatedImage.thumbnailUrl, sourceUrl))
.returning({ id: generatedImage.id });
changed += generatedThumb.length;
const generatedOrig = await db
.update(generatedImage)
.set({ originalUrl: targetUrl })
.where(eq(generatedImage.originalUrl, sourceUrl))
.returning({ id: generatedImage.id });
changed += generatedOrig.length;
const assets = await db
.select()
.from(mediaAsset)
.where(
or(
eq(mediaAsset.originalUrl, sourceUrl),
eq(mediaAsset.previewUrl, sourceUrl),
eq(mediaAsset.thumbnailUrl, sourceUrl),
eq(mediaAsset.normalizedUrl, sourceNormalizedUrl),
sql`${mediaAsset.metadata}::text like ${`%${sourceUrl}%`}`,
sql`${mediaAsset.summary}::text like ${`%${sourceUrl}%`}`
)
)
.limit(500);
for (const item of assets) {
const original = replaceDeepUrls(item.originalUrl, sourceUrl, targetUrl);
const preview = replaceDeepUrls(item.previewUrl, sourceUrl, targetUrl);
const thumbnail = replaceDeepUrls(item.thumbnailUrl, sourceUrl, targetUrl);
const metadata = replaceDeepUrls(item.metadata, sourceUrl, targetUrl);
const summary = replaceDeepUrls(item.summary, sourceUrl, targetUrl);
const shouldNormalize =
item.normalizedUrl === sourceNormalizedUrl || original.changed;
const nextOriginal = (original.value as string) || item.originalUrl;
if (
!original.changed &&
!preview.changed &&
!thumbnail.changed &&
!metadata.changed &&
!summary.changed &&
!shouldNormalize
) {
continue;
}
try {
await db
.update(mediaAsset)
.set({
originalUrl: nextOriginal,
normalizedUrl: shouldNormalize ? targetNormalizedUrl : item.normalizedUrl,
previewUrl: (preview.value as string | null | undefined) ?? item.previewUrl,
thumbnailUrl:
(thumbnail.value as string | null | undefined) ?? item.thumbnailUrl,
metadata: (metadata.value ?? item.metadata) as any,
summary: (summary.value ?? item.summary) as any,
updatedAt: new Date(),
})
.where(eq(mediaAsset.id, item.id));
} catch (error) {
await db
.update(mediaAsset)
.set({
originalUrl: nextOriginal,
previewUrl: (preview.value as string | null | undefined) ?? item.previewUrl,
thumbnailUrl:
(thumbnail.value as string | null | undefined) ?? item.thumbnailUrl,
metadata: (metadata.value ?? item.metadata) as any,
summary: (summary.value ?? item.summary) as any,
updatedAt: new Date(),
})
.where(eq(mediaAsset.id, item.id));
}
changed += 1;
}
const snapshots = await db
.select()
.from(assetMemorySnapshot)
.where(
or(
eq(assetMemorySnapshot.normalizedUrl, sourceNormalizedUrl),
sql`${assetMemorySnapshot.metadata}::text like ${`%${sourceUrl}%`}`,
like(assetMemorySnapshot.summaryText, `%${sourceUrl}%`)
)
)
.limit(500);
for (const item of snapshots) {
const metadata = replaceDeepUrls(item.metadata, sourceUrl, targetUrl);
const summary = replaceDeepUrls(item.summaryText, sourceUrl, targetUrl);
const shouldNormalize = item.normalizedUrl === sourceNormalizedUrl;
if (!metadata.changed && !summary.changed && !shouldNormalize) continue;
await db
.update(assetMemorySnapshot)
.set({
normalizedUrl: shouldNormalize ? targetNormalizedUrl : item.normalizedUrl,
metadata: (metadata.value ?? item.metadata) as any,
summaryText: (summary.value as string | null | undefined) ?? item.summaryText,
updatedAt: new Date(),
})
.where(eq(assetMemorySnapshot.id, item.id));
changed += 1;
}
const messages = await db
.select({
id: message.id,
parts: message.parts,
attachments: message.attachments,
})
.from(message)
.where(
or(
sql`${message.parts}::text like ${`%${sourceUrl}%`}`,
sql`${message.attachments}::text like ${`%${sourceUrl}%`}`
)
)
.limit(500);
for (const item of messages) {
const partsResult = replaceDeepUrls(item.parts, sourceUrl, targetUrl);
const attachmentsResult = replaceDeepUrls(item.attachments, sourceUrl, targetUrl);
if (!partsResult.changed && !attachmentsResult.changed) continue;
await db
.update(message)
.set({
parts: (partsResult.value ?? item.parts) as any,
attachments: (attachmentsResult.value ?? item.attachments) as any,
})
.where(eq(message.id, item.id));
changed += 1;
}
const legacyMessages = await db
.select({
id: messageDeprecated.id,
content: messageDeprecated.content,
})
.from(messageDeprecated)
.where(sql`${messageDeprecated.content}::text like ${`%${sourceUrl}%`}`)
.limit(500);
for (const item of legacyMessages) {
const replaced = replaceDeepUrls(item.content, sourceUrl, targetUrl);
if (!replaced.changed) continue;
await db
.update(messageDeprecated)
.set({ content: replaced.value as any })
.where(eq(messageDeprecated.id, item.id));
changed += 1;
}
const projects = await db
.select({
id: canvasProject.id,
canvasData: canvasProject.canvasData,
})
.from(canvasProject)
.where(sql`${canvasProject.canvasData}::text like ${`%${sourceUrl}%`}`)
.limit(500);
for (const item of projects) {
const replaced = replaceDeepUrls(item.canvasData, sourceUrl, targetUrl);
if (!replaced.changed) continue;
await db
.update(canvasProject)
.set({
canvasData: replaced.value as any,
updatedAt: new Date(),
})
.where(eq(canvasProject.id, item.id));
changed += 1;
}
const nodes = await db
.select({
id: canvasNode.id,
data: canvasNode.data,
})
.from(canvasNode)
.where(sql`${canvasNode.data}::text like ${`%${sourceUrl}%`}`)
.limit(500);
for (const item of nodes) {
const replaced = replaceDeepUrls(item.data, sourceUrl, targetUrl);
if (!replaced.changed) continue;
await db
.update(canvasNode)
.set({
data: replaced.value as any,
updatedAt: new Date(),
})
.where(eq(canvasNode.id, item.id));
changed += 1;
}
const workingStates = await db
.select()
.from(conversationWorkingState)
.where(
or(
sql`${conversationWorkingState.currentFocusImageUrls}::text like ${`%${sourceUrl}%`}`,
sql`${conversationWorkingState.selectedImageUrls}::text like ${`%${sourceUrl}%`}`,
sql`${conversationWorkingState.lastUploadedImageUrls}::text like ${`%${sourceUrl}%`}`,
sql`${conversationWorkingState.lastGeneratedImageUrls}::text like ${`%${sourceUrl}%`}`,
sql`${conversationWorkingState.lastReferencedImageUrls}::text like ${`%${sourceUrl}%`}`,
sql`${conversationWorkingState.lastResolvedTargetImageUrls}::text like ${`%${sourceUrl}%`}`,
sql`${conversationWorkingState.lastCanvasSnapshot}::text like ${`%${sourceUrl}%`}`,
sql`${conversationWorkingState.lastTurnSummary}::text like ${`%${sourceUrl}%`}`
)
)
.limit(500);
for (const item of workingStates) {
const currentFocus = replaceDeepUrls(
item.currentFocusImageUrls,
sourceUrl,
targetUrl
);
const selected = replaceDeepUrls(item.selectedImageUrls, sourceUrl, targetUrl);
const uploaded = replaceDeepUrls(item.lastUploadedImageUrls, sourceUrl, targetUrl);
const generated = replaceDeepUrls(item.lastGeneratedImageUrls, sourceUrl, targetUrl);
const referenced = replaceDeepUrls(item.lastReferencedImageUrls, sourceUrl, targetUrl);
const resolved = replaceDeepUrls(
item.lastResolvedTargetImageUrls,
sourceUrl,
targetUrl
);
const canvasSnapshot = replaceDeepUrls(
item.lastCanvasSnapshot,
sourceUrl,
targetUrl
);
const turnSummary = replaceDeepUrls(item.lastTurnSummary, sourceUrl, targetUrl);
if (
!currentFocus.changed &&
!selected.changed &&
!uploaded.changed &&
!generated.changed &&
!referenced.changed &&
!resolved.changed &&
!canvasSnapshot.changed &&
!turnSummary.changed
) {
continue;
}
await db
.update(conversationWorkingState)
.set({
currentFocusImageUrls: currentFocus.value as any,
selectedImageUrls: selected.value as any,
lastUploadedImageUrls: uploaded.value as any,
lastGeneratedImageUrls: generated.value as any,
lastReferencedImageUrls: referenced.value as any,
lastResolvedTargetImageUrls: resolved.value as any,
lastCanvasSnapshot: (canvasSnapshot.value ?? item.lastCanvasSnapshot) as any,
lastTurnSummary: (turnSummary.value ?? item.lastTurnSummary) as any,
updatedAt: new Date(),
})
.where(eq(conversationWorkingState.id, item.id));
changed += 1;
}
const detailArchives = await db
.select({
id: canvasDetailArchive.id,
coverUrl: canvasDetailArchive.coverUrl,
brief: canvasDetailArchive.brief,
slots: canvasDetailArchive.slots,
uiSurfaceSnapshot: canvasDetailArchive.uiSurfaceSnapshot,
uiSurfaceLatestReview: canvasDetailArchive.uiSurfaceLatestReview,
})
.from(canvasDetailArchive)
.where(
or(
like(canvasDetailArchive.coverUrl, `%${sourceUrl}%`),
sql`${canvasDetailArchive.brief}::text like ${`%${sourceUrl}%`}`,
sql`${canvasDetailArchive.slots}::text like ${`%${sourceUrl}%`}`,
sql`${canvasDetailArchive.uiSurfaceSnapshot}::text like ${`%${sourceUrl}%`}`,
sql`${canvasDetailArchive.uiSurfaceLatestReview}::text like ${`%${sourceUrl}%`}`
)
)
.limit(500);
for (const item of detailArchives) {
const cover = replaceDeepUrls(item.coverUrl, sourceUrl, targetUrl);
const brief = replaceDeepUrls(item.brief, sourceUrl, targetUrl);
const slots = replaceDeepUrls(item.slots, sourceUrl, targetUrl);
const snapshot = replaceDeepUrls(item.uiSurfaceSnapshot, sourceUrl, targetUrl);
const review = replaceDeepUrls(item.uiSurfaceLatestReview, sourceUrl, targetUrl);
if (
!cover.changed &&
!brief.changed &&
!slots.changed &&
!snapshot.changed &&
!review.changed
) {
continue;
}
await db
.update(canvasDetailArchive)
.set({
coverUrl: (cover.value as string | null | undefined) ?? item.coverUrl,
brief: (brief.value ?? item.brief) as any,
slots: (slots.value ?? item.slots) as any,
uiSurfaceSnapshot: (snapshot.value ?? item.uiSurfaceSnapshot) as any,
uiSurfaceLatestReview: (review.value ?? item.uiSurfaceLatestReview) as any,
updatedAt: new Date(),
})
.where(eq(canvasDetailArchive.id, item.id));
changed += 1;
}
const modelAssets = await db
.select()
.from(model3dAsset)
.where(
or(
like(model3dAsset.previewImageUrl, `%${sourceUrl}%`),
sql`${model3dAsset.assetPayload}::text like ${`%${sourceUrl}%`}`
)
)
.limit(500);
for (const item of modelAssets) {
const preview = replaceDeepUrls(item.previewImageUrl, sourceUrl, targetUrl);
const payload = replaceDeepUrls(item.assetPayload, sourceUrl, targetUrl);
if (!preview.changed && !payload.changed) continue;
await db
.update(model3dAsset)
.set({
previewImageUrl:
(preview.value as string | null | undefined) ?? item.previewImageUrl,
assetPayload: (payload.value ?? item.assetPayload) as any,
updatedAt: new Date(),
})
.where(eq(model3dAsset.id, item.id));
changed += 1;
}
const apiLogs = await db
.select()
.from(externalImageApiRequestLog)
.where(
or(
sql`${externalImageApiRequestLog.responsePayload}::text like ${`%${sourceUrl}%`}`,
sql`${externalImageApiRequestLog.generatedImageUrls}::text like ${`%${sourceUrl}%`}`
)
)
.limit(500);
for (const item of apiLogs) {
const responsePayload = replaceDeepUrls(
item.responsePayload,
sourceUrl,
targetUrl
);
const generatedUrls = replaceDeepUrls(
item.generatedImageUrls,
sourceUrl,
targetUrl
);
if (!responsePayload.changed && !generatedUrls.changed) continue;
await db
.update(externalImageApiRequestLog)
.set({
responsePayload: (responsePayload.value ?? item.responsePayload) as any,
generatedImageUrls: (generatedUrls.value ?? item.generatedImageUrls) as any,
})
.where(eq(externalImageApiRequestLog.id, item.id));
changed += 1;
}
return { changed };
}

View File

@ -54,13 +54,14 @@ function getStorageTimeoutMs(name: string, fallback: number, min: number, max: n
} }
async function withStorageOperationTimeout<T>( async function withStorageOperationTimeout<T>(
operation: Promise<T>, operation: (abortSignal: AbortSignal) => Promise<T>,
timeoutMs: number, timeoutMs: number,
label: string label: string
): Promise<T> { ): Promise<T> {
let timedOut = false; let timedOut = false;
let timeoutId: ReturnType<typeof setTimeout> | undefined; let timeoutId: ReturnType<typeof setTimeout> | undefined;
const guardedOperation = operation.catch((error) => { const abortController = new AbortController();
const guardedOperation = operation(abortController.signal).catch((error) => {
if (timedOut) { if (timedOut) {
logger.warn(`[Upload] ${label} failed after timeout:`, error); logger.warn(`[Upload] ${label} failed after timeout:`, error);
return new Promise<T>(() => undefined); return new Promise<T>(() => undefined);
@ -70,6 +71,7 @@ async function withStorageOperationTimeout<T>(
const timeout = new Promise<never>((_, reject) => { const timeout = new Promise<never>((_, reject) => {
timeoutId = setTimeout(() => { timeoutId = setTimeout(() => {
timedOut = true; timedOut = true;
abortController.abort();
reject(new Error(`${label} timed out after ${timeoutMs}ms`)); reject(new Error(`${label} timed out after ${timeoutMs}ms`));
}, timeoutMs); }, timeoutMs);
}); });
@ -256,7 +258,7 @@ async function uploadToS3(
}); });
await withStorageOperationTimeout( await withStorageOperationTimeout(
s3Client.send(command), (abortSignal) => s3Client.send(command, { abortSignal }),
getStorageTimeoutMs( getStorageTimeoutMs(
"S3_UPLOAD_REQUEST_TIMEOUT_MS", "S3_UPLOAD_REQUEST_TIMEOUT_MS",
DEFAULT_S3_REQUEST_TIMEOUT_MS, DEFAULT_S3_REQUEST_TIMEOUT_MS,

View File

@ -3,7 +3,10 @@
{ {
"path": "/api/jobs/archive-videos", "path": "/api/jobs/archive-videos",
"schedule": "0 * * * *" "schedule": "0 * * * *"
},
{
"path": "/api/jobs/archive-images",
"schedule": "5 * * * *"
} }
] ]
} }