From 748273d9173bca1ad3758d283b9eda9932b1f4b6 Mon Sep 17 00:00:00 2001 From: 134 Date: Fri, 10 Jul 2026 20:27:09 +0800 Subject: [PATCH] feat: defer generated image archiving --- app/api/jobs/archive-images/route.ts | 65 +++ lib/ai/providers/alibaba-image.ts | 6 + lib/ai/providers/bfl-image.ts | 6 + lib/ai/providers/fal-image.ts | 6 + lib/ai/providers/gemini-image.ts | 6 + lib/ai/providers/huanke-gemini.ts | 6 + lib/ai/providers/krea.ts | 9 + lib/ai/providers/midjourney.ts | 6 + lib/ai/providers/openai-image.ts | 6 + lib/ai/providers/runway.ts | 9 + lib/ai/providers/ruxa-image.ts | 6 + lib/ai/providers/snapgen-image.ts | 6 + lib/ai/providers/stepfun-image.ts | 6 + lib/ai/providers/volcano-vision.ts | 8 + .../0087_add_image_archive_tasks.sql | 22 + lib/db/migrations/meta/_journal.json | 7 + lib/db/schema.ts | 20 + lib/jobs/image-archive-job.ts | 118 +++++ lib/storage/image-archive-db.ts | 240 +++++++++ lib/storage/image-archive.ts | 198 ++++++++ lib/storage/image-url-replacement.ts | 476 ++++++++++++++++++ lib/storage/service.ts | 8 +- vercel.json | 5 +- 23 files changed, 1241 insertions(+), 4 deletions(-) create mode 100644 app/api/jobs/archive-images/route.ts create mode 100644 lib/db/migrations/0087_add_image_archive_tasks.sql create mode 100644 lib/jobs/image-archive-job.ts create mode 100644 lib/storage/image-archive-db.ts create mode 100644 lib/storage/image-archive.ts create mode 100644 lib/storage/image-url-replacement.ts diff --git a/app/api/jobs/archive-images/route.ts b/app/api/jobs/archive-images/route.ts new file mode 100644 index 000000000..b74dad4d0 --- /dev/null +++ b/app/api/jobs/archive-images/route.ts @@ -0,0 +1,65 @@ +import { NextResponse } from "next/server"; +import { runImageArchiveJob } from "@/lib/jobs/image-archive-job"; +import { getImageArchiveTaskStats } from "@/lib/storage/image-archive-db"; + +function verifyAuthToken(request: Request): boolean { + const authHeader = request.headers.get("authorization"); + const token = authHeader?.replace("Bearer ", ""); + const cronSecret = process.env.CRON_SECRET; + + if (!cronSecret) { + console.error("[ImageArchiveAPI] CRON_SECRET is not configured"); + return false; + } + + return token === cronSecret; +} + +export async function POST(request: Request) { + try { + if (!verifyAuthToken(request)) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const result = await runImageArchiveJob(); + return NextResponse.json({ + ...result, + timestamp: new Date().toISOString(), + }); + } catch (error) { + console.error("[ImageArchiveAPI] Job execution failed:", error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : String(error), + timestamp: new Date().toISOString(), + }, + { status: 500 } + ); + } +} + +export async function GET(request: Request) { + try { + if (!verifyAuthToken(request)) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const stats = await getImageArchiveTaskStats(); + return NextResponse.json({ + success: true, + stats, + timestamp: new Date().toISOString(), + }); + } catch (error) { + console.error("[ImageArchiveAPI] Stats query failed:", error); + return NextResponse.json( + { + success: false, + error: error instanceof Error ? error.message : String(error), + timestamp: new Date().toISOString(), + }, + { status: 500 } + ); + } +} diff --git a/lib/ai/providers/alibaba-image.ts b/lib/ai/providers/alibaba-image.ts index d00f20d2e..17b2e14a8 100644 --- a/lib/ai/providers/alibaba-image.ts +++ b/lib/ai/providers/alibaba-image.ts @@ -10,6 +10,7 @@ import sharp from "sharp"; import { constrainImageDimensions } from "@/lib/ai/image-size-constraints"; +import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; interface AlibabaImageConfig { @@ -358,6 +359,11 @@ async function downloadAndUploadToS3( prefix: string = "alibaba-image" ): Promise<{ thumbnail: string; original: string }> { try { + const deferred = await createDeferredUploadedImage(imageUrl, { + providerType: "alibaba-image", + }); + if (deferred) return deferred; + console.log("[Alibaba Image] Downloading image from:", imageUrl.substring(0, 100) + "..."); const { buffer } = await fetchImageBufferWithRetry(imageUrl, 30000, 3); diff --git a/lib/ai/providers/bfl-image.ts b/lib/ai/providers/bfl-image.ts index f0026fcfd..adddc2f07 100644 --- a/lib/ai/providers/bfl-image.ts +++ b/lib/ai/providers/bfl-image.ts @@ -2,6 +2,7 @@ import "server-only"; import sharp from "sharp"; import { resolveLongestEdgeDimensions } from "@/lib/ai/image-size-constraints"; +import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; type BflImageConfig = { @@ -273,6 +274,11 @@ async function uploadGeneratedBuffer( } async function downloadAndUploadImage(imageUrl: string): Promise { + const deferred = await createDeferredUploadedImage(imageUrl, { + providerType: "bfl-image", + }); + if (deferred) return deferred; + const response = await fetch(imageUrl, { cache: "no-store" }); if (!response.ok) { throw new Error(`Failed to download generated image: ${response.status}`); diff --git a/lib/ai/providers/fal-image.ts b/lib/ai/providers/fal-image.ts index b4e8f37fc..15ed1866c 100644 --- a/lib/ai/providers/fal-image.ts +++ b/lib/ai/providers/fal-image.ts @@ -6,6 +6,7 @@ import { constrainImageDimensions, resolveLongestEdgeDimensions, } from "@/lib/ai/image-size-constraints"; +import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; interface FalImageConfig { @@ -466,6 +467,11 @@ function generateFilename(prefix: string, extension: string) { } async function downloadAndUploadImage(imageUrl: string): Promise { + const deferred = await createDeferredUploadedImage(imageUrl, { + providerType: "fal-image", + }); + if (deferred) return deferred; + const response = await fetch(imageUrl); if (!response.ok) { throw new Error(`Failed to download image: ${response.status}`); diff --git a/lib/ai/providers/gemini-image.ts b/lib/ai/providers/gemini-image.ts index 86b34563d..10f1dfb9f 100644 --- a/lib/ai/providers/gemini-image.ts +++ b/lib/ai/providers/gemini-image.ts @@ -1,6 +1,7 @@ import "server-only"; import { resolveGrsaiGptImageSize } from "@/lib/ai/grsai-gpt-image-sizes"; +import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; import { createLogger } from "@/lib/observability/logger"; @@ -170,6 +171,11 @@ async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/jp * 带重试机制，确保成功下载并上传到国内中转 */ async function downloadAndUploadToS3(imageUrl: string, retries = 3): Promise<{ thumbnail: string; original: string }> { + const deferred = await createDeferredUploadedImage(imageUrl, { + providerType: "gemini-image", + }); + if (deferred) return deferred; + for (let attempt = 0; attempt < retries; attempt++) { try { logger.debug(`[Gemini Image] Downloading image from URL (attempt ${attempt + 1}/${retries}): ${imageUrl.substring(0, 100)}...`); diff --git a/lib/ai/providers/huanke-gemini.ts b/lib/ai/providers/huanke-gemini.ts index 23b1c054b..aa11bee94 100644 --- a/lib/ai/providers/huanke-gemini.ts +++ b/lib/ai/providers/huanke-gemini.ts @@ -1,5 +1,6 @@ import "server-only"; +import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; interface HuankeGeminiConfig { @@ -112,6 +113,11 @@ async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/jp * 下载远程URL图片并上传到 S3 */ async function downloadAndUploadToS3(imageUrl: string, retries = 3): Promise<{ thumbnail: string; original: string }> { + const deferred = await createDeferredUploadedImage(imageUrl, { + providerType: "huanke-gemini", + }); + if (deferred) return deferred; + for (let attempt = 0; attempt < retries; attempt++) { try { console.log(`[Huanke Gemini] Downloading image from URL (attempt ${attempt + 1}/${retries}): ${imageUrl.substring(0, 100)}...`); diff --git a/lib/ai/providers/krea.ts b/lib/ai/providers/krea.ts index b2d3b6772..326de68f3 100644 --- a/lib/ai/providers/krea.ts +++ b/lib/ai/providers/krea.ts @@ -7,6 +7,7 @@ import { inferKreaModelKind, type KreaModelKind, } from "@/lib/ai/krea-models"; +import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; interface KreaConfig { @@ -651,6 +652,14 @@ async function uploadKreaImages(payload: unknown) { const uploadedImages: UploadedImage[] = []; for (const url of collectHttpUrls(payload)) { try { + const deferred = await createDeferredUploadedImage(url, { + providerType: "krea", + }); + if (deferred) { + uploadedImages.push(deferred); + continue; + } + const { buffer, contentType } = await downloadRemoteAsset(url); if (!String(contentType || "").startsWith("image/")) continue; uploadedImages.push(await uploadGeneratedBuffer(buffer, detectExtensionFromContentType(contentType))); diff --git a/lib/ai/providers/midjourney.ts b/lib/ai/providers/midjourney.ts index 8cc722410..cd0ae4df2 100644 --- a/lib/ai/providers/midjourney.ts +++ b/lib/ai/providers/midjourney.ts @@ -1,6 +1,7 @@ import "server-only"; import sharp from "sharp"; +import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; interface MidjourneyConfig { @@ -161,6 +162,11 @@ async function uploadGeneratedBuffer( } async function downloadAndUploadImage(imageUrl: string): Promise { + const deferred = await createDeferredUploadedImage(imageUrl, { + providerType: "midjourney", + }); + if (deferred) return deferred; + const response = await fetch(imageUrl, { cache: "no-store" }); if (!response.ok) { throw new Error(`Failed to download Midjourney image: ${response.status}`); diff --git a/lib/ai/providers/openai-image.ts b/lib/ai/providers/openai-image.ts index 1dd0ee129..afcf8fd82 100644 --- a/lib/ai/providers/openai-image.ts +++ b/lib/ai/providers/openai-image.ts @@ -4,6 +4,7 @@ import sharp from "sharp"; import { resolveLongestEdgeDimensions, } from "@/lib/ai/image-size-constraints"; +import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; interface OpenAIImageConfig { @@ -413,6 +414,11 @@ async function uploadGeneratedBuffer( } async function downloadAndUploadImage(imageUrl: string): Promise { + const deferred = await createDeferredUploadedImage(imageUrl, { + providerType: "openai-image", + }); + if (deferred) return deferred; + const response = await fetch(imageUrl, { cache: "no-store" }); if (!response.ok) { throw new Error(`Failed to download generated image: ${response.status}`); diff --git a/lib/ai/providers/runway.ts b/lib/ai/providers/runway.ts index cfe1b1637..689444a9a 100644 --- a/lib/ai/providers/runway.ts +++ b/lib/ai/providers/runway.ts @@ -8,6 +8,7 @@ import { normalizeRunwayDuration, resolveRunwayApiRatio, } from "@/lib/ai/runway-video-models"; +import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; import { inferRunwayModelKind } from "@/lib/ai/model-classification"; @@ -432,6 +433,14 @@ async function uploadRunwayImages(taskOutput: unknown) { for (const url of outputUrls) { try { + const deferred = await createDeferredUploadedImage(url, { + providerType: "runway", + }); + if (deferred) { + uploadedImages.push(deferred); + continue; + } + const { buffer, contentType } = await downloadRemoteAsset(url); if (!String(contentType || "").startsWith("image/")) { continue; diff --git a/lib/ai/providers/ruxa-image.ts b/lib/ai/providers/ruxa-image.ts index 54580e0a7..83dccce08 100644 --- a/lib/ai/providers/ruxa-image.ts +++ b/lib/ai/providers/ruxa-image.ts @@ -1,6 +1,7 @@ import "server-only"; import sharp from "sharp"; +import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; import { resolveRuxaImageRuntimeModelId, @@ -369,6 +370,11 @@ function generateFilename(prefix: string, extension: string) { } async function downloadAndUploadImage(imageUrl: string): Promise { + const deferred = await createDeferredUploadedImage(imageUrl, { + providerType: "ruxa-image", + }); + if (deferred) return deferred; + const response = await fetch(imageUrl); if (!response.ok) { throw new Error(`Failed to download image: ${response.status}`); diff --git a/lib/ai/providers/snapgen-image.ts b/lib/ai/providers/snapgen-image.ts index d6b86fcf8..ae39b719e 100644 --- a/lib/ai/providers/snapgen-image.ts +++ b/lib/ai/providers/snapgen-image.ts @@ -1,6 +1,7 @@ import "server-only"; import sharp from "sharp"; +import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; import { buildSnapgenReferenceOrderPrompt } from "@/lib/ai/providers/snapgen-reference-order"; import { @@ -690,6 +691,11 @@ async function downloadAndUploadImage( imageUrl: string, timeoutMs = IMAGE_DOWNLOAD_TIMEOUT_MS ): Promise { + const deferred = await createDeferredUploadedImage(imageUrl, { + providerType: "snapgen-image", + }); + if (deferred) return deferred; + const response = await fetchWithTimeout(imageUrl, {}, timeoutMs); if (!response.ok) { throw new Error(`Failed to download SnapGen image: ${response.status}`); diff --git a/lib/ai/providers/stepfun-image.ts b/lib/ai/providers/stepfun-image.ts index 48a074314..59cc55c36 100644 --- a/lib/ai/providers/stepfun-image.ts +++ b/lib/ai/providers/stepfun-image.ts @@ -2,6 +2,7 @@ import "server-only"; import sharp from "sharp"; import { constrainImageDimensions } from "@/lib/ai/image-size-constraints"; +import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; interface StepfunImageConfig { @@ -160,6 +161,11 @@ async function uploadGeneratedBuffer( } async function downloadAndUploadImage(imageUrl: string): Promise { + const deferred = await createDeferredUploadedImage(imageUrl, { + providerType: "stepfun-image", + }); + if (deferred) return deferred; + const response = await fetch(imageUrl, { cache: "no-store" }); if (!response.ok) { throw new Error(`Failed to download StepFun generated image: ${response.status}`); diff --git a/lib/ai/providers/volcano-vision.ts b/lib/ai/providers/volcano-vision.ts index d356c2017..d4f5e60c6 100644 --- a/lib/ai/providers/volcano-vision.ts +++ b/lib/ai/providers/volcano-vision.ts @@ -1,6 +1,7 @@ import "server-only"; import { constrainImageDimensions } from "@/lib/ai/image-size-constraints"; +import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; import { getDefaultStorageConfig } from "@/lib/db/queries"; import { createLogger } from "@/lib/observability/logger"; @@ -429,6 +430,13 @@ async function createThumbnail(buffer: Buffer, mimeType: string): Promise { try { + if (/^https?:\/\//i.test(imageData)) { + const deferred = await createDeferredUploadedImage(imageData, { + providerType: "volcano-vision", + }); + if (deferred) return deferred; + } + let buffer: Buffer; let mimeType = "image/jpeg"; diff --git a/lib/db/migrations/0087_add_image_archive_tasks.sql b/lib/db/migrations/0087_add_image_archive_tasks.sql new file mode 100644 index 000000000..b284fed2b --- /dev/null +++ b/lib/db/migrations/0087_add_image_archive_tasks.sql @@ -0,0 +1,22 @@ +CREATE TABLE IF NOT EXISTS "ImageArchiveTask" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "sourceUrl" text NOT NULL UNIQUE, + "targetUrl" text, + "targetStorageConfigId" uuid REFERENCES "StorageConfig"("id"), + "status" varchar(32) NOT NULL DEFAULT 'pending', + "attemptCount" integer NOT NULL DEFAULT 0, + "lastError" text, + "archiveAfterAt" timestamp NOT NULL, + "replaceAfterAt" timestamp NOT NULL, + "archivedAt" timestamp, + "replacedAt" timestamp, + "metadata" jsonb, + "createdAt" timestamp NOT NULL DEFAULT now(), + "updatedAt" timestamp NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS "ImageArchiveTask_status_archive_idx" + ON "ImageArchiveTask"("status", "archiveAfterAt", "createdAt"); + +CREATE INDEX IF NOT EXISTS "ImageArchiveTask_status_replace_idx" + ON "ImageArchiveTask"("status", "replaceAfterAt", "createdAt"); diff --git a/lib/db/migrations/meta/_journal.json b/lib/db/migrations/meta/_journal.json index 236089b9d..5236c21b5 100644 --- a/lib/db/migrations/meta/_journal.json +++ b/lib/db/migrations/meta/_journal.json @@ -512,6 +512,13 @@ "when": 1781174400000, "tag": "0086_external_image_log_model_delete_set_null", "breakpoints": true + }, + { + "idx": 73, + "version": "7", + "when": 1783680000000, + "tag": "0087_add_image_archive_tasks", + "breakpoints": true } ] } diff --git a/lib/db/schema.ts b/lib/db/schema.ts index b95052b83..bca301c10 100644 --- a/lib/db/schema.ts +++ b/lib/db/schema.ts @@ -441,6 +441,26 @@ export const storageMigrationTask = pgTable("StorageMigrationTask", { export type StorageMigrationTask = InferSelectModel; +// AI 生成图片临时地址归档任务表 +export const imageArchiveTask = pgTable("ImageArchiveTask", { + id: uuid("id").primaryKey().notNull().defaultRandom(), + sourceUrl: text("sourceUrl").notNull().unique(), + targetUrl: text("targetUrl"), + targetStorageConfigId: uuid("targetStorageConfigId").references(() => storageConfig.id), + status: varchar("status", { length: 32 }).notNull().default("pending"), + attemptCount: integer("attemptCount").notNull().default(0), + lastError: text("lastError"), + archiveAfterAt: timestamp("archiveAfterAt").notNull(), + replaceAfterAt: timestamp("replaceAfterAt").notNull(), + archivedAt: timestamp("archivedAt"), + replacedAt: timestamp("replacedAt"), + metadata: jsonb("metadata"), + createdAt: timestamp("createdAt").notNull().defaultNow(), + updatedAt: timestamp("updatedAt").notNull().defaultNow(), +}); + +export type ImageArchiveTask = InferSelectModel; + // 网站设置表 export const siteSettings = pgTable("SiteSettings", { id: uuid("id").primaryKey().notNull().defaultRandom(), diff --git a/lib/jobs/image-archive-job.ts b/lib/jobs/image-archive-job.ts new file mode 100644 index 000000000..3f6589d8d --- /dev/null +++ b/lib/jobs/image-archive-job.ts @@ -0,0 +1,118 @@ +import "server-only"; + +import { archiveTemporaryImage } from "@/lib/storage/image-archive"; +import { + getImageArchiveTaskStats, + getImageArchiveTasksToReplace, + getImageArchiveTasksToStore, + markImageArchiveTaskArchiving, + markImageArchiveTaskCompleted, + markImageArchiveTaskFailed, + markImageArchiveTaskReplaceFailed, + markImageArchiveTaskReplacing, + markImageArchiveTaskStored, +} from "@/lib/storage/image-archive-db"; +import { replaceImageUrlInDatabase } from "@/lib/storage/image-url-replacement"; + +const DEFAULT_BATCH_SIZE = 10; + +function getBatchSize() { + const parsed = Number(process.env.IMAGE_ARCHIVE_BATCH_SIZE || DEFAULT_BATCH_SIZE); + if (!Number.isFinite(parsed)) return DEFAULT_BATCH_SIZE; + return Math.max(1, Math.min(50, Math.round(parsed))); +} + +export async function runImageArchiveJob() { + const batchSize = getBatchSize(); + console.log(`[ImageArchiveJob] Start image archive job (batchSize=${batchSize})`); + + const storeTasks = await getImageArchiveTasksToStore({ limit: batchSize }); + let storeSucceeded = 0; + let storeFailed = 0; + + for (const task of storeTasks) { + try { + await markImageArchiveTaskArchiving(task.id); + const archived = await archiveTemporaryImage(task.sourceUrl); + await markImageArchiveTaskStored({ + id: task.id, + targetUrl: archived.url, + }); + storeSucceeded += 1; + console.log("[ImageArchiveJob] stored", { + taskId: task.id, + bytes: archived.bytes, + sourceUrl: task.sourceUrl, + targetUrl: archived.url, + }); + } catch (error) { + storeFailed += 1; + await markImageArchiveTaskFailed({ id: task.id, error }); + console.error("[ImageArchiveJob] store failed", { + taskId: task.id, + sourceUrl: task.sourceUrl, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + const remainingSlots = Math.max(1, batchSize - storeTasks.length); + const replaceTasks = await getImageArchiveTasksToReplace({ + limit: remainingSlots, + }); + let replaceSucceeded = 0; + let replaceFailed = 0; + + for (const task of replaceTasks) { + if (!task.targetUrl) { + await markImageArchiveTaskReplaceFailed({ + id: task.id, + error: new Error("Missing targetUrl for stored image archive task"), + }); + replaceFailed += 1; + continue; + } + + try { + await markImageArchiveTaskReplacing(task.id); + const result = await replaceImageUrlInDatabase(task.sourceUrl, task.targetUrl); + await markImageArchiveTaskCompleted({ + id: task.id, + replacedCount: result.changed, + }); + replaceSucceeded += 1; + console.log("[ImageArchiveJob] replaced", { + taskId: task.id, + sourceUrl: task.sourceUrl, + targetUrl: task.targetUrl, + replacedCount: result.changed, + }); + } catch (error) { + replaceFailed += 1; + await markImageArchiveTaskReplaceFailed({ id: task.id, error }); + console.error("[ImageArchiveJob] replace failed", { + taskId: task.id, + sourceUrl: task.sourceUrl, + targetUrl: task.targetUrl, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + const stats = await getImageArchiveTaskStats(); + return { + success: true, + message: "Image archive job completed", + store: { + total: storeTasks.length, + succeeded: storeSucceeded, + failed: storeFailed, + }, + replace: { + total: replaceTasks.length, + succeeded: replaceSucceeded, + failed: replaceFailed, + }, + stats, + }; +} diff --git a/lib/storage/image-archive-db.ts b/lib/storage/image-archive-db.ts new file mode 100644 index 000000000..06f56fc44 --- /dev/null +++ b/lib/storage/image-archive-db.ts @@ -0,0 +1,240 @@ +import "server-only"; + +import { and, asc, eq, inArray, lte, sql } from "drizzle-orm"; +import { db } from "@/lib/db"; +import { imageArchiveTask, type ImageArchiveTask } from "@/lib/db/schema"; + +export type ImageArchiveTaskRecord = ImageArchiveTask; + +const DEFAULT_DISPLAY_HOLD_HOURS = 1; + +function getDisplayHoldHours() { + const raw = Number( + process.env.IMAGE_ARCHIVE_DISPLAY_HOLD_HOURS || DEFAULT_DISPLAY_HOLD_HOURS + ); + if (!Number.isFinite(raw)) return DEFAULT_DISPLAY_HOLD_HOURS; + return Math.max(0, Math.min(24 * 30, raw)); +} + +export function getImageArchiveDisplayHoldMs() { + return getDisplayHoldHours() * 60 * 60 * 1000; +} + +function isArchivableHttpUrl(value: string) { + if (!/^https?:\/\//i.test(value)) return false; + if (/^(data|blob):/i.test(value)) return false; + return true; +} + +export async function enqueueImageArchiveTask(params: { + sourceUrl: string; + archiveAfterAt?: Date; + replaceAfterAt?: Date; + metadata?: Record | null; +}) { + const sourceUrl = String(params.sourceUrl || "").trim(); + if (!isArchivableHttpUrl(sourceUrl)) return null; + + const now = new Date(); + const archiveAfterAt = params.archiveAfterAt || now; + const replaceAfterAt = + params.replaceAfterAt || + new Date(now.getTime() + getImageArchiveDisplayHoldMs()); + + const [task] = await db + .insert(imageArchiveTask) + .values({ + sourceUrl, + status: "pending", + archiveAfterAt, + replaceAfterAt, + metadata: params.metadata || null, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: imageArchiveTask.sourceUrl, + set: { + replaceAfterAt, + metadata: params.metadata || undefined, + updatedAt: now, + }, + }) + .returning(); + + return task || null; +} + +export async function enqueueImageArchiveTasks(params: { + urls: string[]; + metadata?: Record | null; +}) { + const urls = Array.from( + new Set( + (params.urls || []) + .map((url) => String(url || "").trim()) + .filter(isArchivableHttpUrl) + ) + ); + + const tasks: ImageArchiveTaskRecord[] = []; + for (const sourceUrl of urls) { + const task = await enqueueImageArchiveTask({ + sourceUrl, + metadata: params.metadata || null, + }); + if (task) tasks.push(task); + } + return tasks; +} + +export async function getImageArchiveTasksToStore(params: { limit: number }) { + const now = new Date(); + return await db + .select() + .from(imageArchiveTask) + .where( + and( + inArray(imageArchiveTask.status, ["pending", "failed"]), + lte(imageArchiveTask.archiveAfterAt, now) + ) + ) + .orderBy(asc(imageArchiveTask.createdAt), asc(imageArchiveTask.id)) + .limit(Math.max(1, Math.min(50, params.limit))); +} + +export async function markImageArchiveTaskArchiving(id: string) { + await db + .update(imageArchiveTask) + .set({ + status: "archiving", + attemptCount: sql`${imageArchiveTask.attemptCount} + 1`, + lastError: null, + updatedAt: new Date(), + }) + .where(eq(imageArchiveTask.id, id)); +} + +export async function markImageArchiveTaskStored(params: { + id: string; + targetUrl: string; + targetStorageConfigId?: string | null; +}) { + await db + .update(imageArchiveTask) + .set({ + status: "stored", + targetUrl: params.targetUrl, + targetStorageConfigId: params.targetStorageConfigId || null, + archivedAt: new Date(), + lastError: null, + updatedAt: new Date(), + }) + .where(eq(imageArchiveTask.id, params.id)); +} + +export async function markImageArchiveTaskFailed(params: { + id: string; + error: unknown; +}) { + const message = String( + params.error instanceof Error ? params.error.message : params.error || "Unknown error" + ).slice(0, 2000); + + await db + .update(imageArchiveTask) + .set({ + status: "failed", + lastError: message, + archiveAfterAt: new Date(Date.now() + 10 * 60 * 1000), + updatedAt: new Date(), + }) + .where(eq(imageArchiveTask.id, params.id)); +} + +export async function getImageArchiveTasksToReplace(params: { limit: number }) { + const now = new Date(); + return await db + .select() + .from(imageArchiveTask) + .where( + and( + eq(imageArchiveTask.status, "stored"), + lte(imageArchiveTask.replaceAfterAt, now) + ) + ) + .orderBy(asc(imageArchiveTask.createdAt), asc(imageArchiveTask.id)) + .limit(Math.max(1, Math.min(50, params.limit))); +} + +export async function markImageArchiveTaskReplacing(id: string) { + await db + .update(imageArchiveTask) + .set({ + status: "replacing", + updatedAt: new Date(), + }) + .where(eq(imageArchiveTask.id, id)); +} + +export async function markImageArchiveTaskCompleted(params: { + id: string; + replacedCount?: number; +}) { + await db + .update(imageArchiveTask) + .set({ + status: "completed", + replacedAt: new Date(), + lastError: null, + metadata: + params.replacedCount === undefined + ? undefined + : sql`coalesce(${imageArchiveTask.metadata}, '{}'::jsonb) || ${JSON.stringify({ + replacedCount: params.replacedCount, + })}::jsonb`, + updatedAt: new Date(), + }) + .where(eq(imageArchiveTask.id, params.id)); +} + +export async function markImageArchiveTaskReplaceFailed(params: { + id: string; + error: unknown; +}) { + const message = String( + params.error instanceof Error ? params.error.message : params.error || "Unknown error" + ).slice(0, 2000); + + await db + .update(imageArchiveTask) + .set({ + status: "stored", + lastError: message, + replaceAfterAt: new Date(Date.now() + 10 * 60 * 1000), + updatedAt: new Date(), + }) + .where(eq(imageArchiveTask.id, params.id)); +} + +export async function getImageArchiveTaskStats() { + const grouped = await db + .select({ + status: imageArchiveTask.status, + count: sql`count(*)::int`, + }) + .from(imageArchiveTask) + .groupBy(imageArchiveTask.status); + + const pendingStore = grouped + .filter((item) => item.status === "pending" || item.status === "failed") + .reduce((sum, item) => sum + (item.count || 0), 0); + const pendingReplace = grouped + .filter((item) => item.status === "stored") + .reduce((sum, item) => sum + (item.count || 0), 0); + + return { + grouped, + pendingStore, + pendingReplace, + }; +} diff --git a/lib/storage/image-archive.ts b/lib/storage/image-archive.ts new file mode 100644 index 000000000..0c5ef4c84 --- /dev/null +++ b/lib/storage/image-archive.ts @@ -0,0 +1,198 @@ +import "server-only"; + +import { createLogger } from "@/lib/observability/logger"; +import { + enqueueImageArchiveTasks, + getImageArchiveDisplayHoldMs, +} from "@/lib/storage/image-archive-db"; +import { uploadFile } from "@/lib/storage/service"; + +const logger = createLogger("image-archive"); + +const DEFAULT_DOWNLOAD_TIMEOUT_MS = 60_000; + +type DeferredImageArchiveMetadata = Record & { + providerType?: string; + modelId?: string; + imageSize?: string; +}; + +function getDownloadTimeoutMs() { + const parsed = Number(process.env.IMAGE_ARCHIVE_DOWNLOAD_TIMEOUT_MS); + if (!Number.isFinite(parsed)) return DEFAULT_DOWNLOAD_TIMEOUT_MS; + return Math.max(5_000, Math.min(5 * 60_000, Math.round(parsed))); +} + +function isHttpUrl(value: string) { + return /^https?:\/\//i.test(String(value || "").trim()); +} + +export function isDeferredImageArchiveEnabled() { + return process.env.AI_IMAGE_DEFERRED_ARCHIVE !== "false"; +} + +export async function registerGeneratedImageTemporaryUrls(params: { + urls: string[]; + metadata?: DeferredImageArchiveMetadata | null; +}) { + if (!isDeferredImageArchiveEnabled()) return []; + const urls = Array.from( + new Set( + (params.urls || []) + .map((url) => String(url || "").trim()) + .filter(isHttpUrl) + ) + ); + if (urls.length === 0) return []; + + try { + return await enqueueImageArchiveTasks({ + urls, + metadata: { + ...(params.metadata || {}), + displayHoldMs: getImageArchiveDisplayHoldMs(), + }, + }); + } catch (error) { + logger.warn("enqueue temporary image archive failed", { + urls, + error: error instanceof Error ? error.message : String(error), + }); + return []; + } +} + +export async function formatDeferredGeneratedImageMarkdown(params: { + urls: string[]; + imageSize: string; + providerType?: string; + modelId?: string; + metadata?: DeferredImageArchiveMetadata | null; +}) { + const urls = Array.from( + new Set( + (params.urls || []) + .map((url) => String(url || "").trim()) + .filter(isHttpUrl) + ) + ); + if (urls.length === 0) { + throw new Error("AI 未返回可用图片地址"); + } + + await registerGeneratedImageTemporaryUrls({ + urls, + metadata: { + providerType: params.providerType, + modelId: params.modelId, + imageSize: params.imageSize, + ...(params.metadata || {}), + }, + }); + + return urls + .map((url) => `![thumb|||${url}|||${params.imageSize}](${url})`) + .join("\n\n"); +} + +export async function createDeferredUploadedImage( + sourceUrl: string, + metadata?: DeferredImageArchiveMetadata | null +) { + const url = String(sourceUrl || "").trim(); + if (!isDeferredImageArchiveEnabled() || !isHttpUrl(url)) return null; + + await registerGeneratedImageTemporaryUrls({ + urls: [url], + metadata: metadata || null, + }); + + return { + thumbnail: url, + original: url, + }; +} + +function getImageExtension(contentType: string | null, sourceUrl: string) { + const normalized = String(contentType || "").split(";")[0].trim().toLowerCase(); + const byType: Record = { + "image/jpeg": "jpg", + "image/jpg": "jpg", + "image/png": "png", + "image/webp": "webp", + "image/gif": "gif", + "image/svg+xml": "svg", + }; + if (byType[normalized]) return byType[normalized]; + + try { + const pathname = new URL(sourceUrl).pathname.toLowerCase(); + const ext = pathname.split(".").pop()?.replace(/[^a-z0-9]+/g, "") || ""; + if (["jpg", "jpeg", "png", "webp", "gif", "svg"].includes(ext)) { + return ext === "jpeg" ? "jpg" : ext; + } + } catch { + // keep fallback + } + + return "png"; +} + +function toArrayBuffer(buffer: Buffer): ArrayBuffer { + return buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) as ArrayBuffer; +} + +export async function downloadTemporaryImage(params: { + sourceUrl: string; + retries?: number; +}) { + const retries = Math.max(0, Math.min(5, params.retries ?? 2)); + + for (let attempt = 0; attempt <= retries; attempt += 1) { + try { + const response = await fetch(params.sourceUrl, { + signal: AbortSignal.timeout(getDownloadTimeoutMs()), + headers: { + "User-Agent": "AI-Chatbot-Image-Archiver/1.0", + Accept: "image/avif,image/webp,image/png,image/jpeg,image/*,*/*;q=0.8", + }, + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status} ${response.statusText}`); + } + + const contentType = response.headers.get("content-type"); + const arrayBuffer = await response.arrayBuffer(); + if (!arrayBuffer || arrayBuffer.byteLength === 0) { + throw new Error("Downloaded image is empty"); + } + + return { + buffer: Buffer.from(arrayBuffer), + contentType, + extension: getImageExtension(contentType, params.sourceUrl), + }; + } catch (error) { + if (attempt >= retries) throw error; + await new Promise((resolve) => setTimeout(resolve, 1000 * (attempt + 1))); + } + } + + throw new Error("Image download failed"); +} + +export async function archiveTemporaryImage(sourceUrl: string) { + const downloaded = await downloadTemporaryImage({ sourceUrl }); + const filename = `ai-generated-archive-${Date.now()}.${downloaded.extension}`; + const uploaded = await uploadFile(filename, toArrayBuffer(downloaded.buffer)); + return { + url: uploaded.url, + pathname: uploaded.pathname, + contentType: uploaded.contentType, + bytes: downloaded.buffer.length, + }; +} diff --git a/lib/storage/image-url-replacement.ts b/lib/storage/image-url-replacement.ts new file mode 100644 index 000000000..aad3500b4 --- /dev/null +++ b/lib/storage/image-url-replacement.ts @@ -0,0 +1,476 @@ +import "server-only"; + +import { eq, like, or, sql } from "drizzle-orm"; +import { db } from "@/lib/db"; +import { + assetMemorySnapshot, + canvasDetailArchive, + canvasNode, + canvasProject, + conversationWorkingState, + externalImageApiRequestLog, + generatedImage, + mediaAsset, + message, + messageDeprecated, + model3dAsset, +} from "@/lib/db/schema"; +import { inferOriginalImageUrl } from "@/lib/image-url"; + +function normalizeMediaUrl(value: string): string { + const raw = String(value || "").trim(); + if (!raw) return ""; + const canonical = inferOriginalImageUrl(raw); + + try { + const parsed = new URL(canonical); + return `${parsed.origin}${parsed.pathname}`.replace(/\/+$/, "").toLowerCase(); + } catch { + return canonical + .split("#")[0] + .split("?")[0] + .replace(/\/+$/, "") + .toLowerCase(); + } +} + +export function replaceDeepUrls( + value: unknown, + sourceUrl: string, + targetUrl: string +): { value: unknown; changed: boolean } { + if (typeof value === "string") { + if (!value.includes(sourceUrl)) { + return { value, changed: false }; + } + return { value: value.split(sourceUrl).join(targetUrl), changed: true }; + } + + if (Array.isArray(value)) { + let changed = false; + const next = value.map((item) => { + const result = replaceDeepUrls(item, sourceUrl, targetUrl); + if (result.changed) changed = true; + return result.value; + }); + return { value: changed ? next : value, changed }; + } + + if (value && typeof value === "object") { + let changed = false; + const next: Record = {}; + for (const [key, item] of Object.entries(value as Record)) { + const result = replaceDeepUrls(item, sourceUrl, targetUrl); + next[key] = result.value; + if (result.changed) changed = true; + } + return { value: changed ? next : value, changed }; + } + + return { value, changed: false }; +} + +export async function replaceImageUrlInDatabase( + sourceUrl: string, + targetUrl: string +) { + if (!sourceUrl || !targetUrl || sourceUrl === targetUrl) { + return { changed: 0 }; + } + + let changed = 0; + const sourceNormalizedUrl = normalizeMediaUrl(sourceUrl); + const targetNormalizedUrl = normalizeMediaUrl(targetUrl); + + const generatedThumb = await db + .update(generatedImage) + .set({ thumbnailUrl: targetUrl }) + .where(eq(generatedImage.thumbnailUrl, sourceUrl)) + .returning({ id: generatedImage.id }); + changed += generatedThumb.length; + + const generatedOrig = await db + .update(generatedImage) + .set({ originalUrl: targetUrl }) + .where(eq(generatedImage.originalUrl, sourceUrl)) + .returning({ id: generatedImage.id }); + changed += generatedOrig.length; + + const assets = await db + .select() + .from(mediaAsset) + .where( + or( + eq(mediaAsset.originalUrl, sourceUrl), + eq(mediaAsset.previewUrl, sourceUrl), + eq(mediaAsset.thumbnailUrl, sourceUrl), + eq(mediaAsset.normalizedUrl, sourceNormalizedUrl), + sql`${mediaAsset.metadata}::text like ${`%${sourceUrl}%`}`, + sql`${mediaAsset.summary}::text like ${`%${sourceUrl}%`}` + ) + ) + .limit(500); + + for (const item of assets) { + const original = replaceDeepUrls(item.originalUrl, sourceUrl, targetUrl); + const preview = replaceDeepUrls(item.previewUrl, sourceUrl, targetUrl); + const thumbnail = replaceDeepUrls(item.thumbnailUrl, sourceUrl, targetUrl); + const metadata = replaceDeepUrls(item.metadata, sourceUrl, targetUrl); + const summary = replaceDeepUrls(item.summary, sourceUrl, targetUrl); + const shouldNormalize = + item.normalizedUrl === sourceNormalizedUrl || original.changed; + const nextOriginal = (original.value as string) || item.originalUrl; + + if ( + !original.changed && + !preview.changed && + !thumbnail.changed && + !metadata.changed && + !summary.changed && + !shouldNormalize + ) { + continue; + } + + try { + await db + .update(mediaAsset) + .set({ + originalUrl: nextOriginal, + normalizedUrl: shouldNormalize ? targetNormalizedUrl : item.normalizedUrl, + previewUrl: (preview.value as string | null | undefined) ?? item.previewUrl, + thumbnailUrl: + (thumbnail.value as string | null | undefined) ?? item.thumbnailUrl, + metadata: (metadata.value ?? item.metadata) as any, + summary: (summary.value ?? item.summary) as any, + updatedAt: new Date(), + }) + .where(eq(mediaAsset.id, item.id)); + } catch (error) { + await db + .update(mediaAsset) + .set({ + originalUrl: nextOriginal, + previewUrl: (preview.value as string | null | undefined) ?? item.previewUrl, + thumbnailUrl: + (thumbnail.value as string | null | undefined) ?? item.thumbnailUrl, + metadata: (metadata.value ?? item.metadata) as any, + summary: (summary.value ?? item.summary) as any, + updatedAt: new Date(), + }) + .where(eq(mediaAsset.id, item.id)); + } + changed += 1; + } + + const snapshots = await db + .select() + .from(assetMemorySnapshot) + .where( + or( + eq(assetMemorySnapshot.normalizedUrl, sourceNormalizedUrl), + sql`${assetMemorySnapshot.metadata}::text like ${`%${sourceUrl}%`}`, + like(assetMemorySnapshot.summaryText, `%${sourceUrl}%`) + ) + ) + .limit(500); + + for (const item of snapshots) { + const metadata = replaceDeepUrls(item.metadata, sourceUrl, targetUrl); + const summary = replaceDeepUrls(item.summaryText, sourceUrl, targetUrl); + const shouldNormalize = item.normalizedUrl === sourceNormalizedUrl; + if (!metadata.changed && !summary.changed && !shouldNormalize) continue; + + await db + .update(assetMemorySnapshot) + .set({ + normalizedUrl: shouldNormalize ? targetNormalizedUrl : item.normalizedUrl, + metadata: (metadata.value ?? item.metadata) as any, + summaryText: (summary.value as string | null | undefined) ?? item.summaryText, + updatedAt: new Date(), + }) + .where(eq(assetMemorySnapshot.id, item.id)); + changed += 1; + } + + const messages = await db + .select({ + id: message.id, + parts: message.parts, + attachments: message.attachments, + }) + .from(message) + .where( + or( + sql`${message.parts}::text like ${`%${sourceUrl}%`}`, + sql`${message.attachments}::text like ${`%${sourceUrl}%`}` + ) + ) + .limit(500); + + for (const item of messages) { + const partsResult = replaceDeepUrls(item.parts, sourceUrl, targetUrl); + const attachmentsResult = replaceDeepUrls(item.attachments, sourceUrl, targetUrl); + if (!partsResult.changed && !attachmentsResult.changed) continue; + + await db + .update(message) + .set({ + parts: (partsResult.value ?? item.parts) as any, + attachments: (attachmentsResult.value ?? item.attachments) as any, + }) + .where(eq(message.id, item.id)); + changed += 1; + } + + const legacyMessages = await db + .select({ + id: messageDeprecated.id, + content: messageDeprecated.content, + }) + .from(messageDeprecated) + .where(sql`${messageDeprecated.content}::text like ${`%${sourceUrl}%`}`) + .limit(500); + + for (const item of legacyMessages) { + const replaced = replaceDeepUrls(item.content, sourceUrl, targetUrl); + if (!replaced.changed) continue; + + await db + .update(messageDeprecated) + .set({ content: replaced.value as any }) + .where(eq(messageDeprecated.id, item.id)); + changed += 1; + } + + const projects = await db + .select({ + id: canvasProject.id, + canvasData: canvasProject.canvasData, + }) + .from(canvasProject) + .where(sql`${canvasProject.canvasData}::text like ${`%${sourceUrl}%`}`) + .limit(500); + + for (const item of projects) { + const replaced = replaceDeepUrls(item.canvasData, sourceUrl, targetUrl); + if (!replaced.changed) continue; + + await db + .update(canvasProject) + .set({ + canvasData: replaced.value as any, + updatedAt: new Date(), + }) + .where(eq(canvasProject.id, item.id)); + changed += 1; + } + + const nodes = await db + .select({ + id: canvasNode.id, + data: canvasNode.data, + }) + .from(canvasNode) + .where(sql`${canvasNode.data}::text like ${`%${sourceUrl}%`}`) + .limit(500); + + for (const item of nodes) { + const replaced = replaceDeepUrls(item.data, sourceUrl, targetUrl); + if (!replaced.changed) continue; + + await db + .update(canvasNode) + .set({ + data: replaced.value as any, + updatedAt: new Date(), + }) + .where(eq(canvasNode.id, item.id)); + changed += 1; + } + + const workingStates = await db + .select() + .from(conversationWorkingState) + .where( + or( + sql`${conversationWorkingState.currentFocusImageUrls}::text like ${`%${sourceUrl}%`}`, + sql`${conversationWorkingState.selectedImageUrls}::text like ${`%${sourceUrl}%`}`, + sql`${conversationWorkingState.lastUploadedImageUrls}::text like ${`%${sourceUrl}%`}`, + sql`${conversationWorkingState.lastGeneratedImageUrls}::text like ${`%${sourceUrl}%`}`, + sql`${conversationWorkingState.lastReferencedImageUrls}::text like ${`%${sourceUrl}%`}`, + sql`${conversationWorkingState.lastResolvedTargetImageUrls}::text like ${`%${sourceUrl}%`}`, + sql`${conversationWorkingState.lastCanvasSnapshot}::text like ${`%${sourceUrl}%`}`, + sql`${conversationWorkingState.lastTurnSummary}::text like ${`%${sourceUrl}%`}` + ) + ) + .limit(500); + + for (const item of workingStates) { + const currentFocus = replaceDeepUrls( + item.currentFocusImageUrls, + sourceUrl, + targetUrl + ); + const selected = replaceDeepUrls(item.selectedImageUrls, sourceUrl, targetUrl); + const uploaded = replaceDeepUrls(item.lastUploadedImageUrls, sourceUrl, targetUrl); + const generated = replaceDeepUrls(item.lastGeneratedImageUrls, sourceUrl, targetUrl); + const referenced = replaceDeepUrls(item.lastReferencedImageUrls, sourceUrl, targetUrl); + const resolved = replaceDeepUrls( + item.lastResolvedTargetImageUrls, + sourceUrl, + targetUrl + ); + const canvasSnapshot = replaceDeepUrls( + item.lastCanvasSnapshot, + sourceUrl, + targetUrl + ); + const turnSummary = replaceDeepUrls(item.lastTurnSummary, sourceUrl, targetUrl); + + if ( + !currentFocus.changed && + !selected.changed && + !uploaded.changed && + !generated.changed && + !referenced.changed && + !resolved.changed && + !canvasSnapshot.changed && + !turnSummary.changed + ) { + continue; + } + + await db + .update(conversationWorkingState) + .set({ + currentFocusImageUrls: currentFocus.value as any, + selectedImageUrls: selected.value as any, + lastUploadedImageUrls: uploaded.value as any, + lastGeneratedImageUrls: generated.value as any, + lastReferencedImageUrls: referenced.value as any, + lastResolvedTargetImageUrls: resolved.value as any, + lastCanvasSnapshot: (canvasSnapshot.value ?? item.lastCanvasSnapshot) as any, + lastTurnSummary: (turnSummary.value ?? item.lastTurnSummary) as any, + updatedAt: new Date(), + }) + .where(eq(conversationWorkingState.id, item.id)); + changed += 1; + } + + const detailArchives = await db + .select({ + id: canvasDetailArchive.id, + coverUrl: canvasDetailArchive.coverUrl, + brief: canvasDetailArchive.brief, + slots: canvasDetailArchive.slots, + uiSurfaceSnapshot: canvasDetailArchive.uiSurfaceSnapshot, + uiSurfaceLatestReview: canvasDetailArchive.uiSurfaceLatestReview, + }) + .from(canvasDetailArchive) + .where( + or( + like(canvasDetailArchive.coverUrl, `%${sourceUrl}%`), + sql`${canvasDetailArchive.brief}::text like ${`%${sourceUrl}%`}`, + sql`${canvasDetailArchive.slots}::text like ${`%${sourceUrl}%`}`, + sql`${canvasDetailArchive.uiSurfaceSnapshot}::text like ${`%${sourceUrl}%`}`, + sql`${canvasDetailArchive.uiSurfaceLatestReview}::text like ${`%${sourceUrl}%`}` + ) + ) + .limit(500); + + for (const item of detailArchives) { + const cover = replaceDeepUrls(item.coverUrl, sourceUrl, targetUrl); + const brief = replaceDeepUrls(item.brief, sourceUrl, targetUrl); + const slots = replaceDeepUrls(item.slots, sourceUrl, targetUrl); + const snapshot = replaceDeepUrls(item.uiSurfaceSnapshot, sourceUrl, targetUrl); + const review = replaceDeepUrls(item.uiSurfaceLatestReview, sourceUrl, targetUrl); + + if ( + !cover.changed && + !brief.changed && + !slots.changed && + !snapshot.changed && + !review.changed + ) { + continue; + } + + await db + .update(canvasDetailArchive) + .set({ + coverUrl: (cover.value as string | null | undefined) ?? item.coverUrl, + brief: (brief.value ?? item.brief) as any, + slots: (slots.value ?? item.slots) as any, + uiSurfaceSnapshot: (snapshot.value ?? item.uiSurfaceSnapshot) as any, + uiSurfaceLatestReview: (review.value ?? item.uiSurfaceLatestReview) as any, + updatedAt: new Date(), + }) + .where(eq(canvasDetailArchive.id, item.id)); + changed += 1; + } + + const modelAssets = await db + .select() + .from(model3dAsset) + .where( + or( + like(model3dAsset.previewImageUrl, `%${sourceUrl}%`), + sql`${model3dAsset.assetPayload}::text like ${`%${sourceUrl}%`}` + ) + ) + .limit(500); + + for (const item of modelAssets) { + const preview = replaceDeepUrls(item.previewImageUrl, sourceUrl, targetUrl); + const payload = replaceDeepUrls(item.assetPayload, sourceUrl, targetUrl); + if (!preview.changed && !payload.changed) continue; + + await db + .update(model3dAsset) + .set({ + previewImageUrl: + (preview.value as string | null | undefined) ?? item.previewImageUrl, + assetPayload: (payload.value ?? item.assetPayload) as any, + updatedAt: new Date(), + }) + .where(eq(model3dAsset.id, item.id)); + changed += 1; + } + + const apiLogs = await db + .select() + .from(externalImageApiRequestLog) + .where( + or( + sql`${externalImageApiRequestLog.responsePayload}::text like ${`%${sourceUrl}%`}`, + sql`${externalImageApiRequestLog.generatedImageUrls}::text like ${`%${sourceUrl}%`}` + ) + ) + .limit(500); + + for (const item of apiLogs) { + const responsePayload = replaceDeepUrls( + item.responsePayload, + sourceUrl, + targetUrl + ); + const generatedUrls = replaceDeepUrls( + item.generatedImageUrls, + sourceUrl, + targetUrl + ); + if (!responsePayload.changed && !generatedUrls.changed) continue; + + await db + .update(externalImageApiRequestLog) + .set({ + responsePayload: (responsePayload.value ?? item.responsePayload) as any, + generatedImageUrls: (generatedUrls.value ?? item.generatedImageUrls) as any, + }) + .where(eq(externalImageApiRequestLog.id, item.id)); + changed += 1; + } + + return { changed }; +} diff --git a/lib/storage/service.ts b/lib/storage/service.ts index 9780be0ee..f678d51c7 100644 --- a/lib/storage/service.ts +++ b/lib/storage/service.ts @@ -54,13 +54,14 @@ function getStorageTimeoutMs(name: string, fallback: number, min: number, max: n } async function withStorageOperationTimeout( - operation: Promise, + operation: (abortSignal: AbortSignal) => Promise, timeoutMs: number, label: string ): Promise { let timedOut = false; let timeoutId: ReturnType | undefined; - const guardedOperation = operation.catch((error) => { + const abortController = new AbortController(); + const guardedOperation = operation(abortController.signal).catch((error) => { if (timedOut) { logger.warn(`[Upload] ${label} failed after timeout:`, error); return new Promise(() => undefined); @@ -70,6 +71,7 @@ async function withStorageOperationTimeout( const timeout = new Promise((_, reject) => { timeoutId = setTimeout(() => { timedOut = true; + abortController.abort(); reject(new Error(`${label} timed out after ${timeoutMs}ms`)); }, timeoutMs); }); @@ -256,7 +258,7 @@ async function uploadToS3( }); await withStorageOperationTimeout( - s3Client.send(command), + (abortSignal) => s3Client.send(command, { abortSignal }), getStorageTimeoutMs( "S3_UPLOAD_REQUEST_TIMEOUT_MS", DEFAULT_S3_REQUEST_TIMEOUT_MS, diff --git a/vercel.json b/vercel.json index e9b5664f0..713b97d05 100644 --- a/vercel.json +++ b/vercel.json @@ -3,7 +3,10 @@ { "path": "/api/jobs/archive-videos", "schedule": "0 * * * *" + }, + { + "path": "/api/jobs/archive-images", + "schedule": "5 * * * *" } ] } -