feat: defer generated image archiving
This commit is contained in:
parent
6c83826e1d
commit
748273d917
65
app/api/jobs/archive-images/route.ts
Normal file
65
app/api/jobs/archive-images/route.ts
Normal file
@ -0,0 +1,65 @@
|
|||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import { runImageArchiveJob } from "@/lib/jobs/image-archive-job";
|
||||||
|
import { getImageArchiveTaskStats } from "@/lib/storage/image-archive-db";
|
||||||
|
|
||||||
|
function verifyAuthToken(request: Request): boolean {
|
||||||
|
const authHeader = request.headers.get("authorization");
|
||||||
|
const token = authHeader?.replace("Bearer ", "");
|
||||||
|
const cronSecret = process.env.CRON_SECRET;
|
||||||
|
|
||||||
|
if (!cronSecret) {
|
||||||
|
console.error("[ImageArchiveAPI] CRON_SECRET is not configured");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return token === cronSecret;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function POST(request: Request) {
|
||||||
|
try {
|
||||||
|
if (!verifyAuthToken(request)) {
|
||||||
|
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await runImageArchiveJob();
|
||||||
|
return NextResponse.json({
|
||||||
|
...result,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.error("[ImageArchiveAPI] Job execution failed:", error);
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
success: false,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
{ status: 500 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function GET(request: Request) {
|
||||||
|
try {
|
||||||
|
if (!verifyAuthToken(request)) {
|
||||||
|
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const stats = await getImageArchiveTaskStats();
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
stats,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.error("[ImageArchiveAPI] Stats query failed:", error);
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
success: false,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
{ status: 500 }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -10,6 +10,7 @@
|
|||||||
|
|
||||||
import sharp from "sharp";
|
import sharp from "sharp";
|
||||||
import { constrainImageDimensions } from "@/lib/ai/image-size-constraints";
|
import { constrainImageDimensions } from "@/lib/ai/image-size-constraints";
|
||||||
|
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||||||
import { uploadFile } from "@/lib/storage/service";
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
|
|
||||||
interface AlibabaImageConfig {
|
interface AlibabaImageConfig {
|
||||||
@ -358,6 +359,11 @@ async function downloadAndUploadToS3(
|
|||||||
prefix: string = "alibaba-image"
|
prefix: string = "alibaba-image"
|
||||||
): Promise<{ thumbnail: string; original: string }> {
|
): Promise<{ thumbnail: string; original: string }> {
|
||||||
try {
|
try {
|
||||||
|
const deferred = await createDeferredUploadedImage(imageUrl, {
|
||||||
|
providerType: "alibaba-image",
|
||||||
|
});
|
||||||
|
if (deferred) return deferred;
|
||||||
|
|
||||||
console.log("[Alibaba Image] Downloading image from:", imageUrl.substring(0, 100) + "...");
|
console.log("[Alibaba Image] Downloading image from:", imageUrl.substring(0, 100) + "...");
|
||||||
|
|
||||||
const { buffer } = await fetchImageBufferWithRetry(imageUrl, 30000, 3);
|
const { buffer } = await fetchImageBufferWithRetry(imageUrl, 30000, 3);
|
||||||
|
|||||||
@ -2,6 +2,7 @@ import "server-only";
|
|||||||
|
|
||||||
import sharp from "sharp";
|
import sharp from "sharp";
|
||||||
import { resolveLongestEdgeDimensions } from "@/lib/ai/image-size-constraints";
|
import { resolveLongestEdgeDimensions } from "@/lib/ai/image-size-constraints";
|
||||||
|
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||||||
import { uploadFile } from "@/lib/storage/service";
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
|
|
||||||
type BflImageConfig = {
|
type BflImageConfig = {
|
||||||
@ -273,6 +274,11 @@ async function uploadGeneratedBuffer(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
||||||
|
const deferred = await createDeferredUploadedImage(imageUrl, {
|
||||||
|
providerType: "bfl-image",
|
||||||
|
});
|
||||||
|
if (deferred) return deferred;
|
||||||
|
|
||||||
const response = await fetch(imageUrl, { cache: "no-store" });
|
const response = await fetch(imageUrl, { cache: "no-store" });
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new Error(`Failed to download generated image: ${response.status}`);
|
throw new Error(`Failed to download generated image: ${response.status}`);
|
||||||
|
|||||||
@ -6,6 +6,7 @@ import {
|
|||||||
constrainImageDimensions,
|
constrainImageDimensions,
|
||||||
resolveLongestEdgeDimensions,
|
resolveLongestEdgeDimensions,
|
||||||
} from "@/lib/ai/image-size-constraints";
|
} from "@/lib/ai/image-size-constraints";
|
||||||
|
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||||||
import { uploadFile } from "@/lib/storage/service";
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
|
|
||||||
interface FalImageConfig {
|
interface FalImageConfig {
|
||||||
@ -466,6 +467,11 @@ function generateFilename(prefix: string, extension: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
||||||
|
const deferred = await createDeferredUploadedImage(imageUrl, {
|
||||||
|
providerType: "fal-image",
|
||||||
|
});
|
||||||
|
if (deferred) return deferred;
|
||||||
|
|
||||||
const response = await fetch(imageUrl);
|
const response = await fetch(imageUrl);
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new Error(`Failed to download image: ${response.status}`);
|
throw new Error(`Failed to download image: ${response.status}`);
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
import "server-only";
|
import "server-only";
|
||||||
|
|
||||||
import { resolveGrsaiGptImageSize } from "@/lib/ai/grsai-gpt-image-sizes";
|
import { resolveGrsaiGptImageSize } from "@/lib/ai/grsai-gpt-image-sizes";
|
||||||
|
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||||||
import { uploadFile } from "@/lib/storage/service";
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
import { createLogger } from "@/lib/observability/logger";
|
import { createLogger } from "@/lib/observability/logger";
|
||||||
|
|
||||||
@ -170,6 +171,11 @@ async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/jp
|
|||||||
* 带重试机制,确保成功下载并上传到国内中转
|
* 带重试机制,确保成功下载并上传到国内中转
|
||||||
*/
|
*/
|
||||||
async function downloadAndUploadToS3(imageUrl: string, retries = 3): Promise<{ thumbnail: string; original: string }> {
|
async function downloadAndUploadToS3(imageUrl: string, retries = 3): Promise<{ thumbnail: string; original: string }> {
|
||||||
|
const deferred = await createDeferredUploadedImage(imageUrl, {
|
||||||
|
providerType: "gemini-image",
|
||||||
|
});
|
||||||
|
if (deferred) return deferred;
|
||||||
|
|
||||||
for (let attempt = 0; attempt < retries; attempt++) {
|
for (let attempt = 0; attempt < retries; attempt++) {
|
||||||
try {
|
try {
|
||||||
logger.debug(`[Gemini Image] Downloading image from URL (attempt ${attempt + 1}/${retries}): ${imageUrl.substring(0, 100)}...`);
|
logger.debug(`[Gemini Image] Downloading image from URL (attempt ${attempt + 1}/${retries}): ${imageUrl.substring(0, 100)}...`);
|
||||||
|
|||||||
@ -1,5 +1,6 @@
|
|||||||
import "server-only";
|
import "server-only";
|
||||||
|
|
||||||
|
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||||||
import { uploadFile } from "@/lib/storage/service";
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
|
|
||||||
interface HuankeGeminiConfig {
|
interface HuankeGeminiConfig {
|
||||||
@ -112,6 +113,11 @@ async function uploadBase64ToS3(base64Data: string, mimeType: string = "image/jp
|
|||||||
* 下载远程URL图片并上传到 S3
|
* 下载远程URL图片并上传到 S3
|
||||||
*/
|
*/
|
||||||
async function downloadAndUploadToS3(imageUrl: string, retries = 3): Promise<{ thumbnail: string; original: string }> {
|
async function downloadAndUploadToS3(imageUrl: string, retries = 3): Promise<{ thumbnail: string; original: string }> {
|
||||||
|
const deferred = await createDeferredUploadedImage(imageUrl, {
|
||||||
|
providerType: "huanke-gemini",
|
||||||
|
});
|
||||||
|
if (deferred) return deferred;
|
||||||
|
|
||||||
for (let attempt = 0; attempt < retries; attempt++) {
|
for (let attempt = 0; attempt < retries; attempt++) {
|
||||||
try {
|
try {
|
||||||
console.log(`[Huanke Gemini] Downloading image from URL (attempt ${attempt + 1}/${retries}): ${imageUrl.substring(0, 100)}...`);
|
console.log(`[Huanke Gemini] Downloading image from URL (attempt ${attempt + 1}/${retries}): ${imageUrl.substring(0, 100)}...`);
|
||||||
|
|||||||
@ -7,6 +7,7 @@ import {
|
|||||||
inferKreaModelKind,
|
inferKreaModelKind,
|
||||||
type KreaModelKind,
|
type KreaModelKind,
|
||||||
} from "@/lib/ai/krea-models";
|
} from "@/lib/ai/krea-models";
|
||||||
|
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||||||
import { uploadFile } from "@/lib/storage/service";
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
|
|
||||||
interface KreaConfig {
|
interface KreaConfig {
|
||||||
@ -651,6 +652,14 @@ async function uploadKreaImages(payload: unknown) {
|
|||||||
const uploadedImages: UploadedImage[] = [];
|
const uploadedImages: UploadedImage[] = [];
|
||||||
for (const url of collectHttpUrls(payload)) {
|
for (const url of collectHttpUrls(payload)) {
|
||||||
try {
|
try {
|
||||||
|
const deferred = await createDeferredUploadedImage(url, {
|
||||||
|
providerType: "krea",
|
||||||
|
});
|
||||||
|
if (deferred) {
|
||||||
|
uploadedImages.push(deferred);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
const { buffer, contentType } = await downloadRemoteAsset(url);
|
const { buffer, contentType } = await downloadRemoteAsset(url);
|
||||||
if (!String(contentType || "").startsWith("image/")) continue;
|
if (!String(contentType || "").startsWith("image/")) continue;
|
||||||
uploadedImages.push(await uploadGeneratedBuffer(buffer, detectExtensionFromContentType(contentType)));
|
uploadedImages.push(await uploadGeneratedBuffer(buffer, detectExtensionFromContentType(contentType)));
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
import "server-only";
|
import "server-only";
|
||||||
|
|
||||||
import sharp from "sharp";
|
import sharp from "sharp";
|
||||||
|
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||||||
import { uploadFile } from "@/lib/storage/service";
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
|
|
||||||
interface MidjourneyConfig {
|
interface MidjourneyConfig {
|
||||||
@ -161,6 +162,11 @@ async function uploadGeneratedBuffer(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
||||||
|
const deferred = await createDeferredUploadedImage(imageUrl, {
|
||||||
|
providerType: "midjourney",
|
||||||
|
});
|
||||||
|
if (deferred) return deferred;
|
||||||
|
|
||||||
const response = await fetch(imageUrl, { cache: "no-store" });
|
const response = await fetch(imageUrl, { cache: "no-store" });
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new Error(`Failed to download Midjourney image: ${response.status}`);
|
throw new Error(`Failed to download Midjourney image: ${response.status}`);
|
||||||
|
|||||||
@ -4,6 +4,7 @@ import sharp from "sharp";
|
|||||||
import {
|
import {
|
||||||
resolveLongestEdgeDimensions,
|
resolveLongestEdgeDimensions,
|
||||||
} from "@/lib/ai/image-size-constraints";
|
} from "@/lib/ai/image-size-constraints";
|
||||||
|
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||||||
import { uploadFile } from "@/lib/storage/service";
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
|
|
||||||
interface OpenAIImageConfig {
|
interface OpenAIImageConfig {
|
||||||
@ -413,6 +414,11 @@ async function uploadGeneratedBuffer(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
||||||
|
const deferred = await createDeferredUploadedImage(imageUrl, {
|
||||||
|
providerType: "openai-image",
|
||||||
|
});
|
||||||
|
if (deferred) return deferred;
|
||||||
|
|
||||||
const response = await fetch(imageUrl, { cache: "no-store" });
|
const response = await fetch(imageUrl, { cache: "no-store" });
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new Error(`Failed to download generated image: ${response.status}`);
|
throw new Error(`Failed to download generated image: ${response.status}`);
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import {
|
|||||||
normalizeRunwayDuration,
|
normalizeRunwayDuration,
|
||||||
resolveRunwayApiRatio,
|
resolveRunwayApiRatio,
|
||||||
} from "@/lib/ai/runway-video-models";
|
} from "@/lib/ai/runway-video-models";
|
||||||
|
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||||||
import { uploadFile } from "@/lib/storage/service";
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
import { inferRunwayModelKind } from "@/lib/ai/model-classification";
|
import { inferRunwayModelKind } from "@/lib/ai/model-classification";
|
||||||
|
|
||||||
@ -432,6 +433,14 @@ async function uploadRunwayImages(taskOutput: unknown) {
|
|||||||
|
|
||||||
for (const url of outputUrls) {
|
for (const url of outputUrls) {
|
||||||
try {
|
try {
|
||||||
|
const deferred = await createDeferredUploadedImage(url, {
|
||||||
|
providerType: "runway",
|
||||||
|
});
|
||||||
|
if (deferred) {
|
||||||
|
uploadedImages.push(deferred);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
const { buffer, contentType } = await downloadRemoteAsset(url);
|
const { buffer, contentType } = await downloadRemoteAsset(url);
|
||||||
if (!String(contentType || "").startsWith("image/")) {
|
if (!String(contentType || "").startsWith("image/")) {
|
||||||
continue;
|
continue;
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
import "server-only";
|
import "server-only";
|
||||||
|
|
||||||
import sharp from "sharp";
|
import sharp from "sharp";
|
||||||
|
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||||||
import { uploadFile } from "@/lib/storage/service";
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
import {
|
import {
|
||||||
resolveRuxaImageRuntimeModelId,
|
resolveRuxaImageRuntimeModelId,
|
||||||
@ -369,6 +370,11 @@ function generateFilename(prefix: string, extension: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
||||||
|
const deferred = await createDeferredUploadedImage(imageUrl, {
|
||||||
|
providerType: "ruxa-image",
|
||||||
|
});
|
||||||
|
if (deferred) return deferred;
|
||||||
|
|
||||||
const response = await fetch(imageUrl);
|
const response = await fetch(imageUrl);
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new Error(`Failed to download image: ${response.status}`);
|
throw new Error(`Failed to download image: ${response.status}`);
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
import "server-only";
|
import "server-only";
|
||||||
|
|
||||||
import sharp from "sharp";
|
import sharp from "sharp";
|
||||||
|
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||||||
import { uploadFile } from "@/lib/storage/service";
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
import { buildSnapgenReferenceOrderPrompt } from "@/lib/ai/providers/snapgen-reference-order";
|
import { buildSnapgenReferenceOrderPrompt } from "@/lib/ai/providers/snapgen-reference-order";
|
||||||
import {
|
import {
|
||||||
@ -690,6 +691,11 @@ async function downloadAndUploadImage(
|
|||||||
imageUrl: string,
|
imageUrl: string,
|
||||||
timeoutMs = IMAGE_DOWNLOAD_TIMEOUT_MS
|
timeoutMs = IMAGE_DOWNLOAD_TIMEOUT_MS
|
||||||
): Promise<UploadedImage> {
|
): Promise<UploadedImage> {
|
||||||
|
const deferred = await createDeferredUploadedImage(imageUrl, {
|
||||||
|
providerType: "snapgen-image",
|
||||||
|
});
|
||||||
|
if (deferred) return deferred;
|
||||||
|
|
||||||
const response = await fetchWithTimeout(imageUrl, {}, timeoutMs);
|
const response = await fetchWithTimeout(imageUrl, {}, timeoutMs);
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new Error(`Failed to download SnapGen image: ${response.status}`);
|
throw new Error(`Failed to download SnapGen image: ${response.status}`);
|
||||||
|
|||||||
@ -2,6 +2,7 @@ import "server-only";
|
|||||||
|
|
||||||
import sharp from "sharp";
|
import sharp from "sharp";
|
||||||
import { constrainImageDimensions } from "@/lib/ai/image-size-constraints";
|
import { constrainImageDimensions } from "@/lib/ai/image-size-constraints";
|
||||||
|
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||||||
import { uploadFile } from "@/lib/storage/service";
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
|
|
||||||
interface StepfunImageConfig {
|
interface StepfunImageConfig {
|
||||||
@ -160,6 +161,11 @@ async function uploadGeneratedBuffer(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
async function downloadAndUploadImage(imageUrl: string): Promise<UploadedImage> {
|
||||||
|
const deferred = await createDeferredUploadedImage(imageUrl, {
|
||||||
|
providerType: "stepfun-image",
|
||||||
|
});
|
||||||
|
if (deferred) return deferred;
|
||||||
|
|
||||||
const response = await fetch(imageUrl, { cache: "no-store" });
|
const response = await fetch(imageUrl, { cache: "no-store" });
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
throw new Error(`Failed to download StepFun generated image: ${response.status}`);
|
throw new Error(`Failed to download StepFun generated image: ${response.status}`);
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
import "server-only";
|
import "server-only";
|
||||||
|
|
||||||
import { constrainImageDimensions } from "@/lib/ai/image-size-constraints";
|
import { constrainImageDimensions } from "@/lib/ai/image-size-constraints";
|
||||||
|
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||||||
import { uploadFile } from "@/lib/storage/service";
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
import { getDefaultStorageConfig } from "@/lib/db/queries";
|
import { getDefaultStorageConfig } from "@/lib/db/queries";
|
||||||
import { createLogger } from "@/lib/observability/logger";
|
import { createLogger } from "@/lib/observability/logger";
|
||||||
@ -429,6 +430,13 @@ async function createThumbnail(buffer: Buffer, mimeType: string): Promise<Buffer
|
|||||||
*/
|
*/
|
||||||
async function uploadImageToS3(imageData: string): Promise<{ thumbnail: string; original: string }> {
|
async function uploadImageToS3(imageData: string): Promise<{ thumbnail: string; original: string }> {
|
||||||
try {
|
try {
|
||||||
|
if (/^https?:\/\//i.test(imageData)) {
|
||||||
|
const deferred = await createDeferredUploadedImage(imageData, {
|
||||||
|
providerType: "volcano-vision",
|
||||||
|
});
|
||||||
|
if (deferred) return deferred;
|
||||||
|
}
|
||||||
|
|
||||||
let buffer: Buffer;
|
let buffer: Buffer;
|
||||||
let mimeType = "image/jpeg";
|
let mimeType = "image/jpeg";
|
||||||
|
|
||||||
|
|||||||
22
lib/db/migrations/0087_add_image_archive_tasks.sql
Normal file
22
lib/db/migrations/0087_add_image_archive_tasks.sql
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
CREATE TABLE IF NOT EXISTS "ImageArchiveTask" (
|
||||||
|
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
|
||||||
|
"sourceUrl" text NOT NULL UNIQUE,
|
||||||
|
"targetUrl" text,
|
||||||
|
"targetStorageConfigId" uuid REFERENCES "StorageConfig"("id"),
|
||||||
|
"status" varchar(32) NOT NULL DEFAULT 'pending',
|
||||||
|
"attemptCount" integer NOT NULL DEFAULT 0,
|
||||||
|
"lastError" text,
|
||||||
|
"archiveAfterAt" timestamp NOT NULL,
|
||||||
|
"replaceAfterAt" timestamp NOT NULL,
|
||||||
|
"archivedAt" timestamp,
|
||||||
|
"replacedAt" timestamp,
|
||||||
|
"metadata" jsonb,
|
||||||
|
"createdAt" timestamp NOT NULL DEFAULT now(),
|
||||||
|
"updatedAt" timestamp NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "ImageArchiveTask_status_archive_idx"
|
||||||
|
ON "ImageArchiveTask"("status", "archiveAfterAt", "createdAt");
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS "ImageArchiveTask_status_replace_idx"
|
||||||
|
ON "ImageArchiveTask"("status", "replaceAfterAt", "createdAt");
|
||||||
@ -512,6 +512,13 @@
|
|||||||
"when": 1781174400000,
|
"when": 1781174400000,
|
||||||
"tag": "0086_external_image_log_model_delete_set_null",
|
"tag": "0086_external_image_log_model_delete_set_null",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"idx": 73,
|
||||||
|
"version": "7",
|
||||||
|
"when": 1783680000000,
|
||||||
|
"tag": "0087_add_image_archive_tasks",
|
||||||
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@ -441,6 +441,26 @@ export const storageMigrationTask = pgTable("StorageMigrationTask", {
|
|||||||
|
|
||||||
export type StorageMigrationTask = InferSelectModel<typeof storageMigrationTask>;
|
export type StorageMigrationTask = InferSelectModel<typeof storageMigrationTask>;
|
||||||
|
|
||||||
|
// AI 生成图片临时地址归档任务表
|
||||||
|
export const imageArchiveTask = pgTable("ImageArchiveTask", {
|
||||||
|
id: uuid("id").primaryKey().notNull().defaultRandom(),
|
||||||
|
sourceUrl: text("sourceUrl").notNull().unique(),
|
||||||
|
targetUrl: text("targetUrl"),
|
||||||
|
targetStorageConfigId: uuid("targetStorageConfigId").references(() => storageConfig.id),
|
||||||
|
status: varchar("status", { length: 32 }).notNull().default("pending"),
|
||||||
|
attemptCount: integer("attemptCount").notNull().default(0),
|
||||||
|
lastError: text("lastError"),
|
||||||
|
archiveAfterAt: timestamp("archiveAfterAt").notNull(),
|
||||||
|
replaceAfterAt: timestamp("replaceAfterAt").notNull(),
|
||||||
|
archivedAt: timestamp("archivedAt"),
|
||||||
|
replacedAt: timestamp("replacedAt"),
|
||||||
|
metadata: jsonb("metadata"),
|
||||||
|
createdAt: timestamp("createdAt").notNull().defaultNow(),
|
||||||
|
updatedAt: timestamp("updatedAt").notNull().defaultNow(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type ImageArchiveTask = InferSelectModel<typeof imageArchiveTask>;
|
||||||
|
|
||||||
// 网站设置表
|
// 网站设置表
|
||||||
export const siteSettings = pgTable("SiteSettings", {
|
export const siteSettings = pgTable("SiteSettings", {
|
||||||
id: uuid("id").primaryKey().notNull().defaultRandom(),
|
id: uuid("id").primaryKey().notNull().defaultRandom(),
|
||||||
|
|||||||
118
lib/jobs/image-archive-job.ts
Normal file
118
lib/jobs/image-archive-job.ts
Normal file
@ -0,0 +1,118 @@
|
|||||||
|
import "server-only";
|
||||||
|
|
||||||
|
import { archiveTemporaryImage } from "@/lib/storage/image-archive";
|
||||||
|
import {
|
||||||
|
getImageArchiveTaskStats,
|
||||||
|
getImageArchiveTasksToReplace,
|
||||||
|
getImageArchiveTasksToStore,
|
||||||
|
markImageArchiveTaskArchiving,
|
||||||
|
markImageArchiveTaskCompleted,
|
||||||
|
markImageArchiveTaskFailed,
|
||||||
|
markImageArchiveTaskReplaceFailed,
|
||||||
|
markImageArchiveTaskReplacing,
|
||||||
|
markImageArchiveTaskStored,
|
||||||
|
} from "@/lib/storage/image-archive-db";
|
||||||
|
import { replaceImageUrlInDatabase } from "@/lib/storage/image-url-replacement";
|
||||||
|
|
||||||
|
const DEFAULT_BATCH_SIZE = 10;
|
||||||
|
|
||||||
|
function getBatchSize() {
|
||||||
|
const parsed = Number(process.env.IMAGE_ARCHIVE_BATCH_SIZE || DEFAULT_BATCH_SIZE);
|
||||||
|
if (!Number.isFinite(parsed)) return DEFAULT_BATCH_SIZE;
|
||||||
|
return Math.max(1, Math.min(50, Math.round(parsed)));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runImageArchiveJob() {
|
||||||
|
const batchSize = getBatchSize();
|
||||||
|
console.log(`[ImageArchiveJob] Start image archive job (batchSize=${batchSize})`);
|
||||||
|
|
||||||
|
const storeTasks = await getImageArchiveTasksToStore({ limit: batchSize });
|
||||||
|
let storeSucceeded = 0;
|
||||||
|
let storeFailed = 0;
|
||||||
|
|
||||||
|
for (const task of storeTasks) {
|
||||||
|
try {
|
||||||
|
await markImageArchiveTaskArchiving(task.id);
|
||||||
|
const archived = await archiveTemporaryImage(task.sourceUrl);
|
||||||
|
await markImageArchiveTaskStored({
|
||||||
|
id: task.id,
|
||||||
|
targetUrl: archived.url,
|
||||||
|
});
|
||||||
|
storeSucceeded += 1;
|
||||||
|
console.log("[ImageArchiveJob] stored", {
|
||||||
|
taskId: task.id,
|
||||||
|
bytes: archived.bytes,
|
||||||
|
sourceUrl: task.sourceUrl,
|
||||||
|
targetUrl: archived.url,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
storeFailed += 1;
|
||||||
|
await markImageArchiveTaskFailed({ id: task.id, error });
|
||||||
|
console.error("[ImageArchiveJob] store failed", {
|
||||||
|
taskId: task.id,
|
||||||
|
sourceUrl: task.sourceUrl,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const remainingSlots = Math.max(1, batchSize - storeTasks.length);
|
||||||
|
const replaceTasks = await getImageArchiveTasksToReplace({
|
||||||
|
limit: remainingSlots,
|
||||||
|
});
|
||||||
|
let replaceSucceeded = 0;
|
||||||
|
let replaceFailed = 0;
|
||||||
|
|
||||||
|
for (const task of replaceTasks) {
|
||||||
|
if (!task.targetUrl) {
|
||||||
|
await markImageArchiveTaskReplaceFailed({
|
||||||
|
id: task.id,
|
||||||
|
error: new Error("Missing targetUrl for stored image archive task"),
|
||||||
|
});
|
||||||
|
replaceFailed += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await markImageArchiveTaskReplacing(task.id);
|
||||||
|
const result = await replaceImageUrlInDatabase(task.sourceUrl, task.targetUrl);
|
||||||
|
await markImageArchiveTaskCompleted({
|
||||||
|
id: task.id,
|
||||||
|
replacedCount: result.changed,
|
||||||
|
});
|
||||||
|
replaceSucceeded += 1;
|
||||||
|
console.log("[ImageArchiveJob] replaced", {
|
||||||
|
taskId: task.id,
|
||||||
|
sourceUrl: task.sourceUrl,
|
||||||
|
targetUrl: task.targetUrl,
|
||||||
|
replacedCount: result.changed,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
replaceFailed += 1;
|
||||||
|
await markImageArchiveTaskReplaceFailed({ id: task.id, error });
|
||||||
|
console.error("[ImageArchiveJob] replace failed", {
|
||||||
|
taskId: task.id,
|
||||||
|
sourceUrl: task.sourceUrl,
|
||||||
|
targetUrl: task.targetUrl,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const stats = await getImageArchiveTaskStats();
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
message: "Image archive job completed",
|
||||||
|
store: {
|
||||||
|
total: storeTasks.length,
|
||||||
|
succeeded: storeSucceeded,
|
||||||
|
failed: storeFailed,
|
||||||
|
},
|
||||||
|
replace: {
|
||||||
|
total: replaceTasks.length,
|
||||||
|
succeeded: replaceSucceeded,
|
||||||
|
failed: replaceFailed,
|
||||||
|
},
|
||||||
|
stats,
|
||||||
|
};
|
||||||
|
}
|
||||||
240
lib/storage/image-archive-db.ts
Normal file
240
lib/storage/image-archive-db.ts
Normal file
@ -0,0 +1,240 @@
|
|||||||
|
import "server-only";
|
||||||
|
|
||||||
|
import { and, asc, eq, inArray, lte, sql } from "drizzle-orm";
|
||||||
|
import { db } from "@/lib/db";
|
||||||
|
import { imageArchiveTask, type ImageArchiveTask } from "@/lib/db/schema";
|
||||||
|
|
||||||
|
export type ImageArchiveTaskRecord = ImageArchiveTask;
|
||||||
|
|
||||||
|
const DEFAULT_DISPLAY_HOLD_HOURS = 1;
|
||||||
|
|
||||||
|
function getDisplayHoldHours() {
|
||||||
|
const raw = Number(
|
||||||
|
process.env.IMAGE_ARCHIVE_DISPLAY_HOLD_HOURS || DEFAULT_DISPLAY_HOLD_HOURS
|
||||||
|
);
|
||||||
|
if (!Number.isFinite(raw)) return DEFAULT_DISPLAY_HOLD_HOURS;
|
||||||
|
return Math.max(0, Math.min(24 * 30, raw));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getImageArchiveDisplayHoldMs() {
|
||||||
|
return getDisplayHoldHours() * 60 * 60 * 1000;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isArchivableHttpUrl(value: string) {
|
||||||
|
if (!/^https?:\/\//i.test(value)) return false;
|
||||||
|
if (/^(data|blob):/i.test(value)) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function enqueueImageArchiveTask(params: {
|
||||||
|
sourceUrl: string;
|
||||||
|
archiveAfterAt?: Date;
|
||||||
|
replaceAfterAt?: Date;
|
||||||
|
metadata?: Record<string, unknown> | null;
|
||||||
|
}) {
|
||||||
|
const sourceUrl = String(params.sourceUrl || "").trim();
|
||||||
|
if (!isArchivableHttpUrl(sourceUrl)) return null;
|
||||||
|
|
||||||
|
const now = new Date();
|
||||||
|
const archiveAfterAt = params.archiveAfterAt || now;
|
||||||
|
const replaceAfterAt =
|
||||||
|
params.replaceAfterAt ||
|
||||||
|
new Date(now.getTime() + getImageArchiveDisplayHoldMs());
|
||||||
|
|
||||||
|
const [task] = await db
|
||||||
|
.insert(imageArchiveTask)
|
||||||
|
.values({
|
||||||
|
sourceUrl,
|
||||||
|
status: "pending",
|
||||||
|
archiveAfterAt,
|
||||||
|
replaceAfterAt,
|
||||||
|
metadata: params.metadata || null,
|
||||||
|
updatedAt: now,
|
||||||
|
})
|
||||||
|
.onConflictDoUpdate({
|
||||||
|
target: imageArchiveTask.sourceUrl,
|
||||||
|
set: {
|
||||||
|
replaceAfterAt,
|
||||||
|
metadata: params.metadata || undefined,
|
||||||
|
updatedAt: now,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.returning();
|
||||||
|
|
||||||
|
return task || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function enqueueImageArchiveTasks(params: {
|
||||||
|
urls: string[];
|
||||||
|
metadata?: Record<string, unknown> | null;
|
||||||
|
}) {
|
||||||
|
const urls = Array.from(
|
||||||
|
new Set(
|
||||||
|
(params.urls || [])
|
||||||
|
.map((url) => String(url || "").trim())
|
||||||
|
.filter(isArchivableHttpUrl)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
const tasks: ImageArchiveTaskRecord[] = [];
|
||||||
|
for (const sourceUrl of urls) {
|
||||||
|
const task = await enqueueImageArchiveTask({
|
||||||
|
sourceUrl,
|
||||||
|
metadata: params.metadata || null,
|
||||||
|
});
|
||||||
|
if (task) tasks.push(task);
|
||||||
|
}
|
||||||
|
return tasks;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getImageArchiveTasksToStore(params: { limit: number }) {
|
||||||
|
const now = new Date();
|
||||||
|
return await db
|
||||||
|
.select()
|
||||||
|
.from(imageArchiveTask)
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
inArray(imageArchiveTask.status, ["pending", "failed"]),
|
||||||
|
lte(imageArchiveTask.archiveAfterAt, now)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.orderBy(asc(imageArchiveTask.createdAt), asc(imageArchiveTask.id))
|
||||||
|
.limit(Math.max(1, Math.min(50, params.limit)));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function markImageArchiveTaskArchiving(id: string) {
|
||||||
|
await db
|
||||||
|
.update(imageArchiveTask)
|
||||||
|
.set({
|
||||||
|
status: "archiving",
|
||||||
|
attemptCount: sql`${imageArchiveTask.attemptCount} + 1`,
|
||||||
|
lastError: null,
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(eq(imageArchiveTask.id, id));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function markImageArchiveTaskStored(params: {
|
||||||
|
id: string;
|
||||||
|
targetUrl: string;
|
||||||
|
targetStorageConfigId?: string | null;
|
||||||
|
}) {
|
||||||
|
await db
|
||||||
|
.update(imageArchiveTask)
|
||||||
|
.set({
|
||||||
|
status: "stored",
|
||||||
|
targetUrl: params.targetUrl,
|
||||||
|
targetStorageConfigId: params.targetStorageConfigId || null,
|
||||||
|
archivedAt: new Date(),
|
||||||
|
lastError: null,
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(eq(imageArchiveTask.id, params.id));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function markImageArchiveTaskFailed(params: {
|
||||||
|
id: string;
|
||||||
|
error: unknown;
|
||||||
|
}) {
|
||||||
|
const message = String(
|
||||||
|
params.error instanceof Error ? params.error.message : params.error || "Unknown error"
|
||||||
|
).slice(0, 2000);
|
||||||
|
|
||||||
|
await db
|
||||||
|
.update(imageArchiveTask)
|
||||||
|
.set({
|
||||||
|
status: "failed",
|
||||||
|
lastError: message,
|
||||||
|
archiveAfterAt: new Date(Date.now() + 10 * 60 * 1000),
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(eq(imageArchiveTask.id, params.id));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getImageArchiveTasksToReplace(params: { limit: number }) {
|
||||||
|
const now = new Date();
|
||||||
|
return await db
|
||||||
|
.select()
|
||||||
|
.from(imageArchiveTask)
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
eq(imageArchiveTask.status, "stored"),
|
||||||
|
lte(imageArchiveTask.replaceAfterAt, now)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.orderBy(asc(imageArchiveTask.createdAt), asc(imageArchiveTask.id))
|
||||||
|
.limit(Math.max(1, Math.min(50, params.limit)));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function markImageArchiveTaskReplacing(id: string) {
|
||||||
|
await db
|
||||||
|
.update(imageArchiveTask)
|
||||||
|
.set({
|
||||||
|
status: "replacing",
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(eq(imageArchiveTask.id, id));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function markImageArchiveTaskCompleted(params: {
|
||||||
|
id: string;
|
||||||
|
replacedCount?: number;
|
||||||
|
}) {
|
||||||
|
await db
|
||||||
|
.update(imageArchiveTask)
|
||||||
|
.set({
|
||||||
|
status: "completed",
|
||||||
|
replacedAt: new Date(),
|
||||||
|
lastError: null,
|
||||||
|
metadata:
|
||||||
|
params.replacedCount === undefined
|
||||||
|
? undefined
|
||||||
|
: sql`coalesce(${imageArchiveTask.metadata}, '{}'::jsonb) || ${JSON.stringify({
|
||||||
|
replacedCount: params.replacedCount,
|
||||||
|
})}::jsonb`,
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(eq(imageArchiveTask.id, params.id));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function markImageArchiveTaskReplaceFailed(params: {
|
||||||
|
id: string;
|
||||||
|
error: unknown;
|
||||||
|
}) {
|
||||||
|
const message = String(
|
||||||
|
params.error instanceof Error ? params.error.message : params.error || "Unknown error"
|
||||||
|
).slice(0, 2000);
|
||||||
|
|
||||||
|
await db
|
||||||
|
.update(imageArchiveTask)
|
||||||
|
.set({
|
||||||
|
status: "stored",
|
||||||
|
lastError: message,
|
||||||
|
replaceAfterAt: new Date(Date.now() + 10 * 60 * 1000),
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(eq(imageArchiveTask.id, params.id));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getImageArchiveTaskStats() {
|
||||||
|
const grouped = await db
|
||||||
|
.select({
|
||||||
|
status: imageArchiveTask.status,
|
||||||
|
count: sql<number>`count(*)::int`,
|
||||||
|
})
|
||||||
|
.from(imageArchiveTask)
|
||||||
|
.groupBy(imageArchiveTask.status);
|
||||||
|
|
||||||
|
const pendingStore = grouped
|
||||||
|
.filter((item) => item.status === "pending" || item.status === "failed")
|
||||||
|
.reduce((sum, item) => sum + (item.count || 0), 0);
|
||||||
|
const pendingReplace = grouped
|
||||||
|
.filter((item) => item.status === "stored")
|
||||||
|
.reduce((sum, item) => sum + (item.count || 0), 0);
|
||||||
|
|
||||||
|
return {
|
||||||
|
grouped,
|
||||||
|
pendingStore,
|
||||||
|
pendingReplace,
|
||||||
|
};
|
||||||
|
}
|
||||||
198
lib/storage/image-archive.ts
Normal file
198
lib/storage/image-archive.ts
Normal file
@ -0,0 +1,198 @@
|
|||||||
|
import "server-only";
|
||||||
|
|
||||||
|
import { createLogger } from "@/lib/observability/logger";
|
||||||
|
import {
|
||||||
|
enqueueImageArchiveTasks,
|
||||||
|
getImageArchiveDisplayHoldMs,
|
||||||
|
} from "@/lib/storage/image-archive-db";
|
||||||
|
import { uploadFile } from "@/lib/storage/service";
|
||||||
|
|
||||||
|
const logger = createLogger("image-archive");
|
||||||
|
|
||||||
|
const DEFAULT_DOWNLOAD_TIMEOUT_MS = 60_000;
|
||||||
|
|
||||||
|
type DeferredImageArchiveMetadata = Record<string, unknown> & {
|
||||||
|
providerType?: string;
|
||||||
|
modelId?: string;
|
||||||
|
imageSize?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
function getDownloadTimeoutMs() {
|
||||||
|
const parsed = Number(process.env.IMAGE_ARCHIVE_DOWNLOAD_TIMEOUT_MS);
|
||||||
|
if (!Number.isFinite(parsed)) return DEFAULT_DOWNLOAD_TIMEOUT_MS;
|
||||||
|
return Math.max(5_000, Math.min(5 * 60_000, Math.round(parsed)));
|
||||||
|
}
|
||||||
|
|
||||||
|
function isHttpUrl(value: string) {
|
||||||
|
return /^https?:\/\//i.test(String(value || "").trim());
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isDeferredImageArchiveEnabled() {
|
||||||
|
return process.env.AI_IMAGE_DEFERRED_ARCHIVE !== "false";
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function registerGeneratedImageTemporaryUrls(params: {
|
||||||
|
urls: string[];
|
||||||
|
metadata?: DeferredImageArchiveMetadata | null;
|
||||||
|
}) {
|
||||||
|
if (!isDeferredImageArchiveEnabled()) return [];
|
||||||
|
const urls = Array.from(
|
||||||
|
new Set(
|
||||||
|
(params.urls || [])
|
||||||
|
.map((url) => String(url || "").trim())
|
||||||
|
.filter(isHttpUrl)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
if (urls.length === 0) return [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await enqueueImageArchiveTasks({
|
||||||
|
urls,
|
||||||
|
metadata: {
|
||||||
|
...(params.metadata || {}),
|
||||||
|
displayHoldMs: getImageArchiveDisplayHoldMs(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
logger.warn("enqueue temporary image archive failed", {
|
||||||
|
urls,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function formatDeferredGeneratedImageMarkdown(params: {
|
||||||
|
urls: string[];
|
||||||
|
imageSize: string;
|
||||||
|
providerType?: string;
|
||||||
|
modelId?: string;
|
||||||
|
metadata?: DeferredImageArchiveMetadata | null;
|
||||||
|
}) {
|
||||||
|
const urls = Array.from(
|
||||||
|
new Set(
|
||||||
|
(params.urls || [])
|
||||||
|
.map((url) => String(url || "").trim())
|
||||||
|
.filter(isHttpUrl)
|
||||||
|
)
|
||||||
|
);
|
||||||
|
if (urls.length === 0) {
|
||||||
|
throw new Error("AI 未返回可用图片地址");
|
||||||
|
}
|
||||||
|
|
||||||
|
await registerGeneratedImageTemporaryUrls({
|
||||||
|
urls,
|
||||||
|
metadata: {
|
||||||
|
providerType: params.providerType,
|
||||||
|
modelId: params.modelId,
|
||||||
|
imageSize: params.imageSize,
|
||||||
|
...(params.metadata || {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
return urls
|
||||||
|
.map((url) => ``)
|
||||||
|
.join("\n\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createDeferredUploadedImage(
|
||||||
|
sourceUrl: string,
|
||||||
|
metadata?: DeferredImageArchiveMetadata | null
|
||||||
|
) {
|
||||||
|
const url = String(sourceUrl || "").trim();
|
||||||
|
if (!isDeferredImageArchiveEnabled() || !isHttpUrl(url)) return null;
|
||||||
|
|
||||||
|
await registerGeneratedImageTemporaryUrls({
|
||||||
|
urls: [url],
|
||||||
|
metadata: metadata || null,
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
thumbnail: url,
|
||||||
|
original: url,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function getImageExtension(contentType: string | null, sourceUrl: string) {
|
||||||
|
const normalized = String(contentType || "").split(";")[0].trim().toLowerCase();
|
||||||
|
const byType: Record<string, string> = {
|
||||||
|
"image/jpeg": "jpg",
|
||||||
|
"image/jpg": "jpg",
|
||||||
|
"image/png": "png",
|
||||||
|
"image/webp": "webp",
|
||||||
|
"image/gif": "gif",
|
||||||
|
"image/svg+xml": "svg",
|
||||||
|
};
|
||||||
|
if (byType[normalized]) return byType[normalized];
|
||||||
|
|
||||||
|
try {
|
||||||
|
const pathname = new URL(sourceUrl).pathname.toLowerCase();
|
||||||
|
const ext = pathname.split(".").pop()?.replace(/[^a-z0-9]+/g, "") || "";
|
||||||
|
if (["jpg", "jpeg", "png", "webp", "gif", "svg"].includes(ext)) {
|
||||||
|
return ext === "jpeg" ? "jpg" : ext;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// keep fallback
|
||||||
|
}
|
||||||
|
|
||||||
|
return "png";
|
||||||
|
}
|
||||||
|
|
||||||
|
function toArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||||||
|
return buffer.buffer.slice(
|
||||||
|
buffer.byteOffset,
|
||||||
|
buffer.byteOffset + buffer.byteLength
|
||||||
|
) as ArrayBuffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function downloadTemporaryImage(params: {
|
||||||
|
sourceUrl: string;
|
||||||
|
retries?: number;
|
||||||
|
}) {
|
||||||
|
const retries = Math.max(0, Math.min(5, params.retries ?? 2));
|
||||||
|
|
||||||
|
for (let attempt = 0; attempt <= retries; attempt += 1) {
|
||||||
|
try {
|
||||||
|
const response = await fetch(params.sourceUrl, {
|
||||||
|
signal: AbortSignal.timeout(getDownloadTimeoutMs()),
|
||||||
|
headers: {
|
||||||
|
"User-Agent": "AI-Chatbot-Image-Archiver/1.0",
|
||||||
|
Accept: "image/avif,image/webp,image/png,image/jpeg,image/*,*/*;q=0.8",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`HTTP ${response.status} ${response.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentType = response.headers.get("content-type");
|
||||||
|
const arrayBuffer = await response.arrayBuffer();
|
||||||
|
if (!arrayBuffer || arrayBuffer.byteLength === 0) {
|
||||||
|
throw new Error("Downloaded image is empty");
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
buffer: Buffer.from(arrayBuffer),
|
||||||
|
contentType,
|
||||||
|
extension: getImageExtension(contentType, params.sourceUrl),
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
if (attempt >= retries) throw error;
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 1000 * (attempt + 1)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error("Image download failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function archiveTemporaryImage(sourceUrl: string) {
|
||||||
|
const downloaded = await downloadTemporaryImage({ sourceUrl });
|
||||||
|
const filename = `ai-generated-archive-${Date.now()}.${downloaded.extension}`;
|
||||||
|
const uploaded = await uploadFile(filename, toArrayBuffer(downloaded.buffer));
|
||||||
|
return {
|
||||||
|
url: uploaded.url,
|
||||||
|
pathname: uploaded.pathname,
|
||||||
|
contentType: uploaded.contentType,
|
||||||
|
bytes: downloaded.buffer.length,
|
||||||
|
};
|
||||||
|
}
|
||||||
476
lib/storage/image-url-replacement.ts
Normal file
476
lib/storage/image-url-replacement.ts
Normal file
@ -0,0 +1,476 @@
|
|||||||
|
import "server-only";
|
||||||
|
|
||||||
|
import { eq, like, or, sql } from "drizzle-orm";
|
||||||
|
import { db } from "@/lib/db";
|
||||||
|
import {
|
||||||
|
assetMemorySnapshot,
|
||||||
|
canvasDetailArchive,
|
||||||
|
canvasNode,
|
||||||
|
canvasProject,
|
||||||
|
conversationWorkingState,
|
||||||
|
externalImageApiRequestLog,
|
||||||
|
generatedImage,
|
||||||
|
mediaAsset,
|
||||||
|
message,
|
||||||
|
messageDeprecated,
|
||||||
|
model3dAsset,
|
||||||
|
} from "@/lib/db/schema";
|
||||||
|
import { inferOriginalImageUrl } from "@/lib/image-url";
|
||||||
|
|
||||||
|
function normalizeMediaUrl(value: string): string {
|
||||||
|
const raw = String(value || "").trim();
|
||||||
|
if (!raw) return "";
|
||||||
|
const canonical = inferOriginalImageUrl(raw);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parsed = new URL(canonical);
|
||||||
|
return `${parsed.origin}${parsed.pathname}`.replace(/\/+$/, "").toLowerCase();
|
||||||
|
} catch {
|
||||||
|
return canonical
|
||||||
|
.split("#")[0]
|
||||||
|
.split("?")[0]
|
||||||
|
.replace(/\/+$/, "")
|
||||||
|
.toLowerCase();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function replaceDeepUrls(
|
||||||
|
value: unknown,
|
||||||
|
sourceUrl: string,
|
||||||
|
targetUrl: string
|
||||||
|
): { value: unknown; changed: boolean } {
|
||||||
|
if (typeof value === "string") {
|
||||||
|
if (!value.includes(sourceUrl)) {
|
||||||
|
return { value, changed: false };
|
||||||
|
}
|
||||||
|
return { value: value.split(sourceUrl).join(targetUrl), changed: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Array.isArray(value)) {
|
||||||
|
let changed = false;
|
||||||
|
const next = value.map((item) => {
|
||||||
|
const result = replaceDeepUrls(item, sourceUrl, targetUrl);
|
||||||
|
if (result.changed) changed = true;
|
||||||
|
return result.value;
|
||||||
|
});
|
||||||
|
return { value: changed ? next : value, changed };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (value && typeof value === "object") {
|
||||||
|
let changed = false;
|
||||||
|
const next: Record<string, unknown> = {};
|
||||||
|
for (const [key, item] of Object.entries(value as Record<string, unknown>)) {
|
||||||
|
const result = replaceDeepUrls(item, sourceUrl, targetUrl);
|
||||||
|
next[key] = result.value;
|
||||||
|
if (result.changed) changed = true;
|
||||||
|
}
|
||||||
|
return { value: changed ? next : value, changed };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { value, changed: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function replaceImageUrlInDatabase(
|
||||||
|
sourceUrl: string,
|
||||||
|
targetUrl: string
|
||||||
|
) {
|
||||||
|
if (!sourceUrl || !targetUrl || sourceUrl === targetUrl) {
|
||||||
|
return { changed: 0 };
|
||||||
|
}
|
||||||
|
|
||||||
|
let changed = 0;
|
||||||
|
const sourceNormalizedUrl = normalizeMediaUrl(sourceUrl);
|
||||||
|
const targetNormalizedUrl = normalizeMediaUrl(targetUrl);
|
||||||
|
|
||||||
|
const generatedThumb = await db
|
||||||
|
.update(generatedImage)
|
||||||
|
.set({ thumbnailUrl: targetUrl })
|
||||||
|
.where(eq(generatedImage.thumbnailUrl, sourceUrl))
|
||||||
|
.returning({ id: generatedImage.id });
|
||||||
|
changed += generatedThumb.length;
|
||||||
|
|
||||||
|
const generatedOrig = await db
|
||||||
|
.update(generatedImage)
|
||||||
|
.set({ originalUrl: targetUrl })
|
||||||
|
.where(eq(generatedImage.originalUrl, sourceUrl))
|
||||||
|
.returning({ id: generatedImage.id });
|
||||||
|
changed += generatedOrig.length;
|
||||||
|
|
||||||
|
const assets = await db
|
||||||
|
.select()
|
||||||
|
.from(mediaAsset)
|
||||||
|
.where(
|
||||||
|
or(
|
||||||
|
eq(mediaAsset.originalUrl, sourceUrl),
|
||||||
|
eq(mediaAsset.previewUrl, sourceUrl),
|
||||||
|
eq(mediaAsset.thumbnailUrl, sourceUrl),
|
||||||
|
eq(mediaAsset.normalizedUrl, sourceNormalizedUrl),
|
||||||
|
sql`${mediaAsset.metadata}::text like ${`%${sourceUrl}%`}`,
|
||||||
|
sql`${mediaAsset.summary}::text like ${`%${sourceUrl}%`}`
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.limit(500);
|
||||||
|
|
||||||
|
for (const item of assets) {
|
||||||
|
const original = replaceDeepUrls(item.originalUrl, sourceUrl, targetUrl);
|
||||||
|
const preview = replaceDeepUrls(item.previewUrl, sourceUrl, targetUrl);
|
||||||
|
const thumbnail = replaceDeepUrls(item.thumbnailUrl, sourceUrl, targetUrl);
|
||||||
|
const metadata = replaceDeepUrls(item.metadata, sourceUrl, targetUrl);
|
||||||
|
const summary = replaceDeepUrls(item.summary, sourceUrl, targetUrl);
|
||||||
|
const shouldNormalize =
|
||||||
|
item.normalizedUrl === sourceNormalizedUrl || original.changed;
|
||||||
|
const nextOriginal = (original.value as string) || item.originalUrl;
|
||||||
|
|
||||||
|
if (
|
||||||
|
!original.changed &&
|
||||||
|
!preview.changed &&
|
||||||
|
!thumbnail.changed &&
|
||||||
|
!metadata.changed &&
|
||||||
|
!summary.changed &&
|
||||||
|
!shouldNormalize
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await db
|
||||||
|
.update(mediaAsset)
|
||||||
|
.set({
|
||||||
|
originalUrl: nextOriginal,
|
||||||
|
normalizedUrl: shouldNormalize ? targetNormalizedUrl : item.normalizedUrl,
|
||||||
|
previewUrl: (preview.value as string | null | undefined) ?? item.previewUrl,
|
||||||
|
thumbnailUrl:
|
||||||
|
(thumbnail.value as string | null | undefined) ?? item.thumbnailUrl,
|
||||||
|
metadata: (metadata.value ?? item.metadata) as any,
|
||||||
|
summary: (summary.value ?? item.summary) as any,
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(eq(mediaAsset.id, item.id));
|
||||||
|
} catch (error) {
|
||||||
|
await db
|
||||||
|
.update(mediaAsset)
|
||||||
|
.set({
|
||||||
|
originalUrl: nextOriginal,
|
||||||
|
previewUrl: (preview.value as string | null | undefined) ?? item.previewUrl,
|
||||||
|
thumbnailUrl:
|
||||||
|
(thumbnail.value as string | null | undefined) ?? item.thumbnailUrl,
|
||||||
|
metadata: (metadata.value ?? item.metadata) as any,
|
||||||
|
summary: (summary.value ?? item.summary) as any,
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(eq(mediaAsset.id, item.id));
|
||||||
|
}
|
||||||
|
changed += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const snapshots = await db
|
||||||
|
.select()
|
||||||
|
.from(assetMemorySnapshot)
|
||||||
|
.where(
|
||||||
|
or(
|
||||||
|
eq(assetMemorySnapshot.normalizedUrl, sourceNormalizedUrl),
|
||||||
|
sql`${assetMemorySnapshot.metadata}::text like ${`%${sourceUrl}%`}`,
|
||||||
|
like(assetMemorySnapshot.summaryText, `%${sourceUrl}%`)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.limit(500);
|
||||||
|
|
||||||
|
for (const item of snapshots) {
|
||||||
|
const metadata = replaceDeepUrls(item.metadata, sourceUrl, targetUrl);
|
||||||
|
const summary = replaceDeepUrls(item.summaryText, sourceUrl, targetUrl);
|
||||||
|
const shouldNormalize = item.normalizedUrl === sourceNormalizedUrl;
|
||||||
|
if (!metadata.changed && !summary.changed && !shouldNormalize) continue;
|
||||||
|
|
||||||
|
await db
|
||||||
|
.update(assetMemorySnapshot)
|
||||||
|
.set({
|
||||||
|
normalizedUrl: shouldNormalize ? targetNormalizedUrl : item.normalizedUrl,
|
||||||
|
metadata: (metadata.value ?? item.metadata) as any,
|
||||||
|
summaryText: (summary.value as string | null | undefined) ?? item.summaryText,
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(eq(assetMemorySnapshot.id, item.id));
|
||||||
|
changed += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const messages = await db
|
||||||
|
.select({
|
||||||
|
id: message.id,
|
||||||
|
parts: message.parts,
|
||||||
|
attachments: message.attachments,
|
||||||
|
})
|
||||||
|
.from(message)
|
||||||
|
.where(
|
||||||
|
or(
|
||||||
|
sql`${message.parts}::text like ${`%${sourceUrl}%`}`,
|
||||||
|
sql`${message.attachments}::text like ${`%${sourceUrl}%`}`
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.limit(500);
|
||||||
|
|
||||||
|
for (const item of messages) {
|
||||||
|
const partsResult = replaceDeepUrls(item.parts, sourceUrl, targetUrl);
|
||||||
|
const attachmentsResult = replaceDeepUrls(item.attachments, sourceUrl, targetUrl);
|
||||||
|
if (!partsResult.changed && !attachmentsResult.changed) continue;
|
||||||
|
|
||||||
|
await db
|
||||||
|
.update(message)
|
||||||
|
.set({
|
||||||
|
parts: (partsResult.value ?? item.parts) as any,
|
||||||
|
attachments: (attachmentsResult.value ?? item.attachments) as any,
|
||||||
|
})
|
||||||
|
.where(eq(message.id, item.id));
|
||||||
|
changed += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const legacyMessages = await db
|
||||||
|
.select({
|
||||||
|
id: messageDeprecated.id,
|
||||||
|
content: messageDeprecated.content,
|
||||||
|
})
|
||||||
|
.from(messageDeprecated)
|
||||||
|
.where(sql`${messageDeprecated.content}::text like ${`%${sourceUrl}%`}`)
|
||||||
|
.limit(500);
|
||||||
|
|
||||||
|
for (const item of legacyMessages) {
|
||||||
|
const replaced = replaceDeepUrls(item.content, sourceUrl, targetUrl);
|
||||||
|
if (!replaced.changed) continue;
|
||||||
|
|
||||||
|
await db
|
||||||
|
.update(messageDeprecated)
|
||||||
|
.set({ content: replaced.value as any })
|
||||||
|
.where(eq(messageDeprecated.id, item.id));
|
||||||
|
changed += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const projects = await db
|
||||||
|
.select({
|
||||||
|
id: canvasProject.id,
|
||||||
|
canvasData: canvasProject.canvasData,
|
||||||
|
})
|
||||||
|
.from(canvasProject)
|
||||||
|
.where(sql`${canvasProject.canvasData}::text like ${`%${sourceUrl}%`}`)
|
||||||
|
.limit(500);
|
||||||
|
|
||||||
|
for (const item of projects) {
|
||||||
|
const replaced = replaceDeepUrls(item.canvasData, sourceUrl, targetUrl);
|
||||||
|
if (!replaced.changed) continue;
|
||||||
|
|
||||||
|
await db
|
||||||
|
.update(canvasProject)
|
||||||
|
.set({
|
||||||
|
canvasData: replaced.value as any,
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(eq(canvasProject.id, item.id));
|
||||||
|
changed += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const nodes = await db
|
||||||
|
.select({
|
||||||
|
id: canvasNode.id,
|
||||||
|
data: canvasNode.data,
|
||||||
|
})
|
||||||
|
.from(canvasNode)
|
||||||
|
.where(sql`${canvasNode.data}::text like ${`%${sourceUrl}%`}`)
|
||||||
|
.limit(500);
|
||||||
|
|
||||||
|
for (const item of nodes) {
|
||||||
|
const replaced = replaceDeepUrls(item.data, sourceUrl, targetUrl);
|
||||||
|
if (!replaced.changed) continue;
|
||||||
|
|
||||||
|
await db
|
||||||
|
.update(canvasNode)
|
||||||
|
.set({
|
||||||
|
data: replaced.value as any,
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(eq(canvasNode.id, item.id));
|
||||||
|
changed += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const workingStates = await db
|
||||||
|
.select()
|
||||||
|
.from(conversationWorkingState)
|
||||||
|
.where(
|
||||||
|
or(
|
||||||
|
sql`${conversationWorkingState.currentFocusImageUrls}::text like ${`%${sourceUrl}%`}`,
|
||||||
|
sql`${conversationWorkingState.selectedImageUrls}::text like ${`%${sourceUrl}%`}`,
|
||||||
|
sql`${conversationWorkingState.lastUploadedImageUrls}::text like ${`%${sourceUrl}%`}`,
|
||||||
|
sql`${conversationWorkingState.lastGeneratedImageUrls}::text like ${`%${sourceUrl}%`}`,
|
||||||
|
sql`${conversationWorkingState.lastReferencedImageUrls}::text like ${`%${sourceUrl}%`}`,
|
||||||
|
sql`${conversationWorkingState.lastResolvedTargetImageUrls}::text like ${`%${sourceUrl}%`}`,
|
||||||
|
sql`${conversationWorkingState.lastCanvasSnapshot}::text like ${`%${sourceUrl}%`}`,
|
||||||
|
sql`${conversationWorkingState.lastTurnSummary}::text like ${`%${sourceUrl}%`}`
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.limit(500);
|
||||||
|
|
||||||
|
for (const item of workingStates) {
|
||||||
|
const currentFocus = replaceDeepUrls(
|
||||||
|
item.currentFocusImageUrls,
|
||||||
|
sourceUrl,
|
||||||
|
targetUrl
|
||||||
|
);
|
||||||
|
const selected = replaceDeepUrls(item.selectedImageUrls, sourceUrl, targetUrl);
|
||||||
|
const uploaded = replaceDeepUrls(item.lastUploadedImageUrls, sourceUrl, targetUrl);
|
||||||
|
const generated = replaceDeepUrls(item.lastGeneratedImageUrls, sourceUrl, targetUrl);
|
||||||
|
const referenced = replaceDeepUrls(item.lastReferencedImageUrls, sourceUrl, targetUrl);
|
||||||
|
const resolved = replaceDeepUrls(
|
||||||
|
item.lastResolvedTargetImageUrls,
|
||||||
|
sourceUrl,
|
||||||
|
targetUrl
|
||||||
|
);
|
||||||
|
const canvasSnapshot = replaceDeepUrls(
|
||||||
|
item.lastCanvasSnapshot,
|
||||||
|
sourceUrl,
|
||||||
|
targetUrl
|
||||||
|
);
|
||||||
|
const turnSummary = replaceDeepUrls(item.lastTurnSummary, sourceUrl, targetUrl);
|
||||||
|
|
||||||
|
if (
|
||||||
|
!currentFocus.changed &&
|
||||||
|
!selected.changed &&
|
||||||
|
!uploaded.changed &&
|
||||||
|
!generated.changed &&
|
||||||
|
!referenced.changed &&
|
||||||
|
!resolved.changed &&
|
||||||
|
!canvasSnapshot.changed &&
|
||||||
|
!turnSummary.changed
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
await db
|
||||||
|
.update(conversationWorkingState)
|
||||||
|
.set({
|
||||||
|
currentFocusImageUrls: currentFocus.value as any,
|
||||||
|
selectedImageUrls: selected.value as any,
|
||||||
|
lastUploadedImageUrls: uploaded.value as any,
|
||||||
|
lastGeneratedImageUrls: generated.value as any,
|
||||||
|
lastReferencedImageUrls: referenced.value as any,
|
||||||
|
lastResolvedTargetImageUrls: resolved.value as any,
|
||||||
|
lastCanvasSnapshot: (canvasSnapshot.value ?? item.lastCanvasSnapshot) as any,
|
||||||
|
lastTurnSummary: (turnSummary.value ?? item.lastTurnSummary) as any,
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(eq(conversationWorkingState.id, item.id));
|
||||||
|
changed += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const detailArchives = await db
|
||||||
|
.select({
|
||||||
|
id: canvasDetailArchive.id,
|
||||||
|
coverUrl: canvasDetailArchive.coverUrl,
|
||||||
|
brief: canvasDetailArchive.brief,
|
||||||
|
slots: canvasDetailArchive.slots,
|
||||||
|
uiSurfaceSnapshot: canvasDetailArchive.uiSurfaceSnapshot,
|
||||||
|
uiSurfaceLatestReview: canvasDetailArchive.uiSurfaceLatestReview,
|
||||||
|
})
|
||||||
|
.from(canvasDetailArchive)
|
||||||
|
.where(
|
||||||
|
or(
|
||||||
|
like(canvasDetailArchive.coverUrl, `%${sourceUrl}%`),
|
||||||
|
sql`${canvasDetailArchive.brief}::text like ${`%${sourceUrl}%`}`,
|
||||||
|
sql`${canvasDetailArchive.slots}::text like ${`%${sourceUrl}%`}`,
|
||||||
|
sql`${canvasDetailArchive.uiSurfaceSnapshot}::text like ${`%${sourceUrl}%`}`,
|
||||||
|
sql`${canvasDetailArchive.uiSurfaceLatestReview}::text like ${`%${sourceUrl}%`}`
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.limit(500);
|
||||||
|
|
||||||
|
for (const item of detailArchives) {
|
||||||
|
const cover = replaceDeepUrls(item.coverUrl, sourceUrl, targetUrl);
|
||||||
|
const brief = replaceDeepUrls(item.brief, sourceUrl, targetUrl);
|
||||||
|
const slots = replaceDeepUrls(item.slots, sourceUrl, targetUrl);
|
||||||
|
const snapshot = replaceDeepUrls(item.uiSurfaceSnapshot, sourceUrl, targetUrl);
|
||||||
|
const review = replaceDeepUrls(item.uiSurfaceLatestReview, sourceUrl, targetUrl);
|
||||||
|
|
||||||
|
if (
|
||||||
|
!cover.changed &&
|
||||||
|
!brief.changed &&
|
||||||
|
!slots.changed &&
|
||||||
|
!snapshot.changed &&
|
||||||
|
!review.changed
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
await db
|
||||||
|
.update(canvasDetailArchive)
|
||||||
|
.set({
|
||||||
|
coverUrl: (cover.value as string | null | undefined) ?? item.coverUrl,
|
||||||
|
brief: (brief.value ?? item.brief) as any,
|
||||||
|
slots: (slots.value ?? item.slots) as any,
|
||||||
|
uiSurfaceSnapshot: (snapshot.value ?? item.uiSurfaceSnapshot) as any,
|
||||||
|
uiSurfaceLatestReview: (review.value ?? item.uiSurfaceLatestReview) as any,
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(eq(canvasDetailArchive.id, item.id));
|
||||||
|
changed += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const modelAssets = await db
|
||||||
|
.select()
|
||||||
|
.from(model3dAsset)
|
||||||
|
.where(
|
||||||
|
or(
|
||||||
|
like(model3dAsset.previewImageUrl, `%${sourceUrl}%`),
|
||||||
|
sql`${model3dAsset.assetPayload}::text like ${`%${sourceUrl}%`}`
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.limit(500);
|
||||||
|
|
||||||
|
for (const item of modelAssets) {
|
||||||
|
const preview = replaceDeepUrls(item.previewImageUrl, sourceUrl, targetUrl);
|
||||||
|
const payload = replaceDeepUrls(item.assetPayload, sourceUrl, targetUrl);
|
||||||
|
if (!preview.changed && !payload.changed) continue;
|
||||||
|
|
||||||
|
await db
|
||||||
|
.update(model3dAsset)
|
||||||
|
.set({
|
||||||
|
previewImageUrl:
|
||||||
|
(preview.value as string | null | undefined) ?? item.previewImageUrl,
|
||||||
|
assetPayload: (payload.value ?? item.assetPayload) as any,
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(eq(model3dAsset.id, item.id));
|
||||||
|
changed += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const apiLogs = await db
|
||||||
|
.select()
|
||||||
|
.from(externalImageApiRequestLog)
|
||||||
|
.where(
|
||||||
|
or(
|
||||||
|
sql`${externalImageApiRequestLog.responsePayload}::text like ${`%${sourceUrl}%`}`,
|
||||||
|
sql`${externalImageApiRequestLog.generatedImageUrls}::text like ${`%${sourceUrl}%`}`
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.limit(500);
|
||||||
|
|
||||||
|
for (const item of apiLogs) {
|
||||||
|
const responsePayload = replaceDeepUrls(
|
||||||
|
item.responsePayload,
|
||||||
|
sourceUrl,
|
||||||
|
targetUrl
|
||||||
|
);
|
||||||
|
const generatedUrls = replaceDeepUrls(
|
||||||
|
item.generatedImageUrls,
|
||||||
|
sourceUrl,
|
||||||
|
targetUrl
|
||||||
|
);
|
||||||
|
if (!responsePayload.changed && !generatedUrls.changed) continue;
|
||||||
|
|
||||||
|
await db
|
||||||
|
.update(externalImageApiRequestLog)
|
||||||
|
.set({
|
||||||
|
responsePayload: (responsePayload.value ?? item.responsePayload) as any,
|
||||||
|
generatedImageUrls: (generatedUrls.value ?? item.generatedImageUrls) as any,
|
||||||
|
})
|
||||||
|
.where(eq(externalImageApiRequestLog.id, item.id));
|
||||||
|
changed += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return { changed };
|
||||||
|
}
|
||||||
@ -54,13 +54,14 @@ function getStorageTimeoutMs(name: string, fallback: number, min: number, max: n
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function withStorageOperationTimeout<T>(
|
async function withStorageOperationTimeout<T>(
|
||||||
operation: Promise<T>,
|
operation: (abortSignal: AbortSignal) => Promise<T>,
|
||||||
timeoutMs: number,
|
timeoutMs: number,
|
||||||
label: string
|
label: string
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
let timedOut = false;
|
let timedOut = false;
|
||||||
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
||||||
const guardedOperation = operation.catch((error) => {
|
const abortController = new AbortController();
|
||||||
|
const guardedOperation = operation(abortController.signal).catch((error) => {
|
||||||
if (timedOut) {
|
if (timedOut) {
|
||||||
logger.warn(`[Upload] ${label} failed after timeout:`, error);
|
logger.warn(`[Upload] ${label} failed after timeout:`, error);
|
||||||
return new Promise<T>(() => undefined);
|
return new Promise<T>(() => undefined);
|
||||||
@ -70,6 +71,7 @@ async function withStorageOperationTimeout<T>(
|
|||||||
const timeout = new Promise<never>((_, reject) => {
|
const timeout = new Promise<never>((_, reject) => {
|
||||||
timeoutId = setTimeout(() => {
|
timeoutId = setTimeout(() => {
|
||||||
timedOut = true;
|
timedOut = true;
|
||||||
|
abortController.abort();
|
||||||
reject(new Error(`${label} timed out after ${timeoutMs}ms`));
|
reject(new Error(`${label} timed out after ${timeoutMs}ms`));
|
||||||
}, timeoutMs);
|
}, timeoutMs);
|
||||||
});
|
});
|
||||||
@ -256,7 +258,7 @@ async function uploadToS3(
|
|||||||
});
|
});
|
||||||
|
|
||||||
await withStorageOperationTimeout(
|
await withStorageOperationTimeout(
|
||||||
s3Client.send(command),
|
(abortSignal) => s3Client.send(command, { abortSignal }),
|
||||||
getStorageTimeoutMs(
|
getStorageTimeoutMs(
|
||||||
"S3_UPLOAD_REQUEST_TIMEOUT_MS",
|
"S3_UPLOAD_REQUEST_TIMEOUT_MS",
|
||||||
DEFAULT_S3_REQUEST_TIMEOUT_MS,
|
DEFAULT_S3_REQUEST_TIMEOUT_MS,
|
||||||
|
|||||||
@ -3,7 +3,10 @@
|
|||||||
{
|
{
|
||||||
"path": "/api/jobs/archive-videos",
|
"path": "/api/jobs/archive-videos",
|
||||||
"schedule": "0 * * * *"
|
"schedule": "0 * * * *"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "/api/jobs/archive-images",
|
||||||
|
"schedule": "5 * * * *"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user