fix: stage vip98k media on reachable storage
This commit is contained in:
134
root
parent
0a364f537f
commit
6a30795bb8
@ -1,13 +1,9 @@
|
||||
import "server-only";
|
||||
|
||||
import { GetObjectCommand } from "@aws-sdk/client-s3";
|
||||
import { GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
|
||||
import { createOpenAI } from "@ai-sdk/openai";
|
||||
import sharp from "sharp";
|
||||
import {
|
||||
getDefaultStorageConfig,
|
||||
getStorageConfigById,
|
||||
getSystemSetting,
|
||||
} from "@/lib/db/queries";
|
||||
import { getDefaultStorageConfig } from "@/lib/db/queries";
|
||||
import {
|
||||
bindVip98kImages,
|
||||
selectVip98kTaskInvocation,
|
||||
@ -22,11 +18,12 @@ import {
|
||||
import { createLogger } from "@/lib/observability/logger";
|
||||
import { createDeferredUploadedImage } from "@/lib/storage/image-archive";
|
||||
import {
|
||||
buildS3PublicUrl,
|
||||
buildStorageObjectKey,
|
||||
createS3Client,
|
||||
type S3UploadConfig,
|
||||
toDbS3UploadConfig,
|
||||
uploadFile,
|
||||
uploadFileWithS3Config,
|
||||
} from "@/lib/storage/service";
|
||||
|
||||
const logger = createLogger("vip98k-provider");
|
||||
@ -68,13 +65,11 @@ const REFERENCE_DOWNLOAD_ATTEMPTS = 3;
|
||||
const REFERENCE_DOWNLOAD_TIMEOUT_MS = 120_000;
|
||||
const MAX_REFERENCE_DOWNLOAD_BYTES = 40 * 1024 * 1024;
|
||||
const REFERENCE_STAGING_CACHE_MS = 60_000;
|
||||
const STORAGE_FAILOVER_CONFIG_KEY = "storage_failover_config";
|
||||
const REFERENCE_STAGING_PREFIX = "vip98k-references";
|
||||
|
||||
type ReferenceStagingConfig = {
|
||||
sourcePublicUrl: string;
|
||||
sourceDownloadConfig?: S3UploadConfig;
|
||||
targetPublicUrl?: string;
|
||||
uploadConfig: S3UploadConfig;
|
||||
storageConfig: S3UploadConfig;
|
||||
};
|
||||
|
||||
let referenceStagingConfigCache:
|
||||
@ -468,22 +463,6 @@ async function archiveOutputImage(imageUrl: string): Promise<UploadedImage> {
|
||||
);
|
||||
}
|
||||
|
||||
function readStorageFailoverConfig(value: unknown) {
|
||||
let parsed = value;
|
||||
if (typeof parsed === "string") {
|
||||
try {
|
||||
parsed = JSON.parse(parsed);
|
||||
} catch {
|
||||
parsed = null;
|
||||
}
|
||||
}
|
||||
const record = toRecord(parsed);
|
||||
return {
|
||||
enabled: record?.enabled === true,
|
||||
backupStorageConfigId: String(record?.backupStorageConfigId || "").trim(),
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveReferenceStagingConfig() {
|
||||
const now = Date.now();
|
||||
if (referenceStagingConfigCache?.expiresAt && referenceStagingConfigCache.expiresAt > now) {
|
||||
@ -492,29 +471,13 @@ async function resolveReferenceStagingConfig() {
|
||||
|
||||
let value: ReferenceStagingConfig | null = null;
|
||||
try {
|
||||
const [sourceStorage, failoverSetting] = await Promise.all([
|
||||
getDefaultStorageConfig(),
|
||||
getSystemSetting({ key: STORAGE_FAILOVER_CONFIG_KEY }),
|
||||
]);
|
||||
const failover = readStorageFailoverConfig(failoverSetting?.value);
|
||||
if (
|
||||
sourceStorage?.publicUrl &&
|
||||
failover.enabled &&
|
||||
failover.backupStorageConfigId &&
|
||||
failover.backupStorageConfigId !== sourceStorage.id
|
||||
) {
|
||||
const targetStorage = await getStorageConfigById({
|
||||
id: failover.backupStorageConfigId,
|
||||
});
|
||||
const uploadConfig = toDbS3UploadConfig(targetStorage);
|
||||
if (uploadConfig) {
|
||||
value = {
|
||||
sourcePublicUrl: sourceStorage.publicUrl,
|
||||
sourceDownloadConfig: toDbS3UploadConfig(sourceStorage) || undefined,
|
||||
targetPublicUrl: targetStorage?.publicUrl || uploadConfig.publicUrl,
|
||||
uploadConfig,
|
||||
};
|
||||
}
|
||||
const sourceStorage = await getDefaultStorageConfig();
|
||||
const storageConfig = toDbS3UploadConfig(sourceStorage);
|
||||
if (sourceStorage?.publicUrl && storageConfig) {
|
||||
value = {
|
||||
sourcePublicUrl: sourceStorage.publicUrl,
|
||||
storageConfig,
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn("reference staging config lookup failed", {
|
||||
@ -596,7 +559,6 @@ async function downloadReferenceImageFromStorage(
|
||||
mediaUrl: string,
|
||||
stagingConfig: ReferenceStagingConfig,
|
||||
) {
|
||||
if (!stagingConfig.sourceDownloadConfig) return null;
|
||||
const objectKey = getStorageObjectKeyFromPublicUrl(
|
||||
mediaUrl,
|
||||
stagingConfig.sourcePublicUrl,
|
||||
@ -604,9 +566,9 @@ async function downloadReferenceImageFromStorage(
|
||||
if (!objectKey) return null;
|
||||
|
||||
try {
|
||||
const response = await createS3Client(stagingConfig.sourceDownloadConfig).send(
|
||||
const response = await createS3Client(stagingConfig.storageConfig).send(
|
||||
new GetObjectCommand({
|
||||
Bucket: stagingConfig.sourceDownloadConfig.bucket,
|
||||
Bucket: stagingConfig.storageConfig.bucket,
|
||||
Key: objectKey,
|
||||
}),
|
||||
{ abortSignal: AbortSignal.timeout(REFERENCE_DOWNLOAD_TIMEOUT_MS) },
|
||||
@ -642,6 +604,27 @@ async function downloadReferenceImageFromStorage(
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadReferenceImage(
|
||||
buffer: Buffer,
|
||||
stagingConfig: ReferenceStagingConfig,
|
||||
) {
|
||||
const key = buildStorageObjectKey("reference.jpg", REFERENCE_STAGING_PREFIX);
|
||||
await createS3Client(stagingConfig.storageConfig).send(
|
||||
new PutObjectCommand({
|
||||
Bucket: stagingConfig.storageConfig.bucket,
|
||||
Key: key,
|
||||
Body: buffer,
|
||||
ContentType: "image/jpeg",
|
||||
CacheControl: "public, max-age=31536000, immutable",
|
||||
...(stagingConfig.storageConfig.provider === "qiniu-kodo"
|
||||
? {}
|
||||
: { ACL: "public-read" as const }),
|
||||
}),
|
||||
{ abortSignal: AbortSignal.timeout(REFERENCE_DOWNLOAD_TIMEOUT_MS) },
|
||||
);
|
||||
return buildS3PublicUrl(stagingConfig.storageConfig, key);
|
||||
}
|
||||
|
||||
async function normalizeReferenceImage(buffer: Buffer, mimeType: string) {
|
||||
try {
|
||||
const normalized = await sharp(buffer, { failOn: "none" })
|
||||
@ -673,19 +656,14 @@ async function stageReferenceImage(
|
||||
downloaded.buffer,
|
||||
downloaded.mimeType,
|
||||
);
|
||||
const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
const uploaded = await uploadFileWithS3Config(
|
||||
`vip98k-reference-${suffix}.${normalized.extension}`,
|
||||
toArrayBuffer(normalized.buffer),
|
||||
stagingConfig.uploadConfig,
|
||||
);
|
||||
const uploadedUrl = await uploadReferenceImage(normalized.buffer, stagingConfig);
|
||||
logger.info("reference image staged", {
|
||||
sourceHost: getUrlHost(mediaUrl),
|
||||
targetHost: getUrlHost(uploaded.url),
|
||||
targetHost: getUrlHost(uploadedUrl),
|
||||
sourceBytes: downloaded.buffer.byteLength,
|
||||
stagedBytes: normalized.buffer.byteLength,
|
||||
});
|
||||
return uploaded.url;
|
||||
return uploadedUrl;
|
||||
}
|
||||
|
||||
async function ensurePublicMediaUrl(mediaUrl: string, mediaKind: "image" | "video") {
|
||||
@ -693,12 +671,15 @@ async function ensurePublicMediaUrl(mediaUrl: string, mediaKind: "image" | "vide
|
||||
if (/^https?:\/\//i.test(normalized)) {
|
||||
if (mediaKind !== "image") return normalized;
|
||||
const stagingConfig = await resolveReferenceStagingConfig();
|
||||
const sourceObjectKey = stagingConfig
|
||||
? getStorageObjectKeyFromPublicUrl(normalized, stagingConfig.sourcePublicUrl)
|
||||
: null;
|
||||
if (
|
||||
!stagingConfig ||
|
||||
sourceObjectKey?.startsWith(`${REFERENCE_STAGING_PREFIX}/`) ||
|
||||
!shouldStageVip98kReferenceUrl({
|
||||
mediaUrl: normalized,
|
||||
sourcePublicUrl: stagingConfig.sourcePublicUrl,
|
||||
targetPublicUrl: stagingConfig.targetPublicUrl,
|
||||
})
|
||||
) {
|
||||
return normalized;
|
||||
@ -717,13 +698,12 @@ async function ensurePublicMediaUrl(mediaUrl: string, mediaKind: "image" | "vide
|
||||
const extension = detectExtension(parsed.mimeType);
|
||||
const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
const stagingConfig = await resolveReferenceStagingConfig();
|
||||
if (stagingConfig) {
|
||||
const uploaded = await uploadFileWithS3Config(
|
||||
`vip98k-ref-${suffix}.${extension}`,
|
||||
toArrayBuffer(parsed.buffer),
|
||||
stagingConfig.uploadConfig,
|
||||
if (stagingConfig && mediaKind === "image") {
|
||||
const normalizedImage = await normalizeReferenceImage(
|
||||
parsed.buffer,
|
||||
parsed.mimeType,
|
||||
);
|
||||
return uploaded.url;
|
||||
return uploadReferenceImage(normalizedImage.buffer, stagingConfig);
|
||||
}
|
||||
const uploaded = await uploadFile(
|
||||
`vip98k-ref-${suffix}.${extension}`,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user