257 lines
10 KiB
TypeScript
257 lines
10 KiB
TypeScript
import "server-only";
|
|
|
|
import { GetObjectCommand, HeadObjectCommand } from "@aws-sdk/client-s3";
|
|
import { createHmac } from "node:crypto";
|
|
import { createLogger } from "@/lib/observability/logger";
|
|
import {
|
|
buildS3PublicUrl,
|
|
createS3Client,
|
|
resolveS3UploadTarget,
|
|
} from "@/lib/storage/service";
|
|
|
|
const QINIU_EAST_CHINA_IO = "iovip-z0.qiniuio.com";
|
|
const PNG_HEADER_LIMIT = 64 * 1024;
|
|
const PNG_SIGNATURE = Buffer.from("89504e470d0a1a0a", "hex");
|
|
const QINIU_FETCH_TIMEOUT_MS = 40_000;
|
|
const RELAY_RETRY_TIMEOUT_MS = 25_000;
|
|
const logger = createLogger("qiniu-image-archive");
|
|
|
|
type QiniuUploadTarget = NonNullable<Awaited<ReturnType<typeof resolveS3UploadTarget>>>;
|
|
|
|
class QiniuFetchError extends Error {
|
|
constructor(readonly status: number, readonly requestId: string | null) {
|
|
super(`Qiniu archive fetch returned HTTP ${status}`);
|
|
}
|
|
}
|
|
|
|
export class QiniuArchiveSourceMissingError extends Error {
|
|
constructor(status: number) {
|
|
super(`98K original image is no longer available (HTTP ${status})`);
|
|
}
|
|
}
|
|
|
|
export type ReadyImageDownload = {
|
|
status: "ready";
|
|
url: string;
|
|
contentType: "image/png";
|
|
bytes: number;
|
|
preparedAt: string;
|
|
};
|
|
|
|
function encodeQiniu(value: string | Buffer) {
|
|
return (typeof value === "string" ? Buffer.from(value) : value)
|
|
.toString("base64").replace(/\+/g, "-").replace(/\//g, "_");
|
|
}
|
|
|
|
function isDownloadReadyPng(prefix: Buffer) {
|
|
if (prefix.length < 41 || !prefix.subarray(0, 8).equals(PNG_SIGNATURE)
|
|
|| prefix.readUInt32BE(8) !== 13 || prefix.toString("ascii", 12, 16) !== "IHDR"
|
|
|| !prefix.readUInt32BE(16) || !prefix.readUInt32BE(20)
|
|
|| (prefix[24] !== 8 && prefix[24] !== 16) || (prefix[25] !== 2 && prefix[25] !== 6)) return false;
|
|
|
|
// Animation and EXIF orientation need the existing full-image preparation.
|
|
// An embedded color profile is fine: the browser gets the exact stored bytes.
|
|
for (let offset = 33; offset + 8 <= prefix.length;) {
|
|
const type = prefix.toString("ascii", offset + 4, offset + 8);
|
|
if (type === "IDAT") return true;
|
|
if (type === "acTL" || type === "eXIf" || type === "IEND") return false;
|
|
offset += prefix.readUInt32BE(offset) + 12;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
export async function getReadyQiniuImageDownload(url: string, target?: Awaited<ReturnType<typeof resolveS3UploadTarget>>) {
|
|
const config = (target ?? await resolveS3UploadTarget())?.config;
|
|
if (config?.provider !== "qiniu-kodo" || !config.publicUrl) return;
|
|
const base = buildS3PublicUrl(config, "");
|
|
if (!url.startsWith(base)) return;
|
|
const key = url.slice(base.length);
|
|
if (!/^uploads\/(?:[\w.-]+\/)*[\w.-]+$/.test(key)) return;
|
|
|
|
const client = createS3Client(config);
|
|
try {
|
|
const response = await client.send(new GetObjectCommand({
|
|
Bucket: config.bucket, Key: key, Range: `bytes=0-${PNG_HEADER_LIMIT - 1}`,
|
|
}), { abortSignal: AbortSignal.timeout(10_000) });
|
|
const range = response.ContentRange?.match(/^bytes 0-(\d+)\/(\d+)$/);
|
|
const length = response.ContentLength ?? 0;
|
|
const bytes = Number(range?.[2]);
|
|
if (response.$metadata.httpStatusCode !== 206 || response.ContentType !== "image/png"
|
|
|| !range || !response.Body || length <= 0 || length > PNG_HEADER_LIMIT
|
|
|| Number(range[1]) + 1 !== length || bytes < length) {
|
|
(response.Body as NodeJS.ReadableStream & { destroy?: () => void } | undefined)?.destroy?.();
|
|
return;
|
|
}
|
|
const prefix = Buffer.from(await response.Body.transformToByteArray());
|
|
if (prefix.length === length && isDownloadReadyPng(prefix)) {
|
|
return { status: "ready", url, contentType: "image/png", bytes, preparedAt: new Date().toISOString() } satisfies ReadyImageDownload;
|
|
}
|
|
} finally {
|
|
client.destroy();
|
|
}
|
|
}
|
|
|
|
export async function archiveImageThroughQiniu(sourceUrl: string, archiveId: string) {
|
|
return archiveThroughQiniu(sourceUrl, archiveId, "image");
|
|
}
|
|
|
|
export async function archiveMediaThroughQiniu(sourceUrl: string, archiveId: string, kind: "audio" | "video") {
|
|
return archiveThroughQiniu(sourceUrl, archiveId, kind);
|
|
}
|
|
|
|
async function archiveThroughQiniu(sourceUrl: string, archiveId: string, kind: "image" | "audio" | "video") {
|
|
let source: URL;
|
|
try { source = new URL(sourceUrl); } catch { return null; }
|
|
if (!/^https?:$/.test(source.protocol) || source.username || source.password) return null;
|
|
const target = await resolveS3UploadTarget();
|
|
const config = target?.config;
|
|
if (config?.provider !== "qiniu-kodo" || config.region !== "cn-east-1" || !config.publicUrl) {
|
|
return null;
|
|
}
|
|
// Files already in our bucket and inline previews need no remote copy.
|
|
const localOrigins = [config.publicUrl, config.endpoint, process.env.CANVAS_QINIU_ORIGIN_URL,
|
|
process.env.NEXT_PUBLIC_APP_URL, process.env.AUTH_URL, process.env.NEXTAUTH_URL];
|
|
if (localOrigins.some(value => value && new URL(value).origin === source.origin)
|
|
|| source.hostname === "localhost" || source.hostname === "[::1]"
|
|
|| /^(?:127\.|10\.|192\.168\.|169\.254\.|172\.(?:1[6-9]|2\d|3[01])\.)/.test(source.hostname)) return null;
|
|
|
|
// The archive task owns this object key, including retries after a timeout.
|
|
const pathname = kind === "image" ? `uploads/ai-generated-archive-${archiveId}` : `uploads/canvas-${kind}-${archiveId}`;
|
|
if (kind === "image" && source.origin === "https://api.vip98k.com" &&
|
|
source.pathname.startsWith("/api/external/assets/")) {
|
|
return fetchRelayImageIntoQiniu(sourceUrl, pathname, target!, archiveId);
|
|
}
|
|
return fetchImageIntoQiniu(sourceUrl, pathname, target!, kind);
|
|
}
|
|
|
|
function isRetryableQiniuFetch(error: unknown) {
|
|
if (error instanceof QiniuFetchError) {
|
|
return [408, 478, 502, 503, 504, 599].includes(error.status);
|
|
}
|
|
if (!(error instanceof Error)) return false;
|
|
const cause = error.cause as { code?: string } | undefined;
|
|
return error.name === "TimeoutError" || error.name === "AbortError" ||
|
|
["ECONNRESET", "ETIMEDOUT", "UND_ERR_SOCKET"].includes(cause?.code ?? "");
|
|
}
|
|
|
|
function relaySourceStatus(error: unknown) {
|
|
if (error instanceof QiniuFetchError && [404, 410].includes(error.status)) return error.status;
|
|
}
|
|
|
|
async function readStoredRelayImage(pathname: string, target: QiniuUploadTarget) {
|
|
const client = createS3Client(target.config);
|
|
try {
|
|
const response = await client.send(new HeadObjectCommand({
|
|
Bucket: target.config.bucket, Key: pathname,
|
|
}), { abortSignal: AbortSignal.timeout(4_000) });
|
|
if (response.ContentType?.startsWith("image/") && response.ContentLength && response.ContentLength > 0) {
|
|
return describeArchivedObject(pathname, target, response.ContentType, response.ContentLength);
|
|
}
|
|
} catch {
|
|
return;
|
|
} finally {
|
|
client.destroy();
|
|
}
|
|
}
|
|
|
|
async function fetchRelayImageIntoQiniu(
|
|
sourceUrl: string,
|
|
pathname: string,
|
|
target: QiniuUploadTarget,
|
|
archiveId: string,
|
|
) {
|
|
const startedAt = Date.now();
|
|
try {
|
|
return await fetchImageIntoQiniu(sourceUrl, pathname, target);
|
|
} catch (error) {
|
|
const missing = relaySourceStatus(error);
|
|
const retryable = isRetryableQiniuFetch(error);
|
|
const alreadyExists = error instanceof QiniuFetchError && error.status === 614;
|
|
if (!missing && !retryable && !alreadyExists) throw error;
|
|
|
|
// Losing the fetch response does not establish whether Kodo stored the file.
|
|
const stored = await readStoredRelayImage(pathname, target);
|
|
if (stored) return stored;
|
|
if (missing) throw new QiniuArchiveSourceMissingError(missing);
|
|
if (!retryable) throw error;
|
|
|
|
logger.warn("relay_fetch_retry", {
|
|
archiveId,
|
|
elapsedMs: Date.now() - startedAt,
|
|
...(error instanceof QiniuFetchError
|
|
? { status: error.status, requestId: error.requestId }
|
|
: { errorType: error instanceof Error ? error.name : "unknown" }),
|
|
});
|
|
}
|
|
|
|
try {
|
|
return await fetchImageIntoQiniu(sourceUrl, pathname, target, "image", RELAY_RETRY_TIMEOUT_MS);
|
|
} catch (error) {
|
|
const stored = await readStoredRelayImage(pathname, target);
|
|
if (stored) return stored;
|
|
const missing = relaySourceStatus(error);
|
|
if (missing) throw new QiniuArchiveSourceMissingError(missing);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/** Prepare uploaded WebP images inside Kodo, retaining their embedded colour profile. */
|
|
export async function prepareQiniuWebpDownload(url: string, taskId: string) {
|
|
const target = await resolveS3UploadTarget();
|
|
const config = target?.config;
|
|
if (config?.provider !== "qiniu-kodo" || config.region !== "cn-east-1" || !config.publicUrl) return;
|
|
const base = buildS3PublicUrl(config, "");
|
|
if (!url.startsWith(base) || !/^uploads\/(?:[\w.-]+\/)*[\w.-]+\.webp$/i.test(url.slice(base.length))) return;
|
|
// No resizing or lossy re-encoding; apply orientation before removing EXIF.
|
|
const source = `${url}?imageMogr2/auto-orient/format/png/strip/meta-keep-list/${encodeQiniu("icc")}`;
|
|
const result = await fetchImageIntoQiniu(source, `uploads/canvas-download-${taskId}.png`, target!);
|
|
return result.downloadAsset;
|
|
}
|
|
|
|
async function fetchImageIntoQiniu(
|
|
sourceUrl: string,
|
|
pathname: string,
|
|
target: QiniuUploadTarget,
|
|
kind: "image" | "audio" | "video" = "image",
|
|
timeoutMs = QINIU_FETCH_TIMEOUT_MS,
|
|
) {
|
|
const config = target.config;
|
|
const path = `/fetch/${encodeQiniu(sourceUrl)}/to/${encodeQiniu(`${config.bucket}:${pathname}`)}`;
|
|
const contentType = "application/x-www-form-urlencoded";
|
|
const signingData = `POST ${path}\nHost: ${QINIU_EAST_CHINA_IO}\nContent-Type: ${contentType}\n\n`;
|
|
const signature = encodeQiniu(createHmac("sha1", config.secretAccessKey).update(signingData).digest());
|
|
const response = await fetch(`https://${QINIU_EAST_CHINA_IO}${path}`, {
|
|
method: "POST",
|
|
headers: {
|
|
Authorization: `Qiniu ${config.accessKeyId}:${signature}`,
|
|
"Content-Type": contentType,
|
|
},
|
|
body: "",
|
|
redirect: "error",
|
|
signal: AbortSignal.timeout(timeoutMs),
|
|
});
|
|
if (!response.ok) {
|
|
await response.body?.cancel();
|
|
throw new QiniuFetchError(response.status, response.headers.get("x-reqid"));
|
|
}
|
|
const result = await response.json() as { key?: string; fsize?: number; mimeType?: string };
|
|
if (result.key !== pathname || !result.fsize || !result.mimeType?.startsWith(`${kind}/`)) {
|
|
throw new Error(`Qiniu archive fetch did not return ${kind}`);
|
|
}
|
|
return describeArchivedObject(pathname, target, result.mimeType, result.fsize);
|
|
}
|
|
|
|
async function describeArchivedObject(pathname: string, target: QiniuUploadTarget, contentType: string, bytes: number) {
|
|
const url = buildS3PublicUrl(target.config, pathname);
|
|
return {
|
|
url,
|
|
thumbnailUrl: url,
|
|
pathname,
|
|
contentType,
|
|
bytes,
|
|
downloadAsset: contentType === "image/png"
|
|
? await getReadyQiniuImageDownload(url, target).catch(() => undefined)
|
|
: undefined,
|
|
};
|
|
}
|