fix: timeout stalled s3 uploads

This commit is contained in:
134 2026-07-04 20:20:21 +08:00
parent 4d4507489f
commit 8e2c6fea1b

View File

@ -45,6 +45,38 @@ export type StorageUploadFunction = (
let lastKnownGoodDbS3Config: S3UploadConfig | null = null;
const STORAGE_FAILOVER_CONFIG_KEY = "storage_failover_config";
const DEFAULT_S3_REQUEST_TIMEOUT_MS = 30_000;
function getStorageTimeoutMs(name: string, fallback: number, min: number, max: number) {
const parsed = Number(process.env[name]);
if (!Number.isFinite(parsed)) return fallback;
return Math.max(min, Math.min(max, Math.round(parsed)));
}
async function withStorageOperationTimeout<T>(
operation: Promise<T>,
timeoutMs: number,
label: string
): Promise<T> {
let timedOut = false;
let timeoutId: ReturnType<typeof setTimeout> | undefined;
const guardedOperation = operation.catch((error) => {
if (timedOut) {
logger.warn(`[Upload] ${label} failed after timeout:`, error);
return new Promise<T>(() => undefined);
}
throw error;
});
const timeout = new Promise<never>((_, reject) => {
timeoutId = setTimeout(() => {
timedOut = true;
reject(new Error(`${label} timed out after ${timeoutMs}ms`));
}, timeoutMs);
});
return Promise.race([guardedOperation, timeout]).finally(() => {
if (timeoutId) clearTimeout(timeoutId);
});
}
function normalizeEndpoint(endpoint: string): string {
const trimmed = endpoint.trim().replace(/\/+$/, "");
@ -224,7 +256,16 @@ async function uploadToS3(
ACL: "public-read",
});
await s3Client.send(command);
await withStorageOperationTimeout(
s3Client.send(command),
getStorageTimeoutMs(
"S3_UPLOAD_REQUEST_TIMEOUT_MS",
DEFAULT_S3_REQUEST_TIMEOUT_MS,
5_000,
120_000
),
"S3 upload"
);
logger.debug(`[S3 Upload] Upload successful: ${key}`);