fix: timeout stalled s3 uploads
This commit is contained in:
parent
4d4507489f
commit
8e2c6fea1b
@ -45,6 +45,38 @@ export type StorageUploadFunction = (
|
||||
|
||||
let lastKnownGoodDbS3Config: S3UploadConfig | null = null;
|
||||
const STORAGE_FAILOVER_CONFIG_KEY = "storage_failover_config";
|
||||
const DEFAULT_S3_REQUEST_TIMEOUT_MS = 30_000;
|
||||
|
||||
function getStorageTimeoutMs(name: string, fallback: number, min: number, max: number) {
|
||||
const parsed = Number(process.env[name]);
|
||||
if (!Number.isFinite(parsed)) return fallback;
|
||||
return Math.max(min, Math.min(max, Math.round(parsed)));
|
||||
}
|
||||
|
||||
async function withStorageOperationTimeout<T>(
|
||||
operation: Promise<T>,
|
||||
timeoutMs: number,
|
||||
label: string
|
||||
): Promise<T> {
|
||||
let timedOut = false;
|
||||
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
||||
const guardedOperation = operation.catch((error) => {
|
||||
if (timedOut) {
|
||||
logger.warn(`[Upload] ${label} failed after timeout:`, error);
|
||||
return new Promise<T>(() => undefined);
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
const timeout = new Promise<never>((_, reject) => {
|
||||
timeoutId = setTimeout(() => {
|
||||
timedOut = true;
|
||||
reject(new Error(`${label} timed out after ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
});
|
||||
return Promise.race([guardedOperation, timeout]).finally(() => {
|
||||
if (timeoutId) clearTimeout(timeoutId);
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeEndpoint(endpoint: string): string {
|
||||
const trimmed = endpoint.trim().replace(/\/+$/, "");
|
||||
@ -224,7 +256,16 @@ async function uploadToS3(
|
||||
ACL: "public-read",
|
||||
});
|
||||
|
||||
await s3Client.send(command);
|
||||
await withStorageOperationTimeout(
|
||||
s3Client.send(command),
|
||||
getStorageTimeoutMs(
|
||||
"S3_UPLOAD_REQUEST_TIMEOUT_MS",
|
||||
DEFAULT_S3_REQUEST_TIMEOUT_MS,
|
||||
5_000,
|
||||
120_000
|
||||
),
|
||||
"S3 upload"
|
||||
);
|
||||
|
||||
logger.debug(`[S3 Upload] Upload successful: ${key}`);
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user