542 lines
13 KiB
TypeScript
542 lines
13 KiB
TypeScript
const DEFAULT_MAX_RETRIES = 3;
|
||
const MAX_BACKOFF_MS = 8000;
|
||
const MAX_CONCURRENT_UPLOADS = 3;
|
||
const DIRECT_UPLOAD_MIN_BYTES = 1;
|
||
const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024;
|
||
const SERVER_UPLOAD_MAX_BYTES = 50 * 1024 * 1024;
|
||
const DIRECT_UPLOAD_PART_CONCURRENCY = 4;
|
||
const DIRECT_UPLOAD_SUPPRESS_MS = 2 * 60 * 1000;
|
||
const CLIENT_UPLOAD_ID_FIELD = "clientUploadId";
|
||
|
||
const RETRYABLE_UPLOAD_STATUSES = new Set([
|
||
408,
|
||
425,
|
||
429,
|
||
500,
|
||
502,
|
||
503,
|
||
504,
|
||
]);
|
||
|
||
const DIRECT_UPLOAD_IMAGE_TYPES = new Set([
|
||
"image/jpeg",
|
||
"image/jpg",
|
||
"image/png",
|
||
"image/webp",
|
||
]);
|
||
|
||
const BROWSER_IMAGE_TYPES_BY_EXTENSION: Record<string, string> = {
|
||
jpg: "image/jpeg",
|
||
jpeg: "image/jpeg",
|
||
png: "image/png",
|
||
webp: "image/webp",
|
||
};
|
||
|
||
let activeUploadCount = 0;
|
||
const uploadQueue: Array<() => void> = [];
|
||
let directUploadSuppressedUntil = 0;
|
||
|
||
type UploadFormFile = {
|
||
blob: Blob;
|
||
filename: string;
|
||
contentType: string;
|
||
purpose: string;
|
||
clientUploadId: string;
|
||
};
|
||
|
||
type DirectUploadPart = {
|
||
partNumber: number;
|
||
url: string;
|
||
};
|
||
|
||
type DirectUploadInitiateResponse =
|
||
| {
|
||
mode: "single";
|
||
key: string;
|
||
url: string;
|
||
headers?: Record<string, string>;
|
||
}
|
||
| {
|
||
mode: "multipart";
|
||
key: string;
|
||
uploadId: string;
|
||
partSize: number;
|
||
partCount: number;
|
||
parts: DirectUploadPart[];
|
||
};
|
||
|
||
function createAbortError() {
|
||
const error = new Error("Upload aborted");
|
||
error.name = "AbortError";
|
||
return error;
|
||
}
|
||
|
||
function sleep(ms: number, signal?: AbortSignal) {
|
||
if (signal?.aborted) {
|
||
return Promise.reject(createAbortError());
|
||
}
|
||
|
||
return new Promise<void>((resolve, reject) => {
|
||
const timeout = globalThis.setTimeout(() => {
|
||
signal?.removeEventListener("abort", onAbort);
|
||
resolve();
|
||
}, ms);
|
||
|
||
const onAbort = () => {
|
||
globalThis.clearTimeout(timeout);
|
||
reject(createAbortError());
|
||
};
|
||
|
||
signal?.addEventListener("abort", onAbort, { once: true });
|
||
});
|
||
}
|
||
|
||
function acquireUploadSlot() {
|
||
return new Promise<void>((resolve) => {
|
||
const start = () => {
|
||
activeUploadCount += 1;
|
||
resolve();
|
||
};
|
||
|
||
if (activeUploadCount < MAX_CONCURRENT_UPLOADS) {
|
||
start();
|
||
} else {
|
||
uploadQueue.push(start);
|
||
}
|
||
});
|
||
}
|
||
|
||
async function withUploadSlot<T>(task: () => Promise<T>): Promise<T> {
|
||
await acquireUploadSlot();
|
||
|
||
try {
|
||
return await task();
|
||
} finally {
|
||
activeUploadCount = Math.max(0, activeUploadCount - 1);
|
||
uploadQueue.shift()?.();
|
||
}
|
||
}
|
||
|
||
function parseRetryAfterMs(value: string | null): number | null {
|
||
if (!value) return null;
|
||
const seconds = Number(value);
|
||
if (Number.isFinite(seconds) && seconds > 0) {
|
||
return Math.round(seconds * 1000);
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function computeBackoffMs(attempt: number, retryAfterMs: number | null) {
|
||
if (retryAfterMs && retryAfterMs > 0) {
|
||
return Math.min(retryAfterMs, MAX_BACKOFF_MS);
|
||
}
|
||
const exp = Math.min(MAX_BACKOFF_MS, 500 * 2 ** attempt);
|
||
const jitter = Math.floor(Math.random() * 300);
|
||
return exp + jitter;
|
||
}
|
||
|
||
function isRetryableUploadStatus(status: number) {
|
||
return RETRYABLE_UPLOAD_STATUSES.has(status);
|
||
}
|
||
|
||
function isAbortError(error: unknown, signal?: AbortSignal) {
|
||
return (
|
||
signal?.aborted ||
|
||
(error instanceof Error && error.name === "AbortError")
|
||
);
|
||
}
|
||
|
||
function cloneFormData(formData: FormData) {
|
||
const copy = new FormData();
|
||
|
||
formData.forEach((value, key) => {
|
||
if (typeof value === "string") {
|
||
copy.append(key, value);
|
||
return;
|
||
}
|
||
|
||
const filename =
|
||
typeof File !== "undefined" && value instanceof File
|
||
? value.name
|
||
: undefined;
|
||
|
||
if (filename) {
|
||
copy.append(key, value, filename);
|
||
} else {
|
||
copy.append(key, value);
|
||
}
|
||
});
|
||
|
||
return copy;
|
||
}
|
||
|
||
function normalizeBrowserImageType(contentType: string, filename: string) {
|
||
const normalized = contentType.trim().toLowerCase();
|
||
if (normalized && normalized !== "application/octet-stream") {
|
||
return normalized;
|
||
}
|
||
|
||
const extension = filename.toLowerCase().split(".").pop() || "";
|
||
return BROWSER_IMAGE_TYPES_BY_EXTENSION[extension] || normalized;
|
||
}
|
||
|
||
function createClientUploadId() {
|
||
if (
|
||
typeof crypto !== "undefined" &&
|
||
typeof crypto.randomUUID === "function"
|
||
) {
|
||
return crypto.randomUUID();
|
||
}
|
||
return `upload-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
|
||
}
|
||
|
||
function ensureClientUploadId(formData: FormData) {
|
||
const existing = formData.get(CLIENT_UPLOAD_ID_FIELD);
|
||
if (typeof existing === "string" && existing.trim()) {
|
||
return existing.trim();
|
||
}
|
||
|
||
const clientUploadId = createClientUploadId();
|
||
formData.set(CLIENT_UPLOAD_ID_FIELD, clientUploadId);
|
||
return clientUploadId;
|
||
}
|
||
|
||
function getUploadFormFile(formData: FormData): UploadFormFile | null {
|
||
const value = formData.get("file");
|
||
if (!(value instanceof Blob)) {
|
||
return null;
|
||
}
|
||
|
||
const filename =
|
||
typeof File !== "undefined" && value instanceof File && value.name
|
||
? value.name
|
||
: `upload-${Date.now()}`;
|
||
const contentType = normalizeBrowserImageType(
|
||
value.type || "application/octet-stream",
|
||
filename
|
||
);
|
||
const purposeValue = formData.get("purpose");
|
||
const clientUploadId = ensureClientUploadId(formData);
|
||
|
||
return {
|
||
blob: value,
|
||
filename,
|
||
contentType,
|
||
purpose: typeof purposeValue === "string" ? purposeValue : "",
|
||
clientUploadId,
|
||
};
|
||
}
|
||
|
||
function shouldTryDirectUpload(file: UploadFormFile) {
|
||
if (
|
||
Date.now() < directUploadSuppressedUntil &&
|
||
file.blob.size <= SERVER_UPLOAD_MAX_BYTES
|
||
) {
|
||
return false;
|
||
}
|
||
if (file.blob.size <= 0 || file.blob.size > DIRECT_UPLOAD_MAX_BYTES) {
|
||
return false;
|
||
}
|
||
if (!DIRECT_UPLOAD_IMAGE_TYPES.has(file.contentType.toLowerCase())) {
|
||
return false;
|
||
}
|
||
if (file.purpose === "agent-workflow-preview") {
|
||
return false;
|
||
}
|
||
|
||
return (
|
||
file.purpose === "canvas-upload" ||
|
||
file.blob.size >= DIRECT_UPLOAD_MIN_BYTES
|
||
);
|
||
}
|
||
|
||
function suppressDirectUploadTemporarily() {
|
||
directUploadSuppressedUntil = Date.now() + DIRECT_UPLOAD_SUPPRESS_MS;
|
||
}
|
||
|
||
async function readErrorPayload(response: Response) {
|
||
try {
|
||
const payload = await response.clone().json();
|
||
return payload?.error || payload?.details || response.statusText;
|
||
} catch {
|
||
return response.statusText;
|
||
}
|
||
}
|
||
|
||
async function postDirectUploadAction<T>(
|
||
payload: Record<string, unknown>,
|
||
signal?: AbortSignal
|
||
): Promise<T> {
|
||
const response = await fetch("/api/files/direct-upload", {
|
||
method: "POST",
|
||
headers: {
|
||
"Content-Type": "application/json",
|
||
},
|
||
body: JSON.stringify(payload),
|
||
signal,
|
||
});
|
||
|
||
if (!response.ok) {
|
||
throw new Error(String(await readErrorPayload(response)));
|
||
}
|
||
|
||
return (await response.json()) as T;
|
||
}
|
||
|
||
async function fetchDirectPartWithRetry(
|
||
url: string,
|
||
init: RequestInit,
|
||
maxRetries: number,
|
||
signal?: AbortSignal
|
||
) {
|
||
for (let attempt = 0; ; attempt += 1) {
|
||
try {
|
||
const response = await fetch(url, {
|
||
...init,
|
||
signal,
|
||
});
|
||
|
||
if (response.ok || !isRetryableUploadStatus(response.status) || attempt >= maxRetries) {
|
||
return response;
|
||
}
|
||
|
||
await sleep(computeBackoffMs(attempt, null), signal);
|
||
} catch (error) {
|
||
if (isAbortError(error, signal) || attempt >= maxRetries) {
|
||
throw error;
|
||
}
|
||
|
||
await sleep(computeBackoffMs(attempt, null), signal);
|
||
}
|
||
}
|
||
}
|
||
|
||
async function uploadMultipartParts(params: {
|
||
file: UploadFormFile;
|
||
upload: Extract<DirectUploadInitiateResponse, { mode: "multipart" }>;
|
||
maxRetries: number;
|
||
signal?: AbortSignal;
|
||
}) {
|
||
const uploadedParts: Array<{ ETag: string; PartNumber: number }> = [];
|
||
let nextIndex = 0;
|
||
|
||
async function worker() {
|
||
for (;;) {
|
||
const part = params.upload.parts[nextIndex];
|
||
nextIndex += 1;
|
||
if (!part) return;
|
||
|
||
const start = (part.partNumber - 1) * params.upload.partSize;
|
||
const end = Math.min(start + params.upload.partSize, params.file.blob.size);
|
||
const body = params.file.blob.slice(start, end);
|
||
const response = await fetchDirectPartWithRetry(
|
||
part.url,
|
||
{
|
||
method: "PUT",
|
||
body,
|
||
},
|
||
params.maxRetries,
|
||
params.signal
|
||
);
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`Direct multipart upload failed: ${response.status}`);
|
||
}
|
||
|
||
const etag = response.headers.get("ETag") || response.headers.get("etag");
|
||
if (!etag) {
|
||
throw new Error("Storage did not expose ETag; please allow ETag in bucket CORS");
|
||
}
|
||
|
||
uploadedParts.push({
|
||
ETag: etag,
|
||
PartNumber: part.partNumber,
|
||
});
|
||
}
|
||
}
|
||
|
||
await Promise.all(
|
||
Array.from(
|
||
{ length: Math.min(DIRECT_UPLOAD_PART_CONCURRENCY, params.upload.parts.length) },
|
||
() => worker()
|
||
)
|
||
);
|
||
|
||
return uploadedParts.sort((a, b) => a.PartNumber - b.PartNumber);
|
||
}
|
||
|
||
async function fetchDirectUpload(
|
||
formData: FormData,
|
||
maxRetries: number,
|
||
signal?: AbortSignal
|
||
): Promise<Response | null> {
|
||
const file = getUploadFormFile(formData);
|
||
if (!file || !shouldTryDirectUpload(file)) {
|
||
return null;
|
||
}
|
||
|
||
let multipartUpload:
|
||
| Extract<DirectUploadInitiateResponse, { mode: "multipart" }>
|
||
| null = null;
|
||
|
||
try {
|
||
const upload = await postDirectUploadAction<DirectUploadInitiateResponse>(
|
||
{
|
||
action: "initiate",
|
||
filename: file.filename,
|
||
contentType: file.contentType,
|
||
size: file.blob.size,
|
||
purpose: file.purpose,
|
||
clientUploadId: file.clientUploadId,
|
||
},
|
||
signal
|
||
);
|
||
|
||
if (upload.mode === "single") {
|
||
const response = await fetchDirectPartWithRetry(
|
||
upload.url,
|
||
{
|
||
method: "PUT",
|
||
headers: upload.headers || {
|
||
"Content-Type": file.contentType,
|
||
},
|
||
body: file.blob,
|
||
},
|
||
maxRetries,
|
||
signal
|
||
);
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`Direct upload failed: ${response.status}`);
|
||
}
|
||
|
||
const payload = await postDirectUploadAction<Record<string, unknown>>(
|
||
{
|
||
action: "complete",
|
||
key: upload.key,
|
||
filename: file.filename,
|
||
contentType: file.contentType,
|
||
size: file.blob.size,
|
||
purpose: file.purpose,
|
||
clientUploadId: file.clientUploadId,
|
||
},
|
||
signal
|
||
);
|
||
|
||
return Response.json(payload);
|
||
}
|
||
|
||
multipartUpload = upload;
|
||
const parts = await uploadMultipartParts({
|
||
file,
|
||
upload,
|
||
maxRetries,
|
||
signal,
|
||
});
|
||
const payload = await postDirectUploadAction<Record<string, unknown>>(
|
||
{
|
||
action: "complete",
|
||
key: upload.key,
|
||
uploadId: upload.uploadId,
|
||
filename: file.filename,
|
||
contentType: file.contentType,
|
||
size: file.blob.size,
|
||
parts,
|
||
purpose: file.purpose,
|
||
clientUploadId: file.clientUploadId,
|
||
},
|
||
signal
|
||
);
|
||
|
||
return Response.json(payload);
|
||
} catch (error) {
|
||
if (multipartUpload && !isAbortError(error, signal)) {
|
||
postDirectUploadAction(
|
||
{
|
||
action: "abort",
|
||
key: multipartUpload.key,
|
||
uploadId: multipartUpload.uploadId,
|
||
clientUploadId: file.clientUploadId,
|
||
},
|
||
signal
|
||
).catch(() => {});
|
||
}
|
||
|
||
if (!isAbortError(error, signal)) {
|
||
if (file.blob.size > SERVER_UPLOAD_MAX_BYTES) {
|
||
return Response.json(
|
||
{
|
||
error:
|
||
"图片超过 50MB，必须使用浏览器直传；当前直传失败，请刷新后重试，或联系管理员检查存储桶 CORS/ETag 配置。",
|
||
details: error instanceof Error ? error.message : String(error),
|
||
clientUploadId: file.clientUploadId,
|
||
},
|
||
{ status: 502 }
|
||
);
|
||
}
|
||
|
||
console.warn("[Upload] Direct upload failed; falling back to server upload", error);
|
||
suppressDirectUploadTemporarily();
|
||
return null;
|
||
}
|
||
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
export async function fetchUploadWithRetry(
|
||
formData: FormData,
|
||
options?: {
|
||
maxRetries?: number;
|
||
signal?: AbortSignal;
|
||
}
|
||
) {
|
||
const maxRetries = options?.maxRetries ?? DEFAULT_MAX_RETRIES;
|
||
|
||
return withUploadSlot(async () => {
|
||
const directResponse = await fetchDirectUpload(
|
||
formData,
|
||
maxRetries,
|
||
options?.signal
|
||
);
|
||
if (directResponse) {
|
||
return directResponse;
|
||
}
|
||
|
||
for (let attempt = 0; ; attempt++) {
|
||
try {
|
||
const response = await fetch("/api/files/upload", {
|
||
method: "POST",
|
||
body: cloneFormData(formData),
|
||
signal: options?.signal,
|
||
});
|
||
|
||
if (!isRetryableUploadStatus(response.status) || attempt >= maxRetries) {
|
||
return response;
|
||
}
|
||
|
||
let retryAfterMs = parseRetryAfterMs(response.headers.get("Retry-After"));
|
||
if (!retryAfterMs) {
|
||
try {
|
||
const payload = await response.clone().json();
|
||
const retryAfterSec = Number(payload?.retryAfter || 0);
|
||
if (Number.isFinite(retryAfterSec) && retryAfterSec > 0) {
|
||
retryAfterMs = Math.round(retryAfterSec * 1000);
|
||
}
|
||
} catch {}
|
||
}
|
||
|
||
const delayMs = computeBackoffMs(attempt, retryAfterMs);
|
||
await sleep(delayMs, options?.signal);
|
||
} catch (error) {
|
||
if (isAbortError(error, options?.signal) || attempt >= maxRetries) {
|
||
throw error;
|
||
}
|
||
|
||
const delayMs = computeBackoffMs(attempt, null);
|
||
await sleep(delayMs, options?.signal);
|
||
}
|
||
}
|
||
});
|
||
}
|