diff --git a/app/(chat)/api/files/direct-upload/route.ts b/app/(chat)/api/files/direct-upload/route.ts new file mode 100644 index 000000000..b83c19a10 --- /dev/null +++ b/app/(chat)/api/files/direct-upload/route.ts @@ -0,0 +1,354 @@ +import { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, + CreateMultipartUploadCommand, + HeadObjectCommand, + PutObjectCommand, + UploadPartCommand, +} from "@aws-sdk/client-s3"; +import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; +import { NextResponse } from "next/server"; +import { z } from "zod"; + +import { auth } from "@/app/(auth)/auth"; +import { upsertMediaAsset } from "@/lib/db/queries"; +import { + buildS3PublicUrl, + buildStorageObjectKey, + createS3Client, + getContentType, + maybeEnqueueStorageMigrationTask, + resolveS3UploadTarget, +} from "@/lib/storage/service"; + +const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024; +const MULTIPART_THRESHOLD_BYTES = 50 * 1024 * 1024; +const DEFAULT_PART_SIZE_BYTES = 8 * 1024 * 1024; +const PRESIGNED_URL_EXPIRES_SECONDS = 15 * 60; +const DIRECT_UPLOAD_PREFIX = "uploads/direct"; + +const allowedDirectUploadTypes = new Set([ + "image/jpeg", + "image/jpg", + "image/png", + "image/webp", +]); + +const InitiateSchema = z.object({ + action: z.literal("initiate"), + filename: z.string().min(1).max(256), + contentType: z.string().min(1).max(128), + size: z.number().int().positive().max(DIRECT_UPLOAD_MAX_BYTES), + purpose: z.string().max(64).optional(), +}); + +const CompleteSchema = z.object({ + action: z.literal("complete"), + key: z.string().min(1).max(512), + uploadId: z.string().min(1).max(512).optional(), + filename: z.string().min(1).max(256), + contentType: z.string().min(1).max(128), + size: z.number().int().positive().max(DIRECT_UPLOAD_MAX_BYTES), + parts: z + .array( + z.object({ + ETag: z.string().min(1), + PartNumber: z.number().int().positive(), + }) + ) + .optional(), + purpose: z.string().max(64).optional(), +}); + +const AbortSchema = z.object({ + action: z.literal("abort"), + key: z.string().min(1).max(512), + uploadId: z.string().min(1).max(512), +}); + +const RequestSchema = z.discriminatedUnion("action", [ + InitiateSchema, + CompleteSchema, + AbortSchema, +]); + +function normalizeContentType(contentType: string, filename: string) { + const trimmed = contentType.trim().toLowerCase(); + return trimmed || getContentType(filename); +} + +function sanitizeDisplayName(filename: string) { + return ( + filename + .trim() + .replace(/[^\u4e00-\u9fa5a-zA-Z0-9._-]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 120) || `upload-${Date.now()}` + ); +} + +function userUploadPrefix(userId: string) { + return `${DIRECT_UPLOAD_PREFIX}/${userId}`; +} + +function assertUserOwnedKey(key: string, userId: string) { + const normalized = String(key || "").replace(/^\/+/, ""); + const prefix = `${userUploadPrefix(userId)}/`; + if (!normalized.startsWith(prefix)) { + throw new Error("Invalid upload object key"); + } + return normalized; +} + +function isAllowedDirectUpload(contentType: string) { + return allowedDirectUploadTypes.has(contentType); +} + +function computePartSize(size: number) { + if (size <= 100 * 1024 * 1024) return DEFAULT_PART_SIZE_BYTES; + if (size <= 300 * 1024 * 1024) return 12 * 1024 * 1024; + return 16 * 1024 * 1024; +} + +async function createPartUrls(params: { + client: ReturnType; + bucket: string; + key: string; + uploadId: string; + partCount: number; +}) { + const urls = []; + for (let partNumber = 1; partNumber <= params.partCount; partNumber += 1) { + const command = new UploadPartCommand({ + Bucket: params.bucket, + Key: params.key, + UploadId: params.uploadId, + PartNumber: partNumber, + }); + urls.push({ + partNumber, + url: await getSignedUrl(params.client, command, { + expiresIn: PRESIGNED_URL_EXPIRES_SECONDS, + }), + }); + } + return urls; +} + +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + let payload: z.infer; + try { + payload = RequestSchema.parse(await request.json()); + } catch (error) { + return NextResponse.json( + { + error: "Invalid direct upload request", + details: error instanceof Error ? error.message : String(error), + }, + { status: 400 } + ); + } + + const target = await resolveS3UploadTarget().catch((error) => { + console.error("[DirectUpload] Failed to resolve storage target:", error); + return null; + }); + + if (!target) { + return NextResponse.json( + { error: "Direct upload is not available for current storage backend" }, + { status: 409 } + ); + } + + const client = createS3Client(target.config); + const bucket = target.config.bucket; + + try { + if (payload.action === "initiate") { + const contentType = normalizeContentType( + payload.contentType, + payload.filename + ); + + if (!isAllowedDirectUpload(contentType)) { + return NextResponse.json( + { error: "Direct upload only supports JPG, PNG and WebP images" }, + { status: 400 } + ); + } + + const displayName = sanitizeDisplayName(payload.filename); + const key = buildStorageObjectKey( + displayName, + userUploadPrefix(session.user.id) + ); + + if (payload.size < MULTIPART_THRESHOLD_BYTES) { + const command = new PutObjectCommand({ + Bucket: bucket, + Key: key, + ContentType: contentType, + ACL: "public-read", + }); + const url = await getSignedUrl(client, command, { + expiresIn: PRESIGNED_URL_EXPIRES_SECONDS, + }); + + return NextResponse.json({ + mode: "single", + key, + url, + headers: { + "Content-Type": contentType, + "x-amz-acl": "public-read", + }, + expiresIn: PRESIGNED_URL_EXPIRES_SECONDS, + }); + } + + const createResult = await client.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ContentType: contentType, + ACL: "public-read", + }) + ); + + const uploadId = createResult.UploadId; + if (!uploadId) { + throw new Error("Storage provider did not return uploadId"); + } + + const partSize = computePartSize(payload.size); + const partCount = Math.ceil(payload.size / partSize); + const parts = await createPartUrls({ + client, + bucket, + key, + uploadId, + partCount, + }); + + return NextResponse.json({ + mode: "multipart", + key, + uploadId, + partSize, + partCount, + parts, + expiresIn: PRESIGNED_URL_EXPIRES_SECONDS, + }); + } + + if (payload.action === "abort") { + const key = assertUserOwnedKey(payload.key, session.user.id); + await client.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: payload.uploadId, + }) + ); + return NextResponse.json({ ok: true }); + } + + const key = assertUserOwnedKey(payload.key, session.user.id); + const contentType = normalizeContentType(payload.contentType, payload.filename); + if (!isAllowedDirectUpload(contentType)) { + return NextResponse.json( + { error: "Direct upload only supports JPG, PNG and WebP images" }, + { status: 400 } + ); + } + + if (payload.uploadId) { + const parts = [...(payload.parts || [])].sort( + (a, b) => a.PartNumber - b.PartNumber + ); + if (parts.length === 0) { + return NextResponse.json( + { error: "Missing multipart upload parts" }, + { status: 400 } + ); + } + + await client.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: payload.uploadId, + MultipartUpload: { + Parts: parts, + }, + }) + ); + } + + const head = await client.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: key, + }) + ); + + const actualSize = + typeof head.ContentLength === "number" ? head.ContentLength : null; + if (actualSize !== null && actualSize <= 0) { + throw new Error("Uploaded object is empty"); + } + + const url = buildS3PublicUrl(target.config, key); + const resolvedContentType = + head.ContentType || contentType || getContentType(payload.filename); + const asset = await upsertMediaAsset({ + userId: session.user.id, + mediaType: "image", + sourceType: "upload", + status: "ready", + originalUrl: url, + previewUrl: url, + thumbnailUrl: url, + mimeType: resolvedContentType, + displayName: payload.filename, + sourceTable: "direct_upload", + metadata: { + pathname: key, + size: actualSize ?? payload.size, + expectedSize: payload.size, + uploadMode: payload.uploadId ? "multipart" : "single", + purpose: payload.purpose || null, + }, + }); + + await maybeEnqueueStorageMigrationTask({ + uploaded: { url, pathname: key }, + storageConfig: target.storageConfig, + migrationConfig: target.migrationConfig, + }).catch((error) => { + console.error("[DirectUpload] Failed to enqueue OSS migration task:", error); + }); + + return NextResponse.json({ + url, + pathname: key, + contentType: resolvedContentType, + assetId: asset?.id ?? null, + }); + } catch (error) { + console.error("[DirectUpload] Request failed:", error); + return NextResponse.json( + { + error: "Direct upload failed", + details: error instanceof Error ? error.message : String(error), + }, + { status: 500 } + ); + } +} diff --git a/lib/client/upload-with-retry.ts b/lib/client/upload-with-retry.ts index 39568a79e..e351065fa 100644 --- a/lib/client/upload-with-retry.ts +++ b/lib/client/upload-with-retry.ts @@ -1,6 +1,10 @@ const DEFAULT_MAX_RETRIES = 3; const MAX_BACKOFF_MS = 8000; const MAX_CONCURRENT_UPLOADS = 3; +const DIRECT_UPLOAD_MIN_BYTES = 5 * 1024 * 1024; +const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024; +const DIRECT_UPLOAD_PART_CONCURRENCY = 4; +const DIRECT_UPLOAD_SUPPRESS_MS = 2 * 60 * 1000; const RETRYABLE_UPLOAD_STATUSES = new Set([ 408, @@ -12,8 +16,44 @@ const RETRYABLE_UPLOAD_STATUSES = new Set([ 504, ]); +const DIRECT_UPLOAD_IMAGE_TYPES = new Set([ + "image/jpeg", + "image/jpg", + "image/png", + "image/webp", +]); + let activeUploadCount = 0; const uploadQueue: Array<() => void> = []; +let directUploadSuppressedUntil = 0; + +type UploadFormFile = { + blob: Blob; + filename: string; + contentType: string; + purpose: string; +}; + +type DirectUploadPart = { + partNumber: number; + url: string; +}; + +type DirectUploadInitiateResponse = + | { + mode: "single"; + key: string; + url: string; + headers?: Record; + } + | { + mode: "multipart"; + key: string; + uploadId: string; + partSize: number; + partCount: number; + parts: DirectUploadPart[]; + }; function createAbortError() { const error = new Error("Upload aborted"); @@ -120,6 +160,263 @@ function cloneFormData(formData: FormData) { return copy; } +function getUploadFormFile(formData: FormData): UploadFormFile | null { + const value = formData.get("file"); + if (!(value instanceof Blob)) { + return null; + } + + const filename = + typeof File !== "undefined" && value instanceof File && value.name + ? value.name + : `upload-${Date.now()}`; + const contentType = value.type || "application/octet-stream"; + const purposeValue = formData.get("purpose"); + + return { + blob: value, + filename, + contentType, + purpose: typeof purposeValue === "string" ? purposeValue : "", + }; +} + +function shouldTryDirectUpload(file: UploadFormFile) { + if (Date.now() < directUploadSuppressedUntil) return false; + if (file.blob.size <= 0 || file.blob.size > DIRECT_UPLOAD_MAX_BYTES) { + return false; + } + if (!DIRECT_UPLOAD_IMAGE_TYPES.has(file.contentType.toLowerCase())) { + return false; + } + if (file.purpose === "agent-workflow-preview") { + return false; + } + + return ( + file.purpose === "canvas-upload" || + file.blob.size >= DIRECT_UPLOAD_MIN_BYTES + ); +} + +function suppressDirectUploadTemporarily() { + directUploadSuppressedUntil = Date.now() + DIRECT_UPLOAD_SUPPRESS_MS; +} + +async function readErrorPayload(response: Response) { + try { + const payload = await response.clone().json(); + return payload?.error || payload?.details || response.statusText; + } catch { + return response.statusText; + } +} + +async function postDirectUploadAction( + payload: Record, + signal?: AbortSignal +): Promise { + const response = await fetch("/api/files/direct-upload", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(payload), + signal, + }); + + if (!response.ok) { + throw new Error(String(await readErrorPayload(response))); + } + + return (await response.json()) as T; +} + +async function fetchDirectPartWithRetry( + url: string, + init: RequestInit, + maxRetries: number, + signal?: AbortSignal +) { + for (let attempt = 0; ; attempt += 1) { + try { + const response = await fetch(url, { + ...init, + signal, + }); + + if (response.ok || !isRetryableUploadStatus(response.status) || attempt >= maxRetries) { + return response; + } + + await sleep(computeBackoffMs(attempt, null), signal); + } catch (error) { + if (isAbortError(error, signal) || attempt >= maxRetries) { + throw error; + } + + await sleep(computeBackoffMs(attempt, null), signal); + } + } +} + +async function uploadMultipartParts(params: { + file: UploadFormFile; + upload: Extract; + maxRetries: number; + signal?: AbortSignal; +}) { + const uploadedParts: Array<{ ETag: string; PartNumber: number }> = []; + let nextIndex = 0; + + async function worker() { + for (;;) { + const part = params.upload.parts[nextIndex]; + nextIndex += 1; + if (!part) return; + + const start = (part.partNumber - 1) * params.upload.partSize; + const end = Math.min(start + params.upload.partSize, params.file.blob.size); + const body = params.file.blob.slice(start, end); + const response = await fetchDirectPartWithRetry( + part.url, + { + method: "PUT", + body, + }, + params.maxRetries, + params.signal + ); + + if (!response.ok) { + throw new Error(`Direct multipart upload failed: ${response.status}`); + } + + const etag = response.headers.get("ETag") || response.headers.get("etag"); + if (!etag) { + throw new Error("Storage did not expose ETag; please allow ETag in bucket CORS"); + } + + uploadedParts.push({ + ETag: etag, + PartNumber: part.partNumber, + }); + } + } + + await Promise.all( + Array.from( + { length: Math.min(DIRECT_UPLOAD_PART_CONCURRENCY, params.upload.parts.length) }, + () => worker() + ) + ); + + return uploadedParts.sort((a, b) => a.PartNumber - b.PartNumber); +} + +async function fetchDirectUpload( + formData: FormData, + maxRetries: number, + signal?: AbortSignal +): Promise { + const file = getUploadFormFile(formData); + if (!file || !shouldTryDirectUpload(file)) { + return null; + } + + let multipartUpload: + | Extract + | null = null; + + try { + const upload = await postDirectUploadAction( + { + action: "initiate", + filename: file.filename, + contentType: file.contentType, + size: file.blob.size, + purpose: file.purpose, + }, + signal + ); + + if (upload.mode === "single") { + const response = await fetchDirectPartWithRetry( + upload.url, + { + method: "PUT", + headers: upload.headers || { + "Content-Type": file.contentType, + }, + body: file.blob, + }, + maxRetries, + signal + ); + + if (!response.ok) { + throw new Error(`Direct upload failed: ${response.status}`); + } + + const payload = await postDirectUploadAction>( + { + action: "complete", + key: upload.key, + filename: file.filename, + contentType: file.contentType, + size: file.blob.size, + purpose: file.purpose, + }, + signal + ); + + return Response.json(payload); + } + + multipartUpload = upload; + const parts = await uploadMultipartParts({ + file, + upload, + maxRetries, + signal, + }); + const payload = await postDirectUploadAction>( + { + action: "complete", + key: upload.key, + uploadId: upload.uploadId, + filename: file.filename, + contentType: file.contentType, + size: file.blob.size, + parts, + purpose: file.purpose, + }, + signal + ); + + return Response.json(payload); + } catch (error) { + if (multipartUpload && !isAbortError(error, signal)) { + postDirectUploadAction( + { + action: "abort", + key: multipartUpload.key, + uploadId: multipartUpload.uploadId, + }, + signal + ).catch(() => {}); + } + + if (!isAbortError(error, signal)) { + console.warn("[Upload] Direct upload failed; falling back to server upload", error); + suppressDirectUploadTemporarily(); + return null; + } + + throw error; + } +} + export async function fetchUploadWithRetry( formData: FormData, options?: { @@ -130,6 +427,15 @@ export async function fetchUploadWithRetry( const maxRetries = options?.maxRetries ?? DEFAULT_MAX_RETRIES; return withUploadSlot(async () => { + const directResponse = await fetchDirectUpload( + formData, + maxRetries, + options?.signal + ); + if (directResponse) { + return directResponse; + } + for (let attempt = 0; ; attempt++) { try { const response = await fetch("/api/files/upload", { diff --git a/lib/storage/service.ts b/lib/storage/service.ts index f11dbfe79..2b8cd5915 100644 --- a/lib/storage/service.ts +++ b/lib/storage/service.ts @@ -63,7 +63,7 @@ function normalizePublicUrl(publicUrl?: string): string | undefined { } // 获取文件的 Content-Type -function getContentType(filename: string): string { +export function getContentType(filename: string): string { const ext = filename.toLowerCase().split(".").pop(); const types: Record = { jpg: "image/jpeg", @@ -100,7 +100,7 @@ function getContentType(filename: string): string { return types[ext ?? ""] ?? "application/octet-stream"; } -function createS3Client(config: S3UploadConfig): S3Client { +export function createS3Client(config: S3UploadConfig): S3Client { const normalizedEndpoint = normalizeEndpoint(config.endpoint); const lowerEndpoint = normalizedEndpoint.toLowerCase(); const isCloudS3 = @@ -118,6 +118,75 @@ function createS3Client(config: S3UploadConfig): S3Client { }); } +export function buildStorageObjectKey(filename: string, prefix = "uploads"): string { + const timestamp = Date.now(); + const randomString = Math.random().toString(36).slice(2, 8); + const cleanExt = String(filename.split(".").pop() || "bin") + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, "") + .slice(0, 16) || "bin"; + const cleanPrefix = prefix.replace(/^\/+|\/+$/g, "") || "uploads"; + return `${cleanPrefix}/${timestamp}-${randomString}.${cleanExt}`; +} + +export function buildS3PublicUrl(config: S3UploadConfig, key: string): string { + const normalizedEndpoint = normalizeEndpoint(config.endpoint); + const normalizedPublicUrl = normalizePublicUrl(config.publicUrl); + const cleanKey = key.replace(/^\/+/, ""); + + if (normalizedPublicUrl) { + return `${normalizedPublicUrl}/${cleanKey}`; + } + + return `${normalizedEndpoint}/${config.bucket}/${cleanKey}`; +} + +export async function resolveS3UploadTarget(): Promise< + | { + config: S3UploadConfig; + storageConfig?: StorageConfig; + migrationConfig: OssImageMigrationConfig; + } + | null +> { + let migrationConfig = defaultOssImageMigrationConfig; + const migrationSetting = await getSystemSetting({ + key: OSS_MIGRATION_CONFIG_KEY, + }).catch(() => undefined); + migrationConfig = normalizeOssImageMigrationConfig(migrationSetting?.value); + + const selectedStorageConfig = await resolvePreferredUploadStorageConfig( + migrationConfig + ); + const dbS3Config = toDbS3UploadConfig(selectedStorageConfig); + if (dbS3Config) { + return { + config: dbS3Config, + storageConfig: selectedStorageConfig, + migrationConfig, + }; + } + + const envS3Config = buildEnvS3Config(); + if (envS3Config) { + return { + config: envS3Config, + migrationConfig, + }; + } + + return null; +} + +export async function maybeEnqueueStorageMigrationTask(params: { + uploaded: { url: string; pathname: string }; + storageConfig?: StorageConfig; + migrationConfig: OssImageMigrationConfig; +}) { + await maybeEnqueueAliyunMigrationTask(params); +} + // S3/MinIO 上传函数(使用 AWS SDK) async function uploadToS3( filename: string, @@ -134,10 +203,7 @@ async function uploadToS3( const normalizedPublicUrl = normalizePublicUrl(publicUrl); // 生成唯一的文件名 - const timestamp = Date.now(); - const randomString = Math.random().toString(36).slice(2, 8); - const ext = filename.split(".").pop(); - const key = `uploads/${timestamp}-${randomString}.${ext}`; + const key = buildStorageObjectKey(filename); const contentType = getContentType(filename); @@ -160,14 +226,10 @@ async function uploadToS3( // 返回文件的公开 URL // MinIO 的公开 URL 格式通常是: http://endpoint:port/bucket/key - let fileUrl: string; - if (normalizedPublicUrl) { - // 如果用户配置了自定义公开 URL,使用它 - fileUrl = `${normalizedPublicUrl}/${key}`; - } else { - // 否则使用 endpoint + bucket + key - fileUrl = `${normalizedEndpoint}/${bucket}/${key}`; - } + const fileUrl = buildS3PublicUrl( + { ...config, endpoint: normalizedEndpoint, publicUrl: normalizedPublicUrl }, + key + ); console.log(`[S3 Upload] File URL: ${fileUrl}`); diff --git a/package.json b/package.json index 6f4653c5f..08e855f6d 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "@ai-sdk/react": "2.0.26", "@ai-sdk/xai": "2.0.13", "@aws-sdk/client-s3": "^3.679.0", + "@aws-sdk/s3-request-presigner": "^3.958.0", "@codemirror/lang-javascript": "^6.2.2", "@codemirror/lang-python": "^6.1.6", "@codemirror/state": "^6.5.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 907377271..87baef3c0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,6 +29,9 @@ importers: '@aws-sdk/client-s3': specifier: ^3.679.0 version: 3.958.0 + '@aws-sdk/s3-request-presigner': + specifier: ^3.958.0 + version: 3.958.0 '@codemirror/lang-javascript': specifier: ^6.2.2 version: 6.2.3 @@ -653,6 +656,10 @@ packages: resolution: {integrity: sha512-V8iY3blh8l2iaOqXWW88HbkY5jDoWjH56jonprG/cpyqqCnprvpMUZWPWYJoI8rHRf2bqzZeql1slxG6EnKI7A==} engines: {node: '>=18.0.0'} + '@aws-sdk/s3-request-presigner@3.958.0': + resolution: {integrity: sha512-bFKsofead/fl3lyhdES+aNo+MZ+qv1ixSPSsF8O1oj6/KgGE0t1UH9AHw2vPq6iSQMTeEuyV0F5pC+Ns40kBgA==} + engines: {node: '>=18.0.0'} + '@aws-sdk/signature-v4-multi-region@3.957.0': resolution: {integrity: sha512-t6UfP1xMUigMMzHcb7vaZcjv7dA2DQkk9C/OAP1dKyrE0vb4lFGDaTApi17GN6Km9zFxJthEMUbBc7DL0hq1Bg==} engines: {node: '>=18.0.0'} @@ -665,6 +672,10 @@ packages: resolution: {integrity: sha512-wzWC2Nrt859ABk6UCAVY/WYEbAd7FjkdrQL6m24+tfmWYDNRByTJ9uOgU/kw9zqLCAwb//CPvrJdhqjTznWXAg==} engines: {node: '>=18.0.0'} + '@aws-sdk/types@3.973.9': + resolution: {integrity: sha512-kuBfgQVdcz5Bmapc4A13YbpVw/pXkesfhetcFYwbntqas8sF41OHyd4o28+/TG2ZQdHBsv90Lsu5y6oitvYCdg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/util-arn-parser@3.957.0': resolution: {integrity: sha512-Aj6m+AyrhWyg8YQ4LDPg2/gIfGHCEcoQdBt5DeSFogN5k9mmJPOJ+IAmNSWmWRjpOxEy6eY813RNDI6qS97M0g==} engines: {node: '>=18.0.0'} @@ -673,6 +684,10 @@ packages: resolution: {integrity: sha512-xwF9K24mZSxcxKS3UKQFeX/dPYkEps9wF1b+MGON7EvnbcucrJGyQyK1v1xFPn1aqXkBTFi+SZaMRx5E5YCVFw==} engines: {node: '>=18.0.0'} + '@aws-sdk/util-format-url@3.957.0': + resolution: {integrity: sha512-Yyo/tlc0iGFGTPPkuxub1uRAv6XrnVnvSNjslZh5jIYA8GZoeEFPgJa3Qdu0GUS/YwoK8GOLnnaL9h/eH5LDJQ==} + engines: {node: '>=18.0.0'} + '@aws-sdk/util-locate-window@3.957.0': resolution: {integrity: sha512-nhmgKHnNV9K+i9daumaIz8JTLsIIML9PE/HUks5liyrjUzenjW/aHoc7WJ9/Td/gPZtayxFnXQSJRb/fDlBuJw==} engines: {node: '>=18.0.0'} @@ -2781,6 +2796,10 @@ packages: resolution: {integrity: sha512-WsSHCPq/neD5G/MkK4csLI5Y5Pkd9c1NMfpYEKeghSGaD4Ja1qLIohRQf2D5c1Uy5aXp76DeKHkzWZ9KAlHroQ==} engines: {node: '>=18.0.0'} + '@smithy/core@3.24.4': + resolution: {integrity: sha512-3UNRKEyQyAgVgM0LGlerCLm+ChZWZ1GPfde+jBEW6bm6bSBGU1p0EbblaUV3unbhwvidjLA5Zs3sOs7mnZwvAw==} + engines: {node: '>=18.0.0'} + '@smithy/credential-provider-imds@4.2.7': resolution: {integrity: sha512-CmduWdCiILCRNbQWFR0OcZlUPVtyE49Sr8yYL0rZQ4D/wKxiNzBNS/YHemvnbkIWj623fplgkexUd/c9CAKdoA==} engines: {node: '>=18.0.0'} @@ -2893,6 +2912,10 @@ packages: resolution: {integrity: sha512-9oNUlqBlFZFOSdxgImA6X5GFuzE7V2H7VG/7E70cdLhidFbdtvxxt81EHgykGK5vq5D3FafH//X+Oy31j3CKOg==} engines: {node: '>=18.0.0'} + '@smithy/signature-v4@5.4.4': + resolution: {integrity: sha512-e5UtkMvsatzBfbeBZjEOt0k0Z3BEsjTFL/n6fdO5vtBLe67tdy0dX7xw2DU7uZ3acwoHyeCqpU2Fzb7pxwHb6Q==} + engines: {node: '>=18.0.0'} + '@smithy/smithy-client@4.10.2': resolution: {integrity: sha512-D5z79xQWpgrGpAHb054Fn2CCTQZpog7JELbVQ6XAvXs5MNKWf28U9gzSBlJkOyMl9LA1TZEjRtwvGXfP0Sl90g==} engines: {node: '>=18.0.0'} @@ -2901,6 +2924,10 @@ packages: resolution: {integrity: sha512-mlrmL0DRDVe3mNrjTcVcZEgkFmufITfUAPBEA+AHYiIeYyJebso/He1qLbP3PssRe22KUzLRpQSdBPbXdgZ2VA==} engines: {node: '>=18.0.0'} + '@smithy/types@4.14.2': + resolution: {integrity: sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw==} + engines: {node: '>=18.0.0'} + '@smithy/url-parser@4.2.7': resolution: {integrity: sha512-/RLtVsRV4uY3qPWhBDsjwahAtt3x2IsMGnP5W1b2VZIe+qgCqkLxI1UOHDZp1Q1QSOrdOR32MF3Ph2JfWT1VHg==} engines: {node: '>=18.0.0'} @@ -3432,6 +3459,7 @@ packages: '@vercel/postgres@0.10.0': resolution: {integrity: sha512-fSD23DxGND40IzSkXjcFcxr53t3Tiym59Is0jSYIFpG4/0f0KO9SGtcp1sXiebvPaGe7N/tU05cH4yt2S6/IPg==} engines: {node: '>=18.14'} + deprecated: '@vercel/postgres is deprecated. If you are setting up a new database, you can choose an alternate storage solution from the Vercel Marketplace. If you had an existing Vercel Postgres database, it should have been migrated to Neon as a native Vercel integration. You can find more details and the guide to migrate to Neon''s SDKs here: https://neon.com/docs/guides/vercel-postgres-transition-guide' '@vitest/expect@3.2.4': resolution: {integrity: sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig==} @@ -6653,13 +6681,13 @@ snapshots: '@aws-crypto/crc32@5.2.0': dependencies: '@aws-crypto/util': 5.2.0 - '@aws-sdk/types': 3.957.0 + '@aws-sdk/types': 3.973.9 tslib: 2.8.1 '@aws-crypto/crc32c@5.2.0': dependencies: '@aws-crypto/util': 5.2.0 - '@aws-sdk/types': 3.957.0 + '@aws-sdk/types': 3.973.9 tslib: 2.8.1 '@aws-crypto/sha1-browser@5.2.0': @@ -6693,7 +6721,7 @@ snapshots: '@aws-crypto/util@5.2.0': dependencies: - '@aws-sdk/types': 3.957.0 + '@aws-sdk/types': 3.973.9 '@smithy/util-utf8': 2.3.0 tslib: 2.8.1 @@ -6785,7 +6813,7 @@ snapshots: '@smithy/node-http-handler': 4.4.7 '@smithy/protocol-http': 5.3.7 '@smithy/smithy-client': 4.10.2 - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 '@smithy/url-parser': 4.2.7 '@smithy/util-base64': 4.3.0 '@smithy/util-body-length-browser': 4.2.0 @@ -6818,7 +6846,7 @@ snapshots: '@aws-sdk/crc64-nvme@3.957.0': dependencies: - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 tslib: 2.8.1 '@aws-sdk/credential-provider-env@3.957.0': @@ -6826,7 +6854,7 @@ snapshots: '@aws-sdk/core': 3.957.0 '@aws-sdk/types': 3.957.0 '@smithy/property-provider': 4.2.7 - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 tslib: 2.8.1 '@aws-sdk/credential-provider-http@3.957.0': @@ -6838,7 +6866,7 @@ snapshots: '@smithy/property-provider': 4.2.7 '@smithy/protocol-http': 5.3.7 '@smithy/smithy-client': 4.10.2 - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 '@smithy/util-stream': 4.5.8 tslib: 2.8.1 @@ -6856,7 +6884,7 @@ snapshots: '@smithy/credential-provider-imds': 4.2.7 '@smithy/property-provider': 4.2.7 '@smithy/shared-ini-file-loader': 4.4.2 - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 tslib: 2.8.1 transitivePeerDependencies: - aws-crt @@ -6869,7 +6897,7 @@ snapshots: '@smithy/property-provider': 4.2.7 '@smithy/protocol-http': 5.3.7 '@smithy/shared-ini-file-loader': 4.4.2 - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 tslib: 2.8.1 transitivePeerDependencies: - aws-crt @@ -6897,7 +6925,7 @@ snapshots: '@aws-sdk/types': 3.957.0 '@smithy/property-provider': 4.2.7 '@smithy/shared-ini-file-loader': 4.4.2 - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 tslib: 2.8.1 '@aws-sdk/credential-provider-sso@3.958.0': @@ -6908,7 +6936,7 @@ snapshots: '@aws-sdk/types': 3.957.0 '@smithy/property-provider': 4.2.7 '@smithy/shared-ini-file-loader': 4.4.2 - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 tslib: 2.8.1 transitivePeerDependencies: - aws-crt @@ -6920,7 +6948,7 @@ snapshots: '@aws-sdk/types': 3.957.0 '@smithy/property-provider': 4.2.7 '@smithy/shared-ini-file-loader': 4.4.2 - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 tslib: 2.8.1 transitivePeerDependencies: - aws-crt @@ -7047,7 +7075,7 @@ snapshots: '@smithy/node-http-handler': 4.4.7 '@smithy/protocol-http': 5.3.7 '@smithy/smithy-client': 4.10.2 - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 '@smithy/url-parser': 4.2.7 '@smithy/util-base64': 4.3.0 '@smithy/util-body-length-browser': 4.2.0 @@ -7070,13 +7098,24 @@ snapshots: '@smithy/types': 4.11.0 tslib: 2.8.1 + '@aws-sdk/s3-request-presigner@3.958.0': + dependencies: + '@aws-sdk/signature-v4-multi-region': 3.957.0 + '@aws-sdk/types': 3.957.0 + '@aws-sdk/util-format-url': 3.957.0 + '@smithy/middleware-endpoint': 4.4.1 + '@smithy/protocol-http': 5.3.7 + '@smithy/smithy-client': 4.10.2 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + '@aws-sdk/signature-v4-multi-region@3.957.0': dependencies: '@aws-sdk/middleware-sdk-s3': 3.957.0 '@aws-sdk/types': 3.957.0 '@smithy/protocol-http': 5.3.7 - '@smithy/signature-v4': 5.3.7 - '@smithy/types': 4.11.0 + '@smithy/signature-v4': 5.4.4 + '@smithy/types': 4.14.2 tslib: 2.8.1 '@aws-sdk/token-providers@3.958.0': @@ -7086,14 +7125,19 @@ snapshots: '@aws-sdk/types': 3.957.0 '@smithy/property-provider': 4.2.7 '@smithy/shared-ini-file-loader': 4.4.2 - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 tslib: 2.8.1 transitivePeerDependencies: - aws-crt '@aws-sdk/types@3.957.0': dependencies: - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + + '@aws-sdk/types@3.973.9': + dependencies: + '@smithy/types': 4.14.2 tslib: 2.8.1 '@aws-sdk/util-arn-parser@3.957.0': @@ -7108,6 +7152,13 @@ snapshots: '@smithy/util-endpoints': 3.2.7 tslib: 2.8.1 + '@aws-sdk/util-format-url@3.957.0': + dependencies: + '@aws-sdk/types': 3.957.0 + '@smithy/querystring-builder': 4.2.7 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + '@aws-sdk/util-locate-window@3.957.0': dependencies: tslib: 2.8.1 @@ -7129,7 +7180,7 @@ snapshots: '@aws-sdk/xml-builder@3.957.0': dependencies: - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 fast-xml-parser: 5.2.5 tslib: 2.8.1 @@ -9090,7 +9141,7 @@ snapshots: '@smithy/abort-controller@4.2.7': dependencies: - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 tslib: 2.8.1 '@smithy/chunked-blob-reader-native@4.2.1': @@ -9124,18 +9175,24 @@ snapshots: '@smithy/uuid': 1.1.0 tslib: 2.8.1 + '@smithy/core@3.24.4': + dependencies: + '@aws-crypto/crc32': 5.2.0 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + '@smithy/credential-provider-imds@4.2.7': dependencies: '@smithy/node-config-provider': 4.3.7 '@smithy/property-provider': 4.2.7 - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 '@smithy/url-parser': 4.2.7 tslib: 2.8.1 '@smithy/eventstream-codec@4.2.7': dependencies: '@aws-crypto/crc32': 5.2.0 - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 '@smithy/util-hex-encoding': 4.2.0 tslib: 2.8.1 @@ -9159,7 +9216,7 @@ snapshots: '@smithy/eventstream-serde-universal@4.2.7': dependencies: '@smithy/eventstream-codec': 4.2.7 - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 tslib: 2.8.1 '@smithy/fetch-http-handler@5.3.8': @@ -9266,7 +9323,7 @@ snapshots: '@smithy/property-provider@4.2.7': dependencies: - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 tslib: 2.8.1 '@smithy/protocol-http@5.3.7': @@ -9276,35 +9333,41 @@ snapshots: '@smithy/querystring-builder@4.2.7': dependencies: - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 '@smithy/util-uri-escape': 4.2.0 tslib: 2.8.1 '@smithy/querystring-parser@4.2.7': dependencies: - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 tslib: 2.8.1 '@smithy/service-error-classification@4.2.7': dependencies: - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 '@smithy/shared-ini-file-loader@4.4.2': dependencies: - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 tslib: 2.8.1 '@smithy/signature-v4@5.3.7': dependencies: '@smithy/is-array-buffer': 4.2.0 '@smithy/protocol-http': 5.3.7 - '@smithy/types': 4.11.0 + '@smithy/types': 4.14.2 '@smithy/util-hex-encoding': 4.2.0 '@smithy/util-middleware': 4.2.7 '@smithy/util-uri-escape': 4.2.0 '@smithy/util-utf8': 4.2.0 tslib: 2.8.1 + '@smithy/signature-v4@5.4.4': + dependencies: + '@smithy/core': 3.24.4 + '@smithy/types': 4.14.2 + tslib: 2.8.1 + '@smithy/smithy-client@4.10.2': dependencies: '@smithy/core': 3.20.0 @@ -9319,6 +9382,10 @@ snapshots: dependencies: tslib: 2.8.1 + '@smithy/types@4.14.2': + dependencies: + tslib: 2.8.1 + '@smithy/url-parser@4.2.7': dependencies: '@smithy/querystring-parser': 4.2.7 diff --git a/proxy.ts b/proxy.ts index b0ec249ce..87ad4660b 100644 --- a/proxy.ts +++ b/proxy.ts @@ -107,7 +107,11 @@ export async function proxy(request: NextRequest) { { status: 429, headers: { "Retry-After": String(retryAfter) } } ); } - } else if (pathname === "/api/files/upload" && request.method === "POST") { + } else if ( + (pathname === "/api/files/upload" || + pathname === "/api/files/direct-upload") && + request.method === "POST" + ) { const token = await resolveToken(); const userId = typeof token?.sub === "string" && token.sub.length > 0 diff --git a/scripts/storage/configure-direct-upload-cors.ts b/scripts/storage/configure-direct-upload-cors.ts new file mode 100644 index 000000000..37217879f --- /dev/null +++ b/scripts/storage/configure-direct-upload-cors.ts @@ -0,0 +1,108 @@ +import "dotenv/config"; + +import { + GetBucketCorsCommand, + PutBucketCorsCommand, + type CORSRule, +} from "@aws-sdk/client-s3"; + +import { + createS3Client, + resolveS3UploadTarget, +} from "@/lib/storage/service"; + +const DEFAULT_ORIGINS = [ + "https://ai.yydd.ai", + "https://yydd.ai", + "https://bbyy.ai", + "https://goyingup.chat", + "http://localhost:3000", + "http://localhost:3001", +]; + +function readOrigins() { + const raw = process.env.DIRECT_UPLOAD_CORS_ORIGINS || ""; + const origins = raw + .split(",") + .map((origin) => origin.trim()) + .filter(Boolean); + + return origins.length > 0 ? origins : DEFAULT_ORIGINS; +} + +function normalizeRule(rule: CORSRule): CORSRule { + return { + AllowedHeaders: Array.from(new Set(rule.AllowedHeaders || [])), + AllowedMethods: Array.from(new Set(rule.AllowedMethods || [])), + AllowedOrigins: Array.from(new Set(rule.AllowedOrigins || [])), + ExposeHeaders: Array.from(new Set(rule.ExposeHeaders || [])), + MaxAgeSeconds: rule.MaxAgeSeconds, + }; +} + +function isDirectUploadRule(rule: CORSRule, origins: string[]) { + const methods = new Set(rule.AllowedMethods || []); + const allowedOrigins = new Set(rule.AllowedOrigins || []); + return ( + methods.has("PUT") && + methods.has("POST") && + origins.some((origin) => allowedOrigins.has(origin)) + ); +} + +async function main() { + const target = await resolveS3UploadTarget(); + if (!target) { + console.log("[direct-upload-cors] skip: no S3-compatible upload target"); + return; + } + + const origins = readOrigins(); + const client = createS3Client(target.config); + const bucket = target.config.bucket; + const directRule: CORSRule = { + AllowedOrigins: origins, + AllowedMethods: ["GET", "PUT", "POST", "HEAD"], + AllowedHeaders: ["*"], + ExposeHeaders: ["ETag", "x-amz-request-id", "x-amz-id-2"], + MaxAgeSeconds: 3600, + }; + + let existingRules: CORSRule[] = []; + try { + const current = await client.send(new GetBucketCorsCommand({ Bucket: bucket })); + existingRules = current.CORSRules || []; + } catch (error) { + console.warn( + "[direct-upload-cors] existing CORS not found or unreadable, replacing with direct upload rule:", + error instanceof Error ? error.message : String(error) + ); + } + + const mergedRules = [ + directRule, + ...existingRules.filter((rule) => !isDirectUploadRule(rule, origins)), + ] + .slice(0, 100) + .map(normalizeRule); + + await client.send( + new PutBucketCorsCommand({ + Bucket: bucket, + CORSConfiguration: { + CORSRules: mergedRules, + }, + }) + ); + + console.log("[direct-upload-cors] configured", { + bucket, + origins, + ruleCount: mergedRules.length, + }); +} + +main().catch((error) => { + console.error("[direct-upload-cors] failed:", error); + process.exitCode = 1; +});