From 607491fc9ec25a6140201704edf37d2fafd4d563 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 18:52:50 +0800 Subject: [PATCH] standardize direct storage uploads --- app/(chat)/api/files/direct-upload/route.ts | 19 +- lib/client/upload-with-retry.ts | 2 +- lib/storage/service.ts | 3 +- .../storage/configure-direct-upload-cors.ts | 147 +++++++++----- .../storage/configure-multipart-lifecycle.ts | 181 ++++++++++++------ 5 files changed, 241 insertions(+), 111 deletions(-) diff --git a/app/(chat)/api/files/direct-upload/route.ts b/app/(chat)/api/files/direct-upload/route.ts index bd96c74ec..dff8785bd 100644 --- a/app/(chat)/api/files/direct-upload/route.ts +++ b/app/(chat)/api/files/direct-upload/route.ts @@ -173,6 +173,7 @@ export async function POST(request: Request) { const client = createS3Client(target.config); const bucket = target.config.bucket; + const usePublicReadAcl = target.config.provider !== "qiniu-kodo"; try { if (payload.action === "initiate") { @@ -207,21 +208,25 @@ export async function POST(request: Request) { Key: key, ContentType: contentType, CacheControl: "public, max-age=31536000, immutable", - ACL: "public-read", + ...(usePublicReadAcl ? { ACL: "public-read" as const } : {}), }); const url = await getSignedUrl(client, command, { expiresIn: PRESIGNED_URL_EXPIRES_SECONDS, }); + const headers: Record = { + "Content-Type": contentType, + "Cache-Control": "public, max-age=31536000, immutable", + }; + if (usePublicReadAcl) { + headers["x-amz-acl"] = "public-read"; + } + return NextResponse.json({ mode: "single", key, url, - headers: { - "Content-Type": contentType, - "Cache-Control": "public, max-age=31536000, immutable", - "x-amz-acl": "public-read", - }, + headers, expiresIn: PRESIGNED_URL_EXPIRES_SECONDS, clientUploadId: payload.clientUploadId || null, }); @@ -233,7 +238,7 @@ export async function POST(request: Request) { Key: key, ContentType: contentType, CacheControl: "public, max-age=31536000, immutable", - ACL: "public-read", + ...(usePublicReadAcl ? { ACL: "public-read" as const } : {}), }) ); diff --git a/lib/client/upload-with-retry.ts b/lib/client/upload-with-retry.ts index 80ded40f2..d64b6ac2c 100644 --- a/lib/client/upload-with-retry.ts +++ b/lib/client/upload-with-retry.ts @@ -1,7 +1,7 @@ const DEFAULT_MAX_RETRIES = 3; const MAX_BACKOFF_MS = 8000; const MAX_CONCURRENT_UPLOADS = 3; -const DIRECT_UPLOAD_MIN_BYTES = 5 * 1024 * 1024; +const DIRECT_UPLOAD_MIN_BYTES = 1; const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024; const SERVER_UPLOAD_MAX_BYTES = 50 * 1024 * 1024; const DIRECT_UPLOAD_PART_CONCURRENCY = 4; diff --git a/lib/storage/service.ts b/lib/storage/service.ts index 6e3b130c9..9780be0ee 100644 --- a/lib/storage/service.ts +++ b/lib/storage/service.ts @@ -252,8 +252,7 @@ async function uploadToS3( Body: Buffer.from(buffer), ContentType: contentType, CacheControl: "public, max-age=31536000, immutable", - // 设置为公开可读（如果您的 MinIO 桶支持） - ACL: "public-read", + ...(config.provider === "qiniu-kodo" ? {} : { ACL: "public-read" as const }), }); await withStorageOperationTimeout( diff --git a/scripts/storage/configure-direct-upload-cors.ts b/scripts/storage/configure-direct-upload-cors.ts index 4d842bdcd..2d88eb173 100644 --- a/scripts/storage/configure-direct-upload-cors.ts +++ b/scripts/storage/configure-direct-upload-cors.ts @@ -6,23 +6,26 @@ import { S3Client, type CORSRule, } from "@aws-sdk/client-s3"; -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; loadEnv({ path: ".env.local" }); loadEnv(); type S3UploadConfig = { + name?: string; endpoint: string; region: string; bucket: string; accessKeyId: string; secretAccessKey: string; + provider?: "s3" | "qiniu-kodo"; }; const DEFAULT_ORIGINS = [ "https://ai.yydd.ai", "https://yydd.ai", "https://bbyy.ai", + "https://yybb.top", "https://goyingup.chat", "http://localhost:3000", "http://localhost:3001", @@ -54,7 +57,7 @@ function createS3Client(config: S3UploadConfig) { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey, }, - forcePathStyle: !isCloudS3, + forcePathStyle: config.provider === "qiniu-kodo" ? true : !isCloudS3, }); } @@ -69,7 +72,7 @@ function toS3Config(value: unknown): S3UploadConfig | null { if ( !raw || - raw.type !== "s3" || + (raw.type !== "s3" && raw.type !== "qiniu-kodo") || raw.isEnabled !== true || !endpoint || !bucket || @@ -80,11 +83,13 @@ function toS3Config(value: unknown): S3UploadConfig | null { } return { + name: raw.name ? String(raw.name) : undefined, endpoint, region: String(raw.region || "auto").trim() || "auto", bucket, accessKeyId, secretAccessKey, + provider: raw.type === "qiniu-kodo" ? "qiniu-kodo" : "s3", }; } @@ -101,6 +106,7 @@ function readEnvS3Config(): S3UploadConfig | null { } return { + name: "Environment S3", endpoint, region: process.env.S3_REGION?.trim() || "auto", bucket, @@ -109,24 +115,31 @@ function readEnvS3Config(): S3UploadConfig | null { }; } -async function resolveS3Config() { +async function resolveS3Configs() { const [{ db }, { storageConfig }] = await Promise.all([ import("@/lib/db"), import("@/lib/db/schema"), ]); - const [defaultConfig] = await db + const storageConfigs = await db .select() .from(storageConfig) .where( and( - eq(storageConfig.type, "s3"), - eq(storageConfig.isDefault, true), + inArray(storageConfig.type, ["s3", "qiniu-kodo"]), eq(storageConfig.isEnabled, true) ) - ) - .limit(1); + ); - return toS3Config(defaultConfig) || readEnvS3Config(); + const configs = storageConfigs + .map(toS3Config) + .filter((config): config is S3UploadConfig => Boolean(config)); + const envConfig = readEnvS3Config(); + + if (configs.length === 0 && envConfig) { + return [envConfig]; + } + + return configs; } function readOrigins() { @@ -159,15 +172,21 @@ function isDirectUploadRule(rule: CORSRule, origins: string[]) { ); } +function isCorsApiUnsupported(error: unknown) { + const value = error as { Code?: string; name?: string; message?: string } | null; + const code = String(value?.Code || value?.name || ""); + const message = String(value?.message || ""); + return code === "NotImplemented" || /not implemented/i.test(message); +} + async function main() { - const config = await resolveS3Config(); - if (!config) { - console.log("[direct-upload-cors] skip: no S3-compatible upload target"); + const configs = await resolveS3Configs(); + if (configs.length === 0) { + console.log("[direct-upload-cors] skip: no S3-compatible upload targets"); return; } const origins = readOrigins(); - const client = createS3Client(config); const directRule: CORSRule = { AllowedOrigins: origins, AllowedMethods: ["GET", "PUT", "POST", "HEAD"], @@ -175,41 +194,77 @@ async function main() { ExposeHeaders: ["ETag", "x-amz-request-id", "x-amz-id-2"], MaxAgeSeconds: 3600, }; + const failures: Array<{ config: S3UploadConfig; error: unknown }> = []; - let existingRules: CORSRule[] = []; - try { - const current = await client.send( - new GetBucketCorsCommand({ Bucket: config.bucket }) - ); - existingRules = current.CORSRules || []; - } catch (error) { - console.warn( - "[direct-upload-cors] existing CORS not found or unreadable, replacing with direct upload rule:", - error instanceof Error ? error.message : String(error) - ); + for (const config of configs) { + const client = createS3Client(config); + let existingRules: CORSRule[] = []; + try { + const current = await client.send( + new GetBucketCorsCommand({ Bucket: config.bucket }) + ); + existingRules = current.CORSRules || []; + } catch (error) { + console.warn( + "[direct-upload-cors] existing CORS not found or unreadable, replacing with direct upload rule:", + { + name: config.name, + bucket: config.bucket, + error: error instanceof Error ? error.message : String(error), + } + ); + } + + const mergedRules = [ + directRule, + ...existingRules.filter((rule) => !isDirectUploadRule(rule, origins)), + ] + .slice(0, 100) + .map(normalizeRule); + + try { + await client.send( + new PutBucketCorsCommand({ + Bucket: config.bucket, + CORSConfiguration: { + CORSRules: mergedRules, + }, + }) + ); + + console.log("[direct-upload-cors] configured", { + name: config.name, + provider: config.provider, + bucket: config.bucket, + origins, + ruleCount: mergedRules.length, + }); + } catch (error) { + if (isCorsApiUnsupported(error)) { + console.warn("[direct-upload-cors] bucket CORS API unsupported; leaving provider/global CORS in place", { + name: config.name, + provider: config.provider, + bucket: config.bucket, + error: error instanceof Error ? error.message : String(error), + }); + continue; + } + + failures.push({ config, error }); + console.error("[direct-upload-cors] configure bucket failed", { + name: config.name, + provider: config.provider, + bucket: config.bucket, + error: error instanceof Error ? error.message : String(error), + }); + } } - const mergedRules = [ - directRule, - ...existingRules.filter((rule) => !isDirectUploadRule(rule, origins)), - ] - .slice(0, 100) - .map(normalizeRule); - - await client.send( - new PutBucketCorsCommand({ - Bucket: config.bucket, - CORSConfiguration: { - CORSRules: mergedRules, - }, - }) - ); - - console.log("[direct-upload-cors] configured", { - bucket: config.bucket, - origins, - ruleCount: mergedRules.length, - }); + if (failures.length > 0) { + throw new Error( + `Failed to configure direct upload CORS for ${failures.length} bucket(s)` + ); + } } main() diff --git a/scripts/storage/configure-multipart-lifecycle.ts b/scripts/storage/configure-multipart-lifecycle.ts index ba8bcb971..7959d1b7e 100644 --- a/scripts/storage/configure-multipart-lifecycle.ts +++ b/scripts/storage/configure-multipart-lifecycle.ts @@ -6,12 +6,13 @@ import { S3Client, type LifecycleRule, } from "@aws-sdk/client-s3"; -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; loadEnv({ path: ".env.local" }); loadEnv(); type S3UploadConfig = { + name?: string; endpoint: string; region: string; bucket: string; @@ -65,7 +66,7 @@ function toS3Config(value: unknown): S3UploadConfig | null { if ( !raw || - raw.type !== "s3" || + (raw.type !== "s3" && raw.type !== "qiniu-kodo") || raw.isEnabled !== true || !endpoint || !bucket || @@ -76,12 +77,13 @@ function toS3Config(value: unknown): S3UploadConfig | null { } return { + name: raw.name ? String(raw.name) : undefined, endpoint, region: String(raw.region || "auto").trim() || "auto", bucket, accessKeyId, secretAccessKey, - provider: raw.provider === "qiniu-kodo" ? "qiniu-kodo" : "s3", + provider: raw.type === "qiniu-kodo" ? "qiniu-kodo" : "s3", }; } @@ -98,6 +100,7 @@ function readEnvS3Config(): S3UploadConfig | null { } return { + name: "Environment S3", endpoint, region: process.env.S3_REGION?.trim() || "auto", bucket, @@ -106,24 +109,31 @@ function readEnvS3Config(): S3UploadConfig | null { }; } -async function resolveS3Config() { +async function resolveS3Configs() { const [{ db }, { storageConfig }] = await Promise.all([ import("@/lib/db"), import("@/lib/db/schema"), ]); - const [defaultConfig] = await db + const storageConfigs = await db .select() .from(storageConfig) .where( and( - eq(storageConfig.type, "s3"), - eq(storageConfig.isDefault, true), + inArray(storageConfig.type, ["s3", "qiniu-kodo"]), eq(storageConfig.isEnabled, true) ) - ) - .limit(1); + ); - return toS3Config(defaultConfig) || readEnvS3Config(); + const configs = storageConfigs + .map(toS3Config) + .filter((config): config is S3UploadConfig => Boolean(config)); + const envConfig = readEnvS3Config(); + + if (configs.length === 0 && envConfig) { + return [envConfig]; + } + + return configs; } function readAbortDays() { @@ -148,62 +158,123 @@ function normalizeRule(rule: LifecycleRule): LifecycleRule { }; } +function isLifecycleApiUnsupported(error: unknown) { + const value = error as { Code?: string; name?: string; message?: string } | null; + const code = String(value?.Code || value?.name || ""); + const message = String(value?.message || ""); + return ( + code === "NotImplemented" || + /not implemented/i.test(message) || + /not well-formed/i.test(message) || + /validate against/i.test(message) + ); +} + async function main() { - const config = await resolveS3Config(); - if (!config) { - console.log("[multipart-lifecycle] skip: no S3-compatible upload target"); + const configs = await resolveS3Configs(); + if (configs.length === 0) { + console.log("[multipart-lifecycle] skip: no S3-compatible upload targets"); return; } - const client = createS3Client(config); const prefix = readRulePrefix(); const abortDays = readAbortDays(); const ruleId = process.env.DIRECT_UPLOAD_LIFECYCLE_RULE_ID || DEFAULT_RULE_ID; + const failures: Array<{ config: S3UploadConfig; error: unknown }> = []; + let configuredCount = 0; - let existingRules: LifecycleRule[] = []; - try { - const current = await client.send( - new GetBucketLifecycleConfigurationCommand({ Bucket: config.bucket }) - ); - existingRules = current.Rules || []; - } catch (error) { - console.warn( - "[multipart-lifecycle] existing lifecycle not found or unreadable, replacing with direct upload cleanup rule:", - error instanceof Error ? error.message : String(error) - ); + for (const config of configs) { + if (config.provider === "qiniu-kodo") { + console.log("[multipart-lifecycle] skip bucket: qiniu-kodo lifecycle is not configured through S3 compatibility", { + name: config.name, + bucket: config.bucket, + }); + continue; + } + + const client = createS3Client(config); + let existingRules: LifecycleRule[] = []; + try { + const current = await client.send( + new GetBucketLifecycleConfigurationCommand({ Bucket: config.bucket }) + ); + existingRules = current.Rules || []; + } catch (error) { + console.warn( + "[multipart-lifecycle] existing lifecycle not found or unreadable, replacing with direct upload cleanup rule:", + { + name: config.name, + bucket: config.bucket, + error: error instanceof Error ? error.message : String(error), + } + ); + } + + const cleanupRule: LifecycleRule = { + ID: ruleId, + Status: "Enabled", + Filter: { Prefix: prefix }, + AbortIncompleteMultipartUpload: { + DaysAfterInitiation: abortDays, + }, + }; + + const mergedRules = [ + cleanupRule, + ...existingRules.filter((rule) => rule.ID !== ruleId), + ] + .slice(0, 1000) + .map(normalizeRule); + + try { + await client.send( + new PutBucketLifecycleConfigurationCommand({ + Bucket: config.bucket, + LifecycleConfiguration: { + Rules: mergedRules, + }, + }) + ); + + configuredCount += 1; + console.log("[multipart-lifecycle] configured", { + name: config.name, + provider: config.provider, + bucket: config.bucket, + prefix, + abortDays, + ruleCount: mergedRules.length, + }); + } catch (error) { + if (isLifecycleApiUnsupported(error)) { + console.warn("[multipart-lifecycle] bucket lifecycle API unsupported; skipping cleanup rule", { + name: config.name, + provider: config.provider, + bucket: config.bucket, + error: error instanceof Error ? error.message : String(error), + }); + continue; + } + + failures.push({ config, error }); + console.error("[multipart-lifecycle] configure bucket failed", { + name: config.name, + provider: config.provider, + bucket: config.bucket, + error: error instanceof Error ? error.message : String(error), + }); + } } - const cleanupRule: LifecycleRule = { - ID: ruleId, - Status: "Enabled", - Filter: { Prefix: prefix }, - AbortIncompleteMultipartUpload: { - DaysAfterInitiation: abortDays, - }, - }; + if (configuredCount === 0) { + console.log("[multipart-lifecycle] no compatible buckets required lifecycle configuration"); + } - const mergedRules = [ - cleanupRule, - ...existingRules.filter((rule) => rule.ID !== ruleId), - ] - .slice(0, 1000) - .map(normalizeRule); - - await client.send( - new PutBucketLifecycleConfigurationCommand({ - Bucket: config.bucket, - LifecycleConfiguration: { - Rules: mergedRules, - }, - }) - ); - - console.log("[multipart-lifecycle] configured", { - bucket: config.bucket, - prefix, - abortDays, - ruleCount: mergedRules.length, - }); + if (failures.length > 0) { + throw new Error( + `Failed to configure multipart lifecycle for ${failures.length} bucket(s)` + ); + } } main()