standardize direct storage uploads

2026-07-08 18:52:50 +08:00 · 2026-07-08 18:52:50 +08:00 · 607491fc9e
commit 607491fc9e
parent fef8947e73
5 changed files with 241 additions and 111 deletions
--- a/app/(chat)/api/files/direct-upload/route.ts
+++ b/app/(chat)/api/files/direct-upload/route.ts
@ -173,6 +173,7 @@ export async function POST(request: Request) {

  const client = createS3Client(target.config);
  const bucket = target.config.bucket;
+  const usePublicReadAcl = target.config.provider !== "qiniu-kodo";

  try {
    if (payload.action === "initiate") {
@ -207,21 +208,25 @@ export async function POST(request: Request) {
          Key: key,
          ContentType: contentType,
          CacheControl: "public, max-age=31536000, immutable",
-          ACL: "public-read",
+          ...(usePublicReadAcl ? { ACL: "public-read" as const } : {}),
        });
        const url = await getSignedUrl(client, command, {
          expiresIn: PRESIGNED_URL_EXPIRES_SECONDS,
        });

+        const headers: Record<string, string> = {
+          "Content-Type": contentType,
+          "Cache-Control": "public, max-age=31536000, immutable",
+        };
+        if (usePublicReadAcl) {
+          headers["x-amz-acl"] = "public-read";
+        }
+
        return NextResponse.json({
          mode: "single",
          key,
          url,
-          headers: {
-            "Content-Type": contentType,
-            "Cache-Control": "public, max-age=31536000, immutable",
-            "x-amz-acl": "public-read",
-          },
+          headers,
          expiresIn: PRESIGNED_URL_EXPIRES_SECONDS,
          clientUploadId: payload.clientUploadId || null,
        });
@ -233,7 +238,7 @@ export async function POST(request: Request) {
          Key: key,
          ContentType: contentType,
          CacheControl: "public, max-age=31536000, immutable",
-          ACL: "public-read",
+          ...(usePublicReadAcl ? { ACL: "public-read" as const } : {}),
        })
      );

--- a/lib/client/upload-with-retry.ts
+++ b/lib/client/upload-with-retry.ts
@ -1,7 +1,7 @@
 const DEFAULT_MAX_RETRIES = 3;
 const MAX_BACKOFF_MS = 8000;
 const MAX_CONCURRENT_UPLOADS = 3;
-const DIRECT_UPLOAD_MIN_BYTES = 5 * 1024 * 1024;
+const DIRECT_UPLOAD_MIN_BYTES = 1;
 const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024;
 const SERVER_UPLOAD_MAX_BYTES = 50 * 1024 * 1024;
 const DIRECT_UPLOAD_PART_CONCURRENCY = 4;
--- a/lib/storage/service.ts
+++ b/lib/storage/service.ts
@ -252,8 +252,7 @@ async function uploadToS3(
      Body: Buffer.from(buffer),
      ContentType: contentType,
      CacheControl: "public, max-age=31536000, immutable",
-      // 设置为公开可读（如果您的 MinIO 桶支持）
-      ACL: "public-read",
+      ...(config.provider === "qiniu-kodo" ? {} : { ACL: "public-read" as const }),
    });

    await withStorageOperationTimeout(
--- a/scripts/storage/configure-direct-upload-cors.ts
+++ b/scripts/storage/configure-direct-upload-cors.ts
@ -6,23 +6,26 @@ import {
  S3Client,
  type CORSRule,
 } from "@aws-sdk/client-s3";
-import { and, eq } from "drizzle-orm";
+import { and, eq, inArray } from "drizzle-orm";

 loadEnv({ path: ".env.local" });
 loadEnv();

 type S3UploadConfig = {
+  name?: string;
  endpoint: string;
  region: string;
  bucket: string;
  accessKeyId: string;
  secretAccessKey: string;
+  provider?: "s3" | "qiniu-kodo";
 };

 const DEFAULT_ORIGINS = [
  "https://ai.yydd.ai",
  "https://yydd.ai",
  "https://bbyy.ai",
+  "https://yybb.top",
  "https://goyingup.chat",
  "http://localhost:3000",
  "http://localhost:3001",
@ -54,7 +57,7 @@ function createS3Client(config: S3UploadConfig) {
      accessKeyId: config.accessKeyId,
      secretAccessKey: config.secretAccessKey,
    },
-    forcePathStyle: !isCloudS3,
+    forcePathStyle: config.provider === "qiniu-kodo" ? true : !isCloudS3,
  });
 }

@ -69,7 +72,7 @@ function toS3Config(value: unknown): S3UploadConfig | null {

  if (
    !raw ||
-    raw.type !== "s3" ||
+    (raw.type !== "s3" && raw.type !== "qiniu-kodo") ||
    raw.isEnabled !== true ||
    !endpoint ||
    !bucket ||
@ -80,11 +83,13 @@ function toS3Config(value: unknown): S3UploadConfig | null {
  }

  return {
+    name: raw.name ? String(raw.name) : undefined,
    endpoint,
    region: String(raw.region || "auto").trim() || "auto",
    bucket,
    accessKeyId,
    secretAccessKey,
+    provider: raw.type === "qiniu-kodo" ? "qiniu-kodo" : "s3",
  };
 }

@ -101,6 +106,7 @@ function readEnvS3Config(): S3UploadConfig | null {
  }

  return {
+    name: "Environment S3",
    endpoint,
    region: process.env.S3_REGION?.trim() || "auto",
    bucket,
@ -109,24 +115,31 @@ function readEnvS3Config(): S3UploadConfig | null {
  };
 }

-async function resolveS3Config() {
+async function resolveS3Configs() {
  const [{ db }, { storageConfig }] = await Promise.all([
    import("@/lib/db"),
    import("@/lib/db/schema"),
  ]);
-  const [defaultConfig] = await db
+  const storageConfigs = await db
    .select()
    .from(storageConfig)
    .where(
      and(
-        eq(storageConfig.type, "s3"),
-        eq(storageConfig.isDefault, true),
+        inArray(storageConfig.type, ["s3", "qiniu-kodo"]),
        eq(storageConfig.isEnabled, true)
      )
-    )
-    .limit(1);
+    );

-  return toS3Config(defaultConfig) || readEnvS3Config();
+  const configs = storageConfigs
+    .map(toS3Config)
+    .filter((config): config is S3UploadConfig => Boolean(config));
+  const envConfig = readEnvS3Config();
+
+  if (configs.length === 0 && envConfig) {
+    return [envConfig];
+  }
+
+  return configs;
 }

 function readOrigins() {
@ -159,15 +172,21 @@ function isDirectUploadRule(rule: CORSRule, origins: string[]) {
  );
 }

+function isCorsApiUnsupported(error: unknown) {
+  const value = error as { Code?: string; name?: string; message?: string } | null;
+  const code = String(value?.Code || value?.name || "");
+  const message = String(value?.message || "");
+  return code === "NotImplemented" || /not implemented/i.test(message);
+}
+
 async function main() {
-  const config = await resolveS3Config();
-  if (!config) {
-    console.log("[direct-upload-cors] skip: no S3-compatible upload target");
+  const configs = await resolveS3Configs();
+  if (configs.length === 0) {
+    console.log("[direct-upload-cors] skip: no S3-compatible upload targets");
    return;
  }

  const origins = readOrigins();
-  const client = createS3Client(config);
  const directRule: CORSRule = {
    AllowedOrigins: origins,
    AllowedMethods: ["GET", "PUT", "POST", "HEAD"],
@ -175,41 +194,77 @@ async function main() {
    ExposeHeaders: ["ETag", "x-amz-request-id", "x-amz-id-2"],
    MaxAgeSeconds: 3600,
  };
+  const failures: Array<{ config: S3UploadConfig; error: unknown }> = [];

-  let existingRules: CORSRule[] = [];
-  try {
-    const current = await client.send(
-      new GetBucketCorsCommand({ Bucket: config.bucket })
-    );
-    existingRules = current.CORSRules || [];
-  } catch (error) {
-    console.warn(
-      "[direct-upload-cors] existing CORS not found or unreadable, replacing with direct upload rule:",
-      error instanceof Error ? error.message : String(error)
-    );
+  for (const config of configs) {
+    const client = createS3Client(config);
+    let existingRules: CORSRule[] = [];
+    try {
+      const current = await client.send(
+        new GetBucketCorsCommand({ Bucket: config.bucket })
+      );
+      existingRules = current.CORSRules || [];
+    } catch (error) {
+      console.warn(
+        "[direct-upload-cors] existing CORS not found or unreadable, replacing with direct upload rule:",
+        {
+          name: config.name,
+          bucket: config.bucket,
+          error: error instanceof Error ? error.message : String(error),
+        }
+      );
+    }
+
+    const mergedRules = [
+      directRule,
+      ...existingRules.filter((rule) => !isDirectUploadRule(rule, origins)),
+    ]
+      .slice(0, 100)
+      .map(normalizeRule);
+
+    try {
+      await client.send(
+        new PutBucketCorsCommand({
+          Bucket: config.bucket,
+          CORSConfiguration: {
+            CORSRules: mergedRules,
+          },
+        })
+      );
+
+      console.log("[direct-upload-cors] configured", {
+        name: config.name,
+        provider: config.provider,
+        bucket: config.bucket,
+        origins,
+        ruleCount: mergedRules.length,
+      });
+    } catch (error) {
+      if (isCorsApiUnsupported(error)) {
+        console.warn("[direct-upload-cors] bucket CORS API unsupported; leaving provider/global CORS in place", {
+          name: config.name,
+          provider: config.provider,
+          bucket: config.bucket,
+          error: error instanceof Error ? error.message : String(error),
+        });
+        continue;
+      }
+
+      failures.push({ config, error });
+      console.error("[direct-upload-cors] configure bucket failed", {
+        name: config.name,
+        provider: config.provider,
+        bucket: config.bucket,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
  }

-  const mergedRules = [
-    directRule,
-    ...existingRules.filter((rule) => !isDirectUploadRule(rule, origins)),
-  ]
-    .slice(0, 100)
-    .map(normalizeRule);
-
-  await client.send(
-    new PutBucketCorsCommand({
-      Bucket: config.bucket,
-      CORSConfiguration: {
-        CORSRules: mergedRules,
-      },
-    })
-  );
-
-  console.log("[direct-upload-cors] configured", {
-    bucket: config.bucket,
-    origins,
-    ruleCount: mergedRules.length,
-  });
+  if (failures.length > 0) {
+    throw new Error(
+      `Failed to configure direct upload CORS for ${failures.length} bucket(s)`
+    );
+  }
 }

 main()
--- a/scripts/storage/configure-multipart-lifecycle.ts
+++ b/scripts/storage/configure-multipart-lifecycle.ts
@ -6,12 +6,13 @@ import {
  S3Client,
  type LifecycleRule,
 } from "@aws-sdk/client-s3";
-import { and, eq } from "drizzle-orm";
+import { and, eq, inArray } from "drizzle-orm";

 loadEnv({ path: ".env.local" });
 loadEnv();

 type S3UploadConfig = {
+  name?: string;
  endpoint: string;
  region: string;
  bucket: string;
@ -65,7 +66,7 @@ function toS3Config(value: unknown): S3UploadConfig | null {

  if (
    !raw ||
-    raw.type !== "s3" ||
+    (raw.type !== "s3" && raw.type !== "qiniu-kodo") ||
    raw.isEnabled !== true ||
    !endpoint ||
    !bucket ||
@ -76,12 +77,13 @@ function toS3Config(value: unknown): S3UploadConfig | null {
  }

  return {
+    name: raw.name ? String(raw.name) : undefined,
    endpoint,
    region: String(raw.region || "auto").trim() || "auto",
    bucket,
    accessKeyId,
    secretAccessKey,
-    provider: raw.provider === "qiniu-kodo" ? "qiniu-kodo" : "s3",
+    provider: raw.type === "qiniu-kodo" ? "qiniu-kodo" : "s3",
  };
 }

@ -98,6 +100,7 @@ function readEnvS3Config(): S3UploadConfig | null {
  }

  return {
+    name: "Environment S3",
    endpoint,
    region: process.env.S3_REGION?.trim() || "auto",
    bucket,
@ -106,24 +109,31 @@ function readEnvS3Config(): S3UploadConfig | null {
  };
 }

-async function resolveS3Config() {
+async function resolveS3Configs() {
  const [{ db }, { storageConfig }] = await Promise.all([
    import("@/lib/db"),
    import("@/lib/db/schema"),
  ]);
-  const [defaultConfig] = await db
+  const storageConfigs = await db
    .select()
    .from(storageConfig)
    .where(
      and(
-        eq(storageConfig.type, "s3"),
-        eq(storageConfig.isDefault, true),
+        inArray(storageConfig.type, ["s3", "qiniu-kodo"]),
        eq(storageConfig.isEnabled, true)
      )
-    )
-    .limit(1);
+    );

-  return toS3Config(defaultConfig) || readEnvS3Config();
+  const configs = storageConfigs
+    .map(toS3Config)
+    .filter((config): config is S3UploadConfig => Boolean(config));
+  const envConfig = readEnvS3Config();
+
+  if (configs.length === 0 && envConfig) {
+    return [envConfig];
+  }
+
+  return configs;
 }

 function readAbortDays() {
@ -148,62 +158,123 @@ function normalizeRule(rule: LifecycleRule): LifecycleRule {
  };
 }

+function isLifecycleApiUnsupported(error: unknown) {
+  const value = error as { Code?: string; name?: string; message?: string } | null;
+  const code = String(value?.Code || value?.name || "");
+  const message = String(value?.message || "");
+  return (
+    code === "NotImplemented" ||
+    /not implemented/i.test(message) ||
+    /not well-formed/i.test(message) ||
+    /validate against/i.test(message)
+  );
+}
+
 async function main() {
-  const config = await resolveS3Config();
-  if (!config) {
-    console.log("[multipart-lifecycle] skip: no S3-compatible upload target");
+  const configs = await resolveS3Configs();
+  if (configs.length === 0) {
+    console.log("[multipart-lifecycle] skip: no S3-compatible upload targets");
    return;
  }

-  const client = createS3Client(config);
  const prefix = readRulePrefix();
  const abortDays = readAbortDays();
  const ruleId = process.env.DIRECT_UPLOAD_LIFECYCLE_RULE_ID || DEFAULT_RULE_ID;
+  const failures: Array<{ config: S3UploadConfig; error: unknown }> = [];
+  let configuredCount = 0;

-  let existingRules: LifecycleRule[] = [];
-  try {
-    const current = await client.send(
-      new GetBucketLifecycleConfigurationCommand({ Bucket: config.bucket })
-    );
-    existingRules = current.Rules || [];
-  } catch (error) {
-    console.warn(
-      "[multipart-lifecycle] existing lifecycle not found or unreadable, replacing with direct upload cleanup rule:",
-      error instanceof Error ? error.message : String(error)
-    );
+  for (const config of configs) {
+    if (config.provider === "qiniu-kodo") {
+      console.log("[multipart-lifecycle] skip bucket: qiniu-kodo lifecycle is not configured through S3 compatibility", {
+        name: config.name,
+        bucket: config.bucket,
+      });
+      continue;
+    }
+
+    const client = createS3Client(config);
+    let existingRules: LifecycleRule[] = [];
+    try {
+      const current = await client.send(
+        new GetBucketLifecycleConfigurationCommand({ Bucket: config.bucket })
+      );
+      existingRules = current.Rules || [];
+    } catch (error) {
+      console.warn(
+        "[multipart-lifecycle] existing lifecycle not found or unreadable, replacing with direct upload cleanup rule:",
+        {
+          name: config.name,
+          bucket: config.bucket,
+          error: error instanceof Error ? error.message : String(error),
+        }
+      );
+    }
+
+    const cleanupRule: LifecycleRule = {
+      ID: ruleId,
+      Status: "Enabled",
+      Filter: { Prefix: prefix },
+      AbortIncompleteMultipartUpload: {
+        DaysAfterInitiation: abortDays,
+      },
+    };
+
+    const mergedRules = [
+      cleanupRule,
+      ...existingRules.filter((rule) => rule.ID !== ruleId),
+    ]
+      .slice(0, 1000)
+      .map(normalizeRule);
+
+    try {
+      await client.send(
+        new PutBucketLifecycleConfigurationCommand({
+          Bucket: config.bucket,
+          LifecycleConfiguration: {
+            Rules: mergedRules,
+          },
+        })
+      );
+
+      configuredCount += 1;
+      console.log("[multipart-lifecycle] configured", {
+        name: config.name,
+        provider: config.provider,
+        bucket: config.bucket,
+        prefix,
+        abortDays,
+        ruleCount: mergedRules.length,
+      });
+    } catch (error) {
+      if (isLifecycleApiUnsupported(error)) {
+        console.warn("[multipart-lifecycle] bucket lifecycle API unsupported; skipping cleanup rule", {
+          name: config.name,
+          provider: config.provider,
+          bucket: config.bucket,
+          error: error instanceof Error ? error.message : String(error),
+        });
+        continue;
+      }
+
+      failures.push({ config, error });
+      console.error("[multipart-lifecycle] configure bucket failed", {
+        name: config.name,
+        provider: config.provider,
+        bucket: config.bucket,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
  }

-  const cleanupRule: LifecycleRule = {
-    ID: ruleId,
-    Status: "Enabled",
-    Filter: { Prefix: prefix },
-    AbortIncompleteMultipartUpload: {
-      DaysAfterInitiation: abortDays,
-    },
-  };
+  if (configuredCount === 0) {
+    console.log("[multipart-lifecycle] no compatible buckets required lifecycle configuration");
+  }

-  const mergedRules = [
-    cleanupRule,
-    ...existingRules.filter((rule) => rule.ID !== ruleId),
-  ]
-    .slice(0, 1000)
-    .map(normalizeRule);
-
-  await client.send(
-    new PutBucketLifecycleConfigurationCommand({
-      Bucket: config.bucket,
-      LifecycleConfiguration: {
-        Rules: mergedRules,
-      },
-    })
-  );
-
-  console.log("[multipart-lifecycle] configured", {
-    bucket: config.bucket,
-    prefix,
-    abortDays,
-    ruleCount: mergedRules.length,
-  });
+  if (failures.length > 0) {
+    throw new Error(
+      `Failed to configure multipart lifecycle for ${failures.length} bucket(s)`
+    );
+  }
 }

 main()