standardize direct storage uploads

This commit is contained in:
134 2026-07-08 18:52:50 +08:00
parent fef8947e73
commit 607491fc9e
5 changed files with 241 additions and 111 deletions

View File

@ -173,6 +173,7 @@ export async function POST(request: Request) {
const client = createS3Client(target.config); const client = createS3Client(target.config);
const bucket = target.config.bucket; const bucket = target.config.bucket;
const usePublicReadAcl = target.config.provider !== "qiniu-kodo";
try { try {
if (payload.action === "initiate") { if (payload.action === "initiate") {
@ -207,21 +208,25 @@ export async function POST(request: Request) {
Key: key, Key: key,
ContentType: contentType, ContentType: contentType,
CacheControl: "public, max-age=31536000, immutable", CacheControl: "public, max-age=31536000, immutable",
ACL: "public-read", ...(usePublicReadAcl ? { ACL: "public-read" as const } : {}),
}); });
const url = await getSignedUrl(client, command, { const url = await getSignedUrl(client, command, {
expiresIn: PRESIGNED_URL_EXPIRES_SECONDS, expiresIn: PRESIGNED_URL_EXPIRES_SECONDS,
}); });
const headers: Record<string, string> = {
"Content-Type": contentType,
"Cache-Control": "public, max-age=31536000, immutable",
};
if (usePublicReadAcl) {
headers["x-amz-acl"] = "public-read";
}
return NextResponse.json({ return NextResponse.json({
mode: "single", mode: "single",
key, key,
url, url,
headers: { headers,
"Content-Type": contentType,
"Cache-Control": "public, max-age=31536000, immutable",
"x-amz-acl": "public-read",
},
expiresIn: PRESIGNED_URL_EXPIRES_SECONDS, expiresIn: PRESIGNED_URL_EXPIRES_SECONDS,
clientUploadId: payload.clientUploadId || null, clientUploadId: payload.clientUploadId || null,
}); });
@ -233,7 +238,7 @@ export async function POST(request: Request) {
Key: key, Key: key,
ContentType: contentType, ContentType: contentType,
CacheControl: "public, max-age=31536000, immutable", CacheControl: "public, max-age=31536000, immutable",
ACL: "public-read", ...(usePublicReadAcl ? { ACL: "public-read" as const } : {}),
}) })
); );

View File

@ -1,7 +1,7 @@
const DEFAULT_MAX_RETRIES = 3; const DEFAULT_MAX_RETRIES = 3;
const MAX_BACKOFF_MS = 8000; const MAX_BACKOFF_MS = 8000;
const MAX_CONCURRENT_UPLOADS = 3; const MAX_CONCURRENT_UPLOADS = 3;
const DIRECT_UPLOAD_MIN_BYTES = 5 * 1024 * 1024; const DIRECT_UPLOAD_MIN_BYTES = 1;
const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024; const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024;
const SERVER_UPLOAD_MAX_BYTES = 50 * 1024 * 1024; const SERVER_UPLOAD_MAX_BYTES = 50 * 1024 * 1024;
const DIRECT_UPLOAD_PART_CONCURRENCY = 4; const DIRECT_UPLOAD_PART_CONCURRENCY = 4;

View File

@ -252,8 +252,7 @@ async function uploadToS3(
Body: Buffer.from(buffer), Body: Buffer.from(buffer),
ContentType: contentType, ContentType: contentType,
CacheControl: "public, max-age=31536000, immutable", CacheControl: "public, max-age=31536000, immutable",
// 设置为公开可读(如果您的 MinIO 桶支持) ...(config.provider === "qiniu-kodo" ? {} : { ACL: "public-read" as const }),
ACL: "public-read",
}); });
await withStorageOperationTimeout( await withStorageOperationTimeout(

View File

@ -6,23 +6,26 @@ import {
S3Client, S3Client,
type CORSRule, type CORSRule,
} from "@aws-sdk/client-s3"; } from "@aws-sdk/client-s3";
import { and, eq } from "drizzle-orm"; import { and, eq, inArray } from "drizzle-orm";
loadEnv({ path: ".env.local" }); loadEnv({ path: ".env.local" });
loadEnv(); loadEnv();
type S3UploadConfig = { type S3UploadConfig = {
name?: string;
endpoint: string; endpoint: string;
region: string; region: string;
bucket: string; bucket: string;
accessKeyId: string; accessKeyId: string;
secretAccessKey: string; secretAccessKey: string;
provider?: "s3" | "qiniu-kodo";
}; };
const DEFAULT_ORIGINS = [ const DEFAULT_ORIGINS = [
"https://ai.yydd.ai", "https://ai.yydd.ai",
"https://yydd.ai", "https://yydd.ai",
"https://bbyy.ai", "https://bbyy.ai",
"https://yybb.top",
"https://goyingup.chat", "https://goyingup.chat",
"http://localhost:3000", "http://localhost:3000",
"http://localhost:3001", "http://localhost:3001",
@ -54,7 +57,7 @@ function createS3Client(config: S3UploadConfig) {
accessKeyId: config.accessKeyId, accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey, secretAccessKey: config.secretAccessKey,
}, },
forcePathStyle: !isCloudS3, forcePathStyle: config.provider === "qiniu-kodo" ? true : !isCloudS3,
}); });
} }
@ -69,7 +72,7 @@ function toS3Config(value: unknown): S3UploadConfig | null {
if ( if (
!raw || !raw ||
raw.type !== "s3" || (raw.type !== "s3" && raw.type !== "qiniu-kodo") ||
raw.isEnabled !== true || raw.isEnabled !== true ||
!endpoint || !endpoint ||
!bucket || !bucket ||
@ -80,11 +83,13 @@ function toS3Config(value: unknown): S3UploadConfig | null {
} }
return { return {
name: raw.name ? String(raw.name) : undefined,
endpoint, endpoint,
region: String(raw.region || "auto").trim() || "auto", region: String(raw.region || "auto").trim() || "auto",
bucket, bucket,
accessKeyId, accessKeyId,
secretAccessKey, secretAccessKey,
provider: raw.type === "qiniu-kodo" ? "qiniu-kodo" : "s3",
}; };
} }
@ -101,6 +106,7 @@ function readEnvS3Config(): S3UploadConfig | null {
} }
return { return {
name: "Environment S3",
endpoint, endpoint,
region: process.env.S3_REGION?.trim() || "auto", region: process.env.S3_REGION?.trim() || "auto",
bucket, bucket,
@ -109,24 +115,31 @@ function readEnvS3Config(): S3UploadConfig | null {
}; };
} }
async function resolveS3Config() { async function resolveS3Configs() {
const [{ db }, { storageConfig }] = await Promise.all([ const [{ db }, { storageConfig }] = await Promise.all([
import("@/lib/db"), import("@/lib/db"),
import("@/lib/db/schema"), import("@/lib/db/schema"),
]); ]);
const [defaultConfig] = await db const storageConfigs = await db
.select() .select()
.from(storageConfig) .from(storageConfig)
.where( .where(
and( and(
eq(storageConfig.type, "s3"), inArray(storageConfig.type, ["s3", "qiniu-kodo"]),
eq(storageConfig.isDefault, true),
eq(storageConfig.isEnabled, true) eq(storageConfig.isEnabled, true)
) )
) );
.limit(1);
return toS3Config(defaultConfig) || readEnvS3Config(); const configs = storageConfigs
.map(toS3Config)
.filter((config): config is S3UploadConfig => Boolean(config));
const envConfig = readEnvS3Config();
if (configs.length === 0 && envConfig) {
return [envConfig];
}
return configs;
} }
function readOrigins() { function readOrigins() {
@ -159,15 +172,21 @@ function isDirectUploadRule(rule: CORSRule, origins: string[]) {
); );
} }
function isCorsApiUnsupported(error: unknown) {
const value = error as { Code?: string; name?: string; message?: string } | null;
const code = String(value?.Code || value?.name || "");
const message = String(value?.message || "");
return code === "NotImplemented" || /not implemented/i.test(message);
}
async function main() { async function main() {
const config = await resolveS3Config(); const configs = await resolveS3Configs();
if (!config) { if (configs.length === 0) {
console.log("[direct-upload-cors] skip: no S3-compatible upload target"); console.log("[direct-upload-cors] skip: no S3-compatible upload targets");
return; return;
} }
const origins = readOrigins(); const origins = readOrigins();
const client = createS3Client(config);
const directRule: CORSRule = { const directRule: CORSRule = {
AllowedOrigins: origins, AllowedOrigins: origins,
AllowedMethods: ["GET", "PUT", "POST", "HEAD"], AllowedMethods: ["GET", "PUT", "POST", "HEAD"],
@ -175,41 +194,77 @@ async function main() {
ExposeHeaders: ["ETag", "x-amz-request-id", "x-amz-id-2"], ExposeHeaders: ["ETag", "x-amz-request-id", "x-amz-id-2"],
MaxAgeSeconds: 3600, MaxAgeSeconds: 3600,
}; };
const failures: Array<{ config: S3UploadConfig; error: unknown }> = [];
let existingRules: CORSRule[] = []; for (const config of configs) {
try { const client = createS3Client(config);
const current = await client.send( let existingRules: CORSRule[] = [];
new GetBucketCorsCommand({ Bucket: config.bucket }) try {
); const current = await client.send(
existingRules = current.CORSRules || []; new GetBucketCorsCommand({ Bucket: config.bucket })
} catch (error) { );
console.warn( existingRules = current.CORSRules || [];
"[direct-upload-cors] existing CORS not found or unreadable, replacing with direct upload rule:", } catch (error) {
error instanceof Error ? error.message : String(error) console.warn(
); "[direct-upload-cors] existing CORS not found or unreadable, replacing with direct upload rule:",
{
name: config.name,
bucket: config.bucket,
error: error instanceof Error ? error.message : String(error),
}
);
}
const mergedRules = [
directRule,
...existingRules.filter((rule) => !isDirectUploadRule(rule, origins)),
]
.slice(0, 100)
.map(normalizeRule);
try {
await client.send(
new PutBucketCorsCommand({
Bucket: config.bucket,
CORSConfiguration: {
CORSRules: mergedRules,
},
})
);
console.log("[direct-upload-cors] configured", {
name: config.name,
provider: config.provider,
bucket: config.bucket,
origins,
ruleCount: mergedRules.length,
});
} catch (error) {
if (isCorsApiUnsupported(error)) {
console.warn("[direct-upload-cors] bucket CORS API unsupported; leaving provider/global CORS in place", {
name: config.name,
provider: config.provider,
bucket: config.bucket,
error: error instanceof Error ? error.message : String(error),
});
continue;
}
failures.push({ config, error });
console.error("[direct-upload-cors] configure bucket failed", {
name: config.name,
provider: config.provider,
bucket: config.bucket,
error: error instanceof Error ? error.message : String(error),
});
}
} }
const mergedRules = [ if (failures.length > 0) {
directRule, throw new Error(
...existingRules.filter((rule) => !isDirectUploadRule(rule, origins)), `Failed to configure direct upload CORS for ${failures.length} bucket(s)`
] );
.slice(0, 100) }
.map(normalizeRule);
await client.send(
new PutBucketCorsCommand({
Bucket: config.bucket,
CORSConfiguration: {
CORSRules: mergedRules,
},
})
);
console.log("[direct-upload-cors] configured", {
bucket: config.bucket,
origins,
ruleCount: mergedRules.length,
});
} }
main() main()

View File

@ -6,12 +6,13 @@ import {
S3Client, S3Client,
type LifecycleRule, type LifecycleRule,
} from "@aws-sdk/client-s3"; } from "@aws-sdk/client-s3";
import { and, eq } from "drizzle-orm"; import { and, eq, inArray } from "drizzle-orm";
loadEnv({ path: ".env.local" }); loadEnv({ path: ".env.local" });
loadEnv(); loadEnv();
type S3UploadConfig = { type S3UploadConfig = {
name?: string;
endpoint: string; endpoint: string;
region: string; region: string;
bucket: string; bucket: string;
@ -65,7 +66,7 @@ function toS3Config(value: unknown): S3UploadConfig | null {
if ( if (
!raw || !raw ||
raw.type !== "s3" || (raw.type !== "s3" && raw.type !== "qiniu-kodo") ||
raw.isEnabled !== true || raw.isEnabled !== true ||
!endpoint || !endpoint ||
!bucket || !bucket ||
@ -76,12 +77,13 @@ function toS3Config(value: unknown): S3UploadConfig | null {
} }
return { return {
name: raw.name ? String(raw.name) : undefined,
endpoint, endpoint,
region: String(raw.region || "auto").trim() || "auto", region: String(raw.region || "auto").trim() || "auto",
bucket, bucket,
accessKeyId, accessKeyId,
secretAccessKey, secretAccessKey,
provider: raw.provider === "qiniu-kodo" ? "qiniu-kodo" : "s3", provider: raw.type === "qiniu-kodo" ? "qiniu-kodo" : "s3",
}; };
} }
@ -98,6 +100,7 @@ function readEnvS3Config(): S3UploadConfig | null {
} }
return { return {
name: "Environment S3",
endpoint, endpoint,
region: process.env.S3_REGION?.trim() || "auto", region: process.env.S3_REGION?.trim() || "auto",
bucket, bucket,
@ -106,24 +109,31 @@ function readEnvS3Config(): S3UploadConfig | null {
}; };
} }
async function resolveS3Config() { async function resolveS3Configs() {
const [{ db }, { storageConfig }] = await Promise.all([ const [{ db }, { storageConfig }] = await Promise.all([
import("@/lib/db"), import("@/lib/db"),
import("@/lib/db/schema"), import("@/lib/db/schema"),
]); ]);
const [defaultConfig] = await db const storageConfigs = await db
.select() .select()
.from(storageConfig) .from(storageConfig)
.where( .where(
and( and(
eq(storageConfig.type, "s3"), inArray(storageConfig.type, ["s3", "qiniu-kodo"]),
eq(storageConfig.isDefault, true),
eq(storageConfig.isEnabled, true) eq(storageConfig.isEnabled, true)
) )
) );
.limit(1);
return toS3Config(defaultConfig) || readEnvS3Config(); const configs = storageConfigs
.map(toS3Config)
.filter((config): config is S3UploadConfig => Boolean(config));
const envConfig = readEnvS3Config();
if (configs.length === 0 && envConfig) {
return [envConfig];
}
return configs;
} }
function readAbortDays() { function readAbortDays() {
@ -148,62 +158,123 @@ function normalizeRule(rule: LifecycleRule): LifecycleRule {
}; };
} }
function isLifecycleApiUnsupported(error: unknown) {
const value = error as { Code?: string; name?: string; message?: string } | null;
const code = String(value?.Code || value?.name || "");
const message = String(value?.message || "");
return (
code === "NotImplemented" ||
/not implemented/i.test(message) ||
/not well-formed/i.test(message) ||
/validate against/i.test(message)
);
}
async function main() { async function main() {
const config = await resolveS3Config(); const configs = await resolveS3Configs();
if (!config) { if (configs.length === 0) {
console.log("[multipart-lifecycle] skip: no S3-compatible upload target"); console.log("[multipart-lifecycle] skip: no S3-compatible upload targets");
return; return;
} }
const client = createS3Client(config);
const prefix = readRulePrefix(); const prefix = readRulePrefix();
const abortDays = readAbortDays(); const abortDays = readAbortDays();
const ruleId = process.env.DIRECT_UPLOAD_LIFECYCLE_RULE_ID || DEFAULT_RULE_ID; const ruleId = process.env.DIRECT_UPLOAD_LIFECYCLE_RULE_ID || DEFAULT_RULE_ID;
const failures: Array<{ config: S3UploadConfig; error: unknown }> = [];
let configuredCount = 0;
let existingRules: LifecycleRule[] = []; for (const config of configs) {
try { if (config.provider === "qiniu-kodo") {
const current = await client.send( console.log("[multipart-lifecycle] skip bucket: qiniu-kodo lifecycle is not configured through S3 compatibility", {
new GetBucketLifecycleConfigurationCommand({ Bucket: config.bucket }) name: config.name,
); bucket: config.bucket,
existingRules = current.Rules || []; });
} catch (error) { continue;
console.warn( }
"[multipart-lifecycle] existing lifecycle not found or unreadable, replacing with direct upload cleanup rule:",
error instanceof Error ? error.message : String(error) const client = createS3Client(config);
); let existingRules: LifecycleRule[] = [];
try {
const current = await client.send(
new GetBucketLifecycleConfigurationCommand({ Bucket: config.bucket })
);
existingRules = current.Rules || [];
} catch (error) {
console.warn(
"[multipart-lifecycle] existing lifecycle not found or unreadable, replacing with direct upload cleanup rule:",
{
name: config.name,
bucket: config.bucket,
error: error instanceof Error ? error.message : String(error),
}
);
}
const cleanupRule: LifecycleRule = {
ID: ruleId,
Status: "Enabled",
Filter: { Prefix: prefix },
AbortIncompleteMultipartUpload: {
DaysAfterInitiation: abortDays,
},
};
const mergedRules = [
cleanupRule,
...existingRules.filter((rule) => rule.ID !== ruleId),
]
.slice(0, 1000)
.map(normalizeRule);
try {
await client.send(
new PutBucketLifecycleConfigurationCommand({
Bucket: config.bucket,
LifecycleConfiguration: {
Rules: mergedRules,
},
})
);
configuredCount += 1;
console.log("[multipart-lifecycle] configured", {
name: config.name,
provider: config.provider,
bucket: config.bucket,
prefix,
abortDays,
ruleCount: mergedRules.length,
});
} catch (error) {
if (isLifecycleApiUnsupported(error)) {
console.warn("[multipart-lifecycle] bucket lifecycle API unsupported; skipping cleanup rule", {
name: config.name,
provider: config.provider,
bucket: config.bucket,
error: error instanceof Error ? error.message : String(error),
});
continue;
}
failures.push({ config, error });
console.error("[multipart-lifecycle] configure bucket failed", {
name: config.name,
provider: config.provider,
bucket: config.bucket,
error: error instanceof Error ? error.message : String(error),
});
}
} }
const cleanupRule: LifecycleRule = { if (configuredCount === 0) {
ID: ruleId, console.log("[multipart-lifecycle] no compatible buckets required lifecycle configuration");
Status: "Enabled", }
Filter: { Prefix: prefix },
AbortIncompleteMultipartUpload: {
DaysAfterInitiation: abortDays,
},
};
const mergedRules = [ if (failures.length > 0) {
cleanupRule, throw new Error(
...existingRules.filter((rule) => rule.ID !== ruleId), `Failed to configure multipart lifecycle for ${failures.length} bucket(s)`
] );
.slice(0, 1000) }
.map(normalizeRule);
await client.send(
new PutBucketLifecycleConfigurationCommand({
Bucket: config.bucket,
LifecycleConfiguration: {
Rules: mergedRules,
},
})
);
console.log("[multipart-lifecycle] configured", {
bucket: config.bucket,
prefix,
abortDays,
ruleCount: mergedRules.length,
});
} }
main() main()