lunanting/qiangdanxitong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

实现S3分块上传支持大文件 - 超过10MB的文件使用分块上传(Multipart Upload) - 每个分块5MB，支持最大5GB文件 - 解决S3服务商413错误

Browse Source
This commit is contained in:
Your Name 2026-01-14 17:07:30 +08:00
parent 2697494934
commit c7728df3fb
1 changed files with 131 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
backend/src/main/java/com/designhub/service/S3Service.java
View File

@ -17,13 +17,24 @@ import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import software.amazon.awssdk.services.s3.model.UploadPartResponse;
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
import software.amazon.awssdk.services.s3.model.CompletedPart;
import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import net.coobird.thumbnailator.Thumbnails;
@ -34,6 +45,11 @@ import net.coobird.thumbnailator.Thumbnails;
public class S3Service {
private static final Logger log = LoggerFactory.getLogger(S3Service.class);
// 分块上传的阈值：超过10MB使用分块上传
private static final long MULTIPART_THRESHOLD = 10 * 1024 * 1024; // 10MB
// 每个分块的大小：5MB（S3要求最小5MB，最大5GB）
private static final int PART_SIZE = 5 * 1024 * 1024; // 5MB
@Autowired
private S3ConfigMapper s3ConfigMapper;
@ -72,15 +88,22 @@ public class S3Service {
try {
// 创建S3客户端
S3Client s3Client = createS3Client(config);
// 上传文件
PutObjectRequest putObjectRequest = PutObjectRequest.builder()
.bucket(config.getBucket())
.key(filePath)
.contentType(file.getContentType())
.build();
s3Client.putObject(putObjectRequest, RequestBody.fromInputStream(file.getInputStream(), file.getSize()));
// 根据文件大小选择上传方式
if (file.getSize() > MULTIPART_THRESHOLD) {
// 大文件使用分块上传
log.info("文件大小 {} bytes，使用分块上传: {}", file.getSize(), originalFilename);
uploadMultipart(s3Client, config, file, filePath);
} else {
// 小文件使用普通上传
log.info("文件大小 {} bytes，使用普通上传: {}", file.getSize(), originalFilename);
PutObjectRequest putObjectRequest = PutObjectRequest.builder()
.bucket(config.getBucket())
.key(filePath)
.contentType(file.getContentType())
.build();
s3Client.putObject(putObjectRequest, RequestBody.fromInputStream(file.getInputStream(), file.getSize()));
}
// 生成文件URL
String fileUrl = config.getPublicDomain() + "/" + filePath;
@ -107,6 +130,105 @@ public class S3Service {
throw new RuntimeException("文件上传失败: " + e.getMessage());
}
}
/**
* 分块上传大文件
*/
private void uploadMultipart(S3Client s3Client, S3Config config, MultipartFile file, String filePath) throws IOException {
String uploadId = null;
try {
// 1. 初始化分块上传
CreateMultipartUploadRequest createRequest = CreateMultipartUploadRequest.builder()
.bucket(config.getBucket())
.key(filePath)
.contentType(file.getContentType())
.build();
CreateMultipartUploadResponse createResponse = s3Client.createMultipartUpload(createRequest);
uploadId = createResponse.uploadId();
log.info("分块上传初始化成功，uploadId: {}", uploadId);
// 2. 上传各个分块
List<CompletedPart> completedParts = new ArrayList<>();
InputStream inputStream = file.getInputStream();
byte[] buffer = new byte[PART_SIZE];
int partNumber = 1;
int bytesRead;
long totalBytesUploaded = 0;
while ((bytesRead = readFully(inputStream, buffer)) > 0) {
UploadPartRequest uploadPartRequest = UploadPartRequest.builder()
.bucket(config.getBucket())
.key(filePath)
.uploadId(uploadId)
.partNumber(partNumber)
.build();
UploadPartResponse uploadPartResponse = s3Client.uploadPart(
uploadPartRequest,
RequestBody.fromBytes(bytesRead == PART_SIZE ? buffer : java.util.Arrays.copyOf(buffer, bytesRead))
);
completedParts.add(CompletedPart.builder()
.partNumber(partNumber)
.eTag(uploadPartResponse.eTag())
.build());
totalBytesUploaded += bytesRead;
log.info("分块 {} 上传完成，已上传 {} / {} bytes", partNumber, totalBytesUploaded, file.getSize());
partNumber++;
}
inputStream.close();
// 3. 完成分块上传
CompletedMultipartUpload completedUpload = CompletedMultipartUpload.builder()
.parts(completedParts)
.build();
CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder()
.bucket(config.getBucket())
.key(filePath)
.uploadId(uploadId)
.multipartUpload(completedUpload)
.build();
s3Client.completeMultipartUpload(completeRequest);
log.info("分块上传完成，共 {} 个分块", completedParts.size());
} catch (Exception e) {
// 如果上传失败，取消分块上传
if (uploadId != null) {
try {
AbortMultipartUploadRequest abortRequest = AbortMultipartUploadRequest.builder()
.bucket(config.getBucket())
.key(filePath)
.uploadId(uploadId)
.build();
s3Client.abortMultipartUpload(abortRequest);
log.info("分块上传已取消，uploadId: {}", uploadId);
} catch (Exception abortEx) {
log.error("取消分块上传失败", abortEx);
}
}
throw e;
}
}
/**
* 读取指定长度的数据到缓冲区
*/
private int readFully(InputStream inputStream, byte[] buffer) throws IOException {
int totalBytesRead = 0;
int bytesRead;
while (totalBytesRead < buffer.length && (bytesRead = inputStream.read(buffer, totalBytesRead, buffer.length - totalBytesRead)) != -1) {
totalBytesRead += bytesRead;
}
return totalBytesRead;
}
/**
* 创建S3客户端