687 lines
22 KiB
TypeScript

import {
AbortMultipartUploadCommand,
CompleteMultipartUploadCommand,
CreateMultipartUploadCommand,
HeadObjectCommand,
ListPartsCommand,
PutObjectCommand,
UploadPartCommand,
} from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
import { NextResponse } from "next/server";
import { z } from "zod";
import { auth } from "@/app/(auth)/auth";
import { upsertMediaAsset } from "@/lib/db/queries";
import {
IMAGE_UPLOAD_MAX_BYTES,
IMAGE_UPLOAD_TOO_LARGE_MESSAGE,
} from "@/lib/image-upload-limits";
import { createLogger } from "@/lib/observability/logger";
import {
buildS3PublicUrl,
buildStorageObjectKey,
createS3Client,
getContentType,
maybeEnqueueStorageMigrationTask,
resolveS3UploadTarget,
} from "@/lib/storage/service";
import {
getOpenClawChatFileContentType,
isOpenClawChatFolderFileAllowed,
OPENCLAW_CHAT_FILE_MAX_BYTES,
OPENCLAW_CHAT_FILE_UPLOAD_PURPOSE,
OPENCLAW_CHAT_FOLDER_UPLOAD_PURPOSE,
OPENCLAW_CHAT_VIDEO_MAX_BYTES,
OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE,
} from "@/lib/openclaw-chat/media-attachments";
const logger = createLogger("direct-upload-api");
const DIRECT_UPLOAD_MAX_BYTES = OPENCLAW_CHAT_FILE_MAX_BYTES;
const DIRECT_UPLOAD_MEDIA_MAX_BYTES = 500 * 1024 * 1024;
const MULTIPART_THRESHOLD_BYTES = 50 * 1024 * 1024;
const DEFAULT_PART_SIZE_BYTES = 8 * 1024 * 1024;
const PRESIGNED_URL_EXPIRES_SECONDS = 6 * 60 * 60;
const DIRECT_UPLOAD_PREFIX = "uploads/direct";
function elapsedMs(startedAt: number) {
return Date.now() - startedAt;
}
const allowedDirectUploadTypes = new Set([
"image/jpeg",
"image/jpg",
"image/png",
"image/webp",
]);
const allowedOpenClawVideoUploadTypes = new Set([
"video/mp4",
"video/quicktime",
"video/webm",
"video/x-m4v",
"video/x-matroska",
]);
const InitiateSchema = z.object({
action: z.literal("initiate"),
filename: z.string().min(1).max(256),
contentType: z.string().min(1).max(128),
size: z.number().int().positive().max(DIRECT_UPLOAD_MAX_BYTES),
purpose: z.string().max(64).optional(),
clientUploadId: z.string().max(128).optional(),
});
const CompleteSchema = z.object({
action: z.literal("complete"),
key: z.string().min(1).max(512),
uploadId: z.string().min(1).max(512).optional(),
filename: z.string().min(1).max(256),
contentType: z.string().min(1).max(128),
size: z.number().int().positive().max(DIRECT_UPLOAD_MAX_BYTES),
parts: z
.array(
z.object({
ETag: z.string().min(1),
PartNumber: z.number().int().positive(),
})
)
.optional(),
purpose: z.string().max(64).optional(),
clientUploadId: z.string().max(128).optional(),
});
const ResumeSchema = z.object({
action: z.literal("resume"),
key: z.string().min(1).max(512),
uploadId: z.string().min(1).max(512),
filename: z.string().min(1).max(256),
contentType: z.string().min(1).max(128),
size: z.number().int().positive().max(DIRECT_UPLOAD_MAX_BYTES),
partSize: z.number().int().min(5 * 1024 * 1024).max(128 * 1024 * 1024),
purpose: z.string().max(64).optional(),
clientUploadId: z.string().max(128).optional(),
});
const AbortSchema = z.object({
action: z.literal("abort"),
key: z.string().min(1).max(512),
uploadId: z.string().min(1).max(512),
clientUploadId: z.string().max(128).optional(),
});
const RequestSchema = z.discriminatedUnion("action", [
InitiateSchema,
ResumeSchema,
CompleteSchema,
AbortSchema,
]);
function normalizeContentType(contentType: string, filename: string) {
const trimmed = contentType.trim().toLowerCase();
return trimmed || getContentType(filename);
}
function sanitizeDisplayName(filename: string) {
return (
filename
.trim()
.replace(/[^\u4e00-\u9fa5a-zA-Z0-9._-]+/g, "-")
.replace(/^-+|-+$/g, "")
.slice(0, 120) || `upload-${Date.now()}`
);
}
function userUploadPrefix(userId: string) {
return `${DIRECT_UPLOAD_PREFIX}/${userId}`;
}
function assertUserOwnedKey(key: string, userId: string) {
const normalized = String(key || "").replace(/^\/+/, "");
const prefix = `${userUploadPrefix(userId)}/`;
if (!normalized.startsWith(prefix)) {
const error = new Error("Invalid upload object key");
error.name = "InvalidUploadObjectKeyError";
throw error;
}
return normalized;
}
function isAllowedDirectUpload(
contentType: string,
purpose: string | undefined,
filename: string
) {
return (
allowedDirectUploadTypes.has(contentType) ||
(purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE &&
allowedOpenClawVideoUploadTypes.has(contentType)) ||
(purpose === OPENCLAW_CHAT_FILE_UPLOAD_PURPOSE &&
Boolean(
getOpenClawChatFileContentType({ name: filename, type: contentType })
)) ||
(purpose === OPENCLAW_CHAT_FOLDER_UPLOAD_PURPOSE &&
isOpenClawChatFolderFileAllowed({ name: filename, type: contentType }))
);
}
function getPurposeMaxBytes(purpose?: string, contentType?: string) {
if (String(contentType || "").toLowerCase().startsWith("image/")) {
return IMAGE_UPLOAD_MAX_BYTES;
}
if (purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE) {
return OPENCLAW_CHAT_VIDEO_MAX_BYTES;
}
if (
purpose === OPENCLAW_CHAT_FILE_UPLOAD_PURPOSE ||
purpose === OPENCLAW_CHAT_FOLDER_UPLOAD_PURPOSE
) {
return DIRECT_UPLOAD_MAX_BYTES;
}
return DIRECT_UPLOAD_MEDIA_MAX_BYTES;
}
function computePartSize(size: number) {
if (size <= 100 * 1024 * 1024) return DEFAULT_PART_SIZE_BYTES;
if (size <= 500 * 1024 * 1024) return 16 * 1024 * 1024;
if (size <= 1024 * 1024 * 1024) return 32 * 1024 * 1024;
return 64 * 1024 * 1024;
}
async function createPartUrls(params: {
client: ReturnType<typeof createS3Client>;
bucket: string;
key: string;
uploadId: string;
partNumbers: number[];
}) {
return Promise.all(
params.partNumbers.map(async (partNumber) => {
const command = new UploadPartCommand({
Bucket: params.bucket,
Key: params.key,
UploadId: params.uploadId,
PartNumber: partNumber,
});
return {
partNumber,
url: await getSignedUrl(params.client, command, {
expiresIn: PRESIGNED_URL_EXPIRES_SECONDS,
}),
};
})
);
}
export async function POST(request: Request) {
const requestStartedAt = Date.now();
const session = await auth();
if (!session?.user?.id) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
let payload: z.infer<typeof RequestSchema>;
try {
payload = RequestSchema.parse(await request.json());
} catch (error) {
return NextResponse.json(
{
error: "Invalid direct upload request",
details: error instanceof Error ? error.message : String(error),
},
{ status: 400 }
);
}
logger.info("direct_upload_request_start", {
action: payload.action,
size: "size" in payload ? payload.size : null,
contentType: "contentType" in payload ? payload.contentType : null,
purpose: "purpose" in payload ? payload.purpose || null : null,
clientUploadId:
"clientUploadId" in payload ? payload.clientUploadId || null : null,
});
const resolveStartedAt = Date.now();
const target = await resolveS3UploadTarget().catch((error) => {
logger.error("resolve_storage_target_failed", error);
return null;
});
if (!target) {
return NextResponse.json(
{ error: "Direct upload is not available for current storage backend" },
{ status: 409 }
);
}
logger.info("direct_upload_storage_target_resolved", {
action: payload.action,
elapsedMs: elapsedMs(resolveStartedAt),
provider: target.config.provider || "s3",
bucket: target.config.bucket,
hasPublicUrl: Boolean(target.config.publicUrl),
});
const client = createS3Client(target.config);
const bucket = target.config.bucket;
const usePublicReadAcl = target.config.provider !== "qiniu-kodo";
try {
if (payload.action === "initiate") {
const contentType = normalizeContentType(
payload.contentType,
payload.filename
);
if (!isAllowedDirectUpload(contentType, payload.purpose, payload.filename)) {
logger.warn("unsupported_direct_upload_type", {
contentType,
filenameExtension: payload.filename.toLowerCase().split(".").pop() || null,
size: payload.size,
purpose: payload.purpose || null,
clientUploadId: payload.clientUploadId || null,
});
return NextResponse.json(
{
error:
"直传仅支持图片、Claw 视频、压缩包、表格和文件夹清单",
},
{ status: 400 }
);
}
const maxBytes = getPurposeMaxBytes(payload.purpose, contentType);
if (payload.size > maxBytes) {
return NextResponse.json(
{
error: contentType.startsWith("image/")
? IMAGE_UPLOAD_TOO_LARGE_MESSAGE
: `文件不能超过 ${maxBytes / 1024 / 1024}MB`,
},
{ status: 413 }
);
}
const displayName = sanitizeDisplayName(payload.filename);
const key = buildStorageObjectKey(
displayName,
userUploadPrefix(session.user.id)
);
const multipartThreshold =
payload.purpose === OPENCLAW_CHAT_FILE_UPLOAD_PURPOSE ||
payload.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE ||
payload.purpose === OPENCLAW_CHAT_FOLDER_UPLOAD_PURPOSE
? DEFAULT_PART_SIZE_BYTES
: MULTIPART_THRESHOLD_BYTES;
if (payload.size < multipartThreshold) {
const presignStartedAt = Date.now();
const command = new PutObjectCommand({
Bucket: bucket,
Key: key,
ContentType: contentType,
CacheControl: "public, max-age=31536000, immutable",
...(usePublicReadAcl ? { ACL: "public-read" as const } : {}),
});
const url = await getSignedUrl(client, command, {
expiresIn: PRESIGNED_URL_EXPIRES_SECONDS,
});
logger.info("direct_upload_single_presigned", {
elapsedMs: elapsedMs(presignStartedAt),
totalElapsedMs: elapsedMs(requestStartedAt),
size: payload.size,
contentType,
clientUploadId: payload.clientUploadId || null,
});
const headers: Record<string, string> = {
"Content-Type": contentType,
"Cache-Control": "public, max-age=31536000, immutable",
};
if (usePublicReadAcl) {
headers["x-amz-acl"] = "public-read";
}
return NextResponse.json({
mode: "single",
key,
url,
headers,
expiresIn: PRESIGNED_URL_EXPIRES_SECONDS,
clientUploadId: payload.clientUploadId || null,
});
}
const multipartStartedAt = Date.now();
const createResult = await client.send(
new CreateMultipartUploadCommand({
Bucket: bucket,
Key: key,
ContentType: contentType,
CacheControl: "public, max-age=31536000, immutable",
...(usePublicReadAcl ? { ACL: "public-read" as const } : {}),
})
);
const uploadId = createResult.UploadId;
if (!uploadId) {
throw new Error("Storage provider did not return uploadId");
}
const partSize = computePartSize(payload.size);
const partCount = Math.ceil(payload.size / partSize);
const parts = await createPartUrls({
client,
bucket,
key,
uploadId,
partNumbers: Array.from({ length: partCount }, (_, index) => index + 1),
});
logger.info("direct_upload_multipart_prepared", {
elapsedMs: elapsedMs(multipartStartedAt),
totalElapsedMs: elapsedMs(requestStartedAt),
size: payload.size,
partCount,
partSize,
contentType,
clientUploadId: payload.clientUploadId || null,
});
return NextResponse.json({
mode: "multipart",
key,
uploadId,
partSize,
partCount,
parts,
expiresIn: PRESIGNED_URL_EXPIRES_SECONDS,
clientUploadId: payload.clientUploadId || null,
});
}
if (payload.action === "resume") {
const key = assertUserOwnedKey(payload.key, session.user.id);
const contentType = normalizeContentType(payload.contentType, payload.filename);
if (!isAllowedDirectUpload(contentType, payload.purpose, payload.filename)) {
return NextResponse.json({ error: "不支持续传该文件类型" }, { status: 400 });
}
if (payload.size > getPurposeMaxBytes(payload.purpose, contentType)) {
return NextResponse.json(
{
error: contentType.startsWith("image/")
? IMAGE_UPLOAD_TOO_LARGE_MESSAGE
: "文件超过允许的上传大小",
},
{ status: 413 }
);
}
const expectedPartSize = computePartSize(payload.size);
if (payload.partSize !== expectedPartSize) {
return NextResponse.json(
{ error: "续传分片参数已失效,请重新开始上传", code: "RESUME_SESSION_EXPIRED" },
{ status: 409 }
);
}
try {
const listed = await client.send(
new ListPartsCommand({
Bucket: bucket,
Key: key,
UploadId: payload.uploadId,
MaxParts: 1000,
})
);
const uploadedParts = (listed.Parts || []).flatMap((part) => {
if (!part.ETag || typeof part.PartNumber !== "number") return [];
return [
{
ETag: part.ETag,
PartNumber: part.PartNumber,
size: Number(part.Size || 0),
},
];
});
const uploadedPartNumbers = new Set(
uploadedParts.map((part) => part.PartNumber)
);
const partCount = Math.ceil(payload.size / expectedPartSize);
const missingPartNumbers = Array.from(
{ length: partCount },
(_, index) => index + 1
).filter((partNumber) => !uploadedPartNumbers.has(partNumber));
const parts = await createPartUrls({
client,
bucket,
key,
uploadId: payload.uploadId,
partNumbers: missingPartNumbers,
});
logger.info("direct_upload_multipart_resumed", {
totalElapsedMs: elapsedMs(requestStartedAt),
size: payload.size,
uploadedPartCount: uploadedParts.length,
missingPartCount: missingPartNumbers.length,
clientUploadId: payload.clientUploadId || null,
});
return NextResponse.json({
mode: "multipart",
key,
uploadId: payload.uploadId,
partSize: expectedPartSize,
partCount,
parts,
uploadedParts,
resumed: true,
expiresIn: PRESIGNED_URL_EXPIRES_SECONDS,
clientUploadId: payload.clientUploadId || null,
});
} catch (error) {
const head = await client
.send(new HeadObjectCommand({ Bucket: bucket, Key: key }))
.catch(() => null);
if (head?.ContentLength === payload.size) {
return NextResponse.json({
mode: "completed",
key,
clientUploadId: payload.clientUploadId || null,
});
}
logger.warn("direct_upload_resume_session_missing", error, {
clientUploadId: payload.clientUploadId || null,
});
return NextResponse.json(
{ error: "续传会话已失效,请重新开始上传", code: "RESUME_SESSION_EXPIRED" },
{ status: 409 }
);
}
}
if (payload.action === "abort") {
const key = assertUserOwnedKey(payload.key, session.user.id);
const abortStartedAt = Date.now();
await client.send(
new AbortMultipartUploadCommand({
Bucket: bucket,
Key: key,
UploadId: payload.uploadId,
})
);
logger.info("direct_upload_multipart_aborted", {
elapsedMs: elapsedMs(abortStartedAt),
totalElapsedMs: elapsedMs(requestStartedAt),
clientUploadId: payload.clientUploadId || null,
});
return NextResponse.json({ ok: true });
}
const key = assertUserOwnedKey(payload.key, session.user.id);
const contentType = normalizeContentType(payload.contentType, payload.filename);
if (!isAllowedDirectUpload(contentType, payload.purpose, payload.filename)) {
return NextResponse.json(
{
error:
"直传仅支持图片、Claw 视频、压缩包、表格和文件夹清单",
},
{ status: 400 }
);
}
if (payload.size > getPurposeMaxBytes(payload.purpose, contentType)) {
return NextResponse.json(
{
error: contentType.startsWith("image/")
? IMAGE_UPLOAD_TOO_LARGE_MESSAGE
: "文件超过允许的上传大小",
},
{ status: 413 }
);
}
if (payload.uploadId) {
const completeMultipartStartedAt = Date.now();
const parts = [...(payload.parts || [])].sort(
(a, b) => a.PartNumber - b.PartNumber
);
if (parts.length === 0) {
return NextResponse.json(
{ error: "Missing multipart upload parts" },
{ status: 400 }
);
}
await client.send(
new CompleteMultipartUploadCommand({
Bucket: bucket,
Key: key,
UploadId: payload.uploadId,
MultipartUpload: {
Parts: parts,
},
})
);
logger.info("direct_upload_multipart_completed", {
elapsedMs: elapsedMs(completeMultipartStartedAt),
partCount: parts.length,
clientUploadId: payload.clientUploadId || null,
});
}
const headStartedAt = Date.now();
const head = await client.send(
new HeadObjectCommand({
Bucket: bucket,
Key: key,
})
);
logger.info("direct_upload_head_object_completed", {
elapsedMs: elapsedMs(headStartedAt),
uploadMode: payload.uploadId ? "multipart" : "single",
clientUploadId: payload.clientUploadId || null,
});
const actualSize =
typeof head.ContentLength === "number" ? head.ContentLength : null;
if (actualSize !== null && actualSize !== payload.size) {
throw new Error(
`Uploaded object size mismatch: expected ${payload.size}, received ${actualSize}`
);
}
const url = buildS3PublicUrl(target.config, key);
const resolvedContentType =
head.ContentType || contentType || getContentType(payload.filename);
const assetStartedAt = Date.now();
const asset = await upsertMediaAsset({
userId: session.user.id,
mediaType:
payload.purpose === OPENCLAW_CHAT_FILE_UPLOAD_PURPOSE ||
payload.purpose === OPENCLAW_CHAT_FOLDER_UPLOAD_PURPOSE
? "file"
: resolvedContentType.startsWith("video/")
? "video"
: "image",
sourceType:
payload.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE
? "openclaw_chat_video_upload"
: payload.purpose === OPENCLAW_CHAT_FILE_UPLOAD_PURPOSE
? "openclaw_chat_file_upload"
: payload.purpose === OPENCLAW_CHAT_FOLDER_UPLOAD_PURPOSE
? "openclaw_chat_folder_file_upload"
: "upload",
status: "ready",
originalUrl: url,
previewUrl:
payload.purpose === OPENCLAW_CHAT_FILE_UPLOAD_PURPOSE ||
payload.purpose === OPENCLAW_CHAT_FOLDER_UPLOAD_PURPOSE
? null
: url,
thumbnailUrl:
payload.purpose === OPENCLAW_CHAT_FILE_UPLOAD_PURPOSE ||
payload.purpose === OPENCLAW_CHAT_FOLDER_UPLOAD_PURPOSE
? null
: url,
mimeType: resolvedContentType,
displayName: payload.filename,
sourceTable:
payload.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE
? "openclaw_chat_video_upload"
: payload.purpose === OPENCLAW_CHAT_FILE_UPLOAD_PURPOSE
? "openclaw_chat_file_upload"
: payload.purpose === OPENCLAW_CHAT_FOLDER_UPLOAD_PURPOSE
? "openclaw_chat_folder_file_upload"
: "direct_upload",
metadata: {
pathname: key,
size: actualSize ?? payload.size,
expectedSize: payload.size,
uploadMode: payload.uploadId ? "multipart" : "single",
purpose: payload.purpose || null,
clientUploadId: payload.clientUploadId || null,
},
});
logger.info("direct_upload_asset_registered", {
elapsedMs: elapsedMs(assetStartedAt),
totalElapsedMs: elapsedMs(requestStartedAt),
uploadMode: payload.uploadId ? "multipart" : "single",
assetId: asset?.id ?? null,
clientUploadId: payload.clientUploadId || null,
});
await maybeEnqueueStorageMigrationTask({
uploaded: { url, pathname: key },
storageConfig: target.storageConfig,
migrationConfig: target.migrationConfig,
}).catch((error) => {
logger.warn("enqueue_storage_migration_failed", error);
});
return NextResponse.json({
url,
pathname: key,
contentType: resolvedContentType,
assetId: asset?.id ?? null,
clientUploadId: payload.clientUploadId || null,
});
} catch (error) {
logger.error("direct_upload_request_failed", error, {
action: payload.action,
size: "size" in payload ? payload.size : null,
contentType: "contentType" in payload ? payload.contentType : null,
purpose: "purpose" in payload ? payload.purpose || null : null,
clientUploadId:
"clientUploadId" in payload ? payload.clientUploadId || null : null,
});
return NextResponse.json(
{
error: "Direct upload failed",
details: error instanceof Error ? error.message : String(error),
},
{
status:
error instanceof Error &&
error.name === "InvalidUploadObjectKeyError"
? 403
: 500,
}
);
}
}