476 lines
14 KiB
TypeScript
476 lines
14 KiB
TypeScript
import {
|
|
AbortMultipartUploadCommand,
|
|
CompleteMultipartUploadCommand,
|
|
CreateMultipartUploadCommand,
|
|
HeadObjectCommand,
|
|
PutObjectCommand,
|
|
UploadPartCommand,
|
|
} from "@aws-sdk/client-s3";
|
|
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
import { NextResponse } from "next/server";
|
|
import { z } from "zod";
|
|
|
|
import { auth } from "@/app/(auth)/auth";
|
|
import { upsertMediaAsset } from "@/lib/db/queries";
|
|
import { createLogger } from "@/lib/observability/logger";
|
|
import {
|
|
buildS3PublicUrl,
|
|
buildStorageObjectKey,
|
|
createS3Client,
|
|
getContentType,
|
|
maybeEnqueueStorageMigrationTask,
|
|
resolveS3UploadTarget,
|
|
} from "@/lib/storage/service";
|
|
import { OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE } from "@/lib/openclaw-chat/media-attachments";
|
|
|
|
const logger = createLogger("direct-upload-api");
|
|
const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024;
|
|
const MULTIPART_THRESHOLD_BYTES = 50 * 1024 * 1024;
|
|
const DEFAULT_PART_SIZE_BYTES = 8 * 1024 * 1024;
|
|
const PRESIGNED_URL_EXPIRES_SECONDS = 15 * 60;
|
|
const DIRECT_UPLOAD_PREFIX = "uploads/direct";
|
|
|
|
function elapsedMs(startedAt: number) {
|
|
return Date.now() - startedAt;
|
|
}
|
|
|
|
const allowedDirectUploadTypes = new Set([
|
|
"image/jpeg",
|
|
"image/jpg",
|
|
"image/png",
|
|
"image/webp",
|
|
]);
|
|
const allowedOpenClawVideoUploadTypes = new Set([
|
|
"video/mp4",
|
|
"video/quicktime",
|
|
"video/webm",
|
|
"video/x-m4v",
|
|
"video/x-matroska",
|
|
]);
|
|
|
|
const InitiateSchema = z.object({
|
|
action: z.literal("initiate"),
|
|
filename: z.string().min(1).max(256),
|
|
contentType: z.string().min(1).max(128),
|
|
size: z.number().int().positive().max(DIRECT_UPLOAD_MAX_BYTES),
|
|
purpose: z.string().max(64).optional(),
|
|
clientUploadId: z.string().max(128).optional(),
|
|
});
|
|
|
|
const CompleteSchema = z.object({
|
|
action: z.literal("complete"),
|
|
key: z.string().min(1).max(512),
|
|
uploadId: z.string().min(1).max(512).optional(),
|
|
filename: z.string().min(1).max(256),
|
|
contentType: z.string().min(1).max(128),
|
|
size: z.number().int().positive().max(DIRECT_UPLOAD_MAX_BYTES),
|
|
parts: z
|
|
.array(
|
|
z.object({
|
|
ETag: z.string().min(1),
|
|
PartNumber: z.number().int().positive(),
|
|
})
|
|
)
|
|
.optional(),
|
|
purpose: z.string().max(64).optional(),
|
|
clientUploadId: z.string().max(128).optional(),
|
|
});
|
|
|
|
const AbortSchema = z.object({
|
|
action: z.literal("abort"),
|
|
key: z.string().min(1).max(512),
|
|
uploadId: z.string().min(1).max(512),
|
|
clientUploadId: z.string().max(128).optional(),
|
|
});
|
|
|
|
const RequestSchema = z.discriminatedUnion("action", [
|
|
InitiateSchema,
|
|
CompleteSchema,
|
|
AbortSchema,
|
|
]);
|
|
|
|
function normalizeContentType(contentType: string, filename: string) {
|
|
const trimmed = contentType.trim().toLowerCase();
|
|
return trimmed || getContentType(filename);
|
|
}
|
|
|
|
function sanitizeDisplayName(filename: string) {
|
|
return (
|
|
filename
|
|
.trim()
|
|
.replace(/[^\u4e00-\u9fa5a-zA-Z0-9._-]+/g, "-")
|
|
.replace(/^-+|-+$/g, "")
|
|
.slice(0, 120) || `upload-${Date.now()}`
|
|
);
|
|
}
|
|
|
|
function userUploadPrefix(userId: string) {
|
|
return `${DIRECT_UPLOAD_PREFIX}/${userId}`;
|
|
}
|
|
|
|
function assertUserOwnedKey(key: string, userId: string) {
|
|
const normalized = String(key || "").replace(/^\/+/, "");
|
|
const prefix = `${userUploadPrefix(userId)}/`;
|
|
if (!normalized.startsWith(prefix)) {
|
|
throw new Error("Invalid upload object key");
|
|
}
|
|
return normalized;
|
|
}
|
|
|
|
function isAllowedDirectUpload(contentType: string, purpose?: string) {
|
|
return (
|
|
allowedDirectUploadTypes.has(contentType) ||
|
|
(purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE &&
|
|
allowedOpenClawVideoUploadTypes.has(contentType))
|
|
);
|
|
}
|
|
|
|
function computePartSize(size: number) {
|
|
if (size <= 100 * 1024 * 1024) return DEFAULT_PART_SIZE_BYTES;
|
|
if (size <= 300 * 1024 * 1024) return 12 * 1024 * 1024;
|
|
return 16 * 1024 * 1024;
|
|
}
|
|
|
|
async function createPartUrls(params: {
|
|
client: ReturnType<typeof createS3Client>;
|
|
bucket: string;
|
|
key: string;
|
|
uploadId: string;
|
|
partCount: number;
|
|
}) {
|
|
const urls = [];
|
|
for (let partNumber = 1; partNumber <= params.partCount; partNumber += 1) {
|
|
const command = new UploadPartCommand({
|
|
Bucket: params.bucket,
|
|
Key: params.key,
|
|
UploadId: params.uploadId,
|
|
PartNumber: partNumber,
|
|
});
|
|
urls.push({
|
|
partNumber,
|
|
url: await getSignedUrl(params.client, command, {
|
|
expiresIn: PRESIGNED_URL_EXPIRES_SECONDS,
|
|
}),
|
|
});
|
|
}
|
|
return urls;
|
|
}
|
|
|
|
export async function POST(request: Request) {
|
|
const requestStartedAt = Date.now();
|
|
const session = await auth();
|
|
if (!session?.user?.id) {
|
|
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
|
}
|
|
|
|
let payload: z.infer<typeof RequestSchema>;
|
|
try {
|
|
payload = RequestSchema.parse(await request.json());
|
|
} catch (error) {
|
|
return NextResponse.json(
|
|
{
|
|
error: "Invalid direct upload request",
|
|
details: error instanceof Error ? error.message : String(error),
|
|
},
|
|
{ status: 400 }
|
|
);
|
|
}
|
|
|
|
logger.info("direct_upload_request_start", {
|
|
action: payload.action,
|
|
size: "size" in payload ? payload.size : null,
|
|
contentType: "contentType" in payload ? payload.contentType : null,
|
|
purpose: "purpose" in payload ? payload.purpose || null : null,
|
|
clientUploadId:
|
|
"clientUploadId" in payload ? payload.clientUploadId || null : null,
|
|
});
|
|
|
|
const resolveStartedAt = Date.now();
|
|
const target = await resolveS3UploadTarget().catch((error) => {
|
|
logger.error("resolve_storage_target_failed", error);
|
|
return null;
|
|
});
|
|
|
|
if (!target) {
|
|
return NextResponse.json(
|
|
{ error: "Direct upload is not available for current storage backend" },
|
|
{ status: 409 }
|
|
);
|
|
}
|
|
logger.info("direct_upload_storage_target_resolved", {
|
|
action: payload.action,
|
|
elapsedMs: elapsedMs(resolveStartedAt),
|
|
provider: target.config.provider || "s3",
|
|
bucket: target.config.bucket,
|
|
hasPublicUrl: Boolean(target.config.publicUrl),
|
|
});
|
|
|
|
const client = createS3Client(target.config);
|
|
const bucket = target.config.bucket;
|
|
const usePublicReadAcl = target.config.provider !== "qiniu-kodo";
|
|
|
|
try {
|
|
if (payload.action === "initiate") {
|
|
const contentType = normalizeContentType(
|
|
payload.contentType,
|
|
payload.filename
|
|
);
|
|
|
|
if (!isAllowedDirectUpload(contentType, payload.purpose)) {
|
|
logger.warn("unsupported_direct_upload_type", {
|
|
contentType,
|
|
filenameExtension: payload.filename.toLowerCase().split(".").pop() || null,
|
|
size: payload.size,
|
|
purpose: payload.purpose || null,
|
|
clientUploadId: payload.clientUploadId || null,
|
|
});
|
|
return NextResponse.json(
|
|
{
|
|
error:
|
|
"直传仅支持 JPG、PNG、WebP 图片以及 Claw 对话的 MP4、MOV、WebM、M4V、MKV 视频",
|
|
},
|
|
{ status: 400 }
|
|
);
|
|
}
|
|
|
|
const displayName = sanitizeDisplayName(payload.filename);
|
|
const key = buildStorageObjectKey(
|
|
displayName,
|
|
userUploadPrefix(session.user.id)
|
|
);
|
|
|
|
if (payload.size < MULTIPART_THRESHOLD_BYTES) {
|
|
const presignStartedAt = Date.now();
|
|
const command = new PutObjectCommand({
|
|
Bucket: bucket,
|
|
Key: key,
|
|
ContentType: contentType,
|
|
CacheControl: "public, max-age=31536000, immutable",
|
|
...(usePublicReadAcl ? { ACL: "public-read" as const } : {}),
|
|
});
|
|
const url = await getSignedUrl(client, command, {
|
|
expiresIn: PRESIGNED_URL_EXPIRES_SECONDS,
|
|
});
|
|
logger.info("direct_upload_single_presigned", {
|
|
elapsedMs: elapsedMs(presignStartedAt),
|
|
totalElapsedMs: elapsedMs(requestStartedAt),
|
|
size: payload.size,
|
|
contentType,
|
|
clientUploadId: payload.clientUploadId || null,
|
|
});
|
|
|
|
const headers: Record<string, string> = {
|
|
"Content-Type": contentType,
|
|
"Cache-Control": "public, max-age=31536000, immutable",
|
|
};
|
|
if (usePublicReadAcl) {
|
|
headers["x-amz-acl"] = "public-read";
|
|
}
|
|
|
|
return NextResponse.json({
|
|
mode: "single",
|
|
key,
|
|
url,
|
|
headers,
|
|
expiresIn: PRESIGNED_URL_EXPIRES_SECONDS,
|
|
clientUploadId: payload.clientUploadId || null,
|
|
});
|
|
}
|
|
|
|
const multipartStartedAt = Date.now();
|
|
const createResult = await client.send(
|
|
new CreateMultipartUploadCommand({
|
|
Bucket: bucket,
|
|
Key: key,
|
|
ContentType: contentType,
|
|
CacheControl: "public, max-age=31536000, immutable",
|
|
...(usePublicReadAcl ? { ACL: "public-read" as const } : {}),
|
|
})
|
|
);
|
|
|
|
const uploadId = createResult.UploadId;
|
|
if (!uploadId) {
|
|
throw new Error("Storage provider did not return uploadId");
|
|
}
|
|
|
|
const partSize = computePartSize(payload.size);
|
|
const partCount = Math.ceil(payload.size / partSize);
|
|
const parts = await createPartUrls({
|
|
client,
|
|
bucket,
|
|
key,
|
|
uploadId,
|
|
partCount,
|
|
});
|
|
logger.info("direct_upload_multipart_prepared", {
|
|
elapsedMs: elapsedMs(multipartStartedAt),
|
|
totalElapsedMs: elapsedMs(requestStartedAt),
|
|
size: payload.size,
|
|
partCount,
|
|
partSize,
|
|
contentType,
|
|
clientUploadId: payload.clientUploadId || null,
|
|
});
|
|
|
|
return NextResponse.json({
|
|
mode: "multipart",
|
|
key,
|
|
uploadId,
|
|
partSize,
|
|
partCount,
|
|
parts,
|
|
expiresIn: PRESIGNED_URL_EXPIRES_SECONDS,
|
|
clientUploadId: payload.clientUploadId || null,
|
|
});
|
|
}
|
|
|
|
if (payload.action === "abort") {
|
|
const key = assertUserOwnedKey(payload.key, session.user.id);
|
|
const abortStartedAt = Date.now();
|
|
await client.send(
|
|
new AbortMultipartUploadCommand({
|
|
Bucket: bucket,
|
|
Key: key,
|
|
UploadId: payload.uploadId,
|
|
})
|
|
);
|
|
logger.info("direct_upload_multipart_aborted", {
|
|
elapsedMs: elapsedMs(abortStartedAt),
|
|
totalElapsedMs: elapsedMs(requestStartedAt),
|
|
clientUploadId: payload.clientUploadId || null,
|
|
});
|
|
return NextResponse.json({ ok: true });
|
|
}
|
|
|
|
const key = assertUserOwnedKey(payload.key, session.user.id);
|
|
const contentType = normalizeContentType(payload.contentType, payload.filename);
|
|
if (!isAllowedDirectUpload(contentType, payload.purpose)) {
|
|
return NextResponse.json(
|
|
{
|
|
error:
|
|
"直传仅支持 JPG、PNG、WebP 图片以及 Claw 对话的 MP4、MOV、WebM、M4V、MKV 视频",
|
|
},
|
|
{ status: 400 }
|
|
);
|
|
}
|
|
|
|
if (payload.uploadId) {
|
|
const completeMultipartStartedAt = Date.now();
|
|
const parts = [...(payload.parts || [])].sort(
|
|
(a, b) => a.PartNumber - b.PartNumber
|
|
);
|
|
if (parts.length === 0) {
|
|
return NextResponse.json(
|
|
{ error: "Missing multipart upload parts" },
|
|
{ status: 400 }
|
|
);
|
|
}
|
|
|
|
await client.send(
|
|
new CompleteMultipartUploadCommand({
|
|
Bucket: bucket,
|
|
Key: key,
|
|
UploadId: payload.uploadId,
|
|
MultipartUpload: {
|
|
Parts: parts,
|
|
},
|
|
})
|
|
);
|
|
logger.info("direct_upload_multipart_completed", {
|
|
elapsedMs: elapsedMs(completeMultipartStartedAt),
|
|
partCount: parts.length,
|
|
clientUploadId: payload.clientUploadId || null,
|
|
});
|
|
}
|
|
|
|
const headStartedAt = Date.now();
|
|
const head = await client.send(
|
|
new HeadObjectCommand({
|
|
Bucket: bucket,
|
|
Key: key,
|
|
})
|
|
);
|
|
logger.info("direct_upload_head_object_completed", {
|
|
elapsedMs: elapsedMs(headStartedAt),
|
|
uploadMode: payload.uploadId ? "multipart" : "single",
|
|
clientUploadId: payload.clientUploadId || null,
|
|
});
|
|
|
|
const actualSize =
|
|
typeof head.ContentLength === "number" ? head.ContentLength : null;
|
|
if (actualSize !== null && actualSize <= 0) {
|
|
throw new Error("Uploaded object is empty");
|
|
}
|
|
|
|
const url = buildS3PublicUrl(target.config, key);
|
|
const resolvedContentType =
|
|
head.ContentType || contentType || getContentType(payload.filename);
|
|
const assetStartedAt = Date.now();
|
|
const asset = await upsertMediaAsset({
|
|
userId: session.user.id,
|
|
mediaType: resolvedContentType.startsWith("video/") ? "video" : "image",
|
|
sourceType:
|
|
payload.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE
|
|
? "openclaw_chat_video_upload"
|
|
: "upload",
|
|
status: "ready",
|
|
originalUrl: url,
|
|
previewUrl: url,
|
|
thumbnailUrl: url,
|
|
mimeType: resolvedContentType,
|
|
displayName: payload.filename,
|
|
sourceTable:
|
|
payload.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE
|
|
? "openclaw_chat_video_upload"
|
|
: "direct_upload",
|
|
metadata: {
|
|
pathname: key,
|
|
size: actualSize ?? payload.size,
|
|
expectedSize: payload.size,
|
|
uploadMode: payload.uploadId ? "multipart" : "single",
|
|
purpose: payload.purpose || null,
|
|
clientUploadId: payload.clientUploadId || null,
|
|
},
|
|
});
|
|
logger.info("direct_upload_asset_registered", {
|
|
elapsedMs: elapsedMs(assetStartedAt),
|
|
totalElapsedMs: elapsedMs(requestStartedAt),
|
|
uploadMode: payload.uploadId ? "multipart" : "single",
|
|
assetId: asset?.id ?? null,
|
|
clientUploadId: payload.clientUploadId || null,
|
|
});
|
|
|
|
await maybeEnqueueStorageMigrationTask({
|
|
uploaded: { url, pathname: key },
|
|
storageConfig: target.storageConfig,
|
|
migrationConfig: target.migrationConfig,
|
|
}).catch((error) => {
|
|
logger.warn("enqueue_storage_migration_failed", error);
|
|
});
|
|
|
|
return NextResponse.json({
|
|
url,
|
|
pathname: key,
|
|
contentType: resolvedContentType,
|
|
assetId: asset?.id ?? null,
|
|
clientUploadId: payload.clientUploadId || null,
|
|
});
|
|
} catch (error) {
|
|
logger.error("direct_upload_request_failed", error, {
|
|
action: payload.action,
|
|
size: "size" in payload ? payload.size : null,
|
|
contentType: "contentType" in payload ? payload.contentType : null,
|
|
purpose: "purpose" in payload ? payload.purpose || null : null,
|
|
clientUploadId:
|
|
"clientUploadId" in payload ? payload.clientUploadId || null : null,
|
|
});
|
|
return NextResponse.json(
|
|
{
|
|
error: "Direct upload failed",
|
|
details: error instanceof Error ? error.message : String(error),
|
|
},
|
|
{ status: 500 }
|
|
);
|
|
}
|
|
}
|