diff --git a/.env.example b/.env.example index 943428edf..019ac065d 100644 --- a/.env.example +++ b/.env.example @@ -139,6 +139,13 @@ EXTERNAL_IMAGE_API_MAX_CONCURRENT=3 EXTERNAL_IMAGE_API_MAX_QUEUE=20 EXTERNAL_IMAGE_API_QUEUE_TIMEOUT_MS=120000 +# Browser direct uploads use signed OSS/S3 URLs. Configure CORS and lifecycle +# with the scripts below after changing the default storage bucket. +DIRECT_UPLOAD_CORS_ORIGINS=https://ai.yydd.ai,https://yydd.ai +DIRECT_UPLOAD_ABORT_INCOMPLETE_DAYS=2 +DIRECT_UPLOAD_LIFECYCLE_PREFIX=uploads/direct/ +DIRECT_UPLOAD_LIFECYCLE_RULE_ID=abort-incomplete-direct-upload-parts + # ----------------------------------------------------------------------------- # Agent / PSD Runtime # ----------------------------------------------------------------------------- diff --git a/app/(chat)/api/files/direct-upload/route.ts b/app/(chat)/api/files/direct-upload/route.ts index 160f9e3c1..bd96c74ec 100644 --- a/app/(chat)/api/files/direct-upload/route.ts +++ b/app/(chat)/api/files/direct-upload/route.ts @@ -42,6 +42,7 @@ const InitiateSchema = z.object({ contentType: z.string().min(1).max(128), size: z.number().int().positive().max(DIRECT_UPLOAD_MAX_BYTES), purpose: z.string().max(64).optional(), + clientUploadId: z.string().max(128).optional(), }); const CompleteSchema = z.object({ @@ -60,12 +61,14 @@ const CompleteSchema = z.object({ ) .optional(), purpose: z.string().max(64).optional(), + clientUploadId: z.string().max(128).optional(), }); const AbortSchema = z.object({ action: z.literal("abort"), key: z.string().min(1).max(512), uploadId: z.string().min(1).max(512), + clientUploadId: z.string().max(128).optional(), }); const RequestSchema = z.discriminatedUnion("action", [ @@ -184,6 +187,7 @@ export async function POST(request: Request) { filenameExtension: payload.filename.toLowerCase().split(".").pop() || null, size: payload.size, purpose: payload.purpose || null, + clientUploadId: payload.clientUploadId || null, }); return NextResponse.json( { error: "Direct upload only supports JPG, PNG and WebP images" }, @@ -219,6 +223,7 @@ export async function POST(request: Request) { "x-amz-acl": "public-read", }, expiresIn: PRESIGNED_URL_EXPIRES_SECONDS, + clientUploadId: payload.clientUploadId || null, }); } @@ -255,6 +260,7 @@ export async function POST(request: Request) { partCount, parts, expiresIn: PRESIGNED_URL_EXPIRES_SECONDS, + clientUploadId: payload.clientUploadId || null, }); } @@ -335,6 +341,7 @@ export async function POST(request: Request) { expectedSize: payload.size, uploadMode: payload.uploadId ? "multipart" : "single", purpose: payload.purpose || null, + clientUploadId: payload.clientUploadId || null, }, }); @@ -351,6 +358,7 @@ export async function POST(request: Request) { pathname: key, contentType: resolvedContentType, assetId: asset?.id ?? null, + clientUploadId: payload.clientUploadId || null, }); } catch (error) { logger.error("direct_upload_request_failed", error, { @@ -358,6 +366,8 @@ export async function POST(request: Request) { size: "size" in payload ? payload.size : null, contentType: "contentType" in payload ? payload.contentType : null, purpose: "purpose" in payload ? payload.purpose || null : null, + clientUploadId: + "clientUploadId" in payload ? payload.clientUploadId || null : null, }); return NextResponse.json( { diff --git a/app/(chat)/api/files/upload/route.ts b/app/(chat)/api/files/upload/route.ts index c78b7e253..4ddb1f601 100644 --- a/app/(chat)/api/files/upload/route.ts +++ b/app/(chat)/api/files/upload/route.ts @@ -155,6 +155,8 @@ export async function POST(request: Request) { const filename = (formData.get("file") as File).name; const contentType = normalizeUploadContentType(file.type, filename); const uploadPurpose = String(formData.get("purpose") || ""); + const clientUploadId = + String(formData.get("clientUploadId") || "").trim() || null; const isSvgUpload = contentType === "image/svg+xml"; if ( isSvgUpload && @@ -190,6 +192,7 @@ export async function POST(request: Request) { filenameExtension: filename.toLowerCase().split(".").pop() || null, size: file.size, purpose: uploadPurpose || null, + clientUploadId, }); return NextResponse.json({ error: errorMessage }, { status: 400 }); } @@ -227,6 +230,7 @@ export async function POST(request: Request) { originalSize: fileBuffer.byteLength, previewSize: preview.buffer.length, purpose: uploadPurpose, + clientUploadId, }, }); assetId = asset?.id ?? null; @@ -292,6 +296,7 @@ export async function POST(request: Request) { metadata: { pathname: data.pathname, size: optimizedBuffer.length, + clientUploadId, }, }); assetId = asset?.id ?? null; @@ -303,6 +308,7 @@ export async function POST(request: Request) { return NextResponse.json({ ...data, assetId, + clientUploadId, }); } catch (_error) { logger.error("[Upload API] Upload failed:", _error); diff --git a/docs/operations.md b/docs/operations.md index 628000cca..7269a0c8b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -159,6 +159,32 @@ curl -I --max-time 30 "http://127.0.0.1:3001/api/featured?page=1&pageSize=24" 首页 HTML 应快速返回。公开轻量接口在数据库慢或不可用时应快速返回缓存或 fallback，不应长时间挂住首屏。 +## 上传与对象存储 + +浏览器图片上传优先走 `/api/files/direct-upload` 获取签名 URL，再由浏览器直传默认 OSS/S3 Bucket；小图或直传失败时才降级到 `/api/files/upload` 服务器中转。排查用户上传失败时，优先按 `clientUploadId` 串联 direct upload、fallback upload 和 `MediaAsset.metadata`。 + +直传 Bucket 必须配置 CORS，否则大文件分片上传会在浏览器侧失败： + +```bash +DIRECT_UPLOAD_CORS_ORIGINS="https://ai.yydd.ai,https://yydd.ai" pnpm storage:configure-upload-cors +``` + +核心要求： + +- Allowed Methods 包含 `GET`、`PUT`、`POST`、`HEAD`。 +- Allowed Headers 允许 `*`，因为签名上传会携带 `Content-Type`、`Cache-Control`、ACL 等请求头。 +- Exposed Headers 必须包含 `ETag`，否则 multipart upload 无法 complete。 + +未完成的 multipart upload 会产生隐藏对象存储费用。生产 Bucket 必须开启 lifecycle 自动清理： + +```bash +DIRECT_UPLOAD_ABORT_INCOMPLETE_DAYS=2 pnpm storage:configure-upload-lifecycle +``` + +默认只清理 `uploads/direct/` 前缀下 2 天未完成的分片；如使用自定义前缀，可设置 `DIRECT_UPLOAD_LIFECYCLE_PREFIX`。 + +当前上传格式支持 JPEG、PNG、WebP，管理员图标额外允许 SVG。HEIC/HEIF 不是可靠的 Web 展示格式，当前会明确提示用户转为 JPG/PNG；若未来要原生支持，应作为独立专项引入客户端或服务端转码，并评估前端包体、Sharp/libvips HEIF 支持和隐私成本。 + ## 路由回归烟测 推荐使用脚本化检查： @@ -190,5 +216,5 @@ curl -I -X POST http://127.0.0.1:3001/api/files/upload - `.next/dev/lock` 阻止启动：已有 dev server 持锁，先确认进程归属，不要误杀其他工作区服务。 - 首页接口慢：先看 `Server-Timing`、`X-Cache`、`X-Fallback`，再查数据库连接池和公开接口 fallback。 - 登录 Host 错误：生产配置 `AUTH_URL`；仅在可信代理后设置 `AUTH_TRUST_HOST=true`。 -- 上传失败：确认后台存储配置、环境变量 S3 fallback 或 `BLOB_READ_WRITE_TOKEN` 至少一组可用。 +- 上传失败：先用 `clientUploadId` 查日志；再确认 Bucket CORS 暴露 `ETag`、生命周期规则已开启、后台存储配置或环境变量 S3 fallback 至少一组可用。 - 日志泄露风险：不要把完整 prompt、OAuth token、签名 URL、cookie、request body 放进日志 message。 diff --git a/lib/client/upload-with-retry.ts b/lib/client/upload-with-retry.ts index f262c4700..80ded40f2 100644 --- a/lib/client/upload-with-retry.ts +++ b/lib/client/upload-with-retry.ts @@ -6,6 +6,7 @@ const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024; const SERVER_UPLOAD_MAX_BYTES = 50 * 1024 * 1024; const DIRECT_UPLOAD_PART_CONCURRENCY = 4; const DIRECT_UPLOAD_SUPPRESS_MS = 2 * 60 * 1000; +const CLIENT_UPLOAD_ID_FIELD = "clientUploadId"; const RETRYABLE_UPLOAD_STATUSES = new Set([ 408, @@ -40,6 +41,7 @@ type UploadFormFile = { filename: string; contentType: string; purpose: string; + clientUploadId: string; }; type DirectUploadPart = { @@ -178,6 +180,27 @@ function normalizeBrowserImageType(contentType: string, filename: string) { return BROWSER_IMAGE_TYPES_BY_EXTENSION[extension] || normalized; } +function createClientUploadId() { + if ( + typeof crypto !== "undefined" && + typeof crypto.randomUUID === "function" + ) { + return crypto.randomUUID(); + } + return `upload-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; +} + +function ensureClientUploadId(formData: FormData) { + const existing = formData.get(CLIENT_UPLOAD_ID_FIELD); + if (typeof existing === "string" && existing.trim()) { + return existing.trim(); + } + + const clientUploadId = createClientUploadId(); + formData.set(CLIENT_UPLOAD_ID_FIELD, clientUploadId); + return clientUploadId; +} + function getUploadFormFile(formData: FormData): UploadFormFile | null { const value = formData.get("file"); if (!(value instanceof Blob)) { @@ -193,12 +216,14 @@ function getUploadFormFile(formData: FormData): UploadFormFile | null { filename ); const purposeValue = formData.get("purpose"); + const clientUploadId = ensureClientUploadId(formData); return { blob: value, filename, contentType, purpose: typeof purposeValue === "string" ? purposeValue : "", + clientUploadId, }; } @@ -362,6 +387,7 @@ async function fetchDirectUpload( contentType: file.contentType, size: file.blob.size, purpose: file.purpose, + clientUploadId: file.clientUploadId, }, signal ); @@ -392,6 +418,7 @@ async function fetchDirectUpload( contentType: file.contentType, size: file.blob.size, purpose: file.purpose, + clientUploadId: file.clientUploadId, }, signal ); @@ -416,6 +443,7 @@ async function fetchDirectUpload( size: file.blob.size, parts, purpose: file.purpose, + clientUploadId: file.clientUploadId, }, signal ); @@ -428,6 +456,7 @@ async function fetchDirectUpload( action: "abort", key: multipartUpload.key, uploadId: multipartUpload.uploadId, + clientUploadId: file.clientUploadId, }, signal ).catch(() => {}); @@ -440,6 +469,7 @@ async function fetchDirectUpload( error: "图片超过 50MB，必须使用浏览器直传；当前直传失败，请刷新后重试，或联系管理员检查存储桶 CORS/ETag 配置。", details: error instanceof Error ? error.message : String(error), + clientUploadId: file.clientUploadId, }, { status: 502 } ); diff --git a/package.json b/package.json index 881febf87..fb0749a3b 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,9 @@ "db:check": "drizzle-kit check", "db:up": "drizzle-kit up", "poc:nanobanana": "pnpm exec tsx scripts/nanobanana-browser-poc.ts", - "test": "export PLAYWRIGHT=True && pnpm exec playwright test" + "test": "export PLAYWRIGHT=True && pnpm exec playwright test", + "storage:configure-upload-cors": "pnpm exec tsx scripts/storage/configure-direct-upload-cors.ts", + "storage:configure-upload-lifecycle": "pnpm exec tsx scripts/storage/configure-multipart-lifecycle.ts" }, "dependencies": { "@ai-sdk/gateway": "^1.0.15", diff --git a/scripts/storage/configure-multipart-lifecycle.ts b/scripts/storage/configure-multipart-lifecycle.ts new file mode 100644 index 000000000..ba8bcb971 --- /dev/null +++ b/scripts/storage/configure-multipart-lifecycle.ts @@ -0,0 +1,216 @@ +import { config as loadEnv } from "dotenv"; + +import { + GetBucketLifecycleConfigurationCommand, + PutBucketLifecycleConfigurationCommand, + S3Client, + type LifecycleRule, +} from "@aws-sdk/client-s3"; +import { and, eq } from "drizzle-orm"; + +loadEnv({ path: ".env.local" }); +loadEnv(); + +type S3UploadConfig = { + endpoint: string; + region: string; + bucket: string; + accessKeyId: string; + secretAccessKey: string; + provider?: "s3" | "qiniu-kodo"; +}; + +const DEFAULT_RULE_ID = "abort-incomplete-direct-upload-parts"; +const DEFAULT_PREFIX = "uploads/direct/"; +const DEFAULT_ABORT_DAYS = 2; + +function normalizeEndpoint(endpoint: string): string { + const trimmed = endpoint.trim().replace(/\/+$/, ""); + if (!trimmed) return ""; + + if (/aliyuncs\.com/i.test(trimmed)) { + const withoutProtocol = trimmed.replace(/^https?:\/\//i, ""); + return `https://${withoutProtocol}`; + } + + return trimmed; +} + +function createS3Client(config: S3UploadConfig) { + const normalizedEndpoint = normalizeEndpoint(config.endpoint); + const lowerEndpoint = normalizedEndpoint.toLowerCase(); + const isCloudS3 = + lowerEndpoint.includes("aliyuncs.com") || + lowerEndpoint.includes("amazonaws.com"); + + return new S3Client({ + endpoint: normalizedEndpoint, + region: config.region || "auto", + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, + forcePathStyle: config.provider === "qiniu-kodo" ? true : !isCloudS3, + }); +} + +function toS3Config(value: unknown): S3UploadConfig | null { + const raw = value as Record | null | undefined; + const endpoint = raw?.endpoint ? normalizeEndpoint(String(raw.endpoint)) : ""; + const bucket = raw?.bucket ? String(raw.bucket).trim() : ""; + const accessKeyId = raw?.accessKeyId ? String(raw.accessKeyId).trim() : ""; + const secretAccessKey = raw?.secretAccessKey + ? String(raw.secretAccessKey).trim() + : ""; + + if ( + !raw || + raw.type !== "s3" || + raw.isEnabled !== true || + !endpoint || + !bucket || + !accessKeyId || + !secretAccessKey + ) { + return null; + } + + return { + endpoint, + region: String(raw.region || "auto").trim() || "auto", + bucket, + accessKeyId, + secretAccessKey, + provider: raw.provider === "qiniu-kodo" ? "qiniu-kodo" : "s3", + }; +} + +function readEnvS3Config(): S3UploadConfig | null { + const endpoint = process.env.S3_ENDPOINT + ? normalizeEndpoint(process.env.S3_ENDPOINT) + : ""; + const bucket = process.env.S3_BUCKET?.trim() || ""; + const accessKeyId = process.env.S3_ACCESS_KEY_ID?.trim() || ""; + const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY?.trim() || ""; + + if (!endpoint || !bucket || !accessKeyId || !secretAccessKey) { + return null; + } + + return { + endpoint, + region: process.env.S3_REGION?.trim() || "auto", + bucket, + accessKeyId, + secretAccessKey, + }; +} + +async function resolveS3Config() { + const [{ db }, { storageConfig }] = await Promise.all([ + import("@/lib/db"), + import("@/lib/db/schema"), + ]); + const [defaultConfig] = await db + .select() + .from(storageConfig) + .where( + and( + eq(storageConfig.type, "s3"), + eq(storageConfig.isDefault, true), + eq(storageConfig.isEnabled, true) + ) + ) + .limit(1); + + return toS3Config(defaultConfig) || readEnvS3Config(); +} + +function readAbortDays() { + const value = Number(process.env.DIRECT_UPLOAD_ABORT_INCOMPLETE_DAYS || ""); + if (Number.isFinite(value) && value >= 1 && value <= 30) { + return Math.floor(value); + } + return DEFAULT_ABORT_DAYS; +} + +function readRulePrefix() { + const raw = process.env.DIRECT_UPLOAD_LIFECYCLE_PREFIX || DEFAULT_PREFIX; + const normalized = raw.trim().replace(/^\/+/, ""); + return normalized || DEFAULT_PREFIX; +} + +function normalizeRule(rule: LifecycleRule): LifecycleRule { + return { + ...rule, + ID: rule.ID || DEFAULT_RULE_ID, + Status: rule.Status || "Enabled", + }; +} + +async function main() { + const config = await resolveS3Config(); + if (!config) { + console.log("[multipart-lifecycle] skip: no S3-compatible upload target"); + return; + } + + const client = createS3Client(config); + const prefix = readRulePrefix(); + const abortDays = readAbortDays(); + const ruleId = process.env.DIRECT_UPLOAD_LIFECYCLE_RULE_ID || DEFAULT_RULE_ID; + + let existingRules: LifecycleRule[] = []; + try { + const current = await client.send( + new GetBucketLifecycleConfigurationCommand({ Bucket: config.bucket }) + ); + existingRules = current.Rules || []; + } catch (error) { + console.warn( + "[multipart-lifecycle] existing lifecycle not found or unreadable, replacing with direct upload cleanup rule:", + error instanceof Error ? error.message : String(error) + ); + } + + const cleanupRule: LifecycleRule = { + ID: ruleId, + Status: "Enabled", + Filter: { Prefix: prefix }, + AbortIncompleteMultipartUpload: { + DaysAfterInitiation: abortDays, + }, + }; + + const mergedRules = [ + cleanupRule, + ...existingRules.filter((rule) => rule.ID !== ruleId), + ] + .slice(0, 1000) + .map(normalizeRule); + + await client.send( + new PutBucketLifecycleConfigurationCommand({ + Bucket: config.bucket, + LifecycleConfiguration: { + Rules: mergedRules, + }, + }) + ); + + console.log("[multipart-lifecycle] configured", { + bucket: config.bucket, + prefix, + abortDays, + ruleCount: mergedRules.length, + }); +} + +main() + .then(() => { + process.exit(0); + }) + .catch((error) => { + console.error("[multipart-lifecycle] failed:", error); + process.exit(1); + });