WuYongJie/furniture-3d-system-Wu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(intake): support large non-integrated uploads

Browse Source
This commit is contained in:
WuYongJie 2026-05-07 18:45:41 +08:00
parent 6dd75c502a
commit 41fb1b0aac
7 changed files with 300 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
furniture-3d-system/next.config.js
View File

@ -85,7 +85,7 @@ const nextConfig = {
// Turbopack配置（Next.js 16默认启用）
turbopack: {},
// Non-integrated uploads can include model ZIPs. Keep Next proxy body limit aligned with the app upload limit.
// Keep single request bodies bounded. Large non-integrated files use the chunked upload route.
experimental: {
proxyClientMaxBodySize: 500 * 1024 * 1024,
},

88
furniture-3d-system/src/app/api/intake/non-integrated/[id]/upload/chunk/route.ts
View File

@ -1,16 +1,18 @@
import { existsSync } from 'fs'
import { mkdir, readFile, rm, stat, writeFile } from 'fs/promises'
import { createReadStream, createWriteStream, existsSync } from 'fs'
import { mkdir, rm, stat, writeFile } from 'fs/promises'
import os from 'os'
import path from 'path'
import { pipeline } from 'stream/promises'
import { NextRequest, NextResponse } from 'next/server'
import { AppError, ErrorCode } from '@/server/core/errors'
import { handleRequest, jsonError, jsonSuccess } from '@/server/core/response'
import { intakeNonIntegratedProductService } from '@/server/services'
import {
completeNonIntegratedStoredUploadFile,
createNonIntegratedUploadTarget,
readOptionalFormValue,
readUploadMetadata,
shouldAttachUploadFile,
storeNonIntegratedUploadFile,
} from '../../../uploadHelpers'
import {
NON_INTEGRATED_UPLOAD_CHUNK_MAX_BYTES,
@ -41,30 +43,46 @@ function getChunkSessionDirectory(productId: string, uploadId: string) {
)
}
async function readMergedChunks(sessionDirectory: string, totalChunks: number) {
const buffers: Buffer[] = []
async function mergeChunksToUploadTarget(
sessionDirectory: string,
totalChunks: number,
targetPath: string,
) {
let totalBytes = 0
let hasWrittenOutput = false
for (let index = 0; index < totalChunks; index += 1) {
const chunkPath = path.join(sessionDirectory, `${index}.part`)
if (!existsSync(chunkPath)) {
return null
}
await rm(targetPath, { force: true })
const chunk = await readFile(chunkPath)
totalBytes += chunk.length
try {
for (let index = 0; index < totalChunks; index += 1) {
const chunkPath = path.join(sessionDirectory, `${index}.part`)
if (!existsSync(chunkPath)) {
await rm(targetPath, { force: true })
return null
}
if (totalBytes > NON_INTEGRATED_UPLOAD_MAX_BYTES) {
throw new AppError(
ErrorCode.FILE_TOO_LARGE,
buildNonIntegratedFileTooLargeMessage(),
const chunkStat = await stat(chunkPath)
totalBytes += chunkStat.size
if (totalBytes > NON_INTEGRATED_UPLOAD_MAX_BYTES) {
throw new AppError(
ErrorCode.FILE_TOO_LARGE,
buildNonIntegratedFileTooLargeMessage(),
)
}
await pipeline(
createReadStream(chunkPath),
createWriteStream(targetPath, { flags: hasWrittenOutput ? 'a' : 'w' }),
)
hasWrittenOutput = true
}
buffers.push(chunk)
} catch (error) {
await rm(targetPath, { force: true })
throw error
}
return Buffer.concat(buffers, totalBytes)
return totalBytes
}
export async function POST(request: NextRequest, context: RouteContext) {
@ -123,14 +141,16 @@ export async function POST(request: NextRequest, context: RouteContext) {
)
}
try {
await intakeNonIntegratedProductService.assertCanWrite(productId, user)
} catch (error) {
if (error instanceof AppError) {
return jsonError(error.code, error.message, error.statusCode, requestId)
}
if (shouldAttach) {
try {
await intakeNonIntegratedProductService.assertCanWrite(productId, user)
} catch (error) {
if (error instanceof AppError) {
return jsonError(error.code, error.message, error.statusCode, requestId)
}
throw error
throw error
}
}
const sessionDirectory = getChunkSessionDirectory(productId, uploadId)
@ -161,18 +181,24 @@ export async function POST(request: NextRequest, context: RouteContext) {
})
}
const mergedBytes = await readMergedChunks(sessionDirectory, totalChunks)
if (!mergedBytes) {
const uploadTarget = await createNonIntegratedUploadTarget(fileName)
const mergedSize = await mergeChunksToUploadTarget(
sessionDirectory,
totalChunks,
uploadTarget.storedFilePath,
)
if (mergedSize === null) {
return jsonError(ErrorCode.FILE_UPLOAD_FAILED, '上传分片不完整，请重新上传', 400, requestId)
}
if (mergedBytes.length !== totalSize) {
if (mergedSize !== totalSize) {
await rm(uploadTarget.storedFilePath, { force: true })
await rm(sessionDirectory, { recursive: true, force: true })
return jsonError(ErrorCode.FILE_UPLOAD_FAILED, '上传文件大小校验失败，请重新上传', 400, requestId)
}
const responseData = await storeNonIntegratedUploadFile({
bytes: mergedBytes,
const responseData = await completeNonIntegratedStoredUploadFile({
storedFileName: uploadTarget.storedFileName,
originalFileName: fileName,
mimeType,
size: totalSize,

16
furniture-3d-system/src/app/api/intake/non-integrated/[id]/upload/route.ts
View File

@ -47,14 +47,16 @@ export async function POST(request: NextRequest, context: RouteContext) {
)
}
try {
await intakeNonIntegratedProductService.assertCanWrite(productId, user)
} catch (error) {
if (error instanceof AppError) {
return jsonError(error.code, error.message, error.statusCode, requestId)
}
if (shouldAttach) {
try {
await intakeNonIntegratedProductService.assertCanWrite(productId, user)
} catch (error) {
if (error instanceof AppError) {
return jsonError(error.code, error.message, error.statusCode, requestId)
}
throw error
throw error
}
}
const bytes = await file.arrayBuffer()

155
furniture-3d-system/src/app/api/intake/non-integrated/upload/route.test.ts
View File

@ -7,6 +7,7 @@ import { afterEach, beforeEach, describe, expect, it, jest } from '@jest/globals
const mockHandleRequest = jest.fn()
const mockGetNonIntegratedAssetDirectory = jest.fn()
const mockAssertCanWrite = jest.fn()
const mockAttachUploadFile = jest.fn()
jest.mock('@/server/core/response', () => {
@ -57,12 +58,13 @@ jest.mock('@/app/intake/non-integrated/nonIntegratedAssetServer', () => ({
jest.mock('@/server/services', () => ({
intakeNonIntegratedProductService: {
assertCanWrite: jest.fn(),
assertCanWrite: (...args: unknown[]) => mockAssertCanWrite(...args),
attachUploadFile: (...args: unknown[]) => mockAttachUploadFile(...args),
},
}))
const { POST } = require('./route')
const { POST: POST_PRODUCT_UPLOAD } = require('../[id]/upload/route')
const { POST: POST_PRODUCT_UPLOAD_CHUNK } = require('../[id]/upload/chunk/route')
const { GET } = require('../assets/[...assetPath]/route')
@ -348,6 +350,153 @@ describe('POST /api/intake/non-integrated/upload', () => {
expect(Buffer.from(storedBytes).toString('utf8')).toBe('chunk-one/chunk-two')
})
it('accepts larger upload chunks and stores the merged asset', async () => {
const firstChunkBytes = new Uint8Array(600 * 1024).fill(65)
const secondChunkBytes = new TextEncoder().encode('/tail')
const totalSize = firstChunkBytes.length + secondChunkBytes.length
async function uploadDetachedChunk(chunkIndex: number, chunkBytes: Uint8Array) {
const formData = new FormData()
formData.append(
'chunk',
createUploadFile(chunkBytes, 'large-model.glb', 'model/gltf-binary'),
)
formData.append('uploadId', 'large-chunk-upload-test')
formData.append('chunkIndex', String(chunkIndex))
formData.append('totalChunks', '2')
formData.append('totalSize', String(totalSize))
formData.append('fileName', 'large-model.glb')
formData.append('mimeType', 'model/gltf-binary')
formData.append('attach', 'false')
return POST_PRODUCT_UPLOAD_CHUNK(
{
formData: async () => formData,
} as any,
{
params: Promise.resolve({ id: 'new-product-large-upload' }),
},
)
}
const firstResponse = await uploadDetachedChunk(0, firstChunkBytes)
expect(firstResponse.status).toBe(200)
expect(firstResponse.headers.get('X-Upload-Chunk-Status')).toBe('pending')
const completedResponse = await uploadDetachedChunk(1, secondChunkBytes)
const completedPayload = await completedResponse.json()
expect(completedResponse.status).toBe(201)
expect(completedPayload).toEqual(
expect.objectContaining({
success: true,
data: expect.objectContaining({
completed: true,
file: null,
fileName: 'large-model.glb',
mimeType: 'model/gltf-binary',
size: totalSize,
storageKey: expect.stringMatching(/\.glb$/),
}),
}),
)
const storedBytes = await readFile(
path.join(assetDirectory, completedPayload.data.storageKey),
)
expect(storedBytes).toHaveLength(totalSize)
expect(storedBytes[0]).toBe(65)
expect(storedBytes.subarray(-5).toString()).toBe('/tail')
})
it('allows detached product-route uploads before the product record exists', async () => {
const fileBytes = new TextEncoder().encode('new-product-image')
const formData = new FormData()
formData.append(
'file',
createUploadFile(fileBytes, 'opening-reference.png', 'image/png'),
)
formData.append('attach', 'false')
const uploadResponse = await POST_PRODUCT_UPLOAD(
{
formData: async () => formData,
} as any,
{
params: Promise.resolve({ id: 'PD20260507173941261-1778146788454' }),
},
)
const uploadPayload = await uploadResponse.json()
expect(uploadResponse.status).toBe(201)
expect(mockAssertCanWrite).not.toHaveBeenCalled()
expect(mockAttachUploadFile).not.toHaveBeenCalled()
expect(uploadPayload).toEqual(
expect.objectContaining({
success: true,
data: expect.objectContaining({
file: null,
fileName: 'opening-reference.png',
mimeType: 'image/png',
size: fileBytes.length,
storageKey: expect.stringMatching(/\.png$/),
}),
}),
)
})
it('allows detached chunked product-route uploads before the product record exists', async () => {
const fileBytes = new TextEncoder().encode('new-product/chunked-image')
async function uploadDetachedChunk(chunkIndex: number, chunkBytes: Uint8Array) {
const formData = new FormData()
formData.append(
'chunk',
createUploadFile(chunkBytes, 'opening-reference.png', 'image/png'),
)
formData.append('uploadId', 'new-product-upload-test')
formData.append('chunkIndex', String(chunkIndex))
formData.append('totalChunks', '2')
formData.append('totalSize', String(fileBytes.length))
formData.append('fileName', 'opening-reference.png')
formData.append('mimeType', 'image/png')
formData.append('attach', 'false')
return POST_PRODUCT_UPLOAD_CHUNK(
{
formData: async () => formData,
} as any,
{
params: Promise.resolve({ id: 'PD20260507173941261-1778146788454' }),
},
)
}
const firstResponse = await uploadDetachedChunk(0, fileBytes.slice(0, 12))
expect(firstResponse.status).toBe(200)
expect(firstResponse.headers.get('X-Upload-Chunk-Status')).toBe('pending')
const completedResponse = await uploadDetachedChunk(1, fileBytes.slice(12))
const completedPayload = await completedResponse.json()
expect(completedResponse.status).toBe(201)
expect(mockAssertCanWrite).not.toHaveBeenCalled()
expect(mockAttachUploadFile).not.toHaveBeenCalled()
expect(completedPayload).toEqual(
expect.objectContaining({
success: true,
data: expect.objectContaining({
completed: true,
file: null,
fileName: 'opening-reference.png',
mimeType: 'image/png',
size: fileBytes.length,
storageKey: expect.stringMatching(/\.png$/),
}),
}),
)
})
it('rejects files above the configured intake upload limit', async () => {
const uploadResponse = await POST({
formData: async () =>
@ -356,7 +505,7 @@ describe('POST /api/intake/non-integrated/upload', () => {
new TextEncoder().encode('too-large'),
'too-large.glb',
'model/gltf-binary',
501 * 1024 * 1024,
10241 * 1024 * 1024,
),
),
} as any)
@ -367,7 +516,7 @@ describe('POST /api/intake/non-integrated/upload', () => {
success: false,
error: expect.objectContaining({
code: 'FILE_003',
message: '文件过大，最大允许 500MB。',
message: '文件过大，最大允许 10GB。',
}),
}),
)

78
furniture-3d-system/src/app/api/intake/non-integrated/uploadHelpers.ts
View File

@ -56,11 +56,26 @@ export function shouldAttachUploadFile(formData: FormData) {
return normalized !== 'false' && normalized !== '0' && normalized !== 'no'
}
export async function storeNonIntegratedUploadFile(input: {
bytes: Buffer
export async function createNonIntegratedUploadTarget(originalFileName: string) {
const uploadDir = getNonIntegratedAssetDirectory()
if (!existsSync(uploadDir)) {
await mkdir(uploadDir, { recursive: true })
}
const extension = sanitizeUploadExtension(originalFileName)
const storedFileName = `${Date.now()}-${randomUUID()}${extension}`
return {
storedFileName,
storedFilePath: path.join(uploadDir, storedFileName),
}
}
export async function completeNonIntegratedStoredUploadFile(input: {
storedFileName: string
originalFileName: string
mimeType: string
size?: number
size: number
productId?: string
category?: string
fileId?: string
@ -68,29 +83,14 @@ export async function storeNonIntegratedUploadFile(input: {
shouldAttach?: boolean
user: AuthUser
}) {
const uploadDir = getNonIntegratedAssetDirectory()
if (!existsSync(uploadDir)) {
await mkdir(uploadDir, { recursive: true })
}
const extension = sanitizeUploadExtension(input.originalFileName)
const storedFileName = `${Date.now()}-${randomUUID()}${extension}`
const storedFilePath = path.join(uploadDir, storedFileName)
await writeFile(storedFilePath, input.bytes)
const url = buildNonIntegratedAssetUrl(storedFileName)
const url = buildNonIntegratedAssetUrl(input.storedFileName)
const mimeType = input.mimeType || 'application/octet-stream'
const fileSize =
typeof input.size === 'number' && Number.isFinite(input.size)
? input.size
: input.bytes.length
const responseBase = {
url,
fileName: input.originalFileName,
mimeType,
size: fileSize,
storageKey: storedFileName,
size: input.size,
storageKey: input.storedFileName,
}
if (!input.productId || input.shouldAttach === false) {
@ -110,8 +110,8 @@ export async function storeNonIntegratedUploadFile(input: {
category: input.category,
dataUrl: url,
mimeType,
size: fileSize,
storageKey: storedFileName,
size: input.size,
storageKey: input.storedFileName,
...(input.metadata ?? {}),
},
input.user,
@ -125,3 +125,35 @@ export async function storeNonIntegratedUploadFile(input: {
return responseData
}
export async function storeNonIntegratedUploadFile(input: {
bytes: Buffer
originalFileName: string
mimeType: string
size?: number
productId?: string
category?: string
fileId?: string
metadata?: Record<string, unknown>
shouldAttach?: boolean
user: AuthUser
}) {
const uploadTarget = await createNonIntegratedUploadTarget(input.originalFileName)
await writeFile(uploadTarget.storedFilePath, input.bytes)
return completeNonIntegratedStoredUploadFile({
storedFileName: uploadTarget.storedFileName,
originalFileName: input.originalFileName,
mimeType: input.mimeType,
size:
typeof input.size === 'number' && Number.isFinite(input.size)
? input.size
: input.bytes.length,
productId: input.productId,
category: input.category,
fileId: input.fileId,
metadata: input.metadata,
shouldAttach: input.shouldAttach,
user: input.user,
})
}

28
furniture-3d-system/src/app/api/intake/non-integrated/uploadLimit.ts
View File

@ -1,4 +1,5 @@
const DEFAULT_NON_INTEGRATED_UPLOAD_MAX_MB = 500
const DEFAULT_NON_INTEGRATED_UPLOAD_MAX_MB = 10 * 1024
const DEFAULT_NON_INTEGRATED_UPLOAD_CHUNK_MB = 10
function readConfiguredUploadLimitMb() {
const rawValue = process.env.NON_INTEGRATED_UPLOAD_MAX_MB
@ -11,12 +12,33 @@ function readConfiguredUploadLimitMb() {
return Math.floor(parsedValue)
}
function readConfiguredUploadChunkMb() {
const rawValue = process.env.NON_INTEGRATED_UPLOAD_CHUNK_MB
const parsedValue = Number(rawValue)
if (!Number.isFinite(parsedValue) || parsedValue <= 0) {
return DEFAULT_NON_INTEGRATED_UPLOAD_CHUNK_MB
}
return Math.floor(parsedValue)
}
export const NON_INTEGRATED_UPLOAD_MAX_MB = readConfiguredUploadLimitMb()
export const NON_INTEGRATED_UPLOAD_MAX_BYTES =
NON_INTEGRATED_UPLOAD_MAX_MB * 1024 * 1024
export const NON_INTEGRATED_UPLOAD_CHUNK_MAX_BYTES = 512 * 1024
export const NON_INTEGRATED_UPLOAD_RECOMMENDED_CHUNK_BYTES = 256 * 1024
export const NON_INTEGRATED_UPLOAD_CHUNK_MB = readConfiguredUploadChunkMb()
export const NON_INTEGRATED_UPLOAD_CHUNK_MAX_BYTES =
NON_INTEGRATED_UPLOAD_CHUNK_MB * 1024 * 1024
export const NON_INTEGRATED_UPLOAD_RECOMMENDED_CHUNK_BYTES =
NON_INTEGRATED_UPLOAD_CHUNK_MAX_BYTES
export function buildNonIntegratedFileTooLargeMessage() {
if (
NON_INTEGRATED_UPLOAD_MAX_MB >= 1024 &&
NON_INTEGRATED_UPLOAD_MAX_MB % 1024 === 0
) {
return `文件过大，最大允许 ${NON_INTEGRATED_UPLOAD_MAX_MB / 1024}GB。`
}
return `文件过大，最大允许 ${NON_INTEGRATED_UPLOAD_MAX_MB}MB。`
}

2
furniture-3d-system/src/app/intake/non-integrated/nonIntegratedApi.ts
View File

@ -8,7 +8,7 @@ type ApiEnvelope<T> = {
}
}
const NON_INTEGRATED_UPLOAD_CHUNK_BYTES = 128 * 1024
const NON_INTEGRATED_UPLOAD_CHUNK_BYTES = 10 * 1024 * 1024
type UploadSharedNonIntegratedFileResponse = {
product: NonIntegratedProductRecord