From 7a5dce6988604affa2393f42fb248f3696f6e449 Mon Sep 17 00:00:00 2001 From: WuYongJie Date: Mon, 8 Jun 2026 11:32:25 +0800 Subject: [PATCH] =?UTF-8?q?fix(storage):=20=E9=9D=9E=E9=9B=86=E6=88=90?= =?UTF-8?q?=E5=8E=86=E5=8F=B2=E7=B4=A0=E6=9D=90=E6=94=B9=E4=B8=BA=E5=AF=B9?= =?UTF-8?q?=E8=B1=A1=E5=AD=98=E5=82=A8=E4=BC=98=E5=85=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../non-integrated/[id]/upload/chunk/route.ts | 11 +- .../assets/[...assetPath]/route.ts | 50 +++-- .../non-integrated/upload/route.test.ts | 175 ++++++++++++++++-- .../intake/non-integrated/uploadHelpers.ts | 31 +++- .../nonIntegratedAssetServer.ts | 57 +++++- 5 files changed, 288 insertions(+), 36 deletions(-) diff --git a/furniture-3d-system/src/app/api/intake/non-integrated/[id]/upload/chunk/route.ts b/furniture-3d-system/src/app/api/intake/non-integrated/[id]/upload/chunk/route.ts index 5e68a0ad..b6c761a6 100644 --- a/furniture-3d-system/src/app/api/intake/non-integrated/[id]/upload/chunk/route.ts +++ b/furniture-3d-system/src/app/api/intake/non-integrated/[id]/upload/chunk/route.ts @@ -1,5 +1,5 @@ import { createReadStream, createWriteStream, existsSync } from 'fs' -import { mkdir, rm, stat, writeFile } from 'fs/promises' +import { mkdir, readFile, rm, stat, writeFile } from 'fs/promises' import os from 'os' import path from 'path' import { pipeline } from 'stream/promises' @@ -13,6 +13,7 @@ import { readOptionalFormValue, readUploadMetadata, shouldAttachUploadFile, + uploadNonIntegratedStoredBytesToObjectStorage, } from '../../../uploadHelpers' import { NON_INTEGRATED_UPLOAD_CHUNK_MAX_BYTES, @@ -203,6 +204,13 @@ export async function POST(request: NextRequest, context: RouteContext) { return jsonError(ErrorCode.FILE_UPLOAD_FAILED, '上传文件大小校验失败，请重新上传', 400, requestId) } + const mergedBytes = await readFile(uploadTarget.storedFilePath) + await uploadNonIntegratedStoredBytesToObjectStorage({ + storedFileName: uploadTarget.storedFileName, + bytes: mergedBytes, + mimeType, + }) + const responseData = await completeNonIntegratedStoredUploadFile({ storedFileName: uploadTarget.storedFileName, originalFileName: fileName, @@ -216,6 +224,7 @@ export async function POST(request: NextRequest, context: RouteContext) { user, }) + await rm(uploadTarget.storedFilePath, { force: true }) await rm(sessionDirectory, { recursive: true, force: true }) return jsonSuccess( diff --git a/furniture-3d-system/src/app/api/intake/non-integrated/assets/[...assetPath]/route.ts b/furniture-3d-system/src/app/api/intake/non-integrated/assets/[...assetPath]/route.ts index eb1b73c5..129483ec 100644 --- a/furniture-3d-system/src/app/api/intake/non-integrated/assets/[...assetPath]/route.ts +++ b/furniture-3d-system/src/app/api/intake/non-integrated/assets/[...assetPath]/route.ts @@ -3,10 +3,18 @@ import { stat } from "fs/promises"; import path from "path"; import { Readable } from "stream"; import { NextRequest, NextResponse } from "next/server"; -import { resolveNonIntegratedAssetRelativeFilePath } from "@/app/intake/non-integrated/nonIntegratedAssetServer"; +import { + buildNonIntegratedAssetObjectStorageKey, + resolveNonIntegratedAssetRelativeFilePath, +} from "@/app/intake/non-integrated/nonIntegratedAssetServer"; import { AppError, ErrorCode } from "@/server/core/errors"; import { handleRequest, jsonError } from "@/server/core/response"; import { intakeNonIntegratedProductService } from "@/server/services/intake-non-integrated-product.service"; +import { createObjectReadUrl, doesObjectExist } from "@/lib/storage"; +import { + buildObjectReadResponse, + ObjectReadProxyError, +} from "@/lib/object-storage-read-response"; export const runtime = "nodejs"; @@ -124,19 +132,39 @@ export async function GET(request: NextRequest, context: RouteContext) { } } + const objectStorageKey = + buildNonIntegratedAssetObjectStorageKey(relativePath); + let objectStorageReadError: ObjectReadProxyError | null = null; + + if (objectStorageKey) { + try { + if (await doesObjectExist(objectStorageKey)) { + return (await buildObjectReadResponse( + createObjectReadUrl(objectStorageKey), + request, + )) as unknown as NextResponse; + } + } catch (error) { + if (error instanceof ObjectReadProxyError) { + objectStorageReadError = error; + } else { + throw error; + } + } + } + const resolvedPath = resolveNonIntegratedAssetRelativeFilePath(relativePath); - if (!resolvedPath) { - return jsonError( - ErrorCode.FILE_NOT_FOUND, - "The requested asset was not found.", - 404, - requestId, - ); - } - - if (!existsSync(resolvedPath)) { + if (!resolvedPath || !existsSync(resolvedPath)) { + if (objectStorageReadError) { + return jsonError( + ErrorCode.SYSTEM_STORAGE, + "瀵硅薄瀛樺偍璇诲彇澶辫触", + 502, + requestId, + ); + } return jsonError( ErrorCode.FILE_NOT_FOUND, "The requested asset was not found.", diff --git a/furniture-3d-system/src/app/api/intake/non-integrated/upload/route.test.ts b/furniture-3d-system/src/app/api/intake/non-integrated/upload/route.test.ts index e17ab964..d607df5d 100644 --- a/furniture-3d-system/src/app/api/intake/non-integrated/upload/route.test.ts +++ b/furniture-3d-system/src/app/api/intake/non-integrated/upload/route.test.ts @@ -1,6 +1,6 @@ export {} -import { readFile, rm } from 'fs/promises' +import { mkdir, rm, writeFile } from 'fs/promises' import os from 'os' import path from 'path' import { afterEach, beforeEach, describe, expect, it, jest } from '@jest/globals' @@ -13,6 +13,34 @@ const mockAssertCanWrite = jest.fn() const mockAssertCanUploadDetachedAsset = jest.fn() const mockAssertCanDownload = jest.fn() const mockAttachUploadFile = jest.fn() +const uploadedObjectBodies = new Map() + +function buildMockObjectStorageKey(relativePath: string) { + return `prod/non-integrated-assets/${relativePath.replace(/^\/+/, '')}` +} + +class MockObjectReadProxyError extends Error { + constructor( + message: string, + public readonly statusCode: number, + ) { + super(message) + this.name = 'ObjectReadProxyError' + } +} + +function getContentTypeFromObjectKey(key: string) { + if (key.endsWith('.pdf')) return 'application/pdf' + if (key.endsWith('.mp4')) return 'video/mp4' + if (key.endsWith('.png')) return 'image/png' + if (key.endsWith('.zip')) return 'application/zip' + if (key.endsWith('.glb')) return 'model/gltf-binary' + return 'application/octet-stream' +} + +function readUploadedObjectBody(relativeStorageKey: string) { + return uploadedObjectBodies.get(buildMockObjectStorageKey(relativeStorageKey)) +} jest.mock('@/server/core/response', () => { return { @@ -58,10 +86,67 @@ jest.mock('@/server/core/response', () => { jest.mock('@/app/intake/non-integrated/nonIntegratedAssetServer', () => ({ getNonIntegratedAssetDirectory: (...args: unknown[]) => mockGetNonIntegratedAssetDirectory(...args), + buildNonIntegratedAssetObjectStorageKey: (relativePath: string) => + buildMockObjectStorageKey(relativePath), resolveNonIntegratedAssetRelativeFilePath: (relativePath: string) => path.join(mockGetNonIntegratedAssetDirectory(), relativePath), })) +jest.mock('@/lib/storage', () => ({ + uploadBufferToObjectStorage: async ( + buffer: Buffer, + key: string, + ) => { + uploadedObjectBodies.set(String(key), Buffer.from(buffer)) + return `/api/storage/object?key=${encodeURIComponent(String(key))}` + }, + createObjectReadUrl: (key: string) => + `https://storage.example.com/read?key=${encodeURIComponent(String(key))}`, + doesObjectExist: async (key: string) => uploadedObjectBodies.has(String(key)), +})) + +jest.mock('@/lib/object-storage-read-response', () => ({ + ObjectReadProxyError: MockObjectReadProxyError, + buildObjectReadResponse: async (readUrl: string, request?: { headers?: { get?: (name: string) => string | null } }) => { + const parsed = new URL(readUrl) + const key = parsed.searchParams.get('key') || '' + const body = uploadedObjectBodies.get(key) + if (!body) { + throw new MockObjectReadProxyError('对象不存在', 404) + } + + const rangeHeader = request?.headers?.get?.('range') ?? null + const contentType = getContentTypeFromObjectKey(key) + if (rangeHeader) { + const match = /^bytes=(\d+)-(\d+)?$/.exec(rangeHeader) + if (match) { + const start = Number(match[1]) + const requestedEnd = match[2] ? Number(match[2]) : body.length - 1 + const end = Math.min(requestedEnd, body.length - 1) + const slice = body.subarray(start, end + 1) + return new Response(slice, { + status: 206, + headers: { + 'content-type': contentType, + 'content-length': String(slice.length), + 'accept-ranges': 'bytes', + 'content-range': `bytes ${start}-${end}/${body.length}`, + }, + }) + } + } + + return new Response(body, { + status: 200, + headers: { + 'content-type': contentType, + 'content-length': String(body.length), + 'accept-ranges': 'bytes', + }, + }) + }, +})) + jest.mock('@/server/services', () => ({ intakeNonIntegratedProductService: { assertCanCreate: (...args: unknown[]) => mockAssertCanCreate(...args), @@ -124,8 +209,10 @@ async function readResponseBody(response: Response) { if (!body) return Buffer.from([]) const chunks: Buffer[] = [] - for await (const chunk of body as unknown as AsyncIterable) { - chunks.push(Buffer.from(chunk)) + for await (const chunk of body as unknown as AsyncIterable) { + chunks.push( + typeof chunk === 'number' ? Buffer.from([chunk]) : Buffer.from(chunk), + ) } return Buffer.concat(chunks) } @@ -133,6 +220,8 @@ async function readResponseBody(response: Response) { function getResponseHeader(response: Response, name: string) { return ( response.headers.get(name) ?? + response.headers.get(name.toLowerCase()) ?? + response.headers.get(name.toUpperCase()) ?? ((response.headers as unknown as Record)[name] || (response.headers as unknown as Record)[ name.toLowerCase() @@ -145,6 +234,7 @@ describe('POST /api/intake/non-integrated/upload', () => { beforeEach(async () => { jest.clearAllMocks() + uploadedObjectBodies.clear() assetDirectory = path.join( os.tmpdir(), @@ -203,10 +293,9 @@ describe('POST /api/intake/non-integrated/upload', () => { }), ) - const storedBytes = await readFile( - path.join(assetDirectory, uploadPayload.data.storageKey), - ) - expect(Buffer.from(storedBytes).toString('utf8')).toBe('patent-certificate') + expect( + readUploadedObjectBody(uploadPayload.data.storageKey)?.toString('utf8'), + ).toBe('patent-certificate') const downloadResponse = await GET( { @@ -229,6 +318,63 @@ describe('POST /api/intake/non-integrated/upload', () => { expect(downloadedBytes.toString('utf8')).toBe('patent-certificate') }) + it('prefers object storage over a legacy local file when both are present', async () => { + const relativeStorageKey = + '1780710044498-fe09c3fa-d272-4842-9399-0bc0f1665781.png' + const localPath = path.join(assetDirectory, relativeStorageKey) + + await mkdir(path.dirname(localPath), { recursive: true }) + await writeFile(localPath, Buffer.from('local-legacy-body')) + uploadedObjectBodies.set( + buildMockObjectStorageKey(relativeStorageKey), + Buffer.from('object-storage-body'), + ) + + const downloadResponse = await GET( + { + headers: { + get: () => null, + }, + } as any, + { + params: Promise.resolve({ + assetPath: [relativeStorageKey], + }), + }, + ) + const downloadedBytes = await readResponseBody(downloadResponse) + + expect(downloadResponse.status).toBe(200) + expect(getResponseHeader(downloadResponse, 'Content-Type')).toBe('image/png') + expect(downloadedBytes.toString('utf8')).toBe('object-storage-body') + }) + + it('falls back to the legacy local file when object storage is unavailable', async () => { + const relativeStorageKey = 'legacy-only.png' + const localPath = path.join(assetDirectory, relativeStorageKey) + + await mkdir(path.dirname(localPath), { recursive: true }) + await writeFile(localPath, Buffer.from('local-legacy-body')) + + const downloadResponse = await GET( + { + headers: { + get: () => null, + }, + } as any, + { + params: Promise.resolve({ + assetPath: [relativeStorageKey], + }), + }, + ) + const downloadedBytes = await readResponseBody(downloadResponse) + + expect(downloadResponse.status).toBe(200) + expect(getResponseHeader(downloadResponse, 'Content-Type')).toBe('image/png') + expect(downloadedBytes.toString('utf8')).toBe('local-legacy-body') + }) + it('checks product and node download permission for scoped asset paths', async () => { const certificateBytes = new TextEncoder().encode('scoped-certificate') const formData = createFormData( @@ -466,10 +612,9 @@ describe('POST /api/intake/non-integrated/upload', () => { expect.objectContaining({ id: 'admin-1' }), ) - const storedBytes = await readFile( - path.join(assetDirectory, completedPayload.data.storageKey), - ) - expect(Buffer.from(storedBytes).toString('utf8')).toBe('chunk-one/chunk-two') + expect( + readUploadedObjectBody(completedPayload.data.storageKey)?.toString('utf8'), + ).toBe('chunk-one/chunk-two') }) it('accepts larger upload chunks and stores the merged asset', async () => { @@ -524,12 +669,10 @@ describe('POST /api/intake/non-integrated/upload', () => { }), ) - const storedBytes = await readFile( - path.join(assetDirectory, completedPayload.data.storageKey), - ) + const storedBytes = readUploadedObjectBody(completedPayload.data.storageKey) expect(storedBytes).toHaveLength(totalSize) - expect(storedBytes[0]).toBe(65) - expect(storedBytes.subarray(-5).toString()).toBe('/tail') + expect(storedBytes?.[0]).toBe(65) + expect(storedBytes?.subarray(-5).toString()).toBe('/tail') }) it('rejects detached chunked product-route uploads when detached upload permission fails', async () => { diff --git a/furniture-3d-system/src/app/api/intake/non-integrated/uploadHelpers.ts b/furniture-3d-system/src/app/api/intake/non-integrated/uploadHelpers.ts index bf4ac822..a67771cf 100644 --- a/furniture-3d-system/src/app/api/intake/non-integrated/uploadHelpers.ts +++ b/furniture-3d-system/src/app/api/intake/non-integrated/uploadHelpers.ts @@ -1,13 +1,17 @@ import { existsSync } from 'fs' -import { mkdir, writeFile } from 'fs/promises' +import { mkdir } from 'fs/promises' import path from 'path' import { randomUUID } from 'crypto' import { buildNonIntegratedAssetUrl } from '@/app/intake/non-integrated/nonIntegratedAssetUrl' -import { getNonIntegratedAssetDirectory } from '@/app/intake/non-integrated/nonIntegratedAssetServer' +import { + buildNonIntegratedAssetObjectStorageKey, + getNonIntegratedAssetDirectory, +} from '@/app/intake/non-integrated/nonIntegratedAssetServer' import type { NonIntegratedProductRecord, NonIntegratedUploadFile, } from '@/app/intake/non-integrated/nonIntegratedStorage' +import { uploadBufferToObjectStorage } from '@/lib/storage' import { intakeNonIntegratedProductService } from '@/server/services/intake-non-integrated-product.service' import type { AuthUser } from '@/server/core/types' @@ -153,6 +157,23 @@ export async function completeNonIntegratedStoredUploadFile(input: { return responseData } +export async function uploadNonIntegratedStoredBytesToObjectStorage(input: { + storedFileName: string + bytes: Buffer + mimeType: string +}) { + const objectKey = buildNonIntegratedAssetObjectStorageKey(input.storedFileName) + if (!objectKey) { + throw new Error('非集成素材对象 key 生成失败') + } + + await uploadBufferToObjectStorage( + input.bytes, + objectKey, + input.mimeType || 'application/octet-stream', + ) +} + export async function storeNonIntegratedUploadFile(input: { bytes: Buffer originalFileName: string @@ -170,7 +191,11 @@ export async function storeNonIntegratedUploadFile(input: { productId: input.productId, category: input.category, }) - await writeFile(uploadTarget.storedFilePath, input.bytes) + await uploadNonIntegratedStoredBytesToObjectStorage({ + storedFileName: uploadTarget.storedFileName, + bytes: input.bytes, + mimeType: input.mimeType, + }) return completeNonIntegratedStoredUploadFile({ storedFileName: uploadTarget.storedFileName, diff --git a/furniture-3d-system/src/app/intake/non-integrated/nonIntegratedAssetServer.ts b/furniture-3d-system/src/app/intake/non-integrated/nonIntegratedAssetServer.ts index e2fc007a..ae2c7c89 100644 --- a/furniture-3d-system/src/app/intake/non-integrated/nonIntegratedAssetServer.ts +++ b/furniture-3d-system/src/app/intake/non-integrated/nonIntegratedAssetServer.ts @@ -1,6 +1,6 @@ import { randomUUID } from "crypto"; import { existsSync } from "fs"; -import { mkdir, writeFile } from "fs/promises"; +import { mkdir, readFile, writeFile } from "fs/promises"; import path from "path"; import { NON_INTEGRATED_ASSET_DIRECTORY_NAME, @@ -13,6 +13,7 @@ import { createObjectReadUrl, normalizeObjectStorageKey } from "@/lib/storage"; const NON_INTEGRATED_PRIVATE_ASSET_DIRECTORY_NAME = "non-integrated-assets"; const OBJECT_STORAGE_ROUTE_PATH = "/api/storage/object"; +const NON_INTEGRATED_OBJECT_STORAGE_PREFIX = "non-integrated-assets"; export type MaterializedNonIntegratedAssetFile = { filePath: string; @@ -79,6 +80,15 @@ export function resolveNonIntegratedAssetFilePath( return resolveNonIntegratedAssetRelativeFilePath(relativePath, cwd); } +export function buildNonIntegratedAssetObjectStorageKey(relativePath: string) { + const normalizedRelativePath = relativePath.trim().replace(/^\/+/, ""); + if (!normalizedRelativePath) return ""; + + return normalizeObjectStorageKey( + `${NON_INTEGRATED_OBJECT_STORAGE_PREFIX}/${normalizedRelativePath}`, + ); +} + export function extractObjectStorageKeyFromUrl(value: string) { const normalizedValue = value.trim(); if (!normalizedValue) return ""; @@ -98,6 +108,15 @@ export function extractObjectStorageKeyFromUrl(value: string) { } } +export function extractNonIntegratedAssetObjectStorageKey(fileUrl: string) { + const relativePath = extractNonIntegratedAssetRelativePath(fileUrl); + if (relativePath) { + return buildNonIntegratedAssetObjectStorageKey(relativePath); + } + + return extractObjectStorageKeyFromUrl(fileUrl); +} + function buildSafeMaterializedFileName(objectKey: string) { const extension = path.extname(objectKey).replace(/[^a-zA-Z0-9.]/g, ""); const baseName = @@ -110,6 +129,34 @@ function buildSafeMaterializedFileName(objectKey: string) { return `${baseName}-${randomUUID()}${extension}`; } +async function fetchObjectStorageBuffer(objectKey: string) { + const response = await fetch(createObjectReadUrl(objectKey), { + cache: "no-store", + }); + if (!response.ok) { + return null; + } + + return Buffer.from(await response.arrayBuffer()); +} + +export async function readNonIntegratedAssetBuffer( + fileUrl: string, + cwd = process.cwd(), +): Promise { + const localPath = resolveNonIntegratedAssetFilePath(fileUrl, cwd); + if (localPath && existsSync(localPath)) { + return readFile(localPath); + } + + const objectKey = extractNonIntegratedAssetObjectStorageKey(fileUrl); + if (!objectKey) { + return null; + } + + return fetchObjectStorageBuffer(objectKey); +} + export async function materializeNonIntegratedAssetFile( fileUrl: string, tempDirectory: string, @@ -124,13 +171,13 @@ export async function materializeNonIntegratedAssetFile( }; } - const objectKey = extractObjectStorageKeyFromUrl(fileUrl); + const objectKey = extractNonIntegratedAssetObjectStorageKey(fileUrl); if (!objectKey) { return null; } - const response = await fetch(createObjectReadUrl(objectKey)); - if (!response.ok) { + const buffer = await fetchObjectStorageBuffer(objectKey); + if (!buffer) { return null; } @@ -139,7 +186,7 @@ export async function materializeNonIntegratedAssetFile( tempDirectory, buildSafeMaterializedFileName(objectKey), ); - await writeFile(filePath, Buffer.from(await response.arrayBuffer())); + await writeFile(filePath, buffer); return { filePath,