fix(storage): 非集成历史素材改为对象存储优先
This commit is contained in:
WuYongJie
parent
768b988843
commit
7a5dce6988
@ -1,5 +1,5 @@
|
||||
import { createReadStream, createWriteStream, existsSync } from 'fs'
|
||||
import { mkdir, rm, stat, writeFile } from 'fs/promises'
|
||||
import { mkdir, readFile, rm, stat, writeFile } from 'fs/promises'
|
||||
import os from 'os'
|
||||
import path from 'path'
|
||||
import { pipeline } from 'stream/promises'
|
||||
@ -13,6 +13,7 @@ import {
|
||||
readOptionalFormValue,
|
||||
readUploadMetadata,
|
||||
shouldAttachUploadFile,
|
||||
uploadNonIntegratedStoredBytesToObjectStorage,
|
||||
} from '../../../uploadHelpers'
|
||||
import {
|
||||
NON_INTEGRATED_UPLOAD_CHUNK_MAX_BYTES,
|
||||
@ -203,6 +204,13 @@ export async function POST(request: NextRequest, context: RouteContext) {
|
||||
return jsonError(ErrorCode.FILE_UPLOAD_FAILED, '上传文件大小校验失败，请重新上传', 400, requestId)
|
||||
}
|
||||
|
||||
const mergedBytes = await readFile(uploadTarget.storedFilePath)
|
||||
await uploadNonIntegratedStoredBytesToObjectStorage({
|
||||
storedFileName: uploadTarget.storedFileName,
|
||||
bytes: mergedBytes,
|
||||
mimeType,
|
||||
})
|
||||
|
||||
const responseData = await completeNonIntegratedStoredUploadFile({
|
||||
storedFileName: uploadTarget.storedFileName,
|
||||
originalFileName: fileName,
|
||||
@ -216,6 +224,7 @@ export async function POST(request: NextRequest, context: RouteContext) {
|
||||
user,
|
||||
})
|
||||
|
||||
await rm(uploadTarget.storedFilePath, { force: true })
|
||||
await rm(sessionDirectory, { recursive: true, force: true })
|
||||
|
||||
return jsonSuccess(
|
||||
|
||||
@ -3,10 +3,18 @@ import { stat } from "fs/promises";
|
||||
import path from "path";
|
||||
import { Readable } from "stream";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { resolveNonIntegratedAssetRelativeFilePath } from "@/app/intake/non-integrated/nonIntegratedAssetServer";
|
||||
import {
|
||||
buildNonIntegratedAssetObjectStorageKey,
|
||||
resolveNonIntegratedAssetRelativeFilePath,
|
||||
} from "@/app/intake/non-integrated/nonIntegratedAssetServer";
|
||||
import { AppError, ErrorCode } from "@/server/core/errors";
|
||||
import { handleRequest, jsonError } from "@/server/core/response";
|
||||
import { intakeNonIntegratedProductService } from "@/server/services/intake-non-integrated-product.service";
|
||||
import { createObjectReadUrl, doesObjectExist } from "@/lib/storage";
|
||||
import {
|
||||
buildObjectReadResponse,
|
||||
ObjectReadProxyError,
|
||||
} from "@/lib/object-storage-read-response";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
|
||||
@ -124,19 +132,39 @@ export async function GET(request: NextRequest, context: RouteContext) {
|
||||
}
|
||||
}
|
||||
|
||||
const objectStorageKey =
|
||||
buildNonIntegratedAssetObjectStorageKey(relativePath);
|
||||
let objectStorageReadError: ObjectReadProxyError | null = null;
|
||||
|
||||
if (objectStorageKey) {
|
||||
try {
|
||||
if (await doesObjectExist(objectStorageKey)) {
|
||||
return (await buildObjectReadResponse(
|
||||
createObjectReadUrl(objectStorageKey),
|
||||
request,
|
||||
)) as unknown as NextResponse;
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof ObjectReadProxyError) {
|
||||
objectStorageReadError = error;
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const resolvedPath =
|
||||
resolveNonIntegratedAssetRelativeFilePath(relativePath);
|
||||
|
||||
if (!resolvedPath) {
|
||||
if (!resolvedPath || !existsSync(resolvedPath)) {
|
||||
if (objectStorageReadError) {
|
||||
return jsonError(
|
||||
ErrorCode.FILE_NOT_FOUND,
|
||||
"The requested asset was not found.",
|
||||
404,
|
||||
ErrorCode.SYSTEM_STORAGE,
|
||||
"瀵硅薄瀛樺偍璇诲彇澶辫触",
|
||||
502,
|
||||
requestId,
|
||||
);
|
||||
}
|
||||
|
||||
if (!existsSync(resolvedPath)) {
|
||||
return jsonError(
|
||||
ErrorCode.FILE_NOT_FOUND,
|
||||
"The requested asset was not found.",
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
export {}
|
||||
|
||||
import { readFile, rm } from 'fs/promises'
|
||||
import { mkdir, rm, writeFile } from 'fs/promises'
|
||||
import os from 'os'
|
||||
import path from 'path'
|
||||
import { afterEach, beforeEach, describe, expect, it, jest } from '@jest/globals'
|
||||
@ -13,6 +13,34 @@ const mockAssertCanWrite = jest.fn()
|
||||
const mockAssertCanUploadDetachedAsset = jest.fn()
|
||||
const mockAssertCanDownload = jest.fn()
|
||||
const mockAttachUploadFile = jest.fn()
|
||||
const uploadedObjectBodies = new Map<string, Buffer>()
|
||||
|
||||
function buildMockObjectStorageKey(relativePath: string) {
|
||||
return `prod/non-integrated-assets/${relativePath.replace(/^\/+/, '')}`
|
||||
}
|
||||
|
||||
class MockObjectReadProxyError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly statusCode: number,
|
||||
) {
|
||||
super(message)
|
||||
this.name = 'ObjectReadProxyError'
|
||||
}
|
||||
}
|
||||
|
||||
function getContentTypeFromObjectKey(key: string) {
|
||||
if (key.endsWith('.pdf')) return 'application/pdf'
|
||||
if (key.endsWith('.mp4')) return 'video/mp4'
|
||||
if (key.endsWith('.png')) return 'image/png'
|
||||
if (key.endsWith('.zip')) return 'application/zip'
|
||||
if (key.endsWith('.glb')) return 'model/gltf-binary'
|
||||
return 'application/octet-stream'
|
||||
}
|
||||
|
||||
function readUploadedObjectBody(relativeStorageKey: string) {
|
||||
return uploadedObjectBodies.get(buildMockObjectStorageKey(relativeStorageKey))
|
||||
}
|
||||
|
||||
jest.mock('@/server/core/response', () => {
|
||||
return {
|
||||
@ -58,10 +86,67 @@ jest.mock('@/server/core/response', () => {
|
||||
jest.mock('@/app/intake/non-integrated/nonIntegratedAssetServer', () => ({
|
||||
getNonIntegratedAssetDirectory: (...args: unknown[]) =>
|
||||
mockGetNonIntegratedAssetDirectory(...args),
|
||||
buildNonIntegratedAssetObjectStorageKey: (relativePath: string) =>
|
||||
buildMockObjectStorageKey(relativePath),
|
||||
resolveNonIntegratedAssetRelativeFilePath: (relativePath: string) =>
|
||||
path.join(mockGetNonIntegratedAssetDirectory(), relativePath),
|
||||
}))
|
||||
|
||||
jest.mock('@/lib/storage', () => ({
|
||||
uploadBufferToObjectStorage: async (
|
||||
buffer: Buffer,
|
||||
key: string,
|
||||
) => {
|
||||
uploadedObjectBodies.set(String(key), Buffer.from(buffer))
|
||||
return `/api/storage/object?key=${encodeURIComponent(String(key))}`
|
||||
},
|
||||
createObjectReadUrl: (key: string) =>
|
||||
`https://storage.example.com/read?key=${encodeURIComponent(String(key))}`,
|
||||
doesObjectExist: async (key: string) => uploadedObjectBodies.has(String(key)),
|
||||
}))
|
||||
|
||||
jest.mock('@/lib/object-storage-read-response', () => ({
|
||||
ObjectReadProxyError: MockObjectReadProxyError,
|
||||
buildObjectReadResponse: async (readUrl: string, request?: { headers?: { get?: (name: string) => string | null } }) => {
|
||||
const parsed = new URL(readUrl)
|
||||
const key = parsed.searchParams.get('key') || ''
|
||||
const body = uploadedObjectBodies.get(key)
|
||||
if (!body) {
|
||||
throw new MockObjectReadProxyError('对象不存在', 404)
|
||||
}
|
||||
|
||||
const rangeHeader = request?.headers?.get?.('range') ?? null
|
||||
const contentType = getContentTypeFromObjectKey(key)
|
||||
if (rangeHeader) {
|
||||
const match = /^bytes=(\d+)-(\d+)?$/.exec(rangeHeader)
|
||||
if (match) {
|
||||
const start = Number(match[1])
|
||||
const requestedEnd = match[2] ? Number(match[2]) : body.length - 1
|
||||
const end = Math.min(requestedEnd, body.length - 1)
|
||||
const slice = body.subarray(start, end + 1)
|
||||
return new Response(slice, {
|
||||
status: 206,
|
||||
headers: {
|
||||
'content-type': contentType,
|
||||
'content-length': String(slice.length),
|
||||
'accept-ranges': 'bytes',
|
||||
'content-range': `bytes ${start}-${end}/${body.length}`,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return new Response(body, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'content-type': contentType,
|
||||
'content-length': String(body.length),
|
||||
'accept-ranges': 'bytes',
|
||||
},
|
||||
})
|
||||
},
|
||||
}))
|
||||
|
||||
jest.mock('@/server/services', () => ({
|
||||
intakeNonIntegratedProductService: {
|
||||
assertCanCreate: (...args: unknown[]) => mockAssertCanCreate(...args),
|
||||
@ -124,8 +209,10 @@ async function readResponseBody(response: Response) {
|
||||
if (!body) return Buffer.from([])
|
||||
|
||||
const chunks: Buffer[] = []
|
||||
for await (const chunk of body as unknown as AsyncIterable<Uint8Array>) {
|
||||
chunks.push(Buffer.from(chunk))
|
||||
for await (const chunk of body as unknown as AsyncIterable<Uint8Array | number>) {
|
||||
chunks.push(
|
||||
typeof chunk === 'number' ? Buffer.from([chunk]) : Buffer.from(chunk),
|
||||
)
|
||||
}
|
||||
return Buffer.concat(chunks)
|
||||
}
|
||||
@ -133,6 +220,8 @@ async function readResponseBody(response: Response) {
|
||||
function getResponseHeader(response: Response, name: string) {
|
||||
return (
|
||||
response.headers.get(name) ??
|
||||
response.headers.get(name.toLowerCase()) ??
|
||||
response.headers.get(name.toUpperCase()) ??
|
||||
((response.headers as unknown as Record<string, string>)[name] ||
|
||||
(response.headers as unknown as Record<string, string>)[
|
||||
name.toLowerCase()
|
||||
@ -145,6 +234,7 @@ describe('POST /api/intake/non-integrated/upload', () => {
|
||||
|
||||
beforeEach(async () => {
|
||||
jest.clearAllMocks()
|
||||
uploadedObjectBodies.clear()
|
||||
|
||||
assetDirectory = path.join(
|
||||
os.tmpdir(),
|
||||
@ -203,10 +293,9 @@ describe('POST /api/intake/non-integrated/upload', () => {
|
||||
}),
|
||||
)
|
||||
|
||||
const storedBytes = await readFile(
|
||||
path.join(assetDirectory, uploadPayload.data.storageKey),
|
||||
)
|
||||
expect(Buffer.from(storedBytes).toString('utf8')).toBe('patent-certificate')
|
||||
expect(
|
||||
readUploadedObjectBody(uploadPayload.data.storageKey)?.toString('utf8'),
|
||||
).toBe('patent-certificate')
|
||||
|
||||
const downloadResponse = await GET(
|
||||
{
|
||||
@ -229,6 +318,63 @@ describe('POST /api/intake/non-integrated/upload', () => {
|
||||
expect(downloadedBytes.toString('utf8')).toBe('patent-certificate')
|
||||
})
|
||||
|
||||
it('prefers object storage over a legacy local file when both are present', async () => {
|
||||
const relativeStorageKey =
|
||||
'1780710044498-fe09c3fa-d272-4842-9399-0bc0f1665781.png'
|
||||
const localPath = path.join(assetDirectory, relativeStorageKey)
|
||||
|
||||
await mkdir(path.dirname(localPath), { recursive: true })
|
||||
await writeFile(localPath, Buffer.from('local-legacy-body'))
|
||||
uploadedObjectBodies.set(
|
||||
buildMockObjectStorageKey(relativeStorageKey),
|
||||
Buffer.from('object-storage-body'),
|
||||
)
|
||||
|
||||
const downloadResponse = await GET(
|
||||
{
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
} as any,
|
||||
{
|
||||
params: Promise.resolve({
|
||||
assetPath: [relativeStorageKey],
|
||||
}),
|
||||
},
|
||||
)
|
||||
const downloadedBytes = await readResponseBody(downloadResponse)
|
||||
|
||||
expect(downloadResponse.status).toBe(200)
|
||||
expect(getResponseHeader(downloadResponse, 'Content-Type')).toBe('image/png')
|
||||
expect(downloadedBytes.toString('utf8')).toBe('object-storage-body')
|
||||
})
|
||||
|
||||
it('falls back to the legacy local file when object storage is unavailable', async () => {
|
||||
const relativeStorageKey = 'legacy-only.png'
|
||||
const localPath = path.join(assetDirectory, relativeStorageKey)
|
||||
|
||||
await mkdir(path.dirname(localPath), { recursive: true })
|
||||
await writeFile(localPath, Buffer.from('local-legacy-body'))
|
||||
|
||||
const downloadResponse = await GET(
|
||||
{
|
||||
headers: {
|
||||
get: () => null,
|
||||
},
|
||||
} as any,
|
||||
{
|
||||
params: Promise.resolve({
|
||||
assetPath: [relativeStorageKey],
|
||||
}),
|
||||
},
|
||||
)
|
||||
const downloadedBytes = await readResponseBody(downloadResponse)
|
||||
|
||||
expect(downloadResponse.status).toBe(200)
|
||||
expect(getResponseHeader(downloadResponse, 'Content-Type')).toBe('image/png')
|
||||
expect(downloadedBytes.toString('utf8')).toBe('local-legacy-body')
|
||||
})
|
||||
|
||||
it('checks product and node download permission for scoped asset paths', async () => {
|
||||
const certificateBytes = new TextEncoder().encode('scoped-certificate')
|
||||
const formData = createFormData(
|
||||
@ -466,10 +612,9 @@ describe('POST /api/intake/non-integrated/upload', () => {
|
||||
expect.objectContaining({ id: 'admin-1' }),
|
||||
)
|
||||
|
||||
const storedBytes = await readFile(
|
||||
path.join(assetDirectory, completedPayload.data.storageKey),
|
||||
)
|
||||
expect(Buffer.from(storedBytes).toString('utf8')).toBe('chunk-one/chunk-two')
|
||||
expect(
|
||||
readUploadedObjectBody(completedPayload.data.storageKey)?.toString('utf8'),
|
||||
).toBe('chunk-one/chunk-two')
|
||||
})
|
||||
|
||||
it('accepts larger upload chunks and stores the merged asset', async () => {
|
||||
@ -524,12 +669,10 @@ describe('POST /api/intake/non-integrated/upload', () => {
|
||||
}),
|
||||
)
|
||||
|
||||
const storedBytes = await readFile(
|
||||
path.join(assetDirectory, completedPayload.data.storageKey),
|
||||
)
|
||||
const storedBytes = readUploadedObjectBody(completedPayload.data.storageKey)
|
||||
expect(storedBytes).toHaveLength(totalSize)
|
||||
expect(storedBytes[0]).toBe(65)
|
||||
expect(storedBytes.subarray(-5).toString()).toBe('/tail')
|
||||
expect(storedBytes?.[0]).toBe(65)
|
||||
expect(storedBytes?.subarray(-5).toString()).toBe('/tail')
|
||||
})
|
||||
|
||||
it('rejects detached chunked product-route uploads when detached upload permission fails', async () => {
|
||||
|
||||
@ -1,13 +1,17 @@
|
||||
import { existsSync } from 'fs'
|
||||
import { mkdir, writeFile } from 'fs/promises'
|
||||
import { mkdir } from 'fs/promises'
|
||||
import path from 'path'
|
||||
import { randomUUID } from 'crypto'
|
||||
import { buildNonIntegratedAssetUrl } from '@/app/intake/non-integrated/nonIntegratedAssetUrl'
|
||||
import { getNonIntegratedAssetDirectory } from '@/app/intake/non-integrated/nonIntegratedAssetServer'
|
||||
import {
|
||||
buildNonIntegratedAssetObjectStorageKey,
|
||||
getNonIntegratedAssetDirectory,
|
||||
} from '@/app/intake/non-integrated/nonIntegratedAssetServer'
|
||||
import type {
|
||||
NonIntegratedProductRecord,
|
||||
NonIntegratedUploadFile,
|
||||
} from '@/app/intake/non-integrated/nonIntegratedStorage'
|
||||
import { uploadBufferToObjectStorage } from '@/lib/storage'
|
||||
import { intakeNonIntegratedProductService } from '@/server/services/intake-non-integrated-product.service'
|
||||
import type { AuthUser } from '@/server/core/types'
|
||||
|
||||
@ -153,6 +157,23 @@ export async function completeNonIntegratedStoredUploadFile(input: {
|
||||
return responseData
|
||||
}
|
||||
|
||||
export async function uploadNonIntegratedStoredBytesToObjectStorage(input: {
|
||||
storedFileName: string
|
||||
bytes: Buffer
|
||||
mimeType: string
|
||||
}) {
|
||||
const objectKey = buildNonIntegratedAssetObjectStorageKey(input.storedFileName)
|
||||
if (!objectKey) {
|
||||
throw new Error('非集成素材对象 key 生成失败')
|
||||
}
|
||||
|
||||
await uploadBufferToObjectStorage(
|
||||
input.bytes,
|
||||
objectKey,
|
||||
input.mimeType || 'application/octet-stream',
|
||||
)
|
||||
}
|
||||
|
||||
export async function storeNonIntegratedUploadFile(input: {
|
||||
bytes: Buffer
|
||||
originalFileName: string
|
||||
@ -170,7 +191,11 @@ export async function storeNonIntegratedUploadFile(input: {
|
||||
productId: input.productId,
|
||||
category: input.category,
|
||||
})
|
||||
await writeFile(uploadTarget.storedFilePath, input.bytes)
|
||||
await uploadNonIntegratedStoredBytesToObjectStorage({
|
||||
storedFileName: uploadTarget.storedFileName,
|
||||
bytes: input.bytes,
|
||||
mimeType: input.mimeType,
|
||||
})
|
||||
|
||||
return completeNonIntegratedStoredUploadFile({
|
||||
storedFileName: uploadTarget.storedFileName,
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { randomUUID } from "crypto";
|
||||
import { existsSync } from "fs";
|
||||
import { mkdir, writeFile } from "fs/promises";
|
||||
import { mkdir, readFile, writeFile } from "fs/promises";
|
||||
import path from "path";
|
||||
import {
|
||||
NON_INTEGRATED_ASSET_DIRECTORY_NAME,
|
||||
@ -13,6 +13,7 @@ import { createObjectReadUrl, normalizeObjectStorageKey } from "@/lib/storage";
|
||||
|
||||
const NON_INTEGRATED_PRIVATE_ASSET_DIRECTORY_NAME = "non-integrated-assets";
|
||||
const OBJECT_STORAGE_ROUTE_PATH = "/api/storage/object";
|
||||
const NON_INTEGRATED_OBJECT_STORAGE_PREFIX = "non-integrated-assets";
|
||||
|
||||
export type MaterializedNonIntegratedAssetFile = {
|
||||
filePath: string;
|
||||
@ -79,6 +80,15 @@ export function resolveNonIntegratedAssetFilePath(
|
||||
return resolveNonIntegratedAssetRelativeFilePath(relativePath, cwd);
|
||||
}
|
||||
|
||||
export function buildNonIntegratedAssetObjectStorageKey(relativePath: string) {
|
||||
const normalizedRelativePath = relativePath.trim().replace(/^\/+/, "");
|
||||
if (!normalizedRelativePath) return "";
|
||||
|
||||
return normalizeObjectStorageKey(
|
||||
`${NON_INTEGRATED_OBJECT_STORAGE_PREFIX}/${normalizedRelativePath}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function extractObjectStorageKeyFromUrl(value: string) {
|
||||
const normalizedValue = value.trim();
|
||||
if (!normalizedValue) return "";
|
||||
@ -98,6 +108,15 @@ export function extractObjectStorageKeyFromUrl(value: string) {
|
||||
}
|
||||
}
|
||||
|
||||
export function extractNonIntegratedAssetObjectStorageKey(fileUrl: string) {
|
||||
const relativePath = extractNonIntegratedAssetRelativePath(fileUrl);
|
||||
if (relativePath) {
|
||||
return buildNonIntegratedAssetObjectStorageKey(relativePath);
|
||||
}
|
||||
|
||||
return extractObjectStorageKeyFromUrl(fileUrl);
|
||||
}
|
||||
|
||||
function buildSafeMaterializedFileName(objectKey: string) {
|
||||
const extension = path.extname(objectKey).replace(/[^a-zA-Z0-9.]/g, "");
|
||||
const baseName =
|
||||
@ -110,6 +129,34 @@ function buildSafeMaterializedFileName(objectKey: string) {
|
||||
return `${baseName}-${randomUUID()}${extension}`;
|
||||
}
|
||||
|
||||
async function fetchObjectStorageBuffer(objectKey: string) {
|
||||
const response = await fetch(createObjectReadUrl(objectKey), {
|
||||
cache: "no-store",
|
||||
});
|
||||
if (!response.ok) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return Buffer.from(await response.arrayBuffer());
|
||||
}
|
||||
|
||||
export async function readNonIntegratedAssetBuffer(
|
||||
fileUrl: string,
|
||||
cwd = process.cwd(),
|
||||
): Promise<Buffer | null> {
|
||||
const localPath = resolveNonIntegratedAssetFilePath(fileUrl, cwd);
|
||||
if (localPath && existsSync(localPath)) {
|
||||
return readFile(localPath);
|
||||
}
|
||||
|
||||
const objectKey = extractNonIntegratedAssetObjectStorageKey(fileUrl);
|
||||
if (!objectKey) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return fetchObjectStorageBuffer(objectKey);
|
||||
}
|
||||
|
||||
export async function materializeNonIntegratedAssetFile(
|
||||
fileUrl: string,
|
||||
tempDirectory: string,
|
||||
@ -124,13 +171,13 @@ export async function materializeNonIntegratedAssetFile(
|
||||
};
|
||||
}
|
||||
|
||||
const objectKey = extractObjectStorageKeyFromUrl(fileUrl);
|
||||
const objectKey = extractNonIntegratedAssetObjectStorageKey(fileUrl);
|
||||
if (!objectKey) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const response = await fetch(createObjectReadUrl(objectKey));
|
||||
if (!response.ok) {
|
||||
const buffer = await fetchObjectStorageBuffer(objectKey);
|
||||
if (!buffer) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@ -139,7 +186,7 @@ export async function materializeNonIntegratedAssetFile(
|
||||
tempDirectory,
|
||||
buildSafeMaterializedFileName(objectKey),
|
||||
);
|
||||
await writeFile(filePath, Buffer.from(await response.arrayBuffer()));
|
||||
await writeFile(filePath, buffer);
|
||||
|
||||
return {
|
||||
filePath,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user