fix(intake): 降低非集成素材读取超时

This commit is contained in:
WuYongJie 2026-06-15 15:30:39 +08:00
parent 68a17b1dff
commit 455686e1e3
6 changed files with 248 additions and 11 deletions

View File

@ -73,7 +73,6 @@ export async function GET(request: NextRequest, context: RouteContext) {
return (await buildObjectReadResponse(
createObjectReadUrl(objectStorageKey),
request,
{ redirectHttps: false },
)) as unknown as NextResponse;
} catch (error) {
if (!(error instanceof ObjectReadProxyError)) {

View File

@ -330,7 +330,7 @@ describe('POST /api/intake/non-integrated/upload', () => {
expect(
mockObjectReadResponseCalls[mockObjectReadResponseCalls.length - 1]
?.options,
).toEqual({ redirectHttps: false })
).toBeUndefined()
expect(getResponseHeader(downloadResponse, 'Content-Type')).toBe(
'application/pdf',
)

View File

@ -1,8 +1,26 @@
import { mkdir, mkdtemp, rm, writeFile } from "fs/promises";
import { EventEmitter } from "node:events";
import os from "os";
import path from "path";
const mockCreateObjectReadUrl = jest.fn();
const mockNormalizeObjectStorageKey = jest.fn();
const mockSpawn = jest.fn();
const CURL_HTTP_STATUS_MARKER = "__NON_INTEGRATED_OBJECT_HTTP_STATUS__";
jest.mock("@/lib/storage", () => ({
createObjectReadUrl: (...args: unknown[]) => mockCreateObjectReadUrl(...args),
normalizeObjectStorageKey: (...args: unknown[]) =>
mockNormalizeObjectStorageKey(...args),
}));
jest.mock("node:child_process", () => ({
spawn: (...args: unknown[]) => mockSpawn(...args),
}));
import {
getNonIntegratedAssetDirectory,
readNonIntegratedAssetBuffer,
resolveNonIntegratedAssetFilePath,
resolveNonIntegratedAssetRelativeFilePath,
} from "./nonIntegratedAssetServer";
@ -10,14 +28,51 @@ import {
describe("nonIntegratedAssetServer", () => {
let cwd: string;
function createMockCurlChild(args: string[], body: string, status = 200) {
const child = new EventEmitter() as EventEmitter & {
stdout: EventEmitter & { setEncoding: jest.Mock };
stderr: EventEmitter & { setEncoding: jest.Mock };
};
child.stdout = Object.assign(new EventEmitter(), {
setEncoding: jest.fn(),
});
child.stderr = Object.assign(new EventEmitter(), {
setEncoding: jest.fn(),
});
const outputIndex = args.indexOf("-o");
const bodyPath = outputIndex >= 0 ? args[outputIndex + 1] : "";
setImmediate(async () => {
if (bodyPath) {
await writeFile(bodyPath, body);
}
child.stdout.emit("data", `${CURL_HTTP_STATUS_MARKER}${status}`);
child.emit("close", 0);
});
return child;
}
beforeEach(async () => {
cwd = await mkdtemp(path.join(os.tmpdir(), "non-integrated-assets-"));
delete process.env.NON_INTEGRATED_ASSET_DIR;
mockSpawn.mockReset();
mockNormalizeObjectStorageKey.mockImplementation((key: string) =>
key.trim().replace(/^\/+/, ""),
);
mockCreateObjectReadUrl.mockImplementation(
(key: string) => `https://storage.example.com/${encodeURIComponent(key)}`,
);
global.fetch = jest.fn(async () => ({
ok: true,
arrayBuffer: async () => new TextEncoder().encode("object-body").buffer,
})) as unknown as typeof fetch;
});
afterEach(async () => {
await rm(cwd, { recursive: true, force: true });
delete process.env.NON_INTEGRATED_ASSET_DIR;
jest.restoreAllMocks();
});
it("stores new non-integrated assets outside the public directory by default", () => {
@ -59,4 +114,43 @@ describe("nonIntegratedAssetServer", () => {
).toBe("");
expect(resolveNonIntegratedAssetRelativeFilePath("../secret.txt", cwd)).toBe("");
});
it("reads protected object storage assets by stable non-integrated url", async () => {
const buffer = await readNonIntegratedAssetBuffer(
"/api/intake/non-integrated/assets/SZ0230/产品模型/model.png",
);
expect(mockCreateObjectReadUrl).toHaveBeenCalledWith(
"non-integrated-assets/SZ0230/产品模型/model.png",
);
expect(fetch).toHaveBeenCalledWith(
"https://storage.example.com/non-integrated-assets%2FSZ0230%2F%E4%BA%A7%E5%93%81%E6%A8%A1%E5%9E%8B%2Fmodel.png",
{ cache: "no-store" },
);
expect(buffer?.toString("utf8")).toBe("object-body");
});
it("falls back to curl when object storage fetch times out", async () => {
global.fetch = jest.fn(async () => {
const error = new TypeError("fetch failed") as TypeError & {
cause?: { code: string };
};
error.cause = { code: "ETIMEDOUT" };
throw error;
}) as unknown as typeof fetch;
mockSpawn.mockImplementationOnce((_command: string, args: string[]) =>
createMockCurlChild(args, "curl-object-body"),
);
const buffer = await readNonIntegratedAssetBuffer(
"/api/intake/non-integrated/assets/SZ0230/产品模型/model.png",
);
expect(mockSpawn).toHaveBeenCalledWith(
"curl",
expect.arrayContaining(["--http1.1", "-L"]),
expect.objectContaining({ windowsHide: true }),
);
expect(buffer?.toString("utf8")).toBe("curl-object-body");
});
});

View File

@ -1,6 +1,8 @@
import { randomUUID } from "crypto";
import { spawn } from "node:child_process";
import { existsSync } from "fs";
import { mkdir, writeFile } from "fs/promises";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "fs/promises";
import os from "os";
import path from "path";
import {
NON_INTEGRATED_DOCUMENT_PREVIEW_DIRECTORY,
@ -13,6 +15,7 @@ import { createObjectReadUrl, normalizeObjectStorageKey } from "@/lib/storage";
const NON_INTEGRATED_PRIVATE_ASSET_DIRECTORY_NAME = "non-integrated-assets";
const OBJECT_STORAGE_ROUTE_PATH = "/api/storage/object";
const NON_INTEGRATED_OBJECT_STORAGE_PREFIX = "non-integrated-assets";
const CURL_HTTP_STATUS_MARKER = "__NON_INTEGRATED_OBJECT_HTTP_STATUS__";
export type MaterializedNonIntegratedAssetFile = {
filePath: string;
@ -117,9 +120,19 @@ function buildSafeMaterializedFileName(objectKey: string) {
}
async function fetchObjectStorageBuffer(objectKey: string) {
const response = await fetch(createObjectReadUrl(objectKey), {
cache: "no-store",
});
const readUrl = createObjectReadUrl(objectKey);
let response: Response;
try {
response = await fetch(readUrl, {
cache: "no-store",
});
} catch (error) {
if (!isNetworkFetchFailure(error)) {
throw error;
}
return fetchObjectStorageBufferViaCurl(readUrl);
}
if (!response.ok) {
return null;
}
@ -127,6 +140,121 @@ async function fetchObjectStorageBuffer(objectKey: string) {
return Buffer.from(await response.arrayBuffer());
}
function collectErrorCodes(error: unknown, codes = new Set<string>()) {
if (!error || typeof error !== "object") {
return codes;
}
const value = error as { code?: unknown; cause?: unknown; errors?: unknown };
if (typeof value.code === "string" && value.code.trim()) {
codes.add(value.code.trim());
}
if (value.cause) {
collectErrorCodes(value.cause, codes);
}
if (Array.isArray(value.errors)) {
value.errors.forEach((entry) => collectErrorCodes(entry, codes));
}
return codes;
}
function isNetworkFetchFailure(error: unknown) {
if (!(error instanceof Error)) {
return false;
}
if (error.message === "fetch failed") {
return true;
}
const codes = collectErrorCodes(error);
return [
"ETIMEDOUT",
"ECONNRESET",
"EPIPE",
"ENETUNREACH",
"EHOSTUNREACH",
"UND_ERR_CONNECT_TIMEOUT",
"UND_ERR_HEADERS_TIMEOUT",
"UND_ERR_SOCKET",
].some((code) => codes.has(code));
}
function runCurl(args: string[]) {
return new Promise<{ stdout: string; stderr: string; exitCode: number | null }>(
(resolve, reject) => {
const child = spawn("curl", args, {
stdio: ["ignore", "pipe", "pipe"],
windowsHide: true,
});
let stdout = "";
let stderr = "";
child.stdout.setEncoding("utf8");
child.stderr.setEncoding("utf8");
child.stdout.on("data", (chunk) => {
stdout += chunk;
});
child.stderr.on("data", (chunk) => {
stderr += chunk;
});
child.on("error", reject);
child.on("close", (exitCode) => {
resolve({ stdout, stderr, exitCode });
});
},
);
}
function parseCurlHttpStatus(stdout: string) {
const markerIndex = stdout.lastIndexOf(CURL_HTTP_STATUS_MARKER);
if (markerIndex < 0) {
return null;
}
const status = Number(
stdout.slice(markerIndex + CURL_HTTP_STATUS_MARKER.length).trim(),
);
return Number.isFinite(status) ? status : null;
}
async function fetchObjectStorageBufferViaCurl(readUrl: string) {
const tempDirectory = await mkdtemp(
path.join(os.tmpdir(), "non-integrated-object-download-"),
);
try {
const bodyPath = path.join(tempDirectory, "response.bin");
const { stdout, stderr, exitCode } = await runCurl([
"--http1.1",
"-sS",
"-L",
"-o",
bodyPath,
"-w",
CURL_HTTP_STATUS_MARKER + "%{http_code}",
readUrl,
]);
const status = parseCurlHttpStatus(stdout);
if (status == null) {
throw new Error(
stderr.trim() ||
stdout.trim() ||
`curl 下载失败(exit code: ${exitCode ?? "unknown"})`,
);
}
if (status < 200 || status >= 300) {
return null;
}
return readFile(bodyPath);
} finally {
await rm(tempDirectory, { recursive: true, force: true }).catch(
() => undefined,
);
}
}
export async function readNonIntegratedAssetBuffer(
fileUrl: string,
_cwd = process.cwd(),

View File

@ -72,10 +72,15 @@ export async function buildObjectReadResponse(
upstreamHeaders.range = range
}
const upstreamResponse = await fetch(readUrl, {
headers: upstreamHeaders,
cache: 'no-store',
})
let upstreamResponse: Response
try {
upstreamResponse = await fetch(readUrl, {
headers: upstreamHeaders,
cache: 'no-store',
})
} catch {
throw new ObjectReadProxyError('对象存储读取失败', 502)
}
if (!upstreamResponse.ok && upstreamResponse.status !== 206) {
throw new ObjectReadProxyError('对象存储读取失败', upstreamResponse.status)

View File

@ -295,7 +295,18 @@ async function readAssetBufferByUrl(url: string) {
);
}
const buffer = await readNonIntegratedAssetBuffer(url);
let buffer: Buffer | null;
try {
buffer = await readNonIntegratedAssetBuffer(url);
} catch (error) {
throw new AppError(
ErrorCode.FILE_UPLOAD_FAILED,
`白底图源图读取失败:${readUnknownErrorMessage(
error,
"对象存储连接失败",
)}`,
);
}
if (!buffer) {
throw new AppError(ErrorCode.FILE_NOT_FOUND, "白底图所需的源图片不存在。");
}