fix(intake): 降低非集成素材读取超时
This commit is contained in:
parent
68a17b1dff
commit
455686e1e3
@ -73,7 +73,6 @@ export async function GET(request: NextRequest, context: RouteContext) {
|
||||
return (await buildObjectReadResponse(
|
||||
createObjectReadUrl(objectStorageKey),
|
||||
request,
|
||||
{ redirectHttps: false },
|
||||
)) as unknown as NextResponse;
|
||||
} catch (error) {
|
||||
if (!(error instanceof ObjectReadProxyError)) {
|
||||
|
||||
@ -330,7 +330,7 @@ describe('POST /api/intake/non-integrated/upload', () => {
|
||||
expect(
|
||||
mockObjectReadResponseCalls[mockObjectReadResponseCalls.length - 1]
|
||||
?.options,
|
||||
).toEqual({ redirectHttps: false })
|
||||
).toBeUndefined()
|
||||
expect(getResponseHeader(downloadResponse, 'Content-Type')).toBe(
|
||||
'application/pdf',
|
||||
)
|
||||
|
||||
@ -1,8 +1,26 @@
|
||||
import { mkdir, mkdtemp, rm, writeFile } from "fs/promises";
|
||||
import { EventEmitter } from "node:events";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
|
||||
const mockCreateObjectReadUrl = jest.fn();
|
||||
const mockNormalizeObjectStorageKey = jest.fn();
|
||||
const mockSpawn = jest.fn();
|
||||
const CURL_HTTP_STATUS_MARKER = "__NON_INTEGRATED_OBJECT_HTTP_STATUS__";
|
||||
|
||||
jest.mock("@/lib/storage", () => ({
|
||||
createObjectReadUrl: (...args: unknown[]) => mockCreateObjectReadUrl(...args),
|
||||
normalizeObjectStorageKey: (...args: unknown[]) =>
|
||||
mockNormalizeObjectStorageKey(...args),
|
||||
}));
|
||||
|
||||
jest.mock("node:child_process", () => ({
|
||||
spawn: (...args: unknown[]) => mockSpawn(...args),
|
||||
}));
|
||||
|
||||
import {
|
||||
getNonIntegratedAssetDirectory,
|
||||
readNonIntegratedAssetBuffer,
|
||||
resolveNonIntegratedAssetFilePath,
|
||||
resolveNonIntegratedAssetRelativeFilePath,
|
||||
} from "./nonIntegratedAssetServer";
|
||||
@ -10,14 +28,51 @@ import {
|
||||
describe("nonIntegratedAssetServer", () => {
|
||||
let cwd: string;
|
||||
|
||||
function createMockCurlChild(args: string[], body: string, status = 200) {
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
stdout: EventEmitter & { setEncoding: jest.Mock };
|
||||
stderr: EventEmitter & { setEncoding: jest.Mock };
|
||||
};
|
||||
child.stdout = Object.assign(new EventEmitter(), {
|
||||
setEncoding: jest.fn(),
|
||||
});
|
||||
child.stderr = Object.assign(new EventEmitter(), {
|
||||
setEncoding: jest.fn(),
|
||||
});
|
||||
|
||||
const outputIndex = args.indexOf("-o");
|
||||
const bodyPath = outputIndex >= 0 ? args[outputIndex + 1] : "";
|
||||
setImmediate(async () => {
|
||||
if (bodyPath) {
|
||||
await writeFile(bodyPath, body);
|
||||
}
|
||||
child.stdout.emit("data", `${CURL_HTTP_STATUS_MARKER}${status}`);
|
||||
child.emit("close", 0);
|
||||
});
|
||||
|
||||
return child;
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
cwd = await mkdtemp(path.join(os.tmpdir(), "non-integrated-assets-"));
|
||||
delete process.env.NON_INTEGRATED_ASSET_DIR;
|
||||
mockSpawn.mockReset();
|
||||
mockNormalizeObjectStorageKey.mockImplementation((key: string) =>
|
||||
key.trim().replace(/^\/+/, ""),
|
||||
);
|
||||
mockCreateObjectReadUrl.mockImplementation(
|
||||
(key: string) => `https://storage.example.com/${encodeURIComponent(key)}`,
|
||||
);
|
||||
global.fetch = jest.fn(async () => ({
|
||||
ok: true,
|
||||
arrayBuffer: async () => new TextEncoder().encode("object-body").buffer,
|
||||
})) as unknown as typeof fetch;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(cwd, { recursive: true, force: true });
|
||||
delete process.env.NON_INTEGRATED_ASSET_DIR;
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("stores new non-integrated assets outside the public directory by default", () => {
|
||||
@ -59,4 +114,43 @@ describe("nonIntegratedAssetServer", () => {
|
||||
).toBe("");
|
||||
expect(resolveNonIntegratedAssetRelativeFilePath("../secret.txt", cwd)).toBe("");
|
||||
});
|
||||
|
||||
it("reads protected object storage assets by stable non-integrated url", async () => {
|
||||
const buffer = await readNonIntegratedAssetBuffer(
|
||||
"/api/intake/non-integrated/assets/SZ0230/产品模型/model.png",
|
||||
);
|
||||
|
||||
expect(mockCreateObjectReadUrl).toHaveBeenCalledWith(
|
||||
"non-integrated-assets/SZ0230/产品模型/model.png",
|
||||
);
|
||||
expect(fetch).toHaveBeenCalledWith(
|
||||
"https://storage.example.com/non-integrated-assets%2FSZ0230%2F%E4%BA%A7%E5%93%81%E6%A8%A1%E5%9E%8B%2Fmodel.png",
|
||||
{ cache: "no-store" },
|
||||
);
|
||||
expect(buffer?.toString("utf8")).toBe("object-body");
|
||||
});
|
||||
|
||||
it("falls back to curl when object storage fetch times out", async () => {
|
||||
global.fetch = jest.fn(async () => {
|
||||
const error = new TypeError("fetch failed") as TypeError & {
|
||||
cause?: { code: string };
|
||||
};
|
||||
error.cause = { code: "ETIMEDOUT" };
|
||||
throw error;
|
||||
}) as unknown as typeof fetch;
|
||||
mockSpawn.mockImplementationOnce((_command: string, args: string[]) =>
|
||||
createMockCurlChild(args, "curl-object-body"),
|
||||
);
|
||||
|
||||
const buffer = await readNonIntegratedAssetBuffer(
|
||||
"/api/intake/non-integrated/assets/SZ0230/产品模型/model.png",
|
||||
);
|
||||
|
||||
expect(mockSpawn).toHaveBeenCalledWith(
|
||||
"curl",
|
||||
expect.arrayContaining(["--http1.1", "-L"]),
|
||||
expect.objectContaining({ windowsHide: true }),
|
||||
);
|
||||
expect(buffer?.toString("utf8")).toBe("curl-object-body");
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
import { randomUUID } from "crypto";
|
||||
import { spawn } from "node:child_process";
|
||||
import { existsSync } from "fs";
|
||||
import { mkdir, writeFile } from "fs/promises";
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "fs/promises";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import {
|
||||
NON_INTEGRATED_DOCUMENT_PREVIEW_DIRECTORY,
|
||||
@ -13,6 +15,7 @@ import { createObjectReadUrl, normalizeObjectStorageKey } from "@/lib/storage";
|
||||
const NON_INTEGRATED_PRIVATE_ASSET_DIRECTORY_NAME = "non-integrated-assets";
|
||||
const OBJECT_STORAGE_ROUTE_PATH = "/api/storage/object";
|
||||
const NON_INTEGRATED_OBJECT_STORAGE_PREFIX = "non-integrated-assets";
|
||||
const CURL_HTTP_STATUS_MARKER = "__NON_INTEGRATED_OBJECT_HTTP_STATUS__";
|
||||
|
||||
export type MaterializedNonIntegratedAssetFile = {
|
||||
filePath: string;
|
||||
@ -117,9 +120,19 @@ function buildSafeMaterializedFileName(objectKey: string) {
|
||||
}
|
||||
|
||||
async function fetchObjectStorageBuffer(objectKey: string) {
|
||||
const response = await fetch(createObjectReadUrl(objectKey), {
|
||||
cache: "no-store",
|
||||
});
|
||||
const readUrl = createObjectReadUrl(objectKey);
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(readUrl, {
|
||||
cache: "no-store",
|
||||
});
|
||||
} catch (error) {
|
||||
if (!isNetworkFetchFailure(error)) {
|
||||
throw error;
|
||||
}
|
||||
return fetchObjectStorageBufferViaCurl(readUrl);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
return null;
|
||||
}
|
||||
@ -127,6 +140,121 @@ async function fetchObjectStorageBuffer(objectKey: string) {
|
||||
return Buffer.from(await response.arrayBuffer());
|
||||
}
|
||||
|
||||
function collectErrorCodes(error: unknown, codes = new Set<string>()) {
|
||||
if (!error || typeof error !== "object") {
|
||||
return codes;
|
||||
}
|
||||
|
||||
const value = error as { code?: unknown; cause?: unknown; errors?: unknown };
|
||||
if (typeof value.code === "string" && value.code.trim()) {
|
||||
codes.add(value.code.trim());
|
||||
}
|
||||
if (value.cause) {
|
||||
collectErrorCodes(value.cause, codes);
|
||||
}
|
||||
if (Array.isArray(value.errors)) {
|
||||
value.errors.forEach((entry) => collectErrorCodes(entry, codes));
|
||||
}
|
||||
|
||||
return codes;
|
||||
}
|
||||
|
||||
function isNetworkFetchFailure(error: unknown) {
|
||||
if (!(error instanceof Error)) {
|
||||
return false;
|
||||
}
|
||||
if (error.message === "fetch failed") {
|
||||
return true;
|
||||
}
|
||||
|
||||
const codes = collectErrorCodes(error);
|
||||
return [
|
||||
"ETIMEDOUT",
|
||||
"ECONNRESET",
|
||||
"EPIPE",
|
||||
"ENETUNREACH",
|
||||
"EHOSTUNREACH",
|
||||
"UND_ERR_CONNECT_TIMEOUT",
|
||||
"UND_ERR_HEADERS_TIMEOUT",
|
||||
"UND_ERR_SOCKET",
|
||||
].some((code) => codes.has(code));
|
||||
}
|
||||
|
||||
function runCurl(args: string[]) {
|
||||
return new Promise<{ stdout: string; stderr: string; exitCode: number | null }>(
|
||||
(resolve, reject) => {
|
||||
const child = spawn("curl", args, {
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
windowsHide: true,
|
||||
});
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout += chunk;
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
child.on("error", reject);
|
||||
child.on("close", (exitCode) => {
|
||||
resolve({ stdout, stderr, exitCode });
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function parseCurlHttpStatus(stdout: string) {
|
||||
const markerIndex = stdout.lastIndexOf(CURL_HTTP_STATUS_MARKER);
|
||||
if (markerIndex < 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const status = Number(
|
||||
stdout.slice(markerIndex + CURL_HTTP_STATUS_MARKER.length).trim(),
|
||||
);
|
||||
return Number.isFinite(status) ? status : null;
|
||||
}
|
||||
|
||||
async function fetchObjectStorageBufferViaCurl(readUrl: string) {
|
||||
const tempDirectory = await mkdtemp(
|
||||
path.join(os.tmpdir(), "non-integrated-object-download-"),
|
||||
);
|
||||
|
||||
try {
|
||||
const bodyPath = path.join(tempDirectory, "response.bin");
|
||||
const { stdout, stderr, exitCode } = await runCurl([
|
||||
"--http1.1",
|
||||
"-sS",
|
||||
"-L",
|
||||
"-o",
|
||||
bodyPath,
|
||||
"-w",
|
||||
CURL_HTTP_STATUS_MARKER + "%{http_code}",
|
||||
readUrl,
|
||||
]);
|
||||
const status = parseCurlHttpStatus(stdout);
|
||||
if (status == null) {
|
||||
throw new Error(
|
||||
stderr.trim() ||
|
||||
stdout.trim() ||
|
||||
`curl 下载失败(exit code: ${exitCode ?? "unknown"})`,
|
||||
);
|
||||
}
|
||||
if (status < 200 || status >= 300) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return readFile(bodyPath);
|
||||
} finally {
|
||||
await rm(tempDirectory, { recursive: true, force: true }).catch(
|
||||
() => undefined,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function readNonIntegratedAssetBuffer(
|
||||
fileUrl: string,
|
||||
_cwd = process.cwd(),
|
||||
|
||||
@ -72,10 +72,15 @@ export async function buildObjectReadResponse(
|
||||
upstreamHeaders.range = range
|
||||
}
|
||||
|
||||
const upstreamResponse = await fetch(readUrl, {
|
||||
headers: upstreamHeaders,
|
||||
cache: 'no-store',
|
||||
})
|
||||
let upstreamResponse: Response
|
||||
try {
|
||||
upstreamResponse = await fetch(readUrl, {
|
||||
headers: upstreamHeaders,
|
||||
cache: 'no-store',
|
||||
})
|
||||
} catch {
|
||||
throw new ObjectReadProxyError('对象存储读取失败', 502)
|
||||
}
|
||||
|
||||
if (!upstreamResponse.ok && upstreamResponse.status !== 206) {
|
||||
throw new ObjectReadProxyError('对象存储读取失败', upstreamResponse.status)
|
||||
|
||||
@ -295,7 +295,18 @@ async function readAssetBufferByUrl(url: string) {
|
||||
);
|
||||
}
|
||||
|
||||
const buffer = await readNonIntegratedAssetBuffer(url);
|
||||
let buffer: Buffer | null;
|
||||
try {
|
||||
buffer = await readNonIntegratedAssetBuffer(url);
|
||||
} catch (error) {
|
||||
throw new AppError(
|
||||
ErrorCode.FILE_UPLOAD_FAILED,
|
||||
`白底图源图读取失败:${readUnknownErrorMessage(
|
||||
error,
|
||||
"对象存储连接失败",
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
if (!buffer) {
|
||||
throw new AppError(ErrorCode.FILE_NOT_FOUND, "白底图所需的源图片不存在。");
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user