258 lines
11 KiB
TypeScript
258 lines
11 KiB
TypeScript
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||
import { timingSafeEqual } from "node:crypto";
|
||
import { open, rm, stat } from "node:fs/promises";
|
||
import { MIMEType } from "node:util";
|
||
import { claimZipTask, cleanupZipTasks, loadZipTask, saveZipTask, type ZipTask } from "../../lib/canvas/download-zip-task-store";
|
||
import { prepareCanvasZip } from "../../lib/canvas/download-zip-prepare";
|
||
import { normalizeCanvasDownloadAsset } from "../../lib/canvas/photoshop-download-normalize";
|
||
import { zipStorageKey, zipWorkerJob, type ZipWorkerJob, type ZipWorkerSnapshot } from "../../lib/canvas/remote-zip-contract";
|
||
import { openPublicMedia } from "../../lib/video-canvas/download-media";
|
||
|
||
const TOKEN = process.env.CANVAS_ZIP_WORKER_TOKEN || "";
|
||
if (TOKEN.length < 32 || !process.env.CANVAS_DOWNLOAD_ZIP_DIR) {
|
||
throw new Error("Set CANVAS_ZIP_WORKER_TOKEN and CANVAS_DOWNLOAD_ZIP_DIR");
|
||
}
|
||
|
||
const OWNER = "zip-worker";
|
||
const IMAGE_LIMIT = 125 * 1024 ** 2;
|
||
const PART_BYTES = 8 * 1024 ** 2;
|
||
const queue = new Map<string, ZipWorkerJob>();
|
||
const accepting = new Set<string>();
|
||
let running: string | undefined;
|
||
|
||
function authorized(request: IncomingMessage) {
|
||
const actual = Buffer.from(request.headers.authorization || "");
|
||
const expected = Buffer.from("Bearer " + TOKEN);
|
||
return actual.length === expected.length && timingSafeEqual(actual, expected);
|
||
}
|
||
|
||
function reply(response: ServerResponse, status: number, body: unknown) {
|
||
response.writeHead(status, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" });
|
||
response.end(JSON.stringify(body));
|
||
}
|
||
|
||
async function readJob(request: IncomingMessage) {
|
||
let bytes = 0;
|
||
const body: Buffer[] = [];
|
||
for await (const chunk of request) {
|
||
const value = Buffer.from(chunk);
|
||
bytes += value.length;
|
||
if (bytes > 1024 ** 2) throw new Error("Invalid job size");
|
||
body.push(value);
|
||
}
|
||
return zipWorkerJob.parse(JSON.parse(Buffer.concat(body).toString("utf8")));
|
||
}
|
||
|
||
function snapshot(task: ZipTask): ZipWorkerSnapshot {
|
||
const { status, stage, error, processedCount, successCount, failedCount, compressPercent } = task;
|
||
return { status, stage, error, processedCount, successCount, failedCount, compressPercent,
|
||
bytes: task.status === "completed" ? task.zipFile?.bytes : undefined };
|
||
}
|
||
|
||
async function fetchInput(job: ZipWorkerJob, index: number, signal: AbortSignal) {
|
||
const item = job.files[index];
|
||
const appHost = new URL(job.appOrigin).hostname;
|
||
const allowUrl = (url: URL) => url.hostname !== appHost && url.hostname !== "ai.yydd.ai";
|
||
for (const url of item.sources) {
|
||
if (signal.aborted) break;
|
||
try {
|
||
const response = await openPublicMedia(url, {
|
||
allowUrl,
|
||
signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]),
|
||
});
|
||
if (response.statusCode !== 200 || Number(response.headers["content-length"] || 0) > IMAGE_LIMIT) {
|
||
response.destroy();
|
||
continue;
|
||
}
|
||
const contentType = new MIMEType(response.headers["content-type"] || "application/octet-stream").essence;
|
||
const buffers: Buffer[] = [];
|
||
let bytes = 0;
|
||
for await (const packet of response) {
|
||
const chunk = Buffer.from(packet);
|
||
bytes += chunk.length;
|
||
if (bytes > IMAGE_LIMIT) throw new Error("Original exceeds the download limit");
|
||
buffers.push(chunk);
|
||
}
|
||
return await normalizeCanvasDownloadAsset({
|
||
item: { url, name: item.name }, index, bytes: Buffer.concat(buffers, bytes),
|
||
contentType, includeSequencePrefix: true,
|
||
});
|
||
} catch { /* Try the next resolved source. */ }
|
||
}
|
||
return { ok: false as const, url: "", name: item.name, error: "宿主机暂时无法读取这张原图" };
|
||
}
|
||
|
||
async function uploadArchive(task: ZipTask, job: ZipWorkerJob) {
|
||
const { bucket, token, publicUrl } = job.storage;
|
||
const { path, bytes } = task.zipFile!;
|
||
const key = zipStorageKey(task.id);
|
||
const endpoint = "https://upload.qiniup.com/buckets/" + encodeURIComponent(bucket)
|
||
+ "/objects/" + encodeURIComponent(Buffer.from(key).toString("base64").replace(/\+/g, "-").replace(/\//g, "_")) + "/uploads";
|
||
const storedUrl = new URL(publicUrl);
|
||
const hasExactSize = async () => {
|
||
try {
|
||
const response = await fetch(storedUrl, { method: "HEAD", redirect: "error", signal: AbortSignal.timeout(8_000) });
|
||
return response.ok && Number(response.headers.get("content-length")) === bytes;
|
||
} catch { return false; }
|
||
};
|
||
if (await hasExactSize()) return;
|
||
const request = async (url: string, method: string, body?: Buffer | string, contentType?: string) => {
|
||
const headers: Record<string, string> = { Authorization: "UpToken " + token };
|
||
if (contentType) headers["Content-Type"] = contentType;
|
||
return await fetch(url, {
|
||
method, headers, body: Buffer.isBuffer(body) ? Uint8Array.from(body) : body,
|
||
redirect: "error", signal: AbortSignal.timeout(method === "PUT" ? 180_000 : 30_000),
|
||
});
|
||
};
|
||
const init = await request(endpoint, "POST");
|
||
if (!init.ok) throw new Error("七牛未能开始接收压缩包(" + init.status + ")");
|
||
const { uploadId } = await init.json() as { uploadId?: string };
|
||
if (!uploadId) throw new Error("七牛未返回上传编号");
|
||
const uploadUrl = endpoint + "/" + encodeURIComponent(uploadId);
|
||
const parts: Array<{ partNumber: number; etag: string }> = [];
|
||
try {
|
||
const handle = await open(path, "r");
|
||
try {
|
||
for (let offset = 0; offset < bytes; offset += PART_BYTES) {
|
||
const data = Buffer.allocUnsafe(Math.min(PART_BYTES, bytes - offset));
|
||
let read = 0;
|
||
while (read < data.length) {
|
||
const result = await handle.read(data, read, data.length - read, offset + read);
|
||
if (!result.bytesRead) throw new Error("压缩包读取不完整");
|
||
read += result.bytesRead;
|
||
}
|
||
const partNumber = parts.length + 1;
|
||
const response = await request(uploadUrl + "/" + partNumber, "PUT", data, "application/octet-stream");
|
||
if (!response.ok) throw new Error("七牛接收分片失败(" + response.status + ")");
|
||
const { etag } = await response.json() as { etag?: string };
|
||
if (!etag) throw new Error("七牛未确认上传分片");
|
||
parts.push({ partNumber, etag });
|
||
task.stage = "正在传送至七牛 " + Math.floor((offset + data.length) / bytes * 100) + "%";
|
||
task.updatedAt = Date.now();
|
||
await saveZipTask(task);
|
||
}
|
||
} finally { await handle.close(); }
|
||
const complete = await request(uploadUrl, "POST", JSON.stringify({
|
||
parts, fname: task.folderName + ".zip", mimeType: "application/zip",
|
||
}), "application/json");
|
||
if (!complete.ok) {
|
||
if (complete.status === 614 && await hasExactSize()) return;
|
||
throw new Error("七牛未确认压缩包(" + complete.status + ")");
|
||
}
|
||
const saved = await complete.json() as { key?: string; fsize?: number };
|
||
if (saved.key !== key || saved.fsize !== bytes) throw new Error("七牛返回的压缩包信息不一致");
|
||
} catch (error) {
|
||
await request(uploadUrl, "DELETE").then(response => response.body?.cancel()).catch(() => undefined);
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
async function run(id: string, job: ZipWorkerJob) {
|
||
const release = await claimZipTask(id);
|
||
if (!release) return;
|
||
let task: ZipTask | null = null;
|
||
try {
|
||
task = await loadZipTask(id, OWNER);
|
||
if (!task || task.status === "completed" || task.status === "failed") return;
|
||
task.status = "running";
|
||
task.stage = "宿主机正在准备原图";
|
||
await saveZipTask(task);
|
||
const present = task.zipFile && await stat(task.zipFile.path).catch(() => null);
|
||
if (!present || present.size !== task.zipFile!.bytes) {
|
||
await prepareCanvasZip(task, ({ index, signal }) => fetchInput(job, index, signal));
|
||
}
|
||
task.stage = "正在传送至七牛";
|
||
await saveZipTask(task);
|
||
await uploadArchive(task, job);
|
||
task.status = "completed";
|
||
task.stage = task.failedCount ? "部分原图未能读取" : "打包完成";
|
||
task.compressPercent = 100;
|
||
task.error = undefined;
|
||
task.updatedAt = Date.now();
|
||
task.expiresAt = Date.now() + 24 * 60 * 60_000;
|
||
await saveZipTask(task);
|
||
if (task.zipFile) await rm(task.zipFile.path, { force: true }).catch(() => undefined);
|
||
} catch (error) {
|
||
if (task) {
|
||
task.status = "failed";
|
||
task.stage = "打包未完成";
|
||
task.error = (error instanceof Error ? error.message : "宿主机打包失败").slice(0, 450);
|
||
task.updatedAt = Date.now();
|
||
await saveZipTask(task);
|
||
}
|
||
} finally { await release(); }
|
||
}
|
||
|
||
function drain() {
|
||
if (running) return;
|
||
const next = queue.entries().next().value;
|
||
if (!next) return;
|
||
const [id, job] = next;
|
||
queue.delete(id);
|
||
running = id;
|
||
void run(id, job).catch(() => console.error("Canvas ZIP job could not be saved")).finally(() => {
|
||
running = undefined;
|
||
drain();
|
||
});
|
||
}
|
||
|
||
export const router = createServer((request, response) => {
|
||
void (async () => {
|
||
if (!authorized(request)) { reply(response, 401, {}); return; }
|
||
if (request.method === "GET" && request.url === "/health") {
|
||
reply(response, 200, { ok: true, active: Number(Boolean(running)), waiting: queue.size });
|
||
return;
|
||
}
|
||
const id = request.url?.match(/^\/v1\/jobs\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i)?.[1];
|
||
if (!id || request.method !== "PUT") { reply(response, 404, {}); return; }
|
||
if (accepting.has(id)) { reply(response, 503, {}); return; }
|
||
accepting.add(id);
|
||
try {
|
||
const job = await readJob(request);
|
||
const publicUrl = new URL(job.storage.publicUrl);
|
||
if (publicUrl.protocol !== "https:" || !publicUrl.pathname.endsWith("/" + zipStorageKey(id))) {
|
||
reply(response, 400, {}); return;
|
||
}
|
||
let task = await loadZipTask(id, OWNER);
|
||
if (running === id || queue.has(id)) {
|
||
if (task) reply(response, 200, snapshot(task)); else reply(response, 503, {});
|
||
return;
|
||
}
|
||
if (queue.size >= 8 && (!task || task.status === "pending" || task.status === "running")) {
|
||
reply(response, 503, {}); return;
|
||
}
|
||
if (task?.status === "failed" && job.attempt > (task.attempt ?? 0)) {
|
||
task.attempt = job.attempt;
|
||
task.status = "pending";
|
||
task.stage = "等待宿主机打包";
|
||
task.error = undefined;
|
||
task.expiresAt = job.expiresAt;
|
||
task.updatedAt = Date.now();
|
||
await saveZipTask(task);
|
||
}
|
||
if (!task) {
|
||
const now = Date.now();
|
||
task = {
|
||
id, userId: OWNER, folderName: job.folderName, attempt: job.attempt,
|
||
files: job.files.map(item => ({ url: item.sources[0] || "", name: item.name })),
|
||
totalCount: job.files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0,
|
||
status: "pending", stage: "等待宿主机打包", createdAt: now, updatedAt: now, expiresAt: job.expiresAt,
|
||
};
|
||
await saveZipTask(task);
|
||
}
|
||
if (task.status === "pending" || task.status === "running") {
|
||
queue.set(id, job);
|
||
drain();
|
||
}
|
||
reply(response, 200, snapshot(task));
|
||
void cleanupZipTasks().catch(() => undefined);
|
||
} finally { accepting.delete(id); }
|
||
})().catch(() => {
|
||
if (!response.headersSent) reply(response, 400, { error: "无效的打包请求" });
|
||
});
|
||
});
|
||
|
||
if (require.main === module) {
|
||
router.listen(Number(process.env.CANVAS_ZIP_PORT || 18343), "127.0.0.1");
|
||
}
|