258 lines
11 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { timingSafeEqual } from "node:crypto";
import { open, rm, stat } from "node:fs/promises";
import { MIMEType } from "node:util";
import { claimZipTask, cleanupZipTasks, loadZipTask, saveZipTask, type ZipTask } from "../../lib/canvas/download-zip-task-store";
import { prepareCanvasZip } from "../../lib/canvas/download-zip-prepare";
import { normalizeCanvasDownloadAsset } from "../../lib/canvas/photoshop-download-normalize";
import { zipStorageKey, zipWorkerJob, type ZipWorkerJob, type ZipWorkerSnapshot } from "../../lib/canvas/remote-zip-contract";
import { openPublicMedia } from "../../lib/video-canvas/download-media";
const TOKEN = process.env.CANVAS_ZIP_WORKER_TOKEN || "";
if (TOKEN.length < 32 || !process.env.CANVAS_DOWNLOAD_ZIP_DIR) {
throw new Error("Set CANVAS_ZIP_WORKER_TOKEN and CANVAS_DOWNLOAD_ZIP_DIR");
}
const OWNER = "zip-worker";
const IMAGE_LIMIT = 125 * 1024 ** 2;
const PART_BYTES = 8 * 1024 ** 2;
const queue = new Map<string, ZipWorkerJob>();
const accepting = new Set<string>();
let running: string | undefined;
function authorized(request: IncomingMessage) {
const actual = Buffer.from(request.headers.authorization || "");
const expected = Buffer.from("Bearer " + TOKEN);
return actual.length === expected.length && timingSafeEqual(actual, expected);
}
function reply(response: ServerResponse, status: number, body: unknown) {
response.writeHead(status, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" });
response.end(JSON.stringify(body));
}
async function readJob(request: IncomingMessage) {
let bytes = 0;
const body: Buffer[] = [];
for await (const chunk of request) {
const value = Buffer.from(chunk);
bytes += value.length;
if (bytes > 1024 ** 2) throw new Error("Invalid job size");
body.push(value);
}
return zipWorkerJob.parse(JSON.parse(Buffer.concat(body).toString("utf8")));
}
function snapshot(task: ZipTask): ZipWorkerSnapshot {
const { status, stage, error, processedCount, successCount, failedCount, compressPercent } = task;
return { status, stage, error, processedCount, successCount, failedCount, compressPercent,
bytes: task.status === "completed" ? task.zipFile?.bytes : undefined };
}
async function fetchInput(job: ZipWorkerJob, index: number, signal: AbortSignal) {
const item = job.files[index];
const appHost = new URL(job.appOrigin).hostname;
const allowUrl = (url: URL) => url.hostname !== appHost && url.hostname !== "ai.yydd.ai";
for (const url of item.sources) {
if (signal.aborted) break;
try {
const response = await openPublicMedia(url, {
allowUrl,
signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]),
});
if (response.statusCode !== 200 || Number(response.headers["content-length"] || 0) > IMAGE_LIMIT) {
response.destroy();
continue;
}
const contentType = new MIMEType(response.headers["content-type"] || "application/octet-stream").essence;
const buffers: Buffer[] = [];
let bytes = 0;
for await (const packet of response) {
const chunk = Buffer.from(packet);
bytes += chunk.length;
if (bytes > IMAGE_LIMIT) throw new Error("Original exceeds the download limit");
buffers.push(chunk);
}
return await normalizeCanvasDownloadAsset({
item: { url, name: item.name }, index, bytes: Buffer.concat(buffers, bytes),
contentType, includeSequencePrefix: true,
});
} catch { /* Try the next resolved source. */ }
}
return { ok: false as const, url: "", name: item.name, error: "宿主机暂时无法读取这张原图" };
}
async function uploadArchive(task: ZipTask, job: ZipWorkerJob) {
const { bucket, token, publicUrl } = job.storage;
const { path, bytes } = task.zipFile!;
const key = zipStorageKey(task.id);
const endpoint = "https://upload.qiniup.com/buckets/" + encodeURIComponent(bucket)
+ "/objects/" + encodeURIComponent(Buffer.from(key).toString("base64").replace(/\+/g, "-").replace(/\//g, "_")) + "/uploads";
const storedUrl = new URL(publicUrl);
const hasExactSize = async () => {
try {
const response = await fetch(storedUrl, { method: "HEAD", redirect: "error", signal: AbortSignal.timeout(8_000) });
return response.ok && Number(response.headers.get("content-length")) === bytes;
} catch { return false; }
};
if (await hasExactSize()) return;
const request = async (url: string, method: string, body?: Buffer | string, contentType?: string) => {
const headers: Record<string, string> = { Authorization: "UpToken " + token };
if (contentType) headers["Content-Type"] = contentType;
return await fetch(url, {
method, headers, body: Buffer.isBuffer(body) ? Uint8Array.from(body) : body,
redirect: "error", signal: AbortSignal.timeout(method === "PUT" ? 180_000 : 30_000),
});
};
const init = await request(endpoint, "POST");
if (!init.ok) throw new Error("七牛未能开始接收压缩包(" + init.status + ")");
const { uploadId } = await init.json() as { uploadId?: string };
if (!uploadId) throw new Error("七牛未返回上传编号");
const uploadUrl = endpoint + "/" + encodeURIComponent(uploadId);
const parts: Array<{ partNumber: number; etag: string }> = [];
try {
const handle = await open(path, "r");
try {
for (let offset = 0; offset < bytes; offset += PART_BYTES) {
const data = Buffer.allocUnsafe(Math.min(PART_BYTES, bytes - offset));
let read = 0;
while (read < data.length) {
const result = await handle.read(data, read, data.length - read, offset + read);
if (!result.bytesRead) throw new Error("压缩包读取不完整");
read += result.bytesRead;
}
const partNumber = parts.length + 1;
const response = await request(uploadUrl + "/" + partNumber, "PUT", data, "application/octet-stream");
if (!response.ok) throw new Error("七牛接收分片失败(" + response.status + ")");
const { etag } = await response.json() as { etag?: string };
if (!etag) throw new Error("七牛未确认上传分片");
parts.push({ partNumber, etag });
task.stage = "正在传送至七牛 " + Math.floor((offset + data.length) / bytes * 100) + "%";
task.updatedAt = Date.now();
await saveZipTask(task);
}
} finally { await handle.close(); }
const complete = await request(uploadUrl, "POST", JSON.stringify({
parts, fname: task.folderName + ".zip", mimeType: "application/zip",
}), "application/json");
if (!complete.ok) {
if (complete.status === 614 && await hasExactSize()) return;
throw new Error("七牛未确认压缩包(" + complete.status + ")");
}
const saved = await complete.json() as { key?: string; fsize?: number };
if (saved.key !== key || saved.fsize !== bytes) throw new Error("七牛返回的压缩包信息不一致");
} catch (error) {
await request(uploadUrl, "DELETE").then(response => response.body?.cancel()).catch(() => undefined);
throw error;
}
}
async function run(id: string, job: ZipWorkerJob) {
const release = await claimZipTask(id);
if (!release) return;
let task: ZipTask | null = null;
try {
task = await loadZipTask(id, OWNER);
if (!task || task.status === "completed" || task.status === "failed") return;
task.status = "running";
task.stage = "宿主机正在准备原图";
await saveZipTask(task);
const present = task.zipFile && await stat(task.zipFile.path).catch(() => null);
if (!present || present.size !== task.zipFile!.bytes) {
await prepareCanvasZip(task, ({ index, signal }) => fetchInput(job, index, signal));
}
task.stage = "正在传送至七牛";
await saveZipTask(task);
await uploadArchive(task, job);
task.status = "completed";
task.stage = task.failedCount ? "部分原图未能读取" : "打包完成";
task.compressPercent = 100;
task.error = undefined;
task.updatedAt = Date.now();
task.expiresAt = Date.now() + 24 * 60 * 60_000;
await saveZipTask(task);
if (task.zipFile) await rm(task.zipFile.path, { force: true }).catch(() => undefined);
} catch (error) {
if (task) {
task.status = "failed";
task.stage = "打包未完成";
task.error = (error instanceof Error ? error.message : "宿主机打包失败").slice(0, 450);
task.updatedAt = Date.now();
await saveZipTask(task);
}
} finally { await release(); }
}
function drain() {
if (running) return;
const next = queue.entries().next().value;
if (!next) return;
const [id, job] = next;
queue.delete(id);
running = id;
void run(id, job).catch(() => console.error("Canvas ZIP job could not be saved")).finally(() => {
running = undefined;
drain();
});
}
export const router = createServer((request, response) => {
void (async () => {
if (!authorized(request)) { reply(response, 401, {}); return; }
if (request.method === "GET" && request.url === "/health") {
reply(response, 200, { ok: true, active: Number(Boolean(running)), waiting: queue.size });
return;
}
const id = request.url?.match(/^\/v1\/jobs\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i)?.[1];
if (!id || request.method !== "PUT") { reply(response, 404, {}); return; }
if (accepting.has(id)) { reply(response, 503, {}); return; }
accepting.add(id);
try {
const job = await readJob(request);
const publicUrl = new URL(job.storage.publicUrl);
if (publicUrl.protocol !== "https:" || !publicUrl.pathname.endsWith("/" + zipStorageKey(id))) {
reply(response, 400, {}); return;
}
let task = await loadZipTask(id, OWNER);
if (running === id || queue.has(id)) {
if (task) reply(response, 200, snapshot(task)); else reply(response, 503, {});
return;
}
if (queue.size >= 8 && (!task || task.status === "pending" || task.status === "running")) {
reply(response, 503, {}); return;
}
if (task?.status === "failed" && job.attempt > (task.attempt ?? 0)) {
task.attempt = job.attempt;
task.status = "pending";
task.stage = "等待宿主机打包";
task.error = undefined;
task.expiresAt = job.expiresAt;
task.updatedAt = Date.now();
await saveZipTask(task);
}
if (!task) {
const now = Date.now();
task = {
id, userId: OWNER, folderName: job.folderName, attempt: job.attempt,
files: job.files.map(item => ({ url: item.sources[0] || "", name: item.name })),
totalCount: job.files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0,
status: "pending", stage: "等待宿主机打包", createdAt: now, updatedAt: now, expiresAt: job.expiresAt,
};
await saveZipTask(task);
}
if (task.status === "pending" || task.status === "running") {
queue.set(id, job);
drain();
}
reply(response, 200, snapshot(task));
void cleanupZipTasks().catch(() => undefined);
} finally { accepting.delete(id); }
})().catch(() => {
if (!response.headersSent) reply(response, 400, { error: "无效的打包请求" });
});
});
if (require.main === module) {
router.listen(Number(process.env.CANVAS_ZIP_PORT || 18343), "127.0.0.1");
}