import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { timingSafeEqual } from "node:crypto"; import { open, rm, stat } from "node:fs/promises"; import { MIMEType } from "node:util"; import { claimZipTask, cleanupZipTasks, loadZipTask, saveZipTask, type ZipTask } from "../../lib/canvas/download-zip-task-store"; import { prepareCanvasZip } from "../../lib/canvas/download-zip-prepare"; import { normalizeCanvasDownloadAsset } from "../../lib/canvas/photoshop-download-normalize"; import { zipStorageKey, zipWorkerJob, type ZipWorkerJob, type ZipWorkerSnapshot } from "../../lib/canvas/remote-zip-contract"; import { openPublicMedia } from "../../lib/video-canvas/download-media"; const TOKEN = process.env.CANVAS_ZIP_WORKER_TOKEN || ""; if (TOKEN.length < 32 || !process.env.CANVAS_DOWNLOAD_ZIP_DIR) { throw new Error("Set CANVAS_ZIP_WORKER_TOKEN and CANVAS_DOWNLOAD_ZIP_DIR"); } const OWNER = "zip-worker"; const IMAGE_LIMIT = 125 * 1024 ** 2; const PART_BYTES = 8 * 1024 ** 2; const queue = new Map(); const accepting = new Set(); let running: string | undefined; function authorized(request: IncomingMessage) { const actual = Buffer.from(request.headers.authorization || ""); const expected = Buffer.from("Bearer " + TOKEN); return actual.length === expected.length && timingSafeEqual(actual, expected); } function reply(response: ServerResponse, status: number, body: unknown) { response.writeHead(status, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" }); response.end(JSON.stringify(body)); } async function readJob(request: IncomingMessage) { let bytes = 0; const body: Buffer[] = []; for await (const chunk of request) { const value = Buffer.from(chunk); bytes += value.length; if (bytes > 1024 ** 2) throw new Error("Invalid job size"); body.push(value); } return zipWorkerJob.parse(JSON.parse(Buffer.concat(body).toString("utf8"))); } function snapshot(task: ZipTask): ZipWorkerSnapshot { const { status, stage, error, processedCount, successCount, failedCount, compressPercent } = task; return { status, stage, error, processedCount, successCount, failedCount, compressPercent, bytes: task.status === "completed" ? task.zipFile?.bytes : undefined }; } async function fetchInput(job: ZipWorkerJob, index: number, signal: AbortSignal) { const item = job.files[index]; const appHost = new URL(job.appOrigin).hostname; const allowUrl = (url: URL) => url.hostname !== appHost && url.hostname !== "ai.yydd.ai"; for (const url of item.sources) { if (signal.aborted) break; try { const response = await openPublicMedia(url, { allowUrl, signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), }); if (response.statusCode !== 200 || Number(response.headers["content-length"] || 0) > IMAGE_LIMIT) { response.destroy(); continue; } const contentType = new MIMEType(response.headers["content-type"] || "application/octet-stream").essence; const buffers: Buffer[] = []; let bytes = 0; for await (const packet of response) { const chunk = Buffer.from(packet); bytes += chunk.length; if (bytes > IMAGE_LIMIT) throw new Error("Original exceeds the download limit"); buffers.push(chunk); } return await normalizeCanvasDownloadAsset({ item: { url, name: item.name }, index, bytes: Buffer.concat(buffers, bytes), contentType, includeSequencePrefix: true, }); } catch { /* Try the next resolved source. */ } } return { ok: false as const, url: "", name: item.name, error: "宿主机暂时无法读取这张原图" }; } async function uploadArchive(task: ZipTask, job: ZipWorkerJob) { const { bucket, token, publicUrl } = job.storage; const { path, bytes } = task.zipFile!; const key = zipStorageKey(task.id); const endpoint = "https://upload.qiniup.com/buckets/" + encodeURIComponent(bucket) + "/objects/" + encodeURIComponent(Buffer.from(key).toString("base64").replace(/\+/g, "-").replace(/\//g, "_")) + "/uploads"; const storedUrl = new URL(publicUrl); const hasExactSize = async () => { try { const response = await fetch(storedUrl, { method: "HEAD", redirect: "error", signal: AbortSignal.timeout(8_000) }); return response.ok && Number(response.headers.get("content-length")) === bytes; } catch { return false; } }; if (await hasExactSize()) return; const request = async (url: string, method: string, body?: Buffer | string, contentType?: string) => { const headers: Record = { Authorization: "UpToken " + token }; if (contentType) headers["Content-Type"] = contentType; return await fetch(url, { method, headers, body: Buffer.isBuffer(body) ? Uint8Array.from(body) : body, redirect: "error", signal: AbortSignal.timeout(method === "PUT" ? 180_000 : 30_000), }); }; const init = await request(endpoint, "POST"); if (!init.ok) throw new Error("七牛未能开始接收压缩包(" + init.status + ")"); const { uploadId } = await init.json() as { uploadId?: string }; if (!uploadId) throw new Error("七牛未返回上传编号"); const uploadUrl = endpoint + "/" + encodeURIComponent(uploadId); const parts: Array<{ partNumber: number; etag: string }> = []; try { const handle = await open(path, "r"); try { for (let offset = 0; offset < bytes; offset += PART_BYTES) { const data = Buffer.allocUnsafe(Math.min(PART_BYTES, bytes - offset)); let read = 0; while (read < data.length) { const result = await handle.read(data, read, data.length - read, offset + read); if (!result.bytesRead) throw new Error("压缩包读取不完整"); read += result.bytesRead; } const partNumber = parts.length + 1; const response = await request(uploadUrl + "/" + partNumber, "PUT", data, "application/octet-stream"); if (!response.ok) throw new Error("七牛接收分片失败(" + response.status + ")"); const { etag } = await response.json() as { etag?: string }; if (!etag) throw new Error("七牛未确认上传分片"); parts.push({ partNumber, etag }); task.stage = "正在传送至七牛 " + Math.floor((offset + data.length) / bytes * 100) + "%"; task.updatedAt = Date.now(); await saveZipTask(task); } } finally { await handle.close(); } const complete = await request(uploadUrl, "POST", JSON.stringify({ parts, fname: task.folderName + ".zip", mimeType: "application/zip", }), "application/json"); if (!complete.ok) { if (complete.status === 614 && await hasExactSize()) return; throw new Error("七牛未确认压缩包(" + complete.status + ")"); } const saved = await complete.json() as { key?: string; fsize?: number }; if (saved.key !== key || saved.fsize !== bytes) throw new Error("七牛返回的压缩包信息不一致"); } catch (error) { await request(uploadUrl, "DELETE").then(response => response.body?.cancel()).catch(() => undefined); throw error; } } async function run(id: string, job: ZipWorkerJob) { const release = await claimZipTask(id); if (!release) return; let task: ZipTask | null = null; try { task = await loadZipTask(id, OWNER); if (!task || task.status === "completed" || task.status === "failed") return; task.status = "running"; task.stage = "宿主机正在准备原图"; await saveZipTask(task); const present = task.zipFile && await stat(task.zipFile.path).catch(() => null); if (!present || present.size !== task.zipFile!.bytes) { await prepareCanvasZip(task, ({ index, signal }) => fetchInput(job, index, signal)); } task.stage = "正在传送至七牛"; await saveZipTask(task); await uploadArchive(task, job); task.status = "completed"; task.stage = task.failedCount ? "部分原图未能读取" : "打包完成"; task.compressPercent = 100; task.error = undefined; task.updatedAt = Date.now(); task.expiresAt = Date.now() + 24 * 60 * 60_000; await saveZipTask(task); if (task.zipFile) await rm(task.zipFile.path, { force: true }).catch(() => undefined); } catch (error) { if (task) { task.status = "failed"; task.stage = "打包未完成"; task.error = (error instanceof Error ? error.message : "宿主机打包失败").slice(0, 450); task.updatedAt = Date.now(); await saveZipTask(task); } } finally { await release(); } } function drain() { if (running) return; const next = queue.entries().next().value; if (!next) return; const [id, job] = next; queue.delete(id); running = id; void run(id, job).catch(() => console.error("Canvas ZIP job could not be saved")).finally(() => { running = undefined; drain(); }); } export const router = createServer((request, response) => { void (async () => { if (!authorized(request)) { reply(response, 401, {}); return; } if (request.method === "GET" && request.url === "/health") { reply(response, 200, { ok: true, active: Number(Boolean(running)), waiting: queue.size }); return; } const id = request.url?.match(/^\/v1\/jobs\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i)?.[1]; if (!id || request.method !== "PUT") { reply(response, 404, {}); return; } if (accepting.has(id)) { reply(response, 503, {}); return; } accepting.add(id); try { const job = await readJob(request); const publicUrl = new URL(job.storage.publicUrl); if (publicUrl.protocol !== "https:" || !publicUrl.pathname.endsWith("/" + zipStorageKey(id))) { reply(response, 400, {}); return; } let task = await loadZipTask(id, OWNER); if (running === id || queue.has(id)) { if (task) reply(response, 200, snapshot(task)); else reply(response, 503, {}); return; } if (queue.size >= 8 && (!task || task.status === "pending" || task.status === "running")) { reply(response, 503, {}); return; } if (task?.status === "failed" && job.attempt > (task.attempt ?? 0)) { task.attempt = job.attempt; task.status = "pending"; task.stage = "等待宿主机打包"; task.error = undefined; task.expiresAt = job.expiresAt; task.updatedAt = Date.now(); await saveZipTask(task); } if (!task) { const now = Date.now(); task = { id, userId: OWNER, folderName: job.folderName, attempt: job.attempt, files: job.files.map(item => ({ url: item.sources[0] || "", name: item.name })), totalCount: job.files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0, status: "pending", stage: "等待宿主机打包", createdAt: now, updatedAt: now, expiresAt: job.expiresAt, }; await saveZipTask(task); } if (task.status === "pending" || task.status === "running") { queue.set(id, job); drain(); } reply(response, 200, snapshot(task)); void cleanupZipTasks().catch(() => undefined); } finally { accepting.delete(id); } })().catch(() => { if (!response.headersSent) reply(response, 400, { error: "无效的打包请求" }); }); }); if (require.main === module) { router.listen(Number(process.env.CANVAS_ZIP_PORT || 18343), "127.0.0.1"); }