Merge pull request 'Run detail image ZIP exports on the video host' (#110) from codex/canvas-zip-worker into main

This commit is contained in:
WuYongJie 2026-11-20 09:50:55 +08:00
commit 7fe5ec24a8
13 changed files with 938 additions and 275 deletions

View File

@ -1,12 +1,11 @@
import { writeDownloadZip, readDownloadZip } from "@/lib/canvas/download-zip-stream"; import { readDownloadZip } from "@/lib/canvas/download-zip-stream";
import { claimZipTask, loadZipTask, saveZipTask, zipTaskPath, assertZipDiskSpace, cleanupZipTasks, assertZipTaskCapacity, type ZipTask } from "@/lib/canvas/download-zip-task-store"; import { claimZipTask, loadZipTask, saveZipTask, zipTaskPath, assertZipDiskSpace, cleanupZipTasks, assertZipTaskCapacity, type ZipTask } from "@/lib/canvas/download-zip-task-store";
import { parseDownloadByteRange } from "@/lib/canvas/download-byte-range"; import { parseDownloadByteRange } from "@/lib/canvas/download-byte-range";
import { createReadStream } from "node:fs"; import { mkdir, stat, rm } from "node:fs/promises";
import { mkdir, stat, writeFile, rm } from "node:fs/promises";
import { join } from "node:path"; import { join } from "node:path";
import { randomUUID } from "node:crypto"; import { randomUUID } from "node:crypto";
import { after, NextRequest, NextResponse } from "next/server"; import { after, NextRequest, NextResponse } from "next/server";
import JSZip from "jszip"; import { prepareCanvasZip } from "@/lib/canvas/download-zip-prepare";
import { auth } from "@/app/(auth)/auth"; import { auth } from "@/app/(auth)/auth";
import { fetchCanvasDownloadAsset } from "@/lib/canvas/photoshop-safe-download"; import { fetchCanvasDownloadAsset } from "@/lib/canvas/photoshop-safe-download";
import { publishCanvasZip } from "@/lib/storage/canvas-zip-delivery"; import { publishCanvasZip } from "@/lib/storage/canvas-zip-delivery";
@ -29,16 +28,6 @@ function sanitizeFolderName(value: string): string {
return cleaned && !/^\.+$/.test(cleaned) ? cleaned.slice(0, 100) : `canvas-images-${Date.now()}`; return cleaned && !/^\.+$/.test(cleaned) ? cleaned.slice(0, 100) : `canvas-images-${Date.now()}`;
} }
function buildContentDisposition(folderName: string): string {
const encodedName = encodeURIComponent(`${folderName}.zip`);
const asciiName =
folderName
.replace(/[^\x20-\x7E]+/g, "")
.replace(/"/g, "")
.trim() || "canvas-images";
return `attachment; filename="${asciiName}.zip"; filename*=UTF-8''${encodedName}`;
}
function getRequestOrigin(request: NextRequest): string { function getRequestOrigin(request: NextRequest): string {
const forwardedHost = (request.headers.get("x-forwarded-host") || "") const forwardedHost = (request.headers.get("x-forwarded-host") || "")
.split(",")[0] .split(",")[0]
@ -99,50 +88,20 @@ async function processTask(id: string, userId: string, requestOrigin: string, co
if (!release) return; if (!release) return;
task = await loadZipTask(id, userId); task = await loadZipTask(id, userId);
if (!task || task.status === "completed" || task.status === "failed") return; if (!task || task.status === "completed" || task.status === "failed") return;
if (task.remote) {
const { updateRemoteCanvasZip } = await import("@/lib/canvas/remote-zip-client");
await updateRemoteCanvasZip(task, requestOrigin);
return;
}
await assertZipDiskSpace(); await assertZipDiskSpace();
const existingZip = task.zipFile && task.contentDisposition && await stat(task.zipFile.path).catch(() => null); const existingZip = task.zipFile && task.contentDisposition && await stat(task.zipFile.path).catch(() => null);
const reuseZip = existingZip && existingZip.size === task.zipFile!.bytes; const reuseZip = existingZip && existingZip.size === task.zipFile!.bytes;
task.status = "running"; task.stage = reuseZip ? "正在准备加速下载" : "正在准备原图"; task.updatedAt = Date.now(); task.status = "running"; task.stage = reuseZip ? "正在准备加速下载" : "正在准备原图"; task.updatedAt = Date.now();
await saveZipTask(task); await saveZipTask(task);
if (!reuseZip) { if (!reuseZip) {
delete task.zipFile; delete task.contentDisposition; delete task.cdnUrl; await prepareCanvasZip(task, (params) =>
// Stage each image on disk, not all PNG buffers in the web process heap. fetchCanvasDownloadAsset({ ...params, requestOrigin, cookie })
// A restart resumes from these files using the current owner's session. );
let cursor = 0;
let writes = Promise.resolve();
const persist = () => { const current = task!; writes = writes.then(() => saveZipTask(current)); return writes; };
const results = await Promise.allSettled(Array.from({ length: Math.min(2, task.files.length) }, async () => {
while (cursor < task!.files.length) {
const index = cursor++; const file = task!.files[index];
const path = join(zipTaskPath(id), `${index}.image`);
if (file.prepared && file.filename && (await stat(path).catch(() => null))?.size) continue;
delete file.prepared; delete file.filename;
const result = await fetchCanvasDownloadAsset({ file, index, requestOrigin, cookie,
signal: AbortSignal.timeout(120_000) }).catch(() => ({ ok: false as const, error: "原图读取超时或源地址暂时不可用" }));
if (result.ok) {
await assertZipDiskSpace(result.bytes.length);
await writeFile(path, result.bytes, { mode: 0o600 });
file.prepared = `${index}.image`; file.filename = result.filename; delete file.error;
} else file.error = result.error;
task!.processedCount = task!.files.filter(f => f.prepared || f.error).length;
task!.successCount = task!.files.filter(f => f.prepared).length;
task!.failedCount = task!.files.filter(f => f.error).length;
task!.stage = `正在准备原图 ${task!.processedCount}/${task!.totalCount}`;
task!.updatedAt = Date.now(); await persist();
}
}));
const rejected = results.find(result => result.status === "rejected");
if (rejected?.status === "rejected") throw rejected.reason;
if (!task.successCount) throw new Error("所选原图暂时都无法读取,请稍后重试;画布图片不会被删除");
const zip = new JSZip(); const folder = zip.folder(task.folderName) || zip;
for (const file of task.files) if (file.prepared && file.filename)
folder.file(file.filename, createReadStream(join(zipTaskPath(id), file.prepared))
.once("close", () => { void rm(join(zipTaskPath(id), file.prepared!), { force: true }).catch(() => undefined); }));
if (task.failedCount) folder.file("_download-failures.txt", task.files.filter(f => f.error)
.map(f => `${f.name || "图片"}:${f.error}`).join("\n"));
task.stage = "正在生成完整 ZIP 文件"; await saveZipTask(task);
task.zipFile = await writeDownloadZip(zip, task.id, percent => { task!.compressPercent = percent; });
task.contentDisposition = buildContentDisposition(task.folderName);
} }
task.stage = "正在准备下载链接"; await saveZipTask(task); task.stage = "正在准备下载链接"; await saveZipTask(task);
const cdn = await publishCanvasZip({ id: task.id, ...task.zipFile!, disposition: task.contentDisposition!, const cdn = await publishCanvasZip({ id: task.id, ...task.zipFile!, disposition: task.contentDisposition!,
@ -195,13 +154,20 @@ export async function POST(request: NextRequest) {
await mkdir(zipTaskPath(id), { mode: 0o700 }); await mkdir(zipTaskPath(id), { mode: 0o700 });
}); });
const now = Date.now(); const now = Date.now();
task = { id, userId: session.user.id, folderName: sanitizeFolderName(String(body.folderName || "")), files, const remote = body.scope === "style-detail" &&
process.env.CANVAS_ZIP_WORKER_URL && process.env.CANVAS_ZIP_WORKER_TOKEN
? { attempt: 0 } : undefined;
task = { id, userId: session.user.id, folderName: sanitizeFolderName(String(body.folderName || "")), files, remote,
totalCount: files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0, totalCount: files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0,
status: "pending", stage: "等待后台准备", createdAt: now, updatedAt: now, expiresAt: now + PREPARATION_TTL_MS }; status: "pending", stage: "等待后台准备", createdAt: now, updatedAt: now, expiresAt: now + PREPARATION_TTL_MS };
await saveZipTask(task); await saveZipTask(task);
} else if ((task.status === "failed" || task.status === "completed" && task.cdnObject && !task.cdnUrl) && body.retry === true) { } else if ((task.status === "failed" || task.status === "completed" && task.cdnObject && !task.cdnUrl) && body.retry === true) {
task.status = "pending"; task.error = undefined; task.status = "pending"; task.error = undefined;
await assertZipTaskCapacity(session.user.id); await assertZipTaskCapacity(session.user.id);
if (task.remote) {
task.remote.attempt++;
delete task.remote.sources;
}
if (!task.zipFile) { if (!task.zipFile) {
for (const file of task.files) delete file.error; for (const file of task.files) delete file.error;
task.processedCount = task.successCount = task.files.filter(file => file.prepared).length; task.processedCount = task.successCount = task.files.filter(file => file.prepared).length;
@ -211,7 +177,10 @@ export async function POST(request: NextRequest) {
await saveZipTask(task); await saveZipTask(task);
} }
if (task.status === "pending" || task.status === "running") schedule(task, request); if (task.status === "pending" || task.status === "running") schedule(task, request);
after(() => cleanupZipTasks().catch(() => { console.warn("[CanvasDownloadZipTask] Expired file cleanup deferred"); })); after(() => cleanupZipTasks(async object => {
const { removeCanvasZipObject } = await import("@/lib/storage/canvas-zip-delivery");
await removeCanvasZipObject(object);
}).catch(() => { console.warn("[CanvasDownloadZipTask] Expired file cleanup deferred"); }));
return NextResponse.json({ success: true, task: serializeTask(task) }, { status: 202, headers: { "Cache-Control": "no-store" } }); return NextResponse.json({ success: true, task: serializeTask(task) }, { status: 202, headers: { "Cache-Control": "no-store" } });
} catch (error) { } catch (error) {
console.warn("[CanvasDownloadZipTask] Create rejected", { message: (error as Error).message }); console.warn("[CanvasDownloadZipTask] Create rejected", { message: (error as Error).message });
@ -226,6 +195,8 @@ export async function GET(request: NextRequest) {
const task = await loadZipTask(id, session.user.id); const task = await loadZipTask(id, session.user.id);
if (!task) return NextResponse.json({ error: "打包任务不存在或已过期,请重新打包" }, { status: 404 }); if (!task) return NextResponse.json({ error: "打包任务不存在或已过期,请重新打包" }, { status: 404 });
if (request.nextUrl.searchParams.get("download") === "1") { if (request.nextUrl.searchParams.get("download") === "1") {
if (task.remote && task.status === "completed" && task.cdnUrl)
return NextResponse.redirect(task.cdnUrl, 307);
if (task.status !== "completed" || !task.zipFile) return NextResponse.json({ error: "ZIP 尚未准备好" }, { status: 409 }); if (task.status !== "completed" || !task.zipFile) return NextResponse.json({ error: "ZIP 尚未准备好" }, { status: 409 });
if (task.cdnUrl && request.nextUrl.searchParams.get("local") !== "1") if (task.cdnUrl && request.nextUrl.searchParams.get("local") !== "1")
return NextResponse.redirect(task.cdnUrl, 307); return NextResponse.redirect(task.cdnUrl, 307);

View File

@ -5892,6 +5892,7 @@ export function AgentGenerationWorkspaceDarkV2({
headers: { "Content-Type": "application/json" }, headers: { "Content-Type": "application/json" },
cache: "no-store", cache: "no-store",
body: JSON.stringify({ body: JSON.stringify({
scope: params.scope === "detail" ? "style-detail" : undefined,
folderName: params.zipName, folderName: params.zipName,
files: normalizedUrls.map((imageUrl) => ({ files: normalizedUrls.map((imageUrl) => ({
url: imageUrl, url: imageUrl,

View File

@ -0,0 +1,57 @@
import { createReadStream } from "node:fs";
import { stat, writeFile, rm } from "node:fs/promises";
import { join } from "node:path";
import JSZip from "jszip";
import { writeDownloadZip } from "./download-zip-stream";
import { assertZipDiskSpace, saveZipTask, zipTaskPath, type ZipTask } from "./download-zip-task-store";
import type { CanvasDownloadAssetResult } from "./photoshop-safe-download";
export function buildZipContentDisposition(folderName: string) {
const name = `${folderName}.zip`;
const asciiName = folderName.replace(/[^\x20-\x7E]+/g, "").replace(/"/g, "").trim() || "canvas-images";
return `attachment; filename="${asciiName}.zip"; filename*=UTF-8''${encodeURIComponent(name)}`;
}
export async function prepareCanvasZip(
task: ZipTask,
fetchAsset: (params: { file: ZipTask["files"][number]; index: number; signal: AbortSignal }) => Promise<CanvasDownloadAssetResult>,
) {
delete task.zipFile; delete task.contentDisposition; delete task.cdnUrl;
// Stage each image on disk, not all PNG buffers in the web process heap.
// A restart resumes from these files using the current owner's session.
let cursor = 0;
let writes = Promise.resolve();
const persist = () => { const current = task; writes = writes.then(() => saveZipTask(current)); return writes; };
const results = await Promise.allSettled(Array.from({ length: Math.min(2, task.files.length) }, async () => {
while (cursor < task.files.length) {
const index = cursor++; const file = task.files[index];
const path = join(zipTaskPath(task.id), `${index}.image`);
if (file.prepared && file.filename && (await stat(path).catch(() => null))?.size) continue;
delete file.prepared; delete file.filename;
const result = await fetchAsset({ file, index,
signal: AbortSignal.timeout(120_000) }).catch(() => ({ ok: false as const, error: "原图读取超时或源地址暂时不可用" }));
if (result.ok) {
await assertZipDiskSpace(result.bytes.length);
await writeFile(path, result.bytes, { mode: 0o600 });
file.prepared = `${index}.image`; file.filename = result.filename; delete file.error;
} else file.error = result.error;
task.processedCount = task.files.filter(f => f.prepared || f.error).length;
task.successCount = task.files.filter(f => f.prepared).length;
task.failedCount = task.files.filter(f => f.error).length;
task.stage = `正在准备原图 ${task.processedCount}/${task.totalCount}`;
task.updatedAt = Date.now(); await persist();
}
}));
const rejected = results.find(result => result.status === "rejected");
if (rejected?.status === "rejected") throw rejected.reason;
if (!task.successCount) throw new Error("所选原图暂时都无法读取,请稍后重试;画布图片不会被删除");
const zip = new JSZip(); const folder = zip.folder(task.folderName) || zip;
for (const file of task.files) if (file.prepared && file.filename)
folder.file(file.filename, createReadStream(join(zipTaskPath(task.id), file.prepared))
.once("close", () => { void rm(join(zipTaskPath(task.id), file.prepared!), { force: true }).catch(() => undefined); }));
if (task.failedCount) folder.file("_download-failures.txt", task.files.filter(f => f.error)
.map(f => `${f.name || "图片"}:${f.error}`).join("\n"));
task.stage = "正在生成完整 ZIP 文件"; await saveZipTask(task);
task.zipFile = await writeDownloadZip(zip, task.id, percent => { task.compressPercent = percent; });
task.contentDisposition = buildZipContentDisposition(task.folderName);
}

View File

@ -10,6 +10,8 @@ export type ZipTask = {
totalCount: number; processedCount: number; successCount: number; failedCount: number; totalCount: number; processedCount: number; successCount: number; failedCount: number;
compressPercent: number; status: "pending" | "running" | "completed" | "failed"; compressPercent: number; status: "pending" | "running" | "completed" | "failed";
stage: string; error?: string; zipFile?: { path: string; bytes: number }; stage: string; error?: string; zipFile?: { path: string; bytes: number };
remote?: { attempt: number; storageConfigId?: string; sources?: string[][] };
attempt?: number;
contentDisposition?: string; cdnUrl?: string; cdnObject?: { storageConfigId: string; key: string }; contentDisposition?: string; cdnUrl?: string; cdnObject?: { storageConfigId: string; key: string };
createdAt: number; updatedAt: number; expiresAt: number; createdAt: number; updatedAt: number; expiresAt: number;
}; };
@ -62,7 +64,7 @@ export async function assertZipDiskSpace(bytes = 0) {
} }
let cleanupAt = 0; let cleanupAt = 0;
export async function cleanupZipTasks() { export async function cleanupZipTasks(removeObject?: (object: NonNullable<ZipTask["cdnObject"]>) => Promise<void>) {
if (Date.now() - cleanupAt < 60_000) return; if (Date.now() - cleanupAt < 60_000) return;
cleanupAt = Date.now(); cleanupAt = Date.now();
for (const id of await readdir(downloadZipDirectory).catch(() => [] as string[])) { for (const id of await readdir(downloadZipDirectory).catch(() => [] as string[])) {
@ -74,10 +76,7 @@ export async function cleanupZipTasks() {
const release = await claimZipTask(id); const release = await claimZipTask(id);
if (!release) continue; if (!release) continue;
try { try {
if (task.cdnObject) { if (task.cdnObject && removeObject) await removeObject(task.cdnObject).catch(() => undefined);
const { removeCanvasZipObject } = await import("@/lib/storage/canvas-zip-delivery");
await removeCanvasZipObject(task.cdnObject).catch(() => undefined);
}
await rm(join(downloadZipDirectory, `${id}.zip`), { force: true }); await rm(join(downloadZipDirectory, `${id}.zip`), { force: true });
await rm(join(downloadZipDirectory, `${id}.zip.partial`), { force: true }); await rm(join(downloadZipDirectory, `${id}.zip.partial`), { force: true });
} finally { await release(); } } finally { await release(); }

View File

@ -0,0 +1,212 @@
export type DownloadFileItem = {
url: string;
name?: string;
};
export type CanvasDownloadAssetResult =
| {
ok: true;
filename: string;
bytes: Buffer;
contentType: string;
}
| {
ok: false;
url: string;
name?: string;
error: string;
};
export function sanitizeCanvasDownloadFileName(value: string, fallback: string): string {
const cleaned = String(value || "")
.replace(/\.[a-zA-Z0-9]{2,6}$/g, "")
.replace(/[\\/:*?"<>|]+/g, "-")
.replace(/\s+/g, "-")
.replace(/-+/g, "-")
.replace(/^-+|-+$/g, "")
.trim()
.slice(0, 120);
return cleaned || fallback;
}
export function buildAttachmentContentDisposition(filename: string): string {
const safeAsciiName =
filename
.replace(/[^\x20-\x7E]+/g, "")
.replace(/"/g, "")
.trim() || `canvas-image-${Date.now()}.png`;
return `attachment; filename="${safeAsciiName}"; filename*=UTF-8''${encodeURIComponent(filename)}`;
}
function getContentTypeMime(contentType: string): string {
return String(contentType || "")
.split(";")[0]
?.trim()
.toLowerCase();
}
function hasAsciiAt(buffer: Buffer, offset: number, value: string): boolean {
return buffer.subarray(offset, offset + value.length).toString("latin1") === value;
}
function looksLikeSvg(buffer: Buffer, contentType: string): boolean {
if (getContentTypeMime(contentType) === "image/svg+xml") return true;
const text = buffer.subarray(0, Math.min(buffer.length, 1024)).toString("utf8").trimStart();
return /^(?:<\?xml[^>]*>\s*)?<svg[\s>]/i.test(text);
}
function looksLikeErrorDocument(buffer: Buffer, contentType: string): boolean {
const mime = getContentTypeMime(contentType);
if (
mime.startsWith("text/") ||
mime === "application/json" ||
mime === "application/xml" ||
mime === "application/xhtml+xml"
) {
return true;
}
const text = buffer.subarray(0, Math.min(buffer.length, 512)).toString("utf8").trimStart();
return /^(?:<!doctype\s+html|<html[\s>]|<\?xml|<error[\s>]|[{[])/i.test(text);
}
function detectAsset(bytes: Buffer, contentType: string): { kind: "image" | "video" | "audio"; mimeType: string; ext: string } | null {
if (bytes.length < 4) return null;
if (bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
return { kind: "image", mimeType: "image/png", ext: "png" };
}
if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
return { kind: "image", mimeType: "image/jpeg", ext: "jpg" };
}
if (hasAsciiAt(bytes, 0, "RIFF") && hasAsciiAt(bytes, 8, "WEBP")) {
return { kind: "image", mimeType: "image/webp", ext: "webp" };
}
if (hasAsciiAt(bytes, 0, "GIF87a") || hasAsciiAt(bytes, 0, "GIF89a")) {
return { kind: "image", mimeType: "image/gif", ext: "gif" };
}
if (hasAsciiAt(bytes, 0, "BM")) {
return { kind: "image", mimeType: "image/bmp", ext: "bmp" };
}
if (
bytes.subarray(0, 4).equals(Buffer.from([0x49, 0x49, 0x2a, 0x00])) ||
bytes.subarray(0, 4).equals(Buffer.from([0x4d, 0x4d, 0x00, 0x2a]))
) {
return { kind: "image", mimeType: "image/tiff", ext: "tiff" };
}
if (looksLikeSvg(bytes, contentType)) {
return { kind: "image", mimeType: "image/svg+xml", ext: "svg" };
}
if (hasAsciiAt(bytes, 0, "RIFF") && hasAsciiAt(bytes, 8, "WAVE")) return {kind:"audio",mimeType:"audio/wav",ext:"wav"};
if (hasAsciiAt(bytes, 0, "ID3") || (bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0 && (bytes[1] & 0x06) !== 0)) return {kind:"audio",mimeType:"audio/mpeg",ext:"mp3"};
if (hasAsciiAt(bytes, 0, "OggS")) return {kind:"audio",mimeType:"audio/ogg",ext:"ogg"};
if (hasAsciiAt(bytes, 0, "fLaC")) return {kind:"audio",mimeType:"audio/flac",ext:"flac"};
if (getContentTypeMime(contentType).startsWith("audio/")) return {kind:"audio",mimeType:getContentTypeMime(contentType),ext:getContentTypeMime(contentType).includes("mp4") ? "m4a" : getContentTypeMime(contentType).split("/")[1]};
if (hasAsciiAt(bytes, 4, "ftyp")) {
const brand = bytes.subarray(8, Math.min(bytes.length, 32)).toString("latin1").toLowerCase();
if (brand.includes("m4a")) return {kind:"audio",mimeType:"audio/mp4",ext:"m4a"};
if (brand.includes("avif")) return { kind: "image", mimeType: "image/avif", ext: "avif" };
if (brand.includes("heic") || brand.includes("heif") || brand.includes("mif1")) {
return { kind: "image", mimeType: "image/heic", ext: "heic" };
}
return { kind: "video", mimeType: "video/mp4", ext: "mp4" };
}
if (bytes.subarray(0, 4).equals(Buffer.from([0x1a, 0x45, 0xdf, 0xa3]))) {
return { kind: "video", mimeType: "video/webm", ext: "webm" };
}
const mime = getContentTypeMime(contentType);
if (mime.startsWith("image/")) {
const ext = mime.includes("jpeg") ? "jpg" : mime.split("/")[1] || "png";
return { kind: "image", mimeType: mime, ext };
}
if (mime.startsWith("video/")) {
const ext = mime.includes("quicktime") ? "mov" : mime.split("/")[1] || "mp4";
return { kind: "video", mimeType: mime, ext };
}
return null;
}
export async function convertImageToPhotoshopPng(bytes: Buffer): Promise<Buffer> {
const sharp = await import("sharp");
const image = sharp.default(bytes, {
animated: false,
failOn: "none",
limitInputPixels: false,
});
const metadata = await image.metadata();
if (!metadata.width || !metadata.height) {
throw new Error("无法解析图片尺寸");
}
// RGB/RGBA, 8-bit, single-frame PNG is already compatible; preserve its exact bytes.
const rgb = (metadata.space === "srgb" && metadata.depth === "uchar")
|| (metadata.space === "rgb16" && metadata.depth === "ushort");
if (metadata.format === "png" && rgb
&& !metadata.isPalette && (metadata.pages || 1) === 1 && (!metadata.orientation || metadata.orientation === 1)) return bytes;
const output = image.rotate().toColorspace("srgb");
if (metadata.icc) output.withIccProfile("srgb");
return output.png({ compressionLevel: 3, adaptiveFiltering: true }).toBuffer();
}
function buildCanvasAssetFilename(params: {
item: DownloadFileItem;
index: number;
ext: string;
includeSequencePrefix: boolean;
}) {
const baseName = sanitizeCanvasDownloadFileName(
params.item.name || `canvas-image-${params.index + 1}`,
`canvas-image-${params.index + 1}`
);
if (!params.includeSequencePrefix) {
return `${baseName}.${params.ext}`;
}
const sequencePrefix = String(params.index + 1).padStart(3, "0");
return `${sequencePrefix}-${baseName}.${params.ext}`;
}
export async function normalizeCanvasDownloadAsset(params: {
item: DownloadFileItem;
index: number;
bytes: Buffer;
contentType: string;
includeSequencePrefix: boolean;
}, convert = convertImageToPhotoshopPng): Promise<CanvasDownloadAssetResult> {
if (params.bytes.length === 0) {
throw new Error("empty response");
}
if (looksLikeErrorDocument(params.bytes, params.contentType) && !looksLikeSvg(params.bytes, params.contentType)) {
throw new Error("源地址返回的不是图片文件");
}
const detected = detectAsset(params.bytes, params.contentType);
if (!detected) {
throw new Error("无法识别图片格式");
}
if (detected.kind !== "image") {
return {
ok: true,
filename: buildCanvasAssetFilename({
item: params.item,
index: params.index,
ext: detected.ext,
includeSequencePrefix: params.includeSequencePrefix,
}),
bytes: params.bytes,
contentType: detected.mimeType,
};
}
const pngBytes = await convert(params.bytes);
return {
ok: true,
filename: buildCanvasAssetFilename({
item: params.item,
index: params.index,
ext: "png",
includeSequencePrefix: params.includeSequencePrefix,
}),
bytes: pngBytes,
contentType: "image/png",
};
}

View File

@ -4,24 +4,15 @@ import { fetchImageBytes, onceImageBytes } from "@/lib/storage/image-byte-cache"
import { extractImageProxySourceUrl, inferOriginalImageUrl } from "@/lib/image-url"; import { extractImageProxySourceUrl, inferOriginalImageUrl } from "@/lib/image-url";
import { getDownloadFailure } from "./download-error"; import { getDownloadFailure } from "./download-error";
type DownloadFileItem = { import {
url: string; convertImageToPhotoshopPng,
name?: string; normalizeCanvasDownloadAsset as normalizeAsset,
}; type CanvasDownloadAssetResult,
type DownloadFileItem,
} from "./photoshop-download-normalize";
export type CanvasDownloadAssetResult = export { buildAttachmentContentDisposition, convertImageToPhotoshopPng, sanitizeCanvasDownloadFileName } from "./photoshop-download-normalize";
| { export type { CanvasDownloadAssetResult } from "./photoshop-download-normalize";
ok: true;
filename: string;
bytes: Buffer;
contentType: string;
}
| {
ok: false;
url: string;
name?: string;
error: string;
};
const DEFAULT_FETCH_TIMEOUT_MS = 30_000; const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
const DEFAULT_FETCH_RETRY_COUNT = 3; const DEFAULT_FETCH_RETRY_COUNT = 3;
@ -37,27 +28,6 @@ function normalizeDownloadError(error: unknown): string {
return "download failed"; return "download failed";
} }
export function sanitizeCanvasDownloadFileName(value: string, fallback: string): string {
const cleaned = String(value || "")
.replace(/\.[a-zA-Z0-9]{2,6}$/g, "")
.replace(/[\\/:*?"<>|]+/g, "-")
.replace(/\s+/g, "-")
.replace(/-+/g, "-")
.replace(/^-+|-+$/g, "")
.trim()
.slice(0, 120);
return cleaned || fallback;
}
export function buildAttachmentContentDisposition(filename: string): string {
const safeAsciiName =
filename
.replace(/[^\x20-\x7E]+/g, "")
.replace(/"/g, "")
.trim() || `canvas-image-${Date.now()}.png`;
return `attachment; filename="${safeAsciiName}"; filename*=UTF-8''${encodeURIComponent(filename)}`;
}
function buildRemoteFetchHeaders(fileUrl: string, requestOrigin: string, cookie: string): HeadersInit { function buildRemoteFetchHeaders(fileUrl: string, requestOrigin: string, cookie: string): HeadersInit {
const headers: Record<string, string> = { const headers: Record<string, string> = {
Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,video/*,*/*;q=0.8", Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,video/*,*/*;q=0.8",
@ -91,7 +61,7 @@ function resolveHttpUrl(value: string, requestOrigin: string): string {
} }
} }
function buildCanvasDownloadSourceCandidates(fileUrl: string, requestOrigin: string) { export function buildCanvasDownloadSourceCandidates(fileUrl: string, requestOrigin: string) {
const candidates: string[] = []; const candidates: string[] = [];
const seen = new Set<string>(); const seen = new Set<string>();
@ -117,180 +87,14 @@ function buildCanvasDownloadSourceCandidates(fileUrl: string, requestOrigin: str
return candidates; return candidates;
} }
function getContentTypeMime(contentType: string): string { export async function normalizeCanvasDownloadAsset(params: Parameters<typeof normalizeAsset>[0]) {
return String(contentType || "") return normalizeAsset(params, async bytes => {
.split(";")[0] const digest = createHash("sha256").update(bytes).digest("hex");
?.trim() const { buffer } = await onceImageBytes(`photoshop:${digest}`, async () => ({
.toLowerCase(); buffer: await convertImageToPhotoshopPng(bytes), contentType: "image/png",
}
function hasAsciiAt(buffer: Buffer, offset: number, value: string): boolean {
return buffer.subarray(offset, offset + value.length).toString("latin1") === value;
}
function looksLikeSvg(buffer: Buffer, contentType: string): boolean {
if (getContentTypeMime(contentType) === "image/svg+xml") return true;
const text = buffer.subarray(0, Math.min(buffer.length, 1024)).toString("utf8").trimStart();
return /^(?:<\?xml[^>]*>\s*)?<svg[\s>]/i.test(text);
}
function looksLikeErrorDocument(buffer: Buffer, contentType: string): boolean {
const mime = getContentTypeMime(contentType);
if (
mime.startsWith("text/") ||
mime === "application/json" ||
mime === "application/xml" ||
mime === "application/xhtml+xml"
) {
return true;
}
const text = buffer.subarray(0, Math.min(buffer.length, 512)).toString("utf8").trimStart();
return /^(?:<!doctype\s+html|<html[\s>]|<\?xml|<error[\s>]|[{[])/i.test(text);
}
function detectAsset(bytes: Buffer, contentType: string): { kind: "image" | "video" | "audio"; mimeType: string; ext: string } | null {
if (bytes.length < 4) return null;
if (bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
return { kind: "image", mimeType: "image/png", ext: "png" };
}
if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
return { kind: "image", mimeType: "image/jpeg", ext: "jpg" };
}
if (hasAsciiAt(bytes, 0, "RIFF") && hasAsciiAt(bytes, 8, "WEBP")) {
return { kind: "image", mimeType: "image/webp", ext: "webp" };
}
if (hasAsciiAt(bytes, 0, "GIF87a") || hasAsciiAt(bytes, 0, "GIF89a")) {
return { kind: "image", mimeType: "image/gif", ext: "gif" };
}
if (hasAsciiAt(bytes, 0, "BM")) {
return { kind: "image", mimeType: "image/bmp", ext: "bmp" };
}
if (
bytes.subarray(0, 4).equals(Buffer.from([0x49, 0x49, 0x2a, 0x00])) ||
bytes.subarray(0, 4).equals(Buffer.from([0x4d, 0x4d, 0x00, 0x2a]))
) {
return { kind: "image", mimeType: "image/tiff", ext: "tiff" };
}
if (looksLikeSvg(bytes, contentType)) {
return { kind: "image", mimeType: "image/svg+xml", ext: "svg" };
}
if (hasAsciiAt(bytes, 0, "RIFF") && hasAsciiAt(bytes, 8, "WAVE")) return {kind:"audio",mimeType:"audio/wav",ext:"wav"};
if (hasAsciiAt(bytes, 0, "ID3") || (bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0 && (bytes[1] & 0x06) !== 0)) return {kind:"audio",mimeType:"audio/mpeg",ext:"mp3"};
if (hasAsciiAt(bytes, 0, "OggS")) return {kind:"audio",mimeType:"audio/ogg",ext:"ogg"};
if (hasAsciiAt(bytes, 0, "fLaC")) return {kind:"audio",mimeType:"audio/flac",ext:"flac"};
if (getContentTypeMime(contentType).startsWith("audio/")) return {kind:"audio",mimeType:getContentTypeMime(contentType),ext:getContentTypeMime(contentType).includes("mp4") ? "m4a" : getContentTypeMime(contentType).split("/")[1]};
if (hasAsciiAt(bytes, 4, "ftyp")) {
const brand = bytes.subarray(8, Math.min(bytes.length, 32)).toString("latin1").toLowerCase();
if (brand.includes("m4a")) return {kind:"audio",mimeType:"audio/mp4",ext:"m4a"};
if (brand.includes("avif")) return { kind: "image", mimeType: "image/avif", ext: "avif" };
if (brand.includes("heic") || brand.includes("heif") || brand.includes("mif1")) {
return { kind: "image", mimeType: "image/heic", ext: "heic" };
}
return { kind: "video", mimeType: "video/mp4", ext: "mp4" };
}
if (bytes.subarray(0, 4).equals(Buffer.from([0x1a, 0x45, 0xdf, 0xa3]))) {
return { kind: "video", mimeType: "video/webm", ext: "webm" };
}
const mime = getContentTypeMime(contentType);
if (mime.startsWith("image/")) {
const ext = mime.includes("jpeg") ? "jpg" : mime.split("/")[1] || "png";
return { kind: "image", mimeType: mime, ext };
}
if (mime.startsWith("video/")) {
const ext = mime.includes("quicktime") ? "mov" : mime.split("/")[1] || "mp4";
return { kind: "video", mimeType: mime, ext };
}
return null;
}
export async function convertImageToPhotoshopPng(bytes: Buffer): Promise<Buffer> {
const sharp = await import("sharp");
const image = sharp.default(bytes, {
animated: false,
failOn: "none",
limitInputPixels: false,
});
const metadata = await image.metadata();
if (!metadata.width || !metadata.height) {
throw new Error("无法解析图片尺寸");
}
// RGB/RGBA, 8-bit, single-frame PNG is already compatible; preserve its exact bytes.
const rgb = (metadata.space === "srgb" && metadata.depth === "uchar")
|| (metadata.space === "rgb16" && metadata.depth === "ushort");
if (metadata.format === "png" && rgb
&& !metadata.isPalette && (metadata.pages || 1) === 1 && (!metadata.orientation || metadata.orientation === 1)) return bytes;
const output = image.rotate().toColorspace("srgb");
if (metadata.icc) output.withIccProfile("srgb");
return output.png({ compressionLevel: 3, adaptiveFiltering: true }).toBuffer();
}
function buildCanvasAssetFilename(params: {
item: DownloadFileItem;
index: number;
ext: string;
includeSequencePrefix: boolean;
}) {
const baseName = sanitizeCanvasDownloadFileName(
params.item.name || `canvas-image-${params.index + 1}`,
`canvas-image-${params.index + 1}`
);
if (!params.includeSequencePrefix) {
return `${baseName}.${params.ext}`;
}
const sequencePrefix = String(params.index + 1).padStart(3, "0");
return `${sequencePrefix}-${baseName}.${params.ext}`;
}
export async function normalizeCanvasDownloadAsset(params: {
item: DownloadFileItem;
index: number;
bytes: Buffer;
contentType: string;
includeSequencePrefix: boolean;
}): Promise<CanvasDownloadAssetResult> {
if (params.bytes.length === 0) {
throw new Error("empty response");
}
if (looksLikeErrorDocument(params.bytes, params.contentType) && !looksLikeSvg(params.bytes, params.contentType)) {
throw new Error("源地址返回的不是图片文件");
}
const detected = detectAsset(params.bytes, params.contentType);
if (!detected) {
throw new Error("无法识别图片格式");
}
if (detected.kind !== "image") {
return {
ok: true,
filename: buildCanvasAssetFilename({
item: params.item,
index: params.index,
ext: detected.ext,
includeSequencePrefix: params.includeSequencePrefix,
}),
bytes: params.bytes,
contentType: detected.mimeType,
};
}
const digest = createHash("sha256").update(params.bytes).digest("hex");
const { buffer: pngBytes } = await onceImageBytes(`photoshop:${digest}`, async () => ({
buffer: await convertImageToPhotoshopPng(params.bytes), contentType: "image/png",
})); }));
return { return buffer;
ok: true, });
filename: buildCanvasAssetFilename({
item: params.item,
index: params.index,
ext: "png",
includeSequencePrefix: params.includeSequencePrefix,
}),
bytes: pngBytes,
contentType: "image/png",
};
} }
export async function fetchCanvasDownloadAsset(params: { export async function fetchCanvasDownloadAsset(params: {

View File

@ -0,0 +1,103 @@
import "server-only";
import { createHmac } from "node:crypto";
import { getDefaultStorageConfig, getStorageConfigById } from "@/lib/db/queries";
import { getArchivedImageUrlMappings } from "@/lib/storage/image-archive-db";
import { buildS3PublicUrl, toDbS3UploadConfig } from "@/lib/storage/service";
import { buildCanvasDownloadSourceCandidates } from "./photoshop-safe-download";
import { saveZipTask, type ZipTask } from "./download-zip-task-store";
import { zipStorageKey, zipWorkerSnapshot, type ZipWorkerJob } from "./remote-zip-contract";
async function resolveSources(task: ZipTask, origin: string) {
const candidates = task.files.map(file => buildCanvasDownloadSourceCandidates(file.url, origin));
const aliases = await getArchivedImageUrlMappings(candidates.flat());
const app = new URL(origin).hostname;
const isDirect = (value: string) => {
try {
const parsed = new URL(value);
return ["http:", "https:"].includes(parsed.protocol)
&& parsed.hostname !== app && parsed.hostname !== "ai.yydd.ai" && !parsed.username && !parsed.password;
} catch { return false; }
};
return candidates.map(values => [...new Set([
...values.flatMap(value => aliases[value] ? [aliases[value]] : []), ...values,
])].filter(isDirect).slice(0, 8));
}
export async function updateRemoteCanvasZip(task: ZipTask, origin: string) {
const remote = task.remote!;
const url = process.env.CANVAS_ZIP_WORKER_URL;
const token = process.env.CANVAS_ZIP_WORKER_TOKEN;
if (!url || !token) throw new Error("独立打包服务暂时不可用,请稍后重试");
const raw = remote.storageConfigId
? await getStorageConfigById({ id: remote.storageConfigId })
: await getDefaultStorageConfig();
const config = toDbS3UploadConfig(raw);
if (!raw?.isEnabled || config?.provider !== "qiniu-kodo" || config.region !== "cn-east-1" || !config.publicUrl?.startsWith("https://")) {
throw new Error("七牛打包存储暂时不可用,请稍后重试");
}
const key = zipStorageKey(task.id);
if (!remote.sources) {
remote.storageConfigId = raw.id;
remote.sources = await resolveSources(task, origin);
task.cdnObject = { storageConfigId: raw.id, key };
await saveZipTask(task);
}
const publicUrl = buildS3PublicUrl(config, key);
const policy = Buffer.from(JSON.stringify({
scope: config.bucket + ":" + key,
deadline: Math.floor(task.expiresAt / 1000),
insertOnly: 1,
returnBody: '{"key":"$(key)","fsize":$(fsize)}',
})).toString("base64").replace(/\+/g, "-").replace(/\//g, "_");
const mac = createHmac("sha1", config.secretAccessKey).update(policy).digest("base64")
.replace(/\+/g, "-").replace(/\//g, "_");
const job: ZipWorkerJob = {
attempt: remote.attempt,
appOrigin: origin,
folderName: task.folderName,
expiresAt: task.expiresAt,
files: task.files.map((file, i) => ({ name: file.name?.slice(0, 200), sources: remote.sources![i] })),
storage: { bucket: config.bucket, token: config.accessKeyId + ":" + mac + ":" + policy, publicUrl },
};
const response = await fetch(new URL("/v1/jobs/" + task.id, url), {
method: "PUT",
body: JSON.stringify(job),
headers: { Authorization: "Bearer " + token, "Content-Type": "application/json" },
cache: "no-store",
redirect: "error",
signal: AbortSignal.timeout(8_000),
}).catch(() => null);
if (!response?.ok) {
if (response?.status === 503) {
task.stage = "等待宿主机打包";
await saveZipTask(task);
return;
}
throw new Error("独立打包服务暂时不可达,请稍后重试,图片不会重新生成");
}
const result = zipWorkerSnapshot.parse(await response.json());
const { status, bytes, ...progress } = result;
Object.assign(task, progress, { updatedAt: Date.now() });
if (status !== "completed") {
task.status = status;
await saveZipTask(task);
return;
}
if (!bytes) throw new Error("宿主机未返回有效的压缩包信息");
const target = new URL(publicUrl);
target.searchParams.set("v", String(bytes));
target.searchParams.set("attname", task.folderName + ".zip");
const verified = await fetch(target, {
method: "HEAD", cache: "no-store", redirect: "error", signal: AbortSignal.timeout(10_000),
}).catch(() => null);
if (!verified?.ok || Number(verified.headers.get("content-length")) !== bytes) {
task.status = "running";
task.stage = "正在确认七牛下载链接";
await saveZipTask(task);
return;
}
task.cdnUrl = target.href;
task.status = "completed";
task.expiresAt = Date.now() + 24 * 60 * 60_000;
await saveZipTask(task);
}

View File

@ -0,0 +1,35 @@
import { z } from "zod";
export const zipWorkerJob = z.object({
attempt: z.number().int().nonnegative(),
folderName: z.string().min(1).max(100).refine(value => !value.includes("/") && !value.includes(String.fromCharCode(92)) && !/^\.+$/.test(value)),
expiresAt: z.number().int().positive(),
appOrigin: z.string().url(),
files: z.array(z.object({
name: z.string().max(200).optional(),
sources: z.array(z.string().url()).max(8),
})).min(1).max(200),
storage: z.object({
bucket: z.string().regex(/^[a-z0-9][a-z0-9-]{0,62}$/),
token: z.string().min(1).max(4096),
publicUrl: z.string().url(),
}),
});
export const zipWorkerSnapshot = z.object({
status: z.enum(["pending", "running", "completed", "failed"]),
stage: z.string().max(300),
error: z.string().max(500).optional(),
processedCount: z.number().int().nonnegative(),
successCount: z.number().int().nonnegative(),
failedCount: z.number().int().nonnegative(),
compressPercent: z.number().min(0).max(100),
bytes: z.number().int().positive().optional(),
});
export type ZipWorkerJob = z.infer<typeof zipWorkerJob>;
export type ZipWorkerSnapshot = z.infer<typeof zipWorkerSnapshot>;
export function zipStorageKey(id: string) {
return `canvas-downloads/${id}.zip`;
}

View File

@ -14,9 +14,13 @@ export function isPublicMediaIpv4(address:string) {
!(p[0]===192&&(p[1]===168||p[1]===0)) && !(p[0]===100&&p[1]>=64&&p[1]<=127) && !(p[0]===192&&(p[1]===168||p[1]===0)) && !(p[0]===100&&p[1]>=64&&p[1]<=127) &&
!(p[0]===198&&(p[1]===18||p[1]===19)); !(p[0]===198&&(p[1]===18||p[1]===19));
} }
export async function downloadCanvasMedia(url:string,output:string,redirects=0):Promise<void> { export async function openPublicMedia(
url: string,
options: { signal?: AbortSignal; allowUrl?: (url: URL) => boolean } = {},
redirects = 0,
): Promise<http.IncomingMessage> {
const target=new URL(url); const target=new URL(url);
if(!['http:','https:'].includes(target.protocol)||target.username||target.password||redirects>4) throw new Error('视频地址无效'); if(!['http:','https:'].includes(target.protocol)||target.username||target.password||redirects>4||options.allowUrl?.(target)===false) throw new Error('视频地址无效');
const addresses=await lookup(target.hostname,{family:4,all:true}); const addresses=await lookup(target.hostname,{family:4,all:true});
if(!addresses.length||addresses.some(v=>!isPublicMediaIpv4(v.address))) throw new Error('视频地址不可访问'); if(!addresses.length||addresses.some(v=>!isPublicMediaIpv4(v.address))) throw new Error('视频地址不可访问');
// Pin the validated address while preserving Host and TLS server name. // Pin the validated address while preserving Host and TLS server name.
@ -24,14 +28,18 @@ export async function downloadCanvasMedia(url:string,output:string,redirects=0):
const request=(target.protocol==='https:'?https:http).get({ const request=(target.protocol==='https:'?https:http).get({
protocol:target.protocol,hostname:addresses[0].address,port:target.port||undefined, protocol:target.protocol,hostname:addresses[0].address,port:target.port||undefined,
path:target.pathname+target.search,headers:{Host:target.host},servername:target.hostname, path:target.pathname+target.search,headers:{Host:target.host},servername:target.hostname,
signal:AbortSignal.timeout(90000), signal:options.signal||AbortSignal.timeout(90000),
},resolve); },resolve);
request.on('error',reject); request.on('error',reject);
}); });
if([301,302,303,307,308].includes(response.statusCode||0)&&response.headers.location) { if([301,302,303,307,308].includes(response.statusCode||0)&&response.headers.location) {
response.destroy(); response.destroy();
return downloadCanvasMedia(new URL(response.headers.location,target).href,output,redirects+1); return openPublicMedia(new URL(response.headers.location,target).href,options,redirects+1);
} }
return response;
}
export async function downloadCanvasMedia(url:string,output:string,redirects=0):Promise<void> {
const response=await openPublicMedia(url,{},redirects);
if(response.statusCode!==200||Number(response.headers['content-length']||0)>MAX_BYTES) {response.destroy();throw new Error('视频下载失败或超过 125 MiB 处理上限');} if(response.statusCode!==200||Number(response.headers['content-length']||0)>MAX_BYTES) {response.destroy();throw new Error('视频下载失败或超过 125 MiB 处理上限');}
let bytes=0; let bytes=0;
const limit=new Transform({transform(chunk,_encoding,callback){bytes+=chunk.length;callback(bytes>MAX_BYTES?new Error('视频超过 125 MiB 处理上限'):null,chunk);}}); const limit=new Transform({transform(chunk,_encoding,callback){bytes+=chunk.length;callback(bytes>MAX_BYTES?new Error('视频超过 125 MiB 处理上限'):null,chunk);}});

View File

@ -0,0 +1,20 @@
const { createRequire } = require("node:module");
const { resolve } = require("node:path");
const { build } = createRequire(require.resolve("tsx"))("esbuild");
build({
entryPoints: [resolve(__dirname, "server.ts")],
outfile: resolve(process.argv[2] || resolve(__dirname, "dist/server.cjs")),
bundle: true,
platform: "node",
target: "node20",
format: "cjs",
external: ["sharp"],
plugins: [{
name: "server-only-marker",
setup(build) {
build.onResolve({ filter: /^server-only$/ }, args => ({ path: args.path, namespace: "empty" }));
build.onLoad({ filter: /.*/, namespace: "empty" }, () => ({ contents: "" }));
},
}],
}).catch(() => { process.exitCode = 1; });

View File

@ -0,0 +1,29 @@
[Unit]
Description=由壹点详情图 ZIP 打包服务
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=canvas-zip
Group=canvas-zip
WorkingDirectory=/opt/canvas-zip/current
EnvironmentFile=/etc/canvas-zip.env
Environment=CANVAS_DOWNLOAD_ZIP_DIR=/var/lib/canvas-zip
Environment=CANVAS_ZIP_PORT=18343
ExecStart=/usr/bin/node /opt/canvas-zip/current/server.cjs
StateDirectory=canvas-zip
StateDirectoryMode=0700
Restart=on-failure
RestartSec=5
UMask=0077
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=true
PrivateTmp=true
MemoryMax=2G
CPUQuota=200%
TasksMax=128
[Install]
WantedBy=multi-user.target

View File

@ -0,0 +1,257 @@
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { timingSafeEqual } from "node:crypto";
import { open, rm, stat } from "node:fs/promises";
import { MIMEType } from "node:util";
import { claimZipTask, cleanupZipTasks, loadZipTask, saveZipTask, type ZipTask } from "../../lib/canvas/download-zip-task-store";
import { prepareCanvasZip } from "../../lib/canvas/download-zip-prepare";
import { normalizeCanvasDownloadAsset } from "../../lib/canvas/photoshop-download-normalize";
import { zipStorageKey, zipWorkerJob, type ZipWorkerJob, type ZipWorkerSnapshot } from "../../lib/canvas/remote-zip-contract";
import { openPublicMedia } from "../../lib/video-canvas/download-media";
const TOKEN = process.env.CANVAS_ZIP_WORKER_TOKEN || "";
if (TOKEN.length < 32 || !process.env.CANVAS_DOWNLOAD_ZIP_DIR) {
throw new Error("Set CANVAS_ZIP_WORKER_TOKEN and CANVAS_DOWNLOAD_ZIP_DIR");
}
const OWNER = "zip-worker";
const IMAGE_LIMIT = 125 * 1024 ** 2;
const PART_BYTES = 8 * 1024 ** 2;
const queue = new Map<string, ZipWorkerJob>();
const accepting = new Set<string>();
let running: string | undefined;
function authorized(request: IncomingMessage) {
const actual = Buffer.from(request.headers.authorization || "");
const expected = Buffer.from("Bearer " + TOKEN);
return actual.length === expected.length && timingSafeEqual(actual, expected);
}
function reply(response: ServerResponse, status: number, body: unknown) {
response.writeHead(status, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" });
response.end(JSON.stringify(body));
}
async function readJob(request: IncomingMessage) {
let bytes = 0;
const body: Buffer[] = [];
for await (const chunk of request) {
const value = Buffer.from(chunk);
bytes += value.length;
if (bytes > 1024 ** 2) throw new Error("Invalid job size");
body.push(value);
}
return zipWorkerJob.parse(JSON.parse(Buffer.concat(body).toString("utf8")));
}
function snapshot(task: ZipTask): ZipWorkerSnapshot {
const { status, stage, error, processedCount, successCount, failedCount, compressPercent } = task;
return { status, stage, error, processedCount, successCount, failedCount, compressPercent,
bytes: task.status === "completed" ? task.zipFile?.bytes : undefined };
}
async function fetchInput(job: ZipWorkerJob, index: number, signal: AbortSignal) {
const item = job.files[index];
const appHost = new URL(job.appOrigin).hostname;
const allowUrl = (url: URL) => url.hostname !== appHost && url.hostname !== "ai.yydd.ai";
for (const url of item.sources) {
if (signal.aborted) break;
try {
const response = await openPublicMedia(url, {
allowUrl,
signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]),
});
if (response.statusCode !== 200 || Number(response.headers["content-length"] || 0) > IMAGE_LIMIT) {
response.destroy();
continue;
}
const contentType = new MIMEType(response.headers["content-type"] || "application/octet-stream").essence;
const buffers: Buffer[] = [];
let bytes = 0;
for await (const packet of response) {
const chunk = Buffer.from(packet);
bytes += chunk.length;
if (bytes > IMAGE_LIMIT) throw new Error("Original exceeds the download limit");
buffers.push(chunk);
}
return await normalizeCanvasDownloadAsset({
item: { url, name: item.name }, index, bytes: Buffer.concat(buffers, bytes),
contentType, includeSequencePrefix: true,
});
} catch { /* Try the next resolved source. */ }
}
return { ok: false as const, url: "", name: item.name, error: "宿主机暂时无法读取这张原图" };
}
async function uploadArchive(task: ZipTask, job: ZipWorkerJob) {
const { bucket, token, publicUrl } = job.storage;
const { path, bytes } = task.zipFile!;
const key = zipStorageKey(task.id);
const endpoint = "https://upload.qiniup.com/buckets/" + encodeURIComponent(bucket)
+ "/objects/" + encodeURIComponent(Buffer.from(key).toString("base64").replace(/\+/g, "-").replace(/\//g, "_")) + "/uploads";
const storedUrl = new URL(publicUrl);
const hasExactSize = async () => {
try {
const response = await fetch(storedUrl, { method: "HEAD", redirect: "error", signal: AbortSignal.timeout(8_000) });
return response.ok && Number(response.headers.get("content-length")) === bytes;
} catch { return false; }
};
if (await hasExactSize()) return;
const request = async (url: string, method: string, body?: Buffer | string, contentType?: string) => {
const headers: Record<string, string> = { Authorization: "UpToken " + token };
if (contentType) headers["Content-Type"] = contentType;
return await fetch(url, {
method, headers, body: Buffer.isBuffer(body) ? Uint8Array.from(body) : body,
redirect: "error", signal: AbortSignal.timeout(method === "PUT" ? 180_000 : 30_000),
});
};
const init = await request(endpoint, "POST");
if (!init.ok) throw new Error("七牛未能开始接收压缩包(" + init.status + ")");
const { uploadId } = await init.json() as { uploadId?: string };
if (!uploadId) throw new Error("七牛未返回上传编号");
const uploadUrl = endpoint + "/" + encodeURIComponent(uploadId);
const parts: Array<{ partNumber: number; etag: string }> = [];
try {
const handle = await open(path, "r");
try {
for (let offset = 0; offset < bytes; offset += PART_BYTES) {
const data = Buffer.allocUnsafe(Math.min(PART_BYTES, bytes - offset));
let read = 0;
while (read < data.length) {
const result = await handle.read(data, read, data.length - read, offset + read);
if (!result.bytesRead) throw new Error("压缩包读取不完整");
read += result.bytesRead;
}
const partNumber = parts.length + 1;
const response = await request(uploadUrl + "/" + partNumber, "PUT", data, "application/octet-stream");
if (!response.ok) throw new Error("七牛接收分片失败(" + response.status + ")");
const { etag } = await response.json() as { etag?: string };
if (!etag) throw new Error("七牛未确认上传分片");
parts.push({ partNumber, etag });
task.stage = "正在传送至七牛 " + Math.floor((offset + data.length) / bytes * 100) + "%";
task.updatedAt = Date.now();
await saveZipTask(task);
}
} finally { await handle.close(); }
const complete = await request(uploadUrl, "POST", JSON.stringify({
parts, fname: task.folderName + ".zip", mimeType: "application/zip",
}), "application/json");
if (!complete.ok) {
if (complete.status === 614 && await hasExactSize()) return;
throw new Error("七牛未确认压缩包(" + complete.status + ")");
}
const saved = await complete.json() as { key?: string; fsize?: number };
if (saved.key !== key || saved.fsize !== bytes) throw new Error("七牛返回的压缩包信息不一致");
} catch (error) {
await request(uploadUrl, "DELETE").then(response => response.body?.cancel()).catch(() => undefined);
throw error;
}
}
async function run(id: string, job: ZipWorkerJob) {
const release = await claimZipTask(id);
if (!release) return;
let task: ZipTask | null = null;
try {
task = await loadZipTask(id, OWNER);
if (!task || task.status === "completed" || task.status === "failed") return;
task.status = "running";
task.stage = "宿主机正在准备原图";
await saveZipTask(task);
const present = task.zipFile && await stat(task.zipFile.path).catch(() => null);
if (!present || present.size !== task.zipFile!.bytes) {
await prepareCanvasZip(task, ({ index, signal }) => fetchInput(job, index, signal));
}
task.stage = "正在传送至七牛";
await saveZipTask(task);
await uploadArchive(task, job);
task.status = "completed";
task.stage = task.failedCount ? "部分原图未能读取" : "打包完成";
task.compressPercent = 100;
task.error = undefined;
task.updatedAt = Date.now();
task.expiresAt = Date.now() + 24 * 60 * 60_000;
await saveZipTask(task);
if (task.zipFile) await rm(task.zipFile.path, { force: true }).catch(() => undefined);
} catch (error) {
if (task) {
task.status = "failed";
task.stage = "打包未完成";
task.error = (error instanceof Error ? error.message : "宿主机打包失败").slice(0, 450);
task.updatedAt = Date.now();
await saveZipTask(task);
}
} finally { await release(); }
}
function drain() {
if (running) return;
const next = queue.entries().next().value;
if (!next) return;
const [id, job] = next;
queue.delete(id);
running = id;
void run(id, job).catch(() => console.error("Canvas ZIP job could not be saved")).finally(() => {
running = undefined;
drain();
});
}
export const router = createServer((request, response) => {
void (async () => {
if (!authorized(request)) { reply(response, 401, {}); return; }
if (request.method === "GET" && request.url === "/health") {
reply(response, 200, { ok: true, active: Number(Boolean(running)), waiting: queue.size });
return;
}
const id = request.url?.match(/^\/v1\/jobs\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i)?.[1];
if (!id || request.method !== "PUT") { reply(response, 404, {}); return; }
if (accepting.has(id)) { reply(response, 503, {}); return; }
accepting.add(id);
try {
const job = await readJob(request);
const publicUrl = new URL(job.storage.publicUrl);
if (publicUrl.protocol !== "https:" || !publicUrl.pathname.endsWith("/" + zipStorageKey(id))) {
reply(response, 400, {}); return;
}
let task = await loadZipTask(id, OWNER);
if (running === id || queue.has(id)) {
if (task) reply(response, 200, snapshot(task)); else reply(response, 503, {});
return;
}
if (queue.size >= 8 && (!task || task.status === "pending" || task.status === "running")) {
reply(response, 503, {}); return;
}
if (task?.status === "failed" && job.attempt > (task.attempt ?? 0)) {
task.attempt = job.attempt;
task.status = "pending";
task.stage = "等待宿主机打包";
task.error = undefined;
task.expiresAt = job.expiresAt;
task.updatedAt = Date.now();
await saveZipTask(task);
}
if (!task) {
const now = Date.now();
task = {
id, userId: OWNER, folderName: job.folderName, attempt: job.attempt,
files: job.files.map(item => ({ url: item.sources[0] || "", name: item.name })),
totalCount: job.files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0,
status: "pending", stage: "等待宿主机打包", createdAt: now, updatedAt: now, expiresAt: job.expiresAt,
};
await saveZipTask(task);
}
if (task.status === "pending" || task.status === "running") {
queue.set(id, job);
drain();
}
reply(response, 200, snapshot(task));
void cleanupZipTasks().catch(() => undefined);
} finally { accepting.delete(id); }
})().catch(() => {
if (!response.headersSent) reply(response, 400, { error: "无效的打包请求" });
});
});
if (require.main === module) {
router.listen(Number(process.env.CANVAS_ZIP_PORT || 18343), "127.0.0.1");
}

View File

@ -0,0 +1,167 @@
import { test, expect } from "@playwright/test";
import JSZip from "jszip";
import { NextRequest, NextResponse } from "next/server";
import sharp from "sharp";
import { once } from "node:events";
import { Readable } from "node:stream";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { randomUUID } from "node:crypto";
import { serverLoader } from "../helpers/canvas-download-server-loader";
test("the worker prepares 13 images and uploads the ZIP through Qiniu multipart", async () => {
const directory = await mkdtemp(join(tmpdir(), "zip-worker-test-"));
const token = "local-test-secret".repeat(3);
const id = randomUUID();
const key = "canvas-downloads/" + id + ".zip";
const rgb = { r: 25, g: 100, b: 170 };
const png = await sharp({ create: { width: 800, height: 800, channels: 3, background: rgb } }).png().toBuffer();
const jpeg = await sharp({ create: { width: 800, height: 800, channels: 3, background: rgb } }).jpeg().toBuffer();
const parts: Buffer[] = [];
let complete = 0, downloads = 0;
const loader = serverLoader({
"../../lib/video-canvas/download-media": {
openPublicMedia: async (url: string, options: { allowUrl: (url: URL) => boolean }) => {
const parsed = new URL(url);
expect(options.allowUrl(parsed)).toBe(true);
expect(options.allowUrl(new URL("https://ai.yydd.ai/api/image-proxy"))).toBe(false);
downloads++;
const original = parsed.pathname.endsWith("original.jpeg") ? jpeg : png;
return Object.assign(Readable.from([original]), {
statusCode: 200, headers: { "content-type": original === png ? "image/png" : "image/jpeg" },
});
},
},
fetch: async (uri: URL | string, init: RequestInit) => {
const url = new URL(uri);
if (init.method === "HEAD") return new Response(null, { status: 404 });
expect(url.hostname).toBe("upload.qiniup.com");
expect(new Headers(init.headers).get("Authorization")).toBe("UpToken scoped-test-token");
if (url.pathname.endsWith("/uploads")) return Response.json({ uploadId: "one-upload" });
if (init.method === "PUT") {
parts.push(Buffer.from(init.body as Uint8Array));
return Response.json({ etag: "one-part" });
}
expect(init.method).toBe("POST");
expect(JSON.parse(init.body as string).mimeType).toBe("application/zip");
complete++;
return Response.json({ key, fsize: Buffer.concat(parts).length });
},
}, { CANVAS_DOWNLOAD_ZIP_DIR: directory, CANVAS_ZIP_WORKER_TOKEN: token });
const { router } = loader("services/canvas-zip/server.ts");
router.listen(0, "127.0.0.1");
await once(router, "listening");
const endpoint = "http://127.0.0.1:" + router.address().port + "/v1/jobs/" + id;
const files = Array.from({ length: 13 }, (_, i) => ({
name: "详情图-" + i,
sources: ["https://cdn.example/" + i + (i === 12 ? "/original.jpeg" : "/original.png")],
}));
const job = {
attempt: 0, folderName: "详情图", expiresAt: Date.now() + 60_000, appOrigin: "https://ai.yydd.ai", files,
storage: { bucket: "qa-bucket", token: "scoped-test-token", publicUrl: "https://cdn.example/" + key },
};
const post = async () => fetch(endpoint, {
method: "PUT", headers: { Authorization: "Bearer " + token, "content-type": "application/json" }, body: JSON.stringify(job),
});
try {
expect((await fetch(endpoint, { method: "PUT" })).status).toBe(401);
expect((await post()).status).toBe(200);
await expect.poll(async () => {
const response = await post();
const state = await response.json();
return state.status;
}, { timeout: 10_000 }).toBe("completed");
await expect((await post()).json()).resolves.toMatchObject({ successCount: 13, failedCount: 0 });
expect(downloads).toBe(13);
expect(complete).toBe(1);
const archive = await JSZip.loadAsync(Buffer.concat(parts), { checkCRC32: true });
const images = Object.values(archive.files).filter(file => !file.dir);
expect(images).toHaveLength(13);
for (const [i, entry] of images.entries()) {
const data = await entry.async("nodebuffer");
expect(entry.name).toMatch(/\.png$/);
if (i < 12) expect(data).toEqual(png);
else {
const metadata = await sharp(data).metadata();
expect(metadata).toMatchObject({ width: 800, height: 800, space: "srgb", format: "png" });
const color = (await sharp(data).extract({ left: 20, top: 20, width: 1, height: 1 }).raw().toBuffer());
expect(Math.abs(color[0] - rgb.r)).toBeLessThan(4);
expect(Math.abs(color[1] - rgb.g)).toBeLessThan(4);
expect(Math.abs(color[2] - rgb.b)).toBeLessThan(4);
}
}
} finally {
await new Promise<void>(resolve => router.close(() => resolve()));
await rm(directory, { recursive: true, force: true });
}
});
test("the style-detail route gives the worker addresses and returns a CDN redirect even when local is requested", async () => {
const directory = await mkdtemp(join(tmpdir(), "zip-remote-route-"));
const id = randomUUID();
const callbacks: Array<() => Promise<void>> = [];
let owner = "owner";
const zipUrl = "https://cdn.example/canvas-downloads/" + id + ".zip";
const raw = { id: "storage", isEnabled: true };
const worker = "http://127.0.0.1:18343";
const config = { provider: "qiniu-kodo", region: "cn-east-1", bucket: "qa", publicUrl: "https://cdn.example",
accessKeyId: "fake-public-id", secretAccessKey: "fake-private-key" };
let workerCalls = 0;
const route = serverLoader({
"@/lib/observability/active-work": { trackRuntimeWork: async (fn: () => Promise<void>) => fn() },
"@/app/(auth)/auth": { auth: async () => ({ user: { id: owner } }) },
"next/server": { NextResponse, after: (fn: () => Promise<void>) => callbacks.push(fn) },
"@/lib/canvas/photoshop-safe-download": {
fetchCanvasDownloadAsset: () => { throw new Error("The production server must not fetch the originals"); },
},
"@/lib/storage/image-archive-db": { getArchivedImageUrlMappings: async () => ({
"https://ai.yydd.ai/archive": "https://cdn.example/archive.png",
}) },
"@/lib/db/queries": { getDefaultStorageConfig: async () => raw, getStorageConfigById: async () => raw },
"@/lib/storage/service": { toDbS3UploadConfig: () => config, buildS3PublicUrl: () => zipUrl },
"@/lib/storage/canvas-zip-delivery": {
publishCanvasZip: () => { throw new Error("The production server must not upload the ZIP"); },
},
fetch: async (uri: URL | string, init: RequestInit) => {
const url = new URL(uri);
if (url.origin === worker) {
workerCalls++;
const job = JSON.parse(init.body as string);
expect(job.files[0].sources).toEqual(["https://cdn.example/archive.png"]);
expect(job.files[1].sources).toEqual(["https://provider.example/result.png"]);
expect(job).not.toHaveProperty("cookie");
expect(job.storage).not.toHaveProperty("secretAccessKey");
const policy = JSON.parse(Buffer.from(job.storage.token.split(":")[2], "base64url").toString());
expect(policy.scope).toBe("qa:canvas-downloads/" + id + ".zip");
return Response.json({ status: "completed", stage: "打包完成", processedCount: 2,
successCount: 2, failedCount: 0, compressPercent: 100, bytes: 12345 });
}
expect(url.href).toContain(zipUrl);
expect(init.method).toBe("HEAD");
return new Response(null, { headers: { "content-length": "12345" } });
},
}, { CANVAS_DOWNLOAD_ZIP_DIR: directory, CANVAS_ZIP_WORKER_URL: worker,
CANVAS_ZIP_WORKER_TOKEN: "local-test-secret".repeat(3) })("app/api/canvas/download-zip-task/route.ts");
const endpoint = "https://ai.yydd.ai/api/canvas/download-zip-task";
try {
const started = await route.POST(new NextRequest(endpoint, {
method: "POST",
body: JSON.stringify({ taskId: id, scope: "style-detail", folderName: "详情图", files: [
{ url: "https://ai.yydd.ai/archive" }, { url: "https://provider.example/result.png" },
] }),
headers: { cookie: "auth=private-test-cookie" },
}));
expect(started.status).toBe(202);
while (callbacks.length) await callbacks.shift()!();
expect(workerCalls).toBe(1);
const download = await route.GET(new NextRequest(endpoint + "?taskId=" + id + "&download=1&local=1"));
expect(download.status).toBe(307);
expect(download.headers.get("location")).toContain(zipUrl);
owner = "another-user";
const denied = await route.GET(new NextRequest(endpoint + "?taskId=" + id + "&download=1"));
expect(denied.status).toBe(404);
} finally { await rm(directory, { recursive: true, force: true }); }
});