From e64bbfef9cdcd32474bc62e5db50a6c4dbe10afe Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Fri, 20 Nov 2026 09:49:04 +0800 Subject: [PATCH] Run detail image ZIP exports on the video host --- app/api/canvas/download-zip-task/route.ts | 79 ++---- .../agents/AgentGenerationWorkspaceDarkV2.tsx | 1 + lib/canvas/download-zip-prepare.ts | 57 ++++ lib/canvas/download-zip-task-store.ts | 9 +- lib/canvas/photoshop-download-normalize.ts | 212 +++++++++++++++ lib/canvas/photoshop-safe-download.ts | 228 ++-------------- lib/canvas/remote-zip-client.ts | 103 +++++++ lib/canvas/remote-zip-contract.ts | 35 +++ lib/video-canvas/download-media.ts | 16 +- services/canvas-zip/build.cjs | 20 ++ services/canvas-zip/canvas-zip.service | 29 ++ services/canvas-zip/server.ts | 257 ++++++++++++++++++ tests/routes/canvas-zip-worker.test.ts | 167 ++++++++++++ 13 files changed, 938 insertions(+), 275 deletions(-) create mode 100644 lib/canvas/download-zip-prepare.ts create mode 100644 lib/canvas/photoshop-download-normalize.ts create mode 100644 lib/canvas/remote-zip-client.ts create mode 100644 lib/canvas/remote-zip-contract.ts create mode 100644 services/canvas-zip/build.cjs create mode 100644 services/canvas-zip/canvas-zip.service create mode 100644 services/canvas-zip/server.ts create mode 100644 tests/routes/canvas-zip-worker.test.ts diff --git a/app/api/canvas/download-zip-task/route.ts b/app/api/canvas/download-zip-task/route.ts index 33f281d1c..16a9a6616 100644 --- a/app/api/canvas/download-zip-task/route.ts +++ b/app/api/canvas/download-zip-task/route.ts @@ -1,12 +1,11 @@ -import { writeDownloadZip, readDownloadZip } from "@/lib/canvas/download-zip-stream"; +import { readDownloadZip } from "@/lib/canvas/download-zip-stream"; import { claimZipTask, loadZipTask, saveZipTask, zipTaskPath, assertZipDiskSpace, cleanupZipTasks, assertZipTaskCapacity, type ZipTask } from "@/lib/canvas/download-zip-task-store"; import { parseDownloadByteRange } from "@/lib/canvas/download-byte-range"; -import { createReadStream } from "node:fs"; -import { mkdir, stat, writeFile, rm } from "node:fs/promises"; +import { mkdir, stat, rm } from "node:fs/promises"; import { join } from "node:path"; import { randomUUID } from "node:crypto"; import { after, NextRequest, NextResponse } from "next/server"; -import JSZip from "jszip"; +import { prepareCanvasZip } from "@/lib/canvas/download-zip-prepare"; import { auth } from "@/app/(auth)/auth"; import { fetchCanvasDownloadAsset } from "@/lib/canvas/photoshop-safe-download"; import { publishCanvasZip } from "@/lib/storage/canvas-zip-delivery"; @@ -29,16 +28,6 @@ function sanitizeFolderName(value: string): string { return cleaned && !/^\.+$/.test(cleaned) ? cleaned.slice(0, 100) : `canvas-images-${Date.now()}`; } -function buildContentDisposition(folderName: string): string { - const encodedName = encodeURIComponent(`${folderName}.zip`); - const asciiName = - folderName - .replace(/[^\x20-\x7E]+/g, "") - .replace(/"/g, "") - .trim() || "canvas-images"; - return `attachment; filename="${asciiName}.zip"; filename*=UTF-8''${encodedName}`; -} - function getRequestOrigin(request: NextRequest): string { const forwardedHost = (request.headers.get("x-forwarded-host") || "") .split(",")[0] @@ -99,50 +88,20 @@ async function processTask(id: string, userId: string, requestOrigin: string, co if (!release) return; task = await loadZipTask(id, userId); if (!task || task.status === "completed" || task.status === "failed") return; + if (task.remote) { + const { updateRemoteCanvasZip } = await import("@/lib/canvas/remote-zip-client"); + await updateRemoteCanvasZip(task, requestOrigin); + return; + } await assertZipDiskSpace(); const existingZip = task.zipFile && task.contentDisposition && await stat(task.zipFile.path).catch(() => null); const reuseZip = existingZip && existingZip.size === task.zipFile!.bytes; task.status = "running"; task.stage = reuseZip ? "正在准备加速下载" : "正在准备原图"; task.updatedAt = Date.now(); await saveZipTask(task); if (!reuseZip) { - delete task.zipFile; delete task.contentDisposition; delete task.cdnUrl; - // Stage each image on disk, not all PNG buffers in the web process heap. - // A restart resumes from these files using the current owner's session. - let cursor = 0; - let writes = Promise.resolve(); - const persist = () => { const current = task!; writes = writes.then(() => saveZipTask(current)); return writes; }; - const results = await Promise.allSettled(Array.from({ length: Math.min(2, task.files.length) }, async () => { - while (cursor < task!.files.length) { - const index = cursor++; const file = task!.files[index]; - const path = join(zipTaskPath(id), `${index}.image`); - if (file.prepared && file.filename && (await stat(path).catch(() => null))?.size) continue; - delete file.prepared; delete file.filename; - const result = await fetchCanvasDownloadAsset({ file, index, requestOrigin, cookie, - signal: AbortSignal.timeout(120_000) }).catch(() => ({ ok: false as const, error: "原图读取超时或源地址暂时不可用" })); - if (result.ok) { - await assertZipDiskSpace(result.bytes.length); - await writeFile(path, result.bytes, { mode: 0o600 }); - file.prepared = `${index}.image`; file.filename = result.filename; delete file.error; - } else file.error = result.error; - task!.processedCount = task!.files.filter(f => f.prepared || f.error).length; - task!.successCount = task!.files.filter(f => f.prepared).length; - task!.failedCount = task!.files.filter(f => f.error).length; - task!.stage = `正在准备原图 ${task!.processedCount}/${task!.totalCount}`; - task!.updatedAt = Date.now(); await persist(); - } - })); - const rejected = results.find(result => result.status === "rejected"); - if (rejected?.status === "rejected") throw rejected.reason; - if (!task.successCount) throw new Error("所选原图暂时都无法读取,请稍后重试;画布图片不会被删除"); - const zip = new JSZip(); const folder = zip.folder(task.folderName) || zip; - for (const file of task.files) if (file.prepared && file.filename) - folder.file(file.filename, createReadStream(join(zipTaskPath(id), file.prepared)) - .once("close", () => { void rm(join(zipTaskPath(id), file.prepared!), { force: true }).catch(() => undefined); })); - if (task.failedCount) folder.file("_download-failures.txt", task.files.filter(f => f.error) - .map(f => `${f.name || "图片"}:${f.error}`).join("\n")); - task.stage = "正在生成完整 ZIP 文件"; await saveZipTask(task); - task.zipFile = await writeDownloadZip(zip, task.id, percent => { task!.compressPercent = percent; }); - task.contentDisposition = buildContentDisposition(task.folderName); + await prepareCanvasZip(task, (params) => + fetchCanvasDownloadAsset({ ...params, requestOrigin, cookie }) + ); } task.stage = "正在准备下载链接"; await saveZipTask(task); const cdn = await publishCanvasZip({ id: task.id, ...task.zipFile!, disposition: task.contentDisposition!, @@ -195,13 +154,20 @@ export async function POST(request: NextRequest) { await mkdir(zipTaskPath(id), { mode: 0o700 }); }); const now = Date.now(); - task = { id, userId: session.user.id, folderName: sanitizeFolderName(String(body.folderName || "")), files, + const remote = body.scope === "style-detail" && + process.env.CANVAS_ZIP_WORKER_URL && process.env.CANVAS_ZIP_WORKER_TOKEN + ? { attempt: 0 } : undefined; + task = { id, userId: session.user.id, folderName: sanitizeFolderName(String(body.folderName || "")), files, remote, totalCount: files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0, status: "pending", stage: "等待后台准备", createdAt: now, updatedAt: now, expiresAt: now + PREPARATION_TTL_MS }; await saveZipTask(task); } else if ((task.status === "failed" || task.status === "completed" && task.cdnObject && !task.cdnUrl) && body.retry === true) { task.status = "pending"; task.error = undefined; await assertZipTaskCapacity(session.user.id); + if (task.remote) { + task.remote.attempt++; + delete task.remote.sources; + } if (!task.zipFile) { for (const file of task.files) delete file.error; task.processedCount = task.successCount = task.files.filter(file => file.prepared).length; @@ -211,7 +177,10 @@ export async function POST(request: NextRequest) { await saveZipTask(task); } if (task.status === "pending" || task.status === "running") schedule(task, request); - after(() => cleanupZipTasks().catch(() => { console.warn("[CanvasDownloadZipTask] Expired file cleanup deferred"); })); + after(() => cleanupZipTasks(async object => { + const { removeCanvasZipObject } = await import("@/lib/storage/canvas-zip-delivery"); + await removeCanvasZipObject(object); + }).catch(() => { console.warn("[CanvasDownloadZipTask] Expired file cleanup deferred"); })); return NextResponse.json({ success: true, task: serializeTask(task) }, { status: 202, headers: { "Cache-Control": "no-store" } }); } catch (error) { console.warn("[CanvasDownloadZipTask] Create rejected", { message: (error as Error).message }); @@ -226,6 +195,8 @@ export async function GET(request: NextRequest) { const task = await loadZipTask(id, session.user.id); if (!task) return NextResponse.json({ error: "打包任务不存在或已过期,请重新打包" }, { status: 404 }); if (request.nextUrl.searchParams.get("download") === "1") { + if (task.remote && task.status === "completed" && task.cdnUrl) + return NextResponse.redirect(task.cdnUrl, 307); if (task.status !== "completed" || !task.zipFile) return NextResponse.json({ error: "ZIP 尚未准备好" }, { status: 409 }); if (task.cdnUrl && request.nextUrl.searchParams.get("local") !== "1") return NextResponse.redirect(task.cdnUrl, 307); diff --git a/components/agents/AgentGenerationWorkspaceDarkV2.tsx b/components/agents/AgentGenerationWorkspaceDarkV2.tsx index 491c8216a..543139ced 100644 --- a/components/agents/AgentGenerationWorkspaceDarkV2.tsx +++ b/components/agents/AgentGenerationWorkspaceDarkV2.tsx @@ -5892,6 +5892,7 @@ export function AgentGenerationWorkspaceDarkV2({ headers: { "Content-Type": "application/json" }, cache: "no-store", body: JSON.stringify({ + scope: params.scope === "detail" ? "style-detail" : undefined, folderName: params.zipName, files: normalizedUrls.map((imageUrl) => ({ url: imageUrl, diff --git a/lib/canvas/download-zip-prepare.ts b/lib/canvas/download-zip-prepare.ts new file mode 100644 index 000000000..ef5736231 --- /dev/null +++ b/lib/canvas/download-zip-prepare.ts @@ -0,0 +1,57 @@ +import { createReadStream } from "node:fs"; +import { stat, writeFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import JSZip from "jszip"; +import { writeDownloadZip } from "./download-zip-stream"; +import { assertZipDiskSpace, saveZipTask, zipTaskPath, type ZipTask } from "./download-zip-task-store"; +import type { CanvasDownloadAssetResult } from "./photoshop-safe-download"; + +export function buildZipContentDisposition(folderName: string) { + const name = `${folderName}.zip`; + const asciiName = folderName.replace(/[^\x20-\x7E]+/g, "").replace(/"/g, "").trim() || "canvas-images"; + return `attachment; filename="${asciiName}.zip"; filename*=UTF-8''${encodeURIComponent(name)}`; +} + +export async function prepareCanvasZip( + task: ZipTask, + fetchAsset: (params: { file: ZipTask["files"][number]; index: number; signal: AbortSignal }) => Promise, +) { + delete task.zipFile; delete task.contentDisposition; delete task.cdnUrl; + // Stage each image on disk, not all PNG buffers in the web process heap. + // A restart resumes from these files using the current owner's session. + let cursor = 0; + let writes = Promise.resolve(); + const persist = () => { const current = task; writes = writes.then(() => saveZipTask(current)); return writes; }; + const results = await Promise.allSettled(Array.from({ length: Math.min(2, task.files.length) }, async () => { + while (cursor < task.files.length) { + const index = cursor++; const file = task.files[index]; + const path = join(zipTaskPath(task.id), `${index}.image`); + if (file.prepared && file.filename && (await stat(path).catch(() => null))?.size) continue; + delete file.prepared; delete file.filename; + const result = await fetchAsset({ file, index, + signal: AbortSignal.timeout(120_000) }).catch(() => ({ ok: false as const, error: "原图读取超时或源地址暂时不可用" })); + if (result.ok) { + await assertZipDiskSpace(result.bytes.length); + await writeFile(path, result.bytes, { mode: 0o600 }); + file.prepared = `${index}.image`; file.filename = result.filename; delete file.error; + } else file.error = result.error; + task.processedCount = task.files.filter(f => f.prepared || f.error).length; + task.successCount = task.files.filter(f => f.prepared).length; + task.failedCount = task.files.filter(f => f.error).length; + task.stage = `正在准备原图 ${task.processedCount}/${task.totalCount}`; + task.updatedAt = Date.now(); await persist(); + } + })); + const rejected = results.find(result => result.status === "rejected"); + if (rejected?.status === "rejected") throw rejected.reason; + if (!task.successCount) throw new Error("所选原图暂时都无法读取,请稍后重试;画布图片不会被删除"); + const zip = new JSZip(); const folder = zip.folder(task.folderName) || zip; + for (const file of task.files) if (file.prepared && file.filename) + folder.file(file.filename, createReadStream(join(zipTaskPath(task.id), file.prepared)) + .once("close", () => { void rm(join(zipTaskPath(task.id), file.prepared!), { force: true }).catch(() => undefined); })); + if (task.failedCount) folder.file("_download-failures.txt", task.files.filter(f => f.error) + .map(f => `${f.name || "图片"}:${f.error}`).join("\n")); + task.stage = "正在生成完整 ZIP 文件"; await saveZipTask(task); + task.zipFile = await writeDownloadZip(zip, task.id, percent => { task.compressPercent = percent; }); + task.contentDisposition = buildZipContentDisposition(task.folderName); +} diff --git a/lib/canvas/download-zip-task-store.ts b/lib/canvas/download-zip-task-store.ts index d7f95a338..4cdb6747d 100644 --- a/lib/canvas/download-zip-task-store.ts +++ b/lib/canvas/download-zip-task-store.ts @@ -10,6 +10,8 @@ export type ZipTask = { totalCount: number; processedCount: number; successCount: number; failedCount: number; compressPercent: number; status: "pending" | "running" | "completed" | "failed"; stage: string; error?: string; zipFile?: { path: string; bytes: number }; + remote?: { attempt: number; storageConfigId?: string; sources?: string[][] }; + attempt?: number; contentDisposition?: string; cdnUrl?: string; cdnObject?: { storageConfigId: string; key: string }; createdAt: number; updatedAt: number; expiresAt: number; }; @@ -62,7 +64,7 @@ export async function assertZipDiskSpace(bytes = 0) { } let cleanupAt = 0; -export async function cleanupZipTasks() { +export async function cleanupZipTasks(removeObject?: (object: NonNullable) => Promise) { if (Date.now() - cleanupAt < 60_000) return; cleanupAt = Date.now(); for (const id of await readdir(downloadZipDirectory).catch(() => [] as string[])) { @@ -74,10 +76,7 @@ export async function cleanupZipTasks() { const release = await claimZipTask(id); if (!release) continue; try { - if (task.cdnObject) { - const { removeCanvasZipObject } = await import("@/lib/storage/canvas-zip-delivery"); - await removeCanvasZipObject(task.cdnObject).catch(() => undefined); - } + if (task.cdnObject && removeObject) await removeObject(task.cdnObject).catch(() => undefined); await rm(join(downloadZipDirectory, `${id}.zip`), { force: true }); await rm(join(downloadZipDirectory, `${id}.zip.partial`), { force: true }); } finally { await release(); } diff --git a/lib/canvas/photoshop-download-normalize.ts b/lib/canvas/photoshop-download-normalize.ts new file mode 100644 index 000000000..4e5aa08c4 --- /dev/null +++ b/lib/canvas/photoshop-download-normalize.ts @@ -0,0 +1,212 @@ +export type DownloadFileItem = { + url: string; + name?: string; +}; + +export type CanvasDownloadAssetResult = + | { + ok: true; + filename: string; + bytes: Buffer; + contentType: string; + } + | { + ok: false; + url: string; + name?: string; + error: string; + }; + +export function sanitizeCanvasDownloadFileName(value: string, fallback: string): string { + const cleaned = String(value || "") + .replace(/\.[a-zA-Z0-9]{2,6}$/g, "") + .replace(/[\\/:*?"<>|]+/g, "-") + .replace(/\s+/g, "-") + .replace(/-+/g, "-") + .replace(/^-+|-+$/g, "") + .trim() + .slice(0, 120); + return cleaned || fallback; +} + +export function buildAttachmentContentDisposition(filename: string): string { + const safeAsciiName = + filename + .replace(/[^\x20-\x7E]+/g, "") + .replace(/"/g, "") + .trim() || `canvas-image-${Date.now()}.png`; + return `attachment; filename="${safeAsciiName}"; filename*=UTF-8''${encodeURIComponent(filename)}`; +} + +function getContentTypeMime(contentType: string): string { + return String(contentType || "") + .split(";")[0] + ?.trim() + .toLowerCase(); +} + +function hasAsciiAt(buffer: Buffer, offset: number, value: string): boolean { + return buffer.subarray(offset, offset + value.length).toString("latin1") === value; +} + +function looksLikeSvg(buffer: Buffer, contentType: string): boolean { + if (getContentTypeMime(contentType) === "image/svg+xml") return true; + const text = buffer.subarray(0, Math.min(buffer.length, 1024)).toString("utf8").trimStart(); + return /^(?:<\?xml[^>]*>\s*)?]/i.test(text); +} + +function looksLikeErrorDocument(buffer: Buffer, contentType: string): boolean { + const mime = getContentTypeMime(contentType); + if ( + mime.startsWith("text/") || + mime === "application/json" || + mime === "application/xml" || + mime === "application/xhtml+xml" + ) { + return true; + } + const text = buffer.subarray(0, Math.min(buffer.length, 512)).toString("utf8").trimStart(); + return /^(?:]|<\?xml|]|[{[])/i.test(text); +} + +function detectAsset(bytes: Buffer, contentType: string): { kind: "image" | "video" | "audio"; mimeType: string; ext: string } | null { + if (bytes.length < 4) return null; + + if (bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) { + return { kind: "image", mimeType: "image/png", ext: "png" }; + } + if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { + return { kind: "image", mimeType: "image/jpeg", ext: "jpg" }; + } + if (hasAsciiAt(bytes, 0, "RIFF") && hasAsciiAt(bytes, 8, "WEBP")) { + return { kind: "image", mimeType: "image/webp", ext: "webp" }; + } + if (hasAsciiAt(bytes, 0, "GIF87a") || hasAsciiAt(bytes, 0, "GIF89a")) { + return { kind: "image", mimeType: "image/gif", ext: "gif" }; + } + if (hasAsciiAt(bytes, 0, "BM")) { + return { kind: "image", mimeType: "image/bmp", ext: "bmp" }; + } + if ( + bytes.subarray(0, 4).equals(Buffer.from([0x49, 0x49, 0x2a, 0x00])) || + bytes.subarray(0, 4).equals(Buffer.from([0x4d, 0x4d, 0x00, 0x2a])) + ) { + return { kind: "image", mimeType: "image/tiff", ext: "tiff" }; + } + if (looksLikeSvg(bytes, contentType)) { + return { kind: "image", mimeType: "image/svg+xml", ext: "svg" }; + } + if (hasAsciiAt(bytes, 0, "RIFF") && hasAsciiAt(bytes, 8, "WAVE")) return {kind:"audio",mimeType:"audio/wav",ext:"wav"}; + if (hasAsciiAt(bytes, 0, "ID3") || (bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0 && (bytes[1] & 0x06) !== 0)) return {kind:"audio",mimeType:"audio/mpeg",ext:"mp3"}; + if (hasAsciiAt(bytes, 0, "OggS")) return {kind:"audio",mimeType:"audio/ogg",ext:"ogg"}; + if (hasAsciiAt(bytes, 0, "fLaC")) return {kind:"audio",mimeType:"audio/flac",ext:"flac"}; + if (getContentTypeMime(contentType).startsWith("audio/")) return {kind:"audio",mimeType:getContentTypeMime(contentType),ext:getContentTypeMime(contentType).includes("mp4") ? "m4a" : getContentTypeMime(contentType).split("/")[1]}; + if (hasAsciiAt(bytes, 4, "ftyp")) { + const brand = bytes.subarray(8, Math.min(bytes.length, 32)).toString("latin1").toLowerCase(); + if (brand.includes("m4a")) return {kind:"audio",mimeType:"audio/mp4",ext:"m4a"}; + if (brand.includes("avif")) return { kind: "image", mimeType: "image/avif", ext: "avif" }; + if (brand.includes("heic") || brand.includes("heif") || brand.includes("mif1")) { + return { kind: "image", mimeType: "image/heic", ext: "heic" }; + } + return { kind: "video", mimeType: "video/mp4", ext: "mp4" }; + } + if (bytes.subarray(0, 4).equals(Buffer.from([0x1a, 0x45, 0xdf, 0xa3]))) { + return { kind: "video", mimeType: "video/webm", ext: "webm" }; + } + + const mime = getContentTypeMime(contentType); + if (mime.startsWith("image/")) { + const ext = mime.includes("jpeg") ? "jpg" : mime.split("/")[1] || "png"; + return { kind: "image", mimeType: mime, ext }; + } + if (mime.startsWith("video/")) { + const ext = mime.includes("quicktime") ? "mov" : mime.split("/")[1] || "mp4"; + return { kind: "video", mimeType: mime, ext }; + } + return null; +} + +export async function convertImageToPhotoshopPng(bytes: Buffer): Promise { + const sharp = await import("sharp"); + const image = sharp.default(bytes, { + animated: false, + failOn: "none", + limitInputPixels: false, + }); + const metadata = await image.metadata(); + if (!metadata.width || !metadata.height) { + throw new Error("无法解析图片尺寸"); + } + // RGB/RGBA, 8-bit, single-frame PNG is already compatible; preserve its exact bytes. + const rgb = (metadata.space === "srgb" && metadata.depth === "uchar") + || (metadata.space === "rgb16" && metadata.depth === "ushort"); + if (metadata.format === "png" && rgb + && !metadata.isPalette && (metadata.pages || 1) === 1 && (!metadata.orientation || metadata.orientation === 1)) return bytes; + const output = image.rotate().toColorspace("srgb"); + if (metadata.icc) output.withIccProfile("srgb"); + return output.png({ compressionLevel: 3, adaptiveFiltering: true }).toBuffer(); +} + +function buildCanvasAssetFilename(params: { + item: DownloadFileItem; + index: number; + ext: string; + includeSequencePrefix: boolean; +}) { + const baseName = sanitizeCanvasDownloadFileName( + params.item.name || `canvas-image-${params.index + 1}`, + `canvas-image-${params.index + 1}` + ); + if (!params.includeSequencePrefix) { + return `${baseName}.${params.ext}`; + } + const sequencePrefix = String(params.index + 1).padStart(3, "0"); + return `${sequencePrefix}-${baseName}.${params.ext}`; +} + +export async function normalizeCanvasDownloadAsset(params: { + item: DownloadFileItem; + index: number; + bytes: Buffer; + contentType: string; + includeSequencePrefix: boolean; +}, convert = convertImageToPhotoshopPng): Promise { + if (params.bytes.length === 0) { + throw new Error("empty response"); + } + if (looksLikeErrorDocument(params.bytes, params.contentType) && !looksLikeSvg(params.bytes, params.contentType)) { + throw new Error("源地址返回的不是图片文件"); + } + + const detected = detectAsset(params.bytes, params.contentType); + if (!detected) { + throw new Error("无法识别图片格式"); + } + + if (detected.kind !== "image") { + return { + ok: true, + filename: buildCanvasAssetFilename({ + item: params.item, + index: params.index, + ext: detected.ext, + includeSequencePrefix: params.includeSequencePrefix, + }), + bytes: params.bytes, + contentType: detected.mimeType, + }; + } + + const pngBytes = await convert(params.bytes); + return { + ok: true, + filename: buildCanvasAssetFilename({ + item: params.item, + index: params.index, + ext: "png", + includeSequencePrefix: params.includeSequencePrefix, + }), + bytes: pngBytes, + contentType: "image/png", + }; +} diff --git a/lib/canvas/photoshop-safe-download.ts b/lib/canvas/photoshop-safe-download.ts index 33b7e2224..0025efd00 100644 --- a/lib/canvas/photoshop-safe-download.ts +++ b/lib/canvas/photoshop-safe-download.ts @@ -4,24 +4,15 @@ import { fetchImageBytes, onceImageBytes } from "@/lib/storage/image-byte-cache" import { extractImageProxySourceUrl, inferOriginalImageUrl } from "@/lib/image-url"; import { getDownloadFailure } from "./download-error"; -type DownloadFileItem = { - url: string; - name?: string; -}; +import { + convertImageToPhotoshopPng, + normalizeCanvasDownloadAsset as normalizeAsset, + type CanvasDownloadAssetResult, + type DownloadFileItem, +} from "./photoshop-download-normalize"; -export type CanvasDownloadAssetResult = - | { - ok: true; - filename: string; - bytes: Buffer; - contentType: string; - } - | { - ok: false; - url: string; - name?: string; - error: string; - }; +export { buildAttachmentContentDisposition, convertImageToPhotoshopPng, sanitizeCanvasDownloadFileName } from "./photoshop-download-normalize"; +export type { CanvasDownloadAssetResult } from "./photoshop-download-normalize"; const DEFAULT_FETCH_TIMEOUT_MS = 30_000; const DEFAULT_FETCH_RETRY_COUNT = 3; @@ -37,27 +28,6 @@ function normalizeDownloadError(error: unknown): string { return "download failed"; } -export function sanitizeCanvasDownloadFileName(value: string, fallback: string): string { - const cleaned = String(value || "") - .replace(/\.[a-zA-Z0-9]{2,6}$/g, "") - .replace(/[\\/:*?"<>|]+/g, "-") - .replace(/\s+/g, "-") - .replace(/-+/g, "-") - .replace(/^-+|-+$/g, "") - .trim() - .slice(0, 120); - return cleaned || fallback; -} - -export function buildAttachmentContentDisposition(filename: string): string { - const safeAsciiName = - filename - .replace(/[^\x20-\x7E]+/g, "") - .replace(/"/g, "") - .trim() || `canvas-image-${Date.now()}.png`; - return `attachment; filename="${safeAsciiName}"; filename*=UTF-8''${encodeURIComponent(filename)}`; -} - function buildRemoteFetchHeaders(fileUrl: string, requestOrigin: string, cookie: string): HeadersInit { const headers: Record = { Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,video/*,*/*;q=0.8", @@ -91,7 +61,7 @@ function resolveHttpUrl(value: string, requestOrigin: string): string { } } -function buildCanvasDownloadSourceCandidates(fileUrl: string, requestOrigin: string) { +export function buildCanvasDownloadSourceCandidates(fileUrl: string, requestOrigin: string) { const candidates: string[] = []; const seen = new Set(); @@ -117,180 +87,14 @@ function buildCanvasDownloadSourceCandidates(fileUrl: string, requestOrigin: str return candidates; } -function getContentTypeMime(contentType: string): string { - return String(contentType || "") - .split(";")[0] - ?.trim() - .toLowerCase(); -} - -function hasAsciiAt(buffer: Buffer, offset: number, value: string): boolean { - return buffer.subarray(offset, offset + value.length).toString("latin1") === value; -} - -function looksLikeSvg(buffer: Buffer, contentType: string): boolean { - if (getContentTypeMime(contentType) === "image/svg+xml") return true; - const text = buffer.subarray(0, Math.min(buffer.length, 1024)).toString("utf8").trimStart(); - return /^(?:<\?xml[^>]*>\s*)?]/i.test(text); -} - -function looksLikeErrorDocument(buffer: Buffer, contentType: string): boolean { - const mime = getContentTypeMime(contentType); - if ( - mime.startsWith("text/") || - mime === "application/json" || - mime === "application/xml" || - mime === "application/xhtml+xml" - ) { - return true; - } - const text = buffer.subarray(0, Math.min(buffer.length, 512)).toString("utf8").trimStart(); - return /^(?:]|<\?xml|]|[{[])/i.test(text); -} - -function detectAsset(bytes: Buffer, contentType: string): { kind: "image" | "video" | "audio"; mimeType: string; ext: string } | null { - if (bytes.length < 4) return null; - - if (bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) { - return { kind: "image", mimeType: "image/png", ext: "png" }; - } - if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { - return { kind: "image", mimeType: "image/jpeg", ext: "jpg" }; - } - if (hasAsciiAt(bytes, 0, "RIFF") && hasAsciiAt(bytes, 8, "WEBP")) { - return { kind: "image", mimeType: "image/webp", ext: "webp" }; - } - if (hasAsciiAt(bytes, 0, "GIF87a") || hasAsciiAt(bytes, 0, "GIF89a")) { - return { kind: "image", mimeType: "image/gif", ext: "gif" }; - } - if (hasAsciiAt(bytes, 0, "BM")) { - return { kind: "image", mimeType: "image/bmp", ext: "bmp" }; - } - if ( - bytes.subarray(0, 4).equals(Buffer.from([0x49, 0x49, 0x2a, 0x00])) || - bytes.subarray(0, 4).equals(Buffer.from([0x4d, 0x4d, 0x00, 0x2a])) - ) { - return { kind: "image", mimeType: "image/tiff", ext: "tiff" }; - } - if (looksLikeSvg(bytes, contentType)) { - return { kind: "image", mimeType: "image/svg+xml", ext: "svg" }; - } - if (hasAsciiAt(bytes, 0, "RIFF") && hasAsciiAt(bytes, 8, "WAVE")) return {kind:"audio",mimeType:"audio/wav",ext:"wav"}; - if (hasAsciiAt(bytes, 0, "ID3") || (bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0 && (bytes[1] & 0x06) !== 0)) return {kind:"audio",mimeType:"audio/mpeg",ext:"mp3"}; - if (hasAsciiAt(bytes, 0, "OggS")) return {kind:"audio",mimeType:"audio/ogg",ext:"ogg"}; - if (hasAsciiAt(bytes, 0, "fLaC")) return {kind:"audio",mimeType:"audio/flac",ext:"flac"}; - if (getContentTypeMime(contentType).startsWith("audio/")) return {kind:"audio",mimeType:getContentTypeMime(contentType),ext:getContentTypeMime(contentType).includes("mp4") ? "m4a" : getContentTypeMime(contentType).split("/")[1]}; - if (hasAsciiAt(bytes, 4, "ftyp")) { - const brand = bytes.subarray(8, Math.min(bytes.length, 32)).toString("latin1").toLowerCase(); - if (brand.includes("m4a")) return {kind:"audio",mimeType:"audio/mp4",ext:"m4a"}; - if (brand.includes("avif")) return { kind: "image", mimeType: "image/avif", ext: "avif" }; - if (brand.includes("heic") || brand.includes("heif") || brand.includes("mif1")) { - return { kind: "image", mimeType: "image/heic", ext: "heic" }; - } - return { kind: "video", mimeType: "video/mp4", ext: "mp4" }; - } - if (bytes.subarray(0, 4).equals(Buffer.from([0x1a, 0x45, 0xdf, 0xa3]))) { - return { kind: "video", mimeType: "video/webm", ext: "webm" }; - } - - const mime = getContentTypeMime(contentType); - if (mime.startsWith("image/")) { - const ext = mime.includes("jpeg") ? "jpg" : mime.split("/")[1] || "png"; - return { kind: "image", mimeType: mime, ext }; - } - if (mime.startsWith("video/")) { - const ext = mime.includes("quicktime") ? "mov" : mime.split("/")[1] || "mp4"; - return { kind: "video", mimeType: mime, ext }; - } - return null; -} - -export async function convertImageToPhotoshopPng(bytes: Buffer): Promise { - const sharp = await import("sharp"); - const image = sharp.default(bytes, { - animated: false, - failOn: "none", - limitInputPixels: false, +export async function normalizeCanvasDownloadAsset(params: Parameters[0]) { + return normalizeAsset(params, async bytes => { + const digest = createHash("sha256").update(bytes).digest("hex"); + const { buffer } = await onceImageBytes(`photoshop:${digest}`, async () => ({ + buffer: await convertImageToPhotoshopPng(bytes), contentType: "image/png", + })); + return buffer; }); - const metadata = await image.metadata(); - if (!metadata.width || !metadata.height) { - throw new Error("无法解析图片尺寸"); - } - // RGB/RGBA, 8-bit, single-frame PNG is already compatible; preserve its exact bytes. - const rgb = (metadata.space === "srgb" && metadata.depth === "uchar") - || (metadata.space === "rgb16" && metadata.depth === "ushort"); - if (metadata.format === "png" && rgb - && !metadata.isPalette && (metadata.pages || 1) === 1 && (!metadata.orientation || metadata.orientation === 1)) return bytes; - const output = image.rotate().toColorspace("srgb"); - if (metadata.icc) output.withIccProfile("srgb"); - return output.png({ compressionLevel: 3, adaptiveFiltering: true }).toBuffer(); -} - -function buildCanvasAssetFilename(params: { - item: DownloadFileItem; - index: number; - ext: string; - includeSequencePrefix: boolean; -}) { - const baseName = sanitizeCanvasDownloadFileName( - params.item.name || `canvas-image-${params.index + 1}`, - `canvas-image-${params.index + 1}` - ); - if (!params.includeSequencePrefix) { - return `${baseName}.${params.ext}`; - } - const sequencePrefix = String(params.index + 1).padStart(3, "0"); - return `${sequencePrefix}-${baseName}.${params.ext}`; -} - -export async function normalizeCanvasDownloadAsset(params: { - item: DownloadFileItem; - index: number; - bytes: Buffer; - contentType: string; - includeSequencePrefix: boolean; -}): Promise { - if (params.bytes.length === 0) { - throw new Error("empty response"); - } - if (looksLikeErrorDocument(params.bytes, params.contentType) && !looksLikeSvg(params.bytes, params.contentType)) { - throw new Error("源地址返回的不是图片文件"); - } - - const detected = detectAsset(params.bytes, params.contentType); - if (!detected) { - throw new Error("无法识别图片格式"); - } - - if (detected.kind !== "image") { - return { - ok: true, - filename: buildCanvasAssetFilename({ - item: params.item, - index: params.index, - ext: detected.ext, - includeSequencePrefix: params.includeSequencePrefix, - }), - bytes: params.bytes, - contentType: detected.mimeType, - }; - } - - const digest = createHash("sha256").update(params.bytes).digest("hex"); - const { buffer: pngBytes } = await onceImageBytes(`photoshop:${digest}`, async () => ({ - buffer: await convertImageToPhotoshopPng(params.bytes), contentType: "image/png", - })); - return { - ok: true, - filename: buildCanvasAssetFilename({ - item: params.item, - index: params.index, - ext: "png", - includeSequencePrefix: params.includeSequencePrefix, - }), - bytes: pngBytes, - contentType: "image/png", - }; } export async function fetchCanvasDownloadAsset(params: { diff --git a/lib/canvas/remote-zip-client.ts b/lib/canvas/remote-zip-client.ts new file mode 100644 index 000000000..bacacbc05 --- /dev/null +++ b/lib/canvas/remote-zip-client.ts @@ -0,0 +1,103 @@ +import "server-only"; +import { createHmac } from "node:crypto"; +import { getDefaultStorageConfig, getStorageConfigById } from "@/lib/db/queries"; +import { getArchivedImageUrlMappings } from "@/lib/storage/image-archive-db"; +import { buildS3PublicUrl, toDbS3UploadConfig } from "@/lib/storage/service"; +import { buildCanvasDownloadSourceCandidates } from "./photoshop-safe-download"; +import { saveZipTask, type ZipTask } from "./download-zip-task-store"; +import { zipStorageKey, zipWorkerSnapshot, type ZipWorkerJob } from "./remote-zip-contract"; + +async function resolveSources(task: ZipTask, origin: string) { + const candidates = task.files.map(file => buildCanvasDownloadSourceCandidates(file.url, origin)); + const aliases = await getArchivedImageUrlMappings(candidates.flat()); + const app = new URL(origin).hostname; + const isDirect = (value: string) => { + try { + const parsed = new URL(value); + return ["http:", "https:"].includes(parsed.protocol) + && parsed.hostname !== app && parsed.hostname !== "ai.yydd.ai" && !parsed.username && !parsed.password; + } catch { return false; } + }; + return candidates.map(values => [...new Set([ + ...values.flatMap(value => aliases[value] ? [aliases[value]] : []), ...values, + ])].filter(isDirect).slice(0, 8)); +} + +export async function updateRemoteCanvasZip(task: ZipTask, origin: string) { + const remote = task.remote!; + const url = process.env.CANVAS_ZIP_WORKER_URL; + const token = process.env.CANVAS_ZIP_WORKER_TOKEN; + if (!url || !token) throw new Error("独立打包服务暂时不可用,请稍后重试"); + const raw = remote.storageConfigId + ? await getStorageConfigById({ id: remote.storageConfigId }) + : await getDefaultStorageConfig(); + const config = toDbS3UploadConfig(raw); + if (!raw?.isEnabled || config?.provider !== "qiniu-kodo" || config.region !== "cn-east-1" || !config.publicUrl?.startsWith("https://")) { + throw new Error("七牛打包存储暂时不可用,请稍后重试"); + } + const key = zipStorageKey(task.id); + if (!remote.sources) { + remote.storageConfigId = raw.id; + remote.sources = await resolveSources(task, origin); + task.cdnObject = { storageConfigId: raw.id, key }; + await saveZipTask(task); + } + const publicUrl = buildS3PublicUrl(config, key); + const policy = Buffer.from(JSON.stringify({ + scope: config.bucket + ":" + key, + deadline: Math.floor(task.expiresAt / 1000), + insertOnly: 1, + returnBody: '{"key":"$(key)","fsize":$(fsize)}', + })).toString("base64").replace(/\+/g, "-").replace(/\//g, "_"); + const mac = createHmac("sha1", config.secretAccessKey).update(policy).digest("base64") + .replace(/\+/g, "-").replace(/\//g, "_"); + const job: ZipWorkerJob = { + attempt: remote.attempt, + appOrigin: origin, + folderName: task.folderName, + expiresAt: task.expiresAt, + files: task.files.map((file, i) => ({ name: file.name?.slice(0, 200), sources: remote.sources![i] })), + storage: { bucket: config.bucket, token: config.accessKeyId + ":" + mac + ":" + policy, publicUrl }, + }; + const response = await fetch(new URL("/v1/jobs/" + task.id, url), { + method: "PUT", + body: JSON.stringify(job), + headers: { Authorization: "Bearer " + token, "Content-Type": "application/json" }, + cache: "no-store", + redirect: "error", + signal: AbortSignal.timeout(8_000), + }).catch(() => null); + if (!response?.ok) { + if (response?.status === 503) { + task.stage = "等待宿主机打包"; + await saveZipTask(task); + return; + } + throw new Error("独立打包服务暂时不可达,请稍后重试,图片不会重新生成"); + } + const result = zipWorkerSnapshot.parse(await response.json()); + const { status, bytes, ...progress } = result; + Object.assign(task, progress, { updatedAt: Date.now() }); + if (status !== "completed") { + task.status = status; + await saveZipTask(task); + return; + } + if (!bytes) throw new Error("宿主机未返回有效的压缩包信息"); + const target = new URL(publicUrl); + target.searchParams.set("v", String(bytes)); + target.searchParams.set("attname", task.folderName + ".zip"); + const verified = await fetch(target, { + method: "HEAD", cache: "no-store", redirect: "error", signal: AbortSignal.timeout(10_000), + }).catch(() => null); + if (!verified?.ok || Number(verified.headers.get("content-length")) !== bytes) { + task.status = "running"; + task.stage = "正在确认七牛下载链接"; + await saveZipTask(task); + return; + } + task.cdnUrl = target.href; + task.status = "completed"; + task.expiresAt = Date.now() + 24 * 60 * 60_000; + await saveZipTask(task); +} diff --git a/lib/canvas/remote-zip-contract.ts b/lib/canvas/remote-zip-contract.ts new file mode 100644 index 000000000..de8e4c6f7 --- /dev/null +++ b/lib/canvas/remote-zip-contract.ts @@ -0,0 +1,35 @@ +import { z } from "zod"; + +export const zipWorkerJob = z.object({ + attempt: z.number().int().nonnegative(), + folderName: z.string().min(1).max(100).refine(value => !value.includes("/") && !value.includes(String.fromCharCode(92)) && !/^\.+$/.test(value)), + expiresAt: z.number().int().positive(), + appOrigin: z.string().url(), + files: z.array(z.object({ + name: z.string().max(200).optional(), + sources: z.array(z.string().url()).max(8), + })).min(1).max(200), + storage: z.object({ + bucket: z.string().regex(/^[a-z0-9][a-z0-9-]{0,62}$/), + token: z.string().min(1).max(4096), + publicUrl: z.string().url(), + }), +}); + +export const zipWorkerSnapshot = z.object({ + status: z.enum(["pending", "running", "completed", "failed"]), + stage: z.string().max(300), + error: z.string().max(500).optional(), + processedCount: z.number().int().nonnegative(), + successCount: z.number().int().nonnegative(), + failedCount: z.number().int().nonnegative(), + compressPercent: z.number().min(0).max(100), + bytes: z.number().int().positive().optional(), +}); + +export type ZipWorkerJob = z.infer; +export type ZipWorkerSnapshot = z.infer; + +export function zipStorageKey(id: string) { + return `canvas-downloads/${id}.zip`; +} diff --git a/lib/video-canvas/download-media.ts b/lib/video-canvas/download-media.ts index 39634230d..16bbc250f 100644 --- a/lib/video-canvas/download-media.ts +++ b/lib/video-canvas/download-media.ts @@ -14,9 +14,13 @@ export function isPublicMediaIpv4(address:string) { !(p[0]===192&&(p[1]===168||p[1]===0)) && !(p[0]===100&&p[1]>=64&&p[1]<=127) && !(p[0]===198&&(p[1]===18||p[1]===19)); } -export async function downloadCanvasMedia(url:string,output:string,redirects=0):Promise { +export async function openPublicMedia( + url: string, + options: { signal?: AbortSignal; allowUrl?: (url: URL) => boolean } = {}, + redirects = 0, +): Promise { const target=new URL(url); - if(!['http:','https:'].includes(target.protocol)||target.username||target.password||redirects>4) throw new Error('视频地址无效'); + if(!['http:','https:'].includes(target.protocol)||target.username||target.password||redirects>4||options.allowUrl?.(target)===false) throw new Error('视频地址无效'); const addresses=await lookup(target.hostname,{family:4,all:true}); if(!addresses.length||addresses.some(v=>!isPublicMediaIpv4(v.address))) throw new Error('视频地址不可访问'); // Pin the validated address while preserving Host and TLS server name. @@ -24,14 +28,18 @@ export async function downloadCanvasMedia(url:string,output:string,redirects=0): const request=(target.protocol==='https:'?https:http).get({ protocol:target.protocol,hostname:addresses[0].address,port:target.port||undefined, path:target.pathname+target.search,headers:{Host:target.host},servername:target.hostname, - signal:AbortSignal.timeout(90000), + signal:options.signal||AbortSignal.timeout(90000), },resolve); request.on('error',reject); }); if([301,302,303,307,308].includes(response.statusCode||0)&&response.headers.location) { response.destroy(); - return downloadCanvasMedia(new URL(response.headers.location,target).href,output,redirects+1); + return openPublicMedia(new URL(response.headers.location,target).href,options,redirects+1); } + return response; +} +export async function downloadCanvasMedia(url:string,output:string,redirects=0):Promise { + const response=await openPublicMedia(url,{},redirects); if(response.statusCode!==200||Number(response.headers['content-length']||0)>MAX_BYTES) {response.destroy();throw new Error('视频下载失败或超过 125 MiB 处理上限');} let bytes=0; const limit=new Transform({transform(chunk,_encoding,callback){bytes+=chunk.length;callback(bytes>MAX_BYTES?new Error('视频超过 125 MiB 处理上限'):null,chunk);}}); diff --git a/services/canvas-zip/build.cjs b/services/canvas-zip/build.cjs new file mode 100644 index 000000000..6cb64b96d --- /dev/null +++ b/services/canvas-zip/build.cjs @@ -0,0 +1,20 @@ +const { createRequire } = require("node:module"); +const { resolve } = require("node:path"); +const { build } = createRequire(require.resolve("tsx"))("esbuild"); + +build({ + entryPoints: [resolve(__dirname, "server.ts")], + outfile: resolve(process.argv[2] || resolve(__dirname, "dist/server.cjs")), + bundle: true, + platform: "node", + target: "node20", + format: "cjs", + external: ["sharp"], + plugins: [{ + name: "server-only-marker", + setup(build) { + build.onResolve({ filter: /^server-only$/ }, args => ({ path: args.path, namespace: "empty" })); + build.onLoad({ filter: /.*/, namespace: "empty" }, () => ({ contents: "" })); + }, + }], +}).catch(() => { process.exitCode = 1; }); diff --git a/services/canvas-zip/canvas-zip.service b/services/canvas-zip/canvas-zip.service new file mode 100644 index 000000000..fdb3217ce --- /dev/null +++ b/services/canvas-zip/canvas-zip.service @@ -0,0 +1,29 @@ +[Unit] +Description=由壹点详情图 ZIP 打包服务 +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=canvas-zip +Group=canvas-zip +WorkingDirectory=/opt/canvas-zip/current +EnvironmentFile=/etc/canvas-zip.env +Environment=CANVAS_DOWNLOAD_ZIP_DIR=/var/lib/canvas-zip +Environment=CANVAS_ZIP_PORT=18343 +ExecStart=/usr/bin/node /opt/canvas-zip/current/server.cjs +StateDirectory=canvas-zip +StateDirectoryMode=0700 +Restart=on-failure +RestartSec=5 +UMask=0077 +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +MemoryMax=2G +CPUQuota=200% +TasksMax=128 + +[Install] +WantedBy=multi-user.target diff --git a/services/canvas-zip/server.ts b/services/canvas-zip/server.ts new file mode 100644 index 000000000..f8f5bf363 --- /dev/null +++ b/services/canvas-zip/server.ts @@ -0,0 +1,257 @@ +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import { timingSafeEqual } from "node:crypto"; +import { open, rm, stat } from "node:fs/promises"; +import { MIMEType } from "node:util"; +import { claimZipTask, cleanupZipTasks, loadZipTask, saveZipTask, type ZipTask } from "../../lib/canvas/download-zip-task-store"; +import { prepareCanvasZip } from "../../lib/canvas/download-zip-prepare"; +import { normalizeCanvasDownloadAsset } from "../../lib/canvas/photoshop-download-normalize"; +import { zipStorageKey, zipWorkerJob, type ZipWorkerJob, type ZipWorkerSnapshot } from "../../lib/canvas/remote-zip-contract"; +import { openPublicMedia } from "../../lib/video-canvas/download-media"; + +const TOKEN = process.env.CANVAS_ZIP_WORKER_TOKEN || ""; +if (TOKEN.length < 32 || !process.env.CANVAS_DOWNLOAD_ZIP_DIR) { + throw new Error("Set CANVAS_ZIP_WORKER_TOKEN and CANVAS_DOWNLOAD_ZIP_DIR"); +} + +const OWNER = "zip-worker"; +const IMAGE_LIMIT = 125 * 1024 ** 2; +const PART_BYTES = 8 * 1024 ** 2; +const queue = new Map(); +const accepting = new Set(); +let running: string | undefined; + +function authorized(request: IncomingMessage) { + const actual = Buffer.from(request.headers.authorization || ""); + const expected = Buffer.from("Bearer " + TOKEN); + return actual.length === expected.length && timingSafeEqual(actual, expected); +} + +function reply(response: ServerResponse, status: number, body: unknown) { + response.writeHead(status, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" }); + response.end(JSON.stringify(body)); +} + +async function readJob(request: IncomingMessage) { + let bytes = 0; + const body: Buffer[] = []; + for await (const chunk of request) { + const value = Buffer.from(chunk); + bytes += value.length; + if (bytes > 1024 ** 2) throw new Error("Invalid job size"); + body.push(value); + } + return zipWorkerJob.parse(JSON.parse(Buffer.concat(body).toString("utf8"))); +} + +function snapshot(task: ZipTask): ZipWorkerSnapshot { + const { status, stage, error, processedCount, successCount, failedCount, compressPercent } = task; + return { status, stage, error, processedCount, successCount, failedCount, compressPercent, + bytes: task.status === "completed" ? task.zipFile?.bytes : undefined }; +} + +async function fetchInput(job: ZipWorkerJob, index: number, signal: AbortSignal) { + const item = job.files[index]; + const appHost = new URL(job.appOrigin).hostname; + const allowUrl = (url: URL) => url.hostname !== appHost && url.hostname !== "ai.yydd.ai"; + for (const url of item.sources) { + if (signal.aborted) break; + try { + const response = await openPublicMedia(url, { + allowUrl, + signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]), + }); + if (response.statusCode !== 200 || Number(response.headers["content-length"] || 0) > IMAGE_LIMIT) { + response.destroy(); + continue; + } + const contentType = new MIMEType(response.headers["content-type"] || "application/octet-stream").essence; + const buffers: Buffer[] = []; + let bytes = 0; + for await (const packet of response) { + const chunk = Buffer.from(packet); + bytes += chunk.length; + if (bytes > IMAGE_LIMIT) throw new Error("Original exceeds the download limit"); + buffers.push(chunk); + } + return await normalizeCanvasDownloadAsset({ + item: { url, name: item.name }, index, bytes: Buffer.concat(buffers, bytes), + contentType, includeSequencePrefix: true, + }); + } catch { /* Try the next resolved source. */ } + } + return { ok: false as const, url: "", name: item.name, error: "宿主机暂时无法读取这张原图" }; +} + +async function uploadArchive(task: ZipTask, job: ZipWorkerJob) { + const { bucket, token, publicUrl } = job.storage; + const { path, bytes } = task.zipFile!; + const key = zipStorageKey(task.id); + const endpoint = "https://upload.qiniup.com/buckets/" + encodeURIComponent(bucket) + + "/objects/" + encodeURIComponent(Buffer.from(key).toString("base64").replace(/\+/g, "-").replace(/\//g, "_")) + "/uploads"; + const storedUrl = new URL(publicUrl); + const hasExactSize = async () => { + try { + const response = await fetch(storedUrl, { method: "HEAD", redirect: "error", signal: AbortSignal.timeout(8_000) }); + return response.ok && Number(response.headers.get("content-length")) === bytes; + } catch { return false; } + }; + if (await hasExactSize()) return; + const request = async (url: string, method: string, body?: Buffer | string, contentType?: string) => { + const headers: Record = { Authorization: "UpToken " + token }; + if (contentType) headers["Content-Type"] = contentType; + return await fetch(url, { + method, headers, body: Buffer.isBuffer(body) ? Uint8Array.from(body) : body, + redirect: "error", signal: AbortSignal.timeout(method === "PUT" ? 180_000 : 30_000), + }); + }; + const init = await request(endpoint, "POST"); + if (!init.ok) throw new Error("七牛未能开始接收压缩包(" + init.status + ")"); + const { uploadId } = await init.json() as { uploadId?: string }; + if (!uploadId) throw new Error("七牛未返回上传编号"); + const uploadUrl = endpoint + "/" + encodeURIComponent(uploadId); + const parts: Array<{ partNumber: number; etag: string }> = []; + try { + const handle = await open(path, "r"); + try { + for (let offset = 0; offset < bytes; offset += PART_BYTES) { + const data = Buffer.allocUnsafe(Math.min(PART_BYTES, bytes - offset)); + let read = 0; + while (read < data.length) { + const result = await handle.read(data, read, data.length - read, offset + read); + if (!result.bytesRead) throw new Error("压缩包读取不完整"); + read += result.bytesRead; + } + const partNumber = parts.length + 1; + const response = await request(uploadUrl + "/" + partNumber, "PUT", data, "application/octet-stream"); + if (!response.ok) throw new Error("七牛接收分片失败(" + response.status + ")"); + const { etag } = await response.json() as { etag?: string }; + if (!etag) throw new Error("七牛未确认上传分片"); + parts.push({ partNumber, etag }); + task.stage = "正在传送至七牛 " + Math.floor((offset + data.length) / bytes * 100) + "%"; + task.updatedAt = Date.now(); + await saveZipTask(task); + } + } finally { await handle.close(); } + const complete = await request(uploadUrl, "POST", JSON.stringify({ + parts, fname: task.folderName + ".zip", mimeType: "application/zip", + }), "application/json"); + if (!complete.ok) { + if (complete.status === 614 && await hasExactSize()) return; + throw new Error("七牛未确认压缩包(" + complete.status + ")"); + } + const saved = await complete.json() as { key?: string; fsize?: number }; + if (saved.key !== key || saved.fsize !== bytes) throw new Error("七牛返回的压缩包信息不一致"); + } catch (error) { + await request(uploadUrl, "DELETE").then(response => response.body?.cancel()).catch(() => undefined); + throw error; + } +} + +async function run(id: string, job: ZipWorkerJob) { + const release = await claimZipTask(id); + if (!release) return; + let task: ZipTask | null = null; + try { + task = await loadZipTask(id, OWNER); + if (!task || task.status === "completed" || task.status === "failed") return; + task.status = "running"; + task.stage = "宿主机正在准备原图"; + await saveZipTask(task); + const present = task.zipFile && await stat(task.zipFile.path).catch(() => null); + if (!present || present.size !== task.zipFile!.bytes) { + await prepareCanvasZip(task, ({ index, signal }) => fetchInput(job, index, signal)); + } + task.stage = "正在传送至七牛"; + await saveZipTask(task); + await uploadArchive(task, job); + task.status = "completed"; + task.stage = task.failedCount ? "部分原图未能读取" : "打包完成"; + task.compressPercent = 100; + task.error = undefined; + task.updatedAt = Date.now(); + task.expiresAt = Date.now() + 24 * 60 * 60_000; + await saveZipTask(task); + if (task.zipFile) await rm(task.zipFile.path, { force: true }).catch(() => undefined); + } catch (error) { + if (task) { + task.status = "failed"; + task.stage = "打包未完成"; + task.error = (error instanceof Error ? error.message : "宿主机打包失败").slice(0, 450); + task.updatedAt = Date.now(); + await saveZipTask(task); + } + } finally { await release(); } +} + +function drain() { + if (running) return; + const next = queue.entries().next().value; + if (!next) return; + const [id, job] = next; + queue.delete(id); + running = id; + void run(id, job).catch(() => console.error("Canvas ZIP job could not be saved")).finally(() => { + running = undefined; + drain(); + }); +} + +export const router = createServer((request, response) => { + void (async () => { + if (!authorized(request)) { reply(response, 401, {}); return; } + if (request.method === "GET" && request.url === "/health") { + reply(response, 200, { ok: true, active: Number(Boolean(running)), waiting: queue.size }); + return; + } + const id = request.url?.match(/^\/v1\/jobs\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i)?.[1]; + if (!id || request.method !== "PUT") { reply(response, 404, {}); return; } + if (accepting.has(id)) { reply(response, 503, {}); return; } + accepting.add(id); + try { + const job = await readJob(request); + const publicUrl = new URL(job.storage.publicUrl); + if (publicUrl.protocol !== "https:" || !publicUrl.pathname.endsWith("/" + zipStorageKey(id))) { + reply(response, 400, {}); return; + } + let task = await loadZipTask(id, OWNER); + if (running === id || queue.has(id)) { + if (task) reply(response, 200, snapshot(task)); else reply(response, 503, {}); + return; + } + if (queue.size >= 8 && (!task || task.status === "pending" || task.status === "running")) { + reply(response, 503, {}); return; + } + if (task?.status === "failed" && job.attempt > (task.attempt ?? 0)) { + task.attempt = job.attempt; + task.status = "pending"; + task.stage = "等待宿主机打包"; + task.error = undefined; + task.expiresAt = job.expiresAt; + task.updatedAt = Date.now(); + await saveZipTask(task); + } + if (!task) { + const now = Date.now(); + task = { + id, userId: OWNER, folderName: job.folderName, attempt: job.attempt, + files: job.files.map(item => ({ url: item.sources[0] || "", name: item.name })), + totalCount: job.files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0, + status: "pending", stage: "等待宿主机打包", createdAt: now, updatedAt: now, expiresAt: job.expiresAt, + }; + await saveZipTask(task); + } + if (task.status === "pending" || task.status === "running") { + queue.set(id, job); + drain(); + } + reply(response, 200, snapshot(task)); + void cleanupZipTasks().catch(() => undefined); + } finally { accepting.delete(id); } + })().catch(() => { + if (!response.headersSent) reply(response, 400, { error: "无效的打包请求" }); + }); +}); + +if (require.main === module) { + router.listen(Number(process.env.CANVAS_ZIP_PORT || 18343), "127.0.0.1"); +} diff --git a/tests/routes/canvas-zip-worker.test.ts b/tests/routes/canvas-zip-worker.test.ts new file mode 100644 index 000000000..00e03c5a4 --- /dev/null +++ b/tests/routes/canvas-zip-worker.test.ts @@ -0,0 +1,167 @@ +import { test, expect } from "@playwright/test"; +import JSZip from "jszip"; +import { NextRequest, NextResponse } from "next/server"; +import sharp from "sharp"; +import { once } from "node:events"; +import { Readable } from "node:stream"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { randomUUID } from "node:crypto"; +import { serverLoader } from "../helpers/canvas-download-server-loader"; + +test("the worker prepares 13 images and uploads the ZIP through Qiniu multipart", async () => { + const directory = await mkdtemp(join(tmpdir(), "zip-worker-test-")); + const token = "local-test-secret".repeat(3); + const id = randomUUID(); + const key = "canvas-downloads/" + id + ".zip"; + const rgb = { r: 25, g: 100, b: 170 }; + const png = await sharp({ create: { width: 800, height: 800, channels: 3, background: rgb } }).png().toBuffer(); + const jpeg = await sharp({ create: { width: 800, height: 800, channels: 3, background: rgb } }).jpeg().toBuffer(); + const parts: Buffer[] = []; + let complete = 0, downloads = 0; + const loader = serverLoader({ + "../../lib/video-canvas/download-media": { + openPublicMedia: async (url: string, options: { allowUrl: (url: URL) => boolean }) => { + const parsed = new URL(url); + expect(options.allowUrl(parsed)).toBe(true); + expect(options.allowUrl(new URL("https://ai.yydd.ai/api/image-proxy"))).toBe(false); + downloads++; + const original = parsed.pathname.endsWith("original.jpeg") ? jpeg : png; + return Object.assign(Readable.from([original]), { + statusCode: 200, headers: { "content-type": original === png ? "image/png" : "image/jpeg" }, + }); + }, + }, + fetch: async (uri: URL | string, init: RequestInit) => { + const url = new URL(uri); + if (init.method === "HEAD") return new Response(null, { status: 404 }); + expect(url.hostname).toBe("upload.qiniup.com"); + expect(new Headers(init.headers).get("Authorization")).toBe("UpToken scoped-test-token"); + if (url.pathname.endsWith("/uploads")) return Response.json({ uploadId: "one-upload" }); + if (init.method === "PUT") { + parts.push(Buffer.from(init.body as Uint8Array)); + return Response.json({ etag: "one-part" }); + } + expect(init.method).toBe("POST"); + expect(JSON.parse(init.body as string).mimeType).toBe("application/zip"); + complete++; + return Response.json({ key, fsize: Buffer.concat(parts).length }); + }, + }, { CANVAS_DOWNLOAD_ZIP_DIR: directory, CANVAS_ZIP_WORKER_TOKEN: token }); + const { router } = loader("services/canvas-zip/server.ts"); + router.listen(0, "127.0.0.1"); + await once(router, "listening"); + const endpoint = "http://127.0.0.1:" + router.address().port + "/v1/jobs/" + id; + const files = Array.from({ length: 13 }, (_, i) => ({ + name: "详情图-" + i, + sources: ["https://cdn.example/" + i + (i === 12 ? "/original.jpeg" : "/original.png")], + })); + const job = { + attempt: 0, folderName: "详情图", expiresAt: Date.now() + 60_000, appOrigin: "https://ai.yydd.ai", files, + storage: { bucket: "qa-bucket", token: "scoped-test-token", publicUrl: "https://cdn.example/" + key }, + }; + const post = async () => fetch(endpoint, { + method: "PUT", headers: { Authorization: "Bearer " + token, "content-type": "application/json" }, body: JSON.stringify(job), + }); + try { + expect((await fetch(endpoint, { method: "PUT" })).status).toBe(401); + expect((await post()).status).toBe(200); + await expect.poll(async () => { + const response = await post(); + const state = await response.json(); + return state.status; + }, { timeout: 10_000 }).toBe("completed"); + await expect((await post()).json()).resolves.toMatchObject({ successCount: 13, failedCount: 0 }); + expect(downloads).toBe(13); + expect(complete).toBe(1); + const archive = await JSZip.loadAsync(Buffer.concat(parts), { checkCRC32: true }); + const images = Object.values(archive.files).filter(file => !file.dir); + expect(images).toHaveLength(13); + for (const [i, entry] of images.entries()) { + const data = await entry.async("nodebuffer"); + expect(entry.name).toMatch(/\.png$/); + if (i < 12) expect(data).toEqual(png); + else { + const metadata = await sharp(data).metadata(); + expect(metadata).toMatchObject({ width: 800, height: 800, space: "srgb", format: "png" }); + const color = (await sharp(data).extract({ left: 20, top: 20, width: 1, height: 1 }).raw().toBuffer()); + expect(Math.abs(color[0] - rgb.r)).toBeLessThan(4); + expect(Math.abs(color[1] - rgb.g)).toBeLessThan(4); + expect(Math.abs(color[2] - rgb.b)).toBeLessThan(4); + } + } + } finally { + await new Promise(resolve => router.close(() => resolve())); + await rm(directory, { recursive: true, force: true }); + } +}); + + + +test("the style-detail route gives the worker addresses and returns a CDN redirect even when local is requested", async () => { + const directory = await mkdtemp(join(tmpdir(), "zip-remote-route-")); + const id = randomUUID(); + const callbacks: Array<() => Promise> = []; + let owner = "owner"; + const zipUrl = "https://cdn.example/canvas-downloads/" + id + ".zip"; + const raw = { id: "storage", isEnabled: true }; + const worker = "http://127.0.0.1:18343"; + const config = { provider: "qiniu-kodo", region: "cn-east-1", bucket: "qa", publicUrl: "https://cdn.example", + accessKeyId: "fake-public-id", secretAccessKey: "fake-private-key" }; + let workerCalls = 0; + const route = serverLoader({ + "@/lib/observability/active-work": { trackRuntimeWork: async (fn: () => Promise) => fn() }, + "@/app/(auth)/auth": { auth: async () => ({ user: { id: owner } }) }, + "next/server": { NextResponse, after: (fn: () => Promise) => callbacks.push(fn) }, + "@/lib/canvas/photoshop-safe-download": { + fetchCanvasDownloadAsset: () => { throw new Error("The production server must not fetch the originals"); }, + }, + "@/lib/storage/image-archive-db": { getArchivedImageUrlMappings: async () => ({ + "https://ai.yydd.ai/archive": "https://cdn.example/archive.png", + }) }, + "@/lib/db/queries": { getDefaultStorageConfig: async () => raw, getStorageConfigById: async () => raw }, + "@/lib/storage/service": { toDbS3UploadConfig: () => config, buildS3PublicUrl: () => zipUrl }, + "@/lib/storage/canvas-zip-delivery": { + publishCanvasZip: () => { throw new Error("The production server must not upload the ZIP"); }, + }, + fetch: async (uri: URL | string, init: RequestInit) => { + const url = new URL(uri); + if (url.origin === worker) { + workerCalls++; + const job = JSON.parse(init.body as string); + expect(job.files[0].sources).toEqual(["https://cdn.example/archive.png"]); + expect(job.files[1].sources).toEqual(["https://provider.example/result.png"]); + expect(job).not.toHaveProperty("cookie"); + expect(job.storage).not.toHaveProperty("secretAccessKey"); + const policy = JSON.parse(Buffer.from(job.storage.token.split(":")[2], "base64url").toString()); + expect(policy.scope).toBe("qa:canvas-downloads/" + id + ".zip"); + return Response.json({ status: "completed", stage: "打包完成", processedCount: 2, + successCount: 2, failedCount: 0, compressPercent: 100, bytes: 12345 }); + } + expect(url.href).toContain(zipUrl); + expect(init.method).toBe("HEAD"); + return new Response(null, { headers: { "content-length": "12345" } }); + }, + }, { CANVAS_DOWNLOAD_ZIP_DIR: directory, CANVAS_ZIP_WORKER_URL: worker, + CANVAS_ZIP_WORKER_TOKEN: "local-test-secret".repeat(3) })("app/api/canvas/download-zip-task/route.ts"); + const endpoint = "https://ai.yydd.ai/api/canvas/download-zip-task"; + try { + const started = await route.POST(new NextRequest(endpoint, { + method: "POST", + body: JSON.stringify({ taskId: id, scope: "style-detail", folderName: "详情图", files: [ + { url: "https://ai.yydd.ai/archive" }, { url: "https://provider.example/result.png" }, + ] }), + headers: { cookie: "auth=private-test-cookie" }, + })); + expect(started.status).toBe(202); + while (callbacks.length) await callbacks.shift()!(); + expect(workerCalls).toBe(1); + const download = await route.GET(new NextRequest(endpoint + "?taskId=" + id + "&download=1&local=1")); + expect(download.status).toBe(307); + expect(download.headers.get("location")).toContain(zipUrl); + owner = "another-user"; + const denied = await route.GET(new NextRequest(endpoint + "?taskId=" + id + "&download=1")); + expect(denied.status).toBe(404); + } finally { await rm(directory, { recursive: true, force: true }); } +});