Run detail image ZIP exports on the video host
This commit is contained in:
parent
a99a63a7bb
commit
e64bbfef9c
@ -1,12 +1,11 @@
|
|||||||
import { writeDownloadZip, readDownloadZip } from "@/lib/canvas/download-zip-stream";
|
import { readDownloadZip } from "@/lib/canvas/download-zip-stream";
|
||||||
import { claimZipTask, loadZipTask, saveZipTask, zipTaskPath, assertZipDiskSpace, cleanupZipTasks, assertZipTaskCapacity, type ZipTask } from "@/lib/canvas/download-zip-task-store";
|
import { claimZipTask, loadZipTask, saveZipTask, zipTaskPath, assertZipDiskSpace, cleanupZipTasks, assertZipTaskCapacity, type ZipTask } from "@/lib/canvas/download-zip-task-store";
|
||||||
import { parseDownloadByteRange } from "@/lib/canvas/download-byte-range";
|
import { parseDownloadByteRange } from "@/lib/canvas/download-byte-range";
|
||||||
import { createReadStream } from "node:fs";
|
import { mkdir, stat, rm } from "node:fs/promises";
|
||||||
import { mkdir, stat, writeFile, rm } from "node:fs/promises";
|
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { randomUUID } from "node:crypto";
|
import { randomUUID } from "node:crypto";
|
||||||
import { after, NextRequest, NextResponse } from "next/server";
|
import { after, NextRequest, NextResponse } from "next/server";
|
||||||
import JSZip from "jszip";
|
import { prepareCanvasZip } from "@/lib/canvas/download-zip-prepare";
|
||||||
import { auth } from "@/app/(auth)/auth";
|
import { auth } from "@/app/(auth)/auth";
|
||||||
import { fetchCanvasDownloadAsset } from "@/lib/canvas/photoshop-safe-download";
|
import { fetchCanvasDownloadAsset } from "@/lib/canvas/photoshop-safe-download";
|
||||||
import { publishCanvasZip } from "@/lib/storage/canvas-zip-delivery";
|
import { publishCanvasZip } from "@/lib/storage/canvas-zip-delivery";
|
||||||
@ -29,16 +28,6 @@ function sanitizeFolderName(value: string): string {
|
|||||||
return cleaned && !/^\.+$/.test(cleaned) ? cleaned.slice(0, 100) : `canvas-images-${Date.now()}`;
|
return cleaned && !/^\.+$/.test(cleaned) ? cleaned.slice(0, 100) : `canvas-images-${Date.now()}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildContentDisposition(folderName: string): string {
|
|
||||||
const encodedName = encodeURIComponent(`${folderName}.zip`);
|
|
||||||
const asciiName =
|
|
||||||
folderName
|
|
||||||
.replace(/[^\x20-\x7E]+/g, "")
|
|
||||||
.replace(/"/g, "")
|
|
||||||
.trim() || "canvas-images";
|
|
||||||
return `attachment; filename="${asciiName}.zip"; filename*=UTF-8''${encodedName}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function getRequestOrigin(request: NextRequest): string {
|
function getRequestOrigin(request: NextRequest): string {
|
||||||
const forwardedHost = (request.headers.get("x-forwarded-host") || "")
|
const forwardedHost = (request.headers.get("x-forwarded-host") || "")
|
||||||
.split(",")[0]
|
.split(",")[0]
|
||||||
@ -99,50 +88,20 @@ async function processTask(id: string, userId: string, requestOrigin: string, co
|
|||||||
if (!release) return;
|
if (!release) return;
|
||||||
task = await loadZipTask(id, userId);
|
task = await loadZipTask(id, userId);
|
||||||
if (!task || task.status === "completed" || task.status === "failed") return;
|
if (!task || task.status === "completed" || task.status === "failed") return;
|
||||||
|
if (task.remote) {
|
||||||
|
const { updateRemoteCanvasZip } = await import("@/lib/canvas/remote-zip-client");
|
||||||
|
await updateRemoteCanvasZip(task, requestOrigin);
|
||||||
|
return;
|
||||||
|
}
|
||||||
await assertZipDiskSpace();
|
await assertZipDiskSpace();
|
||||||
const existingZip = task.zipFile && task.contentDisposition && await stat(task.zipFile.path).catch(() => null);
|
const existingZip = task.zipFile && task.contentDisposition && await stat(task.zipFile.path).catch(() => null);
|
||||||
const reuseZip = existingZip && existingZip.size === task.zipFile!.bytes;
|
const reuseZip = existingZip && existingZip.size === task.zipFile!.bytes;
|
||||||
task.status = "running"; task.stage = reuseZip ? "正在准备加速下载" : "正在准备原图"; task.updatedAt = Date.now();
|
task.status = "running"; task.stage = reuseZip ? "正在准备加速下载" : "正在准备原图"; task.updatedAt = Date.now();
|
||||||
await saveZipTask(task);
|
await saveZipTask(task);
|
||||||
if (!reuseZip) {
|
if (!reuseZip) {
|
||||||
delete task.zipFile; delete task.contentDisposition; delete task.cdnUrl;
|
await prepareCanvasZip(task, (params) =>
|
||||||
// Stage each image on disk, not all PNG buffers in the web process heap.
|
fetchCanvasDownloadAsset({ ...params, requestOrigin, cookie })
|
||||||
// A restart resumes from these files using the current owner's session.
|
);
|
||||||
let cursor = 0;
|
|
||||||
let writes = Promise.resolve();
|
|
||||||
const persist = () => { const current = task!; writes = writes.then(() => saveZipTask(current)); return writes; };
|
|
||||||
const results = await Promise.allSettled(Array.from({ length: Math.min(2, task.files.length) }, async () => {
|
|
||||||
while (cursor < task!.files.length) {
|
|
||||||
const index = cursor++; const file = task!.files[index];
|
|
||||||
const path = join(zipTaskPath(id), `${index}.image`);
|
|
||||||
if (file.prepared && file.filename && (await stat(path).catch(() => null))?.size) continue;
|
|
||||||
delete file.prepared; delete file.filename;
|
|
||||||
const result = await fetchCanvasDownloadAsset({ file, index, requestOrigin, cookie,
|
|
||||||
signal: AbortSignal.timeout(120_000) }).catch(() => ({ ok: false as const, error: "原图读取超时或源地址暂时不可用" }));
|
|
||||||
if (result.ok) {
|
|
||||||
await assertZipDiskSpace(result.bytes.length);
|
|
||||||
await writeFile(path, result.bytes, { mode: 0o600 });
|
|
||||||
file.prepared = `${index}.image`; file.filename = result.filename; delete file.error;
|
|
||||||
} else file.error = result.error;
|
|
||||||
task!.processedCount = task!.files.filter(f => f.prepared || f.error).length;
|
|
||||||
task!.successCount = task!.files.filter(f => f.prepared).length;
|
|
||||||
task!.failedCount = task!.files.filter(f => f.error).length;
|
|
||||||
task!.stage = `正在准备原图 ${task!.processedCount}/${task!.totalCount}`;
|
|
||||||
task!.updatedAt = Date.now(); await persist();
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
const rejected = results.find(result => result.status === "rejected");
|
|
||||||
if (rejected?.status === "rejected") throw rejected.reason;
|
|
||||||
if (!task.successCount) throw new Error("所选原图暂时都无法读取,请稍后重试;画布图片不会被删除");
|
|
||||||
const zip = new JSZip(); const folder = zip.folder(task.folderName) || zip;
|
|
||||||
for (const file of task.files) if (file.prepared && file.filename)
|
|
||||||
folder.file(file.filename, createReadStream(join(zipTaskPath(id), file.prepared))
|
|
||||||
.once("close", () => { void rm(join(zipTaskPath(id), file.prepared!), { force: true }).catch(() => undefined); }));
|
|
||||||
if (task.failedCount) folder.file("_download-failures.txt", task.files.filter(f => f.error)
|
|
||||||
.map(f => `${f.name || "图片"}:${f.error}`).join("\n"));
|
|
||||||
task.stage = "正在生成完整 ZIP 文件"; await saveZipTask(task);
|
|
||||||
task.zipFile = await writeDownloadZip(zip, task.id, percent => { task!.compressPercent = percent; });
|
|
||||||
task.contentDisposition = buildContentDisposition(task.folderName);
|
|
||||||
}
|
}
|
||||||
task.stage = "正在准备下载链接"; await saveZipTask(task);
|
task.stage = "正在准备下载链接"; await saveZipTask(task);
|
||||||
const cdn = await publishCanvasZip({ id: task.id, ...task.zipFile!, disposition: task.contentDisposition!,
|
const cdn = await publishCanvasZip({ id: task.id, ...task.zipFile!, disposition: task.contentDisposition!,
|
||||||
@ -195,13 +154,20 @@ export async function POST(request: NextRequest) {
|
|||||||
await mkdir(zipTaskPath(id), { mode: 0o700 });
|
await mkdir(zipTaskPath(id), { mode: 0o700 });
|
||||||
});
|
});
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
task = { id, userId: session.user.id, folderName: sanitizeFolderName(String(body.folderName || "")), files,
|
const remote = body.scope === "style-detail" &&
|
||||||
|
process.env.CANVAS_ZIP_WORKER_URL && process.env.CANVAS_ZIP_WORKER_TOKEN
|
||||||
|
? { attempt: 0 } : undefined;
|
||||||
|
task = { id, userId: session.user.id, folderName: sanitizeFolderName(String(body.folderName || "")), files, remote,
|
||||||
totalCount: files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0,
|
totalCount: files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0,
|
||||||
status: "pending", stage: "等待后台准备", createdAt: now, updatedAt: now, expiresAt: now + PREPARATION_TTL_MS };
|
status: "pending", stage: "等待后台准备", createdAt: now, updatedAt: now, expiresAt: now + PREPARATION_TTL_MS };
|
||||||
await saveZipTask(task);
|
await saveZipTask(task);
|
||||||
} else if ((task.status === "failed" || task.status === "completed" && task.cdnObject && !task.cdnUrl) && body.retry === true) {
|
} else if ((task.status === "failed" || task.status === "completed" && task.cdnObject && !task.cdnUrl) && body.retry === true) {
|
||||||
task.status = "pending"; task.error = undefined;
|
task.status = "pending"; task.error = undefined;
|
||||||
await assertZipTaskCapacity(session.user.id);
|
await assertZipTaskCapacity(session.user.id);
|
||||||
|
if (task.remote) {
|
||||||
|
task.remote.attempt++;
|
||||||
|
delete task.remote.sources;
|
||||||
|
}
|
||||||
if (!task.zipFile) {
|
if (!task.zipFile) {
|
||||||
for (const file of task.files) delete file.error;
|
for (const file of task.files) delete file.error;
|
||||||
task.processedCount = task.successCount = task.files.filter(file => file.prepared).length;
|
task.processedCount = task.successCount = task.files.filter(file => file.prepared).length;
|
||||||
@ -211,7 +177,10 @@ export async function POST(request: NextRequest) {
|
|||||||
await saveZipTask(task);
|
await saveZipTask(task);
|
||||||
}
|
}
|
||||||
if (task.status === "pending" || task.status === "running") schedule(task, request);
|
if (task.status === "pending" || task.status === "running") schedule(task, request);
|
||||||
after(() => cleanupZipTasks().catch(() => { console.warn("[CanvasDownloadZipTask] Expired file cleanup deferred"); }));
|
after(() => cleanupZipTasks(async object => {
|
||||||
|
const { removeCanvasZipObject } = await import("@/lib/storage/canvas-zip-delivery");
|
||||||
|
await removeCanvasZipObject(object);
|
||||||
|
}).catch(() => { console.warn("[CanvasDownloadZipTask] Expired file cleanup deferred"); }));
|
||||||
return NextResponse.json({ success: true, task: serializeTask(task) }, { status: 202, headers: { "Cache-Control": "no-store" } });
|
return NextResponse.json({ success: true, task: serializeTask(task) }, { status: 202, headers: { "Cache-Control": "no-store" } });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn("[CanvasDownloadZipTask] Create rejected", { message: (error as Error).message });
|
console.warn("[CanvasDownloadZipTask] Create rejected", { message: (error as Error).message });
|
||||||
@ -226,6 +195,8 @@ export async function GET(request: NextRequest) {
|
|||||||
const task = await loadZipTask(id, session.user.id);
|
const task = await loadZipTask(id, session.user.id);
|
||||||
if (!task) return NextResponse.json({ error: "打包任务不存在或已过期,请重新打包" }, { status: 404 });
|
if (!task) return NextResponse.json({ error: "打包任务不存在或已过期,请重新打包" }, { status: 404 });
|
||||||
if (request.nextUrl.searchParams.get("download") === "1") {
|
if (request.nextUrl.searchParams.get("download") === "1") {
|
||||||
|
if (task.remote && task.status === "completed" && task.cdnUrl)
|
||||||
|
return NextResponse.redirect(task.cdnUrl, 307);
|
||||||
if (task.status !== "completed" || !task.zipFile) return NextResponse.json({ error: "ZIP 尚未准备好" }, { status: 409 });
|
if (task.status !== "completed" || !task.zipFile) return NextResponse.json({ error: "ZIP 尚未准备好" }, { status: 409 });
|
||||||
if (task.cdnUrl && request.nextUrl.searchParams.get("local") !== "1")
|
if (task.cdnUrl && request.nextUrl.searchParams.get("local") !== "1")
|
||||||
return NextResponse.redirect(task.cdnUrl, 307);
|
return NextResponse.redirect(task.cdnUrl, 307);
|
||||||
|
|||||||
@ -5892,6 +5892,7 @@ export function AgentGenerationWorkspaceDarkV2({
|
|||||||
headers: { "Content-Type": "application/json" },
|
headers: { "Content-Type": "application/json" },
|
||||||
cache: "no-store",
|
cache: "no-store",
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
|
scope: params.scope === "detail" ? "style-detail" : undefined,
|
||||||
folderName: params.zipName,
|
folderName: params.zipName,
|
||||||
files: normalizedUrls.map((imageUrl) => ({
|
files: normalizedUrls.map((imageUrl) => ({
|
||||||
url: imageUrl,
|
url: imageUrl,
|
||||||
|
|||||||
57
lib/canvas/download-zip-prepare.ts
Normal file
57
lib/canvas/download-zip-prepare.ts
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
import { createReadStream } from "node:fs";
|
||||||
|
import { stat, writeFile, rm } from "node:fs/promises";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import JSZip from "jszip";
|
||||||
|
import { writeDownloadZip } from "./download-zip-stream";
|
||||||
|
import { assertZipDiskSpace, saveZipTask, zipTaskPath, type ZipTask } from "./download-zip-task-store";
|
||||||
|
import type { CanvasDownloadAssetResult } from "./photoshop-safe-download";
|
||||||
|
|
||||||
|
export function buildZipContentDisposition(folderName: string) {
|
||||||
|
const name = `${folderName}.zip`;
|
||||||
|
const asciiName = folderName.replace(/[^\x20-\x7E]+/g, "").replace(/"/g, "").trim() || "canvas-images";
|
||||||
|
return `attachment; filename="${asciiName}.zip"; filename*=UTF-8''${encodeURIComponent(name)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function prepareCanvasZip(
|
||||||
|
task: ZipTask,
|
||||||
|
fetchAsset: (params: { file: ZipTask["files"][number]; index: number; signal: AbortSignal }) => Promise<CanvasDownloadAssetResult>,
|
||||||
|
) {
|
||||||
|
delete task.zipFile; delete task.contentDisposition; delete task.cdnUrl;
|
||||||
|
// Stage each image on disk, not all PNG buffers in the web process heap.
|
||||||
|
// A restart resumes from these files using the current owner's session.
|
||||||
|
let cursor = 0;
|
||||||
|
let writes = Promise.resolve();
|
||||||
|
const persist = () => { const current = task; writes = writes.then(() => saveZipTask(current)); return writes; };
|
||||||
|
const results = await Promise.allSettled(Array.from({ length: Math.min(2, task.files.length) }, async () => {
|
||||||
|
while (cursor < task.files.length) {
|
||||||
|
const index = cursor++; const file = task.files[index];
|
||||||
|
const path = join(zipTaskPath(task.id), `${index}.image`);
|
||||||
|
if (file.prepared && file.filename && (await stat(path).catch(() => null))?.size) continue;
|
||||||
|
delete file.prepared; delete file.filename;
|
||||||
|
const result = await fetchAsset({ file, index,
|
||||||
|
signal: AbortSignal.timeout(120_000) }).catch(() => ({ ok: false as const, error: "原图读取超时或源地址暂时不可用" }));
|
||||||
|
if (result.ok) {
|
||||||
|
await assertZipDiskSpace(result.bytes.length);
|
||||||
|
await writeFile(path, result.bytes, { mode: 0o600 });
|
||||||
|
file.prepared = `${index}.image`; file.filename = result.filename; delete file.error;
|
||||||
|
} else file.error = result.error;
|
||||||
|
task.processedCount = task.files.filter(f => f.prepared || f.error).length;
|
||||||
|
task.successCount = task.files.filter(f => f.prepared).length;
|
||||||
|
task.failedCount = task.files.filter(f => f.error).length;
|
||||||
|
task.stage = `正在准备原图 ${task.processedCount}/${task.totalCount}`;
|
||||||
|
task.updatedAt = Date.now(); await persist();
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
const rejected = results.find(result => result.status === "rejected");
|
||||||
|
if (rejected?.status === "rejected") throw rejected.reason;
|
||||||
|
if (!task.successCount) throw new Error("所选原图暂时都无法读取,请稍后重试;画布图片不会被删除");
|
||||||
|
const zip = new JSZip(); const folder = zip.folder(task.folderName) || zip;
|
||||||
|
for (const file of task.files) if (file.prepared && file.filename)
|
||||||
|
folder.file(file.filename, createReadStream(join(zipTaskPath(task.id), file.prepared))
|
||||||
|
.once("close", () => { void rm(join(zipTaskPath(task.id), file.prepared!), { force: true }).catch(() => undefined); }));
|
||||||
|
if (task.failedCount) folder.file("_download-failures.txt", task.files.filter(f => f.error)
|
||||||
|
.map(f => `${f.name || "图片"}:${f.error}`).join("\n"));
|
||||||
|
task.stage = "正在生成完整 ZIP 文件"; await saveZipTask(task);
|
||||||
|
task.zipFile = await writeDownloadZip(zip, task.id, percent => { task.compressPercent = percent; });
|
||||||
|
task.contentDisposition = buildZipContentDisposition(task.folderName);
|
||||||
|
}
|
||||||
@ -10,6 +10,8 @@ export type ZipTask = {
|
|||||||
totalCount: number; processedCount: number; successCount: number; failedCount: number;
|
totalCount: number; processedCount: number; successCount: number; failedCount: number;
|
||||||
compressPercent: number; status: "pending" | "running" | "completed" | "failed";
|
compressPercent: number; status: "pending" | "running" | "completed" | "failed";
|
||||||
stage: string; error?: string; zipFile?: { path: string; bytes: number };
|
stage: string; error?: string; zipFile?: { path: string; bytes: number };
|
||||||
|
remote?: { attempt: number; storageConfigId?: string; sources?: string[][] };
|
||||||
|
attempt?: number;
|
||||||
contentDisposition?: string; cdnUrl?: string; cdnObject?: { storageConfigId: string; key: string };
|
contentDisposition?: string; cdnUrl?: string; cdnObject?: { storageConfigId: string; key: string };
|
||||||
createdAt: number; updatedAt: number; expiresAt: number;
|
createdAt: number; updatedAt: number; expiresAt: number;
|
||||||
};
|
};
|
||||||
@ -62,7 +64,7 @@ export async function assertZipDiskSpace(bytes = 0) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let cleanupAt = 0;
|
let cleanupAt = 0;
|
||||||
export async function cleanupZipTasks() {
|
export async function cleanupZipTasks(removeObject?: (object: NonNullable<ZipTask["cdnObject"]>) => Promise<void>) {
|
||||||
if (Date.now() - cleanupAt < 60_000) return;
|
if (Date.now() - cleanupAt < 60_000) return;
|
||||||
cleanupAt = Date.now();
|
cleanupAt = Date.now();
|
||||||
for (const id of await readdir(downloadZipDirectory).catch(() => [] as string[])) {
|
for (const id of await readdir(downloadZipDirectory).catch(() => [] as string[])) {
|
||||||
@ -74,10 +76,7 @@ export async function cleanupZipTasks() {
|
|||||||
const release = await claimZipTask(id);
|
const release = await claimZipTask(id);
|
||||||
if (!release) continue;
|
if (!release) continue;
|
||||||
try {
|
try {
|
||||||
if (task.cdnObject) {
|
if (task.cdnObject && removeObject) await removeObject(task.cdnObject).catch(() => undefined);
|
||||||
const { removeCanvasZipObject } = await import("@/lib/storage/canvas-zip-delivery");
|
|
||||||
await removeCanvasZipObject(task.cdnObject).catch(() => undefined);
|
|
||||||
}
|
|
||||||
await rm(join(downloadZipDirectory, `${id}.zip`), { force: true });
|
await rm(join(downloadZipDirectory, `${id}.zip`), { force: true });
|
||||||
await rm(join(downloadZipDirectory, `${id}.zip.partial`), { force: true });
|
await rm(join(downloadZipDirectory, `${id}.zip.partial`), { force: true });
|
||||||
} finally { await release(); }
|
} finally { await release(); }
|
||||||
|
|||||||
212
lib/canvas/photoshop-download-normalize.ts
Normal file
212
lib/canvas/photoshop-download-normalize.ts
Normal file
@ -0,0 +1,212 @@
|
|||||||
|
export type DownloadFileItem = {
|
||||||
|
url: string;
|
||||||
|
name?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type CanvasDownloadAssetResult =
|
||||||
|
| {
|
||||||
|
ok: true;
|
||||||
|
filename: string;
|
||||||
|
bytes: Buffer;
|
||||||
|
contentType: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
ok: false;
|
||||||
|
url: string;
|
||||||
|
name?: string;
|
||||||
|
error: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function sanitizeCanvasDownloadFileName(value: string, fallback: string): string {
|
||||||
|
const cleaned = String(value || "")
|
||||||
|
.replace(/\.[a-zA-Z0-9]{2,6}$/g, "")
|
||||||
|
.replace(/[\\/:*?"<>|]+/g, "-")
|
||||||
|
.replace(/\s+/g, "-")
|
||||||
|
.replace(/-+/g, "-")
|
||||||
|
.replace(/^-+|-+$/g, "")
|
||||||
|
.trim()
|
||||||
|
.slice(0, 120);
|
||||||
|
return cleaned || fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildAttachmentContentDisposition(filename: string): string {
|
||||||
|
const safeAsciiName =
|
||||||
|
filename
|
||||||
|
.replace(/[^\x20-\x7E]+/g, "")
|
||||||
|
.replace(/"/g, "")
|
||||||
|
.trim() || `canvas-image-${Date.now()}.png`;
|
||||||
|
return `attachment; filename="${safeAsciiName}"; filename*=UTF-8''${encodeURIComponent(filename)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getContentTypeMime(contentType: string): string {
|
||||||
|
return String(contentType || "")
|
||||||
|
.split(";")[0]
|
||||||
|
?.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasAsciiAt(buffer: Buffer, offset: number, value: string): boolean {
|
||||||
|
return buffer.subarray(offset, offset + value.length).toString("latin1") === value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function looksLikeSvg(buffer: Buffer, contentType: string): boolean {
|
||||||
|
if (getContentTypeMime(contentType) === "image/svg+xml") return true;
|
||||||
|
const text = buffer.subarray(0, Math.min(buffer.length, 1024)).toString("utf8").trimStart();
|
||||||
|
return /^(?:<\?xml[^>]*>\s*)?<svg[\s>]/i.test(text);
|
||||||
|
}
|
||||||
|
|
||||||
|
function looksLikeErrorDocument(buffer: Buffer, contentType: string): boolean {
|
||||||
|
const mime = getContentTypeMime(contentType);
|
||||||
|
if (
|
||||||
|
mime.startsWith("text/") ||
|
||||||
|
mime === "application/json" ||
|
||||||
|
mime === "application/xml" ||
|
||||||
|
mime === "application/xhtml+xml"
|
||||||
|
) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
const text = buffer.subarray(0, Math.min(buffer.length, 512)).toString("utf8").trimStart();
|
||||||
|
return /^(?:<!doctype\s+html|<html[\s>]|<\?xml|<error[\s>]|[{[])/i.test(text);
|
||||||
|
}
|
||||||
|
|
||||||
|
function detectAsset(bytes: Buffer, contentType: string): { kind: "image" | "video" | "audio"; mimeType: string; ext: string } | null {
|
||||||
|
if (bytes.length < 4) return null;
|
||||||
|
|
||||||
|
if (bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
|
||||||
|
return { kind: "image", mimeType: "image/png", ext: "png" };
|
||||||
|
}
|
||||||
|
if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
|
||||||
|
return { kind: "image", mimeType: "image/jpeg", ext: "jpg" };
|
||||||
|
}
|
||||||
|
if (hasAsciiAt(bytes, 0, "RIFF") && hasAsciiAt(bytes, 8, "WEBP")) {
|
||||||
|
return { kind: "image", mimeType: "image/webp", ext: "webp" };
|
||||||
|
}
|
||||||
|
if (hasAsciiAt(bytes, 0, "GIF87a") || hasAsciiAt(bytes, 0, "GIF89a")) {
|
||||||
|
return { kind: "image", mimeType: "image/gif", ext: "gif" };
|
||||||
|
}
|
||||||
|
if (hasAsciiAt(bytes, 0, "BM")) {
|
||||||
|
return { kind: "image", mimeType: "image/bmp", ext: "bmp" };
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
bytes.subarray(0, 4).equals(Buffer.from([0x49, 0x49, 0x2a, 0x00])) ||
|
||||||
|
bytes.subarray(0, 4).equals(Buffer.from([0x4d, 0x4d, 0x00, 0x2a]))
|
||||||
|
) {
|
||||||
|
return { kind: "image", mimeType: "image/tiff", ext: "tiff" };
|
||||||
|
}
|
||||||
|
if (looksLikeSvg(bytes, contentType)) {
|
||||||
|
return { kind: "image", mimeType: "image/svg+xml", ext: "svg" };
|
||||||
|
}
|
||||||
|
if (hasAsciiAt(bytes, 0, "RIFF") && hasAsciiAt(bytes, 8, "WAVE")) return {kind:"audio",mimeType:"audio/wav",ext:"wav"};
|
||||||
|
if (hasAsciiAt(bytes, 0, "ID3") || (bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0 && (bytes[1] & 0x06) !== 0)) return {kind:"audio",mimeType:"audio/mpeg",ext:"mp3"};
|
||||||
|
if (hasAsciiAt(bytes, 0, "OggS")) return {kind:"audio",mimeType:"audio/ogg",ext:"ogg"};
|
||||||
|
if (hasAsciiAt(bytes, 0, "fLaC")) return {kind:"audio",mimeType:"audio/flac",ext:"flac"};
|
||||||
|
if (getContentTypeMime(contentType).startsWith("audio/")) return {kind:"audio",mimeType:getContentTypeMime(contentType),ext:getContentTypeMime(contentType).includes("mp4") ? "m4a" : getContentTypeMime(contentType).split("/")[1]};
|
||||||
|
if (hasAsciiAt(bytes, 4, "ftyp")) {
|
||||||
|
const brand = bytes.subarray(8, Math.min(bytes.length, 32)).toString("latin1").toLowerCase();
|
||||||
|
if (brand.includes("m4a")) return {kind:"audio",mimeType:"audio/mp4",ext:"m4a"};
|
||||||
|
if (brand.includes("avif")) return { kind: "image", mimeType: "image/avif", ext: "avif" };
|
||||||
|
if (brand.includes("heic") || brand.includes("heif") || brand.includes("mif1")) {
|
||||||
|
return { kind: "image", mimeType: "image/heic", ext: "heic" };
|
||||||
|
}
|
||||||
|
return { kind: "video", mimeType: "video/mp4", ext: "mp4" };
|
||||||
|
}
|
||||||
|
if (bytes.subarray(0, 4).equals(Buffer.from([0x1a, 0x45, 0xdf, 0xa3]))) {
|
||||||
|
return { kind: "video", mimeType: "video/webm", ext: "webm" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const mime = getContentTypeMime(contentType);
|
||||||
|
if (mime.startsWith("image/")) {
|
||||||
|
const ext = mime.includes("jpeg") ? "jpg" : mime.split("/")[1] || "png";
|
||||||
|
return { kind: "image", mimeType: mime, ext };
|
||||||
|
}
|
||||||
|
if (mime.startsWith("video/")) {
|
||||||
|
const ext = mime.includes("quicktime") ? "mov" : mime.split("/")[1] || "mp4";
|
||||||
|
return { kind: "video", mimeType: mime, ext };
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function convertImageToPhotoshopPng(bytes: Buffer): Promise<Buffer> {
|
||||||
|
const sharp = await import("sharp");
|
||||||
|
const image = sharp.default(bytes, {
|
||||||
|
animated: false,
|
||||||
|
failOn: "none",
|
||||||
|
limitInputPixels: false,
|
||||||
|
});
|
||||||
|
const metadata = await image.metadata();
|
||||||
|
if (!metadata.width || !metadata.height) {
|
||||||
|
throw new Error("无法解析图片尺寸");
|
||||||
|
}
|
||||||
|
// RGB/RGBA, 8-bit, single-frame PNG is already compatible; preserve its exact bytes.
|
||||||
|
const rgb = (metadata.space === "srgb" && metadata.depth === "uchar")
|
||||||
|
|| (metadata.space === "rgb16" && metadata.depth === "ushort");
|
||||||
|
if (metadata.format === "png" && rgb
|
||||||
|
&& !metadata.isPalette && (metadata.pages || 1) === 1 && (!metadata.orientation || metadata.orientation === 1)) return bytes;
|
||||||
|
const output = image.rotate().toColorspace("srgb");
|
||||||
|
if (metadata.icc) output.withIccProfile("srgb");
|
||||||
|
return output.png({ compressionLevel: 3, adaptiveFiltering: true }).toBuffer();
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildCanvasAssetFilename(params: {
|
||||||
|
item: DownloadFileItem;
|
||||||
|
index: number;
|
||||||
|
ext: string;
|
||||||
|
includeSequencePrefix: boolean;
|
||||||
|
}) {
|
||||||
|
const baseName = sanitizeCanvasDownloadFileName(
|
||||||
|
params.item.name || `canvas-image-${params.index + 1}`,
|
||||||
|
`canvas-image-${params.index + 1}`
|
||||||
|
);
|
||||||
|
if (!params.includeSequencePrefix) {
|
||||||
|
return `${baseName}.${params.ext}`;
|
||||||
|
}
|
||||||
|
const sequencePrefix = String(params.index + 1).padStart(3, "0");
|
||||||
|
return `${sequencePrefix}-${baseName}.${params.ext}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function normalizeCanvasDownloadAsset(params: {
|
||||||
|
item: DownloadFileItem;
|
||||||
|
index: number;
|
||||||
|
bytes: Buffer;
|
||||||
|
contentType: string;
|
||||||
|
includeSequencePrefix: boolean;
|
||||||
|
}, convert = convertImageToPhotoshopPng): Promise<CanvasDownloadAssetResult> {
|
||||||
|
if (params.bytes.length === 0) {
|
||||||
|
throw new Error("empty response");
|
||||||
|
}
|
||||||
|
if (looksLikeErrorDocument(params.bytes, params.contentType) && !looksLikeSvg(params.bytes, params.contentType)) {
|
||||||
|
throw new Error("源地址返回的不是图片文件");
|
||||||
|
}
|
||||||
|
|
||||||
|
const detected = detectAsset(params.bytes, params.contentType);
|
||||||
|
if (!detected) {
|
||||||
|
throw new Error("无法识别图片格式");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (detected.kind !== "image") {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
filename: buildCanvasAssetFilename({
|
||||||
|
item: params.item,
|
||||||
|
index: params.index,
|
||||||
|
ext: detected.ext,
|
||||||
|
includeSequencePrefix: params.includeSequencePrefix,
|
||||||
|
}),
|
||||||
|
bytes: params.bytes,
|
||||||
|
contentType: detected.mimeType,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const pngBytes = await convert(params.bytes);
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
filename: buildCanvasAssetFilename({
|
||||||
|
item: params.item,
|
||||||
|
index: params.index,
|
||||||
|
ext: "png",
|
||||||
|
includeSequencePrefix: params.includeSequencePrefix,
|
||||||
|
}),
|
||||||
|
bytes: pngBytes,
|
||||||
|
contentType: "image/png",
|
||||||
|
};
|
||||||
|
}
|
||||||
@ -4,24 +4,15 @@ import { fetchImageBytes, onceImageBytes } from "@/lib/storage/image-byte-cache"
|
|||||||
import { extractImageProxySourceUrl, inferOriginalImageUrl } from "@/lib/image-url";
|
import { extractImageProxySourceUrl, inferOriginalImageUrl } from "@/lib/image-url";
|
||||||
import { getDownloadFailure } from "./download-error";
|
import { getDownloadFailure } from "./download-error";
|
||||||
|
|
||||||
type DownloadFileItem = {
|
import {
|
||||||
url: string;
|
convertImageToPhotoshopPng,
|
||||||
name?: string;
|
normalizeCanvasDownloadAsset as normalizeAsset,
|
||||||
};
|
type CanvasDownloadAssetResult,
|
||||||
|
type DownloadFileItem,
|
||||||
|
} from "./photoshop-download-normalize";
|
||||||
|
|
||||||
export type CanvasDownloadAssetResult =
|
export { buildAttachmentContentDisposition, convertImageToPhotoshopPng, sanitizeCanvasDownloadFileName } from "./photoshop-download-normalize";
|
||||||
| {
|
export type { CanvasDownloadAssetResult } from "./photoshop-download-normalize";
|
||||||
ok: true;
|
|
||||||
filename: string;
|
|
||||||
bytes: Buffer;
|
|
||||||
contentType: string;
|
|
||||||
}
|
|
||||||
| {
|
|
||||||
ok: false;
|
|
||||||
url: string;
|
|
||||||
name?: string;
|
|
||||||
error: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
|
const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
|
||||||
const DEFAULT_FETCH_RETRY_COUNT = 3;
|
const DEFAULT_FETCH_RETRY_COUNT = 3;
|
||||||
@ -37,27 +28,6 @@ function normalizeDownloadError(error: unknown): string {
|
|||||||
return "download failed";
|
return "download failed";
|
||||||
}
|
}
|
||||||
|
|
||||||
export function sanitizeCanvasDownloadFileName(value: string, fallback: string): string {
|
|
||||||
const cleaned = String(value || "")
|
|
||||||
.replace(/\.[a-zA-Z0-9]{2,6}$/g, "")
|
|
||||||
.replace(/[\\/:*?"<>|]+/g, "-")
|
|
||||||
.replace(/\s+/g, "-")
|
|
||||||
.replace(/-+/g, "-")
|
|
||||||
.replace(/^-+|-+$/g, "")
|
|
||||||
.trim()
|
|
||||||
.slice(0, 120);
|
|
||||||
return cleaned || fallback;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function buildAttachmentContentDisposition(filename: string): string {
|
|
||||||
const safeAsciiName =
|
|
||||||
filename
|
|
||||||
.replace(/[^\x20-\x7E]+/g, "")
|
|
||||||
.replace(/"/g, "")
|
|
||||||
.trim() || `canvas-image-${Date.now()}.png`;
|
|
||||||
return `attachment; filename="${safeAsciiName}"; filename*=UTF-8''${encodeURIComponent(filename)}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildRemoteFetchHeaders(fileUrl: string, requestOrigin: string, cookie: string): HeadersInit {
|
function buildRemoteFetchHeaders(fileUrl: string, requestOrigin: string, cookie: string): HeadersInit {
|
||||||
const headers: Record<string, string> = {
|
const headers: Record<string, string> = {
|
||||||
Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,video/*,*/*;q=0.8",
|
Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,video/*,*/*;q=0.8",
|
||||||
@ -91,7 +61,7 @@ function resolveHttpUrl(value: string, requestOrigin: string): string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildCanvasDownloadSourceCandidates(fileUrl: string, requestOrigin: string) {
|
export function buildCanvasDownloadSourceCandidates(fileUrl: string, requestOrigin: string) {
|
||||||
const candidates: string[] = [];
|
const candidates: string[] = [];
|
||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
|
|
||||||
@ -117,180 +87,14 @@ function buildCanvasDownloadSourceCandidates(fileUrl: string, requestOrigin: str
|
|||||||
return candidates;
|
return candidates;
|
||||||
}
|
}
|
||||||
|
|
||||||
function getContentTypeMime(contentType: string): string {
|
export async function normalizeCanvasDownloadAsset(params: Parameters<typeof normalizeAsset>[0]) {
|
||||||
return String(contentType || "")
|
return normalizeAsset(params, async bytes => {
|
||||||
.split(";")[0]
|
const digest = createHash("sha256").update(bytes).digest("hex");
|
||||||
?.trim()
|
const { buffer } = await onceImageBytes(`photoshop:${digest}`, async () => ({
|
||||||
.toLowerCase();
|
buffer: await convertImageToPhotoshopPng(bytes), contentType: "image/png",
|
||||||
}
|
|
||||||
|
|
||||||
function hasAsciiAt(buffer: Buffer, offset: number, value: string): boolean {
|
|
||||||
return buffer.subarray(offset, offset + value.length).toString("latin1") === value;
|
|
||||||
}
|
|
||||||
|
|
||||||
function looksLikeSvg(buffer: Buffer, contentType: string): boolean {
|
|
||||||
if (getContentTypeMime(contentType) === "image/svg+xml") return true;
|
|
||||||
const text = buffer.subarray(0, Math.min(buffer.length, 1024)).toString("utf8").trimStart();
|
|
||||||
return /^(?:<\?xml[^>]*>\s*)?<svg[\s>]/i.test(text);
|
|
||||||
}
|
|
||||||
|
|
||||||
function looksLikeErrorDocument(buffer: Buffer, contentType: string): boolean {
|
|
||||||
const mime = getContentTypeMime(contentType);
|
|
||||||
if (
|
|
||||||
mime.startsWith("text/") ||
|
|
||||||
mime === "application/json" ||
|
|
||||||
mime === "application/xml" ||
|
|
||||||
mime === "application/xhtml+xml"
|
|
||||||
) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
const text = buffer.subarray(0, Math.min(buffer.length, 512)).toString("utf8").trimStart();
|
|
||||||
return /^(?:<!doctype\s+html|<html[\s>]|<\?xml|<error[\s>]|[{[])/i.test(text);
|
|
||||||
}
|
|
||||||
|
|
||||||
function detectAsset(bytes: Buffer, contentType: string): { kind: "image" | "video" | "audio"; mimeType: string; ext: string } | null {
|
|
||||||
if (bytes.length < 4) return null;
|
|
||||||
|
|
||||||
if (bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {
|
|
||||||
return { kind: "image", mimeType: "image/png", ext: "png" };
|
|
||||||
}
|
|
||||||
if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
|
|
||||||
return { kind: "image", mimeType: "image/jpeg", ext: "jpg" };
|
|
||||||
}
|
|
||||||
if (hasAsciiAt(bytes, 0, "RIFF") && hasAsciiAt(bytes, 8, "WEBP")) {
|
|
||||||
return { kind: "image", mimeType: "image/webp", ext: "webp" };
|
|
||||||
}
|
|
||||||
if (hasAsciiAt(bytes, 0, "GIF87a") || hasAsciiAt(bytes, 0, "GIF89a")) {
|
|
||||||
return { kind: "image", mimeType: "image/gif", ext: "gif" };
|
|
||||||
}
|
|
||||||
if (hasAsciiAt(bytes, 0, "BM")) {
|
|
||||||
return { kind: "image", mimeType: "image/bmp", ext: "bmp" };
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
bytes.subarray(0, 4).equals(Buffer.from([0x49, 0x49, 0x2a, 0x00])) ||
|
|
||||||
bytes.subarray(0, 4).equals(Buffer.from([0x4d, 0x4d, 0x00, 0x2a]))
|
|
||||||
) {
|
|
||||||
return { kind: "image", mimeType: "image/tiff", ext: "tiff" };
|
|
||||||
}
|
|
||||||
if (looksLikeSvg(bytes, contentType)) {
|
|
||||||
return { kind: "image", mimeType: "image/svg+xml", ext: "svg" };
|
|
||||||
}
|
|
||||||
if (hasAsciiAt(bytes, 0, "RIFF") && hasAsciiAt(bytes, 8, "WAVE")) return {kind:"audio",mimeType:"audio/wav",ext:"wav"};
|
|
||||||
if (hasAsciiAt(bytes, 0, "ID3") || (bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0 && (bytes[1] & 0x06) !== 0)) return {kind:"audio",mimeType:"audio/mpeg",ext:"mp3"};
|
|
||||||
if (hasAsciiAt(bytes, 0, "OggS")) return {kind:"audio",mimeType:"audio/ogg",ext:"ogg"};
|
|
||||||
if (hasAsciiAt(bytes, 0, "fLaC")) return {kind:"audio",mimeType:"audio/flac",ext:"flac"};
|
|
||||||
if (getContentTypeMime(contentType).startsWith("audio/")) return {kind:"audio",mimeType:getContentTypeMime(contentType),ext:getContentTypeMime(contentType).includes("mp4") ? "m4a" : getContentTypeMime(contentType).split("/")[1]};
|
|
||||||
if (hasAsciiAt(bytes, 4, "ftyp")) {
|
|
||||||
const brand = bytes.subarray(8, Math.min(bytes.length, 32)).toString("latin1").toLowerCase();
|
|
||||||
if (brand.includes("m4a")) return {kind:"audio",mimeType:"audio/mp4",ext:"m4a"};
|
|
||||||
if (brand.includes("avif")) return { kind: "image", mimeType: "image/avif", ext: "avif" };
|
|
||||||
if (brand.includes("heic") || brand.includes("heif") || brand.includes("mif1")) {
|
|
||||||
return { kind: "image", mimeType: "image/heic", ext: "heic" };
|
|
||||||
}
|
|
||||||
return { kind: "video", mimeType: "video/mp4", ext: "mp4" };
|
|
||||||
}
|
|
||||||
if (bytes.subarray(0, 4).equals(Buffer.from([0x1a, 0x45, 0xdf, 0xa3]))) {
|
|
||||||
return { kind: "video", mimeType: "video/webm", ext: "webm" };
|
|
||||||
}
|
|
||||||
|
|
||||||
const mime = getContentTypeMime(contentType);
|
|
||||||
if (mime.startsWith("image/")) {
|
|
||||||
const ext = mime.includes("jpeg") ? "jpg" : mime.split("/")[1] || "png";
|
|
||||||
return { kind: "image", mimeType: mime, ext };
|
|
||||||
}
|
|
||||||
if (mime.startsWith("video/")) {
|
|
||||||
const ext = mime.includes("quicktime") ? "mov" : mime.split("/")[1] || "mp4";
|
|
||||||
return { kind: "video", mimeType: mime, ext };
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function convertImageToPhotoshopPng(bytes: Buffer): Promise<Buffer> {
|
|
||||||
const sharp = await import("sharp");
|
|
||||||
const image = sharp.default(bytes, {
|
|
||||||
animated: false,
|
|
||||||
failOn: "none",
|
|
||||||
limitInputPixels: false,
|
|
||||||
});
|
|
||||||
const metadata = await image.metadata();
|
|
||||||
if (!metadata.width || !metadata.height) {
|
|
||||||
throw new Error("无法解析图片尺寸");
|
|
||||||
}
|
|
||||||
// RGB/RGBA, 8-bit, single-frame PNG is already compatible; preserve its exact bytes.
|
|
||||||
const rgb = (metadata.space === "srgb" && metadata.depth === "uchar")
|
|
||||||
|| (metadata.space === "rgb16" && metadata.depth === "ushort");
|
|
||||||
if (metadata.format === "png" && rgb
|
|
||||||
&& !metadata.isPalette && (metadata.pages || 1) === 1 && (!metadata.orientation || metadata.orientation === 1)) return bytes;
|
|
||||||
const output = image.rotate().toColorspace("srgb");
|
|
||||||
if (metadata.icc) output.withIccProfile("srgb");
|
|
||||||
return output.png({ compressionLevel: 3, adaptiveFiltering: true }).toBuffer();
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildCanvasAssetFilename(params: {
|
|
||||||
item: DownloadFileItem;
|
|
||||||
index: number;
|
|
||||||
ext: string;
|
|
||||||
includeSequencePrefix: boolean;
|
|
||||||
}) {
|
|
||||||
const baseName = sanitizeCanvasDownloadFileName(
|
|
||||||
params.item.name || `canvas-image-${params.index + 1}`,
|
|
||||||
`canvas-image-${params.index + 1}`
|
|
||||||
);
|
|
||||||
if (!params.includeSequencePrefix) {
|
|
||||||
return `${baseName}.${params.ext}`;
|
|
||||||
}
|
|
||||||
const sequencePrefix = String(params.index + 1).padStart(3, "0");
|
|
||||||
return `${sequencePrefix}-${baseName}.${params.ext}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function normalizeCanvasDownloadAsset(params: {
|
|
||||||
item: DownloadFileItem;
|
|
||||||
index: number;
|
|
||||||
bytes: Buffer;
|
|
||||||
contentType: string;
|
|
||||||
includeSequencePrefix: boolean;
|
|
||||||
}): Promise<CanvasDownloadAssetResult> {
|
|
||||||
if (params.bytes.length === 0) {
|
|
||||||
throw new Error("empty response");
|
|
||||||
}
|
|
||||||
if (looksLikeErrorDocument(params.bytes, params.contentType) && !looksLikeSvg(params.bytes, params.contentType)) {
|
|
||||||
throw new Error("源地址返回的不是图片文件");
|
|
||||||
}
|
|
||||||
|
|
||||||
const detected = detectAsset(params.bytes, params.contentType);
|
|
||||||
if (!detected) {
|
|
||||||
throw new Error("无法识别图片格式");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (detected.kind !== "image") {
|
|
||||||
return {
|
|
||||||
ok: true,
|
|
||||||
filename: buildCanvasAssetFilename({
|
|
||||||
item: params.item,
|
|
||||||
index: params.index,
|
|
||||||
ext: detected.ext,
|
|
||||||
includeSequencePrefix: params.includeSequencePrefix,
|
|
||||||
}),
|
|
||||||
bytes: params.bytes,
|
|
||||||
contentType: detected.mimeType,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const digest = createHash("sha256").update(params.bytes).digest("hex");
|
|
||||||
const { buffer: pngBytes } = await onceImageBytes(`photoshop:${digest}`, async () => ({
|
|
||||||
buffer: await convertImageToPhotoshopPng(params.bytes), contentType: "image/png",
|
|
||||||
}));
|
}));
|
||||||
return {
|
return buffer;
|
||||||
ok: true,
|
});
|
||||||
filename: buildCanvasAssetFilename({
|
|
||||||
item: params.item,
|
|
||||||
index: params.index,
|
|
||||||
ext: "png",
|
|
||||||
includeSequencePrefix: params.includeSequencePrefix,
|
|
||||||
}),
|
|
||||||
bytes: pngBytes,
|
|
||||||
contentType: "image/png",
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function fetchCanvasDownloadAsset(params: {
|
export async function fetchCanvasDownloadAsset(params: {
|
||||||
|
|||||||
103
lib/canvas/remote-zip-client.ts
Normal file
103
lib/canvas/remote-zip-client.ts
Normal file
@ -0,0 +1,103 @@
|
|||||||
|
import "server-only";
|
||||||
|
import { createHmac } from "node:crypto";
|
||||||
|
import { getDefaultStorageConfig, getStorageConfigById } from "@/lib/db/queries";
|
||||||
|
import { getArchivedImageUrlMappings } from "@/lib/storage/image-archive-db";
|
||||||
|
import { buildS3PublicUrl, toDbS3UploadConfig } from "@/lib/storage/service";
|
||||||
|
import { buildCanvasDownloadSourceCandidates } from "./photoshop-safe-download";
|
||||||
|
import { saveZipTask, type ZipTask } from "./download-zip-task-store";
|
||||||
|
import { zipStorageKey, zipWorkerSnapshot, type ZipWorkerJob } from "./remote-zip-contract";
|
||||||
|
|
||||||
|
async function resolveSources(task: ZipTask, origin: string) {
|
||||||
|
const candidates = task.files.map(file => buildCanvasDownloadSourceCandidates(file.url, origin));
|
||||||
|
const aliases = await getArchivedImageUrlMappings(candidates.flat());
|
||||||
|
const app = new URL(origin).hostname;
|
||||||
|
const isDirect = (value: string) => {
|
||||||
|
try {
|
||||||
|
const parsed = new URL(value);
|
||||||
|
return ["http:", "https:"].includes(parsed.protocol)
|
||||||
|
&& parsed.hostname !== app && parsed.hostname !== "ai.yydd.ai" && !parsed.username && !parsed.password;
|
||||||
|
} catch { return false; }
|
||||||
|
};
|
||||||
|
return candidates.map(values => [...new Set([
|
||||||
|
...values.flatMap(value => aliases[value] ? [aliases[value]] : []), ...values,
|
||||||
|
])].filter(isDirect).slice(0, 8));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateRemoteCanvasZip(task: ZipTask, origin: string) {
|
||||||
|
const remote = task.remote!;
|
||||||
|
const url = process.env.CANVAS_ZIP_WORKER_URL;
|
||||||
|
const token = process.env.CANVAS_ZIP_WORKER_TOKEN;
|
||||||
|
if (!url || !token) throw new Error("独立打包服务暂时不可用,请稍后重试");
|
||||||
|
const raw = remote.storageConfigId
|
||||||
|
? await getStorageConfigById({ id: remote.storageConfigId })
|
||||||
|
: await getDefaultStorageConfig();
|
||||||
|
const config = toDbS3UploadConfig(raw);
|
||||||
|
if (!raw?.isEnabled || config?.provider !== "qiniu-kodo" || config.region !== "cn-east-1" || !config.publicUrl?.startsWith("https://")) {
|
||||||
|
throw new Error("七牛打包存储暂时不可用,请稍后重试");
|
||||||
|
}
|
||||||
|
const key = zipStorageKey(task.id);
|
||||||
|
if (!remote.sources) {
|
||||||
|
remote.storageConfigId = raw.id;
|
||||||
|
remote.sources = await resolveSources(task, origin);
|
||||||
|
task.cdnObject = { storageConfigId: raw.id, key };
|
||||||
|
await saveZipTask(task);
|
||||||
|
}
|
||||||
|
const publicUrl = buildS3PublicUrl(config, key);
|
||||||
|
const policy = Buffer.from(JSON.stringify({
|
||||||
|
scope: config.bucket + ":" + key,
|
||||||
|
deadline: Math.floor(task.expiresAt / 1000),
|
||||||
|
insertOnly: 1,
|
||||||
|
returnBody: '{"key":"$(key)","fsize":$(fsize)}',
|
||||||
|
})).toString("base64").replace(/\+/g, "-").replace(/\//g, "_");
|
||||||
|
const mac = createHmac("sha1", config.secretAccessKey).update(policy).digest("base64")
|
||||||
|
.replace(/\+/g, "-").replace(/\//g, "_");
|
||||||
|
const job: ZipWorkerJob = {
|
||||||
|
attempt: remote.attempt,
|
||||||
|
appOrigin: origin,
|
||||||
|
folderName: task.folderName,
|
||||||
|
expiresAt: task.expiresAt,
|
||||||
|
files: task.files.map((file, i) => ({ name: file.name?.slice(0, 200), sources: remote.sources![i] })),
|
||||||
|
storage: { bucket: config.bucket, token: config.accessKeyId + ":" + mac + ":" + policy, publicUrl },
|
||||||
|
};
|
||||||
|
const response = await fetch(new URL("/v1/jobs/" + task.id, url), {
|
||||||
|
method: "PUT",
|
||||||
|
body: JSON.stringify(job),
|
||||||
|
headers: { Authorization: "Bearer " + token, "Content-Type": "application/json" },
|
||||||
|
cache: "no-store",
|
||||||
|
redirect: "error",
|
||||||
|
signal: AbortSignal.timeout(8_000),
|
||||||
|
}).catch(() => null);
|
||||||
|
if (!response?.ok) {
|
||||||
|
if (response?.status === 503) {
|
||||||
|
task.stage = "等待宿主机打包";
|
||||||
|
await saveZipTask(task);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
throw new Error("独立打包服务暂时不可达,请稍后重试,图片不会重新生成");
|
||||||
|
}
|
||||||
|
const result = zipWorkerSnapshot.parse(await response.json());
|
||||||
|
const { status, bytes, ...progress } = result;
|
||||||
|
Object.assign(task, progress, { updatedAt: Date.now() });
|
||||||
|
if (status !== "completed") {
|
||||||
|
task.status = status;
|
||||||
|
await saveZipTask(task);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!bytes) throw new Error("宿主机未返回有效的压缩包信息");
|
||||||
|
const target = new URL(publicUrl);
|
||||||
|
target.searchParams.set("v", String(bytes));
|
||||||
|
target.searchParams.set("attname", task.folderName + ".zip");
|
||||||
|
const verified = await fetch(target, {
|
||||||
|
method: "HEAD", cache: "no-store", redirect: "error", signal: AbortSignal.timeout(10_000),
|
||||||
|
}).catch(() => null);
|
||||||
|
if (!verified?.ok || Number(verified.headers.get("content-length")) !== bytes) {
|
||||||
|
task.status = "running";
|
||||||
|
task.stage = "正在确认七牛下载链接";
|
||||||
|
await saveZipTask(task);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
task.cdnUrl = target.href;
|
||||||
|
task.status = "completed";
|
||||||
|
task.expiresAt = Date.now() + 24 * 60 * 60_000;
|
||||||
|
await saveZipTask(task);
|
||||||
|
}
|
||||||
35
lib/canvas/remote-zip-contract.ts
Normal file
35
lib/canvas/remote-zip-contract.ts
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
import { z } from "zod";
|
||||||
|
|
||||||
|
export const zipWorkerJob = z.object({
|
||||||
|
attempt: z.number().int().nonnegative(),
|
||||||
|
folderName: z.string().min(1).max(100).refine(value => !value.includes("/") && !value.includes(String.fromCharCode(92)) && !/^\.+$/.test(value)),
|
||||||
|
expiresAt: z.number().int().positive(),
|
||||||
|
appOrigin: z.string().url(),
|
||||||
|
files: z.array(z.object({
|
||||||
|
name: z.string().max(200).optional(),
|
||||||
|
sources: z.array(z.string().url()).max(8),
|
||||||
|
})).min(1).max(200),
|
||||||
|
storage: z.object({
|
||||||
|
bucket: z.string().regex(/^[a-z0-9][a-z0-9-]{0,62}$/),
|
||||||
|
token: z.string().min(1).max(4096),
|
||||||
|
publicUrl: z.string().url(),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const zipWorkerSnapshot = z.object({
|
||||||
|
status: z.enum(["pending", "running", "completed", "failed"]),
|
||||||
|
stage: z.string().max(300),
|
||||||
|
error: z.string().max(500).optional(),
|
||||||
|
processedCount: z.number().int().nonnegative(),
|
||||||
|
successCount: z.number().int().nonnegative(),
|
||||||
|
failedCount: z.number().int().nonnegative(),
|
||||||
|
compressPercent: z.number().min(0).max(100),
|
||||||
|
bytes: z.number().int().positive().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type ZipWorkerJob = z.infer<typeof zipWorkerJob>;
|
||||||
|
export type ZipWorkerSnapshot = z.infer<typeof zipWorkerSnapshot>;
|
||||||
|
|
||||||
|
export function zipStorageKey(id: string) {
|
||||||
|
return `canvas-downloads/${id}.zip`;
|
||||||
|
}
|
||||||
@ -14,9 +14,13 @@ export function isPublicMediaIpv4(address:string) {
|
|||||||
!(p[0]===192&&(p[1]===168||p[1]===0)) && !(p[0]===100&&p[1]>=64&&p[1]<=127) &&
|
!(p[0]===192&&(p[1]===168||p[1]===0)) && !(p[0]===100&&p[1]>=64&&p[1]<=127) &&
|
||||||
!(p[0]===198&&(p[1]===18||p[1]===19));
|
!(p[0]===198&&(p[1]===18||p[1]===19));
|
||||||
}
|
}
|
||||||
export async function downloadCanvasMedia(url:string,output:string,redirects=0):Promise<void> {
|
export async function openPublicMedia(
|
||||||
|
url: string,
|
||||||
|
options: { signal?: AbortSignal; allowUrl?: (url: URL) => boolean } = {},
|
||||||
|
redirects = 0,
|
||||||
|
): Promise<http.IncomingMessage> {
|
||||||
const target=new URL(url);
|
const target=new URL(url);
|
||||||
if(!['http:','https:'].includes(target.protocol)||target.username||target.password||redirects>4) throw new Error('视频地址无效');
|
if(!['http:','https:'].includes(target.protocol)||target.username||target.password||redirects>4||options.allowUrl?.(target)===false) throw new Error('视频地址无效');
|
||||||
const addresses=await lookup(target.hostname,{family:4,all:true});
|
const addresses=await lookup(target.hostname,{family:4,all:true});
|
||||||
if(!addresses.length||addresses.some(v=>!isPublicMediaIpv4(v.address))) throw new Error('视频地址不可访问');
|
if(!addresses.length||addresses.some(v=>!isPublicMediaIpv4(v.address))) throw new Error('视频地址不可访问');
|
||||||
// Pin the validated address while preserving Host and TLS server name.
|
// Pin the validated address while preserving Host and TLS server name.
|
||||||
@ -24,14 +28,18 @@ export async function downloadCanvasMedia(url:string,output:string,redirects=0):
|
|||||||
const request=(target.protocol==='https:'?https:http).get({
|
const request=(target.protocol==='https:'?https:http).get({
|
||||||
protocol:target.protocol,hostname:addresses[0].address,port:target.port||undefined,
|
protocol:target.protocol,hostname:addresses[0].address,port:target.port||undefined,
|
||||||
path:target.pathname+target.search,headers:{Host:target.host},servername:target.hostname,
|
path:target.pathname+target.search,headers:{Host:target.host},servername:target.hostname,
|
||||||
signal:AbortSignal.timeout(90000),
|
signal:options.signal||AbortSignal.timeout(90000),
|
||||||
},resolve);
|
},resolve);
|
||||||
request.on('error',reject);
|
request.on('error',reject);
|
||||||
});
|
});
|
||||||
if([301,302,303,307,308].includes(response.statusCode||0)&&response.headers.location) {
|
if([301,302,303,307,308].includes(response.statusCode||0)&&response.headers.location) {
|
||||||
response.destroy();
|
response.destroy();
|
||||||
return downloadCanvasMedia(new URL(response.headers.location,target).href,output,redirects+1);
|
return openPublicMedia(new URL(response.headers.location,target).href,options,redirects+1);
|
||||||
}
|
}
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
export async function downloadCanvasMedia(url:string,output:string,redirects=0):Promise<void> {
|
||||||
|
const response=await openPublicMedia(url,{},redirects);
|
||||||
if(response.statusCode!==200||Number(response.headers['content-length']||0)>MAX_BYTES) {response.destroy();throw new Error('视频下载失败或超过 125 MiB 处理上限');}
|
if(response.statusCode!==200||Number(response.headers['content-length']||0)>MAX_BYTES) {response.destroy();throw new Error('视频下载失败或超过 125 MiB 处理上限');}
|
||||||
let bytes=0;
|
let bytes=0;
|
||||||
const limit=new Transform({transform(chunk,_encoding,callback){bytes+=chunk.length;callback(bytes>MAX_BYTES?new Error('视频超过 125 MiB 处理上限'):null,chunk);}});
|
const limit=new Transform({transform(chunk,_encoding,callback){bytes+=chunk.length;callback(bytes>MAX_BYTES?new Error('视频超过 125 MiB 处理上限'):null,chunk);}});
|
||||||
|
|||||||
20
services/canvas-zip/build.cjs
Normal file
20
services/canvas-zip/build.cjs
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
const { createRequire } = require("node:module");
|
||||||
|
const { resolve } = require("node:path");
|
||||||
|
const { build } = createRequire(require.resolve("tsx"))("esbuild");
|
||||||
|
|
||||||
|
build({
|
||||||
|
entryPoints: [resolve(__dirname, "server.ts")],
|
||||||
|
outfile: resolve(process.argv[2] || resolve(__dirname, "dist/server.cjs")),
|
||||||
|
bundle: true,
|
||||||
|
platform: "node",
|
||||||
|
target: "node20",
|
||||||
|
format: "cjs",
|
||||||
|
external: ["sharp"],
|
||||||
|
plugins: [{
|
||||||
|
name: "server-only-marker",
|
||||||
|
setup(build) {
|
||||||
|
build.onResolve({ filter: /^server-only$/ }, args => ({ path: args.path, namespace: "empty" }));
|
||||||
|
build.onLoad({ filter: /.*/, namespace: "empty" }, () => ({ contents: "" }));
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
}).catch(() => { process.exitCode = 1; });
|
||||||
29
services/canvas-zip/canvas-zip.service
Normal file
29
services/canvas-zip/canvas-zip.service
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=由壹点详情图 ZIP 打包服务
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=canvas-zip
|
||||||
|
Group=canvas-zip
|
||||||
|
WorkingDirectory=/opt/canvas-zip/current
|
||||||
|
EnvironmentFile=/etc/canvas-zip.env
|
||||||
|
Environment=CANVAS_DOWNLOAD_ZIP_DIR=/var/lib/canvas-zip
|
||||||
|
Environment=CANVAS_ZIP_PORT=18343
|
||||||
|
ExecStart=/usr/bin/node /opt/canvas-zip/current/server.cjs
|
||||||
|
StateDirectory=canvas-zip
|
||||||
|
StateDirectoryMode=0700
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=5
|
||||||
|
UMask=0077
|
||||||
|
NoNewPrivileges=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
PrivateTmp=true
|
||||||
|
MemoryMax=2G
|
||||||
|
CPUQuota=200%
|
||||||
|
TasksMax=128
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
257
services/canvas-zip/server.ts
Normal file
257
services/canvas-zip/server.ts
Normal file
@ -0,0 +1,257 @@
|
|||||||
|
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||||||
|
import { timingSafeEqual } from "node:crypto";
|
||||||
|
import { open, rm, stat } from "node:fs/promises";
|
||||||
|
import { MIMEType } from "node:util";
|
||||||
|
import { claimZipTask, cleanupZipTasks, loadZipTask, saveZipTask, type ZipTask } from "../../lib/canvas/download-zip-task-store";
|
||||||
|
import { prepareCanvasZip } from "../../lib/canvas/download-zip-prepare";
|
||||||
|
import { normalizeCanvasDownloadAsset } from "../../lib/canvas/photoshop-download-normalize";
|
||||||
|
import { zipStorageKey, zipWorkerJob, type ZipWorkerJob, type ZipWorkerSnapshot } from "../../lib/canvas/remote-zip-contract";
|
||||||
|
import { openPublicMedia } from "../../lib/video-canvas/download-media";
|
||||||
|
|
||||||
|
const TOKEN = process.env.CANVAS_ZIP_WORKER_TOKEN || "";
|
||||||
|
if (TOKEN.length < 32 || !process.env.CANVAS_DOWNLOAD_ZIP_DIR) {
|
||||||
|
throw new Error("Set CANVAS_ZIP_WORKER_TOKEN and CANVAS_DOWNLOAD_ZIP_DIR");
|
||||||
|
}
|
||||||
|
|
||||||
|
const OWNER = "zip-worker";
|
||||||
|
const IMAGE_LIMIT = 125 * 1024 ** 2;
|
||||||
|
const PART_BYTES = 8 * 1024 ** 2;
|
||||||
|
const queue = new Map<string, ZipWorkerJob>();
|
||||||
|
const accepting = new Set<string>();
|
||||||
|
let running: string | undefined;
|
||||||
|
|
||||||
|
function authorized(request: IncomingMessage) {
|
||||||
|
const actual = Buffer.from(request.headers.authorization || "");
|
||||||
|
const expected = Buffer.from("Bearer " + TOKEN);
|
||||||
|
return actual.length === expected.length && timingSafeEqual(actual, expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
function reply(response: ServerResponse, status: number, body: unknown) {
|
||||||
|
response.writeHead(status, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" });
|
||||||
|
response.end(JSON.stringify(body));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readJob(request: IncomingMessage) {
|
||||||
|
let bytes = 0;
|
||||||
|
const body: Buffer[] = [];
|
||||||
|
for await (const chunk of request) {
|
||||||
|
const value = Buffer.from(chunk);
|
||||||
|
bytes += value.length;
|
||||||
|
if (bytes > 1024 ** 2) throw new Error("Invalid job size");
|
||||||
|
body.push(value);
|
||||||
|
}
|
||||||
|
return zipWorkerJob.parse(JSON.parse(Buffer.concat(body).toString("utf8")));
|
||||||
|
}
|
||||||
|
|
||||||
|
function snapshot(task: ZipTask): ZipWorkerSnapshot {
|
||||||
|
const { status, stage, error, processedCount, successCount, failedCount, compressPercent } = task;
|
||||||
|
return { status, stage, error, processedCount, successCount, failedCount, compressPercent,
|
||||||
|
bytes: task.status === "completed" ? task.zipFile?.bytes : undefined };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchInput(job: ZipWorkerJob, index: number, signal: AbortSignal) {
|
||||||
|
const item = job.files[index];
|
||||||
|
const appHost = new URL(job.appOrigin).hostname;
|
||||||
|
const allowUrl = (url: URL) => url.hostname !== appHost && url.hostname !== "ai.yydd.ai";
|
||||||
|
for (const url of item.sources) {
|
||||||
|
if (signal.aborted) break;
|
||||||
|
try {
|
||||||
|
const response = await openPublicMedia(url, {
|
||||||
|
allowUrl,
|
||||||
|
signal: AbortSignal.any([signal, AbortSignal.timeout(30_000)]),
|
||||||
|
});
|
||||||
|
if (response.statusCode !== 200 || Number(response.headers["content-length"] || 0) > IMAGE_LIMIT) {
|
||||||
|
response.destroy();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const contentType = new MIMEType(response.headers["content-type"] || "application/octet-stream").essence;
|
||||||
|
const buffers: Buffer[] = [];
|
||||||
|
let bytes = 0;
|
||||||
|
for await (const packet of response) {
|
||||||
|
const chunk = Buffer.from(packet);
|
||||||
|
bytes += chunk.length;
|
||||||
|
if (bytes > IMAGE_LIMIT) throw new Error("Original exceeds the download limit");
|
||||||
|
buffers.push(chunk);
|
||||||
|
}
|
||||||
|
return await normalizeCanvasDownloadAsset({
|
||||||
|
item: { url, name: item.name }, index, bytes: Buffer.concat(buffers, bytes),
|
||||||
|
contentType, includeSequencePrefix: true,
|
||||||
|
});
|
||||||
|
} catch { /* Try the next resolved source. */ }
|
||||||
|
}
|
||||||
|
return { ok: false as const, url: "", name: item.name, error: "宿主机暂时无法读取这张原图" };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function uploadArchive(task: ZipTask, job: ZipWorkerJob) {
|
||||||
|
const { bucket, token, publicUrl } = job.storage;
|
||||||
|
const { path, bytes } = task.zipFile!;
|
||||||
|
const key = zipStorageKey(task.id);
|
||||||
|
const endpoint = "https://upload.qiniup.com/buckets/" + encodeURIComponent(bucket)
|
||||||
|
+ "/objects/" + encodeURIComponent(Buffer.from(key).toString("base64").replace(/\+/g, "-").replace(/\//g, "_")) + "/uploads";
|
||||||
|
const storedUrl = new URL(publicUrl);
|
||||||
|
const hasExactSize = async () => {
|
||||||
|
try {
|
||||||
|
const response = await fetch(storedUrl, { method: "HEAD", redirect: "error", signal: AbortSignal.timeout(8_000) });
|
||||||
|
return response.ok && Number(response.headers.get("content-length")) === bytes;
|
||||||
|
} catch { return false; }
|
||||||
|
};
|
||||||
|
if (await hasExactSize()) return;
|
||||||
|
const request = async (url: string, method: string, body?: Buffer | string, contentType?: string) => {
|
||||||
|
const headers: Record<string, string> = { Authorization: "UpToken " + token };
|
||||||
|
if (contentType) headers["Content-Type"] = contentType;
|
||||||
|
return await fetch(url, {
|
||||||
|
method, headers, body: Buffer.isBuffer(body) ? Uint8Array.from(body) : body,
|
||||||
|
redirect: "error", signal: AbortSignal.timeout(method === "PUT" ? 180_000 : 30_000),
|
||||||
|
});
|
||||||
|
};
|
||||||
|
const init = await request(endpoint, "POST");
|
||||||
|
if (!init.ok) throw new Error("七牛未能开始接收压缩包(" + init.status + ")");
|
||||||
|
const { uploadId } = await init.json() as { uploadId?: string };
|
||||||
|
if (!uploadId) throw new Error("七牛未返回上传编号");
|
||||||
|
const uploadUrl = endpoint + "/" + encodeURIComponent(uploadId);
|
||||||
|
const parts: Array<{ partNumber: number; etag: string }> = [];
|
||||||
|
try {
|
||||||
|
const handle = await open(path, "r");
|
||||||
|
try {
|
||||||
|
for (let offset = 0; offset < bytes; offset += PART_BYTES) {
|
||||||
|
const data = Buffer.allocUnsafe(Math.min(PART_BYTES, bytes - offset));
|
||||||
|
let read = 0;
|
||||||
|
while (read < data.length) {
|
||||||
|
const result = await handle.read(data, read, data.length - read, offset + read);
|
||||||
|
if (!result.bytesRead) throw new Error("压缩包读取不完整");
|
||||||
|
read += result.bytesRead;
|
||||||
|
}
|
||||||
|
const partNumber = parts.length + 1;
|
||||||
|
const response = await request(uploadUrl + "/" + partNumber, "PUT", data, "application/octet-stream");
|
||||||
|
if (!response.ok) throw new Error("七牛接收分片失败(" + response.status + ")");
|
||||||
|
const { etag } = await response.json() as { etag?: string };
|
||||||
|
if (!etag) throw new Error("七牛未确认上传分片");
|
||||||
|
parts.push({ partNumber, etag });
|
||||||
|
task.stage = "正在传送至七牛 " + Math.floor((offset + data.length) / bytes * 100) + "%";
|
||||||
|
task.updatedAt = Date.now();
|
||||||
|
await saveZipTask(task);
|
||||||
|
}
|
||||||
|
} finally { await handle.close(); }
|
||||||
|
const complete = await request(uploadUrl, "POST", JSON.stringify({
|
||||||
|
parts, fname: task.folderName + ".zip", mimeType: "application/zip",
|
||||||
|
}), "application/json");
|
||||||
|
if (!complete.ok) {
|
||||||
|
if (complete.status === 614 && await hasExactSize()) return;
|
||||||
|
throw new Error("七牛未确认压缩包(" + complete.status + ")");
|
||||||
|
}
|
||||||
|
const saved = await complete.json() as { key?: string; fsize?: number };
|
||||||
|
if (saved.key !== key || saved.fsize !== bytes) throw new Error("七牛返回的压缩包信息不一致");
|
||||||
|
} catch (error) {
|
||||||
|
await request(uploadUrl, "DELETE").then(response => response.body?.cancel()).catch(() => undefined);
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function run(id: string, job: ZipWorkerJob) {
|
||||||
|
const release = await claimZipTask(id);
|
||||||
|
if (!release) return;
|
||||||
|
let task: ZipTask | null = null;
|
||||||
|
try {
|
||||||
|
task = await loadZipTask(id, OWNER);
|
||||||
|
if (!task || task.status === "completed" || task.status === "failed") return;
|
||||||
|
task.status = "running";
|
||||||
|
task.stage = "宿主机正在准备原图";
|
||||||
|
await saveZipTask(task);
|
||||||
|
const present = task.zipFile && await stat(task.zipFile.path).catch(() => null);
|
||||||
|
if (!present || present.size !== task.zipFile!.bytes) {
|
||||||
|
await prepareCanvasZip(task, ({ index, signal }) => fetchInput(job, index, signal));
|
||||||
|
}
|
||||||
|
task.stage = "正在传送至七牛";
|
||||||
|
await saveZipTask(task);
|
||||||
|
await uploadArchive(task, job);
|
||||||
|
task.status = "completed";
|
||||||
|
task.stage = task.failedCount ? "部分原图未能读取" : "打包完成";
|
||||||
|
task.compressPercent = 100;
|
||||||
|
task.error = undefined;
|
||||||
|
task.updatedAt = Date.now();
|
||||||
|
task.expiresAt = Date.now() + 24 * 60 * 60_000;
|
||||||
|
await saveZipTask(task);
|
||||||
|
if (task.zipFile) await rm(task.zipFile.path, { force: true }).catch(() => undefined);
|
||||||
|
} catch (error) {
|
||||||
|
if (task) {
|
||||||
|
task.status = "failed";
|
||||||
|
task.stage = "打包未完成";
|
||||||
|
task.error = (error instanceof Error ? error.message : "宿主机打包失败").slice(0, 450);
|
||||||
|
task.updatedAt = Date.now();
|
||||||
|
await saveZipTask(task);
|
||||||
|
}
|
||||||
|
} finally { await release(); }
|
||||||
|
}
|
||||||
|
|
||||||
|
function drain() {
|
||||||
|
if (running) return;
|
||||||
|
const next = queue.entries().next().value;
|
||||||
|
if (!next) return;
|
||||||
|
const [id, job] = next;
|
||||||
|
queue.delete(id);
|
||||||
|
running = id;
|
||||||
|
void run(id, job).catch(() => console.error("Canvas ZIP job could not be saved")).finally(() => {
|
||||||
|
running = undefined;
|
||||||
|
drain();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export const router = createServer((request, response) => {
|
||||||
|
void (async () => {
|
||||||
|
if (!authorized(request)) { reply(response, 401, {}); return; }
|
||||||
|
if (request.method === "GET" && request.url === "/health") {
|
||||||
|
reply(response, 200, { ok: true, active: Number(Boolean(running)), waiting: queue.size });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const id = request.url?.match(/^\/v1\/jobs\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i)?.[1];
|
||||||
|
if (!id || request.method !== "PUT") { reply(response, 404, {}); return; }
|
||||||
|
if (accepting.has(id)) { reply(response, 503, {}); return; }
|
||||||
|
accepting.add(id);
|
||||||
|
try {
|
||||||
|
const job = await readJob(request);
|
||||||
|
const publicUrl = new URL(job.storage.publicUrl);
|
||||||
|
if (publicUrl.protocol !== "https:" || !publicUrl.pathname.endsWith("/" + zipStorageKey(id))) {
|
||||||
|
reply(response, 400, {}); return;
|
||||||
|
}
|
||||||
|
let task = await loadZipTask(id, OWNER);
|
||||||
|
if (running === id || queue.has(id)) {
|
||||||
|
if (task) reply(response, 200, snapshot(task)); else reply(response, 503, {});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (queue.size >= 8 && (!task || task.status === "pending" || task.status === "running")) {
|
||||||
|
reply(response, 503, {}); return;
|
||||||
|
}
|
||||||
|
if (task?.status === "failed" && job.attempt > (task.attempt ?? 0)) {
|
||||||
|
task.attempt = job.attempt;
|
||||||
|
task.status = "pending";
|
||||||
|
task.stage = "等待宿主机打包";
|
||||||
|
task.error = undefined;
|
||||||
|
task.expiresAt = job.expiresAt;
|
||||||
|
task.updatedAt = Date.now();
|
||||||
|
await saveZipTask(task);
|
||||||
|
}
|
||||||
|
if (!task) {
|
||||||
|
const now = Date.now();
|
||||||
|
task = {
|
||||||
|
id, userId: OWNER, folderName: job.folderName, attempt: job.attempt,
|
||||||
|
files: job.files.map(item => ({ url: item.sources[0] || "", name: item.name })),
|
||||||
|
totalCount: job.files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0,
|
||||||
|
status: "pending", stage: "等待宿主机打包", createdAt: now, updatedAt: now, expiresAt: job.expiresAt,
|
||||||
|
};
|
||||||
|
await saveZipTask(task);
|
||||||
|
}
|
||||||
|
if (task.status === "pending" || task.status === "running") {
|
||||||
|
queue.set(id, job);
|
||||||
|
drain();
|
||||||
|
}
|
||||||
|
reply(response, 200, snapshot(task));
|
||||||
|
void cleanupZipTasks().catch(() => undefined);
|
||||||
|
} finally { accepting.delete(id); }
|
||||||
|
})().catch(() => {
|
||||||
|
if (!response.headersSent) reply(response, 400, { error: "无效的打包请求" });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
if (require.main === module) {
|
||||||
|
router.listen(Number(process.env.CANVAS_ZIP_PORT || 18343), "127.0.0.1");
|
||||||
|
}
|
||||||
167
tests/routes/canvas-zip-worker.test.ts
Normal file
167
tests/routes/canvas-zip-worker.test.ts
Normal file
@ -0,0 +1,167 @@
|
|||||||
|
import { test, expect } from "@playwright/test";
|
||||||
|
import JSZip from "jszip";
|
||||||
|
import { NextRequest, NextResponse } from "next/server";
|
||||||
|
import sharp from "sharp";
|
||||||
|
import { once } from "node:events";
|
||||||
|
import { Readable } from "node:stream";
|
||||||
|
import { mkdtemp, rm } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
import { serverLoader } from "../helpers/canvas-download-server-loader";
|
||||||
|
|
||||||
|
test("the worker prepares 13 images and uploads the ZIP through Qiniu multipart", async () => {
|
||||||
|
const directory = await mkdtemp(join(tmpdir(), "zip-worker-test-"));
|
||||||
|
const token = "local-test-secret".repeat(3);
|
||||||
|
const id = randomUUID();
|
||||||
|
const key = "canvas-downloads/" + id + ".zip";
|
||||||
|
const rgb = { r: 25, g: 100, b: 170 };
|
||||||
|
const png = await sharp({ create: { width: 800, height: 800, channels: 3, background: rgb } }).png().toBuffer();
|
||||||
|
const jpeg = await sharp({ create: { width: 800, height: 800, channels: 3, background: rgb } }).jpeg().toBuffer();
|
||||||
|
const parts: Buffer[] = [];
|
||||||
|
let complete = 0, downloads = 0;
|
||||||
|
const loader = serverLoader({
|
||||||
|
"../../lib/video-canvas/download-media": {
|
||||||
|
openPublicMedia: async (url: string, options: { allowUrl: (url: URL) => boolean }) => {
|
||||||
|
const parsed = new URL(url);
|
||||||
|
expect(options.allowUrl(parsed)).toBe(true);
|
||||||
|
expect(options.allowUrl(new URL("https://ai.yydd.ai/api/image-proxy"))).toBe(false);
|
||||||
|
downloads++;
|
||||||
|
const original = parsed.pathname.endsWith("original.jpeg") ? jpeg : png;
|
||||||
|
return Object.assign(Readable.from([original]), {
|
||||||
|
statusCode: 200, headers: { "content-type": original === png ? "image/png" : "image/jpeg" },
|
||||||
|
});
|
||||||
|
},
|
||||||
|
},
|
||||||
|
fetch: async (uri: URL | string, init: RequestInit) => {
|
||||||
|
const url = new URL(uri);
|
||||||
|
if (init.method === "HEAD") return new Response(null, { status: 404 });
|
||||||
|
expect(url.hostname).toBe("upload.qiniup.com");
|
||||||
|
expect(new Headers(init.headers).get("Authorization")).toBe("UpToken scoped-test-token");
|
||||||
|
if (url.pathname.endsWith("/uploads")) return Response.json({ uploadId: "one-upload" });
|
||||||
|
if (init.method === "PUT") {
|
||||||
|
parts.push(Buffer.from(init.body as Uint8Array));
|
||||||
|
return Response.json({ etag: "one-part" });
|
||||||
|
}
|
||||||
|
expect(init.method).toBe("POST");
|
||||||
|
expect(JSON.parse(init.body as string).mimeType).toBe("application/zip");
|
||||||
|
complete++;
|
||||||
|
return Response.json({ key, fsize: Buffer.concat(parts).length });
|
||||||
|
},
|
||||||
|
}, { CANVAS_DOWNLOAD_ZIP_DIR: directory, CANVAS_ZIP_WORKER_TOKEN: token });
|
||||||
|
const { router } = loader("services/canvas-zip/server.ts");
|
||||||
|
router.listen(0, "127.0.0.1");
|
||||||
|
await once(router, "listening");
|
||||||
|
const endpoint = "http://127.0.0.1:" + router.address().port + "/v1/jobs/" + id;
|
||||||
|
const files = Array.from({ length: 13 }, (_, i) => ({
|
||||||
|
name: "详情图-" + i,
|
||||||
|
sources: ["https://cdn.example/" + i + (i === 12 ? "/original.jpeg" : "/original.png")],
|
||||||
|
}));
|
||||||
|
const job = {
|
||||||
|
attempt: 0, folderName: "详情图", expiresAt: Date.now() + 60_000, appOrigin: "https://ai.yydd.ai", files,
|
||||||
|
storage: { bucket: "qa-bucket", token: "scoped-test-token", publicUrl: "https://cdn.example/" + key },
|
||||||
|
};
|
||||||
|
const post = async () => fetch(endpoint, {
|
||||||
|
method: "PUT", headers: { Authorization: "Bearer " + token, "content-type": "application/json" }, body: JSON.stringify(job),
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
expect((await fetch(endpoint, { method: "PUT" })).status).toBe(401);
|
||||||
|
expect((await post()).status).toBe(200);
|
||||||
|
await expect.poll(async () => {
|
||||||
|
const response = await post();
|
||||||
|
const state = await response.json();
|
||||||
|
return state.status;
|
||||||
|
}, { timeout: 10_000 }).toBe("completed");
|
||||||
|
await expect((await post()).json()).resolves.toMatchObject({ successCount: 13, failedCount: 0 });
|
||||||
|
expect(downloads).toBe(13);
|
||||||
|
expect(complete).toBe(1);
|
||||||
|
const archive = await JSZip.loadAsync(Buffer.concat(parts), { checkCRC32: true });
|
||||||
|
const images = Object.values(archive.files).filter(file => !file.dir);
|
||||||
|
expect(images).toHaveLength(13);
|
||||||
|
for (const [i, entry] of images.entries()) {
|
||||||
|
const data = await entry.async("nodebuffer");
|
||||||
|
expect(entry.name).toMatch(/\.png$/);
|
||||||
|
if (i < 12) expect(data).toEqual(png);
|
||||||
|
else {
|
||||||
|
const metadata = await sharp(data).metadata();
|
||||||
|
expect(metadata).toMatchObject({ width: 800, height: 800, space: "srgb", format: "png" });
|
||||||
|
const color = (await sharp(data).extract({ left: 20, top: 20, width: 1, height: 1 }).raw().toBuffer());
|
||||||
|
expect(Math.abs(color[0] - rgb.r)).toBeLessThan(4);
|
||||||
|
expect(Math.abs(color[1] - rgb.g)).toBeLessThan(4);
|
||||||
|
expect(Math.abs(color[2] - rgb.b)).toBeLessThan(4);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await new Promise<void>(resolve => router.close(() => resolve()));
|
||||||
|
await rm(directory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
test("the style-detail route gives the worker addresses and returns a CDN redirect even when local is requested", async () => {
|
||||||
|
const directory = await mkdtemp(join(tmpdir(), "zip-remote-route-"));
|
||||||
|
const id = randomUUID();
|
||||||
|
const callbacks: Array<() => Promise<void>> = [];
|
||||||
|
let owner = "owner";
|
||||||
|
const zipUrl = "https://cdn.example/canvas-downloads/" + id + ".zip";
|
||||||
|
const raw = { id: "storage", isEnabled: true };
|
||||||
|
const worker = "http://127.0.0.1:18343";
|
||||||
|
const config = { provider: "qiniu-kodo", region: "cn-east-1", bucket: "qa", publicUrl: "https://cdn.example",
|
||||||
|
accessKeyId: "fake-public-id", secretAccessKey: "fake-private-key" };
|
||||||
|
let workerCalls = 0;
|
||||||
|
const route = serverLoader({
|
||||||
|
"@/lib/observability/active-work": { trackRuntimeWork: async (fn: () => Promise<void>) => fn() },
|
||||||
|
"@/app/(auth)/auth": { auth: async () => ({ user: { id: owner } }) },
|
||||||
|
"next/server": { NextResponse, after: (fn: () => Promise<void>) => callbacks.push(fn) },
|
||||||
|
"@/lib/canvas/photoshop-safe-download": {
|
||||||
|
fetchCanvasDownloadAsset: () => { throw new Error("The production server must not fetch the originals"); },
|
||||||
|
},
|
||||||
|
"@/lib/storage/image-archive-db": { getArchivedImageUrlMappings: async () => ({
|
||||||
|
"https://ai.yydd.ai/archive": "https://cdn.example/archive.png",
|
||||||
|
}) },
|
||||||
|
"@/lib/db/queries": { getDefaultStorageConfig: async () => raw, getStorageConfigById: async () => raw },
|
||||||
|
"@/lib/storage/service": { toDbS3UploadConfig: () => config, buildS3PublicUrl: () => zipUrl },
|
||||||
|
"@/lib/storage/canvas-zip-delivery": {
|
||||||
|
publishCanvasZip: () => { throw new Error("The production server must not upload the ZIP"); },
|
||||||
|
},
|
||||||
|
fetch: async (uri: URL | string, init: RequestInit) => {
|
||||||
|
const url = new URL(uri);
|
||||||
|
if (url.origin === worker) {
|
||||||
|
workerCalls++;
|
||||||
|
const job = JSON.parse(init.body as string);
|
||||||
|
expect(job.files[0].sources).toEqual(["https://cdn.example/archive.png"]);
|
||||||
|
expect(job.files[1].sources).toEqual(["https://provider.example/result.png"]);
|
||||||
|
expect(job).not.toHaveProperty("cookie");
|
||||||
|
expect(job.storage).not.toHaveProperty("secretAccessKey");
|
||||||
|
const policy = JSON.parse(Buffer.from(job.storage.token.split(":")[2], "base64url").toString());
|
||||||
|
expect(policy.scope).toBe("qa:canvas-downloads/" + id + ".zip");
|
||||||
|
return Response.json({ status: "completed", stage: "打包完成", processedCount: 2,
|
||||||
|
successCount: 2, failedCount: 0, compressPercent: 100, bytes: 12345 });
|
||||||
|
}
|
||||||
|
expect(url.href).toContain(zipUrl);
|
||||||
|
expect(init.method).toBe("HEAD");
|
||||||
|
return new Response(null, { headers: { "content-length": "12345" } });
|
||||||
|
},
|
||||||
|
}, { CANVAS_DOWNLOAD_ZIP_DIR: directory, CANVAS_ZIP_WORKER_URL: worker,
|
||||||
|
CANVAS_ZIP_WORKER_TOKEN: "local-test-secret".repeat(3) })("app/api/canvas/download-zip-task/route.ts");
|
||||||
|
const endpoint = "https://ai.yydd.ai/api/canvas/download-zip-task";
|
||||||
|
try {
|
||||||
|
const started = await route.POST(new NextRequest(endpoint, {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({ taskId: id, scope: "style-detail", folderName: "详情图", files: [
|
||||||
|
{ url: "https://ai.yydd.ai/archive" }, { url: "https://provider.example/result.png" },
|
||||||
|
] }),
|
||||||
|
headers: { cookie: "auth=private-test-cookie" },
|
||||||
|
}));
|
||||||
|
expect(started.status).toBe(202);
|
||||||
|
while (callbacks.length) await callbacks.shift()!();
|
||||||
|
expect(workerCalls).toBe(1);
|
||||||
|
const download = await route.GET(new NextRequest(endpoint + "?taskId=" + id + "&download=1&local=1"));
|
||||||
|
expect(download.status).toBe(307);
|
||||||
|
expect(download.headers.get("location")).toContain(zipUrl);
|
||||||
|
owner = "another-user";
|
||||||
|
const denied = await route.GET(new NextRequest(endpoint + "?taskId=" + id + "&download=1"));
|
||||||
|
expect(denied.status).toBe(404);
|
||||||
|
} finally { await rm(directory, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
Loading…
x
Reference in New Issue
Block a user