Merge pull request '画布多选下载由七牛云直接打包' (#115) from codex/canvas-qiniu-cloud-zip into main

This commit is contained in:
WuYongJie 2026-11-21 14:52:00 +08:00
commit 6c7bebe9d3
7 changed files with 345 additions and 7 deletions

View File

@ -93,6 +93,11 @@ async function processTask(id: string, userId: string, requestOrigin: string, co
await updateRemoteCanvasZip(task, requestOrigin); await updateRemoteCanvasZip(task, requestOrigin);
return; return;
} }
if (task.preferQiniu || task.qiniu) {
const { processQiniuCanvasZip } = await import("@/lib/canvas/qiniu-canvas-zip");
if (await processQiniuCanvasZip(task, requestOrigin)) return;
delete task.preferQiniu;
}
await assertZipDiskSpace(); await assertZipDiskSpace();
const existingZip = task.zipFile && task.contentDisposition && await stat(task.zipFile.path).catch(() => null); const existingZip = task.zipFile && task.contentDisposition && await stat(task.zipFile.path).catch(() => null);
const reuseZip = existingZip && existingZip.size === task.zipFile!.bytes; const reuseZip = existingZip && existingZip.size === task.zipFile!.bytes;
@ -158,6 +163,7 @@ export async function POST(request: NextRequest) {
process.env.CANVAS_ZIP_WORKER_URL && process.env.CANVAS_ZIP_WORKER_TOKEN process.env.CANVAS_ZIP_WORKER_URL && process.env.CANVAS_ZIP_WORKER_TOKEN
? { attempt: 0 } : undefined; ? { attempt: 0 } : undefined;
task = { id, userId: session.user.id, folderName: sanitizeFolderName(String(body.folderName || "")), files, remote, task = { id, userId: session.user.id, folderName: sanitizeFolderName(String(body.folderName || "")), files, remote,
preferQiniu: body.scope === "canvas-selection" || undefined,
totalCount: files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0, totalCount: files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0,
status: "pending", stage: "等待后台准备", createdAt: now, updatedAt: now, expiresAt: now + PREPARATION_TTL_MS }; status: "pending", stage: "等待后台准备", createdAt: now, updatedAt: now, expiresAt: now + PREPARATION_TTL_MS };
await saveZipTask(task); await saveZipTask(task);
@ -168,6 +174,7 @@ export async function POST(request: NextRequest) {
task.remote.attempt++; task.remote.attempt++;
delete task.remote.sources; delete task.remote.sources;
} }
if (task.qiniu?.confirmedFailed) delete task.qiniu;
if (!task.zipFile) { if (!task.zipFile) {
for (const file of task.files) delete file.error; for (const file of task.files) delete file.error;
task.processedCount = task.successCount = task.files.filter(file => file.prepared).length; task.processedCount = task.successCount = task.files.filter(file => file.prepared).length;
@ -195,7 +202,7 @@ export async function GET(request: NextRequest) {
const task = await loadZipTask(id, session.user.id); const task = await loadZipTask(id, session.user.id);
if (!task) return NextResponse.json({ error: "打包任务不存在或已过期,请重新打包" }, { status: 404 }); if (!task) return NextResponse.json({ error: "打包任务不存在或已过期,请重新打包" }, { status: 404 });
if (request.nextUrl.searchParams.get("download") === "1") { if (request.nextUrl.searchParams.get("download") === "1") {
if (task.remote && task.status === "completed" && task.cdnUrl) if ((task.remote || task.qiniu) && task.status === "completed" && task.cdnUrl)
return NextResponse.redirect(task.cdnUrl, 307); return NextResponse.redirect(task.cdnUrl, 307);
if (task.status !== "completed" || !task.zipFile) return NextResponse.json({ error: "ZIP 尚未准备好" }, { status: 409 }); if (task.status !== "completed" || !task.zipFile) return NextResponse.json({ error: "ZIP 尚未准备好" }, { status: 409 });
if (task.cdnUrl && request.nextUrl.searchParams.get("local") !== "1") if (task.cdnUrl && request.nextUrl.searchParams.get("local") !== "1")

View File

@ -1937,6 +1937,7 @@ export function CanvasWorkspace({
const toastId = toast.loading("正在准备所选图片..."); const toastId = toast.loading("正在准备所选图片...");
try { try {
const result = await downloadCanvasSelectionZip({ folderName, files, const result = await downloadCanvasSelectionZip({ folderName, files,
scope: "canvas-selection",
selectionKey: `${useCanvasStore.getState().projectId}:${selectedImageNodes.map(node => node.id).sort().join(",")}`, selectionKey: `${useCanvasStore.getState().projectId}:${selectedImageNodes.map(node => node.id).sort().join(",")}`,
onProgress: task => toast.loading(`${task.stage}(${task.progress}%)`, { id: toastId }), onProgress: task => toast.loading(`${task.stage}(${task.progress}%)`, { id: toastId }),
}); });

View File

@ -11,8 +11,15 @@ export type ZipTask = {
compressPercent: number; status: "pending" | "running" | "completed" | "failed"; compressPercent: number; status: "pending" | "running" | "completed" | "failed";
stage: string; error?: string; zipFile?: { path: string; bytes: number }; stage: string; error?: string; zipFile?: { path: string; bytes: number };
remote?: { attempt: number; storageConfigId?: string; sources?: string[][] }; remote?: { attempt: number; storageConfigId?: string; sources?: string[][] };
preferQiniu?: boolean;
qiniu?: {
storageConfigId: string;
submittedAt: number;
persistentId?: string;
confirmedFailed?: boolean;
};
attempt?: number; attempt?: number;
contentDisposition?: string; cdnUrl?: string; cdnObject?: { storageConfigId: string; key: string }; contentDisposition?: string; cdnUrl?: string; cdnObject?: { storageConfigId: string; key: string; indexKey?: string };
createdAt: number; updatedAt: number; expiresAt: number; createdAt: number; updatedAt: number; expiresAt: number;
}; };
const validId = (id: string) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id); const validId = (id: string) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id);

View File

@ -0,0 +1,209 @@
import "server-only";
import { HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
import { createHmac } from "node:crypto";
import { getDefaultStorageConfig, getStorageConfigById } from "@/lib/db/queries";
import { findImageArchiveDownloadTask } from "@/lib/storage/image-archive-db";
import { getReadyQiniuImageDownload } from "@/lib/storage/qiniu-image-archive";
import {
buildS3PublicUrl,
createS3Client,
resolveS3UploadTarget,
toDbS3UploadConfig,
type S3UploadConfig,
} from "@/lib/storage/service";
import { buildCanvasDownloadSourceCandidates, sanitizeCanvasDownloadFileName } from "./photoshop-safe-download";
import { saveZipTask, type ZipTask } from "./download-zip-task-store";
const qiniuApi = "https://api.qiniu.com";
const preparationTimeout = 10_000;
const uncertainSubmitWindow = 180_000;
const toQiniuBase64 = (value: string | Buffer) =>
(typeof value === "string" ? Buffer.from(value) : value).toString("base64").replace(/\+/g, "-").replace(/\//g, "_");
const compatiblePngQuery = "imageMogr2/auto-orient/format/png/strip/meta-keep-list/aWNj";
function bucketObjectKey(url: string, config: S3UploadConfig) {
const base = buildS3PublicUrl(config, "");
if (!url.startsWith(base)) return;
const key = url.slice(base.length);
if (/^uploads\/(?:[A-Za-z0-9_-][\w.-]*\/)*[A-Za-z0-9_-][\w.-]*$/.test(key)) return key;
}
async function resolveSource(
file: ZipTask["files"][number],
requestOrigin: string,
target: NonNullable<Awaited<ReturnType<typeof resolveS3UploadTarget>>>,
client: ReturnType<typeof createS3Client>,
) {
const { config } = target;
const candidates = buildCanvasDownloadSourceCandidates(file.url, requestOrigin);
if (!candidates.length) return;
const archived = await findImageArchiveDownloadTask(candidates);
const asset = (archived?.metadata as { downloadAsset?: { status?: string; url?: string; contentType?: string } } | null)?.downloadAsset;
if (asset?.status === "ready" && asset.contentType === "image/png" && asset.url && bucketObjectKey(asset.url, config)) {
return asset.url;
}
const source = [archived?.targetUrl, ...candidates].find(url => url && bucketObjectKey(url, config));
const key = source && bucketObjectKey(source, config);
if (!source || !key) return;
const object = await client.send(new HeadObjectCommand({ Bucket: config.bucket, Key: key }), {
abortSignal: AbortSignal.timeout(preparationTimeout),
}).catch(error => {
if (error?.$metadata?.httpStatusCode === 404) return null;
throw error;
});
if (!object?.ContentLength) return;
if (object.ContentType === "image/png" && await getReadyQiniuImageDownload(source, target)) return source;
if (object.ContentType !== "image/jpeg" || object.ContentLength > 20 * 1024 * 1024) return;
const info = await fetch(source + "?imageInfo", {
redirect: "error", signal: AbortSignal.timeout(preparationTimeout),
}).then(response => response.ok ? response.json() : null).catch(() => null) as {
format?: string; colorModel?: string;
} | null;
// Color spaces such as CMYK still use the existing Photoshop conversion path.
if (info?.format === "jpeg" && /^(?:ycbcr|rgb|srgb)$/i.test(info.colorModel || "")) {
return source + "?" + compatiblePngQuery;
}
}
async function markFailed(task: ZipTask, message: string) {
task.qiniu!.confirmedFailed = true;
await saveZipTask(task);
throw new Error(message);
}
function resultKey(id: string) {
return "canvas-downloads/" + id + ".zip";
}
async function finishPublishedZip(task: ZipTask, config: S3UploadConfig) {
const url = new URL(buildS3PublicUrl(config, resultKey(task.id)));
url.searchParams.set("attname", task.folderName + ".zip");
const response = await fetch(url, {
method: "HEAD",
cache: "no-store",
redirect: "error",
signal: AbortSignal.timeout(preparationTimeout),
}).catch(() => null);
if (!response?.ok || !(Number(response.headers.get("content-length")) > 22) ||
!/^(?:application\/(?:zip|x-zip-compressed|octet-stream))(?:;|$)/i.test(response.headers.get("content-type") || "")) {
return false;
}
task.cdnUrl = url.href;
task.successCount = task.totalCount;
task.processedCount = task.totalCount;
task.failedCount = 0;
task.compressPercent = 100;
task.status = "completed";
task.stage = "打包完成";
delete task.error;
task.updatedAt = Date.now();
task.expiresAt = task.updatedAt + 24 * 60 * 60_000;
await saveZipTask(task);
return true;
}
async function updateQiniuZip(task: ZipTask) {
const job = task.qiniu!;
const storage = await getStorageConfigById({ id: job.storageConfigId });
const config = toDbS3UploadConfig(storage);
if (!storage?.isEnabled || config?.provider !== "qiniu-kodo" || !config.publicUrl) {
throw new Error("七牛云打包存储暂时不可用,请稍后重试");
}
if (!job.persistentId) {
if (await finishPublishedZip(task, config)) return;
if (Date.now() - job.submittedAt >= uncertainSubmitWindow) {
throw new Error("七牛云打包结果未确认。再次点击可继续检查原任务");
}
return;
}
const response = await fetch(qiniuApi + "/status/get/prefop?id=" + encodeURIComponent(job.persistentId), {
cache: "no-store",
redirect: "error",
signal: AbortSignal.timeout(preparationTimeout),
}).catch(() => null);
if (!response?.ok) return;
const result = await response.json().catch(() => null) as {
code?: number;
items?: Array<{ code: number; key?: string }>;
} | null;
if (result?.code === 3 || result?.code === 5) {
return markFailed(task, "七牛云未能打包所选图片,请确认原图仍可打开后重试");
}
if (result?.code !== 0) return;
if (!result.items?.some(item => item.code === 0 && item.key === resultKey(task.id))) {
return markFailed(task, "七牛云未返回完整压缩包,请重试");
}
if (task.stage !== "正在确认七牛下载链接") {
task.stage = "正在确认七牛下载链接";
task.compressPercent = 100;
task.updatedAt = Date.now();
await saveZipTask(task);
}
await finishPublishedZip(task, config);
}
export async function processQiniuCanvasZip(task: ZipTask, requestOrigin: string) {
if (task.qiniu) {
await updateQiniuZip(task);
return true;
}
const [storage, target] = await Promise.all([getDefaultStorageConfig(), resolveS3UploadTarget()]);
if (!storage?.isEnabled || target?.storageConfig?.id !== storage.id ||
target.config.provider !== "qiniu-kodo" || !target.config.publicUrl?.startsWith("https://")) return false;
const { config } = target;
const client = createS3Client(config);
try {
const sources: string[] = [];
for (let start = 0; start < task.files.length; start += 6) {
const batch = await Promise.all(task.files.slice(start, start + 6).map(file => resolveSource(file, requestOrigin, target, client)));
if (batch.some(url => !url)) return false;
sources.push(...(batch as string[]));
}
const lines = sources.map((url, index) => {
const fallback = "canvas-image-" + (index + 1);
const name = sanitizeCanvasDownloadFileName(task.files[index].name || fallback, fallback);
const file = String(index + 1).padStart(3, "0") + "-" + name + ".png";
return "/url/" + toQiniuBase64(url) + "/alias/" + toQiniuBase64(task.folderName + "/" + file);
});
const indexKey = "canvas-downloads/" + task.id + ".mkzip.txt";
const key = resultKey(task.id);
task.cdnObject = { storageConfigId: storage.id, key, indexKey };
await saveZipTask(task);
// This file contains addresses only. Kodo handles image reads, conversion and ZIP bytes.
client.config.requestChecksumCalculation = async () => "WHEN_REQUIRED";
await client.send(new PutObjectCommand({
Bucket: config.bucket, Key: indexKey, Body: Buffer.from(lines.join("\n")), ContentType: "text/plain",
}), { abortSignal: AbortSignal.timeout(preparationTimeout) });
const body = new URLSearchParams({ bucket: config.bucket, key: indexKey,
fops: "mkzip/4/|saveas/" + toQiniuBase64(config.bucket + ":" + key) }).toString();
const signature = toQiniuBase64(createHmac("sha1", config.secretAccessKey).update("/pfop/\n" + body).digest());
task.qiniu = { storageConfigId: storage.id, submittedAt: Date.now() };
task.status = "running";
task.stage = "正在七牛云端打包";
task.processedCount = task.totalCount;
task.updatedAt = Date.now();
// Persist before submitting so a lost response is reconciled against the output key.
await saveZipTask(task);
const response = await fetch(qiniuApi + "/pfop/", {
method: "POST", body, redirect: "error",
headers: { Authorization: "QBox " + config.accessKeyId + ":" + signature, "Content-Type": "application/x-www-form-urlencoded" },
signal: AbortSignal.timeout(15_000),
}).catch(() => null);
if (response && response.status >= 400 && response.status < 500) {
await response.body?.cancel().catch(() => undefined);
return markFailed(task, "七牛云暂时无法受理打包,请稍后重试");
}
const result = await response?.json().catch(() => null) as { persistentId?: string } | null;
if (result?.persistentId && response?.ok) {
task.qiniu.persistentId = result.persistentId;
await saveZipTask(task);
}
return true;
} finally {
client.destroy();
}
}

View File

@ -20,10 +20,10 @@ async function taskRequest(url: string, init?: RequestInit) {
} }
export async function downloadCanvasSelectionZip(params: { export async function downloadCanvasSelectionZip(params: {
folderName: string; files: Array<{ url: string; name: string }>; selectionKey: string; folderName: string; files: Array<{ url: string; name: string }>; selectionKey: string;
scope?: "batch-instruction"; scope?: "canvas-selection" | "batch-instruction";
onProgress: (task: CanvasZipTaskView) => void; onProgress: (task: CanvasZipTaskView) => void;
}) { }) {
const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(JSON.stringify([params.selectionKey, params.files, ...(params.scope ? [params.scope] : [])]))); const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(JSON.stringify([params.selectionKey, params.files, ...(params.scope === "batch-instruction" ? [params.scope] : [])])));
const key = "canvas-zip:" + Array.from(new Uint8Array(digest), byte => byte.toString(16).padStart(2, "0")).join(""); const key = "canvas-zip:" + Array.from(new Uint8Array(digest), byte => byte.toString(16).padStart(2, "0")).join("");
let taskId = ""; let taskId = "";
try { taskId = sessionStorage.getItem(key) || ""; } catch { /* Storage is optional. */ } try { taskId = sessionStorage.getItem(key) || ""; } catch { /* Storage is optional. */ }

View File

@ -107,12 +107,18 @@ export async function publishCanvasZip(params: ZipPublication) {
} finally { client.destroy(); } } finally { client.destroy(); }
} }
export async function removeCanvasZipObject(object: { storageConfigId: string; key: string }) { export async function removeCanvasZipObject(object: { storageConfigId: string; key: string; indexKey?: string }) {
if (!/^canvas-downloads\/[0-9a-f-]{36}\.zip$/i.test(object.key)) return; if (!/^canvas-downloads\/[0-9a-f-]{36}\.zip$/i.test(object.key)) return;
const config = toDbS3UploadConfig(await getStorageConfigById({ id: object.storageConfigId })); const config = toDbS3UploadConfig(await getStorageConfigById({ id: object.storageConfigId }));
if (!config || config.provider !== "qiniu-kodo") return; if (!config || config.provider !== "qiniu-kodo") return;
const { DeleteObjectCommand } = await import("@aws-sdk/client-s3"); const { DeleteObjectCommand } = await import("@aws-sdk/client-s3");
const client = createS3Client(config); const client = createS3Client(config);
try { await client.send(new DeleteObjectCommand({ Bucket: config.bucket, Key: object.key }), { abortSignal: AbortSignal.timeout(5000) }); } const keys = [object.key];
finally { client.destroy(); } if (object.indexKey === object.key.slice(0, -4) + ".mkzip.txt") keys.push(object.indexKey);
try {
await Promise.all(keys.map(Key => client.send(new DeleteObjectCommand({ Bucket: config.bucket, Key }),
{ abortSignal: AbortSignal.timeout(5000) })));
} finally {
client.destroy();
}
} }

View File

@ -296,3 +296,111 @@ test("small ZIP delivery preserves the complete file and download filename",asyn
try{expect((await publish({id:randomUUID(),path:file,bytes:bytes.length,disposition:'attachment; filename="small.zip"'})).url).toContain('https://cdn.example/');} try{expect((await publish({id:randomUUID(),path:file,bytes:bytes.length,disposition:'attachment; filename="small.zip"'})).url).toContain('https://cdn.example/');}
finally{await rm(dir,{recursive:true,force:true});} finally{await rm(dir,{recursive:true,force:true});}
}); });
test("canvas selection submits a Qiniu index and recovers an accepted ZIP without resubmitting", async () => {
const directory = await mkdtemp(join(tmpdir(), "canvas-cloud-zip-"));
const taskId = randomUUID();
const endpoint = "https://canvas.example/api/canvas/download-zip-task";
const original = "https://cdn.example/uploads/bookshelf.jpg";
const prepared = "https://cdn.example/uploads/prepared.png";
const config = {
provider: "qiniu-kodo", region: "cn-east-1", bucket: "assets",
publicUrl: "https://cdn.example", accessKeyId: "test-access-key", secretAccessKey: "test-secret",
};
const storage = { id: "qiniu-config", isEnabled: true };
const callbacks: Array<() => Promise<void>> = [];
let owner = "owner";
let submitted = 0;
let published = false;
let saves = 0;
const load = serverLoader({
"@/app/(auth)/auth": { auth: async () => ({ user: { id: owner } }) },
"@/lib/observability/active-work": runtimeWork,
"next/server": { NextResponse, after: (callback: () => Promise<void>) => callbacks.push(callback) },
"@/lib/db/queries": { getDefaultStorageConfig: async () => storage, getStorageConfigById: async () => storage },
"@/lib/storage/image-archive-db": {
findImageArchiveDownloadTask: async ([url]: string[]) => url === prepared
? { metadata: { downloadAsset: { status: "ready", url: prepared, contentType: "image/png" } } } : null,
},
"@/lib/storage/qiniu-image-archive": {},
"@/lib/storage/service": {
toDbS3UploadConfig: () => config,
resolveS3UploadTarget: async () => ({ config, storageConfig: storage }),
buildS3PublicUrl: (_config: unknown, key: string) => config.publicUrl + "/" + key,
createS3Client: () => ({
config: {}, destroy() {},
async send(command: HeadObjectCommand | PutObjectCommand) {
if (command instanceof HeadObjectCommand) {
expect(command.input.Key).toBe("uploads/bookshelf.jpg");
return { ContentLength: 200, ContentType: "image/jpeg" };
}
expect(command.input.Key).toBe("canvas-downloads/" + taskId + ".mkzip.txt");
const lines = Buffer.from(command.input.Body as Buffer).toString("utf8").split("\n");
expect(lines).toHaveLength(2);
const parts = lines[0].split("/");
expect(Buffer.from(parts[2], "base64url").toString()).toMatch(/bookshelf\.jpg\?imageMogr2/);
expect(Buffer.from(parts[4], "base64url").toString()).toBe("画布选图/001-书架.png");
expect(Buffer.from(lines[1].split("/")[2], "base64url").toString()).toBe(prepared);
saves++;
return {};
},
}),
},
"./photoshop-safe-download": {
buildCanvasDownloadSourceCandidates: (url: string) => [url],
sanitizeCanvasDownloadFileName: (name: string) => name,
},
"@/lib/canvas/photoshop-safe-download": {
fetchCanvasDownloadAsset: () => { throw new Error("Image bytes should stay in Qiniu"); },
},
"@/lib/storage/canvas-zip-delivery": { removeCanvasZipObject: async () => {} },
fetch: async (url: string | URL, init: RequestInit) => {
const u = new URL(url);
if (u.search === "?imageInfo") {
return Response.json({ format: "jpeg", colorModel: "ycbcr" });
}
if (u.pathname === "/pfop/") {
submitted++;
const payload = new URLSearchParams(String(init.body));
expect(payload.get("key")).toBe("canvas-downloads/" + taskId + ".mkzip.txt");
expect(payload.get("fops")).toMatch(/^mkzip\/4\//);
throw new Error("Accepted by Qiniu, but the response was lost");
}
expect(init.method).toBe("HEAD");
expect(u.hostname).toBe("cdn.example");
return published
? new Response(null, { headers: { "content-length": "700", "content-type": "application/zip" } })
: new Response(null, { status: 404 });
},
}, { CANVAS_DOWNLOAD_ZIP_DIR: directory });
const route = load("app/api/canvas/download-zip-task/route.ts");
const post = () => new NextRequest(endpoint, { method: "POST", body: JSON.stringify({
taskId, folderName: "画布选图", scope: "canvas-selection", retry: true,
files: [{ url: original, name: "书架" }, { url: prepared, name: "底图" }],
}) });
const poll = (extra = "") => new NextRequest(endpoint + "?taskId=" + taskId + extra);
const flush = async () => { while (callbacks.length) await callbacks.shift()!(); };
try {
expect((await route.POST(post())).status).toBe(202);
await flush();
expect((await (await route.GET(poll())).json()).task).toMatchObject({ status: "running", stage: "正在七牛云端打包" });
expect((await route.GET(poll("&download=1"))).status).toBe(409);
published = true;
await route.POST(post());
await flush();
expect((await (await route.GET(poll())).json()).task).toMatchObject({ status: "completed", progress: 100, successCount: 2 });
const download = await route.GET(poll("&download=1"));
expect(download.status).toBe(307);
expect(new URL(download.headers.get("location")!).searchParams.get("attname")).toBe("画布选图.zip");
expect(submitted).toBe(1);
expect(saves).toBe(1);
const stored = JSON.parse(await readFile(join(directory, taskId, "task.json"), "utf8"));
expect(stored.zipFile).toBeUndefined();
owner = "other-user";
expect((await route.GET(poll("&download=1"))).status).toBe(404);
} finally {
await rm(directory, { recursive: true, force: true });
}
});