From 9334591323827d9085a104fc5eb4a487cbee141b Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Sat, 21 Nov 2026 14:51:24 +0800 Subject: [PATCH] Package canvas selections in Qiniu --- app/api/canvas/download-zip-task/route.ts | 9 +- components/canvas/CanvasWorkspace.tsx | 1 + lib/canvas/download-zip-task-store.ts | 9 +- lib/canvas/qiniu-canvas-zip.ts | 209 ++++++++++++++++++++++ lib/client/canvas-zip-download.ts | 4 +- lib/storage/canvas-zip-delivery.ts | 12 +- tests/routes/canvas-zip-download.test.ts | 108 +++++++++++ 7 files changed, 345 insertions(+), 7 deletions(-) create mode 100644 lib/canvas/qiniu-canvas-zip.ts diff --git a/app/api/canvas/download-zip-task/route.ts b/app/api/canvas/download-zip-task/route.ts index 57aa08d0b..1a8b61ae6 100644 --- a/app/api/canvas/download-zip-task/route.ts +++ b/app/api/canvas/download-zip-task/route.ts @@ -93,6 +93,11 @@ async function processTask(id: string, userId: string, requestOrigin: string, co await updateRemoteCanvasZip(task, requestOrigin); return; } + if (task.preferQiniu || task.qiniu) { + const { processQiniuCanvasZip } = await import("@/lib/canvas/qiniu-canvas-zip"); + if (await processQiniuCanvasZip(task, requestOrigin)) return; + delete task.preferQiniu; + } await assertZipDiskSpace(); const existingZip = task.zipFile && task.contentDisposition && await stat(task.zipFile.path).catch(() => null); const reuseZip = existingZip && existingZip.size === task.zipFile!.bytes; @@ -158,6 +163,7 @@ export async function POST(request: NextRequest) { process.env.CANVAS_ZIP_WORKER_URL && process.env.CANVAS_ZIP_WORKER_TOKEN ? { attempt: 0 } : undefined; task = { id, userId: session.user.id, folderName: sanitizeFolderName(String(body.folderName || "")), files, remote, + preferQiniu: body.scope === "canvas-selection" || undefined, totalCount: files.length, processedCount: 0, successCount: 0, failedCount: 0, compressPercent: 0, status: "pending", stage: "等待后台准备", createdAt: now, updatedAt: now, expiresAt: now + PREPARATION_TTL_MS }; await saveZipTask(task); @@ -168,6 +174,7 @@ export async function POST(request: NextRequest) { task.remote.attempt++; delete task.remote.sources; } + if (task.qiniu?.confirmedFailed) delete task.qiniu; if (!task.zipFile) { for (const file of task.files) delete file.error; task.processedCount = task.successCount = task.files.filter(file => file.prepared).length; @@ -195,7 +202,7 @@ export async function GET(request: NextRequest) { const task = await loadZipTask(id, session.user.id); if (!task) return NextResponse.json({ error: "打包任务不存在或已过期,请重新打包" }, { status: 404 }); if (request.nextUrl.searchParams.get("download") === "1") { - if (task.remote && task.status === "completed" && task.cdnUrl) + if ((task.remote || task.qiniu) && task.status === "completed" && task.cdnUrl) return NextResponse.redirect(task.cdnUrl, 307); if (task.status !== "completed" || !task.zipFile) return NextResponse.json({ error: "ZIP 尚未准备好" }, { status: 409 }); if (task.cdnUrl && request.nextUrl.searchParams.get("local") !== "1") diff --git a/components/canvas/CanvasWorkspace.tsx b/components/canvas/CanvasWorkspace.tsx index 2290761a3..535985ce2 100644 --- a/components/canvas/CanvasWorkspace.tsx +++ b/components/canvas/CanvasWorkspace.tsx @@ -1937,6 +1937,7 @@ export function CanvasWorkspace({ const toastId = toast.loading("正在准备所选图片..."); try { const result = await downloadCanvasSelectionZip({ folderName, files, + scope: "canvas-selection", selectionKey: `${useCanvasStore.getState().projectId}:${selectedImageNodes.map(node => node.id).sort().join(",")}`, onProgress: task => toast.loading(`${task.stage}(${task.progress}%)`, { id: toastId }), }); diff --git a/lib/canvas/download-zip-task-store.ts b/lib/canvas/download-zip-task-store.ts index 4cdb6747d..370348cfc 100644 --- a/lib/canvas/download-zip-task-store.ts +++ b/lib/canvas/download-zip-task-store.ts @@ -11,8 +11,15 @@ export type ZipTask = { compressPercent: number; status: "pending" | "running" | "completed" | "failed"; stage: string; error?: string; zipFile?: { path: string; bytes: number }; remote?: { attempt: number; storageConfigId?: string; sources?: string[][] }; + preferQiniu?: boolean; + qiniu?: { + storageConfigId: string; + submittedAt: number; + persistentId?: string; + confirmedFailed?: boolean; + }; attempt?: number; - contentDisposition?: string; cdnUrl?: string; cdnObject?: { storageConfigId: string; key: string }; + contentDisposition?: string; cdnUrl?: string; cdnObject?: { storageConfigId: string; key: string; indexKey?: string }; createdAt: number; updatedAt: number; expiresAt: number; }; const validId = (id: string) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id); diff --git a/lib/canvas/qiniu-canvas-zip.ts b/lib/canvas/qiniu-canvas-zip.ts new file mode 100644 index 000000000..98e12240f --- /dev/null +++ b/lib/canvas/qiniu-canvas-zip.ts @@ -0,0 +1,209 @@ +import "server-only"; + +import { HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"; +import { createHmac } from "node:crypto"; +import { getDefaultStorageConfig, getStorageConfigById } from "@/lib/db/queries"; +import { findImageArchiveDownloadTask } from "@/lib/storage/image-archive-db"; +import { getReadyQiniuImageDownload } from "@/lib/storage/qiniu-image-archive"; +import { + buildS3PublicUrl, + createS3Client, + resolveS3UploadTarget, + toDbS3UploadConfig, + type S3UploadConfig, +} from "@/lib/storage/service"; +import { buildCanvasDownloadSourceCandidates, sanitizeCanvasDownloadFileName } from "./photoshop-safe-download"; +import { saveZipTask, type ZipTask } from "./download-zip-task-store"; + +const qiniuApi = "https://api.qiniu.com"; +const preparationTimeout = 10_000; +const uncertainSubmitWindow = 180_000; +const toQiniuBase64 = (value: string | Buffer) => + (typeof value === "string" ? Buffer.from(value) : value).toString("base64").replace(/\+/g, "-").replace(/\//g, "_"); +const compatiblePngQuery = "imageMogr2/auto-orient/format/png/strip/meta-keep-list/aWNj"; + +function bucketObjectKey(url: string, config: S3UploadConfig) { + const base = buildS3PublicUrl(config, ""); + if (!url.startsWith(base)) return; + const key = url.slice(base.length); + if (/^uploads\/(?:[A-Za-z0-9_-][\w.-]*\/)*[A-Za-z0-9_-][\w.-]*$/.test(key)) return key; +} + +async function resolveSource( + file: ZipTask["files"][number], + requestOrigin: string, + target: NonNullable>>, + client: ReturnType, +) { + const { config } = target; + const candidates = buildCanvasDownloadSourceCandidates(file.url, requestOrigin); + if (!candidates.length) return; + const archived = await findImageArchiveDownloadTask(candidates); + const asset = (archived?.metadata as { downloadAsset?: { status?: string; url?: string; contentType?: string } } | null)?.downloadAsset; + if (asset?.status === "ready" && asset.contentType === "image/png" && asset.url && bucketObjectKey(asset.url, config)) { + return asset.url; + } + + const source = [archived?.targetUrl, ...candidates].find(url => url && bucketObjectKey(url, config)); + const key = source && bucketObjectKey(source, config); + if (!source || !key) return; + const object = await client.send(new HeadObjectCommand({ Bucket: config.bucket, Key: key }), { + abortSignal: AbortSignal.timeout(preparationTimeout), + }).catch(error => { + if (error?.$metadata?.httpStatusCode === 404) return null; + throw error; + }); + if (!object?.ContentLength) return; + if (object.ContentType === "image/png" && await getReadyQiniuImageDownload(source, target)) return source; + if (object.ContentType !== "image/jpeg" || object.ContentLength > 20 * 1024 * 1024) return; + + const info = await fetch(source + "?imageInfo", { + redirect: "error", signal: AbortSignal.timeout(preparationTimeout), + }).then(response => response.ok ? response.json() : null).catch(() => null) as { + format?: string; colorModel?: string; + } | null; + // Color spaces such as CMYK still use the existing Photoshop conversion path. + if (info?.format === "jpeg" && /^(?:ycbcr|rgb|srgb)$/i.test(info.colorModel || "")) { + return source + "?" + compatiblePngQuery; + } +} + +async function markFailed(task: ZipTask, message: string) { + task.qiniu!.confirmedFailed = true; + await saveZipTask(task); + throw new Error(message); +} + +function resultKey(id: string) { + return "canvas-downloads/" + id + ".zip"; +} + +async function finishPublishedZip(task: ZipTask, config: S3UploadConfig) { + const url = new URL(buildS3PublicUrl(config, resultKey(task.id))); + url.searchParams.set("attname", task.folderName + ".zip"); + const response = await fetch(url, { + method: "HEAD", + cache: "no-store", + redirect: "error", + signal: AbortSignal.timeout(preparationTimeout), + }).catch(() => null); + if (!response?.ok || !(Number(response.headers.get("content-length")) > 22) || + !/^(?:application\/(?:zip|x-zip-compressed|octet-stream))(?:;|$)/i.test(response.headers.get("content-type") || "")) { + return false; + } + task.cdnUrl = url.href; + task.successCount = task.totalCount; + task.processedCount = task.totalCount; + task.failedCount = 0; + task.compressPercent = 100; + task.status = "completed"; + task.stage = "打包完成"; + delete task.error; + task.updatedAt = Date.now(); + task.expiresAt = task.updatedAt + 24 * 60 * 60_000; + await saveZipTask(task); + return true; +} + +async function updateQiniuZip(task: ZipTask) { + const job = task.qiniu!; + const storage = await getStorageConfigById({ id: job.storageConfigId }); + const config = toDbS3UploadConfig(storage); + if (!storage?.isEnabled || config?.provider !== "qiniu-kodo" || !config.publicUrl) { + throw new Error("七牛云打包存储暂时不可用,请稍后重试"); + } + if (!job.persistentId) { + if (await finishPublishedZip(task, config)) return; + if (Date.now() - job.submittedAt >= uncertainSubmitWindow) { + throw new Error("七牛云打包结果未确认。再次点击可继续检查原任务"); + } + return; + } + + const response = await fetch(qiniuApi + "/status/get/prefop?id=" + encodeURIComponent(job.persistentId), { + cache: "no-store", + redirect: "error", + signal: AbortSignal.timeout(preparationTimeout), + }).catch(() => null); + if (!response?.ok) return; + const result = await response.json().catch(() => null) as { + code?: number; + items?: Array<{ code: number; key?: string }>; + } | null; + if (result?.code === 3 || result?.code === 5) { + return markFailed(task, "七牛云未能打包所选图片,请确认原图仍可打开后重试"); + } + if (result?.code !== 0) return; + if (!result.items?.some(item => item.code === 0 && item.key === resultKey(task.id))) { + return markFailed(task, "七牛云未返回完整压缩包,请重试"); + } + if (task.stage !== "正在确认七牛下载链接") { + task.stage = "正在确认七牛下载链接"; + task.compressPercent = 100; + task.updatedAt = Date.now(); + await saveZipTask(task); + } + await finishPublishedZip(task, config); +} + +export async function processQiniuCanvasZip(task: ZipTask, requestOrigin: string) { + if (task.qiniu) { + await updateQiniuZip(task); + return true; + } + const [storage, target] = await Promise.all([getDefaultStorageConfig(), resolveS3UploadTarget()]); + if (!storage?.isEnabled || target?.storageConfig?.id !== storage.id || + target.config.provider !== "qiniu-kodo" || !target.config.publicUrl?.startsWith("https://")) return false; + const { config } = target; + const client = createS3Client(config); + try { + const sources: string[] = []; + for (let start = 0; start < task.files.length; start += 6) { + const batch = await Promise.all(task.files.slice(start, start + 6).map(file => resolveSource(file, requestOrigin, target, client))); + if (batch.some(url => !url)) return false; + sources.push(...(batch as string[])); + } + const lines = sources.map((url, index) => { + const fallback = "canvas-image-" + (index + 1); + const name = sanitizeCanvasDownloadFileName(task.files[index].name || fallback, fallback); + const file = String(index + 1).padStart(3, "0") + "-" + name + ".png"; + return "/url/" + toQiniuBase64(url) + "/alias/" + toQiniuBase64(task.folderName + "/" + file); + }); + const indexKey = "canvas-downloads/" + task.id + ".mkzip.txt"; + const key = resultKey(task.id); + task.cdnObject = { storageConfigId: storage.id, key, indexKey }; + await saveZipTask(task); + // This file contains addresses only. Kodo handles image reads, conversion and ZIP bytes. + client.config.requestChecksumCalculation = async () => "WHEN_REQUIRED"; + await client.send(new PutObjectCommand({ + Bucket: config.bucket, Key: indexKey, Body: Buffer.from(lines.join("\n")), ContentType: "text/plain", + }), { abortSignal: AbortSignal.timeout(preparationTimeout) }); + const body = new URLSearchParams({ bucket: config.bucket, key: indexKey, + fops: "mkzip/4/|saveas/" + toQiniuBase64(config.bucket + ":" + key) }).toString(); + const signature = toQiniuBase64(createHmac("sha1", config.secretAccessKey).update("/pfop/\n" + body).digest()); + task.qiniu = { storageConfigId: storage.id, submittedAt: Date.now() }; + task.status = "running"; + task.stage = "正在七牛云端打包"; + task.processedCount = task.totalCount; + task.updatedAt = Date.now(); + // Persist before submitting so a lost response is reconciled against the output key. + await saveZipTask(task); + const response = await fetch(qiniuApi + "/pfop/", { + method: "POST", body, redirect: "error", + headers: { Authorization: "QBox " + config.accessKeyId + ":" + signature, "Content-Type": "application/x-www-form-urlencoded" }, + signal: AbortSignal.timeout(15_000), + }).catch(() => null); + if (response && response.status >= 400 && response.status < 500) { + await response.body?.cancel().catch(() => undefined); + return markFailed(task, "七牛云暂时无法受理打包,请稍后重试"); + } + const result = await response?.json().catch(() => null) as { persistentId?: string } | null; + if (result?.persistentId && response?.ok) { + task.qiniu.persistentId = result.persistentId; + await saveZipTask(task); + } + return true; + } finally { + client.destroy(); + } +} diff --git a/lib/client/canvas-zip-download.ts b/lib/client/canvas-zip-download.ts index 43a228eb4..de3a843cc 100644 --- a/lib/client/canvas-zip-download.ts +++ b/lib/client/canvas-zip-download.ts @@ -20,10 +20,10 @@ async function taskRequest(url: string, init?: RequestInit) { } export async function downloadCanvasSelectionZip(params: { folderName: string; files: Array<{ url: string; name: string }>; selectionKey: string; - scope?: "batch-instruction"; + scope?: "canvas-selection" | "batch-instruction"; onProgress: (task: CanvasZipTaskView) => void; }) { - const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(JSON.stringify([params.selectionKey, params.files, ...(params.scope ? [params.scope] : [])]))); + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(JSON.stringify([params.selectionKey, params.files, ...(params.scope === "batch-instruction" ? [params.scope] : [])]))); const key = "canvas-zip:" + Array.from(new Uint8Array(digest), byte => byte.toString(16).padStart(2, "0")).join(""); let taskId = ""; try { taskId = sessionStorage.getItem(key) || ""; } catch { /* Storage is optional. */ } diff --git a/lib/storage/canvas-zip-delivery.ts b/lib/storage/canvas-zip-delivery.ts index a8b0a18a0..494e392ca 100644 --- a/lib/storage/canvas-zip-delivery.ts +++ b/lib/storage/canvas-zip-delivery.ts @@ -107,12 +107,18 @@ export async function publishCanvasZip(params: ZipPublication) { } finally { client.destroy(); } } -export async function removeCanvasZipObject(object: { storageConfigId: string; key: string }) { +export async function removeCanvasZipObject(object: { storageConfigId: string; key: string; indexKey?: string }) { if (!/^canvas-downloads\/[0-9a-f-]{36}\.zip$/i.test(object.key)) return; const config = toDbS3UploadConfig(await getStorageConfigById({ id: object.storageConfigId })); if (!config || config.provider !== "qiniu-kodo") return; const { DeleteObjectCommand } = await import("@aws-sdk/client-s3"); const client = createS3Client(config); - try { await client.send(new DeleteObjectCommand({ Bucket: config.bucket, Key: object.key }), { abortSignal: AbortSignal.timeout(5000) }); } - finally { client.destroy(); } + const keys = [object.key]; + if (object.indexKey === object.key.slice(0, -4) + ".mkzip.txt") keys.push(object.indexKey); + try { + await Promise.all(keys.map(Key => client.send(new DeleteObjectCommand({ Bucket: config.bucket, Key }), + { abortSignal: AbortSignal.timeout(5000) }))); + } finally { + client.destroy(); + } } diff --git a/tests/routes/canvas-zip-download.test.ts b/tests/routes/canvas-zip-download.test.ts index 3d450b704..a0e638854 100644 --- a/tests/routes/canvas-zip-download.test.ts +++ b/tests/routes/canvas-zip-download.test.ts @@ -296,3 +296,111 @@ test("small ZIP delivery preserves the complete file and download filename",asyn try{expect((await publish({id:randomUUID(),path:file,bytes:bytes.length,disposition:'attachment; filename="small.zip"'})).url).toContain('https://cdn.example/');} finally{await rm(dir,{recursive:true,force:true});} }); + +test("canvas selection submits a Qiniu index and recovers an accepted ZIP without resubmitting", async () => { + const directory = await mkdtemp(join(tmpdir(), "canvas-cloud-zip-")); + const taskId = randomUUID(); + const endpoint = "https://canvas.example/api/canvas/download-zip-task"; + const original = "https://cdn.example/uploads/bookshelf.jpg"; + const prepared = "https://cdn.example/uploads/prepared.png"; + const config = { + provider: "qiniu-kodo", region: "cn-east-1", bucket: "assets", + publicUrl: "https://cdn.example", accessKeyId: "test-access-key", secretAccessKey: "test-secret", + }; + const storage = { id: "qiniu-config", isEnabled: true }; + const callbacks: Array<() => Promise> = []; + let owner = "owner"; + let submitted = 0; + let published = false; + let saves = 0; + + const load = serverLoader({ + "@/app/(auth)/auth": { auth: async () => ({ user: { id: owner } }) }, + "@/lib/observability/active-work": runtimeWork, + "next/server": { NextResponse, after: (callback: () => Promise) => callbacks.push(callback) }, + "@/lib/db/queries": { getDefaultStorageConfig: async () => storage, getStorageConfigById: async () => storage }, + "@/lib/storage/image-archive-db": { + findImageArchiveDownloadTask: async ([url]: string[]) => url === prepared + ? { metadata: { downloadAsset: { status: "ready", url: prepared, contentType: "image/png" } } } : null, + }, + "@/lib/storage/qiniu-image-archive": {}, + "@/lib/storage/service": { + toDbS3UploadConfig: () => config, + resolveS3UploadTarget: async () => ({ config, storageConfig: storage }), + buildS3PublicUrl: (_config: unknown, key: string) => config.publicUrl + "/" + key, + createS3Client: () => ({ + config: {}, destroy() {}, + async send(command: HeadObjectCommand | PutObjectCommand) { + if (command instanceof HeadObjectCommand) { + expect(command.input.Key).toBe("uploads/bookshelf.jpg"); + return { ContentLength: 200, ContentType: "image/jpeg" }; + } + expect(command.input.Key).toBe("canvas-downloads/" + taskId + ".mkzip.txt"); + const lines = Buffer.from(command.input.Body as Buffer).toString("utf8").split("\n"); + expect(lines).toHaveLength(2); + const parts = lines[0].split("/"); + expect(Buffer.from(parts[2], "base64url").toString()).toMatch(/bookshelf\.jpg\?imageMogr2/); + expect(Buffer.from(parts[4], "base64url").toString()).toBe("画布选图/001-书架.png"); + expect(Buffer.from(lines[1].split("/")[2], "base64url").toString()).toBe(prepared); + saves++; + return {}; + }, + }), + }, + "./photoshop-safe-download": { + buildCanvasDownloadSourceCandidates: (url: string) => [url], + sanitizeCanvasDownloadFileName: (name: string) => name, + }, + "@/lib/canvas/photoshop-safe-download": { + fetchCanvasDownloadAsset: () => { throw new Error("Image bytes should stay in Qiniu"); }, + }, + "@/lib/storage/canvas-zip-delivery": { removeCanvasZipObject: async () => {} }, + fetch: async (url: string | URL, init: RequestInit) => { + const u = new URL(url); + if (u.search === "?imageInfo") { + return Response.json({ format: "jpeg", colorModel: "ycbcr" }); + } + if (u.pathname === "/pfop/") { + submitted++; + const payload = new URLSearchParams(String(init.body)); + expect(payload.get("key")).toBe("canvas-downloads/" + taskId + ".mkzip.txt"); + expect(payload.get("fops")).toMatch(/^mkzip\/4\//); + throw new Error("Accepted by Qiniu, but the response was lost"); + } + expect(init.method).toBe("HEAD"); + expect(u.hostname).toBe("cdn.example"); + return published + ? new Response(null, { headers: { "content-length": "700", "content-type": "application/zip" } }) + : new Response(null, { status: 404 }); + }, + }, { CANVAS_DOWNLOAD_ZIP_DIR: directory }); + const route = load("app/api/canvas/download-zip-task/route.ts"); + const post = () => new NextRequest(endpoint, { method: "POST", body: JSON.stringify({ + taskId, folderName: "画布选图", scope: "canvas-selection", retry: true, + files: [{ url: original, name: "书架" }, { url: prepared, name: "底图" }], + }) }); + const poll = (extra = "") => new NextRequest(endpoint + "?taskId=" + taskId + extra); + const flush = async () => { while (callbacks.length) await callbacks.shift()!(); }; + + try { + expect((await route.POST(post())).status).toBe(202); + await flush(); + expect((await (await route.GET(poll())).json()).task).toMatchObject({ status: "running", stage: "正在七牛云端打包" }); + expect((await route.GET(poll("&download=1"))).status).toBe(409); + published = true; + await route.POST(post()); + await flush(); + expect((await (await route.GET(poll())).json()).task).toMatchObject({ status: "completed", progress: 100, successCount: 2 }); + const download = await route.GET(poll("&download=1")); + expect(download.status).toBe(307); + expect(new URL(download.headers.get("location")!).searchParams.get("attname")).toBe("画布选图.zip"); + expect(submitted).toBe(1); + expect(saves).toBe(1); + const stored = JSON.parse(await readFile(join(directory, taskId, "task.json"), "utf8")); + expect(stored.zipFile).toBeUndefined(); + owner = "other-user"; + expect((await route.GET(poll("&download=1"))).status).toBe(404); + } finally { + await rm(directory, { recursive: true, force: true }); + } +});