ai-chatbot/lib/canvas/qiniu-canvas-zip.ts

210 lines
9.2 KiB
TypeScript

import "server-only";
import { HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
import { createHmac } from "node:crypto";
import { getDefaultStorageConfig, getStorageConfigById } from "@/lib/db/queries";
import { findImageArchiveDownloadTask } from "@/lib/storage/image-archive-db";
import { getReadyQiniuImageDownload } from "@/lib/storage/qiniu-image-archive";
import {
buildS3PublicUrl,
createS3Client,
resolveS3UploadTarget,
toDbS3UploadConfig,
type S3UploadConfig,
} from "@/lib/storage/service";
import { buildCanvasDownloadSourceCandidates, sanitizeCanvasDownloadFileName } from "./photoshop-safe-download";
import { saveZipTask, type ZipTask } from "./download-zip-task-store";
const qiniuApi = "https://api.qiniu.com";
const preparationTimeout = 10_000;
const uncertainSubmitWindow = 180_000;
const toQiniuBase64 = (value: string | Buffer) =>
(typeof value === "string" ? Buffer.from(value) : value).toString("base64").replace(/\+/g, "-").replace(/\//g, "_");
const compatiblePngQuery = "imageMogr2/auto-orient/format/png/strip/meta-keep-list/aWNj";
function bucketObjectKey(url: string, config: S3UploadConfig) {
const base = buildS3PublicUrl(config, "");
if (!url.startsWith(base)) return;
const key = url.slice(base.length);
if (/^uploads\/(?:[A-Za-z0-9_-][\w.-]*\/)*[A-Za-z0-9_-][\w.-]*$/.test(key)) return key;
}
async function resolveSource(
file: ZipTask["files"][number],
requestOrigin: string,
target: NonNullable<Awaited<ReturnType<typeof resolveS3UploadTarget>>>,
client: ReturnType<typeof createS3Client>,
) {
const { config } = target;
const candidates = buildCanvasDownloadSourceCandidates(file.url, requestOrigin);
if (!candidates.length) return;
const archived = await findImageArchiveDownloadTask(candidates);
const asset = (archived?.metadata as { downloadAsset?: { status?: string; url?: string; contentType?: string } } | null)?.downloadAsset;
if (asset?.status === "ready" && asset.contentType === "image/png" && asset.url && bucketObjectKey(asset.url, config)) {
return asset.url;
}
const source = [archived?.targetUrl, ...candidates].find(url => url && bucketObjectKey(url, config));
const key = source && bucketObjectKey(source, config);
if (!source || !key) return;
const object = await client.send(new HeadObjectCommand({ Bucket: config.bucket, Key: key }), {
abortSignal: AbortSignal.timeout(preparationTimeout),
}).catch(error => {
if (error?.$metadata?.httpStatusCode === 404) return null;
throw error;
});
if (!object?.ContentLength) return;
if (object.ContentType === "image/png" && await getReadyQiniuImageDownload(source, target)) return source;
if (object.ContentType !== "image/jpeg" || object.ContentLength > 20 * 1024 * 1024) return;
const info = await fetch(source + "?imageInfo", {
redirect: "error", signal: AbortSignal.timeout(preparationTimeout),
}).then(response => response.ok ? response.json() : null).catch(() => null) as {
format?: string; colorModel?: string;
} | null;
// Color spaces such as CMYK still use the existing Photoshop conversion path.
if (info?.format === "jpeg" && /^(?:ycbcr|rgb|srgb)$/i.test(info.colorModel || "")) {
return source + "?" + compatiblePngQuery;
}
}
async function markFailed(task: ZipTask, message: string) {
task.qiniu!.confirmedFailed = true;
await saveZipTask(task);
throw new Error(message);
}
function resultKey(id: string) {
return "canvas-downloads/" + id + ".zip";
}
async function finishPublishedZip(task: ZipTask, config: S3UploadConfig) {
const url = new URL(buildS3PublicUrl(config, resultKey(task.id)));
url.searchParams.set("attname", task.folderName + ".zip");
const response = await fetch(url, {
method: "HEAD",
cache: "no-store",
redirect: "error",
signal: AbortSignal.timeout(preparationTimeout),
}).catch(() => null);
if (!response?.ok || !(Number(response.headers.get("content-length")) > 22) ||
!/^(?:application\/(?:zip|x-zip-compressed|octet-stream))(?:;|$)/i.test(response.headers.get("content-type") || "")) {
return false;
}
task.cdnUrl = url.href;
task.successCount = task.totalCount;
task.processedCount = task.totalCount;
task.failedCount = 0;
task.compressPercent = 100;
task.status = "completed";
task.stage = "打包完成";
delete task.error;
task.updatedAt = Date.now();
task.expiresAt = task.updatedAt + 24 * 60 * 60_000;
await saveZipTask(task);
return true;
}
async function updateQiniuZip(task: ZipTask) {
const job = task.qiniu!;
const storage = await getStorageConfigById({ id: job.storageConfigId });
const config = toDbS3UploadConfig(storage);
if (!storage?.isEnabled || config?.provider !== "qiniu-kodo" || !config.publicUrl) {
throw new Error("七牛云打包存储暂时不可用,请稍后重试");
}
if (!job.persistentId) {
if (await finishPublishedZip(task, config)) return;
if (Date.now() - job.submittedAt >= uncertainSubmitWindow) {
throw new Error("七牛云打包结果未确认。再次点击可继续检查原任务");
}
return;
}
const response = await fetch(qiniuApi + "/status/get/prefop?id=" + encodeURIComponent(job.persistentId), {
cache: "no-store",
redirect: "error",
signal: AbortSignal.timeout(preparationTimeout),
}).catch(() => null);
if (!response?.ok) return;
const result = await response.json().catch(() => null) as {
code?: number;
items?: Array<{ code: number; key?: string }>;
} | null;
if (result?.code === 3 || result?.code === 5) {
return markFailed(task, "七牛云未能打包所选图片,请确认原图仍可打开后重试");
}
if (result?.code !== 0) return;
if (!result.items?.some(item => item.code === 0 && item.key === resultKey(task.id))) {
return markFailed(task, "七牛云未返回完整压缩包,请重试");
}
if (task.stage !== "正在确认七牛下载链接") {
task.stage = "正在确认七牛下载链接";
task.compressPercent = 100;
task.updatedAt = Date.now();
await saveZipTask(task);
}
await finishPublishedZip(task, config);
}
export async function processQiniuCanvasZip(task: ZipTask, requestOrigin: string) {
if (task.qiniu) {
await updateQiniuZip(task);
return true;
}
const [storage, target] = await Promise.all([getDefaultStorageConfig(), resolveS3UploadTarget()]);
if (!storage?.isEnabled || target?.storageConfig?.id !== storage.id ||
target.config.provider !== "qiniu-kodo" || !target.config.publicUrl?.startsWith("https://")) return false;
const { config } = target;
const client = createS3Client(config);
try {
const sources: string[] = [];
for (let start = 0; start < task.files.length; start += 6) {
const batch = await Promise.all(task.files.slice(start, start + 6).map(file => resolveSource(file, requestOrigin, target, client)));
if (batch.some(url => !url)) return false;
sources.push(...(batch as string[]));
}
const lines = sources.map((url, index) => {
const fallback = "canvas-image-" + (index + 1);
const name = sanitizeCanvasDownloadFileName(task.files[index].name || fallback, fallback);
const file = String(index + 1).padStart(3, "0") + "-" + name + ".png";
return "/url/" + toQiniuBase64(url) + "/alias/" + toQiniuBase64(task.folderName + "/" + file);
});
const indexKey = "canvas-downloads/" + task.id + ".mkzip.txt";
const key = resultKey(task.id);
task.cdnObject = { storageConfigId: storage.id, key, indexKey };
await saveZipTask(task);
// This file contains addresses only. Kodo handles image reads, conversion and ZIP bytes.
client.config.requestChecksumCalculation = async () => "WHEN_REQUIRED";
await client.send(new PutObjectCommand({
Bucket: config.bucket, Key: indexKey, Body: Buffer.from(lines.join("\n")), ContentType: "text/plain",
}), { abortSignal: AbortSignal.timeout(preparationTimeout) });
const body = new URLSearchParams({ bucket: config.bucket, key: indexKey,
fops: "mkzip/4/|saveas/" + toQiniuBase64(config.bucket + ":" + key) }).toString();
const signature = toQiniuBase64(createHmac("sha1", config.secretAccessKey).update("/pfop/\n" + body).digest());
task.qiniu = { storageConfigId: storage.id, submittedAt: Date.now() };
task.status = "running";
task.stage = "正在七牛云端打包";
task.processedCount = task.totalCount;
task.updatedAt = Date.now();
// Persist before submitting so a lost response is reconciled against the output key.
await saveZipTask(task);
const response = await fetch(qiniuApi + "/pfop/", {
method: "POST", body, redirect: "error",
headers: { Authorization: "QBox " + config.accessKeyId + ":" + signature, "Content-Type": "application/x-www-form-urlencoded" },
signal: AbortSignal.timeout(15_000),
}).catch(() => null);
if (response && response.status >= 400 && response.status < 500) {
await response.body?.cancel().catch(() => undefined);
return markFailed(task, "七牛云暂时无法受理打包,请稍后重试");
}
const result = await response?.json().catch(() => null) as { persistentId?: string } | null;
if (result?.persistentId && response?.ok) {
task.qiniu.persistentId = result.persistentId;
await saveZipTask(task);
}
return true;
} finally {
client.destroy();
}
}