Merge pull request '恢复短暂失败的 98K 七牛原图抓取' (#105) from codex/qiniu-98k-archive-recovery into main

恢复短暂失败的 98K 七牛原图抓取
This commit is contained in:
WuYongJie 2026-11-18 12:19:11 +08:00
commit 18cdbc99c2
4 changed files with 177 additions and 9 deletions

View File

@ -12,7 +12,7 @@ import {
imageArchiveSourceKey,
} from "@/lib/storage/image-archive-db";
import { uploadFile } from "@/lib/storage/service";
import { archiveImageThroughQiniu } from "./qiniu-image-archive";
import { archiveImageThroughQiniu, QiniuArchiveSourceMissingError } from "./qiniu-image-archive";
const logger = createLogger("image-archive");
@ -195,6 +195,7 @@ export async function downloadTemporaryImage(params: { sourceUrl: string; retrie
export async function archiveTemporaryImage(sourceUrl: string, archiveId?: string) {
if (archiveId) {
const direct = await archiveImageThroughQiniu(sourceUrl, archiveId).catch(error => {
if (error instanceof QiniuArchiveSourceMissingError) throw error;
logger.warn("cloud_archive_fallback", error, { archiveId });
return null;
});

View File

@ -1,7 +1,8 @@
import "server-only";
import { GetObjectCommand } from "@aws-sdk/client-s3";
import { GetObjectCommand, HeadObjectCommand } from "@aws-sdk/client-s3";
import { createHmac } from "node:crypto";
import { createLogger } from "@/lib/observability/logger";
import {
buildS3PublicUrl,
createS3Client,
@ -11,6 +12,23 @@ import {
const QINIU_EAST_CHINA_IO = "iovip-z0.qiniuio.com";
const PNG_HEADER_LIMIT = 64 * 1024;
const PNG_SIGNATURE = Buffer.from("89504e470d0a1a0a", "hex");
const QINIU_FETCH_TIMEOUT_MS = 40_000;
const RELAY_RETRY_TIMEOUT_MS = 25_000;
const logger = createLogger("qiniu-image-archive");
type QiniuUploadTarget = NonNullable<Awaited<ReturnType<typeof resolveS3UploadTarget>>>;
class QiniuFetchError extends Error {
constructor(readonly status: number, readonly requestId: string | null) {
super(`Qiniu archive fetch returned HTTP ${status}`);
}
}
export class QiniuArchiveSourceMissingError extends Error {
constructor(status: number) {
super(`98K original image is no longer available (HTTP ${status})`);
}
}
export type ReadyImageDownload = {
status: "ready";
@ -99,9 +117,84 @@ async function archiveThroughQiniu(sourceUrl: string, archiveId: string, kind: "
// The archive task owns this object key, including retries after a timeout.
const pathname = kind === "image" ? `uploads/ai-generated-archive-${archiveId}` : `uploads/canvas-${kind}-${archiveId}`;
if (kind === "image" && source.origin === "https://api.vip98k.com" &&
source.pathname.startsWith("/api/external/assets/")) {
return fetchRelayImageIntoQiniu(sourceUrl, pathname, target!, archiveId);
}
return fetchImageIntoQiniu(sourceUrl, pathname, target!, kind);
}
function isRetryableQiniuFetch(error: unknown) {
if (error instanceof QiniuFetchError) {
return [408, 478, 502, 503, 504, 599].includes(error.status);
}
if (!(error instanceof Error)) return false;
const cause = error.cause as { code?: string } | undefined;
return error.name === "TimeoutError" || error.name === "AbortError" ||
["ECONNRESET", "ETIMEDOUT", "UND_ERR_SOCKET"].includes(cause?.code ?? "");
}
function relaySourceStatus(error: unknown) {
if (error instanceof QiniuFetchError && [404, 410].includes(error.status)) return error.status;
}
async function readStoredRelayImage(pathname: string, target: QiniuUploadTarget) {
const client = createS3Client(target.config);
try {
const response = await client.send(new HeadObjectCommand({
Bucket: target.config.bucket, Key: pathname,
}), { abortSignal: AbortSignal.timeout(4_000) });
if (response.ContentType?.startsWith("image/") && response.ContentLength && response.ContentLength > 0) {
return describeArchivedObject(pathname, target, response.ContentType, response.ContentLength);
}
} catch {
return;
} finally {
client.destroy();
}
}
async function fetchRelayImageIntoQiniu(
sourceUrl: string,
pathname: string,
target: QiniuUploadTarget,
archiveId: string,
) {
const startedAt = Date.now();
try {
return await fetchImageIntoQiniu(sourceUrl, pathname, target);
} catch (error) {
const missing = relaySourceStatus(error);
const retryable = isRetryableQiniuFetch(error);
const alreadyExists = error instanceof QiniuFetchError && error.status === 614;
if (!missing && !retryable && !alreadyExists) throw error;
// Losing the fetch response does not establish whether Kodo stored the file.
const stored = await readStoredRelayImage(pathname, target);
if (stored) return stored;
if (missing) throw new QiniuArchiveSourceMissingError(missing);
if (!retryable) throw error;
logger.warn("relay_fetch_retry", {
archiveId,
elapsedMs: Date.now() - startedAt,
...(error instanceof QiniuFetchError
? { status: error.status, requestId: error.requestId }
: { errorType: error instanceof Error ? error.name : "unknown" }),
});
}
try {
return await fetchImageIntoQiniu(sourceUrl, pathname, target, "image", RELAY_RETRY_TIMEOUT_MS);
} catch (error) {
const stored = await readStoredRelayImage(pathname, target);
if (stored) return stored;
const missing = relaySourceStatus(error);
if (missing) throw new QiniuArchiveSourceMissingError(missing);
throw error;
}
}
/** Prepare uploaded WebP images inside Kodo, retaining their embedded colour profile. */
export async function prepareQiniuWebpDownload(url: string, taskId: string) {
const target = await resolveS3UploadTarget();
@ -118,8 +211,9 @@ export async function prepareQiniuWebpDownload(url: string, taskId: string) {
async function fetchImageIntoQiniu(
sourceUrl: string,
pathname: string,
target: NonNullable<Awaited<ReturnType<typeof resolveS3UploadTarget>>>,
target: QiniuUploadTarget,
kind: "image" | "audio" | "video" = "image",
timeoutMs = QINIU_FETCH_TIMEOUT_MS,
) {
const config = target.config;
const path = `/fetch/${encodeQiniu(sourceUrl)}/to/${encodeQiniu(`${config.bucket}:${pathname}`)}`;
@ -134,24 +228,28 @@ async function fetchImageIntoQiniu(
},
body: "",
redirect: "error",
signal: AbortSignal.timeout(40_000),
signal: AbortSignal.timeout(timeoutMs),
});
if (!response.ok) {
await response.body?.cancel();
throw new Error(`Qiniu archive fetch returned HTTP ${response.status}`);
throw new QiniuFetchError(response.status, response.headers.get("x-reqid"));
}
const result = await response.json() as { key?: string; fsize?: number; mimeType?: string };
if (result.key !== pathname || !result.fsize || !result.mimeType?.startsWith(`${kind}/`)) {
throw new Error(`Qiniu archive fetch did not return ${kind}`);
}
const url = buildS3PublicUrl(config, pathname);
return describeArchivedObject(pathname, target, result.mimeType, result.fsize);
}
async function describeArchivedObject(pathname: string, target: QiniuUploadTarget, contentType: string, bytes: number) {
const url = buildS3PublicUrl(target.config, pathname);
return {
url,
thumbnailUrl: url,
pathname,
contentType: result.mimeType,
bytes: result.fsize,
downloadAsset: result.mimeType === "image/png"
contentType,
bytes,
downloadAsset: contentType === "image/png"
? await getReadyQiniuImageDownload(url, target).catch(() => undefined)
: undefined,
};

View File

@ -132,6 +132,73 @@ test("Qiniu archives 98K originals without relaying full image bytes through the
expect(reads).toHaveLength(3);
});
test("98K fetch checks the original archive key and retries a temporary failure once", async () => {
const source = "https://api.vip98k.com/api/external/assets/test_asset";
const id = "archive-1";
const key = `uploads/ai-generated-archive-${id}`;
const publicUrl = "https://bucket.example";
const config = { provider: "qiniu-kodo", region: "cn-east-1", bucket: "archive",
accessKeyId: "test-key", secretAccessKey: "test-secret", publicUrl };
const calls: string[] = [];
let existing = false;
let firstStatus = 478;
const load = serverLoader({
"@/lib/observability/logger": loggerMock,
"@/lib/storage/service": {
resolveS3UploadTarget: async () => ({ config }),
buildS3PublicUrl: (_: unknown, pathname: string) => `${publicUrl}/${pathname}`,
createS3Client: () => ({
async send(cmd: { input: { Bucket: string; Key: string } }) {
expect(cmd.input).toEqual({ Bucket: "archive", Key: key });
calls.push("head");
if (!existing) throw new Error("Object does not exist");
return { ContentType: "image/jpeg", ContentLength: 20 };
},
destroy() {},
}),
},
fetch: async (url: string) => {
const to = new URL(url).pathname.split("/to/")[1];
expect(Buffer.from(to, "base64url").toString()).toBe(`archive:${key}`);
calls.push("fetch");
return calls.length === 1
? { ok: false, status: firstStatus, headers: new Headers() }
: Response.json({ key, fsize: 20, mimeType: "image/jpeg" });
},
})("lib/storage/qiniu-image-archive.ts");
expect(await load.archiveImageThroughQiniu(source, id)).toMatchObject({ pathname: key, bytes: 20 });
expect(calls.splice(0)).toEqual(["fetch", "head", "fetch"]);
existing = true;
firstStatus = 614;
expect(await load.archiveImageThroughQiniu(source, id)).toMatchObject({ pathname: key, bytes: 20 });
expect(calls.splice(0)).toEqual(["fetch", "head"]);
existing = false;
firstStatus = 404;
await expect(load.archiveImageThroughQiniu(source, id)).rejects.toBeInstanceOf(load.QiniuArchiveSourceMissingError);
expect(calls).toEqual(["fetch", "head"]);
});
test("a confirmed missing 98K image does not start a production transfer", async () => {
class QiniuArchiveSourceMissingError extends Error {}
const sourceError = new QiniuArchiveSourceMissingError("98K original image is no longer available (HTTP 404)");
const load = serverLoader({
"@/lib/storage/image-byte-cache": { fetchImageBytes() { throw new Error("Unexpected production download"); } },
"@/lib/observability/logger": loggerMock,
"@/lib/storage/image-archive-db": {},
"@/lib/storage/service": {},
"./image-archive-worker": {},
"./qiniu-image-archive": {
QiniuArchiveSourceMissingError,
async archiveImageThroughQiniu() { throw sourceError; },
},
})("lib/storage/image-archive.ts");
await expect(load.archiveTemporaryImage("https://api.vip98k.com/api/external/assets/test_asset", "task"))
.rejects.toBe(sourceError);
});
test("160 requests from the same user and model start without admission waiting", async () => {
const module = serverLoader({})("lib/canvas/image-concurrency.ts");
let queued = 0;

View File

@ -62,7 +62,9 @@ test("download preparation only registers unsigned files in the configured bucke
const module = serverLoader({
"@/lib/db/queries": { getDefaultStorageConfig: async () => config },
"./image-archive-db": { findImageArchiveDownloadTask: async () => null,
enqueueImageArchiveTask: async () => ({ id: "waiting", status: "pending", targetUrl: null }),
ensureStoredImageDownloadTask: async (url: string) => { stored.push(url); return { id: "1", targetUrl: url }; } },
"./image-archive-worker": { scheduleImageArchiveWorker() {} },
}, { CANVAS_QINIU_ORIGIN_URL: "https://origin.example" })("lib/storage/canvas-cdn-download.ts");
for (const url of ["https://other.example/uploads/a.png", "https://cdn.example/uploads/a.png?token=signed",
"https://cdn.example/uploads/a.png?imageView2/2/w/160", "https://cdn.example/private/a.png"]) {