Merge pull request '恢复短暂失败的 98K 七牛原图抓取' (#105) from codex/qiniu-98k-archive-recovery into main
恢复短暂失败的 98K 七牛原图抓取
This commit is contained in:
commit
18cdbc99c2
@ -12,7 +12,7 @@ import {
|
||||
imageArchiveSourceKey,
|
||||
} from "@/lib/storage/image-archive-db";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
import { archiveImageThroughQiniu } from "./qiniu-image-archive";
|
||||
import { archiveImageThroughQiniu, QiniuArchiveSourceMissingError } from "./qiniu-image-archive";
|
||||
|
||||
const logger = createLogger("image-archive");
|
||||
|
||||
@ -195,6 +195,7 @@ export async function downloadTemporaryImage(params: { sourceUrl: string; retrie
|
||||
export async function archiveTemporaryImage(sourceUrl: string, archiveId?: string) {
|
||||
if (archiveId) {
|
||||
const direct = await archiveImageThroughQiniu(sourceUrl, archiveId).catch(error => {
|
||||
if (error instanceof QiniuArchiveSourceMissingError) throw error;
|
||||
logger.warn("cloud_archive_fallback", error, { archiveId });
|
||||
return null;
|
||||
});
|
||||
|
||||
@ -1,7 +1,8 @@
|
||||
import "server-only";
|
||||
|
||||
import { GetObjectCommand } from "@aws-sdk/client-s3";
|
||||
import { GetObjectCommand, HeadObjectCommand } from "@aws-sdk/client-s3";
|
||||
import { createHmac } from "node:crypto";
|
||||
import { createLogger } from "@/lib/observability/logger";
|
||||
import {
|
||||
buildS3PublicUrl,
|
||||
createS3Client,
|
||||
@ -11,6 +12,23 @@ import {
|
||||
const QINIU_EAST_CHINA_IO = "iovip-z0.qiniuio.com";
|
||||
const PNG_HEADER_LIMIT = 64 * 1024;
|
||||
const PNG_SIGNATURE = Buffer.from("89504e470d0a1a0a", "hex");
|
||||
const QINIU_FETCH_TIMEOUT_MS = 40_000;
|
||||
const RELAY_RETRY_TIMEOUT_MS = 25_000;
|
||||
const logger = createLogger("qiniu-image-archive");
|
||||
|
||||
type QiniuUploadTarget = NonNullable<Awaited<ReturnType<typeof resolveS3UploadTarget>>>;
|
||||
|
||||
class QiniuFetchError extends Error {
|
||||
constructor(readonly status: number, readonly requestId: string | null) {
|
||||
super(`Qiniu archive fetch returned HTTP ${status}`);
|
||||
}
|
||||
}
|
||||
|
||||
export class QiniuArchiveSourceMissingError extends Error {
|
||||
constructor(status: number) {
|
||||
super(`98K original image is no longer available (HTTP ${status})`);
|
||||
}
|
||||
}
|
||||
|
||||
export type ReadyImageDownload = {
|
||||
status: "ready";
|
||||
@ -99,9 +117,84 @@ async function archiveThroughQiniu(sourceUrl: string, archiveId: string, kind: "
|
||||
|
||||
// The archive task owns this object key, including retries after a timeout.
|
||||
const pathname = kind === "image" ? `uploads/ai-generated-archive-${archiveId}` : `uploads/canvas-${kind}-${archiveId}`;
|
||||
if (kind === "image" && source.origin === "https://api.vip98k.com" &&
|
||||
source.pathname.startsWith("/api/external/assets/")) {
|
||||
return fetchRelayImageIntoQiniu(sourceUrl, pathname, target!, archiveId);
|
||||
}
|
||||
return fetchImageIntoQiniu(sourceUrl, pathname, target!, kind);
|
||||
}
|
||||
|
||||
function isRetryableQiniuFetch(error: unknown) {
|
||||
if (error instanceof QiniuFetchError) {
|
||||
return [408, 478, 502, 503, 504, 599].includes(error.status);
|
||||
}
|
||||
if (!(error instanceof Error)) return false;
|
||||
const cause = error.cause as { code?: string } | undefined;
|
||||
return error.name === "TimeoutError" || error.name === "AbortError" ||
|
||||
["ECONNRESET", "ETIMEDOUT", "UND_ERR_SOCKET"].includes(cause?.code ?? "");
|
||||
}
|
||||
|
||||
function relaySourceStatus(error: unknown) {
|
||||
if (error instanceof QiniuFetchError && [404, 410].includes(error.status)) return error.status;
|
||||
}
|
||||
|
||||
async function readStoredRelayImage(pathname: string, target: QiniuUploadTarget) {
|
||||
const client = createS3Client(target.config);
|
||||
try {
|
||||
const response = await client.send(new HeadObjectCommand({
|
||||
Bucket: target.config.bucket, Key: pathname,
|
||||
}), { abortSignal: AbortSignal.timeout(4_000) });
|
||||
if (response.ContentType?.startsWith("image/") && response.ContentLength && response.ContentLength > 0) {
|
||||
return describeArchivedObject(pathname, target, response.ContentType, response.ContentLength);
|
||||
}
|
||||
} catch {
|
||||
return;
|
||||
} finally {
|
||||
client.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchRelayImageIntoQiniu(
|
||||
sourceUrl: string,
|
||||
pathname: string,
|
||||
target: QiniuUploadTarget,
|
||||
archiveId: string,
|
||||
) {
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
return await fetchImageIntoQiniu(sourceUrl, pathname, target);
|
||||
} catch (error) {
|
||||
const missing = relaySourceStatus(error);
|
||||
const retryable = isRetryableQiniuFetch(error);
|
||||
const alreadyExists = error instanceof QiniuFetchError && error.status === 614;
|
||||
if (!missing && !retryable && !alreadyExists) throw error;
|
||||
|
||||
// Losing the fetch response does not establish whether Kodo stored the file.
|
||||
const stored = await readStoredRelayImage(pathname, target);
|
||||
if (stored) return stored;
|
||||
if (missing) throw new QiniuArchiveSourceMissingError(missing);
|
||||
if (!retryable) throw error;
|
||||
|
||||
logger.warn("relay_fetch_retry", {
|
||||
archiveId,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
...(error instanceof QiniuFetchError
|
||||
? { status: error.status, requestId: error.requestId }
|
||||
: { errorType: error instanceof Error ? error.name : "unknown" }),
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
return await fetchImageIntoQiniu(sourceUrl, pathname, target, "image", RELAY_RETRY_TIMEOUT_MS);
|
||||
} catch (error) {
|
||||
const stored = await readStoredRelayImage(pathname, target);
|
||||
if (stored) return stored;
|
||||
const missing = relaySourceStatus(error);
|
||||
if (missing) throw new QiniuArchiveSourceMissingError(missing);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** Prepare uploaded WebP images inside Kodo, retaining their embedded colour profile. */
|
||||
export async function prepareQiniuWebpDownload(url: string, taskId: string) {
|
||||
const target = await resolveS3UploadTarget();
|
||||
@ -118,8 +211,9 @@ export async function prepareQiniuWebpDownload(url: string, taskId: string) {
|
||||
async function fetchImageIntoQiniu(
|
||||
sourceUrl: string,
|
||||
pathname: string,
|
||||
target: NonNullable<Awaited<ReturnType<typeof resolveS3UploadTarget>>>,
|
||||
target: QiniuUploadTarget,
|
||||
kind: "image" | "audio" | "video" = "image",
|
||||
timeoutMs = QINIU_FETCH_TIMEOUT_MS,
|
||||
) {
|
||||
const config = target.config;
|
||||
const path = `/fetch/${encodeQiniu(sourceUrl)}/to/${encodeQiniu(`${config.bucket}:${pathname}`)}`;
|
||||
@ -134,24 +228,28 @@ async function fetchImageIntoQiniu(
|
||||
},
|
||||
body: "",
|
||||
redirect: "error",
|
||||
signal: AbortSignal.timeout(40_000),
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
throw new Error(`Qiniu archive fetch returned HTTP ${response.status}`);
|
||||
throw new QiniuFetchError(response.status, response.headers.get("x-reqid"));
|
||||
}
|
||||
const result = await response.json() as { key?: string; fsize?: number; mimeType?: string };
|
||||
if (result.key !== pathname || !result.fsize || !result.mimeType?.startsWith(`${kind}/`)) {
|
||||
throw new Error(`Qiniu archive fetch did not return ${kind}`);
|
||||
}
|
||||
const url = buildS3PublicUrl(config, pathname);
|
||||
return describeArchivedObject(pathname, target, result.mimeType, result.fsize);
|
||||
}
|
||||
|
||||
async function describeArchivedObject(pathname: string, target: QiniuUploadTarget, contentType: string, bytes: number) {
|
||||
const url = buildS3PublicUrl(target.config, pathname);
|
||||
return {
|
||||
url,
|
||||
thumbnailUrl: url,
|
||||
pathname,
|
||||
contentType: result.mimeType,
|
||||
bytes: result.fsize,
|
||||
downloadAsset: result.mimeType === "image/png"
|
||||
contentType,
|
||||
bytes,
|
||||
downloadAsset: contentType === "image/png"
|
||||
? await getReadyQiniuImageDownload(url, target).catch(() => undefined)
|
||||
: undefined,
|
||||
};
|
||||
|
||||
@ -132,6 +132,73 @@ test("Qiniu archives 98K originals without relaying full image bytes through the
|
||||
expect(reads).toHaveLength(3);
|
||||
});
|
||||
|
||||
test("98K fetch checks the original archive key and retries a temporary failure once", async () => {
|
||||
const source = "https://api.vip98k.com/api/external/assets/test_asset";
|
||||
const id = "archive-1";
|
||||
const key = `uploads/ai-generated-archive-${id}`;
|
||||
const publicUrl = "https://bucket.example";
|
||||
const config = { provider: "qiniu-kodo", region: "cn-east-1", bucket: "archive",
|
||||
accessKeyId: "test-key", secretAccessKey: "test-secret", publicUrl };
|
||||
const calls: string[] = [];
|
||||
let existing = false;
|
||||
let firstStatus = 478;
|
||||
const load = serverLoader({
|
||||
"@/lib/observability/logger": loggerMock,
|
||||
"@/lib/storage/service": {
|
||||
resolveS3UploadTarget: async () => ({ config }),
|
||||
buildS3PublicUrl: (_: unknown, pathname: string) => `${publicUrl}/${pathname}`,
|
||||
createS3Client: () => ({
|
||||
async send(cmd: { input: { Bucket: string; Key: string } }) {
|
||||
expect(cmd.input).toEqual({ Bucket: "archive", Key: key });
|
||||
calls.push("head");
|
||||
if (!existing) throw new Error("Object does not exist");
|
||||
return { ContentType: "image/jpeg", ContentLength: 20 };
|
||||
},
|
||||
destroy() {},
|
||||
}),
|
||||
},
|
||||
fetch: async (url: string) => {
|
||||
const to = new URL(url).pathname.split("/to/")[1];
|
||||
expect(Buffer.from(to, "base64url").toString()).toBe(`archive:${key}`);
|
||||
calls.push("fetch");
|
||||
return calls.length === 1
|
||||
? { ok: false, status: firstStatus, headers: new Headers() }
|
||||
: Response.json({ key, fsize: 20, mimeType: "image/jpeg" });
|
||||
},
|
||||
})("lib/storage/qiniu-image-archive.ts");
|
||||
|
||||
expect(await load.archiveImageThroughQiniu(source, id)).toMatchObject({ pathname: key, bytes: 20 });
|
||||
expect(calls.splice(0)).toEqual(["fetch", "head", "fetch"]);
|
||||
|
||||
existing = true;
|
||||
firstStatus = 614;
|
||||
expect(await load.archiveImageThroughQiniu(source, id)).toMatchObject({ pathname: key, bytes: 20 });
|
||||
expect(calls.splice(0)).toEqual(["fetch", "head"]);
|
||||
|
||||
existing = false;
|
||||
firstStatus = 404;
|
||||
await expect(load.archiveImageThroughQiniu(source, id)).rejects.toBeInstanceOf(load.QiniuArchiveSourceMissingError);
|
||||
expect(calls).toEqual(["fetch", "head"]);
|
||||
});
|
||||
|
||||
test("a confirmed missing 98K image does not start a production transfer", async () => {
|
||||
class QiniuArchiveSourceMissingError extends Error {}
|
||||
const sourceError = new QiniuArchiveSourceMissingError("98K original image is no longer available (HTTP 404)");
|
||||
const load = serverLoader({
|
||||
"@/lib/storage/image-byte-cache": { fetchImageBytes() { throw new Error("Unexpected production download"); } },
|
||||
"@/lib/observability/logger": loggerMock,
|
||||
"@/lib/storage/image-archive-db": {},
|
||||
"@/lib/storage/service": {},
|
||||
"./image-archive-worker": {},
|
||||
"./qiniu-image-archive": {
|
||||
QiniuArchiveSourceMissingError,
|
||||
async archiveImageThroughQiniu() { throw sourceError; },
|
||||
},
|
||||
})("lib/storage/image-archive.ts");
|
||||
await expect(load.archiveTemporaryImage("https://api.vip98k.com/api/external/assets/test_asset", "task"))
|
||||
.rejects.toBe(sourceError);
|
||||
});
|
||||
|
||||
test("160 requests from the same user and model start without admission waiting", async () => {
|
||||
const module = serverLoader({})("lib/canvas/image-concurrency.ts");
|
||||
let queued = 0;
|
||||
|
||||
@ -62,7 +62,9 @@ test("download preparation only registers unsigned files in the configured bucke
|
||||
const module = serverLoader({
|
||||
"@/lib/db/queries": { getDefaultStorageConfig: async () => config },
|
||||
"./image-archive-db": { findImageArchiveDownloadTask: async () => null,
|
||||
enqueueImageArchiveTask: async () => ({ id: "waiting", status: "pending", targetUrl: null }),
|
||||
ensureStoredImageDownloadTask: async (url: string) => { stored.push(url); return { id: "1", targetUrl: url }; } },
|
||||
"./image-archive-worker": { scheduleImageArchiveWorker() {} },
|
||||
}, { CANVAS_QINIU_ORIGIN_URL: "https://origin.example" })("lib/storage/canvas-cdn-download.ts");
|
||||
for (const url of ["https://other.example/uploads/a.png", "https://cdn.example/uploads/a.png?token=signed",
|
||||
"https://cdn.example/uploads/a.png?imageView2/2/w/160", "https://cdn.example/private/a.png"]) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user