415 lines
14 KiB
TypeScript
415 lines
14 KiB
TypeScript
import { config as loadEnv } from "dotenv";
|
||
import { desc, eq } from "drizzle-orm";
|
||
import { mkdir, writeFile } from "node:fs/promises";
|
||
import Module from "node:module";
|
||
import path from "node:path";
|
||
|
||
loadEnv({ path: ".env.local" });
|
||
loadEnv();
|
||
|
||
const moduleLoader = Module as typeof Module & {
|
||
_load: (request: string, parent: NodeModule | null, isMain: boolean) => unknown;
|
||
};
|
||
const originalLoad = moduleLoader._load;
|
||
moduleLoader._load = function patchedServerOnlyLoad(
|
||
this: typeof moduleLoader,
|
||
request,
|
||
parent,
|
||
isMain
|
||
) {
|
||
if (request === "server-only") return {};
|
||
return originalLoad.call(this, request, parent, isMain);
|
||
};
|
||
|
||
type AssetKind = "glb" | "image" | "spz";
|
||
type StringRecord = Record<string, string>;
|
||
type JsonRecord = Record<string, unknown>;
|
||
|
||
const dryRun = process.argv.includes("--dry-run");
|
||
const worldOnly = process.argv.includes("--world-only");
|
||
const tripoOnly = process.argv.includes("--tripo-only");
|
||
const limitArg = process.argv.find((arg) => arg.startsWith("--limit="));
|
||
const limit = limitArg ? Math.max(1, Number(limitArg.split("=")[1]) || 1) : null;
|
||
const DOWNLOAD_TIMEOUT_MS = 10 * 60 * 1000;
|
||
|
||
function normalizeUrl(value: unknown) {
|
||
return typeof value === "string" ? value.trim() : "";
|
||
}
|
||
|
||
function isHttpUrl(value: string) {
|
||
return /^https?:\/\//i.test(value);
|
||
}
|
||
|
||
function hasGlbMagic(bytes: Uint8Array) {
|
||
return (
|
||
bytes.length >= 12 &&
|
||
bytes[0] === 0x67 &&
|
||
bytes[1] === 0x6c &&
|
||
bytes[2] === 0x54 &&
|
||
bytes[3] === 0x46
|
||
);
|
||
}
|
||
|
||
function hasSpzMagic(bytes: Uint8Array) {
|
||
return (
|
||
(bytes.length >= 4 &&
|
||
bytes[0] === 0x4e &&
|
||
bytes[1] === 0x47 &&
|
||
bytes[2] === 0x53 &&
|
||
bytes[3] === 0x50) ||
|
||
(bytes.length >= 2 && bytes[0] === 0x1f && bytes[1] === 0x8b)
|
||
);
|
||
}
|
||
|
||
function looksLikeErrorPayload(bytes: Uint8Array) {
|
||
const head = new TextDecoder("utf-8", { fatal: false })
|
||
.decode(bytes.slice(0, Math.min(bytes.length, 64)))
|
||
.trimStart();
|
||
return head.startsWith("<") || head.startsWith("{") || head.startsWith("[");
|
||
}
|
||
|
||
function validateBytes(bytes: Uint8Array, kind: AssetKind) {
|
||
if (bytes.byteLength <= 0 || looksLikeErrorPayload(bytes)) return false;
|
||
if (kind === "glb") return hasGlbMagic(bytes);
|
||
if (kind === "spz") return hasSpzMagic(bytes);
|
||
return true;
|
||
}
|
||
|
||
async function probeAsset(url: string, kind: AssetKind) {
|
||
if (!isHttpUrl(url)) return false;
|
||
try {
|
||
const response = await fetch(url, {
|
||
method: "GET",
|
||
headers: { Range: "bytes=0-63" },
|
||
cache: "no-store",
|
||
signal: AbortSignal.timeout(30_000),
|
||
});
|
||
if (!response.ok) return false;
|
||
const bytes = new Uint8Array(await response.arrayBuffer());
|
||
return validateBytes(bytes, kind);
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
async function downloadAsset(url: string, kind: AssetKind) {
|
||
const response = await fetch(url, {
|
||
method: "GET",
|
||
headers: { "User-Agent": "AI-Chatbot-3D-Qiniu-Repair/1.0" },
|
||
cache: "no-store",
|
||
signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS),
|
||
});
|
||
if (!response.ok) {
|
||
throw new Error(`下载失败（HTTP ${response.status}）`);
|
||
}
|
||
const buffer = await response.arrayBuffer();
|
||
if (!validateBytes(new Uint8Array(buffer), kind)) {
|
||
throw new Error(`${kind.toUpperCase()} 文件内容无效`);
|
||
}
|
||
return {
|
||
buffer,
|
||
contentType: response.headers.get("content-type"),
|
||
};
|
||
}
|
||
|
||
function resolveExtension(url: string, kind: AssetKind, contentType?: string | null) {
|
||
if (kind === "glb") return "glb";
|
||
if (kind === "spz") return "spz";
|
||
const type = String(contentType || "").toLowerCase();
|
||
if (type.includes("webp")) return "webp";
|
||
if (type.includes("png")) return "png";
|
||
if (type.includes("gif")) return "gif";
|
||
try {
|
||
const ext = new URL(url).pathname.split(".").pop()?.toLowerCase();
|
||
if (ext && ["jpg", "jpeg", "png", "webp", "gif"].includes(ext)) return ext;
|
||
} catch {
|
||
// ignore
|
||
}
|
||
return "jpg";
|
||
}
|
||
|
||
function replaceUrlsDeep(value: unknown, replacements: Map<string, string>): unknown {
|
||
if (typeof value === "string") return replacements.get(value) || value;
|
||
if (Array.isArray(value)) {
|
||
return value.map((item) => replaceUrlsDeep(item, replacements));
|
||
}
|
||
if (value && typeof value === "object") {
|
||
return Object.fromEntries(
|
||
Object.entries(value as JsonRecord).map(([key, item]) => [
|
||
key,
|
||
replaceUrlsDeep(item, replacements),
|
||
])
|
||
);
|
||
}
|
||
return value;
|
||
}
|
||
|
||
function toStringRecord(value: unknown): StringRecord {
|
||
if (!value || typeof value !== "object" || Array.isArray(value)) return {};
|
||
return Object.fromEntries(
|
||
Object.entries(value as JsonRecord)
|
||
.map(([key, item]) => [key, normalizeUrl(item)] as const)
|
||
.filter(([, item]) => isHttpUrl(item))
|
||
);
|
||
}
|
||
|
||
async function main() {
|
||
const [dbModule, schemaModule, storageModule, tripoClient, tripoServer] =
|
||
await Promise.all([
|
||
import("../../lib/db"),
|
||
import("../../lib/db/schema"),
|
||
import("../../lib/storage/service"),
|
||
import("../../lib/tripo/client"),
|
||
import("../../lib/tripo/server"),
|
||
]);
|
||
const { db } = dbModule;
|
||
const { canvasNode, model3dAsset, storageConfig } = schemaModule;
|
||
const { toDbS3UploadConfig, uploadFileWithS3Config } = storageModule;
|
||
|
||
const qiniuRows = await db
|
||
.select()
|
||
.from(storageConfig)
|
||
.where(eq(storageConfig.type, "qiniu-kodo"))
|
||
.orderBy(desc(storageConfig.isDefault), desc(storageConfig.updatedAt));
|
||
const qiniuRow = qiniuRows.find((item) => item.isEnabled);
|
||
const qiniuConfig = toDbS3UploadConfig(qiniuRow);
|
||
if (!qiniuConfig || qiniuConfig.provider !== "qiniu-kodo") {
|
||
throw new Error("未找到已启用的七牛云 Kodo 配置，已拒绝使用其他存储");
|
||
}
|
||
|
||
let assets = await db
|
||
.select()
|
||
.from(model3dAsset)
|
||
.where(eq(model3dAsset.status, "success"))
|
||
.orderBy(desc(model3dAsset.createdAt));
|
||
if (worldOnly) assets = assets.filter((asset) => asset.provider === "worldlabs-3d");
|
||
if (tripoOnly) assets = assets.filter((asset) => asset.provider === "tripo-3d");
|
||
if (limit) assets = assets.slice(0, limit);
|
||
|
||
const backupPath = path.join(
|
||
process.cwd(),
|
||
"backups",
|
||
`model3d-assets-before-qiniu-repair-${new Date()
|
||
.toISOString()
|
||
.replace(/[:.]/g, "-")}.json`
|
||
);
|
||
if (!dryRun) {
|
||
await mkdir(path.dirname(backupPath), { recursive: true });
|
||
await writeFile(backupPath, JSON.stringify(assets, null, 2));
|
||
console.log(`[repair-3d] backup=${backupPath}`);
|
||
}
|
||
|
||
const summary = {
|
||
scanned: 0,
|
||
repairedAssets: 0,
|
||
repairedFiles: 0,
|
||
failedFiles: 0,
|
||
updatedNodes: 0,
|
||
};
|
||
|
||
for (const asset of assets) {
|
||
summary.scanned += 1;
|
||
const payload =
|
||
asset.assetPayload && typeof asset.assetPayload === "object"
|
||
? ({ ...asset.assetPayload } as JsonRecord)
|
||
: {};
|
||
const replacements = new Map<string, string>();
|
||
let nextModelUrl = normalizeUrl(asset.modelUrl) || null;
|
||
let nextPreviewUrl = normalizeUrl(asset.previewImageUrl) || null;
|
||
let freshTripoOutput: Record<string, unknown> | null = null;
|
||
|
||
const uploadRecoveredAsset = async (params: {
|
||
currentUrl?: string | null;
|
||
sourceUrl?: string | null;
|
||
kind: AssetKind;
|
||
label: string;
|
||
}) => {
|
||
const currentUrl = normalizeUrl(params.currentUrl);
|
||
if (currentUrl && (await probeAsset(currentUrl, params.kind))) return currentUrl;
|
||
|
||
let sourceUrl = normalizeUrl(params.sourceUrl);
|
||
if (
|
||
asset.provider === "tripo-3d" &&
|
||
(!sourceUrl || !(await probeAsset(sourceUrl, params.kind)))
|
||
) {
|
||
if (!freshTripoOutput) {
|
||
const preferredProviderId = normalizeUrl(payload.providerId);
|
||
const provider = await tripoServer.resolveTripoProviderConfigForTaskId(
|
||
asset.taskId,
|
||
preferredProviderId
|
||
);
|
||
const task = await tripoClient.tripoRequest<{
|
||
output?: Record<string, unknown>;
|
||
}>(provider, {
|
||
method: "GET",
|
||
path: `/task/${encodeURIComponent(asset.taskId)}`,
|
||
});
|
||
freshTripoOutput = task.data.output || {};
|
||
}
|
||
sourceUrl =
|
||
params.kind === "glb"
|
||
? normalizeUrl(tripoClient.pickTaskModelUrl(freshTripoOutput))
|
||
: normalizeUrl(tripoClient.pickTaskPreviewImageUrl(freshTripoOutput));
|
||
}
|
||
|
||
if (!sourceUrl || !(await probeAsset(sourceUrl, params.kind))) {
|
||
throw new Error(`${params.label} 没有可恢复的源文件`);
|
||
}
|
||
if (dryRun) return sourceUrl;
|
||
|
||
const downloaded = await downloadAsset(sourceUrl, params.kind);
|
||
const extension = resolveExtension(
|
||
sourceUrl,
|
||
params.kind,
|
||
downloaded.contentType
|
||
);
|
||
const uploaded = await uploadFileWithS3Config(
|
||
`repair-${asset.taskId}-${params.label}.${extension}`,
|
||
downloaded.buffer,
|
||
qiniuConfig
|
||
);
|
||
if (!(await probeAsset(uploaded.url, params.kind))) {
|
||
throw new Error(`${params.label} 上传七牛云后校验失败`);
|
||
}
|
||
if (currentUrl) replacements.set(currentUrl, uploaded.url);
|
||
if (sourceUrl) replacements.set(sourceUrl, uploaded.url);
|
||
summary.repairedFiles += 1;
|
||
console.log(
|
||
`[repair-3d] repaired task=${asset.taskId} file=${params.label} bytes=${downloaded.buffer.byteLength}`
|
||
);
|
||
return uploaded.url;
|
||
};
|
||
|
||
try {
|
||
if (asset.provider === "worldlabs-3d") {
|
||
nextModelUrl = await uploadRecoveredAsset({
|
||
currentUrl: nextModelUrl,
|
||
sourceUrl: normalizeUrl(payload.sourceMeshUrl),
|
||
kind: "glb",
|
||
label: "mesh",
|
||
}).catch((error) => {
|
||
summary.failedFiles += 1;
|
||
console.warn(`[repair-3d] ${asset.taskId} mesh: ${String(error)}`);
|
||
return nextModelUrl;
|
||
});
|
||
|
||
const nextThumbnailUrl = await uploadRecoveredAsset({
|
||
currentUrl: normalizeUrl(payload.thumbnailUrl),
|
||
sourceUrl: normalizeUrl(payload.sourceThumbnailUrl),
|
||
kind: "image",
|
||
label: "thumbnail",
|
||
}).catch((error) => {
|
||
summary.failedFiles += 1;
|
||
console.warn(`[repair-3d] ${asset.taskId} thumbnail: ${String(error)}`);
|
||
return normalizeUrl(payload.thumbnailUrl) || null;
|
||
});
|
||
const nextPanoUrl = await uploadRecoveredAsset({
|
||
currentUrl: normalizeUrl(payload.panoUrl),
|
||
sourceUrl: normalizeUrl(payload.sourcePanoUrl),
|
||
kind: "image",
|
||
label: "pano",
|
||
}).catch((error) => {
|
||
summary.failedFiles += 1;
|
||
console.warn(`[repair-3d] ${asset.taskId} pano: ${String(error)}`);
|
||
return normalizeUrl(payload.panoUrl) || null;
|
||
});
|
||
|
||
const currentSpzUrls = toStringRecord(payload.spzUrls);
|
||
const sourceSpzUrls = toStringRecord(payload.sourceSpzUrls);
|
||
const nextSpzUrls: StringRecord = {};
|
||
for (const [key, sourceUrl] of Object.entries(sourceSpzUrls)) {
|
||
const repairedUrl = await uploadRecoveredAsset({
|
||
currentUrl: currentSpzUrls[key],
|
||
sourceUrl,
|
||
kind: "spz",
|
||
label: `spz-${key}`,
|
||
}).catch((error) => {
|
||
summary.failedFiles += 1;
|
||
console.warn(`[repair-3d] ${asset.taskId} spz-${key}: ${String(error)}`);
|
||
return currentSpzUrls[key] || null;
|
||
});
|
||
if (repairedUrl) nextSpzUrls[key] = repairedUrl;
|
||
}
|
||
for (const [key, currentUrl] of Object.entries(currentSpzUrls)) {
|
||
if (!nextSpzUrls[key] && (await probeAsset(currentUrl, "spz"))) {
|
||
nextSpzUrls[key] = currentUrl;
|
||
}
|
||
}
|
||
|
||
payload.meshUrl = nextModelUrl;
|
||
payload.thumbnailUrl = nextThumbnailUrl;
|
||
payload.panoUrl = nextPanoUrl;
|
||
payload.spzUrls = nextSpzUrls;
|
||
nextPreviewUrl = nextThumbnailUrl || nextPanoUrl || nextPreviewUrl;
|
||
const allSpzRecovered =
|
||
Object.keys(sourceSpzUrls).length > 0 &&
|
||
Object.keys(nextSpzUrls).length === Object.keys(sourceSpzUrls).length;
|
||
payload.assetsArchivingPending = !allSpzRecovered;
|
||
payload.storageArchivedAt = allSpzRecovered ? new Date().toISOString() : null;
|
||
payload.spzArchiveValidatedAt = allSpzRecovered
|
||
? new Date().toISOString()
|
||
: null;
|
||
} else if (asset.provider === "tripo-3d") {
|
||
nextModelUrl = await uploadRecoveredAsset({
|
||
currentUrl: nextModelUrl,
|
||
sourceUrl: normalizeUrl(payload.sourceModelUrl),
|
||
kind: "glb",
|
||
label: "model",
|
||
});
|
||
nextPreviewUrl = await uploadRecoveredAsset({
|
||
currentUrl: nextPreviewUrl,
|
||
sourceUrl: normalizeUrl(payload.sourcePreviewImageUrl),
|
||
kind: "image",
|
||
label: "preview",
|
||
});
|
||
payload.storageArchivedAt = new Date().toISOString();
|
||
payload.assetsArchivingPending = false;
|
||
}
|
||
} catch (error) {
|
||
summary.failedFiles += 1;
|
||
console.warn(`[repair-3d] task=${asset.taskId} skipped: ${String(error)}`);
|
||
}
|
||
|
||
if (replacements.size === 0 || dryRun) continue;
|
||
|
||
await db.transaction(async (tx) => {
|
||
await tx
|
||
.update(model3dAsset)
|
||
.set({
|
||
modelUrl: nextModelUrl,
|
||
previewImageUrl: nextPreviewUrl,
|
||
assetPayload: replaceUrlsDeep(payload, replacements) as JsonRecord,
|
||
updatedAt: new Date(),
|
||
})
|
||
.where(eq(model3dAsset.id, asset.id));
|
||
|
||
const nodes = await tx
|
||
.select({ id: canvasNode.id, data: canvasNode.data })
|
||
.from(canvasNode)
|
||
.where(eq(canvasNode.projectId, asset.projectId));
|
||
for (const node of nodes) {
|
||
const serialized = JSON.stringify(node.data || {});
|
||
if (!serialized.includes(asset.taskId) && ![...replacements.keys()].some((url) => serialized.includes(url))) {
|
||
continue;
|
||
}
|
||
await tx
|
||
.update(canvasNode)
|
||
.set({
|
||
data: replaceUrlsDeep(node.data, replacements),
|
||
updatedAt: new Date(),
|
||
})
|
||
.where(eq(canvasNode.id, node.id));
|
||
summary.updatedNodes += 1;
|
||
}
|
||
});
|
||
summary.repairedAssets += 1;
|
||
}
|
||
|
||
console.log(JSON.stringify({ dryRun, summary }, null, 2));
|
||
}
|
||
|
||
main().catch((error) => {
|
||
console.error(error);
|
||
process.exit(1);
|
||
});
|