gongxiangp2/scripts/export-material-preview-health-manifest.mjs

142 lines
8.7 KiB
JavaScript

// Read-only inventory of every active Material.previewUrl, not HISTORY_IMPORT.
// Public-source probes remain separate; an uncheckable row is never omitted or healthy.
import { createHash } from 'node:crypto'
import * as fs from 'node:fs/promises'
import { createRequire } from 'node:module'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
import { LIMITS, registeredSource } from './check-material-preview-health.mjs'
import { LEGACY_DOMAINS } from './check-legacy-storage-boundaries.mjs'
const sha256 = value => createHash('sha256').update(value).digest('hex')
const runtimeRequire = createRequire(import.meta.url)
export class InventoryError extends Error {
constructor(code) { super(code); this.code = code }
}
function optionsChecked({ pageSize = 500, batchSize = LIMITS.materials, baselineHost, baselineCount } = {}) {
if (!Number.isSafeInteger(pageSize) || pageSize < 1 || pageSize > 1000
|| !Number.isSafeInteger(batchSize) || batchSize < 1 || batchSize > LIMITS.materials) throw new InventoryError('invalid_batch_budget')
if ((baselineHost === undefined) !== (baselineCount === undefined)
|| (baselineHost !== undefined && (!LEGACY_DOMAINS.includes(baselineHost) || !Number.isSafeInteger(baselineCount) || baselineCount < 0))) {
throw new InventoryError('invalid_legacy_baseline')
}
return { pageSize, batchSize, baselineHost, baselineCount }
}
export async function collectMaterialPreviewInventory(prisma, options = {}) {
const { pageSize, batchSize, baselineHost, baselineCount } = optionsChecked(options)
if (typeof prisma?.$transaction !== 'function') throw new InventoryError('database_reader_required')
return prisma.$transaction(async tx => {
await tx.$executeRawUnsafe('SET TRANSACTION READ ONLY')
const activeCount = await tx.material.count({ where: { deletedAt: null } })
if (!Number.isSafeInteger(activeCount) || activeCount < 0) throw new InventoryError('invalid_active_count')
const materials = [], uncheckable = [], seen = new Set(), materialTypeCounts = Object.create(null), legacyHostCounts = Object.fromEntries(LEGACY_DOMAINS.map(h => [h, 0]))
const inventoryDigest = createHash('sha256')
let cursor, pages = 0
while (true) {
const rows = await tx.material.findMany({
where: { deletedAt: null }, select: { id: true, materialType: true, previewUrl: true },
orderBy: { id: 'asc' }, take: pageSize, ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}),
})
pages++
if (!Array.isArray(rows) || rows.length > pageSize) throw new InventoryError('invalid_inventory_page')
if (!rows.length) break
for (const row of rows) {
if (typeof row.id !== 'string' || !/^[A-Za-z0-9._-]{1,128}$/.test(row.id) || seen.has(row.id)) throw new InventoryError('duplicate_or_invalid_inventory_identity')
seen.add(row.id)
if (seen.size > activeCount) throw new InventoryError('inventory_count_mismatch')
const type = typeof row.materialType === 'string' ? row.materialType : '<unknown>'
materialTypeCounts[type] = (materialTypeCounts[type] || 0) + 1
const source = row.previewUrl
inventoryDigest.update(JSON.stringify([row.id, source ?? null]) + '\n')
if (typeof source === 'string') {
for (const host of LEGACY_DOMAINS) {
if ([`https://${host}/`, `http://${host}/`].some(prefix => source.toLowerCase().startsWith(prefix))) legacyHostCounts[host]++
}
}
try {
const url = registeredSource(source)
materials.push({ materialId: row.id, registeredSource: url.href })
} catch (error) {
uncheckable.push({ materialId: row.id, materialType: type, sourceSha256: sha256(typeof source === 'string' ? source : ''),
classification: error?.classification || 'source_rejected', code: error?.code || 'invalid_url' })
}
}
const next = rows.at(-1).id
if (next === cursor) throw new InventoryError('inventory_cursor_stalled')
cursor = next
}
if (seen.size !== activeCount || materials.length + uncheckable.length !== activeCount) throw new InventoryError('inventory_count_mismatch')
const manifests = []
for (let i = 0; i < materials.length; i += batchSize) {
manifests.push({ version: 1, reviewed: true, provenance: 'read-only-active-material-inventory', materials: materials.slice(i, i + batchSize) })
}
const audit = { version: 1, readOnly: true, capturedAt: new Date().toISOString(), inventoryComplete: true,
scope: { table: 'Material', where: { deletedAt: null }, historyImportFilter: false },
transaction: { isolationLevel: 'RepeatableRead', readOnly: true },
activeCount, exportedCount: materials.length, uncheckableCount: uncheckable.length, materialTypeCounts, legacyHostCounts,
...(baselineHost === undefined ? {} : { legacyBaseline: { host: baselineHost, expectedCount: baselineCount, actualCount: legacyHostCounts[baselineHost],
delta: legacyHostCounts[baselineHost] - baselineCount, matches: legacyHostCounts[baselineHost] === baselineCount } }),
inventorySha256: inventoryDigest.digest('hex'), pages, batchSize, batchCount: manifests.length, uncheckable,
readyForHealthChecks: uncheckable.length === 0, healthChecksExecuted: false }
return { audit, manifests }
}, { isolationLevel: 'RepeatableRead', timeout: 60_000, maxWait: 5000 })
}
export async function writeInventoryArtifacts(outputDirectory, inventory, { fsImpl = fs } = {}) {
if (typeof outputDirectory !== 'string' || !path.isAbsolute(outputDirectory)) throw new InventoryError('absolute_new_output_directory_required')
if (!inventory?.audit?.inventoryComplete || !Array.isArray(inventory.manifests)
|| inventory.audit.batchCount !== inventory.manifests.length) throw new InventoryError('complete_inventory_required')
const output = path.resolve(outputDirectory)
// A fresh directory and exclusive files prevent stale acceptance or accidental overwrite.
await fsImpl.mkdir(output, { mode: 0o700, recursive: false })
const batches = []
for (let i = 0; i < inventory.manifests.length; i++) {
const file = `manifest-${String(i + 1).padStart(6, '0')}.json`
const raw = JSON.stringify(inventory.manifests[i], null, 2) + '\n'
await fsImpl.writeFile(path.join(output, file), raw, { flag: 'wx', mode: 0o600 })
batches.push({ file, sha256: sha256(raw), manifestSha256: sha256(JSON.stringify(inventory.manifests[i])), materialCount: inventory.manifests[i].materials.length })
}
const audit = { ...inventory.audit, batches }
// Written last: incomplete directory contents are never a complete inventory receipt.
await fsImpl.writeFile(path.join(output, 'audit.json'), JSON.stringify(audit, null, 2) + '\n', { flag: 'wx', mode: 0o600 })
return { outputDirectory: output, audit }
}
export async function main(args = process.argv.slice(2), { prismaClient, fsImpl = fs } = {}) {
const options = {}
for (let i = 0; i < args.length; i += 2) {
if (!['--output-dir', '--baseline-host', '--baseline-count'].includes(args[i]) || args[i + 1] === undefined || options[args[i]] !== undefined) {
throw new InventoryError('usage_output_directory_required')
}
options[args[i]] = args[i + 1]
}
if (!options['--output-dir']) throw new InventoryError('usage_output_directory_required')
if (!path.isAbsolute(options['--output-dir'])) throw new InventoryError('absolute_new_output_directory_required')
const baseline = options['--baseline-host'] === undefined && options['--baseline-count'] === undefined ? {} : {
baselineHost: options['--baseline-host'], baselineCount: Number(options['--baseline-count']),
}
optionsChecked(baseline)
const owned = !prismaClient
const prisma = prismaClient || new (runtimeRequire('@prisma/client').PrismaClient)({ log: [] })
try {
const inventory = await collectMaterialPreviewInventory(prisma, baseline)
return await writeInventoryArtifacts(options['--output-dir'], inventory, { fsImpl })
} finally {
if (owned) await prisma.$disconnect()
}
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
main().then(({ outputDirectory, audit }) => {
console.log(JSON.stringify({ version: 1, readOnly: true, outputDirectory, inventoryComplete: audit.inventoryComplete,
activeCount: audit.activeCount, exportedCount: audit.exportedCount, uncheckableCount: audit.uncheckableCount,
batchCount: audit.batchCount, legacyHostCounts: audit.legacyHostCounts, legacyBaseline: audit.legacyBaseline, healthChecksExecuted: false }, null, 2))
process.exitCode = audit.readyForHealthChecks ? 0 : 1
}, error => {
console.log(JSON.stringify({ version: 1, readOnly: true, inventoryComplete: false,
code: error instanceof InventoryError ? error.code : 'inventory_export_failed' }, null, 2))
process.exitCode = 1
})
}