gongxiangp2/scripts/audit-legacy-material-previews.mjs

// Opt-in, read-only audit of one exact registered legacy dependency inventory.
// No database/object writes, source substitution, retries or original-file GETs.
import { createHash } from 'node:crypto'
import * as fs from 'node:fs/promises'
import { spawn } from 'node:child_process'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
import { checkMaterialPreviewHealth, directFetch, LIMITS, registeredSource } from './check-material-preview-health.mjs'
import { LEGACY_DOMAINS } from './check-legacy-storage-boundaries.mjs'

export const AUDIT_LIMITS = Object.freeze({ expectedCount: 472, batchSize: 50, concurrency: 2 })
const sha = value => createHash('sha256').update(value).digest('hex')
const assert = (value, code) => { if (!value) throw new Error(code) }
const previewRoles = new Set(['preview', 'main-preview', 'sku-preview', 'detail-preview', 'report-image'])
const imageExtension = /\.(png|jpe?g|webp)$/i

export function metadataCandidates(row) {
  const candidates = [], excludedAssets = []
  const add = (source, origin, identity, size) => {
    let url
    try { url = new URL(source) } catch { return }
    if (!['https:', 'http:'].includes(url.protocol) || url.username || url.password || !imageExtension.test(url.pathname)) return
    let publicCheckable = false
    try { registeredSource(source); publicCheckable = true } catch {}
    candidates.push({ source, origin, identity, ...(size === undefined ? {} : { declaredBytes: String(size) }),
      sameAsRegistered: source === row.previewUrl, legacyHost: LEGACY_DOMAINS.includes(url.hostname), publicCheckable,
      metadataOnly: true, bytesRead: 0, originalIdentityVerified: false })
  }
  for (const [index, source] of (row.displayImages || []).entries()) add(source, 'displayImages', String(index))
  for (const asset of row.assets || []) {
    if (!previewRoles.has(asset.assetType)) {
      excludedAssets.push({ assetId: asset.id, assetType: asset.assetType, reason: 'not_a_preview_role', bytesRead: 0 })
      continue
    }
    const before = candidates.length
    add(asset.fileUrl, 'MaterialAsset', asset.id, asset.fileSize)
    if (before === candidates.length) excludedAssets.push({ assetId: asset.id, assetType: asset.assetType, reason: 'not_a_supported_image_url', bytesRead: 0 })
  }
  for (const sku of row.skus || []) if (sku.previewUrl) add(sku.previewUrl, 'MaterialSku.previewUrl', sku.id)
  return { materialId: row.id, materialType: row.materialType, candidates, excludedAssets }
}

export function validateAuditSnapshot(snapshot, { host, expectedCount = AUDIT_LIMITS.expectedCount } = {}) {
  assert(LEGACY_DOMAINS.includes(host), 'registered_legacy_host_required')
  assert(snapshot?.readOnly === true && snapshot.count === expectedCount && Array.isArray(snapshot.materials)
    && snapshot.materials.length === expectedCount, 'exact_inventory_count_required')
  const seen = new Set()
  for (const row of snapshot.materials) {
    assert(typeof row.id === 'string' && /^[A-Za-z0-9._-]{1,128}$/.test(row.id) && !seen.has(row.id), 'unique_material_identity_required')
    seen.add(row.id)
    assert(typeof row.previewUrl === 'string', 'source_required')
    const url = new URL(row.previewUrl)
    assert(url.hostname === host && ['https:', 'http:'].includes(url.protocol), 'inventory_host_changed')
  }
  return snapshot.materials
}

// Hash every fully consumed response, including the explicit FRP HTML 404 body.
// The checker still owns the byte/time budgets and cancels rejected streams.
export async function auditPreviewBatch(manifest, { fetchImpl = directFetch, ...options } = {}) {
  const proofs = new Map()
  const capture = async (url, fetchOptions) => {
    const response = await fetchImpl(url, fetchOptions)
    if (!response.body) return response
    const reader = response.body.getReader(), digest = createHash('sha256')
    let count = 0
    const stream = new ReadableStream({
      async pull(controller) {
        try {
          const item = await reader.read()
          if (item.done) {
            proofs.set(url, { responseBodySha256: digest.digest('hex'), responseBytes: count, responseBodyComplete: true })
            controller.close()
          } else {
            count += item.value.byteLength
            digest.update(item.value)
            controller.enqueue(item.value)
          }
        } catch (error) { controller.error(error) }
      },
      cancel(reason) { return reader.cancel(reason) },
    })
    const wrapped = new Response(stream, { status: response.status, statusText: response.statusText, headers: response.headers })
    Object.defineProperties(wrapped, { url: { value: response.url }, redirected: { value: response.redirected } })
    return wrapped
  }
  const report = await checkMaterialPreviewHealth(manifest, { ...options, fetchImpl: capture })
  return { ...report, results: report.results.map(result => ({ ...result,
    ...(proofs.get(manifest.materials.find(entry => entry.materialId === result.materialId).registeredSource)
      || { responseBodySha256: null, responseBodyComplete: false }) })) }
}

export async function auditUniquePreviews(materials, { onBatch = async () => {}, ...options } = {}) {
  const byUrl = new Map(), rejected = []
  for (const row of materials) {
    try {
      const source = registeredSource(row.previewUrl).href
      if (!byUrl.has(source)) byUrl.set(source, { materialId: row.id, registeredSource: source, materialIds: [] })
      byUrl.get(source).materialIds.push(row.id)
    } catch (error) {
      rejected.push({ materialId: row.id, ok: false, classification: 'source_rejected', code: error.code || 'invalid_url', attempts: 0 })
    }
  }
  const entries = [...byUrl.values()], batches = []
  for (let index = 0; index < entries.length; index += AUDIT_LIMITS.batchSize) {
    batches.push({ version: 1, reviewed: true, materials: entries.slice(index, index + AUDIT_LIMITS.batchSize)
      .map(({ materialId, registeredSource }) => ({ materialId, registeredSource })) })
  }
  const reports = new Array(batches.length)
  let next = 0
  await Promise.all(Array.from({ length: Math.min(AUDIT_LIMITS.concurrency, batches.length) }, async () => {
    while (next < batches.length) {
      const index = next++
      reports[index] = await auditPreviewBatch(batches[index], options)
      await onBatch({ index, manifest: batches[index], report: reports[index] })
    }
  }))
  const byRepresentative = new Map(reports.flatMap(report => report.results).map(result => [result.materialId, result]))
  const byId = new Map(rejected.map(result => [result.materialId, result]))
  for (const entry of entries) for (const materialId of entry.materialIds) {
    byId.set(materialId, { ...byRepresentative.get(entry.materialId), materialId, checkedRepresentativeId: entry.materialId,
      sharedRegisteredSource: entry.materialIds.length > 1 })
  }
  return { readOnly: true, limits: AUDIT_LIMITS, materialCount: materials.length, uniqueRegisteredSources: entries.length,
    actualGetAttempts: entries.length, rejectedBeforeGet: rejected.length, retries: 0, allRowsAccounted: byId.size === materials.length,
    deduplication: entries, batches, reports, results: materials.map(row => ({ ...byId.get(row.id), materialType: row.materialType })) }
}

async function sshStdin(command, input, onLine) {
  return new Promise((resolve, reject) => {
    const child = spawn('ssh.exe', ['-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes', '-o',
      'UserKnownHostsFile=C:/Users/54972/.ssh/gongxiangp2-prod.known_hosts', 'gongxiangp2-prod-legacy-freenas', command],
    { stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true })
    let output = '', errors = '', pending = '', lineFailure
    child.stdout.on('data', bytes => {
      output += bytes
      if (Buffer.byteLength(output) > 32 * 1024 * 1024) { child.kill(); lineFailure = new Error('ssh_output_budget_exceeded'); return }
      if (onLine) {
        pending += bytes
        let end
        while ((end = pending.indexOf('\n')) !== -1) {
          const line = pending.slice(0, end); pending = pending.slice(end + 1)
          if (line.trim()) { try { onLine(JSON.parse(line)) } catch (error) { lineFailure = error; child.kill() } }
        }
      }
    })
    child.stderr.on('data', bytes => { if (errors.length < 4000) errors += bytes })
    child.on('error', reject)
    child.on('close', code => lineFailure ? reject(lineFailure) : code === 0 ? resolve(output) : reject(new Error('read_only_remote_audit_failed')))
    child.stdin.on('error', () => {})
    child.stdin.end(input)
  })
}

export async function main(args = process.argv.slice(2)) {
  if (!args.includes('--run')) return { planOnly: true, readOnly: true, expectedCount: AUDIT_LIMITS.expectedCount,
    metadata: ['Material.id', 'materialType', 'previewUrl', 'displayImages', 'MaterialAsset preview metadata', 'active MaterialSku.previewUrl'],
    health: { ...AUDIT_LIMITS, ...LIMITS }, modelCalls: 0, productionWrites: 0 }
  assert(args.length === 3 && args[0] === '--run' && args[1] === '--output-dir' && path.isAbsolute(args[2]), 'run_requires_new_absolute_output_directory')
  const directory = path.resolve(args[2])
  await fs.mkdir(directory, { recursive: false, mode: 0o700 })
  const write = (file, value) => fs.writeFile(path.join(directory, file), JSON.stringify(value, null, 2) + '\n', { flag: 'wx', mode: 0o600 })
  const receipt = JSON.parse(await fs.readFile('output/native-brief-panel-20261023/preview-recovery-reviewed-manifest.json', 'utf8'))
  const host = new URL(receipt.materials[0].oldPreviewUrl).hostname
  assert(LEGACY_DOMAINS.includes(host), 'reviewed_legacy_host_required')
  const identityWire = `import json,subprocess\nc=json.loads(subprocess.check_output(['docker','inspect','youyidian-3d-material-blue']))[0]\nf=subprocess.check_output(['docker','exec',c['Id'],'cat','/app/DEPLOYMENT_COMMIT'],text=True).strip()\nprint(json.dumps({'containerId':c['Id'],'containerName':c['Name'].lstrip('/'),'imageName':c['Config']['Image'],'imageId':c['Image'],'running':c['State']['Running'],'deploymentCommit':f}))\n`
  const identity = JSON.parse(await sshStdin('python3 -u -', identityWire))
  assert(identity.running && /^[a-f0-9]{64}$/.test(identity.containerId) && /^[a-f0-9]{40}$/.test(identity.deploymentCommit), 'running_production_identity_required')
  await write('production-identity.json', { readOnly: true, checkedAt: new Date().toISOString(), ...identity })
  const command = `docker exec -i -w /app ${identity.containerId} node --input-type=module`
  const metadataWire = `import {createRequire} from 'node:module';const require=createRequire(import.meta.url),fs=require('node:fs'),assert=require('node:assert/strict');
assert.equal(fs.readFileSync('/app/DEPLOYMENT_COMMIT','utf8').trim(),${JSON.stringify(identity.deploymentCommit)});
const {PrismaClient}=require('@prisma/client');const p=new PrismaClient({log:[]});
try{const result=await p.$transaction(async tx=>{
 await tx.$executeRawUnsafe('SET TRANSACTION READ ONLY');
 const where={deletedAt:null,OR:[{previewUrl:{startsWith:${JSON.stringify('https://' + host + '/')},mode:'insensitive'}},{previewUrl:{startsWith:${JSON.stringify('http://' + host + '/')},mode:'insensitive'}}]};
 const count=await tx.material.count({where});assert.equal(count,${AUDIT_LIMITS.expectedCount});const materials=[];let cursor;
 while(true){const rows=await tx.material.findMany({where,orderBy:{id:'asc'},take:100,...(cursor?{cursor:{id:cursor},skip:1}:{}),select:{id:true,materialType:true,previewUrl:true,displayImages:true,
 assets:{orderBy:[{sortOrder:'asc'},{id:'asc'}],select:{id:true,assetType:true,fileUrl:true,fileSize:true,sortOrder:true}},
 skus:{where:{deletedAt:null},orderBy:{id:'asc'},select:{id:true,previewUrl:true}}}});if(!rows.length)break;materials.push(...rows);assert.ok(materials.length<=count);cursor=rows.at(-1).id;}
 assert.equal(materials.length,count);return{readOnly:true,capturedAt:new Date().toISOString(),count,host:${JSON.stringify(host)},isolationLevel:'RepeatableRead',historyImportFilter:false,materials};
},{isolationLevel:'RepeatableRead',timeout:60000,maxWait:5000});console.log(JSON.stringify(result,(_,v)=>typeof v==='bigint'?v.toString():v));}finally{await p.$disconnect();}`
  const snapshot = JSON.parse(await sshStdin(command, metadataWire))
  const materials = validateAuditSnapshot(snapshot, { host })
  await write('material-metadata.json', snapshot)
  const candidateMapping = materials.map(metadataCandidates)
  await write('preview-candidate-mapping.json', { readOnly: true, metadataOnly: true, candidateGets: 0, materials: candidateMapping })
  const checker = await fs.readFile('scripts/check-material-preview-health.mjs', 'utf8')
  const own = await fs.readFile(fileURLToPath(import.meta.url), 'utf8')
  const checkSource = checker.replace("import { LEGACY_DOMAINS } from './check-legacy-storage-boundaries.mjs'", `const LEGACY_DOMAINS=${JSON.stringify(LEGACY_DOMAINS)}`)
  const helperSource = own.slice(own.indexOf('export async function auditPreviewBatch'), own.indexOf('async function sshStdin'))
  const wire = checkSource + `\nconst AUDIT_LIMITS=${JSON.stringify(AUDIT_LIMITS)};\n` + helperSource + `
const fsRuntime=createRequire(import.meta.url)('node:fs');
if(fsRuntime.readFileSync('/app/DEPLOYMENT_COMMIT','utf8').trim()!==${JSON.stringify(identity.deploymentCommit)})throw Error('production_changed');
console.log(JSON.stringify({event:'started',expectedCount:${materials.length},checkerSha256:${JSON.stringify(sha(checker))}}));
const audit=await auditUniquePreviews(${JSON.stringify(materials.map(({ id, materialType, previewUrl }) => ({ id, materialType, previewUrl })))},
 {onBatch:async value=>console.log(JSON.stringify({event:'batch',...value}))});
if(fsRuntime.readFileSync('/app/DEPLOYMENT_COMMIT','utf8').trim()!==${JSON.stringify(identity.deploymentCommit)})throw Error('production_changed');
console.log(JSON.stringify({event:'complete',audit}));`
  await write('dispatch-audit.json', { readOnly: true, startedAt: new Date().toISOString(), host, materialCount: materials.length,
    checkerSha256: sha(checker), helperSha256: sha(own), wireSha256: sha(wire), maxConcurrentGets: 2, retries: 0, productionWrites: 0, candidateGets: 0 })
  const batchWrites = []
  let completed
  try {
    await sshStdin(command, wire, event => {
      if (event.event === 'batch') batchWrites.push(write(`batch-${String(event.index + 1).padStart(3, '0')}.json`, event))
      else if (event.event === 'complete') completed = event.audit
    })
    await Promise.all(batchWrites)
    assert(completed?.allRowsAccounted && completed.results.length === materials.length, 'complete_response_mapping_required')
    const summary = { readOnly: true, completedAt: new Date().toISOString(), ...identity, materialCount: materials.length,
      uniqueRegisteredSources: completed.uniqueRegisteredSources, actualGetAttempts: completed.actualGetAttempts,
      results: completed.results, healthyCount: completed.results.filter(item => item.ok).length,
      failedCount: completed.results.filter(item => !item.ok).length, statusCounts: {}, classificationCounts: {},
      candidateMetadataCount: candidateMapping.reduce((n, row) => n + row.candidates.length, 0), candidateGets: 0,
      allRowsAccounted: true, fullImageAuditExecuted: true, retries: 0, databaseWrites: 0, storageWrites: 0, modelCalls: 0 }
    for (const item of completed.results) {
      summary.statusCounts[item.status ?? 'not_dispatched'] = (summary.statusCounts[item.status ?? 'not_dispatched'] || 0) + 1
      summary.classificationCounts[item.classification] = (summary.classificationCounts[item.classification] || 0) + 1
    }
    await write('unique-url-audit.json', completed)
    await write('audit-summary.json', summary)
    return { ...summary, results: undefined, outputDirectory: directory }
  } catch (error) {
    await Promise.allSettled(batchWrites)
    await write('audit-incomplete.json', { readOnly: true, complete: false, code: 'read_only_audit_incomplete', completedBatches: batchWrites.length,
      noAutomaticRetry: true, metadataCount: materials.length })
    throw error
  }
}

if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
  main().then(result => console.log(JSON.stringify(result, null, 2)), () => {
    console.log(JSON.stringify({ readOnly: true, complete: false, code: 'read_only_audit_failed', retries: 0 }))
    process.exitCode = 1
  })
}