WuYongJie
a890660853
Preserve known legacy failures by reviewed identity, probe material delivery before and after releases, and retain originals and downloads.
292 lines
14 KiB
JavaScript
292 lines
14 KiB
JavaScript
// Standalone, opt-in read-only acceptance for reviewed, registered material URLs.
|
|
// LEGACY_READ_ONLY: reuse the project's historical host registry without rewriting URLs.
|
|
// node scripts/check-material-preview-health.mjs --manifest <reviewed.json>
|
|
// {"version":1,"reviewed":true,"materials":[{"materialId":"<id>","registeredSource":"<registered object URL>"}]}
|
|
import { createHash } from 'node:crypto'
|
|
import { lookup } from 'node:dns/promises'
|
|
import { open } from 'node:fs/promises'
|
|
import http from 'node:http'
|
|
import https from 'node:https'
|
|
import { isIP } from 'node:net'
|
|
import { createRequire } from 'node:module'
|
|
import path from 'node:path'
|
|
import { Readable } from 'node:stream'
|
|
import { fileURLToPath } from 'node:url'
|
|
import { LEGACY_DOMAINS } from './check-legacy-storage-boundaries.mjs'
|
|
|
|
export const LIMITS = Object.freeze({ timeoutMs: 10_000, maxBytes: 8 * 1024 * 1024, maxPixels: 25_000_000, materials: 50 })
|
|
const CURRENT_HOST = 's3.jiansun.vip'
|
|
const HOSTS = new Set([CURRENT_HOST, ...LEGACY_DOMAINS])
|
|
const MIME_FORMATS = new Map([['image/png', 'png'], ['image/jpeg', 'jpeg'], ['image/jpg', 'jpeg'], ['image/webp', 'webp']])
|
|
const sha256 = value => createHash('sha256').update(value).digest('hex')
|
|
|
|
class CheckError extends Error {
|
|
constructor(classification, code, materialId) {
|
|
super(code)
|
|
this.classification = classification
|
|
this.code = code
|
|
this.materialId = materialId
|
|
}
|
|
}
|
|
|
|
export function registeredSource(value) {
|
|
if (typeof value !== 'string' || value.length > 4096 || value.trim() !== value || /[\r\n\0]/.test(value)) {
|
|
throw new CheckError('source_rejected', 'invalid_url')
|
|
}
|
|
let url
|
|
try { url = new URL(value) } catch { throw new CheckError('source_rejected', 'invalid_url') }
|
|
if (!HOSTS.has(url.hostname) || url.username || url.password || url.port || url.hash) {
|
|
throw new CheckError('source_rejected', 'unregistered_source')
|
|
}
|
|
if (url.protocol !== 'https:' && !(url.protocol === 'http:' && LEGACY_DOMAINS.includes(url.hostname))) {
|
|
throw new CheckError('source_rejected', 'unregistered_protocol')
|
|
}
|
|
if (!url.pathname || url.pathname === '/') throw new CheckError('source_rejected', 'object_path_required')
|
|
return url
|
|
}
|
|
|
|
export function validateManifest(manifest) {
|
|
if (manifest?.version !== 1 || manifest.reviewed !== true || !Array.isArray(manifest.materials)
|
|
|| manifest.materials.length < 1 || manifest.materials.length > LIMITS.materials) {
|
|
throw new CheckError('invalid_manifest', 'reviewed_manifest_required')
|
|
}
|
|
const identities = new Set()
|
|
return manifest.materials.map(item => {
|
|
if (typeof item?.materialId !== 'string' || !/^[A-Za-z0-9._-]{1,128}$/.test(item.materialId) || identities.has(item.materialId)) {
|
|
throw new CheckError('invalid_manifest', 'unique_material_id_required')
|
|
}
|
|
identities.add(item.materialId)
|
|
try {
|
|
const url = registeredSource(item.registeredSource)
|
|
return { materialId: item.materialId, url }
|
|
} catch (error) {
|
|
error.materialId = item.materialId
|
|
throw error
|
|
}
|
|
})
|
|
}
|
|
|
|
// A reviewed hostname must not be rebound to local/private/metadata networks.
|
|
// The direct Node transport resolves once per connection and uses only these
|
|
// checked addresses; environment proxy variables and redirects are not followed.
|
|
export function isPublicAddress(address) {
|
|
if (isIP(address) === 4) {
|
|
const [a, b, c] = address.split('.').map(Number)
|
|
return !(a === 0 || a === 10 || a === 127 || a >= 224
|
|
|| (a === 100 && b >= 64 && b <= 127)
|
|
|| (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31)
|
|
|| (a === 192 && (b === 168 || (b === 0 && (c === 0 || c === 2))))
|
|
|| (a === 198 && (b === 18 || b === 19 || (b === 51 && c === 100)))
|
|
|| (a === 203 && b === 0 && c === 113))
|
|
}
|
|
if (isIP(address) === 6) {
|
|
// Only global unicast; this also rejects mapped IPv4, loopback and link-local.
|
|
const first = Number.parseInt(address.split(':')[0], 16)
|
|
return first >= 0x2000 && first <= 0x3fff && first !== 0x2002 && !/^2001:(?:0*:|0*db8:)/i.test(address)
|
|
}
|
|
return false
|
|
}
|
|
|
|
export function directFetch(input, { signal, headers, method, cache, redirect } = {}) {
|
|
const url = registeredSource(String(input))
|
|
if (method !== 'GET' || cache !== 'no-store' || redirect !== 'manual') {
|
|
throw new CheckError('source_rejected', 'readonly_transport_required')
|
|
}
|
|
return new Promise((resolve, reject) => {
|
|
const transport = url.protocol === 'https:' ? https : http
|
|
const request = transport.request(url, {
|
|
method: 'GET', signal, headers,
|
|
lookup(hostname, options, callback) {
|
|
lookup(hostname, { all: true, verbatim: true }).then(addresses => {
|
|
if (hostname !== url.hostname || !addresses.length || addresses.some(row => !isPublicAddress(row.address))) {
|
|
callback(new CheckError('source_rejected', 'non_public_dns_address'))
|
|
} else if (options?.all) {
|
|
callback(null, addresses)
|
|
} else {
|
|
callback(null, addresses[0].address, addresses[0].family)
|
|
}
|
|
}, () => callback(new CheckError('network_failure', 'dns_lookup_failed')))
|
|
},
|
|
}, response => {
|
|
try {
|
|
const noBody = [204, 205, 304].includes(response.statusCode)
|
|
if (noBody) response.destroy()
|
|
resolve(new Response(noBody ? null : Readable.toWeb(response), {
|
|
status: response.statusCode,
|
|
headers: response.headers,
|
|
}))
|
|
} catch {
|
|
response.destroy()
|
|
reject(new CheckError('network_failure', 'invalid_http_response'))
|
|
}
|
|
})
|
|
request.on('error', reject)
|
|
request.end()
|
|
})
|
|
}
|
|
|
|
function formatOf(bytes) {
|
|
if (bytes.length >= 8 && bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) return 'png'
|
|
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'jpeg'
|
|
if (bytes.length >= 16 && bytes.toString('ascii', 0, 4) === 'RIFF' && bytes.toString('ascii', 8, 12) === 'WEBP'
|
|
&& ['VP8 ', 'VP8L', 'VP8X'].includes(bytes.toString('ascii', 12, 16))) return 'webp'
|
|
return null
|
|
}
|
|
|
|
/** Lazy CJS loading also works when this module's source is run through Node stdin. */
|
|
export function createSharpPreviewDecoder({ requireImpl = createRequire(import.meta.url) } = {}) {
|
|
let sharp
|
|
return async bytes => {
|
|
if (!sharp) {
|
|
try { sharp = requireImpl('sharp') } catch { throw new CheckError('decoder_unavailable', 'image_decoder_unavailable') }
|
|
}
|
|
try {
|
|
const image = sharp(bytes, { failOn: 'error', limitInputPixels: LIMITS.maxPixels, animated: true })
|
|
const metadata = await image.metadata()
|
|
// metadata() alone only parses the header. Materialise pixels so a
|
|
// truncated/corrupt PNG, JPEG or WebP cannot pass from its magic bytes.
|
|
const { info } = await image.raw().toBuffer({ resolveWithObject: true })
|
|
return { format: metadata.format, width: info.width, height: info.height }
|
|
} catch (error) {
|
|
if (error instanceof CheckError) throw error
|
|
throw new CheckError('unexpected_content', 'image_decode_failed')
|
|
}
|
|
}
|
|
}
|
|
const defaultImageDecoder = createSharpPreviewDecoder()
|
|
|
|
function sourceReference(url) {
|
|
return { origin: url.origin, pathname: url.pathname, queryRedacted: Boolean(url.search), sha256: sha256(url.href) }
|
|
}
|
|
|
|
function withAbort(promise, signal) {
|
|
if (signal.aborted) return Promise.reject(signal.reason)
|
|
return new Promise((resolve, reject) => {
|
|
const abort = () => reject(signal.reason)
|
|
signal.addEventListener('abort', abort, { once: true })
|
|
Promise.resolve(promise).then(resolve, reject).finally(() => signal.removeEventListener('abort', abort))
|
|
})
|
|
}
|
|
|
|
async function checkOne(entry, { fetchImpl, decodeImage, timeoutMs, maxBytes }) {
|
|
const base = { materialId: entry.materialId, source: sourceReference(entry.url), attempts: 1 }
|
|
const controller = new AbortController()
|
|
const timer = setTimeout(() => controller.abort(new CheckError('timeout', 'request_timeout')), timeoutMs)
|
|
let reader
|
|
let body
|
|
let status = null
|
|
let contentType = ''
|
|
let bytesRead = 0
|
|
let bodySha256
|
|
const failure = (classification, code) => ({ ...base, ok: false, classification, code, status, contentType, bytes: bytesRead,
|
|
...(bodySha256 ? { bodySha256 } : {}) })
|
|
try {
|
|
const response = await withAbort(fetchImpl(entry.url.href, {
|
|
method: 'GET', cache: 'no-store', redirect: 'manual', signal: controller.signal,
|
|
headers: { 'Cache-Control': 'no-store, no-cache, max-age=0', Pragma: 'no-cache', Accept: 'image/png,image/jpeg,image/webp' },
|
|
}), controller.signal)
|
|
status = response.status
|
|
body = response.body
|
|
contentType = (response.headers.get('content-type') || '').split(';')[0].trim().toLowerCase()
|
|
if (response.redirected || (status >= 300 && status < 400)) return failure('redirect_blocked', 'redirect_not_followed')
|
|
if (response.url && response.url !== entry.url.href) return failure('redirect_blocked', 'response_url_changed')
|
|
const length = response.headers.get('content-length')
|
|
if (length && /^\d+$/.test(length) && Number(length) > maxBytes) return failure('too_large', 'declared_body_limit')
|
|
const chunks = []
|
|
reader = response.body?.getReader()
|
|
while (reader) {
|
|
const { done, value } = await withAbort(reader.read(), controller.signal)
|
|
if (done) break
|
|
bytesRead += value.byteLength
|
|
if (bytesRead > maxBytes) return failure('too_large', 'stream_body_limit')
|
|
chunks.push(Buffer.from(value))
|
|
}
|
|
const bytes = Buffer.concat(chunks)
|
|
// A fully read failure body can identify the known incident without storing
|
|
// its contents. Partial, oversized and unavailable bodies have no digest.
|
|
bodySha256 = sha256(bytes)
|
|
const prefix = bytes.subarray(0, 4096).toString('utf8').replace(/^\uFEFF/, '').trimStart()
|
|
const html = contentType === 'text/html' || /^(?:<!doctype\s+html|<html|<head|<body|<center)\b/i.test(prefix)
|
|
const frpMarker = /(?:powered\s+by\s+frp\b|faithfully\s+yours,\s*frp\b|<center>\s*frp\s*<\/center>)/i.test(prefix)
|
|
if (html) return failure('routing_failure', status === 404 && frpMarker ? 'frp_default_404' : 'html_instead_of_image')
|
|
if (status === 404 && /<Code>\s*NoSuchKey\s*<\/Code>/i.test(prefix)) return failure('missing_object', 's3_no_such_key')
|
|
if (status === 404 && /<Code>\s*NoSuchBucket\s*<\/Code>/i.test(prefix)) return failure('missing_bucket', 's3_no_such_bucket')
|
|
if (!response.ok) return failure('http_failure', 'http_status')
|
|
if (!MIME_FORMATS.has(contentType)) return failure('unexpected_content', 'unsupported_image_content_type')
|
|
const format = formatOf(bytes)
|
|
if (!format || MIME_FORMATS.get(contentType) !== format) return failure('unexpected_content', 'image_signature_mismatch')
|
|
const decoded = await withAbort(decodeImage(bytes), controller.signal)
|
|
if (decoded?.format !== format || !Number.isSafeInteger(decoded.width) || !Number.isSafeInteger(decoded.height)
|
|
|| decoded.width < 1 || decoded.height < 1 || decoded.width * decoded.height > LIMITS.maxPixels) {
|
|
return failure('unexpected_content', 'image_decode_result_invalid')
|
|
}
|
|
return { ...base, ok: true, classification: 'healthy', status, contentType, bytes: bytesRead, format,
|
|
decoded: { width: decoded.width, height: decoded.height }, bodySha256: sha256(bytes) }
|
|
} catch (error) {
|
|
if (controller.signal.aborted) return failure('timeout', 'request_timeout')
|
|
if (error instanceof CheckError) return failure(error.classification, error.code)
|
|
// Transport messages can contain signed query strings. Keep raw exceptions out of reports.
|
|
return failure('network_failure', 'request_failed')
|
|
} finally {
|
|
clearTimeout(timer)
|
|
if (reader) reader.cancel().catch(() => {})
|
|
else if (body) body.cancel().catch(() => {})
|
|
controller.abort()
|
|
}
|
|
}
|
|
|
|
export async function checkMaterialPreviewHealth(manifest, { fetchImpl = directFetch, decodeImage = defaultImageDecoder,
|
|
timeoutMs = LIMITS.timeoutMs, maxBytes = LIMITS.maxBytes } = {}) {
|
|
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > LIMITS.timeoutMs
|
|
|| !Number.isSafeInteger(maxBytes) || maxBytes < 16 || maxBytes > LIMITS.maxBytes || typeof decodeImage !== 'function') {
|
|
throw new CheckError('invalid_manifest', 'limits_must_not_exceed_fixed_budget')
|
|
}
|
|
// Validate the entire manifest before any network request, including mixed lists.
|
|
const entries = validateManifest(manifest)
|
|
const results = []
|
|
for (const entry of entries) results.push(await checkOne(entry, { fetchImpl, decodeImage, timeoutMs, maxBytes }))
|
|
return { version: 1, readOnly: true, checkedAt: new Date().toISOString(), manifestSha256: sha256(JSON.stringify(manifest)),
|
|
passed: results.every(result => result.ok), limits: { timeoutMs, maxBytes, maxPixels: LIMITS.maxPixels }, retries: 0, results }
|
|
}
|
|
|
|
export async function main(args = process.argv.slice(2)) {
|
|
const options = {}
|
|
for (let i = 0; i < args.length; i += 2) {
|
|
if (!['--manifest', '--timeout-ms', '--max-bytes'].includes(args[i]) || args[i + 1] === undefined || options[args[i]] !== undefined) {
|
|
throw new CheckError('invalid_manifest', 'usage_manifest_path_required')
|
|
}
|
|
options[args[i]] = args[i + 1]
|
|
}
|
|
if (!options['--manifest']) throw new CheckError('invalid_manifest', 'usage_manifest_path_required')
|
|
const file = await open(options['--manifest'], 'r')
|
|
const buffer = Buffer.alloc(256 * 1024 + 1)
|
|
let raw
|
|
try {
|
|
const stat = await file.stat()
|
|
if (!stat.isFile() || stat.size > buffer.byteLength - 1) throw new CheckError('invalid_manifest', 'manifest_too_large_or_not_regular')
|
|
const { bytesRead } = await file.read(buffer, 0, buffer.byteLength, 0)
|
|
if (bytesRead >= buffer.byteLength) throw new CheckError('invalid_manifest', 'manifest_too_large')
|
|
raw = buffer.subarray(0, bytesRead)
|
|
} finally {
|
|
await file.close()
|
|
}
|
|
const manifest = JSON.parse(raw.toString('utf8'))
|
|
return checkMaterialPreviewHealth(manifest, {
|
|
...(options['--timeout-ms'] ? { timeoutMs: Number(options['--timeout-ms']) } : {}),
|
|
...(options['--max-bytes'] ? { maxBytes: Number(options['--max-bytes']) } : {}),
|
|
})
|
|
}
|
|
|
|
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
main().then(report => {
|
|
console.log(JSON.stringify(report, null, 2))
|
|
process.exitCode = report.passed ? 0 : 1
|
|
}, error => {
|
|
console.log(JSON.stringify({ version: 1, readOnly: true, passed: false,
|
|
classification: error instanceof CheckError ? error.classification : 'invalid_manifest',
|
|
code: error instanceof CheckError ? error.code : 'manifest_unreadable_or_invalid',
|
|
...(error.materialId ? { materialId: error.materialId } : {}), results: [] }, null, 2))
|
|
process.exitCode = 1
|
|
})
|
|
}
|