From eabcff40f86b31093547b82eccddd78d5cf47866 Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Fri, 23 Oct 2026 19:17:42 +0800 Subject: [PATCH] fix(materials): restore verified cached previews and add health audits --- docs/material-preview-health.md | 72 +++++ docs/material-preview-incident-2026-10-23.md | 96 ++++++ scripts/check-material-preview-health.mjs | 286 ++++++++++++++++++ .../check-material-preview-health.test.mjs | 190 ++++++++++++ ...xport-material-preview-health-manifest.mjs | 141 +++++++++ ...-material-preview-health-manifest.test.mjs | 196 ++++++++++++ scripts/recover-material-previews.mjs | 97 ++++++ scripts/recover-material-previews.test.mjs | 75 +++++ 8 files changed, 1153 insertions(+) create mode 100644 docs/material-preview-health.md create mode 100644 docs/material-preview-incident-2026-10-23.md create mode 100644 scripts/check-material-preview-health.mjs create mode 100644 scripts/check-material-preview-health.test.mjs create mode 100644 scripts/export-material-preview-health-manifest.mjs create mode 100644 scripts/export-material-preview-health-manifest.test.mjs create mode 100644 scripts/recover-material-previews.mjs create mode 100644 scripts/recover-material-previews.test.mjs diff --git a/docs/material-preview-health.md b/docs/material-preview-health.md new file mode 100644 index 00000000..2d81df14 --- /dev/null +++ b/docs/material-preview-health.md @@ -0,0 +1,72 @@ +# 素材预览只读验收 + +当前为独立、显式执行的验收命令，尚未接入发布流程或定时任务。不会修复图片、修改引用、执行 Schema DDL 或调用模型。 + +## 导出并逐批检查 + +在已核对数据库身份、具备项目运行依赖的环境执行，输出目录必须是新目录；不要将连接凭据写入命令或报告。 + +```sh +node scripts/export-material-preview-health-manifest.mjs --output-dir /tmp/material-preview-inventory-20261023 --baseline-host s3.1080p.hk --baseline-count 483 +node scripts/check-material-preview-health.mjs --manifest <新目录>/manifest-000001.json +``` + +导出器在 PostgreSQL `RepeatableRead` 事务中设置 `SET TRANSACTION READ ONLY`，读取全部 `deletedAt: null` 素材，不按 `HISTORY_IMPORT` 标记筛选，不导出人员字段。产物为 `audit.json` 和 `manifest-000001.json` 等清单，每批最多 50 项；须逐批运行检查器，并另存每批 JSON 输出及退出码。 + +输出路径须为运行环境内的绝对路径，父目录须已存在。导出不发图片 GET。先核对 `audit.json` 的 `activeCount`、`exportedCount`、`uncheckableCount`、`inventoryComplete` 和 `legacyHostCounts`。`inventoryComplete` 只说明库存枚举完整，不代表图片健康；缺少预览、源地址不受支持、未检查或检查失败的项目均不能记为健康。2026-10-23 恢复前基线为 483 条活跃旧域名预览引用，只作对照，不能代替本次实时导出。11 项材质精确恢复后，若同期没有其它变动，应为 472；`legacyBaseline.delta = -11` 是预期的本轮变化，并非其余 472 项已恢复。基线不匹配只报告差额，不自动写库或隐藏引用。 + +检查器按清单中的精确登记地址执行有界单次 GET，不重试、不跟随跳转，并核对 HTTP、MIME、图片魔数及 **Sharp 完整像素解码**；仅头部可识别不足以通过。8 MiB、10 秒、2500 万像素上限保持固定，解码器按运行环境惰性 `createRequire` 加载，不执行安装。保留清单、清单哈希和全部批次报告。`audit.batches[].sha256` 对应原文件字节，`manifestSha256` 对应检查器的同名字段（紧凑 JSON 哈希），二者须分别核对。单批 50 项通过、或 12 项材质通过，均不能写成“全库存通过”。 + +完整图片验收需另行安排流量预算，逐一运行所有 `audit.batches` 并对齐哈希、素材 ID、数量及退出码。当前修复验收仅运行材质 12 项的已审核清单；**不因导出了两万余项元数据而自动发出全库存图片 GET**。未运行的批次保留为未检查。 + +## 尚未部署脚本时的 stdin 适配 + +当前生产镜像尚未复制这些新脚本及其 registry 依赖。可从已核对本地源码生成以下 stdin 载荷，在目标容器 `/app` 的依赖环境执行；不向运行目录复制源码，不新增依赖。下例只导出元数据，不读取图片。`/tmp` 输出目录仍必须新建且不与历史验收重名。载荷中不包含数据库 URL、Token 或 Key。 + +在仓库根目录运行以下 JavaScript（本地 Node 的 `--input-type=module -`）。将 stdout 通过已核对的本项目 SSH/`docker exec -i -w /app <当前容器> node --input-type=module -` 管道传入，不要在尚未核对容器身份时执行： + +```js +import { readFile } from 'node:fs/promises' +import { LEGACY_DOMAINS } from './scripts/check-legacy-storage-boundaries.mjs' +const data = s => 'data:text/javascript;base64,' + Buffer.from(s).toString('base64') +const runtimeBase = 'file:///app/package.json' +const adapt = s => s.replaceAll('import.meta.url', JSON.stringify(runtimeBase)) +const registry = data(`export const LEGACY_DOMAINS=${JSON.stringify(LEGACY_DOMAINS)}`) +const health = data(adapt(await readFile('scripts/check-material-preview-health.mjs', 'utf8')) + .replace("'./check-legacy-storage-boundaries.mjs'", JSON.stringify(registry))) +const exporter = data(adapt(await readFile('scripts/export-material-preview-health-manifest.mjs', 'utf8')) + .replace("'./check-legacy-storage-boundaries.mjs'", JSON.stringify(registry)) + .replace("'./check-material-preview-health.mjs'", JSON.stringify(health))) +const args = ['--output-dir', '/tmp/material-preview-inventory-20261023-after11', + '--baseline-host', LEGACY_DOMAINS[0], '--baseline-count', '483'] +process.stdout.write(`const {main}=await import(${JSON.stringify(exporter)}); +try { const r=await main(${JSON.stringify(args)}); + console.log(JSON.stringify({readOnly:true,outputDirectory:r.outputDirectory, + activeCount:r.audit.activeCount,exportedCount:r.audit.exportedCount, + uncheckableCount:r.audit.uncheckableCount,legacyHostCounts:r.audit.legacyHostCounts, + legacyBaseline:r.audit.legacyBaseline,inventoryComplete:true,healthChecksExecuted:false})); + process.exitCode=r.audit.readyForHealthChecks?0:1; +} catch { console.log(JSON.stringify({readOnly:true,inventoryComplete:false,code:'inventory_export_failed'}));process.exitCode=1; }`) +``` + +stdin 中的 `createRequire('file:///app/package.json')` 使用容器已有的 `@prisma/client`/Sharp；不是读取本机另一套数据库配置。若依赖不存在、数据库事务失败或输出中途失败，退出非零，不能把残留文件当完整清单。导出器在所有批次成功写入后才写 `audit.json`。导出完成后再由调用方按需取回只读审计文件，本命令不清理文件。 + +## 发布接入位置（待实施） + +当前发布采用 direct replacement；本轮不修改共享发布脚本。后续接入 `scripts/git-incremental-deploy.ps1` 时按函数语义定位，行号仅供参考： + +| 位置 | 应保留的验收证据 | +| --- | --- | +| `direct_replace_active_container` 中 `docker stop` 前，约 2839–2842 行 | 导出全活跃库存，记录旧容器基线及所有批次结果 | +| `verify_candidate_brand_and_storage_dependencies` 后、切流前，约 2882–2897 行 | 新容器检查同一份清单；核对新增失败，不能重新抽样替换基线 | +| `verify_proxy_target` 成功后、`write_current_container` 前，约 2897–2900 行 | 从公网路径确认同一份清单，保留结果后再记录当前版本 | + +macOS wrapper 从同一 PowerShell 源码抽取发布逻辑，接入时需同时回归，不另复制一套条件。既有故障必须在报告中保留，不将未检查项当通过，也不因独立检查命令存在就宣称发布门禁已启用。 + +11 项材质预览恢复属于独立的精确 CAS 数据补丁：先验证新对象，再以原值为条件更新并保留回退记录；不得混入本只读检查器，也不代表高清原件或其余历史依赖已恢复。 + +相关离线回归： + +```sh +node --test scripts/check-material-preview-health.test.mjs scripts/export-material-preview-health-manifest.test.mjs +``` diff --git a/docs/material-preview-incident-2026-10-23.md b/docs/material-preview-incident-2026-10-23.md new file mode 100644 index 00000000..5fa00ac1 --- /dev/null +++ b/docs/material-preview-incident-2026-10-23.md @@ -0,0 +1,96 @@ +# 旧素材预览失效核查（2026-10-23） + +状态：已确认旧域名入口路由失败、存量迁移覆盖缺口及 11 份真实材质缩略图的转换来源。**11 项临时 160px 预览已恢复，正式 Web 与现有客户端均复核为 12 张缩略图、0 失败；高清源仍未恢复，旧入口的具体触发变更尚未归因，不能记为事故完全解决。** + +## 已确认的事实 + +- 恢复前，正式 Web 材质列表共有 12 项，仅“新花色01”正常，其余 11 项显示图片加载失败；客户端也读取相同素材记录。精确恢复后，Web 刷新及客户端只读重读均显示 12 张缩略图、0 失败。 +- 恢复前，这 11 项 `previewUrl` 指向历史域名及旧桶，创建时间均为 2025-11-07，最晚 `updatedAt` 为 2026-01-14；10 月 19 日留存验收记录也包含相同旧引用。未发现故障前近期正常编辑回写的证据，但不能排除没有审计的历史 SQL。本次只精确变更这 11 项的预览引用及更新时间，原登记来源已保留在恢复日志中。 +- 旧入口真实 TLS 校验通过，HTTP 404 正文是 FRP 默认未命中页面，并非 S3 的 `NoSuchKey` 或 `NoSuchBucket`。这是入口路由层失败，不能据此认定原对象永久丢失。 +- 恢复前，从生产环境运行只读检查器，12 次单次 GET 得到 1 项正常图片、11 项 `routing_failure/frp_default_404`。这一步没有重试、模型调用、生产文件写入或数据库写入；后续恢复的写入范围及验收结果单独记录如下。 + +## 故障时间与变更排除 + +以朴橡柚木色的同一精确文件为准，时区 UTC+8： + +| 时间 | 正式访问记录 | +| --- | --- | +| 2026-10-20 11:36:18 | `/api/proxy-media` 返回 200，15738 字节 | +| 2026-10-21 10:17:30 | 同一文件首次留存 404 | +| 2026-10-22 09:39:01–04 | 其他十项精确文件均有 404 | +| 2026-10-22 12:08 | 后续 Go/frpc 安全升级 | + +因此不能将 10 月 22 日升级认定为本次首次故障原因。故障核查时正式容器实际提交为 `df90e6f4799fcdf69b3b22f9910d8f4b4a596e77`。URL 重写、媒体代理安全策略、媒体代理路由及 S3 模块的生产字节与此前部署的 Git 字节一致；可取得的 10 月 16 日与 20 日配置快照中，S3 端点、公开域名、桶及路径前缀也未变化。 + +四项已升级 frpc 服务配置与升级前 SHA 一致，且均不连接旧域名现指向的入口。故障前后的该旧入口 DNS、FRPS 域名注册、旧客户端启动停止日志目前不在已确认归属的项目运维资料中。**不能据当前 404 推测是谁改了哪个配置，也不能盲目回滚安全升级。** + +## 迁移覆盖与范围 + +`scripts/migrate-history-preview-assets.js` 只选择存在 `HISTORY_IMPORT` 日志标记的素材。这 11 项没有该标记，因此不进入该脚本候选。该脚本不能当作已迁移所有活跃旧预览的证明。 + +恢复前的只读基线中，活跃 `Material.previewUrl` 共有 483 条旧域名引用：3D场景 305、详情模板 87、其他 61、材质 11、模特 7、买家秀图片 6、实拍图片 6。本次仅处理其中 11 项材质预览，没有扩大为批量修改；这 11 项临时预览恢复通过，不代表其余旧依赖已恢复，也不代表高清源或旧入口恢复。 + +恢复前，在当前可信桶中对这 11 个精确旧对象名分别检查桶根路径及配置前缀，共 22 次 Head，均为 404。没有得到能与原源 SHA 对照的迁移副本；后续使用的是单独命名的新临时预览对象，没有直接替换域名、桶名或覆盖旧对象。 + +## 已确认的缓存来源与临时预览恢复 + +已只读复核本项目原会话 2026-10-19 的相关工具代码。会话 JSONL 第 42207 行（UTC+8 17:36:02）记录了下载和转换实现；第 42210 行（17:36:05）返回 `cached=12`、`failed=0`、`productionChanges=false`、退出码 0，其中包括这 11 项旧材质及 1 项当前桶材质。下载按真实素材目录的 `materialId` 与登记 URL 逐项绑定，没有替代图分支。 + +当时的处理流程为：读取 HTTP 响应到内存 `original`，按图片方向归一后等比缩小、不放大，再以 WebP 质量 80 编码，保存到 `output/detail-flow-20261019/material-thumbnails/.webp`： + +```js +sharp(original) + .rotate() + .resize(160, 160, { fit: 'inside', withoutEnlargement: true }) + .webp({ quality: 80 }) +``` + +代码没有裁切、叠字、拼接或显式改色操作，但等比重采样和有损编码不保持逐像素、逐字节一致。`material-thumbnails.json` 的 `sourceSha256` 计算自转换前的 HTTP 响应，而本地文件 SHA 计算自转换后的 WebP，因此二者不同是预期结果，不是仅凭 SHA 差异就能认定的缓存损坏。本机 11 份文件均可完整解码为 160×160 单帧 WebP，并与 10 月 20 日按相同素材 ID 复制的验收记录逐项一致。 + +原响应只存在当时 Node 进程内存中，没有另存原文件、写入 `store` 或作为图片字节返回会话；原会话保留的是统计值、来源和原响应 SHA，不能据此还原高清原图。对项目已知下载、验收、备份及图片缓存的受限检索也未找到这 11 个原响应 SHA；另找到的同像素 PNG/WebP 副本仍是 160px 派生版本。这不证明旧存储中的原对象永久丢失。 + +本次透明地使用这 11 份有来源记录的 160px 图片，**仅恢复材质卡片和选择器的临时缩略预览**。新对象没有放大、没有冒充高清，旧对象、原件及下载源未改。具体实施和已完成的验证如下。 + +## 已实施的精确恢复及双端验收 + +恢复操作编号为 `material-preview-cache-20261023`。2026-10-23 已在当前可信存储的 `由壹点AI/public-display/material-preview-cache-20261023/` 下新增 11 个对象，文件名包含精确素材 ID 及派生文件 SHA，没有覆盖旧对象。对象上传校验后，按原预览 URL 精确条件执行 11 项 CAS 更新，并在同一数据库事务内写入 11 条 `PREVIEW_CACHE_RECOVERY` 日志。日志逐项保留原 URL、新 URL、原响应 SHA、缓存 SHA、160×160 尺寸和操作编号。 + +前后快照逐项复核通过：除 `Material.previewUrl`、`updatedAt` 外，11 项材料的全部字段及关联 `assets`、`skus` 一致；原文件、ZIP 及其它下载路径不改，未改变权限、素材归属或产品资料。快照内容 SHA 为 `7469b9e0d342c075433a8f133b2910dd82cc543be8feef74031a5b726c8d012f`，事后复核回执内容 SHA 为 `2084dac9220da847bdbbe5d7f39ebd896ee1b1109caa99868dbf49e96231aef6`；两者均按 JSON 紧凑序列化、不含尾换行计算。 + +2026-10-23 19:07（UTC+8）的双端复核结果： + +- 正式 Web 刷新后，可访问性树确认 12 个图片节点、0 个图片加载失败标签，并保留实际页面截图。 +- 复用现有唯一客户端窗口（PID `9260`、HWND `2363054`），没有另开窗口、没有更改项目资料；逐项执行 11 次素材图片只读重读，滚动检查上、下区域并截图，汇总确认 12 张真实缩略图、0 个失败标记。 +- 本轮没有重新验收悬停大图；先前的悬停证据不计作这次恢复后的验证。已通过的范围是 Web 与客户端的临时缩略预览，`originalSourceRestored=false`。 + +因此“11 项临时 160px 缩略预览恢复”已通过；高清源、原件可取回性、旧入口故障的具体触发变更及其余旧依赖仍是开放事项。本次未重发结果未知的模型请求。 + +## 防复发检查及操作边界 + +新增独立只读工具 `scripts/check-material-preview-health.mjs`，输入显式审阅的素材 ID 与登记图片 URL 清单。它限制项目既有域名、公网 DNS 地址、字节数与时间；不跟随跳转、不重试，检查真实 GET、Content-Type、图片魔数和 Sharp 完整像素解码，区分 FRP/HTML 路由失败、S3 缺桶、S3 缺对象及网络异常。只有图片头、损坏正文或像素炸弹不能通过。 + +`scripts/export-material-preview-health-manifest.mjs` 在只读 RepeatableRead 事务中清点所有未删除的 Material，按最多 50 项分批，不再用 `HISTORY_IMPORT` 筛选；缺 URL 或不支持地址仍计入审计，不会被静默遗漏。与安全恢复工具合计 31 项离线测试通过。执行方式、生产 stdin 适配与发布前后接入点见 `docs/material-preview-health.md`。目前为显式执行工具，**尚未接入自动发布或定时监控**，不能声称自动防回归已部署。 + +恢复后生产只读清点共 23467 条活跃记录，其中 23263 条为检查器支持的 URL，204 条缺地址或不受该图片检查器支持（不等于 204 条图片损坏）。旧入口引用由 483 减少到 472，只有本次 11 项材质改变；未对其余库存发起全量图片 GET。11 个新对象的实际公网 GET、完整解码及 SHA 均通过。 + +额外区分了“数据库原登记域名”与“应用实际交付地址”：未修改的新花色01仍登记历史兼容域名，直接访问该域名本次网络检查失败；现有应用按原有规则交付到当前存储。对其当前同桶同键地址做单次 GET，完整解码为 768×1600，正文 SHA 与 10 月 19 日成功的原响应完全一致。此项未改库或替换素材。原始登记地址检查报告保留失败状态，不能将其冒充全库存健康报告。 + +下次涉及存储域名、代理、运行时或 frpc 维护，维护前后需要同时检查当前入口和仍有活跃引用的历史入口，并保留前后报告；仅进程在线、首页健康或配置 SHA 未变，不算素材交付验证。桶退役之前必须清点所有活跃引用，并核对“对象真实存在且内容一致，再改引用”，不能只按导入日志标签判断迁移完成。 + +下一步仍需要旧入口故障窗口内的 DNS/FRPS/客户端变更证据，以及原对象或经过校验的迁移清单。恢复时保留原文件，使用精确条件更新和可回退记录；临时 160px 预览必须保留尺寸与来源记录，不能掩盖高清源未恢复或扩大为其它素材已恢复的结论。 + +## 证据 + +本地 `output/native-brief-panel-20261023/` 下保留： + +- `material-regression-code-config-timeline.json`：生产身份、代码哈希、配置及精确访问时间线。 +- `legacy-preview-migration-readonly-audit.json`：11 项日期、迁移筛选条件、22 次 Head 与依赖计数。 +- `legacy-material-cache-mapping-audit.json`：本机派生图的来源与质量边界。 +- `legacy-material-original-cache-search.json`、`legacy-material-original-cache-search-followup.json`：受限原响应检索及同像素派生副本记录，不是原件恢复证明。 +- `material-preview-health-real.json`：正式环境 12 项单次 GET 检查结果。 +- `web-materials-source-check.png`：正式 Web 实际失败状态。 +- `preview-recovery-before.json`、`preview-recovery-apply-result.json`、`preview-recovery-verify-result.json`：恢复前快照、11 个新增对象及 11 项精确更新、完整字段与审计日志复核。 +- `material-previews-ui-acceptance.json`：单个现有客户端窗口与正式 Web 的 12 张缩略图、0 失败验收回执。 +- `material-previews-native-restored-top.png`、`material-previews-native-restored-bottom.png`、`material-previews-web-restored.png`：恢复后的客户端上、下区域及正式 Web 实际截图。 + +本轮生产恢复范围为 11 个新增临时预览对象、11 项精确预览引用更新及 11 条同事务审计日志；没有修改原文件、下载路径、桶或 FRP 配置，也未重发结果未知的模型请求。本文件此次更新只登记已有实施与验收证据，没有另行执行生产操作。 diff --git a/scripts/check-material-preview-health.mjs b/scripts/check-material-preview-health.mjs new file mode 100644 index 00000000..347536c7 --- /dev/null +++ b/scripts/check-material-preview-health.mjs @@ -0,0 +1,286 @@ +// Standalone, opt-in read-only acceptance for reviewed, registered material URLs. +// LEGACY_READ_ONLY: reuse the project's historical host registry without rewriting URLs. +// node scripts/check-material-preview-health.mjs --manifest +// {"version":1,"reviewed":true,"materials":[{"materialId":"","registeredSource":""}]} +import { createHash } from 'node:crypto' +import { lookup } from 'node:dns/promises' +import { open } from 'node:fs/promises' +import http from 'node:http' +import https from 'node:https' +import { isIP } from 'node:net' +import { createRequire } from 'node:module' +import path from 'node:path' +import { Readable } from 'node:stream' +import { fileURLToPath } from 'node:url' +import { LEGACY_DOMAINS } from './check-legacy-storage-boundaries.mjs' + +export const LIMITS = Object.freeze({ timeoutMs: 10_000, maxBytes: 8 * 1024 * 1024, maxPixels: 25_000_000, materials: 50 }) +const CURRENT_HOST = 's3.jiansun.vip' +const HOSTS = new Set([CURRENT_HOST, ...LEGACY_DOMAINS]) +const MIME_FORMATS = new Map([['image/png', 'png'], ['image/jpeg', 'jpeg'], ['image/jpg', 'jpeg'], ['image/webp', 'webp']]) +const sha256 = value => createHash('sha256').update(value).digest('hex') + +class CheckError extends Error { + constructor(classification, code, materialId) { + super(code) + this.classification = classification + this.code = code + this.materialId = materialId + } +} + +export function registeredSource(value) { + if (typeof value !== 'string' || value.length > 4096 || value.trim() !== value || /[\r\n\0]/.test(value)) { + throw new CheckError('source_rejected', 'invalid_url') + } + let url + try { url = new URL(value) } catch { throw new CheckError('source_rejected', 'invalid_url') } + if (!HOSTS.has(url.hostname) || url.username || url.password || url.port || url.hash) { + throw new CheckError('source_rejected', 'unregistered_source') + } + if (url.protocol !== 'https:' && !(url.protocol === 'http:' && LEGACY_DOMAINS.includes(url.hostname))) { + throw new CheckError('source_rejected', 'unregistered_protocol') + } + if (!url.pathname || url.pathname === '/') throw new CheckError('source_rejected', 'object_path_required') + return url +} + +export function validateManifest(manifest) { + if (manifest?.version !== 1 || manifest.reviewed !== true || !Array.isArray(manifest.materials) + || manifest.materials.length < 1 || manifest.materials.length > LIMITS.materials) { + throw new CheckError('invalid_manifest', 'reviewed_manifest_required') + } + const identities = new Set() + return manifest.materials.map(item => { + if (typeof item?.materialId !== 'string' || !/^[A-Za-z0-9._-]{1,128}$/.test(item.materialId) || identities.has(item.materialId)) { + throw new CheckError('invalid_manifest', 'unique_material_id_required') + } + identities.add(item.materialId) + try { + const url = registeredSource(item.registeredSource) + return { materialId: item.materialId, url } + } catch (error) { + error.materialId = item.materialId + throw error + } + }) +} + +// A reviewed hostname must not be rebound to local/private/metadata networks. +// The direct Node transport resolves once per connection and uses only these +// checked addresses; environment proxy variables and redirects are not followed. +export function isPublicAddress(address) { + if (isIP(address) === 4) { + const [a, b, c] = address.split('.').map(Number) + return !(a === 0 || a === 10 || a === 127 || a >= 224 + || (a === 100 && b >= 64 && b <= 127) + || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31) + || (a === 192 && (b === 168 || (b === 0 && (c === 0 || c === 2)))) + || (a === 198 && (b === 18 || b === 19 || (b === 51 && c === 100))) + || (a === 203 && b === 0 && c === 113)) + } + if (isIP(address) === 6) { + // Only global unicast; this also rejects mapped IPv4, loopback and link-local. + const first = Number.parseInt(address.split(':')[0], 16) + return first >= 0x2000 && first <= 0x3fff && first !== 0x2002 && !/^2001:(?:0*:|0*db8:)/i.test(address) + } + return false +} + +export function directFetch(input, { signal, headers, method, cache, redirect } = {}) { + const url = registeredSource(String(input)) + if (method !== 'GET' || cache !== 'no-store' || redirect !== 'manual') { + throw new CheckError('source_rejected', 'readonly_transport_required') + } + return new Promise((resolve, reject) => { + const transport = url.protocol === 'https:' ? https : http + const request = transport.request(url, { + method: 'GET', signal, headers, + lookup(hostname, options, callback) { + lookup(hostname, { all: true, verbatim: true }).then(addresses => { + if (hostname !== url.hostname || !addresses.length || addresses.some(row => !isPublicAddress(row.address))) { + callback(new CheckError('source_rejected', 'non_public_dns_address')) + } else if (options?.all) { + callback(null, addresses) + } else { + callback(null, addresses[0].address, addresses[0].family) + } + }, () => callback(new CheckError('network_failure', 'dns_lookup_failed'))) + }, + }, response => { + try { + const noBody = [204, 205, 304].includes(response.statusCode) + if (noBody) response.destroy() + resolve(new Response(noBody ? null : Readable.toWeb(response), { + status: response.statusCode, + headers: response.headers, + })) + } catch { + response.destroy() + reject(new CheckError('network_failure', 'invalid_http_response')) + } + }) + request.on('error', reject) + request.end() + }) +} + +function formatOf(bytes) { + if (bytes.length >= 8 && bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) return 'png' + if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'jpeg' + if (bytes.length >= 16 && bytes.toString('ascii', 0, 4) === 'RIFF' && bytes.toString('ascii', 8, 12) === 'WEBP' + && ['VP8 ', 'VP8L', 'VP8X'].includes(bytes.toString('ascii', 12, 16))) return 'webp' + return null +} + +/** Lazy CJS loading also works when this module's source is run through Node stdin. */ +export function createSharpPreviewDecoder({ requireImpl = createRequire(import.meta.url) } = {}) { + let sharp + return async bytes => { + if (!sharp) { + try { sharp = requireImpl('sharp') } catch { throw new CheckError('decoder_unavailable', 'image_decoder_unavailable') } + } + try { + const image = sharp(bytes, { failOn: 'error', limitInputPixels: LIMITS.maxPixels, animated: true }) + const metadata = await image.metadata() + // metadata() alone only parses the header. Materialise pixels so a + // truncated/corrupt PNG, JPEG or WebP cannot pass from its magic bytes. + const { info } = await image.raw().toBuffer({ resolveWithObject: true }) + return { format: metadata.format, width: info.width, height: info.height } + } catch (error) { + if (error instanceof CheckError) throw error + throw new CheckError('unexpected_content', 'image_decode_failed') + } + } +} +const defaultImageDecoder = createSharpPreviewDecoder() + +function sourceReference(url) { + return { origin: url.origin, pathname: url.pathname, queryRedacted: Boolean(url.search), sha256: sha256(url.href) } +} + +function withAbort(promise, signal) { + if (signal.aborted) return Promise.reject(signal.reason) + return new Promise((resolve, reject) => { + const abort = () => reject(signal.reason) + signal.addEventListener('abort', abort, { once: true }) + Promise.resolve(promise).then(resolve, reject).finally(() => signal.removeEventListener('abort', abort)) + }) +} + +async function checkOne(entry, { fetchImpl, decodeImage, timeoutMs, maxBytes }) { + const base = { materialId: entry.materialId, source: sourceReference(entry.url), attempts: 1 } + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(new CheckError('timeout', 'request_timeout')), timeoutMs) + let reader + let body + let status = null + let contentType = '' + let bytesRead = 0 + const failure = (classification, code) => ({ ...base, ok: false, classification, code, status, contentType, bytes: bytesRead }) + try { + const response = await withAbort(fetchImpl(entry.url.href, { + method: 'GET', cache: 'no-store', redirect: 'manual', signal: controller.signal, + headers: { 'Cache-Control': 'no-store, no-cache, max-age=0', Pragma: 'no-cache', Accept: 'image/png,image/jpeg,image/webp' }, + }), controller.signal) + status = response.status + body = response.body + contentType = (response.headers.get('content-type') || '').split(';')[0].trim().toLowerCase() + if (response.redirected || (status >= 300 && status < 400)) return failure('redirect_blocked', 'redirect_not_followed') + if (response.url && response.url !== entry.url.href) return failure('redirect_blocked', 'response_url_changed') + const length = response.headers.get('content-length') + if (length && /^\d+$/.test(length) && Number(length) > maxBytes) return failure('too_large', 'declared_body_limit') + const chunks = [] + reader = response.body?.getReader() + while (reader) { + const { done, value } = await withAbort(reader.read(), controller.signal) + if (done) break + bytesRead += value.byteLength + if (bytesRead > maxBytes) return failure('too_large', 'stream_body_limit') + chunks.push(Buffer.from(value)) + } + const bytes = Buffer.concat(chunks) + const prefix = bytes.subarray(0, 4096).toString('utf8').replace(/^\uFEFF/, '').trimStart() + const html = contentType === 'text/html' || /^(?:\s*frp\s*<\/center>)/i.test(prefix) + if (html) return failure('routing_failure', status === 404 && frpMarker ? 'frp_default_404' : 'html_instead_of_image') + if (status === 404 && /\s*NoSuchKey\s*<\/Code>/i.test(prefix)) return failure('missing_object', 's3_no_such_key') + if (status === 404 && /\s*NoSuchBucket\s*<\/Code>/i.test(prefix)) return failure('missing_bucket', 's3_no_such_bucket') + if (!response.ok) return failure('http_failure', 'http_status') + if (!MIME_FORMATS.has(contentType)) return failure('unexpected_content', 'unsupported_image_content_type') + const format = formatOf(bytes) + if (!format || MIME_FORMATS.get(contentType) !== format) return failure('unexpected_content', 'image_signature_mismatch') + const decoded = await withAbort(decodeImage(bytes), controller.signal) + if (decoded?.format !== format || !Number.isSafeInteger(decoded.width) || !Number.isSafeInteger(decoded.height) + || decoded.width < 1 || decoded.height < 1 || decoded.width * decoded.height > LIMITS.maxPixels) { + return failure('unexpected_content', 'image_decode_result_invalid') + } + return { ...base, ok: true, classification: 'healthy', status, contentType, bytes: bytesRead, format, + decoded: { width: decoded.width, height: decoded.height }, bodySha256: sha256(bytes) } + } catch (error) { + if (controller.signal.aborted) return failure('timeout', 'request_timeout') + if (error instanceof CheckError) return failure(error.classification, error.code) + // Transport messages can contain signed query strings. Keep raw exceptions out of reports. + return failure('network_failure', 'request_failed') + } finally { + clearTimeout(timer) + if (reader) reader.cancel().catch(() => {}) + else if (body) body.cancel().catch(() => {}) + controller.abort() + } +} + +export async function checkMaterialPreviewHealth(manifest, { fetchImpl = directFetch, decodeImage = defaultImageDecoder, + timeoutMs = LIMITS.timeoutMs, maxBytes = LIMITS.maxBytes } = {}) { + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > LIMITS.timeoutMs + || !Number.isSafeInteger(maxBytes) || maxBytes < 16 || maxBytes > LIMITS.maxBytes || typeof decodeImage !== 'function') { + throw new CheckError('invalid_manifest', 'limits_must_not_exceed_fixed_budget') + } + // Validate the entire manifest before any network request, including mixed lists. + const entries = validateManifest(manifest) + const results = [] + for (const entry of entries) results.push(await checkOne(entry, { fetchImpl, decodeImage, timeoutMs, maxBytes })) + return { version: 1, readOnly: true, checkedAt: new Date().toISOString(), manifestSha256: sha256(JSON.stringify(manifest)), + passed: results.every(result => result.ok), limits: { timeoutMs, maxBytes, maxPixels: LIMITS.maxPixels }, retries: 0, results } +} + +export async function main(args = process.argv.slice(2)) { + const options = {} + for (let i = 0; i < args.length; i += 2) { + if (!['--manifest', '--timeout-ms', '--max-bytes'].includes(args[i]) || args[i + 1] === undefined || options[args[i]] !== undefined) { + throw new CheckError('invalid_manifest', 'usage_manifest_path_required') + } + options[args[i]] = args[i + 1] + } + if (!options['--manifest']) throw new CheckError('invalid_manifest', 'usage_manifest_path_required') + const file = await open(options['--manifest'], 'r') + const buffer = Buffer.alloc(256 * 1024 + 1) + let raw + try { + const stat = await file.stat() + if (!stat.isFile() || stat.size > buffer.byteLength - 1) throw new CheckError('invalid_manifest', 'manifest_too_large_or_not_regular') + const { bytesRead } = await file.read(buffer, 0, buffer.byteLength, 0) + if (bytesRead >= buffer.byteLength) throw new CheckError('invalid_manifest', 'manifest_too_large') + raw = buffer.subarray(0, bytesRead) + } finally { + await file.close() + } + const manifest = JSON.parse(raw.toString('utf8')) + return checkMaterialPreviewHealth(manifest, { + ...(options['--timeout-ms'] ? { timeoutMs: Number(options['--timeout-ms']) } : {}), + ...(options['--max-bytes'] ? { maxBytes: Number(options['--max-bytes']) } : {}), + }) +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().then(report => { + console.log(JSON.stringify(report, null, 2)) + process.exitCode = report.passed ? 0 : 1 + }, error => { + console.log(JSON.stringify({ version: 1, readOnly: true, passed: false, + classification: error instanceof CheckError ? error.classification : 'invalid_manifest', + code: error instanceof CheckError ? error.code : 'manifest_unreadable_or_invalid', + ...(error.materialId ? { materialId: error.materialId } : {}), results: [] }, null, 2)) + process.exitCode = 1 + }) +} diff --git a/scripts/check-material-preview-health.test.mjs b/scripts/check-material-preview-health.test.mjs new file mode 100644 index 00000000..365926c9 --- /dev/null +++ b/scripts/check-material-preview-health.test.mjs @@ -0,0 +1,190 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { createRequire } from 'node:module' +import { checkMaterialPreviewHealth, createSharpPreviewDecoder, directFetch, isPublicAddress, LIMITS, registeredSource, validateManifest } from './check-material-preview-health.mjs' + +const sharp = createRequire(import.meta.url)('sharp') +const fixture = () => sharp({ create: { width: 3, height: 2, channels: 3, background: '#345678' } }) +const PNG = await fixture().png().toBuffer() +const JPEG = await fixture().jpeg().toBuffer() +const WEBP = await fixture().webp().toBuffer() +const source = 'https://s3.jiansun.vip/gongxiangpan/public-display/example.png' +const manifest = (url = source) => ({ version: 1, reviewed: true, materials: [{ materialId: 'material-a', registeredSource: url }] }) +const response = (body, type, status = 200, extra = {}) => new Response(body, { status, headers: { 'content-type': type, ...extra } }) + +test('normal PNG/JPEG/WebP use one actual uncached GET with no redirects or credentials', async () => { + for (const [body, mime, format] of [[PNG, 'image/png', 'png'], [JPEG, 'image/jpeg', 'jpeg'], [WEBP, 'image/webp', 'webp']]) { + const calls = [] + const report = await checkMaterialPreviewHealth(manifest(), { fetchImpl: async (url, options) => { + calls.push({ url, options }) + return response(body, `${mime}; charset=binary`) + } }) + assert.equal(report.passed, true) + assert.equal(report.readOnly, true) + assert.equal(report.retries, 0) + assert.equal(calls.length, 1) + assert.equal(calls[0].url, source) + assert.equal(calls[0].options.method, 'GET') + assert.equal(calls[0].options.cache, 'no-store') + assert.equal(calls[0].options.redirect, 'manual') + assert.match(calls[0].options.headers['Cache-Control'], /no-store/) + assert.equal(calls[0].options.headers.Authorization, undefined) + assert.equal(report.results[0].classification, 'healthy') + assert.equal(report.results[0].format, format) + assert.deepEqual(report.results[0].decoded, { width: 3, height: 2 }) + assert.equal(report.results[0].bytes, body.length) + assert.match(report.results[0].bodySha256, /^[a-f0-9]{64}$/) + } +}) + +test('correct MIME and magic bytes cannot make truncated image headers healthy', async () => { + for (const [body, mime] of [ + [PNG.subarray(0, 16), 'image/png'], + [JPEG.subarray(0, 16), 'image/jpeg'], + [WEBP.subarray(0, 20), 'image/webp'], + ]) { + const report = await checkMaterialPreviewHealth(manifest(), { fetchImpl: async () => response(body, mime) }) + assert.equal(report.passed, false) + assert.equal(report.results[0].code, 'image_decode_failed') + } +}) + +test('decoder is lazy, injectable, and a missing runtime decoder fails closed', async () => { + let requires = 0 + const unavailable = createSharpPreviewDecoder({ requireImpl() { requires++; throw new Error('sensitive runtime path') } }) + assert.equal(requires, 0) + const report = await checkMaterialPreviewHealth(manifest(), { decodeImage: unavailable, fetchImpl: async () => response(PNG, 'image/png') }) + assert.equal(requires, 1) + assert.equal(report.results[0].classification, 'decoder_unavailable') + assert.equal(report.results[0].code, 'image_decoder_unavailable') + assert.ok(!JSON.stringify(report).includes('sensitive runtime path')) + const invalid = await checkMaterialPreviewHealth(manifest(), { decodeImage: async () => ({ format: 'png', width: 0, height: 2 }), fetchImpl: async () => response(PNG, 'image/png') }) + assert.equal(invalid.results[0].code, 'image_decode_result_invalid') +}) + +test('pixel bombs cannot pass the native decoder budget', async () => { + const large = await sharp({ create: { width: 5001, height: 5000, channels: 3, background: '#ffffff' } }).png().toBuffer() + const report = await checkMaterialPreviewHealth(manifest(), { fetchImpl: async () => response(large, 'image/png') }) + assert.equal(report.passed, false) + assert.equal(report.results[0].code, 'image_decode_failed') +}) + +test('FRP default 404 is distinguished from a missing S3 object and never retried', async () => { + let calls = 0 + const report = await checkMaterialPreviewHealth(manifest('https://s3.1080p.hk/jsxiangche/a.png'), { fetchImpl: async () => { + calls++ + return response('

404 Not Found

frp
', 'text/html', 404) + } }) + assert.equal(calls, 1) + assert.equal(report.passed, false) + assert.equal(report.results[0].classification, 'routing_failure') + assert.equal(report.results[0].code, 'frp_default_404') + const missing = await checkMaterialPreviewHealth(manifest(), { fetchImpl: async () => response('NoSuchKey', 'application/xml', 404) }) + assert.equal(missing.results[0].classification, 'missing_object') +}) + +test('real FRP Not Found page is identified by HTTP status and explicit marker without digits in its body', async () => { + const html = 'Not Found

Not Found

powered by frp

Faithfully yours, frp

' + assert.equal(html.includes('404'), false) + let calls = 0 + const report = await checkMaterialPreviewHealth(manifest(), { fetchImpl: async () => { calls++; return response(html, 'text/html', 404) } }) + assert.equal(calls, 1) + assert.equal(report.results[0].classification, 'routing_failure') + assert.equal(report.results[0].code, 'frp_default_404') + const fakeSuccess = await checkMaterialPreviewHealth(manifest(), { fetchImpl: async () => response(html, 'text/html', 200) }) + assert.equal(fakeSuccess.results[0].code, 'html_instead_of_image') +}) + +test('HTML disguised as successful image content is an entrance routing failure', async () => { + for (const type of ['text/html', 'image/png']) { + const report = await checkMaterialPreviewHealth(manifest(), { fetchImpl: async () => response('wrong gateway', type) }) + assert.equal(report.results[0].classification, 'routing_failure') + assert.equal(report.results[0].code, 'html_instead_of_image') + } +}) + +test('declared and streaming byte limits are enforced without consuming an unbounded body', async () => { + let read = 0 + let cancelled = false + const declared = await checkMaterialPreviewHealth(manifest(), { maxBytes: 16, fetchImpl: async () => response(new ReadableStream({ + pull(controller) { read++; controller.enqueue(new Uint8Array(32)) }, + cancel() { cancelled = true }, + }), 'image/png', 200, { 'content-length': '999999999' }) }) + assert.equal(declared.results[0].code, 'declared_body_limit') + assert.equal(declared.results[0].bytes, 0) + assert.ok(read <= 1, 'the declared limit is checked before reading') + assert.equal(cancelled, true, 'rejected response bodies are cancelled even before acquiring a reader') + const streamed = await checkMaterialPreviewHealth(manifest(), { maxBytes: 16, fetchImpl: async () => response(new ReadableStream({ + start(controller) { controller.enqueue(new Uint8Array(12)); controller.enqueue(new Uint8Array(12)); controller.close() }, + }), 'image/png') }) + assert.equal(streamed.results[0].classification, 'too_large') + assert.equal(streamed.results[0].code, 'stream_body_limit') +}) + +test('redirects never follow a second host, including a permitted destination', async () => { + for (const target of ['http://127.0.0.1/private', 'https://s3.jiansun.vip/gongxiangpan/b.png']) { + const calls = [] + const report = await checkMaterialPreviewHealth(manifest(), { fetchImpl: async (url, options) => { + calls.push({ url, options }) + return response('', 'text/html', 302, { location: target }) + } }) + assert.equal(calls.length, 1) + assert.equal(calls[0].options.redirect, 'manual') + assert.equal(report.results[0].classification, 'redirect_blocked') + } + const changed = response(PNG, 'image/png') + Object.defineProperty(changed, 'url', { value: 'https://unregistered.example/p.png' }) + const report = await checkMaterialPreviewHealth(manifest(), { fetchImpl: async () => changed }) + assert.equal(report.results[0].code, 'response_url_changed') +}) + +test('unreviewed, duplicate and mixed invalid manifests fail before any fetch', async () => { + let calls = 0 + const fetchImpl = async () => { calls++; return response(PNG, 'image/png') } + for (const value of [ + { ...manifest(), reviewed: false }, + { ...manifest(), materials: [...manifest().materials, ...manifest().materials] }, + { ...manifest(), materials: [...manifest().materials, { materialId: 'b', registeredSource: 'http://169.254.169.254/latest/meta-data' }] }, + ]) await assert.rejects(checkMaterialPreviewHealth(value, { fetchImpl })) + assert.equal(calls, 0) +}) + +test('only exact registered hosts and approved schemes/ports are accepted', () => { + for (const url of [ + 'http://127.0.0.1/private', 'http://192.168.58.20:9000/object.png', 'https://s3.jiansun.vip.evil.example/a.png', + 'https://sub.s3.jiansun.vip/a.png', 'https://s3.jiansun.vip:444/a.png', 'https://user:pass@s3.jiansun.vip/a.png', + 'http://s3.jiansun.vip/a.png', 'file:///C:/Windows/a.png', 'https://s3.jiansun.vip/', 'https://s3.jiansun.vip/a.png#secret', + ]) assert.throws(() => registeredSource(url), { classification: 'source_rejected' }) + assert.equal(registeredSource('http://s3.1080p.hk/jsxiangche/a.png').protocol, 'http:') + assert.equal(registeredSource('https://s3.yydd.ai/gongxiangpan/a.png').hostname, 's3.yydd.ai') + assert.equal(validateManifest(manifest()).length, 1) + assert.throws(() => directFetch('https://evil.example/a.png', { method: 'GET', cache: 'no-store', redirect: 'manual' }), { classification: 'source_rejected' }) +}) + +test('DNS guard rejects local, private, mapped and special-purpose addresses', () => { + for (const ip of ['0.0.0.0', '10.0.0.1', '100.64.0.1', '127.0.0.1', '169.254.169.254', '172.16.0.1', '192.168.1.1', '198.18.0.1', '224.0.0.1', '::1', '::ffff:127.0.0.1', 'fe80::1', 'fc00::1', '2001:db8::1', '2002:7f00:1::1']) { + assert.equal(isPublicAddress(ip), false, ip) + } + for (const ip of ['218.93.9.38', '156.233.232.106', '2606:4700::1111']) assert.equal(isPublicAddress(ip), true, ip) +}) + +test('timeouts and network errors remain single-attempt failures and hide signed URL secrets', async () => { + let calls = 0 + const report = await checkMaterialPreviewHealth(manifest(), { timeoutMs: 5, fetchImpl: () => { calls++; return new Promise(() => {}) } }) + assert.equal(report.results[0].classification, 'timeout') + assert.equal(calls, 1) + const secret = 'never-print-this-signature' + const failure = await checkMaterialPreviewHealth(manifest(`${source}?X-Amz-Signature=${secret}`), { fetchImpl: async () => { throw new Error(secret) } }) + assert.equal(failure.results[0].classification, 'network_failure') + assert.equal(failure.results[0].source.queryRedacted, true) + assert.ok(!JSON.stringify(failure).includes(secret)) + await assert.rejects(checkMaterialPreviewHealth(manifest(), { timeoutMs: LIMITS.timeoutMs + 1 })) +}) + +test('image content type and magic must agree; JSON and empty images cannot pass', async () => { + for (const [body, type] of [['{}', 'application/json'], ['', 'image/png'], [JPEG, 'image/png']]) { + const report = await checkMaterialPreviewHealth(manifest(), { fetchImpl: async () => response(body, type) }) + assert.equal(report.passed, false) + assert.equal(report.results[0].classification, 'unexpected_content') + } +}) diff --git a/scripts/export-material-preview-health-manifest.mjs b/scripts/export-material-preview-health-manifest.mjs new file mode 100644 index 00000000..c37a2cc1 --- /dev/null +++ b/scripts/export-material-preview-health-manifest.mjs @@ -0,0 +1,141 @@ +// Read-only inventory of every active Material.previewUrl, not HISTORY_IMPORT. +// Public-source probes remain separate; an uncheckable row is never omitted or healthy. +import { createHash } from 'node:crypto' +import * as fs from 'node:fs/promises' +import { createRequire } from 'node:module' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { LIMITS, registeredSource } from './check-material-preview-health.mjs' +import { LEGACY_DOMAINS } from './check-legacy-storage-boundaries.mjs' + +const sha256 = value => createHash('sha256').update(value).digest('hex') +const runtimeRequire = createRequire(import.meta.url) +export class InventoryError extends Error { + constructor(code) { super(code); this.code = code } +} +function optionsChecked({ pageSize = 500, batchSize = LIMITS.materials, baselineHost, baselineCount } = {}) { + if (!Number.isSafeInteger(pageSize) || pageSize < 1 || pageSize > 1000 + || !Number.isSafeInteger(batchSize) || batchSize < 1 || batchSize > LIMITS.materials) throw new InventoryError('invalid_batch_budget') + if ((baselineHost === undefined) !== (baselineCount === undefined) + || (baselineHost !== undefined && (!LEGACY_DOMAINS.includes(baselineHost) || !Number.isSafeInteger(baselineCount) || baselineCount < 0))) { + throw new InventoryError('invalid_legacy_baseline') + } + return { pageSize, batchSize, baselineHost, baselineCount } +} + +export async function collectMaterialPreviewInventory(prisma, options = {}) { + const { pageSize, batchSize, baselineHost, baselineCount } = optionsChecked(options) + if (typeof prisma?.$transaction !== 'function') throw new InventoryError('database_reader_required') + return prisma.$transaction(async tx => { + await tx.$executeRawUnsafe('SET TRANSACTION READ ONLY') + const activeCount = await tx.material.count({ where: { deletedAt: null } }) + if (!Number.isSafeInteger(activeCount) || activeCount < 0) throw new InventoryError('invalid_active_count') + const materials = [], uncheckable = [], seen = new Set(), materialTypeCounts = Object.create(null), legacyHostCounts = Object.fromEntries(LEGACY_DOMAINS.map(h => [h, 0])) + const inventoryDigest = createHash('sha256') + let cursor, pages = 0 + while (true) { + const rows = await tx.material.findMany({ + where: { deletedAt: null }, select: { id: true, materialType: true, previewUrl: true }, + orderBy: { id: 'asc' }, take: pageSize, ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), + }) + pages++ + if (!Array.isArray(rows) || rows.length > pageSize) throw new InventoryError('invalid_inventory_page') + if (!rows.length) break + for (const row of rows) { + if (typeof row.id !== 'string' || !/^[A-Za-z0-9._-]{1,128}$/.test(row.id) || seen.has(row.id)) throw new InventoryError('duplicate_or_invalid_inventory_identity') + seen.add(row.id) + if (seen.size > activeCount) throw new InventoryError('inventory_count_mismatch') + const type = typeof row.materialType === 'string' ? row.materialType : '' + materialTypeCounts[type] = (materialTypeCounts[type] || 0) + 1 + const source = row.previewUrl + inventoryDigest.update(JSON.stringify([row.id, source ?? null]) + '\n') + if (typeof source === 'string') { + for (const host of LEGACY_DOMAINS) { + if ([`https://${host}/`, `http://${host}/`].some(prefix => source.toLowerCase().startsWith(prefix))) legacyHostCounts[host]++ + } + } + try { + const url = registeredSource(source) + materials.push({ materialId: row.id, registeredSource: url.href }) + } catch (error) { + uncheckable.push({ materialId: row.id, materialType: type, sourceSha256: sha256(typeof source === 'string' ? source : ''), + classification: error?.classification || 'source_rejected', code: error?.code || 'invalid_url' }) + } + } + const next = rows.at(-1).id + if (next === cursor) throw new InventoryError('inventory_cursor_stalled') + cursor = next + } + if (seen.size !== activeCount || materials.length + uncheckable.length !== activeCount) throw new InventoryError('inventory_count_mismatch') + const manifests = [] + for (let i = 0; i < materials.length; i += batchSize) { + manifests.push({ version: 1, reviewed: true, provenance: 'read-only-active-material-inventory', materials: materials.slice(i, i + batchSize) }) + } + const audit = { version: 1, readOnly: true, capturedAt: new Date().toISOString(), inventoryComplete: true, + scope: { table: 'Material', where: { deletedAt: null }, historyImportFilter: false }, + transaction: { isolationLevel: 'RepeatableRead', readOnly: true }, + activeCount, exportedCount: materials.length, uncheckableCount: uncheckable.length, materialTypeCounts, legacyHostCounts, + ...(baselineHost === undefined ? {} : { legacyBaseline: { host: baselineHost, expectedCount: baselineCount, actualCount: legacyHostCounts[baselineHost], + delta: legacyHostCounts[baselineHost] - baselineCount, matches: legacyHostCounts[baselineHost] === baselineCount } }), + inventorySha256: inventoryDigest.digest('hex'), pages, batchSize, batchCount: manifests.length, uncheckable, + readyForHealthChecks: uncheckable.length === 0, healthChecksExecuted: false } + return { audit, manifests } + }, { isolationLevel: 'RepeatableRead', timeout: 60_000, maxWait: 5000 }) +} + +export async function writeInventoryArtifacts(outputDirectory, inventory, { fsImpl = fs } = {}) { + if (typeof outputDirectory !== 'string' || !path.isAbsolute(outputDirectory)) throw new InventoryError('absolute_new_output_directory_required') + if (!inventory?.audit?.inventoryComplete || !Array.isArray(inventory.manifests) + || inventory.audit.batchCount !== inventory.manifests.length) throw new InventoryError('complete_inventory_required') + const output = path.resolve(outputDirectory) + // A fresh directory and exclusive files prevent stale acceptance or accidental overwrite. + await fsImpl.mkdir(output, { mode: 0o700, recursive: false }) + const batches = [] + for (let i = 0; i < inventory.manifests.length; i++) { + const file = `manifest-${String(i + 1).padStart(6, '0')}.json` + const raw = JSON.stringify(inventory.manifests[i], null, 2) + '\n' + await fsImpl.writeFile(path.join(output, file), raw, { flag: 'wx', mode: 0o600 }) + batches.push({ file, sha256: sha256(raw), manifestSha256: sha256(JSON.stringify(inventory.manifests[i])), materialCount: inventory.manifests[i].materials.length }) + } + const audit = { ...inventory.audit, batches } + // Written last: incomplete directory contents are never a complete inventory receipt. + await fsImpl.writeFile(path.join(output, 'audit.json'), JSON.stringify(audit, null, 2) + '\n', { flag: 'wx', mode: 0o600 }) + return { outputDirectory: output, audit } +} + +export async function main(args = process.argv.slice(2), { prismaClient, fsImpl = fs } = {}) { + const options = {} + for (let i = 0; i < args.length; i += 2) { + if (!['--output-dir', '--baseline-host', '--baseline-count'].includes(args[i]) || args[i + 1] === undefined || options[args[i]] !== undefined) { + throw new InventoryError('usage_output_directory_required') + } + options[args[i]] = args[i + 1] + } + if (!options['--output-dir']) throw new InventoryError('usage_output_directory_required') + if (!path.isAbsolute(options['--output-dir'])) throw new InventoryError('absolute_new_output_directory_required') + const baseline = options['--baseline-host'] === undefined && options['--baseline-count'] === undefined ? {} : { + baselineHost: options['--baseline-host'], baselineCount: Number(options['--baseline-count']), + } + optionsChecked(baseline) + const owned = !prismaClient + const prisma = prismaClient || new (runtimeRequire('@prisma/client').PrismaClient)({ log: [] }) + try { + const inventory = await collectMaterialPreviewInventory(prisma, baseline) + return await writeInventoryArtifacts(options['--output-dir'], inventory, { fsImpl }) + } finally { + if (owned) await prisma.$disconnect() + } +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().then(({ outputDirectory, audit }) => { + console.log(JSON.stringify({ version: 1, readOnly: true, outputDirectory, inventoryComplete: audit.inventoryComplete, + activeCount: audit.activeCount, exportedCount: audit.exportedCount, uncheckableCount: audit.uncheckableCount, + batchCount: audit.batchCount, legacyHostCounts: audit.legacyHostCounts, legacyBaseline: audit.legacyBaseline, healthChecksExecuted: false }, null, 2)) + process.exitCode = audit.readyForHealthChecks ? 0 : 1 + }, error => { + console.log(JSON.stringify({ version: 1, readOnly: true, inventoryComplete: false, + code: error instanceof InventoryError ? error.code : 'inventory_export_failed' }, null, 2)) + process.exitCode = 1 + }) +} diff --git a/scripts/export-material-preview-health-manifest.test.mjs b/scripts/export-material-preview-health-manifest.test.mjs new file mode 100644 index 00000000..981a3e2f --- /dev/null +++ b/scripts/export-material-preview-health-manifest.test.mjs @@ -0,0 +1,196 @@ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { readFile } from 'node:fs/promises' +import { spawnSync } from 'node:child_process' +import path from 'node:path' +import test from 'node:test' +import { pathToFileURL } from 'node:url' +import { collectMaterialPreviewInventory, main, writeInventoryArtifacts } from './export-material-preview-health-manifest.mjs' +import { validateManifest } from './check-material-preview-health.mjs' +import { LEGACY_DOMAINS } from './check-legacy-storage-boundaries.mjs' + +const current = id => `https://s3.jiansun.vip/gongxiangpan/display/${id}.webp` +const row = (id, previewUrl = current(id), materialType = '材质') => ({ id, previewUrl, materialType }) +const hash = value => createHash('sha256').update(value).digest('hex') + +function database(rows, { reportedCount = rows.length, findMany } = {}) { + const events = [], reads = [] + const prisma = { + async $transaction(fn, options) { + events.push(['transaction', options]) + return fn({ + async $executeRawUnsafe(sql) { events.push(['sql', sql]) }, + material: { + async count(query) { events.push(['count', query]); return reportedCount }, + async findMany(query) { + reads.push(query) + events.push(['page']) + if (findMany) return findMany(query, reads.length) + const start = query.cursor ? rows.findIndex(item => item.id === query.cursor.id) + query.skip : 0 + return rows.slice(start, start + query.take) + }, + }, + }) + }, + async $disconnect() { events.push(['disconnect']) }, + } + return { prisma, events, reads } +} + +function memoryFs({ failFile } = {}) { + const writes = [], directories = new Set() + return { + writes, + async mkdir(directory, options) { + assert.deepEqual(options, { mode: 0o700, recursive: false }) + if (directories.has(directory)) throw new Error('EEXIST') + directories.add(directory) + }, + async writeFile(file, raw, options) { + assert.deepEqual(options, { flag: 'wx', mode: 0o600 }) + if (file.endsWith(failFile || '\0')) throw new Error('simulated write failure') + writes.push({ file, raw }) + }, + } +} + +test('all active rows are read in a consistent read-only snapshot, with no HISTORY_IMPORT filter', async () => { + const db = database(Array.from({ length: 103 }, (_, i) => row(`m-${String(i).padStart(3, '0')}`))) + const { audit, manifests } = await collectMaterialPreviewInventory(db.prisma, { pageSize: 25 }) + assert.deepEqual(db.events[0], ['transaction', { isolationLevel: 'RepeatableRead', timeout: 60_000, maxWait: 5000 }]) + assert.deepEqual(db.events[1], ['sql', 'SET TRANSACTION READ ONLY']) + assert.deepEqual(db.events[2], ['count', { where: { deletedAt: null } }]) + for (const query of db.reads) { + assert.deepEqual(query.where, { deletedAt: null }) + assert.deepEqual(query.select, { id: true, materialType: true, previewUrl: true }) + assert.deepEqual(query.orderBy, { id: 'asc' }) + assert.equal(query.take, 25) + assert.ok(!JSON.stringify(query).includes('HISTORY_IMPORT')) + } + assert.deepEqual(db.reads[1].cursor, { id: 'm-024' }) + assert.equal(db.reads[1].skip, 1) + assert.equal(audit.activeCount, 103) + assert.equal(audit.exportedCount, 103) + assert.equal(audit.uncheckableCount, 0) + assert.equal(audit.pages, 6, 'the final empty page proves pagination is exhausted') + assert.equal(audit.inventoryComplete, true) + assert.equal(audit.healthChecksExecuted, false) + assert.equal(audit.materialTypeCounts['材质'], 103) + assert.deepEqual(manifests.map(item => item.materials.length), [50, 50, 3]) + assert.equal(new Set(manifests.flatMap(item => validateManifest(item).map(entry => entry.materialId))).size, 103) +}) + +test('missing and unregistered sources remain counted and cannot become a full health pass', async () => { + const secret = 'do-not-print-signed-query' + const db = database([row('one'), row('two', null), row('three', '/uploads/old.webp'), + row('four', `https://unregistered.example/image.webp?signature=${secret}`, '__proto__')]) + const { audit, manifests } = await collectMaterialPreviewInventory(db.prisma) + assert.equal(audit.activeCount, 4) + assert.equal(audit.exportedCount + audit.uncheckableCount, audit.activeCount) + assert.equal(audit.uncheckableCount, 3) + assert.equal(audit.readyForHealthChecks, false) + assert.equal(audit.materialTypeCounts.__proto__, 1) + assert.deepEqual(manifests.flatMap(item => item.materials).map(item => item.materialId), ['one']) + assert.deepEqual(audit.uncheckable.map(item => item.materialId), ['two', 'three', 'four']) + assert.ok(!JSON.stringify(audit).includes(secret)) + assert.ok(audit.uncheckable.every(item => /^[a-f0-9]{64}$/.test(item.sourceSha256))) +}) + +test('legacy baseline is exact and reports drift rather than reusing 483 as current truth', async () => { + const rows = Array.from({ length: 483 }, (_, i) => row(`legacy-${i}`, `https://s3.1080p.hk/jsxiangche/${i}.webp`)) + rows.push(row('different-host', 'https://s3.yydd.ai/gongxiangpan/a.webp'), row('new-source')) + const db = database(rows) + const inventory = await collectMaterialPreviewInventory(db.prisma, { baselineHost: 's3.1080p.hk', baselineCount: 483 }) + assert.deepEqual(inventory.audit.legacyBaseline, { host: 's3.1080p.hk', expectedCount: 483, actualCount: 483, delta: 0, matches: true }) + assert.equal(inventory.audit.activeCount, 485) + assert.equal(inventory.audit.batchCount, 10) + const changed = await collectMaterialPreviewInventory(database(rows.slice(1)).prisma, { baselineHost: 's3.1080p.hk', baselineCount: 483 }) + assert.equal(changed.audit.legacyBaseline.actualCount, 482) + assert.equal(changed.audit.legacyBaseline.matches, false) + assert.equal(changed.audit.legacyBaseline.delta, -1) +}) + +test('duplicates, incomplete counts, invalid pages and changed pagination fail closed', async () => { + await assert.rejects(collectMaterialPreviewInventory(database([row('same'), row('same')]).prisma), { code: 'duplicate_or_invalid_inventory_identity' }) + await assert.rejects(collectMaterialPreviewInventory(database([row('one')], { reportedCount: 2 }).prisma), { code: 'inventory_count_mismatch' }) + await assert.rejects(collectMaterialPreviewInventory(database([row('one')], { reportedCount: 0 }).prisma), { code: 'inventory_count_mismatch' }) + await assert.rejects(collectMaterialPreviewInventory(database([], { findMany: () => null }).prisma), { code: 'invalid_inventory_page' }) + await assert.rejects(collectMaterialPreviewInventory(database([row('one'), row('two')], { findMany: () => [row('one'), row('two')] }).prisma, { pageSize: 1 }), { code: 'invalid_inventory_page' }) + await assert.rejects(collectMaterialPreviewInventory(database([row('one'), row('two')], { findMany: () => [row('one')] }).prisma, { pageSize: 1 }), { code: 'duplicate_or_invalid_inventory_identity' }) +}) + +test('empty inventory is explicit and invalid limits cause no database read', async () => { + const db = database([]) + const { audit, manifests } = await collectMaterialPreviewInventory(db.prisma) + assert.equal(audit.activeCount, 0) + assert.equal(audit.inventoryComplete, true) + assert.deepEqual(manifests, []) + const other = database([]) + for (const options of [{ batchSize: 51 }, { pageSize: 1001 }, { baselineCount: 483 }, { baselineHost: 'evil.example', baselineCount: 1 }]) { + await assert.rejects(collectMaterialPreviewInventory(other.prisma, options)) + } + assert.equal(other.events.length, 0) +}) + +test('fresh private files contain all batches and audit is written last with both hash identities', async () => { + const inventory = await collectMaterialPreviewInventory(database([row('one'), row('two')]).prisma, { batchSize: 1 }) + const fsImpl = memoryFs() + const directory = path.resolve('test-output-not-created') + const { audit } = await writeInventoryArtifacts(directory, inventory, { fsImpl }) + assert.deepEqual(fsImpl.writes.map(item => path.basename(item.file)), ['manifest-000001.json', 'manifest-000002.json', 'audit.json']) + for (let i = 0; i < 2; i++) { + assert.equal(audit.batches[i].sha256, hash(fsImpl.writes[i].raw)) + assert.equal(audit.batches[i].manifestSha256, hash(JSON.stringify(inventory.manifests[i]))) + assert.equal(audit.batches[i].materialCount, 1) + } + await assert.rejects(writeInventoryArtifacts(directory, inventory, { fsImpl }), /EEXIST/) + await assert.rejects(writeInventoryArtifacts('relative-output', inventory, { fsImpl }), { code: 'absolute_new_output_directory_required' }) +}) + +test('partial disk failure cannot leave a complete audit receipt or overwrite another run', async () => { + const inventory = await collectMaterialPreviewInventory(database([row('one'), row('two')]).prisma, { batchSize: 1 }) + const fsImpl = memoryFs({ failFile: 'manifest-000002.json' }) + await assert.rejects(writeInventoryArtifacts(path.resolve('not-created'), inventory, { fsImpl }), /simulated write failure/) + assert.equal(fsImpl.writes.length, 1) + assert.ok(!fsImpl.writes.some(item => item.file.endsWith('audit.json'))) +}) + +test('CLI supports injected runtime dependencies and rejects bad arguments before database access', async () => { + const db = database([row('one')]) + const fsImpl = memoryFs() + const result = await main(['--output-dir', path.resolve('not-created'), '--baseline-host', 's3.1080p.hk', '--baseline-count', '483'], { prismaClient: db.prisma, fsImpl }) + assert.equal(result.audit.activeCount, 1) + assert.equal(result.audit.legacyBaseline.matches, false) + assert.ok(!db.events.some(item => item[0] === 'disconnect'), 'an injected Prisma client is not owned by the CLI') + const before = db.events.length + await assert.rejects(main(['--output-dir', 'relative'], { prismaClient: db.prisma, fsImpl }), { code: 'absolute_new_output_directory_required' }) + await assert.rejects(main(['--output-dir', path.resolve('not-created'), '--batch-size', '500'], { prismaClient: db.prisma, fsImpl }), { code: 'usage_output_directory_required' }) + assert.equal(db.events.length, before) +}) + +test('stdin adapter resolves runtime CJS dependencies without deployed script files or a database connection', async () => { + const data = source => 'data:text/javascript;base64,' + Buffer.from(source).toString('base64') + const runtimeBase = pathToFileURL(path.resolve('package.json')).href + const adapt = source => source.replaceAll('import.meta.url', JSON.stringify(runtimeBase)) + const registry = data(`export const LEGACY_DOMAINS=${JSON.stringify(LEGACY_DOMAINS)}`) + const health = data(adapt(await readFile(new URL('./check-material-preview-health.mjs', import.meta.url), 'utf8')) + .replace("'./check-legacy-storage-boundaries.mjs'", JSON.stringify(registry))) + const exporter = data(adapt(await readFile(new URL('./export-material-preview-health-manifest.mjs', import.meta.url), 'utf8')) + .replace("'./check-legacy-storage-boundaries.mjs'", JSON.stringify(registry)) + .replace("'./check-material-preview-health.mjs'", JSON.stringify(health))) + const script = `import {createRequire} from 'node:module'; + const {collectMaterialPreviewInventory}=await import(${JSON.stringify(exporter)}); + const {checkMaterialPreviewHealth}=await import(${JSON.stringify(health)}); + const sharp=createRequire(${JSON.stringify(runtimeBase)})('sharp'); + const preview=await sharp({create:{width:3,height:2,channels:3,background:'#345678'}}).webp().toBuffer(); + let pages=0; + const inventory=await collectMaterialPreviewInventory({$transaction:async fn=>fn({ + $executeRawUnsafe:async sql=>{if(sql!=='SET TRANSACTION READ ONLY')throw Error('unexpected SQL')}, + material:{count:async()=>1,findMany:async()=>++pages===1?[{id:'one',materialType:'材质',previewUrl:${JSON.stringify(current('one'))}}]:[]} + })}); + const result=await checkMaterialPreviewHealth(inventory.manifests[0],{fetchImpl:async()=>new Response(preview,{headers:{'content-type':'image/webp'}})}); + console.log(JSON.stringify({count:inventory.audit.activeCount,passed:result.passed,decoded:result.results[0].decoded}));` + const child = spawnSync(process.execPath, ['--input-type=module', '-'], { input: script, encoding: 'utf8', timeout: 10_000 }) + assert.equal(child.status, 0, child.stderr) + assert.deepEqual(JSON.parse(child.stdout), { count: 1, passed: true, decoded: { width: 3, height: 2 } }) +}) diff --git a/scripts/recover-material-previews.mjs b/scripts/recover-material-previews.mjs new file mode 100644 index 00000000..ba5bf422 --- /dev/null +++ b/scripts/recover-material-previews.mjs @@ -0,0 +1,97 @@ +// Opt-in, reviewed recovery of authentic cached previews. No model calls, deletes, +// download-source changes or URL-prefix substitution. The caller durably saves +// the returned plan before apply, and provides exact storage/DB adapters. +import { createHash } from 'node:crypto' + +export const OPERATION_ID = 'material-preview-cache-20261023' +export const digest = value => createHash('sha256').update(value).digest('hex') +export const json = value => JSON.stringify(value, (_, v) => typeof v === 'bigint' ? v.toString() : v) +const check = (condition, message) => { if (!condition) throw new Error(message) } +const idPattern = /^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$/ + +export function validateRecovery(manifest) { + check(manifest?.version === 1 && manifest.reviewed === true && manifest.operationId === OPERATION_ID, 'reviewed recovery required') + check(manifest.previewQuality === 'temporary-160px-derived' && manifest.materials?.length === 11, 'exact eleven temporary previews required') + check(idPattern.test(manifest.actorId), 'actor required') + const ids = new Set() + for (const row of manifest.materials) { + check(idPattern.test(row.materialId) && !ids.has(row.materialId), 'unique material identity required') + ids.add(row.materialId) + const old = new URL(row.oldPreviewUrl) + check(old.protocol === 'https:' && !old.username && !old.password && !old.search && !old.hash, 'exact original URL required') + check(/^[a-f0-9]{64}$/.test(row.sourceSha256) && /^[a-f0-9]{64}$/.test(row.cacheSha256), 'source and cache hashes required') + const bytes = Buffer.from(row.bodyBase64, 'base64') + check(bytes.length === row.bytes && bytes.length >= 100 && bytes.length <= 16384 && digest(bytes) === row.cacheSha256, 'cache bytes mismatch') + check(bytes.subarray(0, 4).toString() === 'RIFF' && bytes.subarray(8, 12).toString() === 'WEBP', 'WebP required') + check(row.width === 160 && row.height === 160 && row.provenance === '20261019-sharp-rotate-inside160-webp80', 'verified derivation required') + check(row.destinationKey === `由壹点AI/public-display/${OPERATION_ID}/${row.materialId}-${row.cacheSha256}.webp`, 'scoped immutable destination required') + check(row.newPreviewUrl === `https://s3.jiansun.vip/gongxiangpan/${row.destinationKey}`, 'canonical destination required') + } + return manifest +} + +function verifyActor(actor) { + check(actor && actor.isActive === true && actor.isSuperAdmin === true && actor.status === 'active' && actor.archivedAt === null + && actor.employmentStatus === 'active', 'active authorized recovery actor required') +} + +function verifyRecord(record, row) { + check(record?.id === row.materialId && record.productModel === row.name && record.materialType === '材质' + && record.deletedAt === null && record.previewUrl === row.oldPreviewUrl, 'source record changed') +} + +export async function planRecovery(manifest, db) { + validateRecovery(manifest) + verifyActor(await db.actor(manifest.actorId)) + const records = await db.records(manifest.materials.map(r => r.materialId)) + check(records.length === 11, 'eleven source records required') + const ordered = manifest.materials.map(row => { + const record = records.find(r => r.id === row.materialId) + verifyRecord(record, row) + return record + }) + return { version: 1, operationId: OPERATION_ID, manifestSha256: digest(json(manifest)), capturedAt: new Date().toISOString(), records: ordered } +} + +export async function applyRecovery({ manifest, plan, persistedPlanSha256, db, objects }) { + validateRecovery(manifest) + check(plan.operationId === OPERATION_ID && plan.manifestSha256 === digest(json(manifest)) + && persistedPlanSha256 === digest(json(plan)) && plan.records?.length === 11, 'durably persisted matching plan required') + const originalPlan = await planRecovery(manifest, db) + check(json(originalPlan.records) === json(plan.records), 'source snapshot changed before upload') + check(await db.operationCount(OPERATION_ID) === 0, 'operation already recorded; inspect before continuing') + const uploaded = [] + // Every object is publicly readable and hash-verified before the DB transaction. + // A conflict leaves harmless, named unreferenced objects, never deletes anything. + for (const row of manifest.materials) { + const body = Buffer.from(row.bodyBase64, 'base64') + const existing = await objects.get(row.destinationKey) + if (existing === null) { + await objects.putIfAbsent(row.destinationKey, body, { operationId: OPERATION_ID, materialId: row.materialId, sha256: row.cacheSha256, quality: manifest.previewQuality }) + uploaded.push(row.destinationKey) + } else check(digest(existing) === row.cacheSha256, 'existing destination differs; refusing overwrite') + check(digest(await objects.get(row.destinationKey)) === row.cacheSha256, 'internal object readback mismatch') + check(digest(await objects.publicGet(row.newPreviewUrl)) === row.cacheSha256, 'public object readback mismatch') + } + await db.transaction(async tx => { + verifyActor(await tx.actor(manifest.actorId)) + check(await tx.operationCount(OPERATION_ID) === 0, 'operation concurrently recorded') + const current = await tx.records(manifest.materials.map(r => r.materialId)) + const ordered = manifest.materials.map(row => current.find(r => r.id === row.materialId)) + check(json(ordered) === json(plan.records), 'source snapshot changed; transaction aborted') + for (const row of manifest.materials) { + const before = plan.records.find(r => r.id === row.materialId) + check(await tx.compareAndSet(row, before.updatedAt) === 1, 'concurrent record change; transaction aborted') + await tx.log({ materialId: row.materialId, userId: manifest.actorId, action: 'PREVIEW_CACHE_RECOVERY', changes: json({ operationId: OPERATION_ID, previewQuality: manifest.previewQuality, oldPreviewUrl: row.oldPreviewUrl, newPreviewUrl: row.newPreviewUrl, originalSourceSha256: row.sourceSha256, cacheSha256: row.cacheSha256, width: 160, height: 160, downloadsAndAssetsUnchanged: true, snapshotSha256: persistedPlanSha256 }) }) + } + }) + const after = await db.records(manifest.materials.map(r => r.materialId)) + for (const before of plan.records) { + const actual = after.find(r => r.id === before.id) + const row = manifest.materials.find(r => r.materialId === before.id) + check(actual?.previewUrl === row.newPreviewUrl, 'post-commit preview mismatch; inspect operation logs') + const immutable = ({ previewUrl, updatedAt, ...rest }) => rest + check(json(immutable(actual)) === json(immutable(before)), 'post-commit unrelated data mismatch; inspect operation logs') + } + return { operationId: OPERATION_ID, updated: 11, uploaded, after, temporaryPreviewSize: 160, originalSourceRestored: false, downloadsAndAssetsUnchanged: true, modelCalls: 0 } +} diff --git a/scripts/recover-material-previews.test.mjs b/scripts/recover-material-previews.test.mjs new file mode 100644 index 00000000..f8d5361d --- /dev/null +++ b/scripts/recover-material-previews.test.mjs @@ -0,0 +1,75 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { OPERATION_ID, digest, json, planRecovery, applyRecovery, validateRecovery } from './recover-material-previews.mjs' + +function fixture() { + const body = Buffer.concat([Buffer.from('RIFF'), Buffer.alloc(4), Buffer.from('WEBP'), Buffer.alloc(128)]) + const sha = digest(body) + const manifest = { version: 1, reviewed: true, operationId: OPERATION_ID, previewQuality: 'temporary-160px-derived', actorId: '11111111-1111-4111-8111-111111111111', materials: Array.from({ length: 11 }, (_, i) => { + const materialId = `22222222-2222-4222-8222-${String(i).padStart(12, '0')}` + const destinationKey = `由壹点AI/public-display/${OPERATION_ID}/${materialId}-${sha}.webp` + return { materialId, name: `material-${i}`, oldPreviewUrl: `https://legacy.example.test/${i}.webp`, sourceSha256: '0'.repeat(64), cacheSha256: sha, bodyBase64: body.toString('base64'), bytes: body.length, width: 160, height: 160, provenance: '20261019-sharp-rotate-inside160-webp80', destinationKey, newPreviewUrl: `https://s3.jiansun.vip/gongxiangpan/${destinationKey}` } + }) } + let records = manifest.materials.map(r => ({ id: r.materialId, productModel: r.name, materialType: '材质', deletedAt: null, previewUrl: r.oldPreviewUrl, updatedAt: '2026-01-01T00:00:00.000Z', zipFilePath: 'untouched.zip', zipFileSize: '100', assets: [{ id: 'source', fileUrl: 'original.png' }] })) + let logs = [] + const stored = new Map() + const hooks = { puts: 0, failCas: -1, failLog: false, badPublic: false, actorActive: true } + const db = { + actor: async () => ({ isActive: hooks.actorActive, isSuperAdmin: true, status: hooks.actorActive ? 'active' : 'disabled', employmentStatus: 'active', archivedAt: null }), + records: async () => structuredClone(records), + operationCount: async () => logs.length, + compareAndSet: async (row, updatedAt) => { + const index = records.findIndex(r => r.id === row.materialId && r.previewUrl === row.oldPreviewUrl && r.updatedAt === updatedAt) + if (index < 0 || index === hooks.failCas) return 0 + records[index] = { ...records[index], previewUrl: row.newPreviewUrl, updatedAt: '2026-10-23T00:00:00.000Z' } + return 1 + }, + log: async row => { if (hooks.failLog) throw new Error('log failure'); logs.push(row) }, + transaction: async fn => { + const before = structuredClone(records), previousLogs = structuredClone(logs) + try { await fn(db) } catch (error) { records = before; logs = previousLogs; throw error } + }, + } + const objects = { get: async key => stored.get(key) ?? null, putIfAbsent: async (key, bytes) => { assert.equal(stored.has(key), false); stored.set(key, bytes); hooks.puts++ }, publicGet: async url => hooks.badPublic ? Buffer.from('wrong') : stored.get(decodeURIComponent(new URL(url).pathname).slice('/gongxiangpan/'.length)) } + const prepare = async () => { const plan = await planRecovery(manifest, db); return { manifest, plan, persistedPlanSha256: digest(json(plan)), db, objects } } + return { manifest, db, objects, hooks, stored, prepare, records: () => records, logs: () => logs } +} + +test('requires reviewed exact count, hashes and scoped canonical destinations', () => { + const f = fixture(); assert.equal(validateRecovery(f.manifest), f.manifest) + for (const mutate of [m => { m.reviewed = false }, m => m.materials.pop(), m => { m.materials[0].cacheSha256 = 'f'.repeat(64) }, m => { m.materials[0].destinationKey = 'original.webp' }, m => { m.materials[0].materialId = m.materials[1].materialId }]) { + const m = structuredClone(f.manifest); mutate(m); assert.throws(() => validateRecovery(m)) + } +}) +test('successful recovery changes previews only, preserves source fields and creates eleven audits', async () => { + const f = fixture(), before = structuredClone(f.records()); const result = await applyRecovery(await f.prepare()) + assert.equal(result.updated, 11); assert.equal(f.logs().length, 11); assert.equal(f.hooks.puts, 11) + for (let i = 0; i < 11; i++) { assert.equal(f.records()[i].zipFilePath, before[i].zipFilePath); assert.deepEqual(f.records()[i].assets, before[i].assets); assert.match(f.logs()[i].changes, /temporary-160px-derived/) } +}) +test('rejects missing persisted snapshot proof and changed snapshot before any object write', async () => { + const f = fixture(), args = await f.prepare(); args.persistedPlanSha256 = 'wrong' + await assert.rejects(applyRecovery(args), /persisted/); assert.equal(f.hooks.puts, 0) + const args2 = await f.prepare(); f.records()[0].productModel = 'concurrent edit' + await assert.rejects(applyRecovery(args2), /source record changed/); assert.equal(f.hooks.puts, 0) +}) +test('public readback mismatch cannot change database', async () => { + const f = fixture(), args = await f.prepare(); f.hooks.badPublic = true + await assert.rejects(applyRecovery(args), /public object/); assert.equal(f.logs().length, 0); assert.equal(f.records()[0].previewUrl, f.manifest.materials[0].oldPreviewUrl) +}) +test('existing different object cannot be overwritten', async () => { + const f = fixture(), args = await f.prepare(); f.stored.set(f.manifest.materials[0].destinationKey, Buffer.from('different')) + await assert.rejects(applyRecovery(args), /refusing overwrite/); assert.equal(f.hooks.puts, 0); assert.equal(f.logs().length, 0) +}) +test('CAS conflict rolls back every row and audit, keeps named objects without deleting', async () => { + const f = fixture(), before = structuredClone(f.records()), args = await f.prepare(); f.hooks.failCas = 6 + await assert.rejects(applyRecovery(args), /concurrent record/); assert.deepEqual(f.records(), before); assert.equal(f.logs().length, 0); assert.equal(f.stored.size, 11) +}) +test('audit failure rolls back all database changes', async () => { + const f = fixture(), before = structuredClone(f.records()), args = await f.prepare(); f.hooks.failLog = true + await assert.rejects(applyRecovery(args), /log failure/); assert.deepEqual(f.records(), before); assert.equal(f.logs().length, 0) +}) +test('revoked actor blocks planning and duplicate operation never writes objects again', async () => { + const f = fixture(); f.hooks.actorActive = false; await assert.rejects(f.prepare(), /actor/) + f.hooks.actorActive = true; const args = await f.prepare(); await applyRecovery(args) + await assert.rejects(applyRecovery(args)); assert.equal(f.hooks.puts, 11) +})