From 9da598967a0d707f9bb52994152b461698744a43 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 30 Oct 2026 18:42:59 +0800 Subject: [PATCH] build: snapshot complete release runtime with copy-on-write files --- docs/production-48.md | 22 ++-- scripts/production48.py | 8 +- scripts/release/host.py | 175 +++++++++++++++++++---------- tests/scripts/test_production48.py | 63 ++++++++++- 4 files changed, 196 insertions(+), 72 deletions(-) diff --git a/docs/production-48.md b/docs/production-48.md index 7f8fda681..8872564e3 100644 --- a/docs/production-48.md +++ b/docs/production-48.md @@ -18,7 +18,7 @@ ## 操作入口 -维护电脑需要 Python 3.9+、Paramiko、Git，以及已信任的 SSH `known_hosts`。 +维护电脑需要 Python 3.9+、Paramiko 3.3+、Git，以及已信任的 SSH `known_hosts`。 默认使用 `~/.ssh/45.192.99.48.pem` 和跳板密钥 `~/.ssh/youyidian_prod_ed25519`。 可用 `--key`、`--jump-key`、`--known-hosts` 覆盖本机路径，不要把私钥提交到仓库。 @@ -54,7 +54,7 @@ python scripts/production48.py rollback --release <发布ID> ``` `stage` 校验传输 SHA256，在 `/var/www/ai-chatbot-releases/<发布ID>` 解包。 -它检查压缩包路径、磁盘容量、锁文件、实际依赖包清单和共享 public 文件校验和。 +它检查压缩包路径、磁盘容量、锁文件、实际依赖文件/链接清单和 public 文件校验和。 完成解包后至少保留 1 GiB 空间；不足时停止，不自动删除历史构建。 当前和旧浏览器标签页所需的静态资源通过硬链接保留，不改动旧构建。 @@ -64,13 +64,19 @@ python scripts/production48.py rollback --release <发布ID> 因此会有短暂切换窗口。健康检查同时核对进程目录、构建 ID、首页和路由。 检查失败自动恢复旧配置并再次验活；通过后执行 `pm2 save`，让重启后使用已验收版本。 -## 共享运行资料与边界 +## 运行资料与边界 -初版发布复用原目录 `/var/www/ai-chatbot` 中的 `node_modules`、`public`、`.env.local`、 -`tmp` 和模型配置备份，均通过链接使用，不复制、删除或覆盖。 -生产依赖必须与构建依赖的校验清单相符，Node ABI 必须一致。 -若本次修改了依赖锁文件或 public 中的文件，脚本会停止，须先单独审查并准备对应运行资料， -不能绕过校验强行切换。本流程适用于应用代码迭代；数据库、共享资源和原生 Agent 升级各自单独发布。 +每个发布目录包含构建机按锁文件安装的完整运行依赖和 public 源码资源。 +节点之间不复用 node_modules 链接，避免服务器漏装依赖补丁却被误认为同一版本。 +对内容完全相同的依赖文件，通过 XFS 的写入时复制创建独立 inode，复用底层数据块； +缺失或不同的文件从已校验制品写入。任何一份副本后续修改均不会改动另一份。 +生产文件系统不支持该操作时停止，不自动回退成可能耗尽磁盘的完整复制。 +public 先从当前运行版本创建写入时复制副本，保留未纳入 Git 的运行资源，再叠加本次源码资源。 +完整文件和链接清单在切换前再次验证，Node ABI 必须一致，并在生产机测试 sharp 原生模块。 + +原目录 `/var/www/ai-chatbot` 中的 `.env.local`、`tmp` 和模型配置备份仍通过链接使用， +不会复制、删除或覆盖。新版本也不在生产机执行依赖安装或构建。 +本流程适用于应用代码迭代；数据库、共享资源的删除/迁移和原生 Agent 升级各自单独发布。 原生产 checkout 的支线及未提交改动仍保留，不能据其 Git HEAD 判断已发布版本。 实际发布证据是：运行 cwd、`.next/BUILD_ID`、该目录 `release.json` 和 `activation.json`。 diff --git a/scripts/production48.py b/scripts/production48.py index a6d4af154..d6cbcfbb7 100644 --- a/scripts/production48.py +++ b/scripts/production48.py @@ -194,6 +194,7 @@ with open(workspace/'source/.env.production.local','x') as stream: ["python3", f"{workspace}/host.py", "smoke", "--workspace", workspace]), timeout=240)) write_json(local / "smoke.json", smoke) print(json.dumps(smoke), flush=True) + print("[pack] packaging source, build, pinned dependencies and public assets", flush=True) result = json.loads(connections.execute("build", shlex.join( ["python3", f"{workspace}/host.py", "pack", "--workspace", workspace]), timeout=600)) result.update({"builder": remote, "mainRef": commit, @@ -208,9 +209,12 @@ def stage(connections, args): artifact = local / "artifact.tar.gz" if not artifact.exists(): with connections.sftp("build") as transfer: - transfer.get(f"{BUILD_ROOT}/{args.release}/artifact.tar.gz", str(artifact)) + transfer.get(f"{BUILD_ROOT}/{args.release}/artifact.tar.gz", str(artifact), + max_concurrent_prefetch_requests=32) require(sha(artifact) == receipt["artifactSha256"], "Downloaded artifact checksum mismatch") - connections.helper("status") + state = connections.helper("status") + require(state["freeBytes"] > artifact.stat().st_size + 1024**3 + 64 * 1024**2, + "Insufficient space to upload artifact while retaining 1 GiB reserve") connections.execute("prod", f"install -d -m 700 -- {RELEASE_ROOT}") connections.upload("prod", artifact, f"{RELEASE_ROOT}/{args.release}.tar.gz") result = connections.helper("stage", release=args.release, digest=receipt["artifactSha256"]) diff --git a/scripts/release/host.py b/scripts/release/host.py index 1defd00c7..557d6c841 100644 --- a/scripts/release/host.py +++ b/scripts/release/host.py @@ -53,23 +53,27 @@ def json_write(path, value): os.replace(temporary, path) +def validate_members(members, destination): + for member in members: + path = PurePosixPath(member.name) + require(not path.is_absolute() and ".." not in path.parts, + "Unsafe archive member") + require(member.isdir() or member.isfile() or member.issym(), + "Special files and hard links are not accepted") + if member.issym(): + link = PurePosixPath(member.linkname) + require(not link.is_absolute(), "Absolute archive symlink") + target = (destination / path.parent / member.linkname).resolve() + require(target.is_relative_to(destination.resolve()), + "Archive symlink escapes destination") + + def safe_extract(archive, destination): """Reject traversal, special files and links escaping a new directory.""" require(not destination.exists(), "Extraction destination already exists") with tarfile.open(archive, "r:gz") as bundle: members = bundle.getmembers() - for member in members: - path = PurePosixPath(member.name) - require(not path.is_absolute() and ".." not in path.parts, - "Unsafe archive member") - require(member.isdir() or member.isfile() or member.issym(), - "Special files and hard links are not accepted") - if member.issym(): - link = PurePosixPath(member.linkname) - require(not link.is_absolute(), "Absolute archive symlink") - target = (destination / path.parent / member.linkname).resolve() - require(target.is_relative_to(destination.resolve()), - "Archive symlink escapes destination") + validate_members(members, destination) needed = sum(m.size for m in members if m.isfile()) require(shutil.disk_usage(destination.parent).free > needed + RESERVE, "Insufficient space: at least 1 GiB must remain after extraction") @@ -108,7 +112,7 @@ def app_state(): env = app["pm2_env"] cwd = Path(env["pm_cwd"]) require(cwd == APP or cwd.parent == RELEASES, "Unexpected production cwd") - require(env["pm_exec_path"] == str(APP / "node_modules/next/dist/bin/next"), + require(env["pm_exec_path"] == str(cwd / "node_modules/next/dist/bin/next"), "Unexpected Next.js executable") require(env.get("watch") is False, "PM2 watch must be disabled") require(env.get("CANVAS_E2E_LOCAL") != "true", "Production E2E mode is enabled") @@ -124,6 +128,7 @@ def status(): return {"hostname": socket.gethostname(), "pid": app["pid"], "status": app["pm2_env"]["status"], "cwd": str(cwd), "distDir": dist, "buildId": (cwd / dist / "BUILD_ID").read_text().strip(), + "freeBytes": shutil.disk_usage(APP).free, "freeGiB": round(shutil.disk_usage(APP).free / 1024**3, 2), "node": run(["node", "-v"]).strip()} @@ -169,40 +174,30 @@ const sql=require('postgres')(process.env.POSTGRES_URL,{max:1,connect_timeout:5, return json.loads(run(["node", "-"], cwd=APP, stdin=source, timeout=50)) -def inventory_packages(root): - result = {} - for path in sorted((root / "node_modules/.pnpm").glob("*/node_modules/*/package.json")): - result[path.relative_to(root / "node_modules").as_posix()] = sha(path) - for path in sorted((root / "node_modules/.pnpm").glob("*/node_modules/@*/*/package.json")): - result[path.relative_to(root / "node_modules").as_posix()] = sha(path) - return result - - def build_manifest(workspace): source = workspace / "source" metadata = json.loads((workspace / "input.json").read_text()) - result = {"schema": 1, "sourceCommit": metadata["sourceCommit"], + runtime_files, runtime_links = {}, {} + for name in ["public", "node_modules"]: + for path in sorted((source / name).rglob("*")): + relative = path.relative_to(source).as_posix() + if path.is_symlink(): + runtime_links[relative] = os.readlink(path) + elif path.is_file(): + runtime_files[relative] = {"sha256": sha(path), "bytes": path.stat().st_size} + result = {"schema": 2, "sourceCommit": metadata["sourceCommit"], "sourceTree": metadata["sourceTree"], "releaseId": workspace.name, "nodeMajor": 20, "nodeAbi": "115", "pnpm": "9.12.3", "buildId": (source / ".next/BUILD_ID").read_text().strip(), "lockSha256": sha(source / "pnpm-lock.yaml"), - "publicFiles": {p.relative_to(source).as_posix(): sha(p) - for p in sorted((source / "public").rglob("*")) if p.is_file()}, - "directPackages": {name: sha(source / "node_modules" / name / "package.json") - for name in json.loads((source / "package.json").read_text())["dependencies"]}, - "installedPackages": inventory_packages(source)} - require(result["installedPackages"], "No dependency inventory") + "runtimeFiles": runtime_files, "runtimeLinks": runtime_links} + require(runtime_files, "No runtime inventory") json_write(source / "release.json", result) - with tarfile.open(workspace / "artifact.tar.gz", "w:gz", compresslevel=3) as archive: - for path in sorted(source.iterdir()): - if path.name in {"node_modules", "public", ".env.production.local", ".next"}: - continue - archive.add(path, arcname=path.name) - def filter_build(member): - if member.name == ".next/cache" or member.name.startswith(".next/cache/"): - return None - return member - archive.add(source / ".next", arcname=".next", filter=filter_build) + # Preserve relative package links, but materialize hard links as ordinary + # archive entries so the extractor never has to trust archive hard links. + run(["nice", "-n", "10", "tar", "--hard-dereference", "--exclude=./.next/cache", + "--exclude=./.env.production.local", "-czf", str(workspace / "artifact.tar.gz"), + "-C", str(source), "."], timeout=600) return {"releaseId": workspace.name, "sourceCommit": result["sourceCommit"], "buildId": result["buildId"], "artifactSha256": sha(workspace / "artifact.tar.gz"), "artifactBytes": (workspace / "artifact.tar.gz").stat().st_size} @@ -247,37 +242,95 @@ def smoke(workspace): process.wait() -def verify_shared(manifest): - require(sha(APP / "pnpm-lock.yaml") == manifest["lockSha256"], - "Dependencies changed: provision a separate reviewed runtime dependency set first") - for name, digest in manifest["publicFiles"].items(): - path = PurePosixPath(name) - require(path.parts[0] == "public" and ".." not in path.parts, - "Invalid public path") - require((APP / name).is_file() and sha(APP / name) == digest, - f"Public files differ: {name}; review shared asset publication first") - actual = inventory_packages(APP) - missing = [name for name, value in manifest["installedPackages"].items() - if actual.get(name) != value] - require(not missing, f"Installed dependency mismatch ({len(missing)} packages)") - for name, digest in manifest["directPackages"].items(): - require(sha(APP / "node_modules" / name / "package.json") == digest, - f"Direct dependency link mismatch: {name}") +def verify_runtime(manifest, root): + require(manifest["schema"] == 2, "Unsupported artifact schema") + require(sha(root / "pnpm-lock.yaml") == manifest["lockSha256"], "Release lock mismatch") + for name, metadata in manifest["runtimeFiles"].items(): + require(sha(root / name) == metadata["sha256"], f"Runtime file mismatch: {name}") + for name, target in manifest["runtimeLinks"].items(): + require((root / name).is_symlink() and os.readlink(root / name) == target, + f"Runtime link mismatch: {name}") require(run(["node", "-p", "process.versions.modules"]).strip() == manifest["nodeAbi"], "Node ABI mismatch") +def clone_file(source, destination): + """Independent copy-on-write inode; never hard-link writable dependencies.""" + import fcntl + with open(source, "rb") as origin, open(destination, "xb") as target: + # Linux FICLONE. Stop on unsupported filesystems, with no full-copy fallback. + fcntl.ioctl(target.fileno(), 0x40049409, origin.fileno()) + + +def extract_runtime(archive, destination, current): + require(not destination.exists(), "Release destination already exists") + with tarfile.open(archive, "r:gz") as bundle: + members = bundle.getmembers() + validate_members(members, destination) + names = {m.name.removeprefix("./"): m for m in members} + manifest = json.load(bundle.extractfile(names["release.json"])) + require(manifest["schema"] == 2, "Unsupported artifact schema") + reusable, needed = set(), 0 + for member in members: + if not member.isfile(): + continue + name = member.name.removeprefix("./") + expected = manifest["runtimeFiles"].get(name) + source = current / name + if expected and source.is_file() and source.stat().st_size == member.size and sha(source) == expected["sha256"]: + reusable.add(name) + else: + needed += member.size + # Allow for inode/directory allocation as well as newly written file data. + require(shutil.disk_usage(destination.parent).free > needed + RESERVE + 64 * 1024**2, + "Insufficient space for release delta and 1 GiB reserve") + destination.mkdir(mode=0o700) + run(["cp", "-a", "--reflink=always", "--", str(current / "public"), str(destination / "public")], timeout=120) + for member in members: + if member.issym(): + continue + name = member.name.removeprefix("./") + target = destination / name + require(target.parent.resolve().is_relative_to(destination.resolve()) or target == destination, + "Extraction parent escapes release") + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + continue + target.parent.mkdir(parents=True, exist_ok=True) + if target.is_symlink(): + target.unlink() + if name in reusable: + if not target.exists(): + clone_file(current / name, target) + else: + with bundle.extractfile(member) as origin, open(target, "wb") as output: + shutil.copyfileobj(origin, output) + target.chmod(member.mode & 0o777) + for member in members: + if member.issym(): + target = destination / member.name.removeprefix("./") + require(target.parent.resolve().is_relative_to(destination.resolve()), "Link parent escapes release") + target.parent.mkdir(parents=True, exist_ok=True) + if target.is_symlink() or target.exists(): + require(not target.is_dir(), "Unexpected directory at link path") + target.unlink() + target.symlink_to(member.linkname) + require(shutil.disk_usage(destination).free > RESERVE, "Post-extraction disk reserve exhausted") + return manifest + + def stage(release_id, digest): _, current, dist = app_state() archive = RELEASES / (release_id + ".tar.gz") require(sha(archive) == digest, "Artifact checksum mismatch") destination = release_path(release_id) - safe_extract(archive, destination) - manifest = json.loads((destination / "release.json").read_text()) + manifest = extract_runtime(archive, destination, current) require(manifest["releaseId"] == release_id, "Artifact release ID mismatch") require(manifest["sourceCommit"][:12] == release_id[-12:], "Artifact source mismatch") - verify_shared(manifest) - for name in ["node_modules", "public", ".env.local", "tmp", ".model-config-backups"]: + verify_runtime(manifest, destination) + run(["node", "-e", "const s=require('sharp');s({create:{width:1,height:1,channels:3,background:'white'}}).png().toBuffer().then(b=>{if(!b.length)process.exit(1)})"], + cwd=destination) + for name in [".env.local", "tmp", ".model-config-backups"]: require(not (destination / name).exists(), f"Unexpected bundled shared path: {name}") if (APP / name).exists(): (destination / name).symlink_to(APP / name) @@ -303,7 +356,7 @@ def ecosystem(app, cwd, dist): old = app["pm2_env"] env = {k: v for k, v in old.items() if k.isupper() and isinstance(v, (str, int, bool))} env.update({"NODE_ENV": "production", "PORT": "3125", "PWD": str(cwd), "NEXT_DIST_DIR": dist}) - return {"apps": [{"name": NAME, "cwd": str(cwd), "script": old["pm_exec_path"], + return {"apps": [{"name": NAME, "cwd": str(cwd), "script": str(cwd / "node_modules/next/dist/bin/next"), "args": old["args"], "interpreter": old.get("exec_interpreter", "/usr/bin/node"), "exec_mode": "fork", "instances": 1, "watch": False, "env": env, "autorestart": old.get("autorestart", True), @@ -342,7 +395,7 @@ def activate(release_id): require(str(current) == staged["previousCwd"] and dist == staged["previousDistDir"], "Production changed after staging") require(not (destination / "activation.json").exists(), "Already activated; inspect status") - verify_shared(manifest) + verify_runtime(manifest, destination) jobs = job_status() require(not any(row["recent"] for rows in jobs.values() for row in rows), "Recent active generation jobs: defer activation") diff --git a/tests/scripts/test_production48.py b/tests/scripts/test_production48.py index d307fb70b..385d2a922 100644 --- a/tests/scripts/test_production48.py +++ b/tests/scripts/test_production48.py @@ -83,6 +83,67 @@ class ArchiveTests(unittest.TestCase): self.assertFalse(self.destination.exists()) +class RuntimeSnapshotTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name).resolve() + self.current = self.root / "current" + (self.current / "public").mkdir(parents=True) + (self.current / "node_modules/pkg").mkdir(parents=True) + (self.current / "public/asset.txt").write_bytes(b"old-public") + (self.current / "public/untracked-font.ttf").write_bytes(b"runtime-font") + (self.current / "node_modules/pkg/index.js").write_bytes(b"same-package") + self.destination = self.root / "release" + self.archive = self.root / "artifact.tar.gz" + data = {"public/asset.txt": b"new-public", + "node_modules/pkg/index.js": b"same-package", + "node_modules/patched/index.js": b"new-patch"} + manifest = {"schema": 2, "runtimeFiles": { + name: {"sha256": host.hashlib.sha256(body).hexdigest(), "bytes": len(body)} + for name, body in data.items()}, "runtimeLinks": {}} + data["release.json"] = json.dumps(manifest).encode() + with tarfile.open(self.archive, "w:gz") as bundle: + for name, body in data.items(): + info = tarfile.TarInfo("./" + name) + info.size = len(body) + bundle.addfile(info, io.BytesIO(body)) + + def tearDown(self): + self.temporary.cleanup() + + def copy_public(self, args, **kwargs): + self.assertEqual(args[:3], ["cp", "-a", "--reflink=always"]) + shutil.copytree(args[-2], args[-1]) + return "" + + def test_snapshot_keeps_untracked_assets_and_applies_patched_dependencies(self): + with patch.object(host, "run", side_effect=self.copy_public), \ + patch.object(host, "clone_file", side_effect=shutil.copyfile) as clone: + host.extract_runtime(self.archive, self.destination, self.current) + clone.assert_called_once() + self.assertEqual((self.destination / "public/untracked-font.ttf").read_bytes(), b"runtime-font") + self.assertEqual((self.destination / "public/asset.txt").read_bytes(), b"new-public") + self.assertEqual((self.current / "public/asset.txt").read_bytes(), b"old-public") + self.assertEqual((self.destination / "node_modules/patched/index.js").read_bytes(), b"new-patch") + (self.destination / "node_modules/pkg/index.js").write_bytes(b"changed") + self.assertEqual((self.current / "node_modules/pkg/index.js").read_bytes(), b"same-package") + + def test_space_guard_runs_before_copying_or_mutating(self): + with patch.object(host.shutil, "disk_usage", return_value=shutil._ntuple_diskusage(10, 9, 1)), \ + patch.object(host, "run") as copy: + with self.assertRaisesRegex(RuntimeError, "Insufficient space"): + host.extract_runtime(self.archive, self.destination, self.current) + copy.assert_not_called() + self.assertFalse(self.destination.exists()) + + def test_unsupported_clone_stops_without_falling_back_to_full_copy(self): + with patch.object(host, "run", side_effect=self.copy_public), \ + patch.object(host, "clone_file", side_effect=OSError("unsupported")): + with self.assertRaisesRegex(OSError, "unsupported"): + host.extract_runtime(self.archive, self.destination, self.current) + self.assertEqual((self.current / "node_modules/pkg/index.js").read_bytes(), b"same-package") + + class ActivationTests(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory() @@ -105,7 +166,7 @@ class ActivationTests(unittest.TestCase): "CANVAS_ZCODE_STATE_DIR": "/private/state", "NEXT_DIST_DIR": ".next-old", "pm_err_log_path": "/logs/error", "pm_out_log_path": "/logs/out"}} self.patches = [patch.object(host, "RELEASES", self.releases), - patch.object(host, "verify_shared"), + patch.object(host, "verify_runtime"), patch.object(host, "app_state", return_value=(self.app, self.previous, ".next-old")), patch.object(host.Path, "home", return_value=self.root), patch.object(host, "run", return_value=""),