docs/production-48.md

@@ -18,7 +18,7 @@
 
 ## 操作入口
 
-维护电脑需要 Python 3.9+、Paramiko、Git，以及已信任的 SSH `known_hosts`。
+维护电脑需要 Python 3.9+、Paramiko 3.3+、Git，以及已信任的 SSH `known_hosts`。
 默认使用 `~/.ssh/45.192.99.48.pem` 和跳板密钥 `~/.ssh/youyidian_prod_ed25519`。
 可用 `--key`、`--jump-key`、`--known-hosts` 覆盖本机路径，不要把私钥提交到仓库。
 
@@ -54,7 +54,7 @@ python scripts/production48.py rollback --release <发布ID>
 ```
 
 `stage` 校验传输 SHA256，在 `/var/www/ai-chatbot-releases/<发布ID>` 解包。
-它检查压缩包路径、磁盘容量、锁文件、实际依赖包清单和共享 public 文件校验和。
+它检查压缩包路径、磁盘容量、锁文件、实际依赖文件/链接清单和 public 文件校验和。
 完成解包后至少保留 1 GiB 空间；不足时停止，不自动删除历史构建。
 当前和旧浏览器标签页所需的静态资源通过硬链接保留，不改动旧构建。
 
@@ -64,13 +64,19 @@ python scripts/production48.py rollback --release <发布ID>
 因此会有短暂切换窗口。健康检查同时核对进程目录、构建 ID、首页和路由。
 检查失败自动恢复旧配置并再次验活；通过后执行 `pm2 save`，让重启后使用已验收版本。
 
-## 共享运行资料与边界
+## 运行资料与边界
 
-初版发布复用原目录 `/var/www/ai-chatbot` 中的 `node_modules`、`public`、`.env.local`、
-`tmp` 和模型配置备份，均通过链接使用，不复制、删除或覆盖。
-生产依赖必须与构建依赖的校验清单相符，Node ABI 必须一致。
-若本次修改了依赖锁文件或 public 中的文件，脚本会停止，须先单独审查并准备对应运行资料，
-不能绕过校验强行切换。本流程适用于应用代码迭代；数据库、共享资源和原生 Agent 升级各自单独发布。
+每个发布目录包含构建机按锁文件安装的完整运行依赖和 public 源码资源。
+节点之间不复用 node_modules 链接，避免服务器漏装依赖补丁却被误认为同一版本。
+对内容完全相同的依赖文件，通过 XFS 的写入时复制创建独立 inode，复用底层数据块；
+缺失或不同的文件从已校验制品写入。任何一份副本后续修改均不会改动另一份。
+生产文件系统不支持该操作时停止，不自动回退成可能耗尽磁盘的完整复制。
+public 先从当前运行版本创建写入时复制副本，保留未纳入 Git 的运行资源，再叠加本次源码资源。
+完整文件和链接清单在切换前再次验证，Node ABI 必须一致，并在生产机测试 sharp 原生模块。
+
+原目录 `/var/www/ai-chatbot` 中的 `.env.local`、`tmp` 和模型配置备份仍通过链接使用，
+不会复制、删除或覆盖。新版本也不在生产机执行依赖安装或构建。
+本流程适用于应用代码迭代；数据库、共享资源的删除/迁移和原生 Agent 升级各自单独发布。
 
 原生产 checkout 的支线及未提交改动仍保留，不能据其 Git HEAD 判断已发布版本。
 实际发布证据是：运行 cwd、`.next/BUILD_ID`、该目录 `release.json` 和 `activation.json`。

scripts/production48.py

@@ -194,6 +194,7 @@ with open(workspace/'source/.env.production.local','x') as stream:
         ["python3", f"{workspace}/host.py", "smoke", "--workspace", workspace]), timeout=240))
     write_json(local / "smoke.json", smoke)
     print(json.dumps(smoke), flush=True)
+    print("[pack] packaging source, build, pinned dependencies and public assets", flush=True)
     result = json.loads(connections.execute("build", shlex.join(
         ["python3", f"{workspace}/host.py", "pack", "--workspace", workspace]), timeout=600))
     result.update({"builder": remote, "mainRef": commit,
@@ -208,9 +209,12 @@ def stage(connections, args):
     artifact = local / "artifact.tar.gz"
     if not artifact.exists():
         with connections.sftp("build") as transfer:
-            transfer.get(f"{BUILD_ROOT}/{args.release}/artifact.tar.gz", str(artifact))
+            transfer.get(f"{BUILD_ROOT}/{args.release}/artifact.tar.gz", str(artifact),
+                         max_concurrent_prefetch_requests=32)
     require(sha(artifact) == receipt["artifactSha256"], "Downloaded artifact checksum mismatch")
-    connections.helper("status")
+    state = connections.helper("status")
+    require(state["freeBytes"] > artifact.stat().st_size + 1024**3 + 64 * 1024**2,
+            "Insufficient space to upload artifact while retaining 1 GiB reserve")
     connections.execute("prod", f"install -d -m 700 -- {RELEASE_ROOT}")
     connections.upload("prod", artifact, f"{RELEASE_ROOT}/{args.release}.tar.gz")
     result = connections.helper("stage", release=args.release, digest=receipt["artifactSha256"])

scripts/release/host.py

@@ -53,23 +53,27 @@ def json_write(path, value):
     os.replace(temporary, path)
 
 
+def validate_members(members, destination):
+    for member in members:
+        path = PurePosixPath(member.name)
+        require(not path.is_absolute() and ".." not in path.parts,
+                "Unsafe archive member")
+        require(member.isdir() or member.isfile() or member.issym(),
+                "Special files and hard links are not accepted")
+        if member.issym():
+            link = PurePosixPath(member.linkname)
+            require(not link.is_absolute(), "Absolute archive symlink")
+            target = (destination / path.parent / member.linkname).resolve()
+            require(target.is_relative_to(destination.resolve()),
+                    "Archive symlink escapes destination")
+
+
 def safe_extract(archive, destination):
     """Reject traversal, special files and links escaping a new directory."""
     require(not destination.exists(), "Extraction destination already exists")
     with tarfile.open(archive, "r:gz") as bundle:
         members = bundle.getmembers()
-        for member in members:
-            path = PurePosixPath(member.name)
-            require(not path.is_absolute() and ".." not in path.parts,
-                    "Unsafe archive member")
-            require(member.isdir() or member.isfile() or member.issym(),
-                    "Special files and hard links are not accepted")
-            if member.issym():
-                link = PurePosixPath(member.linkname)
-                require(not link.is_absolute(), "Absolute archive symlink")
-                target = (destination / path.parent / member.linkname).resolve()
-                require(target.is_relative_to(destination.resolve()),
-                        "Archive symlink escapes destination")
+        validate_members(members, destination)
         needed = sum(m.size for m in members if m.isfile())
         require(shutil.disk_usage(destination.parent).free > needed + RESERVE,
                 "Insufficient space: at least 1 GiB must remain after extraction")
@@ -108,7 +112,7 @@ def app_state():
     env = app["pm2_env"]
     cwd = Path(env["pm_cwd"])
     require(cwd == APP or cwd.parent == RELEASES, "Unexpected production cwd")
-    require(env["pm_exec_path"] == str(APP / "node_modules/next/dist/bin/next"),
+    require(env["pm_exec_path"] == str(cwd / "node_modules/next/dist/bin/next"),
             "Unexpected Next.js executable")
     require(env.get("watch") is False, "PM2 watch must be disabled")
     require(env.get("CANVAS_E2E_LOCAL") != "true", "Production E2E mode is enabled")
@@ -124,6 +128,7 @@ def status():
     return {"hostname": socket.gethostname(), "pid": app["pid"],
             "status": app["pm2_env"]["status"], "cwd": str(cwd), "distDir": dist,
             "buildId": (cwd / dist / "BUILD_ID").read_text().strip(),
+            "freeBytes": shutil.disk_usage(APP).free,
             "freeGiB": round(shutil.disk_usage(APP).free / 1024**3, 2),
             "node": run(["node", "-v"]).strip()}
 
@@ -169,40 +174,30 @@ const sql=require('postgres')(process.env.POSTGRES_URL,{max:1,connect_timeout:5,
     return json.loads(run(["node", "-"], cwd=APP, stdin=source, timeout=50))
 
 
-def inventory_packages(root):
-    result = {}
-    for path in sorted((root / "node_modules/.pnpm").glob("*/node_modules/*/package.json")):
-        result[path.relative_to(root / "node_modules").as_posix()] = sha(path)
-    for path in sorted((root / "node_modules/.pnpm").glob("*/node_modules/@*/*/package.json")):
-        result[path.relative_to(root / "node_modules").as_posix()] = sha(path)
-    return result
-
-
 def build_manifest(workspace):
     source = workspace / "source"
     metadata = json.loads((workspace / "input.json").read_text())
-    result = {"schema": 1, "sourceCommit": metadata["sourceCommit"],
+    runtime_files, runtime_links = {}, {}
+    for name in ["public", "node_modules"]:
+        for path in sorted((source / name).rglob("*")):
+            relative = path.relative_to(source).as_posix()
+            if path.is_symlink():
+                runtime_links[relative] = os.readlink(path)
+            elif path.is_file():
+                runtime_files[relative] = {"sha256": sha(path), "bytes": path.stat().st_size}
+    result = {"schema": 2, "sourceCommit": metadata["sourceCommit"],
               "sourceTree": metadata["sourceTree"], "releaseId": workspace.name,
               "nodeMajor": 20, "nodeAbi": "115", "pnpm": "9.12.3",
               "buildId": (source / ".next/BUILD_ID").read_text().strip(),
               "lockSha256": sha(source / "pnpm-lock.yaml"),
-              "publicFiles": {p.relative_to(source).as_posix(): sha(p)
-                              for p in sorted((source / "public").rglob("*")) if p.is_file()},
-              "directPackages": {name: sha(source / "node_modules" / name / "package.json")
-                                 for name in json.loads((source / "package.json").read_text())["dependencies"]},
-              "installedPackages": inventory_packages(source)}
-    require(result["installedPackages"], "No dependency inventory")
+              "runtimeFiles": runtime_files, "runtimeLinks": runtime_links}
+    require(runtime_files, "No runtime inventory")
     json_write(source / "release.json", result)
-    with tarfile.open(workspace / "artifact.tar.gz", "w:gz", compresslevel=3) as archive:
-        for path in sorted(source.iterdir()):
-            if path.name in {"node_modules", "public", ".env.production.local", ".next"}:
-                continue
-            archive.add(path, arcname=path.name)
-        def filter_build(member):
-            if member.name == ".next/cache" or member.name.startswith(".next/cache/"):
-                return None
-            return member
-        archive.add(source / ".next", arcname=".next", filter=filter_build)
+    # Preserve relative package links, but materialize hard links as ordinary
+    # archive entries so the extractor never has to trust archive hard links.
+    run(["nice", "-n", "10", "tar", "--hard-dereference", "--exclude=./.next/cache",
+         "--exclude=./.env.production.local", "-czf", str(workspace / "artifact.tar.gz"),
+         "-C", str(source), "."], timeout=600)
     return {"releaseId": workspace.name, "sourceCommit": result["sourceCommit"],
             "buildId": result["buildId"], "artifactSha256": sha(workspace / "artifact.tar.gz"),
             "artifactBytes": (workspace / "artifact.tar.gz").stat().st_size}
@@ -247,37 +242,95 @@ def smoke(workspace):
                 process.wait()
 
 
-def verify_shared(manifest):
-    require(sha(APP / "pnpm-lock.yaml") == manifest["lockSha256"],
-            "Dependencies changed: provision a separate reviewed runtime dependency set first")
-    for name, digest in manifest["publicFiles"].items():
-        path = PurePosixPath(name)
-        require(path.parts[0] == "public" and ".." not in path.parts,
-                "Invalid public path")
-        require((APP / name).is_file() and sha(APP / name) == digest,
-                f"Public files differ: {name}; review shared asset publication first")
-    actual = inventory_packages(APP)
-    missing = [name for name, value in manifest["installedPackages"].items()
-               if actual.get(name) != value]
-    require(not missing, f"Installed dependency mismatch ({len(missing)} packages)")
-    for name, digest in manifest["directPackages"].items():
-        require(sha(APP / "node_modules" / name / "package.json") == digest,
-                f"Direct dependency link mismatch: {name}")
+def verify_runtime(manifest, root):
+    require(manifest["schema"] == 2, "Unsupported artifact schema")
+    require(sha(root / "pnpm-lock.yaml") == manifest["lockSha256"], "Release lock mismatch")
+    for name, metadata in manifest["runtimeFiles"].items():
+        require(sha(root / name) == metadata["sha256"], f"Runtime file mismatch: {name}")
+    for name, target in manifest["runtimeLinks"].items():
+        require((root / name).is_symlink() and os.readlink(root / name) == target,
+                f"Runtime link mismatch: {name}")
     require(run(["node", "-p", "process.versions.modules"]).strip() == manifest["nodeAbi"],
             "Node ABI mismatch")
 
 
+def clone_file(source, destination):
+    """Independent copy-on-write inode; never hard-link writable dependencies."""
+    import fcntl
+    with open(source, "rb") as origin, open(destination, "xb") as target:
+        # Linux FICLONE. Stop on unsupported filesystems, with no full-copy fallback.
+        fcntl.ioctl(target.fileno(), 0x40049409, origin.fileno())
+
+
+def extract_runtime(archive, destination, current):
+    require(not destination.exists(), "Release destination already exists")
+    with tarfile.open(archive, "r:gz") as bundle:
+        members = bundle.getmembers()
+        validate_members(members, destination)
+        names = {m.name.removeprefix("./"): m for m in members}
+        manifest = json.load(bundle.extractfile(names["release.json"]))
+        require(manifest["schema"] == 2, "Unsupported artifact schema")
+        reusable, needed = set(), 0
+        for member in members:
+            if not member.isfile():
+                continue
+            name = member.name.removeprefix("./")
+            expected = manifest["runtimeFiles"].get(name)
+            source = current / name
+            if expected and source.is_file() and source.stat().st_size == member.size and sha(source) == expected["sha256"]:
+                reusable.add(name)
+            else:
+                needed += member.size
+        # Allow for inode/directory allocation as well as newly written file data.
+        require(shutil.disk_usage(destination.parent).free > needed + RESERVE + 64 * 1024**2,
+                "Insufficient space for release delta and 1 GiB reserve")
+        destination.mkdir(mode=0o700)
+        run(["cp", "-a", "--reflink=always", "--", str(current / "public"), str(destination / "public")], timeout=120)
+        for member in members:
+            if member.issym():
+                continue
+            name = member.name.removeprefix("./")
+            target = destination / name
+            require(target.parent.resolve().is_relative_to(destination.resolve()) or target == destination,
+                    "Extraction parent escapes release")
+            if member.isdir():
+                target.mkdir(parents=True, exist_ok=True)
+                continue
+            target.parent.mkdir(parents=True, exist_ok=True)
+            if target.is_symlink():
+                target.unlink()
+            if name in reusable:
+                if not target.exists():
+                    clone_file(current / name, target)
+            else:
+                with bundle.extractfile(member) as origin, open(target, "wb") as output:
+                    shutil.copyfileobj(origin, output)
+            target.chmod(member.mode & 0o777)
+        for member in members:
+            if member.issym():
+                target = destination / member.name.removeprefix("./")
+                require(target.parent.resolve().is_relative_to(destination.resolve()), "Link parent escapes release")
+                target.parent.mkdir(parents=True, exist_ok=True)
+                if target.is_symlink() or target.exists():
+                    require(not target.is_dir(), "Unexpected directory at link path")
+                    target.unlink()
+                target.symlink_to(member.linkname)
+    require(shutil.disk_usage(destination).free > RESERVE, "Post-extraction disk reserve exhausted")
+    return manifest
+
+
 def stage(release_id, digest):
     _, current, dist = app_state()
     archive = RELEASES / (release_id + ".tar.gz")
     require(sha(archive) == digest, "Artifact checksum mismatch")
     destination = release_path(release_id)
-    safe_extract(archive, destination)
-    manifest = json.loads((destination / "release.json").read_text())
+    manifest = extract_runtime(archive, destination, current)
     require(manifest["releaseId"] == release_id, "Artifact release ID mismatch")
     require(manifest["sourceCommit"][:12] == release_id[-12:], "Artifact source mismatch")
-    verify_shared(manifest)
-    for name in ["node_modules", "public", ".env.local", "tmp", ".model-config-backups"]:
+    verify_runtime(manifest, destination)
+    run(["node", "-e", "const s=require('sharp');s({create:{width:1,height:1,channels:3,background:'white'}}).png().toBuffer().then(b=>{if(!b.length)process.exit(1)})"],
+        cwd=destination)
+    for name in [".env.local", "tmp", ".model-config-backups"]:
         require(not (destination / name).exists(), f"Unexpected bundled shared path: {name}")
         if (APP / name).exists():
             (destination / name).symlink_to(APP / name)
@@ -303,7 +356,7 @@ def ecosystem(app, cwd, dist):
     old = app["pm2_env"]
     env = {k: v for k, v in old.items() if k.isupper() and isinstance(v, (str, int, bool))}
     env.update({"NODE_ENV": "production", "PORT": "3125", "PWD": str(cwd), "NEXT_DIST_DIR": dist})
-    return {"apps": [{"name": NAME, "cwd": str(cwd), "script": old["pm_exec_path"],
+    return {"apps": [{"name": NAME, "cwd": str(cwd), "script": str(cwd / "node_modules/next/dist/bin/next"),
              "args": old["args"], "interpreter": old.get("exec_interpreter", "/usr/bin/node"),
              "exec_mode": "fork", "instances": 1, "watch": False, "env": env,
              "autorestart": old.get("autorestart", True),
@@ -342,7 +395,7 @@ def activate(release_id):
     require(str(current) == staged["previousCwd"] and dist == staged["previousDistDir"],
             "Production changed after staging")
     require(not (destination / "activation.json").exists(), "Already activated; inspect status")
-    verify_shared(manifest)
+    verify_runtime(manifest, destination)
     jobs = job_status()
     require(not any(row["recent"] for rows in jobs.values() for row in rows),
             "Recent active generation jobs: defer activation")

tests/scripts/test_production48.py

@@ -83,6 +83,67 @@ class ArchiveTests(unittest.TestCase):
         self.assertFalse(self.destination.exists())
 
 
+class RuntimeSnapshotTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.root = Path(self.temporary.name).resolve()
+        self.current = self.root / "current"
+        (self.current / "public").mkdir(parents=True)
+        (self.current / "node_modules/pkg").mkdir(parents=True)
+        (self.current / "public/asset.txt").write_bytes(b"old-public")
+        (self.current / "public/untracked-font.ttf").write_bytes(b"runtime-font")
+        (self.current / "node_modules/pkg/index.js").write_bytes(b"same-package")
+        self.destination = self.root / "release"
+        self.archive = self.root / "artifact.tar.gz"
+        data = {"public/asset.txt": b"new-public",
+                "node_modules/pkg/index.js": b"same-package",
+                "node_modules/patched/index.js": b"new-patch"}
+        manifest = {"schema": 2, "runtimeFiles": {
+            name: {"sha256": host.hashlib.sha256(body).hexdigest(), "bytes": len(body)}
+            for name, body in data.items()}, "runtimeLinks": {}}
+        data["release.json"] = json.dumps(manifest).encode()
+        with tarfile.open(self.archive, "w:gz") as bundle:
+            for name, body in data.items():
+                info = tarfile.TarInfo("./" + name)
+                info.size = len(body)
+                bundle.addfile(info, io.BytesIO(body))
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+    def copy_public(self, args, **kwargs):
+        self.assertEqual(args[:3], ["cp", "-a", "--reflink=always"])
+        shutil.copytree(args[-2], args[-1])
+        return ""
+
+    def test_snapshot_keeps_untracked_assets_and_applies_patched_dependencies(self):
+        with patch.object(host, "run", side_effect=self.copy_public), \
+                patch.object(host, "clone_file", side_effect=shutil.copyfile) as clone:
+            host.extract_runtime(self.archive, self.destination, self.current)
+        clone.assert_called_once()
+        self.assertEqual((self.destination / "public/untracked-font.ttf").read_bytes(), b"runtime-font")
+        self.assertEqual((self.destination / "public/asset.txt").read_bytes(), b"new-public")
+        self.assertEqual((self.current / "public/asset.txt").read_bytes(), b"old-public")
+        self.assertEqual((self.destination / "node_modules/patched/index.js").read_bytes(), b"new-patch")
+        (self.destination / "node_modules/pkg/index.js").write_bytes(b"changed")
+        self.assertEqual((self.current / "node_modules/pkg/index.js").read_bytes(), b"same-package")
+
+    def test_space_guard_runs_before_copying_or_mutating(self):
+        with patch.object(host.shutil, "disk_usage", return_value=shutil._ntuple_diskusage(10, 9, 1)), \
+                patch.object(host, "run") as copy:
+            with self.assertRaisesRegex(RuntimeError, "Insufficient space"):
+                host.extract_runtime(self.archive, self.destination, self.current)
+        copy.assert_not_called()
+        self.assertFalse(self.destination.exists())
+
+    def test_unsupported_clone_stops_without_falling_back_to_full_copy(self):
+        with patch.object(host, "run", side_effect=self.copy_public), \
+                patch.object(host, "clone_file", side_effect=OSError("unsupported")):
+            with self.assertRaisesRegex(OSError, "unsupported"):
+                host.extract_runtime(self.archive, self.destination, self.current)
+        self.assertEqual((self.current / "node_modules/pkg/index.js").read_bytes(), b"same-package")
+
+
 class ActivationTests(unittest.TestCase):
     def setUp(self):
         self.temporary = tempfile.TemporaryDirectory()
@@ -105,7 +166,7 @@ class ActivationTests(unittest.TestCase):
                     "CANVAS_ZCODE_STATE_DIR": "/private/state", "NEXT_DIST_DIR": ".next-old",
                     "pm_err_log_path": "/logs/error", "pm_out_log_path": "/logs/out"}}
         self.patches = [patch.object(host, "RELEASES", self.releases),
-                        patch.object(host, "verify_shared"),
+                        patch.object(host, "verify_runtime"),
                         patch.object(host, "app_state", return_value=(self.app, self.previous, ".next-old")),
                         patch.object(host.Path, "home", return_value=self.root),
                         patch.object(host, "run", return_value=""),