diff --git a/docs/production-48.md b/docs/production-48.md index 8872564e3..64bdd8e16 100644 --- a/docs/production-48.md +++ b/docs/production-48.md @@ -60,10 +60,22 @@ python scripts/production48.py rollback --release <发布ID> `activate` 会重新核对生产版本和任务状态，拒绝在最近一小时仍有未结束生成任务时切换。 更早的历史挂起记录只写入发布回执，不会自动修改数据库中的状态。 -它保存旧 PM2 配置和启动快照，然后仅更新 `ai-chatbot` 的工作目录并重启这个应用， +它保存旧 PM2 配置和启动快照，然后仅重建 `ai-chatbot` 这一项 PM2 定义， +让新版本的工作目录、可执行路径和环境变量全部生效，不改变其他应用， 因此会有短暂切换窗口。健康检查同时核对进程目录、构建 ID、首页和路由。 检查失败自动恢复旧配置并再次验活；通过后执行 `pm2 save`，让重启后使用已验收版本。 +PM2 的普通 restart 会保留旧工作目录和程序路径，因此不能用它替代本流程的单应用定义重建。 +若一次切换已经明确回滚并验证原版本健康，修复发布工具后，可保留已校验制品并显式重试： + +```powershell +python scripts/production48.py activate --release <发布ID> --retry-rolled-back +``` + +只有日志阶段为 `rolled-back` 且当前工作目录/构建目录仍匹配原版本才允许重试。 +旧尝试日志另外保留。运行制品的 `sourceCommit` 始终指向当时实际构建的 main 提交， +后续工具修复不改写这个来源；`operatorCommit` 单独记录执行切换的维护工具版本。 + ## 运行资料与边界 每个发布目录包含构建机按锁文件安装的完整运行依赖和 public 源码资源。 diff --git a/scripts/production48.py b/scripts/production48.py index d6cbcfbb7..ab1db7330 100644 --- a/scripts/production48.py +++ b/scripts/production48.py @@ -226,11 +226,13 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("action", choices=["status", "jobs", "build", "stage", "activate", "rollback"]) parser.add_argument("--release") + parser.add_argument("--retry-rolled-back", action="store_true") parser.add_argument("--key", type=Path, default=Path.home() / ".ssh/45.192.99.48.pem") parser.add_argument("--jump-key", type=Path, default=Path.home() / ".ssh/youyidian_prod_ed25519") parser.add_argument("--known-hosts", type=Path, default=Path.home() / ".ssh/known_hosts") parser.add_argument("--receipts", type=Path, default=ROOT / ".git/releases") args = parser.parse_args() + require(not args.retry_rolled_back or args.action == "activate", "Retry flag only applies to activation") if args.action in {"stage", "activate", "rollback"}: require(args.release and re.fullmatch(r"\d{8}T\d{6}Z-[0-9a-f]{12}", args.release), "--release is required") args.receipts = args.receipts.resolve() @@ -241,6 +243,10 @@ def main(): stage(connections, args) else: kwargs = {"release": args.release} if args.release else {} + if args.action == "activate": + kwargs["operator-commit"] = git("rev-parse", "HEAD") + if args.retry_rolled_back: + kwargs["retry"] = "rolled-back" result = connections.helper(args.action, **kwargs) if args.release: write_json(args.receipts / args.release / (args.action + ".json"), result) diff --git a/scripts/release/host.py b/scripts/release/host.py index 557d6c841..fc499689a 100644 --- a/scripts/release/host.py +++ b/scripts/release/host.py @@ -384,21 +384,34 @@ def healthy(cwd, build_id): def restart(config): - run(["pm2", "startOrRestart", str(config), "--only", NAME, "--update-env"], timeout=150) + apps = json.loads(config.read_text())["apps"] + require(len(apps) == 1 and apps[0]["name"] == NAME, "Refusing a multi-application PM2 definition") + current = [app for app in json.loads(run(["pm2", "jlist"])) if app["name"] == NAME] + require(len(current) <= 1, "Duplicate ai-chatbot definitions; operator required") + # PM2 restart updates env but retains the old executable/cwd. Recreate only + # this name so the new immutable release's process definition really applies. + if current: + run(["pm2", "delete", NAME], timeout=150) + run(["pm2", "start", str(config), "--only", NAME], timeout=150) -def activate(release_id): +def activate(release_id, retry=False, operator_commit=None): destination = release_path(release_id) manifest = json.loads((destination / "release.json").read_text()) staged = json.loads((destination / "staged.json").read_text()) app, current, dist = app_state() require(str(current) == staged["previousCwd"] and dist == staged["previousDistDir"], "Production changed after staging") - require(not (destination / "activation.json").exists(), "Already activated; inspect status") + journal = destination / "activation.json" + prior = json.loads(journal.read_text()) if journal.exists() else None + require(not prior or (retry and prior.get("phase") == "rolled-back"), + "Already activated; inspect status before explicitly retrying a rolled-back attempt") verify_runtime(manifest, destination) jobs = job_status() require(not any(row["recent"] for rows in jobs.values() for row in rows), "Recent active generation jobs: defer activation") + if prior: + json_write(destination / f"activation-history-{time.time_ns()}.json", prior) before = ecosystem(app, current, dist) previous_build = (current / dist / "BUILD_ID").read_text().strip() json_write(destination / "previous.pm2.json", before) @@ -410,7 +423,7 @@ def activate(release_id): "previousBuildId": previous_build, "previousCwd": str(current), "activatedAtUtc": datetime.datetime.now(datetime.timezone.utc).isoformat(), "historicalTaskStates": jobs, "artifactSha256": staged["artifactSha256"], - "phase": "activating"} + "phase": "activating", "operatorCommit": operator_commit} json_write(destination / "activation.json", receipt) try: restart(destination / "candidate.pm2.json") @@ -453,6 +466,8 @@ def main(): parser.add_argument("--release") parser.add_argument("--digest") parser.add_argument("--workspace", type=Path) + parser.add_argument("--retry", choices=["rolled-back"]) + parser.add_argument("--operator-commit") args = parser.parse_args() os.umask(0o077) if args.action in {"pack", "smoke"}: @@ -469,7 +484,8 @@ def main(): if args.action == "stage": result = stage(args.release, args.digest) elif args.action == "activate": - result = activate(args.release) + result = activate(args.release, retry=args.retry == "rolled-back", + operator_commit=args.operator_commit) else: result = rollback(args.release) print(json.dumps(result)) diff --git a/tests/scripts/test_production48.py b/tests/scripts/test_production48.py index 385d2a922..79d9b9501 100644 --- a/tests/scripts/test_production48.py +++ b/tests/scripts/test_production48.py @@ -222,6 +222,59 @@ class ActivationTests(unittest.TestCase): with self.assertRaisesRegex(RuntimeError, "Rollback health check failed"): host.activate(self.release_id) + def test_explicit_retry_preserves_the_failed_attempt(self): + host.json_write(self.target / "activation.json", {"phase": "rolled-back", "previousBuildId": "old-build"}) + with patch.object(host, "healthy", return_value=True): + result = host.activate(self.release_id, retry=True, operator_commit="tool-commit") + self.assertEqual(result["phase"], "active") + self.assertEqual(result["operatorCommit"], "tool-commit") + history = list(self.target.glob("activation-history-*.json")) + self.assertEqual(len(history), 1) + self.assertEqual(json.loads(history[0].read_text())["phase"], "rolled-back") + + def test_retry_cannot_replace_an_active_attempt(self): + host.json_write(self.target / "activation.json", {"phase": "active"}) + with self.assertRaisesRegex(RuntimeError, "Already activated"): + host.activate(self.release_id, retry=True) + self.restart.assert_not_called() + + def test_failed_process_start_also_restores_previous_definition(self): + self.restart.side_effect = [RuntimeError("candidate start failed"), None] + with patch.object(host, "healthy", return_value=True): + with self.assertRaisesRegex(RuntimeError, "candidate start failed"): + host.activate(self.release_id) + self.assertEqual(self.restart.call_args_list[-1].args[0].name, "previous.pm2.json") + + +class ProcessDefinitionTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.config = Path(self.temporary.name) / "candidate.json" + self.config.write_text(json.dumps({"apps": [{"name": "ai-chatbot", "cwd": "/new/release"}]})) + + def tearDown(self): + self.temporary.cleanup() + + def test_recreates_only_target_application(self): + with patch.object(host, "run", side_effect=[json.dumps([{"name": "ai-chatbot"}, {"name": "other-service"}]), "", ""]) as run: + host.restart(self.config) + self.assertEqual([call.args[0] for call in run.call_args_list], [ + ["pm2", "jlist"], ["pm2", "delete", "ai-chatbot"], + ["pm2", "start", str(self.config), "--only", "ai-chatbot"]]) + + def test_can_restore_after_failed_candidate_left_no_process(self): + with patch.object(host, "run", side_effect=["[]", ""]) as run: + host.restart(self.config) + self.assertEqual(len(run.call_args_list), 2) + self.assertEqual(run.call_args_list[-1].args[0][1], "start") + + def test_refuses_definitions_that_include_other_applications(self): + self.config.write_text(json.dumps({"apps": [{"name": "ai-chatbot"}, {"name": "other"}]})) + with patch.object(host, "run") as run: + with self.assertRaisesRegex(RuntimeError, "multi-application"): + host.restart(self.config) + run.assert_not_called() + class HelperTests(unittest.TestCase): def test_production_e2e_switch_blocks_release(self):