Merge pull request 'fix(deploy): 切换时重建单个 PM2 应用定义并保留回滚日志' (#4) from codex/production-48-pm2-definition into main
This commit is contained in:
commit
ae4605d479
@ -60,10 +60,22 @@ python scripts/production48.py rollback --release <发布ID>
|
||||
|
||||
`activate` 会重新核对生产版本和任务状态，拒绝在最近一小时仍有未结束生成任务时切换。
|
||||
更早的历史挂起记录只写入发布回执，不会自动修改数据库中的状态。
|
||||
它保存旧 PM2 配置和启动快照，然后仅更新 `ai-chatbot` 的工作目录并重启这个应用，
|
||||
它保存旧 PM2 配置和启动快照，然后仅重建 `ai-chatbot` 这一项 PM2 定义，
|
||||
让新版本的工作目录、可执行路径和环境变量全部生效，不改变其他应用，
|
||||
因此会有短暂切换窗口。健康检查同时核对进程目录、构建 ID、首页和路由。
|
||||
检查失败自动恢复旧配置并再次验活；通过后执行 `pm2 save`，让重启后使用已验收版本。
|
||||
|
||||
PM2 的普通 restart 会保留旧工作目录和程序路径，因此不能用它替代本流程的单应用定义重建。
|
||||
若一次切换已经明确回滚并验证原版本健康，修复发布工具后，可保留已校验制品并显式重试：
|
||||
|
||||
```powershell
|
||||
python scripts/production48.py activate --release <发布ID> --retry-rolled-back
|
||||
```
|
||||
|
||||
只有日志阶段为 `rolled-back` 且当前工作目录/构建目录仍匹配原版本才允许重试。
|
||||
旧尝试日志另外保留。运行制品的 `sourceCommit` 始终指向当时实际构建的 main 提交，
|
||||
后续工具修复不改写这个来源；`operatorCommit` 单独记录执行切换的维护工具版本。
|
||||
|
||||
## 运行资料与边界
|
||||
|
||||
每个发布目录包含构建机按锁文件安装的完整运行依赖和 public 源码资源。
|
||||
|
||||
@ -226,11 +226,13 @@ def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("action", choices=["status", "jobs", "build", "stage", "activate", "rollback"])
|
||||
parser.add_argument("--release")
|
||||
parser.add_argument("--retry-rolled-back", action="store_true")
|
||||
parser.add_argument("--key", type=Path, default=Path.home() / ".ssh/45.192.99.48.pem")
|
||||
parser.add_argument("--jump-key", type=Path, default=Path.home() / ".ssh/youyidian_prod_ed25519")
|
||||
parser.add_argument("--known-hosts", type=Path, default=Path.home() / ".ssh/known_hosts")
|
||||
parser.add_argument("--receipts", type=Path, default=ROOT / ".git/releases")
|
||||
args = parser.parse_args()
|
||||
require(not args.retry_rolled_back or args.action == "activate", "Retry flag only applies to activation")
|
||||
if args.action in {"stage", "activate", "rollback"}:
|
||||
require(args.release and re.fullmatch(r"\d{8}T\d{6}Z-[0-9a-f]{12}", args.release), "--release is required")
|
||||
args.receipts = args.receipts.resolve()
|
||||
@ -241,6 +243,10 @@ def main():
|
||||
stage(connections, args)
|
||||
else:
|
||||
kwargs = {"release": args.release} if args.release else {}
|
||||
if args.action == "activate":
|
||||
kwargs["operator-commit"] = git("rev-parse", "HEAD")
|
||||
if args.retry_rolled_back:
|
||||
kwargs["retry"] = "rolled-back"
|
||||
result = connections.helper(args.action, **kwargs)
|
||||
if args.release:
|
||||
write_json(args.receipts / args.release / (args.action + ".json"), result)
|
||||
|
||||
@ -384,21 +384,34 @@ def healthy(cwd, build_id):
|
||||
|
||||
|
||||
def restart(config):
|
||||
run(["pm2", "startOrRestart", str(config), "--only", NAME, "--update-env"], timeout=150)
|
||||
apps = json.loads(config.read_text())["apps"]
|
||||
require(len(apps) == 1 and apps[0]["name"] == NAME, "Refusing a multi-application PM2 definition")
|
||||
current = [app for app in json.loads(run(["pm2", "jlist"])) if app["name"] == NAME]
|
||||
require(len(current) <= 1, "Duplicate ai-chatbot definitions; operator required")
|
||||
# PM2 restart updates env but retains the old executable/cwd. Recreate only
|
||||
# this name so the new immutable release's process definition really applies.
|
||||
if current:
|
||||
run(["pm2", "delete", NAME], timeout=150)
|
||||
run(["pm2", "start", str(config), "--only", NAME], timeout=150)
|
||||
|
||||
|
||||
def activate(release_id):
|
||||
def activate(release_id, retry=False, operator_commit=None):
|
||||
destination = release_path(release_id)
|
||||
manifest = json.loads((destination / "release.json").read_text())
|
||||
staged = json.loads((destination / "staged.json").read_text())
|
||||
app, current, dist = app_state()
|
||||
require(str(current) == staged["previousCwd"] and dist == staged["previousDistDir"],
|
||||
"Production changed after staging")
|
||||
require(not (destination / "activation.json").exists(), "Already activated; inspect status")
|
||||
journal = destination / "activation.json"
|
||||
prior = json.loads(journal.read_text()) if journal.exists() else None
|
||||
require(not prior or (retry and prior.get("phase") == "rolled-back"),
|
||||
"Already activated; inspect status before explicitly retrying a rolled-back attempt")
|
||||
verify_runtime(manifest, destination)
|
||||
jobs = job_status()
|
||||
require(not any(row["recent"] for rows in jobs.values() for row in rows),
|
||||
"Recent active generation jobs: defer activation")
|
||||
if prior:
|
||||
json_write(destination / f"activation-history-{time.time_ns()}.json", prior)
|
||||
before = ecosystem(app, current, dist)
|
||||
previous_build = (current / dist / "BUILD_ID").read_text().strip()
|
||||
json_write(destination / "previous.pm2.json", before)
|
||||
@ -410,7 +423,7 @@ def activate(release_id):
|
||||
"previousBuildId": previous_build, "previousCwd": str(current),
|
||||
"activatedAtUtc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"historicalTaskStates": jobs, "artifactSha256": staged["artifactSha256"],
|
||||
"phase": "activating"}
|
||||
"phase": "activating", "operatorCommit": operator_commit}
|
||||
json_write(destination / "activation.json", receipt)
|
||||
try:
|
||||
restart(destination / "candidate.pm2.json")
|
||||
@ -453,6 +466,8 @@ def main():
|
||||
parser.add_argument("--release")
|
||||
parser.add_argument("--digest")
|
||||
parser.add_argument("--workspace", type=Path)
|
||||
parser.add_argument("--retry", choices=["rolled-back"])
|
||||
parser.add_argument("--operator-commit")
|
||||
args = parser.parse_args()
|
||||
os.umask(0o077)
|
||||
if args.action in {"pack", "smoke"}:
|
||||
@ -469,7 +484,8 @@ def main():
|
||||
if args.action == "stage":
|
||||
result = stage(args.release, args.digest)
|
||||
elif args.action == "activate":
|
||||
result = activate(args.release)
|
||||
result = activate(args.release, retry=args.retry == "rolled-back",
|
||||
operator_commit=args.operator_commit)
|
||||
else:
|
||||
result = rollback(args.release)
|
||||
print(json.dumps(result))
|
||||
|
||||
@ -222,6 +222,59 @@ class ActivationTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(RuntimeError, "Rollback health check failed"):
|
||||
host.activate(self.release_id)
|
||||
|
||||
def test_explicit_retry_preserves_the_failed_attempt(self):
|
||||
host.json_write(self.target / "activation.json", {"phase": "rolled-back", "previousBuildId": "old-build"})
|
||||
with patch.object(host, "healthy", return_value=True):
|
||||
result = host.activate(self.release_id, retry=True, operator_commit="tool-commit")
|
||||
self.assertEqual(result["phase"], "active")
|
||||
self.assertEqual(result["operatorCommit"], "tool-commit")
|
||||
history = list(self.target.glob("activation-history-*.json"))
|
||||
self.assertEqual(len(history), 1)
|
||||
self.assertEqual(json.loads(history[0].read_text())["phase"], "rolled-back")
|
||||
|
||||
def test_retry_cannot_replace_an_active_attempt(self):
|
||||
host.json_write(self.target / "activation.json", {"phase": "active"})
|
||||
with self.assertRaisesRegex(RuntimeError, "Already activated"):
|
||||
host.activate(self.release_id, retry=True)
|
||||
self.restart.assert_not_called()
|
||||
|
||||
def test_failed_process_start_also_restores_previous_definition(self):
|
||||
self.restart.side_effect = [RuntimeError("candidate start failed"), None]
|
||||
with patch.object(host, "healthy", return_value=True):
|
||||
with self.assertRaisesRegex(RuntimeError, "candidate start failed"):
|
||||
host.activate(self.release_id)
|
||||
self.assertEqual(self.restart.call_args_list[-1].args[0].name, "previous.pm2.json")
|
||||
|
||||
|
||||
class ProcessDefinitionTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.config = Path(self.temporary.name) / "candidate.json"
|
||||
self.config.write_text(json.dumps({"apps": [{"name": "ai-chatbot", "cwd": "/new/release"}]}))
|
||||
|
||||
def tearDown(self):
|
||||
self.temporary.cleanup()
|
||||
|
||||
def test_recreates_only_target_application(self):
|
||||
with patch.object(host, "run", side_effect=[json.dumps([{"name": "ai-chatbot"}, {"name": "other-service"}]), "", ""]) as run:
|
||||
host.restart(self.config)
|
||||
self.assertEqual([call.args[0] for call in run.call_args_list], [
|
||||
["pm2", "jlist"], ["pm2", "delete", "ai-chatbot"],
|
||||
["pm2", "start", str(self.config), "--only", "ai-chatbot"]])
|
||||
|
||||
def test_can_restore_after_failed_candidate_left_no_process(self):
|
||||
with patch.object(host, "run", side_effect=["[]", ""]) as run:
|
||||
host.restart(self.config)
|
||||
self.assertEqual(len(run.call_args_list), 2)
|
||||
self.assertEqual(run.call_args_list[-1].args[0][1], "start")
|
||||
|
||||
def test_refuses_definitions_that_include_other_applications(self):
|
||||
self.config.write_text(json.dumps({"apps": [{"name": "ai-chatbot"}, {"name": "other"}]}))
|
||||
with patch.object(host, "run") as run:
|
||||
with self.assertRaisesRegex(RuntimeError, "multi-application"):
|
||||
host.restart(self.config)
|
||||
run.assert_not_called()
|
||||
|
||||
|
||||
class HelperTests(unittest.TestCase):
|
||||
def test_production_e2e_switch_blocks_release(self):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user