build: 建立 main 到 48 / PM2 的可回滚发布流程 #2

Merged
WuYongJie merged 1 commits from codex/production-48-release-pipeline into main 2026-10-30 18:23:22 +08:00
12 changed files with 992 additions and 485 deletions

1
.gitattributes vendored
View File

@ -15,5 +15,6 @@
*.html text eol=lf
*.md text eol=lf
*.sh text eol=lf
*.py text eol=lf
*.bat text eol=crlf
*.cmd text eol=crlf

4
.gitignore vendored
View File

@ -134,3 +134,7 @@ services/video-upscale/vendor/__pycache__/
/services/audio-processing/dist/
/services/audio-processing/__pycache__/
# Python release-tool and test bytecode.
__pycache__/
*.pyc

View File

@ -1,303 +1,6 @@
#!/bin/bash
# 自动部署脚本 - 包含密码
# 使用方法: ./auto-deploy.sh
set -e
# 服务器配置
SERVER="156.225.27.52"
PORT="5204"
USER="root"
PASSWORD="${DEPLOY_SSH_PASSWORD:?set DEPLOY_SSH_PASSWORD}"
REMOTE_DIR="/var/www/ai-chatbot"
# 颜色输出
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'
echo -e "${GREEN}开始自动部署到服务器...${NC}"
# 检查是否安装了 sshpass
if ! command -v sshpass &> /dev/null; then
echo -e "${YELLOW}安装 sshpass...${NC}"
if [[ "$OSTYPE" == "darwin"* ]]; then
# macOS
if ! command -v brew &> /dev/null; then
echo -e "${RED}请先安装 Homebrew: /bin/bash -c \"\$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)\"${NC}"
exit 1
fi
brew install hudochenkov/sshpass/sshpass
else
# Linux
sudo apt-get update && sudo apt-get install -y sshpass
fi
fi
# SSH 连接函数
ssh_cmd() {
sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no -p $PORT $USER@$SERVER "$1"
}
scp_cmd() {
sshpass -p "$PASSWORD" scp -o StrictHostKeyChecking=no -P $PORT "$1" $USER@$SERVER:"$2"
}
scp_dir() {
sshpass -p "$PASSWORD" scp -o StrictHostKeyChecking=no -P $PORT -r "$1" $USER@$SERVER:"$2"
}
echo -e "${GREEN}步骤 1: 检查服务器连接...${NC}"
ssh_cmd "echo '连接成功' && uname -a"
echo -e "${GREEN}步骤 2: 检查并安装必要软件...${NC}"
ssh_cmd "
# 检查 Node.js
if ! command -v node &> /dev/null; then
echo '安装 Node.js...'
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
export NVM_DIR=\"\$HOME/.nvm\"
[ -s \"\$NVM_DIR/nvm.sh\" ] && . \"\$NVM_DIR/nvm.sh\"
nvm install 20
nvm use 20
nvm alias default 20
fi
# 检查 pnpm
if ! command -v pnpm &> /dev/null; then
echo '安装 pnpm...'
npm install -g pnpm
fi
# 检查 PM2
if ! command -v pm2 &> /dev/null; then
echo '安装 PM2...'
npm install -g pm2
fi
# 检查 PostgreSQL
if ! command -v psql &> /dev/null; then
echo '安装 PostgreSQL...'
apt-get update -y
apt-get install -y postgresql postgresql-contrib
systemctl start postgresql
systemctl enable postgresql
fi
# 检查 Nginx
if ! command -v nginx &> /dev/null; then
echo '安装 Nginx...'
apt-get install -y nginx
systemctl start nginx
systemctl enable nginx
fi
# 创建项目目录
mkdir -p $REMOTE_DIR
mkdir -p /var/log/pm2
mkdir -p /root/backups/ai-chatbot
"
echo -e "${GREEN}步骤 3: 配置数据库...${NC}"
ssh_cmd "
# 检查并创建/更新数据库用户
echo '配置数据库用户...'
sudo -u postgres psql << 'PSQL'
-- 删除已存在的用户(如果存在)
DROP USER IF EXISTS chatbot_user;
-- 创建新用户
CREATE USER chatbot_user WITH PASSWORD 'ChatBot@2024!Secure';
-- 检查数据库是否存在
SELECT 1 FROM pg_database WHERE datname='ai_chatbot'\\gexec
-- 如果不存在则创建
DO \\$\\$
BEGIN
IF NOT EXISTS (SELECT FROM pg_database WHERE datname = 'ai_chatbot') THEN
CREATE DATABASE ai_chatbot;
END IF;
END
\\$\\$;
-- 授予权限
GRANT ALL PRIVILEGES ON DATABASE ai_chatbot TO chatbot_user;
-- 连接到数据库并授予schema权限
\\c ai_chatbot
GRANT ALL ON SCHEMA public TO chatbot_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO chatbot_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO chatbot_user;
\\q
PSQL
echo '数据库配置完成'
"
echo -e "${GREEN}步骤 4: 上传项目文件...${NC}"
# 创建临时部署包
echo "准备部署文件..."
tar --exclude='node_modules' \
--exclude='.next' \
--exclude='.git' \
--exclude='.env.local' \
--exclude='*.log' \
--exclude='deploy.tar.gz' \
-czf /tmp/deploy.tar.gz .
# 上传部署包
scp_cmd "/tmp/deploy.tar.gz" "/tmp/"
# 在服务器上解压
ssh_cmd "
cd $REMOTE_DIR
tar -xzf /tmp/deploy.tar.gz
rm /tmp/deploy.tar.gz
if [ -f \"$REMOTE_DIR/scripts/security/install-security-stack.sh\" ]; then
echo '安装 security-stack...'
bash \"$REMOTE_DIR/scripts/security/install-security-stack.sh\" || echo 'security-stack 安装失败,继续部署'
elif [ -f \"$REMOTE_DIR/scripts/security/install-perf-guard.sh\" ]; then
echo '安装 perf-guard...'
bash \"$REMOTE_DIR/scripts/security/install-perf-guard.sh\" || echo 'perf-guard 安装失败,继续部署'
fi
"
echo -e "${GREEN}步骤 5: 配置环境变量...${NC}"
ssh_cmd "
cd $REMOTE_DIR
# 生成 AUTH_SECRET
AUTH_SECRET=\$(openssl rand -base64 32)
# 创建 .env.local 文件
cat > .env.local << EOF
POSTGRES_URL=postgresql://chatbot_user:ChatBot@2024!Secure@localhost:5432/ai_chatbot
AUTH_SECRET=\$AUTH_SECRET
AUTH_URL=http://$SERVER
NODE_ENV=production
EOF
echo '环境变量文件已创建'
"
echo -e "${GREEN}步骤 6: 安装依赖和构建...${NC}"
ssh_cmd "
cd $REMOTE_DIR
export NVM_DIR=\"\$HOME/.nvm\"
[ -s \"\$NVM_DIR/nvm.sh\" ] && . \"\$NVM_DIR/nvm.sh\"
echo '安装依赖...'
pnpm install --production=false
echo '运行数据库迁移...'
pnpm db:migrate || echo '迁移可能已存在,继续...'
echo '构建项目...'
pnpm build
"
echo -e "${GREEN}步骤 7: 配置 PM2...${NC}"
# 上传 PM2 配置文件
scp_cmd "ecosystem.config.js" "$REMOTE_DIR/"
ssh_cmd "
cd $REMOTE_DIR
export NVM_DIR=\"\$HOME/.nvm\"
[ -s \"\$NVM_DIR/nvm.sh\" ] && . \"\$NVM_DIR/nvm.sh\"
# 停止旧进程(如果存在)
pm2 delete ai-chatbot 2>/dev/null || true
# 启动应用
pm2 start ecosystem.config.js
pm2 save
# 设置开机自启
pm2 startup systemd -u root --hp /root 2>/dev/null || echo 'PM2 startup 已配置'
"
echo -e "${GREEN}步骤 8: 配置 Nginx...${NC}"
ssh_cmd "
# 创建 Nginx 配置
cat > /etc/nginx/sites-available/ai-chatbot << 'NGINX'
server {
listen 80;
server_name $SERVER;
location / {
proxy_pass http://localhost:3000;
proxy_http_version 1.1;
proxy_set_header Upgrade \$http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host \$host;
proxy_set_header X-Real-IP \$remote_addr;
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto \$scheme;
proxy_cache_bypass \$http_upgrade;
proxy_read_timeout 300s;
proxy_connect_timeout 75s;
}
}
NGINX
# 启用站点
ln -sf /etc/nginx/sites-available/ai-chatbot /etc/nginx/sites-enabled/
# 删除默认配置(如果存在)
rm -f /etc/nginx/sites-enabled/default
# 测试配置
nginx -t
# 重启 Nginx
systemctl restart nginx
"
echo -e "${GREEN}步骤 9: 配置防火墙...${NC}"
ssh_cmd "
ufw --force enable
ufw allow 80/tcp
ufw allow 443/tcp
ufw allow 5204/tcp
ufw allow 3000/tcp
"
echo -e "${GREEN}步骤 10: 检查部署状态...${NC}"
ssh_cmd "
echo '=== PM2 状态 ==='
pm2 status
echo ''
echo '=== 应用日志(最后20行)==='
pm2 logs ai-chatbot --lines 20 --nostream
echo ''
echo '=== Nginx 状态 ==='
systemctl status nginx --no-pager -l
echo ''
echo '=== 端口监听 ==='
netstat -tulpn | grep -E ':(80|3000|5432)' || ss -tulpn | grep -E ':(80|3000|5432)'
"
# 清理本地临时文件
rm -f /tmp/deploy.tar.gz
echo -e "${GREEN}========================================${NC}"
echo -e "${GREEN}部署完成!${NC}"
echo -e "${GREEN}========================================${NC}"
echo -e "${YELLOW}访问地址: http://$SERVER${NC}"
echo -e "${YELLOW}管理员设置: http://$SERVER/admin/settings${NC}"
echo -e "${YELLOW}查看日志: ssh -p $PORT $USER@$SERVER 'pm2 logs ai-chatbot'${NC}"
echo -e "${YELLOW}查看状态: ssh -p $PORT $USER@$SERVER 'pm2 status'${NC}"
echo ""
echo -e "${YELLOW}数据库信息:${NC}"
echo -e " 数据库名: ai_chatbot"
echo -e " 用户名: chatbot_user"
echo -e " 密码: ChatBot@2024!Secure"
echo ""
echo -e "${YELLOW}注意: 请尽快修改数据库密码和服务器密码!${NC}"
#!/usr/bin/env bash
set -euo pipefail
printf '%s\n' 'Legacy deployment entry point disabled: it targets an obsolete host/runtime.' >&2
printf '%s\n' 'Use: python scripts/production48.py status' >&2
printf '%s\n' 'Then follow docs/production-48.md for build, stage, activate and rollback.' >&2
exit 2

114
deploy.sh
View File

@ -1,108 +1,6 @@
#!/bin/bash
# AI Chatbot 部署脚本
# 使用方法: ./deploy.sh
set -e
# 配置
SERVER="156.225.27.52"
PORT="5204"
USER="root"
PASSWORD="${DEPLOY_SSH_PASSWORD:?set DEPLOY_SSH_PASSWORD}"
REMOTE_DIR="/var/www/ai-chatbot"
LOCAL_DIR="."
# 颜色输出
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
echo -e "${GREEN}开始部署到服务器...${NC}"
# 检查本地是否有未提交的更改
if [ -n "$(git status --porcelain)" ]; then
echo -e "${YELLOW}警告: 有未提交的更改${NC}"
read -p "是否继续? (y/n) " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
exit 1
fi
fi
# 1. 准备部署文件(跳过本地构建,在服务器上构建)
echo -e "${GREEN}步骤 1: 准备部署文件...${NC}"
tar --exclude='node_modules' \
--exclude='.next' \
--exclude='.git' \
--exclude='.env.local' \
--exclude='.env.production' \
--exclude='*.log' \
--exclude='deploy.tar.gz' \
-czf deploy.tar.gz .
# 2. 上传到服务器
echo -e "${GREEN}步骤 2: 上传文件到服务器...${NC}"
sshpass -p "$PASSWORD" scp -o StrictHostKeyChecking=no -P $PORT deploy.tar.gz $USER@$SERVER:/tmp/
# 3. 在服务器上执行部署
echo -e "${GREEN}步骤 3: 在服务器上部署...${NC}"
sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no -p $PORT $USER@$SERVER << 'ENDSSH'
set -e
REMOTE_DIR="/var/www/ai-chatbot"
BACKUP_DIR="/root/backups/ai-chatbot"
# 创建目录
mkdir -p $REMOTE_DIR
mkdir -p $BACKUP_DIR
# 备份现有部署(如果存在)
if [ -d "$REMOTE_DIR/.next" ]; then
echo "备份现有部署..."
tar -czf $BACKUP_DIR/backup_$(date +%Y%m%d_%H%M%S).tar.gz -C $REMOTE_DIR .
fi
# 解压新文件
echo "解压新文件..."
cd $REMOTE_DIR
tar -xzf /tmp/deploy.tar.gz
# 安装安全基线(恶意进程守护 + 服务器硬化)
if [ -f "$REMOTE_DIR/scripts/security/install-security-stack.sh" ]; then
echo "安装 security-stack..."
bash "$REMOTE_DIR/scripts/security/install-security-stack.sh" || echo "security-stack 安装失败,继续部署"
elif [ -f "$REMOTE_DIR/scripts/security/install-perf-guard.sh" ]; then
echo "安装 perf-guard..."
bash "$REMOTE_DIR/scripts/security/install-perf-guard.sh" || echo "perf-guard 安装失败,继续部署"
fi
# 安装依赖
echo "安装依赖..."
pnpm install --production=false
# 运行数据库迁移
echo "运行数据库迁移..."
pnpm db:migrate || echo "数据库迁移失败,请检查配置"
# 构建项目
echo "构建项目..."
pnpm build
# 重启应用
echo "重启应用..."
pm2 restart ai-chatbot || pm2 start ecosystem.config.js
# 清理
rm /tmp/deploy.tar.gz
echo "部署完成!"
ENDSSH
# 4. 清理本地临时文件
echo -e "${GREEN}步骤 4: 清理临时文件...${NC}"
rm deploy.tar.gz
echo -e "${GREEN}部署完成!${NC}"
echo -e "${YELLOW}请检查应用状态: ssh -p $PORT $USER@$SERVER 'pm2 status'${NC}"
#!/usr/bin/env bash
set -euo pipefail
printf '%s\n' 'Legacy deployment entry point disabled: it targets an obsolete host/runtime.' >&2
printf '%s\n' 'Use: python scripts/production48.py status' >&2
printf '%s\n' 'Then follow docs/production-48.md for build, stage, activate and rollback.' >&2
exit 2

View File

@ -2,6 +2,9 @@
本文档用于生产或准生产环境部署。所有真实密钥只应配置在运行环境中,不要提交到 Git。
当前 `ai.yydd.ai` 的 48 / PM2 生产环境使用 [48 生产发布流程](./production-48.md)。
下方是通用安装说明,不能直接在当前生产目录执行迁移或构建。
## 运行环境
- Node.js 20

81
docs/production-48.md Normal file
View File

@ -0,0 +1,81 @@
# 48 生产发布
本流程由维护电脑发起,使用已有 SSH 密钥。生产只更新 `ai-chatbot`,不运行数据库迁移,
不更新 ZCode 原生运行时,不重启主机、Nginx 或其他 PM2 应用。
## 已核实的机器
| 用途 | SSH 入口 | 主机 |
| --- | --- | --- |
| 构建机 | `root@192.168.102.146:22` | `dao02-Default-string`,Ubuntu 24.04,Node 20 |
| 同一构建机的公网入口 | `root@45.192.99.48:23322` | FRP 转发,不是生产宿主机 |
| 生产机 | `root@45.192.99.48:64192` | `SfSTE0ttmsycO`,CentOS Stream 9,Node 20 |
| 维护连接跳板 | `root@192.168.58.20:22` | 已信任的 NAS,仅转发 SSH |
公网页面 `https://ai.yydd.ai/` 由生产 Nginx 转发至 `127.0.0.1:3125` 的 PM2 应用。
脚本固定验证生产、构建机的 SSH 指纹和主机名;私钥留在维护电脑,不转发 SSH Agent。
构建机原有 `/var/www/ai-chatbot` 是旧实例,本流程不会使用或覆盖它。
## 操作入口
维护电脑需要 Python 3.9+、Paramiko、Git,以及已信任的 SSH `known_hosts`。
默认使用 `~/.ssh/45.192.99.48.pem` 和跳板密钥 `~/.ssh/youyidian_prod_ed25519`。
可用 `--key`、`--jump-key`、`--known-hosts` 覆盖本机路径,不要把私钥提交到仓库。
```powershell
python scripts/production48.py status
python scripts/production48.py jobs
# 完成 PR 合并后,确认远端 main 的最新提交,工作区必须干净。
git fetch origin
git switch main
git merge --ff-only origin/main
python scripts/production48.py build
```
`build` 将确定的 `origin/main` 提交通过 `git archive` 送到构建机,
在 `/opt/ai-chatbot-builds/<时间-提交>/source` 安装锁定依赖并构建。
使用 pnpm 9.12.3,不改变全局 pnpm。安装时不运行依赖脚本,构建时禁用自动安装系统服务。
构建保持 TypeScript 检查,限制 Next 构建并行度和 Node 内存,并降低进程优先级。
构建机仅收到 `NEXT_PUBLIC_*` 等公开页面配置,以及指向本机关闭端口的占位数据库配置。
不会收到生产数据库密码、AI Key、认证密钥或存储凭据。随后在构建机回环地址启动隔离候选实例,
核对构建 ID 并执行公开路由烟测,测试结束自动关闭该候选实例。
成功后会输出发布 ID,例如 `20261030T120000Z-0123456789ab`。后续步骤使用这个完整 ID:
```powershell
python scripts/production48.py stage --release <发布ID>
python scripts/production48.py activate --release <发布ID>
python scripts/production48.py status
# 仅对当前生效的这一版执行回滚:
python scripts/production48.py rollback --release <发布ID>
```
`stage` 校验传输 SHA256,在 `/var/www/ai-chatbot-releases/<发布ID>` 解包。
它检查压缩包路径、磁盘容量、锁文件、实际依赖包清单和共享 public 文件校验和。
完成解包后至少保留 1 GiB 空间;不足时停止,不自动删除历史构建。
当前和旧浏览器标签页所需的静态资源通过硬链接保留,不改动旧构建。
`activate` 会重新核对生产版本和任务状态,拒绝在最近一小时仍有未结束生成任务时切换。
更早的历史挂起记录只写入发布回执,不会自动修改数据库中的状态。
它保存旧 PM2 配置和启动快照,然后仅更新 `ai-chatbot` 的工作目录并重启这个应用,
因此会有短暂切换窗口。健康检查同时核对进程目录、构建 ID、首页和路由。
检查失败自动恢复旧配置并再次验活;通过后执行 `pm2 save`,让重启后使用已验收版本。
## 共享运行资料与边界
初版发布复用原目录 `/var/www/ai-chatbot` 中的 `node_modules`、`public`、`.env.local`、
`tmp` 和模型配置备份,均通过链接使用,不复制、删除或覆盖。
生产依赖必须与构建依赖的校验清单相符,Node ABI 必须一致。
若本次修改了依赖锁文件或 public 中的文件,脚本会停止,须先单独审查并准备对应运行资料,
不能绕过校验强行切换。本流程适用于应用代码迭代;数据库、共享资源和原生 Agent 升级各自单独发布。
原生产 checkout 的支线及未提交改动仍保留,不能据其 Git HEAD 判断已发布版本。
实际发布证据是:运行 cwd、`.next/BUILD_ID`、该目录 `release.json` 和 `activation.json`。
维护电脑的 `.git/releases/<发布ID>/` 保留提交、构建日志、烟测、传输校验和及操作回执。
服务器回滚配置可能含环境元数据,目录权限为 0700、文件为 0600,不放入公开目录或 Git。
仓库历史全量 lint 仍有已记录的问题,不能声称全部质量门禁通过。
每次 PR 须记录本次改动的验证、历史失败和未覆盖项目;当前流程不会自动触发生产发布。

View File

@ -1,69 +1,6 @@
#!/bin/sh
set -eu
app_dir="/srv/ai-chatbot"
compose_dir="/srv/ai-chatbot-deploy"
candidate_dir="$(mktemp -d "${app_dir}/.next-candidate.XXXXXX")"
previous_dir="${app_dir}/.next-previous-$(date +%Y%m%dT%H%M%S)"
build_container="ai-chatbot-build-$(date +%s)"
if grep -Eq '^[[:space:]]*CANVAS_E2E_LOCAL[[:space:]]*=[[:space:]]*"?true"?[[:space:]]*$' "${app_dir}/.env.local"; then
echo "Refusing production deployment while CANVAS_E2E_LOCAL=true." >&2
exit 1
fi
cleanup_candidate() {
if [ -d "$candidate_dir" ]; then
rm -rf -- "$candidate_dir"
fi
}
trap cleanup_candidate EXIT INT TERM
docker run --rm \
--name "$build_container" \
--network host \
-e NODE_OPTIONS=--max-old-space-size=6144 \
-v "${app_dir}:/workspace" \
-v "${candidate_dir}:/workspace/.next" \
-w /workspace \
node:20.20.0-bookworm-slim \
sh -lc 'corepack enable && pnpm build'
test -s "${candidate_dir}/BUILD_ID"
docker stop ai-chatbot-app >/dev/null
docker rm ai-chatbot-app >/dev/null
if [ -d "${app_dir}/.next" ]; then
mv "${app_dir}/.next" "$previous_dir"
fi
mv "$candidate_dir" "${app_dir}/.next"
candidate_dir=""
docker compose -f "${compose_dir}/docker-compose.yml" up -d app
health_ok=false
attempt=1
while [ "$attempt" -le 15 ]; do
if curl -fsS --max-time 10 http://127.0.0.1:3125/login >/dev/null; then
health_ok=true
break
fi
sleep 2
attempt=$((attempt + 1))
done
if [ "$health_ok" != true ]; then
docker stop ai-chatbot-app >/dev/null 2>&1 || true
docker rm ai-chatbot-app >/dev/null 2>&1 || true
failed_dir="${app_dir}/.next-failed-$(date +%Y%m%dT%H%M%S)"
mv "${app_dir}/.next" "$failed_dir"
if [ -d "$previous_dir" ]; then
mv "$previous_dir" "${app_dir}/.next"
docker compose -f "${compose_dir}/docker-compose.yml" up -d app
fi
echo "Deployment health check failed; previous build restored." >&2
exit 1
fi
echo "Production build activated. Previous build: ${previous_dir}"
#!/usr/bin/env bash
set -euo pipefail
printf '%s\n' 'Legacy deployment entry point disabled: it targets an obsolete host/runtime.' >&2
printf '%s\n' 'Use: python scripts/production48.py status' >&2
printf '%s\n' 'Then follow docs/production-48.md for build, stage, activate and rollback.' >&2
exit 2

247
scripts/production48.py Normal file
View File

@ -0,0 +1,247 @@
#!/usr/bin/env python3
"""Build on .146 and release to native production :64192, with local SSH keys.
Requires Python 3.9+ and Paramiko on the operator computer only. Start with
`python scripts/production48.py status`; see docs/production-48.md.
"""
import argparse
import contextlib
import datetime
import hashlib
import json
import os
from pathlib import Path
import re
import shlex
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parent.parent
BUILD_ROOT = "/opt/ai-chatbot-builds"
RELEASE_ROOT = "/var/www/ai-chatbot-releases"
PROD_FINGERPRINT = "SHA256:3tFAg02kzAfGNBDba+je+vgSB7TifvJPje9XIyVBgTM"
BUILD_FINGERPRINT = "SHA256:MXW5w48SFSTUaqS+dS1G9hY228hI6SLepjhCmItHL04"
def require(value, message):
if not value:
raise RuntimeError(message)
def git(*args):
return subprocess.check_output(["git", *args], cwd=ROOT, text=True).strip()
def sha(path):
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def write_json(path, value):
path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n", encoding="utf8")
class Connections:
def __init__(self, args):
import paramiko
self.paramiko = paramiko
self.args = args
self.stack = contextlib.ExitStack()
self.clients = {}
def __enter__(self):
return self
def __exit__(self, *args):
self.stack.close()
def client(self, kind):
import base64
if kind in self.clients:
return self.clients[kind]
client = self.paramiko.SSHClient()
self.stack.callback(client.close)
client.load_host_keys(str(self.args.known_hosts))
client.set_missing_host_key_policy(self.paramiko.RejectPolicy())
options = dict(username="root", allow_agent=False, look_for_keys=False,
timeout=20, banner_timeout=20, auth_timeout=20)
if kind == "jump":
host, port, key = "192.168.58.20", 22, self.args.jump_key
elif kind == "build":
host, port, key = "192.168.102.146", 22, self.args.key
# The LAN entrance has the same verified key as the existing FRP entry.
known = client.get_host_keys().lookup("[45.192.99.48]:23322")
require(known and "ssh-ed25519" in known, "Missing trusted builder host key")
client.get_host_keys().add(host, "ssh-ed25519", known["ssh-ed25519"])
else:
host, port, key = "45.192.99.48", 64192, self.args.key
pin = self.paramiko.Ed25519Key(data=base64.b64decode(
"AAAAC3NzaC1lZDI1NTE5AAAAIK7afV+YwLqvUFBlxDe/q2dUrpBePcLdG+mZ0JekHYPC"))
client.get_host_keys().add("[45.192.99.48]:64192", "ssh-ed25519", pin)
jump = self.client("jump")
options["sock"] = jump.get_transport().open_channel(
"direct-tcpip", (host, port), ("127.0.0.1", 0), timeout=20)
options.update(hostname=host, port=port, key_filename=str(key))
client.connect(**options)
actual = "SHA256:" + base64.b64encode(hashlib.sha256(
client.get_transport().get_remote_server_key().asbytes()).digest()).decode().rstrip("=")
expected = {"build": BUILD_FINGERPRINT, "prod": PROD_FINGERPRINT}.get(kind)
require(not expected or actual == expected, f"{kind} host fingerprint mismatch")
client.get_transport().set_keepalive(20)
self.clients[kind] = client
return client
def execute(self, kind, command, stdin=None, stream=False, timeout=1800):
channel = self.client(kind).get_transport().open_session(timeout=20)
channel.exec_command(command)
if stdin is not None:
channel.sendall(stdin.encode("utf8"))
channel.shutdown_write()
output, errors = bytearray(), bytearray()
deadline = time.monotonic() + timeout
try:
while True:
if channel.recv_ready():
chunk = channel.recv(65536)
output.extend(chunk)
if stream:
print(chunk.decode("utf8", "replace"), end="", flush=True)
if channel.recv_stderr_ready():
chunk = channel.recv_stderr(65536)
errors.extend(chunk)
if stream:
print(chunk.decode("utf8", "replace"), end="", file=sys.stderr, flush=True)
if channel.exit_status_ready() and not channel.recv_ready() and not channel.recv_stderr_ready():
break
require(time.monotonic() < deadline, f"{kind} operation timed out; inspect remote state")
time.sleep(0.03)
code = channel.recv_exit_status()
require(code == 0, f"{kind} command failed ({code}): {errors.decode('utf8', 'replace')[-1800:]}")
return output.decode("utf8")
finally:
channel.close()
def helper(self, action, **kwargs):
command = ["python3", "-", action]
for key, value in kwargs.items():
command += ["--" + key, str(value)]
source = (ROOT / "scripts/release/host.py").read_text(encoding="utf8")
return json.loads(self.execute("prod", shlex.join(command), stdin=source))
def sftp(self, kind):
return self.paramiko.SFTPClient.from_transport(
self.client(kind).get_transport(), window_size=16 * 1024**2, max_packet_size=65536)
def upload(self, kind, local, remote):
last = [0]
def progress(done, total):
if done - last[0] >= 32 * 1024**2:
print(f"[{kind}] transferred {done // 1024**2}/{total // 1024**2} MiB", flush=True)
last[0] = done
with self.sftp(kind) as transfer:
transfer.put(str(local), remote, callback=progress)
transfer.chmod(remote, 0o600)
def build(connections, args):
require(not git("status", "--porcelain"), "Commit requested changes before building")
commit = git("rev-parse", "origin/main^{commit}")
require(git("branch", "--show-current") == "main" and git("rev-parse", "HEAD") == commit,
"Fetch origin and switch to an up-to-date, clean main before building")
remote = connections.execute("build", "hostname").strip()
require(remote == "dao02-Default-string", "Unexpected builder")
require(connections.execute("build", "node -v").strip().startswith("v20."), "Builder requires Node 20")
release_id = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ-") + commit[:12]
local = args.receipts / release_id
local.mkdir(parents=True)
source = local / "source.tar.gz"
subprocess.run(["git", "archive", "--format=tar.gz", f"--output={source}", commit], cwd=ROOT, check=True)
metadata = {"sourceCommit": commit, "sourceTree": git("rev-parse", f"{commit}^{{tree}}"),
"sourceArchiveSha256": sha(source)}
write_json(local / "input.json", metadata)
environment = connections.helper("build-env")
environment.update({"AUTH_SECRET": os.urandom(32).hex(), "POSTGRES_URL": "postgres://build:build@127.0.0.1:9/build",
"AUTH_TRUST_HOST": "true", "NODE_ENV": "production"})
workspace = f"{BUILD_ROOT}/{release_id}"
connections.execute("build", f"install -d -m 700 -- {shlex.quote(workspace)}")
for name in ["source.tar.gz", "input.json"]:
connections.upload("build", local / name, f"{workspace}/{name}")
# Helpers are carried separately, with the reviewed operator checkout's hashes.
for name in ["host.py", "build.sh"]:
connections.upload("build", ROOT / "scripts/release" / name, f"{workspace}/{name}")
setup = f"""import sys,json,pathlib,hashlib,importlib.util,os
os.umask(0o077)
workspace=pathlib.Path({workspace!r})
assert hashlib.sha256((workspace/'source.tar.gz').read_bytes()).hexdigest()=={metadata['sourceArchiveSha256']!r}
spec=importlib.util.spec_from_file_location('release_host',workspace/'host.py'); module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
module.safe_extract(workspace/'source.tar.gz',workspace/'source')
environment=json.load(sys.stdin)
with open(workspace/'source/.env.production.local','x') as stream:
for key,value in environment.items():stream.write(key+'='+json.dumps(value)+'\\n')
"""
# stdin belongs to the program, so the program itself travels as a quoted -c argument.
connections.execute("build", shlex.join(["python3", "-c", setup]), stdin=json.dumps(environment))
print(f"[build] {release_id} from main {commit}", flush=True)
log = connections.execute("build", shlex.join(["nice", "-n", "10", "bash", f"{workspace}/build.sh", workspace]),
stream=True, timeout=2400)
(local / "build.log").write_text(log, encoding="utf8")
smoke = json.loads(connections.execute("build", shlex.join(
["python3", f"{workspace}/host.py", "smoke", "--workspace", workspace]), timeout=240))
write_json(local / "smoke.json", smoke)
print(json.dumps(smoke), flush=True)
result = json.loads(connections.execute("build", shlex.join(
["python3", f"{workspace}/host.py", "pack", "--workspace", workspace]), timeout=600))
result.update({"builder": remote, "mainRef": commit,
"helperHashes": {name: sha(ROOT / "scripts/release" / name) for name in ["host.py", "build.sh"]}})
write_json(local / "build.json", result)
print(json.dumps(result), flush=True)
def stage(connections, args):
local = args.receipts / args.release
receipt = json.loads((local / "build.json").read_text(encoding="utf8"))
artifact = local / "artifact.tar.gz"
if not artifact.exists():
with connections.sftp("build") as transfer:
transfer.get(f"{BUILD_ROOT}/{args.release}/artifact.tar.gz", str(artifact))
require(sha(artifact) == receipt["artifactSha256"], "Downloaded artifact checksum mismatch")
connections.helper("status")
connections.execute("prod", f"install -d -m 700 -- {RELEASE_ROOT}")
connections.upload("prod", artifact, f"{RELEASE_ROOT}/{args.release}.tar.gz")
result = connections.helper("stage", release=args.release, digest=receipt["artifactSha256"])
write_json(local / "stage.json", result)
print(json.dumps(result))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("action", choices=["status", "jobs", "build", "stage", "activate", "rollback"])
parser.add_argument("--release")
parser.add_argument("--key", type=Path, default=Path.home() / ".ssh/45.192.99.48.pem")
parser.add_argument("--jump-key", type=Path, default=Path.home() / ".ssh/youyidian_prod_ed25519")
parser.add_argument("--known-hosts", type=Path, default=Path.home() / ".ssh/known_hosts")
parser.add_argument("--receipts", type=Path, default=ROOT / ".git/releases")
args = parser.parse_args()
if args.action in {"stage", "activate", "rollback"}:
require(args.release and re.fullmatch(r"\d{8}T\d{6}Z-[0-9a-f]{12}", args.release), "--release is required")
args.receipts = args.receipts.resolve()
with Connections(args) as connections:
if args.action == "build":
build(connections, args)
elif args.action == "stage":
stage(connections, args)
else:
kwargs = {"release": args.release} if args.release else {}
result = connections.helper(args.action, **kwargs)
if args.release:
write_json(args.receipts / args.release / (args.action + ".json"), result)
print(json.dumps(result))
if __name__ == "__main__":
main()

18
scripts/release/build.sh Normal file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env bash
set -euo pipefail
umask 077
# Runs only inside the unique workspace created by production48.py.
cd -- "${1:?workspace required}/source"
export DISABLE_PERF_GUARD_AUTO_INSTALL=1
export CI=1 NEXT_TELEMETRY_DISABLED=1 NEXT_IGNORE_BUILD_TYPECHECK=0
export NEXT_DIST_DIR=.next CIRCLE_NODE_TOTAL=4
export NODE_OPTIONS=--max-old-space-size=8192
export npm_config_cache="$(dirname "$PWD")/npm-cache"
export npm_config_registry="${BUILD_NPM_REGISTRY:-https://registry.npmjs.org}"
# Do not change the machine's global pnpm or install any system service.
pnpm9() { npm exec --yes --package=pnpm@9.12.3 -- pnpm "$@"; }
test "$(pnpm9 --version)" = 9.12.3
pnpm9 install --frozen-lockfile --ignore-scripts
pnpm9 build

426
scripts/release/host.py Normal file
View File

@ -0,0 +1,426 @@
"""Linux release operations. No SSH dependency; invoked by production48.py.
Only the ai-chatbot PM2 application is changed. The original checkout, public
files, credentials, dependencies and historical builds remain in place.
"""
import argparse
import datetime
import hashlib
import json
import os
from pathlib import Path, PurePosixPath
import re
import shutil
import socket
import subprocess
import tarfile
import time
import urllib.request
APP = Path("/var/www/ai-chatbot")
RELEASES = Path("/var/www/ai-chatbot-releases")
NAME = "ai-chatbot"
RESERVE = 1024**3
def require(value, message):
if not value:
raise RuntimeError(message)
def run(args, cwd=None, stdin=None, timeout=60):
result = subprocess.run(args, cwd=cwd, input=stdin, text=True,
capture_output=True, timeout=timeout)
require(result.returncode == 0, f"{args[0]} failed (exit {result.returncode})")
return result.stdout
def sha(path):
digest = hashlib.sha256()
with open(path, "rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def json_write(path, value):
temporary = path.with_name(path.name + ".tmp")
with open(temporary, "x", encoding="utf8") as stream:
os.chmod(temporary, 0o600)
json.dump(value, stream, indent=2)
stream.write("\n")
os.replace(temporary, path)
def safe_extract(archive, destination):
"""Reject traversal, special files and links escaping a new directory."""
require(not destination.exists(), "Extraction destination already exists")
with tarfile.open(archive, "r:gz") as bundle:
members = bundle.getmembers()
for member in members:
path = PurePosixPath(member.name)
require(not path.is_absolute() and ".." not in path.parts,
"Unsafe archive member")
require(member.isdir() or member.isfile() or member.issym(),
"Special files and hard links are not accepted")
if member.issym():
link = PurePosixPath(member.linkname)
require(not link.is_absolute(), "Absolute archive symlink")
target = (destination / path.parent / member.linkname).resolve()
require(target.is_relative_to(destination.resolve()),
"Archive symlink escapes destination")
needed = sum(m.size for m in members if m.isfile())
require(shutil.disk_usage(destination.parent).free > needed + RESERVE,
"Insufficient space: at least 1 GiB must remain after extraction")
destination.mkdir(mode=0o700)
# Extract normal entries first; never follow an archive-created link.
for member in members:
if member.issym():
continue
path = destination / member.name
path.parent.mkdir(parents=True, exist_ok=True)
if member.isdir():
path.mkdir(exist_ok=True)
else:
with bundle.extractfile(member) as source, open(path, "xb") as target:
shutil.copyfileobj(source, target)
path.chmod(member.mode & 0o777)
for member in members:
if member.issym():
path = destination / member.name
path.parent.mkdir(parents=True, exist_ok=True)
path.symlink_to(member.linkname)
def release_path(release_id):
require(re.fullmatch(r"\d{8}T\d{6}Z-[0-9a-f]{12}", release_id),
"Invalid release ID")
return RELEASES / release_id
def app_state():
require(socket.gethostname() == "SfSTE0ttmsycO", "Not the verified production host")
apps = json.loads(run(["pm2", "jlist"]))
matching = [app for app in apps if app["name"] == NAME]
require(len(matching) == 1, "Expected exactly one ai-chatbot PM2 application")
app = matching[0]
env = app["pm2_env"]
cwd = Path(env["pm_cwd"])
require(cwd == APP or cwd.parent == RELEASES, "Unexpected production cwd")
require(env["pm_exec_path"] == str(APP / "node_modules/next/dist/bin/next"),
"Unexpected Next.js executable")
require(env.get("watch") is False, "PM2 watch must be disabled")
require(env.get("CANVAS_E2E_LOCAL") != "true", "Production E2E mode is enabled")
require(env.get("args") == ["start", "-p", "3125", "-H", "127.0.0.1"],
"Unexpected production listen arguments")
dist = env.get("NEXT_DIST_DIR", ".next")
require(re.fullmatch(r"\.next[\w.-]*", dist), "Unsafe active build directory")
return app, cwd, dist
def status():
app, cwd, dist = app_state()
return {"hostname": socket.gethostname(), "pid": app["pid"],
"status": app["pm2_env"]["status"], "cwd": str(cwd), "distDir": dist,
"buildId": (cwd / dist / "BUILD_ID").read_text().strip(),
"freeGiB": round(shutil.disk_usage(APP).free / 1024**3, 2),
"node": run(["node", "-v"]).strip()}
ENV_LOADER = """
const {createRequire}=require('node:module');
const nextRequire=createRequire(require.resolve('next/package.json'));
nextRequire('@next/env').loadEnvConfig(process.cwd(),false,{info(){},error(){}});
"""
def build_environment():
# Only public build-time configuration leaves production.
source = ENV_LOADER + """
const extra=['AUTH_URL','NEXTAUTH_URL','PASCAL_EDITOR_URL','CANVAS_DOWNLOAD_CDN_ORIGIN'];
console.log(JSON.stringify(Object.fromEntries(Object.entries(process.env)
.filter(([key])=>key.startsWith('NEXT_PUBLIC_')||extra.includes(key)))));
"""
app_state()
return json.loads(run(["node", "-"], cwd=APP, stdin=source))
def job_status():
source = ENV_LOADER + r"""
const sql=require('postgres')(process.env.POSTGRES_URL,{max:1,connect_timeout:5,onnotice(){}});
(async()=>{try{
const tables=['CanvasImageTask','CanvasCapabilityTask','CanvasAgentRun',
'EnhanceProTask','AgentPsdRenderTask','VideoStoryboardTask'];const result={};
await sql.begin('read only',async tx=>{
await tx.unsafe("set local statement_timeout='5s'");
for(const table of tables){
const exists=await tx`select to_regclass(${JSON.stringify(table)}) as name`;
if(!exists[0].name)throw new Error('Missing task table');
result[table]=await tx.unsafe('select status,count(*)::int as count,'+
'count(*) filter (where "updatedAt">now()-interval \'1 hour\')::int as recent '+
'from "'+table+'" where status not in '+
'(\'succeeded\',\'failed\',\'completed\',\'cancelled\',\'interrupted\',\'needs_review\') group by status');
}
});console.log(JSON.stringify(result));
}catch(e){console.error('Task preflight failed');process.exitCode=1;}
finally{await sql.end();}})();
"""
return json.loads(run(["node", "-"], cwd=APP, stdin=source, timeout=50))
def inventory_packages(root):
result = {}
for path in sorted((root / "node_modules/.pnpm").glob("*/node_modules/*/package.json")):
result[path.relative_to(root / "node_modules").as_posix()] = sha(path)
for path in sorted((root / "node_modules/.pnpm").glob("*/node_modules/@*/*/package.json")):
result[path.relative_to(root / "node_modules").as_posix()] = sha(path)
return result
def build_manifest(workspace):
source = workspace / "source"
metadata = json.loads((workspace / "input.json").read_text())
result = {"schema": 1, "sourceCommit": metadata["sourceCommit"],
"sourceTree": metadata["sourceTree"], "releaseId": workspace.name,
"nodeMajor": 20, "nodeAbi": "115", "pnpm": "9.12.3",
"buildId": (source / ".next/BUILD_ID").read_text().strip(),
"lockSha256": sha(source / "pnpm-lock.yaml"),
"publicFiles": {p.relative_to(source).as_posix(): sha(p)
for p in sorted((source / "public").rglob("*")) if p.is_file()},
"directPackages": {name: sha(source / "node_modules" / name / "package.json")
for name in json.loads((source / "package.json").read_text())["dependencies"]},
"installedPackages": inventory_packages(source)}
require(result["installedPackages"], "No dependency inventory")
json_write(source / "release.json", result)
with tarfile.open(workspace / "artifact.tar.gz", "w:gz", compresslevel=3) as archive:
for path in sorted(source.iterdir()):
if path.name in {"node_modules", "public", ".env.production.local", ".next"}:
continue
archive.add(path, arcname=path.name)
def filter_build(member):
if member.name == ".next/cache" or member.name.startswith(".next/cache/"):
return None
return member
archive.add(source / ".next", arcname=".next", filter=filter_build)
return {"releaseId": workspace.name, "sourceCommit": result["sourceCommit"],
"buildId": result["buildId"], "artifactSha256": sha(workspace / "artifact.tar.gz"),
"artifactBytes": (workspace / "artifact.tar.gz").stat().st_size}
def smoke(workspace):
source = workspace / "source"
expected = (source / ".next/BUILD_ID").read_text().strip()
with socket.socket() as listener:
listener.bind(("127.0.0.1", 0))
port = listener.getsockname()[1]
environment = os.environ.copy()
environment.update({"NODE_ENV": "production", "NEXT_DIST_DIR": ".next",
"PORT": str(port), "POSTGRES_CONNECT_TIMEOUT_SECONDS": "1"})
with open(workspace / "smoke.log", "w") as log:
process = subprocess.Popen(["node", "node_modules/next/dist/bin/next", "start",
"-H", "127.0.0.1", "-p", str(port)],
cwd=source, env=environment, stdout=log, stderr=log)
try:
ready = False
for _ in range(45):
require(process.poll() is None, "Isolated candidate exited; inspect smoke.log")
try:
with urllib.request.urlopen(f"http://127.0.0.1:{port}/", timeout=3) as response:
ready = response.status == 200 and expected in response.read().decode("utf8")
if ready:
break
except Exception:
pass
time.sleep(1)
require(ready, "Isolated candidate readiness failed; inspect smoke.log")
output = run(["env", f"SMOKE_BASE_URL=http://127.0.0.1:{port}", "node",
"scripts/smoke-routes.mjs"], cwd=source, timeout=100)
return {"candidateSmoke": "passed", "buildId": expected,
"checks": output.strip().splitlines()}
finally:
process.terminate()
try:
process.wait(timeout=20)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
def verify_shared(manifest):
require(sha(APP / "pnpm-lock.yaml") == manifest["lockSha256"],
"Dependencies changed: provision a separate reviewed runtime dependency set first")
for name, digest in manifest["publicFiles"].items():
path = PurePosixPath(name)
require(path.parts[0] == "public" and ".." not in path.parts,
"Invalid public path")
require((APP / name).is_file() and sha(APP / name) == digest,
f"Public files differ: {name}; review shared asset publication first")
actual = inventory_packages(APP)
missing = [name for name, value in manifest["installedPackages"].items()
if actual.get(name) != value]
require(not missing, f"Installed dependency mismatch ({len(missing)} packages)")
for name, digest in manifest["directPackages"].items():
require(sha(APP / "node_modules" / name / "package.json") == digest,
f"Direct dependency link mismatch: {name}")
require(run(["node", "-p", "process.versions.modules"]).strip() == manifest["nodeAbi"],
"Node ABI mismatch")
def stage(release_id, digest):
_, current, dist = app_state()
archive = RELEASES / (release_id + ".tar.gz")
require(sha(archive) == digest, "Artifact checksum mismatch")
destination = release_path(release_id)
safe_extract(archive, destination)
manifest = json.loads((destination / "release.json").read_text())
require(manifest["releaseId"] == release_id, "Artifact release ID mismatch")
require(manifest["sourceCommit"][:12] == release_id[-12:], "Artifact source mismatch")
verify_shared(manifest)
for name in ["node_modules", "public", ".env.local", "tmp", ".model-config-backups"]:
require(not (destination / name).exists(), f"Unexpected bundled shared path: {name}")
if (APP / name).exists():
(destination / name).symlink_to(APP / name)
# Keep already open browser tabs' content-hashed assets available.
for path in (current / dist / "static").rglob("*"):
if not path.is_file():
continue
target = destination / ".next/static" / path.relative_to(current / dist / "static")
if target.exists():
require(sha(target) == sha(path), "Static asset name collision")
else:
target.parent.mkdir(parents=True, exist_ok=True)
os.link(path, target)
json_write(destination / "staged.json", {"artifactSha256": digest,
"previousCwd": str(current), "previousDistDir": dist})
# The verified unpacked release is retained; the transport copy is redundant.
archive.unlink()
return {"staged": release_id, "buildId": manifest["buildId"],
"freeGiB": round(shutil.disk_usage(APP).free / 1024**3, 2)}
def ecosystem(app, cwd, dist):
old = app["pm2_env"]
env = {k: v for k, v in old.items() if k.isupper() and isinstance(v, (str, int, bool))}
env.update({"NODE_ENV": "production", "PORT": "3125", "PWD": str(cwd), "NEXT_DIST_DIR": dist})
return {"apps": [{"name": NAME, "cwd": str(cwd), "script": old["pm_exec_path"],
"args": old["args"], "interpreter": old.get("exec_interpreter", "/usr/bin/node"),
"exec_mode": "fork", "instances": 1, "watch": False, "env": env,
"autorestart": old.get("autorestart", True),
"min_uptime": old.get("min_uptime", "30s"), "max_restarts": old.get("max_restarts", 10),
"restart_delay": old.get("restart_delay", 3000), "kill_timeout": old.get("kill_timeout", 10000),
"max_memory_restart": old.get("max_memory_restart", "3G"),
"error_file": old["pm_err_log_path"], "out_file": old["pm_out_log_path"],
"merge_logs": True, "log_date_format": "YYYY-MM-DD HH:mm:ss Z"}]}
def healthy(cwd, build_id):
for _ in range(30):
try:
app, active_cwd, _ = app_state()
require(active_cwd == cwd and app["pm2_env"]["status"] == "online", "Wrong process")
request = urllib.request.Request("http://127.0.0.1:3125/", headers={"Host": "ai.yydd.ai"})
with urllib.request.urlopen(request, timeout=5) as response:
body = response.read().decode("utf8")
if response.status == 200 and build_id in body:
return True
except Exception:
pass
time.sleep(2)
return False
def restart(config):
run(["pm2", "startOrRestart", str(config), "--only", NAME, "--update-env"], timeout=150)
def activate(release_id):
destination = release_path(release_id)
manifest = json.loads((destination / "release.json").read_text())
staged = json.loads((destination / "staged.json").read_text())
app, current, dist = app_state()
require(str(current) == staged["previousCwd"] and dist == staged["previousDistDir"],
"Production changed after staging")
require(not (destination / "activation.json").exists(), "Already activated; inspect status")
verify_shared(manifest)
jobs = job_status()
require(not any(row["recent"] for rows in jobs.values() for row in rows),
"Recent active generation jobs: defer activation")
before = ecosystem(app, current, dist)
previous_build = (current / dist / "BUILD_ID").read_text().strip()
json_write(destination / "previous.pm2.json", before)
json_write(destination / "candidate.pm2.json", ecosystem(app, destination, ".next"))
# Backup contains environment metadata and stays root-readable only.
shutil.copy2(Path.home() / ".pm2/dump.pm2", destination / "pm2-dump.before.json")
(destination / "pm2-dump.before.json").chmod(0o600)
receipt = {"sourceCommit": manifest["sourceCommit"], "buildId": manifest["buildId"],
"previousBuildId": previous_build, "previousCwd": str(current),
"activatedAtUtc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"historicalTaskStates": jobs, "artifactSha256": staged["artifactSha256"],
"phase": "activating"}
json_write(destination / "activation.json", receipt)
try:
restart(destination / "candidate.pm2.json")
require(healthy(destination, manifest["buildId"]), "Candidate health check failed")
run(["env", "SMOKE_BASE_URL=http://127.0.0.1:3125", "node", str(destination / "scripts/smoke-routes.mjs")], cwd=destination,
timeout=100)
run(["pm2", "save"])
except BaseException:
restart(destination / "previous.pm2.json")
require(healthy(current, previous_build), "Rollback health check failed; operator required")
run(["pm2", "save"])
receipt["phase"] = "rolled-back"
json_write(destination / "activation.json", receipt)
raise
receipt["phase"] = "active"
json_write(destination / "activation.json", receipt)
return receipt
def rollback(release_id):
destination = release_path(release_id)
receipt = json.loads((destination / "activation.json").read_text())
_, current, _ = app_state()
require(current == destination, "Rollback target is not currently active")
jobs = job_status()
require(not any(row["recent"] for rows in jobs.values() for row in rows),
"Recent active generation jobs: defer rollback")
restart(destination / "previous.pm2.json")
require(healthy(Path(receipt["previousCwd"]), receipt["previousBuildId"]),
"Rollback health check failed")
run(["pm2", "save"])
receipt["phase"] = "rolled-back"
json_write(destination / "activation.json", receipt)
return status()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("action", choices=["status", "build-env", "jobs", "smoke", "pack", "stage", "activate", "rollback"])
parser.add_argument("--release")
parser.add_argument("--digest")
parser.add_argument("--workspace", type=Path)
args = parser.parse_args()
os.umask(0o077)
if args.action in {"pack", "smoke"}:
require(socket.gethostname() == "dao02-Default-string", "Not the verified build host")
require(args.workspace.parent == Path("/opt/ai-chatbot-builds"), "Invalid build workspace")
result = build_manifest(args.workspace) if args.action == "pack" else smoke(args.workspace)
elif args.action in {"status", "build-env", "jobs"}:
result = {"status": status, "build-env": build_environment, "jobs": job_status}[args.action]()
else:
import fcntl
require(RELEASES.is_dir() and not RELEASES.is_symlink(), "Invalid release root")
with open(RELEASES / ".deploy.lock", "a") as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if args.action == "stage":
result = stage(args.release, args.digest)
elif args.action == "activate":
result = activate(args.release)
else:
result = rollback(args.release)
print(json.dumps(result))
if __name__ == "__main__":
main()

View File

@ -42,14 +42,13 @@ test("production deploy stages the main build and retains active static chunks",
expect(deployScript).toContain('curl -fsS "$MAIN_INTERNAL_URL/ping"');
});
test("atomic production deploy rejects the local E2E fallback switch", () => {
test("legacy atomic deployment cannot target the obsolete Docker runtime", () => {
const deployScript = readFileSync(
resolve(process.cwd(), "scripts/deploy-production-atomic.sh"),
"utf8"
);
expect(deployScript).toContain("CANVAS_E2E_LOCAL");
expect(deployScript).toContain(
"Refusing production deployment while CANVAS_E2E_LOCAL=true."
);
expect(deployScript).toContain("exit 2");
expect(deployScript).toContain("scripts/production48.py status");
expect(deployScript).not.toContain("docker compose");
});

View File

@ -0,0 +1,190 @@
"""Release boundary and rollback regression tests; no SSH or real PM2 changes."""
import importlib.util
import io
import json
from pathlib import Path
import shutil
import subprocess
import tarfile
import tempfile
import unittest
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[2]
spec = importlib.util.spec_from_file_location("release_host", ROOT / "scripts/release/host.py")
host = importlib.util.module_from_spec(spec)
spec.loader.exec_module(host)
class ArchiveTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.root = Path(self.temporary.name).resolve()
self.archive = self.root / "artifact.tar.gz"
self.destination = self.root / "release"
def tearDown(self):
self.temporary.cleanup()
def archive_with(self, name, link=None, kind=None):
with tarfile.open(self.archive, "w:gz") as bundle:
member = tarfile.TarInfo(name)
if link is not None:
member.type, member.linkname = tarfile.SYMTYPE, link
elif kind is not None:
member.type = kind
else:
member.size = 4
bundle.addfile(member, None if link is not None or kind is not None else io.BytesIO(b"test"))
def test_extracts_a_regular_file(self):
self.archive_with("app/version.txt")
host.safe_extract(self.archive, self.destination)
self.assertEqual((self.destination / "app/version.txt").read_bytes(), b"test")
def test_refuses_parent_traversal(self):
self.archive_with("../outside")
with self.assertRaisesRegex(RuntimeError, "Unsafe archive"):
host.safe_extract(self.archive, self.destination)
self.assertFalse((self.root / "outside").exists())
def test_refuses_absolute_paths(self):
self.archive_with("/outside")
with self.assertRaisesRegex(RuntimeError, "Unsafe archive"):
host.safe_extract(self.archive, self.destination)
def test_refuses_symlink_escape(self):
self.archive_with("app/link", "../../outside")
with self.assertRaisesRegex(RuntimeError, "escapes"):
host.safe_extract(self.archive, self.destination)
def test_refuses_absolute_symlink(self):
self.archive_with("app/link", "/etc/passwd")
with self.assertRaisesRegex(RuntimeError, "Absolute archive"):
host.safe_extract(self.archive, self.destination)
def test_refuses_special_files(self):
self.archive_with("device", kind=tarfile.FIFOTYPE)
with self.assertRaisesRegex(RuntimeError, "Special files"):
host.safe_extract(self.archive, self.destination)
def test_refuses_existing_release(self):
self.destination.mkdir()
self.archive_with("file")
with self.assertRaisesRegex(RuntimeError, "already exists"):
host.safe_extract(self.archive, self.destination)
def test_keeps_disk_reserve_without_deleting_anything(self):
self.archive_with("file")
with patch.object(host.shutil, "disk_usage", return_value=shutil._ntuple_diskusage(10, 9, 1)):
with self.assertRaisesRegex(RuntimeError, "Insufficient space"):
host.safe_extract(self.archive, self.destination)
self.assertFalse(self.destination.exists())
class ActivationTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.root = Path(self.temporary.name).resolve()
self.releases = self.root / "releases"
self.release_id = "20261030T120000Z-0123456789ab"
self.target = self.releases / self.release_id
self.target.mkdir(parents=True)
self.previous = self.root / "old"
(self.previous / ".next-old").mkdir(parents=True)
(self.previous / ".next-old/BUILD_ID").write_text("old-build")
(self.root / ".pm2").mkdir()
(self.root / ".pm2/dump.pm2").write_text("[]")
host.json_write(self.target / "release.json", {"sourceCommit": "0123456789ab" + "0" * 28,
"buildId": "new-build"})
host.json_write(self.target / "staged.json", {"previousCwd": str(self.previous),
"previousDistDir": ".next-old", "artifactSha256": "artifact-hash"})
self.app = {"pid": 100, "pm2_env": {"pm_exec_path": "/old/node_modules/next/dist/bin/next",
"args": ["start", "-p", "3125", "-H", "127.0.0.1"],
"CANVAS_ZCODE_STATE_DIR": "/private/state", "NEXT_DIST_DIR": ".next-old",
"pm_err_log_path": "/logs/error", "pm_out_log_path": "/logs/out"}}
self.patches = [patch.object(host, "RELEASES", self.releases),
patch.object(host, "verify_shared"),
patch.object(host, "app_state", return_value=(self.app, self.previous, ".next-old")),
patch.object(host.Path, "home", return_value=self.root),
patch.object(host, "run", return_value=""),
patch.object(host, "restart"),
patch.object(host, "job_status", return_value={"CanvasAgentRun": []})]
self.mocks = [p.start() for p in self.patches]
self.restart = self.mocks[-2]
def tearDown(self):
for p in reversed(self.patches):
p.stop()
self.temporary.cleanup()
def test_active_job_blocks_before_restart(self):
self.mocks[-1].return_value = {"CanvasAgentRun": [{"count": 1, "recent": 1}]}
with self.assertRaisesRegex(RuntimeError, "Recent active"):
host.activate(self.release_id)
self.restart.assert_not_called()
self.assertFalse((self.target / "activation.json").exists())
def test_new_release_health_failure_restores_previous_build(self):
with patch.object(host, "healthy", side_effect=[False, True]):
with self.assertRaisesRegex(RuntimeError, "Candidate health"):
host.activate(self.release_id)
self.assertEqual([call.args[0].name for call in self.restart.call_args_list],
["candidate.pm2.json", "previous.pm2.json"])
self.assertEqual(json.loads((self.target / "activation.json").read_text())["phase"], "rolled-back")
def test_success_records_provenance_and_keeps_native_runtime_state(self):
with patch.object(host, "healthy", return_value=True):
result = host.activate(self.release_id)
self.assertEqual(result["phase"], "active")
config = json.loads((self.target / "candidate.pm2.json").read_text())["apps"][0]
self.assertEqual(config["name"], "ai-chatbot")
self.assertEqual(config["cwd"], str(self.target))
self.assertEqual(config["env"]["CANVAS_ZCODE_STATE_DIR"], "/private/state")
self.assertNotIn("/private/state", json.dumps(result))
def test_changed_production_blocks_activation(self):
self.mocks[2].return_value = (self.app, self.root / "other", ".next")
with self.assertRaisesRegex(RuntimeError, "changed after staging"):
host.activate(self.release_id)
self.restart.assert_not_called()
def test_never_retries_an_activation_without_inspection(self):
host.json_write(self.target / "activation.json", {"phase": "activating"})
with self.assertRaisesRegex(RuntimeError, "Already activated"):
host.activate(self.release_id)
self.restart.assert_not_called()
def test_rollback_failure_is_reported(self):
with patch.object(host, "healthy", return_value=False):
with self.assertRaisesRegex(RuntimeError, "Rollback health check failed"):
host.activate(self.release_id)
class HelperTests(unittest.TestCase):
def test_production_e2e_switch_blocks_release(self):
app = {"name": "ai-chatbot", "pm2_env": {
"pm_cwd": "/var/www/ai-chatbot",
"pm_exec_path": str(host.APP / "node_modules/next/dist/bin/next"),
"watch": False, "CANVAS_E2E_LOCAL": "true"}}
with patch.object(host.socket, "gethostname", return_value="SfSTE0ttmsycO"), \
patch.object(host, "run", return_value=json.dumps([app])):
with self.assertRaisesRegex(RuntimeError, "Production E2E mode"):
host.app_state()
def test_release_ids_cannot_escape_root(self):
for value in ["../app", "/etc", "20261030T120000Z-0123456789ab/../../outside"]:
with self.assertRaisesRegex(RuntimeError, "Invalid release"):
host.release_path(value)
def test_task_query_javascript_is_valid(self):
sources = []
with patch.object(host, "run", side_effect=lambda *a, **k: sources.append(k["stdin"]) or "{}"):
host.job_status()
result = subprocess.run(["node", "--check"], input=sources[0], text=True, capture_output=True)
self.assertEqual(result.returncode, 0, result.stderr)
if __name__ == "__main__":
unittest.main()