From 7439792d400686ef7715bf315b049e602739ad0a Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 30 Oct 2026 18:22:11 +0800 Subject: [PATCH] build: add verified PM2 release workflow for production 48 --- .gitattributes | 1 + .gitignore | 4 + auto-deploy.sh | 309 +------------ deploy.sh | 114 +---- docs/deployment.md | 3 + docs/production-48.md | 81 ++++ scripts/deploy-production-atomic.sh | 75 +-- scripts/production48.py | 247 ++++++++++ scripts/release/build.sh | 18 + scripts/release/host.py | 426 ++++++++++++++++++ .../routes/production-local-fallbacks.test.ts | 9 +- tests/scripts/test_production48.py | 190 ++++++++ 12 files changed, 992 insertions(+), 485 deletions(-) create mode 100644 docs/production-48.md create mode 100644 scripts/production48.py create mode 100644 scripts/release/build.sh create mode 100644 scripts/release/host.py create mode 100644 tests/scripts/test_production48.py diff --git a/.gitattributes b/.gitattributes index e4f7b7c34..12a8e4e3a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -15,5 +15,6 @@ *.html text eol=lf *.md text eol=lf *.sh text eol=lf +*.py text eol=lf *.bat text eol=crlf *.cmd text eol=crlf diff --git a/.gitignore b/.gitignore index 9523ce24d..ed88848ba 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,7 @@ services/video-upscale/vendor/__pycache__/ /services/audio-processing/dist/ /services/audio-processing/__pycache__/ + +# Python release-tool and test bytecode. +__pycache__/ +*.pyc diff --git a/auto-deploy.sh b/auto-deploy.sh index 816eeebd7..28bd5bfc5 100755 --- a/auto-deploy.sh +++ b/auto-deploy.sh @@ -1,303 +1,6 @@ -#!/bin/bash - -# 自动部署脚本 - 包含密码 -# 使用方法: ./auto-deploy.sh - -set -e - -# 服务器配置 -SERVER="156.225.27.52" -PORT="5204" -USER="root" -PASSWORD="${DEPLOY_SSH_PASSWORD:?set DEPLOY_SSH_PASSWORD}" -REMOTE_DIR="/var/www/ai-chatbot" - -# 颜色输出 -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -echo -e "${GREEN}开始自动部署到服务器...${NC}" - -# 检查是否安装了 sshpass -if ! command -v sshpass &> /dev/null; then - echo -e "${YELLOW}安装 sshpass...${NC}" - if [[ "$OSTYPE" == "darwin"* ]]; then - # macOS - if ! command -v brew &> /dev/null; then - echo -e "${RED}请先安装 Homebrew: /bin/bash -c \"\$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)\"${NC}" - exit 1 - fi - brew install hudochenkov/sshpass/sshpass - else - # Linux - sudo apt-get update && sudo apt-get install -y sshpass - fi -fi - -# SSH 连接函数 -ssh_cmd() { - sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no -p $PORT $USER@$SERVER "$1" -} - -scp_cmd() { - sshpass -p "$PASSWORD" scp -o StrictHostKeyChecking=no -P $PORT "$1" $USER@$SERVER:"$2" -} - -scp_dir() { - sshpass -p "$PASSWORD" scp -o StrictHostKeyChecking=no -P $PORT -r "$1" $USER@$SERVER:"$2" -} - -echo -e "${GREEN}步骤 1: 检查服务器连接...${NC}" -ssh_cmd "echo '连接成功' && uname -a" - -echo -e "${GREEN}步骤 2: 检查并安装必要软件...${NC}" -ssh_cmd " - # 检查 Node.js - if ! command -v node &> /dev/null; then - echo '安装 Node.js...' - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash - export NVM_DIR=\"\$HOME/.nvm\" - [ -s \"\$NVM_DIR/nvm.sh\" ] && . \"\$NVM_DIR/nvm.sh\" - nvm install 20 - nvm use 20 - nvm alias default 20 - fi - - # 检查 pnpm - if ! command -v pnpm &> /dev/null; then - echo '安装 pnpm...' - npm install -g pnpm - fi - - # 检查 PM2 - if ! command -v pm2 &> /dev/null; then - echo '安装 PM2...' - npm install -g pm2 - fi - - # 检查 PostgreSQL - if ! command -v psql &> /dev/null; then - echo '安装 PostgreSQL...' - apt-get update -y - apt-get install -y postgresql postgresql-contrib - systemctl start postgresql - systemctl enable postgresql - fi - - # 检查 Nginx - if ! command -v nginx &> /dev/null; then - echo '安装 Nginx...' - apt-get install -y nginx - systemctl start nginx - systemctl enable nginx - fi - - # 创建项目目录 - mkdir -p $REMOTE_DIR - mkdir -p /var/log/pm2 - mkdir -p /root/backups/ai-chatbot -" - -echo -e "${GREEN}步骤 3: 配置数据库...${NC}" -ssh_cmd " - # 检查并创建/更新数据库用户 - echo '配置数据库用户...' - sudo -u postgres psql << 'PSQL' --- 删除已存在的用户（如果存在） -DROP USER IF EXISTS chatbot_user; - --- 创建新用户 -CREATE USER chatbot_user WITH PASSWORD 'ChatBot@2024!Secure'; - --- 检查数据库是否存在 -SELECT 1 FROM pg_database WHERE datname='ai_chatbot'\\gexec - --- 如果不存在则创建 -DO \\$\\$ -BEGIN - IF NOT EXISTS (SELECT FROM pg_database WHERE datname = 'ai_chatbot') THEN - CREATE DATABASE ai_chatbot; - END IF; -END -\\$\\$; - --- 授予权限 -GRANT ALL PRIVILEGES ON DATABASE ai_chatbot TO chatbot_user; - --- 连接到数据库并授予schema权限 -\\c ai_chatbot -GRANT ALL ON SCHEMA public TO chatbot_user; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO chatbot_user; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO chatbot_user; - -\\q -PSQL - echo '数据库配置完成' -" - -echo -e "${GREEN}步骤 4: 上传项目文件...${NC}" -# 创建临时部署包 -echo "准备部署文件..." -tar --exclude='node_modules' \ - --exclude='.next' \ - --exclude='.git' \ - --exclude='.env.local' \ - --exclude='*.log' \ - --exclude='deploy.tar.gz' \ - -czf /tmp/deploy.tar.gz . - -# 上传部署包 -scp_cmd "/tmp/deploy.tar.gz" "/tmp/" - -# 在服务器上解压 -ssh_cmd " - cd $REMOTE_DIR - tar -xzf /tmp/deploy.tar.gz - rm /tmp/deploy.tar.gz - - if [ -f \"$REMOTE_DIR/scripts/security/install-security-stack.sh\" ]; then - echo '安装 security-stack...' - bash \"$REMOTE_DIR/scripts/security/install-security-stack.sh\" || echo 'security-stack 安装失败，继续部署' - elif [ -f \"$REMOTE_DIR/scripts/security/install-perf-guard.sh\" ]; then - echo '安装 perf-guard...' - bash \"$REMOTE_DIR/scripts/security/install-perf-guard.sh\" || echo 'perf-guard 安装失败，继续部署' - fi -" - -echo -e "${GREEN}步骤 5: 配置环境变量...${NC}" -ssh_cmd " - cd $REMOTE_DIR - - # 生成 AUTH_SECRET - AUTH_SECRET=\$(openssl rand -base64 32) - - # 创建 .env.local 文件 - cat > .env.local << EOF -POSTGRES_URL=postgresql://chatbot_user:ChatBot@2024!Secure@localhost:5432/ai_chatbot -AUTH_SECRET=\$AUTH_SECRET -AUTH_URL=http://$SERVER -NODE_ENV=production -EOF - - echo '环境变量文件已创建' -" - -echo -e "${GREEN}步骤 6: 安装依赖和构建...${NC}" -ssh_cmd " - cd $REMOTE_DIR - export NVM_DIR=\"\$HOME/.nvm\" - [ -s \"\$NVM_DIR/nvm.sh\" ] && . \"\$NVM_DIR/nvm.sh\" - - echo '安装依赖...' - pnpm install --production=false - - echo '运行数据库迁移...' - pnpm db:migrate || echo '迁移可能已存在，继续...' - - echo '构建项目...' - pnpm build -" - -echo -e "${GREEN}步骤 7: 配置 PM2...${NC}" -# 上传 PM2 配置文件 -scp_cmd "ecosystem.config.js" "$REMOTE_DIR/" - -ssh_cmd " - cd $REMOTE_DIR - export NVM_DIR=\"\$HOME/.nvm\" - [ -s \"\$NVM_DIR/nvm.sh\" ] && . \"\$NVM_DIR/nvm.sh\" - - # 停止旧进程（如果存在） - pm2 delete ai-chatbot 2>/dev/null || true - - # 启动应用 - pm2 start ecosystem.config.js - pm2 save - - # 设置开机自启 - pm2 startup systemd -u root --hp /root 2>/dev/null || echo 'PM2 startup 已配置' -" - -echo -e "${GREEN}步骤 8: 配置 Nginx...${NC}" -ssh_cmd " - # 创建 Nginx 配置 - cat > /etc/nginx/sites-available/ai-chatbot << 'NGINX' -server { - listen 80; - server_name $SERVER; - - location / { - proxy_pass http://localhost:3000; - proxy_http_version 1.1; - proxy_set_header Upgrade \$http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host \$host; - proxy_set_header X-Real-IP \$remote_addr; - proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto \$scheme; - proxy_cache_bypass \$http_upgrade; - proxy_read_timeout 300s; - proxy_connect_timeout 75s; - } -} -NGINX - - # 启用站点 - ln -sf /etc/nginx/sites-available/ai-chatbot /etc/nginx/sites-enabled/ - - # 删除默认配置（如果存在） - rm -f /etc/nginx/sites-enabled/default - - # 测试配置 - nginx -t - - # 重启 Nginx - systemctl restart nginx -" - -echo -e "${GREEN}步骤 9: 配置防火墙...${NC}" -ssh_cmd " - ufw --force enable - ufw allow 80/tcp - ufw allow 443/tcp - ufw allow 5204/tcp - ufw allow 3000/tcp -" - -echo -e "${GREEN}步骤 10: 检查部署状态...${NC}" -ssh_cmd " - echo '=== PM2 状态 ===' - pm2 status - - echo '' - echo '=== 应用日志（最后20行）===' - pm2 logs ai-chatbot --lines 20 --nostream - - echo '' - echo '=== Nginx 状态 ===' - systemctl status nginx --no-pager -l - - echo '' - echo '=== 端口监听 ===' - netstat -tulpn | grep -E ':(80|3000|5432)' || ss -tulpn | grep -E ':(80|3000|5432)' -" - -# 清理本地临时文件 -rm -f /tmp/deploy.tar.gz - -echo -e "${GREEN}========================================${NC}" -echo -e "${GREEN}部署完成！${NC}" -echo -e "${GREEN}========================================${NC}" -echo -e "${YELLOW}访问地址: http://$SERVER${NC}" -echo -e "${YELLOW}管理员设置: http://$SERVER/admin/settings${NC}" -echo -e "${YELLOW}查看日志: ssh -p $PORT $USER@$SERVER 'pm2 logs ai-chatbot'${NC}" -echo -e "${YELLOW}查看状态: ssh -p $PORT $USER@$SERVER 'pm2 status'${NC}" -echo "" -echo -e "${YELLOW}数据库信息:${NC}" -echo -e " 数据库名: ai_chatbot" -echo -e " 用户名: chatbot_user" -echo -e " 密码: ChatBot@2024!Secure" -echo "" -echo -e "${YELLOW}注意: 请尽快修改数据库密码和服务器密码！${NC}" +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' 'Legacy deployment entry point disabled: it targets an obsolete host/runtime.' >&2 +printf '%s\n' 'Use: python scripts/production48.py status' >&2 +printf '%s\n' 'Then follow docs/production-48.md for build, stage, activate and rollback.' >&2 +exit 2 diff --git a/deploy.sh b/deploy.sh index 14d16c132..28bd5bfc5 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,108 +1,6 @@ -#!/bin/bash - -# AI Chatbot 部署脚本 -# 使用方法: ./deploy.sh - -set -e - -# 配置 -SERVER="156.225.27.52" -PORT="5204" -USER="root" -PASSWORD="${DEPLOY_SSH_PASSWORD:?set DEPLOY_SSH_PASSWORD}" -REMOTE_DIR="/var/www/ai-chatbot" -LOCAL_DIR="." - -# 颜色输出 -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -echo -e "${GREEN}开始部署到服务器...${NC}" - -# 检查本地是否有未提交的更改 -if [ -n "$(git status --porcelain)" ]; then - echo -e "${YELLOW}警告: 有未提交的更改${NC}" - read -p "是否继续? (y/n) " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - exit 1 - fi -fi - -# 1. 准备部署文件（跳过本地构建，在服务器上构建） -echo -e "${GREEN}步骤 1: 准备部署文件...${NC}" -tar --exclude='node_modules' \ - --exclude='.next' \ - --exclude='.git' \ - --exclude='.env.local' \ - --exclude='.env.production' \ - --exclude='*.log' \ - --exclude='deploy.tar.gz' \ - -czf deploy.tar.gz . - -# 2. 上传到服务器 -echo -e "${GREEN}步骤 2: 上传文件到服务器...${NC}" -sshpass -p "$PASSWORD" scp -o StrictHostKeyChecking=no -P $PORT deploy.tar.gz $USER@$SERVER:/tmp/ - -# 3. 在服务器上执行部署 -echo -e "${GREEN}步骤 3: 在服务器上部署...${NC}" -sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no -p $PORT $USER@$SERVER << 'ENDSSH' -set -e - -REMOTE_DIR="/var/www/ai-chatbot" -BACKUP_DIR="/root/backups/ai-chatbot" - -# 创建目录 -mkdir -p $REMOTE_DIR -mkdir -p $BACKUP_DIR - -# 备份现有部署（如果存在） -if [ -d "$REMOTE_DIR/.next" ]; then - echo "备份现有部署..." - tar -czf $BACKUP_DIR/backup_$(date +%Y%m%d_%H%M%S).tar.gz -C $REMOTE_DIR . -fi - -# 解压新文件 -echo "解压新文件..." -cd $REMOTE_DIR -tar -xzf /tmp/deploy.tar.gz - -# 安装安全基线（恶意进程守护 + 服务器硬化） -if [ -f "$REMOTE_DIR/scripts/security/install-security-stack.sh" ]; then - echo "安装 security-stack..." - bash "$REMOTE_DIR/scripts/security/install-security-stack.sh" || echo "security-stack 安装失败，继续部署" -elif [ -f "$REMOTE_DIR/scripts/security/install-perf-guard.sh" ]; then - echo "安装 perf-guard..." - bash "$REMOTE_DIR/scripts/security/install-perf-guard.sh" || echo "perf-guard 安装失败，继续部署" -fi - -# 安装依赖 -echo "安装依赖..." -pnpm install --production=false - -# 运行数据库迁移 -echo "运行数据库迁移..." -pnpm db:migrate || echo "数据库迁移失败，请检查配置" - -# 构建项目 -echo "构建项目..." -pnpm build - -# 重启应用 -echo "重启应用..." -pm2 restart ai-chatbot || pm2 start ecosystem.config.js - -# 清理 -rm /tmp/deploy.tar.gz - -echo "部署完成！" -ENDSSH - -# 4. 清理本地临时文件 -echo -e "${GREEN}步骤 4: 清理临时文件...${NC}" -rm deploy.tar.gz - -echo -e "${GREEN}部署完成！${NC}" -echo -e "${YELLOW}请检查应用状态: ssh -p $PORT $USER@$SERVER 'pm2 status'${NC}" +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' 'Legacy deployment entry point disabled: it targets an obsolete host/runtime.' >&2 +printf '%s\n' 'Use: python scripts/production48.py status' >&2 +printf '%s\n' 'Then follow docs/production-48.md for build, stage, activate and rollback.' >&2 +exit 2 diff --git a/docs/deployment.md b/docs/deployment.md index 4eaa91730..6e1c98cd5 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -2,6 +2,9 @@ 本文档用于生产或准生产环境部署。所有真实密钥只应配置在运行环境中，不要提交到 Git。 +当前 `ai.yydd.ai` 的 48 / PM2 生产环境使用 [48 生产发布流程](./production-48.md)。 +下方是通用安装说明，不能直接在当前生产目录执行迁移或构建。 + ## 运行环境 - Node.js 20 diff --git a/docs/production-48.md b/docs/production-48.md new file mode 100644 index 000000000..7f8fda681 --- /dev/null +++ b/docs/production-48.md @@ -0,0 +1,81 @@ +# 48 生产发布 + +本流程由维护电脑发起，使用已有 SSH 密钥。生产只更新 `ai-chatbot`，不运行数据库迁移， +不更新 ZCode 原生运行时，不重启主机、Nginx 或其他 PM2 应用。 + +## 已核实的机器 + +| 用途 | SSH 入口 | 主机 | +| --- | --- | --- | +| 构建机 | `root@192.168.102.146:22` | `dao02-Default-string`，Ubuntu 24.04，Node 20 | +| 同一构建机的公网入口 | `root@45.192.99.48:23322` | FRP 转发，不是生产宿主机 | +| 生产机 | `root@45.192.99.48:64192` | `SfSTE0ttmsycO`，CentOS Stream 9，Node 20 | +| 维护连接跳板 | `root@192.168.58.20:22` | 已信任的 NAS，仅转发 SSH | + +公网页面 `https://ai.yydd.ai/` 由生产 Nginx 转发至 `127.0.0.1:3125` 的 PM2 应用。 +脚本固定验证生产、构建机的 SSH 指纹和主机名；私钥留在维护电脑，不转发 SSH Agent。 +构建机原有 `/var/www/ai-chatbot` 是旧实例，本流程不会使用或覆盖它。 + +## 操作入口 + +维护电脑需要 Python 3.9+、Paramiko、Git，以及已信任的 SSH `known_hosts`。 +默认使用 `~/.ssh/45.192.99.48.pem` 和跳板密钥 `~/.ssh/youyidian_prod_ed25519`。 +可用 `--key`、`--jump-key`、`--known-hosts` 覆盖本机路径，不要把私钥提交到仓库。 + +```powershell +python scripts/production48.py status +python scripts/production48.py jobs + +# 完成 PR 合并后，确认远端 main 的最新提交，工作区必须干净。 +git fetch origin +git switch main +git merge --ff-only origin/main +python scripts/production48.py build +``` + +`build` 将确定的 `origin/main` 提交通过 `git archive` 送到构建机， +在 `/opt/ai-chatbot-builds/<时间-提交>/source` 安装锁定依赖并构建。 +使用 pnpm 9.12.3，不改变全局 pnpm。安装时不运行依赖脚本，构建时禁用自动安装系统服务。 +构建保持 TypeScript 检查，限制 Next 构建并行度和 Node 内存，并降低进程优先级。 + +构建机仅收到 `NEXT_PUBLIC_*` 等公开页面配置，以及指向本机关闭端口的占位数据库配置。 +不会收到生产数据库密码、AI Key、认证密钥或存储凭据。随后在构建机回环地址启动隔离候选实例， +核对构建 ID 并执行公开路由烟测，测试结束自动关闭该候选实例。 + +成功后会输出发布 ID，例如 `20261030T120000Z-0123456789ab`。后续步骤使用这个完整 ID： + +```powershell +python scripts/production48.py stage --release <发布ID> +python scripts/production48.py activate --release <发布ID> +python scripts/production48.py status + +# 仅对当前生效的这一版执行回滚： +python scripts/production48.py rollback --release <发布ID> +``` + +`stage` 校验传输 SHA256，在 `/var/www/ai-chatbot-releases/<发布ID>` 解包。 +它检查压缩包路径、磁盘容量、锁文件、实际依赖包清单和共享 public 文件校验和。 +完成解包后至少保留 1 GiB 空间；不足时停止，不自动删除历史构建。 +当前和旧浏览器标签页所需的静态资源通过硬链接保留，不改动旧构建。 + +`activate` 会重新核对生产版本和任务状态，拒绝在最近一小时仍有未结束生成任务时切换。 +更早的历史挂起记录只写入发布回执，不会自动修改数据库中的状态。 +它保存旧 PM2 配置和启动快照，然后仅更新 `ai-chatbot` 的工作目录并重启这个应用， +因此会有短暂切换窗口。健康检查同时核对进程目录、构建 ID、首页和路由。 +检查失败自动恢复旧配置并再次验活；通过后执行 `pm2 save`，让重启后使用已验收版本。 + +## 共享运行资料与边界 + +初版发布复用原目录 `/var/www/ai-chatbot` 中的 `node_modules`、`public`、`.env.local`、 +`tmp` 和模型配置备份，均通过链接使用，不复制、删除或覆盖。 +生产依赖必须与构建依赖的校验清单相符，Node ABI 必须一致。 +若本次修改了依赖锁文件或 public 中的文件，脚本会停止，须先单独审查并准备对应运行资料， +不能绕过校验强行切换。本流程适用于应用代码迭代；数据库、共享资源和原生 Agent 升级各自单独发布。 + +原生产 checkout 的支线及未提交改动仍保留，不能据其 Git HEAD 判断已发布版本。 +实际发布证据是：运行 cwd、`.next/BUILD_ID`、该目录 `release.json` 和 `activation.json`。 +维护电脑的 `.git/releases/<发布ID>/` 保留提交、构建日志、烟测、传输校验和及操作回执。 +服务器回滚配置可能含环境元数据，目录权限为 0700、文件为 0600，不放入公开目录或 Git。 + +仓库历史全量 lint 仍有已记录的问题，不能声称全部质量门禁通过。 +每次 PR 须记录本次改动的验证、历史失败和未覆盖项目；当前流程不会自动触发生产发布。 diff --git a/scripts/deploy-production-atomic.sh b/scripts/deploy-production-atomic.sh index e11e3c90b..28bd5bfc5 100755 --- a/scripts/deploy-production-atomic.sh +++ b/scripts/deploy-production-atomic.sh @@ -1,69 +1,6 @@ -#!/bin/sh -set -eu - -app_dir="/srv/ai-chatbot" -compose_dir="/srv/ai-chatbot-deploy" -candidate_dir="$(mktemp -d "${app_dir}/.next-candidate.XXXXXX")" -previous_dir="${app_dir}/.next-previous-$(date +%Y%m%dT%H%M%S)" -build_container="ai-chatbot-build-$(date +%s)" - -if grep -Eq '^[[:space:]]*CANVAS_E2E_LOCAL[[:space:]]*=[[:space:]]*"?true"?[[:space:]]*$' "${app_dir}/.env.local"; then - echo "Refusing production deployment while CANVAS_E2E_LOCAL=true." >&2 - exit 1 -fi - -cleanup_candidate() { - if [ -d "$candidate_dir" ]; then - rm -rf -- "$candidate_dir" - fi -} - -trap cleanup_candidate EXIT INT TERM - -docker run --rm \ - --name "$build_container" \ - --network host \ - -e NODE_OPTIONS=--max-old-space-size=6144 \ - -v "${app_dir}:/workspace" \ - -v "${candidate_dir}:/workspace/.next" \ - -w /workspace \ - node:20.20.0-bookworm-slim \ - sh -lc 'corepack enable && pnpm build' - -test -s "${candidate_dir}/BUILD_ID" - -docker stop ai-chatbot-app >/dev/null -docker rm ai-chatbot-app >/dev/null -if [ -d "${app_dir}/.next" ]; then - mv "${app_dir}/.next" "$previous_dir" -fi -mv "$candidate_dir" "${app_dir}/.next" -candidate_dir="" - -docker compose -f "${compose_dir}/docker-compose.yml" up -d app - -health_ok=false -attempt=1 -while [ "$attempt" -le 15 ]; do - if curl -fsS --max-time 10 http://127.0.0.1:3125/login >/dev/null; then - health_ok=true - break - fi - sleep 2 - attempt=$((attempt + 1)) -done - -if [ "$health_ok" != true ]; then - docker stop ai-chatbot-app >/dev/null 2>&1 || true - docker rm ai-chatbot-app >/dev/null 2>&1 || true - failed_dir="${app_dir}/.next-failed-$(date +%Y%m%dT%H%M%S)" - mv "${app_dir}/.next" "$failed_dir" - if [ -d "$previous_dir" ]; then - mv "$previous_dir" "${app_dir}/.next" - docker compose -f "${compose_dir}/docker-compose.yml" up -d app - fi - echo "Deployment health check failed; previous build restored." >&2 - exit 1 -fi - -echo "Production build activated. Previous build: ${previous_dir}" +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' 'Legacy deployment entry point disabled: it targets an obsolete host/runtime.' >&2 +printf '%s\n' 'Use: python scripts/production48.py status' >&2 +printf '%s\n' 'Then follow docs/production-48.md for build, stage, activate and rollback.' >&2 +exit 2 diff --git a/scripts/production48.py b/scripts/production48.py new file mode 100644 index 000000000..a6d4af154 --- /dev/null +++ b/scripts/production48.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""Build on .146 and release to native production :64192, with local SSH keys. + +Requires Python 3.9+ and Paramiko on the operator computer only. Start with +`python scripts/production48.py status`; see docs/production-48.md. +""" + +import argparse +import contextlib +import datetime +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parent.parent +BUILD_ROOT = "/opt/ai-chatbot-builds" +RELEASE_ROOT = "/var/www/ai-chatbot-releases" +PROD_FINGERPRINT = "SHA256:3tFAg02kzAfGNBDba+je+vgSB7TifvJPje9XIyVBgTM" +BUILD_FINGERPRINT = "SHA256:MXW5w48SFSTUaqS+dS1G9hY228hI6SLepjhCmItHL04" + + +def require(value, message): + if not value: + raise RuntimeError(message) + + +def git(*args): + return subprocess.check_output(["git", *args], cwd=ROOT, text=True).strip() + + +def sha(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n", encoding="utf8") + + +class Connections: + def __init__(self, args): + import paramiko + self.paramiko = paramiko + self.args = args + self.stack = contextlib.ExitStack() + self.clients = {} + + def __enter__(self): + return self + + def __exit__(self, *args): + self.stack.close() + + def client(self, kind): + import base64 + if kind in self.clients: + return self.clients[kind] + client = self.paramiko.SSHClient() + self.stack.callback(client.close) + client.load_host_keys(str(self.args.known_hosts)) + client.set_missing_host_key_policy(self.paramiko.RejectPolicy()) + options = dict(username="root", allow_agent=False, look_for_keys=False, + timeout=20, banner_timeout=20, auth_timeout=20) + if kind == "jump": + host, port, key = "192.168.58.20", 22, self.args.jump_key + elif kind == "build": + host, port, key = "192.168.102.146", 22, self.args.key + # The LAN entrance has the same verified key as the existing FRP entry. + known = client.get_host_keys().lookup("[45.192.99.48]:23322") + require(known and "ssh-ed25519" in known, "Missing trusted builder host key") + client.get_host_keys().add(host, "ssh-ed25519", known["ssh-ed25519"]) + else: + host, port, key = "45.192.99.48", 64192, self.args.key + pin = self.paramiko.Ed25519Key(data=base64.b64decode( + "AAAAC3NzaC1lZDI1NTE5AAAAIK7afV+YwLqvUFBlxDe/q2dUrpBePcLdG+mZ0JekHYPC")) + client.get_host_keys().add("[45.192.99.48]:64192", "ssh-ed25519", pin) + jump = self.client("jump") + options["sock"] = jump.get_transport().open_channel( + "direct-tcpip", (host, port), ("127.0.0.1", 0), timeout=20) + options.update(hostname=host, port=port, key_filename=str(key)) + client.connect(**options) + actual = "SHA256:" + base64.b64encode(hashlib.sha256( + client.get_transport().get_remote_server_key().asbytes()).digest()).decode().rstrip("=") + expected = {"build": BUILD_FINGERPRINT, "prod": PROD_FINGERPRINT}.get(kind) + require(not expected or actual == expected, f"{kind} host fingerprint mismatch") + client.get_transport().set_keepalive(20) + self.clients[kind] = client + return client + + def execute(self, kind, command, stdin=None, stream=False, timeout=1800): + channel = self.client(kind).get_transport().open_session(timeout=20) + channel.exec_command(command) + if stdin is not None: + channel.sendall(stdin.encode("utf8")) + channel.shutdown_write() + output, errors = bytearray(), bytearray() + deadline = time.monotonic() + timeout + try: + while True: + if channel.recv_ready(): + chunk = channel.recv(65536) + output.extend(chunk) + if stream: + print(chunk.decode("utf8", "replace"), end="", flush=True) + if channel.recv_stderr_ready(): + chunk = channel.recv_stderr(65536) + errors.extend(chunk) + if stream: + print(chunk.decode("utf8", "replace"), end="", file=sys.stderr, flush=True) + if channel.exit_status_ready() and not channel.recv_ready() and not channel.recv_stderr_ready(): + break + require(time.monotonic() < deadline, f"{kind} operation timed out; inspect remote state") + time.sleep(0.03) + code = channel.recv_exit_status() + require(code == 0, f"{kind} command failed ({code}): {errors.decode('utf8', 'replace')[-1800:]}") + return output.decode("utf8") + finally: + channel.close() + + def helper(self, action, **kwargs): + command = ["python3", "-", action] + for key, value in kwargs.items(): + command += ["--" + key, str(value)] + source = (ROOT / "scripts/release/host.py").read_text(encoding="utf8") + return json.loads(self.execute("prod", shlex.join(command), stdin=source)) + + def sftp(self, kind): + return self.paramiko.SFTPClient.from_transport( + self.client(kind).get_transport(), window_size=16 * 1024**2, max_packet_size=65536) + + def upload(self, kind, local, remote): + last = [0] + def progress(done, total): + if done - last[0] >= 32 * 1024**2: + print(f"[{kind}] transferred {done // 1024**2}/{total // 1024**2} MiB", flush=True) + last[0] = done + with self.sftp(kind) as transfer: + transfer.put(str(local), remote, callback=progress) + transfer.chmod(remote, 0o600) + + +def build(connections, args): + require(not git("status", "--porcelain"), "Commit requested changes before building") + commit = git("rev-parse", "origin/main^{commit}") + require(git("branch", "--show-current") == "main" and git("rev-parse", "HEAD") == commit, + "Fetch origin and switch to an up-to-date, clean main before building") + remote = connections.execute("build", "hostname").strip() + require(remote == "dao02-Default-string", "Unexpected builder") + require(connections.execute("build", "node -v").strip().startswith("v20."), "Builder requires Node 20") + release_id = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ-") + commit[:12] + local = args.receipts / release_id + local.mkdir(parents=True) + source = local / "source.tar.gz" + subprocess.run(["git", "archive", "--format=tar.gz", f"--output={source}", commit], cwd=ROOT, check=True) + metadata = {"sourceCommit": commit, "sourceTree": git("rev-parse", f"{commit}^{{tree}}"), + "sourceArchiveSha256": sha(source)} + write_json(local / "input.json", metadata) + environment = connections.helper("build-env") + environment.update({"AUTH_SECRET": os.urandom(32).hex(), "POSTGRES_URL": "postgres://build:build@127.0.0.1:9/build", + "AUTH_TRUST_HOST": "true", "NODE_ENV": "production"}) + workspace = f"{BUILD_ROOT}/{release_id}" + connections.execute("build", f"install -d -m 700 -- {shlex.quote(workspace)}") + for name in ["source.tar.gz", "input.json"]: + connections.upload("build", local / name, f"{workspace}/{name}") + # Helpers are carried separately, with the reviewed operator checkout's hashes. + for name in ["host.py", "build.sh"]: + connections.upload("build", ROOT / "scripts/release" / name, f"{workspace}/{name}") + setup = f"""import sys,json,pathlib,hashlib,importlib.util,os +os.umask(0o077) +workspace=pathlib.Path({workspace!r}) +assert hashlib.sha256((workspace/'source.tar.gz').read_bytes()).hexdigest()=={metadata['sourceArchiveSha256']!r} +spec=importlib.util.spec_from_file_location('release_host',workspace/'host.py'); module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module) +module.safe_extract(workspace/'source.tar.gz',workspace/'source') +environment=json.load(sys.stdin) +with open(workspace/'source/.env.production.local','x') as stream: + for key,value in environment.items():stream.write(key+'='+json.dumps(value)+'\\n') +""" + # stdin belongs to the program, so the program itself travels as a quoted -c argument. + connections.execute("build", shlex.join(["python3", "-c", setup]), stdin=json.dumps(environment)) + print(f"[build] {release_id} from main {commit}", flush=True) + log = connections.execute("build", shlex.join(["nice", "-n", "10", "bash", f"{workspace}/build.sh", workspace]), + stream=True, timeout=2400) + (local / "build.log").write_text(log, encoding="utf8") + smoke = json.loads(connections.execute("build", shlex.join( + ["python3", f"{workspace}/host.py", "smoke", "--workspace", workspace]), timeout=240)) + write_json(local / "smoke.json", smoke) + print(json.dumps(smoke), flush=True) + result = json.loads(connections.execute("build", shlex.join( + ["python3", f"{workspace}/host.py", "pack", "--workspace", workspace]), timeout=600)) + result.update({"builder": remote, "mainRef": commit, + "helperHashes": {name: sha(ROOT / "scripts/release" / name) for name in ["host.py", "build.sh"]}}) + write_json(local / "build.json", result) + print(json.dumps(result), flush=True) + + +def stage(connections, args): + local = args.receipts / args.release + receipt = json.loads((local / "build.json").read_text(encoding="utf8")) + artifact = local / "artifact.tar.gz" + if not artifact.exists(): + with connections.sftp("build") as transfer: + transfer.get(f"{BUILD_ROOT}/{args.release}/artifact.tar.gz", str(artifact)) + require(sha(artifact) == receipt["artifactSha256"], "Downloaded artifact checksum mismatch") + connections.helper("status") + connections.execute("prod", f"install -d -m 700 -- {RELEASE_ROOT}") + connections.upload("prod", artifact, f"{RELEASE_ROOT}/{args.release}.tar.gz") + result = connections.helper("stage", release=args.release, digest=receipt["artifactSha256"]) + write_json(local / "stage.json", result) + print(json.dumps(result)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=["status", "jobs", "build", "stage", "activate", "rollback"]) + parser.add_argument("--release") + parser.add_argument("--key", type=Path, default=Path.home() / ".ssh/45.192.99.48.pem") + parser.add_argument("--jump-key", type=Path, default=Path.home() / ".ssh/youyidian_prod_ed25519") + parser.add_argument("--known-hosts", type=Path, default=Path.home() / ".ssh/known_hosts") + parser.add_argument("--receipts", type=Path, default=ROOT / ".git/releases") + args = parser.parse_args() + if args.action in {"stage", "activate", "rollback"}: + require(args.release and re.fullmatch(r"\d{8}T\d{6}Z-[0-9a-f]{12}", args.release), "--release is required") + args.receipts = args.receipts.resolve() + with Connections(args) as connections: + if args.action == "build": + build(connections, args) + elif args.action == "stage": + stage(connections, args) + else: + kwargs = {"release": args.release} if args.release else {} + result = connections.helper(args.action, **kwargs) + if args.release: + write_json(args.receipts / args.release / (args.action + ".json"), result) + print(json.dumps(result)) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/build.sh b/scripts/release/build.sh new file mode 100644 index 000000000..ca8a9a157 --- /dev/null +++ b/scripts/release/build.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +# Runs only inside the unique workspace created by production48.py. +cd -- "${1:?workspace required}/source" +export DISABLE_PERF_GUARD_AUTO_INSTALL=1 +export CI=1 NEXT_TELEMETRY_DISABLED=1 NEXT_IGNORE_BUILD_TYPECHECK=0 +export NEXT_DIST_DIR=.next CIRCLE_NODE_TOTAL=4 +export NODE_OPTIONS=--max-old-space-size=8192 +export npm_config_cache="$(dirname "$PWD")/npm-cache" +export npm_config_registry="${BUILD_NPM_REGISTRY:-https://registry.npmjs.org}" + +# Do not change the machine's global pnpm or install any system service. +pnpm9() { npm exec --yes --package=pnpm@9.12.3 -- pnpm "$@"; } +test "$(pnpm9 --version)" = 9.12.3 +pnpm9 install --frozen-lockfile --ignore-scripts +pnpm9 build diff --git a/scripts/release/host.py b/scripts/release/host.py new file mode 100644 index 000000000..1defd00c7 --- /dev/null +++ b/scripts/release/host.py @@ -0,0 +1,426 @@ +"""Linux release operations. No SSH dependency; invoked by production48.py. + +Only the ai-chatbot PM2 application is changed. The original checkout, public +files, credentials, dependencies and historical builds remain in place. +""" + +import argparse +import datetime +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import socket +import subprocess +import tarfile +import time +import urllib.request + +APP = Path("/var/www/ai-chatbot") +RELEASES = Path("/var/www/ai-chatbot-releases") +NAME = "ai-chatbot" +RESERVE = 1024**3 + + +def require(value, message): + if not value: + raise RuntimeError(message) + + +def run(args, cwd=None, stdin=None, timeout=60): + result = subprocess.run(args, cwd=cwd, input=stdin, text=True, + capture_output=True, timeout=timeout) + require(result.returncode == 0, f"{args[0]} failed (exit {result.returncode})") + return result.stdout + + +def sha(path): + digest = hashlib.sha256() + with open(path, "rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def json_write(path, value): + temporary = path.with_name(path.name + ".tmp") + with open(temporary, "x", encoding="utf8") as stream: + os.chmod(temporary, 0o600) + json.dump(value, stream, indent=2) + stream.write("\n") + os.replace(temporary, path) + + +def safe_extract(archive, destination): + """Reject traversal, special files and links escaping a new directory.""" + require(not destination.exists(), "Extraction destination already exists") + with tarfile.open(archive, "r:gz") as bundle: + members = bundle.getmembers() + for member in members: + path = PurePosixPath(member.name) + require(not path.is_absolute() and ".." not in path.parts, + "Unsafe archive member") + require(member.isdir() or member.isfile() or member.issym(), + "Special files and hard links are not accepted") + if member.issym(): + link = PurePosixPath(member.linkname) + require(not link.is_absolute(), "Absolute archive symlink") + target = (destination / path.parent / member.linkname).resolve() + require(target.is_relative_to(destination.resolve()), + "Archive symlink escapes destination") + needed = sum(m.size for m in members if m.isfile()) + require(shutil.disk_usage(destination.parent).free > needed + RESERVE, + "Insufficient space: at least 1 GiB must remain after extraction") + destination.mkdir(mode=0o700) + # Extract normal entries first; never follow an archive-created link. + for member in members: + if member.issym(): + continue + path = destination / member.name + path.parent.mkdir(parents=True, exist_ok=True) + if member.isdir(): + path.mkdir(exist_ok=True) + else: + with bundle.extractfile(member) as source, open(path, "xb") as target: + shutil.copyfileobj(source, target) + path.chmod(member.mode & 0o777) + for member in members: + if member.issym(): + path = destination / member.name + path.parent.mkdir(parents=True, exist_ok=True) + path.symlink_to(member.linkname) + + +def release_path(release_id): + require(re.fullmatch(r"\d{8}T\d{6}Z-[0-9a-f]{12}", release_id), + "Invalid release ID") + return RELEASES / release_id + + +def app_state(): + require(socket.gethostname() == "SfSTE0ttmsycO", "Not the verified production host") + apps = json.loads(run(["pm2", "jlist"])) + matching = [app for app in apps if app["name"] == NAME] + require(len(matching) == 1, "Expected exactly one ai-chatbot PM2 application") + app = matching[0] + env = app["pm2_env"] + cwd = Path(env["pm_cwd"]) + require(cwd == APP or cwd.parent == RELEASES, "Unexpected production cwd") + require(env["pm_exec_path"] == str(APP / "node_modules/next/dist/bin/next"), + "Unexpected Next.js executable") + require(env.get("watch") is False, "PM2 watch must be disabled") + require(env.get("CANVAS_E2E_LOCAL") != "true", "Production E2E mode is enabled") + require(env.get("args") == ["start", "-p", "3125", "-H", "127.0.0.1"], + "Unexpected production listen arguments") + dist = env.get("NEXT_DIST_DIR", ".next") + require(re.fullmatch(r"\.next[\w.-]*", dist), "Unsafe active build directory") + return app, cwd, dist + + +def status(): + app, cwd, dist = app_state() + return {"hostname": socket.gethostname(), "pid": app["pid"], + "status": app["pm2_env"]["status"], "cwd": str(cwd), "distDir": dist, + "buildId": (cwd / dist / "BUILD_ID").read_text().strip(), + "freeGiB": round(shutil.disk_usage(APP).free / 1024**3, 2), + "node": run(["node", "-v"]).strip()} + + +ENV_LOADER = """ +const {createRequire}=require('node:module'); +const nextRequire=createRequire(require.resolve('next/package.json')); +nextRequire('@next/env').loadEnvConfig(process.cwd(),false,{info(){},error(){}}); +""" + + +def build_environment(): + # Only public build-time configuration leaves production. + source = ENV_LOADER + """ +const extra=['AUTH_URL','NEXTAUTH_URL','PASCAL_EDITOR_URL','CANVAS_DOWNLOAD_CDN_ORIGIN']; +console.log(JSON.stringify(Object.fromEntries(Object.entries(process.env) + .filter(([key])=>key.startsWith('NEXT_PUBLIC_')||extra.includes(key))))); +""" + app_state() + return json.loads(run(["node", "-"], cwd=APP, stdin=source)) + + +def job_status(): + source = ENV_LOADER + r""" +const sql=require('postgres')(process.env.POSTGRES_URL,{max:1,connect_timeout:5,onnotice(){}}); +(async()=>{try{ + const tables=['CanvasImageTask','CanvasCapabilityTask','CanvasAgentRun', + 'EnhanceProTask','AgentPsdRenderTask','VideoStoryboardTask'];const result={}; + await sql.begin('read only',async tx=>{ + await tx.unsafe("set local statement_timeout='5s'"); + for(const table of tables){ + const exists=await tx`select to_regclass(${JSON.stringify(table)}) as name`; + if(!exists[0].name)throw new Error('Missing task table'); + result[table]=await tx.unsafe('select status,count(*)::int as count,'+ + 'count(*) filter (where "updatedAt">now()-interval \'1 hour\')::int as recent '+ + 'from "'+table+'" where status not in '+ + '(\'succeeded\',\'failed\',\'completed\',\'cancelled\',\'interrupted\',\'needs_review\') group by status'); + } + });console.log(JSON.stringify(result)); + }catch(e){console.error('Task preflight failed');process.exitCode=1;} + finally{await sql.end();}})(); +""" + return json.loads(run(["node", "-"], cwd=APP, stdin=source, timeout=50)) + + +def inventory_packages(root): + result = {} + for path in sorted((root / "node_modules/.pnpm").glob("*/node_modules/*/package.json")): + result[path.relative_to(root / "node_modules").as_posix()] = sha(path) + for path in sorted((root / "node_modules/.pnpm").glob("*/node_modules/@*/*/package.json")): + result[path.relative_to(root / "node_modules").as_posix()] = sha(path) + return result + + +def build_manifest(workspace): + source = workspace / "source" + metadata = json.loads((workspace / "input.json").read_text()) + result = {"schema": 1, "sourceCommit": metadata["sourceCommit"], + "sourceTree": metadata["sourceTree"], "releaseId": workspace.name, + "nodeMajor": 20, "nodeAbi": "115", "pnpm": "9.12.3", + "buildId": (source / ".next/BUILD_ID").read_text().strip(), + "lockSha256": sha(source / "pnpm-lock.yaml"), + "publicFiles": {p.relative_to(source).as_posix(): sha(p) + for p in sorted((source / "public").rglob("*")) if p.is_file()}, + "directPackages": {name: sha(source / "node_modules" / name / "package.json") + for name in json.loads((source / "package.json").read_text())["dependencies"]}, + "installedPackages": inventory_packages(source)} + require(result["installedPackages"], "No dependency inventory") + json_write(source / "release.json", result) + with tarfile.open(workspace / "artifact.tar.gz", "w:gz", compresslevel=3) as archive: + for path in sorted(source.iterdir()): + if path.name in {"node_modules", "public", ".env.production.local", ".next"}: + continue + archive.add(path, arcname=path.name) + def filter_build(member): + if member.name == ".next/cache" or member.name.startswith(".next/cache/"): + return None + return member + archive.add(source / ".next", arcname=".next", filter=filter_build) + return {"releaseId": workspace.name, "sourceCommit": result["sourceCommit"], + "buildId": result["buildId"], "artifactSha256": sha(workspace / "artifact.tar.gz"), + "artifactBytes": (workspace / "artifact.tar.gz").stat().st_size} + + +def smoke(workspace): + source = workspace / "source" + expected = (source / ".next/BUILD_ID").read_text().strip() + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + environment = os.environ.copy() + environment.update({"NODE_ENV": "production", "NEXT_DIST_DIR": ".next", + "PORT": str(port), "POSTGRES_CONNECT_TIMEOUT_SECONDS": "1"}) + with open(workspace / "smoke.log", "w") as log: + process = subprocess.Popen(["node", "node_modules/next/dist/bin/next", "start", + "-H", "127.0.0.1", "-p", str(port)], + cwd=source, env=environment, stdout=log, stderr=log) + try: + ready = False + for _ in range(45): + require(process.poll() is None, "Isolated candidate exited; inspect smoke.log") + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/", timeout=3) as response: + ready = response.status == 200 and expected in response.read().decode("utf8") + if ready: + break + except Exception: + pass + time.sleep(1) + require(ready, "Isolated candidate readiness failed; inspect smoke.log") + output = run(["env", f"SMOKE_BASE_URL=http://127.0.0.1:{port}", "node", + "scripts/smoke-routes.mjs"], cwd=source, timeout=100) + return {"candidateSmoke": "passed", "buildId": expected, + "checks": output.strip().splitlines()} + finally: + process.terminate() + try: + process.wait(timeout=20) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def verify_shared(manifest): + require(sha(APP / "pnpm-lock.yaml") == manifest["lockSha256"], + "Dependencies changed: provision a separate reviewed runtime dependency set first") + for name, digest in manifest["publicFiles"].items(): + path = PurePosixPath(name) + require(path.parts[0] == "public" and ".." not in path.parts, + "Invalid public path") + require((APP / name).is_file() and sha(APP / name) == digest, + f"Public files differ: {name}; review shared asset publication first") + actual = inventory_packages(APP) + missing = [name for name, value in manifest["installedPackages"].items() + if actual.get(name) != value] + require(not missing, f"Installed dependency mismatch ({len(missing)} packages)") + for name, digest in manifest["directPackages"].items(): + require(sha(APP / "node_modules" / name / "package.json") == digest, + f"Direct dependency link mismatch: {name}") + require(run(["node", "-p", "process.versions.modules"]).strip() == manifest["nodeAbi"], + "Node ABI mismatch") + + +def stage(release_id, digest): + _, current, dist = app_state() + archive = RELEASES / (release_id + ".tar.gz") + require(sha(archive) == digest, "Artifact checksum mismatch") + destination = release_path(release_id) + safe_extract(archive, destination) + manifest = json.loads((destination / "release.json").read_text()) + require(manifest["releaseId"] == release_id, "Artifact release ID mismatch") + require(manifest["sourceCommit"][:12] == release_id[-12:], "Artifact source mismatch") + verify_shared(manifest) + for name in ["node_modules", "public", ".env.local", "tmp", ".model-config-backups"]: + require(not (destination / name).exists(), f"Unexpected bundled shared path: {name}") + if (APP / name).exists(): + (destination / name).symlink_to(APP / name) + # Keep already open browser tabs' content-hashed assets available. + for path in (current / dist / "static").rglob("*"): + if not path.is_file(): + continue + target = destination / ".next/static" / path.relative_to(current / dist / "static") + if target.exists(): + require(sha(target) == sha(path), "Static asset name collision") + else: + target.parent.mkdir(parents=True, exist_ok=True) + os.link(path, target) + json_write(destination / "staged.json", {"artifactSha256": digest, + "previousCwd": str(current), "previousDistDir": dist}) + # The verified unpacked release is retained; the transport copy is redundant. + archive.unlink() + return {"staged": release_id, "buildId": manifest["buildId"], + "freeGiB": round(shutil.disk_usage(APP).free / 1024**3, 2)} + + +def ecosystem(app, cwd, dist): + old = app["pm2_env"] + env = {k: v for k, v in old.items() if k.isupper() and isinstance(v, (str, int, bool))} + env.update({"NODE_ENV": "production", "PORT": "3125", "PWD": str(cwd), "NEXT_DIST_DIR": dist}) + return {"apps": [{"name": NAME, "cwd": str(cwd), "script": old["pm_exec_path"], + "args": old["args"], "interpreter": old.get("exec_interpreter", "/usr/bin/node"), + "exec_mode": "fork", "instances": 1, "watch": False, "env": env, + "autorestart": old.get("autorestart", True), + "min_uptime": old.get("min_uptime", "30s"), "max_restarts": old.get("max_restarts", 10), + "restart_delay": old.get("restart_delay", 3000), "kill_timeout": old.get("kill_timeout", 10000), + "max_memory_restart": old.get("max_memory_restart", "3G"), + "error_file": old["pm_err_log_path"], "out_file": old["pm_out_log_path"], + "merge_logs": True, "log_date_format": "YYYY-MM-DD HH:mm:ss Z"}]} + + +def healthy(cwd, build_id): + for _ in range(30): + try: + app, active_cwd, _ = app_state() + require(active_cwd == cwd and app["pm2_env"]["status"] == "online", "Wrong process") + request = urllib.request.Request("http://127.0.0.1:3125/", headers={"Host": "ai.yydd.ai"}) + with urllib.request.urlopen(request, timeout=5) as response: + body = response.read().decode("utf8") + if response.status == 200 and build_id in body: + return True + except Exception: + pass + time.sleep(2) + return False + + +def restart(config): + run(["pm2", "startOrRestart", str(config), "--only", NAME, "--update-env"], timeout=150) + + +def activate(release_id): + destination = release_path(release_id) + manifest = json.loads((destination / "release.json").read_text()) + staged = json.loads((destination / "staged.json").read_text()) + app, current, dist = app_state() + require(str(current) == staged["previousCwd"] and dist == staged["previousDistDir"], + "Production changed after staging") + require(not (destination / "activation.json").exists(), "Already activated; inspect status") + verify_shared(manifest) + jobs = job_status() + require(not any(row["recent"] for rows in jobs.values() for row in rows), + "Recent active generation jobs: defer activation") + before = ecosystem(app, current, dist) + previous_build = (current / dist / "BUILD_ID").read_text().strip() + json_write(destination / "previous.pm2.json", before) + json_write(destination / "candidate.pm2.json", ecosystem(app, destination, ".next")) + # Backup contains environment metadata and stays root-readable only. + shutil.copy2(Path.home() / ".pm2/dump.pm2", destination / "pm2-dump.before.json") + (destination / "pm2-dump.before.json").chmod(0o600) + receipt = {"sourceCommit": manifest["sourceCommit"], "buildId": manifest["buildId"], + "previousBuildId": previous_build, "previousCwd": str(current), + "activatedAtUtc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "historicalTaskStates": jobs, "artifactSha256": staged["artifactSha256"], + "phase": "activating"} + json_write(destination / "activation.json", receipt) + try: + restart(destination / "candidate.pm2.json") + require(healthy(destination, manifest["buildId"]), "Candidate health check failed") + run(["env", "SMOKE_BASE_URL=http://127.0.0.1:3125", "node", str(destination / "scripts/smoke-routes.mjs")], cwd=destination, + timeout=100) + run(["pm2", "save"]) + except BaseException: + restart(destination / "previous.pm2.json") + require(healthy(current, previous_build), "Rollback health check failed; operator required") + run(["pm2", "save"]) + receipt["phase"] = "rolled-back" + json_write(destination / "activation.json", receipt) + raise + receipt["phase"] = "active" + json_write(destination / "activation.json", receipt) + return receipt + + +def rollback(release_id): + destination = release_path(release_id) + receipt = json.loads((destination / "activation.json").read_text()) + _, current, _ = app_state() + require(current == destination, "Rollback target is not currently active") + jobs = job_status() + require(not any(row["recent"] for rows in jobs.values() for row in rows), + "Recent active generation jobs: defer rollback") + restart(destination / "previous.pm2.json") + require(healthy(Path(receipt["previousCwd"]), receipt["previousBuildId"]), + "Rollback health check failed") + run(["pm2", "save"]) + receipt["phase"] = "rolled-back" + json_write(destination / "activation.json", receipt) + return status() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("action", choices=["status", "build-env", "jobs", "smoke", "pack", "stage", "activate", "rollback"]) + parser.add_argument("--release") + parser.add_argument("--digest") + parser.add_argument("--workspace", type=Path) + args = parser.parse_args() + os.umask(0o077) + if args.action in {"pack", "smoke"}: + require(socket.gethostname() == "dao02-Default-string", "Not the verified build host") + require(args.workspace.parent == Path("/opt/ai-chatbot-builds"), "Invalid build workspace") + result = build_manifest(args.workspace) if args.action == "pack" else smoke(args.workspace) + elif args.action in {"status", "build-env", "jobs"}: + result = {"status": status, "build-env": build_environment, "jobs": job_status}[args.action]() + else: + import fcntl + require(RELEASES.is_dir() and not RELEASES.is_symlink(), "Invalid release root") + with open(RELEASES / ".deploy.lock", "a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + if args.action == "stage": + result = stage(args.release, args.digest) + elif args.action == "activate": + result = activate(args.release) + else: + result = rollback(args.release) + print(json.dumps(result)) + + +if __name__ == "__main__": + main() diff --git a/tests/routes/production-local-fallbacks.test.ts b/tests/routes/production-local-fallbacks.test.ts index a9dce008f..9c3253f26 100644 --- a/tests/routes/production-local-fallbacks.test.ts +++ b/tests/routes/production-local-fallbacks.test.ts @@ -42,14 +42,13 @@ test("production deploy stages the main build and retains active static chunks", expect(deployScript).toContain('curl -fsS "$MAIN_INTERNAL_URL/ping"'); }); -test("atomic production deploy rejects the local E2E fallback switch", () => { +test("legacy atomic deployment cannot target the obsolete Docker runtime", () => { const deployScript = readFileSync( resolve(process.cwd(), "scripts/deploy-production-atomic.sh"), "utf8" ); - expect(deployScript).toContain("CANVAS_E2E_LOCAL"); - expect(deployScript).toContain( - "Refusing production deployment while CANVAS_E2E_LOCAL=true." - ); + expect(deployScript).toContain("exit 2"); + expect(deployScript).toContain("scripts/production48.py status"); + expect(deployScript).not.toContain("docker compose"); }); diff --git a/tests/scripts/test_production48.py b/tests/scripts/test_production48.py new file mode 100644 index 000000000..d307fb70b --- /dev/null +++ b/tests/scripts/test_production48.py @@ -0,0 +1,190 @@ +"""Release boundary and rollback regression tests; no SSH or real PM2 changes.""" + +import importlib.util +import io +import json +from pathlib import Path +import shutil +import subprocess +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location("release_host", ROOT / "scripts/release/host.py") +host = importlib.util.module_from_spec(spec) +spec.loader.exec_module(host) + + +class ArchiveTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name).resolve() + self.archive = self.root / "artifact.tar.gz" + self.destination = self.root / "release" + + def tearDown(self): + self.temporary.cleanup() + + def archive_with(self, name, link=None, kind=None): + with tarfile.open(self.archive, "w:gz") as bundle: + member = tarfile.TarInfo(name) + if link is not None: + member.type, member.linkname = tarfile.SYMTYPE, link + elif kind is not None: + member.type = kind + else: + member.size = 4 + bundle.addfile(member, None if link is not None or kind is not None else io.BytesIO(b"test")) + + def test_extracts_a_regular_file(self): + self.archive_with("app/version.txt") + host.safe_extract(self.archive, self.destination) + self.assertEqual((self.destination / "app/version.txt").read_bytes(), b"test") + + def test_refuses_parent_traversal(self): + self.archive_with("../outside") + with self.assertRaisesRegex(RuntimeError, "Unsafe archive"): + host.safe_extract(self.archive, self.destination) + self.assertFalse((self.root / "outside").exists()) + + def test_refuses_absolute_paths(self): + self.archive_with("/outside") + with self.assertRaisesRegex(RuntimeError, "Unsafe archive"): + host.safe_extract(self.archive, self.destination) + + def test_refuses_symlink_escape(self): + self.archive_with("app/link", "../../outside") + with self.assertRaisesRegex(RuntimeError, "escapes"): + host.safe_extract(self.archive, self.destination) + + def test_refuses_absolute_symlink(self): + self.archive_with("app/link", "/etc/passwd") + with self.assertRaisesRegex(RuntimeError, "Absolute archive"): + host.safe_extract(self.archive, self.destination) + + def test_refuses_special_files(self): + self.archive_with("device", kind=tarfile.FIFOTYPE) + with self.assertRaisesRegex(RuntimeError, "Special files"): + host.safe_extract(self.archive, self.destination) + + def test_refuses_existing_release(self): + self.destination.mkdir() + self.archive_with("file") + with self.assertRaisesRegex(RuntimeError, "already exists"): + host.safe_extract(self.archive, self.destination) + + def test_keeps_disk_reserve_without_deleting_anything(self): + self.archive_with("file") + with patch.object(host.shutil, "disk_usage", return_value=shutil._ntuple_diskusage(10, 9, 1)): + with self.assertRaisesRegex(RuntimeError, "Insufficient space"): + host.safe_extract(self.archive, self.destination) + self.assertFalse(self.destination.exists()) + + +class ActivationTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name).resolve() + self.releases = self.root / "releases" + self.release_id = "20261030T120000Z-0123456789ab" + self.target = self.releases / self.release_id + self.target.mkdir(parents=True) + self.previous = self.root / "old" + (self.previous / ".next-old").mkdir(parents=True) + (self.previous / ".next-old/BUILD_ID").write_text("old-build") + (self.root / ".pm2").mkdir() + (self.root / ".pm2/dump.pm2").write_text("[]") + host.json_write(self.target / "release.json", {"sourceCommit": "0123456789ab" + "0" * 28, + "buildId": "new-build"}) + host.json_write(self.target / "staged.json", {"previousCwd": str(self.previous), + "previousDistDir": ".next-old", "artifactSha256": "artifact-hash"}) + self.app = {"pid": 100, "pm2_env": {"pm_exec_path": "/old/node_modules/next/dist/bin/next", + "args": ["start", "-p", "3125", "-H", "127.0.0.1"], + "CANVAS_ZCODE_STATE_DIR": "/private/state", "NEXT_DIST_DIR": ".next-old", + "pm_err_log_path": "/logs/error", "pm_out_log_path": "/logs/out"}} + self.patches = [patch.object(host, "RELEASES", self.releases), + patch.object(host, "verify_shared"), + patch.object(host, "app_state", return_value=(self.app, self.previous, ".next-old")), + patch.object(host.Path, "home", return_value=self.root), + patch.object(host, "run", return_value=""), + patch.object(host, "restart"), + patch.object(host, "job_status", return_value={"CanvasAgentRun": []})] + self.mocks = [p.start() for p in self.patches] + self.restart = self.mocks[-2] + + def tearDown(self): + for p in reversed(self.patches): + p.stop() + self.temporary.cleanup() + + def test_active_job_blocks_before_restart(self): + self.mocks[-1].return_value = {"CanvasAgentRun": [{"count": 1, "recent": 1}]} + with self.assertRaisesRegex(RuntimeError, "Recent active"): + host.activate(self.release_id) + self.restart.assert_not_called() + self.assertFalse((self.target / "activation.json").exists()) + + def test_new_release_health_failure_restores_previous_build(self): + with patch.object(host, "healthy", side_effect=[False, True]): + with self.assertRaisesRegex(RuntimeError, "Candidate health"): + host.activate(self.release_id) + self.assertEqual([call.args[0].name for call in self.restart.call_args_list], + ["candidate.pm2.json", "previous.pm2.json"]) + self.assertEqual(json.loads((self.target / "activation.json").read_text())["phase"], "rolled-back") + + def test_success_records_provenance_and_keeps_native_runtime_state(self): + with patch.object(host, "healthy", return_value=True): + result = host.activate(self.release_id) + self.assertEqual(result["phase"], "active") + config = json.loads((self.target / "candidate.pm2.json").read_text())["apps"][0] + self.assertEqual(config["name"], "ai-chatbot") + self.assertEqual(config["cwd"], str(self.target)) + self.assertEqual(config["env"]["CANVAS_ZCODE_STATE_DIR"], "/private/state") + self.assertNotIn("/private/state", json.dumps(result)) + + def test_changed_production_blocks_activation(self): + self.mocks[2].return_value = (self.app, self.root / "other", ".next") + with self.assertRaisesRegex(RuntimeError, "changed after staging"): + host.activate(self.release_id) + self.restart.assert_not_called() + + def test_never_retries_an_activation_without_inspection(self): + host.json_write(self.target / "activation.json", {"phase": "activating"}) + with self.assertRaisesRegex(RuntimeError, "Already activated"): + host.activate(self.release_id) + self.restart.assert_not_called() + + def test_rollback_failure_is_reported(self): + with patch.object(host, "healthy", return_value=False): + with self.assertRaisesRegex(RuntimeError, "Rollback health check failed"): + host.activate(self.release_id) + + +class HelperTests(unittest.TestCase): + def test_production_e2e_switch_blocks_release(self): + app = {"name": "ai-chatbot", "pm2_env": { + "pm_cwd": "/var/www/ai-chatbot", + "pm_exec_path": str(host.APP / "node_modules/next/dist/bin/next"), + "watch": False, "CANVAS_E2E_LOCAL": "true"}} + with patch.object(host.socket, "gethostname", return_value="SfSTE0ttmsycO"), \ + patch.object(host, "run", return_value=json.dumps([app])): + with self.assertRaisesRegex(RuntimeError, "Production E2E mode"): + host.app_state() + + def test_release_ids_cannot_escape_root(self): + for value in ["../app", "/etc", "20261030T120000Z-0123456789ab/../../outside"]: + with self.assertRaisesRegex(RuntimeError, "Invalid release"): + host.release_path(value) + + def test_task_query_javascript_is_valid(self): + sources = [] + with patch.object(host, "run", side_effect=lambda *a, **k: sources.append(k["stdin"]) or "{}"): + host.job_status() + result = subprocess.run(["node", "--check"], input=sources[0], text=True, capture_output=True) + self.assertEqual(result.returncode, 0, result.stderr) + + +if __name__ == "__main__": + unittest.main()