diff --git a/deploy/activate-incremental-release.sh b/deploy/activate-incremental-release.sh index cd67feeb..354930ff 100644 --- a/deploy/activate-incremental-release.sh +++ b/deploy/activate-incremental-release.sh @@ -7,6 +7,11 @@ REMOTE_DIR="${3:-/opt/caiwuxitong}" DEPLOY_SHA="${4:?DEPLOY_SHA is required}" RUN_MIGRATIONS="${5:-0}" KEEP_IMAGES="${6:-2}" +EXPECTED_PREVIOUS_SHA="${7:?Checked production revision is required; update the deployment scripts}" +DEPLOY_ACTOR="${8:-unknown}" +RELEASE_APP_URL="${9:-}" +RELEASE_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +. "$RELEASE_DIR/release-guard.sh" IMAGE_NAME="caiwu_akaunting" LOCK_FILE="$REMOTE_DIR/.deploy.lock" REGISTRY_API="http://127.0.0.1:5000/v2/caiwuxitong/app" @@ -65,6 +70,12 @@ if ! flock -n 9; then exit 1 fi +RELEASE_STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +RELEASE_RESULT=blocked +trap 'release_status=$?; caiwu_record_release_result "$release_status"' EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + # 容器替换会终止其中的同步进程;与生产刷新任务共享同一把锁。 exec 8>/run/caiwu-upstream-sync.lock echo "Waiting for active upstream synchronization before replacing the application container" @@ -74,7 +85,8 @@ if ! flock -w 1800 8; then fi PREVIOUS_IMAGE="$(docker inspect caiwu_app --format '{{.Config.Image}}' 2>/dev/null || true)" -PREVIOUS_SHA="$(cat "$REMOTE_DIR/.deployed-sha" 2>/dev/null || true)" +caiwu_verify_release_base +RELEASE_RESULT=running restore_previous_image() { case "$PREVIOUS_IMAGE" in @@ -96,6 +108,8 @@ if [ "$LOADED_SHA" != "$DEPLOY_SHA" ]; then exit 1 fi +caiwu_install_release_files + cd "$REMOTE_DIR" export IMAGE_TAG RUN_MIGRATIONS diff --git a/deploy/activate-release.sh b/deploy/activate-release.sh index 8d68be89..191b0504 100644 --- a/deploy/activate-release.sh +++ b/deploy/activate-release.sh @@ -7,6 +7,11 @@ REMOTE_DIR="${3:-/opt/caiwuxitong}" DEPLOY_SHA="${4:?DEPLOY_SHA is required}" RUN_MIGRATIONS="${5:-0}" KEEP_IMAGES="${6:-2}" +EXPECTED_PREVIOUS_SHA="${7:?Checked production revision is required; update the deployment scripts}" +DEPLOY_ACTOR="${8:-unknown}" +RELEASE_APP_URL="${9:-}" +RELEASE_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +. "$RELEASE_DIR/release-guard.sh" IMAGE_NAME="caiwu_akaunting" LOCK_FILE="$REMOTE_DIR/.deploy.lock" @@ -14,7 +19,9 @@ cleanup() { rm -f "$IMAGE_ARCHIVE" } -trap cleanup EXIT INT TERM +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM if [ "$(id -u)" -ne 0 ]; then echo "activate-release.sh must run as root" >&2 @@ -69,6 +76,10 @@ if ! flock -n 9; then exit 1 fi +RELEASE_STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +RELEASE_RESULT=blocked +trap 'release_status=$?; caiwu_record_release_result "$release_status"; cleanup' EXIT + # 容器替换会终止其中的同步进程;与生产刷新任务共享同一把锁。 exec 8>/run/caiwu-upstream-sync.lock echo "Waiting for active upstream synchronization before replacing the application container" @@ -78,7 +89,8 @@ if ! flock -w 1800 8; then fi PREVIOUS_IMAGE="$(docker inspect caiwu_app --format '{{.Config.Image}}' 2>/dev/null || true)" -PREVIOUS_SHA="$(cat "$REMOTE_DIR/.deployed-sha" 2>/dev/null || true)" +caiwu_verify_release_base +RELEASE_RESULT=running restore_previous_image() { case "$PREVIOUS_IMAGE" in @@ -100,6 +112,8 @@ if [ "$LOADED_SHA" != "$DEPLOY_SHA" ]; then exit 1 fi +caiwu_install_release_files + cd "$REMOTE_DIR" export IMAGE_TAG RUN_MIGRATIONS diff --git a/deploy/install-release-protection.sh b/deploy/install-release-protection.sh new file mode 100644 index 00000000..d5b09cec --- /dev/null +++ b/deploy/install-release-protection.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Caller holds the production deployment lock. Does not restart any service. +set -eu +REMOTE_DIR="${1:-/opt/caiwuxitong}" +SOURCE_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +LEGACY_STUB="$SOURCE_DIR/retired-activation.sh" +[ "$(id -u)" -eq 0 ] || { echo 'Release protection must be installed as root' >&2; exit 1; } +[ -f "$LEGACY_STUB" ] || { echo 'Missing retired activation stub' >&2; exit 1; } + +is_immutable() { lsattr -d "$1" 2>/dev/null | awk '{print $1}' | grep -q i; } +needs_install=0 +for legacy in activate-release.sh activate-incremental-release.sh; do + if cmp -s "$LEGACY_STUB" "$REMOTE_DIR/$legacy" && is_immutable "$REMOTE_DIR/$legacy"; then + continue + fi + if is_immutable "$REMOTE_DIR/$legacy"; then + echo "Unexpected immutable legacy entry point: $legacy; verify it before changing protection" >&2 + exit 1 + fi + needs_install=1 +done +[ "$needs_install" -eq 1 ] || exit 0 + +# Test support on an owned temporary file before touching either entry point. +probe="$(mktemp "$REMOTE_DIR/.release-protection.XXXXXX")" +cleanup_probe() { chattr -i "$probe" 2>/dev/null || true; rm -f "$probe"; } +trap cleanup_probe EXIT +chattr +i "$probe" || { echo 'Filesystem cannot protect legacy entry points; no application was changed' >&2; exit 1; } +chattr -i "$probe" +backup="$REMOTE_DIR/runtime/private/release-protection/$(date -u +%Y%m%dT%H%M%SZ)-$$" +umask 077 +mkdir -p "$backup" +for legacy in activate-release.sh activate-incremental-release.sh; do + if cmp -s "$LEGACY_STUB" "$REMOTE_DIR/$legacy" && is_immutable "$REMOTE_DIR/$legacy"; then + continue + fi + [ ! -f "$REMOTE_DIR/$legacy" ] || cp -p "$REMOTE_DIR/$legacy" "$backup/$legacy" + install -m 700 "$LEGACY_STUB" "$REMOTE_DIR/.$legacy.next" + mv "$REMOTE_DIR/.$legacy.next" "$REMOTE_DIR/$legacy" + chattr +i "$REMOTE_DIR/$legacy" + cmp -s "$LEGACY_STUB" "$REMOTE_DIR/$legacy" && is_immutable "$REMOTE_DIR/$legacy" +done +echo 'Legacy activation paths protected; current application and databases were not restarted' diff --git a/deploy/release-guard.sh b/deploy/release-guard.sh new file mode 100644 index 00000000..d8cf6ea1 --- /dev/null +++ b/deploy/release-guard.sh @@ -0,0 +1,72 @@ +#!/bin/sh +# Sourced by both activation paths after the deployment and synchronization locks. + +caiwu_verify_release_base() { + case "$EXPECTED_PREVIOUS_SHA" in + none) ;; + *[!0-9a-f]*|'') echo 'Missing or invalid checked production revision; update the deployment scripts' >&2; return 1 ;; + *) [ "${#EXPECTED_PREVIOUS_SHA}" -eq 40 ] || return 1 ;; + esac + PREVIOUS_SHA="$(cat "$REMOTE_DIR/.deployed-sha" 2>/dev/null || true)" + [ -n "$PREVIOUS_SHA" ] || PREVIOUS_SHA=none + if [ "$PREVIOUS_SHA" != none ] && ! printf '%s\n' "$PREVIOUS_SHA" | grep -Eq '^[0-9a-f]{40}$'; then + echo 'Invalid production revision marker; refusing activation' >&2 + return 1 + fi + if [ "$PREVIOUS_SHA" != "$EXPECTED_PREVIOUS_SHA" ]; then + echo "Production changed while this release was being prepared: checked=$EXPECTED_PREVIOUS_SHA current=$PREVIOUS_SHA requested=$DEPLOY_SHA. Fetch latest main and retry; the running application was not changed." >&2 + return 1 + fi + + # A failed/interrupted older activation must not leave the marker ahead of the app. + running_sha="$(docker inspect caiwu_app --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' 2>/dev/null || true)" + if [ "$PREVIOUS_SHA" != none ] && [ "$running_sha" != "$PREVIOUS_SHA" ]; then + echo "Running image and production marker disagree; refusing activation: marker=$PREVIOUS_SHA image=$running_sha" >&2 + return 1 + fi + if [ "$PREVIOUS_SHA" = none ] && [ -n "$running_sha" ]; then + echo 'An application is already running without a production marker; verify its revision before deploying' >&2 + return 1 + fi +} + +caiwu_install_release_files() { + # Each uploader has its own directory. Shared runtime files change only inside + # the activation lock and after the production revision has been rechecked. + [ "$RELEASE_DIR" != "$REMOTE_DIR" ] || return 0 + release_files='compose.jiangling.yml activate-release.sh activate-incremental-release.sh release-guard.sh retired-activation.sh install-release-protection.sh config-db-init.sh run-finance-batch.sh clickhouse-resource-policy.xml update-runtime-url.sh configure-dingtalk-callback.sh host-nginx-8442-dingtalk.conf setup-incremental-registry.sh registry-config.yml' + for release_file in $release_files; do + [ -f "$RELEASE_DIR/$release_file" ] || { echo "Incomplete staged release: $release_file" >&2; return 1; } + done + sh "$RELEASE_DIR/install-release-protection.sh" "$REMOTE_DIR" || return 1 + for release_file in $release_files; do + case "$release_file" in activate-release.sh|activate-incremental-release.sh) continue ;; esac + case "$release_file" in *.sh) release_mode=700 ;; *) release_mode=644 ;; esac + [ "$release_file" != config-db-init.sh ] || release_mode=644 + install -m "$release_mode" "$RELEASE_DIR/$release_file" "$REMOTE_DIR/.$release_file.next" + mv "$REMOTE_DIR/.$release_file.next" "$REMOTE_DIR/$release_file" + done + if [ -n "$RELEASE_APP_URL" ]; then + "$REMOTE_DIR/update-runtime-url.sh" "$RELEASE_APP_URL" "$REMOTE_DIR" + fi + printf '%s\n' "$DEPLOY_SHA" > "$REMOTE_DIR/.release-controls-sha.next" + mv "$REMOTE_DIR/.release-controls-sha.next" "$REMOTE_DIR/.release-controls-sha" +} + +caiwu_record_release_result() { + release_exit="$1" + [ "$RELEASE_RESULT" != running ] || RELEASE_RESULT=failed + [ "$release_exit" -ne 0 ] || RELEASE_RESULT=success + # Client identity is declared by the workstation; the SSH source is recorded + # separately. Neither field is inferred from the commit author. + release_actor="$(printf '%s' "$DEPLOY_ACTOR" | tr -cd 'A-Za-z0-9_.@:-' | cut -c1-160)" + release_peer="$(printf '%s' "${SSH_CONNECTION:-local}" | tr -cd 'A-Za-z0-9 .:-' | cut -c1-160)" + ( + umask 077 + mkdir -p "$REMOTE_DIR/runtime/private" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$RELEASE_STARTED_AT" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$release_actor" "$release_peer" \ + "${PREVIOUS_SHA:-unknown}" "$DEPLOY_SHA" "$RELEASE_RESULT" "$release_exit" "$EXPECTED_PREVIOUS_SHA" \ + >> "$REMOTE_DIR/runtime/private/deployments.tsv" + ) || echo 'Could not write the private deployment receipt' >&2 +} diff --git a/deploy/retired-activation.sh b/deploy/retired-activation.sh new file mode 100644 index 00000000..f378ffe8 --- /dev/null +++ b/deploy/retired-activation.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# This legacy shared path is intentionally retired and immutable on production. +echo 'Legacy activation is disabled. Fetch latest main and use the workstation deployment script with an isolated release directory and checked production revision. The running application was not changed.' >&2 +exit 1 diff --git a/docs/开发与部署SOP.md b/docs/开发与部署SOP.md index 9cb55480..feee2d7e 100644 --- a/docs/开发与部署SOP.md +++ b/docs/开发与部署SOP.md @@ -129,11 +129,22 @@ bash scripts/deploy-from-mac.sh 8. 验证唯一私有七牛桶的写入、读取和删除,并把旧本机业务上传幂等复制到七牛;不删除本机原件; 9. 检查服务器本机后端页面; 10. 任一存储验证或页面验证失败时恢复上一个镜像; -11. 使用服务器互斥锁阻止两个人同时切换版本; +11. 每次上传使用独立暂存目录;服务器取得发布锁和同步锁后,再核对线上 SHA 与发布机已检查的 SHA,确认未变化才切换配置和应用; 12. 删除本地临时镜像包并关闭 SSH 隧道; 13. 开发机、服务器 Docker 和服务器 Registry 均仅保留当前和上一个本项目版本; 14. 清理 7 天前的本项目构建缓存,并将总量压到最多 3GB。 +### 5.0 多人发布保护 + +- 远端 `main` 禁止直接推送和强推,任务分支通过 PR 合并;不额外要求人工审批或新增 CI 门槛。 +- 发布机先刷新远端 `main`,正式发布必须与最新主线一致,并检查本次版本包含线上提交。 +- 上传期间若别人已发布,服务器在锁内拒绝本次激活,运行中的应用保持原版本。先 `git fetch`、快进到最新 `main` 并重新发布,不用回滚开关绕过并发校验。 +- Windows 和 macOS 发布机都必须使用最新脚本。共享根目录的两个旧激活入口已停用并设置 Linux immutable 属性,旧发布机无法用上传覆盖它们;新脚本从每次独立暂存目录执行激活,缺少已检查线上版本参数也会被拒绝。 +- 运行配置按每次发布独立暂存,核对通过后才在锁内安装;上传文件不会提前覆盖其他发布使用的配置。 +- 服务器私有 `runtime/private/deployments.tsv` 记录起止 UTC 时间、发布机声明的身份、SSH 来源、前后 SHA、结果、退出码和已检查的 SHA。声明身份及提交作者不能单独证明具体操作者;记录不包含凭据或业务数据。 +- 安装旧入口保护前自动备份到服务器私有 `runtime/private/release-protection/`。普通发布不解除 immutable 属性;维护这些停用入口时先核对备份和目的,再由维护人员使用 `chattr -i` 处理指定文件。应用镜像回滚不需要启用旧入口。 +- 仅维护发布控制脚本时,可在发布锁内备份并安装已合并主线的控制脚本,记录独立的 `.release-controls-sha`,不重建应用容器、不修改表示应用镜像版本的 `.deployed-sha`。 + ### 5.1 安全软件拦截 regctl 时的降级 第 6 步默认用项目专用 `regctl`(未签名)把 BuildKit 原生 OCI 分层包推入服务器本机 Registry。少数开发机的 360 安全卫士、金山毒霸等安全软件会拦截未签名程序,表现为 `regctl-v0.11.5.exe` 启动即报“拒绝访问”。 diff --git a/scripts/deploy-from-mac.sh b/scripts/deploy-from-mac.sh index ec5d0180..0fa018a9 100755 --- a/scripts/deploy-from-mac.sh +++ b/scripts/deploy-from-mac.sh @@ -416,7 +416,8 @@ if [ -n "$(git status --porcelain --untracked-files=no)" ]; then fail '工作区存在未提交的受控文件改动,拒绝构建。请先提交或处理清楚。' fi -if [ "$branch" = 'main' ] && git rev-parse --verify --quiet origin/main >/dev/null 2>&1; then +git fetch origin main --quiet || fail '无法更新远端 main,已停止发布。' +if [ "$branch" = 'main' ]; then if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/main)" ]; then fail '本地 main 与 origin/main 不一致,请先执行 git pull --ff-only 再发布。' fi @@ -457,7 +458,6 @@ ssh_control_path="/tmp/caiwu-ssh-$$-$RANDOM.sock" ssh_options+=(-o ControlMaster=auto -o ControlPersist=600 -o "ControlPath=$ssh_control_path") tunnel_ssh_options=("${ssh_options[@]}" -o ControlPath=none) -remote_archive="$remote_path/image-$image_tag.tar.gz" temp_root="$(mktemp -d "${TMPDIR:-/tmp}/caiwu-deploy-XXXXXX")" local_tar="$temp_root/image.tar" local_archive="$temp_root/image.tar.gz" @@ -512,6 +512,13 @@ for pair in \ done previous_sha="$(ssh "${ssh_options[@]}" "$ssh_target" "cat '$remote_path/.deployed-sha' 2>/dev/null || true" | tr -d '[:space:]')" +if [ -n "$previous_sha" ] && ! printf '%s\n' "$previous_sha" | grep -Eq '^[0-9a-f]{40}$'; then + fail '线上版本标记格式无效,已停止发布。' +fi +expected_previous_sha="${previous_sha:-none}" +deploy_actor="$(printf '%s@%s' "$(git config user.name)" "$(hostname)" | tr -c 'A-Za-z0-9_.@:-' '_')" +remote_release_path="$remote_path/releases/$image_tag-$(uuidgen | tr 'A-Z' 'a-z')" +remote_archive="$remote_release_path/image-$image_tag.tar.gz" if [ -n "$previous_sha" ]; then if git cat-file -e "${previous_sha}^{commit}" 2>/dev/null; then @@ -556,7 +563,7 @@ if [ "$bootstrap_server" -eq 1 ]; then bootstrap_remote='' fi -ssh "${ssh_options[@]}" "$ssh_target" "docker version >/dev/null && docker compose version >/dev/null && install -d -m 700 '$remote_path'" \ +ssh "${ssh_options[@]}" "$ssh_target" "docker version >/dev/null && docker compose version >/dev/null && install -d -m 700 '$remote_path' '$remote_release_path'" \ || fail '服务器 Docker 未就绪;首次发布请添加 --bootstrap-server。' ensure_qiniu_storage_runtime @@ -570,6 +577,9 @@ scp_to_remote \ "$repo_root/compose.jiangling.yml" \ "$repo_root/deploy/activate-release.sh" \ "$repo_root/deploy/activate-incremental-release.sh" \ + "$repo_root/deploy/release-guard.sh" \ + "$repo_root/deploy/retired-activation.sh" \ + "$repo_root/deploy/install-release-protection.sh" \ "$repo_root/deploy/config-db-init.sh" \ "$repo_root/deploy/run-finance-batch.sh" \ "$repo_root/deploy/clickhouse-resource-policy.xml" \ @@ -578,10 +588,7 @@ scp_to_remote \ "$repo_root/deploy/host-nginx-8442-dingtalk.conf" \ "$repo_root/deploy/setup-incremental-registry.sh" \ "$repo_root/deploy/registry-config.yml" \ - "${ssh_target}:$remote_path/" || fail '上传运行配置失败。' - -ssh "${ssh_options[@]}" "$ssh_target" "chmod 700 '$remote_path/update-runtime-url.sh' && '$remote_path/update-runtime-url.sh' '$app_url' '$remote_path'" \ - || fail '更新服务器访问地址失败。' + "${ssh_target}:$remote_release_path/" || fail '上传运行配置失败。' if [ "$bootstrap_server" -eq 1 ]; then if ! docker image inspect "$database_image" >/dev/null 2>&1; then @@ -598,11 +605,11 @@ if [ "$bootstrap_server" -eq 1 ]; then log "首次初始化全量包:${archive_size} MiB" scp_to_remote "$local_archive" "${ssh_target}:$remote_archive" || fail '上传镜像失败。' - ssh "${ssh_options[@]}" "$ssh_target" "chmod 700 '$remote_path/'*.sh && chmod 644 '$remote_path/config-db-init.sh' && '$remote_path/activate-release.sh' '$remote_archive' '$image_tag' '$remote_path' '$deploy_sha' '$migration_flag' '$keep_images' && '$remote_path/setup-incremental-registry.sh' '$remote_path' '$remote_path/registry-config.yml'" \ + ssh "${ssh_options[@]}" "$ssh_target" "chmod 700 '$remote_release_path/'*.sh && '$remote_release_path/activate-release.sh' '$remote_archive' '$image_tag' '$remote_path' '$deploy_sha' '$migration_flag' '$keep_images' '$expected_previous_sha' '$deploy_actor' '$app_url' && '$remote_path/setup-incremental-registry.sh' '$remote_path' '$remote_path/registry-config.yml'" \ || fail '服务器激活、健康检查或增量仓库初始化失败。' else ssh "${ssh_options[@]}" -o ConnectTimeout=10 -o ConnectionAttempts=1 "$ssh_target" \ - "chmod 700 '$remote_path/'*.sh && chmod 644 '$remote_path/config-db-init.sh' && (curl -fsS -o /dev/null http://127.0.0.1:5000/v2/ || '$remote_path/setup-incremental-registry.sh' '$remote_path' '$remote_path/registry-config.yml')" \ + "chmod 700 '$remote_release_path/'*.sh && (curl -fsS -o /dev/null http://127.0.0.1:5000/v2/ || '$remote_release_path/setup-incremental-registry.sh' '$remote_path' '$remote_release_path/registry-config.yml')" \ || fail '服务器增量镜像仓库初始化失败。' docker buildx build \ @@ -685,7 +692,7 @@ else incremental_mib="$(awk -v a="$after_bytes" -v b="$before_bytes" 'BEGIN { v = (a - b) / 1048576; if (v < 0) v = 0; printf "%.2f", v }')" log "增量新增镜像层:${incremental_mib} MiB;传输耗时:${upload_seconds} 秒" - ssh "${ssh_options[@]}" "$ssh_target" "'$remote_path/activate-incremental-release.sh' '$server_registry_image' '$image_tag' '$remote_path' '$deploy_sha' '$migration_flag' '$keep_images'" \ + ssh "${ssh_options[@]}" "$ssh_target" "'$remote_release_path/activate-incremental-release.sh' '$server_registry_image' '$image_tag' '$remote_path' '$deploy_sha' '$migration_flag' '$keep_images' '$expected_previous_sha' '$deploy_actor' '$app_url'" \ || fail '服务器增量激活或健康检查失败。' fi diff --git a/scripts/deploy-from-workstation.ps1 b/scripts/deploy-from-workstation.ps1 index 7bb70aeb..bc54c179 100644 --- a/scripts/deploy-from-workstation.ps1 +++ b/scripts/deploy-from-workstation.ps1 @@ -348,6 +348,11 @@ try { throw '工作区存在未提交的受控文件改动,拒绝构建。请先提交或处理清楚。' } + Invoke-Native { git fetch origin main --quiet } '无法更新远端 main,已停止发布。' + if ($branch -eq 'main' -and (git rev-parse HEAD).Trim() -ne (git rev-parse origin/main).Trim()) { + throw '本地 main 不是最新 origin/main,请先 git pull --ff-only 再发布。' + } + $deploySha = (git rev-parse HEAD).Trim() $imageTag = $deploySha.Substring(0, 12) $fullImage = "${imageName}:$imageTag" @@ -372,7 +377,6 @@ try { } Invoke-NativeWithRetry { ssh @sshOptions $sshTarget true } '服务器登录失败或已取消,已停止部署。' -Attempts 3 if ($authSession.Directory) { $env:CAIWU_DEPLOY_PASSWORD_UPDATE = $null } - $remoteArchive = "$RemotePath/image-$imageTag.tar.gz" # deploy-auth.ps1 is dot-sourced and can clear the automatic script path # variable on PowerShell Core. Restore it before invoking the build helper. @@ -436,6 +440,11 @@ try { if ($LASTEXITCODE -ne 0) { throw '无法连接服务器。' } + if ($previousSha -and $previousSha -notmatch '^[0-9a-f]{40}$') { throw '线上版本标记格式无效,已停止发布。' } + $expectedPreviousSha = if ($previousSha) { $previousSha } else { 'none' } + $deployActor = (((git config user.name) -join '') + '@' + [Environment]::MachineName) -replace '[^A-Za-z0-9_.@:-]', '_' + $remoteReleasePath = "$RemotePath/releases/$imageTag-$([Guid]::NewGuid().ToString('N'))" + $remoteArchive = "$remoteReleasePath/image-$imageTag.tar.gz" $previousShaExists = $false if ($previousSha) { @@ -487,7 +496,7 @@ try { } Invoke-Native { - ssh @sshOptions $sshTarget "docker version >/dev/null && docker compose version >/dev/null && install -d -m 700 '$RemotePath'" + ssh @sshOptions $sshTarget "docker version >/dev/null && docker compose version >/dev/null && install -d -m 700 '$RemotePath' '$remoteReleasePath'" } '服务器 Docker 未就绪;首次发布请添加 -BootstrapServer。' Ensure-QiniuBusinessStorageRuntime @@ -498,6 +507,9 @@ try { compose.jiangling.yml ` deploy/activate-release.sh ` deploy/activate-incremental-release.sh ` + deploy/release-guard.sh ` + deploy/retired-activation.sh ` + deploy/install-release-protection.sh ` deploy/config-db-init.sh ` deploy/run-finance-batch.sh ` deploy/clickhouse-resource-policy.xml ` @@ -506,13 +518,9 @@ try { deploy/host-nginx-8442-dingtalk.conf ` deploy/setup-incremental-registry.sh ` deploy/registry-config.yml ` - "${sshTarget}:$RemotePath/" + "${sshTarget}:$remoteReleasePath/" } '上传运行配置失败。' - Invoke-Native { - ssh @sshOptions $sshTarget "chmod 700 '$RemotePath/update-runtime-url.sh' && '$RemotePath/update-runtime-url.sh' '$AppUrl' '$RemotePath'" - } '更新服务器访问地址失败。' - if ($BootstrapServer) { docker image inspect $DatabaseImage 2>$null | Out-Null if ($LASTEXITCODE -ne 0) { @@ -532,12 +540,12 @@ try { Write-Host "首次初始化全量包:$archiveSizeMB MiB" Invoke-Native { scp @scpOptions @sshOptions $localArchive "${sshTarget}:$remoteArchive" } '上传镜像失败。' Invoke-Native { - ssh @sshOptions $sshTarget "chmod 700 '$RemotePath/'*.sh && chmod 644 '$RemotePath/config-db-init.sh' && '$RemotePath/activate-release.sh' '$remoteArchive' '$imageTag' '$RemotePath' '$deploySha' '$migrationFlag' '$KeepImages' && '$RemotePath/setup-incremental-registry.sh' '$RemotePath' '$RemotePath/registry-config.yml'" + ssh @sshOptions $sshTarget "chmod 700 '$remoteReleasePath/'*.sh && '$remoteReleasePath/activate-release.sh' '$remoteArchive' '$imageTag' '$RemotePath' '$deploySha' '$migrationFlag' '$KeepImages' '$expectedPreviousSha' '$deployActor' '$AppUrl' && '$RemotePath/setup-incremental-registry.sh' '$RemotePath' '$RemotePath/registry-config.yml'" } '服务器激活、健康检查或增量仓库初始化失败。' } else { Invoke-NativeWithRetry { - ssh @sshOptions -o ConnectTimeout=10 -o ConnectionAttempts=1 $sshTarget "chmod 700 '$RemotePath/'*.sh && chmod 644 '$RemotePath/config-db-init.sh' && (curl -fsS -o /dev/null http://127.0.0.1:5000/v2/ || '$RemotePath/setup-incremental-registry.sh' '$RemotePath' '$RemotePath/registry-config.yml')" + ssh @sshOptions -o ConnectTimeout=10 -o ConnectionAttempts=1 $sshTarget "chmod 700 '$remoteReleasePath/'*.sh && (curl -fsS -o /dev/null http://127.0.0.1:5000/v2/ || '$remoteReleasePath/setup-incremental-registry.sh' '$RemotePath' '$remoteReleasePath/registry-config.yml')" } '服务器增量镜像仓库初始化失败。' # docker save 导出未压缩层,每次导入重新压缩会改变摘要,导致 Registry 无法复用。 @@ -653,7 +661,7 @@ try { Write-Host "增量新增镜像层:$incrementalMiB MiB;传输耗时:$([math]::Round($uploadTimer.Elapsed.TotalSeconds, 1)) 秒" Invoke-Native { - ssh @sshOptions $sshTarget "'$RemotePath/activate-incremental-release.sh' '$serverRegistryImage' '$imageTag' '$RemotePath' '$deploySha' '$migrationFlag' '$KeepImages'" + ssh @sshOptions $sshTarget "'$remoteReleasePath/activate-incremental-release.sh' '$serverRegistryImage' '$imageTag' '$RemotePath' '$deploySha' '$migrationFlag' '$KeepImages' '$expectedPreviousSha' '$deployActor' '$AppUrl'" } '服务器增量激活或健康检查失败。' } diff --git a/tests/scripts/release-guard.sh b/tests/scripts/release-guard.sh new file mode 100644 index 00000000..eab0d8c9 --- /dev/null +++ b/tests/scripts/release-guard.sh @@ -0,0 +1,148 @@ +#!/bin/sh +# Run in a disposable Linux container, never on a production host. +set -eu +REPO="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)" +fixture="$(mktemp -d /tmp/caiwu-release-guard.XXXXXX)" +cleanup() { + for legacy in activate-release.sh activate-incremental-release.sh; do + chattr -i "$fixture/legacy-entrypoints/$legacy" 2>/dev/null || true + done + rm -rf "$fixture" +} +trap cleanup EXIT +mkdir -p "$fixture/bin" +cat > "$fixture/bin/docker" <<'EOF' +#!/bin/sh +printf '%s\n' "$*" >> "$DOCKER_CALLS" +if [ "$1" = inspect ]; then + case "$*" in + *Config.Image*) printf 'caiwu_akaunting:old\n' ;; + *) printf '%s\n' "${FAKE_RUNNING_SHA:-}" ;; + esac + exit 0 +fi +exit 91 +EOF +chmod +x "$fixture/bin/docker" +PATH="$fixture/bin:$PATH" +export PATH +old=1111111111111111111111111111111111111111 +new=2222222222222222222222222222222222222222 +requested=3333333333333333333333333333333333333333 +passed=0 +pass() { passed=$((passed + 1)); printf 'PASS: %s\n' "$*"; } +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } + +new_runtime() { + REMOTE_DIR="$fixture/$1" + mkdir -p "$REMOTE_DIR/runtime" + : > "$REMOTE_DIR/runtime/.env" + : > "$REMOTE_DIR/runtime/config-db.env" + printf 'CLICKHOUSE_PASSWORD=fixture\nBUSINESS_CLICKHOUSE_PASSWORD=fixture\n' > "$REMOTE_DIR/runtime/clickhouse.env" + printf 'BUSINESS_STORAGE_DRIVER=qiniu\nQINIU_ACCESS_KEY=fixture\nQINIU_SECRET_KEY=fixture\n' > "$REMOTE_DIR/runtime/qiniu-storage.env" + printf '%s\n' "$old" > "$REMOTE_DIR/.deployed-sha" + printf 'previous runtime configuration\n' > "$REMOTE_DIR/compose.jiangling.yml" + DOCKER_CALLS="$REMOTE_DIR/docker-calls" + : > "$DOCKER_CALLS" + FAKE_RUNNING_SHA="$old" + export DOCKER_CALLS FAKE_RUNNING_SHA +} + +for activation in activate-release.sh activate-incremental-release.sh; do + for scenario in stale missing-protocol marker-image-mismatch unchanged; do + new_runtime "$activation-$scenario" + : > "$REMOTE_DIR/upload.tar.gz" + expected="$old" + case "$scenario" in + stale) printf '%s\n' "$new" > "$REMOTE_DIR/.deployed-sha"; FAKE_RUNNING_SHA="$new" ;; + marker-image-mismatch) FAKE_RUNNING_SHA="$new" ;; + esac + set +e + if [ "$scenario" = missing-protocol ]; then + sh "$REPO/deploy/$activation" "$REMOTE_DIR/upload.tar.gz" tag "$REMOTE_DIR" "$requested" 0 2 > "$REMOTE_DIR/output" 2>&1 + else + sh "$REPO/deploy/$activation" "$REMOTE_DIR/upload.tar.gz" tag "$REMOTE_DIR" "$requested" 0 2 "$expected" fixture > "$REMOTE_DIR/output" 2>&1 + fi + status=$? + set -e + if [ "$scenario" = unchanged ]; then + [ "$status" -eq 91 ] || { cat "$REMOTE_DIR/output"; fail 'accepted release did not reach image loading'; } + else + [ "$status" -ne 0 ] || fail "$activation accepted $scenario" + if grep -Ev '^inspect ' "$DOCKER_CALLS" | grep -q .; then + fail "$activation mutated Docker before rejecting $scenario" + fi + [ "$(cat "$REMOTE_DIR/compose.jiangling.yml")" = 'previous runtime configuration' ] || fail 'rejected release overwrote runtime configuration' + fi + if [ "$scenario" != missing-protocol ]; then + [ -s "$REMOTE_DIR/runtime/private/deployments.tsv" ] || fail 'deployment receipt missing' + if [ "$scenario" != unchanged ]; then + grep -q 'blocked' "$REMOTE_DIR/runtime/private/deployments.tsv" || fail 'rejection not recorded' + fi + fi + pass "$activation: $scenario" + done +done + +. "$REPO/deploy/release-guard.sh" +new_runtime bootstrap +rm "$REMOTE_DIR/.deployed-sha" +FAKE_RUNNING_SHA='' +EXPECTED_PREVIOUS_SHA=none +DEPLOY_SHA="$new" +caiwu_verify_release_base +pass 'first deployment with no running application' +EXPECTED_PREVIOUS_SHA='' +if caiwu_verify_release_base 2>/dev/null; then fail 'empty baseline accepted'; fi +pass 'empty baseline rejected' + +new_runtime ordered-uploaders +EXPECTED_PREVIOUS_SHA="$old" +DEPLOY_SHA="$new" +# Both uploaders checked the same old revision. The newer one activates first. +( + exec 9> "$REMOTE_DIR/.deploy.lock" + flock -x 9 + caiwu_verify_release_base + printf '%s\n' "$new" > "$REMOTE_DIR/.deployed-sha" +) +FAKE_RUNNING_SHA="$new" +DEPLOY_SHA="$requested" +if ( + exec 9> "$REMOTE_DIR/.deploy.lock" + flock -x 9 + caiwu_verify_release_base +) 2>/dev/null; then fail 'delayed uploader overwrote the newer release'; fi +[ "$(cat "$REMOTE_DIR/.deployed-sha")" = "$new" ] || fail 'production marker regressed' +pass 'delayed uploader is rejected after a newer activation' + +new_runtime staged-config +RELEASE_DIR="$fixture/staging" +RELEASE_APP_URL='' +mkdir -p "$RELEASE_DIR" +for file in compose.jiangling.yml activate-release.sh activate-incremental-release.sh release-guard.sh retired-activation.sh install-release-protection.sh config-db-init.sh run-finance-batch.sh clickhouse-resource-policy.xml update-runtime-url.sh configure-dingtalk-callback.sh host-nginx-8442-dingtalk.conf setup-incremental-registry.sh registry-config.yml; do + printf 'new staged runtime configuration\n' > "$RELEASE_DIR/$file" +done +printf '#!/bin/sh\nexit 0\n' > "$RELEASE_DIR/install-release-protection.sh" +rm "$RELEASE_DIR/registry-config.yml" +if caiwu_install_release_files 2>/dev/null; then fail 'partial upload accepted'; fi +[ "$(cat "$REMOTE_DIR/compose.jiangling.yml")" = 'previous runtime configuration' ] || fail 'partial upload replaced runtime files' +pass 'incomplete staging directory leaves runtime files untouched' +printf 'complete\n' > "$RELEASE_DIR/registry-config.yml" +caiwu_install_release_files +[ "$(cat "$REMOTE_DIR/compose.jiangling.yml")" = 'new staged runtime configuration' ] || fail 'staged configuration not installed' +pass 'complete isolated upload installs the intended runtime files' + +new_runtime legacy-entrypoints +for legacy in activate-release.sh activate-incremental-release.sh; do + printf '#!/bin/sh\nexit 0\n' > "$REMOTE_DIR/$legacy" +done +sh "$REPO/deploy/install-release-protection.sh" "$REMOTE_DIR" +for legacy in activate-release.sh activate-incremental-release.sh; do + if cp "$REPO/deploy/$legacy" "$REMOTE_DIR/$legacy" 2>/dev/null; then fail 'old uploader replaced the protected entry point'; fi + if sh "$REMOTE_DIR/$legacy" old-arguments 2>/dev/null; then fail 'old activation path accepted a release'; fi + pass "$legacy: legacy uploader and activation both rejected" +done +sh "$REPO/deploy/install-release-protection.sh" "$REMOTE_DIR" +pass 'installing entry point protection again is idempotent' +printf 'PASS: %s release protection scenarios\n' "$passed"