修复多人发布的并发版本校验与旧入口覆盖风险

This commit is contained in:
WuYongJie 2026-10-20 18:12:21 +08:00
parent 0ec6f703fe
commit 06aa0d999e
9 changed files with 345 additions and 24 deletions

View File

@ -7,6 +7,11 @@ REMOTE_DIR="${3:-/opt/caiwuxitong}"
DEPLOY_SHA="${4:?DEPLOY_SHA is required}" DEPLOY_SHA="${4:?DEPLOY_SHA is required}"
RUN_MIGRATIONS="${5:-0}" RUN_MIGRATIONS="${5:-0}"
KEEP_IMAGES="${6:-2}" KEEP_IMAGES="${6:-2}"
EXPECTED_PREVIOUS_SHA="${7:?Checked production revision is required; update the deployment scripts}"
DEPLOY_ACTOR="${8:-unknown}"
RELEASE_APP_URL="${9:-}"
RELEASE_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
. "$RELEASE_DIR/release-guard.sh"
IMAGE_NAME="caiwu_akaunting" IMAGE_NAME="caiwu_akaunting"
LOCK_FILE="$REMOTE_DIR/.deploy.lock" LOCK_FILE="$REMOTE_DIR/.deploy.lock"
REGISTRY_API="http://127.0.0.1:5000/v2/caiwuxitong/app" REGISTRY_API="http://127.0.0.1:5000/v2/caiwuxitong/app"
@ -65,6 +70,12 @@ if ! flock -n 9; then
exit 1 exit 1
fi fi
RELEASE_STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
RELEASE_RESULT=blocked
trap 'release_status=$?; caiwu_record_release_result "$release_status"' EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
# 容器替换会终止其中的同步进程;与生产刷新任务共享同一把锁。 # 容器替换会终止其中的同步进程;与生产刷新任务共享同一把锁。
exec 8>/run/caiwu-upstream-sync.lock exec 8>/run/caiwu-upstream-sync.lock
echo "Waiting for active upstream synchronization before replacing the application container" echo "Waiting for active upstream synchronization before replacing the application container"
@ -74,7 +85,8 @@ if ! flock -w 1800 8; then
fi fi
PREVIOUS_IMAGE="$(docker inspect caiwu_app --format '{{.Config.Image}}' 2>/dev/null || true)" PREVIOUS_IMAGE="$(docker inspect caiwu_app --format '{{.Config.Image}}' 2>/dev/null || true)"
PREVIOUS_SHA="$(cat "$REMOTE_DIR/.deployed-sha" 2>/dev/null || true)" caiwu_verify_release_base
RELEASE_RESULT=running
restore_previous_image() { restore_previous_image() {
case "$PREVIOUS_IMAGE" in case "$PREVIOUS_IMAGE" in
@ -96,6 +108,8 @@ if [ "$LOADED_SHA" != "$DEPLOY_SHA" ]; then
exit 1 exit 1
fi fi
caiwu_install_release_files
cd "$REMOTE_DIR" cd "$REMOTE_DIR"
export IMAGE_TAG RUN_MIGRATIONS export IMAGE_TAG RUN_MIGRATIONS

View File

@ -7,6 +7,11 @@ REMOTE_DIR="${3:-/opt/caiwuxitong}"
DEPLOY_SHA="${4:?DEPLOY_SHA is required}" DEPLOY_SHA="${4:?DEPLOY_SHA is required}"
RUN_MIGRATIONS="${5:-0}" RUN_MIGRATIONS="${5:-0}"
KEEP_IMAGES="${6:-2}" KEEP_IMAGES="${6:-2}"
EXPECTED_PREVIOUS_SHA="${7:?Checked production revision is required; update the deployment scripts}"
DEPLOY_ACTOR="${8:-unknown}"
RELEASE_APP_URL="${9:-}"
RELEASE_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
. "$RELEASE_DIR/release-guard.sh"
IMAGE_NAME="caiwu_akaunting" IMAGE_NAME="caiwu_akaunting"
LOCK_FILE="$REMOTE_DIR/.deploy.lock" LOCK_FILE="$REMOTE_DIR/.deploy.lock"
@ -14,7 +19,9 @@ cleanup() {
rm -f "$IMAGE_ARCHIVE" rm -f "$IMAGE_ARCHIVE"
} }
trap cleanup EXIT INT TERM trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
if [ "$(id -u)" -ne 0 ]; then if [ "$(id -u)" -ne 0 ]; then
echo "activate-release.sh must run as root" >&2 echo "activate-release.sh must run as root" >&2
@ -69,6 +76,10 @@ if ! flock -n 9; then
exit 1 exit 1
fi fi
RELEASE_STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
RELEASE_RESULT=blocked
trap 'release_status=$?; caiwu_record_release_result "$release_status"; cleanup' EXIT
# 容器替换会终止其中的同步进程;与生产刷新任务共享同一把锁。 # 容器替换会终止其中的同步进程;与生产刷新任务共享同一把锁。
exec 8>/run/caiwu-upstream-sync.lock exec 8>/run/caiwu-upstream-sync.lock
echo "Waiting for active upstream synchronization before replacing the application container" echo "Waiting for active upstream synchronization before replacing the application container"
@ -78,7 +89,8 @@ if ! flock -w 1800 8; then
fi fi
PREVIOUS_IMAGE="$(docker inspect caiwu_app --format '{{.Config.Image}}' 2>/dev/null || true)" PREVIOUS_IMAGE="$(docker inspect caiwu_app --format '{{.Config.Image}}' 2>/dev/null || true)"
PREVIOUS_SHA="$(cat "$REMOTE_DIR/.deployed-sha" 2>/dev/null || true)" caiwu_verify_release_base
RELEASE_RESULT=running
restore_previous_image() { restore_previous_image() {
case "$PREVIOUS_IMAGE" in case "$PREVIOUS_IMAGE" in
@ -100,6 +112,8 @@ if [ "$LOADED_SHA" != "$DEPLOY_SHA" ]; then
exit 1 exit 1
fi fi
caiwu_install_release_files
cd "$REMOTE_DIR" cd "$REMOTE_DIR"
export IMAGE_TAG RUN_MIGRATIONS export IMAGE_TAG RUN_MIGRATIONS

View File

@ -0,0 +1,43 @@
#!/bin/sh
# Caller holds the production deployment lock. Does not restart any service.
set -eu
REMOTE_DIR="${1:-/opt/caiwuxitong}"
SOURCE_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
LEGACY_STUB="$SOURCE_DIR/retired-activation.sh"
[ "$(id -u)" -eq 0 ] || { echo 'Release protection must be installed as root' >&2; exit 1; }
[ -f "$LEGACY_STUB" ] || { echo 'Missing retired activation stub' >&2; exit 1; }
is_immutable() { lsattr -d "$1" 2>/dev/null | awk '{print $1}' | grep -q i; }
needs_install=0
for legacy in activate-release.sh activate-incremental-release.sh; do
if cmp -s "$LEGACY_STUB" "$REMOTE_DIR/$legacy" && is_immutable "$REMOTE_DIR/$legacy"; then
continue
fi
if is_immutable "$REMOTE_DIR/$legacy"; then
echo "Unexpected immutable legacy entry point: $legacy; verify it before changing protection" >&2
exit 1
fi
needs_install=1
done
[ "$needs_install" -eq 1 ] || exit 0
# Test support on an owned temporary file before touching either entry point.
probe="$(mktemp "$REMOTE_DIR/.release-protection.XXXXXX")"
cleanup_probe() { chattr -i "$probe" 2>/dev/null || true; rm -f "$probe"; }
trap cleanup_probe EXIT
chattr +i "$probe" || { echo 'Filesystem cannot protect legacy entry points; no application was changed' >&2; exit 1; }
chattr -i "$probe"
backup="$REMOTE_DIR/runtime/private/release-protection/$(date -u +%Y%m%dT%H%M%SZ)-$$"
umask 077
mkdir -p "$backup"
for legacy in activate-release.sh activate-incremental-release.sh; do
if cmp -s "$LEGACY_STUB" "$REMOTE_DIR/$legacy" && is_immutable "$REMOTE_DIR/$legacy"; then
continue
fi
[ ! -f "$REMOTE_DIR/$legacy" ] || cp -p "$REMOTE_DIR/$legacy" "$backup/$legacy"
install -m 700 "$LEGACY_STUB" "$REMOTE_DIR/.$legacy.next"
mv "$REMOTE_DIR/.$legacy.next" "$REMOTE_DIR/$legacy"
chattr +i "$REMOTE_DIR/$legacy"
cmp -s "$LEGACY_STUB" "$REMOTE_DIR/$legacy" && is_immutable "$REMOTE_DIR/$legacy"
done
echo 'Legacy activation paths protected; current application and databases were not restarted'

72
deploy/release-guard.sh Normal file
View File

@ -0,0 +1,72 @@
#!/bin/sh
# Sourced by both activation paths after the deployment and synchronization locks.
caiwu_verify_release_base() {
case "$EXPECTED_PREVIOUS_SHA" in
none) ;;
*[!0-9a-f]*|'') echo 'Missing or invalid checked production revision; update the deployment scripts' >&2; return 1 ;;
*) [ "${#EXPECTED_PREVIOUS_SHA}" -eq 40 ] || return 1 ;;
esac
PREVIOUS_SHA="$(cat "$REMOTE_DIR/.deployed-sha" 2>/dev/null || true)"
[ -n "$PREVIOUS_SHA" ] || PREVIOUS_SHA=none
if [ "$PREVIOUS_SHA" != none ] && ! printf '%s\n' "$PREVIOUS_SHA" | grep -Eq '^[0-9a-f]{40}$'; then
echo 'Invalid production revision marker; refusing activation' >&2
return 1
fi
if [ "$PREVIOUS_SHA" != "$EXPECTED_PREVIOUS_SHA" ]; then
echo "Production changed while this release was being prepared: checked=$EXPECTED_PREVIOUS_SHA current=$PREVIOUS_SHA requested=$DEPLOY_SHA. Fetch latest main and retry; the running application was not changed." >&2
return 1
fi
# A failed/interrupted older activation must not leave the marker ahead of the app.
running_sha="$(docker inspect caiwu_app --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' 2>/dev/null || true)"
if [ "$PREVIOUS_SHA" != none ] && [ "$running_sha" != "$PREVIOUS_SHA" ]; then
echo "Running image and production marker disagree; refusing activation: marker=$PREVIOUS_SHA image=$running_sha" >&2
return 1
fi
if [ "$PREVIOUS_SHA" = none ] && [ -n "$running_sha" ]; then
echo 'An application is already running without a production marker; verify its revision before deploying' >&2
return 1
fi
}
caiwu_install_release_files() {
# Each uploader has its own directory. Shared runtime files change only inside
# the activation lock and after the production revision has been rechecked.
[ "$RELEASE_DIR" != "$REMOTE_DIR" ] || return 0
release_files='compose.jiangling.yml activate-release.sh activate-incremental-release.sh release-guard.sh retired-activation.sh install-release-protection.sh config-db-init.sh run-finance-batch.sh clickhouse-resource-policy.xml update-runtime-url.sh configure-dingtalk-callback.sh host-nginx-8442-dingtalk.conf setup-incremental-registry.sh registry-config.yml'
for release_file in $release_files; do
[ -f "$RELEASE_DIR/$release_file" ] || { echo "Incomplete staged release: $release_file" >&2; return 1; }
done
sh "$RELEASE_DIR/install-release-protection.sh" "$REMOTE_DIR" || return 1
for release_file in $release_files; do
case "$release_file" in activate-release.sh|activate-incremental-release.sh) continue ;; esac
case "$release_file" in *.sh) release_mode=700 ;; *) release_mode=644 ;; esac
[ "$release_file" != config-db-init.sh ] || release_mode=644
install -m "$release_mode" "$RELEASE_DIR/$release_file" "$REMOTE_DIR/.$release_file.next"
mv "$REMOTE_DIR/.$release_file.next" "$REMOTE_DIR/$release_file"
done
if [ -n "$RELEASE_APP_URL" ]; then
"$REMOTE_DIR/update-runtime-url.sh" "$RELEASE_APP_URL" "$REMOTE_DIR"
fi
printf '%s\n' "$DEPLOY_SHA" > "$REMOTE_DIR/.release-controls-sha.next"
mv "$REMOTE_DIR/.release-controls-sha.next" "$REMOTE_DIR/.release-controls-sha"
}
caiwu_record_release_result() {
release_exit="$1"
[ "$RELEASE_RESULT" != running ] || RELEASE_RESULT=failed
[ "$release_exit" -ne 0 ] || RELEASE_RESULT=success
# Client identity is declared by the workstation; the SSH source is recorded
# separately. Neither field is inferred from the commit author.
release_actor="$(printf '%s' "$DEPLOY_ACTOR" | tr -cd 'A-Za-z0-9_.@:-' | cut -c1-160)"
release_peer="$(printf '%s' "${SSH_CONNECTION:-local}" | tr -cd 'A-Za-z0-9 .:-' | cut -c1-160)"
(
umask 077
mkdir -p "$REMOTE_DIR/runtime/private"
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
"$RELEASE_STARTED_AT" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$release_actor" "$release_peer" \
"${PREVIOUS_SHA:-unknown}" "$DEPLOY_SHA" "$RELEASE_RESULT" "$release_exit" "$EXPECTED_PREVIOUS_SHA" \
>> "$REMOTE_DIR/runtime/private/deployments.tsv"
) || echo 'Could not write the private deployment receipt' >&2
}

View File

@ -0,0 +1,4 @@
#!/bin/sh
# This legacy shared path is intentionally retired and immutable on production.
echo 'Legacy activation is disabled. Fetch latest main and use the workstation deployment script with an isolated release directory and checked production revision. The running application was not changed.' >&2
exit 1

View File

@ -129,11 +129,22 @@ bash scripts/deploy-from-mac.sh
8. 验证唯一私有七牛桶的写入、读取和删除,并把旧本机业务上传幂等复制到七牛;不删除本机原件; 8. 验证唯一私有七牛桶的写入、读取和删除,并把旧本机业务上传幂等复制到七牛;不删除本机原件;
9. 检查服务器本机后端页面; 9. 检查服务器本机后端页面;
10. 任一存储验证或页面验证失败时恢复上一个镜像; 10. 任一存储验证或页面验证失败时恢复上一个镜像;
11. 使用服务器互斥锁阻止两个人同时切换版本; 11. 每次上传使用独立暂存目录;服务器取得发布锁和同步锁后,再核对线上 SHA 与发布机已检查的 SHA,确认未变化才切换配置和应用;
12. 删除本地临时镜像包并关闭 SSH 隧道; 12. 删除本地临时镜像包并关闭 SSH 隧道;
13. 开发机、服务器 Docker 和服务器 Registry 均仅保留当前和上一个本项目版本; 13. 开发机、服务器 Docker 和服务器 Registry 均仅保留当前和上一个本项目版本;
14. 清理 7 天前的本项目构建缓存,并将总量压到最多 3GB。 14. 清理 7 天前的本项目构建缓存,并将总量压到最多 3GB。
### 5.0 多人发布保护
- 远端 `main` 禁止直接推送和强推,任务分支通过 PR 合并;不额外要求人工审批或新增 CI 门槛。
- 发布机先刷新远端 `main`,正式发布必须与最新主线一致,并检查本次版本包含线上提交。
- 上传期间若别人已发布,服务器在锁内拒绝本次激活,运行中的应用保持原版本。先 `git fetch`、快进到最新 `main` 并重新发布,不用回滚开关绕过并发校验。
- Windows 和 macOS 发布机都必须使用最新脚本。共享根目录的两个旧激活入口已停用并设置 Linux immutable 属性,旧发布机无法用上传覆盖它们;新脚本从每次独立暂存目录执行激活,缺少已检查线上版本参数也会被拒绝。
- 运行配置按每次发布独立暂存,核对通过后才在锁内安装;上传文件不会提前覆盖其他发布使用的配置。
- 服务器私有 `runtime/private/deployments.tsv` 记录起止 UTC 时间、发布机声明的身份、SSH 来源、前后 SHA、结果、退出码和已检查的 SHA。声明身份及提交作者不能单独证明具体操作者;记录不包含凭据或业务数据。
- 安装旧入口保护前自动备份到服务器私有 `runtime/private/release-protection/`。普通发布不解除 immutable 属性;维护这些停用入口时先核对备份和目的,再由维护人员使用 `chattr -i` 处理指定文件。应用镜像回滚不需要启用旧入口。
- 仅维护发布控制脚本时,可在发布锁内备份并安装已合并主线的控制脚本,记录独立的 `.release-controls-sha`,不重建应用容器、不修改表示应用镜像版本的 `.deployed-sha`。
### 5.1 安全软件拦截 regctl 时的降级 ### 5.1 安全软件拦截 regctl 时的降级
第 6 步默认用项目专用 `regctl`(未签名)把 BuildKit 原生 OCI 分层包推入服务器本机 Registry。少数开发机的 360 安全卫士、金山毒霸等安全软件会拦截未签名程序,表现为 `regctl-v0.11.5.exe` 启动即报“拒绝访问”。 第 6 步默认用项目专用 `regctl`(未签名)把 BuildKit 原生 OCI 分层包推入服务器本机 Registry。少数开发机的 360 安全卫士、金山毒霸等安全软件会拦截未签名程序,表现为 `regctl-v0.11.5.exe` 启动即报“拒绝访问”。

View File

@ -416,7 +416,8 @@ if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
fail '工作区存在未提交的受控文件改动,拒绝构建。请先提交或处理清楚。' fail '工作区存在未提交的受控文件改动,拒绝构建。请先提交或处理清楚。'
fi fi
if [ "$branch" = 'main' ] && git rev-parse --verify --quiet origin/main >/dev/null 2>&1; then git fetch origin main --quiet || fail '无法更新远端 main,已停止发布。'
if [ "$branch" = 'main' ]; then
if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/main)" ]; then if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/main)" ]; then
fail '本地 main 与 origin/main 不一致,请先执行 git pull --ff-only 再发布。' fail '本地 main 与 origin/main 不一致,请先执行 git pull --ff-only 再发布。'
fi fi
@ -457,7 +458,6 @@ ssh_control_path="/tmp/caiwu-ssh-$$-$RANDOM.sock"
ssh_options+=(-o ControlMaster=auto -o ControlPersist=600 -o "ControlPath=$ssh_control_path") ssh_options+=(-o ControlMaster=auto -o ControlPersist=600 -o "ControlPath=$ssh_control_path")
tunnel_ssh_options=("${ssh_options[@]}" -o ControlPath=none) tunnel_ssh_options=("${ssh_options[@]}" -o ControlPath=none)
remote_archive="$remote_path/image-$image_tag.tar.gz"
temp_root="$(mktemp -d "${TMPDIR:-/tmp}/caiwu-deploy-XXXXXX")" temp_root="$(mktemp -d "${TMPDIR:-/tmp}/caiwu-deploy-XXXXXX")"
local_tar="$temp_root/image.tar" local_tar="$temp_root/image.tar"
local_archive="$temp_root/image.tar.gz" local_archive="$temp_root/image.tar.gz"
@ -512,6 +512,13 @@ for pair in \
done done
previous_sha="$(ssh "${ssh_options[@]}" "$ssh_target" "cat '$remote_path/.deployed-sha' 2>/dev/null || true" | tr -d '[:space:]')" previous_sha="$(ssh "${ssh_options[@]}" "$ssh_target" "cat '$remote_path/.deployed-sha' 2>/dev/null || true" | tr -d '[:space:]')"
if [ -n "$previous_sha" ] && ! printf '%s\n' "$previous_sha" | grep -Eq '^[0-9a-f]{40}$'; then
fail '线上版本标记格式无效,已停止发布。'
fi
expected_previous_sha="${previous_sha:-none}"
deploy_actor="$(printf '%s@%s' "$(git config user.name)" "$(hostname)" | tr -c 'A-Za-z0-9_.@:-' '_')"
remote_release_path="$remote_path/releases/$image_tag-$(uuidgen | tr 'A-Z' 'a-z')"
remote_archive="$remote_release_path/image-$image_tag.tar.gz"
if [ -n "$previous_sha" ]; then if [ -n "$previous_sha" ]; then
if git cat-file -e "${previous_sha}^{commit}" 2>/dev/null; then if git cat-file -e "${previous_sha}^{commit}" 2>/dev/null; then
@ -556,7 +563,7 @@ if [ "$bootstrap_server" -eq 1 ]; then
bootstrap_remote='' bootstrap_remote=''
fi fi
ssh "${ssh_options[@]}" "$ssh_target" "docker version >/dev/null && docker compose version >/dev/null && install -d -m 700 '$remote_path'" \ ssh "${ssh_options[@]}" "$ssh_target" "docker version >/dev/null && docker compose version >/dev/null && install -d -m 700 '$remote_path' '$remote_release_path'" \
|| fail '服务器 Docker 未就绪;首次发布请添加 --bootstrap-server。' || fail '服务器 Docker 未就绪;首次发布请添加 --bootstrap-server。'
ensure_qiniu_storage_runtime ensure_qiniu_storage_runtime
@ -570,6 +577,9 @@ scp_to_remote \
"$repo_root/compose.jiangling.yml" \ "$repo_root/compose.jiangling.yml" \
"$repo_root/deploy/activate-release.sh" \ "$repo_root/deploy/activate-release.sh" \
"$repo_root/deploy/activate-incremental-release.sh" \ "$repo_root/deploy/activate-incremental-release.sh" \
"$repo_root/deploy/release-guard.sh" \
"$repo_root/deploy/retired-activation.sh" \
"$repo_root/deploy/install-release-protection.sh" \
"$repo_root/deploy/config-db-init.sh" \ "$repo_root/deploy/config-db-init.sh" \
"$repo_root/deploy/run-finance-batch.sh" \ "$repo_root/deploy/run-finance-batch.sh" \
"$repo_root/deploy/clickhouse-resource-policy.xml" \ "$repo_root/deploy/clickhouse-resource-policy.xml" \
@ -578,10 +588,7 @@ scp_to_remote \
"$repo_root/deploy/host-nginx-8442-dingtalk.conf" \ "$repo_root/deploy/host-nginx-8442-dingtalk.conf" \
"$repo_root/deploy/setup-incremental-registry.sh" \ "$repo_root/deploy/setup-incremental-registry.sh" \
"$repo_root/deploy/registry-config.yml" \ "$repo_root/deploy/registry-config.yml" \
"${ssh_target}:$remote_path/" || fail '上传运行配置失败。' "${ssh_target}:$remote_release_path/" || fail '上传运行配置失败。'
ssh "${ssh_options[@]}" "$ssh_target" "chmod 700 '$remote_path/update-runtime-url.sh' && '$remote_path/update-runtime-url.sh' '$app_url' '$remote_path'" \
|| fail '更新服务器访问地址失败。'
if [ "$bootstrap_server" -eq 1 ]; then if [ "$bootstrap_server" -eq 1 ]; then
if ! docker image inspect "$database_image" >/dev/null 2>&1; then if ! docker image inspect "$database_image" >/dev/null 2>&1; then
@ -598,11 +605,11 @@ if [ "$bootstrap_server" -eq 1 ]; then
log "首次初始化全量包:${archive_size} MiB" log "首次初始化全量包:${archive_size} MiB"
scp_to_remote "$local_archive" "${ssh_target}:$remote_archive" || fail '上传镜像失败。' scp_to_remote "$local_archive" "${ssh_target}:$remote_archive" || fail '上传镜像失败。'
ssh "${ssh_options[@]}" "$ssh_target" "chmod 700 '$remote_path/'*.sh && chmod 644 '$remote_path/config-db-init.sh' && '$remote_path/activate-release.sh' '$remote_archive' '$image_tag' '$remote_path' '$deploy_sha' '$migration_flag' '$keep_images' && '$remote_path/setup-incremental-registry.sh' '$remote_path' '$remote_path/registry-config.yml'" \ ssh "${ssh_options[@]}" "$ssh_target" "chmod 700 '$remote_release_path/'*.sh && '$remote_release_path/activate-release.sh' '$remote_archive' '$image_tag' '$remote_path' '$deploy_sha' '$migration_flag' '$keep_images' '$expected_previous_sha' '$deploy_actor' '$app_url' && '$remote_path/setup-incremental-registry.sh' '$remote_path' '$remote_path/registry-config.yml'" \
|| fail '服务器激活、健康检查或增量仓库初始化失败。' || fail '服务器激活、健康检查或增量仓库初始化失败。'
else else
ssh "${ssh_options[@]}" -o ConnectTimeout=10 -o ConnectionAttempts=1 "$ssh_target" \ ssh "${ssh_options[@]}" -o ConnectTimeout=10 -o ConnectionAttempts=1 "$ssh_target" \
"chmod 700 '$remote_path/'*.sh && chmod 644 '$remote_path/config-db-init.sh' && (curl -fsS -o /dev/null http://127.0.0.1:5000/v2/ || '$remote_path/setup-incremental-registry.sh' '$remote_path' '$remote_path/registry-config.yml')" \ "chmod 700 '$remote_release_path/'*.sh && (curl -fsS -o /dev/null http://127.0.0.1:5000/v2/ || '$remote_release_path/setup-incremental-registry.sh' '$remote_path' '$remote_release_path/registry-config.yml')" \
|| fail '服务器增量镜像仓库初始化失败。' || fail '服务器增量镜像仓库初始化失败。'
docker buildx build \ docker buildx build \
@ -685,7 +692,7 @@ else
incremental_mib="$(awk -v a="$after_bytes" -v b="$before_bytes" 'BEGIN { v = (a - b) / 1048576; if (v < 0) v = 0; printf "%.2f", v }')" incremental_mib="$(awk -v a="$after_bytes" -v b="$before_bytes" 'BEGIN { v = (a - b) / 1048576; if (v < 0) v = 0; printf "%.2f", v }')"
log "增量新增镜像层:${incremental_mib} MiB;传输耗时:${upload_seconds} 秒" log "增量新增镜像层:${incremental_mib} MiB;传输耗时:${upload_seconds} 秒"
ssh "${ssh_options[@]}" "$ssh_target" "'$remote_path/activate-incremental-release.sh' '$server_registry_image' '$image_tag' '$remote_path' '$deploy_sha' '$migration_flag' '$keep_images'" \ ssh "${ssh_options[@]}" "$ssh_target" "'$remote_release_path/activate-incremental-release.sh' '$server_registry_image' '$image_tag' '$remote_path' '$deploy_sha' '$migration_flag' '$keep_images' '$expected_previous_sha' '$deploy_actor' '$app_url'" \
|| fail '服务器增量激活或健康检查失败。' || fail '服务器增量激活或健康检查失败。'
fi fi

View File

@ -348,6 +348,11 @@ try {
throw '工作区存在未提交的受控文件改动,拒绝构建。请先提交或处理清楚。' throw '工作区存在未提交的受控文件改动,拒绝构建。请先提交或处理清楚。'
} }
Invoke-Native { git fetch origin main --quiet } '无法更新远端 main,已停止发布。'
if ($branch -eq 'main' -and (git rev-parse HEAD).Trim() -ne (git rev-parse origin/main).Trim()) {
throw '本地 main 不是最新 origin/main,请先 git pull --ff-only 再发布。'
}
$deploySha = (git rev-parse HEAD).Trim() $deploySha = (git rev-parse HEAD).Trim()
$imageTag = $deploySha.Substring(0, 12) $imageTag = $deploySha.Substring(0, 12)
$fullImage = "${imageName}:$imageTag" $fullImage = "${imageName}:$imageTag"
@ -372,7 +377,6 @@ try {
} }
Invoke-NativeWithRetry { ssh @sshOptions $sshTarget true } '服务器登录失败或已取消,已停止部署。' -Attempts 3 Invoke-NativeWithRetry { ssh @sshOptions $sshTarget true } '服务器登录失败或已取消,已停止部署。' -Attempts 3
if ($authSession.Directory) { $env:CAIWU_DEPLOY_PASSWORD_UPDATE = $null } if ($authSession.Directory) { $env:CAIWU_DEPLOY_PASSWORD_UPDATE = $null }
$remoteArchive = "$RemotePath/image-$imageTag.tar.gz"
# deploy-auth.ps1 is dot-sourced and can clear the automatic script path # deploy-auth.ps1 is dot-sourced and can clear the automatic script path
# variable on PowerShell Core. Restore it before invoking the build helper. # variable on PowerShell Core. Restore it before invoking the build helper.
@ -436,6 +440,11 @@ try {
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
throw '无法连接服务器。' throw '无法连接服务器。'
} }
if ($previousSha -and $previousSha -notmatch '^[0-9a-f]{40}$') { throw '线上版本标记格式无效,已停止发布。' }
$expectedPreviousSha = if ($previousSha) { $previousSha } else { 'none' }
$deployActor = (((git config user.name) -join '') + '@' + [Environment]::MachineName) -replace '[^A-Za-z0-9_.@:-]', '_'
$remoteReleasePath = "$RemotePath/releases/$imageTag-$([Guid]::NewGuid().ToString('N'))"
$remoteArchive = "$remoteReleasePath/image-$imageTag.tar.gz"
$previousShaExists = $false $previousShaExists = $false
if ($previousSha) { if ($previousSha) {
@ -487,7 +496,7 @@ try {
} }
Invoke-Native { Invoke-Native {
ssh @sshOptions $sshTarget "docker version >/dev/null && docker compose version >/dev/null && install -d -m 700 '$RemotePath'" ssh @sshOptions $sshTarget "docker version >/dev/null && docker compose version >/dev/null && install -d -m 700 '$RemotePath' '$remoteReleasePath'"
} '服务器 Docker 未就绪;首次发布请添加 -BootstrapServer。' } '服务器 Docker 未就绪;首次发布请添加 -BootstrapServer。'
Ensure-QiniuBusinessStorageRuntime Ensure-QiniuBusinessStorageRuntime
@ -498,6 +507,9 @@ try {
compose.jiangling.yml ` compose.jiangling.yml `
deploy/activate-release.sh ` deploy/activate-release.sh `
deploy/activate-incremental-release.sh ` deploy/activate-incremental-release.sh `
deploy/release-guard.sh `
deploy/retired-activation.sh `
deploy/install-release-protection.sh `
deploy/config-db-init.sh ` deploy/config-db-init.sh `
deploy/run-finance-batch.sh ` deploy/run-finance-batch.sh `
deploy/clickhouse-resource-policy.xml ` deploy/clickhouse-resource-policy.xml `
@ -506,13 +518,9 @@ try {
deploy/host-nginx-8442-dingtalk.conf ` deploy/host-nginx-8442-dingtalk.conf `
deploy/setup-incremental-registry.sh ` deploy/setup-incremental-registry.sh `
deploy/registry-config.yml ` deploy/registry-config.yml `
"${sshTarget}:$RemotePath/" "${sshTarget}:$remoteReleasePath/"
} '上传运行配置失败。' } '上传运行配置失败。'
Invoke-Native {
ssh @sshOptions $sshTarget "chmod 700 '$RemotePath/update-runtime-url.sh' && '$RemotePath/update-runtime-url.sh' '$AppUrl' '$RemotePath'"
} '更新服务器访问地址失败。'
if ($BootstrapServer) { if ($BootstrapServer) {
docker image inspect $DatabaseImage 2>$null | Out-Null docker image inspect $DatabaseImage 2>$null | Out-Null
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
@ -532,12 +540,12 @@ try {
Write-Host "首次初始化全量包:$archiveSizeMB MiB" Write-Host "首次初始化全量包:$archiveSizeMB MiB"
Invoke-Native { scp @scpOptions @sshOptions $localArchive "${sshTarget}:$remoteArchive" } '上传镜像失败。' Invoke-Native { scp @scpOptions @sshOptions $localArchive "${sshTarget}:$remoteArchive" } '上传镜像失败。'
Invoke-Native { Invoke-Native {
ssh @sshOptions $sshTarget "chmod 700 '$RemotePath/'*.sh && chmod 644 '$RemotePath/config-db-init.sh' && '$RemotePath/activate-release.sh' '$remoteArchive' '$imageTag' '$RemotePath' '$deploySha' '$migrationFlag' '$KeepImages' && '$RemotePath/setup-incremental-registry.sh' '$RemotePath' '$RemotePath/registry-config.yml'" ssh @sshOptions $sshTarget "chmod 700 '$remoteReleasePath/'*.sh && '$remoteReleasePath/activate-release.sh' '$remoteArchive' '$imageTag' '$RemotePath' '$deploySha' '$migrationFlag' '$KeepImages' '$expectedPreviousSha' '$deployActor' '$AppUrl' && '$RemotePath/setup-incremental-registry.sh' '$RemotePath' '$RemotePath/registry-config.yml'"
} '服务器激活、健康检查或增量仓库初始化失败。' } '服务器激活、健康检查或增量仓库初始化失败。'
} }
else { else {
Invoke-NativeWithRetry { Invoke-NativeWithRetry {
ssh @sshOptions -o ConnectTimeout=10 -o ConnectionAttempts=1 $sshTarget "chmod 700 '$RemotePath/'*.sh && chmod 644 '$RemotePath/config-db-init.sh' && (curl -fsS -o /dev/null http://127.0.0.1:5000/v2/ || '$RemotePath/setup-incremental-registry.sh' '$RemotePath' '$RemotePath/registry-config.yml')" ssh @sshOptions -o ConnectTimeout=10 -o ConnectionAttempts=1 $sshTarget "chmod 700 '$remoteReleasePath/'*.sh && (curl -fsS -o /dev/null http://127.0.0.1:5000/v2/ || '$remoteReleasePath/setup-incremental-registry.sh' '$RemotePath' '$remoteReleasePath/registry-config.yml')"
} '服务器增量镜像仓库初始化失败。' } '服务器增量镜像仓库初始化失败。'
# docker save 导出未压缩层,每次导入重新压缩会改变摘要,导致 Registry 无法复用。 # docker save 导出未压缩层,每次导入重新压缩会改变摘要,导致 Registry 无法复用。
@ -653,7 +661,7 @@ try {
Write-Host "增量新增镜像层:$incrementalMiB MiB;传输耗时:$([math]::Round($uploadTimer.Elapsed.TotalSeconds, 1)) 秒" Write-Host "增量新增镜像层:$incrementalMiB MiB;传输耗时:$([math]::Round($uploadTimer.Elapsed.TotalSeconds, 1)) 秒"
Invoke-Native { Invoke-Native {
ssh @sshOptions $sshTarget "'$RemotePath/activate-incremental-release.sh' '$serverRegistryImage' '$imageTag' '$RemotePath' '$deploySha' '$migrationFlag' '$KeepImages'" ssh @sshOptions $sshTarget "'$remoteReleasePath/activate-incremental-release.sh' '$serverRegistryImage' '$imageTag' '$RemotePath' '$deploySha' '$migrationFlag' '$KeepImages' '$expectedPreviousSha' '$deployActor' '$AppUrl'"
} '服务器增量激活或健康检查失败。' } '服务器增量激活或健康检查失败。'
} }

View File

@ -0,0 +1,148 @@
#!/bin/sh
# Run in a disposable Linux container, never on a production host.
set -eu
REPO="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)"
fixture="$(mktemp -d /tmp/caiwu-release-guard.XXXXXX)"
cleanup() {
for legacy in activate-release.sh activate-incremental-release.sh; do
chattr -i "$fixture/legacy-entrypoints/$legacy" 2>/dev/null || true
done
rm -rf "$fixture"
}
trap cleanup EXIT
mkdir -p "$fixture/bin"
cat > "$fixture/bin/docker" <<'EOF'
#!/bin/sh
printf '%s\n' "$*" >> "$DOCKER_CALLS"
if [ "$1" = inspect ]; then
case "$*" in
*Config.Image*) printf 'caiwu_akaunting:old\n' ;;
*) printf '%s\n' "${FAKE_RUNNING_SHA:-}" ;;
esac
exit 0
fi
exit 91
EOF
chmod +x "$fixture/bin/docker"
PATH="$fixture/bin:$PATH"
export PATH
old=1111111111111111111111111111111111111111
new=2222222222222222222222222222222222222222
requested=3333333333333333333333333333333333333333
passed=0
pass() { passed=$((passed + 1)); printf 'PASS: %s\n' "$*"; }
fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }
new_runtime() {
REMOTE_DIR="$fixture/$1"
mkdir -p "$REMOTE_DIR/runtime"
: > "$REMOTE_DIR/runtime/.env"
: > "$REMOTE_DIR/runtime/config-db.env"
printf 'CLICKHOUSE_PASSWORD=fixture\nBUSINESS_CLICKHOUSE_PASSWORD=fixture\n' > "$REMOTE_DIR/runtime/clickhouse.env"
printf 'BUSINESS_STORAGE_DRIVER=qiniu\nQINIU_ACCESS_KEY=fixture\nQINIU_SECRET_KEY=fixture\n' > "$REMOTE_DIR/runtime/qiniu-storage.env"
printf '%s\n' "$old" > "$REMOTE_DIR/.deployed-sha"
printf 'previous runtime configuration\n' > "$REMOTE_DIR/compose.jiangling.yml"
DOCKER_CALLS="$REMOTE_DIR/docker-calls"
: > "$DOCKER_CALLS"
FAKE_RUNNING_SHA="$old"
export DOCKER_CALLS FAKE_RUNNING_SHA
}
for activation in activate-release.sh activate-incremental-release.sh; do
for scenario in stale missing-protocol marker-image-mismatch unchanged; do
new_runtime "$activation-$scenario"
: > "$REMOTE_DIR/upload.tar.gz"
expected="$old"
case "$scenario" in
stale) printf '%s\n' "$new" > "$REMOTE_DIR/.deployed-sha"; FAKE_RUNNING_SHA="$new" ;;
marker-image-mismatch) FAKE_RUNNING_SHA="$new" ;;
esac
set +e
if [ "$scenario" = missing-protocol ]; then
sh "$REPO/deploy/$activation" "$REMOTE_DIR/upload.tar.gz" tag "$REMOTE_DIR" "$requested" 0 2 > "$REMOTE_DIR/output" 2>&1
else
sh "$REPO/deploy/$activation" "$REMOTE_DIR/upload.tar.gz" tag "$REMOTE_DIR" "$requested" 0 2 "$expected" fixture > "$REMOTE_DIR/output" 2>&1
fi
status=$?
set -e
if [ "$scenario" = unchanged ]; then
[ "$status" -eq 91 ] || { cat "$REMOTE_DIR/output"; fail 'accepted release did not reach image loading'; }
else
[ "$status" -ne 0 ] || fail "$activation accepted $scenario"
if grep -Ev '^inspect ' "$DOCKER_CALLS" | grep -q .; then
fail "$activation mutated Docker before rejecting $scenario"
fi
[ "$(cat "$REMOTE_DIR/compose.jiangling.yml")" = 'previous runtime configuration' ] || fail 'rejected release overwrote runtime configuration'
fi
if [ "$scenario" != missing-protocol ]; then
[ -s "$REMOTE_DIR/runtime/private/deployments.tsv" ] || fail 'deployment receipt missing'
if [ "$scenario" != unchanged ]; then
grep -q 'blocked' "$REMOTE_DIR/runtime/private/deployments.tsv" || fail 'rejection not recorded'
fi
fi
pass "$activation: $scenario"
done
done
. "$REPO/deploy/release-guard.sh"
new_runtime bootstrap
rm "$REMOTE_DIR/.deployed-sha"
FAKE_RUNNING_SHA=''
EXPECTED_PREVIOUS_SHA=none
DEPLOY_SHA="$new"
caiwu_verify_release_base
pass 'first deployment with no running application'
EXPECTED_PREVIOUS_SHA=''
if caiwu_verify_release_base 2>/dev/null; then fail 'empty baseline accepted'; fi
pass 'empty baseline rejected'
new_runtime ordered-uploaders
EXPECTED_PREVIOUS_SHA="$old"
DEPLOY_SHA="$new"
# Both uploaders checked the same old revision. The newer one activates first.
(
exec 9> "$REMOTE_DIR/.deploy.lock"
flock -x 9
caiwu_verify_release_base
printf '%s\n' "$new" > "$REMOTE_DIR/.deployed-sha"
)
FAKE_RUNNING_SHA="$new"
DEPLOY_SHA="$requested"
if (
exec 9> "$REMOTE_DIR/.deploy.lock"
flock -x 9
caiwu_verify_release_base
) 2>/dev/null; then fail 'delayed uploader overwrote the newer release'; fi
[ "$(cat "$REMOTE_DIR/.deployed-sha")" = "$new" ] || fail 'production marker regressed'
pass 'delayed uploader is rejected after a newer activation'
new_runtime staged-config
RELEASE_DIR="$fixture/staging"
RELEASE_APP_URL=''
mkdir -p "$RELEASE_DIR"
for file in compose.jiangling.yml activate-release.sh activate-incremental-release.sh release-guard.sh retired-activation.sh install-release-protection.sh config-db-init.sh run-finance-batch.sh clickhouse-resource-policy.xml update-runtime-url.sh configure-dingtalk-callback.sh host-nginx-8442-dingtalk.conf setup-incremental-registry.sh registry-config.yml; do
printf 'new staged runtime configuration\n' > "$RELEASE_DIR/$file"
done
printf '#!/bin/sh\nexit 0\n' > "$RELEASE_DIR/install-release-protection.sh"
rm "$RELEASE_DIR/registry-config.yml"
if caiwu_install_release_files 2>/dev/null; then fail 'partial upload accepted'; fi
[ "$(cat "$REMOTE_DIR/compose.jiangling.yml")" = 'previous runtime configuration' ] || fail 'partial upload replaced runtime files'
pass 'incomplete staging directory leaves runtime files untouched'
printf 'complete\n' > "$RELEASE_DIR/registry-config.yml"
caiwu_install_release_files
[ "$(cat "$REMOTE_DIR/compose.jiangling.yml")" = 'new staged runtime configuration' ] || fail 'staged configuration not installed'
pass 'complete isolated upload installs the intended runtime files'
new_runtime legacy-entrypoints
for legacy in activate-release.sh activate-incremental-release.sh; do
printf '#!/bin/sh\nexit 0\n' > "$REMOTE_DIR/$legacy"
done
sh "$REPO/deploy/install-release-protection.sh" "$REMOTE_DIR"
for legacy in activate-release.sh activate-incremental-release.sh; do
if cp "$REPO/deploy/$legacy" "$REMOTE_DIR/$legacy" 2>/dev/null; then fail 'old uploader replaced the protected entry point'; fi
if sh "$REMOTE_DIR/$legacy" old-arguments 2>/dev/null; then fail 'old activation path accepted a release'; fi
pass "$legacy: legacy uploader and activation both rejected"
done
sh "$REPO/deploy/install-release-protection.sh" "$REMOTE_DIR"
pass 'installing entry point protection again is idempotent'
printf 'PASS: %s release protection scenarios\n' "$passed"