修复多人发布并发覆盖及旧客户端绕过保护 (#409)
This commit is contained in:
parent
0ec6f703fe
commit
d68548646d
@ -7,6 +7,11 @@ REMOTE_DIR="${3:-/opt/caiwuxitong}"
|
||||
DEPLOY_SHA="${4:?DEPLOY_SHA is required}"
|
||||
RUN_MIGRATIONS="${5:-0}"
|
||||
KEEP_IMAGES="${6:-2}"
|
||||
EXPECTED_PREVIOUS_SHA="${7:?Checked production revision is required; update the deployment scripts}"
|
||||
DEPLOY_ACTOR="${8:-unknown}"
|
||||
RELEASE_APP_URL="${9:-}"
|
||||
RELEASE_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
|
||||
. "$RELEASE_DIR/release-guard.sh"
|
||||
IMAGE_NAME="caiwu_akaunting"
|
||||
LOCK_FILE="$REMOTE_DIR/.deploy.lock"
|
||||
REGISTRY_API="http://127.0.0.1:5000/v2/caiwuxitong/app"
|
||||
@ -65,6 +70,12 @@ if ! flock -n 9; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RELEASE_STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
RELEASE_RESULT=blocked
|
||||
trap 'release_status=$?; caiwu_record_release_result "$release_status"' EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
# 容器替换会终止其中的同步进程;与生产刷新任务共享同一把锁。
|
||||
exec 8>/run/caiwu-upstream-sync.lock
|
||||
echo "Waiting for active upstream synchronization before replacing the application container"
|
||||
@ -74,7 +85,8 @@ if ! flock -w 1800 8; then
|
||||
fi
|
||||
|
||||
PREVIOUS_IMAGE="$(docker inspect caiwu_app --format '{{.Config.Image}}' 2>/dev/null || true)"
|
||||
PREVIOUS_SHA="$(cat "$REMOTE_DIR/.deployed-sha" 2>/dev/null || true)"
|
||||
caiwu_verify_release_base
|
||||
RELEASE_RESULT=running
|
||||
|
||||
restore_previous_image() {
|
||||
case "$PREVIOUS_IMAGE" in
|
||||
@ -96,6 +108,8 @@ if [ "$LOADED_SHA" != "$DEPLOY_SHA" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
caiwu_install_release_files
|
||||
|
||||
cd "$REMOTE_DIR"
|
||||
export IMAGE_TAG RUN_MIGRATIONS
|
||||
|
||||
|
||||
@ -7,6 +7,11 @@ REMOTE_DIR="${3:-/opt/caiwuxitong}"
|
||||
DEPLOY_SHA="${4:?DEPLOY_SHA is required}"
|
||||
RUN_MIGRATIONS="${5:-0}"
|
||||
KEEP_IMAGES="${6:-2}"
|
||||
EXPECTED_PREVIOUS_SHA="${7:?Checked production revision is required; update the deployment scripts}"
|
||||
DEPLOY_ACTOR="${8:-unknown}"
|
||||
RELEASE_APP_URL="${9:-}"
|
||||
RELEASE_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
|
||||
. "$RELEASE_DIR/release-guard.sh"
|
||||
IMAGE_NAME="caiwu_akaunting"
|
||||
LOCK_FILE="$REMOTE_DIR/.deploy.lock"
|
||||
|
||||
@ -14,7 +19,9 @@ cleanup() {
|
||||
rm -f "$IMAGE_ARCHIVE"
|
||||
}
|
||||
|
||||
trap cleanup EXIT INT TERM
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
echo "activate-release.sh must run as root" >&2
|
||||
@ -69,6 +76,10 @@ if ! flock -n 9; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RELEASE_STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
RELEASE_RESULT=blocked
|
||||
trap 'release_status=$?; caiwu_record_release_result "$release_status"; cleanup' EXIT
|
||||
|
||||
# 容器替换会终止其中的同步进程;与生产刷新任务共享同一把锁。
|
||||
exec 8>/run/caiwu-upstream-sync.lock
|
||||
echo "Waiting for active upstream synchronization before replacing the application container"
|
||||
@ -78,7 +89,8 @@ if ! flock -w 1800 8; then
|
||||
fi
|
||||
|
||||
PREVIOUS_IMAGE="$(docker inspect caiwu_app --format '{{.Config.Image}}' 2>/dev/null || true)"
|
||||
PREVIOUS_SHA="$(cat "$REMOTE_DIR/.deployed-sha" 2>/dev/null || true)"
|
||||
caiwu_verify_release_base
|
||||
RELEASE_RESULT=running
|
||||
|
||||
restore_previous_image() {
|
||||
case "$PREVIOUS_IMAGE" in
|
||||
@ -100,6 +112,8 @@ if [ "$LOADED_SHA" != "$DEPLOY_SHA" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
caiwu_install_release_files
|
||||
|
||||
cd "$REMOTE_DIR"
|
||||
export IMAGE_TAG RUN_MIGRATIONS
|
||||
|
||||
|
||||
43
deploy/install-release-protection.sh
Normal file
43
deploy/install-release-protection.sh
Normal file
@ -0,0 +1,43 @@
|
||||
#!/bin/sh
|
||||
# Caller holds the production deployment lock. Does not restart any service.
|
||||
set -eu
|
||||
REMOTE_DIR="${1:-/opt/caiwuxitong}"
|
||||
SOURCE_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
|
||||
LEGACY_STUB="$SOURCE_DIR/retired-activation.sh"
|
||||
[ "$(id -u)" -eq 0 ] || { echo 'Release protection must be installed as root' >&2; exit 1; }
|
||||
[ -f "$LEGACY_STUB" ] || { echo 'Missing retired activation stub' >&2; exit 1; }
|
||||
|
||||
is_immutable() { lsattr -d "$1" 2>/dev/null | awk '{print $1}' | grep -q i; }
|
||||
needs_install=0
|
||||
for legacy in activate-release.sh activate-incremental-release.sh; do
|
||||
if cmp -s "$LEGACY_STUB" "$REMOTE_DIR/$legacy" && is_immutable "$REMOTE_DIR/$legacy"; then
|
||||
continue
|
||||
fi
|
||||
if is_immutable "$REMOTE_DIR/$legacy"; then
|
||||
echo "Unexpected immutable legacy entry point: $legacy; verify it before changing protection" >&2
|
||||
exit 1
|
||||
fi
|
||||
needs_install=1
|
||||
done
|
||||
[ "$needs_install" -eq 1 ] || exit 0
|
||||
|
||||
# Test support on an owned temporary file before touching either entry point.
|
||||
probe="$(mktemp "$REMOTE_DIR/.release-protection.XXXXXX")"
|
||||
cleanup_probe() { chattr -i "$probe" 2>/dev/null || true; rm -f "$probe"; }
|
||||
trap cleanup_probe EXIT
|
||||
chattr +i "$probe" || { echo 'Filesystem cannot protect legacy entry points; no application was changed' >&2; exit 1; }
|
||||
chattr -i "$probe"
|
||||
backup="$REMOTE_DIR/runtime/private/release-protection/$(date -u +%Y%m%dT%H%M%SZ)-$$"
|
||||
umask 077
|
||||
mkdir -p "$backup"
|
||||
for legacy in activate-release.sh activate-incremental-release.sh; do
|
||||
if cmp -s "$LEGACY_STUB" "$REMOTE_DIR/$legacy" && is_immutable "$REMOTE_DIR/$legacy"; then
|
||||
continue
|
||||
fi
|
||||
[ ! -f "$REMOTE_DIR/$legacy" ] || cp -p "$REMOTE_DIR/$legacy" "$backup/$legacy"
|
||||
install -m 700 "$LEGACY_STUB" "$REMOTE_DIR/.$legacy.next"
|
||||
mv "$REMOTE_DIR/.$legacy.next" "$REMOTE_DIR/$legacy"
|
||||
chattr +i "$REMOTE_DIR/$legacy"
|
||||
cmp -s "$LEGACY_STUB" "$REMOTE_DIR/$legacy" && is_immutable "$REMOTE_DIR/$legacy"
|
||||
done
|
||||
echo 'Legacy activation paths protected; current application and databases were not restarted'
|
||||
72
deploy/release-guard.sh
Normal file
72
deploy/release-guard.sh
Normal file
@ -0,0 +1,72 @@
|
||||
#!/bin/sh
|
||||
# Sourced by both activation paths after the deployment and synchronization locks.
|
||||
|
||||
caiwu_verify_release_base() {
|
||||
case "$EXPECTED_PREVIOUS_SHA" in
|
||||
none) ;;
|
||||
*[!0-9a-f]*|'') echo 'Missing or invalid checked production revision; update the deployment scripts' >&2; return 1 ;;
|
||||
*) [ "${#EXPECTED_PREVIOUS_SHA}" -eq 40 ] || return 1 ;;
|
||||
esac
|
||||
PREVIOUS_SHA="$(cat "$REMOTE_DIR/.deployed-sha" 2>/dev/null || true)"
|
||||
[ -n "$PREVIOUS_SHA" ] || PREVIOUS_SHA=none
|
||||
if [ "$PREVIOUS_SHA" != none ] && ! printf '%s\n' "$PREVIOUS_SHA" | grep -Eq '^[0-9a-f]{40}$'; then
|
||||
echo 'Invalid production revision marker; refusing activation' >&2
|
||||
return 1
|
||||
fi
|
||||
if [ "$PREVIOUS_SHA" != "$EXPECTED_PREVIOUS_SHA" ]; then
|
||||
echo "Production changed while this release was being prepared: checked=$EXPECTED_PREVIOUS_SHA current=$PREVIOUS_SHA requested=$DEPLOY_SHA. Fetch latest main and retry; the running application was not changed." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
# A failed/interrupted older activation must not leave the marker ahead of the app.
|
||||
running_sha="$(docker inspect caiwu_app --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' 2>/dev/null || true)"
|
||||
if [ "$PREVIOUS_SHA" != none ] && [ "$running_sha" != "$PREVIOUS_SHA" ]; then
|
||||
echo "Running image and production marker disagree; refusing activation: marker=$PREVIOUS_SHA image=$running_sha" >&2
|
||||
return 1
|
||||
fi
|
||||
if [ "$PREVIOUS_SHA" = none ] && [ -n "$running_sha" ]; then
|
||||
echo 'An application is already running without a production marker; verify its revision before deploying' >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
caiwu_install_release_files() {
|
||||
# Each uploader has its own directory. Shared runtime files change only inside
|
||||
# the activation lock and after the production revision has been rechecked.
|
||||
[ "$RELEASE_DIR" != "$REMOTE_DIR" ] || return 0
|
||||
release_files='compose.jiangling.yml activate-release.sh activate-incremental-release.sh release-guard.sh retired-activation.sh install-release-protection.sh config-db-init.sh run-finance-batch.sh clickhouse-resource-policy.xml update-runtime-url.sh configure-dingtalk-callback.sh host-nginx-8442-dingtalk.conf setup-incremental-registry.sh registry-config.yml'
|
||||
for release_file in $release_files; do
|
||||
[ -f "$RELEASE_DIR/$release_file" ] || { echo "Incomplete staged release: $release_file" >&2; return 1; }
|
||||
done
|
||||
sh "$RELEASE_DIR/install-release-protection.sh" "$REMOTE_DIR" || return 1
|
||||
for release_file in $release_files; do
|
||||
case "$release_file" in activate-release.sh|activate-incremental-release.sh) continue ;; esac
|
||||
case "$release_file" in *.sh) release_mode=700 ;; *) release_mode=644 ;; esac
|
||||
[ "$release_file" != config-db-init.sh ] || release_mode=644
|
||||
install -m "$release_mode" "$RELEASE_DIR/$release_file" "$REMOTE_DIR/.$release_file.next"
|
||||
mv "$REMOTE_DIR/.$release_file.next" "$REMOTE_DIR/$release_file"
|
||||
done
|
||||
if [ -n "$RELEASE_APP_URL" ]; then
|
||||
"$REMOTE_DIR/update-runtime-url.sh" "$RELEASE_APP_URL" "$REMOTE_DIR"
|
||||
fi
|
||||
printf '%s\n' "$DEPLOY_SHA" > "$REMOTE_DIR/.release-controls-sha.next"
|
||||
mv "$REMOTE_DIR/.release-controls-sha.next" "$REMOTE_DIR/.release-controls-sha"
|
||||
}
|
||||
|
||||
caiwu_record_release_result() {
|
||||
release_exit="$1"
|
||||
[ "$RELEASE_RESULT" != running ] || RELEASE_RESULT=failed
|
||||
[ "$release_exit" -ne 0 ] || RELEASE_RESULT=success
|
||||
# Client identity is declared by the workstation; the SSH source is recorded
|
||||
# separately. Neither field is inferred from the commit author.
|
||||
release_actor="$(printf '%s' "$DEPLOY_ACTOR" | tr -cd 'A-Za-z0-9_.@:-' | cut -c1-160)"
|
||||
release_peer="$(printf '%s' "${SSH_CONNECTION:-local}" | tr -cd 'A-Za-z0-9 .:-' | cut -c1-160)"
|
||||
(
|
||||
umask 077
|
||||
mkdir -p "$REMOTE_DIR/runtime/private"
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$RELEASE_STARTED_AT" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$release_actor" "$release_peer" \
|
||||
"${PREVIOUS_SHA:-unknown}" "$DEPLOY_SHA" "$RELEASE_RESULT" "$release_exit" "$EXPECTED_PREVIOUS_SHA" \
|
||||
>> "$REMOTE_DIR/runtime/private/deployments.tsv"
|
||||
) || echo 'Could not write the private deployment receipt' >&2
|
||||
}
|
||||
4
deploy/retired-activation.sh
Normal file
4
deploy/retired-activation.sh
Normal file
@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
# This legacy shared path is intentionally retired and immutable on production.
|
||||
echo 'Legacy activation is disabled. Fetch latest main and use the workstation deployment script with an isolated release directory and checked production revision. The running application was not changed.' >&2
|
||||
exit 1
|
||||
@ -129,11 +129,22 @@ bash scripts/deploy-from-mac.sh
|
||||
8. 验证唯一私有七牛桶的写入、读取和删除,并把旧本机业务上传幂等复制到七牛;不删除本机原件;
|
||||
9. 检查服务器本机后端页面;
|
||||
10. 任一存储验证或页面验证失败时恢复上一个镜像;
|
||||
11. 使用服务器互斥锁阻止两个人同时切换版本;
|
||||
11. 每次上传使用独立暂存目录;服务器取得发布锁和同步锁后,再核对线上 SHA 与发布机已检查的 SHA,确认未变化才切换配置和应用;
|
||||
12. 删除本地临时镜像包并关闭 SSH 隧道;
|
||||
13. 开发机、服务器 Docker 和服务器 Registry 均仅保留当前和上一个本项目版本;
|
||||
14. 清理 7 天前的本项目构建缓存,并将总量压到最多 3GB。
|
||||
|
||||
### 5.0 多人发布保护
|
||||
|
||||
- 远端 `main` 禁止直接推送和强推,任务分支通过 PR 合并;不额外要求人工审批或新增 CI 门槛。
|
||||
- 发布机先刷新远端 `main`,正式发布必须与最新主线一致,并检查本次版本包含线上提交。
|
||||
- 上传期间若别人已发布,服务器在锁内拒绝本次激活,运行中的应用保持原版本。先 `git fetch`、快进到最新 `main` 并重新发布,不用回滚开关绕过并发校验。
|
||||
- Windows 和 macOS 发布机都必须使用最新脚本。共享根目录的两个旧激活入口已停用并设置 Linux immutable 属性,旧发布机无法用上传覆盖它们;新脚本从每次独立暂存目录执行激活,缺少已检查线上版本参数也会被拒绝。
|
||||
- 运行配置按每次发布独立暂存,核对通过后才在锁内安装;上传文件不会提前覆盖其他发布使用的配置。
|
||||
- 服务器私有 `runtime/private/deployments.tsv` 记录起止 UTC 时间、发布机声明的身份、SSH 来源、前后 SHA、结果、退出码和已检查的 SHA。声明身份及提交作者不能单独证明具体操作者;记录不包含凭据或业务数据。
|
||||
- 安装旧入口保护前自动备份到服务器私有 `runtime/private/release-protection/`。普通发布不解除 immutable 属性;维护这些停用入口时先核对备份和目的,再由维护人员使用 `chattr -i` 处理指定文件。应用镜像回滚不需要启用旧入口。
|
||||
- 仅维护发布控制脚本时,可在发布锁内备份并安装已合并主线的控制脚本,记录独立的 `.release-controls-sha`,不重建应用容器、不修改表示应用镜像版本的 `.deployed-sha`。
|
||||
|
||||
### 5.1 安全软件拦截 regctl 时的降级
|
||||
|
||||
第 6 步默认用项目专用 `regctl`(未签名)把 BuildKit 原生 OCI 分层包推入服务器本机 Registry。少数开发机的 360 安全卫士、金山毒霸等安全软件会拦截未签名程序,表现为 `regctl-v0.11.5.exe` 启动即报“拒绝访问”。
|
||||
|
||||
@ -416,7 +416,8 @@ if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
|
||||
fail '工作区存在未提交的受控文件改动,拒绝构建。请先提交或处理清楚。'
|
||||
fi
|
||||
|
||||
if [ "$branch" = 'main' ] && git rev-parse --verify --quiet origin/main >/dev/null 2>&1; then
|
||||
git fetch origin main --quiet || fail '无法更新远端 main,已停止发布。'
|
||||
if [ "$branch" = 'main' ]; then
|
||||
if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/main)" ]; then
|
||||
fail '本地 main 与 origin/main 不一致,请先执行 git pull --ff-only 再发布。'
|
||||
fi
|
||||
@ -457,7 +458,6 @@ ssh_control_path="/tmp/caiwu-ssh-$$-$RANDOM.sock"
|
||||
ssh_options+=(-o ControlMaster=auto -o ControlPersist=600 -o "ControlPath=$ssh_control_path")
|
||||
tunnel_ssh_options=("${ssh_options[@]}" -o ControlPath=none)
|
||||
|
||||
remote_archive="$remote_path/image-$image_tag.tar.gz"
|
||||
temp_root="$(mktemp -d "${TMPDIR:-/tmp}/caiwu-deploy-XXXXXX")"
|
||||
local_tar="$temp_root/image.tar"
|
||||
local_archive="$temp_root/image.tar.gz"
|
||||
@ -512,6 +512,13 @@ for pair in \
|
||||
done
|
||||
|
||||
previous_sha="$(ssh "${ssh_options[@]}" "$ssh_target" "cat '$remote_path/.deployed-sha' 2>/dev/null || true" | tr -d '[:space:]')"
|
||||
if [ -n "$previous_sha" ] && ! printf '%s\n' "$previous_sha" | grep -Eq '^[0-9a-f]{40}$'; then
|
||||
fail '线上版本标记格式无效,已停止发布。'
|
||||
fi
|
||||
expected_previous_sha="${previous_sha:-none}"
|
||||
deploy_actor="$(printf '%s@%s' "$(git config user.name)" "$(hostname)" | tr -c 'A-Za-z0-9_.@:-' '_')"
|
||||
remote_release_path="$remote_path/releases/$image_tag-$(uuidgen | tr 'A-Z' 'a-z')"
|
||||
remote_archive="$remote_release_path/image-$image_tag.tar.gz"
|
||||
|
||||
if [ -n "$previous_sha" ]; then
|
||||
if git cat-file -e "${previous_sha}^{commit}" 2>/dev/null; then
|
||||
@ -556,7 +563,7 @@ if [ "$bootstrap_server" -eq 1 ]; then
|
||||
bootstrap_remote=''
|
||||
fi
|
||||
|
||||
ssh "${ssh_options[@]}" "$ssh_target" "docker version >/dev/null && docker compose version >/dev/null && install -d -m 700 '$remote_path'" \
|
||||
ssh "${ssh_options[@]}" "$ssh_target" "docker version >/dev/null && docker compose version >/dev/null && install -d -m 700 '$remote_path' '$remote_release_path'" \
|
||||
|| fail '服务器 Docker 未就绪;首次发布请添加 --bootstrap-server。'
|
||||
|
||||
ensure_qiniu_storage_runtime
|
||||
@ -570,6 +577,9 @@ scp_to_remote \
|
||||
"$repo_root/compose.jiangling.yml" \
|
||||
"$repo_root/deploy/activate-release.sh" \
|
||||
"$repo_root/deploy/activate-incremental-release.sh" \
|
||||
"$repo_root/deploy/release-guard.sh" \
|
||||
"$repo_root/deploy/retired-activation.sh" \
|
||||
"$repo_root/deploy/install-release-protection.sh" \
|
||||
"$repo_root/deploy/config-db-init.sh" \
|
||||
"$repo_root/deploy/run-finance-batch.sh" \
|
||||
"$repo_root/deploy/clickhouse-resource-policy.xml" \
|
||||
@ -578,10 +588,7 @@ scp_to_remote \
|
||||
"$repo_root/deploy/host-nginx-8442-dingtalk.conf" \
|
||||
"$repo_root/deploy/setup-incremental-registry.sh" \
|
||||
"$repo_root/deploy/registry-config.yml" \
|
||||
"${ssh_target}:$remote_path/" || fail '上传运行配置失败。'
|
||||
|
||||
ssh "${ssh_options[@]}" "$ssh_target" "chmod 700 '$remote_path/update-runtime-url.sh' && '$remote_path/update-runtime-url.sh' '$app_url' '$remote_path'" \
|
||||
|| fail '更新服务器访问地址失败。'
|
||||
"${ssh_target}:$remote_release_path/" || fail '上传运行配置失败。'
|
||||
|
||||
if [ "$bootstrap_server" -eq 1 ]; then
|
||||
if ! docker image inspect "$database_image" >/dev/null 2>&1; then
|
||||
@ -598,11 +605,11 @@ if [ "$bootstrap_server" -eq 1 ]; then
|
||||
log "首次初始化全量包:${archive_size} MiB"
|
||||
|
||||
scp_to_remote "$local_archive" "${ssh_target}:$remote_archive" || fail '上传镜像失败。'
|
||||
ssh "${ssh_options[@]}" "$ssh_target" "chmod 700 '$remote_path/'*.sh && chmod 644 '$remote_path/config-db-init.sh' && '$remote_path/activate-release.sh' '$remote_archive' '$image_tag' '$remote_path' '$deploy_sha' '$migration_flag' '$keep_images' && '$remote_path/setup-incremental-registry.sh' '$remote_path' '$remote_path/registry-config.yml'" \
|
||||
ssh "${ssh_options[@]}" "$ssh_target" "chmod 700 '$remote_release_path/'*.sh && '$remote_release_path/activate-release.sh' '$remote_archive' '$image_tag' '$remote_path' '$deploy_sha' '$migration_flag' '$keep_images' '$expected_previous_sha' '$deploy_actor' '$app_url' && '$remote_path/setup-incremental-registry.sh' '$remote_path' '$remote_path/registry-config.yml'" \
|
||||
|| fail '服务器激活、健康检查或增量仓库初始化失败。'
|
||||
else
|
||||
ssh "${ssh_options[@]}" -o ConnectTimeout=10 -o ConnectionAttempts=1 "$ssh_target" \
|
||||
"chmod 700 '$remote_path/'*.sh && chmod 644 '$remote_path/config-db-init.sh' && (curl -fsS -o /dev/null http://127.0.0.1:5000/v2/ || '$remote_path/setup-incremental-registry.sh' '$remote_path' '$remote_path/registry-config.yml')" \
|
||||
"chmod 700 '$remote_release_path/'*.sh && (curl -fsS -o /dev/null http://127.0.0.1:5000/v2/ || '$remote_release_path/setup-incremental-registry.sh' '$remote_path' '$remote_release_path/registry-config.yml')" \
|
||||
|| fail '服务器增量镜像仓库初始化失败。'
|
||||
|
||||
docker buildx build \
|
||||
@ -685,7 +692,7 @@ else
|
||||
incremental_mib="$(awk -v a="$after_bytes" -v b="$before_bytes" 'BEGIN { v = (a - b) / 1048576; if (v < 0) v = 0; printf "%.2f", v }')"
|
||||
log "增量新增镜像层:${incremental_mib} MiB;传输耗时:${upload_seconds} 秒"
|
||||
|
||||
ssh "${ssh_options[@]}" "$ssh_target" "'$remote_path/activate-incremental-release.sh' '$server_registry_image' '$image_tag' '$remote_path' '$deploy_sha' '$migration_flag' '$keep_images'" \
|
||||
ssh "${ssh_options[@]}" "$ssh_target" "'$remote_release_path/activate-incremental-release.sh' '$server_registry_image' '$image_tag' '$remote_path' '$deploy_sha' '$migration_flag' '$keep_images' '$expected_previous_sha' '$deploy_actor' '$app_url'" \
|
||||
|| fail '服务器增量激活或健康检查失败。'
|
||||
fi
|
||||
|
||||
|
||||
@ -348,6 +348,11 @@ try {
|
||||
throw '工作区存在未提交的受控文件改动,拒绝构建。请先提交或处理清楚。'
|
||||
}
|
||||
|
||||
Invoke-Native { git fetch origin main --quiet } '无法更新远端 main,已停止发布。'
|
||||
if ($branch -eq 'main' -and (git rev-parse HEAD).Trim() -ne (git rev-parse origin/main).Trim()) {
|
||||
throw '本地 main 不是最新 origin/main,请先 git pull --ff-only 再发布。'
|
||||
}
|
||||
|
||||
$deploySha = (git rev-parse HEAD).Trim()
|
||||
$imageTag = $deploySha.Substring(0, 12)
|
||||
$fullImage = "${imageName}:$imageTag"
|
||||
@ -372,7 +377,6 @@ try {
|
||||
}
|
||||
Invoke-NativeWithRetry { ssh @sshOptions $sshTarget true } '服务器登录失败或已取消,已停止部署。' -Attempts 3
|
||||
if ($authSession.Directory) { $env:CAIWU_DEPLOY_PASSWORD_UPDATE = $null }
|
||||
$remoteArchive = "$RemotePath/image-$imageTag.tar.gz"
|
||||
|
||||
# deploy-auth.ps1 is dot-sourced and can clear the automatic script path
|
||||
# variable on PowerShell Core. Restore it before invoking the build helper.
|
||||
@ -436,6 +440,11 @@ try {
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw '无法连接服务器。'
|
||||
}
|
||||
if ($previousSha -and $previousSha -notmatch '^[0-9a-f]{40}$') { throw '线上版本标记格式无效,已停止发布。' }
|
||||
$expectedPreviousSha = if ($previousSha) { $previousSha } else { 'none' }
|
||||
$deployActor = (((git config user.name) -join '') + '@' + [Environment]::MachineName) -replace '[^A-Za-z0-9_.@:-]', '_'
|
||||
$remoteReleasePath = "$RemotePath/releases/$imageTag-$([Guid]::NewGuid().ToString('N'))"
|
||||
$remoteArchive = "$remoteReleasePath/image-$imageTag.tar.gz"
|
||||
|
||||
$previousShaExists = $false
|
||||
if ($previousSha) {
|
||||
@ -487,7 +496,7 @@ try {
|
||||
}
|
||||
|
||||
Invoke-Native {
|
||||
ssh @sshOptions $sshTarget "docker version >/dev/null && docker compose version >/dev/null && install -d -m 700 '$RemotePath'"
|
||||
ssh @sshOptions $sshTarget "docker version >/dev/null && docker compose version >/dev/null && install -d -m 700 '$RemotePath' '$remoteReleasePath'"
|
||||
} '服务器 Docker 未就绪;首次发布请添加 -BootstrapServer。'
|
||||
|
||||
Ensure-QiniuBusinessStorageRuntime
|
||||
@ -498,6 +507,9 @@ try {
|
||||
compose.jiangling.yml `
|
||||
deploy/activate-release.sh `
|
||||
deploy/activate-incremental-release.sh `
|
||||
deploy/release-guard.sh `
|
||||
deploy/retired-activation.sh `
|
||||
deploy/install-release-protection.sh `
|
||||
deploy/config-db-init.sh `
|
||||
deploy/run-finance-batch.sh `
|
||||
deploy/clickhouse-resource-policy.xml `
|
||||
@ -506,13 +518,9 @@ try {
|
||||
deploy/host-nginx-8442-dingtalk.conf `
|
||||
deploy/setup-incremental-registry.sh `
|
||||
deploy/registry-config.yml `
|
||||
"${sshTarget}:$RemotePath/"
|
||||
"${sshTarget}:$remoteReleasePath/"
|
||||
} '上传运行配置失败。'
|
||||
|
||||
Invoke-Native {
|
||||
ssh @sshOptions $sshTarget "chmod 700 '$RemotePath/update-runtime-url.sh' && '$RemotePath/update-runtime-url.sh' '$AppUrl' '$RemotePath'"
|
||||
} '更新服务器访问地址失败。'
|
||||
|
||||
if ($BootstrapServer) {
|
||||
docker image inspect $DatabaseImage 2>$null | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
@ -532,12 +540,12 @@ try {
|
||||
Write-Host "首次初始化全量包:$archiveSizeMB MiB"
|
||||
Invoke-Native { scp @scpOptions @sshOptions $localArchive "${sshTarget}:$remoteArchive" } '上传镜像失败。'
|
||||
Invoke-Native {
|
||||
ssh @sshOptions $sshTarget "chmod 700 '$RemotePath/'*.sh && chmod 644 '$RemotePath/config-db-init.sh' && '$RemotePath/activate-release.sh' '$remoteArchive' '$imageTag' '$RemotePath' '$deploySha' '$migrationFlag' '$KeepImages' && '$RemotePath/setup-incremental-registry.sh' '$RemotePath' '$RemotePath/registry-config.yml'"
|
||||
ssh @sshOptions $sshTarget "chmod 700 '$remoteReleasePath/'*.sh && '$remoteReleasePath/activate-release.sh' '$remoteArchive' '$imageTag' '$RemotePath' '$deploySha' '$migrationFlag' '$KeepImages' '$expectedPreviousSha' '$deployActor' '$AppUrl' && '$RemotePath/setup-incremental-registry.sh' '$RemotePath' '$RemotePath/registry-config.yml'"
|
||||
} '服务器激活、健康检查或增量仓库初始化失败。'
|
||||
}
|
||||
else {
|
||||
Invoke-NativeWithRetry {
|
||||
ssh @sshOptions -o ConnectTimeout=10 -o ConnectionAttempts=1 $sshTarget "chmod 700 '$RemotePath/'*.sh && chmod 644 '$RemotePath/config-db-init.sh' && (curl -fsS -o /dev/null http://127.0.0.1:5000/v2/ || '$RemotePath/setup-incremental-registry.sh' '$RemotePath' '$RemotePath/registry-config.yml')"
|
||||
ssh @sshOptions -o ConnectTimeout=10 -o ConnectionAttempts=1 $sshTarget "chmod 700 '$remoteReleasePath/'*.sh && (curl -fsS -o /dev/null http://127.0.0.1:5000/v2/ || '$remoteReleasePath/setup-incremental-registry.sh' '$RemotePath' '$remoteReleasePath/registry-config.yml')"
|
||||
} '服务器增量镜像仓库初始化失败。'
|
||||
|
||||
# docker save 导出未压缩层,每次导入重新压缩会改变摘要,导致 Registry 无法复用。
|
||||
@ -653,7 +661,7 @@ try {
|
||||
Write-Host "增量新增镜像层:$incrementalMiB MiB;传输耗时:$([math]::Round($uploadTimer.Elapsed.TotalSeconds, 1)) 秒"
|
||||
|
||||
Invoke-Native {
|
||||
ssh @sshOptions $sshTarget "'$RemotePath/activate-incremental-release.sh' '$serverRegistryImage' '$imageTag' '$RemotePath' '$deploySha' '$migrationFlag' '$KeepImages'"
|
||||
ssh @sshOptions $sshTarget "'$remoteReleasePath/activate-incremental-release.sh' '$serverRegistryImage' '$imageTag' '$RemotePath' '$deploySha' '$migrationFlag' '$KeepImages' '$expectedPreviousSha' '$deployActor' '$AppUrl'"
|
||||
} '服务器增量激活或健康检查失败。'
|
||||
}
|
||||
|
||||
|
||||
148
tests/scripts/release-guard.sh
Normal file
148
tests/scripts/release-guard.sh
Normal file
@ -0,0 +1,148 @@
|
||||
#!/bin/sh
|
||||
# Run in a disposable Linux container, never on a production host.
|
||||
set -eu
|
||||
REPO="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)"
|
||||
fixture="$(mktemp -d /tmp/caiwu-release-guard.XXXXXX)"
|
||||
cleanup() {
|
||||
for legacy in activate-release.sh activate-incremental-release.sh; do
|
||||
chattr -i "$fixture/legacy-entrypoints/$legacy" 2>/dev/null || true
|
||||
done
|
||||
rm -rf "$fixture"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
mkdir -p "$fixture/bin"
|
||||
cat > "$fixture/bin/docker" <<'EOF'
|
||||
#!/bin/sh
|
||||
printf '%s\n' "$*" >> "$DOCKER_CALLS"
|
||||
if [ "$1" = inspect ]; then
|
||||
case "$*" in
|
||||
*Config.Image*) printf 'caiwu_akaunting:old\n' ;;
|
||||
*) printf '%s\n' "${FAKE_RUNNING_SHA:-}" ;;
|
||||
esac
|
||||
exit 0
|
||||
fi
|
||||
exit 91
|
||||
EOF
|
||||
chmod +x "$fixture/bin/docker"
|
||||
PATH="$fixture/bin:$PATH"
|
||||
export PATH
|
||||
old=1111111111111111111111111111111111111111
|
||||
new=2222222222222222222222222222222222222222
|
||||
requested=3333333333333333333333333333333333333333
|
||||
passed=0
|
||||
pass() { passed=$((passed + 1)); printf 'PASS: %s\n' "$*"; }
|
||||
fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
new_runtime() {
|
||||
REMOTE_DIR="$fixture/$1"
|
||||
mkdir -p "$REMOTE_DIR/runtime"
|
||||
: > "$REMOTE_DIR/runtime/.env"
|
||||
: > "$REMOTE_DIR/runtime/config-db.env"
|
||||
printf 'CLICKHOUSE_PASSWORD=fixture\nBUSINESS_CLICKHOUSE_PASSWORD=fixture\n' > "$REMOTE_DIR/runtime/clickhouse.env"
|
||||
printf 'BUSINESS_STORAGE_DRIVER=qiniu\nQINIU_ACCESS_KEY=fixture\nQINIU_SECRET_KEY=fixture\n' > "$REMOTE_DIR/runtime/qiniu-storage.env"
|
||||
printf '%s\n' "$old" > "$REMOTE_DIR/.deployed-sha"
|
||||
printf 'previous runtime configuration\n' > "$REMOTE_DIR/compose.jiangling.yml"
|
||||
DOCKER_CALLS="$REMOTE_DIR/docker-calls"
|
||||
: > "$DOCKER_CALLS"
|
||||
FAKE_RUNNING_SHA="$old"
|
||||
export DOCKER_CALLS FAKE_RUNNING_SHA
|
||||
}
|
||||
|
||||
for activation in activate-release.sh activate-incremental-release.sh; do
|
||||
for scenario in stale missing-protocol marker-image-mismatch unchanged; do
|
||||
new_runtime "$activation-$scenario"
|
||||
: > "$REMOTE_DIR/upload.tar.gz"
|
||||
expected="$old"
|
||||
case "$scenario" in
|
||||
stale) printf '%s\n' "$new" > "$REMOTE_DIR/.deployed-sha"; FAKE_RUNNING_SHA="$new" ;;
|
||||
marker-image-mismatch) FAKE_RUNNING_SHA="$new" ;;
|
||||
esac
|
||||
set +e
|
||||
if [ "$scenario" = missing-protocol ]; then
|
||||
sh "$REPO/deploy/$activation" "$REMOTE_DIR/upload.tar.gz" tag "$REMOTE_DIR" "$requested" 0 2 > "$REMOTE_DIR/output" 2>&1
|
||||
else
|
||||
sh "$REPO/deploy/$activation" "$REMOTE_DIR/upload.tar.gz" tag "$REMOTE_DIR" "$requested" 0 2 "$expected" fixture > "$REMOTE_DIR/output" 2>&1
|
||||
fi
|
||||
status=$?
|
||||
set -e
|
||||
if [ "$scenario" = unchanged ]; then
|
||||
[ "$status" -eq 91 ] || { cat "$REMOTE_DIR/output"; fail 'accepted release did not reach image loading'; }
|
||||
else
|
||||
[ "$status" -ne 0 ] || fail "$activation accepted $scenario"
|
||||
if grep -Ev '^inspect ' "$DOCKER_CALLS" | grep -q .; then
|
||||
fail "$activation mutated Docker before rejecting $scenario"
|
||||
fi
|
||||
[ "$(cat "$REMOTE_DIR/compose.jiangling.yml")" = 'previous runtime configuration' ] || fail 'rejected release overwrote runtime configuration'
|
||||
fi
|
||||
if [ "$scenario" != missing-protocol ]; then
|
||||
[ -s "$REMOTE_DIR/runtime/private/deployments.tsv" ] || fail 'deployment receipt missing'
|
||||
if [ "$scenario" != unchanged ]; then
|
||||
grep -q 'blocked' "$REMOTE_DIR/runtime/private/deployments.tsv" || fail 'rejection not recorded'
|
||||
fi
|
||||
fi
|
||||
pass "$activation: $scenario"
|
||||
done
|
||||
done
|
||||
|
||||
. "$REPO/deploy/release-guard.sh"
|
||||
new_runtime bootstrap
|
||||
rm "$REMOTE_DIR/.deployed-sha"
|
||||
FAKE_RUNNING_SHA=''
|
||||
EXPECTED_PREVIOUS_SHA=none
|
||||
DEPLOY_SHA="$new"
|
||||
caiwu_verify_release_base
|
||||
pass 'first deployment with no running application'
|
||||
EXPECTED_PREVIOUS_SHA=''
|
||||
if caiwu_verify_release_base 2>/dev/null; then fail 'empty baseline accepted'; fi
|
||||
pass 'empty baseline rejected'
|
||||
|
||||
new_runtime ordered-uploaders
|
||||
EXPECTED_PREVIOUS_SHA="$old"
|
||||
DEPLOY_SHA="$new"
|
||||
# Both uploaders checked the same old revision. The newer one activates first.
|
||||
(
|
||||
exec 9> "$REMOTE_DIR/.deploy.lock"
|
||||
flock -x 9
|
||||
caiwu_verify_release_base
|
||||
printf '%s\n' "$new" > "$REMOTE_DIR/.deployed-sha"
|
||||
)
|
||||
FAKE_RUNNING_SHA="$new"
|
||||
DEPLOY_SHA="$requested"
|
||||
if (
|
||||
exec 9> "$REMOTE_DIR/.deploy.lock"
|
||||
flock -x 9
|
||||
caiwu_verify_release_base
|
||||
) 2>/dev/null; then fail 'delayed uploader overwrote the newer release'; fi
|
||||
[ "$(cat "$REMOTE_DIR/.deployed-sha")" = "$new" ] || fail 'production marker regressed'
|
||||
pass 'delayed uploader is rejected after a newer activation'
|
||||
|
||||
new_runtime staged-config
|
||||
RELEASE_DIR="$fixture/staging"
|
||||
RELEASE_APP_URL=''
|
||||
mkdir -p "$RELEASE_DIR"
|
||||
for file in compose.jiangling.yml activate-release.sh activate-incremental-release.sh release-guard.sh retired-activation.sh install-release-protection.sh config-db-init.sh run-finance-batch.sh clickhouse-resource-policy.xml update-runtime-url.sh configure-dingtalk-callback.sh host-nginx-8442-dingtalk.conf setup-incremental-registry.sh registry-config.yml; do
|
||||
printf 'new staged runtime configuration\n' > "$RELEASE_DIR/$file"
|
||||
done
|
||||
printf '#!/bin/sh\nexit 0\n' > "$RELEASE_DIR/install-release-protection.sh"
|
||||
rm "$RELEASE_DIR/registry-config.yml"
|
||||
if caiwu_install_release_files 2>/dev/null; then fail 'partial upload accepted'; fi
|
||||
[ "$(cat "$REMOTE_DIR/compose.jiangling.yml")" = 'previous runtime configuration' ] || fail 'partial upload replaced runtime files'
|
||||
pass 'incomplete staging directory leaves runtime files untouched'
|
||||
printf 'complete\n' > "$RELEASE_DIR/registry-config.yml"
|
||||
caiwu_install_release_files
|
||||
[ "$(cat "$REMOTE_DIR/compose.jiangling.yml")" = 'new staged runtime configuration' ] || fail 'staged configuration not installed'
|
||||
pass 'complete isolated upload installs the intended runtime files'
|
||||
|
||||
new_runtime legacy-entrypoints
|
||||
for legacy in activate-release.sh activate-incremental-release.sh; do
|
||||
printf '#!/bin/sh\nexit 0\n' > "$REMOTE_DIR/$legacy"
|
||||
done
|
||||
sh "$REPO/deploy/install-release-protection.sh" "$REMOTE_DIR"
|
||||
for legacy in activate-release.sh activate-incremental-release.sh; do
|
||||
if cp "$REPO/deploy/$legacy" "$REMOTE_DIR/$legacy" 2>/dev/null; then fail 'old uploader replaced the protected entry point'; fi
|
||||
if sh "$REMOTE_DIR/$legacy" old-arguments 2>/dev/null; then fail 'old activation path accepted a release'; fi
|
||||
pass "$legacy: legacy uploader and activation both rejected"
|
||||
done
|
||||
sh "$REPO/deploy/install-release-protection.sh" "$REMOTE_DIR"
|
||||
pass 'installing entry point protection again is idempotent'
|
||||
printf 'PASS: %s release protection scenarios\n' "$passed"
|
||||
Loading…
x
Reference in New Issue
Block a user