WuYongJie/gongxiangp2
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: prevent local proxy drift during deploys

Browse Source
This commit is contained in:
chenpei 2026-08-07 11:05:55 +08:00
parent caeabb3b4f
commit ebe4325db2
3 changed files with 1083 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

488
scripts/git-incremental-deploy.ps1
View File

@ -28,6 +28,7 @@ param(
[string]$BuildServerIP = '192.168.40.206',
[string]$BuildServerUser = 'root',
[string]$BuildServerPassword = $env:BUILD_SERVER_PASSWORD,
[string]$BuildServerProxyCommand = $env:BUILD_SERVER_PROXY_COMMAND,
[string]$RemoteBuildDir = '/root/youyidian-3d-git-build',
[string]$ProdServerIP = '192.168.58.20',
@ -168,8 +169,13 @@ function Invoke-BuildServerScript {
$pscpPath = $null
$sshPath = $null
$scpPath = $null
$sshCommonArgs = @()
$scpCommonArgs = @()
if ($usePasswordAuth) {
if (-not [string]::IsNullOrWhiteSpace($BuildServerProxyCommand)) {
throw 'BuildServerProxyCommand is supported only with OpenSSH key authentication.'
}
$plinkPath = Get-PlinkPath
$pscpPath = Get-PscpPath
}
@ -181,6 +187,12 @@ function Invoke-BuildServerScript {
}
$sshPath = $sshCommand.Source
$scpPath = $scpCommand.Source
$sshCommonArgs = @('-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=10')
$scpCommonArgs = @('-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=10')
if (-not [string]::IsNullOrWhiteSpace($BuildServerProxyCommand)) {
$sshCommonArgs += @('-o', "ProxyCommand=$BuildServerProxyCommand")
$scpCommonArgs += @('-o', "ProxyCommand=$BuildServerProxyCommand")
}
}
$scriptId = [guid]::NewGuid().ToString('N')
@ -192,7 +204,15 @@ function Invoke-BuildServerScript {
$normalizedScriptText = $ScriptText.Replace("`r`n", "`n").Replace("`r", "`n")
[System.IO.File]::WriteAllText($localTemp, $normalizedScriptText, [System.Text.UTF8Encoding]::new($false))
$authLabel = if ($usePasswordAuth) { 'password auth' } else { 'SSH key auth' }
$authLabel = if ($usePasswordAuth) {
'password auth'
}
elseif (-not [string]::IsNullOrWhiteSpace($BuildServerProxyCommand)) {
'SSH key auth via proxy command'
}
else {
'SSH key auth'
}
Write-Info "Connecting to build server $BuildServerUser@$BuildServerIP ($authLabel)"
# 原生程序（plink/pscp）会把 git 等子进程的提示信息写到 stderr。
@ -210,7 +230,7 @@ function Invoke-BuildServerScript {
$copyOutput = & $pscpPath -batch -pw $BuildServerPassword $localTemp $remoteSpec 2>&1
}
else {
$copyOutput = & $scpPath -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10 $localTemp $remoteSpec 2>&1
$copyOutput = & $scpPath @scpCommonArgs $localTemp $remoteSpec 2>&1
}
$copyOutput |
ForEach-Object {
@ -230,7 +250,7 @@ function Invoke-BuildServerScript {
$runOutput = & $plinkPath -batch -pw $BuildServerPassword "$BuildServerUser@$BuildServerIP" $command 2>&1
}
else {
$runOutput = & $sshPath -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10 "$BuildServerUser@$BuildServerIP" $command 2>&1
$runOutput = & $sshPath @sshCommonArgs "$BuildServerUser@$BuildServerIP" $command 2>&1
}
$runOutput |
ForEach-Object {
@ -269,6 +289,14 @@ if ($RepoUrl -match '^https?://' -and [string]::IsNullOrWhiteSpace($GitAuthHeade
$cfg = $targetConfig[$Target]
$rollbackFlag = if ($NoRollback) { '0' } else { '1' }
$proxyReconcilerPath = Join-Path $PSScriptRoot 'local-proxy-reconcile.sh'
if (-not (Test-Path -LiteralPath $proxyReconcilerPath)) {
throw "Local proxy reconciler is missing: $proxyReconcilerPath"
}
$proxyReconcilerText = [System.IO.File]::ReadAllText($proxyReconcilerPath)
$proxyReconcilerText = $proxyReconcilerText.Replace("`r`n", "`n").Replace("`r", "`n")
$proxyReconcilerBytes = [System.Text.UTF8Encoding]::new($false).GetBytes($proxyReconcilerText)
$proxyReconcilerBase64 = [Convert]::ToBase64String($proxyReconcilerBytes)
$tokens = @{
'__REPO_URL__' = ConvertTo-ShellSingleQuoted $RepoUrl
@ -291,6 +319,7 @@ $tokens = @{
'__DINGTALK_INTERNAL_RETURN_ORIGINS__' = ConvertTo-ShellSingleQuoted $cfg.DingtalkInternalReturnOrigins
'__S3_PUBLIC_DOMAIN__' = ConvertTo-ShellSingleQuoted $cfg.S3PublicDomain
'__DOWNLOAD_TOKEN_SECRET__' = ConvertTo-ShellSingleQuoted $DownloadTokenSecret
'__PROXY_RECONCILE_SCRIPT_B64__' = ConvertTo-ShellSingleQuoted $proxyReconcilerBase64
'__DRY_RUN__' = ConvertTo-ShellSingleQuoted ($(if ($DryRun) { '1' } else { '0' }))
'__NO_CACHE__' = ConvertTo-ShellSingleQuoted ($(if ($NoCache) { '1' } else { '0' }))
'__SKIP_BUILD__' = ConvertTo-ShellSingleQuoted ($(if ($SkipBuild) { '1' } else { '0' }))
@ -324,6 +353,7 @@ DINGTALK_CALLBACK_ORIGIN=__DINGTALK_CALLBACK_ORIGIN__
DINGTALK_INTERNAL_RETURN_ORIGINS=__DINGTALK_INTERNAL_RETURN_ORIGINS__
S3_PUBLIC_DOMAIN=__S3_PUBLIC_DOMAIN__
DOWNLOAD_TOKEN_SECRET=__DOWNLOAD_TOKEN_SECRET__
PROXY_RECONCILE_SCRIPT_B64=__PROXY_RECONCILE_SCRIPT_B64__
DRY_RUN=__DRY_RUN__
NO_CACHE=__NO_CACHE__
SKIP_BUILD=__SKIP_BUILD__
@ -554,7 +584,12 @@ require_file() {
}
command -v docker >/dev/null 2>&1 || fail "Missing docker on production server"
command -v curl >/dev/null 2>&1 || warn "curl not found; health check will use docker logs only"
command -v curl >/dev/null 2>&1 || fail "Missing curl on production server"
command -v flock >/dev/null 2>&1 || fail "Missing flock on production server"
command -v ss >/dev/null 2>&1 || fail "Missing ss on production server; safe connection draining cannot be verified"
command -v base64 >/dev/null 2>&1 || fail "Missing base64 on production server"
command -v nginx >/dev/null 2>&1 || fail "Missing nginx on production server"
command -v systemctl >/dev/null 2>&1 || fail "Missing systemctl on production server"
require_file "$ENV_FILE_PATH"
[ -d "$UPLOADS_MOUNT" ] || fail "Uploads mount does not exist: $UPLOADS_MOUNT"
@ -573,6 +608,10 @@ PROXY_CONF="${PROXY_ROOT}/nginx.conf"
PROXY_PID="${PROXY_ROOT}/nginx.pid"
PROXY_SERVICE="${CONTAINER_NAME}-local-proxy.service"
CURRENT_FILE="${PROXY_ROOT}/current-container"
PROXY_RECONCILE_SCRIPT="${PROXY_ROOT}/reconcile-upstream.sh"
PROXY_RECONCILE_SERVICE="${CONTAINER_NAME}-local-proxy-reconcile.service"
PROXY_RECONCILE_TIMER="${CONTAINER_NAME}-local-proxy-reconcile.timer"
PROXY_RECONCILE_LOCK="/run/${CONTAINER_NAME}-local-proxy-reconcile.lock"
BLUE_CONTAINER="${CONTAINER_NAME}-blue"
GREEN_CONTAINER="${CONTAINER_NAME}-green"
PUBLIC_TRAFFIC_SYNC_ROOT="/mnt/PoolOne/apps/youyidian-3d/public-traffic-sync"
@ -721,8 +760,23 @@ container_ip() {
write_proxy_config() {
local upstream="$1"
local target_container="$2"
local temp_conf=""
local backup_conf=""
case "$target_container" in
"$BLUE_CONTAINER"|"$GREEN_CONTAINER") ;;
*)
warn "Refusing to write proxy config for invalid container: ${target_container}"
return 1
;;
esac
mkdir -p "$PROXY_ROOT"
cat > "$PROXY_CONF" <<EOF
if ! temp_conf="$(mktemp "${PROXY_CONF}.tmp.XXXXXX")"; then
return 1
fi
cat > "$temp_conf" <<EOF
worker_processes 1;
pid ${PROXY_PID};
@ -756,6 +810,8 @@ http {
location / {
proxy_pass http://app_backend;
proxy_hide_header X-Material-Proxy-Target;
add_header X-Material-Proxy-Target ${target_container} always;
proxy_http_version 1.1;
proxy_set_header Host \$http_host;
# Preserve the incoming LAN host and port for same-origin auth handoff.
@ -772,13 +828,39 @@ http {
}
}
EOF
if [ "$?" -ne 0 ]; then
rm -f "$temp_conf"
return 1
fi
nginx -p "${PROXY_ROOT}/" -c "$PROXY_CONF" -t
if ! nginx -p "${PROXY_ROOT}/" -c "$temp_conf" -t; then
rm -f "$temp_conf"
return 1
fi
if [ -f "$PROXY_CONF" ]; then
backup_conf="${PROXY_CONF}.bak.deploy.$(date +%Y%m%d-%H%M%S)"
if ! cp -a "$PROXY_CONF" "$backup_conf"; then
rm -f "$temp_conf"
return 1
fi
fi
if ! mv "$temp_conf" "$PROXY_CONF"; then
rm -f "$temp_conf"
return 1
fi
}
ensure_proxy_service() {
mkdir -p "$PROXY_ROOT"
cat > "/etc/systemd/system/${PROXY_SERVICE}" <<EOF
local service_file="/etc/systemd/system/${PROXY_SERVICE}"
local service_temp=""
mkdir -p "$PROXY_ROOT" || return 1
if ! service_temp="$(mktemp "${service_file}.tmp.XXXXXX")"; then
return 1
fi
cat > "$service_temp" <<EOF
[Unit]
Description=Youyidian material local entry proxy (${CONTAINER_NAME})
After=network-online.target docker.service
@ -797,13 +879,19 @@ RestartSec=2
[Install]
WantedBy=multi-user.target
EOF
if [ "$?" -ne 0 ]; then
rm -f "$service_temp"
return 1
fi
chmod 0644 "$service_temp" || return 1
mv "$service_temp" "$service_file" || return 1
systemctl daemon-reload
systemctl enable "$PROXY_SERVICE" >/dev/null 2>&1 || true
systemctl daemon-reload || return 1
systemctl enable "$PROXY_SERVICE" >/dev/null 2>&1 || return 1
}
switch_proxy() {
ensure_proxy_service
ensure_proxy_service || return 1
if is_proxy_running; then
systemctl reload "$PROXY_SERVICE"
else
@ -811,6 +899,155 @@ switch_proxy() {
fi
}
runtime_proxy_container() {
local target=""
target="$(
curl -sS --max-time 5 -D - -o /dev/null "http://127.0.0.1:${PORT}/api/health" 2>/dev/null |
tr -d '\r' |
sed -nE 's/^X-Material-Proxy-Target:[[:space:]]*(.*)$/\1/Ip' |
tail -n 1 || true
)"
case "$target" in
"$BLUE_CONTAINER"|"$GREEN_CONTAINER") printf '%s\n' "$target" ;;
*) printf '\n' ;;
esac
}
verify_proxy_target() {
local expected_container="$1"
local observed_container=""
for _ in $(seq 1 5); do
observed_container="$(runtime_proxy_container)"
if [ "$observed_container" = "$expected_container" ] && \
curl -fsS --max-time 8 "http://127.0.0.1:${PORT}/api/health" >/tmp/git-deploy-entry-health.json 2>/tmp/git-deploy-entry-health.err; then
return 0
fi
sleep 1
done
warn "Proxy target verification failed: expected=${expected_container}, observed=${observed_container:-missing}"
return 1
}
install_proxy_reconciler() {
local script_temp=""
local service_file="/etc/systemd/system/${PROXY_RECONCILE_SERVICE}"
local service_temp=""
local timer_file="/etc/systemd/system/${PROXY_RECONCILE_TIMER}"
local timer_temp=""
mkdir -p "$PROXY_ROOT" || return 1
script_temp="$(mktemp "${PROXY_RECONCILE_SCRIPT}.tmp.XXXXXX")"
printf '%s' "$PROXY_RECONCILE_SCRIPT_B64" | base64 -d > "$script_temp"
bash -n "$script_temp"
chmod 700 "$script_temp"
mv "$script_temp" "$PROXY_RECONCILE_SCRIPT"
service_temp="$(mktemp "${service_file}.tmp.XXXXXX")"
cat > "$service_temp" <<EOF
[Unit]
Description=Reconcile ${CONTAINER_NAME} local proxy upstream
After=docker.service
Requires=docker.service
[Service]
Type=oneshot
Environment=CONTAINER_NAME=${CONTAINER_NAME}
Environment=PORT=${PORT}
Environment=PROXY_ROOT=${PROXY_ROOT}
Environment=PROXY_SERVICE=${PROXY_SERVICE}
ExecStart=${PROXY_RECONCILE_SCRIPT}
TimeoutStartSec=180
EOF
chmod 0644 "$service_temp"
mv "$service_temp" "$service_file"
timer_temp="$(mktemp "${timer_file}.tmp.XXXXXX")"
cat > "$timer_temp" <<EOF
[Unit]
Description=Check ${CONTAINER_NAME} local proxy upstream
[Timer]
OnBootSec=20s
OnUnitActiveSec=15s
AccuracySec=1s
Persistent=true
[Install]
WantedBy=timers.target
EOF
chmod 0644 "$timer_temp"
mv "$timer_temp" "$timer_file"
systemctl daemon-reload
systemctl enable "$PROXY_RECONCILE_TIMER" >/dev/null 2>&1 || return 1
systemctl restart "$PROXY_RECONCILE_TIMER"
}
container_running() {
[ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || true)" = "true" ]
}
read_recorded_container() {
local name=""
if [ -s "$CURRENT_FILE" ]; then
name="$(tr -d '\r\n' < "$CURRENT_FILE")"
fi
case "$name" in
"$BLUE_CONTAINER"|"$GREEN_CONTAINER") printf '%s\n' "$name" ;;
*) printf '\n' ;;
esac
}
write_current_container() {
local name="$1"
local temp_file=""
mkdir -p "$PROXY_ROOT"
temp_file="$(mktemp "${CURRENT_FILE}.tmp.XXXXXX")"
printf '%s\n' "$name" > "$temp_file"
chmod 0644 "$temp_file"
mv "$temp_file" "$CURRENT_FILE"
}
discover_current_container() {
local recorded_container=""
local runtime_container=""
local running_containers=()
local name=""
recorded_container="$(read_recorded_container)"
runtime_container="$(runtime_proxy_container)"
if [ -n "$runtime_container" ] && container_running "$runtime_container"; then
printf '%s\n' "$runtime_container"
return 0
fi
for name in "$BLUE_CONTAINER" "$GREEN_CONTAINER"; do
if container_running "$name"; then
running_containers+=("$name")
fi
done
if [ "${#running_containers[@]}" -eq 1 ]; then
printf '%s\n' "${running_containers[0]}"
return 0
fi
if [ "${#running_containers[@]}" -gt 1 ]; then
warn "Multiple running material containers found without a verified runtime proxy target (recorded=${recorded_container:-missing})"
return 1
fi
printf '\n'
}
stop_proxy() {
systemctl stop "$PROXY_SERVICE" >/dev/null 2>&1 || true
}
@ -834,9 +1071,150 @@ wait_for_container_health() {
return 1
}
CURRENT_CONTAINER=""
if [ -f "$CURRENT_FILE" ]; then
CURRENT_CONTAINER="$(cat "$CURRENT_FILE" 2>/dev/null || true)"
container_healthy() {
local name="$1"
local ip=""
container_running "$name" || return 1
ip="$(container_ip "$name")"
[ -n "$ip" ] || return 1
curl -fsS --max-time 8 "http://${ip}:3000/api/health" >/dev/null 2>&1
}
container_upstream_connection_count() {
local name="$1"
local ip=""
command -v ss >/dev/null 2>&1 || return 2
container_running "$name" || {
printf '0\n'
return 0
}
ip="$(container_ip "$name")"
[ -n "$ip" ] || return 2
# Let ss filter the peer endpoint itself; column positions differ between
# iproute2 versions when the State column is omitted by -H.
ss -Htan state established dst "${ip}:3000" 2>/dev/null |
awk 'END { print NR + 0 }'
}
wait_for_container_drain() {
local name="$1"
local max_checks="${2:-15}"
local count=""
local zero_samples=0
for _ in $(seq 1 "$max_checks"); do
if ! count="$(container_upstream_connection_count "$name")"; then
warn "Cannot prove that upstream connections drained for ${name}; leaving it running"
return 1
fi
if [ "$count" = "0" ]; then
zero_samples=$((zero_samples + 1))
if [ "$zero_samples" -ge 2 ]; then
return 0
fi
else
zero_samples=0
log "Waiting for ${count} active upstream connection(s) to drain from ${name}"
fi
sleep 2
done
warn "Active upstream connections did not drain from ${name}; leaving it running"
return 1
}
rollback_to_previous_entry() {
local previous_ip=""
[ "$ROLLBACK_ON_FAILURE" = "1" ] || return 1
if [ "$PROXY_WAS_RUNNING" = "1" ] && [ -n "$CURRENT_CONTAINER" ]; then
if ! container_running "$CURRENT_CONTAINER"; then
if ! docker start "$CURRENT_CONTAINER" >/dev/null; then
warn "Failed to restart previous container $CURRENT_CONTAINER"
return 1
fi
if ! wait_for_container_health "$CURRENT_CONTAINER" >/dev/null; then
warn "Previous container did not become healthy: $CURRENT_CONTAINER"
return 1
fi
elif ! container_healthy "$CURRENT_CONTAINER"; then
warn "Previous container is not healthy: $CURRENT_CONTAINER"
return 1
fi
previous_ip="$(container_ip "$CURRENT_CONTAINER")"
[ -n "$previous_ip" ] || return 1
if ! write_proxy_config "${previous_ip}:3000" "$CURRENT_CONTAINER"; then
return 1
fi
if ! switch_proxy; then
return 1
fi
if ! verify_proxy_target "$CURRENT_CONTAINER"; then
return 1
fi
write_current_container "$CURRENT_CONTAINER"
return 0
fi
if [ "$PROXY_WAS_RUNNING" != "1" ] && [ -n "$LEGACY_IMAGE" ]; then
if ! wait_for_container_drain "$NEXT_CONTAINER" 15; then
warn "Candidate still has an in-flight request; direct-container rollback was not attempted"
return 1
fi
stop_proxy >/dev/null 2>&1 || true
docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
if ! start_container "$CONTAINER_NAME" "$LEGACY_IMAGE" 1 >/tmp/git-deploy-rollback-container-id.txt; then
return 1
fi
cat /tmp/git-deploy-rollback-container-id.txt
for _ in $(seq 1 12); do
if ! is_proxy_running && curl -fsS --max-time 8 "http://127.0.0.1:${PORT}/api/health" >/dev/null 2>&1; then
return 0
fi
sleep 2
done
fi
return 1
}
abort_after_uncertain_switch() {
local reason="$1"
warn "$reason"
if rollback_to_previous_entry; then
ok "Previous entry was restored and verified"
if wait_for_container_drain "$NEXT_CONTAINER" 15; then
docker rm -f "$NEXT_CONTAINER" >/dev/null 2>&1 || true
else
warn "Rolled-back candidate was retained because an in-flight request is still using it"
fi
fail "$reason; deployment was rolled back"
fi
warn "Rollback could not be verified; candidate container is being kept for recovery"
fail "$reason; both containers were preserved"
}
mkdir -p "$PROXY_ROOT"
exec 8>"$PROXY_RECONCILE_LOCK"
flock -w 30 8 || fail "Another proxy switch or reconciliation is already running"
install_proxy_reconciler
RECORDED_CONTAINER="$(read_recorded_container)"
if ! CURRENT_CONTAINER="$(discover_current_container)"; then
fail "Cannot safely determine the active material container"
fi
if [ -n "$CURRENT_CONTAINER" ] && [ "$CURRENT_CONTAINER" != "$RECORDED_CONTAINER" ]; then
warn "Repairing stale current-container state: ${RECORDED_CONTAINER:-missing} -> ${CURRENT_CONTAINER}"
write_current_container "$CURRENT_CONTAINER"
fi
if [ "$CURRENT_CONTAINER" = "$BLUE_CONTAINER" ]; then
@ -858,6 +1236,9 @@ fi
ensure_public_traffic_rollup_token
log "Starting candidate container $NEXT_CONTAINER"
if container_running "$NEXT_CONTAINER" && ! wait_for_container_drain "$NEXT_CONTAINER" 15; then
fail "Inactive deployment slot $NEXT_CONTAINER is still serving requests; retry after those requests finish"
fi
docker rm -f "$NEXT_CONTAINER" >/dev/null 2>&1 || true
start_container "$NEXT_CONTAINER" "$REGISTRY_COMMIT" 0 >/tmp/git-deploy-new-container-id.txt
cat /tmp/git-deploy-new-container-id.txt
@ -870,7 +1251,10 @@ if ! NEXT_IP="$(wait_for_container_health "$NEXT_CONTAINER")"; then
fi
ok "Candidate container is healthy at ${NEXT_IP}:3000"
write_proxy_config "${NEXT_IP}:3000"
if ! write_proxy_config "${NEXT_IP}:3000" "$NEXT_CONTAINER"; then
docker rm -f "$NEXT_CONTAINER" >/dev/null 2>&1 || true
fail "Candidate proxy configuration could not be prepared; existing entry was kept"
fi
if [ "$PROXY_WAS_RUNNING" != "1" ]; then
warn "Stable local proxy is not running; migrating public entry from direct container to proxy"
@ -880,39 +1264,53 @@ if [ "$PROXY_WAS_RUNNING" != "1" ]; then
fi
fi
SWITCH_REPORTED_FAILURE=0
if ! switch_proxy; then
warn "Local proxy switch failed"
if [ "$PROXY_WAS_RUNNING" != "1" ] && [ "$ROLLBACK_ON_FAILURE" = "1" ] && [ -n "$LEGACY_IMAGE" ]; then
warn "Rolling back to direct container $LEGACY_IMAGE"
stop_proxy
start_container "$CONTAINER_NAME" "$LEGACY_IMAGE" 1 >/tmp/git-deploy-rollback-container-id.txt
cat /tmp/git-deploy-rollback-container-id.txt
fi
docker rm -f "$NEXT_CONTAINER" >/dev/null 2>&1 || true
fail "Deployment failed during local proxy switch"
SWITCH_REPORTED_FAILURE=1
fi
if ! curl -fsS --max-time 8 "http://127.0.0.1:${PORT}/api/health" >/tmp/git-deploy-entry-health.json 2>/tmp/git-deploy-entry-health.err; then
warn "Stable local entry health check failed after switch"
cat /tmp/git-deploy-entry-health.err >&2 || true
if [ "$PROXY_WAS_RUNNING" != "1" ] && [ "$ROLLBACK_ON_FAILURE" = "1" ] && [ -n "$LEGACY_IMAGE" ]; then
warn "Rolling back to direct container $LEGACY_IMAGE"
stop_proxy
start_container "$CONTAINER_NAME" "$LEGACY_IMAGE" 1 >/tmp/git-deploy-rollback-container-id.txt
cat /tmp/git-deploy-rollback-container-id.txt
elif [ "$PROXY_WAS_RUNNING" = "1" ] && [ -n "$CURRENT_CONTAINER" ]; then
CURRENT_IP="$(container_ip "$CURRENT_CONTAINER")"
if [ -n "$CURRENT_IP" ]; then
warn "Reloading proxy back to previous container $CURRENT_CONTAINER"
write_proxy_config "${CURRENT_IP}:3000" >/dev/null 2>&1 || true
switch_proxy >/dev/null 2>&1 || true
if ! verify_proxy_target "$NEXT_CONTAINER"; then
cat /tmp/git-deploy-entry-health.err >&2 2>/dev/null || true
abort_after_uncertain_switch "Stable local entry did not verify the candidate target"
fi
if [ "$SWITCH_REPORTED_FAILURE" = "1" ]; then
warn "Proxy switch command reported failure, but the runtime candidate target was independently verified"
fi
write_current_container "$NEXT_CONTAINER"
sleep 3
if ! verify_proxy_target "$NEXT_CONTAINER"; then
abort_after_uncertain_switch "Candidate target did not remain healthy during the post-switch check"
fi
if [ -n "$CURRENT_CONTAINER" ] && [ "$CURRENT_CONTAINER" != "$NEXT_CONTAINER" ]; then
if [ "$(runtime_proxy_container)" != "$NEXT_CONTAINER" ] || \
[ "$(read_recorded_container)" != "$NEXT_CONTAINER" ] || \
! container_healthy "$NEXT_CONTAINER"; then
abort_after_uncertain_switch "Proxy state changed before the previous container could be drained"
fi
if wait_for_container_drain "$CURRENT_CONTAINER" 15; then
if docker stop --time 30 "$CURRENT_CONTAINER" >/dev/null 2>&1; then
ok "Previous container stopped and retained for rollback: $CURRENT_CONTAINER"
else
warn "Previous container could not be stopped and was retained: $CURRENT_CONTAINER"
fi
else
warn "Previous container is still serving an in-flight request and was retained: $CURRENT_CONTAINER"
fi
docker rm -f "$NEXT_CONTAINER" >/dev/null 2>&1 || true
fail "Deployment failed after local proxy switch"
fi
printf '%s\n' "$NEXT_CONTAINER" > "$CURRENT_FILE"
if ! verify_proxy_target "$NEXT_CONTAINER"; then
abort_after_uncertain_switch "Candidate target failed after the previous container drain"
fi
flock -u 8
if ! systemctl start "$PROXY_RECONCILE_SERVICE"; then
fail "Proxy reconciler failed its post-deployment verification"
fi
ok "Stable local entry is healthy on port $PORT"
docker logs --tail 20 "$NEXT_CONTAINER" 2>&1 || true
@ -938,10 +1336,6 @@ else
warn "Public traffic statistics rollup timers were not installed"
fi
if [ -n "$CURRENT_CONTAINER" ] && [ "$CURRENT_CONTAINER" != "$NEXT_CONTAINER" ]; then
docker rm -f "$CURRENT_CONTAINER" >/dev/null 2>&1 || true
fi
if [ -n "$LEGACY_IMAGE" ]; then
docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
fi
@ -950,7 +1344,7 @@ PROD_SCRIPT
export_script="$(printf '%s' "$PROD_PAYLOAD" | base64 -w 0)"
ssh -o StrictHostKeyChecking=no "$PROD_SERVER_USER@$PROD_SERVER_IP" \
"TARGET='$TARGET' CONTAINER_NAME='$CONTAINER_NAME' PORT='$PORT' REGISTRY_COMMIT='$REGISTRY_COMMIT' COMMIT_SHA='$COMMIT_SHA' ENV_FILE_PATH='/mnt/PoolOne/apps/youyidian-3d/$ENV_FILE' PUBLIC_APP_URL='$PUBLIC_APP_URL' CORS_ALLOWED_ORIGINS='$CORS_ALLOWED_ORIGINS' DINGTALK_CALLBACK_ORIGIN='$DINGTALK_CALLBACK_ORIGIN' DINGTALK_INTERNAL_RETURN_ORIGINS='$DINGTALK_INTERNAL_RETURN_ORIGINS' S3_PUBLIC_DOMAIN='$S3_PUBLIC_DOMAIN' DOWNLOAD_TOKEN_SECRET='$DOWNLOAD_TOKEN_SECRET' NETWORK_NAME='youyidian-3d_app-network' UPLOADS_MOUNT='/mnt/PoolOne/apps/youyidian-3d/uploads' ROLLBACK_ON_FAILURE='$ROLLBACK_ON_FAILURE' bash -lc \"printf '%s' '$export_script' | base64 -d | bash\""
"TARGET='$TARGET' CONTAINER_NAME='$CONTAINER_NAME' PORT='$PORT' REGISTRY_COMMIT='$REGISTRY_COMMIT' COMMIT_SHA='$COMMIT_SHA' ENV_FILE_PATH='/mnt/PoolOne/apps/youyidian-3d/$ENV_FILE' PUBLIC_APP_URL='$PUBLIC_APP_URL' CORS_ALLOWED_ORIGINS='$CORS_ALLOWED_ORIGINS' DINGTALK_CALLBACK_ORIGIN='$DINGTALK_CALLBACK_ORIGIN' DINGTALK_INTERNAL_RETURN_ORIGINS='$DINGTALK_INTERNAL_RETURN_ORIGINS' S3_PUBLIC_DOMAIN='$S3_PUBLIC_DOMAIN' DOWNLOAD_TOKEN_SECRET='$DOWNLOAD_TOKEN_SECRET' PROXY_RECONCILE_SCRIPT_B64='$PROXY_RECONCILE_SCRIPT_B64' NETWORK_NAME='youyidian-3d_app-network' UPLOADS_MOUNT='/mnt/PoolOne/apps/youyidian-3d/uploads' ROLLBACK_ON_FAILURE='$ROLLBACK_ON_FAILURE' bash -lc \"printf '%s' '$export_script' | base64 -d | bash\""
ok "Production deployment command completed"
if [ "$TARGET" = "production" ]; then
@ -980,7 +1374,7 @@ else
fi
ssh -o StrictHostKeyChecking=no "$PROD_SERVER_USER@$PROD_SERVER_IP" \
"docker image prune -af --filter until=6h >/dev/null 2>&1 || true; docker container prune -f --filter until=6h >/dev/null 2>&1 || true"
"docker image prune -af --filter until=6h >/dev/null 2>&1 || true"
ok "Production server cleanup completed"
fi

325
scripts/local-proxy-reconcile.sh Normal file
View File

@ -0,0 +1,325 @@
#!/usr/bin/env bash
set -euo pipefail
CONTAINER_NAME="${CONTAINER_NAME:?CONTAINER_NAME is required}"
PORT="${PORT:?PORT is required}"
PROXY_ROOT="${PROXY_ROOT:?PROXY_ROOT is required}"
PROXY_SERVICE="${PROXY_SERVICE:?PROXY_SERVICE is required}"
CURRENT_FILE="${PROXY_ROOT}/current-container"
PROXY_CONF="${PROXY_ROOT}/nginx.conf"
LOCK_FILE="${PROXY_RECONCILE_LOCK_FILE:-/run/${CONTAINER_NAME}-local-proxy-reconcile.lock}"
NGINX_BIN="${NGINX_BIN:-/usr/sbin/nginx}"
BLUE_CONTAINER="${CONTAINER_NAME}-blue"
GREEN_CONTAINER="${CONTAINER_NAME}-green"
log() {
printf '%s [%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$1" "$2"
}
container_running() {
[ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || true)" = "true" ]
}
container_ip() {
docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1" 2>/dev/null || true
}
container_healthy() {
local name="$1"
local ip=""
container_running "$name" || return 1
ip="$(container_ip "$name")"
[ -n "$ip" ] || return 1
curl -fsS --max-time 5 "http://${ip}:3000/api/health" >/dev/null 2>&1
}
container_for_ip() {
local expected_ip="$1"
local name=""
local ip=""
[ -n "$expected_ip" ] || return 1
for name in "$BLUE_CONTAINER" "$GREEN_CONTAINER"; do
container_running "$name" || continue
ip="$(container_ip "$name")"
if [ "$ip" = "$expected_ip" ]; then
printf '%s\n' "$name"
return 0
fi
done
return 1
}
read_current_container() {
local name=""
if [ -s "$CURRENT_FILE" ]; then
name="$(tr -d '\r\n' < "$CURRENT_FILE")"
fi
case "$name" in
"$BLUE_CONTAINER"|"$GREEN_CONTAINER") printf '%s\n' "$name" ;;
*) printf '\n' ;;
esac
}
write_current_container() {
local name="$1"
local temp_file=""
temp_file="$(mktemp "${CURRENT_FILE}.tmp.XXXXXX")"
printf '%s\n' "$name" > "$temp_file"
chmod 0644 "$temp_file"
mv "$temp_file" "$CURRENT_FILE"
}
configured_ip() {
sed -nE 's/^[[:space:]]*server ([0-9.]+):3000 .*/\1/p' "$PROXY_CONF" | head -n 1
}
configured_header_container() {
local target=""
target="$(
sed -nE 's/^[[:space:]]*add_header X-Material-Proxy-Target ([^;[:space:]]+) always;.*/\1/p' "$PROXY_CONF" |
tail -n 1
)"
case "$target" in
"$BLUE_CONTAINER"|"$GREEN_CONTAINER") printf '%s\n' "$target" ;;
*) printf '\n' ;;
esac
}
configured_container() {
local header_container=""
local ip_container=""
header_container="$(configured_header_container)"
ip_container="$(container_for_ip "$(configured_ip)" 2>/dev/null || true)"
if [ -n "$header_container" ] && [ -n "$ip_container" ] && [ "$header_container" != "$ip_container" ]; then
printf '\n'
elif [ -n "$header_container" ]; then
printf '%s\n' "$header_container"
else
printf '%s\n' "$ip_container"
fi
}
runtime_proxy_container() {
local target=""
target="$(
curl -sS --max-time 5 -D - -o /dev/null "http://127.0.0.1:${PORT}/api/health" 2>/dev/null |
tr -d '\r' |
sed -nE 's/^X-Material-Proxy-Target:[[:space:]]*(.*)$/\1/Ip' |
tail -n 1 || true
)"
case "$target" in
"$BLUE_CONTAINER"|"$GREEN_CONTAINER") printf '%s\n' "$target" ;;
*) printf '\n' ;;
esac
}
entry_healthy() {
curl -fsS --max-time 5 "http://127.0.0.1:${PORT}/api/health" >/dev/null 2>&1
}
verify_runtime_target() {
local expected_container="$1"
local observed_container=""
for _ in $(seq 1 5); do
observed_container="$(runtime_proxy_container)"
if [ "$observed_container" = "$expected_container" ] && entry_healthy; then
return 0
fi
sleep 1
done
return 1
}
reload_proxy() {
if systemctl is-active --quiet "$PROXY_SERVICE"; then
systemctl reload "$PROXY_SERVICE"
else
systemctl start "$PROXY_SERVICE"
fi
}
backup=""
write_proxy_config_for_target() {
local target_container="$1"
local target_ip="$2"
local temp_conf=""
backup=""
temp_conf="$(mktemp "${PROXY_CONF}.tmp.XXXXXX")"
trap 'if [ -n "${temp_conf:-}" ]; then rm -f "$temp_conf"; fi' EXIT
if ! sed -E \
-e "0,/server [0-9.]+:3000 /s//server ${target_ip}:3000 /" \
-e '/^[[:space:]]*proxy_hide_header X-Material-Proxy-Target;[[:space:]]*$/d' \
-e '/^[[:space:]]*add_header X-Material-Proxy-Target [^;]+ always;[[:space:]]*$/d' \
"$PROXY_CONF" |
awk -v target="$target_container" '
{ print }
/^[[:space:]]*location \/ \{/ {
print " proxy_hide_header X-Material-Proxy-Target;"
print " add_header X-Material-Proxy-Target " target " always;"
}
' > "$temp_conf"; then
return 1
fi
if ! grep -q "server ${target_ip}:3000 " "$temp_conf" ||
! grep -q "add_header X-Material-Proxy-Target ${target_container} always;" "$temp_conf"; then
log "ERROR" "failed to prepare nginx configuration for ${target_container}"
return 1
fi
if ! "$NGINX_BIN" -t -p "${PROXY_ROOT}/" -c "$temp_conf"; then
return 1
fi
backup="${PROXY_CONF}.bak.reconcile.$(date +%Y%m%d-%H%M%S)"
cp -a "$PROXY_CONF" "$backup" || return 1
mv "$temp_conf" "$PROXY_CONF" || return 1
trap - EXIT
temp_conf=""
}
restore_previous_proxy() {
local restore_temp=""
if [ -n "$backup" ] && [ -f "$backup" ]; then
restore_temp="$(mktemp "${PROXY_CONF}.restore.XXXXXX")" || return 1
if ! cp -a "$backup" "$restore_temp"; then
rm -f "$restore_temp"
return 1
fi
if ! "$NGINX_BIN" -t -p "${PROXY_ROOT}/" -c "$restore_temp" >/dev/null 2>&1; then
rm -f "$restore_temp"
return 1
fi
if ! mv "$restore_temp" "$PROXY_CONF"; then
rm -f "$restore_temp"
return 1
fi
reload_proxy >/dev/null 2>&1 || return 1
fi
}
command -v flock >/dev/null 2>&1 || {
log "ERROR" "flock is required"
exit 1
}
command -v awk >/dev/null 2>&1 || {
log "ERROR" "awk is required"
exit 1
}
exec 9>"$LOCK_FILE"
flock -n 9 || exit 0
if [ ! -s "$PROXY_CONF" ]; then
log "WARN" "proxy configuration is missing: ${PROXY_CONF}"
exit 1
fi
current_container="$(read_current_container)"
configured_container_value="$(configured_container)"
runtime_container="$(runtime_proxy_container)"
entry_ok=0
if entry_healthy; then
entry_ok=1
fi
target_container=""
if [ "$entry_ok" = "1" ]; then
if [ -n "$runtime_container" ] && container_healthy "$runtime_container"; then
target_container="$runtime_container"
elif [ -n "$current_container" ] &&
[ -n "$configured_container_value" ] &&
[ "$current_container" != "$configured_container_value" ] &&
container_healthy "$current_container" &&
container_healthy "$configured_container_value"; then
log "ERROR" "healthy entry has no runtime marker and state/config point to different healthy containers"
exit 1
elif [ -n "$current_container" ] && container_healthy "$current_container"; then
# Without a runtime marker, preserve the last verified target instead of
# trusting a disk config that may have been written before a failed reload.
target_container="$current_container"
elif [ -n "$configured_container_value" ] && container_healthy "$configured_container_value"; then
target_container="$configured_container_value"
fi
else
if [ -n "$runtime_container" ] && container_healthy "$runtime_container"; then
target_container="$runtime_container"
elif [ -n "$configured_container_value" ] && container_healthy "$configured_container_value"; then
target_container="$configured_container_value"
elif [ -n "$current_container" ] && container_healthy "$current_container"; then
target_container="$current_container"
fi
fi
if [ -z "$target_container" ]; then
healthy_containers=()
for name in "$BLUE_CONTAINER" "$GREEN_CONTAINER"; do
if container_healthy "$name"; then
healthy_containers+=("$name")
fi
done
if [ "${#healthy_containers[@]}" -eq 1 ]; then
target_container="${healthy_containers[0]}"
elif [ "${#healthy_containers[@]}" -gt 1 ]; then
log "ERROR" "multiple healthy containers found without an authoritative runtime target"
exit 1
fi
fi
if [ -z "$target_container" ]; then
log "WARN" "no healthy material container is available"
exit 1
fi
target_ip="$(container_ip "$target_container")"
if [ -z "$target_ip" ]; then
log "WARN" "target container has no network address: ${target_container}"
exit 1
fi
if [ "$entry_ok" = "1" ] && [ "$runtime_container" = "$target_container" ] && [ "$configured_container_value" = "$target_container" ]; then
if [ "$current_container" != "$target_container" ]; then
write_current_container "$target_container"
log "OK" "state aligned to verified runtime target: ${target_container}"
fi
exit 0
fi
if ! write_proxy_config_for_target "$target_container" "$target_ip"; then
log "ERROR" "failed to prepare proxy configuration for ${target_container}"
exit 1
fi
reload_reported_failure=0
if ! reload_proxy; then
reload_reported_failure=1
fi
if ! verify_runtime_target "$target_container"; then
restore_previous_proxy || true
log "ERROR" "failed to verify proxy target ${target_container}; previous configuration was retained when possible"
exit 1
fi
write_current_container "$target_container"
if [ "$reload_reported_failure" = "1" ]; then
log "WARN" "reload command reported failure, but the runtime target was independently verified"
fi
log "OK" "proxy reconciled to verified runtime target: ${target_container} (${target_ip})"

317
scripts/local-proxy-reconcile.test.sh Normal file
View File

@ -0,0 +1,317 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
RECONCILER="${SCRIPT_DIR}/local-proxy-reconcile.sh"
ORIGINAL_PATH="$PATH"
fail() {
printf 'FAIL: %s\n' "$1" >&2
exit 1
}
assert_equals() {
local expected="$1"
local actual="$2"
local message="$3"
if [ "$expected" != "$actual" ]; then
fail "${message}: expected=${expected}, actual=${actual}"
fi
}
new_fixture() {
FIXTURE_ROOT="$(mktemp -d)"
export FIXTURE_ROOT
export FAKE_STATE_DIR="${FIXTURE_ROOT}/state"
export PROXY_ROOT="${FIXTURE_ROOT}/proxy"
export PROXY_RECONCILE_LOCK_FILE="${FIXTURE_ROOT}/reconcile.lock"
export NGINX_BIN="${FIXTURE_ROOT}/bin/nginx"
export CONTAINER_NAME="material"
export PROXY_SERVICE="material-local-proxy.service"
export PORT="4320"
export PATH="${FIXTURE_ROOT}/bin:${ORIGINAL_PATH}"
mkdir -p "$FAKE_STATE_DIR" "$PROXY_ROOT" "${FIXTURE_ROOT}/bin"
cat > "${FIXTURE_ROOT}/bin/docker" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
template="$3"
name="$4"
if [[ "$template" == *'.State.Running'* ]]; then
cat "${FAKE_STATE_DIR}/${name}.running" 2>/dev/null || printf 'false\n'
else
cat "${FAKE_STATE_DIR}/${name}.ip" 2>/dev/null || true
fi
EOF
cat > "${FIXTURE_ROOT}/bin/curl" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
url="${!#}"
if [[ "$url" == http://127.0.0.1:* ]]; then
if [ -f "${FAKE_STATE_DIR}/entry.connect-fail" ]; then
exit 7
fi
if printf '%s\n' "$@" | grep -qx -- '-D'; then
printf 'HTTP/1.1 %s Test\r\n' "$([ -f "${FAKE_STATE_DIR}/entry.ok" ] && printf '200' || printf '502')"
if [ -s "${FAKE_STATE_DIR}/runtime.target" ]; then
printf 'X-Material-Proxy-Target: %s\r\n' "$(cat "${FAKE_STATE_DIR}/runtime.target")"
fi
printf '\r\n'
exit 0
fi
[ -f "${FAKE_STATE_DIR}/entry.ok" ]
exit
fi
ip="${url#http://}"
ip="${ip%%:*}"
[ -f "${FAKE_STATE_DIR}/healthy.${ip}" ]
EOF
cat > "${FIXTURE_ROOT}/bin/systemctl" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
case "$1" in
is-active) exit 0 ;;
reload|start)
printf '%s\n' "$1" >> "${FAKE_STATE_DIR}/systemctl.log"
target="$(sed -nE 's/^[[:space:]]*add_header X-Material-Proxy-Target ([^;[:space:]]+) always;.*/\1/p' "${PROXY_ROOT}/nginx.conf" | tail -n 1)"
if [ -n "$target" ]; then
printf '%s\n' "$target" > "${FAKE_STATE_DIR}/runtime.target"
fi
if [ "${FAKE_RECOVER_ON_RELOAD:-0}" = "1" ]; then
rm -f "${FAKE_STATE_DIR}/entry.connect-fail"
touch "${FAKE_STATE_DIR}/entry.ok"
fi
count=0
if [ -f "${FAKE_STATE_DIR}/reload.count" ]; then
count="$(cat "${FAKE_STATE_DIR}/reload.count")"
fi
count=$((count + 1))
printf '%s\n' "$count" > "${FAKE_STATE_DIR}/reload.count"
if [ "${FAKE_RELOAD_FAIL_FIRST:-0}" = "1" ] && [ "$count" -eq 1 ]; then
exit 1
fi
;;
esac
EOF
cat > "${FIXTURE_ROOT}/bin/nginx" <<'EOF'
#!/usr/bin/env bash
exit 0
EOF
cat > "${FIXTURE_ROOT}/bin/sleep" <<'EOF'
#!/usr/bin/env bash
exit 0
EOF
cat > "${FIXTURE_ROOT}/bin/flock" <<'EOF'
#!/usr/bin/env bash
[ "${FAKE_FLOCK_FAIL:-0}" = "1" ] && exit 1
exit 0
EOF
chmod +x "${FIXTURE_ROOT}/bin/"*
}
write_proxy_config() {
local ip="$1"
local target="${2:-}"
cat > "${PROXY_ROOT}/nginx.conf" <<EOF
events {}
http {
upstream app_backend {
server ${ip}:3000 max_fails=3 fail_timeout=5s;
}
server {
location / {
$(if [ -n "$target" ]; then printf ' proxy_hide_header X-Material-Proxy-Target;\n add_header X-Material-Proxy-Target %s always;\n' "$target"; fi)
}
}
}
EOF
}
set_container() {
local name="$1"
local ip="$2"
local healthy="$3"
printf 'true\n' > "${FAKE_STATE_DIR}/${name}.running"
printf '%s\n' "$ip" > "${FAKE_STATE_DIR}/${name}.ip"
if [ "$healthy" = "1" ]; then
touch "${FAKE_STATE_DIR}/healthy.${ip}"
fi
}
cleanup_fixture() {
PATH="$ORIGINAL_PATH"
unset FAKE_RECOVER_ON_RELOAD || true
unset FAKE_RELOAD_FAIL_FIRST || true
unset FAKE_FLOCK_FAIL || true
rm -rf "$FIXTURE_ROOT"
}
test_healthy_entry_wins_over_stale_state() {
new_fixture
set_container material-blue 10.0.0.2 1
set_container material-green 10.0.0.3 1
printf 'material-blue\n' > "${PROXY_ROOT}/current-container"
write_proxy_config 10.0.0.3 material-green
printf 'material-green\n' > "${FAKE_STATE_DIR}/runtime.target"
touch "${FAKE_STATE_DIR}/entry.ok"
"$RECONCILER"
assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "healthy proxy target should repair stale state"
[ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "healthy entry must not be reloaded"
grep -q 'server 10.0.0.3:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "healthy proxy target must not be changed"
cleanup_fixture
}
test_disk_config_does_not_override_verified_runtime() {
new_fixture
set_container material-blue 10.0.0.2 1
set_container material-green 10.0.0.3 1
printf 'material-blue\n' > "${PROXY_ROOT}/current-container"
write_proxy_config 10.0.0.3 material-green
printf 'material-blue\n' > "${FAKE_STATE_DIR}/runtime.target"
touch "${FAKE_STATE_DIR}/entry.ok"
"$RECONCILER"
assert_equals "material-blue" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "verified runtime should win over an un-reloaded disk config"
grep -q 'server 10.0.0.2:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "disk config should be restored to the verified runtime target"
assert_equals "material-blue" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "reload should keep the verified runtime target"
cleanup_fixture
}
test_markerless_healthy_conflict_fails_closed() {
new_fixture
set_container material-blue 10.0.0.2 1
set_container material-green 10.0.0.3 1
printf 'material-blue\n' > "${PROXY_ROOT}/current-container"
write_proxy_config 10.0.0.3 material-green
touch "${FAKE_STATE_DIR}/entry.ok"
if "$RECONCILER"; then
fail "markerless healthy state/config conflict should fail closed"
fi
assert_equals "material-blue" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "ambiguous markerless state must remain untouched"
grep -q 'server 10.0.0.3:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "ambiguous markerless config must remain untouched"
[ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "ambiguous markerless state must not reload the proxy"
cleanup_fixture
}
test_broken_entry_prefers_configured_target_over_stale_state() {
new_fixture
set_container material-blue 10.0.0.2 1
set_container material-green 10.0.0.3 1
printf 'material-blue\n' > "${PROXY_ROOT}/current-container"
write_proxy_config 10.0.0.3 material-green
export FAKE_RECOVER_ON_RELOAD=1
"$RECONCILER"
assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "broken entry should prefer the configured healthy target"
assert_equals "material-green" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "configured target should become the verified runtime"
cleanup_fixture
}
test_reload_error_with_verified_target_succeeds() {
new_fixture
set_container material-blue 10.0.0.2 1
set_container material-green 10.0.0.3 1
printf 'material-blue\n' > "${PROXY_ROOT}/current-container"
write_proxy_config 10.0.0.2 material-blue
printf 'material-blue\n' > "${FAKE_STATE_DIR}/runtime.target"
rm -f "${FAKE_STATE_DIR}/entry.ok"
# Make the disk target drift while the recorded state remains blue.
write_proxy_config 10.0.0.3 material-green
export FAKE_RECOVER_ON_RELOAD=1
export FAKE_RELOAD_FAIL_FIRST=1
"$RECONCILER"
assert_equals "material-blue" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "verified runtime should remain authoritative"
assert_equals "material-blue" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "verified target should be accepted despite the command status"
cleanup_fixture
}
test_deployment_lock_blocks_reconciliation() {
new_fixture
set_container material-green 10.0.0.3 1
printf 'material-blue\n' > "${PROXY_ROOT}/current-container"
write_proxy_config 10.0.0.4
export FAKE_FLOCK_FAIL=1
"$RECONCILER"
grep -q 'server 10.0.0.4:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "deployment lock must leave proxy configuration untouched"
[ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "reconciler must not reload while deployment lock is held"
cleanup_fixture
}
test_connection_failure_recovers_to_configured_target() {
new_fixture
set_container material-green 10.0.0.3 1
printf 'material-green\n' > "${PROXY_ROOT}/current-container"
write_proxy_config 10.0.0.3 material-green
touch "${FAKE_STATE_DIR}/entry.connect-fail"
export FAKE_RECOVER_ON_RELOAD=1
"$RECONCILER"
assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "connection failure should recover to the configured healthy target"
assert_equals "material-green" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "recovered target should be verified at runtime"
grep -q '^reload$' "${FAKE_STATE_DIR}/systemctl.log" || fail "connection failure recovery should reload the proxy"
cleanup_fixture
}
test_single_healthy_container_repairs_broken_proxy() {
new_fixture
set_container material-green 10.0.0.3 1
printf 'material-blue\n' > "${PROXY_ROOT}/current-container"
write_proxy_config 10.0.0.4
export FAKE_RECOVER_ON_RELOAD=1
"$RECONCILER"
assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "recovery should persist the selected container"
grep -q 'server 10.0.0.3:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "recovery should update the upstream address"
grep -q '^reload$' "${FAKE_STATE_DIR}/systemctl.log" || fail "recovery should reload the proxy"
cleanup_fixture
}
test_ambiguous_containers_fail_closed() {
new_fixture
set_container material-blue 10.0.0.2 1
set_container material-green 10.0.0.3 1
printf 'invalid-container\n' > "${PROXY_ROOT}/current-container"
write_proxy_config 10.0.0.4
if "$RECONCILER"; then
fail "ambiguous healthy containers should not be switched automatically"
fi
grep -q 'server 10.0.0.4:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "ambiguous state must leave proxy configuration untouched"
[ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "ambiguous state must not reload the proxy"
cleanup_fixture
}
trap 'if [ -n "${FIXTURE_ROOT:-}" ] && [ -d "${FIXTURE_ROOT}" ]; then rm -rf "$FIXTURE_ROOT"; fi' EXIT
test_healthy_entry_wins_over_stale_state
test_single_healthy_container_repairs_broken_proxy
test_ambiguous_containers_fail_closed
test_disk_config_does_not_override_verified_runtime
test_markerless_healthy_conflict_fails_closed
test_broken_entry_prefers_configured_target_over_stale_state
test_reload_error_with_verified_target_succeeds
test_deployment_lock_blocks_reconciliation
test_connection_failure_recovers_to_configured_target
printf 'PASS: local proxy reconciler scenarios\n'