diff --git a/scripts/git-incremental-deploy.ps1 b/scripts/git-incremental-deploy.ps1 index 42d8d61..60c0f0e 100644 --- a/scripts/git-incremental-deploy.ps1 +++ b/scripts/git-incremental-deploy.ps1 @@ -28,6 +28,7 @@ param( [string]$BuildServerIP = '192.168.40.206', [string]$BuildServerUser = 'root', [string]$BuildServerPassword = $env:BUILD_SERVER_PASSWORD, + [string]$BuildServerProxyCommand = $env:BUILD_SERVER_PROXY_COMMAND, [string]$RemoteBuildDir = '/root/youyidian-3d-git-build', [string]$ProdServerIP = '192.168.58.20', @@ -168,8 +169,13 @@ function Invoke-BuildServerScript { $pscpPath = $null $sshPath = $null $scpPath = $null + $sshCommonArgs = @() + $scpCommonArgs = @() if ($usePasswordAuth) { + if (-not [string]::IsNullOrWhiteSpace($BuildServerProxyCommand)) { + throw 'BuildServerProxyCommand is supported only with OpenSSH key authentication.' + } $plinkPath = Get-PlinkPath $pscpPath = Get-PscpPath } @@ -181,6 +187,12 @@ function Invoke-BuildServerScript { } $sshPath = $sshCommand.Source $scpPath = $scpCommand.Source + $sshCommonArgs = @('-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=10') + $scpCommonArgs = @('-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=10') + if (-not [string]::IsNullOrWhiteSpace($BuildServerProxyCommand)) { + $sshCommonArgs += @('-o', "ProxyCommand=$BuildServerProxyCommand") + $scpCommonArgs += @('-o', "ProxyCommand=$BuildServerProxyCommand") + } } $scriptId = [guid]::NewGuid().ToString('N') @@ -192,7 +204,15 @@ function Invoke-BuildServerScript { $normalizedScriptText = $ScriptText.Replace("`r`n", "`n").Replace("`r", "`n") [System.IO.File]::WriteAllText($localTemp, $normalizedScriptText, [System.Text.UTF8Encoding]::new($false)) - $authLabel = if ($usePasswordAuth) { 'password auth' } else { 'SSH key auth' } + $authLabel = if ($usePasswordAuth) { + 'password auth' + } + elseif (-not [string]::IsNullOrWhiteSpace($BuildServerProxyCommand)) { + 'SSH key auth via proxy command' + } + else { + 'SSH key auth' + } Write-Info "Connecting to build server $BuildServerUser@$BuildServerIP ($authLabel)" # 原生程序(plink/pscp)会把 git 等子进程的提示信息写到 stderr。 @@ -210,7 +230,7 @@ function Invoke-BuildServerScript { $copyOutput = & $pscpPath -batch -pw $BuildServerPassword $localTemp $remoteSpec 2>&1 } else { - $copyOutput = & $scpPath -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10 $localTemp $remoteSpec 2>&1 + $copyOutput = & $scpPath @scpCommonArgs $localTemp $remoteSpec 2>&1 } $copyOutput | ForEach-Object { @@ -230,7 +250,7 @@ function Invoke-BuildServerScript { $runOutput = & $plinkPath -batch -pw $BuildServerPassword "$BuildServerUser@$BuildServerIP" $command 2>&1 } else { - $runOutput = & $sshPath -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10 "$BuildServerUser@$BuildServerIP" $command 2>&1 + $runOutput = & $sshPath @sshCommonArgs "$BuildServerUser@$BuildServerIP" $command 2>&1 } $runOutput | ForEach-Object { @@ -269,6 +289,14 @@ if ($RepoUrl -match '^https?://' -and [string]::IsNullOrWhiteSpace($GitAuthHeade $cfg = $targetConfig[$Target] $rollbackFlag = if ($NoRollback) { '0' } else { '1' } +$proxyReconcilerPath = Join-Path $PSScriptRoot 'local-proxy-reconcile.sh' +if (-not (Test-Path -LiteralPath $proxyReconcilerPath)) { + throw "Local proxy reconciler is missing: $proxyReconcilerPath" +} +$proxyReconcilerText = [System.IO.File]::ReadAllText($proxyReconcilerPath) +$proxyReconcilerText = $proxyReconcilerText.Replace("`r`n", "`n").Replace("`r", "`n") +$proxyReconcilerBytes = [System.Text.UTF8Encoding]::new($false).GetBytes($proxyReconcilerText) +$proxyReconcilerBase64 = [Convert]::ToBase64String($proxyReconcilerBytes) $tokens = @{ '__REPO_URL__' = ConvertTo-ShellSingleQuoted $RepoUrl @@ -291,6 +319,7 @@ $tokens = @{ '__DINGTALK_INTERNAL_RETURN_ORIGINS__' = ConvertTo-ShellSingleQuoted $cfg.DingtalkInternalReturnOrigins '__S3_PUBLIC_DOMAIN__' = ConvertTo-ShellSingleQuoted $cfg.S3PublicDomain '__DOWNLOAD_TOKEN_SECRET__' = ConvertTo-ShellSingleQuoted $DownloadTokenSecret + '__PROXY_RECONCILE_SCRIPT_B64__' = ConvertTo-ShellSingleQuoted $proxyReconcilerBase64 '__DRY_RUN__' = ConvertTo-ShellSingleQuoted ($(if ($DryRun) { '1' } else { '0' })) '__NO_CACHE__' = ConvertTo-ShellSingleQuoted ($(if ($NoCache) { '1' } else { '0' })) '__SKIP_BUILD__' = ConvertTo-ShellSingleQuoted ($(if ($SkipBuild) { '1' } else { '0' })) @@ -324,6 +353,7 @@ DINGTALK_CALLBACK_ORIGIN=__DINGTALK_CALLBACK_ORIGIN__ DINGTALK_INTERNAL_RETURN_ORIGINS=__DINGTALK_INTERNAL_RETURN_ORIGINS__ S3_PUBLIC_DOMAIN=__S3_PUBLIC_DOMAIN__ DOWNLOAD_TOKEN_SECRET=__DOWNLOAD_TOKEN_SECRET__ +PROXY_RECONCILE_SCRIPT_B64=__PROXY_RECONCILE_SCRIPT_B64__ DRY_RUN=__DRY_RUN__ NO_CACHE=__NO_CACHE__ SKIP_BUILD=__SKIP_BUILD__ @@ -554,7 +584,12 @@ require_file() { } command -v docker >/dev/null 2>&1 || fail "Missing docker on production server" -command -v curl >/dev/null 2>&1 || warn "curl not found; health check will use docker logs only" +command -v curl >/dev/null 2>&1 || fail "Missing curl on production server" +command -v flock >/dev/null 2>&1 || fail "Missing flock on production server" +command -v ss >/dev/null 2>&1 || fail "Missing ss on production server; safe connection draining cannot be verified" +command -v base64 >/dev/null 2>&1 || fail "Missing base64 on production server" +command -v nginx >/dev/null 2>&1 || fail "Missing nginx on production server" +command -v systemctl >/dev/null 2>&1 || fail "Missing systemctl on production server" require_file "$ENV_FILE_PATH" [ -d "$UPLOADS_MOUNT" ] || fail "Uploads mount does not exist: $UPLOADS_MOUNT" @@ -573,6 +608,10 @@ PROXY_CONF="${PROXY_ROOT}/nginx.conf" PROXY_PID="${PROXY_ROOT}/nginx.pid" PROXY_SERVICE="${CONTAINER_NAME}-local-proxy.service" CURRENT_FILE="${PROXY_ROOT}/current-container" +PROXY_RECONCILE_SCRIPT="${PROXY_ROOT}/reconcile-upstream.sh" +PROXY_RECONCILE_SERVICE="${CONTAINER_NAME}-local-proxy-reconcile.service" +PROXY_RECONCILE_TIMER="${CONTAINER_NAME}-local-proxy-reconcile.timer" +PROXY_RECONCILE_LOCK="/run/${CONTAINER_NAME}-local-proxy-reconcile.lock" BLUE_CONTAINER="${CONTAINER_NAME}-blue" GREEN_CONTAINER="${CONTAINER_NAME}-green" PUBLIC_TRAFFIC_SYNC_ROOT="/mnt/PoolOne/apps/youyidian-3d/public-traffic-sync" @@ -721,8 +760,23 @@ container_ip() { write_proxy_config() { local upstream="$1" + local target_container="$2" + local temp_conf="" + local backup_conf="" + + case "$target_container" in + "$BLUE_CONTAINER"|"$GREEN_CONTAINER") ;; + *) + warn "Refusing to write proxy config for invalid container: ${target_container}" + return 1 + ;; + esac + mkdir -p "$PROXY_ROOT" - cat > "$PROXY_CONF" < "$temp_conf" < "/etc/systemd/system/${PROXY_SERVICE}" < "$service_temp" </dev/null 2>&1 || true + systemctl daemon-reload || return 1 + systemctl enable "$PROXY_SERVICE" >/dev/null 2>&1 || return 1 } switch_proxy() { - ensure_proxy_service + ensure_proxy_service || return 1 if is_proxy_running; then systemctl reload "$PROXY_SERVICE" else @@ -811,6 +899,155 @@ switch_proxy() { fi } +runtime_proxy_container() { + local target="" + + target="$( + curl -sS --max-time 5 -D - -o /dev/null "http://127.0.0.1:${PORT}/api/health" 2>/dev/null | + tr -d '\r' | + sed -nE 's/^X-Material-Proxy-Target:[[:space:]]*(.*)$/\1/Ip' | + tail -n 1 || true + )" + + case "$target" in + "$BLUE_CONTAINER"|"$GREEN_CONTAINER") printf '%s\n' "$target" ;; + *) printf '\n' ;; + esac +} + +verify_proxy_target() { + local expected_container="$1" + local observed_container="" + + for _ in $(seq 1 5); do + observed_container="$(runtime_proxy_container)" + if [ "$observed_container" = "$expected_container" ] && \ + curl -fsS --max-time 8 "http://127.0.0.1:${PORT}/api/health" >/tmp/git-deploy-entry-health.json 2>/tmp/git-deploy-entry-health.err; then + return 0 + fi + sleep 1 + done + + warn "Proxy target verification failed: expected=${expected_container}, observed=${observed_container:-missing}" + return 1 +} + +install_proxy_reconciler() { + local script_temp="" + local service_file="/etc/systemd/system/${PROXY_RECONCILE_SERVICE}" + local service_temp="" + local timer_file="/etc/systemd/system/${PROXY_RECONCILE_TIMER}" + local timer_temp="" + + mkdir -p "$PROXY_ROOT" || return 1 + script_temp="$(mktemp "${PROXY_RECONCILE_SCRIPT}.tmp.XXXXXX")" + printf '%s' "$PROXY_RECONCILE_SCRIPT_B64" | base64 -d > "$script_temp" + bash -n "$script_temp" + chmod 700 "$script_temp" + mv "$script_temp" "$PROXY_RECONCILE_SCRIPT" + + service_temp="$(mktemp "${service_file}.tmp.XXXXXX")" + cat > "$service_temp" < "$timer_temp" </dev/null 2>&1 || return 1 + systemctl restart "$PROXY_RECONCILE_TIMER" +} + +container_running() { + [ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || true)" = "true" ] +} + +read_recorded_container() { + local name="" + + if [ -s "$CURRENT_FILE" ]; then + name="$(tr -d '\r\n' < "$CURRENT_FILE")" + fi + + case "$name" in + "$BLUE_CONTAINER"|"$GREEN_CONTAINER") printf '%s\n' "$name" ;; + *) printf '\n' ;; + esac +} + +write_current_container() { + local name="$1" + local temp_file="" + + mkdir -p "$PROXY_ROOT" + temp_file="$(mktemp "${CURRENT_FILE}.tmp.XXXXXX")" + printf '%s\n' "$name" > "$temp_file" + chmod 0644 "$temp_file" + mv "$temp_file" "$CURRENT_FILE" +} + +discover_current_container() { + local recorded_container="" + local runtime_container="" + local running_containers=() + local name="" + + recorded_container="$(read_recorded_container)" + runtime_container="$(runtime_proxy_container)" + + if [ -n "$runtime_container" ] && container_running "$runtime_container"; then + printf '%s\n' "$runtime_container" + return 0 + fi + + for name in "$BLUE_CONTAINER" "$GREEN_CONTAINER"; do + if container_running "$name"; then + running_containers+=("$name") + fi + done + + if [ "${#running_containers[@]}" -eq 1 ]; then + printf '%s\n' "${running_containers[0]}" + return 0 + fi + + if [ "${#running_containers[@]}" -gt 1 ]; then + warn "Multiple running material containers found without a verified runtime proxy target (recorded=${recorded_container:-missing})" + return 1 + fi + + printf '\n' +} + stop_proxy() { systemctl stop "$PROXY_SERVICE" >/dev/null 2>&1 || true } @@ -834,9 +1071,150 @@ wait_for_container_health() { return 1 } -CURRENT_CONTAINER="" -if [ -f "$CURRENT_FILE" ]; then - CURRENT_CONTAINER="$(cat "$CURRENT_FILE" 2>/dev/null || true)" +container_healthy() { + local name="$1" + local ip="" + + container_running "$name" || return 1 + ip="$(container_ip "$name")" + [ -n "$ip" ] || return 1 + curl -fsS --max-time 8 "http://${ip}:3000/api/health" >/dev/null 2>&1 +} + +container_upstream_connection_count() { + local name="$1" + local ip="" + + command -v ss >/dev/null 2>&1 || return 2 + container_running "$name" || { + printf '0\n' + return 0 + } + ip="$(container_ip "$name")" + [ -n "$ip" ] || return 2 + + # Let ss filter the peer endpoint itself; column positions differ between + # iproute2 versions when the State column is omitted by -H. + ss -Htan state established dst "${ip}:3000" 2>/dev/null | + awk 'END { print NR + 0 }' +} + +wait_for_container_drain() { + local name="$1" + local max_checks="${2:-15}" + local count="" + local zero_samples=0 + + for _ in $(seq 1 "$max_checks"); do + if ! count="$(container_upstream_connection_count "$name")"; then + warn "Cannot prove that upstream connections drained for ${name}; leaving it running" + return 1 + fi + + if [ "$count" = "0" ]; then + zero_samples=$((zero_samples + 1)) + if [ "$zero_samples" -ge 2 ]; then + return 0 + fi + else + zero_samples=0 + log "Waiting for ${count} active upstream connection(s) to drain from ${name}" + fi + sleep 2 + done + + warn "Active upstream connections did not drain from ${name}; leaving it running" + return 1 +} + +rollback_to_previous_entry() { + local previous_ip="" + + [ "$ROLLBACK_ON_FAILURE" = "1" ] || return 1 + + if [ "$PROXY_WAS_RUNNING" = "1" ] && [ -n "$CURRENT_CONTAINER" ]; then + if ! container_running "$CURRENT_CONTAINER"; then + if ! docker start "$CURRENT_CONTAINER" >/dev/null; then + warn "Failed to restart previous container $CURRENT_CONTAINER" + return 1 + fi + if ! wait_for_container_health "$CURRENT_CONTAINER" >/dev/null; then + warn "Previous container did not become healthy: $CURRENT_CONTAINER" + return 1 + fi + elif ! container_healthy "$CURRENT_CONTAINER"; then + warn "Previous container is not healthy: $CURRENT_CONTAINER" + return 1 + fi + + previous_ip="$(container_ip "$CURRENT_CONTAINER")" + [ -n "$previous_ip" ] || return 1 + if ! write_proxy_config "${previous_ip}:3000" "$CURRENT_CONTAINER"; then + return 1 + fi + if ! switch_proxy; then + return 1 + fi + if ! verify_proxy_target "$CURRENT_CONTAINER"; then + return 1 + fi + write_current_container "$CURRENT_CONTAINER" + return 0 + fi + + if [ "$PROXY_WAS_RUNNING" != "1" ] && [ -n "$LEGACY_IMAGE" ]; then + if ! wait_for_container_drain "$NEXT_CONTAINER" 15; then + warn "Candidate still has an in-flight request; direct-container rollback was not attempted" + return 1 + fi + stop_proxy >/dev/null 2>&1 || true + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + if ! start_container "$CONTAINER_NAME" "$LEGACY_IMAGE" 1 >/tmp/git-deploy-rollback-container-id.txt; then + return 1 + fi + cat /tmp/git-deploy-rollback-container-id.txt + for _ in $(seq 1 12); do + if ! is_proxy_running && curl -fsS --max-time 8 "http://127.0.0.1:${PORT}/api/health" >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + fi + + return 1 +} + +abort_after_uncertain_switch() { + local reason="$1" + + warn "$reason" + if rollback_to_previous_entry; then + ok "Previous entry was restored and verified" + if wait_for_container_drain "$NEXT_CONTAINER" 15; then + docker rm -f "$NEXT_CONTAINER" >/dev/null 2>&1 || true + else + warn "Rolled-back candidate was retained because an in-flight request is still using it" + fi + fail "$reason; deployment was rolled back" + fi + + warn "Rollback could not be verified; candidate container is being kept for recovery" + fail "$reason; both containers were preserved" +} + +mkdir -p "$PROXY_ROOT" +exec 8>"$PROXY_RECONCILE_LOCK" +flock -w 30 8 || fail "Another proxy switch or reconciliation is already running" +install_proxy_reconciler + +RECORDED_CONTAINER="$(read_recorded_container)" +if ! CURRENT_CONTAINER="$(discover_current_container)"; then + fail "Cannot safely determine the active material container" +fi + +if [ -n "$CURRENT_CONTAINER" ] && [ "$CURRENT_CONTAINER" != "$RECORDED_CONTAINER" ]; then + warn "Repairing stale current-container state: ${RECORDED_CONTAINER:-missing} -> ${CURRENT_CONTAINER}" + write_current_container "$CURRENT_CONTAINER" fi if [ "$CURRENT_CONTAINER" = "$BLUE_CONTAINER" ]; then @@ -858,6 +1236,9 @@ fi ensure_public_traffic_rollup_token log "Starting candidate container $NEXT_CONTAINER" +if container_running "$NEXT_CONTAINER" && ! wait_for_container_drain "$NEXT_CONTAINER" 15; then + fail "Inactive deployment slot $NEXT_CONTAINER is still serving requests; retry after those requests finish" +fi docker rm -f "$NEXT_CONTAINER" >/dev/null 2>&1 || true start_container "$NEXT_CONTAINER" "$REGISTRY_COMMIT" 0 >/tmp/git-deploy-new-container-id.txt cat /tmp/git-deploy-new-container-id.txt @@ -870,7 +1251,10 @@ if ! NEXT_IP="$(wait_for_container_health "$NEXT_CONTAINER")"; then fi ok "Candidate container is healthy at ${NEXT_IP}:3000" -write_proxy_config "${NEXT_IP}:3000" +if ! write_proxy_config "${NEXT_IP}:3000" "$NEXT_CONTAINER"; then + docker rm -f "$NEXT_CONTAINER" >/dev/null 2>&1 || true + fail "Candidate proxy configuration could not be prepared; existing entry was kept" +fi if [ "$PROXY_WAS_RUNNING" != "1" ]; then warn "Stable local proxy is not running; migrating public entry from direct container to proxy" @@ -880,39 +1264,53 @@ if [ "$PROXY_WAS_RUNNING" != "1" ]; then fi fi +SWITCH_REPORTED_FAILURE=0 if ! switch_proxy; then - warn "Local proxy switch failed" - if [ "$PROXY_WAS_RUNNING" != "1" ] && [ "$ROLLBACK_ON_FAILURE" = "1" ] && [ -n "$LEGACY_IMAGE" ]; then - warn "Rolling back to direct container $LEGACY_IMAGE" - stop_proxy - start_container "$CONTAINER_NAME" "$LEGACY_IMAGE" 1 >/tmp/git-deploy-rollback-container-id.txt - cat /tmp/git-deploy-rollback-container-id.txt - fi - docker rm -f "$NEXT_CONTAINER" >/dev/null 2>&1 || true - fail "Deployment failed during local proxy switch" + SWITCH_REPORTED_FAILURE=1 fi -if ! curl -fsS --max-time 8 "http://127.0.0.1:${PORT}/api/health" >/tmp/git-deploy-entry-health.json 2>/tmp/git-deploy-entry-health.err; then - warn "Stable local entry health check failed after switch" - cat /tmp/git-deploy-entry-health.err >&2 || true - if [ "$PROXY_WAS_RUNNING" != "1" ] && [ "$ROLLBACK_ON_FAILURE" = "1" ] && [ -n "$LEGACY_IMAGE" ]; then - warn "Rolling back to direct container $LEGACY_IMAGE" - stop_proxy - start_container "$CONTAINER_NAME" "$LEGACY_IMAGE" 1 >/tmp/git-deploy-rollback-container-id.txt - cat /tmp/git-deploy-rollback-container-id.txt - elif [ "$PROXY_WAS_RUNNING" = "1" ] && [ -n "$CURRENT_CONTAINER" ]; then - CURRENT_IP="$(container_ip "$CURRENT_CONTAINER")" - if [ -n "$CURRENT_IP" ]; then - warn "Reloading proxy back to previous container $CURRENT_CONTAINER" - write_proxy_config "${CURRENT_IP}:3000" >/dev/null 2>&1 || true - switch_proxy >/dev/null 2>&1 || true +if ! verify_proxy_target "$NEXT_CONTAINER"; then + cat /tmp/git-deploy-entry-health.err >&2 2>/dev/null || true + abort_after_uncertain_switch "Stable local entry did not verify the candidate target" +fi + +if [ "$SWITCH_REPORTED_FAILURE" = "1" ]; then + warn "Proxy switch command reported failure, but the runtime candidate target was independently verified" +fi + +write_current_container "$NEXT_CONTAINER" + +sleep 3 +if ! verify_proxy_target "$NEXT_CONTAINER"; then + abort_after_uncertain_switch "Candidate target did not remain healthy during the post-switch check" +fi + +if [ -n "$CURRENT_CONTAINER" ] && [ "$CURRENT_CONTAINER" != "$NEXT_CONTAINER" ]; then + if [ "$(runtime_proxy_container)" != "$NEXT_CONTAINER" ] || \ + [ "$(read_recorded_container)" != "$NEXT_CONTAINER" ] || \ + ! container_healthy "$NEXT_CONTAINER"; then + abort_after_uncertain_switch "Proxy state changed before the previous container could be drained" + fi + + if wait_for_container_drain "$CURRENT_CONTAINER" 15; then + if docker stop --time 30 "$CURRENT_CONTAINER" >/dev/null 2>&1; then + ok "Previous container stopped and retained for rollback: $CURRENT_CONTAINER" + else + warn "Previous container could not be stopped and was retained: $CURRENT_CONTAINER" fi + else + warn "Previous container is still serving an in-flight request and was retained: $CURRENT_CONTAINER" fi - docker rm -f "$NEXT_CONTAINER" >/dev/null 2>&1 || true - fail "Deployment failed after local proxy switch" fi -printf '%s\n' "$NEXT_CONTAINER" > "$CURRENT_FILE" +if ! verify_proxy_target "$NEXT_CONTAINER"; then + abort_after_uncertain_switch "Candidate target failed after the previous container drain" +fi + +flock -u 8 +if ! systemctl start "$PROXY_RECONCILE_SERVICE"; then + fail "Proxy reconciler failed its post-deployment verification" +fi ok "Stable local entry is healthy on port $PORT" docker logs --tail 20 "$NEXT_CONTAINER" 2>&1 || true @@ -938,10 +1336,6 @@ else warn "Public traffic statistics rollup timers were not installed" fi -if [ -n "$CURRENT_CONTAINER" ] && [ "$CURRENT_CONTAINER" != "$NEXT_CONTAINER" ]; then - docker rm -f "$CURRENT_CONTAINER" >/dev/null 2>&1 || true -fi - if [ -n "$LEGACY_IMAGE" ]; then docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true fi @@ -950,7 +1344,7 @@ PROD_SCRIPT export_script="$(printf '%s' "$PROD_PAYLOAD" | base64 -w 0)" ssh -o StrictHostKeyChecking=no "$PROD_SERVER_USER@$PROD_SERVER_IP" \ - "TARGET='$TARGET' CONTAINER_NAME='$CONTAINER_NAME' PORT='$PORT' REGISTRY_COMMIT='$REGISTRY_COMMIT' COMMIT_SHA='$COMMIT_SHA' ENV_FILE_PATH='/mnt/PoolOne/apps/youyidian-3d/$ENV_FILE' PUBLIC_APP_URL='$PUBLIC_APP_URL' CORS_ALLOWED_ORIGINS='$CORS_ALLOWED_ORIGINS' DINGTALK_CALLBACK_ORIGIN='$DINGTALK_CALLBACK_ORIGIN' DINGTALK_INTERNAL_RETURN_ORIGINS='$DINGTALK_INTERNAL_RETURN_ORIGINS' S3_PUBLIC_DOMAIN='$S3_PUBLIC_DOMAIN' DOWNLOAD_TOKEN_SECRET='$DOWNLOAD_TOKEN_SECRET' NETWORK_NAME='youyidian-3d_app-network' UPLOADS_MOUNT='/mnt/PoolOne/apps/youyidian-3d/uploads' ROLLBACK_ON_FAILURE='$ROLLBACK_ON_FAILURE' bash -lc \"printf '%s' '$export_script' | base64 -d | bash\"" + "TARGET='$TARGET' CONTAINER_NAME='$CONTAINER_NAME' PORT='$PORT' REGISTRY_COMMIT='$REGISTRY_COMMIT' COMMIT_SHA='$COMMIT_SHA' ENV_FILE_PATH='/mnt/PoolOne/apps/youyidian-3d/$ENV_FILE' PUBLIC_APP_URL='$PUBLIC_APP_URL' CORS_ALLOWED_ORIGINS='$CORS_ALLOWED_ORIGINS' DINGTALK_CALLBACK_ORIGIN='$DINGTALK_CALLBACK_ORIGIN' DINGTALK_INTERNAL_RETURN_ORIGINS='$DINGTALK_INTERNAL_RETURN_ORIGINS' S3_PUBLIC_DOMAIN='$S3_PUBLIC_DOMAIN' DOWNLOAD_TOKEN_SECRET='$DOWNLOAD_TOKEN_SECRET' PROXY_RECONCILE_SCRIPT_B64='$PROXY_RECONCILE_SCRIPT_B64' NETWORK_NAME='youyidian-3d_app-network' UPLOADS_MOUNT='/mnt/PoolOne/apps/youyidian-3d/uploads' ROLLBACK_ON_FAILURE='$ROLLBACK_ON_FAILURE' bash -lc \"printf '%s' '$export_script' | base64 -d | bash\"" ok "Production deployment command completed" if [ "$TARGET" = "production" ]; then @@ -980,7 +1374,7 @@ else fi ssh -o StrictHostKeyChecking=no "$PROD_SERVER_USER@$PROD_SERVER_IP" \ - "docker image prune -af --filter until=6h >/dev/null 2>&1 || true; docker container prune -f --filter until=6h >/dev/null 2>&1 || true" + "docker image prune -af --filter until=6h >/dev/null 2>&1 || true" ok "Production server cleanup completed" fi diff --git a/scripts/local-proxy-reconcile.sh b/scripts/local-proxy-reconcile.sh new file mode 100644 index 0000000..b5fc6eb --- /dev/null +++ b/scripts/local-proxy-reconcile.sh @@ -0,0 +1,325 @@ +#!/usr/bin/env bash +set -euo pipefail + +CONTAINER_NAME="${CONTAINER_NAME:?CONTAINER_NAME is required}" +PORT="${PORT:?PORT is required}" +PROXY_ROOT="${PROXY_ROOT:?PROXY_ROOT is required}" +PROXY_SERVICE="${PROXY_SERVICE:?PROXY_SERVICE is required}" + +CURRENT_FILE="${PROXY_ROOT}/current-container" +PROXY_CONF="${PROXY_ROOT}/nginx.conf" +LOCK_FILE="${PROXY_RECONCILE_LOCK_FILE:-/run/${CONTAINER_NAME}-local-proxy-reconcile.lock}" +NGINX_BIN="${NGINX_BIN:-/usr/sbin/nginx}" +BLUE_CONTAINER="${CONTAINER_NAME}-blue" +GREEN_CONTAINER="${CONTAINER_NAME}-green" + +log() { + printf '%s [%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$1" "$2" +} + +container_running() { + [ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || true)" = "true" ] +} + +container_ip() { + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1" 2>/dev/null || true +} + +container_healthy() { + local name="$1" + local ip="" + + container_running "$name" || return 1 + ip="$(container_ip "$name")" + [ -n "$ip" ] || return 1 + curl -fsS --max-time 5 "http://${ip}:3000/api/health" >/dev/null 2>&1 +} + +container_for_ip() { + local expected_ip="$1" + local name="" + local ip="" + + [ -n "$expected_ip" ] || return 1 + for name in "$BLUE_CONTAINER" "$GREEN_CONTAINER"; do + container_running "$name" || continue + ip="$(container_ip "$name")" + if [ "$ip" = "$expected_ip" ]; then + printf '%s\n' "$name" + return 0 + fi + done + + return 1 +} + +read_current_container() { + local name="" + + if [ -s "$CURRENT_FILE" ]; then + name="$(tr -d '\r\n' < "$CURRENT_FILE")" + fi + + case "$name" in + "$BLUE_CONTAINER"|"$GREEN_CONTAINER") printf '%s\n' "$name" ;; + *) printf '\n' ;; + esac +} + +write_current_container() { + local name="$1" + local temp_file="" + + temp_file="$(mktemp "${CURRENT_FILE}.tmp.XXXXXX")" + printf '%s\n' "$name" > "$temp_file" + chmod 0644 "$temp_file" + mv "$temp_file" "$CURRENT_FILE" +} + +configured_ip() { + sed -nE 's/^[[:space:]]*server ([0-9.]+):3000 .*/\1/p' "$PROXY_CONF" | head -n 1 +} + +configured_header_container() { + local target="" + + target="$( + sed -nE 's/^[[:space:]]*add_header X-Material-Proxy-Target ([^;[:space:]]+) always;.*/\1/p' "$PROXY_CONF" | + tail -n 1 + )" + case "$target" in + "$BLUE_CONTAINER"|"$GREEN_CONTAINER") printf '%s\n' "$target" ;; + *) printf '\n' ;; + esac +} + +configured_container() { + local header_container="" + local ip_container="" + + header_container="$(configured_header_container)" + ip_container="$(container_for_ip "$(configured_ip)" 2>/dev/null || true)" + + if [ -n "$header_container" ] && [ -n "$ip_container" ] && [ "$header_container" != "$ip_container" ]; then + printf '\n' + elif [ -n "$header_container" ]; then + printf '%s\n' "$header_container" + else + printf '%s\n' "$ip_container" + fi +} + +runtime_proxy_container() { + local target="" + + target="$( + curl -sS --max-time 5 -D - -o /dev/null "http://127.0.0.1:${PORT}/api/health" 2>/dev/null | + tr -d '\r' | + sed -nE 's/^X-Material-Proxy-Target:[[:space:]]*(.*)$/\1/Ip' | + tail -n 1 || true + )" + case "$target" in + "$BLUE_CONTAINER"|"$GREEN_CONTAINER") printf '%s\n' "$target" ;; + *) printf '\n' ;; + esac +} + +entry_healthy() { + curl -fsS --max-time 5 "http://127.0.0.1:${PORT}/api/health" >/dev/null 2>&1 +} + +verify_runtime_target() { + local expected_container="$1" + local observed_container="" + + for _ in $(seq 1 5); do + observed_container="$(runtime_proxy_container)" + if [ "$observed_container" = "$expected_container" ] && entry_healthy; then + return 0 + fi + sleep 1 + done + + return 1 +} + +reload_proxy() { + if systemctl is-active --quiet "$PROXY_SERVICE"; then + systemctl reload "$PROXY_SERVICE" + else + systemctl start "$PROXY_SERVICE" + fi +} + +backup="" +write_proxy_config_for_target() { + local target_container="$1" + local target_ip="$2" + local temp_conf="" + + backup="" + temp_conf="$(mktemp "${PROXY_CONF}.tmp.XXXXXX")" + trap 'if [ -n "${temp_conf:-}" ]; then rm -f "$temp_conf"; fi' EXIT + + if ! sed -E \ + -e "0,/server [0-9.]+:3000 /s//server ${target_ip}:3000 /" \ + -e '/^[[:space:]]*proxy_hide_header X-Material-Proxy-Target;[[:space:]]*$/d' \ + -e '/^[[:space:]]*add_header X-Material-Proxy-Target [^;]+ always;[[:space:]]*$/d' \ + "$PROXY_CONF" | + awk -v target="$target_container" ' + { print } + /^[[:space:]]*location \/ \{/ { + print " proxy_hide_header X-Material-Proxy-Target;" + print " add_header X-Material-Proxy-Target " target " always;" + } + ' > "$temp_conf"; then + return 1 + fi + + if ! grep -q "server ${target_ip}:3000 " "$temp_conf" || + ! grep -q "add_header X-Material-Proxy-Target ${target_container} always;" "$temp_conf"; then + log "ERROR" "failed to prepare nginx configuration for ${target_container}" + return 1 + fi + + if ! "$NGINX_BIN" -t -p "${PROXY_ROOT}/" -c "$temp_conf"; then + return 1 + fi + + backup="${PROXY_CONF}.bak.reconcile.$(date +%Y%m%d-%H%M%S)" + cp -a "$PROXY_CONF" "$backup" || return 1 + mv "$temp_conf" "$PROXY_CONF" || return 1 + trap - EXIT + temp_conf="" +} + +restore_previous_proxy() { + local restore_temp="" + + if [ -n "$backup" ] && [ -f "$backup" ]; then + restore_temp="$(mktemp "${PROXY_CONF}.restore.XXXXXX")" || return 1 + if ! cp -a "$backup" "$restore_temp"; then + rm -f "$restore_temp" + return 1 + fi + if ! "$NGINX_BIN" -t -p "${PROXY_ROOT}/" -c "$restore_temp" >/dev/null 2>&1; then + rm -f "$restore_temp" + return 1 + fi + if ! mv "$restore_temp" "$PROXY_CONF"; then + rm -f "$restore_temp" + return 1 + fi + reload_proxy >/dev/null 2>&1 || return 1 + fi +} + +command -v flock >/dev/null 2>&1 || { + log "ERROR" "flock is required" + exit 1 +} +command -v awk >/dev/null 2>&1 || { + log "ERROR" "awk is required" + exit 1 +} + +exec 9>"$LOCK_FILE" +flock -n 9 || exit 0 + +if [ ! -s "$PROXY_CONF" ]; then + log "WARN" "proxy configuration is missing: ${PROXY_CONF}" + exit 1 +fi + +current_container="$(read_current_container)" +configured_container_value="$(configured_container)" +runtime_container="$(runtime_proxy_container)" +entry_ok=0 +if entry_healthy; then + entry_ok=1 +fi + +target_container="" +if [ "$entry_ok" = "1" ]; then + if [ -n "$runtime_container" ] && container_healthy "$runtime_container"; then + target_container="$runtime_container" + elif [ -n "$current_container" ] && + [ -n "$configured_container_value" ] && + [ "$current_container" != "$configured_container_value" ] && + container_healthy "$current_container" && + container_healthy "$configured_container_value"; then + log "ERROR" "healthy entry has no runtime marker and state/config point to different healthy containers" + exit 1 + elif [ -n "$current_container" ] && container_healthy "$current_container"; then + # Without a runtime marker, preserve the last verified target instead of + # trusting a disk config that may have been written before a failed reload. + target_container="$current_container" + elif [ -n "$configured_container_value" ] && container_healthy "$configured_container_value"; then + target_container="$configured_container_value" + fi +else + if [ -n "$runtime_container" ] && container_healthy "$runtime_container"; then + target_container="$runtime_container" + elif [ -n "$configured_container_value" ] && container_healthy "$configured_container_value"; then + target_container="$configured_container_value" + elif [ -n "$current_container" ] && container_healthy "$current_container"; then + target_container="$current_container" + fi +fi + +if [ -z "$target_container" ]; then + healthy_containers=() + for name in "$BLUE_CONTAINER" "$GREEN_CONTAINER"; do + if container_healthy "$name"; then + healthy_containers+=("$name") + fi + done + + if [ "${#healthy_containers[@]}" -eq 1 ]; then + target_container="${healthy_containers[0]}" + elif [ "${#healthy_containers[@]}" -gt 1 ]; then + log "ERROR" "multiple healthy containers found without an authoritative runtime target" + exit 1 + fi +fi + +if [ -z "$target_container" ]; then + log "WARN" "no healthy material container is available" + exit 1 +fi + +target_ip="$(container_ip "$target_container")" +if [ -z "$target_ip" ]; then + log "WARN" "target container has no network address: ${target_container}" + exit 1 +fi + +if [ "$entry_ok" = "1" ] && [ "$runtime_container" = "$target_container" ] && [ "$configured_container_value" = "$target_container" ]; then + if [ "$current_container" != "$target_container" ]; then + write_current_container "$target_container" + log "OK" "state aligned to verified runtime target: ${target_container}" + fi + exit 0 +fi + +if ! write_proxy_config_for_target "$target_container" "$target_ip"; then + log "ERROR" "failed to prepare proxy configuration for ${target_container}" + exit 1 +fi + +reload_reported_failure=0 +if ! reload_proxy; then + reload_reported_failure=1 +fi + +if ! verify_runtime_target "$target_container"; then + restore_previous_proxy || true + log "ERROR" "failed to verify proxy target ${target_container}; previous configuration was retained when possible" + exit 1 +fi + +write_current_container "$target_container" +if [ "$reload_reported_failure" = "1" ]; then + log "WARN" "reload command reported failure, but the runtime target was independently verified" +fi +log "OK" "proxy reconciled to verified runtime target: ${target_container} (${target_ip})" diff --git a/scripts/local-proxy-reconcile.test.sh b/scripts/local-proxy-reconcile.test.sh new file mode 100644 index 0000000..76ab5d5 --- /dev/null +++ b/scripts/local-proxy-reconcile.test.sh @@ -0,0 +1,317 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RECONCILER="${SCRIPT_DIR}/local-proxy-reconcile.sh" +ORIGINAL_PATH="$PATH" + +fail() { + printf 'FAIL: %s\n' "$1" >&2 + exit 1 +} + +assert_equals() { + local expected="$1" + local actual="$2" + local message="$3" + + if [ "$expected" != "$actual" ]; then + fail "${message}: expected=${expected}, actual=${actual}" + fi +} + +new_fixture() { + FIXTURE_ROOT="$(mktemp -d)" + export FIXTURE_ROOT + export FAKE_STATE_DIR="${FIXTURE_ROOT}/state" + export PROXY_ROOT="${FIXTURE_ROOT}/proxy" + export PROXY_RECONCILE_LOCK_FILE="${FIXTURE_ROOT}/reconcile.lock" + export NGINX_BIN="${FIXTURE_ROOT}/bin/nginx" + export CONTAINER_NAME="material" + export PROXY_SERVICE="material-local-proxy.service" + export PORT="4320" + export PATH="${FIXTURE_ROOT}/bin:${ORIGINAL_PATH}" + mkdir -p "$FAKE_STATE_DIR" "$PROXY_ROOT" "${FIXTURE_ROOT}/bin" + + cat > "${FIXTURE_ROOT}/bin/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +template="$3" +name="$4" +if [[ "$template" == *'.State.Running'* ]]; then + cat "${FAKE_STATE_DIR}/${name}.running" 2>/dev/null || printf 'false\n' +else + cat "${FAKE_STATE_DIR}/${name}.ip" 2>/dev/null || true +fi +EOF + + cat > "${FIXTURE_ROOT}/bin/curl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +url="${!#}" +if [[ "$url" == http://127.0.0.1:* ]]; then + if [ -f "${FAKE_STATE_DIR}/entry.connect-fail" ]; then + exit 7 + fi + if printf '%s\n' "$@" | grep -qx -- '-D'; then + printf 'HTTP/1.1 %s Test\r\n' "$([ -f "${FAKE_STATE_DIR}/entry.ok" ] && printf '200' || printf '502')" + if [ -s "${FAKE_STATE_DIR}/runtime.target" ]; then + printf 'X-Material-Proxy-Target: %s\r\n' "$(cat "${FAKE_STATE_DIR}/runtime.target")" + fi + printf '\r\n' + exit 0 + fi + [ -f "${FAKE_STATE_DIR}/entry.ok" ] + exit +fi +ip="${url#http://}" +ip="${ip%%:*}" +[ -f "${FAKE_STATE_DIR}/healthy.${ip}" ] +EOF + + cat > "${FIXTURE_ROOT}/bin/systemctl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +case "$1" in + is-active) exit 0 ;; + reload|start) + printf '%s\n' "$1" >> "${FAKE_STATE_DIR}/systemctl.log" + target="$(sed -nE 's/^[[:space:]]*add_header X-Material-Proxy-Target ([^;[:space:]]+) always;.*/\1/p' "${PROXY_ROOT}/nginx.conf" | tail -n 1)" + if [ -n "$target" ]; then + printf '%s\n' "$target" > "${FAKE_STATE_DIR}/runtime.target" + fi + if [ "${FAKE_RECOVER_ON_RELOAD:-0}" = "1" ]; then + rm -f "${FAKE_STATE_DIR}/entry.connect-fail" + touch "${FAKE_STATE_DIR}/entry.ok" + fi + count=0 + if [ -f "${FAKE_STATE_DIR}/reload.count" ]; then + count="$(cat "${FAKE_STATE_DIR}/reload.count")" + fi + count=$((count + 1)) + printf '%s\n' "$count" > "${FAKE_STATE_DIR}/reload.count" + if [ "${FAKE_RELOAD_FAIL_FIRST:-0}" = "1" ] && [ "$count" -eq 1 ]; then + exit 1 + fi + ;; +esac +EOF + + cat > "${FIXTURE_ROOT}/bin/nginx" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF + + cat > "${FIXTURE_ROOT}/bin/sleep" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF + + cat > "${FIXTURE_ROOT}/bin/flock" <<'EOF' +#!/usr/bin/env bash +[ "${FAKE_FLOCK_FAIL:-0}" = "1" ] && exit 1 +exit 0 +EOF + + chmod +x "${FIXTURE_ROOT}/bin/"* +} + +write_proxy_config() { + local ip="$1" + local target="${2:-}" + cat > "${PROXY_ROOT}/nginx.conf" < "${FAKE_STATE_DIR}/${name}.running" + printf '%s\n' "$ip" > "${FAKE_STATE_DIR}/${name}.ip" + if [ "$healthy" = "1" ]; then + touch "${FAKE_STATE_DIR}/healthy.${ip}" + fi +} + +cleanup_fixture() { + PATH="$ORIGINAL_PATH" + unset FAKE_RECOVER_ON_RELOAD || true + unset FAKE_RELOAD_FAIL_FIRST || true + unset FAKE_FLOCK_FAIL || true + rm -rf "$FIXTURE_ROOT" +} + +test_healthy_entry_wins_over_stale_state() { + new_fixture + set_container material-blue 10.0.0.2 1 + set_container material-green 10.0.0.3 1 + printf 'material-blue\n' > "${PROXY_ROOT}/current-container" + write_proxy_config 10.0.0.3 material-green + printf 'material-green\n' > "${FAKE_STATE_DIR}/runtime.target" + touch "${FAKE_STATE_DIR}/entry.ok" + + "$RECONCILER" + + assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "healthy proxy target should repair stale state" + [ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "healthy entry must not be reloaded" + grep -q 'server 10.0.0.3:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "healthy proxy target must not be changed" + cleanup_fixture +} + +test_disk_config_does_not_override_verified_runtime() { + new_fixture + set_container material-blue 10.0.0.2 1 + set_container material-green 10.0.0.3 1 + printf 'material-blue\n' > "${PROXY_ROOT}/current-container" + write_proxy_config 10.0.0.3 material-green + printf 'material-blue\n' > "${FAKE_STATE_DIR}/runtime.target" + touch "${FAKE_STATE_DIR}/entry.ok" + + "$RECONCILER" + + assert_equals "material-blue" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "verified runtime should win over an un-reloaded disk config" + grep -q 'server 10.0.0.2:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "disk config should be restored to the verified runtime target" + assert_equals "material-blue" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "reload should keep the verified runtime target" + cleanup_fixture +} + +test_markerless_healthy_conflict_fails_closed() { + new_fixture + set_container material-blue 10.0.0.2 1 + set_container material-green 10.0.0.3 1 + printf 'material-blue\n' > "${PROXY_ROOT}/current-container" + write_proxy_config 10.0.0.3 material-green + touch "${FAKE_STATE_DIR}/entry.ok" + + if "$RECONCILER"; then + fail "markerless healthy state/config conflict should fail closed" + fi + + assert_equals "material-blue" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "ambiguous markerless state must remain untouched" + grep -q 'server 10.0.0.3:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "ambiguous markerless config must remain untouched" + [ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "ambiguous markerless state must not reload the proxy" + cleanup_fixture +} + +test_broken_entry_prefers_configured_target_over_stale_state() { + new_fixture + set_container material-blue 10.0.0.2 1 + set_container material-green 10.0.0.3 1 + printf 'material-blue\n' > "${PROXY_ROOT}/current-container" + write_proxy_config 10.0.0.3 material-green + export FAKE_RECOVER_ON_RELOAD=1 + + "$RECONCILER" + + assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "broken entry should prefer the configured healthy target" + assert_equals "material-green" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "configured target should become the verified runtime" + cleanup_fixture +} + +test_reload_error_with_verified_target_succeeds() { + new_fixture + set_container material-blue 10.0.0.2 1 + set_container material-green 10.0.0.3 1 + printf 'material-blue\n' > "${PROXY_ROOT}/current-container" + write_proxy_config 10.0.0.2 material-blue + printf 'material-blue\n' > "${FAKE_STATE_DIR}/runtime.target" + rm -f "${FAKE_STATE_DIR}/entry.ok" + # Make the disk target drift while the recorded state remains blue. + write_proxy_config 10.0.0.3 material-green + export FAKE_RECOVER_ON_RELOAD=1 + export FAKE_RELOAD_FAIL_FIRST=1 + + "$RECONCILER" + + assert_equals "material-blue" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "verified runtime should remain authoritative" + assert_equals "material-blue" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "verified target should be accepted despite the command status" + cleanup_fixture +} + +test_deployment_lock_blocks_reconciliation() { + new_fixture + set_container material-green 10.0.0.3 1 + printf 'material-blue\n' > "${PROXY_ROOT}/current-container" + write_proxy_config 10.0.0.4 + export FAKE_FLOCK_FAIL=1 + + "$RECONCILER" + + grep -q 'server 10.0.0.4:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "deployment lock must leave proxy configuration untouched" + [ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "reconciler must not reload while deployment lock is held" + cleanup_fixture +} + +test_connection_failure_recovers_to_configured_target() { + new_fixture + set_container material-green 10.0.0.3 1 + printf 'material-green\n' > "${PROXY_ROOT}/current-container" + write_proxy_config 10.0.0.3 material-green + touch "${FAKE_STATE_DIR}/entry.connect-fail" + export FAKE_RECOVER_ON_RELOAD=1 + + "$RECONCILER" + + assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "connection failure should recover to the configured healthy target" + assert_equals "material-green" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "recovered target should be verified at runtime" + grep -q '^reload$' "${FAKE_STATE_DIR}/systemctl.log" || fail "connection failure recovery should reload the proxy" + cleanup_fixture +} + +test_single_healthy_container_repairs_broken_proxy() { + new_fixture + set_container material-green 10.0.0.3 1 + printf 'material-blue\n' > "${PROXY_ROOT}/current-container" + write_proxy_config 10.0.0.4 + export FAKE_RECOVER_ON_RELOAD=1 + + "$RECONCILER" + + assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "recovery should persist the selected container" + grep -q 'server 10.0.0.3:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "recovery should update the upstream address" + grep -q '^reload$' "${FAKE_STATE_DIR}/systemctl.log" || fail "recovery should reload the proxy" + cleanup_fixture +} + +test_ambiguous_containers_fail_closed() { + new_fixture + set_container material-blue 10.0.0.2 1 + set_container material-green 10.0.0.3 1 + printf 'invalid-container\n' > "${PROXY_ROOT}/current-container" + write_proxy_config 10.0.0.4 + + if "$RECONCILER"; then + fail "ambiguous healthy containers should not be switched automatically" + fi + + grep -q 'server 10.0.0.4:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "ambiguous state must leave proxy configuration untouched" + [ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "ambiguous state must not reload the proxy" + cleanup_fixture +} + +trap 'if [ -n "${FIXTURE_ROOT:-}" ] && [ -d "${FIXTURE_ROOT}" ]; then rm -rf "$FIXTURE_ROOT"; fi' EXIT + +test_healthy_entry_wins_over_stale_state +test_single_healthy_container_repairs_broken_proxy +test_ambiguous_containers_fail_closed +test_disk_config_does_not_override_verified_runtime +test_markerless_healthy_conflict_fails_closed +test_broken_entry_prefers_configured_target_over_stale_state +test_reload_error_with_verified_target_succeeds +test_deployment_lock_blocks_reconciliation +test_connection_failure_recovers_to_configured_target + +printf 'PASS: local proxy reconciler scenarios\n'