#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" RECONCILER="${SCRIPT_DIR}/local-proxy-reconcile.sh" ORIGINAL_PATH="$PATH" fail() { printf 'FAIL: %s\n' "$1" >&2 exit 1 } assert_equals() { local expected="$1" local actual="$2" local message="$3" if [ "$expected" != "$actual" ]; then fail "${message}: expected=${expected}, actual=${actual}" fi } new_fixture() { FIXTURE_ROOT="$(mktemp -d)" export FIXTURE_ROOT export FAKE_STATE_DIR="${FIXTURE_ROOT}/state" export PROXY_ROOT="${FIXTURE_ROOT}/proxy" export PROXY_RECONCILE_LOCK_FILE="${FIXTURE_ROOT}/reconcile.lock" export NGINX_BIN="${FIXTURE_ROOT}/bin/nginx" export CONTAINER_NAME="material" export PROXY_SERVICE="material-local-proxy.service" export PORT="4320" export PATH="${FIXTURE_ROOT}/bin:${ORIGINAL_PATH}" mkdir -p "$FAKE_STATE_DIR" "$PROXY_ROOT" "${FIXTURE_ROOT}/bin" cat > "${FIXTURE_ROOT}/bin/docker" <<'EOF' #!/usr/bin/env bash set -euo pipefail template="$3" name="$4" if [[ "$template" == *'.State.Running'* ]]; then cat "${FAKE_STATE_DIR}/${name}.running" 2>/dev/null || printf 'false\n' else cat "${FAKE_STATE_DIR}/${name}.ip" 2>/dev/null || true fi EOF cat > "${FIXTURE_ROOT}/bin/curl" <<'EOF' #!/usr/bin/env bash set -euo pipefail url="${!#}" if [[ "$url" == http://127.0.0.1:* ]]; then if [ -f "${FAKE_STATE_DIR}/entry.connect-fail" ]; then exit 7 fi if printf '%s\n' "$@" | grep -qx -- '-D'; then printf 'HTTP/1.1 %s Test\r\n' "$([ -f "${FAKE_STATE_DIR}/entry.ok" ] && printf '200' || printf '502')" if [ -s "${FAKE_STATE_DIR}/runtime.target" ]; then printf 'X-Material-Proxy-Target: %s\r\n' "$(cat "${FAKE_STATE_DIR}/runtime.target")" fi printf '\r\n' exit 0 fi [ -f "${FAKE_STATE_DIR}/entry.ok" ] exit fi ip="${url#http://}" ip="${ip%%:*}" [ -f "${FAKE_STATE_DIR}/healthy.${ip}" ] EOF cat > "${FIXTURE_ROOT}/bin/systemctl" <<'EOF' #!/usr/bin/env bash set -euo pipefail case "$1" in is-active) exit 0 ;; reload|start) printf '%s\n' "$1" >> "${FAKE_STATE_DIR}/systemctl.log" target="$(sed -nE 's/^[[:space:]]*add_header X-Material-Proxy-Target ([^;[:space:]]+) always;.*/\1/p' "${PROXY_ROOT}/nginx.conf" | tail -n 1)" if [ -n "$target" ]; then printf '%s\n' "$target" > "${FAKE_STATE_DIR}/runtime.target" fi if [ "${FAKE_RECOVER_ON_RELOAD:-0}" = "1" ]; then rm -f "${FAKE_STATE_DIR}/entry.connect-fail" touch "${FAKE_STATE_DIR}/entry.ok" fi count=0 if [ -f "${FAKE_STATE_DIR}/reload.count" ]; then count="$(cat "${FAKE_STATE_DIR}/reload.count")" fi count=$((count + 1)) printf '%s\n' "$count" > "${FAKE_STATE_DIR}/reload.count" if [ "${FAKE_RELOAD_FAIL_FIRST:-0}" = "1" ] && [ "$count" -eq 1 ]; then exit 1 fi ;; esac EOF cat > "${FIXTURE_ROOT}/bin/nginx" <<'EOF' #!/usr/bin/env bash exit 0 EOF cat > "${FIXTURE_ROOT}/bin/sleep" <<'EOF' #!/usr/bin/env bash exit 0 EOF cat > "${FIXTURE_ROOT}/bin/flock" <<'EOF' #!/usr/bin/env bash [ "${FAKE_FLOCK_FAIL:-0}" = "1" ] && exit 1 exit 0 EOF chmod +x "${FIXTURE_ROOT}/bin/"* } write_proxy_config() { local ip="$1" local target="${2:-}" cat > "${PROXY_ROOT}/nginx.conf" < "${FAKE_STATE_DIR}/${name}.running" printf '%s\n' "$ip" > "${FAKE_STATE_DIR}/${name}.ip" if [ "$healthy" = "1" ]; then touch "${FAKE_STATE_DIR}/healthy.${ip}" fi } cleanup_fixture() { PATH="$ORIGINAL_PATH" unset FAKE_RECOVER_ON_RELOAD || true unset FAKE_RELOAD_FAIL_FIRST || true unset FAKE_FLOCK_FAIL || true rm -rf "$FIXTURE_ROOT" } test_healthy_entry_wins_over_stale_state() { new_fixture set_container material-blue 10.0.0.2 1 set_container material-green 10.0.0.3 1 printf 'material-blue\n' > "${PROXY_ROOT}/current-container" write_proxy_config 10.0.0.3 material-green printf 'material-green\n' > "${FAKE_STATE_DIR}/runtime.target" touch "${FAKE_STATE_DIR}/entry.ok" "$RECONCILER" assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "healthy proxy target should repair stale state" [ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "healthy entry must not be reloaded" grep -q 'server 10.0.0.3:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "healthy proxy target must not be changed" cleanup_fixture } test_disk_config_does_not_override_verified_runtime() { new_fixture set_container material-blue 10.0.0.2 1 set_container material-green 10.0.0.3 1 printf 'material-blue\n' > "${PROXY_ROOT}/current-container" write_proxy_config 10.0.0.3 material-green printf 'material-blue\n' > "${FAKE_STATE_DIR}/runtime.target" touch "${FAKE_STATE_DIR}/entry.ok" "$RECONCILER" assert_equals "material-blue" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "verified runtime should win over an un-reloaded disk config" grep -q 'server 10.0.0.2:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "disk config should be restored to the verified runtime target" assert_equals "material-blue" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "reload should keep the verified runtime target" cleanup_fixture } test_markerless_healthy_conflict_fails_closed() { new_fixture set_container material-blue 10.0.0.2 1 set_container material-green 10.0.0.3 1 printf 'material-blue\n' > "${PROXY_ROOT}/current-container" write_proxy_config 10.0.0.3 material-green touch "${FAKE_STATE_DIR}/entry.ok" if "$RECONCILER"; then fail "markerless healthy state/config conflict should fail closed" fi assert_equals "material-blue" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "ambiguous markerless state must remain untouched" grep -q 'server 10.0.0.3:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "ambiguous markerless config must remain untouched" [ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "ambiguous markerless state must not reload the proxy" cleanup_fixture } test_broken_entry_prefers_configured_target_over_stale_state() { new_fixture set_container material-blue 10.0.0.2 1 set_container material-green 10.0.0.3 1 printf 'material-blue\n' > "${PROXY_ROOT}/current-container" write_proxy_config 10.0.0.3 material-green export FAKE_RECOVER_ON_RELOAD=1 "$RECONCILER" assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "broken entry should prefer the configured healthy target" assert_equals "material-green" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "configured target should become the verified runtime" cleanup_fixture } test_reload_error_with_verified_target_succeeds() { new_fixture set_container material-blue 10.0.0.2 1 set_container material-green 10.0.0.3 1 printf 'material-blue\n' > "${PROXY_ROOT}/current-container" write_proxy_config 10.0.0.2 material-blue printf 'material-blue\n' > "${FAKE_STATE_DIR}/runtime.target" rm -f "${FAKE_STATE_DIR}/entry.ok" # Make the disk target drift while the recorded state remains blue. write_proxy_config 10.0.0.3 material-green export FAKE_RECOVER_ON_RELOAD=1 export FAKE_RELOAD_FAIL_FIRST=1 "$RECONCILER" assert_equals "material-blue" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "verified runtime should remain authoritative" assert_equals "material-blue" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "verified target should be accepted despite the command status" cleanup_fixture } test_deployment_lock_blocks_reconciliation() { new_fixture set_container material-green 10.0.0.3 1 printf 'material-blue\n' > "${PROXY_ROOT}/current-container" write_proxy_config 10.0.0.4 export FAKE_FLOCK_FAIL=1 "$RECONCILER" grep -q 'server 10.0.0.4:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "deployment lock must leave proxy configuration untouched" [ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "reconciler must not reload while deployment lock is held" cleanup_fixture } test_connection_failure_recovers_to_configured_target() { new_fixture set_container material-green 10.0.0.3 1 printf 'material-green\n' > "${PROXY_ROOT}/current-container" write_proxy_config 10.0.0.3 material-green touch "${FAKE_STATE_DIR}/entry.connect-fail" export FAKE_RECOVER_ON_RELOAD=1 "$RECONCILER" assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "connection failure should recover to the configured healthy target" assert_equals "material-green" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "recovered target should be verified at runtime" grep -q '^reload$' "${FAKE_STATE_DIR}/systemctl.log" || fail "connection failure recovery should reload the proxy" cleanup_fixture } test_single_healthy_container_repairs_broken_proxy() { new_fixture set_container material-green 10.0.0.3 1 printf 'material-blue\n' > "${PROXY_ROOT}/current-container" write_proxy_config 10.0.0.4 export FAKE_RECOVER_ON_RELOAD=1 "$RECONCILER" assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "recovery should persist the selected container" grep -q 'server 10.0.0.3:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "recovery should update the upstream address" grep -q '^reload$' "${FAKE_STATE_DIR}/systemctl.log" || fail "recovery should reload the proxy" cleanup_fixture } test_ambiguous_containers_fail_closed() { new_fixture set_container material-blue 10.0.0.2 1 set_container material-green 10.0.0.3 1 printf 'invalid-container\n' > "${PROXY_ROOT}/current-container" write_proxy_config 10.0.0.4 if "$RECONCILER"; then fail "ambiguous healthy containers should not be switched automatically" fi grep -q 'server 10.0.0.4:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "ambiguous state must leave proxy configuration untouched" [ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "ambiguous state must not reload the proxy" cleanup_fixture } trap 'if [ -n "${FIXTURE_ROOT:-}" ] && [ -d "${FIXTURE_ROOT}" ]; then rm -rf "$FIXTURE_ROOT"; fi' EXIT test_healthy_entry_wins_over_stale_state test_single_healthy_container_repairs_broken_proxy test_ambiguous_containers_fail_closed test_disk_config_does_not_override_verified_runtime test_markerless_healthy_conflict_fails_closed test_broken_entry_prefers_configured_target_over_stale_state test_reload_error_with_verified_target_succeeds test_deployment_lock_blocks_reconciliation test_connection_failure_recovers_to_configured_target printf 'PASS: local proxy reconciler scenarios\n'