318 lines
11 KiB
Bash
318 lines
11 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
RECONCILER="${SCRIPT_DIR}/local-proxy-reconcile.sh"
|
|
ORIGINAL_PATH="$PATH"
|
|
|
|
fail() {
|
|
printf 'FAIL: %s\n' "$1" >&2
|
|
exit 1
|
|
}
|
|
|
|
assert_equals() {
|
|
local expected="$1"
|
|
local actual="$2"
|
|
local message="$3"
|
|
|
|
if [ "$expected" != "$actual" ]; then
|
|
fail "${message}: expected=${expected}, actual=${actual}"
|
|
fi
|
|
}
|
|
|
|
new_fixture() {
|
|
FIXTURE_ROOT="$(mktemp -d)"
|
|
export FIXTURE_ROOT
|
|
export FAKE_STATE_DIR="${FIXTURE_ROOT}/state"
|
|
export PROXY_ROOT="${FIXTURE_ROOT}/proxy"
|
|
export PROXY_RECONCILE_LOCK_FILE="${FIXTURE_ROOT}/reconcile.lock"
|
|
export NGINX_BIN="${FIXTURE_ROOT}/bin/nginx"
|
|
export CONTAINER_NAME="material"
|
|
export PROXY_SERVICE="material-local-proxy.service"
|
|
export PORT="4320"
|
|
export PATH="${FIXTURE_ROOT}/bin:${ORIGINAL_PATH}"
|
|
mkdir -p "$FAKE_STATE_DIR" "$PROXY_ROOT" "${FIXTURE_ROOT}/bin"
|
|
|
|
cat > "${FIXTURE_ROOT}/bin/docker" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
template="$3"
|
|
name="$4"
|
|
if [[ "$template" == *'.State.Running'* ]]; then
|
|
cat "${FAKE_STATE_DIR}/${name}.running" 2>/dev/null || printf 'false\n'
|
|
else
|
|
cat "${FAKE_STATE_DIR}/${name}.ip" 2>/dev/null || true
|
|
fi
|
|
EOF
|
|
|
|
cat > "${FIXTURE_ROOT}/bin/curl" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
url="${!#}"
|
|
if [[ "$url" == http://127.0.0.1:* ]]; then
|
|
if [ -f "${FAKE_STATE_DIR}/entry.connect-fail" ]; then
|
|
exit 7
|
|
fi
|
|
if printf '%s\n' "$@" | grep -qx -- '-D'; then
|
|
printf 'HTTP/1.1 %s Test\r\n' "$([ -f "${FAKE_STATE_DIR}/entry.ok" ] && printf '200' || printf '502')"
|
|
if [ -s "${FAKE_STATE_DIR}/runtime.target" ]; then
|
|
printf 'X-Material-Proxy-Target: %s\r\n' "$(cat "${FAKE_STATE_DIR}/runtime.target")"
|
|
fi
|
|
printf '\r\n'
|
|
exit 0
|
|
fi
|
|
[ -f "${FAKE_STATE_DIR}/entry.ok" ]
|
|
exit
|
|
fi
|
|
ip="${url#http://}"
|
|
ip="${ip%%:*}"
|
|
[ -f "${FAKE_STATE_DIR}/healthy.${ip}" ]
|
|
EOF
|
|
|
|
cat > "${FIXTURE_ROOT}/bin/systemctl" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
case "$1" in
|
|
is-active) exit 0 ;;
|
|
reload|start)
|
|
printf '%s\n' "$1" >> "${FAKE_STATE_DIR}/systemctl.log"
|
|
target="$(sed -nE 's/^[[:space:]]*add_header X-Material-Proxy-Target ([^;[:space:]]+) always;.*/\1/p' "${PROXY_ROOT}/nginx.conf" | tail -n 1)"
|
|
if [ -n "$target" ]; then
|
|
printf '%s\n' "$target" > "${FAKE_STATE_DIR}/runtime.target"
|
|
fi
|
|
if [ "${FAKE_RECOVER_ON_RELOAD:-0}" = "1" ]; then
|
|
rm -f "${FAKE_STATE_DIR}/entry.connect-fail"
|
|
touch "${FAKE_STATE_DIR}/entry.ok"
|
|
fi
|
|
count=0
|
|
if [ -f "${FAKE_STATE_DIR}/reload.count" ]; then
|
|
count="$(cat "${FAKE_STATE_DIR}/reload.count")"
|
|
fi
|
|
count=$((count + 1))
|
|
printf '%s\n' "$count" > "${FAKE_STATE_DIR}/reload.count"
|
|
if [ "${FAKE_RELOAD_FAIL_FIRST:-0}" = "1" ] && [ "$count" -eq 1 ]; then
|
|
exit 1
|
|
fi
|
|
;;
|
|
esac
|
|
EOF
|
|
|
|
cat > "${FIXTURE_ROOT}/bin/nginx" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
exit 0
|
|
EOF
|
|
|
|
cat > "${FIXTURE_ROOT}/bin/sleep" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
exit 0
|
|
EOF
|
|
|
|
cat > "${FIXTURE_ROOT}/bin/flock" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
[ "${FAKE_FLOCK_FAIL:-0}" = "1" ] && exit 1
|
|
exit 0
|
|
EOF
|
|
|
|
chmod +x "${FIXTURE_ROOT}/bin/"*
|
|
}
|
|
|
|
write_proxy_config() {
|
|
local ip="$1"
|
|
local target="${2:-}"
|
|
cat > "${PROXY_ROOT}/nginx.conf" <<EOF
|
|
events {}
|
|
http {
|
|
upstream app_backend {
|
|
server ${ip}:3000 max_fails=3 fail_timeout=5s;
|
|
}
|
|
server {
|
|
location / {
|
|
$(if [ -n "$target" ]; then printf ' proxy_hide_header X-Material-Proxy-Target;\n add_header X-Material-Proxy-Target %s always;\n' "$target"; fi)
|
|
}
|
|
}
|
|
}
|
|
EOF
|
|
}
|
|
|
|
set_container() {
|
|
local name="$1"
|
|
local ip="$2"
|
|
local healthy="$3"
|
|
|
|
printf 'true\n' > "${FAKE_STATE_DIR}/${name}.running"
|
|
printf '%s\n' "$ip" > "${FAKE_STATE_DIR}/${name}.ip"
|
|
if [ "$healthy" = "1" ]; then
|
|
touch "${FAKE_STATE_DIR}/healthy.${ip}"
|
|
fi
|
|
}
|
|
|
|
cleanup_fixture() {
|
|
PATH="$ORIGINAL_PATH"
|
|
unset FAKE_RECOVER_ON_RELOAD || true
|
|
unset FAKE_RELOAD_FAIL_FIRST || true
|
|
unset FAKE_FLOCK_FAIL || true
|
|
rm -rf "$FIXTURE_ROOT"
|
|
}
|
|
|
|
test_healthy_entry_wins_over_stale_state() {
|
|
new_fixture
|
|
set_container material-blue 10.0.0.2 1
|
|
set_container material-green 10.0.0.3 1
|
|
printf 'material-blue\n' > "${PROXY_ROOT}/current-container"
|
|
write_proxy_config 10.0.0.3 material-green
|
|
printf 'material-green\n' > "${FAKE_STATE_DIR}/runtime.target"
|
|
touch "${FAKE_STATE_DIR}/entry.ok"
|
|
|
|
"$RECONCILER"
|
|
|
|
assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "healthy proxy target should repair stale state"
|
|
[ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "healthy entry must not be reloaded"
|
|
grep -q 'server 10.0.0.3:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "healthy proxy target must not be changed"
|
|
cleanup_fixture
|
|
}
|
|
|
|
test_disk_config_does_not_override_verified_runtime() {
|
|
new_fixture
|
|
set_container material-blue 10.0.0.2 1
|
|
set_container material-green 10.0.0.3 1
|
|
printf 'material-blue\n' > "${PROXY_ROOT}/current-container"
|
|
write_proxy_config 10.0.0.3 material-green
|
|
printf 'material-blue\n' > "${FAKE_STATE_DIR}/runtime.target"
|
|
touch "${FAKE_STATE_DIR}/entry.ok"
|
|
|
|
"$RECONCILER"
|
|
|
|
assert_equals "material-blue" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "verified runtime should win over an un-reloaded disk config"
|
|
grep -q 'server 10.0.0.2:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "disk config should be restored to the verified runtime target"
|
|
assert_equals "material-blue" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "reload should keep the verified runtime target"
|
|
cleanup_fixture
|
|
}
|
|
|
|
test_markerless_healthy_conflict_fails_closed() {
|
|
new_fixture
|
|
set_container material-blue 10.0.0.2 1
|
|
set_container material-green 10.0.0.3 1
|
|
printf 'material-blue\n' > "${PROXY_ROOT}/current-container"
|
|
write_proxy_config 10.0.0.3 material-green
|
|
touch "${FAKE_STATE_DIR}/entry.ok"
|
|
|
|
if "$RECONCILER"; then
|
|
fail "markerless healthy state/config conflict should fail closed"
|
|
fi
|
|
|
|
assert_equals "material-blue" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "ambiguous markerless state must remain untouched"
|
|
grep -q 'server 10.0.0.3:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "ambiguous markerless config must remain untouched"
|
|
[ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "ambiguous markerless state must not reload the proxy"
|
|
cleanup_fixture
|
|
}
|
|
|
|
test_broken_entry_prefers_configured_target_over_stale_state() {
|
|
new_fixture
|
|
set_container material-blue 10.0.0.2 1
|
|
set_container material-green 10.0.0.3 1
|
|
printf 'material-blue\n' > "${PROXY_ROOT}/current-container"
|
|
write_proxy_config 10.0.0.3 material-green
|
|
export FAKE_RECOVER_ON_RELOAD=1
|
|
|
|
"$RECONCILER"
|
|
|
|
assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "broken entry should prefer the configured healthy target"
|
|
assert_equals "material-green" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "configured target should become the verified runtime"
|
|
cleanup_fixture
|
|
}
|
|
|
|
test_reload_error_with_verified_target_succeeds() {
|
|
new_fixture
|
|
set_container material-blue 10.0.0.2 1
|
|
set_container material-green 10.0.0.3 1
|
|
printf 'material-blue\n' > "${PROXY_ROOT}/current-container"
|
|
write_proxy_config 10.0.0.2 material-blue
|
|
printf 'material-blue\n' > "${FAKE_STATE_DIR}/runtime.target"
|
|
rm -f "${FAKE_STATE_DIR}/entry.ok"
|
|
# Make the disk target drift while the recorded state remains blue.
|
|
write_proxy_config 10.0.0.3 material-green
|
|
export FAKE_RECOVER_ON_RELOAD=1
|
|
export FAKE_RELOAD_FAIL_FIRST=1
|
|
|
|
"$RECONCILER"
|
|
|
|
assert_equals "material-blue" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "verified runtime should remain authoritative"
|
|
assert_equals "material-blue" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "verified target should be accepted despite the command status"
|
|
cleanup_fixture
|
|
}
|
|
|
|
test_deployment_lock_blocks_reconciliation() {
|
|
new_fixture
|
|
set_container material-green 10.0.0.3 1
|
|
printf 'material-blue\n' > "${PROXY_ROOT}/current-container"
|
|
write_proxy_config 10.0.0.4
|
|
export FAKE_FLOCK_FAIL=1
|
|
|
|
"$RECONCILER"
|
|
|
|
grep -q 'server 10.0.0.4:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "deployment lock must leave proxy configuration untouched"
|
|
[ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "reconciler must not reload while deployment lock is held"
|
|
cleanup_fixture
|
|
}
|
|
|
|
test_connection_failure_recovers_to_configured_target() {
|
|
new_fixture
|
|
set_container material-green 10.0.0.3 1
|
|
printf 'material-green\n' > "${PROXY_ROOT}/current-container"
|
|
write_proxy_config 10.0.0.3 material-green
|
|
touch "${FAKE_STATE_DIR}/entry.connect-fail"
|
|
export FAKE_RECOVER_ON_RELOAD=1
|
|
|
|
"$RECONCILER"
|
|
|
|
assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "connection failure should recover to the configured healthy target"
|
|
assert_equals "material-green" "$(tr -d '\r\n' < "${FAKE_STATE_DIR}/runtime.target")" "recovered target should be verified at runtime"
|
|
grep -q '^reload$' "${FAKE_STATE_DIR}/systemctl.log" || fail "connection failure recovery should reload the proxy"
|
|
cleanup_fixture
|
|
}
|
|
|
|
test_single_healthy_container_repairs_broken_proxy() {
|
|
new_fixture
|
|
set_container material-green 10.0.0.3 1
|
|
printf 'material-blue\n' > "${PROXY_ROOT}/current-container"
|
|
write_proxy_config 10.0.0.4
|
|
export FAKE_RECOVER_ON_RELOAD=1
|
|
|
|
"$RECONCILER"
|
|
|
|
assert_equals "material-green" "$(tr -d '\r\n' < "${PROXY_ROOT}/current-container")" "recovery should persist the selected container"
|
|
grep -q 'server 10.0.0.3:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "recovery should update the upstream address"
|
|
grep -q '^reload$' "${FAKE_STATE_DIR}/systemctl.log" || fail "recovery should reload the proxy"
|
|
cleanup_fixture
|
|
}
|
|
|
|
test_ambiguous_containers_fail_closed() {
|
|
new_fixture
|
|
set_container material-blue 10.0.0.2 1
|
|
set_container material-green 10.0.0.3 1
|
|
printf 'invalid-container\n' > "${PROXY_ROOT}/current-container"
|
|
write_proxy_config 10.0.0.4
|
|
|
|
if "$RECONCILER"; then
|
|
fail "ambiguous healthy containers should not be switched automatically"
|
|
fi
|
|
|
|
grep -q 'server 10.0.0.4:3000 ' "${PROXY_ROOT}/nginx.conf" || fail "ambiguous state must leave proxy configuration untouched"
|
|
[ ! -f "${FAKE_STATE_DIR}/systemctl.log" ] || fail "ambiguous state must not reload the proxy"
|
|
cleanup_fixture
|
|
}
|
|
|
|
trap 'if [ -n "${FIXTURE_ROOT:-}" ] && [ -d "${FIXTURE_ROOT}" ]; then rm -rf "$FIXTURE_ROOT"; fi' EXIT
|
|
|
|
test_healthy_entry_wins_over_stale_state
|
|
test_single_healthy_container_repairs_broken_proxy
|
|
test_ambiguous_containers_fail_closed
|
|
test_disk_config_does_not_override_verified_runtime
|
|
test_markerless_healthy_conflict_fails_closed
|
|
test_broken_entry_prefers_configured_target_over_stale_state
|
|
test_reload_error_with_verified_target_succeeds
|
|
test_deployment_lock_blocks_reconciliation
|
|
test_connection_failure_recovers_to_configured_target
|
|
|
|
printf 'PASS: local proxy reconciler scenarios\n'
|