#Requires -Version 5.1 <# .SYNOPSIS One-time, fail-closed handoff from the legacy direct container to the stable local maintenance proxy before the first worker-drain-aware deploy. .DESCRIPTION This script does not deploy the candidate and never fabricates a worker capability for the legacy image. Its only write path is an explicitly authorized, one-time transition that: 1. proves the exact legacy container, image, upstream and uploads mount; 2. proves the exact pre-staged target image and 40-character Git SHA; 3. blocks new traffic while allowing established HTTP connections to drain; 4. proves database queues and host/container FFmpeg activity are empty; 5. pauses, rechecks, force-stops and rechecks the legacy worker to close both the check/use and paused-container graceful-stop races; 6. preserves the stopped legacy container under a deterministic name; and 7. starts the stable local entry in maintenance mode for the normal blue/ green deployment script to take over. Mode=Audit is read-only with respect to application data and traffic. Mode=EnterMaintenance and Mode=RestoreLegacy both require -Apply and -AllowLegacyDrainBootstrap. The normal deployment script remains responsible for creating the candidate, verifying real capabilities and switching traffic. #> [CmdletBinding()] param( [ValidateSet('Audit', 'EnterMaintenance', 'RestoreLegacy')] [string]$Mode = 'Audit', [switch]$Apply, [switch]$AllowLegacyDrainBootstrap, [Parameter(Mandatory = $true)] [ValidatePattern('^[0-9a-f]{40}$')] [string]$TargetCommit, [Parameter(Mandatory = $true)] [ValidatePattern('^[0-9a-f]{64}$')] [string]$LegacyContainerId, [Parameter(Mandatory = $true)] [ValidatePattern('^sha256:[0-9a-f]{64}$')] [string]$LegacyImageId, [Parameter(Mandatory = $true)] [ValidatePattern('^[A-Za-z0-9._:/-]+@sha256:[0-9a-f]{64}$')] [string]$LegacyImageRepoDigest, [Parameter(Mandatory = $true)] [ValidatePattern('^[0-9]{1,3}(\.[0-9]{1,3}){3}:3000$')] [string]$LegacyUpstream, [Parameter(Mandatory = $true)] [ValidatePattern('^[0-9A-Fa-f]+:[0-9A-Fa-f]+$')] [string]$UploadsStatfsSignature, [Parameter(Mandatory = $true)] [ValidatePattern('^[A-Za-z0-9._:/-]+@sha256:[0-9a-f]{64}$')] [string]$TargetImageReference, [Parameter(Mandatory = $true)] [ValidatePattern('^sha256:[0-9a-f]{64}$')] [string]$TargetImageId, [Parameter(Mandatory = $true)] [ValidatePattern('^[A-Za-z0-9._:/-]+@sha256:[0-9a-f]{64}$')] [string]$TargetImageRepoDigest, [ValidateSet('/mnt/PoolOne/youyidian-uploads', '/mnt/PoolOne/apps/youyidian-3d/uploads')] [string]$UploadsMount = '/mnt/PoolOne/youyidian-uploads', [ValidateSet('/mnt/PoolOne/youyidian-uploads', '/mnt/PoolOne')] [string]$UploadsMountTarget = '/mnt/PoolOne/youyidian-uploads', [ValidateSet('PoolOne/youyidian-uploads', 'PoolOne')] [string]$UploadsMountSource = 'PoolOne/youyidian-uploads', [ValidateSet('0.0.0.0:4320,[::]:4320')] [string]$PublishedEndpoints = '0.0.0.0:4320,[::]:4320', [ValidatePattern('^[A-Za-z0-9._-]+$')] [string]$ContainerName = 'youyidian-3d-material', [ValidatePattern('^[A-Za-z0-9._-]+$')] [string]$NetworkName = 'youyidian-3d_app-network', [ValidatePattern('^/[A-Za-z0-9._/-]+$')] [string]$EnvironmentFile = '/mnt/PoolOne/apps/youyidian-3d/.env', [ValidateSet(4320)] [int]$Port = 4320, [Parameter(Mandatory = $true)] [ValidatePattern('^[A-Za-z0-9._-]+$')] [string]$ProdServerIP, [ValidatePattern('^[A-Za-z0-9._-]+$')] [string]$ProdServerUser = 'root', [string]$ProdServerProxyCommand = '' ) $ErrorActionPreference = 'Stop' if ($Mode -eq 'Audit') { if ($Apply -or $AllowLegacyDrainBootstrap) { throw 'Mode=Audit does not accept write authorization switches.' } } elseif (-not ($Apply -and $AllowLegacyDrainBootstrap)) { throw "Mode=$Mode is disabled by default. Pass both -Apply and -AllowLegacyDrainBootstrap after reviewing the audit output." } if (($UploadsMount -eq '/mnt/PoolOne/youyidian-uploads') -and (($UploadsMountTarget -ne '/mnt/PoolOne/youyidian-uploads') -or ($UploadsMountSource -ne 'PoolOne/youyidian-uploads'))) { throw 'The dedicated uploads dataset requires target=/mnt/PoolOne/youyidian-uploads and source=PoolOne/youyidian-uploads.' } if (($UploadsMount -eq '/mnt/PoolOne/apps/youyidian-3d/uploads') -and (($UploadsMountTarget -ne '/mnt/PoolOne') -or ($UploadsMountSource -ne 'PoolOne'))) { throw 'The compatibility uploads path requires target=/mnt/PoolOne and source=PoolOne.' } if ($TargetImageReference -cne $TargetImageRepoDigest) { throw 'TargetImageReference and TargetImageRepoDigest must be the same immutable repository@sha256 reference.' } function ConvertTo-ShellSingleQuoted { param([AllowEmptyString()][string]$Value) return "'" + $Value.Replace("'", "'\''") + "'" } $tokens = @{ '__MODE__' = ConvertTo-ShellSingleQuoted $Mode '__TARGET_COMMIT__' = ConvertTo-ShellSingleQuoted $TargetCommit '__LEGACY_CONTAINER_ID__' = ConvertTo-ShellSingleQuoted $LegacyContainerId '__LEGACY_IMAGE_ID__' = ConvertTo-ShellSingleQuoted $LegacyImageId '__LEGACY_IMAGE_REPO_DIGEST__' = ConvertTo-ShellSingleQuoted $LegacyImageRepoDigest '__LEGACY_UPSTREAM__' = ConvertTo-ShellSingleQuoted $LegacyUpstream '__UPLOADS_MOUNT__' = ConvertTo-ShellSingleQuoted $UploadsMount '__UPLOADS_MOUNT_TARGET__' = ConvertTo-ShellSingleQuoted $UploadsMountTarget '__UPLOADS_MOUNT_SOURCE__' = ConvertTo-ShellSingleQuoted $UploadsMountSource '__UPLOADS_STATFS_SIGNATURE__' = ConvertTo-ShellSingleQuoted $UploadsStatfsSignature.ToLowerInvariant() '__PUBLISHED_ENDPOINTS__' = ConvertTo-ShellSingleQuoted $PublishedEndpoints '__TARGET_IMAGE_REFERENCE__' = ConvertTo-ShellSingleQuoted $TargetImageReference '__TARGET_IMAGE_ID__' = ConvertTo-ShellSingleQuoted $TargetImageId '__TARGET_IMAGE_REPO_DIGEST__' = ConvertTo-ShellSingleQuoted $TargetImageRepoDigest '__CONTAINER_NAME__' = ConvertTo-ShellSingleQuoted $ContainerName '__NETWORK_NAME__' = ConvertTo-ShellSingleQuoted $NetworkName '__ENVIRONMENT_FILE__' = ConvertTo-ShellSingleQuoted $EnvironmentFile '__PORT__' = ConvertTo-ShellSingleQuoted ([string]$Port) } $remoteTemplate = @' set -Eeuo pipefail MODE=__MODE__ TARGET_COMMIT=__TARGET_COMMIT__ LEGACY_CONTAINER_ID=__LEGACY_CONTAINER_ID__ LEGACY_IMAGE_ID=__LEGACY_IMAGE_ID__ LEGACY_IMAGE_REPO_DIGEST=__LEGACY_IMAGE_REPO_DIGEST__ LEGACY_UPSTREAM=__LEGACY_UPSTREAM__ UPLOADS_MOUNT=__UPLOADS_MOUNT__ UPLOADS_MOUNT_TARGET=__UPLOADS_MOUNT_TARGET__ UPLOADS_MOUNT_SOURCE=__UPLOADS_MOUNT_SOURCE__ UPLOADS_STATFS_SIGNATURE=__UPLOADS_STATFS_SIGNATURE__ PUBLISHED_ENDPOINTS=__PUBLISHED_ENDPOINTS__ TARGET_IMAGE_REFERENCE=__TARGET_IMAGE_REFERENCE__ TARGET_IMAGE_ID=__TARGET_IMAGE_ID__ TARGET_IMAGE_REPO_DIGEST=__TARGET_IMAGE_REPO_DIGEST__ CONTAINER_NAME=__CONTAINER_NAME__ NETWORK_NAME=__NETWORK_NAME__ ENVIRONMENT_FILE=__ENVIRONMENT_FILE__ PORT=__PORT__ PROXY_ROOT="/mnt/PoolOne/apps/youyidian-3d/local-proxy/${CONTAINER_NAME}" PROXY_CONF="${PROXY_ROOT}/nginx.conf" PROXY_PID="${PROXY_ROOT}/nginx.pid" MAINTENANCE_SOCKET="${PROXY_ROOT}/maintenance.sock" PROXY_SERVICE="${CONTAINER_NAME}-local-proxy.service" PROXY_SERVICE_FILE="/etc/systemd/system/${PROXY_SERVICE}" PROXY_RECONCILE_TIMER="${CONTAINER_NAME}-local-proxy-reconcile.timer" PROXY_RECONCILE_LOCK="/run/${CONTAINER_NAME}-local-proxy-reconcile.lock" DEPLOY_LOCK="/run/${CONTAINER_NAME}-production-deploy.lock" CURRENT_FILE="${PROXY_ROOT}/current-container" BLUE_CONTAINER="${CONTAINER_NAME}-blue" GREEN_CONTAINER="${CONTAINER_NAME}-green" CANDIDATE_CONTAINER="$BLUE_CONTAINER" PRESERVED_CONTAINER="${CONTAINER_NAME}-legacy-${TARGET_COMMIT:0:12}-${LEGACY_CONTAINER_ID:0:12}" HANDOFF_ID="${TARGET_COMMIT}-${LEGACY_CONTAINER_ID:0:12}" HANDOFF_DIR="${PROXY_ROOT}/legacy-bootstrap/${HANDOFF_ID}" VIDEO_DRAIN_FILE="${UPLOADS_MOUNT}/.video-transcode-runtime/drain/${CANDIDATE_CONTAINER}" TASK_DETAIL_DRAIN_FILE="${UPLOADS_MOUNT}/.task-detail-upload-runtime/drain/${CANDIDATE_CONTAINER}" FIREWALL_COMMENT="gx-bootstrap-${TARGET_COMMIT:0:12}-${LEGACY_CONTAINER_ID:0:12}" log() { printf '[INFO] %s\n' "$*"; } ok() { printf '[OK] %s\n' "$*"; } warn() { printf '[WARN] %s\n' "$*" >&2; } fail() { printf '[ERR] %s\n' "$*" >&2; exit 1; } require_cmd() { command -v "$1" >/dev/null 2>&1 || fail "Missing required production command: $1" } container_running() { [ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || true)" = "true" ] } container_paused() { [ "$(docker inspect -f '{{.State.Paused}}' "$1" 2>/dev/null || true)" = "true" ] } proxy_running() { systemctl is-active --quiet "$PROXY_SERVICE" 2>/dev/null || { [ -s "$PROXY_PID" ] && kill -0 "$(cat "$PROXY_PID")" >/dev/null 2>&1 } } image_has_repo_digest() { local image="$1" local expected="$2" docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$image" 2>/dev/null | tr -d '\r' | grep -Fqx -- "$expected" } normalise_endpoint_list() { tr ',' '\n' <<< "$1" | sed '/^$/d' | sort -u | paste -sd, - } assert_exact_container_and_image() { local actual_id="" local actual_image_id="" local actual_upstream="" local actual_endpoints="" local expected_endpoints="" actual_id="$(docker inspect -f '{{.Id}}' "$CONTAINER_NAME" 2>/dev/null || true)" [ "$actual_id" = "$LEGACY_CONTAINER_ID" ] || fail "Legacy container ID changed or is missing: expected=${LEGACY_CONTAINER_ID}, actual=${actual_id:-missing}" container_running "$CONTAINER_NAME" || fail "Pinned legacy container is not running" container_paused "$CONTAINER_NAME" && fail "Pinned legacy container is unexpectedly paused" actual_image_id="$(docker inspect -f '{{.Image}}' "$CONTAINER_NAME" 2>/dev/null || true)" [ "$actual_image_id" = "$LEGACY_IMAGE_ID" ] || fail "Legacy image ID changed: expected=${LEGACY_IMAGE_ID}, actual=${actual_image_id:-missing}" image_has_repo_digest "$LEGACY_IMAGE_ID" "$LEGACY_IMAGE_REPO_DIGEST" || fail "Pinned legacy RepoDigest is not attached to the legacy image ID" actual_upstream="$(docker inspect -f "{{with index .NetworkSettings.Networks \"${NETWORK_NAME}\"}}{{.IPAddress}}{{end}}" "$CONTAINER_NAME" 2>/dev/null || true):3000" [ "$actual_upstream" = "$LEGACY_UPSTREAM" ] || fail "Legacy upstream changed: expected=${LEGACY_UPSTREAM}, actual=${actual_upstream:-missing}" actual_endpoints="$(docker port "$CONTAINER_NAME" 3000/tcp 2>/dev/null | tr -d '\r' | sort -u | paste -sd, -)" expected_endpoints="$(normalise_endpoint_list "$PUBLISHED_ENDPOINTS")" [ "$actual_endpoints" = "$expected_endpoints" ] || fail "Published endpoint set changed: expected=${expected_endpoints}, actual=${actual_endpoints:-missing}" } assert_target_image() { local actual_id="" local revision="" actual_id="$(docker image inspect -f '{{.Id}}' "$TARGET_IMAGE_REFERENCE" 2>/dev/null || true)" [ "$actual_id" = "$TARGET_IMAGE_ID" ] || fail "Target image is missing or its immutable ID changed: expected=${TARGET_IMAGE_ID}, actual=${actual_id:-missing}" image_has_repo_digest "$TARGET_IMAGE_ID" "$TARGET_IMAGE_REPO_DIGEST" || fail "Pinned target RepoDigest is not attached to the target image ID" revision="$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$TARGET_IMAGE_REFERENCE" 2>/dev/null || true)" [ "$revision" = "$TARGET_COMMIT" ] || fail "Target image OCI revision does not equal the exact target Git SHA" } assert_target_baked_commit() { local baked="" baked="$(docker run --rm --entrypoint sh "$TARGET_IMAGE_REFERENCE" -c 'cat /app/DEPLOYMENT_COMMIT 2>/dev/null' 2>/dev/null | tr -d '\r\n' || true)" [ "$baked" = "$TARGET_COMMIT" ] || fail "Target image baked DEPLOYMENT_COMMIT does not equal the exact target Git SHA" } assert_uploads_mount_identity() { local mount_line="" local actual_target="" local actual_source="" local actual_fstype="" local extra="" local actual_signature="" local container_metadata="" local container_signature="" mount_line="$(findmnt -rn -o TARGET,SOURCE,FSTYPE -T "$UPLOADS_MOUNT" 2>/dev/null || true)" read -r actual_target actual_source actual_fstype extra <<< "$mount_line" [ -z "${extra:-}" ] || fail "Uploads findmnt returned an ambiguous multi-field identity" [ "$actual_target" = "$UPLOADS_MOUNT_TARGET" ] || fail "Uploads mount target changed: ${actual_target:-missing}" [ "$actual_source" = "$UPLOADS_MOUNT_SOURCE" ] || fail "Uploads mount source changed: ${actual_source:-missing}" [ "$actual_fstype" = "zfs" ] || fail "Uploads filesystem is not the pinned ZFS dataset" actual_signature="$(stat -f -c '%i:%t' -- "$UPLOADS_MOUNT" | tr 'A-F' 'a-f')" [ "$actual_signature" = "$UPLOADS_STATFS_SIGNATURE" ] || fail "Uploads statfs signature changed: expected=${UPLOADS_STATFS_SIGNATURE}, actual=${actual_signature:-missing}" container_metadata="$(docker inspect --format '{{range .Mounts}}{{printf "%s\t%s\t%s\t%t\n" .Destination .Type .Source .RW}}{{end}}' "$CONTAINER_NAME")" [ "$(printf '%s\n' "$container_metadata" | awk -F '\t' -v source="$UPLOADS_MOUNT" '$1 == "/app/uploads" && $2 == "bind" && $3 == source && $4 == "true" { count++ } END { print count + 0 }')" = "1" ] || fail "Legacy container does not have exactly one pinned RW /app/uploads bind" [ "$(printf '%s\n' "$container_metadata" | awk -F '\t' '$1 == "/app/uploads" { count++ } END { print count + 0 }')" = "1" ] || fail "Legacy container has an ambiguous /app/uploads mount set" container_signature="$(docker exec "$CONTAINER_NAME" stat -f -c '%i:%t' -- /app/uploads 2>/dev/null | tr 'A-F' 'a-f' || true)" [ "$container_signature" = "$UPLOADS_STATFS_SIGNATURE" ] || fail "Legacy container uploads statfs identity differs from the pinned host dataset" } assert_topology_is_legacy_only() { local name="" local running_container_metadata="" local running_shadow_containers="" proxy_running && fail "Stable local proxy is already active; this one-time direct-to-proxy bootstrap is not applicable" systemctl is-active --quiet "$PROXY_RECONCILE_TIMER" 2>/dev/null && fail "Proxy reconciler timer is active in legacy-direct topology" for name in "$BLUE_CONTAINER" "$GREEN_CONTAINER"; do docker inspect "$name" >/dev/null 2>&1 && fail "Blue/green slot already exists (${name}); reconcile that topology before legacy bootstrap" done docker inspect "$PRESERVED_CONTAINER" >/dev/null 2>&1 && fail "Deterministic preserved-container name already exists: ${PRESERVED_CONTAINER}" if ! running_container_metadata="$(docker ps --format '{{.Names}}\t{{.Label "com.docker.compose.service"}}')"; then fail "Cannot inspect running containers for shadow topology" fi if ! running_shadow_containers="$(printf '%s\n' "$running_container_metadata" | awk -F '\t' '$1 == "youyidian-3d-material-shadow" || $1 ~ /^youyidian-3d-shadow($|[-_])/ || $2 == "app-shadow" { print $1 }')"; then fail "Cannot parse running container metadata for shadow topology" fi if [ -n "$running_shadow_containers" ]; then printf '%s\n' "$running_shadow_containers" >&2 fail "A running shadow container makes shared uploads worker activity unknowable" fi } DB_NODE_SCRIPT="$(cat <<'NODE' const { PrismaClient } = require('@prisma/client'); const prisma = new PrismaClient(); Promise.all([ prisma.$queryRawUnsafe(`SELECT current_database() AS database, current_user AS user, COALESCE(inet_server_addr()::text, '') AS server_address, COALESCE(inet_server_port()::text, '') AS server_port`), prisma.$queryRawUnsafe(`SELECT COUNT(*)::int AS count FROM materials WHERE "videoProcessingStatus" IN ('pending', 'processing')`), prisma.$queryRawUnsafe(`SELECT id, "productModel" AS product_model, "videoProcessingStatus" AS status, "deletedAt" AS deleted_at FROM materials WHERE "videoProcessingStatus" IN ('pending', 'processing') ORDER BY "updatedAt" ASC LIMIT 100`), prisma.$queryRawUnsafe(`SELECT COUNT(*)::int AS count FROM product_package_upload_jobs WHERE status IN ('queued', 'processing')`), prisma.$queryRawUnsafe(`SELECT id, material_id, status, stage, updated_at FROM product_package_upload_jobs WHERE status IN ('queued', 'processing') ORDER BY updated_at ASC LIMIT 100`), prisma.$queryRawUnsafe(`SELECT COUNT(*)::int AS count FROM materials WHERE task_status = 'uploading'`), prisma.$queryRawUnsafe(`SELECT id, "productModel" AS product_model, "materialType" AS material_type, task_no, task_status, "deletedAt" AS deleted_at, "updatedAt" AS updated_at FROM materials WHERE task_status = 'uploading' ORDER BY "updatedAt" ASC LIMIT 100`), ]).then(([identityRows, videoCount, videos, jobCount, jobs, uploadingCount, uploading]) => { const identity = identityRows[0] || {}; process.stdout.write(JSON.stringify({ capturedAt: new Date().toISOString(), databaseIdentity: { database: String(identity.database || ''), user: String(identity.user || ''), serverAddress: String(identity.server_address || ''), serverPort: String(identity.server_port || ''), }, videoQueuedOrProcessingCount: Number(videoCount[0]?.count || 0), videoQueuedOrProcessingRows: videos, taskJobQueuedOrProcessingCount: Number(jobCount[0]?.count || 0), taskJobQueuedOrProcessingRows: jobs, taskStatusUploadingCount: Number(uploadingCount[0]?.count || 0), taskStatusUploadingRows: uploading, })); }).catch((error) => { process.stderr.write(String(error?.stack || error)); process.exitCode = 1; }).finally(() => prisma.$disconnect()); NODE )" capture_db_snapshot_from_legacy() { docker exec "$CONTAINER_NAME" node -e "$DB_NODE_SCRIPT" } capture_db_snapshot_from_target_image() { docker run --rm --network "$NETWORK_NAME" --env-file "$ENVIRONMENT_FILE" --entrypoint node "$TARGET_IMAGE_REFERENCE" -e "$DB_NODE_SCRIPT" } assert_snapshot_idle() { local snapshot_file="$1" local expected_identity_file="${2:-}" python3 - "$snapshot_file" <<'PY' import json import sys path = sys.argv[1] with open(path, 'r', encoding='utf-8') as handle: snapshot = json.load(handle) print('[INFO] database worker snapshot:') print(json.dumps(snapshot, ensure_ascii=False, indent=2, default=str)) blocked = [] for field in ( 'videoQueuedOrProcessingCount', 'taskJobQueuedOrProcessingCount', 'taskStatusUploadingCount', ): value = snapshot.get(field) if not isinstance(value, int): blocked.append(f'{field}=invalid') elif value != 0: blocked.append(f'{field}={value}') if blocked: print('[ERR] non-terminal database work prevents bootstrap: ' + ', '.join(blocked), file=sys.stderr) raise SystemExit(1) PY [ "$?" -eq 0 ] || return 1 if [ -n "$expected_identity_file" ]; then python3 - "$snapshot_file" "$expected_identity_file" <<'PY' import json import sys with open(sys.argv[1], 'r', encoding='utf-8') as handle: current = json.load(handle).get('databaseIdentity') with open(sys.argv[2], 'r', encoding='utf-8') as handle: expected = json.load(handle).get('databaseIdentity') required = ('database', 'user', 'serverAddress', 'serverPort') if not isinstance(current, dict) or not isinstance(expected, dict): raise SystemExit('database identity evidence is missing') if any(not isinstance(current.get(key), str) or not current.get(key) for key in required): raise SystemExit('current database identity evidence is incomplete') if any(current.get(key) != expected.get(key) for key in required): raise SystemExit('target-image database identity differs from the pinned legacy database') PY [ "$?" -eq 0 ] || return 1 fi } PINNED_LEGACY_DB_SNAPSHOT="" assert_database_idle_from_legacy() { local snapshot="" snapshot="$(mktemp /tmp/gx-legacy-db.XXXXXX.json)" if ! capture_db_snapshot_from_legacy > "$snapshot"; then rm -f "$snapshot" fail "Cannot obtain a read-only worker snapshot through the legacy container" fi if ! assert_snapshot_idle "$snapshot"; then rm -f "$snapshot" fail "Legacy database snapshot is not idle" fi if [ -z "$PINNED_LEGACY_DB_SNAPSHOT" ]; then PINNED_LEGACY_DB_SNAPSHOT="$(mktemp /tmp/gx-pinned-legacy-db.XXXXXX.json)" cp -- "$snapshot" "$PINNED_LEGACY_DB_SNAPSHOT" chmod 600 "$PINNED_LEGACY_DB_SNAPSHOT" elif ! assert_snapshot_idle "$snapshot" "$PINNED_LEGACY_DB_SNAPSHOT"; then rm -f "$snapshot" fail "Legacy database identity changed during bootstrap" fi rm -f "$snapshot" } assert_database_idle_via_target_image() { local snapshot="" snapshot="$(mktemp /tmp/gx-target-db.XXXXXX.json)" if ! capture_db_snapshot_from_target_image > "$snapshot"; then rm -f "$snapshot" fail "Cannot obtain the read-only worker snapshot through the pinned target image" fi [ -n "$PINNED_LEGACY_DB_SNAPSHOT" ] && [ -f "$PINNED_LEGACY_DB_SNAPSHOT" ] || { rm -f "$snapshot" fail "Pinned legacy database identity evidence is missing" } if ! assert_snapshot_idle "$snapshot" "$PINNED_LEGACY_DB_SNAPSHOT"; then rm -f "$snapshot" fail "Pinned target-image database snapshot is not idle" fi rm -f "$snapshot" } assert_no_host_ffmpeg() { local host_snapshot="" local host_rows="" host_snapshot="$(mktemp /tmp/gx-host-processes.XXXXXX.txt)" || fail "Cannot allocate host process evidence file" if ! ps -eo pid=,comm=,args= > "$host_snapshot"; then rm -f "$host_snapshot" fail "Cannot capture production host process evidence" fi if ! host_rows="$(awk '$2 ~ /(^|\/)ffmpeg$/ { print }' "$host_snapshot")"; then rm -f "$host_snapshot" fail "Cannot parse production host process evidence" fi rm -f "$host_snapshot" if [ -n "$host_rows" ]; then printf '%s\n' "$host_rows" >&2 fail "Production host still has FFmpeg activity; global worker idleness is not proven" fi ok "No FFmpeg process exists on the production host" } assert_no_ffmpeg_running_container() { local container_snapshot="" local container_rows="" container_snapshot="$(mktemp /tmp/gx-container-processes.XXXXXX.txt)" || fail "Cannot allocate container process evidence file" if ! docker top "$CONTAINER_NAME" -eo pid,comm,args > "$container_snapshot"; then rm -f "$container_snapshot" fail "Cannot capture pinned legacy container process evidence" fi if ! container_rows="$(awk 'NR > 1 && $2 ~ /(^|\/)ffmpeg$/ { print }' "$container_snapshot")"; then rm -f "$container_snapshot" fail "Cannot parse pinned legacy container process evidence" fi rm -f "$container_snapshot" if [ -n "$container_rows" ]; then printf '%s\n' "$container_rows" >&2 fail "Pinned legacy container still has FFmpeg activity" fi assert_no_host_ffmpeg ok "No FFmpeg process exists in the pinned running legacy container" } host_legacy_connection_count() { ss -Htan state established dst "$LEGACY_UPSTREAM" 2>/dev/null | awk 'END { print NR + 0 }' } container_legacy_connection_count() { local actual_id="" local container_pid="" local current_pid="" local namespace_identity="" local current_namespace_identity="" local count="" actual_id="$(docker inspect -f '{{.Id}}' "$CONTAINER_NAME" 2>/dev/null || true)" [ "$actual_id" = "$LEGACY_CONTAINER_ID" ] || { warn "Cannot sample the legacy network namespace because the container ID pin changed" return 1 } container_pid="$(docker inspect -f '{{.State.Pid}}' "$LEGACY_CONTAINER_ID" 2>/dev/null || true)" case "$container_pid" in ''|*[!0-9]*) warn "Cannot obtain a numeric PID for the pinned legacy network namespace" return 1 ;; esac [ "$container_pid" -gt 1 ] && [ -e "/proc/${container_pid}/ns/net" ] || { warn "Pinned legacy network namespace is unavailable for connection sampling" return 1 } namespace_identity="$(stat -Lc '%d:%i' -- "/proc/${container_pid}/ns/net" 2>/dev/null || true)" [ -n "$namespace_identity" ] || { warn "Pinned legacy network namespace identity cannot be recorded" return 1 } # Use the host's nsenter and ss binaries. The legacy image is not trusted to # contain diagnostic tools. Server-side sockets have local/source port 3000. count="$(nsenter -t "$container_pid" -n -- ss -Htan state established '( sport = :3000 )' 2>/dev/null | awk 'END { print NR + 0 }')" || return 1 # Recheck after sampling so a stopped container, PID reuse, or namespace swap # cannot turn an unrelated empty namespace into a false zero. [ "$(docker inspect -f '{{.Id}}' "$CONTAINER_NAME" 2>/dev/null || true)" = "$LEGACY_CONTAINER_ID" ] || return 1 current_pid="$(docker inspect -f '{{.State.Pid}}' "$LEGACY_CONTAINER_ID" 2>/dev/null || true)" [ "$current_pid" = "$container_pid" ] || return 1 container_running "$LEGACY_CONTAINER_ID" || return 1 current_namespace_identity="$(stat -Lc '%d:%i' -- "/proc/${container_pid}/ns/net" 2>/dev/null || true)" [ "$current_namespace_identity" = "$namespace_identity" ] || return 1 printf '%s\n' "$count" } legacy_connection_counts() { local host_count="" local container_count="" host_count="$(host_legacy_connection_count)" || return 1 container_count="$(container_legacy_connection_count)" || return 1 case "$host_count" in ''|*[!0-9]*) return 1 ;; esac case "$container_count" in ''|*[!0-9]*) return 1 ;; esac printf '%s %s\n' "$host_count" "$container_count" } wait_for_two_zero_connection_samples() { local counts="" local host_count="" local container_count="" local extra="" local zero_samples=0 local attempt=0 for attempt in $(seq 1 60); do counts="$(legacy_connection_counts)" || return 1 read -r host_count container_count extra <<< "$counts" [ -z "${extra:-}" ] || return 1 case "$host_count" in ''|*[!0-9]*) return 1 ;; esac case "$container_count" in ''|*[!0-9]*) return 1 ;; esac if [ "$host_count" = "0" ] && [ "$container_count" = "0" ]; then zero_samples=$((zero_samples + 1)) if [ "$zero_samples" -ge 2 ]; then ok "Host and pinned-container netns both have two consecutive zero-connection samples" return 0 fi else zero_samples=0 log "Waiting for established HTTP connections to leave ${LEGACY_UPSTREAM}: host=${host_count}, container-netns=${container_count}" fi sleep 2 done return 1 } firewall_rule() { local action="$1" local chain="$2" shift 2 iptables -w 5 "$action" "$chain" "$@" -m conntrack --ctstate NEW -m comment --comment "$FIREWALL_COMMENT" -j REJECT --reject-with tcp-reset } firewall_original_port_rule() { local action="$1" iptables -w 5 "$action" DOCKER-USER -p tcp -m conntrack --ctstate NEW --ctorigdstport "$PORT" -m comment --comment "$FIREWALL_COMMENT" -j REJECT --reject-with tcp-reset } firewall_ipv6_rule() { local action="$1" local chain="$2" ip6tables -w 5 "$action" "$chain" -p tcp --dport "$PORT" -m conntrack --ctstate NEW -m comment --comment "$FIREWALL_COMMENT" -j REJECT --reject-with tcp-reset } firewall_ipv6_original_port_rule() { local action="$1" ip6tables -w 5 "$action" DOCKER-USER -p tcp -m conntrack --ctstate NEW --ctorigdstport "$PORT" -m comment --comment "$FIREWALL_COMMENT" -j REJECT --reject-with tcp-reset } delete_firewall_rules() { local legacy_ip="${LEGACY_UPSTREAM%:3000}" local status=0 local deletion_failed=0 while true; do if firewall_rule -C OUTPUT -p tcp -d "$legacy_ip" --dport 3000 >/dev/null 2>&1; then if ! firewall_rule -D OUTPUT -p tcp -d "$legacy_ip" --dport 3000 >/dev/null 2>&1; then warn "Failed to remove bootstrap IPv4 OUTPUT rule" deletion_failed=1 break fi else status=$? if [ "$status" -ne 1 ]; then warn "Cannot verify bootstrap IPv4 OUTPUT rule state (exit ${status})" deletion_failed=1 fi break fi done while true; do if firewall_original_port_rule -C >/dev/null 2>&1; then if ! firewall_original_port_rule -D >/dev/null 2>&1; then warn "Failed to remove bootstrap IPv4 DOCKER-USER rule" deletion_failed=1 break fi else status=$? if [ "$status" -ne 1 ]; then warn "Cannot verify bootstrap IPv4 DOCKER-USER rule state (exit ${status})" deletion_failed=1 fi break fi done while true; do if firewall_rule -C INPUT -p tcp --dport "$PORT" >/dev/null 2>&1; then if ! firewall_rule -D INPUT -p tcp --dport "$PORT" >/dev/null 2>&1; then warn "Failed to remove bootstrap IPv4 INPUT rule" deletion_failed=1 break fi else status=$? if [ "$status" -ne 1 ]; then warn "Cannot verify bootstrap IPv4 INPUT rule state (exit ${status})" deletion_failed=1 fi break fi done while true; do if firewall_ipv6_rule -C OUTPUT >/dev/null 2>&1; then if ! firewall_ipv6_rule -D OUTPUT >/dev/null 2>&1; then warn "Failed to remove bootstrap IPv6 OUTPUT rule" deletion_failed=1 break fi else status=$? if [ "$status" -ne 1 ]; then warn "Cannot verify bootstrap IPv6 OUTPUT rule state (exit ${status})" deletion_failed=1 fi break fi done while true; do if firewall_ipv6_rule -C INPUT >/dev/null 2>&1; then if ! firewall_ipv6_rule -D INPUT >/dev/null 2>&1; then warn "Failed to remove bootstrap IPv6 INPUT rule" deletion_failed=1 break fi else status=$? if [ "$status" -ne 1 ]; then warn "Cannot verify bootstrap IPv6 INPUT rule state (exit ${status})" deletion_failed=1 fi break fi done while true; do if firewall_ipv6_original_port_rule -C >/dev/null 2>&1; then if ! firewall_ipv6_original_port_rule -D >/dev/null 2>&1; then warn "Failed to remove bootstrap IPv6 DOCKER-USER rule" deletion_failed=1 break fi else status=$? if [ "$status" -ne 1 ]; then warn "Cannot verify bootstrap IPv6 DOCKER-USER rule state (exit ${status})" deletion_failed=1 fi break fi done if firewall_rule -C OUTPUT -p tcp -d "$legacy_ip" --dport 3000 >/dev/null 2>&1; then warn "Bootstrap IPv4 OUTPUT rule remains after cleanup" deletion_failed=1 else status=$? [ "$status" -eq 1 ] || { warn "Cannot prove bootstrap IPv4 OUTPUT rule is absent (exit ${status})"; deletion_failed=1; } fi if firewall_original_port_rule -C >/dev/null 2>&1; then warn "Bootstrap IPv4 DOCKER-USER rule remains after cleanup" deletion_failed=1 else status=$? [ "$status" -eq 1 ] || { warn "Cannot prove bootstrap IPv4 DOCKER-USER rule is absent (exit ${status})"; deletion_failed=1; } fi if firewall_rule -C INPUT -p tcp --dport "$PORT" >/dev/null 2>&1; then warn "Bootstrap IPv4 INPUT rule remains after cleanup" deletion_failed=1 else status=$? [ "$status" -eq 1 ] || { warn "Cannot prove bootstrap IPv4 INPUT rule is absent (exit ${status})"; deletion_failed=1; } fi if firewall_ipv6_rule -C OUTPUT >/dev/null 2>&1; then warn "Bootstrap IPv6 OUTPUT rule remains after cleanup" deletion_failed=1 else status=$? [ "$status" -eq 1 ] || { warn "Cannot prove bootstrap IPv6 OUTPUT rule is absent (exit ${status})"; deletion_failed=1; } fi if firewall_ipv6_rule -C INPUT >/dev/null 2>&1; then warn "Bootstrap IPv6 INPUT rule remains after cleanup" deletion_failed=1 else status=$? [ "$status" -eq 1 ] || { warn "Cannot prove bootstrap IPv6 INPUT rule is absent (exit ${status})"; deletion_failed=1; } fi if firewall_ipv6_original_port_rule -C >/dev/null 2>&1; then warn "Bootstrap IPv6 DOCKER-USER rule remains after cleanup" deletion_failed=1 else status=$? [ "$status" -eq 1 ] || { warn "Cannot prove bootstrap IPv6 DOCKER-USER rule is absent (exit ${status})"; deletion_failed=1; } fi [ "$deletion_failed" -eq 0 ] } install_firewall_rules() { local legacy_ip="${LEGACY_UPSTREAM%:3000}" firewall_rule -C OUTPUT -p tcp -d "$legacy_ip" --dport 3000 >/dev/null 2>&1 || firewall_rule -I OUTPUT -p tcp -d "$legacy_ip" --dport 3000 || return 1 firewall_original_port_rule -C >/dev/null 2>&1 || firewall_original_port_rule -I || return 1 firewall_rule -C INPUT -p tcp --dport "$PORT" >/dev/null 2>&1 || firewall_rule -I INPUT -p tcp --dport "$PORT" || return 1 firewall_ipv6_rule -C OUTPUT >/dev/null 2>&1 || firewall_ipv6_rule -I OUTPUT || return 1 firewall_ipv6_rule -C INPUT >/dev/null 2>&1 || firewall_ipv6_rule -I INPUT || return 1 firewall_ipv6_original_port_rule -C >/dev/null 2>&1 || firewall_ipv6_original_port_rule -I || return 1 firewall_rule -C OUTPUT -p tcp -d "$legacy_ip" --dport 3000 >/dev/null || return 1 firewall_original_port_rule -C >/dev/null || return 1 firewall_rule -C INPUT -p tcp --dport "$PORT" >/dev/null || return 1 firewall_ipv6_rule -C OUTPUT >/dev/null || return 1 firewall_ipv6_rule -C INPUT >/dev/null || return 1 firewall_ipv6_original_port_rule -C >/dev/null || return 1 } archive_restored_handoff_if_present() { [ -e "$HANDOFF_DIR" ] || return 0 [ -d "$HANDOFF_DIR" ] && [ ! -L "$HANDOFF_DIR" ] || fail "Existing handoff path is not a physical directory: $HANDOFF_DIR" # Only a handoff whose pinned legacy service was completely restored may be # retried. Ready, partial or ambiguous evidence remains fail-closed. [ -f "$HANDOFF_DIR/restored" ] && [ ! -L "$HANDOFF_DIR/restored" ] || fail "Existing handoff is not terminally restored and cannot be reused: $HANDOFF_DIR" [ "$(tr -d '\r\n' < "$HANDOFF_DIR/restored")" = "restored" ] || fail "Existing handoff restored marker is invalid: $HANDOFF_DIR/restored" [ "$(tr -d '\r\n' < "$HANDOFF_DIR/target-commit")" = "$TARGET_COMMIT" ] || fail "Restored handoff target commit does not match the requested retry" [ "$(tr -d '\r\n' < "$HANDOFF_DIR/legacy-container-id")" = "$LEGACY_CONTAINER_ID" ] || fail "Restored handoff legacy container ID does not match the requested retry" [ "$(tr -d '\r\n' < "$HANDOFF_DIR/legacy-image-id")" = "$LEGACY_IMAGE_ID" ] || fail "Restored handoff legacy image ID does not match the requested retry" [ "$(tr -d '\r\n' < "$HANDOFF_DIR/legacy-upstream")" = "$LEGACY_UPSTREAM" ] || fail "Restored handoff legacy upstream does not match the requested retry" local archive_dir="${HANDOFF_DIR}.restored.$(date -u +%Y%m%dT%H%M%SZ).$$" [ ! -e "$archive_dir" ] || fail "Handoff evidence archive already exists: $archive_dir" mv -- "$HANDOFF_DIR" "$archive_dir" chmod -R a-w "$archive_dir" ok "Archived terminally restored handoff evidence at ${archive_dir}" } backup_proxy_files() { archive_restored_handoff_if_present [ ! -e "$HANDOFF_DIR" ] || fail "Handoff directory already exists; use the recorded state or choose the matching restore action: $HANDOFF_DIR" mkdir -p "$HANDOFF_DIR" chmod 700 "$HANDOFF_DIR" if [ -f "$PROXY_CONF" ]; then cp -a "$PROXY_CONF" "$HANDOFF_DIR/nginx.conf.before" : > "$HANDOFF_DIR/had-proxy-conf" fi if [ -f "$PROXY_SERVICE_FILE" ]; then cp -a "$PROXY_SERVICE_FILE" "$HANDOFF_DIR/proxy.service.before" : > "$HANDOFF_DIR/had-proxy-service" fi if [ -f "$CURRENT_FILE" ]; then cp -a "$CURRENT_FILE" "$HANDOFF_DIR/current-container.before" : > "$HANDOFF_DIR/had-current-container" fi printf '%s\n' "$TARGET_COMMIT" > "$HANDOFF_DIR/target-commit" printf '%s\n' "$LEGACY_CONTAINER_ID" > "$HANDOFF_DIR/legacy-container-id" printf '%s\n' "$LEGACY_IMAGE_ID" > "$HANDOFF_DIR/legacy-image-id" printf '%s\n' "$LEGACY_IMAGE_REPO_DIGEST" > "$HANDOFF_DIR/legacy-image-repo-digest" printf '%s\n' "$LEGACY_UPSTREAM" > "$HANDOFF_DIR/legacy-upstream" printf '%s\n' "$UPLOADS_STATFS_SIGNATURE" > "$HANDOFF_DIR/uploads-statfs-signature" printf '%s\n' "$PRESERVED_CONTAINER" > "$HANDOFF_DIR/preserved-container" chmod 600 "$HANDOFF_DIR"/* } published_endpoints_include_ipv4() { case ",${PUBLISHED_ENDPOINTS}," in *",0.0.0.0:${PORT},"*|*",127.0.0.1:${PORT},"*) return 0 ;; *) return 1 ;; esac } published_endpoints_include_ipv6() { case ",${PUBLISHED_ENDPOINTS}," in *",[::]:${PORT},"*) return 0 ;; *) return 1 ;; esac } write_maintenance_proxy_files() { local tcp_listens="" if published_endpoints_include_ipv4; then tcp_listens=" listen 0.0.0.0:${PORT};" fi if published_endpoints_include_ipv6; then tcp_listens="${tcp_listens} listen [::]:${PORT} ipv6only=on;" fi [ -n "$tcp_listens" ] || { warn "Pinned published endpoints do not contain a supported IPv4 or IPv6 listener" return 1 } mkdir -p "$PROXY_ROOT" cat > "$HANDOFF_DIR/nginx.conf.maintenance" < "$HANDOFF_DIR/proxy.service.maintenance" </dev/null 2>&1 || true if [ -f "$HANDOFF_DIR/had-proxy-conf" ]; then cp -a "$HANDOFF_DIR/nginx.conf.before" "$PROXY_CONF" else rm -f "$PROXY_CONF" fi if [ -f "$HANDOFF_DIR/had-proxy-service" ]; then cp -a "$HANDOFF_DIR/proxy.service.before" "$PROXY_SERVICE_FILE" else rm -f "$PROXY_SERVICE_FILE" fi if [ -f "$HANDOFF_DIR/had-current-container" ]; then cp -a "$HANDOFF_DIR/current-container.before" "$CURRENT_FILE" else rm -f "$CURRENT_FILE" fi systemctl daemon-reload >/dev/null 2>&1 || true } start_maintenance_fallback() { write_maintenance_proxy_files } publish_maintenance_entry() { # INPUT/DOCKER-USER rules deliberately block even loopback traffic on the # published TCP port. Prove nginx through its private Unix socket first, then # remove every rule and finally prove the real stable TCP entry. maintenance_socket_is_ready || { warn "Maintenance Unix-socket probe failed while traffic was still blocked" return 1 } delete_firewall_rules || { warn "Maintenance entry is running but firewall cleanup could not be proven" return 1 } maintenance_entry_is_ready || { warn "Maintenance TCP entry failed after firewall cleanup; attempting to block traffic again" install_firewall_rules || warn "Traffic could not be re-blocked after the maintenance TCP probe failed" return 1 } } leave_restore_failure_in_maintenance() { local reason="$1" warn "$reason" start_maintenance_fallback || { warn "Maintenance entry could not be started; traffic remains blocked" return 1 } publish_maintenance_entry || { warn "Maintenance handoff could not be proven; restoration remains failed closed" return 1 } warn "Pinned legacy restoration failed; the explicit maintenance entry remains active" } verify_container_self_health() { local name="$1" docker exec "$name" node -e ' fetch("http://127.0.0.1:3000/api/health", { signal: AbortSignal.timeout(8000) }) .then(async (response) => { const body = await response.text(); if (!response.ok || !body.includes("healthy")) process.exitCode = 1; }) .catch(() => { process.exitCode = 1; }); ' >/dev/null 2>&1 } legacy_tcp_endpoint_is_healthy() { curl --noproxy '*' -g -fsS --max-time 8 "$1" >/dev/null } legacy_entry_is_healthy() { if published_endpoints_include_ipv4; then legacy_tcp_endpoint_is_healthy "http://127.0.0.1:${PORT}/api/health" || return 1 fi if published_endpoints_include_ipv6; then legacy_tcp_endpoint_is_healthy "http://[::1]:${PORT}/api/health" || return 1 fi } rollback_restored_legacy_to_maintenance() { # This path is used only after the restored public entry failed its final # health probe. Block every new host/Docker-forwarded connection first, then # prove both HTTP and background work are idle before stopping anything. if ! install_firewall_rules; then warn "Cannot prove traffic is blocked after restored-entry health failure; preserving the legacy container and failing closed for manual recovery" return 1 fi wait_for_two_zero_connection_samples || { warn "Restored legacy HTTP connections did not drain during rollback; traffic remains blocked" return 1 } assert_database_idle_from_legacy || { warn "Restored legacy background work is not idle during rollback; traffic remains blocked" return 1 } assert_no_ffmpeg_running_container || { warn "Restored legacy FFmpeg activity is not idle during rollback; traffic remains blocked" return 1 } docker pause "$CONTAINER_NAME" >/dev/null || { warn "Restored legacy container could not be paused during rollback; traffic remains blocked" return 1 } container_paused "$CONTAINER_NAME" || { warn "Restored legacy paused state could not be proven during rollback; traffic remains blocked" return 1 } wait_for_two_zero_connection_samples || { warn "Restored legacy connections reappeared after rollback pause; traffic remains blocked" return 1 } assert_database_idle_via_target_image || { warn "Restored legacy database state changed after rollback pause; traffic remains blocked" return 1 } assert_no_ffmpeg_running_container || { warn "Restored legacy FFmpeg activity appeared after rollback pause; traffic remains blocked" return 1 } [ "$(docker inspect -f '{{.Id}}' "$CONTAINER_NAME" 2>/dev/null || true)" = "$LEGACY_CONTAINER_ID" ] || { warn "Restored legacy container ID changed before rollback stop; traffic remains blocked" return 1 } [ "$(docker inspect -f '{{.Image}}' "$CONTAINER_NAME" 2>/dev/null || true)" = "$LEGACY_IMAGE_ID" ] || { warn "Restored legacy image ID changed before rollback stop; traffic remains blocked" return 1 } docker stop --signal SIGKILL --time 0 "$CONTAINER_NAME" >/dev/null || { warn "Restored legacy container could not be force-stopped from paused state; traffic remains blocked" return 1 } container_running "$CONTAINER_NAME" && { warn "Restored legacy container is still running after rollback stop; traffic remains blocked" return 1 } [ "$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME" 2>/dev/null || true)" = "exited" ] || { warn "Restored legacy stopped state could not be proven during rollback; traffic remains blocked" return 1 } container_paused "$CONTAINER_NAME" && { warn "Restored legacy container remained paused after rollback stop; traffic remains blocked" return 1 } [ "$(docker inspect -f '{{.Id}}' "$CONTAINER_NAME" 2>/dev/null || true)" = "$LEGACY_CONTAINER_ID" ] || { warn "Restored legacy container ID changed after rollback stop; traffic remains blocked" return 1 } [ "$(docker inspect -f '{{.Image}}' "$CONTAINER_NAME" 2>/dev/null || true)" = "$LEGACY_IMAGE_ID" ] || { warn "Restored legacy image ID changed after rollback stop; traffic remains blocked" return 1 } assert_database_idle_via_target_image || { warn "Restored legacy database state is not idle after rollback stop; traffic remains blocked" return 1 } assert_no_host_ffmpeg || { warn "Production host still has FFmpeg activity after rollback stop; traffic remains blocked" return 1 } docker rename "$CONTAINER_NAME" "$PRESERVED_CONTAINER" >/dev/null || { warn "Restored legacy container could not be preserved during rollback; traffic remains blocked" return 1 } start_maintenance_fallback || { warn "Maintenance entry could not be restarted after restored-entry health failure; traffic remains blocked" return 1 } publish_maintenance_entry || { warn "Maintenance entry could not be safely published after restored-entry health failure" return 1 } } restore_legacy_entry() { local source_name="" local current_id="" local restored_upstream="" if ! install_firewall_rules; then warn "Could not prove every traffic block before legacy restoration; maintenance entry remains unchanged" return 1 fi systemctl stop "$PROXY_RECONCILE_TIMER" >/dev/null 2>&1 || true systemctl disable "$PROXY_RECONCILE_TIMER" >/dev/null 2>&1 || true systemctl stop "$PROXY_SERVICE" >/dev/null 2>&1 || true if docker inspect "$PRESERVED_CONTAINER" >/dev/null 2>&1; then source_name="$PRESERVED_CONTAINER" elif docker inspect "$CONTAINER_NAME" >/dev/null 2>&1; then source_name="$CONTAINER_NAME" else leave_restore_failure_in_maintenance "Pinned legacy container is missing during restoration" || true return 1 fi current_id="$(docker inspect -f '{{.Id}}' "$source_name" 2>/dev/null || true)" [ "$current_id" = "$LEGACY_CONTAINER_ID" ] || { leave_restore_failure_in_maintenance "Refusing to restore a container whose full ID does not match the pin" || true return 1 } if [ "$source_name" = "$PRESERVED_CONTAINER" ]; then docker inspect "$CONTAINER_NAME" >/dev/null 2>&1 && { leave_restore_failure_in_maintenance "Original legacy container name is occupied; restoration is blocked" || true return 1 } docker rename "$PRESERVED_CONTAINER" "$CONTAINER_NAME" || { leave_restore_failure_in_maintenance "Pinned legacy container could not be renamed for restoration" || true return 1 } fi if container_paused "$CONTAINER_NAME"; then docker unpause "$CONTAINER_NAME" >/dev/null || { leave_restore_failure_in_maintenance "Pinned legacy container could not be unpaused during restoration" || true return 1 } container_paused "$CONTAINER_NAME" && { leave_restore_failure_in_maintenance "Pinned legacy container paused state remained ambiguous after unpause" || true return 1 } fi if ! container_running "$CONTAINER_NAME"; then docker start "$CONTAINER_NAME" >/dev/null || { leave_restore_failure_in_maintenance "Pinned legacy container could not be restarted" || true return 1 } fi restored_upstream="$(docker inspect -f "{{with index .NetworkSettings.Networks \"${NETWORK_NAME}\"}}{{.IPAddress}}{{end}}" "$CONTAINER_NAME" 2>/dev/null || true):3000" if [ "$restored_upstream" != "$LEGACY_UPSTREAM" ]; then warn "Restored legacy upstream identity changed: expected=${LEGACY_UPSTREAM}, actual=${restored_upstream:-missing}" rollback_restored_legacy_to_maintenance || true return 1 fi if ! verify_container_self_health "$CONTAINER_NAME"; then warn "Pinned legacy container restarted but did not pass its internal health check" rollback_restored_legacy_to_maintenance || true return 1 fi restore_proxy_files delete_firewall_rules || { warn "Legacy identity was restarted, but bootstrap firewall removal could not be proven; restoration remains failed closed" rollback_restored_legacy_to_maintenance || true return 1 } if ! legacy_entry_is_healthy; then warn "Restored legacy public entry failed its final health probe; rolling back to explicit maintenance" rollback_restored_legacy_to_maintenance || true return 1 fi local restored_tmp="$HANDOFF_DIR/.restored.$$" printf '%s\n' "restored" > "$restored_tmp" chmod 600 "$restored_tmp" mv -f -- "$restored_tmp" "$HANDOFF_DIR/restored" ok "Pinned legacy direct container and entry were restored" } assert_restore_is_safe() { local name="" local mount_line="" local actual_target="" local actual_source="" local actual_fstype="" local extra="" local actual_signature="" local preserved_metadata="" local actual_endpoints="" local expected_endpoints="" [ -d "$HANDOFF_DIR" ] || fail "Bootstrap handoff evidence is missing: $HANDOFF_DIR" [ "$(tr -d '\r\n' < "$HANDOFF_DIR/target-commit")" = "$TARGET_COMMIT" ] || fail "Handoff target commit does not match" [ "$(tr -d '\r\n' < "$HANDOFF_DIR/legacy-container-id")" = "$LEGACY_CONTAINER_ID" ] || fail "Handoff legacy container ID does not match" [ "$(tr -d '\r\n' < "$HANDOFF_DIR/legacy-image-id")" = "$LEGACY_IMAGE_ID" ] || fail "Handoff legacy image ID does not match" [ "$(tr -d '\r\n' < "$HANDOFF_DIR/legacy-image-repo-digest")" = "$LEGACY_IMAGE_REPO_DIGEST" ] || fail "Handoff legacy RepoDigest does not match" [ "$(tr -d '\r\n' < "$HANDOFF_DIR/legacy-upstream")" = "$LEGACY_UPSTREAM" ] || fail "Handoff legacy upstream does not match" [ "$(tr -d '\r\n' < "$HANDOFF_DIR/uploads-statfs-signature")" = "$UPLOADS_STATFS_SIGNATURE" ] || fail "Handoff uploads statfs signature does not match" for name in "$BLUE_CONTAINER" "$GREEN_CONTAINER"; do docker inspect "$name" >/dev/null 2>&1 && fail "${name} exists; do not restore legacy after a candidate may have accepted traffic" done maintenance_entry_is_ready || fail "Runtime entry is not the bootstrap maintenance target on every pinned address family; automatic legacy restore is unsafe" [ "$(docker inspect -f '{{.Id}}' "$PRESERVED_CONTAINER" 2>/dev/null || true)" = "$LEGACY_CONTAINER_ID" ] || fail "Preserved legacy container full ID changed or is missing" [ "$(docker inspect -f '{{.Image}}' "$PRESERVED_CONTAINER" 2>/dev/null || true)" = "$LEGACY_IMAGE_ID" ] || fail "Preserved legacy image ID changed" image_has_repo_digest "$LEGACY_IMAGE_ID" "$LEGACY_IMAGE_REPO_DIGEST" || fail "Preserved legacy RepoDigest is missing" mount_line="$(findmnt -rn -o TARGET,SOURCE,FSTYPE -T "$UPLOADS_MOUNT" 2>/dev/null || true)" read -r actual_target actual_source actual_fstype extra <<< "$mount_line" [ -z "${extra:-}" ] || fail "Uploads findmnt identity is ambiguous during restore" [ "$actual_target" = "$UPLOADS_MOUNT_TARGET" ] || fail "Uploads mount target changed before restore" [ "$actual_source" = "$UPLOADS_MOUNT_SOURCE" ] || fail "Uploads mount source changed before restore" [ "$actual_fstype" = "zfs" ] || fail "Uploads filesystem changed before restore" actual_signature="$(stat -f -c '%i:%t' -- "$UPLOADS_MOUNT" | tr 'A-F' 'a-f')" [ "$actual_signature" = "$UPLOADS_STATFS_SIGNATURE" ] || fail "Uploads statfs identity changed before restore" preserved_metadata="$(docker inspect --format '{{range .Mounts}}{{printf "%s\t%s\t%s\t%t\n" .Destination .Type .Source .RW}}{{end}}' "$PRESERVED_CONTAINER")" [ "$(printf '%s\n' "$preserved_metadata" | awk -F '\t' -v source="$UPLOADS_MOUNT" '$1 == "/app/uploads" && $2 == "bind" && $3 == source && $4 == "true" { count++ } END { print count + 0 }')" = "1" ] || fail "Preserved legacy container uploads bind changed before restore" [ "$(printf '%s\n' "$preserved_metadata" | awk -F '\t' '$1 == "/app/uploads" { count++ } END { print count + 0 }')" = "1" ] || fail "Preserved legacy container has ambiguous uploads mounts" actual_endpoints="$(docker port "$PRESERVED_CONTAINER" 3000/tcp 2>/dev/null | tr -d '\r' | sort -u | paste -sd, -)" expected_endpoints="$(normalise_endpoint_list "$PUBLISHED_ENDPOINTS")" [ "$actual_endpoints" = "$expected_endpoints" ] || fail "Preserved legacy published endpoint set changed before restore" } for command_name in docker curl flock ss nsenter findmnt stat nginx systemctl iptables ip6tables python3 awk sed grep ps; do require_cmd "$command_name" done [ -f "$ENVIRONMENT_FILE" ] || fail "Production environment file is missing: $ENVIRONMENT_FILE" docker network inspect "$NETWORK_NAME" >/dev/null 2>&1 || fail "Pinned application network is missing: $NETWORK_NAME" if [ "$MODE" = "RestoreLegacy" ]; then exec 9>"$DEPLOY_LOCK" flock -w 30 9 || fail "Another legacy bootstrap/deployment transition holds the deployment lock" exec 8>"$PROXY_RECONCILE_LOCK" flock -w 30 8 || fail "Another proxy switch or reconciliation holds the proxy lock" assert_restore_is_safe restore_legacy_entry || fail "Automatic legacy restoration failed; maintenance entry or preserved container was retained for manual recovery" ok "Legacy bootstrap handoff ${HANDOFF_ID} was restored without deleting the old container or image" exit 0 fi assert_exact_container_and_image verify_container_self_health "$CONTAINER_NAME" || fail "Pinned legacy container does not pass its internal health endpoint" assert_target_image assert_uploads_mount_identity assert_topology_is_legacy_only assert_database_idle_from_legacy assert_no_ffmpeg_running_container if [ "$MODE" = "Audit" ]; then current_connections="$(legacy_connection_counts)" || fail "Cannot obtain both host and pinned-container connection evidence" read -r current_host_connections current_container_connections current_extra <<< "$current_connections" [ -z "${current_extra:-}" ] || fail "Legacy connection evidence is ambiguous" log "Current established legacy connections: host=${current_host_connections}, container-netns=${current_container_connections}" ok "Read-only legacy bootstrap audit passed; no traffic, container, proxy or database state was changed" exit 0 fi # The target image must already be staged. Running only `cat` through an # overridden entrypoint verifies immutable build provenance before traffic is # touched; it does not start the application or its background workers. assert_target_baked_commit exec 9>"$DEPLOY_LOCK" flock -w 30 9 || fail "Another legacy bootstrap/deployment transition holds the deployment lock" exec 8>"$PROXY_RECONCILE_LOCK" flock -w 30 8 || fail "Another proxy switch or reconciliation holds the proxy lock" # Repeat every volatile pin under both locks. No production mutation occurs # before these checks pass. assert_exact_container_and_image verify_container_self_health "$CONTAINER_NAME" || fail "Pinned legacy container health changed under the transition locks" assert_target_image assert_uploads_mount_identity assert_topology_is_legacy_only assert_database_idle_from_legacy assert_no_ffmpeg_running_container backup_proxy_files HANDOFF_COMMITTED=0 AUTO_RESTORE_ALLOWED=1 trap 'status=$?; trap - EXIT; if [ "$status" -ne 0 ] && [ "$HANDOFF_COMMITTED" != "1" ]; then if [ "$AUTO_RESTORE_ALLOWED" = "1" ]; then restore_legacy_entry || true; else warn "Automatic restart is intentionally disabled because the force-stop or its post-stop evidence was not proven; keep traffic blocked and recover manually"; fi; fi; exit "$status"' EXIT mkdir -p "$(dirname "$VIDEO_DRAIN_FILE")" "$(dirname "$TASK_DETAIL_DRAIN_FILE")" : > "$VIDEO_DRAIN_FILE" : > "$TASK_DETAIL_DRAIN_FILE" chmod 600 "$VIDEO_DRAIN_FILE" "$TASK_DETAIL_DRAIN_FILE" ok "Candidate video and task-detail workers are pre-drained without fabricating capability files" install_firewall_rules ok "New host-local, Docker-forwarded and host-input connections to the legacy upstream are blocked" wait_for_two_zero_connection_samples || fail "Legacy HTTP connections did not reach two consecutive zero samples" assert_database_idle_from_legacy assert_no_ffmpeg_running_container # Pausing closes the final worker check/use race. Database snapshots are # executed by the pinned target image with an overridden node entrypoint, so no # candidate application or background worker is started. docker pause "$CONTAINER_NAME" >/dev/null container_paused "$CONTAINER_NAME" || fail "Legacy container pause could not be proven" wait_for_two_zero_connection_samples || fail "Legacy upstream connections reappeared after pause" assert_database_idle_via_target_image assert_no_ffmpeg_running_container [ "$(docker inspect -f '{{.Id}}' "$CONTAINER_NAME")" = "$LEGACY_CONTAINER_ID" ] || fail "Legacy container ID changed immediately before stop" [ "$(docker inspect -f '{{.Image}}' "$CONTAINER_NAME")" = "$LEGACY_IMAGE_ID" ] || fail "Legacy image ID changed immediately before stop" # A normal `docker stop` resumes a paused container after sending its graceful # signal. Use SIGKILL before that resume so legacy user code cannot run again. # Until stopped state plus fresh DB/host evidence are both proven, failure stays # closed instead of automatically restarting an identity whose state is unknown. AUTO_RESTORE_ALLOWED=0 docker stop --signal SIGKILL --time 0 "$CONTAINER_NAME" >/dev/null container_running "$CONTAINER_NAME" && fail "Legacy container is still running after SIGKILL stop" [ "$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME" 2>/dev/null || true)" = "exited" ] || fail "Legacy container stopped state could not be proven after SIGKILL" container_paused "$CONTAINER_NAME" && fail "Legacy container remained paused after SIGKILL stop" [ "$(docker inspect -f '{{.Id}}' "$CONTAINER_NAME")" = "$LEGACY_CONTAINER_ID" ] || fail "Legacy container ID changed immediately after SIGKILL stop" [ "$(docker inspect -f '{{.Image}}' "$CONTAINER_NAME")" = "$LEGACY_IMAGE_ID" ] || fail "Legacy image ID changed immediately after SIGKILL stop" assert_database_idle_via_target_image assert_no_host_ffmpeg AUTO_RESTORE_ALLOWED=1 docker rename "$CONTAINER_NAME" "$PRESERVED_CONTAINER" [ "$(docker inspect -f '{{.Id}}' "$PRESERVED_CONTAINER")" = "$LEGACY_CONTAINER_ID" ] || fail "Preserved legacy container ID does not match after rename" [ "$(docker inspect -f '{{.Image}}' "$PRESERVED_CONTAINER")" = "$LEGACY_IMAGE_ID" ] || fail "Preserved legacy image ID does not match after rename" write_maintenance_proxy_files publish_maintenance_entry || fail "Stable maintenance entry could not be proven through both its private socket and public TCP endpoint" printf '%s\n' "ready" > "$HANDOFF_DIR/handoff.ready" chmod 600 "$HANDOFF_DIR/handoff.ready" HANDOFF_COMMITTED=1 trap - EXIT ok "Legacy container is stopped and preserved as ${PRESERVED_CONTAINER}; image ${LEGACY_IMAGE_ID} was not removed" ok "Stable local entry is serving an explicit 503 maintenance response on port ${PORT}" ok "Handoff ${HANDOFF_ID} is ready for the normal exact-SHA blue/green deployment" log "Run scripts/git-incremental-deploy.ps1 immediately with -Target production -Commit ${TARGET_COMMIT}." log "Do not bypass its video/task-detail capability, schema, image-provenance or health gates." log "If that deployment fails before any blue/green target accepts traffic, run this script again with -Mode RestoreLegacy -Apply -AllowLegacyDrainBootstrap and the identical pins." '@ $remoteScript = $remoteTemplate foreach ($entry in $tokens.GetEnumerator()) { $remoteScript = $remoteScript.Replace($entry.Key, $entry.Value) } $remoteScript = $remoteScript.Replace("`r`n", "`n").Replace("`r", "`n") $ssh = (Get-Command 'ssh.exe' -ErrorAction Stop).Source $sshArgs = @( '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes', '-o', 'ConnectTimeout=10' ) if (-not [string]::IsNullOrWhiteSpace($ProdServerProxyCommand)) { $sshArgs += '-o' $sshArgs += "ProxyCommand=$ProdServerProxyCommand" } $sshArgs += "${ProdServerUser}@${ProdServerIP}" $sshArgs += 'bash -s' $previousOutputEncoding = $OutputEncoding try { # Keep the large remote program on stdin. Passing its Base64 form as one # command-line argument exceeds Windows CreateProcess limits. $OutputEncoding = [System.Text.UTF8Encoding]::new($false) $remoteScript | & $ssh @sshArgs $remoteExitCode = $LASTEXITCODE } finally { $OutputEncoding = $previousOutputEncoding } if ($remoteExitCode -ne 0) { throw "Legacy worker bootstrap mode $Mode failed with exit code $remoteExitCode. Review the remote evidence; no automatic capability bypass is available." }