7326e273e1
Protect upload cleanup and atomic completion paths, add durable media worker leases and brand asset validation, and introduce verified migration/bootstrap/deployment gates.
1425 lines
61 KiB
PowerShell
1425 lines
61 KiB
PowerShell
#Requires -Version 5.1
|
|
|
|
<#
|
|
.SYNOPSIS
|
|
One-time, fail-closed handoff from the legacy direct container to the
|
|
stable local maintenance proxy before the first worker-drain-aware deploy.
|
|
|
|
.DESCRIPTION
|
|
This script does not deploy the candidate and never fabricates a worker
|
|
capability for the legacy image. Its only write path is an explicitly
|
|
authorized, one-time transition that:
|
|
|
|
1. proves the exact legacy container, image, upstream and uploads mount;
|
|
2. proves the exact pre-staged target image and 40-character Git SHA;
|
|
3. blocks new traffic while allowing established HTTP connections to drain;
|
|
4. proves database queues and host/container FFmpeg activity are empty;
|
|
5. pauses, rechecks, force-stops and rechecks the legacy worker to close
|
|
both the check/use and paused-container graceful-stop races;
|
|
6. preserves the stopped legacy container under a deterministic name; and
|
|
7. starts the stable local entry in maintenance mode for the normal blue/
|
|
green deployment script to take over.
|
|
|
|
Mode=Audit is read-only with respect to application data and traffic.
|
|
Mode=EnterMaintenance and Mode=RestoreLegacy both require -Apply and
|
|
-AllowLegacyDrainBootstrap. The normal deployment script remains responsible
|
|
for creating the candidate, verifying real capabilities and switching traffic.
|
|
#>
|
|
|
|
[CmdletBinding()]
|
|
param(
|
|
[ValidateSet('Audit', 'EnterMaintenance', 'RestoreLegacy')]
|
|
[string]$Mode = 'Audit',
|
|
|
|
[switch]$Apply,
|
|
[switch]$AllowLegacyDrainBootstrap,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern('^[0-9a-f]{40}$')]
|
|
[string]$TargetCommit,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern('^[0-9a-f]{64}$')]
|
|
[string]$LegacyContainerId,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern('^sha256:[0-9a-f]{64}$')]
|
|
[string]$LegacyImageId,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern('^[A-Za-z0-9._:/-]+@sha256:[0-9a-f]{64}$')]
|
|
[string]$LegacyImageRepoDigest,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern('^[0-9]{1,3}(\.[0-9]{1,3}){3}:3000$')]
|
|
[string]$LegacyUpstream,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern('^[0-9A-Fa-f]+:[0-9A-Fa-f]+$')]
|
|
[string]$UploadsStatfsSignature,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern('^[A-Za-z0-9._:/-]+@sha256:[0-9a-f]{64}$')]
|
|
[string]$TargetImageReference,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern('^sha256:[0-9a-f]{64}$')]
|
|
[string]$TargetImageId,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern('^[A-Za-z0-9._:/-]+@sha256:[0-9a-f]{64}$')]
|
|
[string]$TargetImageRepoDigest,
|
|
|
|
[ValidateSet('/mnt/PoolOne/youyidian-uploads', '/mnt/PoolOne/apps/youyidian-3d/uploads')]
|
|
[string]$UploadsMount = '/mnt/PoolOne/youyidian-uploads',
|
|
|
|
[ValidateSet('/mnt/PoolOne/youyidian-uploads', '/mnt/PoolOne')]
|
|
[string]$UploadsMountTarget = '/mnt/PoolOne/youyidian-uploads',
|
|
|
|
[ValidateSet('PoolOne/youyidian-uploads', 'PoolOne')]
|
|
[string]$UploadsMountSource = 'PoolOne/youyidian-uploads',
|
|
|
|
[ValidateSet('0.0.0.0:4320,[::]:4320')]
|
|
[string]$PublishedEndpoints = '0.0.0.0:4320,[::]:4320',
|
|
|
|
[ValidatePattern('^[A-Za-z0-9._-]+$')]
|
|
[string]$ContainerName = 'youyidian-3d-material',
|
|
|
|
[ValidatePattern('^[A-Za-z0-9._-]+$')]
|
|
[string]$NetworkName = 'youyidian-3d_app-network',
|
|
|
|
[ValidatePattern('^/[A-Za-z0-9._/-]+$')]
|
|
[string]$EnvironmentFile = '/mnt/PoolOne/apps/youyidian-3d/.env',
|
|
|
|
[ValidateSet(4320)]
|
|
[int]$Port = 4320,
|
|
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidatePattern('^[A-Za-z0-9._-]+$')]
|
|
[string]$ProdServerIP,
|
|
|
|
[ValidatePattern('^[A-Za-z0-9._-]+$')]
|
|
[string]$ProdServerUser = 'root',
|
|
|
|
[string]$ProdServerProxyCommand = ''
|
|
)
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
|
|
if ($Mode -eq 'Audit') {
|
|
if ($Apply -or $AllowLegacyDrainBootstrap) {
|
|
throw 'Mode=Audit does not accept write authorization switches.'
|
|
}
|
|
}
|
|
elseif (-not ($Apply -and $AllowLegacyDrainBootstrap)) {
|
|
throw "Mode=$Mode is disabled by default. Pass both -Apply and -AllowLegacyDrainBootstrap after reviewing the audit output."
|
|
}
|
|
|
|
if (($UploadsMount -eq '/mnt/PoolOne/youyidian-uploads') -and
|
|
(($UploadsMountTarget -ne '/mnt/PoolOne/youyidian-uploads') -or ($UploadsMountSource -ne 'PoolOne/youyidian-uploads'))) {
|
|
throw 'The dedicated uploads dataset requires target=/mnt/PoolOne/youyidian-uploads and source=PoolOne/youyidian-uploads.'
|
|
}
|
|
if (($UploadsMount -eq '/mnt/PoolOne/apps/youyidian-3d/uploads') -and
|
|
(($UploadsMountTarget -ne '/mnt/PoolOne') -or ($UploadsMountSource -ne 'PoolOne'))) {
|
|
throw 'The compatibility uploads path requires target=/mnt/PoolOne and source=PoolOne.'
|
|
}
|
|
if ($TargetImageReference -cne $TargetImageRepoDigest) {
|
|
throw 'TargetImageReference and TargetImageRepoDigest must be the same immutable repository@sha256 reference.'
|
|
}
|
|
|
|
function ConvertTo-ShellSingleQuoted {
|
|
param([AllowEmptyString()][string]$Value)
|
|
return "'" + $Value.Replace("'", "'\''") + "'"
|
|
}
|
|
|
|
$tokens = @{
|
|
'__MODE__' = ConvertTo-ShellSingleQuoted $Mode
|
|
'__TARGET_COMMIT__' = ConvertTo-ShellSingleQuoted $TargetCommit
|
|
'__LEGACY_CONTAINER_ID__' = ConvertTo-ShellSingleQuoted $LegacyContainerId
|
|
'__LEGACY_IMAGE_ID__' = ConvertTo-ShellSingleQuoted $LegacyImageId
|
|
'__LEGACY_IMAGE_REPO_DIGEST__' = ConvertTo-ShellSingleQuoted $LegacyImageRepoDigest
|
|
'__LEGACY_UPSTREAM__' = ConvertTo-ShellSingleQuoted $LegacyUpstream
|
|
'__UPLOADS_MOUNT__' = ConvertTo-ShellSingleQuoted $UploadsMount
|
|
'__UPLOADS_MOUNT_TARGET__' = ConvertTo-ShellSingleQuoted $UploadsMountTarget
|
|
'__UPLOADS_MOUNT_SOURCE__' = ConvertTo-ShellSingleQuoted $UploadsMountSource
|
|
'__UPLOADS_STATFS_SIGNATURE__' = ConvertTo-ShellSingleQuoted $UploadsStatfsSignature.ToLowerInvariant()
|
|
'__PUBLISHED_ENDPOINTS__' = ConvertTo-ShellSingleQuoted $PublishedEndpoints
|
|
'__TARGET_IMAGE_REFERENCE__' = ConvertTo-ShellSingleQuoted $TargetImageReference
|
|
'__TARGET_IMAGE_ID__' = ConvertTo-ShellSingleQuoted $TargetImageId
|
|
'__TARGET_IMAGE_REPO_DIGEST__' = ConvertTo-ShellSingleQuoted $TargetImageRepoDigest
|
|
'__CONTAINER_NAME__' = ConvertTo-ShellSingleQuoted $ContainerName
|
|
'__NETWORK_NAME__' = ConvertTo-ShellSingleQuoted $NetworkName
|
|
'__ENVIRONMENT_FILE__' = ConvertTo-ShellSingleQuoted $EnvironmentFile
|
|
'__PORT__' = ConvertTo-ShellSingleQuoted ([string]$Port)
|
|
}
|
|
|
|
$remoteTemplate = @'
|
|
set -Eeuo pipefail
|
|
|
|
MODE=__MODE__
|
|
TARGET_COMMIT=__TARGET_COMMIT__
|
|
LEGACY_CONTAINER_ID=__LEGACY_CONTAINER_ID__
|
|
LEGACY_IMAGE_ID=__LEGACY_IMAGE_ID__
|
|
LEGACY_IMAGE_REPO_DIGEST=__LEGACY_IMAGE_REPO_DIGEST__
|
|
LEGACY_UPSTREAM=__LEGACY_UPSTREAM__
|
|
UPLOADS_MOUNT=__UPLOADS_MOUNT__
|
|
UPLOADS_MOUNT_TARGET=__UPLOADS_MOUNT_TARGET__
|
|
UPLOADS_MOUNT_SOURCE=__UPLOADS_MOUNT_SOURCE__
|
|
UPLOADS_STATFS_SIGNATURE=__UPLOADS_STATFS_SIGNATURE__
|
|
PUBLISHED_ENDPOINTS=__PUBLISHED_ENDPOINTS__
|
|
TARGET_IMAGE_REFERENCE=__TARGET_IMAGE_REFERENCE__
|
|
TARGET_IMAGE_ID=__TARGET_IMAGE_ID__
|
|
TARGET_IMAGE_REPO_DIGEST=__TARGET_IMAGE_REPO_DIGEST__
|
|
CONTAINER_NAME=__CONTAINER_NAME__
|
|
NETWORK_NAME=__NETWORK_NAME__
|
|
ENVIRONMENT_FILE=__ENVIRONMENT_FILE__
|
|
PORT=__PORT__
|
|
|
|
PROXY_ROOT="/mnt/PoolOne/apps/youyidian-3d/local-proxy/${CONTAINER_NAME}"
|
|
PROXY_CONF="${PROXY_ROOT}/nginx.conf"
|
|
PROXY_PID="${PROXY_ROOT}/nginx.pid"
|
|
MAINTENANCE_SOCKET="${PROXY_ROOT}/maintenance.sock"
|
|
PROXY_SERVICE="${CONTAINER_NAME}-local-proxy.service"
|
|
PROXY_SERVICE_FILE="/etc/systemd/system/${PROXY_SERVICE}"
|
|
PROXY_RECONCILE_TIMER="${CONTAINER_NAME}-local-proxy-reconcile.timer"
|
|
PROXY_RECONCILE_LOCK="/run/${CONTAINER_NAME}-local-proxy-reconcile.lock"
|
|
DEPLOY_LOCK="/run/${CONTAINER_NAME}-production-deploy.lock"
|
|
CURRENT_FILE="${PROXY_ROOT}/current-container"
|
|
BLUE_CONTAINER="${CONTAINER_NAME}-blue"
|
|
GREEN_CONTAINER="${CONTAINER_NAME}-green"
|
|
CANDIDATE_CONTAINER="$BLUE_CONTAINER"
|
|
PRESERVED_CONTAINER="${CONTAINER_NAME}-legacy-${TARGET_COMMIT:0:12}-${LEGACY_CONTAINER_ID:0:12}"
|
|
HANDOFF_ID="${TARGET_COMMIT}-${LEGACY_CONTAINER_ID:0:12}"
|
|
HANDOFF_DIR="${PROXY_ROOT}/legacy-bootstrap/${HANDOFF_ID}"
|
|
VIDEO_DRAIN_FILE="${UPLOADS_MOUNT}/.video-transcode-runtime/drain/${CANDIDATE_CONTAINER}"
|
|
TASK_DETAIL_DRAIN_FILE="${UPLOADS_MOUNT}/.task-detail-upload-runtime/drain/${CANDIDATE_CONTAINER}"
|
|
FIREWALL_COMMENT="gx-bootstrap-${TARGET_COMMIT:0:12}-${LEGACY_CONTAINER_ID:0:12}"
|
|
|
|
log() { printf '[INFO] %s\n' "$*"; }
|
|
ok() { printf '[OK] %s\n' "$*"; }
|
|
warn() { printf '[WARN] %s\n' "$*" >&2; }
|
|
fail() { printf '[ERR] %s\n' "$*" >&2; exit 1; }
|
|
|
|
require_cmd() {
|
|
command -v "$1" >/dev/null 2>&1 || fail "Missing required production command: $1"
|
|
}
|
|
|
|
container_running() {
|
|
[ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || true)" = "true" ]
|
|
}
|
|
|
|
container_paused() {
|
|
[ "$(docker inspect -f '{{.State.Paused}}' "$1" 2>/dev/null || true)" = "true" ]
|
|
}
|
|
|
|
proxy_running() {
|
|
systemctl is-active --quiet "$PROXY_SERVICE" 2>/dev/null || {
|
|
[ -s "$PROXY_PID" ] && kill -0 "$(cat "$PROXY_PID")" >/dev/null 2>&1
|
|
}
|
|
}
|
|
|
|
image_has_repo_digest() {
|
|
local image="$1"
|
|
local expected="$2"
|
|
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$image" 2>/dev/null |
|
|
tr -d '\r' |
|
|
grep -Fqx -- "$expected"
|
|
}
|
|
|
|
normalise_endpoint_list() {
|
|
tr ',' '\n' <<< "$1" | sed '/^$/d' | sort -u | paste -sd, -
|
|
}
|
|
|
|
assert_exact_container_and_image() {
|
|
local actual_id=""
|
|
local actual_image_id=""
|
|
local actual_upstream=""
|
|
local actual_endpoints=""
|
|
local expected_endpoints=""
|
|
|
|
actual_id="$(docker inspect -f '{{.Id}}' "$CONTAINER_NAME" 2>/dev/null || true)"
|
|
[ "$actual_id" = "$LEGACY_CONTAINER_ID" ] || fail "Legacy container ID changed or is missing: expected=${LEGACY_CONTAINER_ID}, actual=${actual_id:-missing}"
|
|
container_running "$CONTAINER_NAME" || fail "Pinned legacy container is not running"
|
|
container_paused "$CONTAINER_NAME" && fail "Pinned legacy container is unexpectedly paused"
|
|
|
|
actual_image_id="$(docker inspect -f '{{.Image}}' "$CONTAINER_NAME" 2>/dev/null || true)"
|
|
[ "$actual_image_id" = "$LEGACY_IMAGE_ID" ] || fail "Legacy image ID changed: expected=${LEGACY_IMAGE_ID}, actual=${actual_image_id:-missing}"
|
|
image_has_repo_digest "$LEGACY_IMAGE_ID" "$LEGACY_IMAGE_REPO_DIGEST" || fail "Pinned legacy RepoDigest is not attached to the legacy image ID"
|
|
|
|
actual_upstream="$(docker inspect -f "{{with index .NetworkSettings.Networks \"${NETWORK_NAME}\"}}{{.IPAddress}}{{end}}" "$CONTAINER_NAME" 2>/dev/null || true):3000"
|
|
[ "$actual_upstream" = "$LEGACY_UPSTREAM" ] || fail "Legacy upstream changed: expected=${LEGACY_UPSTREAM}, actual=${actual_upstream:-missing}"
|
|
|
|
actual_endpoints="$(docker port "$CONTAINER_NAME" 3000/tcp 2>/dev/null | tr -d '\r' | sort -u | paste -sd, -)"
|
|
expected_endpoints="$(normalise_endpoint_list "$PUBLISHED_ENDPOINTS")"
|
|
[ "$actual_endpoints" = "$expected_endpoints" ] || fail "Published endpoint set changed: expected=${expected_endpoints}, actual=${actual_endpoints:-missing}"
|
|
}
|
|
|
|
assert_target_image() {
|
|
local actual_id=""
|
|
local revision=""
|
|
|
|
actual_id="$(docker image inspect -f '{{.Id}}' "$TARGET_IMAGE_REFERENCE" 2>/dev/null || true)"
|
|
[ "$actual_id" = "$TARGET_IMAGE_ID" ] || fail "Target image is missing or its immutable ID changed: expected=${TARGET_IMAGE_ID}, actual=${actual_id:-missing}"
|
|
image_has_repo_digest "$TARGET_IMAGE_ID" "$TARGET_IMAGE_REPO_DIGEST" || fail "Pinned target RepoDigest is not attached to the target image ID"
|
|
revision="$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$TARGET_IMAGE_REFERENCE" 2>/dev/null || true)"
|
|
[ "$revision" = "$TARGET_COMMIT" ] || fail "Target image OCI revision does not equal the exact target Git SHA"
|
|
}
|
|
|
|
assert_target_baked_commit() {
|
|
local baked=""
|
|
baked="$(docker run --rm --entrypoint sh "$TARGET_IMAGE_REFERENCE" -c 'cat /app/DEPLOYMENT_COMMIT 2>/dev/null' 2>/dev/null | tr -d '\r\n' || true)"
|
|
[ "$baked" = "$TARGET_COMMIT" ] || fail "Target image baked DEPLOYMENT_COMMIT does not equal the exact target Git SHA"
|
|
}
|
|
|
|
assert_uploads_mount_identity() {
|
|
local mount_line=""
|
|
local actual_target=""
|
|
local actual_source=""
|
|
local actual_fstype=""
|
|
local extra=""
|
|
local actual_signature=""
|
|
local container_metadata=""
|
|
local container_signature=""
|
|
|
|
mount_line="$(findmnt -rn -o TARGET,SOURCE,FSTYPE -T "$UPLOADS_MOUNT" 2>/dev/null || true)"
|
|
read -r actual_target actual_source actual_fstype extra <<< "$mount_line"
|
|
[ -z "${extra:-}" ] || fail "Uploads findmnt returned an ambiguous multi-field identity"
|
|
[ "$actual_target" = "$UPLOADS_MOUNT_TARGET" ] || fail "Uploads mount target changed: ${actual_target:-missing}"
|
|
[ "$actual_source" = "$UPLOADS_MOUNT_SOURCE" ] || fail "Uploads mount source changed: ${actual_source:-missing}"
|
|
[ "$actual_fstype" = "zfs" ] || fail "Uploads filesystem is not the pinned ZFS dataset"
|
|
|
|
actual_signature="$(stat -f -c '%i:%t' -- "$UPLOADS_MOUNT" | tr 'A-F' 'a-f')"
|
|
[ "$actual_signature" = "$UPLOADS_STATFS_SIGNATURE" ] || fail "Uploads statfs signature changed: expected=${UPLOADS_STATFS_SIGNATURE}, actual=${actual_signature:-missing}"
|
|
|
|
container_metadata="$(docker inspect --format '{{range .Mounts}}{{printf "%s\t%s\t%s\t%t\n" .Destination .Type .Source .RW}}{{end}}' "$CONTAINER_NAME")"
|
|
[ "$(printf '%s\n' "$container_metadata" | awk -F '\t' -v source="$UPLOADS_MOUNT" '$1 == "/app/uploads" && $2 == "bind" && $3 == source && $4 == "true" { count++ } END { print count + 0 }')" = "1" ] ||
|
|
fail "Legacy container does not have exactly one pinned RW /app/uploads bind"
|
|
[ "$(printf '%s\n' "$container_metadata" | awk -F '\t' '$1 == "/app/uploads" { count++ } END { print count + 0 }')" = "1" ] ||
|
|
fail "Legacy container has an ambiguous /app/uploads mount set"
|
|
|
|
container_signature="$(docker exec "$CONTAINER_NAME" stat -f -c '%i:%t' -- /app/uploads 2>/dev/null | tr 'A-F' 'a-f' || true)"
|
|
[ "$container_signature" = "$UPLOADS_STATFS_SIGNATURE" ] || fail "Legacy container uploads statfs identity differs from the pinned host dataset"
|
|
}
|
|
|
|
assert_topology_is_legacy_only() {
|
|
local name=""
|
|
local running_container_metadata=""
|
|
local running_shadow_containers=""
|
|
proxy_running && fail "Stable local proxy is already active; this one-time direct-to-proxy bootstrap is not applicable"
|
|
systemctl is-active --quiet "$PROXY_RECONCILE_TIMER" 2>/dev/null && fail "Proxy reconciler timer is active in legacy-direct topology"
|
|
|
|
for name in "$BLUE_CONTAINER" "$GREEN_CONTAINER"; do
|
|
docker inspect "$name" >/dev/null 2>&1 && fail "Blue/green slot already exists (${name}); reconcile that topology before legacy bootstrap"
|
|
done
|
|
docker inspect "$PRESERVED_CONTAINER" >/dev/null 2>&1 && fail "Deterministic preserved-container name already exists: ${PRESERVED_CONTAINER}"
|
|
|
|
if ! running_container_metadata="$(docker ps --format '{{.Names}}\t{{.Label "com.docker.compose.service"}}')"; then
|
|
fail "Cannot inspect running containers for shadow topology"
|
|
fi
|
|
if ! running_shadow_containers="$(printf '%s\n' "$running_container_metadata" |
|
|
awk -F '\t' '$1 == "youyidian-3d-material-shadow" || $1 ~ /^youyidian-3d-shadow($|[-_])/ || $2 == "app-shadow" { print $1 }')"; then
|
|
fail "Cannot parse running container metadata for shadow topology"
|
|
fi
|
|
if [ -n "$running_shadow_containers" ]; then
|
|
printf '%s\n' "$running_shadow_containers" >&2
|
|
fail "A running shadow container makes shared uploads worker activity unknowable"
|
|
fi
|
|
}
|
|
|
|
DB_NODE_SCRIPT="$(cat <<'NODE'
|
|
const { PrismaClient } = require('@prisma/client');
|
|
const prisma = new PrismaClient();
|
|
Promise.all([
|
|
prisma.$queryRawUnsafe(`SELECT current_database() AS database, current_user AS user, COALESCE(inet_server_addr()::text, '') AS server_address, COALESCE(inet_server_port()::text, '') AS server_port`),
|
|
prisma.$queryRawUnsafe(`SELECT COUNT(*)::int AS count FROM materials WHERE "videoProcessingStatus" IN ('pending', 'processing')`),
|
|
prisma.$queryRawUnsafe(`SELECT id, "productModel" AS product_model, "videoProcessingStatus" AS status, "deletedAt" AS deleted_at FROM materials WHERE "videoProcessingStatus" IN ('pending', 'processing') ORDER BY "updatedAt" ASC LIMIT 100`),
|
|
prisma.$queryRawUnsafe(`SELECT COUNT(*)::int AS count FROM product_package_upload_jobs WHERE status IN ('queued', 'processing')`),
|
|
prisma.$queryRawUnsafe(`SELECT id, material_id, status, stage, updated_at FROM product_package_upload_jobs WHERE status IN ('queued', 'processing') ORDER BY updated_at ASC LIMIT 100`),
|
|
prisma.$queryRawUnsafe(`SELECT COUNT(*)::int AS count FROM materials WHERE task_status = 'uploading'`),
|
|
prisma.$queryRawUnsafe(`SELECT id, "productModel" AS product_model, "materialType" AS material_type, task_no, task_status, "deletedAt" AS deleted_at, "updatedAt" AS updated_at FROM materials WHERE task_status = 'uploading' ORDER BY "updatedAt" ASC LIMIT 100`),
|
|
]).then(([identityRows, videoCount, videos, jobCount, jobs, uploadingCount, uploading]) => {
|
|
const identity = identityRows[0] || {};
|
|
process.stdout.write(JSON.stringify({
|
|
capturedAt: new Date().toISOString(),
|
|
databaseIdentity: {
|
|
database: String(identity.database || ''),
|
|
user: String(identity.user || ''),
|
|
serverAddress: String(identity.server_address || ''),
|
|
serverPort: String(identity.server_port || ''),
|
|
},
|
|
videoQueuedOrProcessingCount: Number(videoCount[0]?.count || 0),
|
|
videoQueuedOrProcessingRows: videos,
|
|
taskJobQueuedOrProcessingCount: Number(jobCount[0]?.count || 0),
|
|
taskJobQueuedOrProcessingRows: jobs,
|
|
taskStatusUploadingCount: Number(uploadingCount[0]?.count || 0),
|
|
taskStatusUploadingRows: uploading,
|
|
}));
|
|
}).catch((error) => {
|
|
process.stderr.write(String(error?.stack || error));
|
|
process.exitCode = 1;
|
|
}).finally(() => prisma.$disconnect());
|
|
NODE
|
|
)"
|
|
|
|
capture_db_snapshot_from_legacy() {
|
|
docker exec "$CONTAINER_NAME" node -e "$DB_NODE_SCRIPT"
|
|
}
|
|
|
|
capture_db_snapshot_from_target_image() {
|
|
docker run --rm --network "$NETWORK_NAME" --env-file "$ENVIRONMENT_FILE" --entrypoint node "$TARGET_IMAGE_REFERENCE" -e "$DB_NODE_SCRIPT"
|
|
}
|
|
|
|
assert_snapshot_idle() {
|
|
local snapshot_file="$1"
|
|
local expected_identity_file="${2:-}"
|
|
python3 - "$snapshot_file" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
path = sys.argv[1]
|
|
with open(path, 'r', encoding='utf-8') as handle:
|
|
snapshot = json.load(handle)
|
|
|
|
print('[INFO] database worker snapshot:')
|
|
print(json.dumps(snapshot, ensure_ascii=False, indent=2, default=str))
|
|
|
|
blocked = []
|
|
for field in (
|
|
'videoQueuedOrProcessingCount',
|
|
'taskJobQueuedOrProcessingCount',
|
|
'taskStatusUploadingCount',
|
|
):
|
|
value = snapshot.get(field)
|
|
if not isinstance(value, int):
|
|
blocked.append(f'{field}=invalid')
|
|
elif value != 0:
|
|
blocked.append(f'{field}={value}')
|
|
|
|
if blocked:
|
|
print('[ERR] non-terminal database work prevents bootstrap: ' + ', '.join(blocked), file=sys.stderr)
|
|
raise SystemExit(1)
|
|
PY
|
|
[ "$?" -eq 0 ] || return 1
|
|
|
|
if [ -n "$expected_identity_file" ]; then
|
|
python3 - "$snapshot_file" "$expected_identity_file" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
with open(sys.argv[1], 'r', encoding='utf-8') as handle:
|
|
current = json.load(handle).get('databaseIdentity')
|
|
with open(sys.argv[2], 'r', encoding='utf-8') as handle:
|
|
expected = json.load(handle).get('databaseIdentity')
|
|
|
|
required = ('database', 'user', 'serverAddress', 'serverPort')
|
|
if not isinstance(current, dict) or not isinstance(expected, dict):
|
|
raise SystemExit('database identity evidence is missing')
|
|
if any(not isinstance(current.get(key), str) or not current.get(key) for key in required):
|
|
raise SystemExit('current database identity evidence is incomplete')
|
|
if any(current.get(key) != expected.get(key) for key in required):
|
|
raise SystemExit('target-image database identity differs from the pinned legacy database')
|
|
PY
|
|
[ "$?" -eq 0 ] || return 1
|
|
fi
|
|
}
|
|
|
|
PINNED_LEGACY_DB_SNAPSHOT=""
|
|
|
|
assert_database_idle_from_legacy() {
|
|
local snapshot=""
|
|
snapshot="$(mktemp /tmp/gx-legacy-db.XXXXXX.json)"
|
|
if ! capture_db_snapshot_from_legacy > "$snapshot"; then
|
|
rm -f "$snapshot"
|
|
fail "Cannot obtain a read-only worker snapshot through the legacy container"
|
|
fi
|
|
if ! assert_snapshot_idle "$snapshot"; then
|
|
rm -f "$snapshot"
|
|
fail "Legacy database snapshot is not idle"
|
|
fi
|
|
if [ -z "$PINNED_LEGACY_DB_SNAPSHOT" ]; then
|
|
PINNED_LEGACY_DB_SNAPSHOT="$(mktemp /tmp/gx-pinned-legacy-db.XXXXXX.json)"
|
|
cp -- "$snapshot" "$PINNED_LEGACY_DB_SNAPSHOT"
|
|
chmod 600 "$PINNED_LEGACY_DB_SNAPSHOT"
|
|
elif ! assert_snapshot_idle "$snapshot" "$PINNED_LEGACY_DB_SNAPSHOT"; then
|
|
rm -f "$snapshot"
|
|
fail "Legacy database identity changed during bootstrap"
|
|
fi
|
|
rm -f "$snapshot"
|
|
}
|
|
|
|
assert_database_idle_via_target_image() {
|
|
local snapshot=""
|
|
snapshot="$(mktemp /tmp/gx-target-db.XXXXXX.json)"
|
|
if ! capture_db_snapshot_from_target_image > "$snapshot"; then
|
|
rm -f "$snapshot"
|
|
fail "Cannot obtain the read-only worker snapshot through the pinned target image"
|
|
fi
|
|
[ -n "$PINNED_LEGACY_DB_SNAPSHOT" ] && [ -f "$PINNED_LEGACY_DB_SNAPSHOT" ] || {
|
|
rm -f "$snapshot"
|
|
fail "Pinned legacy database identity evidence is missing"
|
|
}
|
|
if ! assert_snapshot_idle "$snapshot" "$PINNED_LEGACY_DB_SNAPSHOT"; then
|
|
rm -f "$snapshot"
|
|
fail "Pinned target-image database snapshot is not idle"
|
|
fi
|
|
rm -f "$snapshot"
|
|
}
|
|
|
|
assert_no_host_ffmpeg() {
|
|
local host_snapshot=""
|
|
local host_rows=""
|
|
|
|
host_snapshot="$(mktemp /tmp/gx-host-processes.XXXXXX.txt)" || fail "Cannot allocate host process evidence file"
|
|
if ! ps -eo pid=,comm=,args= > "$host_snapshot"; then
|
|
rm -f "$host_snapshot"
|
|
fail "Cannot capture production host process evidence"
|
|
fi
|
|
if ! host_rows="$(awk '$2 ~ /(^|\/)ffmpeg$/ { print }' "$host_snapshot")"; then
|
|
rm -f "$host_snapshot"
|
|
fail "Cannot parse production host process evidence"
|
|
fi
|
|
rm -f "$host_snapshot"
|
|
if [ -n "$host_rows" ]; then
|
|
printf '%s\n' "$host_rows" >&2
|
|
fail "Production host still has FFmpeg activity; global worker idleness is not proven"
|
|
fi
|
|
ok "No FFmpeg process exists on the production host"
|
|
}
|
|
|
|
assert_no_ffmpeg_running_container() {
|
|
local container_snapshot=""
|
|
local container_rows=""
|
|
|
|
container_snapshot="$(mktemp /tmp/gx-container-processes.XXXXXX.txt)" || fail "Cannot allocate container process evidence file"
|
|
if ! docker top "$CONTAINER_NAME" -eo pid,comm,args > "$container_snapshot"; then
|
|
rm -f "$container_snapshot"
|
|
fail "Cannot capture pinned legacy container process evidence"
|
|
fi
|
|
if ! container_rows="$(awk 'NR > 1 && $2 ~ /(^|\/)ffmpeg$/ { print }' "$container_snapshot")"; then
|
|
rm -f "$container_snapshot"
|
|
fail "Cannot parse pinned legacy container process evidence"
|
|
fi
|
|
rm -f "$container_snapshot"
|
|
if [ -n "$container_rows" ]; then
|
|
printf '%s\n' "$container_rows" >&2
|
|
fail "Pinned legacy container still has FFmpeg activity"
|
|
fi
|
|
assert_no_host_ffmpeg
|
|
ok "No FFmpeg process exists in the pinned running legacy container"
|
|
}
|
|
|
|
host_legacy_connection_count() {
|
|
ss -Htan state established dst "$LEGACY_UPSTREAM" 2>/dev/null | awk 'END { print NR + 0 }'
|
|
}
|
|
|
|
container_legacy_connection_count() {
|
|
local actual_id=""
|
|
local container_pid=""
|
|
local current_pid=""
|
|
local namespace_identity=""
|
|
local current_namespace_identity=""
|
|
local count=""
|
|
|
|
actual_id="$(docker inspect -f '{{.Id}}' "$CONTAINER_NAME" 2>/dev/null || true)"
|
|
[ "$actual_id" = "$LEGACY_CONTAINER_ID" ] || {
|
|
warn "Cannot sample the legacy network namespace because the container ID pin changed"
|
|
return 1
|
|
}
|
|
container_pid="$(docker inspect -f '{{.State.Pid}}' "$LEGACY_CONTAINER_ID" 2>/dev/null || true)"
|
|
case "$container_pid" in ''|*[!0-9]*)
|
|
warn "Cannot obtain a numeric PID for the pinned legacy network namespace"
|
|
return 1
|
|
;;
|
|
esac
|
|
[ "$container_pid" -gt 1 ] && [ -e "/proc/${container_pid}/ns/net" ] || {
|
|
warn "Pinned legacy network namespace is unavailable for connection sampling"
|
|
return 1
|
|
}
|
|
namespace_identity="$(stat -Lc '%d:%i' -- "/proc/${container_pid}/ns/net" 2>/dev/null || true)"
|
|
[ -n "$namespace_identity" ] || {
|
|
warn "Pinned legacy network namespace identity cannot be recorded"
|
|
return 1
|
|
}
|
|
|
|
# Use the host's nsenter and ss binaries. The legacy image is not trusted to
|
|
# contain diagnostic tools. Server-side sockets have local/source port 3000.
|
|
count="$(nsenter -t "$container_pid" -n -- ss -Htan state established '( sport = :3000 )' 2>/dev/null |
|
|
awk 'END { print NR + 0 }')" || return 1
|
|
|
|
# Recheck after sampling so a stopped container, PID reuse, or namespace swap
|
|
# cannot turn an unrelated empty namespace into a false zero.
|
|
[ "$(docker inspect -f '{{.Id}}' "$CONTAINER_NAME" 2>/dev/null || true)" = "$LEGACY_CONTAINER_ID" ] || return 1
|
|
current_pid="$(docker inspect -f '{{.State.Pid}}' "$LEGACY_CONTAINER_ID" 2>/dev/null || true)"
|
|
[ "$current_pid" = "$container_pid" ] || return 1
|
|
container_running "$LEGACY_CONTAINER_ID" || return 1
|
|
current_namespace_identity="$(stat -Lc '%d:%i' -- "/proc/${container_pid}/ns/net" 2>/dev/null || true)"
|
|
[ "$current_namespace_identity" = "$namespace_identity" ] || return 1
|
|
printf '%s\n' "$count"
|
|
}
|
|
|
|
legacy_connection_counts() {
|
|
local host_count=""
|
|
local container_count=""
|
|
|
|
host_count="$(host_legacy_connection_count)" || return 1
|
|
container_count="$(container_legacy_connection_count)" || return 1
|
|
case "$host_count" in ''|*[!0-9]*) return 1 ;; esac
|
|
case "$container_count" in ''|*[!0-9]*) return 1 ;; esac
|
|
printf '%s %s\n' "$host_count" "$container_count"
|
|
}
|
|
|
|
wait_for_two_zero_connection_samples() {
|
|
local counts=""
|
|
local host_count=""
|
|
local container_count=""
|
|
local extra=""
|
|
local zero_samples=0
|
|
local attempt=0
|
|
|
|
for attempt in $(seq 1 60); do
|
|
counts="$(legacy_connection_counts)" || return 1
|
|
read -r host_count container_count extra <<< "$counts"
|
|
[ -z "${extra:-}" ] || return 1
|
|
case "$host_count" in ''|*[!0-9]*) return 1 ;; esac
|
|
case "$container_count" in ''|*[!0-9]*) return 1 ;; esac
|
|
if [ "$host_count" = "0" ] && [ "$container_count" = "0" ]; then
|
|
zero_samples=$((zero_samples + 1))
|
|
if [ "$zero_samples" -ge 2 ]; then
|
|
ok "Host and pinned-container netns both have two consecutive zero-connection samples"
|
|
return 0
|
|
fi
|
|
else
|
|
zero_samples=0
|
|
log "Waiting for established HTTP connections to leave ${LEGACY_UPSTREAM}: host=${host_count}, container-netns=${container_count}"
|
|
fi
|
|
sleep 2
|
|
done
|
|
return 1
|
|
}
|
|
|
|
firewall_rule() {
|
|
local action="$1"
|
|
local chain="$2"
|
|
shift 2
|
|
iptables -w 5 "$action" "$chain" "$@" -m conntrack --ctstate NEW -m comment --comment "$FIREWALL_COMMENT" -j REJECT --reject-with tcp-reset
|
|
}
|
|
|
|
firewall_original_port_rule() {
|
|
local action="$1"
|
|
iptables -w 5 "$action" DOCKER-USER -p tcp -m conntrack --ctstate NEW --ctorigdstport "$PORT" -m comment --comment "$FIREWALL_COMMENT" -j REJECT --reject-with tcp-reset
|
|
}
|
|
|
|
firewall_ipv6_rule() {
|
|
local action="$1"
|
|
local chain="$2"
|
|
ip6tables -w 5 "$action" "$chain" -p tcp --dport "$PORT" -m conntrack --ctstate NEW -m comment --comment "$FIREWALL_COMMENT" -j REJECT --reject-with tcp-reset
|
|
}
|
|
|
|
firewall_ipv6_original_port_rule() {
|
|
local action="$1"
|
|
ip6tables -w 5 "$action" DOCKER-USER -p tcp -m conntrack --ctstate NEW --ctorigdstport "$PORT" -m comment --comment "$FIREWALL_COMMENT" -j REJECT --reject-with tcp-reset
|
|
}
|
|
|
|
delete_firewall_rules() {
|
|
local legacy_ip="${LEGACY_UPSTREAM%:3000}"
|
|
local status=0
|
|
local deletion_failed=0
|
|
|
|
while true; do
|
|
if firewall_rule -C OUTPUT -p tcp -d "$legacy_ip" --dport 3000 >/dev/null 2>&1; then
|
|
if ! firewall_rule -D OUTPUT -p tcp -d "$legacy_ip" --dport 3000 >/dev/null 2>&1; then
|
|
warn "Failed to remove bootstrap IPv4 OUTPUT rule"
|
|
deletion_failed=1
|
|
break
|
|
fi
|
|
else
|
|
status=$?
|
|
if [ "$status" -ne 1 ]; then
|
|
warn "Cannot verify bootstrap IPv4 OUTPUT rule state (exit ${status})"
|
|
deletion_failed=1
|
|
fi
|
|
break
|
|
fi
|
|
done
|
|
while true; do
|
|
if firewall_original_port_rule -C >/dev/null 2>&1; then
|
|
if ! firewall_original_port_rule -D >/dev/null 2>&1; then
|
|
warn "Failed to remove bootstrap IPv4 DOCKER-USER rule"
|
|
deletion_failed=1
|
|
break
|
|
fi
|
|
else
|
|
status=$?
|
|
if [ "$status" -ne 1 ]; then
|
|
warn "Cannot verify bootstrap IPv4 DOCKER-USER rule state (exit ${status})"
|
|
deletion_failed=1
|
|
fi
|
|
break
|
|
fi
|
|
done
|
|
while true; do
|
|
if firewall_rule -C INPUT -p tcp --dport "$PORT" >/dev/null 2>&1; then
|
|
if ! firewall_rule -D INPUT -p tcp --dport "$PORT" >/dev/null 2>&1; then
|
|
warn "Failed to remove bootstrap IPv4 INPUT rule"
|
|
deletion_failed=1
|
|
break
|
|
fi
|
|
else
|
|
status=$?
|
|
if [ "$status" -ne 1 ]; then
|
|
warn "Cannot verify bootstrap IPv4 INPUT rule state (exit ${status})"
|
|
deletion_failed=1
|
|
fi
|
|
break
|
|
fi
|
|
done
|
|
while true; do
|
|
if firewall_ipv6_rule -C OUTPUT >/dev/null 2>&1; then
|
|
if ! firewall_ipv6_rule -D OUTPUT >/dev/null 2>&1; then
|
|
warn "Failed to remove bootstrap IPv6 OUTPUT rule"
|
|
deletion_failed=1
|
|
break
|
|
fi
|
|
else
|
|
status=$?
|
|
if [ "$status" -ne 1 ]; then
|
|
warn "Cannot verify bootstrap IPv6 OUTPUT rule state (exit ${status})"
|
|
deletion_failed=1
|
|
fi
|
|
break
|
|
fi
|
|
done
|
|
while true; do
|
|
if firewall_ipv6_rule -C INPUT >/dev/null 2>&1; then
|
|
if ! firewall_ipv6_rule -D INPUT >/dev/null 2>&1; then
|
|
warn "Failed to remove bootstrap IPv6 INPUT rule"
|
|
deletion_failed=1
|
|
break
|
|
fi
|
|
else
|
|
status=$?
|
|
if [ "$status" -ne 1 ]; then
|
|
warn "Cannot verify bootstrap IPv6 INPUT rule state (exit ${status})"
|
|
deletion_failed=1
|
|
fi
|
|
break
|
|
fi
|
|
done
|
|
while true; do
|
|
if firewall_ipv6_original_port_rule -C >/dev/null 2>&1; then
|
|
if ! firewall_ipv6_original_port_rule -D >/dev/null 2>&1; then
|
|
warn "Failed to remove bootstrap IPv6 DOCKER-USER rule"
|
|
deletion_failed=1
|
|
break
|
|
fi
|
|
else
|
|
status=$?
|
|
if [ "$status" -ne 1 ]; then
|
|
warn "Cannot verify bootstrap IPv6 DOCKER-USER rule state (exit ${status})"
|
|
deletion_failed=1
|
|
fi
|
|
break
|
|
fi
|
|
done
|
|
|
|
if firewall_rule -C OUTPUT -p tcp -d "$legacy_ip" --dport 3000 >/dev/null 2>&1; then
|
|
warn "Bootstrap IPv4 OUTPUT rule remains after cleanup"
|
|
deletion_failed=1
|
|
else
|
|
status=$?
|
|
[ "$status" -eq 1 ] || { warn "Cannot prove bootstrap IPv4 OUTPUT rule is absent (exit ${status})"; deletion_failed=1; }
|
|
fi
|
|
if firewall_original_port_rule -C >/dev/null 2>&1; then
|
|
warn "Bootstrap IPv4 DOCKER-USER rule remains after cleanup"
|
|
deletion_failed=1
|
|
else
|
|
status=$?
|
|
[ "$status" -eq 1 ] || { warn "Cannot prove bootstrap IPv4 DOCKER-USER rule is absent (exit ${status})"; deletion_failed=1; }
|
|
fi
|
|
if firewall_rule -C INPUT -p tcp --dport "$PORT" >/dev/null 2>&1; then
|
|
warn "Bootstrap IPv4 INPUT rule remains after cleanup"
|
|
deletion_failed=1
|
|
else
|
|
status=$?
|
|
[ "$status" -eq 1 ] || { warn "Cannot prove bootstrap IPv4 INPUT rule is absent (exit ${status})"; deletion_failed=1; }
|
|
fi
|
|
if firewall_ipv6_rule -C OUTPUT >/dev/null 2>&1; then
|
|
warn "Bootstrap IPv6 OUTPUT rule remains after cleanup"
|
|
deletion_failed=1
|
|
else
|
|
status=$?
|
|
[ "$status" -eq 1 ] || { warn "Cannot prove bootstrap IPv6 OUTPUT rule is absent (exit ${status})"; deletion_failed=1; }
|
|
fi
|
|
if firewall_ipv6_rule -C INPUT >/dev/null 2>&1; then
|
|
warn "Bootstrap IPv6 INPUT rule remains after cleanup"
|
|
deletion_failed=1
|
|
else
|
|
status=$?
|
|
[ "$status" -eq 1 ] || { warn "Cannot prove bootstrap IPv6 INPUT rule is absent (exit ${status})"; deletion_failed=1; }
|
|
fi
|
|
if firewall_ipv6_original_port_rule -C >/dev/null 2>&1; then
|
|
warn "Bootstrap IPv6 DOCKER-USER rule remains after cleanup"
|
|
deletion_failed=1
|
|
else
|
|
status=$?
|
|
[ "$status" -eq 1 ] || { warn "Cannot prove bootstrap IPv6 DOCKER-USER rule is absent (exit ${status})"; deletion_failed=1; }
|
|
fi
|
|
|
|
[ "$deletion_failed" -eq 0 ]
|
|
}
|
|
|
|
install_firewall_rules() {
|
|
local legacy_ip="${LEGACY_UPSTREAM%:3000}"
|
|
firewall_rule -C OUTPUT -p tcp -d "$legacy_ip" --dport 3000 >/dev/null 2>&1 ||
|
|
firewall_rule -I OUTPUT -p tcp -d "$legacy_ip" --dport 3000 || return 1
|
|
firewall_original_port_rule -C >/dev/null 2>&1 || firewall_original_port_rule -I || return 1
|
|
firewall_rule -C INPUT -p tcp --dport "$PORT" >/dev/null 2>&1 ||
|
|
firewall_rule -I INPUT -p tcp --dport "$PORT" || return 1
|
|
firewall_ipv6_rule -C OUTPUT >/dev/null 2>&1 || firewall_ipv6_rule -I OUTPUT || return 1
|
|
firewall_ipv6_rule -C INPUT >/dev/null 2>&1 || firewall_ipv6_rule -I INPUT || return 1
|
|
firewall_ipv6_original_port_rule -C >/dev/null 2>&1 || firewall_ipv6_original_port_rule -I || return 1
|
|
firewall_rule -C OUTPUT -p tcp -d "$legacy_ip" --dport 3000 >/dev/null || return 1
|
|
firewall_original_port_rule -C >/dev/null || return 1
|
|
firewall_rule -C INPUT -p tcp --dport "$PORT" >/dev/null || return 1
|
|
firewall_ipv6_rule -C OUTPUT >/dev/null || return 1
|
|
firewall_ipv6_rule -C INPUT >/dev/null || return 1
|
|
firewall_ipv6_original_port_rule -C >/dev/null || return 1
|
|
}
|
|
|
|
archive_restored_handoff_if_present() {
|
|
[ -e "$HANDOFF_DIR" ] || return 0
|
|
[ -d "$HANDOFF_DIR" ] && [ ! -L "$HANDOFF_DIR" ] ||
|
|
fail "Existing handoff path is not a physical directory: $HANDOFF_DIR"
|
|
|
|
# Only a handoff whose pinned legacy service was completely restored may be
|
|
# retried. Ready, partial or ambiguous evidence remains fail-closed.
|
|
[ -f "$HANDOFF_DIR/restored" ] && [ ! -L "$HANDOFF_DIR/restored" ] ||
|
|
fail "Existing handoff is not terminally restored and cannot be reused: $HANDOFF_DIR"
|
|
[ "$(tr -d '\r\n' < "$HANDOFF_DIR/restored")" = "restored" ] ||
|
|
fail "Existing handoff restored marker is invalid: $HANDOFF_DIR/restored"
|
|
[ "$(tr -d '\r\n' < "$HANDOFF_DIR/target-commit")" = "$TARGET_COMMIT" ] ||
|
|
fail "Restored handoff target commit does not match the requested retry"
|
|
[ "$(tr -d '\r\n' < "$HANDOFF_DIR/legacy-container-id")" = "$LEGACY_CONTAINER_ID" ] ||
|
|
fail "Restored handoff legacy container ID does not match the requested retry"
|
|
[ "$(tr -d '\r\n' < "$HANDOFF_DIR/legacy-image-id")" = "$LEGACY_IMAGE_ID" ] ||
|
|
fail "Restored handoff legacy image ID does not match the requested retry"
|
|
[ "$(tr -d '\r\n' < "$HANDOFF_DIR/legacy-upstream")" = "$LEGACY_UPSTREAM" ] ||
|
|
fail "Restored handoff legacy upstream does not match the requested retry"
|
|
|
|
local archive_dir="${HANDOFF_DIR}.restored.$(date -u +%Y%m%dT%H%M%SZ).$$"
|
|
[ ! -e "$archive_dir" ] || fail "Handoff evidence archive already exists: $archive_dir"
|
|
mv -- "$HANDOFF_DIR" "$archive_dir"
|
|
chmod -R a-w "$archive_dir"
|
|
ok "Archived terminally restored handoff evidence at ${archive_dir}"
|
|
}
|
|
|
|
backup_proxy_files() {
|
|
archive_restored_handoff_if_present
|
|
[ ! -e "$HANDOFF_DIR" ] || fail "Handoff directory already exists; use the recorded state or choose the matching restore action: $HANDOFF_DIR"
|
|
mkdir -p "$HANDOFF_DIR"
|
|
chmod 700 "$HANDOFF_DIR"
|
|
|
|
if [ -f "$PROXY_CONF" ]; then
|
|
cp -a "$PROXY_CONF" "$HANDOFF_DIR/nginx.conf.before"
|
|
: > "$HANDOFF_DIR/had-proxy-conf"
|
|
fi
|
|
if [ -f "$PROXY_SERVICE_FILE" ]; then
|
|
cp -a "$PROXY_SERVICE_FILE" "$HANDOFF_DIR/proxy.service.before"
|
|
: > "$HANDOFF_DIR/had-proxy-service"
|
|
fi
|
|
if [ -f "$CURRENT_FILE" ]; then
|
|
cp -a "$CURRENT_FILE" "$HANDOFF_DIR/current-container.before"
|
|
: > "$HANDOFF_DIR/had-current-container"
|
|
fi
|
|
printf '%s\n' "$TARGET_COMMIT" > "$HANDOFF_DIR/target-commit"
|
|
printf '%s\n' "$LEGACY_CONTAINER_ID" > "$HANDOFF_DIR/legacy-container-id"
|
|
printf '%s\n' "$LEGACY_IMAGE_ID" > "$HANDOFF_DIR/legacy-image-id"
|
|
printf '%s\n' "$LEGACY_IMAGE_REPO_DIGEST" > "$HANDOFF_DIR/legacy-image-repo-digest"
|
|
printf '%s\n' "$LEGACY_UPSTREAM" > "$HANDOFF_DIR/legacy-upstream"
|
|
printf '%s\n' "$UPLOADS_STATFS_SIGNATURE" > "$HANDOFF_DIR/uploads-statfs-signature"
|
|
printf '%s\n' "$PRESERVED_CONTAINER" > "$HANDOFF_DIR/preserved-container"
|
|
chmod 600 "$HANDOFF_DIR"/*
|
|
}
|
|
|
|
published_endpoints_include_ipv4() {
|
|
case ",${PUBLISHED_ENDPOINTS}," in
|
|
*",0.0.0.0:${PORT},"*|*",127.0.0.1:${PORT},"*) return 0 ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
published_endpoints_include_ipv6() {
|
|
case ",${PUBLISHED_ENDPOINTS}," in
|
|
*",[::]:${PORT},"*) return 0 ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
write_maintenance_proxy_files() {
|
|
local tcp_listens=""
|
|
if published_endpoints_include_ipv4; then
|
|
tcp_listens=" listen 0.0.0.0:${PORT};"
|
|
fi
|
|
if published_endpoints_include_ipv6; then
|
|
tcp_listens="${tcp_listens}
|
|
listen [::]:${PORT} ipv6only=on;"
|
|
fi
|
|
[ -n "$tcp_listens" ] || {
|
|
warn "Pinned published endpoints do not contain a supported IPv4 or IPv6 listener"
|
|
return 1
|
|
}
|
|
mkdir -p "$PROXY_ROOT"
|
|
cat > "$HANDOFF_DIR/nginx.conf.maintenance" <<EOF
|
|
worker_processes 1;
|
|
pid ${PROXY_PID};
|
|
|
|
events { worker_connections 256; }
|
|
|
|
http {
|
|
access_log off;
|
|
server {
|
|
${tcp_listens}
|
|
listen unix:${MAINTENANCE_SOCKET};
|
|
server_name _;
|
|
default_type application/json;
|
|
location / {
|
|
add_header Retry-After 60 always;
|
|
add_header Cache-Control "no-store" always;
|
|
add_header X-Material-Proxy-Target legacy-bootstrap-maintenance always;
|
|
return 503 '{"status":"maintenance","reason":"legacy-worker-bootstrap"}';
|
|
}
|
|
}
|
|
}
|
|
EOF
|
|
cat > "$HANDOFF_DIR/proxy.service.maintenance" <<EOF
|
|
[Unit]
|
|
Description=Youyidian material local entry proxy (${CONTAINER_NAME})
|
|
After=network-online.target docker.service
|
|
Wants=network-online.target
|
|
Requires=docker.service
|
|
|
|
[Service]
|
|
Type=simple
|
|
ExecStartPre=/usr/sbin/nginx -p ${PROXY_ROOT}/ -c ${PROXY_CONF} -t
|
|
ExecStart=/usr/sbin/nginx -p ${PROXY_ROOT}/ -c ${PROXY_CONF} -g 'daemon off;'
|
|
ExecReload=/usr/sbin/nginx -p ${PROXY_ROOT}/ -c ${PROXY_CONF} -s reload
|
|
ExecStop=/usr/sbin/nginx -p ${PROXY_ROOT}/ -c ${PROXY_CONF} -s quit
|
|
Restart=always
|
|
RestartSec=2
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
EOF
|
|
chmod 600 "$HANDOFF_DIR/nginx.conf.maintenance" "$HANDOFF_DIR/proxy.service.maintenance"
|
|
cp -a "$HANDOFF_DIR/nginx.conf.maintenance" "$PROXY_CONF"
|
|
cp -a "$HANDOFF_DIR/proxy.service.maintenance" "$PROXY_SERVICE_FILE"
|
|
rm -f "$CURRENT_FILE"
|
|
rm -f "$MAINTENANCE_SOCKET"
|
|
nginx -p "${PROXY_ROOT}/" -c "$PROXY_CONF" -t
|
|
systemctl daemon-reload
|
|
systemctl start "$PROXY_SERVICE"
|
|
}
|
|
|
|
maintenance_socket_is_ready() {
|
|
local headers=""
|
|
local body=""
|
|
local status=""
|
|
headers="$(mktemp /tmp/gx-maintenance-socket-headers.XXXXXX)" || return 1
|
|
body="$(mktemp /tmp/gx-maintenance-socket-body.XXXXXX)" || {
|
|
rm -f "$headers"
|
|
return 1
|
|
}
|
|
status="$(curl -sS --max-time 5 --unix-socket "$MAINTENANCE_SOCKET" -D "$headers" -o "$body" -w '%{http_code}' "http://localhost/api/health" || true)"
|
|
if [ "$status" != "503" ] || ! tr -d '\r' < "$headers" | grep -Fqi 'X-Material-Proxy-Target: legacy-bootstrap-maintenance'; then
|
|
rm -f "$headers" "$body"
|
|
return 1
|
|
fi
|
|
rm -f "$headers" "$body"
|
|
}
|
|
|
|
maintenance_tcp_endpoint_is_ready() {
|
|
local endpoint="$1"
|
|
local headers=""
|
|
local body=""
|
|
local status=""
|
|
headers="$(mktemp /tmp/gx-maintenance-headers.XXXXXX)" || return 1
|
|
body="$(mktemp /tmp/gx-maintenance-body.XXXXXX)" || {
|
|
rm -f "$headers"
|
|
return 1
|
|
}
|
|
status="$(curl --noproxy '*' -g -sS --max-time 5 -D "$headers" -o "$body" -w '%{http_code}' "$endpoint" || true)"
|
|
if [ "$status" != "503" ] || ! tr -d '\r' < "$headers" | grep -Fqi 'X-Material-Proxy-Target: legacy-bootstrap-maintenance'; then
|
|
rm -f "$headers" "$body"
|
|
return 1
|
|
fi
|
|
rm -f "$headers" "$body"
|
|
}
|
|
|
|
maintenance_entry_is_ready() {
|
|
if published_endpoints_include_ipv4; then
|
|
maintenance_tcp_endpoint_is_ready "http://127.0.0.1:${PORT}/api/health" || return 1
|
|
fi
|
|
if published_endpoints_include_ipv6; then
|
|
maintenance_tcp_endpoint_is_ready "http://[::1]:${PORT}/api/health" || return 1
|
|
fi
|
|
}
|
|
|
|
restore_proxy_files() {
|
|
systemctl stop "$PROXY_SERVICE" >/dev/null 2>&1 || true
|
|
if [ -f "$HANDOFF_DIR/had-proxy-conf" ]; then
|
|
cp -a "$HANDOFF_DIR/nginx.conf.before" "$PROXY_CONF"
|
|
else
|
|
rm -f "$PROXY_CONF"
|
|
fi
|
|
if [ -f "$HANDOFF_DIR/had-proxy-service" ]; then
|
|
cp -a "$HANDOFF_DIR/proxy.service.before" "$PROXY_SERVICE_FILE"
|
|
else
|
|
rm -f "$PROXY_SERVICE_FILE"
|
|
fi
|
|
if [ -f "$HANDOFF_DIR/had-current-container" ]; then
|
|
cp -a "$HANDOFF_DIR/current-container.before" "$CURRENT_FILE"
|
|
else
|
|
rm -f "$CURRENT_FILE"
|
|
fi
|
|
systemctl daemon-reload >/dev/null 2>&1 || true
|
|
}
|
|
|
|
start_maintenance_fallback() {
|
|
write_maintenance_proxy_files
|
|
}
|
|
|
|
publish_maintenance_entry() {
|
|
# INPUT/DOCKER-USER rules deliberately block even loopback traffic on the
|
|
# published TCP port. Prove nginx through its private Unix socket first, then
|
|
# remove every rule and finally prove the real stable TCP entry.
|
|
maintenance_socket_is_ready || {
|
|
warn "Maintenance Unix-socket probe failed while traffic was still blocked"
|
|
return 1
|
|
}
|
|
delete_firewall_rules || {
|
|
warn "Maintenance entry is running but firewall cleanup could not be proven"
|
|
return 1
|
|
}
|
|
maintenance_entry_is_ready || {
|
|
warn "Maintenance TCP entry failed after firewall cleanup; attempting to block traffic again"
|
|
install_firewall_rules || warn "Traffic could not be re-blocked after the maintenance TCP probe failed"
|
|
return 1
|
|
}
|
|
}
|
|
|
|
leave_restore_failure_in_maintenance() {
|
|
local reason="$1"
|
|
warn "$reason"
|
|
start_maintenance_fallback || {
|
|
warn "Maintenance entry could not be started; traffic remains blocked"
|
|
return 1
|
|
}
|
|
publish_maintenance_entry || {
|
|
warn "Maintenance handoff could not be proven; restoration remains failed closed"
|
|
return 1
|
|
}
|
|
warn "Pinned legacy restoration failed; the explicit maintenance entry remains active"
|
|
}
|
|
|
|
verify_container_self_health() {
|
|
local name="$1"
|
|
docker exec "$name" node -e '
|
|
fetch("http://127.0.0.1:3000/api/health", { signal: AbortSignal.timeout(8000) })
|
|
.then(async (response) => {
|
|
const body = await response.text();
|
|
if (!response.ok || !body.includes("healthy")) process.exitCode = 1;
|
|
})
|
|
.catch(() => { process.exitCode = 1; });
|
|
' >/dev/null 2>&1
|
|
}
|
|
|
|
legacy_tcp_endpoint_is_healthy() {
|
|
curl --noproxy '*' -g -fsS --max-time 8 "$1" >/dev/null
|
|
}
|
|
|
|
legacy_entry_is_healthy() {
|
|
if published_endpoints_include_ipv4; then
|
|
legacy_tcp_endpoint_is_healthy "http://127.0.0.1:${PORT}/api/health" || return 1
|
|
fi
|
|
if published_endpoints_include_ipv6; then
|
|
legacy_tcp_endpoint_is_healthy "http://[::1]:${PORT}/api/health" || return 1
|
|
fi
|
|
}
|
|
|
|
rollback_restored_legacy_to_maintenance() {
|
|
# This path is used only after the restored public entry failed its final
|
|
# health probe. Block every new host/Docker-forwarded connection first, then
|
|
# prove both HTTP and background work are idle before stopping anything.
|
|
if ! install_firewall_rules; then
|
|
warn "Cannot prove traffic is blocked after restored-entry health failure; preserving the legacy container and failing closed for manual recovery"
|
|
return 1
|
|
fi
|
|
wait_for_two_zero_connection_samples || {
|
|
warn "Restored legacy HTTP connections did not drain during rollback; traffic remains blocked"
|
|
return 1
|
|
}
|
|
assert_database_idle_from_legacy || {
|
|
warn "Restored legacy background work is not idle during rollback; traffic remains blocked"
|
|
return 1
|
|
}
|
|
assert_no_ffmpeg_running_container || {
|
|
warn "Restored legacy FFmpeg activity is not idle during rollback; traffic remains blocked"
|
|
return 1
|
|
}
|
|
|
|
docker pause "$CONTAINER_NAME" >/dev/null || {
|
|
warn "Restored legacy container could not be paused during rollback; traffic remains blocked"
|
|
return 1
|
|
}
|
|
container_paused "$CONTAINER_NAME" || {
|
|
warn "Restored legacy paused state could not be proven during rollback; traffic remains blocked"
|
|
return 1
|
|
}
|
|
wait_for_two_zero_connection_samples || {
|
|
warn "Restored legacy connections reappeared after rollback pause; traffic remains blocked"
|
|
return 1
|
|
}
|
|
assert_database_idle_via_target_image || {
|
|
warn "Restored legacy database state changed after rollback pause; traffic remains blocked"
|
|
return 1
|
|
}
|
|
assert_no_ffmpeg_running_container || {
|
|
warn "Restored legacy FFmpeg activity appeared after rollback pause; traffic remains blocked"
|
|
return 1
|
|
}
|
|
[ "$(docker inspect -f '{{.Id}}' "$CONTAINER_NAME" 2>/dev/null || true)" = "$LEGACY_CONTAINER_ID" ] || {
|
|
warn "Restored legacy container ID changed before rollback stop; traffic remains blocked"
|
|
return 1
|
|
}
|
|
[ "$(docker inspect -f '{{.Image}}' "$CONTAINER_NAME" 2>/dev/null || true)" = "$LEGACY_IMAGE_ID" ] || {
|
|
warn "Restored legacy image ID changed before rollback stop; traffic remains blocked"
|
|
return 1
|
|
}
|
|
docker stop --signal SIGKILL --time 0 "$CONTAINER_NAME" >/dev/null || {
|
|
warn "Restored legacy container could not be force-stopped from paused state; traffic remains blocked"
|
|
return 1
|
|
}
|
|
container_running "$CONTAINER_NAME" && {
|
|
warn "Restored legacy container is still running after rollback stop; traffic remains blocked"
|
|
return 1
|
|
}
|
|
[ "$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME" 2>/dev/null || true)" = "exited" ] || {
|
|
warn "Restored legacy stopped state could not be proven during rollback; traffic remains blocked"
|
|
return 1
|
|
}
|
|
container_paused "$CONTAINER_NAME" && {
|
|
warn "Restored legacy container remained paused after rollback stop; traffic remains blocked"
|
|
return 1
|
|
}
|
|
[ "$(docker inspect -f '{{.Id}}' "$CONTAINER_NAME" 2>/dev/null || true)" = "$LEGACY_CONTAINER_ID" ] || {
|
|
warn "Restored legacy container ID changed after rollback stop; traffic remains blocked"
|
|
return 1
|
|
}
|
|
[ "$(docker inspect -f '{{.Image}}' "$CONTAINER_NAME" 2>/dev/null || true)" = "$LEGACY_IMAGE_ID" ] || {
|
|
warn "Restored legacy image ID changed after rollback stop; traffic remains blocked"
|
|
return 1
|
|
}
|
|
assert_database_idle_via_target_image || {
|
|
warn "Restored legacy database state is not idle after rollback stop; traffic remains blocked"
|
|
return 1
|
|
}
|
|
assert_no_host_ffmpeg || {
|
|
warn "Production host still has FFmpeg activity after rollback stop; traffic remains blocked"
|
|
return 1
|
|
}
|
|
docker rename "$CONTAINER_NAME" "$PRESERVED_CONTAINER" >/dev/null || {
|
|
warn "Restored legacy container could not be preserved during rollback; traffic remains blocked"
|
|
return 1
|
|
}
|
|
start_maintenance_fallback || {
|
|
warn "Maintenance entry could not be restarted after restored-entry health failure; traffic remains blocked"
|
|
return 1
|
|
}
|
|
publish_maintenance_entry || {
|
|
warn "Maintenance entry could not be safely published after restored-entry health failure"
|
|
return 1
|
|
}
|
|
}
|
|
|
|
restore_legacy_entry() {
|
|
local source_name=""
|
|
local current_id=""
|
|
local restored_upstream=""
|
|
|
|
if ! install_firewall_rules; then
|
|
warn "Could not prove every traffic block before legacy restoration; maintenance entry remains unchanged"
|
|
return 1
|
|
fi
|
|
systemctl stop "$PROXY_RECONCILE_TIMER" >/dev/null 2>&1 || true
|
|
systemctl disable "$PROXY_RECONCILE_TIMER" >/dev/null 2>&1 || true
|
|
systemctl stop "$PROXY_SERVICE" >/dev/null 2>&1 || true
|
|
|
|
if docker inspect "$PRESERVED_CONTAINER" >/dev/null 2>&1; then
|
|
source_name="$PRESERVED_CONTAINER"
|
|
elif docker inspect "$CONTAINER_NAME" >/dev/null 2>&1; then
|
|
source_name="$CONTAINER_NAME"
|
|
else
|
|
leave_restore_failure_in_maintenance "Pinned legacy container is missing during restoration" || true
|
|
return 1
|
|
fi
|
|
|
|
current_id="$(docker inspect -f '{{.Id}}' "$source_name" 2>/dev/null || true)"
|
|
[ "$current_id" = "$LEGACY_CONTAINER_ID" ] || {
|
|
leave_restore_failure_in_maintenance "Refusing to restore a container whose full ID does not match the pin" || true
|
|
return 1
|
|
}
|
|
|
|
if [ "$source_name" = "$PRESERVED_CONTAINER" ]; then
|
|
docker inspect "$CONTAINER_NAME" >/dev/null 2>&1 && {
|
|
leave_restore_failure_in_maintenance "Original legacy container name is occupied; restoration is blocked" || true
|
|
return 1
|
|
}
|
|
docker rename "$PRESERVED_CONTAINER" "$CONTAINER_NAME" || {
|
|
leave_restore_failure_in_maintenance "Pinned legacy container could not be renamed for restoration" || true
|
|
return 1
|
|
}
|
|
fi
|
|
|
|
if container_paused "$CONTAINER_NAME"; then
|
|
docker unpause "$CONTAINER_NAME" >/dev/null || {
|
|
leave_restore_failure_in_maintenance "Pinned legacy container could not be unpaused during restoration" || true
|
|
return 1
|
|
}
|
|
container_paused "$CONTAINER_NAME" && {
|
|
leave_restore_failure_in_maintenance "Pinned legacy container paused state remained ambiguous after unpause" || true
|
|
return 1
|
|
}
|
|
fi
|
|
if ! container_running "$CONTAINER_NAME"; then
|
|
docker start "$CONTAINER_NAME" >/dev/null || {
|
|
leave_restore_failure_in_maintenance "Pinned legacy container could not be restarted" || true
|
|
return 1
|
|
}
|
|
fi
|
|
restored_upstream="$(docker inspect -f "{{with index .NetworkSettings.Networks \"${NETWORK_NAME}\"}}{{.IPAddress}}{{end}}" "$CONTAINER_NAME" 2>/dev/null || true):3000"
|
|
if [ "$restored_upstream" != "$LEGACY_UPSTREAM" ]; then
|
|
warn "Restored legacy upstream identity changed: expected=${LEGACY_UPSTREAM}, actual=${restored_upstream:-missing}"
|
|
rollback_restored_legacy_to_maintenance || true
|
|
return 1
|
|
fi
|
|
if ! verify_container_self_health "$CONTAINER_NAME"; then
|
|
warn "Pinned legacy container restarted but did not pass its internal health check"
|
|
rollback_restored_legacy_to_maintenance || true
|
|
return 1
|
|
fi
|
|
|
|
restore_proxy_files
|
|
delete_firewall_rules || {
|
|
warn "Legacy identity was restarted, but bootstrap firewall removal could not be proven; restoration remains failed closed"
|
|
rollback_restored_legacy_to_maintenance || true
|
|
return 1
|
|
}
|
|
if ! legacy_entry_is_healthy; then
|
|
warn "Restored legacy public entry failed its final health probe; rolling back to explicit maintenance"
|
|
rollback_restored_legacy_to_maintenance || true
|
|
return 1
|
|
fi
|
|
local restored_tmp="$HANDOFF_DIR/.restored.$$"
|
|
printf '%s\n' "restored" > "$restored_tmp"
|
|
chmod 600 "$restored_tmp"
|
|
mv -f -- "$restored_tmp" "$HANDOFF_DIR/restored"
|
|
ok "Pinned legacy direct container and entry were restored"
|
|
}
|
|
|
|
assert_restore_is_safe() {
|
|
local name=""
|
|
local mount_line=""
|
|
local actual_target=""
|
|
local actual_source=""
|
|
local actual_fstype=""
|
|
local extra=""
|
|
local actual_signature=""
|
|
local preserved_metadata=""
|
|
local actual_endpoints=""
|
|
local expected_endpoints=""
|
|
|
|
[ -d "$HANDOFF_DIR" ] || fail "Bootstrap handoff evidence is missing: $HANDOFF_DIR"
|
|
[ "$(tr -d '\r\n' < "$HANDOFF_DIR/target-commit")" = "$TARGET_COMMIT" ] || fail "Handoff target commit does not match"
|
|
[ "$(tr -d '\r\n' < "$HANDOFF_DIR/legacy-container-id")" = "$LEGACY_CONTAINER_ID" ] || fail "Handoff legacy container ID does not match"
|
|
[ "$(tr -d '\r\n' < "$HANDOFF_DIR/legacy-image-id")" = "$LEGACY_IMAGE_ID" ] || fail "Handoff legacy image ID does not match"
|
|
[ "$(tr -d '\r\n' < "$HANDOFF_DIR/legacy-image-repo-digest")" = "$LEGACY_IMAGE_REPO_DIGEST" ] || fail "Handoff legacy RepoDigest does not match"
|
|
[ "$(tr -d '\r\n' < "$HANDOFF_DIR/legacy-upstream")" = "$LEGACY_UPSTREAM" ] || fail "Handoff legacy upstream does not match"
|
|
[ "$(tr -d '\r\n' < "$HANDOFF_DIR/uploads-statfs-signature")" = "$UPLOADS_STATFS_SIGNATURE" ] || fail "Handoff uploads statfs signature does not match"
|
|
|
|
for name in "$BLUE_CONTAINER" "$GREEN_CONTAINER"; do
|
|
docker inspect "$name" >/dev/null 2>&1 && fail "${name} exists; do not restore legacy after a candidate may have accepted traffic"
|
|
done
|
|
maintenance_entry_is_ready || fail "Runtime entry is not the bootstrap maintenance target on every pinned address family; automatic legacy restore is unsafe"
|
|
|
|
[ "$(docker inspect -f '{{.Id}}' "$PRESERVED_CONTAINER" 2>/dev/null || true)" = "$LEGACY_CONTAINER_ID" ] || fail "Preserved legacy container full ID changed or is missing"
|
|
[ "$(docker inspect -f '{{.Image}}' "$PRESERVED_CONTAINER" 2>/dev/null || true)" = "$LEGACY_IMAGE_ID" ] || fail "Preserved legacy image ID changed"
|
|
image_has_repo_digest "$LEGACY_IMAGE_ID" "$LEGACY_IMAGE_REPO_DIGEST" || fail "Preserved legacy RepoDigest is missing"
|
|
|
|
mount_line="$(findmnt -rn -o TARGET,SOURCE,FSTYPE -T "$UPLOADS_MOUNT" 2>/dev/null || true)"
|
|
read -r actual_target actual_source actual_fstype extra <<< "$mount_line"
|
|
[ -z "${extra:-}" ] || fail "Uploads findmnt identity is ambiguous during restore"
|
|
[ "$actual_target" = "$UPLOADS_MOUNT_TARGET" ] || fail "Uploads mount target changed before restore"
|
|
[ "$actual_source" = "$UPLOADS_MOUNT_SOURCE" ] || fail "Uploads mount source changed before restore"
|
|
[ "$actual_fstype" = "zfs" ] || fail "Uploads filesystem changed before restore"
|
|
actual_signature="$(stat -f -c '%i:%t' -- "$UPLOADS_MOUNT" | tr 'A-F' 'a-f')"
|
|
[ "$actual_signature" = "$UPLOADS_STATFS_SIGNATURE" ] || fail "Uploads statfs identity changed before restore"
|
|
|
|
preserved_metadata="$(docker inspect --format '{{range .Mounts}}{{printf "%s\t%s\t%s\t%t\n" .Destination .Type .Source .RW}}{{end}}' "$PRESERVED_CONTAINER")"
|
|
[ "$(printf '%s\n' "$preserved_metadata" | awk -F '\t' -v source="$UPLOADS_MOUNT" '$1 == "/app/uploads" && $2 == "bind" && $3 == source && $4 == "true" { count++ } END { print count + 0 }')" = "1" ] ||
|
|
fail "Preserved legacy container uploads bind changed before restore"
|
|
[ "$(printf '%s\n' "$preserved_metadata" | awk -F '\t' '$1 == "/app/uploads" { count++ } END { print count + 0 }')" = "1" ] ||
|
|
fail "Preserved legacy container has ambiguous uploads mounts"
|
|
|
|
actual_endpoints="$(docker port "$PRESERVED_CONTAINER" 3000/tcp 2>/dev/null | tr -d '\r' | sort -u | paste -sd, -)"
|
|
expected_endpoints="$(normalise_endpoint_list "$PUBLISHED_ENDPOINTS")"
|
|
[ "$actual_endpoints" = "$expected_endpoints" ] || fail "Preserved legacy published endpoint set changed before restore"
|
|
}
|
|
|
|
for command_name in docker curl flock ss nsenter findmnt stat nginx systemctl iptables ip6tables python3 awk sed grep ps; do
|
|
require_cmd "$command_name"
|
|
done
|
|
[ -f "$ENVIRONMENT_FILE" ] || fail "Production environment file is missing: $ENVIRONMENT_FILE"
|
|
docker network inspect "$NETWORK_NAME" >/dev/null 2>&1 || fail "Pinned application network is missing: $NETWORK_NAME"
|
|
|
|
if [ "$MODE" = "RestoreLegacy" ]; then
|
|
exec 9>"$DEPLOY_LOCK"
|
|
flock -w 30 9 || fail "Another legacy bootstrap/deployment transition holds the deployment lock"
|
|
exec 8>"$PROXY_RECONCILE_LOCK"
|
|
flock -w 30 8 || fail "Another proxy switch or reconciliation holds the proxy lock"
|
|
assert_restore_is_safe
|
|
restore_legacy_entry || fail "Automatic legacy restoration failed; maintenance entry or preserved container was retained for manual recovery"
|
|
ok "Legacy bootstrap handoff ${HANDOFF_ID} was restored without deleting the old container or image"
|
|
exit 0
|
|
fi
|
|
|
|
assert_exact_container_and_image
|
|
verify_container_self_health "$CONTAINER_NAME" || fail "Pinned legacy container does not pass its internal health endpoint"
|
|
assert_target_image
|
|
assert_uploads_mount_identity
|
|
assert_topology_is_legacy_only
|
|
assert_database_idle_from_legacy
|
|
assert_no_ffmpeg_running_container
|
|
|
|
if [ "$MODE" = "Audit" ]; then
|
|
current_connections="$(legacy_connection_counts)" || fail "Cannot obtain both host and pinned-container connection evidence"
|
|
read -r current_host_connections current_container_connections current_extra <<< "$current_connections"
|
|
[ -z "${current_extra:-}" ] || fail "Legacy connection evidence is ambiguous"
|
|
log "Current established legacy connections: host=${current_host_connections}, container-netns=${current_container_connections}"
|
|
ok "Read-only legacy bootstrap audit passed; no traffic, container, proxy or database state was changed"
|
|
exit 0
|
|
fi
|
|
|
|
# The target image must already be staged. Running only `cat` through an
|
|
# overridden entrypoint verifies immutable build provenance before traffic is
|
|
# touched; it does not start the application or its background workers.
|
|
assert_target_baked_commit
|
|
|
|
exec 9>"$DEPLOY_LOCK"
|
|
flock -w 30 9 || fail "Another legacy bootstrap/deployment transition holds the deployment lock"
|
|
exec 8>"$PROXY_RECONCILE_LOCK"
|
|
flock -w 30 8 || fail "Another proxy switch or reconciliation holds the proxy lock"
|
|
|
|
# Repeat every volatile pin under both locks. No production mutation occurs
|
|
# before these checks pass.
|
|
assert_exact_container_and_image
|
|
verify_container_self_health "$CONTAINER_NAME" || fail "Pinned legacy container health changed under the transition locks"
|
|
assert_target_image
|
|
assert_uploads_mount_identity
|
|
assert_topology_is_legacy_only
|
|
assert_database_idle_from_legacy
|
|
assert_no_ffmpeg_running_container
|
|
|
|
backup_proxy_files
|
|
HANDOFF_COMMITTED=0
|
|
AUTO_RESTORE_ALLOWED=1
|
|
trap 'status=$?; trap - EXIT; if [ "$status" -ne 0 ] && [ "$HANDOFF_COMMITTED" != "1" ]; then if [ "$AUTO_RESTORE_ALLOWED" = "1" ]; then restore_legacy_entry || true; else warn "Automatic restart is intentionally disabled because the force-stop or its post-stop evidence was not proven; keep traffic blocked and recover manually"; fi; fi; exit "$status"' EXIT
|
|
mkdir -p "$(dirname "$VIDEO_DRAIN_FILE")" "$(dirname "$TASK_DETAIL_DRAIN_FILE")"
|
|
: > "$VIDEO_DRAIN_FILE"
|
|
: > "$TASK_DETAIL_DRAIN_FILE"
|
|
chmod 600 "$VIDEO_DRAIN_FILE" "$TASK_DETAIL_DRAIN_FILE"
|
|
ok "Candidate video and task-detail workers are pre-drained without fabricating capability files"
|
|
|
|
install_firewall_rules
|
|
ok "New host-local, Docker-forwarded and host-input connections to the legacy upstream are blocked"
|
|
wait_for_two_zero_connection_samples || fail "Legacy HTTP connections did not reach two consecutive zero samples"
|
|
assert_database_idle_from_legacy
|
|
assert_no_ffmpeg_running_container
|
|
|
|
# Pausing closes the final worker check/use race. Database snapshots are
|
|
# executed by the pinned target image with an overridden node entrypoint, so no
|
|
# candidate application or background worker is started.
|
|
docker pause "$CONTAINER_NAME" >/dev/null
|
|
container_paused "$CONTAINER_NAME" || fail "Legacy container pause could not be proven"
|
|
wait_for_two_zero_connection_samples || fail "Legacy upstream connections reappeared after pause"
|
|
assert_database_idle_via_target_image
|
|
assert_no_ffmpeg_running_container
|
|
|
|
[ "$(docker inspect -f '{{.Id}}' "$CONTAINER_NAME")" = "$LEGACY_CONTAINER_ID" ] || fail "Legacy container ID changed immediately before stop"
|
|
[ "$(docker inspect -f '{{.Image}}' "$CONTAINER_NAME")" = "$LEGACY_IMAGE_ID" ] || fail "Legacy image ID changed immediately before stop"
|
|
# A normal `docker stop` resumes a paused container after sending its graceful
|
|
# signal. Use SIGKILL before that resume so legacy user code cannot run again.
|
|
# Until stopped state plus fresh DB/host evidence are both proven, failure stays
|
|
# closed instead of automatically restarting an identity whose state is unknown.
|
|
AUTO_RESTORE_ALLOWED=0
|
|
docker stop --signal SIGKILL --time 0 "$CONTAINER_NAME" >/dev/null
|
|
container_running "$CONTAINER_NAME" && fail "Legacy container is still running after SIGKILL stop"
|
|
[ "$(docker inspect -f '{{.State.Status}}' "$CONTAINER_NAME" 2>/dev/null || true)" = "exited" ] || fail "Legacy container stopped state could not be proven after SIGKILL"
|
|
container_paused "$CONTAINER_NAME" && fail "Legacy container remained paused after SIGKILL stop"
|
|
[ "$(docker inspect -f '{{.Id}}' "$CONTAINER_NAME")" = "$LEGACY_CONTAINER_ID" ] || fail "Legacy container ID changed immediately after SIGKILL stop"
|
|
[ "$(docker inspect -f '{{.Image}}' "$CONTAINER_NAME")" = "$LEGACY_IMAGE_ID" ] || fail "Legacy image ID changed immediately after SIGKILL stop"
|
|
assert_database_idle_via_target_image
|
|
assert_no_host_ffmpeg
|
|
AUTO_RESTORE_ALLOWED=1
|
|
docker rename "$CONTAINER_NAME" "$PRESERVED_CONTAINER"
|
|
[ "$(docker inspect -f '{{.Id}}' "$PRESERVED_CONTAINER")" = "$LEGACY_CONTAINER_ID" ] || fail "Preserved legacy container ID does not match after rename"
|
|
[ "$(docker inspect -f '{{.Image}}' "$PRESERVED_CONTAINER")" = "$LEGACY_IMAGE_ID" ] || fail "Preserved legacy image ID does not match after rename"
|
|
|
|
write_maintenance_proxy_files
|
|
publish_maintenance_entry || fail "Stable maintenance entry could not be proven through both its private socket and public TCP endpoint"
|
|
printf '%s\n' "ready" > "$HANDOFF_DIR/handoff.ready"
|
|
chmod 600 "$HANDOFF_DIR/handoff.ready"
|
|
HANDOFF_COMMITTED=1
|
|
trap - EXIT
|
|
|
|
ok "Legacy container is stopped and preserved as ${PRESERVED_CONTAINER}; image ${LEGACY_IMAGE_ID} was not removed"
|
|
ok "Stable local entry is serving an explicit 503 maintenance response on port ${PORT}"
|
|
ok "Handoff ${HANDOFF_ID} is ready for the normal exact-SHA blue/green deployment"
|
|
log "Run scripts/git-incremental-deploy.ps1 immediately with -Target production -Commit ${TARGET_COMMIT}."
|
|
log "Do not bypass its video/task-detail capability, schema, image-provenance or health gates."
|
|
log "If that deployment fails before any blue/green target accepts traffic, run this script again with -Mode RestoreLegacy -Apply -AllowLegacyDrainBootstrap and the identical pins."
|
|
'@
|
|
|
|
$remoteScript = $remoteTemplate
|
|
foreach ($entry in $tokens.GetEnumerator()) {
|
|
$remoteScript = $remoteScript.Replace($entry.Key, $entry.Value)
|
|
}
|
|
|
|
$remoteScript = $remoteScript.Replace("`r`n", "`n").Replace("`r", "`n")
|
|
$ssh = (Get-Command 'ssh.exe' -ErrorAction Stop).Source
|
|
$sshArgs = @(
|
|
'-o', 'BatchMode=yes',
|
|
'-o', 'StrictHostKeyChecking=yes',
|
|
'-o', 'ConnectTimeout=10'
|
|
)
|
|
if (-not [string]::IsNullOrWhiteSpace($ProdServerProxyCommand)) {
|
|
$sshArgs += '-o'
|
|
$sshArgs += "ProxyCommand=$ProdServerProxyCommand"
|
|
}
|
|
$sshArgs += "${ProdServerUser}@${ProdServerIP}"
|
|
$sshArgs += 'bash -s'
|
|
|
|
$previousOutputEncoding = $OutputEncoding
|
|
try {
|
|
# Keep the large remote program on stdin. Passing its Base64 form as one
|
|
# command-line argument exceeds Windows CreateProcess limits.
|
|
$OutputEncoding = [System.Text.UTF8Encoding]::new($false)
|
|
$remoteScript | & $ssh @sshArgs
|
|
$remoteExitCode = $LASTEXITCODE
|
|
}
|
|
finally {
|
|
$OutputEncoding = $previousOutputEncoding
|
|
}
|
|
if ($remoteExitCode -ne 0) {
|
|
throw "Legacy worker bootstrap mode $Mode failed with exit code $remoteExitCode. Review the remote evidence; no automatic capability bypass is available."
|
|
}
|