import assert from 'node:assert/strict' import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import os from 'node:os' import path from 'node:path' import { spawnSync } from 'node:child_process' import test from 'node:test' const scriptPath = path.join(process.cwd(), 'scripts', 'legacy-worker-bootstrap-deploy.ps1') const script = readFileSync(scriptPath, 'utf8').replace(/\r\n/g, '\n') function indexAfter(needle, after = 0) { const index = script.indexOf(needle, after) assert.ok(index > after, `missing or out-of-order marker: ${needle}`) return index } function extractRemoteBash() { const marker = "$remoteTemplate = @'" const start = script.indexOf(marker) assert.ok(start >= 0, 'remote bash here-string start is missing') const bodyStart = start + marker.length const end = script.indexOf("\n'@", bodyStart) assert.ok(end > bodyStart, 'remote bash here-string terminator is missing') return script.slice(bodyStart, end).replace(/^\r?\n/, '') } function extractBashFunction(name) { const payload = extractRemoteBash() const match = payload.match(new RegExp(`(?:^|\\n)${name}\\(\\) \\{[\\s\\S]*?\\n\\}`, 'm')) assert.ok(match, `remote Bash function is missing: ${name}`) return match[0].trimStart() } function findBash() { const candidates = process.platform === 'win32' ? [ 'C:\\Program Files\\Git\\bin\\bash.exe', 'C:\\Program Files\\Git\\usr\\bin\\bash.exe', ] : ['/bin/bash', '/usr/bin/bash'] return candidates.find(existsSync) || null } test('PowerShell 5 parser accepts the one-time bootstrap entry', () => { const command = [ '$tokens=$null', '$errors=$null', `[System.Management.Automation.Language.Parser]::ParseFile('${scriptPath.replaceAll("'", "''")}',[ref]$tokens,[ref]$errors) | Out-Null`, 'if ($errors.Count -ne 0) { $errors | ForEach-Object { Write-Error $_.Message }; exit 1 }', ].join('; ') const result = spawnSync('powershell.exe', ['-NoProfile', '-Command', command], { encoding: 'utf8' }) assert.equal(result.status, 0, result.stderr || result.stdout) }) test('embedded remote Bash passes syntax validation after token substitution', () => { const bash = findBash() assert.ok(bash, 'Git Bash or system bash is required for syntax validation') let payload = extractRemoteBash() const replacements = { __MODE__: "'Audit'", __TARGET_COMMIT__: `'${'1'.repeat(40)}'`, __LEGACY_CONTAINER_ID__: `'${'2'.repeat(64)}'`, __LEGACY_IMAGE_ID__: `'sha256:${'3'.repeat(64)}'`, __LEGACY_IMAGE_REPO_DIGEST__: `'repo/image@sha256:${'4'.repeat(64)}'`, __LEGACY_UPSTREAM__: "'172.18.0.2:3000'", __UPLOADS_MOUNT__: "'/mnt/PoolOne/youyidian-uploads'", __UPLOADS_MOUNT_TARGET__: "'/mnt/PoolOne/youyidian-uploads'", __UPLOADS_MOUNT_SOURCE__: "'PoolOne/youyidian-uploads'", __UPLOADS_STATFS_SIGNATURE__: "'1234:abcd'", __PUBLISHED_ENDPOINTS__: "'0.0.0.0:4320,[::]:4320'", __TARGET_IMAGE_REFERENCE__: `'repo/image@sha256:${'5'.repeat(64)}'`, __TARGET_IMAGE_ID__: `'sha256:${'6'.repeat(64)}'`, __TARGET_IMAGE_REPO_DIGEST__: `'repo/image@sha256:${'5'.repeat(64)}'`, __CONTAINER_NAME__: "'youyidian-3d-material'", __NETWORK_NAME__: "'youyidian-3d_app-network'", __ENVIRONMENT_FILE__: "'/mnt/PoolOne/apps/youyidian-3d/.env'", __PORT__: "'4320'", } for (const [token, value] of Object.entries(replacements)) { payload = payload.replaceAll(token, value) } const result = spawnSync(bash, ['-n'], { input: payload, encoding: 'utf8' }) assert.equal(result.status, 0, result.stderr || result.stdout) }) test('connection-drain simulation requires two consecutive zero samples from both host and container netns', () => { const bash = findBash() assert.ok(bash, 'Git Bash or system bash is required for drain simulation') const fixture = mkdtempSync(path.join(os.tmpdir(), 'gx-bootstrap-drain-')) const sequencePath = path.join(fixture, 'sequence.txt') const helper = extractBashFunction('wait_for_two_zero_connection_samples') const harness = ` set -u ${helper} sleep() { :; } log() { :; } ok() { :; } legacy_connection_counts() { local value="" IFS= read -r value < "$SEQUENCE_FILE" tail -n +2 "$SEQUENCE_FILE" > "${sequencePath.replaceAll('\\', '/')}.next" mv "${sequencePath.replaceAll('\\', '/')}.next" "$SEQUENCE_FILE" printf '%s\\n' "$value" } wait_for_two_zero_connection_samples ` try { writeFileSync(sequencePath, '0 3\n0 0\n0 2\n0 0\n0 0\n', 'utf8') const succeeds = spawnSync(bash, ['-c', harness], { encoding: 'utf8', env: { ...process.env, MSYS2_ARG_CONV_EXCL: '*', MSYS_NO_PATHCONV: '1', LEGACY_UPSTREAM: '172.18.0.2:3000', SEQUENCE_FILE: sequencePath.replaceAll('\\', '/'), }, }) assert.equal(succeeds.status, 0, succeeds.stderr || succeeds.stdout) assert.equal(readFileSync(sequencePath, 'utf8').trim(), '', 'container-netns activity must reset the consecutive-zero counter') writeFileSync(sequencePath, Array.from({ length: 60 }, (_, index) => index % 2 ? '0 1' : '0 0').join('\n') + '\n', 'utf8') const alternating = spawnSync(bash, ['-c', harness], { encoding: 'utf8', env: { ...process.env, MSYS2_ARG_CONV_EXCL: '*', MSYS_NO_PATHCONV: '1', LEGACY_UPSTREAM: '172.18.0.2:3000', SEQUENCE_FILE: sequencePath.replaceAll('\\', '/'), }, }) assert.notEqual(alternating.status, 0, 'container-netns activity between zero samples must prevent drain') writeFileSync(sequencePath, '0\n', 'utf8') const missingNetnsEvidence = spawnSync(bash, ['-c', harness], { encoding: 'utf8', env: { ...process.env, MSYS2_ARG_CONV_EXCL: '*', MSYS_NO_PATHCONV: '1', LEGACY_UPSTREAM: '172.18.0.2:3000', SEQUENCE_FILE: sequencePath.replaceAll('\\', '/'), }, }) assert.notEqual(missingNetnsEvidence.status, 0, 'missing container-netns evidence must fail closed') } finally { rmSync(fixture, { recursive: true, force: true }) } }) test('firewall cleanup simulation fails closed on delete failure or residual rule', () => { const bash = findBash() assert.ok(bash, 'Git Bash or system bash is required for firewall simulation') const helper = extractBashFunction('delete_firewall_rules') const harness = ` set -u ${helper} warn() { printf '%s\\n' "$*" >&2; } state_v4_output=1 state_v4_docker=1 state_v4_input=1 state_v6_output=1 state_v6_input=1 state_v6_docker=1 firewall_rule() { local action="$1" chain="$2" key="" [ "$chain" = OUTPUT ] && key=v4_output || key=v4_input if [ "$action" = -C ]; then [ "$(eval "printf '%s' \\"\\$state_\${key}\\"")" = 1 ] return fi if [ "$action" = -D ]; then [ "\${FAIL_TARGET:-}" = "$key" ] && return 2 eval "state_\${key}=0" return 0 fi return 2 } firewall_original_port_rule() { if [ "$1" = -C ]; then [ "$state_v4_docker" = 1 ]; return; fi if [ "$1" = -D ]; then [ "\${FAIL_TARGET:-}" = v4_docker ] && return 2 state_v4_docker=0 return 0 fi return 2 } firewall_ipv6_rule() { local action="$1" chain="$2" key="" [ "$chain" = OUTPUT ] && key=v6_output || key=v6_input if [ "$action" = -C ]; then [ "$(eval "printf '%s' \\"\\$state_\${key}\\"")" = 1 ] return fi if [ "$action" = -D ]; then [ "\${FAIL_TARGET:-}" = "$key" ] && return 2 eval "state_\${key}=0" return 0 fi return 2 } firewall_ipv6_original_port_rule() { if [ "$1" = -C ]; then [ "$state_v6_docker" = 1 ]; return; fi if [ "$1" = -D ]; then [ "\${FAIL_TARGET:-}" = v6_docker ] && return 2 state_v6_docker=0 return 0 fi return 2 } delete_firewall_rules result=$? printf 'result=%s states=%s%s%s%s%s%s\\n' "$result" "$state_v4_output" "$state_v4_docker" "$state_v4_input" "$state_v6_output" "$state_v6_input" "$state_v6_docker" exit "$result" ` const run = (failTarget = '') => spawnSync(bash, ['-c', harness], { encoding: 'utf8', env: { ...process.env, FAIL_TARGET: failTarget, LEGACY_UPSTREAM: '172.18.0.2:3000', FIREWALL_COMMENT: 'test-bootstrap', PORT: '4320', }, }) const succeeds = run() assert.equal(succeeds.status, 0, succeeds.stderr || succeeds.stdout) assert.match(succeeds.stdout, /result=0 states=000000/) const deleteFails = run('v4_docker') assert.notEqual(deleteFails.status, 0, 'a failed -D operation must fail cleanup') assert.match(deleteFails.stderr, /Failed to remove bootstrap IPv4 DOCKER-USER rule/) assert.match(deleteFails.stderr, /remains after cleanup/) const ipv6DeleteFails = run('v6_docker') assert.notEqual(ipv6DeleteFails.status, 0, 'a failed IPv6 DOCKER-USER -D operation must fail cleanup') assert.match(ipv6DeleteFails.stderr, /Failed to remove bootstrap IPv6 DOCKER-USER rule/) assert.match(ipv6DeleteFails.stderr, /remains after cleanup/) }) test('handoff ready and restored markers are unreachable until firewall absence is proven', () => { const handoffPublish = script.lastIndexOf('publish_maintenance_entry || fail') const handoffReadyFile = indexAfter('"ready" > "$HANDOFF_DIR/handoff.ready"', handoffPublish) const restoreFunction = script.indexOf('restore_legacy_entry() {') const restoreCleanup = script.indexOf('delete_firewall_rules || {', restoreFunction) const restoreHealth = script.indexOf('if ! legacy_entry_is_healthy; then', restoreCleanup) const restoredMarker = script.indexOf('mv -f -- "$restored_tmp" "$HANDOFF_DIR/restored"', restoreHealth) const restoreModeCall = script.indexOf('restore_legacy_entry || fail', restoredMarker) assert.ok(handoffPublish > 0) assert.ok(handoffReadyFile > handoffPublish) assert.ok(restoreCleanup > restoreFunction) assert.ok(restoreHealth > restoreCleanup) assert.ok(restoredMarker > restoreHealth) assert.ok(restoreModeCall > restoredMarker) }) test('maintenance publication proves a private Unix socket before unblocking the public TCP entry', () => { const publishFunction = script.indexOf('publish_maintenance_entry() {') const socketProbe = indexAfter('maintenance_socket_is_ready', publishFunction) const cleanup = indexAfter('delete_firewall_rules', socketProbe) const tcpProbe = indexAfter('maintenance_entry_is_ready', cleanup) const reblock = indexAfter('install_firewall_rules', tcpProbe) assert.ok(socketProbe > publishFunction) assert.ok(cleanup > socketProbe) assert.ok(tcpProbe > cleanup) assert.ok(reblock > tcpProbe) assert.match(script, /MAINTENANCE_SOCKET="\$\{PROXY_ROOT\}\/maintenance\.sock"/) assert.match(script, /listen unix:\$\{MAINTENANCE_SOCKET\};/) assert.match(script, /listen 0\.0\.0\.0:\$\{PORT\};/) assert.match(script, /listen \[::\]:\$\{PORT\} ipv6only=on;/) assert.match(script, /curl .*--unix-socket "\$MAINTENANCE_SOCKET".*http:\/\/localhost\/api\/health/) assert.match(script, /maintenance_tcp_endpoint_is_ready "http:\/\/127\.0\.0\.1:\$\{PORT\}\/api\/health"/) assert.match(script, /maintenance_tcp_endpoint_is_ready "http:\/\/\[::1\]:\$\{PORT\}\/api\/health"/) assert.match(script, /curl --noproxy '\*' -g/) }) test('maintenance publication simulation never removes rules before the private probe', () => { const bash = findBash() assert.ok(bash, 'Git Bash or system bash is required for maintenance publication simulation') const helper = extractBashFunction('publish_maintenance_entry') const harness = ` set -u ${helper} warn() { printf 'warn:%s\\n' "$*"; } maintenance_socket_is_ready() { printf 'socket\\n'; return "\${SOCKET_RESULT}"; } delete_firewall_rules() { printf 'cleanup\\n'; return "\${CLEANUP_RESULT}"; } maintenance_entry_is_ready() { printf 'tcp\\n'; return "\${TCP_RESULT}"; } install_firewall_rules() { printf 'reblock\\n'; return 0; } publish_maintenance_entry ` const run = (socketResult, cleanupResult, tcpResult) => spawnSync(bash, ['-c', harness], { encoding: 'utf8', env: { ...process.env, MSYS2_ARG_CONV_EXCL: '*', MSYS_NO_PATHCONV: '1', SOCKET_RESULT: String(socketResult), CLEANUP_RESULT: String(cleanupResult), TCP_RESULT: String(tcpResult), }, }) const succeeds = run(0, 0, 0) assert.equal(succeeds.status, 0, succeeds.stderr || succeeds.stdout) assert.equal(succeeds.stdout.trim(), 'socket\ncleanup\ntcp') const socketFails = run(1, 0, 0) assert.notEqual(socketFails.status, 0) assert.equal(socketFails.stdout.trim(), 'socket\nwarn:Maintenance Unix-socket probe failed while traffic was still blocked') assert.doesNotMatch(socketFails.stdout, /cleanup|tcp|reblock/) const tcpFails = run(0, 0, 1) assert.notEqual(tcpFails.status, 0) assert.match(tcpFails.stdout.trim(), /^socket\ncleanup\ntcp\nwarn:.*\nreblock$/) }) test('write modes are disabled unless both explicit authorization switches are present', () => { assert.match(script, /\[ValidateSet\('Audit', 'EnterMaintenance', 'RestoreLegacy'\)\]/) assert.match(script, /\[switch\]\$Apply/) assert.match(script, /\[switch\]\$AllowLegacyDrainBootstrap/) assert.match(script, /-not \(\$Apply -and \$AllowLegacyDrainBootstrap\)/) assert.match(script, /Mode=Audit does not accept write authorization switches/) }) test('write authorization guard rejects an actual PowerShell invocation before SSH', () => { const args = [ '-NoProfile', '-File', scriptPath, '-Mode', 'EnterMaintenance', '-TargetCommit', '1'.repeat(40), '-LegacyContainerId', '2'.repeat(64), '-LegacyImageId', `sha256:${'3'.repeat(64)}`, '-LegacyImageRepoDigest', `repo/legacy@sha256:${'4'.repeat(64)}`, '-LegacyUpstream', '172.18.0.2:3000', '-UploadsStatfsSignature', '1234:abcd', '-TargetImageReference', `repo/target@sha256:${'5'.repeat(64)}`, '-TargetImageId', `sha256:${'6'.repeat(64)}`, '-TargetImageRepoDigest', `repo/target@sha256:${'5'.repeat(64)}`, '-ProdServerIP', 'prod.example.internal', ] const result = spawnSync('powershell.exe', args, { encoding: 'utf8' }) assert.notEqual(result.status, 0) assert.match(`${result.stdout}\n${result.stderr}`, /disabled by default/) assert.doesNotMatch(`${result.stdout}\n${result.stderr}`, /ssh/i) }) test('all mutable identities are explicitly pinned before the maintenance window', () => { for (const expected of [ '^[0-9a-f]{40}$', '^[0-9a-f]{64}$', '^sha256:[0-9a-f]{64}$', 'LegacyImageRepoDigest', 'LegacyUpstream', 'UploadsStatfsSignature', 'TargetImageReference', 'TargetImageId', 'TargetImageRepoDigest', 'PUBLISHED_ENDPOINTS', ]) { assert.ok(script.includes(expected), `missing identity pin: ${expected}`) } assert.match(script, /actual_id=.*docker inspect -f '\{\{\.Id\}\}'/) assert.match(script, /actual_image_id=.*docker inspect -f '\{\{\.Image\}\}'/) assert.match(script, /actual_upstream=.*NetworkSettings\.Networks/) assert.match(script, /findmnt -rn -o TARGET,SOURCE,FSTYPE/) assert.match(script, /stat -f -c '%i:%t'/) assert.match(script, /org\.opencontainers\.image\.revision/) assert.match(script, /cat \/app\/DEPLOYMENT_COMMIT/) }) test('critical mutation is serialized by both deployment and proxy flocks', () => { const deployLock = indexAfter('exec 9>"$DEPLOY_LOCK"') const deployFlock = indexAfter('flock -w 30 9', deployLock) const proxyLock = indexAfter('exec 8>"$PROXY_RECONCILE_LOCK"', deployFlock) const proxyFlock = indexAfter('flock -w 30 8', proxyLock) const repeatedPin = indexAfter('assert_exact_container_and_image', proxyFlock) const firstDrainWrite = indexAfter(': > "$VIDEO_DRAIN_FILE"', repeatedPin) assert.ok(firstDrainWrite > proxyFlock) assert.match(script, /DEPLOY_LOCK="\/run\/\$\{CONTAINER_NAME\}-production-deploy\.lock"/) assert.match(script, /PROXY_RECONCILE_LOCK="\/run\/\$\{CONTAINER_NAME\}-local-proxy-reconcile\.lock"/) }) test('new traffic is rejected while two zero HTTP samples and worker-idle evidence are collected', () => { const installFirewall = script.lastIndexOf('install_firewall_rules\n') assert.ok(installFirewall > 0) const twoZeros = indexAfter('wait_for_two_zero_connection_samples || fail', installFirewall) const prePauseDb = indexAfter('assert_database_idle_from_legacy', twoZeros) const prePauseFfmpeg = indexAfter('assert_no_ffmpeg_running_container', prePauseDb) const pause = indexAfter('docker pause "$CONTAINER_NAME"', prePauseFfmpeg) const postPauseZeros = indexAfter('wait_for_two_zero_connection_samples || fail', pause) const postPauseDb = indexAfter('assert_database_idle_via_target_image', postPauseZeros) const postPauseFfmpeg = indexAfter('assert_no_ffmpeg_running_container', postPauseDb) const disableAutomaticRestore = indexAfter('AUTO_RESTORE_ALLOWED=0', postPauseFfmpeg) const stop = indexAfter('docker stop --signal SIGKILL --time 0 "$CONTAINER_NAME"', disableAutomaticRestore) const stoppedState = indexAfter("{{.State.Status}}", stop) const postStopDb = indexAfter('assert_database_idle_via_target_image', stoppedState) const postStopFfmpeg = indexAfter('assert_no_host_ffmpeg', postStopDb) const enableAutomaticRestore = indexAfter('AUTO_RESTORE_ALLOWED=1', postStopFfmpeg) assert.ok(enableAutomaticRestore > stop) assert.match(script, /--ctorigdstport "\$PORT"/) assert.match(script, /ip6tables -w 5 "\$action" DOCKER-USER.*--ctorigdstport "\$PORT"/) assert.match(script, /--ctstate NEW/) assert.match(script, /ip6tables -w 5/) assert.match(script, /zero_samples" -ge 2/) assert.match(script, /nsenter -t "\$container_pid" -n -- ss -Htan state established '\( sport = :3000 \)'/) assert.match(script, /host_count" = "0".*container_count" = "0"/s) assert.match(script, /stat -Lc '%d:%i'.*\/proc\/\$\{container_pid\}\/ns\/net/s) assert.match(script, /current_pid" = "\$container_pid"/) assert.match(script, /videoQueuedOrProcessingCount/) assert.match(script, /taskJobQueuedOrProcessingCount/) assert.match(script, /taskStatusUploadingCount/) assert.match(script, /taskStatusUploadingRows/) }) test('only terminally restored handoffs are archived before an identical retry', () => { const archiveFunction = script.indexOf('archive_restored_handoff_if_present() {') const terminalMarker = indexAfter('[ -f "$HANDOFF_DIR/restored" ]', archiveFunction) const identityCommit = indexAfter('Restored handoff target commit does not match', terminalMarker) const identityContainer = indexAfter('Restored handoff legacy container ID does not match', identityCommit) const archiveMove = indexAfter('mv -- "$HANDOFF_DIR" "$archive_dir"', identityContainer) const makeReadonly = indexAfter('chmod -R a-w "$archive_dir"', archiveMove) const backupFunction = script.indexOf('backup_proxy_files() {') const archiveCall = indexAfter('archive_restored_handoff_if_present', backupFunction) const createNew = indexAfter('mkdir -p "$HANDOFF_DIR"', archiveCall) const restoredWrite = script.indexOf('mv -f -- "$restored_tmp" "$HANDOFF_DIR/restored"') const restoreSuccess = indexAfter('ok "Pinned legacy direct container and entry were restored"', restoredWrite) assert.ok(archiveFunction > 0) assert.ok(makeReadonly > archiveMove) assert.ok(createNew > archiveCall) assert.ok(restoreSuccess > restoredWrite) assert.match(script, /Existing handoff is not terminally restored and cannot be reused/) assert.doesNotMatch(script, /rm\s+-rf\s+.*HANDOFF_DIR/) }) test('candidate is pre-drained without writing any forged capability', () => { assert.match(script, /VIDEO_DRAIN_FILE=.*\.video-transcode-runtime\/drain/) assert.match(script, /TASK_DETAIL_DRAIN_FILE=.*\.task-detail-upload-runtime\/drain/) assert.match(script, /: > "\$VIDEO_DRAIN_FILE"/) assert.match(script, /: > "\$TASK_DETAIL_DRAIN_FILE"/) assert.doesNotMatch(script, /\/capabilities\//) assert.doesNotMatch(script, /VIDEO_TRANSCODE_DRAIN_PROTOCOL_V1/) assert.doesNotMatch(script, /TASK_DETAIL_UPLOAD_DRAIN_PROTOCOL_V1/) }) test('legacy container and image are preserved, and ordinary deploy remains the only candidate cutover path', () => { const stop = script.lastIndexOf('docker stop --signal SIGKILL --time 0 "$CONTAINER_NAME"') const rename = indexAfter('docker rename "$CONTAINER_NAME" "$PRESERVED_CONTAINER"', stop) const maintenance = indexAfter('write_maintenance_proxy_files', rename) const handoff = indexAfter('Handoff ${HANDOFF_ID} is ready for the normal exact-SHA blue/green deployment', maintenance) assert.ok(handoff > maintenance) assert.doesNotMatch(script, /docker\s+rm\s+/) assert.doesNotMatch(script, /docker create --name "\$CANDIDATE_CONTAINER"/) assert.match(script, /status\":\"maintenance/) assert.match(script, /X-Material-Proxy-Target legacy-bootstrap-maintenance/) assert.match(script, /scripts\/git-incremental-deploy\.ps1 immediately/) }) test('failure trap restores the exact legacy identity and unsafe post-candidate restore is refused', () => { const trap = indexAfter("trap 'status=$?") const firewall = indexAfter('install_firewall_rules', trap) assert.ok(firewall > trap) assert.match(script, /restore_legacy_entry \|\| true/) assert.match(script, /AUTO_RESTORE_ALLOWED.*force-stop or its post-stop evidence was not proven/s) assert.match(script, /current_id.*LEGACY_CONTAINER_ID/) assert.match(script, /Preserved legacy image ID changed/) assert.match(script, /docker inspect "\$name" .*&& fail .*candidate may have accepted traffic/) assert.match(script, /maintenance_entry_is_ready \|\| fail .*every pinned address family/) assert.match(script, /leave_restore_failure_in_maintenance/) }) test('legacy restore never mutates topology without proven blocks and unhealthy exposure rolls back to maintenance', () => { const restoreFunction = script.indexOf('restore_legacy_entry() {') const requireBlocks = indexAfter('if ! install_firewall_rules; then', restoreFunction) const stopMaintenance = indexAfter('systemctl stop "$PROXY_SERVICE"', requireBlocks) const startLegacy = indexAfter('docker start "$CONTAINER_NAME"', stopMaintenance) const unpause = script.indexOf('docker unpause "$CONTAINER_NAME"', stopMaintenance) const proveUnpaused = indexAfter('container_paused "$CONTAINER_NAME"', unpause) const upstreamMismatch = indexAfter('Restored legacy upstream identity changed', startLegacy) const upstreamRollback = indexAfter('rollback_restored_legacy_to_maintenance || true', upstreamMismatch) const selfHealthFailure = indexAfter('Pinned legacy container restarted but did not pass its internal health check', upstreamRollback) const selfHealthRollback = indexAfter('rollback_restored_legacy_to_maintenance || true', selfHealthFailure) const exposeLegacy = indexAfter('delete_firewall_rules || {', startLegacy) const cleanupRollback = indexAfter('rollback_restored_legacy_to_maintenance || true', exposeLegacy) const publicHealth = indexAfter('if ! legacy_entry_is_healthy; then', cleanupRollback) const rollbackCall = indexAfter('rollback_restored_legacy_to_maintenance || true', publicHealth) const restoredMarker = indexAfter('mv -f -- "$restored_tmp" "$HANDOFF_DIR/restored"', rollbackCall) const rollbackFunction = script.indexOf('rollback_restored_legacy_to_maintenance() {') const rollbackBlocks = indexAfter('if ! install_firewall_rules; then', rollbackFunction) const rollbackHttpIdle = indexAfter('wait_for_two_zero_connection_samples', rollbackBlocks) const rollbackDbIdle = indexAfter('assert_database_idle_from_legacy', rollbackHttpIdle) const rollbackFfmpegIdle = indexAfter('assert_no_ffmpeg_running_container', rollbackDbIdle) const rollbackPause = indexAfter('docker pause "$CONTAINER_NAME"', rollbackFfmpegIdle) const rollbackPaused = indexAfter('container_paused "$CONTAINER_NAME"', rollbackPause) const rollbackPostPauseHttp = indexAfter('wait_for_two_zero_connection_samples', rollbackPaused) const rollbackPostPauseDb = indexAfter('assert_database_idle_via_target_image', rollbackPostPauseHttp) const rollbackPostPauseFfmpeg = indexAfter('assert_no_ffmpeg_running_container', rollbackPostPauseDb) const rollbackIdentityBeforeStop = indexAfter("{{.Id}}", rollbackPostPauseFfmpeg) const rollbackStop = indexAfter('docker stop --signal SIGKILL --time 0 "$CONTAINER_NAME"', rollbackIdentityBeforeStop) const rollbackExited = indexAfter("{{.State.Status}}", rollbackStop) const rollbackPostStopDb = indexAfter('assert_database_idle_via_target_image', rollbackExited) const rollbackPostStopFfmpeg = indexAfter('assert_no_host_ffmpeg', rollbackPostStopDb) const rollbackPreserve = indexAfter('docker rename "$CONTAINER_NAME" "$PRESERVED_CONTAINER"', rollbackPostStopFfmpeg) const rollbackMaintenance = indexAfter('start_maintenance_fallback', rollbackPreserve) const publishMaintenance = indexAfter('publish_maintenance_entry', rollbackMaintenance) assert.ok(stopMaintenance > requireBlocks) assert.ok(proveUnpaused > unpause) assert.ok(startLegacy > stopMaintenance) assert.ok(upstreamRollback > upstreamMismatch) assert.ok(selfHealthRollback > selfHealthFailure) assert.ok(cleanupRollback > exposeLegacy) assert.ok(rollbackCall > publicHealth) assert.ok(restoredMarker > rollbackCall) assert.ok(rollbackHttpIdle > rollbackBlocks) assert.ok(rollbackDbIdle > rollbackHttpIdle) assert.ok(rollbackFfmpegIdle > rollbackDbIdle) assert.ok(rollbackPause > rollbackFfmpegIdle) assert.ok(rollbackPaused > rollbackPause) assert.ok(rollbackPostPauseHttp > rollbackPaused) assert.ok(rollbackPostPauseDb > rollbackPostPauseHttp) assert.ok(rollbackPostPauseFfmpeg > rollbackPostPauseDb) assert.ok(rollbackStop > rollbackIdentityBeforeStop) assert.ok(rollbackExited > rollbackStop) assert.ok(rollbackPostStopDb > rollbackExited) assert.ok(rollbackPostStopFfmpeg > rollbackPostStopDb) assert.ok(rollbackPreserve > rollbackPostStopFfmpeg) assert.ok(rollbackMaintenance > rollbackPreserve) assert.ok(publishMaintenance > rollbackMaintenance) const restoreBody = script.slice(restoreFunction, script.indexOf('assert_restore_is_safe() {', restoreFunction)) assert.doesNotMatch(restoreBody, /docker stop --time 30/) assert.doesNotMatch(restoreBody, /docker unpause "\$CONTAINER_NAME"[^\n]*\|\| true/) assert.doesNotMatch(restoreBody, /delete_firewall_rules \|\| true/) const restoreSafetyBody = script.slice( script.indexOf('assert_restore_is_safe() {'), script.indexOf('assert_exact_container_and_image', script.indexOf('assert_restore_is_safe() {')), ) assert.match(restoreSafetyBody, /docker inspect "\$name"/) assert.doesNotMatch(restoreSafetyBody, /container_running "\$name"/) for (const failure of [ 'Pinned legacy container is missing during restoration', 'Refusing to restore a container whose full ID does not match the pin', 'Original legacy container name is occupied; restoration is blocked', 'Pinned legacy container could not be renamed for restoration', 'Pinned legacy container could not be unpaused during restoration', 'Pinned legacy container paused state remained ambiguous after unpause', 'Pinned legacy container could not be restarted', ]) { assert.ok( restoreBody.includes(`leave_restore_failure_in_maintenance "${failure}"`), `restore failure must remain behind the maintenance entry: ${failure}`, ) } })