563 lines
27 KiB
JavaScript
563 lines
27 KiB
JavaScript
import assert from 'node:assert/strict'
|
|
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
|
import os from 'node:os'
|
|
import path from 'node:path'
|
|
import { spawnSync } from 'node:child_process'
|
|
import test from 'node:test'
|
|
|
|
const scriptPath = path.join(process.cwd(), 'scripts', 'legacy-worker-bootstrap-deploy.ps1')
|
|
const script = readFileSync(scriptPath, 'utf8').replace(/\r\n/g, '\n')
|
|
|
|
function indexAfter(needle, after = 0) {
|
|
const index = script.indexOf(needle, after)
|
|
assert.ok(index > after, `missing or out-of-order marker: ${needle}`)
|
|
return index
|
|
}
|
|
|
|
function extractRemoteBash() {
|
|
const marker = "$remoteTemplate = @'"
|
|
const start = script.indexOf(marker)
|
|
assert.ok(start >= 0, 'remote bash here-string start is missing')
|
|
const bodyStart = start + marker.length
|
|
const end = script.indexOf("\n'@", bodyStart)
|
|
assert.ok(end > bodyStart, 'remote bash here-string terminator is missing')
|
|
return script.slice(bodyStart, end).replace(/^\r?\n/, '')
|
|
}
|
|
|
|
function extractBashFunction(name) {
|
|
const payload = extractRemoteBash()
|
|
const match = payload.match(new RegExp(`(?:^|\\n)${name}\\(\\) \\{[\\s\\S]*?\\n\\}`, 'm'))
|
|
assert.ok(match, `remote Bash function is missing: ${name}`)
|
|
return match[0].trimStart()
|
|
}
|
|
|
|
function findBash() {
|
|
const candidates = process.platform === 'win32'
|
|
? [
|
|
'C:\\Program Files\\Git\\bin\\bash.exe',
|
|
'C:\\Program Files\\Git\\usr\\bin\\bash.exe',
|
|
]
|
|
: ['/bin/bash', '/usr/bin/bash']
|
|
return candidates.find(existsSync) || null
|
|
}
|
|
|
|
test('PowerShell 5 parser accepts the one-time bootstrap entry', () => {
|
|
const command = [
|
|
'$tokens=$null',
|
|
'$errors=$null',
|
|
`[System.Management.Automation.Language.Parser]::ParseFile('${scriptPath.replaceAll("'", "''")}',[ref]$tokens,[ref]$errors) | Out-Null`,
|
|
'if ($errors.Count -ne 0) { $errors | ForEach-Object { Write-Error $_.Message }; exit 1 }',
|
|
].join('; ')
|
|
const result = spawnSync('powershell.exe', ['-NoProfile', '-Command', command], { encoding: 'utf8' })
|
|
assert.equal(result.status, 0, result.stderr || result.stdout)
|
|
})
|
|
|
|
test('embedded remote Bash passes syntax validation after token substitution', () => {
|
|
const bash = findBash()
|
|
assert.ok(bash, 'Git Bash or system bash is required for syntax validation')
|
|
let payload = extractRemoteBash()
|
|
const replacements = {
|
|
__MODE__: "'Audit'",
|
|
__TARGET_COMMIT__: `'${'1'.repeat(40)}'`,
|
|
__LEGACY_CONTAINER_ID__: `'${'2'.repeat(64)}'`,
|
|
__LEGACY_IMAGE_ID__: `'sha256:${'3'.repeat(64)}'`,
|
|
__LEGACY_IMAGE_REPO_DIGEST__: `'repo/image@sha256:${'4'.repeat(64)}'`,
|
|
__LEGACY_UPSTREAM__: "'172.18.0.2:3000'",
|
|
__UPLOADS_MOUNT__: "'/mnt/PoolOne/youyidian-uploads'",
|
|
__UPLOADS_MOUNT_TARGET__: "'/mnt/PoolOne/youyidian-uploads'",
|
|
__UPLOADS_MOUNT_SOURCE__: "'PoolOne/youyidian-uploads'",
|
|
__UPLOADS_STATFS_SIGNATURE__: "'1234:abcd'",
|
|
__PUBLISHED_ENDPOINTS__: "'0.0.0.0:4320,[::]:4320'",
|
|
__TARGET_IMAGE_REFERENCE__: `'repo/image@sha256:${'5'.repeat(64)}'`,
|
|
__TARGET_IMAGE_ID__: `'sha256:${'6'.repeat(64)}'`,
|
|
__TARGET_IMAGE_REPO_DIGEST__: `'repo/image@sha256:${'5'.repeat(64)}'`,
|
|
__CONTAINER_NAME__: "'youyidian-3d-material'",
|
|
__NETWORK_NAME__: "'youyidian-3d_app-network'",
|
|
__ENVIRONMENT_FILE__: "'/mnt/PoolOne/apps/youyidian-3d/.env'",
|
|
__PORT__: "'4320'",
|
|
}
|
|
for (const [token, value] of Object.entries(replacements)) {
|
|
payload = payload.replaceAll(token, value)
|
|
}
|
|
const result = spawnSync(bash, ['-n'], { input: payload, encoding: 'utf8' })
|
|
assert.equal(result.status, 0, result.stderr || result.stdout)
|
|
})
|
|
|
|
test('connection-drain simulation requires two consecutive zero samples from both host and container netns', () => {
|
|
const bash = findBash()
|
|
assert.ok(bash, 'Git Bash or system bash is required for drain simulation')
|
|
const fixture = mkdtempSync(path.join(os.tmpdir(), 'gx-bootstrap-drain-'))
|
|
const sequencePath = path.join(fixture, 'sequence.txt')
|
|
const helper = extractBashFunction('wait_for_two_zero_connection_samples')
|
|
const harness = `
|
|
set -u
|
|
${helper}
|
|
sleep() { :; }
|
|
log() { :; }
|
|
ok() { :; }
|
|
legacy_connection_counts() {
|
|
local value=""
|
|
IFS= read -r value < "$SEQUENCE_FILE"
|
|
tail -n +2 "$SEQUENCE_FILE" > "${sequencePath.replaceAll('\\', '/')}.next"
|
|
mv "${sequencePath.replaceAll('\\', '/')}.next" "$SEQUENCE_FILE"
|
|
printf '%s\\n' "$value"
|
|
}
|
|
wait_for_two_zero_connection_samples
|
|
`
|
|
try {
|
|
writeFileSync(sequencePath, '0 3\n0 0\n0 2\n0 0\n0 0\n', 'utf8')
|
|
const succeeds = spawnSync(bash, ['-c', harness], {
|
|
encoding: 'utf8',
|
|
env: {
|
|
...process.env,
|
|
MSYS2_ARG_CONV_EXCL: '*',
|
|
MSYS_NO_PATHCONV: '1',
|
|
LEGACY_UPSTREAM: '172.18.0.2:3000',
|
|
SEQUENCE_FILE: sequencePath.replaceAll('\\', '/'),
|
|
},
|
|
})
|
|
assert.equal(succeeds.status, 0, succeeds.stderr || succeeds.stdout)
|
|
assert.equal(readFileSync(sequencePath, 'utf8').trim(), '', 'container-netns activity must reset the consecutive-zero counter')
|
|
|
|
writeFileSync(sequencePath, Array.from({ length: 60 }, (_, index) => index % 2 ? '0 1' : '0 0').join('\n') + '\n', 'utf8')
|
|
const alternating = spawnSync(bash, ['-c', harness], {
|
|
encoding: 'utf8',
|
|
env: {
|
|
...process.env,
|
|
MSYS2_ARG_CONV_EXCL: '*',
|
|
MSYS_NO_PATHCONV: '1',
|
|
LEGACY_UPSTREAM: '172.18.0.2:3000',
|
|
SEQUENCE_FILE: sequencePath.replaceAll('\\', '/'),
|
|
},
|
|
})
|
|
assert.notEqual(alternating.status, 0, 'container-netns activity between zero samples must prevent drain')
|
|
|
|
writeFileSync(sequencePath, '0\n', 'utf8')
|
|
const missingNetnsEvidence = spawnSync(bash, ['-c', harness], {
|
|
encoding: 'utf8',
|
|
env: {
|
|
...process.env,
|
|
MSYS2_ARG_CONV_EXCL: '*',
|
|
MSYS_NO_PATHCONV: '1',
|
|
LEGACY_UPSTREAM: '172.18.0.2:3000',
|
|
SEQUENCE_FILE: sequencePath.replaceAll('\\', '/'),
|
|
},
|
|
})
|
|
assert.notEqual(missingNetnsEvidence.status, 0, 'missing container-netns evidence must fail closed')
|
|
} finally {
|
|
rmSync(fixture, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
test('firewall cleanup simulation fails closed on delete failure or residual rule', () => {
|
|
const bash = findBash()
|
|
assert.ok(bash, 'Git Bash or system bash is required for firewall simulation')
|
|
const helper = extractBashFunction('delete_firewall_rules')
|
|
const harness = `
|
|
set -u
|
|
${helper}
|
|
warn() { printf '%s\\n' "$*" >&2; }
|
|
state_v4_output=1
|
|
state_v4_docker=1
|
|
state_v4_input=1
|
|
state_v6_output=1
|
|
state_v6_input=1
|
|
state_v6_docker=1
|
|
firewall_rule() {
|
|
local action="$1" chain="$2" key=""
|
|
[ "$chain" = OUTPUT ] && key=v4_output || key=v4_input
|
|
if [ "$action" = -C ]; then
|
|
[ "$(eval "printf '%s' \\"\\$state_\${key}\\"")" = 1 ]
|
|
return
|
|
fi
|
|
if [ "$action" = -D ]; then
|
|
[ "\${FAIL_TARGET:-}" = "$key" ] && return 2
|
|
eval "state_\${key}=0"
|
|
return 0
|
|
fi
|
|
return 2
|
|
}
|
|
firewall_original_port_rule() {
|
|
if [ "$1" = -C ]; then [ "$state_v4_docker" = 1 ]; return; fi
|
|
if [ "$1" = -D ]; then
|
|
[ "\${FAIL_TARGET:-}" = v4_docker ] && return 2
|
|
state_v4_docker=0
|
|
return 0
|
|
fi
|
|
return 2
|
|
}
|
|
firewall_ipv6_rule() {
|
|
local action="$1" chain="$2" key=""
|
|
[ "$chain" = OUTPUT ] && key=v6_output || key=v6_input
|
|
if [ "$action" = -C ]; then
|
|
[ "$(eval "printf '%s' \\"\\$state_\${key}\\"")" = 1 ]
|
|
return
|
|
fi
|
|
if [ "$action" = -D ]; then
|
|
[ "\${FAIL_TARGET:-}" = "$key" ] && return 2
|
|
eval "state_\${key}=0"
|
|
return 0
|
|
fi
|
|
return 2
|
|
}
|
|
firewall_ipv6_original_port_rule() {
|
|
if [ "$1" = -C ]; then [ "$state_v6_docker" = 1 ]; return; fi
|
|
if [ "$1" = -D ]; then
|
|
[ "\${FAIL_TARGET:-}" = v6_docker ] && return 2
|
|
state_v6_docker=0
|
|
return 0
|
|
fi
|
|
return 2
|
|
}
|
|
delete_firewall_rules
|
|
result=$?
|
|
printf 'result=%s states=%s%s%s%s%s%s\\n' "$result" "$state_v4_output" "$state_v4_docker" "$state_v4_input" "$state_v6_output" "$state_v6_input" "$state_v6_docker"
|
|
exit "$result"
|
|
`
|
|
const run = (failTarget = '') => spawnSync(bash, ['-c', harness], {
|
|
encoding: 'utf8',
|
|
env: {
|
|
...process.env,
|
|
FAIL_TARGET: failTarget,
|
|
LEGACY_UPSTREAM: '172.18.0.2:3000',
|
|
FIREWALL_COMMENT: 'test-bootstrap',
|
|
PORT: '4320',
|
|
},
|
|
})
|
|
|
|
const succeeds = run()
|
|
assert.equal(succeeds.status, 0, succeeds.stderr || succeeds.stdout)
|
|
assert.match(succeeds.stdout, /result=0 states=000000/)
|
|
|
|
const deleteFails = run('v4_docker')
|
|
assert.notEqual(deleteFails.status, 0, 'a failed -D operation must fail cleanup')
|
|
assert.match(deleteFails.stderr, /Failed to remove bootstrap IPv4 DOCKER-USER rule/)
|
|
assert.match(deleteFails.stderr, /remains after cleanup/)
|
|
|
|
const ipv6DeleteFails = run('v6_docker')
|
|
assert.notEqual(ipv6DeleteFails.status, 0, 'a failed IPv6 DOCKER-USER -D operation must fail cleanup')
|
|
assert.match(ipv6DeleteFails.stderr, /Failed to remove bootstrap IPv6 DOCKER-USER rule/)
|
|
assert.match(ipv6DeleteFails.stderr, /remains after cleanup/)
|
|
})
|
|
|
|
test('handoff ready and restored markers are unreachable until firewall absence is proven', () => {
|
|
const handoffPublish = script.lastIndexOf('publish_maintenance_entry || fail')
|
|
const handoffReadyFile = indexAfter('"ready" > "$HANDOFF_DIR/handoff.ready"', handoffPublish)
|
|
const restoreFunction = script.indexOf('restore_legacy_entry() {')
|
|
const restoreCleanup = script.indexOf('delete_firewall_rules || {', restoreFunction)
|
|
const restoreHealth = script.indexOf('if ! legacy_entry_is_healthy; then', restoreCleanup)
|
|
const restoredMarker = script.indexOf('mv -f -- "$restored_tmp" "$HANDOFF_DIR/restored"', restoreHealth)
|
|
const restoreModeCall = script.indexOf('restore_legacy_entry || fail', restoredMarker)
|
|
|
|
assert.ok(handoffPublish > 0)
|
|
assert.ok(handoffReadyFile > handoffPublish)
|
|
assert.ok(restoreCleanup > restoreFunction)
|
|
assert.ok(restoreHealth > restoreCleanup)
|
|
assert.ok(restoredMarker > restoreHealth)
|
|
assert.ok(restoreModeCall > restoredMarker)
|
|
})
|
|
|
|
test('maintenance publication proves a private Unix socket before unblocking the public TCP entry', () => {
|
|
const publishFunction = script.indexOf('publish_maintenance_entry() {')
|
|
const socketProbe = indexAfter('maintenance_socket_is_ready', publishFunction)
|
|
const cleanup = indexAfter('delete_firewall_rules', socketProbe)
|
|
const tcpProbe = indexAfter('maintenance_entry_is_ready', cleanup)
|
|
const reblock = indexAfter('install_firewall_rules', tcpProbe)
|
|
|
|
assert.ok(socketProbe > publishFunction)
|
|
assert.ok(cleanup > socketProbe)
|
|
assert.ok(tcpProbe > cleanup)
|
|
assert.ok(reblock > tcpProbe)
|
|
assert.match(script, /MAINTENANCE_SOCKET="\$\{PROXY_ROOT\}\/maintenance\.sock"/)
|
|
assert.match(script, /listen unix:\$\{MAINTENANCE_SOCKET\};/)
|
|
assert.match(script, /listen 0\.0\.0\.0:\$\{PORT\};/)
|
|
assert.match(script, /listen \[::\]:\$\{PORT\} ipv6only=on;/)
|
|
assert.match(script, /curl .*--unix-socket "\$MAINTENANCE_SOCKET".*http:\/\/localhost\/api\/health/)
|
|
assert.match(script, /maintenance_tcp_endpoint_is_ready "http:\/\/127\.0\.0\.1:\$\{PORT\}\/api\/health"/)
|
|
assert.match(script, /maintenance_tcp_endpoint_is_ready "http:\/\/\[::1\]:\$\{PORT\}\/api\/health"/)
|
|
assert.match(script, /curl --noproxy '\*' -g/)
|
|
})
|
|
|
|
test('maintenance publication simulation never removes rules before the private probe', () => {
|
|
const bash = findBash()
|
|
assert.ok(bash, 'Git Bash or system bash is required for maintenance publication simulation')
|
|
const helper = extractBashFunction('publish_maintenance_entry')
|
|
const harness = `
|
|
set -u
|
|
${helper}
|
|
warn() { printf 'warn:%s\\n' "$*"; }
|
|
maintenance_socket_is_ready() { printf 'socket\\n'; return "\${SOCKET_RESULT}"; }
|
|
delete_firewall_rules() { printf 'cleanup\\n'; return "\${CLEANUP_RESULT}"; }
|
|
maintenance_entry_is_ready() { printf 'tcp\\n'; return "\${TCP_RESULT}"; }
|
|
install_firewall_rules() { printf 'reblock\\n'; return 0; }
|
|
publish_maintenance_entry
|
|
`
|
|
const run = (socketResult, cleanupResult, tcpResult) => spawnSync(bash, ['-c', harness], {
|
|
encoding: 'utf8',
|
|
env: {
|
|
...process.env,
|
|
MSYS2_ARG_CONV_EXCL: '*',
|
|
MSYS_NO_PATHCONV: '1',
|
|
SOCKET_RESULT: String(socketResult),
|
|
CLEANUP_RESULT: String(cleanupResult),
|
|
TCP_RESULT: String(tcpResult),
|
|
},
|
|
})
|
|
|
|
const succeeds = run(0, 0, 0)
|
|
assert.equal(succeeds.status, 0, succeeds.stderr || succeeds.stdout)
|
|
assert.equal(succeeds.stdout.trim(), 'socket\ncleanup\ntcp')
|
|
|
|
const socketFails = run(1, 0, 0)
|
|
assert.notEqual(socketFails.status, 0)
|
|
assert.equal(socketFails.stdout.trim(), 'socket\nwarn:Maintenance Unix-socket probe failed while traffic was still blocked')
|
|
assert.doesNotMatch(socketFails.stdout, /cleanup|tcp|reblock/)
|
|
|
|
const tcpFails = run(0, 0, 1)
|
|
assert.notEqual(tcpFails.status, 0)
|
|
assert.match(tcpFails.stdout.trim(), /^socket\ncleanup\ntcp\nwarn:.*\nreblock$/)
|
|
})
|
|
|
|
test('write modes are disabled unless both explicit authorization switches are present', () => {
|
|
assert.match(script, /\[ValidateSet\('Audit', 'EnterMaintenance', 'RestoreLegacy'\)\]/)
|
|
assert.match(script, /\[switch\]\$Apply/)
|
|
assert.match(script, /\[switch\]\$AllowLegacyDrainBootstrap/)
|
|
assert.match(script, /-not \(\$Apply -and \$AllowLegacyDrainBootstrap\)/)
|
|
assert.match(script, /Mode=Audit does not accept write authorization switches/)
|
|
})
|
|
|
|
test('write authorization guard rejects an actual PowerShell invocation before SSH', () => {
|
|
const args = [
|
|
'-NoProfile',
|
|
'-File', scriptPath,
|
|
'-Mode', 'EnterMaintenance',
|
|
'-TargetCommit', '1'.repeat(40),
|
|
'-LegacyContainerId', '2'.repeat(64),
|
|
'-LegacyImageId', `sha256:${'3'.repeat(64)}`,
|
|
'-LegacyImageRepoDigest', `repo/legacy@sha256:${'4'.repeat(64)}`,
|
|
'-LegacyUpstream', '172.18.0.2:3000',
|
|
'-UploadsStatfsSignature', '1234:abcd',
|
|
'-TargetImageReference', `repo/target@sha256:${'5'.repeat(64)}`,
|
|
'-TargetImageId', `sha256:${'6'.repeat(64)}`,
|
|
'-TargetImageRepoDigest', `repo/target@sha256:${'5'.repeat(64)}`,
|
|
'-ProdServerIP', 'prod.example.internal',
|
|
]
|
|
const result = spawnSync('powershell.exe', args, { encoding: 'utf8' })
|
|
assert.notEqual(result.status, 0)
|
|
assert.match(`${result.stdout}\n${result.stderr}`, /disabled by default/)
|
|
assert.doesNotMatch(`${result.stdout}\n${result.stderr}`, /ssh/i)
|
|
})
|
|
|
|
test('all mutable identities are explicitly pinned before the maintenance window', () => {
|
|
for (const expected of [
|
|
'^[0-9a-f]{40}$',
|
|
'^[0-9a-f]{64}$',
|
|
'^sha256:[0-9a-f]{64}$',
|
|
'LegacyImageRepoDigest',
|
|
'LegacyUpstream',
|
|
'UploadsStatfsSignature',
|
|
'TargetImageReference',
|
|
'TargetImageId',
|
|
'TargetImageRepoDigest',
|
|
'PUBLISHED_ENDPOINTS',
|
|
]) {
|
|
assert.ok(script.includes(expected), `missing identity pin: ${expected}`)
|
|
}
|
|
assert.match(script, /actual_id=.*docker inspect -f '\{\{\.Id\}\}'/)
|
|
assert.match(script, /actual_image_id=.*docker inspect -f '\{\{\.Image\}\}'/)
|
|
assert.match(script, /actual_upstream=.*NetworkSettings\.Networks/)
|
|
assert.match(script, /findmnt -rn -o TARGET,SOURCE,FSTYPE/)
|
|
assert.match(script, /stat -f -c '%i:%t'/)
|
|
assert.match(script, /org\.opencontainers\.image\.revision/)
|
|
assert.match(script, /cat \/app\/DEPLOYMENT_COMMIT/)
|
|
})
|
|
|
|
test('critical mutation is serialized by both deployment and proxy flocks', () => {
|
|
const deployLock = indexAfter('exec 9>"$DEPLOY_LOCK"')
|
|
const deployFlock = indexAfter('flock -w 30 9', deployLock)
|
|
const proxyLock = indexAfter('exec 8>"$PROXY_RECONCILE_LOCK"', deployFlock)
|
|
const proxyFlock = indexAfter('flock -w 30 8', proxyLock)
|
|
const repeatedPin = indexAfter('assert_exact_container_and_image', proxyFlock)
|
|
const firstDrainWrite = indexAfter(': > "$VIDEO_DRAIN_FILE"', repeatedPin)
|
|
|
|
assert.ok(firstDrainWrite > proxyFlock)
|
|
assert.match(script, /DEPLOY_LOCK="\/run\/\$\{CONTAINER_NAME\}-production-deploy\.lock"/)
|
|
assert.match(script, /PROXY_RECONCILE_LOCK="\/run\/\$\{CONTAINER_NAME\}-local-proxy-reconcile\.lock"/)
|
|
})
|
|
|
|
test('new traffic is rejected while two zero HTTP samples and worker-idle evidence are collected', () => {
|
|
const installFirewall = script.lastIndexOf('install_firewall_rules\n')
|
|
assert.ok(installFirewall > 0)
|
|
const twoZeros = indexAfter('wait_for_two_zero_connection_samples || fail', installFirewall)
|
|
const prePauseDb = indexAfter('assert_database_idle_from_legacy', twoZeros)
|
|
const prePauseFfmpeg = indexAfter('assert_no_ffmpeg_running_container', prePauseDb)
|
|
const pause = indexAfter('docker pause "$CONTAINER_NAME"', prePauseFfmpeg)
|
|
const postPauseZeros = indexAfter('wait_for_two_zero_connection_samples || fail', pause)
|
|
const postPauseDb = indexAfter('assert_database_idle_via_target_image', postPauseZeros)
|
|
const postPauseFfmpeg = indexAfter('assert_no_ffmpeg_running_container', postPauseDb)
|
|
const disableAutomaticRestore = indexAfter('AUTO_RESTORE_ALLOWED=0', postPauseFfmpeg)
|
|
const stop = indexAfter('docker stop --signal SIGKILL --time 0 "$CONTAINER_NAME"', disableAutomaticRestore)
|
|
const stoppedState = indexAfter("{{.State.Status}}", stop)
|
|
const postStopDb = indexAfter('assert_database_idle_via_target_image', stoppedState)
|
|
const postStopFfmpeg = indexAfter('assert_no_host_ffmpeg', postStopDb)
|
|
const enableAutomaticRestore = indexAfter('AUTO_RESTORE_ALLOWED=1', postStopFfmpeg)
|
|
|
|
assert.ok(enableAutomaticRestore > stop)
|
|
assert.match(script, /--ctorigdstport "\$PORT"/)
|
|
assert.match(script, /ip6tables -w 5 "\$action" DOCKER-USER.*--ctorigdstport "\$PORT"/)
|
|
assert.match(script, /--ctstate NEW/)
|
|
assert.match(script, /ip6tables -w 5/)
|
|
assert.match(script, /zero_samples" -ge 2/)
|
|
assert.match(script, /nsenter -t "\$container_pid" -n -- ss -Htan state established '\( sport = :3000 \)'/)
|
|
assert.match(script, /host_count" = "0".*container_count" = "0"/s)
|
|
assert.match(script, /stat -Lc '%d:%i'.*\/proc\/\$\{container_pid\}\/ns\/net/s)
|
|
assert.match(script, /current_pid" = "\$container_pid"/)
|
|
assert.match(script, /videoQueuedOrProcessingCount/)
|
|
assert.match(script, /taskJobQueuedOrProcessingCount/)
|
|
assert.match(script, /taskStatusUploadingCount/)
|
|
assert.match(script, /taskStatusUploadingRows/)
|
|
})
|
|
|
|
test('only terminally restored handoffs are archived before an identical retry', () => {
|
|
const archiveFunction = script.indexOf('archive_restored_handoff_if_present() {')
|
|
const terminalMarker = indexAfter('[ -f "$HANDOFF_DIR/restored" ]', archiveFunction)
|
|
const identityCommit = indexAfter('Restored handoff target commit does not match', terminalMarker)
|
|
const identityContainer = indexAfter('Restored handoff legacy container ID does not match', identityCommit)
|
|
const archiveMove = indexAfter('mv -- "$HANDOFF_DIR" "$archive_dir"', identityContainer)
|
|
const makeReadonly = indexAfter('chmod -R a-w "$archive_dir"', archiveMove)
|
|
const backupFunction = script.indexOf('backup_proxy_files() {')
|
|
const archiveCall = indexAfter('archive_restored_handoff_if_present', backupFunction)
|
|
const createNew = indexAfter('mkdir -p "$HANDOFF_DIR"', archiveCall)
|
|
const restoredWrite = script.indexOf('mv -f -- "$restored_tmp" "$HANDOFF_DIR/restored"')
|
|
const restoreSuccess = indexAfter('ok "Pinned legacy direct container and entry were restored"', restoredWrite)
|
|
|
|
assert.ok(archiveFunction > 0)
|
|
assert.ok(makeReadonly > archiveMove)
|
|
assert.ok(createNew > archiveCall)
|
|
assert.ok(restoreSuccess > restoredWrite)
|
|
assert.match(script, /Existing handoff is not terminally restored and cannot be reused/)
|
|
assert.doesNotMatch(script, /rm\s+-rf\s+.*HANDOFF_DIR/)
|
|
})
|
|
|
|
test('candidate is pre-drained without writing any forged capability', () => {
|
|
assert.match(script, /VIDEO_DRAIN_FILE=.*\.video-transcode-runtime\/drain/)
|
|
assert.match(script, /TASK_DETAIL_DRAIN_FILE=.*\.task-detail-upload-runtime\/drain/)
|
|
assert.match(script, /: > "\$VIDEO_DRAIN_FILE"/)
|
|
assert.match(script, /: > "\$TASK_DETAIL_DRAIN_FILE"/)
|
|
assert.doesNotMatch(script, /\/capabilities\//)
|
|
assert.doesNotMatch(script, /VIDEO_TRANSCODE_DRAIN_PROTOCOL_V1/)
|
|
assert.doesNotMatch(script, /TASK_DETAIL_UPLOAD_DRAIN_PROTOCOL_V1/)
|
|
})
|
|
|
|
test('legacy container and image are preserved, and ordinary deploy remains the only candidate cutover path', () => {
|
|
const stop = script.lastIndexOf('docker stop --signal SIGKILL --time 0 "$CONTAINER_NAME"')
|
|
const rename = indexAfter('docker rename "$CONTAINER_NAME" "$PRESERVED_CONTAINER"', stop)
|
|
const maintenance = indexAfter('write_maintenance_proxy_files', rename)
|
|
const handoff = indexAfter('Handoff ${HANDOFF_ID} is ready for the normal exact-SHA blue/green deployment', maintenance)
|
|
|
|
assert.ok(handoff > maintenance)
|
|
assert.doesNotMatch(script, /docker\s+rm\s+/)
|
|
assert.doesNotMatch(script, /docker create --name "\$CANDIDATE_CONTAINER"/)
|
|
assert.match(script, /status\":\"maintenance/)
|
|
assert.match(script, /X-Material-Proxy-Target legacy-bootstrap-maintenance/)
|
|
assert.match(script, /scripts\/git-incremental-deploy\.ps1 immediately/)
|
|
})
|
|
|
|
test('failure trap restores the exact legacy identity and unsafe post-candidate restore is refused', () => {
|
|
const trap = indexAfter("trap 'status=$?")
|
|
const firewall = indexAfter('install_firewall_rules', trap)
|
|
assert.ok(firewall > trap)
|
|
assert.match(script, /restore_legacy_entry \|\| true/)
|
|
assert.match(script, /AUTO_RESTORE_ALLOWED.*force-stop or its post-stop evidence was not proven/s)
|
|
assert.match(script, /current_id.*LEGACY_CONTAINER_ID/)
|
|
assert.match(script, /Preserved legacy image ID changed/)
|
|
assert.match(script, /docker inspect "\$name" .*&& fail .*candidate may have accepted traffic/)
|
|
assert.match(script, /maintenance_entry_is_ready \|\| fail .*every pinned address family/)
|
|
assert.match(script, /leave_restore_failure_in_maintenance/)
|
|
})
|
|
|
|
test('legacy restore never mutates topology without proven blocks and unhealthy exposure rolls back to maintenance', () => {
|
|
const restoreFunction = script.indexOf('restore_legacy_entry() {')
|
|
const requireBlocks = indexAfter('if ! install_firewall_rules; then', restoreFunction)
|
|
const stopMaintenance = indexAfter('systemctl stop "$PROXY_SERVICE"', requireBlocks)
|
|
const startLegacy = indexAfter('docker start "$CONTAINER_NAME"', stopMaintenance)
|
|
const unpause = script.indexOf('docker unpause "$CONTAINER_NAME"', stopMaintenance)
|
|
const proveUnpaused = indexAfter('container_paused "$CONTAINER_NAME"', unpause)
|
|
const upstreamMismatch = indexAfter('Restored legacy upstream identity changed', startLegacy)
|
|
const upstreamRollback = indexAfter('rollback_restored_legacy_to_maintenance || true', upstreamMismatch)
|
|
const selfHealthFailure = indexAfter('Pinned legacy container restarted but did not pass its internal health check', upstreamRollback)
|
|
const selfHealthRollback = indexAfter('rollback_restored_legacy_to_maintenance || true', selfHealthFailure)
|
|
const exposeLegacy = indexAfter('delete_firewall_rules || {', startLegacy)
|
|
const cleanupRollback = indexAfter('rollback_restored_legacy_to_maintenance || true', exposeLegacy)
|
|
const publicHealth = indexAfter('if ! legacy_entry_is_healthy; then', cleanupRollback)
|
|
const rollbackCall = indexAfter('rollback_restored_legacy_to_maintenance || true', publicHealth)
|
|
const restoredMarker = indexAfter('mv -f -- "$restored_tmp" "$HANDOFF_DIR/restored"', rollbackCall)
|
|
|
|
const rollbackFunction = script.indexOf('rollback_restored_legacy_to_maintenance() {')
|
|
const rollbackBlocks = indexAfter('if ! install_firewall_rules; then', rollbackFunction)
|
|
const rollbackHttpIdle = indexAfter('wait_for_two_zero_connection_samples', rollbackBlocks)
|
|
const rollbackDbIdle = indexAfter('assert_database_idle_from_legacy', rollbackHttpIdle)
|
|
const rollbackFfmpegIdle = indexAfter('assert_no_ffmpeg_running_container', rollbackDbIdle)
|
|
const rollbackPause = indexAfter('docker pause "$CONTAINER_NAME"', rollbackFfmpegIdle)
|
|
const rollbackPaused = indexAfter('container_paused "$CONTAINER_NAME"', rollbackPause)
|
|
const rollbackPostPauseHttp = indexAfter('wait_for_two_zero_connection_samples', rollbackPaused)
|
|
const rollbackPostPauseDb = indexAfter('assert_database_idle_via_target_image', rollbackPostPauseHttp)
|
|
const rollbackPostPauseFfmpeg = indexAfter('assert_no_ffmpeg_running_container', rollbackPostPauseDb)
|
|
const rollbackIdentityBeforeStop = indexAfter("{{.Id}}", rollbackPostPauseFfmpeg)
|
|
const rollbackStop = indexAfter('docker stop --signal SIGKILL --time 0 "$CONTAINER_NAME"', rollbackIdentityBeforeStop)
|
|
const rollbackExited = indexAfter("{{.State.Status}}", rollbackStop)
|
|
const rollbackPostStopDb = indexAfter('assert_database_idle_via_target_image', rollbackExited)
|
|
const rollbackPostStopFfmpeg = indexAfter('assert_no_host_ffmpeg', rollbackPostStopDb)
|
|
const rollbackPreserve = indexAfter('docker rename "$CONTAINER_NAME" "$PRESERVED_CONTAINER"', rollbackPostStopFfmpeg)
|
|
const rollbackMaintenance = indexAfter('start_maintenance_fallback', rollbackPreserve)
|
|
const publishMaintenance = indexAfter('publish_maintenance_entry', rollbackMaintenance)
|
|
|
|
assert.ok(stopMaintenance > requireBlocks)
|
|
assert.ok(proveUnpaused > unpause)
|
|
assert.ok(startLegacy > stopMaintenance)
|
|
assert.ok(upstreamRollback > upstreamMismatch)
|
|
assert.ok(selfHealthRollback > selfHealthFailure)
|
|
assert.ok(cleanupRollback > exposeLegacy)
|
|
assert.ok(rollbackCall > publicHealth)
|
|
assert.ok(restoredMarker > rollbackCall)
|
|
assert.ok(rollbackHttpIdle > rollbackBlocks)
|
|
assert.ok(rollbackDbIdle > rollbackHttpIdle)
|
|
assert.ok(rollbackFfmpegIdle > rollbackDbIdle)
|
|
assert.ok(rollbackPause > rollbackFfmpegIdle)
|
|
assert.ok(rollbackPaused > rollbackPause)
|
|
assert.ok(rollbackPostPauseHttp > rollbackPaused)
|
|
assert.ok(rollbackPostPauseDb > rollbackPostPauseHttp)
|
|
assert.ok(rollbackPostPauseFfmpeg > rollbackPostPauseDb)
|
|
assert.ok(rollbackStop > rollbackIdentityBeforeStop)
|
|
assert.ok(rollbackExited > rollbackStop)
|
|
assert.ok(rollbackPostStopDb > rollbackExited)
|
|
assert.ok(rollbackPostStopFfmpeg > rollbackPostStopDb)
|
|
assert.ok(rollbackPreserve > rollbackPostStopFfmpeg)
|
|
assert.ok(rollbackMaintenance > rollbackPreserve)
|
|
assert.ok(publishMaintenance > rollbackMaintenance)
|
|
const restoreBody = script.slice(restoreFunction, script.indexOf('assert_restore_is_safe() {', restoreFunction))
|
|
assert.doesNotMatch(restoreBody, /docker stop --time 30/)
|
|
assert.doesNotMatch(restoreBody, /docker unpause "\$CONTAINER_NAME"[^\n]*\|\| true/)
|
|
assert.doesNotMatch(restoreBody, /delete_firewall_rules \|\| true/)
|
|
const restoreSafetyBody = script.slice(
|
|
script.indexOf('assert_restore_is_safe() {'),
|
|
script.indexOf('assert_exact_container_and_image', script.indexOf('assert_restore_is_safe() {')),
|
|
)
|
|
assert.match(restoreSafetyBody, /docker inspect "\$name"/)
|
|
assert.doesNotMatch(restoreSafetyBody, /container_running "\$name"/)
|
|
for (const failure of [
|
|
'Pinned legacy container is missing during restoration',
|
|
'Refusing to restore a container whose full ID does not match the pin',
|
|
'Original legacy container name is occupied; restoration is blocked',
|
|
'Pinned legacy container could not be renamed for restoration',
|
|
'Pinned legacy container could not be unpaused during restoration',
|
|
'Pinned legacy container paused state remained ambiguous after unpause',
|
|
'Pinned legacy container could not be restarted',
|
|
]) {
|
|
assert.ok(
|
|
restoreBody.includes(`leave_restore_failure_in_maintenance "${failure}"`),
|
|
`restore failure must remain behind the maintenance entry: ${failure}`,
|
|
)
|
|
}
|
|
})
|