7326e273e1
Protect upload cleanup and atomic completion paths, add durable media worker leases and brand asset validation, and introduce verified migration/bootstrap/deployment gates.
328 lines
12 KiB
JavaScript
328 lines
12 KiB
JavaScript
import assert from 'node:assert/strict'
|
|
import { createHash } from 'node:crypto'
|
|
import {
|
|
cpSync,
|
|
mkdirSync,
|
|
mkdtempSync,
|
|
readFileSync,
|
|
readdirSync,
|
|
rmSync,
|
|
writeFileSync,
|
|
} from 'node:fs'
|
|
import os from 'node:os'
|
|
import path from 'node:path'
|
|
import { spawnSync } from 'node:child_process'
|
|
import test from 'node:test'
|
|
|
|
import {
|
|
assertProductionAndIsolationAreDistinct,
|
|
} from './create-verified-production-backup.mjs'
|
|
import {
|
|
assertBackupReceiptDocument,
|
|
databaseFingerprintHash,
|
|
} from './safe-production-migrate.mjs'
|
|
|
|
const COMMIT = 'a'.repeat(40)
|
|
const PASSWORDS = ['production-password-must-not-leak', 'isolated-password-must-not-leak']
|
|
|
|
const FAKE_DRIVER = String.raw`
|
|
import { existsSync, writeFileSync } from 'node:fs'
|
|
|
|
const [tool, ...args] = process.argv.slice(2)
|
|
const failTool = process.env.FAKE_FAIL_TOOL || ''
|
|
const secret = process.env.PGPASSWORD || ''
|
|
if (process.env.FAKE_EMIT_SECRET === '1' && secret) process.stderr.write(secret + '\n')
|
|
|
|
if (args.includes('--version')) {
|
|
process.stdout.write(tool + ' (PostgreSQL) 15.14\n')
|
|
process.exit(0)
|
|
}
|
|
if (failTool === tool) process.exit(42)
|
|
|
|
if (tool === 'pg_dump') {
|
|
const outputIndex = args.indexOf('--file')
|
|
if (outputIndex < 0 || !args[outputIndex + 1]) process.exit(43)
|
|
writeFileSync(args[outputIndex + 1], Buffer.from('PGDMP-fake-custom-backup-fixture'))
|
|
process.exit(0)
|
|
}
|
|
|
|
if (tool === 'pg_restore') {
|
|
const artifact = args.at(-1)
|
|
if (!artifact || !existsSync(artifact)) process.exit(44)
|
|
process.exit(0)
|
|
}
|
|
|
|
if (tool !== 'psql') process.exit(45)
|
|
const commandIndex = args.indexOf('--command')
|
|
const sql = commandIndex >= 0 ? args[commandIndex + 1] || '' : ''
|
|
const isolated = String(process.env.PGDATABASE || '').startsWith('gongxiangp2_restore_')
|
|
if (sql.includes('verified-backup-identity-v1')) {
|
|
process.stdout.write(JSON.stringify({
|
|
database: process.env.PGDATABASE,
|
|
user: process.env.PGUSER,
|
|
serverAddress: isolated ? process.env.FAKE_ISOLATED_SERVER_ADDRESS : process.env.FAKE_PRODUCTION_SERVER_ADDRESS,
|
|
serverPort: process.env.PGPORT,
|
|
}) + '\n')
|
|
process.exit(0)
|
|
}
|
|
if (sql.includes('verified-backup-summary-v1')) {
|
|
const mismatch = isolated && process.env.FAKE_SUMMARY_MODE === 'mismatch'
|
|
process.stdout.write(JSON.stringify({
|
|
schemaVersion: 1,
|
|
sampleLimit: 32,
|
|
materials: { count: mismatch ? '1000' : '999', sampleSha256InputMd5: 'a'.repeat(32) },
|
|
users: { count: '12', sampleSha256InputMd5: 'b'.repeat(32) },
|
|
systemSettings: { count: '1', sampleSha256InputMd5: 'c'.repeat(32) },
|
|
prismaMigrations: { count: '24', sampleSha256InputMd5: 'd'.repeat(32) },
|
|
productPackageUploadJobs: { count: '7', sampleSha256InputMd5: 'e'.repeat(32) },
|
|
}) + '\n')
|
|
process.exit(0)
|
|
}
|
|
if (sql.includes('verified-backup-reset-isolated-v1')) process.exit(0)
|
|
process.exit(46)
|
|
`
|
|
|
|
function createFixture() {
|
|
const root = mkdtempSync(path.join(os.tmpdir(), 'gx-verified-backup-'))
|
|
const repo = path.join(root, 'repo')
|
|
const scripts = path.join(repo, 'scripts')
|
|
const output = path.join(root, 'output')
|
|
mkdirSync(scripts, { recursive: true })
|
|
mkdirSync(output)
|
|
cpSync(new URL('./create-verified-production-backup.mjs', import.meta.url), path.join(scripts, 'create-verified-production-backup.mjs'))
|
|
cpSync(new URL('./safe-production-migrate.mjs', import.meta.url), path.join(scripts, 'safe-production-migrate.mjs'))
|
|
writeFileSync(path.join(repo, 'DEPLOYMENT_COMMIT'), `${COMMIT}\n`)
|
|
const driver = path.join(root, 'fake-pg-driver.mjs')
|
|
writeFileSync(driver, FAKE_DRIVER)
|
|
return { root, repo, output, driver }
|
|
}
|
|
|
|
function baseEnvironment(fixture) {
|
|
return {
|
|
...process.env,
|
|
NODE_ENV: 'test',
|
|
VERIFIED_BACKUP_TEST_PG_DRIVER: fixture.driver,
|
|
EXPECTED_COMMIT_SHA: COMMIT,
|
|
BACKUP_OUTPUT_DIR: fixture.output,
|
|
PRODUCTION_PGHOST: 'production.service',
|
|
PRODUCTION_PGPORT: '5432',
|
|
PRODUCTION_PGDATABASE: 'lobechat',
|
|
PRODUCTION_PGUSER: 'gongxiangpan',
|
|
PRODUCTION_PGPASSWORD: PASSWORDS[0],
|
|
ISOLATED_PGHOST: 'restore.service',
|
|
ISOLATED_PGPORT: '5433',
|
|
ISOLATED_PGDATABASE: 'gongxiangp2_restore_fixture01',
|
|
ISOLATED_PGUSER: 'restore_operator',
|
|
ISOLATED_PGPASSWORD: PASSWORDS[1],
|
|
EXPECTED_DATABASE_HOST: 'production.service',
|
|
EXPECTED_DATABASE_PORT: '5432',
|
|
EXPECTED_DATABASE_NAME: 'lobechat',
|
|
EXPECTED_DATABASE_USER: 'gongxiangpan',
|
|
EXPECTED_DATABASE_SERVER_ADDRESS: '10.10.0.10',
|
|
EXPECTED_DATABASE_SERVER_PORT: '5432',
|
|
EXPECTED_ISOLATED_DATABASE_HOST: 'restore.service',
|
|
EXPECTED_ISOLATED_DATABASE_PORT: '5433',
|
|
EXPECTED_ISOLATED_DATABASE_NAME: 'gongxiangp2_restore_fixture01',
|
|
EXPECTED_ISOLATED_DATABASE_USER: 'restore_operator',
|
|
EXPECTED_ISOLATED_DATABASE_SERVER_ADDRESS: '10.20.0.20',
|
|
EXPECTED_ISOLATED_DATABASE_SERVER_PORT: '5433',
|
|
FAKE_PRODUCTION_SERVER_ADDRESS: '10.10.0.10',
|
|
FAKE_ISOLATED_SERVER_ADDRESS: '10.20.0.20',
|
|
FAKE_EMIT_SECRET: '1',
|
|
}
|
|
}
|
|
|
|
function runFixture(fixture, overrides = {}, args = ['--create-and-restore']) {
|
|
return spawnSync(
|
|
process.execPath,
|
|
[path.join(fixture.repo, 'scripts', 'create-verified-production-backup.mjs'), ...args],
|
|
{
|
|
cwd: fixture.repo,
|
|
env: { ...baseEnvironment(fixture), ...overrides },
|
|
encoding: 'utf8',
|
|
timeout: 20_000,
|
|
},
|
|
)
|
|
}
|
|
|
|
function authorize() {
|
|
return {
|
|
ALLOW_VERIFIED_BACKUP: 'CREATE_AND_RESTORE',
|
|
ALLOW_ISOLATED_DATABASE_RESET: 'RESET_VERIFIED_ISOLATED_TARGET',
|
|
CONFIRM_MAINTENANCE_WINDOW: 'NO_WRITES_DURING_SNAPSHOT',
|
|
}
|
|
}
|
|
|
|
function outputFiles(fixture) {
|
|
return readdirSync(fixture.output).sort()
|
|
}
|
|
|
|
function assertNoReceipt(fixture) {
|
|
assert.equal(outputFiles(fixture).some((name) => name.endsWith('.receipt.json')), false)
|
|
assert.equal(outputFiles(fixture).some((name) => name.includes('.partial-')), false)
|
|
}
|
|
|
|
function assertSecretsRedacted(result) {
|
|
for (const password of PASSWORDS) {
|
|
assert.equal(result.stdout.includes(password), false)
|
|
assert.equal(result.stderr.includes(password), false)
|
|
}
|
|
}
|
|
|
|
test('rejects a production endpoint or actual server reused as the isolated target', () => {
|
|
const production = {
|
|
configuredHost: 'production.service', configuredPort: '5432', configuredDatabase: 'lobechat',
|
|
configuredUser: 'app', serverAddress: '10.0.0.10', serverPort: '5432',
|
|
}
|
|
assert.throws(() => assertProductionAndIsolationAreDistinct(production, {
|
|
...production,
|
|
configuredDatabase: 'gongxiangp2_restore_fixture01',
|
|
}), /different PostgreSQL server/)
|
|
assert.throws(() => assertProductionAndIsolationAreDistinct(production, {
|
|
...production,
|
|
configuredHost: 'restore.service',
|
|
configuredPort: '5433',
|
|
configuredDatabase: 'lobechat',
|
|
serverAddress: '10.20.0.20',
|
|
serverPort: '5433',
|
|
}), /database name/)
|
|
})
|
|
|
|
test('subprocess workflow rejects a production endpoint reused as the isolated target', () => {
|
|
const fixture = createFixture()
|
|
try {
|
|
const result = runFixture(fixture, {
|
|
...authorize(),
|
|
ISOLATED_PGHOST: 'production.service',
|
|
ISOLATED_PGPORT: '5432',
|
|
EXPECTED_ISOLATED_DATABASE_HOST: 'production.service',
|
|
EXPECTED_ISOLATED_DATABASE_PORT: '5432',
|
|
})
|
|
assert.notEqual(result.status, 0)
|
|
assert.match(result.stderr, /different PostgreSQL server/)
|
|
assert.deepEqual(outputFiles(fixture), [])
|
|
assertSecretsRedacted(result)
|
|
} finally {
|
|
rmSync(fixture.root, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
test('write mode refuses before any PostgreSQL subprocess when explicit authorization is absent', () => {
|
|
const fixture = createFixture()
|
|
try {
|
|
const result = runFixture(fixture, {
|
|
ALLOW_VERIFIED_BACKUP: '',
|
|
FAKE_FAIL_TOOL: 'psql',
|
|
})
|
|
assert.notEqual(result.status, 0)
|
|
assert.match(result.stderr, /ALLOW_VERIFIED_BACKUP=CREATE_AND_RESTORE/)
|
|
assert.deepEqual(outputFiles(fixture), [])
|
|
assertSecretsRedacted(result)
|
|
} finally {
|
|
rmSync(fixture.root, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
for (const failedTool of ['pg_dump', 'pg_restore']) {
|
|
test(`${failedTool} failure never creates a receipt or leaves a partial file`, () => {
|
|
const fixture = createFixture()
|
|
try {
|
|
const result = runFixture(fixture, { ...authorize(), FAKE_FAIL_TOOL: failedTool })
|
|
assert.notEqual(result.status, 0)
|
|
assert.match(result.stderr, new RegExp(`${failedTool} failed with exit code 42`))
|
|
assertNoReceipt(fixture)
|
|
assertSecretsRedacted(result)
|
|
} finally {
|
|
rmSync(fixture.root, { recursive: true, force: true })
|
|
}
|
|
})
|
|
}
|
|
|
|
test('restored bounded summary mismatch refuses the receipt', () => {
|
|
const fixture = createFixture()
|
|
try {
|
|
const result = runFixture(fixture, { ...authorize(), FAKE_SUMMARY_MODE: 'mismatch' })
|
|
assert.notEqual(result.status, 0)
|
|
assert.match(result.stderr, /Restored database summary does not match/)
|
|
assertNoReceipt(fixture)
|
|
assertSecretsRedacted(result)
|
|
} finally {
|
|
rmSync(fixture.root, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
test('write mode refuses a backup directory containing any existing file', () => {
|
|
const fixture = createFixture()
|
|
try {
|
|
writeFileSync(path.join(fixture.output, 'old-evidence.txt'), 'do not overwrite\n')
|
|
const result = runFixture(fixture, authorize())
|
|
assert.notEqual(result.status, 0)
|
|
assert.match(result.stderr, /BACKUP_OUTPUT_DIR must be empty/)
|
|
assert.deepEqual(outputFiles(fixture), ['old-evidence.txt'])
|
|
assertSecretsRedacted(result)
|
|
} finally {
|
|
rmSync(fixture.root, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
test('success atomically creates a custom dump and a machine-generated compatible receipt', () => {
|
|
const fixture = createFixture()
|
|
try {
|
|
const result = runFixture(fixture, authorize())
|
|
assert.equal(result.status, 0, result.stderr)
|
|
assertSecretsRedacted(result)
|
|
const files = outputFiles(fixture)
|
|
assert.equal(files.some((name) => name.includes('.partial-')), false)
|
|
assert.equal(files.filter((name) => name.endsWith('.dump')).length, 1)
|
|
assert.equal(files.filter((name) => name.endsWith('.receipt.json')).length, 1)
|
|
|
|
const artifactPath = path.join(fixture.output, files.find((name) => name.endsWith('.dump')))
|
|
const receiptPath = path.join(fixture.output, files.find((name) => name.endsWith('.receipt.json')))
|
|
const artifact = readFileSync(artifactPath)
|
|
const receiptBytes = readFileSync(receiptPath)
|
|
const receipt = JSON.parse(receiptBytes.toString('utf8'))
|
|
const expectedArtifactHash = createHash('sha256').update(artifact).digest('hex')
|
|
assert.equal(receipt.generatedBy, 'scripts/create-verified-production-backup.mjs')
|
|
assert.equal(receipt.sourceCommit, COMMIT)
|
|
assert.equal(receipt.backup.artifactPath, artifactPath)
|
|
assert.equal(receipt.backup.artifactSize, artifact.length)
|
|
assert.equal(receipt.backup.artifactSha256, expectedArtifactHash)
|
|
assert.equal(receipt.restoreValidation.status, 'passed')
|
|
assert.equal(receipt.restoreValidation.artifactSha256, expectedArtifactHash)
|
|
assert.equal(
|
|
receipt.restoreValidation.databaseFingerprint,
|
|
databaseFingerprintHash(receipt.database),
|
|
)
|
|
assert.notEqual(
|
|
receipt.restoreValidation.isolatedDatabaseFingerprint,
|
|
receipt.restoreValidation.databaseFingerprint,
|
|
)
|
|
assert.equal(
|
|
receipt.restoreValidation.verificationSummary.sourceSummarySha256,
|
|
receipt.restoreValidation.verificationSummary.restoredSummarySha256,
|
|
)
|
|
assert.doesNotThrow(() => assertBackupReceiptDocument(receipt, {
|
|
expectedCommit: COMMIT,
|
|
database: receipt.database,
|
|
maxAgeMinutes: 240,
|
|
}))
|
|
const receiptHash = createHash('sha256').update(receiptBytes).digest('hex')
|
|
assert.ok(result.stdout.includes(receiptHash))
|
|
} finally {
|
|
rmSync(fixture.root, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
test('default audit mode is read-only and produces no backup files', () => {
|
|
const fixture = createFixture()
|
|
try {
|
|
const result = runFixture(fixture, {}, [])
|
|
assert.equal(result.status, 0, result.stderr)
|
|
assert.match(result.stdout, /Read-only verified backup audit completed/)
|
|
assert.deepEqual(outputFiles(fixture), [])
|
|
assertSecretsRedacted(result)
|
|
} finally {
|
|
rmSync(fixture.root, { recursive: true, force: true })
|
|
}
|
|
})
|