diff --git a/docs/生产数据库安全增量迁移SOP-2026-08-12.md b/docs/生产数据库安全增量迁移SOP-2026-08-12.md index c69f468e..1e375586 100644 --- a/docs/生产数据库安全增量迁移SOP-2026-08-12.md +++ b/docs/生产数据库安全增量迁移SOP-2026-08-12.md @@ -166,6 +166,8 @@ PRODUCTION_VERIFY 禁止手写或补填 receipt。`scripts/create-verified-production-backup.mjs` 会在同一次受控运行中自动完成: +恢复使用原始 custom 归档分段执行 `pg_restore`。pre-data 完成后,先验证两个既有回收站函数的确定签名、语言、返回类型、不可变性及空配置;仅在隔离库给它们临时设置 `pg_catalog, public` 搜索路径,再完整恢复 data 和 post-data。恢复后 RESET,并核对函数定义与全部已捕获配置完全一致,才进行摘要验证和生成 receipt。此步骤解决空搜索路径下重建回收站 GIN 索引时的旧函数内部调用问题;不修改归档、生产函数或全局恢复搜索路径。 + 1. 只读核对生产库与隔离库的配置身份和数据库实际身份; 2. 在生产库读取迁移表、素材、用户、系统设置、产品包作业的有界摘要; 3. 用 `pg_dump --format=custom` 写入本次唯一的 `.partial` 文件,刷新到磁盘后原子改名; diff --git a/scripts/create-verified-production-backup.mjs b/scripts/create-verified-production-backup.mjs index 4defcaba..510e3e04 100644 --- a/scripts/create-verified-production-backup.mjs +++ b/scripts/create-verified-production-backup.mjs @@ -109,6 +109,39 @@ DROP SCHEMA IF EXISTS public CASCADE; CREATE SCHEMA public; ` +export const PURGE_RESTORE_FUNCTION_EVIDENCE_SQL = String.raw` +/* verified-backup-purge-functions-v1 */ +SELECT COALESCE(jsonb_agg(jsonb_build_object( + 'signature', CASE p.proname WHEN 'purge_reference_keys' THEN 'public.purge_reference_keys(text[],boolean)' ELSE 'public.purge_json_reference_strings(jsonb)' END, + 'definition', pg_catalog.pg_get_functiondef(p.oid), 'config', to_jsonb(p.proconfig), + 'language', l.lanname, 'returnType', pg_catalog.format_type(p.prorettype, NULL), + 'volatility', p.provolatile, 'strict', p.proisstrict, 'securityDefiner', p.prosecdef, 'parallel', p.proparallel +) ORDER BY p.proname), '[]'::jsonb)::text +FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_language l ON l.oid=p.prolang +WHERE p.oid IN (pg_catalog.to_regprocedure('public.purge_reference_keys(text[],boolean)'), + pg_catalog.to_regprocedure('public.purge_json_reference_strings(jsonb)')); +` +export const PURGE_RESTORE_SET_SQL = String.raw` +/* verified-backup-purge-set-v1 */ +ALTER FUNCTION public.purge_reference_keys(text[],boolean) SET search_path TO pg_catalog, public; +ALTER FUNCTION public.purge_json_reference_strings(jsonb) SET search_path TO pg_catalog, public; +` +export const PURGE_RESTORE_RESET_SQL = String.raw` +/* verified-backup-purge-reset-v1 */ +ALTER FUNCTION public.purge_reference_keys(text[],boolean) RESET search_path; +ALTER FUNCTION public.purge_json_reference_strings(jsonb) RESET search_path; +` + +export function assertPurgeRestoreFunctions(rows) { + const expected = ['public.purge_json_reference_strings(jsonb)', 'public.purge_reference_keys(text[],boolean)'] + if (!Array.isArray(rows) || rows.length !== 2 || rows.some((row, index) => row.signature !== expected[index] + || row.language !== 'plpgsql' || row.returnType !== 'text[]' || row.volatility !== 'i' + || row.strict !== true || row.securityDefiner !== false || row.parallel !== 'u' + || row.config !== null || typeof row.definition !== 'string' || !row.definition.includes('CREATE OR REPLACE FUNCTION public.'))) { + fail('Restored purge function metadata is unsupported; backup receipt refused') + } +} + function fail(message) { throw new Error(message) } @@ -501,17 +534,21 @@ async function createDump(connection, outputDirectory, basename) { async function restoreDump(connection, artifactPath) { await executeSql(connection, RESET_ISOLATED_SQL, 'isolated database reset') - await runPgClient('pg_restore', [ - '--exit-on-error', - '--no-owner', - '--no-privileges', - '--dbname', - connection.database, - artifactPath, - ], { - env: pgEnvironment(connection), - label: 'pg_restore', - }) + const restoreSections = async sections => runPgClient('pg_restore', [ + '--exit-on-error', '--no-owner', '--no-privileges', ...sections.map(section => `--section=${section}`), + '--dbname', connection.database, artifactPath, + ], { env: pgEnvironment(connection), label: 'pg_restore' }) + await restoreSections(['pre-data']) + const originalFunctions = await queryJson(connection, PURGE_RESTORE_FUNCTION_EVIDENCE_SQL, 'restored purge function metadata') + assertPurgeRestoreFunctions(originalFunctions) + // Only the two known legacy functions need a temporary resolver while GIN indexes + // are recreated. pg_restore keeps its global empty search_path throughout. + await executeSql(connection, PURGE_RESTORE_SET_SQL, 'isolated purge function resolver') + await restoreSections(['data', 'post-data']) + await executeSql(connection, PURGE_RESTORE_RESET_SQL, 'isolated purge function resolver reset') + const restoredFunctions = await queryJson(connection, PURGE_RESTORE_FUNCTION_EVIDENCE_SQL, 'restored purge function final metadata') + assertPurgeRestoreFunctions(restoredFunctions) + if (JSON.stringify(originalFunctions) !== JSON.stringify(restoredFunctions)) fail('Restored purge function definitions or configurations changed; backup receipt refused') } function sanitizedAudit(mode, sourceCommit, production, isolated, versions) { diff --git a/scripts/create-verified-production-backup.test.mjs b/scripts/create-verified-production-backup.test.mjs index 5445ecca..14adaa29 100644 --- a/scripts/create-verified-production-backup.test.mjs +++ b/scripts/create-verified-production-backup.test.mjs @@ -16,6 +16,7 @@ import test from 'node:test' import { assertProductionAndIsolationAreDistinct, + assertPurgeRestoreFunctions, } from './create-verified-production-backup.mjs' import { assertBackupReceiptDocument, @@ -26,12 +27,13 @@ const COMMIT = 'a'.repeat(40) const PASSWORDS = ['production-password-must-not-leak', 'isolated-password-must-not-leak'] const FAKE_DRIVER = String.raw` -import { existsSync, writeFileSync } from 'node:fs' +import { appendFileSync, existsSync, writeFileSync } from 'node:fs' const [tool, ...args] = process.argv.slice(2) const failTool = process.env.FAKE_FAIL_TOOL || '' const secret = process.env.PGPASSWORD || '' if (process.env.FAKE_EMIT_SECRET === '1' && secret) process.stderr.write(secret + '\n') +if (process.env.FAKE_TOOL_TRACE_FILE) appendFileSync(process.env.FAKE_TOOL_TRACE_FILE, JSON.stringify({ tool, args }) + '\n') if (args.includes('--version')) { process.stdout.write(tool + ' (PostgreSQL) 15.14\n') @@ -47,6 +49,7 @@ if (tool === 'pg_dump') { } if (tool === 'pg_restore') { + if (process.env.FAKE_FAIL_RESTORE_SECTION && args.includes('--section=' + process.env.FAKE_FAIL_RESTORE_SECTION)) process.exit(42) const artifact = args.at(-1) if (!artifact || !existsSync(artifact)) process.exit(44) process.exit(0) @@ -79,6 +82,18 @@ if (sql.includes('verified-backup-summary-v1')) { process.exit(0) } if (sql.includes('verified-backup-reset-isolated-v1')) process.exit(0) +if (sql.includes('verified-backup-purge-set-v1') || sql.includes('verified-backup-purge-reset-v1')) process.exit(0) +if (sql.includes('verified-backup-purge-functions-v1')) { + const names = ['public.purge_json_reference_strings(jsonb)', 'public.purge_reference_keys(text[],boolean)']; + const metadata = names.map(signature => ({ signature, definition: 'CREATE OR REPLACE FUNCTION public.' + signature.slice(7) + ' legacy-body', config: null, language: 'plpgsql', returnType: 'text[]', volatility: 'i', strict: true, securityDefiner: false, parallel: 'u' })); + if (process.env.FAKE_PURGE_METADATA_MODE === 'missing') metadata.pop(); + if (process.env.FAKE_PURGE_METADATA_MODE === 'config') metadata[0].config = ['search_path=public']; + if (process.env.FAKE_PURGE_METADATA_MODE === 'changed') { + if (existsSync(process.env.FAKE_PURGE_STATE_FILE)) metadata[0].definition += '-changed'; + else writeFileSync(process.env.FAKE_PURGE_STATE_FILE, 'captured'); + } + process.stdout.write(JSON.stringify(metadata) + '\n'); process.exit(0); +} process.exit(46) ` @@ -238,6 +253,53 @@ for (const failedTool of ['pg_dump', 'pg_restore']) { }) } +test('restore covers all three sections and resets only the two scoped legacy functions before receipt', () => { + const fixture = createFixture() + try { + const trace = path.join(fixture.root, 'tools.jsonl') + const result = runFixture(fixture, { ...authorize(), FAKE_TOOL_TRACE_FILE: trace }) + assert.equal(result.status, 0, result.stderr) + const calls = readFileSync(trace, 'utf8').trim().split('\n').map(line => JSON.parse(line)) + const restores = calls.filter(call => call.tool === 'pg_restore' && !call.args.includes('--version')) + assert.equal(restores.length, 2) + assert.ok(restores[0].args.includes('--section=pre-data')) + assert.ok(restores[1].args.includes('--section=data') && restores[1].args.includes('--section=post-data')) + assert.ok(restores.every(call => call.args.includes('--exit-on-error'))) + const sql = calls.filter(call => call.tool === 'psql').map(call => call.args.at(-1)) + assert.ok(sql.some(value => value.includes('ALTER FUNCTION public.purge_reference_keys(text[],boolean) SET search_path TO pg_catalog, public'))) + assert.ok(sql.some(value => value.includes('ALTER FUNCTION public.purge_json_reference_strings(jsonb) RESET search_path'))) + assert.equal(sql.filter(value => value.includes('verified-backup-purge-functions-v1')).length, 2) + } finally { rmSync(fixture.root, { recursive: true, force: true }) } +}) + +for (const mode of ['missing', 'config', 'changed']) { + test(`unsupported or changed purge metadata (${mode}) never produces a backup receipt`, () => { + const fixture = createFixture() + try { + const result = runFixture(fixture, { ...authorize(), FAKE_PURGE_METADATA_MODE: mode, FAKE_PURGE_STATE_FILE: path.join(fixture.root, 'purge-state') }) + assert.notEqual(result.status, 0) + assert.match(result.stderr, mode === 'changed' ? /definitions or configurations changed/ : /metadata is unsupported/) + assert.equal(readdirSync(fixture.output).some(name => name.endsWith('.receipt.json')), false) + } finally { rmSync(fixture.root, { recursive: true, force: true }) } + }) +} + +test('post-data index restore failure remains fatal and cannot issue a receipt', () => { + const fixture = createFixture() + try { + const result = runFixture(fixture, { ...authorize(), FAKE_FAIL_RESTORE_SECTION: 'post-data' }) + assert.notEqual(result.status, 0) + assert.match(result.stderr, /pg_restore failed with exit code 42/) + assert.equal(readdirSync(fixture.output).some(name => name.endsWith('.receipt.json')), false) + } finally { rmSync(fixture.root, { recursive: true, force: true }) } +}) + +test('legacy resolver metadata requires exact signatures and excludes privileged or mutable functions', () => { + const rows = ['public.purge_json_reference_strings(jsonb)', 'public.purge_reference_keys(text[],boolean)'].map(signature => ({ signature, definition: 'CREATE OR REPLACE FUNCTION public.' + signature.slice(7), config: null, language: 'plpgsql', returnType: 'text[]', volatility: 'i', strict: true, securityDefiner: false, parallel: 'u' })) + assert.doesNotThrow(() => assertPurgeRestoreFunctions(rows)) + for (const change of [{ securityDefiner: true }, { volatility: 'v' }, { strict: false }, { signature: 'public.other_function(jsonb)' }]) assert.throws(() => assertPurgeRestoreFunctions([{ ...rows[0], ...change }, rows[1]]), /metadata is unsupported/) +}) + test('restored bounded summary mismatch refuses the receipt', () => { const fixture = createFixture() try { diff --git a/scripts/verify-purge-backup-restore.mjs b/scripts/verify-purge-backup-restore.mjs new file mode 100644 index 00000000..cf8c2eb0 --- /dev/null +++ b/scripts/verify-purge-backup-restore.mjs @@ -0,0 +1,68 @@ +/** Real, disposable LOCAL PostgreSQL restore regression; never connects to production. */ +import assert from 'node:assert/strict' +import { randomUUID } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { spawnSync } from 'node:child_process' +import { PURGE_RESTORE_FUNCTION_EVIDENCE_SQL, PURGE_RESTORE_SET_SQL, PURGE_RESTORE_RESET_SQL, assertPurgeRestoreFunctions } from './create-verified-production-backup.mjs' + +const container = 'gongxiang-detail-test-20261012' +function docker(args, options = {}) { + const result = spawnSync('docker', ['exec', '-i', container, ...args], { encoding: 'utf8', maxBuffer: 2 * 1024 * 1024, ...options }) + if (options.allowFailure) return result + assert.equal(result.status, 0, result.stderr || 'Local PostgreSQL command failed') + return result.stdout.trim() +} +const inspect = spawnSync('docker', ['inspect', '--format', '{{json .NetworkSettings.Ports}}', container], { encoding: 'utf8' }) +assert.equal(inspect.status, 0) +assert.deepEqual(JSON.parse(inspect.stdout)['5432/tcp'], [{ HostIp: '127.0.0.1', HostPort: '15437' }], 'Only the existing disposable local database is allowed') +const user = docker(['sh', '-c', 'printf "%s" "$POSTGRES_USER"']) || 'postgres' +assert.match(user, /^[a-zA-Z0-9_]+$/) +const suffix = randomUUID().replaceAll('-', '').slice(0, 12) +const source = `gongxiangp2_restore_purge_source_${suffix}` +const broken = `gongxiangp2_restore_purge_broken_${suffix}` +const restored = `gongxiangp2_restore_purge_passed_${suffix}` +const dump = `/tmp/purge-restore-${suffix}.dump` +const psql = (database, sql) => docker(['psql', '-X', '-A', '-t', '-v', 'ON_ERROR_STOP=1', '-U', user, '-d', database, '-c', sql]) +const restore = (database, sections = [], allowFailure = false) => docker(['pg_restore', '--exit-on-error', '--no-owner', '--no-privileges', ...sections.map(section => `--section=${section}`), '-U', user, '--dbname', database, dump], { allowFailure }) +const schema = ` +CREATE TABLE materials ("previewUrl" text, "zipFilePath" text, "videoSourceUrl" text, "videoPreview540Url" text, "videoPreviewClipUrl" text, "videoCoverUrl" text, "displayImages" text[]); +CREATE TABLE material_assets (file_url text); +CREATE TABLE material_skus (file_path text, preview_url text); +CREATE TABLE brand_knowledge (file_path text, preview_image_path text, preview_image_slices text[]); +CREATE TABLE system_settings ("headerLogoUrl" text, "loginLogoUrl" text); +CREATE TABLE performance_survey_work_items (image_urls jsonb); +CREATE TABLE work_review_versions (image_urls jsonb); +CREATE TABLE product_package_upload_jobs (assets jsonb, status text); +CREATE TABLE task_detail_change_requests (operations jsonb, status text); +INSERT INTO brand_knowledge VALUES ('uploads/%E4%BD%A0%E5%A5%BD.png', NULL, '{}'); +INSERT INTO work_review_versions VALUES ('{"nested":["uploads/%E4%BD%A0%E5%A5%BD.png"]}'); +` +const migration = readFileSync(new URL('../prisma/migrations/20261009_material_purge_reference_indexes/migration.sql', import.meta.url), 'utf8') +try { + for (const database of [source, broken, restored]) psql('postgres', `CREATE DATABASE ${database};`) + psql(source, schema) + psql(source, migration) + docker(['pg_dump', '--format=custom', '--no-owner', '--no-privileges', '-U', user, '--dbname', source, '--file', dump]) + const originalHash = docker(['sha256sum', dump]).split(' ')[0] + const failed = restore(broken, [], true) + assert.notEqual(failed.status, 0, 'Original restore unexpectedly accepted the unqualified legacy functions') + assert.match(failed.stderr, /purge_decode_reference.*does not exist/) + restore(restored, ['pre-data']) + const before = JSON.parse(psql(restored, PURGE_RESTORE_FUNCTION_EVIDENCE_SQL)) + assertPurgeRestoreFunctions(before) + psql(restored, PURGE_RESTORE_SET_SQL) + restore(restored, ['data', 'post-data']) + psql(restored, PURGE_RESTORE_RESET_SQL) + const after = JSON.parse(psql(restored, PURGE_RESTORE_FUNCTION_EVIDENCE_SQL)) + assertPurgeRestoreFunctions(after) + assert.deepEqual(after, before, 'Function definition or final configuration changed') + const indexes = Number(psql(restored, "SELECT count(*) FROM pg_catalog.pg_index i JOIN pg_catalog.pg_class c ON c.oid=i.indexrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relname LIKE '%purge_references_idx' AND i.indisvalid AND i.indisready;")) + assert.equal(indexes, 9, 'Every original GIN index must be restored and valid') + assert.equal(psql(restored, 'SELECT count(*) FROM brand_knowledge;'), '1') + assert.equal(psql(restored, 'SELECT count(*) FROM work_review_versions;'), '1') + assert.equal(docker(['sha256sum', dump]).split(' ')[0], originalHash, 'Original dump changed') + console.log(JSON.stringify({ passed: ['original empty-search-path restore failure reproduced', 'all pre-data/data/post-data restored', '9 original GIN indexes valid', 'original function definitions/configuration restored', 'dump hash unchanged'], productionWrites: 0 })) +} finally { + for (const database of [source, broken, restored]) psql('postgres', `DROP DATABASE IF EXISTS ${database} WITH (FORCE);`) + docker(['rm', '-f', dump]) +}