furniture-3d-system-Wu/furniture-3d-system/scripts/apply-master-data-foundation-migrations.js

581 lines
20 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

const fs = require('node:fs')
const crypto = require('node:crypto')
const path = require('node:path')
const { spawnSync } = require('node:child_process')
const { Pool } = require('pg')
const { loadDatabaseUrl, describeDatabaseUrl, getDatabaseIdentity } = require('./diagnose-prisma-migration-baseline')
const {
REQUIRED_MIGRATIONS,
applyMissingMigrationFiles,
classifyAppliedMigrationPaths,
validateRequiredMigrationCatalog,
} = require('./master-data-required-migrations')
const {
getRequiredMigrationReadiness,
} = require('./verify-master-data-foundation')
const ROOT_DIR = path.resolve(__dirname, '..')
const PRISMA_CLI = path.join(ROOT_DIR, 'node_modules/prisma/build/index.js')
const CONFIRM_TOKEN = 'apply-master-data-foundation-migrations'
const APPLY_PHASES = new Set(['finalize', 'bootstrap-repair'])
function parseArgs(argv = process.argv.slice(2)) {
const options = {
apply: false,
confirm: '',
expectedDatabaseIdentity: '',
expectedBackupSha256: '',
outputPath: '',
phase: 'finalize',
}
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index]
if (arg === '--apply') {
options.apply = true
continue
}
if (arg.startsWith('--confirm=')) {
options.confirm = arg.slice('--confirm='.length)
continue
}
if (arg === '--expected-database-identity') {
const value = argv[index + 1]
if (!value || value.startsWith('--')) {
throw new Error('缺少 --expected-database-identity <identity>。')
}
options.expectedDatabaseIdentity = value
index += 1
continue
}
if (arg.startsWith('--expected-database-identity=')) {
options.expectedDatabaseIdentity = arg.slice('--expected-database-identity='.length)
continue
}
if (arg === '--expected-backup-sha256') {
const value = argv[index + 1]
if (!value || value.startsWith('--')) {
throw new Error('缺少 --expected-backup-sha256 <sha256>。')
}
options.expectedBackupSha256 = value
index += 1
continue
}
if (arg.startsWith('--expected-backup-sha256=')) {
options.expectedBackupSha256 = arg.slice('--expected-backup-sha256='.length)
continue
}
if (arg === '--output') {
const value = argv[index + 1]
if (!value || value.startsWith('--')) {
throw new Error('缺少 --output <json-file>。')
}
options.outputPath = value
index += 1
continue
}
if (arg.startsWith('--output=')) {
options.outputPath = arg.slice('--output='.length)
if (!options.outputPath) {
throw new Error('缺少 --output <json-file>。')
}
continue
}
if (arg === '--phase') {
const value = argv[index + 1]
if (!value || value.startsWith('--')) {
throw new Error('缺少 --phase <finalize|bootstrap-repair>。')
}
if (!APPLY_PHASES.has(value)) {
throw new Error(`未知 phase:${value}`)
}
options.phase = value
index += 1
continue
}
if (arg.startsWith('--phase=')) {
const value = arg.slice('--phase='.length)
if (!APPLY_PHASES.has(value)) {
throw new Error(`未知 phase:${value}`)
}
options.phase = value
continue
}
throw new Error(`未知参数:${arg}`)
}
return options
}
function writeJsonReport(outputPath, payload, rootDir = ROOT_DIR) {
if (!outputPath) return ''
const absolutePath = path.resolve(rootDir, outputPath)
fs.mkdirSync(path.dirname(absolutePath), { recursive: true })
fs.writeFileSync(absolutePath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8')
return absolutePath
}
function migrationHistoryReady(migrationHistory = {}) {
const unresolvedRolledBackMigrationRows =
migrationHistory.unresolvedRolledBackMigrationRows == null
? migrationHistory.rolledBackMigrationRows
: migrationHistory.unresolvedRolledBackMigrationRows
return (
migrationHistory.tableExists === true &&
migrationHistory.unfinishedMigrationRows === 0 &&
unresolvedRolledBackMigrationRows === 0
)
}
function normalizeSha256(value) {
return String(value || '').trim().toLowerCase()
}
function getMigrationCatalogByPath(migrations = REQUIRED_MIGRATIONS) {
return new Map(migrations.map((migration) => [migration.path, migration]))
}
function targetRelationsReady(migrationReadiness = {}) {
return !Array.isArray(migrationReadiness.targetRelations) ||
migrationReadiness.targetRelations.every((relation) => relation.exists === true)
}
function getFailedDataChecks(readiness = {}) {
if (!Array.isArray(readiness.checks)) return []
return readiness.checks.flatMap((migration) => {
if (!targetRelationsReady(migration) || !Array.isArray(migration.dataChecks)) return []
return migration.dataChecks
.filter((check) => check.matches === false)
.map((check) => ({
path: migration.path,
kind: migration.kind,
name: check.name,
expectedValue: check.expectedValue,
}))
})
}
function getMigrationReadinessByPath(readiness = {}) {
return new Map((readiness.checks || []).map((migration) => [migration.path, migration]))
}
function getMigrationCatalogEntry(path, migrations = REQUIRED_MIGRATIONS) {
return migrations.find((migration) => migration.path === path)
}
function isIdempotentGuardRepairDataCheck(failedCheck, readiness = {}, migrations = REQUIRED_MIGRATIONS) {
if (failedCheck.kind !== 'idempotent') return false
const migration = getMigrationCatalogEntry(failedCheck.path, migrations)
if (!migration?.alwaysApply) return false
if (
!Array.isArray(migration.guardRepairDataChecks) ||
!migration.guardRepairDataChecks.includes(failedCheck.name)
) {
return false
}
const migrationReadiness = getMigrationReadinessByPath(readiness).get(failedCheck.path)
if (!migrationReadiness || !targetRelationsReady(migrationReadiness)) return false
return Array.isArray(migration.dataChecks) &&
migration.dataChecks.some((check) => check.name === failedCheck.name)
}
function getBlockingFailedDataChecks(readiness = {}, migrations = REQUIRED_MIGRATIONS) {
return getFailedDataChecks(readiness).filter(
(check) => !isIdempotentGuardRepairDataCheck(check, readiness, migrations),
)
}
function getBootstrapRepairPlan(readiness = {}, migrations = REQUIRED_MIGRATIONS) {
const catalogByPath = getMigrationCatalogByPath(migrations)
const candidates = []
const blockers = []
for (const migrationReadiness of readiness.checks || []) {
if (migrationReadiness.kind !== 'structure' || migrationReadiness.ready === true) {
continue
}
const migration = catalogByPath.get(migrationReadiness.path)
if (!migration) {
blockers.push({
key: 'bootstrapRepair.catalogMissing',
path: migrationReadiness.path,
message: '结构修复阶段发现 readiness 里存在未登记的治理迁移,不能继续。'
})
continue
}
const targetReady = targetRelationsReady(migrationReadiness)
const hasRepairSql = Boolean(migration.repairSql)
if (!targetReady || hasRepairSql) {
candidates.push({
path: migration.path,
strategy: targetReady ? 'repair-sql' : 'migration-sql',
})
continue
}
blockers.push({
key: 'bootstrapRepair.unrepairableStructureMigration',
path: migration.path,
message: '结构迁移目标已存在但仍未就绪,且未声明 repairSql;不能用 bootstrap 阶段盲目执行。'
})
}
return { candidates, blockers }
}
function getBackupFingerprint(env = process.env, rootDir = ROOT_DIR) {
const rawPath = env.DATABASE_BACKUP_FILE || env.DB_BACKUP_FILE || ''
if (!rawPath) {
return { provided: false, path: '', exists: false, sha256: '' }
}
const absolutePath = path.resolve(rootDir, rawPath)
if (!fs.existsSync(absolutePath)) {
return { provided: true, path: absolutePath, exists: false, sha256: '' }
}
const bytes = fs.readFileSync(absolutePath)
return {
provided: true,
path: absolutePath,
exists: true,
sizeBytes: bytes.length,
sha256: crypto.createHash('sha256').update(bytes).digest('hex'),
}
}
function buildFoundationRequiredMigrationApplyPlan(readiness, options = {}) {
const phase = options.phase || 'finalize'
const blockers = []
const failedDataChecks = getFailedDataChecks(readiness)
const blockingFailedDataChecks = getBlockingFailedDataChecks(readiness)
const bootstrapRepairPlan = getBootstrapRepairPlan(readiness)
if (!migrationHistoryReady(readiness.migrationHistory)) {
blockers.push({
key: 'migrationHistory.notReady',
message: '治理迁移/护栏真实应用前,必须先补齐 _prisma_migrations 历史且无失败/未解决回滚记录。',
})
}
if (phase === 'finalize' && readiness.pendingStructureMigrationCount > 0) {
blockers.push({
key: 'pendingStructureMigrationCount',
value: readiness.pendingStructureMigrationCount,
message: '仍有结构迁移未就绪,不能先应用治理护栏;必须先完成结构修复和 baseline resolve。',
})
}
if (phase === 'finalize' && blockingFailedDataChecks.length > 0) {
blockers.push({
key: 'dataChecks.notReady',
value: blockingFailedDataChecks.length,
message: '治理迁移应用前仍有历史数据不满足数据检查,必须先清洗真实主数据,不能靠约束迁移硬压过去。',
evidence: blockingFailedDataChecks,
})
}
if (phase === 'bootstrap-repair') {
blockers.push(...bootstrapRepairPlan.blockers)
if (bootstrapRepairPlan.candidates.length === 0) {
blockers.push({
key: 'bootstrapRepair.noWork',
message: '当前没有可由 bootstrap-repair 阶段处理的结构迁移或历史回填。'
})
}
}
if (options.apply && options.confirm !== CONFIRM_TOKEN) {
blockers.push({
key: 'confirm',
message: `真实执行必须显式传入 --confirm=${CONFIRM_TOKEN}。`,
})
}
if (options.apply && !options.expectedDatabaseIdentity) {
blockers.push({
key: 'expectedDatabaseIdentity.missing',
message: '治理迁移真实执行必须显式传入 --expected-database-identity,避免误连错库。',
})
} else if (
options.apply &&
options.actualDatabaseIdentity &&
options.expectedDatabaseIdentity !== options.actualDatabaseIdentity
) {
blockers.push({
key: 'expectedDatabaseIdentity.mismatch',
message: '命令行指定的目标库身份和当前 DATABASE_URL 不一致,禁止应用治理迁移。',
})
}
if (options.apply && !options.expectedBackupSha256) {
blockers.push({
key: 'expectedBackupSha256.missing',
message: '治理迁移真实执行必须显式传入 --expected-backup-sha256,避免在没有确认备份的情况下写库。',
})
} else if (options.apply && options.backupFingerprint?.sha256) {
if (normalizeSha256(options.expectedBackupSha256) !== normalizeSha256(options.backupFingerprint.sha256)) {
blockers.push({
key: 'expectedBackupSha256.mismatch',
message: '命令行指定的备份 sha256 和当前 DATABASE_BACKUP_FILE 不一致,禁止应用治理迁移。',
})
}
}
if (options.apply && options.backupFingerprint?.provided !== true) {
blockers.push({
key: 'backupFingerprint.missing',
message: '治理迁移真实执行前必须通过 DATABASE_BACKUP_FILE 或 DB_BACKUP_FILE 指定当前数据库备份。',
})
} else if (options.apply && options.backupFingerprint?.exists !== true) {
blockers.push({
key: 'backupFingerprint.notFound',
message: '命令指定的数据库备份文件不存在,禁止应用治理迁移。',
})
}
return {
mode: 'master-data-foundation-required-migration-apply-plan',
phase,
dryRun: !options.apply,
readyToApply: options.apply === true && blockers.length === 0,
confirmToken: CONFIRM_TOKEN,
migrationHistory: readiness.migrationHistory,
migrationCount: readiness.migrationCount,
readyMigrationCount: readiness.readyMigrationCount,
pendingMigrationCount: readiness.pendingMigrationCount,
pendingStructureMigrationCount: readiness.pendingStructureMigrationCount,
pendingIdempotentMigrationCount: readiness.pendingIdempotentMigrationCount,
pendingStructureMigrationPaths: readiness.pendingStructureMigrationPaths,
pendingIdempotentMigrationPaths: readiness.pendingIdempotentMigrationPaths,
failedDataChecks,
blockingFailedDataChecks,
bootstrapRepairCandidates: bootstrapRepairPlan.candidates,
blockers,
actualDatabaseIdentity: options.actualDatabaseIdentity || '',
expectedDatabaseIdentity: options.expectedDatabaseIdentity || '',
backupFingerprint: options.backupFingerprint || null,
commands: [
{
name: 'apply-bootstrap-structure-and-backfill-repairs',
command: `node scripts/apply-master-data-foundation-migrations.js --phase=bootstrap-repair --apply --confirm=${CONFIRM_TOKEN} --expected-database-identity <database-identity> --expected-backup-sha256 <backup-sha256>`,
purpose: '在 migration history 已补齐后,只应用缺失结构和可证明的 repairSql 回填,不提前应用最终治理护栏。',
dangerous: true,
},
{
name: 'apply-required-foundation-migrations',
command: `node scripts/apply-master-data-foundation-migrations.js --apply --confirm=${CONFIRM_TOKEN} --expected-database-identity <database-identity> --expected-backup-sha256 <backup-sha256>`,
purpose: '在结构修复和 baseline resolve 完成后,事务性应用治理迁移、视图、约束和触发器。',
dangerous: true,
},
{
name: 'verify-readiness',
command: 'npm run db:verify-master-data:readiness',
purpose: '应用后重新验证迁移历史、结构化主数据、外部只读契约和数据库护栏。',
dangerous: false,
},
],
}
}
function getMigrationsByPaths(paths, migrations = REQUIRED_MIGRATIONS) {
const pathSet = new Set(paths)
return migrations.filter((migration) => pathSet.has(migration.path))
}
function getUnreadyPathsAfterApply(after, appliedPaths) {
const appliedPathSet = new Set(appliedPaths)
return (after.checks || [])
.filter((migration) => appliedPathSet.has(migration.path) && migration.ready !== true)
.map((migration) => migration.path)
}
function getMigrationIdFromPath(migrationPath) {
const normalized = String(migrationPath || '').replace(/\\/g, '/')
const parts = normalized.split('/').filter(Boolean)
const migrationIndex = parts.lastIndexOf('migrations')
const migrationId = migrationIndex >= 0 ? parts[migrationIndex + 1] : ''
if (!/^\d{14}_[A-Za-z0-9_]+$/.test(migrationId)) {
throw new Error(`无法从迁移路径解析 migration id:${migrationPath}`)
}
return migrationId
}
async function getAppliedPrismaMigrationNames(client) {
const tableResult = await client.query(`
SELECT to_regclass('_prisma_migrations') AS relation_name
`)
const tableExists = tableResult.rows[0]?.relation_name !== null
if (!tableExists) {
throw new Error('数据库缺少 _prisma_migrations,不能为治理迁移补 Prisma 历史。')
}
const result = await client.query(`
SELECT migration_name
FROM _prisma_migrations
WHERE finished_at IS NOT NULL
AND rolled_back_at IS NULL
`)
return new Set(result.rows.map((row) => row.migration_name).filter(Boolean))
}
function runPrismaMigrateResolveApplied(migrationId, env = process.env) {
return spawnSync(process.execPath, [PRISMA_CLI, 'migrate', 'resolve', '--applied', migrationId], {
cwd: ROOT_DIR,
encoding: 'utf8',
env: { ...process.env, ...env },
windowsHide: true,
})
}
async function resolveAppliedMigrationHistory({
client,
migrationPaths,
env = process.env,
runner = runPrismaMigrateResolveApplied,
}) {
const uniqueMigrationIds = Array.from(
new Set((migrationPaths || []).map((migrationPath) => getMigrationIdFromPath(migrationPath))),
)
if (uniqueMigrationIds.length === 0) return []
const appliedMigrationNames = await getAppliedPrismaMigrationNames(client)
const resolved = []
for (const migrationId of uniqueMigrationIds) {
if (appliedMigrationNames.has(migrationId)) {
continue
}
const result = runner(migrationId, env)
const status = result.status ?? 1
if (status !== 0) {
throw new Error(
[
`治理迁移 SQL 已提交,但 Prisma 历史补登记失败:${migrationId}`,
String(result.stdout ?? '').trim(),
String(result.stderr ?? '').trim(),
result.error ? String(result.error.message ?? result.error) : '',
].filter(Boolean).join('\n'),
)
}
resolved.push({
migrationId,
stdout: String(result.stdout ?? '').trim(),
stderr: String(result.stderr ?? '').trim(),
})
appliedMigrationNames.add(migrationId)
}
return resolved
}
async function runApply(options = parseArgs(), env = process.env) {
validateRequiredMigrationCatalog()
const databaseUrl = loadDatabaseUrl(ROOT_DIR, env)
const actualDatabaseIdentity = getDatabaseIdentity(databaseUrl)
const backupFingerprint = getBackupFingerprint(env)
const pool = new Pool({ connectionString: databaseUrl, connectionTimeoutMillis: 5000 })
const client = await pool.connect()
try {
const before = await getRequiredMigrationReadiness(client)
const plan = buildFoundationRequiredMigrationApplyPlan(before, {
...options,
actualDatabaseIdentity,
backupFingerprint,
})
const reportBase = {
...plan,
database: describeDatabaseUrl(databaseUrl),
checkedAt: new Date().toISOString(),
applied: null,
after: null,
}
if (!plan.readyToApply) {
return reportBase
}
await client.query('BEGIN')
let committed = false
try {
const targetMigrations = options.phase === 'bootstrap-repair'
? getMigrationsByPaths(plan.bootstrapRepairCandidates.map((candidate) => candidate.path))
: REQUIRED_MIGRATIONS
const appliedMigrationPaths = await applyMissingMigrationFiles(client, targetMigrations)
const after = await getRequiredMigrationReadiness(client)
if (options.phase === 'bootstrap-repair') {
const unreadyAppliedPaths = getUnreadyPathsAfterApply(after, appliedMigrationPaths)
if (unreadyAppliedPaths.length > 0) {
throw new Error(
`bootstrap-repair 应用后仍有目标迁移未就绪,已回滚本次事务:${unreadyAppliedPaths.join(', ')}`,
)
}
} else if (!after.fullyReady) {
throw new Error('治理迁移应用后仍未达到 fullyReady,已回滚本次事务。')
}
await client.query('COMMIT')
committed = true
const resolvedMigrationHistory = await resolveAppliedMigrationHistory({
client,
migrationPaths: appliedMigrationPaths,
env: { ...env, DATABASE_URL: databaseUrl },
})
const afterResolve = resolvedMigrationHistory.length > 0
? await getRequiredMigrationReadiness(client)
: after
return {
...reportBase,
applied: {
phase: options.phase,
migrationCount: appliedMigrationPaths.length,
...classifyAppliedMigrationPaths(appliedMigrationPaths),
migrationPaths: appliedMigrationPaths,
},
resolvedMigrationHistory,
after: afterResolve,
}
} catch (error) {
if (!committed) {
await client.query('ROLLBACK')
}
throw error
}
} finally {
client.release()
await pool.end()
}
}
async function main() {
const options = parseArgs()
const report = await runApply(options)
writeJsonReport(options.outputPath, report)
console.log(JSON.stringify(report, null, 2))
if (options.apply && !report.readyToApply) {
process.exit(1)
}
}
if (require.main === module) {
main().catch((error) => {
console.error(error instanceof Error ? error.message : String(error))
process.exit(1)
})
}
module.exports = {
CONFIRM_TOKEN,
getBootstrapRepairPlan,
buildFoundationRequiredMigrationApplyPlan,
getBackupFingerprint,
getFailedDataChecks,
getMigrationsByPaths,
getUnreadyPathsAfterApply,
getAppliedPrismaMigrationNames,
getMigrationIdFromPath,
migrationHistoryReady,
normalizeSha256,
parseArgs,
resolveAppliedMigrationHistory,
runApply,
runPrismaMigrateResolveApplied,
writeJsonReport,
}