强化升级回滚和外部备份确认
This commit is contained in:
WuYongJie
parent
3cdfaa89a9
commit
b6246044b8
@ -22,6 +22,7 @@ param(
|
||||
[int]$RemoteHealthTimeoutSec = 120,
|
||||
[switch]$NoRemoteDbBackup,
|
||||
[switch]$NoRemoteObjectStorageBackup,
|
||||
[switch]$ExternalDataBackupConfirmed,
|
||||
[string]$RemoteBackupHelperImage = "busybox:1.36",
|
||||
[switch]$NoRollback,
|
||||
|
||||
@ -134,6 +135,7 @@ function Show-Help {
|
||||
Write-Host " -NoUpdateRemoteEnv 只让本次 compose 命令临时生效，不持久写入远程 .env"
|
||||
Write-Host " -NoRemoteDbBackup 跳过远程 PostgreSQL 备份；默认会在升级前尝试备份 Compose 管理的数据库"
|
||||
Write-Host " -NoRemoteObjectStorageBackup 跳过远程 MinIO 数据卷备份；默认会在升级前尝试备份 Compose 管理的素材文件"
|
||||
Write-Host " -ExternalDataBackupConfirmed 使用外部 PostgreSQL / S3 时，确认客户侧备份已完成后才允许继续升级"
|
||||
Write-Host " -RemoteBackupHelperImage 远程备份对象存储卷时使用的 helper 镜像，默认 busybox:1.36"
|
||||
Write-Host " -NoRollback 远程部署失败时不自动恢复旧 APP_IMAGE；默认启用失败回滚"
|
||||
Write-Host " -RemoteHealthTimeoutSec 远程容器健康等待秒数，默认 120"
|
||||
@ -518,6 +520,7 @@ function New-RemoteDeployScript {
|
||||
$quotedRemoteBackupHelperImage = ConvertTo-ShellSingleQuoted -Value $RemoteBackupHelperImage
|
||||
$remoteBackupEnabled = if ($NoRemoteDbBackup) { "0" } else { "1" }
|
||||
$remoteObjectBackupEnabled = if ($NoRemoteObjectStorageBackup) { "0" } else { "1" }
|
||||
$externalDataBackupConfirmed = if ($ExternalDataBackupConfirmed) { "1" } else { "0" }
|
||||
$rollbackEnabled = if ($NoRollback) { "0" } else { "1" }
|
||||
|
||||
$lines = New-Object System.Collections.Generic.List[string]
|
||||
@ -535,6 +538,7 @@ function New-RemoteDeployScript {
|
||||
$lines.Add("remote_health_timeout=$RemoteHealthTimeoutSec")
|
||||
$lines.Add("remote_db_backup_enabled=$remoteBackupEnabled")
|
||||
$lines.Add("remote_object_backup_enabled=$remoteObjectBackupEnabled")
|
||||
$lines.Add("external_data_backup_confirmed=$externalDataBackupConfirmed")
|
||||
$lines.Add("remote_backup_helper_image=$quotedRemoteBackupHelperImage")
|
||||
$lines.Add("rollback_enabled=$rollbackEnabled")
|
||||
$lines.Add("export APP_IMAGE=$quotedRegistryImage")
|
||||
@ -579,6 +583,26 @@ read_env_value() {
|
||||
' .env | tail -n 1
|
||||
}
|
||||
|
||||
shell_single_quote() {
|
||||
printf "'"
|
||||
printf '%s' "$1" | sed "s/'/'\\\\''/g"
|
||||
printf "'"
|
||||
}
|
||||
|
||||
write_deploy_context() {
|
||||
context_file="./backups/LAST_DEPLOY_CONTEXT.env"
|
||||
{
|
||||
printf 'compose_file=%s\n' "$(shell_single_quote "$compose_file")"
|
||||
printf 'compose_service=%s\n' "$(shell_single_quote "$compose_service")"
|
||||
printf 'backup_dir=%s\n' "$(shell_single_quote "$backup_dir")"
|
||||
printf 'env_backup=%s\n' "$(shell_single_quote "$env_backup")"
|
||||
printf 'old_container_image=%s\n' "$(shell_single_quote "$old_container_image")"
|
||||
printf 'old_env_app_image=%s\n' "$(shell_single_quote "$old_env_app_image")"
|
||||
printf 'rollback_enabled=%s\n' "$(shell_single_quote "$rollback_enabled")"
|
||||
} > "$context_file"
|
||||
echo "回滚上下文：$context_file"
|
||||
}
|
||||
|
||||
is_external_database() {
|
||||
database_url="$(read_env_value DATABASE_URL || true)"
|
||||
if [ -z "$database_url" ]; then
|
||||
@ -675,6 +699,7 @@ elif [ -n "$old_env_app_image" ]; then
|
||||
else
|
||||
echo "当前未发现已运行的 app 镜像，按首次部署处理"
|
||||
fi
|
||||
write_deploy_context
|
||||
|
||||
rollback() {
|
||||
code="$?"
|
||||
@ -732,6 +757,10 @@ backup_postgres() {
|
||||
if is_external_database; then
|
||||
echo "检测到 DATABASE_URL 指向外部数据库，跳过 Compose 内置 postgres 备份"
|
||||
printf '%s\n' "检测到 DATABASE_URL 指向外部数据库。本脚本不会自动备份外部数据库，请按客户现场数据库方案单独备份。" > "$backup_dir/EXTERNAL_DATABASE_BACKUP_REQUIRED.txt"
|
||||
if [ "$external_data_backup_confirmed" != "1" ]; then
|
||||
echo "外部数据库备份未确认，已中止升级。确认客户侧数据库备份完成后，重新执行并加 -ExternalDataBackupConfirmed。"
|
||||
exit 33
|
||||
fi
|
||||
return
|
||||
fi
|
||||
|
||||
@ -758,6 +787,10 @@ backup_minio_volume() {
|
||||
if is_external_object_storage; then
|
||||
echo "检测到 S3_ENDPOINT 指向外部对象存储，跳过 Compose 内置 MinIO 卷备份"
|
||||
printf '%s\n' "检测到 S3_ENDPOINT 指向外部 S3 / MinIO。本脚本不会自动备份外部对象存储，请按客户现场方案单独备份素材对象。" > "$backup_dir/EXTERNAL_OBJECT_STORAGE_BACKUP_REQUIRED.txt"
|
||||
if [ "$external_data_backup_confirmed" != "1" ]; then
|
||||
echo "外部对象存储备份未确认，已中止升级。确认客户侧素材对象备份完成后，重新执行并加 -ExternalDataBackupConfirmed。"
|
||||
exit 34
|
||||
fi
|
||||
return
|
||||
fi
|
||||
|
||||
@ -870,6 +903,50 @@ write_restore_plan
|
||||
return ($lines -join "`n")
|
||||
}
|
||||
|
||||
function New-RemoteRollbackScript {
|
||||
param([string]$Reason)
|
||||
|
||||
$quotedRemotePath = ConvertTo-ShellSingleQuoted -Value $RemotePath
|
||||
$quotedReason = ConvertTo-ShellSingleQuoted -Value $Reason
|
||||
|
||||
return @"
|
||||
set -Eeuo pipefail
|
||||
cd $quotedRemotePath
|
||||
reason=$quotedReason
|
||||
context_file="./backups/LAST_DEPLOY_CONTEXT.env"
|
||||
echo "[ROLLBACK] 本地最终健康检查失败，准备远程回滚：`$reason"
|
||||
if [ ! -f "`$context_file" ]; then
|
||||
echo "[ROLLBACK] 找不到回滚上下文：`$context_file"
|
||||
exit 50
|
||||
fi
|
||||
. "`$context_file"
|
||||
if [ "`${rollback_enabled:-1}" != "1" ]; then
|
||||
echo "[ROLLBACK] 当前部署已关闭自动回滚，保留现场供人工排查"
|
||||
exit 51
|
||||
fi
|
||||
if [ -n "`${env_backup:-}" ] && [ -f "`$env_backup" ]; then
|
||||
cp "`$env_backup" .env
|
||||
echo "[ROLLBACK] 已恢复升级前 .env"
|
||||
else
|
||||
echo "[ROLLBACK] 找不到升级前 .env 备份：`${env_backup:-}"
|
||||
exit 52
|
||||
fi
|
||||
rollback_image="`${old_container_image:-}"
|
||||
if [ -z "`$rollback_image" ]; then
|
||||
rollback_image="`${old_env_app_image:-}"
|
||||
fi
|
||||
if [ -n "`$rollback_image" ]; then
|
||||
export APP_IMAGE="`$rollback_image"
|
||||
docker compose -f "`${compose_file:-docker-compose.prod.yml}" up -d --no-deps --pull never --force-recreate "`${compose_service:-app}"
|
||||
docker compose -f "`${compose_file:-docker-compose.prod.yml}" ps
|
||||
echo "[ROLLBACK] 已恢复旧镜像：`$rollback_image"
|
||||
else
|
||||
echo "[ROLLBACK] 未找到旧镜像，无法自动恢复应用容器"
|
||||
exit 53
|
||||
fi
|
||||
"@
|
||||
}
|
||||
|
||||
function Test-RemoteHealth {
|
||||
Assert-RemoteServerConfigured
|
||||
|
||||
@ -1002,7 +1079,23 @@ function Invoke-DeployApp {
|
||||
if (-not $SkipRemote) {
|
||||
$remoteScript = New-RemoteDeployScript -RegistryImage $registryImage -ResolvedNextAuthUrl $resolvedNextAuthUrl -ResolvedAppPublicUrl $resolvedAppPublicUrl
|
||||
Invoke-RemoteCommand -Command $remoteScript
|
||||
Test-RemoteHealth
|
||||
try {
|
||||
Test-RemoteHealth
|
||||
} catch {
|
||||
$healthError = $_.Exception.Message
|
||||
if ($NoRollback) {
|
||||
Write-Warn "应用健康检查失败，但已关闭自动回滚：$healthError"
|
||||
} else {
|
||||
Write-Warn "应用健康检查失败，尝试远程回滚：$healthError"
|
||||
$rollbackScript = New-RemoteRollbackScript -Reason $healthError
|
||||
try {
|
||||
Invoke-RemoteCommand -Command $rollbackScript -NoRegistryPasswordInput
|
||||
} catch {
|
||||
Write-Warn "远程回滚命令执行失败：$($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
throw
|
||||
}
|
||||
}
|
||||
|
||||
Write-Success "发布流程完成"
|
||||
|
||||
@ -98,11 +98,13 @@ MAX_UPLOAD_FIELD_SIZE_BYTES=2097152
|
||||
|
||||
- 内置 PostgreSQL：`DATABASE_URL` 留空或不写，由 `docker-compose.prod.yml` 按 `POSTGRES_USER`、`POSTGRES_PASSWORD`、`POSTGRES_DB` 自动拼接到 `postgres` 服务。
|
||||
- 外部 PostgreSQL：才填写真实 `DATABASE_URL`，例如 `postgresql://flnxi:<password>@192.168.1.10:5432/flnxi?sslmode=disable`。
|
||||
- 商业首版更推荐内置 PostgreSQL，升级脚本可以自动备份和回滚。若客户坚持使用外部 PostgreSQL，升级前必须先完成客户侧数据库备份；执行 `deploy-registry.ps1` 时需显式加 `-ExternalDataBackupConfirmed`，否则脚本会中止升级。
|
||||
|
||||
对象存储也有两种模式：
|
||||
|
||||
- 内置 MinIO：`S3_ENDPOINT=http://minio:9000`，`S3_ACCESS_KEY_ID` / `S3_SECRET_ACCESS_KEY` 可不填，Compose 会使用 `MINIO_ROOT_USER` / `MINIO_ROOT_PASSWORD`。`S3_PUBLIC_DOMAIN` 必须写成用户浏览器能访问的地址，例如 `http://192.168.1.20:9000/flnxi-materials`。
|
||||
- 外部 S3 / 外部 MinIO：需要填写真实 `S3_ENDPOINT`、`S3_ACCESS_KEY_ID`、`S3_SECRET_ACCESS_KEY`、`S3_BUCKET`、`S3_PUBLIC_DOMAIN`，并确认 bucket 已存在或当前账号有创建和读写权限。
|
||||
- 商业首版更推荐内置 MinIO，升级脚本可以自动备份素材对象。若使用外部 S3 / MinIO，升级前必须先完成客户侧 bucket / 对象备份；执行 `deploy-registry.ps1` 时同样需显式加 `-ExternalDataBackupConfirmed`。
|
||||
|
||||
NAS 或 Linux 服务器上建议提前准备挂载目录：
|
||||
|
||||
|
||||
@ -10,6 +10,7 @@ describe('deploy-registry.ps1', () => {
|
||||
it('远程部署默认包含升级前数据库、对象存储和 .env 备份', () => {
|
||||
expect(script).toContain('[switch]$NoRemoteDbBackup')
|
||||
expect(script).toContain('[switch]$NoRemoteObjectStorageBackup')
|
||||
expect(script).toContain('[switch]$ExternalDataBackupConfirmed')
|
||||
expect(script).toContain('[string]$RemoteBackupHelperImage = "busybox:1.36"')
|
||||
expect(script).toContain('env_backup="$backup_dir/.env.before-upgrade"')
|
||||
expect(script).toContain('enter_maintenance_for_backup')
|
||||
@ -26,6 +27,9 @@ describe('deploy-registry.ps1', () => {
|
||||
expect(script).toContain('is_external_object_storage()')
|
||||
expect(script).toContain('EXTERNAL_DATABASE_BACKUP_REQUIRED.txt')
|
||||
expect(script).toContain('EXTERNAL_OBJECT_STORAGE_BACKUP_REQUIRED.txt')
|
||||
expect(script).toContain('external_data_backup_confirmed=$externalDataBackupConfirmed')
|
||||
expect(script).toContain('外部数据库备份未确认，已中止升级')
|
||||
expect(script).toContain('外部对象存储备份未确认，已中止升级')
|
||||
})
|
||||
|
||||
it('远程部署失败时默认会恢复旧 APP_IMAGE 或旧容器镜像', () => {
|
||||
@ -33,11 +37,23 @@ describe('deploy-registry.ps1', () => {
|
||||
expect(script).toContain('rollback()')
|
||||
expect(script).toContain('trap rollback ERR')
|
||||
expect(script.indexOf('trap rollback ERR')).toBeLessThan(script.indexOf('upsert_env APP_IMAGE'))
|
||||
expect(script).toContain('write_deploy_context')
|
||||
expect(script).toContain('LAST_DEPLOY_CONTEXT.env')
|
||||
expect(script).toContain('rollback_image="$old_container_image"')
|
||||
expect(script).toContain('docker compose -f "$compose_file" up -d --no-deps --pull never --force-recreate "$compose_service" || true')
|
||||
expect(script).toContain('自动回滚只恢复应用镜像和 .env，不自动恢复数据库')
|
||||
})
|
||||
|
||||
it('本地最终健康检查失败时会触发远程回滚', () => {
|
||||
expect(script).toContain('function New-RemoteRollbackScript')
|
||||
expect(script).toContain('应用健康检查失败，尝试远程回滚')
|
||||
expect(script).toContain('New-RemoteRollbackScript -Reason $healthError')
|
||||
expect(script).toContain('Invoke-RemoteCommand -Command $rollbackScript -NoRegistryPasswordInput')
|
||||
expect(script).toContain('本地最终健康检查失败，准备远程回滚')
|
||||
expect(script).toContain('cp "$env_backup" .env')
|
||||
expect(script).toContain('docker compose -f "`${compose_file:-docker-compose.prod.yml}" up -d --no-deps --pull never --force-recreate "`${compose_service:-app}"')
|
||||
})
|
||||
|
||||
it('部署后校验运行镜像、版本文件和数据库迁移状态', () => {
|
||||
expect(script).toContain('verify_running_image')
|
||||
expect(script).toContain('actual_image="$(docker inspect "$container_id" --format')
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user