3744 lines
154 KiB
PowerShell
3744 lines
154 KiB
PowerShell
#Requires -Version 5.1
|
||
|
||
<#
|
||
.SYNOPSIS
|
||
Git-based incremental deployment for gongxiangp2.
|
||
|
||
.DESCRIPTION
|
||
Source of truth is the Git repository. The build server fetches only Git
|
||
deltas, builds a Docker image with cache, pushes commit-tagged images to the
|
||
internal registry, and the production server pulls the selected image.
|
||
|
||
This script intentionally does not upload a source tarball from the local
|
||
workstation and does not build on the production server.
|
||
#>
|
||
|
||
param(
|
||
[ValidateSet('production', 'shadow')]
|
||
[string]$Target = 'production',
|
||
|
||
[string]$RepoUrl = 'http://git.1080p.hk/WuYongJie/gongxiangp2.git',
|
||
[string]$Branch = 'main',
|
||
[string]$Commit = '',
|
||
|
||
[string]$GitUsername = $env:GIT_DEPLOY_USERNAME,
|
||
[string]$GitPassword = $env:GIT_DEPLOY_PASSWORD,
|
||
[string]$GitAuthHeader = $env:GIT_AUTH_HEADER,
|
||
|
||
[string]$BuildServerIP = '192.168.40.206',
|
||
[string]$BuildServerUser = 'root',
|
||
[string]$BuildServerPassword = $env:BUILD_SERVER_PASSWORD,
|
||
[string]$BuildServerProxyCommand = $env:BUILD_SERVER_PROXY_COMMAND,
|
||
[string]$BuildServerHostKey = $env:BUILD_SERVER_HOST_KEY,
|
||
[string]$BuildServerKnownHostsFile = $env:BUILD_SERVER_KNOWN_HOSTS_FILE,
|
||
[string]$RemoteBuildDir = '/root/youyidian-3d-git-build',
|
||
|
||
[string]$ProdServerIP = '192.168.58.20',
|
||
[string]$ProdServerUser = 'root',
|
||
[string]$ProdServerHostKey = $env:PROD_SERVER_HOST_KEY,
|
||
[string]$ProdServerKnownHostsFile = $env:PROD_SERVER_KNOWN_HOSTS_FILE,
|
||
|
||
[int]$RegistryPort = 5000,
|
||
[string]$ImageName = 'youyidian-3d-material',
|
||
|
||
[string]$UploadsMount = $env:GONGXIANGP2_UPLOADS_MOUNT,
|
||
|
||
[string]$MigrationAuditFile = $env:GONGXIANGP2_MIGRATION_AUDIT_FILE,
|
||
[string]$MigrationAuditSha256 = $env:GONGXIANGP2_MIGRATION_AUDIT_SHA256,
|
||
[string]$MigrationAuditSourceCommit = $env:GONGXIANGP2_MIGRATION_AUDIT_SOURCE_COMMIT,
|
||
# Normal application releases do not run the database migration workflow.
|
||
# This explicit switch is only for a migration batch that already has an
|
||
# independently created and verified audit receipt.
|
||
[switch]$RequireMigrationAudit,
|
||
|
||
[string]$DownloadTokenSecret = $env:DOWNLOAD_TOKEN_SECRET,
|
||
|
||
[switch]$DryRun,
|
||
[switch]$NoCache,
|
||
[switch]$SkipBuild,
|
||
[switch]$SkipPush,
|
||
# Deploy a previously built, immutable exact-SHA image only after the
|
||
# build server and registry both prove its identity. This is intended for
|
||
# a script-only rollout or a retry; it never falls back to an image tag
|
||
# that has not been verified for the requested commit.
|
||
[switch]$ReuseVerifiedImage,
|
||
[switch]$SkipDeploy,
|
||
[switch]$KeepOldImages,
|
||
[switch]$AggressiveBuildCacheCleanup,
|
||
[switch]$NoRollback,
|
||
|
||
# Production returns to the original single-slot replacement workflow:
|
||
# stop the active slot, start the same slot with the new exact image, and
|
||
# roll it back to the previous image if health fails. Keep the old
|
||
# blue/green implementation below only as historical code while existing
|
||
# runtime artefacts are gradually retired; it is not a release path.
|
||
[ValidateSet('direct')]
|
||
[string]$DeploymentStrategy = 'direct'
|
||
)
|
||
|
||
$ErrorActionPreference = 'Stop'
|
||
|
||
# Shadow still shares production uploads, transcode runtime markers, and Redis,
|
||
# while its separate database lease cannot serialize FFmpeg across environments.
|
||
# Keep this entry fail-closed until a dedicated shadow topology is verified.
|
||
if ($Target -eq 'shadow') {
|
||
throw 'Target=shadow is disabled fail-closed: use the dedicated shadow deployment entry after uploads/runtime/Redis isolation is configured and verified.'
|
||
}
|
||
|
||
if ($ReuseVerifiedImage -and ($SkipBuild -or $SkipPush -or $SkipDeploy -or $NoCache)) {
|
||
throw '-ReuseVerifiedImage cannot be combined with -SkipBuild, -SkipPush, -SkipDeploy, or -NoCache.'
|
||
}
|
||
|
||
if (-not $SkipDeploy -and ($SkipBuild -or $SkipPush) -and -not $ReuseVerifiedImage) {
|
||
throw 'Production deployment requires a freshly built and pushed exact-SHA image; -SkipBuild and -SkipPush are only allowed together with -SkipDeploy.'
|
||
}
|
||
|
||
if (-not $DryRun -and $Commit -cnotmatch '^[0-9a-f]{40}$') {
|
||
throw 'HTTP Git transport requires -Commit with the exact 40-character lowercase SHA selected on the trusted workstation.'
|
||
}
|
||
|
||
if ($RequireMigrationAudit -and [string]::IsNullOrWhiteSpace($MigrationAuditSourceCommit)) {
|
||
$MigrationAuditSourceCommit = $Commit
|
||
}
|
||
|
||
if ($RequireMigrationAudit -and -not $DryRun -and -not $SkipDeploy) {
|
||
if ($MigrationAuditFile -cnotmatch '^/root/gongxiangp2-migration-audit/[A-Za-z0-9._-]+/applied-[0-9]{8}T[0-9]{6}Z\.json$') {
|
||
throw 'Production deployment requires the immutable apply audit file under /root/gongxiangp2-migration-audit/<batch>/applied-<UTC>.json.'
|
||
}
|
||
if ($MigrationAuditSha256 -cnotmatch '^[0-9a-f]{64}$') {
|
||
throw 'Production deployment requires the exact lowercase SHA-256 of the immutable apply audit file.'
|
||
}
|
||
if ($MigrationAuditSourceCommit -cnotmatch '^[0-9a-f]{40}$') {
|
||
throw 'Production deployment requires the exact lowercase source commit recorded by the approved migration audit.'
|
||
}
|
||
}
|
||
|
||
$trustedRemoteBuildDir = '/root/youyidian-3d-git-build'
|
||
if ($RemoteBuildDir -cne $trustedRemoteBuildDir) {
|
||
throw "RemoteBuildDir is fixed to the dedicated build worktree $trustedRemoteBuildDir; arbitrary cleanup targets are forbidden."
|
||
}
|
||
|
||
if (
|
||
-not [string]::IsNullOrWhiteSpace($GitUsername) -or
|
||
-not [string]::IsNullOrWhiteSpace($GitPassword) -or
|
||
-not [string]::IsNullOrWhiteSpace($GitAuthHeader)
|
||
) {
|
||
throw 'Inline Git credentials are forbidden. Configure a read-only credential helper on the workstation and build server instead.'
|
||
}
|
||
|
||
$trustedRepoUrl = 'http://git.1080p.hk/WuYongJie/gongxiangp2.git'
|
||
$localDryRunRepository = $false
|
||
if ($DryRun -and [System.IO.Path]::IsPathRooted($RepoUrl)) {
|
||
$localDryRunRepository = Test-Path -LiteralPath $RepoUrl -PathType Container
|
||
}
|
||
if (-not $localDryRunRepository -and $RepoUrl -cne $trustedRepoUrl) {
|
||
throw "Production Git repository must be the exact pinned HTTP repository $trustedRepoUrl. URL credentials and arbitrary repositories are forbidden."
|
||
}
|
||
|
||
# Production evidence from 2026-08-02 identifies the dedicated ZFS dataset as
|
||
# the current source of truth. The older application-directory path remains an
|
||
# explicit, controlled compatibility choice only; never infer a path from an
|
||
# existing container because that would preserve a bad mount automatically.
|
||
$defaultUploadsMount = '/mnt/PoolOne/youyidian-uploads'
|
||
$trustedUploadsMounts = @(
|
||
$defaultUploadsMount,
|
||
'/mnt/PoolOne/apps/youyidian-3d/uploads'
|
||
)
|
||
if ([string]::IsNullOrWhiteSpace($UploadsMount)) {
|
||
$UploadsMount = $defaultUploadsMount
|
||
}
|
||
if ($trustedUploadsMounts -cnotcontains $UploadsMount) {
|
||
throw "UploadsMount must be one of the explicitly trusted production paths: $($trustedUploadsMounts -join ', ')"
|
||
}
|
||
|
||
$targetConfig = @{
|
||
production = @{
|
||
Port = 4320
|
||
ContainerName = 'youyidian-3d-material'
|
||
EnvFile = '.env'
|
||
PublicAppUrl = 'https://sc.jiansun.vip'
|
||
CorsAllowedOrigins = 'https://sc.jiansun.vip,http://192.168.58.20:4320'
|
||
DingtalkCallbackOrigin = 'https://sc.jiansun.vip'
|
||
DingtalkInternalReturnOrigins = 'http://192.168.58.20:4320'
|
||
S3PublicDomain = 'https://s3.jiansun.vip/gongxiangpan'
|
||
}
|
||
shadow = @{
|
||
Port = 5320
|
||
ContainerName = 'youyidian-3d-shadow'
|
||
EnvFile = '.env.shadow'
|
||
PublicAppUrl = 'http://192.168.58.20:5320'
|
||
CorsAllowedOrigins = 'http://192.168.58.20:5320'
|
||
DingtalkCallbackOrigin = 'http://192.168.58.20:5320'
|
||
DingtalkInternalReturnOrigins = 'http://192.168.58.20:5320'
|
||
S3PublicDomain = 'https://s3.jiansun.vip/gongxiangpan'
|
||
}
|
||
}
|
||
|
||
function Write-Info {
|
||
param([string]$Message)
|
||
Write-Host "[INFO] $Message" -ForegroundColor Cyan
|
||
}
|
||
|
||
function Write-Success {
|
||
param([string]$Message)
|
||
Write-Host "[OK] $Message" -ForegroundColor Green
|
||
}
|
||
|
||
function Write-Warning-Custom {
|
||
param([string]$Message)
|
||
Write-Host "[WARN] $Message" -ForegroundColor Yellow
|
||
}
|
||
|
||
function Write-Error-Custom {
|
||
param([string]$Message)
|
||
Write-Host "[ERR] $Message" -ForegroundColor Red
|
||
}
|
||
|
||
function Write-Step {
|
||
param([string]$Message)
|
||
Write-Host ""
|
||
Write-Host "==================================================" -ForegroundColor Cyan
|
||
Write-Host $Message -ForegroundColor Magenta
|
||
Write-Host "==================================================" -ForegroundColor Cyan
|
||
}
|
||
|
||
function Invoke-PreflightGit {
|
||
param(
|
||
[Parameter(Mandatory = $true)][string]$GitExecutable,
|
||
[Parameter(Mandatory = $true)][string[]]$Arguments,
|
||
[switch]$CaptureOutput
|
||
)
|
||
|
||
$previousErrorAction = $ErrorActionPreference
|
||
$previousGitPrompt = $env:GIT_TERMINAL_PROMPT
|
||
$ErrorActionPreference = 'Continue'
|
||
$env:GIT_TERMINAL_PROMPT = '0'
|
||
try {
|
||
$output = & $GitExecutable `
|
||
-c 'http.lowSpeedLimit=1' `
|
||
-c 'http.lowSpeedTime=30' `
|
||
@Arguments 2>&1
|
||
$exitCode = $LASTEXITCODE
|
||
}
|
||
finally {
|
||
$ErrorActionPreference = $previousErrorAction
|
||
$env:GIT_TERMINAL_PROMPT = $previousGitPrompt
|
||
}
|
||
|
||
if ($exitCode -ne 0) {
|
||
$detail = Protect-LogLine -Line ((@($output) | ForEach-Object { $_.ToString() }) -join [Environment]::NewLine)
|
||
if ([string]::IsNullOrWhiteSpace($detail)) {
|
||
$detail = 'no diagnostic output'
|
||
}
|
||
throw "Git target preflight failed with exit code ${exitCode}: $detail"
|
||
}
|
||
|
||
if ($CaptureOutput) {
|
||
return ((@($output) | ForEach-Object { $_.ToString() }) -join "`n").Trim()
|
||
}
|
||
}
|
||
|
||
function Invoke-LegacyStorageBoundaryPreflight {
|
||
param(
|
||
[Parameter(Mandatory = $true)][string]$RepositoryUrl,
|
||
[Parameter(Mandatory = $true)][string]$TargetBranch,
|
||
[AllowEmptyString()][string]$RequestedCommit
|
||
)
|
||
|
||
$scannerPath = Join-Path $PSScriptRoot 'check-legacy-storage-boundaries.mjs'
|
||
if (-not (Test-Path -LiteralPath $scannerPath -PathType Leaf)) {
|
||
throw "Legacy storage boundary scanner is missing: $scannerPath"
|
||
}
|
||
|
||
$nodeCommand = Get-Command node -ErrorAction SilentlyContinue
|
||
if (-not $nodeCommand) {
|
||
throw 'Node.js is required for the legacy storage boundary preflight.'
|
||
}
|
||
|
||
$gitCommand = Get-Command git -ErrorAction SilentlyContinue
|
||
if (-not $gitCommand) {
|
||
throw 'Git is required to resolve and inspect the exact deployment target.'
|
||
}
|
||
if ([string]::IsNullOrWhiteSpace($RepositoryUrl) -or $RepositoryUrl.StartsWith('-')) {
|
||
throw 'Repository URL is invalid for exact-target preflight.'
|
||
}
|
||
|
||
$tempBase = [System.IO.Path]::GetFullPath([System.IO.Path]::GetTempPath()).TrimEnd([char[]]@('\', '/'))
|
||
$tempRoot = Join-Path $tempBase ("gongxiangp2-deploy-preflight-{0}" -f [Guid]::NewGuid().ToString('N'))
|
||
$bareRepository = Join-Path $tempRoot 'target.git'
|
||
$targetCheckout = Join-Path $tempRoot 'target-tree'
|
||
|
||
try {
|
||
New-Item -ItemType Directory -Path $tempRoot -Force | Out-Null
|
||
Invoke-PreflightGit -GitExecutable $gitCommand.Source -Arguments @(
|
||
'check-ref-format', '--branch', $TargetBranch
|
||
)
|
||
|
||
$remoteHeadOutput = Invoke-PreflightGit -GitExecutable $gitCommand.Source -CaptureOutput -Arguments @(
|
||
'ls-remote', '--heads', $RepositoryUrl, "refs/heads/$TargetBranch"
|
||
)
|
||
$remoteHeadLines = @($remoteHeadOutput -split "`r?`n" | Where-Object { -not [string]::IsNullOrWhiteSpace($_) })
|
||
if ($remoteHeadLines.Count -ne 1) {
|
||
throw "Remote branch $TargetBranch did not resolve to exactly one ref."
|
||
}
|
||
$remoteHeadParts = @($remoteHeadLines[0].Trim() -split '\s+')
|
||
if (
|
||
$remoteHeadParts.Count -ne 2 -or
|
||
$remoteHeadParts[0] -cnotmatch '^[0-9a-f]{40}$' -or
|
||
$remoteHeadParts[1] -cne "refs/heads/$TargetBranch"
|
||
) {
|
||
throw 'Remote branch did not resolve to one exact commit SHA.'
|
||
}
|
||
$branchCommit = $remoteHeadParts[0]
|
||
|
||
$targetCommit = $branchCommit
|
||
if (-not [string]::IsNullOrWhiteSpace($RequestedCommit)) {
|
||
if ($RequestedCommit -cnotmatch '^[0-9a-f]{40}$') {
|
||
throw 'Commit override must be the exact 40-character lowercase Git commit SHA.'
|
||
}
|
||
$targetCommit = $RequestedCommit
|
||
}
|
||
|
||
$localRepository = $null
|
||
try {
|
||
$localRepositoryCandidate = Invoke-PreflightGit -GitExecutable $gitCommand.Source -CaptureOutput -Arguments @(
|
||
'-C', $PSScriptRoot, 'rev-parse', '--show-toplevel'
|
||
)
|
||
if (Test-Path -LiteralPath $localRepositoryCandidate -PathType Container) {
|
||
$localRepository = [System.IO.Path]::GetFullPath($localRepositoryCandidate)
|
||
}
|
||
}
|
||
catch {
|
||
$localRepository = $null
|
||
}
|
||
|
||
$useLocalObjects = $false
|
||
if ($localRepository) {
|
||
try {
|
||
$resolvedBranchCommit = Invoke-PreflightGit -GitExecutable $gitCommand.Source -CaptureOutput -Arguments @(
|
||
'-C', $localRepository, 'rev-parse', '--verify', "${branchCommit}^{commit}"
|
||
)
|
||
$resolvedTargetCommit = Invoke-PreflightGit -GitExecutable $gitCommand.Source -CaptureOutput -Arguments @(
|
||
'-C', $localRepository, 'rev-parse', '--verify', "${targetCommit}^{commit}"
|
||
)
|
||
if ($resolvedBranchCommit -cne $branchCommit -or $resolvedTargetCommit -cne $targetCommit) {
|
||
throw 'Local Git objects did not resolve to the exact remote commits.'
|
||
}
|
||
Invoke-PreflightGit -GitExecutable $gitCommand.Source -Arguments @(
|
||
'-C', $localRepository, 'merge-base', '--is-ancestor', $targetCommit, $branchCommit
|
||
)
|
||
$useLocalObjects = $true
|
||
}
|
||
catch {
|
||
$useLocalObjects = $false
|
||
}
|
||
}
|
||
|
||
if ($useLocalObjects) {
|
||
Invoke-PreflightGit -GitExecutable $gitCommand.Source -Arguments @(
|
||
'clone', '--no-checkout', '--shared', $localRepository, $targetCheckout
|
||
)
|
||
Invoke-PreflightGit -GitExecutable $gitCommand.Source -Arguments @(
|
||
'-C', $targetCheckout, 'checkout', '--detach', '--force', $targetCommit
|
||
)
|
||
}
|
||
else {
|
||
Invoke-PreflightGit -GitExecutable $gitCommand.Source -Arguments @('init', '--bare', $bareRepository)
|
||
Invoke-PreflightGit -GitExecutable $gitCommand.Source -Arguments @(
|
||
"--git-dir=$bareRepository",
|
||
'fetch', '--no-tags', '--force',
|
||
$RepositoryUrl,
|
||
"+refs/heads/${TargetBranch}:refs/preflight/branch"
|
||
)
|
||
$fetchedBranchCommit = Invoke-PreflightGit -GitExecutable $gitCommand.Source -CaptureOutput -Arguments @(
|
||
"--git-dir=$bareRepository", 'rev-parse', '--verify', 'refs/preflight/branch^{commit}'
|
||
)
|
||
if ($fetchedBranchCommit -cne $branchCommit) {
|
||
throw 'Fetched branch changed after the exact remote ref was resolved; retry the deployment.'
|
||
}
|
||
$resolvedTargetCommit = Invoke-PreflightGit -GitExecutable $gitCommand.Source -CaptureOutput -Arguments @(
|
||
"--git-dir=$bareRepository", 'rev-parse', '--verify', "${targetCommit}^{commit}"
|
||
)
|
||
if ($resolvedTargetCommit -cne $targetCommit) {
|
||
throw 'Commit override did not resolve to one exact commit SHA.'
|
||
}
|
||
try {
|
||
Invoke-PreflightGit -GitExecutable $gitCommand.Source -Arguments @(
|
||
"--git-dir=$bareRepository", 'merge-base', '--is-ancestor', $targetCommit, $branchCommit
|
||
)
|
||
}
|
||
catch {
|
||
throw "Commit override $targetCommit is not reachable from fetched branch $TargetBranch."
|
||
}
|
||
Invoke-PreflightGit -GitExecutable $gitCommand.Source -Arguments @(
|
||
"--git-dir=$bareRepository", 'worktree', 'add', '--detach', $targetCheckout, $targetCommit
|
||
)
|
||
}
|
||
|
||
Push-Location $targetCheckout
|
||
try {
|
||
$previousErrorAction = $ErrorActionPreference
|
||
$ErrorActionPreference = 'Continue'
|
||
try {
|
||
$scannerOutput = & $nodeCommand.Source $scannerPath 2>&1
|
||
$scannerExitCode = $LASTEXITCODE
|
||
}
|
||
finally {
|
||
$ErrorActionPreference = $previousErrorAction
|
||
}
|
||
}
|
||
finally {
|
||
Pop-Location
|
||
}
|
||
|
||
foreach ($line in @($scannerOutput)) {
|
||
Write-Host (Protect-LogLine -Line $line.ToString())
|
||
}
|
||
if ($scannerExitCode -ne 0) {
|
||
throw "Legacy storage boundary preflight failed for target $targetCommit with exit code $scannerExitCode"
|
||
}
|
||
|
||
return $targetCommit
|
||
}
|
||
finally {
|
||
$resolvedTempRoot = [System.IO.Path]::GetFullPath($tempRoot)
|
||
$expectedPrefix = "$tempBase$([System.IO.Path]::DirectorySeparatorChar)gongxiangp2-deploy-preflight-"
|
||
if ($resolvedTempRoot.StartsWith($expectedPrefix, [StringComparison]::OrdinalIgnoreCase)) {
|
||
Remove-Item -LiteralPath $resolvedTempRoot -Recurse -Force -ErrorAction SilentlyContinue
|
||
}
|
||
else {
|
||
Write-Warning-Custom "Skipped unexpected preflight cleanup path: $resolvedTempRoot"
|
||
}
|
||
}
|
||
}
|
||
|
||
function ConvertTo-ShellSingleQuoted {
|
||
param([AllowNull()][string]$Value)
|
||
if ($null -eq $Value) {
|
||
return "''"
|
||
}
|
||
return "'" + $Value.Replace("'", "'\''") + "'"
|
||
}
|
||
|
||
function Get-PlinkPath {
|
||
$candidates = @(
|
||
'C:\Program Files\PuTTY\plink.exe',
|
||
'C:\Program Files (x86)\PuTTY\plink.exe'
|
||
)
|
||
|
||
foreach ($candidate in $candidates) {
|
||
if (Test-Path $candidate) {
|
||
return $candidate
|
||
}
|
||
}
|
||
|
||
$fromPath = Get-Command plink.exe -ErrorAction SilentlyContinue
|
||
if ($fromPath) {
|
||
return $fromPath.Source
|
||
}
|
||
|
||
throw 'Cannot find plink.exe. Please install PuTTY or add plink.exe to PATH.'
|
||
}
|
||
|
||
function Get-PscpPath {
|
||
$candidates = @(
|
||
'C:\Program Files\PuTTY\pscp.exe',
|
||
'C:\Program Files (x86)\PuTTY\pscp.exe'
|
||
)
|
||
|
||
foreach ($candidate in $candidates) {
|
||
if (Test-Path $candidate) {
|
||
return $candidate
|
||
}
|
||
}
|
||
|
||
$fromPath = Get-Command pscp.exe -ErrorAction SilentlyContinue
|
||
if ($fromPath) {
|
||
return $fromPath.Source
|
||
}
|
||
|
||
throw 'Cannot find pscp.exe. Please install PuTTY or add pscp.exe to PATH.'
|
||
}
|
||
|
||
function Assert-Sha256HostFingerprint {
|
||
param(
|
||
[Parameter(Mandatory = $true)][string]$Fingerprint,
|
||
[Parameter(Mandatory = $true)][string]$Label
|
||
)
|
||
|
||
if ($Fingerprint -notmatch '^SHA256:[A-Za-z0-9+/]{20,}={0,2}$') {
|
||
throw "$Label must be an explicit SHA256 host fingerprint."
|
||
}
|
||
}
|
||
|
||
function Assert-LocalKnownHostFingerprint {
|
||
param(
|
||
[Parameter(Mandatory = $true)][string]$KnownHostsFile,
|
||
[Parameter(Mandatory = $true)][string]$HostName,
|
||
[Parameter(Mandatory = $true)][string]$ExpectedFingerprint
|
||
)
|
||
|
||
if ([string]::IsNullOrWhiteSpace($KnownHostsFile) -or -not (Test-Path -LiteralPath $KnownHostsFile -PathType Leaf)) {
|
||
throw 'BuildServerKnownHostsFile must be a pre-provisioned regular file.'
|
||
}
|
||
$knownHostsItem = Get-Item -LiteralPath $KnownHostsFile -Force
|
||
if (($knownHostsItem.Attributes -band [System.IO.FileAttributes]::ReparsePoint) -ne 0) {
|
||
throw 'BuildServerKnownHostsFile must not be a symbolic link or reparse point.'
|
||
}
|
||
|
||
$sshKeygen = Get-Command ssh-keygen.exe -ErrorAction SilentlyContinue
|
||
if (-not $sshKeygen) {
|
||
throw 'ssh-keygen.exe is required to verify the pinned build-server host fingerprint.'
|
||
}
|
||
|
||
$lookup = & $sshKeygen.Source -F $HostName -f $KnownHostsFile 2>$null
|
||
if ($LASTEXITCODE -ne 0) {
|
||
throw "Build server $HostName is absent from the pinned known_hosts file."
|
||
}
|
||
|
||
$fingerprints = @()
|
||
foreach ($line in @($lookup)) {
|
||
$keyLine = $line.ToString().Trim()
|
||
if (-not $keyLine -or $keyLine.StartsWith('#')) {
|
||
continue
|
||
}
|
||
$fingerprintOutput = $keyLine | & $sshKeygen.Source -l -f - -E sha256 2>$null
|
||
if ($LASTEXITCODE -ne 0) {
|
||
throw 'Could not calculate the build-server key fingerprint from known_hosts.'
|
||
}
|
||
foreach ($fingerprintLine in @($fingerprintOutput)) {
|
||
if ($fingerprintLine.ToString() -match '(SHA256:[A-Za-z0-9+/]{20,}={0,2})') {
|
||
$fingerprints += $Matches[1]
|
||
}
|
||
}
|
||
}
|
||
|
||
$uniqueFingerprints = @($fingerprints | Sort-Object -Unique)
|
||
if ($uniqueFingerprints.Count -ne 1 -or $uniqueFingerprints[0] -cne $ExpectedFingerprint) {
|
||
throw 'Build-server known_hosts fingerprint does not match BuildServerHostKey exactly.'
|
||
}
|
||
}
|
||
|
||
function Protect-LogLine {
|
||
param([string]$Line)
|
||
$masked = $Line
|
||
foreach ($secret in @($GitPassword, $BuildServerPassword, $DownloadTokenSecret, $GitAuthHeader)) {
|
||
if (-not [string]::IsNullOrEmpty($secret)) {
|
||
$masked = $masked.Replace($secret, '***')
|
||
}
|
||
}
|
||
return $masked
|
||
}
|
||
|
||
function Invoke-BuildServerScript {
|
||
param([string]$ScriptText)
|
||
|
||
$usePasswordAuth = -not [string]::IsNullOrWhiteSpace($BuildServerPassword)
|
||
$plinkPath = $null
|
||
$pscpPath = $null
|
||
$sshPath = $null
|
||
$scpPath = $null
|
||
$sshCommonArgs = @()
|
||
$scpCommonArgs = @()
|
||
|
||
Assert-Sha256HostFingerprint -Fingerprint $BuildServerHostKey -Label 'BuildServerHostKey'
|
||
Assert-Sha256HostFingerprint -Fingerprint $ProdServerHostKey -Label 'ProdServerHostKey'
|
||
if ([string]::IsNullOrWhiteSpace($ProdServerKnownHostsFile)) {
|
||
throw 'ProdServerKnownHostsFile is required on the build server for the production SSH hop.'
|
||
}
|
||
|
||
if ($usePasswordAuth) {
|
||
if (-not [string]::IsNullOrWhiteSpace($BuildServerProxyCommand)) {
|
||
throw 'BuildServerProxyCommand is supported only with OpenSSH key authentication.'
|
||
}
|
||
$plinkPath = Get-PlinkPath
|
||
$pscpPath = Get-PscpPath
|
||
}
|
||
else {
|
||
Assert-LocalKnownHostFingerprint `
|
||
-KnownHostsFile $BuildServerKnownHostsFile `
|
||
-HostName $BuildServerIP `
|
||
-ExpectedFingerprint $BuildServerHostKey
|
||
$sshCommand = Get-Command ssh.exe -ErrorAction SilentlyContinue
|
||
$scpCommand = Get-Command scp.exe -ErrorAction SilentlyContinue
|
||
if (-not $sshCommand -or -not $scpCommand) {
|
||
throw 'Build server password is empty, and OpenSSH ssh.exe/scp.exe were not found for key-based deployment.'
|
||
}
|
||
$sshPath = $sshCommand.Source
|
||
$scpPath = $scpCommand.Source
|
||
$sshCommonArgs = @('-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes', '-o', "UserKnownHostsFile=$BuildServerKnownHostsFile", '-o', 'ConnectTimeout=10')
|
||
$scpCommonArgs = @('-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=yes', '-o', "UserKnownHostsFile=$BuildServerKnownHostsFile", '-o', 'ConnectTimeout=10')
|
||
if (-not [string]::IsNullOrWhiteSpace($BuildServerProxyCommand)) {
|
||
$sshCommonArgs += @('-o', "ProxyCommand=$BuildServerProxyCommand")
|
||
$scpCommonArgs += @('-o', "ProxyCommand=$BuildServerProxyCommand")
|
||
}
|
||
}
|
||
|
||
$scriptId = [guid]::NewGuid().ToString('N')
|
||
$localTemp = Join-Path ([System.IO.Path]::GetTempPath()) "git-incremental-deploy-$scriptId.sh"
|
||
$remoteTempDir = "/tmp/gongxiangp2-deploy-$scriptId"
|
||
$remoteTemp = "$remoteTempDir/payload.sh"
|
||
$remotePrepared = $false
|
||
# PowerShell here-string 在 Windows 下使用 CRLF；远端 Bash 会把行尾的
|
||
# `\r` 当作参数内容，导致 `set -euo pipefail` 在脚本第一行就失败。
|
||
# 上传前统一为 LF，保证脚本在 Linux 构建机上可直接执行。
|
||
$normalizedScriptText = $ScriptText.Replace("`r`n", "`n").Replace("`r", "`n")
|
||
[System.IO.File]::WriteAllText($localTemp, $normalizedScriptText, [System.Text.UTF8Encoding]::new($false))
|
||
|
||
$authLabel = if ($usePasswordAuth) {
|
||
'password auth'
|
||
}
|
||
elseif (-not [string]::IsNullOrWhiteSpace($BuildServerProxyCommand)) {
|
||
'SSH key auth via proxy command'
|
||
}
|
||
else {
|
||
'SSH key auth'
|
||
}
|
||
Write-Info "Connecting to build server $BuildServerUser@$BuildServerIP ($authLabel)"
|
||
|
||
# 原生程序（plink/pscp）会把 git 等子进程的提示信息写到 stderr。
|
||
# 在 ErrorActionPreference=Stop 下，PowerShell 会把这些非致命的 stderr
|
||
# 当成 NativeCommandError 抛出，导致部署在 Git Sync 等阶段误中断。
|
||
# 这里在调用原生程序的局部作用域内改为 Continue，让 stderr 通过 2>&1
|
||
# 合并进管道正常显示；真实失败仍由 $LASTEXITCODE 捕获，不会掩盖错误。
|
||
$previousErrorAction = $ErrorActionPreference
|
||
$ErrorActionPreference = 'Continue'
|
||
|
||
try {
|
||
$prepareCommand = "set -eu; umask 077; test ! -e '$remoteTempDir'; mkdir -m 700 '$remoteTempDir'; test -d '$remoteTempDir'; test ! -L '$remoteTempDir'; test -O '$remoteTempDir'; test `"`$(stat -c %a '$remoteTempDir')`" = '700'"
|
||
if ($usePasswordAuth) {
|
||
$prepareOutput = & $plinkPath -batch -hostkey $BuildServerHostKey -pw $BuildServerPassword "$BuildServerUser@$BuildServerIP" $prepareCommand 2>&1
|
||
}
|
||
else {
|
||
$prepareOutput = & $sshPath @sshCommonArgs "$BuildServerUser@$BuildServerIP" $prepareCommand 2>&1
|
||
}
|
||
$prepareExitCode = $LASTEXITCODE
|
||
$prepareOutput |
|
||
ForEach-Object {
|
||
$line = Protect-LogLine -Line ($_.ToString())
|
||
if ($line.Trim()) {
|
||
Write-Host $line -ForegroundColor Gray
|
||
}
|
||
}
|
||
if ($prepareExitCode -ne 0) {
|
||
return $prepareExitCode
|
||
}
|
||
$remotePrepared = $true
|
||
|
||
$remoteSpec = "${BuildServerUser}@${BuildServerIP}:$remoteTemp"
|
||
$copyOutput = $null
|
||
if ($usePasswordAuth) {
|
||
$copyOutput = & $pscpPath -batch -hostkey $BuildServerHostKey -pw $BuildServerPassword $localTemp $remoteSpec 2>&1
|
||
}
|
||
else {
|
||
$copyOutput = & $scpPath @scpCommonArgs $localTemp $remoteSpec 2>&1
|
||
}
|
||
$copyOutput |
|
||
ForEach-Object {
|
||
$line = Protect-LogLine -Line ($_.ToString())
|
||
if ($line.Trim()) {
|
||
Write-Host $line -ForegroundColor Gray
|
||
}
|
||
}
|
||
|
||
if ($LASTEXITCODE -ne 0) {
|
||
return $LASTEXITCODE
|
||
}
|
||
|
||
$command = "set -eu; umask 077; test -f '$remoteTemp'; test ! -L '$remoteTemp'; test -O '$remoteTemp'; chmod 600 '$remoteTemp'; test `"`$(stat -c %a '$remoteTemp')`" = '600'; set +e; bash '$remoteTemp'; rc=`$?; set -e; rm -f '$remoteTemp'; rmdir '$remoteTempDir'; exit `$rc"
|
||
$runOutput = $null
|
||
if ($usePasswordAuth) {
|
||
$runOutput = & $plinkPath -batch -hostkey $BuildServerHostKey -pw $BuildServerPassword "$BuildServerUser@$BuildServerIP" $command 2>&1
|
||
}
|
||
else {
|
||
$runOutput = & $sshPath @sshCommonArgs "$BuildServerUser@$BuildServerIP" $command 2>&1
|
||
}
|
||
$runOutput |
|
||
ForEach-Object {
|
||
$line = Protect-LogLine -Line ($_.ToString())
|
||
if ($line -match '^\[OK\]') {
|
||
Write-Host $line -ForegroundColor Green
|
||
}
|
||
elseif ($line -match '^\[WARN\]') {
|
||
Write-Host $line -ForegroundColor Yellow
|
||
}
|
||
elseif ($line -match '^\[ERR\]|error|failed|fatal' ) {
|
||
Write-Host $line -ForegroundColor Red
|
||
}
|
||
else {
|
||
Write-Host $line -ForegroundColor Gray
|
||
}
|
||
}
|
||
|
||
return $LASTEXITCODE
|
||
}
|
||
finally {
|
||
if ($remotePrepared) {
|
||
$cleanupCommand = "rm -f '$remoteTemp'; rmdir '$remoteTempDir' >/dev/null 2>&1 || true"
|
||
if ($usePasswordAuth) {
|
||
$null = & $plinkPath -batch -hostkey $BuildServerHostKey -pw $BuildServerPassword "$BuildServerUser@$BuildServerIP" $cleanupCommand 2>$null
|
||
}
|
||
elseif ($sshPath) {
|
||
$null = & $sshPath @sshCommonArgs "$BuildServerUser@$BuildServerIP" $cleanupCommand 2>$null
|
||
}
|
||
}
|
||
$ErrorActionPreference = $previousErrorAction
|
||
Remove-Item $localTemp -Force -ErrorAction SilentlyContinue
|
||
}
|
||
}
|
||
|
||
$cfg = $targetConfig[$Target]
|
||
$rollbackFlag = if ($NoRollback) { '0' } else { '1' }
|
||
$proxyReconcilerPath = Join-Path $PSScriptRoot 'local-proxy-reconcile.sh'
|
||
if (-not (Test-Path -LiteralPath $proxyReconcilerPath)) {
|
||
throw "Local proxy reconciler is missing: $proxyReconcilerPath"
|
||
}
|
||
$proxyReconcilerText = [System.IO.File]::ReadAllText($proxyReconcilerPath)
|
||
$proxyReconcilerText = $proxyReconcilerText.Replace("`r`n", "`n").Replace("`r", "`n")
|
||
$proxyReconcilerBytes = [System.Text.UTF8Encoding]::new($false).GetBytes($proxyReconcilerText)
|
||
$proxyReconcilerBase64 = [Convert]::ToBase64String($proxyReconcilerBytes)
|
||
|
||
$tokens = @{
|
||
'__REPO_URL__' = ConvertTo-ShellSingleQuoted $RepoUrl
|
||
'__BRANCH__' = ConvertTo-ShellSingleQuoted $Branch
|
||
'__COMMIT__' = ConvertTo-ShellSingleQuoted ''
|
||
'__REMOTE_BUILD_DIR__' = ConvertTo-ShellSingleQuoted $RemoteBuildDir
|
||
'__TARGET__' = ConvertTo-ShellSingleQuoted $Target
|
||
'__BUILD_SERVER_IP__' = ConvertTo-ShellSingleQuoted $BuildServerIP
|
||
'__PROD_SERVER_IP__' = ConvertTo-ShellSingleQuoted $ProdServerIP
|
||
'__PROD_SERVER_USER__' = ConvertTo-ShellSingleQuoted $ProdServerUser
|
||
'__PROD_SERVER_HOST_KEY__' = ConvertTo-ShellSingleQuoted $ProdServerHostKey
|
||
'__PROD_KNOWN_HOSTS_FILE__' = ConvertTo-ShellSingleQuoted $ProdServerKnownHostsFile
|
||
'__REGISTRY_PORT__' = ConvertTo-ShellSingleQuoted ([string]$RegistryPort)
|
||
'__IMAGE_NAME__' = ConvertTo-ShellSingleQuoted $ImageName
|
||
'__UPLOADS_MOUNT__' = ConvertTo-ShellSingleQuoted $UploadsMount
|
||
'__MIGRATION_AUDIT_FILE__' = ConvertTo-ShellSingleQuoted $MigrationAuditFile
|
||
'__MIGRATION_AUDIT_SHA256__' = ConvertTo-ShellSingleQuoted $MigrationAuditSha256
|
||
'__MIGRATION_AUDIT_SOURCE_COMMIT__' = ConvertTo-ShellSingleQuoted $MigrationAuditSourceCommit
|
||
'__REQUIRE_MIGRATION_AUDIT__' = ConvertTo-ShellSingleQuoted ($(if ($RequireMigrationAudit) { '1' } else { '0' }))
|
||
'__CONTAINER_NAME__' = ConvertTo-ShellSingleQuoted $cfg.ContainerName
|
||
'__PORT__' = ConvertTo-ShellSingleQuoted ([string]$cfg.Port)
|
||
'__ENV_FILE__' = ConvertTo-ShellSingleQuoted $cfg.EnvFile
|
||
'__PUBLIC_APP_URL__' = ConvertTo-ShellSingleQuoted $cfg.PublicAppUrl
|
||
'__CORS_ALLOWED_ORIGINS__' = ConvertTo-ShellSingleQuoted $cfg.CorsAllowedOrigins
|
||
'__DINGTALK_CALLBACK_ORIGIN__' = ConvertTo-ShellSingleQuoted $cfg.DingtalkCallbackOrigin
|
||
'__DINGTALK_INTERNAL_RETURN_ORIGINS__' = ConvertTo-ShellSingleQuoted $cfg.DingtalkInternalReturnOrigins
|
||
'__S3_PUBLIC_DOMAIN__' = ConvertTo-ShellSingleQuoted $cfg.S3PublicDomain
|
||
'__DOWNLOAD_TOKEN_SECRET__' = ConvertTo-ShellSingleQuoted $DownloadTokenSecret
|
||
'__PROXY_RECONCILE_SCRIPT_B64__' = ConvertTo-ShellSingleQuoted $proxyReconcilerBase64
|
||
'__DRY_RUN__' = ConvertTo-ShellSingleQuoted ($(if ($DryRun) { '1' } else { '0' }))
|
||
'__NO_CACHE__' = ConvertTo-ShellSingleQuoted ($(if ($NoCache) { '1' } else { '0' }))
|
||
'__SKIP_BUILD__' = ConvertTo-ShellSingleQuoted ($(if ($SkipBuild) { '1' } else { '0' }))
|
||
'__SKIP_PUSH__' = ConvertTo-ShellSingleQuoted ($(if ($SkipPush) { '1' } else { '0' }))
|
||
'__REUSE_VERIFIED_IMAGE__' = ConvertTo-ShellSingleQuoted ($(if ($ReuseVerifiedImage) { '1' } else { '0' }))
|
||
'__SKIP_DEPLOY__' = ConvertTo-ShellSingleQuoted ($(if ($SkipDeploy) { '1' } else { '0' }))
|
||
'__KEEP_OLD_IMAGES__' = ConvertTo-ShellSingleQuoted ($(if ($KeepOldImages) { '1' } else { '0' }))
|
||
'__AGGRESSIVE_CLEANUP__' = ConvertTo-ShellSingleQuoted ($(if ($AggressiveBuildCacheCleanup) { '1' } else { '0' }))
|
||
'__ROLLBACK_ON_FAILURE__' = ConvertTo-ShellSingleQuoted $rollbackFlag
|
||
'__DEPLOYMENT_STRATEGY__' = ConvertTo-ShellSingleQuoted $DeploymentStrategy
|
||
}
|
||
|
||
$remoteTemplate = @'
|
||
set -Eeuo pipefail
|
||
|
||
REPO_URL=__REPO_URL__
|
||
BRANCH=__BRANCH__
|
||
COMMIT_INPUT=__COMMIT__
|
||
REMOTE_BUILD_DIR=__REMOTE_BUILD_DIR__
|
||
TARGET=__TARGET__
|
||
BUILD_SERVER_IP=__BUILD_SERVER_IP__
|
||
PROD_SERVER_IP=__PROD_SERVER_IP__
|
||
PROD_SERVER_USER=__PROD_SERVER_USER__
|
||
PROD_SERVER_HOST_KEY=__PROD_SERVER_HOST_KEY__
|
||
PROD_KNOWN_HOSTS_FILE=__PROD_KNOWN_HOSTS_FILE__
|
||
REGISTRY_PORT=__REGISTRY_PORT__
|
||
IMAGE_NAME=__IMAGE_NAME__
|
||
UPLOADS_MOUNT=__UPLOADS_MOUNT__
|
||
MIGRATION_AUDIT_FILE=__MIGRATION_AUDIT_FILE__
|
||
MIGRATION_AUDIT_SHA256=__MIGRATION_AUDIT_SHA256__
|
||
MIGRATION_AUDIT_SOURCE_COMMIT=__MIGRATION_AUDIT_SOURCE_COMMIT__
|
||
REQUIRE_MIGRATION_AUDIT=__REQUIRE_MIGRATION_AUDIT__
|
||
CONTAINER_NAME=__CONTAINER_NAME__
|
||
PORT=__PORT__
|
||
ENV_FILE=__ENV_FILE__
|
||
PUBLIC_APP_URL=__PUBLIC_APP_URL__
|
||
CORS_ALLOWED_ORIGINS=__CORS_ALLOWED_ORIGINS__
|
||
DINGTALK_CALLBACK_ORIGIN=__DINGTALK_CALLBACK_ORIGIN__
|
||
DINGTALK_INTERNAL_RETURN_ORIGINS=__DINGTALK_INTERNAL_RETURN_ORIGINS__
|
||
S3_PUBLIC_DOMAIN=__S3_PUBLIC_DOMAIN__
|
||
DOWNLOAD_TOKEN_SECRET=__DOWNLOAD_TOKEN_SECRET__
|
||
PROXY_RECONCILE_SCRIPT_B64=__PROXY_RECONCILE_SCRIPT_B64__
|
||
DRY_RUN=__DRY_RUN__
|
||
NO_CACHE=__NO_CACHE__
|
||
SKIP_BUILD=__SKIP_BUILD__
|
||
SKIP_PUSH=__SKIP_PUSH__
|
||
REUSE_VERIFIED_IMAGE=__REUSE_VERIFIED_IMAGE__
|
||
SKIP_DEPLOY=__SKIP_DEPLOY__
|
||
KEEP_OLD_IMAGES=__KEEP_OLD_IMAGES__
|
||
AGGRESSIVE_CLEANUP=__AGGRESSIVE_CLEANUP__
|
||
ROLLBACK_ON_FAILURE=__ROLLBACK_ON_FAILURE__
|
||
DEPLOYMENT_STRATEGY=__DEPLOYMENT_STRATEGY__
|
||
|
||
log() { printf '[INFO] %s\n' "$*"; }
|
||
ok() { printf '[OK] %s\n' "$*"; }
|
||
warn() { printf '[WARN] %s\n' "$*"; }
|
||
fail() { printf '[ERR] %s\n' "$*" >&2; exit 1; }
|
||
|
||
git_cmd() {
|
||
GIT_TERMINAL_PROMPT=0 git \
|
||
-c http.lowSpeedLimit=1 \
|
||
-c http.lowSpeedTime=30 \
|
||
"$@"
|
||
}
|
||
|
||
require_cmd() {
|
||
command -v "$1" >/dev/null 2>&1 || fail "Missing command on build server: $1"
|
||
}
|
||
|
||
start_remote_step() {
|
||
printf '\n==================================================\n'
|
||
printf '%s\n' "$1"
|
||
printf '==================================================\n'
|
||
}
|
||
|
||
start_remote_step "Preflight"
|
||
require_cmd git
|
||
require_cmd docker
|
||
require_cmd curl
|
||
require_cmd node
|
||
require_cmd ssh
|
||
require_cmd ssh-keygen
|
||
[ "$REPO_URL" = 'http://git.1080p.hk/WuYongJie/gongxiangp2.git' ] || fail "Build server received an untrusted Git repository URL"
|
||
[ "$REMOTE_BUILD_DIR" = '/root/youyidian-3d-git-build' ] || fail "Build directory is not the pinned dedicated worktree"
|
||
test -f "$PROD_KNOWN_HOSTS_FILE" || fail "Pinned production known_hosts file is missing on build server"
|
||
test ! -L "$PROD_KNOWN_HOSTS_FILE" || fail "Pinned production known_hosts file must not be a symlink"
|
||
ssh-keygen -F "$PROD_SERVER_IP" -f "$PROD_KNOWN_HOSTS_FILE" >/dev/null 2>&1 || fail "Pinned production host is absent from known_hosts"
|
||
actual_prod_host_key="$(ssh-keygen -F "$PROD_SERVER_IP" -f "$PROD_KNOWN_HOSTS_FILE" 2>/dev/null | awk '!/^#/ { print $2 " " $3 }' | ssh-keygen -lf - -E sha256 2>/dev/null | awk '{print $2}' | sort -u)"
|
||
[ "$(printf '%s\n' "$actual_prod_host_key" | sed '/^$/d' | wc -l | tr -d ' ')" = "1" ] || fail "Production known_hosts must resolve to exactly one SHA256 fingerprint"
|
||
[ "$actual_prod_host_key" = "$PROD_SERVER_HOST_KEY" ] || fail "Production host key fingerprint does not match the pinned value"
|
||
ok "Build server tools are available"
|
||
|
||
prod_ssh() {
|
||
ssh -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$PROD_KNOWN_HOSTS_FILE" -o ConnectTimeout=8 "$PROD_SERVER_USER@$PROD_SERVER_IP" "$@"
|
||
}
|
||
|
||
prod_ssh_script() {
|
||
local remote_command='bash -s --'
|
||
local argument=''
|
||
local quoted_argument=''
|
||
for argument in "$@"; do
|
||
printf -v quoted_argument '%q' "$argument"
|
||
remote_command+=" $quoted_argument"
|
||
done
|
||
prod_ssh "$remote_command"
|
||
}
|
||
|
||
ssh_prod_command() {
|
||
local log_file="$1"
|
||
shift
|
||
prod_ssh "$@" >"$log_file" 2>&1
|
||
}
|
||
|
||
retry_prod_ssh() {
|
||
local log_file="$1"
|
||
shift
|
||
local max_attempts=5
|
||
local attempt
|
||
|
||
for attempt in $(seq 1 "$max_attempts"); do
|
||
if ssh_prod_command "$log_file" "$@"; then
|
||
return 0
|
||
fi
|
||
|
||
warn "Production SSH check failed, retrying (${attempt}/${max_attempts})"
|
||
cat "$log_file" >&2 || true
|
||
sleep $((attempt * 2))
|
||
done
|
||
|
||
return 1
|
||
}
|
||
|
||
REMOTE_HEAD="$(git_cmd ls-remote --heads "$REPO_URL" "$BRANCH" | awk '{print $1}' | head -n 1 || true)"
|
||
if [ -z "$REMOTE_HEAD" ]; then
|
||
fail "Cannot read Git branch $BRANCH from $REPO_URL"
|
||
fi
|
||
ok "Git branch $BRANCH is reachable: ${REMOTE_HEAD:0:12}"
|
||
[ -n "$COMMIT_INPUT" ] || fail "Exact preflighted deployment commit is missing"
|
||
|
||
retry_prod_ssh /tmp/git-deploy-prod-check.log "echo OK" || {
|
||
cat /tmp/git-deploy-prod-check.log >&2 || true
|
||
fail "Cannot reach production server from build server"
|
||
}
|
||
ok "Production server is reachable from build server"
|
||
|
||
curl -fsS "http://localhost:${REGISTRY_PORT}/v2/" >/dev/null 2>&1 || {
|
||
fail "Docker registry is not reachable on build server port ${REGISTRY_PORT}"
|
||
}
|
||
ok "Docker registry is reachable"
|
||
|
||
if [ "$DRY_RUN" = "1" ]; then
|
||
log "Dry run completed. No code, image, or container was changed."
|
||
exit 0
|
||
fi
|
||
|
||
start_remote_step "Git Sync"
|
||
test ! -L "$REMOTE_BUILD_DIR" || fail "Pinned build directory must not be a symbolic link"
|
||
if [ -e "$REMOTE_BUILD_DIR" ]; then
|
||
test -d "$REMOTE_BUILD_DIR" || fail "Pinned build path exists but is not a directory; refusing destructive cleanup"
|
||
test -d "$REMOTE_BUILD_DIR/.git" || fail "Pinned build directory is not the expected Git worktree; refusing destructive cleanup"
|
||
test ! -L "$REMOTE_BUILD_DIR/.git" || fail "Pinned build worktree metadata must not be a symbolic link"
|
||
[ "$(realpath -e "$REMOTE_BUILD_DIR")" = "$REMOTE_BUILD_DIR" ] || fail "Pinned build directory resolves to an unexpected path"
|
||
existing_origin="$(git -C "$REMOTE_BUILD_DIR" remote get-url origin 2>/dev/null || true)"
|
||
[ "$existing_origin" = "$REPO_URL" ] || fail "Pinned build worktree origin does not match the trusted repository"
|
||
log "Using verified dedicated Git worktree: $REMOTE_BUILD_DIR"
|
||
else
|
||
[ "$(dirname "$REMOTE_BUILD_DIR")" = '/root' ] || fail "Pinned build directory parent is not trusted"
|
||
test -d /root && test ! -L /root || fail "Pinned build directory parent is not a trusted physical directory"
|
||
log "Creating a fresh trusted clone at $REMOTE_BUILD_DIR"
|
||
git_cmd clone --origin origin "$REPO_URL" "$REMOTE_BUILD_DIR"
|
||
fi
|
||
|
||
cd "$REMOTE_BUILD_DIR"
|
||
git_cmd fetch --prune origin "$BRANCH"
|
||
|
||
TARGET_REF="$COMMIT_INPUT"
|
||
|
||
git checkout -B "$BRANCH" "origin/$BRANCH" >/dev/null 2>&1 || git checkout "$BRANCH"
|
||
git reset --hard "$TARGET_REF"
|
||
git clean -fdx
|
||
|
||
COMMIT_SHA="$(git rev-parse HEAD)"
|
||
[ "$COMMIT_SHA" = "$COMMIT_INPUT" ] || fail "Git sync did not land on the exact preflighted commit"
|
||
if [ "$REQUIRE_MIGRATION_AUDIT" = "1" ] && [ "$SKIP_DEPLOY" != "1" ]; then
|
||
git cat-file -e "${MIGRATION_AUDIT_SOURCE_COMMIT}^{commit}" 2>/dev/null || \
|
||
fail "Migration audit source commit is unavailable in the trusted build worktree"
|
||
git merge-base --is-ancestor "$MIGRATION_AUDIT_SOURCE_COMMIT" "$COMMIT_SHA" || \
|
||
fail "Migration audit source commit is not an ancestor of the deployment commit"
|
||
if [ "$MIGRATION_AUDIT_SOURCE_COMMIT" != "$COMMIT_SHA" ]; then
|
||
git diff --quiet "$MIGRATION_AUDIT_SOURCE_COMMIT" "$COMMIT_SHA" -- prisma/schema.prisma prisma/migrations || \
|
||
fail "Database schema or migrations changed after the approved audit; create a fresh verified migration audit"
|
||
ok "Reusing verified migration audit because Prisma schema and migrations are unchanged"
|
||
fi
|
||
fi
|
||
SHORT_SHA="$(git rev-parse --short=12 HEAD)"
|
||
COMMIT_TAG="${TARGET}-${SHORT_SHA}"
|
||
MOVING_TAG="${TARGET}-latest"
|
||
REGISTRY_TAG="${BUILD_SERVER_IP}:${REGISTRY_PORT}/${IMAGE_NAME}:${COMMIT_TAG}"
|
||
REGISTRY_COMMIT="$REGISTRY_TAG"
|
||
REGISTRY_MOVING="${BUILD_SERVER_IP}:${REGISTRY_PORT}/${IMAGE_NAME}:${MOVING_TAG}"
|
||
LOCAL_REGISTRY_COMMIT="localhost:${REGISTRY_PORT}/${IMAGE_NAME}:${COMMIT_TAG}"
|
||
LOCAL_REGISTRY_MOVING="localhost:${REGISTRY_PORT}/${IMAGE_NAME}:${MOVING_TAG}"
|
||
|
||
test -f Dockerfile || fail "Dockerfile missing after Git sync"
|
||
test -f package.json || fail "package.json missing after Git sync"
|
||
test -f "app/api/uploads/[...path]/route.ts" || fail "app/api/uploads fallback route missing"
|
||
test -f "app/uploads/[...path]/route.ts" || fail "app/uploads fallback route missing"
|
||
|
||
ok "Synced to commit $COMMIT_SHA"
|
||
log "Image tag: $COMMIT_TAG"
|
||
|
||
if [ "$REUSE_VERIFIED_IMAGE" = "1" ]; then
|
||
start_remote_step "Verify Existing Image"
|
||
docker pull "$LOCAL_REGISTRY_COMMIT"
|
||
[ "$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$LOCAL_REGISTRY_COMMIT" 2>/dev/null || true)" = "$COMMIT_SHA" ] || \
|
||
fail "Existing exact-SHA image does not contain the requested Git revision"
|
||
docker tag "$LOCAL_REGISTRY_COMMIT" "$REGISTRY_TAG"
|
||
ok "Reusing verified exact-SHA image without rebuilding: $LOCAL_REGISTRY_COMMIT"
|
||
elif [ "$SKIP_BUILD" != "1" ]; then
|
||
start_remote_step "Docker Build"
|
||
docker pull "$LOCAL_REGISTRY_MOVING" >/dev/null 2>&1 || true
|
||
|
||
build_args=()
|
||
if [ "$NO_CACHE" = "1" ]; then
|
||
build_args+=(--no-cache)
|
||
elif docker image inspect "$LOCAL_REGISTRY_MOVING" >/dev/null 2>&1; then
|
||
build_args+=(--cache-from "$LOCAL_REGISTRY_MOVING")
|
||
fi
|
||
|
||
DOCKER_BUILDKIT=1 docker build "${build_args[@]}" \
|
||
--build-arg "VCS_REF=$COMMIT_SHA" \
|
||
-t "$IMAGE_NAME:$COMMIT_TAG" \
|
||
-t "$IMAGE_NAME:$MOVING_TAG" \
|
||
-t "$LOCAL_REGISTRY_COMMIT" \
|
||
-t "$LOCAL_REGISTRY_MOVING" \
|
||
-t "$REGISTRY_COMMIT" \
|
||
-t "$REGISTRY_MOVING" \
|
||
.
|
||
|
||
ok "Docker image built: $REGISTRY_COMMIT"
|
||
else
|
||
warn "Skipping Docker build by request"
|
||
fi
|
||
|
||
if [ "$REUSE_VERIFIED_IMAGE" = "1" ]; then
|
||
ok "Reusing immutable registry image; no image push is required"
|
||
elif [ "$SKIP_PUSH" != "1" ]; then
|
||
start_remote_step "Push Image"
|
||
docker push "$LOCAL_REGISTRY_COMMIT"
|
||
docker push "$LOCAL_REGISTRY_MOVING"
|
||
ok "Pushed image tags: $COMMIT_TAG, $MOVING_TAG"
|
||
else
|
||
warn "Skipping image push by request"
|
||
fi
|
||
|
||
resolve_published_image_identity() {
|
||
EXPECTED_MANIFEST_DIGEST="$(curl -fsSI \
|
||
-H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \
|
||
"http://localhost:${REGISTRY_PORT}/v2/${IMAGE_NAME}/manifests/${COMMIT_TAG}" \
|
||
| tr -d '\r' \
|
||
| awk 'tolower($1) == "docker-content-digest:" { print $2 }' \
|
||
| tail -n 1)"
|
||
[[ "$EXPECTED_MANIFEST_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "Registry did not return one immutable manifest digest for the exact commit tag"
|
||
REGISTRY_MANIFEST_JSON="$(curl -fsS \
|
||
-H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \
|
||
"http://localhost:${REGISTRY_PORT}/v2/${IMAGE_NAME}/manifests/${COMMIT_TAG}")"
|
||
EXPECTED_PLATFORM_MANIFEST_DIGEST="$(printf '%s' "$REGISTRY_MANIFEST_JSON" | node -e '
|
||
const document = JSON.parse(require("node:fs").readFileSync(0, "utf8"))
|
||
if (!Array.isArray(document.manifests)) {
|
||
if (!document.config || typeof document.config.digest !== "string") process.exit(2)
|
||
process.stdout.write("self")
|
||
process.exit(0)
|
||
}
|
||
const matches = document.manifests.filter((entry) =>
|
||
entry && entry.platform && entry.platform.os === "linux" && entry.platform.architecture === "amd64"
|
||
)
|
||
if (matches.length !== 1 || typeof matches[0].digest !== "string") process.exit(3)
|
||
process.stdout.write(matches[0].digest)
|
||
')"
|
||
if [ "$EXPECTED_PLATFORM_MANIFEST_DIGEST" = "self" ]; then
|
||
REGISTRY_PLATFORM_MANIFEST_JSON="$REGISTRY_MANIFEST_JSON"
|
||
else
|
||
[[ "$EXPECTED_PLATFORM_MANIFEST_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "Registry index did not contain exactly one linux/amd64 manifest"
|
||
REGISTRY_PLATFORM_MANIFEST_JSON="$(curl -fsS \
|
||
-H 'Accept: application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \
|
||
"http://localhost:${REGISTRY_PORT}/v2/${IMAGE_NAME}/manifests/${EXPECTED_PLATFORM_MANIFEST_DIGEST}")"
|
||
fi
|
||
EXPECTED_IMAGE_ID="$(printf '%s' "$REGISTRY_PLATFORM_MANIFEST_JSON" | node -e '
|
||
const document = JSON.parse(require("node:fs").readFileSync(0, "utf8"))
|
||
const digest = document && document.config && document.config.digest
|
||
if (typeof digest !== "string" || !/^sha256:[0-9a-f]{64}$/.test(digest)) process.exit(2)
|
||
process.stdout.write(digest)
|
||
')"
|
||
[[ "$EXPECTED_IMAGE_ID" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "Registry platform manifest did not contain one exact config digest"
|
||
REGISTRY_COMMIT="${BUILD_SERVER_IP}:${REGISTRY_PORT}/${IMAGE_NAME}@${EXPECTED_MANIFEST_DIGEST}"
|
||
}
|
||
|
||
if [ "$SKIP_DEPLOY" != "1" ]; then
|
||
resolve_published_image_identity
|
||
ok "Pinned production image: $REGISTRY_COMMIT ($EXPECTED_IMAGE_ID)"
|
||
start_remote_step "Deploy on Production Server"
|
||
log "Ensuring production server can access image $REGISTRY_COMMIT"
|
||
if prod_ssh "docker pull '$REGISTRY_COMMIT'" >/tmp/git-deploy-prod-pull.log 2>&1; then
|
||
ok "Production server pulled image from registry"
|
||
else
|
||
warn "Production server could not pull image directly; streaming image from build server"
|
||
cat /tmp/git-deploy-prod-pull.log >&2 || true
|
||
docker save "$REGISTRY_TAG" | prod_ssh "docker load"
|
||
REGISTRY_COMMIT="$REGISTRY_TAG"
|
||
ok "Image loaded on production server via SSH stream"
|
||
fi
|
||
|
||
ACTUAL_PROD_IMAGE_ID="$(prod_ssh "docker image inspect --format '{{.Id}}' '$REGISTRY_COMMIT' 2>/dev/null" | tr -d '\r\n' || true)"
|
||
if [ "$ACTUAL_PROD_IMAGE_ID" != "$EXPECTED_IMAGE_ID" ]; then
|
||
fail "Production image ID differs from the image built on the pinned build server"
|
||
fi
|
||
ok "Production image ID matches the trusted build result: $EXPECTED_IMAGE_ID"
|
||
|
||
if ! prod_ssh \
|
||
"test \"\$(docker image inspect --format '{{ index .Config.Labels \"org.opencontainers.image.revision\" }}' '$REGISTRY_COMMIT' 2>/dev/null)\" = '$COMMIT_SHA'"; then
|
||
fail "Production image revision label does not match the exact preflighted Git commit"
|
||
fi
|
||
ok "Production image revision label matches $COMMIT_SHA"
|
||
|
||
IFS= read -r -d '' PROD_PAYLOAD <<'PROD_SCRIPT' || true
|
||
set -Eeuo pipefail
|
||
|
||
TARGET="$1"
|
||
CONTAINER_NAME="$2"
|
||
PORT="$3"
|
||
REGISTRY_COMMIT="$4"
|
||
EXPECTED_IMAGE_ID="$5"
|
||
COMMIT_SHA="$6"
|
||
ENV_FILE_PATH="$7"
|
||
PUBLIC_APP_URL="$8"
|
||
CORS_ALLOWED_ORIGINS="$9"
|
||
shift 9
|
||
DINGTALK_CALLBACK_ORIGIN="$1"
|
||
DINGTALK_INTERNAL_RETURN_ORIGINS="$2"
|
||
S3_PUBLIC_DOMAIN="$3"
|
||
DOWNLOAD_TOKEN_SECRET="$4"
|
||
PROXY_RECONCILE_SCRIPT_B64="$5"
|
||
NETWORK_NAME="$6"
|
||
UPLOADS_MOUNT="$7"
|
||
ROLLBACK_ON_FAILURE="$8"
|
||
MIGRATION_AUDIT_FILE="$9"
|
||
shift 9
|
||
MIGRATION_AUDIT_SHA256="$1"
|
||
MIGRATION_AUDIT_SOURCE_COMMIT="$2"
|
||
REQUIRE_MIGRATION_AUDIT="$3"
|
||
DEPLOYMENT_STRATEGY="${4:-direct}"
|
||
|
||
log() { printf '[INFO] %s\n' "$*"; }
|
||
ok() { printf '[OK] %s\n' "$*"; }
|
||
warn() { printf '[WARN] %s\n' "$*"; }
|
||
fail() { printf '[ERR] %s\n' "$*" >&2; exit 1; }
|
||
|
||
create_container() {
|
||
local name="$1"
|
||
local image="$2"
|
||
local publish_host_port="${3:-0}"
|
||
local deploy_version
|
||
local transcode_instance_id
|
||
local sanitized_env_file
|
||
local create_status
|
||
deploy_version="$(date +%s)"
|
||
transcode_instance_id="${name}-${COMMIT_SHA}-${deploy_version}-$$"
|
||
sanitized_env_file="$(mktemp "/run/${CONTAINER_NAME}.env.XXXXXX")" || return 1
|
||
chmod 600 "$sanitized_env_file"
|
||
if ! awk '!/^UPLOAD_DIR=/' "$ENV_FILE_PATH" >"$sanitized_env_file"; then
|
||
rm -f "$sanitized_env_file"
|
||
return 1
|
||
fi
|
||
|
||
run_args=(
|
||
docker create
|
||
--name "$name"
|
||
--network "$NETWORK_NAME"
|
||
-v "${UPLOADS_MOUNT}:/app/uploads"
|
||
-v "/mnt/PoolOne/apps/youyidian-3d/public-traffic-logs:/var/log/nginx:ro"
|
||
--env-file "$sanitized_env_file"
|
||
-e "UPLOAD_DIR=/app/uploads"
|
||
-e "USE_REDIS=true"
|
||
-e "REDIS_URL=redis://gongxiangpan-redis:6379"
|
||
-e "NEXTAUTH_URL=$PUBLIC_APP_URL"
|
||
-e "APP_PUBLIC_URL=$PUBLIC_APP_URL"
|
||
-e "CORS_ALLOWED_ORIGINS=$CORS_ALLOWED_ORIGINS"
|
||
-e "NEXT_PUBLIC_DINGTALK_CALLBACK_ORIGIN=$DINGTALK_CALLBACK_ORIGIN"
|
||
-e "DINGTALK_INTERNAL_RETURN_ORIGINS=$DINGTALK_INTERNAL_RETURN_ORIGINS"
|
||
-e "S3_PUBLIC_DOMAIN=$S3_PUBLIC_DOMAIN"
|
||
-e "DISABLE_CACHE=true"
|
||
-e "DEPLOYMENT_VERSION=$deploy_version"
|
||
-e "VIDEO_TRANSCODE_WORKER_ID=$name"
|
||
-e "VIDEO_TRANSCODE_INSTANCE_ID=$transcode_instance_id"
|
||
-e "VIDEO_TRANSCODE_RUNTIME_DIR=/app/uploads/.video-transcode-runtime"
|
||
-e "TASK_DETAIL_UPLOAD_WORKER_ID=$name"
|
||
-e "TASK_DETAIL_UPLOAD_INSTANCE_ID=$transcode_instance_id"
|
||
-e "TASK_DETAIL_UPLOAD_RUNTIME_DIR=/app/uploads/.task-detail-upload-runtime"
|
||
--memory=6g
|
||
--memory-reservation=512m
|
||
--restart=unless-stopped
|
||
)
|
||
|
||
if [ "$publish_host_port" = "1" ]; then
|
||
run_args+=(-p "${PORT}:3000")
|
||
fi
|
||
|
||
if [ -n "$DOWNLOAD_TOKEN_SECRET" ]; then
|
||
run_args+=(-e "DOWNLOAD_TOKEN_SECRET=$DOWNLOAD_TOKEN_SECRET")
|
||
fi
|
||
|
||
if [ -n "${PUBLIC_TRAFFIC_ROLLUP_TOKEN:-}" ]; then
|
||
run_args+=(-e "PUBLIC_TRAFFIC_ROLLUP_TOKEN=$PUBLIC_TRAFFIC_ROLLUP_TOKEN")
|
||
fi
|
||
|
||
if "${run_args[@]}" "$image"; then
|
||
rm -f "$sanitized_env_file"
|
||
return 0
|
||
fi
|
||
create_status=$?
|
||
rm -f "$sanitized_env_file"
|
||
return "$create_status"
|
||
}
|
||
|
||
verify_image_revision_label() {
|
||
local image="$1"
|
||
local expected_commit="$2"
|
||
local revision=""
|
||
local image_id=""
|
||
revision="$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$image" 2>/dev/null || true)"
|
||
image_id="$(docker image inspect --format '{{.Id}}' "$image" 2>/dev/null || true)"
|
||
[ "$revision" = "$expected_commit" ] && [ "$image_id" = "$EXPECTED_IMAGE_ID" ]
|
||
}
|
||
|
||
verify_container_baked_revision() {
|
||
local name="$1"
|
||
local expected_commit="$2"
|
||
local revision=""
|
||
local baked=""
|
||
local image_id=""
|
||
revision="$(docker inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$name" 2>/dev/null || true)"
|
||
image_id="$(docker inspect --format '{{.Image}}' "$name" 2>/dev/null || true)"
|
||
baked="$(docker exec "$name" sh -c 'cat /app/DEPLOYMENT_COMMIT 2>/dev/null' 2>/dev/null | tr -d '\r\n' || true)"
|
||
[ "$revision" = "$expected_commit" ] && [ "$baked" = "$expected_commit" ] && [ "$image_id" = "$EXPECTED_IMAGE_ID" ]
|
||
}
|
||
|
||
require_file() {
|
||
[ -f "$1" ] || fail "Missing required file: $1"
|
||
}
|
||
|
||
command -v docker >/dev/null 2>&1 || fail "Missing docker on production server"
|
||
command -v curl >/dev/null 2>&1 || fail "Missing curl on production server"
|
||
command -v flock >/dev/null 2>&1 || fail "Missing flock on production server"
|
||
command -v ss >/dev/null 2>&1 || fail "Missing ss on production server; safe connection draining cannot be verified"
|
||
command -v base64 >/dev/null 2>&1 || fail "Missing base64 on production server"
|
||
command -v findmnt >/dev/null 2>&1 || fail "Missing findmnt on production server; uploads storage identity cannot be verified"
|
||
command -v stat >/dev/null 2>&1 || fail "Missing stat on production server; uploads storage identity cannot be verified"
|
||
command -v nginx >/dev/null 2>&1 || fail "Missing nginx on production server"
|
||
command -v systemctl >/dev/null 2>&1 || fail "Missing systemctl on production server"
|
||
command -v sha256sum >/dev/null 2>&1 || fail "Missing sha256sum on production server"
|
||
command -v timeout >/dev/null 2>&1 || fail "Missing timeout on production server"
|
||
|
||
require_file "$ENV_FILE_PATH"
|
||
|
||
uploads_findmnt_metadata_is_trusted() {
|
||
local mount_path="$1"
|
||
local mount_line="$2"
|
||
local root_line="$3"
|
||
local mount_target=""
|
||
local mount_source=""
|
||
local mount_fstype=""
|
||
local mount_extra=""
|
||
local root_target=""
|
||
local root_source=""
|
||
local root_fstype=""
|
||
local root_extra=""
|
||
|
||
[ -n "$mount_line" ] || return 1
|
||
[ -n "$root_line" ] || return 1
|
||
case "$mount_line" in *$'\n'*|*$'\r'*) return 1 ;; esac
|
||
case "$root_line" in *$'\n'*|*$'\r'*) return 1 ;; esac
|
||
|
||
read -r mount_target mount_source mount_fstype mount_extra <<< "$mount_line"
|
||
read -r root_target root_source root_fstype root_extra <<< "$root_line"
|
||
[ -n "$mount_target" ] && [ -n "$mount_source" ] && [ -n "$mount_fstype" ] && [ -z "$mount_extra" ] || return 1
|
||
[ "$root_target" = "/" ] && [ -n "$root_source" ] && [ -n "$root_fstype" ] && [ -z "$root_extra" ] || return 1
|
||
|
||
case "${mount_path}|${mount_target}|${mount_source}|${mount_fstype}" in
|
||
'/mnt/PoolOne/youyidian-uploads|/mnt/PoolOne/youyidian-uploads|PoolOne/youyidian-uploads|zfs') ;;
|
||
'/mnt/PoolOne/apps/youyidian-3d/uploads|/mnt/PoolOne|PoolOne|zfs') ;;
|
||
*) return 1 ;;
|
||
esac
|
||
|
||
[ "$mount_target" != "$root_target" ] || return 1
|
||
[ "$mount_source" != "$root_source" ] || return 1
|
||
UPLOADS_MOUNT_TARGET="$mount_target"
|
||
UPLOADS_MOUNT_SOURCE="$mount_source"
|
||
UPLOADS_MOUNT_FSTYPE="$mount_fstype"
|
||
}
|
||
|
||
host_statfs_signature_is_trusted() {
|
||
local uploads_signature="$1"
|
||
local root_signature="$2"
|
||
|
||
case "$uploads_signature" in ''|*$'\n'*|*$'\r'*|*[!0-9a-fA-F:]*) return 1 ;; esac
|
||
case "$root_signature" in ''|*$'\n'*|*$'\r'*|*[!0-9a-fA-F:]*) return 1 ;; esac
|
||
[ "$uploads_signature" != "$root_signature" ]
|
||
}
|
||
|
||
statfs_signatures_match() {
|
||
local host_signature="$1"
|
||
local container_signature="$2"
|
||
|
||
case "$host_signature" in ''|*$'\n'*|*$'\r'*|*[!0-9a-fA-F:]*) return 1 ;; esac
|
||
case "$container_signature" in ''|*$'\n'*|*$'\r'*|*[!0-9a-fA-F:]*) return 1 ;; esac
|
||
[ "$host_signature" = "$container_signature" ]
|
||
}
|
||
|
||
uploads_container_mount_metadata_is_trusted() {
|
||
local expected_source="$1"
|
||
local metadata="$2"
|
||
local allow_resolved_source_alias="${3:-0}"
|
||
local matching_metadata=""
|
||
local matching_count=""
|
||
local destination=""
|
||
local mount_type=""
|
||
local source=""
|
||
local read_write=""
|
||
local extra=""
|
||
local resolved_source=""
|
||
local resolved_expected_source=""
|
||
|
||
[ -n "$metadata" ] || return 1
|
||
case "$metadata" in *$'\r'*) return 1 ;; esac
|
||
matching_metadata="$(printf '%s\n' "$metadata" | awk -F '\t' '$1 == "/app/uploads" { print }')"
|
||
matching_count="$(printf '%s\n' "$matching_metadata" | awk 'NF { count++ } END { print count + 0 }')"
|
||
[ "$matching_count" = "1" ] || return 1
|
||
|
||
IFS=$'\t' read -r destination mount_type source read_write extra <<< "$matching_metadata"
|
||
[ "$destination" = "/app/uploads" ] || return 1
|
||
[ "$mount_type" = "bind" ] || return 1
|
||
if [ "$source" != "$expected_source" ]; then
|
||
# Old production containers were created before UPLOADS_MOUNT was
|
||
# canonicalised. They may retain the former path spelling in Docker
|
||
# metadata, while resolving to exactly the same verified ZFS dataset.
|
||
# This compatibility is for draining an existing container only; newly
|
||
# created candidates always use the canonical string above.
|
||
[ "$allow_resolved_source_alias" = "1" ] || return 1
|
||
resolved_source="$(readlink -f -- "$source")" || return 1
|
||
resolved_expected_source="$(readlink -f -- "$expected_source")" || return 1
|
||
[ "$resolved_source" = "$resolved_expected_source" ] || return 1
|
||
fi
|
||
[ "$read_write" = "true" ] || return 1
|
||
[ -z "$extra" ]
|
||
}
|
||
|
||
verify_host_uploads_mount() {
|
||
local mount_line=""
|
||
local root_line=""
|
||
local uploads_signature=""
|
||
local root_signature=""
|
||
|
||
[ -d "$UPLOADS_MOUNT" ] || {
|
||
warn "Uploads mount path does not exist: $UPLOADS_MOUNT"
|
||
return 1
|
||
}
|
||
if ! mount_line="$(findmnt -rn -o TARGET,SOURCE,FSTYPE -T "$UPLOADS_MOUNT")"; then
|
||
warn "findmnt failed for the configured uploads path"
|
||
return 1
|
||
fi
|
||
if ! root_line="$(findmnt -rn -o TARGET,SOURCE,FSTYPE -T /)"; then
|
||
warn "findmnt failed for the host root filesystem"
|
||
return 1
|
||
fi
|
||
if ! uploads_findmnt_metadata_is_trusted "$UPLOADS_MOUNT" "$mount_line" "$root_line"; then
|
||
warn "Uploads findmnt identity is not an explicitly trusted production tuple"
|
||
return 1
|
||
fi
|
||
if ! uploads_signature="$(stat -f -c '%i:%t' -- "$UPLOADS_MOUNT")"; then
|
||
warn "Cannot read uploads statfs identity"
|
||
return 1
|
||
fi
|
||
if ! root_signature="$(stat -f -c '%i:%t' -- /)"; then
|
||
warn "Cannot read root statfs identity"
|
||
return 1
|
||
fi
|
||
if ! host_statfs_signature_is_trusted "$uploads_signature" "$root_signature"; then
|
||
warn "Uploads path resolves to the root filesystem or has an invalid statfs identity"
|
||
return 1
|
||
fi
|
||
|
||
HOST_UPLOADS_STATFS_SIGNATURE="$uploads_signature"
|
||
ok "Uploads mount verified: TARGET=${UPLOADS_MOUNT_TARGET} SOURCE=${UPLOADS_MOUNT_SOURCE} FSTYPE=${UPLOADS_MOUNT_FSTYPE}"
|
||
}
|
||
|
||
container_uploads_mount_metadata() {
|
||
local name="$1"
|
||
docker inspect --format '{{range .Mounts}}{{printf "%s\t%s\t%s\t%t\n" .Destination .Type .Source .RW}}{{end}}' "$name"
|
||
}
|
||
|
||
verify_container_uploads_mount_metadata() {
|
||
local name="$1"
|
||
local allow_resolved_source_alias="${2:-0}"
|
||
local metadata=""
|
||
|
||
if ! metadata="$(container_uploads_mount_metadata "$name")"; then
|
||
warn "Cannot inspect uploads mount metadata for container ${name}"
|
||
return 1
|
||
fi
|
||
if ! uploads_container_mount_metadata_is_trusted "$UPLOADS_MOUNT" "$metadata" "$allow_resolved_source_alias"; then
|
||
warn "Container ${name} does not have exactly one trusted RW bind mount at /app/uploads"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
verify_container_upload_dir_config() {
|
||
local name="$1"
|
||
local require_exact="${2:-0}"
|
||
local env_lines=""
|
||
local count="0"
|
||
local value=""
|
||
|
||
if ! env_lines="$(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$name")"; then
|
||
warn "Cannot inspect UPLOAD_DIR for container ${name}"
|
||
return 1
|
||
fi
|
||
count="$(printf '%s\n' "$env_lines" | awk '/^UPLOAD_DIR=/{count++} END{print count+0}')"
|
||
value="$(printf '%s\n' "$env_lines" | awk -F= '/^UPLOAD_DIR=/{sub(/^UPLOAD_DIR=/, ""); print}')"
|
||
if [ "$require_exact" = "1" ]; then
|
||
if [ "$count" != "1" ] || [ "$value" != "/app/uploads" ]; then
|
||
warn "Container ${name} must declare exactly UPLOAD_DIR=/app/uploads"
|
||
return 1
|
||
fi
|
||
return 0
|
||
fi
|
||
if [ "$count" = "0" ]; then
|
||
return 0
|
||
fi
|
||
if [ "$count" = "1" ] && { [ "$value" = "/app/uploads" ] || [ "$value" = "./uploads" ]; }; then
|
||
return 0
|
||
fi
|
||
warn "Existing container ${name} has an unsafe UPLOAD_DIR override"
|
||
return 1
|
||
}
|
||
|
||
verify_container_upload_dir_runtime() {
|
||
local name="$1"
|
||
local require_exact="${2:-0}"
|
||
verify_container_upload_dir_config "$name" "$require_exact" || return 1
|
||
docker exec "$name" node -e '
|
||
const path = require("path");
|
||
const configured = String(process.env.UPLOAD_DIR || "").trim();
|
||
const resolved = configured ? path.resolve(configured) : path.join(process.cwd(), "uploads");
|
||
if (resolved !== "/app/uploads") process.exit(1);
|
||
' >/dev/null 2>&1 || {
|
||
warn "Container ${name} application upload root does not resolve to /app/uploads"
|
||
return 1
|
||
}
|
||
}
|
||
|
||
verify_container_uploads_statfs() {
|
||
local name="$1"
|
||
local container_signature=""
|
||
|
||
if ! container_signature="$(docker exec "$name" stat -f -c '%i:%t' -- /app/uploads)"; then
|
||
warn "Cannot read /app/uploads statfs identity from container ${name}"
|
||
return 1
|
||
fi
|
||
if ! statfs_signatures_match "$HOST_UPLOADS_STATFS_SIGNATURE" "$container_signature"; then
|
||
warn "Container ${name} /app/uploads statfs identity differs from the verified host data disk"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
verify_container_uploads_storage() {
|
||
local name="$1"
|
||
local require_exact="${2:-0}"
|
||
local allow_resolved_source_alias="0"
|
||
[ "$require_exact" = "1" ] || allow_resolved_source_alias="1"
|
||
verify_container_uploads_mount_metadata "$name" "$allow_resolved_source_alias" || return 1
|
||
verify_container_uploads_statfs "$name" || return 1
|
||
verify_container_upload_dir_runtime "$name" "$require_exact"
|
||
}
|
||
|
||
# This gate is intentionally before every container discovery, drain, start,
|
||
# stop, or proxy mutation below. A directory shell on the root disk must never
|
||
# be allowed to seed a second uploads/runtime universe.
|
||
if ! verify_host_uploads_mount; then
|
||
fail "Production uploads data disk could not be proven; no container or traffic was changed"
|
||
fi
|
||
docker network inspect "$NETWORK_NAME" >/dev/null 2>&1 || fail "Docker network missing: $NETWORK_NAME"
|
||
|
||
log "Checking image $REGISTRY_COMMIT"
|
||
if docker image inspect "$REGISTRY_COMMIT" >/dev/null 2>&1; then
|
||
ok "Image already available on production server"
|
||
else
|
||
docker pull "$REGISTRY_COMMIT"
|
||
fi
|
||
[[ "$EXPECTED_IMAGE_ID" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "Trusted build image ID is missing or malformed"
|
||
ACTUAL_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$REGISTRY_COMMIT" 2>/dev/null || true)"
|
||
[ "$ACTUAL_IMAGE_ID" = "$EXPECTED_IMAGE_ID" ] || fail "Production image differs from the exact artifact built on the pinned build server"
|
||
ok "Production image identity is pinned to $EXPECTED_IMAGE_ID"
|
||
|
||
PROXY_ROOT="/mnt/PoolOne/apps/youyidian-3d/local-proxy/${CONTAINER_NAME}"
|
||
PROXY_CONF="${PROXY_ROOT}/nginx.conf"
|
||
PROXY_PID="${PROXY_ROOT}/nginx.pid"
|
||
PROXY_SERVICE="${CONTAINER_NAME}-local-proxy.service"
|
||
CURRENT_FILE="${PROXY_ROOT}/current-container"
|
||
PROXY_RECONCILE_SCRIPT="${PROXY_ROOT}/reconcile-upstream.sh"
|
||
PROXY_RECONCILE_SERVICE="${CONTAINER_NAME}-local-proxy-reconcile.service"
|
||
PROXY_RECONCILE_TIMER="${CONTAINER_NAME}-local-proxy-reconcile.timer"
|
||
PROXY_RECONCILE_LOCK="/run/${CONTAINER_NAME}-local-proxy-reconcile.lock"
|
||
DEPLOY_LOCK="/run/${CONTAINER_NAME}-production-deploy.lock"
|
||
BOOTSTRAP_ROOT="${PROXY_ROOT}/legacy-bootstrap"
|
||
PROXY_CONFIG_REPLACED=0
|
||
PROXY_CONFIG_BACKUP=""
|
||
BLUE_CONTAINER="${CONTAINER_NAME}-blue"
|
||
GREEN_CONTAINER="${CONTAINER_NAME}-green"
|
||
PUBLIC_TRAFFIC_SYNC_ROOT="/mnt/PoolOne/apps/youyidian-3d/public-traffic-sync"
|
||
PUBLIC_TRAFFIC_ROLLUP_TOKEN_FILE="${PUBLIC_TRAFFIC_SYNC_ROOT}/rollup.token"
|
||
PUBLIC_TRAFFIC_LOG_SYNC_SCRIPT="${PUBLIC_TRAFFIC_SYNC_ROOT}/public-traffic-log-sync.sh"
|
||
PUBLIC_TRAFFIC_ROLLUP_SCRIPT="${PUBLIC_TRAFFIC_SYNC_ROOT}/public-traffic-rollup-sync.sh"
|
||
PUBLIC_TRAFFIC_ROLLUP_SERVICE="public-traffic-rollup-sync.service"
|
||
PUBLIC_TRAFFIC_ROLLUP_TIMER="public-traffic-rollup-sync.timer"
|
||
PUBLIC_TRAFFIC_ROLLUP_TOKEN=""
|
||
READY_BOOTSTRAP_HANDOFF=""
|
||
|
||
exec 9>"$DEPLOY_LOCK"
|
||
flock -w 30 9 || fail "Another production deployment or legacy bootstrap transition is already running"
|
||
|
||
verify_migration_audit_evidence() {
|
||
if [ "$REQUIRE_MIGRATION_AUDIT" != "1" ]; then
|
||
ok "Migration audit is not required for this normal application release"
|
||
return 0
|
||
fi
|
||
[ "$TARGET" = "production" ] || return 0
|
||
case "$MIGRATION_AUDIT_FILE" in
|
||
/root/gongxiangp2-migration-audit/*/applied-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]Z.json) ;;
|
||
*) fail "Migration apply audit path is outside the immutable production audit tree" ;;
|
||
esac
|
||
[ -f "$MIGRATION_AUDIT_FILE" ] && [ ! -L "$MIGRATION_AUDIT_FILE" ] || fail "Migration apply audit file is missing or not a regular file"
|
||
[ "$(stat -c '%a' "$MIGRATION_AUDIT_FILE")" = "600" ] || fail "Migration apply audit file permissions must be 0600"
|
||
[ "$(sha256sum "$MIGRATION_AUDIT_FILE" | awk '{print $1}')" = "$MIGRATION_AUDIT_SHA256" ] || fail "Migration apply audit SHA-256 does not match approved evidence"
|
||
docker run --rm -i --network none --read-only --user 0:0 --cap-drop ALL --security-opt no-new-privileges --cap-add DAC_READ_SEARCH \
|
||
-e MIGRATION_AUDIT_SOURCE_COMMIT="$MIGRATION_AUDIT_SOURCE_COMMIT" \
|
||
-v "$MIGRATION_AUDIT_FILE:/approved-migration-audit.json:ro" \
|
||
-v "$ENV_FILE_PATH:/approved-production.env:ro" \
|
||
--entrypoint node "$REGISTRY_COMMIT" <<'NODE'
|
||
const fs = require('fs');
|
||
const audit = JSON.parse(fs.readFileSync('/approved-migration-audit.json', 'utf8'));
|
||
if (audit.mode !== 'apply' || audit.verified !== true || audit.sourceCommit !== process.env.MIGRATION_AUDIT_SOURCE_COMMIT) process.exit(1);
|
||
const env = Object.fromEntries(fs.readFileSync('/approved-production.env', 'utf8').split(/\r?\n/)
|
||
.filter((line) => line && !line.trimStart().startsWith('#') && line.includes('='))
|
||
.map((line) => { const index = line.indexOf('='); return [line.slice(0, index), line.slice(index + 1)]; }));
|
||
const url = new URL(env.DATABASE_URL || '');
|
||
const database = audit.database || {};
|
||
const expected = {
|
||
configuredHost: url.hostname,
|
||
configuredPort: url.port || '5432',
|
||
configuredDatabase: decodeURIComponent(url.pathname.replace(/^\//, '')),
|
||
configuredUser: decodeURIComponent(url.username),
|
||
};
|
||
if (Object.entries(expected).some(([key, value]) => String(database[key] || '') !== value)) process.exit(1);
|
||
const migrations = Array.isArray(audit.targetMigrations) ? audit.targetMigrations : [];
|
||
const expectedMigrationNames = new Set([
|
||
'20260811_add_video_transcode_lease',
|
||
'20260811_update_brand_logo_defaults',
|
||
'20260812_add_product_package_upload_job_lease',
|
||
'20260812_align_material_upload_idempotency_index',
|
||
'20260813_add_material_custody_department',
|
||
'20260813_add_organization_units_and_employment_state',
|
||
'20260813_add_user_employee_binding',
|
||
]);
|
||
if (migrations.length !== expectedMigrationNames.size
|
||
|| migrations.some((item) => !expectedMigrationNames.has(item && item.name)
|
||
|| item.completed !== true
|
||
|| item.sourceChecksum !== item.databaseChecksum)
|
||
|| new Set(migrations.map((item) => item.name)).size !== expectedMigrationNames.size) process.exit(1);
|
||
if (!database.actualDatabase || !database.actualUser || !database.serverAddress || !database.serverPort) process.exit(1);
|
||
NODE
|
||
}
|
||
|
||
verify_migration_audit_evidence
|
||
|
||
verify_candidate_database_identity() {
|
||
[ "$REQUIRE_MIGRATION_AUDIT" = "1" ] || return 0
|
||
local name="$1"
|
||
docker exec -i "$name" node -e '
|
||
const { PrismaClient } = require("@prisma/client");
|
||
(async () => {
|
||
const chunks = [];
|
||
for await (const chunk of process.stdin) chunks.push(Buffer.from(chunk));
|
||
const input = Buffer.concat(chunks).toString("utf8");
|
||
const audit = JSON.parse(input);
|
||
const expected = audit.database || {};
|
||
const url = new URL(process.env.DATABASE_URL || "");
|
||
const configured = {
|
||
configuredHost: url.hostname,
|
||
configuredPort: url.port || "5432",
|
||
configuredDatabase: decodeURIComponent(url.pathname.replace(/^\//, "")),
|
||
configuredUser: decodeURIComponent(url.username),
|
||
};
|
||
const prisma = new PrismaClient();
|
||
try {
|
||
const rows = await prisma.$queryRawUnsafe(`SELECT current_database() AS database, current_user AS user, inet_server_addr()::text AS server_address, inet_server_port()::text AS server_port`);
|
||
const actual = rows[0] || {};
|
||
const ok = Object.entries(configured).every(([key, value]) => String(expected[key] || "") === value)
|
||
&& String(actual.database || "") === String(expected.actualDatabase || "")
|
||
&& String(actual.user || "") === String(expected.actualUser || "")
|
||
&& String(actual.server_address || "") === String(expected.serverAddress || "")
|
||
&& String(actual.server_port || "") === String(expected.serverPort || "");
|
||
if (!ok) process.exitCode = 1;
|
||
} finally {
|
||
await prisma.$disconnect();
|
||
}
|
||
})().catch((error) => { process.stderr.write(String(error && error.message || error)); process.exitCode = 1; });
|
||
' < "$MIGRATION_AUDIT_FILE"
|
||
}
|
||
|
||
verify_candidate_authenticated_s3() {
|
||
local name="$1"
|
||
docker exec "$name" node -e '
|
||
const { S3Client, HeadBucketCommand } = require("@aws-sdk/client-s3");
|
||
const required = ["S3_ENDPOINT", "S3_BUCKET", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"];
|
||
if (required.some((name) => !String(process.env[name] || "").trim())) process.exit(1);
|
||
const endpoint = String(process.env.S3_ENDPOINT).split("#")[0].trim();
|
||
const client = new S3Client({
|
||
endpoint,
|
||
region: process.env.S3_REGION || "us-east-1",
|
||
credentials: { accessKeyId: process.env.S3_ACCESS_KEY_ID, secretAccessKey: process.env.S3_SECRET_ACCESS_KEY },
|
||
forcePathStyle: true,
|
||
maxAttempts: 2,
|
||
});
|
||
(async () => {
|
||
try {
|
||
await client.send(new HeadBucketCommand({ Bucket: process.env.S3_BUCKET }), { abortSignal: AbortSignal.timeout(12000) });
|
||
} finally {
|
||
client.destroy();
|
||
}
|
||
})().catch((error) => { process.stderr.write(String(error && error.message || error)); process.exitCode = 1; });
|
||
'
|
||
}
|
||
|
||
decode_image_stream_with_candidate() {
|
||
local name="$1"
|
||
docker exec -i "$name" node -e '
|
||
const sharp = require("sharp");
|
||
(async () => {
|
||
const chunks = [];
|
||
for await (const chunk of process.stdin) chunks.push(Buffer.from(chunk));
|
||
const input = Buffer.concat(chunks);
|
||
if (!input.length) throw new Error("image input is empty");
|
||
await sharp(input, { failOn: "error", limitInputPixels: 25000000, animated: false }).rotate().toBuffer();
|
||
})().catch((error) => { process.stderr.write(String(error && error.message || error)); process.exitCode = 1; });
|
||
'
|
||
}
|
||
|
||
verify_candidate_baked_brand_assets() {
|
||
local name="$1"
|
||
docker exec "$name" node -e '
|
||
const sharp = require("sharp");
|
||
Promise.all(["header-logo.png", "login-logo.png", "favicon.png"].map((name) =>
|
||
sharp(`/app/public/brand/${name}`, { failOn: "error", limitInputPixels: 25000000, animated: false }).rotate().toBuffer()
|
||
)).catch((error) => { process.stderr.write(String(error && error.message || error)); process.exitCode = 1; });
|
||
'
|
||
}
|
||
|
||
verify_candidate_configured_brand_assets() {
|
||
local name="$1"
|
||
local candidate_ip="$2"
|
||
local prefix="/tmp/git-deploy-candidate-brand"
|
||
local logo_index=0
|
||
local logo_url=""
|
||
local resolved_logo_url=""
|
||
local logo_content_type=""
|
||
|
||
curl -fsS --max-time 12 "http://${candidate_ip}:3000/api/system-settings" > "${prefix}-settings.json" || return 1
|
||
docker exec -i -e PUBLIC_APP_URL="$PUBLIC_APP_URL" "$name" node -e '
|
||
(async () => {
|
||
const chunks = [];
|
||
for await (const chunk of process.stdin) chunks.push(Buffer.from(chunk));
|
||
const settings = JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
||
const origin = new URL(process.env.PUBLIC_APP_URL);
|
||
const values = [settings.headerLogoUrl, settings.loginLogoUrl];
|
||
if (!values.every((value) => typeof value === "string" && value.length > 0)) process.exit(1);
|
||
const normalized = values.map((value) => {
|
||
const parsed = new URL(value, origin);
|
||
const local = parsed.origin === origin.origin && /^\/brand\/[A-Za-z0-9][A-Za-z0-9._/-]*$/.test(parsed.pathname)
|
||
&& !parsed.search && !parsed.hash;
|
||
const s3 = parsed.protocol === "https:" && parsed.hostname === "s3.jiansun.vip"
|
||
&& parsed.port === "" && parsed.pathname.startsWith("/gongxiangpan/") && !parsed.search && !parsed.hash;
|
||
if (!local && !s3) process.exit(1);
|
||
return local ? parsed.pathname : parsed.href;
|
||
});
|
||
process.stdout.write(normalized.join("\n") + "\n");
|
||
})().catch((error) => { process.stderr.write(String(error && error.message || error)); process.exitCode = 1; });
|
||
' < "${prefix}-settings.json" > "${prefix}-urls.txt" || return 1
|
||
|
||
while IFS= read -r logo_url; do
|
||
[ -n "$logo_url" ] || return 1
|
||
logo_index=$((logo_index + 1))
|
||
case "$logo_url" in
|
||
/brand/*) resolved_logo_url="http://${candidate_ip}:3000${logo_url}" ;;
|
||
https://s3.jiansun.vip/gongxiangpan/*) resolved_logo_url="$logo_url" ;;
|
||
*) return 1 ;;
|
||
esac
|
||
curl -fsS --max-time 12 -D "${prefix}-${logo_index}.headers" \
|
||
-o "${prefix}-${logo_index}.body" "$resolved_logo_url" || return 1
|
||
logo_content_type="$(tr -d '\r' <"${prefix}-${logo_index}.headers" | sed -nE 's/^Content-Type:[[:space:]]*([^;[:space:]]+).*$/\1/Ip' | tail -n 1 | tr 'A-Z' 'a-z')"
|
||
case "$logo_content_type" in image/*) ;; *) return 1 ;; esac
|
||
decode_image_stream_with_candidate "$name" < "${prefix}-${logo_index}.body" || return 1
|
||
done < "${prefix}-urls.txt"
|
||
|
||
[ "$logo_index" -eq 2 ]
|
||
}
|
||
|
||
verify_candidate_brand_and_storage_dependencies() {
|
||
local name="$1"
|
||
local candidate_ip="$2"
|
||
verify_candidate_authenticated_s3 "$name" || return 1
|
||
verify_candidate_baked_brand_assets "$name" || return 1
|
||
verify_candidate_configured_brand_assets "$name" "$candidate_ip" || return 1
|
||
}
|
||
|
||
verify_ready_bootstrap_handoff_evidence() {
|
||
local handoff="$1"
|
||
local evidence_name=""
|
||
local legacy_container_id=""
|
||
local legacy_image_id=""
|
||
local legacy_image_repo_digest=""
|
||
local legacy_upstream=""
|
||
local legacy_ip=""
|
||
local uploads_signature=""
|
||
local preserved_container=""
|
||
local expected_preserved_container=""
|
||
local preserved_identity=""
|
||
local expected_identity=""
|
||
local repo_digests=""
|
||
local mount_metadata=""
|
||
local published_endpoints=""
|
||
local octet=""
|
||
local -a legacy_octets=()
|
||
|
||
[ -d "$handoff" ] && [ ! -L "$handoff" ] || fail "Bootstrap handoff evidence is not a physical directory"
|
||
[ "$(stat -c '%a' "$handoff")" = "700" ] || fail "Bootstrap handoff directory permissions must be 0700"
|
||
for evidence_name in target-commit legacy-container-id legacy-image-id legacy-image-repo-digest legacy-upstream uploads-statfs-signature preserved-container handoff.ready; do
|
||
[ -f "$handoff/$evidence_name" ] && [ ! -L "$handoff/$evidence_name" ] || fail "Bootstrap handoff evidence is missing or not a regular file: $evidence_name"
|
||
[ "$(stat -c '%a' "$handoff/$evidence_name")" = "600" ] || fail "Bootstrap handoff evidence permissions must be 0600: $evidence_name"
|
||
done
|
||
|
||
[ "$(tr -d '\r\n' < "$handoff/target-commit")" = "$COMMIT_SHA" ] || fail "Bootstrap handoff targets a different Git commit"
|
||
[ "$(tr -d '\r\n' < "$handoff/handoff.ready")" = "ready" ] || fail "Bootstrap handoff ready marker is invalid"
|
||
legacy_container_id="$(tr -d '\r\n' < "$handoff/legacy-container-id")"
|
||
legacy_image_id="$(tr -d '\r\n' < "$handoff/legacy-image-id")"
|
||
legacy_image_repo_digest="$(tr -d '\r\n' < "$handoff/legacy-image-repo-digest")"
|
||
legacy_upstream="$(tr -d '\r\n' < "$handoff/legacy-upstream")"
|
||
uploads_signature="$(tr 'A-F' 'a-f' < "$handoff/uploads-statfs-signature" | tr -d '\r\n')"
|
||
preserved_container="$(tr -d '\r\n' < "$handoff/preserved-container")"
|
||
|
||
[[ "$legacy_container_id" =~ ^[0-9a-f]{64}$ ]] || fail "Bootstrap handoff legacy container ID is malformed"
|
||
[[ "$legacy_image_id" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "Bootstrap handoff legacy image ID is malformed"
|
||
[[ "$legacy_image_repo_digest" =~ ^[A-Za-z0-9._:/-]+@sha256:[0-9a-f]{64}$ ]] || fail "Bootstrap handoff legacy RepoDigest is malformed"
|
||
[[ "$legacy_upstream" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}:3000$ ]] || fail "Bootstrap handoff legacy upstream is malformed"
|
||
legacy_ip="${legacy_upstream%:3000}"
|
||
IFS='.' read -r -a legacy_octets <<< "$legacy_ip"
|
||
[ "${#legacy_octets[@]}" = "4" ] || fail "Bootstrap handoff legacy IPv4 address is malformed"
|
||
for octet in "${legacy_octets[@]}"; do
|
||
(( 10#$octet <= 255 )) || fail "Bootstrap handoff legacy IPv4 address is out of range"
|
||
done
|
||
[ "$uploads_signature" = "$(printf '%s' "$HOST_UPLOADS_STATFS_SIGNATURE" | tr 'A-F' 'a-f')" ] || fail "Bootstrap handoff uploads statfs identity changed"
|
||
|
||
expected_preserved_container="${CONTAINER_NAME}-legacy-${COMMIT_SHA:0:12}-${legacy_container_id:0:12}"
|
||
[ "$preserved_container" = "$expected_preserved_container" ] || fail "Bootstrap handoff preserved container name is not deterministic"
|
||
if ! preserved_identity="$(docker inspect --format '{{printf "%s\t%s\t%s\t%t\t%t" .Id .Image .State.Status .State.Running .State.Paused}}' "$preserved_container")"; then
|
||
fail "Preserved legacy container is missing"
|
||
fi
|
||
expected_identity="${legacy_container_id}"$'\t'"${legacy_image_id}"$'\t'exited$'\t'false$'\t'false
|
||
[ "$preserved_identity" = "$expected_identity" ] || fail "Preserved legacy container is not the exact stopped handoff identity"
|
||
|
||
if ! repo_digests="$(docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$legacy_image_id")"; then
|
||
fail "Preserved legacy image is missing"
|
||
fi
|
||
[ "$(printf '%s\n' "$repo_digests" | grep -Fxc -- "$legacy_image_repo_digest")" = "1" ] || fail "Preserved legacy image RepoDigest changed"
|
||
if ! mount_metadata="$(container_uploads_mount_metadata "$preserved_container")"; then
|
||
fail "Preserved legacy uploads mount metadata is unavailable"
|
||
fi
|
||
uploads_container_mount_metadata_is_trusted "$UPLOADS_MOUNT" "$mount_metadata" || fail "Preserved legacy uploads bind changed"
|
||
if ! published_endpoints="$(docker port "$preserved_container" 3000/tcp 2>/dev/null | tr -d '\r' | sort -u | paste -sd, -)"; then
|
||
fail "Preserved legacy published endpoints are unavailable"
|
||
fi
|
||
[ "$published_endpoints" = "0.0.0.0:${PORT},[::]:${PORT}" ] || fail "Preserved legacy published endpoints changed"
|
||
}
|
||
|
||
verify_bootstrap_handoff_if_present() {
|
||
[ "$TARGET" = "production" ] || return 0
|
||
local handoff=""
|
||
local ready_count=0
|
||
local runtime_target=""
|
||
while IFS= read -r handoff; do
|
||
[ -n "$handoff" ] || continue
|
||
[ ! -e "$handoff/consumed" ] || continue
|
||
ready_count=$((ready_count + 1))
|
||
READY_BOOTSTRAP_HANDOFF="$handoff"
|
||
done < <(find "$BOOTSTRAP_ROOT" -mindepth 2 -maxdepth 2 -type f -name handoff.ready -printf '%h\n' 2>/dev/null)
|
||
[ "$ready_count" -le 1 ] || fail "Multiple ready bootstrap handoffs are ambiguous"
|
||
runtime_target="$(runtime_proxy_container)"
|
||
if [ -n "$runtime_target" ]; then
|
||
[ "$ready_count" -eq 0 ] || fail "A normal blue-green entry cannot consume a stale bootstrap handoff"
|
||
elif is_proxy_running; then
|
||
[ "$ready_count" -eq 1 ] || fail "The maintenance entry requires exactly one unconsumed bootstrap handoff for this Git commit"
|
||
verify_ready_bootstrap_handoff_evidence "$READY_BOOTSTRAP_HANDOFF"
|
||
maintenance_entry_is_ready_after_restore || fail "Bootstrap maintenance entry is not an exact dual-stack 503 handoff"
|
||
elif [ "$ready_count" -ne 0 ]; then
|
||
fail "Bootstrap handoff evidence exists but its maintenance proxy is not running"
|
||
fi
|
||
}
|
||
|
||
ensure_public_traffic_rollup_token() {
|
||
if [ "$TARGET" != "production" ]; then
|
||
return 0
|
||
fi
|
||
|
||
mkdir -p "$PUBLIC_TRAFFIC_SYNC_ROOT"
|
||
chmod 700 "$PUBLIC_TRAFFIC_SYNC_ROOT"
|
||
|
||
if [ ! -s "$PUBLIC_TRAFFIC_ROLLUP_TOKEN_FILE" ]; then
|
||
if command -v openssl >/dev/null 2>&1; then
|
||
openssl rand -hex 32 > "$PUBLIC_TRAFFIC_ROLLUP_TOKEN_FILE"
|
||
else
|
||
tr -dc 'A-Za-z0-9' </dev/urandom | head -c 64 > "$PUBLIC_TRAFFIC_ROLLUP_TOKEN_FILE"
|
||
fi
|
||
fi
|
||
|
||
chmod 600 "$PUBLIC_TRAFFIC_ROLLUP_TOKEN_FILE"
|
||
PUBLIC_TRAFFIC_ROLLUP_TOKEN="$(tr -d '\r\n' < "$PUBLIC_TRAFFIC_ROLLUP_TOKEN_FILE")"
|
||
}
|
||
|
||
install_public_traffic_rollup_timers() {
|
||
if [ "$TARGET" != "production" ]; then
|
||
return 0
|
||
fi
|
||
|
||
mkdir -p "$PUBLIC_TRAFFIC_SYNC_ROOT"
|
||
cat > "$PUBLIC_TRAFFIC_LOG_SYNC_SCRIPT" <<'EOF'
|
||
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
remote='trafficlog@218.93.9.38'
|
||
key='/root/.ssh/public_traffic_log_sync_ed25519'
|
||
dest='/mnt/PoolOne/apps/youyidian-3d/public-traffic-logs'
|
||
ssh_opts="ssh -i ${key} -o BatchMode=yes -o StrictHostKeyChecking=yes -o ConnectTimeout=10"
|
||
domains=(
|
||
sc.jiansun.vip
|
||
s3.jiansun.vip
|
||
cp.jiansun.vip
|
||
)
|
||
mkdir -p "$dest"
|
||
chmod 755 "$dest"
|
||
status=0
|
||
for domain in "${domains[@]}"; do
|
||
if ! rsync -rtz --timeout=45 --chmod=F644,D755 -e "$ssh_opts" "${remote}:/var/log/nginx/${domain}.access.log*" "$dest/"; then
|
||
echo "WARN: failed to sync ${domain}.access.log*" >&2
|
||
status=1
|
||
fi
|
||
done
|
||
find "$dest" -maxdepth 1 -type f -name '*.access.log*' -exec chmod 0644 {} +
|
||
exit "$status"
|
||
EOF
|
||
chmod 700 "$PUBLIC_TRAFFIC_LOG_SYNC_SCRIPT"
|
||
|
||
cat > "/etc/systemd/system/public-traffic-log-sync.service" <<EOF
|
||
[Unit]
|
||
Description=Sync public traffic nginx logs from tunnel server
|
||
After=network-online.target
|
||
Wants=network-online.target
|
||
|
||
[Service]
|
||
Type=oneshot
|
||
ExecStart=${PUBLIC_TRAFFIC_LOG_SYNC_SCRIPT}
|
||
TimeoutStartSec=180
|
||
EOF
|
||
|
||
cat > "/etc/systemd/system/public-traffic-log-sync.timer" <<EOF
|
||
[Unit]
|
||
Description=Run public traffic log sync every 5 minutes
|
||
|
||
[Timer]
|
||
OnBootSec=1min
|
||
OnUnitActiveSec=5min
|
||
Persistent=true
|
||
|
||
[Install]
|
||
WantedBy=timers.target
|
||
EOF
|
||
|
||
cat > "$PUBLIC_TRAFFIC_ROLLUP_SCRIPT" <<EOF
|
||
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
current_file='${CURRENT_FILE}'
|
||
container="\$(cat "\$current_file" 2>/dev/null || true)"
|
||
if [ -z "\$container" ] || ! docker inspect "\$container" >/dev/null 2>&1; then
|
||
echo "No active material container found" >&2
|
||
exit 1
|
||
fi
|
||
systemctl start public-traffic-log-sync.service >/dev/null 2>&1 || true
|
||
docker exec "\$container" python3 /app/scripts/public-traffic-rollup-from-nginx.py --granularity 5m
|
||
yesterday="\$(TZ=Asia/Shanghai date -d 'yesterday' +%F)"
|
||
docker exec "\$container" python3 /app/scripts/public-traffic-rollup-from-nginx.py --granularity day --start-date "\$yesterday" --end-date "\$yesterday"
|
||
EOF
|
||
chmod 700 "$PUBLIC_TRAFFIC_ROLLUP_SCRIPT"
|
||
|
||
cat > "/etc/systemd/system/${PUBLIC_TRAFFIC_ROLLUP_SERVICE}" <<EOF
|
||
[Unit]
|
||
Description=Sync public traffic statistics rollups
|
||
After=network-online.target ${PROXY_SERVICE}
|
||
Wants=network-online.target
|
||
|
||
[Service]
|
||
Type=oneshot
|
||
ExecStart=${PUBLIC_TRAFFIC_ROLLUP_SCRIPT}
|
||
TimeoutStartSec=360
|
||
EOF
|
||
|
||
cat > "/etc/systemd/system/${PUBLIC_TRAFFIC_ROLLUP_TIMER}" <<EOF
|
||
[Unit]
|
||
Description=Run public traffic statistics rollup sync
|
||
|
||
[Timer]
|
||
OnBootSec=2min
|
||
OnUnitActiveSec=5min
|
||
RandomizedDelaySec=30s
|
||
Persistent=true
|
||
|
||
[Install]
|
||
WantedBy=timers.target
|
||
EOF
|
||
|
||
systemctl daemon-reload
|
||
systemctl enable public-traffic-log-sync.timer >/dev/null 2>&1 || true
|
||
systemctl enable "$PUBLIC_TRAFFIC_ROLLUP_TIMER" >/dev/null 2>&1 || true
|
||
systemctl restart public-traffic-log-sync.timer >/dev/null 2>&1 || true
|
||
systemctl restart "$PUBLIC_TRAFFIC_ROLLUP_TIMER" >/dev/null 2>&1 || true
|
||
}
|
||
|
||
is_proxy_running() {
|
||
systemctl is-active --quiet "$PROXY_SERVICE" 2>/dev/null || {
|
||
[ -f "$PROXY_PID" ] && kill -0 "$(cat "$PROXY_PID")" >/dev/null 2>&1
|
||
}
|
||
}
|
||
|
||
container_ip() {
|
||
docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1" 2>/dev/null || true
|
||
}
|
||
|
||
write_proxy_config() {
|
||
local upstream="$1"
|
||
local target_container="$2"
|
||
local temp_conf=""
|
||
local backup_conf=""
|
||
|
||
PROXY_CONFIG_REPLACED=0
|
||
PROXY_CONFIG_BACKUP=""
|
||
|
||
case "$target_container" in
|
||
"$BLUE_CONTAINER"|"$GREEN_CONTAINER") ;;
|
||
*)
|
||
warn "Refusing to write proxy config for invalid container: ${target_container}"
|
||
return 1
|
||
;;
|
||
esac
|
||
|
||
mkdir -p "$PROXY_ROOT"
|
||
if ! temp_conf="$(mktemp "${PROXY_CONF}.tmp.XXXXXX")"; then
|
||
return 1
|
||
fi
|
||
cat > "$temp_conf" <<EOF
|
||
worker_processes 1;
|
||
pid ${PROXY_PID};
|
||
|
||
events {
|
||
worker_connections 2048;
|
||
}
|
||
|
||
http {
|
||
access_log off;
|
||
error_log ${PROXY_ROOT}/error.log warn;
|
||
# Unknown routes must never inherit a gigabyte-scale request allowance.
|
||
# Every supported large upload path below declares its own explicit limit.
|
||
client_max_body_size 16m;
|
||
|
||
map \$http_upgrade \$connection_upgrade {
|
||
default upgrade;
|
||
'' close;
|
||
}
|
||
|
||
map \$http_x_forwarded_proto \$forwarded_proto {
|
||
default \$http_x_forwarded_proto;
|
||
'' \$scheme;
|
||
}
|
||
|
||
upstream app_backend {
|
||
server ${upstream} max_fails=3 fail_timeout=5s;
|
||
keepalive 64;
|
||
}
|
||
|
||
server {
|
||
listen 0.0.0.0:${PORT};
|
||
listen [::]:${PORT} ipv6only=on;
|
||
server_name _;
|
||
|
||
# Public pre-auth endpoint: only a short DingTalk authorization code is
|
||
# accepted. Keep the proxy and application limits aligned so anonymous
|
||
# requests can never make Node parse a large form body.
|
||
location = /api/dingtalk/h5-login {
|
||
client_max_body_size 64k;
|
||
proxy_request_buffering off;
|
||
proxy_pass http://app_backend;
|
||
proxy_hide_header X-Material-Proxy-Target;
|
||
add_header X-Material-Proxy-Target ${target_container} always;
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Host \$http_host;
|
||
proxy_set_header X-Real-IP \$remote_addr;
|
||
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
||
proxy_set_header X-Forwarded-Proto \$forwarded_proto;
|
||
proxy_connect_timeout 10s;
|
||
proxy_read_timeout 60s;
|
||
proxy_send_timeout 60s;
|
||
}
|
||
|
||
location ~ "^/api/(?:user/avatar|system-settings/upload-logo)\$" {
|
||
client_max_body_size 7m;
|
||
proxy_request_buffering off;
|
||
proxy_pass http://app_backend;
|
||
proxy_hide_header X-Material-Proxy-Target;
|
||
add_header X-Material-Proxy-Target ${target_container} always;
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Host \$http_host;
|
||
proxy_set_header X-Real-IP \$remote_addr;
|
||
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
||
proxy_set_header X-Forwarded-Proto \$forwarded_proto;
|
||
proxy_read_timeout 3600s;
|
||
proxy_send_timeout 3600s;
|
||
proxy_connect_timeout 10s;
|
||
proxy_buffering off;
|
||
}
|
||
|
||
location = /api/upload/preview {
|
||
client_max_body_size 53m;
|
||
proxy_request_buffering off;
|
||
proxy_pass http://app_backend;
|
||
proxy_hide_header X-Material-Proxy-Target;
|
||
add_header X-Material-Proxy-Target ${target_container} always;
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Host \$http_host;
|
||
proxy_set_header X-Real-IP \$remote_addr;
|
||
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
||
proxy_set_header X-Forwarded-Proto \$forwarded_proto;
|
||
proxy_read_timeout 3600s;
|
||
proxy_send_timeout 3600s;
|
||
proxy_connect_timeout 10s;
|
||
proxy_buffering off;
|
||
}
|
||
|
||
location ~ "^/api/upload/s3(?:-original)?\$" {
|
||
client_max_body_size 103m;
|
||
proxy_request_buffering off;
|
||
proxy_pass http://app_backend;
|
||
proxy_hide_header X-Material-Proxy-Target;
|
||
add_header X-Material-Proxy-Target ${target_container} always;
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Host \$http_host;
|
||
proxy_set_header X-Real-IP \$remote_addr;
|
||
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
||
proxy_set_header X-Forwarded-Proto \$forwarded_proto;
|
||
proxy_read_timeout 3600s;
|
||
proxy_send_timeout 3600s;
|
||
proxy_connect_timeout 10s;
|
||
proxy_buffering off;
|
||
}
|
||
|
||
location ~ "^/(?:api/upload/psd|api/materials/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/skus/upload)\$" {
|
||
client_max_body_size 131m;
|
||
proxy_request_buffering off;
|
||
proxy_pass http://app_backend;
|
||
proxy_hide_header X-Material-Proxy-Target;
|
||
add_header X-Material-Proxy-Target ${target_container} always;
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Host \$http_host;
|
||
proxy_set_header X-Real-IP \$remote_addr;
|
||
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
||
proxy_set_header X-Forwarded-Proto \$forwarded_proto;
|
||
proxy_read_timeout 3600s;
|
||
proxy_send_timeout 3600s;
|
||
proxy_connect_timeout 10s;
|
||
proxy_buffering off;
|
||
}
|
||
|
||
location ~ "^/(?:api/materials/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/white-background|api/external/standard-model-white-background/(?:by-spu|by-barcode))\$" {
|
||
client_max_body_size 106m;
|
||
proxy_request_buffering off;
|
||
proxy_pass http://app_backend;
|
||
proxy_hide_header X-Material-Proxy-Target;
|
||
add_header X-Material-Proxy-Target ${target_container} always;
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Host \$http_host;
|
||
proxy_set_header X-Real-IP \$remote_addr;
|
||
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
||
proxy_set_header X-Forwarded-Proto \$forwarded_proto;
|
||
proxy_read_timeout 3600s;
|
||
proxy_send_timeout 3600s;
|
||
proxy_connect_timeout 10s;
|
||
proxy_buffering off;
|
||
}
|
||
|
||
location = /api/materials/update {
|
||
client_max_body_size 11m;
|
||
proxy_request_buffering off;
|
||
proxy_pass http://app_backend;
|
||
proxy_hide_header X-Material-Proxy-Target;
|
||
add_header X-Material-Proxy-Target ${target_container} always;
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Host \$http_host;
|
||
proxy_set_header X-Real-IP \$remote_addr;
|
||
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
||
proxy_set_header X-Forwarded-Proto \$forwarded_proto;
|
||
proxy_read_timeout 3600s;
|
||
proxy_send_timeout 3600s;
|
||
proxy_connect_timeout 10s;
|
||
proxy_buffering off;
|
||
}
|
||
|
||
location ~ "^/api/materials/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\$" {
|
||
client_max_body_size 11m;
|
||
proxy_request_buffering off;
|
||
proxy_pass http://app_backend;
|
||
proxy_hide_header X-Material-Proxy-Target;
|
||
add_header X-Material-Proxy-Target ${target_container} always;
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Host \$http_host;
|
||
proxy_set_header X-Real-IP \$remote_addr;
|
||
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
||
proxy_set_header X-Forwarded-Proto \$forwarded_proto;
|
||
proxy_read_timeout 3600s;
|
||
proxy_send_timeout 3600s;
|
||
proxy_connect_timeout 10s;
|
||
proxy_buffering off;
|
||
}
|
||
|
||
# Streaming request bodies keeps each upstream connection visible to the
|
||
# blue-green drain gate for the full client upload instead of only after
|
||
# nginx has buffered the complete request. Keep the 128 MiB legacy upload,
|
||
# 20 MiB raw chunk, and remaining sub-routes separate so multipart framing
|
||
# has explicit headroom without changing the existing general-route ceiling.
|
||
location = /api/materials/upload {
|
||
client_max_body_size 129m;
|
||
proxy_request_buffering off;
|
||
proxy_pass http://app_backend;
|
||
proxy_hide_header X-Material-Proxy-Target;
|
||
add_header X-Material-Proxy-Target ${target_container} always;
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Host \$http_host;
|
||
proxy_set_header X-Real-IP \$remote_addr;
|
||
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
||
proxy_set_header X-Forwarded-Proto \$forwarded_proto;
|
||
proxy_set_header Upgrade \$http_upgrade;
|
||
proxy_set_header Connection \$connection_upgrade;
|
||
proxy_read_timeout 3600s;
|
||
proxy_send_timeout 3600s;
|
||
proxy_connect_timeout 10s;
|
||
proxy_buffering off;
|
||
}
|
||
|
||
location = /api/materials/upload/chunk {
|
||
client_max_body_size 21m;
|
||
proxy_request_buffering off;
|
||
proxy_pass http://app_backend;
|
||
proxy_hide_header X-Material-Proxy-Target;
|
||
add_header X-Material-Proxy-Target ${target_container} always;
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Host \$http_host;
|
||
proxy_set_header X-Real-IP \$remote_addr;
|
||
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
||
proxy_set_header X-Forwarded-Proto \$forwarded_proto;
|
||
proxy_set_header Upgrade \$http_upgrade;
|
||
proxy_set_header Connection \$connection_upgrade;
|
||
proxy_read_timeout 3600s;
|
||
proxy_send_timeout 3600s;
|
||
proxy_connect_timeout 10s;
|
||
proxy_buffering off;
|
||
}
|
||
|
||
location ^~ /api/materials/upload/ {
|
||
client_max_body_size 2m;
|
||
proxy_request_buffering off;
|
||
proxy_pass http://app_backend;
|
||
proxy_hide_header X-Material-Proxy-Target;
|
||
add_header X-Material-Proxy-Target ${target_container} always;
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Host \$http_host;
|
||
proxy_set_header X-Real-IP \$remote_addr;
|
||
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
||
proxy_set_header X-Forwarded-Proto \$forwarded_proto;
|
||
proxy_set_header Upgrade \$http_upgrade;
|
||
proxy_set_header Connection \$connection_upgrade;
|
||
proxy_read_timeout 3600s;
|
||
proxy_send_timeout 3600s;
|
||
proxy_connect_timeout 10s;
|
||
proxy_buffering off;
|
||
}
|
||
|
||
location / {
|
||
proxy_pass http://app_backend;
|
||
proxy_hide_header X-Material-Proxy-Target;
|
||
add_header X-Material-Proxy-Target ${target_container} always;
|
||
proxy_http_version 1.1;
|
||
proxy_set_header Host \$http_host;
|
||
# Preserve the incoming LAN host and port for same-origin auth handoff.
|
||
proxy_set_header X-Real-IP \$remote_addr;
|
||
proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
|
||
proxy_set_header X-Forwarded-Proto \$forwarded_proto;
|
||
proxy_set_header Upgrade \$http_upgrade;
|
||
proxy_set_header Connection \$connection_upgrade;
|
||
proxy_read_timeout 3600s;
|
||
proxy_send_timeout 3600s;
|
||
proxy_connect_timeout 10s;
|
||
proxy_buffering off;
|
||
}
|
||
}
|
||
}
|
||
EOF
|
||
if [ "$?" -ne 0 ]; then
|
||
rm -f "$temp_conf"
|
||
return 1
|
||
fi
|
||
|
||
if ! nginx -p "${PROXY_ROOT}/" -c "$temp_conf" -t; then
|
||
rm -f "$temp_conf"
|
||
return 1
|
||
fi
|
||
|
||
if [ -f "$PROXY_CONF" ]; then
|
||
backup_conf="${PROXY_CONF}.bak.deploy.$(date +%Y%m%d-%H%M%S)"
|
||
if ! cp -a "$PROXY_CONF" "$backup_conf"; then
|
||
rm -f "$temp_conf"
|
||
return 1
|
||
fi
|
||
PROXY_CONFIG_BACKUP="$backup_conf"
|
||
fi
|
||
|
||
if ! mv "$temp_conf" "$PROXY_CONF"; then
|
||
rm -f "$temp_conf"
|
||
return 1
|
||
fi
|
||
PROXY_CONFIG_REPLACED=1
|
||
}
|
||
|
||
ensure_proxy_service() {
|
||
local service_file="/etc/systemd/system/${PROXY_SERVICE}"
|
||
local service_temp=""
|
||
|
||
mkdir -p "$PROXY_ROOT" || return 1
|
||
if ! service_temp="$(mktemp "${service_file}.tmp.XXXXXX")"; then
|
||
return 1
|
||
fi
|
||
cat > "$service_temp" <<EOF
|
||
[Unit]
|
||
Description=Youyidian material local entry proxy (${CONTAINER_NAME})
|
||
After=network-online.target docker.service
|
||
Wants=network-online.target
|
||
Requires=docker.service
|
||
|
||
[Service]
|
||
Type=simple
|
||
ExecStartPre=/usr/sbin/nginx -p ${PROXY_ROOT}/ -c ${PROXY_CONF} -t
|
||
ExecStart=/usr/sbin/nginx -p ${PROXY_ROOT}/ -c ${PROXY_CONF} -g 'daemon off;'
|
||
ExecReload=/usr/sbin/nginx -p ${PROXY_ROOT}/ -c ${PROXY_CONF} -s reload
|
||
ExecStop=/usr/sbin/nginx -p ${PROXY_ROOT}/ -c ${PROXY_CONF} -s quit
|
||
Restart=always
|
||
RestartSec=2
|
||
|
||
[Install]
|
||
WantedBy=multi-user.target
|
||
EOF
|
||
if [ "$?" -ne 0 ]; then
|
||
rm -f "$service_temp"
|
||
return 1
|
||
fi
|
||
chmod 0644 "$service_temp" || return 1
|
||
mv "$service_temp" "$service_file" || return 1
|
||
|
||
systemctl daemon-reload || return 1
|
||
systemctl enable "$PROXY_SERVICE" >/dev/null 2>&1 || return 1
|
||
}
|
||
|
||
switch_proxy() {
|
||
ensure_proxy_service || return 1
|
||
if is_proxy_running; then
|
||
systemctl reload "$PROXY_SERVICE"
|
||
else
|
||
systemctl start "$PROXY_SERVICE"
|
||
fi
|
||
}
|
||
|
||
runtime_proxy_container_at_url() {
|
||
local url="$1"
|
||
local target=""
|
||
|
||
target="$(
|
||
curl --noproxy '*' -g -sS --max-time 5 -D - -o /dev/null "$url" 2>/dev/null |
|
||
tr -d '\r' |
|
||
sed -nE 's/^X-Material-Proxy-Target:[[:space:]]*(.*)$/\1/Ip' |
|
||
tail -n 1 || true
|
||
)"
|
||
|
||
case "$target" in
|
||
"$BLUE_CONTAINER"|"$GREEN_CONTAINER") printf '%s\n' "$target" ;;
|
||
*) printf '\n' ;;
|
||
esac
|
||
}
|
||
|
||
runtime_proxy_container() {
|
||
runtime_proxy_container_at_url "http://127.0.0.1:${PORT}/api/health"
|
||
}
|
||
|
||
verify_proxy_target() {
|
||
local expected_container="$1"
|
||
local expected_commit="${2:-}"
|
||
local observed_container=""
|
||
local observed_ipv6_container=""
|
||
|
||
for _ in $(seq 1 5); do
|
||
observed_container="$(runtime_proxy_container)"
|
||
observed_ipv6_container="$(runtime_proxy_container_at_url "http://[::1]:${PORT}/api/health")"
|
||
if [ "$observed_container" = "$expected_container" ] && \
|
||
[ "$observed_ipv6_container" = "$expected_container" ] && \
|
||
curl -fsS --max-time 8 "http://127.0.0.1:${PORT}/api/health" >/tmp/git-deploy-entry-health.json 2>/tmp/git-deploy-entry-health.err && \
|
||
health_payload_is_ready /tmp/git-deploy-entry-health.json "$expected_commit" && \
|
||
curl --noproxy '*' -g -fsS --max-time 8 "http://[::1]:${PORT}/api/health" >/tmp/git-deploy-entry-health-ipv6.json 2>/tmp/git-deploy-entry-health-ipv6.err && \
|
||
health_payload_is_ready /tmp/git-deploy-entry-health-ipv6.json "$expected_commit"; then
|
||
return 0
|
||
fi
|
||
sleep 1
|
||
done
|
||
|
||
warn "Proxy target verification failed: expected=${expected_container}, observed_ipv4=${observed_container:-missing}, observed_ipv6=${observed_ipv6_container:-missing}"
|
||
return 1
|
||
}
|
||
|
||
install_proxy_reconciler() {
|
||
local script_temp=""
|
||
local service_file="/etc/systemd/system/${PROXY_RECONCILE_SERVICE}"
|
||
local service_temp=""
|
||
local timer_file="/etc/systemd/system/${PROXY_RECONCILE_TIMER}"
|
||
local timer_temp=""
|
||
|
||
mkdir -p "$PROXY_ROOT" || return 1
|
||
script_temp="$(mktemp "${PROXY_RECONCILE_SCRIPT}.tmp.XXXXXX")"
|
||
printf '%s' "$PROXY_RECONCILE_SCRIPT_B64" | base64 -d > "$script_temp"
|
||
bash -n "$script_temp"
|
||
chmod 700 "$script_temp"
|
||
mv "$script_temp" "$PROXY_RECONCILE_SCRIPT"
|
||
|
||
service_temp="$(mktemp "${service_file}.tmp.XXXXXX")"
|
||
cat > "$service_temp" <<EOF
|
||
[Unit]
|
||
Description=Reconcile ${CONTAINER_NAME} local proxy upstream
|
||
After=docker.service
|
||
Requires=docker.service
|
||
|
||
[Service]
|
||
Type=oneshot
|
||
Environment=CONTAINER_NAME=${CONTAINER_NAME}
|
||
Environment=PORT=${PORT}
|
||
Environment=PROXY_ROOT=${PROXY_ROOT}
|
||
Environment=PROXY_SERVICE=${PROXY_SERVICE}
|
||
ExecStart=${PROXY_RECONCILE_SCRIPT}
|
||
TimeoutStartSec=180
|
||
EOF
|
||
chmod 0644 "$service_temp"
|
||
mv "$service_temp" "$service_file"
|
||
|
||
timer_temp="$(mktemp "${timer_file}.tmp.XXXXXX")"
|
||
cat > "$timer_temp" <<EOF
|
||
[Unit]
|
||
Description=Check ${CONTAINER_NAME} local proxy upstream
|
||
|
||
[Timer]
|
||
OnBootSec=20s
|
||
OnUnitActiveSec=15s
|
||
AccuracySec=1s
|
||
Persistent=true
|
||
|
||
[Install]
|
||
WantedBy=timers.target
|
||
EOF
|
||
chmod 0644 "$timer_temp"
|
||
mv "$timer_temp" "$timer_file"
|
||
|
||
systemctl daemon-reload
|
||
systemctl enable "$PROXY_RECONCILE_TIMER" >/dev/null 2>&1 || return 1
|
||
systemctl restart "$PROXY_RECONCILE_TIMER"
|
||
}
|
||
|
||
container_running() {
|
||
[ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || true)" = "true" ]
|
||
}
|
||
|
||
read_recorded_container() {
|
||
local name=""
|
||
|
||
if [ -s "$CURRENT_FILE" ]; then
|
||
name="$(tr -d '\r\n' < "$CURRENT_FILE")"
|
||
fi
|
||
|
||
case "$name" in
|
||
"$BLUE_CONTAINER"|"$GREEN_CONTAINER") printf '%s\n' "$name" ;;
|
||
*) printf '\n' ;;
|
||
esac
|
||
}
|
||
|
||
write_current_container() {
|
||
local name="$1"
|
||
local temp_file=""
|
||
|
||
mkdir -p "$PROXY_ROOT"
|
||
temp_file="$(mktemp "${CURRENT_FILE}.tmp.XXXXXX")"
|
||
printf '%s\n' "$name" > "$temp_file"
|
||
chmod 0644 "$temp_file"
|
||
mv "$temp_file" "$CURRENT_FILE"
|
||
}
|
||
|
||
discover_current_container() {
|
||
local recorded_container=""
|
||
local runtime_container=""
|
||
local running_containers=()
|
||
local name=""
|
||
|
||
recorded_container="$(read_recorded_container)"
|
||
runtime_container="$(runtime_proxy_container)"
|
||
|
||
if [ -n "$runtime_container" ] && container_running "$runtime_container"; then
|
||
printf '%s\n' "$runtime_container"
|
||
return 0
|
||
fi
|
||
|
||
for name in "$BLUE_CONTAINER" "$GREEN_CONTAINER"; do
|
||
if container_running "$name"; then
|
||
running_containers+=("$name")
|
||
fi
|
||
done
|
||
|
||
if [ "${#running_containers[@]}" -eq 1 ]; then
|
||
printf '%s\n' "${running_containers[0]}"
|
||
return 0
|
||
fi
|
||
|
||
if [ "${#running_containers[@]}" -gt 1 ]; then
|
||
warn "Multiple running material containers found without a verified runtime proxy target (recorded=${recorded_container:-missing})"
|
||
return 1
|
||
fi
|
||
|
||
printf '\n'
|
||
}
|
||
|
||
health_payload_is_ready() {
|
||
local payload_file="$1"
|
||
local expected_commit="${2:-}"
|
||
|
||
grep -Fq -- '"status":"healthy"' "$payload_file" || return 1
|
||
grep -Fq -- '"database":{"status":"ok"' "$payload_file" || return 1
|
||
grep -Fq -- '"redis":{"status":"ok"' "$payload_file" || return 1
|
||
if [ -n "$expected_commit" ]; then
|
||
grep -Fq -- "\"deploymentCommit\":\"${expected_commit}\"" "$payload_file" || return 1
|
||
fi
|
||
}
|
||
|
||
wait_for_container_health() {
|
||
local name="$1"
|
||
local expected_commit="${2:-}"
|
||
local ip=""
|
||
|
||
for i in $(seq 1 24); do
|
||
running="$(docker inspect -f '{{.State.Running}}' "$name" 2>/dev/null || echo false)"
|
||
ip="$(container_ip "$name")"
|
||
if [ "$running" = "true" ] && [ -n "$ip" ]; then
|
||
if curl -fsS --max-time 8 "http://${ip}:3000/api/health" >/tmp/git-deploy-health.json 2>/tmp/git-deploy-health.err &&
|
||
health_payload_is_ready /tmp/git-deploy-health.json "$expected_commit"; then
|
||
printf '%s\n' "$ip"
|
||
return 0
|
||
fi
|
||
fi
|
||
sleep 5
|
||
done
|
||
|
||
return 1
|
||
}
|
||
|
||
container_healthy() {
|
||
local name="$1"
|
||
local expected_commit="${2:-}"
|
||
local ip=""
|
||
|
||
container_running "$name" || return 1
|
||
ip="$(container_ip "$name")"
|
||
[ -n "$ip" ] || return 1
|
||
curl -fsS --max-time 8 "http://${ip}:3000/api/health" >/tmp/git-deploy-container-health.json 2>/dev/null &&
|
||
health_payload_is_ready /tmp/git-deploy-container-health.json "$expected_commit"
|
||
}
|
||
|
||
# This is intentionally a one-slot replacement. It never starts a second
|
||
# application worker beside the running one: the active slot is stopped before
|
||
# the new image is created with that same name. It therefore matches the
|
||
# former short deployment model and avoids the video/transcode drain protocol
|
||
# required only by the retired blue/green hand-off path.
|
||
restore_direct_previous_container() {
|
||
local name="$1"
|
||
local previous_image="$2"
|
||
local previous_ip=""
|
||
|
||
docker rm -f "$name" >/dev/null 2>&1 || return 1
|
||
create_container "$name" "$previous_image" 0 || return 1
|
||
verify_container_uploads_mount_metadata "$name" || return 1
|
||
verify_container_upload_dir_config "$name" 1 || return 1
|
||
docker start "$name" >/dev/null || return 1
|
||
verify_container_uploads_storage "$name" 1 || return 1
|
||
previous_ip="$(wait_for_container_health "$name")" || return 1
|
||
write_proxy_config "${previous_ip}:3000" "$name" || return 1
|
||
switch_proxy || return 1
|
||
verify_proxy_target "$name" || return 1
|
||
write_current_container "$name" || return 1
|
||
}
|
||
|
||
fail_direct_replace_with_restore() {
|
||
local reason="$1"
|
||
local name="$2"
|
||
local previous_image="$3"
|
||
|
||
warn "$reason"
|
||
if restore_direct_previous_container "$name" "$previous_image"; then
|
||
fail "$reason; previous container was restored"
|
||
fi
|
||
fail "$reason; previous container could not be restored automatically"
|
||
}
|
||
|
||
retire_inactive_direct_slot() {
|
||
local active_name="$1"
|
||
local inactive_name=""
|
||
|
||
case "$active_name" in
|
||
"$BLUE_CONTAINER") inactive_name="$GREEN_CONTAINER" ;;
|
||
"$GREEN_CONTAINER") inactive_name="$BLUE_CONTAINER" ;;
|
||
*) return 1 ;;
|
||
esac
|
||
|
||
docker inspect "$inactive_name" >/dev/null 2>&1 || return 0
|
||
if container_running "$inactive_name"; then
|
||
log "Retiring inactive historical container: $inactive_name"
|
||
docker stop --time 10 "$inactive_name" >/dev/null || return 1
|
||
fi
|
||
docker rm "$inactive_name" >/dev/null || return 1
|
||
}
|
||
|
||
direct_replace_active_container() {
|
||
local name="$1"
|
||
local previous_image=""
|
||
local replacement_ip=""
|
||
|
||
case "$name" in
|
||
"$BLUE_CONTAINER"|"$GREEN_CONTAINER") ;;
|
||
*) fail "Direct replacement requires one active material slot" ;;
|
||
esac
|
||
|
||
previous_image="$(docker inspect -f '{{.Config.Image}}' "$name" 2>/dev/null || true)"
|
||
[ -n "$previous_image" ] || fail "Active container image could not be determined: $name"
|
||
verify_container_uploads_storage "$name" || fail "Active container uploads storage could not be verified before direct replacement"
|
||
verify_image_revision_label "$REGISTRY_COMMIT" "$COMMIT_SHA" || fail "Direct replacement image label does not match the exact Git commit"
|
||
prepare_worker_runtime_directories || fail "Worker runtime directories could not be prepared"
|
||
|
||
log "Replacing active container directly: $name"
|
||
if ! docker stop --time 10 "$name" >/dev/null; then
|
||
fail "Active container could not be stopped for direct replacement"
|
||
fi
|
||
if ! docker rm "$name" >/dev/null; then
|
||
docker start "$name" >/dev/null 2>&1 || true
|
||
fail "Stopped container could not be removed for direct replacement"
|
||
fi
|
||
if ! create_container "$name" "$REGISTRY_COMMIT" 0; then
|
||
fail_direct_replace_with_restore "New direct container could not be created" "$name" "$previous_image"
|
||
fi
|
||
if ! verify_container_uploads_mount_metadata "$name" || ! verify_container_upload_dir_config "$name" 1; then
|
||
fail_direct_replace_with_restore "New direct container uploads configuration is invalid" "$name" "$previous_image"
|
||
fi
|
||
if ! docker start "$name" >/dev/null; then
|
||
fail_direct_replace_with_restore "New direct container could not be started" "$name" "$previous_image"
|
||
fi
|
||
if ! verify_container_uploads_storage "$name" 1 || ! verify_container_baked_revision "$name" "$COMMIT_SHA"; then
|
||
fail_direct_replace_with_restore "New direct container identity or uploads storage verification failed" "$name" "$previous_image"
|
||
fi
|
||
if ! replacement_ip="$(wait_for_container_health "$name" "$COMMIT_SHA")"; then
|
||
fail_direct_replace_with_restore "New direct container failed its health check" "$name" "$previous_image"
|
||
fi
|
||
if ! write_proxy_config "${replacement_ip}:3000" "$name" || ! switch_proxy || ! verify_proxy_target "$name" "$COMMIT_SHA"; then
|
||
fail_direct_replace_with_restore "New direct container could not become the verified proxy target" "$name" "$previous_image"
|
||
fi
|
||
write_current_container "$name" || fail "Direct replacement traffic is healthy but the current-slot record could not be written"
|
||
retire_inactive_direct_slot "$name" || warn "The inactive historical slot could not be removed; the active direct container remains healthy"
|
||
rm -f -- "$(transcode_drain_file "$name")" "$(task_detail_drain_file "$name")" || true
|
||
ok "Direct replacement completed: $name"
|
||
}
|
||
|
||
container_upstream_connection_count() {
|
||
local name="$1"
|
||
local ip=""
|
||
|
||
command -v ss >/dev/null 2>&1 || return 2
|
||
container_running "$name" || {
|
||
printf '0\n'
|
||
return 0
|
||
}
|
||
ip="$(container_ip "$name")"
|
||
[ -n "$ip" ] || return 2
|
||
|
||
# Let ss filter the peer endpoint itself; column positions differ between
|
||
# iproute2 versions when the State column is omitted by -H.
|
||
ss -Htan state established dst "${ip}:3000" 2>/dev/null |
|
||
awk 'END { print NR + 0 }'
|
||
}
|
||
|
||
wait_for_container_drain() {
|
||
local name="$1"
|
||
local max_checks="${2:-15}"
|
||
local count=""
|
||
local zero_samples=0
|
||
|
||
for _ in $(seq 1 "$max_checks"); do
|
||
if ! count="$(container_upstream_connection_count "$name")"; then
|
||
warn "Cannot prove that upstream connections drained for ${name}; leaving it running"
|
||
return 1
|
||
fi
|
||
|
||
if [ "$count" = "0" ]; then
|
||
zero_samples=$((zero_samples + 1))
|
||
if [ "$zero_samples" -ge 2 ]; then
|
||
return 0
|
||
fi
|
||
else
|
||
zero_samples=0
|
||
log "Waiting for ${count} active upstream connection(s) to drain from ${name}"
|
||
fi
|
||
sleep 2
|
||
done
|
||
|
||
warn "Active upstream connections did not drain from ${name}; leaving it running"
|
||
return 1
|
||
}
|
||
|
||
TRANSCODE_RUNTIME_ROOT="${UPLOADS_MOUNT}/.video-transcode-runtime"
|
||
TRANSCODE_DRAIN_TARGET=""
|
||
TRANSCODE_DRAIN_TARGETS=""
|
||
|
||
transcode_capability_file() {
|
||
printf '%s/capabilities/%s.v1\n' "$TRANSCODE_RUNTIME_ROOT" "$1"
|
||
}
|
||
|
||
transcode_drain_file() {
|
||
printf '%s/drain/%s\n' "$TRANSCODE_RUNTIME_ROOT" "$1"
|
||
}
|
||
|
||
transcode_active_file() {
|
||
printf '%s/active/%s.json\n' "$TRANSCODE_RUNTIME_ROOT" "$1"
|
||
}
|
||
|
||
container_transcode_instance_id() {
|
||
local name="$1"
|
||
docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$name" 2>/dev/null |
|
||
while IFS='=' read -r key value; do
|
||
if [ "$key" = "VIDEO_TRANSCODE_INSTANCE_ID" ]; then
|
||
printf '%s\n' "$value"
|
||
break
|
||
fi
|
||
done
|
||
}
|
||
|
||
verify_video_transcode_capability() {
|
||
local name="$1"
|
||
local capability
|
||
local instance_id
|
||
capability="$(transcode_capability_file "$name")"
|
||
instance_id="$(container_transcode_instance_id "$name")"
|
||
|
||
[ -n "$instance_id" ] || return 1
|
||
[ -s "$capability" ] || return 1
|
||
grep -Fqx -- 'VIDEO_TRANSCODE_DRAIN_PROTOCOL_V1' "$capability" || return 1
|
||
grep -Fqx -- "workerId=$name" "$capability" || return 1
|
||
grep -Fqx -- "instanceId=$instance_id" "$capability"
|
||
}
|
||
|
||
resume_video_transcode() {
|
||
local name="$1"
|
||
local marker
|
||
marker="$(transcode_drain_file "$name")"
|
||
rm -f -- "$marker" || return 1
|
||
log "Video transcode drain resumed for ${name}"
|
||
}
|
||
|
||
wait_for_video_transcode_drain() {
|
||
local name="$1"
|
||
local max_checks="${2:-90}"
|
||
local active
|
||
local zero_samples=0
|
||
active="$(transcode_active_file "$name")"
|
||
|
||
for _ in $(seq 1 "$max_checks"); do
|
||
if [ -e "$active" ]; then
|
||
zero_samples=0
|
||
log "Waiting for active video transcode lease in ${name}"
|
||
else
|
||
zero_samples=$((zero_samples + 1))
|
||
if [ "$zero_samples" -ge 2 ]; then
|
||
return 0
|
||
fi
|
||
fi
|
||
sleep 2
|
||
done
|
||
|
||
warn "Cannot prove video transcode worker ${name} is inactive; container will not be stopped"
|
||
return 1
|
||
}
|
||
|
||
request_video_transcode_drain() {
|
||
local name="$1"
|
||
local marker
|
||
marker="$(transcode_drain_file "$name")"
|
||
|
||
if ! verify_video_transcode_capability "$name"; then
|
||
warn "Container ${name} does not prove current-instance VIDEO_TRANSCODE_DRAIN_PROTOCOL_V1 support"
|
||
return 2
|
||
fi
|
||
|
||
mkdir -p "$(dirname "$marker")"
|
||
: > "$marker"
|
||
chmod 600 "$marker"
|
||
TRANSCODE_DRAIN_TARGET="$name"
|
||
case " $TRANSCODE_DRAIN_TARGETS " in
|
||
*" $name "*) ;;
|
||
*) TRANSCODE_DRAIN_TARGETS="${TRANSCODE_DRAIN_TARGETS} ${name}" ;;
|
||
esac
|
||
wait_for_video_transcode_drain "$name"
|
||
}
|
||
|
||
TASK_DETAIL_RUNTIME_ROOT="${UPLOADS_MOUNT}/.task-detail-upload-runtime"
|
||
TASK_DETAIL_DRAIN_TARGETS=""
|
||
|
||
prepare_worker_runtime_directories() {
|
||
local app_uid=''
|
||
local app_gid=''
|
||
local runtime_root=''
|
||
app_uid="$(docker run --rm --network none --entrypoint id "$REGISTRY_COMMIT" -u nextjs 2>/dev/null || true)"
|
||
app_gid="$(docker run --rm --network none --entrypoint id "$REGISTRY_COMMIT" -g nextjs 2>/dev/null || true)"
|
||
[[ "$app_uid" =~ ^[1-9][0-9]*$ ]] || return 1
|
||
[[ "$app_gid" =~ ^[1-9][0-9]*$ ]] || return 1
|
||
|
||
for runtime_root in "$TRANSCODE_RUNTIME_ROOT" "$TASK_DETAIL_RUNTIME_ROOT"; do
|
||
if [ -e "$runtime_root" ] && { [ ! -d "$runtime_root" ] || [ -L "$runtime_root" ]; }; then
|
||
return 1
|
||
fi
|
||
install -d -m 0750 -o "$app_uid" -g "$app_gid" "$runtime_root" || return 1
|
||
install -d -m 0750 -o "$app_uid" -g "$app_gid" "$runtime_root/capabilities" "$runtime_root/active" || return 1
|
||
if [ -e "$runtime_root/drain" ] && { [ ! -d "$runtime_root/drain" ] || [ -L "$runtime_root/drain" ]; }; then
|
||
return 1
|
||
fi
|
||
install -d -m 0755 -o 0 -g 0 "$runtime_root/drain" || return 1
|
||
done
|
||
}
|
||
|
||
task_detail_capability_file() {
|
||
printf '%s/capabilities/%s.v1\n' "$TASK_DETAIL_RUNTIME_ROOT" "$1"
|
||
}
|
||
|
||
task_detail_drain_file() {
|
||
printf '%s/drain/%s\n' "$TASK_DETAIL_RUNTIME_ROOT" "$1"
|
||
}
|
||
|
||
task_detail_active_file() {
|
||
printf '%s/active/%s.json\n' "$TASK_DETAIL_RUNTIME_ROOT" "$1"
|
||
}
|
||
|
||
container_task_detail_instance_id() {
|
||
local name="$1"
|
||
docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$name" 2>/dev/null |
|
||
while IFS='=' read -r key value; do
|
||
if [ "$key" = "TASK_DETAIL_UPLOAD_INSTANCE_ID" ]; then
|
||
printf '%s\n' "$value"
|
||
break
|
||
fi
|
||
done
|
||
}
|
||
|
||
verify_task_detail_upload_capability() {
|
||
local name="$1"
|
||
local capability
|
||
local instance_id
|
||
capability="$(task_detail_capability_file "$name")"
|
||
instance_id="$(container_task_detail_instance_id "$name")"
|
||
|
||
[ -n "$instance_id" ] || return 1
|
||
[ -s "$capability" ] || return 1
|
||
grep -Fqx -- 'TASK_DETAIL_UPLOAD_DRAIN_PROTOCOL_V1' "$capability" || return 1
|
||
grep -Fqx -- "workerId=$name" "$capability" || return 1
|
||
grep -Fqx -- "instanceId=$instance_id" "$capability"
|
||
}
|
||
|
||
resume_task_detail_upload() {
|
||
local name="$1"
|
||
rm -f -- "$(task_detail_drain_file "$name")" || return 1
|
||
log "Task-detail upload drain resumed for ${name}"
|
||
}
|
||
|
||
wait_for_task_detail_upload_drain() {
|
||
local name="$1"
|
||
local max_checks="${2:-900}"
|
||
local active
|
||
local zero_samples=0
|
||
active="$(task_detail_active_file "$name")"
|
||
|
||
for _ in $(seq 1 "$max_checks"); do
|
||
if [ -e "$active" ]; then
|
||
zero_samples=0
|
||
log "Waiting for active task-detail upload job in ${name}"
|
||
else
|
||
zero_samples=$((zero_samples + 1))
|
||
if [ "$zero_samples" -ge 2 ]; then
|
||
return 0
|
||
fi
|
||
fi
|
||
sleep 2
|
||
done
|
||
|
||
warn "Cannot prove task-detail upload worker ${name} is inactive; container will not be stopped"
|
||
return 1
|
||
}
|
||
|
||
request_task_detail_upload_drain() {
|
||
local name="$1"
|
||
local marker
|
||
marker="$(task_detail_drain_file "$name")"
|
||
|
||
if ! verify_task_detail_upload_capability "$name"; then
|
||
warn "Container ${name} does not prove current-instance TASK_DETAIL_UPLOAD_DRAIN_PROTOCOL_V1 support"
|
||
return 2
|
||
fi
|
||
|
||
mkdir -p "$(dirname "$marker")"
|
||
: > "$marker"
|
||
chmod 600 "$marker"
|
||
case " $TASK_DETAIL_DRAIN_TARGETS " in
|
||
*" $name "*) ;;
|
||
*) TASK_DETAIL_DRAIN_TARGETS="${TASK_DETAIL_DRAIN_TARGETS} ${name}" ;;
|
||
esac
|
||
wait_for_task_detail_upload_drain "$name"
|
||
}
|
||
|
||
forget_task_detail_drain_target() {
|
||
local forgotten="$1"
|
||
local retained=""
|
||
local target
|
||
for target in $TASK_DETAIL_DRAIN_TARGETS; do
|
||
if [ "$target" != "$forgotten" ]; then
|
||
retained="${retained} ${target}"
|
||
fi
|
||
done
|
||
TASK_DETAIL_DRAIN_TARGETS="$retained"
|
||
}
|
||
|
||
verify_task_detail_upload_lease_schema() {
|
||
local name="$1"
|
||
local result
|
||
result="$(docker exec "$name" node -e '
|
||
const { createHash } = require("crypto");
|
||
const { readFileSync } = require("fs");
|
||
const { PrismaClient } = require("@prisma/client");
|
||
const p = new PrismaClient();
|
||
const migrationName = "20260812_add_product_package_upload_job_lease";
|
||
const migrationPath = `/app/prisma/migrations/${migrationName}/migration.sql`;
|
||
let sourceChecksum = "";
|
||
try {
|
||
sourceChecksum = createHash("sha256")
|
||
.update(readFileSync(migrationPath, "utf8").replace(/\r\n/g, "\n"), "utf8")
|
||
.digest("hex");
|
||
}
|
||
catch (error) { process.stderr.write(String(error && error.message || error)); process.exit(1); }
|
||
Promise.all([
|
||
p.$queryRawUnsafe("SELECT column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = $1 AND column_name IN ($2, $3, $4)", "product_package_upload_jobs", "worker_id", "lease_token", "lease_expires_at"),
|
||
p.$queryRawUnsafe("SELECT indexdef FROM pg_indexes WHERE schemaname = current_schema() AND tablename = $1 AND indexname = $2", "product_package_upload_jobs", "product_package_upload_jobs_lease_idx"),
|
||
p.$queryRawUnsafe("SELECT checksum FROM \"_prisma_migrations\" WHERE migration_name = $1 AND finished_at IS NOT NULL AND rolled_back_at IS NULL", migrationName)
|
||
]).then(([columns, indexes, migrations]) => {
|
||
const byName = new Map(columns.map((row) => [String(row.column_name), row]));
|
||
const textColumnOk = (column) => byName.has(column)
|
||
&& byName.get(column).data_type === "text"
|
||
&& byName.get(column).is_nullable === "YES";
|
||
const timestamp = byName.get("lease_expires_at");
|
||
const columnsOk = textColumnOk("worker_id") && textColumnOk("lease_token")
|
||
&& timestamp && timestamp.data_type === "timestamp without time zone" && timestamp.is_nullable === "YES";
|
||
const indexDefinition = String(indexes[0] && indexes[0].indexdef || "");
|
||
const normalizedIndexDefinition = indexDefinition.replace(/"/g, "");
|
||
const indexOk = indexes.length === 1
|
||
&& /\(\s*status\s*,\s*lease_expires_at\s*\)/i.test(normalizedIndexDefinition);
|
||
const migrationOk = migrations.length === 1 && String(migrations[0].checksum || "") === sourceChecksum;
|
||
if (!columnsOk || !indexOk || !migrationOk) {
|
||
process.stderr.write(JSON.stringify({ columnsOk, indexOk, migrationOk, indexDefinition, sourceChecksum }));
|
||
process.exitCode = 1;
|
||
return;
|
||
}
|
||
process.stdout.write("OK");
|
||
}).catch((error) => {
|
||
process.stderr.write(String(error && error.message || error));
|
||
process.exitCode = 1;
|
||
}).finally(() => p.$disconnect());
|
||
' 2>/dev/null || true)"
|
||
[ "$result" = "OK" ]
|
||
}
|
||
|
||
verify_transcode_lease_schema() {
|
||
local name="$1"
|
||
local result
|
||
result="$(docker exec "$name" node -e '
|
||
const { createHash } = require("crypto");
|
||
const { readFileSync } = require("fs");
|
||
const { PrismaClient } = require("@prisma/client");
|
||
const p = new PrismaClient();
|
||
const migrationName = "20260811_add_video_transcode_lease";
|
||
const migrationPath = `/app/prisma/migrations/${migrationName}/migration.sql`;
|
||
let sourceChecksum = "";
|
||
try {
|
||
sourceChecksum = createHash("sha256")
|
||
.update(readFileSync(migrationPath, "utf8").replace(/\r\n/g, "\n"), "utf8")
|
||
.digest("hex");
|
||
}
|
||
catch (error) { process.stderr.write(String(error && error.message || error)); process.exit(1); }
|
||
Promise.all([
|
||
p.$queryRawUnsafe("SELECT column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = $1 AND column_name IN ($2, $3)", "materials", "transcode_lease_token", "transcode_lease_expires_at"),
|
||
p.$queryRawUnsafe("SELECT indexdef FROM pg_indexes WHERE schemaname = current_schema() AND tablename = $1 AND indexname = $2", "materials", "materials_transcode_lease_idx"),
|
||
p.$queryRawUnsafe("SELECT checksum FROM \"_prisma_migrations\" WHERE migration_name = $1 AND finished_at IS NOT NULL AND rolled_back_at IS NULL", migrationName)
|
||
]).then(([columns, indexes, migrations]) => {
|
||
const byName = new Map(columns.map((row) => [String(row.column_name), row]));
|
||
const token = byName.get("transcode_lease_token");
|
||
const expires = byName.get("transcode_lease_expires_at");
|
||
const columnsOk = columns.length === 2
|
||
&& token && token.data_type === "text" && token.is_nullable === "YES"
|
||
&& expires && expires.data_type === "timestamp without time zone" && expires.is_nullable === "YES";
|
||
const indexDefinition = String(indexes[0] && indexes[0].indexdef || "");
|
||
const indexOk = indexes.length === 1
|
||
&& /^CREATE INDEX\s+\S+\s+ON\s+\S+\s+USING\s+btree\s*\(\s*transcode_lease_expires_at\s*,\s*transcode_lease_token\s*\)\s*$/i.test(indexDefinition.replace(/"/g, ""))
|
||
&& !/\bUNIQUE\b|\bWHERE\b/i.test(indexDefinition);
|
||
const migrationOk = migrations.length === 1 && String(migrations[0].checksum || "") === sourceChecksum;
|
||
if (!columnsOk || !indexOk || !migrationOk) {
|
||
process.stderr.write(JSON.stringify({ columnsOk, indexOk, migrationOk, indexDefinition, sourceChecksum }));
|
||
process.exitCode = 1;
|
||
return;
|
||
}
|
||
process.stdout.write("OK");
|
||
}).catch((error) => {
|
||
process.stderr.write(String(error && error.message || error));
|
||
process.exitCode = 1;
|
||
}).finally(() => p.$disconnect());
|
||
' 2>/dev/null || true)"
|
||
[ "$result" = "OK" ]
|
||
}
|
||
|
||
verify_brand_logo_defaults() {
|
||
local name="$1"
|
||
local result
|
||
result="$(docker exec "$name" node -e '
|
||
const { PrismaClient } = require("@prisma/client");
|
||
const p = new PrismaClient();
|
||
Promise.all([
|
||
p.$queryRawUnsafe("SELECT column_name, column_default FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = $1 AND column_name IN ($2, $3, $4)", "system_settings", "headerLogoUrl", "loginLogoUrl", "logoUrl"),
|
||
p.$queryRawUnsafe("SELECT COUNT(*)::int AS count FROM \"_prisma_migrations\" WHERE migration_name = $1 AND finished_at IS NOT NULL AND rolled_back_at IS NULL", "20260811_update_brand_logo_defaults"),
|
||
p.$queryRawUnsafe("SELECT id, \"headerLogoUrl\", \"loginLogoUrl\" FROM \"system_settings\" ORDER BY id")
|
||
]).then(([columns, migrations, brandRows]) => {
|
||
const defaults = new Map(columns.map((row) => [String(row.column_name), row.column_default]));
|
||
const quote = String.fromCharCode(39);
|
||
const normalize = (value) => {
|
||
let normalized = String(value == null ? "" : value).trim().replace(/::(?:text|character varying)$/i, "");
|
||
if (normalized.startsWith(quote) && normalized.endsWith(quote)) normalized = normalized.slice(1, -1);
|
||
return normalized;
|
||
};
|
||
const headerOk = defaults.has("headerLogoUrl") && normalize(defaults.get("headerLogoUrl")) === "/brand/header-logo.png";
|
||
const loginOk = defaults.has("loginLogoUrl") && normalize(defaults.get("loginLogoUrl")) === "/brand/login-logo.png";
|
||
const legacyOk = !defaults.has("logoUrl") || normalize(defaults.get("logoUrl")) === "/brand/header-logo.png";
|
||
const migrationOk = Number(migrations[0] && migrations[0].count || 0) === 1;
|
||
const isTrustedBrandValue = (value, localDefault) => {
|
||
const normalized = String(value == null ? "" : value).trim();
|
||
if (normalized === localDefault) return true;
|
||
try {
|
||
const parsed = new URL(normalized);
|
||
return parsed.protocol === "https:"
|
||
&& parsed.hostname === "s3.jiansun.vip"
|
||
&& parsed.port === ""
|
||
&& parsed.username === ""
|
||
&& parsed.password === ""
|
||
&& parsed.pathname.startsWith("/gongxiangpan/")
|
||
&& parsed.search === ""
|
||
&& parsed.hash === "";
|
||
} catch {
|
||
return false;
|
||
}
|
||
};
|
||
const rowsOk = brandRows.every((row) =>
|
||
isTrustedBrandValue(row.headerLogoUrl, "/brand/header-logo.png")
|
||
&& isTrustedBrandValue(row.loginLogoUrl, "/brand/login-logo.png")
|
||
);
|
||
if (!headerOk || !loginOk || !legacyOk || !migrationOk || !rowsOk) {
|
||
process.stderr.write(JSON.stringify({ headerOk, loginOk, legacyOk, migrationOk, rowsOk }));
|
||
process.exitCode = 1;
|
||
return;
|
||
}
|
||
process.stdout.write("OK");
|
||
}).catch((error) => {
|
||
process.stderr.write(String(error && error.message || error));
|
||
process.exitCode = 1;
|
||
}).finally(() => p.$disconnect());
|
||
' 2>/dev/null || true)"
|
||
[ "$result" = "OK" ]
|
||
}
|
||
|
||
verify_material_upload_idempotency_schema() {
|
||
local name="$1"
|
||
local result
|
||
result="$(docker exec "$name" node -e '
|
||
const { createHash } = require("crypto");
|
||
const { readFileSync } = require("fs");
|
||
const { PrismaClient } = require("@prisma/client");
|
||
const p = new PrismaClient();
|
||
const historicalMigrationName = "20251118000000_add_upload_id_to_materials";
|
||
const normalizationMigrationName = "20260812_align_material_upload_idempotency_index";
|
||
const migrationChecksum = (name) => createHash("sha256")
|
||
.update(readFileSync(`/app/prisma/migrations/${name}/migration.sql`, "utf8").replace(/\r\n/g, "\n"), "utf8")
|
||
.digest("hex");
|
||
let historicalSourceChecksum = "";
|
||
let normalizationSourceChecksum = "";
|
||
try {
|
||
historicalSourceChecksum = migrationChecksum(historicalMigrationName);
|
||
normalizationSourceChecksum = migrationChecksum(normalizationMigrationName);
|
||
}
|
||
catch (error) { process.stderr.write(String(error && error.message || error)); process.exit(1); }
|
||
Promise.all([
|
||
p.$queryRawUnsafe("SELECT column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = $1 AND column_name = $2", "materials", "uploadId"),
|
||
p.$queryRawUnsafe(`
|
||
SELECT i.indisunique, i.indisprimary, i.indisvalid, i.indisready,
|
||
am.amname AS access_method, i.indnkeyatts, i.indnatts,
|
||
pg_get_expr(i.indpred, i.indrelid) AS predicate,
|
||
pg_get_indexdef(i.indexrelid) AS indexdef,
|
||
ARRAY(
|
||
SELECT a.attname
|
||
FROM unnest(i.indkey) WITH ORDINALITY AS k(attnum, ordinality)
|
||
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum
|
||
WHERE k.ordinality <= i.indnkeyatts
|
||
ORDER BY k.ordinality
|
||
) AS key_columns
|
||
FROM pg_class t
|
||
JOIN pg_namespace n ON n.oid = t.relnamespace
|
||
JOIN pg_index i ON i.indrelid = t.oid
|
||
JOIN pg_class idx ON idx.oid = i.indexrelid
|
||
JOIN pg_am am ON am.oid = idx.relam
|
||
WHERE n.nspname = current_schema() AND t.relname = $1 AND idx.relname = $2
|
||
`, "materials", "materials_uploadId_key"),
|
||
p.$queryRawUnsafe("SELECT migration_name, checksum FROM \"_prisma_migrations\" WHERE migration_name IN ($1, $2) AND finished_at IS NOT NULL AND rolled_back_at IS NULL ORDER BY migration_name", historicalMigrationName, normalizationMigrationName)
|
||
]).then(([columns, indexes, migrations]) => {
|
||
const column = columns[0];
|
||
const columnOk = columns.length === 1 && column
|
||
&& column.column_name === "uploadId"
|
||
&& column.data_type === "text"
|
||
&& column.is_nullable === "YES";
|
||
const index = indexes[0];
|
||
const indexDefinition = String(index && index.indexdef || "");
|
||
const predicate = String(index && index.predicate || "");
|
||
const normalizedPredicate = predicate.replace(/[\"()\s]/g, "").toLowerCase();
|
||
const keyColumns = Array.isArray(index && index.key_columns)
|
||
? index.key_columns.map(String)
|
||
: [];
|
||
const indexOk = indexes.length === 1 && index
|
||
&& index.indisunique === true
|
||
&& index.indisprimary === false
|
||
&& index.indisvalid === true
|
||
&& index.indisready === true
|
||
&& index.access_method === "btree"
|
||
&& Number(index.indnkeyatts) === 1
|
||
&& Number(index.indnatts) === 1
|
||
&& keyColumns.length === 1 && keyColumns[0] === "uploadId"
|
||
&& (normalizedPredicate === "uploadidisnotnullanddeletedatisnull"
|
||
|| normalizedPredicate === "deletedatisnullanduploadidisnotnull");
|
||
const migrationByName = new Map(migrations.map((row) => [String(row.migration_name), String(row.checksum || "")]));
|
||
const migrationOk = migrations.length === 2
|
||
&& migrationByName.get(historicalMigrationName) === historicalSourceChecksum
|
||
&& migrationByName.get(normalizationMigrationName) === normalizationSourceChecksum;
|
||
if (!columnOk || !indexOk || !migrationOk) {
|
||
process.stderr.write(JSON.stringify({ columnOk, indexOk, migrationOk, indexDefinition, predicate, keyColumns, historicalSourceChecksum, normalizationSourceChecksum }));
|
||
process.exitCode = 1;
|
||
return;
|
||
}
|
||
process.stdout.write("OK");
|
||
}).catch((error) => {
|
||
process.stderr.write(String(error && error.message || error));
|
||
process.exitCode = 1;
|
||
}).finally(() => p.$disconnect());
|
||
' 2>/dev/null || true)"
|
||
[ "$result" = "OK" ]
|
||
}
|
||
|
||
BACKGROUND_WORKERS_COMMITTED=0
|
||
|
||
resume_transcode_after_failed_deploy() {
|
||
local status=$?
|
||
trap - EXIT
|
||
if [ "$status" -ne 0 ] && [ "$BACKGROUND_WORKERS_COMMITTED" != "1" ]; then
|
||
for target in $TRANSCODE_DRAIN_TARGETS; do
|
||
resume_video_transcode "$target" || \
|
||
warn "Deployment failed and video worker drain marker could not be removed: $target"
|
||
done
|
||
for target in $TASK_DETAIL_DRAIN_TARGETS; do
|
||
resume_task_detail_upload "$target" || \
|
||
warn "Deployment failed and task-detail worker drain marker could not be removed: $target"
|
||
done
|
||
elif [ "$status" -ne 0 ]; then
|
||
warn "Deployment failed after worker commit; previous background-worker drain markers remain enabled"
|
||
fi
|
||
exit "$status"
|
||
}
|
||
|
||
trap resume_transcode_after_failed_deploy EXIT
|
||
|
||
forget_transcode_drain_target() {
|
||
local forgotten="$1"
|
||
local retained=""
|
||
local target
|
||
for target in $TRANSCODE_DRAIN_TARGETS; do
|
||
if [ "$target" != "$forgotten" ]; then
|
||
retained="${retained} ${target}"
|
||
fi
|
||
done
|
||
TRANSCODE_DRAIN_TARGETS="$retained"
|
||
}
|
||
|
||
remove_video_container_safely() {
|
||
local name="$1"
|
||
if docker inspect "$name" >/dev/null 2>&1; then
|
||
if container_running "$name"; then
|
||
if ! wait_for_video_transcode_drain "$name" 2; then
|
||
warn "Container $name still has active video transcode work and will be preserved"
|
||
return 1
|
||
fi
|
||
if ! wait_for_task_detail_upload_drain "$name" 2; then
|
||
warn "Container $name still has active task-detail upload work and will be preserved"
|
||
return 1
|
||
fi
|
||
if ! docker stop --time 30 "$name" >/dev/null 2>&1; then
|
||
warn "Container $name could not be stopped and will be preserved"
|
||
return 1
|
||
fi
|
||
fi
|
||
if [ -e "$(transcode_active_file "$name")" ]; then
|
||
warn "Container $name left an active marker and will not be removed"
|
||
return 1
|
||
fi
|
||
if [ -e "$(task_detail_active_file "$name")" ]; then
|
||
warn "Container $name left a task-detail active marker and will not be removed"
|
||
return 1
|
||
fi
|
||
docker rm "$name" >/dev/null 2>&1 || return 1
|
||
fi
|
||
rm -f -- \
|
||
"$(transcode_drain_file "$name")" \
|
||
"$(transcode_capability_file "$name")" \
|
||
"$(task_detail_drain_file "$name")" \
|
||
"$(task_detail_capability_file "$name")"
|
||
}
|
||
|
||
restore_unreloaded_proxy_config() {
|
||
[ "$PROXY_CONFIG_REPLACED" = "1" ] || return 0
|
||
if [ -n "$PROXY_CONFIG_BACKUP" ] && [ -f "$PROXY_CONFIG_BACKUP" ]; then
|
||
cp -a "$PROXY_CONFIG_BACKUP" "$PROXY_CONF" || return 1
|
||
else
|
||
rm -f -- "$PROXY_CONF" || return 1
|
||
fi
|
||
PROXY_CONFIG_REPLACED=0
|
||
PROXY_CONFIG_BACKUP=""
|
||
}
|
||
|
||
fail_candidate_before_traffic() {
|
||
local reason="$1"
|
||
local proxy_config_restored=1
|
||
local candidate_removed=1
|
||
|
||
if ! restore_unreloaded_proxy_config; then
|
||
proxy_config_restored=0
|
||
warn "Previous proxy configuration could not be restored after candidate rejection"
|
||
fi
|
||
if ! remove_video_container_safely "$NEXT_CONTAINER"; then
|
||
candidate_removed=0
|
||
warn "Rejected candidate could not be removed safely: $NEXT_CONTAINER"
|
||
fi
|
||
if [ "$proxy_config_restored" = "1" ] && [ "$candidate_removed" = "1" ]; then
|
||
fail "${reason}; candidate was removed and existing entry was kept"
|
||
fi
|
||
fail "${reason}; existing traffic was not switched, but rejection cleanup needs operator attention"
|
||
}
|
||
|
||
maintenance_entry_is_ready_after_restore() {
|
||
local suffix=""
|
||
local url=""
|
||
local http_code=""
|
||
local target_count=""
|
||
local target=""
|
||
|
||
for suffix in ipv4 ipv6; do
|
||
if [ "$suffix" = "ipv4" ]; then
|
||
url="http://127.0.0.1:${PORT}/api/health"
|
||
else
|
||
url="http://[::1]:${PORT}/api/health"
|
||
fi
|
||
http_code="$(curl --noproxy '*' -g -sS --max-time 8 \
|
||
-D "/tmp/git-deploy-maintenance-${suffix}.headers" \
|
||
-o "/tmp/git-deploy-maintenance-${suffix}.body" \
|
||
-w '%{http_code}' "$url")" || return 1
|
||
[ "$http_code" = "503" ] || return 1
|
||
target_count="$(tr -d '\r' <"/tmp/git-deploy-maintenance-${suffix}.headers" | grep -Fic 'X-Material-Proxy-Target:' || true)"
|
||
target="$(tr -d '\r' <"/tmp/git-deploy-maintenance-${suffix}.headers" | sed -nE 's/^X-Material-Proxy-Target:[[:space:]]*(.*)$/\1/Ip' | tail -n 1 || true)"
|
||
[ "$target_count" = "1" ] && [ "$target" = "legacy-bootstrap-maintenance" ] || return 1
|
||
done
|
||
}
|
||
|
||
restore_bootstrap_maintenance_entry() {
|
||
local temp_conf=""
|
||
|
||
[ -n "$READY_BOOTSTRAP_HANDOFF" ] || return 1
|
||
[ -n "$PROXY_CONFIG_BACKUP" ] && [ -f "$PROXY_CONFIG_BACKUP" ] && [ ! -L "$PROXY_CONFIG_BACKUP" ] || return 1
|
||
temp_conf="$(mktemp "${PROXY_CONF}.restore.XXXXXX")" || return 1
|
||
cp -a -- "$PROXY_CONFIG_BACKUP" "$temp_conf" || { rm -f -- "$temp_conf"; return 1; }
|
||
nginx -p "${PROXY_ROOT}/" -c "$temp_conf" -t || { rm -f -- "$temp_conf"; return 1; }
|
||
mv -- "$temp_conf" "$PROXY_CONF" || { rm -f -- "$temp_conf"; return 1; }
|
||
systemctl reload "$PROXY_SERVICE" || return 1
|
||
maintenance_entry_is_ready_after_restore || return 1
|
||
rm -f -- "$CURRENT_FILE" || return 1
|
||
PROXY_CONFIG_REPLACED=0
|
||
PROXY_CONFIG_BACKUP=""
|
||
}
|
||
|
||
rollback_to_previous_entry() {
|
||
local previous_ip=""
|
||
|
||
[ "$ROLLBACK_ON_FAILURE" = "1" ] || return 1
|
||
|
||
if [ "$PROXY_WAS_RUNNING" = "1" ] && [ -n "$CURRENT_CONTAINER" ]; then
|
||
if ! container_running "$CURRENT_CONTAINER"; then
|
||
if ! verify_container_uploads_mount_metadata "$CURRENT_CONTAINER"; then
|
||
warn "Previous container uploads mount metadata is invalid: $CURRENT_CONTAINER"
|
||
return 1
|
||
fi
|
||
if ! verify_container_upload_dir_config "$CURRENT_CONTAINER" 0; then
|
||
warn "Previous container upload root configuration is invalid: $CURRENT_CONTAINER"
|
||
return 1
|
||
fi
|
||
if ! docker start "$CURRENT_CONTAINER" >/dev/null; then
|
||
warn "Failed to restart previous container $CURRENT_CONTAINER"
|
||
return 1
|
||
fi
|
||
if ! verify_container_uploads_statfs "$CURRENT_CONTAINER"; then
|
||
warn "Previous container uploads statfs identity is invalid: $CURRENT_CONTAINER"
|
||
return 1
|
||
fi
|
||
if ! verify_container_upload_dir_runtime "$CURRENT_CONTAINER" 0; then
|
||
warn "Previous container application upload root is invalid: $CURRENT_CONTAINER"
|
||
return 1
|
||
fi
|
||
if ! wait_for_container_health "$CURRENT_CONTAINER" >/dev/null; then
|
||
warn "Previous container did not become healthy: $CURRENT_CONTAINER"
|
||
return 1
|
||
fi
|
||
else
|
||
if ! verify_container_uploads_storage "$CURRENT_CONTAINER"; then
|
||
warn "Previous container uploads storage identity is invalid: $CURRENT_CONTAINER"
|
||
return 1
|
||
fi
|
||
if ! container_healthy "$CURRENT_CONTAINER"; then
|
||
warn "Previous container is not healthy: $CURRENT_CONTAINER"
|
||
return 1
|
||
fi
|
||
fi
|
||
|
||
previous_ip="$(container_ip "$CURRENT_CONTAINER")"
|
||
[ -n "$previous_ip" ] || return 1
|
||
if ! write_proxy_config "${previous_ip}:3000" "$CURRENT_CONTAINER"; then
|
||
return 1
|
||
fi
|
||
if ! switch_proxy; then
|
||
return 1
|
||
fi
|
||
if ! verify_proxy_target "$CURRENT_CONTAINER"; then
|
||
return 1
|
||
fi
|
||
if ! write_current_container "$CURRENT_CONTAINER"; then
|
||
return 1
|
||
fi
|
||
resume_video_transcode "$CURRENT_CONTAINER" || {
|
||
warn "Previous container traffic was restored but video transcode drain could not be resumed"
|
||
return 1
|
||
}
|
||
resume_task_detail_upload "$CURRENT_CONTAINER" || {
|
||
warn "Previous container traffic was restored but task-detail upload drain could not be resumed"
|
||
return 1
|
||
}
|
||
return 0
|
||
fi
|
||
|
||
return 1
|
||
}
|
||
|
||
abort_after_uncertain_switch() {
|
||
local reason="$1"
|
||
|
||
warn "$reason"
|
||
# Traffic may already have reached the candidate. From this point the EXIT
|
||
# trap must never blindly resume either old or new workers; rollback handles
|
||
# the chosen worker explicitly and otherwise leaves both sides drained.
|
||
BACKGROUND_WORKERS_COMMITTED=1
|
||
if [ "${PROXY_LOCK_HELD:-0}" != "1" ]; then
|
||
if ! flock -w 30 8; then
|
||
fail "$reason; proxy reconciliation lock could not be reacquired, so both containers remain drained"
|
||
fi
|
||
PROXY_LOCK_HELD=1
|
||
fi
|
||
if ! request_video_transcode_drain "$NEXT_CONTAINER"; then
|
||
warn "Candidate video transcode worker could not be safely drained; rollback was not attempted"
|
||
fail "$reason; both containers were preserved"
|
||
fi
|
||
if ! request_task_detail_upload_drain "$NEXT_CONTAINER"; then
|
||
warn "Candidate task-detail upload worker could not be safely drained; rollback was not attempted"
|
||
fail "$reason; both containers were preserved"
|
||
fi
|
||
if rollback_to_previous_entry; then
|
||
ok "Previous entry was restored and verified"
|
||
if wait_for_container_drain "$NEXT_CONTAINER" 15; then
|
||
remove_video_container_safely "$NEXT_CONTAINER" || \
|
||
warn "Rolled-back candidate was retained because video activity or stop state is uncertain"
|
||
else
|
||
warn "Rolled-back candidate was retained because an in-flight request is still using it"
|
||
fi
|
||
fail "$reason; deployment was rolled back"
|
||
fi
|
||
|
||
if restore_bootstrap_maintenance_entry; then
|
||
ok "Bootstrap maintenance entry was restored and verified"
|
||
if wait_for_container_drain "$NEXT_CONTAINER" 15; then
|
||
remove_video_container_safely "$NEXT_CONTAINER" || \
|
||
warn "Candidate was retained after maintenance rollback because its stop state is uncertain"
|
||
else
|
||
warn "Candidate was retained after maintenance rollback because an in-flight request is still using it"
|
||
fi
|
||
fail "$reason; deployment returned to the verified bootstrap maintenance entry"
|
||
fi
|
||
|
||
warn "Rollback could not be verified; candidate container is being kept for recovery"
|
||
fail "$reason; both containers were preserved"
|
||
}
|
||
|
||
# The historical shadow topology shared production uploads/runtime while using
|
||
# a separate database lease. A running shadow app therefore makes global FFmpeg
|
||
# exclusivity unknowable; inspect both known names and the Compose service label.
|
||
verify_bootstrap_handoff_if_present
|
||
|
||
if [ "$TARGET" = "production" ]; then
|
||
if ! RUNNING_CONTAINER_METADATA="$(docker ps --format '{{.Names}}\t{{.Label "com.docker.compose.service"}}')"; then
|
||
fail "Cannot inspect running containers for shadow topology; production deployment is blocked"
|
||
fi
|
||
RUNNING_SHADOW_CONTAINERS="$(
|
||
printf '%s\n' "$RUNNING_CONTAINER_METADATA" |
|
||
awk -F '\t' '$1 == "youyidian-3d-material-shadow" || $1 ~ /^youyidian-3d-shadow($|[-_])/ || $2 == "app-shadow" { print $1 }' |
|
||
sort -u
|
||
)"
|
||
if [ -n "$RUNNING_SHADOW_CONTAINERS" ]; then
|
||
fail "Running shadow container(s) detected; production deployment is blocked until they are stopped and verified: $RUNNING_SHADOW_CONTAINERS"
|
||
fi
|
||
fi
|
||
|
||
# This general deployment entry must never perform the one-time direct-to-proxy
|
||
# topology migration. Detect the legacy direct container before starting a
|
||
# candidate, changing traffic, draining workers, or stopping any container.
|
||
LEGACY_IMAGE="$(docker inspect -f '{{.Config.Image}}' "$CONTAINER_NAME" 2>/dev/null || true)"
|
||
PROXY_WAS_RUNNING=0
|
||
if is_proxy_running; then
|
||
PROXY_WAS_RUNNING=1
|
||
elif [ -n "$LEGACY_IMAGE" ]; then
|
||
fail "Stable local proxy is not running while legacy direct container $CONTAINER_NAME exists; no container or traffic was changed. Use the controlled one-time direct-to-proxy migration SOP before rerunning this deployment."
|
||
fi
|
||
if [ -n "$LEGACY_IMAGE" ]; then
|
||
log "Legacy direct container detected behind an already-running stable proxy: $LEGACY_IMAGE"
|
||
fi
|
||
|
||
mkdir -p "$PROXY_ROOT"
|
||
exec 8>"$PROXY_RECONCILE_LOCK"
|
||
flock -w 30 8 || fail "Another proxy switch or reconciliation is already running"
|
||
PROXY_LOCK_HELD=1
|
||
install_proxy_reconciler
|
||
|
||
RECORDED_CONTAINER="$(read_recorded_container)"
|
||
if ! CURRENT_CONTAINER="$(discover_current_container)"; then
|
||
fail "Cannot safely determine the active material container"
|
||
fi
|
||
|
||
if [ -n "$CURRENT_CONTAINER" ] && [ "$CURRENT_CONTAINER" != "$RECORDED_CONTAINER" ]; then
|
||
warn "Repairing stale current-container state: ${RECORDED_CONTAINER:-missing} -> ${CURRENT_CONTAINER}"
|
||
write_current_container "$CURRENT_CONTAINER"
|
||
fi
|
||
|
||
if [ "$DEPLOYMENT_STRATEGY" = "direct" ]; then
|
||
[ -n "$CURRENT_CONTAINER" ] || fail "Direct replacement requires an active material container"
|
||
direct_replace_active_container "$CURRENT_CONTAINER"
|
||
exit 0
|
||
fi
|
||
|
||
if [ "$CURRENT_CONTAINER" = "$BLUE_CONTAINER" ]; then
|
||
NEXT_CONTAINER="$GREEN_CONTAINER"
|
||
else
|
||
NEXT_CONTAINER="$BLUE_CONTAINER"
|
||
fi
|
||
|
||
ensure_public_traffic_rollup_token
|
||
|
||
TRANSCODE_DRAIN_SOURCES=""
|
||
if [ -n "$CURRENT_CONTAINER" ]; then
|
||
TRANSCODE_DRAIN_SOURCES="$CURRENT_CONTAINER"
|
||
fi
|
||
if [ -n "$LEGACY_IMAGE" ] && container_running "$CONTAINER_NAME"; then
|
||
case " $TRANSCODE_DRAIN_SOURCES " in
|
||
*" $CONTAINER_NAME "*) ;;
|
||
*) TRANSCODE_DRAIN_SOURCES="${TRANSCODE_DRAIN_SOURCES} ${CONTAINER_NAME}" ;;
|
||
esac
|
||
fi
|
||
|
||
# 蓝绿入口和意外残留的直连容器都可能运行 worker；必须逐个证明当前协议、
|
||
# 设置 drain 并等待 active 消失。任一状态不明都保留原容器并停止发布。
|
||
for TRANSCODE_DRAIN_SOURCE in $TRANSCODE_DRAIN_SOURCES; do
|
||
if ! verify_container_uploads_storage "$TRANSCODE_DRAIN_SOURCE"; then
|
||
fail "Existing container uploads mount could not be verified before drain; candidate was not created"
|
||
fi
|
||
if ! verify_transcode_lease_schema "$TRANSCODE_DRAIN_SOURCE"; then
|
||
fail "Video transcode lease columns are missing or could not be verified; apply the additive Prisma migration before deployment"
|
||
fi
|
||
if ! verify_task_detail_upload_lease_schema "$TRANSCODE_DRAIN_SOURCE"; then
|
||
fail "Task-detail upload job lease schema or migration checksum is missing; apply and audit the additive Prisma migration before deployment"
|
||
fi
|
||
if ! verify_material_upload_idempotency_schema "$TRANSCODE_DRAIN_SOURCE"; then
|
||
fail "Existing container cannot verify the exact material uploadId column, unique index, and migration checksum; candidate was not created"
|
||
fi
|
||
if ! request_video_transcode_drain "$TRANSCODE_DRAIN_SOURCE"; then
|
||
fail "Existing container cannot be safely drained for video transcode; candidate was not started (first rollout fails closed)"
|
||
fi
|
||
if ! request_task_detail_upload_drain "$TRANSCODE_DRAIN_SOURCE"; then
|
||
fail "Existing container cannot be safely drained for task-detail uploads; candidate was not started (first rollout fails closed)"
|
||
fi
|
||
ok "Existing video transcode worker is drained: $TRANSCODE_DRAIN_SOURCE"
|
||
ok "Existing task-detail upload worker is drained: $TRANSCODE_DRAIN_SOURCE"
|
||
done
|
||
|
||
log "Starting candidate container $NEXT_CONTAINER"
|
||
if ! verify_image_revision_label "$REGISTRY_COMMIT" "$COMMIT_SHA"; then
|
||
fail "Candidate image label does not match the exact preflighted Git commit; no container or traffic was changed"
|
||
fi
|
||
if container_running "$NEXT_CONTAINER"; then
|
||
if ! verify_container_uploads_storage "$NEXT_CONTAINER"; then
|
||
fail "Inactive deployment slot $NEXT_CONTAINER has an untrusted uploads mount and cannot be safely drained"
|
||
fi
|
||
if ! request_video_transcode_drain "$NEXT_CONTAINER"; then
|
||
fail "Inactive deployment slot $NEXT_CONTAINER cannot prove video transcode drain support"
|
||
fi
|
||
if ! request_task_detail_upload_drain "$NEXT_CONTAINER"; then
|
||
fail "Inactive deployment slot $NEXT_CONTAINER cannot prove task-detail upload drain support"
|
||
fi
|
||
if ! wait_for_container_drain "$NEXT_CONTAINER" 15; then
|
||
fail "Inactive deployment slot $NEXT_CONTAINER is still serving requests; retry after those requests finish"
|
||
fi
|
||
if ! wait_for_video_transcode_drain "$NEXT_CONTAINER" 2; then
|
||
fail "Inactive deployment slot $NEXT_CONTAINER has unknown video activity and will not be removed"
|
||
fi
|
||
if ! wait_for_task_detail_upload_drain "$NEXT_CONTAINER" 2; then
|
||
fail "Inactive deployment slot $NEXT_CONTAINER has unknown task-detail upload activity and will not be removed"
|
||
fi
|
||
fi
|
||
if ! remove_video_container_safely "$NEXT_CONTAINER"; then
|
||
fail "Inactive deployment slot $NEXT_CONTAINER could not be removed safely"
|
||
fi
|
||
forget_transcode_drain_target "$NEXT_CONTAINER"
|
||
forget_task_detail_drain_target "$NEXT_CONTAINER"
|
||
if ! prepare_worker_runtime_directories; then
|
||
fail "Worker runtime directories could not be safely assigned to the verified application UID/GID"
|
||
fi
|
||
mkdir -p "$(dirname "$(transcode_drain_file "$NEXT_CONTAINER")")"
|
||
: > "$(transcode_drain_file "$NEXT_CONTAINER")"
|
||
chmod 600 "$(transcode_drain_file "$NEXT_CONTAINER")"
|
||
mkdir -p "$(dirname "$(task_detail_drain_file "$NEXT_CONTAINER")")"
|
||
: > "$(task_detail_drain_file "$NEXT_CONTAINER")"
|
||
chmod 600 "$(task_detail_drain_file "$NEXT_CONTAINER")"
|
||
if ! create_container "$NEXT_CONTAINER" "$REGISTRY_COMMIT" 0 >/tmp/git-deploy-new-container-id.txt; then
|
||
fail_candidate_before_traffic "Candidate container could not be created"
|
||
fi
|
||
cat /tmp/git-deploy-new-container-id.txt
|
||
if ! verify_container_uploads_mount_metadata "$NEXT_CONTAINER"; then
|
||
fail_candidate_before_traffic "Candidate uploads bind metadata failed before docker start"
|
||
fi
|
||
if ! verify_container_upload_dir_config "$NEXT_CONTAINER" 1; then
|
||
fail_candidate_before_traffic "Candidate UPLOAD_DIR is not pinned to /app/uploads before docker start"
|
||
fi
|
||
if ! docker start "$NEXT_CONTAINER" >/dev/null; then
|
||
fail_candidate_before_traffic "Candidate container could not be started"
|
||
fi
|
||
if ! verify_container_uploads_statfs "$NEXT_CONTAINER"; then
|
||
fail_candidate_before_traffic "Candidate host/container statfs identity differs before health checks"
|
||
fi
|
||
if ! verify_container_upload_dir_runtime "$NEXT_CONTAINER" 1; then
|
||
fail_candidate_before_traffic "Candidate application upload root is not exactly /app/uploads"
|
||
fi
|
||
if ! verify_container_baked_revision "$NEXT_CONTAINER" "$COMMIT_SHA"; then
|
||
fail_candidate_before_traffic "Candidate image label or baked commit file does not match the exact Git commit"
|
||
fi
|
||
|
||
if ! NEXT_IP="$(wait_for_container_health "$NEXT_CONTAINER" "$COMMIT_SHA")"; then
|
||
warn "Candidate container failed health check"
|
||
if ! docker logs --tail 120 "$NEXT_CONTAINER" 2>&1; then
|
||
warn "Candidate logs could not be read"
|
||
fi
|
||
fail_candidate_before_traffic "Deployment failed before traffic switch"
|
||
fi
|
||
|
||
ok "Candidate container is healthy at ${NEXT_IP}:3000"
|
||
if ! verify_transcode_lease_schema "$NEXT_CONTAINER"; then
|
||
fail_candidate_before_traffic "Candidate cannot verify required video transcode lease columns; traffic was not switched"
|
||
fi
|
||
if ! verify_task_detail_upload_lease_schema "$NEXT_CONTAINER"; then
|
||
fail_candidate_before_traffic "Candidate cannot verify task-detail upload job lease columns, index, and migration checksum; traffic was not switched"
|
||
fi
|
||
if ! verify_brand_logo_defaults "$NEXT_CONTAINER"; then
|
||
fail_candidate_before_traffic "Candidate cannot verify safe brand logo database defaults and completed migration; traffic was not switched"
|
||
fi
|
||
if ! verify_material_upload_idempotency_schema "$NEXT_CONTAINER"; then
|
||
fail_candidate_before_traffic "Candidate cannot verify material upload idempotency column, unique index, and completed migration; traffic was not switched"
|
||
fi
|
||
if ! verify_candidate_database_identity "$NEXT_CONTAINER"; then
|
||
fail_candidate_before_traffic "Candidate database identity differs from the approved migration audit; traffic was not switched"
|
||
fi
|
||
if ! verify_candidate_brand_and_storage_dependencies "$NEXT_CONTAINER" "$NEXT_IP"; then
|
||
fail_candidate_before_traffic "Candidate cannot authenticate to S3 or fully decode its configured and baked brand images; traffic was not switched"
|
||
fi
|
||
if ! verify_video_transcode_capability "$NEXT_CONTAINER"; then
|
||
fail_candidate_before_traffic "Candidate did not publish the required video transcode drain capability"
|
||
fi
|
||
if ! verify_task_detail_upload_capability "$NEXT_CONTAINER"; then
|
||
fail_candidate_before_traffic "Candidate did not publish the required task-detail upload drain capability"
|
||
fi
|
||
if ! wait_for_video_transcode_drain "$NEXT_CONTAINER" 2; then
|
||
fail_candidate_before_traffic "Candidate started despite its pre-set drain marker; traffic was not switched"
|
||
fi
|
||
if ! wait_for_task_detail_upload_drain "$NEXT_CONTAINER" 2; then
|
||
fail_candidate_before_traffic "Candidate started despite its pre-set task-detail drain marker; traffic was not switched"
|
||
fi
|
||
if ! write_proxy_config "${NEXT_IP}:3000" "$NEXT_CONTAINER"; then
|
||
fail_candidate_before_traffic "Candidate proxy configuration could not be prepared"
|
||
fi
|
||
if ! verify_container_uploads_storage "$NEXT_CONTAINER" 1; then
|
||
fail_candidate_before_traffic "Candidate uploads storage identity changed immediately before traffic switch"
|
||
fi
|
||
|
||
SWITCH_REPORTED_FAILURE=0
|
||
# nginx reload can complete before the shell receives its status. From this
|
||
# instruction onward traffic may have reached the candidate, so the generic
|
||
# EXIT trap must never resume the old workers on its own.
|
||
BACKGROUND_WORKERS_COMMITTED=1
|
||
if ! switch_proxy; then
|
||
SWITCH_REPORTED_FAILURE=1
|
||
fi
|
||
|
||
if ! verify_proxy_target "$NEXT_CONTAINER"; then
|
||
cat /tmp/git-deploy-entry-health.err >&2 2>/dev/null || true
|
||
abort_after_uncertain_switch "Stable local entry did not verify the candidate target"
|
||
fi
|
||
|
||
if [ "$SWITCH_REPORTED_FAILURE" = "1" ]; then
|
||
warn "Proxy switch command reported failure, but the runtime candidate target was independently verified"
|
||
fi
|
||
|
||
if ! write_current_container "$NEXT_CONTAINER"; then
|
||
abort_after_uncertain_switch "Candidate traffic was verified but the committed target record could not be written"
|
||
fi
|
||
|
||
sleep 3
|
||
if ! verify_proxy_target "$NEXT_CONTAINER" "$COMMIT_SHA"; then
|
||
abort_after_uncertain_switch "Candidate target did not remain healthy during the post-switch check"
|
||
fi
|
||
|
||
if ! resume_video_transcode "$NEXT_CONTAINER"; then
|
||
abort_after_uncertain_switch "Candidate traffic is stable but its video transcode drain could not be resumed"
|
||
fi
|
||
if ! resume_task_detail_upload "$NEXT_CONTAINER"; then
|
||
abort_after_uncertain_switch "Candidate traffic is stable but its task-detail upload drain could not be resumed"
|
||
fi
|
||
BACKGROUND_WORKERS_COMMITTED=1
|
||
ok "Candidate is the committed background worker: $NEXT_CONTAINER"
|
||
|
||
if [ -n "$CURRENT_CONTAINER" ] && [ "$CURRENT_CONTAINER" != "$NEXT_CONTAINER" ]; then
|
||
if [ "$(runtime_proxy_container)" != "$NEXT_CONTAINER" ] || \
|
||
[ "$(read_recorded_container)" != "$NEXT_CONTAINER" ] || \
|
||
! container_healthy "$NEXT_CONTAINER" "$COMMIT_SHA" || \
|
||
! verify_container_uploads_storage "$CURRENT_CONTAINER"; then
|
||
abort_after_uncertain_switch "Proxy state changed before the previous container could be drained"
|
||
fi
|
||
|
||
if wait_for_container_drain "$CURRENT_CONTAINER" 15; then
|
||
if wait_for_video_transcode_drain "$CURRENT_CONTAINER" 2 && \
|
||
wait_for_task_detail_upload_drain "$CURRENT_CONTAINER" 2 && \
|
||
docker stop --time 30 "$CURRENT_CONTAINER" >/dev/null 2>&1; then
|
||
ok "Previous container stopped and retained for rollback: $CURRENT_CONTAINER"
|
||
else
|
||
warn "Previous container background-worker activity is not proven idle; it was retained in drain mode: $CURRENT_CONTAINER"
|
||
fi
|
||
else
|
||
warn "Previous container is still serving an in-flight request and was retained: $CURRENT_CONTAINER"
|
||
fi
|
||
fi
|
||
|
||
if ! verify_proxy_target "$NEXT_CONTAINER" "$COMMIT_SHA"; then
|
||
abort_after_uncertain_switch "Candidate target failed after the previous container drain"
|
||
fi
|
||
|
||
if ! flock -u 8; then
|
||
PROXY_LOCK_HELD=0
|
||
abort_after_uncertain_switch "Proxy reconciliation lock could not be released for post-deployment verification"
|
||
fi
|
||
PROXY_LOCK_HELD=0
|
||
if ! systemctl start "$PROXY_RECONCILE_SERVICE"; then
|
||
abort_after_uncertain_switch "Proxy reconciler failed its post-deployment verification"
|
||
fi
|
||
ok "Stable local entry is healthy on port $PORT"
|
||
docker logs --tail 20 "$NEXT_CONTAINER" 2>&1 || true
|
||
|
||
if install_public_traffic_rollup_timers; then
|
||
if [ "$TARGET" = "production" ]; then
|
||
if systemctl start "$PUBLIC_TRAFFIC_ROLLUP_SERVICE" >/tmp/public-traffic-rollup-sync.log 2>/tmp/public-traffic-rollup-sync.err; then
|
||
ok "Public traffic statistics rollups synced"
|
||
else
|
||
warn "Public traffic statistics rollup sync failed; page remains available, check /tmp/public-traffic-rollup-sync.err"
|
||
cat /tmp/public-traffic-rollup-sync.err >&2 || true
|
||
fi
|
||
|
||
backfill_start="$(TZ=Asia/Shanghai date -d '7 days ago' +%F)"
|
||
backfill_end="$(TZ=Asia/Shanghai date -d 'yesterday' +%F)"
|
||
if docker exec "$NEXT_CONTAINER" python3 /app/scripts/public-traffic-rollup-from-nginx.py --granularity day --start-date "$backfill_start" --end-date "$backfill_end" >/tmp/public-traffic-rollup-backfill.log 2>/tmp/public-traffic-rollup-backfill.err; then
|
||
ok "Public traffic historical day rollups backfilled from ${backfill_start} to ${backfill_end}"
|
||
else
|
||
warn "Public traffic historical backfill failed; check /tmp/public-traffic-rollup-backfill.err"
|
||
cat /tmp/public-traffic-rollup-backfill.err >&2 || true
|
||
fi
|
||
fi
|
||
else
|
||
warn "Public traffic statistics rollup timers were not installed"
|
||
fi
|
||
|
||
if [ -n "$LEGACY_IMAGE" ]; then
|
||
if ! remove_video_container_safely "$CONTAINER_NAME"; then
|
||
abort_after_uncertain_switch "Legacy direct container could not be proven inactive and was preserved"
|
||
fi
|
||
fi
|
||
|
||
if [ "$TARGET" = "production" ]; then
|
||
log "Checking public FRP, authenticated S3, and Logo decode acceptance under the deployment lock"
|
||
COMMITTED_PUBLIC_TARGET="$(tr -d '\r\n' < "$CURRENT_FILE" 2>/dev/null || true)"
|
||
[ "$COMMITTED_PUBLIC_TARGET" = "$NEXT_CONTAINER" ] || abort_after_uncertain_switch "Production did not report the committed candidate target after cutover"
|
||
curl -fsS --max-time 12 -D /tmp/git-deploy-public-health.headers \
|
||
"$PUBLIC_APP_URL/api/health" >/tmp/git-deploy-public-health.json || \
|
||
abort_after_uncertain_switch "Public application endpoint is unreachable after cutover"
|
||
health_payload_is_ready /tmp/git-deploy-public-health.json "$COMMIT_SHA" || \
|
||
abort_after_uncertain_switch "Public application endpoint does not prove the expected commit and healthy database/Redis after cutover"
|
||
public_target_count="$(tr -d '\r' </tmp/git-deploy-public-health.headers | grep -Fic 'X-Material-Proxy-Target:' || true)"
|
||
public_target="$(tr -d '\r' </tmp/git-deploy-public-health.headers | sed -nE 's/^X-Material-Proxy-Target:[[:space:]]*(.*)$/\1/Ip' | tail -n 1 || true)"
|
||
[ "$public_target_count" = "1" ] && [ "$public_target" = "$COMMITTED_PUBLIC_TARGET" ] || \
|
||
abort_after_uncertain_switch "Public application endpoint does not point uniquely to the deployed candidate after cutover"
|
||
|
||
verify_candidate_authenticated_s3 "$NEXT_CONTAINER" || abort_after_uncertain_switch "Candidate lost authenticated access to the configured S3 bucket after cutover"
|
||
curl -fsS --max-time 12 "https://s3.jiansun.vip/minio/health/live" >/tmp/git-deploy-public-s3-health.txt || \
|
||
abort_after_uncertain_switch "Public S3 endpoint is unhealthy after cutover"
|
||
curl -fsS --max-time 12 "$PUBLIC_APP_URL/api/system-settings" >/tmp/git-deploy-public-settings.json || \
|
||
abort_after_uncertain_switch "Public system settings are unavailable after cutover"
|
||
PUBLIC_APP_URL="$PUBLIC_APP_URL" docker exec -i -e PUBLIC_APP_URL="$PUBLIC_APP_URL" "$NEXT_CONTAINER" node -e '
|
||
const fs = require("fs");
|
||
(async () => {
|
||
const chunks = [];
|
||
for await (const chunk of process.stdin) chunks.push(Buffer.from(chunk));
|
||
const input = Buffer.concat(chunks).toString("utf8");
|
||
const settings = JSON.parse(input);
|
||
const origin = new URL(process.env.PUBLIC_APP_URL);
|
||
const values = [settings.headerLogoUrl, settings.loginLogoUrl];
|
||
if (!values.every((value) => typeof value === "string" && value.length > 0)) process.exit(1);
|
||
for (const value of values) {
|
||
const parsed = new URL(value, origin);
|
||
const local = parsed.origin === origin.origin && /^\/brand\/[A-Za-z0-9][A-Za-z0-9._/-]*$/.test(parsed.pathname)
|
||
&& !parsed.search && !parsed.hash;
|
||
const s3 = parsed.protocol === "https:" && parsed.hostname === "s3.jiansun.vip"
|
||
&& parsed.port === "" && parsed.pathname.startsWith("/gongxiangpan/") && !parsed.search && !parsed.hash;
|
||
if (!local && !s3) process.exit(1);
|
||
}
|
||
process.stdout.write(values.join("\n") + "\n");
|
||
})().catch((error) => { process.stderr.write(String(error && error.message || error)); process.exitCode = 1; });
|
||
' </tmp/git-deploy-public-settings.json >/tmp/git-deploy-public-logo-urls.txt || \
|
||
abort_after_uncertain_switch "Public system settings did not expose two trusted Logo URLs"
|
||
|
||
logo_index=0
|
||
while IFS= read -r logo_url; do
|
||
logo_index=$((logo_index + 1))
|
||
case "$logo_url" in /*) resolved_logo_url="${PUBLIC_APP_URL%/}${logo_url}" ;; *) resolved_logo_url="$logo_url" ;; esac
|
||
curl -fsS --max-time 12 -D "/tmp/git-deploy-logo-${logo_index}.headers" \
|
||
-o "/tmp/git-deploy-logo-${logo_index}.body" "$resolved_logo_url" || \
|
||
abort_after_uncertain_switch "Configured public Logo ${logo_index} is unavailable after cutover"
|
||
logo_content_type="$(tr -d '\r' <"/tmp/git-deploy-logo-${logo_index}.headers" | sed -nE 's/^Content-Type:[[:space:]]*([^;[:space:]]+).*$/\1/Ip' | tail -n 1 | tr 'A-Z' 'a-z')"
|
||
case "$logo_content_type" in image/*) ;; *) abort_after_uncertain_switch "Configured public Logo ${logo_index} did not return an image Content-Type" ;; esac
|
||
decode_image_stream_with_candidate "$NEXT_CONTAINER" < "/tmp/git-deploy-logo-${logo_index}.body" || abort_after_uncertain_switch "Configured public Logo ${logo_index} cannot be fully decoded"
|
||
done </tmp/git-deploy-public-logo-urls.txt
|
||
for fallback_logo in header-logo.png login-logo.png; do
|
||
curl -fsS --max-time 12 -D /tmp/git-deploy-fallback-logo.headers \
|
||
-o /tmp/git-deploy-fallback-logo.body "$PUBLIC_APP_URL/brand/$fallback_logo" || \
|
||
abort_after_uncertain_switch "Local fallback Logo $fallback_logo is unavailable after cutover"
|
||
fallback_content_type="$(tr -d '\r' </tmp/git-deploy-fallback-logo.headers | sed -nE 's/^Content-Type:[[:space:]]*([^;[:space:]]+).*$/\1/Ip' | tail -n 1 | tr 'A-Z' 'a-z')"
|
||
case "$fallback_content_type" in image/*) ;; *) abort_after_uncertain_switch "Local fallback Logo $fallback_logo did not return an image Content-Type" ;; esac
|
||
decode_image_stream_with_candidate "$NEXT_CONTAINER" </tmp/git-deploy-fallback-logo.body || abort_after_uncertain_switch "Local fallback Logo $fallback_logo cannot be fully decoded"
|
||
done
|
||
verify_candidate_baked_brand_assets "$NEXT_CONTAINER" || abort_after_uncertain_switch "Baked brand image assets cannot be fully decoded after cutover"
|
||
ok "Public commit, target, database, Redis, authenticated S3, and all Logo images passed acceptance"
|
||
fi
|
||
|
||
[ -z "$READY_BOOTSTRAP_HANDOFF" ] || {
|
||
mv -- "$READY_BOOTSTRAP_HANDOFF/handoff.ready" "$READY_BOOTSTRAP_HANDOFF/consumed" || \
|
||
abort_after_uncertain_switch "Bootstrap handoff could not be atomically consumed after acceptance"
|
||
chmod 600 "$READY_BOOTSTRAP_HANDOFF/consumed" || \
|
||
abort_after_uncertain_switch "Consumed bootstrap handoff permissions could not be secured"
|
||
}
|
||
PROD_SCRIPT
|
||
|
||
printf '%s' "$PROD_PAYLOAD" | prod_ssh_script \
|
||
"$TARGET" "$CONTAINER_NAME" "$PORT" "$REGISTRY_COMMIT" "$EXPECTED_IMAGE_ID" "$COMMIT_SHA" \
|
||
"/mnt/PoolOne/apps/youyidian-3d/$ENV_FILE" "$PUBLIC_APP_URL" "$CORS_ALLOWED_ORIGINS" \
|
||
"$DINGTALK_CALLBACK_ORIGIN" "$DINGTALK_INTERNAL_RETURN_ORIGINS" "$S3_PUBLIC_DOMAIN" \
|
||
"$DOWNLOAD_TOKEN_SECRET" "$PROXY_RECONCILE_SCRIPT_B64" 'youyidian-3d_app-network' \
|
||
"$UPLOADS_MOUNT" "$ROLLBACK_ON_FAILURE" "$MIGRATION_AUDIT_FILE" "$MIGRATION_AUDIT_SHA256" \
|
||
"$MIGRATION_AUDIT_SOURCE_COMMIT" "$REQUIRE_MIGRATION_AUDIT" "$DEPLOYMENT_STRATEGY"
|
||
ok "Production deployment command completed"
|
||
|
||
if [ "$TARGET" = "production" ]; then
|
||
ok "Public commit, target, database, Redis, authenticated S3 and Logo decode acceptance completed under the production deployment lock"
|
||
fi
|
||
else
|
||
warn "Skipping production deployment by request"
|
||
fi
|
||
|
||
start_remote_step "Cleanup"
|
||
if [ "$KEEP_OLD_IMAGES" = "1" ]; then
|
||
warn "Skipping cleanup by request"
|
||
else
|
||
docker image prune -f >/dev/null 2>&1 || true
|
||
if [ "$AGGRESSIVE_CLEANUP" = "1" ]; then
|
||
docker builder prune -af --filter until=168h >/dev/null 2>&1 || true
|
||
ok "Build server cleanup completed, including old build cache"
|
||
else
|
||
ok "Build server image cleanup completed; build cache preserved"
|
||
fi
|
||
|
||
prod_ssh \
|
||
"docker image prune -af --filter until=6h >/dev/null 2>&1 || true"
|
||
ok "Production server cleanup completed"
|
||
fi
|
||
|
||
start_remote_step "Deployment Summary"
|
||
ok "Target: $TARGET"
|
||
ok "Commit: $COMMIT_SHA"
|
||
ok "Image: $REGISTRY_COMMIT"
|
||
ok "Container: $CONTAINER_NAME"
|
||
'@
|
||
|
||
Write-Step "Git Incremental Deploy"
|
||
Write-Info "Target: $Target"
|
||
Write-Info "Repository: $RepoUrl"
|
||
Write-Info "Branch: $Branch"
|
||
if ($Commit) {
|
||
Write-Info "Commit override: $Commit"
|
||
}
|
||
if ($DryRun) {
|
||
Write-Warning-Custom 'DryRun mode: no code, image, or production container will be changed.'
|
||
}
|
||
|
||
Write-Info 'Resolving and scanning the exact Git tree that will be deployed...'
|
||
$resolvedTargetCommit = Invoke-LegacyStorageBoundaryPreflight `
|
||
-RepositoryUrl $RepoUrl `
|
||
-TargetBranch $Branch `
|
||
-RequestedCommit $Commit
|
||
$tokens['__COMMIT__'] = ConvertTo-ShellSingleQuoted $resolvedTargetCommit
|
||
Write-Success "Exact deployment target passed legacy storage boundary preflight: $resolvedTargetCommit"
|
||
|
||
$remoteScript = $remoteTemplate
|
||
foreach ($key in $tokens.Keys) {
|
||
$remoteScript = $remoteScript.Replace($key, $tokens[$key])
|
||
}
|
||
|
||
$exitCode = Invoke-BuildServerScript -ScriptText $remoteScript
|
||
if ($exitCode -ne 0) {
|
||
Write-Error-Custom "Git incremental deployment failed with exit code $exitCode"
|
||
exit $exitCode
|
||
}
|
||
|
||
Write-Success 'Git incremental deployment finished.'
|