diff --git a/scripts/erp-daily-bills/README.md b/scripts/erp-daily-bills/README.md index bf67e1f6..9c5c2336 100644 --- a/scripts/erp-daily-bills/README.md +++ b/scripts/erp-daily-bills/README.md @@ -26,6 +26,8 @@ pwsh -File scripts\erp-daily-bills\run-erp-daily.ps1 -Platform toutiao -Through - **无差异跳过**:该平台该月已导入且笔数/收入/支出完全一致时跳过导入(`-ForceImport` 可强制重刷);状态记在 `runtime/private/erp-exports/imported-state.json`。 - 超过胜算同步导出上限(响应 `超出同步导出允许最大数据量`)时,脚本自动轮询「异步导出下载管理」取回本次任务的全部分片,逐片转 CSV 后合并成一份再导入;仍可用 `-Start/-Through` 分段导出。 - 导入前用页面笔数(共 N 条)与导出行数核对,不一致就跳过导入(`-ForceImport` 可强制),避免只导入一部分。 +- 稳定性:导出失败自动重试一次(胜算页面偶发日历面板/授权弹窗异常);上传与"导入+回验"失败自动重试 3 次(含重新上传 CSV),用于吸收服务器 SSH 偶发断连。 +- 失败即失败:导出/转 CSV/笔数核对/上传/导入回验任一步失败,都会**不写本地状态、不归档**,并在结束时以退出码 1 结束(下次运行会重新导入),不会把失败当成功记入 `imported-state.json`。 ## 组成 diff --git a/scripts/erp-daily-bills/erp-remote.ps1 b/scripts/erp-daily-bills/erp-remote.ps1 new file mode 100644 index 00000000..536bd554 --- /dev/null +++ b/scripts/erp-daily-bills/erp-remote.ps1 @@ -0,0 +1,77 @@ +<# + 每日账单脚本的远程执行辅助:上传 + 远程执行 + 失败重试。 + + 远端命令失败(包括 SSH 连接被服务器断开)时不再静默当成功: + 统一返回 { Ok; Output; Attempts },由调用方决定是否写本地状态、是否归档。 + 上传与远程执行都按 Attempts 重试;重试时会重新上传 CSV, + 避免上一次失败已经把远端文件删掉导致二次失败。 +#> + +function New-ErpRemoteResult { + param([bool]$Ok, [string]$Output, [int]$Attempts) + + return [pscustomobject]@{ Ok = $Ok; Output = $Output; Attempts = $Attempts } +} + +function Invoke-ErpRemoteUpload { + param( + [Parameter(Mandatory = $true)][string]$PwshPath, + [Parameter(Mandatory = $true)][string]$UploaderScript, + [Parameter(Mandatory = $true)][string]$LocalFile, + [Parameter(Mandatory = $true)][string]$RemoteFile + ) + + $out = (& $PwshPath -NoProfile -ExecutionPolicy Bypass -File $UploaderScript -LocalPath $LocalFile -RemotePath $RemoteFile 2>&1 | Out-String) + + return (New-ErpRemoteResult -Ok ($LASTEXITCODE -eq 0) -Output $out -Attempts 1) +} + +function Invoke-ErpRemote { + param( + [Parameter(Mandatory = $true)][string]$PwshPath, + [Parameter(Mandatory = $true)][string]$SshScript, + [Parameter(Mandatory = $true)][string]$Body, + [string]$LocalFile = '', + [string]$RemoteFile = '', + [string]$UploaderScript = '', + [int]$Attempts = 3, + [int]$RetryDelaySeconds = 10, + [scriptblock]$Log = $null + ) + + $write = { param([string]$Message) if ($Log) { & $Log $Message } } + $tmp = [IO.Path]::GetTempFileName() + '.sh' + [IO.File]::WriteAllText($tmp, ($Body -replace "`r`n", "`n")) + try { + $output = '' + for ($attempt = 1; $attempt -le $Attempts; $attempt++) { + if ($LocalFile -and $RemoteFile -and $UploaderScript) { + $upload = Invoke-ErpRemoteUpload -PwshPath $PwshPath -UploaderScript $UploaderScript -LocalFile $LocalFile -RemoteFile $RemoteFile + if (-not $upload.Ok) { + $output = $upload.Output + & $write ('上传失败(第 {0}/{1} 次):{2}' -f $attempt, $Attempts, ($output.Trim() -replace "`r?`n", ' | ')) + if ($attempt -lt $Attempts) { + Start-Sleep -Seconds $RetryDelaySeconds + continue + } + + return (New-ErpRemoteResult -Ok $false -Output $output -Attempts $attempt) + } + } + + $output = (& $PwshPath -NoProfile -ExecutionPolicy Bypass -File $SshScript -ScriptPath $tmp 2>&1 | Out-String) + if ($LASTEXITCODE -eq 0) { + return (New-ErpRemoteResult -Ok $true -Output $output -Attempts $attempt) + } + + & $write ('远程执行失败(第 {0}/{1} 次,exit={2}):{3}' -f $attempt, $Attempts, $LASTEXITCODE, ($output.Trim() -replace "`r?`n", ' | ')) + if ($attempt -lt $Attempts) { + Start-Sleep -Seconds $RetryDelaySeconds + } + } + + return (New-ErpRemoteResult -Ok $false -Output $output -Attempts $Attempts) + } finally { + Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue + } +} diff --git a/scripts/erp-daily-bills/run-erp-daily.ps1 b/scripts/erp-daily-bills/run-erp-daily.ps1 index fcd76949..00b9fa7a 100644 --- a/scripts/erp-daily-bills/run-erp-daily.ps1 +++ b/scripts/erp-daily-bills/run-erp-daily.ps1 @@ -53,30 +53,51 @@ $state = if (Test-Path -LiteralPath $stateFile) { (Get-Content -LiteralPath $sta $auth = Start-CaiwuDeploymentAuth -Server '218.93.190.75' -SshUser 'root' $sshOptions = $auth.Options + @('-o', 'PreferredAuthentications=password', '-o', 'PubkeyAuthentication=no', '-o', 'NumberOfPasswordPrompts=1', '-o', 'ConnectTimeout=15') +. (Join-Path $root 'erp-remote.ps1') +$remoteSshScript = Join-Path $root '..\..\runtime\private\server-ssh.ps1' +$remoteUploaderScript = Join-Path $root '..\..\runtime\private\upload-to-server.ps1' + +# 远程执行包装:失败自动重试(含重新上传 CSV),并返回 { Ok; Output; Attempts } function Invoke-Remote { - param([Parameter(Mandatory = $true)][string]$Body) - $tmp = [IO.Path]::GetTempFileName() + '.sh' - [IO.File]::WriteAllText($tmp, ($Body -replace "`r`n", "`n")) - try { - $out = (& $self -NoProfile -ExecutionPolicy Bypass -File (Join-Path $root '..\..\runtime\private\server-ssh.ps1') -ScriptPath $tmp 2>&1 | Out-String) - return $out - } finally { - Remove-Item -LiteralPath $tmp -Force -ErrorAction SilentlyContinue - } + param( + [Parameter(Mandatory = $true)][string]$Body, + [string]$LocalFile = '', + [string]$RemoteFile = '' + ) + + return Invoke-ErpRemote ` + -PwshPath $self ` + -SshScript $remoteSshScript ` + -UploaderScript $remoteUploaderScript ` + -Body $Body ` + -LocalFile $LocalFile ` + -RemoteFile $RemoteFile ` + -Log { param($message) Write-Log $message } } +$failed = $false try { foreach ($target in $targets) { $spec = $specs[$target] Write-Log ('=== ' + $spec.label + ':开始 ===') - # 1) 导出 - $args = @((Join-Path $root 'export-daily.mjs'), ('--platform=' + $target), ('--out=' + $exportDir)) - if ($Start) { $args += ('--start=' + $Start) } - if ($Through) { $args += ('--through=' + $Through) } - $exportOut = (& $node @args 2>&1 | Out-String) - Write-Log ('导出:' + ($exportOut.Trim() -replace "`r?`n", ' | ')) - if ($LASTEXITCODE -ne 0) { Write-Log ($spec.label + ' 导出失败(exit=' + $LASTEXITCODE + ')'); continue } + # 1) 导出(胜算页面偶发日历面板/授权弹窗异常,失败后自动重试一次) + $exportArgs = @((Join-Path $root 'export-daily.mjs'), ('--platform=' + $target), ('--out=' + $exportDir)) + if ($Start) { $exportArgs += ('--start=' + $Start) } + if ($Through) { $exportArgs += ('--through=' + $Through) } + $exportAttempts = 2 + $exportOk = $false + for ($exportTry = 1; $exportTry -le $exportAttempts -and -not $exportOk; $exportTry++) { + $exportOut = (& $node @exportArgs 2>&1 | Out-String) + Write-Log ('导出:' + ($exportOut.Trim() -replace "`r?`n", ' | ')) + if ($LASTEXITCODE -eq 0) { + $exportOk = $true + break + } + Write-Log ($spec.label + ' 导出失败(exit=' + $LASTEXITCODE + ',第 ' + $exportTry + '/' + $exportAttempts + ' 次)') + if ($exportTry -lt $exportAttempts) { Start-Sleep -Seconds 20 } + } + if (-not $exportOk) { $failed = $true; continue } $summary = Get-Content -LiteralPath (Join-Path $exportDir 'last-export.json') -Raw | ConvertFrom-Json if ($summary.empty) { @@ -87,7 +108,7 @@ try { $rawFiles = @() if (($summary.PSObject.Properties.Name -contains 'files') -and $summary.files) { $rawFiles = @($summary.files) } if ($rawFiles.Count -eq 0 -and $summary.file) { $rawFiles = @($summary.file) } - if ($rawFiles.Count -eq 0) { Write-Log ($spec.label + ' 导出结果里没有文件名,跳过'); continue } + if ($rawFiles.Count -eq 0) { Write-Log ($spec.label + ' 导出结果里没有文件名'); $failed = $true; continue } $rawFile = $rawFiles[0] # 多分片时 CSV 用合并后的名字(不带 _partN),避免看起来只落了第一片 $csvFile = if ($rawFiles.Count -gt 1) { @@ -102,7 +123,7 @@ try { $conv = (& $python -X utf8 (Join-Path $root 'xlsx-to-csv.py') $rawFile $csvFile 2>&1 | Out-String) } $line = (($conv -split "`n") | Where-Object { $_ -match '^csv=' } | Select-Object -First 1) - if (-not $line) { Write-Log ($spec.label + ' 转 CSV 失败:' + ($conv.Trim() -replace "`r?`n", ' | ')); continue } + if (-not $line) { Write-Log ($spec.label + ' 转 CSV 失败:' + ($conv.Trim() -replace "`r?`n", ' | ')); $failed = $true; continue } $rows = [int]([regex]::Match($line, 'rows=(\d+)').Groups[1].Value) $income = [double]([regex]::Match($line, 'income=([\-\d\.]+)').Groups[1].Value) $expense = [double]([regex]::Match($line, 'expense=([\-\d\.]+)').Groups[1].Value) @@ -114,6 +135,7 @@ try { $pageCount = if ($summary.pageCount) { [int]$summary.pageCount } else { 0 } if ($pageCount -gt 0 -and $pageCount -ne $rows -and -not $ForceImport) { Write-Log ('跳过导入:页面笔数 {0} 与导出行数 {1} 不一致(分片可能未下全),未改动生产' -f $pageCount, $rows) + $failed = $true continue } @@ -127,10 +149,15 @@ try { } if ($SkipImport) { Write-Log '已指定 -SkipImport,跳过导入'; continue } - # 3) 上传 + 导入 + 页面回验 + # 3) 上传 + 导入 + 页面回验(远端失败/断连自动重试;整段失败则本平台失败,不写状态、不归档) $remoteFile = '/tmp/' + $target + '_daily_' + (Get-Date -Format 'yyyyMMdd_HHmmss') + '.csv' - $upload = (& $self -NoProfile -ExecutionPolicy Bypass -File (Join-Path $root '..\..\runtime\private\upload-to-server.ps1') -LocalPath $csvFile -RemotePath $remoteFile 2>&1 | Out-String) - Write-Log ('上传:' + ($upload.Trim() -replace "`r?`n", ' | ')) + $upload = Invoke-ErpRemoteUpload -PwshPath $self -UploaderScript $remoteUploaderScript -LocalFile $csvFile -RemoteFile $remoteFile + Write-Log ('上传:' + ($upload.Output.Trim() -replace "`r?`n", ' | ')) + if (-not $upload.Ok) { + Write-Log ($spec.label + ' 上传失败:未写入本地状态、未归档(下次运行会重新尝试)') + $failed = $true + continue + } $verifyPhp = @" cat > /tmp/verify_daily.php <<'PHP' @@ -159,8 +186,13 @@ docker cp $remoteFile caiwu_app:/tmp/daily_bills.csv $verifyPhp rm -f /tmp/daily_bills.csv $remoteFile "@ - $importOut = Invoke-Remote -Body $body - Write-Log ('导入/回验:' + ($importOut.Trim() -replace "`r?`n", ' | ')) + $import = Invoke-Remote -Body $body -LocalFile $csvFile -RemoteFile $remoteFile + Write-Log ('导入/回验:' + ($import.Output.Trim() -replace "`r?`n", ' | ')) + if (-not $import.Ok) { + Write-Log ($spec.label + ' 导入/回验失败(已尝试 ' + $import.Attempts + ' 次):未写入本地状态、未归档,下次运行会重新导入') + $failed = $true + continue + } # 4) 记录状态 + 归档 $state | Add-Member -NotePropertyName $key -NotePropertyValue ([pscustomobject]@{ rows = $rows; income = $income; expense = $expense; file = $csvFile; at = (Get-Date -Format 'yyyy-MM-dd HH:mm:ss') }) -Force @@ -180,6 +212,11 @@ rm -f /tmp/daily_bills.csv $remoteFile } Write-Log ('=== ' + $spec.label + ':完成 ===') } + + if ($failed) { + Write-Log '本次运行存在失败平台(详见上面日志):相关平台未写状态、未归档' + exit 1 + } } finally { Stop-CaiwuDeploymentAuth $auth }