diff --git a/scripts/erp-daily-bills/README.md b/scripts/erp-daily-bills/README.md index c3abdf44..d96b99b3 100644 --- a/scripts/erp-daily-bills/README.md +++ b/scripts/erp-daily-bills/README.md @@ -22,11 +22,13 @@ pwsh -File scripts\erp-daily-bills\run-erp-daily.ps1 -Platform jd -Through 2026- # 只跑京东钱包，补历史月份 pwsh -File scripts\erp-daily-bills\run-erp-daily.ps1 -Platform jd-wallet -Through 2026-07-31 -# 只导出不导入（排查） +# 只导出不导入（排查；仍会做覆盖核查） pwsh -File scripts\erp-daily-bills\run-erp-daily.ps1 -Platform toutiao -Through 2026-10-05 -SkipImport ``` - 归期：`--through`（默认昨日）所在月份；每月 1 日自动导出**上一个整月**。 +- **先补旧账再拉新账**：每次运行先核对 `2026-07-01` 至本次截止日的覆盖情况。发现历史月份缺失、截止日不足、行数或金额不一致时，按月份从旧到新重导并复核；历史补数未通过时，不继续拉取该平台的新日期。 +- 每日失败自动顺延：当天任一平台导出、上传或导入失败，不写成功状态；次日覆盖核查会重新发现缺口并优先补齐。 - **无差异跳过**：该平台该月已导入且笔数/收入/支出完全一致时跳过导入（`-ForceImport` 可强制重刷）；状态记在 `runtime/private/erp-exports/imported-state.json`。 - 超过胜算同步导出上限（响应 `超出同步导出允许最大数据量`）时，脚本自动轮询「异步导出下载管理」取回本次任务的全部分片，逐片转 CSV 后合并成一份再导入；仍可用 `-Start/-Through` 分段导出。 - 导入前用页面笔数（共 N 条）与导出行数核对，不一致就跳过导入（`-ForceImport` 可强制），避免只导入一部分。 @@ -47,7 +49,7 @@ docker exec -u www-data caiwu_app php artisan business-mirror:verify-erp-daily-c | 文件 | 作用 | | --- | --- | -| `run-erp-daily.ps1` | 统一入口：导出 → 转 CSV → 差异判断 → 上传 → 受限批处理容器导入 → 页面回验 → 归档 → 日志 | +| `run-erp-daily.ps1` | 统一入口：导出 → 转 CSV → 差异判断 → 上传 → 受控导入 → 页面回验 → 归档 → 日志 | | `export-daily.mjs` | 浏览器自动化导出（登录、切页签、设日期、查询、导出、取下载直链） | | `erp-session.mjs` | 胜算登录与会话工具（凭据读 `runtime/private/敏感信息.md`，可用 `ERP_SECRET_FILE` 覆盖） | | `cdp.mjs` | 无依赖的 Edge DevTools Protocol 驱动 | @@ -55,6 +57,6 @@ docker exec -u www-data caiwu_app php artisan business-mirror:verify-erp-daily-c ## 说明 -- 导入走财务系统既有受控导入命令，并通过生产 `run-finance-batch.sh` 进入 1 CPU / 1 GiB 的独立批处理容器；命令带笔数/收支核对、原子切换并保留旧版本可回滚，不改上游数据。 +- 导入走财务系统既有受控导入命令（带笔数/收支核对、原子切换、保留旧版本可回滚），不改上游数据。 - 凭据、导出文件、日志都写 `runtime/private/`（`.gitignore` 忽略），不进仓库。 - 定时：Windows 计划任务每天 09:30 调 `run-erp-daily.ps1`（京东取昨日；头条取月初→昨日）。 diff --git a/scripts/erp-daily-bills/erp-remote.ps1 b/scripts/erp-daily-bills/erp-remote.ps1 index 536bd554..65dbcaf1 100644 --- a/scripts/erp-daily-bills/erp-remote.ps1 +++ b/scripts/erp-daily-bills/erp-remote.ps1 @@ -13,6 +13,25 @@ function New-ErpRemoteResult { return [pscustomobject]@{ Ok = $Ok; Output = $Output; Attempts = $Attempts } } +function Get-ErpCoverageBackfillJobs { + param( + [object[]]$CoverageReport, + [Parameter(Mandatory = $true)][string]$Dataset, + [Parameter(Mandatory = $true)][string]$CurrentMonth + ) + + # 只返回当前拉取月份之前的缺口；当前月由本轮整月重导自然覆盖。 + return @( + $CoverageReport | + Where-Object { + $_.dataset -eq $Dataset -and + $_.month -lt $CurrentMonth -and + $_.status -ne 'ok' + } | + Sort-Object month + ) +} + function Invoke-ErpRemoteUpload { param( [Parameter(Mandatory = $true)][string]$PwshPath, diff --git a/scripts/erp-daily-bills/run-erp-daily.ps1 b/scripts/erp-daily-bills/run-erp-daily.ps1 index a83f17a2..00c8f557 100644 --- a/scripts/erp-daily-bills/run-erp-daily.ps1 +++ b/scripts/erp-daily-bills/run-erp-daily.ps1 @@ -5,7 +5,8 @@ jd = 京东商城→京东日账单（一次导入按店铺名拆 京东账单 + 京喜账单） jd-wallet = 京东商城→京东钱包（落 京东网银钱包账户） - 流程：导出 → xlsx 转 CSV → 与已导入期间比对（无差异则跳过）→ 上传 → 受限批处理容器导入 → 页面回验 → 归档 → 写日志 + 流程：覆盖核查 → 补齐历史缺口 → 导出 → xlsx 转 CSV → 与已导入期间比对（无差异则跳过） + → 上传 → 受限批处理容器导入 → 页面回验 → 归档 → 写日志 用法： pwsh -File run-erp-daily.ps1 # 三个平台，区间「月初 → 昨日」 @@ -18,8 +19,10 @@ param( [ValidateSet('toutiao', 'jd', 'jd-wallet', 'both')][string]$Platform = 'both', [string]$Through, [string]$Start, + [string]$CoverageFrom = '2026-07-01', [switch]$SkipImport, - [switch]$ForceImport + [switch]$ForceImport, + [switch]$SkipCoverageCheck ) $ErrorActionPreference = 'Stop' @@ -49,6 +52,11 @@ $specs = @{ jd = @{ label = '京东日账单'; artifact = 'jd'; importCmd = 'business-mirror:import-jd-daily'; verifyTabs = @('jd-transactions', 'jingxi'); split = $true } 'jd-wallet' = @{ label = '京东钱包'; artifact = 'jd-wallet'; importCmd = 'business-mirror:import-jd-wallet'; verifyTabs = @('jd-wallet'); split = $false } } +$coverageKeys = @{ + toutiao = 'douyin' + jd = 'jd-daily' + 'jd-wallet' = 'jd-wallet' +} $targets = if ($Platform -eq 'both') { @('toutiao', 'jd', 'jd-wallet') } else { @($Platform) } $state = if (Test-Path -LiteralPath $stateFile) { (Get-Content -LiteralPath $stateFile -Raw | ConvertFrom-Json) } else { New-Object psobject } @@ -79,11 +87,125 @@ function Invoke-Remote { -Log { param($message) Write-Log $message } } +function Get-ErpDailyCoverage { + param([Parameter(Mandatory = $true)][string]$CoverageFrom, [Parameter(Mandatory = $true)][string]$Through) + + # 覆盖检查在有缺口时按设计返回 exit 1；报告本身仍是完整 JSON，不能把它当成远程执行失败重试。 + $body = @" +set +e +output=`$(docker exec -u www-data caiwu_app php /var/www/html/artisan business-mirror:verify-erp-daily-coverage --from=$CoverageFrom --to=$Through --json 2>&1) +status=`$? +printf '%s\n' "`$output" +if [ "`$status" -eq 1 ]; then + exit 0 +fi +exit "`$status" +"@ + $result = Invoke-Remote -Body $body + if (-not $result.Ok) { + throw ('覆盖核查失败：' + ($result.Output.Trim() -replace "`r?`n", ' | ')) + } + + $jsonLine = (($result.Output -split "`r?`n") | Where-Object { $_.TrimStart().StartsWith('[') } | Select-Object -Last 1) + if (-not $jsonLine) { + throw ('覆盖核查没有返回 JSON：' + ($result.Output.Trim() -replace "`r?`n", ' | ')) + } + + return @($jsonLine | ConvertFrom-Json) +} + $failed = $false try { + $pullThroughDate = if ($Through) { + $parsed = [datetime]::MinValue + if (-not [datetime]::TryParseExact($Through, 'yyyy-MM-dd', [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::None, [ref]$parsed)) { + throw '截止日期格式无效，应为 YYYY-MM-DD。' + } + $parsed + } else { + (Get-Date).Date.AddDays(-1) + } + $pullThrough = $pullThroughDate.ToString('yyyy-MM-dd') + $pullMonth = $pullThrough.Substring(0, 7) + + $coverageFromDate = [datetime]::MinValue + if (-not [datetime]::TryParseExact($CoverageFrom, 'yyyy-MM-dd', [Globalization.CultureInfo]::InvariantCulture, [Globalization.DateTimeStyles]::None, [ref]$coverageFromDate)) { + throw '覆盖核查起始日期格式无效，应为 YYYY-MM-DD。' + } + if ($coverageFromDate.Date -gt $pullThroughDate.Date) { + throw '覆盖核查起始日期不能晚于本次账单截止日期。' + } + + $coverageReady = [bool]$SkipCoverageCheck + $coverageReport = @() + if (-not $SkipCoverageCheck) { + Write-Log ('=== 每日覆盖核查：{0} ~ {1} ===' -f $CoverageFrom, $pullThrough) + try { + $coverageReport = @(Get-ErpDailyCoverage -CoverageFrom $CoverageFrom -Through $pullThrough) + $coverageIssues = @($coverageReport | Where-Object { $_.status -ne 'ok' }) + Write-Log ('覆盖核查完成：{0} 项，异常 {1} 项' -f $coverageReport.Count, $coverageIssues.Count) + foreach ($issue in $coverageIssues) { + Write-Log (' 待补：{0} {1}，{2} ~ {3}，状态 {4}（{5}）' -f $issue.label, $issue.month, $issue.required_start, $issue.required_through, $issue.status, $issue.issue) + } + $coverageReady = $true + } catch { + Write-Log ('覆盖核查失败，本次不拉取新账单：' + $_.Exception.Message) + $failed = $true + } + } + + if (-not $coverageReady) { + $targets = @() + } + foreach ($target in $targets) { $spec = $specs[$target] Write-Log ('=== ' + $spec.label + '：开始 ===') + $coverageKey = $coverageKeys[$target] + $currentIssue = @($coverageReport | Where-Object { $_.dataset -eq $coverageKey -and $_.month -eq $pullMonth -and $_.status -ne 'ok' }) + $forceThisImport = [bool]$ForceImport -or $currentIssue.Count -gt 0 + + # 先补此前月份的缺口；历史仍不完整时，不继续拉取本月新数据。 + $historicalIssues = @(Get-ErpCoverageBackfillJobs -CoverageReport $coverageReport -Dataset $coverageKey -CurrentMonth $pullMonth) + if ($historicalIssues.Count -gt 0 -and -not $SkipImport) { + $catchUpFailed = $false + foreach ($issue in $historicalIssues) { + Write-Log ('发现历史缺口，先补数：{0} {1}，{2} ~ {3}（{4}）' -f $issue.label, $issue.month, $issue.required_start, $issue.required_through, $issue.status) + & $self -NoProfile -ExecutionPolicy Bypass -File $PSCommandPath ` + -Platform $target ` + -Start $issue.required_start ` + -Through $issue.required_through ` + -CoverageFrom $CoverageFrom ` + -ForceImport ` + -SkipCoverageCheck + if ($LASTEXITCODE -ne 0) { + Write-Log ('历史补数失败：{0} {1}，已停止该平台的新数据拉取' -f $issue.label, $issue.month) + $catchUpFailed = $true + break + } + } + + if (-not $catchUpFailed) { + try { + $coverageReport = @(Get-ErpDailyCoverage -CoverageFrom $CoverageFrom -Through $pullThrough) + $remaining = @(Get-ErpCoverageBackfillJobs -CoverageReport $coverageReport -Dataset $coverageKey -CurrentMonth $pullMonth) + if ($remaining.Count -gt 0) { + foreach ($issue in $remaining) { + Write-Log ('历史补数后仍未完整：{0} {1}，状态 {2}' -f $issue.label, $issue.month, $issue.status) + } + $catchUpFailed = $true + } + } catch { + Write-Log ('历史补数后复核失败：' + $_.Exception.Message) + $catchUpFailed = $true + } + } + + if ($catchUpFailed) { + $failed = $true + continue + } + } # 1) 导出（胜算页面偶发日历面板/授权弹窗/胜算标签页注册慢，失败后自动重试） $exportArgs = @((Join-Path $root 'export-daily.mjs'), ('--platform=' + $target), ('--out=' + $exportDir)) @@ -149,21 +271,28 @@ try { $key = $target + ':' + $month $prev = $state.$key $same = $prev -and ([string]$prev.through -eq [string]$summary.through) -and ([int]$prev.rows -eq $rows) -and ([math]::Abs([double]$prev.income - $income) -lt 0.005) -and ([math]::Abs([double]$prev.expense - $expense) -lt 0.005) - if ($same -and -not $ForceImport) { - Write-Log ('该月已导入且无差异（{0} 笔 / {1} / {2}），跳过导入' -f $rows, $income, $expense) + $currentCoverage = @($coverageReport | Where-Object { $_.dataset -eq $coverageKey -and $_.month -eq $month } | Select-Object -First 1) + $serverMatchesExport = $currentCoverage.Count -eq 1 -and + $currentCoverage[0].status -eq 'ok' -and + [int]$currentCoverage[0].row_count -eq $rows -and + [math]::Abs([double]$currentCoverage[0].income - $income) -lt 0.005 -and + [math]::Abs([double]$currentCoverage[0].expense - $expense) -lt 0.005 + if (($same -or $serverMatchesExport) -and -not $forceThisImport) { + if (-not $same -and $serverMatchesExport) { + # 服务端覆盖已正确、只是本机状态滞后时，回填状态，避免每天重复导入同一批账单。 + $state | Add-Member -NotePropertyName $key -NotePropertyValue ([pscustomobject]@{ rows = $rows; income = $income; expense = $expense; through = $summary.through; file = $csvFile; at = (Get-Date -Format 'yyyy-MM-dd HH:mm:ss') }) -Force + $state | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $stateFile -Encoding UTF8 + Write-Log ('服务端覆盖已核对且与导出无差异，已回填本机状态（{0} 笔 / {1} / {2}），跳过导入' -f $rows, $income, $expense) + } else { + Write-Log ('该月已导入且无差异（{0} 笔 / {1} / {2}），跳过导入' -f $rows, $income, $expense) + } continue } if ($SkipImport) { Write-Log '已指定 -SkipImport，跳过导入'; continue } - # 3) 上传 + 导入 + 页面回验（远端失败/断连自动重试；整段失败则本平台失败，不写状态、不归档） + # 3) 上传 + 导入 + 页面回验。统一交给 Invoke-Remote，上传断开时会重新上传并重试， + # 整段失败则本平台失败，不写状态、不归档。 $remoteFile = '/tmp/' + $target + '_daily_' + (Get-Date -Format 'yyyyMMdd_HHmmss') + '.csv' - $upload = Invoke-ErpRemoteUpload -PwshPath $self -UploaderScript $remoteUploaderScript -LocalFile $csvFile -RemoteFile $remoteFile - Write-Log ('上传：' + ($upload.Output.Trim() -replace "`r?`n", ' | ')) - if (-not $upload.Ok) { - Write-Log ($spec.label + ' 上传失败：未写入本地状态、未归档（下次运行会重新尝试）') - $failed = $true - continue - } $verifyPhp = @" cat > /tmp/verify_daily.php <<'PHP' @@ -173,19 +302,42 @@ require '/var/www/html/vendor/autoload.php'; `$app->make(Illuminate\Contracts\Console\Kernel::class)->bootstrap(); `$repo = app(Modules\EcommerceFinance\Repositories\StoreReceivingDouyinBillMirrorRepository::class); `$tabs = ['__TABS__']; +`$total = 0; +`$incomeCents = 0; +`$expenseCents = 0; foreach (`$tabs as `$tab) { `$r = match (`$tab) { 'douyin' => `$repo->paginate(['start' => '__START__', 'end' => '__THROUGH__'], 'occurred_at', 'desc', 1, 1), 'jd-wallet' => app(Modules\EcommerceFinance\Repositories\StoreReceivingJdWalletBillRepository::class)->paginate(['start' => '__START__', 'end' => '__THROUGH__'], 'occurred_at', 'desc', 1, 1), - default => app(Modules\EcommerceFinance\Repositories\JdDailyBillRepository::class)->paginate(['start' => '__START__', 'end' => '__THROUGH__'], 'occurred_at', 'desc', 1, 1, `$tab), + default => app(Modules\EcommerceFinance\Repositories\JdDailyBillRepository::class) + ->forPlatform(`$tab === 'jingxi' ? 'jingxi' : 'jd') + ->paginate(['start' => '__START__', 'end' => '__THROUGH__'], 'occurred_at', 'desc', 1, 1), }; + `$total += (int) `$r['total']; + `$incomeCents += (int) round((float) `$r['summary']['income'] * 100); + `$expenseCents += (int) round((float) `$r['summary']['expense'] * 100); echo 'PAGE ' . `$tab . ' total=' . `$r['total'] . ' ' . json_encode(`$r['summary'], JSON_UNESCAPED_UNICODE) . PHP_EOL; } +if (`$total !== __COUNT__ + || `$incomeCents !== (int) round(__INCOME__ * 100) + || `$expenseCents !== (int) round(__EXPENSE__ * 100)) { + fwrite(STDERR, sprintf( + '页面回验不一致：导入 %d 笔 / %s / %s，页面 %d 笔 / %.2f / %.2f%s', + __COUNT__, __INCOME__, __EXPENSE__, `$total, `$incomeCents / 100, `$expenseCents / 100, PHP_EOL + )); + exit(1); +} PHP docker exec -i -u www-data caiwu_app php < /tmp/verify_daily.php rm -f /tmp/verify_daily.php "@ - $verifyPhp = $verifyPhp.Replace('__TABS__', ($spec.verifyTabs -join "','")).Replace('__START__', $summary.monthStart).Replace('__THROUGH__', $summary.through) + $invariant = [Globalization.CultureInfo]::InvariantCulture + $verifyPhp = $verifyPhp.Replace('__TABS__', ($spec.verifyTabs -join "','")) + $verifyPhp = $verifyPhp.Replace('__START__', $summary.monthStart) + $verifyPhp = $verifyPhp.Replace('__THROUGH__', $summary.through) + $verifyPhp = $verifyPhp.Replace('__COUNT__', [string]$rows) + $verifyPhp = $verifyPhp.Replace('__INCOME__', $income.ToString('0.00', $invariant)) + $verifyPhp = $verifyPhp.Replace('__EXPENSE__', $expense.ToString('0.00', $invariant)) # 生产重型导入必须进入发布时安装的受限批处理容器；由远端 runner # 持有同步锁、校验可用内存、复制只读输入并限制为 1 CPU / 1 GiB。 @@ -212,9 +364,10 @@ $verifyPhp $state | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $stateFile -Encoding UTF8 $monthNumber = [int]$month.Substring(5, 2) - $shareDir = Join-Path 'Z:\高祝兰\财务系统' ('{0}月' -f $monthNumber) - if (Test-Path -LiteralPath 'Z:\高祝兰\财务系统') { + $shareRoot = 'Z:\高祝兰\财务系统' + if (Test-Path -LiteralPath $shareRoot) { try { + $shareDir = Join-Path $shareRoot ('{0}月' -f $monthNumber) New-Item -ItemType Directory -Path $shareDir -Force | Out-Null foreach ($f in $rawFiles) { Copy-Item -LiteralPath $f -Destination (Join-Path $shareDir ([IO.Path]::GetFileName($f))) -Force