param( [int]$StartBatch = 2, [int]$EndBatch = 15, [string]$BatchDir = "D:\AI Code\gongxiangp2\output\tianlan-spu-ledger\remaining-batches-1778239500000", [string]$Operator = "222" ) $ErrorActionPreference = "Stop" Set-Location "D:\AI Code\gongxiangp2" function Set-ImportEnvironment { $env:DATABASE_URL = ((Get-Content .env | Where-Object { $_ -match '^DATABASE_URL=' } | Select-Object -First 1) -replace '^DATABASE_URL=', '').Trim() $env:S3_ENDPOINT = "http://192.168.58.20:9000" $env:S3_BUCKET = "gongxiangpan" $env:S3_PUBLIC_DOMAIN = "https://s3.jiansun.vip/gongxiangpan" $env:S3_FILE_PATH = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("55Sx5aO554K5QUkv")) $sshArgs = @( "-o", "BatchMode=yes", "-o", "ConnectTimeout=15", "-o", "ServerAliveInterval=15", "-o", "ServerAliveCountMax=2", "prod" ) $env:S3_ACCESS_KEY_ID = Invoke-TextWithRetry "read S3_ACCESS_KEY_ID" { ssh @sshArgs "docker exec youyidian-3d-material printenv S3_ACCESS_KEY_ID" } $env:S3_SECRET_ACCESS_KEY = Invoke-TextWithRetry "read S3_SECRET_ACCESS_KEY" { ssh @sshArgs "docker exec youyidian-3d-material printenv S3_SECRET_ACCESS_KEY" } $env:HISTORY_IMPORT_REMOTE_UPLOADS_HOST = "prod" $env:HISTORY_IMPORT_REMOTE_UPLOADS_DIR = "/mnt/PoolOne/youyidian-uploads" } function Get-NewestImportResult([datetime]$StartedAt) { Get-ChildItem -LiteralPath $BatchDir -Filter "history-material-import-result-*.json" | Where-Object { $_.Name -notlike "*.verification.json" -and $_.LastWriteTime -ge $StartedAt } | Sort-Object LastWriteTime -Descending | Select-Object -First 1 } function Write-GoalProgress($Payload) { $progressPath = Join-Path $BatchDir "goal-progress.json" $Payload | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $progressPath -Encoding UTF8 } function Invoke-NpxToLog([string[]]$Arguments, [string]$LogPath) { $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { & npx @Arguments *> $LogPath return $LASTEXITCODE } finally { $script:ErrorActionPreference = $previousErrorActionPreference } } function Invoke-TextWithRetry([string]$Label, [scriptblock]$Command, [int]$Retries = 3) { $lastOutput = $null for ($attempt = 1; $attempt -le $Retries; $attempt += 1) { $lastOutput = & $Command if ($LASTEXITCODE -eq 0 -and -not [string]::IsNullOrWhiteSpace(($lastOutput | Out-String))) { return ($lastOutput | Out-String).Trim() } if ($attempt -lt $Retries) { Write-Host ("[Goal] {0} failed on attempt {1}/{2}; retrying..." -f $Label, $attempt, $Retries) Start-Sleep -Seconds ([Math]::Min(10, 2 * $attempt)) } } throw ("{0} failed after {1} attempts" -f $Label, $Retries) } Set-ImportEnvironment $runStartedAt = Get-Date $completed = @() [pscustomobject]@{ pid = $PID startBatch = $StartBatch endBatch = $EndBatch batchDir = $BatchDir startedAt = $runStartedAt.ToString("o") command = "scripts/run-tianlan-remaining-batches.ps1 -StartBatch $StartBatch -EndBatch $EndBatch" } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $BatchDir "active-goal-process.json") -Encoding UTF8 for ($batchNumber = $StartBatch; $batchNumber -le $EndBatch; $batchNumber += 1) { $batchName = "batch-{0:D3}.json" -f $batchNumber $batchPath = Join-Path $BatchDir $batchName if (-not (Test-Path -LiteralPath $batchPath)) { throw "Batch file not found: $batchPath" } $stamp = Get-Date -Format "yyyyMMdd-HHmmss" $importLog = Join-Path $BatchDir ("batch-{0:D3}-{1}.import.log" -f $batchNumber, $stamp) $verifyLog = Join-Path $BatchDir ("batch-{0:D3}-{1}.verify.log" -f $batchNumber, $stamp) $startedAt = Get-Date Write-Host "" Write-Host ("[Goal] Importing {0}" -f $batchName) Write-GoalProgress ([pscustomobject]@{ mode = "tianlan-remaining-batches" runStartedAt = $runStartedAt.ToString("o") updatedAt = (Get-Date).ToString("o") startBatch = $StartBatch endBatch = $EndBatch completed = $completed current = [pscustomobject]@{ batch = $batchName phase = "importing" importLog = Split-Path -Leaf $importLog verifyLog = Split-Path -Leaf $verifyLog startedAt = $startedAt.ToString("o") } }) $importExitCode = Invoke-NpxToLog @( "tsx", "scripts/import-history-materials.ts", "--input=$batchPath", "--operator=$Operator", "--apply" ) $importLog $resultFile = Get-NewestImportResult $startedAt if ($importExitCode -ne 0 -or -not $resultFile) { Write-Host ("[Goal] Import failed for {0}; log: {1}" -f $batchName, $importLog) if (Test-Path -LiteralPath $importLog) { Get-Content -LiteralPath $importLog -Tail 80 } exit 1 } Write-Host ("[Goal] Result: {0}" -f $resultFile.Name) Write-GoalProgress ([pscustomobject]@{ mode = "tianlan-remaining-batches" runStartedAt = $runStartedAt.ToString("o") updatedAt = (Get-Date).ToString("o") startBatch = $StartBatch endBatch = $EndBatch completed = $completed current = [pscustomobject]@{ batch = $batchName phase = "validating" result = $resultFile.Name importLog = Split-Path -Leaf $importLog verifyLog = Split-Path -Leaf $verifyLog } }) $verifyExitCode = Invoke-NpxToLog @( "tsx", "scripts/validate-history-import-result.ts", "--input=$($resultFile.FullName)", "--remote-host=prod", "--remote-uploads-dir=/mnt/PoolOne/youyidian-uploads" ) $verifyLog $verificationPath = $resultFile.FullName -replace '\.json$', '.verification.json' $verification = Get-Content -LiteralPath $verificationPath -Raw | ConvertFrom-Json $completed += [pscustomobject]@{ batch = $batchName result = $resultFile.Name total = $verification.total imported = $verification.imported skipped = $verification.skipped failed = $verification.failed checked = $verification.checked ok = $verification.ok issueCount = $verification.issueCount importLog = Split-Path -Leaf $importLog verifyLog = Split-Path -Leaf $verifyLog finishedAt = (Get-Date).ToString("o") } Write-GoalProgress ([pscustomobject]@{ mode = "tianlan-remaining-batches" runStartedAt = $runStartedAt.ToString("o") updatedAt = (Get-Date).ToString("o") startBatch = $StartBatch endBatch = $EndBatch completed = $completed current = $completed[-1] }) Write-Host ("[Goal] Verified {0}: imported={1}, skipped={2}, failed={3}, ok={4}, issues={5}" -f $batchName, $verification.imported, $verification.skipped, $verification.failed, $verification.ok, $verification.issueCount) if ($verifyExitCode -ne 0 -or $verification.failed -gt 0 -or $verification.issueCount -gt 0) { Write-Host ("[Goal] Stopped on verification issue for {0}; log: {1}" -f $batchName, $verifyLog) Get-Content -LiteralPath $verifyLog -Tail 120 exit 1 } } Write-Host "" Write-Host "[Goal] All requested batches completed." Write-GoalProgress ([pscustomobject]@{ mode = "tianlan-remaining-batches" runStartedAt = $runStartedAt.ToString("o") updatedAt = (Get-Date).ToString("o") startBatch = $StartBatch endBatch = $EndBatch completed = $completed done = $true })