gongxiangp2/scripts/run-tianlan-remaining-batches.ps1

param(
  [int]$StartBatch = 2,
  [int]$EndBatch = 15,
  [string]$BatchDir = "D:\AI Code\gongxiangp2\output\tianlan-spu-ledger\remaining-batches-1778239500000",
  [string]$Operator = "222"
)

$ErrorActionPreference = "Stop"
Set-Location "D:\AI Code\gongxiangp2"

function Set-ImportEnvironment {
  $env:DATABASE_URL = ((Get-Content .env | Where-Object { $_ -match '^DATABASE_URL=' } | Select-Object -First 1) -replace '^DATABASE_URL=', '').Trim()
  $env:S3_ENDPOINT = "http://192.168.58.20:9000"
  $env:S3_BUCKET = "gongxiangpan"
  $env:S3_PUBLIC_DOMAIN = "https://s3.jiansun.vip/gongxiangpan"
  $env:S3_FILE_PATH = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String("55Sx5aO554K5QUkv"))
  $sshArgs = @(
    "-o", "BatchMode=yes",
    "-o", "ConnectTimeout=15",
    "-o", "ServerAliveInterval=15",
    "-o", "ServerAliveCountMax=2",
    "prod"
  )
  $env:S3_ACCESS_KEY_ID = Invoke-TextWithRetry "read S3_ACCESS_KEY_ID" { ssh @sshArgs "docker exec youyidian-3d-material printenv S3_ACCESS_KEY_ID" }
  $env:S3_SECRET_ACCESS_KEY = Invoke-TextWithRetry "read S3_SECRET_ACCESS_KEY" { ssh @sshArgs "docker exec youyidian-3d-material printenv S3_SECRET_ACCESS_KEY" }
  $env:HISTORY_IMPORT_REMOTE_UPLOADS_HOST = "prod"
  $env:HISTORY_IMPORT_REMOTE_UPLOADS_DIR = "/mnt/PoolOne/youyidian-uploads"
}

function Get-NewestImportResult([datetime]$StartedAt) {
  Get-ChildItem -LiteralPath $BatchDir -Filter "history-material-import-result-*.json" |
    Where-Object { $_.Name -notlike "*.verification.json" -and $_.LastWriteTime -ge $StartedAt } |
    Sort-Object LastWriteTime -Descending |
    Select-Object -First 1
}

function Write-GoalProgress($Payload) {
  $progressPath = Join-Path $BatchDir "goal-progress.json"
  $Payload | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $progressPath -Encoding UTF8
}

function Invoke-NpxToLog([string[]]$Arguments, [string]$LogPath) {
  $previousErrorActionPreference = $ErrorActionPreference
  $ErrorActionPreference = "Continue"
  try {
    & npx @Arguments *> $LogPath
    return $LASTEXITCODE
  } finally {
    $script:ErrorActionPreference = $previousErrorActionPreference
  }
}

function Invoke-TextWithRetry([string]$Label, [scriptblock]$Command, [int]$Retries = 3) {
  $lastOutput = $null
  for ($attempt = 1; $attempt -le $Retries; $attempt += 1) {
    $lastOutput = & $Command
    if ($LASTEXITCODE -eq 0 -and -not [string]::IsNullOrWhiteSpace(($lastOutput | Out-String))) {
      return ($lastOutput | Out-String).Trim()
    }

    if ($attempt -lt $Retries) {
      Write-Host ("[Goal] {0} failed on attempt {1}/{2}; retrying..." -f $Label, $attempt, $Retries)
      Start-Sleep -Seconds ([Math]::Min(10, 2 * $attempt))
    }
  }

  throw ("{0} failed after {1} attempts" -f $Label, $Retries)
}

Set-ImportEnvironment

$runStartedAt = Get-Date
$completed = @()

[pscustomobject]@{
  pid = $PID
  startBatch = $StartBatch
  endBatch = $EndBatch
  batchDir = $BatchDir
  startedAt = $runStartedAt.ToString("o")
  command = "scripts/run-tianlan-remaining-batches.ps1 -StartBatch $StartBatch -EndBatch $EndBatch"
} | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $BatchDir "active-goal-process.json") -Encoding UTF8

for ($batchNumber = $StartBatch; $batchNumber -le $EndBatch; $batchNumber += 1) {
  $batchName = "batch-{0:D3}.json" -f $batchNumber
  $batchPath = Join-Path $BatchDir $batchName

  if (-not (Test-Path -LiteralPath $batchPath)) {
    throw "Batch file not found: $batchPath"
  }

  $stamp = Get-Date -Format "yyyyMMdd-HHmmss"
  $importLog = Join-Path $BatchDir ("batch-{0:D3}-{1}.import.log" -f $batchNumber, $stamp)
  $verifyLog = Join-Path $BatchDir ("batch-{0:D3}-{1}.verify.log" -f $batchNumber, $stamp)
  $startedAt = Get-Date

  Write-Host ""
  Write-Host ("[Goal] Importing {0}" -f $batchName)
  Write-GoalProgress ([pscustomobject]@{
    mode = "tianlan-remaining-batches"
    runStartedAt = $runStartedAt.ToString("o")
    updatedAt = (Get-Date).ToString("o")
    startBatch = $StartBatch
    endBatch = $EndBatch
    completed = $completed
    current = [pscustomobject]@{
      batch = $batchName
      phase = "importing"
      importLog = Split-Path -Leaf $importLog
      verifyLog = Split-Path -Leaf $verifyLog
      startedAt = $startedAt.ToString("o")
    }
  })

  $importExitCode = Invoke-NpxToLog @(
    "tsx",
    "scripts/import-history-materials.ts",
    "--input=$batchPath",
    "--operator=$Operator",
    "--apply"
  ) $importLog
  $resultFile = Get-NewestImportResult $startedAt

  if ($importExitCode -ne 0 -or -not $resultFile) {
    Write-Host ("[Goal] Import failed for {0}; log: {1}" -f $batchName, $importLog)
    if (Test-Path -LiteralPath $importLog) {
      Get-Content -LiteralPath $importLog -Tail 80
    }
    exit 1
  }

  Write-Host ("[Goal] Result: {0}" -f $resultFile.Name)
  Write-GoalProgress ([pscustomobject]@{
    mode = "tianlan-remaining-batches"
    runStartedAt = $runStartedAt.ToString("o")
    updatedAt = (Get-Date).ToString("o")
    startBatch = $StartBatch
    endBatch = $EndBatch
    completed = $completed
    current = [pscustomobject]@{
      batch = $batchName
      phase = "validating"
      result = $resultFile.Name
      importLog = Split-Path -Leaf $importLog
      verifyLog = Split-Path -Leaf $verifyLog
    }
  })
  $verifyExitCode = Invoke-NpxToLog @(
    "tsx",
    "scripts/validate-history-import-result.ts",
    "--input=$($resultFile.FullName)",
    "--remote-host=prod",
    "--remote-uploads-dir=/mnt/PoolOne/youyidian-uploads"
  ) $verifyLog
  $verificationPath = $resultFile.FullName -replace '\.json$', '.verification.json'
  $verification = Get-Content -LiteralPath $verificationPath -Raw | ConvertFrom-Json

  $completed += [pscustomobject]@{
    batch = $batchName
    result = $resultFile.Name
    total = $verification.total
    imported = $verification.imported
    skipped = $verification.skipped
    failed = $verification.failed
    checked = $verification.checked
    ok = $verification.ok
    issueCount = $verification.issueCount
    importLog = Split-Path -Leaf $importLog
    verifyLog = Split-Path -Leaf $verifyLog
    finishedAt = (Get-Date).ToString("o")
  }

  Write-GoalProgress ([pscustomobject]@{
    mode = "tianlan-remaining-batches"
    runStartedAt = $runStartedAt.ToString("o")
    updatedAt = (Get-Date).ToString("o")
    startBatch = $StartBatch
    endBatch = $EndBatch
    completed = $completed
    current = $completed[-1]
  })

  Write-Host ("[Goal] Verified {0}: imported={1}, skipped={2}, failed={3}, ok={4}, issues={5}" -f $batchName, $verification.imported, $verification.skipped, $verification.failed, $verification.ok, $verification.issueCount)

  if ($verifyExitCode -ne 0 -or $verification.failed -gt 0 -or $verification.issueCount -gt 0) {
    Write-Host ("[Goal] Stopped on verification issue for {0}; log: {1}" -f $batchName, $verifyLog)
    Get-Content -LiteralPath $verifyLog -Tail 120
    exit 1
  }
}

Write-Host ""
Write-Host "[Goal] All requested batches completed."
Write-GoalProgress ([pscustomobject]@{
  mode = "tianlan-remaining-batches"
  runStartedAt = $runStartedAt.ToString("o")
  updatedAt = (Get-Date).ToString("o")
  startBatch = $StartBatch
  endBatch = $EndBatch
  completed = $completed
  done = $true
})