153 lines
4.5 KiB
PowerShell
153 lines
4.5 KiB
PowerShell
param(
|
|
[int]$Minutes = 60,
|
|
[string]$FrpHost = "frp",
|
|
[string]$ProdHost = "prod"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
function Get-FrpWindowJson {
|
|
param(
|
|
[string]$RemoteHost,
|
|
[int]$WindowMinutes
|
|
)
|
|
|
|
$cmd = @"
|
|
python3 - <<'PY'
|
|
from datetime import datetime, timedelta
|
|
import json, re
|
|
|
|
minutes = $WindowMinutes
|
|
cut = datetime.now() - timedelta(minutes=minutes)
|
|
|
|
timeline = {}
|
|
for ln in open('/var/log/yydd-net-probe-frp.log', 'r', errors='ignore'):
|
|
m = re.match(r'^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\s+status=(\w+)\s+login=(\S+)\s+csrf=(\S+)\s+upstream2320=(\S+)\s+frps_anom=(\S+)\s+nginx_anom=(\S+)', ln.strip())
|
|
if not m:
|
|
continue
|
|
ts = datetime.strptime(m.group(1), '%Y-%m-%d %H:%M:%S')
|
|
if ts < cut:
|
|
continue
|
|
minute = ts.strftime('%Y-%m-%d %H:%M')
|
|
timeline[minute] = {
|
|
'status': m.group(2),
|
|
'login': m.group(3),
|
|
'csrf': m.group(4),
|
|
'upstream': m.group(5),
|
|
'frps_anom': m.group(6),
|
|
'nginx_anom': m.group(7),
|
|
'ts': ts.strftime('%H:%M:%S')
|
|
}
|
|
|
|
print(json.dumps({'timeline': timeline}, ensure_ascii=True))
|
|
PY
|
|
"@
|
|
|
|
return (& ssh -o BatchMode=yes -o ConnectTimeout=8 $RemoteHost $cmd)
|
|
}
|
|
|
|
function Get-ProdWindowJson {
|
|
param(
|
|
[string]$RemoteHost,
|
|
[int]$WindowMinutes
|
|
)
|
|
|
|
$cmd = @"
|
|
python3 - <<'PY'
|
|
from datetime import datetime, timedelta
|
|
import json, re
|
|
|
|
minutes = $WindowMinutes
|
|
cut = datetime.now() - timedelta(minutes=minutes)
|
|
|
|
timeline = {}
|
|
for ln in open('/root/frp/yydd-net-probe-prod.log', 'r', errors='ignore'):
|
|
m = re.match(r'^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\s+status=(\w+)\s+app_health=(\S+)\s+health_code=(\S+)\s+frps_tcp=(\S+)\s+frpc_count=(\S+)\s+frpc_anom=(\S+)\s+ext_login=(\S+)', ln.strip())
|
|
if not m:
|
|
continue
|
|
ts = datetime.strptime(m.group(1), '%Y-%m-%d %H:%M:%S')
|
|
if ts < cut:
|
|
continue
|
|
minute = ts.strftime('%Y-%m-%d %H:%M')
|
|
timeline[minute] = {
|
|
'status': m.group(2),
|
|
'app_health': m.group(3),
|
|
'health_code': m.group(4),
|
|
'frps_tcp': m.group(5),
|
|
'frpc_count': m.group(6),
|
|
'frpc_anom': m.group(7),
|
|
'ext_login': m.group(8),
|
|
'ts': ts.strftime('%H:%M:%S')
|
|
}
|
|
|
|
print(json.dumps({'timeline': timeline}, ensure_ascii=True))
|
|
PY
|
|
"@
|
|
|
|
return (& ssh -o BatchMode=yes -o ConnectTimeout=8 $RemoteHost $cmd)
|
|
}
|
|
|
|
function Judge-Minute {
|
|
param(
|
|
$Frp,
|
|
$Prod
|
|
)
|
|
|
|
if ($null -eq $Frp -and $null -eq $Prod) { return "NO_DATA" }
|
|
if ($null -eq $Frp) { return "FRP_LOG_GAP" }
|
|
if ($null -eq $Prod) { return "PROD_LOG_GAP" }
|
|
|
|
if ($Frp.status -eq "FAIL" -and $Prod.status -eq "FAIL") {
|
|
return "LINK_OR_ENTRY_FAIL"
|
|
}
|
|
|
|
if ($Frp.status -eq "FAIL" -and $Prod.status -eq "OK") {
|
|
return "FRP_EDGE_LAYER_FAIL"
|
|
}
|
|
|
|
if ($Frp.status -eq "OK" -and $Prod.status -eq "FAIL") {
|
|
return "PROD_SIDE_OR_PATH_FAIL"
|
|
}
|
|
|
|
if ($Frp.status -eq "WARN" -or $Prod.status -eq "WARN") {
|
|
return "WARN_NEED_CHECK"
|
|
}
|
|
|
|
return "OK"
|
|
}
|
|
|
|
Write-Host "=== YYDD minute-level network timeline (last $Minutes minutes) ==="
|
|
Write-Host "FRP host: $FrpHost, PROD host: $ProdHost"
|
|
|
|
$frpJson = Get-FrpWindowJson -RemoteHost $FrpHost -WindowMinutes $Minutes | ConvertFrom-Json
|
|
$prodJson = Get-ProdWindowJson -RemoteHost $ProdHost -WindowMinutes $Minutes | ConvertFrom-Json
|
|
|
|
$allMinutes = @($frpJson.timeline.PSObject.Properties.Name + $prodJson.timeline.PSObject.Properties.Name | Sort-Object -Unique)
|
|
|
|
if ($allMinutes.Count -eq 0) {
|
|
Write-Host "No probe samples found. Check both timers first."
|
|
exit 1
|
|
}
|
|
|
|
$rows = foreach ($minute in $allMinutes) {
|
|
$frp = $frpJson.timeline.$minute
|
|
$prod = $prodJson.timeline.$minute
|
|
[PSCustomObject]@{
|
|
minute = $minute
|
|
frp = if ($null -ne $frp) { $frp.status } else { "-" }
|
|
prod = if ($null -ne $prod) { $prod.status } else { "-" }
|
|
judge = Judge-Minute -Frp $frp -Prod $prod
|
|
frp_detail = if ($null -ne $frp) { "login=$($frp.login),csrf=$($frp.csrf),up=$($frp.upstream),anom=$($frp.frps_anom)/$($frp.nginx_anom)" } else { "-" }
|
|
prod_detail = if ($null -ne $prod) { "health=$($prod.app_health),tcp=$($prod.frps_tcp),cnt=$($prod.frpc_count),anom=$($prod.frpc_anom),ext=$($prod.ext_login)" } else { "-" }
|
|
}
|
|
}
|
|
|
|
$rows | Sort-Object minute | Format-Table -AutoSize
|
|
|
|
Write-Host ""
|
|
Write-Host "=== Rules ==="
|
|
Write-Host "- LINK_OR_ENTRY_FAIL: both FRP and PROD fail in same minute."
|
|
Write-Host "- FRP_EDGE_LAYER_FAIL: only FRP fails; check Nginx/FRPS/upstream 2320."
|
|
Write-Host "- PROD_SIDE_OR_PATH_FAIL: FRP is OK but PROD fails; check PROD->FRP path/FRPC."
|
|
Write-Host "- WARN_NEED_CHECK: service still works but anomaly counters exist."
|