1777 lines
55 KiB
TypeScript
1777 lines
55 KiB
TypeScript
import { createReadStream, existsSync, statSync } from 'fs'
|
|
import { createInterface } from 'readline'
|
|
import { createGunzip } from 'zlib'
|
|
import path from 'path'
|
|
|
|
type TrafficStatusBucket = '2xx' | '3xx' | '4xx' | '5xx' | 'other'
|
|
|
|
export interface PublicTrafficRange {
|
|
start: Date
|
|
end: Date
|
|
}
|
|
|
|
export interface PublicTrafficSourceIssue {
|
|
path: string
|
|
reason: string
|
|
}
|
|
|
|
export interface PublicTrafficLogEntry {
|
|
timestamp: Date
|
|
domain: string
|
|
remoteAddress: string
|
|
method: string
|
|
path: string
|
|
status: number
|
|
responseBytes: number
|
|
requestBytes: number | null
|
|
}
|
|
|
|
export interface PublicTrafficPurpose {
|
|
key: string
|
|
label: string
|
|
category: string
|
|
detail: string
|
|
}
|
|
|
|
export interface PublicTrafficStats {
|
|
generatedAt: string
|
|
available: boolean
|
|
range: {
|
|
start: string
|
|
end: string
|
|
}
|
|
summary: {
|
|
responseBytes: number
|
|
requestBytes: number | null
|
|
requestBytesKnownCount: number
|
|
totalRequests: number
|
|
uniqueIpCount: number
|
|
statusCounts: Record<TrafficStatusBucket, number>
|
|
firstSeen: string | null
|
|
lastSeen: string | null
|
|
}
|
|
byDay: Array<{
|
|
date: string
|
|
responseBytes: number
|
|
requestBytes: number | null
|
|
requestBytesKnownCount: number
|
|
totalRequests: number
|
|
}>
|
|
byDomain: Array<{
|
|
domain: string
|
|
responseBytes: number
|
|
requestBytes: number | null
|
|
requestBytesKnownCount: number
|
|
totalRequests: number
|
|
statusCounts: Record<TrafficStatusBucket, number>
|
|
}>
|
|
byPurpose: Array<{
|
|
key: string
|
|
label: string
|
|
category: string
|
|
detail: string
|
|
responseBytes: number
|
|
requestBytes: number | null
|
|
requestBytesKnownCount: number
|
|
totalRequests: number
|
|
}>
|
|
topPaths: Array<{
|
|
path: string
|
|
method: string
|
|
purposeLabel: string
|
|
purposeCategory: string
|
|
purposeDetail: string
|
|
responseBytes: number
|
|
totalRequests: number
|
|
}>
|
|
source: {
|
|
logPaths: string[]
|
|
scannedFiles: number
|
|
scannedLines: number
|
|
matchedLines: number
|
|
skippedLines: number
|
|
truncated: boolean
|
|
issues: PublicTrafficSourceIssue[]
|
|
requestBytesAvailable: boolean
|
|
cache?: {
|
|
persisted: boolean
|
|
hit: boolean
|
|
stale: boolean
|
|
refreshInProgress: boolean
|
|
refreshedAt: string | null
|
|
expiresAt: string | null
|
|
}
|
|
}
|
|
}
|
|
|
|
interface PublicTrafficAggregationState {
|
|
uniqueIps: Set<string>
|
|
dayMap: Map<string, PublicTrafficStats['byDay'][number]>
|
|
domainMap: Map<string, PublicTrafficStats['byDomain'][number]>
|
|
purposeMap: Map<string, PublicTrafficStats['byPurpose'][number]>
|
|
topPathMap: Map<string, PublicTrafficStats['topPaths'][number]>
|
|
}
|
|
|
|
interface PublicTrafficCacheEntry {
|
|
key: string
|
|
createdAt: number
|
|
expiresAt: number
|
|
stats: PublicTrafficStats
|
|
}
|
|
|
|
interface PublicTrafficStatsOptions {
|
|
forceRefresh?: boolean
|
|
}
|
|
|
|
export type PublicTrafficPreset = 'today' | 'last7' | 'last30'
|
|
|
|
export type PublicTrafficRollupGranularity = '5m' | 'day'
|
|
|
|
export interface PublicTrafficRollupInput {
|
|
bucketStart: string
|
|
bucketEnd: string
|
|
businessDate?: string | null
|
|
granularity: PublicTrafficRollupGranularity
|
|
domain: string
|
|
method?: string | null
|
|
path: string
|
|
responseBytes: number
|
|
requestBytes?: number | null
|
|
requestBytesKnownCount?: number | null
|
|
totalRequests: number
|
|
statusCounts?: Partial<Record<TrafficStatusBucket, number>> | null
|
|
sourceServer?: string | null
|
|
generatedAt?: string | null
|
|
frozen?: boolean | null
|
|
}
|
|
|
|
export interface PublicTrafficRollupImportResult {
|
|
received: number
|
|
accepted: number
|
|
skipped: number
|
|
}
|
|
|
|
interface PublicTrafficScanContext {
|
|
logPaths: string[]
|
|
maxLines: number
|
|
stableCacheKey: string
|
|
sourceSignature: string
|
|
ttlSeconds: number
|
|
}
|
|
|
|
interface PersistentTrafficCacheRow {
|
|
stats: PublicTrafficStats | string
|
|
refreshed_at: Date | string
|
|
expires_at: Date | string
|
|
}
|
|
|
|
interface PublicTrafficRollupRow {
|
|
bucket_start: Date | string
|
|
bucket_end: Date | string
|
|
business_date: string
|
|
granularity: PublicTrafficRollupGranularity
|
|
domain: string
|
|
method: string
|
|
path: string
|
|
purpose_key: string
|
|
purpose_label: string
|
|
purpose_category: string
|
|
purpose_detail: string
|
|
response_bytes: bigint | number | string
|
|
request_bytes: bigint | number | string | null
|
|
request_bytes_known_count: number
|
|
total_requests: number
|
|
status_2xx: number
|
|
status_3xx: number
|
|
status_4xx: number
|
|
status_5xx: number
|
|
status_other: number
|
|
}
|
|
|
|
interface PublicTrafficRollupAggregationState {
|
|
dayMap: Map<string, PublicTrafficStats['byDay'][number]>
|
|
domainMap: Map<string, PublicTrafficStats['byDomain'][number]>
|
|
purposeMap: Map<string, PublicTrafficStats['byPurpose'][number]>
|
|
topPathMap: Map<string, PublicTrafficStats['topPaths'][number]>
|
|
}
|
|
|
|
const MONTHS: Record<string, number> = {
|
|
Jan: 0,
|
|
Feb: 1,
|
|
Mar: 2,
|
|
Apr: 3,
|
|
May: 4,
|
|
Jun: 5,
|
|
Jul: 6,
|
|
Aug: 7,
|
|
Sep: 8,
|
|
Oct: 9,
|
|
Nov: 10,
|
|
Dec: 11,
|
|
}
|
|
|
|
const DEFAULT_LOG_PATHS = [
|
|
'/var/log/nginx/sc.jiansun.vip.access.log',
|
|
'/var/log/nginx/sc.jiansun.vip.access.log.1',
|
|
'/var/log/nginx/s3.jiansun.vip.access.log',
|
|
'/var/log/nginx/s3.jiansun.vip.access.log.1',
|
|
'/var/log/nginx/cp.jiansun.vip.access.log',
|
|
'/var/log/nginx/cp.jiansun.vip.access.log.1',
|
|
]
|
|
|
|
const statsCache = new Map<string, PublicTrafficCacheEntry>()
|
|
const MAX_CACHE_ENTRIES = 20
|
|
const refreshPromises = new Map<string, Promise<void>>()
|
|
let persistentCacheTableEnsured = false
|
|
let rollupTableEnsured = false
|
|
|
|
function emptyStatusCounts(): Record<TrafficStatusBucket, number> {
|
|
return {
|
|
'2xx': 0,
|
|
'3xx': 0,
|
|
'4xx': 0,
|
|
'5xx': 0,
|
|
other: 0,
|
|
}
|
|
}
|
|
|
|
function createAggregationState(): PublicTrafficAggregationState {
|
|
return {
|
|
uniqueIps: new Set<string>(),
|
|
dayMap: new Map(),
|
|
domainMap: new Map(),
|
|
purposeMap: new Map(),
|
|
topPathMap: new Map(),
|
|
}
|
|
}
|
|
|
|
function createRollupAggregationState(): PublicTrafficRollupAggregationState {
|
|
return {
|
|
dayMap: new Map(),
|
|
domainMap: new Map(),
|
|
purposeMap: new Map(),
|
|
topPathMap: new Map(),
|
|
}
|
|
}
|
|
|
|
function splitEnvList(value: string | undefined): string[] {
|
|
return String(value || '')
|
|
.split(/[\n,;]/)
|
|
.map((item) => item.trim())
|
|
.filter(Boolean)
|
|
}
|
|
|
|
export function getConfiguredPublicTrafficLogPaths(): string[] {
|
|
const configured = splitEnvList(process.env.PUBLIC_TRAFFIC_LOG_PATHS)
|
|
return configured.length > 0 ? configured : DEFAULT_LOG_PATHS
|
|
}
|
|
|
|
export function normalizeTrafficRange(startValue?: string | null, endValue?: string | null): PublicTrafficRange {
|
|
const now = new Date()
|
|
const fallbackEnd = new Date(now)
|
|
const fallbackStart = new Date(now.getTime() - 6 * 24 * 60 * 60 * 1000)
|
|
fallbackStart.setHours(0, 0, 0, 0)
|
|
fallbackEnd.setHours(23, 59, 59, 999)
|
|
|
|
const parsedStart = startValue ? new Date(`${startValue}T00:00:00+08:00`) : fallbackStart
|
|
const parsedEnd = endValue ? new Date(`${endValue}T23:59:59.999+08:00`) : fallbackEnd
|
|
|
|
const start = Number.isNaN(parsedStart.getTime()) ? fallbackStart : parsedStart
|
|
const end = Number.isNaN(parsedEnd.getTime()) ? fallbackEnd : parsedEnd
|
|
|
|
if (start.getTime() > end.getTime()) {
|
|
return { start: end, end: start }
|
|
}
|
|
|
|
return { start, end }
|
|
}
|
|
|
|
function toChinaDateString(date: Date): string {
|
|
return new Intl.DateTimeFormat('en-CA', {
|
|
timeZone: 'Asia/Shanghai',
|
|
year: 'numeric',
|
|
month: '2-digit',
|
|
day: '2-digit',
|
|
}).format(date)
|
|
}
|
|
|
|
function addDays(date: Date, days: number): Date {
|
|
return new Date(date.getTime() + days * 24 * 60 * 60 * 1000)
|
|
}
|
|
|
|
export function getPublicTrafficPresetRange(preset: PublicTrafficPreset, now = new Date()): PublicTrafficRange {
|
|
const endDate = toChinaDateString(now)
|
|
const daysBack = preset === 'today' ? 0 : preset === 'last7' ? 6 : 29
|
|
const startDate = toChinaDateString(addDays(now, -daysBack))
|
|
return normalizeTrafficRange(startDate, endDate)
|
|
}
|
|
|
|
function parseNginxTime(value: string): Date | null {
|
|
const match = value.match(/^(\d{1,2})\/([A-Za-z]{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2}) ([+-]\d{4})$/)
|
|
if (!match) return null
|
|
|
|
const [, day, monthName, year, hour, minute, second, offset] = match
|
|
const month = MONTHS[monthName]
|
|
if (month === undefined) return null
|
|
|
|
const isoOffset = `${offset.slice(0, 3)}:${offset.slice(3)}`
|
|
const date = new Date(
|
|
`${year}-${String(month + 1).padStart(2, '0')}-${String(day).padStart(2, '0')}T${hour}:${minute}:${second}${isoOffset}`,
|
|
)
|
|
return Number.isNaN(date.getTime()) ? null : date
|
|
}
|
|
|
|
function inferDomainFromPath(logPath: string): string {
|
|
const fileName = path.basename(logPath)
|
|
const matched = fileName.match(/([a-z0-9-]+(?:\.[a-z0-9-]+)+)\.access\.log/i)
|
|
if (matched) return matched[1]
|
|
if (fileName.includes('sc.')) return 'sc.jiansun.vip'
|
|
if (fileName.includes('s3.')) return 's3.jiansun.vip'
|
|
if (fileName.includes('cp.')) return 'cp.jiansun.vip'
|
|
return fileName.replace(/\.log(?:\.\d+|\.gz)?$/i, '')
|
|
}
|
|
|
|
function normalizeRequestTarget(request: string) {
|
|
const parts = request.trim().split(/\s+/)
|
|
const method = parts[0] || 'UNKNOWN'
|
|
const rawPath = parts[1] || '/'
|
|
const withoutQuery = rawPath.split('?')[0] || '/'
|
|
return {
|
|
method,
|
|
path: withoutQuery.length > 180 ? `${withoutQuery.slice(0, 177)}...` : withoutQuery,
|
|
}
|
|
}
|
|
|
|
function statusBucket(status: number): TrafficStatusBucket {
|
|
if (status >= 200 && status < 300) return '2xx'
|
|
if (status >= 300 && status < 400) return '3xx'
|
|
if (status >= 400 && status < 500) return '4xx'
|
|
if (status >= 500 && status < 600) return '5xx'
|
|
return 'other'
|
|
}
|
|
|
|
function parseNumber(value: unknown): number | null {
|
|
if (typeof value === 'number' && Number.isFinite(value)) return Math.max(0, value)
|
|
if (typeof value === 'string' && value.trim() && value !== '-') {
|
|
const parsed = Number.parseInt(value, 10)
|
|
return Number.isFinite(parsed) ? Math.max(0, parsed) : null
|
|
}
|
|
return null
|
|
}
|
|
|
|
function safeDecodePath(value: string): string {
|
|
try {
|
|
return decodeURIComponent(value)
|
|
} catch {
|
|
return value
|
|
}
|
|
}
|
|
|
|
function isAssetPath(pathValue: string): boolean {
|
|
return /\.(?:avif|webp|png|jpe?g|gif|svg|mp4|webm|mp3|wav|zip|rar|7z|pdf|psd|ai|cdr)$/i.test(pathValue)
|
|
}
|
|
|
|
function assetTypeLabel(pathValue: string): string {
|
|
if (/\.(?:avif|webp|png|jpe?g|gif|svg)$/i.test(pathValue)) return '图片直链访问'
|
|
if (/\.(?:mp4|webm)$/i.test(pathValue)) return '视频直链访问'
|
|
if (/\.(?:mp3|wav)$/i.test(pathValue)) return '音频直链访问'
|
|
if (/\.(?:zip|rar|7z)$/i.test(pathValue)) return '压缩包直链访问'
|
|
if (/\.(?:pdf|psd|ai|cdr)$/i.test(pathValue)) return '设计文件直链访问'
|
|
return '资源文件直链访问'
|
|
}
|
|
|
|
function fileNameFromPath(pathValue: string): string {
|
|
const rawName = pathValue.split('/').filter(Boolean).pop() || pathValue
|
|
return safeDecodePath(rawName)
|
|
}
|
|
|
|
export function classifyPublicTrafficPurpose(domain: string, pathValue: string, method = 'GET'): PublicTrafficPurpose {
|
|
const pathLower = pathValue.toLowerCase()
|
|
const domainLower = domain.toLowerCase()
|
|
const methodUpper = method.toUpperCase()
|
|
|
|
if (pathLower === '/api/materials/download') {
|
|
return {
|
|
key: 'material-download',
|
|
label: '素材下载/打包下载',
|
|
category: '素材库',
|
|
detail: '用户下载素材原文件、压缩包或分享页下载时产生的流量。',
|
|
}
|
|
}
|
|
|
|
if (pathLower === '/api/external/download') {
|
|
return {
|
|
key: 'external-material-download',
|
|
label: '外部平台素材下载',
|
|
category: '外部接口',
|
|
detail: '脉象、产品平台等外部系统通过下载 token 拉取素材文件。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/api/external/')) {
|
|
return {
|
|
key: 'external-api',
|
|
label: '外部平台对接接口',
|
|
category: '外部接口',
|
|
detail: '外部系统查询素材、SPU、SKU、条码或图片信息。',
|
|
}
|
|
}
|
|
|
|
if (domainLower === 'cp.jiansun.vip') {
|
|
if (pathLower === '/api/intake/non-integrated') {
|
|
return {
|
|
key: 'product-intake-non-integrated',
|
|
label: '产品设计平台未集成素材列表',
|
|
category: '产品设计平台',
|
|
detail: '产品设计平台查看未集成素材或待处理录入内容。',
|
|
}
|
|
}
|
|
if (pathLower === '/api/intake/product-patents') {
|
|
return {
|
|
key: 'product-intake-submit',
|
|
label: '产品设计平台资料提交',
|
|
category: '产品设计平台',
|
|
detail: '产品设计平台提交产品图片、专利或录入资料。',
|
|
}
|
|
}
|
|
if (pathLower === '/api/storage/object') {
|
|
return {
|
|
key: 'product-storage-object',
|
|
label: '产品设计平台取图跳转',
|
|
category: '产品设计平台',
|
|
detail: '产品设计平台请求对象文件地址，通常后续由存储或 CDN 承接真实图片流量。',
|
|
}
|
|
}
|
|
if (pathLower.startsWith('/api/intake/')) {
|
|
return {
|
|
key: 'product-intake-api',
|
|
label: '产品设计平台录入接口',
|
|
category: '产品设计平台',
|
|
detail: '产品设计平台录入、查询或处理产品资料的接口请求。',
|
|
}
|
|
}
|
|
if (pathLower.startsWith('/api/')) {
|
|
return {
|
|
key: 'product-platform-api',
|
|
label: '产品设计平台业务接口',
|
|
category: '产品设计平台',
|
|
detail: '产品设计平台页面调用的业务接口。',
|
|
}
|
|
}
|
|
return {
|
|
key: 'product-platform-page',
|
|
label: '产品设计平台页面访问',
|
|
category: '产品设计平台',
|
|
detail: '用户打开产品设计平台页面或加载前端静态资源。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/gongxiangpan/public-display/white-background/')) {
|
|
return {
|
|
key: 'white-background-preview',
|
|
label: '标准模型白底图预览',
|
|
category: '资源直链',
|
|
detail: '白底图等标准模型展示图片从资源入口直接返回。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/gongxiangpan/public-display/task-detail/')) {
|
|
return {
|
|
key: 'task-detail-preview',
|
|
label: '任务详情素材预览图',
|
|
category: '资源直链',
|
|
detail: '素材详情、主图、SKU 图、详情图等预览资源直链访问。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/gongxiangpan/public-display/materials/')) {
|
|
return {
|
|
key: 'material-preview',
|
|
label: '素材预览图访问',
|
|
category: '资源直链',
|
|
detail: '素材库列表、详情或分享页展示预览图。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/gongxiangpan/uploads/direct/')) {
|
|
return {
|
|
key: 'direct-upload-asset',
|
|
label: assetTypeLabel(pathValue),
|
|
category: '直传资源',
|
|
detail: '直传上传目录下的图片、视频、压缩包等资源直链汇总，具体文件请看“流量最高路径”。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/gongxiangpan/uploads/')) {
|
|
return {
|
|
key: 'legacy-upload-asset',
|
|
label: assetTypeLabel(pathValue),
|
|
category: '历史上传资源',
|
|
detail: '历史上传目录下的图片、视频、压缩包等资源直链汇总，具体文件请看“流量最高路径”。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/gongxiangpan/product-package/')) {
|
|
return {
|
|
key: 'product-package-asset',
|
|
label: '产品包资源访问',
|
|
category: '资源直链',
|
|
detail: '产品包、原始文件或成套资源通过资源入口直接访问。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/gongxiangpan/') || domainLower === 's3.jiansun.vip') {
|
|
return {
|
|
key: 'storage-asset',
|
|
label: isAssetPath(pathValue) ? assetTypeLabel(pathValue) : '资源存储入口访问',
|
|
category: '资源直链',
|
|
detail: isAssetPath(pathValue)
|
|
? '资源入口直接返回的对象存储文件汇总，具体文件请看“流量最高路径”。'
|
|
: '资源入口上的对象存储、探测或目录类请求。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/api/materials/upload/')) {
|
|
return {
|
|
key: 'material-upload',
|
|
label: methodUpper === 'GET' ? '上传状态查询' : '素材上传接口',
|
|
category: '素材库',
|
|
detail: '素材上传分片、合并、刷新或上传状态相关请求。',
|
|
}
|
|
}
|
|
|
|
if (pathLower === '/api/materials') {
|
|
return {
|
|
key: 'material-list-search',
|
|
label: '素材列表/搜索接口',
|
|
category: '素材库',
|
|
detail: '首页、素材列表或筛选搜索素材时加载的数据。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/api/materials/')) {
|
|
return {
|
|
key: 'material-detail-api',
|
|
label: '素材详情/管理接口',
|
|
category: '素材库',
|
|
detail: '素材详情、SKU、资产排序、恢复、收藏、点赞等接口请求。',
|
|
}
|
|
}
|
|
|
|
if (pathLower === '/api/asset-center/substrates') {
|
|
return {
|
|
key: 'asset-center-substrates',
|
|
label: '资产中心基底素材列表',
|
|
category: '素材库',
|
|
detail: '资产中心加载基底素材、贴图或可选素材列表。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/api/asset-center/')) {
|
|
return {
|
|
key: 'asset-center-api',
|
|
label: '资产中心接口',
|
|
category: '素材库',
|
|
detail: '资产中心页面的数据查询或操作接口。',
|
|
}
|
|
}
|
|
|
|
if (pathLower === '/api/rankings') {
|
|
return {
|
|
key: 'home-ranking',
|
|
label: '首页排行榜接口',
|
|
category: '素材库',
|
|
detail: '首页展示热门、排行或推荐素材数据。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/api/auth/') || pathLower.startsWith('/api/dingtalk/') || pathLower === '/dingtalk-callback') {
|
|
return {
|
|
key: 'login-auth',
|
|
label: '登录/钉钉认证',
|
|
category: '账号',
|
|
detail: '登录状态、钉钉回调、会话检查等账号认证请求。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/api/performance-survey') || pathLower.startsWith('/performance-survey')) {
|
|
return {
|
|
key: 'performance-survey',
|
|
label: '考核调研功能',
|
|
category: '素材库',
|
|
detail: '考核调研页面、列表、提交和结果查看相关请求。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/api/work-review') || pathLower.startsWith('/work-review')) {
|
|
return {
|
|
key: 'work-review',
|
|
label: '作品过稿看板',
|
|
category: '素材库',
|
|
detail: '作品提交、版本留档、主管与元泽评审相关请求。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/admin') || pathLower.startsWith('/api/admin/')) {
|
|
return {
|
|
key: 'admin-console',
|
|
label: '后台管理功能',
|
|
category: '后台',
|
|
detail: '管理员后台页面和管理接口请求。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/share/')) {
|
|
return {
|
|
key: 'share-page',
|
|
label: '素材分享页',
|
|
category: '素材库',
|
|
detail: '外部或内部用户打开素材分享页面。',
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/_next/') || pathLower === '/favicon.ico' || pathLower === '/api/favicon') {
|
|
return {
|
|
key: 'frontend-static',
|
|
label: '前端静态资源',
|
|
category: '页面加载',
|
|
detail: '页面 JS、CSS、图标等前端基础资源。',
|
|
}
|
|
}
|
|
|
|
if (isAssetPath(pathValue)) {
|
|
return {
|
|
key: 'static-asset',
|
|
label: assetTypeLabel(pathValue),
|
|
category: '资源文件',
|
|
detail: `直接访问的文件：${fileNameFromPath(pathValue)}`,
|
|
}
|
|
}
|
|
|
|
if (pathLower.startsWith('/api/')) {
|
|
return {
|
|
key: 'other-api',
|
|
label: '其他业务接口',
|
|
category: '其他',
|
|
detail: '尚未细分到具体模块的接口请求。',
|
|
}
|
|
}
|
|
|
|
return {
|
|
key: 'other-page',
|
|
label: '其他页面访问',
|
|
category: '其他',
|
|
detail: '尚未细分到具体模块的页面或资源请求。',
|
|
}
|
|
}
|
|
|
|
function parseJsonLogLine(line: string, fallbackDomain: string): PublicTrafficLogEntry | null {
|
|
try {
|
|
const raw = JSON.parse(line) as Record<string, unknown>
|
|
const timeValue = String(raw.time_local || raw.time || raw.timestamp || '')
|
|
const timestamp = timeValue.includes('/') ? parseNginxTime(timeValue) : new Date(timeValue)
|
|
if (!timestamp || Number.isNaN(timestamp.getTime())) return null
|
|
|
|
const request = String(raw.request || `${raw.method || ''} ${raw.uri || raw.path || '/'} HTTP/1.1`)
|
|
const target = normalizeRequestTarget(request)
|
|
const status = parseNumber(raw.status) || 0
|
|
const responseBytes =
|
|
parseNumber(raw.body_bytes_sent) ??
|
|
parseNumber(raw.bytes_sent) ??
|
|
parseNumber(raw.response_bytes) ??
|
|
0
|
|
const requestBytes = parseNumber(raw.request_length) ?? parseNumber(raw.request_bytes)
|
|
|
|
return {
|
|
timestamp,
|
|
domain: String(raw.host || raw.server_name || fallbackDomain),
|
|
remoteAddress: String(raw.remote_addr || raw.remoteAddress || ''),
|
|
method: target.method,
|
|
path: target.path,
|
|
status,
|
|
responseBytes,
|
|
requestBytes,
|
|
}
|
|
} catch {
|
|
return null
|
|
}
|
|
}
|
|
|
|
export function parsePublicTrafficLogLine(line: string, fallbackDomain: string): PublicTrafficLogEntry | null {
|
|
const trimmed = line.trim()
|
|
if (!trimmed) return null
|
|
|
|
if (trimmed.startsWith('{')) {
|
|
return parseJsonLogLine(trimmed, fallbackDomain)
|
|
}
|
|
|
|
const match = trimmed.match(
|
|
/^(\S+)\s+\S+\s+\S+\s+\[([^\]]+)]\s+"([^"]*)"\s+(\d{3})\s+(\d+|-)(?:\s+"[^"]*"\s+"[^"]*")?(.*)$/,
|
|
)
|
|
if (!match) return null
|
|
|
|
const [, remoteAddress, timeText, request, statusText, responseBytesText, tail] = match
|
|
const timestamp = parseNginxTime(timeText)
|
|
if (!timestamp) return null
|
|
|
|
const target = normalizeRequestTarget(request)
|
|
const bytesSentMatch = tail.match(/(?:bytes_sent|sent_bytes|response_bytes)=(\d+)/i)
|
|
const requestLengthMatch = tail.match(/(?:request_length|req_len|requestBytes|request_bytes)=(\d+)/i)
|
|
const requestBytes = requestLengthMatch ? parseNumber(requestLengthMatch[1]) : null
|
|
|
|
return {
|
|
timestamp,
|
|
domain: fallbackDomain,
|
|
remoteAddress,
|
|
method: target.method,
|
|
path: target.path,
|
|
status: Number.parseInt(statusText, 10),
|
|
responseBytes: bytesSentMatch ? parseNumber(bytesSentMatch[1]) || 0 : parseNumber(responseBytesText) || 0,
|
|
requestBytes,
|
|
}
|
|
}
|
|
|
|
function dateKey(date: Date): string {
|
|
const formatter = new Intl.DateTimeFormat('en-CA', {
|
|
timeZone: 'Asia/Shanghai',
|
|
year: 'numeric',
|
|
month: '2-digit',
|
|
day: '2-digit',
|
|
})
|
|
return formatter.format(date)
|
|
}
|
|
|
|
function addEntry(stats: PublicTrafficStats, aggregation: PublicTrafficAggregationState, entry: PublicTrafficLogEntry) {
|
|
const bucket = statusBucket(entry.status)
|
|
stats.summary.responseBytes += entry.responseBytes
|
|
stats.summary.totalRequests += 1
|
|
stats.summary.statusCounts[bucket] += 1
|
|
if (entry.remoteAddress) aggregation.uniqueIps.add(entry.remoteAddress)
|
|
|
|
if (entry.requestBytes !== null) {
|
|
stats.summary.requestBytes = (stats.summary.requestBytes || 0) + entry.requestBytes
|
|
stats.summary.requestBytesKnownCount += 1
|
|
}
|
|
|
|
const seenIso = entry.timestamp.toISOString()
|
|
if (!stats.summary.firstSeen || seenIso < stats.summary.firstSeen) stats.summary.firstSeen = seenIso
|
|
if (!stats.summary.lastSeen || seenIso > stats.summary.lastSeen) stats.summary.lastSeen = seenIso
|
|
|
|
const key = dateKey(entry.timestamp)
|
|
let day = aggregation.dayMap.get(key)
|
|
if (!day) {
|
|
day = {
|
|
date: key,
|
|
responseBytes: 0,
|
|
requestBytes: null,
|
|
requestBytesKnownCount: 0,
|
|
totalRequests: 0,
|
|
}
|
|
aggregation.dayMap.set(key, day)
|
|
stats.byDay.push(day)
|
|
}
|
|
day.responseBytes += entry.responseBytes
|
|
day.totalRequests += 1
|
|
if (entry.requestBytes !== null) {
|
|
day.requestBytes = (day.requestBytes || 0) + entry.requestBytes
|
|
day.requestBytesKnownCount += 1
|
|
}
|
|
|
|
let domain = aggregation.domainMap.get(entry.domain)
|
|
if (!domain) {
|
|
domain = {
|
|
domain: entry.domain,
|
|
responseBytes: 0,
|
|
requestBytes: null,
|
|
requestBytesKnownCount: 0,
|
|
totalRequests: 0,
|
|
statusCounts: emptyStatusCounts(),
|
|
}
|
|
aggregation.domainMap.set(entry.domain, domain)
|
|
stats.byDomain.push(domain)
|
|
}
|
|
domain.responseBytes += entry.responseBytes
|
|
domain.totalRequests += 1
|
|
domain.statusCounts[bucket] += 1
|
|
if (entry.requestBytes !== null) {
|
|
domain.requestBytes = (domain.requestBytes || 0) + entry.requestBytes
|
|
domain.requestBytesKnownCount += 1
|
|
}
|
|
|
|
const purpose = classifyPublicTrafficPurpose(entry.domain, entry.path, entry.method)
|
|
let purposeStat = aggregation.purposeMap.get(purpose.key)
|
|
if (!purposeStat) {
|
|
purposeStat = {
|
|
key: purpose.key,
|
|
label: purpose.label,
|
|
category: purpose.category,
|
|
detail: purpose.detail,
|
|
responseBytes: 0,
|
|
requestBytes: null,
|
|
requestBytesKnownCount: 0,
|
|
totalRequests: 0,
|
|
}
|
|
aggregation.purposeMap.set(purpose.key, purposeStat)
|
|
stats.byPurpose.push(purposeStat)
|
|
}
|
|
purposeStat.responseBytes += entry.responseBytes
|
|
purposeStat.totalRequests += 1
|
|
if (entry.requestBytes !== null) {
|
|
purposeStat.requestBytes = (purposeStat.requestBytes || 0) + entry.requestBytes
|
|
purposeStat.requestBytesKnownCount += 1
|
|
}
|
|
|
|
const pathKey = `${entry.method} ${entry.path}`
|
|
let topPath = aggregation.topPathMap.get(pathKey)
|
|
if (!topPath) {
|
|
topPath = {
|
|
method: entry.method,
|
|
path: entry.path,
|
|
purposeLabel: purpose.label,
|
|
purposeCategory: purpose.category,
|
|
purposeDetail: purpose.detail,
|
|
responseBytes: 0,
|
|
totalRequests: 0,
|
|
}
|
|
aggregation.topPathMap.set(pathKey, topPath)
|
|
stats.topPaths.push(topPath)
|
|
}
|
|
topPath.responseBytes += entry.responseBytes
|
|
topPath.totalRequests += 1
|
|
}
|
|
|
|
async function scanLogFile(
|
|
logPath: string,
|
|
range: PublicTrafficRange,
|
|
stats: PublicTrafficStats,
|
|
aggregation: PublicTrafficAggregationState,
|
|
maxLines: number,
|
|
) {
|
|
if (!existsSync(logPath)) {
|
|
stats.source.issues.push({ path: logPath, reason: '文件不存在' })
|
|
return
|
|
}
|
|
|
|
const fileStat = statSync(logPath)
|
|
if (!fileStat.isFile()) {
|
|
stats.source.issues.push({ path: logPath, reason: '不是文件' })
|
|
return
|
|
}
|
|
|
|
const fallbackDomain = inferDomainFromPath(logPath)
|
|
const fileStream = createReadStream(logPath)
|
|
const input = logPath.endsWith('.gz') ? fileStream.pipe(createGunzip()) : fileStream
|
|
const reader = createInterface({ input, crlfDelay: Infinity })
|
|
|
|
stats.source.scannedFiles += 1
|
|
|
|
for await (const line of reader) {
|
|
if (stats.source.scannedLines >= maxLines) {
|
|
stats.source.truncated = true
|
|
reader.close()
|
|
break
|
|
}
|
|
|
|
stats.source.scannedLines += 1
|
|
const entry = parsePublicTrafficLogLine(line, fallbackDomain)
|
|
if (!entry) {
|
|
stats.source.skippedLines += 1
|
|
continue
|
|
}
|
|
|
|
if (entry.timestamp < range.start || entry.timestamp > range.end) continue
|
|
|
|
stats.source.matchedLines += 1
|
|
addEntry(stats, aggregation, entry)
|
|
}
|
|
}
|
|
|
|
function buildStableCacheKey(range: PublicTrafficRange, logPaths: string[], maxLines: number): string {
|
|
return [
|
|
'public-traffic-v2',
|
|
range.start.toISOString(),
|
|
range.end.toISOString(),
|
|
maxLines,
|
|
logPaths.join('|'),
|
|
].join('::')
|
|
}
|
|
|
|
function buildSourceSignature(logPaths: string[]): string {
|
|
return logPaths
|
|
.map((logPath) => {
|
|
try {
|
|
const fileStat = statSync(logPath)
|
|
return `${logPath}:${fileStat.size}:${Math.round(fileStat.mtimeMs)}`
|
|
} catch {
|
|
return `${logPath}:missing`
|
|
}
|
|
})
|
|
.join('|')
|
|
}
|
|
|
|
function getCacheTtlSeconds() {
|
|
const parsed = Number.parseInt(process.env.PUBLIC_TRAFFIC_STATS_CACHE_TTL_SECONDS || '900', 10)
|
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : 900
|
|
}
|
|
|
|
function getScanContext(range: PublicTrafficRange): PublicTrafficScanContext {
|
|
const logPaths = getConfiguredPublicTrafficLogPaths()
|
|
const maxLines = Math.max(1000, Number.parseInt(process.env.PUBLIC_TRAFFIC_MAX_LINES || '1000000', 10) || 1000000)
|
|
return {
|
|
logPaths,
|
|
maxLines,
|
|
stableCacheKey: buildStableCacheKey(range, logPaths, maxLines),
|
|
sourceSignature: buildSourceSignature(logPaths),
|
|
ttlSeconds: getCacheTtlSeconds(),
|
|
}
|
|
}
|
|
|
|
function cloneStats(stats: PublicTrafficStats): PublicTrafficStats {
|
|
return JSON.parse(JSON.stringify(stats)) as PublicTrafficStats
|
|
}
|
|
|
|
function withCacheMetadata(
|
|
stats: PublicTrafficStats,
|
|
cache: NonNullable<PublicTrafficStats['source']['cache']>,
|
|
): PublicTrafficStats {
|
|
const nextStats = cloneStats(stats)
|
|
nextStats.source.cache = cache
|
|
return nextStats
|
|
}
|
|
|
|
function persistentTrafficCacheEnabled() {
|
|
return process.env.NODE_ENV !== 'test' && process.env.PUBLIC_TRAFFIC_STATS_DB_CACHE !== '0' && Boolean(process.env.DATABASE_URL)
|
|
}
|
|
|
|
async function ensurePersistentTrafficCacheTable() {
|
|
if (persistentCacheTableEnsured) return true
|
|
if (!persistentTrafficCacheEnabled()) return false
|
|
|
|
const { prisma } = await import('./prisma')
|
|
|
|
await prisma.$executeRawUnsafe(`
|
|
CREATE TABLE IF NOT EXISTS "public_traffic_stats_cache" (
|
|
"cache_key" TEXT PRIMARY KEY,
|
|
"range_start" TIMESTAMPTZ NOT NULL,
|
|
"range_end" TIMESTAMPTZ NOT NULL,
|
|
"stats" JSONB NOT NULL,
|
|
"source_signature" TEXT NOT NULL DEFAULT '',
|
|
"refreshed_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
"expires_at" TIMESTAMPTZ NOT NULL,
|
|
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
"updated_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
`)
|
|
await prisma.$executeRawUnsafe(`
|
|
CREATE INDEX IF NOT EXISTS "public_traffic_stats_cache_range_idx"
|
|
ON "public_traffic_stats_cache" ("range_start", "range_end")
|
|
`)
|
|
await prisma.$executeRawUnsafe(`
|
|
CREATE INDEX IF NOT EXISTS "public_traffic_stats_cache_expires_at_idx"
|
|
ON "public_traffic_stats_cache" ("expires_at")
|
|
`)
|
|
|
|
persistentCacheTableEnsured = true
|
|
return true
|
|
}
|
|
|
|
async function readPersistentTrafficCache(cacheKey: string): Promise<PersistentTrafficCacheRow | null> {
|
|
if (!(await ensurePersistentTrafficCacheTable())) return null
|
|
|
|
const { prisma } = await import('./prisma')
|
|
const rows = await prisma.$queryRawUnsafe<PersistentTrafficCacheRow[]>(
|
|
`
|
|
SELECT "stats", "refreshed_at", "expires_at"
|
|
FROM "public_traffic_stats_cache"
|
|
WHERE "cache_key" = $1
|
|
LIMIT 1
|
|
`,
|
|
cacheKey,
|
|
)
|
|
|
|
return rows[0] || null
|
|
}
|
|
|
|
async function writePersistentTrafficCache(
|
|
cacheKey: string,
|
|
range: PublicTrafficRange,
|
|
context: PublicTrafficScanContext,
|
|
stats: PublicTrafficStats,
|
|
) {
|
|
if (!(await ensurePersistentTrafficCacheTable())) return
|
|
|
|
const { prisma } = await import('./prisma')
|
|
const now = new Date()
|
|
const expiresAt = new Date(now.getTime() + context.ttlSeconds * 1000)
|
|
const statsForStorage = cloneStats(stats)
|
|
delete statsForStorage.source.cache
|
|
|
|
await prisma.$executeRawUnsafe(
|
|
`
|
|
INSERT INTO "public_traffic_stats_cache" (
|
|
"cache_key",
|
|
"range_start",
|
|
"range_end",
|
|
"stats",
|
|
"source_signature",
|
|
"refreshed_at",
|
|
"expires_at",
|
|
"updated_at"
|
|
)
|
|
VALUES ($1, $2::timestamptz, $3::timestamptz, $4::jsonb, $5, $6::timestamptz, $7::timestamptz, $6::timestamptz)
|
|
ON CONFLICT ("cache_key") DO UPDATE SET
|
|
"range_start" = EXCLUDED."range_start",
|
|
"range_end" = EXCLUDED."range_end",
|
|
"stats" = EXCLUDED."stats",
|
|
"source_signature" = EXCLUDED."source_signature",
|
|
"refreshed_at" = EXCLUDED."refreshed_at",
|
|
"expires_at" = EXCLUDED."expires_at",
|
|
"updated_at" = EXCLUDED."updated_at"
|
|
`,
|
|
cacheKey,
|
|
range.start.toISOString(),
|
|
range.end.toISOString(),
|
|
JSON.stringify(statsForStorage),
|
|
context.sourceSignature,
|
|
now.toISOString(),
|
|
expiresAt.toISOString(),
|
|
)
|
|
}
|
|
|
|
function bigintToNumber(value: bigint | number | string | null | undefined): number {
|
|
if (value === null || value === undefined) return 0
|
|
if (typeof value === 'bigint') return Number(value)
|
|
if (typeof value === 'number') return Number.isFinite(value) ? value : 0
|
|
const parsed = Number(value)
|
|
return Number.isFinite(parsed) ? parsed : 0
|
|
}
|
|
|
|
function normalizeStatusCounts(
|
|
counts?: Partial<Record<TrafficStatusBucket, number>> | null,
|
|
): Record<TrafficStatusBucket, number> {
|
|
return {
|
|
'2xx': Math.max(0, Math.floor(Number(counts?.['2xx'] || 0))),
|
|
'3xx': Math.max(0, Math.floor(Number(counts?.['3xx'] || 0))),
|
|
'4xx': Math.max(0, Math.floor(Number(counts?.['4xx'] || 0))),
|
|
'5xx': Math.max(0, Math.floor(Number(counts?.['5xx'] || 0))),
|
|
other: Math.max(0, Math.floor(Number(counts?.other || 0))),
|
|
}
|
|
}
|
|
|
|
function normalizeRollupInput(input: PublicTrafficRollupInput) {
|
|
const bucketStart = new Date(input.bucketStart)
|
|
const bucketEnd = new Date(input.bucketEnd)
|
|
const domain = String(input.domain || '').trim().toLowerCase()
|
|
const method = String(input.method || 'GET').trim().toUpperCase() || 'GET'
|
|
const pathValue = normalizeRequestTarget(`${method} ${String(input.path || '').trim() || '/'} HTTP/1.1`).path
|
|
const responseBytes = Math.max(0, Math.floor(Number(input.responseBytes || 0)))
|
|
const totalRequests = Math.max(0, Math.floor(Number(input.totalRequests || 0)))
|
|
const requestBytes =
|
|
input.requestBytes === null || input.requestBytes === undefined
|
|
? null
|
|
: Math.max(0, Math.floor(Number(input.requestBytes || 0)))
|
|
const requestBytesKnownCount = Math.max(0, Math.floor(Number(input.requestBytesKnownCount || 0)))
|
|
const statusCounts = normalizeStatusCounts(input.statusCounts)
|
|
|
|
if (
|
|
Number.isNaN(bucketStart.getTime()) ||
|
|
Number.isNaN(bucketEnd.getTime()) ||
|
|
bucketStart >= bucketEnd ||
|
|
!domain ||
|
|
!pathValue ||
|
|
totalRequests <= 0 ||
|
|
!['5m', 'day'].includes(input.granularity)
|
|
) {
|
|
return null
|
|
}
|
|
|
|
const purpose = classifyPublicTrafficPurpose(domain, pathValue, method)
|
|
const businessDate = input.businessDate && /^\d{4}-\d{2}-\d{2}$/.test(input.businessDate)
|
|
? input.businessDate
|
|
: toChinaDateString(bucketStart)
|
|
const generatedAt = input.generatedAt ? new Date(input.generatedAt) : new Date()
|
|
|
|
return {
|
|
bucketStart: bucketStart.toISOString(),
|
|
bucketEnd: bucketEnd.toISOString(),
|
|
businessDate,
|
|
granularity: input.granularity,
|
|
domain,
|
|
method,
|
|
path: pathValue,
|
|
purposeKey: purpose.key,
|
|
purposeLabel: purpose.label,
|
|
purposeCategory: purpose.category,
|
|
purposeDetail: purpose.detail,
|
|
responseBytes,
|
|
requestBytes,
|
|
requestBytesKnownCount,
|
|
totalRequests,
|
|
status2xx: statusCounts['2xx'],
|
|
status3xx: statusCounts['3xx'],
|
|
status4xx: statusCounts['4xx'],
|
|
status5xx: statusCounts['5xx'],
|
|
statusOther: statusCounts.other,
|
|
sourceServer: String(input.sourceServer || '').trim(),
|
|
generatedAt: Number.isNaN(generatedAt.getTime()) ? new Date().toISOString() : generatedAt.toISOString(),
|
|
frozen: Boolean(input.frozen),
|
|
}
|
|
}
|
|
|
|
type NormalizedPublicTrafficRollupRow = NonNullable<ReturnType<typeof normalizeRollupInput>>
|
|
|
|
function getRollupConflictKey(row: NormalizedPublicTrafficRollupRow) {
|
|
return [
|
|
row.businessDate,
|
|
row.granularity,
|
|
row.bucketStart,
|
|
row.bucketEnd,
|
|
row.domain,
|
|
row.method,
|
|
row.path,
|
|
].join('\u001f')
|
|
}
|
|
|
|
function mergeRollupRows(rows: NormalizedPublicTrafficRollupRow[]) {
|
|
const rowMap = new Map<string, NormalizedPublicTrafficRollupRow>()
|
|
|
|
for (const row of rows) {
|
|
const key = getRollupConflictKey(row)
|
|
const existing = rowMap.get(key)
|
|
|
|
if (!existing) {
|
|
rowMap.set(key, { ...row })
|
|
continue
|
|
}
|
|
|
|
existing.responseBytes += row.responseBytes
|
|
existing.totalRequests += row.totalRequests
|
|
existing.status2xx += row.status2xx
|
|
existing.status3xx += row.status3xx
|
|
existing.status4xx += row.status4xx
|
|
existing.status5xx += row.status5xx
|
|
existing.statusOther += row.statusOther
|
|
|
|
if (row.requestBytesKnownCount > 0) {
|
|
existing.requestBytes = (existing.requestBytes || 0) + (row.requestBytes || 0)
|
|
existing.requestBytesKnownCount += row.requestBytesKnownCount
|
|
}
|
|
|
|
if (row.generatedAt > existing.generatedAt) {
|
|
existing.generatedAt = row.generatedAt
|
|
}
|
|
|
|
existing.frozen = existing.frozen || row.frozen
|
|
if (row.sourceServer && !existing.sourceServer.includes(row.sourceServer)) {
|
|
existing.sourceServer = existing.sourceServer ? `${existing.sourceServer},${row.sourceServer}` : row.sourceServer
|
|
}
|
|
}
|
|
|
|
return Array.from(rowMap.values())
|
|
}
|
|
|
|
async function ensurePublicTrafficRollupTable() {
|
|
if (rollupTableEnsured) return true
|
|
if (!persistentTrafficCacheEnabled()) return false
|
|
|
|
const { prisma } = await import('./prisma')
|
|
|
|
await prisma.$executeRawUnsafe(`
|
|
CREATE TABLE IF NOT EXISTS "public_traffic_rollups" (
|
|
"id" BIGSERIAL PRIMARY KEY,
|
|
"bucket_start" TIMESTAMPTZ NOT NULL,
|
|
"bucket_end" TIMESTAMPTZ NOT NULL,
|
|
"business_date" TEXT NOT NULL,
|
|
"granularity" TEXT NOT NULL,
|
|
"domain" TEXT NOT NULL,
|
|
"method" TEXT NOT NULL,
|
|
"path" TEXT NOT NULL,
|
|
"purpose_key" TEXT NOT NULL,
|
|
"purpose_label" TEXT NOT NULL,
|
|
"purpose_category" TEXT NOT NULL,
|
|
"purpose_detail" TEXT NOT NULL,
|
|
"response_bytes" BIGINT NOT NULL DEFAULT 0,
|
|
"request_bytes" BIGINT,
|
|
"request_bytes_known_count" INTEGER NOT NULL DEFAULT 0,
|
|
"total_requests" INTEGER NOT NULL DEFAULT 0,
|
|
"status_2xx" INTEGER NOT NULL DEFAULT 0,
|
|
"status_3xx" INTEGER NOT NULL DEFAULT 0,
|
|
"status_4xx" INTEGER NOT NULL DEFAULT 0,
|
|
"status_5xx" INTEGER NOT NULL DEFAULT 0,
|
|
"status_other" INTEGER NOT NULL DEFAULT 0,
|
|
"source_server" TEXT NOT NULL DEFAULT '',
|
|
"generated_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
"frozen" BOOLEAN NOT NULL DEFAULT false,
|
|
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
"updated_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
CONSTRAINT "public_traffic_rollups_unique_bucket_path"
|
|
UNIQUE ("business_date", "granularity", "bucket_start", "bucket_end", "domain", "method", "path")
|
|
)
|
|
`)
|
|
await prisma.$executeRawUnsafe(`
|
|
CREATE INDEX IF NOT EXISTS "public_traffic_rollups_date_granularity_idx"
|
|
ON "public_traffic_rollups" ("business_date", "granularity")
|
|
`)
|
|
await prisma.$executeRawUnsafe(`
|
|
CREATE INDEX IF NOT EXISTS "public_traffic_rollups_bucket_idx"
|
|
ON "public_traffic_rollups" ("bucket_start", "bucket_end")
|
|
`)
|
|
|
|
rollupTableEnsured = true
|
|
return true
|
|
}
|
|
|
|
export async function upsertPublicTrafficRollups(
|
|
inputs: PublicTrafficRollupInput[],
|
|
): Promise<PublicTrafficRollupImportResult> {
|
|
if (!(await ensurePublicTrafficRollupTable())) {
|
|
throw new Error('公网流量汇总表未启用')
|
|
}
|
|
|
|
const rows: NormalizedPublicTrafficRollupRow[] = []
|
|
for (const input of inputs) {
|
|
const row = normalizeRollupInput(input)
|
|
if (row) rows.push(row)
|
|
}
|
|
|
|
if (rows.length === 0) {
|
|
return { received: inputs.length, accepted: 0, skipped: inputs.length }
|
|
}
|
|
|
|
const { prisma } = await import('./prisma')
|
|
const chunkSize = 500
|
|
const mergedRows = mergeRollupRows(rows)
|
|
|
|
for (let index = 0; index < mergedRows.length; index += chunkSize) {
|
|
const chunk = mergedRows.slice(index, index + chunkSize)
|
|
await prisma.$executeRawUnsafe(
|
|
`
|
|
INSERT INTO "public_traffic_rollups" (
|
|
"bucket_start",
|
|
"bucket_end",
|
|
"business_date",
|
|
"granularity",
|
|
"domain",
|
|
"method",
|
|
"path",
|
|
"purpose_key",
|
|
"purpose_label",
|
|
"purpose_category",
|
|
"purpose_detail",
|
|
"response_bytes",
|
|
"request_bytes",
|
|
"request_bytes_known_count",
|
|
"total_requests",
|
|
"status_2xx",
|
|
"status_3xx",
|
|
"status_4xx",
|
|
"status_5xx",
|
|
"status_other",
|
|
"source_server",
|
|
"generated_at",
|
|
"frozen",
|
|
"updated_at"
|
|
)
|
|
SELECT
|
|
"bucketStart"::timestamptz,
|
|
"bucketEnd"::timestamptz,
|
|
"businessDate",
|
|
"granularity",
|
|
"domain",
|
|
"method",
|
|
"path",
|
|
"purposeKey",
|
|
"purposeLabel",
|
|
"purposeCategory",
|
|
"purposeDetail",
|
|
"responseBytes"::bigint,
|
|
NULLIF("requestBytes", '')::bigint,
|
|
"requestBytesKnownCount"::integer,
|
|
"totalRequests"::integer,
|
|
"status2xx"::integer,
|
|
"status3xx"::integer,
|
|
"status4xx"::integer,
|
|
"status5xx"::integer,
|
|
"statusOther"::integer,
|
|
"sourceServer",
|
|
"generatedAt"::timestamptz,
|
|
"frozen"::boolean,
|
|
CURRENT_TIMESTAMP
|
|
FROM jsonb_to_recordset($1::jsonb) AS x(
|
|
"bucketStart" text,
|
|
"bucketEnd" text,
|
|
"businessDate" text,
|
|
"granularity" text,
|
|
"domain" text,
|
|
"method" text,
|
|
"path" text,
|
|
"purposeKey" text,
|
|
"purposeLabel" text,
|
|
"purposeCategory" text,
|
|
"purposeDetail" text,
|
|
"responseBytes" text,
|
|
"requestBytes" text,
|
|
"requestBytesKnownCount" text,
|
|
"totalRequests" text,
|
|
"status2xx" text,
|
|
"status3xx" text,
|
|
"status4xx" text,
|
|
"status5xx" text,
|
|
"statusOther" text,
|
|
"sourceServer" text,
|
|
"generatedAt" text,
|
|
"frozen" text
|
|
)
|
|
ON CONFLICT ("business_date", "granularity", "bucket_start", "bucket_end", "domain", "method", "path")
|
|
DO UPDATE SET
|
|
"purpose_key" = EXCLUDED."purpose_key",
|
|
"purpose_label" = EXCLUDED."purpose_label",
|
|
"purpose_category" = EXCLUDED."purpose_category",
|
|
"purpose_detail" = EXCLUDED."purpose_detail",
|
|
"response_bytes" = EXCLUDED."response_bytes",
|
|
"request_bytes" = EXCLUDED."request_bytes",
|
|
"request_bytes_known_count" = EXCLUDED."request_bytes_known_count",
|
|
"total_requests" = EXCLUDED."total_requests",
|
|
"status_2xx" = EXCLUDED."status_2xx",
|
|
"status_3xx" = EXCLUDED."status_3xx",
|
|
"status_4xx" = EXCLUDED."status_4xx",
|
|
"status_5xx" = EXCLUDED."status_5xx",
|
|
"status_other" = EXCLUDED."status_other",
|
|
"source_server" = EXCLUDED."source_server",
|
|
"generated_at" = EXCLUDED."generated_at",
|
|
"frozen" = EXCLUDED."frozen",
|
|
"updated_at" = CURRENT_TIMESTAMP
|
|
`,
|
|
JSON.stringify(
|
|
chunk.map((row) => ({
|
|
...row,
|
|
requestBytes: row.requestBytes === null ? '' : String(row.requestBytes),
|
|
responseBytes: String(row.responseBytes),
|
|
requestBytesKnownCount: String(row.requestBytesKnownCount),
|
|
totalRequests: String(row.totalRequests),
|
|
status2xx: String(row.status2xx),
|
|
status3xx: String(row.status3xx),
|
|
status4xx: String(row.status4xx),
|
|
status5xx: String(row.status5xx),
|
|
statusOther: String(row.statusOther),
|
|
frozen: String(row.frozen),
|
|
})),
|
|
),
|
|
)
|
|
}
|
|
|
|
return { received: inputs.length, accepted: rows.length, skipped: inputs.length - rows.length }
|
|
}
|
|
|
|
function addRollupRow(
|
|
stats: PublicTrafficStats,
|
|
aggregation: PublicTrafficRollupAggregationState,
|
|
row: PublicTrafficRollupRow,
|
|
) {
|
|
const responseBytes = bigintToNumber(row.response_bytes)
|
|
const requestBytes = row.request_bytes === null ? null : bigintToNumber(row.request_bytes)
|
|
const requestBytesKnownCount = Number(row.request_bytes_known_count || 0)
|
|
const totalRequests = Number(row.total_requests || 0)
|
|
const bucketStart = row.bucket_start instanceof Date ? row.bucket_start : new Date(row.bucket_start)
|
|
const bucketEnd = row.bucket_end instanceof Date ? row.bucket_end : new Date(row.bucket_end)
|
|
const statusCounts: Record<TrafficStatusBucket, number> = {
|
|
'2xx': Number(row.status_2xx || 0),
|
|
'3xx': Number(row.status_3xx || 0),
|
|
'4xx': Number(row.status_4xx || 0),
|
|
'5xx': Number(row.status_5xx || 0),
|
|
other: Number(row.status_other || 0),
|
|
}
|
|
|
|
stats.summary.responseBytes += responseBytes
|
|
if (requestBytesKnownCount > 0) {
|
|
stats.summary.requestBytes = (stats.summary.requestBytes || 0) + (requestBytes || 0)
|
|
stats.summary.requestBytesKnownCount += requestBytesKnownCount
|
|
}
|
|
stats.summary.totalRequests += totalRequests
|
|
for (const bucket of Object.keys(statusCounts) as TrafficStatusBucket[]) {
|
|
stats.summary.statusCounts[bucket] += statusCounts[bucket]
|
|
}
|
|
|
|
const firstSeen = Number.isNaN(bucketStart.getTime()) ? null : bucketStart.toISOString()
|
|
const lastSeen = Number.isNaN(bucketEnd.getTime()) ? null : bucketEnd.toISOString()
|
|
if (firstSeen && (!stats.summary.firstSeen || firstSeen < stats.summary.firstSeen)) stats.summary.firstSeen = firstSeen
|
|
if (lastSeen && (!stats.summary.lastSeen || lastSeen > stats.summary.lastSeen)) stats.summary.lastSeen = lastSeen
|
|
|
|
let dayItem = aggregation.dayMap.get(row.business_date)
|
|
if (!dayItem) {
|
|
dayItem = {
|
|
date: row.business_date,
|
|
responseBytes: 0,
|
|
requestBytes: null,
|
|
requestBytesKnownCount: 0,
|
|
totalRequests: 0,
|
|
}
|
|
stats.byDay.push(dayItem)
|
|
aggregation.dayMap.set(row.business_date, dayItem)
|
|
}
|
|
dayItem.responseBytes += responseBytes
|
|
dayItem.totalRequests += totalRequests
|
|
if (requestBytesKnownCount > 0) {
|
|
dayItem.requestBytes = (dayItem.requestBytes || 0) + (requestBytes || 0)
|
|
dayItem.requestBytesKnownCount += requestBytesKnownCount
|
|
}
|
|
|
|
let domainItem = aggregation.domainMap.get(row.domain)
|
|
if (!domainItem) {
|
|
domainItem = {
|
|
domain: row.domain,
|
|
responseBytes: 0,
|
|
requestBytes: null,
|
|
requestBytesKnownCount: 0,
|
|
totalRequests: 0,
|
|
statusCounts: emptyStatusCounts(),
|
|
}
|
|
stats.byDomain.push(domainItem)
|
|
aggregation.domainMap.set(row.domain, domainItem)
|
|
}
|
|
domainItem.responseBytes += responseBytes
|
|
domainItem.totalRequests += totalRequests
|
|
if (requestBytesKnownCount > 0) {
|
|
domainItem.requestBytes = (domainItem.requestBytes || 0) + (requestBytes || 0)
|
|
domainItem.requestBytesKnownCount += requestBytesKnownCount
|
|
}
|
|
for (const bucket of Object.keys(statusCounts) as TrafficStatusBucket[]) {
|
|
domainItem.statusCounts[bucket] += statusCounts[bucket]
|
|
}
|
|
|
|
let purposeItem = aggregation.purposeMap.get(row.purpose_key)
|
|
if (!purposeItem) {
|
|
purposeItem = {
|
|
key: row.purpose_key,
|
|
label: row.purpose_label,
|
|
category: row.purpose_category,
|
|
detail: row.purpose_detail,
|
|
responseBytes: 0,
|
|
requestBytes: null,
|
|
requestBytesKnownCount: 0,
|
|
totalRequests: 0,
|
|
}
|
|
stats.byPurpose.push(purposeItem)
|
|
aggregation.purposeMap.set(row.purpose_key, purposeItem)
|
|
}
|
|
purposeItem.responseBytes += responseBytes
|
|
purposeItem.totalRequests += totalRequests
|
|
if (requestBytesKnownCount > 0) {
|
|
purposeItem.requestBytes = (purposeItem.requestBytes || 0) + (requestBytes || 0)
|
|
purposeItem.requestBytesKnownCount += requestBytesKnownCount
|
|
}
|
|
|
|
const topPathKey = `${row.method}\u001f${row.path}`
|
|
let topPathItem = aggregation.topPathMap.get(topPathKey)
|
|
if (!topPathItem) {
|
|
topPathItem = {
|
|
path: row.path,
|
|
method: row.method,
|
|
purposeLabel: row.purpose_label,
|
|
purposeCategory: row.purpose_category,
|
|
purposeDetail: row.purpose_detail,
|
|
responseBytes: 0,
|
|
totalRequests: 0,
|
|
}
|
|
stats.topPaths.push(topPathItem)
|
|
aggregation.topPathMap.set(topPathKey, topPathItem)
|
|
}
|
|
topPathItem.responseBytes += responseBytes
|
|
topPathItem.totalRequests += totalRequests
|
|
}
|
|
|
|
async function getPublicTrafficStatsFromRollups(range: PublicTrafficRange): Promise<PublicTrafficStats | null> {
|
|
if (!(await ensurePublicTrafficRollupTable())) return null
|
|
|
|
const startDate = toChinaDateString(range.start)
|
|
const endDate = toChinaDateString(range.end)
|
|
const todayDate = toChinaDateString(new Date())
|
|
const { prisma } = await import('./prisma')
|
|
const rows = await prisma.$queryRawUnsafe<PublicTrafficRollupRow[]>(
|
|
`
|
|
SELECT
|
|
"bucket_start",
|
|
"bucket_end",
|
|
"business_date",
|
|
"granularity",
|
|
"domain",
|
|
"method",
|
|
"path",
|
|
"purpose_key",
|
|
"purpose_label",
|
|
"purpose_category",
|
|
"purpose_detail",
|
|
"response_bytes",
|
|
"request_bytes",
|
|
"request_bytes_known_count",
|
|
"total_requests",
|
|
"status_2xx",
|
|
"status_3xx",
|
|
"status_4xx",
|
|
"status_5xx",
|
|
"status_other"
|
|
FROM "public_traffic_rollups"
|
|
WHERE "business_date" >= $1
|
|
AND "business_date" <= $2
|
|
AND (
|
|
("business_date" = $3 AND "granularity" = '5m')
|
|
OR ("business_date" <> $3 AND "granularity" = 'day')
|
|
)
|
|
ORDER BY "bucket_start" ASC
|
|
`,
|
|
startDate,
|
|
endDate,
|
|
todayDate,
|
|
)
|
|
|
|
if (!rows.length) return null
|
|
|
|
const stats: PublicTrafficStats = {
|
|
generatedAt: new Date().toISOString(),
|
|
available: true,
|
|
range: {
|
|
start: range.start.toISOString(),
|
|
end: range.end.toISOString(),
|
|
},
|
|
summary: {
|
|
responseBytes: 0,
|
|
requestBytes: null,
|
|
requestBytesKnownCount: 0,
|
|
totalRequests: 0,
|
|
uniqueIpCount: 0,
|
|
statusCounts: emptyStatusCounts(),
|
|
firstSeen: null,
|
|
lastSeen: null,
|
|
},
|
|
byDay: [],
|
|
byDomain: [],
|
|
byPurpose: [],
|
|
topPaths: [],
|
|
source: {
|
|
logPaths: [],
|
|
scannedFiles: 0,
|
|
scannedLines: 0,
|
|
matchedLines: 0,
|
|
skippedLines: 0,
|
|
truncated: false,
|
|
issues: [],
|
|
requestBytesAvailable: false,
|
|
cache: {
|
|
persisted: true,
|
|
hit: true,
|
|
stale: false,
|
|
refreshInProgress: false,
|
|
refreshedAt: null,
|
|
expiresAt: null,
|
|
},
|
|
},
|
|
}
|
|
|
|
const aggregation = createRollupAggregationState()
|
|
for (const row of rows) addRollupRow(stats, aggregation, row)
|
|
|
|
stats.source.matchedLines = stats.summary.totalRequests
|
|
stats.source.requestBytesAvailable = stats.summary.requestBytesKnownCount > 0
|
|
stats.byDay.sort((a, b) => a.date.localeCompare(b.date))
|
|
stats.byDomain.sort((a, b) => b.responseBytes - a.responseBytes)
|
|
stats.byPurpose.sort((a, b) => b.responseBytes - a.responseBytes)
|
|
stats.topPaths.sort((a, b) => b.responseBytes - a.responseBytes)
|
|
stats.topPaths = stats.topPaths.slice(0, 20)
|
|
|
|
return stats
|
|
}
|
|
|
|
function getCachedStats(cacheKey: string): PublicTrafficStats | null {
|
|
const cached = statsCache.get(cacheKey)
|
|
if (!cached) return null
|
|
if (cached.expiresAt <= Date.now()) {
|
|
statsCache.delete(cacheKey)
|
|
return null
|
|
}
|
|
statsCache.delete(cacheKey)
|
|
statsCache.set(cacheKey, cached)
|
|
return cached.stats
|
|
}
|
|
|
|
function setCachedStats(cacheKey: string, stats: PublicTrafficStats, ttlSeconds = 60) {
|
|
statsCache.set(cacheKey, {
|
|
key: cacheKey,
|
|
createdAt: Date.now(),
|
|
expiresAt: Date.now() + ttlSeconds * 1000,
|
|
stats,
|
|
})
|
|
|
|
while (statsCache.size > MAX_CACHE_ENTRIES) {
|
|
const oldestKey = statsCache.keys().next().value
|
|
if (!oldestKey) break
|
|
statsCache.delete(oldestKey)
|
|
}
|
|
}
|
|
|
|
async function scanPublicTrafficStats(range: PublicTrafficRange, context: PublicTrafficScanContext): Promise<PublicTrafficStats> {
|
|
const aggregation = createAggregationState()
|
|
|
|
const stats: PublicTrafficStats = {
|
|
generatedAt: new Date().toISOString(),
|
|
available: false,
|
|
range: {
|
|
start: range.start.toISOString(),
|
|
end: range.end.toISOString(),
|
|
},
|
|
summary: {
|
|
responseBytes: 0,
|
|
requestBytes: null,
|
|
requestBytesKnownCount: 0,
|
|
totalRequests: 0,
|
|
uniqueIpCount: 0,
|
|
statusCounts: emptyStatusCounts(),
|
|
firstSeen: null,
|
|
lastSeen: null,
|
|
},
|
|
byDay: [],
|
|
byDomain: [],
|
|
byPurpose: [],
|
|
topPaths: [],
|
|
source: {
|
|
logPaths: context.logPaths,
|
|
scannedFiles: 0,
|
|
scannedLines: 0,
|
|
matchedLines: 0,
|
|
skippedLines: 0,
|
|
truncated: false,
|
|
issues: [],
|
|
requestBytesAvailable: false,
|
|
},
|
|
}
|
|
|
|
for (const logPath of context.logPaths) {
|
|
try {
|
|
await scanLogFile(logPath, range, stats, aggregation, context.maxLines)
|
|
} catch (error) {
|
|
stats.source.issues.push({
|
|
path: logPath,
|
|
reason: error instanceof Error ? error.message : '读取失败',
|
|
})
|
|
}
|
|
|
|
if (stats.source.truncated) break
|
|
}
|
|
|
|
stats.available = stats.source.scannedFiles > 0
|
|
stats.summary.uniqueIpCount = aggregation.uniqueIps.size
|
|
stats.source.requestBytesAvailable = stats.summary.requestBytesKnownCount > 0
|
|
stats.byDay.sort((a, b) => a.date.localeCompare(b.date))
|
|
stats.byDomain.sort((a, b) => b.responseBytes - a.responseBytes)
|
|
stats.byPurpose.sort((a, b) => b.responseBytes - a.responseBytes)
|
|
stats.topPaths.sort((a, b) => b.responseBytes - a.responseBytes)
|
|
stats.topPaths = stats.topPaths.slice(0, 20)
|
|
|
|
return stats
|
|
}
|
|
|
|
async function refreshPublicTrafficStatsInBackground(
|
|
cacheKey: string,
|
|
range: PublicTrafficRange,
|
|
context: PublicTrafficScanContext,
|
|
) {
|
|
if (refreshPromises.has(cacheKey)) return
|
|
|
|
const promise = (async () => {
|
|
try {
|
|
const stats = await scanPublicTrafficStats(range, context)
|
|
await writePersistentTrafficCache(cacheKey, range, context, stats)
|
|
setCachedStats(cacheKey, stats, context.ttlSeconds)
|
|
} catch (error) {
|
|
console.error('[public-traffic] 后台刷新公网流量统计失败:', error)
|
|
} finally {
|
|
refreshPromises.delete(cacheKey)
|
|
}
|
|
})()
|
|
|
|
refreshPromises.set(cacheKey, promise)
|
|
}
|
|
|
|
function parsePersistentStats(row: PersistentTrafficCacheRow): PublicTrafficStats | null {
|
|
const rawStats = typeof row.stats === 'string' ? JSON.parse(row.stats) : row.stats
|
|
if (!rawStats || typeof rawStats !== 'object') return null
|
|
return rawStats as PublicTrafficStats
|
|
}
|
|
|
|
export async function getPublicTrafficStats(
|
|
range: PublicTrafficRange,
|
|
options: PublicTrafficStatsOptions = {},
|
|
): Promise<PublicTrafficStats> {
|
|
try {
|
|
const rollupStats = await getPublicTrafficStatsFromRollups(range)
|
|
if (rollupStats) return rollupStats
|
|
} catch (error) {
|
|
console.error('[public-traffic] 读取公网流量汇总表失败:', error)
|
|
}
|
|
|
|
const context = getScanContext(range)
|
|
const cacheKey = context.stableCacheKey
|
|
|
|
if (!options.forceRefresh) {
|
|
const cachedStats = getCachedStats(cacheKey)
|
|
if (cachedStats) {
|
|
return withCacheMetadata(cachedStats, {
|
|
persisted: false,
|
|
hit: true,
|
|
stale: false,
|
|
refreshInProgress: refreshPromises.has(cacheKey),
|
|
refreshedAt: cachedStats.generatedAt,
|
|
expiresAt: null,
|
|
})
|
|
}
|
|
|
|
try {
|
|
const persistentRow = await readPersistentTrafficCache(cacheKey)
|
|
if (persistentRow) {
|
|
const persistentStats = parsePersistentStats(persistentRow)
|
|
if (persistentStats) {
|
|
const refreshedAt = new Date(persistentRow.refreshed_at)
|
|
const expiresAt = new Date(persistentRow.expires_at)
|
|
const stale = expiresAt.getTime() <= Date.now()
|
|
|
|
if (stale) {
|
|
await refreshPublicTrafficStatsInBackground(cacheKey, range, context)
|
|
} else {
|
|
const secondsToExpire = Math.floor((expiresAt.getTime() - Date.now()) / 1000)
|
|
setCachedStats(cacheKey, persistentStats, Math.max(10, secondsToExpire))
|
|
}
|
|
|
|
return withCacheMetadata(persistentStats, {
|
|
persisted: true,
|
|
hit: true,
|
|
stale,
|
|
refreshInProgress: refreshPromises.has(cacheKey),
|
|
refreshedAt: Number.isNaN(refreshedAt.getTime()) ? null : refreshedAt.toISOString(),
|
|
expiresAt: Number.isNaN(expiresAt.getTime()) ? null : expiresAt.toISOString(),
|
|
})
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error('[public-traffic] 读取公网流量缓存失败，回退实时统计:', error)
|
|
}
|
|
}
|
|
|
|
const stats = await scanPublicTrafficStats(range, context)
|
|
|
|
try {
|
|
await writePersistentTrafficCache(cacheKey, range, context, stats)
|
|
} catch (error) {
|
|
console.error('[public-traffic] 写入公网流量缓存失败:', error)
|
|
}
|
|
|
|
setCachedStats(cacheKey, stats, context.ttlSeconds)
|
|
|
|
return withCacheMetadata(stats, {
|
|
persisted: persistentTrafficCacheEnabled(),
|
|
hit: false,
|
|
stale: false,
|
|
refreshInProgress: false,
|
|
refreshedAt: stats.generatedAt,
|
|
expiresAt: new Date(Date.now() + context.ttlSeconds * 1000).toISOString(),
|
|
})
|
|
}
|