fix: cache public traffic stats

This commit is contained in:
chenpei 2026-07-04 15:17:56 +08:00
parent b3b973ed63
commit da8a8399e4
3 changed files with 297 additions and 23 deletions

View File

@ -18,7 +18,8 @@ export async function GET(request: NextRequest) {
const { searchParams } = new URL(request.url)
const range = normalizeTrafficRange(searchParams.get('start'), searchParams.get('end'))
const stats = await getPublicTrafficStats(range)
const forceRefresh = searchParams.get('refresh') === '1' || searchParams.get('forceRefresh') === '1'
const stats = await getPublicTrafficStats(range, { forceRefresh })
return NextResponse.json(stats)
} catch (error) {

View File

@ -61,6 +61,14 @@ interface PublicTrafficStats {
truncated: boolean
issues: Array<{ path: string; reason: string }>
requestBytesAvailable: boolean
cache?: {
persisted: boolean
hit: boolean
stale: boolean
refreshInProgress: boolean
refreshedAt: string | null
expiresAt: string | null
}
}
}
@ -233,12 +241,13 @@ export default function PublicTrafficTab({ showToast }: PublicTrafficTabProps) {
const [stats, setStats] = useState<PublicTrafficStats | null>(null)
const [expandedPaths, setExpandedPaths] = useState<Set<string>>(new Set())
const loadStats = useCallback(async () => {
const loadStats = useCallback(async (options?: { forceRefresh?: boolean }) => {
setLoading(true)
try {
const params = new URLSearchParams()
params.set('start', startDate)
params.set('end', endDate)
if (options?.forceRefresh) params.set('refresh', '1')
const response = await fetch(`/api/admin/public-traffic?${params.toString()}`, { cache: 'no-store' })
if (!response.ok) {
const data = await response.json().catch(() => null)
@ -364,11 +373,11 @@ export default function PublicTrafficTab({ showToast }: PublicTrafficTabProps) {
</label>
<button
type="button"
onClick={loadStats}
onClick={() => loadStats({ forceRefresh: true })}
disabled={loading}
className="btn-primary text-sm"
>
{loading ? '刷新中...' : '刷新'}
{loading ? '刷新中...' : '重新统计'}
</button>
</div>
</div>
@ -651,6 +660,14 @@ export default function PublicTrafficTab({ showToast }: PublicTrafficTabProps) {
日志未含 request_length 字段,上传入站流量无法精确拆分。
</div>
)}
{stats.source.cache && (
<div className="mt-1 text-gray-400 dark:text-gray-500">
数据缓存:
{stats.source.cache.hit ? '已读取固化结果' : '本次重新统计'}
{stats.source.cache.refreshedAt && `,更新时间 ${formatTime(stats.source.cache.refreshedAt)}`}
{stats.source.cache.stale && ',正在后台刷新'}
</div>
)}
</div>
</>
)}

View File

@ -93,6 +93,14 @@ export interface PublicTrafficStats {
truncated: boolean
issues: PublicTrafficSourceIssue[]
requestBytesAvailable: boolean
cache?: {
persisted: boolean
hit: boolean
stale: boolean
refreshInProgress: boolean
refreshedAt: string | null
expiresAt: string | null
}
}
}
@ -107,9 +115,28 @@ interface PublicTrafficAggregationState {
interface PublicTrafficCacheEntry {
key: string
createdAt: number
expiresAt: number
stats: PublicTrafficStats
}
interface PublicTrafficStatsOptions {
forceRefresh?: boolean
}
interface PublicTrafficScanContext {
logPaths: string[]
maxLines: number
stableCacheKey: string
sourceSignature: string
ttlSeconds: number
}
interface PersistentTrafficCacheRow {
stats: PublicTrafficStats | string
refreshed_at: Date | string
expires_at: Date | string
}
const MONTHS: Record<string, number> = {
Jan: 0,
Feb: 1,
@ -136,6 +163,8 @@ const DEFAULT_LOG_PATHS = [
const statsCache = new Map<string, PublicTrafficCacheEntry>()
const MAX_CACHE_ENTRIES = 20
const refreshPromises = new Map<string, Promise<void>>()
let persistentCacheTableEnsured = false
function emptyStatusCounts(): Record<TrafficStatusBucket, number> {
return {
@ -760,8 +789,18 @@ async function scanLogFile(
}
}
function buildCacheKey(range: PublicTrafficRange, logPaths: string[], maxLines: number): string {
const sourceSignature = logPaths
function buildStableCacheKey(range: PublicTrafficRange, logPaths: string[], maxLines: number): string {
return [
'public-traffic-v2',
range.start.toISOString(),
range.end.toISOString(),
maxLines,
logPaths.join('|'),
].join('::')
}
function buildSourceSignature(logPaths: string[]): string {
return logPaths
.map((logPath) => {
try {
const fileStat = statSync(logPath)
@ -771,27 +810,153 @@ function buildCacheKey(range: PublicTrafficRange, logPaths: string[], maxLines:
}
})
.join('|')
}
return [
function getCacheTtlSeconds() {
const parsed = Number.parseInt(process.env.PUBLIC_TRAFFIC_STATS_CACHE_TTL_SECONDS || '900', 10)
return Number.isFinite(parsed) && parsed > 0 ? parsed : 900
}
function getScanContext(range: PublicTrafficRange): PublicTrafficScanContext {
const logPaths = getConfiguredPublicTrafficLogPaths()
const maxLines = Math.max(1000, Number.parseInt(process.env.PUBLIC_TRAFFIC_MAX_LINES || '1000000', 10) || 1000000)
return {
logPaths,
maxLines,
stableCacheKey: buildStableCacheKey(range, logPaths, maxLines),
sourceSignature: buildSourceSignature(logPaths),
ttlSeconds: getCacheTtlSeconds(),
}
}
function cloneStats(stats: PublicTrafficStats): PublicTrafficStats {
return JSON.parse(JSON.stringify(stats)) as PublicTrafficStats
}
function withCacheMetadata(
stats: PublicTrafficStats,
cache: NonNullable<PublicTrafficStats['source']['cache']>,
): PublicTrafficStats {
const nextStats = cloneStats(stats)
nextStats.source.cache = cache
return nextStats
}
function persistentTrafficCacheEnabled() {
return process.env.NODE_ENV !== 'test' && process.env.PUBLIC_TRAFFIC_STATS_DB_CACHE !== '0' && Boolean(process.env.DATABASE_URL)
}
async function ensurePersistentTrafficCacheTable() {
if (persistentCacheTableEnsured || !persistentTrafficCacheEnabled()) return false
const { prisma } = await import('./prisma')
await prisma.$executeRawUnsafe(`
CREATE TABLE IF NOT EXISTS "public_traffic_stats_cache" (
"cache_key" TEXT PRIMARY KEY,
"range_start" TIMESTAMPTZ NOT NULL,
"range_end" TIMESTAMPTZ NOT NULL,
"stats" JSONB NOT NULL,
"source_signature" TEXT NOT NULL DEFAULT '',
"refreshed_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
"expires_at" TIMESTAMPTZ NOT NULL,
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
)
`)
await prisma.$executeRawUnsafe(`
CREATE INDEX IF NOT EXISTS "public_traffic_stats_cache_range_idx"
ON "public_traffic_stats_cache" ("range_start", "range_end")
`)
await prisma.$executeRawUnsafe(`
CREATE INDEX IF NOT EXISTS "public_traffic_stats_cache_expires_at_idx"
ON "public_traffic_stats_cache" ("expires_at")
`)
persistentCacheTableEnsured = true
return true
}
async function readPersistentTrafficCache(cacheKey: string): Promise<PersistentTrafficCacheRow | null> {
if (!(await ensurePersistentTrafficCacheTable())) return null
const { prisma } = await import('./prisma')
const rows = await prisma.$queryRawUnsafe<PersistentTrafficCacheRow[]>(
`
SELECT "stats", "refreshed_at", "expires_at"
FROM "public_traffic_stats_cache"
WHERE "cache_key" = $1
LIMIT 1
`,
cacheKey,
)
return rows[0] || null
}
async function writePersistentTrafficCache(
cacheKey: string,
range: PublicTrafficRange,
context: PublicTrafficScanContext,
stats: PublicTrafficStats,
) {
if (!(await ensurePersistentTrafficCacheTable())) return
const { prisma } = await import('./prisma')
const now = new Date()
const expiresAt = new Date(now.getTime() + context.ttlSeconds * 1000)
const statsForStorage = cloneStats(stats)
delete statsForStorage.source.cache
await prisma.$executeRawUnsafe(
`
INSERT INTO "public_traffic_stats_cache" (
"cache_key",
"range_start",
"range_end",
"stats",
"source_signature",
"refreshed_at",
"expires_at",
"updated_at"
)
VALUES ($1, $2::timestamptz, $3::timestamptz, $4::jsonb, $5, $6::timestamptz, $7::timestamptz, $6::timestamptz)
ON CONFLICT ("cache_key") DO UPDATE SET
"range_start" = EXCLUDED."range_start",
"range_end" = EXCLUDED."range_end",
"stats" = EXCLUDED."stats",
"source_signature" = EXCLUDED."source_signature",
"refreshed_at" = EXCLUDED."refreshed_at",
"expires_at" = EXCLUDED."expires_at",
"updated_at" = EXCLUDED."updated_at"
`,
cacheKey,
range.start.toISOString(),
range.end.toISOString(),
maxLines,
sourceSignature,
].join('::')
JSON.stringify(statsForStorage),
context.sourceSignature,
now.toISOString(),
expiresAt.toISOString(),
)
}
function getCachedStats(cacheKey: string): PublicTrafficStats | null {
const cached = statsCache.get(cacheKey)
if (!cached) return null
if (cached.expiresAt <= Date.now()) {
statsCache.delete(cacheKey)
return null
}
statsCache.delete(cacheKey)
statsCache.set(cacheKey, cached)
return cached.stats
}
function setCachedStats(cacheKey: string, stats: PublicTrafficStats) {
function setCachedStats(cacheKey: string, stats: PublicTrafficStats, ttlSeconds = 60) {
statsCache.set(cacheKey, {
key: cacheKey,
createdAt: Date.now(),
expiresAt: Date.now() + ttlSeconds * 1000,
stats,
})
@ -802,13 +967,7 @@ function setCachedStats(cacheKey: string, stats: PublicTrafficStats) {
}
}
export async function getPublicTrafficStats(range: PublicTrafficRange): Promise<PublicTrafficStats> {
const logPaths = getConfiguredPublicTrafficLogPaths()
const maxLines = Math.max(1000, Number.parseInt(process.env.PUBLIC_TRAFFIC_MAX_LINES || '1000000', 10) || 1000000)
const cacheKey = buildCacheKey(range, logPaths, maxLines)
const cachedStats = getCachedStats(cacheKey)
if (cachedStats) return cachedStats
async function scanPublicTrafficStats(range: PublicTrafficRange, context: PublicTrafficScanContext): Promise<PublicTrafficStats> {
const aggregation = createAggregationState()
const stats: PublicTrafficStats = {
@ -833,7 +992,7 @@ export async function getPublicTrafficStats(range: PublicTrafficRange): Promise<
byPurpose: [],
topPaths: [],
source: {
logPaths,
logPaths: context.logPaths,
scannedFiles: 0,
scannedLines: 0,
matchedLines: 0,
@ -844,9 +1003,9 @@ export async function getPublicTrafficStats(range: PublicTrafficRange): Promise<
},
}
for (const logPath of logPaths) {
for (const logPath of context.logPaths) {
try {
await scanLogFile(logPath, range, stats, aggregation, maxLines)
await scanLogFile(logPath, range, stats, aggregation, context.maxLines)
} catch (error) {
stats.source.issues.push({
path: logPath,
@ -866,7 +1025,104 @@ export async function getPublicTrafficStats(range: PublicTrafficRange): Promise<
stats.topPaths.sort((a, b) => b.responseBytes - a.responseBytes)
stats.topPaths = stats.topPaths.slice(0, 20)
setCachedStats(cacheKey, stats)
return stats
}
async function refreshPublicTrafficStatsInBackground(
cacheKey: string,
range: PublicTrafficRange,
context: PublicTrafficScanContext,
) {
if (refreshPromises.has(cacheKey)) return
const promise = (async () => {
try {
const stats = await scanPublicTrafficStats(range, context)
await writePersistentTrafficCache(cacheKey, range, context, stats)
setCachedStats(cacheKey, stats, context.ttlSeconds)
} catch (error) {
console.error('[public-traffic] 后台刷新公网流量统计失败:', error)
} finally {
refreshPromises.delete(cacheKey)
}
})()
refreshPromises.set(cacheKey, promise)
}
function parsePersistentStats(row: PersistentTrafficCacheRow): PublicTrafficStats | null {
const rawStats = typeof row.stats === 'string' ? JSON.parse(row.stats) : row.stats
if (!rawStats || typeof rawStats !== 'object') return null
return rawStats as PublicTrafficStats
}
export async function getPublicTrafficStats(
range: PublicTrafficRange,
options: PublicTrafficStatsOptions = {},
): Promise<PublicTrafficStats> {
const context = getScanContext(range)
const cacheKey = context.stableCacheKey
if (!options.forceRefresh) {
const cachedStats = getCachedStats(cacheKey)
if (cachedStats) {
return withCacheMetadata(cachedStats, {
persisted: false,
hit: true,
stale: false,
refreshInProgress: refreshPromises.has(cacheKey),
refreshedAt: cachedStats.generatedAt,
expiresAt: null,
})
}
try {
const persistentRow = await readPersistentTrafficCache(cacheKey)
if (persistentRow) {
const persistentStats = parsePersistentStats(persistentRow)
if (persistentStats) {
const refreshedAt = new Date(persistentRow.refreshed_at)
const expiresAt = new Date(persistentRow.expires_at)
const stale = expiresAt.getTime() <= Date.now()
if (stale) {
await refreshPublicTrafficStatsInBackground(cacheKey, range, context)
} else {
const secondsToExpire = Math.floor((expiresAt.getTime() - Date.now()) / 1000)
setCachedStats(cacheKey, persistentStats, Math.max(10, secondsToExpire))
}
return withCacheMetadata(persistentStats, {
persisted: true,
hit: true,
stale,
refreshInProgress: refreshPromises.has(cacheKey),
refreshedAt: Number.isNaN(refreshedAt.getTime()) ? null : refreshedAt.toISOString(),
expiresAt: Number.isNaN(expiresAt.getTime()) ? null : expiresAt.toISOString(),
})
}
}
} catch (error) {
console.error('[public-traffic] 读取公网流量缓存失败,回退实时统计:', error)
}
}
const stats = await scanPublicTrafficStats(range, context)
try {
await writePersistentTrafficCache(cacheKey, range, context, stats)
} catch (error) {
console.error('[public-traffic] 写入公网流量缓存失败:', error)
}
setCachedStats(cacheKey, stats, context.ttlSeconds)
return withCacheMetadata(stats, {
persisted: persistentTrafficCacheEnabled(),
hit: false,
stale: false,
refreshInProgress: false,
refreshedAt: stats.generatedAt,
expiresAt: new Date(Date.now() + context.ttlSeconds * 1000).toISOString(),
})
}