From da8a8399e4ae09273bd25b1b45759f81abbdeeb9 Mon Sep 17 00:00:00 2001 From: chenpei Date: Sat, 4 Jul 2026 15:17:56 +0800 Subject: [PATCH] fix: cache public traffic stats --- app/api/admin/public-traffic/route.ts | 3 +- components/admin/PublicTrafficTab.tsx | 23 +- lib/public-traffic-stats.ts | 294 ++++++++++++++++++++++++-- 3 files changed, 297 insertions(+), 23 deletions(-) diff --git a/app/api/admin/public-traffic/route.ts b/app/api/admin/public-traffic/route.ts index d8772434..269a9b07 100644 --- a/app/api/admin/public-traffic/route.ts +++ b/app/api/admin/public-traffic/route.ts @@ -18,7 +18,8 @@ export async function GET(request: NextRequest) { const { searchParams } = new URL(request.url) const range = normalizeTrafficRange(searchParams.get('start'), searchParams.get('end')) - const stats = await getPublicTrafficStats(range) + const forceRefresh = searchParams.get('refresh') === '1' || searchParams.get('forceRefresh') === '1' + const stats = await getPublicTrafficStats(range, { forceRefresh }) return NextResponse.json(stats) } catch (error) { diff --git a/components/admin/PublicTrafficTab.tsx b/components/admin/PublicTrafficTab.tsx index 3bcc3ec8..23992fd6 100644 --- a/components/admin/PublicTrafficTab.tsx +++ b/components/admin/PublicTrafficTab.tsx @@ -61,6 +61,14 @@ interface PublicTrafficStats { truncated: boolean issues: Array<{ path: string; reason: string }> requestBytesAvailable: boolean + cache?: { + persisted: boolean + hit: boolean + stale: boolean + refreshInProgress: boolean + refreshedAt: string | null + expiresAt: string | null + } } } @@ -233,12 +241,13 @@ export default function PublicTrafficTab({ showToast }: PublicTrafficTabProps) { const [stats, setStats] = useState(null) const [expandedPaths, setExpandedPaths] = useState>(new Set()) - const loadStats = useCallback(async () => { + const loadStats = useCallback(async (options?: { forceRefresh?: boolean }) => { setLoading(true) try { const params = new URLSearchParams() params.set('start', startDate) params.set('end', endDate) + if (options?.forceRefresh) params.set('refresh', '1') const response = await fetch(`/api/admin/public-traffic?${params.toString()}`, { cache: 'no-store' }) if (!response.ok) { const data = await response.json().catch(() => null) @@ -364,11 +373,11 @@ export default function PublicTrafficTab({ showToast }: PublicTrafficTabProps) { @@ -651,6 +660,14 @@ export default function PublicTrafficTab({ showToast }: PublicTrafficTabProps) { 日志未含 request_length 字段,上传入站流量无法精确拆分。 )} + {stats.source.cache && ( +
+ 数据缓存: + {stats.source.cache.hit ? '已读取固化结果' : '本次重新统计'} + {stats.source.cache.refreshedAt && `,更新时间 ${formatTime(stats.source.cache.refreshedAt)}`} + {stats.source.cache.stale && ',正在后台刷新'} +
+ )} )} diff --git a/lib/public-traffic-stats.ts b/lib/public-traffic-stats.ts index 196886bc..5e3e01b4 100644 --- a/lib/public-traffic-stats.ts +++ b/lib/public-traffic-stats.ts @@ -93,6 +93,14 @@ export interface PublicTrafficStats { truncated: boolean issues: PublicTrafficSourceIssue[] requestBytesAvailable: boolean + cache?: { + persisted: boolean + hit: boolean + stale: boolean + refreshInProgress: boolean + refreshedAt: string | null + expiresAt: string | null + } } } @@ -107,9 +115,28 @@ interface PublicTrafficAggregationState { interface PublicTrafficCacheEntry { key: string createdAt: number + expiresAt: number stats: PublicTrafficStats } +interface PublicTrafficStatsOptions { + forceRefresh?: boolean +} + +interface PublicTrafficScanContext { + logPaths: string[] + maxLines: number + stableCacheKey: string + sourceSignature: string + ttlSeconds: number +} + +interface PersistentTrafficCacheRow { + stats: PublicTrafficStats | string + refreshed_at: Date | string + expires_at: Date | string +} + const MONTHS: Record = { Jan: 0, Feb: 1, @@ -136,6 +163,8 @@ const DEFAULT_LOG_PATHS = [ const statsCache = new Map() const MAX_CACHE_ENTRIES = 20 +const refreshPromises = new Map>() +let persistentCacheTableEnsured = false function emptyStatusCounts(): Record { return { @@ -760,8 +789,18 @@ async function scanLogFile( } } -function buildCacheKey(range: PublicTrafficRange, logPaths: string[], maxLines: number): string { - const sourceSignature = logPaths +function buildStableCacheKey(range: PublicTrafficRange, logPaths: string[], maxLines: number): string { + return [ + 'public-traffic-v2', + range.start.toISOString(), + range.end.toISOString(), + maxLines, + logPaths.join('|'), + ].join('::') +} + +function buildSourceSignature(logPaths: string[]): string { + return logPaths .map((logPath) => { try { const fileStat = statSync(logPath) @@ -771,27 +810,153 @@ function buildCacheKey(range: PublicTrafficRange, logPaths: string[], maxLines: } }) .join('|') +} - return [ +function getCacheTtlSeconds() { + const parsed = Number.parseInt(process.env.PUBLIC_TRAFFIC_STATS_CACHE_TTL_SECONDS || '900', 10) + return Number.isFinite(parsed) && parsed > 0 ? parsed : 900 +} + +function getScanContext(range: PublicTrafficRange): PublicTrafficScanContext { + const logPaths = getConfiguredPublicTrafficLogPaths() + const maxLines = Math.max(1000, Number.parseInt(process.env.PUBLIC_TRAFFIC_MAX_LINES || '1000000', 10) || 1000000) + return { + logPaths, + maxLines, + stableCacheKey: buildStableCacheKey(range, logPaths, maxLines), + sourceSignature: buildSourceSignature(logPaths), + ttlSeconds: getCacheTtlSeconds(), + } +} + +function cloneStats(stats: PublicTrafficStats): PublicTrafficStats { + return JSON.parse(JSON.stringify(stats)) as PublicTrafficStats +} + +function withCacheMetadata( + stats: PublicTrafficStats, + cache: NonNullable, +): PublicTrafficStats { + const nextStats = cloneStats(stats) + nextStats.source.cache = cache + return nextStats +} + +function persistentTrafficCacheEnabled() { + return process.env.NODE_ENV !== 'test' && process.env.PUBLIC_TRAFFIC_STATS_DB_CACHE !== '0' && Boolean(process.env.DATABASE_URL) +} + +async function ensurePersistentTrafficCacheTable() { + if (persistentCacheTableEnsured || !persistentTrafficCacheEnabled()) return false + + const { prisma } = await import('./prisma') + + await prisma.$executeRawUnsafe(` + CREATE TABLE IF NOT EXISTS "public_traffic_stats_cache" ( + "cache_key" TEXT PRIMARY KEY, + "range_start" TIMESTAMPTZ NOT NULL, + "range_end" TIMESTAMPTZ NOT NULL, + "stats" JSONB NOT NULL, + "source_signature" TEXT NOT NULL DEFAULT '', + "refreshed_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + "expires_at" TIMESTAMPTZ NOT NULL, + "created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + `) + await prisma.$executeRawUnsafe(` + CREATE INDEX IF NOT EXISTS "public_traffic_stats_cache_range_idx" + ON "public_traffic_stats_cache" ("range_start", "range_end") + `) + await prisma.$executeRawUnsafe(` + CREATE INDEX IF NOT EXISTS "public_traffic_stats_cache_expires_at_idx" + ON "public_traffic_stats_cache" ("expires_at") + `) + + persistentCacheTableEnsured = true + return true +} + +async function readPersistentTrafficCache(cacheKey: string): Promise { + if (!(await ensurePersistentTrafficCacheTable())) return null + + const { prisma } = await import('./prisma') + const rows = await prisma.$queryRawUnsafe( + ` + SELECT "stats", "refreshed_at", "expires_at" + FROM "public_traffic_stats_cache" + WHERE "cache_key" = $1 + LIMIT 1 + `, + cacheKey, + ) + + return rows[0] || null +} + +async function writePersistentTrafficCache( + cacheKey: string, + range: PublicTrafficRange, + context: PublicTrafficScanContext, + stats: PublicTrafficStats, +) { + if (!(await ensurePersistentTrafficCacheTable())) return + + const { prisma } = await import('./prisma') + const now = new Date() + const expiresAt = new Date(now.getTime() + context.ttlSeconds * 1000) + const statsForStorage = cloneStats(stats) + delete statsForStorage.source.cache + + await prisma.$executeRawUnsafe( + ` + INSERT INTO "public_traffic_stats_cache" ( + "cache_key", + "range_start", + "range_end", + "stats", + "source_signature", + "refreshed_at", + "expires_at", + "updated_at" + ) + VALUES ($1, $2::timestamptz, $3::timestamptz, $4::jsonb, $5, $6::timestamptz, $7::timestamptz, $6::timestamptz) + ON CONFLICT ("cache_key") DO UPDATE SET + "range_start" = EXCLUDED."range_start", + "range_end" = EXCLUDED."range_end", + "stats" = EXCLUDED."stats", + "source_signature" = EXCLUDED."source_signature", + "refreshed_at" = EXCLUDED."refreshed_at", + "expires_at" = EXCLUDED."expires_at", + "updated_at" = EXCLUDED."updated_at" + `, + cacheKey, range.start.toISOString(), range.end.toISOString(), - maxLines, - sourceSignature, - ].join('::') + JSON.stringify(statsForStorage), + context.sourceSignature, + now.toISOString(), + expiresAt.toISOString(), + ) } function getCachedStats(cacheKey: string): PublicTrafficStats | null { const cached = statsCache.get(cacheKey) if (!cached) return null + if (cached.expiresAt <= Date.now()) { + statsCache.delete(cacheKey) + return null + } statsCache.delete(cacheKey) statsCache.set(cacheKey, cached) return cached.stats } -function setCachedStats(cacheKey: string, stats: PublicTrafficStats) { +function setCachedStats(cacheKey: string, stats: PublicTrafficStats, ttlSeconds = 60) { statsCache.set(cacheKey, { key: cacheKey, createdAt: Date.now(), + expiresAt: Date.now() + ttlSeconds * 1000, stats, }) @@ -802,13 +967,7 @@ function setCachedStats(cacheKey: string, stats: PublicTrafficStats) { } } -export async function getPublicTrafficStats(range: PublicTrafficRange): Promise { - const logPaths = getConfiguredPublicTrafficLogPaths() - const maxLines = Math.max(1000, Number.parseInt(process.env.PUBLIC_TRAFFIC_MAX_LINES || '1000000', 10) || 1000000) - const cacheKey = buildCacheKey(range, logPaths, maxLines) - const cachedStats = getCachedStats(cacheKey) - if (cachedStats) return cachedStats - +async function scanPublicTrafficStats(range: PublicTrafficRange, context: PublicTrafficScanContext): Promise { const aggregation = createAggregationState() const stats: PublicTrafficStats = { @@ -833,7 +992,7 @@ export async function getPublicTrafficStats(range: PublicTrafficRange): Promise< byPurpose: [], topPaths: [], source: { - logPaths, + logPaths: context.logPaths, scannedFiles: 0, scannedLines: 0, matchedLines: 0, @@ -844,9 +1003,9 @@ export async function getPublicTrafficStats(range: PublicTrafficRange): Promise< }, } - for (const logPath of logPaths) { + for (const logPath of context.logPaths) { try { - await scanLogFile(logPath, range, stats, aggregation, maxLines) + await scanLogFile(logPath, range, stats, aggregation, context.maxLines) } catch (error) { stats.source.issues.push({ path: logPath, @@ -866,7 +1025,104 @@ export async function getPublicTrafficStats(range: PublicTrafficRange): Promise< stats.topPaths.sort((a, b) => b.responseBytes - a.responseBytes) stats.topPaths = stats.topPaths.slice(0, 20) - setCachedStats(cacheKey, stats) - return stats } + +async function refreshPublicTrafficStatsInBackground( + cacheKey: string, + range: PublicTrafficRange, + context: PublicTrafficScanContext, +) { + if (refreshPromises.has(cacheKey)) return + + const promise = (async () => { + try { + const stats = await scanPublicTrafficStats(range, context) + await writePersistentTrafficCache(cacheKey, range, context, stats) + setCachedStats(cacheKey, stats, context.ttlSeconds) + } catch (error) { + console.error('[public-traffic] 后台刷新公网流量统计失败:', error) + } finally { + refreshPromises.delete(cacheKey) + } + })() + + refreshPromises.set(cacheKey, promise) +} + +function parsePersistentStats(row: PersistentTrafficCacheRow): PublicTrafficStats | null { + const rawStats = typeof row.stats === 'string' ? JSON.parse(row.stats) : row.stats + if (!rawStats || typeof rawStats !== 'object') return null + return rawStats as PublicTrafficStats +} + +export async function getPublicTrafficStats( + range: PublicTrafficRange, + options: PublicTrafficStatsOptions = {}, +): Promise { + const context = getScanContext(range) + const cacheKey = context.stableCacheKey + + if (!options.forceRefresh) { + const cachedStats = getCachedStats(cacheKey) + if (cachedStats) { + return withCacheMetadata(cachedStats, { + persisted: false, + hit: true, + stale: false, + refreshInProgress: refreshPromises.has(cacheKey), + refreshedAt: cachedStats.generatedAt, + expiresAt: null, + }) + } + + try { + const persistentRow = await readPersistentTrafficCache(cacheKey) + if (persistentRow) { + const persistentStats = parsePersistentStats(persistentRow) + if (persistentStats) { + const refreshedAt = new Date(persistentRow.refreshed_at) + const expiresAt = new Date(persistentRow.expires_at) + const stale = expiresAt.getTime() <= Date.now() + + if (stale) { + await refreshPublicTrafficStatsInBackground(cacheKey, range, context) + } else { + const secondsToExpire = Math.floor((expiresAt.getTime() - Date.now()) / 1000) + setCachedStats(cacheKey, persistentStats, Math.max(10, secondsToExpire)) + } + + return withCacheMetadata(persistentStats, { + persisted: true, + hit: true, + stale, + refreshInProgress: refreshPromises.has(cacheKey), + refreshedAt: Number.isNaN(refreshedAt.getTime()) ? null : refreshedAt.toISOString(), + expiresAt: Number.isNaN(expiresAt.getTime()) ? null : expiresAt.toISOString(), + }) + } + } + } catch (error) { + console.error('[public-traffic] 读取公网流量缓存失败,回退实时统计:', error) + } + } + + const stats = await scanPublicTrafficStats(range, context) + + try { + await writePersistentTrafficCache(cacheKey, range, context, stats) + } catch (error) { + console.error('[public-traffic] 写入公网流量缓存失败:', error) + } + + setCachedStats(cacheKey, stats, context.ttlSeconds) + + return withCacheMetadata(stats, { + persisted: persistentTrafficCacheEnabled(), + hit: false, + stale: false, + refreshInProgress: false, + refreshedAt: stats.generatedAt, + expiresAt: new Date(Date.now() + context.ttlSeconds * 1000).toISOString(), + }) +}