fix: cache public traffic stats
This commit is contained in:
parent
b3b973ed63
commit
da8a8399e4
@ -18,7 +18,8 @@ export async function GET(request: NextRequest) {
|
||||
|
||||
const { searchParams } = new URL(request.url)
|
||||
const range = normalizeTrafficRange(searchParams.get('start'), searchParams.get('end'))
|
||||
const stats = await getPublicTrafficStats(range)
|
||||
const forceRefresh = searchParams.get('refresh') === '1' || searchParams.get('forceRefresh') === '1'
|
||||
const stats = await getPublicTrafficStats(range, { forceRefresh })
|
||||
|
||||
return NextResponse.json(stats)
|
||||
} catch (error) {
|
||||
|
||||
@ -61,6 +61,14 @@ interface PublicTrafficStats {
|
||||
truncated: boolean
|
||||
issues: Array<{ path: string; reason: string }>
|
||||
requestBytesAvailable: boolean
|
||||
cache?: {
|
||||
persisted: boolean
|
||||
hit: boolean
|
||||
stale: boolean
|
||||
refreshInProgress: boolean
|
||||
refreshedAt: string | null
|
||||
expiresAt: string | null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -233,12 +241,13 @@ export default function PublicTrafficTab({ showToast }: PublicTrafficTabProps) {
|
||||
const [stats, setStats] = useState<PublicTrafficStats | null>(null)
|
||||
const [expandedPaths, setExpandedPaths] = useState<Set<string>>(new Set())
|
||||
|
||||
const loadStats = useCallback(async () => {
|
||||
const loadStats = useCallback(async (options?: { forceRefresh?: boolean }) => {
|
||||
setLoading(true)
|
||||
try {
|
||||
const params = new URLSearchParams()
|
||||
params.set('start', startDate)
|
||||
params.set('end', endDate)
|
||||
if (options?.forceRefresh) params.set('refresh', '1')
|
||||
const response = await fetch(`/api/admin/public-traffic?${params.toString()}`, { cache: 'no-store' })
|
||||
if (!response.ok) {
|
||||
const data = await response.json().catch(() => null)
|
||||
@ -364,11 +373,11 @@ export default function PublicTrafficTab({ showToast }: PublicTrafficTabProps) {
|
||||
</label>
|
||||
<button
|
||||
type="button"
|
||||
onClick={loadStats}
|
||||
onClick={() => loadStats({ forceRefresh: true })}
|
||||
disabled={loading}
|
||||
className="btn-primary text-sm"
|
||||
>
|
||||
{loading ? '刷新中...' : '刷新'}
|
||||
{loading ? '刷新中...' : '重新统计'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
@ -651,6 +660,14 @@ export default function PublicTrafficTab({ showToast }: PublicTrafficTabProps) {
|
||||
日志未含 request_length 字段,上传入站流量无法精确拆分。
|
||||
</div>
|
||||
)}
|
||||
{stats.source.cache && (
|
||||
<div className="mt-1 text-gray-400 dark:text-gray-500">
|
||||
数据缓存:
|
||||
{stats.source.cache.hit ? '已读取固化结果' : '本次重新统计'}
|
||||
{stats.source.cache.refreshedAt && `,更新时间 ${formatTime(stats.source.cache.refreshedAt)}`}
|
||||
{stats.source.cache.stale && ',正在后台刷新'}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
@ -93,6 +93,14 @@ export interface PublicTrafficStats {
|
||||
truncated: boolean
|
||||
issues: PublicTrafficSourceIssue[]
|
||||
requestBytesAvailable: boolean
|
||||
cache?: {
|
||||
persisted: boolean
|
||||
hit: boolean
|
||||
stale: boolean
|
||||
refreshInProgress: boolean
|
||||
refreshedAt: string | null
|
||||
expiresAt: string | null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -107,9 +115,28 @@ interface PublicTrafficAggregationState {
|
||||
interface PublicTrafficCacheEntry {
|
||||
key: string
|
||||
createdAt: number
|
||||
expiresAt: number
|
||||
stats: PublicTrafficStats
|
||||
}
|
||||
|
||||
interface PublicTrafficStatsOptions {
|
||||
forceRefresh?: boolean
|
||||
}
|
||||
|
||||
interface PublicTrafficScanContext {
|
||||
logPaths: string[]
|
||||
maxLines: number
|
||||
stableCacheKey: string
|
||||
sourceSignature: string
|
||||
ttlSeconds: number
|
||||
}
|
||||
|
||||
interface PersistentTrafficCacheRow {
|
||||
stats: PublicTrafficStats | string
|
||||
refreshed_at: Date | string
|
||||
expires_at: Date | string
|
||||
}
|
||||
|
||||
const MONTHS: Record<string, number> = {
|
||||
Jan: 0,
|
||||
Feb: 1,
|
||||
@ -136,6 +163,8 @@ const DEFAULT_LOG_PATHS = [
|
||||
|
||||
const statsCache = new Map<string, PublicTrafficCacheEntry>()
|
||||
const MAX_CACHE_ENTRIES = 20
|
||||
const refreshPromises = new Map<string, Promise<void>>()
|
||||
let persistentCacheTableEnsured = false
|
||||
|
||||
function emptyStatusCounts(): Record<TrafficStatusBucket, number> {
|
||||
return {
|
||||
@ -760,8 +789,18 @@ async function scanLogFile(
|
||||
}
|
||||
}
|
||||
|
||||
function buildCacheKey(range: PublicTrafficRange, logPaths: string[], maxLines: number): string {
|
||||
const sourceSignature = logPaths
|
||||
function buildStableCacheKey(range: PublicTrafficRange, logPaths: string[], maxLines: number): string {
|
||||
return [
|
||||
'public-traffic-v2',
|
||||
range.start.toISOString(),
|
||||
range.end.toISOString(),
|
||||
maxLines,
|
||||
logPaths.join('|'),
|
||||
].join('::')
|
||||
}
|
||||
|
||||
function buildSourceSignature(logPaths: string[]): string {
|
||||
return logPaths
|
||||
.map((logPath) => {
|
||||
try {
|
||||
const fileStat = statSync(logPath)
|
||||
@ -771,27 +810,153 @@ function buildCacheKey(range: PublicTrafficRange, logPaths: string[], maxLines:
|
||||
}
|
||||
})
|
||||
.join('|')
|
||||
}
|
||||
|
||||
return [
|
||||
function getCacheTtlSeconds() {
|
||||
const parsed = Number.parseInt(process.env.PUBLIC_TRAFFIC_STATS_CACHE_TTL_SECONDS || '900', 10)
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : 900
|
||||
}
|
||||
|
||||
function getScanContext(range: PublicTrafficRange): PublicTrafficScanContext {
|
||||
const logPaths = getConfiguredPublicTrafficLogPaths()
|
||||
const maxLines = Math.max(1000, Number.parseInt(process.env.PUBLIC_TRAFFIC_MAX_LINES || '1000000', 10) || 1000000)
|
||||
return {
|
||||
logPaths,
|
||||
maxLines,
|
||||
stableCacheKey: buildStableCacheKey(range, logPaths, maxLines),
|
||||
sourceSignature: buildSourceSignature(logPaths),
|
||||
ttlSeconds: getCacheTtlSeconds(),
|
||||
}
|
||||
}
|
||||
|
||||
function cloneStats(stats: PublicTrafficStats): PublicTrafficStats {
|
||||
return JSON.parse(JSON.stringify(stats)) as PublicTrafficStats
|
||||
}
|
||||
|
||||
function withCacheMetadata(
|
||||
stats: PublicTrafficStats,
|
||||
cache: NonNullable<PublicTrafficStats['source']['cache']>,
|
||||
): PublicTrafficStats {
|
||||
const nextStats = cloneStats(stats)
|
||||
nextStats.source.cache = cache
|
||||
return nextStats
|
||||
}
|
||||
|
||||
function persistentTrafficCacheEnabled() {
|
||||
return process.env.NODE_ENV !== 'test' && process.env.PUBLIC_TRAFFIC_STATS_DB_CACHE !== '0' && Boolean(process.env.DATABASE_URL)
|
||||
}
|
||||
|
||||
async function ensurePersistentTrafficCacheTable() {
|
||||
if (persistentCacheTableEnsured || !persistentTrafficCacheEnabled()) return false
|
||||
|
||||
const { prisma } = await import('./prisma')
|
||||
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE TABLE IF NOT EXISTS "public_traffic_stats_cache" (
|
||||
"cache_key" TEXT PRIMARY KEY,
|
||||
"range_start" TIMESTAMPTZ NOT NULL,
|
||||
"range_end" TIMESTAMPTZ NOT NULL,
|
||||
"stats" JSONB NOT NULL,
|
||||
"source_signature" TEXT NOT NULL DEFAULT '',
|
||||
"refreshed_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"expires_at" TIMESTAMPTZ NOT NULL,
|
||||
"created_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_at" TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
`)
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE INDEX IF NOT EXISTS "public_traffic_stats_cache_range_idx"
|
||||
ON "public_traffic_stats_cache" ("range_start", "range_end")
|
||||
`)
|
||||
await prisma.$executeRawUnsafe(`
|
||||
CREATE INDEX IF NOT EXISTS "public_traffic_stats_cache_expires_at_idx"
|
||||
ON "public_traffic_stats_cache" ("expires_at")
|
||||
`)
|
||||
|
||||
persistentCacheTableEnsured = true
|
||||
return true
|
||||
}
|
||||
|
||||
async function readPersistentTrafficCache(cacheKey: string): Promise<PersistentTrafficCacheRow | null> {
|
||||
if (!(await ensurePersistentTrafficCacheTable())) return null
|
||||
|
||||
const { prisma } = await import('./prisma')
|
||||
const rows = await prisma.$queryRawUnsafe<PersistentTrafficCacheRow[]>(
|
||||
`
|
||||
SELECT "stats", "refreshed_at", "expires_at"
|
||||
FROM "public_traffic_stats_cache"
|
||||
WHERE "cache_key" = $1
|
||||
LIMIT 1
|
||||
`,
|
||||
cacheKey,
|
||||
)
|
||||
|
||||
return rows[0] || null
|
||||
}
|
||||
|
||||
async function writePersistentTrafficCache(
|
||||
cacheKey: string,
|
||||
range: PublicTrafficRange,
|
||||
context: PublicTrafficScanContext,
|
||||
stats: PublicTrafficStats,
|
||||
) {
|
||||
if (!(await ensurePersistentTrafficCacheTable())) return
|
||||
|
||||
const { prisma } = await import('./prisma')
|
||||
const now = new Date()
|
||||
const expiresAt = new Date(now.getTime() + context.ttlSeconds * 1000)
|
||||
const statsForStorage = cloneStats(stats)
|
||||
delete statsForStorage.source.cache
|
||||
|
||||
await prisma.$executeRawUnsafe(
|
||||
`
|
||||
INSERT INTO "public_traffic_stats_cache" (
|
||||
"cache_key",
|
||||
"range_start",
|
||||
"range_end",
|
||||
"stats",
|
||||
"source_signature",
|
||||
"refreshed_at",
|
||||
"expires_at",
|
||||
"updated_at"
|
||||
)
|
||||
VALUES ($1, $2::timestamptz, $3::timestamptz, $4::jsonb, $5, $6::timestamptz, $7::timestamptz, $6::timestamptz)
|
||||
ON CONFLICT ("cache_key") DO UPDATE SET
|
||||
"range_start" = EXCLUDED."range_start",
|
||||
"range_end" = EXCLUDED."range_end",
|
||||
"stats" = EXCLUDED."stats",
|
||||
"source_signature" = EXCLUDED."source_signature",
|
||||
"refreshed_at" = EXCLUDED."refreshed_at",
|
||||
"expires_at" = EXCLUDED."expires_at",
|
||||
"updated_at" = EXCLUDED."updated_at"
|
||||
`,
|
||||
cacheKey,
|
||||
range.start.toISOString(),
|
||||
range.end.toISOString(),
|
||||
maxLines,
|
||||
sourceSignature,
|
||||
].join('::')
|
||||
JSON.stringify(statsForStorage),
|
||||
context.sourceSignature,
|
||||
now.toISOString(),
|
||||
expiresAt.toISOString(),
|
||||
)
|
||||
}
|
||||
|
||||
function getCachedStats(cacheKey: string): PublicTrafficStats | null {
|
||||
const cached = statsCache.get(cacheKey)
|
||||
if (!cached) return null
|
||||
if (cached.expiresAt <= Date.now()) {
|
||||
statsCache.delete(cacheKey)
|
||||
return null
|
||||
}
|
||||
statsCache.delete(cacheKey)
|
||||
statsCache.set(cacheKey, cached)
|
||||
return cached.stats
|
||||
}
|
||||
|
||||
function setCachedStats(cacheKey: string, stats: PublicTrafficStats) {
|
||||
function setCachedStats(cacheKey: string, stats: PublicTrafficStats, ttlSeconds = 60) {
|
||||
statsCache.set(cacheKey, {
|
||||
key: cacheKey,
|
||||
createdAt: Date.now(),
|
||||
expiresAt: Date.now() + ttlSeconds * 1000,
|
||||
stats,
|
||||
})
|
||||
|
||||
@ -802,13 +967,7 @@ function setCachedStats(cacheKey: string, stats: PublicTrafficStats) {
|
||||
}
|
||||
}
|
||||
|
||||
export async function getPublicTrafficStats(range: PublicTrafficRange): Promise<PublicTrafficStats> {
|
||||
const logPaths = getConfiguredPublicTrafficLogPaths()
|
||||
const maxLines = Math.max(1000, Number.parseInt(process.env.PUBLIC_TRAFFIC_MAX_LINES || '1000000', 10) || 1000000)
|
||||
const cacheKey = buildCacheKey(range, logPaths, maxLines)
|
||||
const cachedStats = getCachedStats(cacheKey)
|
||||
if (cachedStats) return cachedStats
|
||||
|
||||
async function scanPublicTrafficStats(range: PublicTrafficRange, context: PublicTrafficScanContext): Promise<PublicTrafficStats> {
|
||||
const aggregation = createAggregationState()
|
||||
|
||||
const stats: PublicTrafficStats = {
|
||||
@ -833,7 +992,7 @@ export async function getPublicTrafficStats(range: PublicTrafficRange): Promise<
|
||||
byPurpose: [],
|
||||
topPaths: [],
|
||||
source: {
|
||||
logPaths,
|
||||
logPaths: context.logPaths,
|
||||
scannedFiles: 0,
|
||||
scannedLines: 0,
|
||||
matchedLines: 0,
|
||||
@ -844,9 +1003,9 @@ export async function getPublicTrafficStats(range: PublicTrafficRange): Promise<
|
||||
},
|
||||
}
|
||||
|
||||
for (const logPath of logPaths) {
|
||||
for (const logPath of context.logPaths) {
|
||||
try {
|
||||
await scanLogFile(logPath, range, stats, aggregation, maxLines)
|
||||
await scanLogFile(logPath, range, stats, aggregation, context.maxLines)
|
||||
} catch (error) {
|
||||
stats.source.issues.push({
|
||||
path: logPath,
|
||||
@ -866,7 +1025,104 @@ export async function getPublicTrafficStats(range: PublicTrafficRange): Promise<
|
||||
stats.topPaths.sort((a, b) => b.responseBytes - a.responseBytes)
|
||||
stats.topPaths = stats.topPaths.slice(0, 20)
|
||||
|
||||
setCachedStats(cacheKey, stats)
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
async function refreshPublicTrafficStatsInBackground(
|
||||
cacheKey: string,
|
||||
range: PublicTrafficRange,
|
||||
context: PublicTrafficScanContext,
|
||||
) {
|
||||
if (refreshPromises.has(cacheKey)) return
|
||||
|
||||
const promise = (async () => {
|
||||
try {
|
||||
const stats = await scanPublicTrafficStats(range, context)
|
||||
await writePersistentTrafficCache(cacheKey, range, context, stats)
|
||||
setCachedStats(cacheKey, stats, context.ttlSeconds)
|
||||
} catch (error) {
|
||||
console.error('[public-traffic] 后台刷新公网流量统计失败:', error)
|
||||
} finally {
|
||||
refreshPromises.delete(cacheKey)
|
||||
}
|
||||
})()
|
||||
|
||||
refreshPromises.set(cacheKey, promise)
|
||||
}
|
||||
|
||||
function parsePersistentStats(row: PersistentTrafficCacheRow): PublicTrafficStats | null {
|
||||
const rawStats = typeof row.stats === 'string' ? JSON.parse(row.stats) : row.stats
|
||||
if (!rawStats || typeof rawStats !== 'object') return null
|
||||
return rawStats as PublicTrafficStats
|
||||
}
|
||||
|
||||
export async function getPublicTrafficStats(
|
||||
range: PublicTrafficRange,
|
||||
options: PublicTrafficStatsOptions = {},
|
||||
): Promise<PublicTrafficStats> {
|
||||
const context = getScanContext(range)
|
||||
const cacheKey = context.stableCacheKey
|
||||
|
||||
if (!options.forceRefresh) {
|
||||
const cachedStats = getCachedStats(cacheKey)
|
||||
if (cachedStats) {
|
||||
return withCacheMetadata(cachedStats, {
|
||||
persisted: false,
|
||||
hit: true,
|
||||
stale: false,
|
||||
refreshInProgress: refreshPromises.has(cacheKey),
|
||||
refreshedAt: cachedStats.generatedAt,
|
||||
expiresAt: null,
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
const persistentRow = await readPersistentTrafficCache(cacheKey)
|
||||
if (persistentRow) {
|
||||
const persistentStats = parsePersistentStats(persistentRow)
|
||||
if (persistentStats) {
|
||||
const refreshedAt = new Date(persistentRow.refreshed_at)
|
||||
const expiresAt = new Date(persistentRow.expires_at)
|
||||
const stale = expiresAt.getTime() <= Date.now()
|
||||
|
||||
if (stale) {
|
||||
await refreshPublicTrafficStatsInBackground(cacheKey, range, context)
|
||||
} else {
|
||||
const secondsToExpire = Math.floor((expiresAt.getTime() - Date.now()) / 1000)
|
||||
setCachedStats(cacheKey, persistentStats, Math.max(10, secondsToExpire))
|
||||
}
|
||||
|
||||
return withCacheMetadata(persistentStats, {
|
||||
persisted: true,
|
||||
hit: true,
|
||||
stale,
|
||||
refreshInProgress: refreshPromises.has(cacheKey),
|
||||
refreshedAt: Number.isNaN(refreshedAt.getTime()) ? null : refreshedAt.toISOString(),
|
||||
expiresAt: Number.isNaN(expiresAt.getTime()) ? null : expiresAt.toISOString(),
|
||||
})
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[public-traffic] 读取公网流量缓存失败,回退实时统计:', error)
|
||||
}
|
||||
}
|
||||
|
||||
const stats = await scanPublicTrafficStats(range, context)
|
||||
|
||||
try {
|
||||
await writePersistentTrafficCache(cacheKey, range, context, stats)
|
||||
} catch (error) {
|
||||
console.error('[public-traffic] 写入公网流量缓存失败:', error)
|
||||
}
|
||||
|
||||
setCachedStats(cacheKey, stats, context.ttlSeconds)
|
||||
|
||||
return withCacheMetadata(stats, {
|
||||
persisted: persistentTrafficCacheEnabled(),
|
||||
hit: false,
|
||||
stale: false,
|
||||
refreshInProgress: false,
|
||||
refreshedAt: stats.generatedAt,
|
||||
expiresAt: new Date(Date.now() + context.ttlSeconds * 1000).toISOString(),
|
||||
})
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user