const crypto = require('crypto'); const redis = require('../config/redis'); const { runHeavyDatabaseTask } = require('./databaseLoadGuard'); const memoryStore = new Map(); const pendingLoads = new Map(); const MAX_RECENT_EVENTS = 200; const CACHE_MISS_LOADER_GUARD_ENABLED = String( process.env.DATA_PAGE_CACHE_MISS_GUARD_ENABLED || 'true' ).toLowerCase() !== 'false'; const cacheStats = { total: 0, memoryHits: 0, redisHits: 0, pendingHits: 0, misses: 0, lastResetAt: new Date().toISOString(), labels: new Map(), recentEvents: [], }; function isPlainObject(value) { return Object.prototype.toString.call(value) === '[object Object]'; } function normalizeValue(value) { if (Array.isArray(value)) { return value.map((item) => normalizeValue(item)); } if (isPlainObject(value)) { return Object.keys(value) .sort() .reduce((acc, key) => { acc[key] = normalizeValue(value[key]); return acc; }, {}); } if (typeof value === 'string') { return value.trim(); } return value; } function buildHashedCacheKey(namespace, payload, version = 'v1') { const normalizedPayload = normalizeValue(payload); const hash = crypto .createHash('md5') .update(JSON.stringify(normalizedPayload)) .digest('hex'); return `${namespace}:${version}:${hash}`; } function getMemoryEntry(key) { const cached = memoryStore.get(key); if (!cached) { return null; } if (cached.expiresAt <= Date.now()) { memoryStore.delete(key); return null; } return cached.value; } function setMemoryEntry(key, value, ttlSeconds) { if (!ttlSeconds || ttlSeconds <= 0) { return; } memoryStore.set(key, { value, expiresAt: Date.now() + ttlSeconds * 1000, }); } function recordCacheEvent({ label, hit, elapsedMs, cacheStored }) { cacheStats.total += 1; if (hit === 'memory') cacheStats.memoryHits += 1; if (hit === 'redis') cacheStats.redisHits += 1; if (hit === 'pending') cacheStats.pendingHits += 1; if (hit === 'miss') cacheStats.misses += 1; if (label) { const labelStats = cacheStats.labels.get(label) || { label, total: 0, memoryHits: 0, redisHits: 0, pendingHits: 0, misses: 0, totalElapsedMs: 0, lastHit: null, lastSeenAt: null, }; labelStats.total += 1; labelStats.totalElapsedMs += elapsedMs; labelStats.lastHit = hit; labelStats.lastSeenAt = new Date().toISOString(); if (hit === 'memory') labelStats.memoryHits += 1; if (hit === 'redis') labelStats.redisHits += 1; if (hit === 'pending') labelStats.pendingHits += 1; if (hit === 'miss') labelStats.misses += 1; cacheStats.labels.set(label, labelStats); } cacheStats.recentEvents.unshift({ label: label || '(no-label)', hit, elapsedMs, cacheStored, timestamp: new Date().toISOString(), }); if (cacheStats.recentEvents.length > MAX_RECENT_EVENTS) { cacheStats.recentEvents.length = MAX_RECENT_EVENTS; } } async function getOrSetCachedPayload({ key, memoryTtlSeconds = 60, redisTtlSeconds = 300, staleKey = '', staleMemoryTtlSeconds = memoryTtlSeconds, staleRedisTtlSeconds = redisTtlSeconds, returnStaleWhileRevalidate = false, loader, shouldCache, label, guardLoader = CACHE_MISS_LOADER_GUARD_ENABLED, }) { const startedAt = Date.now(); const canUseStaleCache = Boolean(returnStaleWhileRevalidate && staleKey); const readStaleCache = async () => { if (!canUseStaleCache) { return null; } const staleMemoryValue = getMemoryEntry(staleKey); if (staleMemoryValue !== null) { return { value: staleMemoryValue, hit: 'stale-memory', }; } const staleRedisValue = await redis.get(staleKey); if (staleRedisValue !== null) { setMemoryEntry(staleKey, staleRedisValue, staleMemoryTtlSeconds); return { value: staleRedisValue, hit: 'stale-redis', }; } return null; }; const writeStaleCacheFromFreshValue = (value) => { if (!canUseStaleCache) { return; } setMemoryEntry(staleKey, value, staleMemoryTtlSeconds); redis.setex(staleKey, staleRedisTtlSeconds, value).catch((error) => { console.warn(`[DataPageCache] ${label || staleKey} 写入兜底缓存失败:`, error.message || error); }); }; const startLoad = () => { let allowedToCache = false; const loadPromise = (async () => { const loadedValue = guardLoader ? await runHeavyDatabaseTask(`data-page-cache:${label || key}`, loader) : await loader(); allowedToCache = typeof shouldCache === 'function' ? shouldCache(loadedValue) : true; if (allowedToCache) { setMemoryEntry(key, loadedValue, memoryTtlSeconds); await redis.setex(key, redisTtlSeconds, loadedValue); if (canUseStaleCache) { setMemoryEntry(staleKey, loadedValue, staleMemoryTtlSeconds); await redis.setex(staleKey, staleRedisTtlSeconds, loadedValue); } } return { value: loadedValue, allowedToCache, }; })() .finally(() => { if (pendingLoads.get(key) === loadPromise) { pendingLoads.delete(key); } }); pendingLoads.set(key, loadPromise); return loadPromise; }; const memoryValue = getMemoryEntry(key); if (memoryValue !== null) { writeStaleCacheFromFreshValue(memoryValue); if (label) { console.log(`[DataPageCache] ${label} hit=memory elapsedMs=${Date.now() - startedAt}`); } recordCacheEvent({ label, hit: 'memory', elapsedMs: Date.now() - startedAt, cacheStored: true }); return { value: memoryValue, hit: 'memory', }; } const redisValue = await redis.get(key); if (redisValue !== null) { setMemoryEntry(key, redisValue, memoryTtlSeconds); writeStaleCacheFromFreshValue(redisValue); if (label) { console.log(`[DataPageCache] ${label} hit=redis elapsedMs=${Date.now() - startedAt}`); } recordCacheEvent({ label, hit: 'redis', elapsedMs: Date.now() - startedAt, cacheStored: true }); return { value: redisValue, hit: 'redis', }; } const pendingLoad = pendingLoads.get(key); if (pendingLoad) { const staleResult = await readStaleCache(); if (staleResult) { if (label) { console.log(`[DataPageCache] ${label} hit=${staleResult.hit} refresh=pending elapsedMs=${Date.now() - startedAt}`); } recordCacheEvent({ label, hit: staleResult.hit, elapsedMs: Date.now() - startedAt, cacheStored: true }); return staleResult; } const pendingResult = await pendingLoad; const value = pendingResult?.value; if (pendingResult?.allowedToCache) { setMemoryEntry(key, value, memoryTtlSeconds); } if (label) { console.log(`[DataPageCache] ${label} hit=pending elapsedMs=${Date.now() - startedAt}`); } recordCacheEvent({ label, hit: 'pending', elapsedMs: Date.now() - startedAt, cacheStored: Boolean(pendingResult?.allowedToCache), }); return { value, hit: 'pending', }; } const loadPromise = startLoad(); const staleResult = await readStaleCache(); if (staleResult) { loadPromise.catch((error) => { console.warn(`[DataPageCache] ${label || key} 后台刷新失败:`, error.message || error); }); if (label) { console.log(`[DataPageCache] ${label} hit=${staleResult.hit} refresh=background elapsedMs=${Date.now() - startedAt}`); } recordCacheEvent({ label, hit: staleResult.hit, elapsedMs: Date.now() - startedAt, cacheStored: true }); return staleResult; } let value; let allowedToCache = false; const loadedResult = await loadPromise; value = loadedResult.value; allowedToCache = Boolean(loadedResult.allowedToCache); if (label) { console.log( `[DataPageCache] ${label} hit=miss cacheStored=${allowedToCache ? 'yes' : 'no'} elapsedMs=${Date.now() - startedAt}` ); } recordCacheEvent({ label, hit: 'miss', elapsedMs: Date.now() - startedAt, cacheStored: allowedToCache, }); return { value, hit: 'miss', }; } async function deleteByPrefix(prefix) { for (const key of memoryStore.keys()) { if (key.startsWith(prefix)) { memoryStore.delete(key); } } for (const key of pendingLoads.keys()) { if (key.startsWith(prefix)) { pendingLoads.delete(key); } } await redis.delPattern(`${prefix}*`); } function getStatsSnapshot() { const labels = Array.from(cacheStats.labels.values()) .map((item) => ({ ...item, avgElapsedMs: item.total ? Number((item.totalElapsedMs / item.total).toFixed(2)) : 0, })) .sort((a, b) => b.total - a.total); return { total: cacheStats.total, memoryHits: cacheStats.memoryHits, redisHits: cacheStats.redisHits, pendingHits: cacheStats.pendingHits, misses: cacheStats.misses, memoryHitRate: cacheStats.total ? Number((cacheStats.memoryHits * 100 / cacheStats.total).toFixed(2)) : 0, redisHitRate: cacheStats.total ? Number((cacheStats.redisHits * 100 / cacheStats.total).toFixed(2)) : 0, pendingHitRate: cacheStats.total ? Number((cacheStats.pendingHits * 100 / cacheStats.total).toFixed(2)) : 0, missRate: cacheStats.total ? Number((cacheStats.misses * 100 / cacheStats.total).toFixed(2)) : 0, lastResetAt: cacheStats.lastResetAt, labels, recentEvents: cacheStats.recentEvents, }; } function resetStats() { cacheStats.total = 0; cacheStats.memoryHits = 0; cacheStats.redisHits = 0; cacheStats.pendingHits = 0; cacheStats.misses = 0; cacheStats.labels.clear(); cacheStats.recentEvents = []; cacheStats.lastResetAt = new Date().toISOString(); } module.exports = { buildHashedCacheKey, deleteByPrefix, getOrSetCachedPayload, getStatsSnapshot, resetStats, };