4e89b57798
- Expand batch upload list area and optimize preview image layout - Improve upload speed tracking and display (server-side & client-side) - Reduce network detection log noise - Optimize Docker build process (caching, env validation) - Add shadow environment deployment support
363 lines
11 KiB
TypeScript
363 lines
11 KiB
TypeScript
/**
|
||
* 上传会话存储管理
|
||
*
|
||
* 🔄 支持环境变量切换：
|
||
* - USE_REDIS=true → 使用Redis（生产环境推荐）
|
||
* - USE_REDIS=false 或未设置 → 使用内存Map（开发/测试环境）
|
||
*
|
||
* 📋 内存Map的局限性：
|
||
* 1. 数据不持久化 - 服务器/容器重启后数据丢失
|
||
* 2. 不支持多实例 - 多个容器实例无法共享会话数据
|
||
* 3. 内存管理 - 需要手动清理过期数据
|
||
* 4. 无法跨服务 - 不同服务无法共享数据
|
||
*
|
||
* ✅ Redis的优势：
|
||
* 1. 数据持久化 - 服务器重启后数据保留
|
||
* 2. 支持多实例 - 多个容器实例可以共享数据
|
||
* 3. 自动过期 - 支持TTL，无需手动清理
|
||
* 4. 跨服务共享 - 不同服务可以共享数据
|
||
*
|
||
* 📚 详细说明请查看：docs/upload-sessions-redis-explanation.md
|
||
*/
|
||
|
||
export interface UploadSession {
|
||
fileName: string
|
||
fileSize: number
|
||
totalChunks: number
|
||
chunks: Map<number, string> // chunkIndex -> chunkFilePath
|
||
createdAt: number
|
||
/**
|
||
* 最近一次“确实有上传流量”的时间（用于性能面板判断是否仍在上传）
|
||
* - init 时会写入
|
||
* - 每次 chunk 上传成功后会更新
|
||
*/
|
||
lastActivityAt?: number
|
||
/**
|
||
* 已接收的字节数（用于估算上传速度/进度展示）
|
||
* 注意：这是服务端接收字节，不代表最终文件写入/合并完成
|
||
*/
|
||
uploadedBytes?: number
|
||
/**
|
||
* 当前估算上传速度（字节/秒）
|
||
* 由 chunk 上传端点按增量估算，供性能面板展示
|
||
*/
|
||
currentSpeed?: number
|
||
/**
|
||
* 用于速度估算的上一次采样点
|
||
*/
|
||
_speedSampleAt?: number
|
||
_speedSampleBytes?: number
|
||
userId: string
|
||
}
|
||
|
||
// 检查是否使用Redis
|
||
const USE_REDIS = process.env.USE_REDIS === 'true'
|
||
|
||
// 动态导入Redis实现（如果启用）
|
||
let redisModule: typeof import('./upload-sessions-redis') | null = null
|
||
|
||
if (USE_REDIS) {
|
||
try {
|
||
// 动态导入，避免在未安装ioredis时出错
|
||
redisModule = require('./upload-sessions-redis')
|
||
console.log('[UploadSessions] 使用Redis存储（生产环境）')
|
||
} catch (error) {
|
||
console.warn('[UploadSessions] Redis模块加载失败，降级到内存Map:', error)
|
||
console.warn('[UploadSessions] 请确保已安装ioredis: npm install ioredis')
|
||
}
|
||
} else {
|
||
console.log('[UploadSessions] 使用内存Map存储（开发/测试环境）')
|
||
}
|
||
|
||
// 内存存储（单实例部署可以使用）
|
||
const memorySessions = new Map<string, UploadSession>()
|
||
|
||
function setMemorySession(uploadId: string, session: UploadSession) {
|
||
memorySessions.set(uploadId, session)
|
||
}
|
||
|
||
function deleteMemorySession(uploadId: string) {
|
||
memorySessions.delete(uploadId)
|
||
}
|
||
|
||
function getMemorySession(uploadId: string): UploadSession | null {
|
||
return memorySessions.get(uploadId) || null
|
||
}
|
||
|
||
async function withRedisFallback<T>(
|
||
actionName: string,
|
||
redisAction: () => Promise<T>,
|
||
memoryFallback: () => Promise<T> | T
|
||
): Promise<T> {
|
||
if (USE_REDIS && redisModule) {
|
||
try {
|
||
return await redisAction()
|
||
} catch (error) {
|
||
console.error(`[UploadSessions] Redis ${actionName} failed, fallback to memory:`, error)
|
||
return await memoryFallback()
|
||
}
|
||
}
|
||
return await memoryFallback()
|
||
}
|
||
|
||
// 清理过期会话（8小时）- 仅内存Map需要
|
||
if (!USE_REDIS || !redisModule) {
|
||
setInterval(() => {
|
||
const now = Date.now()
|
||
const expireTime = 8 * 60 * 60 * 1000 // 8小时
|
||
const keysToDelete: string[] = []
|
||
|
||
memorySessions.forEach((session, uploadId) => {
|
||
if (now - session.createdAt > expireTime) {
|
||
keysToDelete.push(uploadId)
|
||
}
|
||
})
|
||
|
||
keysToDelete.forEach(key => {
|
||
memorySessions.delete(key)
|
||
console.log(`[UploadSessions] 清理过期会话: ${key}`)
|
||
})
|
||
|
||
if (keysToDelete.length > 0) {
|
||
console.log(`[UploadSessions] 已清理 ${keysToDelete.length} 个过期会话，当前会话数: ${memorySessions.size}`)
|
||
} else {
|
||
// 即使没有清理，也记录当前会话数（用于监控）
|
||
console.log(`[UploadSessions] 当前会话数: ${memorySessions.size}`)
|
||
}
|
||
}, 60 * 60 * 1000) // 每小时清理一次
|
||
}
|
||
|
||
/**
|
||
* 保存上传会话
|
||
*/
|
||
export async function saveUploadSession(
|
||
uploadId: string,
|
||
session: UploadSession
|
||
): Promise<void> {
|
||
setMemorySession(uploadId, session)
|
||
await withRedisFallback(
|
||
'saveUploadSession',
|
||
() => redisModule!.saveUploadSession(uploadId, session),
|
||
async () => {}
|
||
)
|
||
}
|
||
|
||
/**
|
||
* 获取上传会话
|
||
*/
|
||
export async function getUploadSession(
|
||
uploadId: string
|
||
): Promise<UploadSession | null> {
|
||
return withRedisFallback(
|
||
'getUploadSession',
|
||
async () => {
|
||
const session = await redisModule!.getUploadSession(uploadId)
|
||
if (session) {
|
||
setMemorySession(uploadId, session)
|
||
}
|
||
return session
|
||
},
|
||
async () => getMemorySession(uploadId)
|
||
)
|
||
}
|
||
|
||
/**
|
||
* 获取已上传的完整文件路径（从chunk临时目录）
|
||
* 返回组装后的完整文件路径
|
||
*/
|
||
export async function getUploadedFile(uploadId: string): Promise<string | null> {
|
||
const session = await getUploadSession(uploadId);
|
||
if (!session) {
|
||
return null;
|
||
}
|
||
|
||
const path = await import('path');
|
||
const { readdir, readFile, writeFile, mkdir, access } = await import('fs/promises');
|
||
|
||
// Chunk文件所在目录
|
||
const tempDir = path.join(process.env.UPLOAD_DIR || './uploads', 'temp', uploadId);
|
||
|
||
// 输出目录和路径
|
||
const outputDir = path.join(process.env.UPLOAD_DIR || './uploads', 'assembled');
|
||
const outputPath = path.join(outputDir, `${uploadId}-${session.fileName}`);
|
||
|
||
try {
|
||
// 先检查文件是否已经存在（避免重复组装）
|
||
try {
|
||
await access(outputPath);
|
||
console.log(`[Upload Session] File already exists, skip assembly: ${outputPath}`);
|
||
return outputPath;
|
||
} catch (e) {
|
||
// 文件不存在，需要组装
|
||
}
|
||
|
||
// 读取所有chunk文件
|
||
const files = await readdir(tempDir);
|
||
const chunkFiles = files
|
||
.filter(f => f.startsWith('chunk-'))
|
||
.sort((a, b) => {
|
||
const indexA = parseInt(a.replace('chunk-', ''));
|
||
const indexB = parseInt(b.replace('chunk-', ''));
|
||
return indexA - indexB;
|
||
});
|
||
|
||
if (chunkFiles.length === 0) {
|
||
return null;
|
||
}
|
||
|
||
// 创建输出目录
|
||
await mkdir(outputDir, { recursive: true });
|
||
|
||
// 按顺序读取并写入所有chunk
|
||
const chunks: Buffer[] = [];
|
||
for (const chunkFile of chunkFiles) {
|
||
const chunkPath = path.join(tempDir, chunkFile);
|
||
const chunkBuffer = await readFile(chunkPath);
|
||
chunks.push(chunkBuffer);
|
||
}
|
||
|
||
// 合并所有chunk并写入文件
|
||
const completeBuffer = Buffer.concat(chunks);
|
||
await writeFile(outputPath, completeBuffer);
|
||
|
||
console.log(`[Upload Session] Assembled file: ${outputPath} (${completeBuffer.length} bytes)`);
|
||
return outputPath;
|
||
} catch (error) {
|
||
console.error(`[Upload Session] Failed to get uploaded file for ${uploadId}:`, error);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 删除上传会话
|
||
*/
|
||
export async function deleteUploadSession(uploadId: string): Promise<void> {
|
||
deleteMemorySession(uploadId)
|
||
await withRedisFallback(
|
||
'deleteUploadSession',
|
||
() => redisModule!.deleteUploadSession(uploadId),
|
||
async () => {}
|
||
)
|
||
}
|
||
|
||
/**
|
||
* 检查会话是否存在
|
||
*/
|
||
export async function hasUploadSession(uploadId: string): Promise<boolean> {
|
||
return withRedisFallback(
|
||
'hasUploadSession',
|
||
() => redisModule!.hasUploadSession(uploadId),
|
||
async () => memorySessions.has(uploadId)
|
||
)
|
||
}
|
||
|
||
/**
|
||
* 更新块信息
|
||
*/
|
||
export async function updateChunk(
|
||
uploadId: string,
|
||
chunkIndex: number,
|
||
chunkPath: string
|
||
): Promise<void> {
|
||
let session = getMemorySession(uploadId)
|
||
if (!session) {
|
||
session = await getUploadSession(uploadId)
|
||
}
|
||
if (!session) {
|
||
throw new Error(`上传会话不存在: ${uploadId}`)
|
||
}
|
||
session.chunks.set(chunkIndex, chunkPath)
|
||
setMemorySession(uploadId, session)
|
||
await withRedisFallback(
|
||
'updateChunk',
|
||
() => redisModule!.updateChunk(uploadId, chunkIndex, chunkPath),
|
||
async () => {}
|
||
)
|
||
}
|
||
|
||
/**
|
||
* 更新 chunk 信息 + 写入活动/速度统计
|
||
* - bytesDelta: 本次 chunk 的字节数（通常来自 File.size）
|
||
*/
|
||
export async function updateChunkWithStats(
|
||
uploadId: string,
|
||
chunkIndex: number,
|
||
chunkPath: string,
|
||
bytesDelta: number
|
||
): Promise<void> {
|
||
let session = getMemorySession(uploadId)
|
||
if (!session) {
|
||
session = await getUploadSession(uploadId)
|
||
}
|
||
if (!session) {
|
||
throw new Error(`上传会话不存在: ${uploadId}`)
|
||
}
|
||
|
||
const now = Date.now()
|
||
const safeBytesDelta = Number.isFinite(bytesDelta) && bytesDelta > 0 ? bytesDelta : 0
|
||
|
||
session.chunks.set(chunkIndex, chunkPath)
|
||
session.lastActivityAt = now
|
||
session.uploadedBytes = (session.uploadedBytes || 0) + safeBytesDelta
|
||
|
||
// 速度估算：用上一次采样点做差
|
||
const prevAt = session._speedSampleAt
|
||
const prevBytes = session._speedSampleBytes
|
||
if (prevAt && prevBytes !== undefined) {
|
||
const dtSec = (now - prevAt) / 1000
|
||
const dBytes = session.uploadedBytes - prevBytes
|
||
if (dtSec > 0 && dBytes >= 0) {
|
||
const instant = dBytes / dtSec
|
||
// 轻量平滑，避免抖动（EMA）
|
||
const alpha = 0.3
|
||
session.currentSpeed = session.currentSpeed ? session.currentSpeed * (1 - alpha) + instant * alpha : instant
|
||
}
|
||
}
|
||
session._speedSampleAt = now
|
||
session._speedSampleBytes = session.uploadedBytes
|
||
|
||
setMemorySession(uploadId, session)
|
||
|
||
await withRedisFallback(
|
||
'updateChunkWithStats',
|
||
async () => {
|
||
// 生产环境：直接覆盖保存（redis 侧仍然调用 updateChunk 触发 saveUploadSession）
|
||
// 这里复用 updateChunk，然后再把统计字段写回（避免改动 redis 模块的签名）
|
||
await redisModule!.updateChunk(uploadId, chunkIndex, chunkPath)
|
||
await redisModule!.saveUploadSession(uploadId, session as any)
|
||
return
|
||
},
|
||
async () => {}
|
||
)
|
||
}
|
||
|
||
/**
|
||
* 刷新上传会话（保留已上传进度）
|
||
* 用于会话过期后恢复上传，不清空已上传的分片记录
|
||
*/
|
||
export async function refreshUploadSession(uploadId: string): Promise<boolean> {
|
||
return withRedisFallback(
|
||
'refreshUploadSession',
|
||
async () => {
|
||
return await redisModule!.refreshUploadSession(uploadId)
|
||
},
|
||
async () => {
|
||
// 内存Map版本：更新过期时间
|
||
const session = memorySessions.get(uploadId)
|
||
if (session) {
|
||
// 内存会话自动延长4小时
|
||
const expireTime = 4 * 60 * 60 * 1000 // 4小时
|
||
session.createdAt = Date.now() - expireTime + (4 * 60 * 60 * 1000)
|
||
console.log(`[MemoryMap] 刷新上传会话成功: ${uploadId}`)
|
||
return true
|
||
}
|
||
console.warn(`[MemoryMap] 刷新上传会话失败，会话不存在: ${uploadId}`)
|
||
return false
|
||
}
|
||
)
|
||
}
|
||
|
||
// 导出内存Map（用于向后兼容，但不推荐直接使用）
|
||
export const uploadSessions = memorySessions
|
||
|