diff --git a/backend/src/index.js b/backend/src/index.js index 88daa64..f9802ee 100644 --- a/backend/src/index.js +++ b/backend/src/index.js @@ -1,890 +1,892 @@ -// 后端服务入口 -const path = require('path'); - -require('dotenv').config(); -require('dotenv').config({ path: path.resolve(__dirname, '../.env') }); - -const express = require('express'); -const cors = require('cors'); -const { CubejsServer } = require('@cubejs-backend/server'); -const mysql = require('mysql2/promise'); - -const installRoutes = require('./api/install'); -const authMiddleware = require('./middleware/auth'); -const { createPageDataCache } = require('./middleware/pageDataCache'); -const adminRoutes = require('./api/admin'); -const planningUploadRoutes = require('./api/planningUpload'); -const planningMayProductPublicRoutes = require('./api/planningMayProductPublic'); -const planningSelectionFissionExternalRoutes = require('./api/planningSelectionFissionExternal'); -const presaleExternalRoutes = require('./api/presaleExternal'); -const delistExternalRoutes = require('./api/delistExternal'); -const campaignCloudRoutes = require('./api/campaignCloud'); -const desktopClientRoutes = require('./api/desktopClient'); -const desktopPresenceRoutes = require('./api/desktopPresence'); -const desktopExecutionTaskRoutes = require('./api/desktopExecutionTasks'); -const alibaba1688ProductExternalRoutes = require('./api/alibaba1688ProductExternal'); -const taobaoDetailPlusRoutes = require('./api/taobaoDetailPlus'); -const thirdPartySsoRoutes = require('./api/thirdPartySso'); -const thirdPartySsoAdminRoutes = require('./api/thirdPartySsoAdmin'); -const channelRoutes = require('./api/channel'); -const analysisRoutes = require('./api/analysis'); -const productCodeBoardRoutes = require('./api/productCodeBoard'); -const productAnalysisRoutes = require('./api/productAnalysis'); -const marketingEventsRoutes = require('./api/marketingEvents'); -const promotionModuleRoutes = require('./api/promotionModule'); -const productMonitoringRoutes = require('./api/productMonitoring'); -const modularStockMonitorRoutes = require('./api/modularStockMonitor'); -const mkhBomRoutes = require('./api/mkhBom'); -const database = require('./config/database'); -const cloudDb = require('./config/cloudDatabase'); -const jianSunDb = require('./config/jianSunDatabase'); -const redis = require('./config/redis'); -const { startScheduler, manualRefresh } = require('./jobs/dailyRefresh'); -const { startStockSelectionSchedulers } = require('./jobs/stockSelectionSyncJob'); -const { startTaobaoProductEmbeddingSchedulers } = require('./jobs/taobaoProductEmbeddingSyncJob'); -const { startPddSkuDimensionSnapshotSchedulers } = require('./jobs/pddSkuDimensionSnapshotJob'); -const { startPlanningMayProductSnapshotSchedulers } = require('./jobs/planningMayProductSnapshotJob'); -const { startPlanningDingWorkflowSyncSchedulers } = require('./jobs/planningDingWorkflowSyncJob'); -const { startOpsBIncentiveSnapshotSchedulers } = require('./jobs/opsBIncentiveSnapshotJob'); -const { startGoodsFirstSeenScheduler } = require('./jobs/goodsFirstSeenJob'); -const { - startAlibaba1688ImageSearchExpiryScheduler, -} = require('./jobs/alibaba1688ImageSearchExpiryJob'); -const { - startMkhReplenishAlertScheduler, -} = require('./jobs/mkhReplenishAlertJob'); -const { DEFAULT_SITE_CONFIG, getInstallStatus } = require('./services/installService'); -const { ensureLocalSchema } = require('./services/localSchema'); -const { - prewarmChannelHotData, - prewarmChannelIdDefaultSnapshots, - prewarmChannelIdDashboardDefaultSnapshot, - prewarmChannelStockMonitorDashboardDefaultSnapshot, -} = require('./services/channelAnalysis'); -const { prewarmBrandAnalysisDefault } = require('./services/brandAnalysis'); -const { runHeavyDatabaseTask } = require('./services/databaseLoadGuard'); -const { warmSelectionFissionReadSchema } = require('./services/planningSelectionFissionService'); -const { - getPublicSystemConfigCache, - getStalePublicSystemConfigCache, - setPublicSystemConfigCache, -} = require('./services/publicSystemConfigCache'); -const { filterPublicSystemConfigs } = require('./services/systemConfigVisibility'); -const { - shouldRequireImageSearchCredentials, - verifyImageSearchCredentials, -} = require('./services/imageSearchCredentialGuard'); - -const PLANNING_PAGE_REDIS_TTL_SECONDS = Math.max( - Number(process.env.PLANNING_PAGE_RESPONSE_CACHE_REDIS_TTL_SECONDS) || 23 * 60 * 60, - 60 -); -const BRAND_ANALYSIS_PREWARM_DELAY_MS = Math.max( - Number(process.env.BRAND_ANALYSIS_PREWARM_DELAY_MS) || 20 * 1000, - 5 * 1000 -); -const PUBLIC_SYSTEM_CONFIG_CACHE_TTL_MS = Math.max( - Number(process.env.PUBLIC_SYSTEM_CONFIG_CACHE_TTL_SECONDS) || 300, - 30 -) * 1000; -const PUBLIC_SYSTEM_CONFIG_QUERY_TIMEOUT_MS = Math.max( - Number(process.env.PUBLIC_SYSTEM_CONFIG_QUERY_TIMEOUT_MS) || 1800, - 500 -); -const CUBE_SCHEDULED_REFRESH_TIMER = String(process.env.CUBEJS_SCHEDULED_REFRESH_TIMER || '') - .trim() - .toLowerCase() === 'true'; - -function getBuildIdentity() { - return { - sourceGitSha: String(process.env.SOURCE_GIT_SHA || '').trim() || null, - sourceGitRef: String(process.env.SOURCE_GIT_REF || '').trim() || null, - }; -} -const CHANNEL_STOCK_MONITOR_RESILIENCE_PREWARM_DELAY_MS = Math.max( - Number(process.env.CHANNEL_STOCK_MONITOR_RESILIENCE_PREWARM_DELAY_MS) || 90000, - 10000 -); -const CHANNEL_ID_RESILIENCE_PREWARM_DELAY_MS = Math.max( - Number(process.env.CHANNEL_ID_RESILIENCE_PREWARM_DELAY_MS) || 30000, - 10000 -); - -function shouldCachePlanningUploadResponse(body = {}) { - if (!body || body.error || body.success === false) { - return false; - } - - const payload = body.data || body; - if (payload?.pending || payload?.cache?.backgroundRefresh) { - return false; - } - - return true; -} - -function shouldHandlePlanningUploadPageCache(req) { - // 企划页首页会聚合 MAY 快照统计;MAY 商品页本身也依赖最新成功批次。 - // MAY 渠道洞察也依赖最新店铺/覆盖快照,外层整页缓存会把旧列口径整天顶住。 - // 榜单数据分析现在也依赖实时机会/竞争诊断结果,外层整页缓存会把旧字段结构整天顶住。 - // 抖音/小红书市场页支持实时批次切换,整页缓存会把旧批次和旧字段结构顶住。 - // 这些接口必须直接读实体表/轻缓存,不能再被 23 小时整页 Redis 顶住。 - if ( - req.path === '/' - || req.path.startsWith('/table/may_product_data') - || req.path.startsWith('/analysis/may-channel-insight') - || req.path.startsWith('/analysis/ops-b-incentive') - || req.path.startsWith('/selection-fission') - ) { - return false; - } - - return true; -} - -const analysisPageCache = createPageDataCache({ - namespace: 'analysis', - redisTtlSeconds: 300, -}); -const channelPageCache = createPageDataCache({ - namespace: 'channel', - redisTtlSeconds: 300, - // ID 分析内部已有按口径版本控制的缓存;导出接口返回 CSV,不能走整页 JSON 缓存。 - shouldHandleRequest: (req) => !['/id-analysis', '/export'].includes(req.path) - && !req.path.startsWith('/export-jobs'), -}); -const planningUploadPageCache = createPageDataCache({ - namespace: 'planning-upload', - version: 'v27-bypass-may-snapshot-pages-20260423', - redisTtlSeconds: PLANNING_PAGE_REDIS_TTL_SECONDS, - shouldHandleRequest: shouldHandlePlanningUploadPageCache, - shouldCacheBody: shouldCachePlanningUploadResponse, -}); -let serverMode = 'install'; - -function isStartupPrewarmEnabled() { - return String(process.env.ENABLE_STARTUP_PREWARM || '').toLowerCase() === 'true'; -} - -function runGuardedPrewarm(label, handler) { - return runHeavyDatabaseTask(`prewarm:${label}`, handler); -} - -function withTimeout(promise, timeoutMs, fallback) { - let timer = null; - return Promise.race([ - promise.finally(() => { - if (timer) { - clearTimeout(timer); - } - }), - new Promise((resolve) => { - timer = setTimeout(() => resolve(fallback), timeoutMs); - if (typeof timer.unref === 'function') { - timer.unref(); - } - }), - ]); -} - -function hasCompleteCloudEnvConfig() { - return cloudDb.hasCompleteEnvConfig(); -} - -function getCloudEnvConfig() { - const config = cloudDb.getEnvBusinessDatabaseConfig(); - return { - driver: config.driver || 'mssql', - host: config.host, - port: config.port || 1433, - database: config.database, - user: config.user, - password: config.password, - ssl: Boolean(config.ssl), - }; -} - -function getBootstrapCloudConfig() { - return { - driver: 'mysql', - host: process.env.LOCAL_DB_HOST, - port: parseInt(process.env.LOCAL_DB_PORT || '3306', 10), - database: process.env.LOCAL_DB_NAME, - user: process.env.LOCAL_DB_USER, - password: process.env.LOCAL_DB_PASSWORD, - ssl: false, - bootstrap: true, - }; -} - -// ===================== -// 从数据库获取云数据库配置 -// ===================== -async function getCloudDbConfig() { - let localPool; - try { - // 先连接本地数据库读取配置 - localPool = mysql.createPool({ - host: process.env.LOCAL_DB_HOST, - port: parseInt(process.env.LOCAL_DB_PORT || '3306'), - user: process.env.LOCAL_DB_USER, - password: process.env.LOCAL_DB_PASSWORD, - database: process.env.LOCAL_DB_NAME, - charset: 'utf8mb4', - waitForConnections: true, - connectionLimit: 2, - maxIdle: 2, - idleTimeout: 60000, - queueLimit: 10, - connectTimeout: 5000, - enableKeepAlive: true, - keepAliveInitialDelay: 0, - }); - - const [rows] = await localPool.query( - 'SELECT host, port, database_name, username, password, `ssl`, driver, use_env_config FROM database_configs WHERE is_default = 1 AND status = ? LIMIT 1', - ['active'] - ); - - if (rows.length > 0) { - const config = rows[0]; - if (config.use_env_config === 1 || config.use_env_config === true) { - if (hasCompleteCloudEnvConfig()) { - console.log('📌 使用 .env 业务数据库配置(CLOUD_DB_*)'); - return getCloudEnvConfig(); - } - console.warn('⚠️ 已开启 .env 业务数据库配置,但 CLOUD_DB_* 不完整,继续使用 database_configs 表配置'); - } - - console.log('📌 使用数据库配置(来自 database_configs 表)'); - return { - driver: config.driver || 'mysql', - host: config.host, - port: config.port || 3306, - database: config.database_name, - user: config.username, - password: config.password, - ssl: config.ssl === 1, - }; - } - } catch (error) { - console.warn('⚠️ 无法从数据库读取配置:', error.message); - } finally { - if (localPool) { - await localPool.end().catch((error) => { - console.warn('⚠️ 关闭系统库临时连接池失败:', error.message); - }); - } - } - - // 回退到环境变量配置 - if (hasCompleteCloudEnvConfig()) { - console.log('📌 使用环境变量配置(CLOUD_DB_*)'); - return getCloudEnvConfig(); - } - - console.log('⚙️ 业务数据库尚未配置,已进入系统初始化模式'); - return getBootstrapCloudConfig(); -} - -function applyCommonMiddleware(app) { - app.set('trust proxy', 1); - app.use(cors()); - app.use(express.json({ - limit: '15mb', - verify: (req, _res, buffer) => { - req.rawBody = buffer.toString('utf8'); - }, - })); -} - -function registerPublicSystemConfigRoute(app, fallbackConfig = DEFAULT_SITE_CONFIG) { - app.get('/api/admin/system-config', async (_req, res) => { - const now = Date.now(); - const cachedConfig = getPublicSystemConfigCache(now); - if (cachedConfig) { - return res.json(cachedConfig); - } - - try { - const configs = await Promise.race([ - database.local.query('SELECT config_key, config_value, config_type FROM system_config'), - new Promise((_, reject) => { - setTimeout(() => reject(new Error('public system config query timeout')), PUBLIC_SYSTEM_CONFIG_QUERY_TIMEOUT_MS); - }), - ]); - const result = {}; - filterPublicSystemConfigs(configs).forEach((config) => { - if (config.config_type === 'boolean') { - result[config.config_key] = config.config_value === 'true' || config.config_value === '1'; - } else if (config.config_type === 'number') { - result[config.config_key] = Number(config.config_value); - } else { - result[config.config_key] = config.config_value; - } - }); - setPublicSystemConfigCache(result, PUBLIC_SYSTEM_CONFIG_CACHE_TTL_MS, now); - res.json(result); - } catch (error) { - const staleConfig = getStalePublicSystemConfigCache(); - if (staleConfig) { - return res.json(staleConfig); - } - - if (serverMode === 'install' || serverMode === 'database_unavailable') { - return res.json({ - site_name: fallbackConfig.siteName, - site_description: fallbackConfig.siteDescription, - logo: '', - }); - } - - console.warn('获取系统配置超时或失败,返回兜底配置:', error.message); - return res.json({ - site_name: fallbackConfig.siteName, - site_description: fallbackConfig.siteDescription, - logo: '', - }); - } - }); -} - -function registerInstallModeRoutes(app, installStatus) { - const unavailableHandler = (_req, res) => { - res.status(503).json({ error: '系统尚未完成安装,请先访问 /install 完成初始化' }); - }; - - app.get('/health', async (_req, res) => { - const status = await getInstallStatus(); - res.json({ - status: 'install_required', - timestamp: new Date().toISOString(), - build: getBuildIdentity(), - install: status, - }); - }); - - app.use('/api/install', installRoutes); - registerPublicSystemConfigRoute(app); - - app.get('/api/auth/providers', (_req, res) => { - res.json({ - authBaseUrl: '', - dingtalkEnabled: false, - googleEnabled: false, - installRequired: true, - }); - }); - app.get('/api/auth/dingtalk/login', unavailableHandler); - app.get('/api/auth/google/login', unavailableHandler); - app.post('/api/auth/login', unavailableHandler); - app.post('/api/auth/callback', unavailableHandler); - app.get('/api/auth/verify', (_req, res) => { - res.status(401).json({ valid: false, installRequired: true }); - }); - - app.use('/api/admin', (_req, res) => { - res.status(503).json({ - error: '系统尚未完成安装,请先访问 /install 初始化系统数据库并创建管理员账号', - }); - }); - - console.log('='.repeat(50)); - console.log('🛠️ 系统已进入首次安装模式'); - console.log(`📡 服务地址: http://localhost:${process.env.PORT || 4000}`); - console.log(`🧭 安装入口: http://localhost:${process.env.PORT || 4000}/install`); - console.log(`📦 安装状态: ${installStatus.systemDatabase.reason}`); - console.log('='.repeat(50)); -} - -async function startInstallServer(installStatus) { - serverMode = 'install'; - const app = express(); - applyCommonMiddleware(app); - registerInstallModeRoutes(app, installStatus); - - const port = parseInt(process.env.PORT || '4000', 10); - await new Promise((resolve) => { - app.listen(port, resolve); - }); -} - -function registerDatabaseUnavailableRoutes(app, installStatus) { - const unavailableHandler = (_req, res) => { - res.status(503).json({ - error: '系统数据库暂时无法连接,请检查 MySQL 网络、白名单或安全组后重启服务', - databaseUnavailable: true, - installRequired: false, - }); - }; - - app.get('/health', async (_req, res) => { - const status = await getInstallStatus(); - res.status(503).json({ - status: 'database_unavailable', - timestamp: new Date().toISOString(), - build: getBuildIdentity(), - install: status, - }); - }); - - app.use('/api/install', installRoutes); - registerPublicSystemConfigRoute(app); - - app.get('/api/auth/providers', (_req, res) => { - res.json({ - authBaseUrl: '', - dingtalkEnabled: false, - googleEnabled: false, - installRequired: false, - databaseUnavailable: true, - }); - }); - app.get('/api/auth/verify', (_req, res) => { - res.status(503).json({ valid: false, databaseUnavailable: true, installRequired: false }); - }); - app.post('/api/auth/login', unavailableHandler); - app.post('/api/auth/callback', unavailableHandler); - app.get('/api/auth/dingtalk/login', unavailableHandler); - app.get('/api/auth/google/login', unavailableHandler); - app.use('/api', unavailableHandler); - - console.log('='.repeat(50)); - console.log('🚧 系统数据库暂时无法连接,已进入降级模式'); - console.log(`📡 服务地址: http://localhost:${process.env.PORT || 4000}`); - console.log(`📦 数据库状态: ${installStatus.systemDatabase.reason}`); - console.log('💡 这不是首次安装,请检查 MySQL 白名单/安全组/网络后重启服务'); - console.log('='.repeat(50)); -} - -async function startDatabaseUnavailableServer(installStatus) { - serverMode = 'database_unavailable'; - const app = express(); - applyCommonMiddleware(app); - registerDatabaseUnavailableRoutes(app, installStatus); - - const port = parseInt(process.env.PORT || '4000', 10); - await new Promise((resolve) => { - app.listen(port, resolve); - }); -} - -async function createFullModeApp(cloudConfig) { - const dbType = cloudConfig.driver === 'mssql' ? 'mssql' : 'mysql'; - console.log(`📊 数据库类型: ${dbType}`); - - const port = parseInt(process.env.PORT || '4000', 10); - - if (String(process.env.DISABLE_CUBEJS || '').trim().toLowerCase() === 'true') { - console.warn('⏸️ 已通过 DISABLE_CUBEJS=true 跳过 Cube.js,仅启动业务 API 服务'); - const app = express(); - await new Promise((resolve) => { - app.listen(port, resolve); - }); - - return { - app, - cubeMode: 'disabled', - cubeError: 'DISABLE_CUBEJS=true', - }; - } - - try { - const cubejsServer = new CubejsServer({ - schemaPath: 'model', - dbType: dbType, - logger: (msg, params) => { - console.log(`[Cube.js] ${msg}`, params !== undefined ? JSON.stringify(params) : ''); - }, - driverFactory: () => { - // 根据 driver 类型选择驱动 - if (cloudConfig.driver === 'mssql') { - const MSSQLDriver = require('@cubejs-backend/mssql-driver'); - return new MSSQLDriver({ - server: cloudConfig.host, - port: cloudConfig.port, - database: cloudConfig.database, - user: cloudConfig.user, - password: cloudConfig.password, - options: { - encrypt: cloudConfig.ssl, - trustServerCertificate: true, - }, - }); - } - - // 默认使用 MySQL - const MySQLDriver = require('@cubejs-backend/mysql-driver'); - return new MySQLDriver({ - database: cloudConfig.database, - host: cloudConfig.host, - port: cloudConfig.port, - user: cloudConfig.user, - password: cloudConfig.password, - ssl: cloudConfig.ssl, - }); - }, - apiSecret: process.env.CUBEJS_API_SECRET, - scheduledRefreshTimer: CUBE_SCHEDULED_REFRESH_TIMER, - jwt: { - key: process.env.CUBEJS_SECRET, - algorithms: ['HS256'], - }, - }); - - const { app } = await cubejsServer.listen(); - return { - app, - cubeMode: 'cube', - cubeError: null, - }; - } catch (error) { - console.warn('⚠️ Cube.js 启动失败,已降级为业务 API 服务:', error.message); - const app = express(); - await new Promise((resolve) => { - app.listen(port, resolve); - }); - - return { - app, - cubeMode: 'express-fallback', - cubeError: error.message, - }; - } -} - -// ===================== -// 启动服务 -// ===================== -async function startServer(options = {}) { - const envSkipLocalSchema = String(process.env.SKIP_LOCAL_SCHEMA_CHECK || '').trim().toLowerCase() === 'true'; - const { skipLocalSchema = envSkipLocalSchema } = options; - serverMode = 'full'; - if (skipLocalSchema) { - console.warn('⚠️ 系统数据库暂时不可用,已跳过本次本地 schema 检查,继续启动业务 API'); - } else { - await ensureLocalSchema(); - } - - if (shouldRequireImageSearchCredentials()) { - const credentialStatus = await verifyImageSearchCredentials(); - console.log(`✅ 图搜凭据守卫通过:公司=${credentialStatus.company.source}, 1688=${credentialStatus.alibaba1688.source}`); - } - - // 获取云数据库配置 - const cloudConfig = await getCloudDbConfig(); - if (cloudConfig.bootstrap) { - console.log(`📊 初始化模式占位数据源: ${cloudConfig.host}:${cloudConfig.port}/${cloudConfig.database}`); - } else { - console.log(`📊 云数据库: ${cloudConfig.host}:${cloudConfig.port}/${cloudConfig.database}`); - } - - // 启动 Cube.js 服务器;如 Cube Store 不可用,降级为普通业务 API 服务,避免整站不可用。 - const { app: cubejsApp, cubeMode, cubeError } = await createFullModeApp(cloudConfig); - - applyCommonMiddleware(cubejsApp); - - // ===================== - // 健康检查 (多数据源) - // ===================== - cubejsApp.get('/health', async (_req, res) => { - const [systemTest, businessTest, redisTest] = await Promise.all([ - withTimeout(database.local.test(), 5000, { success: false, message: '系统数据库检测超时' }), - withTimeout(cloudDb.test(), 5000, { success: false, message: '业务数据库检测超时' }), - withTimeout(redis.test(), 3000, { success: false, message: 'Redis 检测超时' }), - ]); - - res.json({ - status: 'ok', - timestamp: new Date().toISOString(), - build: getBuildIdentity(), - databases: { - system: systemTest, - local: systemTest, - cloud: businessTest, - }, - services: { - redis: redisTest, - cube: { - success: cubeMode === 'cube', - mode: cubeMode, - message: cubeMode === 'cube' ? 'Cube.js 服务正常' : `Cube.js 已降级: ${cubeError || '未知错误'}`, - }, - }, - }); - }); - - // ===================== - // 认证 API - // ===================== - cubejsApp.get('/api/auth/providers', require('./api/auth').getProviders); - cubejsApp.get('/api/auth/dingtalk/login', require('./api/auth').startDingTalkLogin); - cubejsApp.get('/api/auth/google/login', require('./api/auth').startGoogleLogin); - cubejsApp.post('/api/auth/login', require('./api/auth').login); - cubejsApp.post('/api/auth/local-preview', require('./api/auth').localPreview); - cubejsApp.post('/api/auth/callback', require('./api/auth').callback); - cubejsApp.get('/api/auth/verify', require('./api/auth').verify); - - // ===================== - // 公开访问的 API(无需认证) - // ===================== - cubejsApp.use('/api/install', installRoutes); - cubejsApp.use('/api/sso', thirdPartySsoRoutes); - cubejsApp.use('/api/desktop-client', desktopClientRoutes); - cubejsApp.use('/api/public/taobao-detail-plus', taobaoDetailPlusRoutes); - cubejsApp.use('/api/public/planning-may-product', planningMayProductPublicRoutes); - cubejsApp.use('/api/external/selection-fission/presale', presaleExternalRoutes); - cubejsApp.use('/api/external/selection-fission/delist', delistExternalRoutes); - cubejsApp.use('/api/campaign', campaignCloudRoutes); - cubejsApp.use('/api/external/selection-fission/1688-products', alibaba1688ProductExternalRoutes); - cubejsApp.use('/api/external/selection-fission', planningSelectionFissionExternalRoutes); - cubejsApp.use('/api/external/presale', presaleExternalRoutes); - - // 系统配置的 GET 接口公开访问(登录页需要获取网站名称和 Logo) - registerPublicSystemConfigRoute(cubejsApp); - - // ===================== - // 需要认证的 API - // ===================== - cubejsApp.use('/api/admin/sso-clients', authMiddleware, authMiddleware.requireRole('admin'), thirdPartySsoAdminRoutes); - cubejsApp.use('/api/admin', authMiddleware, adminRoutes); - cubejsApp.use('/api/desktop-presence', authMiddleware, desktopPresenceRoutes); - cubejsApp.use('/api/desktop-execution', desktopExecutionTaskRoutes); - cubejsApp.use('/api/product-code-board', authMiddleware, productCodeBoardRoutes); - cubejsApp.use('/api/product-analysis', authMiddleware, productAnalysisRoutes); - cubejsApp.use('/api/marketing-events', authMiddleware, marketingEventsRoutes); - cubejsApp.use('/api/promotion-module', authMiddleware, promotionModuleRoutes); - cubejsApp.use('/api/planning-upload', authMiddleware, planningUploadPageCache, planningUploadRoutes); - cubejsApp.use('/api/channel', authMiddleware, channelPageCache, channelRoutes); - cubejsApp.use('/api/product-monitoring', authMiddleware, productMonitoringRoutes); - cubejsApp.use('/api/modular-stock-monitor', authMiddleware, modularStockMonitorRoutes); - cubejsApp.use('/api/mkh-bom', authMiddleware, mkhBomRoutes); - - // ===================== - // 数据分析 API(公开访问,可根据需要添加认证) - // ===================== - cubejsApp.use('/api/analysis', analysisPageCache, analysisRoutes); - - // ===================== - // 数据刷新 API(需要认证) - // ===================== - cubejsApp.post('/api/admin/refresh', authMiddleware, async (req, res) => { - try { - const { date } = req.body; - console.log(`[API] 手动触发数据刷新: ${date || '昨天'}`); - const result = await manualRefresh(date); - res.json({ success: true, ...result }); - } catch (error) { - res.status(500).json({ success: false, error: error.message }); - } - }); - - // ===================== - // 重新加载云数据库配置 API - // ===================== - cubejsApp.post('/api/admin/database/reload', authMiddleware, async (req, res) => { - try { - console.log('🔄 正在重新加载云数据库配置...'); - res.json({ - success: true, - message: '配置已保存,需要重启服务才能生效。请执行: pm2 restart maixiang' - }); - } catch (error) { - res.status(500).json({ success: false, message: error.message }); - } - }); - - console.log('='.repeat(50)); - console.log('🚀 后端服务已启动'); - console.log(`📡 服务地址: http://localhost:${process.env.PORT || 4000}`); - console.log(cubeMode === 'cube' - ? `📊 Cube.js API: http://localhost:${process.env.PORT || 4000}/cubejs-api` - : '📊 Cube.js API: 已降级不可用,业务 API 继续运行'); - - // 测试数据库连接 - console.log('\n📦 数据库连接测试:'); - const [systemTest, businessTest, redisTest] = await Promise.all([ - withTimeout(database.local.test(), 5000, { message: '系统数据库检测超时' }), - withTimeout(cloudDb.test(), 5000, { message: '业务数据库检测超时' }), - withTimeout(redis.test(), 3000, { message: 'Redis 检测超时' }), - ]); - console.log(` [系统] ${systemTest.message}`); - console.log(` [业务] ${businessTest.message}`); - console.log(` [缓存] ${redisTest.message}`); - console.log('='.repeat(50)); - - if (String(process.env.ENABLE_BACKGROUND_JOBS || 'true').toLowerCase() === 'false') { - console.log('⏸️ 后台定时任务和预热已关闭,业务请求按需处理'); - return; - } - - startScheduler(); - startStockSelectionSchedulers(); - startTaobaoProductEmbeddingSchedulers(); - startPddSkuDimensionSnapshotSchedulers(); - startPlanningMayProductSnapshotSchedulers(); - startPlanningDingWorkflowSyncSchedulers(); - startOpsBIncentiveSnapshotSchedulers(); - startGoodsFirstSeenScheduler(); - startAlibaba1688ImageSearchExpiryScheduler(); - startMkhReplenishAlertScheduler(); - - // Keep the first selection-pool page request from paying the remote schema - // inspection round-trip over the planning database tunnel. - setTimeout(() => { - warmSelectionFissionReadSchema() - .then(() => console.log('[SelectionFission] 列表读取结构预热完成')) - .catch((error) => console.warn('[SelectionFission] 列表读取结构预热失败:', error.message)); - }, 1000); - - if (isStartupPrewarmEnabled()) { - setTimeout(() => { - if (typeof prewarmChannelHotData === 'function') { - runGuardedPrewarm('channel-hot-data', () => prewarmChannelHotData({ skipCoverage: true })).catch((error) => { - console.warn('⚠️ 渠道默认查询预热失败:', error.message); - }); - } - }, 8000); - - setTimeout(() => { - if (typeof analysisRoutes.prewarmHotAnalysisPages === 'function') { - runGuardedPrewarm('analysis-hot-pages', () => analysisRoutes.prewarmHotAnalysisPages()).catch((error) => { - console.warn('⚠️ 分析页默认查询预热失败:', error.message); - }); - } - }, 10000); - - - - if (typeof analysisRoutes.prewarmHotAnalysisPages === 'function') { - const analysisPrewarmTimer = setInterval(() => { - runGuardedPrewarm('analysis-hot-pages:interval', () => analysisRoutes.prewarmHotAnalysisPages()).catch((error) => { - console.warn('⚠️ 分析页默认查询定时预热失败:', error.message); - }); - }, 10 * 60 * 1000); - - if (typeof analysisPrewarmTimer.unref === 'function') { - analysisPrewarmTimer.unref(); - } - } - - - } else { - console.log('⏸️ 启动预热已关闭,可通过 ENABLE_STARTUP_PREWARM=true 开启'); - } - - if (!isStartupPrewarmEnabled() && typeof prewarmChannelStockMonitorDashboardDefaultSnapshot === 'function') { - if (typeof prewarmChannelIdDefaultSnapshots === 'function') { - setTimeout(() => { - runGuardedPrewarm( - 'channel-id-analysis-resilience', - () => prewarmChannelIdDefaultSnapshots({ skipCoverage: true }) - ).catch((error) => { - console.warn('⚠️ ID数据分析韧性快照预热失败:', error.message); - }); - }, CHANNEL_ID_RESILIENCE_PREWARM_DELAY_MS); - } - - if (typeof prewarmChannelIdDashboardDefaultSnapshot === 'function') { - setTimeout(() => { - runGuardedPrewarm( - 'channel-id-dashboard-resilience', - () => prewarmChannelIdDashboardDefaultSnapshot({ skipCoverage: true }) - ).catch((error) => { - console.warn('⚠️ ID数据分析仪表盘韧性快照预热失败:', error.message); - }); - }, CHANNEL_ID_RESILIENCE_PREWARM_DELAY_MS + 15000); - } - - setTimeout(() => { - runGuardedPrewarm( - 'channel-stock-monitor-resilience', - () => prewarmChannelStockMonitorDashboardDefaultSnapshot({ skipCoverage: true }) - ).catch((error) => { - console.warn('⚠️ 库存销售影响仪表盘韧性快照预热失败:', error.message); - }); - }, CHANNEL_STOCK_MONITOR_RESILIENCE_PREWARM_DELAY_MS); - } - - if (String(process.env.ENABLE_BRAND_ANALYSIS_PREWARM || 'true').toLowerCase() !== 'false' - && typeof prewarmBrandAnalysisDefault === 'function') { - setTimeout(() => { - runGuardedPrewarm('brand-analysis-default', () => prewarmBrandAnalysisDefault()) - .then((result) => { - console.log( - `[BrandAnalysis] default prewarm completed total=${result.total || 0} elapsedMs=${result.elapsedMs || 0}` - ); - }) - .catch((error) => { - console.warn('⚠️ 品牌维度默认真实数据预热失败:', error.message); - }); - }, BRAND_ANALYSIS_PREWARM_DELAY_MS); - } - - const cron = require('node-cron'); - - if (typeof prewarmChannelHotData === 'function') { - cron.schedule('0 1 * * *', async () => { - console.log('🧊 凌晨 1 点执行渠道 23 小时缓存预热...'); - await runGuardedPrewarm( - 'channel-hot-data:daily', - () => prewarmChannelHotData({ forceRefresh: true, skipCoverage: true }) - ); - }, { - timezone: 'Asia/Shanghai', - }); - console.log('[Scheduler] 渠道 23 小时缓存预热任务已注册: 每天01:00执行'); - } - - if (process.env.NODE_ENV === 'production') { - const alertChecker = require('./services/alertChecker'); - - cron.schedule('*/5 * * * *', async () => { - console.log('🔔 执行预警检查...'); - await alertChecker.checkAllRules(); - }); - } -} - -async function bootstrap() { - const installStatus = await getInstallStatus(); - if (installStatus.required) { - await startInstallServer(installStatus); - return; - } - - if (installStatus.databaseUnavailable) { - console.warn(`⚠️ 系统数据库暂时不可用,不进入降级拦截模式: ${installStatus.systemDatabase?.reason || 'unknown'}`); - await startServer({ skipLocalSchema: true }); - return; - } - - await startServer(); -} - -bootstrap().catch((error) => { - console.error('❌ 服务启动失败:', error); - process.exit(1); -}); - -async function shutdownGracefully() { - await Promise.allSettled([ - cloudDb.close(), - jianSunDb.close(), - database.close(), - redis.close(), - ]); -} - -// 优雅关闭 -process.on('SIGTERM', async () => { - console.log('正在关闭服务...'); - await shutdownGracefully(); - process.exit(0); -}); - -process.on('SIGINT', async () => { - console.log('正在关闭服务...'); - await shutdownGracefully(); - process.exit(0); -}); +// 后端服务入口 +const path = require('path'); + +require('dotenv').config(); +require('dotenv').config({ path: path.resolve(__dirname, '../.env') }); + +const express = require('express'); +const cors = require('cors'); +const { CubejsServer } = require('@cubejs-backend/server'); +const mysql = require('mysql2/promise'); + +const installRoutes = require('./api/install'); +const authMiddleware = require('./middleware/auth'); +const { createPageDataCache } = require('./middleware/pageDataCache'); +const adminRoutes = require('./api/admin'); +const planningUploadRoutes = require('./api/planningUpload'); +const planningMayProductPublicRoutes = require('./api/planningMayProductPublic'); +const planningSelectionFissionExternalRoutes = require('./api/planningSelectionFissionExternal'); +const presaleExternalRoutes = require('./api/presaleExternal'); +const delistExternalRoutes = require('./api/delistExternal'); +const campaignCloudRoutes = require('./api/campaignCloud'); +const desktopClientRoutes = require('./api/desktopClient'); +const desktopPresenceRoutes = require('./api/desktopPresence'); +const desktopExecutionTaskRoutes = require('./api/desktopExecutionTasks'); +const alibaba1688ProductExternalRoutes = require('./api/alibaba1688ProductExternal'); +const taobaoDetailPlusRoutes = require('./api/taobaoDetailPlus'); +const thirdPartySsoRoutes = require('./api/thirdPartySso'); +const thirdPartySsoAdminRoutes = require('./api/thirdPartySsoAdmin'); +const channelRoutes = require('./api/channel'); +const analysisRoutes = require('./api/analysis'); +const productCodeBoardRoutes = require('./api/productCodeBoard'); +const productAnalysisRoutes = require('./api/productAnalysis'); +const marketingEventsRoutes = require('./api/marketingEvents'); +const promotionModuleRoutes = require('./api/promotionModule'); +const productMonitoringRoutes = require('./api/productMonitoring'); +const modularStockMonitorRoutes = require('./api/modularStockMonitor'); +const mkhBomRoutes = require('./api/mkhBom'); +const dailyInventoryRoutes = require('./api/dailyInventory'); +const database = require('./config/database'); +const cloudDb = require('./config/cloudDatabase'); +const jianSunDb = require('./config/jianSunDatabase'); +const redis = require('./config/redis'); +const { startScheduler, manualRefresh } = require('./jobs/dailyRefresh'); +const { startStockSelectionSchedulers } = require('./jobs/stockSelectionSyncJob'); +const { startTaobaoProductEmbeddingSchedulers } = require('./jobs/taobaoProductEmbeddingSyncJob'); +const { startPddSkuDimensionSnapshotSchedulers } = require('./jobs/pddSkuDimensionSnapshotJob'); +const { startPlanningMayProductSnapshotSchedulers } = require('./jobs/planningMayProductSnapshotJob'); +const { startPlanningDingWorkflowSyncSchedulers } = require('./jobs/planningDingWorkflowSyncJob'); +const { startOpsBIncentiveSnapshotSchedulers } = require('./jobs/opsBIncentiveSnapshotJob'); +const { startGoodsFirstSeenScheduler } = require('./jobs/goodsFirstSeenJob'); +const { + startAlibaba1688ImageSearchExpiryScheduler, +} = require('./jobs/alibaba1688ImageSearchExpiryJob'); +const { + startMkhReplenishAlertScheduler, +} = require('./jobs/mkhReplenishAlertJob'); +const { DEFAULT_SITE_CONFIG, getInstallStatus } = require('./services/installService'); +const { ensureLocalSchema } = require('./services/localSchema'); +const { + prewarmChannelHotData, + prewarmChannelIdDefaultSnapshots, + prewarmChannelIdDashboardDefaultSnapshot, + prewarmChannelStockMonitorDashboardDefaultSnapshot, +} = require('./services/channelAnalysis'); +const { prewarmBrandAnalysisDefault } = require('./services/brandAnalysis'); +const { runHeavyDatabaseTask } = require('./services/databaseLoadGuard'); +const { warmSelectionFissionReadSchema } = require('./services/planningSelectionFissionService'); +const { + getPublicSystemConfigCache, + getStalePublicSystemConfigCache, + setPublicSystemConfigCache, +} = require('./services/publicSystemConfigCache'); +const { filterPublicSystemConfigs } = require('./services/systemConfigVisibility'); +const { + shouldRequireImageSearchCredentials, + verifyImageSearchCredentials, +} = require('./services/imageSearchCredentialGuard'); + +const PLANNING_PAGE_REDIS_TTL_SECONDS = Math.max( + Number(process.env.PLANNING_PAGE_RESPONSE_CACHE_REDIS_TTL_SECONDS) || 23 * 60 * 60, + 60 +); +const BRAND_ANALYSIS_PREWARM_DELAY_MS = Math.max( + Number(process.env.BRAND_ANALYSIS_PREWARM_DELAY_MS) || 20 * 1000, + 5 * 1000 +); +const PUBLIC_SYSTEM_CONFIG_CACHE_TTL_MS = Math.max( + Number(process.env.PUBLIC_SYSTEM_CONFIG_CACHE_TTL_SECONDS) || 300, + 30 +) * 1000; +const PUBLIC_SYSTEM_CONFIG_QUERY_TIMEOUT_MS = Math.max( + Number(process.env.PUBLIC_SYSTEM_CONFIG_QUERY_TIMEOUT_MS) || 1800, + 500 +); +const CUBE_SCHEDULED_REFRESH_TIMER = String(process.env.CUBEJS_SCHEDULED_REFRESH_TIMER || '') + .trim() + .toLowerCase() === 'true'; + +function getBuildIdentity() { + return { + sourceGitSha: String(process.env.SOURCE_GIT_SHA || '').trim() || null, + sourceGitRef: String(process.env.SOURCE_GIT_REF || '').trim() || null, + }; +} +const CHANNEL_STOCK_MONITOR_RESILIENCE_PREWARM_DELAY_MS = Math.max( + Number(process.env.CHANNEL_STOCK_MONITOR_RESILIENCE_PREWARM_DELAY_MS) || 90000, + 10000 +); +const CHANNEL_ID_RESILIENCE_PREWARM_DELAY_MS = Math.max( + Number(process.env.CHANNEL_ID_RESILIENCE_PREWARM_DELAY_MS) || 30000, + 10000 +); + +function shouldCachePlanningUploadResponse(body = {}) { + if (!body || body.error || body.success === false) { + return false; + } + + const payload = body.data || body; + if (payload?.pending || payload?.cache?.backgroundRefresh) { + return false; + } + + return true; +} + +function shouldHandlePlanningUploadPageCache(req) { + // 企划页首页会聚合 MAY 快照统计;MAY 商品页本身也依赖最新成功批次。 + // MAY 渠道洞察也依赖最新店铺/覆盖快照,外层整页缓存会把旧列口径整天顶住。 + // 榜单数据分析现在也依赖实时机会/竞争诊断结果,外层整页缓存会把旧字段结构整天顶住。 + // 抖音/小红书市场页支持实时批次切换,整页缓存会把旧批次和旧字段结构顶住。 + // 这些接口必须直接读实体表/轻缓存,不能再被 23 小时整页 Redis 顶住。 + if ( + req.path === '/' + || req.path.startsWith('/table/may_product_data') + || req.path.startsWith('/analysis/may-channel-insight') + || req.path.startsWith('/analysis/ops-b-incentive') + || req.path.startsWith('/selection-fission') + ) { + return false; + } + + return true; +} + +const analysisPageCache = createPageDataCache({ + namespace: 'analysis', + redisTtlSeconds: 300, +}); +const channelPageCache = createPageDataCache({ + namespace: 'channel', + redisTtlSeconds: 300, + // ID 分析内部已有按口径版本控制的缓存;导出接口返回 CSV,不能走整页 JSON 缓存。 + shouldHandleRequest: (req) => !['/id-analysis', '/export'].includes(req.path) + && !req.path.startsWith('/export-jobs'), +}); +const planningUploadPageCache = createPageDataCache({ + namespace: 'planning-upload', + version: 'v27-bypass-may-snapshot-pages-20260423', + redisTtlSeconds: PLANNING_PAGE_REDIS_TTL_SECONDS, + shouldHandleRequest: shouldHandlePlanningUploadPageCache, + shouldCacheBody: shouldCachePlanningUploadResponse, +}); +let serverMode = 'install'; + +function isStartupPrewarmEnabled() { + return String(process.env.ENABLE_STARTUP_PREWARM || '').toLowerCase() === 'true'; +} + +function runGuardedPrewarm(label, handler) { + return runHeavyDatabaseTask(`prewarm:${label}`, handler); +} + +function withTimeout(promise, timeoutMs, fallback) { + let timer = null; + return Promise.race([ + promise.finally(() => { + if (timer) { + clearTimeout(timer); + } + }), + new Promise((resolve) => { + timer = setTimeout(() => resolve(fallback), timeoutMs); + if (typeof timer.unref === 'function') { + timer.unref(); + } + }), + ]); +} + +function hasCompleteCloudEnvConfig() { + return cloudDb.hasCompleteEnvConfig(); +} + +function getCloudEnvConfig() { + const config = cloudDb.getEnvBusinessDatabaseConfig(); + return { + driver: config.driver || 'mssql', + host: config.host, + port: config.port || 1433, + database: config.database, + user: config.user, + password: config.password, + ssl: Boolean(config.ssl), + }; +} + +function getBootstrapCloudConfig() { + return { + driver: 'mysql', + host: process.env.LOCAL_DB_HOST, + port: parseInt(process.env.LOCAL_DB_PORT || '3306', 10), + database: process.env.LOCAL_DB_NAME, + user: process.env.LOCAL_DB_USER, + password: process.env.LOCAL_DB_PASSWORD, + ssl: false, + bootstrap: true, + }; +} + +// ===================== +// 从数据库获取云数据库配置 +// ===================== +async function getCloudDbConfig() { + let localPool; + try { + // 先连接本地数据库读取配置 + localPool = mysql.createPool({ + host: process.env.LOCAL_DB_HOST, + port: parseInt(process.env.LOCAL_DB_PORT || '3306'), + user: process.env.LOCAL_DB_USER, + password: process.env.LOCAL_DB_PASSWORD, + database: process.env.LOCAL_DB_NAME, + charset: 'utf8mb4', + waitForConnections: true, + connectionLimit: 2, + maxIdle: 2, + idleTimeout: 60000, + queueLimit: 10, + connectTimeout: 5000, + enableKeepAlive: true, + keepAliveInitialDelay: 0, + }); + + const [rows] = await localPool.query( + 'SELECT host, port, database_name, username, password, `ssl`, driver, use_env_config FROM database_configs WHERE is_default = 1 AND status = ? LIMIT 1', + ['active'] + ); + + if (rows.length > 0) { + const config = rows[0]; + if (config.use_env_config === 1 || config.use_env_config === true) { + if (hasCompleteCloudEnvConfig()) { + console.log('📌 使用 .env 业务数据库配置(CLOUD_DB_*)'); + return getCloudEnvConfig(); + } + console.warn('⚠️ 已开启 .env 业务数据库配置,但 CLOUD_DB_* 不完整,继续使用 database_configs 表配置'); + } + + console.log('📌 使用数据库配置(来自 database_configs 表)'); + return { + driver: config.driver || 'mysql', + host: config.host, + port: config.port || 3306, + database: config.database_name, + user: config.username, + password: config.password, + ssl: config.ssl === 1, + }; + } + } catch (error) { + console.warn('⚠️ 无法从数据库读取配置:', error.message); + } finally { + if (localPool) { + await localPool.end().catch((error) => { + console.warn('⚠️ 关闭系统库临时连接池失败:', error.message); + }); + } + } + + // 回退到环境变量配置 + if (hasCompleteCloudEnvConfig()) { + console.log('📌 使用环境变量配置(CLOUD_DB_*)'); + return getCloudEnvConfig(); + } + + console.log('⚙️ 业务数据库尚未配置,已进入系统初始化模式'); + return getBootstrapCloudConfig(); +} + +function applyCommonMiddleware(app) { + app.set('trust proxy', 1); + app.use(cors()); + app.use(express.json({ + limit: '15mb', + verify: (req, _res, buffer) => { + req.rawBody = buffer.toString('utf8'); + }, + })); +} + +function registerPublicSystemConfigRoute(app, fallbackConfig = DEFAULT_SITE_CONFIG) { + app.get('/api/admin/system-config', async (_req, res) => { + const now = Date.now(); + const cachedConfig = getPublicSystemConfigCache(now); + if (cachedConfig) { + return res.json(cachedConfig); + } + + try { + const configs = await Promise.race([ + database.local.query('SELECT config_key, config_value, config_type FROM system_config'), + new Promise((_, reject) => { + setTimeout(() => reject(new Error('public system config query timeout')), PUBLIC_SYSTEM_CONFIG_QUERY_TIMEOUT_MS); + }), + ]); + const result = {}; + filterPublicSystemConfigs(configs).forEach((config) => { + if (config.config_type === 'boolean') { + result[config.config_key] = config.config_value === 'true' || config.config_value === '1'; + } else if (config.config_type === 'number') { + result[config.config_key] = Number(config.config_value); + } else { + result[config.config_key] = config.config_value; + } + }); + setPublicSystemConfigCache(result, PUBLIC_SYSTEM_CONFIG_CACHE_TTL_MS, now); + res.json(result); + } catch (error) { + const staleConfig = getStalePublicSystemConfigCache(); + if (staleConfig) { + return res.json(staleConfig); + } + + if (serverMode === 'install' || serverMode === 'database_unavailable') { + return res.json({ + site_name: fallbackConfig.siteName, + site_description: fallbackConfig.siteDescription, + logo: '', + }); + } + + console.warn('获取系统配置超时或失败,返回兜底配置:', error.message); + return res.json({ + site_name: fallbackConfig.siteName, + site_description: fallbackConfig.siteDescription, + logo: '', + }); + } + }); +} + +function registerInstallModeRoutes(app, installStatus) { + const unavailableHandler = (_req, res) => { + res.status(503).json({ error: '系统尚未完成安装,请先访问 /install 完成初始化' }); + }; + + app.get('/health', async (_req, res) => { + const status = await getInstallStatus(); + res.json({ + status: 'install_required', + timestamp: new Date().toISOString(), + build: getBuildIdentity(), + install: status, + }); + }); + + app.use('/api/install', installRoutes); + registerPublicSystemConfigRoute(app); + + app.get('/api/auth/providers', (_req, res) => { + res.json({ + authBaseUrl: '', + dingtalkEnabled: false, + googleEnabled: false, + installRequired: true, + }); + }); + app.get('/api/auth/dingtalk/login', unavailableHandler); + app.get('/api/auth/google/login', unavailableHandler); + app.post('/api/auth/login', unavailableHandler); + app.post('/api/auth/callback', unavailableHandler); + app.get('/api/auth/verify', (_req, res) => { + res.status(401).json({ valid: false, installRequired: true }); + }); + + app.use('/api/admin', (_req, res) => { + res.status(503).json({ + error: '系统尚未完成安装,请先访问 /install 初始化系统数据库并创建管理员账号', + }); + }); + + console.log('='.repeat(50)); + console.log('🛠️ 系统已进入首次安装模式'); + console.log(`📡 服务地址: http://localhost:${process.env.PORT || 4000}`); + console.log(`🧭 安装入口: http://localhost:${process.env.PORT || 4000}/install`); + console.log(`📦 安装状态: ${installStatus.systemDatabase.reason}`); + console.log('='.repeat(50)); +} + +async function startInstallServer(installStatus) { + serverMode = 'install'; + const app = express(); + applyCommonMiddleware(app); + registerInstallModeRoutes(app, installStatus); + + const port = parseInt(process.env.PORT || '4000', 10); + await new Promise((resolve) => { + app.listen(port, resolve); + }); +} + +function registerDatabaseUnavailableRoutes(app, installStatus) { + const unavailableHandler = (_req, res) => { + res.status(503).json({ + error: '系统数据库暂时无法连接,请检查 MySQL 网络、白名单或安全组后重启服务', + databaseUnavailable: true, + installRequired: false, + }); + }; + + app.get('/health', async (_req, res) => { + const status = await getInstallStatus(); + res.status(503).json({ + status: 'database_unavailable', + timestamp: new Date().toISOString(), + build: getBuildIdentity(), + install: status, + }); + }); + + app.use('/api/install', installRoutes); + registerPublicSystemConfigRoute(app); + + app.get('/api/auth/providers', (_req, res) => { + res.json({ + authBaseUrl: '', + dingtalkEnabled: false, + googleEnabled: false, + installRequired: false, + databaseUnavailable: true, + }); + }); + app.get('/api/auth/verify', (_req, res) => { + res.status(503).json({ valid: false, databaseUnavailable: true, installRequired: false }); + }); + app.post('/api/auth/login', unavailableHandler); + app.post('/api/auth/callback', unavailableHandler); + app.get('/api/auth/dingtalk/login', unavailableHandler); + app.get('/api/auth/google/login', unavailableHandler); + app.use('/api', unavailableHandler); + + console.log('='.repeat(50)); + console.log('🚧 系统数据库暂时无法连接,已进入降级模式'); + console.log(`📡 服务地址: http://localhost:${process.env.PORT || 4000}`); + console.log(`📦 数据库状态: ${installStatus.systemDatabase.reason}`); + console.log('💡 这不是首次安装,请检查 MySQL 白名单/安全组/网络后重启服务'); + console.log('='.repeat(50)); +} + +async function startDatabaseUnavailableServer(installStatus) { + serverMode = 'database_unavailable'; + const app = express(); + applyCommonMiddleware(app); + registerDatabaseUnavailableRoutes(app, installStatus); + + const port = parseInt(process.env.PORT || '4000', 10); + await new Promise((resolve) => { + app.listen(port, resolve); + }); +} + +async function createFullModeApp(cloudConfig) { + const dbType = cloudConfig.driver === 'mssql' ? 'mssql' : 'mysql'; + console.log(`📊 数据库类型: ${dbType}`); + + const port = parseInt(process.env.PORT || '4000', 10); + + if (String(process.env.DISABLE_CUBEJS || '').trim().toLowerCase() === 'true') { + console.warn('⏸️ 已通过 DISABLE_CUBEJS=true 跳过 Cube.js,仅启动业务 API 服务'); + const app = express(); + await new Promise((resolve) => { + app.listen(port, resolve); + }); + + return { + app, + cubeMode: 'disabled', + cubeError: 'DISABLE_CUBEJS=true', + }; + } + + try { + const cubejsServer = new CubejsServer({ + schemaPath: 'model', + dbType: dbType, + logger: (msg, params) => { + console.log(`[Cube.js] ${msg}`, params !== undefined ? JSON.stringify(params) : ''); + }, + driverFactory: () => { + // 根据 driver 类型选择驱动 + if (cloudConfig.driver === 'mssql') { + const MSSQLDriver = require('@cubejs-backend/mssql-driver'); + return new MSSQLDriver({ + server: cloudConfig.host, + port: cloudConfig.port, + database: cloudConfig.database, + user: cloudConfig.user, + password: cloudConfig.password, + options: { + encrypt: cloudConfig.ssl, + trustServerCertificate: true, + }, + }); + } + + // 默认使用 MySQL + const MySQLDriver = require('@cubejs-backend/mysql-driver'); + return new MySQLDriver({ + database: cloudConfig.database, + host: cloudConfig.host, + port: cloudConfig.port, + user: cloudConfig.user, + password: cloudConfig.password, + ssl: cloudConfig.ssl, + }); + }, + apiSecret: process.env.CUBEJS_API_SECRET, + scheduledRefreshTimer: CUBE_SCHEDULED_REFRESH_TIMER, + jwt: { + key: process.env.CUBEJS_SECRET, + algorithms: ['HS256'], + }, + }); + + const { app } = await cubejsServer.listen(); + return { + app, + cubeMode: 'cube', + cubeError: null, + }; + } catch (error) { + console.warn('⚠️ Cube.js 启动失败,已降级为业务 API 服务:', error.message); + const app = express(); + await new Promise((resolve) => { + app.listen(port, resolve); + }); + + return { + app, + cubeMode: 'express-fallback', + cubeError: error.message, + }; + } +} + +// ===================== +// 启动服务 +// ===================== +async function startServer(options = {}) { + const envSkipLocalSchema = String(process.env.SKIP_LOCAL_SCHEMA_CHECK || '').trim().toLowerCase() === 'true'; + const { skipLocalSchema = envSkipLocalSchema } = options; + serverMode = 'full'; + if (skipLocalSchema) { + console.warn('⚠️ 系统数据库暂时不可用,已跳过本次本地 schema 检查,继续启动业务 API'); + } else { + await ensureLocalSchema(); + } + + if (shouldRequireImageSearchCredentials()) { + const credentialStatus = await verifyImageSearchCredentials(); + console.log(`✅ 图搜凭据守卫通过:公司=${credentialStatus.company.source}, 1688=${credentialStatus.alibaba1688.source}`); + } + + // 获取云数据库配置 + const cloudConfig = await getCloudDbConfig(); + if (cloudConfig.bootstrap) { + console.log(`📊 初始化模式占位数据源: ${cloudConfig.host}:${cloudConfig.port}/${cloudConfig.database}`); + } else { + console.log(`📊 云数据库: ${cloudConfig.host}:${cloudConfig.port}/${cloudConfig.database}`); + } + + // 启动 Cube.js 服务器;如 Cube Store 不可用,降级为普通业务 API 服务,避免整站不可用。 + const { app: cubejsApp, cubeMode, cubeError } = await createFullModeApp(cloudConfig); + + applyCommonMiddleware(cubejsApp); + + // ===================== + // 健康检查 (多数据源) + // ===================== + cubejsApp.get('/health', async (_req, res) => { + const [systemTest, businessTest, redisTest] = await Promise.all([ + withTimeout(database.local.test(), 5000, { success: false, message: '系统数据库检测超时' }), + withTimeout(cloudDb.test(), 5000, { success: false, message: '业务数据库检测超时' }), + withTimeout(redis.test(), 3000, { success: false, message: 'Redis 检测超时' }), + ]); + + res.json({ + status: 'ok', + timestamp: new Date().toISOString(), + build: getBuildIdentity(), + databases: { + system: systemTest, + local: systemTest, + cloud: businessTest, + }, + services: { + redis: redisTest, + cube: { + success: cubeMode === 'cube', + mode: cubeMode, + message: cubeMode === 'cube' ? 'Cube.js 服务正常' : `Cube.js 已降级: ${cubeError || '未知错误'}`, + }, + }, + }); + }); + + // ===================== + // 认证 API + // ===================== + cubejsApp.get('/api/auth/providers', require('./api/auth').getProviders); + cubejsApp.get('/api/auth/dingtalk/login', require('./api/auth').startDingTalkLogin); + cubejsApp.get('/api/auth/google/login', require('./api/auth').startGoogleLogin); + cubejsApp.post('/api/auth/login', require('./api/auth').login); + cubejsApp.post('/api/auth/local-preview', require('./api/auth').localPreview); + cubejsApp.post('/api/auth/callback', require('./api/auth').callback); + cubejsApp.get('/api/auth/verify', require('./api/auth').verify); + + // ===================== + // 公开访问的 API(无需认证) + // ===================== + cubejsApp.use('/api/install', installRoutes); + cubejsApp.use('/api/sso', thirdPartySsoRoutes); + cubejsApp.use('/api/desktop-client', desktopClientRoutes); + cubejsApp.use('/api/public/taobao-detail-plus', taobaoDetailPlusRoutes); + cubejsApp.use('/api/public/planning-may-product', planningMayProductPublicRoutes); + cubejsApp.use('/api/external/selection-fission/presale', presaleExternalRoutes); + cubejsApp.use('/api/external/selection-fission/delist', delistExternalRoutes); + cubejsApp.use('/api/campaign', campaignCloudRoutes); + cubejsApp.use('/api/external/selection-fission/1688-products', alibaba1688ProductExternalRoutes); + cubejsApp.use('/api/external/selection-fission', planningSelectionFissionExternalRoutes); + cubejsApp.use('/api/external/presale', presaleExternalRoutes); + + // 系统配置的 GET 接口公开访问(登录页需要获取网站名称和 Logo) + registerPublicSystemConfigRoute(cubejsApp); + + // ===================== + // 需要认证的 API + // ===================== + cubejsApp.use('/api/admin/sso-clients', authMiddleware, authMiddleware.requireRole('admin'), thirdPartySsoAdminRoutes); + cubejsApp.use('/api/admin', authMiddleware, adminRoutes); + cubejsApp.use('/api/desktop-presence', authMiddleware, desktopPresenceRoutes); + cubejsApp.use('/api/desktop-execution', desktopExecutionTaskRoutes); + cubejsApp.use('/api/product-code-board', authMiddleware, productCodeBoardRoutes); + cubejsApp.use('/api/product-analysis', authMiddleware, productAnalysisRoutes); + cubejsApp.use('/api/marketing-events', authMiddleware, marketingEventsRoutes); + cubejsApp.use('/api/promotion-module', authMiddleware, promotionModuleRoutes); + cubejsApp.use('/api/planning-upload', authMiddleware, planningUploadPageCache, planningUploadRoutes); + cubejsApp.use('/api/channel', authMiddleware, channelPageCache, channelRoutes); + cubejsApp.use('/api/product-monitoring', authMiddleware, productMonitoringRoutes); + cubejsApp.use('/api/modular-stock-monitor', authMiddleware, modularStockMonitorRoutes); + cubejsApp.use('/api/daily-inventory', authMiddleware, dailyInventoryRoutes); + cubejsApp.use('/api/mkh-bom', authMiddleware, mkhBomRoutes); + + // ===================== + // 数据分析 API(公开访问,可根据需要添加认证) + // ===================== + cubejsApp.use('/api/analysis', analysisPageCache, analysisRoutes); + + // ===================== + // 数据刷新 API(需要认证) + // ===================== + cubejsApp.post('/api/admin/refresh', authMiddleware, async (req, res) => { + try { + const { date } = req.body; + console.log(`[API] 手动触发数据刷新: ${date || '昨天'}`); + const result = await manualRefresh(date); + res.json({ success: true, ...result }); + } catch (error) { + res.status(500).json({ success: false, error: error.message }); + } + }); + + // ===================== + // 重新加载云数据库配置 API + // ===================== + cubejsApp.post('/api/admin/database/reload', authMiddleware, async (req, res) => { + try { + console.log('🔄 正在重新加载云数据库配置...'); + res.json({ + success: true, + message: '配置已保存,需要重启服务才能生效。请执行: pm2 restart maixiang' + }); + } catch (error) { + res.status(500).json({ success: false, message: error.message }); + } + }); + + console.log('='.repeat(50)); + console.log('🚀 后端服务已启动'); + console.log(`📡 服务地址: http://localhost:${process.env.PORT || 4000}`); + console.log(cubeMode === 'cube' + ? `📊 Cube.js API: http://localhost:${process.env.PORT || 4000}/cubejs-api` + : '📊 Cube.js API: 已降级不可用,业务 API 继续运行'); + + // 测试数据库连接 + console.log('\n📦 数据库连接测试:'); + const [systemTest, businessTest, redisTest] = await Promise.all([ + withTimeout(database.local.test(), 5000, { message: '系统数据库检测超时' }), + withTimeout(cloudDb.test(), 5000, { message: '业务数据库检测超时' }), + withTimeout(redis.test(), 3000, { message: 'Redis 检测超时' }), + ]); + console.log(` [系统] ${systemTest.message}`); + console.log(` [业务] ${businessTest.message}`); + console.log(` [缓存] ${redisTest.message}`); + console.log('='.repeat(50)); + + if (String(process.env.ENABLE_BACKGROUND_JOBS || 'true').toLowerCase() === 'false') { + console.log('⏸️ 后台定时任务和预热已关闭,业务请求按需处理'); + return; + } + + startScheduler(); + startStockSelectionSchedulers(); + startTaobaoProductEmbeddingSchedulers(); + startPddSkuDimensionSnapshotSchedulers(); + startPlanningMayProductSnapshotSchedulers(); + startPlanningDingWorkflowSyncSchedulers(); + startOpsBIncentiveSnapshotSchedulers(); + startGoodsFirstSeenScheduler(); + startAlibaba1688ImageSearchExpiryScheduler(); + startMkhReplenishAlertScheduler(); + + // Keep the first selection-pool page request from paying the remote schema + // inspection round-trip over the planning database tunnel. + setTimeout(() => { + warmSelectionFissionReadSchema() + .then(() => console.log('[SelectionFission] 列表读取结构预热完成')) + .catch((error) => console.warn('[SelectionFission] 列表读取结构预热失败:', error.message)); + }, 1000); + + if (isStartupPrewarmEnabled()) { + setTimeout(() => { + if (typeof prewarmChannelHotData === 'function') { + runGuardedPrewarm('channel-hot-data', () => prewarmChannelHotData({ skipCoverage: true })).catch((error) => { + console.warn('⚠️ 渠道默认查询预热失败:', error.message); + }); + } + }, 8000); + + setTimeout(() => { + if (typeof analysisRoutes.prewarmHotAnalysisPages === 'function') { + runGuardedPrewarm('analysis-hot-pages', () => analysisRoutes.prewarmHotAnalysisPages()).catch((error) => { + console.warn('⚠️ 分析页默认查询预热失败:', error.message); + }); + } + }, 10000); + + + + if (typeof analysisRoutes.prewarmHotAnalysisPages === 'function') { + const analysisPrewarmTimer = setInterval(() => { + runGuardedPrewarm('analysis-hot-pages:interval', () => analysisRoutes.prewarmHotAnalysisPages()).catch((error) => { + console.warn('⚠️ 分析页默认查询定时预热失败:', error.message); + }); + }, 10 * 60 * 1000); + + if (typeof analysisPrewarmTimer.unref === 'function') { + analysisPrewarmTimer.unref(); + } + } + + + } else { + console.log('⏸️ 启动预热已关闭,可通过 ENABLE_STARTUP_PREWARM=true 开启'); + } + + if (!isStartupPrewarmEnabled() && typeof prewarmChannelStockMonitorDashboardDefaultSnapshot === 'function') { + if (typeof prewarmChannelIdDefaultSnapshots === 'function') { + setTimeout(() => { + runGuardedPrewarm( + 'channel-id-analysis-resilience', + () => prewarmChannelIdDefaultSnapshots({ skipCoverage: true }) + ).catch((error) => { + console.warn('⚠️ ID数据分析韧性快照预热失败:', error.message); + }); + }, CHANNEL_ID_RESILIENCE_PREWARM_DELAY_MS); + } + + if (typeof prewarmChannelIdDashboardDefaultSnapshot === 'function') { + setTimeout(() => { + runGuardedPrewarm( + 'channel-id-dashboard-resilience', + () => prewarmChannelIdDashboardDefaultSnapshot({ skipCoverage: true }) + ).catch((error) => { + console.warn('⚠️ ID数据分析仪表盘韧性快照预热失败:', error.message); + }); + }, CHANNEL_ID_RESILIENCE_PREWARM_DELAY_MS + 15000); + } + + setTimeout(() => { + runGuardedPrewarm( + 'channel-stock-monitor-resilience', + () => prewarmChannelStockMonitorDashboardDefaultSnapshot({ skipCoverage: true }) + ).catch((error) => { + console.warn('⚠️ 库存销售影响仪表盘韧性快照预热失败:', error.message); + }); + }, CHANNEL_STOCK_MONITOR_RESILIENCE_PREWARM_DELAY_MS); + } + + if (String(process.env.ENABLE_BRAND_ANALYSIS_PREWARM || 'true').toLowerCase() !== 'false' + && typeof prewarmBrandAnalysisDefault === 'function') { + setTimeout(() => { + runGuardedPrewarm('brand-analysis-default', () => prewarmBrandAnalysisDefault()) + .then((result) => { + console.log( + `[BrandAnalysis] default prewarm completed total=${result.total || 0} elapsedMs=${result.elapsedMs || 0}` + ); + }) + .catch((error) => { + console.warn('⚠️ 品牌维度默认真实数据预热失败:', error.message); + }); + }, BRAND_ANALYSIS_PREWARM_DELAY_MS); + } + + const cron = require('node-cron'); + + if (typeof prewarmChannelHotData === 'function') { + cron.schedule('0 1 * * *', async () => { + console.log('🧊 凌晨 1 点执行渠道 23 小时缓存预热...'); + await runGuardedPrewarm( + 'channel-hot-data:daily', + () => prewarmChannelHotData({ forceRefresh: true, skipCoverage: true }) + ); + }, { + timezone: 'Asia/Shanghai', + }); + console.log('[Scheduler] 渠道 23 小时缓存预热任务已注册: 每天01:00执行'); + } + + if (process.env.NODE_ENV === 'production') { + const alertChecker = require('./services/alertChecker'); + + cron.schedule('*/5 * * * *', async () => { + console.log('🔔 执行预警检查...'); + await alertChecker.checkAllRules(); + }); + } +} + +async function bootstrap() { + const installStatus = await getInstallStatus(); + if (installStatus.required) { + await startInstallServer(installStatus); + return; + } + + if (installStatus.databaseUnavailable) { + console.warn(`⚠️ 系统数据库暂时不可用,不进入降级拦截模式: ${installStatus.systemDatabase?.reason || 'unknown'}`); + await startServer({ skipLocalSchema: true }); + return; + } + + await startServer(); +} + +bootstrap().catch((error) => { + console.error('❌ 服务启动失败:', error); + process.exit(1); +}); + +async function shutdownGracefully() { + await Promise.allSettled([ + cloudDb.close(), + jianSunDb.close(), + database.close(), + redis.close(), + ]); +} + +// 优雅关闭 +process.on('SIGTERM', async () => { + console.log('正在关闭服务...'); + await shutdownGracefully(); + process.exit(0); +}); + +process.on('SIGINT', async () => { + console.log('正在关闭服务...'); + await shutdownGracefully(); + process.exit(0); +});