2469 lines
84 KiB
JavaScript
2469 lines
84 KiB
JavaScript
'use strict';
|
||
|
||
const crypto = require('crypto');
|
||
const fs = require('fs/promises');
|
||
const path = require('path');
|
||
const dayjs = require('dayjs');
|
||
const { chromium } = require('playwright');
|
||
const { v4: uuidv4 } = require('uuid');
|
||
|
||
const planningDatabase = require('./planningDatabase');
|
||
const {
|
||
ensurePlanningCanonicalSchemas,
|
||
invalidatePlanningRuntimeCache,
|
||
} = require('./planningUploadService');
|
||
|
||
const TARGET_BASE_URL = 'https://hsds.ecbis.cn';
|
||
const HSDS_API_BASE_URL = 'https://biplugs.ecbis.cn/prod';
|
||
const LOGIN_URL = `${TARGET_BASE_URL}/login`;
|
||
const STORAGE_STATE_PATH = path.resolve(__dirname, '../../runtime/hsds/storage-state/market-insight.json');
|
||
const AES_KEY = 'xishuiwangluoMDF';
|
||
|
||
const TABLE_NAME = 'planning_category_demand_indices';
|
||
const CHECKPOINT_TABLE_NAME = 'planning_category_demand_indices_sync_checkpoints';
|
||
const HOT_SEARCH_WORD_TABLE_NAME = 'planning_category_hot_search_words';
|
||
const RISING_WORD_TABLE_NAME = 'planning_category_rising_words';
|
||
const DEMAND_PRODUCT_STYLE_TABLE_NAME = 'planning_demand_product_styles';
|
||
const PLATFORM = 'TX';
|
||
const DEFAULT_PRIMARY_CATEGORY_NAME = '住宅家具';
|
||
const DEFAULT_CATEGORY_LIMIT = 0;
|
||
const DEFAULT_HISTORY_DAYS = 45;
|
||
const DEFAULT_REQUEST_TIMEOUT_MS = 30000;
|
||
const DEFAULT_REQUEST_DELAY_MS = 3500;
|
||
const DEFAULT_REQUEST_MAX_ATTEMPTS = 8;
|
||
const DEFAULT_REQUEST_RETRY_DELAY_MS = 15000;
|
||
const MAX_INSERT_CHUNK_SIZE = 500;
|
||
|
||
const SOURCE_FILE_NAME = 'sync_hsds_category_demand_indices.js';
|
||
const SOURCE_SHEET_TYPE = 'category_demand_indices';
|
||
const KEYWORD_SNAPSHOT_SOURCE_FILE_NAME = 'sync_hsds_category_keyword_snapshots.js';
|
||
const UPLOADED_BY_USERNAME = 'system';
|
||
const UPLOADED_BY_NAME = '系统同步';
|
||
|
||
const DEMAND_INDEX_TABLE_COLUMNS = Object.freeze([
|
||
{ name: 'upload_batch_id', sql: 'VARCHAR(36) NOT NULL' },
|
||
{ name: 'source_file_name', sql: 'VARCHAR(255) NOT NULL' },
|
||
{ name: 'source_sheet_name', sql: 'VARCHAR(100) NOT NULL' },
|
||
{ name: 'source_sheet_type', sql: 'VARCHAR(64) NOT NULL' },
|
||
{ name: 'source_row_number', sql: 'INT NOT NULL' },
|
||
{ name: 'uploaded_by_user_id', sql: 'BIGINT NULL' },
|
||
{ name: 'uploaded_by_username', sql: 'VARCHAR(100) NULL' },
|
||
{ name: 'uploaded_by_name', sql: 'VARCHAR(100) NULL' },
|
||
{ name: 'uploaded_department_id', sql: 'BIGINT NULL' },
|
||
{ name: 'uploaded_department_name', sql: 'VARCHAR(100) NULL' },
|
||
{ name: 'raw_data', sql: 'JSON NULL' },
|
||
{ name: 'row_fingerprint', sql: "CHAR(64) NULL COMMENT '企划模板内容指纹'" },
|
||
{ name: 'stat_date_key', sql: "TEXT NULL COMMENT '来源表头: 日期'" },
|
||
{ name: 'platform', sql: "TEXT NULL COMMENT '来源表头: 平台'" },
|
||
{ name: 'primary_category_name', sql: "TEXT NULL COMMENT '来源表头: 一级品类'" },
|
||
{ name: 'secondary_category_name', sql: "TEXT NULL COMMENT '来源表头: 二级品类'" },
|
||
{ name: 'tertiary_category_name', sql: "TEXT NULL COMMENT '来源表头: 三级品类'" },
|
||
{ name: 'market_category_name', sql: "TEXT NULL COMMENT '来源表头: 分类名称'" },
|
||
{ name: 'search_keyword', sql: "TEXT NULL COMMENT '来源表头: 搜索词'" },
|
||
{ name: 'root_word', sql: "TEXT NULL COMMENT '来源表头: 词根'" },
|
||
{ name: 'search_index', sql: "TEXT NULL COMMENT '来源表头: 搜索指数'" },
|
||
{ name: 'click_index', sql: "TEXT NULL COMMENT '来源表头: 点击指数'" },
|
||
{ name: 'pay_index', sql: "TEXT NULL COMMENT '来源表头: 支付指数'" },
|
||
{ name: 'click_rate', sql: "TEXT NULL COMMENT '来源表头: 点击率'" },
|
||
{ name: 'pay_conversion_rate', sql: "TEXT NULL COMMENT '来源表头: 支付转化率'" },
|
||
{ name: 'search_index_mom', sql: "TEXT NULL COMMENT '来源表头: 搜索指数环比'" },
|
||
{ name: 'click_index_mom', sql: "TEXT NULL COMMENT '来源表头: 点击指数环比'" },
|
||
{ name: 'pay_index_mom', sql: "TEXT NULL COMMENT '来源表头: 支付指数环比'" },
|
||
{ name: 'click_rate_mom', sql: "TEXT NULL COMMENT '来源表头: 点击率环比'" },
|
||
{ name: 'pay_conversion_rate_mom', sql: "TEXT NULL COMMENT '来源表头: 支付转化率环比'" },
|
||
]);
|
||
|
||
const DEMAND_INDEX_TABLE_INDEXES = Object.freeze([
|
||
{ name: 'idx_batch_id', sql: 'ADD KEY idx_batch_id (`upload_batch_id`)' },
|
||
{ name: 'idx_sheet_type', sql: 'ADD KEY idx_sheet_type (`source_sheet_type`)' },
|
||
{ name: 'idx_row_fingerprint', sql: 'ADD KEY idx_row_fingerprint (`row_fingerprint`)' },
|
||
{ name: 'idx_created_at', sql: 'ADD KEY idx_created_at (`created_at`)' },
|
||
{
|
||
name: 'idx_category_date',
|
||
sql: 'ADD KEY idx_category_date (`platform`(16), `primary_category_name`(100), `secondary_category_name`(100), `tertiary_category_name`(100), `stat_date_key`(8))',
|
||
},
|
||
{
|
||
name: 'idx_keyword_date',
|
||
sql: 'ADD KEY idx_keyword_date (`search_keyword`(191), `stat_date_key`(8))',
|
||
},
|
||
]);
|
||
|
||
let demandIndexSchemaReady = false;
|
||
let demandIndexCheckpointSchemaReady = false;
|
||
let demandProductStyleSchemaReady = false;
|
||
|
||
function normalizeText(value, fallback = '') {
|
||
const text = value === undefined || value === null ? '' : String(value).trim();
|
||
return text || fallback;
|
||
}
|
||
|
||
function normalizeNullableText(value) {
|
||
const text = normalizeText(value);
|
||
if (!text || text === '--' || text.toLowerCase() === 'null') {
|
||
return null;
|
||
}
|
||
return text;
|
||
}
|
||
|
||
function normalizeDateKey(value) {
|
||
const text = normalizeText(value).replace(/[^0-9]/g, '').slice(0, 8);
|
||
return text.length === 8 ? text : '';
|
||
}
|
||
|
||
function normalizeMetricText(value) {
|
||
const text = normalizeNullableText(value);
|
||
return text ? text.replace(/,/g, '') : null;
|
||
}
|
||
|
||
function normalizeBooleanEnv(value, fallback) {
|
||
if (value === undefined || value === null || String(value).trim() === '') {
|
||
return fallback;
|
||
}
|
||
const normalized = String(value).trim().toLowerCase();
|
||
if (['1', 'true', 'yes', 'on'].includes(normalized)) {
|
||
return true;
|
||
}
|
||
if (['0', 'false', 'no', 'off'].includes(normalized)) {
|
||
return false;
|
||
}
|
||
return fallback;
|
||
}
|
||
|
||
function normalizeInteger(value, fallback = 0) {
|
||
const numeric = Number.parseInt(value, 10);
|
||
return Number.isFinite(numeric) ? numeric : fallback;
|
||
}
|
||
|
||
function normalizeLeafFlag(value) {
|
||
if (value === true || value === 1) return true;
|
||
const text = normalizeText(value).toUpperCase();
|
||
return text === 'Y' || text === '1' || text === 'TRUE';
|
||
}
|
||
|
||
function escapeIdentifier(value) {
|
||
return `\`${String(value).replace(/`/g, '``')}\``;
|
||
}
|
||
|
||
async function getExistingTableColumns(connection, tableName) {
|
||
const [rows] = await connection.query(
|
||
`SELECT column_name AS columnName
|
||
FROM information_schema.columns
|
||
WHERE table_schema = DATABASE()
|
||
AND table_name = ?`,
|
||
[tableName]
|
||
);
|
||
return new Set(rows.map((row) => normalizeText(row.columnName)).filter(Boolean));
|
||
}
|
||
|
||
async function getExistingTableIndexes(connection, tableName) {
|
||
const [rows] = await connection.query(
|
||
`SELECT DISTINCT index_name AS indexName
|
||
FROM information_schema.statistics
|
||
WHERE table_schema = DATABASE()
|
||
AND table_name = ?`,
|
||
[tableName]
|
||
);
|
||
return new Set(rows.map((row) => normalizeText(row.indexName)).filter(Boolean));
|
||
}
|
||
|
||
async function ensureDemandIndexTableSchema() {
|
||
if (demandIndexSchemaReady) {
|
||
return;
|
||
}
|
||
|
||
await planningDatabase.withTransaction(async (connection) => {
|
||
await connection.query(
|
||
`CREATE TABLE IF NOT EXISTS ${escapeIdentifier(TABLE_NAME)} (
|
||
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||
${DEMAND_INDEX_TABLE_COLUMNS.map((column) => `${escapeIdentifier(column.name)} ${column.sql}`).join(',\n ')},
|
||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||
PRIMARY KEY (id),
|
||
KEY idx_batch_id (${escapeIdentifier('upload_batch_id')}),
|
||
KEY idx_sheet_type (${escapeIdentifier('source_sheet_type')}),
|
||
KEY idx_row_fingerprint (${escapeIdentifier('row_fingerprint')}),
|
||
KEY idx_created_at (${escapeIdentifier('created_at')}),
|
||
KEY idx_category_date (${escapeIdentifier('platform')}(16), ${escapeIdentifier('primary_category_name')}(100), ${escapeIdentifier('secondary_category_name')}(100), ${escapeIdentifier('tertiary_category_name')}(100), ${escapeIdentifier('stat_date_key')}(8)),
|
||
KEY idx_keyword_date (${escapeIdentifier('search_keyword')}(191), ${escapeIdentifier('stat_date_key')}(8))
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='企划模板原始数据：品类需求指数'`
|
||
);
|
||
|
||
const existingColumns = await getExistingTableColumns(connection, TABLE_NAME);
|
||
for (const column of DEMAND_INDEX_TABLE_COLUMNS) {
|
||
if (existingColumns.has(column.name)) {
|
||
continue;
|
||
}
|
||
await connection.query(
|
||
`ALTER TABLE ${escapeIdentifier(TABLE_NAME)}
|
||
ADD COLUMN ${escapeIdentifier(column.name)} ${column.sql}`
|
||
);
|
||
}
|
||
|
||
if (!existingColumns.has('created_at')) {
|
||
await connection.query(
|
||
`ALTER TABLE ${escapeIdentifier(TABLE_NAME)}
|
||
ADD COLUMN ${escapeIdentifier('created_at')} DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP`
|
||
);
|
||
}
|
||
|
||
const existingIndexes = await getExistingTableIndexes(connection, TABLE_NAME);
|
||
for (const index of DEMAND_INDEX_TABLE_INDEXES) {
|
||
if (existingIndexes.has(index.name)) {
|
||
continue;
|
||
}
|
||
await connection.query(
|
||
`ALTER TABLE ${escapeIdentifier(TABLE_NAME)}
|
||
${index.sql}`
|
||
);
|
||
}
|
||
});
|
||
|
||
demandIndexSchemaReady = true;
|
||
}
|
||
|
||
async function ensureDemandIndexCheckpointSchema() {
|
||
if (demandIndexCheckpointSchemaReady) {
|
||
return;
|
||
}
|
||
|
||
await planningDatabase.withTransaction(async (connection) => {
|
||
await connection.query(
|
||
`CREATE TABLE IF NOT EXISTS ${escapeIdentifier(CHECKPOINT_TABLE_NAME)} (
|
||
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||
platform VARCHAR(16) NOT NULL,
|
||
primary_category_name VARCHAR(100) NOT NULL,
|
||
secondary_category_name VARCHAR(150) NOT NULL DEFAULT '',
|
||
tertiary_category_name VARCHAR(150) NOT NULL DEFAULT '',
|
||
stat_date_key CHAR(8) NOT NULL,
|
||
history_start_date_key CHAR(8) NOT NULL,
|
||
history_end_date_key CHAR(8) NOT NULL,
|
||
history_days INT NOT NULL DEFAULT 45,
|
||
status ENUM('running', 'success', 'failed', 'skipped') NOT NULL DEFAULT 'running',
|
||
row_count INT NOT NULL DEFAULT 0,
|
||
source_batch_id VARCHAR(36) NULL,
|
||
note VARCHAR(500) NULL,
|
||
fetched_at DATETIME NULL,
|
||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||
PRIMARY KEY (id),
|
||
UNIQUE KEY uk_demand_checkpoint_scope (
|
||
platform,
|
||
primary_category_name,
|
||
secondary_category_name,
|
||
tertiary_category_name,
|
||
stat_date_key,
|
||
history_start_date_key,
|
||
history_end_date_key
|
||
),
|
||
KEY idx_demand_checkpoint_status (status, stat_date_key),
|
||
KEY idx_demand_checkpoint_updated_at (updated_at)
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='HSDS 需求指数同步检查点'`
|
||
);
|
||
});
|
||
|
||
demandIndexCheckpointSchemaReady = true;
|
||
}
|
||
|
||
async function ensureDemandProductStyleTableSchema() {
|
||
if (demandProductStyleSchemaReady) {
|
||
return;
|
||
}
|
||
|
||
await planningDatabase.withTransaction(async (connection) => {
|
||
await connection.query(
|
||
`CREATE TABLE IF NOT EXISTS ${escapeIdentifier(DEMAND_PRODUCT_STYLE_TABLE_NAME)} (
|
||
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||
platform VARCHAR(16) NOT NULL DEFAULT 'TX',
|
||
primary_category_name VARCHAR(100) NOT NULL,
|
||
secondary_category_name VARCHAR(150) NOT NULL DEFAULT '',
|
||
tertiary_category_name VARCHAR(150) NOT NULL DEFAULT '',
|
||
market_category_name VARCHAR(500) NOT NULL,
|
||
cate_id VARCHAR(64) NOT NULL DEFAULT '',
|
||
root_word VARCHAR(191) NOT NULL,
|
||
month_key CHAR(7) NOT NULL,
|
||
dates_type VARCHAR(32) NOT NULL,
|
||
rank_position INT NOT NULL DEFAULT 0,
|
||
goods_id VARCHAR(100) NOT NULL DEFAULT '',
|
||
goods_title VARCHAR(500) NOT NULL DEFAULT '',
|
||
image_url TEXT NULL,
|
||
shop_name VARCHAR(255) NULL,
|
||
brand_name VARCHAR(255) NULL,
|
||
sales_amount VARCHAR(64) NULL,
|
||
sale_count VARCHAR(64) NULL,
|
||
unit_price VARCHAR(64) NULL,
|
||
visitor_count VARCHAR(64) NULL,
|
||
conversion_rate VARCHAR(64) NULL,
|
||
detail_url TEXT NULL,
|
||
raw_data JSON NULL,
|
||
row_fingerprint CHAR(64) NOT NULL,
|
||
fetched_at DATETIME NOT NULL,
|
||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||
PRIMARY KEY (id),
|
||
KEY idx_demand_product_scope (
|
||
platform,
|
||
primary_category_name(80),
|
||
secondary_category_name(100),
|
||
tertiary_category_name(100),
|
||
root_word(120),
|
||
month_key,
|
||
dates_type,
|
||
rank_position
|
||
),
|
||
KEY idx_demand_product_fingerprint (row_fingerprint),
|
||
KEY idx_demand_product_fetched_at (fetched_at)
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='HSDS 产品需求分析商品款式快照'`
|
||
);
|
||
});
|
||
|
||
demandProductStyleSchemaReady = true;
|
||
}
|
||
|
||
function sleep(ms) {
|
||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||
}
|
||
|
||
async function withTimeout(promise, timeoutMs, label = 'operation') {
|
||
const ms = normalizeInteger(timeoutMs, 0);
|
||
if (!ms) {
|
||
return promise;
|
||
}
|
||
|
||
let timer = null;
|
||
const timeout = new Promise((_, reject) => {
|
||
timer = setTimeout(() => {
|
||
reject(new Error(`${label} timed out after ${ms}ms`));
|
||
}, ms);
|
||
});
|
||
|
||
try {
|
||
return await Promise.race([promise, timeout]);
|
||
} finally {
|
||
if (timer) {
|
||
clearTimeout(timer);
|
||
}
|
||
if (promise && typeof promise.catch === 'function') {
|
||
promise.catch(() => {});
|
||
}
|
||
}
|
||
}
|
||
|
||
function isHsdsRateLimitResponse(json) {
|
||
return Number(json?.code) === 10006 || normalizeText(json?.content).includes('操作频率过快');
|
||
}
|
||
|
||
function isHsdsOtherDeviceLoginResponse(json) {
|
||
return Number(json?.code) === -7 || normalizeText(json?.content).includes('该账号已在其他设备登录');
|
||
}
|
||
|
||
function isHsdsLoginExpiredResponse(json) {
|
||
return Number(json?.code) === -1 || normalizeText(json?.content).includes('用户未登录或登录状态已过期');
|
||
}
|
||
|
||
async function applyRequestDelay(context, options = {}) {
|
||
const waitMs = Math.max(normalizeInteger(options.requestDelayMs, DEFAULT_REQUEST_DELAY_MS), 0);
|
||
if (!waitMs) {
|
||
return;
|
||
}
|
||
|
||
const lastRequestedAt = Number(context._hsdsLastRequestedAt || 0);
|
||
const elapsedMs = lastRequestedAt ? Date.now() - lastRequestedAt : null;
|
||
const remainingMs = elapsedMs === null ? waitMs : Math.max(waitMs - elapsedMs, 0);
|
||
if (remainingMs > 0) {
|
||
await sleep(remainingMs);
|
||
}
|
||
context._hsdsLastRequestedAt = Date.now();
|
||
}
|
||
|
||
async function pathExists(filePath) {
|
||
const target = normalizeText(filePath);
|
||
if (!target) {
|
||
return false;
|
||
}
|
||
try {
|
||
await fs.access(target);
|
||
return true;
|
||
} catch (_error) {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
async function ensureParentDirectory(filePath) {
|
||
const target = normalizeText(filePath);
|
||
if (!target) {
|
||
return;
|
||
}
|
||
await fs.mkdir(path.dirname(target), { recursive: true });
|
||
}
|
||
|
||
function isMissingBrowserExecutableError(error) {
|
||
const message = normalizeText(error?.message).toLowerCase();
|
||
return message.includes('executable doesn\'t exist') || message.includes('executable does not exist');
|
||
}
|
||
|
||
async function resolveFallbackBrowserExecutable() {
|
||
const candidates = [
|
||
process.env.HSDS_BROWSER_EXECUTABLE,
|
||
process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH,
|
||
'/snap/bin/chromium',
|
||
'/usr/bin/chromium-browser',
|
||
'/usr/bin/chromium',
|
||
'/usr/bin/google-chrome',
|
||
'/usr/bin/google-chrome-stable',
|
||
];
|
||
|
||
for (const candidate of candidates) {
|
||
if (await pathExists(candidate)) {
|
||
return candidate;
|
||
}
|
||
}
|
||
return '';
|
||
}
|
||
|
||
function resolveRuntimeOptions(options = {}) {
|
||
return {
|
||
dryRun: Boolean(options.dryRun),
|
||
sampleOnly: Boolean(options.sampleOnly),
|
||
forceRefresh: Boolean(options.forceRefresh),
|
||
headless: options.headless !== false,
|
||
useStorageState: normalizeBooleanEnv(
|
||
options.useStorageState ?? process.env.HSDS_DEMAND_SYNC_USE_STORAGE_STATE,
|
||
true
|
||
),
|
||
persistStorageState: normalizeBooleanEnv(
|
||
options.persistStorageState ?? process.env.HSDS_DEMAND_SYNC_PERSIST_STORAGE_STATE,
|
||
true
|
||
),
|
||
storageStatePath: normalizeText(
|
||
options.storageStatePath
|
||
|| process.env.HSDS_DEMAND_SYNC_STORAGE_STATE_PATH
|
||
|| STORAGE_STATE_PATH
|
||
),
|
||
browserChannel: normalizeText(
|
||
options.browserChannel
|
||
|| process.env.HSDS_DEMAND_SYNC_BROWSER_CHANNEL
|
||
|| (process.platform === 'darwin' ? 'chrome' : '')
|
||
),
|
||
browserExecutable: normalizeText(
|
||
options.browserExecutable
|
||
|| process.env.HSDS_DEMAND_SYNC_BROWSER_EXECUTABLE
|
||
),
|
||
username: normalizeText(
|
||
options.username
|
||
|| process.env.HSDS_DEMAND_SYNC_USERNAME
|
||
|| process.env.HSDS_USERNAME
|
||
),
|
||
password: normalizeText(
|
||
options.password
|
||
|| process.env.HSDS_DEMAND_SYNC_PASSWORD
|
||
|| process.env.HSDS_PASSWORD
|
||
),
|
||
statDateKey: normalizeDateKey(
|
||
options.statDateKey
|
||
|| process.env.HSDS_DEMAND_SYNC_STAT_DATE
|
||
),
|
||
primaryCategoryName: normalizeText(
|
||
options.primaryCategoryName
|
||
|| process.env.HSDS_DEMAND_SYNC_PRIMARY_CATEGORY
|
||
|| DEFAULT_PRIMARY_CATEGORY_NAME
|
||
),
|
||
secondaryCategoryName: normalizeText(
|
||
options.secondaryCategoryName
|
||
|| process.env.HSDS_DEMAND_SYNC_SECONDARY_CATEGORY
|
||
),
|
||
tertiaryCategoryName: normalizeText(
|
||
options.tertiaryCategoryName
|
||
|| process.env.HSDS_DEMAND_SYNC_TERTIARY_CATEGORY
|
||
),
|
||
categoryLimit: Math.max(
|
||
normalizeInteger(
|
||
options.categoryLimit ?? process.env.HSDS_DEMAND_SYNC_CATEGORY_LIMIT,
|
||
DEFAULT_CATEGORY_LIMIT
|
||
),
|
||
0
|
||
),
|
||
historyDays: Math.max(
|
||
normalizeInteger(
|
||
options.historyDays ?? process.env.HSDS_DEMAND_SYNC_HISTORY_DAYS,
|
||
DEFAULT_HISTORY_DAYS
|
||
),
|
||
1
|
||
),
|
||
requestTimeoutMs: Math.max(
|
||
normalizeInteger(
|
||
options.requestTimeoutMs ?? process.env.HSDS_DEMAND_SYNC_REQUEST_TIMEOUT_MS,
|
||
DEFAULT_REQUEST_TIMEOUT_MS
|
||
),
|
||
1000
|
||
),
|
||
requestDelayMs: Math.max(
|
||
normalizeInteger(
|
||
options.requestDelayMs ?? process.env.HSDS_DEMAND_SYNC_REQUEST_DELAY_MS,
|
||
DEFAULT_REQUEST_DELAY_MS
|
||
),
|
||
0
|
||
),
|
||
maxAttempts: Math.max(
|
||
normalizeInteger(
|
||
options.maxAttempts ?? process.env.HSDS_DEMAND_SYNC_MAX_ATTEMPTS,
|
||
DEFAULT_REQUEST_MAX_ATTEMPTS
|
||
),
|
||
1
|
||
),
|
||
retryDelayMs: Math.max(
|
||
normalizeInteger(
|
||
options.retryDelayMs ?? process.env.HSDS_DEMAND_SYNC_RETRY_DELAY_MS,
|
||
DEFAULT_REQUEST_RETRY_DELAY_MS
|
||
),
|
||
0
|
||
),
|
||
strictMetricCoverage: normalizeBooleanEnv(
|
||
options.strictMetricCoverage ?? process.env.HSDS_DEMAND_SYNC_STRICT_METRIC_COVERAGE,
|
||
true
|
||
),
|
||
};
|
||
}
|
||
|
||
function decryptPayload(payload) {
|
||
if (typeof payload !== 'string') {
|
||
return payload;
|
||
}
|
||
const decipher = crypto.createDecipheriv('aes-128-ecb', Buffer.from(AES_KEY, 'utf8'), null);
|
||
decipher.setAutoPadding(true);
|
||
let decrypted = decipher.update(payload, 'base64', 'utf8');
|
||
decrypted += decipher.final('utf8');
|
||
try {
|
||
return JSON.parse(decrypted);
|
||
} catch (_error) {
|
||
return decrypted;
|
||
}
|
||
}
|
||
|
||
function buildCategoryPathLabel(category = {}) {
|
||
return [
|
||
normalizeText(category.primaryCategoryName),
|
||
normalizeText(category.secondaryCategoryName),
|
||
normalizeText(category.tertiaryCategoryName),
|
||
].filter(Boolean).join(' > ');
|
||
}
|
||
|
||
function buildMarketCategoryName(category = {}) {
|
||
return buildCategoryPathLabel(category) || normalizeText(category.primaryCategoryName, DEFAULT_PRIMARY_CATEGORY_NAME);
|
||
}
|
||
|
||
function buildFingerprint(row = {}) {
|
||
return crypto
|
||
.createHash('sha256')
|
||
.update([
|
||
normalizeText(row.stat_date_key),
|
||
normalizeText(row.platform),
|
||
normalizeText(row.primary_category_name),
|
||
normalizeText(row.secondary_category_name),
|
||
normalizeText(row.tertiary_category_name),
|
||
normalizeText(row.market_category_name),
|
||
normalizeText(row.search_keyword),
|
||
normalizeText(row.root_word),
|
||
normalizeText(row.search_index),
|
||
normalizeText(row.click_index),
|
||
normalizeText(row.pay_index),
|
||
normalizeText(row.click_rate),
|
||
normalizeText(row.pay_conversion_rate),
|
||
].join('||'))
|
||
.digest('hex');
|
||
}
|
||
|
||
function buildKeywordSnapshotFingerprint(row = {}) {
|
||
return crypto
|
||
.createHash('sha256')
|
||
.update([
|
||
normalizeText(row.source_sheet_type),
|
||
normalizeText(row.snapshot_period),
|
||
normalizeText(row.market_category_name),
|
||
normalizeText(row.search_keyword),
|
||
normalizeText(row.search_growth_rate),
|
||
normalizeText(row.search_popularity_band),
|
||
normalizeText(row.click_rate),
|
||
normalizeText(row.pay_conversion_rate),
|
||
].join('||'))
|
||
.digest('hex');
|
||
}
|
||
|
||
function resolveCategoryScopeParts(category = {}) {
|
||
return {
|
||
primaryCategoryName: normalizeText(category.primaryCategoryName || DEFAULT_PRIMARY_CATEGORY_NAME),
|
||
secondaryCategoryName: normalizeText(category.secondaryCategoryName),
|
||
tertiaryCategoryName: normalizeText(category.tertiaryCategoryName),
|
||
};
|
||
}
|
||
|
||
function computeHistoryStartDateKey(currentDateKey, historyDays) {
|
||
const current = normalizeDateKey(currentDateKey);
|
||
if (!current) {
|
||
return '';
|
||
}
|
||
return dayjs(current, 'YYYYMMDD')
|
||
.subtract(Math.max(historyDays || DEFAULT_HISTORY_DAYS, 1) - 1, 'day')
|
||
.format('YYYYMMDD');
|
||
}
|
||
|
||
async function getSuccessfulCheckpoint(category = {}, statDateKey = '', historyStartDateKey = '', historyEndDateKey = '') {
|
||
const scope = resolveCategoryScopeParts(category);
|
||
const rows = await planningDatabase.query(
|
||
`SELECT id, status, row_count AS rowCount, source_batch_id AS sourceBatchId, fetched_at AS fetchedAt, updated_at AS updatedAt, note
|
||
FROM ${escapeIdentifier(CHECKPOINT_TABLE_NAME)}
|
||
WHERE platform = ?
|
||
AND primary_category_name = ?
|
||
AND secondary_category_name = ?
|
||
AND tertiary_category_name = ?
|
||
AND stat_date_key = ?
|
||
AND history_start_date_key = ?
|
||
AND history_end_date_key = ?
|
||
AND status = 'success'
|
||
LIMIT 1`,
|
||
[
|
||
PLATFORM,
|
||
scope.primaryCategoryName,
|
||
scope.secondaryCategoryName,
|
||
scope.tertiaryCategoryName,
|
||
normalizeDateKey(statDateKey),
|
||
normalizeDateKey(historyStartDateKey),
|
||
normalizeDateKey(historyEndDateKey),
|
||
]
|
||
);
|
||
return rows[0] || null;
|
||
}
|
||
|
||
async function detectExistingCoverage(category = {}, historyStartDateKey = '', historyEndDateKey = '') {
|
||
const scope = resolveCategoryScopeParts(category);
|
||
const normalizedStartDateKey = normalizeDateKey(historyStartDateKey);
|
||
const normalizedEndDateKey = normalizeDateKey(historyEndDateKey);
|
||
const expectedMetricDateCount = normalizedStartDateKey && normalizedEndDateKey
|
||
? Math.max(dayjs(normalizedEndDateKey, 'YYYYMMDD').diff(dayjs(normalizedStartDateKey, 'YYYYMMDD'), 'day') + 1, 1)
|
||
: 1;
|
||
const rows = await planningDatabase.query(
|
||
`SELECT
|
||
COUNT(*) AS totalRows,
|
||
MIN(stat_date_key) AS minDateKey,
|
||
MAX(stat_date_key) AS maxDateKey,
|
||
MAX(CASE WHEN stat_date_key = ? THEN 1 ELSE 0 END) AS hasLatestDate,
|
||
SUM(CASE WHEN stat_date_key = ? THEN 1 ELSE 0 END) AS latestRowCount,
|
||
SUM(CASE
|
||
WHEN stat_date_key = ?
|
||
AND click_index IS NOT NULL
|
||
AND LENGTH(click_index) > 0
|
||
AND pay_index IS NOT NULL
|
||
AND LENGTH(pay_index) > 0
|
||
THEN 1 ELSE 0
|
||
END) AS latestMetricRowCount,
|
||
COUNT(DISTINCT CASE
|
||
WHEN click_index IS NOT NULL
|
||
AND LENGTH(click_index) > 0
|
||
AND pay_index IS NOT NULL
|
||
AND LENGTH(pay_index) > 0
|
||
THEN stat_date_key ELSE NULL
|
||
END) AS metricDateCount
|
||
FROM ${escapeIdentifier(TABLE_NAME)}
|
||
WHERE platform = ?
|
||
AND primary_category_name = ?
|
||
AND COALESCE(secondary_category_name, '') = ?
|
||
AND COALESCE(tertiary_category_name, '') = ?
|
||
AND stat_date_key BETWEEN ? AND ?`,
|
||
[
|
||
normalizeDateKey(historyEndDateKey),
|
||
normalizeDateKey(historyEndDateKey),
|
||
normalizeDateKey(historyEndDateKey),
|
||
PLATFORM,
|
||
scope.primaryCategoryName,
|
||
scope.secondaryCategoryName,
|
||
scope.tertiaryCategoryName,
|
||
normalizedStartDateKey,
|
||
normalizedEndDateKey,
|
||
]
|
||
);
|
||
|
||
const summary = rows[0] || {};
|
||
const totalRows = Number(summary.totalRows || 0);
|
||
const minDateKey = normalizeDateKey(summary.minDateKey);
|
||
const maxDateKey = normalizeDateKey(summary.maxDateKey);
|
||
const hasLatestDate = Number(summary.hasLatestDate || 0) === 1;
|
||
const latestRowCount = Number(summary.latestRowCount || 0);
|
||
const latestMetricRowCount = Number(summary.latestMetricRowCount || 0);
|
||
const metricDateCount = Number(summary.metricDateCount || 0);
|
||
|
||
return {
|
||
totalRows,
|
||
minDateKey,
|
||
maxDateKey,
|
||
hasLatestDate,
|
||
latestRowCount,
|
||
latestMetricRowCount,
|
||
metricDateCount,
|
||
covered: totalRows > 0
|
||
&& hasLatestDate
|
||
&& latestRowCount > 0
|
||
&& latestMetricRowCount > 0
|
||
&& metricDateCount >= expectedMetricDateCount
|
||
&& maxDateKey >= normalizedEndDateKey,
|
||
};
|
||
}
|
||
|
||
async function upsertDemandIndexCheckpoint(connection, payload = {}) {
|
||
const scope = resolveCategoryScopeParts(payload.category);
|
||
const values = {
|
||
platform: PLATFORM,
|
||
primary_category_name: scope.primaryCategoryName,
|
||
secondary_category_name: scope.secondaryCategoryName,
|
||
tertiary_category_name: scope.tertiaryCategoryName,
|
||
stat_date_key: normalizeDateKey(payload.statDateKey),
|
||
history_start_date_key: normalizeDateKey(payload.historyStartDateKey),
|
||
history_end_date_key: normalizeDateKey(payload.historyEndDateKey),
|
||
history_days: Math.max(normalizeInteger(payload.historyDays, DEFAULT_HISTORY_DAYS), 1),
|
||
status: normalizeText(payload.status || 'success'),
|
||
row_count: Math.max(normalizeInteger(payload.rowCount, 0), 0),
|
||
source_batch_id: normalizeNullableText(payload.sourceBatchId),
|
||
note: normalizeNullableText(payload.note),
|
||
fetched_at: payload.fetchedAt || dayjs().format('YYYY-MM-DD HH:mm:ss'),
|
||
};
|
||
|
||
await connection.query(
|
||
`INSERT INTO ${escapeIdentifier(CHECKPOINT_TABLE_NAME)} (
|
||
platform,
|
||
primary_category_name,
|
||
secondary_category_name,
|
||
tertiary_category_name,
|
||
stat_date_key,
|
||
history_start_date_key,
|
||
history_end_date_key,
|
||
history_days,
|
||
status,
|
||
row_count,
|
||
source_batch_id,
|
||
note,
|
||
fetched_at
|
||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
ON DUPLICATE KEY UPDATE
|
||
history_days = VALUES(history_days),
|
||
status = VALUES(status),
|
||
row_count = VALUES(row_count),
|
||
source_batch_id = VALUES(source_batch_id),
|
||
note = VALUES(note),
|
||
fetched_at = VALUES(fetched_at)`,
|
||
[
|
||
values.platform,
|
||
values.primary_category_name,
|
||
values.secondary_category_name,
|
||
values.tertiary_category_name,
|
||
values.stat_date_key,
|
||
values.history_start_date_key,
|
||
values.history_end_date_key,
|
||
values.history_days,
|
||
values.status,
|
||
values.row_count,
|
||
values.source_batch_id,
|
||
values.note,
|
||
values.fetched_at,
|
||
]
|
||
);
|
||
}
|
||
|
||
async function createBrowserContext(options = {}) {
|
||
const launchOptions = {
|
||
headless: options.headless !== false,
|
||
};
|
||
if (options.browserChannel) {
|
||
launchOptions.channel = options.browserChannel;
|
||
}
|
||
if (options.browserExecutable) {
|
||
launchOptions.executablePath = options.browserExecutable;
|
||
}
|
||
|
||
let browser;
|
||
try {
|
||
browser = await chromium.launch(launchOptions);
|
||
} catch (error) {
|
||
if (!isMissingBrowserExecutableError(error)) {
|
||
throw error;
|
||
}
|
||
const fallbackExecutable = await resolveFallbackBrowserExecutable();
|
||
if (!fallbackExecutable) {
|
||
throw error;
|
||
}
|
||
browser = await chromium.launch({
|
||
headless: options.headless !== false,
|
||
executablePath: fallbackExecutable,
|
||
});
|
||
}
|
||
const contextOptions = {};
|
||
if (options.useStorageState && options.storageStatePath && await pathExists(options.storageStatePath)) {
|
||
contextOptions.storageState = options.storageStatePath;
|
||
}
|
||
const context = await browser.newContext(contextOptions);
|
||
return { browser, context };
|
||
}
|
||
|
||
async function saveStorageState(context, storageStatePath) {
|
||
if (!storageStatePath) {
|
||
return;
|
||
}
|
||
await ensureParentDirectory(storageStatePath);
|
||
await context.storageState({ path: storageStatePath });
|
||
}
|
||
|
||
async function isSessionUsable(context, requestTimeoutMs) {
|
||
const page = await context.newPage();
|
||
try {
|
||
await page.goto(`${TARGET_BASE_URL}/workTable/workTableHome`, {
|
||
waitUntil: 'domcontentloaded',
|
||
timeout: requestTimeoutMs,
|
||
});
|
||
await page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => {});
|
||
const currentUrl = normalizeText(page.url());
|
||
if (currentUrl.includes('/login')) {
|
||
return false;
|
||
}
|
||
const hasLoginInput = await page.getByPlaceholder('请输入账号').isVisible().catch(() => false);
|
||
if (hasLoginInput) {
|
||
return false;
|
||
}
|
||
return currentUrl.includes('/workTable/');
|
||
} catch (_error) {
|
||
return false;
|
||
} finally {
|
||
await page.close().catch(() => {});
|
||
}
|
||
}
|
||
|
||
async function loginWithCredentials(context, options = {}) {
|
||
const username = normalizeText(options.username);
|
||
const password = normalizeText(options.password);
|
||
if (!username || !password) {
|
||
throw new Error('HSDS 账号或密码未配置，无法自动登录');
|
||
}
|
||
|
||
const page = await context.newPage();
|
||
try {
|
||
await page.goto(LOGIN_URL, { waitUntil: 'domcontentloaded', timeout: options.requestTimeoutMs });
|
||
await page.locator('input[placeholder="请输入账号"]:visible').first().fill(username);
|
||
await page.locator('input[placeholder="请输入密码"]:visible').first().fill(password);
|
||
await page.locator('.el-checkbox__input:visible').last().click({ force: true }).catch(() => {});
|
||
await page.locator('div.x-button:visible').filter({ hasText: '登录' }).first().click();
|
||
|
||
const agreeButton = page.getByRole('button', { name: '同意并登录' });
|
||
if (await agreeButton.count()) {
|
||
await agreeButton.click().catch(() => {});
|
||
}
|
||
|
||
await page.waitForURL('**/workTable/workTableHome', { timeout: options.requestTimeoutMs });
|
||
} finally {
|
||
await page.close().catch(() => {});
|
||
}
|
||
|
||
const usable = await isSessionUsable(context, options.requestTimeoutMs);
|
||
if (!usable) {
|
||
throw new Error('HSDS 登录失败，请检查账号密码或 storage state 是否有效');
|
||
}
|
||
}
|
||
|
||
async function ensureAuthenticatedContext(options = {}) {
|
||
const { browser, context } = await createBrowserContext(options);
|
||
let authenticated = false;
|
||
|
||
try {
|
||
authenticated = await isSessionUsable(context, options.requestTimeoutMs);
|
||
if (!authenticated) {
|
||
await loginWithCredentials(context, options);
|
||
authenticated = true;
|
||
if (options.persistStorageState !== false && options.storageStatePath) {
|
||
await saveStorageState(context, options.storageStatePath);
|
||
}
|
||
}
|
||
|
||
return {
|
||
browser,
|
||
context,
|
||
close: async () => {
|
||
await context.close().catch(() => {});
|
||
await browser.close().catch(() => {});
|
||
},
|
||
};
|
||
} catch (error) {
|
||
await context.close().catch(() => {});
|
||
await browser.close().catch(() => {});
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
function buildUrl(endpoint, query = {}) {
|
||
const url = endpoint.startsWith('http')
|
||
? endpoint
|
||
: `${HSDS_API_BASE_URL}${endpoint}`;
|
||
const searchParams = new URLSearchParams();
|
||
for (const [key, value] of Object.entries(query || {})) {
|
||
const normalized = normalizeText(value);
|
||
if (normalized) {
|
||
searchParams.set(key, normalized);
|
||
}
|
||
}
|
||
const queryString = searchParams.toString();
|
||
return queryString ? `${url}?${queryString}` : url;
|
||
}
|
||
|
||
async function requestHsds(context, endpoint, options = {}) {
|
||
const method = normalizeText(options.method || 'GET').toUpperCase();
|
||
const url = buildUrl(endpoint, options.query);
|
||
const maxAttempts = Math.max(normalizeInteger(options.maxAttempts, DEFAULT_REQUEST_MAX_ATTEMPTS), 1);
|
||
const retryDelayMs = Math.max(normalizeInteger(options.retryDelayMs, DEFAULT_REQUEST_RETRY_DELAY_MS), 0);
|
||
let reloginAttempted = false;
|
||
|
||
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||
await applyRequestDelay(context, options);
|
||
|
||
const requestOptions = {
|
||
timeout: options.requestTimeoutMs,
|
||
failOnStatusCode: false,
|
||
};
|
||
if (method === 'POST') {
|
||
requestOptions.data = options.data || {};
|
||
requestOptions.headers = {
|
||
'content-type': 'application/json;charset=UTF-8',
|
||
};
|
||
}
|
||
|
||
const response = await withTimeout(
|
||
method === 'POST'
|
||
? context.request.post(url, requestOptions)
|
||
: context.request.get(url, requestOptions),
|
||
options.requestTimeoutMs,
|
||
`HSDS ${method} ${endpoint}`
|
||
);
|
||
const json = await withTimeout(
|
||
response.json().catch(() => null),
|
||
options.requestTimeoutMs,
|
||
`HSDS ${method} ${endpoint} json`
|
||
);
|
||
|
||
if (!response.ok()) {
|
||
throw new Error(`HSDS 接口请求失败: ${method} ${url} HTTP ${response.status()}`.trim());
|
||
}
|
||
|
||
if (json && json.success === true && Number(json.code) === 0) {
|
||
return decryptPayload(json.data);
|
||
}
|
||
|
||
if (isHsdsLoginExpiredResponse(json)) {
|
||
if (reloginAttempted) {
|
||
throw new Error(`HSDS 登录状态失效且重登失败: ${method} ${url} code=${json?.code ?? 'unknown'} content=${json?.content || ''}`.trim());
|
||
}
|
||
reloginAttempted = true;
|
||
console.warn(`[HSDS] 登录状态失效，尝试重新登录后重试: ${endpoint}`);
|
||
await loginWithCredentials(context, options);
|
||
if (options.persistStorageState !== false && options.storageStatePath) {
|
||
await saveStorageState(context, options.storageStatePath);
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (isHsdsOtherDeviceLoginResponse(json)) {
|
||
throw new Error(`HSDS 账号被其他设备登录顶掉: ${method} ${url} code=${json?.code ?? 'unknown'} content=${json?.content || ''}`.trim());
|
||
}
|
||
|
||
if (attempt < maxAttempts && isHsdsRateLimitResponse(json)) {
|
||
const waitMs = retryDelayMs * attempt * attempt;
|
||
console.warn(`[HSDS] 需求指数接口触发频率限制，${waitMs}ms 后重试: ${endpoint} (第 ${attempt}/${maxAttempts} 次)`);
|
||
await sleep(waitMs);
|
||
continue;
|
||
}
|
||
|
||
throw new Error(`HSDS 接口请求失败: ${method} ${url} status=${response.status()} code=${json?.code ?? 'unknown'} content=${json?.content || response.statusText()}`.trim());
|
||
}
|
||
}
|
||
|
||
function resolveRequestControlOptions(options = {}) {
|
||
return {
|
||
requestTimeoutMs: options.requestTimeoutMs,
|
||
requestDelayMs: options.requestDelayMs,
|
||
maxAttempts: options.maxAttempts,
|
||
retryDelayMs: options.retryDelayMs,
|
||
};
|
||
}
|
||
|
||
async function fetchPrimaryCategory(context, options = {}) {
|
||
const categories = await requestHsds(context, '/category/categoryFirstList', {
|
||
query: { platform: PLATFORM },
|
||
...resolveRequestControlOptions(options),
|
||
});
|
||
|
||
const matched = (Array.isArray(categories) ? categories : []).find((item) => {
|
||
return normalizeText(item.name) === options.primaryCategoryName;
|
||
});
|
||
|
||
if (!matched) {
|
||
throw new Error(`HSDS 未找到一级类目: ${options.primaryCategoryName}`);
|
||
}
|
||
|
||
return matched;
|
||
}
|
||
|
||
async function fetchCategoryTree(context, primaryCategoryId, options = {}) {
|
||
const rows = await requestHsds(context, '/category/getCategoryById', {
|
||
query: {
|
||
platform: PLATFORM,
|
||
categoryId: primaryCategoryId,
|
||
isAuth: 'true',
|
||
},
|
||
...resolveRequestControlOptions(options),
|
||
});
|
||
return Array.isArray(rows) ? rows : [];
|
||
}
|
||
|
||
function buildNodePath(nodeId, nodeMap) {
|
||
const pathNodes = [];
|
||
const visited = new Set();
|
||
let current = nodeMap.get(String(nodeId));
|
||
while (current && !visited.has(String(current.id))) {
|
||
pathNodes.unshift(current);
|
||
visited.add(String(current.id));
|
||
const parentId = normalizeText(current.pid);
|
||
if (!parentId || parentId === '0') {
|
||
break;
|
||
}
|
||
current = nodeMap.get(parentId);
|
||
}
|
||
return pathNodes;
|
||
}
|
||
|
||
function buildLeafCategories(primaryCategory, flatRows = [], options = {}) {
|
||
const nodeMap = new Map();
|
||
for (const row of flatRows) {
|
||
nodeMap.set(String(row.id), row);
|
||
}
|
||
|
||
const leaves = flatRows.filter((row) => normalizeLeafFlag(row.isLeaf));
|
||
const siblingLeafMap = new Map();
|
||
|
||
for (const leaf of leaves) {
|
||
const parentId = normalizeText(leaf.pid || primaryCategory.id);
|
||
if (!siblingLeafMap.has(parentId)) {
|
||
siblingLeafMap.set(parentId, []);
|
||
}
|
||
siblingLeafMap.get(parentId).push(String(leaf.id));
|
||
}
|
||
|
||
return leaves
|
||
.map((leaf) => {
|
||
const pathNodes = buildNodePath(leaf.id, nodeMap);
|
||
const pathNames = pathNodes.map((item) => normalizeText(item.name)).filter(Boolean);
|
||
const secondaryNode = pathNodes[1] || null;
|
||
const tertiaryPathNames = pathNames.slice(2);
|
||
const tertiaryNode = pathNodes.length >= 3 ? pathNodes[pathNodes.length - 1] : null;
|
||
|
||
return {
|
||
primaryCategoryId: normalizeText(primaryCategory.id),
|
||
primaryCategoryName: normalizeText(primaryCategory.name),
|
||
secondaryCategoryId: normalizeText(secondaryNode?.id),
|
||
secondaryCategoryName: normalizeText(secondaryNode?.name),
|
||
tertiaryCategoryId: normalizeText(tertiaryNode?.id),
|
||
tertiaryCategoryName: normalizeText(tertiaryPathNames.join(' > ') || tertiaryNode?.name),
|
||
categoryId: normalizeText(leaf.id),
|
||
categoryName: normalizeText(leaf.name),
|
||
marketCategoryName: pathNames.join(' > ') || normalizeText(primaryCategory.name),
|
||
siblingCategoryIds: (siblingLeafMap.get(normalizeText(leaf.pid || primaryCategory.id)) || [String(leaf.id)]).join(','),
|
||
};
|
||
})
|
||
.filter((row) => {
|
||
if (options.secondaryCategoryName && row.secondaryCategoryName !== options.secondaryCategoryName) {
|
||
return false;
|
||
}
|
||
if (
|
||
options.tertiaryCategoryName
|
||
&& row.tertiaryCategoryName !== options.tertiaryCategoryName
|
||
&& row.categoryName !== options.tertiaryCategoryName
|
||
) {
|
||
return false;
|
||
}
|
||
return true;
|
||
});
|
||
}
|
||
|
||
async function fetchKeywordTime(context, categoryId, options = {}) {
|
||
const payload = await requestHsds(context, '/demand/getKeywordAnalysisTime', {
|
||
query: {
|
||
platform: PLATFORM,
|
||
categoryId,
|
||
pluginType: 'demandInsight',
|
||
},
|
||
...resolveRequestControlOptions(options),
|
||
});
|
||
return payload || {};
|
||
}
|
||
|
||
async function fetchHotWords(context, categoryId, options = {}) {
|
||
const date = normalizeText(options.hotWordDate || options.date || '7');
|
||
const datesType = normalizeText(options.hotWordDatesType || options.datesType || '7');
|
||
const isTimeChange = normalizeText(
|
||
options.hotWordIsTimeChange ?? options.isTimeChange ?? (datesType === 'day' ? 'true' : 'false')
|
||
);
|
||
const payload = await requestHsds(context, '/demand/hotWords', {
|
||
query: {
|
||
platform: PLATFORM,
|
||
cateId: categoryId,
|
||
date,
|
||
datesType,
|
||
isTimeChange,
|
||
filterBrandKeyWord: 'false',
|
||
},
|
||
...resolveRequestControlOptions(options),
|
||
});
|
||
return payload || {};
|
||
}
|
||
|
||
async function fetchSoaringWords(context, categoryId, options = {}) {
|
||
const date = normalizeText(options.hotWordDate || options.date || '7');
|
||
const datesType = normalizeText(options.hotWordDatesType || options.datesType || '7');
|
||
const payload = await requestHsds(context, '/demand/soaringWords', {
|
||
method: 'POST',
|
||
data: {
|
||
platform: PLATFORM,
|
||
cateId: categoryId,
|
||
date,
|
||
datesType,
|
||
isTimeChange: 'false',
|
||
filterBrandKeyWord: false,
|
||
},
|
||
...resolveRequestControlOptions(options),
|
||
});
|
||
return payload || {};
|
||
}
|
||
|
||
function resolveHsdsDemandDatesType(options = {}) {
|
||
const raw = normalizeText(options.dateShortcut || options.windowType || options.datesType || options.historyDays || '7');
|
||
const normalized = raw.toLowerCase();
|
||
if (['7天', '7d', 'day7', '7'].includes(normalized)) return '7';
|
||
if (['14天', '14d', 'day14', '14'].includes(normalized)) return '14';
|
||
if (['30天', '30d', 'day30', '30'].includes(normalized)) return '30';
|
||
if (['日', 'day'].includes(normalized)) return 'day';
|
||
if (['周', 'week'].includes(normalized)) return 'week';
|
||
if (['月', 'month'].includes(normalized)) return 'month';
|
||
return '7';
|
||
}
|
||
|
||
function resolveHsdsDemandMonth(options = {}) {
|
||
const explicitMonth = normalizeText(options.month || options.statMonth);
|
||
if (/^\d{4}-?\d{2}$/.test(explicitMonth)) {
|
||
return explicitMonth.replace(/^(\d{4})(\d{2})$/, '$1-$2');
|
||
}
|
||
const dateKey = normalizeDateKey(options.endDateKey || options.statDateKey || options.latestStatDateKey);
|
||
if (dateKey) {
|
||
return dayjs(dateKey, 'YYYYMMDD').format('YYYY-MM');
|
||
}
|
||
return dayjs().subtract(1, 'day').format('YYYY-MM');
|
||
}
|
||
|
||
function selectDemandProductCategory(categories = [], options = {}) {
|
||
const secondaryCategoryName = normalizeText(options.secondaryCategoryName || options.twoCategory);
|
||
const tertiaryCategoryName = normalizeText(options.tertiaryCategoryName || options.threeCategory);
|
||
if (tertiaryCategoryName) {
|
||
const exactTertiary = categories.find((category) => {
|
||
return category.secondaryCategoryName === secondaryCategoryName
|
||
&& category.tertiaryCategoryName === tertiaryCategoryName;
|
||
});
|
||
if (exactTertiary) return exactTertiary;
|
||
|
||
const looseTertiary = categories.find((category) => {
|
||
return category.tertiaryCategoryName === tertiaryCategoryName
|
||
|| category.categoryName === tertiaryCategoryName;
|
||
});
|
||
if (looseTertiary) return looseTertiary;
|
||
}
|
||
|
||
if (secondaryCategoryName) {
|
||
return categories.find((category) => category.secondaryCategoryName === secondaryCategoryName) || null;
|
||
}
|
||
|
||
return categories[0] || null;
|
||
}
|
||
|
||
function parseDemandProductStyleRows(payload = {}, category = {}, options = {}) {
|
||
const rows = Array.isArray(payload.columnData)
|
||
? payload.columnData
|
||
: Array.isArray(payload.hotWordList)
|
||
? payload.hotWordList
|
||
: Array.isArray(payload.goodsList)
|
||
? payload.goodsList
|
||
: Array.isArray(payload.list)
|
||
? payload.list
|
||
: [];
|
||
const limit = Math.min(Math.max(normalizeInteger(options.limit || options.pageSize, 50), 1), 100);
|
||
|
||
return rows.slice(0, limit).map((row, index) => {
|
||
const goodsId = normalizeText(row.goodsId || row.itemId || row.item_id || row.id);
|
||
const title = normalizeText(row.goodsName || row.title || row.itemTitle || row.name);
|
||
const imageUrl = normalizeText(
|
||
row.goodsUrl
|
||
|| row.imageUrl
|
||
|| row.picUrl
|
||
|| row.mainPic
|
||
|| row.img
|
||
|| (Array.isArray(row.goodsUrlList) ? row.goodsUrlList[0] : '')
|
||
);
|
||
return {
|
||
rank: normalizeInteger(row.rank || row.rankNo || row.ranking, index + 1),
|
||
goodsId,
|
||
title,
|
||
imageUrl,
|
||
shop: normalizeText(row.shopName || row.shop || row.sellerName, '--'),
|
||
brand: normalizeText(row.brand || row.brandName, '--'),
|
||
sales: normalizeText(row.gmv || row.pay || row.salesAmount || row.saleAmount || row.gmvqj2 || '--'),
|
||
saleCount: normalizeText(row.sales || row.saleCount || row.payCnt || row.payCount || '--'),
|
||
unitPrice: normalizeText(row.jdj || row.unitPrice || row.price || '--'),
|
||
visitor: normalizeText(row.visitor || row.uv || '--'),
|
||
conversionRate: normalizeText(row.conversionRate || row.clickPay || row.payConversionRate || '--'),
|
||
detailUrl: normalizeText(row.detailUrl || row.goodsDetailUrl || row.url),
|
||
categoryPath: buildMarketCategoryName(category),
|
||
source: 'hsds_demand_getGoodsInfoByRoot',
|
||
raw: row,
|
||
};
|
||
}).filter((row) => row.title || row.goodsId);
|
||
}
|
||
|
||
function buildDemandProductStyleFingerprint(scope = {}, row = {}) {
|
||
return crypto
|
||
.createHash('sha256')
|
||
.update([
|
||
normalizeText(scope.platform, PLATFORM),
|
||
normalizeText(scope.primaryCategoryName),
|
||
normalizeText(scope.secondaryCategoryName),
|
||
normalizeText(scope.tertiaryCategoryName),
|
||
normalizeText(scope.rootWord),
|
||
normalizeText(scope.month),
|
||
normalizeText(scope.datesType),
|
||
normalizeText(row.rank),
|
||
normalizeText(row.goodsId),
|
||
normalizeText(row.title),
|
||
normalizeText(row.shop),
|
||
normalizeText(row.sales),
|
||
].join('||'))
|
||
.digest('hex');
|
||
}
|
||
|
||
function mapDemandProductStyleCacheRow(row = {}) {
|
||
return {
|
||
rank: normalizeInteger(row.rankPosition || row.rank_position, 0),
|
||
goodsId: normalizeText(row.goodsId || row.goods_id),
|
||
title: normalizeText(row.title || row.goodsTitle || row.goods_title),
|
||
imageUrl: normalizeText(row.imageUrl || row.image_url),
|
||
shop: normalizeText(row.shop || row.shopName || row.shop_name, '--'),
|
||
brand: normalizeText(row.brand || row.brandName || row.brand_name, '--'),
|
||
sales: normalizeText(row.sales || row.salesAmount || row.sales_amount, '--'),
|
||
saleCount: normalizeText(row.saleCount || row.sale_count, '--'),
|
||
unitPrice: normalizeText(row.unitPrice || row.unit_price, '--'),
|
||
visitor: normalizeText(row.visitor || row.visitorCount || row.visitor_count, '--'),
|
||
conversionRate: normalizeText(row.conversionRate || row.conversion_rate, '--'),
|
||
detailUrl: normalizeText(row.detailUrl || row.detail_url),
|
||
categoryPath: normalizeText(row.marketCategoryName || row.market_category_name),
|
||
source: 'planning_demand_product_styles',
|
||
raw: row.rawData || row.raw_data || null,
|
||
};
|
||
}
|
||
|
||
function buildDemandProductStyleScope(options = {}) {
|
||
return {
|
||
platform: normalizeText(options.platform, PLATFORM),
|
||
primaryCategoryName: normalizeText(
|
||
options.primaryCategoryName || options.category || options.oneCategory || DEFAULT_PRIMARY_CATEGORY_NAME
|
||
),
|
||
secondaryCategoryName: normalizeText(
|
||
options.secondaryCategoryName || options.secondaryCategory || options.twoCategory
|
||
),
|
||
tertiaryCategoryName: normalizeText(
|
||
options.tertiaryCategoryName || options.tertiaryCategory || options.threeCategory
|
||
),
|
||
rootWord: normalizeText(options.rootWord || options.keyword || options.rootWordList),
|
||
month: resolveHsdsDemandMonth(options),
|
||
datesType: resolveHsdsDemandDatesType(options),
|
||
};
|
||
}
|
||
|
||
async function getCachedPlanningHsdsDemandProductStyles(rawOptions = {}) {
|
||
await ensureDemandProductStyleTableSchema();
|
||
const scope = buildDemandProductStyleScope(rawOptions);
|
||
if (!scope.rootWord) {
|
||
return {
|
||
success: false,
|
||
skipped: true,
|
||
reason: 'missing_root_word',
|
||
products: [],
|
||
sourceMeta: {
|
||
source: DEMAND_PRODUCT_STYLE_TABLE_NAME,
|
||
skipped: true,
|
||
reason: 'missing_root_word',
|
||
},
|
||
};
|
||
}
|
||
|
||
const limit = Math.min(Math.max(normalizeInteger(rawOptions.limit || rawOptions.pageSize, 50), 1), 100);
|
||
const rows = await planningDatabase.query(
|
||
`SELECT
|
||
rank_position AS rankPosition,
|
||
goods_id AS goodsId,
|
||
goods_title AS goodsTitle,
|
||
image_url AS imageUrl,
|
||
shop_name AS shopName,
|
||
brand_name AS brandName,
|
||
sales_amount AS salesAmount,
|
||
sale_count AS saleCount,
|
||
unit_price AS unitPrice,
|
||
visitor_count AS visitorCount,
|
||
conversion_rate AS conversionRate,
|
||
detail_url AS detailUrl,
|
||
market_category_name AS marketCategoryName,
|
||
raw_data AS rawData,
|
||
fetched_at AS fetchedAt
|
||
FROM ${escapeIdentifier(DEMAND_PRODUCT_STYLE_TABLE_NAME)}
|
||
WHERE platform = ?
|
||
AND primary_category_name = ?
|
||
AND secondary_category_name = ?
|
||
AND tertiary_category_name = ?
|
||
AND root_word = ?
|
||
AND month_key = ?
|
||
AND dates_type = ?
|
||
ORDER BY rank_position ASC, id ASC
|
||
LIMIT ?`,
|
||
[
|
||
scope.platform,
|
||
scope.primaryCategoryName,
|
||
scope.secondaryCategoryName,
|
||
scope.tertiaryCategoryName,
|
||
scope.rootWord,
|
||
scope.month,
|
||
scope.datesType,
|
||
limit,
|
||
]
|
||
);
|
||
|
||
const products = rows.map(mapDemandProductStyleCacheRow);
|
||
return {
|
||
success: true,
|
||
skipped: products.length === 0,
|
||
reason: products.length ? '' : 'cache_miss',
|
||
products,
|
||
total: products.length,
|
||
sourceMeta: {
|
||
source: DEMAND_PRODUCT_STYLE_TABLE_NAME,
|
||
cached: true,
|
||
total: products.length,
|
||
rootWord: scope.rootWord,
|
||
month: scope.month,
|
||
datesType: scope.datesType,
|
||
fetchedAt: rows[0]?.fetchedAt || null,
|
||
reason: products.length ? '' : 'cache_miss',
|
||
},
|
||
};
|
||
}
|
||
|
||
async function replaceDemandProductStyleRows(scope = {}, category = {}, products = []) {
|
||
await ensureDemandProductStyleTableSchema();
|
||
const fetchedAt = dayjs().format('YYYY-MM-DD HH:mm:ss');
|
||
const marketCategoryName = buildMarketCategoryName(category) || [
|
||
scope.primaryCategoryName,
|
||
scope.secondaryCategoryName,
|
||
scope.tertiaryCategoryName,
|
||
].filter(Boolean).join(' > ');
|
||
|
||
await planningDatabase.withTransaction(async (connection) => {
|
||
await connection.query(
|
||
`DELETE FROM ${escapeIdentifier(DEMAND_PRODUCT_STYLE_TABLE_NAME)}
|
||
WHERE platform = ?
|
||
AND primary_category_name = ?
|
||
AND secondary_category_name = ?
|
||
AND tertiary_category_name = ?
|
||
AND root_word = ?
|
||
AND month_key = ?
|
||
AND dates_type = ?`,
|
||
[
|
||
scope.platform,
|
||
scope.primaryCategoryName,
|
||
scope.secondaryCategoryName,
|
||
scope.tertiaryCategoryName,
|
||
scope.rootWord,
|
||
scope.month,
|
||
scope.datesType,
|
||
]
|
||
);
|
||
|
||
if (!products.length) {
|
||
return;
|
||
}
|
||
|
||
const rows = products.map((product, index) => [
|
||
scope.platform,
|
||
scope.primaryCategoryName,
|
||
scope.secondaryCategoryName,
|
||
scope.tertiaryCategoryName,
|
||
marketCategoryName,
|
||
normalizeText(category.categoryId),
|
||
scope.rootWord,
|
||
scope.month,
|
||
scope.datesType,
|
||
normalizeInteger(product.rank, index + 1),
|
||
normalizeText(product.goodsId),
|
||
normalizeText(product.title),
|
||
normalizeText(product.imageUrl) || null,
|
||
normalizeText(product.shop, '--'),
|
||
normalizeText(product.brand, '--'),
|
||
normalizeText(product.sales, '--'),
|
||
normalizeText(product.saleCount, '--'),
|
||
normalizeText(product.unitPrice, '--'),
|
||
normalizeText(product.visitor, '--'),
|
||
normalizeText(product.conversionRate, '--'),
|
||
normalizeText(product.detailUrl) || null,
|
||
JSON.stringify(product.raw || {}),
|
||
buildDemandProductStyleFingerprint(scope, product),
|
||
fetchedAt,
|
||
]);
|
||
|
||
for (let index = 0; index < rows.length; index += MAX_INSERT_CHUNK_SIZE) {
|
||
const chunk = rows.slice(index, index + MAX_INSERT_CHUNK_SIZE);
|
||
const placeholders = chunk.map(() => '(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)').join(', ');
|
||
await connection.query(
|
||
`INSERT INTO ${escapeIdentifier(DEMAND_PRODUCT_STYLE_TABLE_NAME)} (
|
||
platform,
|
||
primary_category_name,
|
||
secondary_category_name,
|
||
tertiary_category_name,
|
||
market_category_name,
|
||
cate_id,
|
||
root_word,
|
||
month_key,
|
||
dates_type,
|
||
rank_position,
|
||
goods_id,
|
||
goods_title,
|
||
image_url,
|
||
shop_name,
|
||
brand_name,
|
||
sales_amount,
|
||
sale_count,
|
||
unit_price,
|
||
visitor_count,
|
||
conversion_rate,
|
||
detail_url,
|
||
raw_data,
|
||
row_fingerprint,
|
||
fetched_at
|
||
) VALUES ${placeholders}`,
|
||
chunk.flat()
|
||
);
|
||
}
|
||
});
|
||
|
||
return {
|
||
table: DEMAND_PRODUCT_STYLE_TABLE_NAME,
|
||
insertedRows: products.length,
|
||
fetchedAt,
|
||
};
|
||
}
|
||
|
||
async function fetchDemandProductStylesByRoot(context, category, rootWord, options = {}) {
|
||
const datesType = resolveHsdsDemandDatesType(options);
|
||
const payload = await requestHsds(context, '/demand/getGoodsInfoByRoot', {
|
||
query: {
|
||
platform: PLATFORM,
|
||
cateId: category.categoryId,
|
||
month: resolveHsdsDemandMonth(options),
|
||
datesType,
|
||
rootWordList: rootWord,
|
||
customDemandInsightCode: 'useGoodsHooks',
|
||
},
|
||
...resolveRequestControlOptions(options),
|
||
});
|
||
|
||
return {
|
||
rows: parseDemandProductStyleRows(payload, category, options),
|
||
raw: payload,
|
||
datesType,
|
||
month: resolveHsdsDemandMonth(options),
|
||
};
|
||
}
|
||
|
||
function buildDateKeyRange(startDateKey, endDateKey) {
|
||
const start = dayjs(normalizeDateKey(startDateKey), 'YYYYMMDD');
|
||
const end = dayjs(normalizeDateKey(endDateKey), 'YYYYMMDD');
|
||
if (!start.isValid() || !end.isValid() || start.isAfter(end)) {
|
||
return [];
|
||
}
|
||
|
||
const dateKeys = [];
|
||
for (let cursor = start; cursor.isSame(end) || cursor.isBefore(end); cursor = cursor.add(1, 'day')) {
|
||
dateKeys.push(cursor.format('YYYYMMDD'));
|
||
}
|
||
return dateKeys;
|
||
}
|
||
|
||
async function fetchDailyHotWordRows(context, category, dateKeys = [], options = {}) {
|
||
const rows = [];
|
||
const failures = [];
|
||
|
||
for (const dateKey of dateKeys) {
|
||
const statDate = dayjs(dateKey, 'YYYYMMDD');
|
||
if (!statDate.isValid()) {
|
||
continue;
|
||
}
|
||
try {
|
||
const payload = await fetchHotWords(context, category.categoryId, {
|
||
...options,
|
||
hotWordDate: statDate.format('YYYY-MM-DD'),
|
||
hotWordDatesType: 'day',
|
||
hotWordIsTimeChange: 'true',
|
||
});
|
||
rows.push(...parseHotWordRows(payload, category, dateKey, 'keyword_daily'));
|
||
} catch (error) {
|
||
failures.push({
|
||
statDateKey: dateKey,
|
||
message: normalizeText(error?.message || error),
|
||
});
|
||
console.warn(`[HSDS] ${buildCategoryPathLabel(category)} ${dateKey} 每日热词指标抓取失败: ${error.message}`);
|
||
}
|
||
}
|
||
|
||
return { rows, failures };
|
||
}
|
||
|
||
async function fetchLayoutWords(context, category, statDateKey, options = {}) {
|
||
const statDate = dayjs(statDateKey, 'YYYYMMDD');
|
||
const payload = await requestHsds(context, '/demandTrend/getWordsByDateList', {
|
||
method: 'POST',
|
||
data: {
|
||
categoryId: category.categoryId,
|
||
cateIds: category.siblingCategoryIds || category.categoryId,
|
||
date: statDate.isValid() ? statDate.format('YYYY-MM-DD') : normalizeText(statDateKey),
|
||
daysType: 'day',
|
||
},
|
||
...resolveRequestControlOptions(options),
|
||
});
|
||
return payload || {};
|
||
}
|
||
|
||
function parseHotWordRows(hotWordsPayload = {}, category = {}, statDateKey = '', sourceType = 'keyword_current') {
|
||
const rows = Array.isArray(hotWordsPayload.columnData)
|
||
? hotWordsPayload.columnData
|
||
: Array.isArray(hotWordsPayload.hotWordList)
|
||
? hotWordsPayload.hotWordList
|
||
: [];
|
||
|
||
return rows
|
||
.map((item) => ({
|
||
stat_date_key: statDateKey,
|
||
platform: PLATFORM,
|
||
primary_category_name: category.primaryCategoryName,
|
||
secondary_category_name: category.secondaryCategoryName || null,
|
||
tertiary_category_name: category.tertiaryCategoryName || null,
|
||
market_category_name: buildMarketCategoryName(category),
|
||
search_keyword: normalizeText(item.keyWord || item.keyword),
|
||
root_word: normalizeText(item.keyWord || item.keyword),
|
||
search_index: normalizeMetricText(item.searchPeople),
|
||
click_index: normalizeMetricText(item.clickPeople),
|
||
pay_index: normalizeMetricText(item.dealPeople),
|
||
click_rate: normalizeMetricText(item.clickRate),
|
||
pay_conversion_rate: normalizeMetricText(item.dealRate || item.conversionRate),
|
||
search_index_mom: normalizeMetricText(item.searchGrow),
|
||
click_index_mom: null,
|
||
pay_index_mom: null,
|
||
click_rate_mom: null,
|
||
pay_conversion_rate_mom: null,
|
||
raw_data: {
|
||
sourceType,
|
||
source: '需求趋势分析',
|
||
row: item,
|
||
},
|
||
}))
|
||
.filter((row) => row.stat_date_key && row.search_keyword);
|
||
}
|
||
|
||
function parseKeywordSnapshotRows(payload = {}, category = {}, options = {}) {
|
||
const sourceType = normalizeText(options.sourceType || 'hot_search');
|
||
const snapshotPeriod = normalizeText(options.snapshotPeriod);
|
||
const rows = Array.isArray(payload.columnData)
|
||
? payload.columnData
|
||
: Array.isArray(payload.hotWordList)
|
||
? payload.hotWordList
|
||
: Array.isArray(payload.soaringWordList)
|
||
? payload.soaringWordList
|
||
: Array.isArray(payload.list)
|
||
? payload.list
|
||
: [];
|
||
const batchId = uuidv4();
|
||
let rowNumber = 1;
|
||
|
||
return rows
|
||
.map((item, index) => {
|
||
const row = {
|
||
upload_batch_id: batchId,
|
||
source_file_name: KEYWORD_SNAPSHOT_SOURCE_FILE_NAME,
|
||
source_sheet_name: buildCategoryPathLabel(category).slice(0, 100) || DEFAULT_PRIMARY_CATEGORY_NAME,
|
||
source_sheet_type: sourceType === 'rising' ? 'category_rising_words' : 'category_hot_search_words',
|
||
source_row_number: rowNumber,
|
||
uploaded_by_user_id: null,
|
||
uploaded_by_username: UPLOADED_BY_USERNAME,
|
||
uploaded_by_name: UPLOADED_BY_NAME,
|
||
uploaded_department_id: null,
|
||
uploaded_department_name: null,
|
||
raw_data: {
|
||
source: sourceType === 'rising' ? 'HSDS 品类飙升词' : 'HSDS 品类热搜词',
|
||
sourceType,
|
||
row: item,
|
||
},
|
||
snapshot_period: snapshotPeriod,
|
||
rank_position: normalizeMetricText(item.rank || item.ranking || item.rankPosition || index + 1),
|
||
market_category_name: buildMarketCategoryName(category),
|
||
search_keyword: normalizeText(item.keyWord || item.keyword || item.searchKeyword || item.searchWord),
|
||
search_growth_rate: sourceType === 'rising'
|
||
? normalizeMetricText(item.searchGrow || item.searchGrowthRate || item.growthRate)
|
||
: undefined,
|
||
search_popularity_band: normalizeMetricText(item.searchPeopleRange || item.searchPeople || item.searchPopularityBand),
|
||
click_rate: normalizeMetricText(item.clickRate),
|
||
pay_conversion_rate: normalizeMetricText(
|
||
item.dealRateRange
|
||
|| item.dealRate
|
||
|| item.conversionRateRange
|
||
|| item.conversionRate
|
||
),
|
||
row_fingerprint: null,
|
||
};
|
||
row.row_fingerprint = buildKeywordSnapshotFingerprint(row);
|
||
rowNumber += 1;
|
||
return row;
|
||
})
|
||
.filter((row) => row.snapshot_period && row.market_category_name && row.search_keyword);
|
||
}
|
||
|
||
function parseLayoutHistoryRows(layoutPayload = {}, category = {}, options = {}) {
|
||
const currentDate = options.currentStatDateKey || '';
|
||
const minDateKey = computeHistoryStartDateKey(currentDate, options.historyDays || DEFAULT_HISTORY_DAYS);
|
||
|
||
const groups = Array.isArray(layoutPayload.list) ? layoutPayload.list : [];
|
||
const rows = [];
|
||
|
||
for (const group of groups) {
|
||
const rowDateKey = normalizeDateKey(group.date);
|
||
if (!rowDateKey) {
|
||
continue;
|
||
}
|
||
if (minDateKey && rowDateKey < minDateKey) {
|
||
continue;
|
||
}
|
||
for (const word of Array.isArray(group.wordList) ? group.wordList : []) {
|
||
const searchKeyword = normalizeText(word.ssc || word.keyWord || word.keyword);
|
||
if (!searchKeyword) {
|
||
continue;
|
||
}
|
||
rows.push({
|
||
stat_date_key: rowDateKey,
|
||
platform: PLATFORM,
|
||
primary_category_name: category.primaryCategoryName,
|
||
secondary_category_name: category.secondaryCategoryName || null,
|
||
tertiary_category_name: category.tertiaryCategoryName || null,
|
||
market_category_name: buildMarketCategoryName(category),
|
||
search_keyword: searchKeyword,
|
||
root_word: searchKeyword,
|
||
search_index: normalizeMetricText(word.ssrq || word.searchPeople),
|
||
click_index: null,
|
||
pay_index: null,
|
||
click_rate: null,
|
||
pay_conversion_rate: null,
|
||
search_index_mom: normalizeMetricText(word.increase || word.searchGrow),
|
||
click_index_mom: null,
|
||
pay_index_mom: null,
|
||
click_rate_mom: null,
|
||
pay_conversion_rate_mom: null,
|
||
raw_data: {
|
||
sourceType: 'keyword_layout',
|
||
source: '关键词布局分析',
|
||
row: word,
|
||
layoutDate: group.date,
|
||
},
|
||
});
|
||
}
|
||
}
|
||
|
||
return rows;
|
||
}
|
||
|
||
function mergeDemandIndexRows(category = {}, currentRows = [], historyRows = []) {
|
||
const rowMap = new Map();
|
||
|
||
const pushOrMerge = (row = {}) => {
|
||
const rowDateKey = normalizeDateKey(row.stat_date_key);
|
||
const searchKeyword = normalizeText(row.search_keyword);
|
||
const rootWord = normalizeText(row.root_word || searchKeyword);
|
||
if (!rowDateKey || !searchKeyword || !rootWord) {
|
||
return;
|
||
}
|
||
|
||
const key = [
|
||
rowDateKey,
|
||
normalizeText(category.primaryCategoryName),
|
||
normalizeText(category.secondaryCategoryName),
|
||
normalizeText(category.tertiaryCategoryName),
|
||
searchKeyword,
|
||
].join('||');
|
||
|
||
const existing = rowMap.get(key) || {};
|
||
rowMap.set(key, {
|
||
...existing,
|
||
...row,
|
||
root_word: rootWord,
|
||
raw_data: {
|
||
...(existing.raw_data || {}),
|
||
...(row.raw_data || {}),
|
||
},
|
||
});
|
||
};
|
||
|
||
for (const row of historyRows) {
|
||
pushOrMerge(row);
|
||
}
|
||
for (const row of currentRows) {
|
||
pushOrMerge(row);
|
||
}
|
||
|
||
const batchId = uuidv4();
|
||
let sourceRowNumber = 1;
|
||
|
||
return Array.from(rowMap.values())
|
||
.sort((left, right) => {
|
||
const dateDiff = normalizeText(left.stat_date_key).localeCompare(normalizeText(right.stat_date_key));
|
||
if (dateDiff !== 0) return dateDiff;
|
||
return normalizeText(left.search_keyword).localeCompare(normalizeText(right.search_keyword), 'zh-CN');
|
||
})
|
||
.map((row) => {
|
||
const finalRow = {
|
||
upload_batch_id: batchId,
|
||
source_file_name: SOURCE_FILE_NAME,
|
||
source_sheet_name: buildCategoryPathLabel(category).slice(0, 100) || DEFAULT_PRIMARY_CATEGORY_NAME,
|
||
source_sheet_type: SOURCE_SHEET_TYPE,
|
||
source_row_number: sourceRowNumber,
|
||
uploaded_by_user_id: null,
|
||
uploaded_by_username: UPLOADED_BY_USERNAME,
|
||
uploaded_by_name: UPLOADED_BY_NAME,
|
||
uploaded_department_id: null,
|
||
uploaded_department_name: null,
|
||
raw_data: row.raw_data,
|
||
row_fingerprint: null,
|
||
stat_date_key: row.stat_date_key,
|
||
platform: row.platform,
|
||
primary_category_name: row.primary_category_name,
|
||
secondary_category_name: row.secondary_category_name,
|
||
tertiary_category_name: row.tertiary_category_name,
|
||
market_category_name: row.market_category_name,
|
||
search_keyword: row.search_keyword,
|
||
root_word: row.root_word,
|
||
search_index: row.search_index,
|
||
click_index: row.click_index,
|
||
pay_index: row.pay_index,
|
||
click_rate: row.click_rate,
|
||
pay_conversion_rate: row.pay_conversion_rate,
|
||
search_index_mom: row.search_index_mom,
|
||
click_index_mom: row.click_index_mom,
|
||
pay_index_mom: row.pay_index_mom,
|
||
click_rate_mom: row.click_rate_mom,
|
||
pay_conversion_rate_mom: row.pay_conversion_rate_mom,
|
||
};
|
||
finalRow.row_fingerprint = buildFingerprint(finalRow);
|
||
sourceRowNumber += 1;
|
||
return finalRow;
|
||
});
|
||
}
|
||
|
||
async function replaceCategoryRows(connection, category, rows = []) {
|
||
if (!rows.length) {
|
||
return { deletedRows: 0, insertedRows: 0 };
|
||
}
|
||
|
||
const statDateKeys = rows.map((row) => normalizeDateKey(row.stat_date_key)).filter(Boolean).sort();
|
||
const minDateKey = statDateKeys[0];
|
||
const maxDateKey = statDateKeys[statDateKeys.length - 1];
|
||
const deleteParams = [
|
||
PLATFORM,
|
||
category.primaryCategoryName,
|
||
category.secondaryCategoryName || '',
|
||
category.tertiaryCategoryName || '',
|
||
minDateKey,
|
||
maxDateKey,
|
||
];
|
||
|
||
const deleteResult = await connection.query(
|
||
`DELETE FROM ${escapeIdentifier(TABLE_NAME)}
|
||
WHERE platform = ?
|
||
AND primary_category_name = ?
|
||
AND COALESCE(secondary_category_name, '') = ?
|
||
AND COALESCE(tertiary_category_name, '') = ?
|
||
AND stat_date_key BETWEEN ? AND ?`,
|
||
deleteParams
|
||
);
|
||
|
||
const columnNames = Object.keys(rows[0]);
|
||
for (let index = 0; index < rows.length; index += MAX_INSERT_CHUNK_SIZE) {
|
||
const chunk = rows.slice(index, index + MAX_INSERT_CHUNK_SIZE);
|
||
const placeholders = chunk.map(() => `(${columnNames.map(() => '?').join(', ')})`).join(', ');
|
||
const values = [];
|
||
for (const row of chunk) {
|
||
for (const columnName of columnNames) {
|
||
values.push(columnName === 'raw_data' ? JSON.stringify(row[columnName] || {}) : row[columnName]);
|
||
}
|
||
}
|
||
await connection.query(
|
||
`INSERT INTO ${escapeIdentifier(TABLE_NAME)} (${columnNames.map(escapeIdentifier).join(', ')})
|
||
VALUES ${placeholders}`,
|
||
values
|
||
);
|
||
}
|
||
|
||
return {
|
||
deletedRows: Number(deleteResult?.[0]?.affectedRows || deleteResult?.affectedRows || 0),
|
||
insertedRows: rows.length,
|
||
};
|
||
}
|
||
|
||
async function replaceKeywordSnapshotRows(connection, tableName, category, snapshotPeriod, rows = []) {
|
||
const scopeMarketCategoryName = buildMarketCategoryName(category);
|
||
await connection.query(
|
||
`DELETE FROM ${escapeIdentifier(tableName)}
|
||
WHERE snapshot_period = ?
|
||
AND market_category_name = ?`,
|
||
[snapshotPeriod, scopeMarketCategoryName]
|
||
);
|
||
|
||
if (!rows.length) {
|
||
return { deletedRows: 0, insertedRows: 0 };
|
||
}
|
||
|
||
const firstRow = rows[0];
|
||
const columnNames = Object.keys(firstRow).filter((columnName) => firstRow[columnName] !== undefined);
|
||
for (let index = 0; index < rows.length; index += MAX_INSERT_CHUNK_SIZE) {
|
||
const chunk = rows.slice(index, index + MAX_INSERT_CHUNK_SIZE);
|
||
const placeholders = chunk.map(() => `(${columnNames.map(() => '?').join(', ')})`).join(', ');
|
||
const values = [];
|
||
for (const row of chunk) {
|
||
for (const columnName of columnNames) {
|
||
values.push(columnName === 'raw_data' ? JSON.stringify(row[columnName] || {}) : row[columnName]);
|
||
}
|
||
}
|
||
await connection.query(
|
||
`INSERT INTO ${escapeIdentifier(tableName)} (${columnNames.map(escapeIdentifier).join(', ')})
|
||
VALUES ${placeholders}`,
|
||
values
|
||
);
|
||
}
|
||
|
||
return { deletedRows: 0, insertedRows: rows.length };
|
||
}
|
||
|
||
async function syncSingleKeywordSnapshotCategory(context, category, runtimeOptions) {
|
||
const keywordTime = await fetchKeywordTime(context, category.categoryId, runtimeOptions);
|
||
const effectiveStatDateKey = runtimeOptions.statDateKey || normalizeDateKey(keywordTime.maxDayDate);
|
||
if (!effectiveStatDateKey) {
|
||
return {
|
||
categoryPathLabel: buildCategoryPathLabel(category),
|
||
skipped: true,
|
||
reason: 'missing_stat_date',
|
||
};
|
||
}
|
||
|
||
const historyStartDateKey = computeHistoryStartDateKey(effectiveStatDateKey, runtimeOptions.historyDays);
|
||
const snapshotPeriod = `${dayjs(historyStartDateKey, 'YYYYMMDD').format('YYYY-MM-DD')} ~ ${dayjs(effectiveStatDateKey, 'YYYYMMDD').format('YYYY-MM-DD')}`;
|
||
const date = String(runtimeOptions.historyDays);
|
||
const failures = [];
|
||
let hotRows = null;
|
||
let risingRows = null;
|
||
|
||
try {
|
||
const hotPayload = await fetchHotWords(context, category.categoryId, {
|
||
...runtimeOptions,
|
||
date,
|
||
datesType: date,
|
||
hotWordIsTimeChange: 'false',
|
||
});
|
||
hotRows = parseKeywordSnapshotRows(hotPayload, category, {
|
||
sourceType: 'hot_search',
|
||
snapshotPeriod,
|
||
});
|
||
} catch (error) {
|
||
failures.push({
|
||
type: 'hot_search',
|
||
message: normalizeText(error?.message || error),
|
||
});
|
||
}
|
||
|
||
try {
|
||
const risingPayload = await fetchSoaringWords(context, category.categoryId, {
|
||
...runtimeOptions,
|
||
date,
|
||
datesType: date,
|
||
});
|
||
risingRows = parseKeywordSnapshotRows(risingPayload, category, {
|
||
sourceType: 'rising',
|
||
snapshotPeriod,
|
||
});
|
||
} catch (error) {
|
||
failures.push({
|
||
type: 'rising',
|
||
message: normalizeText(error?.message || error),
|
||
});
|
||
}
|
||
|
||
if (runtimeOptions.dryRun) {
|
||
return {
|
||
categoryPathLabel: buildCategoryPathLabel(category),
|
||
snapshotPeriod,
|
||
hotRows: Array.isArray(hotRows) ? hotRows.length : 0,
|
||
risingRows: Array.isArray(risingRows) ? risingRows.length : 0,
|
||
failures,
|
||
sampleHotRow: Array.isArray(hotRows) ? hotRows[0] || null : null,
|
||
sampleRisingRow: Array.isArray(risingRows) ? risingRows[0] || null : null,
|
||
};
|
||
}
|
||
|
||
await planningDatabase.withTransaction(async (connection) => {
|
||
if (Array.isArray(hotRows)) {
|
||
await replaceKeywordSnapshotRows(connection, HOT_SEARCH_WORD_TABLE_NAME, category, snapshotPeriod, hotRows);
|
||
}
|
||
if (Array.isArray(risingRows)) {
|
||
await replaceKeywordSnapshotRows(connection, RISING_WORD_TABLE_NAME, category, snapshotPeriod, risingRows);
|
||
}
|
||
});
|
||
|
||
return {
|
||
categoryPathLabel: buildCategoryPathLabel(category),
|
||
snapshotPeriod,
|
||
success: failures.length === 0,
|
||
hotRows: Array.isArray(hotRows) ? hotRows.length : 0,
|
||
risingRows: Array.isArray(risingRows) ? risingRows.length : 0,
|
||
failures,
|
||
};
|
||
}
|
||
|
||
async function syncSingleCategory(context, category, runtimeOptions, progress = {}) {
|
||
const keywordTime = await fetchKeywordTime(context, category.categoryId, runtimeOptions);
|
||
const effectiveStatDateKey = runtimeOptions.statDateKey || normalizeDateKey(keywordTime.maxDayDate);
|
||
if (!effectiveStatDateKey) {
|
||
return {
|
||
categoryPathLabel: buildCategoryPathLabel(category),
|
||
skipped: true,
|
||
reason: 'missing_stat_date',
|
||
};
|
||
}
|
||
|
||
const historyStartDateKey = computeHistoryStartDateKey(effectiveStatDateKey, runtimeOptions.historyDays);
|
||
const historyEndDateKey = effectiveStatDateKey;
|
||
|
||
if (!runtimeOptions.dryRun && !runtimeOptions.forceRefresh) {
|
||
const coverage = await detectExistingCoverage(category, historyStartDateKey, historyEndDateKey);
|
||
const checkpoint = await getSuccessfulCheckpoint(
|
||
category,
|
||
effectiveStatDateKey,
|
||
historyStartDateKey,
|
||
historyEndDateKey
|
||
);
|
||
if (checkpoint && coverage.covered) {
|
||
return {
|
||
categoryPathLabel: buildCategoryPathLabel(category),
|
||
statDateKey: effectiveStatDateKey,
|
||
skipped: true,
|
||
reason: 'checkpoint_exists',
|
||
currentKeywordCount: 0,
|
||
historyRowCount: 0,
|
||
existingRowCount: Number(checkpoint.rowCount || 0),
|
||
insertedRows: 0,
|
||
deletedRows: 0,
|
||
};
|
||
}
|
||
|
||
if (coverage.covered) {
|
||
await planningDatabase.withTransaction(async (connection) => {
|
||
await upsertDemandIndexCheckpoint(connection, {
|
||
category,
|
||
statDateKey: effectiveStatDateKey,
|
||
historyStartDateKey,
|
||
historyEndDateKey,
|
||
historyDays: runtimeOptions.historyDays,
|
||
rowCount: coverage.totalRows,
|
||
status: 'success',
|
||
note: 'bootstrap_from_existing_rows',
|
||
fetchedAt: dayjs().format('YYYY-MM-DD HH:mm:ss'),
|
||
});
|
||
});
|
||
|
||
return {
|
||
categoryPathLabel: buildCategoryPathLabel(category),
|
||
statDateKey: effectiveStatDateKey,
|
||
skipped: true,
|
||
reason: 'existing_rows_covered',
|
||
currentKeywordCount: 0,
|
||
historyRowCount: 0,
|
||
existingRowCount: coverage.totalRows,
|
||
insertedRows: 0,
|
||
deletedRows: 0,
|
||
};
|
||
}
|
||
}
|
||
|
||
const layoutPayload = await fetchLayoutWords(context, category, effectiveStatDateKey, runtimeOptions);
|
||
const historyRows = parseLayoutHistoryRows(layoutPayload, category, {
|
||
historyDays: runtimeOptions.historyDays,
|
||
currentStatDateKey: effectiveStatDateKey,
|
||
});
|
||
const dateKeys = buildDateKeyRange(historyStartDateKey, historyEndDateKey);
|
||
const dailyHotWords = await fetchDailyHotWordRows(context, category, dateKeys, runtimeOptions);
|
||
const metricRows = dailyHotWords.rows;
|
||
const insertRows = mergeDemandIndexRows(category, metricRows, historyRows);
|
||
const metricDateCount = new Set(metricRows.map((row) => row.stat_date_key).filter(Boolean)).size;
|
||
const expectedMetricDateCount = dateKeys.length;
|
||
|
||
if (
|
||
runtimeOptions.strictMetricCoverage
|
||
&& !runtimeOptions.dryRun
|
||
&& (dailyHotWords.failures.length || metricDateCount < expectedMetricDateCount)
|
||
) {
|
||
await planningDatabase.withTransaction(async (connection) => {
|
||
await upsertDemandIndexCheckpoint(connection, {
|
||
category,
|
||
statDateKey: effectiveStatDateKey,
|
||
historyStartDateKey,
|
||
historyEndDateKey,
|
||
historyDays: runtimeOptions.historyDays,
|
||
rowCount: 0,
|
||
status: 'failed',
|
||
note: `metric_coverage_incomplete:${metricDateCount}/${expectedMetricDateCount};failures:${dailyHotWords.failures.length}`,
|
||
fetchedAt: dayjs().format('YYYY-MM-DD HH:mm:ss'),
|
||
});
|
||
});
|
||
|
||
return {
|
||
categoryPathLabel: buildCategoryPathLabel(category),
|
||
statDateKey: effectiveStatDateKey,
|
||
skipped: true,
|
||
reason: 'metric_coverage_incomplete',
|
||
currentKeywordCount: metricRows.length,
|
||
historyRowCount: historyRows.length,
|
||
metricDateCount,
|
||
expectedMetricDateCount,
|
||
metricFetchFailureCount: dailyHotWords.failures.length,
|
||
metricFetchFailures: dailyHotWords.failures.slice(0, 5),
|
||
insertedRows: 0,
|
||
deletedRows: 0,
|
||
};
|
||
}
|
||
|
||
if (runtimeOptions.dryRun) {
|
||
return {
|
||
categoryPathLabel: buildCategoryPathLabel(category),
|
||
statDateKey: effectiveStatDateKey,
|
||
deletedRows: 0,
|
||
insertedRows: insertRows.length,
|
||
currentKeywordCount: metricRows.length,
|
||
historyRowCount: historyRows.length,
|
||
metricDateCount,
|
||
expectedMetricDateCount,
|
||
metricFetchFailureCount: dailyHotWords.failures.length,
|
||
metricFetchFailures: dailyHotWords.failures.slice(0, 5),
|
||
sampleRow: insertRows[0] || null,
|
||
};
|
||
}
|
||
|
||
const replaceResult = await planningDatabase.withTransaction(async (connection) => {
|
||
const result = await replaceCategoryRows(connection, category, insertRows);
|
||
await upsertDemandIndexCheckpoint(connection, {
|
||
category,
|
||
statDateKey: effectiveStatDateKey,
|
||
historyStartDateKey,
|
||
historyEndDateKey,
|
||
historyDays: runtimeOptions.historyDays,
|
||
rowCount: result.insertedRows,
|
||
sourceBatchId: insertRows[0]?.upload_batch_id || null,
|
||
status: dailyHotWords.failures.length ? 'failed' : 'success',
|
||
note: dailyHotWords.failures.length
|
||
? `partial_metric_fetch_failed:${dailyHotWords.failures.length}`
|
||
: 'synced_from_hsds_daily_metrics',
|
||
fetchedAt: dayjs().format('YYYY-MM-DD HH:mm:ss'),
|
||
});
|
||
return result;
|
||
});
|
||
|
||
return {
|
||
categoryPathLabel: buildCategoryPathLabel(category),
|
||
statDateKey: effectiveStatDateKey,
|
||
currentKeywordCount: metricRows.length,
|
||
historyRowCount: historyRows.length,
|
||
metricDateCount,
|
||
expectedMetricDateCount,
|
||
metricFetchFailureCount: dailyHotWords.failures.length,
|
||
metricFetchFailures: dailyHotWords.failures.slice(0, 5),
|
||
sampleRow: insertRows[0] || null,
|
||
...replaceResult,
|
||
};
|
||
}
|
||
|
||
async function syncPlanningHsdsDemandIndices(rawOptions = {}) {
|
||
const options = resolveRuntimeOptions(rawOptions);
|
||
|
||
if (options.sampleOnly) {
|
||
return {
|
||
success: true,
|
||
dryRun: true,
|
||
sampleOnly: true,
|
||
categoryCount: 0,
|
||
totalInsertedRows: 0,
|
||
totalDeletedRows: 0,
|
||
results: [],
|
||
note: 'HSDS 需求指数同步已切换为真实接口，legacy sampleOnly 模式不再生成样本数据。',
|
||
};
|
||
}
|
||
|
||
if (!options.dryRun) {
|
||
await ensurePlanningCanonicalSchemas();
|
||
await ensureDemandIndexTableSchema();
|
||
await ensureDemandIndexCheckpointSchema();
|
||
}
|
||
|
||
const session = await ensureAuthenticatedContext(options);
|
||
|
||
try {
|
||
const primaryCategory = await fetchPrimaryCategory(session.context, options);
|
||
const categoryTree = await fetchCategoryTree(session.context, primaryCategory.id, options);
|
||
const categories = buildLeafCategories(primaryCategory, categoryTree, options);
|
||
const pickedCategories = options.categoryLimit > 0
|
||
? categories.slice(0, options.categoryLimit)
|
||
: categories;
|
||
|
||
const results = [];
|
||
let totalInsertedRows = 0;
|
||
let totalDeletedRows = 0;
|
||
|
||
let failureCount = 0;
|
||
|
||
for (let index = 0; index < pickedCategories.length; index += 1) {
|
||
const category = pickedCategories[index];
|
||
let result;
|
||
try {
|
||
result = await syncSingleCategory(session.context, category, options, {
|
||
currentIndex: index + 1,
|
||
totalCategories: pickedCategories.length,
|
||
});
|
||
} catch (error) {
|
||
failureCount += 1;
|
||
result = {
|
||
categoryPathLabel: buildCategoryPathLabel(category),
|
||
skipped: true,
|
||
reason: 'request_failed',
|
||
message: normalizeText(error?.message || error),
|
||
insertedRows: 0,
|
||
deletedRows: 0,
|
||
};
|
||
}
|
||
results.push(result);
|
||
totalInsertedRows += Number(result.insertedRows || 0);
|
||
totalDeletedRows += Number(result.deletedRows || 0);
|
||
}
|
||
|
||
if (!options.dryRun) {
|
||
invalidatePlanningRuntimeCache();
|
||
}
|
||
|
||
return {
|
||
success: failureCount === 0,
|
||
dryRun: options.dryRun,
|
||
sampleOnly: false,
|
||
statDateKey: options.statDateKey || '',
|
||
categoryCount: pickedCategories.length,
|
||
failureCount,
|
||
totalInsertedRows,
|
||
totalDeletedRows,
|
||
results,
|
||
};
|
||
} finally {
|
||
await session.close();
|
||
}
|
||
}
|
||
|
||
async function syncPlanningHsdsKeywordSnapshots(rawOptions = {}) {
|
||
const options = resolveRuntimeOptions({
|
||
...rawOptions,
|
||
historyDays: rawOptions.historyDays || process.env.HSDS_KEYWORD_SNAPSHOT_HISTORY_DAYS || 7,
|
||
});
|
||
|
||
if (!options.dryRun) {
|
||
await ensurePlanningCanonicalSchemas();
|
||
}
|
||
|
||
const session = await ensureAuthenticatedContext(options);
|
||
|
||
try {
|
||
const primaryCategory = await fetchPrimaryCategory(session.context, options);
|
||
const categoryTree = await fetchCategoryTree(session.context, primaryCategory.id, options);
|
||
const categories = buildLeafCategories(primaryCategory, categoryTree, options);
|
||
const pickedCategories = options.categoryLimit > 0
|
||
? categories.slice(0, options.categoryLimit)
|
||
: categories;
|
||
|
||
const results = [];
|
||
let totalHotRows = 0;
|
||
let totalRisingRows = 0;
|
||
let failureCount = 0;
|
||
|
||
for (const category of pickedCategories) {
|
||
let result;
|
||
try {
|
||
result = await syncSingleKeywordSnapshotCategory(session.context, category, options);
|
||
} catch (error) {
|
||
result = {
|
||
categoryPathLabel: buildCategoryPathLabel(category),
|
||
skipped: true,
|
||
reason: 'request_failed',
|
||
message: normalizeText(error?.message || error),
|
||
hotRows: 0,
|
||
risingRows: 0,
|
||
failures: [{
|
||
type: 'category',
|
||
message: normalizeText(error?.message || error),
|
||
}],
|
||
};
|
||
}
|
||
results.push(result);
|
||
totalHotRows += Number(result.hotRows || 0);
|
||
totalRisingRows += Number(result.risingRows || 0);
|
||
if (result.skipped || result.success === false || (Array.isArray(result.failures) && result.failures.length)) {
|
||
failureCount += 1;
|
||
}
|
||
}
|
||
|
||
if (!options.dryRun) {
|
||
invalidatePlanningRuntimeCache();
|
||
}
|
||
|
||
return {
|
||
success: failureCount === 0,
|
||
dryRun: options.dryRun,
|
||
historyDays: options.historyDays,
|
||
categoryCount: pickedCategories.length,
|
||
failureCount,
|
||
totalHotRows,
|
||
totalRisingRows,
|
||
results,
|
||
};
|
||
} finally {
|
||
await session.close();
|
||
}
|
||
}
|
||
|
||
async function fetchPlanningHsdsDemandProductStyles(rawOptions = {}) {
|
||
const options = resolveRuntimeOptions({
|
||
...rawOptions,
|
||
primaryCategoryName: rawOptions.primaryCategoryName
|
||
|| rawOptions.category
|
||
|| rawOptions.oneCategory
|
||
|| DEFAULT_PRIMARY_CATEGORY_NAME,
|
||
secondaryCategoryName: rawOptions.secondaryCategoryName
|
||
|| rawOptions.secondaryCategory
|
||
|| rawOptions.twoCategory,
|
||
tertiaryCategoryName: rawOptions.tertiaryCategoryName
|
||
|| rawOptions.tertiaryCategory
|
||
|| rawOptions.threeCategory,
|
||
requestTimeoutMs: rawOptions.requestTimeoutMs || process.env.HSDS_DEMAND_PRODUCT_REQUEST_TIMEOUT_MS || 12000,
|
||
requestDelayMs: rawOptions.requestDelayMs || 0,
|
||
});
|
||
const rootWord = normalizeText(rawOptions.rootWord || rawOptions.keyword || rawOptions.rootWordList);
|
||
const scope = buildDemandProductStyleScope({
|
||
...options,
|
||
...rawOptions,
|
||
rootWord,
|
||
});
|
||
if (!rootWord) {
|
||
return {
|
||
success: false,
|
||
skipped: true,
|
||
reason: 'missing_root_word',
|
||
products: [],
|
||
};
|
||
}
|
||
|
||
const session = await ensureAuthenticatedContext(options);
|
||
|
||
try {
|
||
const primaryCategory = await fetchPrimaryCategory(session.context, options);
|
||
const categoryTree = await fetchCategoryTree(session.context, primaryCategory.id, options);
|
||
const categories = buildLeafCategories(primaryCategory, categoryTree, {
|
||
primaryCategoryName: options.primaryCategoryName,
|
||
secondaryCategoryName: options.secondaryCategoryName,
|
||
tertiaryCategoryName: options.tertiaryCategoryName,
|
||
});
|
||
const category = selectDemandProductCategory(categories, options);
|
||
if (!category) {
|
||
return {
|
||
success: false,
|
||
skipped: true,
|
||
reason: 'category_not_found',
|
||
products: [],
|
||
};
|
||
}
|
||
|
||
const payload = await fetchDemandProductStylesByRoot(session.context, category, rootWord, {
|
||
...options,
|
||
...rawOptions,
|
||
});
|
||
const cacheResult = rawOptions.persist === false
|
||
? null
|
||
: await replaceDemandProductStyleRows(scope, category, payload.rows);
|
||
return {
|
||
success: true,
|
||
skipped: false,
|
||
rootWord,
|
||
category,
|
||
cateId: category.categoryId,
|
||
month: payload.month,
|
||
datesType: payload.datesType,
|
||
products: payload.rows,
|
||
total: payload.rows.length,
|
||
sourceMeta: {
|
||
source: 'hsds_demand_getGoodsInfoByRoot',
|
||
endpoint: '/demand/getGoodsInfoByRoot',
|
||
cachedTo: cacheResult?.table || '',
|
||
cachedRows: cacheResult?.insertedRows || 0,
|
||
categoryPath: buildMarketCategoryName(category),
|
||
cateId: category.categoryId,
|
||
month: payload.month,
|
||
datesType: payload.datesType,
|
||
},
|
||
};
|
||
} finally {
|
||
await session.close();
|
||
}
|
||
}
|
||
|
||
async function getDemandProductStyleRootWords(options = {}) {
|
||
const scope = buildDemandProductStyleScope(options);
|
||
const limit = Math.min(Math.max(normalizeInteger(options.rootWordLimit || options.limit, 20), 1), 100);
|
||
const endDateKey = normalizeDateKey(options.endDateKey || options.statDateKey) || await getLatestDemandIndexStatDateKey(scope);
|
||
if (!endDateKey) {
|
||
return [];
|
||
}
|
||
const rangeStart = dayjs(endDateKey, 'YYYYMMDD')
|
||
.subtract(Math.max(normalizeInteger(options.historyDays || 7, 7), 1) - 1, 'day')
|
||
.format('YYYYMMDD');
|
||
const rows = await planningDatabase.query(
|
||
`SELECT
|
||
COALESCE(NULLIF(TRIM(search_keyword), ''), NULLIF(TRIM(root_word), '')) AS rootWord,
|
||
SUM(CAST(REPLACE(IFNULL(search_index, '0'), ',', '') AS DECIMAL(20, 4))) AS searchScore
|
||
FROM ${escapeIdentifier(TABLE_NAME)}
|
||
WHERE platform = ?
|
||
AND primary_category_name = ?
|
||
AND secondary_category_name = ?
|
||
AND tertiary_category_name = ?
|
||
AND stat_date_key BETWEEN ? AND ?
|
||
GROUP BY rootWord
|
||
HAVING rootWord IS NOT NULL AND rootWord <> ''
|
||
ORDER BY searchScore DESC
|
||
LIMIT ?`,
|
||
[
|
||
scope.platform,
|
||
scope.primaryCategoryName,
|
||
scope.secondaryCategoryName,
|
||
scope.tertiaryCategoryName,
|
||
rangeStart,
|
||
endDateKey,
|
||
limit,
|
||
]
|
||
);
|
||
return rows.map((row) => normalizeText(row.rootWord)).filter(Boolean);
|
||
}
|
||
|
||
async function getLatestDemandIndexStatDateKey(scope = {}) {
|
||
const rows = await planningDatabase.query(
|
||
`SELECT MAX(stat_date_key) AS latestStatDateKey
|
||
FROM ${escapeIdentifier(TABLE_NAME)}
|
||
WHERE platform = ?
|
||
AND primary_category_name = ?
|
||
AND secondary_category_name = ?
|
||
AND tertiary_category_name = ?`,
|
||
[
|
||
normalizeText(scope.platform, PLATFORM),
|
||
normalizeText(scope.primaryCategoryName || DEFAULT_PRIMARY_CATEGORY_NAME),
|
||
normalizeText(scope.secondaryCategoryName),
|
||
normalizeText(scope.tertiaryCategoryName),
|
||
]
|
||
);
|
||
return normalizeDateKey(rows[0]?.latestStatDateKey);
|
||
}
|
||
|
||
async function syncPlanningHsdsDemandProductStyles(rawOptions = {}) {
|
||
const rootWord = normalizeText(rawOptions.rootWord || rawOptions.keyword || rawOptions.rootWordList);
|
||
const rootWords = rootWord
|
||
? [rootWord]
|
||
: await getDemandProductStyleRootWords(rawOptions);
|
||
if (!rootWords.length) {
|
||
return {
|
||
success: true,
|
||
skipped: true,
|
||
reason: 'no_root_words',
|
||
results: [],
|
||
};
|
||
}
|
||
|
||
const results = [];
|
||
let totalProducts = 0;
|
||
let failureCount = 0;
|
||
for (const word of rootWords) {
|
||
let result;
|
||
try {
|
||
result = await fetchPlanningHsdsDemandProductStyles({
|
||
...rawOptions,
|
||
rootWord: word,
|
||
persist: rawOptions.persist !== false,
|
||
});
|
||
} catch (error) {
|
||
failureCount += 1;
|
||
result = {
|
||
success: false,
|
||
skipped: true,
|
||
reason: 'request_failed',
|
||
message: normalizeText(error?.message || error),
|
||
total: 0,
|
||
};
|
||
}
|
||
results.push({
|
||
rootWord: word,
|
||
success: Boolean(result.success),
|
||
skipped: Boolean(result.skipped),
|
||
reason: result.reason || '',
|
||
message: result.message || '',
|
||
total: result.total || 0,
|
||
sourceMeta: result.sourceMeta || null,
|
||
});
|
||
totalProducts += Number(result.total || 0);
|
||
const delayMs = Math.max(normalizeInteger(rawOptions.productRequestDelayMs || rawOptions.requestDelayMs, 5000), 0);
|
||
if (delayMs) {
|
||
await sleep(delayMs);
|
||
}
|
||
}
|
||
|
||
invalidatePlanningRuntimeCache();
|
||
return {
|
||
success: failureCount === 0,
|
||
skipped: false,
|
||
rootWordCount: rootWords.length,
|
||
failureCount,
|
||
totalProducts,
|
||
results,
|
||
};
|
||
}
|
||
|
||
module.exports = {
|
||
TABLE_NAME,
|
||
DEFAULT_REQUEST_TIMEOUT_MS,
|
||
fetchPlanningHsdsDemandProductStyles,
|
||
getCachedPlanningHsdsDemandProductStyles,
|
||
syncPlanningHsdsDemandProductStyles,
|
||
syncPlanningHsdsDemandIndices,
|
||
syncPlanningHsdsKeywordSnapshots,
|
||
syncPlanningYucekjDemandIndices: syncPlanningHsdsDemandIndices,
|
||
};
|