forked from lunanting/cangku
2821 lines
91 KiB
JavaScript
2821 lines
91 KiB
JavaScript
const crypto = require('crypto');
|
||
const XLSX = require('xlsx');
|
||
const { v4: uuidv4 } = require('uuid');
|
||
const planningDatabase = require('./planningDatabase');
|
||
const { getResolvedUserWithPageAccess } = require('./pageAccess');
|
||
const PLANNING_UPLOAD_PAGE_KEY = 'planning_upload';
|
||
const { NOT_CONFIGURED_MESSAGE } = require('./planningDatabase');
|
||
const {
|
||
TAOBAO_PRODUCT_RECORDS_STANDARD_COLUMNS,
|
||
APPEARANCE_RECOGNITION_CODE_COLUMN,
|
||
APPEARANCE_RECOGNITION_CODE_LABEL,
|
||
ensureTaobaoProductRecordsAppearanceCodeColumn,
|
||
} = require('./taobaoProductRecordsAppearance');
|
||
const { getPlanningGoodsAnalysisSummary } = require('./planningGoodsAnalysisService');
|
||
const {
|
||
getPlanningMayProductDataSummary,
|
||
getPlanningMayProductDataTableData,
|
||
getPlanningMayProductSnapshotCacheVersion,
|
||
} = require('./planningMayProductDataService');
|
||
const {
|
||
ensurePlanningAiImageSchema,
|
||
ensurePlanningAppearanceMatchSchema,
|
||
ensurePlanningUploadBatchSchema,
|
||
} = require('./planningSupportSchema');
|
||
|
||
const IMPORT_BATCH_TABLE = 'planning_upload_batches';
|
||
const PLANNING_UPLOAD_TABLE_NAME = '企划数据库模板表（5张）';
|
||
const ROW_FINGERPRINT_COLUMN = 'row_fingerprint';
|
||
const TEMPLATE_SUMMARIES_CACHE_TTL_MS = 5 * 60 * 1000;
|
||
const TABLE_DATA_CACHE_TTL_MS = 60 * 1000;
|
||
const MAY_TABLE_DATA_CACHE_TTL_MS = 5 * 1000;
|
||
const PLANNING_TABLE_EXPORT_MAX_ROWS = Math.max(
|
||
Number.parseInt(process.env.PLANNING_TABLE_EXPORT_MAX_ROWS || '50000', 10),
|
||
1000
|
||
);
|
||
const TAOBAO_PRODUCT_RECORDS_TABLE = 'taobao_product_records';
|
||
const TAOBAO_DETAIL_PLUS_TABLE = 'taobao_detail_plus_data';
|
||
const BUSINESS_SKU_DIMENSION_TABLE = 'dwd.dim_ope_sku_data_df';
|
||
const MARKET_BASIC_COMPARE_KIND = 'market_basic_compare_data';
|
||
const MAY_PRODUCT_DATA_KIND = 'may_product_data';
|
||
const GOODS_ANALYSIS_KIND = 'goods_analysis_dashboard';
|
||
|
||
const SEARCH_SALES_PRICE_BAND_EXTRA_COLUMNS = Object.freeze([
|
||
{ name: APPEARANCE_RECOGNITION_CODE_COLUMN, label: APPEARANCE_RECOGNITION_CODE_LABEL },
|
||
{ name: 'taobao_total_sold', label: '销量' },
|
||
{ name: 'taobao_price_range', label: '价格区间' },
|
||
{ name: 'taobao_min_marked_price', label: '最低标价' },
|
||
{ name: 'taobao_min_discount_price', label: '最低标价折后价' },
|
||
...TAOBAO_PRODUCT_RECORDS_STANDARD_COLUMNS
|
||
.filter((column) => String(column.name || '').startsWith('may_'))
|
||
.map((column) => ({ name: column.name, label: column.label })),
|
||
]);
|
||
const SEARCH_SALES_PRICE_BAND_MAY_COLUMN_NAMES = Object.freeze(
|
||
SEARCH_SALES_PRICE_BAND_EXTRA_COLUMNS
|
||
.map((column) => column.name)
|
||
.filter((columnName) => String(columnName || '').startsWith('may_'))
|
||
);
|
||
|
||
const READONLY_PLANNING_PAGE_CATALOG = Object.freeze([
|
||
{
|
||
kind: GOODS_ANALYSIS_KIND,
|
||
label: '货品分析',
|
||
description: '基于市场基础对比数据评估市场机会，并用 MAY 字段判断我方产品覆盖',
|
||
sourceDatabase: 'cloud',
|
||
readOnly: true,
|
||
isAnalysisPage: true,
|
||
},
|
||
{
|
||
kind: MARKET_BASIC_COMPARE_KIND,
|
||
label: '市场基础对比数据',
|
||
description: '读取企划数据库 taobao_product_records，展示市场商品基础字段',
|
||
targetTableName: TAOBAO_PRODUCT_RECORDS_TABLE,
|
||
sourceDatabase: 'planning',
|
||
readOnly: true,
|
||
},
|
||
{
|
||
kind: MAY_PRODUCT_DATA_KIND,
|
||
label: 'MAY-商品数据',
|
||
description: '以业务商品库 dwd.dim_ope_sku_data_df 的公司 SKU 为准，补充外观编码、限价、库存、销量和动销数据',
|
||
targetTableName: BUSINESS_SKU_DIMENSION_TABLE,
|
||
sourceDatabase: 'business+planning',
|
||
readOnly: true,
|
||
},
|
||
]);
|
||
|
||
const MARKET_BASIC_COMPARE_COLUMN_LABELS = Object.freeze(new Map([
|
||
...TAOBAO_PRODUCT_RECORDS_STANDARD_COLUMNS.map((column) => [column.name, column.label]),
|
||
['analysis_price_range', '价格区间'],
|
||
['lowest_after_discount_price', '最低标价折后价'],
|
||
['similar_group_code', '相似款分组编码'],
|
||
]));
|
||
|
||
const MARKET_BASIC_COMPARE_PRIORITY_COLUMNS = Object.freeze([
|
||
'fulfillment_category_level1',
|
||
'fulfillment_category_level2',
|
||
'fulfillment_category_level3',
|
||
'market_common_category_name',
|
||
'appearance_recognition_code',
|
||
'item_id',
|
||
'main_image_url',
|
||
'title',
|
||
'analysis_sales',
|
||
'analysis_price_range',
|
||
'lowest_list_price',
|
||
'lowest_after_discount_price',
|
||
'sku_price_range',
|
||
'actual_hand_price',
|
||
'price_band',
|
||
'price_band_rank',
|
||
'shop_type',
|
||
'shop_name',
|
||
'search_keyword',
|
||
'product_keywords',
|
||
'industry_rank',
|
||
'paying_buyers',
|
||
'visitors',
|
||
'trend',
|
||
'time_dimension',
|
||
'sales_amount',
|
||
'similar_group_code',
|
||
'may_appearance_recognition_code',
|
||
'may_sku_image_url',
|
||
'may_sku_code',
|
||
'may_core_material',
|
||
'may_min_limit_price',
|
||
'may_size',
|
||
'may_color_name',
|
||
'may_performance_amount',
|
||
'may_goods_id_count',
|
||
'may_d_style_efficiency',
|
||
'may_inventory_quantity',
|
||
'may_estimated_sales_days',
|
||
'may_shipping_warehouse',
|
||
'may_avg_daily_sales_30d',
|
||
'may_avg_daily_sales_7d',
|
||
'may_sales_growth_rate_15d',
|
||
'may_listing_duration_days',
|
||
'may_gross_margin_rate',
|
||
'may_after_sales_ratio',
|
||
'may_active_sales_day_ratio_30d',
|
||
]);
|
||
|
||
const MARKET_BASIC_COMPARE_HIDDEN_COLUMNS = new Set([
|
||
'id',
|
||
'planning_date',
|
||
'category_level1',
|
||
'category_level2',
|
||
'category_level3',
|
||
'product_material',
|
||
'core_material_name',
|
||
'standard_product_type',
|
||
'core_product_type_name',
|
||
'market_category_name',
|
||
'browse_users',
|
||
'search_users',
|
||
'favorite_users',
|
||
'cart_users',
|
||
'purchase_users',
|
||
'conversion_rate',
|
||
'search_user_ratio',
|
||
'cart_rate',
|
||
'requested_item_id',
|
||
'price',
|
||
'original_price',
|
||
'total_sold',
|
||
'category_id',
|
||
'location',
|
||
'shop_id',
|
||
'detail_url',
|
||
'source_url',
|
||
'detail_source',
|
||
'detail_storage_record_id',
|
||
'normalized_item_json',
|
||
'raw_data_json',
|
||
'raw_response_json',
|
||
'analysis_json',
|
||
'analysis_headers_json',
|
||
'analysis_meta_json',
|
||
'analysis_history_json',
|
||
'detail_fetched_at',
|
||
'analysis_updated_at',
|
||
'created_at',
|
||
'updated_at',
|
||
'embedding_provider',
|
||
'embedding_model',
|
||
'embedding_vector_json',
|
||
'planning_upload_batch_id',
|
||
'planning_uploaded_at',
|
||
]);
|
||
|
||
let templateSummariesCache = {
|
||
expiresAt: 0,
|
||
value: null,
|
||
pending: null,
|
||
};
|
||
|
||
const tableDataCache = new Map();
|
||
|
||
const TEMPLATE_CATALOG = Object.freeze([
|
||
{
|
||
kind: 'price_band_rank_board',
|
||
label: '价格带榜单数据',
|
||
description: '识别价格带排名、行业排名、趋势等榜单字段',
|
||
templateFileName: '价格带总榜单模板.xlsx',
|
||
downloadPath: '/templates/planning/planning-price-band-rank-template.xlsx',
|
||
targetTableName: 'planning_price_band_rank_board',
|
||
legacyTableName: '生意参谋价格带总榜单',
|
||
fileKeywords: ['价格带总榜单'],
|
||
sheetKeywords: ['价格带总榜单'],
|
||
requiredHeaders: ['价格带', '价格带排名', '日期', '行业排名', '商品ID'],
|
||
expectedHeaders: ['价格带', '价格带排名', '日期', '行业排名', '趋势', '类目名称', '商品名称', '产品材质', '核心主材质', '标准产品类型', '核心产品类型', '市场分类名', '商品图片', '商品图片链接', '商品ID', '商品关键词', '店铺名称', '店铺类型', '店铺打标', '支付买家数', '访客数', '浏览人数', '搜索人数', '收藏人数', '加购人数', '购买人数', '转化率', '搜索人数占比', '加购率'],
|
||
canonicalHeaders: ['market_price_band', 'market_price_band_rank', 'snapshot_period', 'market_industry_rank', 'market_trend', 'market_category_path', 'market_goods_title', 'product_material_text', 'material_core', 'product_type_standard', 'product_type_core', 'market_category_name', 'market_goods_image_formula', 'market_goods_image_url', 'market_goods_id', 'market_keyword_tags', 'shop_name', 'shop_type', 'shop_tag', 'pay_buyer_band', 'visitor_band', 'view_uv', 'search_uv', 'collect_uv', 'cart_uv', 'buyer_count', 'conversion_rate', 'search_uv_share', 'cart_rate'],
|
||
},
|
||
{
|
||
kind: 'search_sales_price_band',
|
||
label: '搜索销量+价格带',
|
||
description: '识别搜索销量、价格带、市场上榜链接等字段',
|
||
templateFileName: '搜索销量+价格带的数据模板.xlsx',
|
||
downloadPath: '/templates/planning/planning-search-sales-price-band-template.xlsx',
|
||
targetTableName: 'planning_search_sales_price_band',
|
||
legacyTableName: '生意参谋搜索销量价格带',
|
||
fileKeywords: ['搜索销量+价格带', '搜索销量'],
|
||
sheetKeywords: ['搜索销量+价格带', '搜索销量'],
|
||
requiredHeaders: ['月份', '标题', '宝贝id', '关键词', '所属价格带', '关键词下价格带排名'],
|
||
expectedHeaders: ['月份', '标题', '链接', '宝贝id', '主图链接', '到手价', '收货人数', '店铺名称', '店铺类型', '是否品牌店铺', '关键词', '所属价格带', '实际价格段', '产品材质', '核心主材质', '标准产品类型', '核心产品类型', '市场分类名', '销售额', '浏览人数', '搜索人数', '收藏人数', '加购人数', '购买人数', '转化率', '搜索人数占比', '加购率', '是否是市场上榜链接', '上榜链接行业排名', '上榜链接所属价格带', '上榜链接价格带排名', '关键词下价格带排名'],
|
||
canonicalHeaders: ['report_month', 'market_goods_title', 'market_goods_url', 'market_goods_id', 'market_goods_image_url', 'deal_price_text', 'deal_buyer_count', 'shop_name', 'shop_type', 'is_brand_shop', 'search_keyword', 'search_price_band', 'actual_price_band', 'product_material_text', 'material_core', 'product_type_standard', 'product_type_core', 'market_category_name', 'sales_amount', 'view_uv', 'search_uv', 'collect_uv', 'cart_uv', 'buyer_count', 'conversion_rate', 'search_uv_share', 'cart_rate', 'is_ranked_market_item', 'ranked_industry_rank', 'ranked_price_band', 'ranked_price_band_rank', 'keyword_price_band_rank'],
|
||
optionalHeaders: ['店铺打标', '铺货一级类目', '铺货二级类目', '铺货三级类目', '市场共性分类名', '三级铺货类目', APPEARANCE_RECOGNITION_CODE_LABEL],
|
||
optionalCanonicalHeaders: ['shop_tag', 'fulfillment_category_level1', 'fulfillment_category_level2', 'fulfillment_category_level3', 'market_common_category_name', 'official_level3_distribution_category', APPEARANCE_RECOGNITION_CODE_COLUMN],
|
||
},
|
||
{
|
||
kind: 'category_rising_words',
|
||
label: '品类飙升词',
|
||
description: '识别搜索增速、搜索人气、点击率、支付转化率等字段',
|
||
templateFileName: '品类飙升词.xlsx',
|
||
downloadPath: '/templates/planning/planning-category-rising-words.xlsx',
|
||
targetTableName: 'planning_category_rising_words',
|
||
legacyTableName: '生意参谋品类飙升词',
|
||
fileKeywords: ['品类飙升词', '飙升词'],
|
||
sheetKeywords: ['飙升词'],
|
||
requiredHeaders: ['日期', '排名', '分类名称', '搜索词', '搜索增速', '搜索人气'],
|
||
expectedHeaders: ['日期', '排名', '分类名称', '搜索词', '搜索增速', '搜索人气', '点击率', '支付转化率'],
|
||
canonicalHeaders: ['snapshot_period', 'rank_position', 'market_category_name', 'search_keyword', 'search_growth_rate', 'search_popularity_band', 'click_rate', 'pay_conversion_rate'],
|
||
},
|
||
{
|
||
kind: 'category_hot_search_words',
|
||
label: '品类热搜词',
|
||
description: '识别热搜词、搜索人气、点击率、支付转化率等字段',
|
||
templateFileName: '品类热搜词.xlsx',
|
||
downloadPath: '/templates/planning/planning-category-hot-search.xlsx',
|
||
targetTableName: 'planning_category_hot_search_words',
|
||
legacyTableName: '生意参谋品类热搜词',
|
||
fileKeywords: ['品类热搜词', '热搜词'],
|
||
sheetKeywords: ['热搜词'],
|
||
requiredHeaders: ['日期', '排名', '分类名称', '搜索词', '搜索人气'],
|
||
forbiddenHeaders: ['搜索增速'],
|
||
expectedHeaders: ['日期', '排名', '分类名称', '搜索词', '搜索人气', '点击率', '支付转化率'],
|
||
canonicalHeaders: ['snapshot_period', 'rank_position', 'market_category_name', 'search_keyword', 'search_popularity_band', 'click_rate', 'pay_conversion_rate'],
|
||
},
|
||
{
|
||
kind: 'category_demand_indices',
|
||
label: '品类需求指数',
|
||
description: '沉淀搜索指数、点击指数、支付指数与转化率趋势底表',
|
||
templateFileName: '品类需求指数.xlsx',
|
||
downloadPath: '/templates/planning/planning-category-demand-indices.xlsx',
|
||
targetTableName: 'planning_category_demand_indices',
|
||
legacyTableName: '生意参谋品类需求指数',
|
||
fileKeywords: ['品类需求指数', '需求指数'],
|
||
sheetKeywords: ['需求指数'],
|
||
expectedHeaders: [
|
||
'日期',
|
||
'平台',
|
||
'一级品类',
|
||
'二级品类',
|
||
'三级品类',
|
||
'分类名称',
|
||
'搜索词',
|
||
'词根',
|
||
'搜索指数',
|
||
'点击指数',
|
||
'支付指数',
|
||
'点击率',
|
||
'支付转化率',
|
||
],
|
||
optionalHeaders: [
|
||
'搜索指数环比',
|
||
'点击指数环比',
|
||
'支付指数环比',
|
||
'点击率环比',
|
||
'支付转化率环比',
|
||
],
|
||
canonicalHeaders: [
|
||
'stat_date_key',
|
||
'platform',
|
||
'primary_category_name',
|
||
'secondary_category_name',
|
||
'tertiary_category_name',
|
||
'market_category_name',
|
||
'search_keyword',
|
||
'root_word',
|
||
'search_index',
|
||
'click_index',
|
||
'pay_index',
|
||
'click_rate',
|
||
'pay_conversion_rate',
|
||
],
|
||
optionalCanonicalHeaders: [
|
||
'search_index_mom',
|
||
'click_index_mom',
|
||
'pay_index_mom',
|
||
'click_rate_mom',
|
||
'pay_conversion_rate_mom',
|
||
],
|
||
},
|
||
{
|
||
kind: 'category_price_band',
|
||
label: '品类价格带',
|
||
description: '识别价格区间、购买人数、起止时间等价格带字段',
|
||
templateFileName: '品类价格带.xlsx',
|
||
downloadPath: '/templates/planning/planning-category-price-band.xlsx',
|
||
targetTableName: 'planning_category_price_band',
|
||
legacyTableName: '生意参谋品类价格带',
|
||
fileKeywords: ['品类价格带', '价格带'],
|
||
sheetKeywords: ['品类价格带'],
|
||
requiredHeaders: ['采集时间', '采集类目', '价格区间低', '价格区间高', '价格区间低-价格区间高', '开始时间', '结束时间'],
|
||
expectedHeaders: ['采集时间', '采集类目', '价格区间低', '价格区间高', '价格区间低-价格区间高', '购买人数', '购买人数', '开始时间', '结束时间', ''],
|
||
canonicalHeaders: ['snapshot_time', 'market_category_name', 'price_band_low', 'price_band_high', 'price_band', 'buyer_count', 'buyer_count_secondary', 'period_start_date', 'period_end_date', 'report_month'],
|
||
fallbackHeaders: {
|
||
9: '统计月份',
|
||
},
|
||
},
|
||
]);
|
||
|
||
const PLANNING_TABLE_INDEX_SPECS = Object.freeze({
|
||
price_band_rank_board: [
|
||
{
|
||
name: 'idx_price_rank_period_goods',
|
||
columns: [
|
||
{ name: 'snapshot_period', prefixLength: 32 },
|
||
{ name: 'market_goods_id', prefixLength: 64 },
|
||
],
|
||
},
|
||
{
|
||
name: 'idx_price_rank_period_dims',
|
||
columns: [
|
||
{ name: 'snapshot_period', prefixLength: 32 },
|
||
{ name: 'market_category_name', prefixLength: 120 },
|
||
{ name: 'market_common_category_name', prefixLength: 120 },
|
||
{ name: 'product_type_core', prefixLength: 100 },
|
||
{ name: 'material_core', prefixLength: 100 },
|
||
{ name: 'market_price_band', prefixLength: 60 },
|
||
],
|
||
},
|
||
],
|
||
search_sales_price_band: [
|
||
{
|
||
name: 'idx_search_sales_month_goods',
|
||
columns: [
|
||
{ name: 'report_month', prefixLength: 32 },
|
||
{ name: 'market_goods_id', prefixLength: 64 },
|
||
],
|
||
},
|
||
{
|
||
name: 'idx_search_sales_month_dims',
|
||
columns: [
|
||
{ name: 'report_month', prefixLength: 32 },
|
||
{ name: 'market_category_name', prefixLength: 120 },
|
||
{ name: 'product_type_core', prefixLength: 100 },
|
||
{ name: 'material_core', prefixLength: 100 },
|
||
{ name: 'actual_price_band', prefixLength: 60 },
|
||
],
|
||
},
|
||
{
|
||
name: 'idx_search_sales_month_keyword',
|
||
columns: [
|
||
{ name: 'report_month', prefixLength: 32 },
|
||
{ name: 'market_category_name', prefixLength: 120 },
|
||
{ name: 'search_keyword', prefixLength: 120 },
|
||
],
|
||
},
|
||
{
|
||
name: 'idx_search_sales_goods_id',
|
||
columns: [
|
||
{ name: 'market_goods_id', prefixLength: 64 },
|
||
],
|
||
},
|
||
],
|
||
category_rising_words: [
|
||
{
|
||
name: 'idx_rising_words_period_category_keyword',
|
||
columns: [
|
||
{ name: 'snapshot_period', prefixLength: 32 },
|
||
{ name: 'market_category_name', prefixLength: 120 },
|
||
{ name: 'search_keyword', prefixLength: 120 },
|
||
],
|
||
},
|
||
],
|
||
category_hot_search_words: [
|
||
{
|
||
name: 'idx_hot_words_period_category_keyword',
|
||
columns: [
|
||
{ name: 'snapshot_period', prefixLength: 32 },
|
||
{ name: 'market_category_name', prefixLength: 120 },
|
||
{ name: 'search_keyword', prefixLength: 120 },
|
||
],
|
||
},
|
||
],
|
||
category_demand_indices: [
|
||
{
|
||
name: 'idx_demand_indices_date_primary_root',
|
||
columns: [
|
||
{ name: 'stat_date_key', prefixLength: 32 },
|
||
{ name: 'primary_category_name', prefixLength: 80 },
|
||
{ name: 'root_word', prefixLength: 120 },
|
||
],
|
||
},
|
||
{
|
||
name: 'idx_demand_indices_date_category_keyword',
|
||
columns: [
|
||
{ name: 'stat_date_key', prefixLength: 32 },
|
||
{ name: 'market_category_name', prefixLength: 120 },
|
||
{ name: 'search_keyword', prefixLength: 120 },
|
||
],
|
||
},
|
||
],
|
||
category_price_band: [
|
||
{
|
||
name: 'idx_category_price_month_category_band',
|
||
columns: [
|
||
{ name: 'report_month', prefixLength: 32 },
|
||
{ name: 'market_category_name', prefixLength: 120 },
|
||
{ name: 'price_band', prefixLength: 80 },
|
||
],
|
||
},
|
||
{
|
||
name: 'idx_category_price_snapshot_category',
|
||
columns: [
|
||
{ name: 'snapshot_time', prefixLength: 32 },
|
||
{ name: 'market_category_name', prefixLength: 120 },
|
||
],
|
||
},
|
||
],
|
||
});
|
||
|
||
const META_COLUMNS = [
|
||
'upload_batch_id',
|
||
'source_file_name',
|
||
'source_sheet_name',
|
||
'source_sheet_type',
|
||
'source_row_number',
|
||
'uploaded_by_user_id',
|
||
'uploaded_by_username',
|
||
'uploaded_by_name',
|
||
'uploaded_department_id',
|
||
'uploaded_department_name',
|
||
'raw_data',
|
||
ROW_FINGERPRINT_COLUMN,
|
||
];
|
||
|
||
const META_COLUMN_DEFINITIONS = Object.freeze({
|
||
upload_batch_id: 'VARCHAR(36) NULL',
|
||
source_file_name: 'VARCHAR(255) NULL',
|
||
source_sheet_name: 'VARCHAR(100) NULL',
|
||
source_sheet_type: 'VARCHAR(64) NULL',
|
||
source_row_number: 'INT NULL',
|
||
uploaded_by_user_id: 'BIGINT NULL',
|
||
uploaded_by_username: 'VARCHAR(100) NULL',
|
||
uploaded_by_name: 'VARCHAR(100) NULL',
|
||
uploaded_department_id: 'BIGINT NULL',
|
||
uploaded_department_name: 'VARCHAR(100) NULL',
|
||
raw_data: 'JSON NULL',
|
||
});
|
||
|
||
const HIDDEN_VIEW_COLUMNS = new Set([
|
||
'id',
|
||
'upload_batch_id',
|
||
'source_file_name',
|
||
'source_sheet_name',
|
||
'source_sheet_type',
|
||
'source_row_number',
|
||
'uploaded_by_user_id',
|
||
'uploaded_by_username',
|
||
'uploaded_by_name',
|
||
'uploaded_department_id',
|
||
'uploaded_department_name',
|
||
'raw_data',
|
||
'created_at',
|
||
ROW_FINGERPRINT_COLUMN,
|
||
]);
|
||
|
||
const TRAILING_META_COLUMNS = [];
|
||
let canonicalSchemaSyncState = {
|
||
lightReady: false,
|
||
fullReady: false,
|
||
lightPending: null,
|
||
fullPending: null,
|
||
};
|
||
|
||
function clonePlainData(value) {
|
||
if (typeof structuredClone === 'function') {
|
||
return structuredClone(value);
|
||
}
|
||
|
||
return JSON.parse(JSON.stringify(value));
|
||
}
|
||
|
||
function getTableDataCacheKey(kind, page, pageSize, options = {}) {
|
||
const mayMatch = normalizeText(options.mayMatch || 'all') || 'all';
|
||
const reviewAction = normalizeText(options.reviewAction || 'all') || 'all';
|
||
const virtualCategory = normalizeText(options.virtualCategory || options.mode || 'all') || 'all';
|
||
const categoryName = normalizeText(options.categoryName || options.category || options.category_name || 'all') || 'all';
|
||
const supplierName = normalizeText(options.supplierName || options.supplier || options.supplier_name || 'all') || 'all';
|
||
const missingMetric = normalizeText(options.missingMetric || options.missing_metric || 'all') || 'all';
|
||
const inventoryScope = normalizeText(options.inventoryScope || options.inventory_scope || options.stockPath || options.stock_path || 'all') || 'all';
|
||
const brandGroup = normalizeText(options.brandGroup || options.brand_group || 'all') || 'all';
|
||
const brandName = normalizeText(options.brandName || options.brand || options.brand_name || 'all') || 'all';
|
||
const snapshotDate = normalizeText(options.snapshotDate || 'latest') || 'latest';
|
||
const realtime = normalizeText(options.realtime || options.forceRealtime || '') === '1' ? '1' : '0';
|
||
const cacheVersion = normalizeText(options.cacheVersion || 'default') || 'default';
|
||
return `${kind}:${page}:${pageSize}:mayMatch=${mayMatch}:reviewAction=${reviewAction}:virtualCategory=${virtualCategory}:inventoryScope=${inventoryScope}:brandGroup=${brandGroup}:brandName=${brandName}:categoryName=${categoryName}:supplierName=${supplierName}:missingMetric=${missingMetric}:snapshotDate=${snapshotDate}:realtime=${realtime}:version=${cacheVersion}`;
|
||
}
|
||
|
||
function shouldUseTableDataCache(kind, options = {}) {
|
||
if (kind === MAY_PRODUCT_DATA_KIND) {
|
||
return String(options.realtime || options.forceRealtime || '') !== '1';
|
||
}
|
||
return true;
|
||
}
|
||
|
||
function createPlanningExportError(message, status = 400) {
|
||
const error = new Error(message);
|
||
error.status = status;
|
||
return error;
|
||
}
|
||
|
||
function normalizeWorkbookCellValue(value) {
|
||
if (value === undefined || value === null) {
|
||
return '';
|
||
}
|
||
if (value instanceof Date) {
|
||
return value;
|
||
}
|
||
if (Array.isArray(value)) {
|
||
return value.join('、');
|
||
}
|
||
if (typeof value === 'object') {
|
||
return JSON.stringify(value);
|
||
}
|
||
return value;
|
||
}
|
||
|
||
function buildWorkbookColumnWidths(columns = [], rows = []) {
|
||
return columns.map((column) => {
|
||
const label = String(column.label || column.name || '');
|
||
const sampleWidth = rows.slice(0, 200).reduce((maxWidth, row) => {
|
||
const text = String(normalizeWorkbookCellValue(row[column.name]) ?? '');
|
||
return Math.max(maxWidth, Math.min(text.length, 60));
|
||
}, label.length);
|
||
return {
|
||
wch: Math.max(10, Math.min(sampleWidth + 2, 64)),
|
||
};
|
||
});
|
||
}
|
||
|
||
function buildWorkbookBufferFromPlanningRows({ columns = [], rows = [], sheetName = '明细' }) {
|
||
const visibleColumns = columns.filter((column) => column?.name);
|
||
const header = visibleColumns.map((column) => column.label || column.name);
|
||
const body = rows.map((row) => (
|
||
visibleColumns.map((column) => normalizeWorkbookCellValue(row[column.name]))
|
||
));
|
||
const worksheet = XLSX.utils.aoa_to_sheet([header, ...body]);
|
||
worksheet['!cols'] = buildWorkbookColumnWidths(visibleColumns, rows);
|
||
const workbook = XLSX.utils.book_new();
|
||
XLSX.utils.book_append_sheet(workbook, worksheet, sheetName.slice(0, 31) || '明细');
|
||
return XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' });
|
||
}
|
||
|
||
function formatExportTimestamp(date = new Date()) {
|
||
const pad = (value) => String(value).padStart(2, '0');
|
||
return [
|
||
date.getFullYear(),
|
||
pad(date.getMonth() + 1),
|
||
pad(date.getDate()),
|
||
].join('') + '-' + [
|
||
pad(date.getHours()),
|
||
pad(date.getMinutes()),
|
||
pad(date.getSeconds()),
|
||
].join('');
|
||
}
|
||
|
||
function getTableDataCacheTtlMs(kind) {
|
||
if (kind === MAY_PRODUCT_DATA_KIND) {
|
||
return MAY_TABLE_DATA_CACHE_TTL_MS;
|
||
}
|
||
return TABLE_DATA_CACHE_TTL_MS;
|
||
}
|
||
|
||
function invalidatePlanningRuntimeCache() {
|
||
templateSummariesCache = {
|
||
expiresAt: 0,
|
||
value: null,
|
||
pending: null,
|
||
};
|
||
tableDataCache.clear();
|
||
}
|
||
|
||
function normalizeText(value) {
|
||
if (value === undefined || value === null) {
|
||
return '';
|
||
}
|
||
|
||
return String(value).trim();
|
||
}
|
||
|
||
function normalizeHeaderText(value) {
|
||
return normalizeText(value).replace(/\s+/g, '');
|
||
}
|
||
|
||
function normalizeNullableText(value) {
|
||
const normalized = normalizeText(value);
|
||
return normalized || null;
|
||
}
|
||
|
||
function normalizeGoodsId(value) {
|
||
return normalizeText(value).replace(/\.0$/u, '');
|
||
}
|
||
|
||
function safeJsonParse(value) {
|
||
if (!value) {
|
||
return null;
|
||
}
|
||
|
||
if (typeof value === 'object') {
|
||
return value;
|
||
}
|
||
|
||
try {
|
||
return JSON.parse(String(value));
|
||
} catch (_error) {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function normalizePriceNumber(value) {
|
||
if (value === undefined || value === null || value === '') {
|
||
return null;
|
||
}
|
||
|
||
if (typeof value === 'number' && Number.isFinite(value)) {
|
||
return value;
|
||
}
|
||
|
||
const normalized = String(value)
|
||
.replace(/[,，]/g, '')
|
||
.replace(/[￥¥元]/g, '')
|
||
.trim();
|
||
if (!normalized) {
|
||
return null;
|
||
}
|
||
|
||
const match = normalized.match(/\d+(?:\.\d+)?/u);
|
||
if (!match) {
|
||
return null;
|
||
}
|
||
|
||
const parsed = Number(match[0]);
|
||
if (!Number.isFinite(parsed)) {
|
||
return null;
|
||
}
|
||
|
||
return parsed;
|
||
}
|
||
|
||
function normalizeSkuPriceNumber(value) {
|
||
const parsed = normalizePriceNumber(value);
|
||
if (parsed === null) {
|
||
return null;
|
||
}
|
||
|
||
const raw = normalizeText(value);
|
||
if (/^\d+$/u.test(raw) && parsed >= 10000) {
|
||
return parsed / 100;
|
||
}
|
||
|
||
return parsed;
|
||
}
|
||
|
||
function getPriceNumbersFromText(value) {
|
||
const normalized = normalizeText(value)
|
||
.replace(/[,，]/g, '')
|
||
.replace(/[￥¥元]/g, ' ');
|
||
if (!normalized) {
|
||
return [];
|
||
}
|
||
|
||
const matches = normalized.match(/\d+(?:\.\d+)?/gu) || [];
|
||
return matches
|
||
.map((item) => Number(item))
|
||
.filter((item) => Number.isFinite(item));
|
||
}
|
||
|
||
function formatPriceNumber(value) {
|
||
if (!Number.isFinite(value)) {
|
||
return null;
|
||
}
|
||
|
||
return value.toFixed(2).replace(/\.00$/u, '').replace(/(\.\d)0$/u, '$1');
|
||
}
|
||
|
||
function getMinimumPrice(values = []) {
|
||
const numbers = values
|
||
.map((value) => (typeof value === 'number' ? value : normalizePriceNumber(value)))
|
||
.filter((value) => Number.isFinite(value));
|
||
|
||
if (numbers.length === 0) {
|
||
return null;
|
||
}
|
||
|
||
return Math.min(...numbers);
|
||
}
|
||
|
||
function getSkuList(value) {
|
||
const parsed = safeJsonParse(value);
|
||
if (!parsed) {
|
||
return [];
|
||
}
|
||
|
||
if (Array.isArray(parsed)) {
|
||
return parsed;
|
||
}
|
||
|
||
if (Array.isArray(parsed.sku)) {
|
||
return parsed.sku;
|
||
}
|
||
|
||
if (Array.isArray(parsed.skus)) {
|
||
return parsed.skus;
|
||
}
|
||
|
||
if (parsed.skus) {
|
||
return getSkuList(parsed.skus);
|
||
}
|
||
|
||
return [];
|
||
}
|
||
|
||
function getSkuMinimumPrice(skuSources = [], fieldNames = []) {
|
||
const values = [];
|
||
|
||
for (const source of skuSources) {
|
||
for (const sku of getSkuList(source)) {
|
||
for (const fieldName of fieldNames) {
|
||
const price = normalizeSkuPriceNumber(sku?.[fieldName]);
|
||
if (Number.isFinite(price)) {
|
||
values.push(price);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (values.length === 0) {
|
||
return null;
|
||
}
|
||
|
||
return Math.min(...values);
|
||
}
|
||
|
||
function getNestedValue(object, path = []) {
|
||
let current = object;
|
||
for (const key of path) {
|
||
if (!current || typeof current !== 'object') {
|
||
return null;
|
||
}
|
||
current = current[key];
|
||
}
|
||
|
||
return current ?? null;
|
||
}
|
||
|
||
function getPromotionPriceCandidates(...sources) {
|
||
const candidates = [];
|
||
const paths = [
|
||
['mainItemInfo', 'taobaoPromotionModel', 'finalPromotionInfo', 'finalPromotionPrice'],
|
||
['mainItemInfo', 'promotionModel', 'promotionPriceModel', 'promotionPrice'],
|
||
['mainItemInfo', 'afterCouponAmountPrice'],
|
||
['mainItemInfo', 'promotionModel', 'promotionPriceModel', 'zkPrice'],
|
||
];
|
||
|
||
for (const source of sources) {
|
||
const parsed = safeJsonParse(source);
|
||
if (!parsed || typeof parsed !== 'object') {
|
||
continue;
|
||
}
|
||
|
||
for (const path of paths) {
|
||
const price = normalizePriceNumber(getNestedValue(parsed, path));
|
||
if (Number.isFinite(price)) {
|
||
candidates.push(price);
|
||
}
|
||
}
|
||
}
|
||
|
||
return candidates;
|
||
}
|
||
|
||
function countChineseCharacters(value) {
|
||
const matches = String(value || '').match(/[\u4e00-\u9fff]/g);
|
||
return matches ? matches.length : 0;
|
||
}
|
||
|
||
function countMojibakeCharacters(value) {
|
||
const matches = String(value || '').match(/[À-ÿ]/g);
|
||
return matches ? matches.length : 0;
|
||
}
|
||
|
||
function normalizeUploadedFileName(fileName) {
|
||
const normalized = normalizeText(fileName);
|
||
if (!normalized) {
|
||
return '';
|
||
}
|
||
|
||
let decoded = normalized;
|
||
try {
|
||
decoded = Buffer.from(normalized, 'latin1').toString('utf8');
|
||
} catch (_error) {
|
||
return normalized;
|
||
}
|
||
|
||
if (!decoded || decoded.includes('\uFFFD')) {
|
||
return normalized;
|
||
}
|
||
|
||
const originalChineseCount = countChineseCharacters(normalized);
|
||
const decodedChineseCount = countChineseCharacters(decoded);
|
||
const originalMojibakeCount = countMojibakeCharacters(normalized);
|
||
const decodedMojibakeCount = countMojibakeCharacters(decoded);
|
||
|
||
const decodedLooksBetter = decodedChineseCount > originalChineseCount
|
||
|| (originalChineseCount === 0 && originalMojibakeCount > 0 && decodedMojibakeCount < originalMojibakeCount);
|
||
|
||
return decodedLooksBetter ? decoded : normalized;
|
||
}
|
||
|
||
function createPlanningUploadValidationError(message) {
|
||
const error = new Error(message);
|
||
error.status = 400;
|
||
return error;
|
||
}
|
||
|
||
function escapeIdentifier(value) {
|
||
return `\`${String(value).replace(/`/g, '``')}\``;
|
||
}
|
||
|
||
function escapeSqlComment(value) {
|
||
return String(value || '').replace(/\\/g, '\\\\').replace(/'/g, "\\'");
|
||
}
|
||
|
||
function hasAnyValue(row = []) {
|
||
return row.some((cell) => normalizeText(cell) !== '');
|
||
}
|
||
|
||
function matchesKeywords(text, keywords = []) {
|
||
const normalized = normalizeText(text);
|
||
return keywords.some((keyword) => normalized.includes(keyword));
|
||
}
|
||
|
||
function hasRequiredHeaders(headers = [], requiredHeaders = []) {
|
||
const headerSet = new Set(headers.map((header) => normalizeHeaderText(header)).filter(Boolean));
|
||
return requiredHeaders.every((header) => headerSet.has(normalizeHeaderText(header)));
|
||
}
|
||
|
||
function hasForbiddenHeaders(headers = [], forbiddenHeaders = []) {
|
||
const headerSet = new Set(headers.map((header) => normalizeHeaderText(header)).filter(Boolean));
|
||
return forbiddenHeaders.some((header) => headerSet.has(normalizeHeaderText(header)));
|
||
}
|
||
|
||
function hasExactHeaders(headers = [], expectedHeaders = [], optionalHeaders = []) {
|
||
if (!Array.isArray(expectedHeaders) || expectedHeaders.length === 0) {
|
||
return false;
|
||
}
|
||
|
||
const actual = headers.map((header) => normalizeHeaderText(header));
|
||
const expected = expectedHeaders.map((header) => normalizeHeaderText(header));
|
||
const optionalSet = new Set(optionalHeaders.map((header) => normalizeHeaderText(header)).filter(Boolean));
|
||
|
||
if (actual.length < expected.length) {
|
||
return false;
|
||
}
|
||
|
||
const matchesExpectedHeaders = expected.every((header, index) => actual[index] === header);
|
||
if (!matchesExpectedHeaders) {
|
||
return false;
|
||
}
|
||
|
||
return actual.slice(expected.length).every((header) => !header || optionalSet.has(header));
|
||
}
|
||
|
||
function getOptionalCanonicalHeaderMap(template) {
|
||
const optionalHeaders = Array.isArray(template?.optionalHeaders) ? template.optionalHeaders : [];
|
||
const optionalCanonicalHeaders = Array.isArray(template?.optionalCanonicalHeaders) ? template.optionalCanonicalHeaders : [];
|
||
const optionalCanonicalMap = new Map();
|
||
|
||
optionalHeaders.forEach((header, index) => {
|
||
const normalizedHeader = normalizeHeaderText(header);
|
||
const canonicalHeader = normalizeText(optionalCanonicalHeaders[index]);
|
||
if (normalizedHeader && canonicalHeader) {
|
||
optionalCanonicalMap.set(normalizedHeader, canonicalHeader);
|
||
}
|
||
});
|
||
|
||
return optionalCanonicalMap;
|
||
}
|
||
|
||
function getTemplateHeaderLimit(template, headerRow = []) {
|
||
if (!template || !Array.isArray(headerRow)) {
|
||
return 0;
|
||
}
|
||
|
||
const expectedLength = Array.isArray(template.expectedHeaders) && template.expectedHeaders.length > 0
|
||
? template.expectedHeaders.length
|
||
: headerRow.length;
|
||
|
||
if (expectedLength >= headerRow.length) {
|
||
return expectedLength;
|
||
}
|
||
|
||
const optionalSet = new Set((template.optionalHeaders || []).map((header) => normalizeHeaderText(header)).filter(Boolean));
|
||
if (optionalSet.size === 0) {
|
||
return expectedLength;
|
||
}
|
||
|
||
let limit = expectedLength;
|
||
for (let index = expectedLength; index < headerRow.length; index += 1) {
|
||
const header = normalizeHeaderText(headerRow[index]);
|
||
if (!header) {
|
||
continue;
|
||
}
|
||
if (!optionalSet.has(header)) {
|
||
break;
|
||
}
|
||
limit = index + 1;
|
||
}
|
||
|
||
return limit;
|
||
}
|
||
|
||
function getBusinessColumnNames(columnNames = []) {
|
||
const metaColumnSet = new Set(['id', 'created_at', ...META_COLUMNS]);
|
||
return columnNames.filter((columnName) => !metaColumnSet.has(columnName));
|
||
}
|
||
|
||
function getExpectedTemplateColumnNames(template) {
|
||
if (!template || !Array.isArray(template.expectedHeaders) || template.expectedHeaders.length === 0) {
|
||
return [];
|
||
}
|
||
|
||
return buildTemplateSourceColumns(template).map((column) => column.columnName);
|
||
}
|
||
|
||
function buildRowFingerprint(columnNames = [], rowLike = {}) {
|
||
const payload = {};
|
||
for (const columnName of columnNames) {
|
||
payload[columnName] = normalizeNullableText(rowLike[columnName]);
|
||
}
|
||
|
||
return crypto.createHash('sha256').update(JSON.stringify(payload)).digest('hex');
|
||
}
|
||
|
||
function detectSheetTemplate(fileName, sheetName, headers = [], options = {}) {
|
||
const candidates = options.expectedKind
|
||
? TEMPLATE_CATALOG.filter((template) => template.kind === options.expectedKind)
|
||
: TEMPLATE_CATALOG;
|
||
|
||
if (candidates.length === 0) {
|
||
return null;
|
||
}
|
||
|
||
return candidates.find((template) => {
|
||
if (Array.isArray(template.expectedHeaders) && template.expectedHeaders.length > 0) {
|
||
return hasExactHeaders(headers, template.expectedHeaders, template.optionalHeaders);
|
||
}
|
||
|
||
if (!hasRequiredHeaders(headers, template.requiredHeaders)) {
|
||
return false;
|
||
}
|
||
|
||
if (hasForbiddenHeaders(headers, template.forbiddenHeaders)) {
|
||
return false;
|
||
}
|
||
|
||
if (matchesKeywords(fileName, template.fileKeywords)) {
|
||
return true;
|
||
}
|
||
|
||
if (matchesKeywords(sheetName, template.sheetKeywords)) {
|
||
return true;
|
||
}
|
||
|
||
return true;
|
||
}) || null;
|
||
}
|
||
|
||
function getTemplateMergedHeaders(template) {
|
||
if (!template) {
|
||
return [];
|
||
}
|
||
|
||
return [
|
||
...(Array.isArray(template.expectedHeaders) ? template.expectedHeaders : []),
|
||
...(Array.isArray(template.optionalHeaders) ? template.optionalHeaders : []),
|
||
];
|
||
}
|
||
|
||
function getTemplateMergedCanonicalHeaders(template) {
|
||
if (!template) {
|
||
return [];
|
||
}
|
||
|
||
return [
|
||
...(Array.isArray(template.canonicalHeaders) ? template.canonicalHeaders : []),
|
||
...(Array.isArray(template.optionalCanonicalHeaders) ? template.optionalCanonicalHeaders : []),
|
||
];
|
||
}
|
||
|
||
function resolveSourceColumnName(baseColumnName, seen) {
|
||
let columnName = baseColumnName;
|
||
let suffix = 2;
|
||
|
||
while (seen.has(columnName)) {
|
||
columnName = `${baseColumnName}_${suffix}`;
|
||
suffix += 1;
|
||
}
|
||
|
||
seen.add(columnName);
|
||
return columnName;
|
||
}
|
||
|
||
function buildSourceColumns(template, headerRow = [], options = {}) {
|
||
const seen = new Set();
|
||
const legacySeen = new Set();
|
||
const headerLimit = Math.min(
|
||
Math.max(Number(options.headerLimit) || headerRow.length, 0),
|
||
headerRow.length
|
||
);
|
||
const canonicalHeaders = getTemplateMergedCanonicalHeaders(template);
|
||
const optionalCanonicalMap = getOptionalCanonicalHeaderMap(template);
|
||
const expectedHeaderCount = Array.isArray(template.expectedHeaders) && template.expectedHeaders.length > 0
|
||
? template.expectedHeaders.length
|
||
: headerLimit;
|
||
|
||
return headerRow.slice(0, headerLimit).map((header, index) => {
|
||
const fallbackHeader = template.fallbackHeaders?.[index] || '';
|
||
const sourceHeader = normalizeText(header) || fallbackHeader;
|
||
if (!sourceHeader) {
|
||
return null;
|
||
}
|
||
|
||
const legacyBaseColumnName = sourceHeader.replace(/`/g, '').trim() || fallbackHeader;
|
||
const legacyColumnName = resolveSourceColumnName(legacyBaseColumnName, legacySeen);
|
||
const optionalCanonicalHeader = index >= expectedHeaderCount
|
||
? optionalCanonicalMap.get(normalizeHeaderText(sourceHeader))
|
||
: '';
|
||
const canonicalBaseColumnName = normalizeText(optionalCanonicalHeader || canonicalHeaders[index]).replace(/`/g, '').trim() || legacyBaseColumnName;
|
||
const columnName = resolveSourceColumnName(canonicalBaseColumnName, seen);
|
||
|
||
return {
|
||
index,
|
||
sourceHeader,
|
||
resolvedHeader: sourceHeader,
|
||
columnName,
|
||
legacyColumnName,
|
||
};
|
||
}).filter(Boolean);
|
||
}
|
||
|
||
function buildTemplateSourceColumns(template) {
|
||
const mergedHeaders = getTemplateMergedHeaders(template);
|
||
return buildSourceColumns(template, mergedHeaders, {
|
||
headerLimit: mergedHeaders.length,
|
||
});
|
||
}
|
||
|
||
function getTemplateCatalog() {
|
||
return TEMPLATE_CATALOG.map((template) => ({
|
||
kind: template.kind,
|
||
label: template.label,
|
||
description: template.description,
|
||
templateFileName: template.templateFileName,
|
||
downloadPath: template.downloadPath,
|
||
targetTableName: template.targetTableName,
|
||
legacyTableName: template.legacyTableName,
|
||
requiredHeaders: template.requiredHeaders,
|
||
}));
|
||
}
|
||
|
||
function getPlanningTemplateByKind(kind) {
|
||
return [...TEMPLATE_CATALOG, ...READONLY_PLANNING_PAGE_CATALOG].find((template) => template.kind === kind) || null;
|
||
}
|
||
|
||
function isPlanningDbNotConfigured(error) {
|
||
return error && error.message === NOT_CONFIGURED_MESSAGE;
|
||
}
|
||
|
||
function parseWorkbook(buffer, fileName, user, options = {}) {
|
||
const workbook = XLSX.read(buffer, {
|
||
type: 'buffer',
|
||
cellDates: false,
|
||
raw: false,
|
||
dense: false,
|
||
});
|
||
|
||
const batchId = uuidv4();
|
||
const parsedSheets = [];
|
||
let totalRows = 0;
|
||
|
||
for (const sheetName of workbook.SheetNames) {
|
||
const worksheet = workbook.Sheets[sheetName];
|
||
const rows = XLSX.utils.sheet_to_json(worksheet, {
|
||
header: 1,
|
||
defval: '',
|
||
raw: false,
|
||
blankrows: false,
|
||
});
|
||
|
||
if (!Array.isArray(rows) || rows.length === 0) {
|
||
continue;
|
||
}
|
||
|
||
const headerRow = Array.isArray(rows[0]) ? rows[0] : [];
|
||
const template = detectSheetTemplate(fileName, sheetName, headerRow, {
|
||
expectedKind: options.expectedKind,
|
||
});
|
||
if (!template) {
|
||
continue;
|
||
}
|
||
|
||
const sourceColumns = buildSourceColumns(template, headerRow, {
|
||
headerLimit: getTemplateHeaderLimit(template, headerRow),
|
||
});
|
||
const insertRows = [];
|
||
|
||
for (let rowIndex = 1; rowIndex < rows.length; rowIndex += 1) {
|
||
const rawRow = Array.isArray(rows[rowIndex]) ? rows[rowIndex] : [];
|
||
if (!hasAnyValue(rawRow)) {
|
||
continue;
|
||
}
|
||
|
||
const sourceData = {};
|
||
const rawDataColumns = [];
|
||
|
||
for (const column of sourceColumns) {
|
||
const value = normalizeNullableText(rawRow[column.index]);
|
||
sourceData[column.columnName] = value;
|
||
rawDataColumns.push({
|
||
index: column.index + 1,
|
||
sourceHeader: column.sourceHeader,
|
||
resolvedHeader: column.resolvedHeader,
|
||
columnName: column.columnName,
|
||
value,
|
||
});
|
||
}
|
||
|
||
insertRows.push({
|
||
upload_batch_id: batchId,
|
||
source_file_name: fileName,
|
||
source_sheet_name: sheetName,
|
||
source_sheet_type: template.kind,
|
||
source_row_number: rowIndex + 1,
|
||
uploaded_by_user_id: user.id || null,
|
||
uploaded_by_username: user.username || null,
|
||
uploaded_by_name: user.name || null,
|
||
uploaded_department_id: user.departmentId || null,
|
||
uploaded_department_name: user.departmentName || null,
|
||
raw_data: JSON.stringify({
|
||
templateKind: template.kind,
|
||
templateLabel: template.label,
|
||
columns: rawDataColumns,
|
||
}),
|
||
sourceData,
|
||
});
|
||
}
|
||
|
||
if (insertRows.length === 0) {
|
||
continue;
|
||
}
|
||
|
||
parsedSheets.push({
|
||
kind: template.kind,
|
||
label: template.label,
|
||
sheetName,
|
||
targetTableName: template.targetTableName,
|
||
sourceColumns,
|
||
rows: insertRows,
|
||
rowCount: insertRows.length,
|
||
});
|
||
totalRows += insertRows.length;
|
||
}
|
||
|
||
if (parsedSheets.length === 0 || totalRows === 0) {
|
||
if (options.expectedKind) {
|
||
const expectedTemplate = getPlanningTemplateByKind(options.expectedKind);
|
||
if (expectedTemplate) {
|
||
throw createPlanningUploadValidationError(`当前上传入口仅支持「${expectedTemplate.label}」模板，请严格按模板表头上传`);
|
||
}
|
||
}
|
||
throw createPlanningUploadValidationError('未识别到可导入的数据工作表，请使用约定模板上传');
|
||
}
|
||
|
||
return {
|
||
batchId,
|
||
totalRows,
|
||
targetTableNames: Array.from(new Set(parsedSheets.map((sheet) => sheet.targetTableName))),
|
||
parsedSheets,
|
||
};
|
||
}
|
||
|
||
function ensureExpectedTemplate(parsed, expectedKind) {
|
||
if (!expectedKind) {
|
||
return parsed;
|
||
}
|
||
|
||
const expectedTemplate = getPlanningTemplateByKind(expectedKind);
|
||
if (!expectedTemplate) {
|
||
throw createPlanningUploadValidationError('指定的企划模板类型无效');
|
||
}
|
||
|
||
const hasUnexpectedTemplate = parsed.parsedSheets.some((sheet) => sheet.kind !== expectedKind);
|
||
const hasExpectedTemplate = parsed.parsedSheets.some((sheet) => sheet.kind === expectedKind);
|
||
|
||
if (!hasExpectedTemplate || hasUnexpectedTemplate) {
|
||
throw createPlanningUploadValidationError(`当前上传入口仅支持「${expectedTemplate.label}」模板，请上传对应 Excel`);
|
||
}
|
||
|
||
return parsed;
|
||
}
|
||
|
||
async function getExistingColumns(connection, tableName) {
|
||
const [rows] = await connection.query(
|
||
`SELECT column_name AS columnName
|
||
FROM information_schema.columns
|
||
WHERE table_schema = DATABASE()
|
||
AND table_name = ?`,
|
||
[tableName]
|
||
);
|
||
|
||
return new Set(rows.map((row) => row.columnName));
|
||
}
|
||
|
||
async function getTableColumnNames(connection, tableName) {
|
||
const [rows] = await connection.query(
|
||
`SELECT column_name AS columnName
|
||
FROM information_schema.columns
|
||
WHERE table_schema = DATABASE()
|
||
AND table_name = ?
|
||
ORDER BY ordinal_position ASC`,
|
||
[tableName]
|
||
);
|
||
|
||
return rows.map((row) => row.columnName);
|
||
}
|
||
|
||
async function getPlanningStandaloneColumnSet(tableName) {
|
||
const rows = await planningDatabase.query(
|
||
`SELECT column_name AS columnName
|
||
FROM information_schema.columns
|
||
WHERE table_schema = DATABASE()
|
||
AND table_name = ?`,
|
||
[tableName]
|
||
);
|
||
|
||
return new Set(rows.map((row) => row.columnName));
|
||
}
|
||
|
||
async function ensurePlanningMetaColumns(connection, tableName, existingColumns = null) {
|
||
const columnSet = existingColumns || await getExistingColumns(connection, tableName);
|
||
|
||
for (const [columnName, definition] of Object.entries(META_COLUMN_DEFINITIONS)) {
|
||
if (columnSet.has(columnName)) {
|
||
continue;
|
||
}
|
||
|
||
await connection.query(
|
||
`ALTER TABLE ${escapeIdentifier(tableName)}
|
||
ADD COLUMN ${escapeIdentifier(columnName)} ${definition}`
|
||
);
|
||
columnSet.add(columnName);
|
||
}
|
||
|
||
return columnSet;
|
||
}
|
||
|
||
async function getTableIndexes(connection, tableName) {
|
||
const [rows] = await connection.query(
|
||
`SELECT DISTINCT index_name AS indexName
|
||
FROM information_schema.statistics
|
||
WHERE table_schema = DATABASE()
|
||
AND table_name = ?`,
|
||
[tableName]
|
||
);
|
||
|
||
return new Set(rows.map((row) => row.indexName));
|
||
}
|
||
|
||
function buildPlanningIndexColumnSql(columnSpec) {
|
||
const columnName = normalizeText(columnSpec?.name || columnSpec);
|
||
const prefixLength = Number(columnSpec?.prefixLength || 0);
|
||
if (!columnName) {
|
||
return '';
|
||
}
|
||
|
||
return `${escapeIdentifier(columnName)}${Number.isInteger(prefixLength) && prefixLength > 0 ? `(${prefixLength})` : ''}`;
|
||
}
|
||
|
||
async function ensurePlanningTemplateIndexesForTable(connection, tableName, template) {
|
||
const tableMeta = await getPlanningTableMeta(connection, tableName);
|
||
if (!tableMeta || tableMeta.tableType === 'VIEW') {
|
||
return;
|
||
}
|
||
|
||
const indexSpecs = PLANNING_TABLE_INDEX_SPECS[template.kind] || [];
|
||
if (indexSpecs.length === 0) {
|
||
return;
|
||
}
|
||
|
||
const existingColumns = await getExistingColumns(connection, tableName);
|
||
const existingIndexes = await getTableIndexes(connection, tableName);
|
||
|
||
for (const indexSpec of indexSpecs) {
|
||
const indexName = normalizeText(indexSpec.name);
|
||
const columns = Array.isArray(indexSpec.columns) ? indexSpec.columns : [];
|
||
if (!indexName || existingIndexes.has(indexName) || columns.length === 0) {
|
||
continue;
|
||
}
|
||
|
||
const hasRequiredColumns = columns.every((columnSpec) => existingColumns.has(normalizeText(columnSpec?.name || columnSpec)));
|
||
if (!hasRequiredColumns) {
|
||
continue;
|
||
}
|
||
|
||
const columnSql = columns.map((columnSpec) => buildPlanningIndexColumnSql(columnSpec)).filter(Boolean).join(', ');
|
||
if (!columnSql) {
|
||
continue;
|
||
}
|
||
|
||
await connection.query(
|
||
`ALTER TABLE ${escapeIdentifier(tableName)}
|
||
ADD KEY ${escapeIdentifier(indexName)} (${columnSql})`
|
||
);
|
||
existingIndexes.add(indexName);
|
||
}
|
||
}
|
||
|
||
async function getPlanningTableMeta(connection, tableName) {
|
||
const [rows] = await connection.query(
|
||
`SELECT table_name AS tableName, table_type AS tableType
|
||
FROM information_schema.tables
|
||
WHERE table_schema = DATABASE()
|
||
AND table_name = ?
|
||
LIMIT 1`,
|
||
[tableName]
|
||
);
|
||
|
||
return rows[0] || null;
|
||
}
|
||
|
||
async function getAvailablePlanningArchiveTableName(connection, targetTableName) {
|
||
const timestamp = new Date().toISOString().replace(/\D/gu, '').slice(0, 14);
|
||
const baseName = `${targetTableName}_legacy_${timestamp}`;
|
||
let candidate = baseName.slice(0, 64);
|
||
let suffix = 2;
|
||
|
||
while (await getPlanningTableMeta(connection, candidate)) {
|
||
const suffixText = `_${suffix}`;
|
||
candidate = `${baseName.slice(0, 64 - suffixText.length)}${suffixText}`;
|
||
suffix += 1;
|
||
}
|
||
|
||
return candidate;
|
||
}
|
||
|
||
async function syncPlanningCanonicalColumnsForTable(connection, tableName, template, options = {}) {
|
||
const includeIndexes = options.includeIndexes !== false;
|
||
const tableExists = await getPlanningTableMeta(connection, tableName);
|
||
if (!tableExists) {
|
||
return false;
|
||
}
|
||
if (tableExists.tableType === 'VIEW') {
|
||
return false;
|
||
}
|
||
|
||
const expectedColumns = buildTemplateSourceColumns(template);
|
||
const existingColumns = await ensurePlanningMetaColumns(connection, tableName);
|
||
|
||
for (const column of expectedColumns) {
|
||
if (existingColumns.has(column.columnName)) {
|
||
continue;
|
||
}
|
||
|
||
if (column.legacyColumnName && existingColumns.has(column.legacyColumnName)) {
|
||
const comment = column.sourceHeader
|
||
? `来源表头: ${column.sourceHeader}`
|
||
: `来源列序号: ${column.index + 1}`;
|
||
await connection.query(
|
||
`ALTER TABLE ${escapeIdentifier(tableName)}
|
||
CHANGE COLUMN ${escapeIdentifier(column.legacyColumnName)} ${escapeIdentifier(column.columnName)} TEXT NULL COMMENT '${escapeSqlComment(comment)}'`
|
||
);
|
||
existingColumns.delete(column.legacyColumnName);
|
||
existingColumns.add(column.columnName);
|
||
continue;
|
||
}
|
||
|
||
await connection.query(
|
||
`ALTER TABLE ${escapeIdentifier(tableName)}
|
||
ADD COLUMN ${buildSourceColumnSql(column)}`
|
||
);
|
||
existingColumns.add(column.columnName);
|
||
}
|
||
|
||
await ensurePlanningFingerprintSupport(connection, tableName, { includeIndexes });
|
||
if (includeIndexes) {
|
||
await ensurePlanningTemplateIndexesForTable(connection, tableName, template);
|
||
}
|
||
return true;
|
||
}
|
||
|
||
async function migrateLegacyPlanningTable(connection, template, options = {}) {
|
||
const legacyTableName = normalizeText(template.legacyTableName);
|
||
const targetTableName = normalizeText(template.targetTableName);
|
||
if (!legacyTableName || !targetTableName || legacyTableName === targetTableName) {
|
||
return;
|
||
}
|
||
|
||
const legacyMeta = await getPlanningTableMeta(connection, legacyTableName);
|
||
if (!legacyMeta) {
|
||
return;
|
||
}
|
||
|
||
const targetMeta = await getPlanningTableMeta(connection, targetTableName);
|
||
if (legacyMeta.tableType === 'VIEW') {
|
||
if (targetMeta) {
|
||
await connection.query(`DROP VIEW IF EXISTS ${escapeIdentifier(legacyTableName)}`);
|
||
}
|
||
return;
|
||
}
|
||
|
||
if (!targetMeta) {
|
||
await connection.query(
|
||
`RENAME TABLE ${escapeIdentifier(legacyTableName)} TO ${escapeIdentifier(targetTableName)}`
|
||
);
|
||
return;
|
||
}
|
||
|
||
await syncPlanningCanonicalColumnsForTable(connection, targetTableName, template, options);
|
||
await syncPlanningCanonicalColumnsForTable(connection, legacyTableName, template, options);
|
||
await syncPlanningFingerprints(connection, targetTableName);
|
||
await syncPlanningFingerprints(connection, legacyTableName);
|
||
|
||
const targetColumns = await getTableColumnNames(connection, targetTableName);
|
||
const legacyColumns = new Set(await getTableColumnNames(connection, legacyTableName));
|
||
const copyColumns = targetColumns.filter((columnName) => columnName !== 'id' && legacyColumns.has(columnName));
|
||
|
||
if (copyColumns.length > 0) {
|
||
const columnSql = copyColumns.map((columnName) => escapeIdentifier(columnName)).join(', ');
|
||
const selectSql = copyColumns.map((columnName) => `legacy_src.${escapeIdentifier(columnName)}`).join(', ');
|
||
await connection.query(
|
||
`INSERT INTO ${escapeIdentifier(targetTableName)} (${columnSql})
|
||
SELECT ${selectSql}
|
||
FROM ${escapeIdentifier(legacyTableName)} legacy_src
|
||
WHERE legacy_src.${escapeIdentifier(ROW_FINGERPRINT_COLUMN)} IS NULL
|
||
OR NOT EXISTS (
|
||
SELECT 1
|
||
FROM ${escapeIdentifier(targetTableName)} target_rows
|
||
WHERE target_rows.${escapeIdentifier(ROW_FINGERPRINT_COLUMN)} = legacy_src.${escapeIdentifier(ROW_FINGERPRINT_COLUMN)}
|
||
)`
|
||
);
|
||
}
|
||
|
||
const archiveTableName = await getAvailablePlanningArchiveTableName(connection, targetTableName);
|
||
await connection.query(
|
||
`RENAME TABLE ${escapeIdentifier(legacyTableName)} TO ${escapeIdentifier(archiveTableName)}`
|
||
);
|
||
}
|
||
|
||
async function getPlanningTableColumns(tableName) {
|
||
return planningDatabase.query(
|
||
`SELECT
|
||
column_name AS columnName,
|
||
column_comment AS columnComment,
|
||
ordinal_position AS ordinalPosition
|
||
FROM information_schema.columns
|
||
WHERE table_schema = DATABASE()
|
||
AND table_name = ?
|
||
ORDER BY ordinal_position ASC`,
|
||
[tableName]
|
||
);
|
||
}
|
||
|
||
function resolvePlanningViewColumnLabel(column) {
|
||
const rawLabel = normalizeText(column?.columnComment || '');
|
||
if (!rawLabel) {
|
||
return column?.columnName || '';
|
||
}
|
||
|
||
const cleanedLabel = rawLabel
|
||
.replace(/^来源表头[:：]\s*/u, '')
|
||
.replace(/^数据来源[:：]\s*/u, '')
|
||
.trim();
|
||
|
||
if (cleanedLabel && !/^来源列序号[:：]\s*\d+\s*$/u.test(cleanedLabel)) {
|
||
return cleanedLabel;
|
||
}
|
||
|
||
return column?.columnName || rawLabel;
|
||
}
|
||
|
||
async function checkPlanningTableExists(tableName) {
|
||
const rows = await planningDatabase.query(
|
||
`SELECT 1
|
||
FROM information_schema.tables
|
||
WHERE table_schema = DATABASE()
|
||
AND table_name = ?
|
||
LIMIT 1`,
|
||
[tableName]
|
||
);
|
||
|
||
return rows.length > 0;
|
||
}
|
||
|
||
async function checkSystemTableExists(tableName) {
|
||
const rows = await planningDatabase.query(
|
||
`SELECT 1
|
||
FROM information_schema.tables
|
||
WHERE table_schema = DATABASE()
|
||
AND table_name = ?
|
||
LIMIT 1`,
|
||
[tableName]
|
||
);
|
||
|
||
return rows.length > 0;
|
||
}
|
||
|
||
async function getSystemTableColumns(tableName) {
|
||
return planningDatabase.query(
|
||
`SELECT
|
||
column_name AS columnName,
|
||
column_comment AS columnComment,
|
||
ordinal_position AS ordinalPosition
|
||
FROM information_schema.columns
|
||
WHERE table_schema = DATABASE()
|
||
AND table_name = ?
|
||
ORDER BY ordinal_position ASC`,
|
||
[tableName]
|
||
);
|
||
}
|
||
|
||
async function resolvePlanningReadableTableName(template) {
|
||
if (!template) {
|
||
return '';
|
||
}
|
||
|
||
if (await checkPlanningTableExists(template.targetTableName)) {
|
||
return template.targetTableName;
|
||
}
|
||
|
||
const legacyTableName = normalizeText(template.legacyTableName);
|
||
if (legacyTableName && await checkPlanningTableExists(legacyTableName)) {
|
||
return legacyTableName;
|
||
}
|
||
|
||
return '';
|
||
}
|
||
|
||
function sortPlanningViewColumns(columns = []) {
|
||
const visibleColumns = columns.filter((column) => !HIDDEN_VIEW_COLUMNS.has(column.columnName));
|
||
const trailingSet = new Set(TRAILING_META_COLUMNS);
|
||
const mainColumns = visibleColumns.filter((column) => !trailingSet.has(column.columnName));
|
||
const trailingColumns = TRAILING_META_COLUMNS
|
||
.map((name) => visibleColumns.find((column) => column.columnName === name))
|
||
.filter(Boolean);
|
||
|
||
return [...mainColumns, ...trailingColumns];
|
||
}
|
||
|
||
function filterPlanningViewColumnsByTemplate(columns = [], template) {
|
||
const expectedColumnNames = getExpectedTemplateColumnNames(template);
|
||
if (expectedColumnNames.length === 0) {
|
||
return columns;
|
||
}
|
||
|
||
const expectedSet = new Set(expectedColumnNames);
|
||
const trailingSet = new Set(TRAILING_META_COLUMNS);
|
||
|
||
return columns.filter((column) => expectedSet.has(column.columnName) || trailingSet.has(column.columnName));
|
||
}
|
||
|
||
function resolveMarketBasicCompareColumnLabel(column) {
|
||
const columnName = normalizeText(column?.columnName);
|
||
const comment = normalizeText(column?.columnComment);
|
||
return comment || columnName;
|
||
}
|
||
|
||
function getMarketBasicCompareVisibleColumns(columns = []) {
|
||
return [...columns].sort((left, right) => Number(left.ordinalPosition || 0) - Number(right.ordinalPosition || 0));
|
||
}
|
||
|
||
function buildMarketBasicCompareSchemaCheck(allColumns = [], visibleColumns = [], totalRows = 0) {
|
||
const dbColumnNames = allColumns.map((column) => normalizeText(column.columnName)).filter(Boolean);
|
||
const visibleColumnNames = visibleColumns.map((column) => normalizeText(column.columnName)).filter(Boolean);
|
||
const visibleColumnSet = new Set(visibleColumnNames);
|
||
const dbColumnSet = new Set(dbColumnNames);
|
||
|
||
return {
|
||
tableName: TAOBAO_PRODUCT_RECORDS_TABLE,
|
||
dbColumnCount: dbColumnNames.length,
|
||
visibleColumnCount: visibleColumnNames.length,
|
||
missingColumns: dbColumnNames.filter((columnName) => !visibleColumnSet.has(columnName)),
|
||
extraColumns: visibleColumnNames.filter((columnName) => !dbColumnSet.has(columnName)),
|
||
rowCount: Number(totalRows || 0),
|
||
columnsSynced: dbColumnNames.length === visibleColumnNames.length
|
||
&& dbColumnNames.every((columnName, index) => columnName === visibleColumnNames[index]),
|
||
};
|
||
}
|
||
|
||
function isMarketBasicMayColumn(column = {}) {
|
||
const columnName = normalizeText(column.columnName);
|
||
const label = resolveMarketBasicCompareColumnLabel(column);
|
||
return columnName.startsWith('may_') || normalizeText(label).startsWith('MAY-');
|
||
}
|
||
|
||
function getMarketBasicMayProductVisibleColumns(columns = []) {
|
||
const columnMap = new Map(columns.map((column) => [column.columnName, column]));
|
||
const picked = [];
|
||
const pickedNames = new Set();
|
||
|
||
for (const columnName of MARKET_BASIC_COMPARE_PRIORITY_COLUMNS.filter((name) => name.startsWith('may_'))) {
|
||
const column = columnMap.get(columnName);
|
||
if (!column || pickedNames.has(columnName)) {
|
||
continue;
|
||
}
|
||
picked.push(column);
|
||
pickedNames.add(columnName);
|
||
}
|
||
|
||
for (const column of columns) {
|
||
const columnName = normalizeText(column.columnName);
|
||
if (pickedNames.has(columnName) || !isMarketBasicMayColumn(column)) {
|
||
continue;
|
||
}
|
||
picked.push(column);
|
||
pickedNames.add(columnName);
|
||
}
|
||
|
||
return picked;
|
||
}
|
||
|
||
function appendSearchSalesPriceBandExtraColumns(kind, columns = []) {
|
||
if (kind !== 'search_sales_price_band') {
|
||
return columns;
|
||
}
|
||
|
||
const existingNames = new Set(columns.map((column) => column.name));
|
||
return [
|
||
...columns,
|
||
...SEARCH_SALES_PRICE_BAND_EXTRA_COLUMNS.filter((column) => !existingNames.has(column.name)),
|
||
];
|
||
}
|
||
|
||
function firstMeaningfulText(...values) {
|
||
for (const value of values) {
|
||
const normalized = normalizeText(value);
|
||
if (normalized) {
|
||
return normalized;
|
||
}
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
function addTaobaoRecordToLookup(lookup, row, payload) {
|
||
for (const rawId of [row?.item_id, row?.requested_item_id]) {
|
||
const goodsId = normalizeGoodsId(rawId);
|
||
if (goodsId && !lookup.has(goodsId)) {
|
||
lookup.set(goodsId, payload);
|
||
}
|
||
}
|
||
}
|
||
|
||
function buildTaobaoPlanningLookup(productRows = [], detailRows = []) {
|
||
const productLookup = new Map();
|
||
const detailLookup = new Map();
|
||
|
||
for (const product of productRows) {
|
||
addTaobaoRecordToLookup(productLookup, product, product);
|
||
}
|
||
|
||
for (const detail of detailRows) {
|
||
addTaobaoRecordToLookup(detailLookup, detail, detail);
|
||
}
|
||
|
||
return {
|
||
productLookup,
|
||
detailLookup,
|
||
};
|
||
}
|
||
|
||
function getTaobaoPriceRange(product, detail) {
|
||
const analysis = safeJsonParse(product?.analysis_json);
|
||
return firstMeaningfulText(
|
||
detail?.price,
|
||
analysis?.['价格段'],
|
||
product?.price
|
||
);
|
||
}
|
||
|
||
function getTaobaoMinMarkedPrice(product, detail) {
|
||
const analysis = safeJsonParse(product?.analysis_json);
|
||
const normalized = safeJsonParse(product?.normalized_item_json);
|
||
const raw = safeJsonParse(product?.raw_data_json);
|
||
const detailRaw = safeJsonParse(detail?.raw_data_json);
|
||
const skuMinimumPrice = getSkuMinimumPrice(
|
||
[normalized?.skus, raw?.skus, detail?.skus_json, detailRaw?.skus],
|
||
['price', 'sku_price', 'sale_price', 'zk_price']
|
||
);
|
||
const rangeMinimumPrice = getMinimumPrice([
|
||
...getPriceNumbersFromText(analysis?.['价格段']),
|
||
...getPriceNumbersFromText(detail?.price),
|
||
...getPriceNumbersFromText(product?.price),
|
||
]);
|
||
const minimumPrice = getMinimumPrice([skuMinimumPrice, rangeMinimumPrice]);
|
||
|
||
return formatPriceNumber(minimumPrice);
|
||
}
|
||
|
||
function getTaobaoMinDiscountPrice(product, detail, marketRow) {
|
||
const normalized = safeJsonParse(product?.normalized_item_json);
|
||
const raw = safeJsonParse(product?.raw_data_json);
|
||
const detailRaw = safeJsonParse(detail?.raw_data_json);
|
||
const promotionMinimumPrice = getMinimumPrice(getPromotionPriceCandidates(normalized, raw, detailRaw));
|
||
const marketDealPrice = normalizePriceNumber(marketRow?.deal_price_text);
|
||
const markedMinimumPrice = normalizePriceNumber(getTaobaoMinMarkedPrice(product, detail));
|
||
const minimumPrice = getMinimumPrice([promotionMinimumPrice, marketDealPrice, markedMinimumPrice]);
|
||
|
||
return formatPriceNumber(minimumPrice);
|
||
}
|
||
|
||
function getTaobaoTotalSold(product) {
|
||
const normalized = safeJsonParse(product?.normalized_item_json);
|
||
return firstMeaningfulText(
|
||
product?.total_sold,
|
||
normalized?.total_sold,
|
||
normalized?.sales
|
||
);
|
||
}
|
||
|
||
function getTaobaoAppearanceRecognitionCode(product) {
|
||
return firstMeaningfulText(product?.[APPEARANCE_RECOGNITION_CODE_COLUMN]);
|
||
}
|
||
|
||
function enrichSearchSalesPriceBandRow(row, product, detail) {
|
||
return {
|
||
...row,
|
||
[APPEARANCE_RECOGNITION_CODE_COLUMN]: getTaobaoAppearanceRecognitionCode(product),
|
||
...Object.fromEntries(
|
||
SEARCH_SALES_PRICE_BAND_MAY_COLUMN_NAMES.map((columnName) => [columnName, product?.[columnName] ?? ''])
|
||
),
|
||
taobao_total_sold: getTaobaoTotalSold(product),
|
||
taobao_price_range: getTaobaoPriceRange(product, detail),
|
||
taobao_min_marked_price: getTaobaoMinMarkedPrice(product, detail),
|
||
taobao_min_discount_price: getTaobaoMinDiscountPrice(product, detail, row),
|
||
};
|
||
}
|
||
|
||
async function enrichSearchSalesPriceBandRows(kind, rows = []) {
|
||
if (kind !== 'search_sales_price_band' || rows.length === 0) {
|
||
return rows;
|
||
}
|
||
|
||
const goodsIds = Array.from(new Set(
|
||
rows.map((row) => normalizeGoodsId(row.market_goods_id)).filter(Boolean)
|
||
));
|
||
|
||
if (goodsIds.length === 0) {
|
||
return rows.map((row) => enrichSearchSalesPriceBandRow(row, null, null));
|
||
}
|
||
|
||
const [hasProductTable, hasDetailTable] = await Promise.all([
|
||
checkPlanningTableExists(TAOBAO_PRODUCT_RECORDS_TABLE),
|
||
checkPlanningTableExists(TAOBAO_DETAIL_PLUS_TABLE),
|
||
]);
|
||
const placeholders = goodsIds.map(() => '?').join(', ');
|
||
if (hasProductTable) {
|
||
await ensureTaobaoProductRecordsAppearanceCodeColumn();
|
||
}
|
||
|
||
const productColumnSet = hasProductTable
|
||
? await getPlanningStandaloneColumnSet(TAOBAO_PRODUCT_RECORDS_TABLE)
|
||
: new Set();
|
||
const productSelectColumns = [
|
||
'id',
|
||
'requested_item_id',
|
||
'item_id',
|
||
'price',
|
||
'original_price',
|
||
'total_sold',
|
||
'analysis_json',
|
||
'normalized_item_json',
|
||
'raw_data_json',
|
||
APPEARANCE_RECOGNITION_CODE_COLUMN,
|
||
...SEARCH_SALES_PRICE_BAND_MAY_COLUMN_NAMES,
|
||
].filter((columnName) => productColumnSet.has(columnName));
|
||
const productRows = hasProductTable && productSelectColumns.length > 0
|
||
? await planningDatabase.query(
|
||
`SELECT ${productSelectColumns.map((columnName) => escapeIdentifier(columnName)).join(', ')}
|
||
FROM ${escapeIdentifier(TAOBAO_PRODUCT_RECORDS_TABLE)}
|
||
WHERE item_id IN (${placeholders}) OR requested_item_id IN (${placeholders})
|
||
ORDER BY id DESC`,
|
||
[...goodsIds, ...goodsIds]
|
||
)
|
||
: [];
|
||
const detailRows = hasDetailTable
|
||
? await planningDatabase.query(
|
||
`SELECT id, requested_item_id, item_id, price, skus_json, raw_data_json
|
||
FROM ${escapeIdentifier(TAOBAO_DETAIL_PLUS_TABLE)}
|
||
WHERE item_id IN (${placeholders}) OR requested_item_id IN (${placeholders})
|
||
ORDER BY id DESC`,
|
||
[...goodsIds, ...goodsIds]
|
||
)
|
||
: [];
|
||
const { productLookup, detailLookup } = buildTaobaoPlanningLookup(productRows, detailRows);
|
||
|
||
return rows.map((row) => {
|
||
const goodsId = normalizeGoodsId(row.market_goods_id);
|
||
return enrichSearchSalesPriceBandRow(
|
||
row,
|
||
productLookup.get(goodsId) || null,
|
||
detailLookup.get(goodsId) || null
|
||
);
|
||
});
|
||
}
|
||
|
||
function buildSourceColumnSql(column) {
|
||
const comment = column.sourceHeader
|
||
? `来源表头: ${column.sourceHeader}`
|
||
: `来源列序号: ${column.index + 1}`;
|
||
|
||
return `${escapeIdentifier(column.columnName)} TEXT NULL COMMENT '${escapeSqlComment(comment)}'`;
|
||
}
|
||
|
||
async function ensurePlanningFingerprintSupport(connection, tableName, options = {}) {
|
||
const includeIndexes = options.includeIndexes !== false;
|
||
const existingColumns = await getExistingColumns(connection, tableName);
|
||
if (!existingColumns.has(ROW_FINGERPRINT_COLUMN)) {
|
||
await connection.query(
|
||
`ALTER TABLE ${escapeIdentifier(tableName)}
|
||
ADD COLUMN ${escapeIdentifier(ROW_FINGERPRINT_COLUMN)} CHAR(64) NULL COMMENT '企划模板内容指纹'`
|
||
);
|
||
}
|
||
|
||
if (!includeIndexes) {
|
||
return;
|
||
}
|
||
|
||
const indexes = await getTableIndexes(connection, tableName);
|
||
if (!indexes.has('idx_row_fingerprint')) {
|
||
await connection.query(
|
||
`ALTER TABLE ${escapeIdentifier(tableName)}
|
||
ADD KEY idx_row_fingerprint (${escapeIdentifier(ROW_FINGERPRINT_COLUMN)})`
|
||
);
|
||
}
|
||
}
|
||
|
||
async function syncPlanningTemplateCanonicalColumns(connection, template, options = {}) {
|
||
await migrateLegacyPlanningTable(connection, template, options);
|
||
return syncPlanningCanonicalColumnsForTable(connection, template.targetTableName, template, options);
|
||
}
|
||
|
||
async function ensurePlanningCanonicalSchemas(options = {}) {
|
||
const forceRefresh = options.forceRefresh === true;
|
||
const includeIndexes = options.includeIndexes !== false;
|
||
const readyKey = includeIndexes ? 'fullReady' : 'lightReady';
|
||
const pendingKey = includeIndexes ? 'fullPending' : 'lightPending';
|
||
|
||
if (!forceRefresh && (canonicalSchemaSyncState.fullReady || canonicalSchemaSyncState[readyKey])) {
|
||
return;
|
||
}
|
||
|
||
if (!forceRefresh && !includeIndexes && canonicalSchemaSyncState.fullPending) {
|
||
await canonicalSchemaSyncState.fullPending;
|
||
return;
|
||
}
|
||
|
||
if (!forceRefresh && canonicalSchemaSyncState[pendingKey]) {
|
||
await canonicalSchemaSyncState[pendingKey];
|
||
return;
|
||
}
|
||
|
||
if (includeIndexes && canonicalSchemaSyncState.lightPending) {
|
||
await canonicalSchemaSyncState.lightPending;
|
||
if (!forceRefresh && canonicalSchemaSyncState.fullReady) {
|
||
return;
|
||
}
|
||
}
|
||
|
||
const syncPromise = planningDatabase.withTransaction(async (connection) => {
|
||
for (const template of TEMPLATE_CATALOG) {
|
||
await syncPlanningTemplateCanonicalColumns(connection, template, { includeIndexes });
|
||
}
|
||
});
|
||
|
||
canonicalSchemaSyncState[pendingKey] = syncPromise;
|
||
|
||
try {
|
||
await syncPromise;
|
||
await ensurePlanningUploadBatchSchema();
|
||
await ensurePlanningAiImageSchema();
|
||
await ensurePlanningAppearanceMatchSchema();
|
||
await ensureTaobaoProductRecordsAppearanceCodeColumn({ forceRefresh, includeIndexes });
|
||
canonicalSchemaSyncState = {
|
||
lightReady: true,
|
||
fullReady: includeIndexes ? true : canonicalSchemaSyncState.fullReady,
|
||
lightPending: null,
|
||
fullPending: null,
|
||
};
|
||
} catch (error) {
|
||
canonicalSchemaSyncState = {
|
||
lightReady: includeIndexes ? canonicalSchemaSyncState.lightReady : false,
|
||
fullReady: false,
|
||
lightPending: null,
|
||
fullPending: null,
|
||
};
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
async function syncPlanningFingerprints(connection, tableName) {
|
||
const columnNames = await getTableColumnNames(connection, tableName);
|
||
const businessColumns = getBusinessColumnNames(columnNames);
|
||
if (businessColumns.length === 0) {
|
||
return {
|
||
businessColumns,
|
||
backfilledCount: 0,
|
||
dedupedCount: 0,
|
||
};
|
||
}
|
||
|
||
const selectColumns = ['id', ROW_FINGERPRINT_COLUMN, ...businessColumns]
|
||
.map((columnName) => escapeIdentifier(columnName))
|
||
.join(', ');
|
||
const [rows] = await connection.query(
|
||
`SELECT ${selectColumns}
|
||
FROM ${escapeIdentifier(tableName)}
|
||
ORDER BY id DESC`
|
||
);
|
||
|
||
const fingerprintToKeepId = new Set();
|
||
const rowsToUpdate = [];
|
||
const duplicateIds = [];
|
||
|
||
for (const row of rows) {
|
||
const fingerprint = buildRowFingerprint(businessColumns, row);
|
||
if (fingerprintToKeepId.has(fingerprint)) {
|
||
duplicateIds.push(row.id);
|
||
continue;
|
||
}
|
||
|
||
fingerprintToKeepId.add(fingerprint);
|
||
if (row[ROW_FINGERPRINT_COLUMN] !== fingerprint) {
|
||
rowsToUpdate.push({
|
||
id: row.id,
|
||
fingerprint,
|
||
});
|
||
}
|
||
}
|
||
|
||
const chunkSize = 500;
|
||
|
||
for (let index = 0; index < rowsToUpdate.length; index += chunkSize) {
|
||
const chunk = rowsToUpdate.slice(index, index + chunkSize);
|
||
for (const item of chunk) {
|
||
await connection.query(
|
||
`UPDATE ${escapeIdentifier(tableName)}
|
||
SET ${escapeIdentifier(ROW_FINGERPRINT_COLUMN)} = ?
|
||
WHERE id = ?`,
|
||
[item.fingerprint, item.id]
|
||
);
|
||
}
|
||
}
|
||
|
||
for (let index = 0; index < duplicateIds.length; index += chunkSize) {
|
||
const chunk = duplicateIds.slice(index, index + chunkSize);
|
||
const placeholders = chunk.map(() => '?').join(', ');
|
||
await connection.query(
|
||
`DELETE FROM ${escapeIdentifier(tableName)}
|
||
WHERE id IN (${placeholders})`,
|
||
chunk
|
||
);
|
||
}
|
||
|
||
return {
|
||
businessColumns,
|
||
backfilledCount: rowsToUpdate.length,
|
||
dedupedCount: duplicateIds.length,
|
||
};
|
||
}
|
||
|
||
async function ensurePlanningSheetTable(connection, sheet) {
|
||
const sourceColumnSql = sheet.sourceColumns.map((column) => buildSourceColumnSql(column));
|
||
|
||
await connection.query(
|
||
`CREATE TABLE IF NOT EXISTS ${escapeIdentifier(sheet.targetTableName)} (
|
||
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||
upload_batch_id VARCHAR(36) NOT NULL,
|
||
source_file_name VARCHAR(255) NOT NULL,
|
||
source_sheet_name VARCHAR(100) NOT NULL,
|
||
source_sheet_type VARCHAR(64) NOT NULL,
|
||
source_row_number INT NOT NULL,
|
||
uploaded_by_user_id BIGINT NULL,
|
||
uploaded_by_username VARCHAR(100) NULL,
|
||
uploaded_by_name VARCHAR(100) NULL,
|
||
uploaded_department_id BIGINT NULL,
|
||
uploaded_department_name VARCHAR(100) NULL,
|
||
raw_data JSON NULL,
|
||
${escapeIdentifier(ROW_FINGERPRINT_COLUMN)} CHAR(64) NULL COMMENT '企划模板内容指纹',
|
||
${sourceColumnSql.join(',\n ')},
|
||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||
PRIMARY KEY (id),
|
||
KEY idx_batch_id (upload_batch_id),
|
||
KEY idx_sheet_type (source_sheet_type),
|
||
KEY idx_row_fingerprint (${escapeIdentifier(ROW_FINGERPRINT_COLUMN)}),
|
||
KEY idx_created_at (created_at)
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='企划模板原始数据：${escapeSqlComment(sheet.label)}'`
|
||
);
|
||
|
||
const existingColumns = await ensurePlanningMetaColumns(connection, sheet.targetTableName);
|
||
|
||
for (const column of sheet.sourceColumns) {
|
||
if (existingColumns.has(column.columnName)) {
|
||
continue;
|
||
}
|
||
|
||
await connection.query(
|
||
`ALTER TABLE ${escapeIdentifier(sheet.targetTableName)}
|
||
ADD COLUMN ${buildSourceColumnSql(column)}`
|
||
);
|
||
}
|
||
|
||
await ensurePlanningFingerprintSupport(connection, sheet.targetTableName);
|
||
await ensurePlanningTemplateIndexesForTable(connection, sheet.targetTableName, sheet);
|
||
}
|
||
|
||
function assignIncomingFingerprints(sheet, businessColumns = []) {
|
||
const dedupedRows = [];
|
||
const fingerprintToIndex = new Map();
|
||
let replacedInUploadCount = 0;
|
||
|
||
for (const row of sheet.rows) {
|
||
const rowFingerprint = buildRowFingerprint(businessColumns, row.sourceData);
|
||
row[ROW_FINGERPRINT_COLUMN] = rowFingerprint;
|
||
|
||
if (fingerprintToIndex.has(rowFingerprint)) {
|
||
dedupedRows[fingerprintToIndex.get(rowFingerprint)] = row;
|
||
replacedInUploadCount += 1;
|
||
continue;
|
||
}
|
||
|
||
fingerprintToIndex.set(rowFingerprint, dedupedRows.length);
|
||
dedupedRows.push(row);
|
||
}
|
||
|
||
sheet.rows = dedupedRows;
|
||
sheet.rowCount = dedupedRows.length;
|
||
|
||
return {
|
||
replacedInUploadCount,
|
||
};
|
||
}
|
||
|
||
async function deleteOverlappingRows(connection, tableName, rowFingerprints = []) {
|
||
const uniqueFingerprints = Array.from(new Set(rowFingerprints.filter(Boolean)));
|
||
if (uniqueFingerprints.length === 0) {
|
||
return 0;
|
||
}
|
||
|
||
const chunkSize = 500;
|
||
let deletedCount = 0;
|
||
|
||
for (let index = 0; index < uniqueFingerprints.length; index += chunkSize) {
|
||
const chunk = uniqueFingerprints.slice(index, index + chunkSize);
|
||
const placeholders = chunk.map(() => '?').join(', ');
|
||
const [result] = await connection.query(
|
||
`DELETE FROM ${escapeIdentifier(tableName)}
|
||
WHERE ${escapeIdentifier(ROW_FINGERPRINT_COLUMN)} IN (${placeholders})`,
|
||
chunk
|
||
);
|
||
deletedCount += Number(result?.affectedRows || 0);
|
||
}
|
||
|
||
return deletedCount;
|
||
}
|
||
|
||
async function insertSheetRows(connection, sheet) {
|
||
const insertColumns = [...META_COLUMNS, ...sheet.sourceColumns.map((column) => column.columnName)];
|
||
const chunkSize = 200;
|
||
|
||
for (let index = 0; index < sheet.rows.length; index += 1) {
|
||
const row = sheet.rows[index];
|
||
row.__insertValues = insertColumns.map((columnName) => {
|
||
if (META_COLUMNS.includes(columnName)) {
|
||
return row[columnName] ?? null;
|
||
}
|
||
|
||
return row.sourceData[columnName] ?? null;
|
||
});
|
||
}
|
||
|
||
for (let index = 0; index < sheet.rows.length; index += chunkSize) {
|
||
const chunk = sheet.rows.slice(index, index + chunkSize);
|
||
const placeholders = chunk
|
||
.map(() => `(${insertColumns.map(() => '?').join(', ')})`)
|
||
.join(', ');
|
||
const values = chunk.flatMap((row) => row.__insertValues);
|
||
|
||
await connection.query(
|
||
`INSERT INTO ${escapeIdentifier(sheet.targetTableName)} (${insertColumns.map((columnName) => escapeIdentifier(columnName)).join(', ')})
|
||
VALUES ${placeholders}`,
|
||
values
|
||
);
|
||
}
|
||
|
||
for (const row of sheet.rows) {
|
||
delete row.__insertValues;
|
||
}
|
||
}
|
||
|
||
function chunkArray(list, size) {
|
||
const safeList = Array.isArray(list) ? list : [];
|
||
const chunkSize = Math.max(Number(size) || 1, 1);
|
||
const chunks = [];
|
||
for (let index = 0; index < safeList.length; index += chunkSize) {
|
||
chunks.push(safeList.slice(index, index + chunkSize));
|
||
}
|
||
return chunks;
|
||
}
|
||
|
||
async function syncTaobaoAppearanceCodesFromSearchSalesSheet(connection, sheet) {
|
||
if (sheet?.kind !== 'search_sales_price_band' || !Array.isArray(sheet.rows) || sheet.rows.length === 0) {
|
||
return 0;
|
||
}
|
||
|
||
const tableMeta = await getPlanningTableMeta(connection, TAOBAO_PRODUCT_RECORDS_TABLE);
|
||
if (!tableMeta || tableMeta.tableType === 'VIEW') {
|
||
return 0;
|
||
}
|
||
|
||
const existingColumns = await getExistingColumns(connection, TAOBAO_PRODUCT_RECORDS_TABLE);
|
||
if (
|
||
!existingColumns.has(APPEARANCE_RECOGNITION_CODE_COLUMN)
|
||
|| (!existingColumns.has('item_id') && !existingColumns.has('requested_item_id'))
|
||
) {
|
||
return 0;
|
||
}
|
||
|
||
const updateMap = new Map();
|
||
for (const row of sheet.rows) {
|
||
const goodsId = normalizeGoodsId(row?.sourceData?.market_goods_id);
|
||
const appearanceCode = firstMeaningfulText(row?.sourceData?.[APPEARANCE_RECOGNITION_CODE_COLUMN]);
|
||
if (!goodsId || !appearanceCode) {
|
||
continue;
|
||
}
|
||
updateMap.set(goodsId, appearanceCode);
|
||
}
|
||
|
||
const updates = Array.from(updateMap.entries())
|
||
.map(([goodsId, appearanceCode]) => ({ goodsId, appearanceCode }));
|
||
if (updates.length === 0) {
|
||
return 0;
|
||
}
|
||
|
||
let updatedCount = 0;
|
||
for (const chunk of chunkArray(updates, 100)) {
|
||
const caseFragments = [];
|
||
const caseParams = [];
|
||
const whereFragments = [];
|
||
const whereParams = [];
|
||
|
||
for (const item of chunk) {
|
||
const matchFragments = [];
|
||
if (existingColumns.has('item_id')) {
|
||
matchFragments.push(`${escapeIdentifier('item_id')} = ?`);
|
||
caseParams.push(item.goodsId);
|
||
whereParams.push(item.goodsId);
|
||
}
|
||
if (existingColumns.has('requested_item_id')) {
|
||
matchFragments.push(`${escapeIdentifier('requested_item_id')} = ?`);
|
||
caseParams.push(item.goodsId);
|
||
whereParams.push(item.goodsId);
|
||
}
|
||
if (matchFragments.length === 0) {
|
||
continue;
|
||
}
|
||
caseFragments.push(`WHEN ${matchFragments.join(' OR ')} THEN ?`);
|
||
caseParams.push(item.appearanceCode);
|
||
whereFragments.push(...matchFragments);
|
||
}
|
||
|
||
if (caseFragments.length === 0 || whereFragments.length === 0) {
|
||
continue;
|
||
}
|
||
|
||
const result = await connection.query(
|
||
`UPDATE ${escapeIdentifier(TAOBAO_PRODUCT_RECORDS_TABLE)}
|
||
SET ${escapeIdentifier(APPEARANCE_RECOGNITION_CODE_COLUMN)} = CASE ${caseFragments.join(' ')} ELSE ${escapeIdentifier(APPEARANCE_RECOGNITION_CODE_COLUMN)} END
|
||
WHERE NULLIF(TRIM(${escapeIdentifier(APPEARANCE_RECOGNITION_CODE_COLUMN)}), '') IS NULL
|
||
AND (${whereFragments.join(' OR ')})`,
|
||
[...caseParams, ...whereParams]
|
||
);
|
||
updatedCount += Number(result?.[0]?.affectedRows || 0);
|
||
}
|
||
|
||
return updatedCount;
|
||
}
|
||
|
||
async function getPlanningUploadBatches(limit = 20) {
|
||
const safeLimit = Math.min(Math.max(Number(limit) || 20, 1), 100);
|
||
await ensurePlanningUploadBatchSchema();
|
||
const rows = await planningDatabase.query(
|
||
`SELECT
|
||
batch_id AS batchId,
|
||
source_file_name AS sourceFileName,
|
||
target_table_name AS targetTableName,
|
||
row_count AS rowCount,
|
||
success_count AS successCount,
|
||
source_sheet_summary AS sourceSheetSummary,
|
||
uploaded_by_user_id AS uploadedByUserId,
|
||
uploaded_by_username AS uploadedByUsername,
|
||
uploaded_by_name AS uploadedByName,
|
||
uploaded_department_id AS uploadedDepartmentId,
|
||
uploaded_department_name AS uploadedDepartmentName,
|
||
created_at AS createdAt
|
||
FROM ${IMPORT_BATCH_TABLE}
|
||
ORDER BY id DESC
|
||
LIMIT ?`,
|
||
[safeLimit]
|
||
);
|
||
|
||
return rows.map((row) => ({
|
||
...row,
|
||
sourceFileName: normalizeUploadedFileName(row.sourceFileName),
|
||
}));
|
||
}
|
||
|
||
async function getPlanningTemplateSummaries(options = {}) {
|
||
const forceRefresh = options.forceRefresh === true;
|
||
const now = Date.now();
|
||
|
||
if (!forceRefresh && templateSummariesCache.value && templateSummariesCache.expiresAt > now) {
|
||
return clonePlainData(templateSummariesCache.value);
|
||
}
|
||
|
||
if (!forceRefresh && templateSummariesCache.pending) {
|
||
return clonePlainData(await templateSummariesCache.pending);
|
||
}
|
||
|
||
const fetchPromise = (async () => {
|
||
const templates = getTemplateCatalog().map((template) => ({
|
||
...template,
|
||
totalRows: 0,
|
||
tableExists: false,
|
||
databaseConfigured: true,
|
||
}));
|
||
|
||
try {
|
||
const hydratedTemplates = await Promise.all(
|
||
templates.map(async (template) => {
|
||
const readableTableName = await resolvePlanningReadableTableName(template);
|
||
if (!readableTableName) {
|
||
return {
|
||
...template,
|
||
tableExists: false,
|
||
totalRows: 0,
|
||
};
|
||
}
|
||
|
||
const countRows = await planningDatabase.query(
|
||
`SELECT COUNT(*) AS total
|
||
FROM ${escapeIdentifier(readableTableName)}`
|
||
);
|
||
|
||
return {
|
||
...template,
|
||
tableExists: true,
|
||
physicalTableName: readableTableName,
|
||
totalRows: Number(countRows[0]?.total || 0),
|
||
};
|
||
})
|
||
);
|
||
|
||
templateSummariesCache = {
|
||
expiresAt: Date.now() + TEMPLATE_SUMMARIES_CACHE_TTL_MS,
|
||
value: hydratedTemplates,
|
||
pending: null,
|
||
};
|
||
|
||
return hydratedTemplates;
|
||
} catch (error) {
|
||
if (isPlanningDbNotConfigured(error)) {
|
||
const fallbackTemplates = templates.map((template) => ({
|
||
...template,
|
||
databaseConfigured: false,
|
||
}));
|
||
templateSummariesCache = {
|
||
expiresAt: Date.now() + TEMPLATE_SUMMARIES_CACHE_TTL_MS,
|
||
value: fallbackTemplates,
|
||
pending: null,
|
||
};
|
||
return fallbackTemplates;
|
||
}
|
||
|
||
templateSummariesCache = {
|
||
expiresAt: 0,
|
||
value: null,
|
||
pending: null,
|
||
};
|
||
throw error;
|
||
}
|
||
})();
|
||
|
||
templateSummariesCache.pending = fetchPromise;
|
||
const result = await fetchPromise;
|
||
return clonePlainData(result);
|
||
}
|
||
|
||
async function getReadonlyPlanningPageSummaries() {
|
||
return Promise.all(
|
||
READONLY_PLANNING_PAGE_CATALOG.map(async (template) => {
|
||
if (template.kind === GOODS_ANALYSIS_KIND) {
|
||
return {
|
||
...template,
|
||
...getPlanningGoodsAnalysisSummary(),
|
||
};
|
||
}
|
||
|
||
if (template.kind === MAY_PRODUCT_DATA_KIND) {
|
||
return {
|
||
...template,
|
||
...await getPlanningMayProductDataSummary(),
|
||
};
|
||
}
|
||
|
||
const tableExists = await checkSystemTableExists(template.targetTableName);
|
||
if (!tableExists) {
|
||
return {
|
||
...template,
|
||
tableExists: false,
|
||
totalRows: 0,
|
||
databaseConfigured: true,
|
||
};
|
||
}
|
||
|
||
const allColumns = await getSystemTableColumns(template.targetTableName);
|
||
const visibleColumns = template.kind === MAY_PRODUCT_DATA_KIND
|
||
? getMarketBasicMayProductVisibleColumns(allColumns)
|
||
: getMarketBasicCompareVisibleColumns(allColumns);
|
||
const mayProductCondition = template.kind === MAY_PRODUCT_DATA_KIND
|
||
? buildMarketBasicMayMatchCondition(visibleColumns, 'matched')
|
||
: '';
|
||
const whereClause = mayProductCondition ? `WHERE ${mayProductCondition}` : '';
|
||
const countRows = await planningDatabase.query(
|
||
`SELECT COUNT(*) AS total
|
||
FROM ${escapeIdentifier(template.targetTableName)}
|
||
${whereClause}`
|
||
);
|
||
|
||
return {
|
||
...template,
|
||
tableExists: true,
|
||
physicalTableName: template.targetTableName,
|
||
totalRows: Number(countRows[0]?.total || 0),
|
||
databaseConfigured: true,
|
||
};
|
||
})
|
||
);
|
||
}
|
||
|
||
async function getPlanningPageSummaries(options = {}) {
|
||
const [uploadTemplates, readonlyPages] = await Promise.all([
|
||
getPlanningTemplateSummaries(options),
|
||
getReadonlyPlanningPageSummaries(),
|
||
]);
|
||
|
||
return [
|
||
...uploadTemplates,
|
||
...readonlyPages,
|
||
];
|
||
}
|
||
|
||
function buildMarketBasicMayMatchCondition(columns = [], mayMatch = 'all') {
|
||
const matchMode = normalizeText(mayMatch || 'all');
|
||
if (!['matched', 'unmatched'].includes(matchMode)) {
|
||
return '';
|
||
}
|
||
|
||
const mayColumns = columns
|
||
.map((column) => normalizeText(column.columnName))
|
||
.filter((columnName) => columnName.startsWith('may_'));
|
||
if (mayColumns.length === 0) {
|
||
return matchMode === 'matched' ? '1 = 0' : '';
|
||
}
|
||
|
||
const hasMayDataCondition = mayColumns
|
||
.map((columnName) => `NULLIF(TRIM(CAST(${escapeIdentifier(columnName)} AS CHAR)), '') IS NOT NULL`)
|
||
.join(' OR ');
|
||
return matchMode === 'matched'
|
||
? `(${hasMayDataCondition})`
|
||
: `NOT (${hasMayDataCondition})`;
|
||
}
|
||
|
||
async function getMarketBasicCompareTableData(template, page, pageSize) {
|
||
const tableExists = await checkSystemTableExists(template.targetTableName);
|
||
const baseTemplate = {
|
||
...template,
|
||
totalRows: 0,
|
||
tableExists,
|
||
databaseConfigured: true,
|
||
};
|
||
|
||
if (!tableExists) {
|
||
return {
|
||
template: baseTemplate,
|
||
columns: [],
|
||
rows: [],
|
||
total: 0,
|
||
page,
|
||
pageSize,
|
||
};
|
||
}
|
||
|
||
const allColumns = await getSystemTableColumns(template.targetTableName);
|
||
const visibleColumns = template.kind === MAY_PRODUCT_DATA_KIND
|
||
? getMarketBasicMayProductVisibleColumns(allColumns)
|
||
: getMarketBasicCompareVisibleColumns(allColumns);
|
||
const selectColumns = visibleColumns.map((column) => escapeIdentifier(column.columnName)).join(', ');
|
||
const offset = (page - 1) * pageSize;
|
||
const mayMatchCondition = template.kind === MAY_PRODUCT_DATA_KIND
|
||
? buildMarketBasicMayMatchCondition(visibleColumns, 'matched')
|
||
: '';
|
||
const whereClause = mayMatchCondition ? `WHERE ${mayMatchCondition}` : '';
|
||
const countRows = await planningDatabase.query(
|
||
`SELECT COUNT(*) AS total
|
||
FROM ${escapeIdentifier(template.targetTableName)}
|
||
${whereClause}`
|
||
);
|
||
const total = Number(countRows[0]?.total || 0);
|
||
const hasIdColumn = allColumns.some((column) => column.columnName === 'id');
|
||
const rows = selectColumns
|
||
? await planningDatabase.query(
|
||
`SELECT ${selectColumns}
|
||
FROM ${escapeIdentifier(template.targetTableName)}
|
||
${whereClause}
|
||
${hasIdColumn ? `ORDER BY ${escapeIdentifier('id')} DESC` : ''}
|
||
LIMIT ? OFFSET ?`,
|
||
[pageSize, offset]
|
||
)
|
||
: [];
|
||
|
||
return {
|
||
template: {
|
||
...baseTemplate,
|
||
totalRows: total,
|
||
physicalTableName: template.targetTableName,
|
||
schemaCheck: buildMarketBasicCompareSchemaCheck(allColumns, visibleColumns, total),
|
||
},
|
||
columns: visibleColumns.map((column) => ({
|
||
name: column.columnName,
|
||
label: resolveMarketBasicCompareColumnLabel(column),
|
||
})),
|
||
rows,
|
||
total,
|
||
page,
|
||
pageSize,
|
||
};
|
||
}
|
||
|
||
async function getPlanningTableData(kind, options = {}) {
|
||
const template = getPlanningTemplateByKind(kind);
|
||
if (!template) {
|
||
throw new Error('不支持的企划子页面类型');
|
||
}
|
||
|
||
const page = Math.max(Number(options.page) || 1, 1);
|
||
const pageSize = Math.min(Math.max(Number(options.pageSize) || 20, 1), 100);
|
||
const cacheVersion = template.kind === MAY_PRODUCT_DATA_KIND
|
||
? await getPlanningMayProductSnapshotCacheVersion({ snapshotDate: options.snapshotDate })
|
||
: '';
|
||
const cacheOptions = cacheVersion
|
||
? { ...options, cacheVersion }
|
||
: options;
|
||
const useCache = shouldUseTableDataCache(kind, options);
|
||
const cacheTtlMs = getTableDataCacheTtlMs(kind);
|
||
const cacheKey = useCache ? getTableDataCacheKey(kind, page, pageSize, cacheOptions) : '';
|
||
const cachedEntry = useCache ? tableDataCache.get(cacheKey) : null;
|
||
const now = Date.now();
|
||
|
||
if (useCache && cachedEntry?.value && cachedEntry.expiresAt > now) {
|
||
return clonePlainData(cachedEntry.value);
|
||
}
|
||
|
||
if (useCache && cachedEntry?.pending) {
|
||
return clonePlainData(await cachedEntry.pending);
|
||
}
|
||
|
||
const fetchPromise = (async () => {
|
||
if (template.kind === MAY_PRODUCT_DATA_KIND) {
|
||
return getPlanningMayProductDataTableData(template, page, pageSize, options);
|
||
}
|
||
|
||
if (template.kind === MARKET_BASIC_COMPARE_KIND) {
|
||
return getMarketBasicCompareTableData(template, page, pageSize, options);
|
||
}
|
||
|
||
try {
|
||
const summaryList = await getPlanningTemplateSummaries();
|
||
const templateSummary = summaryList.find((item) => item.kind === kind) || null;
|
||
const tableExists = templateSummary?.tableExists === true;
|
||
const databaseConfigured = templateSummary?.databaseConfigured !== false;
|
||
const readableTableName = templateSummary?.physicalTableName || await resolvePlanningReadableTableName(template);
|
||
|
||
if (!databaseConfigured) {
|
||
return {
|
||
template: {
|
||
...getTemplateCatalog().find((item) => item.kind === kind),
|
||
totalRows: 0,
|
||
tableExists: false,
|
||
databaseConfigured: false,
|
||
},
|
||
columns: [],
|
||
rows: [],
|
||
total: 0,
|
||
page,
|
||
pageSize,
|
||
message: NOT_CONFIGURED_MESSAGE,
|
||
};
|
||
}
|
||
|
||
if (!tableExists || !readableTableName) {
|
||
return {
|
||
template: {
|
||
...getTemplateCatalog().find((item) => item.kind === kind),
|
||
totalRows: 0,
|
||
tableExists: false,
|
||
databaseConfigured: true,
|
||
},
|
||
columns: [],
|
||
rows: [],
|
||
total: 0,
|
||
page,
|
||
pageSize,
|
||
};
|
||
}
|
||
|
||
const allColumns = await getPlanningTableColumns(readableTableName);
|
||
const sortedColumns = sortPlanningViewColumns(allColumns);
|
||
const visibleColumns = readableTableName === template.targetTableName
|
||
? filterPlanningViewColumnsByTemplate(sortedColumns, template)
|
||
: sortedColumns;
|
||
const selectColumns = visibleColumns.map((column) => escapeIdentifier(column.columnName)).join(', ');
|
||
const offset = (page - 1) * pageSize;
|
||
const total = Number(templateSummary?.totalRows || 0);
|
||
|
||
const rows = await planningDatabase.query(
|
||
`SELECT ${selectColumns}
|
||
FROM ${escapeIdentifier(readableTableName)}
|
||
ORDER BY id DESC
|
||
LIMIT ? OFFSET ?`,
|
||
[pageSize, offset]
|
||
);
|
||
const enrichedRows = await enrichSearchSalesPriceBandRows(kind, rows);
|
||
|
||
return {
|
||
template: {
|
||
...getTemplateCatalog().find((item) => item.kind === kind),
|
||
totalRows: total,
|
||
tableExists: true,
|
||
databaseConfigured: true,
|
||
},
|
||
columns: appendSearchSalesPriceBandExtraColumns(kind, visibleColumns.map((column) => ({
|
||
name: column.columnName,
|
||
label: resolvePlanningViewColumnLabel(column),
|
||
}))),
|
||
rows: enrichedRows,
|
||
total,
|
||
page,
|
||
pageSize,
|
||
};
|
||
} catch (error) {
|
||
if (isPlanningDbNotConfigured(error)) {
|
||
return {
|
||
template: {
|
||
...getTemplateCatalog().find((item) => item.kind === kind),
|
||
totalRows: 0,
|
||
tableExists: false,
|
||
databaseConfigured: false,
|
||
},
|
||
columns: [],
|
||
rows: [],
|
||
total: 0,
|
||
page,
|
||
pageSize,
|
||
message: error.message,
|
||
};
|
||
}
|
||
|
||
throw error;
|
||
}
|
||
})();
|
||
|
||
if (useCache) {
|
||
tableDataCache.set(cacheKey, {
|
||
expiresAt: 0,
|
||
value: null,
|
||
pending: fetchPromise,
|
||
});
|
||
}
|
||
|
||
try {
|
||
const result = await fetchPromise;
|
||
if (useCache) {
|
||
tableDataCache.set(cacheKey, {
|
||
expiresAt: Date.now() + cacheTtlMs,
|
||
value: result,
|
||
pending: null,
|
||
});
|
||
}
|
||
return clonePlainData(result);
|
||
} catch (error) {
|
||
if (useCache) {
|
||
tableDataCache.delete(cacheKey);
|
||
}
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
async function buildPlanningTableExportBuffer(kind, options = {}) {
|
||
const template = getPlanningTemplateByKind(kind);
|
||
if (!template) {
|
||
throw createPlanningExportError('不支持的企划子页面类型');
|
||
}
|
||
|
||
if (template.kind !== MAY_PRODUCT_DATA_KIND) {
|
||
throw createPlanningExportError('当前页面暂不支持导出全部明细');
|
||
}
|
||
|
||
const countResult = await getPlanningMayProductDataTableData(template, 1, 1, options);
|
||
const totalRows = Number(countResult.total || 0);
|
||
if (totalRows > PLANNING_TABLE_EXPORT_MAX_ROWS) {
|
||
throw createPlanningExportError(
|
||
`当前明细共 ${totalRows} 行，超过单次导出上限 ${PLANNING_TABLE_EXPORT_MAX_ROWS} 行，请先缩小筛选范围后再导出`,
|
||
413
|
||
);
|
||
}
|
||
|
||
const exportResult = await getPlanningMayProductDataTableData(
|
||
template,
|
||
1,
|
||
Math.max(totalRows, 1),
|
||
options
|
||
);
|
||
const rows = Array.isArray(exportResult.rows) ? exportResult.rows : [];
|
||
const columns = Array.isArray(exportResult.columns) ? exportResult.columns : [];
|
||
const activeLabel = exportResult.reviewDashboard?.activeActionLabel || '全部';
|
||
const virtualCategory = normalizeText(exportResult.reviewDashboard?.virtualCategory || options.virtualCategory || options.mode);
|
||
const dashboardFilters = exportResult.reviewDashboard?.filters || {};
|
||
const filterParts = [
|
||
virtualCategory || '全部模式',
|
||
activeLabel,
|
||
dashboardFilters.inventoryScope ? `路径-${dashboardFilters.inventoryScope}` : '',
|
||
dashboardFilters.brandName ? `品牌-${dashboardFilters.brandName}` : '',
|
||
!dashboardFilters.brandName && dashboardFilters.brandGroup ? `品牌分类-${dashboardFilters.brandGroup}` : '',
|
||
dashboardFilters.categoryName ? `类目-${dashboardFilters.categoryName}` : '',
|
||
dashboardFilters.supplierName ? `供应商-${dashboardFilters.supplierName}` : '',
|
||
dashboardFilters.missingMetric ? `缺失-${dashboardFilters.missingMetric}` : '',
|
||
]
|
||
.map((item) => normalizeText(item).replace(/[\\/:*?"<>|]/g, ''))
|
||
.filter(Boolean);
|
||
const snapshotDate = exportResult.snapshotDate || exportResult.template?.snapshotDate || '';
|
||
const fileName = `停产建议分析明细-${filterParts.join('-')}-${snapshotDate || formatExportTimestamp()}.xlsx`;
|
||
|
||
return {
|
||
buffer: buildWorkbookBufferFromPlanningRows({
|
||
columns,
|
||
rows,
|
||
sheetName: `${filterParts.join('-') || '全部明细'}`.slice(0, 31),
|
||
}),
|
||
totalRows: rows.length,
|
||
fileName,
|
||
activeLabel,
|
||
snapshotDate,
|
||
};
|
||
}
|
||
|
||
async function resolvePlanningUploadUser(userId) {
|
||
const resolvedUser = await getResolvedUserWithPageAccess(userId);
|
||
if (!resolvedUser) {
|
||
throw new Error('当前用户不存在');
|
||
}
|
||
|
||
if (resolvedUser.role === 'admin') {
|
||
return {
|
||
...resolvedUser,
|
||
planningUploadAllowed: true,
|
||
};
|
||
}
|
||
|
||
const hasPageAccess = Array.isArray(resolvedUser.allowedPageKeys)
|
||
&& resolvedUser.allowedPageKeys.includes(PLANNING_UPLOAD_PAGE_KEY);
|
||
return {
|
||
...resolvedUser,
|
||
planningUploadAllowed: hasPageAccess,
|
||
};
|
||
}
|
||
|
||
async function importPlanningWorkbook({ fileBuffer, originalName, user, expectedKind }) {
|
||
if (!fileBuffer || !Buffer.isBuffer(fileBuffer)) {
|
||
throw new Error('上传文件内容无效');
|
||
}
|
||
|
||
await ensurePlanningCanonicalSchemas();
|
||
|
||
const normalizedOriginalName = normalizeUploadedFileName(originalName);
|
||
const parsed = ensureExpectedTemplate(
|
||
parseWorkbook(fileBuffer, normalizedOriginalName, user, {
|
||
expectedKind,
|
||
}),
|
||
expectedKind
|
||
);
|
||
|
||
let replacedExistingRows = 0;
|
||
let dedupedExistingRows = 0;
|
||
let replacedInUploadRows = 0;
|
||
|
||
await planningDatabase.withTransaction(async (connection) => {
|
||
for (const sheet of parsed.parsedSheets) {
|
||
await ensurePlanningSheetTable(connection, sheet);
|
||
const fingerprintSyncResult = await syncPlanningFingerprints(connection, sheet.targetTableName);
|
||
const businessColumns = fingerprintSyncResult.businessColumns.length > 0
|
||
? fingerprintSyncResult.businessColumns
|
||
: sheet.sourceColumns.map((column) => column.columnName);
|
||
const incomingFingerprintResult = assignIncomingFingerprints(sheet, businessColumns);
|
||
dedupedExistingRows += fingerprintSyncResult.dedupedCount;
|
||
replacedInUploadRows += incomingFingerprintResult.replacedInUploadCount;
|
||
replacedExistingRows += await deleteOverlappingRows(
|
||
connection,
|
||
sheet.targetTableName,
|
||
sheet.rows.map((row) => row[ROW_FINGERPRINT_COLUMN])
|
||
);
|
||
await insertSheetRows(connection, sheet);
|
||
await syncTaobaoAppearanceCodesFromSearchSalesSheet(connection, sheet);
|
||
}
|
||
});
|
||
|
||
parsed.totalRows = parsed.parsedSheets.reduce((sum, sheet) => sum + sheet.rows.length, 0);
|
||
|
||
await ensurePlanningUploadBatchSchema();
|
||
await planningDatabase.insert(IMPORT_BATCH_TABLE, {
|
||
batch_id: parsed.batchId,
|
||
source_file_name: normalizedOriginalName,
|
||
target_table_name: parsed.targetTableNames.join(' / '),
|
||
row_count: parsed.totalRows,
|
||
success_count: parsed.totalRows,
|
||
source_sheet_summary: JSON.stringify(
|
||
parsed.parsedSheets.map((sheet) => ({
|
||
kind: sheet.kind,
|
||
label: sheet.label,
|
||
sheetName: sheet.sheetName,
|
||
targetTableName: sheet.targetTableName,
|
||
rowCount: sheet.rowCount,
|
||
}))
|
||
),
|
||
uploaded_by_user_id: user.id,
|
||
uploaded_by_username: user.username || null,
|
||
uploaded_by_name: user.name || null,
|
||
uploaded_department_id: user.departmentId || null,
|
||
uploaded_department_name: user.departmentName || null,
|
||
});
|
||
|
||
invalidatePlanningRuntimeCache();
|
||
|
||
return {
|
||
batchId: parsed.batchId,
|
||
tableName: PLANNING_UPLOAD_TABLE_NAME,
|
||
tableNames: parsed.targetTableNames,
|
||
totalRows: parsed.totalRows,
|
||
replacedExistingRows,
|
||
dedupedExistingRows,
|
||
replacedInUploadRows,
|
||
sheets: parsed.parsedSheets.map((sheet) => ({
|
||
kind: sheet.kind,
|
||
label: sheet.label,
|
||
sheetName: sheet.sheetName,
|
||
targetTableName: sheet.targetTableName,
|
||
rowCount: sheet.rowCount,
|
||
})),
|
||
};
|
||
}
|
||
|
||
module.exports = {
|
||
PLANNING_UPLOAD_PAGE_KEY,
|
||
PLANNING_UPLOAD_TABLE_NAME,
|
||
PLANNING_UPLOAD_TEMPLATES: getTemplateCatalog(),
|
||
getPlanningUploadBatches,
|
||
getPlanningTableData,
|
||
buildPlanningTableExportBuffer,
|
||
getPlanningPageSummaries,
|
||
getPlanningTemplateByKind,
|
||
getPlanningUploadTemplateCatalog: getTemplateCatalog,
|
||
getPlanningTemplateSummaries,
|
||
resolvePlanningUploadUser,
|
||
ensurePlanningCanonicalSchemas,
|
||
invalidatePlanningRuntimeCache,
|
||
importPlanningWorkbook,
|
||
};
|