chore(master-data): 增加聚水潭分类匹配计划导出
This commit is contained in:
WuYongJie
parent
4ca58ffaa2
commit
4974869f6a
@ -25,6 +25,7 @@
|
||||
"db:cleanup-master-data:validate": "node scripts/apply-master-data-cleanup-plan.js --validate-plan",
|
||||
"db:export-product-category-review": "node scripts/export-non-integrated-product-category-review.js",
|
||||
"db:build-product-category-cleanup-plan": "node scripts/build-non-integrated-product-category-cleanup-plan.js",
|
||||
"db:match-jst-product-category-cleanup-plan": "node scripts/match-jst-product-category-cleanup-plan.js",
|
||||
"prisma:seed": "tsx prisma/seed.ts",
|
||||
"open-platform:seed": "tsx scripts/seed-open-platform-demo.ts",
|
||||
"type-check": "node scripts/run-type-check.js",
|
||||
|
||||
@ -0,0 +1,870 @@
|
||||
/* eslint-disable @typescript-eslint/no-require-imports */
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const mysql = require('mysql2/promise')
|
||||
const { Pool } = require('pg')
|
||||
const {
|
||||
APPLY_CONFIRMATION,
|
||||
validateCleanupPlanForApply,
|
||||
writeJsonReport,
|
||||
} = require('./apply-master-data-cleanup-plan')
|
||||
|
||||
const ROOT_DIR = path.resolve(__dirname, '..')
|
||||
const DEFAULT_OUTPUT_DIR =
|
||||
'.master-data-cleanup/jst-product-category-match-current'
|
||||
const DEFAULT_CONFIRMED_BY = '聚水潭来源分类匹配'
|
||||
const SPU_RULE_LIBRARY_TYPE = 'SPU_RULE'
|
||||
|
||||
const SOURCE_PROBES = [
|
||||
{
|
||||
label: 'MAY商品快照',
|
||||
table: 'planning_may_product_snapshot_di',
|
||||
mode: 'sku',
|
||||
keyFields: ['may_sku_code', 'product_code', 'spu_code'],
|
||||
categoryFields: ['category_name'],
|
||||
nameFields: ['product_name', 'sku_name'],
|
||||
},
|
||||
{
|
||||
label: 'MAY商品评审快照',
|
||||
table: 'planning_may_product_review_snapshot_di',
|
||||
mode: 'sku',
|
||||
keyFields: ['may_sku_code'],
|
||||
categoryFields: ['category_name'],
|
||||
nameFields: [],
|
||||
},
|
||||
{
|
||||
label: 'MAY渠道活跃SKU索引',
|
||||
table: 'planning_may_channel_active_sku_index_di',
|
||||
mode: 'sku',
|
||||
keyFields: ['may_sku_code'],
|
||||
categoryFields: ['category_name'],
|
||||
nameFields: [],
|
||||
},
|
||||
{
|
||||
label: 'MAY渠道布局快照',
|
||||
table: 'planning_may_channel_layout_snapshot_di',
|
||||
mode: 'sku',
|
||||
keyFields: ['may_sku_code'],
|
||||
categoryFields: ['category_name'],
|
||||
nameFields: [],
|
||||
},
|
||||
{
|
||||
label: '淘宝产品记录商品ID',
|
||||
table: 'taobao_product_records',
|
||||
mode: 'product',
|
||||
keyFields: ['item_id', 'requested_item_id', 'source_item_id'],
|
||||
categoryFields: [
|
||||
'market_category_name',
|
||||
'standard_product_type',
|
||||
'category_level3',
|
||||
'category_level2',
|
||||
'fulfillment_category_level3',
|
||||
'fulfillment_category_level2',
|
||||
],
|
||||
nameFields: ['title'],
|
||||
},
|
||||
{
|
||||
label: '淘宝产品记录MAY-SKU',
|
||||
table: 'taobao_product_records',
|
||||
mode: 'sku',
|
||||
keyFields: ['may_sku_code'],
|
||||
categoryFields: [
|
||||
'market_category_name',
|
||||
'standard_product_type',
|
||||
'category_level3',
|
||||
'category_level2',
|
||||
'fulfillment_category_level3',
|
||||
'fulfillment_category_level2',
|
||||
],
|
||||
nameFields: ['title'],
|
||||
},
|
||||
{
|
||||
label: '淘宝产品记录产品名',
|
||||
table: 'taobao_product_records',
|
||||
mode: 'name',
|
||||
keyFields: ['title'],
|
||||
categoryFields: [
|
||||
'market_category_name',
|
||||
'standard_product_type',
|
||||
'category_level3',
|
||||
'category_level2',
|
||||
'fulfillment_category_level3',
|
||||
'fulfillment_category_level2',
|
||||
],
|
||||
nameFields: [],
|
||||
},
|
||||
{
|
||||
label: '市场Top商品',
|
||||
table: 'ods_lcf_mkt_top_goods_di',
|
||||
mode: 'product',
|
||||
keyFields: ['goods_id'],
|
||||
categoryFields: ['three_category', 'two_category', 'one_category'],
|
||||
nameFields: ['goods_name'],
|
||||
},
|
||||
{
|
||||
label: '市场洞察商品排行',
|
||||
table: 'planning_hsds_market_insight_goods_rankings',
|
||||
mode: 'product',
|
||||
keyFields: ['goods_id'],
|
||||
categoryFields: [
|
||||
'tertiary_category_name',
|
||||
'secondary_category_name',
|
||||
'yz_cate_name',
|
||||
'second_cate_name',
|
||||
'cate_path_name',
|
||||
],
|
||||
nameFields: ['goods_name'],
|
||||
},
|
||||
{
|
||||
label: '排行看板商品项',
|
||||
table: 'planning_rank_board_item_snapshot_di',
|
||||
mode: 'product',
|
||||
keyFields: ['item_id'],
|
||||
categoryFields: [
|
||||
'market_category_name',
|
||||
'category_level3',
|
||||
'category_level2',
|
||||
'category_path',
|
||||
],
|
||||
nameFields: ['title'],
|
||||
},
|
||||
{
|
||||
label: '图片特征库SKU',
|
||||
table: 'ai_image_feature_library',
|
||||
mode: 'sku',
|
||||
keyFields: ['sku_id'],
|
||||
categoryFields: ['category_name', 'category_tag'],
|
||||
nameFields: ['product_name'],
|
||||
},
|
||||
]
|
||||
|
||||
function readText(value) {
|
||||
return value === null || value === undefined ? '' : String(value).trim()
|
||||
}
|
||||
|
||||
function normalizeKey(value) {
|
||||
return readText(value).toUpperCase()
|
||||
}
|
||||
|
||||
function uniqueValues(values) {
|
||||
return Array.from(new Set(values.map(readText).filter(Boolean)))
|
||||
}
|
||||
|
||||
function uniqueKeys(values) {
|
||||
return Array.from(new Set(values.map(normalizeKey).filter(Boolean)))
|
||||
}
|
||||
|
||||
function chunk(values, size) {
|
||||
const chunks = []
|
||||
for (let index = 0; index < values.length; index += size) {
|
||||
chunks.push(values.slice(index, index + size))
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value, name) {
|
||||
const number = Number(value)
|
||||
if (!Number.isInteger(number) || number <= 0) {
|
||||
throw new Error(`${name} 必须是正整数。`)
|
||||
}
|
||||
return number
|
||||
}
|
||||
|
||||
function parseArgs(argv = process.argv.slice(2)) {
|
||||
const options = {
|
||||
outputDir: DEFAULT_OUTPUT_DIR,
|
||||
confirmedBy: DEFAULT_CONFIRMED_BY,
|
||||
limit: 0,
|
||||
summaryStdout: false,
|
||||
}
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index]
|
||||
if (arg === '--output-dir') {
|
||||
const value = argv[index + 1]
|
||||
if (!value || value.startsWith('--')) throw new Error('缺少 --output-dir <directory>。')
|
||||
options.outputDir = value
|
||||
index += 1
|
||||
continue
|
||||
}
|
||||
if (arg.startsWith('--output-dir=')) {
|
||||
options.outputDir = arg.slice('--output-dir='.length)
|
||||
if (!options.outputDir) throw new Error('缺少 --output-dir <directory>。')
|
||||
continue
|
||||
}
|
||||
if (arg === '--confirmed-by') {
|
||||
const value = argv[index + 1]
|
||||
if (!value || value.startsWith('--')) throw new Error('缺少 --confirmed-by <name>。')
|
||||
options.confirmedBy = value
|
||||
index += 1
|
||||
continue
|
||||
}
|
||||
if (arg.startsWith('--confirmed-by=')) {
|
||||
options.confirmedBy = arg.slice('--confirmed-by='.length)
|
||||
if (!options.confirmedBy.trim()) throw new Error('缺少 --confirmed-by <name>。')
|
||||
continue
|
||||
}
|
||||
if (arg === '--limit') {
|
||||
options.limit = parsePositiveInteger(argv[index + 1], '--limit')
|
||||
index += 1
|
||||
continue
|
||||
}
|
||||
if (arg.startsWith('--limit=')) {
|
||||
options.limit = parsePositiveInteger(arg.slice('--limit='.length), '--limit')
|
||||
continue
|
||||
}
|
||||
if (arg === '--summary-stdout') {
|
||||
options.summaryStdout = true
|
||||
continue
|
||||
}
|
||||
throw new Error(`未知参数：${arg}`)
|
||||
}
|
||||
|
||||
if (!options.confirmedBy.trim()) throw new Error('缺少 --confirmed-by <name>。')
|
||||
return options
|
||||
}
|
||||
|
||||
function loadEnvValue(name, rootDir = ROOT_DIR) {
|
||||
if (process.env[name]) return process.env[name]
|
||||
const envPath = path.join(rootDir, '.env')
|
||||
if (!fs.existsSync(envPath)) return ''
|
||||
|
||||
const content = fs.readFileSync(envPath, 'utf8')
|
||||
for (const line of content.split(/\r?\n/)) {
|
||||
const trimmed = line.trim()
|
||||
if (!trimmed || trimmed.startsWith('#')) continue
|
||||
const separatorIndex = trimmed.indexOf('=')
|
||||
if (separatorIndex <= 0) continue
|
||||
const key = trimmed.slice(0, separatorIndex).trim()
|
||||
if (key !== name) continue
|
||||
let value = trimmed.slice(separatorIndex + 1).trim()
|
||||
if (
|
||||
(value.startsWith('"') && value.endsWith('"')) ||
|
||||
(value.startsWith("'") && value.endsWith("'"))
|
||||
) {
|
||||
value = value.slice(1, -1)
|
||||
}
|
||||
return value
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function loadDatabaseUrl() {
|
||||
return loadEnvValue('DATABASE_URL')
|
||||
}
|
||||
|
||||
function createMysqlConnectionConfig() {
|
||||
const databaseUrl = loadEnvValue('JST_SKU_DATABASE_URL')
|
||||
if (databaseUrl) return databaseUrl
|
||||
|
||||
const host = loadEnvValue('JST_SKU_DB_HOST')
|
||||
const database = loadEnvValue('JST_SKU_DB_NAME')
|
||||
const user = loadEnvValue('JST_SKU_DB_USER')
|
||||
const password = loadEnvValue('JST_SKU_DB_PASSWORD')
|
||||
if (!host || !database || !user) {
|
||||
throw new Error('缺少 JST_SKU_DATABASE_URL 或 JST_SKU_DB_HOST/JST_SKU_DB_NAME/JST_SKU_DB_USER。')
|
||||
}
|
||||
|
||||
return {
|
||||
host,
|
||||
port: Number(loadEnvValue('JST_SKU_DB_PORT') || '3306'),
|
||||
database,
|
||||
user,
|
||||
password,
|
||||
charset: loadEnvValue('JST_SKU_DB_CHARSET') || 'utf8mb4',
|
||||
ssl: loadEnvValue('JST_SKU_DB_SSL') === 'true' ? {} : undefined,
|
||||
supportBigNumbers: true,
|
||||
dateStrings: true,
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchMissingProductCategoryProducts(client, options = {}) {
|
||||
const values = []
|
||||
let limitSql = ''
|
||||
if (options.limit && options.limit > 0) {
|
||||
values.push(options.limit)
|
||||
limitSql = `LIMIT $${values.length}`
|
||||
}
|
||||
|
||||
const result = await client.query(`
|
||||
SELECT
|
||||
product."id" AS "productId",
|
||||
product."spu",
|
||||
product."spuBase",
|
||||
product."brand",
|
||||
profile."sourceSpu",
|
||||
profile."productName",
|
||||
profile."productCategory",
|
||||
profile."productDevelopmentKind",
|
||||
profile."sourceCategory",
|
||||
profile."sourceItemType"
|
||||
FROM "IntakeNonIntegratedProductProfile" profile
|
||||
JOIN "IntakeNonIntegratedProduct" product ON product."id" = profile."productId"
|
||||
WHERE product."deletedAt" IS NULL
|
||||
AND NULLIF(btrim(profile."productCategory"), '') IS NULL
|
||||
ORDER BY profile."sourceSpu", product."spu"
|
||||
${limitSql}
|
||||
`, values)
|
||||
|
||||
return result.rows.map((row) => ({
|
||||
productId: readText(row.productId),
|
||||
spu: readText(row.spu),
|
||||
spuBase: readText(row.spuBase),
|
||||
brand: readText(row.brand),
|
||||
sourceSpu: readText(row.sourceSpu),
|
||||
productName: readText(row.productName),
|
||||
productCategory: readText(row.productCategory),
|
||||
productDevelopmentKind: readText(row.productDevelopmentKind),
|
||||
sourceCategory: readText(row.sourceCategory),
|
||||
sourceItemType: readText(row.sourceItemType),
|
||||
}))
|
||||
}
|
||||
|
||||
async function fetchMissingProductCategorySkuRows(client, productIds) {
|
||||
if (productIds.length === 0) return []
|
||||
const result = await client.query(`
|
||||
SELECT
|
||||
item."productId",
|
||||
item."sourceSkuId",
|
||||
item."barcode",
|
||||
item."skuCode",
|
||||
item."model",
|
||||
item."specName"
|
||||
FROM "IntakeNonIntegratedSkuItem" item
|
||||
WHERE item."productId" = ANY($1::text[])
|
||||
`, [productIds])
|
||||
|
||||
return result.rows.map((row) => ({
|
||||
productId: readText(row.productId),
|
||||
sourceSkuId: readText(row.sourceSkuId),
|
||||
barcode: readText(row.barcode),
|
||||
skuCode: readText(row.skuCode),
|
||||
model: readText(row.model),
|
||||
specName: readText(row.specName),
|
||||
}))
|
||||
}
|
||||
|
||||
async function fetchAllowedProductCategories(client) {
|
||||
const result = await client.query(`
|
||||
SELECT DISTINCT btrim("name") AS name
|
||||
FROM "ProductCategory"
|
||||
WHERE "isActive" IS TRUE
|
||||
AND NULLIF(btrim("name"), '') IS NOT NULL
|
||||
UNION
|
||||
SELECT DISTINCT btrim("productCategory") AS name
|
||||
FROM "SpuNumberCounter"
|
||||
WHERE NULLIF(btrim("productCategory"), '') IS NOT NULL
|
||||
UNION
|
||||
SELECT DISTINCT btrim("name") AS name
|
||||
FROM "IntakeReferenceLibraryItem"
|
||||
WHERE "libraryType" = $1
|
||||
AND "isActive" IS TRUE
|
||||
AND NULLIF(btrim("name"), '') IS NOT NULL
|
||||
`, [SPU_RULE_LIBRARY_TYPE])
|
||||
|
||||
return uniqueValues(result.rows.map((row) => row.name))
|
||||
}
|
||||
|
||||
async function fetchExistingColumns(connection, table, columns) {
|
||||
const [rows] = await connection.query(
|
||||
`SELECT column_name AS columnName
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = ?`,
|
||||
[table],
|
||||
)
|
||||
const columnSet = new Set(rows.map((row) => readText(row.columnName)))
|
||||
return columns.filter((column) => columnSet.has(column))
|
||||
}
|
||||
|
||||
async function fetchRowsByValues(connection, table, fields, keyField, values) {
|
||||
if (fields.length === 0 || values.length === 0) return []
|
||||
const rows = []
|
||||
for (const part of chunk(values, 300)) {
|
||||
const placeholders = part.map(() => '?').join(', ')
|
||||
const [batch] = await connection.execute(
|
||||
`SELECT ${fields.map((field) => `\`${field}\``).join(', ')}
|
||||
FROM \`${table}\`
|
||||
WHERE \`${keyField}\` IN (${placeholders})`,
|
||||
part,
|
||||
)
|
||||
rows.push(...batch)
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
function createProductLookup(products) {
|
||||
const byId = new Map()
|
||||
const bySourceKey = new Map()
|
||||
const byName = new Map()
|
||||
for (const product of products) {
|
||||
if (product.productId) byId.set(product.productId, product)
|
||||
for (const key of uniqueKeys([product.sourceSpu, product.spu])) {
|
||||
bySourceKey.set(key, product)
|
||||
}
|
||||
if (product.productName) byName.set(product.productName, product)
|
||||
}
|
||||
return { byId, bySourceKey, byName }
|
||||
}
|
||||
|
||||
function createSkuLookup(products, skuRows, jstRows = []) {
|
||||
const lookup = createProductLookup(products)
|
||||
const bySku = new Map()
|
||||
for (const sku of skuRows) {
|
||||
const product = lookup.byId.get(sku.productId)
|
||||
if (!product) continue
|
||||
for (const key of uniqueKeys([sku.sourceSkuId, sku.barcode, sku.skuCode])) {
|
||||
bySku.set(key, product)
|
||||
}
|
||||
}
|
||||
for (const row of jstRows) {
|
||||
const product = lookup.bySourceKey.get(normalizeKey(row.__source_spu))
|
||||
if (!product) continue
|
||||
const skuCodes = readText(row.sku_codes)
|
||||
.split(/[,，\s]+/)
|
||||
.map(normalizeKey)
|
||||
for (const key of uniqueKeys([row.sku_id, row.sku_code, ...skuCodes])) {
|
||||
bySku.set(key, product)
|
||||
}
|
||||
}
|
||||
return bySku
|
||||
}
|
||||
|
||||
async function fetchJstSkuRows(connection, sourceKeys) {
|
||||
const availableColumns = await fetchExistingColumns(connection, 'jst_sku', [
|
||||
'sku_id',
|
||||
'i_id',
|
||||
'name',
|
||||
'category',
|
||||
'sku_code',
|
||||
'sku_codes',
|
||||
])
|
||||
if (!availableColumns.includes('sku_id')) {
|
||||
throw new Error('jst_sku 缺少 sku_id 字段，无法反查聚水潭来源。')
|
||||
}
|
||||
|
||||
const sourceExpression = availableColumns.includes('i_id')
|
||||
? 'COALESCE(NULLIF(TRIM(CAST(`i_id` AS CHAR)), \'\'), TRIM(CAST(`sku_id` AS CHAR)))'
|
||||
: 'TRIM(CAST(`sku_id` AS CHAR))'
|
||||
const rows = []
|
||||
for (const part of chunk(sourceKeys, 300)) {
|
||||
if (part.length === 0) continue
|
||||
const placeholders = part.map(() => '?').join(', ')
|
||||
const [batch] = await connection.execute(
|
||||
`SELECT ${availableColumns.map((column) => `\`${column}\``).join(', ')},
|
||||
${sourceExpression} AS __source_spu
|
||||
FROM \`jst_sku\`
|
||||
WHERE ${sourceExpression} IN (${placeholders})`,
|
||||
part,
|
||||
)
|
||||
rows.push(...batch)
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
function resolveProductForProbeRow({ mode, row, keyField, productLookup, skuLookup }) {
|
||||
if (mode === 'sku') return skuLookup.get(normalizeKey(row[keyField])) || null
|
||||
if (mode === 'name') return productLookup.byName.get(readText(row[keyField])) || null
|
||||
return productLookup.bySourceKey.get(normalizeKey(row[keyField])) || null
|
||||
}
|
||||
|
||||
function readProbeRowName(row, nameFields) {
|
||||
for (const field of nameFields) {
|
||||
const value = readText(row[field])
|
||||
if (value) return value
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function addEvidence(evidenceByProductId, productId, evidence) {
|
||||
const rows = evidenceByProductId.get(productId) || []
|
||||
rows.push(evidence)
|
||||
evidenceByProductId.set(productId, rows)
|
||||
}
|
||||
|
||||
function splitCategoryPath(value) {
|
||||
return readText(value)
|
||||
.split(/\s*(?:>|＞|,|，|、| - |－|—|–)\s*/)
|
||||
.map(readText)
|
||||
.filter(Boolean)
|
||||
}
|
||||
|
||||
function resolveAllowedCategories(rawCategory, allowedCategories) {
|
||||
const category = readText(rawCategory)
|
||||
if (!category) return []
|
||||
if (allowedCategories.has(category)) return [category]
|
||||
return uniqueValues(splitCategoryPath(category).filter((item) => allowedCategories.has(item)))
|
||||
}
|
||||
|
||||
function normalizeEvidence(rawEvidence, allowedCategories) {
|
||||
return {
|
||||
...rawEvidence,
|
||||
rawCategory: readText(rawEvidence.rawCategory),
|
||||
resolvedCategories: resolveAllowedCategories(rawEvidence.rawCategory, allowedCategories),
|
||||
}
|
||||
}
|
||||
|
||||
function summarizeSourceStats(sourceStats, evidenceByProductId) {
|
||||
return sourceStats.map((stat) => {
|
||||
const uniqueResolvedProducts = new Set()
|
||||
const ambiguousResolvedProducts = new Set()
|
||||
for (const [productId, evidences] of evidenceByProductId.entries()) {
|
||||
const resolvedCategories = uniqueValues(
|
||||
evidences
|
||||
.filter((evidence) => evidence.source === stat.label)
|
||||
.flatMap((evidence) => evidence.resolvedCategories || []),
|
||||
)
|
||||
if (resolvedCategories.length === 1) uniqueResolvedProducts.add(productId)
|
||||
if (resolvedCategories.length > 1) ambiguousResolvedProducts.add(productId)
|
||||
}
|
||||
return {
|
||||
...stat,
|
||||
uniqueResolvedProducts: uniqueResolvedProducts.size,
|
||||
ambiguousResolvedProducts: ambiguousResolvedProducts.size,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async function collectSourceEvidence({
|
||||
connection,
|
||||
products,
|
||||
skuRows,
|
||||
allowedCategories,
|
||||
probes = SOURCE_PROBES,
|
||||
}) {
|
||||
const productLookup = createProductLookup(products)
|
||||
const sourceKeys = uniqueKeys(products.flatMap((product) => [product.sourceSpu, product.spu]))
|
||||
const jstRows = await fetchJstSkuRows(connection, sourceKeys)
|
||||
const skuLookup = createSkuLookup(products, skuRows, jstRows)
|
||||
const structuredSkuCodes = uniqueKeys(
|
||||
skuRows.flatMap((sku) => [sku.sourceSkuId, sku.barcode, sku.skuCode]),
|
||||
)
|
||||
const allSkuCodes = uniqueKeys([...structuredSkuCodes, ...Array.from(skuLookup.keys())])
|
||||
const productNames = uniqueValues(products.map((product) => product.productName))
|
||||
const evidenceByProductId = new Map()
|
||||
const sourceStats = []
|
||||
|
||||
for (const probe of probes) {
|
||||
const keyFields = await fetchExistingColumns(connection, probe.table, probe.keyFields)
|
||||
const categoryFields = await fetchExistingColumns(
|
||||
connection,
|
||||
probe.table,
|
||||
probe.categoryFields,
|
||||
)
|
||||
const nameFields = await fetchExistingColumns(connection, probe.table, probe.nameFields)
|
||||
let rawRows = 0
|
||||
const productsWithEvidence = new Set()
|
||||
const productsWithResolved = new Set()
|
||||
|
||||
if (keyFields.length > 0 && categoryFields.length > 0) {
|
||||
const values =
|
||||
probe.mode === 'sku' ? allSkuCodes : probe.mode === 'name' ? productNames : sourceKeys
|
||||
for (const keyField of keyFields) {
|
||||
const fields = uniqueValues([keyField, ...categoryFields, ...nameFields])
|
||||
const rows = await fetchRowsByValues(connection, probe.table, fields, keyField, values)
|
||||
rawRows += rows.length
|
||||
for (const row of rows) {
|
||||
const product = resolveProductForProbeRow({
|
||||
mode: probe.mode,
|
||||
row,
|
||||
keyField,
|
||||
productLookup,
|
||||
skuLookup,
|
||||
})
|
||||
if (!product) continue
|
||||
|
||||
for (const categoryField of categoryFields) {
|
||||
const rawCategory = readText(row[categoryField])
|
||||
if (!rawCategory) continue
|
||||
const evidence = normalizeEvidence(
|
||||
{
|
||||
source: probe.label,
|
||||
table: probe.table,
|
||||
mode: probe.mode,
|
||||
keyField,
|
||||
key: readText(row[keyField]),
|
||||
categoryField,
|
||||
rawCategory,
|
||||
matchedName: readProbeRowName(row, nameFields),
|
||||
},
|
||||
allowedCategories,
|
||||
)
|
||||
productsWithEvidence.add(product.productId)
|
||||
if (evidence.resolvedCategories.length > 0) {
|
||||
productsWithResolved.add(product.productId)
|
||||
}
|
||||
addEvidence(evidenceByProductId, product.productId, evidence)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sourceStats.push({
|
||||
label: probe.label,
|
||||
table: probe.table,
|
||||
mode: probe.mode,
|
||||
keyFields,
|
||||
categoryFields,
|
||||
rawRows,
|
||||
productsWithEvidence: productsWithEvidence.size,
|
||||
productsWithResolved: productsWithResolved.size,
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
evidenceByProductId,
|
||||
sourceStats: summarizeSourceStats(sourceStats, evidenceByProductId),
|
||||
jstSummary: {
|
||||
jstSkuRows: jstRows.length,
|
||||
jstCategoryNonEmpty: jstRows.filter((row) => readText(row.category)).length,
|
||||
jstDerivedSkuCodes: skuLookup.size,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function summarizeEvidenceForPlan(evidences, category) {
|
||||
return evidences
|
||||
.filter((evidence) => (evidence.resolvedCategories || []).includes(category))
|
||||
.slice(0, 8)
|
||||
.map((evidence) => ({
|
||||
source: evidence.source,
|
||||
table: evidence.table,
|
||||
mode: evidence.mode,
|
||||
keyField: evidence.keyField,
|
||||
key: evidence.key,
|
||||
categoryField: evidence.categoryField,
|
||||
rawCategory: evidence.rawCategory,
|
||||
matchedName: evidence.matchedName,
|
||||
}))
|
||||
}
|
||||
|
||||
function buildProductProfilePlanEntry(match) {
|
||||
return {
|
||||
productId: match.productId,
|
||||
spu: match.spu,
|
||||
sourceSpu: match.sourceSpu,
|
||||
productName: match.productName,
|
||||
current: {
|
||||
productName: match.productName,
|
||||
productCategory: match.currentProductCategory,
|
||||
productDevelopmentKind: match.productDevelopmentKind,
|
||||
missingFields: ['productCategory'],
|
||||
},
|
||||
fill: {
|
||||
productCategory: match.productCategory,
|
||||
},
|
||||
evidence: {
|
||||
source: 'jst-product-category-match',
|
||||
matchedCategory: match.productCategory,
|
||||
sourceCount: match.sources.length,
|
||||
sources: match.sources,
|
||||
rows: match.sampleEvidence,
|
||||
},
|
||||
note: '只读来源库唯一命中正式产品分类字典；冲突和无证据记录不会进入本计划。',
|
||||
}
|
||||
}
|
||||
|
||||
function buildMatchArtifacts({
|
||||
products,
|
||||
evidenceByProductId,
|
||||
allowedCategories,
|
||||
confirmedBy = DEFAULT_CONFIRMED_BY,
|
||||
sourceStats = [],
|
||||
jstSummary = {},
|
||||
generatedAt = new Date().toISOString(),
|
||||
}) {
|
||||
const uniqueMatches = []
|
||||
const conflictMatches = []
|
||||
const unresolvedProducts = []
|
||||
const categoryCounts = new Map()
|
||||
|
||||
for (const product of products) {
|
||||
const evidences = evidenceByProductId.get(product.productId) || []
|
||||
const resolvedCategories = uniqueValues(
|
||||
evidences.flatMap((evidence) => evidence.resolvedCategories || []),
|
||||
)
|
||||
resolvedCategories.forEach((category) => {
|
||||
categoryCounts.set(category, (categoryCounts.get(category) || 0) + 1)
|
||||
})
|
||||
|
||||
if (resolvedCategories.length === 1) {
|
||||
const productCategory = resolvedCategories[0]
|
||||
const sources = uniqueValues(
|
||||
evidences
|
||||
.filter((evidence) => (evidence.resolvedCategories || []).includes(productCategory))
|
||||
.map((evidence) => evidence.source),
|
||||
)
|
||||
uniqueMatches.push({
|
||||
productId: product.productId,
|
||||
spu: product.spu,
|
||||
sourceSpu: product.sourceSpu,
|
||||
productName: product.productName,
|
||||
currentProductCategory: product.productCategory,
|
||||
productDevelopmentKind: product.productDevelopmentKind,
|
||||
productCategory,
|
||||
evidenceCount: evidences.length,
|
||||
sources,
|
||||
sampleEvidence: summarizeEvidenceForPlan(evidences, productCategory),
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
if (resolvedCategories.length > 1) {
|
||||
conflictMatches.push({
|
||||
productId: product.productId,
|
||||
spu: product.spu,
|
||||
sourceSpu: product.sourceSpu,
|
||||
productName: product.productName,
|
||||
categories: resolvedCategories,
|
||||
evidence: evidences.slice(0, 12),
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
unresolvedProducts.push({
|
||||
productId: product.productId,
|
||||
spu: product.spu,
|
||||
sourceSpu: product.sourceSpu,
|
||||
productName: product.productName,
|
||||
evidenceCount: evidences.length,
|
||||
evidence: evidences.slice(0, 8),
|
||||
})
|
||||
}
|
||||
|
||||
const cleanupPlan = {
|
||||
mode: 'jst-product-category-match-cleanup-plan',
|
||||
confirmation: APPLY_CONFIRMATION,
|
||||
confirmedBy: confirmedBy.trim(),
|
||||
skuItems: [],
|
||||
productProfiles: uniqueMatches.map(buildProductProfilePlanEntry),
|
||||
qualityDecisionRepairs: [],
|
||||
preSkuFormalizationMismatches: [],
|
||||
}
|
||||
const validation = cleanupPlan.productProfiles.length > 0
|
||||
? validateCleanupPlanForApply(cleanupPlan)
|
||||
: null
|
||||
const categoryHistogram = Array.from(categoryCounts.entries())
|
||||
.sort((left, right) => right[1] - left[1])
|
||||
.map(([category, count]) => ({ category, count }))
|
||||
const summary = {
|
||||
mode: 'jst-product-category-match',
|
||||
generatedAt,
|
||||
missingProducts: products.length,
|
||||
allowedCategoryCount: allowedCategories.size,
|
||||
totalProductsWithAnyEvidence: Array.from(evidenceByProductId.keys()).length,
|
||||
totalProductsWithResolved: uniqueMatches.length + conflictMatches.length,
|
||||
uniqueResolvedProducts: uniqueMatches.length,
|
||||
ambiguousResolvedProducts: conflictMatches.length,
|
||||
unresolvedProducts: unresolvedProducts.length,
|
||||
cleanupPlanRows: cleanupPlan.productProfiles.length,
|
||||
cleanupPlanAppliedFields: validation?.productProfiles?.appliedFields ?? 0,
|
||||
categoryHistogram,
|
||||
...jstSummary,
|
||||
}
|
||||
|
||||
return {
|
||||
summary,
|
||||
sourceStats,
|
||||
uniqueMatches,
|
||||
conflictMatches,
|
||||
unresolvedProducts,
|
||||
cleanupPlan,
|
||||
validation,
|
||||
}
|
||||
}
|
||||
|
||||
async function run(options) {
|
||||
const databaseUrl = loadDatabaseUrl()
|
||||
if (!databaseUrl) throw new Error('缺少 DATABASE_URL。')
|
||||
|
||||
const pgPool = new Pool({ connectionString: databaseUrl })
|
||||
const mysqlConnection = await mysql.createConnection(createMysqlConnectionConfig())
|
||||
try {
|
||||
const pgClient = await pgPool.connect()
|
||||
let products
|
||||
let skuRows
|
||||
let allowedCategories
|
||||
try {
|
||||
products = await fetchMissingProductCategoryProducts(pgClient, options)
|
||||
skuRows = await fetchMissingProductCategorySkuRows(
|
||||
pgClient,
|
||||
products.map((product) => product.productId),
|
||||
)
|
||||
allowedCategories = new Set(await fetchAllowedProductCategories(pgClient))
|
||||
} finally {
|
||||
pgClient.release()
|
||||
}
|
||||
|
||||
const evidence = await collectSourceEvidence({
|
||||
connection: mysqlConnection,
|
||||
products,
|
||||
skuRows,
|
||||
allowedCategories,
|
||||
})
|
||||
const artifacts = buildMatchArtifacts({
|
||||
products,
|
||||
evidenceByProductId: evidence.evidenceByProductId,
|
||||
allowedCategories,
|
||||
confirmedBy: options.confirmedBy,
|
||||
sourceStats: evidence.sourceStats,
|
||||
jstSummary: evidence.jstSummary,
|
||||
})
|
||||
|
||||
const outputDir = options.outputDir
|
||||
const summaryFile = writeJsonReport(path.join(outputDir, 'summary.json'), artifacts.summary)
|
||||
const reportFile = writeJsonReport(path.join(outputDir, 'report.json'), {
|
||||
summary: artifacts.summary,
|
||||
sourceStats: artifacts.sourceStats,
|
||||
uniqueMatches: artifacts.uniqueMatches,
|
||||
conflictMatches: artifacts.conflictMatches,
|
||||
unresolvedProducts: artifacts.unresolvedProducts,
|
||||
})
|
||||
const cleanupPlanFile = writeJsonReport(
|
||||
path.join(outputDir, 'cleanup-plan.json'),
|
||||
artifacts.cleanupPlan,
|
||||
)
|
||||
|
||||
return {
|
||||
...artifacts.summary,
|
||||
summaryFile,
|
||||
reportFile,
|
||||
cleanupPlanFile,
|
||||
}
|
||||
} finally {
|
||||
await mysqlConnection.end()
|
||||
await pgPool.end()
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2))
|
||||
const result = await run(options)
|
||||
if (options.summaryStdout) {
|
||||
console.log(JSON.stringify(result, null, 2))
|
||||
} else {
|
||||
console.log(`已导出聚水潭来源分类匹配报告：${result.reportFile}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error))
|
||||
process.exit(1)
|
||||
})
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
DEFAULT_CONFIRMED_BY,
|
||||
DEFAULT_OUTPUT_DIR,
|
||||
SOURCE_PROBES,
|
||||
parseArgs,
|
||||
readText,
|
||||
normalizeKey,
|
||||
splitCategoryPath,
|
||||
resolveAllowedCategories,
|
||||
normalizeEvidence,
|
||||
buildMatchArtifacts,
|
||||
buildProductProfilePlanEntry,
|
||||
createProductLookup,
|
||||
createSkuLookup,
|
||||
}
|
||||
@ -1,3 +1,4 @@
|
||||
/* eslint-disable @typescript-eslint/no-require-imports */
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
|
||||
@ -1157,6 +1158,10 @@ const approvedRawSqlMasterDataReadPaths = new Map([
|
||||
'scripts/export-non-integrated-product-category-review.js',
|
||||
'非集成产品分类审阅导出脚本只允许只读读取产品/profile/SKU 样例，生成业务确认清单和 cleanup plan 模板；不得直接写库或猜测分类。',
|
||||
],
|
||||
[
|
||||
'scripts/match-jst-product-category-cleanup-plan.js',
|
||||
'聚水潭来源分类匹配脚本只允许只读读取非集成产品/profile/SKU 和正式分类字典，结合 JST 来源库生成可审计 cleanup plan；不得直接写库或关键词猜分类。',
|
||||
],
|
||||
[
|
||||
'scripts/apply-pre-sku-formalization-repair.js',
|
||||
'准 SKU 生命周期专项修复入口允许读取 PreSku、ProductSKU、DraftGroup、包装和库位关系，用于 currentEvidence 校验、dry-run 和 apply 前事务内复核。',
|
||||
@ -1217,6 +1222,17 @@ const approvedRawSqlMasterDataReadTableScopes = new Map([
|
||||
'IntakeNonIntegratedSkuItem',
|
||||
]),
|
||||
],
|
||||
[
|
||||
'scripts/match-jst-product-category-cleanup-plan.js',
|
||||
new Set([
|
||||
'IntakeNonIntegratedProduct',
|
||||
'IntakeNonIntegratedProductProfile',
|
||||
'IntakeNonIntegratedSkuItem',
|
||||
'IntakeReferenceLibraryItem',
|
||||
'ProductCategory',
|
||||
'SpuNumberCounter',
|
||||
]),
|
||||
],
|
||||
[
|
||||
'scripts/apply-pre-sku-formalization-repair.js',
|
||||
new Set([
|
||||
|
||||
@ -0,0 +1,162 @@
|
||||
const {
|
||||
buildMatchArtifacts,
|
||||
createSkuLookup,
|
||||
normalizeEvidence,
|
||||
parseArgs,
|
||||
resolveAllowedCategories,
|
||||
splitCategoryPath,
|
||||
} = require('../../scripts/match-jst-product-category-cleanup-plan')
|
||||
|
||||
function product(overrides = {}) {
|
||||
return {
|
||||
productId: 'product-1',
|
||||
spu: 'SPU001',
|
||||
sourceSpu: '1001',
|
||||
productName: '书桌 A',
|
||||
productCategory: '',
|
||||
productDevelopmentKind: 'product',
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
function evidence(overrides = {}) {
|
||||
return normalizeEvidence(
|
||||
{
|
||||
source: '市场Top商品',
|
||||
table: 'ods_lcf_mkt_top_goods_di',
|
||||
mode: 'product',
|
||||
keyField: 'goods_id',
|
||||
key: '1001',
|
||||
categoryField: 'three_category',
|
||||
rawCategory: '住宅家具,桌类,书桌',
|
||||
matchedName: '书桌 A',
|
||||
...overrides,
|
||||
},
|
||||
new Set(['书桌', '电脑桌', '书柜']),
|
||||
)
|
||||
}
|
||||
|
||||
describe('match-jst-product-category-cleanup-plan', () => {
|
||||
it('parses CLI options', () => {
|
||||
expect(parseArgs([
|
||||
'--output-dir',
|
||||
'.out',
|
||||
'--confirmed-by=企划负责人',
|
||||
'--limit',
|
||||
'20',
|
||||
'--summary-stdout',
|
||||
])).toEqual({
|
||||
outputDir: '.out',
|
||||
confirmedBy: '企划负责人',
|
||||
limit: 20,
|
||||
summaryStdout: true,
|
||||
})
|
||||
expect(() => parseArgs(['--limit=0'])).toThrow('正整数')
|
||||
})
|
||||
|
||||
it('resolves allowed category from a source category path', () => {
|
||||
const allowed = new Set(['书桌', '电脑桌'])
|
||||
expect(splitCategoryPath('住宅家具,桌类,书桌')).toEqual(['住宅家具', '桌类', '书桌'])
|
||||
expect(resolveAllowedCategories('住宅家具 > 桌类 > 书桌', allowed)).toEqual(['书桌'])
|
||||
expect(resolveAllowedCategories('电脑桌', allowed)).toEqual(['电脑桌'])
|
||||
expect(resolveAllowedCategories('桌类', allowed)).toEqual([])
|
||||
})
|
||||
|
||||
it('builds a validate-ready cleanup plan for unique resolved matches', () => {
|
||||
const products = [product()]
|
||||
const evidenceByProductId = new Map([
|
||||
['product-1', [evidence(), evidence({ rawCategory: '书桌', source: '市场洞察商品排行' })]],
|
||||
])
|
||||
|
||||
const artifacts = buildMatchArtifacts({
|
||||
products,
|
||||
evidenceByProductId,
|
||||
allowedCategories: new Set(['书桌', '电脑桌']),
|
||||
confirmedBy: '企划负责人',
|
||||
sourceStats: [],
|
||||
jstSummary: { jstSkuRows: 1, jstCategoryNonEmpty: 0 },
|
||||
generatedAt: '2026-06-05T00:00:00.000Z',
|
||||
})
|
||||
|
||||
expect(artifacts.summary).toMatchObject({
|
||||
missingProducts: 1,
|
||||
uniqueResolvedProducts: 1,
|
||||
ambiguousResolvedProducts: 0,
|
||||
unresolvedProducts: 0,
|
||||
cleanupPlanRows: 1,
|
||||
cleanupPlanAppliedFields: 1,
|
||||
jstSkuRows: 1,
|
||||
jstCategoryNonEmpty: 0,
|
||||
})
|
||||
expect(artifacts.cleanupPlan).toMatchObject({
|
||||
confirmation: 'I_UNDERSTAND_THIS_UPDATES_MASTER_DATA',
|
||||
confirmedBy: '企划负责人',
|
||||
productProfiles: [
|
||||
expect.objectContaining({
|
||||
productId: 'product-1',
|
||||
sourceSpu: '1001',
|
||||
fill: { productCategory: '书桌' },
|
||||
current: expect.objectContaining({
|
||||
productCategory: '',
|
||||
productDevelopmentKind: 'product',
|
||||
missingFields: ['productCategory'],
|
||||
}),
|
||||
}),
|
||||
],
|
||||
})
|
||||
expect(artifacts.validation.productProfiles).toMatchObject({
|
||||
actionableRows: 1,
|
||||
appliedFields: 1,
|
||||
remainingMissingFieldCounts: {},
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps conflicting source categories out of the cleanup plan', () => {
|
||||
const products = [product()]
|
||||
const evidenceByProductId = new Map([
|
||||
[
|
||||
'product-1',
|
||||
[
|
||||
evidence({ rawCategory: '书桌' }),
|
||||
evidence({ rawCategory: '电脑桌', source: 'MAY商品快照' }),
|
||||
],
|
||||
],
|
||||
])
|
||||
|
||||
const artifacts = buildMatchArtifacts({
|
||||
products,
|
||||
evidenceByProductId,
|
||||
allowedCategories: new Set(['书桌', '电脑桌']),
|
||||
confirmedBy: '企划负责人',
|
||||
})
|
||||
|
||||
expect(artifacts.summary).toMatchObject({
|
||||
uniqueResolvedProducts: 0,
|
||||
ambiguousResolvedProducts: 1,
|
||||
unresolvedProducts: 0,
|
||||
cleanupPlanRows: 0,
|
||||
})
|
||||
expect(artifacts.cleanupPlan.productProfiles).toEqual([])
|
||||
expect(artifacts.conflictMatches[0]).toMatchObject({
|
||||
productId: 'product-1',
|
||||
categories: ['书桌', '电脑桌'],
|
||||
})
|
||||
})
|
||||
|
||||
it('uses jst sku codes to map source rows back to products', () => {
|
||||
const products = [product({ sourceSpu: 'i-001' })]
|
||||
const skuRows = [{ productId: 'product-1', sourceSkuId: '', barcode: '', skuCode: '' }]
|
||||
const skuLookup = createSkuLookup(products, skuRows, [
|
||||
{
|
||||
__source_spu: 'i-001',
|
||||
sku_id: 'sku-1',
|
||||
sku_code: 'may-1',
|
||||
sku_codes: 'alias-1, alias-2',
|
||||
},
|
||||
])
|
||||
|
||||
expect(skuLookup.get('SKU-1')?.productId).toBe('product-1')
|
||||
expect(skuLookup.get('MAY-1')?.productId).toBe('product-1')
|
||||
expect(skuLookup.get('ALIAS-2')?.productId).toBe('product-1')
|
||||
})
|
||||
})
|
||||
Loading…
x
Reference in New Issue
Block a user