From 4974869f6a778f4e2410e801ad1d83d9b372a3cf Mon Sep 17 00:00:00 2001 From: WuYongJie Date: Fri, 5 Jun 2026 10:34:22 +0800 Subject: [PATCH] =?UTF-8?q?chore(master-data):=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E8=81=9A=E6=B0=B4=E6=BD=AD=E5=88=86=E7=B1=BB=E5=8C=B9=E9=85=8D?= =?UTF-8?q?=E8=AE=A1=E5=88=92=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- furniture-3d-system/package.json | 1 + ...match-jst-product-category-cleanup-plan.js | 870 ++++++++++++++++++ .../verify-master-data-code-boundaries.js | 16 + ...-jst-product-category-cleanup-plan.test.js | 162 ++++ 4 files changed, 1049 insertions(+) create mode 100644 furniture-3d-system/scripts/match-jst-product-category-cleanup-plan.js create mode 100644 furniture-3d-system/tests/scripts/match-jst-product-category-cleanup-plan.test.js diff --git a/furniture-3d-system/package.json b/furniture-3d-system/package.json index b68d0b7a..5be29a8e 100644 --- a/furniture-3d-system/package.json +++ b/furniture-3d-system/package.json @@ -25,6 +25,7 @@ "db:cleanup-master-data:validate": "node scripts/apply-master-data-cleanup-plan.js --validate-plan", "db:export-product-category-review": "node scripts/export-non-integrated-product-category-review.js", "db:build-product-category-cleanup-plan": "node scripts/build-non-integrated-product-category-cleanup-plan.js", + "db:match-jst-product-category-cleanup-plan": "node scripts/match-jst-product-category-cleanup-plan.js", "prisma:seed": "tsx prisma/seed.ts", "open-platform:seed": "tsx scripts/seed-open-platform-demo.ts", "type-check": "node scripts/run-type-check.js", diff --git a/furniture-3d-system/scripts/match-jst-product-category-cleanup-plan.js b/furniture-3d-system/scripts/match-jst-product-category-cleanup-plan.js new file mode 100644 index 00000000..cff9b6ca --- /dev/null +++ b/furniture-3d-system/scripts/match-jst-product-category-cleanup-plan.js @@ -0,0 +1,870 @@ +/* eslint-disable @typescript-eslint/no-require-imports */ +const fs = require('node:fs') +const path = require('node:path') +const mysql = require('mysql2/promise') +const { Pool } = require('pg') +const { + APPLY_CONFIRMATION, + validateCleanupPlanForApply, + writeJsonReport, +} = require('./apply-master-data-cleanup-plan') + +const ROOT_DIR = path.resolve(__dirname, '..') +const DEFAULT_OUTPUT_DIR = + '.master-data-cleanup/jst-product-category-match-current' +const DEFAULT_CONFIRMED_BY = '聚水潭来源分类匹配' +const SPU_RULE_LIBRARY_TYPE = 'SPU_RULE' + +const SOURCE_PROBES = [ + { + label: 'MAY商品快照', + table: 'planning_may_product_snapshot_di', + mode: 'sku', + keyFields: ['may_sku_code', 'product_code', 'spu_code'], + categoryFields: ['category_name'], + nameFields: ['product_name', 'sku_name'], + }, + { + label: 'MAY商品评审快照', + table: 'planning_may_product_review_snapshot_di', + mode: 'sku', + keyFields: ['may_sku_code'], + categoryFields: ['category_name'], + nameFields: [], + }, + { + label: 'MAY渠道活跃SKU索引', + table: 'planning_may_channel_active_sku_index_di', + mode: 'sku', + keyFields: ['may_sku_code'], + categoryFields: ['category_name'], + nameFields: [], + }, + { + label: 'MAY渠道布局快照', + table: 'planning_may_channel_layout_snapshot_di', + mode: 'sku', + keyFields: ['may_sku_code'], + categoryFields: ['category_name'], + nameFields: [], + }, + { + label: '淘宝产品记录商品ID', + table: 'taobao_product_records', + mode: 'product', + keyFields: ['item_id', 'requested_item_id', 'source_item_id'], + categoryFields: [ + 'market_category_name', + 'standard_product_type', + 'category_level3', + 'category_level2', + 'fulfillment_category_level3', + 'fulfillment_category_level2', + ], + nameFields: ['title'], + }, + { + label: '淘宝产品记录MAY-SKU', + table: 'taobao_product_records', + mode: 'sku', + keyFields: ['may_sku_code'], + categoryFields: [ + 'market_category_name', + 'standard_product_type', + 'category_level3', + 'category_level2', + 'fulfillment_category_level3', + 'fulfillment_category_level2', + ], + nameFields: ['title'], + }, + { + label: '淘宝产品记录产品名', + table: 'taobao_product_records', + mode: 'name', + keyFields: ['title'], + categoryFields: [ + 'market_category_name', + 'standard_product_type', + 'category_level3', + 'category_level2', + 'fulfillment_category_level3', + 'fulfillment_category_level2', + ], + nameFields: [], + }, + { + label: '市场Top商品', + table: 'ods_lcf_mkt_top_goods_di', + mode: 'product', + keyFields: ['goods_id'], + categoryFields: ['three_category', 'two_category', 'one_category'], + nameFields: ['goods_name'], + }, + { + label: '市场洞察商品排行', + table: 'planning_hsds_market_insight_goods_rankings', + mode: 'product', + keyFields: ['goods_id'], + categoryFields: [ + 'tertiary_category_name', + 'secondary_category_name', + 'yz_cate_name', + 'second_cate_name', + 'cate_path_name', + ], + nameFields: ['goods_name'], + }, + { + label: '排行看板商品项', + table: 'planning_rank_board_item_snapshot_di', + mode: 'product', + keyFields: ['item_id'], + categoryFields: [ + 'market_category_name', + 'category_level3', + 'category_level2', + 'category_path', + ], + nameFields: ['title'], + }, + { + label: '图片特征库SKU', + table: 'ai_image_feature_library', + mode: 'sku', + keyFields: ['sku_id'], + categoryFields: ['category_name', 'category_tag'], + nameFields: ['product_name'], + }, +] + +function readText(value) { + return value === null || value === undefined ? '' : String(value).trim() +} + +function normalizeKey(value) { + return readText(value).toUpperCase() +} + +function uniqueValues(values) { + return Array.from(new Set(values.map(readText).filter(Boolean))) +} + +function uniqueKeys(values) { + return Array.from(new Set(values.map(normalizeKey).filter(Boolean))) +} + +function chunk(values, size) { + const chunks = [] + for (let index = 0; index < values.length; index += size) { + chunks.push(values.slice(index, index + size)) + } + return chunks +} + +function parsePositiveInteger(value, name) { + const number = Number(value) + if (!Number.isInteger(number) || number <= 0) { + throw new Error(`${name} 必须是正整数。`) + } + return number +} + +function parseArgs(argv = process.argv.slice(2)) { + const options = { + outputDir: DEFAULT_OUTPUT_DIR, + confirmedBy: DEFAULT_CONFIRMED_BY, + limit: 0, + summaryStdout: false, + } + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] + if (arg === '--output-dir') { + const value = argv[index + 1] + if (!value || value.startsWith('--')) throw new Error('缺少 --output-dir 。') + options.outputDir = value + index += 1 + continue + } + if (arg.startsWith('--output-dir=')) { + options.outputDir = arg.slice('--output-dir='.length) + if (!options.outputDir) throw new Error('缺少 --output-dir 。') + continue + } + if (arg === '--confirmed-by') { + const value = argv[index + 1] + if (!value || value.startsWith('--')) throw new Error('缺少 --confirmed-by 。') + options.confirmedBy = value + index += 1 + continue + } + if (arg.startsWith('--confirmed-by=')) { + options.confirmedBy = arg.slice('--confirmed-by='.length) + if (!options.confirmedBy.trim()) throw new Error('缺少 --confirmed-by 。') + continue + } + if (arg === '--limit') { + options.limit = parsePositiveInteger(argv[index + 1], '--limit') + index += 1 + continue + } + if (arg.startsWith('--limit=')) { + options.limit = parsePositiveInteger(arg.slice('--limit='.length), '--limit') + continue + } + if (arg === '--summary-stdout') { + options.summaryStdout = true + continue + } + throw new Error(`未知参数：${arg}`) + } + + if (!options.confirmedBy.trim()) throw new Error('缺少 --confirmed-by 。') + return options +} + +function loadEnvValue(name, rootDir = ROOT_DIR) { + if (process.env[name]) return process.env[name] + const envPath = path.join(rootDir, '.env') + if (!fs.existsSync(envPath)) return '' + + const content = fs.readFileSync(envPath, 'utf8') + for (const line of content.split(/\r?\n/)) { + const trimmed = line.trim() + if (!trimmed || trimmed.startsWith('#')) continue + const separatorIndex = trimmed.indexOf('=') + if (separatorIndex <= 0) continue + const key = trimmed.slice(0, separatorIndex).trim() + if (key !== name) continue + let value = trimmed.slice(separatorIndex + 1).trim() + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1) + } + return value + } + return '' +} + +function loadDatabaseUrl() { + return loadEnvValue('DATABASE_URL') +} + +function createMysqlConnectionConfig() { + const databaseUrl = loadEnvValue('JST_SKU_DATABASE_URL') + if (databaseUrl) return databaseUrl + + const host = loadEnvValue('JST_SKU_DB_HOST') + const database = loadEnvValue('JST_SKU_DB_NAME') + const user = loadEnvValue('JST_SKU_DB_USER') + const password = loadEnvValue('JST_SKU_DB_PASSWORD') + if (!host || !database || !user) { + throw new Error('缺少 JST_SKU_DATABASE_URL 或 JST_SKU_DB_HOST/JST_SKU_DB_NAME/JST_SKU_DB_USER。') + } + + return { + host, + port: Number(loadEnvValue('JST_SKU_DB_PORT') || '3306'), + database, + user, + password, + charset: loadEnvValue('JST_SKU_DB_CHARSET') || 'utf8mb4', + ssl: loadEnvValue('JST_SKU_DB_SSL') === 'true' ? {} : undefined, + supportBigNumbers: true, + dateStrings: true, + } +} + +async function fetchMissingProductCategoryProducts(client, options = {}) { + const values = [] + let limitSql = '' + if (options.limit && options.limit > 0) { + values.push(options.limit) + limitSql = `LIMIT $${values.length}` + } + + const result = await client.query(` + SELECT + product."id" AS "productId", + product."spu", + product."spuBase", + product."brand", + profile."sourceSpu", + profile."productName", + profile."productCategory", + profile."productDevelopmentKind", + profile."sourceCategory", + profile."sourceItemType" + FROM "IntakeNonIntegratedProductProfile" profile + JOIN "IntakeNonIntegratedProduct" product ON product."id" = profile."productId" + WHERE product."deletedAt" IS NULL + AND NULLIF(btrim(profile."productCategory"), '') IS NULL + ORDER BY profile."sourceSpu", product."spu" + ${limitSql} + `, values) + + return result.rows.map((row) => ({ + productId: readText(row.productId), + spu: readText(row.spu), + spuBase: readText(row.spuBase), + brand: readText(row.brand), + sourceSpu: readText(row.sourceSpu), + productName: readText(row.productName), + productCategory: readText(row.productCategory), + productDevelopmentKind: readText(row.productDevelopmentKind), + sourceCategory: readText(row.sourceCategory), + sourceItemType: readText(row.sourceItemType), + })) +} + +async function fetchMissingProductCategorySkuRows(client, productIds) { + if (productIds.length === 0) return [] + const result = await client.query(` + SELECT + item."productId", + item."sourceSkuId", + item."barcode", + item."skuCode", + item."model", + item."specName" + FROM "IntakeNonIntegratedSkuItem" item + WHERE item."productId" = ANY($1::text[]) + `, [productIds]) + + return result.rows.map((row) => ({ + productId: readText(row.productId), + sourceSkuId: readText(row.sourceSkuId), + barcode: readText(row.barcode), + skuCode: readText(row.skuCode), + model: readText(row.model), + specName: readText(row.specName), + })) +} + +async function fetchAllowedProductCategories(client) { + const result = await client.query(` + SELECT DISTINCT btrim("name") AS name + FROM "ProductCategory" + WHERE "isActive" IS TRUE + AND NULLIF(btrim("name"), '') IS NOT NULL + UNION + SELECT DISTINCT btrim("productCategory") AS name + FROM "SpuNumberCounter" + WHERE NULLIF(btrim("productCategory"), '') IS NOT NULL + UNION + SELECT DISTINCT btrim("name") AS name + FROM "IntakeReferenceLibraryItem" + WHERE "libraryType" = $1 + AND "isActive" IS TRUE + AND NULLIF(btrim("name"), '') IS NOT NULL + `, [SPU_RULE_LIBRARY_TYPE]) + + return uniqueValues(result.rows.map((row) => row.name)) +} + +async function fetchExistingColumns(connection, table, columns) { + const [rows] = await connection.query( + `SELECT column_name AS columnName + FROM information_schema.columns + WHERE table_schema = DATABASE() + AND table_name = ?`, + [table], + ) + const columnSet = new Set(rows.map((row) => readText(row.columnName))) + return columns.filter((column) => columnSet.has(column)) +} + +async function fetchRowsByValues(connection, table, fields, keyField, values) { + if (fields.length === 0 || values.length === 0) return [] + const rows = [] + for (const part of chunk(values, 300)) { + const placeholders = part.map(() => '?').join(', ') + const [batch] = await connection.execute( + `SELECT ${fields.map((field) => `\`${field}\``).join(', ')} + FROM \`${table}\` + WHERE \`${keyField}\` IN (${placeholders})`, + part, + ) + rows.push(...batch) + } + return rows +} + +function createProductLookup(products) { + const byId = new Map() + const bySourceKey = new Map() + const byName = new Map() + for (const product of products) { + if (product.productId) byId.set(product.productId, product) + for (const key of uniqueKeys([product.sourceSpu, product.spu])) { + bySourceKey.set(key, product) + } + if (product.productName) byName.set(product.productName, product) + } + return { byId, bySourceKey, byName } +} + +function createSkuLookup(products, skuRows, jstRows = []) { + const lookup = createProductLookup(products) + const bySku = new Map() + for (const sku of skuRows) { + const product = lookup.byId.get(sku.productId) + if (!product) continue + for (const key of uniqueKeys([sku.sourceSkuId, sku.barcode, sku.skuCode])) { + bySku.set(key, product) + } + } + for (const row of jstRows) { + const product = lookup.bySourceKey.get(normalizeKey(row.__source_spu)) + if (!product) continue + const skuCodes = readText(row.sku_codes) + .split(/[,，\s]+/) + .map(normalizeKey) + for (const key of uniqueKeys([row.sku_id, row.sku_code, ...skuCodes])) { + bySku.set(key, product) + } + } + return bySku +} + +async function fetchJstSkuRows(connection, sourceKeys) { + const availableColumns = await fetchExistingColumns(connection, 'jst_sku', [ + 'sku_id', + 'i_id', + 'name', + 'category', + 'sku_code', + 'sku_codes', + ]) + if (!availableColumns.includes('sku_id')) { + throw new Error('jst_sku 缺少 sku_id 字段，无法反查聚水潭来源。') + } + + const sourceExpression = availableColumns.includes('i_id') + ? 'COALESCE(NULLIF(TRIM(CAST(`i_id` AS CHAR)), \'\'), TRIM(CAST(`sku_id` AS CHAR)))' + : 'TRIM(CAST(`sku_id` AS CHAR))' + const rows = [] + for (const part of chunk(sourceKeys, 300)) { + if (part.length === 0) continue + const placeholders = part.map(() => '?').join(', ') + const [batch] = await connection.execute( + `SELECT ${availableColumns.map((column) => `\`${column}\``).join(', ')}, + ${sourceExpression} AS __source_spu + FROM \`jst_sku\` + WHERE ${sourceExpression} IN (${placeholders})`, + part, + ) + rows.push(...batch) + } + return rows +} + +function resolveProductForProbeRow({ mode, row, keyField, productLookup, skuLookup }) { + if (mode === 'sku') return skuLookup.get(normalizeKey(row[keyField])) || null + if (mode === 'name') return productLookup.byName.get(readText(row[keyField])) || null + return productLookup.bySourceKey.get(normalizeKey(row[keyField])) || null +} + +function readProbeRowName(row, nameFields) { + for (const field of nameFields) { + const value = readText(row[field]) + if (value) return value + } + return '' +} + +function addEvidence(evidenceByProductId, productId, evidence) { + const rows = evidenceByProductId.get(productId) || [] + rows.push(evidence) + evidenceByProductId.set(productId, rows) +} + +function splitCategoryPath(value) { + return readText(value) + .split(/\s*(?:>|＞|,|，|、| - |－|—|–)\s*/) + .map(readText) + .filter(Boolean) +} + +function resolveAllowedCategories(rawCategory, allowedCategories) { + const category = readText(rawCategory) + if (!category) return [] + if (allowedCategories.has(category)) return [category] + return uniqueValues(splitCategoryPath(category).filter((item) => allowedCategories.has(item))) +} + +function normalizeEvidence(rawEvidence, allowedCategories) { + return { + ...rawEvidence, + rawCategory: readText(rawEvidence.rawCategory), + resolvedCategories: resolveAllowedCategories(rawEvidence.rawCategory, allowedCategories), + } +} + +function summarizeSourceStats(sourceStats, evidenceByProductId) { + return sourceStats.map((stat) => { + const uniqueResolvedProducts = new Set() + const ambiguousResolvedProducts = new Set() + for (const [productId, evidences] of evidenceByProductId.entries()) { + const resolvedCategories = uniqueValues( + evidences + .filter((evidence) => evidence.source === stat.label) + .flatMap((evidence) => evidence.resolvedCategories || []), + ) + if (resolvedCategories.length === 1) uniqueResolvedProducts.add(productId) + if (resolvedCategories.length > 1) ambiguousResolvedProducts.add(productId) + } + return { + ...stat, + uniqueResolvedProducts: uniqueResolvedProducts.size, + ambiguousResolvedProducts: ambiguousResolvedProducts.size, + } + }) +} + +async function collectSourceEvidence({ + connection, + products, + skuRows, + allowedCategories, + probes = SOURCE_PROBES, +}) { + const productLookup = createProductLookup(products) + const sourceKeys = uniqueKeys(products.flatMap((product) => [product.sourceSpu, product.spu])) + const jstRows = await fetchJstSkuRows(connection, sourceKeys) + const skuLookup = createSkuLookup(products, skuRows, jstRows) + const structuredSkuCodes = uniqueKeys( + skuRows.flatMap((sku) => [sku.sourceSkuId, sku.barcode, sku.skuCode]), + ) + const allSkuCodes = uniqueKeys([...structuredSkuCodes, ...Array.from(skuLookup.keys())]) + const productNames = uniqueValues(products.map((product) => product.productName)) + const evidenceByProductId = new Map() + const sourceStats = [] + + for (const probe of probes) { + const keyFields = await fetchExistingColumns(connection, probe.table, probe.keyFields) + const categoryFields = await fetchExistingColumns( + connection, + probe.table, + probe.categoryFields, + ) + const nameFields = await fetchExistingColumns(connection, probe.table, probe.nameFields) + let rawRows = 0 + const productsWithEvidence = new Set() + const productsWithResolved = new Set() + + if (keyFields.length > 0 && categoryFields.length > 0) { + const values = + probe.mode === 'sku' ? allSkuCodes : probe.mode === 'name' ? productNames : sourceKeys + for (const keyField of keyFields) { + const fields = uniqueValues([keyField, ...categoryFields, ...nameFields]) + const rows = await fetchRowsByValues(connection, probe.table, fields, keyField, values) + rawRows += rows.length + for (const row of rows) { + const product = resolveProductForProbeRow({ + mode: probe.mode, + row, + keyField, + productLookup, + skuLookup, + }) + if (!product) continue + + for (const categoryField of categoryFields) { + const rawCategory = readText(row[categoryField]) + if (!rawCategory) continue + const evidence = normalizeEvidence( + { + source: probe.label, + table: probe.table, + mode: probe.mode, + keyField, + key: readText(row[keyField]), + categoryField, + rawCategory, + matchedName: readProbeRowName(row, nameFields), + }, + allowedCategories, + ) + productsWithEvidence.add(product.productId) + if (evidence.resolvedCategories.length > 0) { + productsWithResolved.add(product.productId) + } + addEvidence(evidenceByProductId, product.productId, evidence) + } + } + } + } + + sourceStats.push({ + label: probe.label, + table: probe.table, + mode: probe.mode, + keyFields, + categoryFields, + rawRows, + productsWithEvidence: productsWithEvidence.size, + productsWithResolved: productsWithResolved.size, + }) + } + + return { + evidenceByProductId, + sourceStats: summarizeSourceStats(sourceStats, evidenceByProductId), + jstSummary: { + jstSkuRows: jstRows.length, + jstCategoryNonEmpty: jstRows.filter((row) => readText(row.category)).length, + jstDerivedSkuCodes: skuLookup.size, + }, + } +} + +function summarizeEvidenceForPlan(evidences, category) { + return evidences + .filter((evidence) => (evidence.resolvedCategories || []).includes(category)) + .slice(0, 8) + .map((evidence) => ({ + source: evidence.source, + table: evidence.table, + mode: evidence.mode, + keyField: evidence.keyField, + key: evidence.key, + categoryField: evidence.categoryField, + rawCategory: evidence.rawCategory, + matchedName: evidence.matchedName, + })) +} + +function buildProductProfilePlanEntry(match) { + return { + productId: match.productId, + spu: match.spu, + sourceSpu: match.sourceSpu, + productName: match.productName, + current: { + productName: match.productName, + productCategory: match.currentProductCategory, + productDevelopmentKind: match.productDevelopmentKind, + missingFields: ['productCategory'], + }, + fill: { + productCategory: match.productCategory, + }, + evidence: { + source: 'jst-product-category-match', + matchedCategory: match.productCategory, + sourceCount: match.sources.length, + sources: match.sources, + rows: match.sampleEvidence, + }, + note: '只读来源库唯一命中正式产品分类字典；冲突和无证据记录不会进入本计划。', + } +} + +function buildMatchArtifacts({ + products, + evidenceByProductId, + allowedCategories, + confirmedBy = DEFAULT_CONFIRMED_BY, + sourceStats = [], + jstSummary = {}, + generatedAt = new Date().toISOString(), +}) { + const uniqueMatches = [] + const conflictMatches = [] + const unresolvedProducts = [] + const categoryCounts = new Map() + + for (const product of products) { + const evidences = evidenceByProductId.get(product.productId) || [] + const resolvedCategories = uniqueValues( + evidences.flatMap((evidence) => evidence.resolvedCategories || []), + ) + resolvedCategories.forEach((category) => { + categoryCounts.set(category, (categoryCounts.get(category) || 0) + 1) + }) + + if (resolvedCategories.length === 1) { + const productCategory = resolvedCategories[0] + const sources = uniqueValues( + evidences + .filter((evidence) => (evidence.resolvedCategories || []).includes(productCategory)) + .map((evidence) => evidence.source), + ) + uniqueMatches.push({ + productId: product.productId, + spu: product.spu, + sourceSpu: product.sourceSpu, + productName: product.productName, + currentProductCategory: product.productCategory, + productDevelopmentKind: product.productDevelopmentKind, + productCategory, + evidenceCount: evidences.length, + sources, + sampleEvidence: summarizeEvidenceForPlan(evidences, productCategory), + }) + continue + } + + if (resolvedCategories.length > 1) { + conflictMatches.push({ + productId: product.productId, + spu: product.spu, + sourceSpu: product.sourceSpu, + productName: product.productName, + categories: resolvedCategories, + evidence: evidences.slice(0, 12), + }) + continue + } + + unresolvedProducts.push({ + productId: product.productId, + spu: product.spu, + sourceSpu: product.sourceSpu, + productName: product.productName, + evidenceCount: evidences.length, + evidence: evidences.slice(0, 8), + }) + } + + const cleanupPlan = { + mode: 'jst-product-category-match-cleanup-plan', + confirmation: APPLY_CONFIRMATION, + confirmedBy: confirmedBy.trim(), + skuItems: [], + productProfiles: uniqueMatches.map(buildProductProfilePlanEntry), + qualityDecisionRepairs: [], + preSkuFormalizationMismatches: [], + } + const validation = cleanupPlan.productProfiles.length > 0 + ? validateCleanupPlanForApply(cleanupPlan) + : null + const categoryHistogram = Array.from(categoryCounts.entries()) + .sort((left, right) => right[1] - left[1]) + .map(([category, count]) => ({ category, count })) + const summary = { + mode: 'jst-product-category-match', + generatedAt, + missingProducts: products.length, + allowedCategoryCount: allowedCategories.size, + totalProductsWithAnyEvidence: Array.from(evidenceByProductId.keys()).length, + totalProductsWithResolved: uniqueMatches.length + conflictMatches.length, + uniqueResolvedProducts: uniqueMatches.length, + ambiguousResolvedProducts: conflictMatches.length, + unresolvedProducts: unresolvedProducts.length, + cleanupPlanRows: cleanupPlan.productProfiles.length, + cleanupPlanAppliedFields: validation?.productProfiles?.appliedFields ?? 0, + categoryHistogram, + ...jstSummary, + } + + return { + summary, + sourceStats, + uniqueMatches, + conflictMatches, + unresolvedProducts, + cleanupPlan, + validation, + } +} + +async function run(options) { + const databaseUrl = loadDatabaseUrl() + if (!databaseUrl) throw new Error('缺少 DATABASE_URL。') + + const pgPool = new Pool({ connectionString: databaseUrl }) + const mysqlConnection = await mysql.createConnection(createMysqlConnectionConfig()) + try { + const pgClient = await pgPool.connect() + let products + let skuRows + let allowedCategories + try { + products = await fetchMissingProductCategoryProducts(pgClient, options) + skuRows = await fetchMissingProductCategorySkuRows( + pgClient, + products.map((product) => product.productId), + ) + allowedCategories = new Set(await fetchAllowedProductCategories(pgClient)) + } finally { + pgClient.release() + } + + const evidence = await collectSourceEvidence({ + connection: mysqlConnection, + products, + skuRows, + allowedCategories, + }) + const artifacts = buildMatchArtifacts({ + products, + evidenceByProductId: evidence.evidenceByProductId, + allowedCategories, + confirmedBy: options.confirmedBy, + sourceStats: evidence.sourceStats, + jstSummary: evidence.jstSummary, + }) + + const outputDir = options.outputDir + const summaryFile = writeJsonReport(path.join(outputDir, 'summary.json'), artifacts.summary) + const reportFile = writeJsonReport(path.join(outputDir, 'report.json'), { + summary: artifacts.summary, + sourceStats: artifacts.sourceStats, + uniqueMatches: artifacts.uniqueMatches, + conflictMatches: artifacts.conflictMatches, + unresolvedProducts: artifacts.unresolvedProducts, + }) + const cleanupPlanFile = writeJsonReport( + path.join(outputDir, 'cleanup-plan.json'), + artifacts.cleanupPlan, + ) + + return { + ...artifacts.summary, + summaryFile, + reportFile, + cleanupPlanFile, + } + } finally { + await mysqlConnection.end() + await pgPool.end() + } +} + +async function main() { + const options = parseArgs(process.argv.slice(2)) + const result = await run(options) + if (options.summaryStdout) { + console.log(JSON.stringify(result, null, 2)) + } else { + console.log(`已导出聚水潭来源分类匹配报告：${result.reportFile}`) + } +} + +if (require.main === module) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)) + process.exit(1) + }) +} + +module.exports = { + DEFAULT_CONFIRMED_BY, + DEFAULT_OUTPUT_DIR, + SOURCE_PROBES, + parseArgs, + readText, + normalizeKey, + splitCategoryPath, + resolveAllowedCategories, + normalizeEvidence, + buildMatchArtifacts, + buildProductProfilePlanEntry, + createProductLookup, + createSkuLookup, +} diff --git a/furniture-3d-system/scripts/verify-master-data-code-boundaries.js b/furniture-3d-system/scripts/verify-master-data-code-boundaries.js index 8abdc25b..d0525e4f 100644 --- a/furniture-3d-system/scripts/verify-master-data-code-boundaries.js +++ b/furniture-3d-system/scripts/verify-master-data-code-boundaries.js @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-require-imports */ const fs = require('node:fs') const path = require('node:path') @@ -1157,6 +1158,10 @@ const approvedRawSqlMasterDataReadPaths = new Map([ 'scripts/export-non-integrated-product-category-review.js', '非集成产品分类审阅导出脚本只允许只读读取产品/profile/SKU 样例，生成业务确认清单和 cleanup plan 模板；不得直接写库或猜测分类。', ], + [ + 'scripts/match-jst-product-category-cleanup-plan.js', + '聚水潭来源分类匹配脚本只允许只读读取非集成产品/profile/SKU 和正式分类字典，结合 JST 来源库生成可审计 cleanup plan；不得直接写库或关键词猜分类。', + ], [ 'scripts/apply-pre-sku-formalization-repair.js', '准 SKU 生命周期专项修复入口允许读取 PreSku、ProductSKU、DraftGroup、包装和库位关系，用于 currentEvidence 校验、dry-run 和 apply 前事务内复核。', @@ -1217,6 +1222,17 @@ const approvedRawSqlMasterDataReadTableScopes = new Map([ 'IntakeNonIntegratedSkuItem', ]), ], + [ + 'scripts/match-jst-product-category-cleanup-plan.js', + new Set([ + 'IntakeNonIntegratedProduct', + 'IntakeNonIntegratedProductProfile', + 'IntakeNonIntegratedSkuItem', + 'IntakeReferenceLibraryItem', + 'ProductCategory', + 'SpuNumberCounter', + ]), + ], [ 'scripts/apply-pre-sku-formalization-repair.js', new Set([ diff --git a/furniture-3d-system/tests/scripts/match-jst-product-category-cleanup-plan.test.js b/furniture-3d-system/tests/scripts/match-jst-product-category-cleanup-plan.test.js new file mode 100644 index 00000000..1dfad5f1 --- /dev/null +++ b/furniture-3d-system/tests/scripts/match-jst-product-category-cleanup-plan.test.js @@ -0,0 +1,162 @@ +const { + buildMatchArtifacts, + createSkuLookup, + normalizeEvidence, + parseArgs, + resolveAllowedCategories, + splitCategoryPath, +} = require('../../scripts/match-jst-product-category-cleanup-plan') + +function product(overrides = {}) { + return { + productId: 'product-1', + spu: 'SPU001', + sourceSpu: '1001', + productName: '书桌 A', + productCategory: '', + productDevelopmentKind: 'product', + ...overrides, + } +} + +function evidence(overrides = {}) { + return normalizeEvidence( + { + source: '市场Top商品', + table: 'ods_lcf_mkt_top_goods_di', + mode: 'product', + keyField: 'goods_id', + key: '1001', + categoryField: 'three_category', + rawCategory: '住宅家具,桌类,书桌', + matchedName: '书桌 A', + ...overrides, + }, + new Set(['书桌', '电脑桌', '书柜']), + ) +} + +describe('match-jst-product-category-cleanup-plan', () => { + it('parses CLI options', () => { + expect(parseArgs([ + '--output-dir', + '.out', + '--confirmed-by=企划负责人', + '--limit', + '20', + '--summary-stdout', + ])).toEqual({ + outputDir: '.out', + confirmedBy: '企划负责人', + limit: 20, + summaryStdout: true, + }) + expect(() => parseArgs(['--limit=0'])).toThrow('正整数') + }) + + it('resolves allowed category from a source category path', () => { + const allowed = new Set(['书桌', '电脑桌']) + expect(splitCategoryPath('住宅家具,桌类,书桌')).toEqual(['住宅家具', '桌类', '书桌']) + expect(resolveAllowedCategories('住宅家具 > 桌类 > 书桌', allowed)).toEqual(['书桌']) + expect(resolveAllowedCategories('电脑桌', allowed)).toEqual(['电脑桌']) + expect(resolveAllowedCategories('桌类', allowed)).toEqual([]) + }) + + it('builds a validate-ready cleanup plan for unique resolved matches', () => { + const products = [product()] + const evidenceByProductId = new Map([ + ['product-1', [evidence(), evidence({ rawCategory: '书桌', source: '市场洞察商品排行' })]], + ]) + + const artifacts = buildMatchArtifacts({ + products, + evidenceByProductId, + allowedCategories: new Set(['书桌', '电脑桌']), + confirmedBy: '企划负责人', + sourceStats: [], + jstSummary: { jstSkuRows: 1, jstCategoryNonEmpty: 0 }, + generatedAt: '2026-06-05T00:00:00.000Z', + }) + + expect(artifacts.summary).toMatchObject({ + missingProducts: 1, + uniqueResolvedProducts: 1, + ambiguousResolvedProducts: 0, + unresolvedProducts: 0, + cleanupPlanRows: 1, + cleanupPlanAppliedFields: 1, + jstSkuRows: 1, + jstCategoryNonEmpty: 0, + }) + expect(artifacts.cleanupPlan).toMatchObject({ + confirmation: 'I_UNDERSTAND_THIS_UPDATES_MASTER_DATA', + confirmedBy: '企划负责人', + productProfiles: [ + expect.objectContaining({ + productId: 'product-1', + sourceSpu: '1001', + fill: { productCategory: '书桌' }, + current: expect.objectContaining({ + productCategory: '', + productDevelopmentKind: 'product', + missingFields: ['productCategory'], + }), + }), + ], + }) + expect(artifacts.validation.productProfiles).toMatchObject({ + actionableRows: 1, + appliedFields: 1, + remainingMissingFieldCounts: {}, + }) + }) + + it('keeps conflicting source categories out of the cleanup plan', () => { + const products = [product()] + const evidenceByProductId = new Map([ + [ + 'product-1', + [ + evidence({ rawCategory: '书桌' }), + evidence({ rawCategory: '电脑桌', source: 'MAY商品快照' }), + ], + ], + ]) + + const artifacts = buildMatchArtifacts({ + products, + evidenceByProductId, + allowedCategories: new Set(['书桌', '电脑桌']), + confirmedBy: '企划负责人', + }) + + expect(artifacts.summary).toMatchObject({ + uniqueResolvedProducts: 0, + ambiguousResolvedProducts: 1, + unresolvedProducts: 0, + cleanupPlanRows: 0, + }) + expect(artifacts.cleanupPlan.productProfiles).toEqual([]) + expect(artifacts.conflictMatches[0]).toMatchObject({ + productId: 'product-1', + categories: ['书桌', '电脑桌'], + }) + }) + + it('uses jst sku codes to map source rows back to products', () => { + const products = [product({ sourceSpu: 'i-001' })] + const skuRows = [{ productId: 'product-1', sourceSkuId: '', barcode: '', skuCode: '' }] + const skuLookup = createSkuLookup(products, skuRows, [ + { + __source_spu: 'i-001', + sku_id: 'sku-1', + sku_code: 'may-1', + sku_codes: 'alias-1, alias-2', + }, + ]) + + expect(skuLookup.get('SKU-1')?.productId).toBe('product-1') + expect(skuLookup.get('MAY-1')?.productId).toBe('product-1') + expect(skuLookup.get('ALIAS-2')?.productId).toBe('product-1') + }) +})