cangku/backend/src/services/taobaoProductRecordsAppearance.js

const planningDatabase = require('./planningDatabase');

const TAOBAO_PRODUCT_RECORDS_TABLE = 'taobao_product_records';
const APPEARANCE_RECOGNITION_CODE_COLUMN = 'appearance_recognition_code';
const APPEARANCE_RECOGNITION_CODE_LABEL = '外观识别编码';
const APPEARANCE_RECOGNITION_CODE_INDEX = 'idx_taobao_product_records_appearance_code';
const MAY_APPEARANCE_RECOGNITION_CODE_COLUMN = 'may_appearance_recognition_code';
const MAY_SKU_CODE_COLUMN = 'may_sku_code';

const TAOBAO_PRODUCT_RECORDS_STANDARD_COLUMNS = Object.freeze([
  { name: 'planning_date', label: '日期', type: 'VARCHAR(64)' },
  { name: 'category_level1', label: '一级类目', type: 'VARCHAR(255)' },
  { name: 'category_level2', label: '二级类目', type: 'VARCHAR(255)' },
  { name: 'category_level3', label: '三级类目', type: 'VARCHAR(255)' },
  { name: 'market_common_category_name', label: '市场共性分类名', type: 'VARCHAR(255)' },
  { name: APPEARANCE_RECOGNITION_CODE_COLUMN, label: APPEARANCE_RECOGNITION_CODE_LABEL, type: 'VARCHAR(160)' },
  { name: 'item_id', label: '商品ID', type: 'VARCHAR(100)' },
  { name: 'main_image_url', label: '主图', type: 'VARCHAR(1000)' },
  { name: 'title', label: '商品标题', type: 'VARCHAR(1000)' },
  { name: 'analysis_sales', label: '销量', type: 'VARCHAR(120)' },
  { name: 'lowest_list_price', label: '最低标价', type: 'VARCHAR(120)' },
  { name: 'product_material', label: '产品材质', type: 'VARCHAR(500)' },
  { name: 'core_material_name', label: '核心主材质', type: 'VARCHAR(255)' },
  { name: 'standard_product_type', label: '标准产品类型', type: 'VARCHAR(255)' },
  { name: 'core_product_type_name', label: '核心产品类型', type: 'VARCHAR(255)' },
  { name: 'market_category_name', label: '市场分类名', type: 'VARCHAR(500)' },
  { name: 'sku_price_range', label: 'SKU价格区间', type: 'VARCHAR(120)' },
  { name: 'actual_hand_price', label: '实际到手价', type: 'VARCHAR(120)' },
  { name: 'price_band', label: '价格带', type: 'VARCHAR(120)' },
  { name: 'price_band_rank', label: '价格带排名', type: 'VARCHAR(120)' },
  { name: 'shop_type', label: '店铺类型', type: 'VARCHAR(120)' },
  { name: 'shop_name', label: '店铺名称', type: 'VARCHAR(255)' },
  { name: 'search_keyword', label: '搜索关键词', type: 'VARCHAR(500)' },
  { name: 'product_keywords', label: '商品关键词', type: 'TEXT' },
  { name: 'industry_rank', label: '行业排名', type: 'VARCHAR(120)' },
  { name: 'paying_buyers', label: '支付买家数', type: 'VARCHAR(120)' },
  { name: 'visitors', label: '访客数', type: 'VARCHAR(120)' },
  { name: 'trend', label: '趋势', type: 'VARCHAR(120)' },
  { name: 'time_dimension', label: '时间维度', type: 'VARCHAR(120)' },
  { name: 'sales_amount', label: '销售额', type: 'VARCHAR(120)' },
  { name: 'browse_users', label: '浏览人数', type: 'VARCHAR(120)' },
  { name: 'search_users', label: '搜索人数', type: 'VARCHAR(120)' },
  { name: 'favorite_users', label: '收藏人数', type: 'VARCHAR(120)' },
  { name: 'cart_users', label: '加购人数', type: 'VARCHAR(120)' },
  { name: 'purchase_users', label: '购买人数', type: 'VARCHAR(120)' },
  { name: 'conversion_rate', label: '转化率', type: 'VARCHAR(120)' },
  { name: 'search_user_ratio', label: '搜索人数占比', type: 'VARCHAR(120)' },
  { name: 'cart_rate', label: '加购率', type: 'VARCHAR(120)' },
  { name: MAY_APPEARANCE_RECOGNITION_CODE_COLUMN, label: 'MAY-外观识别编码', type: 'VARCHAR(160)' },
  { name: 'may_sku_image_url', label: 'MAY-SKU图片', type: 'VARCHAR(1000)' },
  { name: MAY_SKU_CODE_COLUMN, label: 'MAY-SKU编码', type: 'VARCHAR(100)' },
  { name: 'may_sales_status', label: 'MAY-销售状态', type: 'VARCHAR(120)' },
  { name: 'may_core_material', label: 'MAY-核心材质', type: 'VARCHAR(255)' },
  { name: 'may_min_limit_price', label: 'MAY-最低限价', type: 'VARCHAR(120)' },
  { name: 'may_size', label: 'MAY-尺寸', type: 'VARCHAR(255)' },
  { name: 'may_color_name', label: 'MAY-颜色名称', type: 'VARCHAR(255)' },
  { name: 'may_performance_amount', label: 'MAY-30天销售额', type: 'VARCHAR(120)' },
  { name: 'may_goods_id_count', label: 'MAY-ID数量', type: 'VARCHAR(120)' },
  { name: 'may_covered_channels', label: 'MAY-ID覆盖的渠道', type: 'VARCHAR(500)' },
  { name: 'may_1688_sales_ratio', label: 'MAY-1688销售占比', type: 'VARCHAR(120)' },
  { name: 'may_taobao_sales_ratio', label: 'MAY-淘宝销售占比', type: 'VARCHAR(120)' },
  { name: 'may_tmall_sales_ratio', label: 'MAY-天猫销售占比', type: 'VARCHAR(120)' },
  { name: 'may_pdd_sales_ratio', label: 'MAY-拼多多销售占比', type: 'VARCHAR(120)' },
  { name: 'may_jd_sales_ratio', label: 'MAY-京东销售占比', type: 'VARCHAR(120)' },
  { name: 'may_douyin_sales_ratio', label: 'MAY-抖音销售占比', type: 'VARCHAR(120)' },
  { name: 'may_vip_sales_ratio', label: 'MAY-唯品会销售占比', type: 'VARCHAR(120)' },
  { name: 'may_other_channel_sales_ratio', label: 'MAY-其他渠道销售占比', type: 'VARCHAR(120)' },
  { name: 'may_promotion_fee_30d', label: 'MAY-近30天推广费用', type: 'VARCHAR(120)' },
  { name: 'may_d_style_efficiency', label: 'MAY-D款效', type: 'VARCHAR(120)' },
  { name: 'may_inventory_quantity', label: 'MAY-库存', type: 'VARCHAR(120)' },
  { name: 'may_real_company_inventory_quantity', label: 'MAY-真实公司库存', type: 'VARCHAR(120)' },
  { name: 'may_estimated_sales_days', label: 'MAY-预计销售时长', type: 'VARCHAR(120)' },
  { name: 'may_shipping_warehouse', label: 'MAY-发货仓库', type: 'VARCHAR(500)' },
  { name: 'may_inventory_scope_remark', label: 'MAY-库存备注', type: 'VARCHAR(255)' },
  { name: 'may_avg_daily_sales_30d', label: 'MAY-30天平均日销', type: 'VARCHAR(120)' },
  { name: 'may_avg_daily_sales_7d', label: 'MAY-7天日均销量', type: 'VARCHAR(120)' },
  { name: 'may_sales_growth_rate_15d', label: 'MAY-15天环比增长率', type: 'VARCHAR(120)' },
  { name: 'may_listing_duration_days', label: 'MAY-铺货时长（店铺铺货时间）', type: 'VARCHAR(120)' },
  { name: 'may_gross_margin_rate', label: 'MAY-销售毛利率', type: 'VARCHAR(120)' },
  { name: 'may_after_sales_ratio', label: 'MAY-售后占比', type: 'VARCHAR(120)' },
  { name: 'may_active_sales_day_ratio_30d', label: 'MAY-动销率（近 30 天有销量的天数占比）', type: 'VARCHAR(120)' },
]);

const TAOBAO_PRODUCT_RECORDS_STANDARD_INDEXES = Object.freeze([
  {
    name: APPEARANCE_RECOGNITION_CODE_INDEX,
    columns: [APPEARANCE_RECOGNITION_CODE_COLUMN],
  },
  {
    name: 'idx_taobao_product_records_may_appearance_code',
    columns: [MAY_APPEARANCE_RECOGNITION_CODE_COLUMN],
  },
  {
    name: 'idx_taobao_product_records_may_sku_code',
    columns: [MAY_SKU_CODE_COLUMN],
  },
]);

const LEGACY_APPEARANCE_CODE_COLUMNS = Object.freeze([
  'similar_group_code',
  'appearance_group_key',
  'appearance_code',
]);

const JSON_APPEARANCE_CODE_COLUMNS = Object.freeze([
  'analysis_json',
  'normalized_item_json',
  'raw_data_json',
  'raw_response_json',
]);

const APPEARANCE_CODE_KEYS = Object.freeze([
  'appearance_recognition_code',
  'appearanceRecognitionCode',
  'appearance_group_key',
  'appearanceGroupKey',
  'appearance_code',
  'appearanceCode',
  '外观识别编码',
  '外观相似款组键',
  '外观编码',
]);

const APPEARANCE_CODE_TEXT_MARKERS = Object.freeze([
  'appearance_recognition_code',
  'appearanceRecognitionCode',
  'appearance_group_key',
  'appearanceGroupKey',
  'appearance_code',
  'appearanceCode',
  '外观识别编码',
  '外观相似款组键',
  '外观编码',
]);

let ensureState = {
  ready: false,
  pending: null,
};

function escapeIdentifier(identifier) {
  return `\`${String(identifier || '').replace(/`/g, '``')}\``;
}

function escapeSqlComment(value) {
  return String(value || '').replace(/\\/g, '\\\\').replace(/'/g, "\\'");
}

function buildColumnDefinition(column) {
  return `${escapeIdentifier(column.name)} ${column.type} NULL COMMENT '${escapeSqlComment(column.label)}'`;
}

function normalizeText(value) {
  if (value === null || value === undefined) {
    return '';
  }
  return String(value).replace(/\u00a0/g, ' ').trim();
}

function firstMeaningfulText(...values) {
  for (const value of values) {
    const normalized = normalizeText(value);
    if (
      normalized
      && !['—', '-', '--', '未配置', '暂无', '无', 'null', 'NULL', 'undefined'].includes(normalized)
    ) {
      return normalized;
    }
  }
  return '';
}

function safeJsonParse(value) {
  if (!value) {
    return null;
  }
  if (typeof value === 'object') {
    return value;
  }
  try {
    return JSON.parse(String(value));
  } catch (_error) {
    return null;
  }
}

function findAppearanceCodeInJson(value, depth = 0, visited = new Set()) {
  if (!value || depth > 6) {
    return '';
  }

  if (typeof value !== 'object') {
    return '';
  }

  if (visited.has(value)) {
    return '';
  }
  visited.add(value);

  if (Array.isArray(value)) {
    for (const item of value.slice(0, 200)) {
      const matched = findAppearanceCodeInJson(item, depth + 1, visited);
      if (matched) {
        return matched;
      }
    }
    return '';
  }

  for (const key of APPEARANCE_CODE_KEYS) {
    const matched = firstMeaningfulText(value[key]);
    if (matched) {
      return matched;
    }
  }

  for (const item of Object.values(value)) {
    const matched = findAppearanceCodeInJson(item, depth + 1, visited);
    if (matched) {
      return matched;
    }
  }

  return '';
}

function resolveAppearanceCodeFromRow(row = {}, columnNames = new Set()) {
  const directCandidates = LEGACY_APPEARANCE_CODE_COLUMNS
    .filter((columnName) => columnNames.has(columnName))
    .map((columnName) => row[columnName]);
  const directCode = firstMeaningfulText(...directCandidates);
  if (directCode) {
    return directCode;
  }

  for (const columnName of JSON_APPEARANCE_CODE_COLUMNS) {
    if (!columnNames.has(columnName)) {
      continue;
    }
    const parsed = safeJsonParse(row[columnName]);
    const matched = findAppearanceCodeInJson(parsed);
    if (matched) {
      return matched;
    }
  }

  return '';
}

async function getColumnNames(tableName = TAOBAO_PRODUCT_RECORDS_TABLE) {
  const rows = await planningDatabase.query(
    `SELECT column_name AS columnName
     FROM information_schema.columns
     WHERE table_schema = DATABASE()
       AND table_name = ?
     ORDER BY ordinal_position ASC`,
    [tableName]
  );

  return rows.map((row) => row.columnName);
}

async function tableExists(tableName = TAOBAO_PRODUCT_RECORDS_TABLE) {
  const rows = await planningDatabase.query(
    `SELECT 1 AS ok
     FROM information_schema.tables
     WHERE table_schema = DATABASE()
       AND table_name = ?
     LIMIT 1`,
    [tableName]
  );
  return rows.length > 0;
}

async function indexExists(tableName, indexName) {
  const rows = await planningDatabase.query(
    `SELECT 1 AS ok
     FROM information_schema.statistics
     WHERE table_schema = DATABASE()
       AND table_name = ?
       AND index_name = ?
     LIMIT 1`,
    [tableName, indexName]
  );
  return rows.length > 0;
}

async function ensureAppearanceColumn(options = {}) {
  const includeIndexes = options.includeIndexes !== false;
  if (!await tableExists(TAOBAO_PRODUCT_RECORDS_TABLE)) {
    return {
      tableExists: false,
      columnAdded: false,
      indexAdded: false,
      addedColumns: [],
      addedIndexes: [],
    };
  }

  const columnNames = new Set(await getColumnNames(TAOBAO_PRODUCT_RECORDS_TABLE));
  const addedColumns = [];
  const addedIndexes = [];

  for (const column of TAOBAO_PRODUCT_RECORDS_STANDARD_COLUMNS) {
    if (columnNames.has(column.name)) {
      continue;
    }
    await planningDatabase.execute(
      `ALTER TABLE ${escapeIdentifier(TAOBAO_PRODUCT_RECORDS_TABLE)}
       ADD COLUMN ${buildColumnDefinition(column)}`
    );
    columnNames.add(column.name);
    addedColumns.push(column.name);
  }

  if (includeIndexes) {
    for (const indexDefinition of TAOBAO_PRODUCT_RECORDS_STANDARD_INDEXES) {
      if (indexDefinition.columns.some((columnName) => !columnNames.has(columnName))) {
        continue;
      }
      if (await indexExists(TAOBAO_PRODUCT_RECORDS_TABLE, indexDefinition.name)) {
        continue;
      }
      await planningDatabase.execute(
        `ALTER TABLE ${escapeIdentifier(TAOBAO_PRODUCT_RECORDS_TABLE)}
         ADD KEY ${escapeIdentifier(indexDefinition.name)} (${indexDefinition.columns.map((columnName) => escapeIdentifier(columnName)).join(', ')})`
      );
      addedIndexes.push(indexDefinition.name);
    }
  }

  return {
    tableExists: true,
    columnAdded: addedColumns.length > 0,
    indexAdded: addedIndexes.length > 0,
    addedColumns,
    addedIndexes,
  };
}

async function backfillAppearanceCodes() {
  const columnNames = new Set(await getColumnNames(TAOBAO_PRODUCT_RECORDS_TABLE));
  if (!columnNames.has(APPEARANCE_RECOGNITION_CODE_COLUMN)) {
    return 0;
  }

  const directSourceColumns = LEGACY_APPEARANCE_CODE_COLUMNS
    .filter((columnName) => columnNames.has(columnName));
  let updatedCount = 0;

  if (directSourceColumns.length > 0) {
    const directExpression = directSourceColumns
      .map((columnName) => `NULLIF(TRIM(${escapeIdentifier(columnName)}), '')`)
      .join(', ');
    const directResult = await planningDatabase.execute(
      `UPDATE ${escapeIdentifier(TAOBAO_PRODUCT_RECORDS_TABLE)}
       SET ${escapeIdentifier(APPEARANCE_RECOGNITION_CODE_COLUMN)} = COALESCE(${directExpression})
       WHERE NULLIF(TRIM(${escapeIdentifier(APPEARANCE_RECOGNITION_CODE_COLUMN)}), '') IS NULL
         AND COALESCE(${directExpression}) IS NOT NULL`
    );
    updatedCount += Number(directResult?.affectedRows || 0);
  }

  if (directSourceColumns.includes('similar_group_code')) {
    return updatedCount;
  }

  const selectColumns = [
    'id',
    ...LEGACY_APPEARANCE_CODE_COLUMNS,
    ...JSON_APPEARANCE_CODE_COLUMNS,
  ].filter((columnName, index, list) => columnNames.has(columnName) && list.indexOf(columnName) === index);

  if (selectColumns.length <= 1) {
    return 0;
  }

  const candidateConditions = [];
  const candidateParams = [];

  LEGACY_APPEARANCE_CODE_COLUMNS.forEach((columnName) => {
    if (columnNames.has(columnName)) {
      candidateConditions.push(`NULLIF(TRIM(${escapeIdentifier(columnName)}), '') IS NOT NULL`);
    }
  });

  JSON_APPEARANCE_CODE_COLUMNS.forEach((columnName) => {
    if (!columnNames.has(columnName)) {
      return;
    }
    APPEARANCE_CODE_TEXT_MARKERS.forEach((marker) => {
      candidateConditions.push(`${escapeIdentifier(columnName)} LIKE ?`);
      candidateParams.push(`%${marker}%`);
    });
  });

  if (candidateConditions.length === 0) {
    return 0;
  }

  let lastId = 0;
  const batchSize = 500;

  while (true) {
    const rows = await planningDatabase.query(
      `SELECT ${selectColumns.map((columnName) => escapeIdentifier(columnName)).join(', ')}
       FROM ${escapeIdentifier(TAOBAO_PRODUCT_RECORDS_TABLE)}
       WHERE id > ?
         AND NULLIF(TRIM(${escapeIdentifier(APPEARANCE_RECOGNITION_CODE_COLUMN)}), '') IS NULL
         AND (${candidateConditions.join(' OR ')})
       ORDER BY id ASC
       LIMIT ?`,
      [lastId, ...candidateParams, batchSize]
    );

    if (rows.length === 0) {
      break;
    }

    const updates = rows
      .map((row) => ({
        id: Number(row.id),
        code: resolveAppearanceCodeFromRow(row, columnNames),
      }))
      .filter((item) => Number.isFinite(item.id) && item.code);

    for (const updateChunk of chunkArray(updates, 100)) {
      const caseFragments = [];
      const params = [];
      const ids = [];

      updateChunk.forEach((item) => {
        caseFragments.push('WHEN ? THEN ?');
        params.push(item.id, item.code);
        ids.push(item.id);
      });

      if (ids.length === 0) {
        continue;
      }

      await planningDatabase.execute(
        `UPDATE ${escapeIdentifier(TAOBAO_PRODUCT_RECORDS_TABLE)}
         SET ${escapeIdentifier(APPEARANCE_RECOGNITION_CODE_COLUMN)} = CASE id ${caseFragments.join(' ')} ELSE ${escapeIdentifier(APPEARANCE_RECOGNITION_CODE_COLUMN)} END
         WHERE id IN (${ids.map(() => '?').join(', ')})`,
        [...params, ...ids]
      );
      updatedCount += ids.length;
    }

    lastId = Number(rows[rows.length - 1]?.id || lastId);
    if (rows.length < batchSize) {
      break;
    }
  }

  return updatedCount;
}

function chunkArray(list, size) {
  const safeList = Array.isArray(list) ? list : [];
  const chunkSize = Math.max(Number(size) || 1, 1);
  const chunks = [];
  for (let index = 0; index < safeList.length; index += chunkSize) {
    chunks.push(safeList.slice(index, index + chunkSize));
  }
  return chunks;
}

async function ensureTaobaoProductRecordsAppearanceCodeColumn(options = {}) {
  const forceRefresh = options.forceRefresh === true;
  if (!forceRefresh && ensureState.ready) {
    return {
      tableExists: true,
      columnAdded: false,
      indexAdded: false,
      backfilledCount: 0,
      cached: true,
    };
  }
  if (!forceRefresh && ensureState.pending) {
    return ensureState.pending;
  }

  ensureState.pending = (async () => {
    const columnResult = await ensureAppearanceColumn(options);
    const shouldBackfillExistingCodes = options.backfillExistingCodes === true;
    const backfilledCount = columnResult.tableExists && shouldBackfillExistingCodes
      ? await backfillAppearanceCodes()
      : 0;
    ensureState.ready = columnResult.tableExists;
    return {
      ...columnResult,
      backfilledCount,
      cached: false,
    };
  })();

  try {
    return await ensureState.pending;
  } finally {
    ensureState.pending = null;
  }
}

module.exports = {
  TAOBAO_PRODUCT_RECORDS_TABLE,
  TAOBAO_PRODUCT_RECORDS_STANDARD_COLUMNS,
  APPEARANCE_RECOGNITION_CODE_COLUMN,
  APPEARANCE_RECOGNITION_CODE_LABEL,
  MAY_APPEARANCE_RECOGNITION_CODE_COLUMN,
  MAY_SKU_CODE_COLUMN,
  ensureTaobaoProductRecordsAppearanceCodeColumn,
};