WuYongJie
6b2b486ce1
## 任务占用与边界 - 任务名称：全项目上游业务镜像目录与通用同步框架 - 实际开发者：WuYongJie - 目标结果：登记当前代码实际使用的上游原表，提供低压、可断点的首次全量、增量重叠和全量校准框架。 - 实际修改：13 个文件；新增通用数据集目录、动态镜像建表、同步状态、同步命令、生产低压轮询、测试和团队清单。 - 将写入的公共契约：`business_mirror_table_states`、`business-mirror:sync-tables`、`BusinessMirrorDatasetRegistry`、`BusinessMirrorSchemaManager`。 - 仅复用、明确不修改：`UpstreamReadOnlyQueryFactory`、`upstream_schema_columns`、既有 `product_price_mirror_rows`。 - 明确不包含：本 PR 不切换其余业务页面 Repository，不执行生产迁移或首次全量，不提交真实业务数据或凭据。 ## 合并前自审 - [x] 已重新拉取最新 `origin/main`，分支仅领先 1 个提交 - [x] `git merge-tree --write-tree origin/main HEAD` 无冲突 - [x] 完整 diff 共 13 个文件，未发现越界文件或敏感内容 - [x] 生产共享数据字典核对 38 个数据集，缺失对象/字段为 0 - [x] 临时 MariaDB 空库迁移实测成功，验证库已删除 - [x] 上游异常空结果不会清空已有镜像 - [x] 定时任务只轮转已完成首次全量的数据集，不会自动扫描未初始化大表 ## 测试 - `BusinessMirrorSynchronizerTest` 等相关测试：14 tests / 65 assertions 全通过 - 新同步器专项：5 tests / 25 assertions 全通过 - 新增 PHP 文件语法检查全部通过 - 两份生产激活脚本 `bash -n` 通过 - `git diff --check` 通过 ## 风险与回滚 - 包含新增迁移，但发布脚本遇到迁移仍会要求独立高风险授权；合并本身不会执行生产迁移。 - 首次全量必须逐数据集显式执行并做数量/翻页/筛选验收，特别是约 280 万行的大表不得随发布自动全扫。 - 回滚代码可回退本合并提交；生产迁移执行前必须备份 `caiwu_system_config` 与 `caiwu_business_mirror`。 Reviewed-on: http://git.1080p.hk/WuYongJie/CaiWuXiTong/pulls/53
86 lines
10 KiB
PHP
86 lines
10 KiB
PHP
<?php
|
|
|
|
return [
|
|
'connection' => env('BUSINESS_MIRROR_DB_CONNECTION', 'business_mirror'),
|
|
|
|
// 只有生产刷新节点写镜像；开发环境和网页请求只读共享结果。
|
|
'refresh_enabled' => env(
|
|
'BUSINESS_MIRROR_REFRESH_ENABLED',
|
|
env('APP_ENV', 'production') === 'production'
|
|
),
|
|
|
|
'product_prices' => [
|
|
'dataset_key' => 'supplier-payables.product-prices',
|
|
'source_key' => 'operations_replica',
|
|
'source_object' => 'rpt.Bulk_Price_Limit',
|
|
'batch_size' => (int) env('PRODUCT_PRICE_MIRROR_BATCH_SIZE', 500),
|
|
'max_batches_per_run' => (int) env('PRODUCT_PRICE_MIRROR_MAX_BATCHES', 4),
|
|
// 上游完成记录通常不再变化；仍回看 48 小时，覆盖迟到或修正数据。
|
|
'overlap_hours' => (int) env('PRODUCT_PRICE_MIRROR_OVERLAP_HOURS', 48),
|
|
],
|
|
|
|
/*
|
|
|--------------------------------------------------------------------------
|
|
| 上游原表镜像目录
|
|
|--------------------------------------------------------------------------
|
|
|
|
|
| 页面不能再按“页码 + 筛选条件”临时生成一份快照。这里登记项目代码实际
|
|
| 使用的上游表/视图，由后台按表同步到 caiwu_business_mirror。首次全量
|
|
| 必须显式执行；日常任务只续传已完成首次全量的数据集。
|
|
|
|
|
| watermark_column 为空时按稳定业务键追加，并依靠周期全量校准修正历史
|
|
| 更新和删除；存在可靠更新时间时使用重叠窗口增量同步。
|
|
*/
|
|
'table_sync' => [
|
|
'batch_size' => (int) env('BUSINESS_MIRROR_TABLE_BATCH_SIZE', 500),
|
|
'max_batches_per_run' => (int) env('BUSINESS_MIRROR_TABLE_MAX_BATCHES', 2),
|
|
'max_datasets_per_run' => (int) env('BUSINESS_MIRROR_MAX_DATASETS', 1),
|
|
'overlap_hours' => (int) env('BUSINESS_MIRROR_OVERLAP_HOURS', 48),
|
|
'full_reconcile_hours' => (int) env('BUSINESS_MIRROR_FULL_RECONCILE_HOURS', 168),
|
|
],
|
|
|
|
'datasets' => [
|
|
// 经营分析副本库：组织资料、订单、退款和聚水潭基础数据。
|
|
'operations_replica.rpt.fys_companyInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'fys_companyInfo', 'target' => 'ops_fys_company_info', 'keys' => ['Id'], 'watermark' => 'updateTIme'],
|
|
'operations_replica.rpt.fys_shopInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'fys_shopInfo', 'target' => 'ops_fys_shop_info', 'keys' => ['Id'], 'watermark' => 'updateTime'],
|
|
'operations_replica.rpt.fys_TrademarkInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'fys_TrademarkInfo', 'target' => 'ops_fys_trademark_info', 'keys' => ['Id'], 'watermark' => 'updateTime'],
|
|
'operations_replica.rpt.jst_Orders' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_Orders', 'target' => 'ops_jst_orders', 'keys' => ['Id'], 'watermark' => 'updateTime'],
|
|
'operations_replica.rpt.jst_OrderItem' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_OrderItem', 'target' => 'ops_jst_order_item', 'keys' => ['Id']],
|
|
'operations_replica.rpt.jst_sku' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_sku', 'target' => 'ops_jst_sku', 'keys' => ['autoid']],
|
|
'operations_replica.rpt.jst_warehouseInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_warehouseInfo', 'target' => 'ops_jst_warehouse_info', 'keys' => ['Id'], 'watermark' => 'updateTime'],
|
|
'operations_replica.rpt.jst_skumap' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_skumap', 'target' => 'ops_jst_sku_map', 'keys' => ['shop_id', 'shop_i_id', 'shop_sku_id'], 'watermark' => 'modifiedTime'],
|
|
'operations_replica.rpt.ads_itemid_sales_cost_ratio_v2_di' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'ads_itemid_sales_cost_ratio_v2_di', 'target' => 'ops_item_sales_cost_ratio', 'keys' => ['stat_date', 'item_id'], 'watermark' => 'item_id_update'],
|
|
'operations_replica.rpt.jst_returnAndRefundInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_returnAndRefundInfo', 'target' => 'ops_jst_return_refund_info', 'keys' => ['Id'], 'watermark' => 'updateTime'],
|
|
'operations_replica.rpt.jst_returnAndRefundItem' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_returnAndRefundItem', 'target' => 'ops_jst_return_refund_item', 'keys' => ['Id']],
|
|
'operations_replica.rpt.jst_shopInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_shopInfo', 'target' => 'ops_jst_shop_info', 'keys' => ['Id'], 'watermark' => 'updateTime'],
|
|
|
|
// JianSun 旧业务库：营销账单、供应商应付、板材、封边条和库存。
|
|
'jiansun.dbo.tmall_spread_bills' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'tmall_spread_bills', 'target' => 'js_tmall_spread_bills', 'keys' => ['id']],
|
|
'jiansun.dbo.pdd_spread_bills' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pdd_spread_bills', 'target' => 'js_pdd_spread_bills', 'keys' => ['id'], 'watermark' => 'createTime'],
|
|
'jiansun.dbo.alibaba_bill' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'alibaba_bill', 'target' => 'js_alibaba_bill', 'keys' => ['id']],
|
|
'jiansun.dbo.jd_spreadBills' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'jd_spreadBills', 'target' => 'js_jd_spread_bills', 'keys' => ['id'], 'watermark' => 'updateTime'],
|
|
'jiansun.dbo.jd_costBills' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'jd_costBills', 'target' => 'js_jd_cost_bills', 'keys' => ['id'], 'watermark' => 'creatTime'],
|
|
'jiansun.dbo.finance_purchaseReturn' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_purchaseReturn', 'target' => 'js_finance_purchase_return', 'keys' => ['id']],
|
|
'jiansun.dbo.finance_invoiceDeduction' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_invoiceDeduction', 'target' => 'js_finance_invoice_deduction', 'keys' => ['id'], 'watermark' => 'createTime'],
|
|
'jiansun.dbo.fys_supplierPaymentSummary' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'fys_supplierPaymentSummary', 'target' => 'js_supplier_payment_summary', 'keys' => ['id'], 'watermark' => 'modifiedTime'],
|
|
'jiansun.dbo.finance_advanceLoans' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_advanceLoans', 'target' => 'js_finance_advance_loans', 'keys' => ['id'], 'watermark' => 'createTime'],
|
|
'jiansun.dbo.pm_unOrderDetails' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_unOrderDetails', 'target' => 'js_pm_unorder_details', 'keys' => ['Id']],
|
|
'jiansun.dbo.finance_sampleDeduction' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_sampleDeduction', 'target' => 'js_finance_sample_deduction', 'keys' => ['id'], 'watermark' => 'finishTime'],
|
|
'jiansun.dbo.fys_express_courier_settlement' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'fys_express_courier_settlement', 'target' => 'js_express_courier_settlement', 'keys' => ['Id'], 'watermark' => 'modify_time'],
|
|
'jiansun.dbo.finance_boardDeduction' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_boardDeduction', 'target' => 'js_finance_board_deduction', 'keys' => ['id']],
|
|
'jiansun.dbo.pm_plainPlateData' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_plainPlateData', 'target' => 'js_pm_plain_plate_data', 'keys' => ['Id'], 'watermark' => 'createTime'],
|
|
'jiansun.dbo.pm_boardPurchasePrice' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_boardPurchasePrice', 'target' => 'js_pm_board_purchase_price', 'keys' => ['id'], 'watermark' => 'updateTime'],
|
|
'jiansun.dbo.pm_boardPriceChangeImport' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_boardPriceChangeImport', 'target' => 'js_pm_board_price_change', 'keys' => ['id'], 'watermark' => 'createTime'],
|
|
'jiansun.dbo.pm_boardApplyDetail' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_boardApplyDetail', 'target' => 'js_pm_board_apply_detail', 'keys' => ['Id']],
|
|
'jiansun.dbo.pm_edgeBandData' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_edgeBandData', 'target' => 'js_pm_edge_band_data', 'keys' => ['id'], 'watermark' => 'createTime'],
|
|
'jiansun.dbo.pm_edgeBandApplyDetail' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_edgeBandApplyDetail', 'target' => 'js_pm_edge_band_apply_detail', 'keys' => ['Id']],
|
|
'jiansun.dbo.pm_edgeBandProfitInfo' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_edgeBandProfitInfo', 'target' => 'js_pm_edge_band_profit_info', 'keys' => ['id']],
|
|
'jiansun.dbo.finance_edgeBandDeductio' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_edgeBandDeductio', 'target' => 'js_finance_edge_band_deduction', 'keys' => ['id']],
|
|
'jiansun.dbo.pm_averageUnitPrice' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_averageUnitPrice', 'target' => 'js_pm_average_unit_price', 'keys' => ['id']],
|
|
'jiansun.dbo.pm_boardInventoryDetail_snapshot' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_boardInventoryDetail_snapshot', 'target' => 'js_pm_board_inventory_snapshot', 'keys' => ['id']],
|
|
'jiansun.dbo.fys_supplierGoods_sttlement_purchase' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'fys_supplierGoods_sttlement_purchase', 'target' => 'js_supplier_goods_purchase', 'keys' => ['Id'], 'watermark' => 'createTime'],
|
|
'jiansun.dbo.fys_supplierGoods_sttlement_sale' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'fys_supplierGoods_sttlement_sale', 'target' => 'js_supplier_goods_sale', 'keys' => ['Id'], 'watermark' => 'createTime'],
|
|
'jiansun.dbo.pm_purchaseIn' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_purchaseIn', 'target' => 'js_pm_purchase_in', 'keys' => ['id'], 'watermark' => 'updateTime'],
|
|
],
|
|
];
|