From 6b2b486ce171e2db0a6a9a3334c9cd22c18941c2 Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Fri, 14 Aug 2026 16:38:33 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E5=BB=BA=E7=AB=8B=E5=85=A8=E9=A1=B9?= =?UTF-8?q?=E7=9B=AE=E4=B8=8A=E6=B8=B8=E4=B8=9A=E5=8A=A1=E9=95=9C=E5=83=8F?= =?UTF-8?q?=E6=A1=86=E6=9E=B6=20(#53)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 任务占用与边界 - 任务名称：全项目上游业务镜像目录与通用同步框架 - 实际开发者：WuYongJie - 目标结果：登记当前代码实际使用的上游原表，提供低压、可断点的首次全量、增量重叠和全量校准框架。 - 实际修改：13 个文件；新增通用数据集目录、动态镜像建表、同步状态、同步命令、生产低压轮询、测试和团队清单。 - 将写入的公共契约：`business_mirror_table_states`、`business-mirror:sync-tables`、`BusinessMirrorDatasetRegistry`、`BusinessMirrorSchemaManager`。 - 仅复用、明确不修改：`UpstreamReadOnlyQueryFactory`、`upstream_schema_columns`、既有 `product_price_mirror_rows`。 - 明确不包含：本 PR 不切换其余业务页面 Repository，不执行生产迁移或首次全量，不提交真实业务数据或凭据。 ## 合并前自审 - [x] 已重新拉取最新 `origin/main`，分支仅领先 1 个提交 - [x] `git merge-tree --write-tree origin/main HEAD` 无冲突 - [x] 完整 diff 共 13 个文件，未发现越界文件或敏感内容 - [x] 生产共享数据字典核对 38 个数据集，缺失对象/字段为 0 - [x] 临时 MariaDB 空库迁移实测成功，验证库已删除 - [x] 上游异常空结果不会清空已有镜像 - [x] 定时任务只轮转已完成首次全量的数据集，不会自动扫描未初始化大表 ## 测试 - `BusinessMirrorSynchronizerTest` 等相关测试：14 tests / 65 assertions 全通过 - 新同步器专项：5 tests / 25 assertions 全通过 - 新增 PHP 文件语法检查全部通过 - 两份生产激活脚本 `bash -n` 通过 - `git diff --check` 通过 ## 风险与回滚 - 包含新增迁移，但发布脚本遇到迁移仍会要求独立高风险授权；合并本身不会执行生产迁移。 - 首次全量必须逐数据集显式执行并做数量/翻页/筛选验收，特别是约 280 万行的大表不得随发布自动全扫。 - 回滚代码可回退本合并提交；生产迁移执行前必须备份 `caiwu_system_config` 与 `caiwu_business_mirror`。 Reviewed-on: http://git.1080p.hk/WuYongJie/CaiWuXiTong/pulls/53 --- app/Console/Commands/SyncBusinessMirrors.php | 107 +++++++ config/upstream_mirrors.php | 64 +++++ ..._create_generic_business_mirror_tables.php | 62 +++++ deploy/activate-incremental-release.sh | 2 +- deploy/activate-release.sh | 2 +- docs/上游业务镜像清单.md | 60 ++++ docs/开发与部署SOP.md | 9 + .../Contracts/BusinessMirrorDataset.php | 21 ++ .../GenericBusinessMirrorRepository.php | 256 +++++++++++++++++ .../BusinessMirrorDatasetRegistry.php | 78 ++++++ .../Services/BusinessMirrorSchemaManager.php | 145 ++++++++++ .../Services/BusinessMirrorSynchronizer.php | 263 ++++++++++++++++++ .../BusinessMirrorSynchronizerTest.php | 223 +++++++++++++++ 13 files changed, 1290 insertions(+), 2 deletions(-) create mode 100644 app/Console/Commands/SyncBusinessMirrors.php create mode 100644 database/migrations/2026_08_14_230000_create_generic_business_mirror_tables.php create mode 100644 docs/上游业务镜像清单.md create mode 100644 modules/EcommerceFinance/Contracts/BusinessMirrorDataset.php create mode 100644 modules/EcommerceFinance/Repositories/GenericBusinessMirrorRepository.php create mode 100644 modules/EcommerceFinance/Services/BusinessMirrorDatasetRegistry.php create mode 100644 modules/EcommerceFinance/Services/BusinessMirrorSchemaManager.php create mode 100644 modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php create mode 100644 tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php diff --git a/app/Console/Commands/SyncBusinessMirrors.php b/app/Console/Commands/SyncBusinessMirrors.php new file mode 100644 index 00000000..5abb3fa5 --- /dev/null +++ b/app/Console/Commands/SyncBusinessMirrors.php @@ -0,0 +1,107 @@ +option('full')) { + return self::SUCCESS; + } + + $requested = $this->argument('dataset'); + $keys = $requested === null + ? $this->scheduledKeys($datasets, $mirror) + : [(string) $requested]; + $maxDatasets = max((int) ($this->option('max-datasets') + ?? config('upstream_mirrors.table_sync.max_datasets_per_run', 1)), 1); + $processed = 0; + $failed = false; + + foreach ($keys as $key) { + if ($processed >= $maxDatasets) { + break; + } + + try { + $result = $synchronizer->sync( + $key, + (bool) $this->option('full'), + $this->option('batch') === null ? null : (int) $this->option('batch'), + $this->option('max-batches') === null ? null : (int) $this->option('max-batches'), + ); + } catch (Throwable $exception) { + $failed = true; + Log::error('共享业务镜像表同步失败。', ['dataset' => $key, 'exception' => $exception::class]); + $this->error($key . '：同步失败，已保留上一版共享数据。'); + $processed++; + continue; + } + + if ($result['status'] === 'requires_full_sync') { + if ($requested !== null) { + $this->warn($key . '：尚未完成首次全量同步，请显式使用 --full。'); + } + continue; + } + + $processed++; + $this->info(sprintf( + '%s：%s，本轮 %d 批，累计处理 %d 行。', + $key, + $result['status'] === 'succeeded' ? '同步完成' : '同步进行中', + $result['batches'], + $result['rows'], + )); + } + + return $failed ? self::FAILURE : self::SUCCESS; + } + + /** @return list */ + private function scheduledKeys( + BusinessMirrorDatasetRegistry $datasets, + GenericBusinessMirrorRepository $mirror, + ): array { + $scheduled = []; + + foreach ($datasets->all() as $key => $dataset) { + $state = $mirror->state($dataset); + if (! is_array($state) || ($state['initial_completed_at'] ?? null) === null) { + continue; + } + + $scheduled[] = [ + 'key' => $key, + 'last_attempted_at' => (string) ($state['last_attempted_at'] ?? ''), + ]; + } + + usort($scheduled, fn (array $left, array $right): int => + [$left['last_attempted_at'], $left['key']] <=> [$right['last_attempted_at'], $right['key']] + ); + + return array_column($scheduled, 'key'); + } +} diff --git a/config/upstream_mirrors.php b/config/upstream_mirrors.php index 68573f37..2df70817 100644 --- a/config/upstream_mirrors.php +++ b/config/upstream_mirrors.php @@ -18,4 +18,68 @@ return [ // 上游完成记录通常不再变化；仍回看 48 小时，覆盖迟到或修正数据。 'overlap_hours' => (int) env('PRODUCT_PRICE_MIRROR_OVERLAP_HOURS', 48), ], + + /* + |-------------------------------------------------------------------------- + | 上游原表镜像目录 + |-------------------------------------------------------------------------- + | + | 页面不能再按“页码 + 筛选条件”临时生成一份快照。这里登记项目代码实际 + | 使用的上游表/视图，由后台按表同步到 caiwu_business_mirror。首次全量 + | 必须显式执行；日常任务只续传已完成首次全量的数据集。 + | + | watermark_column 为空时按稳定业务键追加，并依靠周期全量校准修正历史 + | 更新和删除；存在可靠更新时间时使用重叠窗口增量同步。 + */ + 'table_sync' => [ + 'batch_size' => (int) env('BUSINESS_MIRROR_TABLE_BATCH_SIZE', 500), + 'max_batches_per_run' => (int) env('BUSINESS_MIRROR_TABLE_MAX_BATCHES', 2), + 'max_datasets_per_run' => (int) env('BUSINESS_MIRROR_MAX_DATASETS', 1), + 'overlap_hours' => (int) env('BUSINESS_MIRROR_OVERLAP_HOURS', 48), + 'full_reconcile_hours' => (int) env('BUSINESS_MIRROR_FULL_RECONCILE_HOURS', 168), + ], + + 'datasets' => [ + // 经营分析副本库：组织资料、订单、退款和聚水潭基础数据。 + 'operations_replica.rpt.fys_companyInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'fys_companyInfo', 'target' => 'ops_fys_company_info', 'keys' => ['Id'], 'watermark' => 'updateTIme'], + 'operations_replica.rpt.fys_shopInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'fys_shopInfo', 'target' => 'ops_fys_shop_info', 'keys' => ['Id'], 'watermark' => 'updateTime'], + 'operations_replica.rpt.fys_TrademarkInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'fys_TrademarkInfo', 'target' => 'ops_fys_trademark_info', 'keys' => ['Id'], 'watermark' => 'updateTime'], + 'operations_replica.rpt.jst_Orders' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_Orders', 'target' => 'ops_jst_orders', 'keys' => ['Id'], 'watermark' => 'updateTime'], + 'operations_replica.rpt.jst_OrderItem' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_OrderItem', 'target' => 'ops_jst_order_item', 'keys' => ['Id']], + 'operations_replica.rpt.jst_sku' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_sku', 'target' => 'ops_jst_sku', 'keys' => ['autoid']], + 'operations_replica.rpt.jst_warehouseInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_warehouseInfo', 'target' => 'ops_jst_warehouse_info', 'keys' => ['Id'], 'watermark' => 'updateTime'], + 'operations_replica.rpt.jst_skumap' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_skumap', 'target' => 'ops_jst_sku_map', 'keys' => ['shop_id', 'shop_i_id', 'shop_sku_id'], 'watermark' => 'modifiedTime'], + 'operations_replica.rpt.ads_itemid_sales_cost_ratio_v2_di' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'ads_itemid_sales_cost_ratio_v2_di', 'target' => 'ops_item_sales_cost_ratio', 'keys' => ['stat_date', 'item_id'], 'watermark' => 'item_id_update'], + 'operations_replica.rpt.jst_returnAndRefundInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_returnAndRefundInfo', 'target' => 'ops_jst_return_refund_info', 'keys' => ['Id'], 'watermark' => 'updateTime'], + 'operations_replica.rpt.jst_returnAndRefundItem' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_returnAndRefundItem', 'target' => 'ops_jst_return_refund_item', 'keys' => ['Id']], + 'operations_replica.rpt.jst_shopInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_shopInfo', 'target' => 'ops_jst_shop_info', 'keys' => ['Id'], 'watermark' => 'updateTime'], + + // JianSun 旧业务库：营销账单、供应商应付、板材、封边条和库存。 + 'jiansun.dbo.tmall_spread_bills' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'tmall_spread_bills', 'target' => 'js_tmall_spread_bills', 'keys' => ['id']], + 'jiansun.dbo.pdd_spread_bills' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pdd_spread_bills', 'target' => 'js_pdd_spread_bills', 'keys' => ['id'], 'watermark' => 'createTime'], + 'jiansun.dbo.alibaba_bill' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'alibaba_bill', 'target' => 'js_alibaba_bill', 'keys' => ['id']], + 'jiansun.dbo.jd_spreadBills' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'jd_spreadBills', 'target' => 'js_jd_spread_bills', 'keys' => ['id'], 'watermark' => 'updateTime'], + 'jiansun.dbo.jd_costBills' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'jd_costBills', 'target' => 'js_jd_cost_bills', 'keys' => ['id'], 'watermark' => 'creatTime'], + 'jiansun.dbo.finance_purchaseReturn' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_purchaseReturn', 'target' => 'js_finance_purchase_return', 'keys' => ['id']], + 'jiansun.dbo.finance_invoiceDeduction' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_invoiceDeduction', 'target' => 'js_finance_invoice_deduction', 'keys' => ['id'], 'watermark' => 'createTime'], + 'jiansun.dbo.fys_supplierPaymentSummary' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'fys_supplierPaymentSummary', 'target' => 'js_supplier_payment_summary', 'keys' => ['id'], 'watermark' => 'modifiedTime'], + 'jiansun.dbo.finance_advanceLoans' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_advanceLoans', 'target' => 'js_finance_advance_loans', 'keys' => ['id'], 'watermark' => 'createTime'], + 'jiansun.dbo.pm_unOrderDetails' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_unOrderDetails', 'target' => 'js_pm_unorder_details', 'keys' => ['Id']], + 'jiansun.dbo.finance_sampleDeduction' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_sampleDeduction', 'target' => 'js_finance_sample_deduction', 'keys' => ['id'], 'watermark' => 'finishTime'], + 'jiansun.dbo.fys_express_courier_settlement' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'fys_express_courier_settlement', 'target' => 'js_express_courier_settlement', 'keys' => ['Id'], 'watermark' => 'modify_time'], + 'jiansun.dbo.finance_boardDeduction' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_boardDeduction', 'target' => 'js_finance_board_deduction', 'keys' => ['id']], + 'jiansun.dbo.pm_plainPlateData' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_plainPlateData', 'target' => 'js_pm_plain_plate_data', 'keys' => ['Id'], 'watermark' => 'createTime'], + 'jiansun.dbo.pm_boardPurchasePrice' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_boardPurchasePrice', 'target' => 'js_pm_board_purchase_price', 'keys' => ['id'], 'watermark' => 'updateTime'], + 'jiansun.dbo.pm_boardPriceChangeImport' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_boardPriceChangeImport', 'target' => 'js_pm_board_price_change', 'keys' => ['id'], 'watermark' => 'createTime'], + 'jiansun.dbo.pm_boardApplyDetail' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_boardApplyDetail', 'target' => 'js_pm_board_apply_detail', 'keys' => ['Id']], + 'jiansun.dbo.pm_edgeBandData' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_edgeBandData', 'target' => 'js_pm_edge_band_data', 'keys' => ['id'], 'watermark' => 'createTime'], + 'jiansun.dbo.pm_edgeBandApplyDetail' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_edgeBandApplyDetail', 'target' => 'js_pm_edge_band_apply_detail', 'keys' => ['Id']], + 'jiansun.dbo.pm_edgeBandProfitInfo' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_edgeBandProfitInfo', 'target' => 'js_pm_edge_band_profit_info', 'keys' => ['id']], + 'jiansun.dbo.finance_edgeBandDeductio' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_edgeBandDeductio', 'target' => 'js_finance_edge_band_deduction', 'keys' => ['id']], + 'jiansun.dbo.pm_averageUnitPrice' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_averageUnitPrice', 'target' => 'js_pm_average_unit_price', 'keys' => ['id']], + 'jiansun.dbo.pm_boardInventoryDetail_snapshot' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_boardInventoryDetail_snapshot', 'target' => 'js_pm_board_inventory_snapshot', 'keys' => ['id']], + 'jiansun.dbo.fys_supplierGoods_sttlement_purchase' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'fys_supplierGoods_sttlement_purchase', 'target' => 'js_supplier_goods_purchase', 'keys' => ['Id'], 'watermark' => 'createTime'], + 'jiansun.dbo.fys_supplierGoods_sttlement_sale' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'fys_supplierGoods_sttlement_sale', 'target' => 'js_supplier_goods_sale', 'keys' => ['Id'], 'watermark' => 'createTime'], + 'jiansun.dbo.pm_purchaseIn' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_purchaseIn', 'target' => 'js_pm_purchase_in', 'keys' => ['id'], 'watermark' => 'updateTime'], + ], ]; diff --git a/database/migrations/2026_08_14_230000_create_generic_business_mirror_tables.php b/database/migrations/2026_08_14_230000_create_generic_business_mirror_tables.php new file mode 100644 index 00000000..53f18ce9 --- /dev/null +++ b/database/migrations/2026_08_14_230000_create_generic_business_mirror_tables.php @@ -0,0 +1,62 @@ +environment('testing') + || (string) config('system_database.mode', 'bootstrap') === 'bootstrap') { + return; + } + + $schema = Schema::connection($this->connection()); + + if ($schema->hasTable('business_mirror_table_states')) { + return; + } + + $schema->create('business_mirror_table_states', function (Blueprint $table): void { + $table->string('dataset_key', 191)->primary(); + $table->string('source_key', 64); + $table->string('source_object', 191); + $table->string('target_table', 64); + $table->string('status', 24)->default('pending'); + $table->string('run_mode', 24)->nullable(); + $table->uuid('generation')->nullable(); + $table->json('cursor')->nullable(); + $table->json('high_watermark')->nullable(); + $table->dateTime('window_started_at', 6)->nullable(); + $table->dateTime('window_ends_at', 6)->nullable(); + $table->dateTime('initial_completed_at')->nullable(); + $table->dateTime('last_full_succeeded_at')->nullable(); + $table->dateTime('last_attempted_at')->nullable(); + $table->dateTime('last_succeeded_at')->nullable(); + $table->unsignedBigInteger('last_upserted_rows')->default(0); + $table->unsignedBigInteger('mirror_row_count')->default(0); + $table->string('last_error_code', 120)->nullable(); + $table->timestamps(); + + $table->index(['source_key', 'status'], 'mirror_table_states_source_status'); + $table->index('last_succeeded_at', 'mirror_table_states_last_success'); + }); + } + + public function down(): void + { + if (app()->environment('testing') + || (string) config('system_database.mode', 'bootstrap') === 'bootstrap') { + return; + } + + Schema::connection($this->connection())->dropIfExists('business_mirror_table_states'); + } +}; diff --git a/deploy/activate-incremental-release.sh b/deploy/activate-incremental-release.sh index 9bc68558..2e7f6097 100644 --- a/deploy/activate-incremental-release.sh +++ b/deploy/activate-incremental-release.sh @@ -128,7 +128,7 @@ mv "$REMOTE_DIR/.deployed-sha.tmp" "$REMOTE_DIR/.deployed-sha" # 只运行已登记的上游后台读取，不启用整个 Laravel 调度器，避免意外启动 # 账单提醒等既有任务。同一把锁保证快照与业务镜像不会并发施压上游。 cat > /etc/cron.d/caiwu-jiansun-snapshots <<'EOF' -* * * * * root /usr/bin/flock -n /run/caiwu-upstream-sync.lock /bin/sh -c '/usr/bin/docker exec -u www-data caiwu_app php artisan upstream:snapshots:refresh-all --quiet; /usr/bin/docker exec -u www-data caiwu_app php artisan business-mirror:sync-product-prices --quiet' >/dev/null 2>&1 +* * * * * root /usr/bin/flock -n /run/caiwu-upstream-sync.lock /bin/sh -c '/usr/bin/docker exec -u www-data caiwu_app php artisan upstream:snapshots:refresh-all --quiet; /usr/bin/docker exec -u www-data caiwu_app php artisan business-mirror:sync-product-prices --quiet; /usr/bin/docker exec -u www-data caiwu_app php artisan business-mirror:sync-tables --quiet' >/dev/null 2>&1 EOF chmod 644 /etc/cron.d/caiwu-jiansun-snapshots diff --git a/deploy/activate-release.sh b/deploy/activate-release.sh index 31559d6f..45a4b552 100644 --- a/deploy/activate-release.sh +++ b/deploy/activate-release.sh @@ -132,7 +132,7 @@ mv "$REMOTE_DIR/.deployed-sha.tmp" "$REMOTE_DIR/.deployed-sha" # 只运行已登记的上游后台读取，不启用整个 Laravel 调度器，避免意外启动 # 账单提醒等既有任务。同一把锁保证快照与业务镜像不会并发施压上游。 cat > /etc/cron.d/caiwu-jiansun-snapshots <<'EOF' -* * * * * root /usr/bin/flock -n /run/caiwu-upstream-sync.lock /bin/sh -c '/usr/bin/docker exec -u www-data caiwu_app php artisan upstream:snapshots:refresh-all --quiet; /usr/bin/docker exec -u www-data caiwu_app php artisan business-mirror:sync-product-prices --quiet' >/dev/null 2>&1 +* * * * * root /usr/bin/flock -n /run/caiwu-upstream-sync.lock /bin/sh -c '/usr/bin/docker exec -u www-data caiwu_app php artisan upstream:snapshots:refresh-all --quiet; /usr/bin/docker exec -u www-data caiwu_app php artisan business-mirror:sync-product-prices --quiet; /usr/bin/docker exec -u www-data caiwu_app php artisan business-mirror:sync-tables --quiet' >/dev/null 2>&1 EOF chmod 644 /etc/cron.d/caiwu-jiansun-snapshots diff --git a/docs/上游业务镜像清单.md b/docs/上游业务镜像清单.md new file mode 100644 index 00000000..0cc43440 --- /dev/null +++ b/docs/上游业务镜像清单.md @@ -0,0 +1,60 @@ +# 上游业务镜像清单 + +## 结论 + +项目不是只使用 `rpt.Bulk_Price_Limit` 一张表。当前主线代码直接读取两套上游中的约 40 个表或视图，覆盖组织资料、订单、退款、营销账户、供应商应付、板材、封边条和库存。 + +`caiwu_business_mirror` 的目标是把项目实际使用的上游原表数据保存为共享业务镜像。网页只读镜像，后台刷新节点在低压、可断点的条件下完成首次全量和后续增量；不得再按页码或筛选条件临时生成不完整的数据副本。 + +## 当前迁移状态 + +| 状态 | 说明 | +| --- | --- | +| 已完成 | `operations_replica/rpt.Bulk_Price_Limit` 已进入类型化表 `product_price_mirror_rows`，商品价格页面已读取镜像并支持完整分页。 | +| 本次建立 | `config/upstream_mirrors.php` 中登记全部当前代码引用的原表；通用表级同步框架负责建表、稳定游标、断点续传、增量重叠和全量校准。 | +| 待逐域切换 | 其他业务 Repository 仍需从上游 SQL Server 查询改为读取对应镜像表。切换前保留旧快照作为回滚，不得宣称已经完成。 | + +## 经营分析副本库（operations_replica） + +| 业务范围 | 上游对象 | 镜像目标 | +| --- | --- | --- | +| 公司资料 | `rpt.fys_companyInfo` | `ops_fys_company_info` | +| 店铺资料 | `rpt.fys_shopInfo` | `ops_fys_shop_info` | +| 商标资料 | `rpt.fys_TrademarkInfo` | `ops_fys_trademark_info` | +| 订单 | `rpt.jst_Orders`、`rpt.jst_OrderItem` | `ops_jst_orders`、`ops_jst_order_item` | +| 商品与仓库 | `rpt.jst_sku`、`rpt.jst_warehouseInfo`、`rpt.jst_skumap` | `ops_jst_*` | +| 推广费用 | `rpt.ads_itemid_sales_cost_ratio_v2_di` | `ops_item_sales_cost_ratio` | +| 退款退货 | `rpt.jst_returnAndRefundInfo`、`rpt.jst_returnAndRefundItem` | `ops_jst_return_refund_*` | +| 聚水潭店铺 | `rpt.jst_shopInfo` | `ops_jst_shop_info` | + +商品价格 `rpt.Bulk_Price_Limit` 继续使用已经上线的类型化同步器，不在通用原表任务中重复同步。 + +## JianSun 旧业务库（jiansun） + +| 业务范围 | 上游对象 | +| --- | --- | +| 营销账户 | `dbo.tmall_spread_bills`、`dbo.pdd_spread_bills`、`dbo.alibaba_bill`、`dbo.jd_spreadBills`、`dbo.jd_costBills` | +| 供应商应付 | `dbo.finance_purchaseReturn`、`dbo.finance_invoiceDeduction`、`dbo.fys_supplierPaymentSummary`、`dbo.finance_advanceLoans`、`dbo.pm_unOrderDetails`、`dbo.finance_sampleDeduction`、`dbo.fys_express_courier_settlement` | +| 板材 | `dbo.finance_boardDeduction`、`dbo.pm_plainPlateData`、`dbo.pm_boardPurchasePrice`、`dbo.pm_boardPriceChangeImport`、`dbo.pm_boardApplyDetail` | +| 封边条 | `dbo.pm_edgeBandData`、`dbo.pm_edgeBandApplyDetail`、`dbo.pm_edgeBandProfitInfo`、`dbo.finance_edgeBandDeductio`、`dbo.pm_averageUnitPrice` | +| 库存与货款对账 | `dbo.pm_boardInventoryDetail_snapshot`、`dbo.fys_supplierGoods_sttlement_purchase`、`dbo.fys_supplierGoods_sttlement_sale`、`dbo.pm_purchaseIn` | + +其中 `fys_supplierGoods_sttlement_sale` 当前约 280 万行，是首次全量同步的最大对象。必须在低峰期小批量、可断点执行，禁止把全量同步塞进发布流程。 + +## 同步规则 + +1. 首次全量只能通过 `business-mirror:sync-tables --full` 显式启动；失败或中断后使用同一命令继续。 +2. 日常任务只处理已经完成首次全量的数据集。存在可靠更新时间的表使用 48 小时重叠窗口；没有可靠更新时间的表按稳定主键追加。 +3. 全量校准使用新的同步代次，只有全部完成后才删除镜像中本轮未出现的旧行，因此中途失败不会清空当前可用数据。 +4. 每轮默认只处理 1 个数据集、每个数据集 2 批，每批 500 行，避免同时压迫两个旧系统。 +5. 页面切换必须以“总记录数、第一页、第二页、末页、筛选、汇总、空数据和更新时间”作为验收，不能只看首页有数据。 + +## 后续切换顺序 + +1. 公司、店铺、商标等小型维表。 +2. 营销账户五张单表账单。 +3. 供应商应付与货款对账。 +4. 订单、退款和推广费用。 +5. 板材、封边条和库存的跨表查询。 + +每个阶段都必须先完成首次全量和数量核对，再将对应 Repository 切到镜像；任一阶段失败时保留上一版镜像和旧只读链路用于回滚。 diff --git a/docs/开发与部署SOP.md b/docs/开发与部署SOP.md index bb011ae4..146c0a6d 100644 --- a/docs/开发与部署SOP.md +++ b/docs/开发与部署SOP.md @@ -234,6 +234,15 @@ php artisan business-mirror:sync-product-prices --full php artisan business-mirror:sync-product-prices ``` +其余已在代码中使用的上游表由统一的数据集目录管理。首次全量必须逐个数据集显式执行，普通定时任务只续传已完成首次全量的数据集： + +```bash +php artisan business-mirror:sync-tables --full +php artisan business-mirror:sync-tables +``` + +完整数据集、来源对象、镜像表和分阶段切换状态见 `docs/上游业务镜像清单.md`。不得把“已登记同步目录”误报成“页面已经切换完成”；每个业务域仍需在首份全量核对通过后，把对应 Repository 改为只读镜像。 + 首次 `--full` 属于生产数据写入，必须先备份 `caiwu_system_config` 和 `caiwu_business_mirror`，并由负责人明确授权。后续无参数命令只执行或继续受控增量。 ### 8.3 开发者需要尚未接入的上游数据时 diff --git a/modules/EcommerceFinance/Contracts/BusinessMirrorDataset.php b/modules/EcommerceFinance/Contracts/BusinessMirrorDataset.php new file mode 100644 index 00000000..015d6b41 --- /dev/null +++ b/modules/EcommerceFinance/Contracts/BusinessMirrorDataset.php @@ -0,0 +1,21 @@ + */ + public function keyColumns(): array; + + public function watermarkColumn(): ?string; +} diff --git a/modules/EcommerceFinance/Repositories/GenericBusinessMirrorRepository.php b/modules/EcommerceFinance/Repositories/GenericBusinessMirrorRepository.php new file mode 100644 index 00000000..4dcee6ad --- /dev/null +++ b/modules/EcommerceFinance/Repositories/GenericBusinessMirrorRepository.php @@ -0,0 +1,256 @@ +connection = $database->connection( + (string) config('upstream_mirrors.connection', 'business_mirror') + ); + } + + public function connection(): ConnectionInterface + { + return $this->connection; + } + + public function ready(BusinessMirrorDataset $dataset): bool + { + $schema = Schema::connection($this->connection->getName()); + + if (! $schema->hasTable('business_mirror_table_states') + || ! $schema->hasTable($dataset->targetTable())) { + return false; + } + + $state = $this->state($dataset); + + return is_array($state) && ($state['initial_completed_at'] ?? null) !== null; + } + + /** @return array|null */ + public function state(BusinessMirrorDataset $dataset): ?array + { + if (! Schema::connection($this->connection->getName())->hasTable('business_mirror_table_states')) { + return null; + } + + $row = $this->connection->table('business_mirror_table_states') + ->where('dataset_key', $dataset->key()) + ->first(); + + if ($row === null) { + return null; + } + + $state = (array) $row; + $state['cursor'] = $this->decode($state['cursor'] ?? null); + $state['high_watermark'] = $this->decode($state['high_watermark'] ?? null); + + return $state; + } + + public function startRun( + BusinessMirrorDataset $dataset, + string $mode, + string $generation, + ?string $windowStart, + ?string $windowEnd, + bool $resume, + ): array { + $now = now(); + $values = [ + 'source_key' => $dataset->sourceKey(), + 'source_object' => $dataset->sourceSchema() . '.' . $dataset->sourceObject(), + 'target_table' => $dataset->targetTable(), + 'status' => 'running', + 'run_mode' => $mode, + 'generation' => $generation, + 'last_attempted_at' => $now, + 'last_error_code' => null, + 'updated_at' => $now, + ]; + + if (! $resume) { + $values += [ + 'cursor' => null, + 'window_started_at' => $windowStart, + 'window_ends_at' => $windowEnd, + 'last_upserted_rows' => 0, + ]; + } + + $query = $this->connection->table('business_mirror_table_states') + ->where('dataset_key', $dataset->key()); + + if ($query->exists()) { + $query->update($values); + } else { + $this->connection->table('business_mirror_table_states')->insert( + ['dataset_key' => $dataset->key(), 'created_at' => $now] + $values, + ); + } + + return $this->state($dataset) ?? []; + } + + public function storeBatch( + BusinessMirrorDataset $dataset, + array $columns, + array $rows, + array $cursor, + string $generation, + int $cumulativeRows, + ): void { + $now = now(); + $columnNames = array_map(fn (array $column): string => (string) $column['column_name'], $columns); + $mapped = []; + + foreach ($rows as $row) { + $values = []; + foreach ($columnNames as $columnName) { + $values[$columnName] = $this->value($row, $columnName); + } + + $identity = array_map( + fn (string $column): string => $this->identityValue($this->value($row, $column)), + $dataset->keyColumns(), + ); + + $mapped[] = $values + [ + '_mirror_key' => hash('sha256', implode("\0", $identity)), + '_mirror_generation' => $generation, + '_mirror_synced_at' => $now, + ]; + } + + $this->connection->transaction(function () use ( + $dataset, $columnNames, $mapped, $cursor, $cumulativeRows, $now + ): void { + $updates = [...$columnNames, '_mirror_generation', '_mirror_synced_at']; + $chunkSize = max(1, intdiv(50000, max(count($updates) + 1, 1))); + + foreach (array_chunk($mapped, $chunkSize) as $chunk) { + $this->connection->table($dataset->targetTable())->upsert($chunk, ['_mirror_key'], $updates); + } + + $this->connection->table('business_mirror_table_states') + ->where('dataset_key', $dataset->key()) + ->update([ + 'cursor' => json_encode($cursor, JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION), + 'last_upserted_rows' => $cumulativeRows, + 'updated_at' => $now, + ]); + }); + } + + public function complete( + BusinessMirrorDataset $dataset, + string $mode, + string $generation, + array $highWatermark, + int $rows, + ): void { + $now = now(); + + $this->connection->transaction(function () use ( + $dataset, $mode, $generation, $highWatermark, $rows, $now + ): void { + if ($mode === 'full') { + $existingRows = $this->connection->table($dataset->targetTable())->count(); + + if ($rows === 0 && $existingRows > 0) { + throw new RuntimeException('business_mirror_unexpected_empty_source'); + } + + $this->connection->table($dataset->targetTable()) + ->where(function ($query) use ($generation): void { + $query->whereNull('_mirror_generation') + ->orWhere('_mirror_generation', '!=', $generation); + }) + ->delete(); + } + + $rowCount = $this->connection->table($dataset->targetTable())->count(); + $values = [ + 'status' => 'succeeded', + 'cursor' => null, + 'high_watermark' => json_encode($highWatermark, JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION), + 'last_succeeded_at' => $now, + 'last_upserted_rows' => $rows, + 'mirror_row_count' => $rowCount, + 'last_error_code' => null, + 'updated_at' => $now, + ]; + + if ($mode === 'full') { + $values['initial_completed_at'] = $now; + $values['last_full_succeeded_at'] = $now; + } + + $this->connection->table('business_mirror_table_states') + ->where('dataset_key', $dataset->key()) + ->update($values); + }); + } + + public function fail(BusinessMirrorDataset $dataset, string $errorCode): void + { + $this->connection->table('business_mirror_table_states') + ->where('dataset_key', $dataset->key()) + ->update([ + 'status' => 'failed', + 'last_error_code' => mb_substr($errorCode, 0, 120), + 'updated_at' => now(), + ]); + } + + private function decode(mixed $value): ?array + { + if (is_array($value)) { + return $value; + } + + if (! is_string($value) || $value === '') { + return null; + } + + $decoded = json_decode($value, true); + + return is_array($decoded) ? $decoded : null; + } + + private function value(array $row, string $name): mixed + { + foreach ($row as $key => $value) { + if (strcasecmp((string) $key, $name) === 0) { + return $value; + } + } + + return null; + } + + private function identityValue(mixed $value): string + { + if ($value === null) { + return ''; + } + + if (is_bool($value)) { + return $value ? '1' : '0'; + } + + return (string) $value; + } +} diff --git a/modules/EcommerceFinance/Services/BusinessMirrorDatasetRegistry.php b/modules/EcommerceFinance/Services/BusinessMirrorDatasetRegistry.php new file mode 100644 index 00000000..5df609ee --- /dev/null +++ b/modules/EcommerceFinance/Services/BusinessMirrorDatasetRegistry.php @@ -0,0 +1,78 @@ + */ + public function all(): array + { + $datasets = []; + + foreach ((array) config('upstream_mirrors.datasets', []) as $key => $definition) { + $datasets[(string) $key] = $this->make((string) $key, (array) $definition); + } + + return $datasets; + } + + public function get(string $key): BusinessMirrorDataset + { + $definitions = (array) config('upstream_mirrors.datasets', []); + $definition = (array) ($definitions[$key] ?? []); + + if ($definition === []) { + throw new InvalidArgumentException('未知的业务镜像数据集：' . $key); + } + + return $this->make($key, $definition); + } + + private function make(string $key, array $definition): BusinessMirrorDataset + { + $source = trim((string) ($definition['source_key'] ?? '')); + $schema = trim((string) ($definition['schema'] ?? '')); + $object = trim((string) ($definition['object'] ?? '')); + $target = trim((string) ($definition['target'] ?? '')); + $keys = array_values(array_filter(array_map('strval', (array) ($definition['keys'] ?? [])))); + $watermark = trim((string) ($definition['watermark'] ?? '')); + + foreach ([$source, $schema, $object, $target, ...$keys] as $identifier) { + if (preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/', $identifier) !== 1) { + throw new InvalidArgumentException('业务镜像数据集包含不安全的标识符：' . $key); + } + } + + if ($keys === [] || strlen($target) > 64) { + throw new InvalidArgumentException('业务镜像数据集缺少稳定键或目标表名过长：' . $key); + } + + if ($watermark !== '' && preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/', $watermark) !== 1) { + throw new InvalidArgumentException('业务镜像数据集的增量字段不安全：' . $key); + } + + return new readonly class($key, $source, $schema, $object, $target, $keys, $watermark ?: null) implements BusinessMirrorDataset { + public function __construct( + private string $key, + private string $source, + private string $schema, + private string $object, + private string $target, + private array $keys, + private ?string $watermark, + ) { + } + + public function key(): string { return $this->key; } + public function sourceKey(): string { return $this->source; } + public function sourceSchema(): string { return $this->schema; } + public function sourceObject(): string { return $this->object; } + public function targetTable(): string { return $this->target; } + public function keyColumns(): array { return $this->keys; } + public function watermarkColumn(): ?string { return $this->watermark; } + }; + } +} diff --git a/modules/EcommerceFinance/Services/BusinessMirrorSchemaManager.php b/modules/EcommerceFinance/Services/BusinessMirrorSchemaManager.php new file mode 100644 index 00000000..5608d1fd --- /dev/null +++ b/modules/EcommerceFinance/Services/BusinessMirrorSchemaManager.php @@ -0,0 +1,145 @@ +> */ + public function ensure(BusinessMirrorDataset $dataset): array + { + $columns = $this->columns($dataset); + $connection = (string) config('upstream_mirrors.connection', 'business_mirror'); + $schema = Schema::connection($connection); + + if (! $schema->hasTable($dataset->targetTable())) { + $schema->create($dataset->targetTable(), function (Blueprint $table) use ($columns, $dataset): void { + $table->char('_mirror_key', 64)->primary(); + + foreach ($columns as $column) { + $this->addColumn($table, $column); + } + + $table->uuid('_mirror_generation')->nullable()->index(); + $table->dateTime('_mirror_synced_at', 6)->index(); + + $watermark = $dataset->watermarkColumn(); + if ($watermark !== null && $this->isIndexable($columns, $watermark)) { + $table->index($watermark, $this->indexName($dataset->targetTable(), 'watermark')); + } + }); + + return $columns; + } + + $existing = array_map('strtolower', $schema->getColumnListing($dataset->targetTable())); + $missing = array_values(array_filter( + $columns, + fn (array $column): bool => ! in_array(strtolower((string) $column['column_name']), $existing, true), + )); + + if ($missing !== []) { + $schema->table($dataset->targetTable(), function (Blueprint $table) use ($missing): void { + foreach ($missing as $column) { + $this->addColumn($table, $column); + } + }); + } + + return $columns; + } + + /** @return list> */ + public function columns(BusinessMirrorDataset $dataset): array + { + $rows = DB::table('upstream_schema_columns') + ->where('source_key', $dataset->sourceKey()) + ->whereRaw('LOWER(schema_name) = ?', [strtolower($dataset->sourceSchema())]) + ->whereRaw('LOWER(object_name) = ?', [strtolower($dataset->sourceObject())]) + ->orderBy('ordinal_position') + ->get() + ->map(fn (object $row): array => (array) $row) + ->all(); + + if ($rows === []) { + throw new RuntimeException('business_mirror_schema_catalog_missing:' . $dataset->key()); + } + + $names = array_map(fn (array $row): string => strtolower((string) $row['column_name']), $rows); + if (count($names) !== count(array_unique($names))) { + throw new RuntimeException('business_mirror_duplicate_column:' . $dataset->key()); + } + + foreach ($rows as $row) { + $name = (string) $row['column_name']; + if (strlen($name) > 64 || str_starts_with(strtolower($name), '_mirror_')) { + throw new RuntimeException('business_mirror_unsupported_column:' . $dataset->key()); + } + } + + foreach ([...$dataset->keyColumns(), $dataset->watermarkColumn()] as $required) { + if ($required !== null && ! in_array(strtolower($required), $names, true)) { + throw new RuntimeException('business_mirror_required_column_missing:' . $dataset->key()); + } + } + + return $rows; + } + + private function addColumn(Blueprint $table, array $column): void + { + $name = (string) $column['column_name']; + $type = strtolower((string) $column['data_type']); + $length = (int) ($column['max_length'] ?? 0); + $precision = min(max((int) ($column['numeric_precision'] ?? 20), 1), 65); + $scale = min(max((int) ($column['numeric_scale'] ?? 0), 0), min($precision, 30)); + + $definition = match ($type) { + 'bit' => $table->boolean($name), + 'tinyint' => $table->unsignedTinyInteger($name), + 'smallint' => $table->smallInteger($name), + 'int' => $table->integer($name), + 'bigint' => $table->bigInteger($name), + 'decimal', 'numeric', 'money', 'smallmoney' => $table->decimal($name, $precision, $scale), + 'float', 'real' => $table->double($name), + 'date' => $table->date($name), + 'datetime', 'datetime2', 'smalldatetime' => $table->dateTime($name, 6), + 'time' => $table->time($name, 6), + 'uniqueidentifier' => $table->char($name, 36), + 'binary', 'varbinary', 'image', 'timestamp', 'rowversion' => $table->binary($name), + 'char', 'nchar', 'varchar', 'nvarchar' => $length > 0 && $length <= 1024 + ? $table->string($name, $length) + : $table->longText($name), + default => $table->longText($name), + }; + + $definition->nullable(); + } + + private function isIndexable(array $columns, string $name): bool + { + foreach ($columns as $column) { + if (strcasecmp((string) $column['column_name'], $name) !== 0) { + continue; + } + + $type = strtolower((string) $column['data_type']); + $length = (int) ($column['max_length'] ?? 0); + + return ! in_array($type, ['text', 'ntext', 'xml', 'image', 'binary', 'varbinary'], true) + && (! in_array($type, ['char', 'nchar', 'varchar', 'nvarchar'], true) || ($length > 0 && $length <= 191)); + } + + return false; + } + + private function indexName(string $table, string $suffix): string + { + return substr($table . '_' . $suffix, 0, 64); + } +} diff --git a/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php b/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php new file mode 100644 index 00000000..8ef379b4 --- /dev/null +++ b/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php @@ -0,0 +1,263 @@ +datasets->get($datasetKey); + $state = $this->mirror->state($dataset); + + if (! $full && ($state['initial_completed_at'] ?? null) === null) { + return $this->result($dataset, 'requires_full_sync', 'incremental', 0, 0); + } + + $columns = $this->schemas->ensure($dataset); + + $mode = $full ? 'full' : 'incremental'; + $resume = is_array($state) + && ($state['run_mode'] ?? null) === $mode + && in_array($state['status'] ?? null, ['running', 'failed'], true) + && is_array($state['cursor'] ?? null); + $generation = $resume && is_string($state['generation'] ?? null) + ? (string) $state['generation'] + : (string) Str::uuid(); + [$windowStart, $windowEnd] = $this->window($dataset, $state, $mode, $resume); + $state = $this->mirror->startRun( + $dataset, + $mode, + $generation, + $windowStart?->format('Y-m-d H:i:s.u'), + $windowEnd?->format('Y-m-d H:i:s.u'), + $resume, + ); + $cursor = is_array($state['cursor'] ?? null) ? $state['cursor'] : null; + $highWatermark = is_array($state['high_watermark'] ?? null) ? $state['high_watermark'] : []; + $batchSize = min(max($batchSize ?? (int) config('upstream_mirrors.table_sync.batch_size', 500), 1), 2000); + $maxBatches = max($maxBatches ?? (int) config('upstream_mirrors.table_sync.max_batches_per_run', 2), 0); + $batches = 0; + $rows = (int) ($state['last_upserted_rows'] ?? 0); + + try { + while (true) { + $batch = $this->readBatch( + $dataset, + $columns, + $mode, + $cursor, + $highWatermark, + $windowStart, + $windowEnd, + $batchSize, + ); + + if ($batch === []) { + $this->mirror->complete($dataset, $mode, $generation, $cursor ?? $highWatermark, $rows); + + return $this->result($dataset, 'succeeded', $mode, $batches, $rows); + } + + $cursor = $this->cursor($dataset, $batch[array_key_last($batch)], $mode); + $rows += count($batch); + $batches++; + $this->mirror->storeBatch($dataset, $columns, $this->normalize($columns, $batch), $cursor, $generation, $rows); + + if (count($batch) < $batchSize) { + $this->mirror->complete($dataset, $mode, $generation, $cursor, $rows); + + return $this->result($dataset, 'succeeded', $mode, $batches, $rows); + } + + if ($maxBatches > 0 && $batches >= $maxBatches) { + return $this->result($dataset, 'running', $mode, $batches, $rows); + } + } + } catch (Throwable $exception) { + $this->mirror->fail($dataset, $exception::class); + throw $exception; + } + } + + /** @return list> */ + private function readBatch( + BusinessMirrorDataset $dataset, + array $columns, + string $mode, + ?array $cursor, + array $highWatermark, + ?Carbon $windowStart, + ?Carbon $windowEnd, + int $limit, + ): array { + $names = array_map(fn (array $column): string => (string) $column['column_name'], $columns); + $order = $this->orderColumns($dataset, $mode); + $where = []; + $bindings = []; + + if ($mode === 'incremental' && $dataset->watermarkColumn() !== null) { + $watermark = $this->quote($dataset->watermarkColumn()); + $where[] = $watermark . ' >= ? AND ' . $watermark . ' < ?'; + $bindings[] = $windowStart?->format('Y-m-d H:i:s.u'); + $bindings[] = $windowEnd?->format('Y-m-d H:i:s.u'); + } elseif ($mode === 'incremental' && $cursor === null && $highWatermark !== []) { + [$predicate, $values] = $this->keyset($dataset->keyColumns(), $highWatermark); + $where[] = $predicate; + array_push($bindings, ...$values); + } + + if ($cursor !== null) { + [$predicate, $values] = $this->keyset($order, $cursor); + $where[] = $predicate; + array_push($bindings, ...$values); + } + + $sql = 'SELECT TOP (' . $limit . ') ' + . implode(', ', array_map(fn (string $name): string => $this->quote($name), $names)) + . ' FROM ' . $this->quote($dataset->sourceSchema()) . '.' . $this->quote($dataset->sourceObject()) + . ($where === [] ? '' : ' WHERE (' . implode(') AND (', $where) . ')') + . ' ORDER BY ' . implode(', ', array_map(fn (string $name): string => $this->quote($name), $order)); + + return $this->queries->forSource($dataset->sourceKey())->select($sql, $bindings); + } + + /** @return array{0:string,1:list} */ + private function keyset(array $columns, array $cursor): array + { + $parts = []; + $bindings = []; + + foreach (array_values($columns) as $index => $column) { + $and = []; + for ($previous = 0; $previous < $index; $previous++) { + $name = $columns[$previous]; + $and[] = $this->quote($name) . ' = ?'; + $bindings[] = $this->cursorValue($cursor, $name); + } + + $and[] = $this->quote($column) . ' > ?'; + $bindings[] = $this->cursorValue($cursor, $column); + $parts[] = '(' . implode(' AND ', $and) . ')'; + } + + return ['(' . implode(' OR ', $parts) . ')', $bindings]; + } + + /** @return list */ + private function orderColumns(BusinessMirrorDataset $dataset, string $mode): array + { + $columns = $dataset->keyColumns(); + + if ($mode === 'incremental' && $dataset->watermarkColumn() !== null) { + array_unshift($columns, $dataset->watermarkColumn()); + } + + return array_values(array_unique($columns)); + } + + /** @return array */ + private function cursor(BusinessMirrorDataset $dataset, array $row, string $mode): array + { + $cursor = []; + foreach ($this->orderColumns($dataset, $mode) as $column) { + $cursor[$column] = $this->rowValue($row, $column); + } + + return $cursor; + } + + /** @return array{0:?Carbon,1:?Carbon} */ + private function window(BusinessMirrorDataset $dataset, ?array $state, string $mode, bool $resume): array + { + if ($mode === 'full' || $dataset->watermarkColumn() === null) { + return [null, null]; + } + + if ($resume && ($state['window_started_at'] ?? null) !== null && ($state['window_ends_at'] ?? null) !== null) { + return [Carbon::parse((string) $state['window_started_at']), Carbon::parse((string) $state['window_ends_at'])]; + } + + $overlap = max((int) config('upstream_mirrors.table_sync.overlap_hours', 48), 1); + $last = Carbon::parse((string) ($state['last_succeeded_at'] ?? now()->subHours($overlap))); + + return [$last->subHours($overlap), now()->addSecond()]; + } + + /** @return list> */ + private function normalize(array $columns, array $rows): array + { + $types = []; + foreach ($columns as $column) { + $types[strtolower((string) $column['column_name'])] = strtolower((string) $column['data_type']); + } + + return array_map(function (array $row) use ($types): array { + foreach ($row as $name => $value) { + if (! is_string($value)) { + continue; + } + + $type = $types[strtolower((string) $name)] ?? ''; + if ($type === 'date') { + $row[$name] = substr($value, 0, 10); + } elseif (in_array($type, ['datetime', 'datetime2', 'smalldatetime'], true)) { + $row[$name] = preg_replace('/(\.\d{6})\d+/', '$1', str_replace('T', ' ', $value)); + } + } + + return $row; + }, $rows); + } + + private function quote(string $identifier): string + { + if (preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/', $identifier) !== 1) { + throw new RuntimeException('business_mirror_invalid_identifier'); + } + + return '[' . $identifier . ']'; + } + + private function cursorValue(array $cursor, string $column): mixed + { + foreach ($cursor as $key => $value) { + if (strcasecmp((string) $key, $column) === 0) { + return $value; + } + } + + throw new RuntimeException('business_mirror_cursor_incomplete'); + } + + private function rowValue(array $row, string $column): mixed + { + foreach ($row as $key => $value) { + if (strcasecmp((string) $key, $column) === 0) { + return $value; + } + } + + throw new RuntimeException('business_mirror_row_key_missing'); + } + + private function result(BusinessMirrorDataset $dataset, string $status, string $mode, int $batches, int $rows): array + { + return ['status' => $status, 'mode' => $mode, 'batches' => $batches, 'rows' => $rows, 'dataset' => $dataset->key()]; + } +} diff --git a/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php new file mode 100644 index 00000000..be3455b2 --- /dev/null +++ b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php @@ -0,0 +1,223 @@ + [ + 'driver' => 'sqlite', + 'database' => ':memory:', + 'prefix' => '', + ], + 'upstream_mirrors.connection' => 'business_mirror', + 'upstream_mirrors.datasets' => [ + 'test.dbo.records' => [ + 'source_key' => 'test', + 'schema' => 'dbo', + 'object' => 'records', + 'target' => 'test_records', + 'keys' => ['Id'], + 'watermark' => 'updateTime', + ], + ], + 'upstream_mirrors.table_sync.overlap_hours' => 24, + ]); + DB::purge('business_mirror'); + + Schema::connection('business_mirror')->create('business_mirror_table_states', function (Blueprint $table): void { + $table->string('dataset_key')->primary(); + $table->string('source_key'); + $table->string('source_object'); + $table->string('target_table'); + $table->string('status')->default('pending'); + $table->string('run_mode')->nullable(); + $table->string('generation')->nullable(); + $table->json('cursor')->nullable(); + $table->json('high_watermark')->nullable(); + $table->dateTime('window_started_at')->nullable(); + $table->dateTime('window_ends_at')->nullable(); + $table->dateTime('initial_completed_at')->nullable(); + $table->dateTime('last_full_succeeded_at')->nullable(); + $table->dateTime('last_attempted_at')->nullable(); + $table->dateTime('last_succeeded_at')->nullable(); + $table->unsignedBigInteger('last_upserted_rows')->default(0); + $table->unsignedBigInteger('mirror_row_count')->default(0); + $table->string('last_error_code')->nullable(); + $table->timestamps(); + }); + + DB::table('upstream_schema_columns')->insert([ + $this->column('Id', 1, 'int', null), + $this->column('updateTime', 2, 'datetime2', null), + $this->column('displayName', 3, 'nvarchar', 120), + ]); + + $this->query = new FakeBusinessMirrorQuery(); + $factory = new class($this->query) extends UpstreamReadOnlyQueryFactory { + public function __construct(private readonly UpstreamReadOnlyQuery $query) {} + public function forSource(string $source): UpstreamReadOnlyQuery { return $this->query; } + }; + $registry = new BusinessMirrorDatasetRegistry(); + $this->synchronizer = new BusinessMirrorSynchronizer( + $registry, + new BusinessMirrorSchemaManager(), + new GenericBusinessMirrorRepository(app('db')), + $factory, + ); + } + + public function test_full_sync_materializes_complete_table_and_marks_state_ready(): void + { + $this->query->batches = [[ + ['Id' => 1, 'updateTime' => '2026-08-14 10:00:00.1234567', 'displayName' => '一号'], + ['Id' => 2, 'updateTime' => '2026-08-14 11:00:00.0000000', 'displayName' => '二号'], + ]]; + + $result = $this->synchronizer->sync('test.dbo.records', true, 10, 1); + + $this->assertSame('succeeded', $result['status']); + $this->assertSame(2, DB::connection('business_mirror')->table('test_records')->count()); + $this->assertSame( + '2026-08-14 10:00:00.123456', + DB::connection('business_mirror')->table('test_records')->where('Id', 1)->value('updateTime'), + ); + $state = DB::connection('business_mirror')->table('business_mirror_table_states')->first(); + $this->assertSame('succeeded', $state->status); + $this->assertNotNull($state->initial_completed_at); + $this->assertSame(2, (int) $state->mirror_row_count); + $this->assertStringContainsString('SELECT TOP (10)', $this->query->calls[0]['sql']); + $this->assertStringContainsString('ORDER BY [Id]', $this->query->calls[0]['sql']); + } + + public function test_incremental_sync_uses_overlap_window_and_updates_existing_key(): void + { + $this->query->batches = [[ + ['Id' => 1, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => '旧值'], + ]]; + $this->synchronizer->sync('test.dbo.records', true, 10, 1); + + $this->query->calls = []; + $this->query->batches = [[ + ['Id' => 1, 'updateTime' => '2026-08-14 12:00:00', 'displayName' => '新值'], + ['Id' => 2, 'updateTime' => '2026-08-14 12:30:00', 'displayName' => '新增'], + ]]; + + $result = $this->synchronizer->sync('test.dbo.records', false, 10, 1); + + $this->assertSame('succeeded', $result['status']); + $this->assertSame(2, DB::connection('business_mirror')->table('test_records')->count()); + $this->assertSame('新值', DB::connection('business_mirror')->table('test_records')->where('Id', 1)->value('displayName')); + $this->assertStringContainsString('[updateTime] >= ? AND [updateTime] < ?', $this->query->calls[0]['sql']); + $this->assertStringContainsString('ORDER BY [updateTime], [Id]', $this->query->calls[0]['sql']); + $this->assertCount(2, $this->query->calls[0]['bindings']); + } + + public function test_incremental_sync_refuses_to_guess_before_first_full_sync(): void + { + $result = $this->synchronizer->sync('test.dbo.records', false, 10, 1); + + $this->assertSame('requires_full_sync', $result['status']); + $this->assertSame([], $this->query->calls); + } + + public function test_empty_full_read_does_not_delete_previous_mirror_rows(): void + { + $this->query->batches = [[ + ['Id' => 1, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => '保留'], + ]]; + $this->synchronizer->sync('test.dbo.records', true, 10, 1); + + $this->query->batches = [[]]; + + try { + $this->synchronizer->sync('test.dbo.records', true, 10, 1); + $this->fail('空的上游响应不应覆盖已有镜像。'); + } catch (\RuntimeException $exception) { + $this->assertSame('business_mirror_unexpected_empty_source', $exception->getMessage()); + } + + $this->assertSame(1, DB::connection('business_mirror')->table('test_records')->count()); + $this->assertSame('保留', DB::connection('business_mirror')->table('test_records')->value('displayName')); + $this->assertSame( + 'failed', + DB::connection('business_mirror')->table('business_mirror_table_states')->value('status'), + ); + } + + public function test_project_dataset_registry_has_unique_safe_targets(): void + { + $configured = require base_path('config/upstream_mirrors.php'); + config(['upstream_mirrors.datasets' => $configured['datasets']]); + $datasets = (new BusinessMirrorDatasetRegistry())->all(); + $targets = array_map(fn ($dataset): string => $dataset->targetTable(), $datasets); + + $this->assertCount(38, $datasets); + $this->assertCount(count($targets), array_unique($targets)); + $this->assertContains('operations_replica', array_map(fn ($dataset): string => $dataset->sourceKey(), $datasets)); + $this->assertContains('jiansun', array_map(fn ($dataset): string => $dataset->sourceKey(), $datasets)); + $this->assertNotContains('Bulk_Price_Limit', array_map(fn ($dataset): string => $dataset->sourceObject(), $datasets)); + } + + /** @return array */ + private function column(string $name, int $position, string $type, ?int $length): array + { + $now = now(); + + return [ + 'source_key' => 'test', + 'schema_name' => 'dbo', + 'object_name' => 'records', + 'object_type' => 'table', + 'column_name' => $name, + 'ordinal_position' => $position, + 'data_type' => $type, + 'max_length' => $length, + 'numeric_precision' => null, + 'numeric_scale' => null, + 'is_nullable' => true, + 'synchronized_at' => $now, + 'created_at' => $now, + 'updated_at' => $now, + ]; + } +} + +class FakeBusinessMirrorQuery extends UpstreamReadOnlyQuery +{ + /** @var list>> */ + public array $batches = []; + + /** @var list */ + public array $calls = []; + + public function __construct() + { + parent::__construct('test'); + } + + public function select(string $sql, array $bindings = []): array + { + $this->calls[] = compact('sql', 'bindings'); + + return array_shift($this->batches) ?? []; + } +} From bb9a18eddb72100cd221d2f303e18ef1b51196ff Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Fri, 14 Aug 2026 16:50:34 +0800 Subject: [PATCH 2/9] =?UTF-8?q?=E5=85=AC=E5=8F=B8=E5=BA=97=E9=93=BA?= =?UTF-8?q?=E5=95=86=E6=A0=87=E9=A1=B5=E9=9D=A2=E5=88=87=E6=8D=A2=E5=85=B1?= =?UTF-8?q?=E4=BA=AB=E4=B8=9A=E5=8A=A1=E9=95=9C=E5=83=8F=20(#54)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/Business/UpstreamBusinessInformation.php | 36 ++++- docs/上游业务镜像清单.md | 2 +- .../BusinessInformationMirrorRepository.php | 150 ++++++++++++++++++ ...usinessInformationMirrorRepositoryTest.php | 125 +++++++++++++++ .../UpstreamBusinessInformationTest.php | 44 ++++- 5 files changed, 348 insertions(+), 9 deletions(-) create mode 100644 modules/EcommerceFinance/Repositories/BusinessInformationMirrorRepository.php create mode 100644 tests/Unit/EcommerceFinance/BusinessInformationMirrorRepositoryTest.php diff --git a/app/Business/UpstreamBusinessInformation.php b/app/Business/UpstreamBusinessInformation.php index 53a9a4b9..9598170b 100644 --- a/app/Business/UpstreamBusinessInformation.php +++ b/app/Business/UpstreamBusinessInformation.php @@ -4,6 +4,7 @@ namespace App\Business; use Illuminate\Pagination\LengthAwarePaginator; use InvalidArgumentException; +use Modules\EcommerceFinance\Repositories\BusinessInformationMirrorRepository; use Modules\EcommerceFinance\Services\UpstreamQueryCacheFactory; use Modules\EcommerceFinance\Services\UpstreamReadOnlyQuery; @@ -11,9 +12,10 @@ class UpstreamBusinessInformation { private const SOURCE = 'operations_replica'; - public function __construct(private readonly UpstreamQueryCacheFactory $caches) - { - } + public function __construct( + private readonly UpstreamQueryCacheFactory $caches, + private readonly BusinessInformationMirrorRepository $mirror, + ) {} public function paginate(string $section, string $page, int $perPage = 50): LengthAwarePaginator { @@ -64,7 +66,7 @@ class UpstreamBusinessInformation private function companyPage(int $page, int $perPage): array { - return $this->remember( + return $this->page( 'company-information.read', $page, $perPage, @@ -108,7 +110,7 @@ SQL); private function storePage(int $page, int $perPage): array { - return $this->remember( + return $this->page( 'store-information.read', $page, $perPage, @@ -156,7 +158,7 @@ SQL); private function trademarkPage(int $page, int $perPage): array { - return $this->remember( + return $this->page( 'trademark-information.read', $page, $perPage, @@ -191,8 +193,12 @@ SQL); ]; } - private function remember(string $scope, int $page, int $perPage, callable $resolver): array + private function page(string $scope, int $page, int $perPage, callable $resolver): array { + if ($this->mirror->ready($scope)) { + return $this->normalize($scope, $this->mirror->paginate($scope, $page, $perPage)); + } + $cache = $this->caches->forSource(self::SOURCE); return $cache->remember($scope, compact('page', 'perPage'), function () use ($resolver) { @@ -200,6 +206,22 @@ SQL); }); } + /** @param array{data: array>, total: int} $payload */ + private function normalize(string $scope, array $payload): array + { + $mapper = match ($scope) { + 'company-information.read' => fn (array $row): array => $this->mapCompany($row), + 'store-information.read' => fn (array $row): array => $this->mapStore($row), + 'trademark-information.read' => fn (array $row): array => $this->mapTrademark($row), + default => throw new InvalidArgumentException('未知的经营分析副本库资料类型：' . $scope), + }; + + return [ + 'data' => array_map($mapper, $payload['data']), + 'total' => (int) $payload['total'], + ]; + } + private function mapCompany(array $row): array { $row = $this->toUtf8($row); diff --git a/docs/上游业务镜像清单.md b/docs/上游业务镜像清单.md index 0cc43440..41ee91ea 100644 --- a/docs/上游业务镜像清单.md +++ b/docs/上游业务镜像清单.md @@ -12,7 +12,7 @@ | --- | --- | | 已完成 | `operations_replica/rpt.Bulk_Price_Limit` 已进入类型化表 `product_price_mirror_rows`，商品价格页面已读取镜像并支持完整分页。 | | 本次建立 | `config/upstream_mirrors.php` 中登记全部当前代码引用的原表；通用表级同步框架负责建表、稳定游标、断点续传、增量重叠和全量校准。 | -| 待逐域切换 | 其他业务 Repository 仍需从上游 SQL Server 查询改为读取对应镜像表。切换前保留旧快照作为回滚，不得宣称已经完成。 | +| 已开始逐域切换 | 公司、店铺、商标页面在对应镜像完成首次全量后自动改读完整镜像表；首次全量前保留旧快照作为兼容回滚。其他业务域仍需逐项切换，不得宣称全项目已经完成。 | ## 经营分析副本库（operations_replica） diff --git a/modules/EcommerceFinance/Repositories/BusinessInformationMirrorRepository.php b/modules/EcommerceFinance/Repositories/BusinessInformationMirrorRepository.php new file mode 100644 index 00000000..26bcbbda --- /dev/null +++ b/modules/EcommerceFinance/Repositories/BusinessInformationMirrorRepository.php @@ -0,0 +1,150 @@ + 'operations_replica.rpt.fys_companyInfo', + 'store-information.read' => 'operations_replica.rpt.fys_shopInfo', + 'trademark-information.read' => 'operations_replica.rpt.fys_TrademarkInfo', + ]; + + private readonly ConnectionInterface $connection; + + public function __construct( + private readonly BusinessMirrorDatasetRegistry $datasets, + private readonly GenericBusinessMirrorRepository $mirror, + ) { + $this->connection = $mirror->connection(); + } + + public function ready(string $scope): bool + { + return $this->mirror->ready($this->dataset($scope)); + } + + /** @return array{data: array>, total: int} */ + public function paginate(string $scope, int $page, int $perPage): array + { + if (! $this->ready($scope)) { + throw new RuntimeException('business_mirror_dataset_not_ready'); + } + + $page = max($page, 1); + $perPage = min(max($perPage, 1), 100); + + return match ($scope) { + 'company-information.read' => $this->companies($page, $perPage), + 'store-information.read' => $this->stores($page, $perPage), + 'trademark-information.read' => $this->trademarks($page, $perPage), + default => throw new InvalidArgumentException('未知的业务资料镜像范围：' . $scope), + }; + } + + /** @return array{data: array>, total: int} */ + private function companies(int $page, int $perPage): array + { + $query = $this->connection->table('ops_fys_company_info') + ->selectRaw(<<<'SQL' +Id AS id, +unifiedSocialCreditCode AS unified_social_credit_code, +businessLicenseName AS company_name, +registeredCapital AS subscribed_capital, +businessLicenseType AS company_type, +IncorporationDate AS establishment_date, +legalRepresentative AS legal_representative, +registeredAddress AS registered_address, +businessScope AS business_scope, +accountSerialNumber AS basic_deposit_account_number, +companyStatus AS company_status, +financialManager AS financial_officer, +supervisor AS supervisor, +taxCategoryAndType AS taxpayer_category, +NULL AS shareholders +SQL); + + return $this->page($query, $page, $perPage, 'Id'); + } + + /** @return array{data: array>, total: int} */ + private function stores(int $page, int $perPage): array + { + $latestCompanies = $this->connection->table('ops_fys_company_info') + ->selectRaw('unifiedSocialCreditCode, MAX(Id) AS latest_id') + ->whereNotNull('unifiedSocialCreditCode') + ->where('unifiedSocialCreditCode', '<>', '') + ->groupBy('unifiedSocialCreditCode'); + + $query = $this->connection->table('ops_fys_shop_info AS s') + ->leftJoinSub($latestCompanies, 'company_keys', function ($join): void { + $join->on('company_keys.unifiedSocialCreditCode', '=', 's.unifiedSocialCreditCode'); + }) + ->leftJoin('ops_fys_company_info AS c', 'c.Id', '=', 'company_keys.latest_id') + ->selectRaw(<<<'SQL' +s.Id AS id, +s.openTime AS opened_at, +s.erpShopName AS erp_store_name, +s.unifiedSocialCreditCode AS unified_social_credit_code, +c.businessLicenseName AS company_name, +s.trademarkNumber AS trademark_number, +NULL AS trademark_name, +s.platform AS platform, +s.responsibleDepartment AS profit_center, +s.shopHead AS store_manager, +s.shopState AS store_status, +s.shopType AS store_type, +s.channelType AS channel_mode +SQL); + + return $this->page($query, $page, $perPage, 's.Id'); + } + + /** @return array{data: array>, total: int} */ + private function trademarks(int $page, int $perPage): array + { + $query = $this->connection->table('ops_fys_trademark_info') + ->where('isDelete', 0) + ->selectRaw(<<<'SQL' +Id AS id, +trademarkName AS trademark_name, +internationalClassification AS trademark_type, +trademarkStatus AS trademark_status, +registrationDate AS registration_date, +termOfValidity AS expiry_date, +remark AS notes +SQL); + + return $this->page($query, $page, $perPage, 'Id'); + } + + /** @return array{data: array>, total: int} */ + private function page($query, int $page, int $perPage, string $orderColumn): array + { + $total = (clone $query)->count(); + $rows = $query->orderByDesc($orderColumn) + ->forPage($page, $perPage) + ->get() + ->map(static fn (object $row): array => (array) $row) + ->all(); + + return ['data' => $rows, 'total' => $total]; + } + + private function dataset(string $scope): BusinessMirrorDataset + { + $key = self::DATASETS[$scope] ?? null; + + if ($key === null) { + throw new InvalidArgumentException('未知的业务资料镜像范围：' . $scope); + } + + return $this->datasets->get($key); + } +} diff --git a/tests/Unit/EcommerceFinance/BusinessInformationMirrorRepositoryTest.php b/tests/Unit/EcommerceFinance/BusinessInformationMirrorRepositoryTest.php new file mode 100644 index 00000000..d7cbdb57 --- /dev/null +++ b/tests/Unit/EcommerceFinance/BusinessInformationMirrorRepositoryTest.php @@ -0,0 +1,125 @@ + [ + 'driver' => 'sqlite', + 'database' => ':memory:', + 'prefix' => '', + ], + 'upstream_mirrors.connection' => 'business_mirror', + 'upstream_mirrors.datasets' => [ + 'operations_replica.rpt.fys_companyInfo' => $this->definition('fys_companyInfo', 'ops_fys_company_info'), + 'operations_replica.rpt.fys_shopInfo' => $this->definition('fys_shopInfo', 'ops_fys_shop_info'), + 'operations_replica.rpt.fys_TrademarkInfo' => $this->definition('fys_TrademarkInfo', 'ops_fys_trademark_info'), + ], + ]); + DB::purge('business_mirror'); + + $this->createTables(); + $mirror = new GenericBusinessMirrorRepository(app('db')); + $this->repository = new BusinessInformationMirrorRepository( + new BusinessMirrorDatasetRegistry(), + $mirror, + ); + } + + public function test_it_reads_all_three_ready_mirrors_with_pagination_and_business_filters(): void + { + $connection = DB::connection('business_mirror'); + $connection->table('ops_fys_company_info')->insert([ + $this->company(1, '统一代码', '旧公司'), + $this->company(2, '统一代码', '新公司'), + $this->company(3, '其他代码', '其他公司'), + ]); + $connection->table('ops_fys_shop_info')->insert([ + ['Id' => 8, 'openTime' => '2026-08-14 10:00:00', 'erpShopName' => '测试店', 'unifiedSocialCreditCode' => '统一代码', 'trademarkNumber' => 'TM', 'platform' => '天猫', 'responsibleDepartment' => '电商', 'shopHead' => '负责人', 'shopState' => '0', 'shopType' => '直营', 'channelType' => '线上'], + ]); + $connection->table('ops_fys_trademark_info')->insert([ + ['Id' => 5, 'trademarkName' => '有效商标', 'internationalClassification' => '20', 'trademarkStatus' => '1', 'registrationDate' => '2020-01-01', 'termOfValidity' => '2030-01-01', 'remark' => null, 'isDelete' => 0], + ['Id' => 6, 'trademarkName' => '已删除', 'internationalClassification' => '20', 'trademarkStatus' => '0', 'registrationDate' => null, 'termOfValidity' => null, 'remark' => null, 'isDelete' => 1], + ]); + $this->markReady(); + + $companies = $this->repository->paginate('company-information.read', 2, 2); + $store = $this->repository->paginate('store-information.read', 1, 20); + $trademarks = $this->repository->paginate('trademark-information.read', 1, 20); + + $this->assertSame(3, $companies['total']); + $this->assertSame(1, $companies['data'][0]['id']); + $this->assertSame('新公司', $store['data'][0]['company_name']); + $this->assertSame(1, $trademarks['total']); + $this->assertSame('有效商标', $trademarks['data'][0]['trademark_name']); + } + + public function test_dataset_is_not_ready_until_initial_full_sync_is_marked_complete(): void + { + $this->assertFalse($this->repository->ready('company-information.read')); + $this->markReady(['operations_replica.rpt.fys_companyInfo']); + $this->assertTrue($this->repository->ready('company-information.read')); + } + + private function createTables(): void + { + $schema = Schema::connection('business_mirror'); + $schema->create('business_mirror_table_states', function (Blueprint $table): void { + $table->string('dataset_key')->primary(); + $table->dateTime('initial_completed_at')->nullable(); + }); + $schema->create('ops_fys_company_info', function (Blueprint $table): void { + $table->integer('Id'); $table->string('unifiedSocialCreditCode')->nullable(); $table->string('businessLicenseName')->nullable(); + $table->string('registeredCapital')->nullable(); $table->string('businessLicenseType')->nullable(); $table->dateTime('IncorporationDate')->nullable(); + $table->string('legalRepresentative')->nullable(); $table->string('registeredAddress')->nullable(); $table->text('businessScope')->nullable(); + $table->string('accountSerialNumber')->nullable(); $table->string('companyStatus')->nullable(); $table->string('financialManager')->nullable(); + $table->string('supervisor')->nullable(); $table->string('taxCategoryAndType')->nullable(); + }); + $schema->create('ops_fys_shop_info', function (Blueprint $table): void { + $table->integer('Id'); $table->dateTime('openTime')->nullable(); $table->string('erpShopName')->nullable(); $table->string('unifiedSocialCreditCode')->nullable(); + $table->string('trademarkNumber')->nullable(); $table->string('platform')->nullable(); $table->string('responsibleDepartment')->nullable(); + $table->string('shopHead')->nullable(); $table->string('shopState')->nullable(); $table->string('shopType')->nullable(); $table->string('channelType')->nullable(); + }); + $schema->create('ops_fys_trademark_info', function (Blueprint $table): void { + $table->integer('Id'); $table->string('trademarkName')->nullable(); $table->string('internationalClassification')->nullable(); + $table->string('trademarkStatus')->nullable(); $table->dateTime('registrationDate')->nullable(); $table->dateTime('termOfValidity')->nullable(); + $table->text('remark')->nullable(); $table->integer('isDelete')->default(0); + }); + } + + private function markReady(?array $keys = null): void + { + $keys ??= array_keys((array) config('upstream_mirrors.datasets')); + foreach ($keys as $key) { + DB::connection('business_mirror')->table('business_mirror_table_states')->insert([ + 'dataset_key' => $key, + 'initial_completed_at' => '2026-08-14 12:00:00', + ]); + } + } + + private function definition(string $object, string $target): array + { + return ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => $object, 'target' => $target, 'keys' => ['Id']]; + } + + private function company(int $id, string $code, string $name): array + { + return ['Id' => $id, 'unifiedSocialCreditCode' => $code, 'businessLicenseName' => $name, 'registeredCapital' => null, 'businessLicenseType' => null, 'IncorporationDate' => null, 'legalRepresentative' => null, 'registeredAddress' => null, 'businessScope' => null, 'accountSerialNumber' => null, 'companyStatus' => '0', 'financialManager' => null, 'supervisor' => null, 'taxCategoryAndType' => null]; + } +} diff --git a/tests/Unit/EcommerceFinance/UpstreamBusinessInformationTest.php b/tests/Unit/EcommerceFinance/UpstreamBusinessInformationTest.php index 9adc4839..f0da44f9 100644 --- a/tests/Unit/EcommerceFinance/UpstreamBusinessInformationTest.php +++ b/tests/Unit/EcommerceFinance/UpstreamBusinessInformationTest.php @@ -4,6 +4,8 @@ namespace Tests\Unit\EcommerceFinance; use App\Business\UpstreamBusinessInformation; use Closure; +use Mockery; +use Modules\EcommerceFinance\Repositories\BusinessInformationMirrorRepository; use Modules\EcommerceFinance\Services\UpstreamQueryCache; use Modules\EcommerceFinance\Services\UpstreamQueryCacheFactory; use Tests\TestCase; @@ -44,7 +46,10 @@ class UpstreamBusinessInformationTest extends TestCase } }; - $information = new class($caches) extends UpstreamBusinessInformation { + $mirror = Mockery::mock(BusinessInformationMirrorRepository::class); + $mirror->shouldReceive('ready')->once()->with($scope)->andReturnFalse(); + + $information = new class($caches, $mirror) extends UpstreamBusinessInformation { public array $snapshots = []; public function snapshot(string $scope, array $parameters): array @@ -65,6 +70,43 @@ class UpstreamBusinessInformationTest extends TestCase $this->assertSame($expected, $information->snapshots[0]); } + /** + * @dataProvider businessInformationPages + */ + public function test_ready_business_mirror_is_used_without_registering_a_page_snapshot( + string $section, + string $page, + string $scope, + ): void { + $cache = new class extends UpstreamQueryCache { + public function __construct() {} + + public function remember(string $scope, array $parameters, Closure $resolver): mixed + { + throw new \RuntimeException('镜像就绪后不应再读取分页快照。'); + } + }; + $caches = new class($cache) extends UpstreamQueryCacheFactory { + public function __construct(private readonly UpstreamQueryCache $cache) {} + + public function forSource(string $source): UpstreamQueryCache { return $this->cache; } + }; + $mirror = Mockery::mock(BusinessInformationMirrorRepository::class); + $mirror->shouldReceive('ready')->once()->with($scope)->andReturnTrue(); + $mirror->shouldReceive('paginate')->once()->with($scope, 2, 25)->andReturn([ + 'data' => [['id' => 10]], + 'total' => 51, + ]); + + $this->app['request']->merge(['page' => 2]); + $paginator = (new UpstreamBusinessInformation($caches, $mirror)) + ->paginate($section, $page, 25); + + $this->assertSame(2, $paginator->currentPage()); + $this->assertSame(51, $paginator->total()); + $this->assertSame(10, $paginator->items()[0]->id); + } + public static function businessInformationPages(): array { return [ From 43445fde4b7c762765d7176468d66abdbdd23626 Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Fri, 14 Aug 2026 17:48:23 +0800 Subject: [PATCH 3/9] feat: read marketing bills from business mirror MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 首次全量完成后，营销账户账单、余额、汇总、选项及分页优先读取 caiwu_business_mirror；未就绪时保留旧快照链路。 --- app/Http/Controllers/Business/Pages.php | 239 +++++--- .../Repositories/AlibabaBillRepository.php | 50 +- .../Repositories/JdWalletBillRepository.php | 80 ++- .../MarketingAccountBillMirrorRepository.php | 559 ++++++++++++++++++ .../PinduoduoSpreadBillRepository.php | 63 +- .../TmallTaobaoSpreadBillRepository.php | 65 +- .../MarketingAccountBillsTabsTest.php | 109 +++- ...rketingAccountBillMirrorRepositoryTest.php | 209 +++++++ 8 files changed, 1241 insertions(+), 133 deletions(-) create mode 100644 modules/EcommerceFinance/Repositories/MarketingAccountBillMirrorRepository.php create mode 100644 tests/Unit/EcommerceFinance/MarketingAccountBillMirrorRepositoryTest.php diff --git a/app/Http/Controllers/Business/Pages.php b/app/Http/Controllers/Business/Pages.php index 62898918..2ee0f3c5 100644 --- a/app/Http/Controllers/Business/Pages.php +++ b/app/Http/Controllers/Business/Pages.php @@ -272,18 +272,22 @@ class Pages extends Controller : app(PinduoduoSpreadBillRepository::class); $scope = $isTaobao ? 'taobao-alipay-bills.list' : 'pinduoduo-bills.list'; $cache = $this->upstreamCaches->forSource('jiansun'); + $usingMirror = $repository->usesBusinessMirror(); try { - $result = $cache->remember( - $scope, - ['view' => 'monthly-balances', 'filters' => $filters, 'page' => $page, 'limit' => $pageSize], - fn () => $repository->paginateMonthlyBalances( - $filters['store_name'], - $filters['month'], - $page, - $pageSize, - ), + $resolver = fn () => $repository->paginateMonthlyBalances( + $filters['store_name'], + $filters['month'], + $page, + $pageSize, ); + $result = $usingMirror + ? $resolver() + : $cache->remember( + $scope, + ['view' => 'monthly-balances', 'filters' => $filters, 'page' => $page, 'limit' => $pageSize], + $resolver, + ); } catch (UpstreamSnapshotPendingException $exception) { $content['pending'] = true; $result = ['rows' => [], 'total' => 0]; @@ -297,11 +301,14 @@ class Pages extends Controller } try { - $storeOptions = $cache->remember( - $isTaobao ? 'taobao-alipay-bills.options' : 'pinduoduo-bills.options', - [], - fn () => $repository->storeOptions(), - ); + $resolver = fn () => $repository->storeOptions(); + $storeOptions = $usingMirror + ? $resolver() + : $cache->remember( + $isTaobao ? 'taobao-alipay-bills.options' : 'pinduoduo-bills.options', + [], + $resolver, + ); } catch (Throwable $exception) { report($exception); $storeOptions = []; @@ -352,11 +359,14 @@ class Pages extends Controller ], static fn (mixed $value): bool => $value !== ''); try { - $result = $this->upstreamCaches->forSource('jiansun')->remember( - $scope, - ['view' => 'monthly-balances', 'filters' => $filters, 'page' => 1, 'limit' => 5000, 'export' => true], - fn () => $repository->paginateMonthlyBalances($filters['store_name'], $filters['month'], 1, 5000), - ); + $resolver = fn () => $repository->paginateMonthlyBalances($filters['store_name'], $filters['month'], 1, 5000); + $result = $repository->usesBusinessMirror() + ? $resolver() + : $this->upstreamCaches->forSource('jiansun')->remember( + $scope, + ['view' => 'monthly-balances', 'filters' => $filters, 'page' => 1, 'limit' => 5000, 'export' => true], + $resolver, + ); } catch (ServiceUnavailableHttpException) { return redirect()->route('business.pages.show', $redirectParameters) ->with('error', '首份共享快照正在准备，系统已登记刷新请求；请稍后刷新页面后再导出。'); @@ -880,28 +890,38 @@ class Pages extends Controller ]; $repository = app(AlibabaBillRepository::class); $cache = $this->upstreamCaches->forSource('jiansun'); + $usingMirror = $repository->usesBusinessMirror(); try { - $result = $cache->remember( - 'marketing-account-bills.alibaba.list', - ['filters' => $filters, 'page' => $page, 'limit' => $pageSize], - fn () => $repository->paginate( - $filters['shop_name'], $filters['product_type'], - $filters['from'], $filters['to'], $page, $pageSize, - ), + $listResolver = fn () => $repository->paginate( + $filters['shop_name'], $filters['product_type'], + $filters['from'], $filters['to'], $page, $pageSize, ); - $summary = $cache->remember( - 'marketing-account-bills.alibaba.summary', - ['filters' => $filters], - fn () => $repository->summary( - $filters['shop_name'], $filters['product_type'], $filters['from'], $filters['to'], - ), - ); - $options = $cache->remember( - 'marketing-account-bills.alibaba.options', - [], - fn () => $repository->filterOptions(), + $summaryResolver = fn () => $repository->summary( + $filters['shop_name'], $filters['product_type'], $filters['from'], $filters['to'], ); + $optionsResolver = fn () => $repository->filterOptions(); + $result = $usingMirror + ? $listResolver() + : $cache->remember( + 'marketing-account-bills.alibaba.list', + ['filters' => $filters, 'page' => $page, 'limit' => $pageSize], + $listResolver, + ); + $summary = $usingMirror + ? $summaryResolver() + : $cache->remember( + 'marketing-account-bills.alibaba.summary', + ['filters' => $filters], + $summaryResolver, + ); + $options = $usingMirror + ? $optionsResolver() + : $cache->remember( + 'marketing-account-bills.alibaba.options', + [], + $optionsResolver, + ); } catch (ServiceUnavailableHttpException $exception) { return [ 'view' => 'banking.marketing_alibaba_bills.partials.embedded-table', @@ -966,33 +986,43 @@ class Pages extends Controller 'flow_type' => $flowType !== '' ? $flowType : null, ]; - $result = $cache->remember( - 'taobao-alipay-bills.list', - ['filters' => $filters, 'page' => $page, 'limit' => $pageSize], - fn () => $repository->paginate( - $filters['store_name'], - $filters['from'], - $filters['to'], - $filters['flow_type'], - $page, - $pageSize - ) + $listResolver = fn () => $repository->paginate( + $filters['store_name'], + $filters['from'], + $filters['to'], + $filters['flow_type'], + $page, + $pageSize ); - $summary = $cache->remember( - 'taobao-alipay-bills.summary', - ['filters' => $filters], - fn () => $repository->summary( - $filters['store_name'], - $filters['from'], - $filters['to'], - $filters['flow_type'] - ) - ); - $storeOptions = $cache->remember( - 'taobao-alipay-bills.options', - [], - fn () => $repository->storeOptions() + $summaryResolver = fn () => $repository->summary( + $filters['store_name'], + $filters['from'], + $filters['to'], + $filters['flow_type'] ); + $optionsResolver = fn () => $repository->storeOptions(); + $usingMirror = $repository->usesBusinessMirror(); + $result = $usingMirror + ? $listResolver() + : $cache->remember( + 'taobao-alipay-bills.list', + ['filters' => $filters, 'page' => $page, 'limit' => $pageSize], + $listResolver, + ); + $summary = $usingMirror + ? $summaryResolver() + : $cache->remember( + 'taobao-alipay-bills.summary', + ['filters' => $filters], + $summaryResolver, + ); + $storeOptions = $usingMirror + ? $optionsResolver() + : $cache->remember( + 'taobao-alipay-bills.options', + [], + $optionsResolver, + ); $bills = new LengthAwarePaginator( Collection::make($result['rows']), @@ -1041,34 +1071,43 @@ class Pages extends Controller ]; // 页面只读取共享查询快照；冷查询由统一限流和熔断链路访问上游。 - $result = $cache->remember( - 'pinduoduo-bills.list', - ['filters' => $filters, 'page' => $page, 'limit' => $pageSize], - fn () => $repository->paginate( - $filters['store_name'], - $filters['from'], - $filters['to'], - $filters['flow_type'], - $page, - $pageSize - ) + $listResolver = fn () => $repository->paginate( + $filters['store_name'], + $filters['from'], + $filters['to'], + $filters['flow_type'], + $page, + $pageSize ); - $summary = $cache->remember( - 'pinduoduo-bills.summary', - ['filters' => $filters], - fn () => $repository->summary( - $filters['store_name'], - $filters['from'], - $filters['to'], - $filters['flow_type'] - ) - ); - - $storeOptions = $cache->remember( - 'pinduoduo-bills.options', - [], - fn () => $repository->storeOptions() + $summaryResolver = fn () => $repository->summary( + $filters['store_name'], + $filters['from'], + $filters['to'], + $filters['flow_type'] ); + $optionsResolver = fn () => $repository->storeOptions(); + $usingMirror = $repository->usesBusinessMirror(); + $result = $usingMirror + ? $listResolver() + : $cache->remember( + 'pinduoduo-bills.list', + ['filters' => $filters, 'page' => $page, 'limit' => $pageSize], + $listResolver, + ); + $summary = $usingMirror + ? $summaryResolver() + : $cache->remember( + 'pinduoduo-bills.summary', + ['filters' => $filters], + $summaryResolver, + ); + $storeOptions = $usingMirror + ? $optionsResolver() + : $cache->remember( + 'pinduoduo-bills.options', + [], + $optionsResolver, + ); // 包装成 Laravel 分页器，便于 Blade 用 $bills->links() $bills = new LengthAwarePaginator( @@ -1114,9 +1153,10 @@ class Pages extends Controller $cache = $this->upstreamCaches->forSource('jiansun'); $scopePrefix = 'jd-wallet-bills.' . $ledger; $errors = []; - $remember = function (string $scope, array $parameters, callable $resolver, mixed $fallback) use ($cache, &$errors): mixed { + $usingMirror = $repository->usesBusinessMirror($ledger); + $remember = function (string $scope, array $parameters, callable $resolver, mixed $fallback) use ($cache, $usingMirror, &$errors): mixed { try { - return $cache->remember($scope, $parameters, $resolver); + return $usingMirror ? $resolver() : $cache->remember($scope, $parameters, $resolver); } catch (ServiceUnavailableHttpException $exception) { $errors[] = $exception->getMessage(); } catch (Throwable $exception) { @@ -1145,12 +1185,19 @@ class Pages extends Controller : $repository->summaryRecharge($filters['store_name'], $filters['from'], $filters['to']), ['amount_total' => 0, 'balance_total' => 0, 'count' => 0], ); - $shopOptions = $remember( - 'jd-wallet-bills.options', - [], - fn () => $repository->shopOptions(), - [], - ); + try { + $optionsResolver = fn () => $repository->shopOptions(); + $shopOptions = $repository->usesBusinessMirrorForOptions() + ? $optionsResolver() + : $cache->remember('jd-wallet-bills.options', [], $optionsResolver); + } catch (ServiceUnavailableHttpException $exception) { + $errors[] = $exception->getMessage(); + $shopOptions = []; + } catch (Throwable $exception) { + report($exception); + $errors[] = '京东账单数据暂时无法读取，请稍后重试。'; + $shopOptions = []; + } return [ 'view' => 'banking.marketing_jd_wallet_bills.partials.embedded-table', diff --git a/modules/EcommerceFinance/Repositories/AlibabaBillRepository.php b/modules/EcommerceFinance/Repositories/AlibabaBillRepository.php index f1dea8d9..cb5b8296 100644 --- a/modules/EcommerceFinance/Repositories/AlibabaBillRepository.php +++ b/modules/EcommerceFinance/Repositories/AlibabaBillRepository.php @@ -17,14 +17,25 @@ class AlibabaBillRepository private UpstreamReadOnlyQuery $query; - public function __construct(UpstreamReadOnlyQueryFactory $factory) - { + public function __construct( + UpstreamReadOnlyQueryFactory $factory, + private readonly ?MarketingAccountBillMirrorRepository $mirror = null, + ) { $this->query = $factory->forSource(self::SOURCE); } + public function usesBusinessMirror(): bool + { + return $this->mirror?->ready(MarketingAccountBillMirrorRepository::ALIBABA) ?? false; + } + /** @return array{shops: string[], product_types: string[]} */ public function filterOptions(): array { + if ($this->usesBusinessMirror()) { + return $this->mirror->alibabaFilterOptions(); + } + $shops = $this->query->select( 'SELECT DISTINCT TOP (500) shopName FROM ' . self::TABLE . ' WHERE shopName IS NOT NULL AND LTRIM(RTRIM(shopName)) <> \'\' ORDER BY shopName' @@ -51,6 +62,23 @@ class AlibabaBillRepository ): array { $page = max(1, $page); $pageSize = min(max(1, $pageSize), 100); + + if ($this->usesBusinessMirror()) { + $result = $this->mirror->paginateAlibaba( + $shopName, + $productType, + $fromDate, + $toDate, + $page, + $pageSize, + ); + + return [ + 'rows' => array_map(fn (array $row): array => $this->normalize($row), $result['rows']), + 'total' => $result['total'], + ]; + } + [$where, $bindings] = $this->where($shopName, $productType, $fromDate, $toDate); $total = (int) $this->query->scalar('SELECT COUNT(*) FROM ' . self::TABLE . ' src' . $where, $bindings); @@ -82,13 +110,17 @@ class AlibabaBillRepository ?string $fromDate = null, ?string $toDate = null, ): array { - [$where, $bindings] = $this->where($shopName, $productType, $fromDate, $toDate); - $row = $this->query->first( - 'SELECT COALESCE(SUM(tradeAMTIn), 0) AS income_total, ' - . 'COALESCE(SUM(tradeAMTOut), 0) AS expense_total, COUNT(*) AS cnt ' - . 'FROM ' . self::TABLE . ' src' . $where, - $bindings, - ) ?? []; + if ($this->usesBusinessMirror()) { + $row = $this->mirror->summarizeAlibaba($shopName, $productType, $fromDate, $toDate); + } else { + [$where, $bindings] = $this->where($shopName, $productType, $fromDate, $toDate); + $row = $this->query->first( + 'SELECT COALESCE(SUM(tradeAMTIn), 0) AS income_total, ' + . 'COALESCE(SUM(tradeAMTOut), 0) AS expense_total, COUNT(*) AS cnt ' + . 'FROM ' . self::TABLE . ' src' . $where, + $bindings, + ) ?? []; + } $income = round((float) ($row['income_total'] ?? 0), 2); $expense = round((float) ($row['expense_total'] ?? 0), 2); diff --git a/modules/EcommerceFinance/Repositories/JdWalletBillRepository.php b/modules/EcommerceFinance/Repositories/JdWalletBillRepository.php index 77d9f126..4c22d7d4 100644 --- a/modules/EcommerceFinance/Repositories/JdWalletBillRepository.php +++ b/modules/EcommerceFinance/Repositories/JdWalletBillRepository.php @@ -19,14 +19,34 @@ class JdWalletBillRepository private UpstreamReadOnlyQuery $query; - public function __construct(UpstreamReadOnlyQueryFactory $factory) - { + public function __construct( + UpstreamReadOnlyQueryFactory $factory, + private readonly ?MarketingAccountBillMirrorRepository $mirror = null, + ) { $this->query = $factory->forSource(self::SOURCE); } + public function usesBusinessMirror(string $ledger): bool + { + $dataset = $ledger === 'consumption' + ? MarketingAccountBillMirrorRepository::JD_CONSUMPTION + : MarketingAccountBillMirrorRepository::JD_RECHARGE; + + return $this->mirror?->ready($dataset) ?? false; + } + + public function usesBusinessMirrorForOptions(): bool + { + return $this->usesBusinessMirror('recharge') && $this->usesBusinessMirror('consumption'); + } + /** @return string[] */ public function shopOptions(): array { + if ($this->usesBusinessMirrorForOptions()) { + return $this->mirror->jdShopOptions(); + } + $rows = $this->query->select( "SELECT TOP (500) shopName FROM (SELECT shopName FROM " . self::RECHARGE_TABLE . " UNION SELECT shopName FROM " . self::CONSUMPTION_TABLE @@ -47,6 +67,21 @@ class JdWalletBillRepository int $page = 1, int $pageSize = 25, ): array { + if ($this->usesBusinessMirror('recharge')) { + $result = $this->mirror->paginateJdRecharge( + $shopName, + $fromDate, + $toDate, + $page, + $pageSize, + ); + + return [ + 'rows' => array_map(fn (array $row): array => $this->normalizeRecharge($row), $result['rows']), + 'total' => $result['total'], + ]; + } + return $this->paginate(self::RECHARGE_TABLE, $this->rechargeWhere($shopName, $fromDate, $toDate), $page, $pageSize, [ 'id', 'shopName', 'amount', 'balanceAmount', 'orderTypeName', 'pk', 'createDate', 'updateTime', 'statusName', 'remark', @@ -61,6 +96,21 @@ class JdWalletBillRepository int $page = 1, int $pageSize = 25, ): array { + if ($this->usesBusinessMirror('consumption')) { + $result = $this->mirror->paginateJdConsumption( + $shopName, + $fromDate, + $toDate, + $page, + $pageSize, + ); + + return [ + 'rows' => array_map(fn (array $row): array => $this->normalizeConsumption($row), $result['rows']), + 'total' => $result['total'], + ]; + } + return $this->paginate(self::CONSUMPTION_TABLE, $this->consumptionWhere($shopName, $fromDate, $toDate), $page, $pageSize, [ 'id', 'shopName', 'amount', 'balanceAmount', 'orderTypeName', 'creatTime', 'settleTime', 'pk', 'invoiceApplyTypeName', @@ -70,12 +120,32 @@ class JdWalletBillRepository /** @return array{amount_total: float, balance_total: float, count: int} */ public function summaryRecharge(?string $shopName = null, ?string $fromDate = null, ?string $toDate = null): array { + if ($this->usesBusinessMirror('recharge')) { + return $this->normalizeSummary($this->mirror->summarizeJd( + MarketingAccountBillMirrorRepository::JD_RECHARGE, + 'createDate', + $shopName, + $fromDate, + $toDate, + )); + } + return $this->summary(self::RECHARGE_TABLE, $this->rechargeWhere($shopName, $fromDate, $toDate)); } /** @return array{amount_total: float, balance_total: float, count: int} */ public function summaryConsumption(?string $shopName = null, ?string $fromDate = null, ?string $toDate = null): array { + if ($this->usesBusinessMirror('consumption')) { + return $this->normalizeSummary($this->mirror->summarizeJd( + MarketingAccountBillMirrorRepository::JD_CONSUMPTION, + 'creatTime', + $shopName, + $fromDate, + $toDate, + )); + } + return $this->summary(self::CONSUMPTION_TABLE, $this->consumptionWhere($shopName, $fromDate, $toDate)); } @@ -111,6 +181,12 @@ class JdWalletBillRepository $where['bindings'], ) ?? []; + return $this->normalizeSummary($row); + } + + /** @return array{amount_total: float, balance_total: float, count: int} */ + private function normalizeSummary(array $row): array + { return [ 'amount_total' => round((float) ($row['amount_total'] ?? 0), 2), 'balance_total' => round((float) ($row['balance_total'] ?? 0), 2), diff --git a/modules/EcommerceFinance/Repositories/MarketingAccountBillMirrorRepository.php b/modules/EcommerceFinance/Repositories/MarketingAccountBillMirrorRepository.php new file mode 100644 index 00000000..d76d5dfb --- /dev/null +++ b/modules/EcommerceFinance/Repositories/MarketingAccountBillMirrorRepository.php @@ -0,0 +1,559 @@ + */ + private array $readiness = []; + + public function __construct( + private readonly BusinessMirrorDatasetRegistry $datasets, + private readonly GenericBusinessMirrorRepository $mirror, + ) { + $this->connection = $mirror->connection(); + } + + public function ready(string $datasetKey): bool + { + if (array_key_exists($datasetKey, $this->readiness)) { + return $this->readiness[$datasetKey]; + } + + try { + return $this->readiness[$datasetKey] = $this->mirror->ready($this->dataset($datasetKey)); + } catch (Throwable $exception) { + report($exception); + + return $this->readiness[$datasetKey] = false; + } + } + + /** @return string[] */ + public function taobaoStoreOptions(): array + { + return $this->stringOptions(self::TAOBAO, 'shopName'); + } + + /** @return array{rows:array>,total:int} */ + public function paginateTaobao( + ?string $storeName, + ?string $fromDate, + ?string $toDate, + ?string $flowType, + int $page, + int $pageSize, + ): array { + $query = $this->table(self::TAOBAO, 'src'); + $this->applyTaobaoFilters($query, $storeName, $fromDate, $toDate, $flowType); + + return $this->paginate($query, [ + 'src.id', 'src.shopName', 'src.uniTransTime', 'src.transTime', 'src.amount', + 'src.balance', 'src.comment', 'src.finType', 'src.bizType', 'src.tradeType', + ], ['src.uniTransTime', 'src.id'], $page, min(max($pageSize, 1), 200)); + } + + /** @return array{income_total_raw:float,expense_total_raw:float,cnt:int} */ + public function summarizeTaobao( + ?string $storeName, + ?string $fromDate, + ?string $toDate, + ?string $flowType, + ): array { + $query = $this->table(self::TAOBAO, 'src'); + $this->applyTaobaoFilters($query, $storeName, $fromDate, $toDate, $flowType); + $row = $query->selectRaw( + "COALESCE(SUM(CASE WHEN src.finType = 'DEBIT' THEN src.amount ELSE 0 END), 0) AS income_total_raw, " + . "COALESCE(SUM(CASE WHEN src.finType = 'CREDIT' THEN src.amount ELSE 0 END), 0) AS expense_total_raw, " + . 'COUNT(*) AS cnt' + )->first(); + + return [ + 'income_total_raw' => (float) ($row->income_total_raw ?? 0), + 'expense_total_raw' => (float) ($row->expense_total_raw ?? 0), + 'cnt' => (int) ($row->cnt ?? 0), + ]; + } + + /** @return array{rows:array>,total:int} */ + public function paginateTaobaoMonthlyBalances( + ?string $storeName, + ?string $month, + int $page, + int $pageSize, + ): array { + return $this->monthlyBalances( + self::TAOBAO, + 'transTime', + "CASE WHEN src.finType = 'DEBIT' THEN src.amount ELSE 0 END", + "CASE WHEN src.finType = 'CREDIT' THEN src.amount ELSE 0 END", + $storeName, + $month, + $page, + min(max($pageSize, 1), 5000), + 'opening_balance_raw', + 'income_total_raw', + 'expense_total_raw', + 'system_closing_balance_raw', + ); + } + + /** @return string[] */ + public function pinduoduoStoreOptions(): array + { + return $this->stringOptions(self::PINDUODUO, 'shopName'); + } + + /** @return array{rows:array>,total:int} */ + public function paginatePinduoduo( + ?string $storeName, + ?string $fromDate, + ?string $toDate, + ?string $flowType, + int $page, + int $pageSize, + ): array { + $query = $this->table(self::PINDUODUO, 'src'); + $this->applyPinduoduoFilters($query, $storeName, $fromDate, $toDate, $flowType); + + return $this->paginate($query, [ + 'src.id', 'src.shopName', 'src.amount', 'src.balance', 'src.summary', + 'src.flowType', 'src.toAdvertiserFundType', 'src.createTime', + ], ['src.createTime', 'src.id'], $page, min(max($pageSize, 1), 200)); + } + + /** @return array{income_total:float,expense_total:float,cnt:int} */ + public function summarizePinduoduo( + ?string $storeName, + ?string $fromDate, + ?string $toDate, + ?string $flowType, + ): array { + $query = $this->table(self::PINDUODUO, 'src'); + $this->applyPinduoduoFilters($query, $storeName, $fromDate, $toDate, $flowType); + $row = $query->selectRaw( + 'COALESCE(SUM(CASE WHEN src.flowType = 1 THEN src.amount ELSE 0 END), 0) AS income_total, ' + . 'COALESCE(SUM(CASE WHEN src.flowType = 2 THEN src.amount ELSE 0 END), 0) AS expense_total, ' + . 'COUNT(*) AS cnt' + )->first(); + + return [ + 'income_total' => (float) ($row->income_total ?? 0), + 'expense_total' => (float) ($row->expense_total ?? 0), + 'cnt' => (int) ($row->cnt ?? 0), + ]; + } + + /** @return array{rows:array>,total:int} */ + public function paginatePinduoduoMonthlyBalances( + ?string $storeName, + ?string $month, + int $page, + int $pageSize, + ): array { + return $this->monthlyBalances( + self::PINDUODUO, + 'createTime', + 'CASE WHEN src.flowType = 1 THEN src.amount ELSE 0 END', + 'CASE WHEN src.flowType = 2 THEN src.amount ELSE 0 END', + $storeName, + $month, + $page, + min(max($pageSize, 1), 5000), + 'opening_balance', + 'income_total', + 'expense_total', + 'system_closing_balance', + ); + } + + /** @return array{shops:string[],product_types:string[]} */ + public function alibabaFilterOptions(): array + { + return [ + 'shops' => $this->stringOptions(self::ALIBABA, 'shopName', 500), + 'product_types' => $this->stringOptions(self::ALIBABA, 'tradeTypeName', 500), + ]; + } + + /** @return array{rows:array>,total:int} */ + public function paginateAlibaba( + ?string $shopName, + ?string $productType, + ?string $fromDate, + ?string $toDate, + int $page, + int $pageSize, + ): array { + $query = $this->table(self::ALIBABA, 'src'); + $this->applyAlibabaFilters($query, $shopName, $productType, $fromDate, $toDate); + + return $this->paginate($query, [ + 'src.id', 'src.shopName', 'src.tradeDay', 'src.tradeTypeName', + 'src.tradeAMTIn', 'src.tradeAMTOut', + ], ['src.tradeDay', 'src.id'], $page, min(max($pageSize, 1), 100)); + } + + /** @return array{income_total:float,expense_total:float,cnt:int} */ + public function summarizeAlibaba( + ?string $shopName, + ?string $productType, + ?string $fromDate, + ?string $toDate, + ): array { + $query = $this->table(self::ALIBABA, 'src'); + $this->applyAlibabaFilters($query, $shopName, $productType, $fromDate, $toDate); + $row = $query->selectRaw( + 'COALESCE(SUM(src.tradeAMTIn), 0) AS income_total, ' + . 'COALESCE(SUM(src.tradeAMTOut), 0) AS expense_total, COUNT(*) AS cnt' + )->first(); + + return [ + 'income_total' => (float) ($row->income_total ?? 0), + 'expense_total' => (float) ($row->expense_total ?? 0), + 'cnt' => (int) ($row->cnt ?? 0), + ]; + } + + /** @return string[] */ + public function jdShopOptions(): array + { + $shops = array_merge( + $this->stringOptions(self::JD_RECHARGE, 'shopName', 500), + $this->stringOptions(self::JD_CONSUMPTION, 'shopName', 500), + ); + $shops = array_values(array_unique($shops)); + sort($shops, SORT_NATURAL); + + return array_slice($shops, 0, 500); + } + + /** @return array{rows:array>,total:int} */ + public function paginateJdRecharge( + ?string $shopName, + ?string $fromDate, + ?string $toDate, + int $page, + int $pageSize, + ): array { + return $this->paginateJd( + self::JD_RECHARGE, + 'createDate', + ['src.id', 'src.shopName', 'src.amount', 'src.balanceAmount', 'src.orderTypeName', + 'src.pk', 'src.createDate', 'src.updateTime', 'src.statusName', 'src.remark'], + $shopName, + $fromDate, + $toDate, + $page, + $pageSize, + ); + } + + /** @return array{rows:array>,total:int} */ + public function paginateJdConsumption( + ?string $shopName, + ?string $fromDate, + ?string $toDate, + int $page, + int $pageSize, + ): array { + return $this->paginateJd( + self::JD_CONSUMPTION, + 'creatTime', + ['src.id', 'src.shopName', 'src.amount', 'src.balanceAmount', 'src.orderTypeName', + 'src.creatTime', 'src.settleTime', 'src.pk', 'src.invoiceApplyTypeName'], + $shopName, + $fromDate, + $toDate, + $page, + $pageSize, + ); + } + + /** @return array{amount_total:float,balance_total:float,cnt:int} */ + public function summarizeJd( + string $datasetKey, + string $dateField, + ?string $shopName, + ?string $fromDate, + ?string $toDate, + ): array { + $query = $this->table($datasetKey, 'src'); + $this->applyJdFilters($query, $dateField, $shopName, $fromDate, $toDate); + $row = $query->selectRaw( + 'COALESCE(SUM(ABS(src.amount)), 0) AS amount_total, ' + . 'COALESCE(SUM(src.balanceAmount), 0) AS balance_total, COUNT(*) AS cnt' + )->first(); + + return [ + 'amount_total' => (float) ($row->amount_total ?? 0), + 'balance_total' => (float) ($row->balance_total ?? 0), + 'cnt' => (int) ($row->cnt ?? 0), + ]; + } + + /** @return array{rows:array>,total:int} */ + private function paginateJd( + string $datasetKey, + string $dateField, + array $fields, + ?string $shopName, + ?string $fromDate, + ?string $toDate, + int $page, + int $pageSize, + ): array { + $query = $this->table($datasetKey, 'src'); + $this->applyJdFilters($query, $dateField, $shopName, $fromDate, $toDate); + + return $this->paginate( + $query, + $fields, + ['src.' . $dateField, 'src.id'], + $page, + min(max($pageSize, 1), 100), + ); + } + + private function applyTaobaoFilters( + Builder $query, + ?string $storeName, + ?string $fromDate, + ?string $toDate, + ?string $flowType, + ): void { + $this->whereValue($query, 'src.shopName', $storeName); + $this->whereFromDate($query, 'src.transTime', $fromDate); + $this->whereToDate($query, 'src.transTime', $toDate); + if ($flowType === 'income') { + $query->where('src.finType', 'DEBIT'); + } elseif ($flowType === 'expense') { + $query->where('src.finType', 'CREDIT'); + } + } + + private function applyPinduoduoFilters( + Builder $query, + ?string $storeName, + ?string $fromDate, + ?string $toDate, + ?string $flowType, + ): void { + $this->whereValue($query, 'src.shopName', $storeName); + $this->whereFromDate($query, 'src.createTime', $fromDate, true); + $this->whereToDate($query, 'src.createTime', $toDate); + if ($flowType === 'income') { + $query->where('src.flowType', 1); + } elseif ($flowType === 'expense') { + $query->where('src.flowType', 2); + } + } + + private function applyAlibabaFilters( + Builder $query, + ?string $shopName, + ?string $productType, + ?string $fromDate, + ?string $toDate, + ): void { + $this->whereValue($query, 'src.shopName', $shopName); + $this->whereValue($query, 'src.tradeTypeName', $productType); + if ($this->validDate($fromDate)) { + $query->whereDate('src.tradeDay', '>=', $fromDate); + } + if ($this->validDate($toDate)) { + $query->whereDate('src.tradeDay', '<=', $toDate); + } + } + + private function applyJdFilters( + Builder $query, + string $dateField, + ?string $shopName, + ?string $fromDate, + ?string $toDate, + ): void { + $this->whereValue($query, 'src.shopName', $shopName); + $this->whereFromDate($query, 'src.' . $dateField, $fromDate, true); + $this->whereToDate($query, 'src.' . $dateField, $toDate); + } + + private function whereValue(Builder $query, string $column, ?string $value): void + { + if ($value !== null && $value !== '') { + $query->where($column, $value); + } + } + + private function whereFromDate(Builder $query, string $column, ?string $date, bool $midnight = false): void + { + if ($this->validDate($date)) { + $query->where($column, '>=', $date . ($midnight ? ' 00:00:00' : '')); + } + } + + private function whereToDate(Builder $query, string $column, ?string $date): void + { + if ($this->validDate($date)) { + $query->where($column, '<', date('Y-m-d', strtotime($date . ' +1 day')) . ' 00:00:00'); + } + } + + private function validDate(?string $date): bool + { + return $date !== null && preg_match('/^\d{4}-\d{2}-\d{2}$/', $date) === 1; + } + + /** @return string[] */ + private function stringOptions(string $datasetKey, string $column, ?int $limit = null): array + { + $query = $this->table($datasetKey) + ->whereNotNull($column) + ->where($column, '<>', '') + ->distinct() + ->orderBy($column); + if ($limit !== null) { + $query->limit($limit); + } + + return $query->pluck($column)->map(static fn (mixed $value): string => (string) $value)->all(); + } + + /** @return array{rows:array>,total:int} */ + private function paginate(Builder $query, array $fields, array $orderColumns, int $page, int $pageSize): array + { + $page = max($page, 1); + $total = (clone $query)->count(); + if ($total === 0) { + return ['rows' => [], 'total' => 0]; + } + + foreach ($orderColumns as $column) { + $query->orderByDesc($column); + } + $rows = $query->select($fields) + ->forPage($page, $pageSize) + ->get() + ->map(static fn (object $row): array => (array) $row) + ->all(); + + return ['rows' => $rows, 'total' => $total]; + } + + /** @return array{rows:array>,total:int} */ + private function monthlyBalances( + string $datasetKey, + string $dateField, + string $incomeExpression, + string $expenseExpression, + ?string $storeName, + ?string $month, + int $page, + int $pageSize, + string $openingAlias, + string $incomeAlias, + string $expenseAlias, + string $closingAlias, + ): array { + $page = max($page, 1); + $driver = $this->connection->getDriverName(); + $monthStart = $driver === 'sqlite' + ? "date(src.$dateField, 'start of month')" + : "DATE_FORMAT(src.$dateField, '%Y-%m-01')"; + $currentMonth = $driver === 'sqlite' + ? "date('now', 'start of month')" + : "DATE_FORMAT(CURRENT_DATE, '%Y-%m-01')"; + $nextMonth = $driver === 'sqlite' + ? "date(m.month_start, '+1 month')" + : 'DATE_ADD(m.month_start, INTERVAL 1 MONTH)'; + $lastDay = $driver === 'sqlite' + ? "date(m.month_start, '+1 month', '-1 day')" + : "DATE_FORMAT(LAST_DAY(m.month_start), '%Y-%m-%d')"; + $table = $this->dataset($datasetKey)->targetTable(); + + $groups = $this->connection->table($table . ' AS src') + ->whereNotNull('src.shopName') + ->whereNotNull('src.' . $dateField) + ->whereRaw("src.$dateField < $currentMonth") + ->when($storeName !== null && $storeName !== '', fn (Builder $query) => $query->where('src.shopName', $storeName)) + ->when($month !== null && preg_match('/^\d{4}-\d{2}$/', $month) === 1, function (Builder $query) use ($dateField, $month): void { + $start = $month . '-01'; + $end = date('Y-m-d', strtotime($start . ' +1 month')); + $query->where('src.' . $dateField, '>=', $start)->where('src.' . $dateField, '<', $end); + }) + ->selectRaw("src.shopName, $monthStart AS month_start, COALESCE(SUM($incomeExpression), 0) AS $incomeAlias, COALESCE(SUM($expenseExpression), 0) AS $expenseAlias") + ->groupBy('src.shopName') + ->groupByRaw($monthStart); + + $total = $this->connection->query()->fromSub(clone $groups, 'monthly_count')->count(); + if ($total === 0) { + return ['rows' => [], 'total' => 0]; + } + + $query = $this->connection->query()->fromSub($groups, 'm') + ->selectRaw("m.shopName, $lastDay AS balance_date, m.$incomeAlias, m.$expenseAlias") + ->selectSub(function (Builder $query) use ($table, $dateField): void { + $query->from($table . ' AS opening_entry') + ->select('opening_entry.balance') + ->whereColumn('opening_entry.shopName', 'm.shopName') + ->whereColumn('opening_entry.' . $dateField, '<', 'm.month_start') + ->orderByDesc('opening_entry.' . $dateField) + ->orderByDesc('opening_entry.id') + ->limit(1); + }, $openingAlias) + ->selectSub(function (Builder $query) use ($table, $dateField, $nextMonth): void { + $query->from($table . ' AS closing_entry') + ->select('closing_entry.balance') + ->whereColumn('closing_entry.shopName', 'm.shopName') + ->whereRaw("closing_entry.$dateField < $nextMonth") + ->orderByDesc('closing_entry.' . $dateField) + ->orderByDesc('closing_entry.id') + ->limit(1); + }, $closingAlias) + ->orderByDesc('m.month_start') + ->orderBy('m.shopName') + ->forPage($page, $pageSize); + + return [ + 'rows' => $query->get()->map(static fn (object $row): array => (array) $row)->all(), + 'total' => $total, + ]; + } + + private function table(string $datasetKey, ?string $alias = null): Builder + { + if (! $this->ready($datasetKey)) { + throw new RuntimeException('business_mirror_dataset_not_ready'); + } + + $table = $this->dataset($datasetKey)->targetTable(); + + return $this->connection->table($alias === null ? $table : $table . ' AS ' . $alias); + } + + private function dataset(string $datasetKey): BusinessMirrorDataset + { + return $this->datasets->get($datasetKey); + } +} diff --git a/modules/EcommerceFinance/Repositories/PinduoduoSpreadBillRepository.php b/modules/EcommerceFinance/Repositories/PinduoduoSpreadBillRepository.php index 8858a2f1..022179a8 100644 --- a/modules/EcommerceFinance/Repositories/PinduoduoSpreadBillRepository.php +++ b/modules/EcommerceFinance/Repositories/PinduoduoSpreadBillRepository.php @@ -41,11 +41,18 @@ class PinduoduoSpreadBillRepository private UpstreamReadOnlyQuery $query; - public function __construct(UpstreamReadOnlyQueryFactory $factory) - { + public function __construct( + UpstreamReadOnlyQueryFactory $factory, + private readonly ?MarketingAccountBillMirrorRepository $mirror = null, + ) { $this->query = $factory->forSource(self::SOURCE); } + public function usesBusinessMirror(): bool + { + return $this->mirror?->ready(MarketingAccountBillMirrorRepository::PINDUODUO) ?? false; + } + /** * 店铺下拉选项（全部去重） * @@ -53,6 +60,10 @@ class PinduoduoSpreadBillRepository */ public function storeOptions(): array { + if ($this->usesBusinessMirror()) { + return $this->mirror->pinduoduoStoreOptions(); + } + $rows = $this->query->select( 'SELECT DISTINCT shopName FROM ' . self::TABLE . ' WHERE shopName IS NOT NULL ORDER BY shopName' ); @@ -79,6 +90,22 @@ class PinduoduoSpreadBillRepository $page = max(1, $page); $pageSize = max(1, min(200, $pageSize)); + if ($this->usesBusinessMirror()) { + $result = $this->mirror->paginatePinduoduo( + $storeName, + $fromDate, + $toDate, + $flowType, + $page, + $pageSize, + ); + + return [ + 'rows' => array_map(fn (array $row): array => $this->normalize($row), $result['rows']), + 'total' => $result['total'], + ]; + } + [$whereSql, $params] = $this->buildWhere($storeName, $fromDate, $toDate, $flowType); // 总数 @@ -125,16 +152,20 @@ class PinduoduoSpreadBillRepository ?string $toDate = null, ?string $flowType = null ): array { - [$whereSql, $params] = $this->buildWhere($storeName, $fromDate, $toDate, $flowType); + if ($this->usesBusinessMirror()) { + $row = $this->mirror->summarizePinduoduo($storeName, $fromDate, $toDate, $flowType); + } else { + [$whereSql, $params] = $this->buildWhere($storeName, $fromDate, $toDate, $flowType); - $row = $this->query->first( - 'SELECT ' - . 'COALESCE(SUM(CASE WHEN flowType = 1 THEN amount ELSE 0 END), 0) AS income_total, ' - . 'COALESCE(SUM(CASE WHEN flowType = 2 THEN amount ELSE 0 END), 0) AS expense_total, ' - . 'COUNT(*) AS cnt ' - . 'FROM ' . self::TABLE . $whereSql, - $params - ); + $row = $this->query->first( + 'SELECT ' + . 'COALESCE(SUM(CASE WHEN flowType = 1 THEN amount ELSE 0 END), 0) AS income_total, ' + . 'COALESCE(SUM(CASE WHEN flowType = 2 THEN amount ELSE 0 END), 0) AS expense_total, ' + . 'COUNT(*) AS cnt ' + . 'FROM ' . self::TABLE . $whereSql, + $params + ); + } $income = (float) ($row['income_total'] ?? 0); $expense = (float) ($row['expense_total'] ?? 0); @@ -156,6 +187,16 @@ class PinduoduoSpreadBillRepository ): array { $page = max(1, $page); $pageSize = max(1, min(5000, $pageSize)); + + if ($this->usesBusinessMirror()) { + $result = $this->mirror->paginatePinduoduoMonthlyBalances($storeName, $month, $page, $pageSize); + + return [ + 'rows' => array_map(fn (array $row): array => $this->normalizeMonthlyBalance($row), $result['rows']), + 'total' => $result['total'], + ]; + } + $monthExpression = 'DATEADD(MONTH, DATEDIFF(MONTH, 0, createTime), 0)'; [$whereSql, $params] = $this->buildMonthlyWhere($storeName, $month); diff --git a/modules/EcommerceFinance/Repositories/TmallTaobaoSpreadBillRepository.php b/modules/EcommerceFinance/Repositories/TmallTaobaoSpreadBillRepository.php index ff41da31..ccbf69dc 100644 --- a/modules/EcommerceFinance/Repositories/TmallTaobaoSpreadBillRepository.php +++ b/modules/EcommerceFinance/Repositories/TmallTaobaoSpreadBillRepository.php @@ -58,11 +58,18 @@ class TmallTaobaoSpreadBillRepository private UpstreamReadOnlyQuery $query; - public function __construct(UpstreamReadOnlyQueryFactory $factory) - { + public function __construct( + UpstreamReadOnlyQueryFactory $factory, + private readonly ?MarketingAccountBillMirrorRepository $mirror = null, + ) { $this->query = $factory->forSource(self::SOURCE); } + public function usesBusinessMirror(): bool + { + return $this->mirror?->ready(MarketingAccountBillMirrorRepository::TAOBAO) ?? false; + } + /** * 店铺下拉选项（全部去重） * @@ -70,6 +77,10 @@ class TmallTaobaoSpreadBillRepository */ public function storeOptions(): array { + if ($this->usesBusinessMirror()) { + return $this->mirror->taobaoStoreOptions(); + } + $rows = $this->query->select( 'SELECT DISTINCT shopName FROM ' . self::TABLE . ' WHERE shopName IS NOT NULL ORDER BY shopName' ); @@ -96,6 +107,22 @@ class TmallTaobaoSpreadBillRepository $page = max(1, $page); $pageSize = max(1, min(200, $pageSize)); + if ($this->usesBusinessMirror()) { + $result = $this->mirror->paginateTaobao( + $storeName, + $fromDate, + $toDate, + $flowType, + $page, + $pageSize, + ); + + return [ + 'rows' => array_map(fn (array $row): array => $this->normalize($row), $result['rows']), + 'total' => $result['total'], + ]; + } + [$whereSql, $params] = $this->buildWhere($storeName, $fromDate, $toDate, $flowType); // 总数 @@ -143,17 +170,21 @@ class TmallTaobaoSpreadBillRepository ?string $toDate = null, ?string $flowType = null ): array { - [$whereSql, $params] = $this->buildWhere($storeName, $fromDate, $toDate, $flowType); + if ($this->usesBusinessMirror()) { + $row = $this->mirror->summarizeTaobao($storeName, $fromDate, $toDate, $flowType); + } else { + [$whereSql, $params] = $this->buildWhere($storeName, $fromDate, $toDate, $flowType); - // amount 单位为「分」，汇总时除以 100 转为「元」 - $row = $this->query->first( - 'SELECT ' - . 'COALESCE(SUM(CASE WHEN finType = \'DEBIT\' THEN amount ELSE 0 END), 0) AS income_total_raw, ' - . 'COALESCE(SUM(CASE WHEN finType = \'CREDIT\' THEN amount ELSE 0 END), 0) AS expense_total_raw, ' - . 'COUNT(*) AS cnt ' - . 'FROM ' . self::TABLE . $whereSql, - $params - ); + // amount 单位为「分」，汇总时除以 100 转为「元」 + $row = $this->query->first( + 'SELECT ' + . 'COALESCE(SUM(CASE WHEN finType = \'DEBIT\' THEN amount ELSE 0 END), 0) AS income_total_raw, ' + . 'COALESCE(SUM(CASE WHEN finType = \'CREDIT\' THEN amount ELSE 0 END), 0) AS expense_total_raw, ' + . 'COUNT(*) AS cnt ' + . 'FROM ' . self::TABLE . $whereSql, + $params + ); + } // 分 -> 元 $income = ((float) ($row['income_total_raw'] ?? 0)) / 100; @@ -176,6 +207,16 @@ class TmallTaobaoSpreadBillRepository ): array { $page = max(1, $page); $pageSize = max(1, min(5000, $pageSize)); + + if ($this->usesBusinessMirror()) { + $result = $this->mirror->paginateTaobaoMonthlyBalances($storeName, $month, $page, $pageSize); + + return [ + 'rows' => array_map(fn (array $row): array => $this->normalizeMonthlyBalance($row), $result['rows']), + 'total' => $result['total'], + ]; + } + $monthExpression = 'DATEADD(MONTH, DATEDIFF(MONTH, 0, transTime), 0)'; [$where, $params] = $this->buildMonthlyWhere($storeName, $month); diff --git a/tests/Feature/Business/MarketingAccountBillsTabsTest.php b/tests/Feature/Business/MarketingAccountBillsTabsTest.php index d12be030..537a171f 100644 --- a/tests/Feature/Business/MarketingAccountBillsTabsTest.php +++ b/tests/Feature/Business/MarketingAccountBillsTabsTest.php @@ -6,6 +6,7 @@ use App\Models\Auth\Permission; use Mockery; use Modules\EcommerceFinance\Exceptions\UpstreamSnapshotPendingException; use Modules\EcommerceFinance\Repositories\JdWalletBillRepository; +use Modules\EcommerceFinance\Repositories\MarketingAccountBillMirrorRepository; use Modules\EcommerceFinance\Repositories\PinduoduoSpreadBillRepository; use Modules\EcommerceFinance\Repositories\TmallTaobaoSpreadBillRepository; use Modules\EcommerceFinance\Services\UpstreamQueryCache; @@ -15,10 +16,20 @@ use Tests\Feature\FeatureTestCase; class MarketingAccountBillsTabsTest extends FeatureTestCase { + protected function setUp(): void + { + parent::setUp(); + + $mirror = Mockery::mock(MarketingAccountBillMirrorRepository::class); + $mirror->shouldReceive('ready')->byDefault()->andReturnFalse(); + $this->app->instance(MarketingAccountBillMirrorRepository::class, $mirror); + } + public function testMarketingAccountBalanceRedirectsToTaobaoAndShowsRestoredControls(): void { $this->allowMarketingAccountBalancePage(); $repository = Mockery::mock(PinduoduoSpreadBillRepository::class); + $repository->shouldReceive('usesBusinessMirror')->once()->andReturnFalse(); $this->app->instance(PinduoduoSpreadBillRepository::class, $repository); $cache = Mockery::mock(UpstreamQueryCache::class); $cache->shouldReceive('remember')->twice()->andReturnUsing( @@ -141,6 +152,10 @@ class MarketingAccountBillsTabsTest extends FeatureTestCase $this->app->instance(UpstreamQueryCacheFactory::class, $factory); $this->app->instance(JdWalletBillRepository::class, Mockery::mock(JdWalletBillRepository::class)); + $this->app->make(JdWalletBillRepository::class) + ->shouldReceive('usesBusinessMirror')->once()->with('recharge')->andReturnFalse(); + $this->app->make(JdWalletBillRepository::class) + ->shouldReceive('usesBusinessMirrorForOptions')->once()->andReturnFalse(); $this->loginAs() ->get(route('business.pages.show', [ @@ -177,7 +192,9 @@ class MarketingAccountBillsTabsTest extends FeatureTestCase ->andReturn($cache); $this->app->instance(UpstreamQueryCacheFactory::class, $factory); - $this->app->instance(TmallTaobaoSpreadBillRepository::class, Mockery::mock(TmallTaobaoSpreadBillRepository::class)); + $repository = Mockery::mock(TmallTaobaoSpreadBillRepository::class); + $repository->shouldReceive('usesBusinessMirror')->once()->andReturnFalse(); + $this->app->instance(TmallTaobaoSpreadBillRepository::class, $repository); $this->loginAs() ->get(route('business.pages.export', [ @@ -214,7 +231,9 @@ class MarketingAccountBillsTabsTest extends FeatureTestCase $factory->shouldReceive('forSource')->once()->with('jiansun')->andReturn($cache); $this->app->instance(UpstreamQueryCacheFactory::class, $factory); - $this->app->instance(TmallTaobaoSpreadBillRepository::class, Mockery::mock(TmallTaobaoSpreadBillRepository::class)); + $repository = Mockery::mock(TmallTaobaoSpreadBillRepository::class); + $repository->shouldReceive('usesBusinessMirror')->once()->andReturnFalse(); + $this->app->instance(TmallTaobaoSpreadBillRepository::class, $repository); $this->loginAs() ->get(route('business.pages.show', [ @@ -239,7 +258,9 @@ class MarketingAccountBillsTabsTest extends FeatureTestCase $factory->shouldReceive('forSource')->once()->with('jiansun')->andReturn($cache); $this->app->instance(UpstreamQueryCacheFactory::class, $factory); - $this->app->instance(TmallTaobaoSpreadBillRepository::class, Mockery::mock(TmallTaobaoSpreadBillRepository::class)); + $repository = Mockery::mock(TmallTaobaoSpreadBillRepository::class); + $repository->shouldReceive('usesBusinessMirror')->once()->andReturnFalse(); + $this->app->instance(TmallTaobaoSpreadBillRepository::class, $repository); $this->loginAs() ->get(route('business.pages.export', [ @@ -255,6 +276,88 @@ class MarketingAccountBillsTabsTest extends FeatureTestCase ->assertSessionHas('error', '余额数据导出暂时无法完成，请稍后重试。'); } + public function testMarketingBillSecondPageReadsReadyMirrorWithoutRegisteringAColdSnapshot(): void + { + $this->allowMarketingAccountBillsPage(); + + $repository = Mockery::mock(TmallTaobaoSpreadBillRepository::class); + $repository->shouldReceive('usesBusinessMirror')->once()->andReturnTrue(); + $repository->shouldReceive('paginate') + ->once() + ->with(null, null, null, null, 2, 25) + ->andReturn(['rows' => [['store_name' => '镜像店铺']], 'total' => 26]); + $repository->shouldReceive('summary')->once()->andReturn([ + 'income_total' => 0, + 'expense_total' => 0, + 'balance' => 0, + 'count' => 26, + ]); + $repository->shouldReceive('storeOptions')->once()->andReturn(['镜像店铺']); + $this->app->instance(TmallTaobaoSpreadBillRepository::class, $repository); + + $cache = Mockery::mock(UpstreamQueryCache::class); + $cache->shouldNotReceive('remember'); + $factory = Mockery::mock(UpstreamQueryCacheFactory::class); + $factory->shouldReceive('forSource')->once()->with('jiansun')->andReturn($cache); + $this->app->instance(UpstreamQueryCacheFactory::class, $factory); + + $this->loginAs(); + $url = route('business.pages.show', [ + 'section' => 'billing-transactions', + 'page' => 'marketing-account-bills', + 'tab' => 'taobao-alipay', + ]) . '?page=2'; + + $this->get($url) + ->assertOk() + ->assertSeeText('镜像店铺') + ->assertDontSeeText('首份共享快照正在准备'); + } + + public function testMarketingBalanceSecondPageReadsReadyMirrorWithoutRegisteringAColdSnapshot(): void + { + $this->allowMarketingAccountBalancePage(); + + $repository = Mockery::mock(TmallTaobaoSpreadBillRepository::class); + $repository->shouldReceive('usesBusinessMirror')->once()->andReturnTrue(); + $repository->shouldReceive('paginateMonthlyBalances') + ->once() + ->with(null, null, 2, 25) + ->andReturn(['rows' => [[ + 'date' => '2026-08', + 'store_name' => '镜像余额店铺', + 'opening_balance' => 100, + 'income_total' => 20, + 'expense_total' => 5, + 'closing_balance' => 115, + 'system_closing_balance' => 115, + 'balance_difference' => 0, + 'difference_reason' => '', + 'verification_status' => '一致', + 'notes' => '', + ]], 'total' => 26]); + $repository->shouldReceive('storeOptions')->once()->andReturn(['镜像余额店铺']); + $this->app->instance(TmallTaobaoSpreadBillRepository::class, $repository); + + $cache = Mockery::mock(UpstreamQueryCache::class); + $cache->shouldNotReceive('remember'); + $factory = Mockery::mock(UpstreamQueryCacheFactory::class); + $factory->shouldReceive('forSource')->once()->with('jiansun')->andReturn($cache); + $this->app->instance(UpstreamQueryCacheFactory::class, $factory); + + $this->loginAs(); + $url = route('business.pages.show', [ + 'section' => 'billing-transactions', + 'page' => 'marketing-account-balance', + 'tab' => 'taobao-alipay', + ]) . '?page=2'; + + $this->get($url) + ->assertOk() + ->assertSeeText('镜像余额店铺') + ->assertDontSeeText('后台正在准备首份共享数据'); + } + private function allowMarketingAccountBillsPage(): void { $this->user->update(['locale' => 'zh-CN']); diff --git a/tests/Unit/EcommerceFinance/MarketingAccountBillMirrorRepositoryTest.php b/tests/Unit/EcommerceFinance/MarketingAccountBillMirrorRepositoryTest.php new file mode 100644 index 00000000..856b269f --- /dev/null +++ b/tests/Unit/EcommerceFinance/MarketingAccountBillMirrorRepositoryTest.php @@ -0,0 +1,209 @@ + [ + 'driver' => 'sqlite', + 'database' => ':memory:', + 'prefix' => '', + ], + 'upstream_mirrors.connection' => 'business_mirror', + 'upstream_mirrors.datasets' => [ + MarketingAccountBillMirrorRepository::TAOBAO => $this->definition('tmall_spread_bills', 'js_tmall_spread_bills'), + MarketingAccountBillMirrorRepository::PINDUODUO => $this->definition('pdd_spread_bills', 'js_pdd_spread_bills'), + MarketingAccountBillMirrorRepository::ALIBABA => $this->definition('alibaba_bill', 'js_alibaba_bill'), + MarketingAccountBillMirrorRepository::JD_RECHARGE => $this->definition('jd_spreadBills', 'js_jd_spread_bills'), + MarketingAccountBillMirrorRepository::JD_CONSUMPTION => $this->definition('jd_costBills', 'js_jd_cost_bills'), + ], + ]); + DB::purge('business_mirror'); + $this->createTables(); + + $generic = new GenericBusinessMirrorRepository(app('db')); + $this->mirror = new MarketingAccountBillMirrorRepository( + new BusinessMirrorDatasetRegistry(), + $generic, + ); + } + + public function test_ready_mirrors_drive_lists_summaries_options_monthly_balances_and_second_pages(): void + { + $db = DB::connection('business_mirror'); + $db->table('js_tmall_spread_bills')->insert([ + $this->taobao(1, '2026-05-31 23:00:00', 10000, 10000, 'DEBIT'), + $this->taobao(2, '2026-06-01 09:00:00', 5000, 15000, 'DEBIT'), + $this->taobao(3, '2026-06-02 09:00:00', 2000, 13000, 'CREDIT'), + ]); + $db->table('js_pdd_spread_bills')->insert([ + $this->pinduoduo(1, '2026-05-31 23:00:00', 100, 100, 1), + $this->pinduoduo(2, '2026-06-01 09:00:00', 50, 150, 1), + $this->pinduoduo(3, '2026-06-02 09:00:00', 20, 130, 2), + ]); + $db->table('js_alibaba_bill')->insert([ + ['id' => 1, 'shopName' => '阿里店', 'tradeDay' => '2026-06-01 09:00:00', 'tradeTypeName' => '全站', 'tradeAMTIn' => 10, 'tradeAMTOut' => 1], + ['id' => 2, 'shopName' => '阿里店', 'tradeDay' => '2026-06-02 09:00:00', 'tradeTypeName' => '全站', 'tradeAMTIn' => 20, 'tradeAMTOut' => 2], + ]); + $db->table('js_jd_spread_bills')->insert([ + ['id' => 1, 'shopName' => '京东店', 'amount' => 100, 'balanceAmount' => 100, 'orderTypeName' => '充值', 'pk' => 'R1', 'createDate' => '2026-06-01 10:00:00', 'updateTime' => '2026-06-01 10:01:00', 'statusName' => '成功', 'remark' => ''], + ['id' => 2, 'shopName' => '京东店', 'amount' => 200, 'balanceAmount' => 300, 'orderTypeName' => '充值', 'pk' => 'R2', 'createDate' => '2026-06-02 10:00:00', 'updateTime' => '2026-06-02 10:01:00', 'statusName' => '成功', 'remark' => ''], + ]); + $db->table('js_jd_cost_bills')->insert([ + ['id' => 1, 'shopName' => '京东店', 'amount' => -30, 'balanceAmount' => 270, 'orderTypeName' => '消耗', 'creatTime' => '2026-06-03 10:00:00', 'settleTime' => '2026-06-04 00:00:00', 'pk' => 'C1', 'invoiceApplyTypeName' => '推广费'], + ]); + $this->markReady(); + + $factory = $this->throwingFactory(); + $taobao = new TmallTaobaoSpreadBillRepository($factory, $this->mirror); + $pinduoduo = new PinduoduoSpreadBillRepository($factory, $this->mirror); + $alibaba = new AlibabaBillRepository($factory, $this->mirror); + $jd = new JdWalletBillRepository($factory, $this->mirror); + + $taobaoPage2 = $taobao->paginate('镜像店铺', '2026-06-01', '2026-06-30', null, 2, 1); + $taobaoSummary = $taobao->summary('镜像店铺', '2026-06-01', '2026-06-30'); + $taobaoMonthly = $taobao->paginateMonthlyBalances('镜像店铺', '2026-06', 1, 25); + $pddPage2 = $pinduoduo->paginate('镜像店铺', '2026-06-01', '2026-06-30', null, 2, 1); + $pddSummary = $pinduoduo->summary('镜像店铺', '2026-06-01', '2026-06-30'); + $pddMonthly = $pinduoduo->paginateMonthlyBalances('镜像店铺', '2026-06', 1, 25); + $alibabaPage2 = $alibaba->paginate('阿里店', '全站', '2026-06-01', '2026-06-30', 2, 1); + $jdRechargePage2 = $jd->paginateRecharge('京东店', '2026-06-01', '2026-06-30', 2, 1); + + $this->assertSame(2, $taobaoPage2['total']); + $this->assertSame(50.0, $taobaoPage2['rows'][0]['amount']); + $this->assertSame(['镜像店铺'], $taobao->storeOptions()); + $this->assertSame(30.0, $taobaoSummary['net']); + $this->assertSame(130.0, $taobaoMonthly['rows'][0]['closing_balance']); + $this->assertSame(0.0, $taobaoMonthly['rows'][0]['balance_difference']); + + $this->assertSame(2, $pddPage2['total']); + $this->assertSame(50.0, $pddPage2['rows'][0]['amount']); + $this->assertSame(30.0, $pddSummary['net']); + $this->assertSame(130.0, $pddMonthly['rows'][0]['closing_balance']); + $this->assertSame(0.0, $pddMonthly['rows'][0]['balance_difference']); + + $this->assertSame(2, $alibabaPage2['total']); + $this->assertSame(10.0, $alibabaPage2['rows'][0]['income_amount']); + $this->assertSame(['shops' => ['阿里店'], 'product_types' => ['全站']], $alibaba->filterOptions()); + $this->assertSame(27.0, $alibaba->summary('阿里店', '全站')['net']); + + $this->assertSame(2, $jdRechargePage2['total']); + $this->assertSame('R1', $jdRechargePage2['rows'][0]['order_number']); + $this->assertSame(['京东店'], $jd->shopOptions()); + $this->assertSame(300.0, $jd->summaryRecharge('京东店')['amount_total']); + $this->assertSame(30.0, $jd->summaryConsumption('京东店')['amount_total']); + } + + public function test_unready_dataset_keeps_the_existing_upstream_repository_path(): void + { + $query = new class('jiansun') extends UpstreamReadOnlyQuery { + public function scalar(string $sql, array $bindings = []): mixed { return 1; } + public function select(string $sql, array $bindings = []): array + { + return [['shopName' => '上游回退', 'amount' => 8, 'balance' => 9, 'summary' => '', 'flowType' => 1, 'toAdvertiserFundType' => 1, 'createTime' => '2026-06-01']]; + } + }; + $factory = new class($query) extends UpstreamReadOnlyQueryFactory { + public function __construct(private readonly UpstreamReadOnlyQuery $query) {} + public function forSource(string $source): UpstreamReadOnlyQuery { return $this->query; } + }; + + $result = (new PinduoduoSpreadBillRepository($factory, $this->mirror))->paginate(); + + $this->assertSame(1, $result['total']); + $this->assertSame('上游回退', $result['rows'][0]['store_name']); + } + + private function throwingFactory(): UpstreamReadOnlyQueryFactory + { + $query = new class('jiansun') extends UpstreamReadOnlyQuery { + public function scalar(string $sql, array $bindings = []): mixed { throw new RuntimeException('upstream_should_not_be_called'); } + public function select(string $sql, array $bindings = []): array { throw new RuntimeException('upstream_should_not_be_called'); } + public function first(string $sql, array $bindings = []): ?array { throw new RuntimeException('upstream_should_not_be_called'); } + }; + + return new class($query) extends UpstreamReadOnlyQueryFactory { + public function __construct(private readonly UpstreamReadOnlyQuery $query) {} + public function forSource(string $source): UpstreamReadOnlyQuery { return $this->query; } + }; + } + + private function createTables(): void + { + $schema = Schema::connection('business_mirror'); + $schema->create('business_mirror_table_states', function (Blueprint $table): void { + $table->string('dataset_key')->primary(); + $table->dateTime('initial_completed_at')->nullable(); + }); + $schema->create('js_tmall_spread_bills', function (Blueprint $table): void { + $table->integer('id'); $table->string('shopName')->nullable(); $table->dateTime('uniTransTime')->nullable(); $table->dateTime('transTime')->nullable(); + $table->decimal('amount', 18, 2)->nullable(); $table->decimal('balance', 18, 2)->nullable(); $table->string('comment')->nullable(); + $table->string('finType')->nullable(); $table->string('bizType')->nullable(); $table->string('tradeType')->nullable(); + }); + $schema->create('js_pdd_spread_bills', function (Blueprint $table): void { + $table->integer('id'); $table->string('shopName')->nullable(); $table->decimal('amount', 18, 2)->nullable(); $table->decimal('balance', 18, 2)->nullable(); + $table->string('summary')->nullable(); $table->integer('flowType')->nullable(); $table->integer('toAdvertiserFundType')->nullable(); $table->dateTime('createTime')->nullable(); + }); + $schema->create('js_alibaba_bill', function (Blueprint $table): void { + $table->integer('id'); $table->string('shopName')->nullable(); $table->dateTime('tradeDay')->nullable(); $table->string('tradeTypeName')->nullable(); + $table->decimal('tradeAMTIn', 18, 2)->nullable(); $table->decimal('tradeAMTOut', 18, 2)->nullable(); + }); + $schema->create('js_jd_spread_bills', function (Blueprint $table): void { + $table->integer('id'); $table->string('shopName')->nullable(); $table->decimal('amount', 18, 2)->nullable(); $table->decimal('balanceAmount', 18, 2)->nullable(); + $table->string('orderTypeName')->nullable(); $table->string('pk')->nullable(); $table->dateTime('createDate')->nullable(); $table->dateTime('updateTime')->nullable(); + $table->string('statusName')->nullable(); $table->string('remark')->nullable(); + }); + $schema->create('js_jd_cost_bills', function (Blueprint $table): void { + $table->integer('id'); $table->string('shopName')->nullable(); $table->decimal('amount', 18, 2)->nullable(); $table->decimal('balanceAmount', 18, 2)->nullable(); + $table->string('orderTypeName')->nullable(); $table->dateTime('creatTime')->nullable(); $table->dateTime('settleTime')->nullable(); + $table->string('pk')->nullable(); $table->string('invoiceApplyTypeName')->nullable(); + }); + } + + private function markReady(): void + { + foreach (array_keys((array) config('upstream_mirrors.datasets')) as $key) { + DB::connection('business_mirror')->table('business_mirror_table_states')->insert([ + 'dataset_key' => $key, + 'initial_completed_at' => '2026-08-14 12:00:00', + ]); + } + } + + private function definition(string $object, string $target): array + { + return ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => $object, 'target' => $target, 'keys' => ['id']]; + } + + private function taobao(int $id, string $time, float $amount, float $balance, string $finType): array + { + return ['id' => $id, 'shopName' => '镜像店铺', 'uniTransTime' => $time, 'transTime' => $time, 'amount' => $amount, 'balance' => $balance, 'comment' => '', 'finType' => $finType, 'bizType' => '', 'tradeType' => '']; + } + + private function pinduoduo(int $id, string $time, float $amount, float $balance, int $flowType): array + { + return ['id' => $id, 'shopName' => '镜像店铺', 'amount' => $amount, 'balance' => $balance, 'summary' => '', 'flowType' => $flowType, 'toAdvertiserFundType' => 1, 'createTime' => $time]; + } +} From 38858028771fdbf2491564fe1030fa77c7932c32 Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Fri, 14 Aug 2026 20:15:34 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=9A=E5=8A=A1?= =?UTF-8?q?=E9=95=9C=E5=83=8F=E5=A2=9E=E9=87=8F=E5=90=8C=E6=AD=A5=E4=B8=8E?= =?UTF-8?q?=E8=90=A5=E9=94=80=E8=B4=A6=E5=8D=95=E5=AF=BC=E5=87=BA=20(#56)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../MarketingAccountBillMirrorRepository.php | 46 ++++++------------- .../Services/BusinessMirrorSynchronizer.php | 12 ++++- .../BusinessMirrorSynchronizerTest.php | 4 ++ ...rketingAccountBillMirrorRepositoryTest.php | 2 + 4 files changed, 31 insertions(+), 33 deletions(-) diff --git a/modules/EcommerceFinance/Repositories/MarketingAccountBillMirrorRepository.php b/modules/EcommerceFinance/Repositories/MarketingAccountBillMirrorRepository.php index d76d5dfb..b59b5bb2 100644 --- a/modules/EcommerceFinance/Repositories/MarketingAccountBillMirrorRepository.php +++ b/modules/EcommerceFinance/Repositories/MarketingAccountBillMirrorRepository.php @@ -484,27 +484,29 @@ class MarketingAccountBillMirrorRepository $currentMonth = $driver === 'sqlite' ? "date('now', 'start of month')" : "DATE_FORMAT(CURRENT_DATE, '%Y-%m-01')"; - $nextMonth = $driver === 'sqlite' - ? "date(m.month_start, '+1 month')" - : 'DATE_ADD(m.month_start, INTERVAL 1 MONTH)'; $lastDay = $driver === 'sqlite' ? "date(m.month_start, '+1 month', '-1 day')" : "DATE_FORMAT(LAST_DAY(m.month_start), '%Y-%m-%d')"; $table = $this->dataset($datasetKey)->targetTable(); - $groups = $this->connection->table($table . ' AS src') + $ranked = $this->connection->table($table . ' AS src') ->whereNotNull('src.shopName') ->whereNotNull('src.' . $dateField) ->whereRaw("src.$dateField < $currentMonth") ->when($storeName !== null && $storeName !== '', fn (Builder $query) => $query->where('src.shopName', $storeName)) - ->when($month !== null && preg_match('/^\d{4}-\d{2}$/', $month) === 1, function (Builder $query) use ($dateField, $month): void { - $start = $month . '-01'; - $end = date('Y-m-d', strtotime($start . ' +1 month')); - $query->where('src.' . $dateField, '>=', $start)->where('src.' . $dateField, '<', $end); - }) - ->selectRaw("src.shopName, $monthStart AS month_start, COALESCE(SUM($incomeExpression), 0) AS $incomeAlias, COALESCE(SUM($expenseExpression), 0) AS $expenseAlias") - ->groupBy('src.shopName') - ->groupByRaw($monthStart); + ->selectRaw("src.shopName, src.$dateField AS entry_time, src.id AS entry_id, src.balance AS entry_balance, $monthStart AS month_start") + ->selectRaw("$incomeExpression AS income_value, $expenseExpression AS expense_value") + ->selectRaw("LAG(src.balance) OVER (PARTITION BY src.shopName ORDER BY src.$dateField, src.id) AS opening_balance") + ->selectRaw("ROW_NUMBER() OVER (PARTITION BY src.shopName, $monthStart ORDER BY src.$dateField, src.id) AS month_first_rank") + ->selectRaw("ROW_NUMBER() OVER (PARTITION BY src.shopName, $monthStart ORDER BY src.$dateField DESC, src.id DESC) AS month_last_rank"); + + $groups = $this->connection->query()->fromSub($ranked, 'ranked') + ->when($month !== null && preg_match('/^\d{4}-\d{2}$/', $month) === 1, fn (Builder $query) => $query->where('month_start', $month . '-01')) + ->selectRaw("shopName, month_start, COALESCE(SUM(income_value), 0) AS $incomeAlias, COALESCE(SUM(expense_value), 0) AS $expenseAlias") + ->selectRaw("MAX(CASE WHEN month_first_rank = 1 THEN opening_balance END) AS $openingAlias") + ->selectRaw("MAX(CASE WHEN month_last_rank = 1 THEN entry_balance END) AS $closingAlias") + ->groupBy('shopName') + ->groupBy('month_start'); $total = $this->connection->query()->fromSub(clone $groups, 'monthly_count')->count(); if ($total === 0) { @@ -512,25 +514,7 @@ class MarketingAccountBillMirrorRepository } $query = $this->connection->query()->fromSub($groups, 'm') - ->selectRaw("m.shopName, $lastDay AS balance_date, m.$incomeAlias, m.$expenseAlias") - ->selectSub(function (Builder $query) use ($table, $dateField): void { - $query->from($table . ' AS opening_entry') - ->select('opening_entry.balance') - ->whereColumn('opening_entry.shopName', 'm.shopName') - ->whereColumn('opening_entry.' . $dateField, '<', 'm.month_start') - ->orderByDesc('opening_entry.' . $dateField) - ->orderByDesc('opening_entry.id') - ->limit(1); - }, $openingAlias) - ->selectSub(function (Builder $query) use ($table, $dateField, $nextMonth): void { - $query->from($table . ' AS closing_entry') - ->select('closing_entry.balance') - ->whereColumn('closing_entry.shopName', 'm.shopName') - ->whereRaw("closing_entry.$dateField < $nextMonth") - ->orderByDesc('closing_entry.' . $dateField) - ->orderByDesc('closing_entry.id') - ->limit(1); - }, $closingAlias) + ->selectRaw("m.shopName, $lastDay AS balance_date, m.$incomeAlias, m.$expenseAlias, m.$openingAlias, m.$closingAlias") ->orderByDesc('m.month_start') ->orderBy('m.shopName') ->forPage($page, $pageSize); diff --git a/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php b/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php index 8ef379b4..80802317 100644 --- a/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php +++ b/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php @@ -114,8 +114,8 @@ class BusinessMirrorSynchronizer if ($mode === 'incremental' && $dataset->watermarkColumn() !== null) { $watermark = $this->quote($dataset->watermarkColumn()); $where[] = $watermark . ' >= ? AND ' . $watermark . ' < ?'; - $bindings[] = $windowStart?->format('Y-m-d H:i:s.u'); - $bindings[] = $windowEnd?->format('Y-m-d H:i:s.u'); + $bindings[] = $this->sqlServerDateTime($windowStart); + $bindings[] = $this->sqlServerDateTime($windowEnd); } elseif ($mode === 'incremental' && $cursor === null && $highWatermark !== []) { [$predicate, $values] = $this->keyset($dataset->keyColumns(), $highWatermark); $where[] = $predicate; @@ -137,6 +137,14 @@ class BusinessMirrorSynchronizer return $this->queries->forSource($dataset->sourceKey())->select($sql, $bindings); } + private function sqlServerDateTime(?Carbon $value): ?string + { + // ISO 8601 with millisecond precision is accepted consistently by SQL Server + // through FreeTDS. Six-digit fractions were interpreted as an invalid date by + // some upstream datetime columns and broke every incremental run. + return $value?->format('Y-m-d\TH:i:s.v'); + } + /** @return array{0:string,1:list} */ private function keyset(array $columns, array $cursor): array { diff --git a/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php index be3455b2..29925c62 100644 --- a/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php +++ b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php @@ -129,6 +129,10 @@ class BusinessMirrorSynchronizerTest extends TestCase $this->assertStringContainsString('[updateTime] >= ? AND [updateTime] < ?', $this->query->calls[0]['sql']); $this->assertStringContainsString('ORDER BY [updateTime], [Id]', $this->query->calls[0]['sql']); $this->assertCount(2, $this->query->calls[0]['bindings']); + foreach ($this->query->calls[0]['bindings'] as $binding) { + $this->assertMatchesRegularExpression('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}$/', $binding); + $this->assertStringNotContainsString(' ', $binding); + } } public function test_incremental_sync_refuses_to_guess_before_first_full_sync(): void diff --git a/tests/Unit/EcommerceFinance/MarketingAccountBillMirrorRepositoryTest.php b/tests/Unit/EcommerceFinance/MarketingAccountBillMirrorRepositoryTest.php index 856b269f..609ae15b 100644 --- a/tests/Unit/EcommerceFinance/MarketingAccountBillMirrorRepositoryTest.php +++ b/tests/Unit/EcommerceFinance/MarketingAccountBillMirrorRepositoryTest.php @@ -95,12 +95,14 @@ class MarketingAccountBillMirrorRepositoryTest extends TestCase $this->assertSame(50.0, $taobaoPage2['rows'][0]['amount']); $this->assertSame(['镜像店铺'], $taobao->storeOptions()); $this->assertSame(30.0, $taobaoSummary['net']); + $this->assertSame(100.0, $taobaoMonthly['rows'][0]['opening_balance']); $this->assertSame(130.0, $taobaoMonthly['rows'][0]['closing_balance']); $this->assertSame(0.0, $taobaoMonthly['rows'][0]['balance_difference']); $this->assertSame(2, $pddPage2['total']); $this->assertSame(50.0, $pddPage2['rows'][0]['amount']); $this->assertSame(30.0, $pddSummary['net']); + $this->assertSame(100.0, $pddMonthly['rows'][0]['opening_balance']); $this->assertSame(130.0, $pddMonthly['rows'][0]['closing_balance']); $this->assertSame(0.0, $pddMonthly['rows'][0]['balance_difference']); From bb9937a0cf31a806ee16bff0a786f485d76c29a5 Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Fri, 14 Aug 2026 20:44:59 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=97=A7=E4=B8=9A?= =?UTF-8?q?=E5=8A=A1=E6=95=B0=E6=8D=AE=E5=AD=97=E7=AC=A6=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E9=95=9C=E5=83=8F=E5=90=8C=E6=AD=A5=E4=B8=AD=E6=96=AD=20(#57)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Services/BusinessMirrorSynchronizer.php | 16 ++++++++++++ .../BusinessMirrorSynchronizerTest.php | 25 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php b/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php index 80802317..c46b391a 100644 --- a/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php +++ b/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php @@ -226,6 +226,8 @@ class BusinessMirrorSynchronizer $row[$name] = substr($value, 0, 10); } elseif (in_array($type, ['datetime', 'datetime2', 'smalldatetime'], true)) { $row[$name] = preg_replace('/(\.\d{6})\d+/', '$1', str_replace('T', ' ', $value)); + } elseif (in_array($type, ['char', 'nchar', 'varchar', 'nvarchar', 'text', 'ntext', 'xml'], true)) { + $row[$name] = $this->normalizeText($value); } } @@ -233,6 +235,20 @@ class BusinessMirrorSynchronizer }, $rows); } + private function normalizeText(string $value): string + { + // A small number of legacy SQL Server rows contain embedded NUL bytes or + // malformed byte sequences. MariaDB correctly rejects those values as + // invalid utf8mb4. Do not guess a source encoding: preserve all valid + // UTF-8 and mark only irrecoverable bytes with the Unicode replacement + // character so one corrupt label cannot block the entire table mirror. + $value = str_replace("\0", '', $value); + + return mb_check_encoding($value, 'UTF-8') + ? $value + : mb_scrub($value, 'UTF-8'); + } + private function quote(string $identifier): string { if (preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/', $identifier) !== 1) { diff --git a/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php index 29925c62..a1205baa 100644 --- a/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php +++ b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php @@ -143,6 +143,31 @@ class BusinessMirrorSynchronizerTest extends TestCase $this->assertSame([], $this->query->calls); } + public function test_full_sync_sanitizes_invalid_utf8_text_without_changing_valid_text(): void + { + $this->query->batches = [[ + [ + 'Id' => 1, + 'updateTime' => '2026-08-14 10:00:00', + 'displayName' => hex2bin('e80044495900'), + ], + [ + 'Id' => 2, + 'updateTime' => '2026-08-14 11:00:00', + 'displayName' => '合法名称', + ], + ]]; + + $this->synchronizer->sync('test.dbo.records', true, 10, 1); + + $rows = DB::connection('business_mirror')->table('test_records')->orderBy('Id')->pluck('displayName')->all(); + $this->assertStringEndsWith('DIY', $rows[0]); + $this->assertNotSame(hex2bin('e80044495900'), $rows[0]); + $this->assertSame('合法名称', $rows[1]); + $this->assertTrue(mb_check_encoding($rows[0], 'UTF-8')); + $this->assertStringNotContainsString("\0", $rows[0]); + } + public function test_empty_full_read_does_not_delete_previous_mirror_rows(): void { $this->query->batches = [[ From 9c179f617067594b899b48c5be4fd88859efebe4 Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Fri, 14 Aug 2026 21:02:58 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=9A=E5=8A=A1?= =?UTF-8?q?=E9=95=9C=E5=83=8F=E5=AE=BD=E8=A1=A8=E8=A1=8C=E5=A4=A7=E5=B0=8F?= =?UTF-8?q?=E9=99=90=E5=88=B6=20(#58)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Services/BusinessMirrorSchemaManager.php | 9 +++-- .../BusinessMirrorSynchronizerTest.php | 34 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/modules/EcommerceFinance/Services/BusinessMirrorSchemaManager.php b/modules/EcommerceFinance/Services/BusinessMirrorSchemaManager.php index 5608d1fd..32b1aeb9 100644 --- a/modules/EcommerceFinance/Services/BusinessMirrorSchemaManager.php +++ b/modules/EcommerceFinance/Services/BusinessMirrorSchemaManager.php @@ -112,7 +112,12 @@ class BusinessMirrorSchemaManager 'time' => $table->time($name, 6), 'uniqueidentifier' => $table->char($name, 36), 'binary', 'varbinary', 'image', 'timestamp', 'rowversion' => $table->binary($name), - 'char', 'nchar', 'varchar', 'nvarchar' => $length > 0 && $length <= 1024 + // MariaDB counts every inline varchar column towards the 65,535-byte + // row limit. Wide legacy SQL Server tables can contain dozens of + // nvarchar columns even though each individual column looks small. + // Keep only genuinely short values inline; TEXT is stored off-page + // and lets the mirror preserve the complete source schema. + 'char', 'nchar', 'varchar', 'nvarchar' => $length > 0 && $length <= 64 ? $table->string($name, $length) : $table->longText($name), default => $table->longText($name), @@ -132,7 +137,7 @@ class BusinessMirrorSchemaManager $length = (int) ($column['max_length'] ?? 0); return ! in_array($type, ['text', 'ntext', 'xml', 'image', 'binary', 'varbinary'], true) - && (! in_array($type, ['char', 'nchar', 'varchar', 'nvarchar'], true) || ($length > 0 && $length <= 191)); + && (! in_array($type, ['char', 'nchar', 'varchar', 'nvarchar'], true) || ($length > 0 && $length <= 64)); } return false; diff --git a/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php index a1205baa..4a206e8c 100644 --- a/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php +++ b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php @@ -168,6 +168,40 @@ class BusinessMirrorSynchronizerTest extends TestCase $this->assertStringNotContainsString("\0", $rows[0]); } + public function test_wide_source_schema_stores_only_short_strings_inline(): void + { + $datasets = (array) config('upstream_mirrors.datasets', []); + $datasets['test.dbo.wide_records'] = [ + 'source_key' => 'test', + 'schema' => 'dbo', + 'object' => 'wide_records', + 'target' => 'test_wide_records', + 'keys' => ['Id'], + 'watermark' => 'updateTime', + ]; + config(['upstream_mirrors.datasets' => $datasets]); + + $columns = [ + $this->column('Id', 1, 'int', null), + $this->column('updateTime', 2, 'datetime2', null), + $this->column('shortCode', 3, 'nvarchar', 32), + $this->column('longLabel', 4, 'nvarchar', 120), + ]; + DB::table('upstream_schema_columns')->insert(array_map( + fn (array $column): array => [...$column, 'object_name' => 'wide_records'], + $columns, + )); + + $dataset = (new BusinessMirrorDatasetRegistry())->get('test.dbo.wide_records'); + (new BusinessMirrorSchemaManager())->ensure($dataset); + + $types = DB::connection('business_mirror')->select('PRAGMA table_info(test_wide_records)'); + $types = collect($types)->pluck('type', 'name')->map(fn (string $type): string => strtolower($type)); + + $this->assertSame('varchar', $types['shortCode']); + $this->assertSame('text', $types['longLabel']); + } + public function test_empty_full_read_does_not_delete_previous_mirror_rows(): void { $this->query->batches = [[ From 955ddb5e2619f7484bb08e8b1e72948843a9a6d1 Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Sat, 15 Aug 2026 03:16:16 +0800 Subject: [PATCH 7/9] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=9A=E5=8A=A1?= =?UTF-8?q?=E9=95=9C=E5=83=8F=E6=97=A5=E6=9C=9F=E6=A0=BC=E5=BC=8F=20(#59)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Services/BusinessMirrorSynchronizer.php | 19 +++++++++++++++++-- .../BusinessMirrorSynchronizerTest.php | 7 +++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php b/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php index c46b391a..6d36af0b 100644 --- a/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php +++ b/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php @@ -106,7 +106,6 @@ class BusinessMirrorSynchronizer ?Carbon $windowEnd, int $limit, ): array { - $names = array_map(fn (array $column): string => (string) $column['column_name'], $columns); $order = $this->orderColumns($dataset, $mode); $where = []; $bindings = []; @@ -129,7 +128,7 @@ class BusinessMirrorSynchronizer } $sql = 'SELECT TOP (' . $limit . ') ' - . implode(', ', array_map(fn (string $name): string => $this->quote($name), $names)) + . implode(', ', array_map(fn (array $column): string => $this->selectExpression($column), $columns)) . ' FROM ' . $this->quote($dataset->sourceSchema()) . '.' . $this->quote($dataset->sourceObject()) . ($where === [] ? '' : ' WHERE (' . implode(') AND (', $where) . ')') . ' ORDER BY ' . implode(', ', array_map(fn (string $name): string => $this->quote($name), $order)); @@ -137,6 +136,22 @@ class BusinessMirrorSynchronizer return $this->queries->forSource($dataset->sourceKey())->select($sql, $bindings); } + /** @param array $column */ + private function selectExpression(array $column): string + { + $name = (string) $column['column_name']; + $quoted = $this->quote($name); + + // FreeTDS may return SQL Server date values using the connection locale + // (for example "Jan 1 2026"). Convert temporal columns at the source so + // every mirror receives deterministic ISO values that MariaDB accepts. + return match (strtolower((string) ($column['data_type'] ?? ''))) { + 'date' => 'CONVERT(varchar(10), ' . $quoted . ', 23) AS ' . $quoted, + 'datetime', 'datetime2', 'smalldatetime' => 'CONVERT(varchar(27), ' . $quoted . ', 126) AS ' . $quoted, + default => $quoted, + }; + } + private function sqlServerDateTime(?Carbon $value): ?string { // ISO 8601 with millisecond precision is accepted consistently by SQL Server diff --git a/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php index 4a206e8c..93c65ee1 100644 --- a/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php +++ b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php @@ -69,6 +69,7 @@ class BusinessMirrorSynchronizerTest extends TestCase $this->column('Id', 1, 'int', null), $this->column('updateTime', 2, 'datetime2', null), $this->column('displayName', 3, 'nvarchar', 120), + $this->column('businessDate', 4, 'date', null), ]); $this->query = new FakeBusinessMirrorQuery(); @@ -88,8 +89,8 @@ class BusinessMirrorSynchronizerTest extends TestCase public function test_full_sync_materializes_complete_table_and_marks_state_ready(): void { $this->query->batches = [[ - ['Id' => 1, 'updateTime' => '2026-08-14 10:00:00.1234567', 'displayName' => '一号'], - ['Id' => 2, 'updateTime' => '2026-08-14 11:00:00.0000000', 'displayName' => '二号'], + ['Id' => 1, 'updateTime' => '2026-08-14 10:00:00.1234567', 'displayName' => '一号', 'businessDate' => '2026-08-14'], + ['Id' => 2, 'updateTime' => '2026-08-14 11:00:00.0000000', 'displayName' => '二号', 'businessDate' => '2026-08-15'], ]]; $result = $this->synchronizer->sync('test.dbo.records', true, 10, 1); @@ -105,6 +106,8 @@ class BusinessMirrorSynchronizerTest extends TestCase $this->assertNotNull($state->initial_completed_at); $this->assertSame(2, (int) $state->mirror_row_count); $this->assertStringContainsString('SELECT TOP (10)', $this->query->calls[0]['sql']); + $this->assertStringContainsString('CONVERT(varchar(27), [updateTime], 126) AS [updateTime]', $this->query->calls[0]['sql']); + $this->assertStringContainsString('CONVERT(varchar(10), [businessDate], 23) AS [businessDate]', $this->query->calls[0]['sql']); $this->assertStringContainsString('ORDER BY [Id]', $this->query->calls[0]['sql']); } From fbd05380f394c7ad8f1e20c866b16bee61b6a33e Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Sat, 15 Aug 2026 03:49:49 +0800 Subject: [PATCH 8/9] =?UTF-8?q?=E6=8F=90=E9=AB=98=E9=A6=96=E6=AC=A1?= =?UTF-8?q?=E4=B8=9A=E5=8A=A1=E9=95=9C=E5=83=8F=E6=89=B9=E6=AC=A1=E6=95=88?= =?UTF-8?q?=E7=8E=87=20(#60)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Services/BusinessMirrorSynchronizer.php | 5 ++++- .../BusinessMirrorSynchronizerTest.php | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php b/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php index 6d36af0b..258bb831 100644 --- a/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php +++ b/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php @@ -50,7 +50,10 @@ class BusinessMirrorSynchronizer ); $cursor = is_array($state['cursor'] ?? null) ? $state['cursor'] : null; $highWatermark = is_array($state['high_watermark'] ?? null) ? $state['high_watermark'] : []; - $batchSize = min(max($batchSize ?? (int) config('upstream_mirrors.table_sync.batch_size', 500), 1), 2000); + // Routine runs keep the configured 500-row default. A larger explicit + // batch is reserved for supervised first syncs so wide SQL Server views + // need fewer repeated keyset scans without creating unbounded PHP arrays. + $batchSize = min(max($batchSize ?? (int) config('upstream_mirrors.table_sync.batch_size', 500), 1), 5000); $maxBatches = max($maxBatches ?? (int) config('upstream_mirrors.table_sync.max_batches_per_run', 2), 0); $batches = 0; $rows = (int) ($state['last_upserted_rows'] ?? 0); diff --git a/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php index 93c65ee1..bcc8e7de 100644 --- a/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php +++ b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php @@ -146,6 +146,17 @@ class BusinessMirrorSynchronizerTest extends TestCase $this->assertSame([], $this->query->calls); } + public function test_supervised_full_sync_caps_explicit_batch_size_at_five_thousand(): void + { + $this->query->batches = [[ + ['Id' => 1, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => '一号', 'businessDate' => '2026-08-14'], + ]]; + + $this->synchronizer->sync('test.dbo.records', true, 50000, 1); + + $this->assertStringContainsString('SELECT TOP (5000)', $this->query->calls[0]['sql']); + } + public function test_full_sync_sanitizes_invalid_utf8_text_without_changing_valid_text(): void { $this->query->batches = [[ From 809bca2e5c790444660df243e38fca4ffbe10f43 Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Sat, 15 Aug 2026 05:03:19 +0800 Subject: [PATCH 9/9] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=9A=E5=8A=A1?= =?UTF-8?q?=E9=95=9C=E5=83=8F=E5=A2=9E=E9=87=8F=E5=88=86=E9=A1=B5=E8=B6=85?= =?UTF-8?q?=E6=97=B6=20(#61)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Services/BusinessMirrorSynchronizer.php | 26 ++++++++------- .../BusinessMirrorSynchronizerTest.php | 32 ++++++++++++++++++- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php b/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php index 258bb831..7577037c 100644 --- a/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php +++ b/modules/EcommerceFinance/Services/BusinessMirrorSynchronizer.php @@ -23,6 +23,10 @@ class BusinessMirrorSynchronizer public function sync(string $datasetKey, bool $full = false, ?int $batchSize = null, ?int $maxBatches = null): array { $dataset = $this->datasets->get($datasetKey); + // A long-running scheduler may have retained a timed-out PDO instance + // from the previous dataset. Each dataset starts with a fresh upstream + // connection so one source failure cannot poison subsequent mirrors. + UpstreamOdbcConnection::reset($dataset->sourceKey()); $state = $this->mirror->state($dataset); if (! $full && ($state['initial_completed_at'] ?? null) === null) { @@ -77,7 +81,7 @@ class BusinessMirrorSynchronizer return $this->result($dataset, 'succeeded', $mode, $batches, $rows); } - $cursor = $this->cursor($dataset, $batch[array_key_last($batch)], $mode); + $cursor = $this->cursor($dataset, $batch[array_key_last($batch)]); $rows += count($batch); $batches++; $this->mirror->storeBatch($dataset, $columns, $this->normalize($columns, $batch), $cursor, $generation, $rows); @@ -109,7 +113,7 @@ class BusinessMirrorSynchronizer ?Carbon $windowEnd, int $limit, ): array { - $order = $this->orderColumns($dataset, $mode); + $order = $this->orderColumns($dataset); $where = []; $bindings = []; @@ -186,22 +190,20 @@ class BusinessMirrorSynchronizer } /** @return list */ - private function orderColumns(BusinessMirrorDataset $dataset, string $mode): array + private function orderColumns(BusinessMirrorDataset $dataset): array { - $columns = $dataset->keyColumns(); - - if ($mode === 'incremental' && $dataset->watermarkColumn() !== null) { - array_unshift($columns, $dataset->watermarkColumn()); - } - - return array_values(array_unique($columns)); + // The watermark belongs in the incremental WHERE window only. Sorting + // a very large SQL Server view by an unindexed timestamp forces an + // expensive full-window sort on every keyset page and eventually times + // out. Dataset keys are the stable, unique pagination contract. + return array_values(array_unique($dataset->keyColumns())); } /** @return array */ - private function cursor(BusinessMirrorDataset $dataset, array $row, string $mode): array + private function cursor(BusinessMirrorDataset $dataset, array $row): array { $cursor = []; - foreach ($this->orderColumns($dataset, $mode) as $column) { + foreach ($this->orderColumns($dataset) as $column) { $cursor[$column] = $this->rowValue($row, $column); } diff --git a/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php index bcc8e7de..9c85136a 100644 --- a/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php +++ b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php @@ -40,6 +40,10 @@ class BusinessMirrorSynchronizerTest extends TestCase ], ], 'upstream_mirrors.table_sync.overlap_hours' => 24, + 'upstreams.sources.test' => [ + 'label' => 'Test source', + 'config' => 'test', + ], ]); DB::purge('business_mirror'); @@ -130,7 +134,8 @@ class BusinessMirrorSynchronizerTest extends TestCase $this->assertSame(2, DB::connection('business_mirror')->table('test_records')->count()); $this->assertSame('新值', DB::connection('business_mirror')->table('test_records')->where('Id', 1)->value('displayName')); $this->assertStringContainsString('[updateTime] >= ? AND [updateTime] < ?', $this->query->calls[0]['sql']); - $this->assertStringContainsString('ORDER BY [updateTime], [Id]', $this->query->calls[0]['sql']); + $this->assertStringContainsString('ORDER BY [Id]', $this->query->calls[0]['sql']); + $this->assertStringNotContainsString('ORDER BY [updateTime]', $this->query->calls[0]['sql']); $this->assertCount(2, $this->query->calls[0]['bindings']); foreach ($this->query->calls[0]['bindings'] as $binding) { $this->assertMatchesRegularExpression('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}$/', $binding); @@ -138,6 +143,31 @@ class BusinessMirrorSynchronizerTest extends TestCase } } + public function test_incremental_keyset_keeps_watermark_out_of_pagination_cursor(): void + { + $this->query->batches = [[ + ['Id' => 1, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => '初始'], + ]]; + $this->synchronizer->sync('test.dbo.records', true, 10, 1); + + $this->query->calls = []; + $this->query->batches = [[ + ['Id' => 1, 'updateTime' => '2026-08-14 12:00:00', 'displayName' => '更新一'], + ['Id' => 2, 'updateTime' => '2026-08-14 12:30:00', 'displayName' => '更新二'], + ], []]; + + $result = $this->synchronizer->sync('test.dbo.records', false, 2, 0); + + $this->assertSame('succeeded', $result['status']); + $this->assertCount(2, $this->query->calls); + $this->assertStringContainsString('[updateTime] >= ? AND [updateTime] < ?', $this->query->calls[1]['sql']); + $this->assertStringContainsString('[Id] > ?', $this->query->calls[1]['sql']); + $this->assertStringContainsString('ORDER BY [Id]', $this->query->calls[1]['sql']); + $this->assertStringNotContainsString('[updateTime] > ?', $this->query->calls[1]['sql']); + $this->assertCount(3, $this->query->calls[1]['bindings']); + $this->assertSame(2, $this->query->calls[1]['bindings'][2]); + } + public function test_incremental_sync_refuses_to_guess_before_first_full_sync(): void { $result = $this->synchronizer->sync('test.dbo.records', false, 10, 1);