Merge remote-tracking branch 'origin/main' into feature/limeng-courier-quote-change-workflow

This commit is contained in:
limeng 2026-08-15 16:40:15 +08:00
commit f47fe2acd2
25 changed files with 3020 additions and 143 deletions

View File

@ -4,6 +4,7 @@ namespace App\Business;
use Illuminate\Pagination\LengthAwarePaginator;
use InvalidArgumentException;
use Modules\EcommerceFinance\Repositories\BusinessInformationMirrorRepository;
use Modules\EcommerceFinance\Services\UpstreamQueryCacheFactory;
use Modules\EcommerceFinance\Services\UpstreamReadOnlyQuery;
@ -11,9 +12,10 @@ class UpstreamBusinessInformation
{
private const SOURCE = 'operations_replica';
public function __construct(private readonly UpstreamQueryCacheFactory $caches)
{
}
public function __construct(
private readonly UpstreamQueryCacheFactory $caches,
private readonly BusinessInformationMirrorRepository $mirror,
) {}
public function paginate(string $section, string $page, int $perPage = 50): LengthAwarePaginator
{
@ -64,7 +66,7 @@ class UpstreamBusinessInformation
private function companyPage(int $page, int $perPage): array
{
return $this->remember(
return $this->page(
'company-information.read',
$page,
$perPage,
@ -108,7 +110,7 @@ SQL);
private function storePage(int $page, int $perPage): array
{
return $this->remember(
return $this->page(
'store-information.read',
$page,
$perPage,
@ -156,7 +158,7 @@ SQL);
private function trademarkPage(int $page, int $perPage): array
{
return $this->remember(
return $this->page(
'trademark-information.read',
$page,
$perPage,
@ -191,8 +193,12 @@ SQL);
];
}
private function remember(string $scope, int $page, int $perPage, callable $resolver): array
private function page(string $scope, int $page, int $perPage, callable $resolver): array
{
if ($this->mirror->ready($scope)) {
return $this->normalize($scope, $this->mirror->paginate($scope, $page, $perPage));
}
$cache = $this->caches->forSource(self::SOURCE);
return $cache->remember($scope, compact('page', 'perPage'), function () use ($resolver) {
@ -200,6 +206,22 @@ SQL);
});
}
/** @param array{data: array<int,array<string,mixed>>, total: int} $payload */
private function normalize(string $scope, array $payload): array
{
$mapper = match ($scope) {
'company-information.read' => fn (array $row): array => $this->mapCompany($row),
'store-information.read' => fn (array $row): array => $this->mapStore($row),
'trademark-information.read' => fn (array $row): array => $this->mapTrademark($row),
default => throw new InvalidArgumentException('未知的经营分析副本库资料类型:' . $scope),
};
return [
'data' => array_map($mapper, $payload['data']),
'total' => (int) $payload['total'],
];
}
private function mapCompany(array $row): array
{
$row = $this->toUtf8($row);

View File

@ -0,0 +1,107 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
use Modules\EcommerceFinance\Repositories\GenericBusinessMirrorRepository;
use Modules\EcommerceFinance\Services\BusinessMirrorDatasetRegistry;
use Modules\EcommerceFinance\Services\BusinessMirrorSynchronizer;
use Throwable;
class SyncBusinessMirrors extends Command
{
protected $signature = 'business-mirror:sync-tables
{dataset? : 只同步指定数据集}
{--full : 执行或继续首次全量/校准同步}
{--batch= : 每批最多读取行数}
{--max-batches= : 每个数据集本轮最多批次数,0 表示直到完成}
{--max-datasets= : 本轮最多处理的数据集数量}';
protected $description = '将项目实际使用的上游表按稳定游标同步到共享业务镜像库';
public function handle(
BusinessMirrorDatasetRegistry $datasets,
BusinessMirrorSynchronizer $synchronizer,
GenericBusinessMirrorRepository $mirror,
): int
{
if (! (bool) config('upstream_mirrors.refresh_enabled', false) && ! $this->option('full')) {
return self::SUCCESS;
}
$requested = $this->argument('dataset');
$keys = $requested === null
? $this->scheduledKeys($datasets, $mirror)
: [(string) $requested];
$maxDatasets = max((int) ($this->option('max-datasets')
?? config('upstream_mirrors.table_sync.max_datasets_per_run', 1)), 1);
$processed = 0;
$failed = false;
foreach ($keys as $key) {
if ($processed >= $maxDatasets) {
break;
}
try {
$result = $synchronizer->sync(
$key,
(bool) $this->option('full'),
$this->option('batch') === null ? null : (int) $this->option('batch'),
$this->option('max-batches') === null ? null : (int) $this->option('max-batches'),
);
} catch (Throwable $exception) {
$failed = true;
Log::error('共享业务镜像表同步失败。', ['dataset' => $key, 'exception' => $exception::class]);
$this->error($key . ':同步失败,已保留上一版共享数据。');
$processed++;
continue;
}
if ($result['status'] === 'requires_full_sync') {
if ($requested !== null) {
$this->warn($key . ':尚未完成首次全量同步,请显式使用 --full。');
}
continue;
}
$processed++;
$this->info(sprintf(
'%s:%s,本轮 %d 批,累计处理 %d 行。',
$key,
$result['status'] === 'succeeded' ? '同步完成' : '同步进行中',
$result['batches'],
$result['rows'],
));
}
return $failed ? self::FAILURE : self::SUCCESS;
}
/** @return list<string> */
private function scheduledKeys(
BusinessMirrorDatasetRegistry $datasets,
GenericBusinessMirrorRepository $mirror,
): array {
$scheduled = [];
foreach ($datasets->all() as $key => $dataset) {
$state = $mirror->state($dataset);
if (! is_array($state) || ($state['initial_completed_at'] ?? null) === null) {
continue;
}
$scheduled[] = [
'key' => $key,
'last_attempted_at' => (string) ($state['last_attempted_at'] ?? ''),
];
}
usort($scheduled, fn (array $left, array $right): int =>
[$left['last_attempted_at'], $left['key']] <=> [$right['last_attempted_at'], $right['key']]
);
return array_column($scheduled, 'key');
}
}

View File

@ -272,18 +272,22 @@ class Pages extends Controller
: app(PinduoduoSpreadBillRepository::class);
$scope = $isTaobao ? 'taobao-alipay-bills.list' : 'pinduoduo-bills.list';
$cache = $this->upstreamCaches->forSource('jiansun');
$usingMirror = $repository->usesBusinessMirror();
try {
$result = $cache->remember(
$scope,
['view' => 'monthly-balances', 'filters' => $filters, 'page' => $page, 'limit' => $pageSize],
fn () => $repository->paginateMonthlyBalances(
$filters['store_name'],
$filters['month'],
$page,
$pageSize,
),
$resolver = fn () => $repository->paginateMonthlyBalances(
$filters['store_name'],
$filters['month'],
$page,
$pageSize,
);
$result = $usingMirror
? $resolver()
: $cache->remember(
$scope,
['view' => 'monthly-balances', 'filters' => $filters, 'page' => $page, 'limit' => $pageSize],
$resolver,
);
} catch (UpstreamSnapshotPendingException $exception) {
$content['pending'] = true;
$result = ['rows' => [], 'total' => 0];
@ -297,11 +301,14 @@ class Pages extends Controller
}
try {
$storeOptions = $cache->remember(
$isTaobao ? 'taobao-alipay-bills.options' : 'pinduoduo-bills.options',
[],
fn () => $repository->storeOptions(),
);
$resolver = fn () => $repository->storeOptions();
$storeOptions = $usingMirror
? $resolver()
: $cache->remember(
$isTaobao ? 'taobao-alipay-bills.options' : 'pinduoduo-bills.options',
[],
$resolver,
);
} catch (Throwable $exception) {
report($exception);
$storeOptions = [];
@ -352,11 +359,14 @@ class Pages extends Controller
], static fn (mixed $value): bool => $value !== '');
try {
$result = $this->upstreamCaches->forSource('jiansun')->remember(
$scope,
['view' => 'monthly-balances', 'filters' => $filters, 'page' => 1, 'limit' => 5000, 'export' => true],
fn () => $repository->paginateMonthlyBalances($filters['store_name'], $filters['month'], 1, 5000),
);
$resolver = fn () => $repository->paginateMonthlyBalances($filters['store_name'], $filters['month'], 1, 5000);
$result = $repository->usesBusinessMirror()
? $resolver()
: $this->upstreamCaches->forSource('jiansun')->remember(
$scope,
['view' => 'monthly-balances', 'filters' => $filters, 'page' => 1, 'limit' => 5000, 'export' => true],
$resolver,
);
} catch (ServiceUnavailableHttpException) {
return redirect()->route('business.pages.show', $redirectParameters)
->with('error', '首份共享快照正在准备,系统已登记刷新请求;请稍后刷新页面后再导出。');
@ -880,28 +890,38 @@ class Pages extends Controller
];
$repository = app(AlibabaBillRepository::class);
$cache = $this->upstreamCaches->forSource('jiansun');
$usingMirror = $repository->usesBusinessMirror();
try {
$result = $cache->remember(
'marketing-account-bills.alibaba.list',
['filters' => $filters, 'page' => $page, 'limit' => $pageSize],
fn () => $repository->paginate(
$filters['shop_name'], $filters['product_type'],
$filters['from'], $filters['to'], $page, $pageSize,
),
$listResolver = fn () => $repository->paginate(
$filters['shop_name'], $filters['product_type'],
$filters['from'], $filters['to'], $page, $pageSize,
);
$summary = $cache->remember(
'marketing-account-bills.alibaba.summary',
['filters' => $filters],
fn () => $repository->summary(
$filters['shop_name'], $filters['product_type'], $filters['from'], $filters['to'],
),
);
$options = $cache->remember(
'marketing-account-bills.alibaba.options',
[],
fn () => $repository->filterOptions(),
$summaryResolver = fn () => $repository->summary(
$filters['shop_name'], $filters['product_type'], $filters['from'], $filters['to'],
);
$optionsResolver = fn () => $repository->filterOptions();
$result = $usingMirror
? $listResolver()
: $cache->remember(
'marketing-account-bills.alibaba.list',
['filters' => $filters, 'page' => $page, 'limit' => $pageSize],
$listResolver,
);
$summary = $usingMirror
? $summaryResolver()
: $cache->remember(
'marketing-account-bills.alibaba.summary',
['filters' => $filters],
$summaryResolver,
);
$options = $usingMirror
? $optionsResolver()
: $cache->remember(
'marketing-account-bills.alibaba.options',
[],
$optionsResolver,
);
} catch (ServiceUnavailableHttpException $exception) {
return [
'view' => 'banking.marketing_alibaba_bills.partials.embedded-table',
@ -966,33 +986,43 @@ class Pages extends Controller
'flow_type' => $flowType !== '' ? $flowType : null,
];
$result = $cache->remember(
'taobao-alipay-bills.list',
['filters' => $filters, 'page' => $page, 'limit' => $pageSize],
fn () => $repository->paginate(
$filters['store_name'],
$filters['from'],
$filters['to'],
$filters['flow_type'],
$page,
$pageSize
)
$listResolver = fn () => $repository->paginate(
$filters['store_name'],
$filters['from'],
$filters['to'],
$filters['flow_type'],
$page,
$pageSize
);
$summary = $cache->remember(
'taobao-alipay-bills.summary',
['filters' => $filters],
fn () => $repository->summary(
$filters['store_name'],
$filters['from'],
$filters['to'],
$filters['flow_type']
)
);
$storeOptions = $cache->remember(
'taobao-alipay-bills.options',
[],
fn () => $repository->storeOptions()
$summaryResolver = fn () => $repository->summary(
$filters['store_name'],
$filters['from'],
$filters['to'],
$filters['flow_type']
);
$optionsResolver = fn () => $repository->storeOptions();
$usingMirror = $repository->usesBusinessMirror();
$result = $usingMirror
? $listResolver()
: $cache->remember(
'taobao-alipay-bills.list',
['filters' => $filters, 'page' => $page, 'limit' => $pageSize],
$listResolver,
);
$summary = $usingMirror
? $summaryResolver()
: $cache->remember(
'taobao-alipay-bills.summary',
['filters' => $filters],
$summaryResolver,
);
$storeOptions = $usingMirror
? $optionsResolver()
: $cache->remember(
'taobao-alipay-bills.options',
[],
$optionsResolver,
);
$bills = new LengthAwarePaginator(
Collection::make($result['rows']),
@ -1041,34 +1071,43 @@ class Pages extends Controller
];
// 页面只读取共享查询快照;冷查询由统一限流和熔断链路访问上游。
$result = $cache->remember(
'pinduoduo-bills.list',
['filters' => $filters, 'page' => $page, 'limit' => $pageSize],
fn () => $repository->paginate(
$filters['store_name'],
$filters['from'],
$filters['to'],
$filters['flow_type'],
$page,
$pageSize
)
$listResolver = fn () => $repository->paginate(
$filters['store_name'],
$filters['from'],
$filters['to'],
$filters['flow_type'],
$page,
$pageSize
);
$summary = $cache->remember(
'pinduoduo-bills.summary',
['filters' => $filters],
fn () => $repository->summary(
$filters['store_name'],
$filters['from'],
$filters['to'],
$filters['flow_type']
)
);
$storeOptions = $cache->remember(
'pinduoduo-bills.options',
[],
fn () => $repository->storeOptions()
$summaryResolver = fn () => $repository->summary(
$filters['store_name'],
$filters['from'],
$filters['to'],
$filters['flow_type']
);
$optionsResolver = fn () => $repository->storeOptions();
$usingMirror = $repository->usesBusinessMirror();
$result = $usingMirror
? $listResolver()
: $cache->remember(
'pinduoduo-bills.list',
['filters' => $filters, 'page' => $page, 'limit' => $pageSize],
$listResolver,
);
$summary = $usingMirror
? $summaryResolver()
: $cache->remember(
'pinduoduo-bills.summary',
['filters' => $filters],
$summaryResolver,
);
$storeOptions = $usingMirror
? $optionsResolver()
: $cache->remember(
'pinduoduo-bills.options',
[],
$optionsResolver,
);
// 包装成 Laravel 分页器,便于 Blade 用 $bills->links()
$bills = new LengthAwarePaginator(
@ -1114,9 +1153,10 @@ class Pages extends Controller
$cache = $this->upstreamCaches->forSource('jiansun');
$scopePrefix = 'jd-wallet-bills.' . $ledger;
$errors = [];
$remember = function (string $scope, array $parameters, callable $resolver, mixed $fallback) use ($cache, &$errors): mixed {
$usingMirror = $repository->usesBusinessMirror($ledger);
$remember = function (string $scope, array $parameters, callable $resolver, mixed $fallback) use ($cache, $usingMirror, &$errors): mixed {
try {
return $cache->remember($scope, $parameters, $resolver);
return $usingMirror ? $resolver() : $cache->remember($scope, $parameters, $resolver);
} catch (ServiceUnavailableHttpException $exception) {
$errors[] = $exception->getMessage();
} catch (Throwable $exception) {
@ -1145,12 +1185,19 @@ class Pages extends Controller
: $repository->summaryRecharge($filters['store_name'], $filters['from'], $filters['to']),
['amount_total' => 0, 'balance_total' => 0, 'count' => 0],
);
$shopOptions = $remember(
'jd-wallet-bills.options',
[],
fn () => $repository->shopOptions(),
[],
);
try {
$optionsResolver = fn () => $repository->shopOptions();
$shopOptions = $repository->usesBusinessMirrorForOptions()
? $optionsResolver()
: $cache->remember('jd-wallet-bills.options', [], $optionsResolver);
} catch (ServiceUnavailableHttpException $exception) {
$errors[] = $exception->getMessage();
$shopOptions = [];
} catch (Throwable $exception) {
report($exception);
$errors[] = '京东账单数据暂时无法读取,请稍后重试。';
$shopOptions = [];
}
return [
'view' => 'banking.marketing_jd_wallet_bills.partials.embedded-table',

View File

@ -18,4 +18,68 @@ return [
// 上游完成记录通常不再变化;仍回看 48 小时,覆盖迟到或修正数据。
'overlap_hours' => (int) env('PRODUCT_PRICE_MIRROR_OVERLAP_HOURS', 48),
],
/*
|--------------------------------------------------------------------------
| 上游原表镜像目录
|--------------------------------------------------------------------------
|
| 页面不能再按“页码 + 筛选条件”临时生成一份快照。这里登记项目代码实际
| 使用的上游表/视图,由后台按表同步到 caiwu_business_mirror。首次全量
| 必须显式执行;日常任务只续传已完成首次全量的数据集。
|
| watermark_column 为空时按稳定业务键追加,并依靠周期全量校准修正历史
| 更新和删除;存在可靠更新时间时使用重叠窗口增量同步。
*/
'table_sync' => [
'batch_size' => (int) env('BUSINESS_MIRROR_TABLE_BATCH_SIZE', 500),
'max_batches_per_run' => (int) env('BUSINESS_MIRROR_TABLE_MAX_BATCHES', 2),
'max_datasets_per_run' => (int) env('BUSINESS_MIRROR_MAX_DATASETS', 1),
'overlap_hours' => (int) env('BUSINESS_MIRROR_OVERLAP_HOURS', 48),
'full_reconcile_hours' => (int) env('BUSINESS_MIRROR_FULL_RECONCILE_HOURS', 168),
],
'datasets' => [
// 经营分析副本库:组织资料、订单、退款和聚水潭基础数据。
'operations_replica.rpt.fys_companyInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'fys_companyInfo', 'target' => 'ops_fys_company_info', 'keys' => ['Id'], 'watermark' => 'updateTIme'],
'operations_replica.rpt.fys_shopInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'fys_shopInfo', 'target' => 'ops_fys_shop_info', 'keys' => ['Id'], 'watermark' => 'updateTime'],
'operations_replica.rpt.fys_TrademarkInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'fys_TrademarkInfo', 'target' => 'ops_fys_trademark_info', 'keys' => ['Id'], 'watermark' => 'updateTime'],
'operations_replica.rpt.jst_Orders' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_Orders', 'target' => 'ops_jst_orders', 'keys' => ['Id'], 'watermark' => 'updateTime'],
'operations_replica.rpt.jst_OrderItem' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_OrderItem', 'target' => 'ops_jst_order_item', 'keys' => ['Id']],
'operations_replica.rpt.jst_sku' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_sku', 'target' => 'ops_jst_sku', 'keys' => ['autoid']],
'operations_replica.rpt.jst_warehouseInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_warehouseInfo', 'target' => 'ops_jst_warehouse_info', 'keys' => ['Id'], 'watermark' => 'updateTime'],
'operations_replica.rpt.jst_skumap' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_skumap', 'target' => 'ops_jst_sku_map', 'keys' => ['shop_id', 'shop_i_id', 'shop_sku_id'], 'watermark' => 'modifiedTime'],
'operations_replica.rpt.ads_itemid_sales_cost_ratio_v2_di' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'ads_itemid_sales_cost_ratio_v2_di', 'target' => 'ops_item_sales_cost_ratio', 'keys' => ['stat_date', 'item_id'], 'watermark' => 'item_id_update'],
'operations_replica.rpt.jst_returnAndRefundInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_returnAndRefundInfo', 'target' => 'ops_jst_return_refund_info', 'keys' => ['Id'], 'watermark' => 'updateTime'],
'operations_replica.rpt.jst_returnAndRefundItem' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_returnAndRefundItem', 'target' => 'ops_jst_return_refund_item', 'keys' => ['Id']],
'operations_replica.rpt.jst_shopInfo' => ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => 'jst_shopInfo', 'target' => 'ops_jst_shop_info', 'keys' => ['Id'], 'watermark' => 'updateTime'],
// JianSun 旧业务库:营销账单、供应商应付、板材、封边条和库存。
'jiansun.dbo.tmall_spread_bills' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'tmall_spread_bills', 'target' => 'js_tmall_spread_bills', 'keys' => ['id']],
'jiansun.dbo.pdd_spread_bills' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pdd_spread_bills', 'target' => 'js_pdd_spread_bills', 'keys' => ['id'], 'watermark' => 'createTime'],
'jiansun.dbo.alibaba_bill' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'alibaba_bill', 'target' => 'js_alibaba_bill', 'keys' => ['id']],
'jiansun.dbo.jd_spreadBills' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'jd_spreadBills', 'target' => 'js_jd_spread_bills', 'keys' => ['id'], 'watermark' => 'updateTime'],
'jiansun.dbo.jd_costBills' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'jd_costBills', 'target' => 'js_jd_cost_bills', 'keys' => ['id'], 'watermark' => 'creatTime'],
'jiansun.dbo.finance_purchaseReturn' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_purchaseReturn', 'target' => 'js_finance_purchase_return', 'keys' => ['id']],
'jiansun.dbo.finance_invoiceDeduction' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_invoiceDeduction', 'target' => 'js_finance_invoice_deduction', 'keys' => ['id'], 'watermark' => 'createTime'],
'jiansun.dbo.fys_supplierPaymentSummary' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'fys_supplierPaymentSummary', 'target' => 'js_supplier_payment_summary', 'keys' => ['id'], 'watermark' => 'modifiedTime'],
'jiansun.dbo.finance_advanceLoans' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_advanceLoans', 'target' => 'js_finance_advance_loans', 'keys' => ['id'], 'watermark' => 'createTime'],
'jiansun.dbo.pm_unOrderDetails' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_unOrderDetails', 'target' => 'js_pm_unorder_details', 'keys' => ['Id']],
'jiansun.dbo.finance_sampleDeduction' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_sampleDeduction', 'target' => 'js_finance_sample_deduction', 'keys' => ['id'], 'watermark' => 'finishTime'],
'jiansun.dbo.fys_express_courier_settlement' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'fys_express_courier_settlement', 'target' => 'js_express_courier_settlement', 'keys' => ['Id'], 'watermark' => 'modify_time'],
'jiansun.dbo.finance_boardDeduction' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_boardDeduction', 'target' => 'js_finance_board_deduction', 'keys' => ['id']],
'jiansun.dbo.pm_plainPlateData' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_plainPlateData', 'target' => 'js_pm_plain_plate_data', 'keys' => ['Id'], 'watermark' => 'createTime'],
'jiansun.dbo.pm_boardPurchasePrice' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_boardPurchasePrice', 'target' => 'js_pm_board_purchase_price', 'keys' => ['id'], 'watermark' => 'updateTime'],
'jiansun.dbo.pm_boardPriceChangeImport' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_boardPriceChangeImport', 'target' => 'js_pm_board_price_change', 'keys' => ['id'], 'watermark' => 'createTime'],
'jiansun.dbo.pm_boardApplyDetail' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_boardApplyDetail', 'target' => 'js_pm_board_apply_detail', 'keys' => ['Id']],
'jiansun.dbo.pm_edgeBandData' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_edgeBandData', 'target' => 'js_pm_edge_band_data', 'keys' => ['id'], 'watermark' => 'createTime'],
'jiansun.dbo.pm_edgeBandApplyDetail' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_edgeBandApplyDetail', 'target' => 'js_pm_edge_band_apply_detail', 'keys' => ['Id']],
'jiansun.dbo.pm_edgeBandProfitInfo' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_edgeBandProfitInfo', 'target' => 'js_pm_edge_band_profit_info', 'keys' => ['id']],
'jiansun.dbo.finance_edgeBandDeductio' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_edgeBandDeductio', 'target' => 'js_finance_edge_band_deduction', 'keys' => ['id']],
'jiansun.dbo.pm_averageUnitPrice' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_averageUnitPrice', 'target' => 'js_pm_average_unit_price', 'keys' => ['id']],
'jiansun.dbo.pm_boardInventoryDetail_snapshot' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_boardInventoryDetail_snapshot', 'target' => 'js_pm_board_inventory_snapshot', 'keys' => ['id']],
'jiansun.dbo.fys_supplierGoods_sttlement_purchase' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'fys_supplierGoods_sttlement_purchase', 'target' => 'js_supplier_goods_purchase', 'keys' => ['Id'], 'watermark' => 'createTime'],
'jiansun.dbo.fys_supplierGoods_sttlement_sale' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'fys_supplierGoods_sttlement_sale', 'target' => 'js_supplier_goods_sale', 'keys' => ['Id'], 'watermark' => 'createTime'],
'jiansun.dbo.pm_purchaseIn' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'pm_purchaseIn', 'target' => 'js_pm_purchase_in', 'keys' => ['id'], 'watermark' => 'updateTime'],
],
];

View File

@ -0,0 +1,62 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
private function connection(): string
{
return (string) config('upstream_mirrors.connection', 'business_mirror');
}
public function up(): void
{
if (app()->environment('testing')
|| (string) config('system_database.mode', 'bootstrap') === 'bootstrap') {
return;
}
$schema = Schema::connection($this->connection());
if ($schema->hasTable('business_mirror_table_states')) {
return;
}
$schema->create('business_mirror_table_states', function (Blueprint $table): void {
$table->string('dataset_key', 191)->primary();
$table->string('source_key', 64);
$table->string('source_object', 191);
$table->string('target_table', 64);
$table->string('status', 24)->default('pending');
$table->string('run_mode', 24)->nullable();
$table->uuid('generation')->nullable();
$table->json('cursor')->nullable();
$table->json('high_watermark')->nullable();
$table->dateTime('window_started_at', 6)->nullable();
$table->dateTime('window_ends_at', 6)->nullable();
$table->dateTime('initial_completed_at')->nullable();
$table->dateTime('last_full_succeeded_at')->nullable();
$table->dateTime('last_attempted_at')->nullable();
$table->dateTime('last_succeeded_at')->nullable();
$table->unsignedBigInteger('last_upserted_rows')->default(0);
$table->unsignedBigInteger('mirror_row_count')->default(0);
$table->string('last_error_code', 120)->nullable();
$table->timestamps();
$table->index(['source_key', 'status'], 'mirror_table_states_source_status');
$table->index('last_succeeded_at', 'mirror_table_states_last_success');
});
}
public function down(): void
{
if (app()->environment('testing')
|| (string) config('system_database.mode', 'bootstrap') === 'bootstrap') {
return;
}
Schema::connection($this->connection())->dropIfExists('business_mirror_table_states');
}
};

View File

@ -128,7 +128,7 @@ mv "$REMOTE_DIR/.deployed-sha.tmp" "$REMOTE_DIR/.deployed-sha"
# 只运行已登记的上游后台读取,不启用整个 Laravel 调度器,避免意外启动
# 账单提醒等既有任务。同一把锁保证快照与业务镜像不会并发施压上游。
cat > /etc/cron.d/caiwu-jiansun-snapshots <<'EOF'
* * * * * root /usr/bin/flock -n /run/caiwu-upstream-sync.lock /bin/sh -c '/usr/bin/docker exec -u www-data caiwu_app php artisan upstream:snapshots:refresh-all --quiet; /usr/bin/docker exec -u www-data caiwu_app php artisan business-mirror:sync-product-prices --quiet' >/dev/null 2>&1
* * * * * root /usr/bin/flock -n /run/caiwu-upstream-sync.lock /bin/sh -c '/usr/bin/docker exec -u www-data caiwu_app php artisan upstream:snapshots:refresh-all --quiet; /usr/bin/docker exec -u www-data caiwu_app php artisan business-mirror:sync-product-prices --quiet; /usr/bin/docker exec -u www-data caiwu_app php artisan business-mirror:sync-tables --quiet' >/dev/null 2>&1
EOF
chmod 644 /etc/cron.d/caiwu-jiansun-snapshots

View File

@ -132,7 +132,7 @@ mv "$REMOTE_DIR/.deployed-sha.tmp" "$REMOTE_DIR/.deployed-sha"
# 只运行已登记的上游后台读取,不启用整个 Laravel 调度器,避免意外启动
# 账单提醒等既有任务。同一把锁保证快照与业务镜像不会并发施压上游。
cat > /etc/cron.d/caiwu-jiansun-snapshots <<'EOF'
* * * * * root /usr/bin/flock -n /run/caiwu-upstream-sync.lock /bin/sh -c '/usr/bin/docker exec -u www-data caiwu_app php artisan upstream:snapshots:refresh-all --quiet; /usr/bin/docker exec -u www-data caiwu_app php artisan business-mirror:sync-product-prices --quiet' >/dev/null 2>&1
* * * * * root /usr/bin/flock -n /run/caiwu-upstream-sync.lock /bin/sh -c '/usr/bin/docker exec -u www-data caiwu_app php artisan upstream:snapshots:refresh-all --quiet; /usr/bin/docker exec -u www-data caiwu_app php artisan business-mirror:sync-product-prices --quiet; /usr/bin/docker exec -u www-data caiwu_app php artisan business-mirror:sync-tables --quiet' >/dev/null 2>&1
EOF
chmod 644 /etc/cron.d/caiwu-jiansun-snapshots

View File

@ -0,0 +1,60 @@
# 上游业务镜像清单
## 结论
项目不是只使用 `rpt.Bulk_Price_Limit` 一张表。当前主线代码直接读取两套上游中的约 40 个表或视图,覆盖组织资料、订单、退款、营销账户、供应商应付、板材、封边条和库存。
`caiwu_business_mirror` 的目标是把项目实际使用的上游原表数据保存为共享业务镜像。网页只读镜像,后台刷新节点在低压、可断点的条件下完成首次全量和后续增量;不得再按页码或筛选条件临时生成不完整的数据副本。
## 当前迁移状态
| 状态 | 说明 |
| --- | --- |
| 已完成 | `operations_replica/rpt.Bulk_Price_Limit` 已进入类型化表 `product_price_mirror_rows`,商品价格页面已读取镜像并支持完整分页。 |
| 本次建立 | `config/upstream_mirrors.php` 中登记全部当前代码引用的原表;通用表级同步框架负责建表、稳定游标、断点续传、增量重叠和全量校准。 |
| 已开始逐域切换 | 公司、店铺、商标页面在对应镜像完成首次全量后自动改读完整镜像表;首次全量前保留旧快照作为兼容回滚。其他业务域仍需逐项切换,不得宣称全项目已经完成。 |
## 经营分析副本库(operations_replica)
| 业务范围 | 上游对象 | 镜像目标 |
| --- | --- | --- |
| 公司资料 | `rpt.fys_companyInfo` | `ops_fys_company_info` |
| 店铺资料 | `rpt.fys_shopInfo` | `ops_fys_shop_info` |
| 商标资料 | `rpt.fys_TrademarkInfo` | `ops_fys_trademark_info` |
| 订单 | `rpt.jst_Orders`、`rpt.jst_OrderItem` | `ops_jst_orders`、`ops_jst_order_item` |
| 商品与仓库 | `rpt.jst_sku`、`rpt.jst_warehouseInfo`、`rpt.jst_skumap` | `ops_jst_*` |
| 推广费用 | `rpt.ads_itemid_sales_cost_ratio_v2_di` | `ops_item_sales_cost_ratio` |
| 退款退货 | `rpt.jst_returnAndRefundInfo`、`rpt.jst_returnAndRefundItem` | `ops_jst_return_refund_*` |
| 聚水潭店铺 | `rpt.jst_shopInfo` | `ops_jst_shop_info` |
商品价格 `rpt.Bulk_Price_Limit` 继续使用已经上线的类型化同步器,不在通用原表任务中重复同步。
## JianSun 旧业务库(jiansun)
| 业务范围 | 上游对象 |
| --- | --- |
| 营销账户 | `dbo.tmall_spread_bills`、`dbo.pdd_spread_bills`、`dbo.alibaba_bill`、`dbo.jd_spreadBills`、`dbo.jd_costBills` |
| 供应商应付 | `dbo.finance_purchaseReturn`、`dbo.finance_invoiceDeduction`、`dbo.fys_supplierPaymentSummary`、`dbo.finance_advanceLoans`、`dbo.pm_unOrderDetails`、`dbo.finance_sampleDeduction`、`dbo.fys_express_courier_settlement` |
| 板材 | `dbo.finance_boardDeduction`、`dbo.pm_plainPlateData`、`dbo.pm_boardPurchasePrice`、`dbo.pm_boardPriceChangeImport`、`dbo.pm_boardApplyDetail` |
| 封边条 | `dbo.pm_edgeBandData`、`dbo.pm_edgeBandApplyDetail`、`dbo.pm_edgeBandProfitInfo`、`dbo.finance_edgeBandDeductio`、`dbo.pm_averageUnitPrice` |
| 库存与货款对账 | `dbo.pm_boardInventoryDetail_snapshot`、`dbo.fys_supplierGoods_sttlement_purchase`、`dbo.fys_supplierGoods_sttlement_sale`、`dbo.pm_purchaseIn` |
其中 `fys_supplierGoods_sttlement_sale` 当前约 280 万行,是首次全量同步的最大对象。必须在低峰期小批量、可断点执行,禁止把全量同步塞进发布流程。
## 同步规则
1. 首次全量只能通过 `business-mirror:sync-tables <dataset> --full` 显式启动;失败或中断后使用同一命令继续。
2. 日常任务只处理已经完成首次全量的数据集。存在可靠更新时间的表使用 48 小时重叠窗口;没有可靠更新时间的表按稳定主键追加。
3. 全量校准使用新的同步代次,只有全部完成后才删除镜像中本轮未出现的旧行,因此中途失败不会清空当前可用数据。
4. 每轮默认只处理 1 个数据集、每个数据集 2 批,每批 500 行,避免同时压迫两个旧系统。
5. 页面切换必须以“总记录数、第一页、第二页、末页、筛选、汇总、空数据和更新时间”作为验收,不能只看首页有数据。
## 后续切换顺序
1. 公司、店铺、商标等小型维表。
2. 营销账户五张单表账单。
3. 供应商应付与货款对账。
4. 订单、退款和推广费用。
5. 板材、封边条和库存的跨表查询。
每个阶段都必须先完成首次全量和数量核对,再将对应 Repository 切到镜像;任一阶段失败时保留上一版镜像和旧只读链路用于回滚。

View File

@ -234,6 +234,15 @@ php artisan business-mirror:sync-product-prices --full
php artisan business-mirror:sync-product-prices
```
其余已在代码中使用的上游表由统一的数据集目录管理。首次全量必须逐个数据集显式执行,普通定时任务只续传已完成首次全量的数据集:
```bash
php artisan business-mirror:sync-tables <dataset-key> --full
php artisan business-mirror:sync-tables
```
完整数据集、来源对象、镜像表和分阶段切换状态见 `docs/上游业务镜像清单.md`。不得把“已登记同步目录”误报成“页面已经切换完成”;每个业务域仍需在首份全量核对通过后,把对应 Repository 改为只读镜像。
首次 `--full` 属于生产数据写入,必须先备份 `caiwu_system_config` 和 `caiwu_business_mirror`,并由负责人明确授权。后续无参数命令只执行或继续受控增量。
### 8.3 开发者需要尚未接入的上游数据时

View File

@ -0,0 +1,21 @@
<?php
namespace Modules\EcommerceFinance\Contracts;
interface BusinessMirrorDataset
{
public function key(): string;
public function sourceKey(): string;
public function sourceSchema(): string;
public function sourceObject(): string;
public function targetTable(): string;
/** @return list<string> */
public function keyColumns(): array;
public function watermarkColumn(): ?string;
}

View File

@ -17,14 +17,25 @@ class AlibabaBillRepository
private UpstreamReadOnlyQuery $query;
public function __construct(UpstreamReadOnlyQueryFactory $factory)
{
public function __construct(
UpstreamReadOnlyQueryFactory $factory,
private readonly ?MarketingAccountBillMirrorRepository $mirror = null,
) {
$this->query = $factory->forSource(self::SOURCE);
}
public function usesBusinessMirror(): bool
{
return $this->mirror?->ready(MarketingAccountBillMirrorRepository::ALIBABA) ?? false;
}
/** @return array{shops: string[], product_types: string[]} */
public function filterOptions(): array
{
if ($this->usesBusinessMirror()) {
return $this->mirror->alibabaFilterOptions();
}
$shops = $this->query->select(
'SELECT DISTINCT TOP (500) shopName FROM ' . self::TABLE
. ' WHERE shopName IS NOT NULL AND LTRIM(RTRIM(shopName)) <> \'\' ORDER BY shopName'
@ -51,6 +62,23 @@ class AlibabaBillRepository
): array {
$page = max(1, $page);
$pageSize = min(max(1, $pageSize), 100);
if ($this->usesBusinessMirror()) {
$result = $this->mirror->paginateAlibaba(
$shopName,
$productType,
$fromDate,
$toDate,
$page,
$pageSize,
);
return [
'rows' => array_map(fn (array $row): array => $this->normalize($row), $result['rows']),
'total' => $result['total'],
];
}
[$where, $bindings] = $this->where($shopName, $productType, $fromDate, $toDate);
$total = (int) $this->query->scalar('SELECT COUNT(*) FROM ' . self::TABLE . ' src' . $where, $bindings);
@ -82,13 +110,17 @@ class AlibabaBillRepository
?string $fromDate = null,
?string $toDate = null,
): array {
[$where, $bindings] = $this->where($shopName, $productType, $fromDate, $toDate);
$row = $this->query->first(
'SELECT COALESCE(SUM(tradeAMTIn), 0) AS income_total, '
. 'COALESCE(SUM(tradeAMTOut), 0) AS expense_total, COUNT(*) AS cnt '
. 'FROM ' . self::TABLE . ' src' . $where,
$bindings,
) ?? [];
if ($this->usesBusinessMirror()) {
$row = $this->mirror->summarizeAlibaba($shopName, $productType, $fromDate, $toDate);
} else {
[$where, $bindings] = $this->where($shopName, $productType, $fromDate, $toDate);
$row = $this->query->first(
'SELECT COALESCE(SUM(tradeAMTIn), 0) AS income_total, '
. 'COALESCE(SUM(tradeAMTOut), 0) AS expense_total, COUNT(*) AS cnt '
. 'FROM ' . self::TABLE . ' src' . $where,
$bindings,
) ?? [];
}
$income = round((float) ($row['income_total'] ?? 0), 2);
$expense = round((float) ($row['expense_total'] ?? 0), 2);

View File

@ -0,0 +1,150 @@
<?php
namespace Modules\EcommerceFinance\Repositories;
use Illuminate\Database\ConnectionInterface;
use InvalidArgumentException;
use Modules\EcommerceFinance\Contracts\BusinessMirrorDataset;
use Modules\EcommerceFinance\Services\BusinessMirrorDatasetRegistry;
use RuntimeException;
class BusinessInformationMirrorRepository
{
private const DATASETS = [
'company-information.read' => 'operations_replica.rpt.fys_companyInfo',
'store-information.read' => 'operations_replica.rpt.fys_shopInfo',
'trademark-information.read' => 'operations_replica.rpt.fys_TrademarkInfo',
];
private readonly ConnectionInterface $connection;
public function __construct(
private readonly BusinessMirrorDatasetRegistry $datasets,
private readonly GenericBusinessMirrorRepository $mirror,
) {
$this->connection = $mirror->connection();
}
public function ready(string $scope): bool
{
return $this->mirror->ready($this->dataset($scope));
}
/** @return array{data: array<int,array<string,mixed>>, total: int} */
public function paginate(string $scope, int $page, int $perPage): array
{
if (! $this->ready($scope)) {
throw new RuntimeException('business_mirror_dataset_not_ready');
}
$page = max($page, 1);
$perPage = min(max($perPage, 1), 100);
return match ($scope) {
'company-information.read' => $this->companies($page, $perPage),
'store-information.read' => $this->stores($page, $perPage),
'trademark-information.read' => $this->trademarks($page, $perPage),
default => throw new InvalidArgumentException('未知的业务资料镜像范围:' . $scope),
};
}
/** @return array{data: array<int,array<string,mixed>>, total: int} */
private function companies(int $page, int $perPage): array
{
$query = $this->connection->table('ops_fys_company_info')
->selectRaw(<<<'SQL'
Id AS id,
unifiedSocialCreditCode AS unified_social_credit_code,
businessLicenseName AS company_name,
registeredCapital AS subscribed_capital,
businessLicenseType AS company_type,
IncorporationDate AS establishment_date,
legalRepresentative AS legal_representative,
registeredAddress AS registered_address,
businessScope AS business_scope,
accountSerialNumber AS basic_deposit_account_number,
companyStatus AS company_status,
financialManager AS financial_officer,
supervisor AS supervisor,
taxCategoryAndType AS taxpayer_category,
NULL AS shareholders
SQL);
return $this->page($query, $page, $perPage, 'Id');
}
/** @return array{data: array<int,array<string,mixed>>, total: int} */
private function stores(int $page, int $perPage): array
{
$latestCompanies = $this->connection->table('ops_fys_company_info')
->selectRaw('unifiedSocialCreditCode, MAX(Id) AS latest_id')
->whereNotNull('unifiedSocialCreditCode')
->where('unifiedSocialCreditCode', '<>', '')
->groupBy('unifiedSocialCreditCode');
$query = $this->connection->table('ops_fys_shop_info AS s')
->leftJoinSub($latestCompanies, 'company_keys', function ($join): void {
$join->on('company_keys.unifiedSocialCreditCode', '=', 's.unifiedSocialCreditCode');
})
->leftJoin('ops_fys_company_info AS c', 'c.Id', '=', 'company_keys.latest_id')
->selectRaw(<<<'SQL'
s.Id AS id,
s.openTime AS opened_at,
s.erpShopName AS erp_store_name,
s.unifiedSocialCreditCode AS unified_social_credit_code,
c.businessLicenseName AS company_name,
s.trademarkNumber AS trademark_number,
NULL AS trademark_name,
s.platform AS platform,
s.responsibleDepartment AS profit_center,
s.shopHead AS store_manager,
s.shopState AS store_status,
s.shopType AS store_type,
s.channelType AS channel_mode
SQL);
return $this->page($query, $page, $perPage, 's.Id');
}
/** @return array{data: array<int,array<string,mixed>>, total: int} */
private function trademarks(int $page, int $perPage): array
{
$query = $this->connection->table('ops_fys_trademark_info')
->where('isDelete', 0)
->selectRaw(<<<'SQL'
Id AS id,
trademarkName AS trademark_name,
internationalClassification AS trademark_type,
trademarkStatus AS trademark_status,
registrationDate AS registration_date,
termOfValidity AS expiry_date,
remark AS notes
SQL);
return $this->page($query, $page, $perPage, 'Id');
}
/** @return array{data: array<int,array<string,mixed>>, total: int} */
private function page($query, int $page, int $perPage, string $orderColumn): array
{
$total = (clone $query)->count();
$rows = $query->orderByDesc($orderColumn)
->forPage($page, $perPage)
->get()
->map(static fn (object $row): array => (array) $row)
->all();
return ['data' => $rows, 'total' => $total];
}
private function dataset(string $scope): BusinessMirrorDataset
{
$key = self::DATASETS[$scope] ?? null;
if ($key === null) {
throw new InvalidArgumentException('未知的业务资料镜像范围:' . $scope);
}
return $this->datasets->get($key);
}
}

View File

@ -0,0 +1,256 @@
<?php
namespace Modules\EcommerceFinance\Repositories;
use Illuminate\Database\ConnectionInterface;
use Illuminate\Database\DatabaseManager;
use Illuminate\Support\Facades\Schema;
use Modules\EcommerceFinance\Contracts\BusinessMirrorDataset;
use RuntimeException;
class GenericBusinessMirrorRepository
{
private readonly ConnectionInterface $connection;
public function __construct(DatabaseManager $database)
{
$this->connection = $database->connection(
(string) config('upstream_mirrors.connection', 'business_mirror')
);
}
public function connection(): ConnectionInterface
{
return $this->connection;
}
public function ready(BusinessMirrorDataset $dataset): bool
{
$schema = Schema::connection($this->connection->getName());
if (! $schema->hasTable('business_mirror_table_states')
|| ! $schema->hasTable($dataset->targetTable())) {
return false;
}
$state = $this->state($dataset);
return is_array($state) && ($state['initial_completed_at'] ?? null) !== null;
}
/** @return array<string,mixed>|null */
public function state(BusinessMirrorDataset $dataset): ?array
{
if (! Schema::connection($this->connection->getName())->hasTable('business_mirror_table_states')) {
return null;
}
$row = $this->connection->table('business_mirror_table_states')
->where('dataset_key', $dataset->key())
->first();
if ($row === null) {
return null;
}
$state = (array) $row;
$state['cursor'] = $this->decode($state['cursor'] ?? null);
$state['high_watermark'] = $this->decode($state['high_watermark'] ?? null);
return $state;
}
public function startRun(
BusinessMirrorDataset $dataset,
string $mode,
string $generation,
?string $windowStart,
?string $windowEnd,
bool $resume,
): array {
$now = now();
$values = [
'source_key' => $dataset->sourceKey(),
'source_object' => $dataset->sourceSchema() . '.' . $dataset->sourceObject(),
'target_table' => $dataset->targetTable(),
'status' => 'running',
'run_mode' => $mode,
'generation' => $generation,
'last_attempted_at' => $now,
'last_error_code' => null,
'updated_at' => $now,
];
if (! $resume) {
$values += [
'cursor' => null,
'window_started_at' => $windowStart,
'window_ends_at' => $windowEnd,
'last_upserted_rows' => 0,
];
}
$query = $this->connection->table('business_mirror_table_states')
->where('dataset_key', $dataset->key());
if ($query->exists()) {
$query->update($values);
} else {
$this->connection->table('business_mirror_table_states')->insert(
['dataset_key' => $dataset->key(), 'created_at' => $now] + $values,
);
}
return $this->state($dataset) ?? [];
}
public function storeBatch(
BusinessMirrorDataset $dataset,
array $columns,
array $rows,
array $cursor,
string $generation,
int $cumulativeRows,
): void {
$now = now();
$columnNames = array_map(fn (array $column): string => (string) $column['column_name'], $columns);
$mapped = [];
foreach ($rows as $row) {
$values = [];
foreach ($columnNames as $columnName) {
$values[$columnName] = $this->value($row, $columnName);
}
$identity = array_map(
fn (string $column): string => $this->identityValue($this->value($row, $column)),
$dataset->keyColumns(),
);
$mapped[] = $values + [
'_mirror_key' => hash('sha256', implode("\0", $identity)),
'_mirror_generation' => $generation,
'_mirror_synced_at' => $now,
];
}
$this->connection->transaction(function () use (
$dataset, $columnNames, $mapped, $cursor, $cumulativeRows, $now
): void {
$updates = [...$columnNames, '_mirror_generation', '_mirror_synced_at'];
$chunkSize = max(1, intdiv(50000, max(count($updates) + 1, 1)));
foreach (array_chunk($mapped, $chunkSize) as $chunk) {
$this->connection->table($dataset->targetTable())->upsert($chunk, ['_mirror_key'], $updates);
}
$this->connection->table('business_mirror_table_states')
->where('dataset_key', $dataset->key())
->update([
'cursor' => json_encode($cursor, JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION),
'last_upserted_rows' => $cumulativeRows,
'updated_at' => $now,
]);
});
}
public function complete(
BusinessMirrorDataset $dataset,
string $mode,
string $generation,
array $highWatermark,
int $rows,
): void {
$now = now();
$this->connection->transaction(function () use (
$dataset, $mode, $generation, $highWatermark, $rows, $now
): void {
if ($mode === 'full') {
$existingRows = $this->connection->table($dataset->targetTable())->count();
if ($rows === 0 && $existingRows > 0) {
throw new RuntimeException('business_mirror_unexpected_empty_source');
}
$this->connection->table($dataset->targetTable())
->where(function ($query) use ($generation): void {
$query->whereNull('_mirror_generation')
->orWhere('_mirror_generation', '!=', $generation);
})
->delete();
}
$rowCount = $this->connection->table($dataset->targetTable())->count();
$values = [
'status' => 'succeeded',
'cursor' => null,
'high_watermark' => json_encode($highWatermark, JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION),
'last_succeeded_at' => $now,
'last_upserted_rows' => $rows,
'mirror_row_count' => $rowCount,
'last_error_code' => null,
'updated_at' => $now,
];
if ($mode === 'full') {
$values['initial_completed_at'] = $now;
$values['last_full_succeeded_at'] = $now;
}
$this->connection->table('business_mirror_table_states')
->where('dataset_key', $dataset->key())
->update($values);
});
}
public function fail(BusinessMirrorDataset $dataset, string $errorCode): void
{
$this->connection->table('business_mirror_table_states')
->where('dataset_key', $dataset->key())
->update([
'status' => 'failed',
'last_error_code' => mb_substr($errorCode, 0, 120),
'updated_at' => now(),
]);
}
private function decode(mixed $value): ?array
{
if (is_array($value)) {
return $value;
}
if (! is_string($value) || $value === '') {
return null;
}
$decoded = json_decode($value, true);
return is_array($decoded) ? $decoded : null;
}
private function value(array $row, string $name): mixed
{
foreach ($row as $key => $value) {
if (strcasecmp((string) $key, $name) === 0) {
return $value;
}
}
return null;
}
private function identityValue(mixed $value): string
{
if ($value === null) {
return '<NULL>';
}
if (is_bool($value)) {
return $value ? '1' : '0';
}
return (string) $value;
}
}

View File

@ -19,14 +19,34 @@ class JdWalletBillRepository
private UpstreamReadOnlyQuery $query;
public function __construct(UpstreamReadOnlyQueryFactory $factory)
{
public function __construct(
UpstreamReadOnlyQueryFactory $factory,
private readonly ?MarketingAccountBillMirrorRepository $mirror = null,
) {
$this->query = $factory->forSource(self::SOURCE);
}
public function usesBusinessMirror(string $ledger): bool
{
$dataset = $ledger === 'consumption'
? MarketingAccountBillMirrorRepository::JD_CONSUMPTION
: MarketingAccountBillMirrorRepository::JD_RECHARGE;
return $this->mirror?->ready($dataset) ?? false;
}
public function usesBusinessMirrorForOptions(): bool
{
return $this->usesBusinessMirror('recharge') && $this->usesBusinessMirror('consumption');
}
/** @return string[] */
public function shopOptions(): array
{
if ($this->usesBusinessMirrorForOptions()) {
return $this->mirror->jdShopOptions();
}
$rows = $this->query->select(
"SELECT TOP (500) shopName FROM (SELECT shopName FROM " . self::RECHARGE_TABLE
. " UNION SELECT shopName FROM " . self::CONSUMPTION_TABLE
@ -47,6 +67,21 @@ class JdWalletBillRepository
int $page = 1,
int $pageSize = 25,
): array {
if ($this->usesBusinessMirror('recharge')) {
$result = $this->mirror->paginateJdRecharge(
$shopName,
$fromDate,
$toDate,
$page,
$pageSize,
);
return [
'rows' => array_map(fn (array $row): array => $this->normalizeRecharge($row), $result['rows']),
'total' => $result['total'],
];
}
return $this->paginate(self::RECHARGE_TABLE, $this->rechargeWhere($shopName, $fromDate, $toDate), $page, $pageSize, [
'id', 'shopName', 'amount', 'balanceAmount', 'orderTypeName', 'pk',
'createDate', 'updateTime', 'statusName', 'remark',
@ -61,6 +96,21 @@ class JdWalletBillRepository
int $page = 1,
int $pageSize = 25,
): array {
if ($this->usesBusinessMirror('consumption')) {
$result = $this->mirror->paginateJdConsumption(
$shopName,
$fromDate,
$toDate,
$page,
$pageSize,
);
return [
'rows' => array_map(fn (array $row): array => $this->normalizeConsumption($row), $result['rows']),
'total' => $result['total'],
];
}
return $this->paginate(self::CONSUMPTION_TABLE, $this->consumptionWhere($shopName, $fromDate, $toDate), $page, $pageSize, [
'id', 'shopName', 'amount', 'balanceAmount', 'orderTypeName', 'creatTime', 'settleTime', 'pk',
'invoiceApplyTypeName',
@ -70,12 +120,32 @@ class JdWalletBillRepository
/** @return array{amount_total: float, balance_total: float, count: int} */
public function summaryRecharge(?string $shopName = null, ?string $fromDate = null, ?string $toDate = null): array
{
if ($this->usesBusinessMirror('recharge')) {
return $this->normalizeSummary($this->mirror->summarizeJd(
MarketingAccountBillMirrorRepository::JD_RECHARGE,
'createDate',
$shopName,
$fromDate,
$toDate,
));
}
return $this->summary(self::RECHARGE_TABLE, $this->rechargeWhere($shopName, $fromDate, $toDate));
}
/** @return array{amount_total: float, balance_total: float, count: int} */
public function summaryConsumption(?string $shopName = null, ?string $fromDate = null, ?string $toDate = null): array
{
if ($this->usesBusinessMirror('consumption')) {
return $this->normalizeSummary($this->mirror->summarizeJd(
MarketingAccountBillMirrorRepository::JD_CONSUMPTION,
'creatTime',
$shopName,
$fromDate,
$toDate,
));
}
return $this->summary(self::CONSUMPTION_TABLE, $this->consumptionWhere($shopName, $fromDate, $toDate));
}
@ -111,6 +181,12 @@ class JdWalletBillRepository
$where['bindings'],
) ?? [];
return $this->normalizeSummary($row);
}
/** @return array{amount_total: float, balance_total: float, count: int} */
private function normalizeSummary(array $row): array
{
return [
'amount_total' => round((float) ($row['amount_total'] ?? 0), 2),
'balance_total' => round((float) ($row['balance_total'] ?? 0), 2),

View File

@ -0,0 +1,543 @@
<?php
namespace Modules\EcommerceFinance\Repositories;
use Illuminate\Database\ConnectionInterface;
use Illuminate\Database\Query\Builder;
use Modules\EcommerceFinance\Contracts\BusinessMirrorDataset;
use Modules\EcommerceFinance\Services\BusinessMirrorDatasetRegistry;
use RuntimeException;
use Throwable;
/**
* 营销账户账单的完整业务镜像读取层。
*
* 这里只返回与旧 SQL Server Repository 相同的原始字段;金额换算和业务名称
* 仍由原 Repository 负责,确保镜像切换不会改变页面口径。
*/
class MarketingAccountBillMirrorRepository
{
public const TAOBAO = 'jiansun.dbo.tmall_spread_bills';
public const PINDUODUO = 'jiansun.dbo.pdd_spread_bills';
public const ALIBABA = 'jiansun.dbo.alibaba_bill';
public const JD_RECHARGE = 'jiansun.dbo.jd_spreadBills';
public const JD_CONSUMPTION = 'jiansun.dbo.jd_costBills';
private readonly ConnectionInterface $connection;
/** @var array<string,bool> */
private array $readiness = [];
public function __construct(
private readonly BusinessMirrorDatasetRegistry $datasets,
private readonly GenericBusinessMirrorRepository $mirror,
) {
$this->connection = $mirror->connection();
}
public function ready(string $datasetKey): bool
{
if (array_key_exists($datasetKey, $this->readiness)) {
return $this->readiness[$datasetKey];
}
try {
return $this->readiness[$datasetKey] = $this->mirror->ready($this->dataset($datasetKey));
} catch (Throwable $exception) {
report($exception);
return $this->readiness[$datasetKey] = false;
}
}
/** @return string[] */
public function taobaoStoreOptions(): array
{
return $this->stringOptions(self::TAOBAO, 'shopName');
}
/** @return array{rows:array<int,array<string,mixed>>,total:int} */
public function paginateTaobao(
?string $storeName,
?string $fromDate,
?string $toDate,
?string $flowType,
int $page,
int $pageSize,
): array {
$query = $this->table(self::TAOBAO, 'src');
$this->applyTaobaoFilters($query, $storeName, $fromDate, $toDate, $flowType);
return $this->paginate($query, [
'src.id', 'src.shopName', 'src.uniTransTime', 'src.transTime', 'src.amount',
'src.balance', 'src.comment', 'src.finType', 'src.bizType', 'src.tradeType',
], ['src.uniTransTime', 'src.id'], $page, min(max($pageSize, 1), 200));
}
/** @return array{income_total_raw:float,expense_total_raw:float,cnt:int} */
public function summarizeTaobao(
?string $storeName,
?string $fromDate,
?string $toDate,
?string $flowType,
): array {
$query = $this->table(self::TAOBAO, 'src');
$this->applyTaobaoFilters($query, $storeName, $fromDate, $toDate, $flowType);
$row = $query->selectRaw(
"COALESCE(SUM(CASE WHEN src.finType = 'DEBIT' THEN src.amount ELSE 0 END), 0) AS income_total_raw, "
. "COALESCE(SUM(CASE WHEN src.finType = 'CREDIT' THEN src.amount ELSE 0 END), 0) AS expense_total_raw, "
. 'COUNT(*) AS cnt'
)->first();
return [
'income_total_raw' => (float) ($row->income_total_raw ?? 0),
'expense_total_raw' => (float) ($row->expense_total_raw ?? 0),
'cnt' => (int) ($row->cnt ?? 0),
];
}
/** @return array{rows:array<int,array<string,mixed>>,total:int} */
public function paginateTaobaoMonthlyBalances(
?string $storeName,
?string $month,
int $page,
int $pageSize,
): array {
return $this->monthlyBalances(
self::TAOBAO,
'transTime',
"CASE WHEN src.finType = 'DEBIT' THEN src.amount ELSE 0 END",
"CASE WHEN src.finType = 'CREDIT' THEN src.amount ELSE 0 END",
$storeName,
$month,
$page,
min(max($pageSize, 1), 5000),
'opening_balance_raw',
'income_total_raw',
'expense_total_raw',
'system_closing_balance_raw',
);
}
/** @return string[] */
public function pinduoduoStoreOptions(): array
{
return $this->stringOptions(self::PINDUODUO, 'shopName');
}
/** @return array{rows:array<int,array<string,mixed>>,total:int} */
public function paginatePinduoduo(
?string $storeName,
?string $fromDate,
?string $toDate,
?string $flowType,
int $page,
int $pageSize,
): array {
$query = $this->table(self::PINDUODUO, 'src');
$this->applyPinduoduoFilters($query, $storeName, $fromDate, $toDate, $flowType);
return $this->paginate($query, [
'src.id', 'src.shopName', 'src.amount', 'src.balance', 'src.summary',
'src.flowType', 'src.toAdvertiserFundType', 'src.createTime',
], ['src.createTime', 'src.id'], $page, min(max($pageSize, 1), 200));
}
/** @return array{income_total:float,expense_total:float,cnt:int} */
public function summarizePinduoduo(
?string $storeName,
?string $fromDate,
?string $toDate,
?string $flowType,
): array {
$query = $this->table(self::PINDUODUO, 'src');
$this->applyPinduoduoFilters($query, $storeName, $fromDate, $toDate, $flowType);
$row = $query->selectRaw(
'COALESCE(SUM(CASE WHEN src.flowType = 1 THEN src.amount ELSE 0 END), 0) AS income_total, '
. 'COALESCE(SUM(CASE WHEN src.flowType = 2 THEN src.amount ELSE 0 END), 0) AS expense_total, '
. 'COUNT(*) AS cnt'
)->first();
return [
'income_total' => (float) ($row->income_total ?? 0),
'expense_total' => (float) ($row->expense_total ?? 0),
'cnt' => (int) ($row->cnt ?? 0),
];
}
/** @return array{rows:array<int,array<string,mixed>>,total:int} */
public function paginatePinduoduoMonthlyBalances(
?string $storeName,
?string $month,
int $page,
int $pageSize,
): array {
return $this->monthlyBalances(
self::PINDUODUO,
'createTime',
'CASE WHEN src.flowType = 1 THEN src.amount ELSE 0 END',
'CASE WHEN src.flowType = 2 THEN src.amount ELSE 0 END',
$storeName,
$month,
$page,
min(max($pageSize, 1), 5000),
'opening_balance',
'income_total',
'expense_total',
'system_closing_balance',
);
}
/** @return array{shops:string[],product_types:string[]} */
public function alibabaFilterOptions(): array
{
return [
'shops' => $this->stringOptions(self::ALIBABA, 'shopName', 500),
'product_types' => $this->stringOptions(self::ALIBABA, 'tradeTypeName', 500),
];
}
/** @return array{rows:array<int,array<string,mixed>>,total:int} */
public function paginateAlibaba(
?string $shopName,
?string $productType,
?string $fromDate,
?string $toDate,
int $page,
int $pageSize,
): array {
$query = $this->table(self::ALIBABA, 'src');
$this->applyAlibabaFilters($query, $shopName, $productType, $fromDate, $toDate);
return $this->paginate($query, [
'src.id', 'src.shopName', 'src.tradeDay', 'src.tradeTypeName',
'src.tradeAMTIn', 'src.tradeAMTOut',
], ['src.tradeDay', 'src.id'], $page, min(max($pageSize, 1), 100));
}
/** @return array{income_total:float,expense_total:float,cnt:int} */
public function summarizeAlibaba(
?string $shopName,
?string $productType,
?string $fromDate,
?string $toDate,
): array {
$query = $this->table(self::ALIBABA, 'src');
$this->applyAlibabaFilters($query, $shopName, $productType, $fromDate, $toDate);
$row = $query->selectRaw(
'COALESCE(SUM(src.tradeAMTIn), 0) AS income_total, '
. 'COALESCE(SUM(src.tradeAMTOut), 0) AS expense_total, COUNT(*) AS cnt'
)->first();
return [
'income_total' => (float) ($row->income_total ?? 0),
'expense_total' => (float) ($row->expense_total ?? 0),
'cnt' => (int) ($row->cnt ?? 0),
];
}
/** @return string[] */
public function jdShopOptions(): array
{
$shops = array_merge(
$this->stringOptions(self::JD_RECHARGE, 'shopName', 500),
$this->stringOptions(self::JD_CONSUMPTION, 'shopName', 500),
);
$shops = array_values(array_unique($shops));
sort($shops, SORT_NATURAL);
return array_slice($shops, 0, 500);
}
/** @return array{rows:array<int,array<string,mixed>>,total:int} */
public function paginateJdRecharge(
?string $shopName,
?string $fromDate,
?string $toDate,
int $page,
int $pageSize,
): array {
return $this->paginateJd(
self::JD_RECHARGE,
'createDate',
['src.id', 'src.shopName', 'src.amount', 'src.balanceAmount', 'src.orderTypeName',
'src.pk', 'src.createDate', 'src.updateTime', 'src.statusName', 'src.remark'],
$shopName,
$fromDate,
$toDate,
$page,
$pageSize,
);
}
/** @return array{rows:array<int,array<string,mixed>>,total:int} */
public function paginateJdConsumption(
?string $shopName,
?string $fromDate,
?string $toDate,
int $page,
int $pageSize,
): array {
return $this->paginateJd(
self::JD_CONSUMPTION,
'creatTime',
['src.id', 'src.shopName', 'src.amount', 'src.balanceAmount', 'src.orderTypeName',
'src.creatTime', 'src.settleTime', 'src.pk', 'src.invoiceApplyTypeName'],
$shopName,
$fromDate,
$toDate,
$page,
$pageSize,
);
}
/** @return array{amount_total:float,balance_total:float,cnt:int} */
public function summarizeJd(
string $datasetKey,
string $dateField,
?string $shopName,
?string $fromDate,
?string $toDate,
): array {
$query = $this->table($datasetKey, 'src');
$this->applyJdFilters($query, $dateField, $shopName, $fromDate, $toDate);
$row = $query->selectRaw(
'COALESCE(SUM(ABS(src.amount)), 0) AS amount_total, '
. 'COALESCE(SUM(src.balanceAmount), 0) AS balance_total, COUNT(*) AS cnt'
)->first();
return [
'amount_total' => (float) ($row->amount_total ?? 0),
'balance_total' => (float) ($row->balance_total ?? 0),
'cnt' => (int) ($row->cnt ?? 0),
];
}
/** @return array{rows:array<int,array<string,mixed>>,total:int} */
private function paginateJd(
string $datasetKey,
string $dateField,
array $fields,
?string $shopName,
?string $fromDate,
?string $toDate,
int $page,
int $pageSize,
): array {
$query = $this->table($datasetKey, 'src');
$this->applyJdFilters($query, $dateField, $shopName, $fromDate, $toDate);
return $this->paginate(
$query,
$fields,
['src.' . $dateField, 'src.id'],
$page,
min(max($pageSize, 1), 100),
);
}
private function applyTaobaoFilters(
Builder $query,
?string $storeName,
?string $fromDate,
?string $toDate,
?string $flowType,
): void {
$this->whereValue($query, 'src.shopName', $storeName);
$this->whereFromDate($query, 'src.transTime', $fromDate);
$this->whereToDate($query, 'src.transTime', $toDate);
if ($flowType === 'income') {
$query->where('src.finType', 'DEBIT');
} elseif ($flowType === 'expense') {
$query->where('src.finType', 'CREDIT');
}
}
private function applyPinduoduoFilters(
Builder $query,
?string $storeName,
?string $fromDate,
?string $toDate,
?string $flowType,
): void {
$this->whereValue($query, 'src.shopName', $storeName);
$this->whereFromDate($query, 'src.createTime', $fromDate, true);
$this->whereToDate($query, 'src.createTime', $toDate);
if ($flowType === 'income') {
$query->where('src.flowType', 1);
} elseif ($flowType === 'expense') {
$query->where('src.flowType', 2);
}
}
private function applyAlibabaFilters(
Builder $query,
?string $shopName,
?string $productType,
?string $fromDate,
?string $toDate,
): void {
$this->whereValue($query, 'src.shopName', $shopName);
$this->whereValue($query, 'src.tradeTypeName', $productType);
if ($this->validDate($fromDate)) {
$query->whereDate('src.tradeDay', '>=', $fromDate);
}
if ($this->validDate($toDate)) {
$query->whereDate('src.tradeDay', '<=', $toDate);
}
}
private function applyJdFilters(
Builder $query,
string $dateField,
?string $shopName,
?string $fromDate,
?string $toDate,
): void {
$this->whereValue($query, 'src.shopName', $shopName);
$this->whereFromDate($query, 'src.' . $dateField, $fromDate, true);
$this->whereToDate($query, 'src.' . $dateField, $toDate);
}
private function whereValue(Builder $query, string $column, ?string $value): void
{
if ($value !== null && $value !== '') {
$query->where($column, $value);
}
}
private function whereFromDate(Builder $query, string $column, ?string $date, bool $midnight = false): void
{
if ($this->validDate($date)) {
$query->where($column, '>=', $date . ($midnight ? ' 00:00:00' : ''));
}
}
private function whereToDate(Builder $query, string $column, ?string $date): void
{
if ($this->validDate($date)) {
$query->where($column, '<', date('Y-m-d', strtotime($date . ' +1 day')) . ' 00:00:00');
}
}
private function validDate(?string $date): bool
{
return $date !== null && preg_match('/^\d{4}-\d{2}-\d{2}$/', $date) === 1;
}
/** @return string[] */
private function stringOptions(string $datasetKey, string $column, ?int $limit = null): array
{
$query = $this->table($datasetKey)
->whereNotNull($column)
->where($column, '<>', '')
->distinct()
->orderBy($column);
if ($limit !== null) {
$query->limit($limit);
}
return $query->pluck($column)->map(static fn (mixed $value): string => (string) $value)->all();
}
/** @return array{rows:array<int,array<string,mixed>>,total:int} */
private function paginate(Builder $query, array $fields, array $orderColumns, int $page, int $pageSize): array
{
$page = max($page, 1);
$total = (clone $query)->count();
if ($total === 0) {
return ['rows' => [], 'total' => 0];
}
foreach ($orderColumns as $column) {
$query->orderByDesc($column);
}
$rows = $query->select($fields)
->forPage($page, $pageSize)
->get()
->map(static fn (object $row): array => (array) $row)
->all();
return ['rows' => $rows, 'total' => $total];
}
/** @return array{rows:array<int,array<string,mixed>>,total:int} */
private function monthlyBalances(
string $datasetKey,
string $dateField,
string $incomeExpression,
string $expenseExpression,
?string $storeName,
?string $month,
int $page,
int $pageSize,
string $openingAlias,
string $incomeAlias,
string $expenseAlias,
string $closingAlias,
): array {
$page = max($page, 1);
$driver = $this->connection->getDriverName();
$monthStart = $driver === 'sqlite'
? "date(src.$dateField, 'start of month')"
: "DATE_FORMAT(src.$dateField, '%Y-%m-01')";
$currentMonth = $driver === 'sqlite'
? "date('now', 'start of month')"
: "DATE_FORMAT(CURRENT_DATE, '%Y-%m-01')";
$lastDay = $driver === 'sqlite'
? "date(m.month_start, '+1 month', '-1 day')"
: "DATE_FORMAT(LAST_DAY(m.month_start), '%Y-%m-%d')";
$table = $this->dataset($datasetKey)->targetTable();
$ranked = $this->connection->table($table . ' AS src')
->whereNotNull('src.shopName')
->whereNotNull('src.' . $dateField)
->whereRaw("src.$dateField < $currentMonth")
->when($storeName !== null && $storeName !== '', fn (Builder $query) => $query->where('src.shopName', $storeName))
->selectRaw("src.shopName, src.$dateField AS entry_time, src.id AS entry_id, src.balance AS entry_balance, $monthStart AS month_start")
->selectRaw("$incomeExpression AS income_value, $expenseExpression AS expense_value")
->selectRaw("LAG(src.balance) OVER (PARTITION BY src.shopName ORDER BY src.$dateField, src.id) AS opening_balance")
->selectRaw("ROW_NUMBER() OVER (PARTITION BY src.shopName, $monthStart ORDER BY src.$dateField, src.id) AS month_first_rank")
->selectRaw("ROW_NUMBER() OVER (PARTITION BY src.shopName, $monthStart ORDER BY src.$dateField DESC, src.id DESC) AS month_last_rank");
$groups = $this->connection->query()->fromSub($ranked, 'ranked')
->when($month !== null && preg_match('/^\d{4}-\d{2}$/', $month) === 1, fn (Builder $query) => $query->where('month_start', $month . '-01'))
->selectRaw("shopName, month_start, COALESCE(SUM(income_value), 0) AS $incomeAlias, COALESCE(SUM(expense_value), 0) AS $expenseAlias")
->selectRaw("MAX(CASE WHEN month_first_rank = 1 THEN opening_balance END) AS $openingAlias")
->selectRaw("MAX(CASE WHEN month_last_rank = 1 THEN entry_balance END) AS $closingAlias")
->groupBy('shopName')
->groupBy('month_start');
$total = $this->connection->query()->fromSub(clone $groups, 'monthly_count')->count();
if ($total === 0) {
return ['rows' => [], 'total' => 0];
}
$query = $this->connection->query()->fromSub($groups, 'm')
->selectRaw("m.shopName, $lastDay AS balance_date, m.$incomeAlias, m.$expenseAlias, m.$openingAlias, m.$closingAlias")
->orderByDesc('m.month_start')
->orderBy('m.shopName')
->forPage($page, $pageSize);
return [
'rows' => $query->get()->map(static fn (object $row): array => (array) $row)->all(),
'total' => $total,
];
}
private function table(string $datasetKey, ?string $alias = null): Builder
{
if (! $this->ready($datasetKey)) {
throw new RuntimeException('business_mirror_dataset_not_ready');
}
$table = $this->dataset($datasetKey)->targetTable();
return $this->connection->table($alias === null ? $table : $table . ' AS ' . $alias);
}
private function dataset(string $datasetKey): BusinessMirrorDataset
{
return $this->datasets->get($datasetKey);
}
}

View File

@ -41,11 +41,18 @@ class PinduoduoSpreadBillRepository
private UpstreamReadOnlyQuery $query;
public function __construct(UpstreamReadOnlyQueryFactory $factory)
{
public function __construct(
UpstreamReadOnlyQueryFactory $factory,
private readonly ?MarketingAccountBillMirrorRepository $mirror = null,
) {
$this->query = $factory->forSource(self::SOURCE);
}
public function usesBusinessMirror(): bool
{
return $this->mirror?->ready(MarketingAccountBillMirrorRepository::PINDUODUO) ?? false;
}
/**
* 店铺下拉选项(全部去重)
*
@ -53,6 +60,10 @@ class PinduoduoSpreadBillRepository
*/
public function storeOptions(): array
{
if ($this->usesBusinessMirror()) {
return $this->mirror->pinduoduoStoreOptions();
}
$rows = $this->query->select(
'SELECT DISTINCT shopName FROM ' . self::TABLE . ' WHERE shopName IS NOT NULL ORDER BY shopName'
);
@ -79,6 +90,22 @@ class PinduoduoSpreadBillRepository
$page = max(1, $page);
$pageSize = max(1, min(200, $pageSize));
if ($this->usesBusinessMirror()) {
$result = $this->mirror->paginatePinduoduo(
$storeName,
$fromDate,
$toDate,
$flowType,
$page,
$pageSize,
);
return [
'rows' => array_map(fn (array $row): array => $this->normalize($row), $result['rows']),
'total' => $result['total'],
];
}
[$whereSql, $params] = $this->buildWhere($storeName, $fromDate, $toDate, $flowType);
// 总数
@ -125,16 +152,20 @@ class PinduoduoSpreadBillRepository
?string $toDate = null,
?string $flowType = null
): array {
[$whereSql, $params] = $this->buildWhere($storeName, $fromDate, $toDate, $flowType);
if ($this->usesBusinessMirror()) {
$row = $this->mirror->summarizePinduoduo($storeName, $fromDate, $toDate, $flowType);
} else {
[$whereSql, $params] = $this->buildWhere($storeName, $fromDate, $toDate, $flowType);
$row = $this->query->first(
'SELECT '
. 'COALESCE(SUM(CASE WHEN flowType = 1 THEN amount ELSE 0 END), 0) AS income_total, '
. 'COALESCE(SUM(CASE WHEN flowType = 2 THEN amount ELSE 0 END), 0) AS expense_total, '
. 'COUNT(*) AS cnt '
. 'FROM ' . self::TABLE . $whereSql,
$params
);
$row = $this->query->first(
'SELECT '
. 'COALESCE(SUM(CASE WHEN flowType = 1 THEN amount ELSE 0 END), 0) AS income_total, '
. 'COALESCE(SUM(CASE WHEN flowType = 2 THEN amount ELSE 0 END), 0) AS expense_total, '
. 'COUNT(*) AS cnt '
. 'FROM ' . self::TABLE . $whereSql,
$params
);
}
$income = (float) ($row['income_total'] ?? 0);
$expense = (float) ($row['expense_total'] ?? 0);
@ -156,6 +187,16 @@ class PinduoduoSpreadBillRepository
): array {
$page = max(1, $page);
$pageSize = max(1, min(5000, $pageSize));
if ($this->usesBusinessMirror()) {
$result = $this->mirror->paginatePinduoduoMonthlyBalances($storeName, $month, $page, $pageSize);
return [
'rows' => array_map(fn (array $row): array => $this->normalizeMonthlyBalance($row), $result['rows']),
'total' => $result['total'],
];
}
$monthExpression = 'DATEADD(MONTH, DATEDIFF(MONTH, 0, createTime), 0)';
[$whereSql, $params] = $this->buildMonthlyWhere($storeName, $month);

View File

@ -58,11 +58,18 @@ class TmallTaobaoSpreadBillRepository
private UpstreamReadOnlyQuery $query;
public function __construct(UpstreamReadOnlyQueryFactory $factory)
{
public function __construct(
UpstreamReadOnlyQueryFactory $factory,
private readonly ?MarketingAccountBillMirrorRepository $mirror = null,
) {
$this->query = $factory->forSource(self::SOURCE);
}
public function usesBusinessMirror(): bool
{
return $this->mirror?->ready(MarketingAccountBillMirrorRepository::TAOBAO) ?? false;
}
/**
* 店铺下拉选项(全部去重)
*
@ -70,6 +77,10 @@ class TmallTaobaoSpreadBillRepository
*/
public function storeOptions(): array
{
if ($this->usesBusinessMirror()) {
return $this->mirror->taobaoStoreOptions();
}
$rows = $this->query->select(
'SELECT DISTINCT shopName FROM ' . self::TABLE . ' WHERE shopName IS NOT NULL ORDER BY shopName'
);
@ -96,6 +107,22 @@ class TmallTaobaoSpreadBillRepository
$page = max(1, $page);
$pageSize = max(1, min(200, $pageSize));
if ($this->usesBusinessMirror()) {
$result = $this->mirror->paginateTaobao(
$storeName,
$fromDate,
$toDate,
$flowType,
$page,
$pageSize,
);
return [
'rows' => array_map(fn (array $row): array => $this->normalize($row), $result['rows']),
'total' => $result['total'],
];
}
[$whereSql, $params] = $this->buildWhere($storeName, $fromDate, $toDate, $flowType);
// 总数
@ -143,17 +170,21 @@ class TmallTaobaoSpreadBillRepository
?string $toDate = null,
?string $flowType = null
): array {
[$whereSql, $params] = $this->buildWhere($storeName, $fromDate, $toDate, $flowType);
if ($this->usesBusinessMirror()) {
$row = $this->mirror->summarizeTaobao($storeName, $fromDate, $toDate, $flowType);
} else {
[$whereSql, $params] = $this->buildWhere($storeName, $fromDate, $toDate, $flowType);
// amount 单位为「分」,汇总时除以 100 转为「元」
$row = $this->query->first(
'SELECT '
. 'COALESCE(SUM(CASE WHEN finType = \'DEBIT\' THEN amount ELSE 0 END), 0) AS income_total_raw, '
. 'COALESCE(SUM(CASE WHEN finType = \'CREDIT\' THEN amount ELSE 0 END), 0) AS expense_total_raw, '
. 'COUNT(*) AS cnt '
. 'FROM ' . self::TABLE . $whereSql,
$params
);
// amount 单位为「分」,汇总时除以 100 转为「元」
$row = $this->query->first(
'SELECT '
. 'COALESCE(SUM(CASE WHEN finType = \'DEBIT\' THEN amount ELSE 0 END), 0) AS income_total_raw, '
. 'COALESCE(SUM(CASE WHEN finType = \'CREDIT\' THEN amount ELSE 0 END), 0) AS expense_total_raw, '
. 'COUNT(*) AS cnt '
. 'FROM ' . self::TABLE . $whereSql,
$params
);
}
// 分 -> 元
$income = ((float) ($row['income_total_raw'] ?? 0)) / 100;
@ -176,6 +207,16 @@ class TmallTaobaoSpreadBillRepository
): array {
$page = max(1, $page);
$pageSize = max(1, min(5000, $pageSize));
if ($this->usesBusinessMirror()) {
$result = $this->mirror->paginateTaobaoMonthlyBalances($storeName, $month, $page, $pageSize);
return [
'rows' => array_map(fn (array $row): array => $this->normalizeMonthlyBalance($row), $result['rows']),
'total' => $result['total'],
];
}
$monthExpression = 'DATEADD(MONTH, DATEDIFF(MONTH, 0, transTime), 0)';
[$where, $params] = $this->buildMonthlyWhere($storeName, $month);

View File

@ -0,0 +1,78 @@
<?php
namespace Modules\EcommerceFinance\Services;
use InvalidArgumentException;
use Modules\EcommerceFinance\Contracts\BusinessMirrorDataset;
class BusinessMirrorDatasetRegistry
{
/** @return array<string,BusinessMirrorDataset> */
public function all(): array
{
$datasets = [];
foreach ((array) config('upstream_mirrors.datasets', []) as $key => $definition) {
$datasets[(string) $key] = $this->make((string) $key, (array) $definition);
}
return $datasets;
}
public function get(string $key): BusinessMirrorDataset
{
$definitions = (array) config('upstream_mirrors.datasets', []);
$definition = (array) ($definitions[$key] ?? []);
if ($definition === []) {
throw new InvalidArgumentException('未知的业务镜像数据集:' . $key);
}
return $this->make($key, $definition);
}
private function make(string $key, array $definition): BusinessMirrorDataset
{
$source = trim((string) ($definition['source_key'] ?? ''));
$schema = trim((string) ($definition['schema'] ?? ''));
$object = trim((string) ($definition['object'] ?? ''));
$target = trim((string) ($definition['target'] ?? ''));
$keys = array_values(array_filter(array_map('strval', (array) ($definition['keys'] ?? []))));
$watermark = trim((string) ($definition['watermark'] ?? ''));
foreach ([$source, $schema, $object, $target, ...$keys] as $identifier) {
if (preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/', $identifier) !== 1) {
throw new InvalidArgumentException('业务镜像数据集包含不安全的标识符:' . $key);
}
}
if ($keys === [] || strlen($target) > 64) {
throw new InvalidArgumentException('业务镜像数据集缺少稳定键或目标表名过长:' . $key);
}
if ($watermark !== '' && preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/', $watermark) !== 1) {
throw new InvalidArgumentException('业务镜像数据集的增量字段不安全:' . $key);
}
return new readonly class($key, $source, $schema, $object, $target, $keys, $watermark ?: null) implements BusinessMirrorDataset {
public function __construct(
private string $key,
private string $source,
private string $schema,
private string $object,
private string $target,
private array $keys,
private ?string $watermark,
) {
}
public function key(): string { return $this->key; }
public function sourceKey(): string { return $this->source; }
public function sourceSchema(): string { return $this->schema; }
public function sourceObject(): string { return $this->object; }
public function targetTable(): string { return $this->target; }
public function keyColumns(): array { return $this->keys; }
public function watermarkColumn(): ?string { return $this->watermark; }
};
}
}

View File

@ -0,0 +1,150 @@
<?php
namespace Modules\EcommerceFinance\Services;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
use Modules\EcommerceFinance\Contracts\BusinessMirrorDataset;
use RuntimeException;
class BusinessMirrorSchemaManager
{
/** @return list<array<string,mixed>> */
public function ensure(BusinessMirrorDataset $dataset): array
{
$columns = $this->columns($dataset);
$connection = (string) config('upstream_mirrors.connection', 'business_mirror');
$schema = Schema::connection($connection);
if (! $schema->hasTable($dataset->targetTable())) {
$schema->create($dataset->targetTable(), function (Blueprint $table) use ($columns, $dataset): void {
$table->char('_mirror_key', 64)->primary();
foreach ($columns as $column) {
$this->addColumn($table, $column);
}
$table->uuid('_mirror_generation')->nullable()->index();
$table->dateTime('_mirror_synced_at', 6)->index();
$watermark = $dataset->watermarkColumn();
if ($watermark !== null && $this->isIndexable($columns, $watermark)) {
$table->index($watermark, $this->indexName($dataset->targetTable(), 'watermark'));
}
});
return $columns;
}
$existing = array_map('strtolower', $schema->getColumnListing($dataset->targetTable()));
$missing = array_values(array_filter(
$columns,
fn (array $column): bool => ! in_array(strtolower((string) $column['column_name']), $existing, true),
));
if ($missing !== []) {
$schema->table($dataset->targetTable(), function (Blueprint $table) use ($missing): void {
foreach ($missing as $column) {
$this->addColumn($table, $column);
}
});
}
return $columns;
}
/** @return list<array<string,mixed>> */
public function columns(BusinessMirrorDataset $dataset): array
{
$rows = DB::table('upstream_schema_columns')
->where('source_key', $dataset->sourceKey())
->whereRaw('LOWER(schema_name) = ?', [strtolower($dataset->sourceSchema())])
->whereRaw('LOWER(object_name) = ?', [strtolower($dataset->sourceObject())])
->orderBy('ordinal_position')
->get()
->map(fn (object $row): array => (array) $row)
->all();
if ($rows === []) {
throw new RuntimeException('business_mirror_schema_catalog_missing:' . $dataset->key());
}
$names = array_map(fn (array $row): string => strtolower((string) $row['column_name']), $rows);
if (count($names) !== count(array_unique($names))) {
throw new RuntimeException('business_mirror_duplicate_column:' . $dataset->key());
}
foreach ($rows as $row) {
$name = (string) $row['column_name'];
if (strlen($name) > 64 || str_starts_with(strtolower($name), '_mirror_')) {
throw new RuntimeException('business_mirror_unsupported_column:' . $dataset->key());
}
}
foreach ([...$dataset->keyColumns(), $dataset->watermarkColumn()] as $required) {
if ($required !== null && ! in_array(strtolower($required), $names, true)) {
throw new RuntimeException('business_mirror_required_column_missing:' . $dataset->key());
}
}
return $rows;
}
private function addColumn(Blueprint $table, array $column): void
{
$name = (string) $column['column_name'];
$type = strtolower((string) $column['data_type']);
$length = (int) ($column['max_length'] ?? 0);
$precision = min(max((int) ($column['numeric_precision'] ?? 20), 1), 65);
$scale = min(max((int) ($column['numeric_scale'] ?? 0), 0), min($precision, 30));
$definition = match ($type) {
'bit' => $table->boolean($name),
'tinyint' => $table->unsignedTinyInteger($name),
'smallint' => $table->smallInteger($name),
'int' => $table->integer($name),
'bigint' => $table->bigInteger($name),
'decimal', 'numeric', 'money', 'smallmoney' => $table->decimal($name, $precision, $scale),
'float', 'real' => $table->double($name),
'date' => $table->date($name),
'datetime', 'datetime2', 'smalldatetime' => $table->dateTime($name, 6),
'time' => $table->time($name, 6),
'uniqueidentifier' => $table->char($name, 36),
'binary', 'varbinary', 'image', 'timestamp', 'rowversion' => $table->binary($name),
// MariaDB counts every inline varchar column towards the 65,535-byte
// row limit. Wide legacy SQL Server tables can contain dozens of
// nvarchar columns even though each individual column looks small.
// Keep only genuinely short values inline; TEXT is stored off-page
// and lets the mirror preserve the complete source schema.
'char', 'nchar', 'varchar', 'nvarchar' => $length > 0 && $length <= 64
? $table->string($name, $length)
: $table->longText($name),
default => $table->longText($name),
};
$definition->nullable();
}
private function isIndexable(array $columns, string $name): bool
{
foreach ($columns as $column) {
if (strcasecmp((string) $column['column_name'], $name) !== 0) {
continue;
}
$type = strtolower((string) $column['data_type']);
$length = (int) ($column['max_length'] ?? 0);
return ! in_array($type, ['text', 'ntext', 'xml', 'image', 'binary', 'varbinary'], true)
&& (! in_array($type, ['char', 'nchar', 'varchar', 'nvarchar'], true) || ($length > 0 && $length <= 64));
}
return false;
}
private function indexName(string $table, string $suffix): string
{
return substr($table . '_' . $suffix, 0, 64);
}
}

View File

@ -0,0 +1,307 @@
<?php
namespace Modules\EcommerceFinance\Services;
use Illuminate\Support\Carbon;
use Illuminate\Support\Str;
use Modules\EcommerceFinance\Contracts\BusinessMirrorDataset;
use Modules\EcommerceFinance\Repositories\GenericBusinessMirrorRepository;
use RuntimeException;
use Throwable;
class BusinessMirrorSynchronizer
{
public function __construct(
private readonly BusinessMirrorDatasetRegistry $datasets,
private readonly BusinessMirrorSchemaManager $schemas,
private readonly GenericBusinessMirrorRepository $mirror,
private readonly UpstreamReadOnlyQueryFactory $queries,
) {
}
/** @return array{status:string,mode:string,batches:int,rows:int,dataset:string} */
public function sync(string $datasetKey, bool $full = false, ?int $batchSize = null, ?int $maxBatches = null): array
{
$dataset = $this->datasets->get($datasetKey);
// A long-running scheduler may have retained a timed-out PDO instance
// from the previous dataset. Each dataset starts with a fresh upstream
// connection so one source failure cannot poison subsequent mirrors.
UpstreamOdbcConnection::reset($dataset->sourceKey());
$state = $this->mirror->state($dataset);
if (! $full && ($state['initial_completed_at'] ?? null) === null) {
return $this->result($dataset, 'requires_full_sync', 'incremental', 0, 0);
}
$columns = $this->schemas->ensure($dataset);
$mode = $full ? 'full' : 'incremental';
$resume = is_array($state)
&& ($state['run_mode'] ?? null) === $mode
&& in_array($state['status'] ?? null, ['running', 'failed'], true)
&& is_array($state['cursor'] ?? null);
$generation = $resume && is_string($state['generation'] ?? null)
? (string) $state['generation']
: (string) Str::uuid();
[$windowStart, $windowEnd] = $this->window($dataset, $state, $mode, $resume);
$state = $this->mirror->startRun(
$dataset,
$mode,
$generation,
$windowStart?->format('Y-m-d H:i:s.u'),
$windowEnd?->format('Y-m-d H:i:s.u'),
$resume,
);
$cursor = is_array($state['cursor'] ?? null) ? $state['cursor'] : null;
$highWatermark = is_array($state['high_watermark'] ?? null) ? $state['high_watermark'] : [];
// Routine runs keep the configured 500-row default. A larger explicit
// batch is reserved for supervised first syncs so wide SQL Server views
// need fewer repeated keyset scans without creating unbounded PHP arrays.
$batchSize = min(max($batchSize ?? (int) config('upstream_mirrors.table_sync.batch_size', 500), 1), 5000);
$maxBatches = max($maxBatches ?? (int) config('upstream_mirrors.table_sync.max_batches_per_run', 2), 0);
$batches = 0;
$rows = (int) ($state['last_upserted_rows'] ?? 0);
try {
while (true) {
$batch = $this->readBatch(
$dataset,
$columns,
$mode,
$cursor,
$highWatermark,
$windowStart,
$windowEnd,
$batchSize,
);
if ($batch === []) {
$this->mirror->complete($dataset, $mode, $generation, $cursor ?? $highWatermark, $rows);
return $this->result($dataset, 'succeeded', $mode, $batches, $rows);
}
$cursor = $this->cursor($dataset, $batch[array_key_last($batch)]);
$rows += count($batch);
$batches++;
$this->mirror->storeBatch($dataset, $columns, $this->normalize($columns, $batch), $cursor, $generation, $rows);
if (count($batch) < $batchSize) {
$this->mirror->complete($dataset, $mode, $generation, $cursor, $rows);
return $this->result($dataset, 'succeeded', $mode, $batches, $rows);
}
if ($maxBatches > 0 && $batches >= $maxBatches) {
return $this->result($dataset, 'running', $mode, $batches, $rows);
}
}
} catch (Throwable $exception) {
$this->mirror->fail($dataset, $exception::class);
throw $exception;
}
}
/** @return list<array<string,mixed>> */
private function readBatch(
BusinessMirrorDataset $dataset,
array $columns,
string $mode,
?array $cursor,
array $highWatermark,
?Carbon $windowStart,
?Carbon $windowEnd,
int $limit,
): array {
$order = $this->orderColumns($dataset);
$where = [];
$bindings = [];
if ($mode === 'incremental' && $dataset->watermarkColumn() !== null) {
$watermark = $this->quote($dataset->watermarkColumn());
$where[] = $watermark . ' >= ? AND ' . $watermark . ' < ?';
$bindings[] = $this->sqlServerDateTime($windowStart);
$bindings[] = $this->sqlServerDateTime($windowEnd);
} elseif ($mode === 'incremental' && $cursor === null && $highWatermark !== []) {
[$predicate, $values] = $this->keyset($dataset->keyColumns(), $highWatermark);
$where[] = $predicate;
array_push($bindings, ...$values);
}
if ($cursor !== null) {
[$predicate, $values] = $this->keyset($order, $cursor);
$where[] = $predicate;
array_push($bindings, ...$values);
}
$sql = 'SELECT TOP (' . $limit . ') '
. implode(', ', array_map(fn (array $column): string => $this->selectExpression($column), $columns))
. ' FROM ' . $this->quote($dataset->sourceSchema()) . '.' . $this->quote($dataset->sourceObject())
. ($where === [] ? '' : ' WHERE (' . implode(') AND (', $where) . ')')
. ' ORDER BY ' . implode(', ', array_map(fn (string $name): string => $this->quote($name), $order));
return $this->queries->forSource($dataset->sourceKey())->select($sql, $bindings);
}
/** @param array<string,mixed> $column */
private function selectExpression(array $column): string
{
$name = (string) $column['column_name'];
$quoted = $this->quote($name);
// FreeTDS may return SQL Server date values using the connection locale
// (for example "Jan 1 2026"). Convert temporal columns at the source so
// every mirror receives deterministic ISO values that MariaDB accepts.
return match (strtolower((string) ($column['data_type'] ?? ''))) {
'date' => 'CONVERT(varchar(10), ' . $quoted . ', 23) AS ' . $quoted,
'datetime', 'datetime2', 'smalldatetime' => 'CONVERT(varchar(27), ' . $quoted . ', 126) AS ' . $quoted,
default => $quoted,
};
}
private function sqlServerDateTime(?Carbon $value): ?string
{
// ISO 8601 with millisecond precision is accepted consistently by SQL Server
// through FreeTDS. Six-digit fractions were interpreted as an invalid date by
// some upstream datetime columns and broke every incremental run.
return $value?->format('Y-m-d\TH:i:s.v');
}
/** @return array{0:string,1:list<mixed>} */
private function keyset(array $columns, array $cursor): array
{
$parts = [];
$bindings = [];
foreach (array_values($columns) as $index => $column) {
$and = [];
for ($previous = 0; $previous < $index; $previous++) {
$name = $columns[$previous];
$and[] = $this->quote($name) . ' = ?';
$bindings[] = $this->cursorValue($cursor, $name);
}
$and[] = $this->quote($column) . ' > ?';
$bindings[] = $this->cursorValue($cursor, $column);
$parts[] = '(' . implode(' AND ', $and) . ')';
}
return ['(' . implode(' OR ', $parts) . ')', $bindings];
}
/** @return list<string> */
private function orderColumns(BusinessMirrorDataset $dataset): array
{
// The watermark belongs in the incremental WHERE window only. Sorting
// a very large SQL Server view by an unindexed timestamp forces an
// expensive full-window sort on every keyset page and eventually times
// out. Dataset keys are the stable, unique pagination contract.
return array_values(array_unique($dataset->keyColumns()));
}
/** @return array<string,mixed> */
private function cursor(BusinessMirrorDataset $dataset, array $row): array
{
$cursor = [];
foreach ($this->orderColumns($dataset) as $column) {
$cursor[$column] = $this->rowValue($row, $column);
}
return $cursor;
}
/** @return array{0:?Carbon,1:?Carbon} */
private function window(BusinessMirrorDataset $dataset, ?array $state, string $mode, bool $resume): array
{
if ($mode === 'full' || $dataset->watermarkColumn() === null) {
return [null, null];
}
if ($resume && ($state['window_started_at'] ?? null) !== null && ($state['window_ends_at'] ?? null) !== null) {
return [Carbon::parse((string) $state['window_started_at']), Carbon::parse((string) $state['window_ends_at'])];
}
$overlap = max((int) config('upstream_mirrors.table_sync.overlap_hours', 48), 1);
$last = Carbon::parse((string) ($state['last_succeeded_at'] ?? now()->subHours($overlap)));
return [$last->subHours($overlap), now()->addSecond()];
}
/** @return list<array<string,mixed>> */
private function normalize(array $columns, array $rows): array
{
$types = [];
foreach ($columns as $column) {
$types[strtolower((string) $column['column_name'])] = strtolower((string) $column['data_type']);
}
return array_map(function (array $row) use ($types): array {
foreach ($row as $name => $value) {
if (! is_string($value)) {
continue;
}
$type = $types[strtolower((string) $name)] ?? '';
if ($type === 'date') {
$row[$name] = substr($value, 0, 10);
} elseif (in_array($type, ['datetime', 'datetime2', 'smalldatetime'], true)) {
$row[$name] = preg_replace('/(\.\d{6})\d+/', '$1', str_replace('T', ' ', $value));
} elseif (in_array($type, ['char', 'nchar', 'varchar', 'nvarchar', 'text', 'ntext', 'xml'], true)) {
$row[$name] = $this->normalizeText($value);
}
}
return $row;
}, $rows);
}
private function normalizeText(string $value): string
{
// A small number of legacy SQL Server rows contain embedded NUL bytes or
// malformed byte sequences. MariaDB correctly rejects those values as
// invalid utf8mb4. Do not guess a source encoding: preserve all valid
// UTF-8 and mark only irrecoverable bytes with the Unicode replacement
// character so one corrupt label cannot block the entire table mirror.
$value = str_replace("\0", '', $value);
return mb_check_encoding($value, 'UTF-8')
? $value
: mb_scrub($value, 'UTF-8');
}
private function quote(string $identifier): string
{
if (preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/', $identifier) !== 1) {
throw new RuntimeException('business_mirror_invalid_identifier');
}
return '[' . $identifier . ']';
}
private function cursorValue(array $cursor, string $column): mixed
{
foreach ($cursor as $key => $value) {
if (strcasecmp((string) $key, $column) === 0) {
return $value;
}
}
throw new RuntimeException('business_mirror_cursor_incomplete');
}
private function rowValue(array $row, string $column): mixed
{
foreach ($row as $key => $value) {
if (strcasecmp((string) $key, $column) === 0) {
return $value;
}
}
throw new RuntimeException('business_mirror_row_key_missing');
}
private function result(BusinessMirrorDataset $dataset, string $status, string $mode, int $batches, int $rows): array
{
return ['status' => $status, 'mode' => $mode, 'batches' => $batches, 'rows' => $rows, 'dataset' => $dataset->key()];
}
}

View File

@ -6,6 +6,7 @@ use App\Models\Auth\Permission;
use Mockery;
use Modules\EcommerceFinance\Exceptions\UpstreamSnapshotPendingException;
use Modules\EcommerceFinance\Repositories\JdWalletBillRepository;
use Modules\EcommerceFinance\Repositories\MarketingAccountBillMirrorRepository;
use Modules\EcommerceFinance\Repositories\PinduoduoSpreadBillRepository;
use Modules\EcommerceFinance\Repositories\TmallTaobaoSpreadBillRepository;
use Modules\EcommerceFinance\Services\UpstreamQueryCache;
@ -15,10 +16,20 @@ use Tests\Feature\FeatureTestCase;
class MarketingAccountBillsTabsTest extends FeatureTestCase
{
protected function setUp(): void
{
parent::setUp();
$mirror = Mockery::mock(MarketingAccountBillMirrorRepository::class);
$mirror->shouldReceive('ready')->byDefault()->andReturnFalse();
$this->app->instance(MarketingAccountBillMirrorRepository::class, $mirror);
}
public function testMarketingAccountBalanceRedirectsToTaobaoAndShowsRestoredControls(): void
{
$this->allowMarketingAccountBalancePage();
$repository = Mockery::mock(PinduoduoSpreadBillRepository::class);
$repository->shouldReceive('usesBusinessMirror')->once()->andReturnFalse();
$this->app->instance(PinduoduoSpreadBillRepository::class, $repository);
$cache = Mockery::mock(UpstreamQueryCache::class);
$cache->shouldReceive('remember')->twice()->andReturnUsing(
@ -141,6 +152,10 @@ class MarketingAccountBillsTabsTest extends FeatureTestCase
$this->app->instance(UpstreamQueryCacheFactory::class, $factory);
$this->app->instance(JdWalletBillRepository::class, Mockery::mock(JdWalletBillRepository::class));
$this->app->make(JdWalletBillRepository::class)
->shouldReceive('usesBusinessMirror')->once()->with('recharge')->andReturnFalse();
$this->app->make(JdWalletBillRepository::class)
->shouldReceive('usesBusinessMirrorForOptions')->once()->andReturnFalse();
$this->loginAs()
->get(route('business.pages.show', [
@ -177,7 +192,9 @@ class MarketingAccountBillsTabsTest extends FeatureTestCase
->andReturn($cache);
$this->app->instance(UpstreamQueryCacheFactory::class, $factory);
$this->app->instance(TmallTaobaoSpreadBillRepository::class, Mockery::mock(TmallTaobaoSpreadBillRepository::class));
$repository = Mockery::mock(TmallTaobaoSpreadBillRepository::class);
$repository->shouldReceive('usesBusinessMirror')->once()->andReturnFalse();
$this->app->instance(TmallTaobaoSpreadBillRepository::class, $repository);
$this->loginAs()
->get(route('business.pages.export', [
@ -214,7 +231,9 @@ class MarketingAccountBillsTabsTest extends FeatureTestCase
$factory->shouldReceive('forSource')->once()->with('jiansun')->andReturn($cache);
$this->app->instance(UpstreamQueryCacheFactory::class, $factory);
$this->app->instance(TmallTaobaoSpreadBillRepository::class, Mockery::mock(TmallTaobaoSpreadBillRepository::class));
$repository = Mockery::mock(TmallTaobaoSpreadBillRepository::class);
$repository->shouldReceive('usesBusinessMirror')->once()->andReturnFalse();
$this->app->instance(TmallTaobaoSpreadBillRepository::class, $repository);
$this->loginAs()
->get(route('business.pages.show', [
@ -239,7 +258,9 @@ class MarketingAccountBillsTabsTest extends FeatureTestCase
$factory->shouldReceive('forSource')->once()->with('jiansun')->andReturn($cache);
$this->app->instance(UpstreamQueryCacheFactory::class, $factory);
$this->app->instance(TmallTaobaoSpreadBillRepository::class, Mockery::mock(TmallTaobaoSpreadBillRepository::class));
$repository = Mockery::mock(TmallTaobaoSpreadBillRepository::class);
$repository->shouldReceive('usesBusinessMirror')->once()->andReturnFalse();
$this->app->instance(TmallTaobaoSpreadBillRepository::class, $repository);
$this->loginAs()
->get(route('business.pages.export', [
@ -255,6 +276,88 @@ class MarketingAccountBillsTabsTest extends FeatureTestCase
->assertSessionHas('error', '余额数据导出暂时无法完成,请稍后重试。');
}
public function testMarketingBillSecondPageReadsReadyMirrorWithoutRegisteringAColdSnapshot(): void
{
$this->allowMarketingAccountBillsPage();
$repository = Mockery::mock(TmallTaobaoSpreadBillRepository::class);
$repository->shouldReceive('usesBusinessMirror')->once()->andReturnTrue();
$repository->shouldReceive('paginate')
->once()
->with(null, null, null, null, 2, 25)
->andReturn(['rows' => [['store_name' => '镜像店铺']], 'total' => 26]);
$repository->shouldReceive('summary')->once()->andReturn([
'income_total' => 0,
'expense_total' => 0,
'balance' => 0,
'count' => 26,
]);
$repository->shouldReceive('storeOptions')->once()->andReturn(['镜像店铺']);
$this->app->instance(TmallTaobaoSpreadBillRepository::class, $repository);
$cache = Mockery::mock(UpstreamQueryCache::class);
$cache->shouldNotReceive('remember');
$factory = Mockery::mock(UpstreamQueryCacheFactory::class);
$factory->shouldReceive('forSource')->once()->with('jiansun')->andReturn($cache);
$this->app->instance(UpstreamQueryCacheFactory::class, $factory);
$this->loginAs();
$url = route('business.pages.show', [
'section' => 'billing-transactions',
'page' => 'marketing-account-bills',
'tab' => 'taobao-alipay',
]) . '?page=2';
$this->get($url)
->assertOk()
->assertSeeText('镜像店铺')
->assertDontSeeText('首份共享快照正在准备');
}
public function testMarketingBalanceSecondPageReadsReadyMirrorWithoutRegisteringAColdSnapshot(): void
{
$this->allowMarketingAccountBalancePage();
$repository = Mockery::mock(TmallTaobaoSpreadBillRepository::class);
$repository->shouldReceive('usesBusinessMirror')->once()->andReturnTrue();
$repository->shouldReceive('paginateMonthlyBalances')
->once()
->with(null, null, 2, 25)
->andReturn(['rows' => [[
'date' => '2026-08',
'store_name' => '镜像余额店铺',
'opening_balance' => 100,
'income_total' => 20,
'expense_total' => 5,
'closing_balance' => 115,
'system_closing_balance' => 115,
'balance_difference' => 0,
'difference_reason' => '',
'verification_status' => '一致',
'notes' => '',
]], 'total' => 26]);
$repository->shouldReceive('storeOptions')->once()->andReturn(['镜像余额店铺']);
$this->app->instance(TmallTaobaoSpreadBillRepository::class, $repository);
$cache = Mockery::mock(UpstreamQueryCache::class);
$cache->shouldNotReceive('remember');
$factory = Mockery::mock(UpstreamQueryCacheFactory::class);
$factory->shouldReceive('forSource')->once()->with('jiansun')->andReturn($cache);
$this->app->instance(UpstreamQueryCacheFactory::class, $factory);
$this->loginAs();
$url = route('business.pages.show', [
'section' => 'billing-transactions',
'page' => 'marketing-account-balance',
'tab' => 'taobao-alipay',
]) . '?page=2';
$this->get($url)
->assertOk()
->assertSeeText('镜像余额店铺')
->assertDontSeeText('后台正在准备首份共享数据');
}
private function allowMarketingAccountBillsPage(): void
{
$this->user->update(['locale' => 'zh-CN']);

View File

@ -0,0 +1,125 @@
<?php
namespace Tests\Unit\EcommerceFinance;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
use Modules\EcommerceFinance\Repositories\BusinessInformationMirrorRepository;
use Modules\EcommerceFinance\Repositories\GenericBusinessMirrorRepository;
use Modules\EcommerceFinance\Services\BusinessMirrorDatasetRegistry;
use Tests\TestCase;
class BusinessInformationMirrorRepositoryTest extends TestCase
{
private BusinessInformationMirrorRepository $repository;
protected function setUp(): void
{
parent::setUp();
config([
'database.connections.business_mirror' => [
'driver' => 'sqlite',
'database' => ':memory:',
'prefix' => '',
],
'upstream_mirrors.connection' => 'business_mirror',
'upstream_mirrors.datasets' => [
'operations_replica.rpt.fys_companyInfo' => $this->definition('fys_companyInfo', 'ops_fys_company_info'),
'operations_replica.rpt.fys_shopInfo' => $this->definition('fys_shopInfo', 'ops_fys_shop_info'),
'operations_replica.rpt.fys_TrademarkInfo' => $this->definition('fys_TrademarkInfo', 'ops_fys_trademark_info'),
],
]);
DB::purge('business_mirror');
$this->createTables();
$mirror = new GenericBusinessMirrorRepository(app('db'));
$this->repository = new BusinessInformationMirrorRepository(
new BusinessMirrorDatasetRegistry(),
$mirror,
);
}
public function test_it_reads_all_three_ready_mirrors_with_pagination_and_business_filters(): void
{
$connection = DB::connection('business_mirror');
$connection->table('ops_fys_company_info')->insert([
$this->company(1, '统一代码', '旧公司'),
$this->company(2, '统一代码', '新公司'),
$this->company(3, '其他代码', '其他公司'),
]);
$connection->table('ops_fys_shop_info')->insert([
['Id' => 8, 'openTime' => '2026-08-14 10:00:00', 'erpShopName' => '测试店', 'unifiedSocialCreditCode' => '统一代码', 'trademarkNumber' => 'TM', 'platform' => '天猫', 'responsibleDepartment' => '电商', 'shopHead' => '负责人', 'shopState' => '0', 'shopType' => '直营', 'channelType' => '线上'],
]);
$connection->table('ops_fys_trademark_info')->insert([
['Id' => 5, 'trademarkName' => '有效商标', 'internationalClassification' => '20', 'trademarkStatus' => '1', 'registrationDate' => '2020-01-01', 'termOfValidity' => '2030-01-01', 'remark' => null, 'isDelete' => 0],
['Id' => 6, 'trademarkName' => '已删除', 'internationalClassification' => '20', 'trademarkStatus' => '0', 'registrationDate' => null, 'termOfValidity' => null, 'remark' => null, 'isDelete' => 1],
]);
$this->markReady();
$companies = $this->repository->paginate('company-information.read', 2, 2);
$store = $this->repository->paginate('store-information.read', 1, 20);
$trademarks = $this->repository->paginate('trademark-information.read', 1, 20);
$this->assertSame(3, $companies['total']);
$this->assertSame(1, $companies['data'][0]['id']);
$this->assertSame('新公司', $store['data'][0]['company_name']);
$this->assertSame(1, $trademarks['total']);
$this->assertSame('有效商标', $trademarks['data'][0]['trademark_name']);
}
public function test_dataset_is_not_ready_until_initial_full_sync_is_marked_complete(): void
{
$this->assertFalse($this->repository->ready('company-information.read'));
$this->markReady(['operations_replica.rpt.fys_companyInfo']);
$this->assertTrue($this->repository->ready('company-information.read'));
}
private function createTables(): void
{
$schema = Schema::connection('business_mirror');
$schema->create('business_mirror_table_states', function (Blueprint $table): void {
$table->string('dataset_key')->primary();
$table->dateTime('initial_completed_at')->nullable();
});
$schema->create('ops_fys_company_info', function (Blueprint $table): void {
$table->integer('Id'); $table->string('unifiedSocialCreditCode')->nullable(); $table->string('businessLicenseName')->nullable();
$table->string('registeredCapital')->nullable(); $table->string('businessLicenseType')->nullable(); $table->dateTime('IncorporationDate')->nullable();
$table->string('legalRepresentative')->nullable(); $table->string('registeredAddress')->nullable(); $table->text('businessScope')->nullable();
$table->string('accountSerialNumber')->nullable(); $table->string('companyStatus')->nullable(); $table->string('financialManager')->nullable();
$table->string('supervisor')->nullable(); $table->string('taxCategoryAndType')->nullable();
});
$schema->create('ops_fys_shop_info', function (Blueprint $table): void {
$table->integer('Id'); $table->dateTime('openTime')->nullable(); $table->string('erpShopName')->nullable(); $table->string('unifiedSocialCreditCode')->nullable();
$table->string('trademarkNumber')->nullable(); $table->string('platform')->nullable(); $table->string('responsibleDepartment')->nullable();
$table->string('shopHead')->nullable(); $table->string('shopState')->nullable(); $table->string('shopType')->nullable(); $table->string('channelType')->nullable();
});
$schema->create('ops_fys_trademark_info', function (Blueprint $table): void {
$table->integer('Id'); $table->string('trademarkName')->nullable(); $table->string('internationalClassification')->nullable();
$table->string('trademarkStatus')->nullable(); $table->dateTime('registrationDate')->nullable(); $table->dateTime('termOfValidity')->nullable();
$table->text('remark')->nullable(); $table->integer('isDelete')->default(0);
});
}
private function markReady(?array $keys = null): void
{
$keys ??= array_keys((array) config('upstream_mirrors.datasets'));
foreach ($keys as $key) {
DB::connection('business_mirror')->table('business_mirror_table_states')->insert([
'dataset_key' => $key,
'initial_completed_at' => '2026-08-14 12:00:00',
]);
}
}
private function definition(string $object, string $target): array
{
return ['source_key' => 'operations_replica', 'schema' => 'rpt', 'object' => $object, 'target' => $target, 'keys' => ['Id']];
}
private function company(int $id, string $code, string $name): array
{
return ['Id' => $id, 'unifiedSocialCreditCode' => $code, 'businessLicenseName' => $name, 'registeredCapital' => null, 'businessLicenseType' => null, 'IncorporationDate' => null, 'legalRepresentative' => null, 'registeredAddress' => null, 'businessScope' => null, 'accountSerialNumber' => null, 'companyStatus' => '0', 'financialManager' => null, 'supervisor' => null, 'taxCategoryAndType' => null];
}
}

View File

@ -0,0 +1,330 @@
<?php
namespace Tests\Unit\EcommerceFinance;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
use Modules\EcommerceFinance\Repositories\GenericBusinessMirrorRepository;
use Modules\EcommerceFinance\Services\BusinessMirrorDatasetRegistry;
use Modules\EcommerceFinance\Services\BusinessMirrorSchemaManager;
use Modules\EcommerceFinance\Services\BusinessMirrorSynchronizer;
use Modules\EcommerceFinance\Services\UpstreamReadOnlyQuery;
use Modules\EcommerceFinance\Services\UpstreamReadOnlyQueryFactory;
use Tests\TestCase;
class BusinessMirrorSynchronizerTest extends TestCase
{
private FakeBusinessMirrorQuery $query;
private BusinessMirrorSynchronizer $synchronizer;
protected function setUp(): void
{
parent::setUp();
config([
'database.connections.business_mirror' => [
'driver' => 'sqlite',
'database' => ':memory:',
'prefix' => '',
],
'upstream_mirrors.connection' => 'business_mirror',
'upstream_mirrors.datasets' => [
'test.dbo.records' => [
'source_key' => 'test',
'schema' => 'dbo',
'object' => 'records',
'target' => 'test_records',
'keys' => ['Id'],
'watermark' => 'updateTime',
],
],
'upstream_mirrors.table_sync.overlap_hours' => 24,
'upstreams.sources.test' => [
'label' => 'Test source',
'config' => 'test',
],
]);
DB::purge('business_mirror');
Schema::connection('business_mirror')->create('business_mirror_table_states', function (Blueprint $table): void {
$table->string('dataset_key')->primary();
$table->string('source_key');
$table->string('source_object');
$table->string('target_table');
$table->string('status')->default('pending');
$table->string('run_mode')->nullable();
$table->string('generation')->nullable();
$table->json('cursor')->nullable();
$table->json('high_watermark')->nullable();
$table->dateTime('window_started_at')->nullable();
$table->dateTime('window_ends_at')->nullable();
$table->dateTime('initial_completed_at')->nullable();
$table->dateTime('last_full_succeeded_at')->nullable();
$table->dateTime('last_attempted_at')->nullable();
$table->dateTime('last_succeeded_at')->nullable();
$table->unsignedBigInteger('last_upserted_rows')->default(0);
$table->unsignedBigInteger('mirror_row_count')->default(0);
$table->string('last_error_code')->nullable();
$table->timestamps();
});
DB::table('upstream_schema_columns')->insert([
$this->column('Id', 1, 'int', null),
$this->column('updateTime', 2, 'datetime2', null),
$this->column('displayName', 3, 'nvarchar', 120),
$this->column('businessDate', 4, 'date', null),
]);
$this->query = new FakeBusinessMirrorQuery();
$factory = new class($this->query) extends UpstreamReadOnlyQueryFactory {
public function __construct(private readonly UpstreamReadOnlyQuery $query) {}
public function forSource(string $source): UpstreamReadOnlyQuery { return $this->query; }
};
$registry = new BusinessMirrorDatasetRegistry();
$this->synchronizer = new BusinessMirrorSynchronizer(
$registry,
new BusinessMirrorSchemaManager(),
new GenericBusinessMirrorRepository(app('db')),
$factory,
);
}
public function test_full_sync_materializes_complete_table_and_marks_state_ready(): void
{
$this->query->batches = [[
['Id' => 1, 'updateTime' => '2026-08-14 10:00:00.1234567', 'displayName' => '一号', 'businessDate' => '2026-08-14'],
['Id' => 2, 'updateTime' => '2026-08-14 11:00:00.0000000', 'displayName' => '二号', 'businessDate' => '2026-08-15'],
]];
$result = $this->synchronizer->sync('test.dbo.records', true, 10, 1);
$this->assertSame('succeeded', $result['status']);
$this->assertSame(2, DB::connection('business_mirror')->table('test_records')->count());
$this->assertSame(
'2026-08-14 10:00:00.123456',
DB::connection('business_mirror')->table('test_records')->where('Id', 1)->value('updateTime'),
);
$state = DB::connection('business_mirror')->table('business_mirror_table_states')->first();
$this->assertSame('succeeded', $state->status);
$this->assertNotNull($state->initial_completed_at);
$this->assertSame(2, (int) $state->mirror_row_count);
$this->assertStringContainsString('SELECT TOP (10)', $this->query->calls[0]['sql']);
$this->assertStringContainsString('CONVERT(varchar(27), [updateTime], 126) AS [updateTime]', $this->query->calls[0]['sql']);
$this->assertStringContainsString('CONVERT(varchar(10), [businessDate], 23) AS [businessDate]', $this->query->calls[0]['sql']);
$this->assertStringContainsString('ORDER BY [Id]', $this->query->calls[0]['sql']);
}
public function test_incremental_sync_uses_overlap_window_and_updates_existing_key(): void
{
$this->query->batches = [[
['Id' => 1, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => '旧值'],
]];
$this->synchronizer->sync('test.dbo.records', true, 10, 1);
$this->query->calls = [];
$this->query->batches = [[
['Id' => 1, 'updateTime' => '2026-08-14 12:00:00', 'displayName' => '新值'],
['Id' => 2, 'updateTime' => '2026-08-14 12:30:00', 'displayName' => '新增'],
]];
$result = $this->synchronizer->sync('test.dbo.records', false, 10, 1);
$this->assertSame('succeeded', $result['status']);
$this->assertSame(2, DB::connection('business_mirror')->table('test_records')->count());
$this->assertSame('新值', DB::connection('business_mirror')->table('test_records')->where('Id', 1)->value('displayName'));
$this->assertStringContainsString('[updateTime] >= ? AND [updateTime] < ?', $this->query->calls[0]['sql']);
$this->assertStringContainsString('ORDER BY [Id]', $this->query->calls[0]['sql']);
$this->assertStringNotContainsString('ORDER BY [updateTime]', $this->query->calls[0]['sql']);
$this->assertCount(2, $this->query->calls[0]['bindings']);
foreach ($this->query->calls[0]['bindings'] as $binding) {
$this->assertMatchesRegularExpression('/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}$/', $binding);
$this->assertStringNotContainsString(' ', $binding);
}
}
public function test_incremental_keyset_keeps_watermark_out_of_pagination_cursor(): void
{
$this->query->batches = [[
['Id' => 1, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => '初始'],
]];
$this->synchronizer->sync('test.dbo.records', true, 10, 1);
$this->query->calls = [];
$this->query->batches = [[
['Id' => 1, 'updateTime' => '2026-08-14 12:00:00', 'displayName' => '更新一'],
['Id' => 2, 'updateTime' => '2026-08-14 12:30:00', 'displayName' => '更新二'],
], []];
$result = $this->synchronizer->sync('test.dbo.records', false, 2, 0);
$this->assertSame('succeeded', $result['status']);
$this->assertCount(2, $this->query->calls);
$this->assertStringContainsString('[updateTime] >= ? AND [updateTime] < ?', $this->query->calls[1]['sql']);
$this->assertStringContainsString('[Id] > ?', $this->query->calls[1]['sql']);
$this->assertStringContainsString('ORDER BY [Id]', $this->query->calls[1]['sql']);
$this->assertStringNotContainsString('[updateTime] > ?', $this->query->calls[1]['sql']);
$this->assertCount(3, $this->query->calls[1]['bindings']);
$this->assertSame(2, $this->query->calls[1]['bindings'][2]);
}
public function test_incremental_sync_refuses_to_guess_before_first_full_sync(): void
{
$result = $this->synchronizer->sync('test.dbo.records', false, 10, 1);
$this->assertSame('requires_full_sync', $result['status']);
$this->assertSame([], $this->query->calls);
}
public function test_supervised_full_sync_caps_explicit_batch_size_at_five_thousand(): void
{
$this->query->batches = [[
['Id' => 1, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => '一号', 'businessDate' => '2026-08-14'],
]];
$this->synchronizer->sync('test.dbo.records', true, 50000, 1);
$this->assertStringContainsString('SELECT TOP (5000)', $this->query->calls[0]['sql']);
}
public function test_full_sync_sanitizes_invalid_utf8_text_without_changing_valid_text(): void
{
$this->query->batches = [[
[
'Id' => 1,
'updateTime' => '2026-08-14 10:00:00',
'displayName' => hex2bin('e80044495900'),
],
[
'Id' => 2,
'updateTime' => '2026-08-14 11:00:00',
'displayName' => '合法名称',
],
]];
$this->synchronizer->sync('test.dbo.records', true, 10, 1);
$rows = DB::connection('business_mirror')->table('test_records')->orderBy('Id')->pluck('displayName')->all();
$this->assertStringEndsWith('DIY', $rows[0]);
$this->assertNotSame(hex2bin('e80044495900'), $rows[0]);
$this->assertSame('合法名称', $rows[1]);
$this->assertTrue(mb_check_encoding($rows[0], 'UTF-8'));
$this->assertStringNotContainsString("\0", $rows[0]);
}
public function test_wide_source_schema_stores_only_short_strings_inline(): void
{
$datasets = (array) config('upstream_mirrors.datasets', []);
$datasets['test.dbo.wide_records'] = [
'source_key' => 'test',
'schema' => 'dbo',
'object' => 'wide_records',
'target' => 'test_wide_records',
'keys' => ['Id'],
'watermark' => 'updateTime',
];
config(['upstream_mirrors.datasets' => $datasets]);
$columns = [
$this->column('Id', 1, 'int', null),
$this->column('updateTime', 2, 'datetime2', null),
$this->column('shortCode', 3, 'nvarchar', 32),
$this->column('longLabel', 4, 'nvarchar', 120),
];
DB::table('upstream_schema_columns')->insert(array_map(
fn (array $column): array => [...$column, 'object_name' => 'wide_records'],
$columns,
));
$dataset = (new BusinessMirrorDatasetRegistry())->get('test.dbo.wide_records');
(new BusinessMirrorSchemaManager())->ensure($dataset);
$types = DB::connection('business_mirror')->select('PRAGMA table_info(test_wide_records)');
$types = collect($types)->pluck('type', 'name')->map(fn (string $type): string => strtolower($type));
$this->assertSame('varchar', $types['shortCode']);
$this->assertSame('text', $types['longLabel']);
}
public function test_empty_full_read_does_not_delete_previous_mirror_rows(): void
{
$this->query->batches = [[
['Id' => 1, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => '保留'],
]];
$this->synchronizer->sync('test.dbo.records', true, 10, 1);
$this->query->batches = [[]];
try {
$this->synchronizer->sync('test.dbo.records', true, 10, 1);
$this->fail('空的上游响应不应覆盖已有镜像。');
} catch (\RuntimeException $exception) {
$this->assertSame('business_mirror_unexpected_empty_source', $exception->getMessage());
}
$this->assertSame(1, DB::connection('business_mirror')->table('test_records')->count());
$this->assertSame('保留', DB::connection('business_mirror')->table('test_records')->value('displayName'));
$this->assertSame(
'failed',
DB::connection('business_mirror')->table('business_mirror_table_states')->value('status'),
);
}
public function test_project_dataset_registry_has_unique_safe_targets(): void
{
$configured = require base_path('config/upstream_mirrors.php');
config(['upstream_mirrors.datasets' => $configured['datasets']]);
$datasets = (new BusinessMirrorDatasetRegistry())->all();
$targets = array_map(fn ($dataset): string => $dataset->targetTable(), $datasets);
$this->assertCount(38, $datasets);
$this->assertCount(count($targets), array_unique($targets));
$this->assertContains('operations_replica', array_map(fn ($dataset): string => $dataset->sourceKey(), $datasets));
$this->assertContains('jiansun', array_map(fn ($dataset): string => $dataset->sourceKey(), $datasets));
$this->assertNotContains('Bulk_Price_Limit', array_map(fn ($dataset): string => $dataset->sourceObject(), $datasets));
}
/** @return array<string,mixed> */
private function column(string $name, int $position, string $type, ?int $length): array
{
$now = now();
return [
'source_key' => 'test',
'schema_name' => 'dbo',
'object_name' => 'records',
'object_type' => 'table',
'column_name' => $name,
'ordinal_position' => $position,
'data_type' => $type,
'max_length' => $length,
'numeric_precision' => null,
'numeric_scale' => null,
'is_nullable' => true,
'synchronized_at' => $now,
'created_at' => $now,
'updated_at' => $now,
];
}
}
class FakeBusinessMirrorQuery extends UpstreamReadOnlyQuery
{
/** @var list<list<array<string,mixed>>> */
public array $batches = [];
/** @var list<array{sql:string,bindings:array}> */
public array $calls = [];
public function __construct()
{
parent::__construct('test');
}
public function select(string $sql, array $bindings = []): array
{
$this->calls[] = compact('sql', 'bindings');
return array_shift($this->batches) ?? [];
}
}

View File

@ -0,0 +1,211 @@
<?php
namespace Tests\Unit\EcommerceFinance;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
use Modules\EcommerceFinance\Repositories\AlibabaBillRepository;
use Modules\EcommerceFinance\Repositories\GenericBusinessMirrorRepository;
use Modules\EcommerceFinance\Repositories\JdWalletBillRepository;
use Modules\EcommerceFinance\Repositories\MarketingAccountBillMirrorRepository;
use Modules\EcommerceFinance\Repositories\PinduoduoSpreadBillRepository;
use Modules\EcommerceFinance\Repositories\TmallTaobaoSpreadBillRepository;
use Modules\EcommerceFinance\Services\BusinessMirrorDatasetRegistry;
use Modules\EcommerceFinance\Services\UpstreamReadOnlyQuery;
use Modules\EcommerceFinance\Services\UpstreamReadOnlyQueryFactory;
use RuntimeException;
use Tests\TestCase;
class MarketingAccountBillMirrorRepositoryTest extends TestCase
{
private MarketingAccountBillMirrorRepository $mirror;
protected function setUp(): void
{
parent::setUp();
config([
'database.connections.business_mirror' => [
'driver' => 'sqlite',
'database' => ':memory:',
'prefix' => '',
],
'upstream_mirrors.connection' => 'business_mirror',
'upstream_mirrors.datasets' => [
MarketingAccountBillMirrorRepository::TAOBAO => $this->definition('tmall_spread_bills', 'js_tmall_spread_bills'),
MarketingAccountBillMirrorRepository::PINDUODUO => $this->definition('pdd_spread_bills', 'js_pdd_spread_bills'),
MarketingAccountBillMirrorRepository::ALIBABA => $this->definition('alibaba_bill', 'js_alibaba_bill'),
MarketingAccountBillMirrorRepository::JD_RECHARGE => $this->definition('jd_spreadBills', 'js_jd_spread_bills'),
MarketingAccountBillMirrorRepository::JD_CONSUMPTION => $this->definition('jd_costBills', 'js_jd_cost_bills'),
],
]);
DB::purge('business_mirror');
$this->createTables();
$generic = new GenericBusinessMirrorRepository(app('db'));
$this->mirror = new MarketingAccountBillMirrorRepository(
new BusinessMirrorDatasetRegistry(),
$generic,
);
}
public function test_ready_mirrors_drive_lists_summaries_options_monthly_balances_and_second_pages(): void
{
$db = DB::connection('business_mirror');
$db->table('js_tmall_spread_bills')->insert([
$this->taobao(1, '2026-05-31 23:00:00', 10000, 10000, 'DEBIT'),
$this->taobao(2, '2026-06-01 09:00:00', 5000, 15000, 'DEBIT'),
$this->taobao(3, '2026-06-02 09:00:00', 2000, 13000, 'CREDIT'),
]);
$db->table('js_pdd_spread_bills')->insert([
$this->pinduoduo(1, '2026-05-31 23:00:00', 100, 100, 1),
$this->pinduoduo(2, '2026-06-01 09:00:00', 50, 150, 1),
$this->pinduoduo(3, '2026-06-02 09:00:00', 20, 130, 2),
]);
$db->table('js_alibaba_bill')->insert([
['id' => 1, 'shopName' => '阿里店', 'tradeDay' => '2026-06-01 09:00:00', 'tradeTypeName' => '全站', 'tradeAMTIn' => 10, 'tradeAMTOut' => 1],
['id' => 2, 'shopName' => '阿里店', 'tradeDay' => '2026-06-02 09:00:00', 'tradeTypeName' => '全站', 'tradeAMTIn' => 20, 'tradeAMTOut' => 2],
]);
$db->table('js_jd_spread_bills')->insert([
['id' => 1, 'shopName' => '京东店', 'amount' => 100, 'balanceAmount' => 100, 'orderTypeName' => '充值', 'pk' => 'R1', 'createDate' => '2026-06-01 10:00:00', 'updateTime' => '2026-06-01 10:01:00', 'statusName' => '成功', 'remark' => ''],
['id' => 2, 'shopName' => '京东店', 'amount' => 200, 'balanceAmount' => 300, 'orderTypeName' => '充值', 'pk' => 'R2', 'createDate' => '2026-06-02 10:00:00', 'updateTime' => '2026-06-02 10:01:00', 'statusName' => '成功', 'remark' => ''],
]);
$db->table('js_jd_cost_bills')->insert([
['id' => 1, 'shopName' => '京东店', 'amount' => -30, 'balanceAmount' => 270, 'orderTypeName' => '消耗', 'creatTime' => '2026-06-03 10:00:00', 'settleTime' => '2026-06-04 00:00:00', 'pk' => 'C1', 'invoiceApplyTypeName' => '推广费'],
]);
$this->markReady();
$factory = $this->throwingFactory();
$taobao = new TmallTaobaoSpreadBillRepository($factory, $this->mirror);
$pinduoduo = new PinduoduoSpreadBillRepository($factory, $this->mirror);
$alibaba = new AlibabaBillRepository($factory, $this->mirror);
$jd = new JdWalletBillRepository($factory, $this->mirror);
$taobaoPage2 = $taobao->paginate('镜像店铺', '2026-06-01', '2026-06-30', null, 2, 1);
$taobaoSummary = $taobao->summary('镜像店铺', '2026-06-01', '2026-06-30');
$taobaoMonthly = $taobao->paginateMonthlyBalances('镜像店铺', '2026-06', 1, 25);
$pddPage2 = $pinduoduo->paginate('镜像店铺', '2026-06-01', '2026-06-30', null, 2, 1);
$pddSummary = $pinduoduo->summary('镜像店铺', '2026-06-01', '2026-06-30');
$pddMonthly = $pinduoduo->paginateMonthlyBalances('镜像店铺', '2026-06', 1, 25);
$alibabaPage2 = $alibaba->paginate('阿里店', '全站', '2026-06-01', '2026-06-30', 2, 1);
$jdRechargePage2 = $jd->paginateRecharge('京东店', '2026-06-01', '2026-06-30', 2, 1);
$this->assertSame(2, $taobaoPage2['total']);
$this->assertSame(50.0, $taobaoPage2['rows'][0]['amount']);
$this->assertSame(['镜像店铺'], $taobao->storeOptions());
$this->assertSame(30.0, $taobaoSummary['net']);
$this->assertSame(100.0, $taobaoMonthly['rows'][0]['opening_balance']);
$this->assertSame(130.0, $taobaoMonthly['rows'][0]['closing_balance']);
$this->assertSame(0.0, $taobaoMonthly['rows'][0]['balance_difference']);
$this->assertSame(2, $pddPage2['total']);
$this->assertSame(50.0, $pddPage2['rows'][0]['amount']);
$this->assertSame(30.0, $pddSummary['net']);
$this->assertSame(100.0, $pddMonthly['rows'][0]['opening_balance']);
$this->assertSame(130.0, $pddMonthly['rows'][0]['closing_balance']);
$this->assertSame(0.0, $pddMonthly['rows'][0]['balance_difference']);
$this->assertSame(2, $alibabaPage2['total']);
$this->assertSame(10.0, $alibabaPage2['rows'][0]['income_amount']);
$this->assertSame(['shops' => ['阿里店'], 'product_types' => ['全站']], $alibaba->filterOptions());
$this->assertSame(27.0, $alibaba->summary('阿里店', '全站')['net']);
$this->assertSame(2, $jdRechargePage2['total']);
$this->assertSame('R1', $jdRechargePage2['rows'][0]['order_number']);
$this->assertSame(['京东店'], $jd->shopOptions());
$this->assertSame(300.0, $jd->summaryRecharge('京东店')['amount_total']);
$this->assertSame(30.0, $jd->summaryConsumption('京东店')['amount_total']);
}
public function test_unready_dataset_keeps_the_existing_upstream_repository_path(): void
{
$query = new class('jiansun') extends UpstreamReadOnlyQuery {
public function scalar(string $sql, array $bindings = []): mixed { return 1; }
public function select(string $sql, array $bindings = []): array
{
return [['shopName' => '上游回退', 'amount' => 8, 'balance' => 9, 'summary' => '', 'flowType' => 1, 'toAdvertiserFundType' => 1, 'createTime' => '2026-06-01']];
}
};
$factory = new class($query) extends UpstreamReadOnlyQueryFactory {
public function __construct(private readonly UpstreamReadOnlyQuery $query) {}
public function forSource(string $source): UpstreamReadOnlyQuery { return $this->query; }
};
$result = (new PinduoduoSpreadBillRepository($factory, $this->mirror))->paginate();
$this->assertSame(1, $result['total']);
$this->assertSame('上游回退', $result['rows'][0]['store_name']);
}
private function throwingFactory(): UpstreamReadOnlyQueryFactory
{
$query = new class('jiansun') extends UpstreamReadOnlyQuery {
public function scalar(string $sql, array $bindings = []): mixed { throw new RuntimeException('upstream_should_not_be_called'); }
public function select(string $sql, array $bindings = []): array { throw new RuntimeException('upstream_should_not_be_called'); }
public function first(string $sql, array $bindings = []): ?array { throw new RuntimeException('upstream_should_not_be_called'); }
};
return new class($query) extends UpstreamReadOnlyQueryFactory {
public function __construct(private readonly UpstreamReadOnlyQuery $query) {}
public function forSource(string $source): UpstreamReadOnlyQuery { return $this->query; }
};
}
private function createTables(): void
{
$schema = Schema::connection('business_mirror');
$schema->create('business_mirror_table_states', function (Blueprint $table): void {
$table->string('dataset_key')->primary();
$table->dateTime('initial_completed_at')->nullable();
});
$schema->create('js_tmall_spread_bills', function (Blueprint $table): void {
$table->integer('id'); $table->string('shopName')->nullable(); $table->dateTime('uniTransTime')->nullable(); $table->dateTime('transTime')->nullable();
$table->decimal('amount', 18, 2)->nullable(); $table->decimal('balance', 18, 2)->nullable(); $table->string('comment')->nullable();
$table->string('finType')->nullable(); $table->string('bizType')->nullable(); $table->string('tradeType')->nullable();
});
$schema->create('js_pdd_spread_bills', function (Blueprint $table): void {
$table->integer('id'); $table->string('shopName')->nullable(); $table->decimal('amount', 18, 2)->nullable(); $table->decimal('balance', 18, 2)->nullable();
$table->string('summary')->nullable(); $table->integer('flowType')->nullable(); $table->integer('toAdvertiserFundType')->nullable(); $table->dateTime('createTime')->nullable();
});
$schema->create('js_alibaba_bill', function (Blueprint $table): void {
$table->integer('id'); $table->string('shopName')->nullable(); $table->dateTime('tradeDay')->nullable(); $table->string('tradeTypeName')->nullable();
$table->decimal('tradeAMTIn', 18, 2)->nullable(); $table->decimal('tradeAMTOut', 18, 2)->nullable();
});
$schema->create('js_jd_spread_bills', function (Blueprint $table): void {
$table->integer('id'); $table->string('shopName')->nullable(); $table->decimal('amount', 18, 2)->nullable(); $table->decimal('balanceAmount', 18, 2)->nullable();
$table->string('orderTypeName')->nullable(); $table->string('pk')->nullable(); $table->dateTime('createDate')->nullable(); $table->dateTime('updateTime')->nullable();
$table->string('statusName')->nullable(); $table->string('remark')->nullable();
});
$schema->create('js_jd_cost_bills', function (Blueprint $table): void {
$table->integer('id'); $table->string('shopName')->nullable(); $table->decimal('amount', 18, 2)->nullable(); $table->decimal('balanceAmount', 18, 2)->nullable();
$table->string('orderTypeName')->nullable(); $table->dateTime('creatTime')->nullable(); $table->dateTime('settleTime')->nullable();
$table->string('pk')->nullable(); $table->string('invoiceApplyTypeName')->nullable();
});
}
private function markReady(): void
{
foreach (array_keys((array) config('upstream_mirrors.datasets')) as $key) {
DB::connection('business_mirror')->table('business_mirror_table_states')->insert([
'dataset_key' => $key,
'initial_completed_at' => '2026-08-14 12:00:00',
]);
}
}
private function definition(string $object, string $target): array
{
return ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => $object, 'target' => $target, 'keys' => ['id']];
}
private function taobao(int $id, string $time, float $amount, float $balance, string $finType): array
{
return ['id' => $id, 'shopName' => '镜像店铺', 'uniTransTime' => $time, 'transTime' => $time, 'amount' => $amount, 'balance' => $balance, 'comment' => '', 'finType' => $finType, 'bizType' => '', 'tradeType' => ''];
}
private function pinduoduo(int $id, string $time, float $amount, float $balance, int $flowType): array
{
return ['id' => $id, 'shopName' => '镜像店铺', 'amount' => $amount, 'balance' => $balance, 'summary' => '', 'flowType' => $flowType, 'toAdvertiserFundType' => 1, 'createTime' => $time];
}
}

View File

@ -4,6 +4,8 @@ namespace Tests\Unit\EcommerceFinance;
use App\Business\UpstreamBusinessInformation;
use Closure;
use Mockery;
use Modules\EcommerceFinance\Repositories\BusinessInformationMirrorRepository;
use Modules\EcommerceFinance\Services\UpstreamQueryCache;
use Modules\EcommerceFinance\Services\UpstreamQueryCacheFactory;
use Tests\TestCase;
@ -44,7 +46,10 @@ class UpstreamBusinessInformationTest extends TestCase
}
};
$information = new class($caches) extends UpstreamBusinessInformation {
$mirror = Mockery::mock(BusinessInformationMirrorRepository::class);
$mirror->shouldReceive('ready')->once()->with($scope)->andReturnFalse();
$information = new class($caches, $mirror) extends UpstreamBusinessInformation {
public array $snapshots = [];
public function snapshot(string $scope, array $parameters): array
@ -65,6 +70,43 @@ class UpstreamBusinessInformationTest extends TestCase
$this->assertSame($expected, $information->snapshots[0]);
}
/**
* @dataProvider businessInformationPages
*/
public function test_ready_business_mirror_is_used_without_registering_a_page_snapshot(
string $section,
string $page,
string $scope,
): void {
$cache = new class extends UpstreamQueryCache {
public function __construct() {}
public function remember(string $scope, array $parameters, Closure $resolver): mixed
{
throw new \RuntimeException('镜像就绪后不应再读取分页快照。');
}
};
$caches = new class($cache) extends UpstreamQueryCacheFactory {
public function __construct(private readonly UpstreamQueryCache $cache) {}
public function forSource(string $source): UpstreamQueryCache { return $this->cache; }
};
$mirror = Mockery::mock(BusinessInformationMirrorRepository::class);
$mirror->shouldReceive('ready')->once()->with($scope)->andReturnTrue();
$mirror->shouldReceive('paginate')->once()->with($scope, 2, 25)->andReturn([
'data' => [['id' => 10]],
'total' => 51,
]);
$this->app['request']->merge(['page' => 2]);
$paginator = (new UpstreamBusinessInformation($caches, $mirror))
->paginate($section, $page, 25);
$this->assertSame(2, $paginator->currentPage());
$this->assertSame(51, $paginator->total());
$this->assertSame(10, $paginator->items()[0]->id);
}
public static function businessInformationPages(): array
{
return [