Merge remote-tracking branch 'origin/main' into fix/qiuxiang-vipshop-filter-options

This commit is contained in:
qiuxiang 2026-09-24 10:16:27 +08:00
commit 5b20d11dfa
2 changed files with 127 additions and 16 deletions

View File

@ -39,6 +39,9 @@ class ShippingReplicaSynchronizer
/** @var array<string,array> 窗口内 dws 销售全量商品缓存 sourceno=>items */
private array $dwsCache = [];
/** @var array{byL:array<string,string>,byO:array<string,string>} 窗口内出库单映射 */
private array $outboundMap = ['byL' => [], 'byO' => []];
public function __construct(UpstreamReadOnlyQueryFactory $factory)
{
$this->up = $factory->forSource(self::SOURCE);
@ -70,13 +73,17 @@ class ShippingReplicaSynchronizer
$auto = 0;
if ($windowed) {
$this->preloadDws($from, $to);
$this->preloadOutboundMap($fromS, $toS);
if ($progress) {
$progress('dws 预载完成:' . count($this->dwsCache) . ' 个 sourceno');
$progress(sprintf(
'出库映射预载完成:%d 个快递单号,%d 个内部单号',
count($this->outboundMap['byL']),
count($this->outboundMap['byO']),
));
}
$total = $this->syncExpress(true, $fromS, $toS, $limit, $now, $progress);
$auto = $this->syncAutoShip($fromS, $toS, $now, $progress);
$this->dwsCache = [];
gc_collect_cycles();
$removed = $this->dedupAtPrefix();
if ($progress && $removed > 0) {
$progress('去除@重复单号:' . $removed . ' 条');
@ -90,6 +97,10 @@ class ShippingReplicaSynchronizer
} catch (Throwable $e) {
$this->markFailed($e->getMessage());
throw $e;
} finally {
$this->dwsCache = [];
$this->outboundMap = ['byL' => [], 'byO' => []];
gc_collect_cycles();
}
}
@ -200,7 +211,7 @@ class ShippingReplicaSynchronizer
$logs[$lg] = true;
}
}
[$byOid, $bySo] = $this->loadAllItems(array_keys($oids), array_keys($soIds));
[$byOid, $bySo] = $this->loadAllItems(array_keys($oids), array_keys($soIds), ! $windowed);
$dws = [];
foreach (array_keys($soIds) as $mso) {
if (empty($bySo[$mso]) && ! empty($this->dwsCache[$mso])) {
@ -208,7 +219,7 @@ class ShippingReplicaSynchronizer
}
}
// 主口径:发货数据视图按出库单(io_id)取实发商品
$ioMap = $this->loadIoMap(array_keys($logs), array_keys($oids));
$ioMap = $this->ioMapFor($windowed, array_keys($logs), array_keys($oids));
$ioIds = [];
foreach ($rows as $r) {
$io = $this->ioFor($ioMap, (string) ($r['logistic_id'] ?? ''), (string) ($r['inner_order_id'] ?? ''));
@ -311,14 +322,14 @@ class ShippingReplicaSynchronizer
}
$lastRid = $r['rid'];
}
[$byOid, $bySo] = $this->loadAllItems(array_keys($oids), array_keys($soIds));
[$byOid, $bySo] = $this->loadAllItems(array_keys($oids), array_keys($soIds), false);
$dws = [];
foreach (array_keys($soIds) as $mso) {
if (empty($bySo[$mso]) && ! empty($this->dwsCache[$mso])) {
$dws[$mso] = $this->dwsCache[$mso];
}
}
$ioMap = $this->loadIoMap(array_keys($logs), array_keys($oids));
$ioMap = $this->ioMapFor(true, array_keys($logs), array_keys($oids));
$ioIds = [];
foreach ($rows as $r) {
$io = $this->ioFor($ioMap, (string) ($r['l_id'] ?? ''), (string) ($r['o_id'] ?? ''));
@ -518,7 +529,7 @@ class ShippingReplicaSynchronizer
}
/** @return array{0:array<string,array>,1:array<string,array>} [byOid, bySo] */
private function loadAllItems(array $oids, array $soIds): array
private function loadAllItems(array $oids, array $soIds, bool $loadBySoId = true): array
{
$byOid = [];
$bySo = [];
@ -536,15 +547,21 @@ class ShippingReplicaSynchronizer
}
}
}
foreach (array_chunk($soIds, 800) as $chunk) {
if ($chunk === []) {
continue;
}
$ph = implode(',', array_fill(0, count($chunk), '?'));
foreach ($this->up->select("SELECT {$sel} FROM rpt.jst_OrderItem WHERE raw_so_id IN ({$ph}) ORDER BY Id", $chunk) as $r) {
$k = (string) ($r['raw_so_id'] ?? '');
if ($k !== '') {
$bySo[$k][] = $this->itemArr($r);
// raw_so_id lacks a usable index in the production replica. Windowed sync already
// preloads DWS rows by sourceno, so querying this fallback can turn every batch into
// a full-table scan and exceed the upstream timeout. Keep it only for the small,
// non-windowed diagnostic mode where DWS is not preloaded.
if ($loadBySoId) {
foreach (array_chunk($soIds, 800) as $chunk) {
if ($chunk === []) {
continue;
}
$ph = implode(',', array_fill(0, count($chunk), '?'));
foreach ($this->up->select("SELECT {$sel} FROM rpt.jst_OrderItem WHERE raw_so_id IN ({$ph}) ORDER BY Id", $chunk) as $r) {
$k = (string) ($r['raw_so_id'] ?? '');
if ($k !== '') {
$bySo[$k][] = $this->itemArr($r);
}
}
}
}
@ -644,6 +661,32 @@ class ShippingReplicaSynchronizer
return [];
}
/** 窗口同步按 io_date 一次预载,避免对无可用索引的 l_id/o_id 反复执行 IN 查询。 */
private function preloadOutboundMap(string $fromS, string $toS): void
{
$this->outboundMap = ['byL' => [], 'byO' => []];
$sql = 'SELECT l_id, o_id, io_id FROM ' . self::OUTBOUND . ' WHERE io_date >= ? AND io_date < ?';
foreach ($this->up->select($sql, [$fromS, $toS]) as $r) {
$io = (string) ($r['io_id'] ?? '');
if ($io === '') {
continue;
}
$l = (string) ($r['l_id'] ?? '');
if ($l !== '' && ! isset($this->outboundMap['byL'][$l])) {
$this->outboundMap['byL'][$l] = $io;
}
$o = (string) ($r['o_id'] ?? '');
if ($o !== '' && ! isset($this->outboundMap['byO'][$o])) {
$this->outboundMap['byO'][$o] = $io;
}
}
}
private function ioMapFor(bool $windowed, array $logisticIds, array $oIds): array
{
return $windowed ? $this->outboundMap : $this->loadIoMap($logisticIds, $oIds);
}
/** 快递单号/内部单号 -> 出库单 io_id(优先快递单号,精确到该次出库)。 */
private function loadIoMap(array $logisticIds, array $oIds): array
{

View File

@ -10,6 +10,74 @@ use Tests\TestCase;
class ShippingReplicaSynchronizerTest extends TestCase
{
public function test_windowed_outbound_lookup_preloads_date_range_and_reuses_cache_without_in_queries(): void
{
$from = '2026-09-22 00:00:00';
$to = '2026-09-25 00:00:00';
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->expects($this->once())
->method('select')
->with(
$this->callback(fn (string $sql): bool => str_contains($sql, 'WHERE io_date >= ? AND io_date < ?')
&& ! str_contains($sql, 'l_id IN')
&& ! str_contains($sql, 'o_id IN')),
[$from, $to],
)
->willReturn([
['l_id' => 'logistic-1', 'o_id' => 'order-1', 'io_id' => 'outbound-1'],
['l_id' => 'logistic-2', 'o_id' => 'order-2', 'io_id' => 'outbound-2'],
]);
$factory = $this->createMock(UpstreamReadOnlyQueryFactory::class);
$factory->method('forSource')->with('operations_replica')->willReturn($upstream);
$synchronizer = new ShippingReplicaSynchronizer($factory);
$preload = new \ReflectionMethod($synchronizer, 'preloadOutboundMap');
$preload->setAccessible(true);
$preload->invoke($synchronizer, $from, $to);
$lookup = new \ReflectionMethod($synchronizer, 'ioMapFor');
$lookup->setAccessible(true);
$map = $lookup->invoke($synchronizer, true, ['logistic-1'], ['order-2']);
$this->assertSame('outbound-1', $map['byL']['logistic-1']);
$this->assertSame('outbound-2', $map['byO']['order-2']);
}
public function test_windowed_item_lookup_skips_unindexed_raw_so_id_fallback(): void
{
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->expects($this->once())
->method('select')
->with(
$this->stringContains('WHERE o_id IN'),
['order-1'],
)
->willReturn([
[
'o_id' => 'order-1',
'raw_so_id' => 'trade-1',
'sku_id' => 'sku-1',
'i_id' => 'item-1',
'shop_i_id' => 'shop-item-1',
'name' => '商品',
'qty' => 1,
'amount' => 10,
'send_warehouse' => '仓库',
'Id' => 1,
],
]);
$factory = $this->createMock(UpstreamReadOnlyQueryFactory::class);
$factory->method('forSource')->with('operations_replica')->willReturn($upstream);
$synchronizer = new ShippingReplicaSynchronizer($factory);
$method = new \ReflectionMethod($synchronizer, 'loadAllItems');
$method->setAccessible(true);
[$byOrder, $byTrade] = $method->invoke($synchronizer, ['order-1'], ['trade-1'], false);
$this->assertArrayHasKey('order-1', $byOrder);
$this->assertSame([], $byTrade);
}
public function test_sync_removes_only_the_known_demo_source_ids(): void
{
$db = DB::connection('business_mirror');