fix: remove shipping sync order join timeout

This commit is contained in:
limeng 2026-09-24 18:27:45 +08:00
parent 896e3fefa7
commit e224cbebc3
2 changed files with 125 additions and 23 deletions

View File

@ -82,6 +82,9 @@ class ShippingReplicaSynchronizer
/** @var array<string,array> 窗口内 dws 销售全量商品缓存 sourceno=>items */
private array $dwsCache = [];
/** @var array<string,array{l_id:mixed,send_date:mixed}> 已查询订单元数据缓存 o_id=>字段 */
private array $orderMetadata = [];
/** @var array{byL:array<string,string>,byO:array<string,string>} 窗口内出库单映射 */
private array $outboundMap = ['byL' => [], 'byO' => []];
@ -153,6 +156,10 @@ class ShippingReplicaSynchronizer
$total = $this->syncExpress(false, $fromS, $toS, $limit, $now, $progress);
}
if ($windowed && ! $skipFreight && $this->staleMirrorPackages === 0) {
$this->markCovered(self::FREIGHT_DATASET_PREFIX, $fromS, $toS);
}
if ($windowed && ($this->dwsFailedDays !== [] || $this->packagesWithoutItems > 0
|| $this->packagesRetainedForUnmatchedAmount > 0)) {
$this->markItemsIncomplete($fromS, $toS);
@ -185,6 +192,7 @@ class ShippingReplicaSynchronizer
throw $e;
} finally {
$this->dwsCache = [];
$this->orderMetadata = [];
$this->outboundMap = ['byL' => [], 'byO' => []];
$this->replacementPackages = [];
$this->replacementHasFreight = [];
@ -249,7 +257,7 @@ class ShippingReplicaSynchronizer
$maxBatches = $this->maxExpressBatches;
$cols = 'e.id, e.so_id, e.inner_order_id, e.logistic_id, e.logistic_company_name, e.send_time, e.pay_time, '
. 'e.shop_name, e.platform_name, e.dis_co_name, e.warehouse_name, e.receiver_address, '
. 'e.weight, e.f_weight, e.buyer_message, e.order_labels, jo.l_id AS jo_lid, jo.send_date AS jo_send';
. 'e.weight, e.f_weight, e.buyer_message, e.order_labels';
// 窗口模式:只按主键 e.id 翻页(有索引,毫秒级),send_time 窗口过滤放到 PHP,
// 避免每页都对无索引的 send_time 做全表扫描(那是"卡死"的真凶)。
@ -267,7 +275,6 @@ class ShippingReplicaSynchronizer
$bind[] = $lastId;
}
$sql = "SELECT TOP {$batch} {$cols} FROM rpt.jst_order_express_details e "
. 'LEFT JOIN rpt.jst_Orders jo ON jo.o_id = e.inner_order_id '
. "WHERE {$where} ORDER BY e.id DESC";
$rows = $this->selectWithReconnect($sql, $bind);
if ($rows === []) {
@ -282,6 +289,11 @@ class ShippingReplicaSynchronizer
$lastId = $r['id'];
}
// 主表分页不联接订单表;每页只按 inner_order_id 分块补齐订单字段,
// 并复用已经缓存的结果,避免 LEFT JOIN 在大历史窗口上拖垮游标。
$this->loadOrderMetadataForRows($rows);
$rows = array_map(fn (array $row): array => $this->mergeOrderMetadata($row), $rows);
if ($windowed) {
$kept = [];
foreach ($rows as $r) {
@ -382,6 +394,54 @@ class ShippingReplicaSynchronizer
return $total;
}
/** 按当前页订单键补齐订单字段,并复用已缓存的结果。 */
private function loadOrderMetadataForRows(array $rows): void
{
$oids = [];
foreach ($rows as $row) {
$oid = (string) ($row['inner_order_id'] ?? '');
if ($oid !== '' && ! array_key_exists($oid, $this->orderMetadata)) {
$oids[$oid] = true;
}
}
foreach (array_chunk(array_keys($oids), 800) as $chunk) {
if ($chunk === []) {
continue;
}
$ph = implode(',', array_fill(0, count($chunk), '?'));
$sql = "SELECT o_id, l_id, send_date FROM rpt.jst_Orders om WHERE om.o_id IN ({$ph})";
foreach ($this->selectWithReconnect($sql, $chunk) as $row) {
$oid = (string) ($row['o_id'] ?? '');
if ($oid === '') {
continue;
}
$this->orderMetadata[$oid] = [
'l_id' => $row['l_id'] ?? null,
'send_date' => $row['send_date'] ?? null,
];
}
foreach ($chunk as $oid) {
// 记录无匹配订单,避免后续分页反复查询同一个缺失键。
$this->orderMetadata[$oid] ??= ['l_id' => null, 'send_date' => null];
}
}
}
/** 将订单元数据映射到发货明细,保留测试/兼容来源已提供的字段。 */
private function mergeOrderMetadata(array $row): array
{
$oid = (string) ($row['inner_order_id'] ?? '');
$metadata = $oid !== '' ? ($this->orderMetadata[$oid] ?? []) : [];
if (($row['jo_lid'] ?? '') === '' && ($metadata['l_id'] ?? '') !== '') {
$row['jo_lid'] = $metadata['l_id'];
}
if (($row['jo_send'] ?? '') === '' && ($metadata['send_date'] ?? '') !== '') {
$row['jo_send'] = $metadata['send_date'];
}
return $row;
}
/** @return list<array<string,mixed>> */
private function selectWithReconnect(string $sql, array $bindings): array
{

View File

@ -14,6 +14,48 @@ use Tests\TestCase;
class ShippingReplicaSynchronizerTest extends TestCase
{
public function test_windowed_express_scan_maps_order_metadata_without_joining_orders_per_page(): void
{
$detailQueries = [];
$orderMetadataQueries = [];
$detailCalls = 0;
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql, array $bindings) use (&$detailQueries, &$orderMetadataQueries, &$detailCalls): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
$detailQueries[] = $sql;
return $detailCalls++ === 0
? [['id' => 5000, 'inner_order_id' => 'order-1', 'send_time' => null, 'logistic_id' => '']]
: [];
}
if (str_contains($sql, 'FROM rpt.jst_Orders om WHERE om.o_id IN')) {
$orderMetadataQueries[] = [$sql, $bindings];
return [['o_id' => 'order-1', 'l_id' => 'order-tracking', 'send_date' => '2026-09-02 12:00:00']];
}
if (str_contains($sql, 'FROM rpt.jst_Orders o WHERE o.status =')) {
return [];
}
return [];
});
$factory = $this->createMock(UpstreamReadOnlyQueryFactory::class);
$factory->method('forSource')->with('operations_replica')->willReturn($upstream);
$result = (new ShippingReplicaSynchronizer($factory))->sync(
Carbon::parse('2026-09-01'), Carbon::parse('2026-09-05'), 2000, null, true,
);
$this->assertNotEmpty($detailQueries);
$this->assertStringNotContainsString('LEFT JOIN rpt.jst_Orders', $detailQueries[0]);
$this->assertCount(1, $orderMetadataQueries);
$this->assertSame(['order-1'], $orderMetadataQueries[0][1]);
$this->assertSame(1, $result['processed']);
$row = DB::connection('business_mirror')->table('jst_oms_shipping')->where('source_id', 5000)->sole();
$this->assertSame('order-tracking', $row->express_no);
$this->assertSame('2026-09-02 12:00:00', $row->ship_time);
}
public function test_windowed_outbound_lookup_preloads_date_range_and_reuses_cache_without_in_queries(): void
{
$from = '2026-09-22 00:00:00';
@ -92,7 +134,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
$scanCalls = 0;
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql) use (&$scanCalls, $pages): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return $pages[$scanCalls++] ?? [];
}
@ -123,7 +165,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
$bindings = [];
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql, array $bind) use (&$scanCalls, &$bindings, $firstPage, $lastPage): array {
if (! str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (! str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return [];
}
$scanCalls++;
@ -154,7 +196,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
$calls = 0;
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql) use (&$calls): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
$calls++;
throw new PDOException('SQLSTATE[HY000]: unrelated upstream error');
}
@ -187,7 +229,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
$bindings = [];
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql, array $bind) use (&$scanCalls, &$bindings, $page): array {
if (! str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (! str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return [];
}
$scanCalls++;
@ -215,7 +257,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
{
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
$row = ['id' => 5000, 'send_time' => '2026-09-02 12:00:00'];
return [$row, $row];
}
@ -248,7 +290,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
$scanCalls = 0;
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql) use (&$scanCalls, $pages): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return $pages[$scanCalls++] ?? [];
}
@ -281,7 +323,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
$scanCalls = 0;
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql) use (&$scanCalls): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return $scanCalls++ === 0
? array_map(fn (int $id): array => ['id' => $id], range(1500, 1))
: [];
@ -339,7 +381,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
$orderQueries = [];
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql, array $bindings) use (&$orderQueries): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return [
['id' => 1001, 'inner_order_id' => '123', 'jo_send' => '2026-09-02 13:00:00',
'send_time' => '2026-08-31 12:00:00'],
@ -379,7 +421,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
$orderBindings = [];
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql, array $bindings) use (&$orderBindings): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return array_map(fn (int $oid): array => [
'id' => $oid, 'inner_order_id' => (string) $oid,
'jo_send' => '2026-09-02 13:00:00', 'send_time' => '2026-08-31 12:00:00',
@ -432,7 +474,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
if (str_contains($sql, 'fys_jstFreightCostDetail')) {
$this->fail('Skip-freight mode must not query the freight view.');
}
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return $scanCalls++ === 0 ? [
['id' => 6000, 'send_time' => '2026-09-03 12:00:00', 'weight' => 2, 'logistic_id' => 'new-weight-tracking'],
['id' => 5000, 'send_time' => '2026-09-02 12:00:00', 'weight' => 2, 'logistic_id' => 'new-tracking'],
@ -507,7 +549,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
$scanCalls = 0;
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql) use (&$scanCalls): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return $scanCalls++ === 0 ? [
['id' => 6000, 'so_id' => 'so-6000', 'logistic_id' => 'new-6000',
'send_time' => '2026-09-03 12:00:00', 'weight' => 2],
@ -540,7 +582,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
$this->assertEquals(7.50, $updated->freight);
$this->assertEquals(7.50, $updated->settle_freight);
$this->assertEquals(2.5, $updated->settle_weight_kg);
$this->assertSame('pending', $db->table('jst_sync_state')
$this->assertSame('succeeded', $db->table('jst_sync_state')
->where('dataset_key', 'jst.sctrade.freight.20260901000000-20260905000000')->value('status'));
}
@ -565,7 +607,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
$scanCalls = 0;
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql) use (&$scanCalls): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return $scanCalls++ === 0 ? [[
'id' => 5000, 'inner_order_id' => 'order-1', 'send_time' => '2026-09-02 12:00:00',
]] : [];
@ -629,7 +671,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
if (str_contains($sql, 'rpt.dws_sn_sales_full_info_di') && $bindings === ['2026-09-02']) {
throw new RuntimeException('DWS read timed out');
}
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return [['id' => 5000, 'inner_order_id' => 'order-1',
'send_time' => '2026-09-02 12:00:00']];
}
@ -667,7 +709,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
]);
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return [['id' => 5000, 'inner_order_id' => 'order-1',
'send_time' => '2026-09-02 12:00:00']];
}
@ -710,7 +752,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
]);
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return [
['id' => 5000, 'inner_order_id' => 'order-1', 'send_time' => '2026-09-02 12:00:00'],
['id' => 6000, 'inner_order_id' => 'order-2', 'send_time' => '2026-09-02 13:00:00'],
@ -759,7 +801,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
]);
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return [['id' => 5000, 'inner_order_id' => 'order-1',
'send_time' => '2026-09-02 12:00:00']];
}
@ -787,7 +829,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
{
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return [['id' => 5000, 'send_time' => '2026-09-02 12:00:00']];
}
if (str_contains($sql, 'FROM rpt.jst_Orders o WHERE')) {
@ -819,7 +861,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
$scanCalls = 0;
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql) use (&$scanCalls): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
if ($scanCalls++ > 0) {
throw new RuntimeException('upstream page failed');
}
@ -892,7 +934,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
]);
$upstream = $this->createMock(UpstreamReadOnlyQuery::class);
$upstream->method('select')->willReturnCallback(function (string $sql): array {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return [[
'id' => 5000,
'send_time' => '2026-09-02 12:00:00',
@ -921,7 +963,7 @@ class ShippingReplicaSynchronizerTest extends TestCase
if (str_contains($sql, 'rpt.dws_sn_sales_full_info_di') && $bindings === ['2026-08-31']) {
throw new RuntimeException('read timed out');
}
if (str_contains($sql, 'FROM rpt.jst_order_express_details e LEFT JOIN')) {
if (str_contains($sql, 'FROM rpt.jst_order_express_details e')) {
return [['id' => 1234, 'send_time' => '2026-09-02 12:00:00']];
}