fix(发货数据): 对齐工作平台口径并补足两边差异

- 确认基表为发货明细 jst_order_express_details(聚水潭发货单dlv镜像)
- 快递单号回落: logistic_id -> jst_Orders.l_id -> 运费表 expressNo
- 发货时间 send_time 缺失时回落 jst_Orders.send_date(修滞后单)
- 并入自动发货仓(jst_Orders已发货但不在发货明细表), 约2179条
- 剔除补差虚拟单(商品名含 补收差价/差价专用)
- 窗口重跑先按发货时间删旧行再重建, 批量 insertOrIgnore 提速
- 每日 01:28 滚动同步 --days=3

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014nvGZkg27YMEdhRy6EiUwf
This commit is contained in:
limeng 2026-09-21 03:11:31 +00:00
parent f580807097
commit 8cae97df0d
4 changed files with 518 additions and 12 deletions

View File

@ -0,0 +1,25 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
class JstCount extends Command
{
protected $signature = 'jst:count';
protected $description = '发货数据本地表统计';
public function handle(): int
{
$c = DB::connection((string) config('jushuitan.connection', 'jushuitan_local'));
$t = $c->table('jst_oms_shipping');
$this->line('total='.$t->count());
$this->line('autoship(source>=1e9)='.(clone $t)->where('source_id', '>=', 1000000000)->count());
$this->line('express_rows(source<9e6)='.(clone $t)->where('source_id', '<', 9000000)->count());
$this->line('buchai_left='.(clone $t)->where('product_name', 'like', '%补收差价%')->count());
$this->line('express_no_filled='.(clone $t)->whereNotNull('express_no')->where('express_no', '<>', '')->count());
$this->line('express_no_empty='.(clone $t)->where(function ($q) { $q->whereNull('express_no')->orWhere('express_no', ''); })->count());
return self::SUCCESS;
}
}

View File

@ -58,9 +58,9 @@ class Kernel extends ConsoleKernel
// 聚水潭发货数据增量同步:每 10 分钟按 modified 水位拉销售出库单。
// 默认关闭(JST_SYNC_ENABLED=false);填好凭证并完成迁移后再打开。
if ((bool) env('JST_SYNC_ENABLED', false)) {
$schedule->command('jushuitan:sync-shipping')
->everyTenMinutes()
->withoutOverlapping(30)
$schedule->command('jushuitan:sync-shipping --days=3')
->dailyAt('01:28')
->withoutOverlapping(120)
->runInBackground();
}

View File

@ -4,34 +4,47 @@ namespace Modules\EcommerceFinance\Console;
use Illuminate\Console\Command;
use Illuminate\Support\Carbon;
use Modules\EcommerceFinance\Services\Jushuitan\ShippingDataSynchronizer;
use Modules\EcommerceFinance\Services\Jushuitan\ShippingReplicaSynchronizer;
/**
* 聚水潭发货数据同步命令。
* 发货数据同步:从经营分析副本库(operations_replica)直读聚水潭发货明细,清洗进本地展示表。
*
* php artisan jushuitan:sync-shipping # 增量(按水位 - 重叠窗口)
* php artisan jushuitan:sync-shipping --from=... --to=... # 指定时间窗(首次全量分批用)
* php artisan jushuitan:sync-shipping --from=2026-09-01 --to=2026-09-17 # 按发货时间窗口拉全量
* php artisan jushuitan:sync-shipping --days=3 # 滚动最近 3 天(每日调度用)
* php artisan jushuitan:sync-shipping # 最近 2000 条(快速看效果)
*
* 按 id 去重 upsert,任何时候补拉早期区间都不会重复。
*/
class SyncJushuitanShipping extends Command
{
protected $signature = 'jushuitan:sync-shipping {--from=} {--to=}';
protected $signature = 'jushuitan:sync-shipping {--from=} {--to=} {--days=} {--limit=2000}';
protected $description = '从聚水潭(奇门)拉取销售出库单发货数据到 caiwu 共享镜像库';
protected $description = '从经营分析副本库直读聚水潭发货明细,清洗写入 caiwu 发货数据表';
public function handle(): int
{
$from = $this->option('from') ? Carbon::parse($this->option('from')) : null;
$to = $this->option('to') ? Carbon::parse($this->option('to')) : null;
$this->info('开始同步聚水潭发货数据...');
// --days=N:滚动最近 N 天(含今天)。显式 --from/--to 优先。
if (! $from && ! $to && $this->option('days')) {
$days = max((int) $this->option('days'), 1);
$from = now()->startOfDay()->subDays($days - 1);
$to = now()->startOfDay()->addDay();
}
$limit = (int) ($this->option('limit') ?: 2000);
$range = ($from && $to) ? $from->toDateString() . ' ~ ' . $to->toDateString() : '最近 ' . $limit . ' 条';
$this->info('开始从副本库同步发货明细(' . $range . ')...');
try {
$result = ShippingDataSynchronizer::make()->syncIncremental($from, $to);
$result = ShippingReplicaSynchronizer::make()->sync($from, $to, $limit);
} catch (\Throwable $e) {
$this->error('同步失败:' . $e->getMessage());
return self::FAILURE;
}
$this->info(sprintf('完成:处理 %d 单,水位推进到 %s', $result['processed'], $result['watermark'] ?? '(未变)'));
$this->info('完成:写入 ' . ($result['processed'] ?? 0) . ' 条发货明细。');
return self::SUCCESS;
}
}

View File

@ -0,0 +1,468 @@
<?php
namespace Modules\EcommerceFinance\Services\Jushuitan;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Modules\EcommerceFinance\Services\UpstreamReadOnlyQueryFactory;
use Throwable;
/**
* 直读经营分析副本库(operations_replica)的聚水潭发货明细,清洗写入 jst_oms_shipping。
* 不经工作平台、不用聚水潭 API。
*
* 口径对齐工作平台(erp.dlv.query tradetype=1&sendsgn=1):
* - 主口径:rpt.jst_order_express_details 发货单(dlv)镜像,一行一发货单。logistic_id=快递单号(postcode)。
* - 快递单号回落:logistic_id 空 -> jst_Orders.l_id -> 运费表 expressNo。
* - 发货时间回落:send_time 空 -> jst_Orders.send_date(修滞后单)。
* - 剔除补差虚拟单(商品名含 补收差价/差价专用),与工作平台一致。
* - 并入自动发货仓:jst_Orders 已发货(Sent) 但不在发货明细表的单。
* - 窗口重跑:先按发货时间删掉该窗口旧行再重建,避免历史残留。
*/
class ShippingReplicaSynchronizer
{
private const SOURCE = 'operations_replica';
private const DATASET = 'jst.sctrade';
private const AUTOSHIP_OFFSET = 1000000000; // 自动发货仓行的 source_id 偏移,避免与 express_details.id 撞号
private const FLUSH = 200;
private readonly \Modules\EcommerceFinance\Services\UpstreamReadOnlyQuery $up;
private readonly \Illuminate\Database\ConnectionInterface $db;
/** @var array<int,array> 待批量写入缓冲 */
private array $buffer = [];
public function __construct(UpstreamReadOnlyQueryFactory $factory)
{
$this->up = $factory->forSource(self::SOURCE);
$this->db = DB::connection((string) config('jushuitan.connection', 'jushuitan_local'));
}
public static function make(): self
{
return new self(app(UpstreamReadOnlyQueryFactory::class));
}
public function sync(?Carbon $from = null, ?Carbon $to = null, int $limit = 2000): array
{
$this->markRunning();
try {
$windowed = $from && $to;
$now = now();
$fromS = $windowed ? $from->format('Y-m-d H:i:s') : null;
$toS = $windowed ? $to->format('Y-m-d H:i:s') : null;
// 清 demo 行
$this->db->table('jst_oms_shipping')->where('source_id', '>=', 9000000)
->where('source_id', '<', self::AUTOSHIP_OFFSET)->delete();
// 窗口重跑:先删该窗口旧行(含补差历史残留)再重建
if ($windowed) {
$this->db->table('jst_oms_shipping')
->where('ship_time', '>=', $fromS)->where('ship_time', '<', $toS)->delete();
}
$total = $this->syncExpress($windowed, $fromS, $toS, $limit, $now);
$auto = 0;
if ($windowed) {
$auto = $this->syncAutoShip($fromS, $toS, $now);
}
$this->markSucceeded();
return ['processed' => $total, 'autoship' => $auto];
} catch (Throwable $e) {
$this->markFailed($e->getMessage());
throw $e;
}
}
/** 主口径:发货明细表(dlv 镜像)。 */
private function syncExpress(bool $windowed, ?string $fromS, ?string $toS, int $limit, $now): int
{
$batch = 500;
$maxBatches = 2000;
$cols = 'e.id, e.so_id, e.inner_order_id, e.logistic_id, e.logistic_company_name, e.send_time, e.pay_time, '
. 'e.shop_name, e.platform_name, e.dis_co_name, e.warehouse_name, e.receiver_address, '
. 'e.weight, e.f_weight, e.buyer_message, jo.l_id AS jo_lid, jo.send_date AS jo_send';
$total = 0;
$lastId = null;
for ($b = 0; $b < $maxBatches; $b++) {
if ($windowed) {
$where = '((e.send_time >= ? AND e.send_time < ?) OR (e.send_time IS NULL AND jo.send_date >= ? AND jo.send_date < ?))';
$bind = [$fromS, $toS, $fromS, $toS];
} else {
$where = 'e.send_time IS NOT NULL';
$bind = [];
}
if ($lastId !== null) {
$where .= ' AND e.id < ?';
$bind[] = $lastId;
}
$sql = "SELECT TOP {$batch} {$cols} FROM rpt.jst_order_express_details e "
. "LEFT JOIN rpt.jst_Orders jo ON jo.o_id = e.inner_order_id "
. "WHERE {$where} ORDER BY e.id DESC";
$rows = $this->up->select($sql, $bind);
if ($rows === []) {
break;
}
$soIds = [];
foreach ($rows as $r) {
$so = (string) ($r['so_id'] ?? '');
if ($so !== '') {
$soIds[$so] = true;
}
$lastId = $r['id'];
}
$soIds = array_keys($soIds);
$items = $this->loadItems($soIds);
$skuIds = [];
foreach ($items as $it) {
if (! empty($it['sku'])) {
$skuIds[(string) $it['sku']] = true;
}
}
$skus = $this->loadSkus(array_keys($skuIds));
$freight = $this->loadFreight($soIds);
foreach ($rows as $r) {
$id = $r['id'] ?? null;
if ($id === null) {
continue;
}
$so = (string) ($r['so_id'] ?? '');
$it = $items[$so] ?? [];
if ($this->isPricePatch($it['name'] ?? null)) {
continue; // 剔除补差虚拟单
}
$expno = (string) ($r['logistic_id'] ?? '');
$sk = (! empty($it['sku']) && isset($skus[(string) $it['sku']])) ? $skus[(string) $it['sku']] : [];
$f = $freight[$so . '|' . $expno] ?? ($freight[$so . '|'] ?? []);
$tracking = $expno !== '' ? $expno : ((string) ($r['jo_lid'] ?? '') ?: (string) ($f['no'] ?? ''));
$tracking = $tracking !== '' ? $tracking : null;
$shipTime = $this->ts($r['send_time'] ?? null) ?? $this->ts($r['jo_send'] ?? null);
$this->push((int) $id, $this->buildPayload($r, $it, $sk, $f, $tracking, $shipTime, $now));
$total++;
}
if (count($rows) < $batch) {
break;
}
if (! $windowed && $total >= $limit) {
break;
}
}
$this->flush();
return $total;
}
/** 并入自动发货仓:jst_Orders 已发货但不在发货明细表。 */
private function syncAutoShip(string $fromS, string $toS, $now): int
{
$batch = 500;
$maxBatches = 400;
$cols = 'o.Id AS rid, o.o_id, o.so_id, o.l_id, o.lc_id, o.logistics_company, o.order_from, '
. 'o.shop_name, o.send_date, o.pay_date, o.weight, o.f_weight, o.buyer_message, '
. 'o.receiver_state, o.receiver_city, o.receiver_district, o.receiver_address, '
. 'o.receiver_name, o.receiver_mobile, o.receiver_phone';
$total = 0;
$lastRid = null;
for ($b = 0; $b < $maxBatches; $b++) {
$where = "o.status = 'Sent' AND o.send_date >= ? AND o.send_date < ? "
. 'AND NOT EXISTS (SELECT 1 FROM rpt.jst_order_express_details e WHERE e.inner_order_id = o.o_id)';
$bind = [$fromS, $toS];
if ($lastRid !== null) {
$where .= ' AND o.Id < ?';
$bind[] = $lastRid;
}
$sql = "SELECT TOP {$batch} {$cols} FROM rpt.jst_Orders o WHERE {$where} ORDER BY o.Id DESC";
$rows = $this->up->select($sql, $bind);
if ($rows === []) {
break;
}
$soIds = [];
foreach ($rows as $r) {
$so = (string) ($r['so_id'] ?? '');
if ($so !== '') {
$soIds[$so] = true;
}
$lastRid = $r['rid'];
}
$soIds = array_keys($soIds);
$items = $this->loadItems($soIds);
$skuIds = [];
foreach ($items as $it) {
if (! empty($it['sku'])) {
$skuIds[(string) $it['sku']] = true;
}
}
$skus = $this->loadSkus(array_keys($skuIds));
$freight = $this->loadFreight($soIds);
foreach ($rows as $r) {
$so = (string) ($r['so_id'] ?? '');
$oid = $r['o_id'] ?? null;
if ($oid === null) {
continue;
}
$it = $items[$so] ?? [];
if ($this->isPricePatch($it['name'] ?? null)) {
continue;
}
$sk = (! empty($it['sku']) && isset($skus[(string) $it['sku']])) ? $skus[(string) $it['sku']] : [];
$f = $freight[$so . '|'] ?? [];
$tracking = ((string) ($r['l_id'] ?? '')) ?: ((string) ($f['no'] ?? '')) ?: null;
$shipTime = $this->ts($r['send_date'] ?? null);
$sid = self::AUTOSHIP_OFFSET + (int) $oid;
$mapped = [
'so_id' => $so,
'logistic_id' => $tracking,
'logistic_company_name' => $r['logistics_company'] ?? null,
'send_time' => $shipTime,
'pay_time' => $r['pay_date'] ?? null,
'shop_name' => $r['shop_name'] ?? null,
'platform_name' => $r['order_from'] ?? null,
'dis_co_name' => null,
'warehouse_name' => $it['warehouse'] ?? null,
'receiver_address' => $r['receiver_address'] ?? '',
'weight' => $r['weight'] ?? null,
'buyer_message' => $r['buyer_message'] ?? null,
'r_state' => $r['receiver_state'] ?? null,
'r_city' => $r['receiver_city'] ?? null,
'r_district' => $r['receiver_district'] ?? null,
];
$this->push($sid, $this->buildPayload($mapped, $it, $sk, $f, $tracking, $shipTime, $now, true));
$total++;
}
if (count($rows) < $batch) {
break;
}
}
$this->flush();
return $total;
}
/** 入缓冲,满 FLUSH 条批量写。窗口已预删,用 insert 更快。 */
private function push(int $sid, array $payload): void
{
$payload['source_id'] = $sid;
$payload['sku_source_id'] = $sid;
$this->buffer[] = $payload;
if (count($this->buffer) >= self::FLUSH) {
$this->flush();
}
}
private function flush(): void
{
if ($this->buffer === []) {
return;
}
// 去重同一 source_id(同窗口内极少见,保底)
$this->db->table('jst_oms_shipping')->insertOrIgnore($this->buffer);
$this->buffer = [];
}
/** 组装写入行。$auto=true 时省级市区直接取 jst_Orders 字段。 */
private function buildPayload(array $r, array $it, array $sk, array $f, ?string $tracking, ?string $shipTime, $now, bool $auto = false): array
{
$addr = (string) ($r['receiver_address'] ?? '');
$parts = $addr !== '' ? array_values(array_filter(explode('-', $addr))) : [];
$qty = (float) ($it['qty'] ?? 0);
$skuW = (float) ($sk['weight'] ?? 0);
$onlineW = $skuW > 0 ? round($skuW * ($qty > 0 ? $qty : 1), 4) : null;
$actualW = (float) ($r['weight'] ?? 0) ?: null;
$freightW = (float) ($f['w'] ?? 0);
$settleW = $freightW > 0 ? $freightW : ($actualW ?: $onlineW);
$L = $sk['l'] ?? null;
$W = $sk['w'] ?? null;
$H = $sk['h'] ?? null;
$vol = ($sk['vol'] ?? null);
if (($vol === null || (float) $vol <= 0) && $L && $W && $H) {
$vol = (float) $L * (float) $W * (float) $H;
}
$state = $auto ? ($r['r_state'] ?? null) : (($f['state'] ?? null) ?: ($parts[0] ?? null));
$city = $auto ? ($r['r_city'] ?? null) : ($parts[1] ?? null);
$district = $auto ? ($r['r_district'] ?? null) : ($parts[2] ?? null);
return $this->cleanPayload([
'express_no' => $tracking,
'trade_no' => $r['so_id'] ?? null,
'item_id' => $it['shopid'] ?? null,
'barcode' => $it['sku'] ?? null,
'product_name' => $it['name'] ?? null,
'qty' => $it['qty'] ?? null,
'product_code' => $it['iid'] ?? null,
'express_name1' => $r['logistic_company_name'] ?? null,
'express_name2' => ($f['name'] ?? null) ?: ($r['logistic_company_name'] ?? null),
'ship_time' => $shipTime,
'pay_time' => $this->ts($r['pay_time'] ?? null),
'shop_name' => $r['shop_name'] ?? null,
'platform' => $r['platform_name'] ?? null,
'supplier_name' => ($sk['supplier'] ?? null) ?: ($r['dis_co_name'] ?? null),
'warehouse_name' => $r['warehouse_name'] ?? null,
'receiver_state' => $state,
'receiver_city' => $city,
'receiver_district' => $district,
'receiver_address' => $addr,
'online_weight' => $onlineW,
'actual_weight' => $actualW,
'length_cm' => $L,
'width_cm' => $W,
'height_cm' => $H,
'volume_cm3' => $vol,
'weight_kg' => $settleW,
'settle_weight_kg' => $settleW,
'settle_volume_cm3' => $vol,
'amount' => $it['amount'] ?? null,
'freight' => $f['cost'] ?? null,
'settle_freight' => $f['cost'] ?? null,
'buyer_memo' => $r['buyer_message'] ?? null,
'modified' => $shipTime,
'synced_at' => $now,
]);
}
private function isPricePatch(?string $name): bool
{
if ($name === null || $name === '') {
return false;
}
return str_contains($name, '补收差价') || str_contains($name, '差价专用');
}
/** @return array<string,array> so_id => 第一条商品 */
private function loadItems(array $soIds): array
{
$out = [];
foreach (array_chunk($soIds, 800) as $chunk) {
if ($chunk === []) {
continue;
}
$ph = implode(',', array_fill(0, count($chunk), '?'));
$sql = "SELECT raw_so_id, sku_id, i_id, shop_i_id, name, qty, amount, send_warehouse, Id FROM rpt.jst_OrderItem "
. "WHERE raw_so_id IN ({$ph}) ORDER BY Id";
foreach ($this->up->select($sql, $chunk) as $r) {
$so = (string) ($r['raw_so_id'] ?? '');
if ($so === '' || isset($out[$so])) {
continue;
}
$out[$so] = [
'sku' => $r['sku_id'] ?? null, 'iid' => $r['i_id'] ?? null,
'shopid' => $r['shop_i_id'] ?? null, 'name' => $r['name'] ?? null,
'qty' => $r['qty'] ?? null, 'amount' => $r['amount'] ?? null,
'warehouse' => $r['send_warehouse'] ?? null,
];
}
}
return $out;
}
/** @return array<string,array> sku_id => 尺寸/重量/供应商 */
private function loadSkus(array $skuIds): array
{
$out = [];
foreach (array_chunk($skuIds, 800) as $chunk) {
if ($chunk === []) {
continue;
}
$ph = implode(',', array_fill(0, count($chunk), '?'));
$sql = "SELECT sku_id, l, w, h, pack_volume, weight, supplier_name FROM rpt.jst_sku "
. "WHERE sku_id IN ({$ph})";
foreach ($this->up->select($sql, $chunk) as $r) {
$out[(string) $r['sku_id']] = [
'l' => $r['l'] ?? null, 'w' => $r['w'] ?? null, 'h' => $r['h'] ?? null,
'vol' => $r['pack_volume'] ?? null, 'weight' => $r['weight'] ?? null,
'supplier' => $r['supplier_name'] ?? null,
];
}
}
return $out;
}
/** @return array<string,array> so_id|expressNo => 运费 (并含 so_id| 兜底键) */
private function loadFreight(array $soIds): array
{
$out = [];
foreach (array_chunk($soIds, 800) as $chunk) {
if ($chunk === []) {
continue;
}
$ph = implode(',', array_fill(0, count($chunk), '?'));
$sql = 'SELECT so_id, expressNo, SUM(freightCost) AS cost, SUM(weight) AS w, '
. 'MAX(expressName) AS name, MAX(receiver_state) AS state '
. "FROM rpt.fys_jstFreightCostDetail WHERE so_id IN ({$ph}) GROUP BY so_id, expressNo";
foreach ($this->up->select($sql, $chunk) as $r) {
$expno = (string) ($r['expressNo'] ?? '');
$val = ['cost' => $r['cost'] ?? null, 'w' => $r['w'] ?? null, 'name' => $r['name'] ?? null, 'state' => $r['state'] ?? null, 'no' => $expno ?: null];
$out[((string) $r['so_id']) . '|' . $expno] = $val;
$out[((string) $r['so_id']) . '|'] ??= $val;
}
}
return $out;
}
private function cleanPayload(array $data): array
{
foreach ($data as $k => $v) {
if (is_string($v)) {
$data[$k] = $this->cleanStr($v);
}
}
return $data;
}
private function cleanStr(string $v): string
{
$v = str_replace("\0", '', $v);
$clean = @iconv('UTF-8', 'UTF-8//IGNORE', $v);
if ($clean === false) {
$clean = mb_convert_encoding($v, 'UTF-8', 'UTF-8');
}
return $clean;
}
private function ts(mixed $v): ?string
{
if ($v === null || $v === '') {
return null;
}
try {
return Carbon::parse((string) $v)->format('Y-m-d H:i:s');
} catch (Throwable) {
return null;
}
}
private function markRunning(): void
{
$this->db->table('jst_sync_state')->updateOrInsert(
['dataset_key' => self::DATASET],
['status' => 'running', 'last_started_at' => now(), 'updated_at' => now(), 'created_at' => now()]
);
}
private function markSucceeded(): void
{
$this->db->table('jst_sync_state')->where('dataset_key', self::DATASET)->update([
'status' => 'succeeded', 'last_succeeded_at' => now(), 'last_error' => null, 'watermark' => now(), 'updated_at' => now(),
]);
}
private function markFailed(string $err): void
{
$this->db->table('jst_sync_state')->where('dataset_key', self::DATASET)->update([
'status' => 'failed', 'last_error' => mb_substr($err, 0, 1000), 'updated_at' => now(),
]);
}
}