镜像 ops_jst_orders 只覆盖 2026-04-01 之后,缺的历史区间只能靠财务在聚水潭「导出订单」导文件补。所以:- 新增 JstOrderExportReader:ZipArchive+expat 流式解析聚水潭导出 xlsx。这类文件字符串走 inlineStr、dimension 只写 A1、单文件上万行 ×135 列(解压 78MB),常规读法会读成空表或吃满内存;实测 12286 行 3.5 秒、峰值内存 10MB。 - jst-order-labels:import 新增 --file=:按「线上订单号+标签」去重后拆行写入,可配合 --from/--to 过滤窗口;带 --replace 时先成功读到数据行再清区间,避免文件有问题却先删了线上数据。 - 筛选下拉上限 500 -> 5000:导入 4~7 月后店铺已达 604 个,原来下拉里会缺 104 个店铺;提交筛选值的截断上限同步放开。 - 补两个测试:导出文件去重拆行、窗口过滤+replace。
338 lines
12 KiB
PHP
338 lines
12 KiB
PHP
<?php
|
||
|
||
namespace App\Business;
|
||
|
||
use Illuminate\Database\DatabaseManager;
|
||
use InvalidArgumentException;
|
||
use RuntimeException;
|
||
|
||
/**
|
||
* 聚水潭订单标签明细导入(拆标签写入 jst_order_labels)。
|
||
*
|
||
* 一个标签一行:一单有 N 个标签就写 N 行;订单不去重、标签不汇总。
|
||
* 无标签的订单保留一行,标签记为「(无标签)」,方便与订单总数核对。
|
||
*
|
||
* 两个数据源:
|
||
* - 共享镜像 ops_jst_orders(常规,覆盖镜像有数据的区间);
|
||
* - 聚水潭「导出订单」Excel(镜像没有的历史区间,例如 2026-03,只能靠财务导出补)。
|
||
*/
|
||
class JstOrderLabelImporter
|
||
{
|
||
public const NO_LABEL = '(无标签)';
|
||
|
||
private const READ_CHUNK = 2000;
|
||
|
||
private const INSERT_CHUNK = 2000;
|
||
|
||
/** 聚水潭导出文件里用到的表头。 */
|
||
private const EXPORT_COLUMNS = ['线上订单号', '店铺名称', '下单时间', '标签'];
|
||
|
||
public function __construct(
|
||
private readonly DatabaseManager $databases,
|
||
private readonly ?JstOrderExportReader $exportReader = null,
|
||
) {}
|
||
|
||
/**
|
||
* @return array{from:string,to:string,orders:int,rows:int,no_label_orders:int,labels:int,seconds:float}
|
||
*/
|
||
public function import(string $from, string $to, int $companyId = 1, bool $replace = false, string $source = 'jst-order-label-import'): array
|
||
{
|
||
$from = $this->normalizeDate($from, '起始日期');
|
||
$to = $this->normalizeDate($to, '结束日期');
|
||
if ($from > $to) {
|
||
throw new InvalidArgumentException('起始日期不能晚于结束日期。');
|
||
}
|
||
|
||
$start = $from . ' 00:00:00';
|
||
$end = date('Y-m-d', strtotime($to . ' +1 day')) . ' 00:00:00';
|
||
|
||
$mirror = $this->databases->connection(config('upstream_mirrors.connection', 'business_mirror'));
|
||
if (! $mirror->getSchemaBuilder()->hasTable('ops_jst_orders')) {
|
||
throw new RuntimeException('聚水潭订单镜像尚未建立(ops_jst_orders)。');
|
||
}
|
||
|
||
$target = $this->databases->connection();
|
||
$startedAt = microtime(true);
|
||
|
||
// 清空区间放到第一次成功读到数据行之后:万一文件打不开或表头不对,
|
||
// 不能先把线上已有行删掉又导入失败。
|
||
$clearWindow = function () use ($target, $companyId, $start, $end): void {
|
||
$target->table('jst_order_labels')
|
||
->where('company_id', $companyId)
|
||
->where('ordered_at', '>=', $start)
|
||
->where('ordered_at', '<', $end)
|
||
->delete();
|
||
};
|
||
$cleared = ! $replace;
|
||
|
||
$now = now();
|
||
$orders = 0;
|
||
$rows = 0;
|
||
$noLabelOrders = 0;
|
||
$buffer = [];
|
||
|
||
$flush = function () use (&$buffer, &$rows, $target): void {
|
||
foreach (array_chunk($buffer, self::INSERT_CHUNK) as $chunk) {
|
||
$target->table('jst_order_labels')->insert($chunk);
|
||
$rows += count($chunk);
|
||
}
|
||
$buffer = [];
|
||
};
|
||
|
||
$mirror->table('ops_jst_orders')
|
||
->where('order_date', '>=', $start)
|
||
->where('order_date', '<', $end)
|
||
->orderBy('Id')
|
||
->chunkById(self::READ_CHUNK, function ($records) use (&$buffer, &$orders, &$noLabelOrders, $flush, $companyId, $source, $now): void {
|
||
foreach ($records as $record) {
|
||
$orders++;
|
||
$orderedAt = $record->order_date === null ? null : substr((string) $record->order_date, 0, 19);
|
||
if ($orderedAt === null || $orderedAt === '') {
|
||
continue;
|
||
}
|
||
|
||
$labels = $this->splitLabels((string) ($record->labels ?? ''));
|
||
if ($labels === []) {
|
||
$labels = [self::NO_LABEL];
|
||
$noLabelOrders++;
|
||
}
|
||
|
||
foreach ($labels as $label) {
|
||
$buffer[] = [
|
||
'company_id' => $companyId,
|
||
'order_no' => mb_substr((string) $record->so_id, 0, 100),
|
||
'shop_name' => mb_substr((string) ($record->shop_name ?? ''), 0, 191),
|
||
'ordered_at' => $orderedAt,
|
||
'label' => $label,
|
||
'created_from' => $source,
|
||
'created_at' => $now,
|
||
'updated_at' => $now,
|
||
];
|
||
}
|
||
|
||
if (count($buffer) >= self::INSERT_CHUNK * 2) {
|
||
$flush();
|
||
}
|
||
}
|
||
}, 'Id');
|
||
|
||
$flush();
|
||
|
||
$labels = $target->table('jst_order_labels')
|
||
->where('company_id', $companyId)
|
||
->where('ordered_at', '>=', $start)
|
||
->where('ordered_at', '<', $end)
|
||
->distinct()
|
||
->count('label');
|
||
|
||
// 导入会改变筛选选项,重新预热缓存,避免页面下次打开时做几秒的 distinct 扫描。
|
||
app(JstOrderLabelOptions::class)->forget($companyId);
|
||
app(JstOrderLabelOptions::class)->warm($companyId);
|
||
|
||
return [
|
||
'from' => $from,
|
||
'to' => $to,
|
||
'orders' => $orders,
|
||
'rows' => $rows,
|
||
'no_label_orders' => $noLabelOrders,
|
||
'labels' => (int) $labels,
|
||
'seconds' => round(microtime(true) - $startedAt, 1),
|
||
];
|
||
}
|
||
|
||
/**
|
||
* 从聚水潭「导出订单」Excel 导入。
|
||
*
|
||
* 导出文件是一单多行(商品明细),这里按「线上订单号 + 标签」去重后拆行写入;
|
||
* 同时兼容「导出订单」直接勾选 4 列导出的极简表头。
|
||
*
|
||
* 传 $from/$to 时会顺带按窗口过滤;带 $replace 时必须给窗口,
|
||
* 以便在写入前先清掉该区间,避免重复导入翻倍。
|
||
*
|
||
* @return array{file:string,company_id:int,from:?string,to:?string,orders:int,rows:int,no_label_orders:int,skipped:int,labels:int,seconds:float}
|
||
*/
|
||
public function importFromFile(
|
||
string $path,
|
||
?string $from = null,
|
||
?string $to = null,
|
||
int $companyId = 1,
|
||
bool $replace = false,
|
||
string $source = 'jst-order-label-file-import',
|
||
): array {
|
||
$from = $from === null || trim($from) === '' ? null : $this->normalizeDate(trim($from), '起始日期');
|
||
$to = $to === null || trim($to) === '' ? null : $this->normalizeDate(trim($to), '结束日期');
|
||
if ($from !== null && $to !== null && $from > $to) {
|
||
throw new InvalidArgumentException('起始日期不能晚于结束日期。');
|
||
}
|
||
if ($replace && ($from === null || $to === null)) {
|
||
throw new InvalidArgumentException('按文件导入并使用 --replace 时,必须同时给出 --from 与 --to,才能确定要清空的区间。');
|
||
}
|
||
|
||
$start = $from === null ? null : $from . ' 00:00:00';
|
||
$end = $to === null ? null : date('Y-m-d', strtotime($to . ' +1 day')) . ' 00:00:00';
|
||
|
||
$target = $this->databases->connection();
|
||
$startedAt = microtime(true);
|
||
|
||
if ($replace) {
|
||
$target->table('jst_order_labels')
|
||
->where('company_id', $companyId)
|
||
->where('ordered_at', '>=', $start)
|
||
->where('ordered_at', '<', $end)
|
||
->delete();
|
||
}
|
||
|
||
$now = now();
|
||
$orders = 0;
|
||
$rows = 0;
|
||
$noLabelOrders = 0;
|
||
$skipped = 0;
|
||
$seen = [];
|
||
$orderSeen = [];
|
||
$buffer = [];
|
||
|
||
$flush = function () use (&$buffer, &$rows, $target): void {
|
||
foreach (array_chunk($buffer, self::INSERT_CHUNK) as $chunk) {
|
||
$target->table('jst_order_labels')->insert($chunk);
|
||
$rows += count($chunk);
|
||
}
|
||
$buffer = [];
|
||
};
|
||
|
||
$reader = $this->exportReader ?? app(JstOrderExportReader::class);
|
||
$anyRow = false;
|
||
|
||
foreach ($reader->rows($path, self::EXPORT_COLUMNS) as $line) {
|
||
$anyRow = true;
|
||
if (! $cleared) {
|
||
$clearWindow();
|
||
$cleared = true;
|
||
}
|
||
|
||
$orderNo = trim((string) ($line['线上订单号'] ?? ''));
|
||
$orderedAt = $this->normalizeOrderedAt((string) ($line['下单时间'] ?? ''));
|
||
if ($orderNo === '' || $orderedAt === null) {
|
||
$skipped++;
|
||
|
||
continue;
|
||
}
|
||
if ($start !== null && ($orderedAt < $start || $orderedAt >= $end)) {
|
||
$skipped++;
|
||
|
||
continue;
|
||
}
|
||
|
||
if (! isset($orderSeen[$orderNo])) {
|
||
$orderSeen[$orderNo] = true;
|
||
$orders++;
|
||
}
|
||
|
||
$shopName = mb_substr(trim((string) ($line['店铺名称'] ?? '')), 0, 191);
|
||
$labels = $this->splitLabels((string) ($line['标签'] ?? ''));
|
||
$withoutLabel = $labels === [];
|
||
if ($labels === []) {
|
||
$labels = [self::NO_LABEL];
|
||
}
|
||
|
||
foreach ($labels as $label) {
|
||
$fingerprint = hash('xxh3', $orderNo . "\0" . $label);
|
||
if (isset($seen[$fingerprint])) {
|
||
continue;
|
||
}
|
||
$seen[$fingerprint] = true;
|
||
if ($withoutLabel) {
|
||
$noLabelOrders++;
|
||
}
|
||
|
||
$buffer[] = [
|
||
'company_id' => $companyId,
|
||
'order_no' => mb_substr($orderNo, 0, 100),
|
||
'shop_name' => $shopName,
|
||
'ordered_at' => $orderedAt,
|
||
'label' => $label,
|
||
'created_from' => $source,
|
||
'created_at' => $now,
|
||
'updated_at' => $now,
|
||
];
|
||
}
|
||
|
||
if (count($buffer) >= self::INSERT_CHUNK * 2) {
|
||
$flush();
|
||
}
|
||
}
|
||
|
||
if (! $anyRow) {
|
||
throw new RuntimeException('订单导出文件里没有数据行:' . basename($path));
|
||
}
|
||
if (! $cleared) {
|
||
$clearWindow();
|
||
}
|
||
|
||
$flush();
|
||
|
||
$labelQuery = $target->table('jst_order_labels')->where('company_id', $companyId);
|
||
if ($start !== null) {
|
||
$labelQuery->where('ordered_at', '>=', $start)->where('ordered_at', '<', $end);
|
||
}
|
||
$labels = $labelQuery->distinct()->count('label');
|
||
|
||
app(JstOrderLabelOptions::class)->forget($companyId);
|
||
app(JstOrderLabelOptions::class)->warm($companyId);
|
||
|
||
return [
|
||
'file' => basename($path),
|
||
'company_id' => $companyId,
|
||
'from' => $from,
|
||
'to' => $to,
|
||
'orders' => $orders,
|
||
'rows' => $rows,
|
||
'no_label_orders' => $noLabelOrders,
|
||
'skipped' => $skipped,
|
||
'labels' => (int) $labels,
|
||
'seconds' => round(microtime(true) - $startedAt, 1),
|
||
];
|
||
}
|
||
|
||
/** 下单时间归一成 Y-m-d H:i:s;解析不了就返回 null(该行跳过)。 */
|
||
private function normalizeOrderedAt(string $value): ?string
|
||
{
|
||
$value = trim($value);
|
||
if ($value === '') {
|
||
return null;
|
||
}
|
||
|
||
$timestamp = strtotime($value);
|
||
|
||
return $timestamp === false ? null : date('Y-m-d H:i:s', $timestamp);
|
||
}
|
||
|
||
/** @return list<string> */
|
||
private function splitLabels(string $labels): array
|
||
{
|
||
$out = [];
|
||
foreach (explode(',', $labels) as $label) {
|
||
$label = trim($label);
|
||
if ($label !== '') {
|
||
$out[] = mb_substr($label, 0, 191);
|
||
}
|
||
}
|
||
|
||
return array_values(array_unique($out));
|
||
}
|
||
|
||
private function normalizeDate(string $value, string $label): string
|
||
{
|
||
$value = trim($value);
|
||
if (preg_match('/^\d{4}-\d{2}-\d{2}$/', $value) !== 1) {
|
||
throw new InvalidArgumentException($label . '必须是 YYYY-MM-DD。');
|
||
}
|
||
|
||
[$year, $month, $day] = array_map('intval', explode('-', $value));
|
||
if (! checkdate($month, $day, $year)) {
|
||
throw new InvalidArgumentException($label . '不是有效日期。');
|
||
}
|
||
|
||
return $value;
|
||
}
|
||
}
|