renshen
656c655c91
新增流式 xlsx/csv 读取器与分片导入：上传建行缓存，按片处理（2 万行/25 秒预算），批量 upsert + 批量审计事件，前端进度条自动续跑；上限 20 万行，上传 50MB。
381 lines
13 KiB
PHP
381 lines
13 KiB
PHP
<?php
|
||
|
||
namespace Modules\EcommerceFinance\Services;
|
||
|
||
use Illuminate\Http\UploadedFile;
|
||
use Illuminate\Support\Facades\File;
|
||
use Illuminate\Validation\ValidationException;
|
||
use Throwable;
|
||
|
||
/**
|
||
* 大文件导入（20 万行级）：上传时把工作簿转成紧凑的行缓存（jsonl.gz），
|
||
* 之后由前端按片调用 process()，每片控制行数与耗时，避免超时与内存爆掉。
|
||
*/
|
||
class PayablesFinalImportRunner
|
||
{
|
||
/** 单次导入的行数上限。 */
|
||
public const MAX_ROWS = 200000;
|
||
|
||
/** 每片默认处理的行数。 */
|
||
private const SLICE_ROWS = 20000;
|
||
|
||
/** 每累计多少行落一次库。 */
|
||
private const FLUSH_ROWS = 2000;
|
||
|
||
/** 单片的时间预算（秒），到点就返回让前端继续下一片。 */
|
||
private const SLICE_SECONDS = 25.0;
|
||
|
||
/** 未完成任务的保留时间（小时），过期自动清理。 */
|
||
private const STALE_HOURS = 24;
|
||
|
||
public function __construct(
|
||
private readonly PayablesFieldNormalizer $normalizer,
|
||
private readonly SupplierPayableOverrideStore $store,
|
||
) {
|
||
}
|
||
|
||
public function storagePath(): string
|
||
{
|
||
return storage_path('app/payables-final-imports');
|
||
}
|
||
|
||
/**
|
||
* 上传文件 → 建立行缓存。
|
||
*
|
||
* @param array<string,array{label:string,type:string}> $fields
|
||
* @return array{token:string,total:int}
|
||
*/
|
||
public function prepare(UploadedFile $file, array $fields, string $storeDataset, int $companyId, ?int $userId): array
|
||
{
|
||
$this->cleanupStale();
|
||
|
||
$token = bin2hex(random_bytes(16));
|
||
$directory = $this->directory($token);
|
||
if (! is_dir($directory) && ! mkdir($directory, 0770, true)) {
|
||
throw ValidationException::withMessages(['import' => '无法创建导入任务目录。']);
|
||
}
|
||
|
||
$extension = strtolower((string) $file->getClientOriginalExtension());
|
||
$source = (string) $file->getRealPath();
|
||
|
||
$headerRow = 0;
|
||
|
||
try {
|
||
$reader = PayablesWorkbookStreamReader::forFile($source, $extension);
|
||
$header = $this->locateHeader($reader, $fields);
|
||
if ($header === null) {
|
||
throw ValidationException::withMessages([
|
||
'import' => '没有识别到表头，请先下载本页面的导入模板，按模板列填写后重试。',
|
||
]);
|
||
}
|
||
|
||
[$sheetIndex, $headerRow, $columns] = $header;
|
||
|
||
$rows = 0;
|
||
$handle = gzopen($directory . '/rows.jsonl.gz', 'wb1');
|
||
if ($handle === false) {
|
||
throw ValidationException::withMessages(['import' => '无法写入导入缓存。']);
|
||
}
|
||
|
||
try {
|
||
foreach ($reader->rows($sheetIndex, $headerRow) as $row) {
|
||
$values = [];
|
||
foreach ($columns as $columnIndex => $field) {
|
||
$values[$field] = $row[$columnIndex] ?? null;
|
||
}
|
||
|
||
$rows++;
|
||
if ($rows > self::MAX_ROWS) {
|
||
throw ValidationException::withMessages([
|
||
'import' => '单次最多导入 200,000 条记录，请拆分文件后重试。',
|
||
]);
|
||
}
|
||
|
||
gzwrite($handle, (string) json_encode($values, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE) . "\n");
|
||
}
|
||
} finally {
|
||
gzclose($handle);
|
||
}
|
||
|
||
if ($rows === 0) {
|
||
throw ValidationException::withMessages(['import' => '表格中没有可导入的数据行。']);
|
||
}
|
||
} catch (Throwable $exception) {
|
||
$this->delete($token);
|
||
if ($exception instanceof ValidationException) {
|
||
throw $exception;
|
||
}
|
||
|
||
throw ValidationException::withMessages([
|
||
'import' => '无法读取导入文件，请确认文件未损坏且格式为 xlsx、xls 或 csv。',
|
||
]);
|
||
}
|
||
|
||
$this->writeMeta($token, [
|
||
'dataset' => $storeDataset,
|
||
'company_id' => $companyId,
|
||
'user_id' => $userId,
|
||
'fields' => $fields,
|
||
'first_row' => $headerRow + 1,
|
||
'rows' => $rows,
|
||
'offset' => 0,
|
||
'created' => 0,
|
||
'updated' => 0,
|
||
'status' => 'ready',
|
||
'error' => null,
|
||
'started_at' => now()->toIso8601String(),
|
||
]);
|
||
|
||
return ['token' => $token, 'total' => $rows];
|
||
}
|
||
|
||
/**
|
||
* 处理一片；返回最新进度。
|
||
*
|
||
* @return array{offset:int,total:int,created:int,updated:int,done:bool}
|
||
*/
|
||
public function process(string $token, int $offset, ?int $maxRows = null, ?int $companyId = null): array
|
||
{
|
||
$meta = $this->readMeta($token);
|
||
if ($meta === null) {
|
||
throw ValidationException::withMessages(['import' => '导入任务不存在或已过期，请重新上传文件。']);
|
||
}
|
||
|
||
if ($companyId !== null && (int) $meta['company_id'] !== $companyId) {
|
||
throw ValidationException::withMessages(['import' => '导入任务不存在或已过期，请重新上传文件。']);
|
||
}
|
||
|
||
if (($meta['status'] ?? '') === 'done') {
|
||
return $this->progress($meta);
|
||
}
|
||
|
||
$total = (int) $meta['rows'];
|
||
$firstRow = (int) ($meta['first_row'] ?? 2);
|
||
$dataset = (string) $meta['dataset'];
|
||
$companyId = (int) $meta['company_id'];
|
||
$userId = isset($meta['user_id']) ? (int) $meta['user_id'] : null;
|
||
$fields = is_array($meta['fields'] ?? null) ? $meta['fields'] : [];
|
||
$sliceRows = max(1, $maxRows ?? self::SLICE_ROWS);
|
||
$deadline = microtime(true) + self::SLICE_SECONDS;
|
||
|
||
$created = (int) $meta['created'];
|
||
$updated = (int) $meta['updated'];
|
||
$line = 0;
|
||
$processed = 0;
|
||
$records = [];
|
||
$offset = min(max($offset, 0), $total);
|
||
|
||
$handle = gzopen($this->directory($token) . '/rows.jsonl.gz', 'rb');
|
||
if ($handle === false) {
|
||
throw ValidationException::withMessages(['import' => '导入任务数据已损坏，请重新上传文件。']);
|
||
}
|
||
|
||
try {
|
||
while ($line < $total && $processed < $sliceRows) {
|
||
$raw = gzgets($handle);
|
||
if ($raw === false) {
|
||
break;
|
||
}
|
||
$line++;
|
||
|
||
if ($line <= $offset) {
|
||
continue;
|
||
}
|
||
|
||
$values = json_decode((string) rtrim($raw, "\r\n"), true);
|
||
if (! is_array($values)) {
|
||
$values = [];
|
||
}
|
||
|
||
if (collect($values)->every(static fn (mixed $value): bool => trim((string) $value) === '')) {
|
||
continue;
|
||
}
|
||
|
||
try {
|
||
$normalized = $this->normalizer->normalize($fields, $values);
|
||
} catch (ValidationException $exception) {
|
||
$this->fail($meta, $line - 1, $created, $updated, $this->rowMessage($firstRow, $line, $exception));
|
||
throw ValidationException::withMessages([
|
||
'import' => $this->rowMessage($firstRow, $line, $exception),
|
||
]);
|
||
}
|
||
|
||
$key = $this->store->key($dataset, $normalized);
|
||
if (isset($records[$key])) {
|
||
$message = '第 ' . ($firstRow + $line - 1) . ' 行与前一行内容重复。';
|
||
$this->fail($meta, $line - 1, $created, $updated, $message);
|
||
throw ValidationException::withMessages(['import' => $message]);
|
||
}
|
||
|
||
$records[$key] = $normalized;
|
||
$processed++;
|
||
|
||
if (count($records) >= self::FLUSH_ROWS) {
|
||
$counts = $this->store->saveBatch($dataset, $records, $companyId, $userId, 'import');
|
||
$created += $counts['created'];
|
||
$updated += $counts['updated'];
|
||
$records = [];
|
||
|
||
if (microtime(true) >= $deadline || $processed >= $sliceRows) {
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
if ($records !== []) {
|
||
$counts = $this->store->saveBatch($dataset, $records, $companyId, $userId, 'import');
|
||
$created += $counts['created'];
|
||
$updated += $counts['updated'];
|
||
}
|
||
} finally {
|
||
gzclose($handle);
|
||
}
|
||
|
||
$done = $line >= $total;
|
||
$meta['offset'] = $line;
|
||
$meta['created'] = $created;
|
||
$meta['updated'] = $updated;
|
||
$meta['status'] = $done ? 'done' : 'running';
|
||
$meta['error'] = null;
|
||
$this->writeMeta($token, $meta);
|
||
|
||
if ($done) {
|
||
$this->delete($token);
|
||
}
|
||
|
||
return $this->progress($meta);
|
||
}
|
||
|
||
/** @return array{offset:int,total:int,created:int,updated:int,done:bool} */
|
||
private function progress(array $meta): array
|
||
{
|
||
return [
|
||
'offset' => (int) ($meta['offset'] ?? 0),
|
||
'total' => (int) ($meta['rows'] ?? 0),
|
||
'created' => (int) ($meta['created'] ?? 0),
|
||
'updated' => (int) ($meta['updated'] ?? 0),
|
||
'done' => ($meta['status'] ?? '') === 'done',
|
||
];
|
||
}
|
||
|
||
private function rowMessage(int $firstRow, int $line, ValidationException $exception): string
|
||
{
|
||
return '第 ' . ($firstRow + $line - 1) . ' 行：' . collect($exception->errors())->flatten()->first();
|
||
}
|
||
|
||
private function fail(array $meta, int $offset, int $created, int $updated, string $message): void
|
||
{
|
||
$meta['offset'] = $offset;
|
||
$meta['created'] = $created;
|
||
$meta['updated'] = $updated;
|
||
$meta['status'] = 'failed';
|
||
$meta['error'] = $message;
|
||
$this->writeMeta((string) $meta['token'], $meta);
|
||
}
|
||
|
||
/**
|
||
* 前若干行里找出表头（列序号 => 字段），与旧版打分规则一致。
|
||
*
|
||
* @param array<string,array{label:string,type:string}> $fields
|
||
* @return array{0:int,1:int,2:array<int,string>}|null
|
||
*/
|
||
private function locateHeader(PayablesWorkbookStreamReader $reader, array $fields): ?array
|
||
{
|
||
$lookup = ['记录标识' => '_action_key', 'recordkey' => '_action_key', '_action_key' => '_action_key'];
|
||
foreach ($fields as $field => $definition) {
|
||
$lookup[$this->header($field)] = $field;
|
||
$lookup[$this->header((string) $definition['label'])] = $field;
|
||
}
|
||
|
||
$best = null;
|
||
$score = 0;
|
||
foreach (array_keys($reader->sheetNames()) as $sheetIndex) {
|
||
foreach ($reader->rows($sheetIndex, null, 20) as $rowNumber => $row) {
|
||
$columns = [];
|
||
foreach ($row as $columnIndex => $value) {
|
||
$key = $this->header($value);
|
||
if ($key !== '' && isset($lookup[$key])) {
|
||
$columns[$columnIndex] = $lookup[$key];
|
||
}
|
||
}
|
||
|
||
if (count($columns) > $score) {
|
||
$score = count($columns);
|
||
$best = [$sheetIndex, $rowNumber, $columns];
|
||
}
|
||
}
|
||
}
|
||
|
||
return $best;
|
||
}
|
||
|
||
private function header(mixed $value): string
|
||
{
|
||
return mb_strtolower((string) preg_replace('/[\s\x{FEFF}_\-（）()]+/u', '', trim((string) $value)));
|
||
}
|
||
|
||
private function directory(string $token): string
|
||
{
|
||
return $this->storagePath() . '/' . $token;
|
||
}
|
||
|
||
private function metaPath(string $token): string
|
||
{
|
||
return $this->directory($token) . '/meta.json';
|
||
}
|
||
|
||
/** @return array<string,mixed>|null */
|
||
private function readMeta(string $token): ?array
|
||
{
|
||
if (preg_match('/^[a-f0-9]{32}$/', $token) !== 1) {
|
||
return null;
|
||
}
|
||
|
||
$path = $this->metaPath($token);
|
||
if (! is_file($path)) {
|
||
return null;
|
||
}
|
||
|
||
$meta = json_decode((string) file_get_contents($path), true);
|
||
if (! is_array($meta)) {
|
||
return null;
|
||
}
|
||
|
||
return $meta + ['token' => $token];
|
||
}
|
||
|
||
private function writeMeta(string $token, array $meta): void
|
||
{
|
||
$meta['token'] = $token;
|
||
$meta['updated_at'] = now()->toIso8601String();
|
||
file_put_contents($this->metaPath($token), (string) json_encode($meta, JSON_UNESCAPED_UNICODE));
|
||
}
|
||
|
||
public function delete(string $token): void
|
||
{
|
||
if (preg_match('/^[a-f0-9]{32}$/', $token) !== 1) {
|
||
return;
|
||
}
|
||
|
||
File::deleteDirectory($this->directory($token));
|
||
}
|
||
|
||
/** 清理过期任务（超过 24 小时未完成）。 */
|
||
private function cleanupStale(): void
|
||
{
|
||
$root = $this->storagePath();
|
||
if (! is_dir($root)) {
|
||
return;
|
||
}
|
||
|
||
$deadline = time() - self::STALE_HOURS * 3600;
|
||
foreach (File::directories($root) as $directory) {
|
||
$metaPath = $directory . '/meta.json';
|
||
$modified = is_file($metaPath) ? (int) filemtime($metaPath) : (int) filemtime($directory);
|
||
if ($modified < $deadline) {
|
||
$this->delete(basename($directory));
|
||
}
|
||
}
|
||
}
|
||
}
|