应付款终版导入：支持 20 万行（流式读取 + 分片导入 + 进度显示） (#361)
新增流式 xlsx/csv 读取器与分片导入：上传建行缓存，按片处理（2 万行/25 秒预算），批量 upsert + 批量审计事件，前端进度条自动续跑；上限 20 万行，上传 50MB。
This commit is contained in:
parent
7742163632
commit
656c655c91
@ -7,6 +7,7 @@ use Illuminate\Http\RedirectResponse;
|
||||
use Illuminate\Http\Request;
|
||||
use Illuminate\Routing\Controller;
|
||||
use Modules\EcommerceFinance\Services\PayablesFinalImporter;
|
||||
use Modules\EcommerceFinance\Services\PayablesFinalImportRunner;
|
||||
use Modules\EcommerceFinance\Services\PayablesFinalRegistry;
|
||||
use Modules\EcommerceFinance\Services\PayablesFieldNormalizer;
|
||||
use Modules\EcommerceFinance\Services\SupplierPayableImportTemplate;
|
||||
@ -70,6 +71,38 @@ class PayablesFinalActions extends Controller
|
||||
: back()->with('success', $message);
|
||||
}
|
||||
|
||||
/** 大文件导入第 1 步：上传文件并建立行缓存（前端随后按片拉取处理）。 */
|
||||
public function importUpload(Request $request, PayablesFinalImportRunner $runner): JsonResponse
|
||||
{
|
||||
$request->validate(['import' => ['required', 'file', 'mimes:xlsx,xls,csv', 'max:51200']]);
|
||||
|
||||
$prepared = $runner->prepare(
|
||||
$request->file('import'),
|
||||
$this->fields($request),
|
||||
$this->storeDataset($request),
|
||||
(int) (company_id() ?: 1),
|
||||
user_id() ? (int) user_id() : null,
|
||||
);
|
||||
|
||||
return response()->json($prepared + ['offset' => 0, 'created' => 0, 'updated' => 0, 'done' => false]);
|
||||
}
|
||||
|
||||
/** 大文件导入第 2 步起：每次处理一片，前端拿到进度后继续。 */
|
||||
public function importChunk(Request $request, PayablesFinalImportRunner $runner): JsonResponse
|
||||
{
|
||||
$data = $request->validate([
|
||||
'token' => ['required', 'string', 'regex:/^[a-f0-9]{32}$/'],
|
||||
'offset' => ['required', 'integer', 'min:0'],
|
||||
]);
|
||||
|
||||
return response()->json($runner->process(
|
||||
$data['token'],
|
||||
(int) $data['offset'],
|
||||
null,
|
||||
(int) (company_id() ?: 1),
|
||||
));
|
||||
}
|
||||
|
||||
public function update(Request $request, string $recordKey): JsonResponse|RedirectResponse
|
||||
{
|
||||
$values = $this->normalizer->normalize($this->fields($request), (array) $request->input('values', []));
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
memory_limit=512M
|
||||
max_execution_time=120
|
||||
post_max_size=64M
|
||||
upload_max_filesize=10M
|
||||
upload_max_filesize=64M
|
||||
# 线上必须把错误写进日志而不是显示给用户；否则致命错误（内存/超时）会既看不到、也无处查。
|
||||
display_errors=Off
|
||||
log_errors=On
|
||||
|
||||
380
modules/EcommerceFinance/Services/PayablesFinalImportRunner.php
Normal file
380
modules/EcommerceFinance/Services/PayablesFinalImportRunner.php
Normal file
@ -0,0 +1,380 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\EcommerceFinance\Services;
|
||||
|
||||
use Illuminate\Http\UploadedFile;
|
||||
use Illuminate\Support\Facades\File;
|
||||
use Illuminate\Validation\ValidationException;
|
||||
use Throwable;
|
||||
|
||||
/**
|
||||
* 大文件导入（20 万行级）：上传时把工作簿转成紧凑的行缓存（jsonl.gz），
|
||||
* 之后由前端按片调用 process()，每片控制行数与耗时，避免超时与内存爆掉。
|
||||
*/
|
||||
class PayablesFinalImportRunner
|
||||
{
|
||||
/** 单次导入的行数上限。 */
|
||||
public const MAX_ROWS = 200000;
|
||||
|
||||
/** 每片默认处理的行数。 */
|
||||
private const SLICE_ROWS = 20000;
|
||||
|
||||
/** 每累计多少行落一次库。 */
|
||||
private const FLUSH_ROWS = 2000;
|
||||
|
||||
/** 单片的时间预算（秒），到点就返回让前端继续下一片。 */
|
||||
private const SLICE_SECONDS = 25.0;
|
||||
|
||||
/** 未完成任务的保留时间（小时），过期自动清理。 */
|
||||
private const STALE_HOURS = 24;
|
||||
|
||||
public function __construct(
|
||||
private readonly PayablesFieldNormalizer $normalizer,
|
||||
private readonly SupplierPayableOverrideStore $store,
|
||||
) {
|
||||
}
|
||||
|
||||
public function storagePath(): string
|
||||
{
|
||||
return storage_path('app/payables-final-imports');
|
||||
}
|
||||
|
||||
/**
|
||||
* 上传文件 → 建立行缓存。
|
||||
*
|
||||
* @param array<string,array{label:string,type:string}> $fields
|
||||
* @return array{token:string,total:int}
|
||||
*/
|
||||
public function prepare(UploadedFile $file, array $fields, string $storeDataset, int $companyId, ?int $userId): array
|
||||
{
|
||||
$this->cleanupStale();
|
||||
|
||||
$token = bin2hex(random_bytes(16));
|
||||
$directory = $this->directory($token);
|
||||
if (! is_dir($directory) && ! mkdir($directory, 0770, true)) {
|
||||
throw ValidationException::withMessages(['import' => '无法创建导入任务目录。']);
|
||||
}
|
||||
|
||||
$extension = strtolower((string) $file->getClientOriginalExtension());
|
||||
$source = (string) $file->getRealPath();
|
||||
|
||||
$headerRow = 0;
|
||||
|
||||
try {
|
||||
$reader = PayablesWorkbookStreamReader::forFile($source, $extension);
|
||||
$header = $this->locateHeader($reader, $fields);
|
||||
if ($header === null) {
|
||||
throw ValidationException::withMessages([
|
||||
'import' => '没有识别到表头，请先下载本页面的导入模板，按模板列填写后重试。',
|
||||
]);
|
||||
}
|
||||
|
||||
[$sheetIndex, $headerRow, $columns] = $header;
|
||||
|
||||
$rows = 0;
|
||||
$handle = gzopen($directory . '/rows.jsonl.gz', 'wb1');
|
||||
if ($handle === false) {
|
||||
throw ValidationException::withMessages(['import' => '无法写入导入缓存。']);
|
||||
}
|
||||
|
||||
try {
|
||||
foreach ($reader->rows($sheetIndex, $headerRow) as $row) {
|
||||
$values = [];
|
||||
foreach ($columns as $columnIndex => $field) {
|
||||
$values[$field] = $row[$columnIndex] ?? null;
|
||||
}
|
||||
|
||||
$rows++;
|
||||
if ($rows > self::MAX_ROWS) {
|
||||
throw ValidationException::withMessages([
|
||||
'import' => '单次最多导入 200,000 条记录，请拆分文件后重试。',
|
||||
]);
|
||||
}
|
||||
|
||||
gzwrite($handle, (string) json_encode($values, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE) . "\n");
|
||||
}
|
||||
} finally {
|
||||
gzclose($handle);
|
||||
}
|
||||
|
||||
if ($rows === 0) {
|
||||
throw ValidationException::withMessages(['import' => '表格中没有可导入的数据行。']);
|
||||
}
|
||||
} catch (Throwable $exception) {
|
||||
$this->delete($token);
|
||||
if ($exception instanceof ValidationException) {
|
||||
throw $exception;
|
||||
}
|
||||
|
||||
throw ValidationException::withMessages([
|
||||
'import' => '无法读取导入文件，请确认文件未损坏且格式为 xlsx、xls 或 csv。',
|
||||
]);
|
||||
}
|
||||
|
||||
$this->writeMeta($token, [
|
||||
'dataset' => $storeDataset,
|
||||
'company_id' => $companyId,
|
||||
'user_id' => $userId,
|
||||
'fields' => $fields,
|
||||
'first_row' => $headerRow + 1,
|
||||
'rows' => $rows,
|
||||
'offset' => 0,
|
||||
'created' => 0,
|
||||
'updated' => 0,
|
||||
'status' => 'ready',
|
||||
'error' => null,
|
||||
'started_at' => now()->toIso8601String(),
|
||||
]);
|
||||
|
||||
return ['token' => $token, 'total' => $rows];
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理一片；返回最新进度。
|
||||
*
|
||||
* @return array{offset:int,total:int,created:int,updated:int,done:bool}
|
||||
*/
|
||||
public function process(string $token, int $offset, ?int $maxRows = null, ?int $companyId = null): array
|
||||
{
|
||||
$meta = $this->readMeta($token);
|
||||
if ($meta === null) {
|
||||
throw ValidationException::withMessages(['import' => '导入任务不存在或已过期，请重新上传文件。']);
|
||||
}
|
||||
|
||||
if ($companyId !== null && (int) $meta['company_id'] !== $companyId) {
|
||||
throw ValidationException::withMessages(['import' => '导入任务不存在或已过期，请重新上传文件。']);
|
||||
}
|
||||
|
||||
if (($meta['status'] ?? '') === 'done') {
|
||||
return $this->progress($meta);
|
||||
}
|
||||
|
||||
$total = (int) $meta['rows'];
|
||||
$firstRow = (int) ($meta['first_row'] ?? 2);
|
||||
$dataset = (string) $meta['dataset'];
|
||||
$companyId = (int) $meta['company_id'];
|
||||
$userId = isset($meta['user_id']) ? (int) $meta['user_id'] : null;
|
||||
$fields = is_array($meta['fields'] ?? null) ? $meta['fields'] : [];
|
||||
$sliceRows = max(1, $maxRows ?? self::SLICE_ROWS);
|
||||
$deadline = microtime(true) + self::SLICE_SECONDS;
|
||||
|
||||
$created = (int) $meta['created'];
|
||||
$updated = (int) $meta['updated'];
|
||||
$line = 0;
|
||||
$processed = 0;
|
||||
$records = [];
|
||||
$offset = min(max($offset, 0), $total);
|
||||
|
||||
$handle = gzopen($this->directory($token) . '/rows.jsonl.gz', 'rb');
|
||||
if ($handle === false) {
|
||||
throw ValidationException::withMessages(['import' => '导入任务数据已损坏，请重新上传文件。']);
|
||||
}
|
||||
|
||||
try {
|
||||
while ($line < $total && $processed < $sliceRows) {
|
||||
$raw = gzgets($handle);
|
||||
if ($raw === false) {
|
||||
break;
|
||||
}
|
||||
$line++;
|
||||
|
||||
if ($line <= $offset) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$values = json_decode((string) rtrim($raw, "\r\n"), true);
|
||||
if (! is_array($values)) {
|
||||
$values = [];
|
||||
}
|
||||
|
||||
if (collect($values)->every(static fn (mixed $value): bool => trim((string) $value) === '')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
$normalized = $this->normalizer->normalize($fields, $values);
|
||||
} catch (ValidationException $exception) {
|
||||
$this->fail($meta, $line - 1, $created, $updated, $this->rowMessage($firstRow, $line, $exception));
|
||||
throw ValidationException::withMessages([
|
||||
'import' => $this->rowMessage($firstRow, $line, $exception),
|
||||
]);
|
||||
}
|
||||
|
||||
$key = $this->store->key($dataset, $normalized);
|
||||
if (isset($records[$key])) {
|
||||
$message = '第 ' . ($firstRow + $line - 1) . ' 行与前一行内容重复。';
|
||||
$this->fail($meta, $line - 1, $created, $updated, $message);
|
||||
throw ValidationException::withMessages(['import' => $message]);
|
||||
}
|
||||
|
||||
$records[$key] = $normalized;
|
||||
$processed++;
|
||||
|
||||
if (count($records) >= self::FLUSH_ROWS) {
|
||||
$counts = $this->store->saveBatch($dataset, $records, $companyId, $userId, 'import');
|
||||
$created += $counts['created'];
|
||||
$updated += $counts['updated'];
|
||||
$records = [];
|
||||
|
||||
if (microtime(true) >= $deadline || $processed >= $sliceRows) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($records !== []) {
|
||||
$counts = $this->store->saveBatch($dataset, $records, $companyId, $userId, 'import');
|
||||
$created += $counts['created'];
|
||||
$updated += $counts['updated'];
|
||||
}
|
||||
} finally {
|
||||
gzclose($handle);
|
||||
}
|
||||
|
||||
$done = $line >= $total;
|
||||
$meta['offset'] = $line;
|
||||
$meta['created'] = $created;
|
||||
$meta['updated'] = $updated;
|
||||
$meta['status'] = $done ? 'done' : 'running';
|
||||
$meta['error'] = null;
|
||||
$this->writeMeta($token, $meta);
|
||||
|
||||
if ($done) {
|
||||
$this->delete($token);
|
||||
}
|
||||
|
||||
return $this->progress($meta);
|
||||
}
|
||||
|
||||
/** @return array{offset:int,total:int,created:int,updated:int,done:bool} */
|
||||
private function progress(array $meta): array
|
||||
{
|
||||
return [
|
||||
'offset' => (int) ($meta['offset'] ?? 0),
|
||||
'total' => (int) ($meta['rows'] ?? 0),
|
||||
'created' => (int) ($meta['created'] ?? 0),
|
||||
'updated' => (int) ($meta['updated'] ?? 0),
|
||||
'done' => ($meta['status'] ?? '') === 'done',
|
||||
];
|
||||
}
|
||||
|
||||
private function rowMessage(int $firstRow, int $line, ValidationException $exception): string
|
||||
{
|
||||
return '第 ' . ($firstRow + $line - 1) . ' 行：' . collect($exception->errors())->flatten()->first();
|
||||
}
|
||||
|
||||
private function fail(array $meta, int $offset, int $created, int $updated, string $message): void
|
||||
{
|
||||
$meta['offset'] = $offset;
|
||||
$meta['created'] = $created;
|
||||
$meta['updated'] = $updated;
|
||||
$meta['status'] = 'failed';
|
||||
$meta['error'] = $message;
|
||||
$this->writeMeta((string) $meta['token'], $meta);
|
||||
}
|
||||
|
||||
/**
|
||||
* 前若干行里找出表头（列序号 => 字段），与旧版打分规则一致。
|
||||
*
|
||||
* @param array<string,array{label:string,type:string}> $fields
|
||||
* @return array{0:int,1:int,2:array<int,string>}|null
|
||||
*/
|
||||
private function locateHeader(PayablesWorkbookStreamReader $reader, array $fields): ?array
|
||||
{
|
||||
$lookup = ['记录标识' => '_action_key', 'recordkey' => '_action_key', '_action_key' => '_action_key'];
|
||||
foreach ($fields as $field => $definition) {
|
||||
$lookup[$this->header($field)] = $field;
|
||||
$lookup[$this->header((string) $definition['label'])] = $field;
|
||||
}
|
||||
|
||||
$best = null;
|
||||
$score = 0;
|
||||
foreach (array_keys($reader->sheetNames()) as $sheetIndex) {
|
||||
foreach ($reader->rows($sheetIndex, null, 20) as $rowNumber => $row) {
|
||||
$columns = [];
|
||||
foreach ($row as $columnIndex => $value) {
|
||||
$key = $this->header($value);
|
||||
if ($key !== '' && isset($lookup[$key])) {
|
||||
$columns[$columnIndex] = $lookup[$key];
|
||||
}
|
||||
}
|
||||
|
||||
if (count($columns) > $score) {
|
||||
$score = count($columns);
|
||||
$best = [$sheetIndex, $rowNumber, $columns];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $best;
|
||||
}
|
||||
|
||||
private function header(mixed $value): string
|
||||
{
|
||||
return mb_strtolower((string) preg_replace('/[\s\x{FEFF}_\-（）()]+/u', '', trim((string) $value)));
|
||||
}
|
||||
|
||||
private function directory(string $token): string
|
||||
{
|
||||
return $this->storagePath() . '/' . $token;
|
||||
}
|
||||
|
||||
private function metaPath(string $token): string
|
||||
{
|
||||
return $this->directory($token) . '/meta.json';
|
||||
}
|
||||
|
||||
/** @return array<string,mixed>|null */
|
||||
private function readMeta(string $token): ?array
|
||||
{
|
||||
if (preg_match('/^[a-f0-9]{32}$/', $token) !== 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$path = $this->metaPath($token);
|
||||
if (! is_file($path)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$meta = json_decode((string) file_get_contents($path), true);
|
||||
if (! is_array($meta)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return $meta + ['token' => $token];
|
||||
}
|
||||
|
||||
private function writeMeta(string $token, array $meta): void
|
||||
{
|
||||
$meta['token'] = $token;
|
||||
$meta['updated_at'] = now()->toIso8601String();
|
||||
file_put_contents($this->metaPath($token), (string) json_encode($meta, JSON_UNESCAPED_UNICODE));
|
||||
}
|
||||
|
||||
public function delete(string $token): void
|
||||
{
|
||||
if (preg_match('/^[a-f0-9]{32}$/', $token) !== 1) {
|
||||
return;
|
||||
}
|
||||
|
||||
File::deleteDirectory($this->directory($token));
|
||||
}
|
||||
|
||||
/** 清理过期任务（超过 24 小时未完成）。 */
|
||||
private function cleanupStale(): void
|
||||
{
|
||||
$root = $this->storagePath();
|
||||
if (! is_dir($root)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$deadline = time() - self::STALE_HOURS * 3600;
|
||||
foreach (File::directories($root) as $directory) {
|
||||
$metaPath = $directory . '/meta.json';
|
||||
$modified = is_file($metaPath) ? (int) filemtime($metaPath) : (int) filemtime($directory);
|
||||
if ($modified < $deadline) {
|
||||
$this->delete(basename($directory));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,564 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\EcommerceFinance\Services;
|
||||
|
||||
use Generator;
|
||||
use PhpOffice\PhpSpreadsheet\IOFactory;
|
||||
use PhpOffice\PhpSpreadsheet\Shared\Date as ExcelDate;
|
||||
use RuntimeException;
|
||||
use SplFixedArray;
|
||||
use XMLReader;
|
||||
use ZipArchive;
|
||||
|
||||
/**
|
||||
* 逐行读取导入工作簿，内存有界（20 万行级文件也能跑）：
|
||||
* - xlsx：ZipArchive + XMLReader 流式解析，共享字符串用「拼接串 + 偏移索引」紧凑存放；
|
||||
* - csv：逐行 fgetcsv（自动去 BOM、非 UTF-8 按 GBK 转换）；
|
||||
* - xls：退回 PhpSpreadsheet（老格式少见）。
|
||||
* 日期格式的单元格按 Y-m-d（含时分秒时为 Y-m-d H:i:s）输出，其余保持原值。
|
||||
*/
|
||||
class PayablesWorkbookStreamReader
|
||||
{
|
||||
/** @var array<int,string>|null 工作表内部路径（xlsx） */
|
||||
private ?array $sheetPaths = null;
|
||||
|
||||
/** @var list<int>|null 日期样式的 style 索引 */
|
||||
private ?array $dateStyles = null;
|
||||
|
||||
/** @var string|null 共享字符串拼接串 */
|
||||
private ?string $sharedStrings = null;
|
||||
|
||||
/** @var SplFixedArray<int>|null 共享字符串偏移（第 i 个字符串的起始位置） */
|
||||
private ?SplFixedArray $sharedStringOffsets = null;
|
||||
|
||||
public function __construct(
|
||||
private readonly string $path,
|
||||
private readonly string $extension,
|
||||
) {
|
||||
}
|
||||
|
||||
/** 新建读取器（按扩展名）。 */
|
||||
public static function forFile(string $path, string $extension): self
|
||||
{
|
||||
return new self($path, strtolower($extension));
|
||||
}
|
||||
|
||||
/** @return list<string> 工作表名称（xls 走 PhpSpreadsheet，仅取名称） */
|
||||
public function sheetNames(): array
|
||||
{
|
||||
if ($this->extension === 'xlsx') {
|
||||
return array_map(
|
||||
static fn (array $sheet): string => $sheet['name'],
|
||||
$this->xlsxSheets(),
|
||||
);
|
||||
}
|
||||
|
||||
if ($this->extension === 'csv') {
|
||||
return ['Sheet1'];
|
||||
}
|
||||
|
||||
$spreadsheet = IOFactory::createReader('Xls')->load($this->path);
|
||||
try {
|
||||
return $spreadsheet->getSheetNames();
|
||||
} finally {
|
||||
$spreadsheet->disconnectWorksheets();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 逐行读取指定工作表。
|
||||
*
|
||||
* @param int|null $skipUntilRow 跳过该行号之前的行（含），用于分片续读
|
||||
* @param int|null $stopAfterRow 读到该行号后停止
|
||||
* @return Generator<int,array<int,mixed>> 行号（1 基）=> 列序号（0 基）=> 值
|
||||
*/
|
||||
public function rows(int $sheetIndex = 0, ?int $skipUntilRow = null, ?int $stopAfterRow = null): Generator
|
||||
{
|
||||
yield from match ($this->extension) {
|
||||
'xlsx' => $this->xlsxRows($sheetIndex, $skipUntilRow, $stopAfterRow),
|
||||
'csv' => $this->csvRows($skipUntilRow, $stopAfterRow),
|
||||
default => $this->xlsRows($sheetIndex, $skipUntilRow, $stopAfterRow),
|
||||
};
|
||||
}
|
||||
|
||||
/** @return Generator<int,array<int,mixed>> */
|
||||
private function xlsxRows(int $sheetIndex, ?int $skipUntilRow, ?int $stopAfterRow): Generator
|
||||
{
|
||||
$path = $this->sheetPaths()[$sheetIndex] ?? null;
|
||||
if ($path === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
$zip = new ZipArchive();
|
||||
if ($zip->open($this->path) !== true) {
|
||||
throw new RuntimeException('无法打开工作簿。');
|
||||
}
|
||||
|
||||
try {
|
||||
$stream = $zip->getStream($path);
|
||||
if ($stream === false) {
|
||||
return;
|
||||
}
|
||||
|
||||
$reader = new XMLReader();
|
||||
$reader->XML(stream_get_contents($stream), null, LIBXML_NONET);
|
||||
fclose($stream);
|
||||
|
||||
$dictionary = $this->xlsxSharedStrings();
|
||||
$rowNumber = 0;
|
||||
|
||||
while ($reader->read()) {
|
||||
if ($reader->nodeType !== XMLReader::ELEMENT || $reader->localName !== 'row') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$rowNumber = (int) ($reader->getAttribute('r') ?: $rowNumber + 1);
|
||||
if ($stopAfterRow !== null && $rowNumber > $stopAfterRow) {
|
||||
return;
|
||||
}
|
||||
|
||||
if ($skipUntilRow !== null && $rowNumber <= $skipUntilRow) {
|
||||
$reader->next(); // 跳过整行
|
||||
continue;
|
||||
}
|
||||
|
||||
yield $rowNumber => $this->readRow($reader, $dictionary);
|
||||
}
|
||||
} finally {
|
||||
$zip->close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 读一行里的单元格（当前节点位于 <row>）。
|
||||
*
|
||||
* @param array{0:string,1:SplFixedArray<int>} $dictionary
|
||||
* @return array<int,mixed>
|
||||
*/
|
||||
private function readRow(XMLReader $reader, array $dictionary): array
|
||||
{
|
||||
$cells = [];
|
||||
$depth = $reader->depth;
|
||||
$empty = $reader->isEmptyElement;
|
||||
if ($empty) {
|
||||
return $cells;
|
||||
}
|
||||
|
||||
while ($reader->read()) {
|
||||
if ($reader->nodeType === XMLReader::END_ELEMENT && $reader->depth === $depth) {
|
||||
break;
|
||||
}
|
||||
if ($reader->nodeType !== XMLReader::ELEMENT || $reader->localName !== 'c' || $reader->isEmptyElement) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$reference = (string) $reader->getAttribute('r');
|
||||
$type = (string) $reader->getAttribute('t');
|
||||
$styleIndex = (string) $reader->getAttribute('s');
|
||||
$columnIndex = $reference !== '' ? $this->columnIndex($reference) : null;
|
||||
$value = $this->readCellValue($reader, $type, $dictionary);
|
||||
$columnIndex ??= count($cells);
|
||||
|
||||
if ($value === null || $value === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$cells[$columnIndex] = $this->formatValue($value, $type, $styleIndex);
|
||||
}
|
||||
|
||||
return $cells;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{0:string,1:SplFixedArray<int>} $dictionary
|
||||
*/
|
||||
private function readCellValue(XMLReader $reader, string $type, array $dictionary): mixed
|
||||
{
|
||||
$value = null;
|
||||
$depth = $reader->depth;
|
||||
|
||||
while ($reader->read()) {
|
||||
if ($reader->nodeType === XMLReader::END_ELEMENT && $reader->depth === $depth) {
|
||||
break;
|
||||
}
|
||||
if ($reader->nodeType !== XMLReader::ELEMENT) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($reader->localName === 'v') {
|
||||
$value = $reader->isEmptyElement ? '' : $reader->readString();
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($reader->localName === 'is') {
|
||||
$value = $this->readInlineString($reader);
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($reader->localName === 't' && $type === 'inlineStr') {
|
||||
$value = $reader->isEmptyElement ? '' : $reader->readString();
|
||||
}
|
||||
}
|
||||
|
||||
if ($type === 's') {
|
||||
$index = (int) $value;
|
||||
[$blob, $offsets] = $dictionary;
|
||||
if ($index < 0 || $index + 1 >= $offsets->getSize()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return substr($blob, $offsets[$index], $offsets[$index + 1] - $offsets[$index]);
|
||||
}
|
||||
|
||||
if ($type === 'str' || $type === 'inlineStr') {
|
||||
return $value;
|
||||
}
|
||||
|
||||
if ($type === 'b') {
|
||||
return $value === '1' ? '1' : '0';
|
||||
}
|
||||
|
||||
return $value;
|
||||
}
|
||||
|
||||
/** 富文本 <is> 里可能有多个 <t>。 */
|
||||
private function readInlineString(XMLReader $reader): string
|
||||
{
|
||||
$depth = $reader->depth;
|
||||
$text = '';
|
||||
|
||||
while ($reader->read()) {
|
||||
if ($reader->nodeType === XMLReader::END_ELEMENT && $reader->depth === $depth) {
|
||||
break;
|
||||
}
|
||||
if ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 't' && ! $reader->isEmptyElement) {
|
||||
$text .= $reader->readString();
|
||||
}
|
||||
}
|
||||
|
||||
return $text;
|
||||
}
|
||||
|
||||
/** 日期样式的数字单元格转成日期字符串。 */
|
||||
private function formatValue(mixed $value, string $type, string $styleIndex): mixed
|
||||
{
|
||||
if (! is_string($value) || $value === '' || $styleIndex === '' || ! is_numeric($value)) {
|
||||
return $value;
|
||||
}
|
||||
|
||||
if (! in_array((int) $styleIndex, $this->dateStyles(), true)) {
|
||||
return $value;
|
||||
}
|
||||
|
||||
$serial = (float) $value;
|
||||
$date = ExcelDate::excelToDateTimeObject($serial);
|
||||
|
||||
return fmod($serial, 1.0) === 0.0 ? $date->format('Y-m-d') : $date->format('Y-m-d H:i:s');
|
||||
}
|
||||
|
||||
/** @return Generator<int,array<int,mixed>> */
|
||||
private function csvRows(?int $skipUntilRow, ?int $stopAfterRow): Generator
|
||||
{
|
||||
$handle = fopen($this->path, 'rb');
|
||||
if ($handle === false) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
$delimiter = $this->detectDelimiter((string) fgets($handle));
|
||||
rewind($handle);
|
||||
|
||||
$line = 0;
|
||||
while (($row = fgetcsv($handle, 0, $delimiter)) !== false) {
|
||||
$line++;
|
||||
if ($stopAfterRow !== null && $line > $stopAfterRow) {
|
||||
return;
|
||||
}
|
||||
if ($skipUntilRow !== null && $line <= $skipUntilRow) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$cells = [];
|
||||
foreach ($row as $index => $value) {
|
||||
$value = $this->normalizeEncoding((string) $value);
|
||||
if ($index === 0) {
|
||||
$value = preg_replace('/^\x{FEFF}/u', '', $value) ?? $value;
|
||||
}
|
||||
if ($value !== '') {
|
||||
$cells[$index] = $value;
|
||||
}
|
||||
}
|
||||
|
||||
yield $line => $cells;
|
||||
}
|
||||
} finally {
|
||||
fclose($handle);
|
||||
}
|
||||
}
|
||||
|
||||
/** @return Generator<int,array<int,mixed>> */
|
||||
private function xlsRows(int $sheetIndex, ?int $skipUntilRow, ?int $stopAfterRow): Generator
|
||||
{
|
||||
$spreadsheet = IOFactory::createReader('Xls')->load($this->path);
|
||||
try {
|
||||
$sheet = $spreadsheet->getSheet($sheetIndex);
|
||||
foreach ($sheet->getRowIterator() as $row) {
|
||||
$line = $row->getRowIndex();
|
||||
if ($stopAfterRow !== null && $line > $stopAfterRow) {
|
||||
return;
|
||||
}
|
||||
if ($skipUntilRow !== null && $line <= $skipUntilRow) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$cells = [];
|
||||
foreach ($row->getCellIterator() as $cell) {
|
||||
$value = $cell->getValue();
|
||||
if ($value !== null && $value !== '') {
|
||||
$cells[$this->columnIndex($cell->getColumn())] = $value;
|
||||
}
|
||||
}
|
||||
|
||||
yield $line => $cells;
|
||||
}
|
||||
} finally {
|
||||
$spreadsheet->disconnectWorksheets();
|
||||
}
|
||||
}
|
||||
|
||||
/** @return list<int> 日期样式的 style 索引 */
|
||||
private function dateStyles(): array
|
||||
{
|
||||
if ($this->dateStyles !== null) {
|
||||
return $this->dateStyles;
|
||||
}
|
||||
|
||||
if ($this->extension !== 'xlsx') {
|
||||
return $this->dateStyles = [];
|
||||
}
|
||||
|
||||
$zip = new ZipArchive();
|
||||
if ($zip->open($this->path) !== true) {
|
||||
return $this->dateStyles = [];
|
||||
}
|
||||
|
||||
$styles = [];
|
||||
try {
|
||||
$stylesXml = $zip->getFromName('xl/styles.xml');
|
||||
if ($stylesXml === false) {
|
||||
return $this->dateStyles = [];
|
||||
}
|
||||
|
||||
$reader = new XMLReader();
|
||||
$reader->XML($stylesXml, null, LIBXML_NONET);
|
||||
|
||||
$customDateFormats = [];
|
||||
$cellXfs = [];
|
||||
$inCellXfs = false;
|
||||
|
||||
while ($reader->read()) {
|
||||
if ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 'numFmt') {
|
||||
$id = (int) $reader->getAttribute('numFmtId');
|
||||
$code = (string) $reader->getAttribute('formatCode');
|
||||
if ($id >= 164 && $this->looksLikeDateFormat($code)) {
|
||||
$customDateFormats[$id] = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 'cellXfs') {
|
||||
$inCellXfs = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($reader->nodeType === XMLReader::END_ELEMENT && $reader->localName === 'cellXfs') {
|
||||
$inCellXfs = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($inCellXfs && $reader->nodeType === XMLReader::ELEMENT && $reader->localName === 'xf') {
|
||||
$cellXfs[] = (int) $reader->getAttribute('numFmtId');
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($cellXfs as $index => $numFmtId) {
|
||||
if (($numFmtId >= 14 && $numFmtId <= 22) || ($numFmtId >= 45 && $numFmtId <= 47) || isset($customDateFormats[$numFmtId])) {
|
||||
$styles[] = $index;
|
||||
}
|
||||
}
|
||||
|
||||
$reader->close();
|
||||
} finally {
|
||||
$zip->close();
|
||||
}
|
||||
|
||||
return $this->dateStyles = $styles;
|
||||
}
|
||||
|
||||
private function looksLikeDateFormat(string $code): bool
|
||||
{
|
||||
$code = preg_replace('/"[^"]*"/', '', $code) ?? $code;
|
||||
$code = preg_replace('/\[[^\]]*\]/', '', $code) ?? $code;
|
||||
|
||||
return (bool) preg_match('/[ymdhs]/i', str_replace(['AM/PM', 'am/pm'], '', $code));
|
||||
}
|
||||
|
||||
/** @return list<array{name:string,path:string}> */
|
||||
private function xlsxSheets(): array
|
||||
{
|
||||
if ($this->sheetPaths !== null) {
|
||||
return $this->sheetPaths;
|
||||
}
|
||||
|
||||
$zip = new ZipArchive();
|
||||
if ($zip->open($this->path) !== true) {
|
||||
throw new RuntimeException('无法打开工作簿。');
|
||||
}
|
||||
|
||||
$sheets = [];
|
||||
try {
|
||||
$workbook = $zip->getFromName('xl/workbook.xml');
|
||||
$relationships = $zip->getFromName('xl/_rels/workbook.xml.rels');
|
||||
if ($workbook === false || $relationships === false) {
|
||||
throw new RuntimeException('工作簿结构不完整。');
|
||||
}
|
||||
|
||||
$targets = [];
|
||||
$rels = new XMLReader();
|
||||
$rels->XML($relationships, null, LIBXML_NONET);
|
||||
while ($rels->read()) {
|
||||
if ($rels->nodeType === XMLReader::ELEMENT && $rels->localName === 'Relationship') {
|
||||
$targets[(string) $rels->getAttribute('Id')] = ltrim((string) $rels->getAttribute('Target'), '/');
|
||||
}
|
||||
}
|
||||
$rels->close();
|
||||
|
||||
$book = new XMLReader();
|
||||
$book->XML($workbook, null, LIBXML_NONET);
|
||||
while ($book->read()) {
|
||||
if ($book->nodeType !== XMLReader::ELEMENT || $book->localName !== 'sheet') {
|
||||
continue;
|
||||
}
|
||||
$relId = (string) ($book->getAttributeNs('id', 'http://schemas.openxmlformats.org/officeDocument/2006/relationships') ?? '');
|
||||
$target = $targets[$relId] ?? ('worksheets/sheet' . (count($sheets) + 1) . '.xml');
|
||||
$sheets[] = [
|
||||
'name' => (string) $book->getAttribute('name'),
|
||||
'path' => str_starts_with($target, 'xl/') ? $target : 'xl/' . $target,
|
||||
];
|
||||
}
|
||||
$book->close();
|
||||
} finally {
|
||||
$zip->close();
|
||||
}
|
||||
|
||||
return $this->sheetPaths = $sheets;
|
||||
}
|
||||
|
||||
/** @return list<string> */
|
||||
private function sheetPaths(): array
|
||||
{
|
||||
return array_column($this->xlsxSheets(), 'path');
|
||||
}
|
||||
|
||||
/** @return array{0:string,1:SplFixedArray<int>} */
|
||||
private function xlsxSharedStrings(): array
|
||||
{
|
||||
if ($this->sharedStrings !== null && $this->sharedStringOffsets !== null) {
|
||||
return [$this->sharedStrings, $this->sharedStringOffsets];
|
||||
}
|
||||
|
||||
$zip = new ZipArchive();
|
||||
if ($zip->open($this->path) !== true) {
|
||||
throw new RuntimeException('无法打开工作簿。');
|
||||
}
|
||||
|
||||
$blob = '';
|
||||
try {
|
||||
$xml = $zip->getFromName('xl/sharedStrings.xml');
|
||||
if ($xml === false) {
|
||||
return [$this->sharedStrings = '', $this->sharedStringOffsets = new SplFixedArray(1)];
|
||||
}
|
||||
|
||||
$reader = new XMLReader();
|
||||
$reader->XML($xml, null, LIBXML_NONET);
|
||||
|
||||
$count = 0;
|
||||
$offsets = new SplFixedArray(1024);
|
||||
$offsets[0] = 0;
|
||||
$depth = 0;
|
||||
$collect = false;
|
||||
$text = '';
|
||||
|
||||
while ($reader->read()) {
|
||||
if ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 'si') {
|
||||
$collect = true;
|
||||
$text = '';
|
||||
$depth = $reader->depth;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (! $collect) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 't' && ! $reader->isEmptyElement) {
|
||||
$text .= $reader->readString();
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($reader->nodeType === XMLReader::END_ELEMENT && $reader->localName === 'si' && $reader->depth === $depth) {
|
||||
$blob .= $text;
|
||||
$count++;
|
||||
if ($count + 1 > $offsets->getSize()) {
|
||||
$offsets->setSize($offsets->getSize() * 2);
|
||||
}
|
||||
$offsets[$count] = strlen($blob);
|
||||
$collect = false;
|
||||
}
|
||||
}
|
||||
|
||||
$reader->close();
|
||||
$offsets->setSize($count + 1);
|
||||
} finally {
|
||||
$zip->close();
|
||||
}
|
||||
|
||||
return [$this->sharedStrings = $blob, $this->sharedStringOffsets = $offsets];
|
||||
}
|
||||
|
||||
private function detectDelimiter(string $line): string
|
||||
{
|
||||
$counts = [
|
||||
',' => substr_count($line, ','),
|
||||
';' => substr_count($line, ';'),
|
||||
"\t" => substr_count($line, "\t"),
|
||||
];
|
||||
|
||||
arsort($counts);
|
||||
$delimiter = (string) array_key_first($counts);
|
||||
|
||||
return $counts[$delimiter] > 0 ? $delimiter : ',';
|
||||
}
|
||||
|
||||
private function normalizeEncoding(string $value): string
|
||||
{
|
||||
if ($value === '' || mb_check_encoding($value, 'UTF-8')) {
|
||||
return $value;
|
||||
}
|
||||
|
||||
return mb_convert_encoding($value, 'UTF-8', 'GBK');
|
||||
}
|
||||
|
||||
private function columnIndex(string $reference): int
|
||||
{
|
||||
preg_match('/^([A-Z]+)/i', $reference, $matches);
|
||||
|
||||
$letters = strtoupper($matches[1] ?? 'A');
|
||||
$index = 0;
|
||||
foreach (str_split($letters) as $letter) {
|
||||
$index = $index * 26 + (ord($letter) - 64);
|
||||
}
|
||||
|
||||
return $index - 1;
|
||||
}
|
||||
}
|
||||
@ -95,11 +95,91 @@ class SupplierPayableOverrideStore
|
||||
return ['created' => $existing ? 0 : 1, 'updated' => $existing ? 1 : 0];
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量保存（大文件导入用）：按记录键 upsert，并批量写审计事件。
|
||||
*
|
||||
* @param array<string,array<string,mixed>> $records 记录键 => 字段值
|
||||
* @return array{created:int,updated:int}
|
||||
*/
|
||||
public function saveBatch(string $dataset, array $records, int $companyId, ?int $userId, string $source = 'import'): array
|
||||
{
|
||||
if ($records === []) {
|
||||
return ['created' => 0, 'updated' => 0];
|
||||
}
|
||||
|
||||
$connection = $this->connection();
|
||||
$now = now();
|
||||
$created = 0;
|
||||
$updated = 0;
|
||||
|
||||
$connection->transaction(function () use ($connection, $dataset, $records, $companyId, $userId, $source, $now, &$created, &$updated): void {
|
||||
$existing = [];
|
||||
foreach (array_chunk(array_keys($records), 1000) as $keys) {
|
||||
foreach ($connection->table('supplier_payable_overrides')
|
||||
->where('company_id', $companyId)
|
||||
->where('dataset', $dataset)
|
||||
->whereIn('record_key', $keys)
|
||||
->get() as $row) {
|
||||
$existing[(string) $row->record_key] = $row;
|
||||
}
|
||||
}
|
||||
|
||||
$upserts = [];
|
||||
$events = [];
|
||||
foreach ($records as $key => $values) {
|
||||
$before = $existing[$key] ?? null;
|
||||
$before === null ? $created++ : $updated++;
|
||||
$encoded = $this->encodeValues($values);
|
||||
|
||||
$upserts[] = [
|
||||
'company_id' => $companyId,
|
||||
'dataset' => $dataset,
|
||||
'record_key' => $key,
|
||||
'values' => $encoded,
|
||||
'created_by' => $before->created_by ?? $userId,
|
||||
'updated_by' => $userId,
|
||||
'created_at' => $before->created_at ?? $now,
|
||||
'updated_at' => $now,
|
||||
];
|
||||
$events[] = [
|
||||
'company_id' => $companyId,
|
||||
'dataset' => $dataset,
|
||||
'record_key' => $key,
|
||||
'source' => $source,
|
||||
'before_values' => $before === null ? null : (string) $before->values,
|
||||
'after_values' => $encoded,
|
||||
'user_id' => $userId,
|
||||
'created_at' => $now,
|
||||
];
|
||||
}
|
||||
|
||||
foreach (array_chunk($upserts, 500) as $chunk) {
|
||||
$connection->table('supplier_payable_overrides')->upsert(
|
||||
$chunk,
|
||||
['company_id', 'dataset', 'record_key'],
|
||||
['values', 'updated_by', 'updated_at'],
|
||||
);
|
||||
}
|
||||
|
||||
foreach (array_chunk($events, 500) as $chunk) {
|
||||
$connection->table('supplier_payable_override_events')->insert($chunk);
|
||||
}
|
||||
});
|
||||
|
||||
return ['created' => $created, 'updated' => $updated];
|
||||
}
|
||||
|
||||
public function transaction(Closure $callback): mixed
|
||||
{
|
||||
return $this->connection()->transaction($callback);
|
||||
}
|
||||
|
||||
/** 值编码为 JSON；脏字节（非法 UTF-8）用替代字符，避免整批写入失败。 */
|
||||
private function encodeValues(array $values): string
|
||||
{
|
||||
return (string) json_encode($values, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE);
|
||||
}
|
||||
|
||||
/**
|
||||
* 列出某个数据集已经保存的全部记录（「终版」目录的独立数据就是这些行）。
|
||||
*
|
||||
|
||||
@ -71,6 +71,8 @@
|
||||
@include('business.pages.supplier-payables.maintenance-import-modal', [
|
||||
'supplierImportAction' => $finalActions . '/import',
|
||||
'supplierTemplateUrl' => $finalActions . '/template',
|
||||
'supplierImportUploadUrl' => $finalActions . '/import-upload',
|
||||
'supplierImportChunkUrl' => $finalActions . '/import-chunk',
|
||||
])
|
||||
|
||||
<div class="payables-final-card">
|
||||
|
||||
@ -5,7 +5,9 @@
|
||||
<h2 id="supplier-import-modal-title">导入数据</h2>
|
||||
<button type="button" data-supplier-import-close aria-label="关闭" title="关闭"><span class="material-icons-outlined">close</span></button>
|
||||
</header>
|
||||
<form method="POST" enctype="multipart/form-data" action="{{ $supplierImportAction }}" data-supplier-import-form>
|
||||
<form method="POST" enctype="multipart/form-data" action="{{ $supplierImportAction }}" data-supplier-import-form
|
||||
data-upload-url="{{ $supplierImportUploadUrl ?? $supplierImportAction . '-upload' }}"
|
||||
data-chunk-url="{{ $supplierImportChunkUrl ?? $supplierImportAction . '-chunk' }}">
|
||||
@csrf
|
||||
<div class="supplier-import-dialog">
|
||||
<a href="{{ $supplierTemplateUrl }}" class="supplier-import-template-link">
|
||||
@ -17,6 +19,10 @@
|
||||
<span class="material-icons-outlined" aria-hidden="true">upload_file</span>
|
||||
<span data-supplier-import-file-name>选择导入文件</span>
|
||||
</button>
|
||||
<div class="supplier-import-progress" data-supplier-import-progress hidden>
|
||||
<div class="supplier-import-progress__track"><span data-supplier-import-bar></span></div>
|
||||
<p data-supplier-import-status role="status" aria-live="polite"></p>
|
||||
</div>
|
||||
</div>
|
||||
<footer>
|
||||
<button type="button" class="supplier-maintenance-button" data-supplier-import-close>取消</button>
|
||||
@ -40,6 +46,10 @@
|
||||
.supplier-import-modal .supplier-maintenance-button { height: 34px; padding: 0 14px; border: 1px solid #D8D2C7; border-radius: 8px; background: #fff; color: #171717; font-size: .8125rem; font-weight: 500; cursor: pointer; }
|
||||
.supplier-import-modal .supplier-maintenance-button.is-primary { border-color: #856632; background: #856632; color: #fff; }
|
||||
.supplier-import-modal [data-supplier-import-submit]:disabled { cursor: not-allowed; opacity: .5; }
|
||||
.supplier-import-progress { display: grid; gap: 8px; }
|
||||
.supplier-import-progress__track { height: 6px; overflow: hidden; border-radius: 999px; background: #EFEAE0; }
|
||||
.supplier-import-progress__track > span { display: block; width: 0; height: 100%; background: #856632; transition: width .3s ease; }
|
||||
.supplier-import-progress p { margin: 0; color: #595959; font-size: .75rem; }
|
||||
</style>
|
||||
@endonce
|
||||
|
||||
@ -54,21 +64,47 @@
|
||||
const input = modal.querySelector('[data-supplier-import-file]');
|
||||
const fileName = modal.querySelector('[data-supplier-import-file-name]');
|
||||
const submit = modal.querySelector('[data-supplier-import-submit]');
|
||||
const progress = modal.querySelector('[data-supplier-import-progress]');
|
||||
const bar = modal.querySelector('[data-supplier-import-bar]');
|
||||
const status = modal.querySelector('[data-supplier-import-status]');
|
||||
const openButton = document.querySelector('[data-supplier-import-open]');
|
||||
const csrf = form.querySelector('input[name="_token"]')?.value || '';
|
||||
let returnFocus = null;
|
||||
let running = false;
|
||||
const reset = () => {
|
||||
form.reset();
|
||||
fileName.textContent = '选择导入文件';
|
||||
submit.disabled = true;
|
||||
submit.textContent = '开始导入';
|
||||
delete form.dataset.submitting;
|
||||
progress.hidden = true;
|
||||
bar.style.width = '0%';
|
||||
status.textContent = '';
|
||||
};
|
||||
const close = () => {
|
||||
if (form.dataset.submitting) return;
|
||||
if (running) return;
|
||||
modal.hidden = true;
|
||||
reset();
|
||||
returnFocus?.focus();
|
||||
};
|
||||
const render = state => {
|
||||
const percent = state.total > 0 ? Math.min(100, Math.round((state.offset / state.total) * 100)) : 0;
|
||||
bar.style.width = percent + '%';
|
||||
status.textContent = `正在导入 ${state.offset.toLocaleString()} / ${state.total.toLocaleString()} 条（新增 ${state.created.toLocaleString()}，更新 ${state.updated.toLocaleString()}）`;
|
||||
};
|
||||
const request = async (url, options = {}) => {
|
||||
const response = await fetch(url, {
|
||||
credentials: 'same-origin',
|
||||
...options,
|
||||
headers: { Accept: 'application/json', 'X-CSRF-TOKEN': csrf, ...(options.headers || {}) },
|
||||
});
|
||||
let payload = null;
|
||||
try { payload = await response.json(); } catch (error) { payload = null; }
|
||||
if (!response.ok) {
|
||||
const first = payload?.errors ? Object.values(payload.errors).flat()[0] : null;
|
||||
throw new Error(first || payload?.message || `导入失败（HTTP ${response.status}）`);
|
||||
}
|
||||
return payload;
|
||||
};
|
||||
openButton?.addEventListener('click', () => {
|
||||
returnFocus = openButton;
|
||||
modal.hidden = false;
|
||||
@ -79,14 +115,46 @@
|
||||
fileName.textContent = file?.name || '选择导入文件';
|
||||
submit.disabled = !file;
|
||||
});
|
||||
form?.addEventListener('submit', event => {
|
||||
if (!input.files?.length) {
|
||||
event.preventDefault();
|
||||
return;
|
||||
}
|
||||
form.dataset.submitting = '1';
|
||||
form?.addEventListener('submit', async event => {
|
||||
event.preventDefault();
|
||||
if (running || !input.files?.length) return;
|
||||
|
||||
running = true;
|
||||
submit.disabled = true;
|
||||
submit.textContent = '导入中...';
|
||||
progress.hidden = false;
|
||||
bar.style.width = '0%';
|
||||
status.textContent = '正在上传文件...';
|
||||
|
||||
try {
|
||||
const data = new FormData();
|
||||
data.append('_token', csrf);
|
||||
data.append('import', input.files[0]);
|
||||
|
||||
const uploaded = await request(form.dataset.uploadUrl, { method: 'POST', body: data });
|
||||
const jobToken = uploaded.token;
|
||||
let state = uploaded;
|
||||
render(state);
|
||||
|
||||
while (!state.done) {
|
||||
state = await request(form.dataset.chunkUrl, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ token: jobToken, offset: state.offset }),
|
||||
});
|
||||
render(state);
|
||||
}
|
||||
|
||||
status.textContent = `导入完成：新增 ${state.created.toLocaleString()} 条，更新 ${state.updated.toLocaleString()} 条。`;
|
||||
running = false;
|
||||
submit.textContent = '已导入';
|
||||
setTimeout(() => window.location.reload(), 900);
|
||||
} catch (error) {
|
||||
running = false;
|
||||
submit.disabled = false;
|
||||
submit.textContent = '开始导入';
|
||||
status.textContent = String(error?.message || error);
|
||||
}
|
||||
});
|
||||
modal.querySelectorAll('[data-supplier-import-close]').forEach(node => node.addEventListener('click', close));
|
||||
document.addEventListener('keydown', event => {
|
||||
|
||||
@ -373,6 +373,11 @@ foreach ($payablesFinalDatasets as $finalSection => $finalDatasets) {
|
||||
Route::post('import', [PayablesFinalActions::class, 'import'])
|
||||
->middleware('import')
|
||||
->defaults('dataset', $dataset)->defaults('finalSection', $finalSection)->name('import');
|
||||
Route::post('import-upload', [PayablesFinalActions::class, 'importUpload'])
|
||||
->middleware('import')
|
||||
->defaults('dataset', $dataset)->defaults('finalSection', $finalSection)->name('import.upload');
|
||||
Route::post('import-chunk', [PayablesFinalActions::class, 'importChunk'])
|
||||
->defaults('dataset', $dataset)->defaults('finalSection', $finalSection)->name('import.chunk');
|
||||
Route::patch('{recordKey}', [PayablesFinalActions::class, 'update'])
|
||||
->where('recordKey', '[A-Za-z0-9:_-]+')
|
||||
->defaults('dataset', $dataset)->defaults('finalSection', $finalSection)->name('update');
|
||||
|
||||
196
tests/Feature/Business/PayablesFinalImportRunnerTest.php
Normal file
196
tests/Feature/Business/PayablesFinalImportRunnerTest.php
Normal file
@ -0,0 +1,196 @@
|
||||
<?php
|
||||
|
||||
namespace Tests\Feature\Business;
|
||||
|
||||
use App\Models\Auth\Permission;
|
||||
use Illuminate\Http\UploadedFile;
|
||||
use Modules\EcommerceFinance\Services\PayablesFinalImportRunner;
|
||||
use Modules\EcommerceFinance\Services\PayablesFinalRegistry;
|
||||
use Modules\EcommerceFinance\Services\PayablesWorkbookStreamReader;
|
||||
use Modules\EcommerceFinance\Services\SupplierPayableOverrideStore;
|
||||
use Tests\Feature\FeatureTestCase;
|
||||
use ZipArchive;
|
||||
|
||||
class PayablesFinalImportRunnerTest extends FeatureTestCase
|
||||
{
|
||||
private const HEADER = ['月份', '入库单号', '采购单号', '日期', '供应商', '货号', '商品编码', '规格名称', '数量', '单价',
|
||||
'金额', '预估单价', '预估金额', '供应商商品编码', '备注', '仓库名称', '快递名称', '快递单号', '未发出', '对账人'];
|
||||
|
||||
protected function setUp(): void
|
||||
{
|
||||
parent::setUp();
|
||||
|
||||
$this->user->attachPermission(
|
||||
Permission::where('name', 'read-admin-panel')->firstOrFail()
|
||||
);
|
||||
}
|
||||
|
||||
public function test_stream_reader_reads_xlsx_rows_and_dates(): void
|
||||
{
|
||||
$path = $this->makeXlsxFile([
|
||||
self::HEADER,
|
||||
['2026-09', 'RK-1', 'CG-1', '', '供应商甲', 'X-1', 'P-1', '规格甲', '2', '10', '20', '9', '18', 'SC-1', '备注甲', '华东仓', '顺丰', 'SF-1', '未发出', '对账人甲'],
|
||||
], dateColumnIndexes: [3]);
|
||||
|
||||
try {
|
||||
$reader = PayablesWorkbookStreamReader::forFile($path, 'xlsx');
|
||||
$this->assertSame(['Sheet1'], $reader->sheetNames());
|
||||
|
||||
$rows = iterator_to_array($reader->rows(0), true);
|
||||
$this->assertCount(2, $rows);
|
||||
$this->assertSame('月份', $rows[1][0]);
|
||||
$this->assertSame('供应商甲', $rows[2][4]);
|
||||
$this->assertSame('2026-09-08', $rows[2][3], '日期格式的单元格应转成 Y-m-d');
|
||||
} finally {
|
||||
@unlink($path);
|
||||
}
|
||||
}
|
||||
|
||||
public function test_stream_reader_reads_gbk_csv(): void
|
||||
{
|
||||
$content = mb_convert_encoding(implode(',', self::HEADER) . "\n2026-09,RK-9,CG-9,2026-09-08,供应商九,X-9,P-9,规格甲,2,10,20,9,18,SC-1,备注甲,华东仓,顺丰,SF-9,未发出,对账人甲\n", 'GBK', 'UTF-8');
|
||||
$path = $this->makeFile('gbk.csv', $content);
|
||||
|
||||
try {
|
||||
$reader = PayablesWorkbookStreamReader::forFile($path, 'csv');
|
||||
$rows = iterator_to_array($reader->rows(0), true);
|
||||
$this->assertSame('供应商九', $rows[2][4]);
|
||||
} finally {
|
||||
@unlink($path);
|
||||
}
|
||||
}
|
||||
|
||||
public function test_runner_imports_in_slices_and_cleans_up(): void
|
||||
{
|
||||
$lines = [implode(',', self::HEADER)];
|
||||
for ($i = 1; $i <= 7; $i++) {
|
||||
$lines[] = "2026-09,RK-{$i},CG-{$i},2026-09-08,供应商{$i},X-{$i},P-{$i},规格甲,2,10,20,9,18,SC-1,备注甲,华东仓,顺丰,SF-{$i},未发出,对账人甲";
|
||||
}
|
||||
|
||||
$file = UploadedFile::fake()->createWithContent('rows.csv', "\xEF\xBB\xBF" . implode("\n", $lines) . "\n");
|
||||
$runner = app(PayablesFinalImportRunner::class);
|
||||
$fields = app(PayablesFinalRegistry::class)->fields('supplier-payables-final', 'payment-reconciliation');
|
||||
$dataset = 'final:supplier:payment-reconciliation';
|
||||
|
||||
$prepared = $runner->prepare($file, $fields, $dataset, (int) $this->company->id, (int) $this->user->id);
|
||||
$this->assertSame(7, $prepared['total']);
|
||||
|
||||
$offset = 0;
|
||||
$created = 0;
|
||||
$slices = 0;
|
||||
do {
|
||||
$progress = $runner->process($prepared['token'], $offset, 3);
|
||||
$offset = $progress['offset'];
|
||||
$created = $progress['created'];
|
||||
$slices++;
|
||||
} while (! $progress['done'] && $slices < 10);
|
||||
|
||||
$this->assertTrue($progress['done']);
|
||||
$this->assertGreaterThanOrEqual(3, $slices, '应当分多片处理');
|
||||
$this->assertSame(7, $created);
|
||||
|
||||
$this->assertCount(7, app(SupplierPayableOverrideStore::class)->listRows($dataset, (int) $this->company->id));
|
||||
$this->assertDirectoryDoesNotExist($runner->storagePath() . '/' . $prepared['token'], '完成后应删除缓存');
|
||||
}
|
||||
|
||||
public function test_import_endpoints_accept_uploads_and_chunks(): void
|
||||
{
|
||||
$file = UploadedFile::fake()->createWithContent(
|
||||
'rows.csv',
|
||||
"\xEF\xBB\xBF" . implode(',', self::HEADER) . "\n2026-09,RK-1,CG-1,2026-09-08,供应商甲,X-1,P-1,规格甲,2,10,20,9,18,SC-1,备注甲,华东仓,顺丰,SF-1,未发出,对账人甲\n",
|
||||
);
|
||||
|
||||
$upload = $this->loginAs()->postJson(route('business.supplier-payables-final.payment-reconciliation.import.upload'), [
|
||||
'import' => $file,
|
||||
]);
|
||||
|
||||
$upload->assertOk()->assertJsonPath('total', 1);
|
||||
$token = $upload->json('token');
|
||||
$this->assertIsString($token);
|
||||
|
||||
$chunk = $this->loginAs()->postJson(route('business.supplier-payables-final.payment-reconciliation.import.chunk'), [
|
||||
'token' => $token,
|
||||
'offset' => 0,
|
||||
]);
|
||||
|
||||
$chunk->assertOk()->assertJsonPath('done', true)->assertJsonPath('created', 1);
|
||||
|
||||
$this->withExceptionHandling()
|
||||
->loginAs()
|
||||
->postJson(route('business.supplier-payables-final.payment-reconciliation.import.chunk'), [
|
||||
'token' => str_repeat('0', 32),
|
||||
'offset' => 0,
|
||||
])
|
||||
->assertStatus(422);
|
||||
}
|
||||
|
||||
public function test_final_page_wires_the_chunked_import_endpoints(): void
|
||||
{
|
||||
$this->loginAs()
|
||||
->get(route('business.pages.show', [
|
||||
'section' => 'supplier-payables-final',
|
||||
'page' => 'payment-reconciliation',
|
||||
]))
|
||||
->assertOk()
|
||||
->assertSee('workspace/supplier-payables-final/actions/payment-reconciliation/import-upload', false)
|
||||
->assertSee('workspace/supplier-payables-final/actions/payment-reconciliation/import-chunk', false)
|
||||
->assertSee('data-supplier-import-progress', false);
|
||||
}
|
||||
|
||||
/** @param list<list<string>> $rows */
|
||||
private function makeXlsxFile(array $rows, array $dateColumnIndexes = []): string
|
||||
{
|
||||
$path = $this->makeFile('book.xlsx', '');
|
||||
$strings = [];
|
||||
$sheet = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>';
|
||||
|
||||
foreach ($rows as $rowIndex => $row) {
|
||||
$sheet .= '<row r="' . ($rowIndex + 1) . '">';
|
||||
foreach ($row as $columnIndex => $value) {
|
||||
$cell = chr(65 + $columnIndex) . ($rowIndex + 1);
|
||||
if (in_array($columnIndex, $dateColumnIndexes, true) && $rowIndex > 0) {
|
||||
$sheet .= '<c r="' . $cell . '" s="1"><v>' . (int) \PhpOffice\PhpSpreadsheet\Shared\Date::PHPToExcel('2026-09-08') . '</v></c>';
|
||||
continue;
|
||||
}
|
||||
if ($value === '') {
|
||||
continue;
|
||||
}
|
||||
if (is_numeric($value) && ! in_array($columnIndex, [0, 1, 2], true)) {
|
||||
$sheet .= '<c r="' . $cell . '"><v>' . $value . '</v></c>';
|
||||
continue;
|
||||
}
|
||||
$strings[$value] ??= count($strings);
|
||||
$sheet .= '<c r="' . $cell . '" t="s"><v>' . $strings[$value] . '</v></c>';
|
||||
}
|
||||
$sheet .= '</row>';
|
||||
}
|
||||
$sheet .= '</sheetData></worksheet>';
|
||||
|
||||
$shared = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><sst xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" count="' . count($strings) . '" uniqueCount="' . count($strings) . '">';
|
||||
foreach (array_keys($strings) as $value) {
|
||||
$shared .= '<si><t>' . htmlspecialchars($value, ENT_XML1 | ENT_COMPAT, 'UTF-8') . '</t></si>';
|
||||
}
|
||||
$shared .= '</sst>';
|
||||
|
||||
$zip = new ZipArchive();
|
||||
$zip->open($path, ZipArchive::CREATE | ZipArchive::OVERWRITE);
|
||||
$zip->addFromString('[Content_Types].xml', '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/><Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/><Override PartName="/xl/sharedStrings.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sharedStrings+xml"/><Override PartName="/xl/styles.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml"/></Types>');
|
||||
$zip->addFromString('_rels/.rels', '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>');
|
||||
$zip->addFromString('xl/workbook.xml', '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="Sheet1" sheetId="1" r:id="rId1"/></sheets></workbook>');
|
||||
$zip->addFromString('xl/_rels/workbook.xml.rels', '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/><Relationship Id="rId2" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/sharedStrings" Target="sharedStrings.xml"/><Relationship Id="rId3" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/></Relationships>');
|
||||
$zip->addFromString('xl/styles.xml', '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><cellXfs count="2"><xf numFmtId="0"/><xf numFmtId="14"/></cellXfs></styleSheet>');
|
||||
$zip->addFromString('xl/worksheets/sheet1.xml', $sheet);
|
||||
$zip->addFromString('xl/sharedStrings.xml', $shared);
|
||||
$zip->close();
|
||||
|
||||
return $path;
|
||||
}
|
||||
|
||||
private function makeFile(string $name, string $content): string
|
||||
{
|
||||
$path = sys_get_temp_dir() . '/caiwu-import-test-' . bin2hex(random_bytes(6)) . '-' . $name;
|
||||
file_put_contents($path, $content);
|
||||
|
||||
return $path;
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user