苏木
3ceeacee63
同步导入在 20 万行规模下必然超时/爆内存：新增 xlsx/csv 流式读取器（共享字符串紧凑存放、支持日期单元格、GBK CSV），上传时把工作簿转成行缓存，之后按片处理（每片 2 万行/25 秒预算），前端显示进度并自动续跑；落库改为批量 upsert + 批量审计事件；上限提升到 20 万行，上传限制放宽到 50MB。
565 lines
18 KiB
PHP
565 lines
18 KiB
PHP
<?php
|
||
|
||
namespace Modules\EcommerceFinance\Services;
|
||
|
||
use Generator;
|
||
use PhpOffice\PhpSpreadsheet\IOFactory;
|
||
use PhpOffice\PhpSpreadsheet\Shared\Date as ExcelDate;
|
||
use RuntimeException;
|
||
use SplFixedArray;
|
||
use XMLReader;
|
||
use ZipArchive;
|
||
|
||
/**
|
||
* 逐行读取导入工作簿，内存有界（20 万行级文件也能跑）：
|
||
* - xlsx：ZipArchive + XMLReader 流式解析，共享字符串用「拼接串 + 偏移索引」紧凑存放；
|
||
* - csv：逐行 fgetcsv（自动去 BOM、非 UTF-8 按 GBK 转换）；
|
||
* - xls：退回 PhpSpreadsheet（老格式少见）。
|
||
* 日期格式的单元格按 Y-m-d（含时分秒时为 Y-m-d H:i:s）输出，其余保持原值。
|
||
*/
|
||
class PayablesWorkbookStreamReader
|
||
{
|
||
/** @var array<int,string>|null 工作表内部路径（xlsx） */
|
||
private ?array $sheetPaths = null;
|
||
|
||
/** @var list<int>|null 日期样式的 style 索引 */
|
||
private ?array $dateStyles = null;
|
||
|
||
/** @var string|null 共享字符串拼接串 */
|
||
private ?string $sharedStrings = null;
|
||
|
||
/** @var SplFixedArray<int>|null 共享字符串偏移（第 i 个字符串的起始位置） */
|
||
private ?SplFixedArray $sharedStringOffsets = null;
|
||
|
||
public function __construct(
|
||
private readonly string $path,
|
||
private readonly string $extension,
|
||
) {
|
||
}
|
||
|
||
/** 新建读取器（按扩展名）。 */
|
||
public static function forFile(string $path, string $extension): self
|
||
{
|
||
return new self($path, strtolower($extension));
|
||
}
|
||
|
||
/** @return list<string> 工作表名称（xls 走 PhpSpreadsheet，仅取名称） */
|
||
public function sheetNames(): array
|
||
{
|
||
if ($this->extension === 'xlsx') {
|
||
return array_map(
|
||
static fn (array $sheet): string => $sheet['name'],
|
||
$this->xlsxSheets(),
|
||
);
|
||
}
|
||
|
||
if ($this->extension === 'csv') {
|
||
return ['Sheet1'];
|
||
}
|
||
|
||
$spreadsheet = IOFactory::createReader('Xls')->load($this->path);
|
||
try {
|
||
return $spreadsheet->getSheetNames();
|
||
} finally {
|
||
$spreadsheet->disconnectWorksheets();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 逐行读取指定工作表。
|
||
*
|
||
* @param int|null $skipUntilRow 跳过该行号之前的行（含），用于分片续读
|
||
* @param int|null $stopAfterRow 读到该行号后停止
|
||
* @return Generator<int,array<int,mixed>> 行号（1 基）=> 列序号（0 基）=> 值
|
||
*/
|
||
public function rows(int $sheetIndex = 0, ?int $skipUntilRow = null, ?int $stopAfterRow = null): Generator
|
||
{
|
||
yield from match ($this->extension) {
|
||
'xlsx' => $this->xlsxRows($sheetIndex, $skipUntilRow, $stopAfterRow),
|
||
'csv' => $this->csvRows($skipUntilRow, $stopAfterRow),
|
||
default => $this->xlsRows($sheetIndex, $skipUntilRow, $stopAfterRow),
|
||
};
|
||
}
|
||
|
||
/** @return Generator<int,array<int,mixed>> */
|
||
private function xlsxRows(int $sheetIndex, ?int $skipUntilRow, ?int $stopAfterRow): Generator
|
||
{
|
||
$path = $this->sheetPaths()[$sheetIndex] ?? null;
|
||
if ($path === null) {
|
||
return;
|
||
}
|
||
|
||
$zip = new ZipArchive();
|
||
if ($zip->open($this->path) !== true) {
|
||
throw new RuntimeException('无法打开工作簿。');
|
||
}
|
||
|
||
try {
|
||
$stream = $zip->getStream($path);
|
||
if ($stream === false) {
|
||
return;
|
||
}
|
||
|
||
$reader = new XMLReader();
|
||
$reader->XML(stream_get_contents($stream), null, LIBXML_NONET);
|
||
fclose($stream);
|
||
|
||
$dictionary = $this->xlsxSharedStrings();
|
||
$rowNumber = 0;
|
||
|
||
while ($reader->read()) {
|
||
if ($reader->nodeType !== XMLReader::ELEMENT || $reader->localName !== 'row') {
|
||
continue;
|
||
}
|
||
|
||
$rowNumber = (int) ($reader->getAttribute('r') ?: $rowNumber + 1);
|
||
if ($stopAfterRow !== null && $rowNumber > $stopAfterRow) {
|
||
return;
|
||
}
|
||
|
||
if ($skipUntilRow !== null && $rowNumber <= $skipUntilRow) {
|
||
$reader->next(); // 跳过整行
|
||
continue;
|
||
}
|
||
|
||
yield $rowNumber => $this->readRow($reader, $dictionary);
|
||
}
|
||
} finally {
|
||
$zip->close();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 读一行里的单元格（当前节点位于 <row>）。
|
||
*
|
||
* @param array{0:string,1:SplFixedArray<int>} $dictionary
|
||
* @return array<int,mixed>
|
||
*/
|
||
private function readRow(XMLReader $reader, array $dictionary): array
|
||
{
|
||
$cells = [];
|
||
$depth = $reader->depth;
|
||
$empty = $reader->isEmptyElement;
|
||
if ($empty) {
|
||
return $cells;
|
||
}
|
||
|
||
while ($reader->read()) {
|
||
if ($reader->nodeType === XMLReader::END_ELEMENT && $reader->depth === $depth) {
|
||
break;
|
||
}
|
||
if ($reader->nodeType !== XMLReader::ELEMENT || $reader->localName !== 'c' || $reader->isEmptyElement) {
|
||
continue;
|
||
}
|
||
|
||
$reference = (string) $reader->getAttribute('r');
|
||
$type = (string) $reader->getAttribute('t');
|
||
$styleIndex = (string) $reader->getAttribute('s');
|
||
$columnIndex = $reference !== '' ? $this->columnIndex($reference) : null;
|
||
$value = $this->readCellValue($reader, $type, $dictionary);
|
||
$columnIndex ??= count($cells);
|
||
|
||
if ($value === null || $value === '') {
|
||
continue;
|
||
}
|
||
|
||
$cells[$columnIndex] = $this->formatValue($value, $type, $styleIndex);
|
||
}
|
||
|
||
return $cells;
|
||
}
|
||
|
||
/**
|
||
* @param array{0:string,1:SplFixedArray<int>} $dictionary
|
||
*/
|
||
private function readCellValue(XMLReader $reader, string $type, array $dictionary): mixed
|
||
{
|
||
$value = null;
|
||
$depth = $reader->depth;
|
||
|
||
while ($reader->read()) {
|
||
if ($reader->nodeType === XMLReader::END_ELEMENT && $reader->depth === $depth) {
|
||
break;
|
||
}
|
||
if ($reader->nodeType !== XMLReader::ELEMENT) {
|
||
continue;
|
||
}
|
||
|
||
if ($reader->localName === 'v') {
|
||
$value = $reader->isEmptyElement ? '' : $reader->readString();
|
||
continue;
|
||
}
|
||
|
||
if ($reader->localName === 'is') {
|
||
$value = $this->readInlineString($reader);
|
||
continue;
|
||
}
|
||
|
||
if ($reader->localName === 't' && $type === 'inlineStr') {
|
||
$value = $reader->isEmptyElement ? '' : $reader->readString();
|
||
}
|
||
}
|
||
|
||
if ($type === 's') {
|
||
$index = (int) $value;
|
||
[$blob, $offsets] = $dictionary;
|
||
if ($index < 0 || $index + 1 >= $offsets->getSize()) {
|
||
return null;
|
||
}
|
||
|
||
return substr($blob, $offsets[$index], $offsets[$index + 1] - $offsets[$index]);
|
||
}
|
||
|
||
if ($type === 'str' || $type === 'inlineStr') {
|
||
return $value;
|
||
}
|
||
|
||
if ($type === 'b') {
|
||
return $value === '1' ? '1' : '0';
|
||
}
|
||
|
||
return $value;
|
||
}
|
||
|
||
/** 富文本 <is> 里可能有多个 <t>。 */
|
||
private function readInlineString(XMLReader $reader): string
|
||
{
|
||
$depth = $reader->depth;
|
||
$text = '';
|
||
|
||
while ($reader->read()) {
|
||
if ($reader->nodeType === XMLReader::END_ELEMENT && $reader->depth === $depth) {
|
||
break;
|
||
}
|
||
if ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 't' && ! $reader->isEmptyElement) {
|
||
$text .= $reader->readString();
|
||
}
|
||
}
|
||
|
||
return $text;
|
||
}
|
||
|
||
/** 日期样式的数字单元格转成日期字符串。 */
|
||
private function formatValue(mixed $value, string $type, string $styleIndex): mixed
|
||
{
|
||
if (! is_string($value) || $value === '' || $styleIndex === '' || ! is_numeric($value)) {
|
||
return $value;
|
||
}
|
||
|
||
if (! in_array((int) $styleIndex, $this->dateStyles(), true)) {
|
||
return $value;
|
||
}
|
||
|
||
$serial = (float) $value;
|
||
$date = ExcelDate::excelToDateTimeObject($serial);
|
||
|
||
return fmod($serial, 1.0) === 0.0 ? $date->format('Y-m-d') : $date->format('Y-m-d H:i:s');
|
||
}
|
||
|
||
/** @return Generator<int,array<int,mixed>> */
|
||
private function csvRows(?int $skipUntilRow, ?int $stopAfterRow): Generator
|
||
{
|
||
$handle = fopen($this->path, 'rb');
|
||
if ($handle === false) {
|
||
return;
|
||
}
|
||
|
||
try {
|
||
$delimiter = $this->detectDelimiter((string) fgets($handle));
|
||
rewind($handle);
|
||
|
||
$line = 0;
|
||
while (($row = fgetcsv($handle, 0, $delimiter)) !== false) {
|
||
$line++;
|
||
if ($stopAfterRow !== null && $line > $stopAfterRow) {
|
||
return;
|
||
}
|
||
if ($skipUntilRow !== null && $line <= $skipUntilRow) {
|
||
continue;
|
||
}
|
||
|
||
$cells = [];
|
||
foreach ($row as $index => $value) {
|
||
$value = $this->normalizeEncoding((string) $value);
|
||
if ($index === 0) {
|
||
$value = preg_replace('/^\x{FEFF}/u', '', $value) ?? $value;
|
||
}
|
||
if ($value !== '') {
|
||
$cells[$index] = $value;
|
||
}
|
||
}
|
||
|
||
yield $line => $cells;
|
||
}
|
||
} finally {
|
||
fclose($handle);
|
||
}
|
||
}
|
||
|
||
/** @return Generator<int,array<int,mixed>> */
|
||
private function xlsRows(int $sheetIndex, ?int $skipUntilRow, ?int $stopAfterRow): Generator
|
||
{
|
||
$spreadsheet = IOFactory::createReader('Xls')->load($this->path);
|
||
try {
|
||
$sheet = $spreadsheet->getSheet($sheetIndex);
|
||
foreach ($sheet->getRowIterator() as $row) {
|
||
$line = $row->getRowIndex();
|
||
if ($stopAfterRow !== null && $line > $stopAfterRow) {
|
||
return;
|
||
}
|
||
if ($skipUntilRow !== null && $line <= $skipUntilRow) {
|
||
continue;
|
||
}
|
||
|
||
$cells = [];
|
||
foreach ($row->getCellIterator() as $cell) {
|
||
$value = $cell->getValue();
|
||
if ($value !== null && $value !== '') {
|
||
$cells[$this->columnIndex($cell->getColumn())] = $value;
|
||
}
|
||
}
|
||
|
||
yield $line => $cells;
|
||
}
|
||
} finally {
|
||
$spreadsheet->disconnectWorksheets();
|
||
}
|
||
}
|
||
|
||
/** @return list<int> 日期样式的 style 索引 */
|
||
private function dateStyles(): array
|
||
{
|
||
if ($this->dateStyles !== null) {
|
||
return $this->dateStyles;
|
||
}
|
||
|
||
if ($this->extension !== 'xlsx') {
|
||
return $this->dateStyles = [];
|
||
}
|
||
|
||
$zip = new ZipArchive();
|
||
if ($zip->open($this->path) !== true) {
|
||
return $this->dateStyles = [];
|
||
}
|
||
|
||
$styles = [];
|
||
try {
|
||
$stylesXml = $zip->getFromName('xl/styles.xml');
|
||
if ($stylesXml === false) {
|
||
return $this->dateStyles = [];
|
||
}
|
||
|
||
$reader = new XMLReader();
|
||
$reader->XML($stylesXml, null, LIBXML_NONET);
|
||
|
||
$customDateFormats = [];
|
||
$cellXfs = [];
|
||
$inCellXfs = false;
|
||
|
||
while ($reader->read()) {
|
||
if ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 'numFmt') {
|
||
$id = (int) $reader->getAttribute('numFmtId');
|
||
$code = (string) $reader->getAttribute('formatCode');
|
||
if ($id >= 164 && $this->looksLikeDateFormat($code)) {
|
||
$customDateFormats[$id] = true;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 'cellXfs') {
|
||
$inCellXfs = true;
|
||
continue;
|
||
}
|
||
|
||
if ($reader->nodeType === XMLReader::END_ELEMENT && $reader->localName === 'cellXfs') {
|
||
$inCellXfs = false;
|
||
continue;
|
||
}
|
||
|
||
if ($inCellXfs && $reader->nodeType === XMLReader::ELEMENT && $reader->localName === 'xf') {
|
||
$cellXfs[] = (int) $reader->getAttribute('numFmtId');
|
||
}
|
||
}
|
||
|
||
foreach ($cellXfs as $index => $numFmtId) {
|
||
if (($numFmtId >= 14 && $numFmtId <= 22) || ($numFmtId >= 45 && $numFmtId <= 47) || isset($customDateFormats[$numFmtId])) {
|
||
$styles[] = $index;
|
||
}
|
||
}
|
||
|
||
$reader->close();
|
||
} finally {
|
||
$zip->close();
|
||
}
|
||
|
||
return $this->dateStyles = $styles;
|
||
}
|
||
|
||
private function looksLikeDateFormat(string $code): bool
|
||
{
|
||
$code = preg_replace('/"[^"]*"/', '', $code) ?? $code;
|
||
$code = preg_replace('/\[[^\]]*\]/', '', $code) ?? $code;
|
||
|
||
return (bool) preg_match('/[ymdhs]/i', str_replace(['AM/PM', 'am/pm'], '', $code));
|
||
}
|
||
|
||
/** @return list<array{name:string,path:string}> */
|
||
private function xlsxSheets(): array
|
||
{
|
||
if ($this->sheetPaths !== null) {
|
||
return $this->sheetPaths;
|
||
}
|
||
|
||
$zip = new ZipArchive();
|
||
if ($zip->open($this->path) !== true) {
|
||
throw new RuntimeException('无法打开工作簿。');
|
||
}
|
||
|
||
$sheets = [];
|
||
try {
|
||
$workbook = $zip->getFromName('xl/workbook.xml');
|
||
$relationships = $zip->getFromName('xl/_rels/workbook.xml.rels');
|
||
if ($workbook === false || $relationships === false) {
|
||
throw new RuntimeException('工作簿结构不完整。');
|
||
}
|
||
|
||
$targets = [];
|
||
$rels = new XMLReader();
|
||
$rels->XML($relationships, null, LIBXML_NONET);
|
||
while ($rels->read()) {
|
||
if ($rels->nodeType === XMLReader::ELEMENT && $rels->localName === 'Relationship') {
|
||
$targets[(string) $rels->getAttribute('Id')] = ltrim((string) $rels->getAttribute('Target'), '/');
|
||
}
|
||
}
|
||
$rels->close();
|
||
|
||
$book = new XMLReader();
|
||
$book->XML($workbook, null, LIBXML_NONET);
|
||
while ($book->read()) {
|
||
if ($book->nodeType !== XMLReader::ELEMENT || $book->localName !== 'sheet') {
|
||
continue;
|
||
}
|
||
$relId = (string) ($book->getAttributeNs('id', 'http://schemas.openxmlformats.org/officeDocument/2006/relationships') ?? '');
|
||
$target = $targets[$relId] ?? ('worksheets/sheet' . (count($sheets) + 1) . '.xml');
|
||
$sheets[] = [
|
||
'name' => (string) $book->getAttribute('name'),
|
||
'path' => str_starts_with($target, 'xl/') ? $target : 'xl/' . $target,
|
||
];
|
||
}
|
||
$book->close();
|
||
} finally {
|
||
$zip->close();
|
||
}
|
||
|
||
return $this->sheetPaths = $sheets;
|
||
}
|
||
|
||
/** @return list<string> */
|
||
private function sheetPaths(): array
|
||
{
|
||
return array_column($this->xlsxSheets(), 'path');
|
||
}
|
||
|
||
/** @return array{0:string,1:SplFixedArray<int>} */
|
||
private function xlsxSharedStrings(): array
|
||
{
|
||
if ($this->sharedStrings !== null && $this->sharedStringOffsets !== null) {
|
||
return [$this->sharedStrings, $this->sharedStringOffsets];
|
||
}
|
||
|
||
$zip = new ZipArchive();
|
||
if ($zip->open($this->path) !== true) {
|
||
throw new RuntimeException('无法打开工作簿。');
|
||
}
|
||
|
||
$blob = '';
|
||
try {
|
||
$xml = $zip->getFromName('xl/sharedStrings.xml');
|
||
if ($xml === false) {
|
||
return [$this->sharedStrings = '', $this->sharedStringOffsets = new SplFixedArray(1)];
|
||
}
|
||
|
||
$reader = new XMLReader();
|
||
$reader->XML($xml, null, LIBXML_NONET);
|
||
|
||
$count = 0;
|
||
$offsets = new SplFixedArray(1024);
|
||
$offsets[0] = 0;
|
||
$depth = 0;
|
||
$collect = false;
|
||
$text = '';
|
||
|
||
while ($reader->read()) {
|
||
if ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 'si') {
|
||
$collect = true;
|
||
$text = '';
|
||
$depth = $reader->depth;
|
||
continue;
|
||
}
|
||
|
||
if (! $collect) {
|
||
continue;
|
||
}
|
||
|
||
if ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 't' && ! $reader->isEmptyElement) {
|
||
$text .= $reader->readString();
|
||
continue;
|
||
}
|
||
|
||
if ($reader->nodeType === XMLReader::END_ELEMENT && $reader->localName === 'si' && $reader->depth === $depth) {
|
||
$blob .= $text;
|
||
$count++;
|
||
if ($count + 1 > $offsets->getSize()) {
|
||
$offsets->setSize($offsets->getSize() * 2);
|
||
}
|
||
$offsets[$count] = strlen($blob);
|
||
$collect = false;
|
||
}
|
||
}
|
||
|
||
$reader->close();
|
||
$offsets->setSize($count + 1);
|
||
} finally {
|
||
$zip->close();
|
||
}
|
||
|
||
return [$this->sharedStrings = $blob, $this->sharedStringOffsets = $offsets];
|
||
}
|
||
|
||
private function detectDelimiter(string $line): string
|
||
{
|
||
$counts = [
|
||
',' => substr_count($line, ','),
|
||
';' => substr_count($line, ';'),
|
||
"\t" => substr_count($line, "\t"),
|
||
];
|
||
|
||
arsort($counts);
|
||
$delimiter = (string) array_key_first($counts);
|
||
|
||
return $counts[$delimiter] > 0 ? $delimiter : ',';
|
||
}
|
||
|
||
private function normalizeEncoding(string $value): string
|
||
{
|
||
if ($value === '' || mb_check_encoding($value, 'UTF-8')) {
|
||
return $value;
|
||
}
|
||
|
||
return mb_convert_encoding($value, 'UTF-8', 'GBK');
|
||
}
|
||
|
||
private function columnIndex(string $reference): int
|
||
{
|
||
preg_match('/^([A-Z]+)/i', $reference, $matches);
|
||
|
||
$letters = strtoupper($matches[1] ?? 'A');
|
||
$index = 0;
|
||
foreach (str_split($letters) as $letter) {
|
||
$index = $index * 26 + (ord($letter) - 64);
|
||
}
|
||
|
||
return $index - 1;
|
||
}
|
||
}
|