应付款终版导入:分块读取工作簿,避免大文件把进程撑爆导致 500 #359
@ -2,6 +2,10 @@ memory_limit=512M
|
||||
max_execution_time=120
|
||||
post_max_size=64M
|
||||
upload_max_filesize=10M
|
||||
# 线上必须把错误写进日志而不是显示给用户;否则致命错误(内存/超时)会既看不到、也无处查。
|
||||
display_errors=Off
|
||||
log_errors=On
|
||||
error_log=/var/www/html/storage/logs/php-errors.log
|
||||
date.timezone=Asia/Shanghai
|
||||
expose_php=Off
|
||||
opcache.enable=1
|
||||
|
||||
@ -4,17 +4,29 @@ namespace Modules\EcommerceFinance\Services;
|
||||
|
||||
use Illuminate\Http\UploadedFile;
|
||||
use Illuminate\Validation\ValidationException;
|
||||
use PhpOffice\PhpSpreadsheet\Cell\Coordinate;
|
||||
use PhpOffice\PhpSpreadsheet\IOFactory;
|
||||
use PhpOffice\PhpSpreadsheet\Reader\IReadFilter;
|
||||
use PhpOffice\PhpSpreadsheet\Spreadsheet;
|
||||
use Throwable;
|
||||
|
||||
/**
|
||||
* 「终版」目录按表导入:数据独立存放,可以从空表开始导入,
|
||||
* 不要求基础页面那种“先导出再改”的记录标识列。
|
||||
*
|
||||
* 读取按行分块(每次只把一块行载入内存),避免整表载入把 PHP 进程撑爆
|
||||
* ——大文件或带“幽灵区域”的工作簿以前会让导入直接 500。
|
||||
*/
|
||||
class PayablesFinalImporter
|
||||
{
|
||||
private const MAX_ROWS = 10000;
|
||||
|
||||
/** 分块读取:每块载入的行数。 */
|
||||
private const CHUNK_ROWS = 2000;
|
||||
|
||||
/** 表头只在前若干行里找,避免把数据行误判成表头。 */
|
||||
private const HEADER_SCAN_ROWS = 20;
|
||||
|
||||
public function __construct(
|
||||
private readonly PayablesFieldNormalizer $normalizer,
|
||||
private readonly SupplierPayableOverrideStore $store,
|
||||
@ -41,28 +53,24 @@ class PayablesFinalImporter
|
||||
'csv' => 'Csv',
|
||||
default => throw new \RuntimeException('Unsupported workbook format.'),
|
||||
};
|
||||
$spreadsheet = IOFactory::createReader($readerType)->load($file->getRealPath());
|
||||
[$rows, $headerRow, $columns] = $this->bestSheet($spreadsheet, $lookup);
|
||||
$header = $this->locateHeader($readerType, $file->getRealPath(), $lookup);
|
||||
} catch (Throwable) {
|
||||
throw ValidationException::withMessages([
|
||||
'import' => '无法读取导入文件,请确认文件未损坏且格式为 xlsx、xls 或 csv。',
|
||||
]);
|
||||
} finally {
|
||||
if (isset($spreadsheet)) {
|
||||
$spreadsheet->disconnectWorksheets();
|
||||
}
|
||||
}
|
||||
|
||||
if ($rows === null || $columns === []) {
|
||||
if ($header === null) {
|
||||
throw ValidationException::withMessages([
|
||||
'import' => '没有识别到表头,请先下载本页面的导入模板,按模板列填写后重试。',
|
||||
]);
|
||||
}
|
||||
|
||||
[$sheetIndex, $headerRow, $columns] = $header;
|
||||
|
||||
$store = $this->store;
|
||||
$records = [];
|
||||
foreach (array_slice($rows, $headerRow + 1, null, true) as $rowIndex => $row) {
|
||||
$line = $rowIndex + 1;
|
||||
foreach ($this->rowsAfter($readerType, $file->getRealPath(), $sheetIndex, $headerRow + 1) as $line => $row) {
|
||||
$key = '';
|
||||
$rawValues = [];
|
||||
foreach ($columns as $columnIndex => $field) {
|
||||
@ -116,30 +124,94 @@ class PayablesFinalImporter
|
||||
return $counts + ['rows' => count($records)];
|
||||
}
|
||||
|
||||
private function bestSheet(mixed $spreadsheet, array $lookup): array
|
||||
/**
|
||||
* 载入第一块行,在其中找到表头。
|
||||
*
|
||||
* @param array<string,string> $lookup
|
||||
* @return array{0:int,1:int,2:array<int,string>}|null [sheet 序号, 表头行号(1 基), 列序号 => 字段]
|
||||
*/
|
||||
private function locateHeader(string $readerType, string $path, array $lookup): ?array
|
||||
{
|
||||
$best = [null, null, []];
|
||||
$best = null;
|
||||
$score = 0;
|
||||
foreach ($spreadsheet->getWorksheetIterator() as $worksheet) {
|
||||
$rows = $worksheet->toArray(null, false, false, false);
|
||||
foreach (array_slice($rows, 0, 20, true) as $rowIndex => $row) {
|
||||
$columns = [];
|
||||
foreach ($row as $columnIndex => $heading) {
|
||||
$key = $this->header($heading);
|
||||
if ($key !== '' && isset($lookup[$key])) {
|
||||
$columns[$columnIndex] = $lookup[$key];
|
||||
$spreadsheet = $this->loadChunk($readerType, $path, 1, self::CHUNK_ROWS);
|
||||
|
||||
try {
|
||||
foreach ($spreadsheet->getWorksheetIterator() as $sheetIndex => $worksheet) {
|
||||
foreach ($worksheet->getRowIterator(1, self::HEADER_SCAN_ROWS) as $row) {
|
||||
$columns = [];
|
||||
foreach ($row->getCellIterator() as $cell) {
|
||||
$key = $this->header($cell->getValue());
|
||||
if ($key !== '' && isset($lookup[$key])) {
|
||||
$columns[Coordinate::columnIndexFromString($cell->getColumn()) - 1] = $lookup[$key];
|
||||
}
|
||||
}
|
||||
if (count($columns) > $score) {
|
||||
$score = count($columns);
|
||||
$best = [$sheetIndex, $row->getRowIndex(), $columns];
|
||||
}
|
||||
}
|
||||
if (count($columns) > $score) {
|
||||
$best = [$rows, $rowIndex, $columns];
|
||||
$score = count($columns);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
$spreadsheet->disconnectWorksheets();
|
||||
}
|
||||
|
||||
return $best;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从指定行开始按块读取数据行。
|
||||
*
|
||||
* @return \Generator<int,array<int,mixed>> 行号(1 基)=> 列序号 => 单元格值
|
||||
*/
|
||||
private function rowsAfter(string $readerType, string $path, int $sheetIndex, int $firstRow): \Generator
|
||||
{
|
||||
for ($start = $firstRow; ; $start += self::CHUNK_ROWS) {
|
||||
$spreadsheet = $this->loadChunk($readerType, $path, $start, self::CHUNK_ROWS);
|
||||
|
||||
try {
|
||||
$worksheet = $spreadsheet->getSheet($sheetIndex);
|
||||
$highestRow = $worksheet->getHighestRow();
|
||||
if ($highestRow < $start) {
|
||||
return;
|
||||
}
|
||||
|
||||
$endRow = min($start + self::CHUNK_ROWS - 1, $highestRow);
|
||||
foreach ($worksheet->getRowIterator($start, $endRow) as $row) {
|
||||
$cells = [];
|
||||
foreach ($row->getCellIterator() as $cell) {
|
||||
$cells[Coordinate::columnIndexFromString($cell->getColumn()) - 1] = $cell->getValue();
|
||||
}
|
||||
|
||||
yield $row->getRowIndex() => $cells;
|
||||
}
|
||||
} finally {
|
||||
$spreadsheet->disconnectWorksheets();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 只把 [startRow, endRow] 这些行载入内存。 */
|
||||
private function loadChunk(string $readerType, string $path, int $startRow, int $rows): Spreadsheet
|
||||
{
|
||||
$reader = IOFactory::createReader($readerType);
|
||||
$reader->setReadDataOnly(true);
|
||||
$reader->setReadFilter(new class($startRow, $startRow + $rows - 1) implements IReadFilter {
|
||||
public function __construct(
|
||||
private readonly int $startRow,
|
||||
private readonly int $endRow,
|
||||
) {
|
||||
}
|
||||
|
||||
public function readCell($columnAddress, $row, $worksheetName = ''): bool
|
||||
{
|
||||
return $row >= $this->startRow && $row <= $this->endRow;
|
||||
}
|
||||
});
|
||||
|
||||
return $reader->load($path);
|
||||
}
|
||||
|
||||
private function header(mixed $value): string
|
||||
{
|
||||
return mb_strtolower(preg_replace('/[\s\x{FEFF}_\-()()]+/u', '', trim((string) $value)));
|
||||
|
||||
@ -5,6 +5,9 @@ namespace Tests\Feature\Business;
|
||||
use App\Models\Auth\Permission;
|
||||
use Illuminate\Http\UploadedFile;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Validation\ValidationException;
|
||||
use Modules\EcommerceFinance\Services\PayablesFinalImporter;
|
||||
use Modules\EcommerceFinance\Services\PayablesFinalRegistry;
|
||||
use Tests\Feature\FeatureTestCase;
|
||||
|
||||
class SupplierPayableFinalPagesTest extends FeatureTestCase
|
||||
@ -200,6 +203,63 @@ class SupplierPayableFinalPagesTest extends FeatureTestCase
|
||||
$this->assertStringNotContainsString('快递名称', $export->streamedContent());
|
||||
}
|
||||
|
||||
public function test_final_import_reads_files_larger_than_one_chunk(): void
|
||||
{
|
||||
$lines = ['月份,入库单号,采购单号,日期,供应商,货号,商品编码,规格名称,数量,单价,金额,预估单价,预估金额,供应商商品编码,备注,仓库名称,快递名称,快递单号,未发出,对账人'];
|
||||
for ($i = 1; $i <= 4500; $i++) {
|
||||
$lines[] = "2026-09,RK-{$i},CG-{$i},2026-09-08,供应商{$i},X-{$i},P-{$i},规格甲,2,10,20,9,18,SC-1,备注甲,华东仓,顺丰,SF-{$i},未发出,对账人甲";
|
||||
}
|
||||
|
||||
$file = UploadedFile::fake()->createWithContent(
|
||||
'payment-reconciliation.csv',
|
||||
"\xEF\xBB\xBF" . implode("\n", $lines) . "\n",
|
||||
);
|
||||
|
||||
$result = app(PayablesFinalImporter::class)->import(
|
||||
app(PayablesFinalRegistry::class)->fields('supplier-payables-final', 'payment-reconciliation'),
|
||||
'final:supplier:payment-reconciliation',
|
||||
$file,
|
||||
(int) $this->company->id,
|
||||
(int) $this->user->id,
|
||||
);
|
||||
|
||||
$this->assertSame(['created' => 4500, 'updated' => 0, 'rows' => 4500], $result);
|
||||
}
|
||||
|
||||
public function test_final_import_stops_at_the_row_limit_without_writing(): void
|
||||
{
|
||||
$lines = ['月份,入库单号,采购单号,日期,供应商,货号,商品编码,规格名称,数量,单价,金额,预估单价,预估金额,供应商商品编码,备注,仓库名称,快递名称,快递单号,未发出,对账人'];
|
||||
for ($i = 1; $i <= 10050; $i++) {
|
||||
$lines[] = "2026-09,RK-{$i},CG-{$i},2026-09-08,供应商{$i},X-{$i},P-{$i},规格甲,2,10,20,9,18,SC-1,备注甲,华东仓,顺丰,SF-{$i},未发出,对账人甲";
|
||||
}
|
||||
|
||||
$file = UploadedFile::fake()->createWithContent(
|
||||
'payment-reconciliation.csv',
|
||||
"\xEF\xBB\xBF" . implode("\n", $lines) . "\n",
|
||||
);
|
||||
|
||||
try {
|
||||
app(PayablesFinalImporter::class)->import(
|
||||
app(PayablesFinalRegistry::class)->fields('supplier-payables-final', 'payment-reconciliation'),
|
||||
'final:supplier:payment-reconciliation',
|
||||
$file,
|
||||
(int) $this->company->id,
|
||||
(int) $this->user->id,
|
||||
);
|
||||
$this->fail('超过 1 万行应当拒绝导入。');
|
||||
} catch (ValidationException $exception) {
|
||||
$this->assertSame(
|
||||
['单次最多导入 10,000 条记录,请拆分文件后重试。'],
|
||||
$exception->errors()['import'],
|
||||
);
|
||||
}
|
||||
|
||||
$this->assertSame(0, (int) DB::connection(config('database.supplier_payable_maintenance_connection'))
|
||||
->table('supplier_payable_overrides')
|
||||
->where('dataset', 'final:supplier:payment-reconciliation')
|
||||
->count());
|
||||
}
|
||||
|
||||
public function test_product_price_final_page_reuses_the_price_template_columns(): void
|
||||
{
|
||||
$this->loginAs()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user