diff --git a/app/Http/Controllers/Business/PayablesFinalActions.php b/app/Http/Controllers/Business/PayablesFinalActions.php index c8112f22..67908c1d 100644 --- a/app/Http/Controllers/Business/PayablesFinalActions.php +++ b/app/Http/Controllers/Business/PayablesFinalActions.php @@ -7,6 +7,7 @@ use Illuminate\Http\RedirectResponse; use Illuminate\Http\Request; use Illuminate\Routing\Controller; use Modules\EcommerceFinance\Services\PayablesFinalImporter; +use Modules\EcommerceFinance\Services\PayablesFinalImportRunner; use Modules\EcommerceFinance\Services\PayablesFinalRegistry; use Modules\EcommerceFinance\Services\PayablesFieldNormalizer; use Modules\EcommerceFinance\Services\SupplierPayableImportTemplate; @@ -70,6 +71,38 @@ class PayablesFinalActions extends Controller : back()->with('success', $message); } + /** 大文件导入第 1 步：上传文件并建立行缓存（前端随后按片拉取处理）。 */ + public function importUpload(Request $request, PayablesFinalImportRunner $runner): JsonResponse + { + $request->validate(['import' => ['required', 'file', 'mimes:xlsx,xls,csv', 'max:51200']]); + + $prepared = $runner->prepare( + $request->file('import'), + $this->fields($request), + $this->storeDataset($request), + (int) (company_id() ?: 1), + user_id() ? (int) user_id() : null, + ); + + return response()->json($prepared + ['offset' => 0, 'created' => 0, 'updated' => 0, 'done' => false]); + } + + /** 大文件导入第 2 步起：每次处理一片，前端拿到进度后继续。 */ + public function importChunk(Request $request, PayablesFinalImportRunner $runner): JsonResponse + { + $data = $request->validate([ + 'token' => ['required', 'string', 'regex:/^[a-f0-9]{32}$/'], + 'offset' => ['required', 'integer', 'min:0'], + ]); + + return response()->json($runner->process( + $data['token'], + (int) $data['offset'], + null, + (int) (company_id() ?: 1), + )); + } + public function update(Request $request, string $recordKey): JsonResponse|RedirectResponse { $values = $this->normalizer->normalize($this->fields($request), (array) $request->input('values', [])); diff --git a/deploy/php.ini b/deploy/php.ini index 90a3c984..bd38bb36 100644 --- a/deploy/php.ini +++ b/deploy/php.ini @@ -1,7 +1,7 @@ memory_limit=512M max_execution_time=120 post_max_size=64M -upload_max_filesize=10M +upload_max_filesize=64M # 线上必须把错误写进日志而不是显示给用户；否则致命错误（内存/超时）会既看不到、也无处查。 display_errors=Off log_errors=On diff --git a/modules/EcommerceFinance/Services/PayablesFinalImportRunner.php b/modules/EcommerceFinance/Services/PayablesFinalImportRunner.php new file mode 100644 index 00000000..c574bd39 --- /dev/null +++ b/modules/EcommerceFinance/Services/PayablesFinalImportRunner.php @@ -0,0 +1,380 @@ + $fields + * @return array{token:string,total:int} + */ + public function prepare(UploadedFile $file, array $fields, string $storeDataset, int $companyId, ?int $userId): array + { + $this->cleanupStale(); + + $token = bin2hex(random_bytes(16)); + $directory = $this->directory($token); + if (! is_dir($directory) && ! mkdir($directory, 0770, true)) { + throw ValidationException::withMessages(['import' => '无法创建导入任务目录。']); + } + + $extension = strtolower((string) $file->getClientOriginalExtension()); + $source = (string) $file->getRealPath(); + + $headerRow = 0; + + try { + $reader = PayablesWorkbookStreamReader::forFile($source, $extension); + $header = $this->locateHeader($reader, $fields); + if ($header === null) { + throw ValidationException::withMessages([ + 'import' => '没有识别到表头，请先下载本页面的导入模板，按模板列填写后重试。', + ]); + } + + [$sheetIndex, $headerRow, $columns] = $header; + + $rows = 0; + $handle = gzopen($directory . '/rows.jsonl.gz', 'wb1'); + if ($handle === false) { + throw ValidationException::withMessages(['import' => '无法写入导入缓存。']); + } + + try { + foreach ($reader->rows($sheetIndex, $headerRow) as $row) { + $values = []; + foreach ($columns as $columnIndex => $field) { + $values[$field] = $row[$columnIndex] ?? null; + } + + $rows++; + if ($rows > self::MAX_ROWS) { + throw ValidationException::withMessages([ + 'import' => '单次最多导入 200,000 条记录，请拆分文件后重试。', + ]); + } + + gzwrite($handle, (string) json_encode($values, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE) . "\n"); + } + } finally { + gzclose($handle); + } + + if ($rows === 0) { + throw ValidationException::withMessages(['import' => '表格中没有可导入的数据行。']); + } + } catch (Throwable $exception) { + $this->delete($token); + if ($exception instanceof ValidationException) { + throw $exception; + } + + throw ValidationException::withMessages([ + 'import' => '无法读取导入文件，请确认文件未损坏且格式为 xlsx、xls 或 csv。', + ]); + } + + $this->writeMeta($token, [ + 'dataset' => $storeDataset, + 'company_id' => $companyId, + 'user_id' => $userId, + 'fields' => $fields, + 'first_row' => $headerRow + 1, + 'rows' => $rows, + 'offset' => 0, + 'created' => 0, + 'updated' => 0, + 'status' => 'ready', + 'error' => null, + 'started_at' => now()->toIso8601String(), + ]); + + return ['token' => $token, 'total' => $rows]; + } + + /** + * 处理一片；返回最新进度。 + * + * @return array{offset:int,total:int,created:int,updated:int,done:bool} + */ + public function process(string $token, int $offset, ?int $maxRows = null, ?int $companyId = null): array + { + $meta = $this->readMeta($token); + if ($meta === null) { + throw ValidationException::withMessages(['import' => '导入任务不存在或已过期，请重新上传文件。']); + } + + if ($companyId !== null && (int) $meta['company_id'] !== $companyId) { + throw ValidationException::withMessages(['import' => '导入任务不存在或已过期，请重新上传文件。']); + } + + if (($meta['status'] ?? '') === 'done') { + return $this->progress($meta); + } + + $total = (int) $meta['rows']; + $firstRow = (int) ($meta['first_row'] ?? 2); + $dataset = (string) $meta['dataset']; + $companyId = (int) $meta['company_id']; + $userId = isset($meta['user_id']) ? (int) $meta['user_id'] : null; + $fields = is_array($meta['fields'] ?? null) ? $meta['fields'] : []; + $sliceRows = max(1, $maxRows ?? self::SLICE_ROWS); + $deadline = microtime(true) + self::SLICE_SECONDS; + + $created = (int) $meta['created']; + $updated = (int) $meta['updated']; + $line = 0; + $processed = 0; + $records = []; + $offset = min(max($offset, 0), $total); + + $handle = gzopen($this->directory($token) . '/rows.jsonl.gz', 'rb'); + if ($handle === false) { + throw ValidationException::withMessages(['import' => '导入任务数据已损坏，请重新上传文件。']); + } + + try { + while ($line < $total && $processed < $sliceRows) { + $raw = gzgets($handle); + if ($raw === false) { + break; + } + $line++; + + if ($line <= $offset) { + continue; + } + + $values = json_decode((string) rtrim($raw, "\r\n"), true); + if (! is_array($values)) { + $values = []; + } + + if (collect($values)->every(static fn (mixed $value): bool => trim((string) $value) === '')) { + continue; + } + + try { + $normalized = $this->normalizer->normalize($fields, $values); + } catch (ValidationException $exception) { + $this->fail($meta, $line - 1, $created, $updated, $this->rowMessage($firstRow, $line, $exception)); + throw ValidationException::withMessages([ + 'import' => $this->rowMessage($firstRow, $line, $exception), + ]); + } + + $key = $this->store->key($dataset, $normalized); + if (isset($records[$key])) { + $message = '第 ' . ($firstRow + $line - 1) . ' 行与前一行内容重复。'; + $this->fail($meta, $line - 1, $created, $updated, $message); + throw ValidationException::withMessages(['import' => $message]); + } + + $records[$key] = $normalized; + $processed++; + + if (count($records) >= self::FLUSH_ROWS) { + $counts = $this->store->saveBatch($dataset, $records, $companyId, $userId, 'import'); + $created += $counts['created']; + $updated += $counts['updated']; + $records = []; + + if (microtime(true) >= $deadline || $processed >= $sliceRows) { + break; + } + } + } + + if ($records !== []) { + $counts = $this->store->saveBatch($dataset, $records, $companyId, $userId, 'import'); + $created += $counts['created']; + $updated += $counts['updated']; + } + } finally { + gzclose($handle); + } + + $done = $line >= $total; + $meta['offset'] = $line; + $meta['created'] = $created; + $meta['updated'] = $updated; + $meta['status'] = $done ? 'done' : 'running'; + $meta['error'] = null; + $this->writeMeta($token, $meta); + + if ($done) { + $this->delete($token); + } + + return $this->progress($meta); + } + + /** @return array{offset:int,total:int,created:int,updated:int,done:bool} */ + private function progress(array $meta): array + { + return [ + 'offset' => (int) ($meta['offset'] ?? 0), + 'total' => (int) ($meta['rows'] ?? 0), + 'created' => (int) ($meta['created'] ?? 0), + 'updated' => (int) ($meta['updated'] ?? 0), + 'done' => ($meta['status'] ?? '') === 'done', + ]; + } + + private function rowMessage(int $firstRow, int $line, ValidationException $exception): string + { + return '第 ' . ($firstRow + $line - 1) . ' 行：' . collect($exception->errors())->flatten()->first(); + } + + private function fail(array $meta, int $offset, int $created, int $updated, string $message): void + { + $meta['offset'] = $offset; + $meta['created'] = $created; + $meta['updated'] = $updated; + $meta['status'] = 'failed'; + $meta['error'] = $message; + $this->writeMeta((string) $meta['token'], $meta); + } + + /** + * 前若干行里找出表头（列序号 => 字段），与旧版打分规则一致。 + * + * @param array $fields + * @return array{0:int,1:int,2:array}|null + */ + private function locateHeader(PayablesWorkbookStreamReader $reader, array $fields): ?array + { + $lookup = ['记录标识' => '_action_key', 'recordkey' => '_action_key', '_action_key' => '_action_key']; + foreach ($fields as $field => $definition) { + $lookup[$this->header($field)] = $field; + $lookup[$this->header((string) $definition['label'])] = $field; + } + + $best = null; + $score = 0; + foreach (array_keys($reader->sheetNames()) as $sheetIndex) { + foreach ($reader->rows($sheetIndex, null, 20) as $rowNumber => $row) { + $columns = []; + foreach ($row as $columnIndex => $value) { + $key = $this->header($value); + if ($key !== '' && isset($lookup[$key])) { + $columns[$columnIndex] = $lookup[$key]; + } + } + + if (count($columns) > $score) { + $score = count($columns); + $best = [$sheetIndex, $rowNumber, $columns]; + } + } + } + + return $best; + } + + private function header(mixed $value): string + { + return mb_strtolower((string) preg_replace('/[\s\x{FEFF}_\-（）()]+/u', '', trim((string) $value))); + } + + private function directory(string $token): string + { + return $this->storagePath() . '/' . $token; + } + + private function metaPath(string $token): string + { + return $this->directory($token) . '/meta.json'; + } + + /** @return array|null */ + private function readMeta(string $token): ?array + { + if (preg_match('/^[a-f0-9]{32}$/', $token) !== 1) { + return null; + } + + $path = $this->metaPath($token); + if (! is_file($path)) { + return null; + } + + $meta = json_decode((string) file_get_contents($path), true); + if (! is_array($meta)) { + return null; + } + + return $meta + ['token' => $token]; + } + + private function writeMeta(string $token, array $meta): void + { + $meta['token'] = $token; + $meta['updated_at'] = now()->toIso8601String(); + file_put_contents($this->metaPath($token), (string) json_encode($meta, JSON_UNESCAPED_UNICODE)); + } + + public function delete(string $token): void + { + if (preg_match('/^[a-f0-9]{32}$/', $token) !== 1) { + return; + } + + File::deleteDirectory($this->directory($token)); + } + + /** 清理过期任务（超过 24 小时未完成）。 */ + private function cleanupStale(): void + { + $root = $this->storagePath(); + if (! is_dir($root)) { + return; + } + + $deadline = time() - self::STALE_HOURS * 3600; + foreach (File::directories($root) as $directory) { + $metaPath = $directory . '/meta.json'; + $modified = is_file($metaPath) ? (int) filemtime($metaPath) : (int) filemtime($directory); + if ($modified < $deadline) { + $this->delete(basename($directory)); + } + } + } +} diff --git a/modules/EcommerceFinance/Services/PayablesWorkbookStreamReader.php b/modules/EcommerceFinance/Services/PayablesWorkbookStreamReader.php new file mode 100644 index 00000000..07c29253 --- /dev/null +++ b/modules/EcommerceFinance/Services/PayablesWorkbookStreamReader.php @@ -0,0 +1,564 @@ +|null 工作表内部路径（xlsx） */ + private ?array $sheetPaths = null; + + /** @var list|null 日期样式的 style 索引 */ + private ?array $dateStyles = null; + + /** @var string|null 共享字符串拼接串 */ + private ?string $sharedStrings = null; + + /** @var SplFixedArray|null 共享字符串偏移（第 i 个字符串的起始位置） */ + private ?SplFixedArray $sharedStringOffsets = null; + + public function __construct( + private readonly string $path, + private readonly string $extension, + ) { + } + + /** 新建读取器（按扩展名）。 */ + public static function forFile(string $path, string $extension): self + { + return new self($path, strtolower($extension)); + } + + /** @return list 工作表名称（xls 走 PhpSpreadsheet，仅取名称） */ + public function sheetNames(): array + { + if ($this->extension === 'xlsx') { + return array_map( + static fn (array $sheet): string => $sheet['name'], + $this->xlsxSheets(), + ); + } + + if ($this->extension === 'csv') { + return ['Sheet1']; + } + + $spreadsheet = IOFactory::createReader('Xls')->load($this->path); + try { + return $spreadsheet->getSheetNames(); + } finally { + $spreadsheet->disconnectWorksheets(); + } + } + + /** + * 逐行读取指定工作表。 + * + * @param int|null $skipUntilRow 跳过该行号之前的行（含），用于分片续读 + * @param int|null $stopAfterRow 读到该行号后停止 + * @return Generator> 行号（1 基）=> 列序号（0 基）=> 值 + */ + public function rows(int $sheetIndex = 0, ?int $skipUntilRow = null, ?int $stopAfterRow = null): Generator + { + yield from match ($this->extension) { + 'xlsx' => $this->xlsxRows($sheetIndex, $skipUntilRow, $stopAfterRow), + 'csv' => $this->csvRows($skipUntilRow, $stopAfterRow), + default => $this->xlsRows($sheetIndex, $skipUntilRow, $stopAfterRow), + }; + } + + /** @return Generator> */ + private function xlsxRows(int $sheetIndex, ?int $skipUntilRow, ?int $stopAfterRow): Generator + { + $path = $this->sheetPaths()[$sheetIndex] ?? null; + if ($path === null) { + return; + } + + $zip = new ZipArchive(); + if ($zip->open($this->path) !== true) { + throw new RuntimeException('无法打开工作簿。'); + } + + try { + $stream = $zip->getStream($path); + if ($stream === false) { + return; + } + + $reader = new XMLReader(); + $reader->XML(stream_get_contents($stream), null, LIBXML_NONET); + fclose($stream); + + $dictionary = $this->xlsxSharedStrings(); + $rowNumber = 0; + + while ($reader->read()) { + if ($reader->nodeType !== XMLReader::ELEMENT || $reader->localName !== 'row') { + continue; + } + + $rowNumber = (int) ($reader->getAttribute('r') ?: $rowNumber + 1); + if ($stopAfterRow !== null && $rowNumber > $stopAfterRow) { + return; + } + + if ($skipUntilRow !== null && $rowNumber <= $skipUntilRow) { + $reader->next(); // 跳过整行 + continue; + } + + yield $rowNumber => $this->readRow($reader, $dictionary); + } + } finally { + $zip->close(); + } + } + + /** + * 读一行里的单元格（当前节点位于 ）。 + * + * @param array{0:string,1:SplFixedArray} $dictionary + * @return array + */ + private function readRow(XMLReader $reader, array $dictionary): array + { + $cells = []; + $depth = $reader->depth; + $empty = $reader->isEmptyElement; + if ($empty) { + return $cells; + } + + while ($reader->read()) { + if ($reader->nodeType === XMLReader::END_ELEMENT && $reader->depth === $depth) { + break; + } + if ($reader->nodeType !== XMLReader::ELEMENT || $reader->localName !== 'c' || $reader->isEmptyElement) { + continue; + } + + $reference = (string) $reader->getAttribute('r'); + $type = (string) $reader->getAttribute('t'); + $styleIndex = (string) $reader->getAttribute('s'); + $columnIndex = $reference !== '' ? $this->columnIndex($reference) : null; + $value = $this->readCellValue($reader, $type, $dictionary); + $columnIndex ??= count($cells); + + if ($value === null || $value === '') { + continue; + } + + $cells[$columnIndex] = $this->formatValue($value, $type, $styleIndex); + } + + return $cells; + } + + /** + * @param array{0:string,1:SplFixedArray} $dictionary + */ + private function readCellValue(XMLReader $reader, string $type, array $dictionary): mixed + { + $value = null; + $depth = $reader->depth; + + while ($reader->read()) { + if ($reader->nodeType === XMLReader::END_ELEMENT && $reader->depth === $depth) { + break; + } + if ($reader->nodeType !== XMLReader::ELEMENT) { + continue; + } + + if ($reader->localName === 'v') { + $value = $reader->isEmptyElement ? '' : $reader->readString(); + continue; + } + + if ($reader->localName === 'is') { + $value = $this->readInlineString($reader); + continue; + } + + if ($reader->localName === 't' && $type === 'inlineStr') { + $value = $reader->isEmptyElement ? '' : $reader->readString(); + } + } + + if ($type === 's') { + $index = (int) $value; + [$blob, $offsets] = $dictionary; + if ($index < 0 || $index + 1 >= $offsets->getSize()) { + return null; + } + + return substr($blob, $offsets[$index], $offsets[$index + 1] - $offsets[$index]); + } + + if ($type === 'str' || $type === 'inlineStr') { + return $value; + } + + if ($type === 'b') { + return $value === '1' ? '1' : '0'; + } + + return $value; + } + + /** 富文本 里可能有多个 。 */ + private function readInlineString(XMLReader $reader): string + { + $depth = $reader->depth; + $text = ''; + + while ($reader->read()) { + if ($reader->nodeType === XMLReader::END_ELEMENT && $reader->depth === $depth) { + break; + } + if ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 't' && ! $reader->isEmptyElement) { + $text .= $reader->readString(); + } + } + + return $text; + } + + /** 日期样式的数字单元格转成日期字符串。 */ + private function formatValue(mixed $value, string $type, string $styleIndex): mixed + { + if (! is_string($value) || $value === '' || $styleIndex === '' || ! is_numeric($value)) { + return $value; + } + + if (! in_array((int) $styleIndex, $this->dateStyles(), true)) { + return $value; + } + + $serial = (float) $value; + $date = ExcelDate::excelToDateTimeObject($serial); + + return fmod($serial, 1.0) === 0.0 ? $date->format('Y-m-d') : $date->format('Y-m-d H:i:s'); + } + + /** @return Generator> */ + private function csvRows(?int $skipUntilRow, ?int $stopAfterRow): Generator + { + $handle = fopen($this->path, 'rb'); + if ($handle === false) { + return; + } + + try { + $delimiter = $this->detectDelimiter((string) fgets($handle)); + rewind($handle); + + $line = 0; + while (($row = fgetcsv($handle, 0, $delimiter)) !== false) { + $line++; + if ($stopAfterRow !== null && $line > $stopAfterRow) { + return; + } + if ($skipUntilRow !== null && $line <= $skipUntilRow) { + continue; + } + + $cells = []; + foreach ($row as $index => $value) { + $value = $this->normalizeEncoding((string) $value); + if ($index === 0) { + $value = preg_replace('/^\x{FEFF}/u', '', $value) ?? $value; + } + if ($value !== '') { + $cells[$index] = $value; + } + } + + yield $line => $cells; + } + } finally { + fclose($handle); + } + } + + /** @return Generator> */ + private function xlsRows(int $sheetIndex, ?int $skipUntilRow, ?int $stopAfterRow): Generator + { + $spreadsheet = IOFactory::createReader('Xls')->load($this->path); + try { + $sheet = $spreadsheet->getSheet($sheetIndex); + foreach ($sheet->getRowIterator() as $row) { + $line = $row->getRowIndex(); + if ($stopAfterRow !== null && $line > $stopAfterRow) { + return; + } + if ($skipUntilRow !== null && $line <= $skipUntilRow) { + continue; + } + + $cells = []; + foreach ($row->getCellIterator() as $cell) { + $value = $cell->getValue(); + if ($value !== null && $value !== '') { + $cells[$this->columnIndex($cell->getColumn())] = $value; + } + } + + yield $line => $cells; + } + } finally { + $spreadsheet->disconnectWorksheets(); + } + } + + /** @return list 日期样式的 style 索引 */ + private function dateStyles(): array + { + if ($this->dateStyles !== null) { + return $this->dateStyles; + } + + if ($this->extension !== 'xlsx') { + return $this->dateStyles = []; + } + + $zip = new ZipArchive(); + if ($zip->open($this->path) !== true) { + return $this->dateStyles = []; + } + + $styles = []; + try { + $stylesXml = $zip->getFromName('xl/styles.xml'); + if ($stylesXml === false) { + return $this->dateStyles = []; + } + + $reader = new XMLReader(); + $reader->XML($stylesXml, null, LIBXML_NONET); + + $customDateFormats = []; + $cellXfs = []; + $inCellXfs = false; + + while ($reader->read()) { + if ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 'numFmt') { + $id = (int) $reader->getAttribute('numFmtId'); + $code = (string) $reader->getAttribute('formatCode'); + if ($id >= 164 && $this->looksLikeDateFormat($code)) { + $customDateFormats[$id] = true; + } + continue; + } + + if ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 'cellXfs') { + $inCellXfs = true; + continue; + } + + if ($reader->nodeType === XMLReader::END_ELEMENT && $reader->localName === 'cellXfs') { + $inCellXfs = false; + continue; + } + + if ($inCellXfs && $reader->nodeType === XMLReader::ELEMENT && $reader->localName === 'xf') { + $cellXfs[] = (int) $reader->getAttribute('numFmtId'); + } + } + + foreach ($cellXfs as $index => $numFmtId) { + if (($numFmtId >= 14 && $numFmtId <= 22) || ($numFmtId >= 45 && $numFmtId <= 47) || isset($customDateFormats[$numFmtId])) { + $styles[] = $index; + } + } + + $reader->close(); + } finally { + $zip->close(); + } + + return $this->dateStyles = $styles; + } + + private function looksLikeDateFormat(string $code): bool + { + $code = preg_replace('/"[^"]*"/', '', $code) ?? $code; + $code = preg_replace('/\[[^\]]*\]/', '', $code) ?? $code; + + return (bool) preg_match('/[ymdhs]/i', str_replace(['AM/PM', 'am/pm'], '', $code)); + } + + /** @return list */ + private function xlsxSheets(): array + { + if ($this->sheetPaths !== null) { + return $this->sheetPaths; + } + + $zip = new ZipArchive(); + if ($zip->open($this->path) !== true) { + throw new RuntimeException('无法打开工作簿。'); + } + + $sheets = []; + try { + $workbook = $zip->getFromName('xl/workbook.xml'); + $relationships = $zip->getFromName('xl/_rels/workbook.xml.rels'); + if ($workbook === false || $relationships === false) { + throw new RuntimeException('工作簿结构不完整。'); + } + + $targets = []; + $rels = new XMLReader(); + $rels->XML($relationships, null, LIBXML_NONET); + while ($rels->read()) { + if ($rels->nodeType === XMLReader::ELEMENT && $rels->localName === 'Relationship') { + $targets[(string) $rels->getAttribute('Id')] = ltrim((string) $rels->getAttribute('Target'), '/'); + } + } + $rels->close(); + + $book = new XMLReader(); + $book->XML($workbook, null, LIBXML_NONET); + while ($book->read()) { + if ($book->nodeType !== XMLReader::ELEMENT || $book->localName !== 'sheet') { + continue; + } + $relId = (string) ($book->getAttributeNs('id', 'http://schemas.openxmlformats.org/officeDocument/2006/relationships') ?? ''); + $target = $targets[$relId] ?? ('worksheets/sheet' . (count($sheets) + 1) . '.xml'); + $sheets[] = [ + 'name' => (string) $book->getAttribute('name'), + 'path' => str_starts_with($target, 'xl/') ? $target : 'xl/' . $target, + ]; + } + $book->close(); + } finally { + $zip->close(); + } + + return $this->sheetPaths = $sheets; + } + + /** @return list */ + private function sheetPaths(): array + { + return array_column($this->xlsxSheets(), 'path'); + } + + /** @return array{0:string,1:SplFixedArray} */ + private function xlsxSharedStrings(): array + { + if ($this->sharedStrings !== null && $this->sharedStringOffsets !== null) { + return [$this->sharedStrings, $this->sharedStringOffsets]; + } + + $zip = new ZipArchive(); + if ($zip->open($this->path) !== true) { + throw new RuntimeException('无法打开工作簿。'); + } + + $blob = ''; + try { + $xml = $zip->getFromName('xl/sharedStrings.xml'); + if ($xml === false) { + return [$this->sharedStrings = '', $this->sharedStringOffsets = new SplFixedArray(1)]; + } + + $reader = new XMLReader(); + $reader->XML($xml, null, LIBXML_NONET); + + $count = 0; + $offsets = new SplFixedArray(1024); + $offsets[0] = 0; + $depth = 0; + $collect = false; + $text = ''; + + while ($reader->read()) { + if ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 'si') { + $collect = true; + $text = ''; + $depth = $reader->depth; + continue; + } + + if (! $collect) { + continue; + } + + if ($reader->nodeType === XMLReader::ELEMENT && $reader->localName === 't' && ! $reader->isEmptyElement) { + $text .= $reader->readString(); + continue; + } + + if ($reader->nodeType === XMLReader::END_ELEMENT && $reader->localName === 'si' && $reader->depth === $depth) { + $blob .= $text; + $count++; + if ($count + 1 > $offsets->getSize()) { + $offsets->setSize($offsets->getSize() * 2); + } + $offsets[$count] = strlen($blob); + $collect = false; + } + } + + $reader->close(); + $offsets->setSize($count + 1); + } finally { + $zip->close(); + } + + return [$this->sharedStrings = $blob, $this->sharedStringOffsets = $offsets]; + } + + private function detectDelimiter(string $line): string + { + $counts = [ + ',' => substr_count($line, ','), + ';' => substr_count($line, ';'), + "\t" => substr_count($line, "\t"), + ]; + + arsort($counts); + $delimiter = (string) array_key_first($counts); + + return $counts[$delimiter] > 0 ? $delimiter : ','; + } + + private function normalizeEncoding(string $value): string + { + if ($value === '' || mb_check_encoding($value, 'UTF-8')) { + return $value; + } + + return mb_convert_encoding($value, 'UTF-8', 'GBK'); + } + + private function columnIndex(string $reference): int + { + preg_match('/^([A-Z]+)/i', $reference, $matches); + + $letters = strtoupper($matches[1] ?? 'A'); + $index = 0; + foreach (str_split($letters) as $letter) { + $index = $index * 26 + (ord($letter) - 64); + } + + return $index - 1; + } +} diff --git a/modules/EcommerceFinance/Services/SupplierPayableOverrideStore.php b/modules/EcommerceFinance/Services/SupplierPayableOverrideStore.php index eff299b5..fbe190c6 100644 --- a/modules/EcommerceFinance/Services/SupplierPayableOverrideStore.php +++ b/modules/EcommerceFinance/Services/SupplierPayableOverrideStore.php @@ -95,11 +95,91 @@ class SupplierPayableOverrideStore return ['created' => $existing ? 0 : 1, 'updated' => $existing ? 1 : 0]; } + /** + * 批量保存（大文件导入用）：按记录键 upsert，并批量写审计事件。 + * + * @param array> $records 记录键 => 字段值 + * @return array{created:int,updated:int} + */ + public function saveBatch(string $dataset, array $records, int $companyId, ?int $userId, string $source = 'import'): array + { + if ($records === []) { + return ['created' => 0, 'updated' => 0]; + } + + $connection = $this->connection(); + $now = now(); + $created = 0; + $updated = 0; + + $connection->transaction(function () use ($connection, $dataset, $records, $companyId, $userId, $source, $now, &$created, &$updated): void { + $existing = []; + foreach (array_chunk(array_keys($records), 1000) as $keys) { + foreach ($connection->table('supplier_payable_overrides') + ->where('company_id', $companyId) + ->where('dataset', $dataset) + ->whereIn('record_key', $keys) + ->get() as $row) { + $existing[(string) $row->record_key] = $row; + } + } + + $upserts = []; + $events = []; + foreach ($records as $key => $values) { + $before = $existing[$key] ?? null; + $before === null ? $created++ : $updated++; + $encoded = $this->encodeValues($values); + + $upserts[] = [ + 'company_id' => $companyId, + 'dataset' => $dataset, + 'record_key' => $key, + 'values' => $encoded, + 'created_by' => $before->created_by ?? $userId, + 'updated_by' => $userId, + 'created_at' => $before->created_at ?? $now, + 'updated_at' => $now, + ]; + $events[] = [ + 'company_id' => $companyId, + 'dataset' => $dataset, + 'record_key' => $key, + 'source' => $source, + 'before_values' => $before === null ? null : (string) $before->values, + 'after_values' => $encoded, + 'user_id' => $userId, + 'created_at' => $now, + ]; + } + + foreach (array_chunk($upserts, 500) as $chunk) { + $connection->table('supplier_payable_overrides')->upsert( + $chunk, + ['company_id', 'dataset', 'record_key'], + ['values', 'updated_by', 'updated_at'], + ); + } + + foreach (array_chunk($events, 500) as $chunk) { + $connection->table('supplier_payable_override_events')->insert($chunk); + } + }); + + return ['created' => $created, 'updated' => $updated]; + } + public function transaction(Closure $callback): mixed { return $this->connection()->transaction($callback); } + /** 值编码为 JSON；脏字节（非法 UTF-8）用替代字符，避免整批写入失败。 */ + private function encodeValues(array $values): string + { + return (string) json_encode($values, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE); + } + /** * 列出某个数据集已经保存的全部记录（「终版」目录的独立数据就是这些行）。 * diff --git a/resources/views/business/pages/partials/final-dataset-table.blade.php b/resources/views/business/pages/partials/final-dataset-table.blade.php index 77f36f7b..781ec6aa 100644 --- a/resources/views/business/pages/partials/final-dataset-table.blade.php +++ b/resources/views/business/pages/partials/final-dataset-table.blade.php @@ -71,6 +71,8 @@ @include('business.pages.supplier-payables.maintenance-import-modal', [ 'supplierImportAction' => $finalActions . '/import', 'supplierTemplateUrl' => $finalActions . '/template', + 'supplierImportUploadUrl' => $finalActions . '/import-upload', + 'supplierImportChunkUrl' => $finalActions . '/import-chunk', ])
diff --git a/resources/views/business/pages/supplier-payables/maintenance-import-modal.blade.php b/resources/views/business/pages/supplier-payables/maintenance-import-modal.blade.php index bbe3872d..b2896976 100644 --- a/resources/views/business/pages/supplier-payables/maintenance-import-modal.blade.php +++ b/resources/views/business/pages/supplier-payables/maintenance-import-modal.blade.php @@ -5,7 +5,9 @@

导入数据

-
+ @csrf
@@ -17,6 +19,10 @@ 选择导入文件 +