$fields * @return array{token:string,total:int} */ public function prepare(UploadedFile $file, array $fields, string $storeDataset, int $companyId, ?int $userId): array { $this->cleanupStale(); $token = bin2hex(random_bytes(16)); $directory = $this->directory($token); if (! is_dir($directory) && ! mkdir($directory, 0770, true)) { throw ValidationException::withMessages(['import' => '无法创建导入任务目录。']); } $extension = strtolower((string) $file->getClientOriginalExtension()); $source = (string) $file->getRealPath(); $headerRow = 0; try { $reader = PayablesWorkbookStreamReader::forFile($source, $extension); $header = $this->locateHeader($reader, $fields); if ($header === null) { throw ValidationException::withMessages([ 'import' => '没有识别到表头,请先下载本页面的导入模板,按模板列填写后重试。', ]); } [$sheetIndex, $headerRow, $columns] = $header; $rows = 0; $handle = gzopen($directory . '/rows.jsonl.gz', 'wb1'); if ($handle === false) { throw ValidationException::withMessages(['import' => '无法写入导入缓存。']); } try { foreach ($reader->rows($sheetIndex, $headerRow) as $row) { $values = []; foreach ($columns as $columnIndex => $field) { $values[$field] = $row[$columnIndex] ?? null; } $rows++; if ($rows > self::MAX_ROWS) { throw ValidationException::withMessages([ 'import' => '单次最多导入 200,000 条记录,请拆分文件后重试。', ]); } gzwrite($handle, (string) json_encode($values, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE) . "\n"); } } finally { gzclose($handle); } if ($rows === 0) { throw ValidationException::withMessages(['import' => '表格中没有可导入的数据行。']); } } catch (Throwable $exception) { $this->delete($token); if ($exception instanceof ValidationException) { throw $exception; } throw ValidationException::withMessages([ 'import' => '无法读取导入文件,请确认文件未损坏且格式为 xlsx、xls 或 csv。', ]); } $this->writeMeta($token, [ 'dataset' => $storeDataset, 'company_id' => $companyId, 'user_id' => $userId, 'fields' => $fields, 'first_row' => $headerRow + 1, 'rows' => $rows, 'offset' => 0, 'created' => 0, 'updated' => 0, 'status' => 'ready', 'error' => null, 'started_at' => now()->toIso8601String(), ]); return ['token' => $token, 'total' => $rows]; } /** * 处理一片;返回最新进度。 * * @return array{offset:int,total:int,created:int,updated:int,done:bool} */ public function process(string $token, int $offset, ?int $maxRows = null, ?int $companyId = null): array { $meta = $this->readMeta($token); if ($meta === null) { throw ValidationException::withMessages(['import' => '导入任务不存在或已过期,请重新上传文件。']); } if ($companyId !== null && (int) $meta['company_id'] !== $companyId) { throw ValidationException::withMessages(['import' => '导入任务不存在或已过期,请重新上传文件。']); } if (($meta['status'] ?? '') === 'done') { return $this->progress($meta); } $total = (int) $meta['rows']; $firstRow = (int) ($meta['first_row'] ?? 2); $dataset = (string) $meta['dataset']; $companyId = (int) $meta['company_id']; $userId = isset($meta['user_id']) ? (int) $meta['user_id'] : null; $fields = is_array($meta['fields'] ?? null) ? $meta['fields'] : []; $sliceRows = max(1, $maxRows ?? self::SLICE_ROWS); $deadline = microtime(true) + self::SLICE_SECONDS; $created = (int) $meta['created']; $updated = (int) $meta['updated']; $line = 0; $processed = 0; $records = []; $offset = min(max($offset, 0), $total); $handle = gzopen($this->directory($token) . '/rows.jsonl.gz', 'rb'); if ($handle === false) { throw ValidationException::withMessages(['import' => '导入任务数据已损坏,请重新上传文件。']); } try { while ($line < $total && $processed < $sliceRows) { $raw = gzgets($handle); if ($raw === false) { break; } $line++; if ($line <= $offset) { continue; } $values = json_decode((string) rtrim($raw, "\r\n"), true); if (! is_array($values)) { $values = []; } if (collect($values)->every(static fn (mixed $value): bool => trim((string) $value) === '')) { continue; } try { $normalized = $this->normalizer->normalize($fields, $values); } catch (ValidationException $exception) { $this->fail($meta, $line - 1, $created, $updated, $this->rowMessage($firstRow, $line, $exception)); throw ValidationException::withMessages([ 'import' => $this->rowMessage($firstRow, $line, $exception), ]); } $key = $this->store->key($dataset, $normalized); if (isset($records[$key])) { $message = '第 ' . ($firstRow + $line - 1) . ' 行与前一行内容重复。'; $this->fail($meta, $line - 1, $created, $updated, $message); throw ValidationException::withMessages(['import' => $message]); } $records[$key] = $normalized; $processed++; if (count($records) >= self::FLUSH_ROWS) { $counts = $this->store->saveBatch($dataset, $records, $companyId, $userId, 'import'); $created += $counts['created']; $updated += $counts['updated']; $records = []; if (microtime(true) >= $deadline || $processed >= $sliceRows) { break; } } } if ($records !== []) { $counts = $this->store->saveBatch($dataset, $records, $companyId, $userId, 'import'); $created += $counts['created']; $updated += $counts['updated']; } } finally { gzclose($handle); } $done = $line >= $total; $meta['offset'] = $line; $meta['created'] = $created; $meta['updated'] = $updated; $meta['status'] = $done ? 'done' : 'running'; $meta['error'] = null; $this->writeMeta($token, $meta); if ($done) { $this->delete($token); } return $this->progress($meta); } /** @return array{offset:int,total:int,created:int,updated:int,done:bool} */ private function progress(array $meta): array { return [ 'offset' => (int) ($meta['offset'] ?? 0), 'total' => (int) ($meta['rows'] ?? 0), 'created' => (int) ($meta['created'] ?? 0), 'updated' => (int) ($meta['updated'] ?? 0), 'done' => ($meta['status'] ?? '') === 'done', ]; } private function rowMessage(int $firstRow, int $line, ValidationException $exception): string { return '第 ' . ($firstRow + $line - 1) . ' 行:' . collect($exception->errors())->flatten()->first(); } private function fail(array $meta, int $offset, int $created, int $updated, string $message): void { $meta['offset'] = $offset; $meta['created'] = $created; $meta['updated'] = $updated; $meta['status'] = 'failed'; $meta['error'] = $message; $this->writeMeta((string) $meta['token'], $meta); } /** * 前若干行里找出表头(列序号 => 字段),与旧版打分规则一致。 * * @param array $fields * @return array{0:int,1:int,2:array}|null */ private function locateHeader(PayablesWorkbookStreamReader $reader, array $fields): ?array { $lookup = ['记录标识' => '_action_key', 'recordkey' => '_action_key', '_action_key' => '_action_key']; foreach ($fields as $field => $definition) { $lookup[$this->header($field)] = $field; $lookup[$this->header((string) $definition['label'])] = $field; } $best = null; $score = 0; foreach (array_keys($reader->sheetNames()) as $sheetIndex) { foreach ($reader->rows($sheetIndex, null, 20) as $rowNumber => $row) { $columns = []; foreach ($row as $columnIndex => $value) { $key = $this->header($value); if ($key !== '' && isset($lookup[$key])) { $columns[$columnIndex] = $lookup[$key]; } } if (count($columns) > $score) { $score = count($columns); $best = [$sheetIndex, $rowNumber, $columns]; } } } return $best; } private function header(mixed $value): string { return mb_strtolower((string) preg_replace('/[\s\x{FEFF}_\-()()]+/u', '', trim((string) $value))); } private function directory(string $token): string { return $this->storagePath() . '/' . $token; } private function metaPath(string $token): string { return $this->directory($token) . '/meta.json'; } /** @return array|null */ private function readMeta(string $token): ?array { if (preg_match('/^[a-f0-9]{32}$/', $token) !== 1) { return null; } $path = $this->metaPath($token); if (! is_file($path)) { return null; } $meta = json_decode((string) file_get_contents($path), true); if (! is_array($meta)) { return null; } return $meta + ['token' => $token]; } private function writeMeta(string $token, array $meta): void { $meta['token'] = $token; $meta['updated_at'] = now()->toIso8601String(); file_put_contents($this->metaPath($token), (string) json_encode($meta, JSON_UNESCAPED_UNICODE)); } public function delete(string $token): void { if (preg_match('/^[a-f0-9]{32}$/', $token) !== 1) { return; } File::deleteDirectory($this->directory($token)); } /** 清理过期任务(超过 24 小时未完成)。 */ private function cleanupStale(): void { $root = $this->storagePath(); if (! is_dir($root)) { return; } $deadline = time() - self::STALE_HOURS * 3600; foreach (File::directories($root) as $directory) { $metaPath = $directory . '/meta.json'; $modified = is_file($metaPath) ? (int) filemtime($metaPath) : (int) filemtime($directory); if ($modified < $deadline) { $this->delete(basename($directory)); } } } }