CaiWuXiTong/modules/EcommerceFinance/Services/PinduoduoSettlementBillImporter.php

716 lines
28 KiB
PHP
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<?php
namespace Modules\EcommerceFinance\Services;
use DateTimeImmutable;
use Illuminate\Database\ConnectionInterface;
use Illuminate\Database\DatabaseManager;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Schema;
use Illuminate\Support\Str;
use RuntimeException;
use SimpleXMLElement;
use Throwable;
use XMLReader;
use ZipArchive;
class PinduoduoSettlementBillImporter
{
public const ACCOUNT_NAME = '拼多多货款账户';
private const IMPORT_TABLE = 'pdd_settlement_bill_imports';
private const STAGING_TABLE = 'pdd_settlement_bill_staging';
private const ROW_TABLE = 'pdd_settlement_bill_rows';
private const HEADER_ALIASES = [
'record_code' => ['记录编码'],
'store_name' => ['店铺', '原始店铺'],
'custom_remark' => ['自定义备注'],
'bill_item' => ['账单项目'],
'merchant_order_no' => ['商户订单号'],
'online_order_no' => ['线上单号'],
'business_type' => ['财务类型', '业务类型'],
'business_description' => ['业务描述'],
'account_name' => ['类型', '账户'],
'occurred_at' => ['发生时间'],
'income_amount' => ['收入金额'],
'expense_amount' => ['支出金额'],
'signed_amount' => ['金额'],
'remark' => ['备注'],
'source_updated_at' => ['修改时间'],
'source_modified_by' => ['修改人'],
];
private const OPTIONAL_HEADERS = ['signed_amount'];
private readonly ConnectionInterface $connection;
public function __construct(DatabaseManager $databases)
{
$this->connection = $databases->connection(
(string) config('upstream_mirrors.connection', 'business_mirror'),
);
}
/**
* @return array{status:string,row_count:int,excluded_account_rows:int,store_count:int,bill_item_count:int,account_type_counts:array<string,int>,income_total:string,expense_total:string,net_total:string,period_start:string,period_end:string,source_sha256:string}
*/
public function import(
string $path,
string $fromDate,
string $toDate,
int $expectedCount,
string $expectedIncome,
string $expectedExpense,
int $batchSize = 1000,
): array {
$path = $this->validatedPath($path);
[$from, $to] = $this->period($fromDate, $toDate);
$this->assertSchemaReady();
if ($expectedCount <= 0) {
throw new RuntimeException('权威笔数控制数必须大于零。');
}
$expectedIncome = $this->controlAmount($expectedIncome, '收入');
$expectedExpense = $this->controlAmount($expectedExpense, '支出');
$batchSize = min(max($batchSize, 100), 5000);
$hash = hash_file('sha256', $path);
if (! is_string($hash)) {
throw new RuntimeException('无法计算拼多多完整账单文件摘要。');
}
$existing = $this->connection->table(self::IMPORT_TABLE)
->where('source_sha256', $hash)
->first();
if ($existing !== null && $existing->status === 'succeeded') {
$this->assertSameImportRequest(
$existing,
$from,
$to,
$expectedCount,
$expectedIncome,
$expectedExpense,
);
if ($this->activeRowCount($hash, $from, $to) === (int) $existing->row_count) {
return $this->result((array) $existing, 'unchanged');
}
}
$generation = $existing !== null ? (string) $existing->generation : (string) Str::uuid();
$now = now();
$import = [
'generation' => $generation,
'source_filename' => mb_substr(basename($path), 0, 255),
'source_sha256' => $hash,
'period_start' => $from->format('Y-m-d'),
'period_end' => $to->format('Y-m-d'),
'status' => 'running',
'expected_row_count' => $expectedCount,
'expected_income_total' => $expectedIncome,
'expected_expense_total' => $expectedExpense,
'row_count' => 0,
'excluded_account_rows' => 0,
'store_count' => 0,
'bill_item_count' => 0,
'account_type_counts' => null,
'income_total' => '0.00',
'expense_total' => '0.00',
'net_total' => '0.00',
'completed_at' => null,
'last_error_code' => null,
'updated_at' => $now,
];
if ($existing === null) {
$this->connection->table(self::IMPORT_TABLE)->insert($import + ['created_at' => $now]);
} else {
$this->connection->table(self::IMPORT_TABLE)->where('generation', $generation)->update($import);
}
$stats = null;
try {
$stats = $this->stage($path, $generation, $from, $to, $batchSize);
$this->assertControls($stats, $expectedCount, $expectedIncome, $expectedExpense);
$this->activate($generation, $hash, $from, $to, $stats, $batchSize);
return $this->result([
'row_count' => $stats['row_count'],
'excluded_account_rows' => $stats['excluded_account_rows'],
'store_count' => $stats['store_count'],
'bill_item_count' => $stats['bill_item_count'],
'account_type_counts' => $stats['account_type_counts'],
'income_total' => $stats['income_total'],
'expense_total' => $stats['expense_total'],
'net_total' => $stats['net_total'],
'period_start' => $from->format('Y-m-d'),
'period_end' => $to->format('Y-m-d'),
'source_sha256' => $hash,
], 'succeeded');
} catch (Throwable $exception) {
$this->connection->table(self::STAGING_TABLE)->where('generation', $generation)->delete();
$failure = [
'status' => 'failed',
'last_error_code' => mb_substr(class_basename($exception), 0, 120),
'updated_at' => now(),
];
if (is_array($stats)) {
$failure += [
'row_count' => $stats['row_count'],
'excluded_account_rows' => $stats['excluded_account_rows'],
'store_count' => $stats['store_count'],
'bill_item_count' => $stats['bill_item_count'],
'account_type_counts' => json_encode($stats['account_type_counts'], JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR),
'income_total' => $stats['income_total'],
'expense_total' => $stats['expense_total'],
'net_total' => $stats['net_total'],
];
}
$this->connection->table(self::IMPORT_TABLE)->where('generation', $generation)->update($failure);
throw $exception;
}
}
private function validatedPath(string $path): string
{
$realPath = realpath($path);
if ($realPath === false || ! is_file($realPath) || ! is_readable($realPath)) {
throw new RuntimeException('拼多多完整账单文件不存在或不可读。');
}
if (strtolower((string) pathinfo($realPath, PATHINFO_EXTENSION)) !== 'xlsx') {
throw new RuntimeException('聚胜算完整账单导入只支持 XLSX 文件。');
}
return $realPath;
}
/** @return array{0:DateTimeImmutable,1:DateTimeImmutable} */
private function period(string $fromDate, string $toDate): array
{
$from = DateTimeImmutable::createFromFormat('!Y-m-d', $fromDate);
$to = DateTimeImmutable::createFromFormat('!Y-m-d', $toDate);
if ($from === false || $to === false
|| $from->format('Y-m-d') !== $fromDate
|| $to->format('Y-m-d') !== $toDate
|| $from > $to) {
throw new RuntimeException('拼多多账单导入账期无效。');
}
return [$from, $to];
}
private function assertSchemaReady(): void
{
$schema = Schema::connection($this->connection->getName());
foreach ([self::IMPORT_TABLE, self::STAGING_TABLE, self::ROW_TABLE] as $table) {
if (! $schema->hasTable($table)) {
throw new RuntimeException('拼多多完整账单表尚未完成迁移:' . $table);
}
}
}
/**
* @return array{row_count:int,excluded_account_rows:int,store_count:int,bill_item_count:int,account_type_counts:array<string,int>,income_total:string,expense_total:string,net_total:string}
*/
private function stage(
string $path,
string $generation,
DateTimeImmutable $from,
DateTimeImmutable $to,
int $batchSize,
): array {
$archive = new ZipArchive();
if ($archive->open($path) !== true) {
throw new RuntimeException('无法打开拼多多完整账单 XLSX 文件。');
}
try {
$worksheet = $this->firstWorksheet($archive);
$sharedStrings = $this->sharedStrings($path, $archive);
} finally {
$archive->close();
}
$reader = new XMLReader();
if (! $reader->open($this->zipUri($path, $worksheet), null, LIBXML_NONET | LIBXML_COMPACT)) {
throw new RuntimeException('无法读取拼多多完整账单工作表。');
}
$headerMap = null;
$batch = [];
$rowCount = 0;
$excluded = 0;
$incomeTotal = '0.00';
$expenseTotal = '0.00';
$stores = [];
$billItems = [];
$accountTypeCounts = [];
try {
while ($reader->read()) {
if ($reader->nodeType !== XMLReader::ELEMENT || $reader->localName !== 'row') {
continue;
}
$rowNumber = (int) ($reader->getAttribute('r') ?: 0);
$cells = $this->rowCells($reader->readOuterXml(), $sharedStrings);
if ($headerMap === null) {
$headerMap = $this->headerMap($cells);
continue;
}
$values = [];
foreach ($headerMap as $header => $column) {
$values[$header] = trim((string) ($cells[$column] ?? ''));
}
if (implode('', $values) === '') {
continue;
}
$recordCode = $this->required($values, 'record_code', $rowNumber, '记录编码');
$accountName = $this->required($values, 'account_name', $rowNumber, '账户类型');
$storeName = $this->nullable($values['store_name']);
$billItem = $this->nullable($values['bill_item']);
$occurredAt = $this->dateTime($values['occurred_at'], $rowNumber, '发生时间');
if ($occurredAt < $from->format('Y-m-d') . ' 00:00:00'
|| $occurredAt >= $to->modify('+1 day')->format('Y-m-d') . ' 00:00:00') {
throw new RuntimeException("第 {$rowNumber} 行发生时间超出指定账期。");
}
$income = $this->money($values['income_amount'], $rowNumber, '收入金额', false);
$expense = $this->money($values['expense_amount'], $rowNumber, '支出金额', true);
$calculated = bcsub($income, $expense, 2);
$signed = $calculated;
if (array_key_exists('signed_amount', $headerMap)) {
$signed = $this->money($values['signed_amount'], $rowNumber, '金额', false, true);
if (bccomp($calculated, $signed, 2) !== 0) {
throw new RuntimeException("第 {$rowNumber} 行收入、支出与金额不一致。");
}
}
$batch[] = [
'generation' => $generation,
'record_code' => mb_substr($recordCode, 0, 191),
'store_name' => $storeName,
'custom_remark' => $this->nullable($values['custom_remark']),
'bill_item' => $billItem,
'merchant_order_no' => $this->nullable($values['merchant_order_no'], 191),
'online_order_no' => $this->nullable($values['online_order_no'], 191),
'business_type' => $this->nullable($values['business_type']),
'business_description' => $this->nullable($values['business_description'], 1000),
// Keep the account type from the authoritative export. The
// complete Jushengsuan file contains both wallet and marketing
// account rows, and the finance page must retain both.
'account_name' => mb_substr($accountName, 0, 255),
'occurred_at' => $occurredAt,
'income_amount' => $income,
'expense_amount' => $expense,
'signed_amount' => $signed,
'remark' => $this->nullable($values['remark'], 1000),
'source_updated_at' => $this->optionalDateTime($values['source_updated_at'], $rowNumber, '修改时间'),
'source_modified_by' => $this->nullable($values['source_modified_by']),
'source_row_number' => $rowNumber,
];
$rowCount++;
if ($storeName !== null) {
$stores[$storeName] = true;
}
if ($billItem !== null) {
$billItems[$billItem] = true;
}
$accountTypeCounts[$accountName] = ($accountTypeCounts[$accountName] ?? 0) + 1;
$incomeTotal = bcadd($incomeTotal, $income, 2);
$expenseTotal = bcadd($expenseTotal, $expense, 2);
if (count($batch) >= $batchSize) {
$this->connection->table(self::STAGING_TABLE)->insert($batch);
$batch = [];
}
}
} finally {
$reader->close();
}
if ($headerMap === null) {
throw new RuntimeException('未找到聚胜算拼多多账单标准表头。');
}
if ($batch !== []) {
$this->connection->table(self::STAGING_TABLE)->insert($batch);
}
if ($rowCount === 0) {
throw new RuntimeException('文件中没有拼多多货款账户数据。');
}
return [
'row_count' => $rowCount,
'excluded_account_rows' => $excluded,
'store_count' => count($stores),
'bill_item_count' => count($billItems),
'account_type_counts' => $accountTypeCounts,
'income_total' => $incomeTotal,
'expense_total' => $expenseTotal,
'net_total' => bcsub($incomeTotal, $expenseTotal, 2),
];
}
private function assertControls(array $stats, int $count, string $income, string $expense): void
{
if ($stats['row_count'] !== $count
|| bccomp($stats['income_total'], $income, 2) !== 0
|| bccomp($stats['expense_total'], $expense, 2) !== 0) {
throw new RuntimeException(sprintf(
'权威控制数不一致:实际 %d 笔,收入 %s,支出 %s;期望 %d 笔,收入 %s,支出 %s。',
$stats['row_count'],
$stats['income_total'],
$stats['expense_total'],
$count,
$income,
$expense,
));
}
}
private function assertSameImportRequest(
object $existing,
DateTimeImmutable $from,
DateTimeImmutable $to,
int $expectedCount,
string $expectedIncome,
string $expectedExpense,
): void {
if ((string) $existing->period_start !== $from->format('Y-m-d')
|| (string) $existing->period_end !== $to->format('Y-m-d')
|| (int) $existing->expected_row_count !== $expectedCount
|| bccomp((string) $existing->expected_income_total, $expectedIncome, 2) !== 0
|| bccomp((string) $existing->expected_expense_total, $expectedExpense, 2) !== 0) {
throw new RuntimeException('该文件已有成功导入记录,但本次账期或权威控制数与历史记录不一致。');
}
}
private function activeRowCount(
string $hash,
DateTimeImmutable $from,
DateTimeImmutable $to,
): int {
return $this->connection->table(self::ROW_TABLE)
->where('source_sha256', $hash)
->where('occurred_at', '>=', $from->format('Y-m-d') . ' 00:00:00')
->where('occurred_at', '<', $to->modify('+1 day')->format('Y-m-d') . ' 00:00:00')
->count();
}
private function activate(
string $generation,
string $hash,
DateTimeImmutable $from,
DateTimeImmutable $to,
array $stats,
int $batchSize,
): void {
$now = now();
$columns = [
'store_name', 'custom_remark', 'bill_item', 'merchant_order_no', 'online_order_no',
'business_type', 'business_description', 'account_name', 'occurred_at', 'income_amount',
'expense_amount', 'signed_amount', 'remark', 'source_updated_at', 'source_modified_by',
'source_row_number', 'source_generation', 'source_sha256', 'imported_at',
];
$this->connection->transaction(function () use (
$generation, $hash, $from, $to, $stats, $batchSize, $columns, $now
): void {
$this->connection->table(self::ROW_TABLE)
->where('occurred_at', '>=', $from->format('Y-m-d') . ' 00:00:00')
->where('occurred_at', '<', $to->modify('+1 day')->format('Y-m-d') . ' 00:00:00')
->delete();
$this->connection->table(self::STAGING_TABLE)
->where('generation', $generation)
->orderBy('id')
->chunkById($batchSize, function ($rows) use ($generation, $hash, $columns, $now): void {
$payload = [];
foreach ($rows as $row) {
$payload[] = [
'record_code' => $row->record_code,
'store_name' => $row->store_name,
'custom_remark' => $row->custom_remark,
'bill_item' => $row->bill_item,
'merchant_order_no' => $row->merchant_order_no,
'online_order_no' => $row->online_order_no,
'business_type' => $row->business_type,
'business_description' => $row->business_description,
'account_name' => $row->account_name,
'occurred_at' => $row->occurred_at,
'income_amount' => $row->income_amount,
'expense_amount' => $row->expense_amount,
'signed_amount' => $row->signed_amount,
'remark' => $row->remark,
'source_updated_at' => $row->source_updated_at,
'source_modified_by' => $row->source_modified_by,
'source_row_number' => $row->source_row_number,
'source_generation' => $generation,
'source_sha256' => $hash,
'imported_at' => $now,
];
}
$this->connection->table(self::ROW_TABLE)->upsert($payload, ['record_code'], $columns);
});
$this->connection->table(self::IMPORT_TABLE)->where('generation', $generation)->update([
'status' => 'succeeded',
'row_count' => $stats['row_count'],
'excluded_account_rows' => $stats['excluded_account_rows'],
'store_count' => $stats['store_count'],
'bill_item_count' => $stats['bill_item_count'],
'account_type_counts' => json_encode($stats['account_type_counts'], JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR),
'income_total' => $stats['income_total'],
'expense_total' => $stats['expense_total'],
'net_total' => $stats['net_total'],
'completed_at' => $now,
'last_error_code' => null,
'updated_at' => $now,
]);
});
$this->connection->table(self::STAGING_TABLE)->where('generation', $generation)->delete();
}
/** @return array<int,string> */
private function rowCells(string $xml, array $sharedStrings): array
{
$row = simplexml_load_string($xml);
if (! $row instanceof SimpleXMLElement) {
return [];
}
$cells = [];
foreach ($row->xpath('./*[local-name()="c"]') ?: [] as $cell) {
$reference = (string) ($cell['r'] ?? '');
if (preg_match('/^([A-Z]+)\d+$/i', $reference, $matches) !== 1) {
continue;
}
$type = (string) ($cell['t'] ?? '');
if ($type === 'inlineStr') {
$nodes = $cell->xpath('.//*[local-name()="t"]') ?: [];
$value = implode('', array_map(static fn (SimpleXMLElement $node): string => (string) $node, $nodes));
} else {
$nodes = $cell->xpath('./*[local-name()="v"]') ?: [];
$raw = isset($nodes[0]) ? (string) $nodes[0] : '';
$value = $type === 's' ? (string) ($sharedStrings[(int) $raw] ?? '') : $raw;
}
$cells[$this->columnIndex($matches[1])] = $value;
}
return $cells;
}
/** @param array<int,string> $cells @return array<string,int> */
private function headerMap(array $cells): ?array
{
$lookup = [];
foreach ($cells as $column => $value) {
$lookup[trim($value)] = $column;
}
$mapping = [];
foreach (self::HEADER_ALIASES as $field => $aliases) {
foreach ($aliases as $alias) {
if (array_key_exists($alias, $lookup)) {
$mapping[$field] = $lookup[$alias];
break;
}
}
if (! array_key_exists($field, $mapping)
&& ! in_array($field, self::OPTIONAL_HEADERS, true)) {
return null;
}
}
return $mapping;
}
private function required(array $values, string $key, int $row, ?string $label = null): string
{
$value = trim((string) ($values[$key] ?? ''));
if ($value === '') {
throw new RuntimeException("第 {$row} 行缺少" . ($label ?? $key) . '。');
}
return $value;
}
private function money(
string $value,
int $row,
string $field,
bool $absolute = false,
bool $allowNegative = false,
): string {
$value = str_replace([',', ',', '¥', '¥'], '', trim($value));
$value = $value === '' ? '0' : $value;
if (! is_numeric($value) || (! $allowNegative && ! $absolute && bccomp($value, '0', 6) < 0)) {
throw new RuntimeException("第 {$row} 行{$field}不是有效金额。");
}
if ($absolute && bccomp($value, '0', 6) < 0) {
$value = bcmul($value, '-1', 6);
}
return bcadd($value, '0', 2);
}
private function controlAmount(string $value, string $label): string
{
$value = str_replace([',', ',', '¥', '¥'], '', trim($value));
if (! is_numeric($value) || bccomp($value, '0', 6) < 0) {
throw new RuntimeException("权威{$label}控制数无效。");
}
return bcadd($value, '0', 2);
}
private function dateTime(string $value, int $row, string $field): string
{
$parsed = $this->parseDateTime($value);
if ($parsed === null) {
throw new RuntimeException("第 {$row} 行{$field}不是有效日期时间。");
}
return $parsed;
}
private function optionalDateTime(string $value, int $row, string $field): ?string
{
if (trim($value) === '') {
return null;
}
return $this->dateTime($value, $row, $field);
}
private function parseDateTime(string $value): ?string
{
$value = trim($value);
try {
if (is_numeric($value)) {
$seconds = (int) round((((float) $value) - 25569) * 86400);
return Carbon::createFromTimestampUTC($seconds)->format('Y-m-d H:i:s');
}
return Carbon::parse($value)->format('Y-m-d H:i:s');
} catch (Throwable) {
return null;
}
}
private function nullable(string $value, int $max = 255): ?string
{
$value = trim($value);
return $value === '' ? null : mb_substr($value, 0, $max);
}
private function columnIndex(string $letters): int
{
$index = 0;
foreach (str_split(strtoupper($letters)) as $letter) {
$index = ($index * 26) + ord($letter) - 64;
}
return $index - 1;
}
/** @return list<string> */
private function sharedStrings(string $path, ZipArchive $archive): array
{
if ($archive->locateName('xl/sharedStrings.xml') === false) {
return [];
}
$reader = new XMLReader();
if (! $reader->open($this->zipUri($path, 'xl/sharedStrings.xml'), null, LIBXML_NONET | LIBXML_COMPACT)) {
throw new RuntimeException('无法读取 XLSX 共享文本。');
}
$strings = [];
try {
while ($reader->read()) {
if ($reader->nodeType !== XMLReader::ELEMENT || $reader->localName !== 'si') {
continue;
}
$item = simplexml_load_string($reader->readOuterXml());
$nodes = $item instanceof SimpleXMLElement ? ($item->xpath('.//*[local-name()="t"]') ?: []) : [];
$strings[] = implode('', array_map(
static fn (SimpleXMLElement $node): string => (string) $node,
$nodes,
));
}
} finally {
$reader->close();
}
return $strings;
}
private function firstWorksheet(ZipArchive $archive): string
{
$worksheets = [];
for ($index = 0; $index < $archive->numFiles; $index++) {
$name = $archive->getNameIndex($index);
if (is_string($name) && preg_match('#^xl/worksheets/sheet\d+\.xml$#', $name) === 1) {
$worksheets[] = $name;
}
}
sort($worksheets, SORT_NATURAL);
if ($worksheets === []) {
throw new RuntimeException('XLSX 文件中没有工作表。');
}
return $worksheets[0];
}
private function zipUri(string $path, string $entry): string
{
return 'zip://' . str_replace('\\', '/', $path) . '#' . $entry;
}
private function result(array $row, string $status): array
{
return [
'status' => $status,
'row_count' => (int) ($row['row_count'] ?? 0),
'excluded_account_rows' => (int) ($row['excluded_account_rows'] ?? 0),
'store_count' => (int) ($row['store_count'] ?? 0),
'bill_item_count' => (int) ($row['bill_item_count'] ?? 0),
'account_type_counts' => $this->accountTypeCounts($row['account_type_counts'] ?? []),
'income_total' => bcadd((string) ($row['income_total'] ?? '0'), '0', 2),
'expense_total' => bcadd((string) ($row['expense_total'] ?? '0'), '0', 2),
'net_total' => bcadd((string) ($row['net_total'] ?? '0'), '0', 2),
'period_start' => (string) ($row['period_start'] ?? ''),
'period_end' => (string) ($row['period_end'] ?? ''),
'source_sha256' => (string) ($row['source_sha256'] ?? ''),
];
}
/** @return array<string,int> */
private function accountTypeCounts(mixed $value): array
{
if (is_string($value)) {
try {
$value = json_decode($value, true, 512, JSON_THROW_ON_ERROR);
} catch (Throwable) {
return [];
}
}
if (! is_array($value)) {
return [];
}
$counts = [];
foreach ($value as $account => $count) {
if (is_string($account) && $account !== '' && is_numeric($count)) {
$counts[$account] = (int) $count;
}
}
return $counts;
}
}