825 lines
39 KiB
PHP
825 lines
39 KiB
PHP
<?php
|
|
|
|
namespace Modules\EcommerceFinance\Services;
|
|
|
|
use Illuminate\Database\ConnectionInterface;
|
|
use Illuminate\Database\DatabaseManager;
|
|
use Illuminate\Support\Facades\Schema;
|
|
use Illuminate\Support\Str;
|
|
use Modules\EcommerceFinance\Repositories\StoreReportRepository;
|
|
use Modules\EcommerceFinance\Repositories\StoreReportUpstreamRepository;
|
|
use RuntimeException;
|
|
use Throwable;
|
|
|
|
class StoreReportMirrorSynchronizer
|
|
{
|
|
private readonly ConnectionInterface $connection;
|
|
|
|
public function __construct(
|
|
private readonly StoreReportUpstreamRepository $upstream,
|
|
DatabaseManager $databases,
|
|
) {
|
|
$this->connection = $databases->connection(
|
|
(string) config('upstream_mirrors.connection', 'business_mirror'),
|
|
);
|
|
}
|
|
|
|
/** @return array{lifecycles:array<string,mixed>,period_changes:array<string,mixed>,bills:array<string,mixed>,daily:array<string,mixed>} */
|
|
public function sync(
|
|
?int $batchSize = null,
|
|
?int $maxBatches = null,
|
|
?int $dailyBatchSize = null,
|
|
?int $dailyMaxBatches = null,
|
|
): array
|
|
{
|
|
$this->assertSchemaReady();
|
|
|
|
$lifecycleBatch = min(max((int) config('upstream_mirrors.store_report.lifecycle_batch_size', 1000), 100), 5000);
|
|
$lifecycleMax = max((int) config('upstream_mirrors.store_report.lifecycle_max_batches_per_run', 5), 1);
|
|
$billBatch = min(max($batchSize ?? (int) config('upstream_mirrors.store_report.batch_size', 5000), 100), 10000);
|
|
$billMax = max($maxBatches ?? (int) config('upstream_mirrors.store_report.max_batches_per_run', 4), 1);
|
|
|
|
$lifecycles = $this->syncDataset(
|
|
StoreReportRepository::LIFECYCLES_DATASET,
|
|
'store_report_lifecycle_rows',
|
|
$lifecycleBatch,
|
|
$lifecycleMax,
|
|
fn (int $cursor, int $limit): array => $this->upstream->lifecycleBatch($cursor, $limit),
|
|
fn (array $rows, string $generation): array => $this->mapLifecycles($rows, $generation),
|
|
);
|
|
$periodChanges = $this->syncShopPeriodChanges();
|
|
$bills = $this->syncDataset(
|
|
StoreReportRepository::BILLS_DATASET,
|
|
'store_report_bill_rows',
|
|
$billBatch,
|
|
$billMax,
|
|
fn (int $cursor, int $limit): array => $this->upstream->billBatch($cursor, $limit),
|
|
fn (array $rows, string $generation): array => $this->mapBills($rows, $generation),
|
|
);
|
|
|
|
return [
|
|
'lifecycles' => $lifecycles,
|
|
'period_changes' => $periodChanges,
|
|
'bills' => $bills,
|
|
'daily' => $this->syncDailySummary($dailyBatchSize, $dailyMaxBatches),
|
|
'paid_sales' => $this->syncPaidSales(),
|
|
];
|
|
}
|
|
|
|
/**
|
|
* 把爬虫首次抓到的新主体写入店铺主体期间变更记录。
|
|
*
|
|
* 爬虫主体名称先按税号归一到工商主体资料，避免同一主体的名称笔误被当成变更；
|
|
* 只有与当前期间最后一个主体不同才新增记录，重复执行不会重复插入，
|
|
* 记录下来的首次抓到日期会固定新主体的生效时间，后续爬虫更新不再前移。
|
|
*
|
|
* @return array{status:string,checked:int,created:int,reason?:string}
|
|
*/
|
|
private function syncShopPeriodChanges(): array
|
|
{
|
|
// 控制台没有网页公司上下文；先定位首家公司的期初设置，避免回退到代码文件旧值。
|
|
try {
|
|
\App\Models\Common\Company::query()->orderBy('id')->first()?->makeCurrent();
|
|
} catch (Throwable) {
|
|
// 读取失败时期间构建器会回退到配置文件，不阻断同步。
|
|
}
|
|
|
|
$schema = Schema::connection($this->connection->getName());
|
|
if (! $schema->hasTable('ops_crawl_shop_company')) {
|
|
return ['status' => 'skipped', 'checked' => 0, 'created' => 0, 'reason' => 'crawl_mirror_not_ready'];
|
|
}
|
|
if (! $schema->hasTable(StoreReportPeriodBuilder::CHANGES_TABLE)) {
|
|
return ['status' => 'skipped', 'checked' => 0, 'created' => 0, 'reason' => 'change_table_missing'];
|
|
}
|
|
|
|
$ranked = $this->connection->table('ops_crawl_shop_company')
|
|
->selectRaw('id, shop_name, company_name, tax_no, crawl_date, row_number() over (partition by shop_name order by crawl_date desc, id desc) AS row_rank');
|
|
$latest = $this->connection->table($ranked, 'crawler_latest')
|
|
->where('row_rank', 1)
|
|
->get(['shop_name', 'company_name', 'tax_no', 'crawl_date']);
|
|
if ($latest->isEmpty()) {
|
|
return ['status' => 'succeeded', 'checked' => 0, 'created' => 0];
|
|
}
|
|
|
|
$masterNames = [];
|
|
if ($schema->hasTable('ops_fys_company_info')) {
|
|
$rankedCompanies = $this->connection->table('ops_fys_company_info')
|
|
->selectRaw('unifiedSocialCreditCode, businessLicenseName, row_number() over (partition by unifiedSocialCreditCode order by Id desc) AS row_rank');
|
|
foreach ($this->connection->table($rankedCompanies, 'company_latest')
|
|
->where('row_rank', 1)
|
|
->get(['unifiedSocialCreditCode', 'businessLicenseName']) as $row) {
|
|
$code = trim((string) ($row->unifiedSocialCreditCode ?? ''));
|
|
$name = trim((string) ($row->businessLicenseName ?? ''));
|
|
if ($code !== '' && $name !== '') {
|
|
$masterNames[$code] = $name;
|
|
}
|
|
}
|
|
}
|
|
|
|
$changes = [];
|
|
foreach ($latest as $row) {
|
|
$name = trim((string) ($row->shop_name ?? ''));
|
|
$taxNo = trim((string) ($row->tax_no ?? ''));
|
|
$company = $masterNames[$taxNo] ?? trim((string) ($row->company_name ?? ''));
|
|
$observed = substr(trim((string) ($row->crawl_date ?? '')), 0, 10);
|
|
if ($name === '' || $company === '' || $observed === '') {
|
|
continue;
|
|
}
|
|
$changes[$name] = ['company' => $company, 'tax_no' => $taxNo, 'observed_on' => $observed];
|
|
}
|
|
if ($changes === []) {
|
|
return ['status' => 'succeeded', 'checked' => 0, 'created' => 0];
|
|
}
|
|
|
|
$periods = app(StoreReportPeriodBuilder::class)->build(array_map(
|
|
static fn (string $name): array => ['erp_store_name' => $name],
|
|
array_keys($changes),
|
|
));
|
|
|
|
$created = 0;
|
|
foreach ($changes as $name => $change) {
|
|
$list = $periods[$name] ?? [];
|
|
$last = $list === [] ? null : $list[count($list) - 1];
|
|
if ($this->sameCompany($last['company_name'] ?? null, $change['company'])) {
|
|
continue;
|
|
}
|
|
|
|
$created += (int) $this->connection->table(StoreReportPeriodBuilder::CHANGES_TABLE)->insertOrIgnore([
|
|
'erp_store_name' => $name,
|
|
'company_name' => $change['company'],
|
|
'unified_social_credit_code' => $change['tax_no'] === '' ? null : $change['tax_no'],
|
|
'observed_on' => $change['observed_on'],
|
|
'source' => 'crawl_shop_company',
|
|
'business_key' => hash('sha256', implode('|', [$name, $change['company'], $change['observed_on']])),
|
|
'created_at' => now(),
|
|
'updated_at' => now(),
|
|
]);
|
|
}
|
|
|
|
return ['status' => 'succeeded', 'checked' => count($changes), 'created' => $created];
|
|
}
|
|
|
|
/**
|
|
* @param callable(int,int):array<int,array<string,mixed>> $fetch
|
|
* @param callable(array<int,array<string,mixed>>,string):array{upserts:array<int,array<string,mixed>>,deletes:array<int,int>} $map
|
|
* @return array{status:string,mode:string,batches:int,source_rows:int,stored_rows:int,cursor_id:int}
|
|
*/
|
|
private function syncDataset(
|
|
string $dataset,
|
|
string $table,
|
|
int $batchSize,
|
|
int $maxBatches,
|
|
callable $fetch,
|
|
callable $map,
|
|
): array {
|
|
$state = $this->state($dataset);
|
|
$initial = empty($state->active_generation);
|
|
$mode = $initial ? 'full' : 'incremental';
|
|
$resume = in_array((string) $state->status, ['running', 'failed'], true)
|
|
&& (string) ($state->run_mode ?? '') === $mode;
|
|
$generation = $initial
|
|
? ($resume && ! empty($state->pending_generation)
|
|
? (string) $state->pending_generation
|
|
: (string) Str::uuid())
|
|
: (string) $state->active_generation;
|
|
$cursor = $resume
|
|
? (int) ($state->cursor_id ?? 0)
|
|
: ($initial
|
|
? 0
|
|
: max((int) ($state->cursor_id ?? 0)
|
|
- max((int) config('upstream_mirrors.store_report.overlap_rows', 50000), 0), 0));
|
|
$storedRows = $initial && ! $resume ? 0 : (int) $state->row_count;
|
|
$sourceRows = 0;
|
|
$batches = 0;
|
|
|
|
$this->connection->table('store_report_import_states')->where('dataset_key', $dataset)->update([
|
|
'status' => 'running',
|
|
'run_mode' => $mode,
|
|
'pending_generation' => $initial ? $generation : null,
|
|
'cursor_id' => $cursor,
|
|
'last_attempted_at' => now(),
|
|
'last_error_code' => null,
|
|
'updated_at' => now(),
|
|
]);
|
|
|
|
try {
|
|
while ($batches < $maxBatches) {
|
|
$rows = $fetch($cursor, $batchSize);
|
|
if ($rows === []) {
|
|
return $this->complete($dataset, $generation, $mode, $cursor, $storedRows, $batches, $sourceRows);
|
|
}
|
|
|
|
$nextCursor = max(array_map(static fn (array $row): int => (int) ($row['source_id'] ?? 0), $rows));
|
|
if ($nextCursor <= $cursor) {
|
|
throw new RuntimeException('store_report_cursor_did_not_advance');
|
|
}
|
|
|
|
$mapped = $map($rows, $generation);
|
|
$storedRows = $this->storeBatch(
|
|
$dataset,
|
|
$table,
|
|
$generation,
|
|
$nextCursor,
|
|
$mapped,
|
|
$storedRows,
|
|
);
|
|
$cursor = $nextCursor;
|
|
$sourceRows += count($rows);
|
|
$batches++;
|
|
|
|
if (count($rows) < $batchSize) {
|
|
return $this->complete($dataset, $generation, $mode, $cursor, $storedRows, $batches, $sourceRows);
|
|
}
|
|
}
|
|
|
|
return $this->result('running', $mode, $batches, $sourceRows, $storedRows, $cursor);
|
|
} catch (Throwable $exception) {
|
|
$this->connection->table('store_report_import_states')->where('dataset_key', $dataset)->update([
|
|
'status' => 'failed',
|
|
'last_error_code' => class_basename($exception),
|
|
'updated_at' => now(),
|
|
]);
|
|
throw $exception;
|
|
}
|
|
}
|
|
|
|
private function state(string $dataset): object
|
|
{
|
|
$now = now();
|
|
$this->connection->table('store_report_import_states')->insertOrIgnore([
|
|
'dataset_key' => $dataset,
|
|
'status' => 'pending',
|
|
'row_count' => 0,
|
|
'source_files_json' => json_encode(['jiansun.dbo.jst_originalBillInfo', 'jiansun.dbo.fys_shopInfo_flow']),
|
|
'created_at' => $now,
|
|
'updated_at' => $now,
|
|
]);
|
|
|
|
return $this->connection->table('store_report_import_states')->where('dataset_key', $dataset)->first();
|
|
}
|
|
|
|
/** @param array{upserts:array<int,array<string,mixed>>,deletes:array<int,int>} $mapped */
|
|
private function storeBatch(
|
|
string $dataset,
|
|
string $table,
|
|
string $generation,
|
|
int $cursor,
|
|
array $mapped,
|
|
int $storedRows,
|
|
): int {
|
|
return $this->connection->transaction(function () use (
|
|
$dataset, $table, $generation, $cursor, $mapped, $storedRows,
|
|
): int {
|
|
$upsertIds = [];
|
|
foreach ($mapped['upserts'] as $row) {
|
|
$upsertIds[(int) $row['source_id']] = true;
|
|
}
|
|
$deleteIds = array_fill_keys(array_map('intval', $mapped['deletes']), true);
|
|
$touchedIds = array_keys($upsertIds + $deleteIds);
|
|
$existingIds = [];
|
|
foreach (array_chunk($touchedIds, 1000) as $chunk) {
|
|
foreach ($this->connection->table($table)
|
|
->where('generation', $generation)
|
|
->whereIn('source_id', $chunk)
|
|
->pluck('source_id') as $sourceId) {
|
|
$existingIds[(int) $sourceId] = true;
|
|
}
|
|
}
|
|
$inserted = count(array_diff_key($upsertIds, $existingIds));
|
|
$deleted = count(array_diff_key(array_intersect_key($deleteIds, $existingIds), $upsertIds));
|
|
$nextStoredRows = max($storedRows + $inserted - $deleted, 0);
|
|
|
|
$dailyDeltas = $table === 'store_report_bill_rows' && $this->dailySummaryReady($generation)
|
|
? $this->dailyDeltas($generation, $mapped)
|
|
: [];
|
|
if ($mapped['deletes'] !== []) {
|
|
$this->connection->table($table)
|
|
->where('generation', $generation)
|
|
->whereIn('source_id', $mapped['deletes'])
|
|
->delete();
|
|
}
|
|
if ($mapped['upserts'] !== []) {
|
|
$updates = array_values(array_diff(array_keys($mapped['upserts'][0]), ['generation', 'source_id']));
|
|
// MySQL/MariaDB 对单条预处理语句的占位符数量有限，不能把
|
|
// 数千行、十余列一次塞进同一条 upsert。
|
|
$columnsPerRow = max(count($mapped['upserts'][0]), 1);
|
|
$chunkSize = max(1, min(2000, intdiv(50000, $columnsPerRow)));
|
|
foreach (array_chunk($mapped['upserts'], $chunkSize) as $chunk) {
|
|
$this->connection->table($table)->upsert(
|
|
$chunk, ['generation', 'source_id'], $updates,
|
|
);
|
|
}
|
|
}
|
|
if ($dailyDeltas !== []) {
|
|
$this->applyDailyDeltas($generation, $dailyDeltas);
|
|
}
|
|
$this->connection->table('store_report_import_states')->where('dataset_key', $dataset)->update([
|
|
'cursor_id' => $cursor,
|
|
'row_count' => $nextStoredRows,
|
|
'updated_at' => now(),
|
|
]);
|
|
|
|
return $nextStoredRows;
|
|
});
|
|
}
|
|
|
|
/** @return array{status:string,mode:string,batches:int,source_rows:int,stored_rows:int,cursor_id:int} */
|
|
private function complete(
|
|
string $dataset,
|
|
string $generation,
|
|
string $mode,
|
|
int $cursor,
|
|
int $storedRows,
|
|
int $batches,
|
|
int $sourceRows,
|
|
): array {
|
|
$now = now();
|
|
$values = [
|
|
'status' => 'succeeded',
|
|
'run_mode' => null,
|
|
'cursor_id' => $cursor,
|
|
'last_succeeded_at' => $now,
|
|
'last_error_code' => null,
|
|
'row_count' => $storedRows,
|
|
'updated_at' => $now,
|
|
];
|
|
if ($mode === 'full') {
|
|
$values += [
|
|
'active_generation' => $generation,
|
|
'pending_generation' => null,
|
|
'initial_completed_at' => $now,
|
|
];
|
|
}
|
|
$this->connection->table('store_report_import_states')->where('dataset_key', $dataset)->update($values);
|
|
|
|
return $this->result('succeeded', $mode, $batches, $sourceRows, $storedRows, $cursor);
|
|
}
|
|
|
|
/** @return array{status:string,mode:string,batches:int,source_rows:int,stored_rows:int,cursor_id:int} */
|
|
private function syncDailySummary(?int $batchSize, ?int $maxBatches): array
|
|
{
|
|
$billState = $this->state(StoreReportRepository::BILLS_DATASET);
|
|
$generation = (string) ($billState->active_generation ?? '');
|
|
if ($generation === '') {
|
|
return $this->result('pending', 'full', 0, 0, 0, 0);
|
|
}
|
|
|
|
$batchSize = min(max($batchSize ?? (int) config('upstream_mirrors.store_report.daily_batch_size', 50000), 1000), 100000);
|
|
$maxBatches = max($maxBatches ?? (int) config('upstream_mirrors.store_report.daily_max_batches_per_run', 10), 1);
|
|
$state = $this->state(StoreReportRepository::DAILY_DATASET);
|
|
if ((string) ($state->active_generation ?? '') === $generation && ! empty($state->initial_completed_at)) {
|
|
return $this->result('succeeded', 'incremental', 0, 0, (int) $state->row_count, (int) $state->cursor_id);
|
|
}
|
|
|
|
$resume = in_array((string) $state->status, ['running', 'failed'], true)
|
|
&& (string) ($state->pending_generation ?? '') === $generation;
|
|
$cursor = $resume ? (int) ($state->cursor_id ?? 0) : 0;
|
|
if (! $resume) {
|
|
$this->connection->table('store_report_daily_rows')->where('generation', $generation)->delete();
|
|
}
|
|
$this->connection->table('store_report_import_states')
|
|
->where('dataset_key', StoreReportRepository::DAILY_DATASET)
|
|
->update([
|
|
'status' => 'running', 'run_mode' => 'full', 'pending_generation' => $generation,
|
|
'cursor_id' => $cursor, 'last_attempted_at' => now(), 'last_error_code' => null,
|
|
'updated_at' => now(),
|
|
]);
|
|
|
|
$batches = 0;
|
|
$sourceRows = 0;
|
|
try {
|
|
while ($batches < $maxBatches) {
|
|
$rows = $this->connection->table('store_report_bill_rows')
|
|
->where('generation', $generation)->where('source_id', '>', $cursor)
|
|
->orderBy('source_id')->limit($batchSize)
|
|
->get(['source_id', 'store_name', 'occurred_at', 'income', 'expense'])
|
|
->map(static fn (object $row): array => (array) $row)->all();
|
|
if ($rows === []) {
|
|
return $this->completeDailySummary($generation, $cursor, $batches, $sourceRows);
|
|
}
|
|
$cursor = max(array_map(static fn (array $row): int => (int) $row['source_id'], $rows));
|
|
$deltas = $this->aggregateDailyRows($rows, 1.0);
|
|
$this->connection->transaction(function () use ($generation, $cursor, $deltas): void {
|
|
$this->applyDailyDeltas($generation, $deltas);
|
|
$this->connection->table('store_report_import_states')
|
|
->where('dataset_key', StoreReportRepository::DAILY_DATASET)
|
|
->update(['cursor_id' => $cursor, 'updated_at' => now()]);
|
|
});
|
|
$sourceRows += count($rows);
|
|
$batches++;
|
|
if (count($rows) < $batchSize) {
|
|
return $this->completeDailySummary($generation, $cursor, $batches, $sourceRows);
|
|
}
|
|
}
|
|
} catch (Throwable $exception) {
|
|
$this->connection->table('store_report_import_states')
|
|
->where('dataset_key', StoreReportRepository::DAILY_DATASET)
|
|
->update(['status' => 'failed', 'last_error_code' => class_basename($exception), 'updated_at' => now()]);
|
|
throw $exception;
|
|
}
|
|
|
|
$stored = (int) $this->connection->table('store_report_daily_rows')->where('generation', $generation)->count();
|
|
return $this->result('running', 'full', $batches, $sourceRows, $stored, $cursor);
|
|
}
|
|
|
|
/** @return array{status:string,mode:string,batches:int,source_rows:int,stored_rows:int,cursor_id:int} */
|
|
private function completeDailySummary(string $generation, int $cursor, int $batches, int $sourceRows): array
|
|
{
|
|
$rows = (int) $this->connection->table('store_report_daily_rows')->where('generation', $generation)->count();
|
|
$now = now();
|
|
$this->connection->table('store_report_import_states')
|
|
->where('dataset_key', StoreReportRepository::DAILY_DATASET)
|
|
->update([
|
|
'status' => 'succeeded', 'run_mode' => null, 'active_generation' => $generation,
|
|
'pending_generation' => null, 'cursor_id' => $cursor, 'row_count' => $rows,
|
|
'initial_completed_at' => $now, 'last_succeeded_at' => $now,
|
|
'last_error_code' => null, 'updated_at' => $now,
|
|
]);
|
|
return $this->result('succeeded', 'full', $batches, $sourceRows, $rows, $cursor);
|
|
}
|
|
|
|
private function dailySummaryReady(string $generation): bool
|
|
{
|
|
$state = $this->connection->table('store_report_import_states')
|
|
->where('dataset_key', StoreReportRepository::DAILY_DATASET)->first();
|
|
return $state !== null && (string) ($state->active_generation ?? '') === $generation
|
|
&& ! empty($state->initial_completed_at);
|
|
}
|
|
|
|
/**
|
|
* 店铺维度报表的「销量」：把业绩明细的
|
|
* 「收入金额 + 优惠金额 + 冲减金额 + 退款金额」按「店铺名称 + 发生日期」预汇总。
|
|
*
|
|
* 业绩明细由 store-receipt:order-match:sync 按「账期 + 平台」整体重写，这里用
|
|
* 「行数 + 最近写入时间 + 金额合计」的指纹判断是否变化：没变化就复用当前版本，
|
|
* 变化时全量重建（源表 60 万行左右，汇总后每店铺每天一行）。
|
|
*
|
|
* @return array{status:string,source_rows:int,stored_rows:int,changed:bool}
|
|
*/
|
|
public function syncPaidSales(bool $force = false): array
|
|
{
|
|
$result = \Illuminate\Support\Facades\Cache::store('upstream_shared')->lock('store-report:paid-sales', 600)
|
|
->get(fn () => $this->syncPaidSalesUnlocked($force));
|
|
return $result === false
|
|
? ['status' => 'succeeded', 'source_rows' => 0, 'stored_rows' => 0, 'changed' => false]
|
|
: $result;
|
|
}
|
|
|
|
private function syncPaidSalesUnlocked(bool $force): array
|
|
{
|
|
if (! Schema::connection($this->connection->getName())
|
|
->hasTable(StoreReportRepository::PAID_SALES_SOURCE_TABLE)) {
|
|
return ['status' => 'skipped', 'source_rows' => 0, 'stored_rows' => 0, 'changed' => false];
|
|
}
|
|
|
|
$state = $this->state(StoreReportRepository::PAID_SALES_DATASET);
|
|
$changes = app(StoreReportPaidSalesChanges::class);
|
|
try {
|
|
if (! $force && $changes->available() && ! empty($state->active_generation)
|
|
&& ! empty($state->initial_completed_at)
|
|
&& str_starts_with((string) ($state->source_fingerprint ?? ''), StoreReportPaidSalesChanges::VERSION)) {
|
|
return $this->syncPaidSalesDates($state, $changes);
|
|
}
|
|
return $this->connection->transaction(function () use ($state, $force, $changes): array {
|
|
$pending = $changes->available() ? $changes->pending() : [];
|
|
$fingerprint = $changes->available()
|
|
? StoreReportPaidSalesChanges::VERSION . Str::uuid() : $this->paidSalesFingerprint();
|
|
$active = trim((string) ($state->active_generation ?? ''));
|
|
if (! $force && $active !== '' && ! empty($state->initial_completed_at)
|
|
&& (string) ($state->source_fingerprint ?? '') === $fingerprint) {
|
|
return [
|
|
'status' => 'succeeded',
|
|
'source_rows' => 0,
|
|
'stored_rows' => (int) ($state->row_count ?? 0),
|
|
'changed' => false,
|
|
];
|
|
}
|
|
|
|
$now = now();
|
|
$generation = (string) Str::uuid();
|
|
$this->connection->statement(
|
|
'INSERT INTO ' . StoreReportRepository::PAID_SALES_TABLE
|
|
. ' (generation, store_name, occurred_on, paid_amount, order_count, updated_at)'
|
|
. ' SELECT ?, store_name, occurred_date,'
|
|
. ' SUM(COALESCE(income_amount, 0) + COALESCE(discount_amount, 0)'
|
|
. ' + COALESCE(offset_amount, 0) + COALESCE(refund_amount, 0)), COUNT(*), ?'
|
|
. ' FROM ' . StoreReportRepository::PAID_SALES_SOURCE_TABLE
|
|
. " WHERE store_name IS NOT NULL AND store_name <> '' AND occurred_date IS NOT NULL"
|
|
. ' GROUP BY store_name, occurred_date',
|
|
[$generation, $now->format('Y-m-d H:i:s')],
|
|
);
|
|
$stored = (int) $this->connection->table(StoreReportRepository::PAID_SALES_TABLE)
|
|
->where('generation', $generation)->count();
|
|
$sourceRows = (int) ($this->connection->table(StoreReportRepository::PAID_SALES_SOURCE_TABLE)->count());
|
|
$this->connection->table('store_report_import_states')
|
|
->where('dataset_key', StoreReportRepository::PAID_SALES_DATASET)
|
|
->update([
|
|
'status' => 'succeeded',
|
|
'run_mode' => 'full',
|
|
'active_generation' => $generation,
|
|
'pending_generation' => null,
|
|
'row_count' => $stored,
|
|
'source_fingerprint' => $fingerprint,
|
|
'initial_completed_at' => $state->initial_completed_at ?? $now,
|
|
'last_attempted_at' => $now,
|
|
'last_succeeded_at' => $now,
|
|
'last_error_code' => null,
|
|
'updated_at' => $now,
|
|
]);
|
|
// 版本切换成功后再清理上一版，避免中途失败时页面读到被清空的当前版本。
|
|
$this->connection->table(StoreReportRepository::PAID_SALES_TABLE)
|
|
->where('generation', '<>', $generation)->delete();
|
|
if ($changes->available()) $changes->acknowledge($pending);
|
|
|
|
return ['status' => 'succeeded', 'source_rows' => $sourceRows, 'stored_rows' => $stored, 'changed' => true];
|
|
});
|
|
} catch (Throwable $exception) {
|
|
$this->connection->table('store_report_import_states')
|
|
->where('dataset_key', StoreReportRepository::PAID_SALES_DATASET)
|
|
->update([
|
|
'status' => 'failed',
|
|
'last_attempted_at' => now(),
|
|
'last_error_code' => class_basename($exception),
|
|
'updated_at' => now(),
|
|
]);
|
|
|
|
return [
|
|
'status' => 'failed',
|
|
'source_rows' => 0,
|
|
'stored_rows' => (int) ($state->row_count ?? 0),
|
|
'changed' => false,
|
|
];
|
|
}
|
|
}
|
|
|
|
private function syncPaidSalesDates(object $state, StoreReportPaidSalesChanges $changes): array
|
|
{
|
|
$pending = $changes->pending();
|
|
if ($pending === []) {
|
|
return ['status' => 'succeeded', 'source_rows' => 0, 'stored_rows' => (int) $state->row_count, 'changed' => false];
|
|
}
|
|
$dates = array_column($pending, 'occurred_on');
|
|
$generation = (string) $state->active_generation;
|
|
return $this->connection->transaction(function () use ($changes, $pending, $dates, $generation): array {
|
|
$this->connection->table(StoreReportRepository::PAID_SALES_TABLE)
|
|
->where('generation', $generation)->whereIn('occurred_on', $dates)->delete();
|
|
$this->connection->statement(
|
|
'INSERT INTO ' . StoreReportRepository::PAID_SALES_TABLE
|
|
. ' (generation, store_name, occurred_on, paid_amount, order_count, updated_at)'
|
|
. ' SELECT ?, store_name, occurred_date,'
|
|
. ' SUM(COALESCE(income_amount, 0) + COALESCE(discount_amount, 0)'
|
|
. ' + COALESCE(offset_amount, 0) + COALESCE(refund_amount, 0)), COUNT(*), ?'
|
|
. ' FROM ' . StoreReportRepository::PAID_SALES_SOURCE_TABLE
|
|
. " WHERE store_name IS NOT NULL AND store_name <> '' AND occurred_date IN ("
|
|
. implode(',', array_fill(0, count($dates), '?')) . ') GROUP BY store_name, occurred_date',
|
|
[$generation, now()->format('Y-m-d H:i:s'), ...$dates],
|
|
);
|
|
$stored = (int) $this->connection->table(StoreReportRepository::PAID_SALES_TABLE)->where('generation', $generation)->count();
|
|
$sourceRows = (int) $this->connection->table(StoreReportRepository::PAID_SALES_SOURCE_TABLE)->whereIn('occurred_date', $dates)->count();
|
|
$this->connection->table('store_report_import_states')
|
|
->where('dataset_key', StoreReportRepository::PAID_SALES_DATASET)->update([
|
|
'status' => 'succeeded', 'run_mode' => 'incremental', 'row_count' => $stored,
|
|
'source_fingerprint' => StoreReportPaidSalesChanges::VERSION . Str::uuid(),
|
|
'last_attempted_at' => now(), 'last_succeeded_at' => now(), 'last_error_code' => null, 'updated_at' => now(),
|
|
]);
|
|
$changes->acknowledge($pending);
|
|
return ['status' => 'succeeded', 'source_rows' => $sourceRows, 'stored_rows' => $stored, 'changed' => true];
|
|
});
|
|
}
|
|
|
|
/** 业绩明细是否变化的指纹：行数 + 最近写入时间 + 销量口径四项金额合计。 */
|
|
private function paidSalesFingerprint(): string
|
|
{
|
|
$row = $this->connection->selectOne(
|
|
'SELECT COUNT(*) AS rows_total, MAX(imported_at) AS last_imported_at,'
|
|
. ' COALESCE(SUM(COALESCE(income_amount, 0) + COALESCE(discount_amount, 0)'
|
|
. ' + COALESCE(offset_amount, 0) + COALESCE(refund_amount, 0)), 0) AS sales_total'
|
|
. ' FROM ' . StoreReportRepository::PAID_SALES_SOURCE_TABLE,
|
|
);
|
|
|
|
return hash('sha256', implode('|', [
|
|
(string) ($row->rows_total ?? 0),
|
|
(string) ($row->last_imported_at ?? ''),
|
|
(string) ($row->sales_total ?? '0'),
|
|
]));
|
|
}
|
|
|
|
/** @param array{upserts:array<int,array<string,mixed>>,deletes:array<int,int>} $mapped */
|
|
private function dailyDeltas(string $generation, array $mapped): array
|
|
{
|
|
$ids = array_values(array_unique(array_merge(
|
|
$mapped['deletes'],
|
|
array_map(static fn (array $row): int => (int) $row['source_id'], $mapped['upserts']),
|
|
)));
|
|
$deltas = [];
|
|
foreach (array_chunk($ids, 1000) as $chunk) {
|
|
$old = $this->connection->table('store_report_bill_rows')
|
|
->where('generation', $generation)->whereIn('source_id', $chunk)
|
|
->get(['store_name', 'occurred_at', 'income', 'expense'])
|
|
->map(static fn (object $row): array => (array) $row)->all();
|
|
$deltas = $this->mergeDailyDeltas($deltas, $this->aggregateDailyRows($old, -1.0));
|
|
}
|
|
return $this->mergeDailyDeltas($deltas, $this->aggregateDailyRows($mapped['upserts'], 1.0));
|
|
}
|
|
|
|
private function aggregateDailyRows(array $rows, float $sign): array
|
|
{
|
|
$result = [];
|
|
foreach ($rows as $row) {
|
|
$store = trim((string) ($row['store_name'] ?? ''));
|
|
$date = substr((string) ($row['occurred_at'] ?? ''), 0, 10);
|
|
if ($store === '' || strlen($date) !== 10) continue;
|
|
$key = $store . "\0" . $date;
|
|
$result[$key] ??= ['store_name' => $store, 'occurred_on' => $date, 'income' => 0.0, 'expense' => 0.0];
|
|
$result[$key]['income'] += $sign * (float) ($row['income'] ?? 0);
|
|
$result[$key]['expense'] += $sign * (float) ($row['expense'] ?? 0);
|
|
}
|
|
return $result;
|
|
}
|
|
|
|
private function mergeDailyDeltas(array $left, array $right): array
|
|
{
|
|
foreach ($right as $key => $row) {
|
|
$left[$key] ??= ['store_name' => $row['store_name'], 'occurred_on' => $row['occurred_on'], 'income' => 0.0, 'expense' => 0.0];
|
|
$left[$key]['income'] += $row['income'];
|
|
$left[$key]['expense'] += $row['expense'];
|
|
}
|
|
return $left;
|
|
}
|
|
|
|
private function applyDailyDeltas(string $generation, array $deltas): void
|
|
{
|
|
if ($deltas === []) return;
|
|
if ($this->connection->getDriverName() === 'sqlite') {
|
|
foreach ($deltas as $row) {
|
|
$current = $this->connection->table('store_report_daily_rows')
|
|
->where('generation', $generation)->where('store_name', $row['store_name'])
|
|
->where('occurred_on', $row['occurred_on'])->first();
|
|
$values = ['income' => (float) ($current->income ?? 0) + $row['income'],
|
|
'expense' => (float) ($current->expense ?? 0) + $row['expense'], 'updated_at' => now()];
|
|
$current === null
|
|
? $this->connection->table('store_report_daily_rows')->insert($values + [
|
|
'generation' => $generation, 'store_name' => $row['store_name'], 'occurred_on' => $row['occurred_on'],
|
|
])
|
|
: $this->connection->table('store_report_daily_rows')->where('id', $current->id)->update($values);
|
|
}
|
|
return;
|
|
}
|
|
foreach (array_chunk(array_values($deltas), 500) as $chunk) {
|
|
$bindings = [];
|
|
$values = [];
|
|
foreach ($chunk as $row) {
|
|
$values[] = '(?,?,?,?,?,?)';
|
|
array_push($bindings, $generation, $row['store_name'], $row['occurred_on'],
|
|
$row['income'], $row['expense'], now()->format('Y-m-d H:i:s'));
|
|
}
|
|
$this->connection->statement(
|
|
'INSERT INTO store_report_daily_rows (generation,store_name,occurred_on,income,expense,updated_at) VALUES '
|
|
. implode(',', $values)
|
|
. ' ON DUPLICATE KEY UPDATE income=income+VALUES(income), expense=expense+VALUES(expense), updated_at=VALUES(updated_at)',
|
|
$bindings,
|
|
);
|
|
}
|
|
}
|
|
|
|
/** @return array{status:string,mode:string,batches:int,source_rows:int,stored_rows:int,cursor_id:int} */
|
|
private function result(string $status, string $mode, int $batches, int $sourceRows, int $storedRows, int $cursor): array
|
|
{
|
|
return [
|
|
'status' => $status,
|
|
'mode' => $mode,
|
|
'batches' => $batches,
|
|
'source_rows' => $sourceRows,
|
|
'stored_rows' => $storedRows,
|
|
'cursor_id' => $cursor,
|
|
];
|
|
}
|
|
|
|
/** @return array{upserts:array<int,array<string,mixed>>,deletes:array<int,int>} */
|
|
private function mapBills(array $rows, string $generation): array
|
|
{
|
|
$upserts = [];
|
|
$deletes = [];
|
|
foreach ($rows as $row) {
|
|
$sourceId = (int) ($row['source_id'] ?? 0);
|
|
$storeName = $this->text($row['shop_name'] ?? null, 191);
|
|
$occurredAt = $this->dateTime($row['fee_time'] ?? null);
|
|
$direction = (string) ($row['inout_type'] ?? '');
|
|
if ($sourceId < 1 || (int) ($row['del_flag'] ?? 0) !== 0 || $storeName === null
|
|
|| $occurredAt === null || ! in_array($direction, ['1', '-1'], true)) {
|
|
if ($sourceId > 0) {
|
|
$deletes[] = $sourceId;
|
|
}
|
|
continue;
|
|
}
|
|
$amount = abs((float) ($row['amount'] ?? 0));
|
|
$platform = $this->text($row['channel'] ?? null, 32) ?? 'unknown';
|
|
$business = [$sourceId, $platform, $storeName, $occurredAt, $direction, $amount];
|
|
$upserts[] = [
|
|
'generation' => $generation,
|
|
'source_id' => $sourceId,
|
|
'shop_id' => (int) ($row['shop_id'] ?? 0) ?: null,
|
|
'platform' => $platform,
|
|
'platform_label' => $this->platformLabel($platform),
|
|
'store_name' => $storeName,
|
|
'occurred_at' => $occurredAt,
|
|
'income' => $direction === '1' ? $amount : 0,
|
|
'expense' => $direction === '-1' ? $amount : 0,
|
|
'bill_item' => $this->text($row['fee_name'] ?? null, 255),
|
|
'record_code' => (string) $sourceId,
|
|
'source_file' => 'jiansun.dbo.jst_originalBillInfo',
|
|
'row_hash' => hash('sha256', json_encode($business, JSON_UNESCAPED_UNICODE)),
|
|
'source_updated_at' => $this->dateTime($row['source_updated_at'] ?? null),
|
|
'imported_at' => now(),
|
|
];
|
|
}
|
|
return ['upserts' => $upserts, 'deletes' => $deletes];
|
|
}
|
|
|
|
/** @return array{upserts:array<int,array<string,mixed>>,deletes:array<int,int>} */
|
|
private function mapLifecycles(array $rows, string $generation): array
|
|
{
|
|
$upserts = [];
|
|
$deletes = [];
|
|
foreach ($rows as $row) {
|
|
$sourceId = (int) ($row['source_id'] ?? 0);
|
|
$storeName = $this->text($row['erp_store_name'] ?? null, 191);
|
|
$creditCode = $this->text($row['unified_social_credit_code'] ?? null, 64);
|
|
$effectiveAt = $this->dateTime($row['effective_at'] ?? null);
|
|
if ($sourceId < 1 || $storeName === null || $creditCode === null || $effectiveAt === null) {
|
|
if ($sourceId > 0) {
|
|
$deletes[] = $sourceId;
|
|
}
|
|
continue;
|
|
}
|
|
$business = [$sourceId, $storeName, $creditCode, $effectiveAt];
|
|
$upserts[] = [
|
|
'generation' => $generation,
|
|
'source_id' => $sourceId,
|
|
'business_key' => hash('sha256', (string) $sourceId),
|
|
'erp_store_name' => $storeName,
|
|
'company_name' => $this->text($row['company_name'] ?? null, 191),
|
|
'unified_social_credit_code' => $creditCode,
|
|
'effective_at' => $effectiveAt,
|
|
'expires_at' => null,
|
|
'source_file' => 'jiansun.dbo.fys_shopInfo_flow',
|
|
'row_hash' => hash('sha256', json_encode($business, JSON_UNESCAPED_UNICODE)),
|
|
'imported_at' => now(),
|
|
];
|
|
}
|
|
return ['upserts' => $upserts, 'deletes' => $deletes];
|
|
}
|
|
|
|
private function platformLabel(string $channel): string
|
|
{
|
|
return match ($channel) {
|
|
'alipayCSV' => '淘系',
|
|
'TouTiaoFXGZJ' => '抖音',
|
|
'PinduoduoImport' => '拼多多',
|
|
'Jos' => '京东',
|
|
default => $channel,
|
|
};
|
|
}
|
|
|
|
private function text(mixed $value, int $length): ?string
|
|
{
|
|
$value = trim((string) ($value ?? ''));
|
|
if ($value === '') {
|
|
return null;
|
|
}
|
|
if (! mb_check_encoding($value, 'UTF-8')) {
|
|
$value = mb_convert_encoding($value, 'UTF-8', 'GB18030');
|
|
}
|
|
return mb_substr($value, 0, $length);
|
|
}
|
|
|
|
private function sameCompany(mixed $left, mixed $right): bool
|
|
{
|
|
$left = mb_strtolower(trim((string) ($left ?? '')));
|
|
$right = mb_strtolower(trim((string) ($right ?? '')));
|
|
|
|
return $left !== '' && $left === $right;
|
|
}
|
|
|
|
private function dateTime(mixed $value): ?string
|
|
{
|
|
$value = trim((string) ($value ?? ''));
|
|
return $value === '' ? null : str_replace('T', ' ', substr($value, 0, 26));
|
|
}
|
|
|
|
private function assertSchemaReady(): void
|
|
{
|
|
$schema = Schema::connection($this->connection->getName());
|
|
foreach (['store_report_import_states', 'store_report_bill_rows', 'store_report_lifecycle_rows', 'store_report_daily_rows'] as $table) {
|
|
if (! $schema->hasTable($table)) {
|
|
throw new RuntimeException('store_report_mirror_schema_missing:' . $table);
|
|
}
|
|
}
|
|
foreach (['run_mode', 'cursor_id', 'initial_completed_at'] as $column) {
|
|
if (! $schema->hasColumn('store_report_import_states', $column)) {
|
|
throw new RuntimeException('store_report_mirror_upgrade_missing:' . $column);
|
|
}
|
|
}
|
|
}
|
|
}
|