避免支付宝账单在增量同步期间反复全表计数 #142

Merged
WuYongJie merged 3 commits from fix/WuYongJie-live-mirror-count into main 2026-09-09 14:38:44 +08:00
7 changed files with 132 additions and 9 deletions

View File

@ -0,0 +1,26 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
if (config('system_database.mode', 'bootstrap') === 'bootstrap') return;
$schema = Schema::connection((string) config('upstream_mirrors.connection', 'business_mirror'));
if ($schema->hasTable('business_mirror_table_states') && ! $schema->hasColumn('business_mirror_table_states', 'mirror_row_count_is_exact')) {
$schema->table('business_mirror_table_states', static fn (Blueprint $t) => $t->boolean('mirror_row_count_is_exact')->default(false));
}
}
public function down(): void
{
if (config('system_database.mode', 'bootstrap') === 'bootstrap') return;
$schema = Schema::connection((string) config('upstream_mirrors.connection', 'business_mirror'));
if ($schema->hasTable('business_mirror_table_states') && $schema->hasColumn('business_mirror_table_states', 'mirror_row_count_is_exact')) {
$schema->table('business_mirror_table_states', static fn (Blueprint $t) => $t->dropColumn('mirror_row_count_is_exact'));
}
}
};

View File

@ -11,6 +11,7 @@ use RuntimeException;
class GenericBusinessMirrorRepository
{
private readonly ConnectionInterface $connection;
private ?bool $supportsExactCount = null;
public function __construct(DatabaseManager $database)
{
@ -93,7 +94,14 @@ class GenericBusinessMirrorRepository
$query = $this->connection->table('business_mirror_table_states')
->where('dataset_key', $dataset->key());
if ($query->exists()) {
$existing = $query->first();
if ($this->supportsExactCount()) {
$values['mirror_row_count_is_exact'] = $mode === 'incremental'
&& $existing !== null && $existing->initial_completed_at !== null
&& ctype_digit((string) ($existing->mirror_row_count ?? ''))
&& ($existing->status === 'succeeded' || (bool) ($existing->mirror_row_count_is_exact ?? false));
}
if ($existing !== null) {
$query->update($values);
} else {
$this->connection->table('business_mirror_table_states')->insert(
@ -137,6 +145,16 @@ class GenericBusinessMirrorRepository
$this->connection->transaction(function () use (
$dataset, $columnNames, $mapped, $cursor, $cumulativeRows, $now
): void {
$exactCount = null;
if ($this->supportsExactCount()) {
$state = $this->connection->table('business_mirror_table_states')
->where('dataset_key', $dataset->key())->lockForUpdate()->first();
if ($state && $state->run_mode === 'incremental' && $state->mirror_row_count_is_exact) {
$keys = array_values(array_unique(array_column($mapped, '_mirror_key')));
$existing = $this->connection->table($dataset->targetTable())->whereIn('_mirror_key', $keys)->count();
$exactCount = (int) $state->mirror_row_count + count($keys) - $existing;
}
}
$updates = [...$columnNames, '_mirror_generation', '_mirror_synced_at'];
$chunkSize = max(1, intdiv(50000, max(count($updates) + 1, 1)));
@ -150,7 +168,7 @@ class GenericBusinessMirrorRepository
'cursor' => json_encode($cursor, JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION),
'last_upserted_rows' => $cumulativeRows,
'updated_at' => $now,
]);
] + ($exactCount === null ? [] : ['mirror_row_count' => $exactCount]));
});
}
@ -181,7 +199,12 @@ class GenericBusinessMirrorRepository
->delete();
}
$rowCount = $this->connection->table($dataset->targetTable())->count();
$countState = $this->supportsExactCount()
? $this->connection->table('business_mirror_table_states')->where('dataset_key', $dataset->key())->lockForUpdate()->first()
: null;
$rowCount = $mode === 'incremental' && $countState && $countState->mirror_row_count_is_exact
? (int) $countState->mirror_row_count
: $this->connection->table($dataset->targetTable())->count();
$values = [
'status' => 'succeeded',
'cursor' => null,
@ -197,6 +220,7 @@ class GenericBusinessMirrorRepository
$values['initial_completed_at'] = $now;
$values['last_full_succeeded_at'] = $now;
}
if ($this->supportsExactCount()) $values['mirror_row_count_is_exact'] = true;
$this->connection->table('business_mirror_table_states')
->where('dataset_key', $dataset->key())
@ -215,6 +239,12 @@ class GenericBusinessMirrorRepository
]);
}
private function supportsExactCount(): bool
{
return $this->supportsExactCount ??= Schema::connection($this->connection->getName())
->hasColumn('business_mirror_table_states', 'mirror_row_count_is_exact');
}
private function decode(mixed $value): ?array
{
if (is_array($value)) {

View File

@ -62,11 +62,12 @@ class StoreReceivingAlipayBillMirrorRepository
$this->applyChannel($query, $channel);
$this->applyFilters($query, $filters);
$state = $this->mirror->state($dataset) ?? [];
// The synchronizer persists an exact COUNT in the same transaction that
// marks a run successful. Never substitute a database row estimate.
// Completed runs persist an exact COUNT. New incremental batches keep
// that count exact in their upsert transaction, including inserts.
$hasFilters = collect($filters)->contains(static fn (mixed $value): bool => $value !== null && trim((string) $value) !== '');
$recordedCount = $state['mirror_row_count'] ?? null;
$useRecordedCount = $channel === null && ! $hasFilters && ($state['status'] ?? null) === 'succeeded'
$useRecordedCount = $channel === null && ! $hasFilters
&& (($state['status'] ?? null) === 'succeeded' || (bool) ($state['mirror_row_count_is_exact'] ?? false))
&& $recordedCount !== null && ctype_digit((string) $recordedCount);
$total = $useRecordedCount ? (int) $recordedCount : (clone $query)->count();
$summary = $this->summary($query, $filters, $channel);

View File

@ -94,7 +94,7 @@
@endforeach
</div><div class="sm-column-picker__foot"><button class="sm-button" type="button" data-column-reset><span class="material-icons-outlined" aria-hidden="true">restart_alt</span>恢复默认</button></div></div></div><button class="sm-button" type="button" data-surcharge-export-current><span class="material-icons-outlined" aria-hidden="true">download</span>导出当前列</button><button class="sm-button" type="button" data-surcharge-export-all><span class="material-icons-outlined" aria-hidden="true">file_download</span>导出全部字段</button><span class="sm-card__hint">规则按生效时间应用到后续结算页面</span></div></div>
<div class="sm-table-meta"><span>当前筛选结果(全部):<strong data-surcharge-row-count>{{ $rows->count() }}</strong> 条</span><span>规则编号与操作列固定显示;列设置仅保存于当前用户</span></div>
<div class="sm-table-wrap" data-surcharge-table-wrap><table class="sm-table" data-surcharge-table><thead><tr>
<div class="sm-table-wrap" data-surcharge-table-wrap><table class="sm-table" data-finance-table-native data-surcharge-table><thead><tr>
@foreach ($columns as $column)
<th scope="col" draggable="{{ empty($column['fixed']) ? 'true' : 'false' }}" data-column-header data-column-key="{{ $column['key'] }}" data-column-label="{{ $column['label'] }}" data-fixed="{{ !empty($column['fixed']) ? 'true' : 'false' }}"><span class="sm-table__header">@if (empty($column['fixed']))<span class="sm-table__drag material-icons-outlined" aria-hidden="true">drag_handle</span>@endif<span>{{ $column['label'] }}</span></span></th>
@endforeach

View File

@ -303,7 +303,11 @@
return /^[=+\-@]/u.test(text) ? `'${text}` : text;
};
const csvEscape = (value) => `"${csvText(value).replaceAll('"', '""')}"`;
const cellText = (cell) => (cell?.querySelector('[data-password-status]') || cell)?.textContent.trim() || '';
const cellText = (cell) => {
const passwordStatus = cell?.querySelector('[data-password-status]');
if (passwordStatus) return passwordStatus.textContent.trim();
return cell?.dataset.exportValue ?? cell?.textContent.trim() ?? '';
};
const downloadCsv = (filename, lines) => {
const blob = new Blob([`\ufeff${lines.join('\r\n')}`], { type: 'text/csv;charset=utf-8' });
const url = URL.createObjectURL(blob);
@ -669,7 +673,7 @@
}
function initialise(table) {
if (!table || tableNames.has(table)) return;
if (!table || table.hasAttribute('data-finance-table-native') || tableNames.has(table)) return;
const prepared = prepareColumns(table);
if (!prepared.headers.length) return;
tableNames.set(table, true);

View File

@ -65,6 +65,7 @@ class BusinessMirrorSynchronizerTest extends TestCase
$table->dateTime('last_succeeded_at')->nullable();
$table->unsignedBigInteger('last_upserted_rows')->default(0);
$table->unsignedBigInteger('mirror_row_count')->default(0);
$table->boolean('mirror_row_count_is_exact')->default(false);
$table->string('last_error_code')->nullable();
$table->timestamps();
});
@ -335,6 +336,60 @@ class BusinessMirrorSynchronizerTest extends TestCase
$this->assertSame($value, DB::connection('business_mirror')->table($dataset->targetTable())->value('displayName'));
}
public function test_incremental_batches_keep_exact_counts_without_counting_updates_as_inserts(): void
{
$this->query->batches = [[
['Id' => 1, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => 'one'],
['Id' => 2, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => 'two'],
]];
$this->synchronizer->sync('test.dbo.records', true);
$db = DB::connection('business_mirror');
$state = fn () => $db->table('business_mirror_table_states')->where('dataset_key', 'test.dbo.records')->first();
$this->assertTrue((bool) $state()->mirror_row_count_is_exact);
$this->query->batches = [[
['Id' => 1, 'updateTime' => '2026-08-15 10:00:00', 'displayName' => 'updated'],
['Id' => 3, 'updateTime' => '2026-08-15 10:00:00', 'displayName' => 'new'],
]];
$this->synchronizer->sync('test.dbo.records', false, 2, 1);
$this->assertSame('running', $state()->status);
$this->assertSame(3, (int) $state()->mirror_row_count);
$this->assertTrue((bool) $state()->mirror_row_count_is_exact);
$this->assertSame(3, $db->table('test_records')->count());
$this->query->batches = [[
['Id' => 4, 'updateTime' => '2026-08-15 10:00:00', 'displayName' => 'rollback-a'],
['Id' => 5, 'updateTime' => '2026-08-15 10:00:00', 'displayName' => 'rollback-b'],
]];
$db->beginTransaction();
$this->synchronizer->sync('test.dbo.records', false, 2, 1);
$this->assertSame(5, (int) $state()->mirror_row_count);
$db->rollBack();
$this->assertSame(3, (int) $state()->mirror_row_count);
$this->assertSame(3, $db->table('test_records')->count());
$db->enableQueryLog();
$db->flushQueryLog();
$this->query->batches = [[]];
$this->synchronizer->sync('test.dbo.records');
$this->assertSame(3, (int) $state()->mirror_row_count);
$this->assertCount(0, array_filter($db->getQueryLog(), static fn ($q) => str_contains(strtolower($q['query']), 'count(*)') && str_contains($q['query'], 'test_records')));
$db->disableQueryLog();
}
public function test_legacy_running_count_is_not_trusted_until_the_run_completes(): void
{
$this->query->batches = [[['Id' => 1, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => 'one']]];
$this->synchronizer->sync('test.dbo.records', true);
$db = DB::connection('business_mirror');
$query = $db->table('business_mirror_table_states')->where('dataset_key', 'test.dbo.records');
$query->update(['status' => 'running', 'run_mode' => 'incremental', 'cursor' => '{"Id":1}', 'mirror_row_count' => 999, 'mirror_row_count_is_exact' => false]);
$this->query->batches = [[['Id' => 2, 'updateTime' => '2026-08-15 10:00:00', 'displayName' => 'two']]];
$this->synchronizer->sync('test.dbo.records', false, 1, 1);
$this->assertFalse((bool) $query->first()->mirror_row_count_is_exact);
$this->query->batches = [[]];
$this->synchronizer->sync('test.dbo.records');
$this->assertSame(2, (int) $query->first()->mirror_row_count);
$this->assertTrue((bool) $query->first()->mirror_row_count_is_exact);
}
public function test_empty_full_read_does_not_delete_previous_mirror_rows(): void
{
$this->query->batches = [[

View File

@ -45,6 +45,7 @@ class StoreReceivingAlipayBillMirrorRepositoryTest extends TestCase
$table->string('status')->nullable();
$table->dateTime('last_succeeded_at')->nullable();
$table->unsignedBigInteger('mirror_row_count')->nullable();
$table->boolean('mirror_row_count_is_exact')->default(false);
});
$schema->create('ops_jst_original_bill_info', function (Blueprint $table): void {
$table->integer('Id');
@ -190,6 +191,12 @@ class StoreReceivingAlipayBillMirrorRepositoryTest extends TestCase
$this->assertSame(3, $this->repository->paginate([], 'occurred_at', 'desc', 1, 1)['total']);
$state->update(['status' => 'failed']);
$this->assertSame(3, $this->repository->paginate([], 'occurred_at', 'desc', 1, 1)['total']);
$state->update(['status' => 'running', 'mirror_row_count' => 3, 'mirror_row_count_is_exact' => true]);
$db->flushQueryLog();
$db->enableQueryLog();
$this->assertSame(3, $this->repository->paginate([], 'occurred_at', 'desc', 1, 1)['total']);
$this->assertCount(0, array_filter($db->getQueryLog(), fn ($q) => str_contains(strtolower($q['query']), 'count(*)')));
$db->disableQueryLog();
$state->update(['status' => 'succeeded', 'mirror_row_count' => null]);
$this->assertSame(3, $this->repository->paginate([], 'occurred_at', 'desc', 1, 1)['total']);
}