避免支付宝账单在增量同步期间反复全表计数 #142
@ -0,0 +1,26 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
public function up(): void
|
||||
{
|
||||
if (config('system_database.mode', 'bootstrap') === 'bootstrap') return;
|
||||
$schema = Schema::connection((string) config('upstream_mirrors.connection', 'business_mirror'));
|
||||
if ($schema->hasTable('business_mirror_table_states') && ! $schema->hasColumn('business_mirror_table_states', 'mirror_row_count_is_exact')) {
|
||||
$schema->table('business_mirror_table_states', static fn (Blueprint $t) => $t->boolean('mirror_row_count_is_exact')->default(false));
|
||||
}
|
||||
}
|
||||
|
||||
public function down(): void
|
||||
{
|
||||
if (config('system_database.mode', 'bootstrap') === 'bootstrap') return;
|
||||
$schema = Schema::connection((string) config('upstream_mirrors.connection', 'business_mirror'));
|
||||
if ($schema->hasTable('business_mirror_table_states') && $schema->hasColumn('business_mirror_table_states', 'mirror_row_count_is_exact')) {
|
||||
$schema->table('business_mirror_table_states', static fn (Blueprint $t) => $t->dropColumn('mirror_row_count_is_exact'));
|
||||
}
|
||||
}
|
||||
};
|
||||
@ -11,6 +11,7 @@ use RuntimeException;
|
||||
class GenericBusinessMirrorRepository
|
||||
{
|
||||
private readonly ConnectionInterface $connection;
|
||||
private ?bool $supportsExactCount = null;
|
||||
|
||||
public function __construct(DatabaseManager $database)
|
||||
{
|
||||
@ -93,7 +94,14 @@ class GenericBusinessMirrorRepository
|
||||
$query = $this->connection->table('business_mirror_table_states')
|
||||
->where('dataset_key', $dataset->key());
|
||||
|
||||
if ($query->exists()) {
|
||||
$existing = $query->first();
|
||||
if ($this->supportsExactCount()) {
|
||||
$values['mirror_row_count_is_exact'] = $mode === 'incremental'
|
||||
&& $existing !== null && $existing->initial_completed_at !== null
|
||||
&& ctype_digit((string) ($existing->mirror_row_count ?? ''))
|
||||
&& ($existing->status === 'succeeded' || (bool) ($existing->mirror_row_count_is_exact ?? false));
|
||||
}
|
||||
if ($existing !== null) {
|
||||
$query->update($values);
|
||||
} else {
|
||||
$this->connection->table('business_mirror_table_states')->insert(
|
||||
@ -137,6 +145,16 @@ class GenericBusinessMirrorRepository
|
||||
$this->connection->transaction(function () use (
|
||||
$dataset, $columnNames, $mapped, $cursor, $cumulativeRows, $now
|
||||
): void {
|
||||
$exactCount = null;
|
||||
if ($this->supportsExactCount()) {
|
||||
$state = $this->connection->table('business_mirror_table_states')
|
||||
->where('dataset_key', $dataset->key())->lockForUpdate()->first();
|
||||
if ($state && $state->run_mode === 'incremental' && $state->mirror_row_count_is_exact) {
|
||||
$keys = array_values(array_unique(array_column($mapped, '_mirror_key')));
|
||||
$existing = $this->connection->table($dataset->targetTable())->whereIn('_mirror_key', $keys)->count();
|
||||
$exactCount = (int) $state->mirror_row_count + count($keys) - $existing;
|
||||
}
|
||||
}
|
||||
$updates = [...$columnNames, '_mirror_generation', '_mirror_synced_at'];
|
||||
$chunkSize = max(1, intdiv(50000, max(count($updates) + 1, 1)));
|
||||
|
||||
@ -150,7 +168,7 @@ class GenericBusinessMirrorRepository
|
||||
'cursor' => json_encode($cursor, JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION),
|
||||
'last_upserted_rows' => $cumulativeRows,
|
||||
'updated_at' => $now,
|
||||
]);
|
||||
] + ($exactCount === null ? [] : ['mirror_row_count' => $exactCount]));
|
||||
});
|
||||
}
|
||||
|
||||
@ -181,7 +199,12 @@ class GenericBusinessMirrorRepository
|
||||
->delete();
|
||||
}
|
||||
|
||||
$rowCount = $this->connection->table($dataset->targetTable())->count();
|
||||
$countState = $this->supportsExactCount()
|
||||
? $this->connection->table('business_mirror_table_states')->where('dataset_key', $dataset->key())->lockForUpdate()->first()
|
||||
: null;
|
||||
$rowCount = $mode === 'incremental' && $countState && $countState->mirror_row_count_is_exact
|
||||
? (int) $countState->mirror_row_count
|
||||
: $this->connection->table($dataset->targetTable())->count();
|
||||
$values = [
|
||||
'status' => 'succeeded',
|
||||
'cursor' => null,
|
||||
@ -197,6 +220,7 @@ class GenericBusinessMirrorRepository
|
||||
$values['initial_completed_at'] = $now;
|
||||
$values['last_full_succeeded_at'] = $now;
|
||||
}
|
||||
if ($this->supportsExactCount()) $values['mirror_row_count_is_exact'] = true;
|
||||
|
||||
$this->connection->table('business_mirror_table_states')
|
||||
->where('dataset_key', $dataset->key())
|
||||
@ -215,6 +239,12 @@ class GenericBusinessMirrorRepository
|
||||
]);
|
||||
}
|
||||
|
||||
private function supportsExactCount(): bool
|
||||
{
|
||||
return $this->supportsExactCount ??= Schema::connection($this->connection->getName())
|
||||
->hasColumn('business_mirror_table_states', 'mirror_row_count_is_exact');
|
||||
}
|
||||
|
||||
private function decode(mixed $value): ?array
|
||||
{
|
||||
if (is_array($value)) {
|
||||
|
||||
@ -62,11 +62,12 @@ class StoreReceivingAlipayBillMirrorRepository
|
||||
$this->applyChannel($query, $channel);
|
||||
$this->applyFilters($query, $filters);
|
||||
$state = $this->mirror->state($dataset) ?? [];
|
||||
// The synchronizer persists an exact COUNT in the same transaction that
|
||||
// marks a run successful. Never substitute a database row estimate.
|
||||
// Completed runs persist an exact COUNT. New incremental batches keep
|
||||
// that count exact in their upsert transaction, including inserts.
|
||||
$hasFilters = collect($filters)->contains(static fn (mixed $value): bool => $value !== null && trim((string) $value) !== '');
|
||||
$recordedCount = $state['mirror_row_count'] ?? null;
|
||||
$useRecordedCount = $channel === null && ! $hasFilters && ($state['status'] ?? null) === 'succeeded'
|
||||
$useRecordedCount = $channel === null && ! $hasFilters
|
||||
&& (($state['status'] ?? null) === 'succeeded' || (bool) ($state['mirror_row_count_is_exact'] ?? false))
|
||||
&& $recordedCount !== null && ctype_digit((string) $recordedCount);
|
||||
$total = $useRecordedCount ? (int) $recordedCount : (clone $query)->count();
|
||||
$summary = $this->summary($query, $filters, $channel);
|
||||
|
||||
@ -94,7 +94,7 @@
|
||||
@endforeach
|
||||
</div><div class="sm-column-picker__foot"><button class="sm-button" type="button" data-column-reset><span class="material-icons-outlined" aria-hidden="true">restart_alt</span>恢复默认</button></div></div></div><button class="sm-button" type="button" data-surcharge-export-current><span class="material-icons-outlined" aria-hidden="true">download</span>导出当前列</button><button class="sm-button" type="button" data-surcharge-export-all><span class="material-icons-outlined" aria-hidden="true">file_download</span>导出全部字段</button><span class="sm-card__hint">规则按生效时间应用到后续结算页面</span></div></div>
|
||||
<div class="sm-table-meta"><span>当前筛选结果(全部):<strong data-surcharge-row-count>{{ $rows->count() }}</strong> 条</span><span>规则编号与操作列固定显示;列设置仅保存于当前用户</span></div>
|
||||
<div class="sm-table-wrap" data-surcharge-table-wrap><table class="sm-table" data-surcharge-table><thead><tr>
|
||||
<div class="sm-table-wrap" data-surcharge-table-wrap><table class="sm-table" data-finance-table-native data-surcharge-table><thead><tr>
|
||||
@foreach ($columns as $column)
|
||||
<th scope="col" draggable="{{ empty($column['fixed']) ? 'true' : 'false' }}" data-column-header data-column-key="{{ $column['key'] }}" data-column-label="{{ $column['label'] }}" data-fixed="{{ !empty($column['fixed']) ? 'true' : 'false' }}"><span class="sm-table__header">@if (empty($column['fixed']))<span class="sm-table__drag material-icons-outlined" aria-hidden="true">drag_handle</span>@endif<span>{{ $column['label'] }}</span></span></th>
|
||||
@endforeach
|
||||
|
||||
@ -303,7 +303,11 @@
|
||||
return /^[=+\-@]/u.test(text) ? `'${text}` : text;
|
||||
};
|
||||
const csvEscape = (value) => `"${csvText(value).replaceAll('"', '""')}"`;
|
||||
const cellText = (cell) => (cell?.querySelector('[data-password-status]') || cell)?.textContent.trim() || '';
|
||||
const cellText = (cell) => {
|
||||
const passwordStatus = cell?.querySelector('[data-password-status]');
|
||||
if (passwordStatus) return passwordStatus.textContent.trim();
|
||||
return cell?.dataset.exportValue ?? cell?.textContent.trim() ?? '';
|
||||
};
|
||||
const downloadCsv = (filename, lines) => {
|
||||
const blob = new Blob([`\ufeff${lines.join('\r\n')}`], { type: 'text/csv;charset=utf-8' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
@ -669,7 +673,7 @@
|
||||
}
|
||||
|
||||
function initialise(table) {
|
||||
if (!table || tableNames.has(table)) return;
|
||||
if (!table || table.hasAttribute('data-finance-table-native') || tableNames.has(table)) return;
|
||||
const prepared = prepareColumns(table);
|
||||
if (!prepared.headers.length) return;
|
||||
tableNames.set(table, true);
|
||||
|
||||
@ -65,6 +65,7 @@ class BusinessMirrorSynchronizerTest extends TestCase
|
||||
$table->dateTime('last_succeeded_at')->nullable();
|
||||
$table->unsignedBigInteger('last_upserted_rows')->default(0);
|
||||
$table->unsignedBigInteger('mirror_row_count')->default(0);
|
||||
$table->boolean('mirror_row_count_is_exact')->default(false);
|
||||
$table->string('last_error_code')->nullable();
|
||||
$table->timestamps();
|
||||
});
|
||||
@ -335,6 +336,60 @@ class BusinessMirrorSynchronizerTest extends TestCase
|
||||
$this->assertSame($value, DB::connection('business_mirror')->table($dataset->targetTable())->value('displayName'));
|
||||
}
|
||||
|
||||
public function test_incremental_batches_keep_exact_counts_without_counting_updates_as_inserts(): void
|
||||
{
|
||||
$this->query->batches = [[
|
||||
['Id' => 1, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => 'one'],
|
||||
['Id' => 2, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => 'two'],
|
||||
]];
|
||||
$this->synchronizer->sync('test.dbo.records', true);
|
||||
$db = DB::connection('business_mirror');
|
||||
$state = fn () => $db->table('business_mirror_table_states')->where('dataset_key', 'test.dbo.records')->first();
|
||||
$this->assertTrue((bool) $state()->mirror_row_count_is_exact);
|
||||
$this->query->batches = [[
|
||||
['Id' => 1, 'updateTime' => '2026-08-15 10:00:00', 'displayName' => 'updated'],
|
||||
['Id' => 3, 'updateTime' => '2026-08-15 10:00:00', 'displayName' => 'new'],
|
||||
]];
|
||||
$this->synchronizer->sync('test.dbo.records', false, 2, 1);
|
||||
$this->assertSame('running', $state()->status);
|
||||
$this->assertSame(3, (int) $state()->mirror_row_count);
|
||||
$this->assertTrue((bool) $state()->mirror_row_count_is_exact);
|
||||
$this->assertSame(3, $db->table('test_records')->count());
|
||||
$this->query->batches = [[
|
||||
['Id' => 4, 'updateTime' => '2026-08-15 10:00:00', 'displayName' => 'rollback-a'],
|
||||
['Id' => 5, 'updateTime' => '2026-08-15 10:00:00', 'displayName' => 'rollback-b'],
|
||||
]];
|
||||
$db->beginTransaction();
|
||||
$this->synchronizer->sync('test.dbo.records', false, 2, 1);
|
||||
$this->assertSame(5, (int) $state()->mirror_row_count);
|
||||
$db->rollBack();
|
||||
$this->assertSame(3, (int) $state()->mirror_row_count);
|
||||
$this->assertSame(3, $db->table('test_records')->count());
|
||||
$db->enableQueryLog();
|
||||
$db->flushQueryLog();
|
||||
$this->query->batches = [[]];
|
||||
$this->synchronizer->sync('test.dbo.records');
|
||||
$this->assertSame(3, (int) $state()->mirror_row_count);
|
||||
$this->assertCount(0, array_filter($db->getQueryLog(), static fn ($q) => str_contains(strtolower($q['query']), 'count(*)') && str_contains($q['query'], 'test_records')));
|
||||
$db->disableQueryLog();
|
||||
}
|
||||
|
||||
public function test_legacy_running_count_is_not_trusted_until_the_run_completes(): void
|
||||
{
|
||||
$this->query->batches = [[['Id' => 1, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => 'one']]];
|
||||
$this->synchronizer->sync('test.dbo.records', true);
|
||||
$db = DB::connection('business_mirror');
|
||||
$query = $db->table('business_mirror_table_states')->where('dataset_key', 'test.dbo.records');
|
||||
$query->update(['status' => 'running', 'run_mode' => 'incremental', 'cursor' => '{"Id":1}', 'mirror_row_count' => 999, 'mirror_row_count_is_exact' => false]);
|
||||
$this->query->batches = [[['Id' => 2, 'updateTime' => '2026-08-15 10:00:00', 'displayName' => 'two']]];
|
||||
$this->synchronizer->sync('test.dbo.records', false, 1, 1);
|
||||
$this->assertFalse((bool) $query->first()->mirror_row_count_is_exact);
|
||||
$this->query->batches = [[]];
|
||||
$this->synchronizer->sync('test.dbo.records');
|
||||
$this->assertSame(2, (int) $query->first()->mirror_row_count);
|
||||
$this->assertTrue((bool) $query->first()->mirror_row_count_is_exact);
|
||||
}
|
||||
|
||||
public function test_empty_full_read_does_not_delete_previous_mirror_rows(): void
|
||||
{
|
||||
$this->query->batches = [[
|
||||
|
||||
@ -45,6 +45,7 @@ class StoreReceivingAlipayBillMirrorRepositoryTest extends TestCase
|
||||
$table->string('status')->nullable();
|
||||
$table->dateTime('last_succeeded_at')->nullable();
|
||||
$table->unsignedBigInteger('mirror_row_count')->nullable();
|
||||
$table->boolean('mirror_row_count_is_exact')->default(false);
|
||||
});
|
||||
$schema->create('ops_jst_original_bill_info', function (Blueprint $table): void {
|
||||
$table->integer('Id');
|
||||
@ -190,6 +191,12 @@ class StoreReceivingAlipayBillMirrorRepositoryTest extends TestCase
|
||||
$this->assertSame(3, $this->repository->paginate([], 'occurred_at', 'desc', 1, 1)['total']);
|
||||
$state->update(['status' => 'failed']);
|
||||
$this->assertSame(3, $this->repository->paginate([], 'occurred_at', 'desc', 1, 1)['total']);
|
||||
$state->update(['status' => 'running', 'mirror_row_count' => 3, 'mirror_row_count_is_exact' => true]);
|
||||
$db->flushQueryLog();
|
||||
$db->enableQueryLog();
|
||||
$this->assertSame(3, $this->repository->paginate([], 'occurred_at', 'desc', 1, 1)['total']);
|
||||
$this->assertCount(0, array_filter($db->getQueryLog(), fn ($q) => str_contains(strtolower($q['query']), 'count(*)')));
|
||||
$db->disableQueryLog();
|
||||
$state->update(['status' => 'succeeded', 'mirror_row_count' => null]);
|
||||
$this->assertSame(3, $this->repository->paginate([], 'occurred_at', 'desc', 1, 1)['total']);
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user