From 1469c2b0b4ab3aecf19fd11c625778284f53c7e0 Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Wed, 9 Sep 2026 14:30:44 +0800 Subject: [PATCH 1/2] fix: maintain exact mirror counts during incremental sync --- ...150000_track_exact_mirror_batch_counts.php | 26 +++++++++ .../GenericBusinessMirrorRepository.php | 36 +++++++++++- ...oreReceivingAlipayBillMirrorRepository.php | 7 ++- .../BusinessMirrorSynchronizerTest.php | 55 +++++++++++++++++++ ...eceivingAlipayBillMirrorRepositoryTest.php | 7 +++ 5 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 database/migrations/2026_09_09_150000_track_exact_mirror_batch_counts.php diff --git a/database/migrations/2026_09_09_150000_track_exact_mirror_batch_counts.php b/database/migrations/2026_09_09_150000_track_exact_mirror_batch_counts.php new file mode 100644 index 00000000..0e2879e0 --- /dev/null +++ b/database/migrations/2026_09_09_150000_track_exact_mirror_batch_counts.php @@ -0,0 +1,26 @@ +hasTable('business_mirror_table_states') && ! $schema->hasColumn('business_mirror_table_states', 'mirror_row_count_is_exact')) { + $schema->table('business_mirror_table_states', static fn (Blueprint $t) => $t->boolean('mirror_row_count_is_exact')->default(false)); + } + } + + public function down(): void + { + if (config('system_database.mode', 'bootstrap') === 'bootstrap') return; + $schema = Schema::connection((string) config('upstream_mirrors.connection', 'business_mirror')); + if ($schema->hasTable('business_mirror_table_states') && $schema->hasColumn('business_mirror_table_states', 'mirror_row_count_is_exact')) { + $schema->table('business_mirror_table_states', static fn (Blueprint $t) => $t->dropColumn('mirror_row_count_is_exact')); + } + } +}; diff --git a/modules/EcommerceFinance/Repositories/GenericBusinessMirrorRepository.php b/modules/EcommerceFinance/Repositories/GenericBusinessMirrorRepository.php index 4dcee6ad..e9b6aec4 100644 --- a/modules/EcommerceFinance/Repositories/GenericBusinessMirrorRepository.php +++ b/modules/EcommerceFinance/Repositories/GenericBusinessMirrorRepository.php @@ -11,6 +11,7 @@ use RuntimeException; class GenericBusinessMirrorRepository { private readonly ConnectionInterface $connection; + private ?bool $supportsExactCount = null; public function __construct(DatabaseManager $database) { @@ -93,7 +94,14 @@ class GenericBusinessMirrorRepository $query = $this->connection->table('business_mirror_table_states') ->where('dataset_key', $dataset->key()); - if ($query->exists()) { + $existing = $query->first(); + if ($this->supportsExactCount()) { + $values['mirror_row_count_is_exact'] = $mode === 'incremental' + && $existing !== null && $existing->initial_completed_at !== null + && ctype_digit((string) ($existing->mirror_row_count ?? '')) + && ($existing->status === 'succeeded' || (bool) ($existing->mirror_row_count_is_exact ?? false)); + } + if ($existing !== null) { $query->update($values); } else { $this->connection->table('business_mirror_table_states')->insert( @@ -137,6 +145,16 @@ class GenericBusinessMirrorRepository $this->connection->transaction(function () use ( $dataset, $columnNames, $mapped, $cursor, $cumulativeRows, $now ): void { + $exactCount = null; + if ($this->supportsExactCount()) { + $state = $this->connection->table('business_mirror_table_states') + ->where('dataset_key', $dataset->key())->lockForUpdate()->first(); + if ($state && $state->run_mode === 'incremental' && $state->mirror_row_count_is_exact) { + $keys = array_values(array_unique(array_column($mapped, '_mirror_key'))); + $existing = $this->connection->table($dataset->targetTable())->whereIn('_mirror_key', $keys)->count(); + $exactCount = (int) $state->mirror_row_count + count($keys) - $existing; + } + } $updates = [...$columnNames, '_mirror_generation', '_mirror_synced_at']; $chunkSize = max(1, intdiv(50000, max(count($updates) + 1, 1))); @@ -150,7 +168,7 @@ class GenericBusinessMirrorRepository 'cursor' => json_encode($cursor, JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION), 'last_upserted_rows' => $cumulativeRows, 'updated_at' => $now, - ]); + ] + ($exactCount === null ? [] : ['mirror_row_count' => $exactCount])); }); } @@ -181,7 +199,12 @@ class GenericBusinessMirrorRepository ->delete(); } - $rowCount = $this->connection->table($dataset->targetTable())->count(); + $countState = $this->supportsExactCount() + ? $this->connection->table('business_mirror_table_states')->where('dataset_key', $dataset->key())->lockForUpdate()->first() + : null; + $rowCount = $mode === 'incremental' && $countState && $countState->mirror_row_count_is_exact + ? (int) $countState->mirror_row_count + : $this->connection->table($dataset->targetTable())->count(); $values = [ 'status' => 'succeeded', 'cursor' => null, @@ -197,6 +220,7 @@ class GenericBusinessMirrorRepository $values['initial_completed_at'] = $now; $values['last_full_succeeded_at'] = $now; } + if ($this->supportsExactCount()) $values['mirror_row_count_is_exact'] = true; $this->connection->table('business_mirror_table_states') ->where('dataset_key', $dataset->key()) @@ -215,6 +239,12 @@ class GenericBusinessMirrorRepository ]); } + private function supportsExactCount(): bool + { + return $this->supportsExactCount ??= Schema::connection($this->connection->getName()) + ->hasColumn('business_mirror_table_states', 'mirror_row_count_is_exact'); + } + private function decode(mixed $value): ?array { if (is_array($value)) { diff --git a/modules/EcommerceFinance/Repositories/StoreReceivingAlipayBillMirrorRepository.php b/modules/EcommerceFinance/Repositories/StoreReceivingAlipayBillMirrorRepository.php index 5b7952c1..fa4fe77f 100644 --- a/modules/EcommerceFinance/Repositories/StoreReceivingAlipayBillMirrorRepository.php +++ b/modules/EcommerceFinance/Repositories/StoreReceivingAlipayBillMirrorRepository.php @@ -41,11 +41,12 @@ class StoreReceivingAlipayBillMirrorRepository $query = $this->billQuery(); $this->applyFilters($query, $filters); $state = $this->mirror->state($dataset) ?? []; - // The synchronizer persists an exact COUNT in the same transaction that - // marks a run successful. Never substitute a database row estimate. + // Completed runs persist an exact COUNT. New incremental batches keep + // that count exact in their upsert transaction, including inserts. $hasFilters = collect($filters)->contains(static fn (mixed $value): bool => $value !== null && trim((string) $value) !== ''); $recordedCount = $state['mirror_row_count'] ?? null; - $useRecordedCount = ! $hasFilters && ($state['status'] ?? null) === 'succeeded' + $useRecordedCount = ! $hasFilters + && (($state['status'] ?? null) === 'succeeded' || (bool) ($state['mirror_row_count_is_exact'] ?? false)) && $recordedCount !== null && ctype_digit((string) $recordedCount); $total = $useRecordedCount ? (int) $recordedCount : (clone $query)->count(); $summary = $this->summary($query, $filters); diff --git a/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php index 215bf85a..0e0f2124 100644 --- a/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php +++ b/tests/Unit/EcommerceFinance/BusinessMirrorSynchronizerTest.php @@ -65,6 +65,7 @@ class BusinessMirrorSynchronizerTest extends TestCase $table->dateTime('last_succeeded_at')->nullable(); $table->unsignedBigInteger('last_upserted_rows')->default(0); $table->unsignedBigInteger('mirror_row_count')->default(0); + $table->boolean('mirror_row_count_is_exact')->default(false); $table->string('last_error_code')->nullable(); $table->timestamps(); }); @@ -335,6 +336,60 @@ class BusinessMirrorSynchronizerTest extends TestCase $this->assertSame($value, DB::connection('business_mirror')->table($dataset->targetTable())->value('displayName')); } + public function test_incremental_batches_keep_exact_counts_without_counting_updates_as_inserts(): void + { + $this->query->batches = [[ + ['Id' => 1, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => 'one'], + ['Id' => 2, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => 'two'], + ]]; + $this->synchronizer->sync('test.dbo.records', true); + $db = DB::connection('business_mirror'); + $state = fn () => $db->table('business_mirror_table_states')->where('dataset_key', 'test.dbo.records')->first(); + $this->assertTrue((bool) $state()->mirror_row_count_is_exact); + $this->query->batches = [[ + ['Id' => 1, 'updateTime' => '2026-08-15 10:00:00', 'displayName' => 'updated'], + ['Id' => 3, 'updateTime' => '2026-08-15 10:00:00', 'displayName' => 'new'], + ]]; + $this->synchronizer->sync('test.dbo.records', false, 2, 1); + $this->assertSame('running', $state()->status); + $this->assertSame(3, (int) $state()->mirror_row_count); + $this->assertTrue((bool) $state()->mirror_row_count_is_exact); + $this->assertSame(3, $db->table('test_records')->count()); + $this->query->batches = [[ + ['Id' => 4, 'updateTime' => '2026-08-15 10:00:00', 'displayName' => 'rollback-a'], + ['Id' => 5, 'updateTime' => '2026-08-15 10:00:00', 'displayName' => 'rollback-b'], + ]]; + $db->beginTransaction(); + $this->synchronizer->sync('test.dbo.records', false, 2, 1); + $this->assertSame(5, (int) $state()->mirror_row_count); + $db->rollBack(); + $this->assertSame(3, (int) $state()->mirror_row_count); + $this->assertSame(3, $db->table('test_records')->count()); + $db->enableQueryLog(); + $db->flushQueryLog(); + $this->query->batches = [[]]; + $this->synchronizer->sync('test.dbo.records'); + $this->assertSame(3, (int) $state()->mirror_row_count); + $this->assertCount(0, array_filter($db->getQueryLog(), static fn ($q) => str_contains(strtolower($q['query']), 'count(*)') && str_contains($q['query'], 'test_records'))); + $db->disableQueryLog(); + } + + public function test_legacy_running_count_is_not_trusted_until_the_run_completes(): void + { + $this->query->batches = [[['Id' => 1, 'updateTime' => '2026-08-14 10:00:00', 'displayName' => 'one']]]; + $this->synchronizer->sync('test.dbo.records', true); + $db = DB::connection('business_mirror'); + $query = $db->table('business_mirror_table_states')->where('dataset_key', 'test.dbo.records'); + $query->update(['status' => 'running', 'run_mode' => 'incremental', 'cursor' => '{"Id":1}', 'mirror_row_count' => 999, 'mirror_row_count_is_exact' => false]); + $this->query->batches = [[['Id' => 2, 'updateTime' => '2026-08-15 10:00:00', 'displayName' => 'two']]]; + $this->synchronizer->sync('test.dbo.records', false, 1, 1); + $this->assertFalse((bool) $query->first()->mirror_row_count_is_exact); + $this->query->batches = [[]]; + $this->synchronizer->sync('test.dbo.records'); + $this->assertSame(2, (int) $query->first()->mirror_row_count); + $this->assertTrue((bool) $query->first()->mirror_row_count_is_exact); + } + public function test_empty_full_read_does_not_delete_previous_mirror_rows(): void { $this->query->batches = [[ diff --git a/tests/Unit/EcommerceFinance/StoreReceivingAlipayBillMirrorRepositoryTest.php b/tests/Unit/EcommerceFinance/StoreReceivingAlipayBillMirrorRepositoryTest.php index 0b1c3355..d5a3c633 100644 --- a/tests/Unit/EcommerceFinance/StoreReceivingAlipayBillMirrorRepositoryTest.php +++ b/tests/Unit/EcommerceFinance/StoreReceivingAlipayBillMirrorRepositoryTest.php @@ -44,6 +44,7 @@ class StoreReceivingAlipayBillMirrorRepositoryTest extends TestCase $table->string('status')->nullable(); $table->dateTime('last_succeeded_at')->nullable(); $table->unsignedBigInteger('mirror_row_count')->nullable(); + $table->boolean('mirror_row_count_is_exact')->default(false); }); $schema->create('ops_jst_original_bill_info', function (Blueprint $table): void { $table->integer('Id'); @@ -189,6 +190,12 @@ class StoreReceivingAlipayBillMirrorRepositoryTest extends TestCase $this->assertSame(3, $this->repository->paginate([], 'occurred_at', 'desc', 1, 1)['total']); $state->update(['status' => 'failed']); $this->assertSame(3, $this->repository->paginate([], 'occurred_at', 'desc', 1, 1)['total']); + $state->update(['status' => 'running', 'mirror_row_count' => 3, 'mirror_row_count_is_exact' => true]); + $db->flushQueryLog(); + $db->enableQueryLog(); + $this->assertSame(3, $this->repository->paginate([], 'occurred_at', 'desc', 1, 1)['total']); + $this->assertCount(0, array_filter($db->getQueryLog(), fn ($q) => str_contains(strtolower($q['query']), 'count(*)'))); + $db->disableQueryLog(); $state->update(['status' => 'succeeded', 'mirror_row_count' => null]); $this->assertSame(3, $this->repository->paginate([], 'occurred_at', 'desc', 1, 1)['total']); } -- 2.47.2 From ac775a5fb0409412d5964b40d4cf40d875197db7 Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Wed, 9 Sep 2026 14:35:41 +0800 Subject: [PATCH 2/2] fix: keep explicit empty export cells in shared table controls --- .../pages/partials/finance-table-interactions.blade.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/resources/views/business/pages/partials/finance-table-interactions.blade.php b/resources/views/business/pages/partials/finance-table-interactions.blade.php index 059b2841..6d4b2644 100644 --- a/resources/views/business/pages/partials/finance-table-interactions.blade.php +++ b/resources/views/business/pages/partials/finance-table-interactions.blade.php @@ -303,7 +303,11 @@ return /^[=+\-@]/u.test(text) ? `'${text}` : text; }; const csvEscape = (value) => `"${csvText(value).replaceAll('"', '""')}"`; - const cellText = (cell) => (cell?.querySelector('[data-password-status]') || cell)?.textContent.trim() || ''; + const cellText = (cell) => { + const passwordStatus = cell?.querySelector('[data-password-status]'); + if (passwordStatus) return passwordStatus.textContent.trim(); + return cell?.dataset.exportValue ?? cell?.textContent.trim() ?? ''; + }; const downloadCsv = (filename, lines) => { const blob = new Blob([`\ufeff${lines.join('\r\n')}`], { type: 'text/csv;charset=utf-8' }); const url = URL.createObjectURL(blob); -- 2.47.2