From bfb921c81a284cf7e68cf629d60c4e2f272a6088 Mon Sep 17 00:00:00 2001 From: WuYongJie <549727955@qq.com> Date: Mon, 7 Sep 2026 11:34:29 +0800 Subject: [PATCH] test: verify partitioned parquet archive flow --- .../Commands/BackfillBusinessDataColumnar.php | 5 ++- .../ColumnarBusinessMirrorRepository.php | 40 +++++------------ .../BusinessDataColumnarBackfillTest.php | 43 +++++++++++++++++++ 3 files changed, 56 insertions(+), 32 deletions(-) diff --git a/app/Console/Commands/BackfillBusinessDataColumnar.php b/app/Console/Commands/BackfillBusinessDataColumnar.php index 0b0676ad..671992c3 100644 --- a/app/Console/Commands/BackfillBusinessDataColumnar.php +++ b/app/Console/Commands/BackfillBusinessDataColumnar.php @@ -202,7 +202,8 @@ class BackfillBusinessDataColumnar extends Command } $bytes = (int) $disk->size($objectKey); - $rowCount = $columnar->countPartition($dataset, $generation, $partition); + $stats = $columnar->partitionStats($dataset, $generation, $partition); + $rowCount = $stats['row_count']; if ($bytes <= 0 || ($rowCount > 0 && $bytes < 8)) { throw new RuntimeException('business_data_archive_verification_failed'); } @@ -212,7 +213,7 @@ class BackfillBusinessDataColumnar extends Command 'object_key' => $objectKey, 'row_count' => $rowCount, 'byte_size' => $bytes, - 'logical_checksum' => $columnar->logicalChecksumPartition($dataset, $generation, $partition), + 'logical_checksum' => $stats['logical_checksum'], 'status' => 'verified', 'verified_at' => now(), 'created_at' => now(), diff --git a/modules/EcommerceFinance/Repositories/ColumnarBusinessMirrorRepository.php b/modules/EcommerceFinance/Repositories/ColumnarBusinessMirrorRepository.php index 96fd0a1c..dce06900 100644 --- a/modules/EcommerceFinance/Repositories/ColumnarBusinessMirrorRepository.php +++ b/modules/EcommerceFinance/Repositories/ColumnarBusinessMirrorRepository.php @@ -74,44 +74,24 @@ class ColumnarBusinessMirrorRepository ); } - public function logicalChecksum(BusinessMirrorDataset $dataset, string $generation): string + /** @return array{row_count:int,logical_checksum:string} */ + public function partitionStats(BusinessMirrorDataset $dataset, string $generation, int $partition): array { - return (string) $this->clickHouse->scalar( + $rows = $this->clickHouse->rows( sprintf( - 'SELECT hex(groupBitXor(cityHash64(_mirror_key))) FROM %s FINAL ' - . 'WHERE _mirror_generation = {generation:UUID}', - $this->clickHouse->qualifiedTable($dataset->targetTable()), - ), - ['generation' => $generation], - ); - } - - public function countPartition(BusinessMirrorDataset $dataset, string $generation, int $partition): int - { - return (int) $this->clickHouse->scalar( - sprintf( - 'SELECT count() FROM %s FINAL WHERE _mirror_generation = {generation:UUID} ' + 'SELECT count() AS row_count, hex(groupBitXor(cityHash64(_mirror_key))) AS logical_checksum ' + . 'FROM %s FINAL WHERE _mirror_generation = {generation:UUID} ' . 'AND modulo(cityHash64(_mirror_key), 32) = {partition:UInt8}', $this->clickHouse->qualifiedTable($dataset->targetTable()), ), ['generation' => $generation, 'partition' => $this->partition($partition)], ); - } + $row = $rows[0] ?? []; - public function logicalChecksumPartition( - BusinessMirrorDataset $dataset, - string $generation, - int $partition, - ): string { - return (string) $this->clickHouse->scalar( - sprintf( - 'SELECT hex(groupBitXor(cityHash64(_mirror_key))) FROM %s FINAL ' - . 'WHERE _mirror_generation = {generation:UUID} ' - . 'AND modulo(cityHash64(_mirror_key), 32) = {partition:UInt8}', - $this->clickHouse->qualifiedTable($dataset->targetTable()), - ), - ['generation' => $generation, 'partition' => $this->partition($partition)], - ); + return [ + 'row_count' => (int) ($row['row_count'] ?? 0), + 'logical_checksum' => (string) ($row['logical_checksum'] ?? ''), + ]; } public function parquetPartition( diff --git a/tests/Feature/Console/BusinessDataColumnarBackfillTest.php b/tests/Feature/Console/BusinessDataColumnarBackfillTest.php index 7f95811b..457f1205 100644 --- a/tests/Feature/Console/BusinessDataColumnarBackfillTest.php +++ b/tests/Feature/Console/BusinessDataColumnarBackfillTest.php @@ -5,6 +5,8 @@ namespace Tests\Feature\Console; use Illuminate\Database\Schema\Blueprint; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; +use Illuminate\Support\Facades\Storage; +use GuzzleHttp\Psr7\Utils; use Mockery; use Modules\EcommerceFinance\Repositories\ColumnarBusinessMirrorRepository; use Modules\EcommerceFinance\Services\ClickHouseClient; @@ -91,4 +93,45 @@ class BusinessDataColumnarBackfillTest extends TestCase $this->assertNull($state->activated_at); $this->assertSame(2, DB::connection('business_mirror')->table('fact_rows')->count()); } + + public function test_verified_backfill_can_archive_all_hash_partitions_without_deleting_source(): void + { + config([ + 'business_data.archive.disk' => 'business_data_archive_test', + 'business_data.archive.prefix' => 'archive-root', + ]); + Storage::fake('business_data_archive_test'); + $columns = [ + ['column_name' => 'Id', 'data_type' => 'int'], + ['column_name' => 'name', 'data_type' => 'nvarchar'], + ['column_name' => 'updated_at', 'data_type' => 'datetime2'], + ]; + $schema = Mockery::mock(ClickHouseSchemaManager::class); + $schema->shouldReceive('ensure')->once()->andReturn($columns); + $columnar = Mockery::mock(ColumnarBusinessMirrorRepository::class); + $columnar->shouldReceive('storeBackfillBatch')->once(); + $columnar->shouldReceive('count')->once()->andReturn(2); + $columnar->shouldReceive('distinctKeys')->once()->andReturn(2); + $columnar->shouldReceive('parquetPartition')->times(32) + ->andReturnUsing(static fn () => Utils::streamFor('PAR1-fixture')); + $columnar->shouldReceive('partitionStats')->times(32)->andReturn([ + 'row_count' => 0, + 'logical_checksum' => '0000000000000000', + ]); + $clickHouse = Mockery::mock(ClickHouseClient::class); + $clickHouse->shouldReceive('enabled')->once()->andReturnTrue(); + $clickHouse->shouldReceive('ping')->once()->andReturnTrue(); + $this->app->instance(ClickHouseSchemaManager::class, $schema); + $this->app->instance(ColumnarBusinessMirrorRepository::class, $columnar); + $this->app->instance(ClickHouseClient::class, $clickHouse); + + $this->artisan('business-data:backfill-columnar', [ + 'dataset' => 'source.schema.fact', '--batch' => 100, '--max-batches' => 1, '--archive' => true, + ])->expectsOutputToContain('32 个 Parquet 哈希分区')->assertSuccessful(); + + $archives = DB::connection('business_mirror')->table('business_data_archives')->get(); + $this->assertCount(32, $archives); + $this->assertSame(32, collect(Storage::disk('business_data_archive_test')->allFiles())->count()); + $this->assertSame(2, DB::connection('business_mirror')->table('fact_rows')->count()); + } }