test: verify partitioned parquet archive flow

This commit is contained in:
WuYongJie 2026-09-07 11:34:29 +08:00
parent 9e35a57fb0
commit bfb921c81a
3 changed files with 56 additions and 32 deletions

View File

@ -202,7 +202,8 @@ class BackfillBusinessDataColumnar extends Command
}
$bytes = (int) $disk->size($objectKey);
$rowCount = $columnar->countPartition($dataset, $generation, $partition);
$stats = $columnar->partitionStats($dataset, $generation, $partition);
$rowCount = $stats['row_count'];
if ($bytes <= 0 || ($rowCount > 0 && $bytes < 8)) {
throw new RuntimeException('business_data_archive_verification_failed');
}
@ -212,7 +213,7 @@ class BackfillBusinessDataColumnar extends Command
'object_key' => $objectKey,
'row_count' => $rowCount,
'byte_size' => $bytes,
'logical_checksum' => $columnar->logicalChecksumPartition($dataset, $generation, $partition),
'logical_checksum' => $stats['logical_checksum'],
'status' => 'verified',
'verified_at' => now(),
'created_at' => now(),

View File

@ -74,44 +74,24 @@ class ColumnarBusinessMirrorRepository
);
}
public function logicalChecksum(BusinessMirrorDataset $dataset, string $generation): string
/** @return array{row_count:int,logical_checksum:string} */
public function partitionStats(BusinessMirrorDataset $dataset, string $generation, int $partition): array
{
return (string) $this->clickHouse->scalar(
$rows = $this->clickHouse->rows(
sprintf(
'SELECT hex(groupBitXor(cityHash64(_mirror_key))) FROM %s FINAL '
. 'WHERE _mirror_generation = {generation:UUID}',
$this->clickHouse->qualifiedTable($dataset->targetTable()),
),
['generation' => $generation],
);
}
public function countPartition(BusinessMirrorDataset $dataset, string $generation, int $partition): int
{
return (int) $this->clickHouse->scalar(
sprintf(
'SELECT count() FROM %s FINAL WHERE _mirror_generation = {generation:UUID} '
'SELECT count() AS row_count, hex(groupBitXor(cityHash64(_mirror_key))) AS logical_checksum '
. 'FROM %s FINAL WHERE _mirror_generation = {generation:UUID} '
. 'AND modulo(cityHash64(_mirror_key), 32) = {partition:UInt8}',
$this->clickHouse->qualifiedTable($dataset->targetTable()),
),
['generation' => $generation, 'partition' => $this->partition($partition)],
);
}
$row = $rows[0] ?? [];
public function logicalChecksumPartition(
BusinessMirrorDataset $dataset,
string $generation,
int $partition,
): string {
return (string) $this->clickHouse->scalar(
sprintf(
'SELECT hex(groupBitXor(cityHash64(_mirror_key))) FROM %s FINAL '
. 'WHERE _mirror_generation = {generation:UUID} '
. 'AND modulo(cityHash64(_mirror_key), 32) = {partition:UInt8}',
$this->clickHouse->qualifiedTable($dataset->targetTable()),
),
['generation' => $generation, 'partition' => $this->partition($partition)],
);
return [
'row_count' => (int) ($row['row_count'] ?? 0),
'logical_checksum' => (string) ($row['logical_checksum'] ?? ''),
];
}
public function parquetPartition(

View File

@ -5,6 +5,8 @@ namespace Tests\Feature\Console;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
use Illuminate\Support\Facades\Storage;
use GuzzleHttp\Psr7\Utils;
use Mockery;
use Modules\EcommerceFinance\Repositories\ColumnarBusinessMirrorRepository;
use Modules\EcommerceFinance\Services\ClickHouseClient;
@ -91,4 +93,45 @@ class BusinessDataColumnarBackfillTest extends TestCase
$this->assertNull($state->activated_at);
$this->assertSame(2, DB::connection('business_mirror')->table('fact_rows')->count());
}
public function test_verified_backfill_can_archive_all_hash_partitions_without_deleting_source(): void
{
config([
'business_data.archive.disk' => 'business_data_archive_test',
'business_data.archive.prefix' => 'archive-root',
]);
Storage::fake('business_data_archive_test');
$columns = [
['column_name' => 'Id', 'data_type' => 'int'],
['column_name' => 'name', 'data_type' => 'nvarchar'],
['column_name' => 'updated_at', 'data_type' => 'datetime2'],
];
$schema = Mockery::mock(ClickHouseSchemaManager::class);
$schema->shouldReceive('ensure')->once()->andReturn($columns);
$columnar = Mockery::mock(ColumnarBusinessMirrorRepository::class);
$columnar->shouldReceive('storeBackfillBatch')->once();
$columnar->shouldReceive('count')->once()->andReturn(2);
$columnar->shouldReceive('distinctKeys')->once()->andReturn(2);
$columnar->shouldReceive('parquetPartition')->times(32)
->andReturnUsing(static fn () => Utils::streamFor('PAR1-fixture'));
$columnar->shouldReceive('partitionStats')->times(32)->andReturn([
'row_count' => 0,
'logical_checksum' => '0000000000000000',
]);
$clickHouse = Mockery::mock(ClickHouseClient::class);
$clickHouse->shouldReceive('enabled')->once()->andReturnTrue();
$clickHouse->shouldReceive('ping')->once()->andReturnTrue();
$this->app->instance(ClickHouseSchemaManager::class, $schema);
$this->app->instance(ColumnarBusinessMirrorRepository::class, $columnar);
$this->app->instance(ClickHouseClient::class, $clickHouse);
$this->artisan('business-data:backfill-columnar', [
'dataset' => 'source.schema.fact', '--batch' => 100, '--max-batches' => 1, '--archive' => true,
])->expectsOutputToContain('32 个 Parquet 哈希分区')->assertSuccessful();
$archives = DB::connection('business_mirror')->table('business_data_archives')->get();
$this->assertCount(32, $archives);
$this->assertSame(32, collect(Storage::disk('business_data_archive_test')->allFiles())->count());
$this->assertSame(2, DB::connection('business_mirror')->table('fact_rows')->count());
}
}