test: verify partitioned parquet archive flow
This commit is contained in:
parent
9e35a57fb0
commit
bfb921c81a
@ -202,7 +202,8 @@ class BackfillBusinessDataColumnar extends Command
|
||||
}
|
||||
|
||||
$bytes = (int) $disk->size($objectKey);
|
||||
$rowCount = $columnar->countPartition($dataset, $generation, $partition);
|
||||
$stats = $columnar->partitionStats($dataset, $generation, $partition);
|
||||
$rowCount = $stats['row_count'];
|
||||
if ($bytes <= 0 || ($rowCount > 0 && $bytes < 8)) {
|
||||
throw new RuntimeException('business_data_archive_verification_failed');
|
||||
}
|
||||
@ -212,7 +213,7 @@ class BackfillBusinessDataColumnar extends Command
|
||||
'object_key' => $objectKey,
|
||||
'row_count' => $rowCount,
|
||||
'byte_size' => $bytes,
|
||||
'logical_checksum' => $columnar->logicalChecksumPartition($dataset, $generation, $partition),
|
||||
'logical_checksum' => $stats['logical_checksum'],
|
||||
'status' => 'verified',
|
||||
'verified_at' => now(),
|
||||
'created_at' => now(),
|
||||
|
||||
@ -74,44 +74,24 @@ class ColumnarBusinessMirrorRepository
|
||||
);
|
||||
}
|
||||
|
||||
public function logicalChecksum(BusinessMirrorDataset $dataset, string $generation): string
|
||||
/** @return array{row_count:int,logical_checksum:string} */
|
||||
public function partitionStats(BusinessMirrorDataset $dataset, string $generation, int $partition): array
|
||||
{
|
||||
return (string) $this->clickHouse->scalar(
|
||||
$rows = $this->clickHouse->rows(
|
||||
sprintf(
|
||||
'SELECT hex(groupBitXor(cityHash64(_mirror_key))) FROM %s FINAL '
|
||||
. 'WHERE _mirror_generation = {generation:UUID}',
|
||||
$this->clickHouse->qualifiedTable($dataset->targetTable()),
|
||||
),
|
||||
['generation' => $generation],
|
||||
);
|
||||
}
|
||||
|
||||
public function countPartition(BusinessMirrorDataset $dataset, string $generation, int $partition): int
|
||||
{
|
||||
return (int) $this->clickHouse->scalar(
|
||||
sprintf(
|
||||
'SELECT count() FROM %s FINAL WHERE _mirror_generation = {generation:UUID} '
|
||||
'SELECT count() AS row_count, hex(groupBitXor(cityHash64(_mirror_key))) AS logical_checksum '
|
||||
. 'FROM %s FINAL WHERE _mirror_generation = {generation:UUID} '
|
||||
. 'AND modulo(cityHash64(_mirror_key), 32) = {partition:UInt8}',
|
||||
$this->clickHouse->qualifiedTable($dataset->targetTable()),
|
||||
),
|
||||
['generation' => $generation, 'partition' => $this->partition($partition)],
|
||||
);
|
||||
}
|
||||
$row = $rows[0] ?? [];
|
||||
|
||||
public function logicalChecksumPartition(
|
||||
BusinessMirrorDataset $dataset,
|
||||
string $generation,
|
||||
int $partition,
|
||||
): string {
|
||||
return (string) $this->clickHouse->scalar(
|
||||
sprintf(
|
||||
'SELECT hex(groupBitXor(cityHash64(_mirror_key))) FROM %s FINAL '
|
||||
. 'WHERE _mirror_generation = {generation:UUID} '
|
||||
. 'AND modulo(cityHash64(_mirror_key), 32) = {partition:UInt8}',
|
||||
$this->clickHouse->qualifiedTable($dataset->targetTable()),
|
||||
),
|
||||
['generation' => $generation, 'partition' => $this->partition($partition)],
|
||||
);
|
||||
return [
|
||||
'row_count' => (int) ($row['row_count'] ?? 0),
|
||||
'logical_checksum' => (string) ($row['logical_checksum'] ?? ''),
|
||||
];
|
||||
}
|
||||
|
||||
public function parquetPartition(
|
||||
|
||||
@ -5,6 +5,8 @@ namespace Tests\Feature\Console;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use GuzzleHttp\Psr7\Utils;
|
||||
use Mockery;
|
||||
use Modules\EcommerceFinance\Repositories\ColumnarBusinessMirrorRepository;
|
||||
use Modules\EcommerceFinance\Services\ClickHouseClient;
|
||||
@ -91,4 +93,45 @@ class BusinessDataColumnarBackfillTest extends TestCase
|
||||
$this->assertNull($state->activated_at);
|
||||
$this->assertSame(2, DB::connection('business_mirror')->table('fact_rows')->count());
|
||||
}
|
||||
|
||||
public function test_verified_backfill_can_archive_all_hash_partitions_without_deleting_source(): void
|
||||
{
|
||||
config([
|
||||
'business_data.archive.disk' => 'business_data_archive_test',
|
||||
'business_data.archive.prefix' => 'archive-root',
|
||||
]);
|
||||
Storage::fake('business_data_archive_test');
|
||||
$columns = [
|
||||
['column_name' => 'Id', 'data_type' => 'int'],
|
||||
['column_name' => 'name', 'data_type' => 'nvarchar'],
|
||||
['column_name' => 'updated_at', 'data_type' => 'datetime2'],
|
||||
];
|
||||
$schema = Mockery::mock(ClickHouseSchemaManager::class);
|
||||
$schema->shouldReceive('ensure')->once()->andReturn($columns);
|
||||
$columnar = Mockery::mock(ColumnarBusinessMirrorRepository::class);
|
||||
$columnar->shouldReceive('storeBackfillBatch')->once();
|
||||
$columnar->shouldReceive('count')->once()->andReturn(2);
|
||||
$columnar->shouldReceive('distinctKeys')->once()->andReturn(2);
|
||||
$columnar->shouldReceive('parquetPartition')->times(32)
|
||||
->andReturnUsing(static fn () => Utils::streamFor('PAR1-fixture'));
|
||||
$columnar->shouldReceive('partitionStats')->times(32)->andReturn([
|
||||
'row_count' => 0,
|
||||
'logical_checksum' => '0000000000000000',
|
||||
]);
|
||||
$clickHouse = Mockery::mock(ClickHouseClient::class);
|
||||
$clickHouse->shouldReceive('enabled')->once()->andReturnTrue();
|
||||
$clickHouse->shouldReceive('ping')->once()->andReturnTrue();
|
||||
$this->app->instance(ClickHouseSchemaManager::class, $schema);
|
||||
$this->app->instance(ColumnarBusinessMirrorRepository::class, $columnar);
|
||||
$this->app->instance(ClickHouseClient::class, $clickHouse);
|
||||
|
||||
$this->artisan('business-data:backfill-columnar', [
|
||||
'dataset' => 'source.schema.fact', '--batch' => 100, '--max-batches' => 1, '--archive' => true,
|
||||
])->expectsOutputToContain('32 个 Parquet 哈希分区')->assertSuccessful();
|
||||
|
||||
$archives = DB::connection('business_mirror')->table('business_data_archives')->get();
|
||||
$this->assertCount(32, $archives);
|
||||
$this->assertSame(32, collect(Storage::disk('business_data_archive_test')->allFiles())->count());
|
||||
$this->assertSame(2, DB::connection('business_mirror')->table('fact_rows')->count());
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user