['driver' => 'sqlite', 'database' => ':memory:', 'prefix' => ''], 'upstream_mirrors.connection' => 'business_mirror', 'upstream_mirrors.datasets' => [ 'source.schema.fact' => [ 'source_key' => 'source', 'schema' => 'schema', 'object' => 'fact', 'target' => 'fact_rows', 'keys' => ['Id'], 'watermark' => 'updated_at', ], ], 'business_data.datasets' => [ 'source.schema.fact' => ['role' => 'fact', 'target_store' => 'clickhouse'], ], ]); DB::purge('business_mirror'); $migration = require base_path('database/migrations/2026_09_07_120000_create_business_data_placement_tables.php'); $migration->up(); Schema::connection('business_mirror')->create('fact_rows', function (Blueprint $table): void { $table->string('_mirror_key')->primary(); $table->integer('Id'); $table->string('name')->nullable(); $table->dateTime('updated_at')->nullable(); $table->dateTime('_mirror_synced_at', 6); }); DB::connection('business_mirror')->table('fact_rows')->insert([ ['_mirror_key' => 'key-1', 'Id' => 1, 'name' => '一', 'updated_at' => '2026-09-01 00:00:00', '_mirror_synced_at' => '2026-09-07 00:00:00'], ['_mirror_key' => 'key-2', 'Id' => 2, 'name' => '二', 'updated_at' => '2026-09-02 00:00:00', '_mirror_synced_at' => '2026-09-07 00:00:00'], ]); } protected function tearDown(): void { DB::purge('business_mirror'); parent::tearDown(); } public function test_it_backfills_from_existing_mariadb_and_stops_at_verified_without_cutover_or_delete(): void { $columns = [ ['column_name' => 'Id', 'data_type' => 'int'], ['column_name' => 'name', 'data_type' => 'nvarchar'], ['column_name' => 'updated_at', 'data_type' => 'datetime2'], ]; $schema = Mockery::mock(ClickHouseSchemaManager::class); $schema->shouldReceive('ensure')->once()->andReturn($columns); $columnar = Mockery::mock(ColumnarBusinessMirrorRepository::class); $columnar->shouldReceive('storeBackfillBatch')->once()->withArgs( static fn ($dataset, array $actualColumns, array $rows, string $generation): bool => $dataset->key() === 'source.schema.fact' && $actualColumns === $columns && count($rows) === 2 && $generation !== '', ); $columnar->shouldReceive('count')->once()->andReturn(2); $columnar->shouldReceive('distinctKeys')->once()->andReturn(2); $clickHouse = Mockery::mock(ClickHouseClient::class); $clickHouse->shouldReceive('enabled')->once()->andReturnTrue(); $clickHouse->shouldReceive('ping')->once()->andReturnTrue(); $this->app->instance(ClickHouseSchemaManager::class, $schema); $this->app->instance(ColumnarBusinessMirrorRepository::class, $columnar); $this->app->instance(ClickHouseClient::class, $clickHouse); $exitCode = $this->artisan('business-data:backfill-columnar', [ 'dataset' => 'source.schema.fact', '--batch' => 100, '--max-batches' => 1, ])->run(); $state = DB::connection('business_mirror')->table('business_data_placements') ->where('dataset_key', 'source.schema.fact')->first(); $this->assertSame('verified', $state->phase, (string) $state->last_error_code); $this->assertSame(0, $exitCode); $this->assertSame(2, (int) $state->source_rows); $this->assertSame(2, (int) $state->target_rows); $this->assertNull($state->activated_at); $this->assertSame(2, DB::connection('business_mirror')->table('fact_rows')->count()); } public function test_verified_backfill_can_archive_all_hash_partitions_without_deleting_source(): void { config([ 'business_data.archive.disk' => 'business_data_archive_test', 'business_data.archive.prefix' => 'archive-root', ]); Storage::fake('business_data_archive_test'); $columns = [ ['column_name' => 'Id', 'data_type' => 'int'], ['column_name' => 'name', 'data_type' => 'nvarchar'], ['column_name' => 'updated_at', 'data_type' => 'datetime2'], ]; $schema = Mockery::mock(ClickHouseSchemaManager::class); $schema->shouldReceive('ensure')->twice()->andReturn($columns); $columnar = Mockery::mock(ColumnarBusinessMirrorRepository::class); $columnar->shouldReceive('storeBackfillBatch')->once(); $columnar->shouldReceive('count')->once()->andReturn(2); $columnar->shouldReceive('distinctKeys')->once()->andReturn(2); $columnar->shouldReceive('parquetPartition')->times(32) ->andReturnUsing(static fn () => Utils::streamFor('PAR1-fixture')); $columnar->shouldReceive('partitionStats')->times(32)->andReturn([ 'row_count' => 0, 'logical_checksum' => '0000000000000000', ]); $clickHouse = Mockery::mock(ClickHouseClient::class); $clickHouse->shouldReceive('enabled')->twice()->andReturnTrue(); $clickHouse->shouldReceive('ping')->twice()->andReturnTrue(); $this->app->instance(ClickHouseSchemaManager::class, $schema); $this->app->instance(ColumnarBusinessMirrorRepository::class, $columnar); $this->app->instance(ClickHouseClient::class, $clickHouse); $this->artisan('business-data:backfill-columnar', [ 'dataset' => 'source.schema.fact', '--batch' => 100, '--max-batches' => 1, ])->expectsOutputToContain('已对账')->assertSuccessful(); $this->artisan('business-data:backfill-columnar', [ 'dataset' => 'source.schema.fact', '--archive' => true, ])->expectsOutputToContain('32 个 Parquet 哈希分区')->assertSuccessful(); $archives = DB::connection('business_mirror')->table('business_data_archives')->get(); $this->assertCount(32, $archives); $this->assertSame(32, collect(Storage::disk('business_data_archive_test')->allFiles())->count()); $this->assertSame(2, DB::connection('business_mirror')->table('fact_rows')->count()); } }