CaiWuXiTong/tests/Feature/Console/BusinessDataColumnarBackfillTest.php

<?php

namespace Tests\Feature\Console;

use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
use Illuminate\Support\Facades\Storage;
use GuzzleHttp\Psr7\Utils;
use Mockery;
use Modules\EcommerceFinance\Repositories\ColumnarBusinessMirrorRepository;
use Modules\EcommerceFinance\Services\ClickHouseClient;
use Modules\EcommerceFinance\Services\ClickHouseSchemaManager;
use Tests\TestCase;

class BusinessDataColumnarBackfillTest extends TestCase
{
    protected function setUp(): void
    {
        parent::setUp();
        config([
            'database.connections.business_mirror' => ['driver' => 'sqlite', 'database' => ':memory:', 'prefix' => ''],
            'upstream_mirrors.connection' => 'business_mirror',
            'upstream_mirrors.datasets' => [
                'source.schema.fact' => [
                    'source_key' => 'source', 'schema' => 'schema', 'object' => 'fact',
                    'target' => 'fact_rows', 'keys' => ['Id'], 'watermark' => 'updated_at',
                ],
            ],
            'business_data.datasets' => [
                'source.schema.fact' => ['role' => 'fact', 'target_store' => 'clickhouse'],
            ],
        ]);
        DB::purge('business_mirror');

        $migration = require base_path('database/migrations/2026_09_07_120000_create_business_data_placement_tables.php');
        $migration->up();
        Schema::connection('business_mirror')->create('fact_rows', function (Blueprint $table): void {
            $table->string('_mirror_key')->primary();
            $table->integer('Id');
            $table->string('name')->nullable();
            $table->dateTime('updated_at')->nullable();
            $table->dateTime('_mirror_synced_at', 6);
        });
        DB::connection('business_mirror')->table('fact_rows')->insert([
            ['_mirror_key' => 'key-1', 'Id' => 1, 'name' => '一', 'updated_at' => '2026-09-01 00:00:00', '_mirror_synced_at' => '2026-09-07 00:00:00'],
            ['_mirror_key' => 'key-2', 'Id' => 2, 'name' => '二', 'updated_at' => '2026-09-02 00:00:00', '_mirror_synced_at' => '2026-09-07 00:00:00'],
        ]);
    }

    protected function tearDown(): void
    {
        DB::purge('business_mirror');
        parent::tearDown();
    }

    public function test_it_backfills_from_existing_mariadb_and_stops_at_verified_without_cutover_or_delete(): void
    {
        $columns = [
            ['column_name' => 'Id', 'data_type' => 'int'],
            ['column_name' => 'name', 'data_type' => 'nvarchar'],
            ['column_name' => 'updated_at', 'data_type' => 'datetime2'],
        ];
        $schema = Mockery::mock(ClickHouseSchemaManager::class);
        $schema->shouldReceive('ensure')->once()->andReturn($columns);
        $columnar = Mockery::mock(ColumnarBusinessMirrorRepository::class);
        $columnar->shouldReceive('storeBackfillBatch')->once()->withArgs(
            static fn ($dataset, array $actualColumns, array $rows, string $generation): bool =>
                $dataset->key() === 'source.schema.fact'
                && $actualColumns === $columns
                && count($rows) === 2
                && $generation !== '',
        );
        $columnar->shouldReceive('count')->once()->andReturn(2);
        $columnar->shouldReceive('distinctKeys')->once()->andReturn(2);
        $clickHouse = Mockery::mock(ClickHouseClient::class);
        $clickHouse->shouldReceive('enabled')->once()->andReturnTrue();
        $clickHouse->shouldReceive('ping')->once()->andReturnTrue();
        $this->app->instance(ClickHouseSchemaManager::class, $schema);
        $this->app->instance(ColumnarBusinessMirrorRepository::class, $columnar);
        $this->app->instance(ClickHouseClient::class, $clickHouse);

        $exitCode = $this->artisan('business-data:backfill-columnar', [
            'dataset' => 'source.schema.fact', '--batch' => 100, '--max-batches' => 1,
        ])->run();

        $state = DB::connection('business_mirror')->table('business_data_placements')
            ->where('dataset_key', 'source.schema.fact')->first();
        $this->assertSame('verified', $state->phase, (string) $state->last_error_code);
        $this->assertSame(0, $exitCode);
        $this->assertSame(2, (int) $state->source_rows);
        $this->assertSame(2, (int) $state->target_rows);
        $this->assertNull($state->activated_at);
        $this->assertSame(2, DB::connection('business_mirror')->table('fact_rows')->count());
    }

    public function test_verified_backfill_can_archive_all_hash_partitions_without_deleting_source(): void
    {
        config([
            'business_data.archive.disk' => 'business_data_archive_test',
            'business_data.archive.prefix' => 'archive-root',
        ]);
        Storage::fake('business_data_archive_test');
        $columns = [
            ['column_name' => 'Id', 'data_type' => 'int'],
            ['column_name' => 'name', 'data_type' => 'nvarchar'],
            ['column_name' => 'updated_at', 'data_type' => 'datetime2'],
        ];
        $schema = Mockery::mock(ClickHouseSchemaManager::class);
        $schema->shouldReceive('ensure')->twice()->andReturn($columns);
        $columnar = Mockery::mock(ColumnarBusinessMirrorRepository::class);
        $columnar->shouldReceive('storeBackfillBatch')->once();
        $columnar->shouldReceive('count')->once()->andReturn(2);
        $columnar->shouldReceive('distinctKeys')->once()->andReturn(2);
        $columnar->shouldReceive('parquetPartition')->times(32)
            ->andReturnUsing(static fn () => Utils::streamFor('PAR1-fixture'));
        $columnar->shouldReceive('partitionStats')->times(32)->andReturn([
            'row_count' => 0,
            'logical_checksum' => '0000000000000000',
        ]);
        $clickHouse = Mockery::mock(ClickHouseClient::class);
        $clickHouse->shouldReceive('enabled')->twice()->andReturnTrue();
        $clickHouse->shouldReceive('ping')->twice()->andReturnTrue();
        $this->app->instance(ClickHouseSchemaManager::class, $schema);
        $this->app->instance(ColumnarBusinessMirrorRepository::class, $columnar);
        $this->app->instance(ClickHouseClient::class, $clickHouse);

        $this->artisan('business-data:backfill-columnar', [
            'dataset' => 'source.schema.fact', '--batch' => 100, '--max-batches' => 1,
        ])->expectsOutputToContain('已对账')->assertSuccessful();
        $this->artisan('business-data:backfill-columnar', [
            'dataset' => 'source.schema.fact', '--archive' => true,
        ])->expectsOutputToContain('32 个 Parquet 哈希分区')->assertSuccessful();

        $archives = DB::connection('business_mirror')->table('business_data_archives')->get();
        $this->assertCount(32, $archives);
        $this->assertSame(32, collect(Storage::disk('business_data_archive_test')->allFiles())->count());
        $this->assertSame(2, DB::connection('business_mirror')->table('fact_rows')->count());
    }
}