101 lines
4.2 KiB
PHP
101 lines
4.2 KiB
PHP
<?php
|
|
|
|
namespace Modules\EcommerceFinance\Services;
|
|
|
|
use Modules\EcommerceFinance\Contracts\BusinessMirrorDataset;
|
|
use RuntimeException;
|
|
|
|
class ClickHouseSchemaManager
|
|
{
|
|
public function __construct(
|
|
private readonly BusinessMirrorSchemaManager $sourceSchemas,
|
|
private readonly ClickHouseClient $clickHouse,
|
|
private readonly BusinessDataPlacement $placements,
|
|
) {
|
|
}
|
|
|
|
/** @return list<array<string,mixed>> */
|
|
public function ensure(BusinessMirrorDataset $dataset): array
|
|
{
|
|
if (! in_array($this->placements->targetStore($dataset->key()), ['clickhouse', 'parquet'], true)) {
|
|
throw new RuntimeException('business_data_dataset_not_columnar');
|
|
}
|
|
|
|
$columns = $this->sourceSchemas->columns($dataset);
|
|
$definitions = ['`_mirror_key` String'];
|
|
foreach ($columns as $column) {
|
|
$definitions[] = $this->clickHouse->identifier((string) $column['column_name'])
|
|
. ' Nullable(' . $this->type($column) . ')';
|
|
}
|
|
$definitions[] = '`_mirror_generation` UUID';
|
|
$definitions[] = "`_mirror_synced_at` DateTime64(6, 'Asia/Shanghai')";
|
|
$definitions[] = '`_mirror_version` UInt64';
|
|
|
|
// Hash partitioning keeps every version of one business key in the same
|
|
// partition. The generation must also be part of the replacement key:
|
|
// otherwise a newer backfill generation would erase the retained one.
|
|
$sql = sprintf(
|
|
'CREATE TABLE IF NOT EXISTS %s (%s) '
|
|
. 'ENGINE = ReplacingMergeTree(_mirror_version) '
|
|
. 'PARTITION BY modulo(cityHash64(_mirror_key), 32) '
|
|
. 'ORDER BY (_mirror_generation, _mirror_key) SETTINGS index_granularity = 8192',
|
|
$this->clickHouse->qualifiedTable($dataset->targetTable()),
|
|
implode(', ', $definitions),
|
|
);
|
|
$this->clickHouse->command($sql);
|
|
$this->assertGenerationScopedSortingKey($dataset);
|
|
|
|
return $columns;
|
|
}
|
|
|
|
public function tableExists(BusinessMirrorDataset $dataset): bool
|
|
{
|
|
return (int) $this->clickHouse->scalar(
|
|
'SELECT count() FROM system.tables WHERE database = {database:String} AND name = {table:String}',
|
|
[
|
|
'database' => (string) config('business_data.clickhouse.database', 'caiwu_business_analytics'),
|
|
'table' => $dataset->targetTable(),
|
|
],
|
|
) === 1;
|
|
}
|
|
|
|
private function assertGenerationScopedSortingKey(BusinessMirrorDataset $dataset): void
|
|
{
|
|
$sortingKey = (string) $this->clickHouse->scalar(
|
|
'SELECT sorting_key FROM system.tables '
|
|
. 'WHERE database = {database:String} AND name = {table:String}',
|
|
[
|
|
'database' => (string) config('business_data.clickhouse.database', 'caiwu_business_analytics'),
|
|
'table' => $dataset->targetTable(),
|
|
],
|
|
);
|
|
$normalized = strtolower((string) preg_replace('/[`()\s]+/', '', $sortingKey));
|
|
if ($normalized !== '_mirror_generation,_mirror_key') {
|
|
throw new RuntimeException('business_data_columnar_generation_key_mismatch');
|
|
}
|
|
}
|
|
|
|
/** @param array<string,mixed> $column */
|
|
private function type(array $column): string
|
|
{
|
|
$type = strtolower((string) ($column['data_type'] ?? ''));
|
|
$precision = min(max((int) ($column['numeric_precision'] ?? 20), 1), 76);
|
|
$scale = min(max((int) ($column['numeric_scale'] ?? 0), 0), $precision);
|
|
|
|
return match ($type) {
|
|
'bit', 'boolean' => 'UInt8',
|
|
// SQL Server tinyint is unsigned (0..255), unlike MySQL tinyint.
|
|
'tinyint' => 'UInt8',
|
|
'smallint' => 'Int16',
|
|
'int', 'integer' => 'Int32',
|
|
'bigint' => 'Int64',
|
|
'decimal', 'numeric', 'money', 'smallmoney' => sprintf('Decimal(%d, %d)', $precision, $scale),
|
|
'float', 'real', 'double', 'double precision' => 'Float64',
|
|
'date' => 'Date32',
|
|
'datetime', 'datetime2', 'smalldatetime', 'timestamp without time zone', 'timestamp with time zone' => "DateTime64(6, 'Asia/Shanghai')",
|
|
'uniqueidentifier', 'uuid' => 'UUID',
|
|
default => 'String',
|
|
};
|
|
}
|
|
}
|