> */ public function ensure(BusinessMirrorDataset $dataset): array { if (! in_array($this->placements->targetStore($dataset->key()), ['clickhouse', 'parquet'], true)) { throw new RuntimeException('business_data_dataset_not_columnar'); } $columns = $this->sourceSchemas->columns($dataset); $definitions = ['`_mirror_key` String']; foreach ($columns as $column) { $definitions[] = $this->clickHouse->identifier((string) $column['column_name']) . ' Nullable(' . $this->type($column) . ')'; } $definitions[] = '`_mirror_generation` UUID'; $definitions[] = "`_mirror_synced_at` DateTime64(6, 'Asia/Shanghai')"; $definitions[] = '`_mirror_version` UInt64'; // Hash partitioning keeps every version of one business key in the same // partition. The generation must also be part of the replacement key: // otherwise a newer backfill generation would erase the retained one. $sql = sprintf( 'CREATE TABLE IF NOT EXISTS %s (%s) ' . 'ENGINE = ReplacingMergeTree(_mirror_version) ' . 'PARTITION BY modulo(cityHash64(_mirror_key), 32) ' . 'ORDER BY (_mirror_generation, _mirror_key) SETTINGS index_granularity = 8192', $this->clickHouse->qualifiedTable($dataset->targetTable()), implode(', ', $definitions), ); $this->clickHouse->command($sql); $this->assertGenerationScopedSortingKey($dataset); return $columns; } public function tableExists(BusinessMirrorDataset $dataset): bool { return (int) $this->clickHouse->scalar( 'SELECT count() FROM system.tables WHERE database = {database:String} AND name = {table:String}', [ 'database' => (string) config('business_data.clickhouse.database', 'caiwu_business_analytics'), 'table' => $dataset->targetTable(), ], ) === 1; } private function assertGenerationScopedSortingKey(BusinessMirrorDataset $dataset): void { $sortingKey = (string) $this->clickHouse->scalar( 'SELECT sorting_key FROM system.tables ' . 'WHERE database = {database:String} AND name = {table:String}', [ 'database' => (string) config('business_data.clickhouse.database', 'caiwu_business_analytics'), 'table' => $dataset->targetTable(), ], ); $normalized = strtolower((string) preg_replace('/[`()\s]+/', '', $sortingKey)); if ($normalized !== '_mirror_generation,_mirror_key') { throw new RuntimeException('business_data_columnar_generation_key_mismatch'); } } /** @param array $column */ private function type(array $column): string { $type = strtolower((string) ($column['data_type'] ?? '')); $precision = min(max((int) ($column['numeric_precision'] ?? 20), 1), 76); $scale = min(max((int) ($column['numeric_scale'] ?? 0), 0), $precision); return match ($type) { 'bit', 'boolean' => 'UInt8', // SQL Server tinyint is unsigned (0..255), unlike MySQL tinyint. 'tinyint' => 'UInt8', 'smallint' => 'Int16', 'int', 'integer' => 'Int32', 'bigint' => 'Int64', 'decimal', 'numeric', 'money', 'smallmoney' => sprintf('Decimal(%d, %d)', $precision, $scale), 'float', 'real', 'double', 'double precision' => 'Float64', 'date' => 'Date32', 'datetime', 'datetime2', 'smalldatetime', 'timestamp without time zone', 'timestamp with time zone' => "DateTime64(6, 'Asia/Shanghai')", 'uniqueidentifier', 'uuid' => 'UUID', default => 'String', }; } }