fix: 停止同步抖音旧原始账单
保留抖音 2026-07 至 2026-11 校准月度账单,排除 2025-07 至 2026-06 原始镜像回填。
This commit is contained in:
parent
5f551846d7
commit
64abf3ffd1
@ -600,6 +600,13 @@ return [
|
||||
'target' => 'ops_jst_original_bill_info',
|
||||
'keys' => ['Id'],
|
||||
'watermark' => 'updateTime',
|
||||
// 抖音 2026-07 起已切换到校准月度账单,旧原始镜像不得再同步回来。
|
||||
'excluded_date_range' => [
|
||||
'column' => 'fee_time',
|
||||
'from' => '2025-07-01',
|
||||
'to' => '2026-07-01',
|
||||
'fixed_filters' => ['channel' => 'TouTiaoFXGZJ'],
|
||||
],
|
||||
'columns' => [
|
||||
'Id', 'shop_id', 'channel', 'fee_time', 'del_flag', 'primary_key',
|
||||
'createTime', 'updateTime', 'auto_id', 'fee_name', 'inout_type',
|
||||
|
||||
@ -227,7 +227,7 @@ class BusinessMirrorSynchronizer
|
||||
return $this->queries->forSource($dataset->sourceKey())->select($sql, $bindings);
|
||||
}
|
||||
|
||||
/** @return array{0:?string,1:list<string>} */
|
||||
/** @return array{0:?string,1:list<scalar>} */
|
||||
private function excludedDateRange(BusinessMirrorDataset $dataset): array
|
||||
{
|
||||
$range = $this->datasets->definitionValue($dataset->key(), 'excluded_date_range');
|
||||
@ -242,9 +242,20 @@ class BusinessMirrorSynchronizer
|
||||
throw new \InvalidArgumentException('镜像排除日期范围无效。');
|
||||
}
|
||||
$column = $this->quote((string) $range['column']);
|
||||
[$excludedFilterBypass, $excludedBindings] = $this->excludedRowFilterBypass($range['fixed_filters'] ?? []);
|
||||
if (! isset($range['parent_dataset'])) {
|
||||
if ($excludedFilterBypass !== null) {
|
||||
return [
|
||||
"({$excludedFilterBypass} OR {$column} IS NULL OR {$column} < ? OR {$column} >= ?)",
|
||||
[...$excludedBindings, $from, $to],
|
||||
];
|
||||
}
|
||||
|
||||
return ["({$column} IS NULL OR {$column} < ? OR {$column} >= ?)", [$from, $to]];
|
||||
}
|
||||
if ($excludedFilterBypass !== null) {
|
||||
throw new \InvalidArgumentException('镜像父子日期排除不支持固定过滤条件。');
|
||||
}
|
||||
|
||||
$parent = $this->datasets->get((string) $range['parent_dataset']);
|
||||
if ($parent->sourceKey() !== $dataset->sourceKey()) {
|
||||
@ -267,6 +278,31 @@ class BusinessMirrorSynchronizer
|
||||
];
|
||||
}
|
||||
|
||||
/** @return array{0:?string,1:list<scalar>} */
|
||||
private function excludedRowFilterBypass(mixed $filters): array
|
||||
{
|
||||
if ($filters === null || $filters === []) {
|
||||
return [null, []];
|
||||
}
|
||||
if (! is_array($filters)) {
|
||||
throw new \InvalidArgumentException('镜像排除日期范围的固定过滤条件无效。');
|
||||
}
|
||||
|
||||
$parts = [];
|
||||
$bindings = [];
|
||||
foreach ($filters as $column => $value) {
|
||||
$column = is_string($column) ? trim($column) : '';
|
||||
if (preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/D', $column) !== 1 || ! is_scalar($value)) {
|
||||
throw new \InvalidArgumentException('镜像排除日期范围的固定过滤条件无效。');
|
||||
}
|
||||
$quoted = $this->quote($column);
|
||||
$parts[] = "({$quoted} IS NULL OR {$quoted} <> ?)";
|
||||
$bindings[] = $value;
|
||||
}
|
||||
|
||||
return ['(' . implode(' OR ', $parts) . ')', $bindings];
|
||||
}
|
||||
|
||||
/** @return array{0:?string,1:list<string>} */
|
||||
private function minimumDatetimeFilter(BusinessMirrorDataset $dataset, array $columns): array
|
||||
{
|
||||
|
||||
@ -137,6 +137,35 @@ class BusinessMirrorSynchronizerTest extends TestCase
|
||||
$this->assertSame([1, 4, 5], $statement->fetchAll(\PDO::FETCH_COLUMN));
|
||||
}
|
||||
|
||||
public function test_excluded_date_range_can_be_limited_to_fixed_filters(): void
|
||||
{
|
||||
$definitions = config('upstream_mirrors.datasets');
|
||||
$definitions['test.dbo.records']['excluded_date_range'] = [
|
||||
'column' => 'businessDate',
|
||||
'from' => '2025-07-01',
|
||||
'to' => '2026-07-01',
|
||||
'fixed_filters' => ['channel' => 'TouTiaoFXGZJ'],
|
||||
];
|
||||
config(['upstream_mirrors.datasets' => $definitions]);
|
||||
$source = new \PDO('sqlite::memory:');
|
||||
$source->exec("CREATE TABLE records (Id INTEGER, businessDate TEXT, channel TEXT);
|
||||
INSERT INTO records VALUES
|
||||
(1, '2026-06-30', 'TouTiaoFXGZJ'),
|
||||
(2, '2026-06-30', 'OtherChannel'),
|
||||
(3, '2026-07-01', 'TouTiaoFXGZJ'),
|
||||
(4, NULL, 'TouTiaoFXGZJ'),
|
||||
(5, '2025-06-30', 'TouTiaoFXGZJ'),
|
||||
(6, '2026-06-30', NULL)");
|
||||
[$sql, $bindings] = (new \ReflectionMethod($this->synchronizer, 'excludedDateRange'))->invoke(
|
||||
$this->synchronizer, (new BusinessMirrorDatasetRegistry())->get('test.dbo.records'),
|
||||
);
|
||||
$statement = $source->prepare('SELECT Id FROM records WHERE ' . $sql . ' ORDER BY Id');
|
||||
$statement->execute($bindings);
|
||||
|
||||
$this->assertSame([2, 3, 4, 5, 6], $statement->fetchAll(\PDO::FETCH_COLUMN));
|
||||
$this->assertSame(['TouTiaoFXGZJ', '2025-07-01', '2026-07-01'], $bindings);
|
||||
}
|
||||
|
||||
public function test_excluded_year_details_keep_retained_shared_and_orphan_parents(): void
|
||||
{
|
||||
$definitions = config('upstream_mirrors.datasets');
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user