安装明细按发货时间保留2026年起数据 #575

Merged
qiuxiang merged 1 commits from fix/qiuxiang-install-shipment-cutoff-2026 into main 2026-10-26 09:50:13 +08:00
6 changed files with 283 additions and 9 deletions

View File

@ -684,6 +684,8 @@ return [
'target' => 'js_fys_ticketing_install_view',
'keys' => ['Id'],
'watermark' => null,
// 安装明细只保留 2026 年起的发货记录;空发货时间不进入镜像。
'minimum_datetime' => ['column' => 'ShipTIme', 'value' => '2026-01-01 00:00:00'],
'indexes' => [
['SubmitTime', 'Id'],
],
@ -691,6 +693,14 @@ return [
'jiansun.dbo.fys_Ticketing_install_item' => [
'source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'fys_Ticketing_install_item',
'target' => 'js_fys_ticketing_install_item', 'keys' => ['Id'], 'watermark' => 'modifyTime',
// 主表与明细使用不同的关联字段,明细仅同步仍保留在安装主表中的发货记录。
'parent_minimum_datetime' => [
'column' => 'ShipTIme',
'value' => '2026-01-01 00:00:00',
'parent_dataset' => 'jiansun.dbo.fys_Ticketing_install_view',
'child_key' => 'guid',
'parent_key' => 'SourceGuid',
],
'indexes' => [['guid', 'Id']],
],
'jiansun.dbo.finance_purchaseReturn' => ['source_key' => 'jiansun', 'schema' => 'dbo', 'object' => 'finance_purchaseReturn', 'target' => 'js_finance_purchase_return', 'keys' => ['id']],

View File

@ -11,6 +11,10 @@ class InstallationDetailMirrorRepository
public const DATASET = 'jiansun.dbo.fys_Ticketing_install_view';
public const ITEM_DATASET = 'jiansun.dbo.fys_Ticketing_install_item';
private bool $minimumShipTimeResolved = false;
private ?string $resolvedMinimumShipTime = null;
public function __construct(
private readonly GenericBusinessMirrorRepository $mirror,
private readonly BusinessMirrorDatasetRegistry $datasets,
@ -34,6 +38,7 @@ class InstallationDetailMirrorRepository
$latest = $connection->table($dataset->targetTable())
->selectRaw('MAX(Id) AS id')
->groupBy('SourceGuid');
$this->applyShipmentCutoff($latest);
$query = $connection->table($dataset->targetTable() . ' AS v')
->joinSub($latest, 'latest', static fn ($join) => $join->on('latest.id', '=', 'v.Id'))
->select('v.*');
@ -55,8 +60,9 @@ class InstallationDetailMirrorRepository
}
$mainExists = $connection->table($dataset->targetTable())
->where('SourceGuid', $guid)
->exists();
->where('SourceGuid', $guid);
$this->applyShipmentCutoff($mainExists);
$mainExists = $mainExists->exists();
if (! $mainExists) {
return ['data' => [], 'ready' => true];
}
@ -80,6 +86,24 @@ class InstallationDetailMirrorRepository
&& Schema::connection($connection)->hasTable($items->targetTable());
}
private function applyShipmentCutoff($query): void
{
$minimumShipTime = $this->minimumShipTime();
if ($minimumShipTime !== null) {
$query->whereNotNull('ShipTIme')->where('ShipTIme', '>=', $minimumShipTime);
}
}
private function minimumShipTime(): ?string
{
if (! $this->minimumShipTimeResolved) {
$this->resolvedMinimumShipTime = $this->datasets->minimumDatetime(self::DATASET)['value'] ?? null;
$this->minimumShipTimeResolved = true;
}
return $this->resolvedMinimumShipTime;
}
/** @return array{rows:int,requested:float,settled:float,paid:float} */
public function summary(array $filters = []): array
{
@ -91,6 +115,7 @@ class InstallationDetailMirrorRepository
$latest = $connection->table($dataset->targetTable())
->selectRaw('MAX(Id) AS id')
->groupBy('SourceGuid');
$this->applyShipmentCutoff($latest);
$query = $connection->table($dataset->targetTable() . ' AS v')
->joinSub($latest, 'latest', static fn ($join) => $join->on('latest.id', '=', 'v.Id'));
$this->applyFilters($query, $filters, $this->datasets->get(self::ITEM_DATASET)->targetTable());
@ -123,6 +148,17 @@ class InstallationDetailMirrorRepository
$query = $connection->table($items->targetTable() . ' AS i')
->whereNotNull('i.kfshTime');
$minimumShipTime = $this->minimumShipTime();
if ($minimumShipTime !== null) {
$mainTable = $dataset->targetTable();
$query->whereExists(function ($sub) use ($mainTable, $minimumShipTime): void {
$sub->selectRaw('1')
->from($mainTable . ' AS retained_main')
->whereColumn('retained_main.SourceGuid', 'i.guid')
->whereNotNull('retained_main.ShipTIme')
->where('retained_main.ShipTIme', '>=', $minimumShipTime);
});
}
if ($from !== '') {
$query->where('i.kfshTime', '>=', $from);
@ -145,10 +181,12 @@ class InstallationDetailMirrorRepository
->values()
->all();
foreach (array_chunk($guids, 1000) as $guidChunk) {
$connection->table($dataset->targetTable())
$companyQuery = $connection->table($dataset->targetTable())
->whereIn('SourceGuid', $guidChunk)
->select(['Id', 'SourceGuid', 'installCompanyName'])
->orderBy('Id')
->orderBy('Id');
$this->applyShipmentCutoff($companyQuery);
$companyQuery
->get()
->each(static function (object $row) use (&$companyByGuid): void {
$companyByGuid[(string) $row->SourceGuid] = trim((string) ($row->installCompanyName ?? ''));

View File

@ -90,6 +90,56 @@ class BusinessMirrorDatasetRegistry
return $this->normalizeFixedFilters($key, (array) ($definition['fixed_filters'] ?? []));
}
/** @return array{column:string,value:string}|null */
public function minimumDatetime(string $key): ?array
{
$definitions = $this->definitions();
$definition = (array) ($definitions[$key] ?? []);
if ($definition === []) {
throw new InvalidArgumentException('未知的业务镜像数据集:' . $key);
}
return $this->normalizeMinimumDatetime($key, $definition['minimum_datetime'] ?? null);
}
/** @return array{column:string,value:string,parent_dataset:string,child_key:string,parent_key:string}|null */
public function parentMinimumDatetime(string $key): ?array
{
$definitions = $this->definitions();
$definition = (array) ($definitions[$key] ?? []);
if ($definition === []) {
throw new InvalidArgumentException('未知的业务镜像数据集:' . $key);
}
$minimum = $this->normalizeMinimumDatetime($key, $definition['parent_minimum_datetime'] ?? null);
if ($minimum === null) {
return null;
}
$parentKey = trim((string) ($definition['parent_minimum_datetime']['parent_dataset'] ?? ''));
$childKey = trim((string) ($definition['parent_minimum_datetime']['child_key'] ?? ''));
$parentColumn = trim((string) ($definition['parent_minimum_datetime']['parent_key'] ?? ''));
$parent = (array) ($definitions[$parentKey] ?? []);
if ($parent === []) {
throw new InvalidArgumentException('业务镜像父数据集不存在:' . $key);
}
if ((string) ($parent['source_key'] ?? '') !== (string) ($definition['source_key'] ?? '')) {
throw new InvalidArgumentException('业务镜像父数据集必须使用同一上游来源:' . $key);
}
foreach ([$childKey, $parentColumn] as $identifier) {
if (preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/D', $identifier) !== 1) {
throw new InvalidArgumentException('业务镜像父数据集关联字段无效:' . $key);
}
}
return $minimum + [
'parent_dataset' => $parentKey,
'child_key' => $childKey,
'parent_key' => $parentColumn,
];
}
private function make(string $key, array $definition): BusinessMirrorDataset
{
$source = trim((string) ($definition['source_key'] ?? ''));
@ -180,4 +230,32 @@ class BusinessMirrorDatasetRegistry
return $normalized;
}
/** @return array{column:string,value:string}|null */
private function normalizeMinimumDatetime(string $key, mixed $definition): ?array
{
if ($definition === null) {
return null;
}
if (! is_array($definition)) {
throw new InvalidArgumentException('业务镜像数据集的日期下限无效:' . $key);
}
$column = trim((string) ($definition['column'] ?? ''));
$value = trim((string) ($definition['value'] ?? ''));
if (preg_match('/^[A-Za-z_][A-Za-z0-9_]*$/D', $column) !== 1) {
throw new InvalidArgumentException('业务镜像数据集的日期下限字段无效:' . $key);
}
if (preg_match('/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/D', $value) !== 1) {
throw new InvalidArgumentException('业务镜像数据集的日期下限值无效:' . $key);
}
$date = \DateTimeImmutable::createFromFormat('!Y-m-d H:i:s', $value);
$errors = \DateTimeImmutable::getLastErrors();
if ($date === false || ($errors !== false && ($errors['warning_count'] > 0 || $errors['error_count'] > 0))
|| $date->format('Y-m-d H:i:s') !== $value) {
throw new InvalidArgumentException('业务镜像数据集的日期下限值无效:' . $key);
}
return ['column' => $column, 'value' => $value];
}
}

View File

@ -160,6 +160,18 @@ class BusinessMirrorSynchronizer
array_push($bindings, ...$excludedBindings);
}
[$minimumDatetime, $minimumBindings] = $this->minimumDatetimeFilter($dataset, $columns);
if ($minimumDatetime !== null) {
$where[] = $minimumDatetime;
array_push($bindings, ...$minimumBindings);
}
[$parentMinimumDatetime, $parentMinimumBindings] = $this->parentMinimumDatetimeFilter($dataset);
if ($parentMinimumDatetime !== null) {
$where[] = $parentMinimumDatetime;
array_push($bindings, ...$parentMinimumBindings);
}
foreach ($dataset->fixedFilters() as $column => $value) {
if (is_array($value)) {
$where[] = $this->quote($column) . ' IN (' . implode(', ', array_fill(0, count($value), '?')) . ')';
@ -255,6 +267,60 @@ class BusinessMirrorSynchronizer
];
}
/** @return array{0:?string,1:list<string>} */
private function minimumDatetimeFilter(BusinessMirrorDataset $dataset, array $columns): array
{
$minimum = $this->datasets->minimumDatetime($dataset->key());
if ($minimum === null) {
return [null, []];
}
$available = array_map(
static fn (array $column): string => strtolower((string) ($column['column_name'] ?? '')),
$columns,
);
if (! in_array(strtolower($minimum['column']), $available, true)) {
throw new RuntimeException('business_mirror_minimum_datetime_column_missing');
}
return [
$this->quote($minimum['column']) . ' >= ?',
[$this->minimumDatetimeBinding($minimum['value'])],
];
}
/** @return array{0:?string,1:list<string>} */
private function parentMinimumDatetimeFilter(BusinessMirrorDataset $dataset): array
{
$minimum = $this->datasets->parentMinimumDatetime($dataset->key());
if ($minimum === null) {
return [null, []];
}
$parent = $this->datasets->get($minimum['parent_dataset']);
$parentTable = $this->quote($parent->sourceSchema()) . '.' . $this->quote($parent->sourceObject());
$alias = $this->quote('minimum_parent');
$parentKey = $alias . '.' . $this->quote($minimum['parent_key']);
$outerKey = $this->quote($dataset->sourceObject()) . '.' . $this->quote($minimum['child_key']);
$column = $alias . '.' . $this->quote($minimum['column']);
return [
"EXISTS (SELECT 1 FROM {$parentTable} AS {$alias}"
. " WHERE {$parentKey} = {$outerKey} AND {$column} >= ?)",
[$this->minimumDatetimeBinding($minimum['value'])],
];
}
private function minimumDatetimeBinding(string $value): string
{
$date = Carbon::createFromFormat('!Y-m-d H:i:s', $value);
if ($date === false) {
throw new RuntimeException('business_mirror_minimum_datetime_invalid');
}
return (string) $this->sqlServerDateTime($date);
}
/** @param array<string,mixed> $column */
private function selectExpression(array $column, ?string $alias = null): string
{

View File

@ -163,6 +163,67 @@ class BusinessMirrorSynchronizerTest extends TestCase
$this->assertSame([2, 3, 4, 5, 6], $statement->fetchAll(\PDO::FETCH_COLUMN));
}
public function test_minimum_datetime_filter_excludes_old_and_null_source_rows(): void
{
$definitions = config('upstream_mirrors.datasets');
$definitions['test.dbo.records']['minimum_datetime'] = [
'column' => 'businessDate',
'value' => '2026-01-01 00:00:00',
];
config(['upstream_mirrors.datasets' => $definitions]);
$source = new \PDO('sqlite::memory:');
$source->exec("CREATE TABLE records (Id INTEGER, businessDate TEXT);
INSERT INTO records VALUES (1, '2025-12-31T23:59:59.000'),
(2, '2026-01-01T00:00:00.000'), (3, '2026-06-01T12:00:00.000'), (4, NULL)");
[$sql, $bindings] = (new \ReflectionMethod($this->synchronizer, 'minimumDatetimeFilter'))->invoke(
$this->synchronizer,
(new BusinessMirrorDatasetRegistry())->get('test.dbo.records'),
[
['column_name' => 'Id'],
['column_name' => 'businessDate'],
],
);
$statement = $source->prepare('SELECT Id FROM records WHERE ' . $sql . ' ORDER BY Id');
$statement->execute($bindings);
$this->assertSame([2, 3], $statement->fetchAll(\PDO::FETCH_COLUMN));
$this->assertSame(['2026-01-01T00:00:00.000'], $bindings);
}
public function test_parent_minimum_datetime_filter_excludes_old_null_and_orphan_details(): void
{
$definitions = config('upstream_mirrors.datasets');
$definitions['test.dbo.parents'] = array_replace($definitions['test.dbo.records'], [
'object' => 'parents',
'target' => 'test_parents',
]);
$definitions['test.dbo.children'] = array_replace($definitions['test.dbo.records'], [
'object' => 'children',
'target' => 'test_children',
'parent_minimum_datetime' => [
'column' => 'ShipTIme',
'value' => '2026-01-01 00:00:00',
'parent_dataset' => 'test.dbo.parents',
'child_key' => 'guid',
'parent_key' => 'SourceGuid',
],
]);
config(['upstream_mirrors.datasets' => $definitions]);
$source = new \PDO('sqlite::memory:');
$source->exec("ATTACH DATABASE ':memory:' AS dbo;
CREATE TABLE dbo.parents (SourceGuid TEXT, ShipTIme TEXT);
CREATE TABLE children (Id INTEGER, guid TEXT);
INSERT INTO dbo.parents VALUES ('A', '2025-12-31T23:59:59.000'),
('B', '2026-01-01T00:00:00.000'), ('C', NULL);
INSERT INTO children VALUES (1, 'A'), (2, 'B'), (3, 'C'), (4, 'ORPHAN')");
[$sql, $bindings] = (new \ReflectionMethod($this->synchronizer, 'parentMinimumDatetimeFilter'))->invoke(
$this->synchronizer,
(new BusinessMirrorDatasetRegistry())->get('test.dbo.children'),
);
$statement = $source->prepare('SELECT Id FROM children WHERE ' . $sql . ' ORDER BY Id');
$statement->execute($bindings);
$this->assertSame([2], $statement->fetchAll(\PDO::FETCH_COLUMN));
}
public function test_incremental_sync_uses_overlap_window_and_updates_existing_key(): void
{
$this->query->batches = [[

View File

@ -23,12 +23,16 @@ class InstallationDetailMirrorRepositoryTest extends TestCase
foreach ([InstallationDetailMirrorRepository::DATASET=>'fixture_install',InstallationDetailMirrorRepository::ITEM_DATASET=>'fixture_items'] as $key=>$table) {
$definitions[$key]=['source_key'=>'jiansun','schema'=>'dbo','object'=>explode('.',$key)[2],'target'=>$table,'keys'=>['Id']];
}
$definitions[InstallationDetailMirrorRepository::DATASET]['minimum_datetime'] = [
'column' => 'ShipTIme',
'value' => '2026-01-01 00:00:00',
];
config(['upstream_mirrors.datasets'=>$definitions]);
$schema=Schema::connection('business_mirror');
$schema->create('business_mirror_table_states',function(Blueprint $t){$t->string('dataset_key')->primary();$t->dateTime('initial_completed_at')->nullable();});
$schema->create('fixture_install',function(Blueprint $t){
$t->integer('Id');
foreach(['SourceGuid','OrderNum','ordernum_id','shop_name','SubmitTime','kfshTime','cwshTime','apistatus','kfsh','cwsh','BidStatus','costtaker','installComplete','Delete','installCompanyName','TypeDto'] as $c) $t->string($c)->nullable();
foreach(['SourceGuid','OrderNum','ordernum_id','shop_name','SubmitTime','ShipTIme','kfshTime','cwshTime','apistatus','kfsh','cwsh','BidStatus','costtaker','installComplete','Delete','installCompanyName','TypeDto'] as $c) $t->string($c)->nullable();
$t->integer('StatusId')->nullable()->default(0);
foreach(['installcost','settleAmount','acceptAmount'] as $c) $t->decimal($c,14,2)->default(0);
});
@ -43,10 +47,12 @@ class InstallationDetailMirrorRepositoryTest extends TestCase
});
foreach(array_keys($definitions) as $key) DB::connection('business_mirror')->table('business_mirror_table_states')->insert(['dataset_key'=>$key,'initial_completed_at'=>'2026-08-01']);
DB::connection('business_mirror')->table('fixture_install')->insert([
['Id'=>1,'SourceGuid'=>'A','OrderNum'=>'OLD','shop_name'=>'旧店','SubmitTime'=>'2026-08-01','installcost'=>500],
['Id'=>2,'SourceGuid'=>'A','OrderNum'=>'NEW','shop_name'=>'测试店','SubmitTime'=>'2026-08-20 23:59:59','installcost'=>20],
['Id'=>3,'SourceGuid'=>'B','OrderNum'=>'OTHER','shop_name'=>'其他店','SubmitTime'=>'2026-08-20','installcost'=>30],
['Id'=>4,'SourceGuid'=>'C','OrderNum'=>'NEXT','shop_name'=>'测试店','SubmitTime'=>'2026-08-21','installcost'=>40],
['Id'=>1,'SourceGuid'=>'A','OrderNum'=>'OLD','shop_name'=>'旧店','SubmitTime'=>'2026-08-01','ShipTIme'=>'2026-08-02','installcost'=>500],
['Id'=>2,'SourceGuid'=>'A','OrderNum'=>'NEW','shop_name'=>'测试店','SubmitTime'=>'2026-08-20 23:59:59','ShipTIme'=>'2026-08-21','installcost'=>20],
['Id'=>3,'SourceGuid'=>'B','OrderNum'=>'OTHER','shop_name'=>'其他店','SubmitTime'=>'2026-08-20','ShipTIme'=>'2026-08-21','installcost'=>30],
['Id'=>4,'SourceGuid'=>'C','OrderNum'=>'NEXT','shop_name'=>'测试店','SubmitTime'=>'2026-08-21','ShipTIme'=>'2026-08-22','installcost'=>40],
['Id'=>5,'SourceGuid'=>'D','OrderNum'=>'PRE-2026','shop_name'=>'测试店','SubmitTime'=>'2026-08-21','ShipTIme'=>'2025-12-31 23:59:59','installcost'=>999],
['Id'=>6,'SourceGuid'=>'E','OrderNum'=>'NULL-SHIP','shop_name'=>'测试店','SubmitTime'=>'2026-08-21','ShipTIme'=>null,'installcost'=>999],
]);
DB::connection('business_mirror')->table('fixture_items')->insert([
['Id'=>1,'guid'=>'A','shopproid'=>'PRODUCT-A'],['Id'=>2,'guid'=>'A','shopproid'=>'PRODUCT-A'],
@ -76,6 +82,19 @@ class InstallationDetailMirrorRepositoryTest extends TestCase
$this->assertSame([],$this->repository->details('missing')['data']);
}
public function test_shipment_cutoff_hides_old_and_null_records_from_list_summary_and_details(): void
{
DB::connection('business_mirror')->table('fixture_items')->insert([
['Id'=>30,'guid'=>'D','shopproid'=>'OLD-PRODUCT'],
['Id'=>31,'guid'=>'E','shopproid'=>'NULL-PRODUCT'],
]);
$this->assertSame(3,$this->repository->paginate(1,50)['total']);
$this->assertSame(3,$this->repository->summary()['rows']);
$this->assertSame([],$this->repository->details('D')['data']);
$this->assertSame([],$this->repository->details('E')['data']);
}
public function test_status_filters_and_child_limit_are_explicit(): void
{
DB::connection('business_mirror')->table('fixture_install')->where('Id',2)->update(['kfsh'=>'1','Delete'=>'0','StatusId'=>2]);
@ -124,6 +143,8 @@ class InstallationDetailMirrorRepositoryTest extends TestCase
['Id'=>21,'guid'=>'A','shopproid'=>'P2','Sourceno'=>'A-1','kfshTime'=>'2026-09-30 18:00:00','settleItemAmount'=>39.90,'acceptItemAmount'=>0.00],
['Id'=>22,'guid'=>'B','shopproid'=>'P3','Sourceno'=>'B-1','kfshTime'=>'2026-09-15 10:00:00','settleItemAmount'=>80.00,'acceptItemAmount'=>70.00],
['Id'=>23,'guid'=>'C','shopproid'=>'P4','Sourceno'=>'C-1','kfshTime'=>'2026-10-01 00:00:00','settleItemAmount'=>500.00,'acceptItemAmount'=>500.00],
['Id'=>24,'guid'=>'D','shopproid'=>'OLD','Sourceno'=>'D-1','kfshTime'=>'2026-09-10 10:00:00','settleItemAmount'=>999.00,'acceptItemAmount'=>999.00],
['Id'=>25,'guid'=>'E','shopproid'=>'NULL','Sourceno'=>'E-1','kfshTime'=>'2026-09-10 10:00:00','settleItemAmount'=>999.00,'acceptItemAmount'=>999.00],
]);
$result=$this->repository->companySummary([