diff --git a/app/Business/StoreReceiptDifferenceMaterialization.php b/app/Business/StoreReceiptDifferenceMaterialization.php new file mode 100644 index 00000000..a07609db --- /dev/null +++ b/app/Business/StoreReceiptDifferenceMaterialization.php @@ -0,0 +1,217 @@ + */ + private static array $memo = []; + + public static function connection(): string + { + return (string) config('upstream_mirrors.connection', 'business_mirror'); + } + + public static function table(): string + { + return self::TABLE; + } + + public static function stateTable(): string + { + return self::STATE_TABLE; + } + + /** + * 当前规则表达式的指纹。规则改动后指纹变化，旧的物化结果会自动视为过期。 + */ + public static function ruleVersion(): string + { + return md5(implode('|', [ + Pages::receiptDefaultDifferenceSql(), + Pages::receiptDifferenceRemarkIndexSql(), + Pages::receiptEffectiveDifferenceSql(), + ])); + } + + /** + * 物化列与状态表是否已就绪（迁移是否执行过）。 + * + * 结果缓存 10 分钟：schema 探测在共享镜像库上每次要 0.3～0.5 秒， + * 不能放在每个页面请求的热路径上。迁移执行后由 markMaterialized() 主动失效。 + */ + public static function available(bool $fresh = false): bool + { + static $memo = null; + + if (! $fresh) { + if ($memo !== null) { + return $memo; + } + + return $memo = (bool) Cache::remember( + self::availabilityCacheKey(), + now()->addMinutes(10), + static fn (): bool => self::detectAvailability(), + ); + } + + $available = self::detectAvailability(); + if ($available) { + Cache::put(self::availabilityCacheKey(), true, now()->addMinutes(10)); + } + + return $memo = $available; + } + + private static function detectAvailability(): bool + { + try { + $schema = Schema::connection(self::connection()); + + return $schema->hasTable(self::TABLE) + && $schema->hasTable(self::STATE_TABLE) + && $schema->hasColumn(self::TABLE, self::REASON_INDEX_COLUMN) + && $schema->hasColumn(self::TABLE, self::BASE_DIFFERENCE_COLUMN) + && $schema->hasColumn(self::TABLE, self::HAS_DIFFERENCE_COLUMN); + } catch (\Throwable) { + return false; + } + } + + private static function availabilityCacheKey(): string + { + return 'store-receipt-materialization-available:v1'; + } + + /** + * 该账期是否已经按当前规则物化完成。未物化时页面回退到实时规则表达式。 + */ + public static function materialized(string $month): bool + { + $month = substr(trim($month), 0, 10); + if ($month === '' || ! self::available()) { + return false; + } + + if (array_key_exists($month, self::$memo)) { + return self::$memo[$month]; + } + + $cacheKey = 'store-receipt-materialized:' . $month . ':' . self::ruleVersion(); + + return self::$memo[$month] = (bool) Cache::remember($cacheKey, now()->addMinutes(10), function () use ($month): bool { + $row = DB::connection(self::connection()) + ->table(self::STATE_TABLE) + ->where('report_month', $month) + ->first(); + + if ($row === null || (string) ($row->rule_version ?? '') !== self::ruleVersion()) { + return false; + } + + // 过渡期保护：尚未部署新代码的同步任务重刷账期时不会写物化列， + // 一旦出现 NULL 就回退实时口径，避免页面把未物化的行当成 0 差额。 + return ! DB::connection(self::connection()) + ->table(self::TABLE) + ->where('report_month', $month) + ->whereNull(self::HAS_DIFFERENCE_COLUMN) + ->limit(1) + ->exists(); + }); + } + + /** + * 该账期的数据版本：物化后自增，页面聚合/联动统计的缓存键带上它即可精确失效。 + */ + public static function dataVersion(string $month): string + { + return (string) Cache::remember( + self::dataVersionKey($month), + now()->addDays(30), + static fn (): int => 1, + ); + } + + public static function bumpDataVersion(string $month): void + { + $key = self::dataVersionKey($month); + $current = (int) Cache::get($key, 1); + Cache::put($key, $current + 1, now()->addDays(30)); + } + + private static function dataVersionKey(string $month): string + { + return 'store-receipt-data-version:' . substr(trim($month), 0, 10); + } + + public static function markMaterialized(string $month, int $rowCount): void + { + $month = substr(trim($month), 0, 10); + $connection = self::connection(); + + DB::connection($connection)->table(self::STATE_TABLE)->updateOrInsert( + ['report_month' => $month], + [ + 'rule_version' => self::ruleVersion(), + 'row_count' => $rowCount, + 'materialized_at' => now(), + ], + ); + + Cache::forget('store-receipt-materialized:' . $month . ':' . self::ruleVersion()); + Cache::forget(self::availabilityCacheKey()); + self::bumpDataVersion($month); + unset(self::$memo[$month]); + } + + public static function forget(string $month): void + { + $month = substr(trim($month), 0, 10); + Cache::forget('store-receipt-materialized:' . $month . ':' . self::ruleVersion()); + unset(self::$memo[$month]); + } + + /** + * 账期相邻月份（物化跨月备注规则时需要一起重算）。 + * + * @return list + */ + public static function adjacentMonths(string $month): array + { + $date = \Illuminate\Support\Carbon::parse(substr($month, 0, 10) . ' 00:00:00'); + + return [ + $date->copy()->subMonthNoOverflow()->format('Y-m-01'), + $date->copy()->format('Y-m-01'), + $date->copy()->addMonthNoOverflow()->format('Y-m-01'), + ]; + } +} diff --git a/app/Console/Commands/MaterializeStoreReceiptDifferences.php b/app/Console/Commands/MaterializeStoreReceiptDifferences.php new file mode 100644 index 00000000..d58fc6b6 --- /dev/null +++ b/app/Console/Commands/MaterializeStoreReceiptDifferences.php @@ -0,0 +1,211 @@ +components->error('物化列或状态表不存在，请先执行数据库迁移。'); + + return self::FAILURE; + } + + $months = $this->targetMonths(); + if ($months === []) { + $this->components->warn('镜像里没有可物化的账期。'); + + return self::SUCCESS; + } + + $chunk = max(200, (int) $this->option('chunk')); + $dryRun = (bool) $this->option('dry-run'); + + foreach ($months as $month) { + $this->line(''); + $this->components->info(($dryRun ? '[dry-run] ' : '') . '账期 ' . substr($month, 0, 7)); + + if ($dryRun) { + $this->reportDryRun($month); + continue; + } + + $started = microtime(true); + $updated = $this->materializeMonth($month, $chunk); + if ($updated === null) { + continue; + } + + Materialization::markMaterialized($month, $updated); + $this->line(sprintf(' %s 行，用时 %.1f 秒', number_format($updated), microtime(true) - $started)); + } + + if ($dryRun) { + $this->line(''); + $this->components->info('dry-run 完成，未写入任何数据。'); + } + + return self::SUCCESS; + } + + /** + * @return list + */ + private function targetMonths(): array + { + $month = trim((string) $this->argument('month')); + if ($month !== '') { + if (preg_match('/^\d{4}-\d{2}$/', $month) === 1) { + $month .= '-01'; + } + + // 跨月备注规则依赖相邻月份，指定月份时一起重算前后两个月。 + return array_values(array_unique(Materialization::adjacentMonths($month))); + } + + return DB::connection(Materialization::connection()) + ->table(Materialization::table()) + ->select('report_month') + ->distinct() + ->orderBy('report_month') + ->pluck('report_month') + ->map(static fn ($value): string => substr((string) $value, 0, 10)) + ->all(); + } + + /** + * 分批把规则结果写进物化列；返回更新行数。 + */ + private function materializeMonth(string $month, int $chunk): ?int + { + $connection = Materialization::connection(); + $table = Materialization::table(); + + $exists = DB::connection($connection)->table($table)->where('report_month', $month)->exists(); + if (! $exists) { + $this->components->warn(' 镜像里没有该账期的数据，跳过。'); + + return null; + } + + $baseExpression = Pages::receiptDefaultDifferenceSql(); + $indexExpression = Pages::receiptDifferenceRemarkIndexSql(); + + $updated = 0; + $lastKey = ''; + $batch = 0; + + while (true) { + $keys = DB::connection($connection)->table($table) + ->where('report_month', $month) + ->where('row_key', '>', $lastKey) + ->orderBy('row_key') + ->limit($chunk) + ->pluck('row_key') + ->all(); + + if ($keys === []) { + break; + } + + $lastKey = (string) end($keys); + $batch++; + + // 第一遍：规则表达式只算一次，结果落列。 + DB::connection($connection)->table($table) + ->where('report_month', $month) + ->whereIn('row_key', $keys) + ->update([ + Materialization::BASE_DIFFERENCE_COLUMN => DB::raw($baseExpression), + Materialization::REASON_INDEX_COLUMN => DB::raw($indexExpression), + ]); + + // 第二遍：直接用刚写入的列推导「有/无差额」，避免重复执行规则链， + // 也避免依赖 SET 子句从左到右求值的库间差异。 + DB::connection($connection)->table($table) + ->where('report_month', $month) + ->whereIn('row_key', $keys) + ->update([ + Materialization::HAS_DIFFERENCE_COLUMN => DB::raw($this->hasDifferenceExpression()), + ]); + + $updated += count($keys); + $this->output->write('.'); + } + + if ($updated === 0) { + $this->line(''); + } else { + $this->line(' (' . $batch . ' 批)'); + } + + return $updated; + } + + /** + * 基于物化列的「有/无差额」推导：口径兜底行按 0 计。 + */ + private function hasDifferenceExpression(): string + { + $basisMarks = implode(', ', array_map( + static fn (string $remark): string => "'" . str_replace("'", "''", $remark) . "'", + Pages::RECEIPT_BASIS_FALLBACK_REMARKS, + )); + + return 'CASE WHEN ABS(CASE WHEN remark IN (' . $basisMarks . ')' + . ' AND COALESCE(' . Materialization::REASON_INDEX_COLUMN . ', 0) = 0 THEN 0' + . ' ELSE COALESCE(' . Materialization::BASE_DIFFERENCE_COLUMN . ', 0) END) > 0.005 THEN 1 ELSE 0 END'; + } + + /** + * dry-run：对比「库里的物化值」与「按当前规则重算」的差异，不写库。 + */ + private function reportDryRun(string $month): void + { + $connection = Materialization::connection(); + $table = Materialization::table(); + + $total = (int) DB::connection($connection)->table($table)->where('report_month', $month)->count(); + if ($total === 0) { + $this->components->warn(' 镜像里没有该账期的数据，跳过。'); + + return; + } + + $baseExpression = Pages::receiptDefaultDifferenceSql(); + $indexExpression = Pages::receiptDifferenceRemarkIndexSql(); + + $row = DB::connection($connection)->table($table) + ->where('report_month', $month) + ->selectRaw('COUNT(*) AS total') + ->selectRaw('SUM(CASE WHEN ' . Materialization::REASON_INDEX_COLUMN + . ' IS NULL OR ' . Materialization::REASON_INDEX_COLUMN . ' <> (' . $indexExpression . ') THEN 1 ELSE 0 END) AS reason_mismatch') + ->selectRaw('SUM(CASE WHEN ' . Materialization::BASE_DIFFERENCE_COLUMN + . ' IS NULL OR ABS(' . Materialization::BASE_DIFFERENCE_COLUMN . ' - (' . $baseExpression . ')) > 0.005 THEN 1 ELSE 0 END) AS base_mismatch') + ->selectRaw('SUM(CASE WHEN ' . Materialization::HAS_DIFFERENCE_COLUMN . ' IS NULL THEN 1 ELSE 0 END) AS unreasoned') + ->selectRaw('SUM(CASE WHEN ' . Materialization::HAS_DIFFERENCE_COLUMN . ' IS NOT NULL AND ' + . Materialization::HAS_DIFFERENCE_COLUMN . ' <> (' . $this->hasDifferenceExpression() . ') THEN 1 ELSE 0 END) AS state_mismatch') + ->first(); + + $this->line(sprintf(' 行数 %s｜备注序号不一致 %s｜差额不一致 %s｜有/无不一致 %s｜未物化 %s', + number_format((int) ($row->total ?? 0)), + number_format((int) ($row->reason_mismatch ?? 0)), + number_format((int) ($row->base_mismatch ?? 0)), + number_format((int) ($row->state_mismatch ?? 0)), + number_format((int) ($row->unreasoned ?? 0)), + )); + } +} diff --git a/app/Console/Commands/SyncStoreReceiptOrderMatches.php b/app/Console/Commands/SyncStoreReceiptOrderMatches.php index be5691f3..abfa3cd1 100644 --- a/app/Console/Commands/SyncStoreReceiptOrderMatches.php +++ b/app/Console/Commands/SyncStoreReceiptOrderMatches.php @@ -40,6 +40,7 @@ class SyncStoreReceiptOrderMatches extends Command {--limit=0 : 仅处理前 N 笔账单（调试用，0 表示全量）} {--amounts-only : 只回填「优惠金额 / 冲减金额 / 种菜金额 / 标签」，不重跑订单侧匹配} {--planting-only : 只回填「种菜金额 / 标签」（不读账单，直接按该月已有明细行的订单号刷，一次覆盖全平台）} + {--skip-materialize : 只同步，不重算差额物化列} {--dry-run : 只计算并打印统计，不写库（用于评估口径调整的影响）}'; protected $description = '按店铺收款账户账单匹配订单侧商品金额，生成「业绩明细」核对镜像'; @@ -215,6 +216,8 @@ class SyncStoreReceiptOrderMatches extends Command trim((string) $this->argument('month')), (string) $this->option('platform'), ); + // 跨月标记更新后再物化差额，保证备注分类与最新标记一致。 + $this->materializeDifferences(trim((string) $this->argument('month'))); } } finally { $lock->release(); @@ -225,6 +228,25 @@ class SyncStoreReceiptOrderMatches extends Command return $result; } + /** + * 同步完成后重算本账期与相邻账期的差额物化列。 + * + * 页面「差额异常备注」「差额（有/无）」「差额合计」改为读物化列，不再在查询时 + * 逐行执行规则链；规则本身仍只有 Pages 里的那一份表达式。 + */ + private function materializeDifferences(string $month): void + { + if ((bool) $this->option('skip-materialize')) { + return; + } + + if (! \App\Business\StoreReceiptDifferenceMaterialization::available(true)) { + return; + } + + $this->call('store-receipt:materialize-differences', ['month' => $month]); + } + /** * 同一平台、店铺、交易单号在相邻月份的收入/优惠/冲减/违约赔付合并后， * 与订单实付（各月重复为同一订单级金额，取最大值）在 2 分钱内对平， diff --git a/app/Http/Controllers/Business/Pages.php b/app/Http/Controllers/Business/Pages.php index 617fd6e6..f6abd339 100644 --- a/app/Http/Controllers/Business/Pages.php +++ b/app/Http/Controllers/Business/Pages.php @@ -14,6 +14,7 @@ use App\Business\TableRegistry; use App\Business\UpstreamBusinessInformation; use App\Business\StoreReceivingAccountTemplate; use App\Business\StoreReceivingAccountWorkbookImporter; +use App\Business\StoreReceiptDifferenceMaterialization as DifferenceMaterialization; use App\Business\StoreViolationWorkbookImporter; use App\Business\StoreViolationAlertCalculator; use App\Business\WithdrawalManagementReport; @@ -159,7 +160,7 @@ class Pages extends Controller ]; /** 差额按实收/应付口径兜底时使用的备注；这部分行才需要判断复杂差额原因。 */ - private const RECEIPT_BASIS_FALLBACK_REMARKS = [ + public const RECEIPT_BASIS_FALLBACK_REMARKS = [ '订单实付按卖家实收核对', '订单实付按应付/已付核对', ]; @@ -543,7 +544,7 @@ class Pages extends Controller * 单次 CASE 命中后即停止，筛选统计与备注筛选共用同一口径。相比把 * 18 条条件逐条带上前序 NOT 再分别 SUM，避免重复计算退款等关联子查询。 */ - private static function receiptDifferenceRemarkIndexSql(): string + public static function receiptDifferenceRemarkIndexSql(): string { $effective = 'COALESCE(difference, ' . self::receiptNetSql() . ')'; $sql = 'CASE WHEN ABS(' . $effective . ') <= 0.005 THEN 0'; @@ -642,7 +643,7 @@ class Pages extends Controller * 页面口径的「有效差额」SQL：口径兜底行（备注为空、但能对上实收/应付）按 0 计。 * 合计、差额筛选、联动计数共用，保证三个地方的数字一致。 */ - private static function receiptEffectiveDifferenceSql(): string + public static function receiptEffectiveDifferenceSql(): string { // 备注序号 > 0 表示命中了明确原因；只有口径兜底且没有明确原因时才归零。 $indexSql = self::receiptDifferenceRemarkIndexSql(); @@ -652,7 +653,7 @@ class Pages extends Controller } /** 未命中复杂原因时，差额的默认口径。 */ - private static function receiptDefaultDifferenceSql(): string + public static function receiptDefaultDifferenceSql(): string { return 'COALESCE(difference, ' . self::receiptNetSql() . ')'; } @@ -1504,7 +1505,7 @@ class Pages extends Controller $differenceState = $this->receiptOrderMatchFilterValues(request('difference_state')); $orderNo = trim((string) request('online_order_no', '')); // 闭包里要用 $this->applyReceiptRemarkFilter()（差异原因备注按公式判定），所以不能声明为 static - $applyFilters = function ($query) use ($platformFilter, $storeFilter, $orderNo, $statusFilter, $remarkFilter, $crossMonthOnly, $unmatchedReasonFilter, $differenceState): void { + $applyFilters = function ($query) use ($platformFilter, $storeFilter, $orderNo, $statusFilter, $remarkFilter, $crossMonthOnly, $unmatchedReasonFilter, $differenceState, $month): void { if ($platformFilter !== []) { $query->whereIn('platform', $platformFilter); } @@ -1519,13 +1520,13 @@ class Pages extends Controller $query->whereIn('match_status', $statusFilter); } if ($remarkFilter !== []) { - $this->applyReceiptRemarkFilter($query, $remarkFilter); + $this->applyReceiptRemarkFilter($query, $remarkFilter, $month); } if ($unmatchedReasonFilter !== []) { $this->applyReceiptUnmatchedReasonFilter($query, $unmatchedReasonFilter); } if ($differenceState !== []) { - $this->applyReceiptDifferenceStateFilter($query, $differenceState, $crossMonthOnly); + $this->applyReceiptDifferenceStateFilter($query, $differenceState, $crossMonthOnly, $month); } }; @@ -1535,40 +1536,68 @@ class Pages extends Controller ->where('report_month', $month) ->whereNotIn('platform', self::RECEIPT_MATCH_EXCLUDED_PLATFORMS); - // 计数与合计合并为一次全表扫描：重筛选项（如按备注筛选）下少一次全表重算。 - $aggQuery = $base(); - $applyFilters($aggQuery); - // 口径兜底行（备注为空、口径对上）的差额按 0 计——与页面行展示、差额筛选同一套判定。 - $differenceBaseExpr = self::receiptDefaultDifferenceSql(); - $fallbackSql = self::receiptBasisFallbackSqlList(); - $agg = $aggQuery->selectRaw(" - COUNT(*) as total, - SUM(payment_count) as payments, - SUM(income_amount) as income_total, - SUM(discount_amount) as discount_total, - SUM(offset_amount) as offset_total, - SUM(default_compensation_amount) as default_compensation_total, - SUM(refund_amount) as refund_total, - SUM(planting_amount) as planting_total, - SUM(" . self::receiptComparablePaidSql() . ") as paid_total, - SUM({$differenceBaseExpr}) as difference_base, - SUM(CASE WHEN remark IN ({$fallbackSql}) THEN 1 ELSE 0 END) as basis_fallback_rows, - SUM(CASE WHEN payment_count > 1 THEN 1 ELSE 0 END) as multi_pay, - SUM(CASE WHEN income_amount = 0 THEN 1 ELSE 0 END) as zero_income, - SUM(CASE WHEN match_status = '已匹配' THEN 1 ELSE 0 END) as matched, - SUM(CASE WHEN match_status = '未匹配' THEN 1 ELSE 0 END) as unmatched - ")->first(); + // 计数与合计合并为一次扫描；已物化的账期直接读差额列，不再逐行执行规则链。 + // 数据按天同步，同一组筛选 5 分钟内复用结果，避免来回切换筛选项重复扫整月。 + $differenceMaterialized = DifferenceMaterialization::materialized($month); + $statsCacheKey = 'performance-details-aggregate-v2:' . md5(serialize([ + $month, + [$platformFilter, $storeFilter, $statusFilter, $remarkFilter, $unmatchedReasonFilter, $differenceState, $orderNo], + $crossMonthOnly, + $differenceMaterialized ? 'materialized' : 'live', + DifferenceMaterialization::dataVersion($month), + ])); + + [$agg, $differenceTotal] = cache()->remember($statsCacheKey, now()->addMinutes(5), function () use ( + $base, + $applyFilters, + $crossMonthOnly, + $differenceMaterialized + ): array { + $aggQuery = $base(); + $applyFilters($aggQuery); + // 口径兜底行（备注为空、口径对上）的差额按 0 计——与页面行展示、差额筛选同一套判定。 + $differenceBaseExpr = $differenceMaterialized + ? 'COALESCE(' . DifferenceMaterialization::BASE_DIFFERENCE_COLUMN . ', 0)' + : self::receiptDefaultDifferenceSql(); + $fallbackSql = self::receiptBasisFallbackSqlList(); + $agg = $aggQuery->selectRaw(" + COUNT(*) as total, + SUM(payment_count) as payments, + SUM(income_amount) as income_total, + SUM(discount_amount) as discount_total, + SUM(offset_amount) as offset_total, + SUM(default_compensation_amount) as default_compensation_total, + SUM(refund_amount) as refund_total, + SUM(planting_amount) as planting_total, + SUM(" . self::receiptComparablePaidSql() . ") as paid_total, + SUM({$differenceBaseExpr}) as difference_base, + SUM(CASE WHEN remark IN ({$fallbackSql}) THEN 1 ELSE 0 END) as basis_fallback_rows, + SUM(CASE WHEN payment_count > 1 THEN 1 ELSE 0 END) as multi_pay, + SUM(CASE WHEN income_amount = 0 THEN 1 ELSE 0 END) as zero_income, + SUM(CASE WHEN match_status = '已匹配' THEN 1 ELSE 0 END) as matched, + SUM(CASE WHEN match_status = '未匹配' THEN 1 ELSE 0 END) as unmatched + ")->first(); + + $differenceTotal = (float) ($agg->difference_base ?? 0); + if (! $crossMonthOnly && (int) ($agg->basis_fallback_rows ?? 0) > 0) { + $adjustmentQuery = $base(); + $applyFilters($adjustmentQuery); + $adjustmentExpression = $differenceMaterialized + ? 'CASE WHEN COALESCE(' . DifferenceMaterialization::REASON_INDEX_COLUMN . ', 0) = 0 THEN -COALESCE(' + . DifferenceMaterialization::BASE_DIFFERENCE_COLUMN . ', 0) ELSE 0 END' + : self::receiptFallbackDifferenceAdjustmentSql(); + $adjustment = $adjustmentQuery + ->whereIn('remark', self::RECEIPT_BASIS_FALLBACK_REMARKS) + ->selectRaw('SUM(' . $adjustmentExpression . ') as adjustment') + ->first(); + $differenceTotal += (float) ($adjustment->adjustment ?? 0); + } + + return [(array) $agg, $differenceTotal]; + }); + + $agg = (object) $agg; $total = (int) ($agg->total ?? 0); - $differenceTotal = (float) ($agg->difference_base ?? 0); - if (! $crossMonthOnly && (int) ($agg->basis_fallback_rows ?? 0) > 0) { - $adjustmentQuery = $base(); - $applyFilters($adjustmentQuery); - $adjustment = $adjustmentQuery - ->whereIn('remark', self::RECEIPT_BASIS_FALLBACK_REMARKS) - ->selectRaw('SUM(' . self::receiptFallbackDifferenceAdjustmentSql() . ') as adjustment') - ->first(); - $differenceTotal += (float) ($adjustment->adjustment ?? 0); - } $stats = [ 'month' => $month, @@ -1888,13 +1917,13 @@ class Pages extends Controller $query->whereIn('match_status', $statusFilter); } if ($remarkFilter !== []) { - $this->applyReceiptRemarkFilter($query, $remarkFilter); + $this->applyReceiptRemarkFilter($query, $remarkFilter, $month); } if ($unmatchedReasonFilter !== []) { $this->applyReceiptUnmatchedReasonFilter($query, $unmatchedReasonFilter); } if ($differenceState !== []) { - $this->applyReceiptDifferenceStateFilter($query, $differenceState, $crossMonthOnly); + $this->applyReceiptDifferenceStateFilter($query, $differenceState, $crossMonthOnly, $month); } return $query; @@ -1937,7 +1966,7 @@ class Pages extends Controller * @param \Illuminate\Database\Query\Builder $query * @param list $remarkFilter */ - private function applyReceiptRemarkFilter($query, array $remarkFilter): void + private function applyReceiptRemarkFilter($query, array $remarkFilter, ?string $month = null): void { if ($remarkFilter === []) { return; @@ -1965,12 +1994,16 @@ class Pages extends Controller return; } - $query->where(function ($group) use ($crossMonthSelected, $indexes): void { + $query->where(function ($group) use ($crossMonthSelected, $indexes, $month): void { if ($crossMonthSelected) { $group->where('cross_month_balanced', 1); } if ($indexes !== []) { - $condition = self::receiptDifferenceRemarkIndexSql() + // 已物化的账期直接读规则序号列，避免每次筛选都重算整串规则链。 + $indexExpression = ($month !== null && DifferenceMaterialization::materialized($month)) + ? 'COALESCE(' . DifferenceMaterialization::REASON_INDEX_COLUMN . ', 0)' + : self::receiptDifferenceRemarkIndexSql(); + $condition = $indexExpression . ' IN (' . implode(', ', array_unique($indexes)) . ')'; $crossMonthSelected ? $group->orWhereRaw($condition) : $group->whereRaw($condition); } @@ -2017,7 +2050,7 @@ class Pages extends Controller * @param \Illuminate\Database\Query\Builder $query * @param string|list $state */ - private function applyReceiptDifferenceStateFilter($query, $state, bool $crossMonthOnly = false): void + private function applyReceiptDifferenceStateFilter($query, $state, bool $crossMonthOnly = false, ?string $month = null): void { $states = is_array($state) ? $state : [$state]; $has = in_array('有差额', $states, true); @@ -2025,6 +2058,12 @@ class Pages extends Controller if ($has === $none) { return; } + // 已物化的账期直接命中 has_difference 索引，不再对整月逐行算规则链。 + if ($month !== null && DifferenceMaterialization::materialized($month)) { + $query->where(DifferenceMaterialization::HAS_DIFFERENCE_COLUMN, $has ? 1 : 0); + + return; + } // 已限定跨月第一优先级时，跨月行不会再落入口径兜底归零， // “有/无差额”可直接使用默认差额，避免整串原因 CASE 重复执行。 $effective = $crossMonthOnly @@ -2055,7 +2094,7 @@ class Pages extends Controller continue; } if ($key === 'remark') { - $this->applyReceiptRemarkFilter($query, (array) $values); + $this->applyReceiptRemarkFilter($query, (array) $values, $month); continue; } if ($key === 'unmatched_reason') { @@ -2068,11 +2107,17 @@ class Pages extends Controller $query->where('online_order_no', 'like', '%' . addcslashes($orderNo, '%_\\') . '%'); } - $effective = $crossMonthOnly - ? self::receiptDefaultDifferenceSql() - : self::receiptEffectiveDifferenceSql(); - $row = $query->selectRaw('SUM(CASE WHEN ABS(' . $effective . ') > 0.005 THEN 1 ELSE 0 END) as has_diff, ' - . 'SUM(CASE WHEN ABS(' . $effective . ') <= 0.005 THEN 1 ELSE 0 END) as none_diff')->first(); + if (DifferenceMaterialization::materialized($month)) { + $column = DifferenceMaterialization::HAS_DIFFERENCE_COLUMN; + $row = $query->selectRaw("SUM({$column}) as has_diff, " + . "SUM(1 - {$column}) as none_diff")->first(); + } else { + $effective = $crossMonthOnly + ? self::receiptDefaultDifferenceSql() + : self::receiptEffectiveDifferenceSql(); + $row = $query->selectRaw('SUM(CASE WHEN ABS(' . $effective . ') > 0.005 THEN 1 ELSE 0 END) as has_diff, ' + . 'SUM(CASE WHEN ABS(' . $effective . ') <= 0.005 THEN 1 ELSE 0 END) as none_diff')->first(); + } $counts = []; if ((int) ($row->has_diff ?? 0) > 0) { @@ -2115,14 +2160,22 @@ class Pages extends Controller private function receiptOrderMatchMonths(): array { $connection = (string) config('upstream_mirrors.connection', 'business_mirror'); - $months = DB::connection($connection)->table('store_receipt_order_match_rows') - ->whereNotIn('platform', self::RECEIPT_MATCH_EXCLUDED_PLATFORMS) - ->select('report_month') - ->distinct() - ->orderByDesc('report_month') - ->pluck('report_month') - ->map(static fn ($month): string => substr((string) $month, 0, 10)) - ->all(); + // 月份去重要扫整月索引（近百万行），每次开页都会跑一遍；账期按月增长且数据按天同步， + // 缓存 6 小时即可，同步新月份后最迟 6 小时可见（也可手动清缓存）。 + $months = cache()->remember( + 'performance-details-months-v1:' . md5(implode(',', self::RECEIPT_MATCH_EXCLUDED_PLATFORMS)), + now()->addHours(6), + static function () use ($connection): array { + return DB::connection($connection)->table('store_receipt_order_match_rows') + ->whereNotIn('platform', self::RECEIPT_MATCH_EXCLUDED_PLATFORMS) + ->select('report_month') + ->distinct() + ->orderByDesc('report_month') + ->pluck('report_month') + ->map(static fn ($month): string => substr((string) $month, 0, 10)) + ->all(); + }, + ); $requested = substr(trim((string) request('month', '')), 0, 10); if (preg_match('/^\d{4}-\d{2}$/', $requested) === 1) { @@ -2178,7 +2231,15 @@ class Pages extends Controller $orderNo = trim((string) request('online_order_no', '')); // 联动统计要跑几次全表聚合，短时间内重复请求（页面加载 + 下拉刷新）直接吃缓存 - $cacheKey = 'performance-details-facets:' . md5(serialize([$month, $filters, $orderNo])); + // 缓存键带上规则指纹、物化状态与数据版本：规则调整或重新物化后不会复用旧口径的结果。 + $cacheKey = 'performance-details-facets-v2:' . md5(serialize([ + $month, + $filters, + $orderNo, + DifferenceMaterialization::ruleVersion(), + DifferenceMaterialization::materialized($month), + DifferenceMaterialization::dataVersion($month), + ])); // 数据按天同步；联动统计会扫全月，缓存 1 小时，避免相同筛选反复等待。 return response()->json(cache()->remember($cacheKey, now()->addHour(), function () use ($month, $filters, $orderNo) { @@ -2279,12 +2340,21 @@ class Pages extends Controller $wantNoDifference = in_array('无差额', $differenceValues, true); $connection = (string) config('upstream_mirrors.connection', 'business_mirror'); + // 已物化的账期直接读规则序号列与默认差额列；未物化时回退实时规则表达式。 + $materialized = DifferenceMaterialization::materialized($month); + $baseDifferenceExpression = $materialized + ? 'COALESCE(' . DifferenceMaterialization::BASE_DIFFERENCE_COLUMN . ', 0)' + : self::receiptDefaultDifferenceSql(); + $reasonIndexExpression = $materialized + ? 'COALESCE(' . DifferenceMaterialization::REASON_INDEX_COLUMN . ', 0)' + : self::receiptDifferenceRemarkIndexSql(); + $inner = DB::connection($connection) ->table('store_receipt_order_match_rows') ->where('report_month', $month) ->select(['platform', 'store_name', 'match_status', 'remark']) - ->selectRaw(self::receiptDefaultDifferenceSql() . ' AS base_difference') - ->selectRaw(self::receiptDifferenceRemarkIndexSql() . ' AS reason_index'); + ->selectRaw($baseDifferenceExpression . ' AS base_difference') + ->selectRaw($reasonIndexExpression . ' AS reason_index'); foreach ($filters as $key => $values) { if ($key === 'remark' || $key === 'difference_state' || $values === []) { @@ -2392,7 +2462,7 @@ class Pages extends Controller } // 差额异常备注按公式条件判定 if ($key === 'remark') { - $this->applyReceiptRemarkFilter($query, (array) $values); + $this->applyReceiptRemarkFilter($query, (array) $values, $month); continue; } // 未匹配原因：只看未匹配行、按库里的原因过滤 @@ -2402,7 +2472,7 @@ class Pages extends Controller } // 差额（有/无）：按页面差额口径过滤 if ($key === 'difference_state') { - $this->applyReceiptDifferenceStateFilter($query, (array) $values, $crossMonthOnly); + $this->applyReceiptDifferenceStateFilter($query, (array) $values, $crossMonthOnly, $month); continue; } $query->whereIn($key, $values); @@ -2414,7 +2484,10 @@ class Pages extends Controller $counts = []; if ($column === 'remark') { // 一行只按优先级命中一次，再按序号分组，避免 18 条规则各跑一遍全表。 - $indexSql = self::receiptDifferenceRemarkIndexSql(); + // 已物化的账期直接读规则序号列。 + $indexSql = DifferenceMaterialization::materialized($month) + ? 'COALESCE(' . DifferenceMaterialization::REASON_INDEX_COLUMN . ', 0)' + : self::receiptDifferenceRemarkIndexSql(); $rows = (clone $query) ->selectRaw($indexSql . ' AS reason_index') ->selectRaw('COUNT(*) AS total') @@ -2571,7 +2644,7 @@ class Pages extends Controller $query->whereIn('match_status', $statusValues); } if ($remarkValues !== []) { - $this->applyReceiptRemarkFilter($query, $remarkValues); + $this->applyReceiptRemarkFilter($query, $remarkValues, $month); } $unmatchedReasonValues = $this->receiptOrderMatchFilterValues($request->query('unmatched_reason')); if ($unmatchedReasonValues !== []) { @@ -2579,7 +2652,7 @@ class Pages extends Controller } $differenceState = $this->receiptOrderMatchFilterValues($request->query('difference_state')); if ($differenceState !== []) { - $this->applyReceiptDifferenceStateFilter($query, $differenceState, $crossMonthOnly); + $this->applyReceiptDifferenceStateFilter($query, $differenceState, $crossMonthOnly, $month); } // 「订单实付含已退款部分」「订单实付未减联合营销费」判定所需证据列（与页面同一口径；仅对应平台负差额行会真正计算子查询）。 diff --git a/database/migrations/2026_10_26_160000_materialize_store_receipt_difference_columns.php b/database/migrations/2026_10_26_160000_materialize_store_receipt_difference_columns.php new file mode 100644 index 00000000..38639a6d --- /dev/null +++ b/database/migrations/2026_10_26_160000_materialize_store_receipt_difference_columns.php @@ -0,0 +1,132 @@ +environment('testing') + || (string) config('system_database.mode', 'bootstrap') === 'bootstrap') { + return; + } + + $schema = Schema::connection($this->mirrorConnection()); + + if (! $schema->hasTable(self::TARGET)) { + return; + } + + // 页面上的「差额异常备注」「差额（有/无）」「差额合计」原先都在查询时逐行现算 + // 一套含关联子查询的规则链，整月 30 多万行要几十秒。这里把规则结果落库， + // 查询退化成读列 + 索引聚合。规则表达式仍以 Pages 控制器里的 + // receiptDifferenceRemarkIndexSql()/receiptDefaultDifferenceSql() 为唯一口径， + // 由 store-receipt:materialize-differences 写入。 + if (! $schema->hasColumn(self::TARGET, 'difference_reason_index')) { + $schema->table(self::TARGET, function (Blueprint $table): void { + $table->unsignedSmallInteger('difference_reason_index')->nullable()->after('remark') + ->comment('差额异常备注规则序号；0＝无明确原因，NULL＝尚未物化'); + }); + } + + if (! $schema->hasColumn(self::TARGET, 'base_difference')) { + $schema->table(self::TARGET, function (Blueprint $table): void { + $table->decimal('base_difference', 18, 2)->nullable()->after('difference_reason_index') + ->comment('默认口径差额（COALESCE(difference, 净额)），物化结果'); + }); + } + + if (! $schema->hasColumn(self::TARGET, 'has_difference')) { + $schema->table(self::TARGET, function (Blueprint $table): void { + $table->boolean('has_difference')->nullable()->after('base_difference') + ->comment('有效差额是否非零；用于「差额（有/无）」筛选与计数'); + }); + } + + $this->ensureIndex($schema, self::TARGET, 'store_receipt_difference_reason_index', + ['report_month', 'difference_reason_index']); + $this->ensureIndex($schema, self::TARGET, 'store_receipt_difference_state_index', + ['report_month', 'has_difference']); + // 店铺下拉去重按 (report_month, platform) 过滤后再按 store_name 排序， + // 现有 (report_month, store_name) 不含 platform，必须回表逐行判断。 + $this->ensureIndex($schema, self::TARGET, 'store_receipt_store_platform_index', + ['report_month', 'store_name', 'platform']); + + if (! $schema->hasTable(self::STATE_TABLE)) { + $schema->create(self::STATE_TABLE, function (Blueprint $table): void { + $table->string('report_month', 10)->primary()->comment('账期 Y-m-d'); + $table->string('rule_version', 32)->comment('物化时使用的规则表达式指纹'); + $table->unsignedBigInteger('row_count')->default(0); + $table->dateTime('materialized_at')->nullable(); + }); + } + } + + public function down(): void + { + if (app()->environment('testing') + || (string) config('system_database.mode', 'bootstrap') === 'bootstrap') { + return; + } + + $schema = Schema::connection($this->mirrorConnection()); + + if (! $schema->hasTable(self::TARGET)) { + return; + } + + foreach ([ + 'store_receipt_difference_reason_index', + 'store_receipt_difference_state_index', + 'store_receipt_store_platform_index', + ] as $indexName) { + if ($this->hasIndex($schema, self::TARGET, $indexName)) { + $schema->table(self::TARGET, function (Blueprint $table) use ($indexName): void { + $table->dropIndex($indexName); + }); + } + } + + foreach (['has_difference', 'base_difference', 'difference_reason_index'] as $column) { + if ($schema->hasColumn(self::TARGET, $column)) { + $schema->table(self::TARGET, function (Blueprint $table) use ($column): void { + $table->dropColumn($column); + }); + } + } + + $schema->dropIfExists(self::STATE_TABLE); + } + + private function mirrorConnection(): string + { + return (string) config('upstream_mirrors.connection', 'business_mirror'); + } + + private function hasIndex(\Illuminate\Database\Schema\Builder $schema, string $table, string $indexName): bool + { + return collect($schema->getIndexes($table)) + ->contains(static fn (array $index): bool => ($index['name'] ?? null) === $indexName); + } + + private function ensureIndex( + \Illuminate\Database\Schema\Builder $schema, + string $table, + string $indexName, + array $columns + ): void { + if ($this->hasIndex($schema, $table, $indexName)) { + return; + } + + $schema->table($table, function (Blueprint $blueprint) use ($columns, $indexName): void { + $blueprint->index($columns, $indexName); + }); + } +}; diff --git a/database/migrations/2026_10_26_161000_add_store_receipt_order_sort_index.php b/database/migrations/2026_10_26_161000_add_store_receipt_order_sort_index.php new file mode 100644 index 00000000..69de7b58 --- /dev/null +++ b/database/migrations/2026_10_26_161000_add_store_receipt_order_sort_index.php @@ -0,0 +1,62 @@ +environment('testing') + || (string) config('system_database.mode', 'bootstrap') === 'bootstrap') { + return; + } + + $schema = Schema::connection($this->mirrorConnection()); + if (! $schema->hasTable(self::TABLE) || $this->hasIndex($schema)) { + return; + } + + // 业绩明细默认按「发生时间 → 店铺 → 线上单号」排序并只取当前页， + // 没有这条索引时整月 30 多万行都要走 filesort 才能取前 25 行。 + $schema->table(self::TABLE, function (Blueprint $table): void { + $table->index( + ['report_month', 'occurred_date', 'store_name', 'online_order_no'], + self::INDEX, + ); + }); + } + + public function down(): void + { + if (app()->environment('testing') + || (string) config('system_database.mode', 'bootstrap') === 'bootstrap') { + return; + } + + $schema = Schema::connection($this->mirrorConnection()); + if (! $schema->hasTable(self::TABLE) || ! $this->hasIndex($schema)) { + return; + } + + $schema->table(self::TABLE, function (Blueprint $table): void { + $table->dropIndex(self::INDEX); + }); + } + + private function mirrorConnection(): string + { + return (string) config('upstream_mirrors.connection', 'business_mirror'); + } + + private function hasIndex(\Illuminate\Database\Schema\Builder $schema): bool + { + return collect($schema->getIndexes(self::TABLE)) + ->contains(static fn (array $index): bool => ($index['name'] ?? null) === self::INDEX); + } +}; diff --git a/docs/README.md b/docs/README.md index ca383b45..2f533ed3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -32,6 +32,7 @@ | [电商财务扩展开发说明](ecommerce-finance-development.md) | 订单、结算、成本、资金和利润分析的业务模型与扩展边界 | | [商品成本占位行规则清单](商品成本占位行规则清单.md) | 商品成本同步的取数顺序、占位行去重、价格兜底和月度验收口径 | | [主体维度经营报表需求说明](主体维度经营报表需求说明.md) | 主体维度经营报表的字段、口径、来源和验收契约 | +| [业绩明细差额物化](业绩明细差额物化.md) | 业绩明细差额规则落库、物化命令、发布与回退步骤 | 上游数据库按只读黑盒处理。正式页面优先读取共享业务镜像；尚未迁移的旧页面才使用兼容快照。不要在网页中接受任意 SQL，也不要把真实业务数据或连接凭据写进文档、测试和截图。 diff --git a/docs/业绩明细差额物化.md b/docs/业绩明细差额物化.md new file mode 100644 index 00000000..40215e56 --- /dev/null +++ b/docs/业绩明细差额物化.md @@ -0,0 +1,77 @@ +# 业绩明细差额物化 + +财务经营报表 › 业绩明细（`financial-operations/performance-details`）的「差额合计」「差额异常备注」「差额（有/无）」原先都在查询时按同一套规则链逐行现算。规则链包含 40 多处 `SELECT`、其中 36 处是对 `ops_jst_orders`（140 万行）的关联子查询，生成 SQL 有 24 KB 和 50 KB 两条。 + +实测（2026-10，共享镜像库，91.7 万行；2026-07 全平台 32.4 万行）： + +| 场景 | 优化前 | 优化后（冷缓存） | 优化后（缓存内） | +| --- | --- | --- | --- | +| 切月份后的联动统计（下拉「统计中…」） | 14.5～84.9 秒 | 约 1.7 秒 | 约 2 毫秒 | +| 页面主体（8 个指标卡 + 表格） | 2.7～7.7 秒 | 约 1.7 秒 | 约 24 毫秒 | +| 带平台+备注筛选的联动统计 | 数十秒 | 约 0.8 秒 | 约 2 毫秒 | + +冷缓存下的剩余成本主要是远端镜像库的全月扫描（约 300 毫秒/次 × 5～6 次）， +规则链已不再是瓶颈；进一步优化需要按月预聚合汇总表（未实现）。 + +## 做法 + +把规则结果落成三个列，查询退化为读列 + 索引聚合： + +| 列 | 含义 | +| --- | --- | +| `difference_reason_index` | 差额异常备注规则序号（1 开始，0＝无明确原因，NULL＝尚未物化） | +| `base_difference` | 默认口径差额 `COALESCE(difference, 净额)` | +| `has_difference` | 有效差额是否非零（口径兜底行按 0 计），用于「有/无差额」筛选与计数 | + +迁移同时补了四个索引：`(report_month, difference_reason_index)`、`(report_month, has_difference)`、 +`(report_month, store_name, platform)`（店铺下拉去重）、`(report_month, occurred_date, store_name, online_order_no)` +（明细页默认排序，避免整月 filesort）。 + +规则的唯一口径仍然是 `app/Http/Controllers/Business/Pages.php` 里的 +`receiptDefaultDifferenceSql()` / `receiptDifferenceRemarkIndexSql()` / +`receiptEffectiveDifferenceSql()`；物化命令直接复用这三份表达式，不在 PHP 里重写规则。 + +## 物化命令 + +```bash +# 物化指定账期（会连带重算前一个月和后一个月，跨月备注依赖相邻月份） +php artisan store-receipt:materialize-differences 2026-07 + +# 物化镜像里所有已有数据的月份 +php artisan store-receipt:materialize-differences + +# 只比对不写库：检查已落库的值与按当前规则重算的结果是否一致 +php artisan store-receipt:materialize-differences 2026-07 --dry-run +``` + +分批写入（`--chunk`，默认 5000 行），先写 `base_difference` / `difference_reason_index`，再用这两列推导 `has_difference`，避免重复执行规则链。 + +`store-receipt:order-match:sync` 成功后会自动对同一账期执行物化（可用 `--skip-materialize` 关闭）。 + +## 发布步骤 + +物化列位于 `caiwu_business_mirror`（共享业务镜像库），加列属于数据库迁移： + +1. 按 SOP 取得负责人授权并完成备份后，随正式发布执行迁移； +2. 迁移完成后先跑 `--dry-run` 校验（应显示三项不一致均为 0，`未物化` 行数为总行数）； +3. 再执行物化命令回填所有已有账期； +4. 页面无需额外操作，读到状态表即自动切换到物化列。 + +每台开发机、每个实例共用同一份镜像库，因此物化一次即可全团队生效。 + +## 回退 + +- 页面层有回退：状态表里没有该账期、或规则指纹不一致时，自动回到实时规则表达式，功能不受影响（只是慢）。 +- 需要整体回退时，执行迁移的 `down()` 删除列与状态表即可。 +- 规则调整后，`ruleVersion()` 指纹会变化，旧物化结果自动视为过期；重新执行物化命令即可。 + +## 缓存 + +- 月份下拉：6 小时； +- 指标卡聚合：5 分钟； +- 联动统计：1 小时； +- 三者的缓存键都带规则指纹、物化状态和数据版本；物化完成后数据版本自增，旧缓存自动失效。 + +## 前端 + +联动统计改为按需触发：页面加载不再自动请求，只有展开下拉或改变勾选时才请求（250 ms 防抖）。因此切月份时不再出现空白表格卡在「统计中…」的情况。 diff --git a/resources/views/business/pages/partials/directory-table.blade.php b/resources/views/business/pages/partials/directory-table.blade.php index d1780ee3..bff445b3 100644 --- a/resources/views/business/pages/partials/directory-table.blade.php +++ b/resources/views/business/pages/partials/directory-table.blade.php @@ -2042,7 +2042,7 @@ @endif @endif @if (($page['slug'] ?? '') === 'store-report' && ! $directoryUsesFieldQuery) -
+ @php $storeFilterTextFields = [ ['name' => 'store_name', 'label' => '店铺名称：'], @@ -2151,7 +2151,7 @@ .receipt-remark-tag--warn { background: #fdf3e3; color: #96690a; } .receipt-remark-tag--info { background: #eaf1fb; color: #1f5fa8; } - + {{-- 查询时保留当前每页条数 --}}
@elseif (in_array(($page['slug'] ?? ''), ['inventory-report', 'monthly-weighted-average-price'], true)) - +
- +