修复:隔离单个上游快照刷新失败 #3

Merged
WuYongJie merged 2 commits from fix/WuYongJie-upstream-refresh-failure-isolation into main 2026-08-03 19:15:20 +08:00
2 changed files with 55 additions and 7 deletions

View File

@ -63,6 +63,7 @@ class UpstreamRefreshActiveSnapshots extends Command
$fresh = 0;
$stale = 0;
$failed = 0;
foreach ($entries as $entry) {
if ((microtime(true) - $startedAt) >= $timeBudget) {
@ -74,17 +75,17 @@ class UpstreamRefreshActiveSnapshots extends Command
$status = $refresher->refresh($entry);
$status === 'fresh' ? $fresh++ : $stale++;
} catch (Throwable $exception) {
// 每个来源独立停止和重置,不影响其他来源的连接与刷新队列。
// 单个查询失败只重置当前来源连接并跳过该查询,不能让它
// 每分钟阻断同一来源的其他页面。
UpstreamOdbcConnection::reset($source);
Log::error('上游活跃快照刷新失败,本来源本轮已停止。', [
$failed++;
Log::error('上游活跃快照刷新失败,本轮已跳过该查询。', [
'source' => $definition['key'],
'source_label' => $definition['label'],
'scope' => $entry['scope'] ?? 'unknown',
'exception' => $exception::class,
]);
$this->components->error('上游刷新失败,本轮已停止;错误类型:' . $exception::class);
return self::FAILURE;
$this->components->warn('一个上游查询刷新失败,已跳过并继续;错误类型:' . $exception::class);
}
$pauseMs = max((int) $cache->setting('snapshots.refresh_pause_ms', 300), 0);
@ -94,8 +95,8 @@ class UpstreamRefreshActiveSnapshots extends Command
}
}
$this->components->info("快照刷新完成:新鲜 {$fresh},沿用旧快照 {$stale}。");
$this->components->info("快照刷新完成:新鲜 {$fresh},沿用旧快照 {$stale},失败 {$failed}。");
return self::SUCCESS;
return $failed > 0 ? self::FAILURE : self::SUCCESS;
}
}

View File

@ -6,8 +6,13 @@ use App\Models\Auth\Permission;
use Illuminate\Cache\CacheManager;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
use Mockery;
use Modules\EcommerceFinance\Contracts\UpstreamSnapshotRefresher;
use Modules\EcommerceFinance\Services\UpstreamQueryCache;
use Modules\EcommerceFinance\Services\UpstreamQueryCacheFactory;
use Modules\EcommerceFinance\Services\UpstreamSnapshotRefresherRegistry;
use Modules\EcommerceFinance\Services\UpstreamSourceRegistry;
use RuntimeException;
use Tests\TestCase;
class UpstreamSharedDatabaseCacheTest extends TestCase
@ -93,6 +98,48 @@ class UpstreamSharedDatabaseCacheTest extends TestCase
->assertExitCode(0);
}
public function test_one_failed_snapshot_does_not_block_the_next_query_for_the_same_source(): void
{
$entries = [
['scope' => 'orders.failed', 'parameters' => []],
['scope' => 'orders.healthy', 'parameters' => []],
];
$sources = Mockery::mock(UpstreamSourceRegistry::class);
$sources->shouldReceive('definition')
->twice()
->with('shared_source')
->andReturn(['key' => 'shared_source', 'label' => '共享来源']);
$sources->shouldReceive('enabled')->once()->with('shared_source')->andReturnTrue();
$cache = Mockery::mock(UpstreamQueryCache::class);
$cache->shouldReceive('setting')->andReturnUsing(static fn (string $path, mixed $default = null): mixed => match ($path) {
'snapshots.refresh_limit' => 2,
'snapshots.refresh_time_budget_seconds' => 10,
'snapshots.refresh_pause_ms' => 0,
default => $default,
});
$cache->shouldReceive('activeEntries')->once()->with(2)->andReturn($entries);
$caches = Mockery::mock(UpstreamQueryCacheFactory::class);
$caches->shouldReceive('forSource')->once()->with('shared_source')->andReturn($cache);
$refresher = Mockery::mock(UpstreamSnapshotRefresher::class);
$refresher->shouldReceive('refresh')->once()->with($entries[0])->ordered()
->andThrow(new RuntimeException('first query failed'));
$refresher->shouldReceive('refresh')->once()->with($entries[1])->ordered()->andReturn('fresh');
$refreshers = Mockery::mock(UpstreamSnapshotRefresherRegistry::class);
$refreshers->shouldReceive('forSource')->once()->with('shared_source')->andReturn($refresher);
$this->app->instance(UpstreamSourceRegistry::class, $sources);
$this->app->instance(UpstreamQueryCacheFactory::class, $caches);
$this->app->instance(UpstreamSnapshotRefresherRegistry::class, $refreshers);
$this->artisan('upstream:snapshots:refresh', ['source' => 'shared_source'])
->expectsOutputToContain('失败 1')
->assertExitCode(1);
}
public function test_permission_change_creates_a_separate_snapshot_scope(): void
{
$user = user_model_class()::firstOrFail();