diff --git a/app/Console/Commands/SyncBusinessFilesToQiniu.php b/app/Console/Commands/SyncBusinessFilesToQiniu.php index a1314d4b..f313b19f 100644 --- a/app/Console/Commands/SyncBusinessFilesToQiniu.php +++ b/app/Console/Commands/SyncBusinessFilesToQiniu.php @@ -4,6 +4,7 @@ namespace App\Console\Commands; use App\Utilities\QiniuPrivateUrlSigner; use Illuminate\Console\Command; +use Illuminate\Contracts\Filesystem\Filesystem; use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Storage; @@ -12,6 +13,10 @@ use Throwable; class SyncBusinessFilesToQiniu extends Command { + private const PROBE_CLEANUP_MAX_ATTEMPTS = 10; + + private const PROBE_CLEANUP_RETRY_DELAY_MICROSECONDS = 500_000; + protected $signature = 'storage:business-files:sync {--require-qiniu : Fail unless the private Qiniu backend is enabled} {--probe-only : Verify private read/write/delete without copying retained files} @@ -124,9 +129,26 @@ class SyncBusinessFilesToQiniu extends Command $disk->delete($path); } - if ($disk->exists($path)) { - throw new RuntimeException('Business storage probe cleanup failed.'); + $this->confirmProbeCleanup($disk, $path); + } + + private function confirmProbeCleanup(Filesystem $disk, string $path, ?callable $sleep = null): void + { + $sleep ??= static function (int $microseconds): void { + usleep($microseconds); + }; + + for ($attempt = 0; $attempt < self::PROBE_CLEANUP_MAX_ATTEMPTS; $attempt++) { + if (! $disk->exists($path)) { + return; + } + + if ($attempt < self::PROBE_CLEANUP_MAX_ATTEMPTS - 1) { + $sleep(self::PROBE_CLEANUP_RETRY_DELAY_MICROSECONDS); + } } + + throw new RuntimeException('Business storage probe cleanup failed.'); } private function syncRetainedLocalFiles(): array diff --git a/tests/Feature/Console/SyncBusinessFilesToQiniuTest.php b/tests/Feature/Console/SyncBusinessFilesToQiniuTest.php index c118f9d5..03e6c304 100644 --- a/tests/Feature/Console/SyncBusinessFilesToQiniuTest.php +++ b/tests/Feature/Console/SyncBusinessFilesToQiniuTest.php @@ -2,9 +2,12 @@ namespace Tests\Feature\Console; +use App\Console\Commands\SyncBusinessFilesToQiniu; +use Illuminate\Contracts\Filesystem\Filesystem; use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Storage; +use Mockery; use Tests\TestCase; class SyncBusinessFilesToQiniuTest extends TestCase @@ -61,4 +64,42 @@ class SyncBusinessFilesToQiniuTest extends TestCase $this->artisan('storage:business-files:sync', ['--require-qiniu' => true]) ->assertFailed(); } + + public function test_it_retries_probe_cleanup_until_qiniu_reports_the_object_missing(): void + { + $this->assertSame( + [500_000, 500_000], + $this->invokeProbeCleanup([true, true, false]) + ); + } + + public function test_it_fails_after_bounded_probe_cleanup_retries_when_qiniu_keeps_reporting_the_object(): void + { + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('Business storage probe cleanup failed.'); + + $this->invokeProbeCleanup(array_fill(0, 10, true)); + } + + private function invokeProbeCleanup(array $existsResults): array + { + $waits = []; + $disk = Mockery::mock(Filesystem::class); + $disk->shouldReceive('exists') + ->times(count($existsResults)) + ->andReturn(...$existsResults); + + $method = new \ReflectionMethod(SyncBusinessFilesToQiniu::class, 'confirmProbeCleanup'); + $method->setAccessible(true); + $method->invoke( + app(SyncBusinessFilesToQiniu::class), + $disk, + 'probe-path', + static function (int $microseconds) use (&$waits): void { + $waits[] = $microseconds; + } + ); + + return $waits; + } }