diff --git a/CHANGELOG.md b/CHANGELOG.md index e2f7a3a52..2fd073ac1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # api +## 8x.12.0 - TBD +- Poll wikis for pending MediaWiki jobs and create Kubernetes jobs to process them if needed + ## 8x.11.1 - 18 April 2023 - Do not disable elastic search on wikis after a failure diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 803ca767a..ee5325a7c 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -8,6 +8,7 @@ use App\Jobs\PruneEventPageUpdatesTable; use App\Jobs\PruneQueryserviceBatchesTable; use App\Jobs\SandboxCleanupJob; +use App\Jobs\PollForMediaWikiJobsJob; use Illuminate\Console\Scheduling\Schedule; use Illuminate\Foundation\Console\Kernel as ConsoleKernel; use App\Jobs\PlatformStatsSummaryJob; @@ -43,6 +44,8 @@ protected function schedule(Schedule $schedule) // Schedule site stat updates for each wiki and platform-summary $schedule->command('schedule:stats')->daily(); + + $schedule->job(new PollForMediaWikiJobsJob)->everyMinute(); } /** diff --git a/app/Jobs/PollForMediaWikiJobsJob.php b/app/Jobs/PollForMediaWikiJobsJob.php new file mode 100644 index 000000000..9ea2be226 --- /dev/null +++ b/app/Jobs/PollForMediaWikiJobsJob.php @@ -0,0 +1,45 @@ +pluck('domain'); + foreach ($allWikiDomains as $wikiDomain) { + if ($this->hasPendingJobs($wikiDomain)) { + $this->enqueueWiki($wikiDomain); + } + } + } + + private function hasPendingJobs (string $wikiDomain): bool + { + $response = Http::withHeaders([ + 'host' => $wikiDomain + ])->get( + getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&meta=siteinfo&siprop=statistics&format=json' + ); + + if ($response->failed()) { + $this->job->markAsFailed(); + Log::error( + 'Failure polling wiki '.$wikiDomain.' for pending MediaWiki jobs: '.$response->clientError() + ); + return false; + } + + $pendingJobsCount = data_get($response->json(), 'query.statistics.jobs', 0); + return $pendingJobsCount > 0; + } + + private function enqueueWiki (string $wikiDomain): void + { + dispatch(new ProcessMediaWikiJobsJob($wikiDomain)); + } +} diff --git a/app/Jobs/ProcessMediaWikiJobsJob.php b/app/Jobs/ProcessMediaWikiJobsJob.php new file mode 100644 index 000000000..fe9465472 --- /dev/null +++ b/app/Jobs/ProcessMediaWikiJobsJob.php @@ -0,0 +1,118 @@ +wikiDomain = $wikiDomain; + $this->jobsKubernetesNamespace = env('API_JOB_NAMESPACE', 'api-jobs'); + } + + public function uniqueId(): string + { + return $this->wikiDomain; + } + + public function handle (Client $kubernetesClient): void + { + $kubernetesClient->setNamespace('default'); + $mediawikiPod = $kubernetesClient->pods()->setFieldSelector([ + 'status.phase' => 'Running' + ])->setLabelSelector([ + 'app.kubernetes.io/name' => 'mediawiki', + 'app.kubernetes.io/component' => 'app-backend' + ])->first(); + + if ($mediawikiPod === null) { + $this->fail( + new \RuntimeException( + 'Unable to find a running MediaWiki pod in the cluster, '. + 'cannot continue.' + ) + ); + return; + } + $mediawikiPod = $mediawikiPod->toArray(); + + $kubernetesClient->setNamespace($this->jobsKubernetesNamespace); + $jobSpec = new KubernetesJob([ + 'metadata' => [ + 'name' => 'run-all-mw-jobs-'.hash('sha1', $this->wikiDomain), + 'namespace' => $this->jobsKubernetesNamespace, + 'labels' => [ + 'app.kubernetes.io/instance' => $this->wikiDomain, + 'app.kubernetes.io/name' => 'run-all-mw-jobs' + ] + ], + 'spec' => [ + 'ttlSecondsAfterFinished' => 0, + 'template' => [ + 'metadata' => [ + 'name' => 'run-all-mw-jobs' + ], + 'spec' => [ + 'containers' => [ + 0 => [ + 'name' => 'run-all-mw-jobs', + 'image' => $mediawikiPod['spec']['containers'][0]['image'], + 'env' => array_merge( + $mediawikiPod['spec']['containers'][0]['env'], + [['name' => 'WBS_DOMAIN', 'value' => $this->wikiDomain]] + ), + 'command' => [ + 0 => 'bash', + 1 => '-c', + 2 => <<<'CMD' + JOBS_TO_GO=1 + while [ "$JOBS_TO_GO" != "0" ] + do + echo "Running 1000 jobs" + php w/maintenance/runJobs.php --maxjobs 1000 + echo Waiting for 1 seconds... + sleep 1 + JOBS_TO_GO=$(php w/maintenance/showJobs.php | tr -d '[:space:]') + echo $JOBS_TO_GO jobs to go + done + CMD + ], + ] + ], + 'restartPolicy' => 'Never' + ] + ] + ] + ]); + + $job = $kubernetesClient->jobs()->apply($jobSpec); + $jobName = data_get($job, 'metadata.name'); + if (!$jobName) { + // The k8s client does not fail reliably on 4xx responses, so checking the name + // currently serves as poor man's error handling. + $this->fail( + new \RuntimeException('Job creation for wiki "'.$this->wikiDomain.'" failed.') + ); + return; + } + Log::info( + 'MediaWiki Job for wiki "'.$this->wikiDomain.'" exists or was created with name "'.$jobName.'".' + ); + + return; + } + +} diff --git a/config/queue.php b/config/queue.php index 18fee8ca8..3d008954b 100644 --- a/config/queue.php +++ b/config/queue.php @@ -63,10 +63,9 @@ 'driver' => 'redis', 'connection' => 'default', 'queue' => env('REDIS_QUEUE', 'default'), - 'retry_after' => 90, + 'retry_after' => 100, 'block_for' => null, ], - ], /* diff --git a/phpunit.xml b/phpunit.xml index 8b753d2e6..78e5f5df6 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -14,5 +14,6 @@ + diff --git a/tests/Jobs/PollForMediaWikiJobsJobTest.php b/tests/Jobs/PollForMediaWikiJobsJobTest.php new file mode 100644 index 000000000..dfbf6657a --- /dev/null +++ b/tests/Jobs/PollForMediaWikiJobsJobTest.php @@ -0,0 +1,94 @@ +wiki = Wiki::factory()->create(); + } + + public function testNoJobs() + { + Http::fake([ + getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&meta=siteinfo&siprop=statistics&format=json' => Http::response([ + 'query' => [ + 'statistics' => [ + 'jobs' => 0 + ] + ] + ], 200) + ]); + + Bus::fake(); + $mockJob = $this->createMock(Job::class); + $job = new PollForMediaWikiJobsJob(); + $job->setJob($mockJob); + + $mockJob->expects($this->never())->method('fail'); + $mockJob->expects($this->never())->method('markAsFailed'); + $job->handle(); + Bus::assertNothingDispatched(); + } + + public function testWithJobs() + { + Http::fake([ + getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&meta=siteinfo&siprop=statistics&format=json' => Http::response([ + 'query' => [ + 'statistics' => [ + 'jobs' => 3 + ] + ] + ], 200) + ]); + Bus::fake(); + + $mockJob = $this->createMock(Job::class); + + $job = new PollForMediaWikiJobsJob(); + $job->setJob($mockJob); + + $mockJob->expects($this->never())->method('fail'); + $mockJob->expects($this->never())->method('markAsFailed'); + $job->handle(); + Bus::assertDispatched(ProcessMediaWikiJobsJob::class); + } + + public function testWithFailure() + { + Http::fake([ + getenv('PLATFORM_MW_BACKEND_HOST').'/w/api.php?action=query&meta=siteinfo&siprop=statistics&format=json' => Http::response([ + 'error' => 'Something went wrong' + ], 500) + ]); + Bus::fake(); + + $mockJob = $this->createMock(Job::class); + + $job = new PollForMediaWikiJobsJob(); + $job->setJob($mockJob); + + $mockJob->expects($this->once())->method('markAsFailed'); + $mockJob->expects($this->never())->method('fail'); + $job->handle(); + Bus::assertNothingDispatched(); + } +} diff --git a/tests/Jobs/ProcessMediaWikiJobsJobTest.php b/tests/Jobs/ProcessMediaWikiJobsJobTest.php new file mode 100644 index 000000000..0829a23c4 --- /dev/null +++ b/tests/Jobs/ProcessMediaWikiJobsJobTest.php @@ -0,0 +1,86 @@ +createMock(Job::class); + $mockJob->expects($this->once())->method('fail'); + + $job = new ProcessMediaWikiJobsJob('test.wikibase.cloud'); + $job->setJob($mockJob); + + $mock = new MockHandler([ + new Response(200, [], json_encode([ 'items' => [] ])), + ]); + + $handlerStack = HandlerStack::create($mock); + $mockGuzzle = Guzzle6Client::createWithConfig([ + 'handler' => $handlerStack, + 'verify' => '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', + ]); + + $job->handle(new Client([ + 'master' => 'https://kubernetes.default.svc', + 'token' => '/var/run/secrets/kubernetes.io/serviceaccount/token', + ], null, $mockGuzzle)); + } + + public function testJobDoesNotFail() + { + $mockJob = $this->createMock(Job::class); + $mockJob->expects($this->never())->method('fail'); + + $job = new ProcessMediaWikiJobsJob('test.wikibase.cloud'); + $job->setJob($mockJob); + + $mock = new MockHandler([ + new Response(200, [], json_encode([ 'items' => [ + [ + 'kind' => 'Pod', + 'spec' => [ + 'containers' => [ + [ + 'image' => 'helloworld', + 'env' => [ + 'SOMETHING' => 'something' + ] + ] + ] + ] + ] + ]])), + new Response(200, [], json_encode([ 'items' => [] ])), + new Response(201, [], json_encode([ + 'metadata' => [ + 'name' => 'some-job-name' + ] + ])) + ]); + + $handlerStack = HandlerStack::create($mock); + $mockGuzzle = Guzzle6Client::createWithConfig([ + 'handler' => $handlerStack, + 'verify' => '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', + ]); + + $job->handle(new Client([ + 'master' => 'https://kubernetes.default.svc', + 'token' => '/var/run/secrets/kubernetes.io/serviceaccount/token', + ], null, $mockGuzzle)); + } +}