From 5dae26b3e06047898e792c3afa2bb01e25f08d22 Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Fri, 31 Jan 2025 12:20:23 +0100 Subject: [PATCH 1/7] Add WikiDailyMetric table Add Job for the above table updates daily Bug:T383421 --- app/Console/Kernel.php | 10 ++++ app/Jobs/UpdateWikiMetricDailyJob.php | 47 +++++++++++++++++++ app/WikiDailyMetric.php | 29 ++++++++++++ .../2025_01_28_144045_wiki_daily_metric.php | 32 +++++++++++++ tests/Jobs/UpdateWikiMetricDailyJobTest.php | 40 ++++++++++++++++ 5 files changed, 158 insertions(+) create mode 100755 app/Jobs/UpdateWikiMetricDailyJob.php create mode 100644 app/WikiDailyMetric.php create mode 100755 database/migrations/2025_01_28_144045_wiki_daily_metric.php create mode 100644 tests/Jobs/UpdateWikiMetricDailyJobTest.php diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 33cfa3a04..8b454cb31 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -10,11 +10,13 @@ use App\Jobs\RequeuePendingQsBatchesJob; use App\Jobs\SandboxCleanupJob; use App\Jobs\PollForMediaWikiJobsJob; +use App\Jobs\UpdateWikiMetricDailyJob; use App\Jobs\UpdateWikiSiteStatsJob; use App\Jobs\SendEmptyWikiNotificationsJob; use App\Jobs\CreateQueryserviceBatchesJob; use App\Jobs\FailStalledEntityImportsJob; use App\Jobs\UpdateQueryserviceAllowList; +use App\Wiki; use Illuminate\Console\Scheduling\Schedule; use Illuminate\Foundation\Console\Kernel as ConsoleKernel; @@ -60,6 +62,14 @@ protected function schedule(Schedule $schedule): void $schedule->job(new SendEmptyWikiNotificationsJob)->dailyAt('21:00'); $schedule->job(new UpdateQueryserviceAllowList)->weeklyOn(Schedule::MONDAY, '01:00'); + + $schedule->call(function () { + $wikis = Wiki::all(); + + foreach ($wikis as $wiki) { + dispatch(new UpdateWikiMetricDailyJob($wiki)); + } + })->dailyAt('23:00'); } /** diff --git a/app/Jobs/UpdateWikiMetricDailyJob.php b/app/Jobs/UpdateWikiMetricDailyJob.php new file mode 100755 index 000000000..77d6dd9e6 --- /dev/null +++ b/app/Jobs/UpdateWikiMetricDailyJob.php @@ -0,0 +1,47 @@ +wiki = $wiki; + } + /** + * Execute the job. + */ + public function handle(): void + { + $today = now()->format('Y-m-d'); + $oldRecord = WikiDailyMetric::where('wiki_id', $this->wiki->id)->latest('date')->first(); + $todayPageCount = $this->wiki->wikiSiteStats()->first()->pages ?? 0; + $isDeleted = (bool)$this->wiki->deleted_at; + if( !$oldRecord || $oldRecord->pages != $todayPageCount || !$oldRecord->is_deleted ) { + WikiDailyMetric::create([ + 'id' => $this->wiki->id . '_' . date('Y-m-d'), + 'pages' => $todayPageCount, + 'is_deleted' => $isDeleted, + 'date' => $today, + 'wiki_id' => $this->wiki->id, + ]); + + \Log::info("New metric recorded for Wiki ID {$this->wiki->id}"); + } else { + \Log::info("Metric unchanged for Wiki ID {$this->wiki->id}, no new record added."); + } + } +} diff --git a/app/WikiDailyMetric.php b/app/WikiDailyMetric.php new file mode 100644 index 000000000..aa09642fc --- /dev/null +++ b/app/WikiDailyMetric.php @@ -0,0 +1,29 @@ +string('id')->primary(); + $table->string('wiki_id'); + $table->integer('pages'); + $table->boolean('is_deleted'); + $table->date('date'); + $table->timestamps(); // Created at & Updated at + $table->unique(['wiki_id', 'date']); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('wiki_daily_metrics'); + } +}; diff --git a/tests/Jobs/UpdateWikiMetricDailyJobTest.php b/tests/Jobs/UpdateWikiMetricDailyJobTest.php new file mode 100644 index 000000000..95dff5541 --- /dev/null +++ b/tests/Jobs/UpdateWikiMetricDailyJobTest.php @@ -0,0 +1,40 @@ +create([ + 'domain' => 'thisfake.wikibase.cloud' + ]); + (new UpdateWikiMetricDailyJob($wiki))->handle(); + + // Assert the metric is updated in the database + $this->assertDatabaseHas('wiki_daily_metrics', [ + 'date' => now()->toDateString() + ]); + } + + /** @test + public function doesNotAddDuplicateRecordsWithOnlyDateChange() + { + + }*/ +} + From 4b058adc0171c54fb633025bba4bdb3fbda11f74 Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Tue, 4 Feb 2025 16:39:12 +0100 Subject: [PATCH 2/7] import needed classes in test --- tests/Jobs/UpdateWikiMetricDailyJobTest.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/Jobs/UpdateWikiMetricDailyJobTest.php b/tests/Jobs/UpdateWikiMetricDailyJobTest.php index 95dff5541..27495fa84 100644 --- a/tests/Jobs/UpdateWikiMetricDailyJobTest.php +++ b/tests/Jobs/UpdateWikiMetricDailyJobTest.php @@ -2,7 +2,10 @@ namespace Jobs; +use App\Jobs\UpdateWikiMetricDailyJob; use App\Wiki; +use Illuminate\Foundation\Testing\RefreshDatabase; +use Illuminate\Support\Facades\Queue; use Tests\TestCase; class UpdateWikiMetricDailyJobTest extends TestCase From 694c73c15ed876be8d3abf24ac7b7fc2df8f1e3f Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Thu, 6 Feb 2025 15:30:25 +0100 Subject: [PATCH 3/7] Refactoring: created a wiki class in the metric folde and added tests for it --- app/Console/Kernel.php | 11 ++-- app/Jobs/UpdateWikiDailyMetricJob.php | 27 ++++++++++ app/Jobs/UpdateWikiMetricDailyJob.php | 47 ----------------- app/Metrics/App/WikiMetrics.php | 38 ++++++++++++++ ...kiDailyMetric.php => WikiDailyMetrics.php} | 2 +- tests/Jobs/UpdateWikiDailyMetricJobTest.php | 48 +++++++++++++++++ tests/Jobs/UpdateWikiMetricDailyJobTest.php | 43 --------------- tests/Metrics/WikiMetricsTest.php | 52 +++++++++++++++++++ 8 files changed, 169 insertions(+), 99 deletions(-) create mode 100755 app/Jobs/UpdateWikiDailyMetricJob.php delete mode 100755 app/Jobs/UpdateWikiMetricDailyJob.php create mode 100644 app/Metrics/App/WikiMetrics.php rename app/{WikiDailyMetric.php => WikiDailyMetrics.php} (92%) create mode 100644 tests/Jobs/UpdateWikiDailyMetricJobTest.php delete mode 100644 tests/Jobs/UpdateWikiMetricDailyJobTest.php create mode 100644 tests/Metrics/WikiMetricsTest.php diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 8b454cb31..65e4f00a3 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -10,7 +10,7 @@ use App\Jobs\RequeuePendingQsBatchesJob; use App\Jobs\SandboxCleanupJob; use App\Jobs\PollForMediaWikiJobsJob; -use App\Jobs\UpdateWikiMetricDailyJob; +use App\Jobs\UpdateWikiDailyMetricJob; use App\Jobs\UpdateWikiSiteStatsJob; use App\Jobs\SendEmptyWikiNotificationsJob; use App\Jobs\CreateQueryserviceBatchesJob; @@ -61,15 +61,10 @@ protected function schedule(Schedule $schedule): void $schedule->job(new SendEmptyWikiNotificationsJob)->dailyAt('21:00'); - $schedule->job(new UpdateQueryserviceAllowList)->weeklyOn(Schedule::MONDAY, '01:00'); + $schedule->job(new UpdateWikiDailyMetricJob)->dailyAt('23:00'); - $schedule->call(function () { - $wikis = Wiki::all(); + $schedule->job(new UpdateQueryserviceAllowList)->weeklyOn(Schedule::MONDAY, '01:00'); - foreach ($wikis as $wiki) { - dispatch(new UpdateWikiMetricDailyJob($wiki)); - } - })->dailyAt('23:00'); } /** diff --git a/app/Jobs/UpdateWikiDailyMetricJob.php b/app/Jobs/UpdateWikiDailyMetricJob.php new file mode 100755 index 000000000..2980eab54 --- /dev/null +++ b/app/Jobs/UpdateWikiDailyMetricJob.php @@ -0,0 +1,27 @@ +get(); + foreach ( $wikis as $wiki ) { + (new WikiMetrics())->saveMetrics($wiki); + } + } +} diff --git a/app/Jobs/UpdateWikiMetricDailyJob.php b/app/Jobs/UpdateWikiMetricDailyJob.php deleted file mode 100755 index 77d6dd9e6..000000000 --- a/app/Jobs/UpdateWikiMetricDailyJob.php +++ /dev/null @@ -1,47 +0,0 @@ -wiki = $wiki; - } - /** - * Execute the job. - */ - public function handle(): void - { - $today = now()->format('Y-m-d'); - $oldRecord = WikiDailyMetric::where('wiki_id', $this->wiki->id)->latest('date')->first(); - $todayPageCount = $this->wiki->wikiSiteStats()->first()->pages ?? 0; - $isDeleted = (bool)$this->wiki->deleted_at; - if( !$oldRecord || $oldRecord->pages != $todayPageCount || !$oldRecord->is_deleted ) { - WikiDailyMetric::create([ - 'id' => $this->wiki->id . '_' . date('Y-m-d'), - 'pages' => $todayPageCount, - 'is_deleted' => $isDeleted, - 'date' => $today, - 'wiki_id' => $this->wiki->id, - ]); - - \Log::info("New metric recorded for Wiki ID {$this->wiki->id}"); - } else { - \Log::info("Metric unchanged for Wiki ID {$this->wiki->id}, no new record added."); - } - } -} diff --git a/app/Metrics/App/WikiMetrics.php b/app/Metrics/App/WikiMetrics.php new file mode 100644 index 000000000..c75b03f51 --- /dev/null +++ b/app/Metrics/App/WikiMetrics.php @@ -0,0 +1,38 @@ +format('Y-m-d'); + $oldRecord = WikiDailyMetrics::where('wiki_id', $wiki->id)->latest('date')->first(); + $todayPageCount = $wiki->wikiSiteStats()->first()->pages ?? 0; + $isDeleted = (bool)$wiki->deleted_at; + if ($oldRecord) { + if ($oldRecord->is_deleted) { + \Log::info("WikiMetrics is deleted, no new record for WikiMetrics ID {$wiki->id}."); + return; + } + if ($oldRecord->pages === $todayPageCount) { + \Log::info("Page count unchanged for WikiMetrics ID {$wiki->id}, no new record added."); + return; + } + } + WikiDailyMetrics::create([ + 'id' => $wiki->id . '_' . date('Y-m-d'), + 'pages' => $todayPageCount, + 'is_deleted' => $isDeleted, + 'date' => $today, + 'wiki_id' => $wiki->id, + ]); + \Log::info("New metric recorded for WikiMetrics ID {$wiki->id}"); + } + +} + diff --git a/app/WikiDailyMetric.php b/app/WikiDailyMetrics.php similarity index 92% rename from app/WikiDailyMetric.php rename to app/WikiDailyMetrics.php index aa09642fc..e29eeaf1b 100644 --- a/app/WikiDailyMetric.php +++ b/app/WikiDailyMetrics.php @@ -5,7 +5,7 @@ use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; -class WikiDailyMetric extends Model +class WikiDailyMetrics extends Model { use HasFactory; diff --git a/tests/Jobs/UpdateWikiDailyMetricJobTest.php b/tests/Jobs/UpdateWikiDailyMetricJobTest.php new file mode 100644 index 000000000..12263e367 --- /dev/null +++ b/tests/Jobs/UpdateWikiDailyMetricJobTest.php @@ -0,0 +1,48 @@ +create([ + 'domain' => 'example.wikibase.cloud', + ]); + $deletedWiki = Wiki::factory()->create([ + 'domain' => 'deletedwiki.wikibase.cloud', + ]); + $deletedWiki->delete(); + + (new UpdateWikiDailyMetricJob())->handle(); + + $this->assertDatabaseHas('wiki_daily_metrics', [ + 'wiki_id' => $activeWiki->id, + 'date' => Carbon::today()->toDateString() + ]); + + $this->assertDatabaseHas('wiki_daily_metrics', [ + 'wiki_id' => $deletedWiki->id, + 'date' => Carbon::today()->toDateString() + ]); + } + +} diff --git a/tests/Jobs/UpdateWikiMetricDailyJobTest.php b/tests/Jobs/UpdateWikiMetricDailyJobTest.php deleted file mode 100644 index 27495fa84..000000000 --- a/tests/Jobs/UpdateWikiMetricDailyJobTest.php +++ /dev/null @@ -1,43 +0,0 @@ -create([ - 'domain' => 'thisfake.wikibase.cloud' - ]); - (new UpdateWikiMetricDailyJob($wiki))->handle(); - - // Assert the metric is updated in the database - $this->assertDatabaseHas('wiki_daily_metrics', [ - 'date' => now()->toDateString() - ]); - } - - /** @test - public function doesNotAddDuplicateRecordsWithOnlyDateChange() - { - - }*/ -} - diff --git a/tests/Metrics/WikiMetricsTest.php b/tests/Metrics/WikiMetricsTest.php new file mode 100644 index 000000000..bba606ac6 --- /dev/null +++ b/tests/Metrics/WikiMetricsTest.php @@ -0,0 +1,52 @@ +create([ + 'domain' => 'thisfake.wikibase.cloud' + ]); + + (new WikiMetrics())->saveMetrics($wiki); + // Assert the metric is updated in the database + $this->assertDatabaseHas('wiki_daily_metrics', [ + 'date' => now()->toDateString() + ]); + } + + /** @test*/ + public function doesNotAddDuplicateRecordsWithOnlyDateChange() + { + $wiki = Wiki::factory()->create([ + 'domain' => 'thisfake.wikibase.cloud' + ]); + //Insert an old metric value for a wiki + WikiDailyMetrics::create([ + 'id' => $wiki->id. '_'. Carbon::yesterday()->toDateString(), + 'wiki_id' => $wiki->id, + 'date' => Carbon::yesterday()->toDateString(), + 'pages' => 0, + 'is_deleted' => 0 + ]); + (new WikiMetrics())->saveMetrics($wiki); + + //Assert No new record was created for today + $this->assertDatabaseMissing('wiki_daily_metrics', [ + 'wiki_id' => $wiki->id, + 'date' => Carbon::today()->toDateString() + ]); + } +} + From d9da4185b52ae3fd671b163723d3744f614ab59c Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Thu, 6 Feb 2025 16:13:01 +0100 Subject: [PATCH 4/7] added comments --- tests/Jobs/UpdateWikiDailyMetricJobTest.php | 5 ++++- tests/Metrics/WikiMetricsTest.php | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/Jobs/UpdateWikiDailyMetricJobTest.php b/tests/Jobs/UpdateWikiDailyMetricJobTest.php index 12263e367..8fcd83406 100644 --- a/tests/Jobs/UpdateWikiDailyMetricJobTest.php +++ b/tests/Jobs/UpdateWikiDailyMetricJobTest.php @@ -1,6 +1,6 @@ create([ diff --git a/tests/Metrics/WikiMetricsTest.php b/tests/Metrics/WikiMetricsTest.php index bba606ac6..541f72742 100644 --- a/tests/Metrics/WikiMetricsTest.php +++ b/tests/Metrics/WikiMetricsTest.php @@ -1,6 +1,6 @@ create([ From d4113b40787f08b5c6f6df759ebf0930f2b9c451 Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Mon, 10 Feb 2025 12:54:59 +0100 Subject: [PATCH 5/7] Remove test annotations --- tests/Jobs/UpdateWikiDailyMetricJobTest.php | 4 ++-- tests/Metrics/WikiMetricsTest.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/Jobs/UpdateWikiDailyMetricJobTest.php b/tests/Jobs/UpdateWikiDailyMetricJobTest.php index 8fcd83406..68ded5fbd 100644 --- a/tests/Jobs/UpdateWikiDailyMetricJobTest.php +++ b/tests/Jobs/UpdateWikiDailyMetricJobTest.php @@ -14,7 +14,7 @@ class UpdateWikiDailyMetricJobTest extends TestCase use RefreshDatabase; - /** @test */ + public function dispatchJob() { Queue::fake(); @@ -24,7 +24,7 @@ public function dispatchJob() Queue::assertPushed(UpdateWikiDailyMetricJob::class); } - /** @test */ + public function runJobForAllWikisIncludingDeletedWikis() { $activeWiki = Wiki::factory()->create([ diff --git a/tests/Metrics/WikiMetricsTest.php b/tests/Metrics/WikiMetricsTest.php index 541f72742..f2a510ff7 100644 --- a/tests/Metrics/WikiMetricsTest.php +++ b/tests/Metrics/WikiMetricsTest.php @@ -13,7 +13,7 @@ class WikiMetricsTest extends TestCase { use RefreshDatabase; - /** @test */ + public function successfullyAddRecords() { $wiki = Wiki::factory()->create([ @@ -27,7 +27,7 @@ public function successfullyAddRecords() ]); } - /** @test*/ + public function doesNotAddDuplicateRecordsWithOnlyDateChange() { $wiki = Wiki::factory()->create([ From 7235b1b0ddd1ba21391c26b94bb54fea2601c49b Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Mon, 10 Feb 2025 12:57:30 +0100 Subject: [PATCH 6/7] Update CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9766a50d..e752ac80c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # api + +## 10x.18.3 - 10 February 2024 +- Added new table wiki_daily_metrics +- Added a new Wiki.php class in the metric folder for wiki related metrics +- Added a Job scheduled to run daily and update metrics in the wiki_daily_metric table. + ## 10x.18.2 - 16 December 2024 - Update FAQ link in empty wiki notification email From 99b86b8d33644c13df00b33d64d77812a64ab40e Mon Sep 17 00:00:00 2001 From: Perside Rosalie Date: Mon, 10 Feb 2025 13:05:42 +0100 Subject: [PATCH 7/7] Fix typo in CHANGELOG and tests --- CHANGELOG.md | 2 +- tests/Jobs/UpdateWikiDailyMetricJobTest.php | 4 ++-- tests/Metrics/WikiMetricsTest.php | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e752ac80c..58f11a0b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # api -## 10x.18.3 - 10 February 2024 +## 10x.18.3 - 10 February 2025 - Added new table wiki_daily_metrics - Added a new Wiki.php class in the metric folder for wiki related metrics - Added a Job scheduled to run daily and update metrics in the wiki_daily_metric table. diff --git a/tests/Jobs/UpdateWikiDailyMetricJobTest.php b/tests/Jobs/UpdateWikiDailyMetricJobTest.php index 68ded5fbd..d7a334c5c 100644 --- a/tests/Jobs/UpdateWikiDailyMetricJobTest.php +++ b/tests/Jobs/UpdateWikiDailyMetricJobTest.php @@ -15,7 +15,7 @@ class UpdateWikiDailyMetricJobTest extends TestCase use RefreshDatabase; - public function dispatchJob() + public function testDispatchJob() { Queue::fake(); @@ -25,7 +25,7 @@ public function dispatchJob() } - public function runJobForAllWikisIncludingDeletedWikis() + public function testRunJobForAllWikisIncludingDeletedWikis() { $activeWiki = Wiki::factory()->create([ 'domain' => 'example.wikibase.cloud', diff --git a/tests/Metrics/WikiMetricsTest.php b/tests/Metrics/WikiMetricsTest.php index f2a510ff7..1806d90e9 100644 --- a/tests/Metrics/WikiMetricsTest.php +++ b/tests/Metrics/WikiMetricsTest.php @@ -14,7 +14,7 @@ class WikiMetricsTest extends TestCase use RefreshDatabase; - public function successfullyAddRecords() + public function testSuccessfullyAddRecords() { $wiki = Wiki::factory()->create([ 'domain' => 'thisfake.wikibase.cloud' @@ -28,7 +28,7 @@ public function successfullyAddRecords() } - public function doesNotAddDuplicateRecordsWithOnlyDateChange() + public function testDoesNotAddDuplicateRecordsWithOnlyDateChange() { $wiki = Wiki::factory()->create([ 'domain' => 'thisfake.wikibase.cloud'