Conditionally take snapshots into account

renoki-co · Sep 25, 2021 · 215754e · 215754e
1 parent fb60a8a
commit 215754e
Show file tree

Hide file tree

Showing 6 changed files with 97 additions and 7 deletions.
diff --git a/config/horizon-exporter.php b/config/horizon-exporter.php
@@ -40,4 +40,25 @@
 
     'middleware' => ['web'],
 
+    /*
+    |--------------------------------------------------------------------------
+    | Include Past Snapshots
+    |--------------------------------------------------------------------------
+    |
+    | If you periodically run the horizon:snapshot command, it will reset
+    | the values exposed to Prometheus to 0, this leading to Prometheus
+    | reading counters starting from 0 each time it is ran.
+    |
+    | You can prevent that by enabling this setting. On the current values
+    | for throughput, jobs, etc., the values from snapshots will also be
+    | included. This may lead to graphs showing more jobs if you enable
+    | this after you had it disabled.
+    |
+    | It is recommended to keep it enabled, or to disable it and
+    | make sure the horizon:snapshot is never called.
+    |
+    */
+
+    'include_snapshots' => (bool) env('HORIZON_EXPORTER_SNAPSHOTS', true),
+
 ];
diff --git a/src/Metrics/JobsRuntime.php b/src/Metrics/JobsRuntime.php
@@ -23,8 +23,12 @@ public function update(): void
     {
         if ($jobs = app(MetricsRepository::class)->measuredJobs()) {
             foreach ($jobs as $job) {
+                $value = collect(config('horizon-exporter.include_snapshots') ? app(MetricsRepository::class)->snapshotsForJob($job) : [])
+                    ->merge([(object) ['runtime' => app(MetricsRepository::class)->runtimeForJob($job)]])
+                    ->average('runtime');
+
                 $this->collector->set(
-                    value: app(MetricsRepository::class)->runtimeForJob($job),
+                    value: $value,
                     labels: [
                         'job' => $job,
                     ],

diff --git a/src/Metrics/JobsThroughput.php b/src/Metrics/JobsThroughput.php
@@ -23,8 +23,16 @@ public function update(): void
     {
         if ($jobs = app(MetricsRepository::class)->measuredJobs()) {
             foreach ($jobs as $job) {
+                $value = app(MetricsRepository::class)->throughputForJob($job);
+
+                if (config('horizon-exporter.include_snapshots')) {
+                    foreach (app(MetricsRepository::class)->snapshotsForJob($job) as $snapshot) {
+                        $value += $snapshot->throughput;
+                    }
+                }
+
                 $this->collector->set(
-                    value: app(MetricsRepository::class)->throughputForJob($job),
+                    value: $value,
                     labels: [
                         'job' => $job,
                     ],

diff --git a/src/Metrics/QueuesRuntime.php b/src/Metrics/QueuesRuntime.php
@@ -23,8 +23,12 @@ public function update(): void
     {
         if ($queues = app(MetricsRepository::class)->measuredQueues()) {
             foreach ($queues as $queue) {
+                $value = collect(config('horizon-exporter.include_snapshots') ? app(MetricsRepository::class)->snapshotsForQueue($queue) : [])
+                    ->merge([(object) ['runtime' => app(MetricsRepository::class)->runtimeForQueue($queue)]])
+                    ->average('runtime');
+
                 $this->collector->set(
-                    value: app(MetricsRepository::class)->runtimeForQueue($queue),
+                    value: $value,
                     labels: [
                         'queue' => $queue,
                     ],

diff --git a/src/Metrics/QueuesThroughput.php b/src/Metrics/QueuesThroughput.php
@@ -23,8 +23,16 @@ public function update(): void
     {
         if ($queues = app(MetricsRepository::class)->measuredQueues()) {
             foreach ($queues as $queue) {
+                $value = app(MetricsRepository::class)->throughputForQueue($queue);
+
+                if (config('horizon-exporter.include_snapshots')) {
+                    foreach (app(MetricsRepository::class)->snapshotsForQueue($queue) as $snapshot) {
+                        $value += $snapshot->throughput;
+                    }
+                }
+
                 $this->collector->set(
-                    value: app(MetricsRepository::class)->throughputForQueue($queue),
+                    value: $value,
                     labels: [
                         'queue' => $queue,
                     ],

diff --git a/tests/MetricsTest.php b/tests/MetricsTest.php
@@ -5,6 +5,7 @@
 use Illuminate\Support\Facades\Queue;
 use Laravel\Horizon\Contracts\HorizonCommandQueue;
 use Laravel\Horizon\Contracts\MasterSupervisorRepository;
+use Laravel\Horizon\Contracts\MetricsRepository;
 use Laravel\Horizon\Contracts\SupervisorRepository;
 use Laravel\Horizon\MasterSupervisorCommands\AddSupervisor;
 use Laravel\Horizon\Supervisor;
@@ -71,10 +72,20 @@ public function test_queues_throughput_metric()
         $this->work();
         $this->work();
 
+        resolve(MetricsRepository::class)->snapshot();
+
+        Queue::push(new Jobs\BasicJob);
+        Queue::push(new Jobs\BasicJob);
+        Queue::push(new Jobs\BasicJob);
+
+        $this->work();
+        $this->work();
+        $this->work();
+
         $response = HorizonExporter::exportAsPlainText();
 
         $this->assertStringContainsString(
-            'horizon_queue_throughput{queue="default"} 3',
+            'horizon_queue_throughput{queue="default"} 6',
             $response,
         );
     }
@@ -95,15 +106,27 @@ public function test_jobs_throughput_metric()
         $this->work();
         $this->work();
 
+        resolve(MetricsRepository::class)->snapshot();
+
+        Queue::push(new Jobs\BasicJob);
+        Queue::push(new Jobs\BasicJob);
+        Queue::push(new Jobs\BasicJob);
+        Queue::push(new Jobs\BasicJob2);
+
+        $this->work();
+        $this->work();
+        $this->work();
+        $this->work();
+
         $response = HorizonExporter::exportAsPlainText();
 
         $this->assertStringContainsString(
-            'horizon_job_throughput{job="RenokiCo\\\\HorizonExporter\\\\Test\\\\Jobs\\\\BasicJob"} 3',
+            'horizon_job_throughput{job="RenokiCo\\\\HorizonExporter\\\\Test\\\\Jobs\\\\BasicJob"} 6',
             $response,
         );
 
         $this->assertStringContainsString(
-            'horizon_job_throughput{job="RenokiCo\\\\HorizonExporter\\\\Test\\\\Jobs\\\\BasicJob2"} 1',
+            'horizon_job_throughput{job="RenokiCo\\\\HorizonExporter\\\\Test\\\\Jobs\\\\BasicJob2"} 2',
             $response,
         );
     }
@@ -122,6 +145,16 @@ public function test_queues_runtime_metric()
         $this->work();
         $this->work();
 
+        resolve(MetricsRepository::class)->snapshot();
+
+        Queue::push(new Jobs\BasicJob);
+        Queue::push(new Jobs\BasicJob);
+        Queue::push(new Jobs\BasicJob);
+
+        $this->work();
+        $this->work();
+        $this->work();
+
         $response = HorizonExporter::exportAsPlainText();
 
         $this->assertStringContainsString(
@@ -146,6 +179,18 @@ public function test_jobs_runtime_metric()
         $this->work();
         $this->work();
 
+        resolve(MetricsRepository::class)->snapshot();
+
+        Queue::push(new Jobs\BasicJob);
+        Queue::push(new Jobs\BasicJob);
+        Queue::push(new Jobs\BasicJob);
+        Queue::push(new Jobs\BasicJob2);
+
+        $this->work();
+        $this->work();
+        $this->work();
+        $this->work();
+
         $response = HorizonExporter::exportAsPlainText();
 
         $this->assertStringContainsString(