Fix cc.requests.outstanding.gauge when using puma web server

cloudfoundry#1312 introduced cc.requests.outstanding.gauge
which holds the counter in memory. With the introduction of puma there may be multiple processes, each
would have its own value for this metric. This would cause the gauge to flop between values.

This fix, specifically for puma instead reads the value from prometheus and emits it.

Additionally, add a missing mutux to counter for thread safety

lib/cloud_controller/metrics/prometheus_updater.rb

@@ -69,6 +69,10 @@ def update_gauge_metric(metric, value, labels: {})
       @registry.get(metric).set(value, labels:)
     end
 
+    def get_gauge_metric_value(metric)
+      @registry.get(metric).values[{}]
+    end
+
     def increment_gauge_metric(metric)
       @registry.get(metric).increment
     end

lib/cloud_controller/metrics/request_metrics.rb

@@ -4,32 +4,53 @@ module VCAP::CloudController
   module Metrics
     class RequestMetrics
       def initialize(statsd=CloudController::DependencyLocator.instance.statsd_client, prometheus_updater=CloudController::DependencyLocator.instance.prometheus_updater)
+        @mutex = Mutex.new
         @counter = 0
         @statsd = statsd
         @prometheus_updater = prometheus_updater
       end
 
       def start_request
-        @counter += 1
-        @statsd.gauge('cc.requests.outstanding.gauge', @counter)
         @statsd.increment 'cc.requests.outstanding'
-
-        @prometheus_updater.increment_gauge_metric(:cc_requests_outstanding_total)
+        increment_outstanding_gauge
       end
 
       def complete_request(status)
         http_status_code = "#{status.to_s[0]}XX"
         http_status_metric = "cc.http_status.#{http_status_code}"
-        @counter -= 1
-        @statsd.gauge('cc.requests.outstanding.gauge', @counter)
         @statsd.batch do |batch|
           batch.decrement 'cc.requests.outstanding'
           batch.increment 'cc.requests.completed'
           batch.increment http_status_metric
         end
 
-        @prometheus_updater.decrement_gauge_metric(:cc_requests_outstanding_total)
         @prometheus_updater.increment_counter_metric(:cc_requests_completed_total)
+
+        decrement_outstanding_gauge
+      end
+
+      private
+
+      def increment_outstanding_gauge
+        @prometheus_updater.increment_gauge_metric(:cc_requests_outstanding_total)
+        set_statsd_outstanding_gauge(1)
+      end
+
+      def decrement_outstanding_gauge
+        @prometheus_updater.decrement_gauge_metric(:cc_requests_outstanding_total)
+        set_statsd_outstanding_gauge(-1)
+      end
+
+      def set_statsd_outstanding_gauge(val)
+        if VCAP::CloudController::Config.config.get(:webserver) == 'puma'
+          gauge = @prometheus_updater.get_gauge_metric_value(:cc_requests_outstanding_total)
+        else
+          @mutex.synchronize do
+            @counter += val
+          end
+          gauge = @counter
+        end
+        @statsd.gauge('cc.requests.outstanding.gauge', gauge)
       end
     end
   end

spec/unit/lib/cloud_controller/metrics/request_metrics_spec.rb

@@ -32,6 +32,20 @@ module VCAP::CloudController::Metrics
 
         expect(prometheus_client).to have_received(:increment_gauge_metric).with(:cc_requests_outstanding_total)
       end
+
+      context 'when using puma' do
+        before do
+          allow(VCAP::CloudController::Config.config).to receive(:get).with(:webserver).and_return('puma')
+
+          allow(prometheus_client).to receive(:get_gauge_metric_value).and_return(5)
+        end
+
+        it 'uses the outstanding request value from prometheus for statsd gauge' do
+          request_metrics.start_request
+
+          expect(statsd_client).to have_received(:gauge).with('cc.requests.outstanding.gauge', 5)
+        end
+      end
     end
 
     describe '#complete_request' do
@@ -71,6 +85,20 @@ module VCAP::CloudController::Metrics
         request_metrics.complete_request(400)
         expect(batch).to have_received(:increment).with('cc.http_status.4XX')
       end
+
+      context 'when using puma' do
+        before do
+          allow(VCAP::CloudController::Config.config).to receive(:get).with(:webserver).and_return('puma')
+        end
+
+        it 'uses the outstanding request value from prometheus for statsd gauge' do
+          allow(prometheus_client).to receive(:get_gauge_metric_value).and_return(3)
+
+          request_metrics.complete_request(200)
+
+          expect(statsd_client).to have_received(:gauge).with('cc.requests.outstanding.gauge', 3)
+        end
+      end
     end
   end
 end