feat(poll_rate): metrics to be reported to agent at configured interv…

.circleci/config.yml

@@ -58,7 +58,7 @@ executors:
       - image: quay.io/minio/minio
         command: ["server", "/data"]
       - image: s12v/sns
-      - image: public.ecr.aws/docker/library/rabbitmq:latest
+      - image: public.ecr.aws/docker/library/rabbitmq:4.2
       - image: public.ecr.aws/sprig/elasticmq-native
       - image: public.ecr.aws/docker/library/mongo:5-focal
   mysql2:

lib/instana/backend/host_agent_reporting_observer.rb

@@ -11,30 +11,59 @@ class HostAgentReportingObserver
       TRACES_DATA_URL = "/com.instana.plugin.ruby/traces.%i".freeze
       TRACE_METRICS_URL = "/tracermetrics".freeze
 
-      attr_reader :report_timer
+      attr_reader :metrics_timer, :traces_timer
 
       # @param [RequestClient] client used to make requests to the backend
       # @param [Concurrent::Atom] discovery object used to store discovery response in
       def initialize(client, discovery, logger: ::Instana.logger, timer_class: Concurrent::TimerTask, processor: ::Instana.processor)
         @client = client
         @discovery = discovery
         @logger = logger
-        @report_timer = timer_class.new(execution_interval: 1, run_now: true) { report_to_backend }
+        @timer_class = timer_class
         @nonce = Time.now
         @processor = processor
+
+        # Initialize timers with default 1 second interval
+        @metrics_timer = @timer_class.new(execution_interval: 1, run_now: true) { report_metrics_to_backend }
+        @traces_timer = @timer_class.new(execution_interval: 1, run_now: true) { report_traces_to_backend }
       end
 
       def update(time, _old_version, new_version)
         return unless time > @nonce
 
         @nonce = time
-        new_version.nil? ? @report_timer.shutdown : @report_timer.execute
+
+        if new_version.nil?
+          @metrics_timer&.shutdown
+          @traces_timer&.shutdown
+        else
+          # Read poll_rate from discovery payload - it's nested under plugin.ruby.poll_rate
+          discovery = @discovery.value
+          poll_rate = discovery&.dig('plugin', 'ruby', 'poll_rate') || 1
+
+          # Only recreate metrics_timer if poll_rate is different from current interval
+          if @metrics_timer.nil? || @metrics_timer.execution_interval != poll_rate
+            @metrics_timer&.shutdown
+            @metrics_timer = @timer_class.new(execution_interval: poll_rate, run_now: true) { report_metrics_to_backend }
+          end
+          @metrics_timer.execute
+
+          # Traces timer always uses 1 second interval
+          @traces_timer&.shutdown
+          @traces_timer = @timer_class.new(execution_interval: 1, run_now: true) { report_traces_to_backend }
+          @traces_timer.execute
+        end
       end
 
       private
 
-      def report_to_backend
+      def report_metrics_to_backend
         report_metrics if ::Instana.config[:metrics][:enabled]
+      rescue StandardError => e
+        @logger.error(%(#{e}\n#{e.backtrace.join("\n")}))
+      end
+
+      def report_traces_to_backend
         report_traces if ::Instana.config[:tracing][:enabled]
         report_trace_stats if ::Instana.config[:tracing][:enabled]
       rescue StandardError => e

lib/instana/backend/request_client.rb

@@ -38,7 +38,10 @@ def initialize(host, port, use_ssl: false)
         timeout = Integer(ENV.fetch('INSTANA_TIMEOUT', 500))
         @host = host
         @port = port
-        @client = Net::HTTP.start(host, port, use_ssl: use_ssl, read_timeout: timeout)
+        @use_ssl = use_ssl
+        @timeout = timeout
+        @client_mutex = Mutex.new
+        @client = nil
       end
 
       # Send a request to the backend. If data is a {Hash},
@@ -60,7 +63,10 @@ def send_request(method, path, data = nil, headers = {})
                  data
                end
         begin
-          response = @client.send_request(method, path, body, headers)
+          response = @client_mutex.synchronize do
+            ensure_connection
+            @client.send_request(method, path, body, headers)
+          end
           Response.new(response)
         rescue Errno::ECONNREFUSED => e
           Instana.logger.debug("Connection refused to #{@host}:#{@port} - #{e.message}")
@@ -74,6 +80,11 @@ def send_request(method, path, data = nil, headers = {})
         rescue SocketError => e
           Instana.logger.debug("Socket error connecting to #{@host}:#{@port} - #{e.message}")
           create_error_response('502', 'Socket Error', 'Socket error', e.message)
+        rescue IOError => e
+          Instana.logger.debug("IO error sending request to #{@host}:#{@port} - #{e.message}")
+          # Reset connection on IO errors and retry once
+          @client_mutex.synchronize { reset_connection }
+          create_error_response('500', 'IO Error', 'IOError', e.message)
         rescue StandardError => e
           Instana.logger.debug("Error sending request to #{@host}:#{@port} - #{e.class}: #{e.message}")
           create_error_response('500', 'Internal Error', e.class.to_s, e.message)
@@ -82,6 +93,22 @@ def send_request(method, path, data = nil, headers = {})
 
       private
 
+      def ensure_connection
+        return if @client && !@client.instance_variable_get(:@socket).nil?
+
+        reset_connection
+        @client = Net::HTTP.start(@host, @port, use_ssl: @use_ssl, read_timeout: @timeout)
+      end
+
+      def reset_connection
+        begin
+          @client&.finish
+        rescue
+          nil
+        end
+        @client = nil
+      end
+
       def encode_body(data)
         # :nocov:
         INSTANA_USE_OJ ? Oj.dump(data, mode: :strict) : JSON.dump(data)

lib/instana/util.rb

@@ -27,33 +27,33 @@ def get_rb_source(file)
       def take_snapshot
         data = {}
 
-        data[:sensorVersion] = ::Instana::VERSION
-        data[:ruby_version] = RUBY_VERSION
+        data[:sensorVersion] = ::Instana::VERSION.dup
+        data[:ruby_version] = RUBY_VERSION.dup
         data[:rpl] = RUBY_PATCHLEVEL if defined?(RUBY_PATCHLEVEL)
 
         # Framework Detection
         if defined?(::RailsLts::VERSION)
-          data[:framework] = "Rails on Rails LTS-#{::RailsLts::VERSION}"
+          data[:framework] = "Rails on Rails LTS-#{::RailsLts::VERSION}".dup
 
         elsif defined?(::Rails.version)
-          data[:framework] = "Ruby on Rails #{::Rails.version}"
+          data[:framework] = "Ruby on Rails #{::Rails.version}".dup
 
         elsif defined?(::Grape::VERSION)
-          data[:framework] = "Grape #{::Grape::VERSION}"
+          data[:framework] = "Grape #{::Grape::VERSION}".dup
 
         elsif defined?(::Padrino::VERSION)
-          data[:framework] = "Padrino #{::Padrino::VERSION}"
+          data[:framework] = "Padrino #{::Padrino::VERSION}".dup
 
         elsif defined?(::Sinatra::VERSION)
-          data[:framework] = "Sinatra #{::Sinatra::VERSION}"
+          data[:framework] = "Sinatra #{::Sinatra::VERSION}".dup
         end
 
         # Report Bundle
         if defined?(::Gem) && Gem.respond_to?(:loaded_specs)
           data[:versions] = {}
 
           Gem.loaded_specs.each do |k, v|
-            data[:versions][k] = v.version.to_s
+            data[:versions][k.dup] = v.version.to_s.dup
           end
         end
 

test/backend/host_agent_reporting_observer_test.rb

@@ -10,16 +10,20 @@ def test_start_stop
 
     subject = Instana::Backend::HostAgentReportingObserver.new(client, discovery, timer_class: MockTimer)
 
-    refute subject.report_timer.running
+    refute subject.metrics_timer.running
+    refute subject.traces_timer.running
 
     subject.update(Time.now, nil, true)
-    assert subject.report_timer.running
+    assert subject.metrics_timer.running
+    assert subject.traces_timer.running
 
     subject.update(Time.now, nil, nil)
-    refute subject.report_timer.running
+    refute subject.metrics_timer.running
+    refute subject.traces_timer.running
 
     subject.update(Time.now - 500, nil, true)
-    refute subject.report_timer.running
+    refute subject.metrics_timer.running
+    refute subject.traces_timer.running
   end
 
   def test_report
@@ -33,7 +37,7 @@ def test_report
 
     subject = Instana::Backend::HostAgentReportingObserver.new(client, discovery, timer_class: MockTimer)
 
-    subject.report_timer.block.call
+    subject.metrics_timer.block.call
   end
 
   def test_report_fail
@@ -53,7 +57,7 @@ def test_report_fail
 
     subject = Instana::Backend::HostAgentReportingObserver.new(client, discovery, timer_class: MockTimer)
 
-    subject.report_timer.block.call
+    subject.metrics_timer.block.call
     assert_nil discovery.value
   end
 
@@ -80,7 +84,7 @@ def test_agent_action
 
     subject = Instana::Backend::HostAgentReportingObserver.new(client, discovery, timer_class: MockTimer)
 
-    subject.report_timer.block.call
+    subject.metrics_timer.block.call
   end
 
   def test_agent_actions
@@ -104,7 +108,7 @@ def test_agent_actions
 
     subject = Instana::Backend::HostAgentReportingObserver.new(client, discovery, timer_class: MockTimer)
 
-    subject.report_timer.block.call
+    subject.metrics_timer.block.call
   end
 
   def test_agent_action_error
@@ -119,7 +123,7 @@ def test_agent_action_error
 
     subject = Instana::Backend::HostAgentReportingObserver.new(client, discovery, timer_class: MockTimer)
 
-    subject.report_timer.block.call
+    subject.metrics_timer.block.call
   end
 
   def test_disable_metrics
@@ -130,7 +134,7 @@ def test_disable_metrics
 
     subject = Instana::Backend::HostAgentReportingObserver.new(client, discovery, timer_class: MockTimer)
 
-    subject.report_timer.block.call
+    subject.metrics_timer.block.call
   ensure
     ::Instana.config[:metrics][:enabled] = true
   end
@@ -153,7 +157,7 @@ def test_disable_metrics_memory
 
     subject = Instana::Backend::HostAgentReportingObserver.new(client, discovery, timer_class: MockTimer)
 
-    subject.report_timer.block.call
+    subject.metrics_timer.block.call
   ensure
     ::Instana.config[:metrics][:memory][:enabled] = true
   end
@@ -176,7 +180,7 @@ def test_disable_gc
 
     subject = Instana::Backend::HostAgentReportingObserver.new(client, discovery, timer_class: MockTimer)
 
-    subject.report_timer.block.call
+    subject.metrics_timer.block.call
   ensure
     ::Instana.config[:metrics][:gc][:enabled] = true
   end
@@ -199,7 +203,7 @@ def test_disable_thread
 
     subject = Instana::Backend::HostAgentReportingObserver.new(client, discovery, timer_class: MockTimer)
 
-    subject.report_timer.block.call
+    subject.metrics_timer.block.call
   ensure
     ::Instana.config[:metrics][:thread][:enabled] = true
   end
@@ -212,7 +216,7 @@ def test_disable_tracing
 
     subject = Instana::Backend::HostAgentReportingObserver.new(client, discovery, timer_class: MockTimer)
 
-    subject.report_timer.block.call
+    subject.traces_timer.block.call
   ensure
     ::Instana.config[:tracing][:enabled] = true
   end
@@ -235,7 +239,7 @@ def send
 
     subject = Instana::Backend::HostAgentReportingObserver.new(client, discovery, timer_class: MockTimer, processor: processor)
 
-    subject.report_timer.block.call
+    subject.traces_timer.block.call
     refute_nil discovery.value
   end
 
@@ -264,7 +268,7 @@ def send
 
     subject = Instana::Backend::HostAgentReportingObserver.new(client, discovery, timer_class: MockTimer, processor: processor)
 
-    subject.report_timer.block.call
+    subject.traces_timer.block.call
     assert_nil discovery.value
   end
 
@@ -283,7 +287,35 @@ def send
 
     subject = Instana::Backend::HostAgentReportingObserver.new(client, discovery, timer_class: MockTimer, processor: processor, logger: Logger.new('/dev/null'))
 
-    subject.report_timer.block.call
+    subject.traces_timer.block.call
     assert_equal({"pid" => 1234}, discovery.value)
   end
+
+  def test_poll_rate_changes_metrics_timer_interval
+    client = Instana::Backend::RequestClient.new('10.10.10.10', 9292)
+    discovery = Concurrent::Atom.new(nil)
+
+    subject = Instana::Backend::HostAgentReportingObserver.new(client, discovery, timer_class: MockTimer)
+
+    # Initially, metrics_timer should have 1 second interval (default)
+    assert_equal 1, subject.metrics_timer.opts[:execution_interval]
+    refute subject.metrics_timer.running
+
+    # Simulate first discovery with poll_rate = 1 (should keep 1 second interval)
+    discovery.swap { {'pid' => 1234, 'plugin' => {'ruby' => {'poll_rate' => 1}}} }
+    subject.update(Time.now, nil, true)
+    assert subject.metrics_timer.running
+    assert_equal 1, subject.metrics_timer.opts[:execution_interval]
+    assert_equal({'pid' => 1234, 'plugin' => {'ruby' => {'poll_rate' => 1}}}, discovery.value)
+
+    # Simulate discovery cycle changing poll_rate to 5 seconds
+    discovery.swap { {'pid' => 1234, 'plugin' => {'ruby' => {'poll_rate' => 5}}} }
+    subject.update(Time.now + 1, nil, true)
+    assert subject.metrics_timer.running
+    assert_equal 5, subject.metrics_timer.opts[:execution_interval]
+    assert_equal({'pid' => 1234, 'plugin' => {'ruby' => {'poll_rate' => 5}}}, discovery.value)
+
+    # Verify traces_timer always stays at 1 second
+    assert_equal 1, subject.traces_timer.opts[:execution_interval]
+  end
 end

test/support/mock_timer.rb

@@ -10,6 +10,10 @@ def initialize(*args, &blk)
     @running = false
   end
 
+  def execution_interval
+    @opts[:execution_interval]
+  end
+
   def shutdown
     @running = false
   end