cleaned up exception handling. changed synchronization to stop using …

…Mutex and start using thread.critical.

lib/newrelic/agent/agent.rb

@@ -235,7 +235,7 @@ def connect
       log! "Turning New Relic Agent off."
       return false
 
-    rescue => e
+    rescue Timeout::Error, StandardError => e
       log.error "Error attempting to connect to New Relic RPM Service at #{@remote_host}:#{@remote_port}"
       log.error e.message
       log.debug e.backtrace.join("\n")
@@ -368,10 +368,19 @@ def harvest_and_send_timeslice_data
       @unsent_timeslice_data ||= {}
       @unsent_timeslice_data = @stats_engine.harvest_timeslice_data(@unsent_timeslice_data, @metric_ids)
 
-      metric_ids = invoke_remote(:metric_data, @agent_id, 
-              @last_harvest_time.to_f, 
-              now.to_f, 
-              @unsent_timeslice_data.values)
+
+      begin
+        metric_ids = invoke_remote(:metric_data, @agent_id, 
+                @last_harvest_time.to_f, 
+                now.to_f, 
+                @unsent_timeslice_data.values)
+
+      rescue Timeout::Error
+        # assume that the data was received. chances are that it was
+        metric_ids = nil
+      end
+
+
       @metric_ids.merge! metric_ids unless metric_ids.nil?
 
       log.debug "#{Time.now}: sent #{@unsent_timeslice_data.length} timeslices (#{@agent_id})"
@@ -441,10 +450,14 @@ def invoke_remote(method, *args)
         request.verify_mode = OpenSSL::SSL::VERIFY_NONE
       end
 
+      # set a long timeout on purpose (15 minutes). there are times when the server gets really backed up
+      request.read_timeout = 15 * 60
+
       # we'd like to use to_query but it is not present in all supported rails platforms
       # params = {:method => method, :license_key => license_key, :protocol_version => PROTOCOL_VERSION }
       # uri = "/agent_listener/invoke_raw_method?#{params.to_query}"
       uri = "/agent_listener/invoke_raw_method?method=#{method}&license_key=#{license_key}&protocol_version=#{PROTOCOL_VERSION}"
+
       response = request.start do |http|
         http.post(uri, post_data) 
       end
@@ -495,7 +508,7 @@ def graceful_disconnect
           log.debug "Sending graceful shutdown message to #{remote_host}:#{remote_port}"
           invoke_remote :shutdown, @agent_id, Time.now.to_f
           log.debug "Shutdown Complete"
-        rescue => e
+        rescue Timeout::Error, StandardError => e
           log.warn "Error sending shutdown message to #{remote_host}:#{remote_port}:"
           log.warn e
           log.debug e.backtrace.join("\n")

lib/newrelic/agent/synchronize.rb

@@ -0,0 +1,32 @@
+
+require 'sync'
+
+
+module NewRelic::Agent::Synchronize
+
+  def synchronize_sync(&block)
+    @_local_sync ||= Sync.new
+
+    @_local_sync.synchronize(:EX) do
+      block.call
+    end
+  end
+
+
+  def synchronize_thread
+    old_val = Thread.critical
+
+    Thread.critical = true
+
+    begin
+      yield
+    ensure
+      Thread.critical = old_val
+    end
+  end
+
+  alias synchronize synchronize_thread
+  alias synchronize_quick synchronize_thread
+  alias synchronized_long synchronize_sync
+
+end

lib/newrelic/agent/transaction_sampler.rb

@@ -1,12 +1,14 @@
 require 'newrelic/transaction_sample'
 require 'thread'
 require 'newrelic/agent/method_tracer'
+require 'newrelic/agent/synchronize'
 
 module NewRelic::Agent
   class TransactionSampler
+    include(Synchronize)
+
     def initialize(agent = nil, max_samples = 100)
       @samples = []
-      @mutex = Mutex.new
       @max_samples = max_samples
 
       # when the agent is nil, we are in a unit test.
@@ -56,7 +58,7 @@ def notice_scope_empty
         builder.finish_trace
         reset_builder
 
-        @mutex.synchronize do
+        synchronize do
           sample = builder.sample
 
           # ensure we don't collect more than a specified number of samples in memory
@@ -97,7 +99,7 @@ def notice_sql(sql)
     # and clear the collected sample list. 
 
     def harvest_slowest_sample(previous_slowest = nil)
-      @mutex.synchronize do
+      synchronize do
         slowest = @slowest_sample
         @slowest_sample = nil
 
@@ -113,7 +115,7 @@ def harvest_slowest_sample(previous_slowest = nil)
 
     # get the list of samples without clearing the list.
     def get_samples
-      @mutex.synchronize do
+      synchronize do
         return @samples.clone
       end
     end

lib/newrelic/agent/worker_loop.rb

@@ -1,14 +1,17 @@
 require 'thread'
+require 'newrelic/agent/synchronize'
 
 # A worker loop executes a set of registered tasks on a single thread.  
 # A task is a proc or block with a specified call period in seconds.  
 module NewRelic::Agent
+
   class WorkerLoop
+    include(NewRelic::Agent::Synchronize)
+
     attr_reader :log
 
     def initialize(log = Logger.new(STDERR))
       @tasks = []
-      @mutex = Mutex.new
       @log = log
     end
 
@@ -30,14 +33,14 @@ def add_task(call_period, &task_proc)
         raise ArgumentError, "Invalid Call Period (must be > #{MIN_CALL_PERIOD}): #{call_period}" 
       end
 
-      @mutex.synchronize do 
+      synchronize do 
         @tasks << LoopTask.new(call_period, &task_proc)
       end
     end
 
     private 
       def get_next_task
-        @mutex.synchronize do
+        synchronize do
           return @tasks.inject do |soonest, task|
             (task.next_invocation_time < soonest.next_invocation_time) ? task : soonest
           end
@@ -60,7 +63,7 @@ def run_next_task
 
         begin
           task.execute
-        rescue => e
+        rescue Timeout::Error, StandardError => e
           log.debug "Error running task in Agent Worker Loop: #{e}" 
           log.debug e.backtrace.join("\n")
         end

test/newrelic/agent/mock_http_server.rb

@@ -0,0 +1,40 @@
+require "socket"
+
+
+class MockHTTPServer
+
+  def initialize
+  end
+
+  def start(port, &block)
+      dts = TCPServer.new('localhost', port)
+
+
+
+      t = Thread.start do
+        loop do
+          begin
+            s = dts.accept
+            block.call(s)
+          rescue
+          ensure
+            s.close
+          end
+        end
+
+      end
+    t
+  end
+end
+
+
+server = MockHTTPServer.new
+
+t = server.start(3000) do |s|
+
+  loop do
+    p s.readline
+  end
+end
+
+t.join