Add "Connection Reset by peer" unknown exceptions to retryable except…

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 5.4.0
+  - Introduce retryable unknown exceptions for "connection reset by peer" and "timeout" [#127](https://github.com/logstash-plugins/logstash-output-http/pull/127)
+
 ## 5.3.0
   - Feat: support ssl_verification_mode option [#126](https://github.com/logstash-plugins/logstash-output-http/pull/126)
 

lib/logstash/outputs/http.rb

@@ -23,6 +23,12 @@ class LogStash::Outputs::Http < LogStash::Outputs::Base
     ::Manticore::SocketTimeout
   ]
 
+  RETRYABLE_UNKNOWN_EXCEPTION_STRINGS = [
+    /Connection reset by peer/i,
+    /Read Timed out/i
+  ]
+
+
   # This output lets you send events to a
   # generic HTTP(S) endpoint
   #
@@ -295,7 +301,16 @@ def retryable_response?(response)
   end
 
   def retryable_exception?(exception)
-    RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me) }
+    retryable_manticore_exception?(exception) || retryable_unknown_exception?(exception)
+  end
+
+  def retryable_manticore_exception?(exception)
+    RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me)}
+  end
+
+  def retryable_unknown_exception?(exception)
+    exception.is_a?(::Manticore::UnknownException) &&
+        RETRYABLE_UNKNOWN_EXCEPTION_STRINGS.any? { |snippet| exception.message =~ snippet }
   end
 
   # This is split into a separate method mostly to help testing

logstash-output-http.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-http'
-  s.version         = '5.3.0'
+  s.version         = '5.4.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Sends events to a generic HTTP or HTTPS endpoint"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/outputs/http_spec.rb

@@ -126,6 +126,44 @@ def start_app_and_wait(app, opts = {})
   let(:method) { "post" }
 
   shared_examples("verb behavior") do |method|
+
+    shared_examples("failure log behaviour") do
+      it "logs failure" do
+        expect(subject).to have_received(:log_failure).with(any_args)
+      end
+
+      it "does not log headers" do
+        expect(subject).to have_received(:log_failure).with(anything, hash_not_including(:headers))
+      end
+
+      it "does not log the message body" do
+        expect(subject).to have_received(:log_failure).with(anything, hash_not_including(:body))
+      end
+
+      context "with debug log level" do
+        before :all do
+          @current_log_level = LogStash::Logging::Logger.get_logging_context.get_root_logger.get_level.to_s.downcase
+          LogStash::Logging::Logger.configure_logging "debug"
+        end
+        after :all do
+          LogStash::Logging::Logger.configure_logging @current_log_level
+        end
+
+        it "logs a failure" do
+          expect(subject).to have_received(:log_failure).with(any_args)
+        end
+
+        it "logs headers" do
+          expect(subject).to have_received(:log_failure).with(anything, hash_including(:headers))
+        end
+
+        it "logs the body" do
+          expect(subject).to have_received(:log_failure).with(anything, hash_including(:body))
+        end
+      end
+
+    end
+
     let(:verb_behavior_config) { {"url" => url, "http_method" => method, "pool_max" => 1} }
     subject { LogStash::Outputs::Http.new(verb_behavior_config) }
 
@@ -213,44 +251,96 @@ def start_app_and_wait(app, opts = {})
       end
     end
 
-    context "on exception" do
+    context "on retryable unknown exception" do
       before :each do
-        allow(subject.client).to receive(:send).and_raise RuntimeError
+        raised = false
+        original_method = subject.client.method(:send)
+        allow(subject).to receive(:send_event).and_call_original
+        expect(subject.client).to receive(:send) do |*args|
+          unless raised
+            raised = true
+            raise ::Manticore::UnknownException.new("Read timed out")
+          end
+          original_method.call(args)
+        end
         subject.multi_receive([event])
       end
 
-      it "should not log headers" do
-        expect(subject).to have_received(:log_failure).with(anything, hash_not_including(:headers))
-      end
+      include_examples("failure log behaviour")
 
-      it "should not log the body" do
-        expect(subject).to have_received(:log_failure).with(anything, hash_not_including(:body))
+      it "retries" do
+        expect(subject).to have_received(:send_event).exactly(2).times
       end
+    end
 
-      context "with debug log level" do
-        before :all do
-          @current_log_level = LogStash::Logging::Logger.get_logging_context.get_root_logger.get_level.to_s.downcase
-          LogStash::Logging::Logger.configure_logging "debug"
-        end
-        after :all do
-          LogStash::Logging::Logger.configure_logging @current_log_level
+    context "on non-retryable unknown exception" do
+      before :each do
+        raised = false
+        original_method = subject.client.method(:send)
+        allow(subject).to receive(:send_event).and_call_original
+        expect(subject.client).to receive(:send) do |*args|
+          unless raised
+            raised = true
+            raise ::Manticore::UnknownException.new("broken")
+          end
+          original_method.call(args)
         end
+        subject.multi_receive([event])
+      end
 
-        it "should log a failure" do
-          expect(subject).to have_received(:log_failure).with(any_args)
-        end
+      include_examples("failure log behaviour")
 
-        it "should not log headers" do
-          expect(subject).to have_received(:log_failure).with(anything, hash_including(:headers))
+      it "does not retry" do
+        expect(subject).to have_received(:send_event).exactly(1).times
+      end
+    end
+
+    context "on non-retryable exception" do
+      before :each do
+        raised = false
+        original_method = subject.client.method(:send)
+        allow(subject).to receive(:send_event).and_call_original
+        expect(subject.client).to receive(:send) do |*args|
+          unless raised
+            raised = true
+            raise RuntimeError.new("broken")
+          end
+          original_method.call(args)
         end
+        subject.multi_receive([event])
+      end
 
-        it "should not log the body" do
-          expect(subject).to have_received(:log_failure).with(anything, hash_including(:body))
+      include_examples("failure log behaviour")
+
+      it "does not retry" do
+        expect(subject).to have_received(:send_event).exactly(1).times
+      end
+    end
+
+    context "on retryable exception" do
+      before :each do
+        raised = false
+        original_method = subject.client.method(:send)
+        allow(subject).to receive(:send_event).and_call_original
+        expect(subject.client).to receive(:send) do |*args|
+          unless raised
+            raised = true
+            raise ::Manticore::Timeout.new("broken")
+          end
+          original_method.call(args)
         end
+        subject.multi_receive([event])
       end
+
+      it "retries" do
+        expect(subject).to have_received(:send_event).exactly(2).times
+      end
+
+      include_examples("failure log behaviour")
     end
   end
 
+
   LogStash::Outputs::Http::VALID_METHODS.each do |method|
     context "when using '#{method}'" do
       include_examples("verb behavior", method)