Feat: support ssl_verification_mode => 'full' / 'none'

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 5.3.0
+  - Feat: support ssl_verification_mode option [#126](https://github.com/logstash-plugins/logstash-output-http/pull/126)
+
 ## 5.2.5
   - Reduce amount of default logging on a failed request [#122](https://github.com/logstash-plugins/logstash-output-http/pull/122)
 

docs/index.asciidoc

@@ -66,6 +66,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-retry_non_idempotent>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-retryable_codes>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-socket_timeout>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-ssl_verification_mode>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-truststore_password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-truststore_type>> |<<string,string>>|No
@@ -337,6 +338,22 @@ If encountered as response codes this plugin will retry these requests
 
 Timeout (in seconds) to wait for data on the socket. Default is `10s`
 
+[id="plugins-{type}s-{plugin}-ssl_verification_mode"]
+===== `ssl_verification_mode`
+
+  * Value type is <<string,string>>
+  * Supported values are: `full`, `none`
+  * Default value is `full`
+
+Controls the verification of server certificates.
+The `full` option verifies that the provided certificate is signed by a trusted authority (CA)
+and also that the server’s hostname (or IP address) matches the names identified within the certificate.
+
+The `none` setting performs no verification of the server’s certificate.
+This mode disables many of the security benefits of SSL/TLS and should only be used after cautious consideration.
+It is primarily intended as a temporary diagnostic mechanism when attempting to resolve TLS errors.
+Using `none`  in production environments is strongly discouraged.
+
 [id="plugins-{type}s-{plugin}-truststore"]
 ===== `truststore` 
 

lib/logstash/outputs/http.rb

@@ -138,10 +138,11 @@ def run
   end
 
   def log_retryable_response(response)
+    retry_msg = @retry_failed ? 'will retry' : "won't retry"
     if (response.code == 429)
-      @logger.debug? && @logger.debug("Encountered a 429 response, will retry. This is not serious, just flow control via HTTP")
+      @logger.debug? && @logger.debug("Encountered a 429 response, #{retry_msg}. This is not serious, just flow control via HTTP")
     else
-      @logger.warn("Encountered a retryable HTTP request in HTTP output, will retry", :code => response.code, :body => response.body)
+      @logger.warn("Encountered a retryable HTTP request in HTTP output, #{retry_msg}", :code => response.code, :body => response.body)
     end
   end
 
@@ -299,7 +300,7 @@ def retryable_exception?(exception)
 
   # This is split into a separate method mostly to help testing
   def log_failure(message, opts)
-    @logger.error("[HTTP Output Failure] #{message}", opts)
+    @logger.error(message, opts)
   end
 
   # Format the HTTP body

logstash-output-http.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-http'
-  s.version         = '5.2.5'
+  s.version         = '5.3.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Sends events to a generic HTTP or HTTPS endpoint"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency "logstash-mixin-http_client", ">= 6.0.0", "< 8.0.0"
+  s.add_runtime_dependency "logstash-mixin-http_client", ">= 7.1.0", "< 8.0.0"
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'sinatra'

spec/outputs/http_spec.rb

@@ -3,6 +3,9 @@
 require "logstash/codecs/plain"
 require "thread"
 require "sinatra"
+require "webrick"
+require "webrick/https"
+require 'openssl'
 require_relative "../supports/compressed_requests"
 
 PORT = rand(65535-1024) + 1025
@@ -22,9 +25,20 @@ class TestApp < Sinatra::Base
   # on the fly uncompress gzip content
   use CompressedRequests
 
-  # disable WEBrick logging
+  set :environment, :production
+  set :sessions, false
+
+  @@server_settings = {
+      :AccessLog => [], # disable WEBrick logging
+      :Logger => WEBrick::BasicLog::new(nil, WEBrick::BasicLog::FATAL)
+  }
+
   def self.server_settings
-    { :AccessLog => [], :Logger => WEBrick::BasicLog::new(nil, WEBrick::BasicLog::FATAL) }
+    @@server_settings
+  end
+
+  def self.server_settings=(settings)
+    @@server_settings = settings
   end
 
   def self.multiroute(methods, path, &block)
@@ -72,38 +86,38 @@ def self.retry_fail_count()
   end
 end
 
-RSpec.configure do |config|
+RSpec.configure do
   #http://stackoverflow.com/questions/6557079/start-and-call-ruby-http-server-in-the-same-script
-  def sinatra_run_wait(app, opts)
+  def start_app_and_wait(app, opts = {})
     queue = Queue.new
 
-    t = java.lang.Thread.new(
-      proc do
-        begin
-          app.run!(opts) do |server|
-            queue.push("started")
-          end
-        rescue => e
-          puts "Error in webserver thread #{e}"
-          # ignore
+    Thread.start do
+      begin
+        app.start!({ server: 'WEBrick', port: PORT }.merge opts) do |server|
+          queue.push(server)
         end
+      rescue => e
+        warn "Error starting app: #{e.inspect}" # ignore
       end
-    )
-    t.daemon = true
-    t.start
-    queue.pop # blocks until the run! callback runs
-  end
+    end
 
-  config.before(:suite) do
-    sinatra_run_wait(TestApp, :port => PORT, :server => 'webrick')
-    puts "Test webserver on port #{PORT}"
+    queue.pop # blocks until the start! callback runs
   end
 end
 
 describe LogStash::Outputs::Http do
   # Wait for the async request to finish in this spinlock
   # Requires pool_max to be 1
 
+  before(:all) do
+    @server = start_app_and_wait(TestApp)
+  end
+
+  after(:all) do
+    @server.shutdown # WEBrick::HTTPServer
+    TestApp.stop! rescue nil
+  end
+
   let(:port) { PORT }
   let(:event) {
     LogStash::Event.new({"message" => "hi"})
@@ -398,3 +412,75 @@ def sinatra_run_wait(app, opts)
     end
   end
 end
+
+describe LogStash::Outputs::Http do # different block as we're starting web server with TLS
+
+  @@default_server_settings = TestApp.server_settings.dup
+
+  before do
+    cert, key = WEBrick::Utils.create_self_signed_cert 2048, [["CN", ssl_cert_host]], "Logstash testing"
+    TestApp.server_settings = @@default_server_settings.merge({
+       :SSLEnable       => true,
+       :SSLVerifyClient => OpenSSL::SSL::VERIFY_NONE,
+       :SSLCertificate  => cert,
+       :SSLPrivateKey   => key
+    })
+
+    TestApp.last_request = nil
+
+    @server = start_app_and_wait(TestApp)
+  end
+
+  after do
+    @server.shutdown # WEBrick::HTTPServer
+
+    TestApp.stop! rescue nil
+    TestApp.server_settings = @@default_server_settings
+  end
+
+  let(:ssl_cert_host) { 'localhost' }
+
+  let(:port) { PORT }
+  let(:url) { "https://localhost:#{port}/good" }
+  let(:method) { "post" }
+
+  let(:config) { { "url" => url, "http_method" => method } }
+
+  subject { LogStash::Outputs::Http.new(config) }
+
+  before { subject.register }
+  after  { subject.close }
+
+  let(:last_request) { TestApp.last_request }
+  let(:last_request_body) { last_request.body.read }
+
+  let(:event) { LogStash::Event.new("message" => "hello!") }
+
+  context 'with default (full) verification' do
+
+    let(:config) { super() } # 'ssl_verification_mode' => 'full'
+
+    it "does NOT process the request (due client protocol exception)" do
+      # Manticore's default verification does not accept self-signed certificates!
+      Thread.start do
+        subject.multi_receive [ event ]
+      end
+      sleep 1.5
+
+      expect(last_request).to be nil
+    end
+
+  end
+
+  context 'with verification disabled' do
+
+    let(:config) { super().merge 'ssl_verification_mode' => 'none' }
+
+    it "should process the request" do
+      subject.multi_receive [ event ]
+      expect(last_request_body).to include '"message":"hello!"'
+    end
+
+  end
+
+end