add support for api_key authentication

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.7.0
+  - Added api_key support [#131](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/131)
+
 ## 4.6.2
   - Added scroll clearing and better handling of scroll expiration [#128](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/128)
 

docs/index.asciidoc

@@ -68,6 +68,16 @@ Further documentation describing this syntax can be found
 https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here].
 
 
+[id="plugins-{type}s-{plugin}-auth"]
+==== Authentication
+
+Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options:
+
+* <<plugins-{type}s-{plugin}-user>> AND <<plugins-{type}s-{plugin}-password>>
+* <<plugins-{type}s-{plugin}-cloud_auth>>
+* <<plugins-{type}s-{plugin}-api_key>>
+
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Input Configuration Options
 
@@ -76,6 +86,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
@@ -100,6 +111,16 @@ input plugins.
 
 &nbsp;
 
+[id="plugins-{type}s-{plugin}-api_key"]
+===== `api_key`
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Authenticate using Elasticsearch API key. Note that this option also requires enabling the `ssl` option.
+
+Format is `id:api_key` where `id` and `api_key` are as returned by the Elasticsearch https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key API].
+
 [id="plugins-{type}s-{plugin}-ca_file"]
 ===== `ca_file` 
 
@@ -315,4 +336,4 @@ empty string authentication will be disabled.
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 
-:default_codec!:
+:default_codec!:

lib/logstash/inputs/elasticsearch.rb

@@ -70,11 +70,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # Port defaults to 9200
   config :hosts, :validate => :array
 
-  # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
-  #
-  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
-  config :cloud_id, :validate => :string
-
   # The index or alias to search.
   config :index, :validate => :string, :default => "logstash-*"
 
@@ -140,11 +135,20 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # Basic Auth - password
   config :password, :validate => :password
 
+  # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+  #
+  # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
+  config :cloud_id, :validate => :string
+
   # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
   #
   # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
   config :cloud_auth, :validate => :password
 
+  # Authenticate using Elasticsearch API key.
+  # format is id:api_key (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key])
+  config :api_key, :validate => :password
+
   # Set the address of a forward HTTP proxy.
   config :proxy, :validate => :uri_or_empty
 
@@ -177,28 +181,17 @@ def register
       @slices < 1 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`")
     end
 
-    transport_options = {}
-
+    validate_authentication
     fill_user_password_from_cloud_auth
+    fill_hosts_from_cloud_id
 
-    if @user && @password
-      token = Base64.strict_encode64("#{@user}:#{@password.value}")
-      transport_options[:headers] = { :Authorization => "Basic #{token}" }
-    end
 
-    fill_hosts_from_cloud_id
-    @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
+    transport_options = {:headers => {}}
+    transport_options[:headers].merge!(setup_basic_auth(user, password))
+    transport_options[:headers].merge!(setup_api_key(api_key))
 
-    hosts = if @ssl
-      @hosts.map do |h|
-        host, port = h.split(":")
-        { :host => host, :scheme => 'https', :port => port }
-      end
-    else
-      @hosts
-    end
-    ssl_options = { :ssl  => true, :ca_file => @ca_file } if @ssl && @ca_file
-    ssl_options ||= {}
+    hosts = setup_hosts
+    ssl_options = setup_ssl
 
     @logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
 
@@ -351,25 +344,67 @@ def hosts_default?(hosts)
     hosts.nil? || ( hosts.is_a?(Array) && hosts.empty? )
   end
 
-  def fill_hosts_from_cloud_id
-    return unless @cloud_id
+  def validate_authentication
+    authn_options = 0
+    authn_options += 1 if @cloud_auth
+    authn_options += 1 if (@api_key && @api_key.value)
+    authn_options += 1 if (@user || (@password && @password.value))
 
-    if @hosts && !hosts_default?(@hosts)
-      raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+    if authn_options > 1
+      raise LogStash::ConfigurationError, 'Multiple authentication options are specified, please only use one of user/password, cloud_auth or api_key'
     end
-    @hosts = parse_host_uri_from_cloud_id(@cloud_id)
+
+    if @api_key && @api_key.value && @ssl != true
+      raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl => true` option")
+    end
+  end
+
+  def setup_ssl
+    @ssl && @ca_file ? { :ssl  => true, :ca_file => @ca_file } : {}
+  end
+
+  def setup_hosts
+    @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
+    if @ssl
+      @hosts.map do |h|
+        host, port = h.split(":")
+        { :host => host, :scheme => 'https', :port => port }
+      end
+    else
+      @hosts
+    end
+  end
+
+  def setup_basic_auth(user, password)
+    return {} unless user && password && password.value
+
+    token = ::Base64.strict_encode64("#{user}:#{password.value}")
+    { Authorization: "Basic #{token}" }
+  end
+
+  def setup_api_key(api_key)
+    return {} unless (api_key && api_key.value)
+
+    token = ::Base64.strict_encode64(api_key.value)
+    { Authorization: "ApiKey #{token}" }
   end
 
   def fill_user_password_from_cloud_auth
     return unless @cloud_auth
 
-    if @user || @password
-      raise LogStash::ConfigurationError, 'Both cloud_auth and user/password specified, please only use one.'
-    end
     @user, @password = parse_user_password_from_cloud_auth(@cloud_auth)
     params['user'], params['password'] = @user, @password
   end
 
+  def fill_hosts_from_cloud_id
+    return unless @cloud_id
+
+    if @hosts && !hosts_default?(@hosts)
+      raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
+    end
+    @hosts = parse_host_uri_from_cloud_id(@cloud_id)
+  end
+
   def parse_host_uri_from_cloud_id(cloud_id)
     begin # might not be available on older LS
       require 'logstash/util/cloud_setting_id'

logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.6.2'
+  s.version         = '4.7.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/inputs/elasticsearch_spec.rb

@@ -583,7 +583,37 @@ def synchronize_method!(object, method_name)
         let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
 
         it "should fail" do
-          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth and user/
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
+        end
+      end
+    end if LOGSTASH_VERSION > '6.0'
+
+    describe "api_key" do
+      context "without ssl" do
+        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar') }) }
+
+        it "should fail" do
+          expect { plugin.register }.to raise_error LogStash::ConfigurationError, /api_key authentication requires SSL\/TLS/
+        end
+      end
+
+      context "with ssl" do
+        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
+
+        it "should set authorization" do
+          plugin.register
+          client = plugin.send(:client)
+          auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+
+          expect( auth_header ).to eql "ApiKey #{Base64.strict_encode64('foo:bar')}"
+        end
+
+        context 'user also set' do
+          let(:config) { super.merge({ 'api_key' => 'foo:bar', 'user' => 'another' }) }
+
+          it "should fail" do
+            expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
+          end
         end
       end
     end if LOGSTASH_VERSION > '6.0'