Merge pull request #1084 from kobuskc/awsauth

Add support for AWS IAM authentication using boto3

Changelog.rst

@@ -12,6 +12,11 @@ Changelog
     satisfy this.  Reported in #989 (untergeek)
   * Re-use cached response for nodes in most_available_node function. #1086 (tschroeder-zendesk)
 
+**New Features**
+
+  * IAM Credentials can now be retrieved from the environment using the Boto3
+    Credentials provider. #1084 (kobuskc)
+
 5.1.1 (8 June 2017)
 
 **Bug Fixes**

curator/defaults/client_defaults.py

@@ -14,7 +14,9 @@ def config_client():
         Optional('client_key', default=None): Any(None, str, unicode),
         Optional('aws_key', default=None): Any(None, str, unicode),
         Optional('aws_secret_key', default=None): Any(None, str, unicode),
-        Optional('aws_region', default=None): Any(None, str, unicode),
+        Optional('aws_token', default=None): Any(None, str, unicode),
+        Optional('aws_sign_request', default=False): Boolean(),
+        Optional('aws_region'): Any(None, str, unicode),
         Optional('ssl_no_validate', default=False): Boolean(),
         Optional('http_auth', default=None): Any(None, str, unicode),
         Optional('timeout', default=30): All(

curator/utils.py

@@ -718,11 +718,13 @@ def check_master(client, master_only=False):
 
 def get_client(**kwargs):
     """
-    NOTE: AWS IAM parameters `aws_key`, `aws_secret_key`, and `aws_region` are
-    provided for future compatibility, should AWS ES support the
-    ``/_cluster/state/metadata`` endpoint.  So long as this endpoint does not
-    function in AWS ES, the client will not be able to use
-    :class:`curator.indexlist.IndexList`, which is the backbone of Curator 4
+    NOTE: AWS IAM parameters `aws_sign_request` and `aws_region` are
+     provided to facilitate request signing. The credentials will be
+     fetched from the local environment as per the AWS documentation:
+     http://amzn.to/2fRCGCt
+
+    AWS IAM parameters `aws_key`, `aws_secret_key`, and `aws_region` are
+    provided for users that still have their keys included in the Curator config file.
 
     Return an :class:`elasticsearch.Elasticsearch` client object using the
     provided parameters. Any of the keyword arguments the
@@ -748,6 +750,10 @@ def get_client(**kwargs):
         :mod:`requests-aws4auth` python module is installed)
     :arg aws_region: AWS Region (Only used if the :mod:`requests-aws4auth`
         python module is installed)
+    :arg aws_sign_request: Sign request to AWS (Only used if the :mod:`requests-aws4auth`
+         and :mod:`boto3` python modules are installed)
+     :arg aws_region: AWS Region where the cluster exists (Only used if the :mod:`requests-aws4auth`
+         and :mod:`boto3` python modules are installed)
     :arg ssl_no_validate: If `True`, do not validate the certificate
         chain.  This is an insecure option and you will see warnings in the
         log output.
@@ -816,36 +822,60 @@ def get_client(**kwargs):
                     import certifi
                     kwargs['verify_certs'] = True
                     kwargs['ca_certs'] = certifi.where()
-
-    try:
-        from requests_aws4auth import AWS4Auth
-        kwargs['aws_key'] = False if not 'aws_key' in kwargs \
-            else kwargs['aws_key']
-        kwargs['aws_secret_key'] = False if not 'aws_secret_key' in kwargs \
-            else kwargs['aws_secret_key']
-        kwargs['aws_region'] = False if not 'aws_region' in kwargs \
-            else kwargs['aws_region']
-        if kwargs['aws_key'] or kwargs['aws_secret_key'] or kwargs['aws_region']:
-            if not kwargs['aws_key'] and kwargs['aws_secret_key'] \
-                    and kwargs['aws_region']:
+    kwargs['aws_key'] = False if not 'aws_key' in kwargs \
+        else kwargs['aws_key']
+    kwargs['aws_secret_key'] = False if not 'aws_secret_key' in kwargs \
+        else kwargs['aws_secret_key']
+    kwargs['aws_token='] = '' if not 'aws_token' in kwargs \
+        else kwargs['aws_token']
+    kwargs['aws_sign_request'] = False if not 'aws_sign_request' in kwargs \
+        else kwargs['aws_sign_request']
+    kwargs['aws_region'] = False if not 'aws_region' in kwargs \
+        else kwargs['aws_region']
+    if kwargs['aws_key'] or kwargs['aws_secret_key'] or kwargs['aws_sign_request']:
+        if not kwargs['aws_region']:
+            raise MissingArgument(
+                'Missing "aws_region".'
+            )
+        if kwargs['aws_key'] or kwargs['aws_secret_key']:
+            if not (kwargs['aws_key'] and kwargs['aws_secret_key']):
                 raise MissingArgument(
-                    'Missing one or more of "aws_key", "aws_secret_key", '
-                    'or "aws_region".'
+                    'Missing AWS Access Key or AWS Secret Key'
                 )
+    if kwargs['aws_sign_request']:
+        try:
+            from boto3 import session
+            from botocore import exceptions as botoex
+        # We cannot get credentials without the boto3 library, so we cannot continue
+        except ImportError as e:
+            logger.debug('Failed to import a module: %s' % e)
+            raise ImportError('Failed to import a module: %s' % e)
+        try:
+            session = session.Session()
+            credentials = session.get_credentials()
+            kwargs['aws_key'] = credentials.access_key
+            kwargs['aws_secret_key'] = credentials.secret_key
+            kwargs['aws_token'] = credentials.token
+        # If an attribute doesn't exist, we were not able to retrieve credentials as expected so we can't continue
+        except AttributeError:
+            logger.debug('Unable to locate AWS credentials')
+            raise botoex.NoCredentialsError
+    try:
+        from requests_aws4auth import AWS4Auth
+        if kwargs['aws_key']:
             # Override these kwargs
             kwargs['use_ssl'] = True
             kwargs['verify_certs'] = True
             kwargs['connection_class'] = elasticsearch.RequestsHttpConnection
             kwargs['http_auth'] = (
                 AWS4Auth(
                     kwargs['aws_key'], kwargs['aws_secret_key'],
-                    kwargs['aws_region'], 'es')
+                    kwargs['aws_region'], 'es', session_token=kwargs['aws_token'])
             )
         else:
             logger.debug('"requests_aws4auth" module present, but not used.')
     except ImportError:
         logger.debug('Not using "requests_aws4auth" python module to connect.')
-
     if master_only:
         if len(kwargs['hosts']) > 1:
             raise ConfigurationError(

docs/asciidoc/configuration.asciidoc

@@ -382,6 +382,9 @@ IMPORTANT: You must set your <<hosts,hosts>> to the proper hostname _with_ port.
     It may not work setting <<port,port>> and <<hosts,hosts>> to only a host
     name due to the different connection module used.
 
+IMPORTANT: This feature may be deprecated in a future release. You should consider
+    using <<aws_sign_request,aws_sign_request>> instead.
+
 [[aws_secret_key]]
 === aws_secret_key
 
@@ -401,6 +404,9 @@ IMPORTANT: You must set your <<hosts,hosts>> to the proper hostname _with_ port.
     It may not work setting <<port,port>> and <<hosts,hosts>> to only a host
     name due to the different connection module used.
 
+IMPORTANT: This feature may be deprecated in a future release. You should consider
+    using <<aws_sign_request,aws_sign_request>> instead.
+
 [[aws_region]]
 === aws_region
 
@@ -420,6 +426,39 @@ IMPORTANT: You must set your <<hosts,hosts>> to the proper hostname _with_ port.
     It may not work setting <<port,port>> and <<hosts,hosts>> to only a host
     name due to the different connection module used.
 
+[[aws_sign_request]]
+=== aws_sign_request
+
+WARNING: This feature has not been fully tested and should be considered BETA.
+
+WARNING: If installing via `pip`, this setting will not work unless the `requests-aws4auth`
+    and `boto3` Python modules have been manually installed first.
+
+WARNING: Credentials found in your environment will replace the data specified in
+    <<aws_key,aws_key>> and <<aws_secret_key,aws_secret_key>>
+
+This should be `True` if you want your requests to be signed with credentials retrieved
+    from your environment. The order in which credentials will be searched for is:
+
+    . Environment variables
+    . Shared credential file (~/.aws/credentials)
+    . AWS config file (~/.aws/config)
+    . Boto2 config file (/etc/boto.cfg and ~/.boto)
+    . Instance metadata service on an Amazon EC2 instance that has an IAM role configured.
+
+The default value is `False`.
+
+[source,sh]
+-----------
+aws_sign_request: True
+-----------
+
+IMPORTANT: You must set your <<hosts,hosts>> to the proper hostname _with_ port.
+    It may not work setting <<port,port>> and <<hosts,hosts>> to only a host
+    name due to the different connection module used.
+
+
+
 [[ssl_no_validate]]
 === ssl_no_validate
 

unix_packages/build_official_package.sh

@@ -141,6 +141,7 @@ tar zxf ${FILE}
 
 ${PIPBIN} install -U --user setuptools
 ${PIPBIN} install -U --user requests_aws4auth
+${PIPBIN} install -U --user boto3
 if [ "${CX_VER}" != "$(${PIPBIN} list | grep cx | awk '{print $2}' | tr -d '()')" ]; then
   cd ${WORKDIR}
   rm -rf ${CX_PATH}

unix_packages/build_package_from_source.sh

@@ -138,6 +138,7 @@ fi
 
 ${PIPBIN} install -U --user setuptools
 ${PIPBIN} install -U --user requests_aws4auth
+${PIPBIN} install -U --user boto3
 if [ "${CX_VER}" != "$(${PIPBIN} list | grep cx | awk '{print $2}' | tr -d '()')" ]; then
   cd ${WORKDIR}
   rm -rf ${CX_PATH}