Merge pull request #445 from /issues/435

Fixes #435 - better handling of Throttling

awslimitchecker/checker.py

@@ -203,20 +203,22 @@ def _check_python_version(self):
             warnings.warn(
                 'awslimitchecker has detected that it is running under Python '
                 '2.7. This will no longer be supported as of January 1, 2020. '
-                'Please upgrade to Python 3.5 or newer. Please see the '
-                'changelog for awslimitchecker version 8.0.0 at <https://'
-                'awslimitchecker.readthedocs.io/en/latest/changes.html>'
-                'for further details.',
+                'Please update to a newer Python version (>= 3.5) or switch '
+                'to running via the official Docker image. For further '
+                'information, please see the awslimitchecker 8.0.0 changelog '
+                'at <https://awslimitchecker.readthedocs.io/en/latest/changes.'
+                'html#changelog-8-0-0>',
                 PendingDeprecationWarning
             )
         elif sys.version_info[:2] == (3, 4):  # nocoverage
             warnings.warn(
                 'awslimitchecker has detected that it is running under Python '
                 '3.4. This will no longer be supported as of January 1, 2020. '
-                'Please upgrade to Python 3.5 or newer. Please see the '
-                'changelog for awslimitchecker version 8.0.0 at <https://'
-                'awslimitchecker.readthedocs.io/en/latest/changes.html>'
-                'for further details.',
+                'Please update to a newer Python version (>= 3.5) or switch '
+                'to running via the official Docker image. For further '
+                'information, please see the awslimitchecker 8.0.0 changelog '
+                'at <https://awslimitchecker.readthedocs.io/en/latest/changes.'
+                'html#changelog-8-0-0>',
                 PendingDeprecationWarning
             )
         elif (
@@ -228,7 +230,10 @@ def _check_python_version(self):
                 '%d.%d. This version has reached end-of-life and is no longer '
                 'supported by awslimitchecker, and may not function correctly. '
                 'Please update to a newer Python version (>= 3.5) or switch '
-                'to running via the official Docker image.'
+                'to running via the official Docker image. For further '
+                'information, please see the awslimitchecker 8.0.0 changelog '
+                'at <https://awslimitchecker.readthedocs.io/en/latest/changes.'
+                'html#changelog-8-0-0>'
                 '' % (sys.version_info[0], sys.version_info[1]),
                 DeprecationWarning
             )

awslimitchecker/connectable.py

@@ -37,8 +37,10 @@
 ################################################################################
 """
 
+import os
 import logging
 import boto3
+from botocore.config import Config
 
 logger = logging.getLogger(__name__)
 
@@ -66,12 +68,38 @@ def __init__(self, creds_dict):
 
 
 class Connectable(object):
-
     """
     Mix-in helper class for connecting to AWS APIs. Centralizes logic of
     connecting via regions and/or STS.
     """
 
+    @property
+    def _max_retries_config(self):
+        """
+        If a ``BOTO_MAX_RETRIES_<self.api_name>`` environment variable is set,
+        return a new ``botocore.config.Config`` instance using that number
+        as the retries max_attempts value.
+
+        :rtype: ``botocore.config.Config`` or None
+        """
+        key = 'BOTO_MAX_RETRIES_%s' % self.api_name
+        if key not in os.environ:
+            return None
+        try:
+            max_retries = int(os.environ[key])
+        except Exception:
+            logger.error(
+                'ERROR: Found "%s" environment variable, but unable to '
+                'parse value "%s" to an integer.', key, os.environ[key]
+            )
+            return None
+        logger.debug(
+            'Setting explicit botocore retry config with max_attempts=%d '
+            'for "%s" API based on %s environment variable.',
+            max_retries, self.api_name, key
+        )
+        return Config(retries={'max_attempts': max_retries})
+
     def connect(self):
         """
         Connect to an AWS API via boto3 low-level client and set ``self.conn``
@@ -84,7 +112,9 @@ def connect(self):
         """
         if self.conn is not None:
             return
-        kwargs = self._boto3_connection_kwargs
+        kwargs = dict(self._boto3_connection_kwargs)
+        if self._max_retries_config is not None:
+            kwargs['config'] = self._max_retries_config
         self.conn = boto3.client(self.api_name, **kwargs)
         logger.info("Connected to %s in region %s",
                     self.api_name, self.conn._client_config.region_name)
@@ -102,7 +132,9 @@ def connect_resource(self):
         """
         if self.resource_conn is not None:
             return
-        kwargs = self._boto3_connection_kwargs
+        kwargs = dict(self._boto3_connection_kwargs)
+        if self._max_retries_config is not None:
+            kwargs['config'] = self._max_retries_config
         self.resource_conn = boto3.resource(self.api_name, **kwargs)
         logger.info("Connected to %s (resource) in region %s", self.api_name,
                     self.resource_conn.meta.client._client_config.region_name)

awslimitchecker/tests/test_connectable.py

@@ -40,6 +40,7 @@
 from awslimitchecker.connectable import Connectable, ConnectableCredentials
 from datetime import datetime
 import sys
+import os
 
 # https://code.google.com/p/mock/issues/detail?id=249
 # py>=3.4 should use unittest.mock not the mock package on pypi
@@ -75,6 +76,54 @@ def __init__(self, account_id=None, account_role=None, region=None,
         self.profile_name = profile_name
 
 
+class TestMaxRetriesConfig(object):
+
+    @patch.dict(
+        os.environ,
+        {'BOTO_MAX_RETRIES_myapi': '10'},
+        clear=True
+    )
+    def test_happy_path(self):
+        mock_conf = Mock()
+        cls = ConnectableTester()
+        cls.api_name = 'myapi'
+        with patch('%s.Config' % pbm) as m_conf:
+            m_conf.return_value = mock_conf
+            res = cls._max_retries_config
+        assert res == mock_conf
+        assert m_conf.mock_calls == [call(retries={'max_attempts': 10})]
+
+    @patch.dict(
+        os.environ,
+        {},
+        clear=True
+    )
+    def test_env_var_not_set(self):
+        mock_conf = Mock()
+        cls = ConnectableTester()
+        cls.api_name = 'myapi'
+        with patch('%s.Config' % pbm) as m_conf:
+            m_conf.return_value = mock_conf
+            res = cls._max_retries_config
+        assert res is None
+        assert m_conf.mock_calls == []
+
+    @patch.dict(
+        os.environ,
+        {'BOTO_MAX_RETRIES_myapi': 'hello'},
+        clear=True
+    )
+    def test_cant_parse_int(self):
+        mock_conf = Mock()
+        cls = ConnectableTester()
+        cls.api_name = 'myapi'
+        with patch('%s.Config' % pbm) as m_conf:
+            m_conf.return_value = mock_conf
+            res = cls._max_retries_config
+        assert res is None
+        assert m_conf.mock_calls == []
+
+
 class Test_Connectable(object):
 
     def test_connect(self):
@@ -92,8 +141,12 @@ def test_connect(self):
             mock_kwargs.return_value = kwargs
             with patch('%s.logger' % pbm) as mock_logger:
                 with patch('%s.boto3.client' % pbm) as mock_client:
-                    mock_client.return_value = mock_conn
-                    cls.connect()
+                    with patch(
+                        '%s._max_retries_config' % pb, new_callable=PropertyMock
+                    ) as m_mrc:
+                        m_mrc.return_value = None
+                        mock_client.return_value = mock_conn
+                        cls.connect()
         assert mock_kwargs.mock_calls == [call()]
         assert mock_logger.mock_calls == [
             call.info("Connected to %s in region %s",
@@ -107,6 +160,46 @@ def test_connect(self):
                 bar='barval'
             )
         ]
+        assert m_mrc.mock_calls == [call()]
+        assert cls.conn == mock_client.return_value
+
+    def test_connect_with_retries(self):
+        mock_conn = Mock()
+        mock_cc = Mock()
+        type(mock_cc).region_name = 'myregion'
+        type(mock_conn)._client_config = mock_cc
+
+        cls = ConnectableTester()
+        cls.api_name = 'myapi'
+        kwargs = {'foo': 'fooval', 'bar': 'barval'}
+        mock_conf = Mock()
+
+        with patch('%s._boto3_connection_kwargs' % pb,
+                   new_callable=PropertyMock, create=True) as mock_kwargs:
+            mock_kwargs.return_value = kwargs
+            with patch('%s.logger' % pbm) as mock_logger:
+                with patch('%s.boto3.client' % pbm) as mock_client:
+                    with patch(
+                        '%s._max_retries_config' % pb, new_callable=PropertyMock
+                    ) as m_mrc:
+                        m_mrc.return_value = mock_conf
+                        mock_client.return_value = mock_conn
+                        cls.connect()
+        assert mock_kwargs.mock_calls == [call()]
+        assert mock_logger.mock_calls == [
+            call.info("Connected to %s in region %s",
+                      'myapi',
+                      'myregion')
+        ]
+        assert mock_client.mock_calls == [
+            call(
+                'myapi',
+                foo='fooval',
+                bar='barval',
+                config=mock_conf
+            )
+        ]
+        assert m_mrc.mock_calls == [call(), call()]
         assert cls.conn == mock_client.return_value
 
     def test_connect_again(self):
@@ -125,11 +218,17 @@ def test_connect_again(self):
             mock_kwargs.return_value = kwargs
             with patch('%s.logger' % pbm) as mock_logger:
                 with patch('%s.boto3.client' % pbm) as mock_client:
-                    mock_client.return_value = mock_conn
-                    cls.connect()
+                    with patch(
+                            '%s._max_retries_config' % pb,
+                            new_callable=PropertyMock
+                    ) as m_mrc:
+                        m_mrc.return_value = None
+                        mock_client.return_value = mock_conn
+                        cls.connect()
         assert mock_kwargs.mock_calls == []
         assert mock_logger.mock_calls == []
         assert mock_client.mock_calls == []
+        assert m_mrc.mock_calls == []
         assert cls.conn == mock_conn
 
     def test_connect_resource(self):
@@ -151,8 +250,13 @@ def test_connect_resource(self):
             mock_kwargs.return_value = kwargs
             with patch('%s.logger' % pbm) as mock_logger:
                 with patch('%s.boto3.resource' % pbm) as mock_resource:
-                    mock_resource.return_value = mock_conn
-                    cls.connect_resource()
+                    with patch(
+                            '%s._max_retries_config' % pb,
+                            new_callable=PropertyMock
+                    ) as m_mrc:
+                        m_mrc.return_value = None
+                        mock_resource.return_value = mock_conn
+                        cls.connect_resource()
         assert mock_kwargs.mock_calls == [call()]
         assert mock_logger.mock_calls == [
             call.info("Connected to %s (resource) in region %s",
@@ -166,6 +270,51 @@ def test_connect_resource(self):
                 bar='barval'
             )
         ]
+        assert m_mrc.mock_calls == [call()]
+        assert cls.resource_conn == mock_resource.return_value
+
+    def test_connect_resource_with_max_retries(self):
+        mock_conn = Mock()
+        mock_meta = Mock()
+        mock_client = Mock()
+        mock_cc = Mock()
+        type(mock_cc).region_name = 'myregion'
+        type(mock_client)._client_config = mock_cc
+        type(mock_meta).client = mock_client
+        type(mock_conn).meta = mock_meta
+
+        cls = ConnectableTester()
+        cls.api_name = 'myapi'
+        kwargs = {'foo': 'fooval', 'bar': 'barval'}
+        mock_conf = Mock()
+
+        with patch('%s._boto3_connection_kwargs' % pb,
+                   new_callable=PropertyMock, create=True) as mock_kwargs:
+            mock_kwargs.return_value = kwargs
+            with patch('%s.logger' % pbm) as mock_logger:
+                with patch('%s.boto3.resource' % pbm) as mock_resource:
+                    with patch(
+                            '%s._max_retries_config' % pb,
+                            new_callable=PropertyMock
+                    ) as m_mrc:
+                        m_mrc.return_value = mock_conf
+                        mock_resource.return_value = mock_conn
+                        cls.connect_resource()
+        assert mock_kwargs.mock_calls == [call()]
+        assert mock_logger.mock_calls == [
+            call.info("Connected to %s (resource) in region %s",
+                      'myapi',
+                      'myregion')
+        ]
+        assert mock_resource.mock_calls == [
+            call(
+                'myapi',
+                foo='fooval',
+                bar='barval',
+                config=mock_conf
+            )
+        ]
+        assert m_mrc.mock_calls == [call(), call()]
         assert cls.resource_conn == mock_resource.return_value
 
     def test_connect_resource_again(self):
@@ -189,11 +338,17 @@ def test_connect_resource_again(self):
             mock_kwargs.return_value = kwargs
             with patch('%s.logger' % pbm) as mock_logger:
                 with patch('%s.boto3.resource' % pbm) as mock_resource:
-                    mock_resource.return_value = mock_conn
-                    cls.connect_resource()
+                    with patch(
+                            '%s._max_retries_config' % pb,
+                            new_callable=PropertyMock
+                    ) as m_mrc:
+                        m_mrc.return_value = None
+                        mock_resource.return_value = mock_conn
+                        cls.connect_resource()
         assert mock_kwargs.mock_calls == []
         assert mock_logger.mock_calls == []
         assert mock_resource.mock_calls == []
+        assert m_mrc.mock_calls == []
         assert cls.resource_conn == mock_conn
 
 

docs/source/cli_usage.rst

@@ -651,3 +651,14 @@ Partitions and Trusted Advisor Regions
 ++++++++++++++++++++++++++++++++++++++
 
 awslimitchecker currently supports operating against non-standard `partitions <https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html>`_, such as GovCloud and AWS China (Beijing). Partition names, as seen in the ``partition`` field of ARNs, can be specified with the ``--role-partition`` option to awslimitchecker, like ``--role-partition=aws-cn`` for the China (Beijing) partition. Similarly, the region name to use for the ``support`` API for Trusted Advisor can be specified with the ``--ta-api-region`` option, like ``--ta-api-region=us-gov-west-1``.
+
+.. _cli_usage.throttling:
+
+Handling Throttling and Rate Limiting
++++++++++++++++++++++++++++++++++++++
+
+In some very large and busy AWS accounts, from time to time awslimitchecker might die on unhandled ``Throttling`` or ``RateExceeded`` exceptions. botocore, the underlying low-level AWS API client library that we use, automatically catches these exceptions and retries them up to a per-AWS-API default number of times (generally four for most APIs) with an exponential backoff. In very busy accounts, it may be desirable to increase the default number of retries.
+
+This can be accomplished on a per-API basis (where the API name is the ``service_name`` that would be sent to :py:meth:`boto3.session.Session.client` and is set as the :py:attr:`~.awslimitchecker.services.base._AwsService.api_name` attribute on each :py:class:`~.awslimitchecker.services.base._AwsService` subclass) by setting an environment variable ``BOTO_MAX_RETRIES_<api_name>`` to the maximum number of attempts you'd like for that service.
+
+For example, if you have issues with rate limiting of the ``cloudformation:DescribeStacks`` still failing after the default of four attempts, and you'd like to use ten (10) attempts instead, you could ``export BOTO_MAX_RETRIES_cloudformation=10`` before running ``awslimitchecker``.

docs/source/cli_usage.rst.template

@@ -318,3 +318,14 @@ Partitions and Trusted Advisor Regions
 ++++++++++++++++++++++++++++++++++++++
 
 awslimitchecker currently supports operating against non-standard `partitions <https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html>`_, such as GovCloud and AWS China (Beijing). Partition names, as seen in the ``partition`` field of ARNs, can be specified with the ``--role-partition`` option to awslimitchecker, like ``--role-partition=aws-cn`` for the China (Beijing) partition. Similarly, the region name to use for the ``support`` API for Trusted Advisor can be specified with the ``--ta-api-region`` option, like ``--ta-api-region=us-gov-west-1``.
+
+.. _cli_usage.throttling:
+
+Handling Throttling and Rate Limiting
++++++++++++++++++++++++++++++++++++++
+
+In some very large and busy AWS accounts, from time to time awslimitchecker might die on unhandled ``Throttling`` or ``RateExceeded`` exceptions. botocore, the underlying low-level AWS API client library that we use, automatically catches these exceptions and retries them up to a per-AWS-API default number of times (generally four for most APIs) with an exponential backoff. In very busy accounts, it may be desirable to increase the default number of retries.
+
+This can be accomplished on a per-API basis (where the API name is the ``service_name`` that would be sent to :py:meth:`boto3.session.Session.client` and is set as the :py:attr:`~.awslimitchecker.services.base._AwsService.api_name` attribute on each :py:class:`~.awslimitchecker.services.base._AwsService` subclass) by setting an environment variable ``BOTO_MAX_RETRIES_<api_name>`` to the maximum number of attempts you'd like for that service.
+
+For example, if you have issues with rate limiting of the ``cloudformation:DescribeStacks`` still failing after the default of four attempts, and you'd like to use ten (10) attempts instead, you could ``export BOTO_MAX_RETRIES_cloudformation=10`` before running ``awslimitchecker``.

docs/source/python_usage.rst

@@ -261,6 +261,13 @@ To remove the Firehose and EC2 services:
 
     c.remove_services(['Firehose', 'EC2'])
 
+.. _python_usage.throttling:
+
+Handling Throttling and Rate Limiting
++++++++++++++++++++++++++++++++++++++
+
+See :ref:`CLI Usage - Handling Throttling and Rate Limiting <cli_usage.throttling>`; this is handled the same way in Python, though you'd likely set the environment variables using ``os.environ`` instead of exporting them outside of Python.
+
 Logging
 -------