Merge pull request #93 from /issues/84

Fixes #84 - use account attributes for some EC2 limits

awslimitchecker/checker.py

@@ -166,6 +166,8 @@ def get_limits(self, service=None, use_ta=True):
         if use_ta:
             self.ta.update_limits(to_get)
         for sname, cls in to_get.items():
+            if hasattr(cls, '_update_limits_from_api'):
+                cls._update_limits_from_api()
             res[sname] = cls.get_limits()
         return res
 
@@ -200,6 +202,8 @@ def find_usage(self, service=None, use_ta=True):
         if use_ta:
             self.ta.update_limits(to_get)
         for cls in to_get.values():
+            if hasattr(cls, '_update_limits_from_api'):
+                cls._update_limits_from_api()
             logger.debug("Finding usage for service: %s", cls.service_name)
             cls.find_usage()
 
@@ -394,6 +398,8 @@ def check_thresholds(self, service=None, use_ta=True):
         if use_ta:
             self.ta.update_limits(to_get)
         for sname, cls in to_get.items():
+            if hasattr(cls, '_update_limits_from_api'):
+                cls._update_limits_from_api()
             tmp = cls.check_thresholds()
             if len(tmp) > 0:
                 res[sname] = tmp

awslimitchecker/services/ec2.py

@@ -189,7 +189,31 @@ def get_limits(self):
         limits.update(self._get_limits_instances())
         limits.update(self._get_limits_networking())
         self.limits = limits
-        return limits
+        return self.limits
+
+    def _update_limits_from_api(self):
+        """
+        Query EC2's DescribeAccountAttributes API action, and update limits
+        with the quotas returned. Updates ``self.limits``.
+        """
+        self.connect()
+        logger.info("Querying EC2 DescribeAccountAttributes for limits")
+        attribs = boto_query_wrapper(self.conn.describe_account_attributes)
+        for attrib in attribs:
+            aname = attrib.attribute_name
+            val = attrib.attribute_values[0]
+            lname = None
+            if aname == 'max-elastic-ips':
+                lname = 'Elastic IP addresses (EIPs)'
+            elif aname == 'max-instances':
+                lname = 'Running On-Demand EC2 instances'
+            elif aname == 'vpc-max-elastic-ips':
+                lname = 'VPC Elastic IP addresses (EIPs)'
+            elif aname == 'vpc-max-security-groups-per-interface':
+                lname = 'VPC security groups per elastic network interface'
+            if lname is not None:
+                self.limits[lname]._set_api_limit(int(val))
+        logger.debug("Done setting limits from API")
 
     def _get_limits_instances(self):
         """

awslimitchecker/services/newservice.py.example

@@ -107,13 +107,20 @@ class _XXNewServiceXXService(_AwsService):
         )
 
         """
-        # TODO: if the service has an API that can retrieve current limit information
-        # i.e. ``DescribeAccountAttributes``, pass ``limits`` to a new function called
-        # here that retrieves the limits from the API, and updates each AwsLimit object
-        # via its ``._set_api_limit()`` method.
         self.limits = limits
         return limits
 
+    def _update_limits_from_api(self):
+        """
+        Call the service's API action to retrieve limit/quota information, and
+        update AwsLimit objects in ``self.limits`` with this information.
+        """
+        # TODO: if the service has an API that can retrieve current limit information
+        # i.e. ``DescribeAccountAttributes``, call that action here and update each
+        # relevant AwsLimit object (in ``self.limits``) via its ``._set_api_limit()`` method.
+        # ELSE if the service has no API call for this, remove this method.
+        raise NotImplementedException()
+
     def required_iam_permissions(self):
         """
         Return a list of IAM Actions required for this Service to function

awslimitchecker/tests/services/test_ec2.py

@@ -45,6 +45,8 @@
 from boto.ec2.address import Address
 from boto.ec2.networkinterface import NetworkInterface
 from boto.ec2 import connect_to_region
+from boto.ec2.attributes import AccountAttribute
+from boto.resultset import ResultSet
 from awslimitchecker.services.ec2 import _Ec2Service
 from awslimitchecker.limit import AwsLimit
 
@@ -616,3 +618,52 @@ def test_get_limits_networking(self):
             'VPC security groups per elastic network interface',
         ]
         assert sorted(limits.keys()) == sorted(expected)
+
+    def test_update_limits_from_api(self):
+        mock_conn = Mock(spec_set=EC2Connection)
+
+        rs = ResultSet()
+        a1 = AccountAttribute(connection=mock_conn)
+        a1.attribute_name = 'supported-platforms'
+        a1.attribute_values = ['EC2', 'VPC']
+        rs.append(a1)
+        a2 = AccountAttribute(connection=mock_conn)
+        a2.attribute_name = 'vpc-max-security-groups-per-interface'
+        a2.attribute_values = ['5']
+        rs.append(a2)
+        a3 = AccountAttribute(connection=mock_conn)
+        a3.attribute_name = 'max-elastic-ips'
+        a3.attribute_values = ['40']
+        rs.append(a3)
+        a4 = AccountAttribute(connection=mock_conn)
+        a4.attribute_name = 'max-instances'
+        a4.attribute_values = ['400']
+        rs.append(a4)
+        a5 = AccountAttribute(connection=mock_conn)
+        a5.attribute_name = 'vpc-max-elastic-ips'
+        a5.attribute_values = ['200']
+        rs.append(a5)
+        a6 = AccountAttribute(connection=mock_conn)
+        a6.attribute_name = 'default-vpc'
+        a6.attribute_values = ['none']
+        rs.append(a6)
+
+        cls = _Ec2Service(21, 43)
+        cls.conn = mock_conn
+        with patch('awslimitchecker.services.ec2.logger') as mock_logger:
+            with patch('%s.boto_query_wrapper' % self.pbm) as mock_wrapper:
+                mock_wrapper.return_value = rs
+                cls._update_limits_from_api()
+        assert mock_wrapper.mock_calls == [
+            call(mock_conn.describe_account_attributes)
+        ]
+        assert mock_conn.mock_calls == []
+        assert mock_logger.mock_calls == [
+            call.info("Querying EC2 DescribeAccountAttributes for limits"),
+            call.debug('Done setting limits from API')
+        ]
+        assert cls.limits['Elastic IP addresses (EIPs)'].api_limit == 40
+        assert cls.limits['Running On-Demand EC2 instances'].api_limit == 400
+        assert cls.limits['VPC Elastic IP addresses (EIPs)'].api_limit == 200
+        assert cls.limits['VPC security groups per elastic '
+                          'network interface'].api_limit == 5

awslimitchecker/tests/test_checker.py

@@ -58,6 +58,16 @@
     from unittest.mock import patch, call, Mock, DEFAULT
 
 
+class ApiServiceSpec(_AwsService):
+    """
+    Used for Mock's ``spec_set`` parameter to represent classes that have
+    an ``_update_limits_from_api`` method.
+    """
+
+    def _update_limits_from_api(self):
+        pass
+
+
 class TestAwsLimitChecker(object):
 
     def setup(self):
@@ -70,7 +80,7 @@ def setup(self):
         )
 
         self.mock_svc1 = Mock(spec_set=_AwsService)
-        self.mock_svc2 = Mock(spec_set=_AwsService)
+        self.mock_svc2 = Mock(spec_set=ApiServiceSpec)
         self.mock_foo = Mock(spec_set=_AwsService)
         self.mock_bar = Mock(spec_set=_AwsService)
         self.mock_ta = Mock(spec_set=TrustedAdvisor)
@@ -356,6 +366,13 @@ def test_get_limits(self):
                 'SvcBar': self.mock_svc2,
             })
         ]
+        assert self.mock_svc1.mock_calls == [
+            call.get_limits()
+        ]
+        assert self.mock_svc2.mock_calls == [
+            call._update_limits_from_api(),
+            call.get_limits()
+        ]
 
     def test_get_limits_no_ta(self):
         limits = sample_limits()
@@ -364,6 +381,13 @@ def test_get_limits_no_ta(self):
         res = self.cls.get_limits(use_ta=False)
         assert res == limits
         assert self.mock_ta.mock_calls == []
+        assert self.mock_svc1.mock_calls == [
+            call.get_limits()
+        ]
+        assert self.mock_svc2.mock_calls == [
+            call._update_limits_from_api(),
+            call.get_limits()
+        ]
 
     def test_get_limits_service(self):
         limits = sample_limits()
@@ -376,13 +400,35 @@ def test_get_limits_service(self):
                 'SvcFoo': self.mock_svc1,
             })
         ]
+        assert self.mock_svc1.mock_calls == [
+            call.get_limits()
+        ]
+        assert self.mock_svc2.mock_calls == []
+
+    def test_get_limits_service_with_api(self):
+        limits = sample_limits()
+        self.mock_svc1.get_limits.return_value = limits['SvcFoo']
+        self.mock_svc2.get_limits.return_value = limits['SvcBar']
+        res = self.cls.get_limits(service='SvcBar')
+        assert res == {'SvcBar': limits['SvcBar']}
+        assert self.mock_ta.mock_calls == [
+            call.update_limits({
+                'SvcBar': self.mock_svc2,
+            })
+        ]
+        assert self.mock_svc1.mock_calls == []
+        assert self.mock_svc2.mock_calls == [
+            call._update_limits_from_api(),
+            call.get_limits()
+        ]
 
     def test_find_usage(self):
         self.cls.find_usage()
         assert self.mock_svc1.mock_calls == [
             call.find_usage()
         ]
         assert self.mock_svc2.mock_calls == [
+            call._update_limits_from_api(),
             call.find_usage()
         ]
         assert self.mock_ta.mock_calls == [
@@ -398,6 +444,7 @@ def test_find_usage_no_ta(self):
             call.find_usage()
         ]
         assert self.mock_svc2.mock_calls == [
+            call._update_limits_from_api(),
             call.find_usage()
         ]
         assert self.mock_ta.mock_calls == []
@@ -412,6 +459,17 @@ def test_find_usage_service(self):
             call.update_limits({'SvcFoo': self.mock_svc1})
         ]
 
+    def test_find_usage_service_with_api(self):
+        self.cls.find_usage(service='SvcBar')
+        assert self.mock_svc1.mock_calls == []
+        assert self.mock_svc2.mock_calls == [
+            call._update_limits_from_api(),
+            call.find_usage()
+        ]
+        assert self.mock_ta.mock_calls == [
+            call.update_limits({'SvcBar': self.mock_svc2})
+        ]
+
     def test_set_threshold_overrides(self):
         limits = sample_limits()
         limits['SvcFoo']['zz3'] = AwsLimit(
@@ -617,6 +675,13 @@ def test_check_thresholds(self):
                 'SvcBar': self.mock_svc2
             }),
         ]
+        assert self.mock_svc1.mock_calls == [
+            call.check_thresholds()
+        ]
+        assert self.mock_svc2.mock_calls == [
+            call._update_limits_from_api(),
+            call.check_thresholds()
+        ]
 
     def test_check_thresholds_service(self):
         self.mock_svc1.check_thresholds.return_value = {'foo': 'bar'}
@@ -630,6 +695,28 @@ def test_check_thresholds_service(self):
         assert self.mock_ta.mock_calls == [
             call.update_limits({'SvcFoo': self.mock_svc1})
         ]
+        assert self.mock_svc1.mock_calls == [
+            call.check_thresholds()
+        ]
+        assert self.mock_svc2.mock_calls == []
+
+    def test_check_thresholds_service_api(self):
+        self.mock_svc1.check_thresholds.return_value = {'foo': 'bar'}
+        self.mock_svc2.check_thresholds.return_value = {'baz': 'blam'}
+        res = self.cls.check_thresholds(service='SvcBar')
+        assert res == {
+            'SvcBar': {
+                'baz': 'blam',
+            }
+        }
+        assert self.mock_ta.mock_calls == [
+            call.update_limits({'SvcBar': self.mock_svc2})
+        ]
+        assert self.mock_svc1.mock_calls == []
+        assert self.mock_svc2.mock_calls == [
+            call._update_limits_from_api(),
+            call.check_thresholds()
+        ]
 
     def test_check_thresholds_no_ta(self):
         self.mock_svc1.check_thresholds.return_value = {
@@ -646,3 +733,10 @@ def test_check_thresholds_no_ta(self):
             }
         }
         assert self.mock_ta.mock_calls == []
+        assert self.mock_svc1.mock_calls == [
+            call.check_thresholds()
+        ]
+        assert self.mock_svc2.mock_calls == [
+            call._update_limits_from_api(),
+            call.check_thresholds()
+        ]

awslimitchecker/tests/test_utils.py

@@ -414,10 +414,12 @@ def test_dict_missing_params(self):
         ]
         assert mocks['_paginate_resultset'].mock_calls == []
         assert mocks['_paginate_dict'].mock_calls == []
-        assert mocks['logger'].mock_calls == [
-            call.warning("Query returned a dict, but does not have _paginate_"
-                         "dict params set; cannot paginate")
-        ]
+        assert len(mocks['logger'].mock_calls) == 1
+        args = mocks['logger'].warning.mock_calls[0][1]
+        assert len(args) == 1
+        assert args[0].startswith(
+            "Query returned a dict, but does not have _paginate_dict params "
+            "set; cannot paginate (<Mock id='") is True
 
     def test_other_type(self):
         func = Mock()

awslimitchecker/utils.py

@@ -189,7 +189,8 @@ def paginate_query(function_ref, *argv, **kwargs):
             return _paginate_dict(result, function_ref, *argv, **kwargs)
         else:
             logger.warning("Query returned a dict, but does not have "
-                           "_paginate_dict params set; cannot paginate")
+                           "_paginate_dict params set; cannot paginate (" +
+                           str(function_ref) + ")")
             return result
     logger.warning("Query result of type %s cannot be paginated", type(result))
     return result

docs/build_generated_docs.py

@@ -266,7 +266,20 @@ def format_cmd_output(cmd, output, name):
             if len(line) > 100:
                 lines[idx] = line[:100] + ' (...)'
         if len(lines) > 12:
-            lines = lines[:5] + ['(...)'] + lines[-5:]
+            tmp_lines = lines[:5] + ['(...)'] + lines[-5:]
+            if ' -l' not in cmd:
+                lines = tmp_lines
+            else:
+                # find a line that uses a limit from the API
+                api_line = '(...)'
+                for line in lines:
+                    if '(API)' in line:
+                        api_line = line
+                        break
+                if api_line not in tmp_lines:
+                    tmp_lines = lines[:5] + ['(...)'] + [ api_line ]
+                    tmp_lines = tmp_lines + ['(...)'] + lines[-5:]
+            lines = tmp_lines
     for line in lines:
         if line.strip() == '':
             continue