diff --git a/dev_requirements.txt b/dev_requirements.txt index f808f60..a944d5c 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,3 +1,3 @@ -boto3==1.6.4 -botocore==1.9.4 +boto3==1.10.45 +botocore==1.13.45 tox>=2.3.1 diff --git a/limiter/event_processors.py b/limiter/event_processors.py index 95431d5..8300796 100644 --- a/limiter/event_processors.py +++ b/limiter/event_processors.py @@ -1,6 +1,6 @@ #!/bin/bash/env python -import sys import logging +import functools from boto3.dynamodb.conditions import Key from limiter.utils import validate_table_env_fallback from limiter.clients import dynamodb @@ -8,7 +8,7 @@ logger = logging.getLogger() -class ProcessorPredicate(object): +class ProcessorPredicate: """ Determines if an event matches a specific criteria. @@ -89,7 +89,7 @@ def test(self, event): break return result -class EventProcessor(object): +class EventProcessor: """ Validates and extracts the resource id from events. @@ -130,7 +130,7 @@ def test_and_get_id(self, event): """ return None if self.predicate and not self.predicate.test(event) else _reduce_to_path(event, self.id_path) -class EventProcessorManager(object): +class EventProcessorManager: """ Removes non-fungible tokens from DynamoDB represented by termination events. @@ -222,7 +222,7 @@ def process_event(self, event): } ) else: - logger.warn('Could not find a token for resoure %s', resource_id) + logger.warning('Could not find a token for resoure %s', resource_id) self.cache.append(resource_id) def _get_processor(self, event): @@ -287,11 +287,11 @@ def _reduce_to_path(obj, path): str: If the path is valid, otherwise None. """ try: - if isinstance(path, basestring): + if isinstance(path, str): path = path.split('.') - return reduce(lambda x, y: x[y], path, obj) + return functools.reduce(lambda x, y: x[y], path, obj) except Exception: - sys.exc_clear() + pass return None def _build_processor_key(source, type=None): diff --git a/limiter/exceptions.py b/limiter/exceptions.py index da7b2a2..9d6b1c1 100644 --- a/limiter/exceptions.py +++ b/limiter/exceptions.py @@ -1,14 +1,11 @@ class CapacityExhaustedException(Exception): """ Raised when a token is requested but none are available. """ - pass class ReservationNotFoundException(Exception): """ Raised when the query result for a non-fungible token reservation is empty. """ - pass class ThrottlingException(Exception): """ Raised when the limiter is throttled by AWS. """ - pass class RateLimiterException(Exception): """ Raised by a limiter on unrecoverable errors when fetching a token or account limits. """ diff --git a/limiter/limiters.py b/limiter/limiters.py index 14b34a2..d844285 100644 --- a/limiter/limiters.py +++ b/limiter/limiters.py @@ -2,7 +2,7 @@ from limiter.utils import validate_table_env_fallback from limiter.managers import FungibleTokenManager, NonFungibleTokenManager -class BaseTokenLimiter(object): +class BaseTokenLimiter: """ Base class for both fungible and non-fungible token limiters. @@ -228,7 +228,7 @@ def __enter__(self): def __exit__(self, *args): if any(args): - print str(args) + print(str(args)) self.reservation.delete() def get_reservation(self): diff --git a/limiter/loader.py b/limiter/loader.py index 91f5fc1..e2b9a7e 100644 --- a/limiter/loader.py +++ b/limiter/loader.py @@ -9,7 +9,7 @@ SERVICE_NAME = 'serviceName' CONFIG_VERSION = 'configVersion' -class LimitLoader(object): +class LimitLoader: """ Performs initial limit loading and updates for a specified service. diff --git a/limiter/managers.py b/limiter/managers.py index 2f5221c..7d53d79 100644 --- a/limiter/managers.py +++ b/limiter/managers.py @@ -24,7 +24,7 @@ WINDOW_SEC = 'windowSec' RESERVATION_ID = 'reservationId' -class BaseTokenManager(object): +class BaseTokenManager: """ Base class for both fungible and non-fungible token managers. @@ -96,7 +96,7 @@ def _get_account_resource_limit(self, account_id): except Exception as e: if isinstance(e, ClientError): error_code = e.response['Error']['Code'] - if error_code == 'ProvisionedThroughputExceededException' or error_code == 'TooManyRequestsException': + if error_code in ('ProvisionedThroughputExceededException', 'TooManyRequestsException'): message = 'Throttled getting limit on {} for account {}'.format(self.resource_name, account_id) raise_from(ThrottlingException(message), e) message = 'Failed to get limit on {} for account {}'.format(self.resource_name, account_id) @@ -215,7 +215,7 @@ def _get_bucket_token(self, account_id, exec_time, ms_token): if error_code == 'ConditionalCheckFailedException': message = 'Resource capcity exhausted for {}:{}'.format(self.resource_name, account_id) raise CapacityExhaustedException(message) - elif error_code == 'ProvisionedThroughputExceededException' or error_code == 'TooManyRequestsException': + if error_code in ('ProvisionedThroughputExceededException', 'TooManyRequestsException'): message = 'Throttled by getting limit on {} for account {}'.format(self.resource_name, account_id) raise_from(ThrottlingException(message), e) raise @@ -247,8 +247,8 @@ def _refill_bucket_tokens(self, account_id, tokens, refill_time): ) except Exception as e: if isinstance(e, ClientError) and e.response['Error']['Code'] == 'ConditionalCheckFailedException': - logger.warn('Failed to refill tokens for %s:%s, already refilled with more current state', - self.resource_name, account_id) + logger.warning('Failed to refill tokens for %s:%s, already refilled with more current state', + self.resource_name, account_id) else: logger.exception('Failed to refill tokens for %s:%s', self.resource_name, account_id) @@ -357,7 +357,7 @@ def _buid_coordinate(self, account_id): """ return '{}:{}'.format(self.resource_name, account_id) -class TokenReservation(object): +class TokenReservation: """ Used to represent a temporary placeholder for, and create a non-fungible token, in DynamoDB. @@ -430,11 +430,11 @@ def delete(self): Delete the entry in DynamoDB representing this reservation. """ if self.is_token_created: - logger.warn('Cannot delete, a token has already been created from this reservation [%s]', self.id) + logger.warning('Cannot delete, a token has already been created from this reservation [%s]', self.id) return if self.is_deleted: - logger.warn('Cannot delete, this reservation [%s], has already been deleted', self.id) + logger.warning('Cannot delete, this reservation [%s], has already been deleted', self.id) return self.table.delete_item( diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7a81268 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +boto3==1.10.45 +botocore==1.13.45 diff --git a/setup.py b/setup.py index e8c468a..0b3164b 100644 --- a/setup.py +++ b/setup.py @@ -1,19 +1,19 @@ #!/usr/bin/env python from setuptools import setup, find_packages +with open('./requirements.txt', 'r') as file: + requirements = file.read().splitlines() + setup(name='rate-limiter-py', - version='0.2.1', + version='0.3.0', description='Rate-limiter module which leverages DynamoDB to enforce resource limits.', keywords=['lifeomic', 'dynamodb', 'rate', 'limit'], author='Matthew Tieman', author_email='mjtieman55@gmail.com', url='https://github.com/lifeomic/rate-limiter-py', - download_url='https://github.com/lifeomic/rate-limiter-py/archive/0.2.1.tar.gz', + download_url='https://github.com/lifeomic/rate-limiter-py/archive/0.3.0.tar.gz', packages=find_packages(), license='MIT', - install_requires=[ - 'boto3==1.6.4', - 'botocore==1.9.4', - 'future==0.16.0' - ] + python_requires='>=3.6.0', + install_requires=requirements ) diff --git a/test/test_event_processors.py b/test/test_event_processors.py index b0a86c9..9996483 100644 --- a/test/test_event_processors.py +++ b/test/test_event_processors.py @@ -138,7 +138,7 @@ def test_with_predicate(self): processor = EventProcessor(source, 'detail.id', predicate=mock_predicate) actual_id = processor.test_and_get_id(event) - self.assertEquals(expected_id, actual_id) + self.assertEqual(expected_id, actual_id) def test_without_predicate(self): source = random_string() @@ -148,7 +148,7 @@ def test_without_predicate(self): processor = EventProcessor(source, 'detail.id') actual_id = processor.test_and_get_id(event) - self.assertEquals(expected_id, actual_id) + self.assertEqual(expected_id, actual_id) def test_invalid_path(self): source = random_string() @@ -181,10 +181,10 @@ def test_processor_properties(self): predicate = ProcessorPredicate('detail.state', lambda state: True) processor = EventProcessor(source, id_path, predicate=predicate, type=type) - self.assertEquals(source, processor.source) - self.assertEquals(id_path, processor.id_path) - self.assertEquals(type, processor.type) - self.assertEquals(predicate, processor.predicate) + self.assertEqual(source, processor.source) + self.assertEqual(id_path, processor.id_path) + self.assertEqual(type, processor.type) + self.assertEqual(predicate, processor.predicate) class EventProcessorManagerTest(TestCase): def setUp(self): @@ -229,7 +229,7 @@ def test_env_params(self): with patch.dict('os.environ', env_vars): manager = EventProcessorManager() - self.assertEquals(self.table_name, manager.table_name) + self.assertEqual(self.table_name, manager.table_name) @mock_dynamodb2 def test_delete_token(self): @@ -243,7 +243,7 @@ def test_delete_token(self): mock_table = create_non_fung_table(self.table_name, self.index_name) self._insert_token(mock_table) - self.assertEquals(1, self._get_resource_id_count(mock_table)) + self.assertEqual(1, self._get_resource_id_count(mock_table)) manager = EventProcessorManager(table_name=self.table_name, index_name=self.index_name, @@ -251,7 +251,7 @@ def test_delete_token(self): manager._table = mock_table manager.process_event(event) - self.assertEquals(0, self._get_resource_id_count(mock_table)) + self.assertEqual(0, self._get_resource_id_count(mock_table)) @mock_dynamodb2 def test_delete_no_token_for_id(self): @@ -265,7 +265,7 @@ def test_delete_no_token_for_id(self): mock_table = create_non_fung_table(self.table_name, self.index_name) self._insert_token(mock_table) - self.assertEquals(1, self._get_resource_id_count(mock_table)) + self.assertEqual(1, self._get_resource_id_count(mock_table)) manager = EventProcessorManager(table_name=self.table_name, index_name=self.index_name, @@ -273,7 +273,7 @@ def test_delete_no_token_for_id(self): manager._table = mock_table manager.process_event(event) - self.assertEquals(1, self._get_resource_id_count(mock_table)) + self.assertEqual(1, self._get_resource_id_count(mock_table)) @mock_dynamodb2 def test_delete_no_id_from_processor(self): @@ -287,7 +287,7 @@ def test_delete_no_id_from_processor(self): mock_table = create_non_fung_table(self.table_name, self.index_name) self._insert_token(mock_table) - self.assertEquals(1, self._get_resource_id_count(mock_table)) + self.assertEqual(1, self._get_resource_id_count(mock_table)) manager = EventProcessorManager(table_name=self.table_name, index_name=self.index_name, @@ -295,7 +295,7 @@ def test_delete_no_id_from_processor(self): manager._table = mock_table manager.process_event(event) - self.assertEquals(1, self._get_resource_id_count(mock_table)) + self.assertEqual(1, self._get_resource_id_count(mock_table)) @mock_dynamodb2 def test_delete_on_type(self): @@ -306,7 +306,7 @@ def test_delete_on_type(self): mock_default_processor = Mock() mock_default_processor.source = event_source mock_default_processor.type = None - mock_default_processor.test_and_get_id = MagicMock(side_effect=StandardError('Wrong processor invoked')) + mock_default_processor.test_and_get_id = MagicMock(side_effect=Exception('Wrong processor invoked')) mock_type_processor = Mock() mock_type_processor.source = event_source @@ -315,7 +315,7 @@ def test_delete_on_type(self): mock_table = create_non_fung_table(self.table_name, self.index_name) self._insert_token(mock_table) - self.assertEquals(1, self._get_resource_id_count(mock_table)) + self.assertEqual(1, self._get_resource_id_count(mock_table)) manager = EventProcessorManager(table_name=self.table_name, index_name=self.index_name, @@ -323,7 +323,7 @@ def test_delete_on_type(self): manager._table = mock_table manager.process_event(event) - self.assertEquals(0, self._get_resource_id_count(mock_table)) + self.assertEqual(0, self._get_resource_id_count(mock_table)) @mock_dynamodb2 def test_delete_fallback_no_type(self): @@ -339,11 +339,11 @@ def test_delete_fallback_no_type(self): mock_type_processor = Mock() mock_type_processor.source = event_source mock_type_processor.type = detail_type + random_string() - mock_type_processor.test_and_get_id = MagicMock(side_effect=StandardError('Wrong processor invoked')) + mock_type_processor.test_and_get_id = MagicMock(side_effect=Exception('Wrong processor invoked')) mock_table = create_non_fung_table(self.table_name, self.index_name) self._insert_token(mock_table) - self.assertEquals(1, self._get_resource_id_count(mock_table)) + self.assertEqual(1, self._get_resource_id_count(mock_table)) manager = EventProcessorManager(table_name=self.table_name, index_name=self.index_name, @@ -351,7 +351,7 @@ def test_delete_fallback_no_type(self): manager._table = mock_table manager.process_event(event) - self.assertEquals(0, self._get_resource_id_count(mock_table)) + self.assertEqual(0, self._get_resource_id_count(mock_table)) def _get_resource_id_count(self, mock_table): response = mock_table.query(IndexName=self.index_name, diff --git a/test/test_limiters.py b/test/test_limiters.py index d223a39..5c59b9e 100644 --- a/test/test_limiters.py +++ b/test/test_limiters.py @@ -73,9 +73,9 @@ def test_manager_config_ctor_params(self): self.limit_table_name) manager = limiter.manager - self.assertEquals(self.token_table_name, manager.token_table_name) - self.assertEquals(self.limit_table_name, manager.limit_table_name) - self.assertEquals(self.resource_name, manager.resource_name) + self.assertEqual(self.token_table_name, manager.token_table_name) + self.assertEqual(self.limit_table_name, manager.limit_table_name) + self.assertEqual(self.resource_name, manager.resource_name) def test_manager_config_env_params(self): env_vars = { @@ -87,9 +87,9 @@ def test_manager_config_env_params(self): limiter = rate_limit(self.resource_name, self.limit, self.window) manager = limiter.manager - self.assertEquals(self.token_table_name, manager.token_table_name) - self.assertEquals(self.limit_table_name, manager.limit_table_name) - self.assertEquals(self.resource_name, manager.resource_name) + self.assertEqual(self.token_table_name, manager.token_table_name) + self.assertEqual(self.limit_table_name, manager.limit_table_name) + self.assertEqual(self.resource_name, manager.resource_name) @patch('limiter.limiters.FungibleTokenManager') def test_decoratored_account_id_pos(self, mock_manager_delegate): @@ -213,8 +213,8 @@ def test_delete_on_exception(self, mock_manager_delegate): self.token_table_name, self.limit_table_name) as reservation: self.assertNotNone(reservation) - raise StandardError('Deliberately thrown from test_delete_on_exception') - except StandardError: + raise Exception('Deliberately thrown from test_delete_on_exception') + except Exception: pass mock_reservation.delete.assert_called() diff --git a/test/test_managers.py b/test/test_managers.py index 1f8ff7a..5400590 100644 --- a/test/test_managers.py +++ b/test/test_managers.py @@ -32,7 +32,7 @@ def test_compute_refill_amount(self): expected = 8 actual = _compute_refill_amount(current_tokens, time_since_refill, self.limit, self.token_ms) - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) def test_compute_refill_amount_negative_balance(self): current_tokens = -7 @@ -41,7 +41,7 @@ def test_compute_refill_amount_negative_balance(self): expected = 3 actual = _compute_refill_amount(current_tokens, time_since_refill, self.limit, self.token_ms) - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) def test_compute_refill_amount_refill_lag(self): current_tokens = 0 @@ -49,7 +49,7 @@ def test_compute_refill_amount_refill_lag(self): expected = self.limit - 1 actual = _compute_refill_amount(current_tokens, time_since_refill, self.limit, self.token_ms) - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) def test_get_bucket_token(self): account_id = random_string() @@ -62,10 +62,10 @@ def test_get_bucket_token(self): self.manager._token_table = mock_token_table actual = self.manager._get_bucket_token(account_id, exec_time, self.ms_token) - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) actual_args = mock_token_table.update_item.call_args_list - self.assertEquals(1, len(actual_args)) + self.assertEqual(1, len(actual_args)) expected_args = [ ({ @@ -85,7 +85,7 @@ def test_get_bucket_token(self): }) ] - self.assertEquals(expected_args, actual_args[0]) + self.assertEqual(expected_args, actual_args[0]) def test_get_bucket_token_exhausted(self): exec_time = now_utc_ms() @@ -115,7 +115,7 @@ def test_refill_bucket_tokens(self): self.manager._refill_bucket_tokens(account_id, tokens, refill_time) actual_args = mock_token_table.update_item.call_args_list - self.assertEquals(1, len(actual_args)) + self.assertEqual(1, len(actual_args)) expected_args = [ ({ @@ -133,7 +133,7 @@ def test_refill_bucket_tokens(self): }) ] - self.assertEquals(expected_args, actual_args[0]) + self.assertEqual(expected_args, actual_args[0]) @mock_dynamodb2 def test_account_resource_limit(self): @@ -147,8 +147,8 @@ def test_account_resource_limit(self): self.manager._limit_table = mock_limit_table result = self.manager._get_account_resource_limit(account_id) - self.assertEquals(limit, result['limit']) - self.assertEquals(window, result['windowSec']) + self.assertEqual(limit, result['limit']) + self.assertEqual(window, result['windowSec']) @mock_dynamodb2 def test_account_resource_limit_defaults(self): @@ -158,8 +158,8 @@ def test_account_resource_limit_defaults(self): self.manager._limit_table = mock_limit_table result = self.manager._get_account_resource_limit(account_id) - self.assertEquals(self.limit, result['limit']) - self.assertEquals(self.window, result['windowSec']) + self.assertEqual(self.limit, result['limit']) + self.assertEqual(self.window, result['windowSec']) @mock_dynamodb2 def test_account_resource_limit_blacklist(self): @@ -198,11 +198,11 @@ def test_get_reservation(self): self.manager.get_reservation(account_id) response = mock_token_table.query(KeyConditionExpression=Key('resourceCoordinate').eq(coordinate)) - self.assertEquals(1, response['Count']) + self.assertEqual(1, response['Count']) items = dict(pair for item in response['Items'] for pair in item.items()) - self.assertEquals(self.resource_name, items['resourceName']) - self.assertEquals(account_id, items['accountId']) + self.assertEqual(self.resource_name, items['resourceName']) + self.assertEqual(account_id, items['accountId']) self.assertTrue(items['expirationTime'] > now) self.assertIn('resourceId', items) @@ -282,7 +282,7 @@ def test_get_token_count(self): self.manager._limit_table = mock_limit_table actual_count = self.manager._get_token_count(account_id, now) - self.assertEquals(expected_count, actual_count) + self.assertEqual(expected_count, actual_count) @mock_dynamodb2 def test_get_token_count_no_tokens(self): @@ -299,7 +299,7 @@ def test_get_token_count_no_tokens(self): expected_count = 0 actual_count = self.manager._get_token_count(account_id, now) - self.assertEquals(expected_count, actual_count) + self.assertEqual(expected_count, actual_count) class TokenReservationTest(TestCase): def setUp(self): @@ -317,12 +317,12 @@ def test_delete_reservation(self): _insert_reservation(mock_token_table, reserve) response = mock_token_table.query(KeyConditionExpression=Key('resourceCoordinate').eq(self.coordinate)) - self.assertEquals(1, response['Count']) + self.assertEqual(1, response['Count']) reserve.delete() response = mock_token_table.query(KeyConditionExpression=Key('resourceCoordinate').eq(self.coordinate)) - self.assertEquals(0, response['Count']) + self.assertEqual(0, response['Count']) def test_create_after_delete(self): mock_token_table = Mock() diff --git a/test/utils.py b/test/utils.py index e736e4e..55af50a 100644 --- a/test/utils.py +++ b/test/utils.py @@ -5,7 +5,7 @@ import boto3 def random_string(length=8): - return ''.join(random.choice(string.lowercase) for i in range(length)) + return ''.join(random.choice(string.ascii_lowercase) for i in range(length)) def now_utc_sec(): return int(datetime.utcnow().strftime('%s')) @@ -37,14 +37,6 @@ def create_limit_table(table_name, index_name='idx'): 'AttributeName': 'accountId', 'AttributeType': 'RANGE' }, - { - 'AttributeName': 'limit', - 'AttributeType': 'N' - }, - { - 'AttributeName': 'windowSec', - 'AttributeType': 'N' - }, { 'AttributeName': 'serviceName', 'AttributeType': 'S' @@ -107,18 +99,6 @@ def create_non_fung_table(table_name, index_name='idx'): 'AttributeName': 'reservationId', 'AttributeType': 'S' }, - { - 'AttributeName': 'expirationTime', - 'AttributeType': 'N' - }, - { - 'AttributeName': 'resourceName', - 'AttributeType': 'S' - }, - { - 'AttributeName': 'accountId', - 'AttributeType': 'S' - }, { 'AttributeName': 'resourceId', 'AttributeType': 'S' diff --git a/test_requirements.txt b/test_requirements.txt index e61359e..2aa3455 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,7 +1,7 @@ mock>=1.2.0 pylint>=1.6.4 nose>=1.3.7 -boto3==1.6.4 -botocore==1.9.4 -moto>=1.3.3 +boto3==1.10.45 +botocore==1.13.45 +moto>=1.3.14 future==0.16.0 diff --git a/tox.ini b/tox.ini index 149a3bc..ca2a94a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] skipsdist = True -envlist = py27 +envlist = py36 [testenv] deps = -rtest_requirements.txt