From cb130c16a190aba67ead456855cc005e6933d7a7 Mon Sep 17 00:00:00 2001 From: Robert Stevens Date: Wed, 6 Mar 2019 00:11:23 +0000 Subject: [PATCH 1/4] initial source commit --- .gitignore | 20 ++ .pylintrc | 44 ++++ NOTICE | 4 +- doc/_templates/autosummary/module.rst | 4 + doc/conf.py | 51 ++++ doc/index.rst | 22 ++ requirements.txt | 1 + setup.cfg | 29 ++ setup.py | 24 ++ src/aws_secretsmanager_caching/__init__.py | 18 ++ .../cache/__init__.py | 21 ++ src/aws_secretsmanager_caching/cache/items.py | 247 ++++++++++++++++++ src/aws_secretsmanager_caching/cache/lru.py | 136 ++++++++++ src/aws_secretsmanager_caching/config.py | 70 +++++ src/aws_secretsmanager_caching/decorators.py | 104 ++++++++ .../secret_cache.py | 90 +++++++ test-requirements.txt | 8 + test/unit/__init__.py | 12 + test/unit/test_aws_secretsmanager_caching.py | 175 +++++++++++++ test/unit/test_config.py | 35 +++ test/unit/test_decorators.py | 190 ++++++++++++++ test/unit/test_items.py | 49 ++++ test/unit/test_lru.py | 73 ++++++ tox.ini | 44 ++++ 24 files changed, 1469 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 .pylintrc create mode 100644 doc/_templates/autosummary/module.rst create mode 100644 doc/conf.py create mode 100644 doc/index.rst create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 src/aws_secretsmanager_caching/__init__.py create mode 100644 src/aws_secretsmanager_caching/cache/__init__.py create mode 100644 src/aws_secretsmanager_caching/cache/items.py create mode 100644 src/aws_secretsmanager_caching/cache/lru.py create mode 100644 src/aws_secretsmanager_caching/config.py create mode 100644 src/aws_secretsmanager_caching/decorators.py create mode 100644 src/aws_secretsmanager_caching/secret_cache.py create mode 100644 test-requirements.txt create mode 100644 test/unit/__init__.py create mode 100644 test/unit/test_aws_secretsmanager_caching.py create mode 100644 test/unit/test_config.py create mode 100644 test/unit/test_decorators.py create mode 100644 test/unit/test_items.py create mode 100644 test/unit/test_lru.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..71ccd12 --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +*~ +*# +*.swp +__pycache__/ +*.py[cod] +*$py.class +*.egg-info/ +/.coverage +/.coverage.* +/.cache +/doc/_autosummary/ +/build +*.iml +dist/ +.eggs/ +.pytest_cache/ +.python-version +.tox/ +venv/ +env/ \ No newline at end of file diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..3c3880e --- /dev/null +++ b/.pylintrc @@ -0,0 +1,44 @@ +[MASTER] + +[MESSAGES CONTROL] +disable = I0011, # locally-disabled + R0903 # too-few-public-methods + +[REPORTS] +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html +output-format=colorized + + +[FORMAT] +# Maximum number of characters on a single line. +max-line-length=120 +# Maximum number of lines in a module +#max-module-lines=1000 + +[BASIC] +good-names= + i, + e, + logger + +[SIMILARITIES] +# Minimum lines number of a similarity. +min-similarity-lines=5 +# Ignore comments when computing similarities. +ignore-comments=yes +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +[VARIABLES] +# Tells whether we should check for unused import in __init__ files. +init-import=yes + +[LOGGING] +# Apply logging string format checks to calls on these modules. +logging-modules= + logging + +[TYPECHECK] +ignored-modules= + distutils diff --git a/NOTICE b/NOTICE index ecf60a2..a13915c 100644 --- a/NOTICE +++ b/NOTICE @@ -1,2 +1,2 @@ -AWS Secrets Manager Python caching client -Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +AWS Secrets Manager Python Caching Client +Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/doc/_templates/autosummary/module.rst b/doc/_templates/autosummary/module.rst new file mode 100644 index 0000000..29862a4 --- /dev/null +++ b/doc/_templates/autosummary/module.rst @@ -0,0 +1,4 @@ +{{ fullname }} +{{ underline }} + +.. automodule:: {{ fullname }} diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 0000000..e910d7b --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,51 @@ +from datetime import datetime +import os +import shutil + +version = '1.0' +project = u'AWS Secrets Manager Python Caching Client' + +# If you use autosummary, this ensures that any stale autogenerated files are +# cleaned up first. +if os.path.exists('_autosummary'): + print("cleaning up stale autogenerated files...") + shutil.rmtree('_autosummary') + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.coverage', + 'sphinx.ext.doctest', + 'sphinx.ext.napoleon', + 'sphinx.ext.todo', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +source_suffix = '.rst' # The suffix of source filenames. +master_doc = 'index' # The master toctree document. + +copyright = u'%s, Amazon.com' % datetime.now().year + +# The full version, including alpha/beta/rc tags. +release = version + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = ['_build', '_templates'] + +pygments_style = 'sphinx' + +autoclass_content = "both" +autodoc_default_flags = ['show-inheritance', 'members', 'undoc-members'] +autodoc_member_order = 'bysource' + +html_theme = 'haiku' +html_static_path = ['_static'] +htmlhelp_basename = '%sdoc' % project + +# autosummary +autosummary_generate = True diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 0000000..4b5492b --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,22 @@ +AWS Secrets Manager Python Caching Client +========================================= + +This package provides a client-side caching implementation for AWS Secrets Manager + +Modules +_______ + +.. autosummary:: + :toctree: _autosummary + + .. Add/replace module names you want documented here + aws_secretsmanager_caching + + + +Indices and tables +__________________ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5b9f2b7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +botocore>=1.12 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..00af076 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,29 @@ +[tool:pytest] +xfail_strict = true +addopts = + --verbose + --doctest-modules + --cov aws_secretsmanager_caching + --cov-fail-under 90 + test + +[aliases] +test=pytest + +[metadata] +description-file = README.md +license_file = LICENSE + +[flake8] +max-line-length = 120 +select = C,E,F,W,B +# C812, W503 clash with black +ignore = C812,W503 +exclude = venv,.venv,.tox,dist,doc,build,*.egg + +[isort] +line_length = 120 +multi_line_output = 5 +include_trailing_comma = true +force_grid_wrap = 0 +combine_as_imports = true diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..9741ab8 --- /dev/null +++ b/setup.py @@ -0,0 +1,24 @@ +from setuptools import setup, find_packages + +setup( + name="aws_secretsmanager_caching", + description="Client-side AWS Secrets Manager caching library", + url="https://aws.amazon.com/secrets-manager/", + author="Amazon Web Services", + author_email="aws-secretsmanager-dev@amazon.com", + version="1.0", + packages=find_packages(where="src", exclude=("test",)), + package_dir={"": "src"}, + classifiers=[ + 'Development Status :: 5 - Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache License', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7' + ], + keywords='secretsmanager secrets manager development cache caching client', + python_requires='>3.5', + setup_requires=['pytest-runner'], + tests_require=['pytest<3.8.0', 'pytest-cov', 'pytest-sugar', 'codecov>=1.4.0'] + +) diff --git a/src/aws_secretsmanager_caching/__init__.py b/src/aws_secretsmanager_caching/__init__.py new file mode 100644 index 0000000..06bfc8d --- /dev/null +++ b/src/aws_secretsmanager_caching/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""High level AWS Secrets Manager caching client.""" +from aws_secretsmanager_caching.config import SecretCacheConfig +from aws_secretsmanager_caching.decorators import InjectKeywordedSecretString, InjectSecretString +from aws_secretsmanager_caching.secret_cache import SecretCache + +__all__ = ["SecretCache", "SecretCacheConfig", "InjectSecretString", "InjectKeywordedSecretString"] diff --git a/src/aws_secretsmanager_caching/cache/__init__.py b/src/aws_secretsmanager_caching/cache/__init__.py new file mode 100644 index 0000000..955a16a --- /dev/null +++ b/src/aws_secretsmanager_caching/cache/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Internal Implementation Details +.. warning:: + No guarantee is provided on the modules and APIs within this + namespace staying consistent. Directly reference at your own risk. +""" +from aws_secretsmanager_caching.cache.items import SecretCacheItem, SecretCacheObject, SecretCacheVersion +from aws_secretsmanager_caching.cache.lru import LRUCache + +__all__ = ["SecretCacheObject", "SecretCacheItem", "SecretCacheVersion", "LRUCache"] diff --git a/src/aws_secretsmanager_caching/cache/items.py b/src/aws_secretsmanager_caching/cache/items.py new file mode 100644 index 0000000..1cc6f44 --- /dev/null +++ b/src/aws_secretsmanager_caching/cache/items.py @@ -0,0 +1,247 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Secret cache items""" + +import threading +from abc import ABCMeta, abstractmethod +from copy import deepcopy +from datetime import datetime, timedelta +from random import randint + +from .lru import LRUCache + + +class SecretCacheObject: # pylint: disable=too-many-instance-attributes + """Secret cache object that handles the common refresh logic.""" + + __metaclass__ = ABCMeta + + def __init__(self, config, client, secret_id): + """Construct the secret cache object. + + :type config: aws_secretsmanager_caching.SecretCacheConfig + :param config: Configuration for the cache. + + :type client: botocore.client.BaseClient + :param client: The 'secretsmanager' boto client. + + :type secret_id: str + :param secret_id: The secret identifier to cache. + """ + self._lock = threading.RLock() + self._config = config + self._client = client + self._secret_id = secret_id + self._result = None + self._exception = None + self._exception_count = 0 + self._refresh_needed = True + self._next_retry_time = None + + def _is_refresh_needed(self): + """Determine if the cached object should be refreshed. + + :rtype: bool + :return: True if the object should be refreshed. + """ + if self._refresh_needed: + return True + if self._exception is None: + return False + if self._next_retry_time is None: + return False + return self._next_retry_time <= datetime.utcnow() + + @abstractmethod + def _execute_refresh(self): + """Perform the refresh of the cached object. + + :rtype: object + :return: The cached result of the refresh. + """ + + @abstractmethod + def _get_version(self, version_stage): + """Get a cached secret version based on the given stage. + + :type version_stage: str + :param version_stage: The version stage being requested. + + :rtype: object + :return: The associated cached secret version. + """ + + def __refresh(self): + """Refresh the cached object when needed. + + :rtype: None + :return: None + """ + if not self._is_refresh_needed(): + return + self._refresh_needed = False + try: + self._result = self._execute_refresh() + self._exception = None + self._exception_count = 0 + except Exception as e: # pylint: disable=broad-except + self._exception = e + delay = self._config.exception_retry_delay_base * ( + self._config.exception_retry_growth_factor ** self._exception_count + ) + self._exception_count += 1 + delay = min(delay, self._config.exception_retry_delay_max) + self._next_retry_time = datetime.utcnow() + timedelta(milliseconds=delay) + + def get_secret_value(self, version_stage=None): + """Get the cached secret value for the given version stage. + + :type version_stage: str + :param version_stage: The requested secret version stage. + + :rtype: object + :return: The cached secret value. + """ + if not version_stage: + version_stage = self._config.default_version_stage + with self._lock: + self.__refresh() + value = self._get_version(version_stage) + if not value and self._exception: + raise self._exception + return deepcopy(value) + + +class SecretCacheItem(SecretCacheObject): + """The secret cache item that maintains a cache of secret versions.""" + + def __init__(self, config, client, secret_id): + """Construct a secret cache item. + + :type config: aws_secretsmanager_caching.SecretCacheConfig + :param config: Configuration for the cache. + + :type client: botocore.client.BaseClient + :param client: The 'secretsmanager' boto client. + + :type secret_id: str + :param secret_id: The secret identifier to cache. + """ + super(SecretCacheItem, self).__init__(config, client, secret_id) + self._versions = LRUCache(10) + self._next_refresh_time = datetime.utcnow() + + def _is_refresh_needed(self): + """Determine if the cached item should be refreshed. + + :rtype: bool + :return: True if a refresh is needed. + """ + if super(SecretCacheItem, self)._is_refresh_needed(): + return True + if self._exception: + return False + return self._next_refresh_time <= datetime.utcnow() + + @staticmethod + def _get_version_id(result, version_stage): + """Get the version id for the given version stage. + + :type: dict + :param result: The result of the DescribeSecret request. + + :type version_stage: str + :param version_stage: The version stage being requested. + + :rtype: str + :return: The associated version id. + """ + if not result: + return None + if "VersionIdsToStages" not in result: + return None + ids = [key for (key, value) in result["VersionIdsToStages"].items() if version_stage in value] + if not ids: + return None + return ids[0] + + def _execute_refresh(self): + """Perform the actual refresh of the cached secret information. + + :rtype: dict + :return: The result of the DescribeSecret request. + """ + result = self._client.describe_secret(SecretId=self._secret_id) + ttl = self._config.secret_refresh_interval + self._next_refresh_time = datetime.utcnow() + timedelta(seconds=randint(round(ttl / 2), ttl)) + return result + + def _get_version(self, version_stage): + """Get the version associated with the given stage. + + :type version_stage: str + :param version_stage: The version stage being requested. + + :rtype: dict + :return: The cached secret for the given version stage. + """ + version_id = self._get_version_id(self._result, version_stage) + if not version_id: + return None + version = self._versions.get(version_id) + if version: + return version.get_secret_value() + self._versions.put_if_absent(version_id, SecretCacheVersion(self._config, self._client, self._secret_id, + version_id)) + return self._versions.get(version_id).get_secret_value() + + +class SecretCacheVersion(SecretCacheObject): + """Secret cache object for a secret version.""" + + def __init__(self, config, client, secret_id, version_id): + """Construct the cache object for a secret version. + + :type config: aws_secretsmanager_caching.SecretCacheConfig + :param config: Configuration for the cache. + + :type client: botocore.client.BaseClient + :param client: The 'secretsmanager' boto client. + + :type secret_id: str + :param secret_id: The secret identifier to cache. + + :type version_id: str + :param version_id: The version identifier. + """ + super(SecretCacheVersion, self).__init__(config, client, secret_id) + self._version_id = version_id + + def _execute_refresh(self): + """Perform the actual refresh of the cached secret version. + + :rtype: dict + :return: The result of GetSecretValue for the version. + """ + return self._client.get_secret_value(SecretId=self._secret_id, VersionId=self._version_id) + + def _get_version(self, version_stage): + """Get the cached version information for the given stage. + + :type version_stage: str + :param version_stage: The version stage being requested. + + :rtype: dict + :return: The cached GetSecretValue result. + """ + return self._result diff --git a/src/aws_secretsmanager_caching/cache/lru.py b/src/aws_secretsmanager_caching/cache/lru.py new file mode 100644 index 0000000..2c1bb4b --- /dev/null +++ b/src/aws_secretsmanager_caching/cache/lru.py @@ -0,0 +1,136 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""LRU cache""" + +import threading + + +class LRUCache: + """Least recently used cache""" + + def __init__(self, max_size=1024): + """Construct a new instance of the LRU cache + + :type max_size: int + :param max_size: The maximum number of elements to store in the cache + """ + self._lock = threading.RLock() + self._cache = {} + self._head = None + self._tail = None + self._max_size = max_size + self._size = 0 + + def get(self, key): + """Get the cached item for the given key + + :type key: object + :param key: Key of the cached item + + :rtype: object + :return: The cached item associated with the key + """ + with self._lock: + if key not in self._cache: + return None + item = self._cache[key] + self._update_head(item) + return item.data + + def put_if_absent(self, key, data): + """Associate the given item with the key if the key is not already associated with an item. + + :type key: object + :param key: The key for the item to cache. + + :type data: object + :param data: The item to cache if the key is not already in use. + + :rtype: bool + :return: True if the given data was mapped to the given key. + """ + with self._lock: + if key in self._cache: + return False + item = LRUItem(key=key, data=data) + self._cache[key] = item + self._size += 1 + self._update_head(item) + if self._size > self._max_size: + del self._cache[self._tail.key] + self._unlink(self._tail) + self._size -= 1 + return True + + def _update_head(self, item): + """Update the head item in the list to be the given item. + + :type item: object + :param item: The item that should be updated as the head item. + + :rtype: None + :return: None + """ + if item is self._head: + return + self._unlink(item) + item.next = self._head + if self._head is not None: + self._head.prev = item + self._head = item + if self._tail is None: + self._tail = item + + def _unlink(self, item): + """Unlink the given item from the linked list. + + :type item: object + :param item: The item to unlink from the linked list. + + :rtype: None + :return: None + """ + if item is self._head: + self._head = item.next + if item is self._tail: + self._tail = item.prev + if item.prev is not None: + item.prev.next = item.next + if item.next is not None: + item.next.prev = item.prev + item.next = None + item.prev = None + + +class LRUItem: + """An item for use in the LRU cache.""" + + def __init__(self, key, data=None, prev=None, nxt=None): + """Construct an item for use within the LRU cache. + + :type key: object + :param key: The key associated with the item. + + :type data: object + :param data: The associated data for the key/item. + + :type prev: LRUItem + :param prev: The previous item in the linked list. + + :type nxt: LRUItem + :param nxt: The next item in the linked list. + """ + self.key = key + self.next = nxt + self.prev = prev + self.data = data diff --git a/src/aws_secretsmanager_caching/config.py b/src/aws_secretsmanager_caching/config.py new file mode 100644 index 0000000..5b98388 --- /dev/null +++ b/src/aws_secretsmanager_caching/config.py @@ -0,0 +1,70 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Secret cache configuration object.""" + +from copy import deepcopy +from datetime import timedelta + + +class SecretCacheConfig: + + """Advanced configuration for SecretCache clients. + + :type max_cache_size: int + :param max_cache_size: The maximum number of secrets to cache. + + :type exception_retry_delay_base: int + :param exception_retry_delay_base: The number of seconds to wait + after an exception is encountered and before retrying the request. + + :type exception_retry_growth_factor: int + :param exception_retry_growth_factor: The growth factor to use for + calculating the wait time between retries of failed requests. + + :type exception_retry_delay_max: int + :param exception_retry_delay_max: The maximum amount of time in + seconds to wait between failed requests. + + :type default_version_stage: str + :param default_version_stage: The default version stage to request. + + :type secret_refresh_interval: int + :param secret_refresh_interval: The number of seconds to wait between + refreshing cached secret information. + + """ + + OPTION_DEFAULTS = { + "max_cache_size": 1024, + "exception_retry_delay_base": 1, + "exception_retry_growth_factor": 2, + "exception_retry_delay_max": 3600, + "default_version_stage": "AWSCURRENT", + "secret_refresh_interval": timedelta(hours=1).total_seconds(), + } + + def __init__(self, **kwargs): + options = deepcopy(self.OPTION_DEFAULTS) + + # Set config options based on given values + if kwargs: + for key, value in kwargs.items(): + if key in options: + options[key] = value + # The key must exist in the available options + else: + raise TypeError("Unexpected keyword argument '%s'" % key) + + # Set the attributes based on the config options + for key, value in options.items(): + setattr(self, key, value) diff --git a/src/aws_secretsmanager_caching/decorators.py b/src/aws_secretsmanager_caching/decorators.py new file mode 100644 index 0000000..77caf98 --- /dev/null +++ b/src/aws_secretsmanager_caching/decorators.py @@ -0,0 +1,104 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Decorators for use with caching library """ +import json + + +class InjectSecretString: + """Decorator implementing high-level Secrets Manager caching client""" + + def __init__(self, secret_id, cache): + """ + Constructs a decorator to inject a single non-keyworded argument from a cached secret for a given function. + + :type secret_id: str + :param secret_id: The secret identifier + + :type cache: aws_secretsmanager_caching.SecretCache + :param cache: Secret cache + """ + + self.cache = cache + self.secret_id = secret_id + + def __call__(self, func): + """ + Return a function with cached secret injected as first argument. + + :type func: object + :param func: The function for injecting a single non-keyworded argument too. + :return The function with the injected argument. + """ + + secret = self.cache.get_secret_string(secret_id=self.secret_id) + + def _wrapped_func(*args, **kwargs): + """ + Internal function to execute wrapped function + """ + func(secret, *args, **kwargs) + + return _wrapped_func + + +class InjectKeywordedSecretString: + """Decorator implementing high-level Secrets Manager caching client using JSON-based secrets""" + + def __init__(self, secret_id, cache, **kwargs): + """ + Construct a decorator to inject a variable list of keyword arguments to a given function with resolved values + from a cached secret. + + :type kwargs: dict + :param kwargs: dictionary mapping original keyword argument of wrapped function to JSON-encoded secret key + + :type secret_id: str + :param secret_id: The secret identifier + + :type cache: aws_secretsmanager_caching.SecretCache + :param cache: Secret cache + """ + + self.cache = cache + self.kwarg_map = kwargs + self.secret_id = secret_id + + def __call__(self, func): + """ + Return a function with injected keyword arguments from a cached secret. + + :type func: object + :param func: function for injecting keyword arguments. + :return The original function with injected keyword arguments + """ + + try: + secret = json.loads(self.cache.get_secret_string(secret_id=self.secret_id)) + except json.decoder.JSONDecodeError: + raise RuntimeError('Cached secret is not valid JSON') + + resolved_kwargs = dict() + for orig_kwarg in self.kwarg_map: + secret_key = self.kwarg_map[orig_kwarg] + try: + resolved_kwargs[orig_kwarg] = secret[secret_key] + except KeyError: + raise RuntimeError('Cached secret does not contain key {0}'.format(secret_key)) + + def _wrapped_func(*args, **kwargs): + """ + Internal function to execute wrapped function + """ + func(*args, **resolved_kwargs, **kwargs) + + return _wrapped_func diff --git a/src/aws_secretsmanager_caching/secret_cache.py b/src/aws_secretsmanager_caching/secret_cache.py new file mode 100644 index 0000000..514a174 --- /dev/null +++ b/src/aws_secretsmanager_caching/secret_cache.py @@ -0,0 +1,90 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""High level AWS Secrets Manager caching client.""" +from copy import deepcopy + +import botocore.session + +from .cache import LRUCache, SecretCacheItem +from .config import SecretCacheConfig + + +class SecretCache: + """Secret Cache client for AWS Secrets Manager secrets""" + + def __init__(self, config=SecretCacheConfig(), client=None): + """Construct a secret cache using the given configuration and + AWS Secrets Manager boto client. + + :type config: aws_secretsmanager_caching.SecretCacheConfig + :param config: Secret cache configuration + + :type client: botocore.client.BaseClient + :param client: boto 'secretsmanager' client + """ + self._client = client + self._config = deepcopy(config) + self._cache = LRUCache(max_size=self._config.max_cache_size) + if self._client is None: + self._client = botocore.session.get_session().create_client("secretsmanager") + + def _get_cached_secret(self, secret_id): + """Get a cached secret for the given secret identifier. + + :type secret_id: str + :param secret_id: The secret identifier + + :rtype: aws_secretsmanager_caching.cache.SecretCacheItem + :return: The associated cached secret item + """ + secret = self._cache.get(secret_id) + if secret is not None: + return secret + self._cache.put_if_absent( + secret_id, SecretCacheItem(config=self._config, client=self._client, secret_id=secret_id) + ) + return self._cache.get(secret_id) + + def get_secret_string(self, secret_id, version_stage=None): + """Get the secret string value from the cache. + + :type secret_id: str + :param secret_id: The secret identifier + + :type version_stage: str + :param version_stage: The stage for the requested version. + + :rtype: str + :return: The associated secret string value + """ + secret = self._get_cached_secret(secret_id).get_secret_value(version_stage) + if secret is None: + return secret + return secret.get("SecretString") + + def get_secret_binary(self, secret_id, version_stage=None): + """Get the secret binary value from the cache. + + :type secret_id: str + :param secret_id: The secret identifier + + :type version_stage: str + :param version_stage: The stage for the requested version. + + :rtype: bytes + :return: The associated secret binary value + """ + secret = self._get_cached_secret(secret_id).get_secret_value(version_stage) + if secret is None: + return secret + return secret.get("SecretBinary") diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..258956f --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,8 @@ +pytest<3.8.0 +pytest-cov +pytest-sugar +codecov>=1.4.0 +botocore>=1.12 +pylint>1.9.4 +isort>=4.3.4 +sphinx>=1.8.4 diff --git a/test/unit/__init__.py b/test/unit/__init__.py new file mode 100644 index 0000000..1ccc7fa --- /dev/null +++ b/test/unit/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. diff --git a/test/unit/test_aws_secretsmanager_caching.py b/test/unit/test_aws_secretsmanager_caching.py new file mode 100644 index 0000000..5ecf17e --- /dev/null +++ b/test/unit/test_aws_secretsmanager_caching.py @@ -0,0 +1,175 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +""" +Unit test suite for high-level functions in aws_secretsmanager_caching +""" +import unittest +import pytest +import botocore + +from aws_secretsmanager_caching.config import SecretCacheConfig +from aws_secretsmanager_caching.secret_cache import SecretCache +from botocore.exceptions import NoRegionError +from botocore.exceptions import ClientError +from botocore.stub import Stubber + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +class TestAwsSecretsManagerCaching(unittest.TestCase): + + def setUp(self): + pass + + def get_client(self, response={}, versions=None, version_response=None): + client = botocore.session.get_session().create_client( + 'secretsmanager', region_name='us-west-2') + + stubber = Stubber(client) + expected_params = {'SecretId': 'test'} + if versions: + response['VersionIdsToStages'] = versions + stubber.add_response('describe_secret', response, expected_params) + if version_response is not None: + stubber.add_response('get_secret_value', version_response) + stubber.activate() + return client + + def tearDown(self): + pass + + def test_default_session(self): + try: + SecretCache() + except NoRegionError: + pass + + def test_client_stub(self): + SecretCache(client=self.get_client()) + + def test_get_secret_string_none(self): + cache = SecretCache(client=self.get_client()) + self.assertIsNone(cache.get_secret_string('test')) + + def test_get_secret_string_missing(self): + response = {} + versions = { + '01234567890123456789012345678901': ['AWSCURRENT'] + } + version_response = {'Name': 'test'} + cache = SecretCache(client=self.get_client(response, + versions, + version_response)) + self.assertIsNone(cache.get_secret_string('test')) + + def test_get_secret_string_no_current(self): + response = {} + versions = { + '01234567890123456789012345678901': ['NOTCURRENT'] + } + version_response = {'Name': 'test'} + cache = SecretCache(client=self.get_client(response, + versions, + version_response)) + self.assertIsNone(cache.get_secret_string('test')) + + def test_get_secret_string_no_versions(self): + response = {'Name': 'test'} + cache = SecretCache(client=self.get_client(response)) + self.assertIsNone(cache.get_secret_string('test')) + + def test_get_secret_string_empty(self): + response = {} + versions = { + '01234567890123456789012345678901': ['AWSCURRENT'] + } + version_response = {} + cache = SecretCache(client=self.get_client(response, + versions, + version_response)) + self.assertIsNone(cache.get_secret_string('test')) + + def test_get_secret_string(self): + secret = 'mysecret' + response = {} + versions = { + '01234567890123456789012345678901': ['AWSCURRENT'] + } + version_response = {'SecretString': secret} + cache = SecretCache(client=self.get_client(response, + versions, + version_response)) + for _ in range(10): + self.assertEquals(secret, cache.get_secret_string('test')) + + def test_get_secret_string_refresh(self): + secret = 'mysecret' + response = {} + versions = { + '01234567890123456789012345678901': ['AWSCURRENT'] + } + version_response = {'SecretString': secret} + cache = SecretCache( + config=SecretCacheConfig(secret_refresh_interval=1), + client=self.get_client(response, + versions, + version_response)) + for _ in range(10): + self.assertEquals(secret, cache.get_secret_string('test')) + + def test_get_secret_string_stage(self): + secret = 'mysecret' + response = {} + versions = { + '01234567890123456789012345678901': ['AWSCURRENT'] + } + version_response = {'SecretString': secret} + cache = SecretCache(client=self.get_client(response, + versions, + version_response)) + for _ in range(10): + self.assertEquals(secret, cache.get_secret_string('test', + 'AWSCURRENT')) + + def test_get_secret_string_multiple(self): + cache = SecretCache(client=self.get_client()) + for _ in range(100): + self.assertIsNone(cache.get_secret_string('test')) + + def test_get_secret_binary(self): + secret = b'01010101' + response = {} + versions = { + '01234567890123456789012345678901': ['AWSCURRENT'] + } + version_response = {'SecretBinary': secret} + cache = SecretCache(client=self.get_client(response, + versions, + version_response)) + for _ in range(10): + self.assertEquals(secret, cache.get_secret_binary('test')) + + def test_get_secret_binary_no_versions(self): + cache = SecretCache(client=self.get_client()) + self.assertIsNone(cache.get_secret_binary('test')) + + def test_get_secret_string_exception(self): + client = botocore.session.get_session().create_client( + 'secretsmanager', region_name='us-west-2') + + stubber = Stubber(client) + cache = SecretCache(client=client) + for _ in range(3): + stubber.add_client_error('describe_secret') + stubber.activate() + self.assertRaises(ClientError, cache.get_secret_binary, 'test') diff --git a/test/unit/test_config.py b/test/unit/test_config.py new file mode 100644 index 0000000..af1bdee --- /dev/null +++ b/test/unit/test_config.py @@ -0,0 +1,35 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +""" +Unit test suite for items module +""" +import unittest + +from aws_secretsmanager_caching.config import SecretCacheConfig + + +class TestSecretCacheConfig(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_simple_config(self): + self.assertRaises(TypeError, SecretCacheConfig, no='one') + + def test_config_default_version_stage(self): + stage = 'nothing' + config = SecretCacheConfig(default_version_stage=stage) + self.assertEquals(config.default_version_stage, stage) diff --git a/test/unit/test_decorators.py b/test/unit/test_decorators.py new file mode 100644 index 0000000..fc3d70d --- /dev/null +++ b/test/unit/test_decorators.py @@ -0,0 +1,190 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +""" +Unit test suite for decorators module +""" +from aws_secretsmanager_caching.decorators import InjectSecretString, InjectKeywordedSecretString +from aws_secretsmanager_caching.secret_cache import SecretCache +import botocore +from botocore.stub import Stubber +import unittest +import json + + +class TestAwsSecretsManagerCachingInjectKeywordedSecretStringDecorator(unittest.TestCase): + + def get_client(self, response={}, versions=None, version_response=None): + client = botocore.session.get_session().create_client('secretsmanager', region_name='us-west-2') + stubber = Stubber(client) + expected_params = {'SecretId': 'test'} + if versions: + response['VersionIdsToStages'] = versions + stubber.add_response('describe_secret', response, expected_params) + if version_response is not None: + stubber.add_response('get_secret_value', version_response) + stubber.activate() + return client + + def test_valid_json(self): + secret = { + 'username': 'secret_username', + 'password': 'secret_password' + } + + secret_string = json.dumps(secret) + + response = {} + versions = { + '01234567890123456789012345678901': ['AWSCURRENT'] + } + version_response = {'SecretString': secret_string} + cache = SecretCache(client=self.get_client(response, versions, version_response)) + + @InjectKeywordedSecretString(secret_id='test', cache=cache, func_username='username', func_password='password') + def function_to_be_decorated(func_username, func_password, keyworded_argument='foo'): + self.assertEqual(secret['username'], func_username) + self.assertEqual(secret['password'], func_password) + self.assertEqual(keyworded_argument, 'foo') + + function_to_be_decorated() + + def test_valid_json_with_mixed_args(self): + secret = { + 'username': 'secret_username', + 'password': 'secret_password' + } + + secret_string = json.dumps(secret) + + response = {} + versions = { + '01234567890123456789012345678901': ['AWSCURRENT'] + } + version_response = {'SecretString': secret_string} + cache = SecretCache(client=self.get_client(response, versions, version_response)) + + @InjectKeywordedSecretString(secret_id='test', cache=cache, arg2='username', arg3='password') + def function_to_be_decorated(arg1, arg2, arg3, arg4='bar'): + self.assertEqual(arg1, 'foo') + self.assertEqual(secret['username'], arg2) + self.assertEqual(secret['password'], arg3) + self.assertEqual(arg4, 'bar') + + function_to_be_decorated('foo') + + def test_valid_json_with_no_secret_kwarg(self): + secret = { + 'username': 'secret_username', + 'password': 'secret_password' + } + + secret_string = json.dumps(secret) + + response = {} + versions = { + '01234567890123456789012345678901': ['AWSCURRENT'] + } + version_response = {'SecretString': secret_string} + cache = SecretCache(client=self.get_client(response, versions, version_response)) + + @InjectKeywordedSecretString('test', cache=cache, func_username='username', func_password='password') + def function_to_be_decorated(func_username, func_password, keyworded_argument='foo'): + self.assertEqual(secret['username'], func_username) + self.assertEqual(secret['password'], func_password) + self.assertEqual(keyworded_argument, 'foo') + + function_to_be_decorated() + + def test_invalid_json(self): + secret = 'not json' + response = {} + versions = { + '01234567890123456789012345678901': ['AWSCURRENT'] + } + version_response = {'SecretString': secret} + cache = SecretCache(client=self.get_client(response, versions, version_response)) + + with self.assertRaises((RuntimeError, json.decoder.JSONDecodeError)): + @InjectKeywordedSecretString(secret_id='test', cache=cache, func_username='username', + func_passsword='password') + def function_to_be_decorated(func_username, func_password, keyworded_argument='foo'): + return + + function_to_be_decorated() + + def test_missing_key(self): + secret = {'username': 'secret_username'} + secret_string = json.dumps(secret) + response = {} + versions = { + '01234567890123456789012345678901': ['AWSCURRENT'] + } + version_response = {'SecretString': secret_string} + cache = SecretCache(client=self.get_client(response, versions, version_response)) + + with self.assertRaises((RuntimeError, ValueError)): + @InjectKeywordedSecretString(secret_id='test', cache=cache, func_username='username', + func_passsword='password') + def function_to_be_decorated(func_username, func_password, keyworded_argument='foo'): + return + + function_to_be_decorated() + + +class TestAwsSecretsManagerCachingInjectSecretStringDecorator(unittest.TestCase): + + def get_client(self, response={}, versions=None, version_response=None): + client = botocore.session.get_session().create_client('secretsmanager', region_name='us-west-2') + stubber = Stubber(client) + expected_params = {'SecretId': 'test'} + if versions: + response['VersionIdsToStages'] = versions + stubber.add_response('describe_secret', response, expected_params) + if version_response is not None: + stubber.add_response('get_secret_value', version_response) + stubber.activate() + return client + + def test_string(self): + secret = 'not json' + response = {} + versions = { + '01234567890123456789012345678901': ['AWSCURRENT'] + } + version_response = {'SecretString': secret} + cache = SecretCache(client=self.get_client(response, versions, version_response)) + + @InjectSecretString('test', cache) + def function_to_be_decorated(arg1, arg2, arg3): + self.assertEquals(arg1, secret) + self.assertEqual(arg2, 'foo') + self.assertEqual(arg3, 'bar') + + function_to_be_decorated('foo', 'bar') + + def test_string_with_additional_kwargs(self): + secret = 'not json' + response = {} + versions = { + '01234567890123456789012345678901': ['AWSCURRENT'] + } + version_response = {'SecretString': secret} + cache = SecretCache(client=self.get_client(response, versions, version_response)) + + @InjectSecretString('test', cache) + def function_to_be_decorated(arg1, arg2, arg3): + self.assertEquals(arg1, secret) + self.assertEqual(arg2, 'foo') + self.assertEqual(arg3, 'bar') + + function_to_be_decorated(arg2='foo', arg3='bar') diff --git a/test/unit/test_items.py b/test/unit/test_items.py new file mode 100644 index 0000000..7277202 --- /dev/null +++ b/test/unit/test_items.py @@ -0,0 +1,49 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +""" +Unit test suite for items module +""" +import unittest + +from aws_secretsmanager_caching.config import SecretCacheConfig +from aws_secretsmanager_caching.cache.items import SecretCacheObject + + +class TestSecretCacheObject(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + class TestObject(SecretCacheObject): + + def __init__(self, config, client, secret_id): + super(TestSecretCacheObject.TestObject, self).__init__(config, client, secret_id) + + def _execute_refresh(self): + super(TestSecretCacheObject.TestObject, self)._execute_refresh() + + def _get_version(self, version_stage): + return super(TestSecretCacheObject.TestObject, self)._get_version(version_stage) + + def test_simple(self): + sco = TestSecretCacheObject.TestObject(SecretCacheConfig(), None, None) + self.assertIsNone(sco.get_secret_value()) + + def test_simple_2(self): + sco = TestSecretCacheObject.TestObject(SecretCacheConfig(), None, None) + self.assertIsNone(sco.get_secret_value()) + sco._exception = Exception("test") + self.assertRaises(Exception, sco.get_secret_value) diff --git a/test/unit/test_lru.py b/test/unit/test_lru.py new file mode 100644 index 0000000..b656b4a --- /dev/null +++ b/test/unit/test_lru.py @@ -0,0 +1,73 @@ +# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +""" +Unit test suite for high-level functions in aws_secretsmanager_caching +""" +import unittest + +import pytest + +from aws_secretsmanager_caching.cache.lru import LRUCache + +pytestmark = [pytest.mark.unit, pytest.mark.local] + + +class TestLRUCache(unittest.TestCase): + + def test_lru_cache_max(self): + cache = LRUCache(max_size=10) + for n in range(100): + cache.put_if_absent(n, n) + for n in range(90): + self.assertIsNone(cache.get(n)) + for n in range(91, 100): + self.assertIsNotNone(cache.get(n)) + + def test_lru_cache_none(self): + cache = LRUCache(max_size=10) + self.assertIsNone(cache.get(1)) + + def test_lru_cache_recent(self): + cache = LRUCache(max_size=10) + for n in range(100): + cache.put_if_absent(n, n) + cache.get(0) + for n in range(1, 91): + self.assertIsNone(cache.get(n)) + for n in range(92, 100): + self.assertIsNotNone(cache.get(n)) + self.assertIsNotNone(cache.get(0)) + + def test_lru_cache_zero(self): + cache = LRUCache(max_size=0) + for n in range(100): + cache.put_if_absent(n, n) + self.assertIsNone(cache.get(n)) + for n in range(100): + self.assertIsNone(cache.get(n)) + + def test_lru_cache_one(self): + cache = LRUCache(max_size=1) + for n in range(100): + cache.put_if_absent(n, n) + self.assertEquals(cache.get(n), n) + for n in range(99): + self.assertIsNone(cache.get(n)) + self.assertEquals(cache.get(99), 99) + + def test_lru_cache_if_absent(self): + cache = LRUCache(max_size=1) + for n in range(100): + cache.put_if_absent(1000, 1000) + self.assertIsNone(cache.get(n)) + self.assertEquals(cache.get(1000), 1000) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..2efe3dd --- /dev/null +++ b/tox.ini @@ -0,0 +1,44 @@ +# tox (https://tox.readthedocs.io/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + + +[tox] +envlist = py36, py37, flake8, pylint, isort +skip_missing_interpreters = true + +[testenv] +setenv = PYTHONPATH = {toxinidir}/src +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands= {[testenv:pytest]commands} + +[flake8] +max-line-length = 120 +select = C,E,F,W,B +# C812, W503 clash with black +ignore = C812,W503 + +[testenv:flake8] +commands = flake8 +deps = flake8 + +[testenv:pytest] +commands=py.test \ + {env:pytest_args:} {posargs} + + +[testenv:pylint] +commands = pylint --rcfile=.pylintrc src/aws_secretsmanager_caching + +[testenv:isort] +changedir = {toxinidir} +commands = isort --recursive --check-only --diff src/aws_secretsmanager_caching + + +[testenv:docs] +description = invoke sphinx-build to build the HTML docs +commands = sphinx-build -d "{toxworkdir}/docs_doctree" doc "{toxworkdir}/docs_out" --color -bhtml {posargs} + python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' + From aa937c7340935624935d09fd7abf28045fece808 Mon Sep 17 00:00:00 2001 From: Robert Stevens Date: Wed, 6 Mar 2019 00:14:06 +0000 Subject: [PATCH 2/4] update copyright date --- src/aws_secretsmanager_caching/__init__.py | 2 +- src/aws_secretsmanager_caching/cache/__init__.py | 2 +- src/aws_secretsmanager_caching/cache/items.py | 2 +- src/aws_secretsmanager_caching/cache/lru.py | 2 +- src/aws_secretsmanager_caching/config.py | 2 +- src/aws_secretsmanager_caching/decorators.py | 2 +- src/aws_secretsmanager_caching/secret_cache.py | 2 +- test/unit/__init__.py | 2 +- test/unit/test_aws_secretsmanager_caching.py | 2 +- test/unit/test_config.py | 2 +- test/unit/test_decorators.py | 2 +- test/unit/test_items.py | 2 +- test/unit/test_lru.py | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/aws_secretsmanager_caching/__init__.py b/src/aws_secretsmanager_caching/__init__.py index 06bfc8d..67d114f 100644 --- a/src/aws_secretsmanager_caching/__init__.py +++ b/src/aws_secretsmanager_caching/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of diff --git a/src/aws_secretsmanager_caching/cache/__init__.py b/src/aws_secretsmanager_caching/cache/__init__.py index 955a16a..8fce9dd 100644 --- a/src/aws_secretsmanager_caching/cache/__init__.py +++ b/src/aws_secretsmanager_caching/cache/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of diff --git a/src/aws_secretsmanager_caching/cache/items.py b/src/aws_secretsmanager_caching/cache/items.py index 1cc6f44..264c4d2 100644 --- a/src/aws_secretsmanager_caching/cache/items.py +++ b/src/aws_secretsmanager_caching/cache/items.py @@ -1,4 +1,4 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of diff --git a/src/aws_secretsmanager_caching/cache/lru.py b/src/aws_secretsmanager_caching/cache/lru.py index 2c1bb4b..513e826 100644 --- a/src/aws_secretsmanager_caching/cache/lru.py +++ b/src/aws_secretsmanager_caching/cache/lru.py @@ -1,4 +1,4 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of diff --git a/src/aws_secretsmanager_caching/config.py b/src/aws_secretsmanager_caching/config.py index 5b98388..010f727 100644 --- a/src/aws_secretsmanager_caching/config.py +++ b/src/aws_secretsmanager_caching/config.py @@ -1,4 +1,4 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of diff --git a/src/aws_secretsmanager_caching/decorators.py b/src/aws_secretsmanager_caching/decorators.py index 77caf98..ca8bfb7 100644 --- a/src/aws_secretsmanager_caching/decorators.py +++ b/src/aws_secretsmanager_caching/decorators.py @@ -1,4 +1,4 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of diff --git a/src/aws_secretsmanager_caching/secret_cache.py b/src/aws_secretsmanager_caching/secret_cache.py index 514a174..e6e69d9 100644 --- a/src/aws_secretsmanager_caching/secret_cache.py +++ b/src/aws_secretsmanager_caching/secret_cache.py @@ -1,4 +1,4 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 1ccc7fa..cf81c33 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of diff --git a/test/unit/test_aws_secretsmanager_caching.py b/test/unit/test_aws_secretsmanager_caching.py index 5ecf17e..febfb46 100644 --- a/test/unit/test_aws_secretsmanager_caching.py +++ b/test/unit/test_aws_secretsmanager_caching.py @@ -1,4 +1,4 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of diff --git a/test/unit/test_config.py b/test/unit/test_config.py index af1bdee..43ca4de 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -1,4 +1,4 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of diff --git a/test/unit/test_decorators.py b/test/unit/test_decorators.py index fc3d70d..455e62b 100644 --- a/test/unit/test_decorators.py +++ b/test/unit/test_decorators.py @@ -1,4 +1,4 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of diff --git a/test/unit/test_items.py b/test/unit/test_items.py index 7277202..25d368c 100644 --- a/test/unit/test_items.py +++ b/test/unit/test_items.py @@ -1,4 +1,4 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of diff --git a/test/unit/test_lru.py b/test/unit/test_lru.py index b656b4a..99641e1 100644 --- a/test/unit/test_lru.py +++ b/test/unit/test_lru.py @@ -1,4 +1,4 @@ -# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of From 80fd8b78497c5033f2aa62940a6fb36de626279a Mon Sep 17 00:00:00 2001 From: Robert Stevens Date: Wed, 6 Mar 2019 14:54:33 -0800 Subject: [PATCH 3/4] rework distutil setup, requirements, update code coverage, set user agent string --- .gitignore | 3 ++- test-requirements.txt => dev-requirements.txt | 2 +- setup.cfg | 1 + setup.py | 9 ++++++--- src/aws_secretsmanager_caching/secret_cache.py | 5 ++++- tox.ini | 4 ++-- 6 files changed, 16 insertions(+), 8 deletions(-) rename test-requirements.txt => dev-requirements.txt (82%) diff --git a/.gitignore b/.gitignore index 71ccd12..6216102 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ dist/ .python-version .tox/ venv/ -env/ \ No newline at end of file +env/ +version.txt diff --git a/test-requirements.txt b/dev-requirements.txt similarity index 82% rename from test-requirements.txt rename to dev-requirements.txt index 258956f..dd22b02 100644 --- a/test-requirements.txt +++ b/dev-requirements.txt @@ -2,7 +2,7 @@ pytest<3.8.0 pytest-cov pytest-sugar codecov>=1.4.0 -botocore>=1.12 pylint>1.9.4 isort>=4.3.4 sphinx>=1.8.4 +setuptools_scm>=3.2 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 00af076..357c04d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,6 +5,7 @@ addopts = --doctest-modules --cov aws_secretsmanager_caching --cov-fail-under 90 + --cov-report term-missing test [aliases] diff --git a/setup.py b/setup.py index 9741ab8..18f1d2f 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,6 @@ url="https://aws.amazon.com/secrets-manager/", author="Amazon Web Services", author_email="aws-secretsmanager-dev@amazon.com", - version="1.0", packages=find_packages(where="src", exclude=("test",)), package_dir={"": "src"}, classifiers=[ @@ -17,8 +16,12 @@ 'Programming Language :: Python :: 3.7' ], keywords='secretsmanager secrets manager development cache caching client', + use_scm_version={ + 'write_to': 'version.txt' + }, python_requires='>3.5', - setup_requires=['pytest-runner'], - tests_require=['pytest<3.8.0', 'pytest-cov', 'pytest-sugar', 'codecov>=1.4.0'] + install_requires=['botocore'], + setup_requires=['pytest-runner', 'setuptools-scm'], + tests_require=['pytest', 'pytest-cov', 'pytest-sugar', 'codecov'] ) diff --git a/src/aws_secretsmanager_caching/secret_cache.py b/src/aws_secretsmanager_caching/secret_cache.py index e6e69d9..4352006 100644 --- a/src/aws_secretsmanager_caching/secret_cache.py +++ b/src/aws_secretsmanager_caching/secret_cache.py @@ -14,6 +14,7 @@ from copy import deepcopy import botocore.session +from setuptools_scm import get_version from .cache import LRUCache, SecretCacheItem from .config import SecretCacheConfig @@ -30,7 +31,7 @@ def __init__(self, config=SecretCacheConfig(), client=None): :param config: Secret cache configuration :type client: botocore.client.BaseClient - :param client: boto 'secretsmanager' client + :param client: botocore 'secretsmanager' client """ self._client = client self._config = deepcopy(config) @@ -38,6 +39,8 @@ def __init__(self, config=SecretCacheConfig(), client=None): if self._client is None: self._client = botocore.session.get_session().create_client("secretsmanager") + self._client.meta.config.user_agent_extra = "AwsSecretCache/{}".format(get_version()) + def _get_cached_secret(self, secret_id): """Get a cached secret for the given secret identifier. diff --git a/tox.ini b/tox.ini index 2efe3dd..3835a76 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ skip_missing_interpreters = true [testenv] setenv = PYTHONPATH = {toxinidir}/src deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt + -r{toxinidir}/dev-requirements.txt commands= {[testenv:pytest]commands} [flake8] @@ -26,7 +26,7 @@ deps = flake8 [testenv:pytest] commands=py.test \ - {env:pytest_args:} {posargs} + {env:pytest_args:} {posargs} -s [testenv:pylint] From 94daa3cd3358d198bd85b5133ffd2ef3fe2ca80b Mon Sep 17 00:00:00 2001 From: Robert Stevens Date: Thu, 7 Mar 2019 17:22:18 -0800 Subject: [PATCH 4/4] update version string in docs correctly --- doc/conf.py | 3 ++- tox.ini | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index e910d7b..7f7ce5a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,8 +1,9 @@ from datetime import datetime +from setuptools_scm import get_version import os import shutil -version = '1.0' +version = get_version(root='..') project = u'AWS Secrets Manager Python Caching Client' # If you use autosummary, this ensures that any stale autogenerated files are diff --git a/tox.ini b/tox.ini index 3835a76..5484ded 100644 --- a/tox.ini +++ b/tox.ini @@ -38,6 +38,9 @@ commands = isort --recursive --check-only --diff src/aws_secretsmanager_caching [testenv:docs] +# for dev purposes when git tag is not present or invalid +passenv = SETUPTOOLS_SCM_PRETEND_VERSION +changedir = {toxinidir} description = invoke sphinx-build to build the HTML docs commands = sphinx-build -d "{toxworkdir}/docs_doctree" doc "{toxworkdir}/docs_out" --color -bhtml {posargs} python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))'