From c6c5eb0175bcde07c11d7eb6f66eb820f86e1b61 Mon Sep 17 00:00:00 2001 From: marysieek Date: Thu, 2 Jul 2020 22:53:12 +0200 Subject: [PATCH 1/9] Use correct development file extension --- DEVELOPMENT.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 DEVELOPMENT.rst diff --git a/DEVELOPMENT.rst b/DEVELOPMENT.rst new file mode 100644 index 0000000..e11bb92 --- /dev/null +++ b/DEVELOPMENT.rst @@ -0,0 +1,36 @@ +Installation +------------ + +.. code-block:: console + + $ git clone git@github.com:castle/castle-python.git + $ cd castle-python + $ python3 setup.py install + + +Test +------------ + +.. code-block:: console + + $ python3 setup.py test + +Linting +------------ + +.. code-block:: console + + $ pip3 install pylint + $ pip3 install setuptools-lint + $ pip3 install --upgrade pep8 + $ pip3 install --upgrade autopep8 + $ python3 setup.py lint + $ autopep8 --in-place -r castle + +Coverage +------------ + +.. code-block:: console + + $ pip3 install coverage + $ coverage run setup.py test From b03804ec004937d16e9645e72ea3218acdbf1a5e Mon Sep 17 00:00:00 2001 From: marysieek Date: Thu, 2 Jul 2020 22:59:28 +0200 Subject: [PATCH 2/9] Add allowlisted config in readme --- DEVELOPMENT.md | 36 ------------------------------------ README.rst | 17 +++++++++-------- 2 files changed, 9 insertions(+), 44 deletions(-) delete mode 100644 DEVELOPMENT.md diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md deleted file mode 100644 index e11bb92..0000000 --- a/DEVELOPMENT.md +++ /dev/null @@ -1,36 +0,0 @@ -Installation ------------- - -.. code-block:: console - - $ git clone git@github.com:castle/castle-python.git - $ cd castle-python - $ python3 setup.py install - - -Test ------------- - -.. code-block:: console - - $ python3 setup.py test - -Linting ------------- - -.. code-block:: console - - $ pip3 install pylint - $ pip3 install setuptools-lint - $ pip3 install --upgrade pep8 - $ pip3 install --upgrade autopep8 - $ python3 setup.py lint - $ autopep8 --in-place -r castle - -Coverage ------------- - -.. code-block:: console - - $ pip3 install coverage - $ coverage run setup.py test diff --git a/README.rst b/README.rst index 1591c2e..ee45e1b 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ import and configure the library with your Castle API secret. .. code:: python - from castle.configuration import configuration, WHITELISTED + from castle.configuration import configuration, ALLOWLISTED # Same as setting it through Castle.api_secret configuration.api_secret = ':YOUR-API-SECRET' @@ -32,16 +32,17 @@ import and configure the library with your Castle API secret. # Castle::RequestError is raised when timing out in milliseconds (default: 500 milliseconds) configuration.request_timeout = 1000 - # Whitelisted and Blacklisted headers are case insensitive and allow to use _ and - as a separator, http prefixes are removed + # Allowlisted and Denylisted headers are case insensitive + # and allow to use _ and - as a separator, http prefixes are removed # By default all headers are passed, but some are automatically scrubbed. - # If you need to apply a whitelist, we recommend using the minimum set of - # standard headers that we've exposed in the `WHITELISTED` constant. - # Whitelisted headers - configuration.whitelisted = WHITELISTED + ['X_HEADER'] + # If you need to apply an allowlist, we recommend using the minimum set of + # standard headers that we've exposed in the `ALLOWLISTED` constant. + # Allowlisted headers + configuration.allowlisted = ALLOWLISTED + ['X_HEADER'] - # Blacklisted headers take advantage over whitelisted elements. Note that + # Denylisted headers take advantage over allowlisted elements. Note that # some headers are always scrubbed, for security reasons. - configuration.blacklisted = ['HTTP-X-header'] + configuration.denylisted = ['HTTP-X-header'] # Castle needs the original IP of the client, not the IP of your proxy or load balancer. # The SDK will only trust the proxy chain as defined in the configuration. From 0b9aafc7e15370a870a4f4ff94a06ce9d2de365f Mon Sep 17 00:00:00 2001 From: marysieek Date: Thu, 2 Jul 2020 23:00:20 +0200 Subject: [PATCH 3/9] Use allowlisted and denylisted headers in configuration --- castle/configuration.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/castle/configuration.py b/castle/configuration.py index b857c1c..8caea16 100644 --- a/castle/configuration.py +++ b/castle/configuration.py @@ -44,8 +44,8 @@ def __init__(self): self.host = HOST self.port = PORT self.url_prefix = URL_PREFIX - self.whitelisted = [] - self.blacklisted = [] + self.allowlisted = [] + self.denylisted = [] self.request_timeout = REQUEST_TIMEOUT self.failover_strategy = FAILOVER_STRATEGY self.ip_headers = [] @@ -89,26 +89,26 @@ def url_prefix(self, value): self.__url_prefix = value @property - def whitelisted(self): - return self.__whitelisted + def allowlisted(self): + return self.__allowlisted - @whitelisted.setter - def whitelisted(self, value): + @allowlisted.setter + def allowlisted(self, value): if value: - self.__whitelisted = [HeadersFormatter.call(v) for v in value] + self.__allowlisted = [HeadersFormatter.call(v) for v in value] else: - self.__whitelisted = [] + self.__allowlisted = [] @property - def blacklisted(self): - return self.__blacklisted + def denylisted(self): + return self.__denylisted - @blacklisted.setter - def blacklisted(self, value): + @denylisted.setter + def denylisted(self, value): if value: - self.__blacklisted = [HeadersFormatter.call(v) for v in value] + self.__denylisted = [HeadersFormatter.call(v) for v in value] else: - self.__blacklisted = [] + self.__denylisted = [] @property def request_timeout(self): From f29f512d33eadb9deb04dde4e35c608e0e018450 Mon Sep 17 00:00:00 2001 From: marysieek Date: Thu, 2 Jul 2020 23:12:41 +0200 Subject: [PATCH 4/9] Rename denylist to blocklist --- README.rst | 6 +++--- castle/configuration.py | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index ee45e1b..ecf0ec4 100644 --- a/README.rst +++ b/README.rst @@ -32,7 +32,7 @@ import and configure the library with your Castle API secret. # Castle::RequestError is raised when timing out in milliseconds (default: 500 milliseconds) configuration.request_timeout = 1000 - # Allowlisted and Denylisted headers are case insensitive + # Allowlisted and Blocklisted headers are case insensitive # and allow to use _ and - as a separator, http prefixes are removed # By default all headers are passed, but some are automatically scrubbed. # If you need to apply an allowlist, we recommend using the minimum set of @@ -40,9 +40,9 @@ import and configure the library with your Castle API secret. # Allowlisted headers configuration.allowlisted = ALLOWLISTED + ['X_HEADER'] - # Denylisted headers take advantage over allowlisted elements. Note that + # Blocklisted headers take advantage over allowlisted elements. Note that # some headers are always scrubbed, for security reasons. - configuration.denylisted = ['HTTP-X-header'] + configuration.blocklisted = ['HTTP-X-header'] # Castle needs the original IP of the client, not the IP of your proxy or load balancer. # The SDK will only trust the proxy chain as defined in the configuration. diff --git a/castle/configuration.py b/castle/configuration.py index 8caea16..9f3fc4a 100644 --- a/castle/configuration.py +++ b/castle/configuration.py @@ -45,7 +45,7 @@ def __init__(self): self.port = PORT self.url_prefix = URL_PREFIX self.allowlisted = [] - self.denylisted = [] + self.blocklisted = [] self.request_timeout = REQUEST_TIMEOUT self.failover_strategy = FAILOVER_STRATEGY self.ip_headers = [] @@ -100,15 +100,15 @@ def allowlisted(self, value): self.__allowlisted = [] @property - def denylisted(self): - return self.__denylisted + def blocklisted(self): + return self.__blocklisted - @denylisted.setter - def denylisted(self, value): + @blocklisted.setter + def blocklisted(self, value): if value: - self.__denylisted = [HeadersFormatter.call(v) for v in value] + self.__blocklisted = [HeadersFormatter.call(v) for v in value] else: - self.__denylisted = [] + self.__blocklisted = [] @property def request_timeout(self): From 4e24d8a52b0b6537c15b75452ef7bf99649b5780 Mon Sep 17 00:00:00 2001 From: marysieek Date: Fri, 3 Jul 2020 11:08:43 +0200 Subject: [PATCH 5/9] Use denylisted --- README.rst | 6 ++-- castle/configuration.py | 14 ++++----- castle/extractors/headers.py | 14 ++++----- castle/test/configuration_test.py | 40 +++++++++++++------------- castle/test/extractors/headers_test.py | 22 +++++++------- 5 files changed, 48 insertions(+), 48 deletions(-) diff --git a/README.rst b/README.rst index ecf0ec4..ee45e1b 100644 --- a/README.rst +++ b/README.rst @@ -32,7 +32,7 @@ import and configure the library with your Castle API secret. # Castle::RequestError is raised when timing out in milliseconds (default: 500 milliseconds) configuration.request_timeout = 1000 - # Allowlisted and Blocklisted headers are case insensitive + # Allowlisted and Denylisted headers are case insensitive # and allow to use _ and - as a separator, http prefixes are removed # By default all headers are passed, but some are automatically scrubbed. # If you need to apply an allowlist, we recommend using the minimum set of @@ -40,9 +40,9 @@ import and configure the library with your Castle API secret. # Allowlisted headers configuration.allowlisted = ALLOWLISTED + ['X_HEADER'] - # Blocklisted headers take advantage over allowlisted elements. Note that + # Denylisted headers take advantage over allowlisted elements. Note that # some headers are always scrubbed, for security reasons. - configuration.blocklisted = ['HTTP-X-header'] + configuration.denylisted = ['HTTP-X-header'] # Castle needs the original IP of the client, not the IP of your proxy or load balancer. # The SDK will only trust the proxy chain as defined in the configuration. diff --git a/castle/configuration.py b/castle/configuration.py index 9f3fc4a..8caea16 100644 --- a/castle/configuration.py +++ b/castle/configuration.py @@ -45,7 +45,7 @@ def __init__(self): self.port = PORT self.url_prefix = URL_PREFIX self.allowlisted = [] - self.blocklisted = [] + self.denylisted = [] self.request_timeout = REQUEST_TIMEOUT self.failover_strategy = FAILOVER_STRATEGY self.ip_headers = [] @@ -100,15 +100,15 @@ def allowlisted(self, value): self.__allowlisted = [] @property - def blocklisted(self): - return self.__blocklisted + def denylisted(self): + return self.__denylisted - @blocklisted.setter - def blocklisted(self, value): + @denylisted.setter + def denylisted(self, value): if value: - self.__blocklisted = [HeadersFormatter.call(v) for v in value] + self.__denylisted = [HeadersFormatter.call(v) for v in value] else: - self.__blocklisted = [] + self.__denylisted = [] @property def request_timeout(self): diff --git a/castle/extractors/headers.py b/castle/extractors/headers.py index d42b712..fce20fc 100644 --- a/castle/extractors/headers.py +++ b/castle/extractors/headers.py @@ -1,13 +1,13 @@ from castle.configuration import configuration -ALWAYS_BLACKLISTED = ['Cookie', 'Authorization'] -ALWAYS_WHITELISTED = ['User-Agent'] +ALWAYS_DENYLISTED = ['Cookie', 'Authorization'] +ALWAYS_ALLOWLISTED = ['User-Agent'] class ExtractorsHeaders(object): def __init__(self, headers): self.headers = headers - self.no_whitelist = len(configuration.whitelisted) == 0 + self.no_whitelist = len(configuration.allowlisted) == 0 def call(self): result = dict() @@ -18,13 +18,13 @@ def call(self): return result def _header_value(self, name, value): - if name in ALWAYS_BLACKLISTED: + if name in ALWAYS_DENYLISTED: return True - if name in ALWAYS_WHITELISTED: + if name in ALWAYS_ALLOWLISTED: return value - if name in configuration.blacklisted: + if name in configuration.denylisted: return True - if self.no_whitelist or (name in configuration.whitelisted): + if self.no_whitelist or (name in configuration.allowlisted): return value return True diff --git a/castle/test/configuration_test.py b/castle/test/configuration_test.py index a2aedf1..12bfb92 100644 --- a/castle/test/configuration_test.py +++ b/castle/test/configuration_test.py @@ -10,8 +10,8 @@ def test_default_values(self): self.assertEqual(config.host, 'api.castle.io') self.assertEqual(config.port, 443) self.assertEqual(config.url_prefix, '/v1') - self.assertEqual(config.whitelisted, []) - self.assertEqual(config.blacklisted, []) + self.assertEqual(config.allowlisted, []) + self.assertEqual(config.denylisted, []) self.assertEqual(config.request_timeout, 500) self.assertEqual(config.failover_strategy, 'allow') self.assertEqual(config.ip_headers, []) @@ -37,35 +37,35 @@ def test_url_prefix_setter(self): config.url_prefix = '/v2' self.assertEqual(config.url_prefix, '/v2') - def test_whitelisted_setter_list(self): + def test_allowlisted_setter_list(self): config = Configuration() - config.whitelisted = ['test'] - self.assertEqual(config.whitelisted, ['Test']) + config.allowlisted = ['test'] + self.assertEqual(config.allowlisted, ['Test']) - def test_whitelisted_setter_none(self): + def test_allowlisted_setter_none(self): config = Configuration() - config.whitelisted = None - self.assertEqual(config.whitelisted, []) + config.allowlisted = None + self.assertEqual(config.allowlisted, []) - def test_whitelisted_setter_empty(self): + def test_allowlisted_setter_empty(self): config = Configuration() - config.whitelisted = '' - self.assertEqual(config.whitelisted, []) + config.allowlisted = '' + self.assertEqual(config.allowlisted, []) - def test_blacklisted_setter_list(self): + def test_denylisted_setter_list(self): config = Configuration() - config.blacklisted = ['test'] - self.assertEqual(config.blacklisted, ['Test']) + config.denylisted = ['test'] + self.assertEqual(config.denylisted, ['Test']) - def test_blacklisted_setter_none(self): + def test_denylisted_setter_none(self): config = Configuration() - config.blacklisted = None - self.assertEqual(config.blacklisted, []) + config.denylisted = None + self.assertEqual(config.denylisted, []) - def test_blacklisted_setter_empty(self): + def test_denylisted_setter_empty(self): config = Configuration() - config.blacklisted = '' - self.assertEqual(config.blacklisted, []) + config.denylisted = '' + self.assertEqual(config.denylisted, []) def test_request_timeout_setter(self): config = Configuration() diff --git a/castle/test/extractors/headers_test.py b/castle/test/extractors/headers_test.py index 14ab93f..b9d25a3 100644 --- a/castle/test/extractors/headers_test.py +++ b/castle/test/extractors/headers_test.py @@ -18,8 +18,8 @@ def formatted_headers(): class ExtractorsHeadersTestCase(unittest.TestCase): def tearDown(self): - configuration.whitelisted = [] - configuration.blacklisted = [] + configuration.allowlisted = [] + configuration.denylisted = [] def test_extract_headers(self): self.assertEqual(ExtractorsHeaders(formatted_headers()).call(), @@ -32,8 +32,8 @@ def test_extract_headers(self): 'X-Forwarded-For': '1.2.3.4' }) - def test_whitelisted_headers(self): - configuration.whitelisted = ['Accept', 'OK'] + def test_allowlisted_headers(self): + configuration.allowlisted = ['Accept', 'OK'] self.assertEqual( ExtractorsHeaders(formatted_headers()).call(), {'Accept': 'application/json', @@ -47,8 +47,8 @@ def test_whitelisted_headers(self): ) # - def test_restricted_blacklisted_headers(self): - configuration.blacklisted = ['User-Agent'] + def test_restricted_denylisted_headers(self): + configuration.denylisted = ['User-Agent'] self.assertEqual( ExtractorsHeaders(formatted_headers()).call(), {'Accept': 'application/json', @@ -61,8 +61,8 @@ def test_restricted_blacklisted_headers(self): } ) - def test_blacklisted_headers(self): - configuration.blacklisted = ['Accept'] + def test_denylisted_headers(self): + configuration.denylisted = ['Accept'] self.assertEqual( ExtractorsHeaders(formatted_headers()).call(), {'Accept': True, @@ -76,9 +76,9 @@ def test_blacklisted_headers(self): ) # - def test_blacklisted_and_whitelisted_headers(self): - configuration.blacklisted = ['Accept'] - configuration.whitelisted = ['Accept'] + def test_denylisted_and_allowlisted_headers(self): + configuration.denylisted = ['Accept'] + configuration.allowlisted = ['Accept'] self.assertEqual( ExtractorsHeaders(formatted_headers()).call()['Accept'], True ) From 551830d3541cbc89ce76200348a5152cef863e8c Mon Sep 17 00:00:00 2001 From: marysieek Date: Fri, 3 Jul 2020 11:10:55 +0200 Subject: [PATCH 6/9] Post-lint update --- castle/configuration.py | 1 + castle/extractors/ip.py | 1 + 2 files changed, 2 insertions(+) diff --git a/castle/configuration.py b/castle/configuration.py index 8caea16..a25f48d 100644 --- a/castle/configuration.py +++ b/castle/configuration.py @@ -173,5 +173,6 @@ def trusted_proxy_depth(self, value): else: raise ConfigurationError + # pylint: disable=invalid-name configuration = Configuration() diff --git a/castle/extractors/ip.py b/castle/extractors/ip.py index d2af4f6..a0416c8 100644 --- a/castle/extractors/ip.py +++ b/castle/extractors/ip.py @@ -7,6 +7,7 @@ # list of header which are used with proxy depth setting DEPTH_RELATED = ['X-Forwarded-For'] + class ExtractorsIp(object): def __init__(self, headers): self.headers = headers From 7a6bb2c1638e3b1ba118d8a117b8e2b457c30e01 Mon Sep 17 00:00:00 2001 From: marysieek Date: Fri, 3 Jul 2020 14:31:08 +0200 Subject: [PATCH 7/9] Update default allowlist --- castle/configuration.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/castle/configuration.py b/castle/configuration.py index a25f48d..6875ec4 100644 --- a/castle/configuration.py +++ b/castle/configuration.py @@ -1,7 +1,7 @@ from castle.exceptions import ConfigurationError from castle.headers_formatter import HeadersFormatter -DEFAULT_WHITELIST = [ +DEFAULT_ALLOWLIST = [ "Accept", "Accept-Charset", "Accept-Datetime", @@ -11,11 +11,15 @@ "Connection", "Content-Length", "Content-Type", - "Cookie", + "Dnt", "Host", "Origin", "Pragma", "Referer", + "Sec-Fetch-Dest", + "Sec-Fetch-Mode", + "Sec-Fetch-Site", + "Sec-Fetch-User", "TE", "Upgrade-Insecure-Requests", "User-Agent", From deb9ea3b9ec49d6ab68805ebb1e077c949b6416a Mon Sep 17 00:00:00 2001 From: marysieek Date: Fri, 3 Jul 2020 15:17:10 +0200 Subject: [PATCH 8/9] Add additional headers test context --- README.rst | 6 +++--- castle/test/extractors/headers_test.py | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index ee45e1b..5e196f1 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ import and configure the library with your Castle API secret. .. code:: python - from castle.configuration import configuration, ALLOWLISTED + from castle.configuration import configuration, DEFAULT_ALLOWLIST # Same as setting it through Castle.api_secret configuration.api_secret = ':YOUR-API-SECRET' @@ -36,9 +36,9 @@ import and configure the library with your Castle API secret. # and allow to use _ and - as a separator, http prefixes are removed # By default all headers are passed, but some are automatically scrubbed. # If you need to apply an allowlist, we recommend using the minimum set of - # standard headers that we've exposed in the `ALLOWLISTED` constant. + # standard headers that we've exposed in the `DEFAULT_ALLOWLIST` constant. # Allowlisted headers - configuration.allowlisted = ALLOWLISTED + ['X_HEADER'] + configuration.allowlisted = DEFAULT_ALLOWLIST + ['X_HEADER'] # Denylisted headers take advantage over allowlisted elements. Note that # some headers are always scrubbed, for security reasons. diff --git a/castle/test/extractors/headers_test.py b/castle/test/extractors/headers_test.py index b9d25a3..64fc7af 100644 --- a/castle/test/extractors/headers_test.py +++ b/castle/test/extractors/headers_test.py @@ -1,5 +1,5 @@ from castle.test import unittest -from castle.configuration import configuration +from castle.configuration import configuration, DEFAULT_ALLOWLIST from castle.extractors.headers import ExtractorsHeaders @@ -45,7 +45,20 @@ def test_allowlisted_headers(self): 'X-Forwarded-For': True } ) -# + + def test_only_default_allowlisted_headers(self): + configuration.allowlisted = DEFAULT_ALLOWLIST + self.assertEqual( + ExtractorsHeaders(formatted_headers()).call(), + {'Accept': 'application/json', + 'Authorization': True, + 'Cookie': True, + 'Ok': True, + 'Content-Length': '0', + 'User-Agent': 'Mozilla 1234', + 'X-Forwarded-For': True + } + ) def test_restricted_denylisted_headers(self): configuration.denylisted = ['User-Agent'] @@ -74,7 +87,6 @@ def test_denylisted_headers(self): 'X-Forwarded-For': '1.2.3.4' } ) -# def test_denylisted_and_allowlisted_headers(self): configuration.denylisted = ['Accept'] From 806df6b351de66956d8339b952b8b554d8973dd9 Mon Sep 17 00:00:00 2001 From: marysieek Date: Mon, 6 Jul 2020 10:44:32 +0200 Subject: [PATCH 9/9] Trigger rebuild