From eee83a6035158e47f78c981658b3beff697569bc Mon Sep 17 00:00:00 2001 From: Johanna Larsson Date: Mon, 18 Nov 2019 11:04:35 +0100 Subject: [PATCH 1/9] temp --- castle/configuration.py | 33 ++++++++++++++------------ castle/extractors/headers.py | 3 ++- castle/test/extractors/headers_test.py | 8 +++---- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/castle/configuration.py b/castle/configuration.py index f1049ea..11504f9 100644 --- a/castle/configuration.py +++ b/castle/configuration.py @@ -1,23 +1,26 @@ from castle.exceptions import ConfigurationError from castle.headers_formatter import HeadersFormatter +BLACKLISTED = ['HTTP_COOKIE'] + WHITELISTED = [ - 'User-Agent', - 'Accept-Language', - 'Accept-Encoding', - 'Accept-Charset', - 'Accept', - 'Accept-Datetime', - 'Forwarded', - 'X-Forwarded', - 'X-Real-IP', - 'REMOTE_ADDR', - 'X-Forwarded-For', - 'CF_CONNECTING_IP' + "Accept", + "Accept-Charset", + "Accept-Datetime", + "Accept-Encoding", + "Accept-Language", + "Cache-Control", + "Connection", + "Content-Length", + "Content-Type", + "Cookie", + "Host", + "Origin", + "Pragma", + "Referer", + "User-Agent", ] -BLACKLISTED = ['HTTP_COOKIE'] - # 500 milliseconds REQUEST_TIMEOUT = 500 FAILOVER_STRATEGIES = ['allow', 'deny', 'challenge', 'throw'] @@ -29,7 +32,7 @@ def __init__(self): self.host = 'api.castle.io' self.port = 443 self.url_prefix = '/v1' - self.whitelisted = WHITELISTED + self.whitelisted = [] self.blacklisted = BLACKLISTED self.request_timeout = REQUEST_TIMEOUT self.failover_strategy = 'allow' diff --git a/castle/extractors/headers.py b/castle/extractors/headers.py index d9c7a56..d495cb1 100644 --- a/castle/extractors/headers.py +++ b/castle/extractors/headers.py @@ -9,10 +9,11 @@ def __init__(self, environ): def call(self): headers = dict() + has_whitelist = len(configuration.whitelisted) > 0 for key, value in self.environ.items(): name = self.formatter.call(key) - if name not in configuration.whitelisted: + if has_whitelist and name not in configuration.whitelisted: continue if name in configuration.blacklisted: continue diff --git a/castle/test/extractors/headers_test.py b/castle/test/extractors/headers_test.py index 6d149b1..ee62f8c 100644 --- a/castle/test/extractors/headers_test.py +++ b/castle/test/extractors/headers_test.py @@ -9,7 +9,7 @@ def client_id(): def environ(): return { - 'HTTP_X_FORWARDED_FOR': '1.2.3.4', + 'HTTP_USER_AGENT': 'requests', 'HTTP_OK': 'OK', 'TEST': '1', 'HTTP_COOKIE': "__cid={client_id};other=efgh".format(client_id=client_id) @@ -19,12 +19,12 @@ def environ(): class ExtractorsHeadersTestCase(unittest.TestCase): def test_extract_headers(self): self.assertEqual(ExtractorsHeaders(environ()).call(), - {'X-Forwarded-For': '1.2.3.4'}) + {'Test': '1', 'User-Agent': 'requests'}) - def test_extend_whitelisted_headers(self): + def test_add_whitelisted_headers(self): configuration.whitelisted += ['TEST'] self.assertEqual( ExtractorsHeaders(environ()).call(), - {'X-Forwarded-For': '1.2.3.4', 'Test': '1'} + {'User-Agent': 'requests', 'Test': '1'} ) configuration.whitelisted.remove('Test') From f0aa76643a095d47a80e5b40b6dc202ed8d3fce9 Mon Sep 17 00:00:00 2001 From: Johanna Larsson Date: Mon, 18 Nov 2019 15:21:52 +0100 Subject: [PATCH 2/9] fix tests --- castle/test/client_test.py | 12 ++++++++++-- castle/test/configuration_test.py | 3 +-- castle/test/extractors/headers_test.py | 8 ++++---- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/castle/test/client_test.py b/castle/test/client_test.py index 9f1618d..7a46ae4 100644 --- a/castle/test/client_test.py +++ b/castle/test/client_test.py @@ -31,7 +31,11 @@ def test_init(self): context = { 'active': True, 'client_id': '1234', - 'headers': {'User-Agent': 'test', 'X-Forwarded-For': '217.144.192.112'}, + 'headers': { + 'User-Agent': 'test', + 'X-Forwarded-For': '217.144.192.112', + 'X-Castle-Client-Id': '1234' + }, 'ip': '217.144.192.112', 'library': {'name': 'castle-python', 'version': VERSION}, 'origin': 'web', @@ -189,7 +193,11 @@ def test_to_context(self): context = { 'active': True, 'client_id': '1234', - 'headers': {'User-Agent': 'test', 'X-Forwarded-For': '217.144.192.112'}, + 'headers': { + 'User-Agent': 'test', + 'X-Forwarded-For': '217.144.192.112', + 'X-Castle-Client-Id': '1234' + }, 'ip': '217.144.192.112', 'library': {'name': 'castle-python', 'version': VERSION}, 'origin': 'web', diff --git a/castle/test/configuration_test.py b/castle/test/configuration_test.py index f263133..cbc4b33 100644 --- a/castle/test/configuration_test.py +++ b/castle/test/configuration_test.py @@ -11,8 +11,7 @@ def test_default_values(self): self.assertEqual(config.host, 'api.castle.io') self.assertEqual(config.port, 443) self.assertEqual(config.url_prefix, '/v1') - self.assertEqual(config.whitelisted, [ - HeadersFormatter.call(v) for v in WHITELISTED]) + self.assertEqual(config.whitelisted, []) self.assertEqual(config.blacklisted, [ HeadersFormatter.call(v) for v in BLACKLISTED]) self.assertEqual(config.request_timeout, 500) diff --git a/castle/test/extractors/headers_test.py b/castle/test/extractors/headers_test.py index ee62f8c..d2e3845 100644 --- a/castle/test/extractors/headers_test.py +++ b/castle/test/extractors/headers_test.py @@ -1,5 +1,5 @@ from castle.test import unittest -from castle.configuration import configuration +from castle.configuration import configuration, WHITELISTED from castle.extractors.headers import ExtractorsHeaders @@ -19,12 +19,12 @@ def environ(): class ExtractorsHeadersTestCase(unittest.TestCase): def test_extract_headers(self): self.assertEqual(ExtractorsHeaders(environ()).call(), - {'Test': '1', 'User-Agent': 'requests'}) + {'User-Agent': 'requests', 'Ok': 'OK', 'Test': '1'}) def test_add_whitelisted_headers(self): - configuration.whitelisted += ['TEST'] + configuration.whitelisted = WHITELISTED + ['TEST'] self.assertEqual( ExtractorsHeaders(environ()).call(), {'User-Agent': 'requests', 'Test': '1'} ) - configuration.whitelisted.remove('Test') + configuration.whitelisted = [] From 1cf1e8c4bfb4c4937d838e95d7f5716e568e3203 Mon Sep 17 00:00:00 2001 From: Johanna Larsson Date: Mon, 18 Nov 2019 15:31:34 +0100 Subject: [PATCH 3/9] Rename whitelist/blacklist according to spec --- README.rst | 18 +++++------ castle/configuration.py | 32 +++++++++---------- castle/extractors/headers.py | 6 ++-- castle/test/configuration_test.py | 44 +++++++++++++------------- castle/test/extractors/headers_test.py | 8 ++--- 5 files changed, 54 insertions(+), 54 deletions(-) diff --git a/README.rst b/README.rst index 8a1f154..9526440 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ import and configure the library with your Castle API secret. .. code:: python - from castle.configuration import configuration + from castle.configuration import configuration, WHITE_LIST # Same as setting it through Castle.api_secret configuration.api_secret = ':YOUR-API-SECRET' @@ -33,15 +33,15 @@ import and configure the library with your Castle API secret. configuration.request_timeout = 1000 # Whitelisted and Blacklisted headers are case insensitive and allow to use _ and - as a separator, http prefixes are removed + # By default all headers are passed, but some are automatically scrubbed. + # If you need to apply a whitelist, we recommend using the minimum set of + # standard headers that we've exposed in the `WHITE_LIST` constant. # Whitelisted headers - configuration.whitelisted = ['X_HEADER'] - # or append to default - configuration.whitelisted = configuration.whitelisted + ['http-x-header'] - - # Blacklisted headers take advantage over whitelisted elements - configuration.blacklisted = ['HTTP-X-header'] - # or append to default - configuration.blacklisted = configuration.blacklisted + ['X_HEADER'] + configuration.white_list = WHITE_LIST + ['X_HEADER'] + + # Blacklisted headers take advantage over white_list elements. Note that + # some headers are always scrubbed, for security reasons. + configuration.black_list = ['HTTP-X-header'] Tracking -------- diff --git a/castle/configuration.py b/castle/configuration.py index 11504f9..a182246 100644 --- a/castle/configuration.py +++ b/castle/configuration.py @@ -1,9 +1,9 @@ from castle.exceptions import ConfigurationError from castle.headers_formatter import HeadersFormatter -BLACKLISTED = ['HTTP_COOKIE'] +BLACK_LIST = ['HTTP_COOKIE', 'HTTP_AUTHORIZATION'] -WHITELISTED = [ +WHITE_LIST = [ "Accept", "Accept-Charset", "Accept-Datetime", @@ -32,8 +32,8 @@ def __init__(self): self.host = 'api.castle.io' self.port = 443 self.url_prefix = '/v1' - self.whitelisted = [] - self.blacklisted = BLACKLISTED + self.white_list = [] + self.black_list = BLACK_LIST self.request_timeout = REQUEST_TIMEOUT self.failover_strategy = 'allow' @@ -70,26 +70,26 @@ def url_prefix(self, value): self.__url_prefix = value @property - def whitelisted(self): - return self.__whitelisted + def white_list(self): + return self.__white_list - @whitelisted.setter - def whitelisted(self, value): + @white_list.setter + def white_list(self, value): if value: - self.__whitelisted = [HeadersFormatter.call(v) for v in value] + self.__white_list = [HeadersFormatter.call(v) for v in value] else: - self.__whitelisted = [] + self.__white_list = [] @property - def blacklisted(self): - return self.__blacklisted + def black_list(self): + return self.__black_list - @blacklisted.setter - def blacklisted(self, value): + @black_list.setter + def black_list(self, value): if value: - self.__blacklisted = [HeadersFormatter.call(v) for v in value] + self.__black_list = [HeadersFormatter.call(v) for v in value] + BLACK_LIST else: - self.__blacklisted = [] + self.__black_list = BLACK_LIST @property def request_timeout(self): diff --git a/castle/extractors/headers.py b/castle/extractors/headers.py index d495cb1..daa4173 100644 --- a/castle/extractors/headers.py +++ b/castle/extractors/headers.py @@ -9,13 +9,13 @@ def __init__(self, environ): def call(self): headers = dict() - has_whitelist = len(configuration.whitelisted) > 0 + has_whitelist = len(configuration.white_list) > 0 for key, value in self.environ.items(): name = self.formatter.call(key) - if has_whitelist and name not in configuration.whitelisted: + if has_whitelist and name not in configuration.white_list: continue - if name in configuration.blacklisted: + if name in configuration.black_list: continue headers[name] = value diff --git a/castle/test/configuration_test.py b/castle/test/configuration_test.py index cbc4b33..33db401 100644 --- a/castle/test/configuration_test.py +++ b/castle/test/configuration_test.py @@ -1,6 +1,6 @@ from castle.test import unittest from castle.exceptions import ConfigurationError -from castle.configuration import Configuration, WHITELISTED, BLACKLISTED +from castle.configuration import Configuration, WHITE_LIST, BLACK_LIST from castle.headers_formatter import HeadersFormatter @@ -11,9 +11,9 @@ def test_default_values(self): self.assertEqual(config.host, 'api.castle.io') self.assertEqual(config.port, 443) self.assertEqual(config.url_prefix, '/v1') - self.assertEqual(config.whitelisted, []) - self.assertEqual(config.blacklisted, [ - HeadersFormatter.call(v) for v in BLACKLISTED]) + self.assertEqual(config.white_list, []) + self.assertEqual(config.black_list, [ + HeadersFormatter.call(v) for v in BLACK_LIST]) self.assertEqual(config.request_timeout, 500) self.assertEqual(config.failover_strategy, 'allow') @@ -37,35 +37,35 @@ def test_url_prefix_setter(self): config.url_prefix = '/v2' self.assertEqual(config.url_prefix, '/v2') - def test_whitelisted_setter_list(self): + def test_white_list_setter_list(self): config = Configuration() - config.whitelisted = ['test'] - self.assertEqual(config.whitelisted, ['Test']) + config.white_list = ['test'] + self.assertEqual(config.white_list, ['Test']) - def test_whitelisted_setter_none(self): + def test_white_list_setter_none(self): config = Configuration() - config.whitelisted = None - self.assertEqual(config.whitelisted, []) + config.white_list = None + self.assertEqual(config.white_list, []) - def test_whitelisted_setter_empty(self): + def test_white_list_setter_empty(self): config = Configuration() - config.whitelisted = '' - self.assertEqual(config.whitelisted, []) + config.white_list = '' + self.assertEqual(config.white_list, []) - def test_blacklisted_setter_list(self): + def test_black_list_setter_list(self): config = Configuration() - config.blacklisted = ['test'] - self.assertEqual(config.blacklisted, ['Test']) + config.black_list = ['test'] + self.assertEqual(config.black_list, ['Test']) - def test_blacklisted_setter_none(self): + def test_black_list_setter_none(self): config = Configuration() - config.blacklisted = None - self.assertEqual(config.blacklisted, []) + config.black_list = None + self.assertEqual(config.black_list, []) - def test_blacklisted_setter_empty(self): + def test_black_list_setter_empty(self): config = Configuration() - config.blacklisted = '' - self.assertEqual(config.blacklisted, []) + config.black_list = '' + self.assertEqual(config.black_list, []) def test_request_timeout_setter(self): config = Configuration() diff --git a/castle/test/extractors/headers_test.py b/castle/test/extractors/headers_test.py index d2e3845..b5f8b44 100644 --- a/castle/test/extractors/headers_test.py +++ b/castle/test/extractors/headers_test.py @@ -1,5 +1,5 @@ from castle.test import unittest -from castle.configuration import configuration, WHITELISTED +from castle.configuration import configuration, WHITE_LIST from castle.extractors.headers import ExtractorsHeaders @@ -21,10 +21,10 @@ def test_extract_headers(self): self.assertEqual(ExtractorsHeaders(environ()).call(), {'User-Agent': 'requests', 'Ok': 'OK', 'Test': '1'}) - def test_add_whitelisted_headers(self): - configuration.whitelisted = WHITELISTED + ['TEST'] + def test_add_white_list_headers(self): + configuration.white_list = WHITE_LIST + ['TEST'] self.assertEqual( ExtractorsHeaders(environ()).call(), {'User-Agent': 'requests', 'Test': '1'} ) - configuration.whitelisted = [] + configuration.white_list = [] From f748063c1d6941b75578de300fc2e515363c8b7c Mon Sep 17 00:00:00 2001 From: Johanna Larsson Date: Mon, 18 Nov 2019 15:36:01 +0100 Subject: [PATCH 4/9] Use same standard for header names --- castle/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/castle/configuration.py b/castle/configuration.py index a182246..ea8b5b7 100644 --- a/castle/configuration.py +++ b/castle/configuration.py @@ -1,7 +1,7 @@ from castle.exceptions import ConfigurationError from castle.headers_formatter import HeadersFormatter -BLACK_LIST = ['HTTP_COOKIE', 'HTTP_AUTHORIZATION'] +BLACK_LIST = ['Cookie', 'Authorization'] WHITE_LIST = [ "Accept", From 26ceca851955de61e80a4fc72196f133f68a5bf6 Mon Sep 17 00:00:00 2001 From: Johanna Larsson Date: Mon, 18 Nov 2019 15:55:36 +0100 Subject: [PATCH 5/9] Fix tests --- castle/configuration.py | 4 ++-- castle/extractors/headers.py | 7 +++++-- castle/test/configuration_test.py | 2 +- castle/test/context/default_test.py | 10 +++++++--- castle/test/extractors/headers_test.py | 5 +++-- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/castle/configuration.py b/castle/configuration.py index ea8b5b7..5eff6db 100644 --- a/castle/configuration.py +++ b/castle/configuration.py @@ -87,9 +87,9 @@ def black_list(self): @black_list.setter def black_list(self, value): if value: - self.__black_list = [HeadersFormatter.call(v) for v in value] + BLACK_LIST + self.__black_list = [HeadersFormatter.call(v) for v in value] else: - self.__black_list = BLACK_LIST + self.__black_list = [] @property def request_timeout(self): diff --git a/castle/extractors/headers.py b/castle/extractors/headers.py index daa4173..1732509 100644 --- a/castle/extractors/headers.py +++ b/castle/extractors/headers.py @@ -1,5 +1,5 @@ from castle.headers_formatter import HeadersFormatter -from castle.configuration import configuration +from castle.configuration import configuration, BLACK_LIST class ExtractorsHeaders(object): @@ -10,12 +10,15 @@ def __init__(self, environ): def call(self): headers = dict() has_whitelist = len(configuration.white_list) > 0 + extended_black_list = configuration.black_list + BLACK_LIST for key, value in self.environ.items(): name = self.formatter.call(key) if has_whitelist and name not in configuration.white_list: + headers[name] = True continue - if name in configuration.black_list: + if name in extended_black_list: + headers[name] = True continue headers[name] = value diff --git a/castle/test/configuration_test.py b/castle/test/configuration_test.py index 33db401..c2ef720 100644 --- a/castle/test/configuration_test.py +++ b/castle/test/configuration_test.py @@ -1,6 +1,6 @@ from castle.test import unittest from castle.exceptions import ConfigurationError -from castle.configuration import Configuration, WHITE_LIST, BLACK_LIST +from castle.configuration import Configuration, BLACK_LIST from castle.headers_formatter import HeadersFormatter diff --git a/castle/test/context/default_test.py b/castle/test/context/default_test.py index c756b19..1c45594 100644 --- a/castle/test/context/default_test.py +++ b/castle/test/context/default_test.py @@ -46,7 +46,7 @@ def test_default_context(self): self.assertEqual(context['client_id'], client_id()) self.assertEqual(context['active'], True) self.assertEqual(context['origin'], 'web') - self.assertEqual(context['headers'], {'X-Forwarded-For': request_ip()}) + self.assertEqual(context['headers'], {'X-Forwarded-For': request_ip(), 'Cookie': True}) self.assertEqual(context['ip'], request_ip()) self.assertDictEqual(context['library'], { 'name': 'castle-python', 'version': __version__}) @@ -59,8 +59,12 @@ def test_default_context_with_extras(self): self.assertEqual(context['origin'], 'web') self.assertEqual( context['headers'], - {'X-Forwarded-For': request_ip(), 'Accept-Language': 'en', - 'User-Agent': 'test'} + { + 'X-Forwarded-For': request_ip(), + 'Accept-Language': 'en', + 'User-Agent': 'test', + 'Cookie': True + } ) self.assertEqual(context['ip'], request_ip()) self.assertDictEqual( diff --git a/castle/test/extractors/headers_test.py b/castle/test/extractors/headers_test.py index b5f8b44..ab0fdc9 100644 --- a/castle/test/extractors/headers_test.py +++ b/castle/test/extractors/headers_test.py @@ -18,13 +18,14 @@ def environ(): class ExtractorsHeadersTestCase(unittest.TestCase): def test_extract_headers(self): + configuration.white_list = [] self.assertEqual(ExtractorsHeaders(environ()).call(), - {'User-Agent': 'requests', 'Ok': 'OK', 'Test': '1'}) + {'User-Agent': 'requests', 'Ok': 'OK', 'Test': '1', 'Cookie': True}) def test_add_white_list_headers(self): configuration.white_list = WHITE_LIST + ['TEST'] self.assertEqual( ExtractorsHeaders(environ()).call(), - {'User-Agent': 'requests', 'Test': '1'} + {'User-Agent': 'requests', 'Test': '1', 'Cookie': True, 'Ok': True} ) configuration.white_list = [] From 42b263826fb43659d21244c89e716d59ef3070eb Mon Sep 17 00:00:00 2001 From: Johanna Larsson Date: Mon, 18 Nov 2019 16:28:08 +0100 Subject: [PATCH 6/9] Remove default blacklist --- castle/configuration.py | 2 +- castle/test/configuration_test.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/castle/configuration.py b/castle/configuration.py index 5eff6db..af15ab4 100644 --- a/castle/configuration.py +++ b/castle/configuration.py @@ -33,7 +33,7 @@ def __init__(self): self.port = 443 self.url_prefix = '/v1' self.white_list = [] - self.black_list = BLACK_LIST + self.black_list = [] self.request_timeout = REQUEST_TIMEOUT self.failover_strategy = 'allow' diff --git a/castle/test/configuration_test.py b/castle/test/configuration_test.py index c2ef720..77db7c6 100644 --- a/castle/test/configuration_test.py +++ b/castle/test/configuration_test.py @@ -12,8 +12,7 @@ def test_default_values(self): self.assertEqual(config.port, 443) self.assertEqual(config.url_prefix, '/v1') self.assertEqual(config.white_list, []) - self.assertEqual(config.black_list, [ - HeadersFormatter.call(v) for v in BLACK_LIST]) + self.assertEqual(config.black_list, []) self.assertEqual(config.request_timeout, 500) self.assertEqual(config.failover_strategy, 'allow') From 1ca3a9e49374a2dd7997d3f2d81f36cce677d695 Mon Sep 17 00:00:00 2001 From: Johanna Larsson Date: Tue, 19 Nov 2019 11:58:35 +0100 Subject: [PATCH 7/9] Rename the props --- README.rst | 10 +++--- castle/configuration.py | 32 +++++++++----------- castle/extractors/headers.py | 12 +++++--- castle/test/configuration_test.py | 42 +++++++++++++------------- castle/test/extractors/headers_test.py | 10 +++--- 5 files changed, 53 insertions(+), 53 deletions(-) diff --git a/README.rst b/README.rst index 9526440..22815d3 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ import and configure the library with your Castle API secret. .. code:: python - from castle.configuration import configuration, WHITE_LIST + from castle.configuration import configuration, WHITELISTED # Same as setting it through Castle.api_secret configuration.api_secret = ':YOUR-API-SECRET' @@ -35,13 +35,13 @@ import and configure the library with your Castle API secret. # Whitelisted and Blacklisted headers are case insensitive and allow to use _ and - as a separator, http prefixes are removed # By default all headers are passed, but some are automatically scrubbed. # If you need to apply a whitelist, we recommend using the minimum set of - # standard headers that we've exposed in the `WHITE_LIST` constant. + # standard headers that we've exposed in the `WHITELISTED` constant. # Whitelisted headers - configuration.white_list = WHITE_LIST + ['X_HEADER'] + configuration.whitelisted = WHITELISTED + ['X_HEADER'] - # Blacklisted headers take advantage over white_list elements. Note that + # Blacklisted headers take advantage over whitelisted elements. Note that # some headers are always scrubbed, for security reasons. - configuration.black_list = ['HTTP-X-header'] + configuration.blacklisted = ['HTTP-X-header'] Tracking -------- diff --git a/castle/configuration.py b/castle/configuration.py index af15ab4..5f5e37f 100644 --- a/castle/configuration.py +++ b/castle/configuration.py @@ -1,9 +1,7 @@ from castle.exceptions import ConfigurationError from castle.headers_formatter import HeadersFormatter -BLACK_LIST = ['Cookie', 'Authorization'] - -WHITE_LIST = [ +WHITELISTED = [ "Accept", "Accept-Charset", "Accept-Datetime", @@ -32,8 +30,8 @@ def __init__(self): self.host = 'api.castle.io' self.port = 443 self.url_prefix = '/v1' - self.white_list = [] - self.black_list = [] + self.whitelisted = [] + self.blacklisted = [] self.request_timeout = REQUEST_TIMEOUT self.failover_strategy = 'allow' @@ -70,26 +68,26 @@ def url_prefix(self, value): self.__url_prefix = value @property - def white_list(self): - return self.__white_list + def whitelisted(self): + return self.__whitelisted - @white_list.setter - def white_list(self, value): + @whitelisted.setter + def whitelisted(self, value): if value: - self.__white_list = [HeadersFormatter.call(v) for v in value] + self.__whitelisted = [HeadersFormatter.call(v) for v in value] else: - self.__white_list = [] + self.__whitelisted = [] @property - def black_list(self): - return self.__black_list + def blacklisted(self): + return self.__blacklisted - @black_list.setter - def black_list(self, value): + @blacklisted.setter + def blacklisted(self, value): if value: - self.__black_list = [HeadersFormatter.call(v) for v in value] + self.__blacklisted = [HeadersFormatter.call(v) for v in value] else: - self.__black_list = [] + self.__blacklisted = [] @property def request_timeout(self): diff --git a/castle/extractors/headers.py b/castle/extractors/headers.py index 1732509..35193cc 100644 --- a/castle/extractors/headers.py +++ b/castle/extractors/headers.py @@ -1,5 +1,7 @@ from castle.headers_formatter import HeadersFormatter -from castle.configuration import configuration, BLACK_LIST +from castle.configuration import configuration + +BLACKLISTED = ['Cookie', 'Authorization'] class ExtractorsHeaders(object): @@ -9,15 +11,15 @@ def __init__(self, environ): def call(self): headers = dict() - has_whitelist = len(configuration.white_list) > 0 - extended_black_list = configuration.black_list + BLACK_LIST + has_whitelist = len(configuration.whitelisted) > 0 + extended_blacklisted = configuration.blacklisted + BLACKLISTED for key, value in self.environ.items(): name = self.formatter.call(key) - if has_whitelist and name not in configuration.white_list: + if has_whitelist and name not in configuration.whitelisted: headers[name] = True continue - if name in extended_black_list: + if name in extended_blacklisted: headers[name] = True continue headers[name] = value diff --git a/castle/test/configuration_test.py b/castle/test/configuration_test.py index 77db7c6..5287bcd 100644 --- a/castle/test/configuration_test.py +++ b/castle/test/configuration_test.py @@ -1,6 +1,6 @@ from castle.test import unittest from castle.exceptions import ConfigurationError -from castle.configuration import Configuration, BLACK_LIST +from castle.configuration import Configuration from castle.headers_formatter import HeadersFormatter @@ -11,8 +11,8 @@ def test_default_values(self): self.assertEqual(config.host, 'api.castle.io') self.assertEqual(config.port, 443) self.assertEqual(config.url_prefix, '/v1') - self.assertEqual(config.white_list, []) - self.assertEqual(config.black_list, []) + self.assertEqual(config.whitelisted, []) + self.assertEqual(config.blacklisted, []) self.assertEqual(config.request_timeout, 500) self.assertEqual(config.failover_strategy, 'allow') @@ -36,35 +36,35 @@ def test_url_prefix_setter(self): config.url_prefix = '/v2' self.assertEqual(config.url_prefix, '/v2') - def test_white_list_setter_list(self): + def test_whitelisted_setter_list(self): config = Configuration() - config.white_list = ['test'] - self.assertEqual(config.white_list, ['Test']) + config.whitelisted = ['test'] + self.assertEqual(config.whitelisted, ['Test']) - def test_white_list_setter_none(self): + def test_whitelisted_setter_none(self): config = Configuration() - config.white_list = None - self.assertEqual(config.white_list, []) + config.whitelisted = None + self.assertEqual(config.whitelisted, []) - def test_white_list_setter_empty(self): + def test_whitelisted_setter_empty(self): config = Configuration() - config.white_list = '' - self.assertEqual(config.white_list, []) + config.whitelisted = '' + self.assertEqual(config.whitelisted, []) - def test_black_list_setter_list(self): + def test_blacklisted_setter_list(self): config = Configuration() - config.black_list = ['test'] - self.assertEqual(config.black_list, ['Test']) + config.blacklisted = ['test'] + self.assertEqual(config.blacklisted, ['Test']) - def test_black_list_setter_none(self): + def test_blacklisted_setter_none(self): config = Configuration() - config.black_list = None - self.assertEqual(config.black_list, []) + config.blacklisted = None + self.assertEqual(config.blacklisted, []) - def test_black_list_setter_empty(self): + def test_blacklisted_setter_empty(self): config = Configuration() - config.black_list = '' - self.assertEqual(config.black_list, []) + config.blacklisted = '' + self.assertEqual(config.blacklisted, []) def test_request_timeout_setter(self): config = Configuration() diff --git a/castle/test/extractors/headers_test.py b/castle/test/extractors/headers_test.py index ab0fdc9..8847a1a 100644 --- a/castle/test/extractors/headers_test.py +++ b/castle/test/extractors/headers_test.py @@ -1,5 +1,5 @@ from castle.test import unittest -from castle.configuration import configuration, WHITE_LIST +from castle.configuration import configuration, WHITELISTED from castle.extractors.headers import ExtractorsHeaders @@ -18,14 +18,14 @@ def environ(): class ExtractorsHeadersTestCase(unittest.TestCase): def test_extract_headers(self): - configuration.white_list = [] + configuration.whitelisted = [] self.assertEqual(ExtractorsHeaders(environ()).call(), {'User-Agent': 'requests', 'Ok': 'OK', 'Test': '1', 'Cookie': True}) - def test_add_white_list_headers(self): - configuration.white_list = WHITE_LIST + ['TEST'] + def test_add_whitelisted_headers(self): + configuration.whitelisted = WHITELISTED + ['TEST'] self.assertEqual( ExtractorsHeaders(environ()).call(), {'User-Agent': 'requests', 'Test': '1', 'Cookie': True, 'Ok': True} ) - configuration.white_list = [] + configuration.whitelisted = [] From 57b4e0c1c7fef783620b124c099fe860ac1b5b7a Mon Sep 17 00:00:00 2001 From: Johanna Larsson Date: Tue, 19 Nov 2019 13:13:47 +0100 Subject: [PATCH 8/9] clean up header extraction logic --- castle/extractors/headers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/castle/extractors/headers.py b/castle/extractors/headers.py index 35193cc..20fec29 100644 --- a/castle/extractors/headers.py +++ b/castle/extractors/headers.py @@ -1,7 +1,8 @@ from castle.headers_formatter import HeadersFormatter from castle.configuration import configuration -BLACKLISTED = ['Cookie', 'Authorization'] +DEFAULT_BLACKLIST = ['Cookie', 'Authorization'] +DEFAULT_WHITELIST = ['User-Agent'] class ExtractorsHeaders(object): @@ -12,14 +13,13 @@ def __init__(self, environ): def call(self): headers = dict() has_whitelist = len(configuration.whitelisted) > 0 - extended_blacklisted = configuration.blacklisted + BLACKLISTED for key, value in self.environ.items(): name = self.formatter.call(key) - if has_whitelist and name not in configuration.whitelisted: + if has_whitelist and name not in configuration.whitelisted and name not in DEFAULT_WHITELIST: headers[name] = True continue - if name in extended_blacklisted: + if name in configuration.blacklisted or name in DEFAULT_BLACKLIST: headers[name] = True continue headers[name] = value From 7f1525657944d1045c35935dd3882d848ea63bf9 Mon Sep 17 00:00:00 2001 From: Johanna Larsson Date: Tue, 19 Nov 2019 14:19:19 +0100 Subject: [PATCH 9/9] Update whitelist --- castle/configuration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/castle/configuration.py b/castle/configuration.py index 5f5e37f..6bc66fb 100644 --- a/castle/configuration.py +++ b/castle/configuration.py @@ -16,7 +16,10 @@ "Origin", "Pragma", "Referer", + "TE", + "Upgrade-Insecure-Requests", "User-Agent", + "X-Castle-Client-Id", ] # 500 milliseconds