From 0ae6055f121242556795fc83e5d917dc7170420d Mon Sep 17 00:00:00 2001 From: nitzanpx Date: Tue, 9 May 2017 14:58:19 +0300 Subject: [PATCH 01/35] support cookie v1 + v3, removed uneccerry prints and added more debugs logs, added to debuger INFO, added px_constants class --- perimeterx-python-wsgi.iml | 27 +++ perimeterx/middleware.py | 42 ++-- perimeterx/px_activities_client.py | 2 - perimeterx/px_constants.py | 5 + perimeterx/px_context.py | 15 +- perimeterx/px_cookie.py | 343 ++++++++++++++--------------- perimeterx/px_cookie_v1.py | 35 +++ perimeterx/px_cookie_v3.py | 32 +++ perimeterx/px_cookie_validator.py | 58 +++++ perimeterx/px_logger.py | 4 + setup.py | 2 +- 11 files changed, 355 insertions(+), 210 deletions(-) create mode 100644 perimeterx-python-wsgi.iml create mode 100644 perimeterx/px_constants.py create mode 100644 perimeterx/px_cookie_v1.py create mode 100644 perimeterx/px_cookie_v3.py create mode 100644 perimeterx/px_cookie_validator.py diff --git a/perimeterx-python-wsgi.iml b/perimeterx-python-wsgi.iml new file mode 100644 index 0000000..870de95 --- /dev/null +++ b/perimeterx-python-wsgi.iml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index e20c54d..bab9973 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -1,7 +1,7 @@ from px_logger import Logger import px_context import px_activities_client -import px_cookie +import px_cookie_validator import px_httpc import px_captcha import px_api @@ -9,7 +9,6 @@ import Cookie - class PerimeterX(object): def __init__(self, app, config=None): self.app = app @@ -22,6 +21,7 @@ def __init__(self, app, config=None): 'perimeterx_server_host': 'sapi.perimeterx.net', 'captcha_enabled': True, 'server_calls_enabled': True, + 'encryption_enabled': True, 'sensitive_headers': ['cookie', 'cookies'], 'send_page_activities': True, 'api_timeout': 1, @@ -63,26 +63,30 @@ def custom_start_response(status, headers, exc_info=None): def _verify(self, environ, start_response): logger = self.config['logger'] - ctx = px_context.build_context(environ, self.config) - - if ctx.get('module_mode') == 'inactive' or is_static_file(ctx): - logger.debug('Filter static file request. uri: ' + ctx.get('uri')) - return self.app(environ, start_response) + try: + ctx = px_context.build_context(environ, self.config) - cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE')) - if self.config.get('captcha_enabled') and cookies.get('_pxCaptcha') and cookies.get('_pxCaptcha').value: - pxCaptcha = cookies.get('_pxCaptcha').value - if px_captcha.verify(ctx, self.config, pxCaptcha): - logger.debug('User passed captcha verification. user ip: ' + ctx.get('socket_ip')) + if ctx.get('module_mode') == 'inactive' or is_static_file(ctx): + logger.debug('Filter static file request. uri: ' + ctx.get('uri')) return self.app(environ, start_response) - # PX Cookie verification - if not px_cookie.verify(ctx, self.config) and self.config.get('server_calls_enabled', True): - # Server-to-Server verification fallback - if not px_api.verify(ctx, self.config): - return self.app(environ, start_response) - - return self.handle_verification(ctx, self.config, environ, start_response) + cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE')) + if self.config.get('captcha_enabled') and cookies.get('_pxCaptcha') and cookies.get('_pxCaptcha').value: + pxCaptcha = cookies.get('_pxCaptcha').value + if px_captcha.verify(ctx, self.config, pxCaptcha): + logger.debug('User passed captcha verification. user ip: ' + ctx.get('socket_ip')) + return self.app(environ, start_response) + + # PX Cookie verification + if not px_cookie_validator.verify(ctx, self.config) and self.config.get('server_calls_enabled', True): + # Server-to-Server verification fallback + if not px_api.verify(ctx, self.config): + return self.app(environ, start_response) + + return self.handle_verification(ctx, self.config, environ, start_response) + except: + logger.error("Cought exception, passing request") + self.pass_traffic(environ, start_response, ctx) def handle_verification(self, ctx, config, environ, start_response): score = ctx.get('risk_score', -1) diff --git a/perimeterx/px_activities_client.py b/perimeterx/px_activities_client.py index 7a783cc..0c4556a 100644 --- a/perimeterx/px_activities_client.py +++ b/perimeterx/px_activities_client.py @@ -25,7 +25,6 @@ def send_activities(): def send_to_perimeterx(activity_type, ctx, config, detail): global CONFIG try: - print activity_type if not config.get('server_calls_enabled', True): return @@ -57,7 +56,6 @@ def send_to_perimeterx(activity_type, ctx, config, detail): 'vid': ctx.get('vid', ''), 'uuid': ctx.get('uuid', '') } - print 'appending' ACTIVITIES_BUFFER.append(data) except: print traceback.format_exception(*sys.exc_info()) diff --git a/perimeterx/px_constants.py b/perimeterx/px_constants.py new file mode 100644 index 0000000..dbfdf04 --- /dev/null +++ b/perimeterx/px_constants.py @@ -0,0 +1,5 @@ +PREFIX_PX_COOKIE_V1 = '_px' +PREFIX_PX_COOKIE_V3 = '_px3' + +TRANS_5C = b"".join(chr(x ^ 0x5C) for x in range(256)) +TRANS_36 = b"".join(chr(x ^ 0x36) for x in range(256)) \ No newline at end of file diff --git a/perimeterx/px_context.py b/perimeterx/px_context.py index 869ef2e..716becd 100644 --- a/perimeterx/px_context.py +++ b/perimeterx/px_context.py @@ -1,15 +1,16 @@ import Cookie +from px_constants import * def build_context(environ, config): + logger = config['logger'] headers = {} # Default values http_method = 'GET' http_version = '1.1' http_protocol = 'http://' - px_cookie = '' - uuid = '' + px_cookies = {} # IP Extraction if config.get('ip_handler'): @@ -33,8 +34,12 @@ def build_context(environ, config): http_version = protocol_split[1] cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE', '')) - if cookies.get('_px') and cookies.get('_px').value: - px_cookie = cookies.get('_px').value + cookie_keys = cookies.keys() + + for key in cookie_keys: + if key == PREFIX_PX_COOKIE_V1 or key == PREFIX_PX_COOKIE_V3: + logger.debug('Found cookie prefix:' + key) + px_cookies[key] = cookies.get(key).value user_agent = headers.get('user-agent') uri = environ.get('PATH_INFO') or '' @@ -49,6 +54,6 @@ def build_context(environ, config): 'full_url': full_url, 'uri': uri, 'hostname': hostname, - '_px': px_cookie + 'px_cookies': px_cookies } return ctx diff --git a/perimeterx/px_cookie.py b/perimeterx/px_cookie.py index c3d0043..20539da 100644 --- a/perimeterx/px_cookie.py +++ b/perimeterx/px_cookie.py @@ -1,201 +1,178 @@ +import json +from px_constants import * from Crypto.Cipher import AES from time import time import base64 import hmac import hashlib -import json -import sys, traceback +import sys +import traceback import binascii import struct -_trans_5C = b"".join(chr(x ^ 0x5C) for x in range(256)) -_trans_36 = b"".join(chr(x ^ 0x36) for x in range(256)) - - -def pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None): - """Password based key derivation function 2 (PKCS #5 v2.0) - - This Python implementations based on the hmac module about as fast - as OpenSSL's PKCS5_PBKDF2_HMAC for short passwords and much faster - for long passwords. - """ - if not isinstance(hash_name, str): - raise TypeError(hash_name) - - if not isinstance(password, (bytes, bytearray)): - password = bytes(buffer(password)) - if not isinstance(salt, (bytes, bytearray)): - salt = bytes(buffer(salt)) - - # Fast inline HMAC implementation - inner = hashlib.new(hash_name) - outer = hashlib.new(hash_name) - blocksize = getattr(inner, 'block_size', 64) - if len(password) > blocksize: - password = hashlib.new(hash_name, password).digest() - password = password + b'\x00' * (blocksize - len(password)) - inner.update(password.translate(_trans_36)) - outer.update(password.translate(_trans_5C)) - - def prf(msg, inner=inner, outer=outer): - # PBKDF2_HMAC uses the password as key. We can re-use the same - # digest objects and just update copies to skip initialization. - icpy = inner.copy() - ocpy = outer.copy() - icpy.update(msg) - ocpy.update(icpy.digest()) - return ocpy.digest() - - if iterations < 1: - raise ValueError(iterations) - if dklen is None: - dklen = outer.digest_size - if dklen < 1: - raise ValueError(dklen) - - hex_format_string = "%%0%ix" % (hashlib.new(hash_name).digest_size * 2) - - dkey = b'' - loop = 1 - while len(dkey) < dklen: - prev = prf(salt + struct.pack(b'>I', loop)) - rkey = int(binascii.hexlify(prev), 16) - for i in xrange(iterations - 1): - prev = prf(prev) - rkey ^= int(binascii.hexlify(prev), 16) - loop += 1 - dkey += binascii.unhexlify(hex_format_string % rkey) - - return dkey[:dklen] - - -def is_cookie_expired(cookie): - """ - Checks if cookie validity time expired. - :param cookie: risk object - :type cookie: dictionary - :return: Returns True if valid and False if not - :rtype: Bool - """ - now = int(round(time() * 1000)) - expire = cookie[u't'] - return now > expire - - -def is_cookie_valid(cookie, cookie_key, ctx): - """ - Checks if cookie hmac signing match the request. - :param cookie: risk object - :param cookie_key: cookie secret key - :param ctx: perimeterx request context object - :type cookie: dictionary - :type cookie_key: string - :type ctx: dictionary - :return: Returns True if valid and False if not - :rtype: Bool - """ - user_agent = ctx['user_agent'] - msg = str(cookie['t']) + str(cookie['s']['a']) + str(cookie['s']['b']) + str(cookie['u']) + str( - cookie['v']) + user_agent - - valid_digest = cookie['h'] - try: - calculated_digest = hmac.new(cookie_key, msg, hashlib.sha256).hexdigest() - except: - return False - - return valid_digest == calculated_digest - - -def decrypt_cookie(cookie_key, cookie): - """ - Decrypting the PerimeterX risk cookie using AES - :param cookie_key: cookie secret key - :param cookie: risk cookie - encrypted - :type cookie_key: string - :type cookie: string - :return: Returns decrypted value if valid and False if not - :rtype: Bool|String - """ - try: - parts = cookie.split(':', 3) - if len(parts) != 3: - return False - salt = base64.b64decode(parts[0]) - iterations = int(parts[1]) - if iterations < 1 or iterations > 10000: - return False - data = base64.b64decode(parts[2]) - dk = pbkdf2_hmac('sha256', cookie_key, salt, iterations, dklen=48) - key = dk[:32] - iv = dk[32:] - cipher = AES.new(key, AES.MODE_CBC, iv) - unpad = lambda s: s[0:-ord(s[-1])] - plaintext = unpad(cipher.decrypt(data)) - return plaintext - except: - print traceback.format_exception(*sys.exc_info()) - return False - - -def verify(ctx, config): - """ - main verification function, verifying the content of the perimeterx risk cookie if exists - :param ctx: perimeterx request context object - :param config: global configurations - :type ctx: dictionary - :type config: dictionary - :return: Returns True if verification succeeded and False if not - :rtype: Bool - """ - logger = config['logger'] - px_cookie = ctx['_px'] - try: - if not px_cookie: - logger.debug('No risk cookie on the request') - ctx['s2s_call_reason'] = 'no_cookie' + +class PxCookie: + + def __init__(self): + pass + + @staticmethod + def build_px_cookie(ctx, config): + config["logger"].debug("PxCookie[build_px_cookie]") + px_cookies = ctx['px_cookies'].keys() + + # Check that its not empty + if not px_cookies: + return None + + prefix = px_cookies[0] + if prefix == PREFIX_PX_COOKIE_V1: + config["logger"].debug("PxCookie[build_px_cookie] using cookie v1") + from px_cookie_v1 import PxCookieV1 + return PxCookieV1(ctx, config) + + if prefix == PREFIX_PX_COOKIE_V3: + config["logger"].debug("PxCookie[build_px_cookie] using cookie v3") + from px_cookie_v3 import PxCookieV3 + return PxCookieV3(ctx, config) + + def decode_cookie(self): + self.config['logger'].debug("PxCookie[decode_cookie]") + return base64.b64decode(self.raw_cookie) + + def pbkdf2_hmac(self, hash_name, password, salt, iterations, dklen=None): + """Password based key derivation function 2 (PKCS #5 v2.0) + + This Python implementations based on the hmac module about as fast + as OpenSSL's PKCS5_PBKDF2_HMAC for short passwords and much faster + for long passwords. + """ + if not isinstance(hash_name, str): + raise TypeError(hash_name) + + if not isinstance(password, (bytes, bytearray)): + password = bytes(buffer(password)) + if not isinstance(salt, (bytes, bytearray)): + salt = bytes(buffer(salt)) + + # Fast inline HMAC implementation + inner = hashlib.new(hash_name) + outer = hashlib.new(hash_name) + blocksize = getattr(inner, 'block_size', 64) + if len(password) > blocksize: + password = hashlib.new(hash_name, password).digest() + password = password + b'\x00' * (blocksize - len(password)) + inner.update(password.translate(TRANS_36)) + outer.update(password.translate(TRANS_5C)) + + def prf(msg, inner=inner, outer=outer): + # PBKDF2_HMAC uses the password as key. We can re-use the same + # digest objects and just update copies to skip initialization. + icpy = inner.copy() + ocpy = outer.copy() + icpy.update(msg) + ocpy.update(icpy.digest()) + return ocpy.digest() + + if iterations < 1: + raise ValueError(iterations) + if dklen is None: + dklen = outer.digest_size + if dklen < 1: + raise ValueError(dklen) + + hex_format_string = "%%0%ix" % (hashlib.new(hash_name).digest_size * 2) + + dkey = b'' + loop = 1 + while len(dkey) < dklen: + prev = prf(salt + struct.pack(b'>I', loop)) + rkey = int(binascii.hexlify(prev), 16) + for i in xrange(iterations - 1): + prev = prf(prev) + rkey ^= int(binascii.hexlify(prev), 16) + loop += 1 + dkey += binascii.unhexlify(hex_format_string % rkey) + + return dkey[:dklen] + + def decrypt_cookie(self): + """ + Decrypting the PerimeterX risk cookie using AES + :return: Returns decrypted value if valid and False if not + :rtype: Bool|String + """ + self.config['logger'].debug("PxCookie[decrypt_cookie]") + try: + parts = self.raw_cookie.split(':', 3) + if len(parts) != 3: + return False + salt = base64.b64decode(parts[0]) + iterations = int(parts[1]) + if iterations < 1 or iterations > 10000: + return False + data = base64.b64decode(parts[2]) + dk = self.pbkdf2_hmac('sha256', self.config['cookie_key'], salt, iterations, dklen=48) + key = dk[:32] + iv = dk[32:] + cipher = AES.new(key, AES.MODE_CBC, iv) + unpad = lambda s: s[0:-ord(s[-1])] + plaintext = unpad(cipher.decrypt(data)) + self.config['logger'].debug("PxCookie[decrypt_cookie] cookie decrypted") + return plaintext + except: + print traceback.format_exception(*sys.exc_info()) + return None + + def is_cookie_expired(self): + """ + Checks if cookie validity time expired. + :return: Returns True if valid and False if not + :rtype: Bool + """ + now = int(round(time() * 1000)) + expire = self.get_timestamp() + return now > expire + + def is_cookie_valid(self, str_to_hmac): + """ + Checks if cookie hmac signing match the request. + :return: Returns True if valid and False if not + :rtype: Bool + """ + try: + calculated_digest = hmac.new(self.config['cookie_key'], str_to_hmac, hashlib.sha256).hexdigest() + return self.get_hmac() == calculated_digest + except: + self.config["logger"].debug("failed to calculate hmac") return False - decrypted_cookie = decrypt_cookie(config['cookie_key'], px_cookie) + def deserialize(self): + self.config['logger'].debug("PxCookie[deserialize]") + if self.config.get("encryption_enabled", False): + cookie = self.decrypt_cookie() + else: + cookie = self.decode_cookie() - if not decrypted_cookie: - logger.error('Cookie decryption failed') - ctx['px_orig_cookie'] = px_cookie - ctx['s2s_call_reason'] = 'cookie_decryption_failed' + if not cookie: return False - decoded_cookie = json.loads(decrypted_cookie) - try: - decoded_cookie['s'], decoded_cookie['s']['b'], decoded_cookie['u'], decoded_cookie['t'], decoded_cookie['v'] - except: - logger.error('Cookie decryption failed') - ctx['px_orig_cookie'] = px_cookie - ctx['s2s_call_reason'] = 'cookie_decryption_failed' - return False + self.config['logger'].debug("PxCookie[deserialize] decoded cookie: " + cookie) + self.decoded_cookie = json.loads(cookie) + return self.is_cookie_format_valid() - ctx['risk_score'] = decoded_cookie['s']['b'] - ctx['uuid'] = decoded_cookie.get('u', '') - ctx['vid'] = decoded_cookie.get('v', '') - ctx['decoded_cookie'] = decoded_cookie + def is_high_score(self): + return self.get_score() >= self.config['blocking_score'] + + def get_timestamp(self): + return self.decoded_cookie['t'] + + def get_uuid(self): + return self.decoded_cookie['u'] + + def get_vid(self): + return self.decoded_cookie['v'] - if decoded_cookie['s']['b'] >= config['blocking_score']: - ctx['block_reason'] = 'cookie_high_score' - logger.debug('Cookie with high score: ' + str(ctx['risk_score'])) - return True - if is_cookie_expired(decoded_cookie): - ctx['s2s_call_reason'] = 'cookie_expired' - logger.debug('Cookie expired') - return False - if not is_cookie_valid(decoded_cookie, config['cookie_key'], ctx): - logger.debug('Cookie validation failed') - ctx['s2s_call_reason'] = 'cookie_validation_failed' - return False - logger.debug('Cookie validation passed with good score: ' + str(ctx['risk_score'])) - return True - except: - logger.debug('Cookie validation failed') - ctx['s2s_call_reason'] = 'cookie_validation_failed' - return False diff --git a/perimeterx/px_cookie_v1.py b/perimeterx/px_cookie_v1.py new file mode 100644 index 0000000..959602f --- /dev/null +++ b/perimeterx/px_cookie_v1.py @@ -0,0 +1,35 @@ +from px_cookie import PxCookie +from px_constants import * + + +class PxCookieV1(PxCookie): + + def __init__(self, ctx, config): + self.ctx = ctx + self.config = config + self.raw_cookie = ctx['px_cookies'].get(PREFIX_PX_COOKIE_V1, '') + + def get_score(self): + return self.decoded_cookie['s']['b'] + + def get_hmac(self): + return self.decoded_cookie['h'] + + def get_action(self): + return 'c' + + def is_cookie_format_valid(self): + c = self.decoded_cookie + return 't' in c and 'v' in c and 'u' in c and "s" in c and 'a' in c['s'] and 'h' in c + + def is_secured(self): + c = self.decoded_cookie + user_agent = self.ctx.get('user_agent', '') + ip = self.ctx.get('ip', '') + base_hmac = str(self.get_timestamp()) + str(c['s']['a']) + str(self.get_score()) + self.get_uuid() + self.get_vid() + hmac_with_ip = base_hmac + ip + user_agent + hmac_without_ip = base_hmac + user_agent + + return self.is_cookie_valid(hmac_without_ip) or self.is_cookie_valid(hmac_with_ip) + + diff --git a/perimeterx/px_cookie_v3.py b/perimeterx/px_cookie_v3.py new file mode 100644 index 0000000..00226a8 --- /dev/null +++ b/perimeterx/px_cookie_v3.py @@ -0,0 +1,32 @@ +from px_cookie import PxCookie +from px_constants import * + + +class PxCookieV3(PxCookie): + + def __init__(self, ctx, config): + self.ctx = ctx + self.config = config + spliced_cookie = ctx['px_cookies'].get(PREFIX_PX_COOKIE_V3, '').split(":", 1) + if spliced_cookie.count > 1: + self.hmac = spliced_cookie[0] + self.raw_cookie = spliced_cookie[1] + + def get_score(self): + return self.decoded_cookie['s'] + + def get_hmac(self): + return self.hmac + + def get_action(self): + return self.decoded_cookie['a'] + + def is_cookie_format_valid(self): + c = self.decoded_cookie; + return 't' in c and 'v' in c and 'u' in c and 's' in c and 'a' in c + + def is_secured(self): + user_agent = self.ctx.get('user_agent', '') + str_hmac = self.raw_cookie + user_agent + return self.is_cookie_valid(str_hmac) + diff --git a/perimeterx/px_cookie_validator.py b/perimeterx/px_cookie_validator.py new file mode 100644 index 0000000..2834e56 --- /dev/null +++ b/perimeterx/px_cookie_validator.py @@ -0,0 +1,58 @@ +import traceback + + +def verify(ctx, config): + """ + main verification function, verifying the content of the perimeterx risk cookie if exists + :param ctx: perimeterx request context object + :param config: global configurations + :type ctx: dictionary + :type config: dictionary + :return: Returns True if verification succeeded and False if not + :rtype: Bool + """ + logger = config['logger'] + try: + if not ctx["px_cookies"].keys(): + logger.debug('No risk cookie on the request') + ctx['s2s_call_reason'] = 'no_cookie' + return False + + from px_cookie import PxCookie + px_cookie = PxCookie.build_px_cookie(ctx, config) + + if not px_cookie.deserialize(): + logger.error('Cookie decryption failed') + ctx['px_orig_cookie'] = px_cookie + ctx['s2s_call_reason'] = 'cookie_decryption_failed' + return False + + ctx['risk_score'] = px_cookie.get_score() + ctx['uuid'] = px_cookie.get_uuid() + ctx['vid'] = px_cookie.get_vid() + ctx['decoded_cookie'] = px_cookie.decoded_cookie + ctx['cookie_hmac'] = px_cookie.get_hmac() + ctx['block_action'] = px_cookie.get_action() + + if px_cookie.is_high_score(): + ctx['block_reason'] = 'cookie_high_score' + logger.debug('Cookie with high score: ' + str(ctx['risk_score'])) + return True + + if px_cookie.is_cookie_expired(): + ctx['s2s_call_reason'] = 'cookie_expired' + logger.debug('Cookie expired') + return False + + if not px_cookie.is_secured(): + logger.debug('Cookie validation failed') + ctx['s2s_call_reason'] = 'cookie_validation_failed' + return False + + logger.debug('Cookie validation passed with good score: ' + str(ctx['risk_score'])) + return True + except Exception, e: + traceback.print_exc() + logger.debug('Could not decrypt cookie, exception was thrown, decryption failed ' + e.message) + ctx['s2s_call_reason'] = 'cookie_decryption_failed' + return False diff --git a/perimeterx/px_logger.py b/perimeterx/px_logger.py index 873c34c..afac7c8 100644 --- a/perimeterx/px_logger.py +++ b/perimeterx/px_logger.py @@ -6,5 +6,9 @@ def debug(self, message): if self.debug_mode: print '[PerimeterX DEBUG]: ' + message + def info(self, message): + if self.debug_mode: + print '[PerimeterX INFO]: ' + message + def error(self, message): print '[PerimeterX ERROR]: ' + message diff --git a/setup.py b/setup.py index 57f075c..179045f 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,6 @@ download_url='https://github.com/PerimeterX/perimeterx-python-wsgi/tarball/v1.0.2', package_dir={'perimeterx': 'perimeterx'}, install_requires=[ - "pystache" + "pystache", 'Crypto' ] ) From 82674befe3b32093c8143914d524cb915628f1c0 Mon Sep 17 00:00:00 2001 From: nitzanpx Date: Tue, 9 May 2017 15:42:49 +0300 Subject: [PATCH 02/35] support risk v2, fixed bug in px_orig_cookie --- perimeterx/px_api.py | 16 ++++++++++++---- perimeterx/px_cookie_validator.py | 5 +++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/perimeterx/px_api.py b/perimeterx/px_api.py index a022523..844c99d 100644 --- a/perimeterx/px_api.py +++ b/perimeterx/px_api.py @@ -4,19 +4,24 @@ def send_risk_request(ctx, config): body = prepare_risk_body(ctx, config) - return px_httpc.send('/api/v1/risk', body, config) + return px_httpc.send('/api/v2/risk', body, config) def verify(ctx, config): logger = config['logger'] + logger.debug("PxAPI[verify]") try: response = send_risk_request(ctx, config) if response: - ctx['risk_score'] = response['scores']['non_human'] + score = response['score'] + ctx['score'] = score ctx['uuid'] = response['uuid'] - if ctx['risk_score'] >= config['blocking_score']: + ctx['block_action'] = response['action'] + if score >= config['blocking_score']: + logger.debug("PxAPI[verify] block score threshold reached") ctx['block_reason'] = 's2s_high_score' + logger.debug("PxAPI[verify] S2S completed") return True else: return False @@ -27,6 +32,7 @@ def verify(ctx, config): def prepare_risk_body(ctx, config): logger = config['logger'] + logger.debug("PxAPI[send_risk_request]") body = { 'request': { 'ip': ctx.get('socket_ip'), @@ -41,7 +47,8 @@ def prepare_risk_body(ctx, config): 'http_method': ctx.get('http_method', ''), 'http_version': ctx.get('http_version', ''), 'module_version': config.get('module_version', ''), - 'risk_mode': config.get('module_mode', '') + 'risk_mode': config.get('module_mode', ''), + 'px_cookie_hmac': ctx.get('cookie_hmac', '') } } @@ -53,6 +60,7 @@ def prepare_risk_body(ctx, config): logger.debug('attaching px_cookie to request') body['additional']['px_cookie'] = ctx.get('decoded_cookie') + logger.debug("PxAPI[send_risk_request] request body: " + str(body)) return body diff --git a/perimeterx/px_cookie_validator.py b/perimeterx/px_cookie_validator.py index 2834e56..2aedda6 100644 --- a/perimeterx/px_cookie_validator.py +++ b/perimeterx/px_cookie_validator.py @@ -23,7 +23,7 @@ def verify(ctx, config): if not px_cookie.deserialize(): logger.error('Cookie decryption failed') - ctx['px_orig_cookie'] = px_cookie + ctx['px_orig_cookie'] = px_cookie.raw_cookie ctx['s2s_call_reason'] = 'cookie_decryption_failed' return False @@ -44,7 +44,7 @@ def verify(ctx, config): logger.debug('Cookie expired') return False - if not px_cookie.is_secured(): + if px_cookie.is_secured(): logger.debug('Cookie validation failed') ctx['s2s_call_reason'] = 'cookie_validation_failed' return False @@ -54,5 +54,6 @@ def verify(ctx, config): except Exception, e: traceback.print_exc() logger.debug('Could not decrypt cookie, exception was thrown, decryption failed ' + e.message) + ctx['px_orig_cookie'] = px_cookie.raw_cookie ctx['s2s_call_reason'] = 'cookie_decryption_failed' return False From 9ecac15b3f55b39ce3191384a8168da4884f2a57 Mon Sep 17 00:00:00 2001 From: nitzanpx Date: Thu, 18 May 2017 09:52:20 +0300 Subject: [PATCH 03/35] remvoed iml file --- .gitignore | 1 + perimeterx-python-wsgi.iml | 27 --------------------------- 2 files changed, 1 insertion(+), 27 deletions(-) delete mode 100644 perimeterx-python-wsgi.iml diff --git a/.gitignore b/.gitignore index 7d0865e..5429d7b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .idea +.iml *.pyc db.sqlite3 manage.py diff --git a/perimeterx-python-wsgi.iml b/perimeterx-python-wsgi.iml deleted file mode 100644 index 870de95..0000000 --- a/perimeterx-python-wsgi.iml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - - \ No newline at end of file From 2e86406cd98642dc11b148e7abdd352f73378697 Mon Sep 17 00:00:00 2001 From: nitzanpx Date: Thu, 18 May 2017 10:34:57 +0300 Subject: [PATCH 04/35] added sort to px_cookies keys --- perimeterx/px_cookie.py | 1 + 1 file changed, 1 insertion(+) diff --git a/perimeterx/px_cookie.py b/perimeterx/px_cookie.py index 20539da..afb6f8b 100644 --- a/perimeterx/px_cookie.py +++ b/perimeterx/px_cookie.py @@ -20,6 +20,7 @@ def __init__(self): def build_px_cookie(ctx, config): config["logger"].debug("PxCookie[build_px_cookie]") px_cookies = ctx['px_cookies'].keys() + px_cookies.sort(reverse=True) # Check that its not empty if not px_cookies: From eb8c1da1aa482b721dc4db8e4e4879166fbca720 Mon Sep 17 00:00:00 2001 From: nitzanpx Date: Wed, 24 May 2017 13:44:46 +0300 Subject: [PATCH 05/35] sorting cookie keys after its tested that its not empty --- perimeterx/px_cookie.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perimeterx/px_cookie.py b/perimeterx/px_cookie.py index afb6f8b..005e967 100644 --- a/perimeterx/px_cookie.py +++ b/perimeterx/px_cookie.py @@ -20,12 +20,12 @@ def __init__(self): def build_px_cookie(ctx, config): config["logger"].debug("PxCookie[build_px_cookie]") px_cookies = ctx['px_cookies'].keys() - px_cookies.sort(reverse=True) # Check that its not empty if not px_cookies: return None - + + px_cookies.sort(reverse=True) prefix = px_cookies[0] if prefix == PREFIX_PX_COOKIE_V1: config["logger"].debug("PxCookie[build_px_cookie] using cookie v1") From 339c72b09b935d05dc4f2040af1604c3527356a3 Mon Sep 17 00:00:00 2001 From: Johnny Tordgeman Date: Mon, 19 Nov 2018 10:46:39 +0200 Subject: [PATCH 06/35] APP-1362 added request_cookie_names (#13) * added request_cookie_names * renamed var --- perimeterx/px_api.py | 3 ++- perimeterx/px_context.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/perimeterx/px_api.py b/perimeterx/px_api.py index 844c99d..df2769e 100644 --- a/perimeterx/px_api.py +++ b/perimeterx/px_api.py @@ -48,7 +48,8 @@ def prepare_risk_body(ctx, config): 'http_version': ctx.get('http_version', ''), 'module_version': config.get('module_version', ''), 'risk_mode': config.get('module_mode', ''), - 'px_cookie_hmac': ctx.get('cookie_hmac', '') + 'px_cookie_hmac': ctx.get('cookie_hmac', ''), + 'request_cookie_names': ctx.get('cookie_names', '') } } diff --git a/perimeterx/px_context.py b/perimeterx/px_context.py index 716becd..21fad3f 100644 --- a/perimeterx/px_context.py +++ b/perimeterx/px_context.py @@ -11,6 +11,7 @@ def build_context(environ, config): http_version = '1.1' http_protocol = 'http://' px_cookies = {} + request_cookie_names = list() # IP Extraction if config.get('ip_handler'): @@ -37,6 +38,7 @@ def build_context(environ, config): cookie_keys = cookies.keys() for key in cookie_keys: + request_cookie_names.append(key) if key == PREFIX_PX_COOKIE_V1 or key == PREFIX_PX_COOKIE_V3: logger.debug('Found cookie prefix:' + key) px_cookies[key] = cookies.get(key).value @@ -54,6 +56,7 @@ def build_context(environ, config): 'full_url': full_url, 'uri': uri, 'hostname': hostname, - 'px_cookies': px_cookies + 'px_cookies': px_cookies, + 'cookie_names': request_cookie_names } return ctx From 7a1d824de5c9fa7ca90b6db876605078c706ab35 Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Tue, 20 Nov 2018 13:33:03 +0200 Subject: [PATCH 07/35] Added FP client --- perimeterx/middleware.py | 16 ++++++-- perimeterx/px_constants.py | 9 ++++- perimeterx/px_context.py | 10 ++++- perimeterx/px_httpc.py | 20 ++++++++++ perimeterx/px_proxy.py | 79 ++++++++++++++++++++++++++++++++++++++ perimeterx/px_utils.py | 18 +++++++++ 6 files changed, 146 insertions(+), 6 deletions(-) create mode 100644 perimeterx/px_proxy.py create mode 100644 perimeterx/px_utils.py diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index bab9973..32562e4 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -6,6 +6,7 @@ import px_captcha import px_api import px_template +from px_proxy import PXProxy import Cookie @@ -27,7 +28,10 @@ def __init__(self, app, config=None): 'api_timeout': 1, 'custom_logo': None, 'css_ref': None, - 'js_ref': None + 'js_ref': None, + 'client_host' : 'client.perimeterx.net', + 'first_party': True, + 'first_party_xhr_enabled': True } self.config = dict(self.config.items() + config.items()) @@ -45,6 +49,7 @@ def __init__(self, app, config=None): if not config['cookie_key']: logger.error('PX Cookie Key is missing') raise ValueError('PX Cookie Key is missing') + self.reverse_prefix = self.config['app_id'][2:].lower() px_httpc.init(self.config) @@ -53,7 +58,7 @@ def custom_start_response(status, headers, exc_info=None): cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE')) if cookies.get('_pxCaptcha') and cookies.get('_pxCaptcha').value: cookie = Cookie.SimpleCookie() - cookie['_pxCaptcha'] = ''; + cookie['_pxCaptcha'] = '' cookie['_pxCaptcha']['expires'] = 'Expires=Thu, 01 Jan 1970 00:00:00 GMT'; headers.append(('Set-Cookie', cookie['_pxCaptcha'].OutputString())) self.config['logger'].debug('Cleared Cookie'); @@ -65,9 +70,12 @@ def _verify(self, environ, start_response): logger = self.config['logger'] try: ctx = px_context.build_context(environ, self.config) - + uri = ctx.get('uri') + px_proxy = PXProxy(self.config, ctx) + if px_proxy.should_reverse_request(uri): + return px_proxy.handle_reverse_request(environ, self.config, ctx, start_response) if ctx.get('module_mode') == 'inactive' or is_static_file(ctx): - logger.debug('Filter static file request. uri: ' + ctx.get('uri')) + logger.debug('Filter static file request. uri: ' + uri) return self.app(environ, start_response) cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE')) diff --git a/perimeterx/px_constants.py b/perimeterx/px_constants.py index dbfdf04..0de6ef5 100644 --- a/perimeterx/px_constants.py +++ b/perimeterx/px_constants.py @@ -2,4 +2,11 @@ PREFIX_PX_COOKIE_V3 = '_px3' TRANS_5C = b"".join(chr(x ^ 0x5C) for x in range(256)) -TRANS_36 = b"".join(chr(x ^ 0x36) for x in range(256)) \ No newline at end of file +TRANS_36 = b"".join(chr(x ^ 0x36) for x in range(256)) + +XHR_PATH = 'xhr' +CLIENT_FP_PATH = 'init.js' +CLIENT_TP_PATH = 'main.min.js' +CAPTCHA_PATH = 'captcha' +FIRST_PARTY_HEADER = 'x-px-first-party' +ENFORCER_TRUE_IP_HEADER = 'x-px-enforcer-true-ip' diff --git a/perimeterx/px_context.py b/perimeterx/px_context.py index 716becd..759e3d5 100644 --- a/perimeterx/px_context.py +++ b/perimeterx/px_context.py @@ -11,6 +11,7 @@ def build_context(environ, config): http_version = '1.1' http_protocol = 'http://' px_cookies = {} + request_cookie_names = list() # IP Extraction if config.get('ip_handler'): @@ -37,6 +38,7 @@ def build_context(environ, config): cookie_keys = cookies.keys() for key in cookie_keys: + request_cookie_names.append(key) if key == PREFIX_PX_COOKIE_V1 or key == PREFIX_PX_COOKIE_V3: logger.debug('Found cookie prefix:' + key) px_cookies[key] = cookies.get(key).value @@ -54,6 +56,12 @@ def build_context(environ, config): 'full_url': full_url, 'uri': uri, 'hostname': hostname, - 'px_cookies': px_cookies + 'px_cookies': px_cookies, + 'ip': extract_ip(config, environ), + 'cookie_names': request_cookie_names } return ctx + +def extract_ip(config, environ): + return environ['HTTP_X_FORWARDED_FOR'].split(',')[-1].strip() + diff --git a/perimeterx/px_httpc.py b/perimeterx/px_httpc.py index 33282ee..73f0cd0 100644 --- a/perimeterx/px_httpc.py +++ b/perimeterx/px_httpc.py @@ -32,3 +32,23 @@ def send(uri, body, config): except httplib.HTTPException: init(config) return False + +def sendReverse(url, path, body, headers, config): + logger = config['logger'] + try: + start = time.time() + http_client = httplib.HTTPSConnection(url, timeout=config.get('api_timeout', 1)) + http_client.request('GET', path, "", headers=headers) + response = http_client.getresponse() + + if response.status != 200: + logger.error('error posting server to server call ' + response.reason) + return False + + logger.debug('Server call took ' + str(time.time() - start) + 'ms') + return response + + except httplib.HTTPException: + init(config) + return False + diff --git a/perimeterx/px_proxy.py b/perimeterx/px_proxy.py new file mode 100644 index 0000000..4111347 --- /dev/null +++ b/perimeterx/px_proxy.py @@ -0,0 +1,79 @@ +import px_constants +import px_logger +import px_httpc +import px_utils + +hoppish = { 'connection', 'keep-alive', 'proxy-authenticate', + 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', + 'upgrade' +} + +class PXProxy(object): + def __init__(self, px_config, pxCtx): + reverse_app_id = px_config['app_id'][2:] + self.client_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.CLIENT_FP_PATH) + self.xhr_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.XHR_PATH) + self.captcha_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.CAPTCHA_PATH) + + + + def should_reverse_request(self, uri): + if uri.startswith(self.client_reverse_prefix) or uri.startswith(self.xhr_reverse_prefix) or uri.startswith(self.captcha_reverse_prefix): + return True + return False + + + def handle_reverse_request(self, environ, config, ctx, start_response): + uri = ctx.get('uri') + if uri.startswith(self.client_reverse_prefix): + return self.send_reverse_client_request(config=config, context=ctx, start_response=start_response) + if uri.startswith(self.xhr_reverse_prefix): + return self.send_reverse_xhr_request(config=config, context=ctx, start_response=start_response) + + + def send_reverse_client_request(self, config, context, start_response): + if not config['first_party']: + status = 200 + headers = { + 'Content-Type', 'application/javascript' + } + start_response(status, headers) + return '' + + client_request_uri = '/{}/main.min.js'.format(config['app_id']) + # px_logger.Logger.debug('Forwarding request from {} to client at {}{}'.format(ctx.get('uri').lower(),pxConfig.CLIENT_HOST,clientRequestUri)) + headers = {'host': config['client_host'], + px_constants.FIRST_PARTY_HEADER: 1, + px_constants.ENFORCER_TRUE_IP_HEADER: context['ip']} + filtered_headers = px_utils.filterSensitiveHeaders(context['headers'], config) + filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) + response = px_httpc.sendReverse(url=config['client_host'], path=client_request_uri, body= '', headers=filtered_headers,config=config) + # headers_dict = dict(response.getheaders()) + headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) + start_response(str(response.status) + ' ' + response.reason, headers) + return response.read() + + def send_reverse_xhr_request(self, config, context, start_response): + if None: + print "asdsadas" + # if (!pxConfig.FIRST_PARTY_ENABLED | | !pxConfig.FIRST_PARTY_XHR_ENABLED) { + # if (req.originalUrl.toLowerCase().includes('gif')) { + # res = { + # status: 200, + # header: {key: 'Content-Type', value: 'image/gif'}, + # body: Buffer. + # from + # (pxConfig.EMPTY_GIF_B64, 'base64') + # }; + # } else { + # res = { + # status: 200, + # header: {key: 'Content-Type', value: 'application/json'}, + # body: {} + # }; + # } + # return cb(null, res); + # } + # pass + + diff --git a/perimeterx/px_utils.py b/perimeterx/px_utils.py new file mode 100644 index 0000000..45941dd --- /dev/null +++ b/perimeterx/px_utils.py @@ -0,0 +1,18 @@ + +def filterSensitiveHeaders(headers, config): + + sensitive_keys = config.get('SENSITIVE_HEADERS') + if not sensitive_keys == None: + retval = {} + for header_name in headers: + if not header_name in sensitive_keys: + retval[header_name] = sensitive_keys[header_name] + return retval + else: + return headers + +def merge_two_dicts(x, y): + z = x.copy() # start with x's keys and values + z.update(y) # modifies z with y's keys and values & returns None + return z + From e0c56fe803997363d593c16e80a9b825e9e4f9f9 Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Wed, 21 Nov 2018 13:59:44 +0200 Subject: [PATCH 08/35] XHR --- perimeterx/middleware.py | 5 +- perimeterx/px_constants.py | 5 ++ perimeterx/px_context.py | 15 +++++- perimeterx/px_httpc.py | 4 +- perimeterx/px_proxy.py | 100 +++++++++++++++++++++++++------------ perimeterx/px_utils.py | 13 ++++- 6 files changed, 103 insertions(+), 39 deletions(-) diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index 32562e4..61f70d2 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -8,6 +8,7 @@ import px_template from px_proxy import PXProxy import Cookie +import px_constants class PerimeterX(object): @@ -31,7 +32,7 @@ def __init__(self, app, config=None): 'js_ref': None, 'client_host' : 'client.perimeterx.net', 'first_party': True, - 'first_party_xhr_enabled': True + 'first_party_xhr_enabled': True, } self.config = dict(self.config.items() + config.items()) @@ -39,7 +40,9 @@ def __init__(self, app, config=None): if not config['app_id']: logger.error('PX App ID is missing') raise ValueError('PX App ID is missing') + url = px_constants.COLLECTOR_URL + self.config['collector_url'] = url.format(config.get('app_id').lower()) # if APP_ID is not set, use the deafult perimeterx server - else, use the appid specific sapi. self.config['perimeterx_server_host'] = 'sapi.perimeterx.net' if self.config['app_id'] == 'PX_APP_ID' else 'sapi-' + self.config['app_id'].lower() + '.perimeterx.net' if not config['auth_token']: diff --git a/perimeterx/px_constants.py b/perimeterx/px_constants.py index 0de6ef5..20c5b43 100644 --- a/perimeterx/px_constants.py +++ b/perimeterx/px_constants.py @@ -10,3 +10,8 @@ CAPTCHA_PATH = 'captcha' FIRST_PARTY_HEADER = 'x-px-first-party' ENFORCER_TRUE_IP_HEADER = 'x-px-enforcer-true-ip' +EMPTY_GIF_B64 = 'R0lGODlhAQABAPAAAAAAAAAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==' +COLLECTOR_URL = 'https://collector-{}.perimeterx.net' +COLLECTOR_HOST = 'collector.perimeterx.net' +FIRST_PARTY_FORWARDED_FOR = 'X-FORWARDED-FOR' +CAPTCHA_HOST = 'captcha.px-cdn.net' diff --git a/perimeterx/px_context.py b/perimeterx/px_context.py index 759e3d5..3094d5f 100644 --- a/perimeterx/px_context.py +++ b/perimeterx/px_context.py @@ -42,11 +42,17 @@ def build_context(environ, config): if key == PREFIX_PX_COOKIE_V1 or key == PREFIX_PX_COOKIE_V3: logger.debug('Found cookie prefix:' + key) px_cookies[key] = cookies.get(key).value + vid = None + if '_pxvid' in cookie_keys: + vid = cookies.get('_pxvid').value + else: + vid = '' user_agent = headers.get('user-agent') uri = environ.get('PATH_INFO') or '' full_url = http_protocol + headers.get('host') or environ.get('SERVER_NAME') or '' + uri hostname = headers.get('host') + ctx = { 'headers': headers, 'http_method': http_method, @@ -58,10 +64,15 @@ def build_context(environ, config): 'hostname': hostname, 'px_cookies': px_cookies, 'ip': extract_ip(config, environ), - 'cookie_names': request_cookie_names + 'cookie_names': request_cookie_names, + 'vid': vid } return ctx def extract_ip(config, environ): - return environ['HTTP_X_FORWARDED_FOR'].split(',')[-1].strip() + ip = environ.get('HTTP_X_FORWARDED_FOR') + if not ip == None: + return ip.split(',')[-1].strip() + else: + return '' diff --git a/perimeterx/px_httpc.py b/perimeterx/px_httpc.py index 73f0cd0..8be4dfa 100644 --- a/perimeterx/px_httpc.py +++ b/perimeterx/px_httpc.py @@ -33,12 +33,12 @@ def send(uri, body, config): init(config) return False -def sendReverse(url, path, body, headers, config): +def sendReverse(url, path, body, headers, config, method): logger = config['logger'] try: start = time.time() http_client = httplib.HTTPSConnection(url, timeout=config.get('api_timeout', 1)) - http_client.request('GET', path, "", headers=headers) + http_client.request(method, path, body, headers=headers) response = http_client.getresponse() if response.status != 200: diff --git a/perimeterx/px_proxy.py b/perimeterx/px_proxy.py index 4111347..9ae6db5 100644 --- a/perimeterx/px_proxy.py +++ b/perimeterx/px_proxy.py @@ -2,11 +2,13 @@ import px_logger import px_httpc import px_utils +import base64 + +hoppish = {'connection', 'keep-alive', 'proxy-authenticate', + 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', + 'upgrade' + } -hoppish = { 'connection', 'keep-alive', 'proxy-authenticate', - 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', - 'upgrade' -} class PXProxy(object): def __init__(self, px_config, pxCtx): @@ -15,21 +17,21 @@ def __init__(self, px_config, pxCtx): self.xhr_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.XHR_PATH) self.captcha_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.CAPTCHA_PATH) - - def should_reverse_request(self, uri): - if uri.startswith(self.client_reverse_prefix) or uri.startswith(self.xhr_reverse_prefix) or uri.startswith(self.captcha_reverse_prefix): + if uri.startswith(self.client_reverse_prefix) or uri.startswith(self.xhr_reverse_prefix) or uri.startswith( + self.captcha_reverse_prefix): return True return False - def handle_reverse_request(self, environ, config, ctx, start_response): uri = ctx.get('uri') + if uri.startswith(self.client_reverse_prefix): return self.send_reverse_client_request(config=config, context=ctx, start_response=start_response) if uri.startswith(self.xhr_reverse_prefix): return self.send_reverse_xhr_request(config=config, context=ctx, start_response=start_response) - + if uri.startswith(self.captcha_reverse_prefix): + return self.send_reverse_captcha_request(config=config, context=ctx, start_response=start_response) def send_reverse_client_request(self, config, context, start_response): if not config['first_party']: @@ -42,38 +44,70 @@ def send_reverse_client_request(self, config, context, start_response): client_request_uri = '/{}/main.min.js'.format(config['app_id']) # px_logger.Logger.debug('Forwarding request from {} to client at {}{}'.format(ctx.get('uri').lower(),pxConfig.CLIENT_HOST,clientRequestUri)) + headers = {'host': config['client_host'], px_constants.FIRST_PARTY_HEADER: 1, - px_constants.ENFORCER_TRUE_IP_HEADER: context['ip']} - filtered_headers = px_utils.filterSensitiveHeaders(context['headers'], config) + px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} + filtered_headers = px_utils.filter_sensitive_headers(context['headers'], config) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) - response = px_httpc.sendReverse(url=config['client_host'], path=client_request_uri, body= '', headers=filtered_headers,config=config) + response = px_httpc.sendReverse(url=config['client_host'], path=client_request_uri, body='', + headers=filtered_headers, config=config, method='GET') # headers_dict = dict(response.getheaders()) headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) start_response(str(response.status) + ' ' + response.reason, headers) return response.read() def send_reverse_xhr_request(self, config, context, start_response): - if None: - print "asdsadas" - # if (!pxConfig.FIRST_PARTY_ENABLED | | !pxConfig.FIRST_PARTY_XHR_ENABLED) { - # if (req.originalUrl.toLowerCase().includes('gif')) { - # res = { - # status: 200, - # header: {key: 'Content-Type', value: 'image/gif'}, - # body: Buffer. - # from - # (pxConfig.EMPTY_GIF_B64, 'base64') - # }; - # } else { - # res = { - # status: 200, - # header: {key: 'Content-Type', value: 'application/json'}, - # body: {} - # }; - # } - # return cb(null, res); - # } - # pass + uri = context.get('uri') + if not config.get('first_party') or not config.get('first_party_xhr_enabled'): + body, content_type = self.return_default_response(uri) + + start_response('200 OK', [content_type]) + return body + + xhr_path_index = uri.find('/xhr') + suffix_uri = uri[xhr_path_index + 4:] + + host = config.get('collector_url').replace('https://', '') + headers = {'host': host, + px_constants.FIRST_PARTY_HEADER: 1, + px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} + + if context.get('vid') is not None: + headers['Cookies'] = '_pxvid=' + context.get('vid') + + filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), config, context.get('ip')) + filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) + response = px_httpc.sendReverse(url=host, path=suffix_uri, body='', + headers=filtered_headers, config=config, method=context.get('http_method')) + if response.status >= 400: + body, content_type = self.return_default_response(uri) + + start_response('200 OK', [content_type]) + return body + response_headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) + start_response(str(response.status) + ' ' + response.reason, response_headers) + return response.read() + + # logger.debug(`Host header modified to ${host}`); + + def return_default_response(self, uri): + if 'gif' in uri.lower(): + content_type = tuple('Content-Type', 'image/gif') + body = base64.b64decode(px_constants.EMPTY_GIF_B64) + else: + content_type = tuple('Content-Type', 'application/json') + body = {} + return body, content_type + + def send_reverse_captcha_request(self, config, context, start_response): + if not config['first_party']: + status = 200 + headers = { + 'Content-Type', 'application/javascript' + } + start_response(status, headers) + return '' + uri = context.get('uri') diff --git a/perimeterx/px_utils.py b/perimeterx/px_utils.py index 45941dd..4139fe6 100644 --- a/perimeterx/px_utils.py +++ b/perimeterx/px_utils.py @@ -1,5 +1,7 @@ -def filterSensitiveHeaders(headers, config): +import px_constants + +def filter_sensitive_headers(headers, config): sensitive_keys = config.get('SENSITIVE_HEADERS') if not sensitive_keys == None: @@ -16,3 +18,12 @@ def merge_two_dicts(x, y): z.update(y) # modifies z with y's keys and values & returns None return z + +def handle_proxy_headers(headers, config, ip): + filtered_headers = filter_sensitive_headers(headers, config) + for item in filtered_headers.keys(): + if item.upper() == px_constants.FIRST_PARTY_FORWARDED_FOR: + filtered_headers[item] = ip + else: + filtered_headers[px_constants.FIRST_PARTY_FORWARDED_FOR] = ip + return filtered_headers From 4a91dc5983f26a4b7a626f2850816571a4330f52 Mon Sep 17 00:00:00 2001 From: alexbpx <41577203+alexbpx@users.noreply.github.com> Date: Thu, 22 Nov 2018 11:06:26 +0200 Subject: [PATCH 09/35] Captcha V2 (#14) * Captcha V2 * Fixed PR issues --- perimeterx/middleware.py | 27 +-- perimeterx/px_api.py | 13 +- perimeterx/px_blocker.py | 81 ++++++++ perimeterx/px_captcha.py | 46 ----- perimeterx/px_constants.py | 16 +- perimeterx/px_template.py | 23 +-- perimeterx/templates/block.mustache | 146 --------------- perimeterx/templates/block_template.mustache | 175 ++++++++++++++++++ perimeterx/templates/captcha.mustache | 185 ------------------- perimeterx/templates/ratelimit.mustache | 9 + 10 files changed, 302 insertions(+), 419 deletions(-) create mode 100644 perimeterx/px_blocker.py delete mode 100644 perimeterx/px_captcha.py delete mode 100644 perimeterx/templates/block.mustache create mode 100644 perimeterx/templates/block_template.mustache delete mode 100644 perimeterx/templates/captcha.mustache create mode 100644 perimeterx/templates/ratelimit.mustache diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index bab9973..84ead6c 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -3,9 +3,8 @@ import px_activities_client import px_cookie_validator import px_httpc -import px_captcha +import px_blocker import px_api -import px_template import Cookie @@ -27,7 +26,8 @@ def __init__(self, app, config=None): 'api_timeout': 1, 'custom_logo': None, 'css_ref': None, - 'js_ref': None + 'js_ref': None, + 'is_mobile': False } self.config = dict(self.config.items() + config.items()) @@ -45,7 +45,7 @@ def __init__(self, app, config=None): if not config['cookie_key']: logger.error('PX Cookie Key is missing') raise ValueError('PX Cookie Key is missing') - + self.PXBlocker = px_blocker.PXBlocker() px_httpc.init(self.config) def __call__(self, environ, start_response): @@ -59,7 +59,7 @@ def custom_start_response(status, headers, exc_info=None): self.config['logger'].debug('Cleared Cookie'); return start_response(status, headers, exc_info) - return self._verify(environ, custom_start_response) + return self._verify(environ, start_response) def _verify(self, environ, start_response): logger = self.config['logger'] @@ -71,11 +71,6 @@ def _verify(self, environ, start_response): return self.app(environ, start_response) cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE')) - if self.config.get('captcha_enabled') and cookies.get('_pxCaptcha') and cookies.get('_pxCaptcha').value: - pxCaptcha = cookies.get('_pxCaptcha').value - if px_captcha.verify(ctx, self.config, pxCaptcha): - logger.debug('User passed captcha verification. user ip: ' + ctx.get('socket_ip')) - return self.app(environ, start_response) # PX Cookie verification if not px_cookie_validator.verify(ctx, self.config) and self.config.get('server_calls_enabled', True): @@ -98,17 +93,7 @@ def handle_verification(self, ctx, config, environ, start_response): px_activities_client.send_block_activity(ctx, config) return config['custom_block_handler'](ctx, start_response) elif config.get('module_mode', 'active_monitoring') == 'active_blocking': - vid = ctx.get('vid', '') - uuid = ctx.get('uuid', '') - template = 'block' - if config.get('captcha_enabled', False): - template = 'captcha' - - body = px_template.get_template(template, self.config, uuid, vid) - - px_activities_client.send_block_activity(ctx, config) - start_response("403 Forbidden", [('Content-Type', 'text/html')]) - return [str(body)] + return self.PXBlocker.handle_blocking(ctx=ctx, config=config, start_response=start_response) else: return self.pass_traffic(environ, start_response, ctx) diff --git a/perimeterx/px_api.py b/perimeterx/px_api.py index df2769e..8208f33 100644 --- a/perimeterx/px_api.py +++ b/perimeterx/px_api.py @@ -9,7 +9,7 @@ def send_risk_request(ctx, config): def verify(ctx, config): logger = config['logger'] - logger.debug("PxAPI[verify]") + logger.debug("PXVerify") try: response = send_risk_request(ctx, config) if response: @@ -18,8 +18,17 @@ def verify(ctx, config): ctx['uuid'] = response['uuid'] ctx['block_action'] = response['action'] if score >= config['blocking_score']: - logger.debug("PxAPI[verify] block score threshold reached") + logger.debug("PXVerify block score threshold reached, will initiate blocking") ctx['block_reason'] = 's2s_high_score' + elif response['action'] is 'j' and response.get('action_data') is not None and response.get('action_data').get('body') is not None: + logger.debug("PXVerify received javascript challenge action") + ctx['block_action_data'] = response.get('action_data').get('body') + ctx['block_reason'] = 'challenge' + elif response['action'] is 'r': + logger.debug("PXVerify received javascript ratelimit action") + ctx['block_reason'] = 'exceeded_rate_limit' + else: + ctx['pass_reason'] = 's2s' logger.debug("PxAPI[verify] S2S completed") return True diff --git a/perimeterx/px_blocker.py b/perimeterx/px_blocker.py new file mode 100644 index 0000000..5bc574e --- /dev/null +++ b/perimeterx/px_blocker.py @@ -0,0 +1,81 @@ +import pystache +import px_template +import px_constants +import json + + +class PXBlocker(object): + def __init__(self): + self.mustache_renderer = pystache.Renderer() + self.ratelimit_rendered_page = self.mustache_renderer.render( + px_template.get_template(px_constants.RATELIMIT_TEMPLATE), {}) + + def handle_blocking(self, ctx, config, start_response): + action = ctx.get('block_action') + status = '403 Forbidden' + + is_json_response = self.is_json_response(ctx) + if is_json_response: + content_type = 'application/json' + else: + content_type = 'text/html' + headers = [('Content-Type', content_type)] + + if action is 'j': + blocking_props = ctx['block_action_data'] + blocking_response = blocking_props + elif action is 'r': + blocking_response = self.ratelimit_rendered_page + status = '429 Too Many Requests' + else: + blocking_props = self.prepare_properties(ctx, config) + blocking_response = self.mustache_renderer.render(px_template.get_template(px_constants.BLOCK_TEMPLATE), + blocking_props) + start_response(status, headers) + if is_json_response: + blocking_response = json.dumps(blocking_props) + return str(blocking_response) + + def prepare_properties(self, ctx, config): + app_id = config.get('app_id').lower() + vid = ctx.get('vid') if ctx.get('vid') is not None else '' + uuid = ctx.get('uuid') + custom_logo = config.get('CUSTOM_LOGO') if config.get('CUSTOM_LOGO') is not None else '' + is_mobile_num = 1 if ctx.get('is_mobile') else 0 + captcha_uri = 'captcha.js?a={}&u={}&v={}&m={}'.format(ctx.get('block_action'), uuid, vid, is_mobile_num) + + if config.get('first_party') and not ctx.get('is_mobile'): + prefix = app_id[2:] + js_client_src = '/{}/{}'.format(prefix, px_constants.CLIENT_FP_PATH) + captcha_src = '/{}/{}/{}'.format(prefix, px_constants.CAPTCHA_FP_PATH, captcha_uri) + host_url = '/{}/{}'.format(prefix, px_constants.XHR_FP_PATH) + else: + js_client_src = '//{}/{}/main.min.js'.format(px_constants.CLIENT_HOST, app_id) + captcha_src = '//{}/{}/{}'.format(px_constants.CAPTCHA_HOST, app_id, captcha_uri) + host_url = px_constants.COLLECTOR_URL.format(app_id.lower()) + + return { + 'refId': uuid, + 'appId': app_id, + 'vid': vid, + 'uuid': uuid, + 'customLogo': custom_logo, + 'cssRef': config.get('css_ref'), + 'jsRef': config.get('js_ref'), + 'logoVisibility': 'visible' if custom_logo is not None else 'hidden', + 'hostUrl': host_url, + 'jsClientSrc': js_client_src, + 'firstPartyEnabled': config.get('first_party'), + 'blockScript': captcha_src + } + + def is_json_response(self, ctx): + headers = ctx.get('headers') + if ctx.get('block_action') is not 'r': + for item in headers.keys(): + if (item.lower() == 'accept' or item.lower() == 'content-type'): + item_arr = headers[item].split(',') + for header_item in item_arr: + if header_item.strip() == 'application/json': + return True + return False diff --git a/perimeterx/px_captcha.py b/perimeterx/px_captcha.py deleted file mode 100644 index 489bd7e..0000000 --- a/perimeterx/px_captcha.py +++ /dev/null @@ -1,46 +0,0 @@ -import px_httpc - -def verify(ctx, config, captcha): - if not captcha: - return False - - split_captcha = captcha.split(':') - - if not len(split_captcha) == 3: - return False - - captcha_value = split_captcha[0] - vid = split_captcha[1] - uuid = split_captcha[2] - - if not vid or not captcha_value or not uuid: - return False - - ctx['uuid'] = uuid; - - response = send_captcha_request(vid, uuid, captcha_value, ctx, config) - return response and response.get('status', 1) == 0 - -def send_captcha_request(vid, uuid, captcha_value, ctx, config): - body = { - 'request': { - 'ip': ctx.get('socket_ip'), - 'headers': format_headers(ctx.get('headers')), - 'uri': ctx.get('uri') - }, - 'pxCaptcha': captcha_value, - 'vid': vid, - 'uuid': uuid, - 'hostname': ctx.get('hostname') - } - response = px_httpc.send('/api/v1/risk/captcha', body=body, config=config) - - return response - - -def format_headers(headers): - ret_val = [] - for key in headers.keys(): - ret_val.append({'name': key, 'value': headers[key]}) - return ret_val - diff --git a/perimeterx/px_constants.py b/perimeterx/px_constants.py index dbfdf04..416b856 100644 --- a/perimeterx/px_constants.py +++ b/perimeterx/px_constants.py @@ -2,4 +2,18 @@ PREFIX_PX_COOKIE_V3 = '_px3' TRANS_5C = b"".join(chr(x ^ 0x5C) for x in range(256)) -TRANS_36 = b"".join(chr(x ^ 0x36) for x in range(256)) \ No newline at end of file +TRANS_36 = b"".join(chr(x ^ 0x36) for x in range(256)) + +BLOCK_TEMPLATE = 'block_template.mustache' +RATELIMIT_TEMPLATE = 'ratelimit.mustache' +CAPTCHA_ACTION_CAPTCHA = 'c' +BLOCK_ACTION_CAPTCHA = 'b' +BLOCK_ACTION_CHALLENGE = 'j' +BLOCK_ACTION_RATE = 'r' +CLIENT_HOST = 'client.perimeterx.net' +CAPTCHA_HOST = 'captcha.px-cdn.net' +COLLECTOR_URL = 'https://collector-{}.perimeterx.net' +CLIENT_FP_PATH = 'init.js' +CAPTCHA_FP_PATH = 'captcha' +XHR_FP_PATH = 'xhr' + diff --git a/perimeterx/px_template.py b/perimeterx/px_template.py index a01c0dd..b299f43 100644 --- a/perimeterx/px_template.py +++ b/perimeterx/px_template.py @@ -1,29 +1,16 @@ -import pystache import os -def get_template(template, config, uuid, vid): - template_content = get_content(template) - props = get_props(config, uuid, vid) - generatedHtml = pystache.render(template_content, props) - return generatedHtml def get_path(): return os.path.dirname(os.path.abspath(__file__)) def get_content(template): - templatePath = "%s/templates/%s.mustache" % (get_path(),template) + templatePath = "%s/templates/%s" % (get_path(), template) file = open(templatePath, "r") content = file.read() return content -def get_props(config, uuid, vid): - return { - 'refId': uuid, - 'appId': config.get('app_id'), - 'vid': vid, - 'uuid': uuid, - 'customLogo': config.get('custom_logo'), - 'cssRef': config.get('css_ref'), - 'jsRef': config.get('js_ref'), - 'logoVisibility': 'visible' if config['custom_logo'] else 'hidden' - } + + +def get_template(template_name): + return get_content(template_name) diff --git a/perimeterx/templates/block.mustache b/perimeterx/templates/block.mustache deleted file mode 100644 index b61c371..0000000 --- a/perimeterx/templates/block.mustache +++ /dev/null @@ -1,146 +0,0 @@ - - - - - - Access to this page has been denied. - - - - {{# cssRef }} - - {{/ cssRef }} - - -
-
- -
-
-
-

Access to this page has been denied.

-
-
-
-
-

- You have been blocked because we believe you are using automation tools to browse the website. -

-

- Please note that Javascript and Cookies must be enabled on your browser to access the website. -

-

- If you think you have been blocked by mistake, please contact the website administrator with the reference ID below. -

-

- Reference ID: #{{refId}} -

-
-
- -
- - - - {{# jsRef }} - - {{/ jsRef }} - - diff --git a/perimeterx/templates/block_template.mustache b/perimeterx/templates/block_template.mustache new file mode 100644 index 0000000..3fcef19 --- /dev/null +++ b/perimeterx/templates/block_template.mustache @@ -0,0 +1,175 @@ + + + + + + Access to this page has been denied. + + + + {{#cssRef}} + + {{/cssRef}} + + + +
+
+ +
+
+
+

Please verify you are a human

+
+
+
+
+ +
+
+

+ Access to this page has been denied because we believe you are using automation tools to browse the + website. +

+

+ This may happen as a result of the following: +

+
    +
  • + Javascript is disabled or blocked by an extension (ad blockers for example) +
  • +
  • + Your browser does not support cookies +
  • +
+

+ Please make sure that Javascript and cookies are enabled on your browser and that you are not blocking + them from loading. +

+

+ Reference ID: #{{refId}} +

+
+
+ +
+ + + + + +{{#jsRef}} + +{{/jsRef}} + + diff --git a/perimeterx/templates/captcha.mustache b/perimeterx/templates/captcha.mustache deleted file mode 100644 index 2ede096..0000000 --- a/perimeterx/templates/captcha.mustache +++ /dev/null @@ -1,185 +0,0 @@ - - - - - - Access to this page has been denied. - - - - {{#cssRef}} - - {{/cssRef}} - - - - -
-
- -
-
-
-

Please verify you are a human

-
-
-
-
-

- Please click "I am not a robot" to continue -

-
-
-

- Access to this page has been denied because we believe you are using automation tools to browse the website. -

-

- This may happen as a result of the following: -

-
    -
  • - Javascript is disabled or blocked by an extension (ad blockers for example) -
  • -
  • - Your browser does not support cookies -
  • -
-

- Please make sure that Javascript and cookies are enabled on your browser and that you are not blocking them from loading. -

-

- Reference ID: #{{refId}} -

-
-
- -
- - - - - - {{#jsRef}} - - {{/jsRef}} - - diff --git a/perimeterx/templates/ratelimit.mustache b/perimeterx/templates/ratelimit.mustache new file mode 100644 index 0000000..36fd393 --- /dev/null +++ b/perimeterx/templates/ratelimit.mustache @@ -0,0 +1,9 @@ + + + Too Many Requests + + +

Too Many Requests

+

Reached maximum requests limitation, try again soon.

+ + \ No newline at end of file From 3e8c49166b019b170a764ab1eef62e8117111321 Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Thu, 22 Nov 2018 14:24:36 +0200 Subject: [PATCH 10/35] Fixed empty response http content type header --- perimeterx/middleware.py | 2 +- perimeterx/px_proxy.py | 13 +++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index eb10e67..8228375 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -31,7 +31,7 @@ def __init__(self, app, config=None): 'css_ref': None, 'js_ref': None, 'is_mobile': False, - 'client_host' : 'client.perimeterx.net', + 'client_host': 'client.perimeterx.net', 'first_party': True, 'first_party_xhr_enabled': True, } diff --git a/perimeterx/px_proxy.py b/perimeterx/px_proxy.py index 9ae6db5..ed3a4cc 100644 --- a/perimeterx/px_proxy.py +++ b/perimeterx/px_proxy.py @@ -14,8 +14,8 @@ class PXProxy(object): def __init__(self, px_config, pxCtx): reverse_app_id = px_config['app_id'][2:] self.client_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.CLIENT_FP_PATH) - self.xhr_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.XHR_PATH) - self.captcha_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.CAPTCHA_PATH) + self.xhr_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.XHR_FP_PATH) + self.captcha_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.CAPTCHA_FP_PATH) def should_reverse_request(self, uri): if uri.startswith(self.client_reverse_prefix) or uri.startswith(self.xhr_reverse_prefix) or uri.startswith( @@ -35,12 +35,9 @@ def handle_reverse_request(self, environ, config, ctx, start_response): def send_reverse_client_request(self, config, context, start_response): if not config['first_party']: - status = 200 - headers = { - 'Content-Type', 'application/javascript' - } - start_response(status, headers) - return '' + headers = [('Content-Type', 'application/javascript')] + start_response("200 OK", headers) + return "" client_request_uri = '/{}/main.min.js'.format(config['app_id']) # px_logger.Logger.debug('Forwarding request from {} to client at {}{}'.format(ctx.get('uri').lower(),pxConfig.CLIENT_HOST,clientRequestUri)) From e301f2629a19877e2fa1b60d055bc719c84d7478 Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Thu, 22 Nov 2018 15:45:28 +0200 Subject: [PATCH 11/35] Captcha proxy FP --- perimeterx/middleware.py | 11 ---------- perimeterx/px_context.py | 3 ++- perimeterx/px_proxy.py | 45 ++++++++++++++++++++++++++-------------- 3 files changed, 32 insertions(+), 27 deletions(-) diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index 8228375..382131e 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -31,7 +31,6 @@ def __init__(self, app, config=None): 'css_ref': None, 'js_ref': None, 'is_mobile': False, - 'client_host': 'client.perimeterx.net', 'first_party': True, 'first_party_xhr_enabled': True, } @@ -59,16 +58,6 @@ def __init__(self, app, config=None): px_httpc.init(self.config) def __call__(self, environ, start_response): - def custom_start_response(status, headers, exc_info=None): - cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE')) - if cookies.get('_pxCaptcha') and cookies.get('_pxCaptcha').value: - cookie = Cookie.SimpleCookie() - cookie['_pxCaptcha'] = '' - cookie['_pxCaptcha']['expires'] = 'Expires=Thu, 01 Jan 1970 00:00:00 GMT'; - headers.append(('Set-Cookie', cookie['_pxCaptcha'].OutputString())) - self.config['logger'].debug('Cleared Cookie'); - return start_response(status, headers, exc_info) - return self._verify(environ, start_response) def _verify(self, environ, start_response): diff --git a/perimeterx/px_context.py b/perimeterx/px_context.py index 175ec23..fa33498 100644 --- a/perimeterx/px_context.py +++ b/perimeterx/px_context.py @@ -65,7 +65,8 @@ def build_context(environ, config): 'px_cookies': px_cookies, 'cookie_names': request_cookie_names, 'ip': extract_ip(config, environ), - 'vid': vid + 'vid': vid, + 'query_params': environ['QUERY_STRING'] } return ctx diff --git a/perimeterx/px_proxy.py b/perimeterx/px_proxy.py index ed3a4cc..dd40212 100644 --- a/perimeterx/px_proxy.py +++ b/perimeterx/px_proxy.py @@ -12,19 +12,22 @@ class PXProxy(object): def __init__(self, px_config, pxCtx): + self.logger = px_config['logger'] + reverse_app_id = px_config['app_id'][2:] - self.client_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.CLIENT_FP_PATH) - self.xhr_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.XHR_FP_PATH) - self.captcha_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.CAPTCHA_FP_PATH) + self.client_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.CLIENT_FP_PATH).lower() + self.xhr_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.XHR_FP_PATH).lower() + self.captcha_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.CAPTCHA_FP_PATH).lower() def should_reverse_request(self, uri): + uri = uri.lower() if uri.startswith(self.client_reverse_prefix) or uri.startswith(self.xhr_reverse_prefix) or uri.startswith( self.captcha_reverse_prefix): return True return False def handle_reverse_request(self, environ, config, ctx, start_response): - uri = ctx.get('uri') + uri = ctx.get('uri').lower() if uri.startswith(self.client_reverse_prefix): return self.send_reverse_client_request(config=config, context=ctx, start_response=start_response) @@ -40,14 +43,14 @@ def send_reverse_client_request(self, config, context, start_response): return "" client_request_uri = '/{}/main.min.js'.format(config['app_id']) - # px_logger.Logger.debug('Forwarding request from {} to client at {}{}'.format(ctx.get('uri').lower(),pxConfig.CLIENT_HOST,clientRequestUri)) + self.logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(),px_constants.CLIENT_HOST, client_request_uri)) - headers = {'host': config['client_host'], + headers = {'host': px_constants.CLIENT_HOST, px_constants.FIRST_PARTY_HEADER: 1, px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} filtered_headers = px_utils.filter_sensitive_headers(context['headers'], config) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) - response = px_httpc.sendReverse(url=config['client_host'], path=client_request_uri, body='', + response = px_httpc.sendReverse(url=px_constants.CLIENT_HOST, path=client_request_uri, body='', headers=filtered_headers, config=config, method='GET') # headers_dict = dict(response.getheaders()) headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) @@ -62,7 +65,7 @@ def send_reverse_xhr_request(self, config, context, start_response): start_response('200 OK', [content_type]) return body - xhr_path_index = uri.find('/xhr') + xhr_path_index = uri.find('/' + px_constants.XHR_FP_PATH) suffix_uri = uri[xhr_path_index + 4:] host = config.get('collector_url').replace('https://', '') @@ -75,6 +78,7 @@ def send_reverse_xhr_request(self, config, context, start_response): filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), config, context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) + self.logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, suffix_uri)) response = px_httpc.sendReverse(url=host, path=suffix_uri, body='', headers=filtered_headers, config=config, method=context.get('http_method')) if response.status >= 400: @@ -87,8 +91,6 @@ def send_reverse_xhr_request(self, config, context, start_response): return response.read() - # logger.debug(`Host header modified to ${host}`); - def return_default_response(self, uri): if 'gif' in uri.lower(): content_type = tuple('Content-Type', 'image/gif') @@ -100,11 +102,24 @@ def return_default_response(self, uri): def send_reverse_captcha_request(self, config, context, start_response): if not config['first_party']: - status = 200 - headers = { - 'Content-Type', 'application/javascript' - } + status = '200 OK' + headers = [('Content-Type', 'application/javascript')] start_response(status, headers) return '' - uri = context.get('uri') + uri = '/{}{}?{}'.format(config['app_id'], context.get('uri').lower().replace(self.captcha_reverse_prefix, ''), context['query_params']) + host = px_constants.CAPTCHA_HOST + + headers = {'host': px_constants.CAPTCHA_HOST, + px_constants.FIRST_PARTY_HEADER: 1, + px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} + filtered_headers = px_utils.filter_sensitive_headers(context['headers'], config) + filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) + self.logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, uri)) + response = px_httpc.sendReverse(url=host, path=uri, body='', + headers=filtered_headers, config=config, method='GET') + headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) + start_response(str(response.status) + ' ' + response.reason, headers) + return response.read() + + From a87dba777e803b8b7ffdb2fc05eb8020ce58e04a Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Thu, 22 Nov 2018 16:40:39 +0200 Subject: [PATCH 12/35] Fixed argument alignment --- perimeterx/middleware.py | 2 +- perimeterx/px_proxy.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index 382131e..292ae04 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -67,7 +67,7 @@ def _verify(self, environ, start_response): uri = ctx.get('uri') px_proxy = PXProxy(self.config, ctx) if px_proxy.should_reverse_request(uri): - return px_proxy.handle_reverse_request(environ, self.config, ctx, start_response) + return px_proxy.handle_reverse_request(self.config, ctx, start_response) if ctx.get('module_mode') == 'inactive' or is_static_file(ctx): logger.debug('Filter static file request. uri: ' + uri) return self.app(environ, start_response) diff --git a/perimeterx/px_proxy.py b/perimeterx/px_proxy.py index dd40212..0971562 100644 --- a/perimeterx/px_proxy.py +++ b/perimeterx/px_proxy.py @@ -11,7 +11,7 @@ class PXProxy(object): - def __init__(self, px_config, pxCtx): + def __init__(self, px_config): self.logger = px_config['logger'] reverse_app_id = px_config['app_id'][2:] @@ -26,7 +26,7 @@ def should_reverse_request(self, uri): return True return False - def handle_reverse_request(self, environ, config, ctx, start_response): + def handle_reverse_request(self, config, ctx, start_response): uri = ctx.get('uri').lower() if uri.startswith(self.client_reverse_prefix): From 95283c1e7e1f99775aa676ded8a3d520deb2c940 Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Thu, 22 Nov 2018 17:17:01 +0200 Subject: [PATCH 13/35] Added monitor mode --- perimeterx/middleware.py | 7 ++++--- perimeterx/px_constants.py | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index 84ead6c..bd795e5 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -6,7 +6,7 @@ import px_blocker import px_api import Cookie - +import px_constants class PerimeterX(object): def __init__(self, app, config=None): @@ -27,7 +27,8 @@ def __init__(self, app, config=None): 'custom_logo': None, 'css_ref': None, 'js_ref': None, - 'is_mobile': False + 'is_mobile': False, + 'monitor_mode': px_constants.MONITOR_MODE_BLOCKING } self.config = dict(self.config.items() + config.items()) @@ -92,7 +93,7 @@ def handle_verification(self, ctx, config, environ, start_response): if config.get('custom_block_handler', False): px_activities_client.send_block_activity(ctx, config) return config['custom_block_handler'](ctx, start_response) - elif config.get('module_mode', 'active_monitoring') == 'active_blocking': + elif config.get('module_mode', px_constants.MONITOR_MODE_MONITOR) == px_constants.MONITOR_MODE_BLOCKING: return self.PXBlocker.handle_blocking(ctx=ctx, config=config, start_response=start_response) else: return self.pass_traffic(environ, start_response, ctx) diff --git a/perimeterx/px_constants.py b/perimeterx/px_constants.py index 416b856..7c1feb0 100644 --- a/perimeterx/px_constants.py +++ b/perimeterx/px_constants.py @@ -16,4 +16,6 @@ CLIENT_FP_PATH = 'init.js' CAPTCHA_FP_PATH = 'captcha' XHR_FP_PATH = 'xhr' +MONITOR_MODE_BLOCKING = 'active_blocking' +MONITOR_MODE_MONITOR = 'monitor' From c13e72f9cabacd77b6e8e4c814a1b867d671f028 Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Sat, 24 Nov 2018 11:01:55 +0200 Subject: [PATCH 14/35] Added module mode, risk_rtt and simulated block --- perimeterx/middleware.py | 7 +++---- perimeterx/px_activities_client.py | 11 ++++++++++- perimeterx/px_api.py | 7 +++++++ perimeterx/px_constants.py | 2 ++ perimeterx/px_context.py | 3 ++- perimeterx/px_httpc.py | 6 ------ 6 files changed, 24 insertions(+), 12 deletions(-) diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index bd795e5..bd1e998 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -16,7 +16,7 @@ def __init__(self, app, config=None): 'blocking_score': 60, 'debug_mode': False, 'module_version': 'Python SDK v1.0.3', - 'module_mode': 'active_monitoring', + 'module_mode': 'monitor', 'perimeterx_server_host': 'sapi.perimeterx.net', 'captcha_enabled': True, 'server_calls_enabled': True, @@ -28,15 +28,14 @@ def __init__(self, app, config=None): 'css_ref': None, 'js_ref': None, 'is_mobile': False, - 'monitor_mode': px_constants.MONITOR_MODE_BLOCKING - } + 'monitor_mode': px_constants.MONITOR_MODE_MONITOR, + } self.config = dict(self.config.items() + config.items()) self.config['logger'] = logger = Logger(self.config['debug_mode']) if not config['app_id']: logger.error('PX App ID is missing') raise ValueError('PX App ID is missing') - # if APP_ID is not set, use the deafult perimeterx server - else, use the appid specific sapi. self.config['perimeterx_server_host'] = 'sapi.perimeterx.net' if self.config['app_id'] == 'PX_APP_ID' else 'sapi-' + self.config['app_id'].lower() + '.perimeterx.net' if not config['auth_token']: diff --git a/perimeterx/px_activities_client.py b/perimeterx/px_activities_client.py index 0c4556a..119e737 100644 --- a/perimeterx/px_activities_client.py +++ b/perimeterx/px_activities_client.py @@ -2,6 +2,7 @@ import px_httpc import threading import traceback, sys +import px_constants ACTIVITIES_BUFFER = [] CONFIG = {} @@ -66,5 +67,13 @@ def send_block_activity(ctx, config): send_to_perimeterx('block', ctx, config, { 'block_score': ctx.get('risk_score'), 'client_uuid': ctx.get('uuid'), - 'block_reason': ctx.get('block_reason') + 'block_reason': ctx.get('block_reason'), + 'http_method' : ctx.get('http_method'), + 'http_version': ctx.get('http_version'), + 'px_cookie': ctx.get('decoded_cookie'), + 'risk_rtt': ctx.get('risk_rtt'), + #'cookie_origin':, + 'module_version': px_constants.MODULE_VERSION, + 'simulated_block': config.get('monitor_mode') + }) diff --git a/perimeterx/px_api.py b/perimeterx/px_api.py index 8208f33..f19b8ef 100644 --- a/perimeterx/px_api.py +++ b/perimeterx/px_api.py @@ -1,5 +1,7 @@ import sys import px_httpc +import time + def send_risk_request(ctx, config): @@ -11,12 +13,17 @@ def verify(ctx, config): logger = config['logger'] logger.debug("PXVerify") try: + start = time.time() response = send_risk_request(ctx, config) + risk_rtt = time.time() - start + logger.debug('Risk call took ' + str(risk_rtt) + 'ms') + if response: score = response['score'] ctx['score'] = score ctx['uuid'] = response['uuid'] ctx['block_action'] = response['action'] + ctx['risk_rtt'] = risk_rtt if score >= config['blocking_score']: logger.debug("PXVerify block score threshold reached, will initiate blocking") ctx['block_reason'] = 's2s_high_score' diff --git a/perimeterx/px_constants.py b/perimeterx/px_constants.py index 7c1feb0..98cdfa4 100644 --- a/perimeterx/px_constants.py +++ b/perimeterx/px_constants.py @@ -19,3 +19,5 @@ MONITOR_MODE_BLOCKING = 'active_blocking' MONITOR_MODE_MONITOR = 'monitor' + +MODULE_VERSION = 'Python WSGI Module' diff --git a/perimeterx/px_context.py b/perimeterx/px_context.py index 21fad3f..7df1808 100644 --- a/perimeterx/px_context.py +++ b/perimeterx/px_context.py @@ -57,6 +57,7 @@ def build_context(environ, config): 'uri': uri, 'hostname': hostname, 'px_cookies': px_cookies, - 'cookie_names': request_cookie_names + 'cookie_names': request_cookie_names, + 'risk_rtt': 0 } return ctx diff --git a/perimeterx/px_httpc.py b/perimeterx/px_httpc.py index 33282ee..be6391c 100644 --- a/perimeterx/px_httpc.py +++ b/perimeterx/px_httpc.py @@ -1,6 +1,5 @@ import httplib import json -import time http_client = None @@ -17,17 +16,12 @@ def send(uri, body, config): 'Content-Type': 'application/json' } try: - start = time.time() http_client.request('POST', uri, body=json.dumps(body), headers=headers) r = http_client.getresponse() - if r.status != 200: logger.error('error posting server to server call ' + r.reason) return False - - logger.debug('Server call took ' + str(time.time() - start) + 'ms') response_body = r.read() - return json.loads(response_body) except httplib.HTTPException: init(config) From 62334d816daf0ad077c2488b0fd2b168213d2033 Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Sat, 24 Nov 2018 11:23:26 +0200 Subject: [PATCH 15/35] Changed code style, added http status error handling --- perimeterx/middleware.py | 4 ++-- perimeterx/px_httpc.py | 4 ++-- perimeterx/px_proxy.py | 3 ++- perimeterx/px_utils.py | 19 ++++++++----------- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index 292ae04..5eabf23 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -52,7 +52,7 @@ def __init__(self, app, config=None): if not config['cookie_key']: logger.error('PX Cookie Key is missing') raise ValueError('PX Cookie Key is missing') - self.reverse_prefix = self.config['app_id'][2:].lower() + self.reverse_proxy_prefix = self.config['app_id'][2:].lower() self.PXBlocker = px_blocker.PXBlocker() px_httpc.init(self.config) @@ -101,7 +101,7 @@ def handle_verification(self, ctx, config, environ, start_response): def pass_traffic(self, environ, start_response, ctx): details = {} - if(ctx.get('decoded_cookie','')): + if ctx.get('decoded_cookie', ''): details = {"px_cookie": ctx['decoded_cookie']} px_activities_client.send_to_perimeterx('page_requested', ctx, self.config, details) return self.app(environ, start_response) diff --git a/perimeterx/px_httpc.py b/perimeterx/px_httpc.py index 8be4dfa..c5e631a 100644 --- a/perimeterx/px_httpc.py +++ b/perimeterx/px_httpc.py @@ -21,7 +21,7 @@ def send(uri, body, config): http_client.request('POST', uri, body=json.dumps(body), headers=headers) r = http_client.getresponse() - if r.status != 200: + if r.status >= 400: logger.error('error posting server to server call ' + r.reason) return False @@ -33,7 +33,7 @@ def send(uri, body, config): init(config) return False -def sendReverse(url, path, body, headers, config, method): +def send_reverse(url, path, body, headers, config, method): logger = config['logger'] try: start = time.time() diff --git a/perimeterx/px_proxy.py b/perimeterx/px_proxy.py index 0971562..a56da5f 100644 --- a/perimeterx/px_proxy.py +++ b/perimeterx/px_proxy.py @@ -52,7 +52,8 @@ def send_reverse_client_request(self, config, context, start_response): filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) response = px_httpc.sendReverse(url=px_constants.CLIENT_HOST, path=client_request_uri, body='', headers=filtered_headers, config=config, method='GET') - # headers_dict = dict(response.getheaders()) + + headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) start_response(str(response.status) + ' ' + response.reason, headers) return response.read() diff --git a/perimeterx/px_utils.py b/perimeterx/px_utils.py index 4139fe6..6d1d572 100644 --- a/perimeterx/px_utils.py +++ b/perimeterx/px_utils.py @@ -1,21 +1,18 @@ - import px_constants -def filter_sensitive_headers(headers, config): - sensitive_keys = config.get('SENSITIVE_HEADERS') - if not sensitive_keys == None: - retval = {} - for header_name in headers: - if not header_name in sensitive_keys: - retval[header_name] = sensitive_keys[header_name] - return retval +def filter_sensitive_headers(headers, config): + sensitive_keys = config.get('sensitive_headers') + if not sensitive_keys is not None: + return {header_name: sensitive_keys[header_name] for header_name in headers if + header_name not in sensitive_keys} else: return headers + def merge_two_dicts(x, y): - z = x.copy() # start with x's keys and values - z.update(y) # modifies z with y's keys and values & returns None + z = x.copy() # start with x's keys and values + z.update(y) # modifies z with y's keys and values & returns None return z From b31c31bb0c3e1f7201c4045796610f63aa73e1a2 Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Sat, 24 Nov 2018 11:28:50 +0200 Subject: [PATCH 16/35] fixed http status debugging --- perimeterx/px_httpc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/perimeterx/px_httpc.py b/perimeterx/px_httpc.py index c5e631a..2da2d92 100644 --- a/perimeterx/px_httpc.py +++ b/perimeterx/px_httpc.py @@ -21,7 +21,7 @@ def send(uri, body, config): http_client.request('POST', uri, body=json.dumps(body), headers=headers) r = http_client.getresponse() - if r.status >= 400: + if r.status >= 200: logger.error('error posting server to server call ' + r.reason) return False @@ -41,8 +41,8 @@ def send_reverse(url, path, body, headers, config, method): http_client.request(method, path, body, headers=headers) response = http_client.getresponse() - if response.status != 200: - logger.error('error posting server to server call ' + response.reason) + if response.status != 400: + logger.debug('error reversing the http call ' + response.reason) return False logger.debug('Server call took ' + str(time.time() - start) + 'ms') From 7830e4882d3d788ae490cfb4d88fef90b15b9b0c Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Sat, 24 Nov 2018 11:32:34 +0200 Subject: [PATCH 17/35] Hiding errors from client and captcha calls --- perimeterx/px_httpc.py | 1 - perimeterx/px_proxy.py | 15 ++++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/perimeterx/px_httpc.py b/perimeterx/px_httpc.py index 2da2d92..0fd5369 100644 --- a/perimeterx/px_httpc.py +++ b/perimeterx/px_httpc.py @@ -42,7 +42,6 @@ def send_reverse(url, path, body, headers, config, method): response = http_client.getresponse() if response.status != 400: - logger.debug('error reversing the http call ' + response.reason) return False logger.debug('Server call took ' + str(time.time() - start) + 'ms') diff --git a/perimeterx/px_proxy.py b/perimeterx/px_proxy.py index a56da5f..12a39ee 100644 --- a/perimeterx/px_proxy.py +++ b/perimeterx/px_proxy.py @@ -50,8 +50,8 @@ def send_reverse_client_request(self, config, context, start_response): px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} filtered_headers = px_utils.filter_sensitive_headers(context['headers'], config) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) - response = px_httpc.sendReverse(url=px_constants.CLIENT_HOST, path=client_request_uri, body='', - headers=filtered_headers, config=config, method='GET') + response = px_httpc.send_reverse(url=px_constants.CLIENT_HOST, path=client_request_uri, body='', + headers=filtered_headers, config=config, method='GET') headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) @@ -80,11 +80,12 @@ def send_reverse_xhr_request(self, config, context, start_response): filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), config, context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) self.logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, suffix_uri)) - response = px_httpc.sendReverse(url=host, path=suffix_uri, body='', - headers=filtered_headers, config=config, method=context.get('http_method')) + response = px_httpc.send_reverse(url=host, path=suffix_uri, body='', + headers=filtered_headers, config=config, method=context.get('http_method')) + if response.status >= 400: body, content_type = self.return_default_response(uri) - + px_logger.Logger.debug('error reversing the http call ' + response.reason) start_response('200 OK', [content_type]) return body response_headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) @@ -116,8 +117,8 @@ def send_reverse_captcha_request(self, config, context, start_response): filtered_headers = px_utils.filter_sensitive_headers(context['headers'], config) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) self.logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, uri)) - response = px_httpc.sendReverse(url=host, path=uri, body='', - headers=filtered_headers, config=config, method='GET') + response = px_httpc.send_reverse(url=host, path=uri, body='', + headers=filtered_headers, config=config, method='GET') headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) start_response(str(response.status) + ' ' + response.reason, headers) return response.read() From e2b00c323da74867c77916c525ab844233669632 Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Sat, 24 Nov 2018 11:33:01 +0200 Subject: [PATCH 18/35] Removed newline --- perimeterx/px_proxy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/perimeterx/px_proxy.py b/perimeterx/px_proxy.py index 12a39ee..b3bb3c7 100644 --- a/perimeterx/px_proxy.py +++ b/perimeterx/px_proxy.py @@ -53,7 +53,6 @@ def send_reverse_client_request(self, config, context, start_response): response = px_httpc.send_reverse(url=px_constants.CLIENT_HOST, path=client_request_uri, body='', headers=filtered_headers, config=config, method='GET') - headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) start_response(str(response.status) + ' ' + response.reason, headers) return response.read() From 0a738902c69fe90626e1e75bdda644f1a2caee0d Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Sat, 24 Nov 2018 12:31:42 +0200 Subject: [PATCH 19/35] Changed the name --- perimeterx/middleware.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index bd1e998..9d76903 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -28,7 +28,7 @@ def __init__(self, app, config=None): 'css_ref': None, 'js_ref': None, 'is_mobile': False, - 'monitor_mode': px_constants.MONITOR_MODE_MONITOR, + 'monitor_mode': px_constants.MODULE_MODE_MONITOR, } self.config = dict(self.config.items() + config.items()) @@ -92,7 +92,7 @@ def handle_verification(self, ctx, config, environ, start_response): if config.get('custom_block_handler', False): px_activities_client.send_block_activity(ctx, config) return config['custom_block_handler'](ctx, start_response) - elif config.get('module_mode', px_constants.MONITOR_MODE_MONITOR) == px_constants.MONITOR_MODE_BLOCKING: + elif config.get('module_mode', px_constants.MODULE_MODE_MONITOR) == px_constants.MODULE_MODE_BLOCKING: return self.PXBlocker.handle_blocking(ctx=ctx, config=config, start_response=start_response) else: return self.pass_traffic(environ, start_response, ctx) From 094bfc20a2e744ce32807bdfe3147edfd5009eb3 Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Sat, 24 Nov 2018 12:32:24 +0200 Subject: [PATCH 20/35] Changed the name --- perimeterx/px_constants.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/perimeterx/px_constants.py b/perimeterx/px_constants.py index 98cdfa4..09a4d20 100644 --- a/perimeterx/px_constants.py +++ b/perimeterx/px_constants.py @@ -16,8 +16,7 @@ CLIENT_FP_PATH = 'init.js' CAPTCHA_FP_PATH = 'captcha' XHR_FP_PATH = 'xhr' -MONITOR_MODE_BLOCKING = 'active_blocking' -MONITOR_MODE_MONITOR = 'monitor' - +MODULE_MODE_BLOCKING = 'active_blocking' +MODULE_MODE_MONITOR = 'monitor' MODULE_VERSION = 'Python WSGI Module' From 83d8425a533cf64e31bffebdb8eb0ae0c79ab1d4 Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Sat, 24 Nov 2018 13:14:31 +0200 Subject: [PATCH 21/35] Changed momotproimg mode to module mode --- perimeterx/middleware.py | 4 ++-- perimeterx/px_activities_client.py | 2 +- perimeterx/px_constants.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index 9d76903..1bbd902 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -28,7 +28,7 @@ def __init__(self, app, config=None): 'css_ref': None, 'js_ref': None, 'is_mobile': False, - 'monitor_mode': px_constants.MODULE_MODE_MONITOR, + 'monitor_mode': px_constants.MODULE_MODE_MONITORING, } self.config = dict(self.config.items() + config.items()) @@ -92,7 +92,7 @@ def handle_verification(self, ctx, config, environ, start_response): if config.get('custom_block_handler', False): px_activities_client.send_block_activity(ctx, config) return config['custom_block_handler'](ctx, start_response) - elif config.get('module_mode', px_constants.MODULE_MODE_MONITOR) == px_constants.MODULE_MODE_BLOCKING: + elif config.get('module_mode') == px_constants.MODULE_MODE_BLOCKING: return self.PXBlocker.handle_blocking(ctx=ctx, config=config, start_response=start_response) else: return self.pass_traffic(environ, start_response, ctx) diff --git a/perimeterx/px_activities_client.py b/perimeterx/px_activities_client.py index 119e737..3e06d64 100644 --- a/perimeterx/px_activities_client.py +++ b/perimeterx/px_activities_client.py @@ -74,6 +74,6 @@ def send_block_activity(ctx, config): 'risk_rtt': ctx.get('risk_rtt'), #'cookie_origin':, 'module_version': px_constants.MODULE_VERSION, - 'simulated_block': config.get('monitor_mode') + 'simulated_block': config.get('monitor_mode') is px_constants.MODULE_MODE_MONITORING }) diff --git a/perimeterx/px_constants.py b/perimeterx/px_constants.py index 09a4d20..926843a 100644 --- a/perimeterx/px_constants.py +++ b/perimeterx/px_constants.py @@ -17,6 +17,6 @@ CAPTCHA_FP_PATH = 'captcha' XHR_FP_PATH = 'xhr' MODULE_MODE_BLOCKING = 'active_blocking' -MODULE_MODE_MONITOR = 'monitor' +MODULE_MODE_MONITORING = 'monitor' MODULE_VERSION = 'Python WSGI Module' From af36ee4bfbdbce5588c4d3d31089bf12e630d562 Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Sat, 24 Nov 2018 13:24:45 +0200 Subject: [PATCH 22/35] Fixed according to PR comments --- perimeterx/middleware.py | 1 - perimeterx/px_constants.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index 1bbd902..b67fd7f 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -15,7 +15,6 @@ def __init__(self, app, config=None): self.config = { 'blocking_score': 60, 'debug_mode': False, - 'module_version': 'Python SDK v1.0.3', 'module_mode': 'monitor', 'perimeterx_server_host': 'sapi.perimeterx.net', 'captcha_enabled': True, diff --git a/perimeterx/px_constants.py b/perimeterx/px_constants.py index 926843a..948d8d8 100644 --- a/perimeterx/px_constants.py +++ b/perimeterx/px_constants.py @@ -19,4 +19,4 @@ MODULE_MODE_BLOCKING = 'active_blocking' MODULE_MODE_MONITORING = 'monitor' -MODULE_VERSION = 'Python WSGI Module' +MODULE_VERSION = 'Python WSGI Module v2.0.0' From 79fc4bc0605bac9c5c5040563e7300d331696111 Mon Sep 17 00:00:00 2001 From: alexbpx <41577203+alexbpx@users.noreply.github.com> Date: Tue, 27 Nov 2018 09:10:29 +0200 Subject: [PATCH 23/35] Dev refactor config (#17) * Fxied errors * Config refactoring * Config * Deleted Notes * Feactored static files check * Refactored literals * Block activity * Block action added to block acitivity * Removed socket_ip from context * property * Whitelist and sensitive route capabilities * Update px_constants.py * Update px_activities_client.py * |Removed monitor_mode --- perimeterx/middleware.py | 103 +++++++++---------- perimeterx/px_activities_client.py | 23 ++--- perimeterx/px_api.py | 15 +-- perimeterx/px_blocker.py | 12 +-- perimeterx/px_config.py | 154 +++++++++++++++++++++++++++++ perimeterx/px_constants.py | 7 +- perimeterx/px_context.py | 38 +++---- perimeterx/px_cookie.py | 40 ++++---- perimeterx/px_cookie_v1.py | 8 +- perimeterx/px_cookie_v3.py | 9 +- perimeterx/px_cookie_validator.py | 12 ++- perimeterx/px_httpc.py | 12 +-- perimeterx/px_proxy.py | 30 +++--- perimeterx/px_utils.py | 25 ++--- 14 files changed, 320 insertions(+), 168 deletions(-) create mode 100644 perimeterx/px_config.py diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index b97d8b8..389aaa0 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -1,83 +1,70 @@ -from px_logger import Logger import px_context import px_activities_client import px_cookie_validator import px_httpc import px_blocker import px_api -import px_template -from px_proxy import PXProxy -import Cookie import px_constants +import px_utils +from px_proxy import PXProxy +from px_config import PXConfig + class PerimeterX(object): def __init__(self, app, config=None): self.app = app # merging user's defined configurations with the default one - self.config = { - 'blocking_score': 60, - 'debug_mode': False, - 'perimeterx_server_host': 'sapi.perimeterx.net', - 'captcha_enabled': True, - 'server_calls_enabled': True, - 'encryption_enabled': True, - 'sensitive_headers': ['cookie', 'cookies'], - 'send_page_activities': True, - 'api_timeout': 1, - 'custom_logo': None, - 'css_ref': None, - 'js_ref': None, - 'is_mobile': False, - 'first_party': True, - 'first_party_xhr_enabled': True, - 'monitor_mode': px_constants.MODULE_MODE_MONITORING, - } - - self.config = dict(self.config.items() + config.items()) - self.config['logger'] = logger = Logger(self.config['debug_mode']) - if not config['app_id']: + px_config = PXConfig(config) + logger = px_config.logger + if not px_config.app_id: logger.error('PX App ID is missing') raise ValueError('PX App ID is missing') - url = px_constants.COLLECTOR_URL - self.config['collector_url'] = url.format(config.get('app_id').lower()) # if APP_ID is not set, use the deafult perimeterx server - else, use the appid specific sapi. - self.config['perimeterx_server_host'] = 'sapi.perimeterx.net' if self.config['app_id'] == 'PX_APP_ID' else 'sapi-' + self.config['app_id'].lower() + '.perimeterx.net' - if not config['auth_token']: + if not px_config.auth_token: logger.error('PX Auth Token is missing') raise ValueError('PX Auth Token is missing') - if not config['cookie_key']: + if not px_config.cookie_key: logger.error('PX Cookie Key is missing') raise ValueError('PX Cookie Key is missing') - self.reverse_proxy_prefix = self.config['app_id'][2:].lower() - - self.PXBlocker = px_blocker.PXBlocker() - px_httpc.init(self.config) + self.reverse_proxy_prefix = px_config.app_id[2:].lower() + if px_config.custom_request_handler: + self.handle_verification = px_config.custom_request_handler.__get__(self, PerimeterX) + self._PXBlocker = px_blocker.PXBlocker() + self._config = px_config + px_httpc.init(px_config) def __call__(self, environ, start_response): return self._verify(environ, start_response) def _verify(self, environ, start_response): - logger = self.config['logger'] + config = self.config + logger = config.logger try: - ctx = px_context.build_context(environ, self.config) + ctx = px_context.build_context(environ, config) uri = ctx.get('uri') - px_proxy = PXProxy(self.config, ctx) - if px_proxy.should_reverse_request(uri): + px_proxy = PXProxy(config) + if px_proxy.should_reverse_request(uri): return px_proxy.handle_reverse_request(self.config, ctx, start_response) - if ctx.get('module_mode') == 'inactive' or is_static_file(ctx): + if px_utils.is_static_file(ctx): logger.debug('Filter static file request. uri: ' + uri) return self.app(environ, start_response) + if not self._config._module_enabled: + logger.debug('Module is disabled, request will not be verified') + return self.app(environ, start_response) - cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE')) + if ctx.get('whitelist'): + logger.debug('The requested uri is whitelisted, passing request') + return self.app(environ, start_response) # PX Cookie verification - if not px_cookie_validator.verify(ctx, self.config) and self.config.get('server_calls_enabled', True): + if not px_cookie_validator.verify(ctx, config): # Server-to-Server verification fallback if not px_api.verify(ctx, self.config): return self.app(environ, start_response) - + if config.custom_request_handler: + return config.custom_request_handler(ctx, self.config, environ, start_response) return self.handle_verification(ctx, self.config, environ, start_response) except: logger.error("Cought exception, passing request") @@ -86,14 +73,14 @@ def _verify(self, environ, start_response): def handle_verification(self, ctx, config, environ, start_response): score = ctx.get('risk_score', -1) - if score < config['blocking_score']: + if score < config.blocking_score: return self.pass_traffic(environ, start_response, ctx) - if config.get('custom_block_handler', False): + if config.custom_block_handler: px_activities_client.send_block_activity(ctx, config) - return config['custom_block_handler'](ctx, start_response) - elif config.get('module_mode') == px_constants.MODULE_MODE_BLOCKING: - return self.PXBlocker.handle_blocking(ctx=ctx, config=config, start_response=start_response) + return config.custom_block_handler(ctx, start_response) + elif config.module_mode == px_constants.MODULE_MODE_BLOCKING: + return self.px_blocker.handle_blocking(ctx=ctx, config=config, start_response=start_response) else: return self.pass_traffic(environ, start_response, ctx) @@ -101,18 +88,16 @@ def pass_traffic(self, environ, start_response, ctx): details = {} if ctx.get('decoded_cookie', ''): details = {"px_cookie": ctx['decoded_cookie']} - px_activities_client.send_to_perimeterx('page_requested', ctx, self.config, details) + px_activities_client.send_to_perimeterx(px_constants.PAGE_REQUESTED_ACTIVITY, ctx, self.config, details) return self.app(environ, start_response) + @property + def config(self): + return self._config + + @property + def px_blocker(self): + return self._PXBlocker + -def is_static_file(ctx): - uri = ctx.get('uri', '') - static_extensions = ['.css', '.bmp', '.tif', '.ttf', '.docx', '.woff2', '.js', '.pict', '.tiff', '.eot', - '.xlsx', '.jpg', '.csv', '.eps', '.woff', '.xls', '.jpeg', '.doc', '.ejs', '.otf', '.pptx', - '.gif', '.pdf', '.swf', '.svg', '.ps', '.ico', '.pls', '.midi', '.svgz', '.class', '.png', - '.ppt', '.mid', 'webp', '.jar'] - for ext in static_extensions: - if uri.endswith(ext): - return True - return False diff --git a/perimeterx/px_activities_client.py b/perimeterx/px_activities_client.py index 3e06d64..9ac6b60 100644 --- a/perimeterx/px_activities_client.py +++ b/perimeterx/px_activities_client.py @@ -26,21 +26,18 @@ def send_activities(): def send_to_perimeterx(activity_type, ctx, config, detail): global CONFIG try: - if not config.get('server_calls_enabled', True): - return - - if activity_type == 'page_requested' and not config.get('send_page_activities', False): + if activity_type == 'page_requested' and not config.send_page_activities: print 'Page activities disabled in config - skipping.' return - if len(CONFIG.keys()) == 0: + if not CONFIG: CONFIG = config _details = { 'http_method': ctx.get('http_method', ''), 'http_version': ctx.get('http_version', ''), - 'module_version': config.get('module_version', ''), - 'risk_mode': config.get('module_mode', '') + 'module_version': config.module_version, + 'risk_mode': config.module_mode, } if len(detail.keys()) > 0: @@ -50,8 +47,8 @@ def send_to_perimeterx(activity_type, ctx, config, detail): 'type': activity_type, 'headers': ctx.get('headers'), 'timestamp': int(round(time.time() * 1000)), - 'socket_ip': ctx.get('socket_ip'), - 'px_app_id': config.get('app_id'), + 'socket_ip': ctx.get('ip'), + 'px_app_id': config.app_id, 'url': ctx.get('full_url'), 'details': _details, 'vid': ctx.get('vid', ''), @@ -64,16 +61,16 @@ def send_to_perimeterx(activity_type, ctx, config, detail): def send_block_activity(ctx, config): - send_to_perimeterx('block', ctx, config, { + send_to_perimeterx(px_constants.BLOCK_ACTIVITY, ctx, config, { 'block_score': ctx.get('risk_score'), 'client_uuid': ctx.get('uuid'), 'block_reason': ctx.get('block_reason'), - 'http_method' : ctx.get('http_method'), + 'http_method': ctx.get('http_method'), 'http_version': ctx.get('http_version'), 'px_cookie': ctx.get('decoded_cookie'), 'risk_rtt': ctx.get('risk_rtt'), #'cookie_origin':, + 'block_action': ctx.get('block_action', ''), 'module_version': px_constants.MODULE_VERSION, - 'simulated_block': config.get('monitor_mode') is px_constants.MODULE_MODE_MONITORING - + 'simulated_block': config.monitor_mode is 0, }) diff --git a/perimeterx/px_api.py b/perimeterx/px_api.py index f19b8ef..9ed3b47 100644 --- a/perimeterx/px_api.py +++ b/perimeterx/px_api.py @@ -1,16 +1,17 @@ import sys import px_httpc import time +import px_constants def send_risk_request(ctx, config): body = prepare_risk_body(ctx, config) - return px_httpc.send('/api/v2/risk', body, config) + return px_httpc.send(px_constants.API_RISK, body, config) def verify(ctx, config): - logger = config['logger'] + logger = config.logger logger.debug("PXVerify") try: start = time.time() @@ -24,7 +25,7 @@ def verify(ctx, config): ctx['uuid'] = response['uuid'] ctx['block_action'] = response['action'] ctx['risk_rtt'] = risk_rtt - if score >= config['blocking_score']: + if score >= config.blocking_score: logger.debug("PXVerify block score threshold reached, will initiate blocking") ctx['block_reason'] = 's2s_high_score' elif response['action'] is 'j' and response.get('action_data') is not None and response.get('action_data').get('body') is not None: @@ -47,11 +48,11 @@ def verify(ctx, config): def prepare_risk_body(ctx, config): - logger = config['logger'] + logger = config.logger logger.debug("PxAPI[send_risk_request]") body = { 'request': { - 'ip': ctx.get('socket_ip'), + 'ip': ctx.get('ip'), 'headers': format_headers(ctx.get('headers')), 'uri': ctx.get('uri'), 'url': ctx.get('full_url', '') @@ -62,8 +63,8 @@ def prepare_risk_body(ctx, config): 's2s_call_reason': ctx.get('s2s_call_reason', ''), 'http_method': ctx.get('http_method', ''), 'http_version': ctx.get('http_version', ''), - 'module_version': config.get('module_version', ''), - 'risk_mode': config.get('module_mode', ''), + 'module_version': config.module_version, + 'risk_mode': config.module_mode, 'px_cookie_hmac': ctx.get('cookie_hmac', ''), 'request_cookie_names': ctx.get('cookie_names', '') } diff --git a/perimeterx/px_blocker.py b/perimeterx/px_blocker.py index 5bc574e..add48f9 100644 --- a/perimeterx/px_blocker.py +++ b/perimeterx/px_blocker.py @@ -37,14 +37,14 @@ def handle_blocking(self, ctx, config, start_response): return str(blocking_response) def prepare_properties(self, ctx, config): - app_id = config.get('app_id').lower() + app_id = config.app_id.lower() vid = ctx.get('vid') if ctx.get('vid') is not None else '' uuid = ctx.get('uuid') - custom_logo = config.get('CUSTOM_LOGO') if config.get('CUSTOM_LOGO') is not None else '' + custom_logo = config.custom_logo is_mobile_num = 1 if ctx.get('is_mobile') else 0 captcha_uri = 'captcha.js?a={}&u={}&v={}&m={}'.format(ctx.get('block_action'), uuid, vid, is_mobile_num) - if config.get('first_party') and not ctx.get('is_mobile'): + if config.first_party and not ctx.get('is_mobile'): prefix = app_id[2:] js_client_src = '/{}/{}'.format(prefix, px_constants.CLIENT_FP_PATH) captcha_src = '/{}/{}/{}'.format(prefix, px_constants.CAPTCHA_FP_PATH, captcha_uri) @@ -60,12 +60,12 @@ def prepare_properties(self, ctx, config): 'vid': vid, 'uuid': uuid, 'customLogo': custom_logo, - 'cssRef': config.get('css_ref'), - 'jsRef': config.get('js_ref'), + 'cssRef': config.css_ref, + 'jsRef': config.js_ref, 'logoVisibility': 'visible' if custom_logo is not None else 'hidden', 'hostUrl': host_url, 'jsClientSrc': js_client_src, - 'firstPartyEnabled': config.get('first_party'), + 'firstPartyEnabled': config.first_party, 'blockScript': captcha_src } diff --git a/perimeterx/px_config.py b/perimeterx/px_config.py new file mode 100644 index 0000000..fb6b98c --- /dev/null +++ b/perimeterx/px_config.py @@ -0,0 +1,154 @@ +import px_constants +from px_logger import Logger + + +class PXConfig(object): + def __init__(self, config_dict): + app_id = config_dict.get('app_id') + debug_mode = config_dict.get('debug_mode', False) + module_mode = config_dict.get('module_mode', px_constants.MODULE_MODE_MONITORING) + self._app_id = app_id + self._blocking_score = config_dict.get('blocking_score', 100) + self._debug_mode = debug_mode + self._module_version = config_dict.get('module_version', px_constants.MODULE_VERSION) + self._module_mode = module_mode + self._server_host = 'sapi.perimeterx.net' if app_id is None else px_constants.SERVER_URL.format(app_id.lower()) + self._collector_host = 'collector.perimeterx.net' if app_id is None else px_constants.COLLECTOR_URL.format( + app_id.lower()) + self._encryption_enabled = config_dict.get('encryption_enabled', True) + self._sensitive_headers = config_dict.get('sensitive_headers', ['cookie', 'cookies']) + self._send_page_activities = config_dict.get('send_page_activities', True) + self._api_timeout = config_dict.get('api_timeout', 500) + self._custom_logo = config_dict.get('custom_logo', '') + self._css_ref = config_dict.get('_custom_logo', '') + self._js_ref = config_dict.get('js_ref', '') + self._is_mobile = config_dict.get('is_mobile', False) + self._module_enabled = config_dict.get('module_enabled', True) + self._cookie_key = config_dict.get('cookie_key', None) + self._auth_token = config_dict.get('auth_token', None) + self._is_mobile = config_dict.get('is_mobile', False) + self._first_party = config_dict.get('first_party', True) + self._first_party_xhr_enabled = config_dict.get('first_party_xhr_enabled', True) + self._logger = Logger(debug_mode) + self._ip_headers = config_dict.get('ip_headers', []) + self._proxy_url = config_dict.get('proxy_url', None) + self._max_buffer_len = config_dict.get('max_buffer_len', 30) + self._sensitive_routes = config_dict.get('sensitive_routes', []) + self._whitelist_routes = config_dict.get('whitelist_routes', []) + self.instantiate_user_defined_handlers(config_dict) + + @property + def module_mode(self): + return self._module_mode + + @property + def app_id(self): + return self._app_id + + @property + def logger(self): + return self._logger + + @property + def auth_token(self): + return self._auth_token + + @property + def cookie_key(self): + return self._cookie_key + + @property + def server_host(self): + return self._server_host + + @property + def api_timeout(self): + return self._api_timeout + + @property + def module_enabled(self): + return self._module_enabled + + @property + def ip_headers(self): + return self._ip_headers + + @property + def sensitive_headers(self): + return self._sensitive_headers + + @property + def proxy_url(self): + return self._proxy_url + + @property + def custom_request_handler(self): + return self._custom_request_handler + + @property + def custom_block_handler(self): + return self._custom_block_handler + + @property + def blocking_score(self): + return self._blocking_score + + @property + def encryption_enabled(self): + return self._encryption_enabled + + @property + def module_version(self): + return self._module_version + + @property + def send_page_activities(self): + return self._send_page_activities + + @property + def custom_logo(self): + return self._custom_logo + + @property + def css_ref(self): + return self._css_ref + + @property + def js_ref(self): + return self._js_ref + + @property + def first_party(self): + return self._first_party + + @property + def first_party_xhr_enabled(self): + return self._first_party_xhr_enabled + + @property + def collector_host(self): + return self._collector_host + + @property + def get_user_ip(self): + return self._get_user_ip + + @property + def sensitive_routes(self): + return self._sensitive_routes + + @property + def whitelist_routes(self): + return self._whitelist_routes + + + def instantiate_user_defined_handlers(self, config_dict): + self._custom_request_handler = self.set_handler('custom_request_handler', config_dict) + self._custom_block_handler = self.set_handler('custom_block_handler', config_dict) + self._get_user_ip = self.set_handler('get_user_ip', config_dict) + self._additional_activity_handler = self.set_handler('additional_activity_handler', config_dict) + + + def set_handler(self, function_name, config_dict): + return config_dict.get(function_name) if config_dict.get(function_name) and callable( + config_dict.get(function_name)) else None diff --git a/perimeterx/px_constants.py b/perimeterx/px_constants.py index c68728b..648a3b5 100644 --- a/perimeterx/px_constants.py +++ b/perimeterx/px_constants.py @@ -12,7 +12,8 @@ BLOCK_ACTION_RATE = 'r' CLIENT_HOST = 'client.perimeterx.net' CAPTCHA_HOST = 'captcha.px-cdn.net' -COLLECTOR_URL = 'https://collector-{}.perimeterx.net' +COLLECTOR_URL = 'collector-{}.perimeterx.net' +SERVER_URL = 'sapi-{}.perimeterx.net' CLIENT_FP_PATH = 'init.js' CAPTCHA_FP_PATH = 'captcha' XHR_FP_PATH = 'xhr' @@ -24,5 +25,7 @@ EMPTY_GIF_B64 = 'R0lGODlhAQABAPAAAAAAAAAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==' COLLECTOR_HOST = 'collector.perimeterx.net' FIRST_PARTY_FORWARDED_FOR = 'X-FORWARDED-FOR' - MODULE_VERSION = 'Python WSGI Module v2.0.0' +API_RISK = '/api/v3/risk' +PAGE_REQUESTED_ACTIVITY = 'page_requested' +BLOCK_ACTIVITY = 'block' diff --git a/perimeterx/px_context.py b/perimeterx/px_context.py index dc49c49..ed0f16e 100644 --- a/perimeterx/px_context.py +++ b/perimeterx/px_context.py @@ -3,7 +3,7 @@ def build_context(environ, config): - logger = config['logger'] + logger = config.logger headers = {} # Default values @@ -13,17 +13,11 @@ def build_context(environ, config): px_cookies = {} request_cookie_names = list() - # IP Extraction - if config.get('ip_handler'): - socket_ip = config.get('ip_handler')(environ) - else: - socket_ip = environ.get('REMOTE_ADDR') - # Extracting: Headers, user agent, http method, http version for key in environ.keys(): if key.startswith('HTTP_') and environ.get(key): header_name = key.split('HTTP_')[1].replace('_', '-').lower() - if header_name not in config.get('sensitive_headers'): + if header_name not in config.sensitive_headers: headers[header_name] = environ.get(key) if key == 'REQUEST_METHOD': http_method = environ.get(key) @@ -47,18 +41,17 @@ def build_context(environ, config): vid = cookies.get('_pxvid').value else: vid = '' - user_agent = headers.get('user-agent') uri = environ.get('PATH_INFO') or '' full_url = http_protocol + headers.get('host') or environ.get('SERVER_NAME') or '' + uri hostname = headers.get('host') - + sensitive_route = uri in config.sensitive_routes + whitelist_route = uri in config.whitelist_routes ctx = { 'headers': headers, 'http_method': http_method, 'http_version': http_version, 'user_agent': user_agent, - 'socket_ip': socket_ip, 'full_url': full_url, 'uri': uri, 'hostname': hostname, @@ -67,14 +60,25 @@ def build_context(environ, config): 'risk_rtt': 0, 'ip': extract_ip(config, environ), 'vid': vid, - 'query_params': environ['QUERY_STRING'] + 'query_params': environ['QUERY_STRING'], + 'sensitive_route': sensitive_route, + 'whitelist_route': whitelist_route, } return ctx + def extract_ip(config, environ): ip = environ.get('HTTP_X_FORWARDED_FOR') - if not ip == None: - return ip.split(',')[-1].strip() - else: - return '' - + ip_headers = config.ip_headers + logger = config.logger + if not ip_headers: + try: + for ip_header in ip_headers: + ip_header_name = 'HTTP_' + ip_header.upper() + if environ.get(ip_header_name): + return environ.get(ip_header_name) + except: + logger.debug('Failed to use IP_HEADERS from config') + if config.get_user_ip: + ip = config.get_user_ip(environ) + return ip diff --git a/perimeterx/px_cookie.py b/perimeterx/px_cookie.py index 005e967..8f14128 100644 --- a/perimeterx/px_cookie.py +++ b/perimeterx/px_cookie.py @@ -13,14 +13,14 @@ class PxCookie: - def __init__(self): - pass + def __init__(self, config): + self._config = config + self._logger = config.logger - @staticmethod - def build_px_cookie(ctx, config): - config["logger"].debug("PxCookie[build_px_cookie]") - px_cookies = ctx['px_cookies'].keys() + def build_px_cookie(self, ctx): + self._logger.debug("PxCookie[build_px_cookie]") + px_cookies = ctx['px_cookies'].keys() # Check that its not empty if not px_cookies: return None @@ -28,17 +28,17 @@ def build_px_cookie(ctx, config): px_cookies.sort(reverse=True) prefix = px_cookies[0] if prefix == PREFIX_PX_COOKIE_V1: - config["logger"].debug("PxCookie[build_px_cookie] using cookie v1") + self._logger.debug("PxCookie[build_px_cookie] using cookie v1") from px_cookie_v1 import PxCookieV1 - return PxCookieV1(ctx, config) + return PxCookieV1(ctx, self._config) if prefix == PREFIX_PX_COOKIE_V3: - config["logger"].debug("PxCookie[build_px_cookie] using cookie v3") + self._logger.debug("PxCookie[build_px_cookie] using cookie v3") from px_cookie_v3 import PxCookieV3 - return PxCookieV3(ctx, config) + return PxCookieV3(ctx, self._config) def decode_cookie(self): - self.config['logger'].debug("PxCookie[decode_cookie]") + self._logger.debug("PxCookie[decode_cookie]") return base64.b64decode(self.raw_cookie) def pbkdf2_hmac(self, hash_name, password, salt, iterations, dklen=None): @@ -103,7 +103,7 @@ def decrypt_cookie(self): :return: Returns decrypted value if valid and False if not :rtype: Bool|String """ - self.config['logger'].debug("PxCookie[decrypt_cookie]") + self._logger.debug("PxCookie[decrypt_cookie]") try: parts = self.raw_cookie.split(':', 3) if len(parts) != 3: @@ -113,13 +113,13 @@ def decrypt_cookie(self): if iterations < 1 or iterations > 10000: return False data = base64.b64decode(parts[2]) - dk = self.pbkdf2_hmac('sha256', self.config['cookie_key'], salt, iterations, dklen=48) + dk = self.pbkdf2_hmac('sha256', self._config.cookie_key, salt, iterations, dklen=48) key = dk[:32] iv = dk[32:] cipher = AES.new(key, AES.MODE_CBC, iv) unpad = lambda s: s[0:-ord(s[-1])] plaintext = unpad(cipher.decrypt(data)) - self.config['logger'].debug("PxCookie[decrypt_cookie] cookie decrypted") + self._logger.debug("PxCookie[decrypt_cookie] cookie decrypted") return plaintext except: print traceback.format_exception(*sys.exc_info()) @@ -142,15 +142,15 @@ def is_cookie_valid(self, str_to_hmac): :rtype: Bool """ try: - calculated_digest = hmac.new(self.config['cookie_key'], str_to_hmac, hashlib.sha256).hexdigest() + calculated_digest = hmac.new(self._config.cookie_key, str_to_hmac, hashlib.sha256).hexdigest() return self.get_hmac() == calculated_digest except: - self.config["logger"].debug("failed to calculate hmac") + self._logger.debug("failed to calculate hmac") return False def deserialize(self): - self.config['logger'].debug("PxCookie[deserialize]") - if self.config.get("encryption_enabled", False): + self._logger.debug("PxCookie[deserialize]") + if self._config.encryption_enabled: cookie = self.decrypt_cookie() else: cookie = self.decode_cookie() @@ -158,12 +158,12 @@ def deserialize(self): if not cookie: return False - self.config['logger'].debug("PxCookie[deserialize] decoded cookie: " + cookie) + self._logger.debug("PxCookie[deserialize] decoded cookie: " + cookie) self.decoded_cookie = json.loads(cookie) return self.is_cookie_format_valid() def is_high_score(self): - return self.get_score() >= self.config['blocking_score'] + return self.get_score() >= self._config.blocking_score def get_timestamp(self): return self.decoded_cookie['t'] diff --git a/perimeterx/px_cookie_v1.py b/perimeterx/px_cookie_v1.py index 959602f..fa5e4de 100644 --- a/perimeterx/px_cookie_v1.py +++ b/perimeterx/px_cookie_v1.py @@ -5,8 +5,8 @@ class PxCookieV1(PxCookie): def __init__(self, ctx, config): - self.ctx = ctx - self.config = config + self._ctx = ctx + self._config = config self.raw_cookie = ctx['px_cookies'].get(PREFIX_PX_COOKIE_V1, '') def get_score(self): @@ -24,8 +24,8 @@ def is_cookie_format_valid(self): def is_secured(self): c = self.decoded_cookie - user_agent = self.ctx.get('user_agent', '') - ip = self.ctx.get('ip', '') + user_agent = self._ctx.get('user_agent', '') + ip = self._ctx.get('ip', '') base_hmac = str(self.get_timestamp()) + str(c['s']['a']) + str(self.get_score()) + self.get_uuid() + self.get_vid() hmac_with_ip = base_hmac + ip + user_agent hmac_without_ip = base_hmac + user_agent diff --git a/perimeterx/px_cookie_v3.py b/perimeterx/px_cookie_v3.py index 00226a8..ab70e3a 100644 --- a/perimeterx/px_cookie_v3.py +++ b/perimeterx/px_cookie_v3.py @@ -5,9 +5,10 @@ class PxCookieV3(PxCookie): def __init__(self, ctx, config): - self.ctx = ctx - self.config = config - spliced_cookie = ctx['px_cookies'].get(PREFIX_PX_COOKIE_V3, '').split(":", 1) + self._config = config + self._logger = config.logger + self._ctx = ctx + spliced_cookie = self._ctx['px_cookies'].get(PREFIX_PX_COOKIE_V3, '').split(":", 1) if spliced_cookie.count > 1: self.hmac = spliced_cookie[0] self.raw_cookie = spliced_cookie[1] @@ -26,7 +27,7 @@ def is_cookie_format_valid(self): return 't' in c and 'v' in c and 'u' in c and 's' in c and 'a' in c def is_secured(self): - user_agent = self.ctx.get('user_agent', '') + user_agent = self._ctx.get('user_agent', '') str_hmac = self.raw_cookie + user_agent return self.is_cookie_valid(str_hmac) diff --git a/perimeterx/px_cookie_validator.py b/perimeterx/px_cookie_validator.py index 2aedda6..4d30893 100644 --- a/perimeterx/px_cookie_validator.py +++ b/perimeterx/px_cookie_validator.py @@ -1,4 +1,5 @@ import traceback +from px_cookie import PxCookie def verify(ctx, config): @@ -11,15 +12,15 @@ def verify(ctx, config): :return: Returns True if verification succeeded and False if not :rtype: Bool """ - logger = config['logger'] + logger = config.logger try: if not ctx["px_cookies"].keys(): logger.debug('No risk cookie on the request') ctx['s2s_call_reason'] = 'no_cookie' return False - from px_cookie import PxCookie - px_cookie = PxCookie.build_px_cookie(ctx, config) + px_cookie_builder = PxCookie(config) + px_cookie = px_cookie_builder.build_px_cookie(ctx) if not px_cookie.deserialize(): logger.error('Cookie decryption failed') @@ -49,6 +50,11 @@ def verify(ctx, config): ctx['s2s_call_reason'] = 'cookie_validation_failed' return False + if ctx.get('sensitive_route'): + logger.debug('Sensitive route match, sending Risk API. path: {}'.format(ctx.get('uri'))) + ctx['s2s_call_reason'] = 'sensitive_route' + return False + logger.debug('Cookie validation passed with good score: ' + str(ctx['risk_score'])) return True except Exception, e: diff --git a/perimeterx/px_httpc.py b/perimeterx/px_httpc.py index c44397c..b34f56f 100644 --- a/perimeterx/px_httpc.py +++ b/perimeterx/px_httpc.py @@ -7,37 +7,35 @@ def init(config): global http_client - http_client = httplib.HTTPConnection(config.get('perimeterx_server_host'), timeout=config.get('api_timeout', 1)) + http_client = httplib.HTTPConnection(host=config.server_host, timeout=config.api_timeout) def send(uri, body, config): - logger = config['logger'] + logger = config.logger headers = { - 'Authorization': 'Bearer ' + config.get('auth_token', ''), + 'Authorization': 'Bearer ' + config.auth_token, 'Content-Type': 'application/json' } try: start = time.time() http_client.request('POST', uri, body=json.dumps(body), headers=headers) r = http_client.getresponse() - if r.status != 200: logger.error('error posting server to server call ' + r.reason) return False logger.debug('Server call took ' + str(time.time() - start) + 'ms') response_body = r.read() - return json.loads(response_body) except httplib.HTTPException: init(config) return False def send_reverse(url, path, body, headers, config, method): - logger = config['logger'] + logger = config.logger try: start = time.time() - http_client = httplib.HTTPSConnection(url, timeout=config.get('api_timeout', 1)) + http_client = httplib.HTTPSConnection(url, timeout=config.api_timeout) http_client.request(method, path, body, headers=headers) response = http_client.getresponse() diff --git a/perimeterx/px_proxy.py b/perimeterx/px_proxy.py index b3bb3c7..c9dac3f 100644 --- a/perimeterx/px_proxy.py +++ b/perimeterx/px_proxy.py @@ -11,10 +11,10 @@ class PXProxy(object): - def __init__(self, px_config): - self.logger = px_config['logger'] + def __init__(self, config): + self._logger = config.logger - reverse_app_id = px_config['app_id'][2:] + reverse_app_id = config.app_id[2:] self.client_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.CLIENT_FP_PATH).lower() self.xhr_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.XHR_FP_PATH).lower() self.captcha_reverse_prefix = '/{}/{}'.format(reverse_app_id, px_constants.CAPTCHA_FP_PATH).lower() @@ -37,18 +37,18 @@ def handle_reverse_request(self, config, ctx, start_response): return self.send_reverse_captcha_request(config=config, context=ctx, start_response=start_response) def send_reverse_client_request(self, config, context, start_response): - if not config['first_party']: + if not config.first_party: headers = [('Content-Type', 'application/javascript')] start_response("200 OK", headers) return "" - client_request_uri = '/{}/main.min.js'.format(config['app_id']) - self.logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(),px_constants.CLIENT_HOST, client_request_uri)) + client_request_uri = '/{}/main.min.js'.format(config.app_id) + self._logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(),px_constants.CLIENT_HOST, client_request_uri)) headers = {'host': px_constants.CLIENT_HOST, px_constants.FIRST_PARTY_HEADER: 1, px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} - filtered_headers = px_utils.filter_sensitive_headers(context['headers'], config) + filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) response = px_httpc.send_reverse(url=px_constants.CLIENT_HOST, path=client_request_uri, body='', headers=filtered_headers, config=config, method='GET') @@ -59,7 +59,7 @@ def send_reverse_client_request(self, config, context, start_response): def send_reverse_xhr_request(self, config, context, start_response): uri = context.get('uri') - if not config.get('first_party') or not config.get('first_party_xhr_enabled'): + if not config.first_party or not config.first_party_xhr_enabled: body, content_type = self.return_default_response(uri) start_response('200 OK', [content_type]) @@ -68,7 +68,7 @@ def send_reverse_xhr_request(self, config, context, start_response): xhr_path_index = uri.find('/' + px_constants.XHR_FP_PATH) suffix_uri = uri[xhr_path_index + 4:] - host = config.get('collector_url').replace('https://', '') + host = config.collector_host headers = {'host': host, px_constants.FIRST_PARTY_HEADER: 1, px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} @@ -76,9 +76,9 @@ def send_reverse_xhr_request(self, config, context, start_response): if context.get('vid') is not None: headers['Cookies'] = '_pxvid=' + context.get('vid') - filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), config, context.get('ip')) + filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) - self.logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, suffix_uri)) + self._logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, suffix_uri)) response = px_httpc.send_reverse(url=host, path=suffix_uri, body='', headers=filtered_headers, config=config, method=context.get('http_method')) @@ -102,20 +102,20 @@ def return_default_response(self, uri): return body, content_type def send_reverse_captcha_request(self, config, context, start_response): - if not config['first_party']: + if not config.first_party: status = '200 OK' headers = [('Content-Type', 'application/javascript')] start_response(status, headers) return '' - uri = '/{}{}?{}'.format(config['app_id'], context.get('uri').lower().replace(self.captcha_reverse_prefix, ''), context['query_params']) + uri = '/{}{}?{}'.format(config.app_id, context.get('uri').lower().replace(self.captcha_reverse_prefix, ''), context['query_params']) host = px_constants.CAPTCHA_HOST headers = {'host': px_constants.CAPTCHA_HOST, px_constants.FIRST_PARTY_HEADER: 1, px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} - filtered_headers = px_utils.filter_sensitive_headers(context['headers'], config) + filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) - self.logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, uri)) + self._logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, uri)) response = px_httpc.send_reverse(url=host, path=uri, body='', headers=filtered_headers, config=config, method='GET') headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) diff --git a/perimeterx/px_utils.py b/perimeterx/px_utils.py index 6d1d572..3fc9b73 100644 --- a/perimeterx/px_utils.py +++ b/perimeterx/px_utils.py @@ -1,26 +1,29 @@ import px_constants -def filter_sensitive_headers(headers, config): - sensitive_keys = config.get('sensitive_headers') - if not sensitive_keys is not None: - return {header_name: sensitive_keys[header_name] for header_name in headers if - header_name not in sensitive_keys} - else: - return headers - - def merge_two_dicts(x, y): z = x.copy() # start with x's keys and values z.update(y) # modifies z with y's keys and values & returns None return z -def handle_proxy_headers(headers, config, ip): - filtered_headers = filter_sensitive_headers(headers, config) +def handle_proxy_headers(filtered_headers, ip): for item in filtered_headers.keys(): if item.upper() == px_constants.FIRST_PARTY_FORWARDED_FOR: filtered_headers[item] = ip else: filtered_headers[px_constants.FIRST_PARTY_FORWARDED_FOR] = ip return filtered_headers + + +def is_static_file(ctx): + uri = ctx.get('uri', '') + static_extensions = ['.css', '.bmp', '.tif', '.ttf', '.docx', '.woff2', '.js', '.pict', '.tiff', '.eot', + '.xlsx', '.jpg', '.csv', '.eps', '.woff', '.xls', '.jpeg', '.doc', '.ejs', '.otf', '.pptx', + '.gif', '.pdf', '.swf', '.svg', '.ps', '.ico', '.pls', '.midi', '.svgz', '.class', '.png', + '.ppt', '.mid', 'webp', '.jar'] + + for ext in static_extensions: + if uri.endswith(ext): + return True + return False \ No newline at end of file From 870fb38af2ba65d03dcc44002153032ff4ca323c Mon Sep 17 00:00:00 2001 From: alexbpx <41577203+alexbpx@users.noreply.github.com> Date: Thu, 29 Nov 2018 14:47:53 +0200 Subject: [PATCH 24/35] Dev custom params enforcer tel (#18) * Fxied errors * Config refactoring * Config * Deleted Notes * Feactored static files check * Refactored literals * Block activity * Block action added to block acitivity * Removed socket_ip from context * property * Telemetry * Whitelist and sensitive route capabilities * Changes * px_blocker tests * px_utils tests * px_validator tests * px_validator tests * Fixed a few files after running automation * Fixed unittests * Fxied j challenge response * Added px_proxy tests * Fxied building process of telemetry config * Changed telemtry uri * Empty tests * Empty tests * Fxied captcha loop * Fixed captcha loops * fixed unittests * Fixed spacing * Added custom params - I have no idea where it went * Removed printing of exception in httpc exception handling * fixed api_timeout to work with seconds * Moved the transformation of timeoit to the getter * Moved the transformation of timeoit to the getter --- perimeterx/middleware.py | 63 ++++--- perimeterx/px_activities_client.py | 56 ++++-- perimeterx/px_api.py | 50 ++++-- perimeterx/px_blocker.py | 9 +- perimeterx/px_config.py | 73 ++++++-- perimeterx/px_constants.py | 11 +- perimeterx/px_context.py | 19 +- perimeterx/px_cookie_v1.py | 1 + perimeterx/px_cookie_v3.py | 3 +- perimeterx/px_cookie_validator.py | 2 +- perimeterx/px_httpc.py | 51 ++---- .../{px_proxy.py => px_proxy_handler.py} | 47 ++--- tests/px_activities_client.py | 0 tests/px_api.py | 20 +++ tests/px_blocker.py | 97 ++++++++++ tests/px_blocking_messages/blocking.txt | 169 ++++++++++++++++++ tests/px_blocking_messages/ratelimit.txt | 9 + tests/px_config.py | 0 tests/px_context.py | 0 tests/px_cookie.py | 0 tests/px_cookie_validator.py | 66 +++++++ tests/px_httpc.py | 20 +++ tests/px_proxy_handler.py | 33 ++++ tests/px_utils.py | 26 +++ 24 files changed, 677 insertions(+), 148 deletions(-) rename perimeterx/{px_proxy.py => px_proxy_handler.py} (76%) create mode 100644 tests/px_activities_client.py create mode 100644 tests/px_api.py create mode 100644 tests/px_blocker.py create mode 100644 tests/px_blocking_messages/blocking.txt create mode 100644 tests/px_blocking_messages/ratelimit.txt create mode 100644 tests/px_config.py create mode 100644 tests/px_context.py create mode 100644 tests/px_cookie.py create mode 100644 tests/px_cookie_validator.py create mode 100644 tests/px_httpc.py create mode 100644 tests/px_proxy_handler.py create mode 100644 tests/px_utils.py diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index 389aaa0..2e016ed 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -6,7 +6,7 @@ import px_api import px_constants import px_utils -from px_proxy import PXProxy +from perimeterx.px_proxy_handler import PXProxy from px_config import PXConfig @@ -29,11 +29,10 @@ def __init__(self, app, config=None): logger.error('PX Cookie Key is missing') raise ValueError('PX Cookie Key is missing') self.reverse_proxy_prefix = px_config.app_id[2:].lower() - if px_config.custom_request_handler: - self.handle_verification = px_config.custom_request_handler.__get__(self, PerimeterX) self._PXBlocker = px_blocker.PXBlocker() self._config = px_config - px_httpc.init(px_config) + px_activities_client.init_activities_configuration(px_config) + px_activities_client.send_enforcer_telemetry_activity(config=px_config, update_reason='initial_config') def __call__(self, environ, start_response): return self._verify(environ, start_response) @@ -46,7 +45,7 @@ def _verify(self, environ, start_response): uri = ctx.get('uri') px_proxy = PXProxy(config) if px_proxy.should_reverse_request(uri): - return px_proxy.handle_reverse_request(self.config, ctx, start_response) + return px_proxy.handle_reverse_request(self.config, ctx, start_response, environ) if px_utils.is_static_file(ctx): logger.debug('Filter static file request. uri: ' + uri) return self.app(environ, start_response) @@ -63,33 +62,46 @@ def _verify(self, environ, start_response): # Server-to-Server verification fallback if not px_api.verify(ctx, self.config): return self.app(environ, start_response) - if config.custom_request_handler: - return config.custom_request_handler(ctx, self.config, environ, start_response) return self.handle_verification(ctx, self.config, environ, start_response) except: logger.error("Cought exception, passing request") - self.pass_traffic(environ, start_response, ctx) + self.pass_traffic(ctx) + return self.app(environ, start_response) def handle_verification(self, ctx, config, environ, start_response): - score = ctx.get('risk_score', -1) - + score = ctx.get('score', -1) + result = None + headers = None + status = None + pass_request = True if score < config.blocking_score: - return self.pass_traffic(environ, start_response, ctx) - - if config.custom_block_handler: - px_activities_client.send_block_activity(ctx, config) - return config.custom_block_handler(ctx, start_response) - elif config.module_mode == px_constants.MODULE_MODE_BLOCKING: - return self.px_blocker.handle_blocking(ctx=ctx, config=config, start_response=start_response) + self.pass_traffic(ctx) else: - return self.pass_traffic(environ, start_response, ctx) + pass_request = False + self.block_traffic(ctx) + + if config.additional_activity_handler: + config.additional_activity_handler(ctx, config) + + if config.module_mode == px_constants.MODULE_MODE_BLOCKING and result is None and not pass_request: + result, headers, status = self.px_blocker.handle_blocking(ctx=ctx, config=config) + if config.custom_request_handler: + custom_body, custom_headers, custom_status = config.custom_request_handler(ctx, self.config, environ) + if (custom_body is not None): + start_response(custom_status, custom_headers) + return custom_body + + if headers is not None: + start_response(status, headers) + return result + else: + return self.app(environ, start_response) + + def pass_traffic(self, ctx): + px_activities_client.send_page_requested_activity( ctx, self.config) - def pass_traffic(self, environ, start_response, ctx): - details = {} - if ctx.get('decoded_cookie', ''): - details = {"px_cookie": ctx['decoded_cookie']} - px_activities_client.send_to_perimeterx(px_constants.PAGE_REQUESTED_ACTIVITY, ctx, self.config, details) - return self.app(environ, start_response) + def block_traffic(self, ctx): + px_activities_client.send_block_activity(ctx, self.config) @property def config(self): @@ -98,6 +110,3 @@ def config(self): @property def px_blocker(self): return self._PXBlocker - - - diff --git a/perimeterx/px_activities_client.py b/perimeterx/px_activities_client.py index 9ac6b60..4037464 100644 --- a/perimeterx/px_activities_client.py +++ b/perimeterx/px_activities_client.py @@ -3,36 +3,41 @@ import threading import traceback, sys import px_constants +import socket +import json ACTIVITIES_BUFFER = [] CONFIG = {} +def init_activities_configuration(config): + global CONFIG + CONFIG = config + t1 = threading.Thread(target=send_activities) + t1.daemon = True + t1.start() def send_activities(): global ACTIVITIES_BUFFER + default_headers = { + 'Authorization': 'Bearer ' + CONFIG.auth_token, + 'Content-Type': 'application/json' + } + full_url = CONFIG.server_host + px_constants.API_ACTIVITIES while True: if len(ACTIVITIES_BUFFER) > 0: chunk = ACTIVITIES_BUFFER[:10] ACTIVITIES_BUFFER = ACTIVITIES_BUFFER[10:] - px_httpc.send('/api/v1/collector/s2s', chunk, CONFIG) + px_httpc.send(full_url=full_url, body=json.dumps(chunk), headers=default_headers, config=CONFIG, method='POST') time.sleep(1) -t1 = threading.Thread(target=send_activities) -t1.daemon = True -t1.start() - def send_to_perimeterx(activity_type, ctx, config, detail): - global CONFIG try: if activity_type == 'page_requested' and not config.send_page_activities: print 'Page activities disabled in config - skipping.' return - if not CONFIG: - CONFIG = config - _details = { 'http_method': ctx.get('http_method', ''), 'http_version': ctx.get('http_version', ''), @@ -72,5 +77,36 @@ def send_block_activity(ctx, config): #'cookie_origin':, 'block_action': ctx.get('block_action', ''), 'module_version': px_constants.MODULE_VERSION, - 'simulated_block': config.monitor_mode is 0, + 'simulated_block': config.module_mode is px_constants.MODULE_MODE_MONITORING, }) + +def send_page_requested_activity(ctx, config): + details = {} + if ctx.get('decoded_cookie', ''): + details = {"px_cookie": ctx['decoded_cookie']} + send_to_perimeterx(px_constants.PAGE_REQUESTED_ACTIVITY, ctx, config, details) + +def send_enforcer_telemetry_activity(config, update_reason): + details = { + 'enforcer_configs': config.telemetry_config, + 'node_name': socket.gethostname(), + 'os_name': sys.platform, + 'update_reason': update_reason, + 'module_version': config.module_version + } + body = { + 'type': px_constants.TELEMETRY_ACTIVITY, + 'timestamp': time.time(), + 'px_app_id': config.app_id, + 'details': details + } + headers = { + 'Authorization': 'Bearer ' + config.auth_token, + 'Content-Type': 'application/json' + } + config.logger.debug('Sending telemetry activity to PerimeterX servers') + px_httpc.send(full_url=config.server_host + px_constants.API_ENFORCER_TELEMETRY, body=json.dumps(body), + headers=headers, config=config, method='POST') + + + diff --git a/perimeterx/px_api.py b/perimeterx/px_api.py index 9ed3b47..efb7f83 100644 --- a/perimeterx/px_api.py +++ b/perimeterx/px_api.py @@ -1,14 +1,30 @@ -import sys import px_httpc import time import px_constants +import json +import re - +custom_params = { + 'custom_param1': '', + 'custom_param2': '', + 'custom_param3': '', + 'custom_param4': '', + 'custom_param5': '', + 'custom_param6': '', + 'custom_param7': '', + 'custom_param8': '', + 'custom_param9': '', + 'custom_param10': '' +} def send_risk_request(ctx, config): body = prepare_risk_body(ctx, config) - return px_httpc.send(px_constants.API_RISK, body, config) - + default_headers = { + 'Authorization': 'Bearer ' + config.auth_token, + 'Content-Type': 'application/json' + } + response = px_httpc.send(full_url=config.server_host + px_constants.API_RISK,body=json.dumps(body),config=config, headers=default_headers,method='POST') + return json.loads(response.content) def verify(ctx, config): logger = config.logger @@ -26,15 +42,16 @@ def verify(ctx, config): ctx['block_action'] = response['action'] ctx['risk_rtt'] = risk_rtt if score >= config.blocking_score: - logger.debug("PXVerify block score threshold reached, will initiate blocking") - ctx['block_reason'] = 's2s_high_score' - elif response['action'] is 'j' and response.get('action_data') is not None and response.get('action_data').get('body') is not None: - logger.debug("PXVerify received javascript challenge action") - ctx['block_action_data'] = response.get('action_data').get('body') - ctx['block_reason'] = 'challenge' - elif response['action'] is 'r': - logger.debug("PXVerify received javascript ratelimit action") - ctx['block_reason'] = 'exceeded_rate_limit' + if response['action'] == 'j' and response.get('action_data') is not None and response.get('action_data').get('body') is not None: + logger.debug("PXVerify received javascript challenge action") + ctx['block_action_data'] = response.get('action_data').get('body') + ctx['block_reason'] = 'challenge' + elif response['action'] is 'r': + logger.debug("PXVerify received javascript ratelimit action") + ctx['block_reason'] = 'exceeded_rate_limit' + else: + logger.debug("PXVerify block score threshold reached, will initiate blocking") + ctx['block_reason'] = 's2s_high_score' else: ctx['pass_reason'] = 's2s' @@ -70,6 +87,13 @@ def prepare_risk_body(ctx, config): } } + if config.enrich_custom_parameters: + risk_custom_params = config.enrich_custom_parameters(custom_params) + for param in risk_custom_params: + if re.match('^custom_param\d$',param) and risk_custom_params[param]: + body['additional'][param] = risk_custom_params[param] + + if ctx['s2s_call_reason'] == 'cookie_decryption_failed': logger.debug('attaching orig_cookie to request') body['additional']['px_cookie_orig'] = ctx.get('px_orig_cookie') diff --git a/perimeterx/px_blocker.py b/perimeterx/px_blocker.py index add48f9..c0759df 100644 --- a/perimeterx/px_blocker.py +++ b/perimeterx/px_blocker.py @@ -10,7 +10,7 @@ def __init__(self): self.ratelimit_rendered_page = self.mustache_renderer.render( px_template.get_template(px_constants.RATELIMIT_TEMPLATE), {}) - def handle_blocking(self, ctx, config, start_response): + def handle_blocking(self, ctx, config): action = ctx.get('block_action') status = '403 Forbidden' @@ -31,13 +31,12 @@ def handle_blocking(self, ctx, config, start_response): blocking_props = self.prepare_properties(ctx, config) blocking_response = self.mustache_renderer.render(px_template.get_template(px_constants.BLOCK_TEMPLATE), blocking_props) - start_response(status, headers) if is_json_response: blocking_response = json.dumps(blocking_props) - return str(blocking_response) + return str(blocking_response), headers, status def prepare_properties(self, ctx, config): - app_id = config.app_id.lower() + app_id = config.app_id vid = ctx.get('vid') if ctx.get('vid') is not None else '' uuid = ctx.get('uuid') custom_logo = config.custom_logo @@ -65,7 +64,7 @@ def prepare_properties(self, ctx, config): 'logoVisibility': 'visible' if custom_logo is not None else 'hidden', 'hostUrl': host_url, 'jsClientSrc': js_client_src, - 'firstPartyEnabled': config.first_party, + 'firstPartyEnabled': 'true' if config.first_party else 'false', 'blockScript': captcha_src } diff --git a/perimeterx/px_config.py b/perimeterx/px_config.py index fb6b98c..247859e 100644 --- a/perimeterx/px_config.py +++ b/perimeterx/px_config.py @@ -1,4 +1,5 @@ import px_constants +import json from px_logger import Logger @@ -7,7 +8,8 @@ def __init__(self, config_dict): app_id = config_dict.get('app_id') debug_mode = config_dict.get('debug_mode', False) module_mode = config_dict.get('module_mode', px_constants.MODULE_MODE_MONITORING) - self._app_id = app_id + custom_logo = config_dict.get('custom_logo', None) + self._px_app_id = app_id self._blocking_score = config_dict.get('blocking_score', 100) self._debug_mode = debug_mode self._module_version = config_dict.get('module_version', px_constants.MODULE_VERSION) @@ -18,24 +20,30 @@ def __init__(self, config_dict): self._encryption_enabled = config_dict.get('encryption_enabled', True) self._sensitive_headers = config_dict.get('sensitive_headers', ['cookie', 'cookies']) self._send_page_activities = config_dict.get('send_page_activities', True) - self._api_timeout = config_dict.get('api_timeout', 500) - self._custom_logo = config_dict.get('custom_logo', '') + self._api_timeout_ms = config_dict.get('api_timeout', 500) + self._custom_logo = custom_logo self._css_ref = config_dict.get('_custom_logo', '') self._js_ref = config_dict.get('js_ref', '') self._is_mobile = config_dict.get('is_mobile', False) + self._monitor_mode = 0 if module_mode is px_constants.MODULE_MODE_MONITORING else 1 self._module_enabled = config_dict.get('module_enabled', True) - self._cookie_key = config_dict.get('cookie_key', None) self._auth_token = config_dict.get('auth_token', None) self._is_mobile = config_dict.get('is_mobile', False) self._first_party = config_dict.get('first_party', True) self._first_party_xhr_enabled = config_dict.get('first_party_xhr_enabled', True) - self._logger = Logger(debug_mode) self._ip_headers = config_dict.get('ip_headers', []) self._proxy_url = config_dict.get('proxy_url', None) self._max_buffer_len = config_dict.get('max_buffer_len', 30) self._sensitive_routes = config_dict.get('sensitive_routes', []) self._whitelist_routes = config_dict.get('whitelist_routes', []) - self.instantiate_user_defined_handlers(config_dict) + self._block_html = 'BLOCK' + self._logo_visibility = 'visible' if custom_logo is not None else 'hidden' + self._telemetry_config = self.__create_telemetry_config() + + self._auth_token = config_dict.get('auth_token', None) + self._cookie_key = config_dict.get('cookie_key', None) + self.__instantiate_user_defined_handlers(config_dict) + self._logger = Logger(debug_mode) @property def module_mode(self): @@ -43,7 +51,7 @@ def module_mode(self): @property def app_id(self): - return self._app_id + return self._px_app_id @property def logger(self): @@ -63,7 +71,7 @@ def server_host(self): @property def api_timeout(self): - return self._api_timeout + return self._api_timeout_ms / 1000.000 @property def module_enabled(self): @@ -85,10 +93,6 @@ def proxy_url(self): def custom_request_handler(self): return self._custom_request_handler - @property - def custom_block_handler(self): - return self._custom_block_handler - @property def blocking_score(self): return self._blocking_score @@ -141,14 +145,47 @@ def sensitive_routes(self): def whitelist_routes(self): return self._whitelist_routes + @property + def block_html(self): + return self._block_html - def instantiate_user_defined_handlers(self, config_dict): - self._custom_request_handler = self.set_handler('custom_request_handler', config_dict) - self._custom_block_handler = self.set_handler('custom_block_handler', config_dict) - self._get_user_ip = self.set_handler('get_user_ip', config_dict) - self._additional_activity_handler = self.set_handler('additional_activity_handler', config_dict) + @property + def logo_visibility(self): + return self._logo_visibility + @property + def additional_activity_handler(self): + return self._additional_activity_handler - def set_handler(self, function_name, config_dict): + @property + def debug_mode(self): + return self._debug_mode + + @property + def max_buffer_len(self): + return self._max_buffer_len + + @property + def telemetry_config(self): + return self._telemetry_config + + @property + def enrich_custom_parameters(self): + return self._enrich_custom_parameters + + def __instantiate_user_defined_handlers(self, config_dict): + self._custom_request_handler = self.__set_handler('custom_request_handler', config_dict) + self._get_user_ip = self.__set_handler('get_user_ip', config_dict) + self._additional_activity_handler = self.__set_handler('additional_activity_handler', config_dict) + self._enrich_custom_parameters = self.__set_handler('enrich_custom_parameters', config_dict) + + def __set_handler(self, function_name, config_dict): return config_dict.get(function_name) if config_dict.get(function_name) and callable( config_dict.get(function_name)) else None + + def __create_telemetry_config(self): + config = self.__dict__ + mutated_config = {} + for key, value in config.iteritems(): + mutated_config[key[1:].upper()] = value + return mutated_config diff --git a/perimeterx/px_constants.py b/perimeterx/px_constants.py index 648a3b5..95722ea 100644 --- a/perimeterx/px_constants.py +++ b/perimeterx/px_constants.py @@ -10,10 +10,10 @@ BLOCK_ACTION_CAPTCHA = 'b' BLOCK_ACTION_CHALLENGE = 'j' BLOCK_ACTION_RATE = 'r' -CLIENT_HOST = 'client.perimeterx.net' -CAPTCHA_HOST = 'captcha.px-cdn.net' -COLLECTOR_URL = 'collector-{}.perimeterx.net' -SERVER_URL = 'sapi-{}.perimeterx.net' +CLIENT_HOST = 'https://client.perimeterx.net' +CAPTCHA_HOST = 'https://captcha.px-cdn.net' +COLLECTOR_URL = 'https://collector-{}.perimeterx.net' +SERVER_URL = 'https://sapi-{}.perimeterx.net' CLIENT_FP_PATH = 'init.js' CAPTCHA_FP_PATH = 'captcha' XHR_FP_PATH = 'xhr' @@ -29,3 +29,6 @@ API_RISK = '/api/v3/risk' PAGE_REQUESTED_ACTIVITY = 'page_requested' BLOCK_ACTIVITY = 'block' +API_ENFORCER_TELEMETRY = '/api/v2/risk/telemetry' +API_ACTIVITIES = '/api/v1/collector/s2s' +TELEMETRY_ACTIVITY = 'enforcer_telemetry' diff --git a/perimeterx/px_context.py b/perimeterx/px_context.py index ed0f16e..e21da96 100644 --- a/perimeterx/px_context.py +++ b/perimeterx/px_context.py @@ -27,6 +27,9 @@ def build_context(environ, config): http_protocol = protocol_split[0].lower() + '://' if len(protocol_split) > 1: http_version = protocol_split[1] + if key == 'CONTENT_TYPE' or key == 'CONTENT_LENGTH': + headers['Content-type'.replace('_', '-')] = environ.get(key) + cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE', '')) cookie_keys = cookies.keys() @@ -41,12 +44,12 @@ def build_context(environ, config): vid = cookies.get('_pxvid').value else: vid = '' - user_agent = headers.get('user-agent') + user_agent = environ.get('HTTP_USER_AGENT', '') uri = environ.get('PATH_INFO') or '' - full_url = http_protocol + headers.get('host') or environ.get('SERVER_NAME') or '' + uri + full_url = http_protocol + (headers.get('host') or environ.get('SERVER_NAME') or '') + uri hostname = headers.get('host') - sensitive_route = uri in config.sensitive_routes - whitelist_route = uri in config.whitelist_routes + sensitive_route = len(filter(lambda sensitive_route_item : uri.startswith(sensitive_route_item), config.sensitive_routes)) > 0 + whitelist_route = len(filter(lambda whitelist_route_item : uri.startswith(whitelist_route_item), config.whitelist_routes)) > 0 ctx = { 'headers': headers, 'http_method': http_method, @@ -63,18 +66,20 @@ def build_context(environ, config): 'query_params': environ['QUERY_STRING'], 'sensitive_route': sensitive_route, 'whitelist_route': whitelist_route, + 's2s_call_reason': 'none', + 'cookie_origin': 'cookie' } return ctx def extract_ip(config, environ): - ip = environ.get('HTTP_X_FORWARDED_FOR') + ip = environ.get('HTTP_X_FORWARDED_FOR') if environ.get('HTTP_X_FORWARDED_FOR') else environ.get('REMOTE_ADDR') ip_headers = config.ip_headers logger = config.logger - if not ip_headers: + if ip_headers: try: for ip_header in ip_headers: - ip_header_name = 'HTTP_' + ip_header.upper() + ip_header_name = 'HTTP_' + ip_header.replace('-', '_').upper() if environ.get(ip_header_name): return environ.get(ip_header_name) except: diff --git a/perimeterx/px_cookie_v1.py b/perimeterx/px_cookie_v1.py index fa5e4de..3c3cd4d 100644 --- a/perimeterx/px_cookie_v1.py +++ b/perimeterx/px_cookie_v1.py @@ -7,6 +7,7 @@ class PxCookieV1(PxCookie): def __init__(self, ctx, config): self._ctx = ctx self._config = config + self._logger = config.logger self.raw_cookie = ctx['px_cookies'].get(PREFIX_PX_COOKIE_V1, '') def get_score(self): diff --git a/perimeterx/px_cookie_v3.py b/perimeterx/px_cookie_v3.py index ab70e3a..9b4f8b0 100644 --- a/perimeterx/px_cookie_v3.py +++ b/perimeterx/px_cookie_v3.py @@ -8,8 +8,9 @@ def __init__(self, ctx, config): self._config = config self._logger = config.logger self._ctx = ctx + self.raw_cookie = '' spliced_cookie = self._ctx['px_cookies'].get(PREFIX_PX_COOKIE_V3, '').split(":", 1) - if spliced_cookie.count > 1: + if len(spliced_cookie) > 1: self.hmac = spliced_cookie[0] self.raw_cookie = spliced_cookie[1] diff --git a/perimeterx/px_cookie_validator.py b/perimeterx/px_cookie_validator.py index 4d30893..3a6a00a 100644 --- a/perimeterx/px_cookie_validator.py +++ b/perimeterx/px_cookie_validator.py @@ -45,7 +45,7 @@ def verify(ctx, config): logger.debug('Cookie expired') return False - if px_cookie.is_secured(): + if not px_cookie.is_secured(): logger.debug('Cookie validation failed') ctx['s2s_call_reason'] = 'cookie_validation_failed' return False diff --git a/perimeterx/px_httpc.py b/perimeterx/px_httpc.py index b34f56f..c31fbb6 100644 --- a/perimeterx/px_httpc.py +++ b/perimeterx/px_httpc.py @@ -1,51 +1,22 @@ -import httplib -import json import time +import requests -http_client = None - -def init(config): - global http_client - http_client = httplib.HTTPConnection(host=config.server_host, timeout=config.api_timeout) - - -def send(uri, body, config): - logger = config.logger - headers = { - 'Authorization': 'Bearer ' + config.auth_token, - 'Content-Type': 'application/json' - } - try: - start = time.time() - http_client.request('POST', uri, body=json.dumps(body), headers=headers) - r = http_client.getresponse() - if r.status != 200: - logger.error('error posting server to server call ' + r.reason) - return False - - logger.debug('Server call took ' + str(time.time() - start) + 'ms') - response_body = r.read() - return json.loads(response_body) - except httplib.HTTPException: - init(config) - return False - -def send_reverse(url, path, body, headers, config, method): +def send(full_url, body, headers, config, method): logger = config.logger try: start = time.time() - http_client = httplib.HTTPSConnection(url, timeout=config.api_timeout) - http_client.request(method, path, body, headers=headers) - response = http_client.getresponse() + if method == 'GET': + response = requests.get(url=full_url, headers=headers, timeout=config.api_timeout) + else: + response = requests.post(url=full_url, headers=headers, data=body, timeout=config.api_timeout) - if response.status >= 400: + if response.status_code >= 400: + logger.debug('PerimeterX server call failed') return False - logger.debug('Server call took ' + str(time.time() - start) + 'ms') + logger.debug('PerimeterX server call took ' + str(time.time() - start) + 'ms') return response - - except httplib.HTTPException: - init(config) + except requests.exceptions.RequestException as e: + logger.debug('Received RequestException, message: ' + e.message) return False - diff --git a/perimeterx/px_proxy.py b/perimeterx/px_proxy_handler.py similarity index 76% rename from perimeterx/px_proxy.py rename to perimeterx/px_proxy_handler.py index c9dac3f..f0169fb 100644 --- a/perimeterx/px_proxy.py +++ b/perimeterx/px_proxy_handler.py @@ -26,13 +26,13 @@ def should_reverse_request(self, uri): return True return False - def handle_reverse_request(self, config, ctx, start_response): + def handle_reverse_request(self, config, ctx, start_response, environ): uri = ctx.get('uri').lower() if uri.startswith(self.client_reverse_prefix): return self.send_reverse_client_request(config=config, context=ctx, start_response=start_response) if uri.startswith(self.xhr_reverse_prefix): - return self.send_reverse_xhr_request(config=config, context=ctx, start_response=start_response) + return self.send_reverse_xhr_request(config=config, context=ctx, start_response=start_response, body = environ['wsgi.input'].read(int(environ.get('CONTENT_LENGTH', '0')))) if uri.startswith(self.captcha_reverse_prefix): return self.send_reverse_captcha_request(config=config, context=ctx, start_response=start_response) @@ -46,18 +46,17 @@ def send_reverse_client_request(self, config, context, start_response): self._logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(),px_constants.CLIENT_HOST, client_request_uri)) headers = {'host': px_constants.CLIENT_HOST, - px_constants.FIRST_PARTY_HEADER: 1, + px_constants.FIRST_PARTY_HEADER: '1', px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) - response = px_httpc.send_reverse(url=px_constants.CLIENT_HOST, path=client_request_uri, body='', - headers=filtered_headers, config=config, method='GET') + response = px_httpc.send(full_url=px_constants.CLIENT_HOST + client_request_uri, body='', + headers=filtered_headers, config=config, method='GET') - headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) - start_response(str(response.status) + ' ' + response.reason, headers) - return response.read() + self.handle_proxy_response(response, start_response) + return response.content - def send_reverse_xhr_request(self, config, context, start_response): + def send_reverse_xhr_request(self, config, context, start_response, body): uri = context.get('uri') if not config.first_party or not config.first_party_xhr_enabled: body, content_type = self.return_default_response(uri) @@ -70,7 +69,7 @@ def send_reverse_xhr_request(self, config, context, start_response): host = config.collector_host headers = {'host': host, - px_constants.FIRST_PARTY_HEADER: 1, + px_constants.FIRST_PARTY_HEADER: '1', px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} if context.get('vid') is not None: @@ -79,18 +78,23 @@ def send_reverse_xhr_request(self, config, context, start_response): filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) self._logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, suffix_uri)) - response = px_httpc.send_reverse(url=host, path=suffix_uri, body='', - headers=filtered_headers, config=config, method=context.get('http_method')) + response = px_httpc.send(full_url=host + suffix_uri, body=body, + headers=filtered_headers, config=config, method=context.get('http_method')) - if response.status >= 400: + if response.status_code >= 400: body, content_type = self.return_default_response(uri) px_logger.Logger.debug('error reversing the http call ' + response.reason) start_response('200 OK', [content_type]) return body - response_headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) - start_response(str(response.status) + ' ' + response.reason, response_headers) - return response.read() + self.handle_proxy_response(response, start_response) + return response.content + def handle_proxy_response(self, response, start_response): + headers = [] + for header in response.headers: + if header.lower() not in hoppish: + headers.append((header, response.headers[header])) + start_response(str(response.status_code) + ' ' + response.reason, headers) def return_default_response(self, uri): if 'gif' in uri.lower(): @@ -111,16 +115,15 @@ def send_reverse_captcha_request(self, config, context, start_response): host = px_constants.CAPTCHA_HOST headers = {'host': px_constants.CAPTCHA_HOST, - px_constants.FIRST_PARTY_HEADER: 1, + px_constants.FIRST_PARTY_HEADER: '1', px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) self._logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, uri)) - response = px_httpc.send_reverse(url=host, path=uri, body='', - headers=filtered_headers, config=config, method='GET') - headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) - start_response(str(response.status) + ' ' + response.reason, headers) - return response.read() + response = px_httpc.send(full_url=host + uri, body='', + headers=filtered_headers, config=config, method='GET') + self.handle_proxy_response(response, start_response) + return response.content diff --git a/tests/px_activities_client.py b/tests/px_activities_client.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/px_api.py b/tests/px_api.py new file mode 100644 index 0000000..d7bc19f --- /dev/null +++ b/tests/px_api.py @@ -0,0 +1,20 @@ +import unittest +from perimeterx import px_api +from perimeterx.px_config import PXConfig + +class Test_PXApi(unittest.TestCase): + + def enrich_custom_parameters(self, params): + params['custom_param1'] = '1' + params['custom_param2'] = '5' + params['custom'] = '6' + return params + + def test_prepare_risk_body(self): + config = PXConfig({'app_id': 'app_id', 'enrich_custom_parameters': self.enrich_custom_parameters}) + ctx = {'headers': {}, 's2s_call_reason': 'no_cookie'} + body = px_api.prepare_risk_body(ctx, config) + self.assertEqual(body['additional'].get('custom_param1'), '1') + self.assertEqual(body['additional'].get('custom_param2'), '5') + self.assertFalse(body['additional'].get('custom')) + print diff --git a/tests/px_blocker.py b/tests/px_blocker.py new file mode 100644 index 0000000..9e9b9ee --- /dev/null +++ b/tests/px_blocker.py @@ -0,0 +1,97 @@ +from perimeterx.px_blocker import PXBlocker + + +import unittest +from perimeterx.px_config import PXConfig + + +class Test_PXBlocker(unittest.TestCase): + + def test_is_json_response(self): + px_blocker = PXBlocker() + ctx = { + 'headers': {'Accept': 'text/html'} + } + self.assertFalse(px_blocker.is_json_response(ctx)) + ctx['headers']['Accept'] = 'application/json' + self.assertTrue(px_blocker.is_json_response(ctx)) + + def test_handle_blocking(self): + px_blocker = PXBlocker() + vid = 'bf619be8-94be-458a-b6b1-ee81f154c282' + px_uuid = '8712cef7-bcfa-4bb6-ae99-868025e1908a' + ctx = { + 'headers': {'Accept': 'text/html'}, + 'vid': vid, + 'uuid': px_uuid + } + px_config = PXConfig({'app_id': 'PXfake_app_ip'}) + message, _, _ = px_blocker.handle_blocking(ctx, px_config) + with open('./px_blocking_messages/blocking.txt', 'r') as myfile: + blocking_message = myfile.read() + self.assertEqual(message, blocking_message) + + def test_handle_ratelimit(self): + px_blocker = PXBlocker() + vid = 'bf619be8-94be-458a-b6b1-ee81f154c282' + px_uuid = '8712cef7-bcfa-4bb6-ae99-868025e1908a' + ctx = { + 'headers': {'Accept': 'text/html'}, + 'vid': vid, + 'uuid': px_uuid, + 'block_action': 'r' + } + px_config = PXConfig({'app_id': 'PXfake_app_ip'}) + message, _, _ = px_blocker.handle_blocking(ctx, px_config) + blocking_message = None + with open('./px_blocking_messages/ratelimit.txt', 'r') as myfile: + blocking_message = myfile.read() + self.assertEqual(message, blocking_message) + + def test_handle_challenge(self): + px_blocker = PXBlocker() + vid = 'bf619be8-94be-458a-b6b1-ee81f154c282' + px_uuid = '8712cef7-bcfa-4bb6-ae99-868025e1908a' + ctx = { + 'headers': {'Accept': 'text/html'}, + 'vid': vid, + 'uuid': px_uuid, + 'block_action': 'j', + 'block_action_data': 'Bla' + } + px_config = PXConfig({'app_id': 'PXfake_app_ip'}) + message, _, _ = px_blocker.handle_blocking(ctx, px_config) + blocking_message = 'Bla' + self.assertEqual(message, blocking_message) + + def test_prepare_properties(self): + px_blocker = PXBlocker() + vid = 'bf619be8-94be-458a-b6b1-ee81f154c282' + px_uuid = '8712cef7-bcfa-4bb6-ae99-868025e1908a' + ctx = { + 'headers': {'Accept': 'text/html'}, + 'vid': vid, + 'uuid': px_uuid, + } + px_config = PXConfig({'app_id': 'PXfake_app_ip'}) + message = px_blocker.prepare_properties(ctx, px_config) + expected_message = {'blockScript': '/fake_app_ip/captcha/captcha.js?a=None&u=8712cef7-bcfa-4bb6-ae99-868025e1908a&v=bf619be8-94be-458a-b6b1-ee81f154c282&m=0', + 'vid': 'bf619be8-94be-458a-b6b1-ee81f154c282', + 'jsRef': '', + 'hostUrl': '/fake_app_ip/xhr', + 'customLogo': None, + 'appId': 'PXfake_app_ip', + 'uuid': '8712cef7-bcfa-4bb6-ae99-868025e1908a', + 'logoVisibility': 'hidden', + 'jsClientSrc': '/fake_app_ip/init.js', + 'firstPartyEnabled': 'true', + 'refId': '8712cef7-bcfa-4bb6-ae99-868025e1908a', + 'cssRef': ''} + self.assertDictEqual(message, expected_message) + expected_message['blockScript'] = '/fake_app/captcha/captcha.js?a=None&u=8712cef7-bcfa-4bb6-ae99-868025e1908a&v=bf619be8-94be-458a-b6b1-ee81f154c282&m=0' + self.assertNotEqual(message, expected_message) + + + + + diff --git a/tests/px_blocking_messages/blocking.txt b/tests/px_blocking_messages/blocking.txt new file mode 100644 index 0000000..b42ba1b --- /dev/null +++ b/tests/px_blocking_messages/blocking.txt @@ -0,0 +1,169 @@ + + + + + + Access to this page has been denied. + + + + + + +
+
+ +
+
+
+

Please verify you are a human

+
+
+
+
+ +
+
+

+ Access to this page has been denied because we believe you are using automation tools to browse the + website. +

+

+ This may happen as a result of the following: +

+
    +
  • + Javascript is disabled or blocked by an extension (ad blockers for example) +
  • +
  • + Your browser does not support cookies +
  • +
+

+ Please make sure that Javascript and cookies are enabled on your browser and that you are not blocking + them from loading. +

+

+ Reference ID: #8712cef7-bcfa-4bb6-ae99-868025e1908a +

+
+
+ +
+ + + + + + + diff --git a/tests/px_blocking_messages/ratelimit.txt b/tests/px_blocking_messages/ratelimit.txt new file mode 100644 index 0000000..36fd393 --- /dev/null +++ b/tests/px_blocking_messages/ratelimit.txt @@ -0,0 +1,9 @@ + + + Too Many Requests + + +

Too Many Requests

+

Reached maximum requests limitation, try again soon.

+ + \ No newline at end of file diff --git a/tests/px_config.py b/tests/px_config.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/px_context.py b/tests/px_context.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/px_cookie.py b/tests/px_cookie.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/px_cookie_validator.py b/tests/px_cookie_validator.py new file mode 100644 index 0000000..8f984ab --- /dev/null +++ b/tests/px_cookie_validator.py @@ -0,0 +1,66 @@ +from perimeterx import px_cookie_validator +import unittest +from perimeterx.px_config import PXConfig + + +class Test_PXCookieValidator(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.cookie_key = 'Pyth0nS3crE7K3Y' + + def test_verify_no_cookie(self): + config = PXConfig({'app_id': 'app_id'}) + ctx = {'px_cookies': {}} + verified = px_cookie_validator.verify(ctx, config) + self.assertFalse(verified) + self.assertEqual('no_cookie', ctx['s2s_call_reason']) + + def test_verify_valid_cookie(self): + config = PXConfig({'app_id': 'app_id', + 'cookie_key': self.cookie_key}) + ctx = {'px_cookies': { + '_px3': 'bd078865fa9627f626d6f7d6828ab595028d2c0974065ab6f6c5a9f80c4593cd:OCIluokZHHvqrWyu8zrWSH8Vu7AefCjrd4CMx/NXsX58LzeV40EZIlPG4gsNMoAYzH88s/GoZwv+DpQa76C21A==:1000:zwT+Rht/YGDNWKkzHtJAB7IiI00u4fOePL/3xWMs1nZ93lzW1XvAMGR2hLlHBmOv8O0CpylEQOZZTK1uQMls6O28Y8aQnTo5DETLkrbhpwCVeNjOcf8GVKTckITwuHfXbEcfHbdtb68s1+jHv1+vt/w/6HZqTzanaIsvFVp8vmA='}} + verified = px_cookie_validator.verify(ctx, config) + self.assertTrue(verified) + self.assertEqual(None, ctx.get('s2s_call_reason')) + + def test_verify_decryption_failed(self): + config = PXConfig({'app_id': 'app_id', + 'cookie_key': self.cookie_key}) + ctx = {'px_cookies': { + '_px3': '774958bcc233ea1a876b92ababf47086d8a4d95165bbd6f98b55d7e61afd2a05:ow3Er5dskpt8ZZ11CRiDMAueEi3ozJTqMBnYzsSM7/8vHTDA0so6ekhruiTrXa/taZINotR5PnTo78D5zM2pWw==:1000:uQ3Tdt7D3mSO5CuHDis3GgrnkGMC+XAghbHuNOE9x4H57RAmtxkTcNQ1DaqL8rx79bHl0iPVYlOcRmRgDiBCUoizBdUCjsSIplofPBLIl8WpfHDDtpxPKzz9I2rUEbFgfhFjiTY3rPGob2PUvTsDXTfPUeHnzKqbNTO8z7H6irFnUE='}} + verified = px_cookie_validator.verify(ctx, config) + self.assertFalse(verified) + self.assertEqual('cookie_decryption_failed', ctx.get('s2s_call_reason')) + + def test_verify_cookie_high_score(self): + config = PXConfig({'app_id': 'app_id', + 'cookie_key': self.cookie_key}) + ctx = {'px_cookies': { + '_px3': 'bf46ceff75278ae166f376cbf741a7639060581035dd4e93641892c905dd0d67:EGFGcwQ2rum7KRmQCeSXBAUt1+25mj2DFJYi7KJkEliF3cBspdXtD2X03Csv8N8B6S5Bte/4ccCcETkBNDVxTw==:1000:x9x+oI6BISFhlKEERpf8HpZD2zXBCW9lzVfuRURHaAnbaMnpii+XjPEd7a7EGGUSMch5ramy3y+KOxyuX3F+LbGYwvn3OJb+u40zU+ixT1w5N15QltX+nBMhC7izC1l8QtgMuG/f3Nts5ebnec9j2V7LS5Y1/5b73rd9s7AMnug='}} + verified = px_cookie_validator.verify(ctx, config) + self.assertTrue(verified) + self.assertEqual(None, ctx.get('s2s_call_reason')) + + def test_verify_hmac_validation(self): + config = PXConfig({'app_id': 'app_id', + 'cookie_key': self.cookie_key}) + ctx = {'px_cookies': { + '_px3': '774958bcc232343ea1a876b92ababf47086d8a4d95165bbd6f98b55d7e61afd2a05:ow3Er5dskpt8ZZ11CRiDMAueEi3ozJTqMBnYzsSM7/8vHTDA0so6ekhruiTrXa/taZINotR5PnTo78D5zM2pWw==:1000:uQ3Tdt7D3mSO5CuHDis3GgrnkGMC+XAghbHuNOE9x4H57RAmtxkTcNQ1DaqL8rx79bHl0iPVYlOcRmRgDiBCUoizBdUCjsSIplofPBLIl8WpfHDDtpxPKzz9I2rUEbFFjiTY3rPGob2PUvTsDXTfPUeHnzKqbNTO8z7H6irFnUE='}} + verified = px_cookie_validator.verify(ctx, config) + self.assertFalse(verified) + self.assertEqual(None, ctx.get('cookie_validation_failed')) + + def test_verify_expired_cookie(self): + config = PXConfig({'app_id': 'app_id', + 'cookie_key': self.cookie_key}) + ctx = {'px_cookies': { + '_px3': '0d67bdf4a58c524b55b9cf0f703e4f0f3cbe23a10bd2671530d3c7e0cfa509eb:HOiYSw11ICB2A+HYx+C+l5Naxcl7hMeEo67QNghCQByyHlhWZT571ZKfqV98JFWg7TvbV9QtlrQtXakPYeIEjQ==:1000:+kuXS/iJUoEqrm8Fo4K0cTebsc4YQZu+f5bRGX0lC1T+l0g1gzRUuKiCtWTar28Y0wjch1ZQvkNy523Pxr07agVi/RL0SUktmEl59qGor+m4FLewZBVdcgx/Ya9kU0riis98AAR0zdTpTtoN5wpNbmztIpOZ0YejeD0Esk3vagU='}} + verified = px_cookie_validator.verify(ctx, config) + self.assertFalse(verified) + self.assertEqual(None, ctx.get('cookie_expired')) + + + + diff --git a/tests/px_httpc.py b/tests/px_httpc.py new file mode 100644 index 0000000..16c0cc6 --- /dev/null +++ b/tests/px_httpc.py @@ -0,0 +1,20 @@ +from perimeterx import px_httpc +import unittest +from mock import MagicMock,patch +from perimeterx.px_config import PXConfig +import httplib + + +class Test_PXHTTPC(unittest.TestCase): + + def test_send(self): + # px_config = PXConfig({'app_id': 'fake_app_id', + # 'auth_token': 'fake_auth_token'}) + # http_client = httplib.HTTPConnection(host='host', timeout=1) + # from httplib2 import Response + # http_client.request = MagicMock(return_value= Response({'status':'200'})) + # with patch('perimeterx.px_httpc.httplib', return_value=http_client): + # message = px_httpc.send('uri', 'body', px_config) + + + print 'f' \ No newline at end of file diff --git a/tests/px_proxy_handler.py b/tests/px_proxy_handler.py new file mode 100644 index 0000000..8e14db5 --- /dev/null +++ b/tests/px_proxy_handler.py @@ -0,0 +1,33 @@ +import unittest + +from httplib import HTTPResponse + +from perimeterx.px_proxy_handler import PXProxy +from perimeterx.px_config import PXConfig +from mock import MagicMock,patch + +class Test_PXProxy(unittest.TestCase): + + def test_should_reverse_request(self): + config = PXConfig({'app_id': 'PXfake_app_id'}) + px_proxy = PXProxy(config) + should_reverse = px_proxy.should_reverse_request('/fake_app_id/init.js') + self.assertTrue(should_reverse) + should_reverse = px_proxy.should_reverse_request('/fake_app_id/xhr') + self.assertTrue(should_reverse) + should_reverse = px_proxy.should_reverse_request('/fake_app_id/captcha') + self.assertTrue(should_reverse) + + # def test_send_reverse_client_request(self): + # content = 'client js content' + # config = PXConfig({'app_id': 'PXfake_app_id'}) + # ctx = {'uri': '/fake_app_id/init.js', 'headers': {'X-FORWARDED-FOR': '127.0.0.1'}} + # fake = HttpResponse(content=content, status=200, reason='OK', content_type='text/html') + # px_proxy = PXProxy(config) + # config = PXConfig({'app_id': 'PXfake_app_id'}) + # with patch('perimeterx.px_httpc.send_https', return_value=fake): + # result = px_proxy.handle_reverse_request(config=config, ctx=ctx, start_response= lambda x: x) + # print '' + + + diff --git a/tests/px_utils.py b/tests/px_utils.py new file mode 100644 index 0000000..1d429c7 --- /dev/null +++ b/tests/px_utils.py @@ -0,0 +1,26 @@ +from perimeterx import px_utils +import unittest +from perimeterx import px_constants + + +class Test_PXUtils(unittest.TestCase): + + def test_merge_two_dicts(self): + dict1 = {'a': '1'} + dict2 = {'b': '2'} + merged_dict = px_utils.merge_two_dicts(dict1, dict2) + self.assertDictEqual(merged_dict, {'a': '1', 'b': '2'}) + + def test_handle_proxy_headers(self): + headers_sample = {'ddd': 'not_proxy_url', px_constants.FIRST_PARTY_FORWARDED_FOR: 'proxy_url'} + headers_sample = px_utils.handle_proxy_headers(headers_sample, '127.0.0.1') + self.assertEqual(headers_sample[px_constants.FIRST_PARTY_FORWARDED_FOR], '127.0.0.1') + headers_sample = {'ddd': 'not_proxy_url'} + headers_sample = px_utils.handle_proxy_headers(headers_sample, '127.0.0.1') + self.assertEqual(headers_sample[px_constants.FIRST_PARTY_FORWARDED_FOR], '127.0.0.1') + + def test_is_static_file(self): + ctx = {'uri': '/sample.css'} + self.assertTrue(px_utils.is_static_file(ctx)) + ctx = {'uri': '/sample.html'} + self.assertFalse(px_utils.is_static_file(ctx)) From 3ec6bbfa6ccca602edd09c0b056f884ea87a0002 Mon Sep 17 00:00:00 2001 From: alexbpx <41577203+alexbpx@users.noreply.github.com> Date: Thu, 29 Nov 2018 17:51:35 +0200 Subject: [PATCH 25/35] Dev custom params enforcer tel (#20) * Fxied errors * Config refactoring * Config * Deleted Notes * Feactored static files check * Refactored literals * Block activity * Block action added to block acitivity * Removed socket_ip from context * property * Telemetry * Whitelist and sensitive route capabilities * Changes * px_blocker tests * px_utils tests * px_validator tests * px_validator tests * Fixed a few files after running automation * Fixed unittests * Fxied j challenge response * Added px_proxy tests * Fxied building process of telemetry config * Changed telemtry uri * Empty tests * Empty tests * Fxied captcha loop * Fixed captcha loops * fixed unittests * Fixed spacing * Added custom params - I have no idea where it went * Removed printing of exception in httpc exception handling * fixed api_timeout to work with seconds * Moved the transformation of timeoit to the getter * Moved the transformation of timeoit to the getter * Fixed first party * something * Holla --- perimeterx/middleware.py | 2 +- perimeterx/px_context.py | 2 +- perimeterx/px_cookie_validator.py | 2 +- perimeterx/px_httpc.py | 4 ++-- perimeterx/px_proxy_handler.py | 10 +++++++--- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index 2e016ed..87f0995 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -69,7 +69,7 @@ def _verify(self, environ, start_response): return self.app(environ, start_response) def handle_verification(self, ctx, config, environ, start_response): - score = ctx.get('score', -1) + score = ctx.get('risk_score', -1) result = None headers = None status = None diff --git a/perimeterx/px_context.py b/perimeterx/px_context.py index e21da96..ecd29b8 100644 --- a/perimeterx/px_context.py +++ b/perimeterx/px_context.py @@ -28,7 +28,7 @@ def build_context(environ, config): if len(protocol_split) > 1: http_version = protocol_split[1] if key == 'CONTENT_TYPE' or key == 'CONTENT_LENGTH': - headers['Content-type'.replace('_', '-')] = environ.get(key) + headers[key.replace('_', '-').lower()] = environ.get(key) cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE', '')) diff --git a/perimeterx/px_cookie_validator.py b/perimeterx/px_cookie_validator.py index 3a6a00a..9f3f68c 100644 --- a/perimeterx/px_cookie_validator.py +++ b/perimeterx/px_cookie_validator.py @@ -38,7 +38,7 @@ def verify(ctx, config): if px_cookie.is_high_score(): ctx['block_reason'] = 'cookie_high_score' logger.debug('Cookie with high score: ' + str(ctx['risk_score'])) - return True + return False if px_cookie.is_cookie_expired(): ctx['s2s_call_reason'] = 'cookie_expired' diff --git a/perimeterx/px_httpc.py b/perimeterx/px_httpc.py index c31fbb6..8e189b6 100644 --- a/perimeterx/px_httpc.py +++ b/perimeterx/px_httpc.py @@ -7,9 +7,9 @@ def send(full_url, body, headers, config, method): try: start = time.time() if method == 'GET': - response = requests.get(url=full_url, headers=headers, timeout=config.api_timeout) + response = requests.get(url='https://' + full_url, headers=headers, timeout=500, stream=True) else: - response = requests.post(url=full_url, headers=headers, data=body, timeout=config.api_timeout) + response = requests.post(url='https://' + full_url, headers=headers, data=body, timeout=config.api_timeout) if response.status_code >= 400: logger.debug('PerimeterX server call failed') diff --git a/perimeterx/px_proxy_handler.py b/perimeterx/px_proxy_handler.py index f0169fb..884e54e 100644 --- a/perimeterx/px_proxy_handler.py +++ b/perimeterx/px_proxy_handler.py @@ -50,11 +50,14 @@ def send_reverse_client_request(self, config, context, start_response): px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) + del filtered_headers['content-length'] + del filtered_headers['content-type'] response = px_httpc.send(full_url=px_constants.CLIENT_HOST + client_request_uri, body='', headers=filtered_headers, config=config, method='GET') self.handle_proxy_response(response, start_response) - return response.content + return response.raw.read() + def send_reverse_xhr_request(self, config, context, start_response, body): uri = context.get('uri') @@ -119,11 +122,12 @@ def send_reverse_captcha_request(self, config, context, start_response): px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) + del filtered_headers['content-length'] + del filtered_headers['content-type'] self._logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, uri)) response = px_httpc.send(full_url=host + uri, body='', headers=filtered_headers, config=config, method='GET') self.handle_proxy_response(response, start_response) - return response.content - + return response.raw.read() From 18d7b050c0ddcaa877ce95fd432e40e8ff529f6f Mon Sep 17 00:00:00 2001 From: Yaron Schwimmer Date: Thu, 29 Nov 2018 18:04:19 +0200 Subject: [PATCH 26/35] removed https --- perimeterx/px_constants.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/perimeterx/px_constants.py b/perimeterx/px_constants.py index 95722ea..01e634b 100644 --- a/perimeterx/px_constants.py +++ b/perimeterx/px_constants.py @@ -10,10 +10,10 @@ BLOCK_ACTION_CAPTCHA = 'b' BLOCK_ACTION_CHALLENGE = 'j' BLOCK_ACTION_RATE = 'r' -CLIENT_HOST = 'https://client.perimeterx.net' -CAPTCHA_HOST = 'https://captcha.px-cdn.net' -COLLECTOR_URL = 'https://collector-{}.perimeterx.net' -SERVER_URL = 'https://sapi-{}.perimeterx.net' +CLIENT_HOST = 'client.perimeterx.net' +CAPTCHA_HOST = 'captcha.px-cdn.net' +COLLECTOR_URL = 'collector-{}.perimeterx.net' +SERVER_URL = 'sapi-{}.perimeterx.net' CLIENT_FP_PATH = 'init.js' CAPTCHA_FP_PATH = 'captcha' XHR_FP_PATH = 'xhr' From fec4c60925709bcaaaac84f694280c14f1226ec5 Mon Sep 17 00:00:00 2001 From: alexbpx <41577203+alexbpx@users.noreply.github.com> Date: Sat, 1 Dec 2018 19:24:39 +0200 Subject: [PATCH 27/35] phase 1 (#19) * Fxied errors * Config refactoring * Config * Deleted Notes * Feactored static files check * Refactored literals * Block activity * Block action added to block acitivity * Removed socket_ip from context * property * Telemetry * Whitelist and sensitive route capabilities * Changes * px_blocker tests * px_utils tests * px_validator tests * px_validator tests * Fixed a few files after running automation * Fixed unittests * Fxied j challenge response * Added px_proxy tests * Fxied building process of telemetry config * phase 1 * Changed telemtry uri * Empty tests * Empty tests * Fxied captcha loop * Fixed captcha loops * fixed unittests * Mobile header handling * Fixed tests * fixed empty mobile header handling * asdad * Merge remote-tracking branch 'remotes/origin/dev' into dev-mobile_sdk # Conflicts: # perimeterx/middleware.py # perimeterx/px_api.py # perimeterx/px_config.py # perimeterx/px_context.py # perimeterx/px_cookie_v1.py # perimeterx/px_cookie_v3.py # perimeterx/px_httpc.py # perimeterx/px_proxy_handler.py # tests/px_cookie_validator.py # tests/px_proxy_handler.py * Added px_proxy tests * Fixed code and tests * Added unittests --- perimeterx/middleware.py | 7 ++- perimeterx/px_api.py | 24 ++++++-- perimeterx/px_constants.py | 4 ++ perimeterx/px_context.py | 58 ++++++++++++++------ perimeterx/px_cookie.py | 32 ++++++----- perimeterx/px_cookie_v1.py | 9 +-- perimeterx/px_cookie_v3.py | 18 +++--- perimeterx/px_cookie_validator.py | 26 +++++++-- perimeterx/px_httpc.py | 4 +- perimeterx/px_original_token_validator.py | 39 +++++++++++++ perimeterx/px_proxy_handler.py | 30 ++++++---- perimeterx/px_token_v1.py | 32 +++++++++++ perimeterx/px_token_v3.py | 37 +++++++++++++ tests/px_cookie.py | 38 +++++++++++++ tests/px_cookie_validator.py | 31 +++++------ tests/px_httpc.py | 30 +++++----- tests/px_proxy_handler.py | 67 ++++++++++++++++++----- 17 files changed, 368 insertions(+), 118 deletions(-) create mode 100644 perimeterx/px_original_token_validator.py create mode 100644 perimeterx/px_token_v1.py create mode 100644 perimeterx/px_token_v3.py diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index 87f0995..e98f89d 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -45,7 +45,8 @@ def _verify(self, environ, start_response): uri = ctx.get('uri') px_proxy = PXProxy(config) if px_proxy.should_reverse_request(uri): - return px_proxy.handle_reverse_request(self.config, ctx, start_response, environ) + body = environ['wsgi.input'].read(int(environ.get('CONTENT_LENGTH'))) if environ.get('CONTENT_LENGTH') else '' + return px_proxy.handle_reverse_request(self.config, ctx, start_response, body) if px_utils.is_static_file(ctx): logger.debug('Filter static file request. uri: ' + uri) return self.app(environ, start_response) @@ -65,11 +66,11 @@ def _verify(self, environ, start_response): return self.handle_verification(ctx, self.config, environ, start_response) except: logger.error("Cought exception, passing request") - self.pass_traffic(ctx) + self.pass_traffic({}) return self.app(environ, start_response) def handle_verification(self, ctx, config, environ, start_response): - score = ctx.get('risk_score', -1) + score = ctx.get('score', -1) result = None headers = None status = None diff --git a/perimeterx/px_api.py b/perimeterx/px_api.py index efb7f83..4f6d16d 100644 --- a/perimeterx/px_api.py +++ b/perimeterx/px_api.py @@ -72,7 +72,8 @@ def prepare_risk_body(ctx, config): 'ip': ctx.get('ip'), 'headers': format_headers(ctx.get('headers')), 'uri': ctx.get('uri'), - 'url': ctx.get('full_url', '') + 'url': ctx.get('full_url', ''), + 'firstParty': 'true' if config.first_party else 'false' }, 'vid': ctx.get('vid', ''), 'uuid': ctx.get('uuid', ''), @@ -82,10 +83,15 @@ def prepare_risk_body(ctx, config): 'http_version': ctx.get('http_version', ''), 'module_version': config.module_version, 'risk_mode': config.module_mode, - 'px_cookie_hmac': ctx.get('cookie_hmac', ''), - 'request_cookie_names': ctx.get('cookie_names', '') + 'request_cookie_names': ctx.get('cookie_names', ''), + 'cookie_origin': ctx.get('cookie_origin') } } + if ctx.get('cookie_hmac'): + body['additional']['px_cookie_hmac'] = ctx.get('cookie_hmac') + + + body = add_original_token_data(ctx, body) if config.enrich_custom_parameters: risk_custom_params = config.enrich_custom_parameters(custom_params) @@ -96,7 +102,7 @@ def prepare_risk_body(ctx, config): if ctx['s2s_call_reason'] == 'cookie_decryption_failed': logger.debug('attaching orig_cookie to request') - body['additional']['px_cookie_orig'] = ctx.get('px_orig_cookie') + body['additional']['px_orig_cookie'] = ctx.get('px_orig_cookie') if ctx['s2s_call_reason'] in ['cookie_expired', 'cookie_validation_failed']: logger.debug('attaching px_cookie to request') @@ -105,6 +111,16 @@ def prepare_risk_body(ctx, config): logger.debug("PxAPI[send_risk_request] request body: " + str(body)) return body +def add_original_token_data(ctx, body): + if ctx.get('original_uuid'): + body['additional']['original_uuid'] = ctx.get('original_uuid') + if ctx.get('original_token_error'): + body['additional']['original_token_error'] = ctx.get('original_token_error') + if ctx.get('original_token'): + body['additional']['original_token'] = ctx.get('original_token') + if ctx.get('decoded_original_token'): + body['additional']['decoded_original_token'] = ctx.get('decoded_original_token') + return body def format_headers(headers): ret_val = [] diff --git a/perimeterx/px_constants.py b/perimeterx/px_constants.py index 01e634b..1d99271 100644 --- a/perimeterx/px_constants.py +++ b/perimeterx/px_constants.py @@ -1,5 +1,9 @@ PREFIX_PX_COOKIE_V1 = '_px' PREFIX_PX_COOKIE_V3 = '_px3' +PREFIX_PX_TOKEN_V1 = '1' +PREFIX_PX_TOKEN_V3 = '3' +MOBILE_SDK_HEADER = "x-px-authorization" +MOBILE_SDK_ORIGINAL_HEADER= "x-px-original-token" TRANS_5C = b"".join(chr(x ^ 0x5C) for x in range(256)) TRANS_36 = b"".join(chr(x ^ 0x36) for x in range(256)) diff --git a/perimeterx/px_context.py b/perimeterx/px_context.py index ecd29b8..db9aa84 100644 --- a/perimeterx/px_context.py +++ b/perimeterx/px_context.py @@ -1,7 +1,6 @@ import Cookie from px_constants import * - def build_context(environ, config): logger = config.logger headers = {} @@ -12,6 +11,8 @@ def build_context(environ, config): http_protocol = 'http://' px_cookies = {} request_cookie_names = list() + cookie_origin = "cookie" + original_token = None # Extracting: Headers, user agent, http method, http version for key in environ.keys(): @@ -29,22 +30,34 @@ def build_context(environ, config): http_version = protocol_split[1] if key == 'CONTENT_TYPE' or key == 'CONTENT_LENGTH': headers[key.replace('_', '-').lower()] = environ.get(key) + if key == 'HTTP_' + MOBILE_SDK_HEADER.replace('-','_').upper(): + headers[MOBILE_SDK_HEADER] = environ.get(key, '') - - cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE', '')) - cookie_keys = cookies.keys() - - for key in cookie_keys: - request_cookie_names.append(key) - if key == PREFIX_PX_COOKIE_V1 or key == PREFIX_PX_COOKIE_V3: - logger.debug('Found cookie prefix:' + key) - px_cookies[key] = cookies.get(key).value + mobile_header = headers.get(MOBILE_SDK_HEADER) vid = None - if '_pxvid' in cookie_keys: - vid = cookies.get('_pxvid').value + original_token = None + if mobile_header is None: + cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE', '')) + cookie_keys = cookies.keys() + + for key in cookie_keys: + request_cookie_names.append(key) + if key == PREFIX_PX_COOKIE_V1 or key == PREFIX_PX_COOKIE_V3: + logger.debug('Found cookie prefix:' + key) + px_cookies[key] = cookies.get(key).value + vid = None + if '_pxvid' in cookie_keys: + vid = cookies.get('_pxvid').value + else: + vid = '' else: - vid = '' - user_agent = environ.get('HTTP_USER_AGENT', '') + cookie_origin = "header" + original_token = headers.get(MOBILE_SDK_ORIGINAL_HEADER) + logger.debug('Mobile SDK token detected') + cookie_name, cookie = get_token_object(config, mobile_header) + px_cookies[cookie_name] = cookie + + user_agent = headers.get('user-agent','') uri = environ.get('PATH_INFO') or '' full_url = http_protocol + (headers.get('host') or environ.get('SERVER_NAME') or '') + uri hostname = headers.get('host') @@ -62,15 +75,28 @@ def build_context(environ, config): 'cookie_names': request_cookie_names, 'risk_rtt': 0, 'ip': extract_ip(config, environ), - 'vid': vid, + 'vid': vid if vid else None, 'query_params': environ['QUERY_STRING'], 'sensitive_route': sensitive_route, 'whitelist_route': whitelist_route, 's2s_call_reason': 'none', - 'cookie_origin': 'cookie' + 'cookie_origin':cookie_origin, + 'original_token': original_token, + 'is_mobile': cookie_origin == "header" } + return ctx +def get_token_object(config, token): + result = {} + logger = config.logger + sliced_token = token.split(":", 1) + if len(sliced_token) > 1: + key = sliced_token.pop(0) + if key == PREFIX_PX_TOKEN_V1 or key == PREFIX_PX_TOKEN_V3: + logger.debug('Found token prefix:' + key) + return key, sliced_token[0] + return PREFIX_PX_TOKEN_V3, token def extract_ip(config, environ): ip = environ.get('HTTP_X_FORWARDED_FOR') if environ.get('HTTP_X_FORWARDED_FOR') else environ.get('REMOTE_ADDR') diff --git a/perimeterx/px_cookie.py b/perimeterx/px_cookie.py index 8f14128..a638548 100644 --- a/perimeterx/px_cookie.py +++ b/perimeterx/px_cookie.py @@ -11,31 +11,32 @@ import struct -class PxCookie: +class PxCookie(object): def __init__(self, config): self._config = config self._logger = config.logger - def build_px_cookie(self, ctx): + def build_px_cookie(self, px_cookies, user_agent): self._logger.debug("PxCookie[build_px_cookie]") - px_cookies = ctx['px_cookies'].keys() # Check that its not empty if not px_cookies: return None - - px_cookies.sort(reverse=True) - prefix = px_cookies[0] - if prefix == PREFIX_PX_COOKIE_V1: - self._logger.debug("PxCookie[build_px_cookie] using cookie v1") + px_cookie_keys = px_cookies.keys() + px_cookie_keys.sort(reverse=True) + prefix = px_cookie_keys[0] + if prefix == PREFIX_PX_TOKEN_V1 or prefix == PREFIX_PX_COOKIE_V1: + self._logger.debug("PxCookie[build_px_cookie] using token v1") from px_cookie_v1 import PxCookieV1 - return PxCookieV1(ctx, self._config) - - if prefix == PREFIX_PX_COOKIE_V3: - self._logger.debug("PxCookie[build_px_cookie] using cookie v3") + return PxCookieV1(self._config, px_cookies[prefix]) + if prefix == PREFIX_PX_TOKEN_V3 or prefix == PREFIX_PX_COOKIE_V3: + self._logger.debug("PxCookie[build_px_cookie] using token v3") from px_cookie_v3 import PxCookieV3 - return PxCookieV3(ctx, self._config) + ua = '' + if prefix == PREFIX_PX_COOKIE_V3: + ua = user_agent + return PxCookieV3(self._config, px_cookies[prefix], ua) def decode_cookie(self): self._logger.debug("PxCookie[decode_cookie]") @@ -149,7 +150,8 @@ def is_cookie_valid(self, str_to_hmac): return False def deserialize(self): - self._logger.debug("PxCookie[deserialize]") + logger = self._logger + logger.debug("PxCookie[deserialize]") if self._config.encryption_enabled: cookie = self.decrypt_cookie() else: @@ -158,7 +160,7 @@ def deserialize(self): if not cookie: return False - self._logger.debug("PxCookie[deserialize] decoded cookie: " + cookie) + logger.debug("Original token deserialized : " + cookie) self.decoded_cookie = json.loads(cookie) return self.is_cookie_format_valid() diff --git a/perimeterx/px_cookie_v1.py b/perimeterx/px_cookie_v1.py index 3c3cd4d..a675ea5 100644 --- a/perimeterx/px_cookie_v1.py +++ b/perimeterx/px_cookie_v1.py @@ -4,11 +4,10 @@ class PxCookieV1(PxCookie): - def __init__(self, ctx, config): - self._ctx = ctx + def __init__(self, config, raw_cookie): self._config = config self._logger = config.logger - self.raw_cookie = ctx['px_cookies'].get(PREFIX_PX_COOKIE_V1, '') + self.raw_cookie = raw_cookie def get_score(self): return self.decoded_cookie['s']['b'] @@ -23,10 +22,8 @@ def is_cookie_format_valid(self): c = self.decoded_cookie return 't' in c and 'v' in c and 'u' in c and "s" in c and 'a' in c['s'] and 'h' in c - def is_secured(self): + def is_secured(self, user_agent, ip): c = self.decoded_cookie - user_agent = self._ctx.get('user_agent', '') - ip = self._ctx.get('ip', '') base_hmac = str(self.get_timestamp()) + str(c['s']['a']) + str(self.get_score()) + self.get_uuid() + self.get_vid() hmac_with_ip = base_hmac + ip + user_agent hmac_without_ip = base_hmac + user_agent diff --git a/perimeterx/px_cookie_v3.py b/perimeterx/px_cookie_v3.py index 9b4f8b0..65ab594 100644 --- a/perimeterx/px_cookie_v3.py +++ b/perimeterx/px_cookie_v3.py @@ -1,18 +1,18 @@ from px_cookie import PxCookie -from px_constants import * class PxCookieV3(PxCookie): - def __init__(self, ctx, config): + def __init__(self, config, cookie, user_agent): self._config = config self._logger = config.logger - self._ctx = ctx - self.raw_cookie = '' - spliced_cookie = self._ctx['px_cookies'].get(PREFIX_PX_COOKIE_V3, '').split(":", 1) - if len(spliced_cookie) > 1: + self._user_agent = user_agent + spliced_cookie = cookie.split(':') + if len(spliced_cookie) is 4: self.hmac = spliced_cookie[0] - self.raw_cookie = spliced_cookie[1] + self.raw_cookie = ':'.join(spliced_cookie[1:]) + else: + self.raw_cookie = cookie def get_score(self): return self.decoded_cookie['s'] @@ -24,11 +24,11 @@ def get_action(self): return self.decoded_cookie['a'] def is_cookie_format_valid(self): - c = self.decoded_cookie; + c = self.decoded_cookie return 't' in c and 'v' in c and 'u' in c and 's' in c and 'a' in c def is_secured(self): - user_agent = self._ctx.get('user_agent', '') + user_agent = self._user_agent str_hmac = self.raw_cookie + user_agent return self.is_cookie_valid(str_hmac) diff --git a/perimeterx/px_cookie_validator.py b/perimeterx/px_cookie_validator.py index 9f3f68c..3ff96bc 100644 --- a/perimeterx/px_cookie_validator.py +++ b/perimeterx/px_cookie_validator.py @@ -1,4 +1,6 @@ import traceback +import re +import px_original_token_validator from px_cookie import PxCookie @@ -19,8 +21,22 @@ def verify(ctx, config): ctx['s2s_call_reason'] = 'no_cookie' return False + if not config.cookie_key: + logger.debug('No cookie key found, pause cookie evaluation') + ctx['s2s_call_reason'] = 'no_cookie_key' + return False + px_cookie_builder = PxCookie(config) - px_cookie = px_cookie_builder.build_px_cookie(ctx) + px_cookie = px_cookie_builder.build_px_cookie(px_cookies=ctx.get('px_cookies'), + user_agent=ctx.get('user_agent')) + #Mobile SDK traffic + if px_cookie and ctx['is_mobile']: + pattern = re.compile("^\d+$") + if re.match(pattern, px_cookie.raw_cookie): + ctx['s2s_call_reason'] = "mobile_error_" + px_cookie.raw_cookie + if ctx['original_token'] is not None: + px_original_token_validator.verify(ctx, config) + return False if not px_cookie.deserialize(): logger.error('Cookie decryption failed') @@ -28,7 +44,7 @@ def verify(ctx, config): ctx['s2s_call_reason'] = 'cookie_decryption_failed' return False - ctx['risk_score'] = px_cookie.get_score() + ctx['score'] = px_cookie.get_score() ctx['uuid'] = px_cookie.get_uuid() ctx['vid'] = px_cookie.get_vid() ctx['decoded_cookie'] = px_cookie.decoded_cookie @@ -37,8 +53,8 @@ def verify(ctx, config): if px_cookie.is_high_score(): ctx['block_reason'] = 'cookie_high_score' - logger.debug('Cookie with high score: ' + str(ctx['risk_score'])) - return False + logger.debug('Cookie with high score: ' + str(ctx['score'])) + return True if px_cookie.is_cookie_expired(): ctx['s2s_call_reason'] = 'cookie_expired' @@ -55,7 +71,7 @@ def verify(ctx, config): ctx['s2s_call_reason'] = 'sensitive_route' return False - logger.debug('Cookie validation passed with good score: ' + str(ctx['risk_score'])) + logger.debug('Cookie validation passed with good score: ' + str(ctx['score'])) return True except Exception, e: traceback.print_exc() diff --git a/perimeterx/px_httpc.py b/perimeterx/px_httpc.py index 8e189b6..81e7e68 100644 --- a/perimeterx/px_httpc.py +++ b/perimeterx/px_httpc.py @@ -7,7 +7,7 @@ def send(full_url, body, headers, config, method): try: start = time.time() if method == 'GET': - response = requests.get(url='https://' + full_url, headers=headers, timeout=500, stream=True) + response = requests.get(url='https://' + full_url, headers=headers, timeout=config.api_timeout, stream=True) else: response = requests.post(url='https://' + full_url, headers=headers, data=body, timeout=config.api_timeout) @@ -17,6 +17,6 @@ def send(full_url, body, headers, config, method): logger.debug('PerimeterX server call took ' + str(time.time() - start) + 'ms') return response - except requests.exceptions.RequestException as e: + except requests.exceptions as e: logger.debug('Received RequestException, message: ' + e.message) return False diff --git a/perimeterx/px_original_token_validator.py b/perimeterx/px_original_token_validator.py new file mode 100644 index 0000000..9762483 --- /dev/null +++ b/perimeterx/px_original_token_validator.py @@ -0,0 +1,39 @@ + +from px_cookie import PxCookie + +def verify(ctx, config): + """ + main verification function, verifying the content of the perimeterx original token risk if exists + :param ctx: perimeterx request context object + :param config: global configurations + :type ctx: dictionary + :type config: dictionary + :return: Returns True if verification succeeded and False if not + :rtype: Bool + """ + logger = config.logger + try: + logger.debug('Original token found, Evaluating') + original_token = ctx.get('original_token') + version, no_version_token = original_token.split(':', 1) + px_cookie_builder = PxCookie(config) + px_cookie = px_cookie_builder.build_px_cookie({version: no_version_token}, ctx.get('is_mobile'),'') + + if not px_cookie.deserialize(): + logger.error('Original token decryption failed, value:' + px_cookie.raw_cookie) + ctx['original_token_error'] = 'decryption_failed' + return False + + ctx['decoded_original_token'] = px_cookie.decoded_cookie + ctx['vid'] = px_cookie.get_vid() + ctx['original_uuid'] = px_cookie.get_uuid() + if not px_cookie.is_secured(): + logger.debug('Original token HMAC validation failed, value: ' + str(px_cookie.decoded_cookie)) + ctx['original_token_error'] = 'validation_failed' + return False + return True + + except Exception, e: + logger.debug('Could not decrypt original token, exception was thrown, decryption failed ' + e.message) + ctx['original_token_error'] = 'decryption_failed' + return False diff --git a/perimeterx/px_proxy_handler.py b/perimeterx/px_proxy_handler.py index 884e54e..405bc46 100644 --- a/perimeterx/px_proxy_handler.py +++ b/perimeterx/px_proxy_handler.py @@ -10,6 +10,13 @@ } +def delete_extra_headers(filtered_headers): + if 'content-length' in filtered_headers.keys(): + del filtered_headers['content-length'] + if 'content-type' in filtered_headers.keys(): + del filtered_headers['content-type'] + + class PXProxy(object): def __init__(self, config): self._logger = config.logger @@ -26,13 +33,13 @@ def should_reverse_request(self, uri): return True return False - def handle_reverse_request(self, config, ctx, start_response, environ): + def handle_reverse_request(self, config, ctx, start_response, body): uri = ctx.get('uri').lower() if uri.startswith(self.client_reverse_prefix): return self.send_reverse_client_request(config=config, context=ctx, start_response=start_response) if uri.startswith(self.xhr_reverse_prefix): - return self.send_reverse_xhr_request(config=config, context=ctx, start_response=start_response, body = environ['wsgi.input'].read(int(environ.get('CONTENT_LENGTH', '0')))) + return self.send_reverse_xhr_request(config=config, context=ctx, start_response=start_response, body=body) if uri.startswith(self.captcha_reverse_prefix): return self.send_reverse_captcha_request(config=config, context=ctx, start_response=start_response) @@ -43,21 +50,21 @@ def send_reverse_client_request(self, config, context, start_response): return "" client_request_uri = '/{}/main.min.js'.format(config.app_id) - self._logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(),px_constants.CLIENT_HOST, client_request_uri)) + self._logger.debug( + 'Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), px_constants.CLIENT_HOST, + client_request_uri)) headers = {'host': px_constants.CLIENT_HOST, px_constants.FIRST_PARTY_HEADER: '1', px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) - del filtered_headers['content-length'] - del filtered_headers['content-type'] + delete_extra_headers(filtered_headers) response = px_httpc.send(full_url=px_constants.CLIENT_HOST + client_request_uri, body='', headers=filtered_headers, config=config, method='GET') self.handle_proxy_response(response, start_response) return response.raw.read() - def send_reverse_xhr_request(self, config, context, start_response, body): uri = context.get('uri') @@ -80,7 +87,8 @@ def send_reverse_xhr_request(self, config, context, start_response, body): filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) - self._logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, suffix_uri)) + self._logger.debug( + 'Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, suffix_uri)) response = px_httpc.send(full_url=host + suffix_uri, body=body, headers=filtered_headers, config=config, method=context.get('http_method')) @@ -114,7 +122,8 @@ def send_reverse_captcha_request(self, config, context, start_response): headers = [('Content-Type', 'application/javascript')] start_response(status, headers) return '' - uri = '/{}{}?{}'.format(config.app_id, context.get('uri').lower().replace(self.captcha_reverse_prefix, ''), context['query_params']) + uri = '/{}{}?{}'.format(config.app_id, context.get('uri').lower().replace(self.captcha_reverse_prefix, ''), + context['query_params']) host = px_constants.CAPTCHA_HOST headers = {'host': px_constants.CAPTCHA_HOST, @@ -122,12 +131,9 @@ def send_reverse_captcha_request(self, config, context, start_response): px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) - del filtered_headers['content-length'] - del filtered_headers['content-type'] + delete_extra_headers(filtered_headers) self._logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, uri)) response = px_httpc.send(full_url=host + uri, body='', headers=filtered_headers, config=config, method='GET') self.handle_proxy_response(response, start_response) return response.raw.read() - - diff --git a/perimeterx/px_token_v1.py b/perimeterx/px_token_v1.py new file mode 100644 index 0000000..182f195 --- /dev/null +++ b/perimeterx/px_token_v1.py @@ -0,0 +1,32 @@ +from px_cookie import PxCookie + + +class PxTokenV1(PxCookie): + + def __init__(self, config, token): + self._config = config + self._logger = config.logger + self.raw_cookie = token + + def get_score(self): + return self.decoded_cookie['s']['b'] + + def get_hmac(self): + return self.decoded_cookie['h'] + + def get_action(self): + return 'c' + + def is_cookie_format_valid(self): + c = self.decoded_cookie + return 't' in c and 'v' in c and 'u' in c and "s" in c and 'a' in c['s'] and 'h' in c + + def is_secured(self, ip): + c = self.decoded_cookie + base_hmac = str(self.get_timestamp()) + str(c['s']['a']) + str(self.get_score()) + self.get_uuid() + self.get_vid() + hmac_with_ip = base_hmac + ip + hmac_without_ip = base_hmac + + return self.is_cookie_valid(hmac_without_ip) or self.is_cookie_valid(hmac_with_ip) + + diff --git a/perimeterx/px_token_v3.py b/perimeterx/px_token_v3.py new file mode 100644 index 0000000..057e2e3 --- /dev/null +++ b/perimeterx/px_token_v3.py @@ -0,0 +1,37 @@ +from px_cookie import PxCookie +from px_constants import * + + +class PxTokenV3(PxCookie): + + def __init__(self, config, token): + + self._config = config + self._logger = config.logger + spliced_cookie = token.split(":", 1) + + print ("Count: " + str(len(spliced_cookie))) + + if len(spliced_cookie) > 1: + self.hmac = spliced_cookie[0] + self.raw_cookie = spliced_cookie[1] + else: + self.raw_cookie = token + + + def get_score(self): + return self.decoded_cookie['s'] + + def get_hmac(self): + return self.hmac + + def get_action(self): + return self.decoded_cookie['a'] + + def is_cookie_format_valid(self): + c = self.decoded_cookie; + return 't' in c and 'v' in c and 'u' in c and 's' in c and 'a' in c + + def is_secured(self): + return self.is_cookie_valid(self.raw_cookie) + diff --git a/tests/px_cookie.py b/tests/px_cookie.py index e69de29..d70de11 100644 --- a/tests/px_cookie.py +++ b/tests/px_cookie.py @@ -0,0 +1,38 @@ + +import unittest +from perimeterx.px_cookie import PxCookie +from perimeterx.px_config import PXConfig + +class TestPXCookie(unittest.TestCase): + + @classmethod + def setUpClass(cls): + config = PXConfig({'app_id': 'fake_app_id'}) + cls.px_cookie = PxCookie(config) + cls.config = config + + def test_build_cookie(self): + px_cookies = {'_px3':'bd078865fa9627f626d6f7d6828ab595028d2c0974065ab6f6c5a9f80c4593cd:OCIluokZHHvqrWyu8zrWSH8Vu7AefCjrd4CMx/NXsX58LzeV40EZIlPG4gsNMoAYzH88s/GoZwv+DpQa76C21A==:1000:zwT+Rht/YGDNWKkzHtJAB7IiI00u4fOePL/3xWMs1nZ93lzW1XvAMGR2hLlHBmOv8O0CpylEQOZZTK1uQMls6O28Y8aQnTo5DETLkrbhpwCVeNjOcf8GVKTckITwuHfXbEcfHbdtb68s1+jHv1+vt/w/6HZqTzanaIsvFVp8vmA='} + cookie = self.px_cookie.build_px_cookie(px_cookies=px_cookies, user_agent='') + self.assertEqual('bd078865fa9627f626d6f7d6828ab595028d2c0974065ab6f6c5a9f80c4593cd', cookie.hmac) + self.assertEqual('OCIluokZHHvqrWyu8zrWSH8Vu7AefCjrd4CMx/NXsX58LzeV40EZIlPG4gsNMoAYzH88s/GoZwv+DpQa76C21A==:1000:zwT+Rht/YGDNWKkzHtJAB7IiI00u4fOePL/3xWMs1nZ93lzW1XvAMGR2hLlHBmOv8O0CpylEQOZZTK1uQMls6O28Y8aQnTo5DETLkrbhpwCVeNjOcf8GVKTckITwuHfXbEcfHbdtb68s1+jHv1+vt/w/6HZqTzanaIsvFVp8vmA=', cookie.raw_cookie) + + px_cookies = {'_px3':'OCIluokZHHvqrWyu8zrWSH8Vu7AefCjrd4CMx/NXsX58LzeV40EZIlPG4gsNMoAYzH88s/GoZwv+DpQa76C21A==:1000:zwT+Rht/YGDNWKkzHtJAB7IiI00u4fOePL/3xWMs1nZ93lzW1XvAMGR2hLlHBmOv8O0CpylEQOZZTK1uQMls6O28Y8aQnTo5DETLkrbhpwCVeNjOcf8GVKTckITwuHfXbEcfHbdtb68s1+jHv1+vt/w/6HZqTzanaIsvFVp8vmA='} + cookie = self.px_cookie.build_px_cookie(px_cookies=px_cookies, user_agent='') + if hasattr(cookie, 'hmac'): + self.assertFalse(True) + + def test_build_token(self): + px_cookies = {'3':'bd078865fa9627f626d6f7d6828ab595028d2c0974065ab6f6c5a9f80c4593cd:OCIluokZHHvqrWyu8zrWSH8Vu7AefCjrd4CMx/NXsX58LzeV40EZIlPG4gsNMoAYzH88s/GoZwv+DpQa76C21A==:1000:zwT+Rht/YGDNWKkzHtJAB7IiI00u4fOePL/3xWMs1nZ93lzW1XvAMGR2hLlHBmOv8O0CpylEQOZZTK1uQMls6O28Y8aQnTo5DETLkrbhpwCVeNjOcf8GVKTckITwuHfXbEcfHbdtb68s1+jHv1+vt/w/6HZqTzanaIsvFVp8vmA='} + cookie = self.px_cookie.build_px_cookie(px_cookies=px_cookies, user_agent='') + self.assertEqual('bd078865fa9627f626d6f7d6828ab595028d2c0974065ab6f6c5a9f80c4593cd', cookie.hmac) + self.assertEqual('OCIluokZHHvqrWyu8zrWSH8Vu7AefCjrd4CMx/NXsX58LzeV40EZIlPG4gsNMoAYzH88s/GoZwv+DpQa76C21A==:1000:zwT+Rht/YGDNWKkzHtJAB7IiI00u4fOePL/3xWMs1nZ93lzW1XvAMGR2hLlHBmOv8O0CpylEQOZZTK1uQMls6O28Y8aQnTo5DETLkrbhpwCVeNjOcf8GVKTckITwuHfXbEcfHbdtb68s1+jHv1+vt/w/6HZqTzanaIsvFVp8vmA=', cookie.raw_cookie) + + px_cookies = {'3':'OCIluokZHHvqrWyu8zrWSH8Vu7AefCjrd4CMx/NXsX58LzeV40EZIlPG4gsNMoAYzH88s/GoZwv+DpQa76C21A==:1000:zwT+Rht/YGDNWKkzHtJAB7IiI00u4fOePL/3xWMs1nZ93lzW1XvAMGR2hLlHBmOv8O0CpylEQOZZTK1uQMls6O28Y8aQnTo5DETLkrbhpwCVeNjOcf8GVKTckITwuHfXbEcfHbdtb68s1+jHv1+vt/w/6HZqTzanaIsvFVp8vmA='} + cookie = self.px_cookie.build_px_cookie(px_cookies=px_cookies, user_agent='') + if hasattr(cookie, 'hmac'): + self.assertFalse(True) + + + + diff --git a/tests/px_cookie_validator.py b/tests/px_cookie_validator.py index 8f984ab..94b0ea4 100644 --- a/tests/px_cookie_validator.py +++ b/tests/px_cookie_validator.py @@ -8,54 +8,51 @@ class Test_PXCookieValidator(unittest.TestCase): @classmethod def setUpClass(cls): cls.cookie_key = 'Pyth0nS3crE7K3Y' + cls.config = PXConfig({'app_id': 'app_id', + 'cookie_key': cls.cookie_key}) def test_verify_no_cookie(self): - config = PXConfig({'app_id': 'app_id'}) - ctx = {'px_cookies': {}} + config = self.config + ctx = {'user_agent':'', 'px_cookies': {}, 'is_mobile': False} verified = px_cookie_validator.verify(ctx, config) self.assertFalse(verified) self.assertEqual('no_cookie', ctx['s2s_call_reason']) def test_verify_valid_cookie(self): - config = PXConfig({'app_id': 'app_id', - 'cookie_key': self.cookie_key}) - ctx = {'px_cookies': { + config = self.config + ctx = {'user_agent':'', 'is_mobile': False, 'px_cookies': { '_px3': 'bd078865fa9627f626d6f7d6828ab595028d2c0974065ab6f6c5a9f80c4593cd:OCIluokZHHvqrWyu8zrWSH8Vu7AefCjrd4CMx/NXsX58LzeV40EZIlPG4gsNMoAYzH88s/GoZwv+DpQa76C21A==:1000:zwT+Rht/YGDNWKkzHtJAB7IiI00u4fOePL/3xWMs1nZ93lzW1XvAMGR2hLlHBmOv8O0CpylEQOZZTK1uQMls6O28Y8aQnTo5DETLkrbhpwCVeNjOcf8GVKTckITwuHfXbEcfHbdtb68s1+jHv1+vt/w/6HZqTzanaIsvFVp8vmA='}} verified = px_cookie_validator.verify(ctx, config) self.assertTrue(verified) self.assertEqual(None, ctx.get('s2s_call_reason')) def test_verify_decryption_failed(self): - config = PXConfig({'app_id': 'app_id', - 'cookie_key': self.cookie_key}) - ctx = {'px_cookies': { + config = self.config + ctx = {'user_agent':'', 'is_mobile': False, 'px_cookies': { '_px3': '774958bcc233ea1a876b92ababf47086d8a4d95165bbd6f98b55d7e61afd2a05:ow3Er5dskpt8ZZ11CRiDMAueEi3ozJTqMBnYzsSM7/8vHTDA0so6ekhruiTrXa/taZINotR5PnTo78D5zM2pWw==:1000:uQ3Tdt7D3mSO5CuHDis3GgrnkGMC+XAghbHuNOE9x4H57RAmtxkTcNQ1DaqL8rx79bHl0iPVYlOcRmRgDiBCUoizBdUCjsSIplofPBLIl8WpfHDDtpxPKzz9I2rUEbFgfhFjiTY3rPGob2PUvTsDXTfPUeHnzKqbNTO8z7H6irFnUE='}} verified = px_cookie_validator.verify(ctx, config) self.assertFalse(verified) self.assertEqual('cookie_decryption_failed', ctx.get('s2s_call_reason')) def test_verify_cookie_high_score(self): - config = PXConfig({'app_id': 'app_id', - 'cookie_key': self.cookie_key}) - ctx = {'px_cookies': { + config = self.config + ctx = {'user_agent':'', 'is_mobile': False, 'px_cookies': { '_px3': 'bf46ceff75278ae166f376cbf741a7639060581035dd4e93641892c905dd0d67:EGFGcwQ2rum7KRmQCeSXBAUt1+25mj2DFJYi7KJkEliF3cBspdXtD2X03Csv8N8B6S5Bte/4ccCcETkBNDVxTw==:1000:x9x+oI6BISFhlKEERpf8HpZD2zXBCW9lzVfuRURHaAnbaMnpii+XjPEd7a7EGGUSMch5ramy3y+KOxyuX3F+LbGYwvn3OJb+u40zU+ixT1w5N15QltX+nBMhC7izC1l8QtgMuG/f3Nts5ebnec9j2V7LS5Y1/5b73rd9s7AMnug='}} verified = px_cookie_validator.verify(ctx, config) self.assertTrue(verified) self.assertEqual(None, ctx.get('s2s_call_reason')) def test_verify_hmac_validation(self): - config = PXConfig({'app_id': 'app_id', - 'cookie_key': self.cookie_key}) - ctx = {'px_cookies': { + config = self.config + ctx = {'user_agent':'', 'is_mobile': False, 'px_cookies': { '_px3': '774958bcc232343ea1a876b92ababf47086d8a4d95165bbd6f98b55d7e61afd2a05:ow3Er5dskpt8ZZ11CRiDMAueEi3ozJTqMBnYzsSM7/8vHTDA0so6ekhruiTrXa/taZINotR5PnTo78D5zM2pWw==:1000:uQ3Tdt7D3mSO5CuHDis3GgrnkGMC+XAghbHuNOE9x4H57RAmtxkTcNQ1DaqL8rx79bHl0iPVYlOcRmRgDiBCUoizBdUCjsSIplofPBLIl8WpfHDDtpxPKzz9I2rUEbFFjiTY3rPGob2PUvTsDXTfPUeHnzKqbNTO8z7H6irFnUE='}} verified = px_cookie_validator.verify(ctx, config) self.assertFalse(verified) self.assertEqual(None, ctx.get('cookie_validation_failed')) def test_verify_expired_cookie(self): - config = PXConfig({'app_id': 'app_id', - 'cookie_key': self.cookie_key}) - ctx = {'px_cookies': { + config = self.config + ctx = {'user_agent':'', 'is_mobile': False, 'px_cookies': { '_px3': '0d67bdf4a58c524b55b9cf0f703e4f0f3cbe23a10bd2671530d3c7e0cfa509eb:HOiYSw11ICB2A+HYx+C+l5Naxcl7hMeEo67QNghCQByyHlhWZT571ZKfqV98JFWg7TvbV9QtlrQtXakPYeIEjQ==:1000:+kuXS/iJUoEqrm8Fo4K0cTebsc4YQZu+f5bRGX0lC1T+l0g1gzRUuKiCtWTar28Y0wjch1ZQvkNy523Pxr07agVi/RL0SUktmEl59qGor+m4FLewZBVdcgx/Ya9kU0riis98AAR0zdTpTtoN5wpNbmztIpOZ0YejeD0Esk3vagU='}} verified = px_cookie_validator.verify(ctx, config) self.assertFalse(verified) diff --git a/tests/px_httpc.py b/tests/px_httpc.py index 16c0cc6..71c42f5 100644 --- a/tests/px_httpc.py +++ b/tests/px_httpc.py @@ -1,20 +1,22 @@ -from perimeterx import px_httpc import unittest -from mock import MagicMock,patch +from perimeterx import px_httpc +import requests_mock from perimeterx.px_config import PXConfig -import httplib - - -class Test_PXHTTPC(unittest.TestCase): +class TestPXHttpc(unittest.TestCase): def test_send(self): - # px_config = PXConfig({'app_id': 'fake_app_id', - # 'auth_token': 'fake_auth_token'}) - # http_client = httplib.HTTPConnection(host='host', timeout=1) - # from httplib2 import Response - # http_client.request = MagicMock(return_value= Response({'status':'200'})) - # with patch('perimeterx.px_httpc.httplib', return_value=http_client): - # message = px_httpc.send('uri', 'body', px_config) + with requests_mock.mock() as m: + config = PXConfig({'app_id': 'PXfake_app_id'}) + full_url = 'this_url.com/uri' + method = 'POST' + body = 'content to post' + headers = {'content-type': 'application/json'} - print 'f' \ No newline at end of file + m.post('https://' + full_url) + response = px_httpc.send(full_url=full_url, config=config,method=method,body=body,headers=headers) + m.called + m.get('https://' + full_url) + method = 'GET' + response = px_httpc.send(full_url=full_url, config=config,method=method,body=body,headers=headers) + self.assertEqual(m.call_count, 2) \ No newline at end of file diff --git a/tests/px_proxy_handler.py b/tests/px_proxy_handler.py index 8e14db5..32dc95b 100644 --- a/tests/px_proxy_handler.py +++ b/tests/px_proxy_handler.py @@ -1,10 +1,9 @@ import unittest - -from httplib import HTTPResponse - +import requests_mock from perimeterx.px_proxy_handler import PXProxy from perimeterx.px_config import PXConfig -from mock import MagicMock,patch +from perimeterx import px_constants + class Test_PXProxy(unittest.TestCase): @@ -18,16 +17,54 @@ def test_should_reverse_request(self): should_reverse = px_proxy.should_reverse_request('/fake_app_id/captcha') self.assertTrue(should_reverse) - # def test_send_reverse_client_request(self): - # content = 'client js content' - # config = PXConfig({'app_id': 'PXfake_app_id'}) - # ctx = {'uri': '/fake_app_id/init.js', 'headers': {'X-FORWARDED-FOR': '127.0.0.1'}} - # fake = HttpResponse(content=content, status=200, reason='OK', content_type='text/html') - # px_proxy = PXProxy(config) - # config = PXConfig({'app_id': 'PXfake_app_id'}) - # with patch('perimeterx.px_httpc.send_https', return_value=fake): - # result = px_proxy.handle_reverse_request(config=config, ctx=ctx, start_response= lambda x: x) - # print '' - + @requests_mock.mock() + def test_send_reverse_client_request(self, mock): + content = 'client js content' + config = PXConfig({'app_id': 'PXfake_app_id'}) + ctx = {'uri': '/fake_app_id/init.js', 'headers': {'X-FORWARDED-FOR': '127.0.0.1'}, 'ip': '127.0.0.1'} + headers = {'host': px_constants.CLIENT_HOST, + px_constants.FIRST_PARTY_HEADER: '1', + px_constants.ENFORCER_TRUE_IP_HEADER: ctx.get('ip'), + px_constants.FIRST_PARTY_FORWARDED_FOR: '127.0.0.1'} + mock.get(url='https://client.perimeterx.net/PXfake_app_id/main.min.js', text=content, request_headers=headers, + status_code=200, reason='OK') + px_proxy = PXProxy(config) + response_body = px_proxy.handle_reverse_request(config=config, ctx=ctx, start_response=lambda x, y: x, body='') + self.assertEqual(content, response_body) + @requests_mock.mock() + def test_send_reverse_captcha_request(self, mock): + content = 'captcha js content' + config = PXConfig({'app_id': 'PXfake_app_id'}) + ctx = {'uri': '/fake_app_id/captcha/captcha.js', + 'headers': {'X-FORWARDED-FOR': '127.0.0.1'}, + 'ip': '127.0.0.1', + 'query_params': 'a=c&u=cfe74220-f484-11e8-9b14-d7280325a290&v=0701bb80-f482-11e8-8a31-a37cf9620569&m=0'} + headers = {'host': px_constants.CAPTCHA_HOST, + px_constants.FIRST_PARTY_HEADER: '1', + px_constants.ENFORCER_TRUE_IP_HEADER: ctx.get('ip'), + px_constants.FIRST_PARTY_FORWARDED_FOR: '127.0.0.1'} + mock.get( + url='https://captcha.px-cdn.net/PXfake_app_id/captcha.js?a=c&u=cfe74220-f484-11e8-9b14-d7280325a290&v=0701bb80-f482-11e8-8a31-a37cf9620569&m=0', + text=content, request_headers=headers, status_code=200, reason='OK') + px_proxy = PXProxy(config) + response_body = px_proxy.handle_reverse_request(config=config, ctx=ctx, start_response=lambda x, y: x, body='') + self.assertEqual(content, response_body) + @requests_mock.mock() + def test_send_reverse_xhr_request(self, mock): + content = 'captcha content' + config = PXConfig({'app_id': 'PXfake_app_id'}) + ctx = {'uri': '/fake_app_id/xhr/api/v1/collector', + 'headers': {'X-FORWARDED-FOR': '127.0.0.1'}, + 'ip': '127.0.0.1', + 'http_method': 'POST'} + headers = {'host': config.collector_host, + px_constants.FIRST_PARTY_HEADER: '1', + px_constants.ENFORCER_TRUE_IP_HEADER: ctx.get('ip'), + px_constants.FIRST_PARTY_FORWARDED_FOR: '127.0.0.1'} + mock.post(url='https://collector-pxfake_app_id.perimeterx.net/api/v1/collector', text=content, + request_headers=headers, status_code=200, reason='OK') + px_proxy = PXProxy(config) + response_body = px_proxy.handle_reverse_request(config=config, ctx=ctx, start_response=lambda x, y: x, body='') + self.assertEqual(content, response_body) From 8a6b8ffa66a617288ab97ae647b303ed1c38c3ee Mon Sep 17 00:00:00 2001 From: alexbpx <41577203+alexbpx@users.noreply.github.com> Date: Sun, 2 Dec 2018 14:53:32 +0200 Subject: [PATCH 28/35] Dev refactor context (#21) * Fxied errors * Config refactoring * Config * Deleted Notes * Feactored static files check * Refactored literals * Block activity * Block action added to block acitivity * Removed socket_ip from context * property * Telemetry * Whitelist and sensitive route capabilities * Changes * px_blocker tests * px_utils tests * px_validator tests * px_validator tests * Fixed a few files after running automation * Fixed unittests * Fxied j challenge response * Added px_proxy tests * Fxied building process of telemetry config * phase 1 * Changed telemtry uri * Empty tests * Empty tests * Fxied captcha loop * Fixed captcha loops * fixed unittests * Mobile header handling * Fixed tests * fixed empty mobile header handling * asdad * Merge remote-tracking branch 'remotes/origin/dev' into dev-mobile_sdk # Conflicts: # perimeterx/middleware.py # perimeterx/px_api.py # perimeterx/px_config.py # perimeterx/px_context.py # perimeterx/px_cookie_v1.py # perimeterx/px_cookie_v3.py # perimeterx/px_httpc.py # perimeterx/px_proxy_handler.py # tests/px_cookie_validator.py # tests/px_proxy_handler.py * Added px_proxy tests * Fixed code and tests * Added unittests * Partial fix * Context commited * fixed score * Fixed unittests * Fixed according to PR requests * Fixed risk api message fields setting * Added README and CHANGELOG * Updated README * Updated README * Updated README --- CHANGELOG.md | 5 + README.md | 365 ++++++---------- perimeterx/middleware.py | 26 +- perimeterx/px_activities_client.py | 47 +-- perimeterx/px_api.py | 89 ++-- perimeterx/px_blocker.py | 24 +- perimeterx/px_config.py | 3 +- perimeterx/px_constants.py | 8 +- perimeterx/px_context.py | 482 +++++++++++++++++----- perimeterx/px_cookie.py | 2 +- perimeterx/px_cookie_validator.py | 50 +-- perimeterx/px_original_token_validator.py | 16 +- perimeterx/px_proxy_handler.py | 44 +- perimeterx/px_utils.py | 2 +- requirements.txt | 6 + tests/px_api.py | 8 +- tests/px_blocker.py | 96 +++-- tests/px_blocking_messages/blocking.txt | 6 +- tests/px_cookie.py | 4 +- tests/px_cookie_validator.py | 57 ++- tests/px_httpc.py | 4 +- tests/px_proxy_handler.py | 35 +- tests/px_utils.py | 8 +- 23 files changed, 805 insertions(+), 582 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..dcbc65f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Change Log + +## [v2.0.0](https://github.com/PerimeterX/perimeterx-python-wsgi/compare/v1.0.17...HEAD) (2018-12-03) +- Added Major Enforcer functionalities: Mobile SDK, FirstParty, CaptchaV2, Block handling +- Added unit tests diff --git a/README.md b/README.md index 755327a..3b781cf 100644 --- a/README.md +++ b/README.md @@ -1,329 +1,226 @@ -![image](http://media.marketwire.com/attachments/201604/34215_PerimeterX_logo.jpg) +[![Build Status](https://travis-ci.org/PerimeterX/perimeterx-python-wsgi.svg?branch=master)](https://travis-ci.org/PerimeterX/perimeterx-python-wsgi) -[PerimeterX](http://www.perimeterx.com) Python WSGI Middleware +![image](https://s.perimeterx.net/logo.png) + +[PerimeterX](http://www.perimeterx.com) Python Middleware ============================================================= -> The PerimeterX Python Middleware is supported by all [WSGI based frameworks](https://en.wikipedia.org/wiki/Web_Server_Gateway_Interface#WSGI-compatible_applications_and_frameworks). + +> Latest stable version: [v2.0.0](link to package) Table of Contents ----------------- - -- [Usage](#usage) - * [Dependencies](#dependencies) - * [Installation](#installation) - * [Basic Usage Example](#basic-usage) -- [Configuration](#configuration) - * [Blocking Score](#blocking-score) - * [Customizing Block page](#custom-block-page) - * [Custom Block Action](#custom-block) - * [Enable/Disable Server Calls](#server-calls) - * [Enable/Disable Captcha](#captcha-support) - * [Extracting Real IP Address](#real-ip) - * [Filter Sensitive Headers](#sensitive-headers) - * [API Timeouts](#api-timeout) - * [Send Page Activities](#send-page-activities) - * [Debug Mode](#debug-mode) -- [Contributing](#contributing) - * [Tests](#tests) - - - - Dependencies ----------------------------------------- - -- [Python v2.7](https://www.python.org/download/releases/2.7/) -- [pycrypto v2.6](https://pypi.python.org/pypi/pycrypto) - - Note: pycrypto is a python core module, this need to be manually added to dependencies when using GAE - - - Installation ----------------------------------------- - -Installation can be done using Composer. - -```sh -$ pip install perimeterx-python-wsgi -``` - -### Basic Usage Example -##### Django: +- [Installation](#installation) +- [Basic Usage Example](#basic_usage) +- [Advanced Blocking Response](#advanced_blocking_response) +- [Advanced Configuration Options](#configuration) + * [Module Enabled](#module_enabled) + * [Module Mode](#module_mode) + * [Blocking Score](#blocking_score) + * [Send Page Activities](#send_page_activities) + * [Debug Mode](#debug_mode) + * [Sensitive Routes](#sensitive_routes) + * [Whitelist Routes](#whitelist_routes) + * [Sensitive Headers](#sensitive_headers) + * [IP Headers](#ip_headers) + * [First Party Enabled](#first_party_enabled) + * [Custom Request Handler](#custom_request_handler) + * [Additional Activity Handler](#additional_activity_handler) + +## Installation +PerimeterX python middleware is installed via PIP: +`$ pip install perimeterx-python-wsgi` + +## Basic Usage Example +To use PerimeterX middleware on a specific route follow this example: ```python -from perimeterx.middleware import PerimeterX - px_config = { - 'app_id': 'APP_ID', - 'cookie_key': 'COOKIE_KEY', - 'auth_token': 'AUTH_TOKEN', - 'blocking_score': 70 + 'app_id': 'APP_ID', + 'cookie_key': 'COOKIE_KEY', + 'auth_token': 'AUTH_TOKEN', } application = get_wsgi_application() application = PerimeterX(application, px_config) -``` -##### Google App Engine: -app.yaml: - -```yaml -libraries: -- name: pycrypto - version: 2.6 -``` - -```python -import webapp2 -from perimeterx.middleware import PerimeterX -app = webapp2.WSGIApplication([ - ('/', MainPage), -], debug=True) +**Note:** app id, cookie secret and auth token are required fields. -px_config = { - 'app_id': 'APP_ID', - 'cookie_key': 'COOKIE_KEY', - 'auth_token': 'AUTH_TOKEN', - 'blocking_score': 70 -} -app = PerimeterX(app, px_config) ``` -### Configuration Options -#### Configuring Required Parameters -Configuration options are set in the `px_config` variable. +For details on how to create a custom Captcha page, refer to the [documentation](https://console.perimeterx.com/docs/server_integration_new.html#custom-captcha-section) -#### Required parameters: +## Advanced Configuration Options -- app_id -- cookie_key -- auth_token +In addition to the basic installation configuration [above](#basicUsage), the following configurations options are available: -#### Changing the Minimum Score for Blocking Requests +#### Module Enabled +A boolean flag to enable/disable the PerimeterX Enforcer. -**default:** 70 +**Default:** true ```python -px_config = { - .. - 'blocking_score': 75 - .. +config = { + ... + module_enabled: False + ... } ``` -### Customizing Block Page -#### Customizing logo -Adding a custom logo to the blocking page is by providing the pxConfig a key ```custom_logo``` , the logo will be displayed at the top div of the the block page The logo's ```max-heigh``` property would be 150px and width would be set to ``auto`` +#### Module Mode +Sets the working mode of the Enforcer. -The key customLogo expects a valid URL address such as https://s.perimeterx.net/logo.png +Possible values: -Example below: -```python -px_config = { - .. - 'custom_logo': 'https://s.perimeterx.net/logo.png' - .. -} -``` - -#### Custom JS/CSS - -The block page can be modified with a custom CSS by adding to the pxConfig the key ```css_ref``` and providing a valid URL to the css In addition there is also the option to add a custom JS file by adding ```js_ref``` key to the pxConfig and providing the JS file that will be loaded with the block page, this key also expects a valid URL +* `active_blocking` - Blocking Mode +* `monitor` - Monitoring Mode -On both cases if the URL is not a valid format an exception will be thrown -Example below: +**Default:** `monitor` - Monitor Mode -Example below: ```python -px_config = { - .. - 'js_ref': 'https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js' - 'css_ref': 'https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css' - .. +config = { + ... + module_mode: 'active_blocking' + ... } ``` -#### Custom Blocking Actions -Defining a custom block handler is done by setting the value of `custom_block_handler` to a user-defined function, on the `px_config` variable. - -The custom block handler should contain the action to take when a visitng user is given a high score. Common customizations are to present a reCAPTHA or a custom branded Block Page. +#### Blocking Score +Sets the minimum blocking score of a request. -**default:** return HTTP status code 403 and serve the PerimeterX block page. +Possible values: -```python -def custom_block_handler(ctx, start_response): - start_response('403 Forbidden', [('Content-Type', 'text/html')]) - return ['You have been blocked'] +* Any integer between 0 and 100. +**Default:** 100 -px_config = { - .. - 'custom_block_handler': custom_block_handler, - .. +```python +config = { + ... + blocking_score: 100 + ... } +``` -application = get_wsgi_application() -application = PerimeterX(application, px_config) -``` - -###### Examples +#### Send Page Activities +A boolean flag to enable/disable sending activities and metrics to PerimeterX with each request.
+Enabling this feature allows data to populate the PerimeterX Portal with valuable information, such as the number of requests blocked and additional API usage statistics. -**Serve a Custom HTML Page** +**Default:** true ```python -def custom_block_handler(ctx, start_response): - block_score = ctx.get('risk_score') - block_uuid = ctx.get('uuid') - full_url = ctx.get('full_url') - - html = '
Access to ' + full_url + ' has been blocked.
' \ - '
Block reference - ' + uuid + '
' \ - '
Block score - ' + block_score + '
' - - start_response('403 Forbidden', [('Content-Type', 'text/html')]) - return [html] -}; - -px_config = { - .. - 'custom_block_handler': custom_block_handler, - .. +config = { + ... + send_page_activities: True + ... } - -application = get_wsgi_application() -application = PerimeterX(application, px_config) ``` -#### Module Mode -**default:** `active_monitoring` +#### Debug Mode +A boolean flag to enable/disable the debug log messages. -**Applicable Values:** - `['active_monitoring', 'active_blocking', 'inactive']` +**Default:** False ```python -px_config = { - .. - 'module_mode': 'active_blocking' - .. +config = { + ... + debug_mode: True + ... } ``` -#### Enable/Disable Server Calls +#### Sensitive Routes +An array of route prefixes that trigger a server call to PerimeterX servers every time the page is viewed, regardless of viewing history. -By disabling server calls, the module will only evaluate users by their cookie. Users without a cookie will not generate a request to the PerimeterX servers. - -**default:** `True` +**Default:** Empty ```python -px_config = { - .. - 'server_calls_enabled': False - .. +const config = { + ... + sensitive_routes: ['/login', '/user/checkout'] + ... } ``` -#### Enable/Disable CAPTCHA on the block page +#### Whitelist Routes +An array of route prefixes which will bypass enforcement (will never get scored). -By enabling CAPTCHA support, a CAPTCHA will be served as part of the block page, giving real users the ability to answer, get their score cleaned up and navigate to the requested page. - -**default: True** +**Default:** Empty ```python -px_config = { - .. - 'captcha_enabled': True - .. +config = { + ... + whitelist_routes: ['/about-us', '/careers'] + ... } ``` -#### Extracting the Real User IP Address - -> Note: IP extraction, according to your network setup, is important. It is common to have a load balancer/proxy on top of your applications, in this case the PerimeterX module will send an internal IP as the user's. In order to perform processing and detection for server-to-server calls, PerimeterX's module requires the real user's IP. - -The user's IP can be returned to the PerimeterX module, using a custom user defined function on the `px_config` variable. +#### Sensitive Headers +An array of headers that are not sent to PerimeterX servers on API calls. -**default value:** `environ.get('REMOTE_ADDR')` +**Default:** ['cookie', 'cookies'] ```python -def ip_handler(environ): - for key in environ.keys(): - if key == 'HTTP_X_FORWARDED_FOR': - xff = environ[key].split(' ')[1] - return xff - return '1.2.3.4' - -px_config = { - .. - 'ip_handler': ip_handler, - .. +config = { + ... + sensitive_headers: ['cookie', 'cookies', 'x-sensitive-header'] + ... } - - -application = get_wsgi_application() -application = PerimeterX(application, px_config) ``` -#### Filter sensitive headers - -A user can define a list of sensitive headers that will be excluded from any message sent to PerimeterX's servers (lowere case header names). Filtering the 'cookie' header is set by default (for privacy) and will be overridden if a user specifies otherwise in the configuration. +#### IP Headers +An array of trusted headers that specify an IP to be extracted. -**default value:** `['cookie', 'cookies']` +**Default:** Empty ```python -px_config = { - .. - 'sensitive_headers': ['cookie', 'cookies', 'secret-header'] - .. +config = { + ... + ip_headers: ['x-user-real-ip'] + ... } ``` -#### API Timeouts - -Controls the timeouts for PerimeterX requests. The API is called when the risk cookie does not exist, is expired or is invalid. - -API Timeout in seconds (float) to wait for the PerimeterX servers' API response. - +#### First Party Enabled +A boolean flag to enable/disable first party mode. -**default:** 1 +**Default:** True ```python -px_config = { - .. - 'api_timeout': 2 - .. +const pxConfig = { + ... + first_party_enabled: False + ... } ``` - -#### Send Page Activities - -A boolean flag to determine whether or not to send activities and metrics to -PerimeterX, on each page request. Disabling this feature will prevent PerimeterX from receiving data -populating the PerimeterX portal, containing valuable information such as -the amount of requests blocked and other API usage statistics. - -**default:** True +#### Custom Request Handler +A Python function that adds a custom response handler to the request. +Do not forget to declare the function before using it in the config. +Custom request handler is triggered after PerimeterX's verification. +The custom function should handle the response (probably create a new one) +**Default:** Empty ```python -px_config = { - .. - 'send_page_activities': False - .. +config = { + ... + custom_request_handler: custom_request_handler_function, + ... } ``` -#### Debug Mode - -Enables debug logging. +#### Additional Activity Handler +A Python function that allows interaction with the request data collected by PerimeterX before the data is returned to the PerimeterX servers. Does not alter the response. -**default:** false +**Default:** Empty ```python -px_config = { - .. - 'debug_mode': True - .. +config = { + ... + additional_activity_handler: additional_activity_handler_function, + ... } ``` - Contributing ----------------------------------------- diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index e98f89d..bc7171c 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -1,20 +1,19 @@ -import px_context import px_activities_client import px_cookie_validator -import px_httpc +from px_context import PxContext import px_blocker import px_api import px_constants import px_utils from perimeterx.px_proxy_handler import PXProxy -from px_config import PXConfig +from px_config import PxConfig class PerimeterX(object): def __init__(self, app, config=None): self.app = app # merging user's defined configurations with the default one - px_config = PXConfig(config) + px_config = PxConfig(config) logger = px_config.logger if not px_config.app_id: logger.error('PX App ID is missing') @@ -41,11 +40,12 @@ def _verify(self, environ, start_response): config = self.config logger = config.logger try: - ctx = px_context.build_context(environ, config) - uri = ctx.get('uri') + ctx = PxContext(environ, config) + uri = ctx.uri px_proxy = PXProxy(config) if px_proxy.should_reverse_request(uri): - body = environ['wsgi.input'].read(int(environ.get('CONTENT_LENGTH'))) if environ.get('CONTENT_LENGTH') else '' + body = environ['wsgi.input'].read(int(environ.get('CONTENT_LENGTH'))) if environ.get( + 'CONTENT_LENGTH') else '' return px_proxy.handle_reverse_request(self.config, ctx, start_response, body) if px_utils.is_static_file(ctx): logger.debug('Filter static file request. uri: ' + uri) @@ -54,7 +54,7 @@ def _verify(self, environ, start_response): logger.debug('Module is disabled, request will not be verified') return self.app(environ, start_response) - if ctx.get('whitelist'): + if ctx.whitelist_route: logger.debug('The requested uri is whitelisted, passing request') return self.app(environ, start_response) @@ -65,12 +65,12 @@ def _verify(self, environ, start_response): return self.app(environ, start_response) return self.handle_verification(ctx, self.config, environ, start_response) except: - logger.error("Cought exception, passing request") - self.pass_traffic({}) + logger.error("Caught exception, passing request") + self.pass_traffic(PxContext({}, config)) return self.app(environ, start_response) def handle_verification(self, ctx, config, environ, start_response): - score = ctx.get('score', -1) + score = ctx.score result = None headers = None status = None @@ -88,7 +88,7 @@ def handle_verification(self, ctx, config, environ, start_response): result, headers, status = self.px_blocker.handle_blocking(ctx=ctx, config=config) if config.custom_request_handler: custom_body, custom_headers, custom_status = config.custom_request_handler(ctx, self.config, environ) - if (custom_body is not None): + if custom_body is not None: start_response(custom_status, custom_headers) return custom_body @@ -99,7 +99,7 @@ def handle_verification(self, ctx, config, environ, start_response): return self.app(environ, start_response) def pass_traffic(self, ctx): - px_activities_client.send_page_requested_activity( ctx, self.config) + px_activities_client.send_page_requested_activity(ctx, self.config) def block_traffic(self, ctx): px_activities_client.send_block_activity(ctx, self.config) diff --git a/perimeterx/px_activities_client.py b/perimeterx/px_activities_client.py index 4037464..95f09b4 100644 --- a/perimeterx/px_activities_client.py +++ b/perimeterx/px_activities_client.py @@ -9,6 +9,7 @@ ACTIVITIES_BUFFER = [] CONFIG = {} + def init_activities_configuration(config): global CONFIG CONFIG = config @@ -16,6 +17,7 @@ def init_activities_configuration(config): t1.daemon = True t1.start() + def send_activities(): global ACTIVITIES_BUFFER default_headers = { @@ -27,11 +29,11 @@ def send_activities(): if len(ACTIVITIES_BUFFER) > 0: chunk = ACTIVITIES_BUFFER[:10] ACTIVITIES_BUFFER = ACTIVITIES_BUFFER[10:] - px_httpc.send(full_url=full_url, body=json.dumps(chunk), headers=default_headers, config=CONFIG, method='POST') + px_httpc.send(full_url=full_url, body=json.dumps(chunk), headers=default_headers, config=CONFIG, + method='POST') time.sleep(1) - def send_to_perimeterx(activity_type, ctx, config, detail): try: if activity_type == 'page_requested' and not config.send_page_activities: @@ -39,8 +41,8 @@ def send_to_perimeterx(activity_type, ctx, config, detail): return _details = { - 'http_method': ctx.get('http_method', ''), - 'http_version': ctx.get('http_version', ''), + 'http_method': ctx.http_method, + 'http_version': ctx.http_version, 'module_version': config.module_version, 'risk_mode': config.module_mode, } @@ -50,14 +52,14 @@ def send_to_perimeterx(activity_type, ctx, config, detail): data = { 'type': activity_type, - 'headers': ctx.get('headers'), + 'headers': ctx.headers, 'timestamp': int(round(time.time() * 1000)), - 'socket_ip': ctx.get('ip'), + 'socket_ip': ctx.ip, 'px_app_id': config.app_id, - 'url': ctx.get('full_url'), + 'url': ctx.full_url, 'details': _details, - 'vid': ctx.get('vid', ''), - 'uuid': ctx.get('uuid', '') + 'vid': ctx.vid, + 'uuid': ctx.uuid } ACTIVITIES_BUFFER.append(data) except: @@ -67,25 +69,27 @@ def send_to_perimeterx(activity_type, ctx, config, detail): def send_block_activity(ctx, config): send_to_perimeterx(px_constants.BLOCK_ACTIVITY, ctx, config, { - 'block_score': ctx.get('risk_score'), - 'client_uuid': ctx.get('uuid'), - 'block_reason': ctx.get('block_reason'), - 'http_method': ctx.get('http_method'), - 'http_version': ctx.get('http_version'), - 'px_cookie': ctx.get('decoded_cookie'), - 'risk_rtt': ctx.get('risk_rtt'), - #'cookie_origin':, - 'block_action': ctx.get('block_action', ''), + 'block_score': ctx.score, + 'client_uuid': ctx.uuid, + 'block_reason': ctx.block_reason, + 'http_method': ctx.http_method, + 'http_version': ctx.http_version, + 'px_cookie': ctx.decoded_cookie, + 'risk_rtt': ctx.risk_rtt, + 'cookie_origin': ctx.cookie_origin, + 'block_action': ctx.block_action, 'module_version': px_constants.MODULE_VERSION, 'simulated_block': config.module_mode is px_constants.MODULE_MODE_MONITORING, }) + def send_page_requested_activity(ctx, config): details = {} - if ctx.get('decoded_cookie', ''): - details = {"px_cookie": ctx['decoded_cookie']} + if ctx.decoded_cookie: + details = {"px_cookie": ctx.decoded_cookie} send_to_perimeterx(px_constants.PAGE_REQUESTED_ACTIVITY, ctx, config, details) + def send_enforcer_telemetry_activity(config, update_reason): details = { 'enforcer_configs': config.telemetry_config, @@ -107,6 +111,3 @@ def send_enforcer_telemetry_activity(config, update_reason): config.logger.debug('Sending telemetry activity to PerimeterX servers') px_httpc.send(full_url=config.server_host + px_constants.API_ENFORCER_TELEMETRY, body=json.dumps(body), headers=headers, config=config, method='POST') - - - diff --git a/perimeterx/px_api.py b/perimeterx/px_api.py index 4f6d16d..87cadac 100644 --- a/perimeterx/px_api.py +++ b/perimeterx/px_api.py @@ -17,15 +17,18 @@ 'custom_param10': '' } + def send_risk_request(ctx, config): body = prepare_risk_body(ctx, config) default_headers = { 'Authorization': 'Bearer ' + config.auth_token, 'Content-Type': 'application/json' } - response = px_httpc.send(full_url=config.server_host + px_constants.API_RISK,body=json.dumps(body),config=config, headers=default_headers,method='POST') + response = px_httpc.send(full_url=config.server_host + px_constants.API_RISK, body=json.dumps(body), config=config, + headers=default_headers, method='POST') return json.loads(response.content) + def verify(ctx, config): logger = config.logger logger.debug("PXVerify") @@ -36,24 +39,24 @@ def verify(ctx, config): logger.debug('Risk call took ' + str(risk_rtt) + 'ms') if response: - score = response['score'] - ctx['score'] = score - ctx['uuid'] = response['uuid'] - ctx['block_action'] = response['action'] - ctx['risk_rtt'] = risk_rtt - if score >= config.blocking_score: - if response['action'] == 'j' and response.get('action_data') is not None and response.get('action_data').get('body') is not None: + ctx.score = response.get('score') + ctx.uuid = response.get('uuid') + ctx.block_action = response.get('action') + ctx.risk_rtt = risk_rtt + if ctx.score >= config.blocking_score: + if response.get('action') == px_constants.ACTION_CHALLENGE and response.get('action_data') is not None and response.get( + 'action_data').get('body') is not None: logger.debug("PXVerify received javascript challenge action") - ctx['block_action_data'] = response.get('action_data').get('body') - ctx['block_reason'] = 'challenge' - elif response['action'] is 'r': + ctx.block_action_data = response.get('action_data').get('body') + ctx.block_reason = 'challenge' + elif response.get('action') is px_constants.ACTION_RATELIMIT: logger.debug("PXVerify received javascript ratelimit action") - ctx['block_reason'] = 'exceeded_rate_limit' + ctx.block_reason = 'exceeded_rate_limit' else: logger.debug("PXVerify block score threshold reached, will initiate blocking") - ctx['block_reason'] = 's2s_high_score' + ctx.block_reason = 's2s_high_score' else: - ctx['pass_reason'] = 's2s' + ctx.pass_reason = 's2s' logger.debug("PxAPI[verify] S2S completed") return True @@ -69,26 +72,29 @@ def prepare_risk_body(ctx, config): logger.debug("PxAPI[send_risk_request]") body = { 'request': { - 'ip': ctx.get('ip'), - 'headers': format_headers(ctx.get('headers')), - 'uri': ctx.get('uri'), - 'url': ctx.get('full_url', ''), + 'ip': ctx.ip, + 'headers': format_headers(ctx.headers), + 'uri': ctx.uri, + 'url': ctx.full_url, 'firstParty': 'true' if config.first_party else 'false' }, - 'vid': ctx.get('vid', ''), - 'uuid': ctx.get('uuid', ''), 'additional': { - 's2s_call_reason': ctx.get('s2s_call_reason', ''), - 'http_method': ctx.get('http_method', ''), - 'http_version': ctx.get('http_version', ''), + 's2s_call_reason': ctx.s2s_call_reason, + 'http_method': ctx.http_method, + 'http_version': ctx.http_version, 'module_version': config.module_version, 'risk_mode': config.module_mode, - 'request_cookie_names': ctx.get('cookie_names', ''), - 'cookie_origin': ctx.get('cookie_origin') + 'cookie_origin': ctx.cookie_origin } } - if ctx.get('cookie_hmac'): - body['additional']['px_cookie_hmac'] = ctx.get('cookie_hmac') + if ctx.vid: + body['vid'] = ctx.vid + if ctx.uuid: + body['uuid'] = ctx.uuid + if ctx.cookie_hmac: + body['additional']['px_cookie_hmac'] = ctx.cookie_hmac + if ctx.cookie_names: + body['additional']['request_cookie_names'] = ctx.cookie_names body = add_original_token_data(ctx, body) @@ -96,32 +102,33 @@ def prepare_risk_body(ctx, config): if config.enrich_custom_parameters: risk_custom_params = config.enrich_custom_parameters(custom_params) for param in risk_custom_params: - if re.match('^custom_param\d$',param) and risk_custom_params[param]: + if re.match('^custom_param\d$', param) and risk_custom_params[param]: body['additional'][param] = risk_custom_params[param] - - if ctx['s2s_call_reason'] == 'cookie_decryption_failed': + if ctx.s2s_call_reason == 'cookie_decryption_failed': logger.debug('attaching orig_cookie to request') - body['additional']['px_orig_cookie'] = ctx.get('px_orig_cookie') + body['additional']['px_orig_cookie'] = ctx.px_orig_cookie - if ctx['s2s_call_reason'] in ['cookie_expired', 'cookie_validation_failed']: + if ctx.s2s_call_reason in ['cookie_expired', 'cookie_validation_failed']: logger.debug('attaching px_cookie to request') - body['additional']['px_cookie'] = ctx.get('decoded_cookie') + body['additional']['px_cookie'] = ctx.decoded_cookie logger.debug("PxAPI[send_risk_request] request body: " + str(body)) return body + def add_original_token_data(ctx, body): - if ctx.get('original_uuid'): - body['additional']['original_uuid'] = ctx.get('original_uuid') - if ctx.get('original_token_error'): - body['additional']['original_token_error'] = ctx.get('original_token_error') - if ctx.get('original_token'): - body['additional']['original_token'] = ctx.get('original_token') - if ctx.get('decoded_original_token'): - body['additional']['decoded_original_token'] = ctx.get('decoded_original_token') + if ctx.original_uuid: + body['additional']['original_uuid'] = ctx.original_uuid + if ctx.original_token_error: + body['additional']['original_token_error'] = ctx.original_token_error + if ctx.original_token: + body['additional']['original_token'] = ctx.original_token + if ctx.decoded_original_token: + body['additional']['decoded_original_token'] = ctx.decoded_original_token return body + def format_headers(headers): ret_val = [] for key in headers.keys(): diff --git a/perimeterx/px_blocker.py b/perimeterx/px_blocker.py index c0759df..315b78a 100644 --- a/perimeterx/px_blocker.py +++ b/perimeterx/px_blocker.py @@ -11,7 +11,7 @@ def __init__(self): px_template.get_template(px_constants.RATELIMIT_TEMPLATE), {}) def handle_blocking(self, ctx, config): - action = ctx.get('block_action') + action = ctx.block_action status = '403 Forbidden' is_json_response = self.is_json_response(ctx) @@ -21,10 +21,10 @@ def handle_blocking(self, ctx, config): content_type = 'text/html' headers = [('Content-Type', content_type)] - if action is 'j': - blocking_props = ctx['block_action_data'] + if action is px_constants.ACTION_CHALLENGE: + blocking_props = ctx.block_action_data blocking_response = blocking_props - elif action is 'r': + elif action is px_constants.ACTION_RATELIMIT: blocking_response = self.ratelimit_rendered_page status = '429 Too Many Requests' else: @@ -37,13 +37,13 @@ def handle_blocking(self, ctx, config): def prepare_properties(self, ctx, config): app_id = config.app_id - vid = ctx.get('vid') if ctx.get('vid') is not None else '' - uuid = ctx.get('uuid') + vid = ctx.vid + uuid = ctx.uuid custom_logo = config.custom_logo - is_mobile_num = 1 if ctx.get('is_mobile') else 0 - captcha_uri = 'captcha.js?a={}&u={}&v={}&m={}'.format(ctx.get('block_action'), uuid, vid, is_mobile_num) + is_mobile_num = 1 if ctx.is_mobile else 0 + captcha_uri = 'captcha.js?a={}&u={}&v={}&m={}'.format(ctx.block_action, uuid, vid, is_mobile_num) - if config.first_party and not ctx.get('is_mobile'): + if config.first_party and not ctx.is_mobile: prefix = app_id[2:] js_client_src = '/{}/{}'.format(prefix, px_constants.CLIENT_FP_PATH) captcha_src = '/{}/{}/{}'.format(prefix, px_constants.CAPTCHA_FP_PATH, captcha_uri) @@ -69,10 +69,10 @@ def prepare_properties(self, ctx, config): } def is_json_response(self, ctx): - headers = ctx.get('headers') - if ctx.get('block_action') is not 'r': + headers = ctx.headers + if ctx.block_action is not px_constants.ACTION_RATELIMIT: for item in headers.keys(): - if (item.lower() == 'accept' or item.lower() == 'content-type'): + if item.lower() == 'accept' or item.lower() == 'content-type': item_arr = headers[item].split(',') for header_item in item_arr: if header_item.strip() == 'application/json': diff --git a/perimeterx/px_config.py b/perimeterx/px_config.py index 247859e..41d1263 100644 --- a/perimeterx/px_config.py +++ b/perimeterx/px_config.py @@ -1,9 +1,8 @@ import px_constants -import json from px_logger import Logger -class PXConfig(object): +class PxConfig(object): def __init__(self, config_dict): app_id = config_dict.get('app_id') debug_mode = config_dict.get('debug_mode', False) diff --git a/perimeterx/px_constants.py b/perimeterx/px_constants.py index 1d99271..5fcc489 100644 --- a/perimeterx/px_constants.py +++ b/perimeterx/px_constants.py @@ -10,10 +10,6 @@ BLOCK_TEMPLATE = 'block_template.mustache' RATELIMIT_TEMPLATE = 'ratelimit.mustache' -CAPTCHA_ACTION_CAPTCHA = 'c' -BLOCK_ACTION_CAPTCHA = 'b' -BLOCK_ACTION_CHALLENGE = 'j' -BLOCK_ACTION_RATE = 'r' CLIENT_HOST = 'client.perimeterx.net' CAPTCHA_HOST = 'captcha.px-cdn.net' COLLECTOR_URL = 'collector-{}.perimeterx.net' @@ -36,3 +32,7 @@ API_ENFORCER_TELEMETRY = '/api/v2/risk/telemetry' API_ACTIVITIES = '/api/v1/collector/s2s' TELEMETRY_ACTIVITY = 'enforcer_telemetry' +ACTION_CHALLENGE = 'j' +ACTION_BLOCK = 'b' +ACTION_RATELIMIT = 'r' +ACTION_CAPTCHA = 'c' diff --git a/perimeterx/px_context.py b/perimeterx/px_context.py index db9aa84..d37e818 100644 --- a/perimeterx/px_context.py +++ b/perimeterx/px_context.py @@ -1,115 +1,375 @@ import Cookie from px_constants import * -def build_context(environ, config): - logger = config.logger - headers = {} - - # Default values - http_method = 'GET' - http_version = '1.1' - http_protocol = 'http://' - px_cookies = {} - request_cookie_names = list() - cookie_origin = "cookie" - original_token = None - - # Extracting: Headers, user agent, http method, http version - for key in environ.keys(): - if key.startswith('HTTP_') and environ.get(key): - header_name = key.split('HTTP_')[1].replace('_', '-').lower() - if header_name not in config.sensitive_headers: - headers[header_name] = environ.get(key) - if key == 'REQUEST_METHOD': - http_method = environ.get(key) - if key == 'SERVER_PROTOCOL': - protocol_split = environ.get(key, '').split('/') - if protocol_split[0].startswith('HTTP'): - http_protocol = protocol_split[0].lower() + '://' - if len(protocol_split) > 1: - http_version = protocol_split[1] - if key == 'CONTENT_TYPE' or key == 'CONTENT_LENGTH': - headers[key.replace('_', '-').lower()] = environ.get(key) - if key == 'HTTP_' + MOBILE_SDK_HEADER.replace('-','_').upper(): - headers[MOBILE_SDK_HEADER] = environ.get(key, '') - - mobile_header = headers.get(MOBILE_SDK_HEADER) - vid = None - original_token = None - if mobile_header is None: - cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE', '')) - cookie_keys = cookies.keys() - - for key in cookie_keys: - request_cookie_names.append(key) - if key == PREFIX_PX_COOKIE_V1 or key == PREFIX_PX_COOKIE_V3: - logger.debug('Found cookie prefix:' + key) - px_cookies[key] = cookies.get(key).value - vid = None - if '_pxvid' in cookie_keys: - vid = cookies.get('_pxvid').value + +class PxContext(object): + + def __init__(self, environ, config): + + logger = config.logger + headers = {} + + # Default values + http_method = '' + http_version = '' + http_protocol = '' + px_cookies = {} + request_cookie_names = [] + cookie_origin = "cookie" + vid = '' + + # Extracting: Headers, user agent, http method, http version + for key in environ.keys(): + if key.startswith('HTTP_') and environ.get(key): + header_name = key.split('HTTP_')[1].replace('_', '-').lower() + if header_name not in config.sensitive_headers: + headers[header_name] = environ.get(key) + if key == 'REQUEST_METHOD': + http_method = environ.get(key) + if key == 'SERVER_PROTOCOL': + protocol_split = environ.get(key, '').split('/') + if protocol_split[0].startswith('HTTP'): + http_protocol = protocol_split[0].lower() + '://' + if len(protocol_split) > 1: + http_version = protocol_split[1] + if key == 'CONTENT_TYPE' or key == 'CONTENT_LENGTH': + headers[key.replace('_', '-').lower()] = environ.get(key) + if key == 'HTTP_' + MOBILE_SDK_HEADER.replace('-', '_').upper(): + headers[MOBILE_SDK_HEADER] = environ.get(key, '') + + mobile_header = headers.get(MOBILE_SDK_HEADER) + original_token = '' + if mobile_header is None: + cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE', '')) + cookie_keys = cookies.keys() + + for key in cookie_keys: + request_cookie_names.append(key) + if key == PREFIX_PX_COOKIE_V1 or key == PREFIX_PX_COOKIE_V3: + logger.debug('Found cookie prefix:' + key) + px_cookies[key] = cookies.get(key).value + if '_pxvid' in cookie_keys: + vid = cookies.get('_pxvid').value else: - vid = '' - else: - cookie_origin = "header" - original_token = headers.get(MOBILE_SDK_ORIGINAL_HEADER) - logger.debug('Mobile SDK token detected') - cookie_name, cookie = get_token_object(config, mobile_header) - px_cookies[cookie_name] = cookie - - user_agent = headers.get('user-agent','') - uri = environ.get('PATH_INFO') or '' - full_url = http_protocol + (headers.get('host') or environ.get('SERVER_NAME') or '') + uri - hostname = headers.get('host') - sensitive_route = len(filter(lambda sensitive_route_item : uri.startswith(sensitive_route_item), config.sensitive_routes)) > 0 - whitelist_route = len(filter(lambda whitelist_route_item : uri.startswith(whitelist_route_item), config.whitelist_routes)) > 0 - ctx = { - 'headers': headers, - 'http_method': http_method, - 'http_version': http_version, - 'user_agent': user_agent, - 'full_url': full_url, - 'uri': uri, - 'hostname': hostname, - 'px_cookies': px_cookies, - 'cookie_names': request_cookie_names, - 'risk_rtt': 0, - 'ip': extract_ip(config, environ), - 'vid': vid if vid else None, - 'query_params': environ['QUERY_STRING'], - 'sensitive_route': sensitive_route, - 'whitelist_route': whitelist_route, - 's2s_call_reason': 'none', - 'cookie_origin':cookie_origin, - 'original_token': original_token, - 'is_mobile': cookie_origin == "header" - } - - return ctx - -def get_token_object(config, token): - result = {} - logger = config.logger - sliced_token = token.split(":", 1) - if len(sliced_token) > 1: - key = sliced_token.pop(0) - if key == PREFIX_PX_TOKEN_V1 or key == PREFIX_PX_TOKEN_V3: - logger.debug('Found token prefix:' + key) - return key, sliced_token[0] - return PREFIX_PX_TOKEN_V3, token - -def extract_ip(config, environ): - ip = environ.get('HTTP_X_FORWARDED_FOR') if environ.get('HTTP_X_FORWARDED_FOR') else environ.get('REMOTE_ADDR') - ip_headers = config.ip_headers - logger = config.logger - if ip_headers: - try: - for ip_header in ip_headers: - ip_header_name = 'HTTP_' + ip_header.replace('-', '_').upper() - if environ.get(ip_header_name): - return environ.get(ip_header_name) - except: - logger.debug('Failed to use IP_HEADERS from config') - if config.get_user_ip: - ip = config.get_user_ip(environ) - return ip + cookie_origin = "header" + original_token = headers.get(MOBILE_SDK_ORIGINAL_HEADER) + logger.debug('Mobile SDK token detected') + cookie_name, cookie = self.get_token_object(config, mobile_header) + px_cookies[cookie_name] = cookie + + user_agent = headers.get('user-agent', '') + uri = environ.get('PATH_INFO') or '' + full_url = http_protocol + (headers.get('host') or environ.get('SERVER_NAME') or '') + uri + hostname = headers.get('host') + sensitive_route = len( + filter(lambda sensitive_route_item: uri.startswith(sensitive_route_item), config.sensitive_routes)) > 0 + whitelist_route = len( + filter(lambda whitelist_route_item: uri.startswith(whitelist_route_item), config.whitelist_routes)) > 0 + query_params = environ.get('QUERY_STRING') if environ.get('QUERY_STRING') else '' + self._headers = headers + self._http_method = http_method + self._http_version = http_version + self._user_agent = user_agent + self._full_url = full_url + self._uri = uri + self._hostname = hostname + self._px_cookies = px_cookies + self._cookie_names = request_cookie_names + self._risk_rtt = 0 + self._ip = self.extract_ip(config, environ) + self._vid = vid + self._uuid = '' + self._query_params = query_params + self._sensitive_route = sensitive_route + self._whitelist_route = whitelist_route + self._s2s_call_reason = 'none' + self._cookie_origin = cookie_origin + self._is_mobile = cookie_origin == "header" + self._score = -1 + self._block_reason = '' + self._decoded_cookie = '' + self._block_action = '' + self._block_action_data = '' + self._pass_reason = '' + self._cookie_hmac = '' + self._px_orig_cookie = '' + self._original_token_error = '' + self._original_uuid = '' + self._decoded_original_token = '' + self._original_token = original_token + + + def get_token_object(self, config, token): + result = {} + logger = config.logger + sliced_token = token.split(":", 1) + if len(sliced_token) > 1: + key = sliced_token.pop(0) + if key == PREFIX_PX_TOKEN_V1 or key == PREFIX_PX_TOKEN_V3: + logger.debug('Found token prefix:' + key) + return key, sliced_token[0] + return PREFIX_PX_TOKEN_V3, token + + def extract_ip(self, config, environ): + ip = environ.get('HTTP_X_FORWARDED_FOR') if environ.get('HTTP_X_FORWARDED_FOR') else environ.get('REMOTE_ADDR') + ip_headers = config.ip_headers + logger = config.logger + if ip_headers: + try: + for ip_header in ip_headers: + ip_header_name = 'HTTP_' + ip_header.replace('-', '_').upper() + if environ.get(ip_header_name): + return environ.get(ip_header_name) + except: + logger.debug('Failed to use IP_HEADERS from config') + if config.get_user_ip: + ip = config.get_user_ip(environ) + return ip + + @property + def headers(self): + return self._headers + + @headers.setter + def headers(self, headers): + self._headers = headers + + @property + def http_method(self): + return self._http_method + + @http_method.setter + def http_method(self, http_method): + self._http_method = http_method + + @property + def http_version(self): + return self._http_version + + @http_version.setter + def http_version(self, http_version): + self._http_version = http_version + + @property + def user_agent(self): + return self._user_agent + + @user_agent.setter + def user_agent(self, user_agent): + self._user_agent = user_agent + + @property + def full_url(self): + return self._full_url + + @full_url.setter + def full_url(self, full_url): + self._full_url = full_url + + @property + def uri(self): + return self._uri + + @uri.setter + def uri(self, uri): + self._uri = uri + + @property + def hostname(self): + return self._hostname + + @hostname.setter + def hostname(self, hostname): + self._hostname = hostname + + @property + def px_cookies(self): + return self._px_cookies + + @px_cookies.setter + def px_cookies(self, px_cookies): + self._px_cookies = px_cookies + + @property + def cookie_names(self): + return self._cookie_names + + @cookie_names.setter + def cookie_names(self, cookie_names): + self._cookie_names = cookie_names + + @property + def risk_rtt(self): + return self._risk_rtt + + @risk_rtt.setter + def risk_rtt(self, risk_rtt): + self._risk_rtt = risk_rtt + + @property + def ip(self): + return self._ip + + @ip.setter + def ip(self, ip): + self._ip = ip + + @property + def vid(self): + return self._vid + + @vid.setter + def vid(self, vid): + self._vid = vid + + @property + def query_params(self): + return self._query_params + + @query_params.setter + def query_params(self, query_params): + self._query_params = query_params + + @property + def sensitive_route(self): + return self._sensitive_route + + @sensitive_route.setter + def sensitive_route(self, sensitive_route): + self._sensitive_route = sensitive_route + + @property + def whitelist_route(self): + return self._whitelist_route + + @whitelist_route.setter + def whitelist_route(self, whitelist_route): + self._whitelist_route = whitelist_route + + @property + def s2s_call_reason(self): + return self._s2s_call_reason + + @s2s_call_reason.setter + def s2s_call_reason(self, s2s_call_reason): + self._s2s_call_reason = s2s_call_reason + + @property + def cookie_origin(self): + return self._cookie_origin + + @cookie_origin.setter + def cookie_origin(self, cookie_origin): + self._cookie_origin = cookie_origin + + @property + def original_token(self): + return self._original_token + + @original_token.setter + def original_token(self, original_token): + self._original_token = original_token + + @property + def is_mobile(self): + return self._is_mobile + + @is_mobile.setter + def is_mobile(self, is_mobile): + self._is_mobile = is_mobile + + @property + def score(self): + return self._score + + @score.setter + def score(self, score): + self._score = score + + @property + def uuid(self): + return self._uuid + + @uuid.setter + def uuid(self, uuid): + self._uuid = uuid + + @property + def block_reason(self): + return self._block_reason + + @block_reason.setter + def block_reason(self, block_reason): + self._block_reason = block_reason + + @property + def decoded_cookie(self): + return self._decoded_cookie + + @decoded_cookie.setter + def decoded_cookie(self, decoded_cookie): + self._decoded_cookie = decoded_cookie + + @property + def block_action(self): + return self._block_action + + @block_action.setter + def block_action(self, block_action): + self._block_action = block_action + + @property + def block_action_data(self): + return self._block_action_data + + @block_action_data.setter + def block_action_data(self, block_action_data): + self._block_action_data = block_action_data + + @property + def pass_reason(self): + return self._pass_reason + + @pass_reason.setter + def pass_reason(self, pass_reason): + self._pass_reason = pass_reason + + @property + def cookie_hmac(self): + return self._cookie_hmac + + @cookie_hmac.setter + def cookie_hmac(self, cookie_hmac): + self._cookie_hmac = cookie_hmac + + @property + def px_orig_cookie(self): + return self._px_orig_cookie + + @px_orig_cookie.setter + def px_orig_cookie(self, px_orig_cookie): + self._px_orig_cookie = px_orig_cookie + + @property + def original_token_error(self): + return self._original_token_error + + @original_token_error.setter + def original_token_error(self, original_token_error): + self._original_token_error = original_token_error + + @property + def original_uuid(self): + return self._original_uuid + + @original_uuid.setter + def original_uuid(self, original_uuid): + self._original_uuid = original_uuid + + @property + def decoded_original_token(self): + return self._decoded_original_token + + @decoded_original_token.setter + def decoded_original_token(self, decoded_original_token): + self._decoded_original_token = decoded_original_token diff --git a/perimeterx/px_cookie.py b/perimeterx/px_cookie.py index a638548..46738a6 100644 --- a/perimeterx/px_cookie.py +++ b/perimeterx/px_cookie.py @@ -18,7 +18,7 @@ def __init__(self, config): self._logger = config.logger - def build_px_cookie(self, px_cookies, user_agent): + def build_px_cookie(self, px_cookies, user_agent=''): self._logger.debug("PxCookie[build_px_cookie]") # Check that its not empty if not px_cookies: diff --git a/perimeterx/px_cookie_validator.py b/perimeterx/px_cookie_validator.py index 3ff96bc..d62ada8 100644 --- a/perimeterx/px_cookie_validator.py +++ b/perimeterx/px_cookie_validator.py @@ -16,9 +16,9 @@ def verify(ctx, config): """ logger = config.logger try: - if not ctx["px_cookies"].keys(): + if not ctx.px_cookies.keys(): logger.debug('No risk cookie on the request') - ctx['s2s_call_reason'] = 'no_cookie' + ctx.s2s_call_reason = 'no_cookie' return False if not config.cookie_key: @@ -27,55 +27,55 @@ def verify(ctx, config): return False px_cookie_builder = PxCookie(config) - px_cookie = px_cookie_builder.build_px_cookie(px_cookies=ctx.get('px_cookies'), - user_agent=ctx.get('user_agent')) + px_cookie = px_cookie_builder.build_px_cookie(px_cookies=ctx.px_cookies, + user_agent=ctx.user_agent) #Mobile SDK traffic - if px_cookie and ctx['is_mobile']: + if px_cookie and ctx.is_mobile: pattern = re.compile("^\d+$") if re.match(pattern, px_cookie.raw_cookie): - ctx['s2s_call_reason'] = "mobile_error_" + px_cookie.raw_cookie - if ctx['original_token'] is not None: + ctx.s2s_call_reason = "mobile_error_" + px_cookie.raw_cookie + if ctx.original_token is not None: px_original_token_validator.verify(ctx, config) return False if not px_cookie.deserialize(): logger.error('Cookie decryption failed') - ctx['px_orig_cookie'] = px_cookie.raw_cookie - ctx['s2s_call_reason'] = 'cookie_decryption_failed' + ctx.px_orig_cookie = px_cookie.raw_cookie + ctx.s2s_call_reason = 'cookie_decryption_failed' return False - ctx['score'] = px_cookie.get_score() - ctx['uuid'] = px_cookie.get_uuid() - ctx['vid'] = px_cookie.get_vid() - ctx['decoded_cookie'] = px_cookie.decoded_cookie - ctx['cookie_hmac'] = px_cookie.get_hmac() - ctx['block_action'] = px_cookie.get_action() + ctx.score = px_cookie.get_score() + ctx.uuid = px_cookie.get_uuid() + ctx.vid = px_cookie.get_vid() + ctx.decoded_cookie = px_cookie.decoded_cookie + ctx.cookie_hmac = px_cookie.get_hmac() + ctx.block_action = px_cookie.get_action() if px_cookie.is_high_score(): - ctx['block_reason'] = 'cookie_high_score' - logger.debug('Cookie with high score: ' + str(ctx['score'])) + ctx.block_reason = 'cookie_high_score' + logger.debug('Cookie with high score: ' + str(ctx.score)) return True if px_cookie.is_cookie_expired(): - ctx['s2s_call_reason'] = 'cookie_expired' + ctx.s2s_call_reason = 'cookie_expired' logger.debug('Cookie expired') return False if not px_cookie.is_secured(): logger.debug('Cookie validation failed') - ctx['s2s_call_reason'] = 'cookie_validation_failed' + ctx.s2s_call_reason = 'cookie_validation_failed' return False - if ctx.get('sensitive_route'): - logger.debug('Sensitive route match, sending Risk API. path: {}'.format(ctx.get('uri'))) - ctx['s2s_call_reason'] = 'sensitive_route' + if ctx.sensitive_route: + logger.debug('Sensitive route match, sending Risk API. path: {}'.format(ctx.uri)) + ctx.s2s_call_reason = 'sensitive_route' return False - logger.debug('Cookie validation passed with good score: ' + str(ctx['score'])) + logger.debug('Cookie validation passed with good score: ' + str(ctx.score)) return True except Exception, e: traceback.print_exc() logger.debug('Could not decrypt cookie, exception was thrown, decryption failed ' + e.message) - ctx['px_orig_cookie'] = px_cookie.raw_cookie - ctx['s2s_call_reason'] = 'cookie_decryption_failed' + ctx.px_orig_cookie = px_cookie.raw_cookie + ctx.s2s_call_reason = 'cookie_decryption_failed' return False diff --git a/perimeterx/px_original_token_validator.py b/perimeterx/px_original_token_validator.py index 9762483..c49e8d0 100644 --- a/perimeterx/px_original_token_validator.py +++ b/perimeterx/px_original_token_validator.py @@ -14,26 +14,26 @@ def verify(ctx, config): logger = config.logger try: logger.debug('Original token found, Evaluating') - original_token = ctx.get('original_token') + original_token = ctx.original_token version, no_version_token = original_token.split(':', 1) px_cookie_builder = PxCookie(config) - px_cookie = px_cookie_builder.build_px_cookie({version: no_version_token}, ctx.get('is_mobile'),'') + px_cookie = px_cookie_builder.build_px_cookie({version: no_version_token},'') if not px_cookie.deserialize(): logger.error('Original token decryption failed, value:' + px_cookie.raw_cookie) - ctx['original_token_error'] = 'decryption_failed' + ctx.original_token_error = 'decryption_failed' return False - ctx['decoded_original_token'] = px_cookie.decoded_cookie - ctx['vid'] = px_cookie.get_vid() - ctx['original_uuid'] = px_cookie.get_uuid() + ctx.decoded_original_token = px_cookie.decoded_cookie + ctx.vid = px_cookie.get_vid() + ctx.original_uuid = px_cookie.get_uuid() if not px_cookie.is_secured(): logger.debug('Original token HMAC validation failed, value: ' + str(px_cookie.decoded_cookie)) - ctx['original_token_error'] = 'validation_failed' + ctx.original_token_error = 'validation_failed' return False return True except Exception, e: logger.debug('Could not decrypt original token, exception was thrown, decryption failed ' + e.message) - ctx['original_token_error'] = 'decryption_failed' + ctx.original_token_error = 'decryption_failed' return False diff --git a/perimeterx/px_proxy_handler.py b/perimeterx/px_proxy_handler.py index 405bc46..cfdb968 100644 --- a/perimeterx/px_proxy_handler.py +++ b/perimeterx/px_proxy_handler.py @@ -34,16 +34,16 @@ def should_reverse_request(self, uri): return False def handle_reverse_request(self, config, ctx, start_response, body): - uri = ctx.get('uri').lower() + uri = ctx.uri.lower() if uri.startswith(self.client_reverse_prefix): - return self.send_reverse_client_request(config=config, context=ctx, start_response=start_response) + return self.send_reverse_client_request(config=config, ctx=ctx, start_response=start_response) if uri.startswith(self.xhr_reverse_prefix): - return self.send_reverse_xhr_request(config=config, context=ctx, start_response=start_response, body=body) + return self.send_reverse_xhr_request(config=config, ctx=ctx, start_response=start_response, body=body) if uri.startswith(self.captcha_reverse_prefix): - return self.send_reverse_captcha_request(config=config, context=ctx, start_response=start_response) + return self.send_reverse_captcha_request(config=config, ctx=ctx, start_response=start_response) - def send_reverse_client_request(self, config, context, start_response): + def send_reverse_client_request(self, config, ctx, start_response): if not config.first_party: headers = [('Content-Type', 'application/javascript')] start_response("200 OK", headers) @@ -51,13 +51,13 @@ def send_reverse_client_request(self, config, context, start_response): client_request_uri = '/{}/main.min.js'.format(config.app_id) self._logger.debug( - 'Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), px_constants.CLIENT_HOST, + 'Forwarding request from {} to client at {}{}'.format(ctx.uri.lower(), px_constants.CLIENT_HOST, client_request_uri)) headers = {'host': px_constants.CLIENT_HOST, px_constants.FIRST_PARTY_HEADER: '1', - px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} - filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) + px_constants.ENFORCER_TRUE_IP_HEADER: ctx.ip} + filtered_headers = px_utils.handle_proxy_headers(ctx.headers, ctx.ip) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) delete_extra_headers(filtered_headers) response = px_httpc.send(full_url=px_constants.CLIENT_HOST + client_request_uri, body='', @@ -66,8 +66,8 @@ def send_reverse_client_request(self, config, context, start_response): self.handle_proxy_response(response, start_response) return response.raw.read() - def send_reverse_xhr_request(self, config, context, start_response, body): - uri = context.get('uri') + def send_reverse_xhr_request(self, config, ctx, start_response, body): + uri = ctx.uri if not config.first_party or not config.first_party_xhr_enabled: body, content_type = self.return_default_response(uri) @@ -80,17 +80,17 @@ def send_reverse_xhr_request(self, config, context, start_response, body): host = config.collector_host headers = {'host': host, px_constants.FIRST_PARTY_HEADER: '1', - px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} + px_constants.ENFORCER_TRUE_IP_HEADER: ctx.ip} - if context.get('vid') is not None: - headers['Cookies'] = '_pxvid=' + context.get('vid') + if ctx.vid is not None: + headers['Cookies'] = '_pxvid=' + ctx.vid - filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) + filtered_headers = px_utils.handle_proxy_headers(ctx.headers, ctx.ip) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) self._logger.debug( - 'Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, suffix_uri)) + 'Forwarding request from {} to client at {}{}'.format(ctx.uri.lower(), host, suffix_uri)) response = px_httpc.send(full_url=host + suffix_uri, body=body, - headers=filtered_headers, config=config, method=context.get('http_method')) + headers=filtered_headers, config=config, method=ctx.http_method) if response.status_code >= 400: body, content_type = self.return_default_response(uri) @@ -116,23 +116,23 @@ def return_default_response(self, uri): body = {} return body, content_type - def send_reverse_captcha_request(self, config, context, start_response): + def send_reverse_captcha_request(self, config, ctx, start_response): if not config.first_party: status = '200 OK' headers = [('Content-Type', 'application/javascript')] start_response(status, headers) return '' - uri = '/{}{}?{}'.format(config.app_id, context.get('uri').lower().replace(self.captcha_reverse_prefix, ''), - context['query_params']) + uri = '/{}{}?{}'.format(config.app_id, ctx.uri.lower().replace(self.captcha_reverse_prefix, ''), + ctx.query_params) host = px_constants.CAPTCHA_HOST headers = {'host': px_constants.CAPTCHA_HOST, px_constants.FIRST_PARTY_HEADER: '1', - px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} - filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) + px_constants.ENFORCER_TRUE_IP_HEADER: ctx.ip} + filtered_headers = px_utils.handle_proxy_headers(ctx.headers, ctx.ip) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) delete_extra_headers(filtered_headers) - self._logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, uri)) + self._logger.debug('Forwarding request from {} to client at {}{}'.format(ctx.uri.lower(), host, uri)) response = px_httpc.send(full_url=host + uri, body='', headers=filtered_headers, config=config, method='GET') self.handle_proxy_response(response, start_response) diff --git a/perimeterx/px_utils.py b/perimeterx/px_utils.py index 3fc9b73..dd14177 100644 --- a/perimeterx/px_utils.py +++ b/perimeterx/px_utils.py @@ -17,7 +17,7 @@ def handle_proxy_headers(filtered_headers, ip): def is_static_file(ctx): - uri = ctx.get('uri', '') + uri = ctx.uri static_extensions = ['.css', '.bmp', '.tif', '.ttf', '.docx', '.woff2', '.js', '.pict', '.tiff', '.eot', '.xlsx', '.jpg', '.csv', '.eps', '.woff', '.xls', '.jpeg', '.doc', '.ejs', '.otf', '.pptx', '.gif', '.pdf', '.swf', '.svg', '.ps', '.ico', '.pls', '.midi', '.svgz', '.class', '.png', diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..23ee4f9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +pystache==0.5.4 +requests==2.20.1 +setuptools==40.6.2 +requests_mock==1.5.2 +pycrypto==2.6.1 +webapp2==2.5.2 diff --git a/tests/px_api.py b/tests/px_api.py index d7bc19f..d7d53e7 100644 --- a/tests/px_api.py +++ b/tests/px_api.py @@ -1,6 +1,7 @@ import unittest from perimeterx import px_api -from perimeterx.px_config import PXConfig +from perimeterx.px_config import PxConfig +from perimeterx.px_context import PxContext class Test_PXApi(unittest.TestCase): @@ -11,8 +12,9 @@ def enrich_custom_parameters(self, params): return params def test_prepare_risk_body(self): - config = PXConfig({'app_id': 'app_id', 'enrich_custom_parameters': self.enrich_custom_parameters}) - ctx = {'headers': {}, 's2s_call_reason': 'no_cookie'} + config = PxConfig({'app_id': 'app_id', 'enrich_custom_parameters': self.enrich_custom_parameters}) + ctx = PxContext({},config) + ctx.s2s_call_reason = 'no_cookie' body = px_api.prepare_risk_body(ctx, config) self.assertEqual(body['additional'].get('custom_param1'), '1') self.assertEqual(body['additional'].get('custom_param2'), '5') diff --git a/tests/px_blocker.py b/tests/px_blocker.py index 9e9b9ee..0f3696b 100644 --- a/tests/px_blocker.py +++ b/tests/px_blocker.py @@ -2,30 +2,39 @@ import unittest -from perimeterx.px_config import PXConfig +from perimeterx.px_config import PxConfig +from perimeterx.px_context import PxContext class Test_PXBlocker(unittest.TestCase): def test_is_json_response(self): px_blocker = PXBlocker() - ctx = { - 'headers': {'Accept': 'text/html'} - } + config = PxConfig({'app_id': 'PXfake_app_id'}) + ctx = PxContext({'PATH_INFO': '/fake_app_id/init.js', + 'HTTP_X_FORWARDED_FOR': '127.0.0.1', + 'ip': '127.0.0.1', + 'HTTP_ACCEPT': 'text/html'}, + config) + self.assertFalse(px_blocker.is_json_response(ctx)) - ctx['headers']['Accept'] = 'application/json' + ctx.headers['Accept'] = 'application/json' self.assertTrue(px_blocker.is_json_response(ctx)) def test_handle_blocking(self): px_blocker = PXBlocker() vid = 'bf619be8-94be-458a-b6b1-ee81f154c282' px_uuid = '8712cef7-bcfa-4bb6-ae99-868025e1908a' - ctx = { - 'headers': {'Accept': 'text/html'}, - 'vid': vid, - 'uuid': px_uuid - } - px_config = PXConfig({'app_id': 'PXfake_app_ip'}) + + config = PxConfig({'app_id': 'PXfake_app_id'}) + ctx = PxContext({'PATH_INFO': '/fake_app_id/init.js', + 'HTTP_X_FORWARDED_FOR': '127.0.0.1', + 'ip': '127.0.0.1', + 'HTTP_ACCEPT': 'text/html'}, + config) + ctx.vid = vid + ctx.uuid = px_uuid + px_config = PxConfig({'app_id': 'PXfake_app_ip'}) message, _, _ = px_blocker.handle_blocking(ctx, px_config) with open('./px_blocking_messages/blocking.txt', 'r') as myfile: blocking_message = myfile.read() @@ -35,14 +44,16 @@ def test_handle_ratelimit(self): px_blocker = PXBlocker() vid = 'bf619be8-94be-458a-b6b1-ee81f154c282' px_uuid = '8712cef7-bcfa-4bb6-ae99-868025e1908a' - ctx = { - 'headers': {'Accept': 'text/html'}, - 'vid': vid, - 'uuid': px_uuid, - 'block_action': 'r' - } - px_config = PXConfig({'app_id': 'PXfake_app_ip'}) - message, _, _ = px_blocker.handle_blocking(ctx, px_config) + config = PxConfig({'app_id': 'PXfake_app_id'}) + ctx = PxContext({'PATH_INFO': '/fake_app_id/init.js', + 'HTTP_X_FORWARDED_FOR': '127.0.0.1', + 'ip': '127.0.0.1', + 'HTTP_ACCEPT': 'text/html'}, + config) + ctx.vid = vid + ctx.uuid = px_uuid + ctx.block_action = 'r' + message, _, _ = px_blocker.handle_blocking(ctx, config) blocking_message = None with open('./px_blocking_messages/ratelimit.txt', 'r') as myfile: blocking_message = myfile.read() @@ -52,15 +63,18 @@ def test_handle_challenge(self): px_blocker = PXBlocker() vid = 'bf619be8-94be-458a-b6b1-ee81f154c282' px_uuid = '8712cef7-bcfa-4bb6-ae99-868025e1908a' - ctx = { - 'headers': {'Accept': 'text/html'}, - 'vid': vid, - 'uuid': px_uuid, - 'block_action': 'j', - 'block_action_data': 'Bla' - } - px_config = PXConfig({'app_id': 'PXfake_app_ip'}) - message, _, _ = px_blocker.handle_blocking(ctx, px_config) + config = PxConfig({'app_id': 'PXfake_app_id'}) + ctx = PxContext({'PATH_INFO': '/fake_app_id/init.js', + 'HTTP_X_FORWARDED_FOR': '127.0.0.1', + 'ip': '127.0.0.1', + 'HTTP_ACCEPT': 'text/html'}, + config) + ctx.vid = vid + ctx.uuid = px_uuid + ctx.block_action = 'j' + ctx.block_action_data = 'Bla' + + message, _, _ = px_blocker.handle_blocking(ctx, config) blocking_message = 'Bla' self.assertEqual(message, blocking_message) @@ -68,27 +82,29 @@ def test_prepare_properties(self): px_blocker = PXBlocker() vid = 'bf619be8-94be-458a-b6b1-ee81f154c282' px_uuid = '8712cef7-bcfa-4bb6-ae99-868025e1908a' - ctx = { - 'headers': {'Accept': 'text/html'}, - 'vid': vid, - 'uuid': px_uuid, - } - px_config = PXConfig({'app_id': 'PXfake_app_ip'}) - message = px_blocker.prepare_properties(ctx, px_config) - expected_message = {'blockScript': '/fake_app_ip/captcha/captcha.js?a=None&u=8712cef7-bcfa-4bb6-ae99-868025e1908a&v=bf619be8-94be-458a-b6b1-ee81f154c282&m=0', + config = PxConfig({'app_id': 'PXfake_app_id'}) + ctx = PxContext({'PATH_INFO': '/fake_app_id/xhr', + 'HTTP_X_FORWARDED_FOR': '127.0.0.1', + 'ip': '127.0.0.1', + 'HTTP_ACCEPT': 'text/html'}, + config) + ctx.vid = vid + ctx.uuid = px_uuid + message = px_blocker.prepare_properties(ctx, config) + expected_message = {'blockScript': '/fake_app_id/captcha/captcha.js?a=&u=8712cef7-bcfa-4bb6-ae99-868025e1908a&v=bf619be8-94be-458a-b6b1-ee81f154c282&m=0', 'vid': 'bf619be8-94be-458a-b6b1-ee81f154c282', 'jsRef': '', - 'hostUrl': '/fake_app_ip/xhr', + 'hostUrl': '/fake_app_id/xhr', 'customLogo': None, - 'appId': 'PXfake_app_ip', + 'appId': 'PXfake_app_id', 'uuid': '8712cef7-bcfa-4bb6-ae99-868025e1908a', 'logoVisibility': 'hidden', - 'jsClientSrc': '/fake_app_ip/init.js', + 'jsClientSrc': '/fake_app_id/init.js', 'firstPartyEnabled': 'true', 'refId': '8712cef7-bcfa-4bb6-ae99-868025e1908a', 'cssRef': ''} self.assertDictEqual(message, expected_message) - expected_message['blockScript'] = '/fake_app/captcha/captcha.js?a=None&u=8712cef7-bcfa-4bb6-ae99-868025e1908a&v=bf619be8-94be-458a-b6b1-ee81f154c282&m=0' + expected_message['blockScript'] = '/fake_app/captcha/captcha.js?a=&u=8712cef7-bcfa-4bb6-ae99-868025e1908a&v=bf619be8-94be-458a-b6b1-ee81f154c282&m=0' self.assertNotEqual(message, expected_message) diff --git a/tests/px_blocking_messages/blocking.txt b/tests/px_blocking_messages/blocking.txt index b42ba1b..21071c1 100644 --- a/tests/px_blocking_messages/blocking.txt +++ b/tests/px_blocking_messages/blocking.txt @@ -150,14 +150,14 @@ ' - captcha_code = '' - body_start = ' You have been blocked!' - body_captcha = '

' - px_snippet = '' - body_end = '
Block Reference: #' + uuid + '
' - - print 'user id: ' + vid + ' blocked with score: ' + str(block_score) + ' ref: #' + uuid - if captcha: - custom_block_page = html_head + captcha_code + body_start + body_captcha + px_snippet + body_end - else: - custom_block_page = html_head + body_start + px_snippet + body_end - - start_response("403 Forbidden", [('Content-Type', 'text/html')]) - return [str(custom_block_page)] - - -px_config = { - 'app_id': 'PX_APP_ID', - 'cookie_key': 'PX_COOKIE_KEY', - 'auth_token': 'PX_AUTH_TOKEN', - 'blocking_score': 70, - 'debug_mode': True, - 'ip_handler': ip_handler, - 'captcha_enabled': True, - 'custom_block_handler': custom_block_handler, - 'module_mode': 'active_blocking' -} - -application = get_wsgi_application() -application = PerimeterX(application, px_config) diff --git a/examples/gae.py b/examples/gae.py deleted file mode 100644 index 9904271..0000000 --- a/examples/gae.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -PerimeterX example app implemented in a google app engine based environment -""" - -import webapp2 -from perimeterx.middleware import PerimeterX - - -class MainPage(webapp2.RequestHandler): - def get(self): - index_html = open('index.html').read() - self.response.out.write(index_html) - - -def ip_handler(environ): - for key in environ.keys(): - if key == 'HTTP_X_FORWARDED_FOR': - xff = environ[key].split(' ')[1] - return xff - return '1.2.3.4' - - -def custom_block_handler(ctx, start_response): - uuid = ctx.get('uuid') - block_score = ctx.get('risk_score') - vid = ctx.get('vid') - app_id = 'PX_APP_ID' - captcha = True - html_head = '' - captcha_code = '' - - body_start = ' You have been blocked!' - body_captcha = '

' - px_snippet = '' - body_end = '
Block Reference: #' + uuid + '
' - - print 'user id: ' + vid + ' blocked with score: ' + str(block_score) + ' ref: #' + uuid - if captcha: - custom_block_page = html_head + captcha_code + body_start + body_captcha + px_snippet + body_end - else: - custom_block_page = html_head + body_start + px_snippet + body_end - - start_response("403 Forbidden", [('Content-Type', 'text/html')]) - return [str(custom_block_page)] - - -px_config = { - 'app_id': 'PX_APP_ID', - 'cookie_key': 'PX_COOKIE_KEY', - 'auth_token': 'PX_AUTH_TOKEN', - 'blocking_score': 70, - 'debug_mode': True, - 'ip_handler': ip_handler, - 'captcha_enabled': True, - 'custom_block_handler': custom_block_handler, - 'module_mode': 'active_blocking' -} - -app = webapp2.WSGIApplication([ - ('/', MainPage), -], debug=True) -app = PerimeterX(app, px_config) diff --git a/perimeterx/__init__.py b/perimeterx/__init__.py index fc7e88d..4a6cf4f 100644 --- a/perimeterx/__init__.py +++ b/perimeterx/__init__.py @@ -1,2 +1,3 @@ __author__ = 'bend' __copyright__ = 'Copyright PerimeterX, Inc.' +import perimeterx diff --git a/requirements.txt b/requirements.txt index b8d57dd..dea6cd4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ pystache==0.5.4 +mock==2.0.0 requests==2.20.1 -setuptools==40.6.2 requests_mock==1.5.2 +setuptools==40.6.2 pycrypto==2.6.1 -webapp2==2.5.2 pylint diff --git a/setup.py b/setup.py index 179045f..5b475c7 100644 --- a/setup.py +++ b/setup.py @@ -3,14 +3,18 @@ from setuptools import setup setup(name='perimeterx-python-wsgi', - version='v1.0.3', + packages='perimeterx-python-wsgi', + version='v2.0.0', + license='MIT', description='PerimeterX WSGI middleware', author='Ben Diamant', author_email='ben@perimeterx.com', url='https://github.com/PerimeterX/perimeterx-python-wsgi', - download_url='https://github.com/PerimeterX/perimeterx-python-wsgi/tarball/v1.0.2', + download_url='https://github.com/PerimeterX/perimeterx-python-wsgi/tarball/v2.0.0', package_dir={'perimeterx': 'perimeterx'}, install_requires=[ - "pystache", 'Crypto' - ] - ) + "pystache==0.5.4", 'requests==2.20.1', 'setuptools==40.6.2', 'requests_mock==1.5.2', + 'pycrypto==2.6.1', 'mock==2.0.0', 'pylint'], + classifiers=['Intended Audience :: Developers', + 'Programming Language :: Python :: 2.7'],) + From 760e71b52fb6dad33d6ed8908356cd7510ca7421 Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Mon, 3 Dec 2018 15:37:47 +0200 Subject: [PATCH 33/35] Fixed version declartion in setup.py --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5b475c7..ed7ecc8 100644 --- a/setup.py +++ b/setup.py @@ -2,15 +2,16 @@ from setuptools import setup +version = 'v2.0.0' setup(name='perimeterx-python-wsgi', packages='perimeterx-python-wsgi', - version='v2.0.0', + version=version, license='MIT', description='PerimeterX WSGI middleware', author='Ben Diamant', author_email='ben@perimeterx.com', url='https://github.com/PerimeterX/perimeterx-python-wsgi', - download_url='https://github.com/PerimeterX/perimeterx-python-wsgi/tarball/v2.0.0', + download_url='https://github.com/PerimeterX/perimeterx-python-wsgi/tarball/' + version, package_dir={'perimeterx': 'perimeterx'}, install_requires=[ "pystache==0.5.4", 'requests==2.20.1', 'setuptools==40.6.2', 'requests_mock==1.5.2', From cdfb89b78ec5a9ffdd6732556490510b8c592c0c Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Mon, 3 Dec 2018 15:47:39 +0200 Subject: [PATCH 34/35] Removed packages from setup.py --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index ed7ecc8..6b06b37 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,6 @@ version = 'v2.0.0' setup(name='perimeterx-python-wsgi', - packages='perimeterx-python-wsgi', version=version, license='MIT', description='PerimeterX WSGI middleware', @@ -17,5 +16,5 @@ "pystache==0.5.4", 'requests==2.20.1', 'setuptools==40.6.2', 'requests_mock==1.5.2', 'pycrypto==2.6.1', 'mock==2.0.0', 'pylint'], classifiers=['Intended Audience :: Developers', - 'Programming Language :: Python :: 2.7'],) + 'Programming Language :: Python :: 2.7']) From ba1a428700bb5a749fe373497fd93a54fed0f8a7 Mon Sep 17 00:00:00 2001 From: Alex Bluvstein Date: Mon, 3 Dec 2018 16:22:00 +0200 Subject: [PATCH 35/35] Link to pypi --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1a24f6e..51a0f85 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![image](https://s.perimeterx.net/logo.png) [PerimeterX](http://www.perimeterx.com) Python Middleware ============================================================= -> Latest stable version: [v2.0.0](link to package) +> Latest stable version: [v2.0.0](https://pypi.org/project/perimeterx-python-wsgi/) Table of Contents ----------------- - [Installation](#installation)