diff --git a/perimeterx/middleware.py b/perimeterx/middleware.py index 389aaa0..2e016ed 100644 --- a/perimeterx/middleware.py +++ b/perimeterx/middleware.py @@ -6,7 +6,7 @@ import px_api import px_constants import px_utils -from px_proxy import PXProxy +from perimeterx.px_proxy_handler import PXProxy from px_config import PXConfig @@ -29,11 +29,10 @@ def __init__(self, app, config=None): logger.error('PX Cookie Key is missing') raise ValueError('PX Cookie Key is missing') self.reverse_proxy_prefix = px_config.app_id[2:].lower() - if px_config.custom_request_handler: - self.handle_verification = px_config.custom_request_handler.__get__(self, PerimeterX) self._PXBlocker = px_blocker.PXBlocker() self._config = px_config - px_httpc.init(px_config) + px_activities_client.init_activities_configuration(px_config) + px_activities_client.send_enforcer_telemetry_activity(config=px_config, update_reason='initial_config') def __call__(self, environ, start_response): return self._verify(environ, start_response) @@ -46,7 +45,7 @@ def _verify(self, environ, start_response): uri = ctx.get('uri') px_proxy = PXProxy(config) if px_proxy.should_reverse_request(uri): - return px_proxy.handle_reverse_request(self.config, ctx, start_response) + return px_proxy.handle_reverse_request(self.config, ctx, start_response, environ) if px_utils.is_static_file(ctx): logger.debug('Filter static file request. uri: ' + uri) return self.app(environ, start_response) @@ -63,33 +62,46 @@ def _verify(self, environ, start_response): # Server-to-Server verification fallback if not px_api.verify(ctx, self.config): return self.app(environ, start_response) - if config.custom_request_handler: - return config.custom_request_handler(ctx, self.config, environ, start_response) return self.handle_verification(ctx, self.config, environ, start_response) except: logger.error("Cought exception, passing request") - self.pass_traffic(environ, start_response, ctx) + self.pass_traffic(ctx) + return self.app(environ, start_response) def handle_verification(self, ctx, config, environ, start_response): - score = ctx.get('risk_score', -1) - + score = ctx.get('score', -1) + result = None + headers = None + status = None + pass_request = True if score < config.blocking_score: - return self.pass_traffic(environ, start_response, ctx) - - if config.custom_block_handler: - px_activities_client.send_block_activity(ctx, config) - return config.custom_block_handler(ctx, start_response) - elif config.module_mode == px_constants.MODULE_MODE_BLOCKING: - return self.px_blocker.handle_blocking(ctx=ctx, config=config, start_response=start_response) + self.pass_traffic(ctx) else: - return self.pass_traffic(environ, start_response, ctx) + pass_request = False + self.block_traffic(ctx) + + if config.additional_activity_handler: + config.additional_activity_handler(ctx, config) + + if config.module_mode == px_constants.MODULE_MODE_BLOCKING and result is None and not pass_request: + result, headers, status = self.px_blocker.handle_blocking(ctx=ctx, config=config) + if config.custom_request_handler: + custom_body, custom_headers, custom_status = config.custom_request_handler(ctx, self.config, environ) + if (custom_body is not None): + start_response(custom_status, custom_headers) + return custom_body + + if headers is not None: + start_response(status, headers) + return result + else: + return self.app(environ, start_response) + + def pass_traffic(self, ctx): + px_activities_client.send_page_requested_activity( ctx, self.config) - def pass_traffic(self, environ, start_response, ctx): - details = {} - if ctx.get('decoded_cookie', ''): - details = {"px_cookie": ctx['decoded_cookie']} - px_activities_client.send_to_perimeterx(px_constants.PAGE_REQUESTED_ACTIVITY, ctx, self.config, details) - return self.app(environ, start_response) + def block_traffic(self, ctx): + px_activities_client.send_block_activity(ctx, self.config) @property def config(self): @@ -98,6 +110,3 @@ def config(self): @property def px_blocker(self): return self._PXBlocker - - - diff --git a/perimeterx/px_activities_client.py b/perimeterx/px_activities_client.py index 9ac6b60..4037464 100644 --- a/perimeterx/px_activities_client.py +++ b/perimeterx/px_activities_client.py @@ -3,36 +3,41 @@ import threading import traceback, sys import px_constants +import socket +import json ACTIVITIES_BUFFER = [] CONFIG = {} +def init_activities_configuration(config): + global CONFIG + CONFIG = config + t1 = threading.Thread(target=send_activities) + t1.daemon = True + t1.start() def send_activities(): global ACTIVITIES_BUFFER + default_headers = { + 'Authorization': 'Bearer ' + CONFIG.auth_token, + 'Content-Type': 'application/json' + } + full_url = CONFIG.server_host + px_constants.API_ACTIVITIES while True: if len(ACTIVITIES_BUFFER) > 0: chunk = ACTIVITIES_BUFFER[:10] ACTIVITIES_BUFFER = ACTIVITIES_BUFFER[10:] - px_httpc.send('/api/v1/collector/s2s', chunk, CONFIG) + px_httpc.send(full_url=full_url, body=json.dumps(chunk), headers=default_headers, config=CONFIG, method='POST') time.sleep(1) -t1 = threading.Thread(target=send_activities) -t1.daemon = True -t1.start() - def send_to_perimeterx(activity_type, ctx, config, detail): - global CONFIG try: if activity_type == 'page_requested' and not config.send_page_activities: print 'Page activities disabled in config - skipping.' return - if not CONFIG: - CONFIG = config - _details = { 'http_method': ctx.get('http_method', ''), 'http_version': ctx.get('http_version', ''), @@ -72,5 +77,36 @@ def send_block_activity(ctx, config): #'cookie_origin':, 'block_action': ctx.get('block_action', ''), 'module_version': px_constants.MODULE_VERSION, - 'simulated_block': config.monitor_mode is 0, + 'simulated_block': config.module_mode is px_constants.MODULE_MODE_MONITORING, }) + +def send_page_requested_activity(ctx, config): + details = {} + if ctx.get('decoded_cookie', ''): + details = {"px_cookie": ctx['decoded_cookie']} + send_to_perimeterx(px_constants.PAGE_REQUESTED_ACTIVITY, ctx, config, details) + +def send_enforcer_telemetry_activity(config, update_reason): + details = { + 'enforcer_configs': config.telemetry_config, + 'node_name': socket.gethostname(), + 'os_name': sys.platform, + 'update_reason': update_reason, + 'module_version': config.module_version + } + body = { + 'type': px_constants.TELEMETRY_ACTIVITY, + 'timestamp': time.time(), + 'px_app_id': config.app_id, + 'details': details + } + headers = { + 'Authorization': 'Bearer ' + config.auth_token, + 'Content-Type': 'application/json' + } + config.logger.debug('Sending telemetry activity to PerimeterX servers') + px_httpc.send(full_url=config.server_host + px_constants.API_ENFORCER_TELEMETRY, body=json.dumps(body), + headers=headers, config=config, method='POST') + + + diff --git a/perimeterx/px_api.py b/perimeterx/px_api.py index 9ed3b47..efb7f83 100644 --- a/perimeterx/px_api.py +++ b/perimeterx/px_api.py @@ -1,14 +1,30 @@ -import sys import px_httpc import time import px_constants +import json +import re - +custom_params = { + 'custom_param1': '', + 'custom_param2': '', + 'custom_param3': '', + 'custom_param4': '', + 'custom_param5': '', + 'custom_param6': '', + 'custom_param7': '', + 'custom_param8': '', + 'custom_param9': '', + 'custom_param10': '' +} def send_risk_request(ctx, config): body = prepare_risk_body(ctx, config) - return px_httpc.send(px_constants.API_RISK, body, config) - + default_headers = { + 'Authorization': 'Bearer ' + config.auth_token, + 'Content-Type': 'application/json' + } + response = px_httpc.send(full_url=config.server_host + px_constants.API_RISK,body=json.dumps(body),config=config, headers=default_headers,method='POST') + return json.loads(response.content) def verify(ctx, config): logger = config.logger @@ -26,15 +42,16 @@ def verify(ctx, config): ctx['block_action'] = response['action'] ctx['risk_rtt'] = risk_rtt if score >= config.blocking_score: - logger.debug("PXVerify block score threshold reached, will initiate blocking") - ctx['block_reason'] = 's2s_high_score' - elif response['action'] is 'j' and response.get('action_data') is not None and response.get('action_data').get('body') is not None: - logger.debug("PXVerify received javascript challenge action") - ctx['block_action_data'] = response.get('action_data').get('body') - ctx['block_reason'] = 'challenge' - elif response['action'] is 'r': - logger.debug("PXVerify received javascript ratelimit action") - ctx['block_reason'] = 'exceeded_rate_limit' + if response['action'] == 'j' and response.get('action_data') is not None and response.get('action_data').get('body') is not None: + logger.debug("PXVerify received javascript challenge action") + ctx['block_action_data'] = response.get('action_data').get('body') + ctx['block_reason'] = 'challenge' + elif response['action'] is 'r': + logger.debug("PXVerify received javascript ratelimit action") + ctx['block_reason'] = 'exceeded_rate_limit' + else: + logger.debug("PXVerify block score threshold reached, will initiate blocking") + ctx['block_reason'] = 's2s_high_score' else: ctx['pass_reason'] = 's2s' @@ -70,6 +87,13 @@ def prepare_risk_body(ctx, config): } } + if config.enrich_custom_parameters: + risk_custom_params = config.enrich_custom_parameters(custom_params) + for param in risk_custom_params: + if re.match('^custom_param\d$',param) and risk_custom_params[param]: + body['additional'][param] = risk_custom_params[param] + + if ctx['s2s_call_reason'] == 'cookie_decryption_failed': logger.debug('attaching orig_cookie to request') body['additional']['px_cookie_orig'] = ctx.get('px_orig_cookie') diff --git a/perimeterx/px_blocker.py b/perimeterx/px_blocker.py index add48f9..c0759df 100644 --- a/perimeterx/px_blocker.py +++ b/perimeterx/px_blocker.py @@ -10,7 +10,7 @@ def __init__(self): self.ratelimit_rendered_page = self.mustache_renderer.render( px_template.get_template(px_constants.RATELIMIT_TEMPLATE), {}) - def handle_blocking(self, ctx, config, start_response): + def handle_blocking(self, ctx, config): action = ctx.get('block_action') status = '403 Forbidden' @@ -31,13 +31,12 @@ def handle_blocking(self, ctx, config, start_response): blocking_props = self.prepare_properties(ctx, config) blocking_response = self.mustache_renderer.render(px_template.get_template(px_constants.BLOCK_TEMPLATE), blocking_props) - start_response(status, headers) if is_json_response: blocking_response = json.dumps(blocking_props) - return str(blocking_response) + return str(blocking_response), headers, status def prepare_properties(self, ctx, config): - app_id = config.app_id.lower() + app_id = config.app_id vid = ctx.get('vid') if ctx.get('vid') is not None else '' uuid = ctx.get('uuid') custom_logo = config.custom_logo @@ -65,7 +64,7 @@ def prepare_properties(self, ctx, config): 'logoVisibility': 'visible' if custom_logo is not None else 'hidden', 'hostUrl': host_url, 'jsClientSrc': js_client_src, - 'firstPartyEnabled': config.first_party, + 'firstPartyEnabled': 'true' if config.first_party else 'false', 'blockScript': captcha_src } diff --git a/perimeterx/px_config.py b/perimeterx/px_config.py index fb6b98c..247859e 100644 --- a/perimeterx/px_config.py +++ b/perimeterx/px_config.py @@ -1,4 +1,5 @@ import px_constants +import json from px_logger import Logger @@ -7,7 +8,8 @@ def __init__(self, config_dict): app_id = config_dict.get('app_id') debug_mode = config_dict.get('debug_mode', False) module_mode = config_dict.get('module_mode', px_constants.MODULE_MODE_MONITORING) - self._app_id = app_id + custom_logo = config_dict.get('custom_logo', None) + self._px_app_id = app_id self._blocking_score = config_dict.get('blocking_score', 100) self._debug_mode = debug_mode self._module_version = config_dict.get('module_version', px_constants.MODULE_VERSION) @@ -18,24 +20,30 @@ def __init__(self, config_dict): self._encryption_enabled = config_dict.get('encryption_enabled', True) self._sensitive_headers = config_dict.get('sensitive_headers', ['cookie', 'cookies']) self._send_page_activities = config_dict.get('send_page_activities', True) - self._api_timeout = config_dict.get('api_timeout', 500) - self._custom_logo = config_dict.get('custom_logo', '') + self._api_timeout_ms = config_dict.get('api_timeout', 500) + self._custom_logo = custom_logo self._css_ref = config_dict.get('_custom_logo', '') self._js_ref = config_dict.get('js_ref', '') self._is_mobile = config_dict.get('is_mobile', False) + self._monitor_mode = 0 if module_mode is px_constants.MODULE_MODE_MONITORING else 1 self._module_enabled = config_dict.get('module_enabled', True) - self._cookie_key = config_dict.get('cookie_key', None) self._auth_token = config_dict.get('auth_token', None) self._is_mobile = config_dict.get('is_mobile', False) self._first_party = config_dict.get('first_party', True) self._first_party_xhr_enabled = config_dict.get('first_party_xhr_enabled', True) - self._logger = Logger(debug_mode) self._ip_headers = config_dict.get('ip_headers', []) self._proxy_url = config_dict.get('proxy_url', None) self._max_buffer_len = config_dict.get('max_buffer_len', 30) self._sensitive_routes = config_dict.get('sensitive_routes', []) self._whitelist_routes = config_dict.get('whitelist_routes', []) - self.instantiate_user_defined_handlers(config_dict) + self._block_html = 'BLOCK' + self._logo_visibility = 'visible' if custom_logo is not None else 'hidden' + self._telemetry_config = self.__create_telemetry_config() + + self._auth_token = config_dict.get('auth_token', None) + self._cookie_key = config_dict.get('cookie_key', None) + self.__instantiate_user_defined_handlers(config_dict) + self._logger = Logger(debug_mode) @property def module_mode(self): @@ -43,7 +51,7 @@ def module_mode(self): @property def app_id(self): - return self._app_id + return self._px_app_id @property def logger(self): @@ -63,7 +71,7 @@ def server_host(self): @property def api_timeout(self): - return self._api_timeout + return self._api_timeout_ms / 1000.000 @property def module_enabled(self): @@ -85,10 +93,6 @@ def proxy_url(self): def custom_request_handler(self): return self._custom_request_handler - @property - def custom_block_handler(self): - return self._custom_block_handler - @property def blocking_score(self): return self._blocking_score @@ -141,14 +145,47 @@ def sensitive_routes(self): def whitelist_routes(self): return self._whitelist_routes + @property + def block_html(self): + return self._block_html - def instantiate_user_defined_handlers(self, config_dict): - self._custom_request_handler = self.set_handler('custom_request_handler', config_dict) - self._custom_block_handler = self.set_handler('custom_block_handler', config_dict) - self._get_user_ip = self.set_handler('get_user_ip', config_dict) - self._additional_activity_handler = self.set_handler('additional_activity_handler', config_dict) + @property + def logo_visibility(self): + return self._logo_visibility + @property + def additional_activity_handler(self): + return self._additional_activity_handler - def set_handler(self, function_name, config_dict): + @property + def debug_mode(self): + return self._debug_mode + + @property + def max_buffer_len(self): + return self._max_buffer_len + + @property + def telemetry_config(self): + return self._telemetry_config + + @property + def enrich_custom_parameters(self): + return self._enrich_custom_parameters + + def __instantiate_user_defined_handlers(self, config_dict): + self._custom_request_handler = self.__set_handler('custom_request_handler', config_dict) + self._get_user_ip = self.__set_handler('get_user_ip', config_dict) + self._additional_activity_handler = self.__set_handler('additional_activity_handler', config_dict) + self._enrich_custom_parameters = self.__set_handler('enrich_custom_parameters', config_dict) + + def __set_handler(self, function_name, config_dict): return config_dict.get(function_name) if config_dict.get(function_name) and callable( config_dict.get(function_name)) else None + + def __create_telemetry_config(self): + config = self.__dict__ + mutated_config = {} + for key, value in config.iteritems(): + mutated_config[key[1:].upper()] = value + return mutated_config diff --git a/perimeterx/px_constants.py b/perimeterx/px_constants.py index 648a3b5..95722ea 100644 --- a/perimeterx/px_constants.py +++ b/perimeterx/px_constants.py @@ -10,10 +10,10 @@ BLOCK_ACTION_CAPTCHA = 'b' BLOCK_ACTION_CHALLENGE = 'j' BLOCK_ACTION_RATE = 'r' -CLIENT_HOST = 'client.perimeterx.net' -CAPTCHA_HOST = 'captcha.px-cdn.net' -COLLECTOR_URL = 'collector-{}.perimeterx.net' -SERVER_URL = 'sapi-{}.perimeterx.net' +CLIENT_HOST = 'https://client.perimeterx.net' +CAPTCHA_HOST = 'https://captcha.px-cdn.net' +COLLECTOR_URL = 'https://collector-{}.perimeterx.net' +SERVER_URL = 'https://sapi-{}.perimeterx.net' CLIENT_FP_PATH = 'init.js' CAPTCHA_FP_PATH = 'captcha' XHR_FP_PATH = 'xhr' @@ -29,3 +29,6 @@ API_RISK = '/api/v3/risk' PAGE_REQUESTED_ACTIVITY = 'page_requested' BLOCK_ACTIVITY = 'block' +API_ENFORCER_TELEMETRY = '/api/v2/risk/telemetry' +API_ACTIVITIES = '/api/v1/collector/s2s' +TELEMETRY_ACTIVITY = 'enforcer_telemetry' diff --git a/perimeterx/px_context.py b/perimeterx/px_context.py index ed0f16e..e21da96 100644 --- a/perimeterx/px_context.py +++ b/perimeterx/px_context.py @@ -27,6 +27,9 @@ def build_context(environ, config): http_protocol = protocol_split[0].lower() + '://' if len(protocol_split) > 1: http_version = protocol_split[1] + if key == 'CONTENT_TYPE' or key == 'CONTENT_LENGTH': + headers['Content-type'.replace('_', '-')] = environ.get(key) + cookies = Cookie.SimpleCookie(environ.get('HTTP_COOKIE', '')) cookie_keys = cookies.keys() @@ -41,12 +44,12 @@ def build_context(environ, config): vid = cookies.get('_pxvid').value else: vid = '' - user_agent = headers.get('user-agent') + user_agent = environ.get('HTTP_USER_AGENT', '') uri = environ.get('PATH_INFO') or '' - full_url = http_protocol + headers.get('host') or environ.get('SERVER_NAME') or '' + uri + full_url = http_protocol + (headers.get('host') or environ.get('SERVER_NAME') or '') + uri hostname = headers.get('host') - sensitive_route = uri in config.sensitive_routes - whitelist_route = uri in config.whitelist_routes + sensitive_route = len(filter(lambda sensitive_route_item : uri.startswith(sensitive_route_item), config.sensitive_routes)) > 0 + whitelist_route = len(filter(lambda whitelist_route_item : uri.startswith(whitelist_route_item), config.whitelist_routes)) > 0 ctx = { 'headers': headers, 'http_method': http_method, @@ -63,18 +66,20 @@ def build_context(environ, config): 'query_params': environ['QUERY_STRING'], 'sensitive_route': sensitive_route, 'whitelist_route': whitelist_route, + 's2s_call_reason': 'none', + 'cookie_origin': 'cookie' } return ctx def extract_ip(config, environ): - ip = environ.get('HTTP_X_FORWARDED_FOR') + ip = environ.get('HTTP_X_FORWARDED_FOR') if environ.get('HTTP_X_FORWARDED_FOR') else environ.get('REMOTE_ADDR') ip_headers = config.ip_headers logger = config.logger - if not ip_headers: + if ip_headers: try: for ip_header in ip_headers: - ip_header_name = 'HTTP_' + ip_header.upper() + ip_header_name = 'HTTP_' + ip_header.replace('-', '_').upper() if environ.get(ip_header_name): return environ.get(ip_header_name) except: diff --git a/perimeterx/px_cookie_v1.py b/perimeterx/px_cookie_v1.py index fa5e4de..3c3cd4d 100644 --- a/perimeterx/px_cookie_v1.py +++ b/perimeterx/px_cookie_v1.py @@ -7,6 +7,7 @@ class PxCookieV1(PxCookie): def __init__(self, ctx, config): self._ctx = ctx self._config = config + self._logger = config.logger self.raw_cookie = ctx['px_cookies'].get(PREFIX_PX_COOKIE_V1, '') def get_score(self): diff --git a/perimeterx/px_cookie_v3.py b/perimeterx/px_cookie_v3.py index ab70e3a..9b4f8b0 100644 --- a/perimeterx/px_cookie_v3.py +++ b/perimeterx/px_cookie_v3.py @@ -8,8 +8,9 @@ def __init__(self, ctx, config): self._config = config self._logger = config.logger self._ctx = ctx + self.raw_cookie = '' spliced_cookie = self._ctx['px_cookies'].get(PREFIX_PX_COOKIE_V3, '').split(":", 1) - if spliced_cookie.count > 1: + if len(spliced_cookie) > 1: self.hmac = spliced_cookie[0] self.raw_cookie = spliced_cookie[1] diff --git a/perimeterx/px_cookie_validator.py b/perimeterx/px_cookie_validator.py index 4d30893..3a6a00a 100644 --- a/perimeterx/px_cookie_validator.py +++ b/perimeterx/px_cookie_validator.py @@ -45,7 +45,7 @@ def verify(ctx, config): logger.debug('Cookie expired') return False - if px_cookie.is_secured(): + if not px_cookie.is_secured(): logger.debug('Cookie validation failed') ctx['s2s_call_reason'] = 'cookie_validation_failed' return False diff --git a/perimeterx/px_httpc.py b/perimeterx/px_httpc.py index b34f56f..c31fbb6 100644 --- a/perimeterx/px_httpc.py +++ b/perimeterx/px_httpc.py @@ -1,51 +1,22 @@ -import httplib -import json import time +import requests -http_client = None - -def init(config): - global http_client - http_client = httplib.HTTPConnection(host=config.server_host, timeout=config.api_timeout) - - -def send(uri, body, config): - logger = config.logger - headers = { - 'Authorization': 'Bearer ' + config.auth_token, - 'Content-Type': 'application/json' - } - try: - start = time.time() - http_client.request('POST', uri, body=json.dumps(body), headers=headers) - r = http_client.getresponse() - if r.status != 200: - logger.error('error posting server to server call ' + r.reason) - return False - - logger.debug('Server call took ' + str(time.time() - start) + 'ms') - response_body = r.read() - return json.loads(response_body) - except httplib.HTTPException: - init(config) - return False - -def send_reverse(url, path, body, headers, config, method): +def send(full_url, body, headers, config, method): logger = config.logger try: start = time.time() - http_client = httplib.HTTPSConnection(url, timeout=config.api_timeout) - http_client.request(method, path, body, headers=headers) - response = http_client.getresponse() + if method == 'GET': + response = requests.get(url=full_url, headers=headers, timeout=config.api_timeout) + else: + response = requests.post(url=full_url, headers=headers, data=body, timeout=config.api_timeout) - if response.status >= 400: + if response.status_code >= 400: + logger.debug('PerimeterX server call failed') return False - logger.debug('Server call took ' + str(time.time() - start) + 'ms') + logger.debug('PerimeterX server call took ' + str(time.time() - start) + 'ms') return response - - except httplib.HTTPException: - init(config) + except requests.exceptions.RequestException as e: + logger.debug('Received RequestException, message: ' + e.message) return False - diff --git a/perimeterx/px_proxy.py b/perimeterx/px_proxy_handler.py similarity index 76% rename from perimeterx/px_proxy.py rename to perimeterx/px_proxy_handler.py index c9dac3f..f0169fb 100644 --- a/perimeterx/px_proxy.py +++ b/perimeterx/px_proxy_handler.py @@ -26,13 +26,13 @@ def should_reverse_request(self, uri): return True return False - def handle_reverse_request(self, config, ctx, start_response): + def handle_reverse_request(self, config, ctx, start_response, environ): uri = ctx.get('uri').lower() if uri.startswith(self.client_reverse_prefix): return self.send_reverse_client_request(config=config, context=ctx, start_response=start_response) if uri.startswith(self.xhr_reverse_prefix): - return self.send_reverse_xhr_request(config=config, context=ctx, start_response=start_response) + return self.send_reverse_xhr_request(config=config, context=ctx, start_response=start_response, body = environ['wsgi.input'].read(int(environ.get('CONTENT_LENGTH', '0')))) if uri.startswith(self.captcha_reverse_prefix): return self.send_reverse_captcha_request(config=config, context=ctx, start_response=start_response) @@ -46,18 +46,17 @@ def send_reverse_client_request(self, config, context, start_response): self._logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(),px_constants.CLIENT_HOST, client_request_uri)) headers = {'host': px_constants.CLIENT_HOST, - px_constants.FIRST_PARTY_HEADER: 1, + px_constants.FIRST_PARTY_HEADER: '1', px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) - response = px_httpc.send_reverse(url=px_constants.CLIENT_HOST, path=client_request_uri, body='', - headers=filtered_headers, config=config, method='GET') + response = px_httpc.send(full_url=px_constants.CLIENT_HOST + client_request_uri, body='', + headers=filtered_headers, config=config, method='GET') - headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) - start_response(str(response.status) + ' ' + response.reason, headers) - return response.read() + self.handle_proxy_response(response, start_response) + return response.content - def send_reverse_xhr_request(self, config, context, start_response): + def send_reverse_xhr_request(self, config, context, start_response, body): uri = context.get('uri') if not config.first_party or not config.first_party_xhr_enabled: body, content_type = self.return_default_response(uri) @@ -70,7 +69,7 @@ def send_reverse_xhr_request(self, config, context, start_response): host = config.collector_host headers = {'host': host, - px_constants.FIRST_PARTY_HEADER: 1, + px_constants.FIRST_PARTY_HEADER: '1', px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} if context.get('vid') is not None: @@ -79,18 +78,23 @@ def send_reverse_xhr_request(self, config, context, start_response): filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) self._logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, suffix_uri)) - response = px_httpc.send_reverse(url=host, path=suffix_uri, body='', - headers=filtered_headers, config=config, method=context.get('http_method')) + response = px_httpc.send(full_url=host + suffix_uri, body=body, + headers=filtered_headers, config=config, method=context.get('http_method')) - if response.status >= 400: + if response.status_code >= 400: body, content_type = self.return_default_response(uri) px_logger.Logger.debug('error reversing the http call ' + response.reason) start_response('200 OK', [content_type]) return body - response_headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) - start_response(str(response.status) + ' ' + response.reason, response_headers) - return response.read() + self.handle_proxy_response(response, start_response) + return response.content + def handle_proxy_response(self, response, start_response): + headers = [] + for header in response.headers: + if header.lower() not in hoppish: + headers.append((header, response.headers[header])) + start_response(str(response.status_code) + ' ' + response.reason, headers) def return_default_response(self, uri): if 'gif' in uri.lower(): @@ -111,16 +115,15 @@ def send_reverse_captcha_request(self, config, context, start_response): host = px_constants.CAPTCHA_HOST headers = {'host': px_constants.CAPTCHA_HOST, - px_constants.FIRST_PARTY_HEADER: 1, + px_constants.FIRST_PARTY_HEADER: '1', px_constants.ENFORCER_TRUE_IP_HEADER: context.get('ip')} filtered_headers = px_utils.handle_proxy_headers(context.get('headers'), context.get('ip')) filtered_headers = px_utils.merge_two_dicts(filtered_headers, headers) self._logger.debug('Forwarding request from {} to client at {}{}'.format(context.get('uri').lower(), host, uri)) - response = px_httpc.send_reverse(url=host, path=uri, body='', - headers=filtered_headers, config=config, method='GET') - headers = filter(lambda x: x[0] not in hoppish, response.getheaders()) - start_response(str(response.status) + ' ' + response.reason, headers) - return response.read() + response = px_httpc.send(full_url=host + uri, body='', + headers=filtered_headers, config=config, method='GET') + self.handle_proxy_response(response, start_response) + return response.content diff --git a/tests/px_activities_client.py b/tests/px_activities_client.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/px_api.py b/tests/px_api.py new file mode 100644 index 0000000..d7bc19f --- /dev/null +++ b/tests/px_api.py @@ -0,0 +1,20 @@ +import unittest +from perimeterx import px_api +from perimeterx.px_config import PXConfig + +class Test_PXApi(unittest.TestCase): + + def enrich_custom_parameters(self, params): + params['custom_param1'] = '1' + params['custom_param2'] = '5' + params['custom'] = '6' + return params + + def test_prepare_risk_body(self): + config = PXConfig({'app_id': 'app_id', 'enrich_custom_parameters': self.enrich_custom_parameters}) + ctx = {'headers': {}, 's2s_call_reason': 'no_cookie'} + body = px_api.prepare_risk_body(ctx, config) + self.assertEqual(body['additional'].get('custom_param1'), '1') + self.assertEqual(body['additional'].get('custom_param2'), '5') + self.assertFalse(body['additional'].get('custom')) + print diff --git a/tests/px_blocker.py b/tests/px_blocker.py new file mode 100644 index 0000000..9e9b9ee --- /dev/null +++ b/tests/px_blocker.py @@ -0,0 +1,97 @@ +from perimeterx.px_blocker import PXBlocker + + +import unittest +from perimeterx.px_config import PXConfig + + +class Test_PXBlocker(unittest.TestCase): + + def test_is_json_response(self): + px_blocker = PXBlocker() + ctx = { + 'headers': {'Accept': 'text/html'} + } + self.assertFalse(px_blocker.is_json_response(ctx)) + ctx['headers']['Accept'] = 'application/json' + self.assertTrue(px_blocker.is_json_response(ctx)) + + def test_handle_blocking(self): + px_blocker = PXBlocker() + vid = 'bf619be8-94be-458a-b6b1-ee81f154c282' + px_uuid = '8712cef7-bcfa-4bb6-ae99-868025e1908a' + ctx = { + 'headers': {'Accept': 'text/html'}, + 'vid': vid, + 'uuid': px_uuid + } + px_config = PXConfig({'app_id': 'PXfake_app_ip'}) + message, _, _ = px_blocker.handle_blocking(ctx, px_config) + with open('./px_blocking_messages/blocking.txt', 'r') as myfile: + blocking_message = myfile.read() + self.assertEqual(message, blocking_message) + + def test_handle_ratelimit(self): + px_blocker = PXBlocker() + vid = 'bf619be8-94be-458a-b6b1-ee81f154c282' + px_uuid = '8712cef7-bcfa-4bb6-ae99-868025e1908a' + ctx = { + 'headers': {'Accept': 'text/html'}, + 'vid': vid, + 'uuid': px_uuid, + 'block_action': 'r' + } + px_config = PXConfig({'app_id': 'PXfake_app_ip'}) + message, _, _ = px_blocker.handle_blocking(ctx, px_config) + blocking_message = None + with open('./px_blocking_messages/ratelimit.txt', 'r') as myfile: + blocking_message = myfile.read() + self.assertEqual(message, blocking_message) + + def test_handle_challenge(self): + px_blocker = PXBlocker() + vid = 'bf619be8-94be-458a-b6b1-ee81f154c282' + px_uuid = '8712cef7-bcfa-4bb6-ae99-868025e1908a' + ctx = { + 'headers': {'Accept': 'text/html'}, + 'vid': vid, + 'uuid': px_uuid, + 'block_action': 'j', + 'block_action_data': 'Bla' + } + px_config = PXConfig({'app_id': 'PXfake_app_ip'}) + message, _, _ = px_blocker.handle_blocking(ctx, px_config) + blocking_message = 'Bla' + self.assertEqual(message, blocking_message) + + def test_prepare_properties(self): + px_blocker = PXBlocker() + vid = 'bf619be8-94be-458a-b6b1-ee81f154c282' + px_uuid = '8712cef7-bcfa-4bb6-ae99-868025e1908a' + ctx = { + 'headers': {'Accept': 'text/html'}, + 'vid': vid, + 'uuid': px_uuid, + } + px_config = PXConfig({'app_id': 'PXfake_app_ip'}) + message = px_blocker.prepare_properties(ctx, px_config) + expected_message = {'blockScript': '/fake_app_ip/captcha/captcha.js?a=None&u=8712cef7-bcfa-4bb6-ae99-868025e1908a&v=bf619be8-94be-458a-b6b1-ee81f154c282&m=0', + 'vid': 'bf619be8-94be-458a-b6b1-ee81f154c282', + 'jsRef': '', + 'hostUrl': '/fake_app_ip/xhr', + 'customLogo': None, + 'appId': 'PXfake_app_ip', + 'uuid': '8712cef7-bcfa-4bb6-ae99-868025e1908a', + 'logoVisibility': 'hidden', + 'jsClientSrc': '/fake_app_ip/init.js', + 'firstPartyEnabled': 'true', + 'refId': '8712cef7-bcfa-4bb6-ae99-868025e1908a', + 'cssRef': ''} + self.assertDictEqual(message, expected_message) + expected_message['blockScript'] = '/fake_app/captcha/captcha.js?a=None&u=8712cef7-bcfa-4bb6-ae99-868025e1908a&v=bf619be8-94be-458a-b6b1-ee81f154c282&m=0' + self.assertNotEqual(message, expected_message) + + + + + diff --git a/tests/px_blocking_messages/blocking.txt b/tests/px_blocking_messages/blocking.txt new file mode 100644 index 0000000..b42ba1b --- /dev/null +++ b/tests/px_blocking_messages/blocking.txt @@ -0,0 +1,169 @@ + + + + + + Access to this page has been denied. + + + + + + +
+
+ +
+
+
+

Please verify you are a human

+
+
+
+
+ +
+
+

+ Access to this page has been denied because we believe you are using automation tools to browse the + website. +

+

+ This may happen as a result of the following: +

+
    +
  • + Javascript is disabled or blocked by an extension (ad blockers for example) +
    • +
  • + Your browser does not support cookies +
    • +
+

+ Please make sure that Javascript and cookies are enabled on your browser and that you are not blocking + them from loading. +

+

+ Reference ID: #8712cef7-bcfa-4bb6-ae99-868025e1908a +

+
+
+
+
+

+ Powered by + PerimeterX + , Inc. +

+
+
+
+ + + + + + + diff --git a/tests/px_blocking_messages/ratelimit.txt b/tests/px_blocking_messages/ratelimit.txt new file mode 100644 index 0000000..36fd393 --- /dev/null +++ b/tests/px_blocking_messages/ratelimit.txt @@ -0,0 +1,9 @@ + + + Too Many Requests + + +

Too Many Requests

+

Reached maximum requests limitation, try again soon.

+ + \ No newline at end of file diff --git a/tests/px_config.py b/tests/px_config.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/px_context.py b/tests/px_context.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/px_cookie.py b/tests/px_cookie.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/px_cookie_validator.py b/tests/px_cookie_validator.py new file mode 100644 index 0000000..8f984ab --- /dev/null +++ b/tests/px_cookie_validator.py @@ -0,0 +1,66 @@ +from perimeterx import px_cookie_validator +import unittest +from perimeterx.px_config import PXConfig + + +class Test_PXCookieValidator(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.cookie_key = 'Pyth0nS3crE7K3Y' + + def test_verify_no_cookie(self): + config = PXConfig({'app_id': 'app_id'}) + ctx = {'px_cookies': {}} + verified = px_cookie_validator.verify(ctx, config) + self.assertFalse(verified) + self.assertEqual('no_cookie', ctx['s2s_call_reason']) + + def test_verify_valid_cookie(self): + config = PXConfig({'app_id': 'app_id', + 'cookie_key': self.cookie_key}) + ctx = {'px_cookies': { + '_px3': 'bd078865fa9627f626d6f7d6828ab595028d2c0974065ab6f6c5a9f80c4593cd:OCIluokZHHvqrWyu8zrWSH8Vu7AefCjrd4CMx/NXsX58LzeV40EZIlPG4gsNMoAYzH88s/GoZwv+DpQa76C21A==:1000:zwT+Rht/YGDNWKkzHtJAB7IiI00u4fOePL/3xWMs1nZ93lzW1XvAMGR2hLlHBmOv8O0CpylEQOZZTK1uQMls6O28Y8aQnTo5DETLkrbhpwCVeNjOcf8GVKTckITwuHfXbEcfHbdtb68s1+jHv1+vt/w/6HZqTzanaIsvFVp8vmA='}} + verified = px_cookie_validator.verify(ctx, config) + self.assertTrue(verified) + self.assertEqual(None, ctx.get('s2s_call_reason')) + + def test_verify_decryption_failed(self): + config = PXConfig({'app_id': 'app_id', + 'cookie_key': self.cookie_key}) + ctx = {'px_cookies': { + '_px3': '774958bcc233ea1a876b92ababf47086d8a4d95165bbd6f98b55d7e61afd2a05:ow3Er5dskpt8ZZ11CRiDMAueEi3ozJTqMBnYzsSM7/8vHTDA0so6ekhruiTrXa/taZINotR5PnTo78D5zM2pWw==:1000:uQ3Tdt7D3mSO5CuHDis3GgrnkGMC+XAghbHuNOE9x4H57RAmtxkTcNQ1DaqL8rx79bHl0iPVYlOcRmRgDiBCUoizBdUCjsSIplofPBLIl8WpfHDDtpxPKzz9I2rUEbFgfhFjiTY3rPGob2PUvTsDXTfPUeHnzKqbNTO8z7H6irFnUE='}} + verified = px_cookie_validator.verify(ctx, config) + self.assertFalse(verified) + self.assertEqual('cookie_decryption_failed', ctx.get('s2s_call_reason')) + + def test_verify_cookie_high_score(self): + config = PXConfig({'app_id': 'app_id', + 'cookie_key': self.cookie_key}) + ctx = {'px_cookies': { + '_px3': 'bf46ceff75278ae166f376cbf741a7639060581035dd4e93641892c905dd0d67:EGFGcwQ2rum7KRmQCeSXBAUt1+25mj2DFJYi7KJkEliF3cBspdXtD2X03Csv8N8B6S5Bte/4ccCcETkBNDVxTw==:1000:x9x+oI6BISFhlKEERpf8HpZD2zXBCW9lzVfuRURHaAnbaMnpii+XjPEd7a7EGGUSMch5ramy3y+KOxyuX3F+LbGYwvn3OJb+u40zU+ixT1w5N15QltX+nBMhC7izC1l8QtgMuG/f3Nts5ebnec9j2V7LS5Y1/5b73rd9s7AMnug='}} + verified = px_cookie_validator.verify(ctx, config) + self.assertTrue(verified) + self.assertEqual(None, ctx.get('s2s_call_reason')) + + def test_verify_hmac_validation(self): + config = PXConfig({'app_id': 'app_id', + 'cookie_key': self.cookie_key}) + ctx = {'px_cookies': { + '_px3': '774958bcc232343ea1a876b92ababf47086d8a4d95165bbd6f98b55d7e61afd2a05:ow3Er5dskpt8ZZ11CRiDMAueEi3ozJTqMBnYzsSM7/8vHTDA0so6ekhruiTrXa/taZINotR5PnTo78D5zM2pWw==:1000:uQ3Tdt7D3mSO5CuHDis3GgrnkGMC+XAghbHuNOE9x4H57RAmtxkTcNQ1DaqL8rx79bHl0iPVYlOcRmRgDiBCUoizBdUCjsSIplofPBLIl8WpfHDDtpxPKzz9I2rUEbFFjiTY3rPGob2PUvTsDXTfPUeHnzKqbNTO8z7H6irFnUE='}} + verified = px_cookie_validator.verify(ctx, config) + self.assertFalse(verified) + self.assertEqual(None, ctx.get('cookie_validation_failed')) + + def test_verify_expired_cookie(self): + config = PXConfig({'app_id': 'app_id', + 'cookie_key': self.cookie_key}) + ctx = {'px_cookies': { + '_px3': '0d67bdf4a58c524b55b9cf0f703e4f0f3cbe23a10bd2671530d3c7e0cfa509eb:HOiYSw11ICB2A+HYx+C+l5Naxcl7hMeEo67QNghCQByyHlhWZT571ZKfqV98JFWg7TvbV9QtlrQtXakPYeIEjQ==:1000:+kuXS/iJUoEqrm8Fo4K0cTebsc4YQZu+f5bRGX0lC1T+l0g1gzRUuKiCtWTar28Y0wjch1ZQvkNy523Pxr07agVi/RL0SUktmEl59qGor+m4FLewZBVdcgx/Ya9kU0riis98AAR0zdTpTtoN5wpNbmztIpOZ0YejeD0Esk3vagU='}} + verified = px_cookie_validator.verify(ctx, config) + self.assertFalse(verified) + self.assertEqual(None, ctx.get('cookie_expired')) + + + + diff --git a/tests/px_httpc.py b/tests/px_httpc.py new file mode 100644 index 0000000..16c0cc6 --- /dev/null +++ b/tests/px_httpc.py @@ -0,0 +1,20 @@ +from perimeterx import px_httpc +import unittest +from mock import MagicMock,patch +from perimeterx.px_config import PXConfig +import httplib + + +class Test_PXHTTPC(unittest.TestCase): + + def test_send(self): + # px_config = PXConfig({'app_id': 'fake_app_id', + # 'auth_token': 'fake_auth_token'}) + # http_client = httplib.HTTPConnection(host='host', timeout=1) + # from httplib2 import Response + # http_client.request = MagicMock(return_value= Response({'status':'200'})) + # with patch('perimeterx.px_httpc.httplib', return_value=http_client): + # message = px_httpc.send('uri', 'body', px_config) + + + print 'f' \ No newline at end of file diff --git a/tests/px_proxy_handler.py b/tests/px_proxy_handler.py new file mode 100644 index 0000000..8e14db5 --- /dev/null +++ b/tests/px_proxy_handler.py @@ -0,0 +1,33 @@ +import unittest + +from httplib import HTTPResponse + +from perimeterx.px_proxy_handler import PXProxy +from perimeterx.px_config import PXConfig +from mock import MagicMock,patch + +class Test_PXProxy(unittest.TestCase): + + def test_should_reverse_request(self): + config = PXConfig({'app_id': 'PXfake_app_id'}) + px_proxy = PXProxy(config) + should_reverse = px_proxy.should_reverse_request('/fake_app_id/init.js') + self.assertTrue(should_reverse) + should_reverse = px_proxy.should_reverse_request('/fake_app_id/xhr') + self.assertTrue(should_reverse) + should_reverse = px_proxy.should_reverse_request('/fake_app_id/captcha') + self.assertTrue(should_reverse) + + # def test_send_reverse_client_request(self): + # content = 'client js content' + # config = PXConfig({'app_id': 'PXfake_app_id'}) + # ctx = {'uri': '/fake_app_id/init.js', 'headers': {'X-FORWARDED-FOR': '127.0.0.1'}} + # fake = HttpResponse(content=content, status=200, reason='OK', content_type='text/html') + # px_proxy = PXProxy(config) + # config = PXConfig({'app_id': 'PXfake_app_id'}) + # with patch('perimeterx.px_httpc.send_https', return_value=fake): + # result = px_proxy.handle_reverse_request(config=config, ctx=ctx, start_response= lambda x: x) + # print '' + + + diff --git a/tests/px_utils.py b/tests/px_utils.py new file mode 100644 index 0000000..1d429c7 --- /dev/null +++ b/tests/px_utils.py @@ -0,0 +1,26 @@ +from perimeterx import px_utils +import unittest +from perimeterx import px_constants + + +class Test_PXUtils(unittest.TestCase): + + def test_merge_two_dicts(self): + dict1 = {'a': '1'} + dict2 = {'b': '2'} + merged_dict = px_utils.merge_two_dicts(dict1, dict2) + self.assertDictEqual(merged_dict, {'a': '1', 'b': '2'}) + + def test_handle_proxy_headers(self): + headers_sample = {'ddd': 'not_proxy_url', px_constants.FIRST_PARTY_FORWARDED_FOR: 'proxy_url'} + headers_sample = px_utils.handle_proxy_headers(headers_sample, '127.0.0.1') + self.assertEqual(headers_sample[px_constants.FIRST_PARTY_FORWARDED_FOR], '127.0.0.1') + headers_sample = {'ddd': 'not_proxy_url'} + headers_sample = px_utils.handle_proxy_headers(headers_sample, '127.0.0.1') + self.assertEqual(headers_sample[px_constants.FIRST_PARTY_FORWARDED_FOR], '127.0.0.1') + + def test_is_static_file(self): + ctx = {'uri': '/sample.css'} + self.assertTrue(px_utils.is_static_file(ctx)) + ctx = {'uri': '/sample.html'} + self.assertFalse(px_utils.is_static_file(ctx))