From 99c89f2239f8baef761e152e0126ec137ae98e19 Mon Sep 17 00:00:00 2001 From: Maksym Kucherov Date: Fri, 7 Feb 2025 13:45:36 -0500 Subject: [PATCH 1/2] Revert "feat(4.5): add hardcoded evidence keys to `evidence_key_filter`" This reverts commit d2d938b997d6c4bcf10ec3bd9f4289828aafb54e. --- .../constants.py | 12 +----- .../devicedetection_onpremise.py | 43 ++++++++----------- .../tests/test_devicedetection.py | 37 +--------------- 3 files changed, 20 insertions(+), 72 deletions(-) diff --git a/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py b/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py index 704bc80be..d0709a61e 100644 --- a/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py +++ b/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py @@ -35,14 +35,4 @@ ITERATIONS_DESCRIPTION = ('The number of iterations carried out in order to ' 'find a match. This is the number of nodes in the ' 'graph which have been visited.') -METHOD_DESCRIPTION = 'The method used to determine the match result.' -HARDCODED_EVIDENCE = { - "GHEV": { - "COOKIE": "cookie.51d_gethighentropyvalues", - "QUERY": "query.51d_gethighentropyvalues", - }, - "SUA": { - "COOKIE": "cookie.51d_structureduseragent", - "QUERY": "query.51d_structureduseragent", - }, -} +METHOD_DESCRIPTION = 'The method used to determine the match result.' \ No newline at end of file diff --git a/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py b/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py index b4da1ba81..d60e021e5 100644 --- a/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py +++ b/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py @@ -74,7 +74,7 @@ def __init__( **kwargs ): """! - + Constructor for the DeviceDetection On Premise Engine. @type data_file_path: string @@ -91,7 +91,7 @@ def __init__( @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service @type max_matched_useragents_length : int @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false - @type update_matched_useragent: + @type update_matched_useragent: @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent @type drift: int @param drift: Set maximum drift in hash position to allow when processing HTTP headers @@ -108,7 +108,7 @@ def __init__( @type update_on_start : bool @param update_on_start : When this is set to true the datafile is updated / downloaded immediately on initialization. This is useful if no initial datafile is present. @type file_system_watcher: bool - @param file_system_watcher: whether to check the datafile's path for changes and update the connected FlowElement's data + @param file_system_watcher: whether to check the datafile's path for changes and update the connected FlowElement's data @type polling_interval: int @param polling_interval: How often to poll for updates to the datafile (minutes) @type update_time_maximum_randomisation : int @@ -120,7 +120,7 @@ def __init__( @type data_update_url: string @param data_update_url: base url for the datafile update service @type use_performance_graph: bool - @param use_performance_graph: True if the performance optimized graph should be used for processing + @param use_performance_graph: True if the performance optimized graph should be used for processing @type use_predictive_graph: bool @param use_predictive_graph: True if the predictive optimized graph should be used for processing @param data_update_use_url_formatter: This must be set to false to prevent the API from appending the query string parameters that are required when calling the 51Degrees Distributor service. @@ -133,11 +133,11 @@ def __init__( self.datakey = "device" if not data_file_path and not data: - raise Exception("data_file_path or data is required") - + raise Exception("data_file_path or data is required") + if data_file_path: if os.path.isfile(data_file_path) == False and not update_on_start: - raise Exception("There is no file at " + data_file_path) + raise Exception("There is no file at " + data_file_path) extention = os.path.splitext(data_file_path)[1] @@ -171,7 +171,7 @@ def __init__( available_performance = ["LowMemory", "MaxPerformance", "Balanced", "BalancedTemp", "HighPerformance"] if performance_profile not in available_performance: - raise Exception("Not a valid performance profile") + raise Exception("Not a valid performance profile") if performance_profile == "LowMemory": config.setLowMemory() @@ -192,7 +192,7 @@ def __init__( if(use_performance_graph): config.setUsePerformanceGraph(True) - + if(use_predictive_graph): config.setUsePredictiveGraph(True) @@ -228,7 +228,7 @@ def __init__( self.data_file_path = data_file_path # Disable features that require a licence key if one was not supplied. - + if licence_keys: auto_update = auto_update and (len(licence_keys) > 0) update_on_start = update_on_start and (len(licence_keys) > 0) @@ -237,7 +237,7 @@ def __init__( update_on_start = False if auto_update or update_on_start or file_system_watcher: - + # Construct DataFile update_url_params = { @@ -261,7 +261,7 @@ def __init__( def init_engine(self): """! - + Function for initialising the engine, wrapped like this so that an engine can be initialised once the datafile is retrieved if update_on_start is set to true. If this is the case, processing is held until the data file is downloaded and available. """ @@ -284,13 +284,6 @@ def init_engine(self): for x in range(evidence_keys.size()): evidence_keys_list.append(evidence_keys.__getitem__(x).lower()) - evidence_keys_list.extend([ - HARDCODED_EVIDENCE["GHEV"]["COOKIE"], - HARDCODED_EVIDENCE["GHEV"]["QUERY"], - HARDCODED_EVIDENCE["SUA"]["COOKIE"], - HARDCODED_EVIDENCE["SUA"]["QUERY"], - ]) - self.evidence_keys_list = evidence_keys_list # Get properties list @@ -310,7 +303,7 @@ def init_engine(self): "description": current_property.getDescription(), "component": self.get_component(current_property) } - + self.properties = properties # Special properties @@ -372,16 +365,16 @@ def get_evidence_key_filter(self): """! Returns evidence key filter for the on premise engine. Generated via a list of evidence keys from the swig engine - + """ - + return BasicListEvidenceKeyFilter(self.evidence_keys_list) - + def on_registration(self, pipeline): """! - + Function called after the engine is registered with a pipeline the engine base class registers the datafile but we also need to initialise the SWIG wrapper so call self.init_engine() here @@ -395,7 +388,7 @@ def on_registration(self, pipeline): def process_internal(self, flow_data): """! - + Internal process method of the Device Detection On Premise engine Gets properties through the SWIG wrapper and stores them in a SwigData extension of the ElementData class. Each property value (or lack of) is returned in an AspectPropertyData wrapper from diff --git a/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py b/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py index eb22b7b5b..8d28cd4b0 100644 --- a/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py +++ b/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py @@ -27,8 +27,6 @@ import csv import platform -from fiftyone_devicedetection_onpremise.constants import HARDCODED_EVIDENCE - from fiftyone_devicedetection_onpremise.devicedetection_onpremise_pipelinebuilder import DeviceDetectionOnPremisePipelineBuilder from fiftyone_devicedetection_onpremise.devicedetection_onpremise import DeviceDetectionOnPremise @@ -67,7 +65,7 @@ def test_pipeline_builder_shareusage_init(self): def test_on_premise_engine_datafile(self): """! - Tests whether a datafile (for the update service) is added when auto_update is set + Tests whether a datafile (for the update service) is added when auto_update is set """ pipeline = DeviceDetectionOnPremisePipelineBuilder( @@ -360,39 +358,6 @@ def test_evidencekey_filter_overrides(self): self.assertTrue(evidence_key_filter.filter_evidence_key( "cookie.51d_profileids")) - def test_evidencekey_filter_hardcoded_keys(self): - """! - Test evidence key filter with hardcoded keys in cookies and query string - """ - - pipeline = DeviceDetectionOnPremisePipelineBuilder( - data_file_path=data_file, usage_sharing=False, licence_keys="").build() - - evidence_key_filter = pipeline.flow_elements[0].get_evidence_key_filter( - ) - self.assertTrue(evidence_key_filter.filter_evidence_key( - HARDCODED_EVIDENCE["GHEV"]["QUERY"])) - self.assertTrue(evidence_key_filter.filter_evidence_key( - HARDCODED_EVIDENCE["GHEV"]["COOKIE"])) - self.assertTrue(evidence_key_filter.filter_evidence_key( - HARDCODED_EVIDENCE["SUA"]["QUERY"])) - self.assertTrue(evidence_key_filter.filter_evidence_key( - HARDCODED_EVIDENCE["SUA"]["COOKIE"])) - - self.assertTrue(evidence_key_filter.filter_evidence_key( - "query.51d_gethighentropyvalues")) - self.assertTrue(evidence_key_filter.filter_evidence_key( - "cookie.51d_gethighentropyvalues")) - self.assertTrue(evidence_key_filter.filter_evidence_key( - "query.51d_structureduseragent")) - self.assertTrue(evidence_key_filter.filter_evidence_key( - "cookie.51d_structureduseragent")) - - self.assertTrue(evidence_key_filter.filter_evidence_key( - "header.user-agent")) - self.assertFalse(evidence_key_filter.filter_evidence_key( - "header.nonexistent")) - def test_profile_overrides(self): """! Test profile overrides From 1240294e7b0ec7ae0438ea768e97631032771b07 Mon Sep 17 00:00:00 2001 From: Bohdan <25197509+BohdanVV@users.noreply.github.com> Date: Fri, 7 Feb 2025 22:28:22 +0100 Subject: [PATCH 2/2] Fix styling/remove redundant whitespaces --- .../constants.py | 2 +- .../devicedetection_onpremise.py | 36 +++++++++---------- .../tests/test_devicedetection.py | 2 +- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py b/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py index d0709a61e..6a2fe5fab 100644 --- a/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py +++ b/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/constants.py @@ -35,4 +35,4 @@ ITERATIONS_DESCRIPTION = ('The number of iterations carried out in order to ' 'find a match. This is the number of nodes in the ' 'graph which have been visited.') -METHOD_DESCRIPTION = 'The method used to determine the match result.' \ No newline at end of file +METHOD_DESCRIPTION = 'The method used to determine the match result.' diff --git a/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py b/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py index d60e021e5..d0dadcb4d 100644 --- a/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py +++ b/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/devicedetection_onpremise.py @@ -74,7 +74,7 @@ def __init__( **kwargs ): """! - + Constructor for the DeviceDetection On Premise Engine. @type data_file_path: string @@ -91,7 +91,7 @@ def __init__( @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service @type max_matched_useragents_length : int @param Number of characters to consider in the matched User-Agent. Ignored if update_matched_useragent is false - @type update_matched_useragent: + @type update_matched_useragent: @param: update_matched_useragent: True if the detection should record the matched characters from the target User-Agent @type drift: int @param drift: Set maximum drift in hash position to allow when processing HTTP headers @@ -108,7 +108,7 @@ def __init__( @type update_on_start : bool @param update_on_start : When this is set to true the datafile is updated / downloaded immediately on initialization. This is useful if no initial datafile is present. @type file_system_watcher: bool - @param file_system_watcher: whether to check the datafile's path for changes and update the connected FlowElement's data + @param file_system_watcher: whether to check the datafile's path for changes and update the connected FlowElement's data @type polling_interval: int @param polling_interval: How often to poll for updates to the datafile (minutes) @type update_time_maximum_randomisation : int @@ -120,7 +120,7 @@ def __init__( @type data_update_url: string @param data_update_url: base url for the datafile update service @type use_performance_graph: bool - @param use_performance_graph: True if the performance optimized graph should be used for processing + @param use_performance_graph: True if the performance optimized graph should be used for processing @type use_predictive_graph: bool @param use_predictive_graph: True if the predictive optimized graph should be used for processing @param data_update_use_url_formatter: This must be set to false to prevent the API from appending the query string parameters that are required when calling the 51Degrees Distributor service. @@ -133,11 +133,11 @@ def __init__( self.datakey = "device" if not data_file_path and not data: - raise Exception("data_file_path or data is required") - + raise Exception("data_file_path or data is required") + if data_file_path: if os.path.isfile(data_file_path) == False and not update_on_start: - raise Exception("There is no file at " + data_file_path) + raise Exception("There is no file at " + data_file_path) extention = os.path.splitext(data_file_path)[1] @@ -171,7 +171,7 @@ def __init__( available_performance = ["LowMemory", "MaxPerformance", "Balanced", "BalancedTemp", "HighPerformance"] if performance_profile not in available_performance: - raise Exception("Not a valid performance profile") + raise Exception("Not a valid performance profile") if performance_profile == "LowMemory": config.setLowMemory() @@ -192,7 +192,7 @@ def __init__( if(use_performance_graph): config.setUsePerformanceGraph(True) - + if(use_predictive_graph): config.setUsePredictiveGraph(True) @@ -228,7 +228,7 @@ def __init__( self.data_file_path = data_file_path # Disable features that require a licence key if one was not supplied. - + if licence_keys: auto_update = auto_update and (len(licence_keys) > 0) update_on_start = update_on_start and (len(licence_keys) > 0) @@ -237,7 +237,7 @@ def __init__( update_on_start = False if auto_update or update_on_start or file_system_watcher: - + # Construct DataFile update_url_params = { @@ -261,7 +261,7 @@ def __init__( def init_engine(self): """! - + Function for initialising the engine, wrapped like this so that an engine can be initialised once the datafile is retrieved if update_on_start is set to true. If this is the case, processing is held until the data file is downloaded and available. """ @@ -303,7 +303,7 @@ def init_engine(self): "description": current_property.getDescription(), "component": self.get_component(current_property) } - + self.properties = properties # Special properties @@ -365,16 +365,16 @@ def get_evidence_key_filter(self): """! Returns evidence key filter for the on premise engine. Generated via a list of evidence keys from the swig engine - + """ - + return BasicListEvidenceKeyFilter(self.evidence_keys_list) - + def on_registration(self, pipeline): """! - + Function called after the engine is registered with a pipeline the engine base class registers the datafile but we also need to initialise the SWIG wrapper so call self.init_engine() here @@ -388,7 +388,7 @@ def on_registration(self, pipeline): def process_internal(self, flow_data): """! - + Internal process method of the Device Detection On Premise engine Gets properties through the SWIG wrapper and stores them in a SwigData extension of the ElementData class. Each property value (or lack of) is returned in an AspectPropertyData wrapper from diff --git a/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py b/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py index 8d28cd4b0..252dbd0f2 100644 --- a/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py +++ b/fiftyone_devicedetection_onpremise/tests/test_devicedetection.py @@ -65,7 +65,7 @@ def test_pipeline_builder_shareusage_init(self): def test_on_premise_engine_datafile(self): """! - Tests whether a datafile (for the update service) is added when auto_update is set + Tests whether a datafile (for the update service) is added when auto_update is set """ pipeline = DeviceDetectionOnPremisePipelineBuilder(