diff --git a/Samples/BasicTelemetry.sample.json b/Samples/BasicTelemetry.sample.json new file mode 100644 index 0000000..85e4f88 --- /dev/null +++ b/Samples/BasicTelemetry.sample.json @@ -0,0 +1,33 @@ +{ + "feature_management": { + "feature_flags": [ + { + "id": "TelemetryVariant", + "description": "", + "enabled": "true", + "conditions": { + "client_filters": [] + }, + "variants": [ + { + "name": "True_Override", + "configuration_value": "default", + "status_override": "Disabled" + } + ], + "allocation": { + "default_when_enabled": "True_Override" + }, + "telemetry": { + "enabled": "true", + "metadata": { + "ETag": "cmwBRcIAq1jUyKL3Kj8bvf9jtxBrFg-R-ayExStMC90", + "FeatureFlagReference": "https://fake-config-store/kv/.appconfig.featureflag/TelemetryVariant", + "FeatureFlagId": "7vpkRJe452WVvlKXfA5XF3ASllwKsYZfC7D4w05rIoo", + "AllocationId": "MExY1waco2tqen4EcJKK" + } + } + } + ] + } +} diff --git a/Samples/BasicTelemetry.tests.json b/Samples/BasicTelemetry.tests.json new file mode 100644 index 0000000..f3352c4 --- /dev/null +++ b/Samples/BasicTelemetry.tests.json @@ -0,0 +1,33 @@ +[ + { + "FeatureFlagName": "TelemetryVariant", + "Inputs": {"User":"Aiden"}, + "IsEnabled": { + "Result": "false" + }, + "Variant": { + "Result": { + "Name": "True_Override", + "ConfigurationValue": "default" + } + }, + "Telemetry": { + "EventName": "FeatureEvaluation", + "EventProperties": { + "FeatureName": "TelemetryVariant", + "Enabled": "False", + "Version": "1.0.0", + "Variant": "True_Override", + "VariantAssignmentReason": "DefaultWhenEnabled", + "VariantAssignmentPercentage": "100", + "DefaultWhenEnabled": "True_Override", + "AllocationId": "MExY1waco2tqen4EcJKK", + "ETag": "cmwBRcIAq1jUyKL3Kj8bvf9jtxBrFg-R-ayExStMC90", + "FeatureFlagReference": "https://fake-config-store/kv/.appconfig.featureflag/TelemetryVariant", + "FeatureFlagId": "7vpkRJe452WVvlKXfA5XF3ASllwKsYZfC7D4w05rIoo", + "TargetingId": "Aiden" + } + }, + "Description": "A basic feature flag with telemetry." + } +] diff --git a/libraryValidations/Python/requirements.txt b/libraryValidations/Python/requirements.txt index dc99de6..435aee3 100644 --- a/libraryValidations/Python/requirements.txt +++ b/libraryValidations/Python/requirements.txt @@ -1,3 +1,3 @@ pytest -featuremanagement==2.0.0b2 -azure-appconfiguration-provider==2.0.0b2 +featuremanagement["AzureMonitor"]==2.0.0b3 +azure-appconfiguration-provider==2.0.0b3 diff --git a/libraryValidations/Python/test_json_validations.py b/libraryValidations/Python/test_json_validations.py index 16911d7..1f6bf76 100644 --- a/libraryValidations/Python/test_json_validations.py +++ b/libraryValidations/Python/test_json_validations.py @@ -4,11 +4,12 @@ # license information. # -------------------------------------------------------------------------- -import logging import json import unittest from pytest import raises from featuremanagement import FeatureManager, TargetingContext +from featuremanagement.azuremonitor import publish_telemetry +from unittest.mock import patch, call FILE_PATH = "../../Samples/" SAMPLE_JSON_KEY = ".sample.json" @@ -16,6 +17,9 @@ FRIENDLY_NAME_KEY = "FriendlyName" IS_ENABLED_KEY = "IsEnabled" GET_VARIANT_KEY = "Variant" +GET_TELEMETRY_KEY = "Telemetry" +EVENT_NAME_KEY = "EventName" +EVENT_PROPERTIES_KEY = "EventProperties" RESULT_KEY = "Result" VARIANT_NAME_KEY = "Name" CONFIGURATION_VALUE_KEY = "ConfigurationValue" @@ -26,9 +30,6 @@ EXCEPTION_KEY = "Exception" DESCRIPTION_KEY = "Description" -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - def convert_boolean_value(enabled): if enabled is None: @@ -43,63 +44,82 @@ def convert_boolean_value(enabled): class TestFromFile(unittest.TestCase): - # method: is_enabled + def test_no_filters(self): test_key = "NoFilters" self.run_tests(test_key) - # method: is_enabled def test_time_window_filter(self): test_key = "TimeWindowFilter" self.run_tests(test_key) - # method: is_enabled def test_targeting_filter(self): test_key = "TargetingFilter" self.run_tests(test_key) - # method: is_enabled def test_targeting_filter_modified(self): test_key = "TargetingFilter.modified" self.run_tests(test_key) - # method: is_enabled def test_requirement_type(self): test_key = "RequirementType" self.run_tests(test_key) - # method: is_enabled def test_basic_variant(self): test_key = "BasicVariant" self.run_tests(test_key) - - # method: is_enabled def test_variant_assignment(self): test_key = "VariantAssignment" self.run_tests(test_key) + @patch("featuremanagement.azuremonitor._send_telemetry.azure_monitor_track_event") + def test_basic_telemetry(self, track_event_mock): + test_key = "BasicTelemetry" + self._ran_callback = False + self._mock_track_event = track_event_mock + self.run_tests(test_key, telemetry_callback=self.telemetry_callback) + assert self._ran_callback + @staticmethod - def load_from_file(file): + def load_from_file(file, telemetry_callback=None): with open(FILE_PATH + file, "r", encoding="utf-8") as feature_flags_file: feature_flags = json.load(feature_flags_file) - feature_manager = FeatureManager(feature_flags) + feature_manager = FeatureManager( + feature_flags, on_feature_evaluated=telemetry_callback + ) assert feature_manager is not None return feature_manager - # method: is_enabled - def run_tests(self, test_key): - feature_manager = self.load_from_file(test_key + SAMPLE_JSON_KEY) - - with open(FILE_PATH + test_key + TESTS_JSON_KEY, "r", encoding="utf-8") as feature_flag_test_file: + def telemetry_callback(self, evaluation_event): + publish_telemetry(evaluation_event) + expected_telemetry = self._feature_flag_test.get(GET_TELEMETRY_KEY, {}) + self._mock_track_event.assert_called_once() + self.assertEqual(self._mock_track_event.call_args[0][0], expected_telemetry.get(EVENT_NAME_KEY, None)) + (event_properties) = self._mock_track_event.call_args[0][1] + self.assertEqual(sorted(event_properties), sorted(expected_telemetry.get(EVENT_PROPERTIES_KEY, {}))) + self._ran_callback = True + self._mock_track_event.reset_mock() + + def run_tests(self, test_key, telemetry_callback=None): + feature_manager = self.load_from_file( + test_key + SAMPLE_JSON_KEY, telemetry_callback=telemetry_callback + ) + + with open( + FILE_PATH + test_key + TESTS_JSON_KEY, "r", encoding="utf-8" + ) as feature_flag_test_file: feature_flag_tests = json.load(feature_flag_test_file) for feature_flag_test in feature_flag_tests: + self._feature_flag_test = feature_flag_test is_enabled = feature_flag_test[IS_ENABLED_KEY] get_variant = feature_flag_test.get(GET_VARIANT_KEY, None) - expected_is_enabled_result = convert_boolean_value(is_enabled.get(RESULT_KEY)) + expected_is_enabled_result = convert_boolean_value( + is_enabled.get(RESULT_KEY) + ) feature_flag_id = test_key + "." + feature_flag_test[FEATURE_FLAG_NAME_KEY] failed_description = f"Test {feature_flag_id} failed. Description: {feature_flag_test[DESCRIPTION_KEY]}" @@ -109,25 +129,47 @@ def run_tests(self, test_key): groups = feature_flag_test[INPUTS_KEY].get(GROUPS_KEY, []) assert ( feature_manager.is_enabled( - feature_flag_test[FEATURE_FLAG_NAME_KEY], TargetingContext(user_id=user, groups=groups) + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), ) == expected_is_enabled_result ), failed_description else: with raises(ValueError) as ex_info: - feature_manager.is_enabled(feature_flag_test[FEATURE_FLAG_NAME_KEY]) + feature_manager.is_enabled( + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), + ) expected_message = is_enabled.get(EXCEPTION_KEY) assert str(ex_info.value) == expected_message, failed_description - if get_variant is not None and RESULT_KEY in get_variant: + if get_variant: user = feature_flag_test[INPUTS_KEY].get(USER_KEY, None) groups = feature_flag_test[INPUTS_KEY].get(GROUPS_KEY, []) - variant = feature_manager.get_variant(feature_flag_test[FEATURE_FLAG_NAME_KEY], TargetingContext(user_id=user, groups=groups)) - - if get_variant[RESULT_KEY] == None: - assert variant == None - else: - if VARIANT_NAME_KEY in get_variant[RESULT_KEY]: - assert variant.name == get_variant[RESULT_KEY][VARIANT_NAME_KEY], failed_description - - assert variant.configuration == get_variant[RESULT_KEY][CONFIGURATION_VALUE_KEY], failed_description + + if RESULT_KEY not in get_variant: + with raises(ValueError) as ex_info: + feature_manager.get_variant( + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), + ) + expected_message = get_variant.get(EXCEPTION_KEY) + assert str(ex_info.value) == expected_message, failed_description + continue + + variant = feature_manager.get_variant( + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), + ) + if not get_variant[RESULT_KEY]: + assert not variant + continue + if VARIANT_NAME_KEY in get_variant[RESULT_KEY]: + assert ( + variant.name == get_variant[RESULT_KEY][VARIANT_NAME_KEY] + ), failed_description + + assert ( + variant.configuration + == get_variant[RESULT_KEY][CONFIGURATION_VALUE_KEY] + ), failed_description diff --git a/libraryValidations/Python/test_json_validations_with_provider.py b/libraryValidations/Python/test_json_validations_with_provider.py index 835db5d..46e902f 100644 --- a/libraryValidations/Python/test_json_validations_with_provider.py +++ b/libraryValidations/Python/test_json_validations_with_provider.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- + from unittest.mock import patch import json import unittest @@ -28,6 +29,7 @@ EXCEPTION_KEY = "Exception" DESCRIPTION_KEY = "Description" + def convert_boolean_value(enabled): if enabled is None: return None @@ -44,36 +46,45 @@ class TestFromProvider(unittest.TestCase): def test_provider_telemetry(self): test_key = "ProviderTelemetry" - with patch("featuremanagement.azuremonitor._send_telemetry.azure_monitor_track_event") as mock_track_event: + with patch( + "featuremanagement.azuremonitor._send_telemetry.azure_monitor_track_event" + ) as mock_track_event: self.run_tests(test_key, mock_track_event) def test_complete_provider_telemetry(self): test_key = "ProviderTelemetryComplete" - with patch("featuremanagement.azuremonitor._send_telemetry.azure_monitor_track_event") as mock_track_event: + with patch( + "featuremanagement.azuremonitor._send_telemetry.azure_monitor_track_event" + ) as mock_track_event: self.run_tests(test_key, mock_track_event) @staticmethod def load_from_provider(): connection_string = os.getenv("APP_CONFIG_VALIDATION_CONNECTION_STRING") - config = load(connection_string=connection_string, selects=[], feature_flag_enabled=True) + config = load( + connection_string=connection_string, selects=[], feature_flag_enabled=True + ) feature_manager = FeatureManager(config, on_feature_evaluated=publish_telemetry) assert feature_manager is not None return feature_manager - def run_tests(self, test_key, track_event_mock): feature_manager = self.load_from_provider() - with open(FILE_PATH + test_key + TESTS_JSON_KEY, "r", encoding="utf-8") as feature_flag_test_file: + with open( + FILE_PATH + test_key + TESTS_JSON_KEY, "r", encoding="utf-8" + ) as feature_flag_test_file: feature_flag_tests = json.load(feature_flag_test_file) for feature_flag_test in feature_flag_tests: track_event_mock.reset_mock() is_enabled = feature_flag_test[IS_ENABLED_KEY] get_variant = feature_flag_test.get(GET_VARIANT_KEY, None) - expected_is_enabled_result = convert_boolean_value(is_enabled.get(RESULT_KEY)) + expected_is_enabled_result = convert_boolean_value( + is_enabled.get(RESULT_KEY) + ) feature_flag_id = test_key + "." + feature_flag_test[FEATURE_FLAG_NAME_KEY] telemetry = feature_flag_test.get("Telemetry", None) @@ -84,24 +95,50 @@ def run_tests(self, test_key, track_event_mock): groups = feature_flag_test[INPUTS_KEY].get(GROUPS_KEY, []) assert ( feature_manager.is_enabled( - feature_flag_test[FEATURE_FLAG_NAME_KEY], TargetingContext(user_id=user, groups=groups) + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), ) == expected_is_enabled_result ), failed_description else: with raises(ValueError) as ex_info: - feature_manager.is_enabled(feature_flag_test[FEATURE_FLAG_NAME_KEY]) + feature_manager.is_enabled( + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), + ) expected_message = is_enabled.get(EXCEPTION_KEY) assert str(ex_info.value) == expected_message, failed_description - if get_variant is not None and get_variant[RESULT_KEY]: + if get_variant: user = feature_flag_test[INPUTS_KEY].get(USER_KEY, None) groups = feature_flag_test[INPUTS_KEY].get(GROUPS_KEY, []) - variant = feature_manager.get_variant(feature_flag_test[FEATURE_FLAG_NAME_KEY], TargetingContext(user_id=user, groups=groups)) - assert variant, failed_description + + if RESULT_KEY not in get_variant: + with raises(ValueError) as ex_info: + feature_manager.get_variant( + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), + ) + expected_message = get_variant.get(EXCEPTION_KEY) + assert str(ex_info.value) == expected_message, failed_description + continue + + variant = feature_manager.get_variant( + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), + ) + if not get_variant[RESULT_KEY]: + assert not variant + continue if VARIANT_NAME_KEY in get_variant[RESULT_KEY]: - assert variant.name == get_variant[RESULT_KEY][VARIANT_NAME_KEY], failed_description - assert variant.configuration == get_variant[RESULT_KEY][CONFIGURATION_VALUE_KEY], failed_description + assert ( + variant.name == get_variant[RESULT_KEY][VARIANT_NAME_KEY] + ), failed_description + + assert ( + variant.configuration + == get_variant[RESULT_KEY][CONFIGURATION_VALUE_KEY] + ), failed_description if telemetry: assert track_event_mock.called @@ -110,22 +147,43 @@ def run_tests(self, test_key, track_event_mock): event = track_event_mock.call_args[0][1] event_properties = telemetry["EventProperties"] - connection_string = os.getenv("APP_CONFIG_VALIDATION_CONNECTION_STRING") + connection_string = os.getenv( + "APP_CONFIG_VALIDATION_CONNECTION_STRING" + ) endpoint = endpoint_from_connection_string(connection_string) assert event["FeatureName"] == event_properties["FeatureName"] assert event["Enabled"] == event_properties["Enabled"] assert event["Version"] == event_properties["Version"] assert event["Variant"] == event_properties["Variant"] - assert event["VariantAssignmentReason"] == event_properties["VariantAssignmentReason"] - - if "VariantAssignmentPercentage" in event: # User/Group assignment doesn't have this property - assert event["VariantAssignmentPercentage"] == event_properties["VariantAssignmentPercentage"] - - assert event["DefaultWhenEnabled"] == event_properties["DefaultWhenEnabled"] - assert event["ETag"] # ETag will be different for each store, just assert it exists - assert event["FeatureFlagReference"] == endpoint + event_properties["FeatureFlagReference"] - assert event["FeatureFlagId"].decode("utf-8") == event_properties["FeatureFlagId"] + assert ( + event["VariantAssignmentReason"] + == event_properties["VariantAssignmentReason"] + ) + + if ( + "VariantAssignmentPercentage" in event + ): # User/Group assignment doesn't have this property + assert ( + event["VariantAssignmentPercentage"] + == event_properties["VariantAssignmentPercentage"] + ) + + assert ( + event["DefaultWhenEnabled"] + == event_properties["DefaultWhenEnabled"] + ) + assert event[ + "ETag" + ] # ETag will be different for each store, just assert it exists + assert ( + event["FeatureFlagReference"] + == endpoint + event_properties["FeatureFlagReference"] + ) + assert ( + event["FeatureFlagId"].decode("utf-8") + == event_properties["FeatureFlagId"] + ) assert event["AllocationId"] == event_properties["AllocationId"] assert event["TargetingId"] == event_properties["TargetingId"]