From 91d5e4e6676d6417a61f074a36d068309fb7152f Mon Sep 17 00:00:00 2001 From: Matt Metcalf Date: Thu, 10 Oct 2024 15:35:55 -0700 Subject: [PATCH 1/5] Basic Telemetry --- Samples/BasicTelemetry.sample.json | 33 ++++++++++ Samples/BasicTelemetry.tests.json | 30 +++++++++ libraryValidations/Python/requirements.txt | 2 +- .../Python/test_json_validations.py | 66 +++++++++++++++---- 4 files changed, 119 insertions(+), 12 deletions(-) create mode 100644 Samples/BasicTelemetry.sample.json create mode 100644 Samples/BasicTelemetry.tests.json diff --git a/Samples/BasicTelemetry.sample.json b/Samples/BasicTelemetry.sample.json new file mode 100644 index 0000000..85e4f88 --- /dev/null +++ b/Samples/BasicTelemetry.sample.json @@ -0,0 +1,33 @@ +{ + "feature_management": { + "feature_flags": [ + { + "id": "TelemetryVariant", + "description": "", + "enabled": "true", + "conditions": { + "client_filters": [] + }, + "variants": [ + { + "name": "True_Override", + "configuration_value": "default", + "status_override": "Disabled" + } + ], + "allocation": { + "default_when_enabled": "True_Override" + }, + "telemetry": { + "enabled": "true", + "metadata": { + "ETag": "cmwBRcIAq1jUyKL3Kj8bvf9jtxBrFg-R-ayExStMC90", + "FeatureFlagReference": "https://fake-config-store/kv/.appconfig.featureflag/TelemetryVariant", + "FeatureFlagId": "7vpkRJe452WVvlKXfA5XF3ASllwKsYZfC7D4w05rIoo", + "AllocationId": "MExY1waco2tqen4EcJKK" + } + } + } + ] + } +} diff --git a/Samples/BasicTelemetry.tests.json b/Samples/BasicTelemetry.tests.json new file mode 100644 index 0000000..bd214d8 --- /dev/null +++ b/Samples/BasicTelemetry.tests.json @@ -0,0 +1,30 @@ +[ + { + "FeatureFlagName": "TelemetryVariant", + "Inputs": {"user":"Aiden"}, + "IsEnabled": { + "Result": "false" + }, + "Variant": { + "Result": "default" + }, + "Telemetry": { + "event_name": "FeatureEvaluation", + "event_properties": { + "FeatureName": "TelemetryVariant", + "Enabled": "False", + "Version": "1.0.0", + "Variant": "True_Override", + "VariantAssignmentReason": "DefaultWhenEnabled", + "VariantAssignmentPercentage": "100", + "DefaultWhenEnabled": "True_Override", + "AllocationId": "MExY1waco2tqen4EcJKK", + "ETag": "cmwBRcIAq1jUyKL3Kj8bvf9jtxBrFg-R-ayExStMC90", + "FeatureFlagReference": "https://fake-config-store/kv/.appconfig.featureflag/TelemetryVariant", + "FeatureFlagId": "7vpkRJe452WVvlKXfA5XF3ASllwKsYZfC7D4w05rIoo", + "TargetingId": "Aiden" + } + }, + "Description": "A basic feature flag with telemetry." + } +] diff --git a/libraryValidations/Python/requirements.txt b/libraryValidations/Python/requirements.txt index 64ae258..ae175b1 100644 --- a/libraryValidations/Python/requirements.txt +++ b/libraryValidations/Python/requirements.txt @@ -1,2 +1,2 @@ pytest -featuremanagement +featuremanagement["AzureMonitor"] diff --git a/libraryValidations/Python/test_json_validations.py b/libraryValidations/Python/test_json_validations.py index abb4988..f3371ac 100644 --- a/libraryValidations/Python/test_json_validations.py +++ b/libraryValidations/Python/test_json_validations.py @@ -8,6 +8,8 @@ import unittest from pytest import raises from featuremanagement import FeatureManager, TargetingContext +from featuremanagement.azuremonitor import publish_telemetry +from unittest.mock import patch, call FILE_PATH = "../../Samples/" SAMPLE_JSON_KEY = ".sample.json" @@ -15,6 +17,9 @@ FRIENDLY_NAME_KEY = "FriendlyName" IS_ENABLED_KEY = "IsEnabled" GET_VARIANT_KEY = "Variant" +GET_TELEMETRY_KEY = "Telemetry" +EVENT_NAME_KEY = "event_name" +EVENT_PROPERTIES_KEY = "event_properties" RESULT_KEY = "Result" FEATURE_FLAG_NAME_KEY = "FeatureFlagName" INPUTS_KEY = "Inputs" @@ -75,27 +80,60 @@ def test_variant_assignment(self): test_key = "VariantAssignment" self.run_tests(test_key) + # method: is_enabled + @patch("featuremanagement.azuremonitor._send_telemetry.azure_monitor_track_event") + def test_basic_telemetry(self, track_event_mock): + test_key = "BasicTelemetry" + self._ran_callback = False + self._mock_track_event = track_event_mock + self.run_tests(test_key, telemetry_callback=self.telemetry_callback) + assert self._ran_callback + @staticmethod - def load_from_file(file): + def load_from_file(file, telemetry_callback=None): with open(FILE_PATH + file, "r", encoding="utf-8") as feature_flags_file: feature_flags = json.load(feature_flags_file) - feature_manager = FeatureManager(feature_flags) + if telemetry_callback: + feature_manager = FeatureManager( + feature_flags, on_feature_evaluated=telemetry_callback + ) + else: + feature_manager = FeatureManager(feature_flags) assert feature_manager is not None return feature_manager - # method: is_enabled - def run_tests(self, test_key): - feature_manager = self.load_from_file(test_key + SAMPLE_JSON_KEY) - - with open(FILE_PATH + test_key + TESTS_JSON_KEY, "r", encoding="utf-8") as feature_flag_test_file: + def telemetry_callback(self, evaluation_event): + publish_telemetry(evaluation_event) + expected_telemetry = self._feature_flag_test.get(GET_TELEMETRY_KEY, {}) + self._mock_track_event.assert_has_calls( + [ + call( + expected_telemetry.get(EVENT_NAME_KEY, None), + expected_telemetry.get(EVENT_PROPERTIES_KEY, None), + ) + ] + ) + self._ran_callback = True + + def run_tests(self, test_key, telemetry_callback=None): + feature_manager = self.load_from_file( + test_key + SAMPLE_JSON_KEY, telemetry_callback=telemetry_callback + ) + + with open( + FILE_PATH + test_key + TESTS_JSON_KEY, "r", encoding="utf-8" + ) as feature_flag_test_file: feature_flag_tests = json.load(feature_flag_test_file) for feature_flag_test in feature_flag_tests: + self._feature_flag_test = feature_flag_test is_enabled = feature_flag_test[IS_ENABLED_KEY] get_variant = feature_flag_test.get(GET_VARIANT_KEY, None) - expected_is_enabled_result = convert_boolean_value(is_enabled.get(RESULT_KEY)) + expected_is_enabled_result = convert_boolean_value( + is_enabled.get(RESULT_KEY) + ) feature_flag_id = test_key + "." + feature_flag_test[FEATURE_FLAG_NAME_KEY] failed_description = f"Test {feature_flag_id} failed. Description: {feature_flag_test[DESCRIPTION_KEY]}" @@ -105,7 +143,8 @@ def run_tests(self, test_key): groups = feature_flag_test[INPUTS_KEY].get(GROUPS_KEY, []) assert ( feature_manager.is_enabled( - feature_flag_test[FEATURE_FLAG_NAME_KEY], TargetingContext(user_id=user, groups=groups) + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), ) == expected_is_enabled_result ), failed_description @@ -118,8 +157,13 @@ def run_tests(self, test_key): if get_variant is not None and get_variant[RESULT_KEY]: user = feature_flag_test[INPUTS_KEY].get(USER_KEY, None) groups = feature_flag_test[INPUTS_KEY].get(GROUPS_KEY, []) - variant = feature_manager.get_variant(feature_flag_test[FEATURE_FLAG_NAME_KEY], TargetingContext(user_id=user, groups=groups)) + variant = feature_manager.get_variant( + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), + ) if not variant: logger.error(f"Variant is None for {feature_flag_id}") assert False, failed_description - assert variant.configuration == get_variant[RESULT_KEY], failed_description + assert ( + variant.configuration == get_variant[RESULT_KEY] + ), failed_description From 7682616ce94c8181fa92cc98650df52ae2f29aac Mon Sep 17 00:00:00 2001 From: Ross Grambo Date: Thu, 31 Oct 2024 13:51:53 -0700 Subject: [PATCH 2/5] Validation tests for dotnet (#16) * Python tests + Spring Start * debugging * fixed * requirementType * Python Readme & More Spring Tests * Spring Readme and Updates * Python variants + basic variants * Adds .net Validation Tests * Adjusts variant result to use dynamic configuration value * Uses Pascal Case in tests * Merge with main * Updates to pascal case * Resovles attribute Users instead of User * Adjusts tests to all work with new schema * Fixes test call in shared script * Update pom.xml * Fixes path after spring tests * Fixes schema typo * Adds name to variant result expectation * Misc. dotnet fixes * Adds more names --------- Co-authored-by: Matt Metcalf --- .gitignore | 1 + Samples/BasicVariant.tests.json | 24 +- Samples/NoFilters.tests.json | 2 +- Samples/ProviderTelemetry.tests.json | 57 ++-- Samples/ProviderTelemetryComplete.tests.json | 48 ++-- Samples/RequirementType.tests.json | 4 +- Samples/TargetingFilter.modified.tests.json | 16 +- Samples/TargetingFilter.tests.json | 38 +-- Samples/VariantAssignment.tests.json | 76 ++++-- libraryValidations/Dotnet/Dotnet.csproj | 23 ++ libraryValidations/Dotnet/Dotnet.sln | 25 ++ libraryValidations/Dotnet/MSTestSettings.cs | 1 + .../Dotnet/Properties/launchSettings.json | 10 + libraryValidations/Dotnet/README.md | 19 ++ libraryValidations/Dotnet/SharedTest.cs | 43 +++ libraryValidations/Dotnet/TestValidations.cs | 251 ++++++++++++++++++ .../Dotnet/UnknownJsonFieldConverter.cs | 42 +++ .../Python/test_json_validations.py | 22 +- .../test_json_validations_with_provider.py | 14 +- .../Spring/validation-tests/pom.xml | 2 +- .../ValidationTestsApplicationTests.java | 4 +- .../models/ValidationTestCase.java | 6 +- .../validation_tests/models/Variant.java | 28 +- .../models/VariantResult.java | 36 +++ libraryValidations/run_all_tests.ps1 | 17 ++ 25 files changed, 673 insertions(+), 136 deletions(-) create mode 100644 libraryValidations/Dotnet/Dotnet.csproj create mode 100644 libraryValidations/Dotnet/Dotnet.sln create mode 100644 libraryValidations/Dotnet/MSTestSettings.cs create mode 100644 libraryValidations/Dotnet/Properties/launchSettings.json create mode 100644 libraryValidations/Dotnet/README.md create mode 100644 libraryValidations/Dotnet/SharedTest.cs create mode 100644 libraryValidations/Dotnet/TestValidations.cs create mode 100644 libraryValidations/Dotnet/UnknownJsonFieldConverter.cs create mode 100644 libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/models/VariantResult.java create mode 100644 libraryValidations/run_all_tests.ps1 diff --git a/.gitignore b/.gitignore index b1a5041..8e3678e 100644 --- a/.gitignore +++ b/.gitignore @@ -399,6 +399,7 @@ FodyWeavers.xsd # Python env env/ +venv/ .classpath diff --git a/Samples/BasicVariant.tests.json b/Samples/BasicVariant.tests.json index 5729d4e..514a578 100644 --- a/Samples/BasicVariant.tests.json +++ b/Samples/BasicVariant.tests.json @@ -6,7 +6,10 @@ "Result": "false" }, "Variant": { - "Result": "default" + "Result": { + "Name": "True_Override", + "ConfigurationValue": "default" + } }, "Description": "An Enabled Feature Flag with no Filters." }, @@ -17,33 +20,42 @@ "Result": "false" }, "Variant": { - "Result": "default" + "Result": { + "Name": "False_Override", + "ConfigurationValue": "default" + } }, "Description": "A Disabled Feature Flag with no Filters." }, { "FeatureFlagName": "TestVariants", "Inputs": { - "user": "Adam" + "User": "Adam" }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": "The Variant Alpha." + "Result": { + "Name": "Alpha", + "ConfigurationValue": "The Variant Alpha." + } }, "Description": "A Feature Flag with two Variants, one for Adam and one for Britney." }, { "FeatureFlagName": "TestVariants", "Inputs": { - "user": "Britney" + "User": "Britney" }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": "The Variant Beta." + "Result": { + "Name": "Beta", + "ConfigurationValue": "The Variant Beta." + } }, "Description": "A Feature Flag with two Variants, one for Adam and one for Britney." } diff --git a/Samples/NoFilters.tests.json b/Samples/NoFilters.tests.json index 01445d6..1b98c83 100644 --- a/Samples/NoFilters.tests.json +++ b/Samples/NoFilters.tests.json @@ -28,7 +28,7 @@ "Exception": "Invalid setting 'enabled' with value 'invalid' for feature 'InvalidEnabled'." }, "Variant": { - "Result": null + "Exception": "Invalid setting 'enabled' with value 'invalid' for feature 'InvalidEnabled'." }, "Description": "A Feature Flag with an invalid Enabled Value." }, diff --git a/Samples/ProviderTelemetry.tests.json b/Samples/ProviderTelemetry.tests.json index f54d31d..c6e1914 100644 --- a/Samples/ProviderTelemetry.tests.json +++ b/Samples/ProviderTelemetry.tests.json @@ -2,30 +2,33 @@ { "FeatureFlagName": "TelemetryVariantPercentile", "Inputs": { - "user": "Sami" + "User": "Sami" }, "IsEnabled": { "Result": "true" }, "Variant": { "Result": { - "someOtherKey": { - "someSubKey": "someSubValue" - }, - "someKey4": [ - 3, - 1, - 4, - true - ], - "someKey": "someValue", - "someKey3": 3.14, - "someKey2": 3 + "Name": "True_Override", + "ConfigurationValue": { + "someOtherKey": { + "someSubKey": "someSubValue" + }, + "someKey4": [ + 3, + 1, + 4, + true + ], + "someKey": "someValue", + "someKey3": 3.14, + "someKey2": 3 + } } }, "Telemetry": { - "event_name": "FeatureEvaluation", - "event_properties": { + "EventName": "FeatureEvaluation", + "EventProperties": { "FeatureName": "TelemetryVariantPercentile", "Enabled": "True", "Version": "1.0.0", @@ -44,17 +47,20 @@ { "FeatureFlagName": "Background.Colors", "Inputs": { - "user": "Aiden" + "User": "Aiden" }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": {"r": 75, "g": 0, "b": 130} + "Result": { + "Name": "Indigo", + "ConfigurationValue": { "r": 75, "g": 0, "b": 130 } + } }, "Telemetry": { - "event_name": "FeatureEvaluation", - "event_properties": { + "EventName": "FeatureEvaluation", + "EventProperties": { "FeatureName": "Background.Colors", "Enabled": "True", "Version": "1.0.0", @@ -73,17 +79,22 @@ { "FeatureFlagName": "Background.Colors", "Inputs": { - "user": "Brittney" + "User": "Brittney" }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": {"r": 255, "g": 162, "b": 0} + "Result": { + "Name": "Orange", + "ConfigurationValue": { + "r": 255, "g": 162, "b": 0 + } + } }, "Telemetry": { - "event_name": "FeatureEvaluation", - "event_properties": { + "EventName": "FeatureEvaluation", + "EventProperties": { "FeatureName": "Background.Colors", "Enabled": "True", "Version": "1.0.0", diff --git a/Samples/ProviderTelemetryComplete.tests.json b/Samples/ProviderTelemetryComplete.tests.json index c0a667e..b018b3b 100644 --- a/Samples/ProviderTelemetryComplete.tests.json +++ b/Samples/ProviderTelemetryComplete.tests.json @@ -2,17 +2,20 @@ { "FeatureFlagName": "Complete", "Inputs": { - "user": "Aiden" + "User": "Aiden" }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": 100 + "Result": { + "Name": "Large", + "ConfigurationValue": 100 + } }, "Telemetry": { - "event_name": "FeatureEvaluation", - "event_properties": { + "EventName": "FeatureEvaluation", + "EventProperties": { "FeatureName": "Complete", "Enabled": "True", "Version": "1.0.0", @@ -31,17 +34,20 @@ { "FeatureFlagName": "Complete", "Inputs": { - "user": "Rachel" + "User": "Rachel" }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": 10 + "Result": { + "Name": "Small", + "ConfigurationValue": 10 + } }, "Telemetry": { - "event_name": "FeatureEvaluation", - "event_properties": { + "EventName": "FeatureEvaluation", + "EventProperties": { "FeatureName": "Complete", "Enabled": "True", "Version": "1.0.0", @@ -59,18 +65,21 @@ { "FeatureFlagName": "Complete", "Inputs": { - "user": "Aiden", - "groups": ["beta"] + "User": "Aiden", + "Groups": ["beta"] }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": 100 + "Result": { + "Name": "Large", + "ConfigurationValue": 100 + } }, "Telemetry": { - "event_name": "FeatureEvaluation", - "event_properties": { + "EventName": "FeatureEvaluation", + "EventProperties": { "FeatureName": "Complete", "Enabled": "True", "Version": "1.0.0", @@ -88,18 +97,21 @@ { "FeatureFlagName": "Complete", "Inputs": { - "user": "Rachel", - "group": "beta" + "User": "Rachel", + "Groups": ["beta"] }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": 10 + "Result": { + "Name": "Small", + "ConfigurationValue": 10 + } }, "Telemetry": { - "event_name": "FeatureEvaluation", - "event_properties": { + "EventName": "FeatureEvaluation", + "EventProperties": { "FeatureName": "Complete", "Enabled": "True", "Version": "1.0.0", diff --git a/Samples/RequirementType.tests.json b/Samples/RequirementType.tests.json index 74df4d8..0524194 100644 --- a/Samples/RequirementType.tests.json +++ b/Samples/RequirementType.tests.json @@ -45,7 +45,7 @@ }, { "FeatureFlagName": "RequirementTypeAllPassed", - "Inputs": {"user":"Adam"}, + "Inputs": {"User":"Adam"}, "IsEnabled": { "Result": "true" }, @@ -56,7 +56,7 @@ }, { "FeatureFlagName": "RequirementTypeAllLastFilterFailed", - "Inputs": {"user":"Adam"}, + "Inputs": {"User":"Adam"}, "IsEnabled": { "Result": "false" }, diff --git a/Samples/TargetingFilter.modified.tests.json b/Samples/TargetingFilter.modified.tests.json index 1c1425b..8d32379 100644 --- a/Samples/TargetingFilter.modified.tests.json +++ b/Samples/TargetingFilter.modified.tests.json @@ -2,7 +2,7 @@ { "FriendlyName": "Aiden62", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Aiden"}, + "Inputs": {"User":"Aiden"}, "IsEnabled": { "Result": "true" }, @@ -14,7 +14,7 @@ { "FriendlyName": "Aiden62 - Stage1", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Aiden", "groups":["Stage1"]}, + "Inputs": {"User":"Aiden", "Groups":["Stage1"]}, "IsEnabled": { "Result": "true" }, @@ -26,7 +26,7 @@ { "FriendlyName": "Aiden62 - Stage2", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Aiden", "groups":["Stage2"]}, + "Inputs": {"User":"Aiden", "Groups":["Stage2"]}, "IsEnabled": { "Result": "true" }, @@ -38,7 +38,7 @@ { "FriendlyName": "Aiden62 - Stage3", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Aiden", "groups":["Stage3"]}, + "Inputs": {"User":"Aiden", "Groups":["Stage3"]}, "IsEnabled": { "Result": "true" }, @@ -50,7 +50,7 @@ { "FriendlyName": "Brittney62", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Brittney"}, + "Inputs": {"User":"Brittney"}, "IsEnabled": { "Result": "true" }, @@ -62,7 +62,7 @@ { "FriendlyName": "Brittney62 - Stage1", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Brittney", "groups":["Stage1"]}, + "Inputs": {"User":"Brittney", "Groups":["Stage1"]}, "IsEnabled": { "Result": "true" }, @@ -74,7 +74,7 @@ { "FriendlyName": "Brittney62 - Stage2", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Brittney", "groups":["Stage2"]}, + "Inputs": {"User":"Brittney", "Groups":["Stage2"]}, "IsEnabled": { "Result": "true" }, @@ -86,7 +86,7 @@ { "FriendlyName": "Brittney62 - Stage3", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Brittney", "groups":["Stage3"]}, + "Inputs": {"User":"Brittney", "Groups":["Stage3"]}, "IsEnabled": { "Result": "true" }, diff --git a/Samples/TargetingFilter.tests.json b/Samples/TargetingFilter.tests.json index 60e2f41..a3b88df 100644 --- a/Samples/TargetingFilter.tests.json +++ b/Samples/TargetingFilter.tests.json @@ -2,7 +2,7 @@ { "FriendlyName": "DisabledDefaultRollout", "FeatureFlagName": "ComplexTargeting", - "Inputs": {"user":"Aiden"}, + "Inputs": {"User":"Aiden"}, "IsEnabled": { "Result": "false" }, @@ -14,7 +14,7 @@ { "FriendlyName": "EnabledDefaultRollout", "FeatureFlagName": "ComplexTargeting", - "Inputs": {"user":"Blossom"}, + "Inputs": {"User":"Blossom"}, "IsEnabled": { "Result": "true" }, @@ -26,7 +26,7 @@ { "FriendlyName": "TargetedUser", "FeatureFlagName": "ComplexTargeting", - "Inputs": {"user":"Alice"}, + "Inputs": {"User":"Alice"}, "IsEnabled": { "Result": "true" }, @@ -38,7 +38,7 @@ { "FriendlyName": "TargetedGroup", "FeatureFlagName": "ComplexTargeting", - "Inputs": {"user":"Aiden", "groups":["Stage1"]}, + "Inputs": {"User":"Aiden", "Groups":["Stage1"]}, "IsEnabled": { "Result": "true" }, @@ -50,7 +50,7 @@ { "FriendlyName": "DisabledTargetedGroup", "FeatureFlagName": "ComplexTargeting", - "Inputs": {"groups":["Stage2"]}, + "Inputs": {"Groups":["Stage2"]}, "IsEnabled": { "Result": "false" }, @@ -62,7 +62,7 @@ { "FriendlyName": "EnabledTargetedGroup50", "FeatureFlagName": "ComplexTargeting", - "Inputs": {"user":"Aiden", "groups":["Stage2"]}, + "Inputs": {"User":"Aiden", "Groups":["Stage2"]}, "IsEnabled": { "Result": "true" }, @@ -74,7 +74,7 @@ { "FriendlyName": "DisabledTargetedGroup50", "FeatureFlagName": "ComplexTargeting", - "Inputs": {"user":"Chris", "groups":["Stage2"]}, + "Inputs": {"User":"Chris", "Groups":["Stage2"]}, "IsEnabled": { "Result": "false" }, @@ -86,7 +86,7 @@ { "FriendlyName": "ExcludedGroup", "FeatureFlagName": "ComplexTargeting", - "Inputs": {"groups":["Stage3"]}, + "Inputs": {"Groups":["Stage3"]}, "IsEnabled": { "Result": "false" }, @@ -98,7 +98,7 @@ { "FriendlyName": "ExcludedGroupTargetedUser", "FeatureFlagName": "ComplexTargeting", - "Inputs": {"user":"Alice", "groups":["Stage3"]}, + "Inputs": {"User":"Alice", "Groups":["Stage3"]}, "IsEnabled": { "Result": "false" }, @@ -110,7 +110,7 @@ { "FriendlyName": "ExcludedGroupDefaultRollout", "FeatureFlagName": "ComplexTargeting", - "Inputs": {"user":"Blossom", "groups":["Stage3"]}, + "Inputs": {"User":"Blossom", "Groups":["Stage3"]}, "IsEnabled": { "Result": "false" }, @@ -122,7 +122,7 @@ { "FriendlyName": "ExcludedUser", "FeatureFlagName": "ComplexTargeting", - "Inputs": {"user":"Dave", "groups":["Stage1"]}, + "Inputs": {"User":"Dave", "Groups":["Stage1"]}, "IsEnabled": { "Result": "false" }, @@ -134,7 +134,7 @@ { "FriendlyName": "Aiden61", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Aiden"}, + "Inputs": {"User":"Aiden"}, "IsEnabled": { "Result": "true" }, @@ -146,7 +146,7 @@ { "FriendlyName": "Aiden61 - Stage1", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Aiden", "groups":["Stage1"]}, + "Inputs": {"User":"Aiden", "Groups":["Stage1"]}, "IsEnabled": { "Result": "true" }, @@ -158,7 +158,7 @@ { "FriendlyName": "Aiden61 - Stage2", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Aiden", "groups":["Stage2"]}, + "Inputs": {"User":"Aiden", "Groups":["Stage2"]}, "IsEnabled": { "Result": "true" }, @@ -170,7 +170,7 @@ { "FriendlyName": "Aiden61 - Stage3", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Aiden", "groups":["Stage3"]}, + "Inputs": {"User":"Aiden", "Groups":["Stage3"]}, "IsEnabled": { "Result": "true" }, @@ -182,7 +182,7 @@ { "FriendlyName": "Brittney61", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Brittney"}, + "Inputs": {"User":"Brittney"}, "IsEnabled": { "Result": "false" }, @@ -194,7 +194,7 @@ { "FriendlyName": "Brittney61 - Stage1", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Brittney", "groups":["Stage1"]}, + "Inputs": {"User":"Brittney", "Groups":["Stage1"]}, "IsEnabled": { "Result": "false" }, @@ -206,7 +206,7 @@ { "FriendlyName": "Brittney61 - Stage2", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Brittney", "groups":["Stage2"]}, + "Inputs": {"User":"Brittney", "Groups":["Stage2"]}, "IsEnabled": { "Result": "false" }, @@ -218,7 +218,7 @@ { "FriendlyName": "Brittney61 - Stage3", "FeatureFlagName": "RolloutPercentageUpdate", - "Inputs": {"user":"Brittney", "groups":["Stage3"]}, + "Inputs": {"User":"Brittney", "Groups":["Stage3"]}, "IsEnabled": { "Result": "false" }, diff --git a/Samples/VariantAssignment.tests.json b/Samples/VariantAssignment.tests.json index f91f1fa..6fa8323 100644 --- a/Samples/VariantAssignment.tests.json +++ b/Samples/VariantAssignment.tests.json @@ -2,148 +2,170 @@ { "FeatureFlagName": "UserAssignedVariant", "Inputs": { - "user": "Britney" + "User": "Britney" }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": "The Variant Beta." + "Result": { + "ConfigurationValue": "The Variant Beta." + } }, "Description": "A Feature Flag with two Variants, one for Adam and one for Britney." }, { "FeatureFlagName": "UserAssignedVariant", "Inputs": { - "user": "Adam" + "User": "Adam" }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": "The Variant Alpha." + "Result": { + "ConfigurationValue": "The Variant Alpha." + } }, "Description": "A Feature Flag with two Variants, one for Adam and one for Britney." }, { "FeatureFlagName": "GroupAssignedVariant", "Inputs": { - "user": "Britney", - "groups": ["Ring2"] + "User": "Britney", + "Groups": ["Ring2"] }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": "The Variant Beta." + "Result": { + "ConfigurationValue": "The Variant Beta." + } }, "Description": "A Feature Flag with two Variants, one for Ring1 and one for Ring2." }, { "FeatureFlagName": "GroupAssignedVariant", "Inputs": { - "user": "Britney", - "groups": ["Ring1"] + "User": "Britney", + "Groups": ["Ring1"] }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": "The Variant Alpha." + "Result": { + "ConfigurationValue": "The Variant Alpha." + } }, "Description": "A Feature Flag with two Variants, one for Ring1 and one for Ring2." }, { "FeatureFlagName": "AllocationAssignedVariant", "Inputs": { - "user": "Britney" + "User": "Britney" }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": "The Variant Beta." + "Result": { + "ConfigurationValue": "The Variant Beta." + } }, "Description": "A Feature Flag with two Variants, each with 50%." }, { "FeatureFlagName": "AllocationAssignedVariant", "Inputs": { - "user": "Adam" + "User": "Adam" }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": "The Variant Alpha." + "Result": { + "ConfigurationValue": "The Variant Alpha." + } }, "Description": "A Feature Flag with two Variants, each with 50%." }, { "FeatureFlagName": "ComplexAssignment", "Inputs": { - "user": "Adam" + "User": "Adam" }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": "The Variant Alpha." + "Result": { + "ConfigurationValue": "The Variant Alpha." + } }, "Description": "A Feature Flag with two Variants, with user, group, and percientile assignment." }, { "FeatureFlagName": "ComplexAssignment", "Inputs": { - "user": "Britney" + "User": "Britney" }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": "The Variant Beta." + "Result": { + "ConfigurationValue": "The Variant Beta." + } }, "Description": "A Feature Flag with two Variants, with user, group, and percientile assignment." }, { "FeatureFlagName": "ComplexAssignment", "Inputs": { - "user": "John", - "groups": ["Ring1"] + "User": "John", + "Groups": ["Ring1"] }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": "The Variant Alpha." + "Result": { + "ConfigurationValue": "The Variant Alpha." + } }, "Description": "A Feature Flag with two Variants, with user, group, and percientile assignment." }, { "FeatureFlagName": "ComplexAssignment", "Inputs": { - "user": "Jane", - "groups": ["Ring3"] + "User": "Jane", + "Groups": ["Ring3"] }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": "The Variant Alpha." + "Result": { + "ConfigurationValue": "The Variant Alpha." + } }, "Description": "A Feature Flag with two Variants, with user, group, and percientile assignment." }, { "FeatureFlagName": "ComplexAssignment", "Inputs": { - "user": "Selena", - "groups": ["Ring4"] + "User": "Selena", + "Groups": ["Ring4"] }, "IsEnabled": { "Result": "true" }, "Variant": { - "Result": "The Variant Beta." + "Result": { + "ConfigurationValue": "The Variant Beta." + } }, "Description": "A Feature Flag with two Variants, with user, group, and percientile assignment." } diff --git a/libraryValidations/Dotnet/Dotnet.csproj b/libraryValidations/Dotnet/Dotnet.csproj new file mode 100644 index 0000000..402face --- /dev/null +++ b/libraryValidations/Dotnet/Dotnet.csproj @@ -0,0 +1,23 @@ + + + + net9.0 + latest + enable + enable + + true + + + + + + + + + + + diff --git a/libraryValidations/Dotnet/Dotnet.sln b/libraryValidations/Dotnet/Dotnet.sln new file mode 100644 index 0000000..61f7364 --- /dev/null +++ b/libraryValidations/Dotnet/Dotnet.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.002.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Dotnet", "Dotnet.csproj", "{F19DEACC-9B50-45D3-80F2-C619344722D9}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {F19DEACC-9B50-45D3-80F2-C619344722D9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F19DEACC-9B50-45D3-80F2-C619344722D9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F19DEACC-9B50-45D3-80F2-C619344722D9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F19DEACC-9B50-45D3-80F2-C619344722D9}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {08D18984-BEEF-4BF5-94DC-DBD8096154DC} + EndGlobalSection +EndGlobal diff --git a/libraryValidations/Dotnet/MSTestSettings.cs b/libraryValidations/Dotnet/MSTestSettings.cs new file mode 100644 index 0000000..aaf278c --- /dev/null +++ b/libraryValidations/Dotnet/MSTestSettings.cs @@ -0,0 +1 @@ +[assembly: Parallelize(Scope = ExecutionScope.MethodLevel)] diff --git a/libraryValidations/Dotnet/Properties/launchSettings.json b/libraryValidations/Dotnet/Properties/launchSettings.json new file mode 100644 index 0000000..0bed4f3 --- /dev/null +++ b/libraryValidations/Dotnet/Properties/launchSettings.json @@ -0,0 +1,10 @@ +{ + "profiles": { + "Dotnet": { + "commandName": "Project", + "environmentVariables": { + "APP_CONFIG_VALIDATION_CONNECTION_STRING": "Endpoint=https://appconfig4ndakx7twshlm.azconfig.io;Id=A3jj;Secret=1WWpbi6jB5Bw9sYAvrHfSteqInR0UCZhwA2ynipVN1LcutGRncMyJQQJ99AJACYeBjFAArohAAACAZACIycQ" + } + } + } +} \ No newline at end of file diff --git a/libraryValidations/Dotnet/README.md b/libraryValidations/Dotnet/README.md new file mode 100644 index 0000000..514e7bc --- /dev/null +++ b/libraryValidations/Dotnet/README.md @@ -0,0 +1,19 @@ +# .NET Validation Tests + +This directory contains a .NET script that can be used to validate the correctness of the library against the files in the `Samples` directory. + +## Prerequisites + +* .NET 9 or later + +## Running the tests + +To run the tests, execute the following command: + +```bash +dotnet test +``` + +## Update to run more tests + +To add more tests, after creating the required json files in the `Samples` directory, add a new test method in the `TestValidations.cs` file. diff --git a/libraryValidations/Dotnet/SharedTest.cs b/libraryValidations/Dotnet/SharedTest.cs new file mode 100644 index 0000000..69a6509 --- /dev/null +++ b/libraryValidations/Dotnet/SharedTest.cs @@ -0,0 +1,43 @@ +using System.Text.Json; + +namespace DotnetFeatureManagementTests; + +internal class SharedTest +{ + public required string FeatureFlagName { get; set; } + public InputsSection? Inputs { get; set; } + public IsEnabledSection? IsEnabled { get; set; } + public VariantSection? Variant { get; set; } + public TelemetrySection? Telemetry { get; set; } + public string? Description { get; set; } + + + internal class InputsSection + { + public string? User { get; set; } + public string[]? Groups { get; set; } + } + + internal class IsEnabledSection + { + public string? Result { get; set; } + public string? Exception { get; set; } + } + + internal class VariantSection + { + public VariantResultSection? Result { get; set; } + public string? Exception { get; set; } + } + + internal class VariantResultSection + { + public JsonElement? ConfigurationValue { get; set; } + } + + internal class TelemetrySection + { + public string? EventName { get; set; } + public Dictionary? EventProperties { get; set; } + } +} diff --git a/libraryValidations/Dotnet/TestValidations.cs b/libraryValidations/Dotnet/TestValidations.cs new file mode 100644 index 0000000..1b2629d --- /dev/null +++ b/libraryValidations/Dotnet/TestValidations.cs @@ -0,0 +1,251 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.FeatureManagement; +using Microsoft.FeatureManagement.FeatureFilters; +using System.Diagnostics; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace DotnetFeatureManagementTests; + +[TestClass] +public class Tests +{ + private const string FilePath = "../../../../../Samples/"; + private const string SampleJsonKey = ".sample.json"; + private const string TestsJsonKey = ".tests.json"; + + private ActivityListener? _activityListener; + + private readonly JsonSerializerOptions options = new() + { + Converters = { new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) } + }; + + [TestMethod] + [DataRow("NoFilters")] + [DataRow("TimeWindowFilter")] + [DataRow("TargetingFilter")] + [DataRow("TargetingFilter.modified")] + [DataRow("RequirementType")] + [DataRow("BasicVariant")] + public async Task RunTestFile(string fileName) + { + // Use Sample JSON as Configuration + string file = Directory.GetFiles(FilePath, fileName + SampleJsonKey).First(); + + ConfigurationBuilder builder = new(); + + builder.AddJsonFile(Path.GetFullPath(file)); + + IConfiguration configuration = builder.Build(); + + // Setup FeatureManagement + IServiceCollection services = new ServiceCollection(); + + services.AddSingleton(configuration) + .AddFeatureManagement(); + + ServiceProvider serviceProvider = services.BuildServiceProvider(); + IVariantFeatureManager featureManager = serviceProvider.GetRequiredService(); + + await RunTests(featureManager, fileName); + } + + [TestMethod] + public async Task RunProviderTestFile() + { + // Use Provider for Configuration + IConfiguration configuration = new ConfigurationBuilder() + .AddAzureAppConfiguration(o => + { + o.Connect(Environment.GetEnvironmentVariable("APP_CONFIG_VALIDATION_CONNECTION_STRING")); + + o.UseFeatureFlags(); + }) + .Build(); + + // Setup FeatureManagement + IServiceCollection services = new ServiceCollection(); + + services.AddSingleton(configuration) + .AddFeatureManagement(); + + ServiceProvider serviceProvider = services.BuildServiceProvider(); + IVariantFeatureManager featureManager = serviceProvider.GetRequiredService(); + + // Can't use DataRow because ActivityListener requires these are run sequentially + await RunTests(featureManager, "ProviderTelemetry"); + await RunTests(featureManager, "ProviderTelemetryComplete"); + } + + private async Task RunTests(IVariantFeatureManager featureManager, string fileName) + { + // Get Test Suite JSON + var featureFlagTests = JsonSerializer.Deserialize(File.ReadAllText(FilePath + fileName + TestsJsonKey), options); + + if (featureFlagTests == null) { + throw new Exception("Test failed to parse JSON"); + } + + // Run each test + foreach (var featureFlagTest in featureFlagTests) + { + string featureFlagId = fileName + "." + featureFlagTest.FeatureFlagName; + string failedDescription = $"Test {featureFlagId} failed. Description: {featureFlagTest.Description}"; + + // Setup Activity Listener if we're validating Telemetry + if (featureFlagTest.Telemetry != null) + { + SetupActivityListener(featureFlagTest, failedDescription); + } + + // IsEnabledAsync + if (featureFlagTest.IsEnabled != null) + { + bool? expectedIsEnabledResult = featureFlagTest.IsEnabled.Result != null ? Convert.ToBoolean(featureFlagTest.IsEnabled.Result) : null; + + if (featureFlagTest.IsEnabled.Exception != null) + { + await Assert.ThrowsExceptionAsync(async () => await featureManager.IsEnabledAsync(featureFlagTest.FeatureFlagName), featureFlagTest.IsEnabled.Exception); + } + else + { + bool isEnabledResult = await featureManager.IsEnabledAsync(featureFlagTest.FeatureFlagName, new TargetingContext { UserId = featureFlagTest.Inputs?.User, Groups = featureFlagTest.Inputs?.Groups }); + Assert.AreEqual(expectedIsEnabledResult, isEnabledResult, failedDescription); + } + } + + // GetVariantAsync + if (featureFlagTest.Variant != null) + { + if (featureFlagTest.Variant.Exception != null) + { + await Assert.ThrowsExceptionAsync(async () => await featureManager.GetVariantAsync(featureFlagTest.FeatureFlagName), featureFlagTest.Variant.Exception); + } + else + { + Variant variantResult = await featureManager.GetVariantAsync(featureFlagTest.FeatureFlagName, new TargetingContext { UserId = featureFlagTest.Inputs?.User, Groups = featureFlagTest.Inputs?.Groups }); + + if (featureFlagTest.Variant.Result == null) + { + Assert.IsNull(variantResult); + } + else + { + ValidateJsonConfigurationValue(featureFlagTest.Variant.Result.ConfigurationValue, variantResult.Configuration); + } + } + } + + if (featureFlagTest.Telemetry != null) + { + _activityListener?.Dispose(); + } + } + } + + private void SetupActivityListener(SharedTest featureFlagTest, string failedDescription) + { + _activityListener = new ActivityListener + { + ShouldListenTo = (activitySource) => activitySource.Name == "Microsoft.FeatureManagement", + Sample = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllData, + ActivityStopped = (activity) => + { + ActivityEvent? evaluationEvent = activity.Events.FirstOrDefault((activityEvent) => activityEvent.Name == "FeatureFlag"); + + if (evaluationEvent.HasValue && evaluationEvent.Value.Tags.Any()) + { + Dictionary evaluationEventProperties = evaluationEvent.Value.Tags.ToDictionary((tag) => tag.Key, (tag) => tag.Value); + + if (featureFlagTest.Telemetry != null && featureFlagTest.Telemetry.EventProperties != null) + { + foreach (var property in featureFlagTest.Telemetry.EventProperties) + { + Assert.IsTrue(evaluationEventProperties.ContainsKey(property.Key), failedDescription); + + if (property.Key == "FeatureFlagReference") + { + Assert.IsTrue(evaluationEventProperties[property.Key]?.ToString()?.EndsWith(property.Value)); + } + else + { + Assert.AreEqual(property.Value, evaluationEventProperties[property.Key]?.ToString(), failedDescription); + } + } + } + } + } + }; + + ActivitySource.AddActivityListener(_activityListener); + } + + private void ValidateJsonConfigurationValue(JsonElement? ele, IConfigurationSection configuration) + { + if (ele == null) + { + Assert.IsNull(configuration.Get()); + } + else + { + if (ele.Value.ValueKind == JsonValueKind.Object) + { + foreach (var property in ele.Value.EnumerateObject()) + { + ValidateProperty(property.Value, configuration.GetSection(property.Name)); + } + } + else if (ele.Value.ValueKind == JsonValueKind.Array) + { + for (int i = 0; i < ele.Value.GetArrayLength(); i++) + { + ValidateProperty(ele.Value[i], configuration.GetSection(i.ToString())); + } + } + else + { + ValidateProperty(ele.Value, configuration); + } + } + } + + private void ValidateProperty(JsonElement value, IConfigurationSection configuration) + { + if (value.ValueKind == JsonValueKind.String) + { + Assert.AreEqual(value.GetString(), configuration?.Get()); + } + else if (value.ValueKind == JsonValueKind.Number) + { + Assert.AreEqual(value.GetDouble(), configuration?.Get()); + } + else if (value.ValueKind == JsonValueKind.Null) + { + Assert.IsNull(configuration?.Get()); + } + else if (value.ValueKind == JsonValueKind.False) + { + Assert.IsFalse(configuration?.Get()); + } + else if (value.ValueKind == JsonValueKind.True) + { + Assert.IsTrue(configuration?.Get()); + } + else if (value.ValueKind == JsonValueKind.Object) + { + ValidateJsonConfigurationValue(value, configuration); + } + else if (value.ValueKind == JsonValueKind.Array) + { + ValidateJsonConfigurationValue(value, configuration); + } else + { + Assert.Fail(); + } + } +} \ No newline at end of file diff --git a/libraryValidations/Dotnet/UnknownJsonFieldConverter.cs b/libraryValidations/Dotnet/UnknownJsonFieldConverter.cs new file mode 100644 index 0000000..c9cbcd8 --- /dev/null +++ b/libraryValidations/Dotnet/UnknownJsonFieldConverter.cs @@ -0,0 +1,42 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace DotnetFeatureManagementTests; + +internal class UnknownJsonFieldConverter : JsonConverter +{ + public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + switch (reader.TokenType) + { + case JsonTokenType.String: + return reader.GetString(); + case JsonTokenType.Number: + if (reader.TryGetInt32(out int intValue)) + return intValue; + if (reader.TryGetDouble(out double doubleValue)) + return doubleValue; + break; + case JsonTokenType.True: + return reader.GetBoolean(); + case JsonTokenType.False: + return reader.GetBoolean(); + case JsonTokenType.StartObject: + using (JsonDocument doc = JsonDocument.ParseValue(ref reader)) + { + return doc.RootElement.Clone(); + } + case JsonTokenType.StartArray: + using (JsonDocument doc = JsonDocument.ParseValue(ref reader)) + { + return doc.RootElement.Clone(); + } + } + throw new JsonException("Unknown JSON token type."); + } + + public override void Write(Utf8JsonWriter writer, object value, JsonSerializerOptions options) + { + JsonSerializer.Serialize(writer, value, value.GetType(), options); + } +} diff --git a/libraryValidations/Python/test_json_validations.py b/libraryValidations/Python/test_json_validations.py index b6ab5db..16911d7 100644 --- a/libraryValidations/Python/test_json_validations.py +++ b/libraryValidations/Python/test_json_validations.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- + import logging import json import unittest @@ -16,10 +17,12 @@ IS_ENABLED_KEY = "IsEnabled" GET_VARIANT_KEY = "Variant" RESULT_KEY = "Result" +VARIANT_NAME_KEY = "Name" +CONFIGURATION_VALUE_KEY = "ConfigurationValue" FEATURE_FLAG_NAME_KEY = "FeatureFlagName" INPUTS_KEY = "Inputs" -USER_KEY = "user" -GROUPS_KEY = "groups" +USER_KEY = "User" +GROUPS_KEY = "Groups" EXCEPTION_KEY = "Exception" DESCRIPTION_KEY = "Description" @@ -70,6 +73,7 @@ def test_basic_variant(self): test_key = "BasicVariant" self.run_tests(test_key) + # method: is_enabled def test_variant_assignment(self): test_key = "VariantAssignment" @@ -115,11 +119,15 @@ def run_tests(self, test_key): expected_message = is_enabled.get(EXCEPTION_KEY) assert str(ex_info.value) == expected_message, failed_description - if get_variant is not None and get_variant[RESULT_KEY]: + if get_variant is not None and RESULT_KEY in get_variant: user = feature_flag_test[INPUTS_KEY].get(USER_KEY, None) groups = feature_flag_test[INPUTS_KEY].get(GROUPS_KEY, []) variant = feature_manager.get_variant(feature_flag_test[FEATURE_FLAG_NAME_KEY], TargetingContext(user_id=user, groups=groups)) - if not variant: - logger.error(f"Variant is None for {feature_flag_id}") - assert False, failed_description - assert variant.configuration == get_variant[RESULT_KEY], failed_description + + if get_variant[RESULT_KEY] == None: + assert variant == None + else: + if VARIANT_NAME_KEY in get_variant[RESULT_KEY]: + assert variant.name == get_variant[RESULT_KEY][VARIANT_NAME_KEY], failed_description + + assert variant.configuration == get_variant[RESULT_KEY][CONFIGURATION_VALUE_KEY], failed_description diff --git a/libraryValidations/Python/test_json_validations_with_provider.py b/libraryValidations/Python/test_json_validations_with_provider.py index 7464d49..835db5d 100644 --- a/libraryValidations/Python/test_json_validations_with_provider.py +++ b/libraryValidations/Python/test_json_validations_with_provider.py @@ -19,10 +19,12 @@ IS_ENABLED_KEY = "IsEnabled" GET_VARIANT_KEY = "Variant" RESULT_KEY = "Result" +VARIANT_NAME_KEY = "Name" +CONFIGURATION_VALUE_KEY = "ConfigurationValue" FEATURE_FLAG_NAME_KEY = "FeatureFlagName" INPUTS_KEY = "Inputs" -USER_KEY = "user" -GROUPS_KEY = "groups" +USER_KEY = "User" +GROUPS_KEY = "Groups" EXCEPTION_KEY = "Exception" DESCRIPTION_KEY = "Description" @@ -97,15 +99,17 @@ def run_tests(self, test_key, track_event_mock): groups = feature_flag_test[INPUTS_KEY].get(GROUPS_KEY, []) variant = feature_manager.get_variant(feature_flag_test[FEATURE_FLAG_NAME_KEY], TargetingContext(user_id=user, groups=groups)) assert variant, failed_description - assert variant.configuration == get_variant[RESULT_KEY], failed_description + if VARIANT_NAME_KEY in get_variant[RESULT_KEY]: + assert variant.name == get_variant[RESULT_KEY][VARIANT_NAME_KEY], failed_description + assert variant.configuration == get_variant[RESULT_KEY][CONFIGURATION_VALUE_KEY], failed_description if telemetry: assert track_event_mock.called assert track_event_mock.call_count == 2 - assert track_event_mock.call_args[0][0] == telemetry["event_name"] + assert track_event_mock.call_args[0][0] == telemetry["EventName"] event = track_event_mock.call_args[0][1] - event_properties = telemetry["event_properties"] + event_properties = telemetry["EventProperties"] connection_string = os.getenv("APP_CONFIG_VALIDATION_CONNECTION_STRING") endpoint = endpoint_from_connection_string(connection_string) diff --git a/libraryValidations/Spring/validation-tests/pom.xml b/libraryValidations/Spring/validation-tests/pom.xml index a2b4d19..b13988a 100644 --- a/libraryValidations/Spring/validation-tests/pom.xml +++ b/libraryValidations/Spring/validation-tests/pom.xml @@ -37,7 +37,7 @@ com.azure.spring spring-cloud-azure-feature-management - 5.15.0 + 5.17.0 diff --git a/libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/ValidationTestsApplicationTests.java b/libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/ValidationTestsApplicationTests.java index cb0ad24..c266cdd 100644 --- a/libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/ValidationTestsApplicationTests.java +++ b/libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/ValidationTestsApplicationTests.java @@ -31,9 +31,9 @@ class ValidationTestsApplicationTests { static final String TEST_FILE_POSTFIX = ".tests.json"; - private final String inputsUser = "user"; + private final String inputsUser = "User"; - private final String inputsGroups = "groups"; + private final String inputsGroups = "Groups"; @Autowired private FeatureManager featureManager; diff --git a/libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/models/ValidationTestCase.java b/libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/models/ValidationTestCase.java index bc0c243..218d1c5 100644 --- a/libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/models/ValidationTestCase.java +++ b/libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/models/ValidationTestCase.java @@ -9,7 +9,7 @@ public class ValidationTestCase { private String featureFlagName; private LinkedHashMap inputs; private IsEnabled isEnabled; - private Variant variant; + private VariantResult variant; private String description; /** @@ -71,14 +71,14 @@ public void setIsEnabled(IsEnabled isEnabled) { /** * @return variant * */ - public Variant getVariant() { + public VariantResult getVariant() { return variant; } /** * @param variant the variant of test case * */ - public void setVariant(Variant variant) { + public void setVariant(VariantResult variant) { this.variant = variant; } diff --git a/libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/models/Variant.java b/libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/models/Variant.java index 2bcb4b9..16c8981 100644 --- a/libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/models/Variant.java +++ b/libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/models/Variant.java @@ -3,34 +3,34 @@ package com.microsoft.validation_tests.models; public class Variant { - private String result; - private String exception; + private String name; + private Object configurationValue; /** - * @return result + * @return name * */ - public String getResult() { - return result; + public String getName() { + return name; } /** - * @param result the result of variant feature flag + * @param name the name of variant feature flag * */ - public void setResult(String result) { - this.result = result; + public void setName(String name) { + this.name = name; } /** - * @return exception + * @return configurationValue * */ - public String getException() { - return exception; + public Object getConfigurationValue() { + return configurationValue; } /** - * @param exception the exception message throws when run variant test case + * @param configurationValue the configurationValue of the variant * */ - public void setException(String exception) { - this.exception = exception; + public void setConfigurationValue(Object exception) { + this.configurationValue = exception; } } diff --git a/libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/models/VariantResult.java b/libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/models/VariantResult.java new file mode 100644 index 0000000..e45590c --- /dev/null +++ b/libraryValidations/Spring/validation-tests/src/test/java/com/microsoft/validation_tests/models/VariantResult.java @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +package com.microsoft.validation_tests.models; + +public class VariantResult { + private Variant result; + private String exception; + + /** + * @return result + * */ + public Variant getResult() { + return result; + } + + /** + * @param result the variant result of variant feature flag + * */ + public void setResult(Variant result) { + this.result = result; + } + + /** + * @return exception + * */ + public String getException() { + return exception; + } + + /** + * @param exception the exception message throws when run variant test case + * */ + public void setException(String exception) { + this.exception = exception; + } +} diff --git a/libraryValidations/run_all_tests.ps1 b/libraryValidations/run_all_tests.ps1 new file mode 100644 index 0000000..c2961ff --- /dev/null +++ b/libraryValidations/run_all_tests.ps1 @@ -0,0 +1,17 @@ +# Navigate to the Dotnet directory and run dotnet test +Set-Location -Path "./Dotnet" +dotnet test + +# Navigate to the Python directory and run pytest +Set-Location -Path "../Python" +python -m venv venv +.\venv\Scripts\activate.ps1 +pip install -r requirements.txt +pytest +deactivate + +# Navigate to the Spring directory and run mvn test +Set-Location -Path "../Spring/validation-tests" +mvn test + +Set-Location -Path "../.." \ No newline at end of file From b89eb3046c2c8991d15ab1a03618b47257e21599 Mon Sep 17 00:00:00 2001 From: Matt Metcalf Date: Thu, 7 Nov 2024 11:18:38 -0800 Subject: [PATCH 3/5] PascelCase & Formatting --- Samples/BasicTelemetry.tests.json | 11 +- .../Python/test_json_validations.py | 69 +++++++----- .../test_json_validations_with_provider.py | 106 ++++++++++++++---- 3 files changed, 128 insertions(+), 58 deletions(-) diff --git a/Samples/BasicTelemetry.tests.json b/Samples/BasicTelemetry.tests.json index bd214d8..f3352c4 100644 --- a/Samples/BasicTelemetry.tests.json +++ b/Samples/BasicTelemetry.tests.json @@ -1,16 +1,19 @@ [ { "FeatureFlagName": "TelemetryVariant", - "Inputs": {"user":"Aiden"}, + "Inputs": {"User":"Aiden"}, "IsEnabled": { "Result": "false" }, "Variant": { - "Result": "default" + "Result": { + "Name": "True_Override", + "ConfigurationValue": "default" + } }, "Telemetry": { - "event_name": "FeatureEvaluation", - "event_properties": { + "EventName": "FeatureEvaluation", + "EventProperties": { "FeatureName": "TelemetryVariant", "Enabled": "False", "Version": "1.0.0", diff --git a/libraryValidations/Python/test_json_validations.py b/libraryValidations/Python/test_json_validations.py index 25d5394..6c206d7 100644 --- a/libraryValidations/Python/test_json_validations.py +++ b/libraryValidations/Python/test_json_validations.py @@ -4,7 +4,6 @@ # license information. # -------------------------------------------------------------------------- -import logging import json import unittest from pytest import raises @@ -19,8 +18,8 @@ IS_ENABLED_KEY = "IsEnabled" GET_VARIANT_KEY = "Variant" GET_TELEMETRY_KEY = "Telemetry" -EVENT_NAME_KEY = "event_name" -EVENT_PROPERTIES_KEY = "event_properties" +EVENT_NAME_KEY = "EventName" +EVENT_PROPERTIES_KEY = "EventProperties" RESULT_KEY = "Result" VARIANT_NAME_KEY = "Name" CONFIGURATION_VALUE_KEY = "ConfigurationValue" @@ -31,9 +30,6 @@ EXCEPTION_KEY = "Exception" DESCRIPTION_KEY = "Description" -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - def convert_boolean_value(enabled): if enabled is None: @@ -48,43 +44,35 @@ def convert_boolean_value(enabled): class TestFromFile(unittest.TestCase): - # method: is_enabled + def test_no_filters(self): test_key = "NoFilters" self.run_tests(test_key) - # method: is_enabled def test_time_window_filter(self): test_key = "TimeWindowFilter" self.run_tests(test_key) - # method: is_enabled def test_targeting_filter(self): test_key = "TargetingFilter" self.run_tests(test_key) - # method: is_enabled def test_targeting_filter_modified(self): test_key = "TargetingFilter.modified" self.run_tests(test_key) - # method: is_enabled def test_requirement_type(self): test_key = "RequirementType" self.run_tests(test_key) - # method: is_enabled def test_basic_variant(self): test_key = "BasicVariant" self.run_tests(test_key) - - # method: is_enabled def test_variant_assignment(self): test_key = "VariantAssignment" self.run_tests(test_key) - # method: is_enabled @patch("featuremanagement.azuremonitor._send_telemetry.azure_monitor_track_event") def test_basic_telemetry(self, track_event_mock): test_key = "BasicTelemetry" @@ -98,12 +86,9 @@ def load_from_file(file, telemetry_callback=None): with open(FILE_PATH + file, "r", encoding="utf-8") as feature_flags_file: feature_flags = json.load(feature_flags_file) - if telemetry_callback: - feature_manager = FeatureManager( - feature_flags, on_feature_evaluated=telemetry_callback - ) - else: - feature_manager = FeatureManager(feature_flags) + feature_manager = FeatureManager( + feature_flags, on_feature_evaluated=telemetry_callback + ) assert feature_manager is not None return feature_manager @@ -154,20 +139,42 @@ def run_tests(self, test_key, telemetry_callback=None): ), failed_description else: with raises(ValueError) as ex_info: - feature_manager.is_enabled(feature_flag_test[FEATURE_FLAG_NAME_KEY]) + feature_manager.is_enabled( + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), + ) expected_message = is_enabled.get(EXCEPTION_KEY) assert str(ex_info.value) == expected_message, failed_description - if get_variant is not None and RESULT_KEY in get_variant: + if get_variant and ( + RESULT_KEY in get_variant or EXCEPTION_KEY in get_variant + ): user = feature_flag_test[INPUTS_KEY].get(USER_KEY, None) groups = feature_flag_test[INPUTS_KEY].get(GROUPS_KEY, []) - variant = feature_manager.get_variant(feature_flag_test[FEATURE_FLAG_NAME_KEY], TargetingContext(user_id=user, groups=groups)) + if RESULT_KEY not in get_variant: + with raises(ValueError) as ex_info: + feature_manager.get_variant( + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), + ) + expected_message = get_variant.get(EXCEPTION_KEY) + assert str(ex_info.value) == expected_message, failed_description + continue + + variant = feature_manager.get_variant( + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), + ) + if not get_variant[RESULT_KEY]: + assert not variant + continue + if VARIANT_NAME_KEY in get_variant[RESULT_KEY]: + assert ( + variant.name == get_variant[RESULT_KEY][VARIANT_NAME_KEY] + ), failed_description - if get_variant[RESULT_KEY] == None: - assert variant == None - else: - if VARIANT_NAME_KEY in get_variant[RESULT_KEY]: - assert variant.name == get_variant[RESULT_KEY][VARIANT_NAME_KEY], failed_description - - assert variant.configuration == get_variant[RESULT_KEY][CONFIGURATION_VALUE_KEY], failed_description + assert ( + variant.configuration + == get_variant[RESULT_KEY][CONFIGURATION_VALUE_KEY] + ), failed_description diff --git a/libraryValidations/Python/test_json_validations_with_provider.py b/libraryValidations/Python/test_json_validations_with_provider.py index 835db5d..b219dc2 100644 --- a/libraryValidations/Python/test_json_validations_with_provider.py +++ b/libraryValidations/Python/test_json_validations_with_provider.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- + from unittest.mock import patch import json import unittest @@ -28,6 +29,7 @@ EXCEPTION_KEY = "Exception" DESCRIPTION_KEY = "Description" + def convert_boolean_value(enabled): if enabled is None: return None @@ -44,36 +46,45 @@ class TestFromProvider(unittest.TestCase): def test_provider_telemetry(self): test_key = "ProviderTelemetry" - with patch("featuremanagement.azuremonitor._send_telemetry.azure_monitor_track_event") as mock_track_event: + with patch( + "featuremanagement.azuremonitor._send_telemetry.azure_monitor_track_event" + ) as mock_track_event: self.run_tests(test_key, mock_track_event) def test_complete_provider_telemetry(self): test_key = "ProviderTelemetryComplete" - with patch("featuremanagement.azuremonitor._send_telemetry.azure_monitor_track_event") as mock_track_event: + with patch( + "featuremanagement.azuremonitor._send_telemetry.azure_monitor_track_event" + ) as mock_track_event: self.run_tests(test_key, mock_track_event) @staticmethod def load_from_provider(): connection_string = os.getenv("APP_CONFIG_VALIDATION_CONNECTION_STRING") - config = load(connection_string=connection_string, selects=[], feature_flag_enabled=True) + config = load( + connection_string=connection_string, selects=[], feature_flag_enabled=True + ) feature_manager = FeatureManager(config, on_feature_evaluated=publish_telemetry) assert feature_manager is not None return feature_manager - def run_tests(self, test_key, track_event_mock): feature_manager = self.load_from_provider() - with open(FILE_PATH + test_key + TESTS_JSON_KEY, "r", encoding="utf-8") as feature_flag_test_file: + with open( + FILE_PATH + test_key + TESTS_JSON_KEY, "r", encoding="utf-8" + ) as feature_flag_test_file: feature_flag_tests = json.load(feature_flag_test_file) for feature_flag_test in feature_flag_tests: track_event_mock.reset_mock() is_enabled = feature_flag_test[IS_ENABLED_KEY] get_variant = feature_flag_test.get(GET_VARIANT_KEY, None) - expected_is_enabled_result = convert_boolean_value(is_enabled.get(RESULT_KEY)) + expected_is_enabled_result = convert_boolean_value( + is_enabled.get(RESULT_KEY) + ) feature_flag_id = test_key + "." + feature_flag_test[FEATURE_FLAG_NAME_KEY] telemetry = feature_flag_test.get("Telemetry", None) @@ -84,24 +95,52 @@ def run_tests(self, test_key, track_event_mock): groups = feature_flag_test[INPUTS_KEY].get(GROUPS_KEY, []) assert ( feature_manager.is_enabled( - feature_flag_test[FEATURE_FLAG_NAME_KEY], TargetingContext(user_id=user, groups=groups) + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), ) == expected_is_enabled_result ), failed_description else: with raises(ValueError) as ex_info: - feature_manager.is_enabled(feature_flag_test[FEATURE_FLAG_NAME_KEY]) + feature_manager.is_enabled( + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), + ) expected_message = is_enabled.get(EXCEPTION_KEY) assert str(ex_info.value) == expected_message, failed_description - if get_variant is not None and get_variant[RESULT_KEY]: + if get_variant and ( + RESULT_KEY in get_variant or EXCEPTION_KEY in get_variant + ): user = feature_flag_test[INPUTS_KEY].get(USER_KEY, None) groups = feature_flag_test[INPUTS_KEY].get(GROUPS_KEY, []) - variant = feature_manager.get_variant(feature_flag_test[FEATURE_FLAG_NAME_KEY], TargetingContext(user_id=user, groups=groups)) - assert variant, failed_description + + if RESULT_KEY not in get_variant: + with raises(ValueError) as ex_info: + feature_manager.get_variant( + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), + ) + expected_message = get_variant.get(EXCEPTION_KEY) + assert str(ex_info.value) == expected_message, failed_description + continue + + variant = feature_manager.get_variant( + feature_flag_test[FEATURE_FLAG_NAME_KEY], + TargetingContext(user_id=user, groups=groups), + ) + if not get_variant[RESULT_KEY]: + assert not variant + continue if VARIANT_NAME_KEY in get_variant[RESULT_KEY]: - assert variant.name == get_variant[RESULT_KEY][VARIANT_NAME_KEY], failed_description - assert variant.configuration == get_variant[RESULT_KEY][CONFIGURATION_VALUE_KEY], failed_description + assert ( + variant.name == get_variant[RESULT_KEY][VARIANT_NAME_KEY] + ), failed_description + + assert ( + variant.configuration + == get_variant[RESULT_KEY][CONFIGURATION_VALUE_KEY] + ), failed_description if telemetry: assert track_event_mock.called @@ -110,22 +149,43 @@ def run_tests(self, test_key, track_event_mock): event = track_event_mock.call_args[0][1] event_properties = telemetry["EventProperties"] - connection_string = os.getenv("APP_CONFIG_VALIDATION_CONNECTION_STRING") + connection_string = os.getenv( + "APP_CONFIG_VALIDATION_CONNECTION_STRING" + ) endpoint = endpoint_from_connection_string(connection_string) assert event["FeatureName"] == event_properties["FeatureName"] assert event["Enabled"] == event_properties["Enabled"] assert event["Version"] == event_properties["Version"] assert event["Variant"] == event_properties["Variant"] - assert event["VariantAssignmentReason"] == event_properties["VariantAssignmentReason"] - - if "VariantAssignmentPercentage" in event: # User/Group assignment doesn't have this property - assert event["VariantAssignmentPercentage"] == event_properties["VariantAssignmentPercentage"] - - assert event["DefaultWhenEnabled"] == event_properties["DefaultWhenEnabled"] - assert event["ETag"] # ETag will be different for each store, just assert it exists - assert event["FeatureFlagReference"] == endpoint + event_properties["FeatureFlagReference"] - assert event["FeatureFlagId"].decode("utf-8") == event_properties["FeatureFlagId"] + assert ( + event["VariantAssignmentReason"] + == event_properties["VariantAssignmentReason"] + ) + + if ( + "VariantAssignmentPercentage" in event + ): # User/Group assignment doesn't have this property + assert ( + event["VariantAssignmentPercentage"] + == event_properties["VariantAssignmentPercentage"] + ) + + assert ( + event["DefaultWhenEnabled"] + == event_properties["DefaultWhenEnabled"] + ) + assert event[ + "ETag" + ] # ETag will be different for each store, just assert it exists + assert ( + event["FeatureFlagReference"] + == endpoint + event_properties["FeatureFlagReference"] + ) + assert ( + event["FeatureFlagId"].decode("utf-8") + == event_properties["FeatureFlagId"] + ) assert event["AllocationId"] == event_properties["AllocationId"] assert event["TargetingId"] == event_properties["TargetingId"] From a3bac4f4548308ee06d26a698530755c49e7ead4 Mon Sep 17 00:00:00 2001 From: Matt Metcalf Date: Mon, 11 Nov 2024 10:44:03 -0800 Subject: [PATCH 4/5] Delete launchSettings.json --- .../Dotnet/Properties/launchSettings.json | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 libraryValidations/Dotnet/Properties/launchSettings.json diff --git a/libraryValidations/Dotnet/Properties/launchSettings.json b/libraryValidations/Dotnet/Properties/launchSettings.json deleted file mode 100644 index 0bed4f3..0000000 --- a/libraryValidations/Dotnet/Properties/launchSettings.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "profiles": { - "Dotnet": { - "commandName": "Project", - "environmentVariables": { - "APP_CONFIG_VALIDATION_CONNECTION_STRING": "Endpoint=https://appconfig4ndakx7twshlm.azconfig.io;Id=A3jj;Secret=1WWpbi6jB5Bw9sYAvrHfSteqInR0UCZhwA2ynipVN1LcutGRncMyJQQJ99AJACYeBjFAArohAAACAZACIycQ" - } - } - } -} \ No newline at end of file From 868192eda6407c7fcaa5f14a3def4eba87876bf1 Mon Sep 17 00:00:00 2001 From: Matt Metcalf Date: Wed, 11 Dec 2024 12:07:38 -0800 Subject: [PATCH 5/5] fixings tests and review comments --- libraryValidations/Python/requirements.txt | 4 ++-- .../Python/test_json_validations.py | 17 ++++++----------- .../test_json_validations_with_provider.py | 4 +--- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/libraryValidations/Python/requirements.txt b/libraryValidations/Python/requirements.txt index 9afc3b4..435aee3 100644 --- a/libraryValidations/Python/requirements.txt +++ b/libraryValidations/Python/requirements.txt @@ -1,3 +1,3 @@ pytest -featuremanagement["AzureMonitor"]==2.0.0b2 -azure-appconfiguration-provider==2.0.0b2 +featuremanagement["AzureMonitor"]==2.0.0b3 +azure-appconfiguration-provider==2.0.0b3 diff --git a/libraryValidations/Python/test_json_validations.py b/libraryValidations/Python/test_json_validations.py index 6c206d7..1f6bf76 100644 --- a/libraryValidations/Python/test_json_validations.py +++ b/libraryValidations/Python/test_json_validations.py @@ -96,15 +96,12 @@ def load_from_file(file, telemetry_callback=None): def telemetry_callback(self, evaluation_event): publish_telemetry(evaluation_event) expected_telemetry = self._feature_flag_test.get(GET_TELEMETRY_KEY, {}) - self._mock_track_event.assert_has_calls( - [ - call( - expected_telemetry.get(EVENT_NAME_KEY, None), - expected_telemetry.get(EVENT_PROPERTIES_KEY, None), - ) - ] - ) + self._mock_track_event.assert_called_once() + self.assertEqual(self._mock_track_event.call_args[0][0], expected_telemetry.get(EVENT_NAME_KEY, None)) + (event_properties) = self._mock_track_event.call_args[0][1] + self.assertEqual(sorted(event_properties), sorted(expected_telemetry.get(EVENT_PROPERTIES_KEY, {}))) self._ran_callback = True + self._mock_track_event.reset_mock() def run_tests(self, test_key, telemetry_callback=None): feature_manager = self.load_from_file( @@ -146,9 +143,7 @@ def run_tests(self, test_key, telemetry_callback=None): expected_message = is_enabled.get(EXCEPTION_KEY) assert str(ex_info.value) == expected_message, failed_description - if get_variant and ( - RESULT_KEY in get_variant or EXCEPTION_KEY in get_variant - ): + if get_variant: user = feature_flag_test[INPUTS_KEY].get(USER_KEY, None) groups = feature_flag_test[INPUTS_KEY].get(GROUPS_KEY, []) diff --git a/libraryValidations/Python/test_json_validations_with_provider.py b/libraryValidations/Python/test_json_validations_with_provider.py index b219dc2..46e902f 100644 --- a/libraryValidations/Python/test_json_validations_with_provider.py +++ b/libraryValidations/Python/test_json_validations_with_provider.py @@ -109,9 +109,7 @@ def run_tests(self, test_key, track_event_mock): expected_message = is_enabled.get(EXCEPTION_KEY) assert str(ex_info.value) == expected_message, failed_description - if get_variant and ( - RESULT_KEY in get_variant or EXCEPTION_KEY in get_variant - ): + if get_variant: user = feature_flag_test[INPUTS_KEY].get(USER_KEY, None) groups = feature_flag_test[INPUTS_KEY].get(GROUPS_KEY, [])