diff --git a/ddtrace/appsec/_remoteconfiguration.py b/ddtrace/appsec/_remoteconfiguration.py index 50b2f0b3f32..88a72a8b054 100644 --- a/ddtrace/appsec/_remoteconfiguration.py +++ b/ddtrace/appsec/_remoteconfiguration.py @@ -20,6 +20,7 @@ if TYPE_CHECKING: # pragma: no cover from typing import Any + from typing import Dict try: from typing import Literal @@ -31,7 +32,6 @@ from typing import Union from ddtrace import Tracer - from ddtrace.internal.remoteconfig.client import ConfigMetadata log = get_logger(__name__) @@ -45,28 +45,31 @@ def enable_appsec_rc(test_tracer=None): else: tracer = test_tracer - appsec_features_callback = RCAppSecFeaturesCallBack(tracer) - appsec_callback = RCAppSecCallBack(tracer) + asm_features_callback = RCAppSecFeaturesCallBack(tracer) + asm_dd_callback = RCASMDDCallBack(tracer) + asm_callback = RCAppSecCallBack(tracer) if _appsec_rc_features_is_enabled(): from ddtrace.internal.remoteconfig import RemoteConfig - RemoteConfig.register(PRODUCTS.ASM_FEATURES, appsec_features_callback) + RemoteConfig.register(PRODUCTS.ASM_FEATURES, asm_features_callback) if tracer._appsec_enabled: from ddtrace.internal.remoteconfig import RemoteConfig - RemoteConfig.register(PRODUCTS.ASM_DATA, appsec_callback) # IP Blocking - RemoteConfig.register(PRODUCTS.ASM, appsec_callback) # Exclusion Filters & Custom Rules - RemoteConfig.register(PRODUCTS.ASM_DD, appsec_callback) # DD Rules + RemoteConfig.register(PRODUCTS.ASM_DATA, asm_callback) # IP Blocking + RemoteConfig.register(PRODUCTS.ASM, asm_callback) # Exclusion Filters & Custom Rules + RemoteConfig.register(PRODUCTS.ASM_DD, asm_dd_callback) # DD Rules -def _add_rules_to_list(features, feature, message, rule_list): - # type: (Mapping[str, Any], str, str, list[Any]) -> None - rules = features.get(feature, []) - if rules: +def _add_rules_to_list(features, feature, message, ruleset): + # type: (Mapping[str, Any], str, str, Dict[str, Any]) -> None + rules = features.get(feature, None) + if rules is not None: try: - rule_list += rules + if ruleset.get(feature) is None: + ruleset[feature] = [] + ruleset[feature] += rules log.debug("Reloading Appsec %s: %s", message, rules) except JSONDecodeError: log.error("ERROR Appsec %s: invalid JSON content from remote configuration", message) @@ -75,18 +78,32 @@ def _add_rules_to_list(features, feature, message, rule_list): def _appsec_rules_data(tracer, features): # type: (Tracer, Mapping[str, Any]) -> bool if features and tracer._appsec_processor: - ruleset = {"rules": [], "rules_data": [], "exclusions": [], "rules_override": []} # type: dict[str, list[Any]] - _add_rules_to_list(features, "rules_data", "rules data", ruleset["rules_data"]) - _add_rules_to_list(features, "custom_rules", "custom rules", ruleset["rules"]) - _add_rules_to_list(features, "rules", "Datadog rules", ruleset["rules"]) - _add_rules_to_list(features, "exclusions", "exclusion filters", ruleset["exclusions"]) - _add_rules_to_list(features, "rules_override", "rules override", ruleset["rules_override"]) - if any(ruleset.values()): - return tracer._appsec_processor._update_rules({k: v for k, v in ruleset.items() if v}) + ruleset = { + "rules": None, + "rules_data": None, + "exclusions": None, + "rules_override": None, + } # type: dict[str, Optional[list[Any]]] + _add_rules_to_list(features, "rules_data", "rules data", ruleset) + _add_rules_to_list(features, "custom_rules", "custom rules", ruleset) + _add_rules_to_list(features, "rules", "Datadog rules", ruleset) + _add_rules_to_list(features, "exclusions", "exclusion filters", ruleset) + _add_rules_to_list(features, "rules_override", "rules override", ruleset) + return tracer._appsec_processor._update_rules({k: v for k, v in ruleset.items() if v is not None}) return False +class RCASMDDCallBack(RemoteConfigCallBack): + def __init__(self, tracer): + # type: (Tracer) -> None + self.tracer = tracer + + def __call__(self, metadata, features): + if features is not None: + _appsec_rules_data(self.tracer, features) + + class RCAppSecFeaturesCallBack(RemoteConfigCallBack): def __init__(self, tracer): # type: (Tracer) -> None @@ -126,10 +143,11 @@ def _appsec_1click_activation(self, features): log.debug("Updating ASM Remote Configuration ASM_FEATURES: %s", rc_appsec_enabled) if rc_appsec_enabled: - appsec_callback = RCAppSecCallBack(self.tracer) - RemoteConfig.register(PRODUCTS.ASM_DATA, appsec_callback) # IP Blocking - RemoteConfig.register(PRODUCTS.ASM, appsec_callback) # Exclusion Filters & Custom Rules - RemoteConfig.register(PRODUCTS.ASM_DD, appsec_callback) # DD Rules + asm_dd_callback = RCASMDDCallBack(self.tracer) + asm_callback = RCAppSecCallBack(self.tracer) + RemoteConfig.register(PRODUCTS.ASM_DATA, asm_callback) # IP Blocking + RemoteConfig.register(PRODUCTS.ASM, asm_callback) # Exclusion Filters & Custom Rules + RemoteConfig.register(PRODUCTS.ASM_DD, asm_dd_callback) # DD Rules if not self.tracer._appsec_enabled: self.tracer.configure(appsec_enabled=True) else: @@ -146,13 +164,12 @@ def _appsec_1click_activation(self, features): class RCAppSecCallBack(RemoteConfigCallBackAfterMerge): - configs = {} - def __init__(self, tracer): # type: (Tracer) -> None + super(RCAppSecCallBack, self).__init__() self.tracer = tracer - def __call__(self, metadata, features): - # type: (Optional[ConfigMetadata], Any) -> None + def __call__(self, target, features): + # type: (str, Any) -> None if features is not None: _appsec_rules_data(self.tracer, features) diff --git a/ddtrace/internal/remoteconfig/client.py b/ddtrace/internal/remoteconfig/client.py index 0d41f43244c..55d8f86f0a2 100644 --- a/ddtrace/internal/remoteconfig/client.py +++ b/ddtrace/internal/remoteconfig/client.py @@ -154,17 +154,39 @@ def __call__(self, metadata, config): pass -class RemoteConfigCallBackAfterMerge(RemoteConfigCallBack): +class RemoteConfigCallBackAfterMerge(six.with_metaclass(abc.ABCMeta)): configs = {} # type: Dict[str, Any] - def append(self, config): - self.configs.update(config) + @abc.abstractmethod + def __call__(self, target, config): + # type: (str, Any) -> None + pass + + def append(self, target, config): + if not self.configs.get(target): + self.configs[target] = {} + if config is False: + # Remove old config from the configs dict. _remove_previously_applied_configurations function should + # call to this method + del self.configs[target] + elif config is not None: + # Append the new config to the configs dict. _load_new_configurations function should + # call to this method + if isinstance(config, dict): + self.configs[target].update(config) + else: + raise ValueError("target %s config %s has type of %s" % (target, config, type(config))) def dispatch(self): - try: - self.__call__(None, self.configs) - finally: - self.configs = {} + config_result = {} + for target, config in self.configs.items(): + for key, value in config.items(): + if isinstance(value, list): + config_result[key] = config_result.get(key, []) + value + else: + raise ValueError("target %s key %s has type of %s" % (target, key, type(value))) + if config_result: + self.__call__("", config_result) class RemoteConfigClient(object): @@ -351,10 +373,22 @@ def _process_targets(self, payload): backend_state = signed.custom.get("opaque_backend_state") return signed.version, backend_state, targets + @staticmethod + def _apply_callback(list_callbacks, callback, config_content, target, config): + # type: (List[RemoteConfigCallBackAfterMerge], Any, Any, str, ConfigMetadata) -> None + if isinstance(callback, RemoteConfigCallBackAfterMerge): + callback.append(target, config_content) + if callback not in list_callbacks and not any( + filter(lambda x: isinstance(x, type(callback)), list_callbacks) + ): + list_callbacks.append(callback) + else: + callback(config, config_content) + def _remove_previously_applied_configurations(self, applied_configs, client_configs, targets): # type: (AppliedConfigType, TargetsType, TargetsType) -> None + list_callbacks = [] # type: List[RemoteConfigCallBackAfterMerge] for target, config in self._applied_configs.items(): - callback_action = None if target in client_configs and targets.get(target) == config: # The configuration has not changed. applied_configs[target] = config @@ -362,18 +396,24 @@ def _remove_previously_applied_configurations(self, applied_configs, client_conf elif target not in client_configs: log.debug("Disable configuration: %s", target) callback_action = False + else: + continue callback = self._products.get(config.product_name) if callback: try: - callback(config, callback_action) + log.debug("Disable configuration 2: %s. ", target) + self._apply_callback(list_callbacks, callback, callback_action, target, config) except Exception: log.debug("error while removing product %s config %r", config.product_name, config) continue + for callback_to_dispach in list_callbacks: + callback_to_dispach.dispatch() + def _load_new_configurations(self, applied_configs, client_configs, payload): # type: (AppliedConfigType, TargetsType, AgentPayload) -> None - list_callbacks = [] + list_callbacks = [] # type: List[RemoteConfigCallBackAfterMerge] for target, config in client_configs.items(): callback = self._products.get(config.product_name) if callback: @@ -387,12 +427,7 @@ def _load_new_configurations(self, applied_configs, client_configs, payload): try: log.debug("Load new configuration: %s. content ", target) - if isinstance(callback, RemoteConfigCallBackAfterMerge): - callback.append(config_content) - if callback not in list_callbacks: - list_callbacks.append(callback) - else: - callback(config, config_content) + self._apply_callback(list_callbacks, callback, config_content, target, config) except Exception: log.debug( "Failed to load configuration %s for product %r", config, config.product_name, exc_info=True @@ -400,8 +435,9 @@ def _load_new_configurations(self, applied_configs, client_configs, payload): continue else: applied_configs[target] = config - for callback in list_callbacks: - callback.dispatch() + + for callback_to_dispach in list_callbacks: + callback_to_dispach.dispatch() def _add_apply_config_to_cache(self): if self._applied_configs: diff --git a/tests/appsec/test_remoteconfiguration.py b/tests/appsec/test_remoteconfiguration.py index bf731e6f9db..12b6fadef6d 100644 --- a/tests/appsec/test_remoteconfiguration.py +++ b/tests/appsec/test_remoteconfiguration.py @@ -15,6 +15,7 @@ from ddtrace.appsec._remoteconfiguration import RCAppSecFeaturesCallBack from ddtrace.appsec._remoteconfiguration import _appsec_rules_data from ddtrace.appsec._remoteconfiguration import enable_appsec_rc +from ddtrace.appsec.processor import AppSecSpanProcessor from ddtrace.appsec.utils import _appsec_rc_capabilities from ddtrace.appsec.utils import _appsec_rc_features_is_enabled from ddtrace.constants import APPSEC_ENV @@ -160,6 +161,38 @@ def test_rc_activation_check_asm_features_product_disables_rest_of_products(trac assert RemoteConfig._worker._client._products.get(PRODUCTS.ASM_FEATURES) +def test_load_new_configurations_invalid_content(remote_config_worker, tracer): + with override_global_config(dict(_appsec_enabled=True, api_version="v0.4")): + tracer.configure(appsec_enabled=True, api_version="v0.4") + enable_appsec_rc(tracer) + asm_features_data = b'{"asm":{"enabled":true}}' + asm_data_data = b'{"data": "data"}' + payload = AgentPayload( + target_files=[ + TargetFile(path="mock/ASM_FEATURES", raw=base64.b64encode(asm_features_data)), + TargetFile(path="mock/ASM_DATA", raw=base64.b64encode(asm_data_data)), + ] + ) + client_configs = { + "mock/ASM_FEATURES": ConfigMetadata( + id="", + product_name="ASM_FEATURES", + sha256_hash=hashlib.sha256(asm_features_data).hexdigest(), + length=5, + tuf_version=5, + ), + "mock/ASM_DATA": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data).hexdigest(), + length=5, + tuf_version=5, + ), + } + with pytest.raises(ValueError): + RemoteConfig._worker._client._load_new_configurations({}, client_configs, payload=payload) + + @mock.patch.object(RCAppSecFeaturesCallBack, "_appsec_1click_activation") @mock.patch("ddtrace.appsec._remoteconfiguration._appsec_rules_data") def test_load_new_configurations_dispatch_applied_configs( @@ -169,7 +202,45 @@ def test_load_new_configurations_dispatch_applied_configs( tracer.configure(appsec_enabled=True, api_version="v0.4") enable_appsec_rc(tracer) asm_features_data = b'{"asm":{"enabled":true}}' - asm_data_data = b'{"data":{}}' + asm_data_data = b'{"data": [{"test": "data"}]}' + payload = AgentPayload( + target_files=[ + TargetFile(path="mock/ASM_FEATURES", raw=base64.b64encode(asm_features_data)), + TargetFile(path="mock/ASM_DATA", raw=base64.b64encode(asm_data_data)), + ] + ) + client_configs = { + "mock/ASM_FEATURES": ConfigMetadata( + id="", + product_name="ASM_FEATURES", + sha256_hash=hashlib.sha256(asm_features_data).hexdigest(), + length=5, + tuf_version=5, + ), + "mock/ASM_DATA": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data).hexdigest(), + length=5, + tuf_version=5, + ), + } + + RemoteConfig._worker._client._load_new_configurations({}, client_configs, payload=payload) + mock_appsec_rules_data.assert_called_with(ANY, {"data": [{"test": "data"}]}) + mock_appsec_1click_activation.assert_called_with({"asm": {"enabled": True}}) + + +@mock.patch.object(RCAppSecFeaturesCallBack, "_appsec_1click_activation") +@mock.patch("ddtrace.appsec._remoteconfiguration._appsec_rules_data") +def test_load_new_configurations_empty_config( + mock_appsec_rules_data, mock_appsec_1click_activation, remote_config_worker, tracer +): + with override_global_config(dict(_appsec_enabled=True, api_version="v0.4")): + tracer.configure(appsec_enabled=True, api_version="v0.4") + enable_appsec_rc(tracer) + asm_features_data = b'{"asm":{"enabled":true}}' + asm_data_data = b'{"data": []}' payload = AgentPayload( target_files=[ TargetFile(path="mock/ASM_FEATURES", raw=base64.b64encode(asm_features_data)), @@ -194,7 +265,7 @@ def test_load_new_configurations_dispatch_applied_configs( } RemoteConfig._worker._client._load_new_configurations({}, client_configs, payload=payload) - mock_appsec_rules_data.assert_called_with(ANY, {"data": {}}) + mock_appsec_rules_data.assert_called_with(ANY, {"data": []}) mock_appsec_1click_activation.assert_called_with({"asm": {"enabled": True}}) @@ -283,6 +354,487 @@ def test_load_new_configurations_remove_config_and_dispatch_applied_configs_erro RemoteConfig._worker._client._load_new_configurations({}, client_configs, payload=payload) +@pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason="Mock return order is different in python <= 3.5") +@mock.patch.object(RCAppSecFeaturesCallBack, "_appsec_1click_activation") +@mock.patch("ddtrace.appsec._remoteconfiguration._appsec_rules_data") +def test_load_multiple_targets_file_same_product( + mock_appsec_rules_data, mock_appsec_1click_activation, remote_config_worker, tracer +): + with override_global_config(dict(_appsec_enabled=True, api_version="v0.4")): + tracer.configure(appsec_enabled=True, api_version="v0.4") + enable_appsec_rc(tracer) + asm_features_data = b'{"asm":{"enabled":true}}' + asm_data_data1 = b'{"data": [{"a":1}]}' + asm_data_data2 = b'{"data": [{"b":2}]}' + payload = AgentPayload( + target_files=[ + TargetFile(path="mock/ASM_FEATURES", raw=base64.b64encode(asm_features_data)), + TargetFile(path="mock/ASM_DATA/1", raw=base64.b64encode(asm_data_data1)), + TargetFile(path="mock/ASM_DATA/2", raw=base64.b64encode(asm_data_data2)), + ] + ) + client_configs = { + "mock/ASM_FEATURES": ConfigMetadata( + id="", + product_name="ASM_FEATURES", + sha256_hash=hashlib.sha256(asm_features_data).hexdigest(), + length=5, + tuf_version=5, + ), + "mock/ASM_DATA/1": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data1).hexdigest(), + length=5, + tuf_version=5, + ), + "mock/ASM_DATA/2": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data2).hexdigest(), + length=5, + tuf_version=5, + ), + } + + RemoteConfig._worker._client._load_new_configurations({}, client_configs, payload=payload) + mock_appsec_rules_data.assert_called_with(ANY, {"data": [{"a": 1}, {"b": 2}]}) + mock_appsec_1click_activation.assert_called_with({"asm": {"enabled": True}}) + + +@mock.patch("ddtrace.appsec._remoteconfiguration._appsec_rules_data") +def test_remove_targets_file_with_previous_configuration(mock_appsec_rules_data, remote_config_worker, tracer): + with override_global_config(dict(_appsec_enabled=True, api_version="v0.4")): + RCAppSecCallBack.configs = {} + tracer.configure(appsec_enabled=True, api_version="v0.4") + enable_appsec_rc(tracer) + asm_data_data1 = b'{"data": [{"a":1}]}' + asm_data_data2 = b'{"data": [{"b":2}]}' + payload = AgentPayload( + target_files=[ + TargetFile(path="mock/ASM_DATA/1", raw=base64.b64encode(asm_data_data1)), + TargetFile(path="mock/ASM_DATA/2", raw=base64.b64encode(asm_data_data2)), + ] + ) + applied_configs = { + "mock/ASM_DATA/1": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data1).hexdigest(), + length=5, + tuf_version=5, + ), + "mock/ASM_DATA/2": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data2).hexdigest(), + length=5, + tuf_version=5, + ), + } + + client_configs = { + "mock/ASM_DATA/1": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data1).hexdigest(), + length=5, + tuf_version=5, + ), + } + + target_file = { + "mock/ASM_DATA/2": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data2).hexdigest(), + length=5, + tuf_version=5, + ) + } + + RemoteConfig._worker._client._applied_configs = applied_configs + RemoteConfig._worker._client._remove_previously_applied_configurations({}, client_configs, target_file) + + RemoteConfig._worker._client._load_new_configurations({}, client_configs, payload=payload) + mock_appsec_rules_data.assert_not_called() + + +@pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason="Mock return order is different in python <= 3.5") +@mock.patch.object(RCAppSecFeaturesCallBack, "_appsec_1click_activation") +@mock.patch("ddtrace.appsec._remoteconfiguration._appsec_rules_data") +def test_load_new_config_and_remove_targets_file_same_product( + mock_appsec_rules_data, mock_appsec_1click_activation, remote_config_worker, tracer +): + with override_global_config(dict(_appsec_enabled=True, api_version="v0.4")): + tracer.configure(appsec_enabled=True, api_version="v0.4") + applied_configs = {} + enable_appsec_rc(tracer) + asm_features_data = b'{"asm":{"enabled":true}}' + asm_data_data1 = b'{"data": [{"a":1}]}' + asm_data_data2 = b'{"data": [{"b":2}]}' + payload = AgentPayload( + target_files=[ + TargetFile(path="mock/ASM_FEATURES", raw=base64.b64encode(asm_features_data)), + TargetFile(path="mock/ASM_DATA/1", raw=base64.b64encode(asm_data_data1)), + TargetFile(path="mock/ASM_DATA/2", raw=base64.b64encode(asm_data_data2)), + ] + ) + first_config = { + "mock/ASM_FEATURES": ConfigMetadata( + id="", + product_name="ASM_FEATURES", + sha256_hash=hashlib.sha256(asm_features_data).hexdigest(), + length=5, + tuf_version=5, + ), + "mock/ASM_DATA/1": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data1).hexdigest(), + length=5, + tuf_version=5, + ), + "mock/ASM_DATA/2": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data2).hexdigest(), + length=5, + tuf_version=5, + ), + } + + second_config = { + "mock/ASM_FEATURES": ConfigMetadata( + id="", + product_name="ASM_FEATURES", + sha256_hash=hashlib.sha256(asm_features_data).hexdigest(), + length=5, + tuf_version=5, + ), + "mock/ASM_DATA/1": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data1).hexdigest(), + length=5, + tuf_version=5, + ), + } + + target_file = { + "mock/ASM_DATA/2": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data2).hexdigest(), + length=5, + tuf_version=5, + ) + } + RemoteConfig._worker._client._remove_previously_applied_configurations({}, first_config, {}) + + RemoteConfig._worker._client._load_new_configurations(applied_configs, first_config, payload=payload) + RemoteConfig._worker._client._applied_configs = applied_configs + + mock_appsec_rules_data.assert_called_with(ANY, {"data": [{"a": 1}, {"b": 2}]}) + mock_appsec_rules_data.reset_mock() + + RemoteConfig._worker._client._remove_previously_applied_configurations({}, second_config, target_file) + + RemoteConfig._worker._client._load_new_configurations({}, second_config, payload=payload) + mock_appsec_rules_data.assert_called_with(ANY, {"data": [{"a": 1}]}) + + +@pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason="Mock return order is different in python <= 3.5") +@mock.patch.object(AppSecSpanProcessor, "_update_rules") +def test_fullpath_appsec_rules_data(mock_update_rules, remote_config_worker, tracer): + with override_global_config(dict(_appsec_enabled=True, api_version="v0.4")): + tracer.configure(appsec_enabled=True, api_version="v0.4") + applied_configs = {} + enable_appsec_rc(tracer) + asm_features_data = b'{"asm":{"enabled":true}}' + asm_data_data1 = b'{"exclusions": [{"a":1}]}' + asm_data_data2 = b'{"exclusions": [{"b":2}]}' + payload = AgentPayload( + target_files=[ + TargetFile(path="mock/ASM_FEATURES", raw=base64.b64encode(asm_features_data)), + TargetFile(path="mock/ASM_DATA/1", raw=base64.b64encode(asm_data_data1)), + TargetFile(path="mock/ASM_DATA/2", raw=base64.b64encode(asm_data_data2)), + ] + ) + first_config = { + "mock/ASM_FEATURES": ConfigMetadata( + id="", + product_name="ASM_FEATURES", + sha256_hash=hashlib.sha256(asm_features_data).hexdigest(), + length=5, + tuf_version=5, + ), + "mock/ASM_DATA/1": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data1).hexdigest(), + length=5, + tuf_version=5, + ), + "mock/ASM_DATA/2": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data2).hexdigest(), + length=5, + tuf_version=5, + ), + } + + second_config = { + "mock/ASM_FEATURES": ConfigMetadata( + id="", + product_name="ASM_FEATURES", + sha256_hash=hashlib.sha256(asm_features_data).hexdigest(), + length=5, + tuf_version=5, + ), + "mock/ASM_DATA/1": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data1).hexdigest(), + length=5, + tuf_version=5, + ), + } + + target_file = { + "mock/ASM_DATA/2": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data2).hexdigest(), + length=5, + tuf_version=5, + ) + } + RemoteConfig._worker._client._remove_previously_applied_configurations({}, first_config, {}) + + RemoteConfig._worker._client._load_new_configurations(applied_configs, first_config, payload=payload) + RemoteConfig._worker._client._applied_configs = applied_configs + + mock_update_rules.assert_called_with({"exclusions": [{"a": 1}, {"b": 2}]}) + mock_update_rules.reset_mock() + + RemoteConfig._worker._client._remove_previously_applied_configurations({}, second_config, target_file) + + RemoteConfig._worker._client._load_new_configurations({}, second_config, payload=payload) + mock_update_rules.assert_called_with({"exclusions": [{"a": 1}]}) + + +@pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason="Mock return order is different in python <= 3.5") +@mock.patch.object(AppSecSpanProcessor, "_update_rules") +def test_fullpath_appsec_rules_data_empty_data(mock_update_rules, remote_config_worker, tracer): + with override_global_config(dict(_appsec_enabled=True, api_version="v0.4")): + tracer.configure(appsec_enabled=True, api_version="v0.4") + applied_configs = {} + enable_appsec_rc(tracer) + asm_data_data1 = b'{"exclusions": [{"a":1}]}' + asm_data_data2 = b'{"exclusions": []}' + payload = AgentPayload( + target_files=[ + TargetFile(path="mock/ASM_DATA/1", raw=base64.b64encode(asm_data_data1)), + TargetFile(path="mock/ASM_DATA/2", raw=base64.b64encode(asm_data_data2)), + ] + ) + first_config = { + "mock/ASM_DATA/1": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data1).hexdigest(), + length=5, + tuf_version=5, + ), + "mock/ASM_DATA/2": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data2).hexdigest(), + length=5, + tuf_version=5, + ), + } + + second_config = { + "mock/ASM_DATA/1": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data1).hexdigest(), + length=5, + tuf_version=5, + ), + } + + target_file = { + "mock/ASM_DATA/2": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data2).hexdigest(), + length=5, + tuf_version=5, + ) + } + RemoteConfig._worker._client._remove_previously_applied_configurations({}, first_config, {}) + + RemoteConfig._worker._client._load_new_configurations(applied_configs, first_config, payload=payload) + RemoteConfig._worker._client._applied_configs = applied_configs + + mock_update_rules.assert_called_with({"exclusions": [{"a": 1}]}) + mock_update_rules.reset_mock() + + RemoteConfig._worker._client._remove_previously_applied_configurations({}, second_config, target_file) + + RemoteConfig._worker._client._load_new_configurations({}, second_config, payload=payload) + mock_update_rules.assert_called_with({"exclusions": [{"a": 1}]}) + + +@pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason="Mock return order is different in python <= 3.5") +@mock.patch.object(AppSecSpanProcessor, "_update_rules") +def test_fullpath_appsec_rules_data_add_delete_file(mock_update_rules, remote_config_worker, tracer): + with override_global_config(dict(_appsec_enabled=True, api_version="v0.4")): + tracer.configure(appsec_enabled=True, api_version="v0.4") + applied_configs = {} + enable_appsec_rc(tracer) + asm_data_data1 = b'{"exclusions": [{"a":1}]}' + asm_data_data2 = b'{"exclusions": []}' + payload = AgentPayload( + target_files=[ + TargetFile(path="mock/ASM_DATA/1", raw=base64.b64encode(asm_data_data1)), + ] + ) + first_config = { + "mock/ASM_DATA/1": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data1).hexdigest(), + length=5, + tuf_version=5, + ), + } + + second_payload = AgentPayload( + target_files=[ + TargetFile(path="mock/ASM_DATA/1", raw=base64.b64encode(asm_data_data2)), + ] + ) + second_config = { + "mock/ASM_DATA/1": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data2).hexdigest(), + length=5, + tuf_version=5, + ), + } + + target_file = { + "mock/ASM_DATA/1": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data1).hexdigest(), + length=5, + tuf_version=5, + ) + } + RemoteConfig._worker._client._remove_previously_applied_configurations({}, first_config, {}) + + RemoteConfig._worker._client._load_new_configurations(applied_configs, first_config, payload=payload) + RemoteConfig._worker._client._applied_configs = applied_configs + + mock_update_rules.assert_called_with({"exclusions": [{"a": 1}]}) + mock_update_rules.reset_mock() + + RemoteConfig._worker._client._remove_previously_applied_configurations({}, second_config, target_file) + + RemoteConfig._worker._client._load_new_configurations({}, second_config, payload=second_payload) + mock_update_rules.assert_called_with({"exclusions": []}) + + +@pytest.mark.skipif(sys.version_info[:2] < (3, 6), reason="Mock return order is different in python <= 3.5") +@mock.patch.object(RCAppSecFeaturesCallBack, "_appsec_1click_activation") +@mock.patch("ddtrace.appsec._remoteconfiguration._appsec_rules_data") +def test_load_new_empty_config_and_remove_targets_file_same_product( + mock_appsec_rules_data, remote_config_worker, tracer +): + with override_global_config(dict(_appsec_enabled=True, api_version="v0.4")): + tracer.configure(appsec_enabled=True, api_version="v0.4") + RCAppSecCallBack.configs = {} + applied_configs = {} + enable_appsec_rc(tracer) + asm_features_data = b'{"asm":{"enabled":true}}' + asm_data_data1 = b'{"data": [{"a":1}]}' + asm_data_data2 = b'{"data2": [{"b":2}]}' + asm_data_data_empty = b'{"data2": []}' + payload = AgentPayload( + target_files=[ + TargetFile(path="mock/ASM_FEATURES", raw=base64.b64encode(asm_features_data)), + TargetFile(path="mock/ASM_DATA/1", raw=base64.b64encode(asm_data_data1)), + TargetFile(path="mock/ASM_DATA/2", raw=base64.b64encode(asm_data_data2)), + ] + ) + first_config = { + "mock/ASM_FEATURES": ConfigMetadata( + id="", + product_name="ASM_FEATURES", + sha256_hash=hashlib.sha256(asm_features_data).hexdigest(), + length=5, + tuf_version=5, + ), + "mock/ASM_DATA/1": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data1).hexdigest(), + length=5, + tuf_version=5, + ), + "mock/ASM_DATA/2": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data2).hexdigest(), + length=5, + tuf_version=5, + ), + } + + second_config = { + "mock/ASM_FEATURES": ConfigMetadata( + id="", + product_name="ASM_FEATURES", + sha256_hash=hashlib.sha256(asm_features_data).hexdigest(), + length=5, + tuf_version=5, + ), + "mock/ASM_DATA/2": ConfigMetadata( + id="", + product_name="ASM_DATA", + sha256_hash=hashlib.sha256(asm_data_data_empty).hexdigest(), + length=5, + tuf_version=5, + ), + } + + RemoteConfig._worker._client._remove_previously_applied_configurations({}, first_config, {}) + + RemoteConfig._worker._client._load_new_configurations(applied_configs, first_config, payload=payload) + RemoteConfig._worker._client._applied_configs = {} + + mock_appsec_rules_data.assert_called_with(ANY, {"data": [{"a": 1}], "data2": [{"b": 2}]}) + mock_appsec_rules_data.reset_mock() + + payload = AgentPayload( + target_files=[ + TargetFile(path="mock/ASM_FEATURES", raw=base64.b64encode(asm_features_data)), + TargetFile(path="mock/ASM_DATA/1", raw=base64.b64encode(asm_data_data1)), + TargetFile(path="mock/ASM_DATA/2", raw=base64.b64encode(asm_data_data_empty)), + ] + ) + + RemoteConfig._worker._client._load_new_configurations({}, second_config, payload=payload) + mock_appsec_rules_data.assert_called_with(ANY, {"data": [{"a": 1}], "data2": []}) + + def test_rc_activation_ip_blocking_data(tracer, remote_config_worker): with override_env({APPSEC_ENV: "true"}): tracer.configure(appsec_enabled=True, api_version="v0.4") diff --git a/tests/internal/remoteconfig/test_remoteconfig_client.py b/tests/internal/remoteconfig/test_remoteconfig_client.py index 1bf9af580cb..aac648ac9b0 100644 --- a/tests/internal/remoteconfig/test_remoteconfig_client.py +++ b/tests/internal/remoteconfig/test_remoteconfig_client.py @@ -5,6 +5,7 @@ import pytest from ddtrace.internal.remoteconfig.client import ConfigMetadata +from ddtrace.internal.remoteconfig.client import RemoteConfigCallBack from ddtrace.internal.remoteconfig.client import RemoteConfigCallBackAfterMerge from ddtrace.internal.remoteconfig.client import RemoteConfigClient from ddtrace.internal.remoteconfig.client import RemoteConfigError @@ -47,7 +48,7 @@ class MockExtractFile: def __call__(self, payload, target, config): self.counter += 1 - result = {"test{}".format(self.counter): target} + result = {"test{}".format(self.counter): [target]} expected_results.update(result) return result @@ -72,7 +73,7 @@ def __call__(self, payload, target, config): rc_client._load_new_configurations(applied_configs, client_configs, payload=payload) - mock_callback.assert_called_once_with(None, expected_results) + mock_callback.assert_called_once_with("", expected_results) assert applied_configs == client_configs rc_client._products = {} @@ -265,3 +266,137 @@ def test_remote_config_client_tags_override(): assert tags["env"] == "foo" assert tags["version"] == "bar" + + +def test_apply_default_callback(): + class callbackClass(RemoteConfigCallBack): + result = None + + def __call__(self, *args, **kwargs): + self.result = args + + callback = callbackClass() + callback_content = {"a": 1} + target = "1/ASM/2" + config = {"Config": "data"} + test_list_callbacks = [] + RemoteConfigClient._apply_callback(test_list_callbacks, callback, callback_content, target, config) + + assert callback.result == ({"Config": "data"}, {"a": 1}) + assert test_list_callbacks == [] + + +def test_apply_merge_callback(): + class callbackClass(RemoteConfigCallBackAfterMerge): + result = None + + def __call__(self, *args, **kwargs): + self.result = args + + callback = callbackClass() + callback_content = {"a": [1]} + target = "1/ASM/2" + config = {"Config": "data"} + test_list_callbacks = [] + RemoteConfigClient._apply_callback(test_list_callbacks, callback, callback_content, target, config) + + assert len(test_list_callbacks) == 1 + test_list_callbacks[0].dispatch() + + assert callback.result == ("", {"a": [1]}) + + +def test_apply_merge_multiple_callback(): + class callbackClass(RemoteConfigCallBackAfterMerge): + result = None + + def __call__(self, *args, **kwargs): + self.result = args + + callback1 = callbackClass() + callback2 = callbackClass() + callback_content1 = {"a": [1]} + callback_content2 = {"b": [2]} + target = "1/ASM/2" + config = {"Config": "data"} + test_list_callbacks = [] + RemoteConfigClient._apply_callback(test_list_callbacks, callback1, callback_content1, target, config) + RemoteConfigClient._apply_callback(test_list_callbacks, callback2, callback_content2, target, config) + + assert len(test_list_callbacks) == 1 + test_list_callbacks[0].dispatch() + + assert callback1.result == ("", {"a": [1], "b": [2]}) + assert callback2.result is None + + +def test_apply_merge_different_callback(): + class callback1And2Class(RemoteConfigCallBackAfterMerge): + configs = {} + result = None + + def __call__(self, *args, **kwargs): + self.result = args + + class callback3Class(RemoteConfigCallBackAfterMerge): + configs = {} + result = None + + def __call__(self, *args, **kwargs): + self.result = args + + callback1 = callback1And2Class() + callback2 = callback1And2Class() + callback3 = callback3Class() + callback_content1 = {"a": [1]} + callback_content2 = {"b": [2]} + target = "1/ASM/2" + config = {"Config": "data"} + test_list_callbacks = [] + RemoteConfigClient._apply_callback(test_list_callbacks, callback1, callback_content1, target, config) + RemoteConfigClient._apply_callback(test_list_callbacks, callback2, callback_content2, target, config) + RemoteConfigClient._apply_callback(test_list_callbacks, callback3, callback_content2, target, config) + + assert len(test_list_callbacks) == 2 + test_list_callbacks[0].dispatch() + test_list_callbacks[1].dispatch() + + assert callback1.result == ("", {"a": [1], "b": [2]}) + assert callback2.result is None + assert callback3.result == ("", {"b": [2]}) + + +def test_apply_merge_different_target_callback(): + class callback1And2Class(RemoteConfigCallBackAfterMerge): + configs = {} + result = None + + def __call__(self, *args, **kwargs): + self.result = args + + class callback3Class(RemoteConfigCallBackAfterMerge): + configs = {} + result = None + + def __call__(self, *args, **kwargs): + self.result = args + + callback1 = callback1And2Class() + callback2 = callback1And2Class() + callback3 = callback3Class() + callback_content1 = {"a": [1]} + callback_content2 = {"b": [2]} + callback_content3 = {"b": [3]} + config = {"Config": "data"} + test_list_callbacks = [] + RemoteConfigClient._apply_callback(test_list_callbacks, callback1, callback_content1, "1/ASM/1", config) + RemoteConfigClient._apply_callback(test_list_callbacks, callback2, callback_content2, "1/ASM/2", config) + RemoteConfigClient._apply_callback(test_list_callbacks, callback3, callback_content3, "1/ASM/3", config) + + assert len(test_list_callbacks) == 2 + test_list_callbacks[0].dispatch() + test_list_callbacks[1].dispatch() + + assert callback1.result == ("", {"a": [1], "b": [2]}) + assert callback2.result is None + assert callback3.result == ("", {"b": [3]}) diff --git a/tests/internal/remoteconfig/test_remoteconfig_client_e2e.py b/tests/internal/remoteconfig/test_remoteconfig_client_e2e.py index 75dfaf4d54f..cda08b37e18 100644 --- a/tests/internal/remoteconfig/test_remoteconfig_client_e2e.py +++ b/tests/internal/remoteconfig/test_remoteconfig_client_e2e.py @@ -4,8 +4,8 @@ import mock +from ddtrace.appsec._remoteconfiguration import RCASMDDCallBack from ddtrace.internal import runtime -from ddtrace.internal.remoteconfig.client import RemoteConfigCallBackAfterMerge from ddtrace.internal.remoteconfig.client import RemoteConfigClient from ddtrace.internal.utils.version import _pep440_to_semver from tests.utils import override_env @@ -73,9 +73,7 @@ def test_remote_config_client_steps(mock_appsec_rc_capabilities, mock_send_reque with open(MOCK_AGENT_RESPONSES_FILE, "r") as f: MOCK_AGENT_RESPONSES = json.load(f) - class RCAppSecCallBack(RemoteConfigCallBackAfterMerge): - configs = {} - + class RCAppSecMockCallBack(RCASMDDCallBack): def __call__(self, metadata, features): mock_callback(metadata, features) @@ -83,7 +81,7 @@ def __call__(self, metadata, features): rc_client = RemoteConfigClient() mock_callback = mock.mock.MagicMock() - callback = RCAppSecCallBack() + callback = RCAppSecMockCallBack(mock.mock.MagicMock()) rc_client.register_product("ASM_FEATURES", callback) with override_env(dict(DD_REMOTE_CONFIGURATION_ENABLED="false")):