diff --git a/CHANGELOG.md b/CHANGELOG.md index c63df30..0f7a874 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# v0.0.6 + +## Improvements +- Added fallback for configuration spaces with conditions resulting in all configurations being filtered out. +- Added caching and a function in RandomConfigSpaceSearcher to ensure monotonicity of the value function. + # v0.0.5 ## Features diff --git a/src/hypershap/utils.py b/src/hypershap/utils.py index 294ad6b..9bce172 100644 --- a/src/hypershap/utils.py +++ b/src/hypershap/utils.py @@ -161,8 +161,67 @@ def search(self, coalition: np.ndarray) -> float: ) # predict performance values with the help of the surrogate model for the filtered configurations - vals: np.ndarray = np.array(self.explanation_task.get_single_surrogate_model().evaluate(filtered_samples)) + if len(filtered_samples) > 0: + vals: np.ndarray = np.array( + self.explanation_task.get_single_surrogate_model().evaluate(filtered_samples), + ) + else: + logger.warning( + "WARNING: After filtering for conditions, no configurations were left, thus, using baseline value.", + ) + vals = np.array([self.search(np.array([False] * len(coalition)))]) else: vals: np.ndarray = np.array(self.explanation_task.get_single_surrogate_model().evaluate(temp_random_sample)) - return evaluate_aggregation(self.mode, vals) + # determine the final, aggregated value of the coalition + res = evaluate_aggregation(self.mode, vals) + + # in case we are maximizing or minimizing, ensure that the value function is monotone + if self.mode in (Aggregation.MAX, Aggregation.MIN): + res = self._ensure_monotonicity(coalition, res) + + # cache the coalition's value + self.coalition_cache[str(coalition.tolist())] = res + + return res + + def _ensure_monotonicity(self, coalition: np.ndarray, value: float) -> float: + """Ensure that the value function is monotonically increasing/decreasing depending on whether we want to maximize or minimize respectively. + + Args: + coalition: The current coalition. + value: The value of the coalition as determined by searching. + + Returns: The monotonicity-ensured value of the coalition. + + """ + monotone_value = value + checked_one = False + + for i in range(len(coalition)): + if coalition[i]: # check whether the entry is True and set it to False to check for a cached result + temp_coalition = coalition.copy() + temp_coalition[i] = False + if str(temp_coalition.tolist()) in self.coalition_cache: + checked_one = True + monotone_value = evaluate_aggregation( + self.mode, + np.array([ + monotone_value, + self.coalition_cache[str(temp_coalition.tolist())], + ]), + ) + + if not checked_one and coalition.any(): + logger.warning( + "Could not ensure monotonicity as none of the coalitions with one player less has been cached so far.", + ) + + if value < monotone_value: # pragma: no cover + logger.debug( + "Ensured monotonicity with a sub-coalition's value. Increased the value of the current coalition from %s to %s.", + value, + monotone_value, + ) + + return monotone_value diff --git a/tests/fixtures/simple_setup.py b/tests/fixtures/simple_setup.py index 087f245..76bb3bc 100644 --- a/tests/fixtures/simple_setup.py +++ b/tests/fixtures/simple_setup.py @@ -3,7 +3,13 @@ from __future__ import annotations import pytest -from ConfigSpace import Configuration, ConfigurationSpace, LessThanCondition, UniformFloatHyperparameter +from ConfigSpace import ( + Configuration, + ConfigurationSpace, + GreaterThanCondition, + LessThanCondition, + UniformFloatHyperparameter, +) from hypershap import ExplanationTask @@ -88,6 +94,21 @@ def simple_cond_config_space() -> ConfigurationSpace: return config_space +@pytest.fixture(scope="session") +def simple_act_config_space() -> ConfigurationSpace: + """Return a simple config space with activation structure for testing.""" + config_space = ConfigurationSpace() + config_space.seed(42) + + a = UniformFloatHyperparameter("a", 0, 1, 0) + b = UniformFloatHyperparameter("b", 0, 1, 0) + config_space.add(a) + config_space.add(b) + + config_space.add(GreaterThanCondition(b, a, 0.3)) + return config_space + + @pytest.fixture(scope="session") def simple_cond_base_et( simple_cond_config_space: ConfigurationSpace, @@ -95,3 +116,12 @@ def simple_cond_base_et( ) -> ExplanationTask: """Return a base explanation task for the simple setup with conditions.""" return ExplanationTask.from_function(simple_cond_config_space, simple_blackbox_function.evaluate) + + +@pytest.fixture(scope="session") +def simple_act_base_et( + simple_act_config_space: ConfigurationSpace, + simple_blackbox_function: SimpleBlackboxFunction, +) -> ExplanationTask: + """Return a base explanation task for the simple setup with conditions.""" + return ExplanationTask.from_function(simple_act_config_space, simple_blackbox_function.evaluate) diff --git a/tests/test_extended_settings.py b/tests/test_extended_settings.py index b01d1e0..01fb666 100644 --- a/tests/test_extended_settings.py +++ b/tests/test_extended_settings.py @@ -79,3 +79,10 @@ def test_tunability_with_conditions(simple_cond_base_et: ExplanationTask) -> Non hypershap = HyperSHAP(simple_cond_base_et) iv = hypershap.tunability(simple_cond_base_et.config_space.get_default_configuration()) assert iv is not None, "Interaction values should not be none." + + +def test_tunability_with_activation_structures(simple_act_base_et: ExplanationTask) -> None: + """Test the tunability task with a configuration space that has conditions.""" + hypershap = HyperSHAP(simple_act_base_et) + iv = hypershap.tunability(simple_act_base_et.config_space.get_default_configuration()) + assert iv is not None, "Interaction values should not be none." diff --git a/tests/test_utils.py b/tests/test_utils.py index f149321..28cafb8 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -12,6 +12,8 @@ from hypershap import ExplanationTask from tests.fixtures.simple_setup import SimpleBlackboxFunction +from ConfigSpace import Configuration + from hypershap.task import BaselineExplanationTask from hypershap.utils import Aggregation, RandomConfigSpaceSearcher, evaluate_aggregation @@ -144,3 +146,31 @@ def test_evaluate_aggregation() -> None: assert evaluate_aggregation(Aggregation.MAX, vals) == AGG_LIST[2] assert evaluate_aggregation(Aggregation.AVG, vals) == np.array(AGG_LIST).mean() assert abs(evaluate_aggregation(Aggregation.VAR, vals) - np.array(AGG_LIST).var()) < EPSILON + + +def test_fallback_unfulfilled_conditions(simple_act_base_et: ExplanationTask) -> None: + """Test the fallback strategy when no configurations are left in random sample after filtering for conditions.""" + bet = BaselineExplanationTask( + simple_act_base_et.config_space, + simple_act_base_et.surrogate_model, + simple_act_base_et.config_space.get_default_configuration(), + ) + rcss = RandomConfigSpaceSearcher(bet) + rcss.random_sample = np.array([ + Configuration( + configuration_space=simple_act_base_et.config_space, + values={ + "a": 0.4, + "b": 0.1, + }, + ).get_array(), + Configuration( + configuration_space=simple_act_base_et.config_space, + values={ + "a": 0.5, + "b": 0.1, + }, + ).get_array(), + ]) + value = rcss.search(np.array([False, True])) + assert value is not None