Skip to content

Commit

Permalink
test: Fix upper bounds for test_transformer() and no random in `tes…
Browse files Browse the repository at this point in the history 
…t_transformer_conditional()` (#1102)

* test: Don't rely on randomization for the test

* test: Include import

* test: Fix boundaries for catgoricals in test

* test: Missing import

* doc: Make todo not more clear about when to upgrade
  • Loading branch information
@eddiebergman
eddiebergman committed Apr 17, 2024
1 parent 658a6b3 commit 6da055b
Showing 1 changed file with 109 additions and 29 deletions.
138 changes: 109 additions & 29 deletions tests/test_runhistory/test_runhistory_encoder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
from smac.runhistory.encoder.encoder import RunHistoryEncoder
from smac.runner.abstract_runner import StatusType

from ConfigSpace import Configuration
from ConfigSpace.hyperparameters import CategoricalHyperparameter


@pytest.fixture
def configs(configspace_small):
Expand All @@ -39,70 +42,127 @@ def test_transform(runhistory, make_scenario, configspace_small, configs):
)

# Normal encoder
encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS])
encoder = RunHistoryEncoder(
scenario=scenario, considered_states=[StatusType.SUCCESS]
)
encoder.runhistory = runhistory

# TODO: Please replace with the more general solution once ConfigSpace 1.0
# upper = np.array([hp.upper_vectorized for hp in space.values()])
# lower = np.array([hp.lower_vectorized for hp in space.values()])
# -
# Categoricals are upperbounded by their size, rest of hyperparameters are
# upperbounded by 1.
upper_bounds = {
hp.name: (hp.get_size() - 1)
if isinstance(hp, CategoricalHyperparameter)
else 1.0
for hp in configspace_small.get_hyperparameters()
}
# Need to ensure they match the order in the Configuration vectorized form
sorted_by_indices = sorted(
upper_bounds.items(),
key=lambda x: configspace_small._hyperparameter_idx[x[0]],
)
upper = np.array([upper_bound for _, upper_bound in sorted_by_indices])
lower = 0.0

X1, Y1 = encoder.transform()

assert Y1.tolist() == [[1.0], [5.0]]
assert ((X1 <= 1.0) & (X1 >= 0.0)).all()
assert ((X1 <= upper) & (X1 >= lower)).all()

# Log encoder
encoder = RunHistoryLogEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS])
encoder = RunHistoryLogEncoder(
scenario=scenario, considered_states=[StatusType.SUCCESS]
)
encoder.runhistory = runhistory
X, Y = encoder.transform()
assert Y.tolist() != Y1.tolist()
assert ((X <= 1.0) & (X >= 0.0)).all()
assert ((X <= upper) & (X >= lower)).all()

encoder = RunHistoryLogScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS])
encoder = RunHistoryLogScaledEncoder(
scenario=scenario, considered_states=[StatusType.SUCCESS]
)
encoder.runhistory = runhistory
X, Y = encoder.transform()
assert Y.tolist() != Y1.tolist()
assert ((X <= 1.0) & (X >= 0.0)).all()
assert ((X <= upper) & (X >= lower)).all()

encoder = RunHistoryScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS])
encoder = RunHistoryScaledEncoder(
scenario=scenario, considered_states=[StatusType.SUCCESS]
)
encoder.runhistory = runhistory
X, Y = encoder.transform()
assert Y.tolist() != Y1.tolist()
assert ((X <= 1.0) & (X >= 0.0)).all()
assert ((X <= upper) & (X >= lower)).all()

encoder = RunHistoryInverseScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS])
encoder = RunHistoryInverseScaledEncoder(
scenario=scenario, considered_states=[StatusType.SUCCESS]
)
encoder.runhistory = runhistory
X, Y = encoder.transform()
assert Y.tolist() != Y1.tolist()
assert ((X <= 1.0) & (X >= 0.0)).all()
assert ((X <= upper) & (X >= lower)).all()

encoder = RunHistorySqrtScaledEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS])
encoder = RunHistorySqrtScaledEncoder(
scenario=scenario, considered_states=[StatusType.SUCCESS]
)
encoder.runhistory = runhistory
X, Y = encoder.transform()
assert Y.tolist() != Y1.tolist()
assert ((X <= 1.0) & (X >= 0.0)).all()
assert ((X <= upper) & (X >= lower)).all()

encoder = RunHistoryEIPSEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS])
encoder = RunHistoryEIPSEncoder(
scenario=scenario, considered_states=[StatusType.SUCCESS]
)
encoder.runhistory = runhistory
X, Y = encoder.transform()
assert Y.tolist() != Y1.tolist()
assert ((X <= 1.0) & (X >= 0.0)).all()
assert ((X <= upper) & (X >= lower)).all()


def test_transform_conditionals(runhistory, make_scenario, configspace_large, configs):
configs = configspace_large.sample_configuration(20)
def test_transform_conditionals(runhistory, make_scenario, configspace_large):
scenario = make_scenario(configspace_large)

config_1 = Configuration(
configspace_large,
values={
"activation": "tanh",
"n_layer": 5,
"n_neurons": 27,
"solver": "lbfgs",
},
)
config_2 = Configuration(
configspace_large,
values={
"activation": "tanh",
"batch_size": 47,
"learning_rate": "adaptive",
"learning_rate_init": 0.6673206111956781,
"n_layer": 3,
"n_neurons": 88,
"solver": "sgd",
},
)
runhistory.add(
config=configs[0],
config=config_1,
cost=1,
time=1,
status=StatusType.SUCCESS,
)

runhistory.add(
config=configs[2],
config=config_2,
cost=5,
time=4,
status=StatusType.SUCCESS,
)

encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS])
encoder = RunHistoryEncoder(
scenario=scenario, considered_states=[StatusType.SUCCESS]
)
encoder.runhistory = runhistory
X, Y = encoder.transform()

Expand All @@ -124,7 +184,9 @@ def test_multi_objective(runhistory, make_scenario, configspace_small, configs):

# Multi objective algorithm must be set
with pytest.raises(AssertionError):
encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS])
encoder = RunHistoryEncoder(
scenario=scenario, considered_states=[StatusType.SUCCESS]
)
encoder.runhistory = runhistory
_, Y = encoder.transform()

Expand Down Expand Up @@ -180,7 +242,9 @@ def test_ignore(runhistory, make_scenario, configspace_small, configs):
)

# Normal encoder
encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS])
encoder = RunHistoryEncoder(
scenario=scenario, considered_states=[StatusType.SUCCESS]
)
encoder.runhistory = runhistory
X1, Y1 = encoder.transform()

Expand Down Expand Up @@ -219,10 +283,14 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs):
budget=2,
)

runhistory.add(config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2)
runhistory.add(
config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2
)

# Normal encoder
encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS])
encoder = RunHistoryEncoder(
scenario=scenario, considered_states=[StatusType.SUCCESS]
)
encoder.runhistory = runhistory
X, Y = encoder.transform(budget_subset=[2])
assert Y.tolist() == [[99999999]]
Expand Down Expand Up @@ -251,10 +319,14 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs):
budget=2,
)

runhistory.add(config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2)
runhistory.add(
config=configs[1], cost=5, time=4, status=StatusType.SUCCESS, budget=2
)

# Normal encoder
encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS])
encoder = RunHistoryEncoder(
scenario=scenario, considered_states=[StatusType.SUCCESS]
)
encoder.runhistory = runhistory
X, Y = encoder.transform(budget_subset=[2])
assert Y.tolist() == [[99999999]]
Expand All @@ -266,12 +338,20 @@ def test_budgets(runhistory, make_scenario, configspace_small, configs):
def test_lower_budget_states(runhistory, make_scenario, configspace_small, configs):
"""Tests lower budgets based on budget subset and considered states."""
scenario = make_scenario(configspace_small)
encoder = RunHistoryEncoder(scenario=scenario, considered_states=[StatusType.SUCCESS])
encoder = RunHistoryEncoder(
scenario=scenario, considered_states=[StatusType.SUCCESS]
)
encoder.runhistory = runhistory

runhistory.add(config=configs[0], cost=1, time=1, status=StatusType.SUCCESS, budget=3)
runhistory.add(config=configs[0], cost=2, time=2, status=StatusType.SUCCESS, budget=4)
runhistory.add(config=configs[0], cost=3, time=4, status=StatusType.TIMEOUT, budget=5)
runhistory.add(
config=configs[0], cost=1, time=1, status=StatusType.SUCCESS, budget=3
)
runhistory.add(
config=configs[0], cost=2, time=2, status=StatusType.SUCCESS, budget=4
)
runhistory.add(
config=configs[0], cost=3, time=4, status=StatusType.TIMEOUT, budget=5
)

# We request a higher budget but can't find it, so we expect an empty list
X, Y = encoder.transform(budget_subset=[500])
Expand Down

0 comments on commit 6da055b

Please sign in to comment.