diff --git a/docs/requirements.txt b/docs/requirements.txt index 0555874f0..38bd1e244 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,6 +3,7 @@ sphinx_rtd_theme sphinxcontrib.httpdomain sphinx-autoapi sphinx_gallery +sphinx-autodoc-typehints numpydoc plotly matplotlib diff --git a/docs/src/code/algo/space.rst b/docs/src/code/algo/space.rst index 7e0cd1595..4584cfb4a 100644 --- a/docs/src/code/algo/space.rst +++ b/docs/src/code/algo/space.rst @@ -1,5 +1,6 @@ Space search ============ + .. automodule:: orion.algo.space :members: diff --git a/docs/src/conf.py b/docs/src/conf.py index 9263e047d..c06b60a0a 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -213,7 +213,7 @@ # -- Autodoc configuration ----------------------------------------------- -autodoc_mock_imports = ["_version", "utils._appdirs", "nevergrad"] +autodoc_mock_imports = ["_version", "utils._appdirs", "nevergrad", "torch"] # -- Gallery configuration ----------------------------------------------- @@ -303,6 +303,7 @@ "orion.core.utils.tree.T", "orion.core.utils.tree.NodeType", "orion.core.utils.tree.Self", + "orion.algo.space.T", "Self", "AlgoType", "T", diff --git a/setup.py b/setup.py index 0f6e60354..06905353d 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ "dask": ["dask[complete]"], "track": ["track @ git+https://github.com/Delaunay/track@master#egg=track"], "profet": ["emukit", "GPy", "torch", "pybnn"], + "configspace": ["ConfigSpace"], "ax": [ "ax-platform", "numpy", diff --git a/src/orion/algo/space.py b/src/orion/algo/space/__init__.py similarity index 95% rename from src/orion/algo/space.py rename to src/orion/algo/space/__init__.py index e36a2c138..eafb9794a 100644 --- a/src/orion/algo/space.py +++ b/src/orion/algo/space/__init__.py @@ -27,9 +27,13 @@ unless noted otherwise! """ +from __future__ import annotations + import copy import logging import numbers +from functools import singledispatch +from typing import Any, Generic, TypeVar import numpy from scipy.stats import distributions @@ -66,6 +70,59 @@ def __repr__(self): return "..." +def _to_snake_case(name: str) -> str: + """Transform a class name ``MyClassName`` to snakecase ``my_class_name``""" + frags = [] + + frag = [] + for c in name: + if c.isupper() and frag: + frags.append("".join(frag).lower()) + frag = [] + + frag.append(c) + + if frag: + frags.append("".join(frag).lower()) + + return "_".join(frags) + + +T = TypeVar("T") + + +class SpaceConverter(Generic[T]): + """SpaceConverter iterates over an Orion search space. + This can be used to implement new features for ``orion.algo.space.Space`` + outside of Orion's code base. + + """ + + def convert_dimension(self, dimension: Dimension) -> T: + """Call the dimension conversion handler""" + return getattr(self, _to_snake_case(type(dimension).__name__))(dimension) + + def dimension(self, dim: Dimension) -> T: + """Called when the dimension does not have a decicated handler""" + + def real(self, dim: Real) -> T: + """Called by real dimension""" + + def integer(self, dim: Integer) -> T: + """Called by integer dimension""" + + def categorical(self, dim: Categorical) -> T: + """Called by categorical dimension""" + + def fidelity(self, dim: Fidelity) -> T: + """Called by fidelity dimension""" + + def space(self, space: Space) -> None: + """Iterate through a research space and visit each dimensions""" + for _, dim in space.items(): + self.visit(dim) + + class Dimension: """Base class for search space dimensions. @@ -1095,3 +1152,14 @@ def cardinality(self): for dim in self.values(): capacities *= dim.cardinality return capacities + + +@singledispatch +def to_orionspace(space: Any) -> Space: + """Convert a third party search space into an Orion compatible space + + Raises + ------ + NotImplementedError if no conversion was registered + """ + raise NotImplementedError() diff --git a/src/orion/algo/space/configspace.py b/src/orion/algo/space/configspace.py new file mode 100644 index 000000000..f36980490 --- /dev/null +++ b/src/orion/algo/space/configspace.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +from functools import singledispatch +from math import log10 + +from orion.algo.space import ( + Categorical, + Dimension, + Fidelity, + Integer, + Real, + Space, + SpaceConverter, + to_orionspace, +) + +try: + from ConfigSpace import ConfigurationSpace + from ConfigSpace.hyperparameters import ( + CategoricalHyperparameter, + FloatHyperparameter, + Hyperparameter, + IntegerHyperparameter, + NormalFloatHyperparameter, + NormalIntegerHyperparameter, + UniformFloatHyperparameter, + UniformIntegerHyperparameter, + ) + + IMPORT_ERROR = None + +except ImportError as err: + IMPORT_ERROR = err + + class DummyType: + """Dummy type for type hints""" + + IntegerHyperparameter = DummyType + FloatHyperparameter = DummyType + ConfigurationSpace = DummyType + Hyperparameter = DummyType + UniformFloatHyperparameter = DummyType + NormalFloatHyperparameter = DummyType + UniformIntegerHyperparameter = DummyType + NormalIntegerHyperparameter = DummyType + CategoricalHyperparameter = DummyType + + +class UnsupportedPrior(Exception): + pass + + +def _qantization(dim: Dimension) -> float: + """Convert precision to the quantization factor""" + if dim.precision: + return 10 ** (-dim.precision) + return None + + +class ToConfigSpace(SpaceConverter[Hyperparameter]): + """Convert an Orion space into a configspace""" + + def __init__(self) -> None: + if IMPORT_ERROR is not None: + raise IMPORT_ERROR + + def dimension(self, dim: Dimension) -> None: + """Raise an error if the visitor is called on an abstract class""" + raise NotImplementedError() + + def real(self, dim: Real) -> FloatHyperparameter: + """Convert a real dimension into a configspace equivalent""" + if dim.prior_name in ("reciprocal", "uniform"): + a, b = dim._args + + return UniformFloatHyperparameter( + name=dim.name, + lower=a, + upper=b, + default_value=dim.default_value, + q=_qantization(dim), + log=dim.prior_name == "reciprocal", + ) + + if dim.prior_name in ("normal", "norm"): + a, b = dim._args + + kwargs = dict( + name=dim.name, + mu=a, + sigma=b, + default_value=dim.default_value, + q=_qantization(dim), + log=False, + lower=dim.low if hasattr(dim, "low") else None, + upper=dim.high if hasattr(dim, "high") else None, + ) + + return NormalFloatHyperparameter(**kwargs) + + raise UnsupportedPrior(f'Prior "{dim.prior_name}" is not supported') + + def integer(self, dim: Integer) -> IntegerHyperparameter: + """Convert a integer dimension into a configspace equivalent""" + if dim.prior_name in ("int_uniform", "int_reciprocal"): + a, b = dim._args + + return UniformIntegerHyperparameter( + name=dim.name, + lower=a, + upper=b, + default_value=dim.default_value, + q=_qantization(dim), + log=dim.prior_name == "int_reciprocal", + ) + + if dim.prior_name in ("int_norm", "normal"): + a, b = dim._args + + kwargs = dict( + name=dim.name, + mu=a, + sigma=b, + default_value=dim.default_value, + q=_qantization(dim), + log=False, + lower=dim.low if hasattr(dim, "low") else None, + upper=dim.high if hasattr(dim, "high") else None, + ) + + return NormalIntegerHyperparameter(**kwargs) + + raise UnsupportedPrior(f'Prior "{dim.prior_name}" is not supported') + + def categorical(self, dim: Categorical) -> CategoricalHyperparameter: + """Convert a categorical dimension into a configspace equivalent""" + return CategoricalHyperparameter( + name=dim.name, + choices=dim.categories, + weights=dim._probs, + ) + + def fidelity(self, dim: Fidelity) -> None: + """Ignores fidelity dimension as configspace does not have an equivalent""" + return None + + def space(self, space: Space) -> ConfigurationSpace: + """Convert orion space to configspace""" + cspace = ConfigurationSpace() + dims = [] + + for _, dim in space.items(): + cdim = self.convert_dimension(dim) + + if cdim: + dims.append(cdim) + + cspace.add_hyperparameters(dims) + return cspace + + +def to_configspace(space: Space) -> ConfigurationSpace: + """Convert orion space to configspace + + Notes + ----- + ``ConfigurationSpace`` will set its own default values + if not set inside ``Space`` + + """ + conversion = ToConfigSpace() + return conversion.space(space) + + +@singledispatch +def to_oriondim(dim: Hyperparameter) -> Dimension: + """Convert a config space hyperparameter to an orion dimension""" + raise NotImplementedError(f"Dimension {dim} is not supported by Orion") + + +@to_oriondim.register +def _from_categorical(dim: CategoricalHyperparameter) -> Categorical: + """Builds a categorical dimension from a categorical hyperparameter""" + choices = {k: w for k, w in zip(dim.choices, dim.probabilities)} + return Categorical(dim.name, choices) + + +@to_oriondim.register(UniformIntegerHyperparameter) +@to_oriondim.register(UniformFloatHyperparameter) +def _from_uniform(dim: Hyperparameter) -> Integer | Real: + """Builds a uniform dimension from a uniform hyperparameter""" + + klass = Integer + args = [] + kwargs = dict( + # NOTE: Config space always has a config value + # so orion-space would get it as well + default_value=dim.default_value + ) + + if isinstance(dim, UniformFloatHyperparameter): + klass = Real + else: + kwargs["precision"] = int(-log10(dim.q)) if dim.q else 4 + + dist = "uniform" + args.append(dim.lower) + args.append(dim.upper) + + if dim.log: + dist = "reciprocal" + + return klass(dim.name, dist, *args, **kwargs) + + +@to_oriondim.register(NormalFloatHyperparameter) +@to_oriondim.register(NormalIntegerHyperparameter) +def _from_normal(dim: Hyperparameter) -> Integer | Real: + """Builds a normal dimension from a normal hyperparameter""" + + klass = Integer + args = [] + kwargs = dict( + # NOTE: Config space always has a config value + # so orion-space would get it as well + default_value=dim.default_value + ) + + if isinstance(dim, NormalFloatHyperparameter): + klass = Real + else: + kwargs["precision"] = int(-log10(dim.q)) if dim.q else 4 + + dist = "norm" + args.append(dim.mu) + args.append(dim.sigma) + + if dim.lower: + kwargs["low"] = dim.lower + kwargs["high"] = dim.upper + + return klass(dim.name, dist, *args, **kwargs) + + +@to_orionspace.register +def configspace_to_orionspace(cspace: ConfigurationSpace) -> Space: + """Convert from orion space to configspace + + Notes + ----- + ``ConfigurationSpace`` will set default values for each dimensions of ``Space`` + + """ + space = Space() + + for cdim in cspace.get_hyperparameters_dict().values(): + odim = to_oriondim(cdim) + space.register(odim) + + return space diff --git a/tests/unittests/algo/test_configspace.py b/tests/unittests/algo/test_configspace.py new file mode 100644 index 000000000..b3084d758 --- /dev/null +++ b/tests/unittests/algo/test_configspace.py @@ -0,0 +1,69 @@ +import pytest + +from orion.algo.space import Categorical, Fidelity, Integer, Real, Space, to_orionspace +from orion.algo.space.configspace import IMPORT_ERROR, UnsupportedPrior, to_configspace + +if IMPORT_ERROR: + pytest.skip("Running without ConfigSpace", allow_module_level=True) + + +def test_orion_configspace(): + space = Space() + + space.register(Integer("r1i", "reciprocal", 1, 6)) + space.register(Integer("u1i", "uniform", -3, 6)) + space.register(Integer("u2i", "uniform", -3, 6)) + space.register(Integer("u3i", "uniform", -3, 6, default_value=2)) + + space.register(Real("r1f", "reciprocal", 1, 6)) + space.register(Real("u1f", "uniform", -3, 6)) + space.register(Real("u2f", "uniform", -3, 6)) + space.register(Real("name.u2f", "uniform", -3, 6)) + + space.register(Categorical("c1", ("asdfa", 2))) + space.register(Categorical("c2", dict(a=0.2, b=0.8))) + space.register(Fidelity("f1", 1, 9, 3)) + + space.register(Real("n1", "norm", 0.9, 0.1, precision=6)) + space.register(Real("n2", "norm", 0.9, 0.1, precision=None)) + space.register(Real("n3", "norm", 0.9, 0.1)) + space.register(Integer("n4", "norm", 1, 2)) + + newspace = to_configspace(space) + + roundtrip = to_orionspace(newspace) + + for k, original in space.items(): + # ConfigSpace does not have a fidelity dimension + # or the alpha prior + if k in ("f1", "a1i"): + continue + + converted = roundtrip[k] + + # Orion space did not have default values + # but ConfigSpace always set them + if not original.default_value: + converted._default_value = None + + assert type(original) == type(converted) + assert original == converted + + +def test_configspace_to_orion_unsupported(): + from ConfigSpace import ConfigurationSpace + from ConfigSpace.hyperparameters import OrdinalHyperparameter + + cspace = ConfigurationSpace() + cspace.add_hyperparameters([OrdinalHyperparameter("a", (1, 2, 0, 3))]) + + with pytest.raises(NotImplementedError): + _ = to_orionspace(cspace) + + +def test_orion_configspace_unsupported(): + space = Space() + space.register(Integer("a1i", "alpha", 1, 6)) + + with pytest.raises(UnsupportedPrior): + _ = to_configspace(space)