Skip to content

Commit

Permalink
support for truncated normal distribution (#188)
Browse files Browse the repository at this point in the history 
* adding trucated normal support

* adding trucated normal support

* updating the documentation

* edited setup.py with scipy

* minor editions to improve integration of truncated normal

* removing dublicated definition of public parameters lower/upper

* improving documentation consistency, adding tests for quantization and border case, correcting representation

* Update setup.py

Co-authored-by: Matthias Feurer <lists@matthiasfeurer.de>

* fixed NormalInteger with truncated normal conversion to integer, added test cases

* adaptation of get_neighbors with tests

* pre-commit tests passing

Co-authored-by: Deathn0t <romainegele@gmail.com>
Co-authored-by: Matthias Feurer <lists@matthiasfeurer.de>
  • Loading branch information
@pbalapra @Deathn0t @mfeurer
3 people committed Oct 18, 2021
1 parent 22b979c commit e9573c1
Show file tree
Hide file tree
Showing 3 changed files with 234 additions and 36 deletions.
197 changes: 162 additions & 35 deletions ConfigSpace/hyperparameters.pyx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ from collections import OrderedDict, Counter
from typing import List, Any, Dict, Union, Set, Tuple, Optional

import numpy as np
from scipy.stats import truncnorm
cimport numpy as np


Expand Down Expand Up @@ -200,9 +201,9 @@ cdef class Constant(Hyperparameter):

Additionally, it defines the __ne__() as stated in the
documentation from python:
By default, object implements __eq__() by using is, returning NotImplemented
in the case of a false comparison: True if x is y else NotImplemented.
For __ne__(), by default it delegates to __eq__() and inverts the result
By default, object implements __eq__() by using is, returning NotImplemented
in the case of a false comparison: True if x is y else NotImplemented.
For __ne__(), by default it delegates to __eq__() and inverts the result
unless it is NotImplemented.

"""
Expand Down Expand Up @@ -298,9 +299,9 @@ cdef class NumericalHyperparameter(Hyperparameter):

Additionally, it defines the __ne__() as stated in the
documentation from python:
By default, object implements __eq__() by using is, returning NotImplemented
in the case of a false comparison: True if x is y else NotImplemented.
For __ne__(), by default it delegates to __eq__() and inverts the result
By default, object implements __eq__() by using is, returning NotImplemented
in the case of a false comparison: True if x is y else NotImplemented.
For __ne__(), by default it delegates to __eq__() and inverts the result
unless it is NotImplemented.

"""
Expand Down Expand Up @@ -432,7 +433,7 @@ cdef class UniformFloatHyperparameter(FloatHyperparameter):
----------
name : str
Name of the hyperparameter, with which it can be accessed
lower : int, floor
lower : int, float
Lower bound of a range of values from which the hyperparameter will be sampled
upper : int, float
Upper bound
Expand Down Expand Up @@ -610,6 +611,8 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter):
def __init__(self, name: str, mu: Union[int, float], sigma: Union[int, float],
default_value: Union[None, float] = None,
q: Union[int, float, None] = None, log: bool = False,
lower: Optional[Union[float, int]] = None,
upper: Optional[Union[float, int]] = None,
meta: Optional[Dict] = None) -> None:
r"""
A float hyperparameter.
Expand Down Expand Up @@ -643,6 +646,10 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter):
log : bool, optional
If ``True``, the values of the hyperparameter will be sampled
on a logarithmic scale. Default to ``False``
lower : int, float, optional
Lower bound of a range of values from which the hyperparameter will be sampled
upper : int, float, optional
Upper bound of a range of values from which the hyperparameter will be sampled
meta : Dict, optional
Field for holding meta data provided by the user.
Not used by the configuration space.
Expand All @@ -655,11 +662,58 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter):
self.default_value = self.check_default(default_value)
self.normalized_default_value = self._inverse_transform(self.default_value)

if (lower is not None) ^ (upper is not None):
raise ValueError("Only one bound was provided when both lower and upper bounds must be provided.")

if lower is not None and upper is not None:
self.lower = float(lower)
self.upper = float(upper)

if self.lower >= self.upper:
raise ValueError("Upper bound %f must be larger than lower bound "
"%f for hyperparameter %s" %
(self.upper, self.lower, name))
elif log and self.lower <= 0:
raise ValueError("Negative lower bound (%f) for log-scale "
"hyperparameter %s is forbidden." %
(self.lower, name))

self.default_value = self.check_default(default_value)

if self.log:
if self.q is not None:
lower = self.lower - (np.float64(self.q) / 2. + 0.0001)
upper = self.upper + (np.float64(self.q) / 2. - 0.0001)
else:
lower = self.lower
upper = self.upper
self._lower = np.log(lower)
self._upper = np.log(upper)
else:
if self.q is not None:
self._lower = self.lower - (self.q / 2. + 0.0001)
self._upper = self.upper + (self.q / 2. - 0.0001)
else:
self._lower = self.lower
self._upper = self.upper
if self.q is not None:
# There can be weird rounding errors, so we compare the result against self.q, see
# In [13]: 2.4 % 0.2
# Out[13]: 0.1999999999999998
if np.round((self.upper - self.lower) % self.q, 10) not in (0, self.q):
raise ValueError(
'Upper bound (%f) - lower bound (%f) must be a multiple of q (%f)'
% (self.upper, self.lower, self.q)
)

def __repr__(self) -> str:
repr_str = io.StringIO()
repr_str.write("%s, Type: NormalFloat, Mu: %s Sigma: %s, Default: %s" %
(self.name, repr(self.mu), repr(self.sigma),
repr(self.default_value)))

if self.lower is None or self.upper is None:
repr_str.write("%s, Type: NormalFloat, Mu: %s Sigma: %s, Default: %s" % (self.name, repr(self.mu), repr(self.sigma), repr(self.default_value)))
else:
repr_str.write("%s, Type: NormalFloat, Mu: %s Sigma: %s, Range: [%s, %s], Default: %s" % (self.name, repr(self.mu), repr(self.sigma), repr(self.lower), repr(self.upper), repr(self.default_value)))

if self.log:
repr_str.write(", on log-scale")
if self.q is not None:
Expand All @@ -674,9 +728,9 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter):

Additionally, it defines the __ne__() as stated in the
documentation from python:
By default, object implements __eq__() by using is, returning NotImplemented
in the case of a false comparison: True if x is y else NotImplemented.
For __ne__(), by default it delegates to __eq__() and inverts the result
By default, object implements __eq__() by using is, returning NotImplemented
in the case of a false comparison: True if x is y else NotImplemented.
For __ne__(), by default it delegates to __eq__() and inverts the result
unless it is NotImplemented.

"""
Expand All @@ -689,7 +743,9 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter):
self.mu == other.mu and
self.sigma == other.sigma and
self.log == other.log and
self.q == other.q
self.q == other.q and
self.lower == other.lower and
self.upper == other.upper
)

def __copy__(self):
Expand All @@ -700,16 +756,25 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter):
sigma=self.sigma,
log=self.log,
q=self.q,
lower=self.lower,
upper=self.upper,
meta=self.meta
)

def __hash__(self):
return hash((self.name, self.mu, self.sigma, self.log, self.q))

def to_uniform(self, z: int = 3) -> 'UniformFloatHyperparameter':
if self.lower is None or self.upper is None:
lb = self.mu - (z * self.sigma)
ub = self.mu + (z * self.sigma)
else:
lb = self.lower
ub = self.upper

return UniformFloatHyperparameter(self.name,
self.mu - (z * self.sigma),
self.mu + (z * self.sigma),
lb,
ub,
default_value=int(
np.round(self.default_value, 0)),
q=self.q, log=self.log)
Expand Down Expand Up @@ -740,9 +805,19 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter):

def _sample(self, rs: np.random.RandomState, size: Optional[int] = None
) -> Union[np.ndarray, float]:
mu = self.mu
sigma = self.sigma
return rs.normal(mu, sigma, size=size)

if self.lower == None:
mu = self.mu
sigma = self.sigma
return rs.normal(mu, sigma, size=size)
else:
mu = self.mu
sigma = self.sigma
lower = self.lower
upper = self.upper
a = (self.lower - mu) / sigma
b = (upper - mu) / sigma
return truncnorm.rvs(a, b, loc=mu, scale=sigma, size=size, random_state=rs)

cpdef np.ndarray _transform_vector(self, np.ndarray vector):
if np.isnan(vector).any():
Expand Down Expand Up @@ -774,7 +849,12 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter):
transform: bool = False) -> List[float]:
neighbors = []
for i in range(number):
neighbors.append(rs.normal(value, self.sigma))
new_value = rs.normal(value, self.sigma)

if self.lower is not None and self.upper is not None:
new_value = min(max(new_value, self.lower), self.upper)

neighbors.append(new_value)
return neighbors


Expand Down Expand Up @@ -1034,9 +1114,12 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter):
cdef public sigma
cdef nfhp


def __init__(self, name: str, mu: int, sigma: Union[int, float],
default_value: Union[int, None] = None, q: Union[None, int] = None,
log: bool = False,
lower: Optional[int] = None,
upper: Optional[int] = None,
meta: Optional[Dict] = None) -> None:
r"""
An integer hyperparameter.
Expand Down Expand Up @@ -1071,12 +1154,17 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter):
log : bool, optional
If ``True``, the values of the hyperparameter will be sampled
on a logarithmic scale. Defaults to ``False``
lower : int, float, optional
Lower bound of a range of values from which the hyperparameter will be sampled
upper : int, float, optional
Upper bound of a range of values from which the hyperparameter will be sampled
meta : Dict, optional
Field for holding meta data provided by the user.
Not used by the configuration space.

"""
super(NormalIntegerHyperparameter, self).__init__(name, default_value, meta)

self.mu = mu
self.sigma = sigma

Expand All @@ -1097,20 +1185,43 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter):

self.default_value = self.check_default(default_value)

if (lower is not None) ^ (upper is not None):
raise ValueError("Only one bound was provided when both lower and upper bounds must be provided.")

if lower is not None and upper is not None:
self.upper = self.check_int(upper, "upper")
self.lower = self.check_int(lower, "lower")
if self.lower >= self.upper:
raise ValueError("Upper bound %d must be larger than lower bound "
"%d for hyperparameter %s" %
(self.lower, self.upper, name))
elif log and self.lower <= 0:
raise ValueError("Negative lower bound (%d) for log-scale "
"hyperparameter %s is forbidden." %
(self.lower, name))
self.lower = lower
self.upper = upper


self.nfhp = NormalFloatHyperparameter(self.name,
self.mu,
self.sigma,
log=self.log,
q=self.q,
lower=self.lower,
upper=self.upper,
default_value=self.default_value)

self.normalized_default_value = self._inverse_transform(self.default_value)

def __repr__(self) -> str:
repr_str = io.StringIO()
repr_str.write("%s, Type: NormalInteger, Mu: %s Sigma: %s, Default: "
"%s" % (self.name, repr(self.mu),
repr(self.sigma), repr(self.default_value)))

if self.lower is None or self.upper is None:
repr_str.write("%s, Type: NormalInteger, Mu: %s Sigma: %s, Default: %s" % (self.name, repr(self.mu), repr(self.sigma), repr(self.default_value)))
else:
repr_str.write("%s, Type: NormalInteger, Mu: %s Sigma: %s, Range: [%s, %s], Default: %s" % (self.name, repr(self.mu), repr(self.sigma), repr(self.lower), repr(self.upper), repr(self.default_value)))

if self.log:
repr_str.write(", on log-scale")
if self.q is not None:
Expand All @@ -1125,9 +1236,9 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter):

Additionally, it defines the __ne__() as stated in the
documentation from python:
By default, object implements __eq__() by using is, returning NotImplemented
in the case of a false comparison: True if x is y else NotImplemented.
For __ne__(), by default it delegates to __eq__() and inverts the result
By default, object implements __eq__() by using is, returning NotImplemented
in the case of a false comparison: True if x is y else NotImplemented.
For __ne__(), by default it delegates to __eq__() and inverts the result
unless it is NotImplemented.

"""
Expand All @@ -1139,7 +1250,9 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter):
self.mu == other.mu and
self.sigma == other.sigma and
self.log == other.log and
self.q == other.q
self.q == other.q and
self.lower == other.lower and
self.upper == other.upper
)

def __hash__(self):
Expand All @@ -1153,14 +1266,23 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter):
sigma=self.sigma,
log=self.log,
q=self.q,
lower=self.lower,
upper=self.upper,
meta=self.meta
)

# todo check if conversion should be done in initiation call or inside class itsel
def to_uniform(self, z: int = 3) -> 'UniformIntegerHyperparameter':
if self.lower is None or self.upper is None:
lb = np.round(int(self.mu - (z * self.sigma)))
ub = np.round(int(self.mu + (z * self.sigma)))
else:
lb = self.lower
ub = self.upper

return UniformIntegerHyperparameter(self.name,
np.round(int(self.mu - (z * self.sigma))),
np.round(int(self.mu + (z * self.sigma))),
lb,
ub,
default_value=self.default_value,
q=self.q, log=self.log)

Expand Down Expand Up @@ -1220,6 +1342,11 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter):
new_value = rs.normal(value, self.sigma)
int_value = self._transform(value)
new_int_value = self._transform(new_value)

if self.lower is not None and self.upper is not None:
int_value = min(max(int_value, self.lower), self.upper)
new_int_value = min(max(new_int_value, self.lower), self.upper)

if int_value != new_int_value:
rejected = False
elif iteration > 100000:
Expand Down Expand Up @@ -1332,9 +1459,9 @@ cdef class CategoricalHyperparameter(Hyperparameter):

Additionally, it defines the __ne__() as stated in the
documentation from python:
By default, object implements __eq__() by using is, returning NotImplemented
in the case of a false comparison: True if x is y else NotImplemented.
For __ne__(), by default it delegates to __eq__() and inverts the result
By default, object implements __eq__() by using is, returning NotImplemented
in the case of a false comparison: True if x is y else NotImplemented.
For __ne__(), by default it delegates to __eq__() and inverts the result
unless it is NotImplemented.

"""
Expand Down Expand Up @@ -1588,9 +1715,9 @@ cdef class OrdinalHyperparameter(Hyperparameter):

Additionally, it defines the __ne__() as stated in the
documentation from python:
By default, object implements __eq__() by using is, returning NotImplemented
in the case of a false comparison: True if x is y else NotImplemented.
For __ne__(), by default it delegates to __eq__() and inverts the result
By default, object implements __eq__() by using is, returning NotImplemented
in the case of a false comparison: True if x is y else NotImplemented.
For __ne__(), by default it delegates to __eq__() and inverts the result
unless it is NotImplemented.

"""
Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def finalize_options(self):
AUTHOR_EMAIL = 'feurerm@informatik.uni-freiburg.de'
TEST_SUITE = "pytest"
SETUP_REQS = ['numpy', 'cython']
INSTALL_REQS = ['numpy', 'cython', 'pyparsing']
INSTALL_REQS = ['numpy', 'cython', 'pyparsing', 'scipy']
MIN_PYTHON_VERSION = '>=3.7'
CLASSIFIERS = ['Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
Expand Down
Loading

0 comments on commit e9573c1

Please sign in to comment.