Merge 3715f30 into 0765ec3

.github/workflows/pytest.yml

@@ -7,10 +7,11 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - 3.6
-          - 3.7
-          - 3.8
-          - 3.9
+          - '3.6'
+          - '3.7'
+          - '3.8'
+          - '3.9'
+          - '3.10'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017-2021 Jungtaek Kim
+Copyright (c) 2017-2022 Jungtaek Kim
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -72,9 +72,10 @@ We test our package in the following versions.
 * Python 3.7
 * Python 3.8
 * Python 3.9
+* Python 3.10
 
 ## Contributor
-* [Jungtaek Kim](https://jungtaek.github.io) (POSTECH)
+* [Jungtaek Kim](https://jungtaek.github.io)
 
 ## Citation
 ```
@@ -87,7 +88,7 @@ We test our package in the following versions.
 ```
 
 ## Contact
-* Jungtaek Kim: [jtkim@postech.ac.kr](mailto:jtkim@postech.ac.kr)
+* [Jungtaek Kim](https://jungtaek.github.io)
 
 ## License
 [MIT License](LICENSE)

bayeso/__init__.py

@@ -1,9 +1,9 @@
 #
 # author: Jungtaek Kim (jtkim@postech.ac.kr)
-# last updated: October 23, 2021
+# last updated: February 3, 2022
 #
 """BayesO is a simple, but essential Bayesian optimization
 package, implemented in Python."""
 
 
-__version__ = '0.5.2'
+__version__ = '0.5.3'

bayeso/bo/bo_w_gp.py

@@ -1,6 +1,6 @@
 #
 # author: Jungtaek Kim (jtkim@postech.ac.kr)
-# last updated: October 8, 2021
+# last updated: February 4, 2022
 #
 """It defines a class of Bayesian optimization
 with Gaussian process regression."""
@@ -453,6 +453,10 @@ def optimize(self, X_train: np.ndarray, Y_train: np.ndarray,
         )
         next_point, next_points = self._optimize(fun_negative_acquisition,
             str_sampling_method=str_sampling_method, num_samples=num_samples)
+
+        next_point = utils_bo.check_points_in_bounds(next_point[np.newaxis, ...], np.array(self._get_bounds()))[0]
+        next_points = utils_bo.check_points_in_bounds(next_points, np.array(self._get_bounds()))
+
         time_end_acq = time.time()
 
         acquisitions = fun_negative_acquisition(next_points)

bayeso/bo/bo_w_tp.py

@@ -1,6 +1,6 @@
 #
 # author: Jungtaek Kim (jtkim@postech.ac.kr)
-# last updated: October 8, 2021
+# last updated: February 4, 2022
 #
 """It defines a class of Bayesian optimization
 with Student-:math:`t` process regression."""
@@ -375,6 +375,10 @@ def optimize(self, X_train: np.ndarray, Y_train: np.ndarray,
         )
         next_point, next_points = self._optimize(fun_negative_acquisition,
             str_sampling_method=str_sampling_method, num_samples=num_samples)
+
+        next_point = utils_bo.check_points_in_bounds(next_point[np.newaxis, ...], np.array(self._get_bounds()))[0]
+        next_points = utils_bo.check_points_in_bounds(next_points, np.array(self._get_bounds()))
+
         time_end_acq = time.time()
 
         acquisitions = fun_negative_acquisition(next_points)

bayeso/bo/bo_w_trees.py

@@ -1,6 +1,6 @@
 #
 # author: Jungtaek Kim (jtkim@postech.ac.kr)
-# last updated: October 8, 2021
+# last updated: February 4, 2022
 #
 """It defines a class of Bayesian optimization
 with tree-based surrogate models."""
@@ -253,6 +253,8 @@ def optimize(self, X_train: np.ndarray, Y_train: np.ndarray,
             num_samples=num_samples
         )
 
+        next_points = utils_bo.check_points_in_bounds(next_points, np.array(self._get_bounds()))
+
         fun_negative_acquisition = lambda X_test: -1.0 * self.compute_acquisitions(
             X_test, X_train, Y_train, trees
         )

bayeso/constants.py

@@ -128,6 +128,7 @@
 TYPING_TUPLE_FIVE_ARRAYS = typing.Tuple[TYPE_ARR, TYPE_ARR, TYPE_ARR, TYPE_ARR, TYPE_ARR]
 TYPING_TUPLE_FLOAT_THREE_ARRAYS = typing.Tuple[float, TYPE_ARR, TYPE_ARR, TYPE_ARR]
 TYPING_TUPLE_FLOAT_ARRAY = typing.Tuple[float, TYPE_ARR]
+TYPING_TUPLE_INT_FLOAT_TUPLE = typing.Tuple[int, float, typing.Tuple]
 
 TYPING_UNION_INT_NONE = typing.Union[int, TYPE_NONE]
 TYPING_UNION_INT_FLOAT = typing.Union[int, float]

bayeso/trees/trees_common.py

@@ -189,16 +189,65 @@ def _split_left_right(
     assert X.shape[0] == Y.shape[0]
     assert Y.shape[1] == 1
 
-    left = []
-    right = []
+    indices_left = X[:, dim_to_split] < val_to_split
+    left = list(zip(X[indices_left], Y[indices_left]))
+
+    indices_right = X[:, dim_to_split] >= val_to_split
+    right = list(zip(X[indices_right], Y[indices_right]))
 
-    for bx, by in zip(X, Y):
-        if bx[dim_to_split] < val_to_split:
-            left.append((bx, by))
-        else:
-            right.append((bx, by))
     return left, right
 
+@utils_common.validate_types
+def _split_deterministic(X: np.ndarray, Y: np.ndarray, dim_to_split: int
+) -> constants.TYPING_TUPLE_INT_FLOAT_TUPLE:
+    candidates_loc = np.sort(np.unique(X[:, dim_to_split]))
+    num_evaluations = 1
+    if candidates_loc.shape[0] > 1:
+        num_evaluations = candidates_loc.shape[0] - 1
+
+    cur_ind = np.inf
+    cur_val = np.inf
+    cur_score = np.inf
+    cur_left_right = None
+
+    indices_loc = np.random.choice(
+        num_evaluations, np.minimum(20, num_evaluations), replace=False)
+
+    for ind_loc in indices_loc:
+        if candidates_loc.shape[0] > 1:
+            val_to_split = np.mean(candidates_loc[ind_loc:ind_loc+2])
+        else:
+            val_to_split = X[0, dim_to_split]
+
+        left_right = _split_left_right(X, Y, dim_to_split, val_to_split)
+        score = mse(left_right)
+
+        if score < cur_score:
+            cur_ind = dim_to_split
+            cur_val = val_to_split
+            cur_score = score
+            cur_left_right = left_right
+
+    return cur_ind, cur_val, cur_score, cur_left_right
+
+@utils_common.validate_types
+def _split_random(X: np.ndarray, Y: np.ndarray, dim_to_split: int
+) -> constants.TYPING_TUPLE_INT_FLOAT_TUPLE:
+    candidates_loc = np.sort(np.unique(X[:, dim_to_split]))
+
+    if candidates_loc.shape[0] > 1:
+        min_bx = np.min(X[:, dim_to_split])
+        max_bx = np.max(X[:, dim_to_split])
+
+        val_to_split = np.random.uniform(low=min_bx, high=max_bx)
+    else:
+        val_to_split = X[0, dim_to_split]
+
+    left_right = _split_left_right(X, Y, dim_to_split, val_to_split)
+    score = mse(left_right)
+
+    return dim_to_split, val_to_split, score, left_right
+
 @utils_common.validate_types
 def _split(
     X: np.ndarray, Y: np.ndarray,
@@ -235,43 +284,28 @@ def _split(
     assert X.shape[0] > 0
     assert Y.shape[1] == 1
 
+    features = np.random.choice(X.shape[1], num_features, replace=False)
+
     cur_ind = np.inf
     cur_val = np.inf
     cur_score = np.inf
     cur_left_right = None
 
-    features = np.random.choice(X.shape[1], num_features, replace=False)
-
     for ind in features:
         dim_to_split = int(ind)
-        num_evaluations = 1
-        candidates_loc = np.sort(np.unique(X[:, dim_to_split]))
 
-        if candidates_loc.shape[0] > 1:
-            if split_random_location:
-                min_bx = np.min(X[:, dim_to_split])
-                max_bx = np.max(X[:, dim_to_split])
-            else:
-                num_evaluations = candidates_loc.shape[0] - 1
-
-        for ind_loc in range(0, num_evaluations):
-            if candidates_loc.shape[0] > 1:
-                if split_random_location:
-                    val_to_split = np.random.uniform(low=min_bx, high=max_bx)
-                else:
-                    val_to_split = np.mean(candidates_loc[ind_loc:ind_loc+2])
-            else:
-                val_to_split = X[0, dim_to_split]
-
-            left_right = _split_left_right(X, Y, dim_to_split, val_to_split)
-#            left, right = left_right
-            score = mse(left_right)
-
-            if score < cur_score:
-                cur_ind = dim_to_split
-                cur_val = val_to_split
-                cur_score = score
-                cur_left_right = left_right
+        if split_random_location:
+            _ind, _val, _score, _left_right = _split_random(
+                X, Y, dim_to_split)
+        else:
+            _ind, _val, _score, _left_right = _split_deterministic(
+                X, Y, dim_to_split)
+
+        if _score < cur_score:
+            cur_ind = _ind
+            cur_val = _val
+            cur_score = _score
+            cur_left_right = _left_right
 
     return {
         'index': cur_ind,

bayeso/utils/utils_bo.py

@@ -1,6 +1,6 @@
 #
 # author: Jungtaek Kim (jtkim@postech.ac.kr)
-# last updated: November 5, 2020
+# last updated: February 4, 2022
 #
 """It is utilities for Bayesian optimization."""
 
@@ -286,3 +286,33 @@ def check_hyps_convergence(list_hyps: constants.TYPING_LIST[dict], hyps: dict,
         if np.linalg.norm(hyps_converted - target_hyps_converted, ord=2) < threshold:
             converged = True
     return converged
+
+@utils_common.validate_types
+def check_points_in_bounds(points: np.ndarray, bounds: np.ndarray
+) -> np.ndarray:
+    """
+    It checks whether every instance of `points` is located in `bounds`.
+
+    :param points: points to check. Shape: (n, d).
+    :type points: numpy.ndarray
+    :param bounds: upper and lower bounds. Shape: (d, 2).
+    :type bounds: numpy.ndarray
+
+    :returns: points in `bounds`. Shape: (n, d).
+    :rtype: numpy.ndarray
+
+    :raises: AssertionError
+
+    """
+
+    assert isinstance(points, np.ndarray)
+    assert isinstance(bounds, np.ndarray)
+    assert len(points.shape) == 2
+    assert len(bounds.shape) == 2
+    assert points.shape[1] == bounds.shape[0]
+    assert bounds.shape[1] == 2
+
+    assert np.all(points >= bounds[:, 0])
+    assert np.all(points <= bounds[:, 1])
+
+    return points

bayeso/wrappers/wrappers_bo_class.py

@@ -13,8 +13,6 @@
 from bayeso.utils import utils_bo
 from bayeso.utils import utils_logger
 
-#logger = utils_logger.get_logger('wrappers_bo_class')
-
 
 class BayesianOptimization:
     """

bayeso/wrappers/wrappers_bo_function.py

@@ -13,8 +13,6 @@
 from bayeso.utils import utils_common
 from bayeso.utils import utils_logger
 
-#logger = utils_logger.get_logger('wrappers_bo_function')
-
 
 @utils_common.validate_types
 def run_single_round_with_all_initial_information(model_bo: bo.BO,