UPDATE: bo tests and utils_bo tests

bayeso/bo.py

@@ -65,23 +65,28 @@ def _get_initial_uniform(self, int_samples, int_seed=None):
         arr_initials = np.array(list_initials)
         return arr_initials
 
+    # TODO: I am not sure, but noise can be added.
     def _get_initial_sobol(self, int_samples, int_seed=None):
         assert isinstance(int_seed, int) or int_seed is None
 
         if int_seed is None:
             int_seed = np.random.randint(0, 10000)
+        if self.debug:
+            print('DEBUG: _get_initial_sobol: int_seed')
+            print(int_seed)
         arr_samples = sobol_seq.i4_sobol_generate(self.num_dim, int_samples, int_seed)
         arr_samples = arr_samples * (self.arr_range[:, 1].flatten() - self.arr_range[:, 0].flatten()) + self.arr_range[:, 0].flatten()
         return arr_samples
 
     def _get_initial_latin(self, int_samples):
         pass
 
-    def _get_initial(self, str_initial_method,
+    def get_initial(self, str_initial_method,
         fun_objective=None,
         int_samples=constants.NUM_ACQ_SAMPLES,
         int_seed=None,
     ):
+        assert isinstance(str_initial_method, str)
         assert callable(fun_objective) or fun_objective is None
         assert isinstance(int_samples, int)
         assert isinstance(int_seed, int) or int_seed is None
@@ -95,11 +100,11 @@ def _get_initial(self, str_initial_method,
         elif str_initial_method == 'sobol':
             arr_initials = self._get_initial_sobol(int_samples, int_seed=int_seed)
         elif str_initial_method == 'latin':
-            raise NotImplementedError('_get_initial: latin')
+            raise NotImplementedError('get_initial: latin')
         else:
-            raise ValueError('_get_initial: missing condition for str_initial_method')
+            raise ValueError('get_initial: missing condition for str_initial_method')
         if self.debug:
-            print('DEBUG: _get_initial: arr_initials')
+            print('DEBUG: get_initial: arr_initials')
             print(arr_initials)
         return arr_initials
 
@@ -110,10 +115,11 @@ def _optimize_objective(self, fun_acquisition, X_train, Y_train, X_test, cov_X_X
         return acquisitions
 
     def _optimize(self, fun_objective, str_initial_method):
+        assert str_initial_method in constants.ALLOWED_INITIALIZATIONS_OPTIMIZER
         list_bounds = []
         for elem in self.arr_range:
             list_bounds.append(tuple(elem))
-        arr_initials = self._get_initial(str_initial_method, fun_objective=fun_objective)
+        arr_initials = self.get_initial(str_initial_method, fun_objective=fun_objective)
         list_next_point = []
         for arr_initial in arr_initials:
             next_point = minimize(
@@ -129,7 +135,7 @@ def _optimize(self, fun_objective, str_initial_method):
         next_point = utils_bo.get_best_acquisition(np.array(list_next_point), fun_objective)
         return next_point.flatten()
 
-    def optimize(self, X_train, Y_train, str_initial_method='sobol'):
+    def optimize(self, X_train, Y_train, str_initial_method=constants.STR_OPTIMIZER_INITIALIZATION):
         assert isinstance(X_train, np.ndarray)
         assert isinstance(Y_train, np.ndarray)
         assert isinstance(str_initial_method, str)

bayeso/constants.py

@@ -9,6 +9,8 @@
 STR_OPTIMIZER_METHOD_GP = 'L-BFGS-B'
 STR_GP_COV = 'se'
 STR_BO_ACQ = 'ei'
+STR_BO_INITIALIZATION = 'sobol'
+STR_OPTIMIZER_INITIALIZATION = 'sobol'
 
 NUM_BO_GRID = 50
 NUM_BO_RANDOM = 1000
@@ -19,6 +21,9 @@
 TIME_PAUSE = 2.0
 RANGE_SHADE = 1.96
 
+ALLOWED_INITIALIZATIONS_BO = ['sobol', 'uniform', 'latin']
+ALLOWED_INITIALIZATIONS_OPTIMIZER = ALLOWED_INITIALIZATIONS_BO + ['grid']
+
 COLORS = [
     'red',
     'green',

bayeso/utils/utils_bo.py

@@ -1,9 +1,12 @@
 # utils_bo
 # author: Jungtaek Kim (jtkim@postech.ac.kr)
-# last updated: June 22, 2018
+# last updated: June 23, 2018
 
 import numpy as np
 
+from bayeso import bo
+from bayeso import constants
+
 
 def get_grid(arr_ranges, int_grid):
     assert isinstance(arr_ranges, np.ndarray)
@@ -23,6 +26,10 @@ def get_grid(arr_ranges, int_grid):
     return arr_grid
 
 def get_best_acquisition(arr_initials, fun_objective):
+    assert isinstance(arr_initials, np.ndarray)
+    assert callable(fun_objective)
+    assert len(arr_initials.shape) == 2
+
     cur_best = np.inf
     cur_initial = None
     for arr_initial in arr_initials:
@@ -32,33 +39,51 @@ def get_best_acquisition(arr_initials, fun_objective):
             cur_best = cur_acq
     return np.expand_dims(cur_initial, axis=0)
 
-def optimize_many_(model_bo, fun_target, X_train, Y_train, num_iter):
+def optimize_many_(model_bo, fun_target, X_train, Y_train, int_iter):
+    assert isinstance(model_bo, bo.BO)
+    assert callable(fun_target)
+    assert isinstance(X_train, np.ndarray)
+    assert isinstance(Y_train, np.ndarray)
+    assert isinstance(int_iter, int)
+    assert len(X_train.shape) == 2
+    assert len(Y_train.shape) == 2
+    assert X_train.shape[0] == Y_train.shape[0]
+    assert Y_train.shape[1] == 1
+
     X_final = X_train
     Y_final = Y_train
-    for _ in range(0, num_iter):
-        result_bo, _, _, _ = model_bo.optimize(X_final, Y_final)
-        X_final = np.vstack((X_final, result_bo))
-        Y_final = np.vstack((Y_final, fun_target(result_bo)))
+    for _ in range(0, int_iter):
+        next_point, _, _, _ = model_bo.optimize(X_final, Y_final)
+        X_final = np.vstack((X_final, next_point))
+        Y_final = np.vstack((Y_final, fun_target(next_point)))
     return X_final, Y_final
 
-def optimize_many(model_bo, fun_target, X_train, num_iter):
+def optimize_many(model_bo, fun_target, X_train, int_iter):
+    assert isinstance(model_bo, bo.BO)
+    assert callable(fun_target)
+    assert isinstance(X_train, np.ndarray)
+    assert isinstance(int_iter, int)
+    assert len(X_train.shape) == 2
+
     Y_train = []
     for elem in X_train:
         Y_train.append(fun_target(elem))
     Y_train = np.array(Y_train)
     Y_train = np.reshape(Y_train, (Y_train.shape[0], 1))
-    X_final, Y_final = optimize_many_(model_bo, fun_target, X_train, Y_train, num_iter)
+    X_final, Y_final = optimize_many_(model_bo, fun_target, X_train, Y_train, int_iter)
     return X_final, Y_final
 
-def optimize_many_with_random_init(model_bo, fun_target, num_init, num_iter, int_seed=None):
-    list_init = []
-    for ind_init in range(0, num_init):
-        if int_seed is None or int_seed == 0:
-            print('REMIND: seed is None or 0.')
-            list_init.append(model_bo._get_initial(is_random=True, is_grid=False))
-        else:
-            list_init.append(model_bo._get_initial(is_random=True, is_grid=False, int_seed=int_seed**2 * (ind_init+1)))
-    X_init = np.array(list_init)
-    X_final, Y_final = optimize_many(model_bo, fun_target, X_init, num_iter)
-    return X_final, Y_final
+def optimize_many_with_random_init(model_bo, fun_target, int_init, int_iter, str_initial_method=constants.STR_BO_INITIALIZATION, int_seed=None):
+    assert isinstance(model_bo, bo.BO)
+    assert callable(fun_target)
+    assert isinstance(int_init, int)
+    assert isinstance(int_iter, int)
+    assert isinstance(int_seed, int) or int_seed is None
+    assert str_initial_method in constants.ALLOWED_INITIALIZATIONS_BO
 
+    X_init = model_bo.get_initial(str_initial_method, fun_objective=fun_target, int_samples=int_init, int_seed=int_seed)
+    if model_bo.debug:
+        print('DEBUG: optimize_many_with_random_init: X_init')
+        print(X_init)
+    X_final, Y_final = optimize_many(model_bo, fun_target, X_init, int_iter)
+    return X_final, Y_final

tests/test_bo.py

@@ -6,8 +6,6 @@
 import pytest
 
 from bayeso import bo
-from bayeso import gp
-from bayeso.utils import utils_covariance
 
 
 TEST_EPSILON = 1e-5
@@ -60,14 +58,49 @@ def test_load_bo():
     model_bo = bo.BO(arr_range_1)
     model_bo = bo.BO(arr_range_2)
 
+def test_get_initial():
+    np.random.seed(42)
+    arr_range = np.array([
+        [0.0, 10.0],
+        [-2.0, 2.0],
+        [-5.0, 5.0],
+    ])
+    dim_X = arr_range.shape[0]
+    num_X = 5
+    X = np.random.randn(num_X, dim_X)
+    Y = np.random.randn(num_X, 1)
+    model_bo = bo.BO(arr_range)
+
+    with pytest.raises(AssertionError) as error:
+        model_bo.get_initial(1)
+    with pytest.raises(AssertionError) as error:
+        model_bo.get_initial('grid', fun_objective=None)
+    with pytest.raises(AssertionError) as error:
+        model_bo.get_initial('uniform', fun_objective=1)
+    with pytest.raises(AssertionError) as error:
+        model_bo.get_initial('uniform', int_samples='abc')
+    with pytest.raises(AssertionError) as error:
+        model_bo.get_initial('uniform', int_seed='abc')
+    with pytest.raises(ValueError) as error:
+        model_bo.get_initial('abc')
+
+    arr_initials = model_bo.get_initial('uniform', int_samples=3, int_seed=42)
+    truth_arr_initials = np.array([
+        [3.74540119, 1.80285723, 2.31993942],
+        [5.98658484, -1.37592544, -3.4400548],
+        [0.58083612, 1.46470458, 1.01115012],
+    ])
+
+    assert (arr_initials - truth_arr_initials < TEST_EPSILON).all()
+
 def test_optimize():
     np.random.seed(42)
     arr_range_1 = np.array([
         [0.0, 10.0],
         [-2.0, 2.0],
         [-5.0, 5.0],
     ])
-    dim_X = 3
+    dim_X = arr_range_1.shape[0]
     num_X = 5
     X = np.random.randn(num_X, dim_X)
     Y = np.random.randn(num_X, 1)
@@ -89,7 +122,7 @@ def test_optimize():
         model_bo.optimize(X, np.random.randn(3, 1))
     with pytest.raises(AssertionError) as error:
         model_bo.optimize(X, Y, str_initial_method=1)
-    with pytest.raises(ValueError) as error:
+    with pytest.raises(AssertionError) as error:
         model_bo.optimize(X, Y, str_initial_method='abc')
 
     next_point, cov_X_X, inv_cov_X_X, hyps = model_bo.optimize(X, Y)