Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

UPDATE: add or modify debug codes #1

Merged
merged 1 commit into from Jun 29, 2018
Merged

UPDATE: add or modify debug codes #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 5 additions & 5 deletions bayeso/bo.py
View file
Open in desktop
Expand Up @@ -70,7 +70,7 @@ def _get_initial_sobol(self, int_samples, int_seed=None):
if int_seed is None:
int_seed = np.random.randint(0, 10000)
if self.debug:
print('[DEBUG] _get_initial_sobol: int_seed', int_seed)
print('[DEBUG] _get_initial_sobol in bo.py: int_seed', int_seed)
arr_samples = sobol_seq.i4_sobol_generate(self.num_dim, int_samples, int_seed)
arr_samples = arr_samples * (self.arr_range[:, 1].flatten() - self.arr_range[:, 0].flatten()) + self.arr_range[:, 0].flatten()
return arr_samples
Expand All @@ -92,7 +92,7 @@ def get_initial(self, str_initial_method,
if str_initial_method == 'grid':
assert fun_objective is not None
if self.debug:
print('[DEBUG] get_initial: int_samples is ignored, because grid is chosen.')
print('[DEBUG] get_initial in bo.py: int_samples is ignored, because grid is chosen.')
arr_initials = self._get_initial_grid()
arr_initials = utils_bo.get_best_acquisition(arr_initials, fun_objective)
elif str_initial_method == 'uniform':
Expand All @@ -104,7 +104,7 @@ def get_initial(self, str_initial_method,
else:
raise ValueError('get_initial: missing condition for str_initial_method')
if self.debug:
print('[DEBUG] get_initial: arr_initials')
print('[DEBUG] get_initial in bo.py: arr_initials')
print(arr_initials)
return arr_initials

Expand All @@ -131,7 +131,7 @@ def _optimize(self, fun_objective, str_initial_method, int_samples):
)
list_next_point.append(next_point.x)
if self.debug:
print('[DEBUG] _optimize: optimized point for acq', next_point.x)
print('[DEBUG] _optimize in bo.py: optimized point for acq', next_point.x)
next_point = utils_bo.get_best_acquisition(np.array(list_next_point), fun_objective)
return next_point.flatten()

Expand Down Expand Up @@ -167,5 +167,5 @@ def optimize(self, X_train, Y_train,
time_end = time.time()

if self.debug:
print('[DEBUG] optimize: time consumed', time_end - time_start, 'sec.')
print('[DEBUG] optimize in bo.py: time consumed', time_end - time_start, 'sec.')
return next_point, cov_X_X, inv_cov_X_X, hyps
2 changes: 1 addition & 1 deletion bayeso/constants.py
View file
Open in desktop
Expand Up @@ -6,7 +6,7 @@
JITTER_COV = 1e-5

STR_OPTIMIZER_METHOD_BO = 'L-BFGS-B'
STR_OPTIMIZER_METHOD_GP = 'L-BFGS-B'
STR_OPTIMIZER_METHOD_GP = 'BFGS'
STR_GP_COV = 'se'
STR_BO_ACQ = 'ei'
STR_BO_INITIALIZATION = 'sobol'
Expand Down
30 changes: 20 additions & 10 deletions bayeso/gp.py
View file
Open in desktop
Expand Up @@ -24,29 +24,33 @@ def get_prior_mu(prior_mu, X):
assert len(prior_mu_X.shape) == 2
return prior_mu_X

def get_kernels(X_train, hyps, str_cov):
def get_kernels(X_train, hyps, str_cov, debug=False):
assert isinstance(X_train, np.ndarray)
assert len(X_train.shape) == 2
assert isinstance(hyps, dict)
assert isinstance(str_cov, str)
assert isinstance(debug, bool)
assert len(X_train.shape) == 2
if debug:
pass

cov_X_X = covariance.cov_main(str_cov, X_train, X_train, hyps) + hyps['noise']**2 * np.eye(X_train.shape[0])
cov_X_X = (cov_X_X + cov_X_X.T) / 2.0
inv_cov_X_X = np.linalg.inv(cov_X_X)
return cov_X_X, inv_cov_X_X

def log_ml(X_train, Y_train, hyps, str_cov, prior_mu_train):
def log_ml(X_train, Y_train, hyps, str_cov, prior_mu_train, debug=False):
assert isinstance(X_train, np.ndarray)
assert isinstance(Y_train, np.ndarray)
assert isinstance(hyps, np.ndarray)
assert isinstance(str_cov, str)
assert isinstance(debug, bool)
assert len(X_train.shape) == 2
assert len(Y_train.shape) == 2
assert len(prior_mu_train.shape) == 2
assert X_train.shape[0] == Y_train.shape[0] == prior_mu_train.shape[0]

hyps = utils_covariance.restore_hyps(str_cov, hyps)
cov_X_X, inv_cov_X_X = get_kernels(X_train, hyps, str_cov)
cov_X_X, inv_cov_X_X = get_kernels(X_train, hyps, str_cov, debug)
new_Y_train = Y_train - prior_mu_train

first_term = -0.5 * np.dot(np.dot(new_Y_train.T, inv_cov_X_X), new_Y_train)
Expand All @@ -65,22 +69,28 @@ def get_optimized_kernel(X_train, Y_train, prior_mu, str_cov, str_optimizer_meth
assert len(Y_train.shape) == 2
assert X_train.shape[0] == Y_train.shape[0]

time_start = time.time()

prior_mu_train = get_prior_mu(prior_mu, X_train)
num_dim = X_train.shape[1]
neg_log_ml = lambda hyps: -1.0 * log_ml(X_train, Y_train, hyps, str_cov, prior_mu_train)
neg_log_ml = lambda hyps: -1.0 * log_ml(X_train, Y_train, hyps, str_cov, prior_mu_train, debug)
result_optimized = scipy.optimize.minimize(
neg_log_ml,
utils_covariance.convert_hyps(
str_cov,
utils_covariance.get_hyps(str_cov, num_dim)
utils_covariance.get_hyps(str_cov, num_dim),
),
method=str_optimizer_method
method=str_optimizer_method,
)
result_optimized = result_optimized.x
hyps = utils_covariance.restore_hyps(str_cov, result_optimized)
if debug:
print('[DEBUG] get_optimized_kernel: optimized hyps for gpr ', hyps)
cov_X_X, inv_cov_X_X = get_kernels(X_train, hyps, str_cov)

time_end = time.time()

if debug:
print('[DEBUG] get_optimized_kernel in gp.py: optimized hyps for gpr', hyps)
print('[DEBUG] get_optimized_kernel in gp.py: time consumed', time_end - time_start, 'sec.')
return cov_X_X, inv_cov_X_X, hyps

def predict_test_(X_train, Y_train, X_test, cov_X_X, inv_cov_X_X, hyps, str_cov=constants.STR_GP_COV, prior_mu=None):
Expand Down Expand Up @@ -147,5 +157,5 @@ def predict_optimized(X_train, Y_train, X_test, str_cov=constants.STR_GP_COV, pr

time_end = time.time()
if debug:
print('[DEBUG] predict_optimized: time consumed', time_end - time_start, 'sec.')
print('[DEBUG] predict_optimized in gp.py: time consumed', time_end - time_start, 'sec.')
return mu_Xs, sigma_Xs
17 changes: 15 additions & 2 deletions bayeso/utils/utils_bo.py
View file
Open in desktop
Expand Up @@ -56,12 +56,25 @@ def optimize_many_(model_bo, fun_target, X_train, Y_train, int_iter,
assert X_train.shape[0] == Y_train.shape[0]
assert Y_train.shape[1] == 1

time_start = time.time()

X_final = X_train
Y_final = Y_train
for _ in range(0, int_iter):
next_point, _, _, _ = model_bo.optimize(X_final, Y_final, str_initial_method=str_initial_method_optimizer, int_samples=int_samples_ao)
X_final = np.vstack((X_final, next_point))

time_to_evaluate_start = time.time()
Y_final = np.vstack((Y_final, fun_target(next_point)))
time_to_evaluate_end = time.time()
if model_bo.debug:
print('[DEBUG] optimize_many_ in utils_bo.py: time consumed to evaluate', time_to_evaluate_end - time_to_evaluate_start, 'sec.')


time_end = time.time()

if model_bo.debug:
print('[DEBUG] optimize_many_ in utils_bo.py: time consumed', time_end - time_start, 'sec.')
return X_final, Y_final

def optimize_many(model_bo, fun_target, X_train, int_iter,
Expand Down Expand Up @@ -112,7 +125,7 @@ def optimize_many_with_random_init(model_bo, fun_target, int_init, int_iter,

X_init = model_bo.get_initial(str_initial_method_bo, fun_objective=fun_target, int_samples=int_init, int_seed=int_seed)
if model_bo.debug:
print('[DEBUG] optimize_many_with_random_init: X_init')
print('[DEBUG] optimize_many_with_random_init in utils_bo.py: X_init')
print(X_init)
X_final, Y_final = optimize_many(model_bo, fun_target, X_init, int_iter,
str_initial_method_optimizer=str_initial_method_optimizer,
Expand All @@ -122,6 +135,6 @@ def optimize_many_with_random_init(model_bo, fun_target, int_init, int_iter,
time_end = time.time()

if model_bo.debug:
print('[DEBUG] optimize_many_with_random_init', time_end - time_start, 'sec.')
print('[DEBUG] optimize_many_with_random_init in utils_bo.py: time consumed', time_end - time_start, 'sec.')

return X_final, Y_final
2 changes: 1 addition & 1 deletion bayeso/utils/utils_covariance.py
View file
Open in desktop
Expand Up @@ -15,7 +15,7 @@ def get_hyps(str_cov, num_dim, is_ard=True):
if str_cov == 'se':
hyps['signal'] = 1.0
if is_ard:
hyps['lengthscales'] = np.zeros(num_dim) + 1.0
hyps['lengthscales'] = np.ones(num_dim)
else:
hyps['lengthscales'] = 1.0
elif str_cov == 'matern52' or str_cov == 'matern32':
Expand Down
2 changes: 1 addition & 1 deletion examples/example_hpo_xgboost_ei.py
View file
Open in desktop
Expand Up @@ -34,7 +34,7 @@ def main():
model_bo = bo.BO(np.array([[1, 10], [100, 500]]), debug=True)
list_Y = []
for _ in range(0, 5):
X_final, Y_final = utils_bo.optimize_many_with_random_init(model_bo, fun_target, int_init, 50, str_initial_method_bo='uniform', str_initial_method_optimizer='grid', int_samples_ao=100)
X_final, Y_final = utils_bo.optimize_many_with_random_init(model_bo, fun_target, int_init, 20, str_initial_method_bo='uniform', str_initial_method_optimizer='grid', int_samples_ao=100)
list_Y.append(Y_final)
arr_Y = np.array(list_Y)
arr_Y = np.expand_dims(np.squeeze(arr_Y), axis=0)
Expand Down