From 584e676d369ae1c70ba7494cc77d1c7275e87192 Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Tue, 28 May 2019 14:17:20 +0800 Subject: [PATCH 1/6] feat: Support bayesianoptimization Signed-off-by: Ce Gao --- .../bayesianoptimization/v1alpha2/Dockerfile | 8 + .../bayesianoptimization/v1alpha2/README.md | 14 + .../bayesianoptimization/v1alpha2/__init__.py | 0 .../bayesianoptimization/v1alpha2/main.py | 25 ++ .../v1alpha2/requirements.txt | 9 + .../bayesianoptimization/deployment.yaml | 23 ++ .../bayesianoptimization/service.yaml | 17 + pkg/suggestion/v1alpha2/bayesian_service.py | 209 ++++++++++++ .../bayesianoptimization/src/__init__.py | 0 .../src/acquisition_func.py | 36 +++ .../src/algorithm_manager.py | 235 ++++++++++++++ .../src/bayesian_optimization_algorithm.py | 65 ++++ .../src/global_optimizer.py | 298 ++++++++++++++++++ .../src/model/__init__.py | 0 .../bayesianoptimization/src/model/gp.py | 38 +++ .../bayesianoptimization/src/model/rf.py | 24 ++ .../bayesianoptimization/src/utils.py | 17 + scripts/v1alpha2/build.sh | 42 +++ scripts/v1alpha2/deploy.sh | 13 +- scripts/v1alpha2/undeploy.sh | 12 +- test/scripts/v1alpha2/build-suggestion-bo.sh | 8 +- test/scripts/v1alpha2/run-tests.sh | 9 +- 22 files changed, 1082 insertions(+), 20 deletions(-) create mode 100644 cmd/suggestion/bayesianoptimization/v1alpha2/Dockerfile create mode 100644 cmd/suggestion/bayesianoptimization/v1alpha2/README.md create mode 100644 cmd/suggestion/bayesianoptimization/v1alpha2/__init__.py create mode 100644 cmd/suggestion/bayesianoptimization/v1alpha2/main.py create mode 100644 cmd/suggestion/bayesianoptimization/v1alpha2/requirements.txt create mode 100644 manifests/v1alpha2/katib/suggestion/bayesianoptimization/deployment.yaml create mode 100644 manifests/v1alpha2/katib/suggestion/bayesianoptimization/service.yaml create mode 100644 pkg/suggestion/v1alpha2/bayesian_service.py create mode 100644 pkg/suggestion/v1alpha2/bayesianoptimization/src/__init__.py create mode 100644 pkg/suggestion/v1alpha2/bayesianoptimization/src/acquisition_func.py create mode 100644 pkg/suggestion/v1alpha2/bayesianoptimization/src/algorithm_manager.py create mode 100644 pkg/suggestion/v1alpha2/bayesianoptimization/src/bayesian_optimization_algorithm.py create mode 100644 pkg/suggestion/v1alpha2/bayesianoptimization/src/global_optimizer.py create mode 100644 pkg/suggestion/v1alpha2/bayesianoptimization/src/model/__init__.py create mode 100644 pkg/suggestion/v1alpha2/bayesianoptimization/src/model/gp.py create mode 100644 pkg/suggestion/v1alpha2/bayesianoptimization/src/model/rf.py create mode 100644 pkg/suggestion/v1alpha2/bayesianoptimization/src/utils.py create mode 100755 scripts/v1alpha2/build.sh diff --git a/cmd/suggestion/bayesianoptimization/v1alpha2/Dockerfile b/cmd/suggestion/bayesianoptimization/v1alpha2/Dockerfile new file mode 100644 index 00000000000..35d53f595b4 --- /dev/null +++ b/cmd/suggestion/bayesianoptimization/v1alpha2/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3 + +ADD . /usr/src/app/github.com/kubeflow/katib +WORKDIR /usr/src/app/github.com/kubeflow/katib/cmd/suggestion/bayesianoptimization/v1alpha2 +RUN pip install --no-cache-dir -r requirements.txt +ENV PYTHONPATH /usr/src/app/github.com/kubeflow/katib:/usr/src/app/github.com/kubeflow/katib/pkg/api/v1alpha2/python + +ENTRYPOINT ["python", "main.py"] diff --git a/cmd/suggestion/bayesianoptimization/v1alpha2/README.md b/cmd/suggestion/bayesianoptimization/v1alpha2/README.md new file mode 100644 index 00000000000..89788a8ae67 --- /dev/null +++ b/cmd/suggestion/bayesianoptimization/v1alpha2/README.md @@ -0,0 +1,14 @@ +- start the service + +``` +python suggestion/bayesian/main.py +``` + +- start the testing client + +``` +python suggestion/test_client.py +``` + +note: +the testing client uses the [Franke's function](http://www.sfu.ca/~ssurjano/franke2d.html) as the black box, and the maximum of Franke's function is around 1.22 diff --git a/cmd/suggestion/bayesianoptimization/v1alpha2/__init__.py b/cmd/suggestion/bayesianoptimization/v1alpha2/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cmd/suggestion/bayesianoptimization/v1alpha2/main.py b/cmd/suggestion/bayesianoptimization/v1alpha2/main.py new file mode 100644 index 00000000000..e914919cc7d --- /dev/null +++ b/cmd/suggestion/bayesianoptimization/v1alpha2/main.py @@ -0,0 +1,25 @@ +import grpc +from concurrent import futures + +import time + +from pkg.api.v1alpha2.python import api_pb2_grpc +from pkg.suggestion.v1alpha2.bayesian_service import BayesianService + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 +DEFAULT_PORT = "0.0.0.0:6789" + +def serve(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + api_pb2_grpc.add_SuggestionServicer_to_server(BayesianService(), server) + server.add_insecure_port(DEFAULT_PORT) + print("Listening...") + server.start() + try: + while True: + time.sleep(_ONE_DAY_IN_SECONDS) + except KeyboardInterrupt: + server.stop(0) + +if __name__ == "__main__": + serve() diff --git a/cmd/suggestion/bayesianoptimization/v1alpha2/requirements.txt b/cmd/suggestion/bayesianoptimization/v1alpha2/requirements.txt new file mode 100644 index 00000000000..8d2c9d4bda7 --- /dev/null +++ b/cmd/suggestion/bayesianoptimization/v1alpha2/requirements.txt @@ -0,0 +1,9 @@ +grpcio +duecredit +cloudpickle==0.5.6 +numpy>=1.13.3 +scikit-learn>=0.19.0 +scipy>=0.19.1 +forestci +protobuf +googleapis-common-protos diff --git a/manifests/v1alpha2/katib/suggestion/bayesianoptimization/deployment.yaml b/manifests/v1alpha2/katib/suggestion/bayesianoptimization/deployment.yaml new file mode 100644 index 00000000000..b589af3c218 --- /dev/null +++ b/manifests/v1alpha2/katib/suggestion/bayesianoptimization/deployment.yaml @@ -0,0 +1,23 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: katib-suggestion-bayesianoptimization + namespace: kubeflow + labels: + app: katib + component: suggestion-bayesianoptimization +spec: + replicas: 1 + template: + metadata: + name: katib-suggestion-bayesianoptimization + labels: + app: katib + component: suggestion-bayesianoptimization + spec: + containers: + - name: katib-suggestion-bayesianoptimization + image: katib/v1alpha2/suggestion-bayesianoptimization + ports: + - name: api + containerPort: 6789 diff --git a/manifests/v1alpha2/katib/suggestion/bayesianoptimization/service.yaml b/manifests/v1alpha2/katib/suggestion/bayesianoptimization/service.yaml new file mode 100644 index 00000000000..30bd0be9096 --- /dev/null +++ b/manifests/v1alpha2/katib/suggestion/bayesianoptimization/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: katib-suggestion-bayesianoptimization + namespace: kubeflow + labels: + app: katib + component: suggestion-bayesianoptimization +spec: + type: ClusterIP + ports: + - port: 6789 + protocol: TCP + name: api + selector: + app: katib + component: katib-bayesianoptimization diff --git a/pkg/suggestion/v1alpha2/bayesian_service.py b/pkg/suggestion/v1alpha2/bayesian_service.py new file mode 100644 index 00000000000..7693256ed5a --- /dev/null +++ b/pkg/suggestion/v1alpha2/bayesian_service.py @@ -0,0 +1,209 @@ +import random +import string + +import grpc +import numpy as np + +from pkg.api.v1alpha2.python import api_pb2 +from pkg.api.v1alpha2.python import api_pb2_grpc +from pkg.suggestion.v1alpha2.bayesianoptimization.src.bayesian_optimization_algorithm import BOAlgorithm +from pkg.suggestion.v1alpha2.bayesianoptimization.src.algorithm_manager import AlgorithmManager +import logging +from logging import getLogger, StreamHandler, INFO, DEBUG + + +class BayesianService(api_pb2_grpc.SuggestionServicer): + def __init__(self, logger=None): + self.manager_addr = "katib-manager" + self.manager_port = 6789 + if logger == None: + self.logger = getLogger(__name__) + FORMAT = '%(asctime)-15s StudyID %(studyid)s %(message)s' + logging.basicConfig(format=FORMAT) + handler = StreamHandler() + handler.setLevel(INFO) + self.logger.setLevel(INFO) + self.logger.addHandler(handler) + self.logger.propagate = False + else: + self.logger = logger + + def _get_experiment(self, name): + channel = grpc.beta.implementations.insecure_channel( + self.manager_addr, self.manager_port) + with api_pb2.beta_create_Manager_stub(channel) as client: + exp = client.GetExperiment( + api_pb2.GetExperimentRequest(experiment_name=name), 10) + return exp.experiment + + def GetSuggestions(self, request, context): + """ + Main function to provide suggestion. + """ + service_params = self.parseParameters(request.experiment_name) + experiment = self._get_experiment(request.experiment_name) + X_train, y_train = self.getEvalHistory( + request.experiment_name, experiment.spec.objective.objective_metric_name, service_params["burn_in"]) + + algo_manager = AlgorithmManager( + experiment_name=request.experiment_name, + experiment=experiment, + X_train=X_train, + y_train=y_train, + logger=self.logger, + ) + + lowerbound = np.array(algo_manager.lower_bound) + upperbound = np.array(algo_manager.upper_bound) + self.logger.debug("lowerbound: %r", lowerbound, + extra={"StudyID": request.study_id}) + self.logger.debug("upperbound: %r", upperbound, + extra={"StudyID": request.study_id}) + alg = BOAlgorithm( + dim=algo_manager.dim, + N=int(service_params["N"]), + lowerbound=lowerbound, + upperbound=upperbound, + X_train=algo_manager.X_train, + y_train=algo_manager.y_train, + mode=service_params["mode"], + trade_off=service_params["trade_off"], + # todo: support length_scale with array type + length_scale=service_params["length_scale"], + noise=service_params["noise"], + nu=service_params["nu"], + kernel_type=service_params["kernel_type"], + n_estimators=service_params["n_estimators"], + max_features=service_params["max_features"], + model_type=service_params["model_type"], + logger=self.logger, + ) + trials = [] + x_next_list = alg.get_suggestion(request.request_number) + for x_next in x_next_list: + x_next = x_next.squeeze() + self.logger.debug("xnext: %r ", x_next, extra={ + "StudyID": request.study_id}) + x_next = algo_manager.parse_x_next(x_next) + x_next = algo_manager.convert_to_dict(x_next) + trials.append(api_pb2.Trial( + spec=api_pb2.TrialSpec( + experiment_name=request.experiment_name, + parameter_assignments=api_pb2.TrialSpec.ParameterAssignments( + assignments=[ + api_pb2.ParameterAssignment( + name=x["name"], + value=str(x["value"]), + ) for x in x_next + ] + ) + ) + )) + return api_pb2.GetSuggestionsReply( + trials=trials + ) + + def getEvalHistory(self, experiment_name, obj_name, burn_in): + worker_hist = [] + x_train = [] + y_train = [] + channel = grpc.beta.implementations.insecure_channel( + self.manager_addr, self.manager_port) + with api_pb2.beta_create_Manager_stub(channel) as client: + trialsrep = client.GetTrialList(api_pb2.GetTrialListRequest( + experiment_name=experiment_name + )) + for t in trialsrep.trials: + if t.status.condition == 2: + gwfrep = client.GetObservationLog( + api_pb2.GetObservationLogRequest( + trial_name=t.name, + metric_name=obj_name)) + w = gwfrep.observation_log + for ml in w.metrics_logs: + if ml.name == obj_name: + y_train.append(float(ml.values[-1].value)) + x_train.append(w.parameter_set) + break + self.logger.info("%d completed trials are found.", + len(x_train), extra={"Experiment": experiment_name}) + if len(x_train) <= burn_in: + x_train = [] + y_train = [] + self.logger.info("Trials will be sampled until %d trials for burn-in are completed.", + burn_in, extra={"Experiment": experiment_name}) + else: + self.logger.debug("Completed trials: %r", x_train, + extra={"Experiment": experiment_name}) + + return x_train, y_train + + def parseParameters(self, experiment_name): + channel = grpc.beta.implementations.insecure_channel( + self.manager_addr, self.manager_port) + params = [] + with api_pb2.beta_create_Manager_stub(channel) as client: + gsprep = client.GetAlgorithmExtraSettings( + api_pb2.GetAlgorithmExtraSettingsRequest(param_id=experiment_name), 10) + params = gsprep.extra_algorithm_settings + + parsed_service_params = { + "N": 100, + "model_type": "gp", + "max_features": "auto", + "length_scale": 0.5, + "noise": 0.0005, + "nu": 1.5, + "kernel_type": "matern", + "n_estimators": 50, + "mode": "pi", + "trade_off": 0.01, + "trial_hist": "", + "burn_in": 10, + } + modes = ["pi", "ei"] + model_types = ["gp", "rf"] + kernel_types = ["matern", "rbf"] + + for param in params: + if param.name in parsed_service_params.keys(): + if param.name == "length_scale" or param.name == "noise" or param.name == "nu" or param.name == "trade_off": + try: + float(param.value) + except ValueError: + self.logger.warning( + "Parameter must be float for %s: %s back to default value", param.name, param.value) + else: + parsed_service_params[param.name] = float(param.value) + + elif param.name == "N" or param.name == "n_estimators" or param.name == "burn_in": + try: + int(param.value) + except ValueError: + self.logger.warning( + "Parameter must be int for %s: %s back to default value", param.name, param.value) + else: + parsed_service_params[param.name] = int(param.value) + + elif param.name == "kernel_type": + if param.value != "rbf" and param.value != "matern": + parsed_service_params[param.name] = param.value + else: + self.logger.warning( + "Unknown Parameter for %s: %s back to default value", param.name, param.value) + elif param.name == "mode" and param.value in modes: + if param.value != "lcb" and param.value != "ei" and param.value != "pi": + parsed_service_params[param.name] = param.value + else: + self.logger.warning( + "Unknown Parameter for %s: %s back to default value", param.name, param.value) + elif param.name == "model_type" and param.value in model_types: + if param.value != "rf" and param.value != "gp": + parsed_service_params[param.name] = param.value + else: + self.logger.warning( + "Unknown Parameter for %s: %s back to default value", param.name, param.value) + else: + self.logger.warning("Unknown Parameter name: %s ", param.name) + + return parsed_service_params diff --git a/pkg/suggestion/v1alpha2/bayesianoptimization/src/__init__.py b/pkg/suggestion/v1alpha2/bayesianoptimization/src/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pkg/suggestion/v1alpha2/bayesianoptimization/src/acquisition_func.py b/pkg/suggestion/v1alpha2/bayesianoptimization/src/acquisition_func.py new file mode 100644 index 00000000000..9e061c6bd76 --- /dev/null +++ b/pkg/suggestion/v1alpha2/bayesianoptimization/src/acquisition_func.py @@ -0,0 +1,36 @@ +""" module for acquisition function""" +import numpy as np +from scipy.stats import norm + + +class AcquisitionFunc: + """ + Class for acquisition function with options for expected improvement, + probability of improvement, or lower confident bound. + """ + + def __init__(self, model, current_optimal, mode="ei", trade_off=0.01): + """ + :param mode: pi: probability of improvement, ei: expected improvement, lcb: lower confident bound + :param trade_off: a parameter to control the trade off between exploiting and exploring + :param model_type: gp: gaussian process, rf: random forest + """ + self.model = model + self.current_optimal = current_optimal + self.mode = mode + self.trade_off = trade_off + + def compute(self, X_test): + y_mean, y_std, y_variance = self.model.predict(X_test) + + z = (y_mean - self.current_optimal - self.trade_off) / y_std + + if self.mode == "ei": + if y_std.any() < 0.000001: + return 0, y_mean, y_variance + result = y_std * (z * norm.cdf(z) + norm.pdf(z)) + elif self.mode == "pi": + result = norm.cdf(z) + else: + result = - (y_mean - self.trade_off * y_std) + return np.squeeze(result), np.squeeze(y_mean), np.squeeze(y_variance) diff --git a/pkg/suggestion/v1alpha2/bayesianoptimization/src/algorithm_manager.py b/pkg/suggestion/v1alpha2/bayesianoptimization/src/algorithm_manager.py new file mode 100644 index 00000000000..6aebd01bd7b --- /dev/null +++ b/pkg/suggestion/v1alpha2/bayesianoptimization/src/algorithm_manager.py @@ -0,0 +1,235 @@ +""" module for algorithm manager """ +import numpy as np + +from pkg.api.v1alpha2.python import api_pb2 + +from .utils import get_logger + + +def deal_with_discrete(feasible_values, current_value): + """ function to embed the current values to the feasible discrete space""" + diff = np.subtract(feasible_values, current_value) + diff = np.absolute(diff) + return feasible_values[np.argmin(diff)] + + +def deal_with_categorical(feasible_values, one_hot_values): + """ function to do the one hot encoding of the categorical values """ + index = np.argmax(one_hot_values) + #index = one_hot_values.argmax() + return feasible_values[int(index)] + + +class AlgorithmManager: + """ class for the algorithm manager + provide some helper functions + """ + + def __init__(self, experiment_name, experiment, X_train, y_train, logger=None): + self.logger = logger if (logger is not None) else get_logger() + self._experiment_name = experiment_name + self._experiment = experiment + self._goal = self._experiment.spec.objective.type + self._dim = 0 + self._lowerbound = [] + self._upperbound = [] + self._types = [] + self._names = [] + # record all the feasible values of discrete type variables + self._discrete_info = [] + self._categorical_info = [] + self._name_id = {} + + self._parse_config() + + self._X_train = self._mapping_params(X_train) + self.parse_X() + + self._y_train = y_train + self._parse_metric() + + @property + def experiment_name(self): + """ return the study id """ + return self._experiment_name + + @property + def experiment(self): + """ return the study configuration """ + return self._experiment + + @property + def goal(self): + """ return the optimization goal""" + return self._goal + + @property + def dim(self): + """ return the dimension """ + return self._dim + + @property + def lower_bound(self): + """ return the lower bound of all the parameters """ + return self._lowerbound + + @property + def upper_bound(self): + """ return the upper bound of all the parameters """ + return self._upperbound + + @property + def types(self): + """ return the types of all the parameters """ + return self._types + + @property + def names(self): + """ return the names of all the parameters """ + return self._names + + @property + def discrete_info(self): + """ return the info of all the discrete parameters """ + return self._discrete_info + + @property + def categorical_info(self): + """ return the info of all the categorical parameters """ + return self._categorical_info + + @property + def X_train(self): + """ return the training data """ + return self._X_train + + @property + def y_train(self): + """ return the target of the training data""" + return self._y_train + + def _parse_config(self): + """ extract info from the study configuration """ + for i, param in enumerate(self._experiment.spec.parameter_specs.parameters): + self._name_id[param.name] = i + self._types.append(param.parameter_type) + self._names.append(param.name) + if param.parameter_type in [api_pb2.DOUBLE, api_pb2.INT]: + self._dim = self._dim + 1 + self._lowerbound.append(float(param.feasible_space.min)) + self._upperbound.append(float(param.feasible_space.max)) + elif param.parameter_type == api_pb2.DISCRETE: + self._dim = self._dim + 1 + discrete_values = [int(x) for x in param.feasible_space.list] + min_value = min(discrete_values) + max_value = max(discrete_values) + self._lowerbound.append(min_value) + self._upperbound.append(max_value) + self._discrete_info.append(dict({ + "name": param.name, + "values": discrete_values, + })) + # one hot encoding for categorical type + elif param.parameter_type == api_pb2.CATEGORICAL: + num_feasible = len(param.feasible.list) + for i in range(num_feasible): + self._lowerbound.append(0) + self._upperbound.append(1) + self._categorical_info.append(dict({ + "name": param.name, + "values": param.feasible.list, + "number": num_feasible, + })) + self._dim += num_feasible + + def _mapping_params(self, parameters_list): + if len(parameters_list) == 0: + return None + ret = [] + for parameters in parameters_list: + maplist = [np.zeros(1)]*len(self._names) + for p in parameters: + self.logger.debug("mapping: %r", p, extra={ + "Experiment": self._experiment_name}) + map_id = self._name_id[p.name] + if self._types[map_id] in [api_pb2.DOUBLE, api_pb2.INT, api_pb2.DISCRETE]: + maplist[map_id] = float(p.value) + elif self._types[map_id] == api_pb2.CATEGORICAL: + for ci in self._categorical_info: + if ci["name"] == p.name: + maplist[map_id] = np.zeros(ci["number"]) + for i, v in enumerate(ci["values"]): + if v == p.value: + maplist[map_id][i] = 1 + break + self.logger.debug("mapped: %r", maplist, extra={ + "Experiment": self._experiment_name}) + ret.append(np.hstack(maplist)) + return ret + + def _parse_metric(self): + """ parse the metric to the dictionary """ + if not self._y_train: + self._y_train = None + return + y = [] + for metric in self._y_train: + if self._goal == api_pb2.MAXIMIZE: + y.append(float(metric)) + else: + y.append(-float(metric)) + self.logger.debug("Ytrain: %r", y, extra={ + "Experiment": self._experiment_name}) + self._y_train = np.array(y) + + def parse_X(self): + if not self._X_train: + self._X_train = None + return + self.logger.debug("Xtrain: %r", self._X_train, extra={ + "Experiment": self._experiment_name}) + self._X_train = np.array(self._X_train) + + def parse_x_next(self, x_next): + """ parse the next suggestion to the proper format """ + counter = 0 + result = [] + for i in range(len(self._types)): + if self._types[i] == api_pb2.INT: + result.append(int(round(x_next[counter], 0))) + counter = counter + 1 + elif self._types[i] == api_pb2.DISCRETE: + for param in self._discrete_info: + if param["name"] == self._names[i]: + result.append( + deal_with_discrete( + param["values"], x_next[counter]) + ) + counter = counter + 1 + break + elif self._types[i] == api_pb2.CATEGORICAL: + for param in self._categorical_info: + if param["name"] == self._names[i]: + result.append(deal_with_categorical( + feasible_values=param["values"], + one_hot_values=x_next[counter:counter + + param["number"]], + )) + counter = counter + param["number"] + break + elif self._types[i] == api_pb2.DOUBLE: + result.append(x_next[counter]) + counter = counter + 1 + return result + + def convert_to_dict(self, x_next): + """ convert the next suggestion to the dictionary """ + result = [] + for i in range(len(x_next)): + tmp = dict({ + "name": self._names[i], + "value": x_next[i], + "type": self._types[i], + }) + result.append(tmp) + return result diff --git a/pkg/suggestion/v1alpha2/bayesianoptimization/src/bayesian_optimization_algorithm.py b/pkg/suggestion/v1alpha2/bayesianoptimization/src/bayesian_optimization_algorithm.py new file mode 100644 index 00000000000..6b2e57c3f9d --- /dev/null +++ b/pkg/suggestion/v1alpha2/bayesianoptimization/src/bayesian_optimization_algorithm.py @@ -0,0 +1,65 @@ +""" module for bayesian optimization algorithm """ +import numpy as np +from sklearn.preprocessing import MinMaxScaler + +from .global_optimizer import GlobalOptimizer + + +class BOAlgorithm: + """ class for bayesian optimization """ + def __init__(self, dim, N, lowerbound, upperbound, X_train, y_train, mode, trade_off, + length_scale, noise, nu, kernel_type, n_estimators, max_features, model_type, logger=None): + # np.random.seed(0) + self.dim = dim + self.N = N or 100 + self.l = np.zeros((1, dim)) + self.u = np.ones((1, dim)) + self.lowerbound = lowerbound.reshape(1, dim) + self.upperbound = upperbound.reshape(1, dim) + + # normalize the upperbound and lowerbound to [0, 1] + self.scaler = MinMaxScaler() + self.scaler.fit(np.append(self.lowerbound, self.upperbound, axis=0)) + + self.X_train = X_train + self.y_train = y_train + if self.y_train is None: + self.current_optimal = None + else: + self.current_optimal = max(self.y_train) + + # initialize the global optimizer + self.optimizer = GlobalOptimizer( + N, + self.l, + self.u, + self.scaler, + self.X_train, + self.y_train, + self.current_optimal, + mode=mode, + trade_off=trade_off, + length_scale=length_scale, + noise=noise, + nu=nu, + kernel_type=kernel_type, + n_estimators=n_estimators, + max_features=max_features, + model_type=model_type, + logger=logger, + ) + + def get_suggestion(self, request_num): + """ main function to provide suggestion """ + x_next_list = [] + if self.X_train is None and self.y_train is None and self.current_optimal is None: + # randomly pick a point as the first trial + for _ in range(request_num): + x_next_list.append(np.random.uniform(self.lowerbound, self.upperbound, size=(1, self.dim))) + else: + _, x_next_list_que = self.optimizer.direct(request_num) + for xn in x_next_list_que: + x = np.array(xn).reshape(1, self.dim) + x = self.scaler.inverse_transform(x) + x_next_list.append(x) + return x_next_list diff --git a/pkg/suggestion/v1alpha2/bayesianoptimization/src/global_optimizer.py b/pkg/suggestion/v1alpha2/bayesianoptimization/src/global_optimizer.py new file mode 100644 index 00000000000..7a307c86af9 --- /dev/null +++ b/pkg/suggestion/v1alpha2/bayesianoptimization/src/global_optimizer.py @@ -0,0 +1,298 @@ +""" module for the global optimizer +DIRECT algorithm is used in this case +""" +import copy + +import numpy as np + +from .acquisition_func import AcquisitionFunc +from .model.gp import GaussianProcessModel +from .model.rf import RandomForestModel +from .utils import get_logger + + +class RectPack: + """ class for the rectangular + including border, center and acquisition function value + """ + + def __init__(self, l, u, division_num, dim, scaler, aq_func): + self.l = l + self.u = u + self.center = (l + u) / 2 + j = np.mod(division_num, dim) + k = (division_num - j) / dim + self.d = np.sqrt(j * np.power(3, float(-2 * (k + 1))) + (dim - j) * np.power(3, float(-2 * k))) / 2 + self.division_num = division_num + self.fc, _, _ = aq_func.compute(scaler.inverse_transform(self.center)) + self.fc = -self.fc + + +class RectBucket: + """ class for the rectangular bucket + rectangular with the same size are put in the same bucket + the rectangular is sorted by the acquisition function value + """ + + def __init__(self, diff, pack): + self.diff = diff + self.array = [pack] + + def insert(self, new_pack): + """ insert a new rectangular to a bucket """ + for i in range(len(self.array)): + if new_pack.fc < self.array[i].fc: + self.array.insert(i, new_pack) + return + self.array.append(new_pack) + + def delete(self): + """ delete the first rectangular""" + del self.array[0] + + def diff_exist(self, diff): + """ detect the size difference """ + return abs(self.diff - diff) < 0.00001 + + +class OptimalPoint: + """ helper class to find potential optimal points""" + + def __init__(self, point, prev, slope): + self.point = point + self.prev = prev + self.slope = slope + + +class DimPack: + def __init__(self, dim, fc): + self.dim = dim + self.fc = fc + + +class GlobalOptimizer: + """ class for the global optimizer """ + + def __init__(self, N, l, u, scaler, X_train, y_train, current_optimal, mode, trade_off, length_scale, + noise, nu, kernel_type, n_estimators, max_features, model_type, logger=None): + self.logger = logger if (logger is not None) else get_logger() + self.N = N + self.l = l + self.u = u + self.scaler = scaler + self.buckets = [] + self.dim = None + if model_type == "gp": + model = GaussianProcessModel( + length_scale=length_scale, + noise=noise, + nu=nu, + kernel_type=kernel_type, + ) + else: + model = RandomForestModel( + n_estimators=n_estimators, + max_features=max_features, + ) + model.fit(X_train, y_train) + self.aq_func = AcquisitionFunc( + model=model, + current_optimal=current_optimal, + mode=mode, + trade_off=trade_off, + ) + + def potential_opt(self, f_min): + """ find the potential optimal rectangular """ + b = [] + for i in range(len(self.buckets)): + b.append(self.buckets[i].array[0]) + b.sort(key=lambda x: x.d) + index = 0 + min_fc = b[0].fc + for i in range(len(b)): + if b[i].fc < min_fc: + min_fc = b[i].fc + index = i + + opt_list = [OptimalPoint(b[index], 0, 0)] + for i in range(index + 1, len(b)): + prev = len(opt_list) - 1 + diff1 = b[i].d + diff2 = opt_list[prev].point.d + current_slope = (b[i].fc - opt_list[prev].point.fc) / (diff1 - diff2) + prev_slope = opt_list[prev].slope + + while prev >= 0 and current_slope < prev_slope: + temp = opt_list[prev].prev + opt_list[prev].prev = -1 + prev = temp + prev_slope = opt_list[prev].slope + diff1 = b[i].d + diff2 = opt_list[prev].point.d + current_slope = (b[i].fc - opt_list[prev].point.fc) / (diff1 - diff2) + + opt_list.append(OptimalPoint(b[i], prev, current_slope)) + + opt_list2 = [] + for i in range(len(opt_list)): + if opt_list[i].prev != -1: + opt_list2.append(opt_list[i]) + + for i in range(len(opt_list2) - 1): + c1 = opt_list2[i].point.d + c2 = opt_list2[i + 1].point.d + fc1 = opt_list2[i].point.fc + fc2 = opt_list2[i + 1].point.fc + if fc1 - c1 * (fc1 - fc2) / (c1 - c2) > (1 - 0.001) * f_min: + # if abs(fc1-fc2)<0.0001: + opt_list2[i] = None + while None in opt_list2: + index = opt_list2.index(None) + del opt_list2[index] + # for opt in opt_list2: + # print(opt.point.fc) + return opt_list2 + + def direct(self, request_num): + """ main algorithm """ + self.dim = self.l.shape[1] + division_num = 0 + + # create the first rectangle and put it in the first bucket + first_rect = RectPack(self.l, self.u, division_num, self.dim, + self.scaler, self.aq_func) + self.buckets.append(RectBucket(first_rect.d, first_rect)) + + ei_min = [] + f_min = first_rect.fc + x_next = first_rect.center + ei_min.append(f_min) + + for _ in range(self.N): + opt_set = self.potential_opt(f_min) + + # for bucket in self.buckets: + # for i in range(len(bucket.array)): + # print(bucket.array[i].fc) + # plt.plot(bucket.diff, bucket.array[i].fc, 'b.') + # + # for opt in opt_set: + # plt.plot(opt.point.d, opt.point.fc, 'r.') + # plt.show() + + for opt in opt_set: + f_min, x_next = self.divide_rect( + opt.point, + f_min, + x_next, + self.aq_func, + self.scaler + ) + for bucket in self.buckets: + if bucket.diff_exist(opt.point.d): + bucket.delete() + if not bucket.array: + index = self.buckets.index(bucket) + del self.buckets[index] + ei_min.append(f_min) + x_next_candidate = self.sample_buckets(request_num) + return f_min, x_next_candidate + + def sample_buckets(self, request_num): + self.logger.debug("In lne self.buckets: %r", len(self.buckets)) + bucket_index = [] + fc_sum = 0.0 + x_next_candidate = [] + for bucket in self.buckets: + for a in bucket.array: + self.logger.debug("fc: %r, %r", a.fc, a.center) + fc_sum -= a.fc + bucket_index.append([-a.fc, a.center]) + bucket_index = sorted(bucket_index, key=lambda x: x[0]) + for _ in range(request_num): + sample = np.random.rand() + stick = 0.0 + for b in bucket_index: + stick += b[0]/fc_sum + if stick > sample: + x_next_candidate.append(b[1]) + break + return x_next_candidate + + def divide_rect(self, opt_rect, f_min, x_next, aq_func, scaler): + """ divide the rectangular into smaller ones """ + rect = copy.deepcopy(opt_rect) + division_num = rect.division_num + j = np.mod(division_num, self.dim) + k = (division_num - j) / self.dim + max_side_len = np.power(3, float(-k)) + delta = max_side_len / 3 + dim_set = [] + for i in range(self.dim): + if abs(max_side_len - (rect.u[0, i] - rect.l[0, i])) < 0.0000001: + dim_set.append(i) + + dim_list = [] + for i in dim_set: + e = np.zeros((1, self.dim)) + e[0, i] = 1 + function_value = min( + aq_func.compute(scaler.inverse_transform(rect.center + delta * e)), + aq_func.compute(scaler.inverse_transform(rect.center - delta * e)) + ) + dim_list.append(DimPack(i, function_value)) + dim_list.sort(key=lambda x: x.fc) + + for i in range(len(dim_list)): + division_num = division_num + 1 + temp = np.zeros((1, self.dim)) + temp[0, dim_list[i].dim] = delta + left_rect = RectPack( + rect.l, + rect.u - 2 * temp, + division_num, + self.dim, + self.scaler, + aq_func + ) + middle_rect = RectPack( + rect.l + temp, + rect.u - temp, + division_num, + self.dim, + self.scaler, + aq_func + ) + right_rect = RectPack( + rect.l + 2 * temp, + rect.u, + division_num, + self.dim, + self.scaler, + aq_func + ) + if left_rect.fc < f_min: + f_min = left_rect.fc + x_next = left_rect.center + if right_rect.fc < f_min: + f_min = right_rect.fc + x_next = right_rect.center + + insert = 0 + for bucket in self.buckets: + if bucket.diff_exist(left_rect.d): + bucket.insert(left_rect) + bucket.insert(right_rect) + if i == len(dim_list) - 1: + bucket.insert(middle_rect) + insert = 1 + break + if insert == 0: + new_bucket = RectBucket(left_rect.d, left_rect) + new_bucket.insert(right_rect) + if i == len(dim_list) - 1: + new_bucket.insert(middle_rect) + self.buckets.append(new_bucket) + rect = middle_rect + return f_min, x_next diff --git a/pkg/suggestion/v1alpha2/bayesianoptimization/src/model/__init__.py b/pkg/suggestion/v1alpha2/bayesianoptimization/src/model/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pkg/suggestion/v1alpha2/bayesianoptimization/src/model/gp.py b/pkg/suggestion/v1alpha2/bayesianoptimization/src/model/gp.py new file mode 100644 index 00000000000..446238c0669 --- /dev/null +++ b/pkg/suggestion/v1alpha2/bayesianoptimization/src/model/gp.py @@ -0,0 +1,38 @@ +""" module for gaussian process prior """ +from sklearn.gaussian_process.kernels import RBF, Matern +from sklearn.gaussian_process import GaussianProcessRegressor + + +class GaussianProcessModel: + """ use the gaussian process as a prior """ + def __init__(self, length_scale=0.5, noise=0.00005, + nu=1.5, kernel_type="matern"): + """ + :param length_scale: the larger the length_scale is, the smoother the gaussian prior is. If a float, + an isotropic kernel is used. If an array, an anisotropic kernel is used where each dimension of it defines + the length-scale of the respective feature dimension. + :param noise: + :param nu: control the smoothness of the prior using Matern kernel. The larger nu is, the smoother the + approximate function is. + :param kernel_type: "rbf": squared exponential kernel, "matern": Matern kernel. + """ + if kernel_type == "rbf": + kernel = RBF(length_scale=length_scale) + elif kernel_type == "matern": + kernel = Matern(length_scale=length_scale, nu=nu) + else: + raise Exception("kernel_type must be 'rbf' or 'matern'") + self.gp = GaussianProcessRegressor( + kernel=kernel, + alpha=noise, + random_state=0, + optimizer=None, + ) + + def fit(self, X_train, y_train): + self.gp.fit(X_train, y_train) + + def predict(self, X_test): + y_mean, y_std = self.gp.predict(X_test, return_std=True) + y_variance = y_std ** 2 + return y_mean, y_std, y_variance diff --git a/pkg/suggestion/v1alpha2/bayesianoptimization/src/model/rf.py b/pkg/suggestion/v1alpha2/bayesianoptimization/src/model/rf.py new file mode 100644 index 00000000000..8778b921e78 --- /dev/null +++ b/pkg/suggestion/v1alpha2/bayesianoptimization/src/model/rf.py @@ -0,0 +1,24 @@ +import numpy as np +import forestci as fci +from sklearn.ensemble import RandomForestRegressor + + +class RandomForestModel: + + def __init__(self, n_estimators=50, max_features="auto"): + self.rf = RandomForestRegressor( + n_estimators=n_estimators, + max_features=max_features, + ) + self.X_train = None + + def fit(self, X_train, y_train): + print(X_train.shape, y_train.shape) + self.X_train = X_train + self.rf.fit(X_train, y_train) + + def predict(self, X_test): + y_mean = self.rf.predict(X_test) + y_variance = fci.random_forest_error(self.rf, self.X_train, X_test) + y_std = np.sqrt(y_variance) + return y_mean, y_std, y_variance diff --git a/pkg/suggestion/v1alpha2/bayesianoptimization/src/utils.py b/pkg/suggestion/v1alpha2/bayesianoptimization/src/utils.py new file mode 100644 index 00000000000..7fafc3af390 --- /dev/null +++ b/pkg/suggestion/v1alpha2/bayesianoptimization/src/utils.py @@ -0,0 +1,17 @@ +import os +import logging +from logging import getLogger, StreamHandler + + +FORMAT = '%(asctime)-15s StudyID %(studyid)s %(message)s' +LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO") + + +def get_logger(name=__name__): + logger = getLogger(name) + logging.basicConfig(format=FORMAT) + handler = StreamHandler() + logger.setLevel(LOG_LEVEL) + logger.addHandler(handler) + logger.propagate = False + return logger diff --git a/scripts/v1alpha2/build.sh b/scripts/v1alpha2/build.sh new file mode 100755 index 00000000000..328d674a3ee --- /dev/null +++ b/scripts/v1alpha2/build.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright 2018 The Kubeflow Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +PREFIX="katib" +CMD_PREFIX="cmd" + +SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/../.. + +cd ${SCRIPT_ROOT} + +echo "Building core image..." +docker build -t ${PREFIX}/v1alpha2/katib-controller -f ${CMD_PREFIX}/katib-controller/v1alpha2/Dockerfile . +docker build -t ${PREFIX}/v1alpha2/katib-manager -f ${CMD_PREFIX}/manager/v1alpha2/Dockerfile . +docker build -t ${PREFIX}/v1alpha2/katib-manager-rest -f ${CMD_PREFIX}/manager-rest/v1alpha2/Dockerfile . +docker build -t ${PREFIX}/v1alpha2/metrics-collector -f ${CMD_PREFIX}/metricscollector/v1alpha2/Dockerfile . + +echo "Building UI image..." +docker build -t ${PREFIX}/v1alpha2/katib-ui -f ${CMD_PREFIX}/ui/v1alpha2/Dockerfile . + +echo "Building TF Event metrics collector image..." +docker build -t ${PREFIX}/v1alpha2/tfevent-metrics-collector -f ${CMD_PREFIX}/tfevent-metricscollector/v1alpha2/Dockerfile . + +echo "Building suggestion images..." +docker build -t ${PREFIX}/v1alpha2/suggestion-random -f ${CMD_PREFIX}/suggestion/random/v1alpha2/Dockerfile . +docker build -t ${PREFIX}/v1alpha2/suggestion-bayesianoptimization -f ${CMD_PREFIX}/suggestion/bayesianoptimization/v1alpha2/Dockerfile . diff --git a/scripts/v1alpha2/deploy.sh b/scripts/v1alpha2/deploy.sh index a6c76fbb311..0c0e89e6de5 100755 --- a/scripts/v1alpha2/deploy.sh +++ b/scripts/v1alpha2/deploy.sh @@ -24,10 +24,11 @@ SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/../.. cd ${SCRIPT_ROOT} kubectl apply -f manifests/v1alpha2 kubectl apply -f manifests/v1alpha2/katib-controller -kubectl apply -f manifests/v1alpha2/manager -kubectl apply -f manifests/v1alpha2/manager-rest -kubectl apply -f manifests/v1alpha2/pv -kubectl apply -f manifests/v1alpha2/db -kubectl apply -f manifests/v1alpha2/ui -kubectl apply -f manifests/v1alpha2/suggestion/random +kubectl apply -f manifests/v1alpha2/katib/manager +kubectl apply -f manifests/v1alpha2/katib/manager-rest +kubectl apply -f manifests/v1alpha2/katib/pv +kubectl apply -f manifests/v1alpha2/katib/db +kubectl apply -f manifests/v1alpha2/katib/ui +kubectl apply -f manifests/v1alpha2/katib/suggestion/random +kubectl apply -f manifests/v1alpha2/katib/suggestion/bayesianoptimization cd - > /dev/null diff --git a/scripts/v1alpha2/undeploy.sh b/scripts/v1alpha2/undeploy.sh index 78ecf22d5ed..2139c3f8f66 100755 --- a/scripts/v1alpha2/undeploy.sh +++ b/scripts/v1alpha2/undeploy.sh @@ -30,10 +30,12 @@ SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/../.. cd ${SCRIPT_ROOT} kubectl delete -f manifests/v1alpha2/katib-controller -kubectl delete -f manifests/v1alpha2/manager -kubectl delete -f manifests/v1alpha2/manager-rest -kubectl delete -f manifests/v1alpha2/db -kubectl delete -f manifests/v1alpha2/pv -kubectl delete -f manifests/v1alpha2/suggestion/random +kubectl delete -f manifests/v1alpha2/katib/manager +kubectl delete -f manifests/v1alpha2/katib/manager-rest +kubectl delete -f manifests/v1alpha2/katib/db +kubectl delete -f manifests/v1alpha2/katib/ui +kubectl delete -f manifests/v1alpha2/katib/pv +kubectl delete -f manifests/v1alpha2/katib/suggestion/random +kubectl delete -f manifests/v1alpha2/katib/suggestion/bayesianoptimization kubectl delete -f manifests/v1alpha2 cd - > /dev/null diff --git a/test/scripts/v1alpha2/build-suggestion-bo.sh b/test/scripts/v1alpha2/build-suggestion-bo.sh index 2b13ae5e05f..cacdd25c41d 100755 --- a/test/scripts/v1alpha2/build-suggestion-bo.sh +++ b/test/scripts/v1alpha2/build-suggestion-bo.sh @@ -16,8 +16,6 @@ # This shell script is used to build an image from our argo workflow -exit 0 - set -o errexit set -o nounset set -o pipefail @@ -39,6 +37,6 @@ cp -r vendor ${GO_DIR}/vendor cd ${GO_DIR} -#cp cmd/suggestion/bayesianoptimization/Dockerfile . -#gcloud builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-bayesianoptimization:${VERSION} --project=${PROJECT} -#gcloud container images add-tag --quiet ${REGISTRY}/${REPO_NAME}/suggestion-bayesianoptimization:${VERSION} ${REGISTRY}/${REPO_NAME}/suggestion-bayesianoptimization:latest --verbosity=info +cp cmd/suggestion/bayesianoptimization/v1alpha2/Dockerfile . +gcloud builds submit . --tag=${REGISTRY}/${REPO_NAME}/v1alpha2/suggestion-bayesianoptimization:${VERSION} --project=${PROJECT} +gcloud container images add-tag --quiet ${REGISTRY}/${REPO_NAME}/v1alpha2/suggestion-bayesianoptimization:${VERSION} ${REGISTRY}/${REPO_NAME}/suggestion-bayesianoptimization:latest --verbosity=info diff --git a/test/scripts/v1alpha2/run-tests.sh b/test/scripts/v1alpha2/run-tests.sh index d4d3330a5fa..992d36af958 100755 --- a/test/scripts/v1alpha2/run-tests.sh +++ b/test/scripts/v1alpha2/run-tests.sh @@ -66,10 +66,11 @@ echo "REPO_NAME ${REPO_NAME}" echo "VERSION ${VERSION}" sed -i -e "s@image: katib\/v1alpha2\/katib-controller@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-controller:${VERSION}@" manifests/v1alpha2/katib-controller/katib-controller.yaml -sed -i -e "s@image: katib\/v1alpha2\/katib-manager@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-manager:${VERSION}@" manifests/v1alpha2/manager/deployment.yaml -sed -i -e "s@image: katib\/v1alpha2\/katib-manager-rest@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-manager-rest:${VERSION}@" manifests/v1alpha2/manager-rest/deployment.yaml -sed -i -e "s@image: katib\/v1alpha2\/katib-ui@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-ui:${VERSION}@" manifests/v1alpha2/ui/deployment.yaml -sed -i -e "s@image: katib\/v1alpha2\/suggestion-random@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/suggestion-random:${VERSION}@" manifests/v1alpha2/suggestion/random/deployment.yaml +sed -i -e "s@image: katib\/v1alpha2\/katib-manager@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-manager:${VERSION}@" manifests/v1alpha2/katib/manager/deployment.yaml +sed -i -e "s@image: katib\/v1alpha2\/katib-manager-rest@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-manager-rest:${VERSION}@" manifests/v1alpha2/katib/manager-rest/deployment.yaml +sed -i -e "s@image: katib\/v1alpha2\/katib-ui@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-ui:${VERSION}@" manifests/v1alpha2/katib/ui/deployment.yaml +sed -i -e "s@image: katib\/v1alpha2\/suggestion-random@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/suggestion-random:${VERSION}@" manifests/v1alpha2/katib/suggestion/random/deployment.yaml +sed -i -e "s@image: katib\/v1alpha2\/suggestion-bayesianoptimization@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/suggestion-bayesianoptimization:${VERSION}@" manifests/v1alpha2/katib/suggestion/bayesianoptimization/deployment.yaml ./scripts/v1alpha2/deploy.sh From de96df83e2e9f0b5ca70a6645b2c702d82ac9e91 Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Wed, 29 May 2019 10:03:27 +0800 Subject: [PATCH 2/6] feat: Suport bo Signed-off-by: Ce Gao --- examples/v1alpha2/bayseopt-example.yaml | 64 +++++++++++++++++++ .../bayesianoptimization/deployment.yaml | 1 + pkg/suggestion/v1alpha2/bayesian_service.py | 3 + 3 files changed, 68 insertions(+) create mode 100644 examples/v1alpha2/bayseopt-example.yaml diff --git a/examples/v1alpha2/bayseopt-example.yaml b/examples/v1alpha2/bayseopt-example.yaml new file mode 100644 index 00000000000..70e69f37e48 --- /dev/null +++ b/examples/v1alpha2/bayseopt-example.yaml @@ -0,0 +1,64 @@ +apiVersion: "kubeflow.org/v1alpha2" +kind: Experiment +metadata: + namespace: kubeflow + labels: + controller-tools.k8s.io: "1.0" + name: bayseopt-example +spec: + objective: + type: maximize + goal: 0.99 + objectiveMetricName: Validation-accuracy + additionalMetricNames: + - accuracy + algorithm: + algorithmName: bayesianoptimization + algorithmSettings: + - name: "burn_in" + value: "5" + parallelTrialCount: 3 + maxTrialCount: 12 + maxFailedTrialCount: 3 + parameters: + - name: --lr + parameterType: double + feasibleSpace: + min: "0.01" + max: "0.03" + - name: --num-layers + parameterType: int + feasibleSpace: + min: "2" + max: "5" + - name: --optimizer + parameterType: categorical + feasibleSpace: + list: + - sgd + - adam + - ftrl + trialTemplate: + goTemplate: + rawTemplate: |- + apiVersion: batch/v1 + kind: Job + metadata: + name: {{.Trial}} + namespace: {{.NameSpace}} + spec: + template: + spec: + containers: + - name: {{.Trial}} + image: katib/mxnet-mnist-example + command: + - "python" + - "/mxnet/example/image-classification/train_mnist.py" + - "--batch-size=64" + {{- with .HyperParameters}} + {{- range .}} + - "{{.Name}}={{.Value}}" + {{- end}} + {{- end}} + restartPolicy: Never diff --git a/manifests/v1alpha2/katib/suggestion/bayesianoptimization/deployment.yaml b/manifests/v1alpha2/katib/suggestion/bayesianoptimization/deployment.yaml index b589af3c218..3ceeee9e094 100644 --- a/manifests/v1alpha2/katib/suggestion/bayesianoptimization/deployment.yaml +++ b/manifests/v1alpha2/katib/suggestion/bayesianoptimization/deployment.yaml @@ -18,6 +18,7 @@ spec: containers: - name: katib-suggestion-bayesianoptimization image: katib/v1alpha2/suggestion-bayesianoptimization + imagePullPolicy: IfNotPresent ports: - name: api containerPort: 6789 diff --git a/pkg/suggestion/v1alpha2/bayesian_service.py b/pkg/suggestion/v1alpha2/bayesian_service.py index 7693256ed5a..cb331c5ad18 100644 --- a/pkg/suggestion/v1alpha2/bayesian_service.py +++ b/pkg/suggestion/v1alpha2/bayesian_service.py @@ -36,6 +36,9 @@ def _get_experiment(self, name): api_pb2.GetExperimentRequest(experiment_name=name), 10) return exp.experiment + def ValidateAlgorithmSettings(self, request, context): + return api_pb2.ValidateAlgorithmSettingsReply() + def GetSuggestions(self, request, context): """ Main function to provide suggestion. From 7f5b0a4cbd003f1dd1a160bda9f2060e7ad3f20f Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Wed, 29 May 2019 10:46:16 +0800 Subject: [PATCH 3/6] fix: Resolve conflicts Signed-off-by: Ce Gao --- .../bayesianoptimization/deployment.yaml | 0 .../suggestion/bayesianoptimization/service.yaml | 0 scripts/v1alpha2/deploy.sh | 14 +++++++------- scripts/v1alpha2/undeploy.sh | 14 +++++++------- test/scripts/v1alpha2/run-tests.sh | 10 +++++----- 5 files changed, 19 insertions(+), 19 deletions(-) rename manifests/v1alpha2/{katib => }/suggestion/bayesianoptimization/deployment.yaml (100%) rename manifests/v1alpha2/{katib => }/suggestion/bayesianoptimization/service.yaml (100%) diff --git a/manifests/v1alpha2/katib/suggestion/bayesianoptimization/deployment.yaml b/manifests/v1alpha2/suggestion/bayesianoptimization/deployment.yaml similarity index 100% rename from manifests/v1alpha2/katib/suggestion/bayesianoptimization/deployment.yaml rename to manifests/v1alpha2/suggestion/bayesianoptimization/deployment.yaml diff --git a/manifests/v1alpha2/katib/suggestion/bayesianoptimization/service.yaml b/manifests/v1alpha2/suggestion/bayesianoptimization/service.yaml similarity index 100% rename from manifests/v1alpha2/katib/suggestion/bayesianoptimization/service.yaml rename to manifests/v1alpha2/suggestion/bayesianoptimization/service.yaml diff --git a/scripts/v1alpha2/deploy.sh b/scripts/v1alpha2/deploy.sh index 0c0e89e6de5..19e5238ab08 100755 --- a/scripts/v1alpha2/deploy.sh +++ b/scripts/v1alpha2/deploy.sh @@ -24,11 +24,11 @@ SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/../.. cd ${SCRIPT_ROOT} kubectl apply -f manifests/v1alpha2 kubectl apply -f manifests/v1alpha2/katib-controller -kubectl apply -f manifests/v1alpha2/katib/manager -kubectl apply -f manifests/v1alpha2/katib/manager-rest -kubectl apply -f manifests/v1alpha2/katib/pv -kubectl apply -f manifests/v1alpha2/katib/db -kubectl apply -f manifests/v1alpha2/katib/ui -kubectl apply -f manifests/v1alpha2/katib/suggestion/random -kubectl apply -f manifests/v1alpha2/katib/suggestion/bayesianoptimization +kubectl apply -f manifests/v1alpha2/manager +kubectl apply -f manifests/v1alpha2/manager-rest +kubectl apply -f manifests/v1alpha2/pv +kubectl apply -f manifests/v1alpha2/db +kubectl apply -f manifests/v1alpha2/ui +kubectl apply -f manifests/v1alpha2/suggestion/random +kubectl apply -f manifests/v1alpha2/suggestion/bayesianoptimization cd - > /dev/null diff --git a/scripts/v1alpha2/undeploy.sh b/scripts/v1alpha2/undeploy.sh index 2139c3f8f66..d1f1c86ef5f 100755 --- a/scripts/v1alpha2/undeploy.sh +++ b/scripts/v1alpha2/undeploy.sh @@ -30,12 +30,12 @@ SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/../.. cd ${SCRIPT_ROOT} kubectl delete -f manifests/v1alpha2/katib-controller -kubectl delete -f manifests/v1alpha2/katib/manager -kubectl delete -f manifests/v1alpha2/katib/manager-rest -kubectl delete -f manifests/v1alpha2/katib/db -kubectl delete -f manifests/v1alpha2/katib/ui -kubectl delete -f manifests/v1alpha2/katib/pv -kubectl delete -f manifests/v1alpha2/katib/suggestion/random -kubectl delete -f manifests/v1alpha2/katib/suggestion/bayesianoptimization +kubectl delete -f manifests/v1alpha2/manager +kubectl delete -f manifests/v1alpha2/manager-rest +kubectl delete -f manifests/v1alpha2/db +kubectl delete -f manifests/v1alpha2/ui +kubectl delete -f manifests/v1alpha2/pv +kubectl delete -f manifests/v1alpha2/suggestion/random +kubectl delete -f manifests/v1alpha2/suggestion/bayesianoptimization kubectl delete -f manifests/v1alpha2 cd - > /dev/null diff --git a/test/scripts/v1alpha2/run-tests.sh b/test/scripts/v1alpha2/run-tests.sh index 992d36af958..1437ba12e0e 100755 --- a/test/scripts/v1alpha2/run-tests.sh +++ b/test/scripts/v1alpha2/run-tests.sh @@ -66,11 +66,11 @@ echo "REPO_NAME ${REPO_NAME}" echo "VERSION ${VERSION}" sed -i -e "s@image: katib\/v1alpha2\/katib-controller@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-controller:${VERSION}@" manifests/v1alpha2/katib-controller/katib-controller.yaml -sed -i -e "s@image: katib\/v1alpha2\/katib-manager@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-manager:${VERSION}@" manifests/v1alpha2/katib/manager/deployment.yaml -sed -i -e "s@image: katib\/v1alpha2\/katib-manager-rest@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-manager-rest:${VERSION}@" manifests/v1alpha2/katib/manager-rest/deployment.yaml -sed -i -e "s@image: katib\/v1alpha2\/katib-ui@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-ui:${VERSION}@" manifests/v1alpha2/katib/ui/deployment.yaml -sed -i -e "s@image: katib\/v1alpha2\/suggestion-random@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/suggestion-random:${VERSION}@" manifests/v1alpha2/katib/suggestion/random/deployment.yaml -sed -i -e "s@image: katib\/v1alpha2\/suggestion-bayesianoptimization@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/suggestion-bayesianoptimization:${VERSION}@" manifests/v1alpha2/katib/suggestion/bayesianoptimization/deployment.yaml +sed -i -e "s@image: katib\/v1alpha2\/katib-manager@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-manager:${VERSION}@" manifests/v1alpha2/manager/deployment.yaml +sed -i -e "s@image: katib\/v1alpha2\/katib-manager-rest@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-manager-rest:${VERSION}@" manifests/v1alpha2/manager-rest/deployment.yaml +sed -i -e "s@image: katib\/v1alpha2\/katib-ui@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/katib-ui:${VERSION}@" manifests/v1alpha2/ui/deployment.yaml +sed -i -e "s@image: katib\/v1alpha2\/suggestion-random@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/suggestion-random:${VERSION}@" manifests/v1alpha2/suggestion/random/deployment.yaml +sed -i -e "s@image: katib\/v1alpha2\/suggestion-bayesianoptimization@image: ${REGISTRY}\/${REPO_NAME}\/v1alpha2\/suggestion-bayesianoptimization:${VERSION}@" manifests/v1alpha2/suggestion/bayesianoptimization/deployment.yaml ./scripts/v1alpha2/deploy.sh From c676244976be0e627e0cd575f95b91f9e75ca7d2 Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Thu, 30 May 2019 11:44:58 +0800 Subject: [PATCH 4/6] fix: Fix runtime error Signed-off-by: Ce Gao --- pkg/suggestion/v1alpha2/bayesian_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/suggestion/v1alpha2/bayesian_service.py b/pkg/suggestion/v1alpha2/bayesian_service.py index cb331c5ad18..ef1b056e380 100644 --- a/pkg/suggestion/v1alpha2/bayesian_service.py +++ b/pkg/suggestion/v1alpha2/bayesian_service.py @@ -59,9 +59,9 @@ def GetSuggestions(self, request, context): lowerbound = np.array(algo_manager.lower_bound) upperbound = np.array(algo_manager.upper_bound) self.logger.debug("lowerbound: %r", lowerbound, - extra={"StudyID": request.study_id}) + extra={"experimeng_name": request.experiment_name}) self.logger.debug("upperbound: %r", upperbound, - extra={"StudyID": request.study_id}) + extra={"experimeng_name": request.experiment_name}) alg = BOAlgorithm( dim=algo_manager.dim, N=int(service_params["N"]), @@ -86,7 +86,7 @@ def GetSuggestions(self, request, context): for x_next in x_next_list: x_next = x_next.squeeze() self.logger.debug("xnext: %r ", x_next, extra={ - "StudyID": request.study_id}) + "experiment_name": request.experiment_name}) x_next = algo_manager.parse_x_next(x_next) x_next = algo_manager.convert_to_dict(x_next) trials.append(api_pb2.Trial( From af4edaa594a1c31ecab79c689b199161f598c6b3 Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Thu, 30 May 2019 11:47:07 +0800 Subject: [PATCH 5/6] fix: Fix format Signed-off-by: Ce Gao --- pkg/suggestion/v1alpha2/bayesian_service.py | 10 +++++----- .../v1alpha2/bayesianoptimization/src/utils.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/suggestion/v1alpha2/bayesian_service.py b/pkg/suggestion/v1alpha2/bayesian_service.py index ef1b056e380..a9c4d5517aa 100644 --- a/pkg/suggestion/v1alpha2/bayesian_service.py +++ b/pkg/suggestion/v1alpha2/bayesian_service.py @@ -18,7 +18,7 @@ def __init__(self, logger=None): self.manager_port = 6789 if logger == None: self.logger = getLogger(__name__) - FORMAT = '%(asctime)-15s StudyID %(studyid)s %(message)s' + FORMAT = '%(asctime)-15s Experiment %(experiment_name)s %(message)s' logging.basicConfig(format=FORMAT) handler = StreamHandler() handler.setLevel(INFO) @@ -59,9 +59,9 @@ def GetSuggestions(self, request, context): lowerbound = np.array(algo_manager.lower_bound) upperbound = np.array(algo_manager.upper_bound) self.logger.debug("lowerbound: %r", lowerbound, - extra={"experimeng_name": request.experiment_name}) + extra={"experiment_name": request.experiment_name}) self.logger.debug("upperbound: %r", upperbound, - extra={"experimeng_name": request.experiment_name}) + extra={"experiment_name": request.experiment_name}) alg = BOAlgorithm( dim=algo_manager.dim, N=int(service_params["N"]), @@ -134,10 +134,10 @@ def getEvalHistory(self, experiment_name, obj_name, burn_in): x_train = [] y_train = [] self.logger.info("Trials will be sampled until %d trials for burn-in are completed.", - burn_in, extra={"Experiment": experiment_name}) + burn_in, extra={"experiment_name": experiment_name}) else: self.logger.debug("Completed trials: %r", x_train, - extra={"Experiment": experiment_name}) + extra={"experiment_name": experiment_name}) return x_train, y_train diff --git a/pkg/suggestion/v1alpha2/bayesianoptimization/src/utils.py b/pkg/suggestion/v1alpha2/bayesianoptimization/src/utils.py index 7fafc3af390..a63bfb3b169 100644 --- a/pkg/suggestion/v1alpha2/bayesianoptimization/src/utils.py +++ b/pkg/suggestion/v1alpha2/bayesianoptimization/src/utils.py @@ -3,7 +3,7 @@ from logging import getLogger, StreamHandler -FORMAT = '%(asctime)-15s StudyID %(studyid)s %(message)s' +FORMAT = '%(asctime)-15s Experiment %(experiment_name)s %(message)s' LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO") From f2b64e48ad508326aa8da4f9248097235aa0b693 Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Thu, 30 May 2019 11:48:37 +0800 Subject: [PATCH 6/6] fix: Fix comments Signed-off-by: Ce Gao --- .../v1alpha2/bayesianoptimization/src/algorithm_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/suggestion/v1alpha2/bayesianoptimization/src/algorithm_manager.py b/pkg/suggestion/v1alpha2/bayesianoptimization/src/algorithm_manager.py index 6aebd01bd7b..1fd7318813b 100644 --- a/pkg/suggestion/v1alpha2/bayesianoptimization/src/algorithm_manager.py +++ b/pkg/suggestion/v1alpha2/bayesianoptimization/src/algorithm_manager.py @@ -50,12 +50,12 @@ def __init__(self, experiment_name, experiment, X_train, y_train, logger=None): @property def experiment_name(self): - """ return the study id """ + """ return the experiment_name """ return self._experiment_name @property def experiment(self): - """ return the study configuration """ + """ return the experiment """ return self._experiment @property